deimos-ruby 1.8.0.pre.beta1 → 1.8.1.pre.beta4

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '1.8.0-beta1'
+  VERSION = '1.8.1-beta4'
 end

data/lib/generators/deimos/active_record/templates/migration.rb.tt

@@ -0,0 +1,28 @@
+class <%= migration_class_name %> < ActiveRecord::Migration<%= migration_version %>
+  def up
+    if table_exists?(:<%= table_name %>)
+      warn "<%= table_name %> already exists, exiting"
+      return
+    end
+    create_table :<%= table_name %> do |t|
+      <%- fields.each do |key| -%>
+      <%- next if %w(id message_id timestamp).include?(key.name) -%>
+      <%- sql_type = schema_base.sql_type(key)
+         if %w(record array map).include?(sql_type)
+           conn = ActiveRecord::Base.connection
+           sql_type = conn.respond_to?(:supports_json?) && conn.supports_json? ? :json : :string
+         end
+      -%>
+      t.<%= sql_type %> :<%= key.name %>
+      <%- end -%>
+    end
+
+    # TODO add indexes as necessary
+  end
+
+  def down
+    return unless table_exists?(:<%= table_name %>)
+    drop_table :<%= table_name %>
+  end
+
+end

data/lib/generators/deimos/active_record/templates/model.rb.tt

@@ -0,0 +1,5 @@
+class <%= table_name.classify %> < ApplicationRecord
+<%- fields.select { |f| f.enum_values.any? }.each do |field| -%>
+  enum <%= field.name %>: {<%= field.enum_values.map { |v| "#{v}: '#{v}'"}.join(', ') %>}
+<% end -%>
+end

data/lib/generators/deimos/active_record_generator.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require 'rails/generators'
+require 'rails/generators/active_record/migration'
+require 'rails/version'
+
+# Generates a new consumer.
+module Deimos
+  module Generators
+    # Generator for ActiveRecord model and migration.
+    class ActiveRecordGenerator < Rails::Generators::Base
+      include Rails::Generators::Migration
+      if Rails.version < '4'
+        extend(ActiveRecord::Generators::Migration)
+      else
+        include ActiveRecord::Generators::Migration
+      end
+      source_root File.expand_path('active_record/templates', __dir__)
+
+      argument :table_name, desc: 'The table to create.', required: true
+      argument :full_schema, desc: 'The fully qualified schema name.', required: true
+
+      no_commands do
+
+        # @return [String]
+        def db_migrate_path
+          if defined?(Rails.application) && Rails.application
+            paths = Rails.application.config.paths['db/migrate']
+            paths.respond_to?(:to_ary) ? paths.to_ary.first : paths.to_a.first
+          else
+            'db/migrate'
+          end
+        end
+
+        # @return [String]
+        def migration_version
+          "[#{ActiveRecord::Migration.current_version}]"
+        rescue StandardError
+          ''
+        end
+
+        # @return [String]
+        def table_class
+          self.table_name.classify
+        end
+
+        # @return [String]
+        def schema
+          last_dot = self.full_schema.rindex('.')
+          self.full_schema[last_dot + 1..-1]
+        end
+
+        # @return [String]
+        def namespace
+          last_dot = self.full_schema.rindex('.')
+          self.full_schema[0...last_dot]
+        end
+
+        # @return [Deimos::SchemaBackends::Base]
+        def schema_base
+          @schema_base ||= Deimos.schema_backend_class.new(schema: schema, namespace: namespace)
+        end
+
+        # @return [Array<SchemaField>]
+        def fields
+          schema_base.schema_fields
+        end
+
+      end
+
+      desc 'Generate migration for a table based on an existing schema.'
+      # :nodoc:
+      def generate
+        migration_template('migration.rb', "db/migrate/create_#{table_name.underscore}.rb")
+        template('model.rb', "app/models/#{table_name.underscore}.rb")
+      end
+    end
+  end
+end

data/lib/generators/deimos/db_backend/templates/migration

@@ -16,6 +16,7 @@ class <%= migration_class_name %> < ActiveRecord::Migration<%= migration_version
       t.datetime :locked_at
       t.boolean :error, null: false, default: false
       t.integer :retries, null: false, default: 0
+      t.datetime :last_processed_at
     end
     add_index :kafka_topic_info, :topic, unique: true
     add_index :kafka_topic_info, [:locked_by, :error]

data/lib/generators/deimos/db_backend/templates/rails3_migration

@@ -16,6 +16,7 @@ class <%= migration_class_name %> < ActiveRecord::Migration<%= migration_version
       t.datetime :locked_at
       t.boolean :error, null: false, default: false
       t.integer :retries, null: false, default: 0
+      t.datetime :last_processed_at
     end
     add_index :kafka_topic_info, :topic, unique: true
     add_index :kafka_topic_info, [:locked_by, :error]

data/spec/batch_consumer_spec.rb

@@ -111,6 +111,7 @@ module ConsumerTest
         test_consume_batch('my_batch_consume_topic', batch, keys: keys) do |_received, metadata|
           # Mock decode_key extracts the value of the first field as the key
           expect(metadata[:keys]).to eq(%w(foo bar))
+          expect(metadata[:first_offset]).to eq(1)
         end
       end
 

data/spec/generators/active_record_generator_spec.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require 'generators/deimos/active_record_generator'
+
+RSpec.describe Deimos::Generators::ActiveRecordGenerator do
+
+  after(:each) do
+    FileUtils.rm_rf('db') if File.exist?('db')
+    FileUtils.rm_rf('app') if File.exist?('app')
+  end
+
+  it 'should generate a migration' do
+    expect(Dir['db/migrate/*.rb']).to be_empty
+    expect(Dir['app/models/*.rb']).to be_empty
+    described_class.start(['generated_table', 'com.my-namespace.Generated'])
+    files = Dir['db/migrate/*.rb']
+    expect(files.length).to eq(1)
+    results = <<~MIGRATION
+      class CreateGeneratedTable < ActiveRecord::Migration[6.0]
+        def up
+          if table_exists?(:generated_table)
+            warn "generated_table already exists, exiting"
+            return
+          end
+          create_table :generated_table do |t|
+            t.string :a_string
+            t.integer :a_int
+            t.bigint :a_long
+            t.float :a_float
+            t.float :a_double
+            t.string :an_enum
+            t.json :an_array
+            t.json :a_map
+            t.json :a_record
+          end
+      
+          # TODO add indexes as necessary
+        end
+      
+        def down
+          return unless table_exists?(:generated_table)
+          drop_table :generated_table
+        end
+      
+      end
+    MIGRATION
+    expect(File.read(files[0])).to eq(results)
+    model = <<~MODEL
+      class GeneratedTable < ApplicationRecord
+        enum an_enum: {sym1: 'sym1', sym2: 'sym2'}
+      end
+    MODEL
+    expect(File.read('app/models/generated_table.rb')).to eq(model)
+  end
+
+end

data/spec/kafka_listener_spec.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+describe Deimos::KafkaListener do
+  include_context 'with widgets'
+
+  prepend_before(:each) do
+    producer_class = Class.new(Deimos::Producer) do
+      schema 'MySchema'
+      namespace 'com.my-namespace'
+      topic 'my-topic'
+      key_config none: true
+    end
+    stub_const('MyProducer', producer_class)
+  end
+
+  before(:each) do
+    Deimos.configure do |c|
+      c.producers.backend = :kafka
+      c.schema.backend = :avro_local
+    end
+    allow_any_instance_of(Kafka::Cluster).to receive(:add_target_topics)
+    allow_any_instance_of(Kafka::Cluster).to receive(:partitions_for).
+      and_raise(Kafka::Error)
+  end
+
+  describe '.send_produce_error' do
+    let(:payloads) do
+      [{ 'test_id' => 'foo', 'some_int' => 123 },
+       { 'test_id' => 'bar', 'some_int' => 124 }]
+    end
+
+    it 'should listen to publishing errors and republish as Deimos events' do
+      Deimos.subscribe('produce_error') do |event|
+        expect(event.payload).to include(
+          producer: MyProducer,
+          topic: 'my-topic',
+          payloads: payloads
+        )
+      end
+      expect(Deimos.config.metrics).to receive(:increment).
+        with('publish_error', tags: %w(topic:my-topic), by: 2)
+      expect { MyProducer.publish_list(payloads) }.to raise_error(Kafka::DeliveryFailed)
+    end
+
+    it 'should not send any notifications when producer is not found' do
+      Deimos.subscribe('produce_error') do |_|
+        raise 'OH NOES'
+      end
+      allow(Deimos::Producer).to receive(:descendants).and_return([])
+      expect(Deimos.config.metrics).not_to receive(:increment).with('publish_error', anything)
+      expect { MyProducer.publish_list(payloads) }.to raise_error(Kafka::DeliveryFailed)
+    end
+  end
+end

data/spec/kafka_topic_info_spec.rb

@@ -37,22 +37,45 @@ each_db_config(Deimos::KafkaTopicInfo) do
   end
 
   specify '#clear_lock' do
-    described_class.create!(topic: 'my-topic', locked_by: 'abc',
-                            locked_at: 10.seconds.ago, error: true, retries: 1)
-    described_class.create!(topic: 'my-topic2', locked_by: 'def',
-                            locked_at: 10.seconds.ago, error: true, retries: 1)
-    described_class.clear_lock('my-topic', 'abc')
-    expect(described_class.count).to eq(2)
-    record = described_class.first
-    expect(record.locked_by).to eq(nil)
-    expect(record.locked_at).to eq(nil)
-    expect(record.error).to eq(false)
-    expect(record.retries).to eq(0)
-    record = described_class.last
-    expect(record.locked_by).not_to eq(nil)
-    expect(record.locked_at).not_to eq(nil)
-    expect(record.error).not_to eq(false)
-    expect(record.retries).not_to eq(0)
+    freeze_time do
+      Deimos::KafkaTopicInfo.create!(topic: 'my-topic', locked_by: 'abc',
+                                     locked_at: 10.seconds.ago, error: true, retries: 1,
+                                     last_processed_at: 20.seconds.ago)
+      Deimos::KafkaTopicInfo.create!(topic: 'my-topic2', locked_by: 'def',
+                                     locked_at: 10.seconds.ago, error: true, retries: 1,
+                                     last_processed_at: 20.seconds.ago)
+      Deimos::KafkaTopicInfo.clear_lock('my-topic', 'abc')
+      expect(Deimos::KafkaTopicInfo.count).to eq(2)
+      record = Deimos::KafkaTopicInfo.first
+      expect(record.locked_by).to eq(nil)
+      expect(record.locked_at).to eq(nil)
+      expect(record.error).to eq(false)
+      expect(record.retries).to eq(0)
+      expect(record.last_processed_at.to_s).to eq(Time.zone.now.to_s)
+      record = Deimos::KafkaTopicInfo.last
+      expect(record.locked_by).not_to eq(nil)
+      expect(record.locked_at).not_to eq(nil)
+      expect(record.error).not_to eq(false)
+      expect(record.retries).not_to eq(0)
+      expect(record.last_processed_at.to_s).to eq(20.seconds.ago.to_s)
+    end
+  end
+
+  specify '#ping_empty_topics' do
+    freeze_time do
+      old_time = 1.hour.ago.to_s
+      t1 = Deimos::KafkaTopicInfo.create!(topic: 'topic1', last_processed_at: old_time)
+      t2 = Deimos::KafkaTopicInfo.create!(topic: 'topic2', last_processed_at: old_time)
+      t3 = Deimos::KafkaTopicInfo.create!(topic: 'topic3', last_processed_at: old_time,
+                                          locked_by: 'me', locked_at: 1.minute.ago)
+
+      expect(Deimos::KafkaTopicInfo.count).to eq(3)
+      Deimos::KafkaTopicInfo.all.each { |t| expect(t.last_processed_at.to_s).to eq(old_time) }
+      Deimos::KafkaTopicInfo.ping_empty_topics(%w(topic1))
+      expect(t1.reload.last_processed_at.to_s).to eq(old_time) # was passed as an exception
+      expect(t2.reload.last_processed_at.to_s).to eq(Time.zone.now.to_s)
+      expect(t3.reload.last_processed_at.to_s).to eq(old_time) # is locked
+    end
   end
 
   specify '#register_error' do

data/spec/producer_spec.rb

@@ -41,6 +41,14 @@ module ProducerTest
       end
       stub_const('MyNoKeyProducer', producer_class)
 
+      producer_class = Class.new(Deimos::Producer) do
+        schema 'MyNestedSchema'
+        namespace 'com.my-namespace'
+        topic 'my-topic'
+        key_config field: 'test_id'
+      end
+      stub_const('MyNestedSchemaProducer', producer_class)
+
       producer_class = Class.new(Deimos::Producer) do
         schema 'MySchema'
         namespace 'com.my-namespace'
@@ -233,6 +241,34 @@ module ProducerTest
       )
     end
 
+    it 'should properly encode and coerce values with a nested record' do
+      expect(MyNestedSchemaProducer.encoder).to receive(:encode_key).with('test_id', 'foo', topic: 'my-topic-key')
+      MyNestedSchemaProducer.publish(
+        'test_id' => 'foo',
+        'test_float' => BigDecimal('123.456'),
+        'test_array' => ['1'],
+        'some_nested_record' => {
+          'some_int' => 123,
+          'some_float' => BigDecimal('456.789'),
+          'some_string' => '123',
+          'some_optional_int' => nil
+        },
+        'some_optional_record' => nil
+      )
+      expect(MyNestedSchemaProducer.topic).to have_sent(
+        'test_id' => 'foo',
+        'test_float' => 123.456,
+        'test_array' => ['1'],
+        'some_nested_record' => {
+          'some_int' => 123,
+          'some_float' => 456.789,
+          'some_string' => '123',
+          'some_optional_int' => nil
+        },
+        'some_optional_record' => nil
+      )
+    end
+
     it 'should error with nothing set' do
       expect {
         MyErrorProducer.publish_list(

data/spec/schemas/com/my-namespace/Generated.avsc

@@ -0,0 +1,71 @@
+{
+  "namespace": "com.my-namespace",
+  "name": "Generated",
+  "type": "record",
+  "doc": "Test schema",
+  "fields": [
+    {
+      "name": "a_string",
+      "type": "string"
+    },
+    {
+      "name": "a_int",
+      "type": "int"
+    },
+    {
+      "name": "a_long",
+      "type": "long"
+    },
+    {
+      "name": "a_float",
+      "type": "float"
+    },
+    {
+      "name": "a_double",
+      "type": "double"
+    },
+    {
+      "name": "an_enum",
+      "type": {
+        "type": "enum",
+        "name": "AnEnum",
+        "symbols": ["sym1", "sym2"]
+      }
+    },
+    {
+      "name": "an_array",
+      "type": {
+        "type": "array",
+        "items": "int"
+      }
+    },
+    {
+      "name": "a_map",
+      "type": {
+        "type": "map",
+        "values": "string"
+      }
+    },
+    {
+      "name": "timestamp",
+      "type": "string"
+    },
+    {
+      "name": "message_id",
+      "type": "string"
+    },
+    {
+      "name": "a_record",
+      "type": {
+        "type": "record",
+        "name": "ARecord",
+        "fields": [
+          {
+            "name": "a_record_field",
+            "type": "string"
+          }
+        ]
+      }
+    }
+  ]
+}