deimos-ruby 1.7.0.pre.beta1 → 1.8.1.pre.beta3

data/spec/generators/active_record_generator_spec.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require 'generators/deimos/active_record_generator'
+
+RSpec.describe Deimos::Generators::ActiveRecordGenerator do
+
+  after(:each) do
+    FileUtils.rm_rf('db') if File.exist?('db')
+    FileUtils.rm_rf('app') if File.exist?('app')
+  end
+
+  it 'should generate a migration' do
+    expect(Dir['db/migrate/*.rb']).to be_empty
+    expect(Dir['app/models/*.rb']).to be_empty
+    described_class.start(['generated_table', 'com.my-namespace.Generated'])
+    files = Dir['db/migrate/*.rb']
+    expect(files.length).to eq(1)
+    results = <<~MIGRATION
+      class CreateGeneratedTable < ActiveRecord::Migration[6.0]
+        def up
+          if table_exists?(:generated_table)
+            warn "generated_table already exists, exiting"
+            return
+          end
+          create_table :generated_table do |t|
+            t.string :a_string
+            t.integer :a_int
+            t.bigint :a_long
+            t.float :a_float
+            t.float :a_double
+            t.string :an_enum
+            t.json :an_array
+            t.json :a_map
+            t.json :a_record
+          end
+      
+          # TODO add indexes as necessary
+        end
+      
+        def down
+          return unless table_exists?(:generated_table)
+          drop_table :generated_table
+        end
+      
+      end
+    MIGRATION
+    expect(File.read(files[0])).to eq(results)
+    model = <<~MODEL
+      class GeneratedTable < ApplicationRecord
+        enum an_enum: {sym1: 'sym1', sym2: 'sym2'}
+      end
+    MODEL
+    expect(File.read('app/models/generated_table.rb')).to eq(model)
+  end
+
+end

data/spec/handlers/my_batch_consumer.rb

@@ -1,5 +1,10 @@
 # frozen_string_literal: true
 
 module ConsumerTest
-  class MyBatchConsumer < Deimos::BatchConsumer; end
+  # Mock consumer
+  class MyBatchConsumer < Deimos::Consumer
+    # :no-doc:
+    def consume_batch
+    end
+  end
 end

data/spec/handlers/my_consumer.rb

@@ -1,5 +1,10 @@
 # frozen_string_literal: true
 
 module ConsumerTest
-  class MyConsumer < Deimos::Consumer; end
+  # Mock consumer
+  class MyConsumer < Deimos::Consumer
+    # :no-doc:
+    def consume
+    end
+  end
 end

data/spec/kafka_listener_spec.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+describe Deimos::KafkaListener do
+  include_context 'with widgets'
+
+  prepend_before(:each) do
+    producer_class = Class.new(Deimos::Producer) do
+      schema 'MySchema'
+      namespace 'com.my-namespace'
+      topic 'my-topic'
+      key_config none: true
+    end
+    stub_const('MyProducer', producer_class)
+  end
+
+  before(:each) do
+    Deimos.configure do |c|
+      c.producers.backend = :kafka
+      c.schema.backend = :avro_local
+    end
+    allow_any_instance_of(Kafka::Cluster).to receive(:add_target_topics)
+    allow_any_instance_of(Kafka::Cluster).to receive(:partitions_for).
+      and_raise(Kafka::Error)
+  end
+
+  describe '.send_produce_error' do
+    let(:payloads) do
+      [{ 'test_id' => 'foo', 'some_int' => 123 },
+       { 'test_id' => 'bar', 'some_int' => 124 }]
+    end
+
+    it 'should listen to publishing errors and republish as Deimos events' do
+      Deimos.subscribe('produce_error') do |event|
+        expect(event.payload).to include(
+          producer: MyProducer,
+          topic: 'my-topic',
+          payloads: payloads
+        )
+      end
+      expect(Deimos.config.metrics).to receive(:increment).
+        with('publish_error', tags: %w(topic:my-topic), by: 2)
+      expect { MyProducer.publish_list(payloads) }.to raise_error(Kafka::DeliveryFailed)
+    end
+
+    it 'should not send any notifications when producer is not found' do
+      Deimos.subscribe('produce_error') do |_|
+        raise 'OH NOES'
+      end
+      allow(Deimos::Producer).to receive(:descendants).and_return([])
+      expect(Deimos.config.metrics).not_to receive(:increment).with('publish_error', anything)
+      expect { MyProducer.publish_list(payloads) }.to raise_error(Kafka::DeliveryFailed)
+    end
+  end
+end

data/spec/kafka_topic_info_spec.rb

@@ -37,22 +37,45 @@ each_db_config(Deimos::KafkaTopicInfo) do
   end
 
   specify '#clear_lock' do
-    described_class.create!(topic: 'my-topic', locked_by: 'abc',
-                            locked_at: 10.seconds.ago, error: true, retries: 1)
-    described_class.create!(topic: 'my-topic2', locked_by: 'def',
-                            locked_at: 10.seconds.ago, error: true, retries: 1)
-    described_class.clear_lock('my-topic', 'abc')
-    expect(described_class.count).to eq(2)
-    record = described_class.first
-    expect(record.locked_by).to eq(nil)
-    expect(record.locked_at).to eq(nil)
-    expect(record.error).to eq(false)
-    expect(record.retries).to eq(0)
-    record = described_class.last
-    expect(record.locked_by).not_to eq(nil)
-    expect(record.locked_at).not_to eq(nil)
-    expect(record.error).not_to eq(false)
-    expect(record.retries).not_to eq(0)
+    freeze_time do
+      Deimos::KafkaTopicInfo.create!(topic: 'my-topic', locked_by: 'abc',
+                                     locked_at: 10.seconds.ago, error: true, retries: 1,
+                                     last_processed_at: 20.seconds.ago)
+      Deimos::KafkaTopicInfo.create!(topic: 'my-topic2', locked_by: 'def',
+                                     locked_at: 10.seconds.ago, error: true, retries: 1,
+                                     last_processed_at: 20.seconds.ago)
+      Deimos::KafkaTopicInfo.clear_lock('my-topic', 'abc')
+      expect(Deimos::KafkaTopicInfo.count).to eq(2)
+      record = Deimos::KafkaTopicInfo.first
+      expect(record.locked_by).to eq(nil)
+      expect(record.locked_at).to eq(nil)
+      expect(record.error).to eq(false)
+      expect(record.retries).to eq(0)
+      expect(record.last_processed_at.to_s).to eq(Time.zone.now.to_s)
+      record = Deimos::KafkaTopicInfo.last
+      expect(record.locked_by).not_to eq(nil)
+      expect(record.locked_at).not_to eq(nil)
+      expect(record.error).not_to eq(false)
+      expect(record.retries).not_to eq(0)
+      expect(record.last_processed_at.to_s).to eq(20.seconds.ago.to_s)
+    end
+  end
+
+  specify '#ping_empty_topics' do
+    freeze_time do
+      old_time = 1.hour.ago.to_s
+      t1 = Deimos::KafkaTopicInfo.create!(topic: 'topic1', last_processed_at: old_time)
+      t2 = Deimos::KafkaTopicInfo.create!(topic: 'topic2', last_processed_at: old_time)
+      t3 = Deimos::KafkaTopicInfo.create!(topic: 'topic3', last_processed_at: old_time,
+                                          locked_by: 'me', locked_at: 1.minute.ago)
+
+      expect(Deimos::KafkaTopicInfo.count).to eq(3)
+      Deimos::KafkaTopicInfo.all.each { |t| expect(t.last_processed_at.to_s).to eq(old_time) }
+      Deimos::KafkaTopicInfo.ping_empty_topics(%w(topic1))
+      expect(t1.reload.last_processed_at.to_s).to eq(old_time) # was passed as an exception
+      expect(t2.reload.last_processed_at.to_s).to eq(Time.zone.now.to_s)
+      expect(t3.reload.last_processed_at.to_s).to eq(old_time) # is locked
+    end
   end
 
   specify '#register_error' do

data/spec/message_spec.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+RSpec.describe(Deimos::Message) do
+  it 'should detect tombstones' do
+    expect(described_class.new(nil, nil, key: 'key1')).
+      to be_tombstone
+    expect(described_class.new({ v: 'val1' }, nil, key: 'key1')).
+      not_to be_tombstone
+    expect(described_class.new({ v: '' }, nil, key: 'key1')).
+      not_to be_tombstone
+    expect(described_class.new({ v: 'val1' }, nil, key: nil)).
+      not_to be_tombstone
+  end
+
+  it 'can support complex keys/values' do
+    expect { described_class.new({ a: 1, b: 2 }, nil, key: { c: 3, d: 4 }) }.
+      not_to raise_exception
+  end
+end

data/spec/producer_spec.rb

@@ -41,6 +41,14 @@ module ProducerTest
       end
       stub_const('MyNoKeyProducer', producer_class)
 
+      producer_class = Class.new(Deimos::Producer) do
+        schema 'MyNestedSchema'
+        namespace 'com.my-namespace'
+        topic 'my-topic'
+        key_config field: 'test_id'
+      end
+      stub_const('MyNestedSchemaProducer', producer_class)
+
       producer_class = Class.new(Deimos::Producer) do
         schema 'MySchema'
         namespace 'com.my-namespace'
@@ -233,6 +241,32 @@ module ProducerTest
       )
     end
 
+    it 'should properly encode and coerce values with a nested record' do
+      expect(MyNestedSchemaProducer.encoder).to receive(:encode_key).with('test_id', 'foo', topic: 'my-topic-key')
+      MyNestedSchemaProducer.publish(
+        'test_id' => 'foo',
+        'test_float' => BigDecimal('123.456'),
+        'some_nested_record' => {
+          'some_int' => 123,
+          'some_float' => BigDecimal('456.789'),
+          'some_string' => '123',
+          'some_optional_int' => nil
+        },
+        'some_optional_record' => nil
+      )
+      expect(MyNestedSchemaProducer.topic).to have_sent(
+        'test_id' => 'foo',
+        'test_float' => 123.456,
+        'some_nested_record' => {
+          'some_int' => 123,
+          'some_float' => 456.789,
+          'some_string' => '123',
+          'some_optional_int' => nil
+        },
+        'some_optional_record' => nil
+      )
+    end
+
     it 'should error with nothing set' do
       expect {
         MyErrorProducer.publish_list(

data/spec/schemas/com/my-namespace/Generated.avsc

@@ -0,0 +1,71 @@
+{
+  "namespace": "com.my-namespace",
+  "name": "Generated",
+  "type": "record",
+  "doc": "Test schema",
+  "fields": [
+    {
+      "name": "a_string",
+      "type": "string"
+    },
+    {
+      "name": "a_int",
+      "type": "int"
+    },
+    {
+      "name": "a_long",
+      "type": "long"
+    },
+    {
+      "name": "a_float",
+      "type": "float"
+    },
+    {
+      "name": "a_double",
+      "type": "double"
+    },
+    {
+      "name": "an_enum",
+      "type": {
+        "type": "enum",
+        "name": "AnEnum",
+        "symbols": ["sym1", "sym2"]
+      }
+    },
+    {
+      "name": "an_array",
+      "type": {
+        "type": "array",
+        "items": "int"
+      }
+    },
+    {
+      "name": "a_map",
+      "type": {
+        "type": "map",
+        "values": "string"
+      }
+    },
+    {
+      "name": "timestamp",
+      "type": "string"
+    },
+    {
+      "name": "message_id",
+      "type": "string"
+    },
+    {
+      "name": "a_record",
+      "type": {
+        "type": "record",
+        "name": "ARecord",
+        "fields": [
+          {
+            "name": "a_record_field",
+            "type": "string"
+          }
+        ]
+      }
+    }
+  ]
+}

data/spec/schemas/com/my-namespace/MyNestedSchema.avsc

@@ -0,0 +1,55 @@
+{
+  "namespace": "com.my-namespace",
+  "name": "MyNestedSchema",
+  "type": "record",
+  "doc": "Test schema",
+  "fields": [
+    {
+      "name": "test_id",
+      "type": "string",
+      "doc": "test string"
+    },
+    {
+      "name": "test_float",
+      "type": "float",
+      "doc": "test float"
+    },
+    {
+      "name": "some_nested_record",
+      "doc": "some nested record",
+      "type": {
+        "name": "MyNestedRecord",
+        "type": "record",
+        "fields": [
+          {
+            "name": "some_int",
+            "type": "int",
+            "doc": "some int"
+          },
+          {
+            "name": "some_float",
+            "type": "float",
+            "doc": "some float"
+          },
+          {
+            "name": "some_string",
+            "type": "string",
+            "doc": "some string"
+          },
+          {
+            "name": "some_optional_int",
+            "type": [ "null", "int" ],
+            "doc": "some optional int",
+            "default": null
+          }
+        ]
+      }
+    },
+    {
+      "name": "some_optional_record",
+      "doc": "some optional record",
+      "type": [ "null", "MyNestedRecord" ],
+      "default": null
+    }
+  ]
+}

data/spec/schemas/com/my-namespace/MySchemaCompound-key.avsc

@@ -0,0 +1,18 @@
+{
+  "namespace": "com.my-namespace",
+  "name": "MySchemaCompound-key",
+  "type": "record",
+  "doc": "Test schema",
+  "fields": [
+    {
+      "name": "part_one",
+      "type": "string",
+      "doc": "test string one"
+    },
+    {
+      "name": "part_two",
+      "type": "string",
+      "doc": "test string two"
+    }
+  ]
+}

data/spec/schemas/com/my-namespace/Wibble.avsc

@@ -0,0 +1,43 @@
+{
+  "namespace": "com.my-namespace",
+  "name": "Wibble",
+  "type": "record",
+  "fields": [
+    {
+      "name": "id",
+      "type": "long"
+    },
+    {
+      "name": "wibble_id",
+      "type": "long"
+    },
+    {
+      "name": "name",
+      "type": "string"
+    },
+    {
+      "name": "floop",
+      "type": "string"
+    },
+    {
+      "name": "birthday_int",
+      "type": "int"
+    },
+    {
+      "name": "birthday_long",
+      "type": "long"
+    },
+    {
+      "name": "birthday_optional",
+      "type": ["null", "int"]
+    },
+    {
+      "name": "updated_at",
+      "type": "long"
+    },
+    {
+      "name": "created_at",
+      "type": "long"
+    }
+  ]
+}