deimos-ruby 1.6.3 → 1.8.1.pre.beta1

data/spec/deimos_spec.rb

@@ -68,11 +68,11 @@ describe Deimos do
 
   describe '#start_db_backend!' do
     it 'should start if backend is db and thread_count is > 0' do
-      signal_handler = instance_double(Deimos::Utils::SignalHandler)
+      signal_handler = instance_double(Sigurd::SignalHandler)
       allow(signal_handler).to receive(:run!)
-      expect(Deimos::Utils::Executor).to receive(:new).
+      expect(Sigurd::Executor).to receive(:new).
         with(anything, sleep_seconds: 5, logger: anything).and_call_original
-      expect(Deimos::Utils::SignalHandler).to receive(:new) do |executor|
+      expect(Sigurd::SignalHandler).to receive(:new) do |executor|
         expect(executor.runners.size).to eq(2)
         signal_handler
       end
@@ -83,7 +83,7 @@ describe Deimos do
     end
 
     it 'should not start if backend is not db' do
-      expect(Deimos::Utils::SignalHandler).not_to receive(:new)
+      expect(Sigurd::SignalHandler).not_to receive(:new)
       described_class.configure do |config|
         config.producers.backend = :kafka
       end
@@ -92,7 +92,7 @@ describe Deimos do
     end
 
     it 'should not start if thread_count is nil' do
-      expect(Deimos::Utils::SignalHandler).not_to receive(:new)
+      expect(Sigurd::SignalHandler).not_to receive(:new)
       described_class.configure do |config|
         config.producers.backend = :db
       end
@@ -101,61 +101,69 @@ describe Deimos do
     end
 
     it 'should not start if thread_count is 0' do
-      expect(Deimos::Utils::SignalHandler).not_to receive(:new)
+      expect(Sigurd::SignalHandler).not_to receive(:new)
       described_class.configure do |config|
         config.producers.backend = :db
       end
       expect { described_class.start_db_backend!(thread_count: 0) }.
         to raise_error('Thread count is not given or set to zero, exiting')
     end
+  end
 
-    describe 'delivery configuration' do
-      before(:each) do
-        allow(YAML).to receive(:load).and_return(phobos_configuration)
-      end
-
-      it 'should not raise an error with properly configured handlers' do
-        path = config_path # for scope issues in the block below
-        # Add explicit consumers
-        phobos_configuration['listeners'] << { 'handler' => 'ConsumerTest::MyConsumer',
-                                               'delivery' => 'message' }
-        phobos_configuration['listeners'] << { 'handler' => 'ConsumerTest::MyConsumer',
-                                               'delivery' => 'batch' }
-
-        expect {
-          described_class.configure { |c| c.phobos_config_file = path }
-        }.not_to raise_error
-      end
-
-      it 'should raise an error if BatchConsumers do not have inline_batch delivery' do
-        path = config_path # for scope issues in the block below
-        phobos_configuration['listeners'] = [{ 'handler' => 'ConsumerTest::MyBatchConsumer',
-                                               'delivery' => 'message' }]
-
-        expect {
-          described_class.configure { |c| c.phobos_config_file = path }
-        }.to raise_error('BatchConsumer ConsumerTest::MyBatchConsumer must have delivery set to `inline_batch`')
-      end
-
-      it 'should raise an error if Consumers do not have message or batch delivery' do
-        path = config_path # for scope issues in the block below
-        phobos_configuration['listeners'] = [{ 'handler' => 'ConsumerTest::MyConsumer',
-                                               'delivery' => 'inline_batch' }]
+  describe 'delivery configuration' do
+    before(:each) do
+      allow(YAML).to receive(:load).and_return(phobos_configuration)
+    end
 
-        expect {
-          described_class.configure { |c| c.phobos_config_file = path }
-        }.to raise_error('Non-batch Consumer ConsumerTest::MyConsumer must have delivery set to `message` or `batch`')
-      end
+    it 'should not raise an error with properly configured handlers' do
+      expect {
+        described_class.configure do
+          consumer do
+            class_name 'ConsumerTest::MyConsumer'
+            delivery :message
+          end
+          consumer do
+            class_name 'ConsumerTest::MyConsumer'
+            delivery :batch
+          end
+          consumer do
+            class_name 'ConsumerTest::MyBatchConsumer'
+            delivery :inline_batch
+          end
+        end
+      }.not_to raise_error
+    end
 
-      it 'should treat nil as `batch`' do
-        path = config_path # for scope issues in the block below
-        phobos_configuration['listeners'] = [{ 'handler' => 'ConsumerTest::MyConsumer' }]
+    it 'should raise an error if inline_batch listeners do not implement consume_batch' do
+      expect {
+        described_class.configure do
+          consumer do
+            class_name 'ConsumerTest::MyConsumer'
+            delivery :inline_batch
+          end
+        end
+      }.to raise_error('BatchConsumer ConsumerTest::MyConsumer does not implement `consume_batch`')
+    end
 
-        expect {
-          described_class.configure { |c| c.phobos_config_file = path }
-        }.not_to raise_error
-      end
+    it 'should raise an error if Consumers do not have message or batch delivery' do
+      expect {
+        described_class.configure do
+          consumer do
+            class_name 'ConsumerTest::MyBatchConsumer'
+            delivery :message
+          end
+        end
+      }.to raise_error('Non-batch Consumer ConsumerTest::MyBatchConsumer does not implement `consume`')
+    end
 
+    it 'should treat nil as `batch`' do
+      expect {
+        described_class.configure do
+          consumer do
+            class_name 'ConsumerTest::MyConsumer'
+          end
+        end
+      }.not_to raise_error
     end
   end
 end

data/spec/generators/active_record_generator_spec.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require 'generators/deimos/active_record_generator'
+
+RSpec.describe Deimos::Generators::ActiveRecordGenerator do
+
+  after(:each) do
+    FileUtils.rm_rf('db') if File.exist?('db')
+    FileUtils.rm_rf('app') if File.exist?('app')
+  end
+
+  it 'should generate a migration' do
+    expect(Dir['db/migrate/*.rb']).to be_empty
+    expect(Dir['app/models/*.rb']).to be_empty
+    described_class.start(['generated_table', 'com.my-namespace.Generated'])
+    files = Dir['db/migrate/*.rb']
+    expect(files.length).to eq(1)
+    results = <<~MIGRATION
+      class CreateGeneratedTable < ActiveRecord::Migration[6.0]
+        def up
+          if table_exists?(:generated_table)
+            warn "generated_table already exists, exiting"
+            return
+          end
+          create_table :generated_table do |t|
+            t.string :a_string
+            t.integer :a_int
+            t.bigint :a_long
+            t.float :a_float
+            t.float :a_double
+            t.string :an_enum
+            t.json :an_array
+            t.json :a_map
+            t.json :a_record
+          end
+      
+          # TODO add indexes as necessary
+        end
+      
+        def down
+          return unless table_exists?(:generated_table)
+          drop_table :generated_table
+        end
+      
+      end
+    MIGRATION
+    expect(File.read(files[0])).to eq(results)
+    model = <<~MODEL
+      class GeneratedTable < ApplicationRecord
+        enum an_enum: {sym1: 'sym1', sym2: 'sym2'}
+      end
+    MODEL
+    expect(File.read('app/models/generated_table.rb')).to eq(model)
+  end
+
+end

data/spec/handlers/my_batch_consumer.rb

@@ -1,5 +1,10 @@
 # frozen_string_literal: true
 
 module ConsumerTest
-  class MyBatchConsumer < Deimos::BatchConsumer; end
+  # Mock consumer
+  class MyBatchConsumer < Deimos::Consumer
+    # :no-doc:
+    def consume_batch
+    end
+  end
 end

data/spec/handlers/my_consumer.rb

@@ -1,5 +1,10 @@
 # frozen_string_literal: true
 
 module ConsumerTest
-  class MyConsumer < Deimos::Consumer; end
+  # Mock consumer
+  class MyConsumer < Deimos::Consumer
+    # :no-doc:
+    def consume
+    end
+  end
 end

data/spec/kafka_topic_info_spec.rb

@@ -37,22 +37,45 @@ each_db_config(Deimos::KafkaTopicInfo) do
   end
 
   specify '#clear_lock' do
-    described_class.create!(topic: 'my-topic', locked_by: 'abc',
-                            locked_at: 10.seconds.ago, error: true, retries: 1)
-    described_class.create!(topic: 'my-topic2', locked_by: 'def',
-                            locked_at: 10.seconds.ago, error: true, retries: 1)
-    described_class.clear_lock('my-topic', 'abc')
-    expect(described_class.count).to eq(2)
-    record = described_class.first
-    expect(record.locked_by).to eq(nil)
-    expect(record.locked_at).to eq(nil)
-    expect(record.error).to eq(false)
-    expect(record.retries).to eq(0)
-    record = described_class.last
-    expect(record.locked_by).not_to eq(nil)
-    expect(record.locked_at).not_to eq(nil)
-    expect(record.error).not_to eq(false)
-    expect(record.retries).not_to eq(0)
+    freeze_time do
+      Deimos::KafkaTopicInfo.create!(topic: 'my-topic', locked_by: 'abc',
+                                     locked_at: 10.seconds.ago, error: true, retries: 1,
+                                     last_processed_at: 20.seconds.ago)
+      Deimos::KafkaTopicInfo.create!(topic: 'my-topic2', locked_by: 'def',
+                                     locked_at: 10.seconds.ago, error: true, retries: 1,
+                                     last_processed_at: 20.seconds.ago)
+      Deimos::KafkaTopicInfo.clear_lock('my-topic', 'abc')
+      expect(Deimos::KafkaTopicInfo.count).to eq(2)
+      record = Deimos::KafkaTopicInfo.first
+      expect(record.locked_by).to eq(nil)
+      expect(record.locked_at).to eq(nil)
+      expect(record.error).to eq(false)
+      expect(record.retries).to eq(0)
+      expect(record.last_processed_at.to_s).to eq(Time.zone.now.to_s)
+      record = Deimos::KafkaTopicInfo.last
+      expect(record.locked_by).not_to eq(nil)
+      expect(record.locked_at).not_to eq(nil)
+      expect(record.error).not_to eq(false)
+      expect(record.retries).not_to eq(0)
+      expect(record.last_processed_at.to_s).to eq(20.seconds.ago.to_s)
+    end
+  end
+
+  specify '#ping_empty_topics' do
+    freeze_time do
+      old_time = 1.hour.ago.to_s
+      t1 = Deimos::KafkaTopicInfo.create!(topic: 'topic1', last_processed_at: old_time)
+      t2 = Deimos::KafkaTopicInfo.create!(topic: 'topic2', last_processed_at: old_time)
+      t3 = Deimos::KafkaTopicInfo.create!(topic: 'topic3', last_processed_at: old_time,
+                                          locked_by: 'me', locked_at: 1.minute.ago)
+
+      expect(Deimos::KafkaTopicInfo.count).to eq(3)
+      Deimos::KafkaTopicInfo.all.each { |t| expect(t.last_processed_at.to_s).to eq(old_time) }
+      Deimos::KafkaTopicInfo.ping_empty_topics(%w(topic1))
+      expect(t1.reload.last_processed_at.to_s).to eq(old_time) # was passed as an exception
+      expect(t2.reload.last_processed_at.to_s).to eq(Time.zone.now.to_s)
+      expect(t3.reload.last_processed_at.to_s).to eq(old_time) # is locked
+    end
   end
 
   specify '#register_error' do

data/spec/message_spec.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+RSpec.describe(Deimos::Message) do
+  it 'should detect tombstones' do
+    expect(described_class.new(nil, nil, key: 'key1')).
+      to be_tombstone
+    expect(described_class.new({ v: 'val1' }, nil, key: 'key1')).
+      not_to be_tombstone
+    expect(described_class.new({ v: '' }, nil, key: 'key1')).
+      not_to be_tombstone
+    expect(described_class.new({ v: 'val1' }, nil, key: nil)).
+      not_to be_tombstone
+  end
+
+  it 'can support complex keys/values' do
+    expect { described_class.new({ a: 1, b: 2 }, nil, key: { c: 3, d: 4 }) }.
+      not_to raise_exception
+  end
+end

data/spec/producer_spec.rb

@@ -148,7 +148,7 @@ module ProducerTest
         Deimos.disable_producers do
           raise 'OH NOES'
         end
-      }      .to raise_error('OH NOES')
+      }.to raise_error('OH NOES')
       expect(Deimos).not_to be_producers_disabled
     end
 
@@ -246,7 +246,7 @@ module ProducerTest
         MyNonEncodedProducer.publish_list(
           [{ 'test_id' => 'foo', 'some_int' => 123 }]
         )
-      }      .to raise_error('No key given but a key is required! Use `key_config none: true` to avoid using keys.')
+      }.to raise_error('No key given but a key is required! Use `key_config none: true` to avoid using keys.')
     end
 
     it 'should allow nil keys if none: true is configured' do
@@ -254,7 +254,7 @@ module ProducerTest
         MyNoKeyProducer.publish_list(
           [{ 'test_id' => 'foo', 'some_int' => 123 }]
         )
-      }      .not_to raise_error
+      }.not_to raise_error
     end
 
     it 'should use a partition key' do

data/spec/rake_spec.rb

@@ -9,7 +9,7 @@ if Rake.application.lookup(:environment).nil?
   Rake::Task.define_task(:environment)
 end
 
-describe 'Rakefile' do # rubocop:disable RSpec/DescribeClass
+describe 'Rakefile' do
   it 'should start listeners' do
     runner = instance_double(Phobos::CLI::Runner)
     expect(Phobos::CLI::Runner).to receive(:new).and_return(runner)

data/spec/schemas/com/my-namespace/Generated.avsc

@@ -0,0 +1,71 @@
+{
+  "namespace": "com.my-namespace",
+  "name": "Generated",
+  "type": "record",
+  "doc": "Test schema",
+  "fields": [
+    {
+      "name": "a_string",
+      "type": "string"
+    },
+    {
+      "name": "a_int",
+      "type": "int"
+    },
+    {
+      "name": "a_long",
+      "type": "long"
+    },
+    {
+      "name": "a_float",
+      "type": "float"
+    },
+    {
+      "name": "a_double",
+      "type": "double"
+    },
+    {
+      "name": "an_enum",
+      "type": {
+        "type": "enum",
+        "name": "AnEnum",
+        "symbols": ["sym1", "sym2"]
+      }
+    },
+    {
+      "name": "an_array",
+      "type": {
+        "type": "array",
+        "items": "int"
+      }
+    },
+    {
+      "name": "a_map",
+      "type": {
+        "type": "map",
+        "values": "string"
+      }
+    },
+    {
+      "name": "timestamp",
+      "type": "string"
+    },
+    {
+      "name": "message_id",
+      "type": "string"
+    },
+    {
+      "name": "a_record",
+      "type": {
+        "type": "record",
+        "name": "ARecord",
+        "fields": [
+          {
+            "name": "a_record_field",
+            "type": "string"
+          }
+        ]
+      }
+    }
+  ]
+}

data/spec/schemas/com/my-namespace/MySchemaCompound-key.avsc

@@ -0,0 +1,18 @@
+{
+  "namespace": "com.my-namespace",
+  "name": "MySchemaCompound-key",
+  "type": "record",
+  "doc": "Test schema",
+  "fields": [
+    {
+      "name": "part_one",
+      "type": "string",
+      "doc": "test string one"
+    },
+    {
+      "name": "part_two",
+      "type": "string",
+      "doc": "test string two"
+    }
+  ]
+}

data/spec/schemas/com/my-namespace/Wibble.avsc

@@ -0,0 +1,43 @@
+{
+  "namespace": "com.my-namespace",
+  "name": "Wibble",
+  "type": "record",
+  "fields": [
+    {
+      "name": "id",
+      "type": "long"
+    },
+    {
+      "name": "wibble_id",
+      "type": "long"
+    },
+    {
+      "name": "name",
+      "type": "string"
+    },
+    {
+      "name": "floop",
+      "type": "string"
+    },
+    {
+      "name": "birthday_int",
+      "type": "int"
+    },
+    {
+      "name": "birthday_long",
+      "type": "long"
+    },
+    {
+      "name": "birthday_optional",
+      "type": ["null", "int"]
+    },
+    {
+      "name": "updated_at",
+      "type": "long"
+    },
+    {
+      "name": "created_at",
+      "type": "long"
+    }
+  ]
+}