deimos-ruby 1.11.2 → 1.12.3

data/spec/config/configuration_spec.rb

@@ -21,6 +21,7 @@ describe Deimos, 'configuration' do
       kafka_logger logger
       reraise_consumer_errors true
       schema_registry_url 'http://schema.registry'
+      schema.use_schema_classes false
       seed_broker 'whatever'
       schema_path 'some_path'
       producer_schema_namespace 'namespace'
@@ -37,6 +38,7 @@ describe Deimos, 'configuration' do
     expect(described_class.config.kafka.logger).to eq(logger)
     expect(described_class.config.consumers.reraise_errors).to eq(true)
     expect(described_class.config.schema.registry_url).to eq('http://schema.registry')
+    expect(described_class.config.schema.use_schema_classes).to eq(false)
     expect(described_class.config.kafka.seed_brokers).to eq('whatever')
     expect(described_class.config.producers.schema_namespace).to eq('namespace')
     expect(described_class.config.producers.topic_prefix).to eq('prefix')
@@ -86,7 +88,8 @@ describe Deimos, 'configuration' do
           offset_commit_threshold: 0,
           offset_retention_time: nil,
           heartbeat_interval: 10,
-          handler: 'ConsumerTest::MyConsumer'
+          handler: 'ConsumerTest::MyConsumer',
+          use_schema_classes: nil
         }, {
           topic: 'my_batch_consume_topic',
           group_id: 'my_batch_group_id',
@@ -102,7 +105,8 @@ describe Deimos, 'configuration' do
           offset_commit_threshold: 0,
           offset_retention_time: nil,
           heartbeat_interval: 10,
-          handler: 'ConsumerTest::MyBatchConsumer'
+          handler: 'ConsumerTest::MyBatchConsumer',
+          use_schema_classes: nil
         }
       ],
       producer: {
@@ -178,6 +182,7 @@ describe Deimos, 'configuration' do
         offset_commit_threshold 13
         offset_retention_time 13
         heartbeat_interval 13
+        use_schema_classes false
       end
       consumer do
         disabled true
@@ -185,6 +190,7 @@ describe Deimos, 'configuration' do
         schema 'blah2'
         topic 'blah2'
         group_id 'myconsumerid2'
+        use_schema_classes false
       end
     end
 
@@ -227,7 +233,8 @@ describe Deimos, 'configuration' do
             offset_commit_threshold: 13,
             offset_retention_time: 13,
             heartbeat_interval: 13,
-            handler: 'MyConfigConsumer'
+            handler: 'MyConfigConsumer',
+            use_schema_classes: false
           }
         ],
         producer: {

data/spec/consumer_spec.rb

@@ -3,7 +3,6 @@
 # :nodoc:
 module ConsumerTest
   describe Deimos::Consumer, 'Message Consumer' do
-
     prepend_before(:each) do
       # :nodoc:
       consumer_class = Class.new(described_class) do
@@ -13,7 +12,7 @@ module ConsumerTest
 
         # :nodoc:
         def fatal_error?(_exception, payload, _metadata)
-          payload == 'fatal'
+          payload.to_s == 'fatal'
         end
 
         # :nodoc:
@@ -24,98 +23,110 @@ module ConsumerTest
       stub_const('ConsumerTest::MyConsumer', consumer_class)
     end
 
-    it 'should consume a message' do
-      test_consume_message(MyConsumer,
-                           'test_id' => 'foo',
-                           'some_int' => 123) do |payload, _metadata|
-                             expect(payload['test_id']).to eq('foo')
-                           end
-    end
+    describe 'consume' do
+      SCHEMA_CLASS_SETTINGS.each do |setting, use_schema_classes|
+        context "with Schema Class consumption #{setting}" do
+          before(:each) do
+            Deimos.configure { |config| config.schema.use_schema_classes = use_schema_classes }
+          end
 
-    it 'should consume a nil message' do
-      test_consume_message(MyConsumer, nil) do |payload, _metadata|
-        expect(payload).to be_nil
-      end
-    end
+          it 'should consume a message' do
+            test_consume_message(MyConsumer,
+                                 'test_id' => 'foo',
+                                 'some_int' => 123) do |payload, _metadata|
+                                   expect(payload['test_id']).to eq('foo')
+                                   expect(payload['some_int']).to eq(123)
+                                 end
+          end
 
-    it 'should consume a message idempotently' do
-      # testing for a crash and re-consuming the same message/metadata
-      key = { 'test_id' => 'foo' }
-      test_metadata = { key: key }
-      allow_any_instance_of(MyConsumer).to(receive(:decode_key)) do |_instance, k|
-        k['test_id']
-      end
-      MyConsumer.new.around_consume({ 'test_id' => 'foo',
-                                      'some_int' => 123 }, test_metadata) do |_payload, metadata|
-                                        expect(metadata[:key]).to eq('foo')
-                                      end
-      MyConsumer.new.around_consume({ 'test_id' => 'foo',
-                                      'some_int' => 123 }, test_metadata) do |_payload, metadata|
-                                        expect(metadata[:key]).to eq('foo')
-                                      end
-    end
+          it 'should consume a nil message' do
+            test_consume_message(MyConsumer, nil) do |payload, _metadata|
+              expect(payload).to be_nil
+            end
+          end
 
-    it 'should consume a message on a topic' do
-      test_consume_message('my_consume_topic',
-                           'test_id' => 'foo',
-                           'some_int' => 123) do |payload, _metadata|
-                             expect(payload['test_id']).to eq('foo')
-                           end
-    end
+          it 'should consume a message idempotently' do
+            # testing for a crash and re-consuming the same message/metadata
+            key = { 'test_id' => 'foo' }
+            test_metadata = { key: key }
+            allow_any_instance_of(MyConsumer).to(receive(:decode_key)) do |_instance, k|
+              k['test_id']
+            end
+            MyConsumer.new.around_consume({ 'test_id' => 'foo',
+                                            'some_int' => 123 }, test_metadata) do |_payload, metadata|
+                                              expect(metadata[:key]).to eq('foo')
+                                            end
+            MyConsumer.new.around_consume({ 'test_id' => 'foo',
+                                            'some_int' => 123 }, test_metadata) do |_payload, metadata|
+                                              expect(metadata[:key]).to eq('foo')
+                                            end
+          end
 
-    it 'should fail on invalid message' do
-      test_consume_invalid_message(MyConsumer, 'invalid' => 'key')
-    end
+          it 'should consume a message on a topic' do
+            test_consume_message('my_consume_topic',
+                                 'test_id' => 'foo',
+                                 'some_int' => 123) do |payload, _metadata|
+                                   expect(payload['test_id']).to eq('foo')
+                                   expect(payload['some_int']).to eq(123)
+                                 end
+          end
 
-    it 'should fail if reraise is false but fatal_error is true' do
-      Deimos.configure { |config| config.consumers.reraise_errors = false }
-      test_consume_invalid_message(MyConsumer, 'fatal')
-    end
+          it 'should fail on invalid message' do
+            test_consume_invalid_message(MyConsumer, 'invalid' => 'key')
+          end
 
-    it 'should fail if fatal_error is true globally' do
-      Deimos.configure do |config|
-        config.consumers.fatal_error = proc { true }
-        config.consumers.reraise_errors = false
-      end
-      test_consume_invalid_message(MyConsumer, 'invalid' => 'key')
-    end
+          it 'should fail if reraise is false but fatal_error is true' do
+            Deimos.configure { |config| config.consumers.reraise_errors = false }
+            test_consume_invalid_message(MyConsumer, 'fatal')
+          end
 
-    it 'should fail on message with extra fields' do
-      test_consume_invalid_message(MyConsumer,
-                                   'test_id' => 'foo',
-                                   'some_int' => 123,
-                                   'extra_field' => 'field name')
-    end
+          it 'should fail if fatal_error is true globally' do
+            Deimos.configure do |config|
+              config.consumers.fatal_error = proc { true }
+              config.consumers.reraise_errors = false
+            end
+            test_consume_invalid_message(MyConsumer, 'invalid' => 'key')
+          end
 
-    it 'should not fail when before_consume fails without reraising errors' do
-      Deimos.configure { |config| config.consumers.reraise_errors = false }
-      expect {
-        test_consume_message(
-          MyConsumer,
-          { 'test_id' => 'foo',
-            'some_int' => 123 },
-          { skip_expectation: true }
-        ) { raise 'OH NOES' }
-      }.not_to raise_error
-    end
+          it 'should fail on message with extra fields' do
+            test_consume_invalid_message(MyConsumer,
+                                         'test_id' => 'foo',
+                                         'some_int' => 123,
+                                         'extra_field' => 'field name')
+          end
 
-    it 'should not fail when consume fails without reraising errors' do
-      Deimos.configure { |config| config.consumers.reraise_errors = false }
-      expect {
-        test_consume_message(
-          MyConsumer,
-          { 'invalid' => 'key' },
-          { skip_expectation: true }
-        )
-      }.not_to raise_error
-    end
+          it 'should not fail when before_consume fails without reraising errors' do
+            Deimos.configure { |config| config.consumers.reraise_errors = false }
+            expect {
+              test_consume_message(
+                MyConsumer,
+                { 'test_id' => 'foo',
+                  'some_int' => 123 },
+                { skip_expectation: true }
+              ) { raise 'OH NOES' }
+            }.not_to raise_error
+          end
+
+          it 'should not fail when consume fails without reraising errors' do
+            Deimos.configure { |config| config.consumers.reraise_errors = false }
+            expect {
+              test_consume_message(
+                MyConsumer,
+                { 'invalid' => 'key' },
+                { skip_expectation: true }
+              )
+            }.not_to raise_error
+          end
 
-    it 'should call original' do
-      expect {
-        test_consume_message(MyConsumer,
-                             { 'test_id' => 'foo', 'some_int' => 123 },
-                             { call_original: true })
-      }.to raise_error('This should not be called unless call_original is set')
+          it 'should call original' do
+            expect {
+              test_consume_message(MyConsumer,
+                                   { 'test_id' => 'foo', 'some_int' => 123 },
+                                   { call_original: true })
+            }.to raise_error('This should not be called unless call_original is set')
+          end
+        end
+      end
     end
 
     describe 'decode_key' do

data/spec/generators/active_record_generator_spec.rb

@@ -28,6 +28,7 @@ RSpec.describe Deimos::Generators::ActiveRecordGenerator do
             t.bigint :a_long
             t.float :a_float
             t.float :a_double
+            t.string :an_optional_int
             t.string :an_enum
             t.json :an_array
             t.json :a_map

data/spec/generators/schema_class/my_schema_with_complex_types_spec.rb

@@ -0,0 +1,206 @@
+# frozen_string_literal: true
+
+# For testing the generated class.
+RSpec.describe Schemas::MySchemaWithComplexTypes do
+  let(:payload_hash) do
+    {
+      test_id: 'test id',
+      test_float: 1.2,
+      test_string_array: %w(abc def),
+      test_int_array: [123, 456],
+      some_integer_map: { 'int_1' => 1, 'int_2' => 2 },
+      some_record: Schemas::ARecord.new(a_record_field: 'field 1'),
+      some_optional_record: Schemas::ARecord.new(a_record_field: 'field 2'),
+      some_record_array: [Schemas::ARecord.new(a_record_field: 'field 3'),
+                          Schemas::ARecord.new(a_record_field: 'field 4')],
+      some_record_map: {
+        'record_1' => Schemas::ARecord.new(a_record_field: 'field 5'),
+        'record_2' => Schemas::ARecord.new(a_record_field: 'field 6')
+      },
+      some_enum_array: [Schemas::AnEnum.new('sym1'),
+                        Schemas::AnEnum.new('sym2')]
+    }
+  end
+
+  describe 'class initialization' do
+
+    it 'should initialize the class from keyword arguments' do
+      klass = described_class.new(
+        test_id: payload_hash[:test_id],
+        test_float: payload_hash[:test_float],
+        test_string_array: payload_hash[:test_string_array],
+        some_record: payload_hash[:some_record],
+        some_optional_record: payload_hash[:some_optional_record],
+        some_record_array: payload_hash[:some_record_array],
+        some_record_map: payload_hash[:some_record_map],
+        some_enum_array: payload_hash[:some_enum_array]
+      )
+      expect(klass).to be_instance_of(described_class)
+    end
+
+    it 'should initialize the class from a hash with symbols as keys' do
+      klass = described_class.new(**payload_hash)
+      expect(klass).to be_instance_of(described_class)
+    end
+
+    it 'should initialize the class when missing attributes' do
+      payload_hash.delete(:test_id)
+      klass = described_class.new(**payload_hash)
+      expect(klass).to be_instance_of(described_class)
+    end
+
+  end
+
+  describe 'base class methods' do
+    let(:klass) do
+      described_class.new(**payload_hash)
+    end
+
+    let(:schema_fields) do
+      %w(test_id test_float test_int_array test_optional_int test_string_array some_integer_map some_record some_optional_record some_record_array some_record_map some_enum_array)
+    end
+
+    it 'should return the name of the schema and namespace' do
+      expect(klass.schema).to eq('MySchemaWithComplexTypes')
+      expect(klass.namespace).to eq('com.my-namespace')
+      expect(klass.full_schema).to eq('com.my-namespace.MySchemaWithComplexTypes')
+    end
+
+    it 'should return an array of all fields in the schema' do
+      expect(klass.schema_fields).to match_array(schema_fields)
+    end
+
+    it 'should return a json version of the payload' do
+      described_class.new(**payload_hash)
+      payload_h = {
+        'test_id' => 'test id',
+        'test_float' => 1.2,
+        'test_string_array' => %w(abc def),
+        'test_int_array' => [123, 456],
+        'test_optional_int' => 123,
+        'some_integer_map' => { 'int_1' => 1, 'int_2' => 2 },
+        'some_record' => { 'a_record_field' => 'field 1' },
+        'some_optional_record' => { 'a_record_field' => 'field 2' },
+        'some_record_array' => [
+          { 'a_record_field' => 'field 3' },
+          { 'a_record_field' => 'field 4' }
+        ],
+        'some_record_map' => {
+          'record_1' => { 'a_record_field' => 'field 5' },
+          'record_2' => { 'a_record_field' => 'field 6' }
+        },
+        'some_enum_array' => %w(sym1 sym2)
+      }
+
+      expect(klass.as_json).to eq(payload_h)
+    end
+
+    it 'should return a JSON string of the payload' do
+      s = '{"test_id":"test id","test_float":1.2,"test_string_array":["abc","def"],"test_int_array":[123,456],"test_optional_int":123,"some_integer_map":{"int_1":1,"int_2":2},"some_record":{"a_record_field":"field 1"},"some_optional_record":{"a_record_field":"field 2"},"some_record_array":[{"a_record_field":"field 3"},{"a_record_field":"field 4"}],"some_record_map":{"record_1":{"a_record_field":"field 5"},"record_2":{"a_record_field":"field 6"}},"some_enum_array":["sym1","sym2"]}'
+      expect(klass.to_json).to eq(s)
+    end
+  end
+
+  describe 'defaults' do
+    it 'should set an_optional_int if it is not provided' do
+      payload_hash.delete(:an_optional_int)
+      klass = described_class.new(**payload_hash)
+      expect(klass.test_optional_int).to eq(123)
+    end
+
+    it 'should set some_record if it is not provided' do
+      payload_hash.delete(:some_record)
+      klass = described_class.new(**payload_hash)
+      expect(klass.some_record).to eq(Schemas::ARecord.new(a_record_field: 'Test String'))
+    end
+
+    it 'should set some_record to nil' do
+      klass = described_class.new(**payload_hash.merge(some_record: nil))
+      expect(klass.some_record).to be_nil
+    end
+  end
+
+  describe 'getters and setters' do
+    let(:klass) do
+      described_class.new(**payload_hash)
+    end
+
+    context 'when getting attributes' do
+      it 'should get of values of primitive types' do
+        expect(klass.test_id).to eq('test id')
+        expect(klass.test_float).to eq(1.2)
+        expect(klass.test_string_array).to eq(%w(abc def))
+      end
+
+      it 'should get the value of some_record_array' do
+        some_record_array = klass.some_record_array
+        expect(some_record_array.first).to be_instance_of(Schemas::ARecord)
+        expect(some_record_array.first.a_record_field).to eq('field 3')
+      end
+
+      it 'should get the value of some_record_map' do
+        some_record_map = klass.some_record_map
+        expect(some_record_map['record_1']).to be_instance_of(Schemas::ARecord)
+        expect(some_record_map['record_1'].a_record_field).to eq('field 5')
+      end
+
+      it 'should get the value of some_enum_array' do
+        some_enum_array = klass.some_enum_array
+        expect(some_enum_array.first).to be_instance_of(Schemas::AnEnum)
+        expect(some_enum_array.first.an_enum).to eq('sym1')
+      end
+
+      it 'should get the value of some_record' do
+        record = klass.some_record
+        expect(record).to be_instance_of(Schemas::ARecord)
+        expect(record.a_record_field).to eq('field 1')
+        expect(record.to_h).to eq({ 'a_record_field' => 'field 1' })
+      end
+
+      it 'should support Hash-style element access of values' do
+        expect(klass['test_id']).to eq('test id')
+        expect(klass['test_float']).to eq(1.2)
+        expect(klass['test_string_array']).to eq(%w(abc def))
+      end
+    end
+
+    context 'when setting attributes' do
+      it 'should modify the value of test_id' do
+        expect(klass.test_id).to eq('test id')
+
+        klass.test_id = 'something different'
+        expect(klass.test_id).to eq('something different')
+      end
+
+      it 'should modify the value of some_optional_record' do
+        expect(klass.some_optional_record).to eq(Schemas::ARecord.new(a_record_field: 'field 2'))
+        klass.some_optional_record = Schemas::ARecord.new(a_record_field: 'new field')
+
+        expect(klass.some_optional_record).to eq(Schemas::ARecord.new(a_record_field: 'new field'))
+        expect(klass.some_optional_record.as_json).to eq({ 'a_record_field' => 'new field' })
+      end
+
+      it 'should accept a hash object inner records' do
+        klass.some_optional_record = { a_record_field: 'new field' }
+        expect(klass.some_optional_record).to eq(Schemas::ARecord.new(a_record_field: 'new field'))
+        expect(klass.some_optional_record.as_json).to eq({ 'a_record_field' => 'new field' })
+      end
+
+      it 'should modify the value of some_enum_array' do
+        klass.some_enum_array.first.an_enum = 'new_sym'
+        expect(klass.some_enum_array.first).to eq(Schemas::AnEnum.new('new_sym'))
+
+        klass.some_enum_array.second.an_enum = Schemas::AnEnum.new('other_sym')
+        expect(klass.some_enum_array.second.an_enum).to eq('other_sym')
+      end
+
+      it 'should modify the value of some_record_map' do
+        klass.some_record_map['record_1'].a_record_field = 'new field'
+        expect(klass.some_record_map['record_1']).to eq(Schemas::ARecord.new(a_record_field: 'new field'))
+
+        klass.some_record_map['record_2'] = Schemas::ARecord.new(a_record_field: 'other field')
+        expect(klass.some_record_map['record_2']).to eq(Schemas::ARecord.new(a_record_field: 'other field'))
+      end
+    end
+  end
+end

data/spec/generators/schema_class_generator_spec.rb

@@ -0,0 +1,186 @@
+# frozen_string_literal: true
+
+require 'generators/deimos/schema_class_generator'
+require 'fileutils'
+
+RSpec.describe Deimos::Generators::SchemaClassGenerator do
+  let(:schema_class_path) { 'app/lib/schema_classes/com/my-namespace' }
+  let(:expected_files) { Dir['spec/schema_classes/*.rb'] }
+  let(:files) { Dir["#{schema_class_path}/*.rb"] }
+
+  before(:each) do
+    Deimos.config.reset!
+    Deimos.configure do
+      schema.path 'spec/schemas/'
+      schema.generated_class_path 'app/lib/schema_classes'
+      schema.backend :avro_local
+    end
+  end
+
+  after(:each) do
+    FileUtils.rm_rf(schema_class_path) if File.exist?(schema_class_path)
+  end
+
+  context 'with a Consumers Schema' do
+    before(:each) do
+      Deimos.configure do
+        consumer do
+          class_name 'ConsumerTest::MyConsumer'
+          topic 'MyTopic'
+          schema 'Generated'
+          namespace 'com.my-namespace'
+          key_config field: :a_string
+        end
+      end
+      described_class.start
+    end
+
+    it 'should generate the correct number of classes' do
+      expect(files.length).to eq(1)
+    end
+
+    it 'should generate a schema class for generated' do
+      generated_path = files.select { |f| f =~ /generated/ }.first
+      expected_path = expected_files.select { |f| f =~ /generated/ }.first
+
+      expect(File.read(generated_path)).to eq(File.read(expected_path))
+    end
+  end
+
+  context 'with a Consumers Schema with Complex types' do
+    before(:each) do
+      Deimos.configure do
+        consumer do
+          class_name 'ConsumerTest::MyConsumer'
+          topic 'MyTopic'
+          schema 'MySchemaWithComplexTypes'
+          namespace 'com.my-namespace'
+          key_config field: :a_string
+        end
+      end
+      described_class.start
+    end
+
+    it 'should generate the correct number of classes' do
+      expect(files.length).to eq(1)
+    end
+
+    it 'should generate a schema class for my_schema_with_complex_types' do
+      generated_path = files.select { |f| f =~ /my_schema_with_complex_types/ }.first
+      expected_path = expected_files.select { |f| f =~ /my_schema_with_complex_types/ }.first
+
+      expect(File.read(generated_path)).to eq(File.read(expected_path))
+    end
+  end
+
+  context 'with a Producers Schema and a Key' do
+    before(:each) do
+      Deimos.configure do
+        producer do
+          class_name 'ConsumerTest::MyConsumer'
+          topic 'MyTopic'
+          schema 'MySchema'
+          namespace 'com.my-namespace'
+          key_config schema: 'MySchema-key'
+        end
+      end
+      described_class.start
+    end
+
+    it 'should generate the correct number of classes' do
+      expect(files.length).to eq(2)
+    end
+
+    %w(my_schema my_schema_key).each do |klass|
+      it "should generate a schema class for #{klass}" do
+        generated_path = files.select { |f| f =~ /#{klass}\.rb/ }.first
+        expected_path = expected_files.select { |f| f =~ /#{klass}\.rb/ }.first
+
+        expect(File.read(generated_path)).to eq(File.read(expected_path))
+      end
+    end
+  end
+
+  context 'with a Consumers Nested Schema' do
+    before(:each) do
+      Deimos.configure do
+        consumer do
+          class_name 'ConsumerTest::MyConsumer'
+          topic 'MyTopic'
+          schema 'MyNestedSchema'
+          namespace 'com.my-namespace'
+          key_config field: :test_id
+        end
+      end
+      described_class.start
+    end
+
+    it 'should generate the correct number of classes' do
+      expect(files.length).to eq(1)
+    end
+
+    it 'should generate a schema class for my_nested_schema' do
+      generated_path = files.select { |f| f =~ /my_nested_schema/ }.first
+      expected_path = expected_files.select { |f| f =~ /my_nested_schema/ }.first
+
+      expect(File.read(generated_path)).to eq(File.read(expected_path))
+    end
+  end
+
+  context 'with a mix of Consumer and Producer Schemas' do
+    before(:each) do
+      Deimos.configure do
+        consumer do
+          class_name 'ConsumerTest::MyConsumer'
+          topic 'MyTopic'
+          schema 'Generated'
+          namespace 'com.my-namespace'
+          key_config field: :a_string
+        end
+
+        producer do
+          class_name 'ConsumerTest::MyConsumer'
+          topic 'MyTopic'
+          schema 'MySchema'
+          namespace 'com.my-namespace'
+          key_config schema: 'MySchema-key'
+        end
+
+        producer do
+          class_name 'ConsumerTest::MyConsumer'
+          topic 'MyTopic'
+          schema 'MyNestedSchema'
+          namespace 'com.my-namespace'
+          key_config field: :test_id
+        end
+      end
+      described_class.start
+    end
+
+    it 'should generate the correct number of classes' do
+      expect(files.length).to eq(4)
+    end
+
+    %w(generated my_schema my_schema_key my_nested_schema).each do |klass|
+      it "should generate a schema class for #{klass}" do
+        generated_path = files.select { |f| f =~ /#{klass}\.rb/ }.first
+        expected_path = expected_files.select { |f| f =~ /#{klass}\.rb/ }.first
+
+        expect(File.read(generated_path)).to eq(File.read(expected_path))
+      end
+    end
+  end
+
+  context 'with non-avro schema backends' do
+    before(:each) do
+      Deimos.config.schema.backend :mock
+    end
+
+    it 'should fail to start schema class generation' do
+      expect {
+        described_class.start
+      }.to raise_error(message='Schema Class Generation requires an Avro-based Schema Backend')
+    end
+  end
+
+end