deimos-ruby 1.11.0 → 1.12.1

data/spec/generators/schema_class/my_schema_with_complex_types_spec.rb

@@ -0,0 +1,206 @@
+# frozen_string_literal: true
+
+# For testing the generated class.
+RSpec.describe Schemas::MySchemaWithComplexTypes do
+  let(:payload_hash) do
+    {
+      test_id: 'test id',
+      test_float: 1.2,
+      test_string_array: %w(abc def),
+      test_int_array: [123, 456],
+      some_integer_map: { 'int_1' => 1, 'int_2' => 2 },
+      some_record: Schemas::ARecord.new(a_record_field: 'field 1'),
+      some_optional_record: Schemas::ARecord.new(a_record_field: 'field 2'),
+      some_record_array: [Schemas::ARecord.new(a_record_field: 'field 3'),
+                          Schemas::ARecord.new(a_record_field: 'field 4')],
+      some_record_map: {
+        'record_1' => Schemas::ARecord.new(a_record_field: 'field 5'),
+        'record_2' => Schemas::ARecord.new(a_record_field: 'field 6')
+      },
+      some_enum_array: [Schemas::AnEnum.new('sym1'),
+                        Schemas::AnEnum.new('sym2')]
+    }
+  end
+
+  describe 'class initialization' do
+
+    it 'should initialize the class from keyword arguments' do
+      klass = described_class.new(
+        test_id: payload_hash[:test_id],
+        test_float: payload_hash[:test_float],
+        test_string_array: payload_hash[:test_string_array],
+        some_record: payload_hash[:some_record],
+        some_optional_record: payload_hash[:some_optional_record],
+        some_record_array: payload_hash[:some_record_array],
+        some_record_map: payload_hash[:some_record_map],
+        some_enum_array: payload_hash[:some_enum_array]
+      )
+      expect(klass).to be_instance_of(described_class)
+    end
+
+    it 'should initialize the class from a hash with symbols as keys' do
+      klass = described_class.new(**payload_hash)
+      expect(klass).to be_instance_of(described_class)
+    end
+
+    it 'should initialize the class when missing attributes' do
+      payload_hash.delete(:test_id)
+      klass = described_class.new(**payload_hash)
+      expect(klass).to be_instance_of(described_class)
+    end
+
+  end
+
+  describe 'base class methods' do
+    let(:klass) do
+      described_class.new(**payload_hash)
+    end
+
+    let(:schema_fields) do
+      %w(test_id test_float test_int_array test_optional_int test_string_array some_integer_map some_record some_optional_record some_record_array some_record_map some_enum_array)
+    end
+
+    it 'should return the name of the schema and namespace' do
+      expect(klass.schema).to eq('MySchemaWithComplexTypes')
+      expect(klass.namespace).to eq('com.my-namespace')
+      expect(klass.full_schema).to eq('com.my-namespace.MySchemaWithComplexTypes')
+    end
+
+    it 'should return an array of all fields in the schema' do
+      expect(klass.schema_fields).to match_array(schema_fields)
+    end
+
+    it 'should return a json version of the payload' do
+      described_class.new(**payload_hash)
+      payload_h = {
+        'test_id' => 'test id',
+        'test_float' => 1.2,
+        'test_string_array' => %w(abc def),
+        'test_int_array' => [123, 456],
+        'test_optional_int' => 123,
+        'some_integer_map' => { 'int_1' => 1, 'int_2' => 2 },
+        'some_record' => { 'a_record_field' => 'field 1' },
+        'some_optional_record' => { 'a_record_field' => 'field 2' },
+        'some_record_array' => [
+          { 'a_record_field' => 'field 3' },
+          { 'a_record_field' => 'field 4' }
+        ],
+        'some_record_map' => {
+          'record_1' => { 'a_record_field' => 'field 5' },
+          'record_2' => { 'a_record_field' => 'field 6' }
+        },
+        'some_enum_array' => %w(sym1 sym2)
+      }
+
+      expect(klass.as_json).to eq(payload_h)
+    end
+
+    it 'should return a JSON string of the payload' do
+      s = '{"test_id":"test id","test_float":1.2,"test_string_array":["abc","def"],"test_int_array":[123,456],"test_optional_int":123,"some_integer_map":{"int_1":1,"int_2":2},"some_record":{"a_record_field":"field 1"},"some_optional_record":{"a_record_field":"field 2"},"some_record_array":[{"a_record_field":"field 3"},{"a_record_field":"field 4"}],"some_record_map":{"record_1":{"a_record_field":"field 5"},"record_2":{"a_record_field":"field 6"}},"some_enum_array":["sym1","sym2"]}'
+      expect(klass.to_json).to eq(s)
+    end
+  end
+
+  describe 'defaults' do
+    it 'should set an_optional_int if it is not provided' do
+      payload_hash.delete(:an_optional_int)
+      klass = described_class.new(**payload_hash)
+      expect(klass.test_optional_int).to eq(123)
+    end
+
+    it 'should set some_record if it is not provided' do
+      payload_hash.delete(:some_record)
+      klass = described_class.new(**payload_hash)
+      expect(klass.some_record).to eq(Schemas::ARecord.new(a_record_field: 'Test String'))
+    end
+
+    it 'should set some_record to nil' do
+      klass = described_class.new(**payload_hash.merge(some_record: nil))
+      expect(klass.some_record).to be_nil
+    end
+  end
+
+  describe 'getters and setters' do
+    let(:klass) do
+      described_class.new(**payload_hash)
+    end
+
+    context 'when getting attributes' do
+      it 'should get of values of primitive types' do
+        expect(klass.test_id).to eq('test id')
+        expect(klass.test_float).to eq(1.2)
+        expect(klass.test_string_array).to eq(%w(abc def))
+      end
+
+      it 'should get the value of some_record_array' do
+        some_record_array = klass.some_record_array
+        expect(some_record_array.first).to be_instance_of(Schemas::ARecord)
+        expect(some_record_array.first.a_record_field).to eq('field 3')
+      end
+
+      it 'should get the value of some_record_map' do
+        some_record_map = klass.some_record_map
+        expect(some_record_map['record_1']).to be_instance_of(Schemas::ARecord)
+        expect(some_record_map['record_1'].a_record_field).to eq('field 5')
+      end
+
+      it 'should get the value of some_enum_array' do
+        some_enum_array = klass.some_enum_array
+        expect(some_enum_array.first).to be_instance_of(Schemas::AnEnum)
+        expect(some_enum_array.first.an_enum).to eq('sym1')
+      end
+
+      it 'should get the value of some_record' do
+        record = klass.some_record
+        expect(record).to be_instance_of(Schemas::ARecord)
+        expect(record.a_record_field).to eq('field 1')
+        expect(record.to_h).to eq({ 'a_record_field' => 'field 1' })
+      end
+
+      it 'should support Hash-style element access of values' do
+        expect(klass['test_id']).to eq('test id')
+        expect(klass['test_float']).to eq(1.2)
+        expect(klass['test_string_array']).to eq(%w(abc def))
+      end
+    end
+
+    context 'when setting attributes' do
+      it 'should modify the value of test_id' do
+        expect(klass.test_id).to eq('test id')
+
+        klass.test_id = 'something different'
+        expect(klass.test_id).to eq('something different')
+      end
+
+      it 'should modify the value of some_optional_record' do
+        expect(klass.some_optional_record).to eq(Schemas::ARecord.new(a_record_field: 'field 2'))
+        klass.some_optional_record = Schemas::ARecord.new(a_record_field: 'new field')
+
+        expect(klass.some_optional_record).to eq(Schemas::ARecord.new(a_record_field: 'new field'))
+        expect(klass.some_optional_record.as_json).to eq({ 'a_record_field' => 'new field' })
+      end
+
+      it 'should accept a hash object inner records' do
+        klass.some_optional_record = { a_record_field: 'new field' }
+        expect(klass.some_optional_record).to eq(Schemas::ARecord.new(a_record_field: 'new field'))
+        expect(klass.some_optional_record.as_json).to eq({ 'a_record_field' => 'new field' })
+      end
+
+      it 'should modify the value of some_enum_array' do
+        klass.some_enum_array.first.an_enum = 'new_sym'
+        expect(klass.some_enum_array.first).to eq(Schemas::AnEnum.new('new_sym'))
+
+        klass.some_enum_array.second.an_enum = Schemas::AnEnum.new('other_sym')
+        expect(klass.some_enum_array.second.an_enum).to eq('other_sym')
+      end
+
+      it 'should modify the value of some_record_map' do
+        klass.some_record_map['record_1'].a_record_field = 'new field'
+        expect(klass.some_record_map['record_1']).to eq(Schemas::ARecord.new(a_record_field: 'new field'))
+
+        klass.some_record_map['record_2'] = Schemas::ARecord.new(a_record_field: 'other field')
+        expect(klass.some_record_map['record_2']).to eq(Schemas::ARecord.new(a_record_field: 'other field'))
+      end
+    end
+  end
+end

data/spec/generators/schema_class_generator_spec.rb

@@ -0,0 +1,186 @@
+# frozen_string_literal: true
+
+require 'generators/deimos/schema_class_generator'
+require 'fileutils'
+
+RSpec.describe Deimos::Generators::SchemaClassGenerator do
+  let(:schema_class_path) { 'app/lib/schema_classes/com/my-namespace' }
+  let(:expected_files) { Dir['spec/schema_classes/*.rb'] }
+  let(:files) { Dir["#{schema_class_path}/*.rb"] }
+
+  before(:each) do
+    Deimos.config.reset!
+    Deimos.configure do
+      schema.path 'spec/schemas/'
+      schema.generated_class_path 'app/lib/schema_classes'
+      schema.backend :avro_local
+    end
+  end
+
+  after(:each) do
+    FileUtils.rm_rf(schema_class_path) if File.exist?(schema_class_path)
+  end
+
+  context 'with a Consumers Schema' do
+    before(:each) do
+      Deimos.configure do
+        consumer do
+          class_name 'ConsumerTest::MyConsumer'
+          topic 'MyTopic'
+          schema 'Generated'
+          namespace 'com.my-namespace'
+          key_config field: :a_string
+        end
+      end
+      described_class.start
+    end
+
+    it 'should generate the correct number of classes' do
+      expect(files.length).to eq(1)
+    end
+
+    it 'should generate a schema class for generated' do
+      generated_path = files.select { |f| f =~ /generated/ }.first
+      expected_path = expected_files.select { |f| f =~ /generated/ }.first
+
+      expect(File.read(generated_path)).to eq(File.read(expected_path))
+    end
+  end
+
+  context 'with a Consumers Schema with Complex types' do
+    before(:each) do
+      Deimos.configure do
+        consumer do
+          class_name 'ConsumerTest::MyConsumer'
+          topic 'MyTopic'
+          schema 'MySchemaWithComplexTypes'
+          namespace 'com.my-namespace'
+          key_config field: :a_string
+        end
+      end
+      described_class.start
+    end
+
+    it 'should generate the correct number of classes' do
+      expect(files.length).to eq(1)
+    end
+
+    it 'should generate a schema class for my_schema_with_complex_types' do
+      generated_path = files.select { |f| f =~ /my_schema_with_complex_types/ }.first
+      expected_path = expected_files.select { |f| f =~ /my_schema_with_complex_types/ }.first
+
+      expect(File.read(generated_path)).to eq(File.read(expected_path))
+    end
+  end
+
+  context 'with a Producers Schema and a Key' do
+    before(:each) do
+      Deimos.configure do
+        producer do
+          class_name 'ConsumerTest::MyConsumer'
+          topic 'MyTopic'
+          schema 'MySchema'
+          namespace 'com.my-namespace'
+          key_config schema: 'MySchema-key'
+        end
+      end
+      described_class.start
+    end
+
+    it 'should generate the correct number of classes' do
+      expect(files.length).to eq(2)
+    end
+
+    %w(my_schema my_schema_key).each do |klass|
+      it "should generate a schema class for #{klass}" do
+        generated_path = files.select { |f| f =~ /#{klass}\.rb/ }.first
+        expected_path = expected_files.select { |f| f =~ /#{klass}\.rb/ }.first
+
+        expect(File.read(generated_path)).to eq(File.read(expected_path))
+      end
+    end
+  end
+
+  context 'with a Consumers Nested Schema' do
+    before(:each) do
+      Deimos.configure do
+        consumer do
+          class_name 'ConsumerTest::MyConsumer'
+          topic 'MyTopic'
+          schema 'MyNestedSchema'
+          namespace 'com.my-namespace'
+          key_config field: :test_id
+        end
+      end
+      described_class.start
+    end
+
+    it 'should generate the correct number of classes' do
+      expect(files.length).to eq(1)
+    end
+
+    it 'should generate a schema class for my_nested_schema' do
+      generated_path = files.select { |f| f =~ /my_nested_schema/ }.first
+      expected_path = expected_files.select { |f| f =~ /my_nested_schema/ }.first
+
+      expect(File.read(generated_path)).to eq(File.read(expected_path))
+    end
+  end
+
+  context 'with a mix of Consumer and Producer Schemas' do
+    before(:each) do
+      Deimos.configure do
+        consumer do
+          class_name 'ConsumerTest::MyConsumer'
+          topic 'MyTopic'
+          schema 'Generated'
+          namespace 'com.my-namespace'
+          key_config field: :a_string
+        end
+
+        producer do
+          class_name 'ConsumerTest::MyConsumer'
+          topic 'MyTopic'
+          schema 'MySchema'
+          namespace 'com.my-namespace'
+          key_config schema: 'MySchema-key'
+        end
+
+        producer do
+          class_name 'ConsumerTest::MyConsumer'
+          topic 'MyTopic'
+          schema 'MyNestedSchema'
+          namespace 'com.my-namespace'
+          key_config field: :test_id
+        end
+      end
+      described_class.start
+    end
+
+    it 'should generate the correct number of classes' do
+      expect(files.length).to eq(4)
+    end
+
+    %w(generated my_schema my_schema_key my_nested_schema).each do |klass|
+      it "should generate a schema class for #{klass}" do
+        generated_path = files.select { |f| f =~ /#{klass}\.rb/ }.first
+        expected_path = expected_files.select { |f| f =~ /#{klass}\.rb/ }.first
+
+        expect(File.read(generated_path)).to eq(File.read(expected_path))
+      end
+    end
+  end
+
+  context 'with non-avro schema backends' do
+    before(:each) do
+      Deimos.config.schema.backend :mock
+    end
+
+    it 'should fail to start schema class generation' do
+      expect {
+        described_class.start
+      }.to raise_error(message='Schema Class Generation requires an Avro-based Schema Backend')
+    end
+  end
+
+end

data/spec/producer_spec.rb

@@ -390,6 +390,116 @@ module ProducerTest
                                                       }, '456', '4561')
     end
 
+    context 'with Schema Class payloads' do
+      it 'should fail on invalid message with error handler' do
+        subscriber = Deimos.subscribe('produce') do |event|
+          expect(event.payload[:payloads]).to eq([{ 'invalid' => 'key' }])
+        end
+        expect(MyProducer.encoder).to receive(:validate).and_raise('OH NOES')
+        expect {
+          MyProducer.publish(Schemas::MySchema.new(test_id: 'foo', some_int: 'invalid'))
+        }.to raise_error('OH NOES')
+        Deimos.unsubscribe(subscriber)
+      end
+
+      it 'should produce a message' do
+        expect(described_class).to receive(:produce_batch).once.with(
+          Deimos::Backends::Test,
+          [
+            Deimos::Message.new({ 'test_id' => 'foo', 'some_int' => 123 },
+                                MyProducer,
+                                topic: 'my-topic',
+                                partition_key: 'foo',
+                                key: 'foo'),
+            Deimos::Message.new({ 'test_id' => 'bar', 'some_int' => 124 },
+                                MyProducer,
+                                topic: 'my-topic',
+                                partition_key: 'bar',
+                                key: 'bar')
+          ]
+        ).and_call_original
+
+        MyProducer.publish_list(
+          [Schemas::MySchema.new(test_id: 'foo', some_int: 123),
+           Schemas::MySchema.new(test_id: 'bar', some_int: 124)]
+        )
+        expect('my-topic').to have_sent('test_id' => 'foo', 'some_int' => 123)
+        expect('your-topic').not_to have_sent('test_id' => 'foo', 'some_int' => 123)
+        expect('my-topic').not_to have_sent('test_id' => 'foo2', 'some_int' => 123)
+      end
+
+      it 'should not publish if publish disabled' do
+        expect(described_class).not_to receive(:produce_batch)
+        Deimos.configure { |c| c.producers.disabled = true }
+        MyProducer.publish_list(
+          [Schemas::MySchema.new(test_id: 'foo', some_int: 123),
+           Schemas::MySchema.new(test_id: 'bar', some_int: 124)]
+        )
+        expect(MyProducer.topic).not_to have_sent(anything)
+      end
+
+      it 'should encode the key' do
+        Deimos.configure { |c| c.producers.topic_prefix = nil }
+        expect(MyProducer.encoder).to receive(:encode_key).with('test_id', 'foo', topic: 'my-topic-key')
+        expect(MyProducer.encoder).to receive(:encode_key).with('test_id', 'bar', topic: 'my-topic-key')
+        expect(MyProducer.encoder).to receive(:encode).with({
+                                                              'test_id' => 'foo',
+                                                              'some_int' => 123
+                                                            }, { topic: 'my-topic-value' })
+        expect(MyProducer.encoder).to receive(:encode).with({
+                                                              'test_id' => 'bar',
+                                                              'some_int' => 124
+                                                            }, { topic: 'my-topic-value' })
+
+        MyProducer.publish_list(
+          [Schemas::MySchema.new(test_id: 'foo', some_int: 123),
+           Schemas::MySchema.new(test_id: 'bar', some_int: 124)]
+        )
+      end
+
+      it 'should encode with a schema' do
+        expect(MySchemaProducer.key_encoder).to receive(:encode).with({ 'test_id' => 'foo_key' },
+                                                                      { topic: 'my-topic2-key' })
+        expect(MySchemaProducer.key_encoder).to receive(:encode).with({ 'test_id' => 'bar_key' },
+                                                                      { topic: 'my-topic2-key' })
+
+        MySchemaProducer.publish_list(
+          [Schemas::MySchema.new(test_id: 'foo', some_int: 123, payload_key: { 'test_id' => 'foo_key' }),
+           Schemas::MySchema.new(test_id: 'bar', some_int: 124, payload_key: { 'test_id' => 'bar_key' })]
+        )
+      end
+
+      it 'should properly encode and coerce values with a nested record' do
+        expect(MyNestedSchemaProducer.encoder).to receive(:encode_key).with('test_id', 'foo', topic: 'my-topic-key')
+        MyNestedSchemaProducer.publish(
+          Schemas::MyNestedSchema.new(
+            test_id: 'foo',
+            test_float: BigDecimal('123.456'),
+            test_array: ['1'],
+            some_nested_record: Schemas::MyNestedRecord.new(
+              some_int: 123,
+              some_float: BigDecimal('456.789'),
+              some_string: '123',
+              some_optional_int: nil
+            ),
+            some_optional_record: nil
+          )
+        )
+        expect(MyNestedSchemaProducer.topic).to have_sent(
+          'test_id' => 'foo',
+          'test_float' => 123.456,
+          'test_array' => ['1'],
+          'some_nested_record' => {
+            'some_int' => 123,
+            'some_float' => 456.789,
+            'some_string' => '123',
+            'some_optional_int' => nil
+          },
+          'some_optional_record' => nil
+        )
+      end
+    end
+
     describe 'disabling' do
       it 'should disable globally' do
         Deimos.disable_producers do

data/spec/schema_classes/generated.rb

@@ -0,0 +1,156 @@
+# frozen_string_literal: true
+
+# This file is autogenerated by Deimos, Do NOT modify
+module Schemas
+  ### Secondary Schema Classes ###
+  # Autogenerated Schema for Enum at com.my-namespace.AnEnum
+  class AnEnum < Deimos::SchemaClass::Enum
+    # @return ['sym1', 'sym2']
+    attr_accessor :an_enum
+
+    # :nodoc:
+    def initialize(an_enum)
+      super
+      self.an_enum = an_enum
+    end
+
+    # @override
+    def symbols
+      %w(sym1 sym2)
+    end
+
+    # @override
+    def to_h
+      @an_enum
+    end
+  end
+
+  # Autogenerated Schema for Record at com.my-namespace.ARecord
+  class ARecord < Deimos::SchemaClass::Record
+    ### Attribute Accessors ###
+    # @param value [String]
+    attr_accessor :a_record_field
+
+    # @override
+    def initialize(a_record_field: nil)
+      super
+      self.a_record_field = a_record_field
+    end
+
+    # @override
+    def schema
+      'ARecord'
+    end
+
+    # @override
+    def namespace
+      'com.my-namespace'
+    end
+
+    # @override
+    def to_h
+      {
+        'a_record_field' => @a_record_field
+      }
+    end
+  end
+
+  ### Primary Schema Class ###
+  # Autogenerated Schema for Record at com.my-namespace.Generated
+  class Generated < Deimos::SchemaClass::Record
+    ### Attribute Readers ###
+    # @return [AnEnum]
+    attr_reader :an_enum
+    # @return [ARecord]
+    attr_reader :a_record
+
+    ### Attribute Accessors ###
+    # @param value [String]
+    attr_accessor :a_string
+    # @param value [Integer]
+    attr_accessor :a_int
+    # @param value [Integer]
+    attr_accessor :a_long
+    # @param value [Float]
+    attr_accessor :a_float
+    # @param value [Float]
+    attr_accessor :a_double
+    # @param value [nil, Integer]
+    attr_accessor :an_optional_int
+    # @param values [Array<Integer>]
+    attr_accessor :an_array
+    # @param values [Hash<String, String>]
+    attr_accessor :a_map
+    # @param value [String]
+    attr_accessor :timestamp
+    # @param value [String]
+    attr_accessor :message_id
+
+    ### Attribute Writers ###
+    # @param value [AnEnum]
+    def an_enum=(value)
+      @an_enum = AnEnum.initialize_from_value(value)
+    end
+
+    # @param value [ARecord]
+    def a_record=(value)
+      @a_record = ARecord.initialize_from_value(value)
+    end
+
+    # @override
+    def initialize(a_string: nil,
+                   a_int: nil,
+                   a_long: nil,
+                   a_float: nil,
+                   a_double: nil,
+                   an_optional_int: nil,
+                   an_enum: nil,
+                   an_array: nil,
+                   a_map: nil,
+                   timestamp: nil,
+                   message_id: nil,
+                   a_record: nil)
+      super
+      self.a_string = a_string
+      self.a_int = a_int
+      self.a_long = a_long
+      self.a_float = a_float
+      self.a_double = a_double
+      self.an_optional_int = an_optional_int
+      self.an_enum = an_enum
+      self.an_array = an_array
+      self.a_map = a_map
+      self.timestamp = timestamp
+      self.message_id = message_id
+      self.a_record = a_record
+    end
+
+    # @override
+    def schema
+      'Generated'
+    end
+
+    # @override
+    def namespace
+      'com.my-namespace'
+    end
+
+    # @override
+    def to_h
+      {
+        'a_string' => @a_string,
+        'a_int' => @a_int,
+        'a_long' => @a_long,
+        'a_float' => @a_float,
+        'a_double' => @a_double,
+        'an_optional_int' => @an_optional_int,
+        'an_enum' => @an_enum&.to_h,
+        'an_array' => @an_array,
+        'a_map' => @a_map,
+        'timestamp' => @timestamp,
+        'message_id' => @message_id,
+        'a_record' => @a_record&.to_h
+      }
+    end
+  end
+end