deimos-ruby 1.22.5 → 1.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +2 -2
- data/CHANGELOG.md +12 -0
- data/README.md +6 -0
- data/docs/CONFIGURATION.md +4 -0
- data/lib/deimos/active_record_consume/batch_consumption.rb +43 -20
- data/lib/deimos/active_record_consume/batch_record.rb +5 -4
- data/lib/deimos/active_record_consume/batch_record_list.rb +11 -2
- data/lib/deimos/active_record_consume/mass_updater.rb +13 -4
- data/lib/deimos/active_record_consumer.rb +10 -0
- data/lib/deimos/config/configuration.rb +25 -2
- data/lib/deimos/consume/batch_consumption.rb +2 -2
- data/lib/deimos/test_helpers.rb +1 -0
- data/lib/deimos/tracing/datadog.rb +12 -6
- data/lib/deimos/tracing/mock.rb +31 -2
- data/lib/deimos/tracing/provider.rb +6 -0
- data/lib/deimos/utils/schema_class.rb +10 -2
- data/lib/deimos/version.rb +1 -1
- data/lib/deimos.rb +14 -1
- data/spec/active_record_batch_consumer_association_spec.rb +32 -5
- data/spec/active_record_batch_consumer_spec.rb +376 -59
- data/spec/active_record_consume/mass_updater_spec.rb +46 -3
- data/spec/active_record_consumer_spec.rb +74 -1
- data/spec/active_record_producer_spec.rb +4 -1
- data/spec/batch_consumer_spec.rb +4 -1
- data/spec/config/configuration_spec.rb +42 -3
- data/spec/consumer_spec.rb +42 -1
- data/spec/schemas/my_namespace/my_updated_schema.rb +18 -0
- metadata +5 -3
| @@ -91,7 +91,8 @@ describe Deimos, 'configuration' do | |
| 91 91 | 
             
                      heartbeat_interval: 10,
         | 
| 92 92 | 
             
                      handler: 'ConsumerTest::MyConsumer',
         | 
| 93 93 | 
             
                      use_schema_classes: nil,
         | 
| 94 | 
            -
                      max_db_batch_size: nil
         | 
| 94 | 
            +
                      max_db_batch_size: nil,
         | 
| 95 | 
            +
                      bulk_import_id_generator: nil
         | 
| 95 96 | 
             
                    }, {
         | 
| 96 97 | 
             
                      topic: 'my_batch_consume_topic',
         | 
| 97 98 | 
             
                      group_id: 'my_batch_group_id',
         | 
| @@ -109,7 +110,8 @@ describe Deimos, 'configuration' do | |
| 109 110 | 
             
                      heartbeat_interval: 10,
         | 
| 110 111 | 
             
                      handler: 'ConsumerTest::MyBatchConsumer',
         | 
| 111 112 | 
             
                      use_schema_classes: nil,
         | 
| 112 | 
            -
                      max_db_batch_size: nil
         | 
| 113 | 
            +
                      max_db_batch_size: nil,
         | 
| 114 | 
            +
                      bulk_import_id_generator: nil
         | 
| 113 115 | 
             
                    }
         | 
| 114 116 | 
             
                  ],
         | 
| 115 117 | 
             
                  producer: {
         | 
| @@ -261,7 +263,8 @@ describe Deimos, 'configuration' do | |
| 261 263 | 
             
                        heartbeat_interval: 13,
         | 
| 262 264 | 
             
                        handler: 'MyConfigConsumer',
         | 
| 263 265 | 
             
                        use_schema_classes: false,
         | 
| 264 | 
            -
                        max_db_batch_size: nil
         | 
| 266 | 
            +
                        max_db_batch_size: nil,
         | 
| 267 | 
            +
                        bulk_import_id_generator: nil
         | 
| 265 268 | 
             
                      }
         | 
| 266 269 | 
             
                    ],
         | 
| 267 270 | 
             
                    producer: {
         | 
| @@ -279,4 +282,40 @@ describe Deimos, 'configuration' do | |
| 279 282 | 
             
                    }
         | 
| 280 283 | 
             
                  )
         | 
| 281 284 | 
             
              end
         | 
| 285 | 
            +
             | 
| 286 | 
            +
              it 'should override global configurations' do
         | 
| 287 | 
            +
                described_class.configure do
         | 
| 288 | 
            +
                  consumers.bulk_import_id_generator(-> { 'global' })
         | 
| 289 | 
            +
                  consumers.replace_associations true
         | 
| 290 | 
            +
             | 
| 291 | 
            +
                  consumer do
         | 
| 292 | 
            +
                    class_name 'MyConfigConsumer'
         | 
| 293 | 
            +
                    schema 'blah'
         | 
| 294 | 
            +
                    topic 'blah'
         | 
| 295 | 
            +
                    group_id 'myconsumerid'
         | 
| 296 | 
            +
                    bulk_import_id_generator(-> { 'consumer' })
         | 
| 297 | 
            +
                    replace_associations false
         | 
| 298 | 
            +
                  end
         | 
| 299 | 
            +
             | 
| 300 | 
            +
                  consumer do
         | 
| 301 | 
            +
                    class_name 'MyConfigConsumer2'
         | 
| 302 | 
            +
                    schema 'blah'
         | 
| 303 | 
            +
                    topic 'blah'
         | 
| 304 | 
            +
                    group_id 'myconsumerid'
         | 
| 305 | 
            +
                  end
         | 
| 306 | 
            +
                end
         | 
| 307 | 
            +
             | 
| 308 | 
            +
                consumers = described_class.config.consumers
         | 
| 309 | 
            +
                expect(consumers.replace_associations).to eq(true)
         | 
| 310 | 
            +
                expect(consumers.bulk_import_id_generator.call).to eq('global')
         | 
| 311 | 
            +
             | 
| 312 | 
            +
                custom = MyConfigConsumer.config
         | 
| 313 | 
            +
                expect(custom[:replace_associations]).to eq(false)
         | 
| 314 | 
            +
                expect(custom[:bulk_import_id_generator].call).to eq('consumer')
         | 
| 315 | 
            +
             | 
| 316 | 
            +
                default = MyConfigConsumer2.config
         | 
| 317 | 
            +
                expect(default[:replace_associations]).to eq(true)
         | 
| 318 | 
            +
                expect(default[:bulk_import_id_generator].call).to eq('global')
         | 
| 319 | 
            +
             | 
| 320 | 
            +
              end
         | 
| 282 321 | 
             
            end
         | 
    
        data/spec/consumer_spec.rb
    CHANGED
    
    | @@ -1,6 +1,7 @@ | |
| 1 1 | 
             
            # frozen_string_literal: true
         | 
| 2 2 |  | 
| 3 3 | 
             
            # :nodoc:
         | 
| 4 | 
            +
            # rubocop:disable Metrics/ModuleLength
         | 
| 4 5 | 
             
            module ConsumerTest
         | 
| 5 6 | 
             
              describe Deimos::Consumer, 'Message Consumer' do
         | 
| 6 7 | 
             
                prepend_before(:each) do
         | 
| @@ -26,8 +27,12 @@ module ConsumerTest | |
| 26 27 | 
             
                describe 'consume' do
         | 
| 27 28 | 
             
                  SCHEMA_CLASS_SETTINGS.each do |setting, use_schema_classes|
         | 
| 28 29 | 
             
                    context "with Schema Class consumption #{setting}" do
         | 
| 30 | 
            +
             | 
| 29 31 | 
             
                      before(:each) do
         | 
| 30 | 
            -
                        Deimos.configure  | 
| 32 | 
            +
                        Deimos.configure do |config|
         | 
| 33 | 
            +
                          config.schema.use_schema_classes = use_schema_classes
         | 
| 34 | 
            +
                          config.schema.generate_namespace_folders = true
         | 
| 35 | 
            +
                        end
         | 
| 31 36 | 
             
                      end
         | 
| 32 37 |  | 
| 33 38 | 
             
                      it 'should consume a message' do
         | 
| @@ -127,6 +132,41 @@ module ConsumerTest | |
| 127 132 | 
             
                      end
         | 
| 128 133 | 
             
                    end
         | 
| 129 134 | 
             
                  end
         | 
| 135 | 
            +
             | 
| 136 | 
            +
                  context 'with overriden schema classes' do
         | 
| 137 | 
            +
             | 
| 138 | 
            +
                    before(:each) do
         | 
| 139 | 
            +
                      Deimos.configure do |config|
         | 
| 140 | 
            +
                        config.schema.use_schema_classes = true
         | 
| 141 | 
            +
                        config.schema.generate_namespace_folders = true
         | 
| 142 | 
            +
                      end
         | 
| 143 | 
            +
                    end
         | 
| 144 | 
            +
             | 
| 145 | 
            +
                    prepend_before(:each) do
         | 
| 146 | 
            +
                      consumer_class = Class.new(described_class) do
         | 
| 147 | 
            +
                        schema 'MyUpdatedSchema'
         | 
| 148 | 
            +
                        namespace 'com.my-namespace'
         | 
| 149 | 
            +
                        key_config field: 'test_id'
         | 
| 150 | 
            +
             | 
| 151 | 
            +
                        # :nodoc:
         | 
| 152 | 
            +
                        def consume(_payload, _metadata)
         | 
| 153 | 
            +
                          raise 'This should not be called unless call_original is set'
         | 
| 154 | 
            +
                        end
         | 
| 155 | 
            +
                      end
         | 
| 156 | 
            +
                      stub_const('ConsumerTest::MyConsumer', consumer_class)
         | 
| 157 | 
            +
                    end
         | 
| 158 | 
            +
             | 
| 159 | 
            +
                    it 'should consume messages' do
         | 
| 160 | 
            +
                      test_consume_message('my_consume_topic',
         | 
| 161 | 
            +
                                           { 'test_id' => 'foo',
         | 
| 162 | 
            +
                                             'some_int' => 1 }) do |payload, _metadata|
         | 
| 163 | 
            +
                        expect(payload['test_id']).to eq('foo')
         | 
| 164 | 
            +
                        expect(payload['some_int']).to eq(1)
         | 
| 165 | 
            +
                        expect(payload['super_int']).to eq(9000)
         | 
| 166 | 
            +
                                             end
         | 
| 167 | 
            +
                    end
         | 
| 168 | 
            +
             | 
| 169 | 
            +
                  end
         | 
| 130 170 | 
             
                end
         | 
| 131 171 |  | 
| 132 172 | 
             
                describe 'decode_key' do
         | 
| @@ -218,3 +258,4 @@ module ConsumerTest | |
| 218 258 | 
             
                end
         | 
| 219 259 | 
             
              end
         | 
| 220 260 | 
             
            end
         | 
| 261 | 
            +
            # rubocop:enable Metrics/ModuleLength
         | 
| @@ -0,0 +1,18 @@ | |
| 1 | 
            +
            # frozen_string_literal: true
         | 
| 2 | 
            +
             | 
| 3 | 
            +
            # This file is autogenerated by Deimos, Do NOT modify
         | 
| 4 | 
            +
            module Schemas; module MyNamespace
         | 
| 5 | 
            +
              ### Primary Schema Class ###
         | 
| 6 | 
            +
              # Autogenerated Schema for Record at com.my-namespace.MySchema
         | 
| 7 | 
            +
              class MyUpdatedSchema < Schemas::MyNamespace::MySchema
         | 
| 8 | 
            +
             | 
| 9 | 
            +
                attr_accessor :super_int
         | 
| 10 | 
            +
             | 
| 11 | 
            +
                def initialize(test_id: nil,
         | 
| 12 | 
            +
                               some_int: nil)
         | 
| 13 | 
            +
                  super
         | 
| 14 | 
            +
                  self.super_int = some_int.nil? ? 10 : some_int * 9000
         | 
| 15 | 
            +
                end
         | 
| 16 | 
            +
              end
         | 
| 17 | 
            +
            end
         | 
| 18 | 
            +
            end
         | 
    
        metadata
    CHANGED
    
    | @@ -1,14 +1,14 @@ | |
| 1 1 | 
             
            --- !ruby/object:Gem::Specification
         | 
| 2 2 | 
             
            name: deimos-ruby
         | 
| 3 3 | 
             
            version: !ruby/object:Gem::Version
         | 
| 4 | 
            -
              version: 1. | 
| 4 | 
            +
              version: 1.23.0
         | 
| 5 5 | 
             
            platform: ruby
         | 
| 6 6 | 
             
            authors:
         | 
| 7 7 | 
             
            - Daniel Orner
         | 
| 8 8 | 
             
            autorequire:
         | 
| 9 9 | 
             
            bindir: bin
         | 
| 10 10 | 
             
            cert_chain: []
         | 
| 11 | 
            -
            date:  | 
| 11 | 
            +
            date: 2024-01-09 00:00:00.000000000 Z
         | 
| 12 12 | 
             
            dependencies:
         | 
| 13 13 | 
             
            - !ruby/object:Gem::Dependency
         | 
| 14 14 | 
             
              name: avro_turf
         | 
| @@ -579,6 +579,7 @@ files: | |
| 579 579 | 
             
            - spec/schemas/my_namespace/my_schema_with_date_time.rb
         | 
| 580 580 | 
             
            - spec/schemas/my_namespace/my_schema_with_id.rb
         | 
| 581 581 | 
             
            - spec/schemas/my_namespace/my_schema_with_unique_id.rb
         | 
| 582 | 
            +
            - spec/schemas/my_namespace/my_updated_schema.rb
         | 
| 582 583 | 
             
            - spec/schemas/my_namespace/wibble.rb
         | 
| 583 584 | 
             
            - spec/schemas/my_namespace/widget.rb
         | 
| 584 585 | 
             
            - spec/schemas/my_namespace/widget_the_second.rb
         | 
| @@ -632,7 +633,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement | |
| 632 633 | 
             
                - !ruby/object:Gem::Version
         | 
| 633 634 | 
             
                  version: '0'
         | 
| 634 635 | 
             
            requirements: []
         | 
| 635 | 
            -
            rubygems_version: 3. | 
| 636 | 
            +
            rubygems_version: 3.1.2
         | 
| 636 637 | 
             
            signing_key:
         | 
| 637 638 | 
             
            specification_version: 4
         | 
| 638 639 | 
             
            summary: Kafka libraries for Ruby.
         | 
| @@ -706,6 +707,7 @@ test_files: | |
| 706 707 | 
             
            - spec/schemas/my_namespace/my_schema_with_date_time.rb
         | 
| 707 708 | 
             
            - spec/schemas/my_namespace/my_schema_with_id.rb
         | 
| 708 709 | 
             
            - spec/schemas/my_namespace/my_schema_with_unique_id.rb
         | 
| 710 | 
            +
            - spec/schemas/my_namespace/my_updated_schema.rb
         | 
| 709 711 | 
             
            - spec/schemas/my_namespace/wibble.rb
         | 
| 710 712 | 
             
            - spec/schemas/my_namespace/widget.rb
         | 
| 711 713 | 
             
            - spec/schemas/my_namespace/widget_the_second.rb
         |