deimos-ruby 1.24.3 → 2.0.0.pre.alpha1

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deimos-ruby
 version: !ruby/object:Gem::Version
-  version: 1.24.3
+  version: 2.0.0.pre.alpha1
 platform: ruby
 authors:
 - Daniel Orner
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-05-13 00:00:00.000000000 Z
+date: 2024-11-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro_turf
@@ -31,53 +31,33 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2'
 - !ruby/object:Gem::Dependency
-  name: fig_tree
+  name: karafka
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.0.2
+        version: '2.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.0.2
-- !ruby/object:Gem::Dependency
-  name: phobos
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '1.9'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '3.0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '1.9'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
-  name: ruby-kafka
+  name: fig_tree
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "<"
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '2'
+        version: 0.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "<"
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '2'
+        version: 0.2.0
 - !ruby/object:Gem::Dependency
   name: sigurd
   requirement: !ruby/object:Gem::Requirement
@@ -225,19 +205,33 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1'
 - !ruby/object:Gem::Dependency
-  name: mysql2
+  name: karafka-testing
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.5'
+        version: '2.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.5'
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: trilogy
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.1'
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement
@@ -422,6 +416,7 @@ files:
 - ".rubocop.yml"
 - ".rubocop_todo.yml"
 - ".ruby-version"
+- ".tool-versions"
 - CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Dockerfile
@@ -452,26 +447,29 @@ files:
 - lib/deimos/active_record_consumer.rb
 - lib/deimos/active_record_producer.rb
 - lib/deimos/backends/base.rb
-- lib/deimos/backends/db.rb
 - lib/deimos/backends/kafka.rb
 - lib/deimos/backends/kafka_async.rb
+- lib/deimos/backends/outbox.rb
 - lib/deimos/backends/test.rb
-- lib/deimos/batch_consumer.rb
 - lib/deimos/config/configuration.rb
-- lib/deimos/config/phobos_config.rb
 - lib/deimos/consume/batch_consumption.rb
 - lib/deimos/consume/message_consumption.rb
 - lib/deimos/consumer.rb
 - lib/deimos/exceptions.rb
-- lib/deimos/instrumentation.rb
+- lib/deimos/ext/consumer_route.rb
+- lib/deimos/ext/producer_middleware.rb
+- lib/deimos/ext/producer_route.rb
+- lib/deimos/ext/redraw.rb
+- lib/deimos/ext/routing_defaults.rb
+- lib/deimos/ext/schema_route.rb
 - lib/deimos/kafka_message.rb
 - lib/deimos/kafka_source.rb
 - lib/deimos/kafka_topic_info.rb
+- lib/deimos/logging.rb
 - lib/deimos/message.rb
 - lib/deimos/metrics/datadog.rb
 - lib/deimos/metrics/mock.rb
 - lib/deimos/metrics/provider.rb
-- lib/deimos/monkey_patches/phobos_cli.rb
 - lib/deimos/poll_info.rb
 - lib/deimos/producer.rb
 - lib/deimos/railtie.rb
@@ -482,6 +480,7 @@ files:
 - lib/deimos/schema_backends/avro_validation.rb
 - lib/deimos/schema_backends/base.rb
 - lib/deimos/schema_backends/mock.rb
+- lib/deimos/schema_backends/plain.rb
 - lib/deimos/schema_class/base.rb
 - lib/deimos/schema_class/enum.rb
 - lib/deimos/schema_class/record.rb
@@ -490,32 +489,32 @@ files:
 - lib/deimos/tracing/datadog.rb
 - lib/deimos/tracing/mock.rb
 - lib/deimos/tracing/provider.rb
+- lib/deimos/transcoder.rb
 - lib/deimos/utils/db_poller.rb
 - lib/deimos/utils/db_poller/base.rb
 - lib/deimos/utils/db_poller/state_based.rb
 - lib/deimos/utils/db_poller/time_based.rb
-- lib/deimos/utils/db_producer.rb
 - lib/deimos/utils/deadlock_retry.rb
-- lib/deimos/utils/inline_consumer.rb
-- lib/deimos/utils/lag_reporter.rb
+- lib/deimos/utils/outbox_producer.rb
 - lib/deimos/utils/schema_class.rb
-- lib/deimos/utils/schema_controller_mixin.rb
 - lib/deimos/version.rb
 - lib/generators/deimos/active_record/templates/migration.rb.tt
 - lib/generators/deimos/active_record/templates/model.rb.tt
 - lib/generators/deimos/active_record_generator.rb
 - lib/generators/deimos/bulk_import_id/templates/migration.rb.tt
 - lib/generators/deimos/bulk_import_id_generator.rb
-- lib/generators/deimos/db_backend/templates/migration
-- lib/generators/deimos/db_backend/templates/rails3_migration
-- lib/generators/deimos/db_backend_generator.rb
 - lib/generators/deimos/db_poller/templates/migration
 - lib/generators/deimos/db_poller/templates/rails3_migration
 - lib/generators/deimos/db_poller_generator.rb
+- lib/generators/deimos/outbox_backend/templates/migration
+- lib/generators/deimos/outbox_backend/templates/rails3_migration
+- lib/generators/deimos/outbox_backend_generator.rb
 - lib/generators/deimos/schema_class/templates/schema_class.rb.tt
 - lib/generators/deimos/schema_class/templates/schema_enum.rb.tt
 - lib/generators/deimos/schema_class/templates/schema_record.rb.tt
 - lib/generators/deimos/schema_class_generator.rb
+- lib/generators/deimos/v2/templates/karafka.rb.tt
+- lib/generators/deimos/v2_generator.rb
 - lib/tasks/deimos.rake
 - rbs_collection.lock.yaml
 - rbs_collection.yaml
@@ -532,11 +531,10 @@ files:
 - spec/active_record_consumer_spec.rb
 - spec/active_record_producer_spec.rb
 - spec/backends/base_spec.rb
-- spec/backends/db_spec.rb
 - spec/backends/kafka_async_spec.rb
 - spec/backends/kafka_spec.rb
+- spec/backends/outbox_spec.rb
 - spec/batch_consumer_spec.rb
-- spec/config/configuration_spec.rb
 - spec/consumer_spec.rb
 - spec/deimos_spec.rb
 - spec/generators/active_record_generator_spec.rb
@@ -546,12 +544,12 @@ files:
 - spec/generators/schema_class_generator_spec.rb
 - spec/handlers/my_batch_consumer.rb
 - spec/handlers/my_consumer.rb
-- spec/kafka_listener_spec.rb
 - spec/kafka_source_spec.rb
 - spec/kafka_topic_info_spec.rb
+- spec/karafka/karafka.rb
+- spec/karafka_config/karafka_spec.rb
+- spec/logging_spec.rb
 - spec/message_spec.rb
-- spec/phobos.bad_db.yml
-- spec/phobos.yml
 - spec/producer_spec.rb
 - spec/rake_spec.rb
 - spec/schema_backends/avro_base_shared.rb
@@ -569,6 +567,7 @@ files:
 - spec/schemas/com/my-namespace/MySchemaWithComplexTypes.avsc
 - spec/schemas/com/my-namespace/MySchemaWithDateTimes.avsc
 - spec/schemas/com/my-namespace/MySchemaWithId.avsc
+- spec/schemas/com/my-namespace/MySchemaWithTitle.avsc
 - spec/schemas/com/my-namespace/MySchemaWithUniqueId.avsc
 - spec/schemas/com/my-namespace/MySchema_key.avsc
 - spec/schemas/com/my-namespace/Wibble.avsc
@@ -620,12 +619,8 @@ files:
 - spec/snapshots/producers_with_key.snap
 - spec/spec_helper.rb
 - spec/utils/db_poller_spec.rb
-- spec/utils/db_producer_spec.rb
 - spec/utils/deadlock_retry_spec.rb
-- spec/utils/inline_consumer_spec.rb
-- spec/utils/lag_reporter_spec.rb
-- spec/utils/platform_schema_validation_spec.rb
-- spec/utils/schema_controller_mixin_spec.rb
+- spec/utils/outbox_producer_spec.rb
 - support/deimos-solo.png
 - support/deimos-with-name-next.png
 - support/deimos-with-name.png
@@ -645,9 +640,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubygems_version: 3.4.10
 signing_key:
@@ -663,11 +658,10 @@ test_files:
 - spec/active_record_consumer_spec.rb
 - spec/active_record_producer_spec.rb
 - spec/backends/base_spec.rb
-- spec/backends/db_spec.rb
 - spec/backends/kafka_async_spec.rb
 - spec/backends/kafka_spec.rb
+- spec/backends/outbox_spec.rb
 - spec/batch_consumer_spec.rb
-- spec/config/configuration_spec.rb
 - spec/consumer_spec.rb
 - spec/deimos_spec.rb
 - spec/generators/active_record_generator_spec.rb
@@ -677,12 +671,12 @@ test_files:
 - spec/generators/schema_class_generator_spec.rb
 - spec/handlers/my_batch_consumer.rb
 - spec/handlers/my_consumer.rb
-- spec/kafka_listener_spec.rb
 - spec/kafka_source_spec.rb
 - spec/kafka_topic_info_spec.rb
+- spec/karafka/karafka.rb
+- spec/karafka_config/karafka_spec.rb
+- spec/logging_spec.rb
 - spec/message_spec.rb
-- spec/phobos.bad_db.yml
-- spec/phobos.yml
 - spec/producer_spec.rb
 - spec/rake_spec.rb
 - spec/schema_backends/avro_base_shared.rb
@@ -700,6 +694,7 @@ test_files:
 - spec/schemas/com/my-namespace/MySchemaWithComplexTypes.avsc
 - spec/schemas/com/my-namespace/MySchemaWithDateTimes.avsc
 - spec/schemas/com/my-namespace/MySchemaWithId.avsc
+- spec/schemas/com/my-namespace/MySchemaWithTitle.avsc
 - spec/schemas/com/my-namespace/MySchemaWithUniqueId.avsc
 - spec/schemas/com/my-namespace/MySchema_key.avsc
 - spec/schemas/com/my-namespace/Wibble.avsc
@@ -751,9 +746,5 @@ test_files:
 - spec/snapshots/producers_with_key.snap
 - spec/spec_helper.rb
 - spec/utils/db_poller_spec.rb
-- spec/utils/db_producer_spec.rb
 - spec/utils/deadlock_retry_spec.rb
-- spec/utils/inline_consumer_spec.rb
-- spec/utils/lag_reporter_spec.rb
-- spec/utils/platform_schema_validation_spec.rb
-- spec/utils/schema_controller_mixin_spec.rb
+- spec/utils/outbox_producer_spec.rb

data/lib/deimos/batch_consumer.rb

@@ -1,7 +0,0 @@
-# frozen_string_literal: true
-
-module Deimos
-  # @deprecated Use Deimos::Consumer with `delivery: inline_batch` configured instead
-  class BatchConsumer < Consumer
-  end
-end

data/lib/deimos/config/phobos_config.rb

@@ -1,164 +0,0 @@
-# frozen_string_literal: true
-
-require 'active_support/core_ext/array'
-
-module Deimos
-  # Module to handle phobos.yml as well as outputting the configuration to save
-  # to Phobos itself.
-  module PhobosConfig
-    extend ActiveSupport::Concern
-
-    # @return [Hash]
-    def to_h
-      (FIELDS + [:handler]).map { |f|
-        val = self.send(f)
-        if f == :backoff && val
-          [:backoff, _backoff(val)]
-        elsif val.present?
-          [f, val]
-        end
-      }.to_h
-    end
-
-    # @return [void]
-    def reset!
-      super
-      Phobos.configure(self.phobos_config)
-    end
-
-    # Create a hash representing the config that Phobos expects.
-    # @return [Hash]
-    def phobos_config
-      p_config = {
-        logger: Logger.new(STDOUT),
-        custom_logger: self.phobos_logger,
-        custom_kafka_logger: self.kafka.logger,
-        kafka: {
-          client_id: self.kafka.client_id,
-          connect_timeout: self.kafka.connect_timeout,
-          socket_timeout: self.kafka.socket_timeout,
-          ssl_verify_hostname: self.kafka.ssl.verify_hostname,
-          ssl_ca_certs_from_system: self.kafka.ssl.ca_certs_from_system,
-          seed_brokers: Array.wrap(self.kafka.seed_brokers)
-        },
-        producer: {
-          ack_timeout: self.producers.ack_timeout,
-          required_acks: self.producers.required_acks,
-          max_retries: self.producers.max_retries,
-          retry_backoff: self.producers.retry_backoff,
-          max_buffer_size: self.producers.max_buffer_size,
-          max_buffer_bytesize: self.producers.max_buffer_bytesize,
-          compression_codec: self.producers.compression_codec,
-          compression_threshold: self.producers.compression_threshold,
-          max_queue_size: self.producers.max_queue_size,
-          delivery_threshold: self.producers.delivery_threshold,
-          delivery_interval: self.producers.delivery_interval,
-          persistent_connections: self.producers.persistent_connections
-        },
-        consumer: {
-          session_timeout: self.consumers.session_timeout,
-          offset_commit_interval: self.consumers.offset_commit_interval,
-          offset_commit_threshold: self.consumers.offset_commit_threshold,
-          heartbeat_interval: self.consumers.heartbeat_interval
-        },
-        backoff: _backoff(self.consumers.backoff.to_a)
-      }
-
-      p_config[:listeners] = self.consumer_objects.map do |consumer|
-        next nil if consumer.disabled
-
-        hash = consumer.to_h.reject do |k, _|
-          %i(class_name schema namespace key_config backoff disabled replace_associations
-             bulk_import_id_column).include?(k)
-        end
-        hash = hash.map { |k, v| [k, v.is_a?(Symbol) ? v.to_s : v] }.to_h
-        hash[:handler] = consumer.class_name
-        if consumer.backoff
-          hash[:backoff] = _backoff(consumer.backoff.to_a)
-        end
-        hash
-      end
-      p_config[:listeners].compact!
-
-      if self.kafka.ssl.enabled
-        %w(ca_cert client_cert client_cert_key).each do |key|
-          next if self.kafka.ssl.send(key).blank?
-
-          p_config[:kafka]["ssl_#{key}".to_sym] = ssl_var_contents(self.kafka.ssl.send(key))
-        end
-      end
-
-      if self.kafka.sasl.enabled
-        p_config[:kafka][:sasl_over_ssl] = self.kafka.sasl.enforce_ssl
-        %w(
-          gssapi_principal
-          gssapi_keytab
-          plain_authzid
-          plain_username
-          plain_password
-          scram_username
-          scram_password
-          scram_mechanism
-          oauth_token_provider
-        ).each do |key|
-          value = self.kafka.sasl.send(key)
-          next if value.blank?
-
-          p_config[:kafka]["sasl_#{key}".to_sym] = value
-        end
-      end
-      p_config
-    end
-
-    # @param key [String]
-    # @return [String]
-    def ssl_var_contents(key)
-      File.exist?(key) ? File.read(key) : key
-    end
-
-    # Legacy method to parse Phobos config file
-    # @!visibility private
-    def phobos_config_file=(file)
-      pconfig = YAML.load(ERB.new(File.read(File.expand_path(file))).result). # rubocop:disable Security/YAMLLoad
-        with_indifferent_access
-      self.logger&.warn('phobos.yml is deprecated - use direct configuration instead.')
-      pconfig[:kafka].each do |k, v|
-        if k.starts_with?('ssl')
-          k = k.sub('ssl_', '')
-          self.kafka.ssl.send("#{k}=", v)
-        elsif k.starts_with?('sasl')
-          k = (k == 'sasl_over_ssl') ? 'enforce_ssl' : k.sub('sasl_', '')
-          self.kafka.sasl.send("#{k}=", v)
-        else
-          self.kafka.send("#{k}=", v)
-        end
-      end
-      pconfig[:producer].each do |k, v|
-        self.producers.send("#{k}=", v)
-      end
-      pconfig[:consumer].each do |k, v|
-        self.consumers.send("#{k}=", v)
-      end
-      self.consumers.backoff = pconfig[:backoff][:min_ms]..pconfig[:backoff][:max_ms]
-      pconfig[:listeners].each do |listener_hash|
-        self.consumer do
-          listener_hash.each do |k, v|
-            k = 'class_name' if k == 'handler'
-            send(k, v)
-          end
-        end
-      end
-    end
-
-  private
-
-    # @param values [Array<Integer>]
-    # @return [Hash<Integer>]
-    def _backoff(values)
-      {
-        min_ms: values[0],
-        max_ms: values[-1]
-      }
-    end
-  end
-end

data/lib/deimos/instrumentation.rb

@@ -1,95 +0,0 @@
-# frozen_string_literal: true
-
-require 'active_support/notifications'
-require 'active_support/concern'
-
-# :nodoc:
-module Deimos
-  # Copied from Phobos instrumentation.
-  module Instrumentation
-    extend ActiveSupport::Concern
-
-    # @return [String]
-    NAMESPACE = 'Deimos'
-
-    # :nodoc:
-    module ClassMethods
-      # @param event [String]
-      # @return [void]
-      def subscribe(event)
-        ActiveSupport::Notifications.subscribe("#{NAMESPACE}.#{event}") do |*args|
-          yield(ActiveSupport::Notifications::Event.new(*args)) if block_given?
-        end
-      end
-
-      # @param subscriber [ActiveSupport::Subscriber]
-      # @return [void]
-      def unsubscribe(subscriber)
-        ActiveSupport::Notifications.unsubscribe(subscriber)
-      end
-
-      # @param event [String]
-      # @param extra [Hash]
-      # @return [void]
-      def instrument(event, extra={})
-        ActiveSupport::Notifications.instrument("#{NAMESPACE}.#{event}", extra) do |extra2|
-          yield(extra2) if block_given?
-        end
-      end
-    end
-  end
-
-  include Instrumentation
-
-  # This module listens to events published by RubyKafka.
-  module KafkaListener
-    # @param exception [Exception]
-    def self.handle_exception_with_messages(exception)
-      messages = exception.failed_messages
-      messages.group_by(&:topic).each do |topic, batch|
-        producer = Deimos::Producer.descendants.find { |c| c.topic == topic }
-        next if batch.empty? || !producer
-
-        decoder = Deimos.schema_backend(schema: producer.config[:schema],
-                                        namespace: producer.config[:namespace])
-        payloads = batch.map { |m| decoder.decode(m.value) }
-
-        Deimos.config.metrics&.increment(
-          'publish_error',
-          tags: %W(topic:#{topic}),
-          by: payloads.size
-        )
-        Deimos.instrument(
-          'produce_error',
-          producer: producer,
-          topic: topic,
-          exception_object: exception,
-          payloads: payloads
-        )
-      end
-    end
-
-    # Listens for any exceptions that happen during publishing and re-publishes
-    # as a Deimos event.
-    # @param event [ActiveSupport::Notifications::Event]
-    # @return [void]
-    def self.send_produce_error(event)
-      exception = event.payload[:exception_object]
-      return unless exception
-
-      if exception.respond_to?(:failed_messages)
-        handle_exception_with_messages(exception)
-      else
-        Deimos.config.metrics&.increment(
-          'publish_error',
-          by: event.payload[:message_count] || 1
-        )
-      end
-    end
-  end
-
-  ActiveSupport::Notifications.subscribe('deliver_messages.producer.kafka') do |*args|
-    event = ActiveSupport::Notifications::Event.new(*args)
-    KafkaListener.send_produce_error(event)
-  end
-end

data/lib/deimos/monkey_patches/phobos_cli.rb

@@ -1,35 +0,0 @@
-# frozen_string_literal: true
-
-require 'phobos/cli/start'
-
-#@!visibility private
-module Phobos
-  # :nodoc:
-  module CLI
-    # :nodoc:
-    class Start
-      # :nodoc:
-      def validate_listeners!
-        Phobos.config.listeners.each do |listener|
-          handler = listener.handler
-          begin
-            handler.constantize
-          rescue NameError
-            error_exit("Handler '#{handler}' not defined")
-          end
-
-          delivery = listener.delivery
-          if delivery.nil?
-            Phobos::CLI.logger.warn do
-              Hash(message: "Delivery option should be specified, defaulting to 'batch'"\
-               ' - specify this option to silence this message')
-            end
-          elsif !Listener::DELIVERY_OPTS.include?(delivery)
-            error_exit("Invalid delivery option '#{delivery}'. Please specify one of: "\
-              "#{Listener::DELIVERY_OPTS.join(', ')}")
-          end
-        end
-      end
-    end
-  end
-end