sbmt-kafka_consumer 2.8.0 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5c5c4115c8a5cc26e7d4798244f8e3052f1b6abdddb11a4a96699a1e4f950659
-  data.tar.gz: 7e6d290f5d0866f4e6ea0b424e07afd95770840d6a5a1324ee4d01b28c62b49d
+  metadata.gz: 7af7925acd4a61d131a66dd3cd290064ba11aac749ee64ca00841912bd038adc
+  data.tar.gz: 9cd1e1f789c6e4252adedf3dd3115f512c62240b75edadc38d1590e4a85b6000
 SHA512:
-  metadata.gz: 4f07ac9ad6a9e65898e9d3a48ed86b0df7fdd52f38ea04b2e41aa0406351d314f0a5aa72e33334e9a619779564afb6105481d19ec2a27706adf197346af66776
-  data.tar.gz: 1d65a2e194afade6fc2230bd9d031c457101f799342ea688be292a119ae0661754f4cbca62ce66d8257af2616faef675bdc638ef03059a4eaa9611f41de8a674
+  metadata.gz: 60dbd904786ddc419e9efbb0f02904c6035089a3b23fcb4035d41c85c8db303ae807138fa1e26c042688521c34a355f0314799e82addc73ef6308fc3fe6cad15
+  data.tar.gz: ca1d3af3c319695cbb33e11f9f66a384efb2c1884a387a233d5b197ae9ed5d3a3852c49662586a67ba15eb497ed325b92363cfe65ce8e086c6319b48dfb0569e

data/Appraisals

@@ -3,10 +3,10 @@
 # See compatibility table at https://www.fastruby.io/blog/ruby/rails/versions/compatibility-table.html
 
 versions_map = {
-  "6.0" => %w[2.7],
   "6.1" => %w[2.7 3.0],
   "7.0" => %w[3.1],
-  "7.1" => %w[3.2, 3.3]
+  "7.1" => %w[3.2, 3.3],
+  "7.2" => %w[3.3]
 }
 
 current_ruby_version = RUBY_VERSION.split(".").first(2).join(".")

data/CHANGELOG.md

@@ -13,11 +13,17 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ### Fixed
 
-## [2.8.0] - 2024-09-09
+## [3.0.0] - 2024-09-04
+
+## BREAKING
+
+- Drop support for Ruby 2.7
+- Drop support for Rails 6.0
+- Add support for Karafka 2.4
 
 ### Fixed
 
-- Refactor consumer class initialization
+- Support consumer group mappers to support backward compatibility of consumer group naming
 
 ## [2.7.1] - 2024-08-01
 

data/LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2024 SberMarket Tech
+Copyright (c) 2024 Kuper Tech
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -1,9 +1,9 @@
 [![Gem Version](https://badge.fury.io/rb/sbmt-kafka_consumer.svg)](https://badge.fury.io/rb/sbmt-kafka_consumer)
-[![Build Status](https://github.com/SberMarket-Tech/sbmt-kafka_consumer/actions/workflows/tests.yml/badge.svg?branch=master)](https://github.com/SberMarket-Tech/sbmt-kafka_consumer/actions?query=branch%3Amaster)
+[![Build Status](https://github.com/Kuper-Tech/sbmt-kafka_consumer/actions/workflows/tests.yml/badge.svg?branch=master)](https://github.com/Kuper-Tech/sbmt-kafka_consumer/actions?query=branch%3Amaster)
 
 # Sbmt-KafkaConsumer
 
-This gem is used to consume Kafka messages. It is a wrapper over the [Karafka](https://github.com/karafka/karafka) gem, and is recommended for use as a transport with the [sbmt-outbox](https://github.com/SberMarket-Tech/sbmt-outbox) gem.
+This gem is used to consume Kafka messages. It is a wrapper over the [Karafka](https://github.com/karafka/karafka) gem, and is recommended for use as a transport with the [sbmt-outbox](https://github.com/Kuper-Tech/sbmt-outbox) gem.
 
 ## Installation
 
@@ -21,7 +21,7 @@ bundle install
 
 ## Demo
 
-Learn how to use this gem and how it works with Ruby on Rails at here https://github.com/SberMarket-Tech/outbox-example-apps
+Learn how to use this gem and how it works with Ruby on Rails at here https://github.com/Kuper-Tech/outbox-example-apps
 
 ## Auto configuration
 
@@ -47,7 +47,7 @@ rails g kafka_consumer:consumer MaybeNamespaced::Name
 
 ### Inbox consumer
 
-To generate an Inbox consumer for use with gem [sbmt-outbox](https://github.com/SberMarket-Tech/sbmt-outbox), run the following command:
+To generate an Inbox consumer for use with gem [sbmt-outbox](https://github.com/Kuper-Tech/sbmt-outbox), run the following command:
 
 ```shell
 rails g kafka_consumer:inbox_consumer MaybeNamespaced::Name some-consumer-group some-topic

data/dip.yml

@@ -1,7 +1,7 @@
 version: '7'
 
 environment:
-  RUBY_VERSION: '3.2'
+  RUBY_VERSION: '3.3'
 
 compose:
   files:
@@ -35,14 +35,14 @@ interaction:
     subcommands:
       all:
         command: bundle exec appraisal rspec
-      rails-6.0:
-        command: bundle exec appraisal rails-6.0 rspec
       rails-6.1:
         command: bundle exec appraisal rails-6.1 rspec
       rails-7.0:
         command: bundle exec appraisal rails-7.0 rspec
       rails-7.1:
         command: bundle exec appraisal rails-7.1 rspec
+      rails-7.2:
+        command: bundle exec appraisal rails-7.2 rspec
 
   rubocop:
     description: Run Ruby linter

data/lib/sbmt/kafka_consumer/base_consumer.rb

@@ -3,20 +3,15 @@
 module Sbmt
   module KafkaConsumer
     class BaseConsumer < Karafka::BaseConsumer
-      class_attribute :skip_on_error, instance_writer: false, default: false
-      class_attribute :middlewares, instance_writer: false, default: []
+      def self.consumer_klass(skip_on_error: false, middlewares: [])
+        Class.new(self) do
+          const_set(:SKIP_ON_ERROR, skip_on_error)
+          const_set(:MIDDLEWARES, middlewares.map(&:constantize))
 
-      def self.consumer_klass(skip_on_error: nil, middlewares: nil)
-        klass = Class.new(self) do
           def self.name
             superclass.name
           end
         end
-
-        # defaults are set in class_attribute definition
-        klass.skip_on_error = skip_on_error if skip_on_error
-        klass.middlewares = middlewares.map(&:constantize) if middlewares
-        klass
       end
 
       def consume
@@ -117,6 +112,14 @@ module Sbmt
         end
       end
 
+      def skip_on_error
+        self.class::SKIP_ON_ERROR
+      end
+
+      def middlewares
+        self.class::MIDDLEWARES
+      end
+
       # can be overridden in consumer to enable message logging
       def log_payload?
         false

data/lib/sbmt/kafka_consumer/client_configurer.rb

@@ -6,10 +6,8 @@ class Sbmt::KafkaConsumer::ClientConfigurer
     Karafka::App.setup do |karafka_config|
       karafka_config.monitor = config.monitor_class.classify.constantize.new
       karafka_config.logger = Sbmt::KafkaConsumer.logger
-      karafka_config.deserializer = config.deserializer_class.classify.constantize.new
 
       karafka_config.client_id = config.client_id
-      karafka_config.consumer_mapper = config.consumer_mapper_class.classify.constantize.new
       karafka_config.kafka = config.to_kafka_options
 
       karafka_config.pause_timeout = config.pause_timeout * 1_000 if config.pause_timeout.present?
@@ -43,13 +41,16 @@ class Sbmt::KafkaConsumer::ClientConfigurer
 
     raise "No configured consumer groups found, exiting" if target_consumer_groups.blank?
 
+    consumer_mapper = config.consumer_mapper_class.classify.constantize.new
+
     # clear routes in case CLI runner tries to reconfigure them
     # but railtie initializer had already executed and did the same
     # otherwise we'll get duplicate routes error from sbmt-karafka internal config validation process
     Karafka::App.routes.clear
     Karafka::App.routes.draw do
       target_consumer_groups.each do |cg|
-        consumer_group cg.name do
+        group_id = consumer_mapper.call(cg.name)
+        consumer_group group_id do
           cg.topics.each do |t|
             topic t.name do
               active t.active
@@ -66,7 +67,7 @@ class Sbmt::KafkaConsumer::ClientConfigurer
 
   def self.routes
     Karafka::App.routes.map do |cg|
-      topics = cg.topics.map { |t| {name: t.name, deserializer: t.deserializer} }
+      topics = cg.topics.map { |t| {name: t.name, deserializer: t.deserializers.payload} }
       {group: cg.id, topics: topics}
     end
   end

data/lib/sbmt/kafka_consumer/inbox_consumer.rb

@@ -6,20 +6,17 @@ module Sbmt
       IDEMPOTENCY_HEADER_NAME = "Idempotency-Key"
       DEFAULT_SOURCE = "KAFKA"
 
-      class_attribute :inbox_item_class, instance_writer: false, default: nil
-      class_attribute :event_name, instance_writer: false, default: nil
-
-      def self.consumer_klass(inbox_item:, event_name: nil, skip_on_error: nil, name: nil, middlewares: nil)
-        # defaults are set in class_attribute definition
-        klass = super(skip_on_error: skip_on_error, middlewares: middlewares)
-        klass.inbox_item_class = inbox_item.constantize
-        klass.event_name = event_name if event_name
-        klass
-      end
-
-      def initialize
-        raise Sbmt::KafkaConsumer::Error, "inbox_item param is not set" if inbox_item_class.blank?
-        super
+      def self.consumer_klass(inbox_item:, event_name: nil, skip_on_error: false, name: nil, middlewares: [])
+        Class.new(self) do
+          const_set(:INBOX_ITEM_CLASS_NAME, inbox_item)
+          const_set(:EVENT_NAME, event_name)
+          const_set(:SKIP_ON_ERROR, skip_on_error)
+          const_set(:MIDDLEWARES, middlewares.map(&:constantize))
+
+          def self.name
+            superclass.name
+          end
+        end
       end
 
       def extra_message_attrs(_message)
@@ -104,6 +101,14 @@ module Sbmt
         message.metadata.headers.fetch(IDEMPOTENCY_HEADER_NAME, nil).presence
       end
 
+      def inbox_item_class
+        @inbox_item_class ||= self.class::INBOX_ITEM_CLASS_NAME.constantize
+      end
+
+      def event_name
+        @event_name ||= self.class::EVENT_NAME
+      end
+
       def inbox_name
         inbox_item_class.box_name
       end

data/lib/sbmt/kafka_consumer/routing/consumer_mapper/base.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module KafkaConsumer
+    module Routing
+      module ConsumerMapper
+        class Base
+          # @param raw_consumer_group_name [String, Symbol] string or symbolized consumer group name
+          # @return [String] remapped final consumer group name
+          def call(raw_consumer_group_name)
+            raise "Implement #call in a subclass"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/kafka_consumer/routing/karafka_v1_consumer_mapper.rb

@@ -1,7 +1,12 @@
+# frozen_string_literal: true
+
+require_relative "consumer_mapper/base"
+
 module Sbmt
   module KafkaConsumer
     module Routing
-      class KarafkaV1ConsumerMapper < Karafka::Routing::ConsumerMapper
+      class KarafkaV1ConsumerMapper < ConsumerMapper::Base
+        # karafka v1 consumer group name mapper
         def call(raw_consumer_group_name)
           client_id = ActiveSupport::Inflector.underscore(Karafka::App.config.client_id).tr("/", "_")
           "#{client_id}_#{raw_consumer_group_name}"

data/lib/sbmt/kafka_consumer/routing/karafka_v2_consumer_mapper.rb

@@ -1,9 +1,16 @@
+# frozen_string_literal: true
+
+require_relative "consumer_mapper/base"
+
 module Sbmt
   module KafkaConsumer
     module Routing
-      # uses default karafka v2 mapper
-      # exists just for naming consistency with KarafkaV1ConsumerMapper
-      class KarafkaV2ConsumerMapper < Karafka::Routing::ConsumerMapper; end
+      # karafka v2 (before 2.4) consumer group name mapper
+      class KarafkaV2ConsumerMapper < ConsumerMapper::Base
+        def call(raw_consumer_group_name)
+          "#{Karafka::App.config.client_id}_#{raw_consumer_group_name}"
+        end
+      end
     end
   end
 end

data/lib/sbmt/kafka_consumer/testing/shared_contexts/with_sbmt_karafka_consumer.rb

@@ -2,7 +2,7 @@
 
 RSpec.shared_context "with sbmt karafka consumer" do
   subject(:consume_with_sbmt_karafka) do
-    coordinator.increment
+    coordinator.increment(:consume)
     consumer.on_consume
   end
 
@@ -16,8 +16,9 @@ RSpec.shared_context "with sbmt karafka consumer" do
   let(:kafka_client) { instance_double(Karafka::Connection::Client) }
   let(:null_deserializer) { Sbmt::KafkaConsumer::Serialization::NullDeserializer.new }
 
-  let(:consumer_class) { described_class.consumer_klass }
-  let(:consumer) { build_consumer(consumer_class.new) }
+  let(:consumer) {
+    build_consumer(described_class.new)
+  }
 
   before {
     Sbmt::KafkaConsumer::ClientConfigurer.configure!
@@ -27,27 +28,27 @@ RSpec.shared_context "with sbmt karafka consumer" do
   }
 
   def publish_to_sbmt_karafka(raw_payload, opts = {})
-    message = Karafka::Messages::Message.new(raw_payload, Karafka::Messages::Metadata.new(metadata_defaults.merge(opts)))
+    message = Karafka::Messages::Message.new(raw_payload, Karafka::Messages::Metadata.new(build_metadata_hash(opts)))
     consumer.messages = consumer_messages([message])
   end
 
   def publish_to_sbmt_karafka_batch(raw_payloads, opts = {})
     messages = raw_payloads.map do |p|
-      Karafka::Messages::Message.new(p, Karafka::Messages::Metadata.new(metadata_defaults.merge(opts)))
+      Karafka::Messages::Message.new(p, Karafka::Messages::Metadata.new(build_metadata_hash(opts)))
     end
     consumer.messages = consumer_messages(messages)
   end
 
   # @return [Hash] message default options
-  def metadata_defaults
+  def build_metadata_hash(opts)
     {
-      deserializer: null_deserializer,
-      headers: {},
-      key: nil,
-      offset: 0,
-      partition: 0,
-      received_at: Time.current,
-      topic: test_topic.name
+      deserializers: test_topic.deserializers(payload: opts[:deserializer] || null_deserializer),
+      raw_headers: opts[:headers] || {},
+      raw_key: opts[:key],
+      offset: opts[:offset] || 0,
+      partition: opts[:partition] || 0,
+      received_at: opts[:received_at] || Time.current,
+      topic: opts[:topic] || test_topic.name
     }
   end
 

data/lib/sbmt/kafka_consumer/version.rb

@@ -2,6 +2,6 @@
 
 module Sbmt
   module KafkaConsumer
-    VERSION = "2.8.0"
+    VERSION = "3.0.0"
   end
 end

data/sbmt-kafka_consumer.gemspec

@@ -6,12 +6,12 @@ Gem::Specification.new do |spec|
   spec.name = "sbmt-kafka_consumer"
   spec.license = "MIT"
   spec.version = Sbmt::KafkaConsumer::VERSION
-  spec.authors = ["Sbermarket Ruby-Platform Team"]
+  spec.authors = ["Kuper Ruby-Platform Team"]
 
   spec.summary = "Ruby gem for consuming Kafka messages"
   spec.description = "This gem is used for consuming Kafka messages. It represents a wrapper over Karafka gem and is recommended for using as a transport with sbmt-outbox"
-  spec.homepage = "https://github.com/SberMarket-Tech/sbmt-kafka_consumer"
-  spec.required_ruby_version = ">= 2.7.0"
+  spec.homepage = "https://github.com/Kuper-Tech/sbmt-kafka_consumer"
+  spec.required_ruby_version = ">= 3.0.0"
 
   spec.metadata["allowed_push_host"] = "https://rubygems.org"
 
@@ -31,9 +31,9 @@ Gem::Specification.new do |spec|
   spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "rails", ">= 6.0"
+  spec.add_dependency "rails", ">= 6.1"
   spec.add_dependency "zeitwerk", "~> 2.3"
-  spec.add_dependency "karafka", "~> 2.2", "< 2.4" # [Breaking] Drop the concept of consumer group mapping.
+  spec.add_dependency "karafka", "~> 2.4"
   spec.add_dependency "yabeda", ">= 0.11"
   spec.add_dependency "anyway_config", ">= 2.4.0"
   spec.add_dependency "thor"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sbmt-kafka_consumer
 version: !ruby/object:Gem::Version
-  version: 2.8.0
+  version: 3.0.0
 platform: ruby
 authors:
-- Sbermarket Ruby-Platform Team
+- Kuper Ruby-Platform Team
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-09-09 00:00:00.000000000 Z
+date: 2024-09-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '6.0'
+        version: '6.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '6.0'
+        version: '6.1'
 - !ruby/object:Gem::Dependency
   name: zeitwerk
   requirement: !ruby/object:Gem::Requirement
@@ -43,9 +43,6 @@ dependencies:
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.2'
-    - - "<"
       - !ruby/object:Gem::Version
         version: '2.4'
   type: :runtime
@@ -53,9 +50,6 @@ dependencies:
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.2'
-    - - "<"
       - !ruby/object:Gem::Version
         version: '2.4'
 - !ruby/object:Gem::Dependency
@@ -531,6 +525,7 @@ files:
 - lib/sbmt/kafka_consumer/probes/host.rb
 - lib/sbmt/kafka_consumer/probes/probe.rb
 - lib/sbmt/kafka_consumer/railtie.rb
+- lib/sbmt/kafka_consumer/routing/consumer_mapper/base.rb
 - lib/sbmt/kafka_consumer/routing/karafka_v1_consumer_mapper.rb
 - lib/sbmt/kafka_consumer/routing/karafka_v2_consumer_mapper.rb
 - lib/sbmt/kafka_consumer/serialization/base_deserializer.rb
@@ -546,14 +541,14 @@ files:
 - lib/sbmt/kafka_consumer/yabeda_configurer.rb
 - rubocop/rspec.yml
 - sbmt-kafka_consumer.gemspec
-homepage: https://github.com/SberMarket-Tech/sbmt-kafka_consumer
+homepage: https://github.com/Kuper-Tech/sbmt-kafka_consumer
 licenses:
 - MIT
 metadata:
   allowed_push_host: https://rubygems.org
-  homepage_uri: https://github.com/SberMarket-Tech/sbmt-kafka_consumer
-  source_code_uri: https://github.com/SberMarket-Tech/sbmt-kafka_consumer
-  changelog_uri: https://github.com/SberMarket-Tech/sbmt-kafka_consumer/blob/master/CHANGELOG.md
+  homepage_uri: https://github.com/Kuper-Tech/sbmt-kafka_consumer
+  source_code_uri: https://github.com/Kuper-Tech/sbmt-kafka_consumer
+  changelog_uri: https://github.com/Kuper-Tech/sbmt-kafka_consumer/blob/master/CHANGELOG.md
   rubygems_mfa_required: 'false'
 post_install_message:
 rdoc_options: []
@@ -563,7 +558,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.7.0
+      version: 3.0.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="