sbmt-kafka_consumer 2.0.0 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fe979c0a187c1aac9f2f8d571d99801e564912db57b86c8219b569fc3faef5c1
-  data.tar.gz: af13a117d78170019ee3f0ae0930c5b65c0dd027efa39a46eb51f1a32998f47b
+  metadata.gz: 260ec80f2e17f9aef25d51a0eba28f43cb8ea524c3d2d956dea2fd5a240123f3
+  data.tar.gz: b302aac60dd45b5bb3da4af3e52e1a21f1fa5f4cd126837c68cbb50c20cc894a
 SHA512:
-  metadata.gz: 7105c49b6c2ecf06769bb310c9a4bf836aa5cbad1b2ee994860b814b71059b8217610429dada55b408063597e32893b1efb404be11902d22693b37e4233ddb18
-  data.tar.gz: 2a6195f0db6b8a0a7f9435e6b93745422a97f2238c073598155b0c2d81212891028a67db9029160735754189610a3ab4687c433044c67bf9d6d2b6802bdd1149
+  metadata.gz: db36d1846f7b91745464ccd2a1e11b97553760e43715aae41e4cf7c0671c51cf7e35efc6b14eab9ed572ccabd21b5cdd10a4bf97b6fbf898bf4ddc83c728b2ac
+  data.tar.gz: 631e701e0c5cc15e059faa00c6e5a55ef8d92f2ad45b0c4a8d7467150f668569f3f1796453b5e1596ab489b3dc32ac3d3ca6124431077c0ffd6d97158308adf7

data/.rubocop.yml

@@ -19,6 +19,8 @@ AllCops:
   SuggestExtensions: false
   TargetRubyVersion: 2.7
   TargetRailsVersion: 5.2
+  Exclude:
+    - spec/internal/pkg/**/*
 
 RSpec/FilePath:
   Enabled: false

data/CHANGELOG.md

@@ -13,6 +13,18 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ### Fixed
 
+## [2.1.0] - 2024-05-13
+
+### Added
+
+- Implemented method `export_batch` for processing messages in batches
+
+## [2.0.1] - 2024-05-08
+
+### Fixed
+
+- Limit the Karafka version to less than `2.4` because they dropped the consumer group mapping
+
 ## [2.0.0] - 2024-01-30
 
 ### Changed

data/README.md

@@ -1,5 +1,5 @@
 [![Gem Version](https://badge.fury.io/rb/sbmt-kafka_consumer.svg)](https://badge.fury.io/rb/sbmt-kafka_consumer)
-[![Build Status](https://github.com/SberMarket-Tech/sbmt-kafka_consumer/actions/workflows/tests.yml/badge.svg?branch=master)](https://github.com/SberMarket-Tech/sbmt-outbox/actions?query=branch%3Amaster)
+[![Build Status](https://github.com/SberMarket-Tech/sbmt-kafka_consumer/actions/workflows/tests.yml/badge.svg?branch=master)](https://github.com/SberMarket-Tech/sbmt-kafka_consumer/actions?query=branch%3Amaster)
 
 # Sbmt-KafkaConsumer
 
@@ -19,6 +19,10 @@ And then execute:
 bundle install
 ```
 
+## Demo
+
+Learn how to use this gem and how it works with Ruby on Rails at here https://github.com/SberMarket-Tech/outbox-example-apps
+
 ## Auto configuration
 
 We recommend going through the configuration and file creation process using the following Rails generators. Each generator can be run by using the `--help` option to learn more about the available arguments.
@@ -227,6 +231,22 @@ require_relative "config/environment"
 some-extra-configuration
 ```
 
+### `Export batch`
+
+To process messages in batches, you need to add the `export_batch` method in the consumer
+
+```ruby
+# app/consumers/some_consumer.rb
+class SomeConsumer < Sbmt::KafkaConsumer::BaseConsumer
+  def export_batch(messages)
+    # some code 
+  end
+end
+```
+__CAUTION__:
+- ⚠️ Inbox does not support batch insertion.
+- ⚠️ If you want to use this feature, you need to process the stack atomically (eg: insert it into clickhouse in one request).
+
 ## CLI
 
 Run the following command to execute a server
@@ -255,6 +275,7 @@ Also pay attention to the number of processes of the server:
 
 To test your consumer with Rspec, please use [this shared context](./lib/sbmt/kafka_consumer/testing/shared_contexts/with_sbmt_karafka_consumer.rb)
 
+### for payload
 ```ruby
 require "sbmt/kafka_consumer/testing"
 
@@ -268,6 +289,20 @@ RSpec.describe OrderCreatedConsumer do
 end
 ```
 
+### for payloads
+```ruby
+require "sbmt/kafka_consumer/testing"
+
+RSpec.describe OrderCreatedConsumer do
+  include_context "with sbmt karafka consumer"
+
+  it "works" do
+    publish_to_sbmt_karafka_batch(payloads, deserializer: deserializer)
+    expect { consume_with_sbmt_karafka }.to change(Order, :count).by(1)
+  end
+end
+```
+
 ## Development
 
 1. Prepare environment

data/lib/sbmt/kafka_consumer/base_consumer.rb

@@ -17,12 +17,26 @@ module Sbmt
 
       def consume
         ::Rails.application.executor.wrap do
-          messages.each do |message|
-            with_instrumentation(message) { do_consume(message) }
+          if export_batch?
+            with_batch_instrumentation(messages) do
+              export_batch(messages)
+              mark_as_consumed!(messages.last)
+            end
+          else
+            messages.each do |message|
+              with_instrumentation(message) { do_consume(message) }
+            end
           end
         end
       end
 
+      def export_batch?
+        if @export_batch_memoized.nil?
+          @export_batch_memoized = respond_to?(:export_batch)
+        end
+        @export_batch_memoized
+      end
+
       private
 
       def with_instrumentation(message)
@@ -53,6 +67,25 @@ module Sbmt
         end
       end
 
+      def with_batch_instrumentation(messages)
+        @trace_id = SecureRandom.base58
+
+        logger.tagged(
+          trace_id: trace_id,
+          first_offset: messages.first.metadata.offset,
+          last_offset: messages.last.metadata.offset
+        ) do
+          ::Sbmt::KafkaConsumer.monitor.instrument(
+            "consumer.consumed_batch",
+            caller: self,
+            messages: messages,
+            trace_id: trace_id
+          ) do
+            yield
+          end
+        end
+      end
+
       def do_consume(message)
         log_message(message) if log_payload?
 

data/lib/sbmt/kafka_consumer/instrumentation/base_monitor.rb

@@ -9,6 +9,7 @@ module Sbmt
         SBMT_KAFKA_CONSUMER_EVENTS = %w[
           consumer.consumed_one
           consumer.inbox.consumed_one
+          consumer.consumed_batch
         ].freeze
 
         def initialize

data/lib/sbmt/kafka_consumer/instrumentation/open_telemetry_tracer.rb

@@ -20,6 +20,7 @@ module Sbmt
 
         def trace(&block)
           return handle_consumed_one(&block) if @event_id == "consumer.consumed_one"
+          return handle_consumed_batch(&block) if @event_id == "consumer.consumed_batch"
           return handle_inbox_consumed_one(&block) if @event_id == "consumer.inbox.consumed_one"
           return handle_error(&block) if @event_id == "error.occurred"
 
@@ -43,6 +44,23 @@ module Sbmt
           end
         end
 
+        def handle_consumed_batch
+          return yield unless enabled?
+
+          consumer = @payload[:caller]
+          messages = @payload[:messages]
+
+          links = messages.filter_map do |m|
+            parent_context = ::OpenTelemetry.propagation.extract(m.headers, getter: ::OpenTelemetry::Context::Propagation.text_map_getter)
+            span_context = ::OpenTelemetry::Trace.current_span(parent_context).context
+            ::OpenTelemetry::Trace::Link.new(span_context) if span_context.valid?
+          end
+
+          tracer.in_span("consume batch", links: links, attributes: batch_attrs(consumer, messages), kind: :consumer) do
+            yield
+          end
+        end
+
         def handle_inbox_consumed_one
           return yield unless enabled?
 
@@ -92,6 +110,19 @@ module Sbmt
           attributes.compact
         end
 
+        def batch_attrs(consumer, messages)
+          message = messages.first
+          {
+            "messaging.system" => "kafka",
+            "messaging.destination" => message.topic,
+            "messaging.destination_kind" => "topic",
+            "messaging.kafka.consumer_group" => consumer.topic.consumer_group.id,
+            "messaging.batch_size" => messages.count,
+            "messaging.first_offset" => messages.first.offset,
+            "messaging.last_offset" => messages.last.offset
+          }.compact
+        end
+
         def extract_message_key(key)
           # skip encode if already valid utf8
           return key if key.nil? || (key.encoding == Encoding::UTF_8 && key.valid_encoding?)

data/lib/sbmt/kafka_consumer/instrumentation/sentry_tracer.rb

@@ -9,34 +9,48 @@ module Sbmt
       class SentryTracer < ::Sbmt::KafkaConsumer::Instrumentation::Tracer
         CONSUMER_ERROR_TYPES = %w[
           consumer.base.consume_one
+          consumer.base.consumed_batch
           consumer.inbox.consume_one
         ].freeze
 
         def trace(&block)
           return handle_consumed_one(&block) if @event_id == "consumer.consumed_one"
+          return handle_consumed_batch(&block) if @event_id == "consumer.consumed_batch"
           return handle_error(&block) if @event_id == "error.occurred"
 
           yield
         end
 
         def handle_consumed_one
-          return yield unless ::Sentry.initialized?
-
-          consumer = @payload[:caller]
-          message = @payload[:message]
-          trace_id = @payload[:trace_id]
-
-          scope, transaction = start_transaction(trace_id, consumer, message)
-
-          begin
+          message = {
+            trace_id: @payload[:trace_id],
+            topic: @payload[:message].topic,
+            offset: @payload[:message].offset
+          }
+
+          with_sentry_transaction(
+            @payload[:caller],
+            message
+          ) do
             yield
-          rescue
-            finish_transaction(transaction, 500)
-            raise
           end
+        end
 
-          finish_transaction(transaction, 200)
-          scope.clear
+        def handle_consumed_batch
+          message_first = @payload[:messages].first
+          message = {
+            trace_id: @payload[:trace_id],
+            topic: message_first.topic,
+            first_offset: message_first.offset,
+            last_offset: @payload[:messages].last.offset
+          }
+
+          with_sentry_transaction(
+            @payload[:caller],
+            message
+          ) do
+            yield
+          end
         end
 
         def handle_error
@@ -64,9 +78,9 @@ module Sbmt
 
         private
 
-        def start_transaction(trace_id, consumer, message)
+        def start_transaction(consumer, message)
           scope = ::Sentry.get_current_scope
-          scope.set_tags(trace_id: trace_id, topic: message.topic, offset: message.offset)
+          scope.set_tags(message)
           scope.set_transaction_name("Sbmt/KafkaConsumer/#{consumer.class.name}")
 
           transaction = ::Sentry.start_transaction(name: scope.transaction_name, op: "kafka-consumer")
@@ -97,6 +111,22 @@ module Sbmt
           # so in that case we return raw_payload
           message.raw_payload
         end
+
+        def with_sentry_transaction(consumer, message)
+          return yield unless ::Sentry.initialized?
+
+          scope, transaction = start_transaction(consumer, message)
+
+          begin
+            yield
+          rescue
+            finish_transaction(transaction, 500)
+            raise
+          end
+
+          finish_transaction(transaction, 200)
+          scope.clear
+        end
       end
     end
   end

data/lib/sbmt/kafka_consumer/testing/shared_contexts/with_sbmt_karafka_consumer.rb

@@ -28,15 +28,14 @@ RSpec.shared_context "with sbmt karafka consumer" do
 
   def publish_to_sbmt_karafka(raw_payload, opts = {})
     message = Karafka::Messages::Message.new(raw_payload, Karafka::Messages::Metadata.new(metadata_defaults.merge(opts)))
-    consumer.messages = Karafka::Messages::Messages.new(
-      [message],
-      Karafka::Messages::BatchMetadata.new(
-        topic: test_topic.name,
-        partition: 0,
-        processed_at: Time.zone.now,
-        created_at: Time.zone.now
-      )
-    )
+    consumer.messages = consumer_messages([message])
+  end
+
+  def publish_to_sbmt_karafka_batch(raw_payloads, opts = {})
+    messages = raw_payloads.map do |p|
+      Karafka::Messages::Message.new(p, Karafka::Messages::Metadata.new(metadata_defaults.merge(opts)))
+    end
+    consumer.messages = consumer_messages(messages)
   end
 
   # @return [Hash] message default options
@@ -58,4 +57,18 @@ RSpec.shared_context "with sbmt karafka consumer" do
     instance.singleton_class.include Karafka::Processing::Strategies::Default
     instance
   end
+
+  private
+
+  def consumer_messages(messages)
+    Karafka::Messages::Messages.new(
+      messages,
+      Karafka::Messages::BatchMetadata.new(
+        topic: test_topic.name,
+        partition: 0,
+        processed_at: Time.zone.now,
+        created_at: Time.zone.now
+      )
+    )
+  end
 end

data/lib/sbmt/kafka_consumer/version.rb

@@ -2,6 +2,6 @@
 
 module Sbmt
   module KafkaConsumer
-    VERSION = "2.0.0"
+    VERSION = "2.1.0"
   end
 end

data/sbmt-kafka_consumer.gemspec

@@ -33,7 +33,7 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency "rails", ">= 5.2"
   spec.add_dependency "zeitwerk", "~> 2.3"
-  spec.add_dependency "karafka", "~> 2.2"
+  spec.add_dependency "karafka", "~> 2.2", "< 2.4" # [Breaking] Drop the concept of consumer group mapping.
   spec.add_dependency "yabeda", ">= 0.11"
   spec.add_dependency "anyway_config", ">= 2.4.0"
   spec.add_dependency "thor"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sbmt-kafka_consumer
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 2.1.0
 platform: ruby
 authors:
 - Sbermarket Ruby-Platform Team
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-02-27 00:00:00.000000000 Z
+date: 2024-06-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -45,6 +45,9 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -52,6 +55,9 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2.4'
 - !ruby/object:Gem::Dependency
   name: yabeda
   requirement: !ruby/object:Gem::Requirement
@@ -564,7 +570,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.10
+rubygems_version: 3.5.11
 signing_key:
 specification_version: 4
 summary: Ruby gem for consuming Kafka messages