sbmt-kafka_consumer 2.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: fe979c0a187c1aac9f2f8d571d99801e564912db57b86c8219b569fc3faef5c1
+  data.tar.gz: af13a117d78170019ee3f0ae0930c5b65c0dd027efa39a46eb51f1a32998f47b
+SHA512:
+  metadata.gz: 7105c49b6c2ecf06769bb310c9a4bf836aa5cbad1b2ee994860b814b71059b8217610429dada55b408063597e32893b1efb404be11902d22693b37e4233ddb18
+  data.tar.gz: 2a6195f0db6b8a0a7f9435e6b93745422a97f2238c073598155b0c2d81212891028a67db9029160735754189610a3ab4687c433044c67bf9d6d2b6802bdd1149

data/.rspec

@@ -0,0 +1,3 @@
+--color
+--require spec_helper
+--require rails_helper

data/.rubocop.yml

@@ -0,0 +1,34 @@
+inherit_mode:
+  merge:
+    - Exclude
+
+require:
+  - rubocop-performance
+  - rubocop-rails
+  - rubocop-rspec
+  - standard
+
+inherit_gem:
+  standard: config/base.yml
+
+inherit_from:
+  - rubocop/rspec.yml
+
+AllCops:
+  NewCops: enable
+  SuggestExtensions: false
+  TargetRubyVersion: 2.7
+  TargetRailsVersion: 5.2
+
+RSpec/FilePath:
+  Enabled: false
+
+RSpec/VerifiedDoubles:
+  Exclude:
+    - spec/**/*_spec.rb
+
+Style/SingleLineMethods:
+  Enabled: false
+
+Style/EmptyMethod:
+  Enabled: false

data/Appraisals

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+# See compatibility table at https://www.fastruby.io/blog/ruby/rails/versions/compatibility-table.html
+
+versions_map = {
+  "5.2" => %w[2.7],
+  "6.0" => %w[2.7],
+  "6.1" => %w[2.7 3.0],
+  "7.0" => %w[3.1],
+  "7.1" => %w[3.2]
+}
+
+current_ruby_version = RUBY_VERSION.split(".").first(2).join(".")
+
+versions_map.each do |rails_version, ruby_versions|
+  ruby_versions.each do |ruby_version|
+    next if ruby_version != current_ruby_version
+
+    appraise "rails-#{rails_version}" do
+      gem "rails", "~> #{rails_version}.0"
+    end
+  end
+end

data/CHANGELOG.md

@@ -0,0 +1,292 @@
+# Change Log
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](http://keepachangelog.com/)
+and this project adheres to [Semantic Versioning](http://semver.org/).
+
+## [Unreleased] - yyyy-mm-dd
+
+### Added
+
+### Changed
+
+### Fixed
+
+## [2.0.0] - 2024-01-30
+
+### Changed
+
+- Remove `sbmt-dev`
+
+## [1.0.0] - 2024-01-12
+
+### Added
+
+- Use mainstream karafka instead of custom fork
+
+## [0.23.0] - 2024-01-12
+
+### Added
+
+- ability to override `kafka_options` for topic
+
+## [0.22.0] - 2024-01-09
+
+### Added
+
+- removed useless `outbox_producer` param for `InboxConsumer` class
+- removed useless log messages from `InboxConsumer` class
+
+## [0.21.0] - 2024-01-09
+
+### Fixed
+
+- initialization of proxy consumer classes
+- consumer class name in sentry's transaction name
+
+## [0.20.0] - 2024-01-09
+
+### Added
+
+- New config options `metrics`
+- `metrics.port` for a metrics port that is different from the probes port
+- `metrics.path` for a metrics path
+
+## [0.19.2] - 2023-10-18
+
+### Fixed
+
+- Stub kafka_client to prevent calls to librdkafka: fixes SEGFAULT in parallel tests
+
+## [0.19.1] - 2023-10-05
+
+### Fixed
+
+- disable karafka's `config.strict_topics_namespacing`
+
+## [0.19.0] - 2023-09-29
+
+### Added
+
+- `outbox_producer` configuration flag
+
+## [0.18.4] - 2023-09-26
+
+### Fixed
+
+- Use `Rails.application.executor.wrap` instead manual AR connection clearing
+
+## [0.18.3] - 2023-09-15
+
+### Fixed
+
+- Fix broken outbox item generator call in the `kafka_consumer:inbox_consumer` generator
+
+## [0.18.2] - 2023-09-14
+
+### Fixed
+
+- Properly extract opentelemetry context from kafka message headers
+
+## [0.18.1] - 2023-09-13
+
+### Fixed
+
+- Port `v0.17.5` (properly clear `ActiveRecord` connections in case `skip_on_error` option is used) to master (v0.18)
+
+## [0.18.0] - 2023-09-11
+
+### Added
+
+- OpenTelemetry tracing
+
+## [0.17.5] - 2023-09-13
+
+### Fixed
+
+- Properly clear `ActiveRecord` connections in case `skip_on_error` option is used
+
+## [0.17.4] - 2023-09-05
+
+### Fixed
+
+- Latency metrics in seconds instead ms
+
+## [0.17.3] - 2023-08-31
+
+### Fixed
+
+- Decreased sleep time on db error in a consumer
+
+## [0.17.2] - 2023-08-16
+
+### Fixed
+
+- Fix `message.metadata.key` validation if key is an empty string
+
+## [0.17.1] - 2023-08-08
+
+### Fixed
+
+- Check Idempotency-Key for a empty string
+
+## [0.17.0] - 2023-08-07
+
+### Added
+
+- ability to configure consumer group mapper in `kafka_consumer.yml` (needed for proper migration from existing karafka v2 based consumers)
+- ability to define/override inbox-item attributes in InboxConsumer
+
+### Fixed
+- report `kafka_consumer_inbox_consumes` metric with tag `status = skipped` (instead `failure`) if skip_on_error is enabled on InboxConsumer
+
+## [0.16.0] - 2023-07-27
+
+### Added
+
+- additional tags (client, group_id, partition, topic) for metric `kafka_consumer_inbox_consumes`
+
+## [0.15.0] - 2023-07-21
+
+### Added
+
+- `kafka_consumer:install` generator
+- `kafka_consumer:consumer_group` generator
+- `kafka_consumer:consumer` generator
+
+## [0.14.2] - 2023-07-19
+
+### Changed
+- `.clear_all_connections!` is now called for all DB roles
+
+## [0.14.1] - yyyy-mm-dd
+
+### Added
+- add label `api` for group `kafka_api`
+
+### Changed
+- README improvements
+
+## [0.14.0] - 2023-07-06
+
+### Added
+- report message payload and headers to Sentry if consumer detailed logging is enabled
+
+## [0.13.1] - 2023-07-05
+
+### Added
+- `event_key` callback added to `Sbmt::KafkaConsumer::InboxConsumer`
+
+## [0.13.0] - 2023-06-20
+
+### Changed
+- logging / instrumentation improvements
+
+## [0.12.0] - 2023-06-20
+
+### Changed
+- README improvements
+- update sbmt-waterdrop (via sbmt-karafka) to fix karafka-rdkafka 0.13 compatibility issue
+
+## [0.11.0] - 2023-06-13
+
+### Added
+- `skip_on_error` consumer option to skip message processing (and commit offsets) if exception was raised
+
+## [0.10.0] - 2023-06-07
+
+### Added
+- `SimpleLoggingConsumer`, which just consumes/logs messages, can be used for debug purposes
+
+## [0.9.0] - 2023-06-06
+
+### Changed
+- add custom `ConsumerMapper` to be consistent with KarafkaV1 consumer-group naming conventions (e.g. karafka v1 uses underscored client-id in consumer group name)
+- reuse with_db_retry: release ActiveRecord conn everytime after message processing, in case there's a connection-pool degradation
+
+## [0.8.0] - 2023-06-01
+
+### Changed
+- update sbmt-karafka to 2.1.3
+- remove db retries logic as `ActiveRecord::Base::clear_active_connections!` is already handled by karafka v2 after processing a batch
+- async metrics reporting for `statistics.emitted` event to prevent rdkafka's main thread hanging, see https://github.com/karafka/karafka/pull/1420/files
+- use Rails logger by default
+- use `$stdout.sync = true` in consumer server process to avoid STDOUT buffering issues in docker/k8s
+
+## [0.7.1] - 2023-05-31
+
+### Fixed
+- db error logging in base consumer
+
+## [0.7.0] - 2023-05-30
+
+### Added
+- add `Sbmt::KafkaConsumer::Instrumentation::LivenessListener` and `Sbmt::KafkaConsumer::Instrumentation::ReadinessListener` listeners
+- add `probes` option
+- add `HttpHealthCheck` server with probes' endpoints
+
+## [0.6.1] - 2023-05-30
+
+### Added
+- set default `source: "KAFKA"` option when creating `inbox_item` in `InboxConsumer`
+
+## [0.6.0] - 2023-05-29
+
+### Added
+- add `manual_offset_management` topic's option (defaults to true)
+- add consumer `group_id` to inbox-item metadata (InboxConsumer)
+
+## [0.5.1] - 2023-05-25
+
+### Fixed
+- sentry tracing when instrumentation event is not an exception
+- payload deserialization if skip_decoding_error is enabled
+
+## [0.5.0] - 2023-05-23
+
+### Changed
+- add default deserializer (NullDeserializer) to config
+- refactor logging
+
+## [0.4.0] - 2023-05-19
+
+### Changed
+- refactor consumer groups config
+
+## [0.3.0] - 2023-05-19
+
+### Added
+- add timeout aliases to kafka config
+- README actualization
+
+## [0.2.0] - 2023-05-16
+
+### Added
+- implement consumer metrics
+
+## [Unreleased] - 2023-05-03
+
+### Added
+- base config loader via AnywayConfig
+
+### Changed
+
+### Fixed
+
+## [Unreleased] - 2023-04-26
+
+### Added
+- BaseConsumer
+- InboxConsumer
+- Instrumentation listeners: sentry, logger, yabeda
+
+### Changed
+
+### Fixed
+
+## [Unreleased]
+
+## [0.1.0] - 2023-04-19
+
+- Initial release

data/Gemfile

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+gemspec

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 SberMarket Tech
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,296 @@
+[![Gem Version](https://badge.fury.io/rb/sbmt-kafka_consumer.svg)](https://badge.fury.io/rb/sbmt-kafka_consumer)
+[![Build Status](https://github.com/SberMarket-Tech/sbmt-kafka_consumer/actions/workflows/tests.yml/badge.svg?branch=master)](https://github.com/SberMarket-Tech/sbmt-outbox/actions?query=branch%3Amaster)
+
+# Sbmt-KafkaConsumer
+
+This gem is used to consume Kafka messages. It is a wrapper over the [Karafka](https://github.com/karafka/karafka) gem, and is recommended for use as a transport with the [sbmt-outbox](https://github.com/SberMarket-Tech/sbmt-outbox) gem.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem "sbmt-kafka_consumer"
+```
+
+And then execute:
+
+```bash
+bundle install
+```
+
+## Auto configuration
+
+We recommend going through the configuration and file creation process using the following Rails generators. Each generator can be run by using the `--help` option to learn more about the available arguments.
+
+### Initial configuration
+
+If you plug the gem into your application for the first time, you can generate the initial configuration:
+
+```shell
+rails g kafka_consumer:install
+```
+
+As the result, the `config/kafka_consumer.yml` file will be created.
+
+### Consumer class
+
+A consumer class can be generated with the following command:
+
+```shell
+rails g kafka_consumer:consumer MaybeNamespaced::Name
+```
+
+### Inbox consumer
+
+To generate an Inbox consumer for use with gem [sbmt-outbox](https://github.com/SberMarket-Tech/sbmt-outbox), run the following command:
+
+```shell
+rails g kafka_consumer:inbox_consumer MaybeNamespaced::Name some-consumer-group some-topic
+```
+
+## Manual configuration
+
+The `config/kafka_consumer.yml` file is a main configuration for the gem.
+
+Example config with a full set of options:
+
+```yaml
+default: &default
+  client_id: "my-app-consumer"
+  concurrency: 4 # max number of threads
+  # optional Karafka options
+  max_wait_time: 1
+  shutdown_timeout: 60
+  pause_timeout: 1
+  pause_max_timeout: 30
+  pause_with_exponential_backoff: true
+  auth:
+    kind: plaintext
+  kafka:
+    servers: "kafka:9092"
+    # optional Kafka options
+    heartbeat_timeout: 5
+    session_timeout: 30
+    reconnect_timeout: 3
+    connect_timeout: 5
+    socket_timeout: 30
+    kafka_options:
+      allow.auto.create.topics: true
+  probes: # optional section
+    port: 9394
+    endpoints:
+      readiness:
+        enabled: true
+        path: "/readiness"
+      liveness:
+        enabled: true
+        path: "/liveness"
+        timeout: 15
+  metrics: # optional section
+    port: 9090
+    path: "/metrics"
+  consumer_groups:
+    group_ref_id_1:
+      name: cg_with_single_topic
+      topics:
+        - name: topic_with_inbox_items
+          consumer:
+            klass: "Sbmt::KafkaConsumer::InboxConsumer"
+            init_attrs:
+              name: "test_items"
+              inbox_item: "TestInboxItem"
+          deserializer:
+            klass: "Sbmt::KafkaConsumer::Serialization::NullDeserializer"
+          kafka_options:
+            auto.offset.reset: latest
+    group_ref_id_2:
+      name: cg_with_multiple_topics
+      topics:
+        - name: topic_with_json_data
+          consumer:
+            klass: "SomeConsumer"
+          deserializer:
+            klass: "Sbmt::KafkaConsumer::Serialization::JsonDeserializer"
+        - name: topic_with_protobuf_data
+          consumer:
+            klass: "SomeConsumer"
+          deserializer:
+            klass: "Sbmt::KafkaConsumer::Serialization::ProtobufDeserializer"
+            init_attrs:
+              message_decoder_klass: "SomeDecoder"
+              skip_decoding_error: true
+
+development:
+  <<: *default
+
+test:
+  <<: *default
+  deliver: false
+
+production:
+  <<: *default
+```
+
+### `auth` config section
+
+The gem supports 2 variants: plaintext (default) and SASL-plaintext
+
+SASL-plaintext:
+
+```yaml
+auth:
+  kind: sasl_plaintext
+  sasl_username: user
+  sasl_password: pwd
+  sasl_mechanism: SCRAM-SHA-512
+```
+
+### `kafka` config section
+
+The `servers` key is required and should be in rdkafka format: without `kafka://` prefix, for example: `srv1:port1,srv2:port2,...`.
+
+The `kafka_config` section may contain any [rdkafka option](https://github.com/confluentinc/librdkafka/blob/master/CONFIGURATION.md). Also, `kafka_options` may be redefined for each topic.
+
+### `consumer_groups` config section
+
+```yaml
+consumer_groups:
+  # group id can be used when starting a consumer process (see CLI section below)
+  group_id:
+    name: some_group_name # required
+    topics:
+    - name: some_topic_name # required
+      active: true # optional, default true
+      consumer:
+        klass: SomeConsumerClass # required, a consumer class inherited from Sbmt::KafkaConsumer::BaseConsumer
+        init_attrs: # optional, consumer class attributes (see below)
+          key: value
+      deserializer:
+        klass: SomeDeserializerClass # optional, default NullDeserializer, a deserializer class inherited from Sbmt::KafkaConsumer::Serialization::NullDeserializer
+        init_attrs: # optional, deserializer class attributes (see below)
+          key: value
+      kafka_options: # optional, this section allows to redefine the root rdkafka options for each topic
+        auto.offset.reset: latest
+```
+
+#### `consumer.init_attrs` options for `BaseConsumer`
+
+- `skip_on_error` - optional, default false, omit consumer errors in message processing and commit the offset to Kafka
+
+#### `consumer.init_attrs` options for `InboxConsumer`
+
+- `inbox_item` - required, name of the inbox item class
+- `event_name` - optional, default nil, used when the inbox item keep several event types
+- `skip_on_error` - optional, default false, omit consumer errors in message processing and commit the offset to Kafka
+
+#### `deserializer.init_attrs` options
+
+- `skip_decoding_error` — don't raise an exception when cannot deserialize the message
+
+### `probes` config section
+
+In Kubernetes, probes are mechanisms used to assess the health of your application running within a container.
+
+```yaml
+probes:
+  port: 9394 # optional, default 9394
+  endpoints:
+    liveness:
+      enabled: true # optional, default true
+      path: /liveness # optional, default "/liveness"
+      timeout: 10 # optional, default 10, timeout in seconds after which the group is considered dead
+    readiness:
+      enabled: true # optional, default true
+      path: /readiness/kafka_consumer # optional, default "/readiness/kafka_consumer"
+```
+
+### `metrics` config section
+
+We use [Yabeda](https://github.com/yabeda-rb/yabeda) to collect [all kind of metrics](./lib/sbmt/kafka_consumer/yabeda_configurer.rb).
+
+```yaml
+metrics:
+  port: 9090 # optional, default is probes.port
+  path: /metrics # optional, default "/metrics"
+```
+
+### `Kafkafile`
+
+You can create a `Kafkafile` in the root of your app to configure additional settings for your needs.
+
+Example:
+
+```ruby
+require_relative "config/environment"
+
+some-extra-configuration
+```
+
+## CLI
+
+Run the following command to execute a server
+
+```shell
+kafka_consumer -g some_group_id_1 -g some_group_id_2 -c 5
+```
+
+Where:
+- `-g` - `group`, a consumer group id, if not specified, all groups from the config will be processed
+- `-c` - `concurrency`, a number of threads, default is 4
+
+### `concurrency` argument
+
+[Concurrency and Multithreading](https://karafka.io/docs/Concurrency-and-multithreading/).
+
+Don't forget to properly calculate and set the size of the ActiveRecord connection pool:
+- each thread will utilize one db connection from the pool
+- an application can have monitoring threads which can use db connections from the pool
+
+Also pay attention to the number of processes of the server:
+- `number_of_processes x concurrency` for topics with high data intensity can be equal to the number of partitions of the consumed topic
+- `number_sof_processes x concurrency` for topics with low data intensity can be less than the number of partitions of the consumed topic
+
+## Testing
+
+To test your consumer with Rspec, please use [this shared context](./lib/sbmt/kafka_consumer/testing/shared_contexts/with_sbmt_karafka_consumer.rb)
+
+```ruby
+require "sbmt/kafka_consumer/testing"
+
+RSpec.describe OrderCreatedConsumer do
+  include_context "with sbmt karafka consumer"
+
+  it "works" do
+    publish_to_sbmt_karafka(payload, deserializer: deserializer)
+    expect { consume_with_sbmt_karafka }.to change(Order, :count).by(1)
+  end
+end
+```
+
+## Development
+
+1. Prepare environment
+```shell
+dip provision
+```
+
+2. Run tests
+```shell
+dip rspec
+```
+
+3. Run linter
+```shell
+dip rubocop
+```
+
+4. Run Kafka server
+```shell
+dip up
+```
+
+5. Run consumer server
+```shell
+dip kafka-consumer
+```

data/Rakefile

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+require "rubocop/rake_task"
+
+RuboCop::RakeTask.new
+
+task default: %i[spec rubocop]

data/config.ru

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require "rubygems"
+require "bundler"
+
+Bundler.require :default, :development
+
+Combustion.initialize! :all
+run Combustion::Application