sbmt-kafka_producer 2.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 3b6c10d4e49a462afad448f607d1d9c202b59a7b70661ef17cf7ec8dd4d8112e
+  data.tar.gz: 4cf15927ca33194dc5639afcf7967b3965ac817ad8813380e33f7dc003aa11d0
+SHA512:
+  metadata.gz: 467ccc822998a5a9bb174557b6a72be74855060cb5e24e3097efb9ac9be746adfede6c4c77721e637ae6b14db4dc8983c7b76a23c6127016345f3f7fdcf1f681
+  data.tar.gz: bd1df75213ff393acf379fe4d3bbf013747af8f193a0970effbff0782e6c7dd1adeb8cd8cec4704cebb6fc214a1631dc330a679d77ca83c786067786adda329f

data/.rspec

@@ -0,0 +1,3 @@
+--color
+--require spec_helper
+--require rails_helper

data/.rubocop.yml

@@ -0,0 +1,34 @@
+inherit_mode:
+  merge:
+    - Exclude
+
+require:
+  - rubocop-performance
+  - rubocop-rails
+  - rubocop-rspec
+  - standard
+
+inherit_gem:
+  standard: config/base.yml
+
+inherit_from:
+  - rubocop/rspec.yml
+
+AllCops:
+  NewCops: enable
+  SuggestExtensions: false
+  TargetRubyVersion: 2.7
+  TargetRailsVersion: 5.2
+
+RSpec/FilePath:
+  Enabled: false
+
+RSpec/VerifiedDoubles:
+  Exclude:
+    - spec/**/*_spec.rb
+
+Style/SingleLineMethods:
+  Enabled: false
+
+Style/EmptyMethod:
+  Enabled: false

data/Appraisals

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+# See compatibility table at https://www.fastruby.io/blog/ruby/rails/versions/compatibility-table.html
+
+versions_map = {
+  "5.1" => %w[2.7],
+  "5.2" => %w[2.7],
+  "6.0" => %w[2.7],
+  "6.1" => %w[2.7 3.0],
+  "7.0" => %w[3.1],
+  "7.1" => %w[3.2]
+}
+
+current_ruby_version = RUBY_VERSION.split(".").first(2).join(".")
+
+versions_map.each do |rails_version, ruby_versions|
+  ruby_versions.each do |ruby_version|
+    next if ruby_version != current_ruby_version
+
+    appraise "rails-#{rails_version}" do
+      gem "rails", "~> #{rails_version}.0"
+    end
+  end
+end

data/CHANGELOG.md

@@ -0,0 +1,166 @@
+# Change Log
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](http://keepachangelog.com/)
+and this project adheres to [Semantic Versioning](http://semver.org/).
+
+## [Unreleased] - yyyy-mm-dd
+
+### Added
+
+### Changed
+
+### Fixed
+
+## [2.0.0] - 2024-01-29
+
+### Changed
+
+- Remove `sbmt-dev`
+
+## [1.0.0] - 2024-01-12
+
+### Added
+
+- Use mainstream karafka instead of custom fork
+
+## [0.8.0] - 2023-10-05
+
+### Added
+
+- Errors' `cause` handling
+
+### Fixed
+
+- change from `double` to `instance_double`
+
+## [0.7.0] - 2023-09-14
+
+### Added
+
+- Plug OpenTelemetry
+
+## [0.6.3] - 2023-08-10
+
+### Fixed
+
+- Return True when publishing with bang methods
+
+## [0.6.2] - 2023-08-08
+
+### Added
+
+- add ErrorTracker for Sentry
+
+## [0.6.1] - 2023-08-07
+
+### Fixed
+
+- Don't catch an exception when publishing through the Sbmt::KafkaProducer::OutboxProducer
+
+## [0.6.0] - 2023-07-23
+
+### Added
+
+- rails generator for initial configuration
+- rails generator for producer/outbox_producer creation
+
+## [0.5.1] - 2023-07-21
+
+### Fixed
+
+- change sentry method from capture_message to capture_exception
+
+## [0.5.0] - 2023-06-26
+
+### Fixed
+- Mock BaseProducer for rspec
+
+## [Unreleased] - 2023-06-21
+
+### Changed
+- update README
+
+## [0.4.2] - 2023-06-20
+
+### Fixed
+- fixed version **sbmt-waterdrop**
+
+## [0.4.1] - 2023-06-19
+
+### Fixed
+- fixed error handling in the method **on_error_occurred**
+
+## [0.4.0] - 2023-06-13
+
+### Changed
+- config changed from anyway to Dry::Struct
+
+## [0.3.0] - 2023-06-01
+
+### Added
+- implement producer metrics
+
+## [0.2.3] - 2023-05-19
+
+### Added
+- for outbox, if the default settings for the kafka section are overridden, they are overwritten
+
+### Changed
+
+### Fixed
+
+## [0.2.2] - 2023-05-18
+
+### Added
+- arbitrary parameters from kafka
+
+### Changed
+
+### Fixed
+
+## [0.2.1] - 2023-05-16
+
+### Added
+- fix logger
+
+### Changed
+
+### Fixed
+
+## [0.2.0] - 2023-05-16
+
+### Added
+- basic options for producer
+
+### Changed
+
+### Fixed
+
+## [Unreleased] - 2023-05-04
+
+### Added
+- basic config for producer via gem anyway_config
+
+### Changed
+
+### Fixed
+
+## [Unreleased] - 2023-05-02
+
+### Added
+- BaseProducer
+- OutboxProducer
+- Sentry, logger
+
+### Changed
+
+### Fixed
+
+
+## [Unreleased]
+
+## [0.1.0] - 2023-04-17
+
+- Initial release

data/Gemfile

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+gemspec

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 SberMarket Tech
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,167 @@
+[![Gem Version](https://badge.fury.io/rb/sbmt-kafka_producer.svg)](https://badge.fury.io/rb/sbmt-kafka_producer)
+[![Build Status](https://github.com/SberMarket-Tech/sbmt-kafka_producer/actions/workflows/tests.yml/badge.svg?branch=master)](https://github.com/SberMarket-Tech/sbmt-kafka_producer/actions?query=branch%3Amaster)
+
+# Sbmt-KafkaProducer
+
+This gem is used to produce Kafka messages. It is a wrapper over the [waterdrop](https://github.com/karafka/waterdrop) gem, and it is recommended for use as a transport with the [sbmt-outbox](https://github.com/SberMarket-Tech/sbmt-outbox) gem.
+
+## Installation
+
+Add this line to your app's Gemfile:
+
+```ruby
+gem "sbmt-kafka_producer"
+```
+
+And then execute:
+
+```bash
+bundle install
+```
+
+## Auto configuration
+
+We recommend going through the configuration and file creation process using the following Rails generators. Each generator can be run by using the `--help` option to learn more about the available arguments.
+
+### Initial configuration
+
+If you plug the gem into your application for the first time, you can generate the initial configuration:
+
+```shell
+rails g kafka_producer:install
+```
+
+As a result, the `config/kafka_producer.yml` file will be created.
+
+### Producer class
+
+A producer class can be generated with the following command:
+
+```shell
+rails g kafka_producer:producer MaybeNamespaced::Name sync topic
+```
+
+As the result, a sync producer will be created.
+
+### Outbox producer
+
+To generate an Outbox producer for use with Gem [sbmt-Outbox](https://github.com/SberMarket-Tech/sbmt-outbox), run the following command:
+
+```shell
+rails g kafka_producer:outbox_producer SomeOutboxItem
+```
+
+## Manual configuration
+
+The `config/kafka_producer.yml` file is the main configuration for this gem.
+
+```yaml
+default: &default
+  deliver: true
+  ignore_kafka_error: true
+  # see more options at https://github.com/karafka/waterdrop/blob/master/lib/waterdrop/config.rb
+  wait_on_queue_full: true
+  max_payload_size: 1000012
+  max_wait_timeout: 5
+  wait_timeout: 0.005
+  auth:
+    kind: plaintext
+  kafka:
+    servers: "kafka:9092" # required
+    max_retries: 2 # optional, default: 2
+    required_acks: -1 # optional, default: -1
+    ack_timeout: 1 # in seconds, optional, default: 1
+    retry_backoff: 1 # in seconds, optional, default: 1
+    connect_timeout: 1 # in seconds, optional, default: 1
+    kafka_config: # low-level custom Kafka options
+      queue.buffering.max.messages: 1
+      queue.buffering.max.ms: 10_000
+
+development:
+  <<: *default
+
+test:
+  <<: *default
+  deliver: false
+  wait_on_queue_full: false
+
+production:
+  <<: *default
+```
+
+### `auth` config section
+
+The gem supports 2 variants: plaintext (default) and SASL-plaintext
+
+SASL-plaintext:
+
+```yaml
+auth:
+  kind: sasl_plaintext
+  sasl_username: user
+  sasl_password: pwd
+  sasl_mechanism: SCRAM-SHA-512
+```
+
+### `kafka` config section
+
+The `servers` key is required and should be in rdkafka format: without `kafka://` prefix, for example: `srv1:port1,srv2:port2,...`.
+
+The `kafka_config` section may contain any [rdkafka option](https://github.com/confluentinc/librdkafka/blob/master/CONFIGURATION.md)
+
+### Producer class
+
+To create a producer that will be responsible for sending messages to Kafka, copy the following code:
+
+```ruby
+# app/producers/some_producer.rb
+class SomeProducer < Sbmt::KafkaProducer::BaseProducer
+  option :topic, default: -> { "topic" }
+
+  def publish(payload, **options)
+    sync_publish(payload, options)
+    # async_publish(payload, options)
+  end
+end
+```
+
+### Outbox producer config
+
+Add the following lines to your `config/outbox.yml` file in the `transports` section:
+
+```yaml
+outbox_items:
+  some_outbox_item:
+    transports:
+      sbmt/kafka_producer:
+        topic: 'topic'
+        kafka: # optional kafka options
+          required_acks: -1
+```
+
+## Usage
+
+To send a message to a Kafka topic, execute the following command:
+
+```ruby
+SomeProducer.new.publish(payload, key: "123", headers: {"some-header" => "some-value"})
+```
+
+## Metrics
+
+The gem collects base producing metrics using [Yabeda](https://github.com/yabeda-rb/yabeda). See metrics at [YabedaConfigurer](./lib/sbmt/kafka_producer/yabeda_configurer.rb).
+
+## Testing
+
+To stub a producer request to real Kafka broker, you can use a mock. To do this, please add `require "sbmt/kafka_producer/testing"` to the `spec/rails_helper.rb`.
+
+## Development
+
+Install [dip](https://github.com/bibendi/dip).
+
+And run:
+
+```shell
+dip provision
+dip rspec
+```

data/Rakefile

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+require "rubocop/rake_task"
+
+RuboCop::RakeTask.new
+
+task default: %i[spec rubocop]

data/dip.yml

@@ -0,0 +1,67 @@
+version: '7'
+
+environment:
+  RUBY_VERSION: '3.2'
+
+compose:
+  files:
+    - docker-compose.yml
+
+interaction:
+  bash:
+    description: Open the Bash shell in app's container
+    service: ruby
+    command: /bin/bash
+
+  bundle:
+    description: Run Bundler commands
+    service: ruby
+    command: bundle
+
+  rails:
+    description: Run RoR commands
+    service: ruby
+    command: bundle exec rails
+
+  appraisal:
+    description: Run Appraisal commands
+    service: ruby
+    command: bundle exec appraisal
+
+  rspec:
+    description: Run Rspec commands
+    service: ruby
+    command: bundle exec rspec
+    subcommands:
+      all:
+        command: bundle exec appraisal rspec
+      rails-6.0:
+        command: bundle exec appraisal rails-6.0 rspec
+      rails-6.1:
+        command: bundle exec appraisal rails-6.1 rspec
+      rails-7.0:
+        command: bundle exec appraisal rails-7.0 rspec
+      rails-7.1:
+        command: bundle exec appraisal rails-7.1 rspec
+
+  rubocop:
+    description: Run Ruby linter
+    service: ruby
+    command: bundle exec rubocop
+
+  setup:
+    description: Install deps
+    service: ruby
+    command: bin/setup
+
+  test:
+    description: Run linters, run all tests
+    service: ruby
+    command: bin/test
+
+provision:
+  - dip compose down --volumes
+  - cp -f lefthook-local.dip_example.yml lefthook-local.yml
+  - rm -f Gemfile.lock
+  - rm -f gemfiles/*gemfile*
+  - dip setup

data/docker-compose.yml

@@ -0,0 +1,15 @@
+services:
+  ruby:
+    image: ruby:${RUBY_VERSION:-3.2}
+    environment:
+      HISTFILE: /app/tmp/.bash_history
+      BUNDLE_PATH: /usr/local/bundle
+      BUNDLE_CONFIG: /app/.bundle/config
+    command: bash
+    working_dir: /app
+    volumes:
+      - .:/app:cached
+      - bundler_data:/usr/local/bundle
+
+volumes:
+  bundler_data:

data/lefthook-local.dip_example.yml

@@ -0,0 +1,4 @@
+pre-commit:
+  commands:
+    rubocop:
+      run: dip {cmd}

data/lefthook.yml

@@ -0,0 +1,6 @@
+pre-commit:
+  commands:
+    rubocop:
+      tags: backend
+      glob: "{*.rb,**/*.rb,Gemfile,Rakefile}"
+      run: bundle exec rubocop -A --force-exclusion {staged_files} && git add {staged_files}

data/lib/generators/kafka_producer/concerns/configuration.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module KafkaProducer
+  module Generators
+    module Concerns
+      module Configuration
+        extend ActiveSupport::Concern
+
+        CONFIG_PATH = "config/kafka_producer.yml"
+
+        def check_config_file!
+          config_path = File.expand_path(CONFIG_PATH)
+          return if File.exist?(config_path)
+
+          generator_name = "kafka_producer:install"
+          if yes?(
+            "The file #{config_path} does not appear to exist." \
+            " Would you like to generate it?"
+          )
+            generate generator_name
+          else
+            raise Rails::Generators::Error, "Please generate #{config_path} " \
+                           "by running `bin/rails g #{generator_name}` " \
+                           "or add this file manually."
+          end
+        end
+      end
+    end
+  end
+end

data/lib/generators/kafka_producer/install/USAGE

@@ -0,0 +1,8 @@
+Description:
+    Generates the Kafka producer's initial setup
+
+Example:
+    bin/rails generate kafka_producer:install
+
+    This will create:
+      config/kafka_producer.yml

data/lib/generators/kafka_producer/install/install_generator.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+require "generators/kafka_producer/concerns/configuration"
+
+module KafkaProducer
+  module Generators
+    class InstallGenerator < Rails::Generators::Base
+      include Concerns::Configuration
+
+      source_root File.expand_path("templates", __dir__)
+
+      def create_kafka_producer_yml
+        copy_file "kafka_producer.yml", CONFIG_PATH
+      end
+    end
+  end
+end

data/lib/generators/kafka_producer/install/templates/kafka_producer.yml

@@ -0,0 +1,36 @@
+default: &default
+  deliver: true
+  wait_on_queue_full: true
+  max_payload_size: 1000012
+  max_wait_timeout: 5
+  wait_timeout: 0.005
+  ignore_kafka_error: true
+
+  auth:
+    sasl_username: <%= ENV.fetch('KAFKA_BROKERS'){ 'SCRAM-SHA-512:kafka_login:kafka_password' }.split(':').second %>
+    sasl_password: <%= ENV.fetch('KAFKA_BROKERS'){ 'SCRAM-SHA-512:kafka_login:kafka_password' }.split(':').last %>
+    sasl_mechanism: <%= ENV.fetch('KAFKA_BROKERS'){ 'SCRAM-SHA-512:kafka_login:kafka_password' }.split(':').first %>
+    kind: 'sasl_plaintext'
+
+  kafka:
+    servers: "kafka:9092"
+    max_retries: 2
+    required_acks: -1
+    ack_timeout: 1
+    retry_backoff: 1
+    connect_timeout: 1
+
+development:
+  <<: *default
+  auth:
+    kind: plaintext
+test:
+  <<: *default
+  deliver: false
+  wait_on_queue_full: false
+  auth:
+    kind: plaintext
+staging: &staging
+  <<: *default
+production:
+  <<: *staging

data/lib/generators/kafka_producer/outbox_producer/USAGE

@@ -0,0 +1,9 @@
+Description:
+    Modifies the config/outbox.yml file to work with the outbox gem
+    Pass in the outbox item, either CamelCased or under_scored
+
+Example:
+    bin/rails generate kafka_producer:outbox_producer some_namespace/outbox_item
+
+    This will modify:
+        config/outbox.yml

data/lib/generators/kafka_producer/outbox_producer/outbox_producer_generator.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+
+module KafkaProducer
+  module Generators
+    class OutboxProducerGenerator < Rails::Generators::NamedBase
+      source_root File.expand_path("templates", __dir__)
+
+      class_option :skip_item, type: :boolean, default: false, desc: "Skip creating InboxItem"
+
+      def insert_outbox_producer
+        generate "outbox:item", "#{item_name.underscore} --kind outbox" unless options[:skip_item]
+        generate "outbox:transport", "#{item_name.underscore} sbmt/kafka_producer --kind outbox"
+      end
+
+      private
+
+      def item_name
+        file_path
+      end
+    end
+  end
+end

data/lib/generators/kafka_producer/producer/USAGE

@@ -0,0 +1,10 @@
+Description:
+    Stubs out a new non-outbox producer. Pass the producer name, either
+         CamelCased or under_scored
+    It takes the producer type, topic as arguments.
+
+Example:
+    bin/rails generate kafka_producer:producer Test producer_type topic
+
+    This will create:
+      app/producers/test_producer.rb