sbmt-kafka_consumer 2.0.0

data/lib/sbmt/kafka_consumer/routing/karafka_v2_consumer_mapper.rb

@@ -0,0 +1,9 @@
+module Sbmt
+  module KafkaConsumer
+    module Routing
+      # uses default karafka v2 mapper
+      # exists just for naming consistency with KarafkaV1ConsumerMapper
+      class KarafkaV2ConsumerMapper < Karafka::Routing::ConsumerMapper; end
+    end
+  end
+end

data/lib/sbmt/kafka_consumer/serialization/base_deserializer.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module KafkaConsumer
+    module Serialization
+      class BaseDeserializer
+        attr_reader :skip_decoding_error
+
+        def initialize(skip_decoding_error: false)
+          @skip_decoding_error = skip_decoding_error
+        end
+
+        def call(_message)
+          raise NotImplementedError, "Implement this in a subclass"
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/kafka_consumer/serialization/json_deserializer.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module KafkaConsumer
+    module Serialization
+      class JsonDeserializer < BaseDeserializer
+        def call(message)
+          # nil payload can be present for example for tombstone messages
+          message.raw_payload.nil? ? nil : ::JSON.parse(message.raw_payload)
+        rescue JSON::ParserError => e
+          raise Sbmt::KafkaConsumer::SkipUndeserializableMessage, "cannot decode message: #{e.message}, payload: #{message.raw_payload}" if skip_decoding_error
+
+          raise
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/kafka_consumer/serialization/null_deserializer.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module KafkaConsumer
+    module Serialization
+      class NullDeserializer < BaseDeserializer
+        def call(message)
+          message.raw_payload
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/kafka_consumer/serialization/protobuf_deserializer.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require "google/protobuf"
+
+module Sbmt
+  module KafkaConsumer
+    module Serialization
+      class ProtobufDeserializer < BaseDeserializer
+        attr_reader :message_decoder
+
+        def initialize(message_decoder_klass:, skip_decoding_error: false)
+          super(skip_decoding_error: skip_decoding_error)
+
+          @message_decoder = message_decoder_klass.constantize
+        end
+
+        def call(message)
+          message_decoder.decode(message.raw_payload)
+        rescue Google::Protobuf::ParseError, ArgumentError => e
+          raise Sbmt::KafkaConsumer::SkipUndeserializableMessage, "cannot decode message: #{e.message}, payload: #{message.raw_payload}" if skip_decoding_error
+
+          raise
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/kafka_consumer/server.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module KafkaConsumer
+    class Server < Karafka::Server
+      class << self
+        # original klass tries to validate karafka-specific server cli-options which we override
+        # see Karafka::Server for details
+        def run
+          Karafka::Server.listeners = []
+          Karafka::Server.workers = []
+
+          process.on_sigint { Karafka::Server.stop }
+          process.on_sigquit { Karafka::Server.stop }
+          process.on_sigterm { Karafka::Server.stop }
+          process.on_sigtstp { Karafka::Server.quiet }
+          process.supervise
+
+          $stdout.puts "Starting server"
+          Karafka::Server.start
+
+          sleep(0.1) until Karafka::App.terminated?
+          # rubocop:disable Lint/RescueException
+        rescue Exception => e
+          $stdout.puts "Cannot start server: #{e.message}"
+
+          # rubocop:enable Lint/RescueException
+          Karafka::Server.stop
+
+          raise e
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/kafka_consumer/simple_logging_consumer.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class Sbmt::KafkaConsumer::SimpleLoggingConsumer < Sbmt::KafkaConsumer::BaseConsumer
+  private
+
+  def log_payload?
+    true
+  end
+
+  def process_message(_message); end
+end

data/lib/sbmt/kafka_consumer/testing/shared_contexts/with_sbmt_karafka_consumer.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+RSpec.shared_context "with sbmt karafka consumer" do
+  subject(:consume_with_sbmt_karafka) do
+    coordinator.increment
+    consumer.on_consume
+  end
+
+  let(:coordinator) {
+    instance = Karafka::Processing::Coordinator.new(test_topic, 0, instance_double(Karafka::TimeTrackers::Pause))
+    instance.instance_variable_set(:@seek_offset, -1)
+    instance
+  }
+  let(:test_consumer_group) { Karafka::Routing::ConsumerGroup.new(:test_group) }
+  let(:test_topic) { Karafka::Routing::Topic.new(:test_topic, test_consumer_group) }
+  let(:kafka_client) { instance_double(Karafka::Connection::Client) }
+  let(:null_deserializer) { Sbmt::KafkaConsumer::Serialization::NullDeserializer.new }
+
+  let(:consumer) {
+    build_consumer(described_class.new)
+  }
+
+  before {
+    Sbmt::KafkaConsumer::ClientConfigurer.configure!
+    allow(kafka_client).to receive(:assignment_lost?).and_return(false)
+    allow(kafka_client).to receive(:mark_as_consumed!).and_return(true)
+  }
+
+  def publish_to_sbmt_karafka(raw_payload, opts = {})
+    message = Karafka::Messages::Message.new(raw_payload, Karafka::Messages::Metadata.new(metadata_defaults.merge(opts)))
+    consumer.messages = Karafka::Messages::Messages.new(
+      [message],
+      Karafka::Messages::BatchMetadata.new(
+        topic: test_topic.name,
+        partition: 0,
+        processed_at: Time.zone.now,
+        created_at: Time.zone.now
+      )
+    )
+  end
+
+  # @return [Hash] message default options
+  def metadata_defaults
+    {
+      deserializer: null_deserializer,
+      headers: {},
+      key: nil,
+      offset: 0,
+      partition: 0,
+      received_at: Time.current,
+      topic: test_topic.name
+    }
+  end
+
+  def build_consumer(instance)
+    instance.coordinator = coordinator
+    instance.client = kafka_client
+    instance.singleton_class.include Karafka::Processing::Strategies::Default
+    instance
+  end
+end

data/lib/sbmt/kafka_consumer/testing.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+require "rspec"
+
+Dir["#{__dir__}/testing/shared_contexts/*.rb"].sort.each { |f| require f }

data/lib/sbmt/kafka_consumer/types.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module KafkaConsumer
+    module Types
+      include Dry.Types
+
+      ConfigAttrs = Dry::Types["hash"].constructor { |hsh| hsh.deep_symbolize_keys }
+
+      ConfigConsumer = Types.Constructor(Config::Consumer)
+      ConfigDeserializer = Types.Constructor(Config::Deserializer)
+      ConfigTopic = Types.Constructor(Config::Topic)
+    end
+  end
+end

data/lib/sbmt/kafka_consumer/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module KafkaConsumer
+    VERSION = "2.0.0"
+  end
+end

data/lib/sbmt/kafka_consumer/yabeda_configurer.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module KafkaConsumer
+    class YabedaConfigurer
+      SIZE_BUCKETS = [1, 10, 100, 1000, 10_000, 100_000, 1_000_000].freeze
+      LATENCY_BUCKETS = [0.0001, 0.001, 0.01, 0.1, 1.0, 10, 100, 1000].freeze
+      DELAY_BUCKETS = [1, 3, 10, 30, 100, 300, 1000, 3000, 10_000, 30_000].freeze
+      def self.configure
+        Yabeda.configure do
+          group :kafka_api do
+            counter :calls,
+              tags: %i[client broker api],
+              comment: "API calls"
+            histogram :latency,
+              tags: %i[client broker api],
+              buckets: LATENCY_BUCKETS,
+              comment: "API latency"
+            histogram :request_size,
+              tags: %i[client broker api],
+              buckets: SIZE_BUCKETS,
+              comment: "API request size"
+            histogram :response_size,
+              tags: %i[client broker api],
+              buckets: SIZE_BUCKETS,
+              comment: "API response size"
+            counter :errors,
+              tags: %i[client broker api],
+              comment: "API errors"
+          end
+
+          group :kafka_consumer do
+            counter :consumer_group_rebalances,
+              tags: %i[client group_id state],
+              comment: "Consumer group rebalances"
+
+            counter :process_messages,
+              tags: %i[client group_id topic partition],
+              comment: "Messages consumed"
+
+            counter :process_message_errors,
+              tags: %i[client group_id topic partition],
+              comment: "Messages failed to process"
+
+            histogram :process_message_latency,
+              tags: %i[client group_id topic partition],
+              buckets: LATENCY_BUCKETS,
+              comment: "Consumer latency"
+
+            gauge :offset_lag,
+              tags: %i[client group_id topic partition],
+              comment: "Consumer offset lag"
+
+            gauge :time_lag,
+              tags: %i[client group_id topic partition],
+              comment: "Consumer time lag"
+
+            counter :process_batch_errors,
+              tags: %i[client group_id topic partition],
+              comment: "Messages failed to process"
+
+            histogram :process_batch_latency,
+              tags: %i[client group_id topic partition],
+              buckets: LATENCY_BUCKETS,
+              comment: "Consumer batch latency"
+
+            histogram :batch_size,
+              tags: %i[client group_id topic partition],
+              buckets: SIZE_BUCKETS,
+              comment: "Consumer batch size"
+
+            counter :leave_group_errors,
+              tags: %i[client group_id],
+              comment: "Consumer group leave errors"
+
+            gauge :pause_duration,
+              tags: %i[client group_id topic partition],
+              comment: "Consumer pause duration"
+
+            counter :inbox_consumes,
+              tags: %i[
+                client group_id topic partition
+                inbox_name event_name status
+              ],
+              comment: "Inbox item consumes"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/kafka_consumer.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require "zeitwerk"
+require "karafka"
+require "active_record"
+require "yabeda"
+require "anyway_config"
+require "thor"
+require "dry/types"
+require "dry-struct"
+
+begin
+  require "sbmt/outbox"
+rescue LoadError
+  # sbmt-outbox is an optional dependency
+end
+
+require "anyway/rails" if defined?(Rails)
+require_relative "kafka_consumer/railtie" if defined?(Rails::Railtie)
+
+module Sbmt
+  module KafkaConsumer
+    class << self
+      delegate :monitor, to: Karafka
+
+      def logger
+        @logger ||= Rails.logger
+      end
+    end
+
+    class Error < StandardError; end
+
+    class SkipUndeserializableMessage < Error; end
+  end
+end
+
+loader = Zeitwerk::Loader.new
+# we need to set parent dir as gem autoloading root
+# see https://github.com/fxn/zeitwerk/issues/138#issuecomment-709640940 for details
+loader.push_dir(File.join(__dir__, ".."))
+loader.tag = "sbmt-kafka_consumer"
+
+# protobuf is an optional dependency
+loader.do_not_eager_load("#{__dir__}/kafka_consumer/serialization/protobuf_deserializer.rb")
+loader.do_not_eager_load("#{__dir__}/kafka_consumer/instrumentation/open_telemetry_loader.rb")
+loader.do_not_eager_load("#{__dir__}/kafka_consumer/instrumentation/open_telemetry_tracer.rb")
+loader.do_not_eager_load("#{__dir__}/kafka_consumer/instrumentation/sentry_tracer.rb")
+
+# completely ignore testing helpers
+# because testing.rb just requires some files and does not contain any constants (e.g. Testing) which Zeitwerk expects
+loader.ignore("#{__dir__}/kafka_consumer/testing.rb")
+loader.ignore("#{__dir__}/kafka_consumer/testing")
+loader.ignore("#{File.expand_path("../", __dir__)}/generators")
+
+loader.inflector.inflect("cli" => "CLI")
+loader.inflector.inflect("version" => "VERSION")
+
+loader.setup
+loader.eager_load

data/rubocop/rspec.yml

@@ -0,0 +1,29 @@
+RSpec/AnyInstance:
+  Enabled: false
+
+RSpec/MultipleExpectations:
+  Enabled: false
+
+RSpec/LetSetup:
+  Enabled: false
+
+RSpec/StubbedMock:
+  Enabled: false
+
+RSpec/MessageSpies:
+  Enabled: false
+
+RSpec/NestedGroups:
+  Enabled: false
+
+RSpec/EmptyExampleGroup:
+  Enabled: false
+
+RSpec/ExampleLength:
+  Enabled: false
+
+RSpec/MultipleMemoizedHelpers:
+  Enabled: false
+
+RSpec/VariableName:
+  Enabled: false

data/sbmt-kafka_consumer.gemspec

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require_relative "lib/sbmt/kafka_consumer/version"
+
+Gem::Specification.new do |spec|
+  spec.name = "sbmt-kafka_consumer"
+  spec.license = "MIT"
+  spec.version = Sbmt::KafkaConsumer::VERSION
+  spec.authors = ["Sbermarket Ruby-Platform Team"]
+
+  spec.summary = "Ruby gem for consuming Kafka messages"
+  spec.description = "This gem is used for consuming Kafka messages. It represents a wrapper over Karafka gem and is recommended for using as a transport with sbmt-outbox"
+  spec.homepage = "https://github.com/SberMarket-Tech/sbmt-kafka_consumer"
+  spec.required_ruby_version = ">= 2.7.0"
+
+  spec.metadata["allowed_push_host"] = "https://rubygems.org"
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = spec.homepage
+  spec.metadata["changelog_uri"] = "#{spec.homepage}/blob/master/CHANGELOG.md"
+  spec.metadata["rubygems_mfa_required"] = "false" # rubocop:disable Gemspec/RequireMFA
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(__dir__) do
+    `git ls-files -z`.split("\x0").reject do |f|
+      (f == __FILE__) || f.match(%r{\A(?:(?:bin|test|spec|features)/|\.(?:git|travis|circleci)|appveyor)})
+    end
+  end
+  spec.bindir = "exe"
+  spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "rails", ">= 5.2"
+  spec.add_dependency "zeitwerk", "~> 2.3"
+  spec.add_dependency "karafka", "~> 2.2"
+  spec.add_dependency "yabeda", ">= 0.11"
+  spec.add_dependency "anyway_config", ">= 2.4.0"
+  spec.add_dependency "thor"
+  spec.add_dependency "dry-struct"
+
+  spec.add_development_dependency "appraisal", ">= 2.4"
+  spec.add_development_dependency "bundler", ">= 2.1"
+  spec.add_development_dependency "combustion", ">= 1.3"
+  spec.add_development_dependency "rake", ">= 13.0"
+  spec.add_development_dependency "dry-monads", ">= 1.3"
+  spec.add_development_dependency "factory_bot_rails"
+  spec.add_development_dependency "pg"
+  spec.add_development_dependency "google-protobuf"
+  spec.add_development_dependency "sentry-rails", ">= 5.2"
+  spec.add_development_dependency "opentelemetry-sdk"
+  spec.add_development_dependency "opentelemetry-api", ">= 0.17"
+  spec.add_development_dependency "opentelemetry-common", ">= 0.17"
+  spec.add_development_dependency "opentelemetry-instrumentation-base", ">= 0.17"
+  spec.add_development_dependency "rspec", ">= 3.0"
+  spec.add_development_dependency "rspec_junit_formatter", ">= 0.6"
+  spec.add_development_dependency "rspec-rails", ">= 4.0"
+  spec.add_development_dependency "rubocop-rails", ">= 2.5"
+  spec.add_development_dependency "rubocop-rspec", ">= 2.11"
+  spec.add_development_dependency "sbmt-outbox", ">= 5.0"
+  spec.add_development_dependency "simplecov", ">= 0.16"
+  spec.add_development_dependency "standard", ">= 1.12"
+
+  # let metrics and probes work in dev-mode with combustion
+  # e.g. RAILS_ENV=development bundle exec kafka_consumer
+  spec.add_development_dependency "yabeda-prometheus-mmap"
+  spec.add_development_dependency "webrick"
+  spec.add_development_dependency "rack"
+  spec.add_development_dependency "http_health_check"
+end