sbmt-kafka_consumer 2.0.0

data/dip.yml

@@ -0,0 +1,84 @@
+version: '7'
+
+environment:
+  RUBY_VERSION: '3.2'
+
+compose:
+  files:
+    - docker-compose.yml
+
+interaction:
+  bash:
+    description: Open the Bash shell in app's container
+    service: ruby
+    command: /bin/bash
+
+  bundle:
+    description: Run Bundler commands
+    service: ruby
+    command: bundle
+
+  rails:
+    description: Run RoR commands
+    service: ruby
+    command: bundle exec rails
+
+  appraisal:
+    description: Run Appraisal commands
+    service: ruby
+    command: bundle exec appraisal
+
+  rspec:
+    description: Run Rspec commands
+    service: ruby
+    command: bundle exec rspec
+    subcommands:
+      all:
+        command: bundle exec appraisal rspec
+      rails-6.0:
+        command: bundle exec appraisal rails-6.0 rspec
+      rails-6.1:
+        command: bundle exec appraisal rails-6.1 rspec
+      rails-7.0:
+        command: bundle exec appraisal rails-7.0 rspec
+      rails-7.1:
+        command: bundle exec appraisal rails-7.1 rspec
+
+  rubocop:
+    description: Run Ruby linter
+    service: ruby
+    command: bundle exec rubocop
+
+  setup:
+    description: Install deps
+    service: ruby
+    command: bin/setup
+
+  test:
+    description: Run linters, run all tests
+    service: ruby
+    command: bin/test
+
+  kafka-consumer:
+    description: Run kafka consumer
+    service: ruby
+    command: bundle exec kafka_consumer
+
+  kafka-producer:
+    description: Run kafka producer commands
+    service: kafka
+    command: kafka-console-producer.sh --bootstrap-server kafka:9092
+    subcommands:
+      inbox:
+        command: kafka-console-producer.sh --bootstrap-server kafka:9092 --topic topic_with_inbox_items
+      json:
+        command: kafka-console-producer.sh --bootstrap-server kafka:9092 --topic topic_with_json_data
+      protobuf:
+        command: kafka-console-producer.sh --bootstrap-server kafka:9092 --topic topic_with_protobuf_data
+
+provision:
+  - dip compose down --volumes
+  - cp -f lefthook-local.dip_example.yml lefthook-local.yml
+  - rm -f Gemfile.lock
+  - rm -f gemfiles/*gemfile*
+  - dip setup

data/docker-compose.yml

@@ -0,0 +1,68 @@
+services:
+  ruby:
+    image: ruby:${RUBY_VERSION:-3.2}
+    environment:
+      HISTFILE: /app/tmp/.bash_history
+      BUNDLE_PATH: /usr/local/bundle
+      BUNDLE_CONFIG: /app/.bundle/config
+      DATABASE_URL: postgres://postgres:@postgres:5432
+      KAFKAFILE: spec/internal/Kafkafile
+    depends_on:
+      kafka:
+        condition: service_started
+      postgres:
+        condition: service_started
+    command: bash
+    working_dir: /app
+    volumes:
+      - .:/app:cached
+      - bundler_data:/usr/local/bundle
+
+  postgres:
+    image: postgres:13
+    environment:
+      POSTGRES_HOST_AUTH_METHOD: trust
+    ports:
+      - 5432
+    healthcheck:
+      test: pg_isready -U postgres -h 127.0.0.1
+      interval: 10s
+
+  kafka:
+    image: bitnami/kafka:2.7.0
+    ports:
+      - '9092:9092'
+    environment:
+      - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
+      - ALLOW_PLAINTEXT_LISTENER=yes
+      - KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true
+      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CLIENT:PLAINTEXT,INTERNAL:PLAINTEXT
+      - KAFKA_CFG_LISTENERS=CLIENT://:9092,INTERNAL://:9091
+      - KAFKA_CFG_ADVERTISED_LISTENERS=CLIENT://kafka:9092,INTERNAL://kafka:9091
+      - KAFKA_INTER_BROKER_LISTENER_NAME=INTERNAL
+    depends_on:
+      - zookeeper
+    healthcheck:
+      # we don't have `nc` installed in kafka image :(
+      test:
+        - CMD-SHELL
+        - echo 'exit' | curl --silent -f telnet://0.0.0.0:9092
+      interval: 15s
+      timeout: 5s
+      retries: 15
+
+  zookeeper:
+    image: bitnami/zookeeper:3.5
+    ports:
+      - '2181:2181'
+    environment:
+      - ALLOW_ANONYMOUS_LOGIN=yes
+    healthcheck:
+      test: ["CMD-SHELL", "echo ruok | nc localhost 2181"]
+      interval: 2s
+      timeout: 2s
+      retries: 15
+
+volumes:
+  bundler_data:
+  kafka:

data/exe/kafka_consumer

@@ -0,0 +1,16 @@
+#!/usr/local/bin/ruby
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "sbmt/kafka_consumer"
+
+# rubocop:disable Lint/RescueException
+begin
+  Sbmt::KafkaConsumer::CLI.start(ARGV)
+rescue Exception => e
+  warn "KafkaConsumer exited with error"
+  warn(e.message) if e.respond_to?(:message)
+  warn(e.backtrace.join("\n")) if e.respond_to?(:backtrace) && e.backtrace.respond_to?(:join)
+  exit 1
+end
+# rubocop:enable Lint/RescueException

data/lefthook-local.dip_example.yml

@@ -0,0 +1,4 @@
+pre-commit:
+  commands:
+    rubocop:
+      run: dip {cmd}

data/lefthook.yml

@@ -0,0 +1,6 @@
+pre-commit:
+  commands:
+    rubocop:
+      tags: backend
+      glob: "{*.rb,**/*.rb,Gemfile,Rakefile}"
+      run: bundle exec rubocop -A --force-exclusion {staged_files} && git add {staged_files}

data/lib/generators/kafka_consumer/concerns/configuration.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module KafkaConsumer
+  module Generators
+    module Concerns
+      module Configuration
+        extend ActiveSupport::Concern
+
+        CONFIG_PATH = "config/kafka_consumer.yml"
+
+        def check_config_file!
+          config_path = File.expand_path(CONFIG_PATH)
+          return if File.exist?(config_path)
+
+          generate = ask "The file #{config_path} does not appear to exist. " \
+                       "Would you like to generate it? [Yn]"
+
+          generator_name = "kafka_consumer:install"
+          if (generate.presence || "y").casecmp("y").zero?
+            generate generator_name
+          else
+            raise Rails::Generators::Error, "Please generate #{config_path} " \
+                           "by running `bin/rails g #{generator_name}` " \
+                           "or add this file manually."
+          end
+        end
+      end
+    end
+  end
+end

data/lib/generators/kafka_consumer/consumer/USAGE

@@ -0,0 +1,24 @@
+Description:
+    Stubs out a new non-inbox consumer. Pass the consumer name, either
+     CamelCased or under_scored.
+
+Example:
+    bin/rails generate kafka_consumer:consumer Test
+
+    This will create:
+        app/consumers/test_consumer.rb
+
+    This will optionally insert:
+        'group_key':
+          name: <%= ENV.fetch('ENV_VARIABLE_WITH_GROUP_NAME'){ 'group.name' } %><%= ENV.fetch('ENV_VARIABLE_WITH_GROUP_SUFFIX'){ '' } %>
+          topics:
+            - name: 'topic.name'
+              consumer:
+                klass: "TestConsumer"
+#                init_attrs:
+#                  skip_on_error: false # This is the default value
+              deserializer:
+                klass: "Sbmt::KafkaConsumer::Serialization::ProtobufDeserializer"
+                init_attrs:
+                  message_decoder_klass: "YourMessageDecoderClassName"
+#                  skip_decoding_error: false # This is the default value

data/lib/generators/kafka_consumer/consumer/consumer_generator.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require "rails/generators/named_base"
+require "generators/kafka_consumer/concerns/configuration"
+
+module KafkaConsumer
+  module Generators
+    class ConsumerGenerator < Rails::Generators::NamedBase
+      include Concerns::Configuration
+
+      source_root File.expand_path("templates", __dir__)
+
+      def insert_consumer_class
+        @consumer_name = "#{name.classify}Consumer"
+        template "consumer.rb.erb", "app/consumers/#{file_path}_consumer.rb"
+      end
+
+      def configure_consumer_group
+        @group_key = ask "Would you also configure a consumer group?" \
+                    " Type the group's key (e.g. my_consumer_group) or press Enter to skip this action"
+        return if @group_key.blank?
+
+        check_config_file!
+
+        @group_name = ask "Type the group's name (e.g. my.consumer.group)"
+        @topic = ask "Type the group topic's name"
+        insert_into_file CONFIG_PATH, group_template.result(binding), after: "consumer_groups:\n"
+      end
+
+      private
+
+      def group_template_path
+        File.join(ConsumerGenerator.source_root, "consumer_group.yml.erb")
+      end
+
+      def group_template
+        ERB.new(File.read(group_template_path), trim_mode: "%-")
+      end
+    end
+  end
+end

data/lib/generators/kafka_consumer/consumer/templates/consumer.rb.erb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+<%- module_namespacing do -%>
+class <%= @consumer_name %> < Sbmt::KafkaConsumer::BaseConsumer
+  def process_message(_message)
+    # Add message processing here
+  end
+end
+<%- end -%>

data/lib/generators/kafka_consumer/consumer/templates/consumer_group.yml.erb

@@ -0,0 +1,13 @@
+    '<%= @group_key %>':
+      name: <%%= ENV.fetch('ENV_VARIABLE_WITH_GROUP_NAME'){ "<%= @group_name %>" } %><%%= ENV.fetch('ENV_VARIABLE_WITH_GROUP_SUFFIX'){ "" } %>
+      topics:
+        - name: "<%= @topic.presence || "insert-your-topic-name-here" %>"
+          consumer:
+            klass: "<%= @consumer_name %>"
+#            init_attrs:
+#              skip_on_error: false # This is the default value
+          deserializer:
+            klass: "Sbmt::KafkaConsumer::Serialization::ProtobufDeserializer"
+            init_attrs:
+              message_decoder_klass: "YourMessageDecoderClassName"
+#              skip_decoding_error: false # This is the default value

data/lib/generators/kafka_consumer/inbox_consumer/USAGE

@@ -0,0 +1,22 @@
+Description:
+    Inserts a consumer group's default configuration.
+    It accepts a group key, a group name and an optional array of topics as arguments.
+
+
+Example:
+    bin/rails generate kafka_consumer:inbox_consumer group_key group.name topic.name
+
+    This will insert:
+        'group_key':
+          name: <%= ENV.fetch('ENV_VARIABLE_WITH_GROUP_NAME'){ 'group.name' } %><%= ENV.fetch('ENV_VARIABLE_WITH_GROUP_SUFFIX'){ '' } %>
+          topics:
+            - name: 'topic.name'
+              consumer:
+                # Change the line below to the desired consumer
+                # if InboxConsumer doesn't suit your needs
+                klass: "Sbmt::KafkaConsumer::InboxConsumer"
+                init_attrs:
+                  name: "test_items"
+                  inbox_item: "SomeModelInboxItem" # Change this to your item class name
+    #          deserializer: # This deserializer is used by default
+    #            klass: "Sbmt::KafkaConsumer::Serialization::NullDeserializer"

data/lib/generators/kafka_consumer/inbox_consumer/inbox_consumer_generator.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require "rails/generators/named_base"
+require "generators/kafka_consumer/concerns/configuration"
+
+module KafkaConsumer
+  module Generators
+    class InboxConsumerGenerator < Rails::Generators::NamedBase
+      include Concerns::Configuration
+
+      source_root File.expand_path("templates", __dir__)
+
+      argument :group_name, type: :string, banner: "group.name"
+      argument :topics, type: :array, default: [], banner: "topic topic"
+
+      def process_topics
+        check_config_file!
+
+        @items = {}
+        topics.each do |topic|
+          inbox_item = ask "Would you also add an InboxItem class for topic '#{topic}'?" \
+                           " Type item's name in the form of SomeModel::InboxItem or press Enter" \
+                           " to skip creating item's class"
+          @items[topic] = if inbox_item.blank?
+            nil
+          else
+            generate "outbox:item", inbox_item, "--kind inbox"
+            inbox_item.classify
+          end
+        end
+      end
+
+      def insert_consumer_group
+        insert_into_file CONFIG_PATH, group_template.result(binding), after: "consumer_groups:\n"
+      end
+
+      private
+
+      def group_template_path
+        File.join(InboxConsumerGenerator.source_root, "consumer_group.yml.erb")
+      end
+
+      def group_template
+        ERB.new(File.read(group_template_path), trim_mode: "%-")
+      end
+    end
+  end
+end

data/lib/generators/kafka_consumer/inbox_consumer/templates/consumer_group.yml.erb

@@ -0,0 +1,22 @@
+    '<%= file_name %>':
+      name: <%%= ENV.fetch('ENV_VARIABLE_WITH_GROUP_NAME'){ '<%= group_name %>' } %><%%= ENV.fetch('CONSUMER_GROUP_SUFFIX'){ '' } %>
+<%- if @items.empty? -%>
+      topics: []
+<%- else -%>
+      topics:
+      <%- @items.each do |topic, item_name| -%>
+        <%- next if topic.blank? -%>
+        <%- inbox_item = item_name.presence || "YourModelName::InboxItem" -%>
+        <%- consumer_name = inbox_item.split('::').first.presence || "#{topic}_item" -%>
+        - name: "<%= topic %>"
+          consumer:
+            # Change the line below to the desired consumer
+            # if InboxConsumer doesn't suit your needs
+            klass: "Sbmt::KafkaConsumer::InboxConsumer"
+            init_attrs:
+              name: "<%= consumer_name.underscore.pluralize %>"
+              inbox_item: "<%= inbox_item %>"
+#          deserializer: # This deserializer is used by default
+#            klass: "Sbmt::KafkaConsumer::Serialization::NullDeserializer"
+      <%- end -%>
+<%- end -%>

data/lib/generators/kafka_consumer/install/USAGE

@@ -0,0 +1,9 @@
+Description:
+    Generates the Kafka consumer's initial setup
+
+Example:
+    bin/rails generate kafka_consumer:install
+
+    This will create:
+        Kafkafile
+        config/kafka_consumer.yml

data/lib/generators/kafka_consumer/install/install_generator.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require "rails/generators/base"
+require "generators/kafka_consumer/concerns/configuration"
+
+module KafkaConsumer
+  module Generators
+    class InstallGenerator < Rails::Generators::Base
+      include Concerns::Configuration
+
+      source_root File.expand_path("templates", __dir__)
+
+      def create_kafkafile
+        copy_file "Kafkafile", "./Kafkafile"
+      end
+
+      def create_kafka_consumer_yml
+        copy_file "kafka_consumer.yml", CONFIG_PATH
+      end
+    end
+  end
+end

data/lib/generators/kafka_consumer/install/templates/Kafkafile

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+require_relative "config/environment"

data/lib/generators/kafka_consumer/install/templates/kafka_consumer.yml

@@ -0,0 +1,59 @@
+default: &default
+  client_id: 'some-name'
+  max_wait_time: 1
+  shutdown_timeout: 60
+  concurrency: 4
+  pause_timeout: 1
+  pause_max_timeout: 30
+  pause_with_exponential_backoff: true
+  auth:
+    kind: plaintext
+  kafka:
+    servers: "kafka:9092"
+    heartbeat_timeout: 5
+    session_timeout: 30
+    reconnect_timeout: 3
+    connect_timeout: 5
+    socket_timeout: 30
+    kafka_options:
+      allow.auto.create.topics: true
+  consumer_groups:
+#    group_ref_id_1:
+#      name: cg_with_single_topic
+#      topics:
+#        - name: topic_with_inbox_items
+#          consumer:
+#            klass: "Sbmt::KafkaConsumer::InboxConsumer"
+#            init_attrs:
+#              name: "test_items"
+#              inbox_item: "TestInboxItem"
+#          deserializer:
+#            klass: "Sbmt::KafkaConsumer::Serialization::NullDeserializer"
+#    group_ref_id_2:
+#      name: cg_with_multiple_topics
+#      topics:
+#        - name: topic_with_json_data
+#          consumer:
+#            klass: "Sbmt::KafkaConsumer::SimpleLoggingConsumer"
+#          deserializer:
+#            klass: "Sbmt::KafkaConsumer::Serialization::JsonDeserializer"
+#        - name: topic_with_protobuf_data
+#          consumer:
+#            klass: "Sbmt::KafkaConsumer::SimpleLoggingConsumer"
+#          deserializer:
+#            klass: "Sbmt::KafkaConsumer::Serialization::ProtobufDeserializer"
+#            init_attrs:
+#              message_decoder_klass: "Sso::UserRegistration"
+#              skip_decoding_error: true
+  probes:
+    port: 9394
+
+development:
+  <<: *default
+test:
+  <<: *default
+  deliver: false
+staging: &staging
+  <<: *default
+production:
+  <<: *staging

data/lib/sbmt/kafka_consumer/app_initializer.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module KafkaConsumer
+    module AppInitializer
+      extend self
+
+      def initialize!
+        ClientConfigurer.configure!
+      end
+    end
+  end
+end

data/lib/sbmt/kafka_consumer/base_consumer.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module KafkaConsumer
+    class BaseConsumer < Karafka::BaseConsumer
+      attr_reader :trace_id
+
+      def self.consumer_klass(skip_on_error: false)
+        Class.new(self) do
+          const_set(:SKIP_ON_ERROR, skip_on_error)
+
+          def self.name
+            superclass.name
+          end
+        end
+      end
+
+      def consume
+        ::Rails.application.executor.wrap do
+          messages.each do |message|
+            with_instrumentation(message) { do_consume(message) }
+          end
+        end
+      end
+
+      private
+
+      def with_instrumentation(message)
+        @trace_id = SecureRandom.base58
+
+        logger.tagged(
+          trace_id: trace_id,
+          topic: message.metadata.topic, partition: message.metadata.partition,
+          key: message.metadata.key, offset: message.metadata.offset
+        ) do
+          ::Sbmt::KafkaConsumer.monitor.instrument(
+            "consumer.consumed_one",
+            caller: self, message: message, trace_id: trace_id
+          ) do
+            do_consume(message)
+          rescue SkipUndeserializableMessage => ex
+            instrument_error(ex, message)
+            logger.warn("skipping undeserializable message: #{ex.message}")
+          rescue => ex
+            instrument_error(ex, message)
+
+            if skip_on_error
+              logger.warn("skipping unprocessable message: #{ex.message}, message: #{message_payload(message).inspect}")
+            else
+              raise ex
+            end
+          end
+        end
+      end
+
+      def do_consume(message)
+        log_message(message) if log_payload?
+
+        # deserialization process is lazy (and cached)
+        # so we trigger it explicitly to catch undeserializable message early
+        message.payload
+
+        process_message(message)
+
+        mark_as_consumed!(message)
+      end
+
+      def skip_on_error
+        self.class::SKIP_ON_ERROR
+      end
+
+      # can be overridden in consumer to enable message logging
+      def log_payload?
+        false
+      end
+
+      def logger
+        ::Sbmt::KafkaConsumer.logger
+      end
+
+      def process_message(_message)
+        raise NotImplementedError, "Implement this in a subclass"
+      end
+
+      def log_message(message)
+        logger.info("#{message_payload(message).inspect}, message_key: #{message.metadata.key}, message_headers: #{message.metadata.headers}")
+      end
+
+      def instrument_error(error, message)
+        ::Sbmt::KafkaConsumer.monitor.instrument(
+          "error.occurred",
+          error: error,
+          caller: self,
+          message: message,
+          type: "consumer.base.consume_one"
+        )
+      end
+
+      def message_payload(message)
+        message.payload || message.raw_payload
+      end
+    end
+  end
+end

data/lib/sbmt/kafka_consumer/cli.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module KafkaConsumer
+    class CLI < Thor
+      def self.exit_on_failure?
+        true
+      end
+
+      default_command :start
+
+      desc "start", "Start kafka_consumer worker"
+      option :consumer_group_id,
+        aliases: "-g",
+        desc: "Consumer group id to start",
+        repeatable: true
+      option :concurrency,
+        aliases: "-c",
+        type: :numeric,
+        default: 5,
+        desc: "Number of threads, overrides global kafka.concurrency config"
+      def start
+        $stdout.puts "Initializing KafkaConsumer"
+        $stdout.puts "Version: #{VERSION}"
+
+        load_environment
+
+        $stdout.sync = true
+
+        $stdout.puts "Configuring client"
+        ClientConfigurer.configure!(
+          consumer_groups: options[:consumer_group_id],
+          concurrency: options[:concurrency]
+        )
+        $stdout.puts "Client configured routes: #{ClientConfigurer.routes.inspect}"
+
+        $stdout.puts "Starting probes/metrics http-server"
+        Sbmt::KafkaConsumer::Probes::Host.run_async
+
+        Sbmt::KafkaConsumer::Server.run
+      end
+
+      private
+
+      def load_environment
+        env_file_path = ENV["KAFKAFILE"] || "#{Dir.pwd}/Kafkafile"
+
+        if File.exist?(env_file_path)
+          $stdout.puts "Loading env from Kafkafile: #{env_file_path}"
+          load(env_file_path)
+        end
+      end
+    end
+  end
+end