rimless 2.9.0 → 3.0.0

data/lib/rimless/consumer/app.rb

@@ -0,0 +1,182 @@
+# frozen_string_literal: true
+
+module Rimless
+  module Consumer
+    # The consumer application which adds some convenience helpers and
+    # library-related configurations.
+    class App < Karafka::App
+      # Allow accessing the class-level configuration methods from our instance
+      delegate :setup, :routes, :config, to: self
+
+      # Creates a new Rimless/Karafka application instance while configuring
+      # our library defaults.
+      #
+      # @return [Rimless::Consumer::App] the configured consumer application
+      #
+      # rubocop:disable Metrics/MethodLength -- because of the Karafka
+      #   configuration
+      def initialize
+        # Run the parent class initialization
+        super
+
+        setup do |config|
+          # See: https://bit.ly/3OtIfeu (+config.kafka+ settings)
+
+          # An optional identifier of a Kafka consumer (in a consumer group)
+          # that is passed to a Kafka broker with every request. A logical
+          # application name to be included in Kafka logs and monitoring
+          # aggregates.
+          config.kafka[:'client.id'] = Rimless.configuration.client_id
+          # All the known brokers, at least one. The ruby-kafka driver will
+          # discover the whole cluster structure once and when issues occur
+          # to dynamically adjust scaling operations.
+          config.kafka[:'bootstrap.servers'] =
+            Rimless.configuration.kafka_brokers
+          # All brokers MUST acknowledge a new message by default
+          config.kafka[:'request.required.acks'] = -1
+
+          # See: https://bit.ly/3MAF6Jk (+config.*+ settings)
+
+          # Used to uniquely identify given client instance - for logging only
+          config.client_id = [
+            Rimless.configuration.client_id,
+            Process.pid,
+            Socket.gethostname
+          ].join('-')
+
+          # Should be unique per application to properly track message
+          # consumption. See: Kafka Consumer Groups.
+          config.group_id = Rimless.configuration.client_id
+
+          # We use dots (parts separation) and underscores for topic names, by
+          # convention.
+          config.strict_topics_namespacing = false
+
+          # Number of milliseconds after which Karafka no longer waits for the
+          # consumers to stop gracefully but instead we force terminate
+          # everything.
+          config.shutdown_timeout = 10.seconds.in_milliseconds
+
+          # Recreate consumers with each batch. This will allow Rails code
+          # reload to work in the development mode. Otherwise Karafka process
+          # would not be aware of code changes.
+          config.consumer_persistence = Rimless.env.production?
+
+          # Use our logger instead
+          config.logger = Rimless.logger
+        end
+
+        # Add the logging listener to Karafka in order to facilitate our gem
+        # logger. When the user configuration results in an falsy value (eg.
+        # +nil+ or +false+), we skip it.
+        listener = Rimless.configuration.consumer_logger_listener
+        Karafka.monitor.subscribe(listener) if listener
+
+        # Configure some routing defaults
+        routes.draw do
+          defaults do
+            deserializers(
+              payload: Rimless.configuration.avro_deserializer_class.new
+            )
+          end
+        end
+
+        # Call the user-configurable block with our configuration
+        # for customizations
+        setup(&Rimless.configuration.consumer_configure)
+      end
+      # rubocop:enable Metrics/MethodLength
+
+      # Configure the topics-consumer routing table in a lean way.
+      #
+      # Examples:
+      #
+      #   topics({ app: :test_app, name: :admins } => YourConsumer)
+      #   topics({ app: :test_app, names: %i[users admins] } => YourConsumer)
+      #
+      # Examples:
+      #
+      #   topics(
+      #     { app: :test_app, name: :admins } => lambda { |topic|
+      #       consumer Rimless::Consumer::JobBridge.build(dest_consumer)
+      #     }
+      #   )
+      #
+      # Examples:
+      #
+      #   topics do
+      #     topic('name') do
+      #       consumer CustomConsumer
+      #     end
+      #   end
+      #
+      # @param topics [Hash{Hash => Class, Proc}] the topic to consumer mapping
+      # @yield the given block on the routing table
+      # @return [Rimless::Consumer::App] the application instance for chaining
+      def topics(topics = [], &block)
+        routes.draw do
+          consumer_group(Rimless.configuration.client_id) do
+            instance_exec(&block) if block_given?
+
+            topics.each do |topic_parts, dest_consumer|
+              Rimless.consumer.topic_names(topic_parts).each do |topic_name|
+                configure = proc do
+                  consumer(
+                    Rimless.configuration.job_bridge_class.build(dest_consumer)
+                  )
+                  deserializers(
+                    payload: Rimless.configuration.avro_deserializer_class.new
+                  )
+                end
+                configure = dest_consumer if dest_consumer.is_a? Proc
+                topic(topic_name, &configure)
+              end
+            end
+          end
+        end
+
+        self
+      end
+
+      # Build the conventional Apache Kafka topic names from the given parts.
+      # This allows various forms like single strings/symbols and a hash in the
+      # form of +{ app: [String, Symbol], name: [String, Symbol], names:
+      # [Array<String, Symbol>] }+. This allows the maximum of flexibility.
+      #
+      # @param parts [String, Symbol, Hash{Symbol => Mixed}] the topic
+      #   name parts
+      # @return [Array<String>] the full topic names
+      def topic_names(parts)
+        # We have a single app, but multiple names so we handle them
+        if parts.is_a?(Hash) && parts.key?(:names)
+          return parts[:names].map do |name|
+            Rimless.topic(parts.merge(name: name))
+          end
+        end
+
+        # We cannot handle the given input
+        [Rimless.topic(parts)]
+      end
+
+      # Allows the user to re-configure the Karafka application if this is
+      # needed. (eg. to set some kafka driver settings, etc)
+      #
+      # @yield [Karafka::Setup::ConfigProxy] the given block to allow
+      #   configuration manipulation
+      # @return [Rimless::Consumer::App] our self for chaining
+      def configure(&)
+        setup(&)
+        self
+      end
+
+      # Check if we run as the Karafka server (consumer) process or not.
+      # Unfortunately Karafka still does not offer a solution for it like
+      # +Sidekiq.server?+. (Last tested with Karafka version +2.5.7+)
+      #
+      # @return [Boolean] whenever we run as the Karafka server or not
+      def server?
+        $PROGRAM_NAME.end_with?('karafka') && ARGV.include?('server')
+      end
+    end
+  end
+end

data/lib/rimless/{karafka → consumer}/avro_deserializer.rb

@@ -1,24 +1,26 @@
 # frozen_string_literal: true
 
 module Rimless
-  module Karafka
+  module Consumer
     # A custom Apache Avro compatible message deserializer.
     class AvroDeserializer
       # Deserialize an Apache Avro encoded Apache Kafka message.
       #
-      # @param params [Karafka::Params::Params] the Karafka message parameters
-      # @return [Hash{Symbol => Mixed}] the deserialized Apache Avro message
-      def call(params)
+      # @param message [Karafka::Messages::Message] the Karafka message to
+      #   deserialize
+      # @return [Hash{Symbol => Mixed}, nil] the deserialized Apache Avro
+      #   message, or +nil+ when we received a tombstone message
+      def call(message)
         # When the Kafka message does not have a payload, we won't fail.
         # This is for Kafka users which use log compaction with a nil payload.
-        return if params.raw_payload.nil?
+        return if message.raw_payload.nil?
 
         # We use sparsed hashes inside of Apache Avro messages for schema-less
         # blobs of data, such as loosely structured metadata blobs.  That's a
         # somewhat bad idea on strictly typed and defined messages, but their
         # occurrence should be rare.
         Rimless
-          .decode(params.raw_payload)
+          .decode(message.raw_payload)
           .then { |data| Sparsify(data, sparse_array: true) }
           .then { |data| data.transform_keys { |key| key.delete('\\') } }
           .then { |data| Unsparsify(data, sparse_array: true) }

data/lib/rimless/consumer/base.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+module Rimless
+  module Consumer
+    # The base consumer where all Apache Kafka messages will be processed,
+    # within an ActiveJob job. It comes with some simple conventions to keep
+    # the actual application code simple to use. Example usage on an
+    # application:
+    #
+    #   app/consumers/my_consumer.rb
+    #
+    #   class IdentityApiConsumer < ApplicationConsumer
+    #     # Handle +identity-api.users/user_locked+ messages.
+    #     #
+    #     # @param user [Hash{Symbol => Mixed}] the event user data
+    #     # @param args [Hash{Symbol => Mixed}] additional event data
+    #     def user_locked(user:, **args)
+    #       # ..
+    #     end
+    #   end
+    #
+    # Despite its default usage within an ActiveJob context, it still directly
+    # usable by Karafka. Just be warned that, when running inside a ActiveJob
+    # context, it lack support for various Karafka internals (eg. coordinator,
+    # client, etc).
+    class Base < Karafka::BaseConsumer
+      # Allow to handle a single message, each at a time
+      attr_accessor :message
+
+      # Allow older clients to access the current message as +params+, and the
+      # current messages batch as +params_batch+ (Karafka 1.4 style)
+      alias params message
+      alias params_batch messages
+
+      # A structure for Karafka::BaseConsumer#coodinator, within job contexts.
+      #
+      # rubocop:disable Lint/StructNewOverride -- because +:partition+ is
+      #   expected to be overwritten
+      JobContextCoordinator = Struct.new(:topic, :partition)
+      # rubocop:enable Lint/StructNewOverride
+
+      # Build a new disposable consumer instance for a single Apache Kafka
+      # message, which should be processed within the ActiveJob context.
+      #
+      # @param payload [Mixed] the (already) decoded Kafka message payload
+      # @param metadata [Hash] the Kafka message metadata (string/symbol
+      #   keys are allowed)
+      # @return [Rimless::Consumer::Base] the job context-aware consumer
+      #   instance
+      def self.build_for_job(payload:, metadata:)
+        new.tap do |consumer|
+          metadata = metadata.symbolize_keys
+
+          consumer.coordinator = JobContextCoordinator.new(
+            topic: metadata[:topic],
+            partition: metadata[:partition]
+          )
+          consumer.producer = Rimless.producer
+
+          metadata = Karafka::Messages::Metadata.new(
+            **metadata.except(:key, :headers),
+            raw_key: metadata[:key],
+            raw_headers: metadata[:headers],
+            deserializers: job_deserializers
+          )
+          consumer.messages =
+            [Karafka::Messages::Message.new(payload, metadata)]
+        end
+      end
+
+      # A custom set of Karafka deserializers, exclusive for the ActiveJob
+      # context. As we already get the deserialized details (payload, message
+      # key, message headers), we just want to pass the values through.
+      #
+      # @return [Karafka::Routing::Features::Deserializers::Config] the
+      #   deserializers config object
+      def self.job_deserializers
+        @job_deserializers ||=
+          Karafka::Routing::Features::Deserializers::Config.new(
+            active: true,
+            payload: ->(message) { message.raw_payload },
+            key: Karafka::Deserializers::Key.new,
+            headers: Karafka::Deserializers::Headers.new
+          )
+      end
+
+      # A generic message consuming handler which resolves the message event
+      # name to an actual method. All message data (top-level keys) is passed
+      # down to the event method as symbol arguments.
+      def consume
+        messages.each do |message|
+          self.message = message
+
+          # We ignore events we do not handle by definition
+          send(event, **arguments) if !event.nil? && respond_to?(event)
+        end
+      end
+
+      # Prepare the message payload as event method arguments.
+      #
+      # @return [Hash{Symbol => Mixed}] the event method arguments
+      def arguments
+        event_name_key = :event
+        event_name_key = 'event' if message.payload.key? 'event'
+        message.payload.except(event_name_key)
+      end
+
+      # A shortcut to fetch the event name from the Kafka message.
+      #
+      # @return [Symbol] the event name of the current message
+      def event
+        event_name = message.payload[:event]
+        event_name ||= message.payload['event']
+        event_name&.to_sym
+      end
+    end
+  end
+end

data/lib/rimless/consumer/job.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Rimless
+  module Consumer
+    # A simple consumer job, enqueued by the job bridge, after a message was
+    # consumed from an Apache Kafka topic.
+    class Job < ActiveJob::Base
+      # Configure the default job queue
+      queue_as Rimless.configuration.consumer_job_queue
+
+      # Receive a single message/event from the Karafka process, consuming it
+      # from a Apache Kafka topic. Within the context we "simulate" a Karafka
+      # consumer context and run the configured consumer class (a user
+      # application class, from +app/consumers/+) with the single message.
+      #
+      # The Karafka consumer context is just "simulated", as it does not
+      # feature all components accessible by a regular Karafka consumer
+      # context. This includes access to the real +coordinator+, or +client+.
+      # But access to an +producer+ is provided. Check the
+      # Rimless::Consumer::Base for more details.
+      #
+      # @param payload [Mixed] the (already) decoded Kafka message payload
+      # @param consumer [String] the consumer class name to use
+      # @param metadata [Hash] the Kafka message metadata (string/symbol
+      #   keys are allowed)
+      def perform(payload:, consumer:, metadata:)
+        # Try to lookup the given consumer and create a new instance for it,
+        # which is configured for the job context we're running in
+        consumer = consumer.constantize.build_for_job(payload:, metadata:)
+        # Run the actual consumer logic
+        consumer.consume
+      end
+    end
+  end
+end

data/lib/rimless/consumer/job_bridge.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+module Rimless
+  module Consumer
+    # This is our default consumer for Karafka, which wraps the actual user
+    # consumer classes, consumes all message of a batch and enqueues them as
+    # ActiveJob job. It builds a bridge between the Karafka topic/message
+    # consumer process and a later running ActiveJob processor (eg. Sidekiq or
+    # Solid Queue).
+    class JobBridge < Karafka::BaseConsumer
+      # Persist the wrapped consumer, to pass it later while enqueuing jobs
+      class_attribute :consumer
+
+      class << self
+        # Build a new anonymous wrapper class, based on the given destination
+        # consumer class.
+        #
+        # @param consumer [Class] the consumer to pass down to the jobs
+        # @return [Class] the new and configured wrapper class
+        def build(consumer)
+          # We cannot serialize anonymous classes, as they need to cross
+          # process borders via ActiveJob here, and the resulting job needs to
+          # constantize the serialized class name again
+          raise ArgumentError, "Anonymous consumer class passed: #{consumer}" \
+            unless consumer.name
+
+          Class.new(self).tap do |wrapper|
+            wrapper.consumer = consumer.name
+          end
+        end
+
+        # A custom object/class inspection helper to allow pretty printing of
+        # the anonymous class.
+        #
+        # @return [String] the pretty-printed class/instance
+        def inspect
+          # When not an anonymous class
+          return super unless name.nil?
+
+          # Otherwise the anonymous wrapper class
+          "#{Rimless::Consumer::JobBridge.name}[consumer=#{consumer.inspect}]"
+        end
+        alias to_s inspect
+      end
+
+      # Consume all messages of the current batch, and mark each message
+      # afterwards as processed (asynchronous). You can simply overwrite this
+      # method if you need more precise control of the message processing,
+      # eg. just using marking the whole batch processed, or custom error
+      # handling.
+      #
+      # This method provides *at-least-once* delivery semantics. Each message
+      # is enqueued to ActiveJob and then marked as consumed via
+      # +mark_as_consumed+ (asynchronous commit). In the unlikely event that
+      # the Karafka process crashes after +perform_later+ succeeds but before
+      # the offset is committed to the broker (e.g. OOM kill, hardware
+      # failure), the message will be re-delivered and enqueued again on
+      # restart. Downstream consumers should therefore be idempotent. For
+      # stronger guarantees, use +mark_as_consumed!+ (synchronous commit) at
+      # the cost of throughput, by providing a custom +job_bridge_class+ via
+      # +Rimless.configuration+.
+      #
+      # See: https://bit.ly/4aPXaai - then configure your own
+      # `Rimless.configuration.job_bridge_class`.
+      def consume
+        messages.each do |message|
+          enqueue_job(message)
+          mark_as_consumed(message)
+        end
+      end
+
+      # Enqueue a new job for the given message.
+      #
+      # @param message [Karafka::Messages::Message] the message to enqueue
+      def enqueue_job(message)
+        Rimless.configuration.consumer_job_class.perform_later(
+          **message_to_job_args(message)
+        )
+      end
+
+      # Convert the given +Karafka::Messages::Message+ instance to a simple
+      # hash, which can be transported by ActiveJob.
+      #
+      # @param message [Karafka::Messages::Message] the message to enqueue
+      # @return [Hash{Symbol => Mixed}] the job argument
+      def message_to_job_args(message)
+        {
+          payload: message.payload,
+          consumer:,
+          metadata: message.metadata.to_h.slice(
+            :topic,
+            :partition,
+            :offset,
+            :timestamp,
+            :received_at
+          ).merge(
+            key: message.metadata.key,
+            headers: message.metadata.headers
+          )
+        }
+      end
+
+      # A custom object/class inspection helper to allow pretty printing of
+      # the anonymous class.
+      #
+      # @return [String] the pretty-printed class/instance
+      def inspect
+        "#<#{Rimless::Consumer::JobBridge.name} consumer=#{consumer.inspect}>"
+      end
+      alias to_s inspect
+    end
+  end
+end

data/lib/rimless/extensions/avro_helpers.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module Rimless
+  module Extensions
+    # The top-level Apache Avro helpers.
+    module AvroHelpers
+      extend ActiveSupport::Concern
+
+      class_methods do
+        # A top-level +AvroTurf::Messaging+ instance
+        mattr_accessor :avro
+        # A shared +Rimless::AvroUtils+ instance
+        mattr_accessor :avro_utils
+
+        # A shortcut to encode data using the specified schema to the Apache
+        # Avro format. This also applies data sanitation to avoid issues with
+        # the low level Apache Avro library (symbolized keys, etc) and it
+        # allows deep-relative schema names. When you pass +.deep.deep+ for
+        # example (leading period) it will prefix the schema name with the
+        # local namespace (so it becomes absolute).
+        #
+        # @param data [Mixed] the data structure to encode
+        # @param schema [String, Symbol] name of the schema that should be used
+        # @param opts [Hash{Symbol => Mixed}] additional options
+        # @return [String] the Apache Avro blob
+        def avro_encode(data, schema:, **opts)
+          data = avro_sanitize(data)
+
+          # When the deep-relative form (+.deep.deep[..]+) is present, we add
+          # our local namespace, so Avro can resolve it
+          schema = avro_utils.namespace + schema.to_s \
+            if schema.to_s.start_with? '.'
+
+          avro.encode(data, schema_name: schema.to_s, **opts)
+        end
+        alias_method :encode, :avro_encode
+
+        # A shortcut to parse a blob of Apache Avro data.
+        #
+        # @param data [String] the Apache Avro blob
+        # @param opts [Hash{Symbol => Mixed}] additional options
+        # @return [Mixed] the decoded data structure
+        def avro_decode(data, **opts)
+          avro.decode(data, **opts).deep_symbolize_keys!
+        end
+        alias_method :decode, :avro_decode
+
+        # The Apache Avro Ruby gem requires simple typed hashes for encoding.
+        # This forces us to convert eg. Grape entity representations into
+        # simple string-keyed hashes. Use this method to prepare a hash for the
+        # Apache Avro serialization.
+        #
+        # Note about the implementation: JSON serialization and parsing is the
+        # simplest and fastest way to accomplish this.
+        #
+        # @param hash [Hash{Mixed => Mixed}] the hash to sanitize
+        # @return [Hash{String => Mixed}] the simple typed input hash
+        def avro_to_h(hash)
+          JSON.parse(hash.to_json)
+        end
+        alias_method :avro_sanitize, :avro_to_h
+
+        # Convert the given deep hash into a sparsified flat hash while
+        # transforming all values to strings. This allows to convert a
+        # schema-less hash to a Apache Avro compatible map.
+        #
+        # @see http://avro.apache.org/docs/current/spec.html#Maps
+        # @example Convert schema-less hash
+        #   avro_schemaless_map(a: { b: { c: true } })
+        #   # => { "a.b.c" => "true" }
+        #
+        # @param hash [Hash{Mixed => Mixed}] the deep hash
+        # @return [Hash{String => String}] the flattened and sparsified hash
+        def avro_schemaless_h(hash)
+          Sparsify(hash, sparse_array: true)
+            .transform_values(&:to_s)
+            .transform_keys { |key| key.delete('\\') }
+        end
+        alias_method :avro_schemaless_map, :avro_schemaless_h
+      end
+    end
+  end
+end

data/lib/rimless/extensions/configuration_handling.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module Rimless
+  module Extensions
+    # The top-level configuration handling.
+    #
+    # rubocop:disable Style/ClassVars -- because we split module code
+    module ConfigurationHandling
+      extend ActiveSupport::Concern
+
+      class_methods do
+        # Retrieve the current configuration object.
+        #
+        # @return [Configuration]
+        def configuration
+          @@configuration ||= Configuration.new
+        end
+
+        # Configure the concern by providing a block which takes
+        # care of this task. Example:
+        #
+        #   FactoryBot::Instrumentation.configure do |conf|
+        #     # conf.xyz = [..]
+        #   end
+        def configure
+          yield(configuration)
+          configure_dependencies
+        end
+
+        # Reset the current configuration with the default one.
+        def reset_configuration!
+          @@configuration = Configuration.new
+        end
+
+        # Retrieve the current configured environment. You can use it like
+        # +Rails.env+ to query it. E.g. +Rimless.env.production?+.
+        #
+        # @return [ActiveSupport::StringInquirer] the environment
+        def env
+          @@env = ActiveSupport::StringInquirer.new(configuration.env.to_s) \
+            if @env.to_s != configuration.env.to_s
+          @@env
+        end
+
+        # A simple convention helper to setup Apache Kafka topic names.
+        #
+        # @param app [String] the application namespace
+        # @return [String] the Apache Kafka topic name prefix
+        def topic_prefix(app = Rimless.configuration.app_name)
+          "#{Rimless.env}.#{app}."
+        end
+
+        # Pass back the local application name. When we are loaded together
+        # with a Rails application we use the application class name. This
+        # application name is URI/GID compatible. When no local application is
+        # available, we just pass back +nil+.
+        #
+        # @return [String, nil] the Rails application name, or +nil+
+        def local_app_name
+          # Check for non-Rails integration
+          return unless defined? Rails
+          # Check if a application is defined
+          return if Rails.application.nil?
+
+          # Pass back the URI compatible application name
+          Rails.application.class.module_parent_name.underscore.dasherize
+        end
+
+        # Retrieve the current configured logger instance.
+        #
+        # @return [Logger] the logger instance
+        delegate :logger, to: :configuration
+      end
+    end
+    # rubocop:enable Style/ClassVars
+  end
+end

data/lib/rimless/extensions/consumer.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Rimless
+  module Extensions
+    # The top-level Apache Kafka message consumer integration.
+    module Consumer
+      extend ActiveSupport::Concern
+
+      class_methods do
+        # A simple shortcut to fetch the Karafka-wrapping consumer application.
+        #
+        # @return [Rimless::Consumer::App] the internal consumer
+        #   application class
+        def consumer
+          @consumer ||= Rimless::Consumer::App.new
+        end
+      end
+    end
+  end
+end