rimless 2.8.0 → 3.0.0

data/lib/rimless/consumer.rb

@@ -1,209 +0,0 @@
-# frozen_string_literal: true
-
-module Rimless
-  # The global rimless Apache Kafka consumer application based on
-  # the Karafka framework.
-  #
-  # rubocop:disable Style/ClassVars -- because we just work as a singleton
-  class ConsumerApp < ::Karafka::App
-    # We track our own initialization with this class variable
-    @@rimless_initialized = false
-
-    class << self
-      # Initialize the Karafka framework and our global consumer application
-      # with all our conventions/opinions.
-      #
-      # @return [Rimless::ConsumerApp] our self for chaining
-      def initialize!
-        # When already initialized, skip it
-        return self if @@rimless_initialized
-
-        # Initialize all the parts one by one
-        initialize_rails!
-        initialize_monitors!
-        initialize_karafka!
-        initialize_logger!
-        initialize_code_reload!
-
-        # Load the custom Karafka boot file when it exists, it contains
-        # custom configurations and the topic/consumer routing table
-        require ::Karafka.boot_file if ::Karafka.boot_file.exist?
-
-        # Set our custom initialization process as completed to
-        # skip subsequent calls
-        @@rimless_initialized = true
-        self
-      end
-
-      # Check if Rails is available and not already initialized, then
-      # initialize it.
-      def initialize_rails!
-        rails_env = ::Karafka.root.join('config', 'environment.rb')
-
-        # Stop, when Rails is already initialized
-        return if defined? Rails
-
-        # Stop, when there is no Rails at all
-        return unless rails_env.exist?
-
-        ENV['RAILS_ENV'] ||= 'development'
-        ENV['KARAFKA_ENV'] = ENV.fetch('RAILS_ENV', nil)
-
-        # This is relevant for Karafka server processes, as the +karafka.rb+
-        # root file just requires +rimless+ and then we require
-        # +karafka-sidekiq-backend+ which in fact requires +sidekiq+ before
-        # +rails+ was required. We cannot change the order here, but we can
-        # explicitly load the Sidekiq/Rails integration as we know at this
-        # point that we should load Rails and we're going to use Sidekiq, too.
-        # See: https://bit.ly/3D8ZHj3
-        require 'sidekiq/rails'
-
-        require rails_env
-        require 'rimless/railtie'
-        Rails.application.eager_load!
-      end
-
-      # We like to listen to instrumentation and logging events to allow our
-      # users to handle them like they need.
-      def initialize_monitors!
-        [
-          WaterDrop::Instrumentation::StdoutListener.new,
-          ::Karafka::Instrumentation::StdoutListener.new,
-          ::Karafka::Instrumentation::ProctitleListener.new
-        ].each do |listener|
-          ::Karafka.monitor.subscribe(listener)
-        end
-      end
-
-      # Configure the pure basics on the Karafka application.
-      def initialize_karafka!
-        setup do |config|
-          mapper = Rimless::Karafka::PassthroughMapper.new
-          config.consumer_mapper = config.topic_mapper = mapper
-          config.deserializer = Rimless::Karafka::AvroDeserializer.new
-          config.kafka.seed_brokers = Rimless.configuration.kafka_brokers
-          config.client_id = Rimless.configuration.client_id
-          config.logger = Rimless.logger
-          config.backend = :sidekiq
-          config.batch_fetching = true
-          config.batch_consuming = false
-          config.shutdown_timeout = 10
-        end
-      end
-
-      # When we run in development mode, we want to write the logs
-      # to file and to stdout.
-      def initialize_logger!
-        # Skip when configured not to extend the logger
-        return unless Rimless.configuration.extend_dev_logger
-
-        # Skip when not in development environment or in the server process
-        return unless Rimless.env.development? && server?
-
-        $stdout.sync = true
-        Rimless.logger.extend(ActiveSupport::Logger.broadcast(
-                                ActiveSupport::Logger.new($stdout)
-                              ))
-      end
-
-      # Perform code hot-reloading when we are in Rails and in development
-      # mode.
-      def initialize_code_reload!
-        return unless defined?(Rails) && Rails.env.development?
-
-        ::Karafka.monitor.subscribe(::Karafka::CodeReloader.new(
-                                      *Rails.application.reloaders
-                                    ))
-      end
-
-      # Allows the user to re-configure the Karafka application if this is
-      # needed. (eg. to set some ruby-kafka driver default settings, etc)
-      #
-      # @return [Rimless::ConsumerApp] our self for chaining
-      def configure(&)
-        setup(&)
-        self
-      end
-
-      # Configure the topics-consumer routing table in a lean way.
-      #
-      # Examples:
-      #
-      #   topics({ app: :test_app, name: :admins } => YourConsumer)
-      #   topics({ app: :test_app, names: %i[users admins] } => YourConsumer)
-      #
-      # Examples:
-      #
-      #   topics do
-      #     topic('name') do
-      #       consumer CustomConsumer
-      #     end
-      #   end
-      #
-      # @param topics [Hash{Hash => Class}] the topic to consumer mapping
-      # @yield the given block on the routing table
-      def topics(topics = [], &block)
-        consumer_groups.draw do
-          consumer_group(Rimless.configuration.client_id) do
-            instance_exec(&block) if block_given?
-
-            topics.each do |topic_parts, dest_consumer|
-              Rimless.consumer.topic_names(topic_parts).each do |topic_name|
-                topic(topic_name) do
-                  consumer dest_consumer
-                  worker Rimless::ConsumerJob
-                  interchanger Rimless::Karafka::Base64Interchanger.new
-                end
-              end
-            end
-          end
-        end
-
-        self
-      end
-
-      # Build the conventional Apache Kafka topic names from the given parts.
-      # This allows various forms like single strings/symbols and a hash in the
-      # form of +{ app: [String, Symbol], name: [String, Symbol], names:
-      # [Array<String, Symbol>] }+. This allows the maximum of flexibility.
-      #
-      # @param parts [String, Symbol, Hash{Symbol => Mixed}] the topic
-      #   name parts
-      # @return [Array<String>] the full topic names
-      def topic_names(parts)
-        # We have a single app, but multiple names so we handle them
-        if parts.is_a?(Hash) && parts.key?(:names)
-          return parts[:names].map do |name|
-            Rimless.topic(parts.merge(name: name))
-          end
-        end
-
-        # We cannot handle the given input
-        [Rimless.topic(parts)]
-      end
-
-      # Check if we run as the Karafka server (consumer) process or not.
-      #
-      # @return [Boolean] whenever we run as the Karafka server or not
-      def server?
-        $PROGRAM_NAME.end_with?('karafka') && ARGV.include?('server')
-      end
-    end
-  end
-  # rubocop:enable Style/ClassVars
-
-  # A rimless top-level concern which adds lean access to
-  # the consumer application.
-  module Consumer
-    extend ActiveSupport::Concern
-
-    class_methods do
-      # A simple shortcut to fetch the Karafka consumer application.
-      #
-      # @return [Rimless::ConsumerApp] the Karafka consumer application class
-      def consumer
-        ConsumerApp.initialize!
-      end
-    end
-  end
-end

data/lib/rimless/consumer_job.rb

@@ -1,10 +0,0 @@
-# frozen_string_literal: true
-
-module Rimless
-  # The base consumer job where each message is processed asynchronous via
-  # Sidekiq. We need to inherit the Karafka base worker class into a custom
-  # one, otherwise it fails.
-  class ConsumerJob < ::Karafka::BaseWorker
-    sidekiq_options queue: Rimless.configuration.consumer_job_queue
-  end
-end

data/lib/rimless/dependencies.rb

@@ -1,69 +0,0 @@
-# frozen_string_literal: true
-
-module Rimless
-  # The top-level dependencies helpers.
-  module Dependencies
-    extend ActiveSupport::Concern
-
-    class_methods do
-      # (Re)configure our gem dependencies. We take care of setting up
-      # +WaterDrop+, our Apache Kafka driver and +AvroTurf+, our Confluent
-      # Schema Registry driver.
-      def configure_dependencies
-        configure_waterdrop
-        configure_avro_turf
-      end
-
-      # Set sensible defaults for the +WaterDrop+ gem.
-      def configure_waterdrop
-        # Skip WaterDrop configuration when no brokers/client id is available,
-        # because it will raise. Its fine to have none available for situations
-        # like Rails asset precompilations, etc. - on runtime the settings
-        # should be available, otherwise the message producing just
-        # fails/raise.
-        return if Rimless.configuration.kafka_brokers.empty? \
-          || Rimless.configuration.client_id.blank?
-
-        WaterDrop.setup do |config|
-          # Activate message delivery and use the default logger
-          config.deliver = true
-          config.logger = Rimless.logger
-          # An optional identifier of a Kafka consumer (in a consumer group)
-          # that is passed to a Kafka broker with every request. A logical
-          # application name to be included in Kafka logs and monitoring
-          # aggregates.
-          config.client_id = Rimless.configuration.client_id
-          # All the known brokers, at least one. The ruby-kafka driver will
-          # discover the whole cluster structure once and when issues occur to
-          # dynamically adjust scaling operations.
-          config.kafka.seed_brokers = Rimless.configuration.kafka_brokers
-          # All brokers MUST acknowledge a new message
-          config.kafka.required_acks = -1
-        end
-      end
-
-      # Set sensible defaults for the +AvroTurf+ gem and (re)compile the Apache
-      # Avro schema templates (ERB), so the gem can handle them properly.
-      def configure_avro_turf
-        # No need to configure AvroTurf when no schema registry URL is
-        # available. Its fine to skip this for scenarios where not the full
-        # application configuration is available (eg. on Rails asset
-        # precompilations, etc)
-        return if Rimless.configuration.schema_registry_url.blank?
-
-        # Setup a global available Apache Avro decoder/encoder with support for
-        # the Confluent Schema Registry, but first create a helper instance
-        Rimless.avro_utils = Rimless::AvroUtils.new
-        # Compile our Avro schema templates to ready-to-consume Avro schemas
-        Rimless.avro_utils.recompile_schemas
-        # Register a global Avro messaging instance
-        Rimless.avro = AvroTurf::Messaging.new(
-          logger: Rimless.logger,
-          namespace: Rimless.avro_utils.namespace,
-          schemas_path: Rimless.avro_utils.output_path,
-          registry_url: Rimless.configuration.schema_registry_url
-        )
-      end
-    end
-  end
-end

data/lib/rimless/kafka_helpers.rb

@@ -1,104 +0,0 @@
-# frozen_string_literal: true
-
-module Rimless
-  # The top-level Apache Kafka helpers.
-  module KafkaHelpers
-    extend ActiveSupport::Concern
-
-    class_methods do
-      # Generate a common topic name for Apache Kafka while taking care of
-      # configured prefixes.
-      #
-      # @param args [Array<Mixed>] the relative topic name
-      # @return [String] the complete topic name
-      #
-      # @example Name only
-      #   Rimless.topic(:users)
-      # @example Name with app
-      #   Rimless.topic(:users, app: 'test-api')
-      # @example Mix and match
-      #   Rimless.topic(name: 'test', app: :fancy_app)
-      # @example Full name - use as is
-      #   Rimless.topic(full_name: 'my.custom.topic.name')
-      def topic(*args)
-        opts = args.last
-        name = args.first if [String, Symbol].member?(args.first.class)
-
-        if opts.is_a?(Hash)
-          # When we got a full name, we use it as is
-          return opts[:full_name] if opts.key? :full_name
-
-          name = opts[:name] if opts.key?(:name)
-          app = opts[:app] if opts.key?(:app)
-        end
-
-        name ||= nil
-        app ||= Rimless.configuration.app_name
-
-        raise ArgumentError, 'No name given' if name.nil?
-
-        "#{Rimless.topic_prefix(app)}#{name}".tr('_', '-')
-      end
-
-      # Send a single message to Apache Kafka. The data is encoded according to
-      # the given Apache Avro schema. The destination Kafka topic may be a
-      # relative name, or a hash which is passed to the +.topic+ method to
-      # manipulate the application details. The message is send is a
-      # synchronous, blocking way.
-      #
-      # @param data [Hash{Symbol => Mixed}] the raw data, unencoded
-      # @param schema [String, Symbol] the Apache Avro schema to use
-      # @param topic [String, Symbol, Hash{Symbol => Mixed}] the destination
-      #   Apache Kafka topic
-      def sync_message(data:, schema:, topic:, **args)
-        encoded = Rimless.encode(data, schema: schema)
-        sync_raw_message(data: encoded, topic: topic, **args)
-      end
-      alias_method :message, :sync_message
-
-      # Send a single message to Apache Kafka. The data is encoded according to
-      # the given Apache Avro schema. The destination Kafka topic may be a
-      # relative name, or a hash which is passed to the +.topic+ method to
-      # manipulate the application details. The message is send is an
-      # asynchronous, non-blocking way.
-      #
-      # @param data [Hash{Symbol => Mixed}] the raw data, unencoded
-      # @param schema [String, Symbol] the Apache Avro schema to use
-      # @param topic [String, Symbol, Hash{Symbol => Mixed}] the destination
-      #   Apache Kafka topic
-      def async_message(data:, schema:, topic:, **args)
-        encoded = Rimless.encode(data, schema: schema)
-        async_raw_message(data: encoded, topic: topic, **args)
-      end
-
-      # Send a single message to Apache Kafka. The data is not touched, so you
-      # need to encode it yourself before you pass it in. The destination Kafka
-      # topic may be a relative name, or a hash which is passed to the +.topic+
-      # method to manipulate the application details. The message is send is a
-      # synchronous, blocking way.
-      #
-      # @param data [Hash{Symbol => Mixed}] the raw data, unencoded
-      # @param topic [String, Symbol, Hash{Symbol => Mixed}] the destination
-      #   Apache Kafka topic
-      def sync_raw_message(data:, topic:, **args)
-        args = args.merge(topic: topic(topic))
-        WaterDrop::SyncProducer.call(data, **args)
-      end
-      alias_method :raw_message, :sync_raw_message
-
-      # Send a single message to Apache Kafka. The data is not touched, so you
-      # need to encode it yourself before you pass it in. The destination Kafka
-      # topic may be a relative name, or a hash which is passed to the +.topic+
-      # method to manipulate the application details. The message is send is an
-      # asynchronous, non-blocking way.
-      #
-      # @param data [Hash{Symbol => Mixed}] the raw data, unencoded
-      # @param topic [String, Symbol, Hash{Symbol => Mixed}] the destination
-      #   Apache Kafka topic
-      def async_raw_message(data:, topic:, **args)
-        args = args.merge(topic: topic(topic))
-        WaterDrop::AsyncProducer.call(data, **args)
-      end
-    end
-  end
-end

data/lib/rimless/karafka/base64_interchanger.rb

@@ -1,32 +0,0 @@
-# frozen_string_literal: true
-
-module Rimless
-  module Karafka
-    # Allow the +karafka-sidekiq-backend+ gem to transfer binary Apache Kafka
-    # messages to the actual Sidekiq job.
-    #
-    # rubocop:disable Security/MarshalLoad -- because we encode/decode the
-    #   messages in our own controlled context
-    class Base64Interchanger < ::Karafka::Interchanger
-      # Encode a binary Apache Kafka message(s) so they can be passed to the
-      # Sidekiq +Rimless::ConsumerJob+.
-      #
-      # @param params_batch [Karafka::Params::ParamsBatch] the karafka params
-      #   batch object
-      # @return [String] the marshaled+base64 encoded data
-      def encode(params_batch)
-        Base64.encode64(Marshal.dump(super))
-      end
-
-      # Decode the binary Apache Kafka message(s) so they can be processed by
-      # the Sidekiq +Rimless::ConsumerJob+.
-      #
-      # @param params_string [String] the marshaled+base64 encoded data
-      # @return [Array<Hash>] the unmarshaled+base64 decoded data
-      def decode(params_batch)
-        super(Marshal.load(Base64.decode64(params_batch))).map(&:stringify_keys)
-      end
-    end
-    # rubocop:enable Security/MarshalLoad
-  end
-end

data/lib/rimless/karafka/passthrough_mapper.rb

@@ -1,29 +0,0 @@
-# frozen_string_literal: true
-
-module Rimless
-  module Karafka
-    # The Karafka framework makes some assumptions about the consumer group and
-    # topic names. We have our own opinions/conventions, so we just pass them
-    # through unmodified.
-    class PassthroughMapper
-      # We do not want to modify the given consumer group name, so we
-      # pass it through.
-      #
-      # @param raw_consumer_group_name [String, Symbol] the original
-      #   consumer group name
-      # @return [String, Symbol] the original consumer group name
-      def call(raw_consumer_group_name)
-        raw_consumer_group_name
-      end
-
-      # We do not want to modify the given topic name, so we pass it through.
-      #
-      # @param topic [String, Symbol] the original topic name
-      # @return [String, Symbol] the original topic name
-      def incoming(topic)
-        topic
-      end
-      alias outgoing incoming
-    end
-  end
-end

data/lib/rimless/tasks/stats.rake

@@ -1,22 +0,0 @@
-# frozen_string_literal: true
-
-# TODO: Remove this file, when Rails >= 8.0 is the minimum requirement
-if defined?(Rails) && Rails.env.development? && Rails::VERSION::STRING < '8.0.0'
-  require 'rspec/core/rake_task'
-
-  # rubocop:disable Rails/RakeEnvironment -- because this is just an helper
-  #   command, no need for an application bootstrap
-  task :stats do
-    require 'rails/code_statistics'
-
-    [
-      [:unshift, 'Consumer', 'app/consumers']
-    ].each do |method, type, dir|
-      next unless File.directory? dir
-
-      STATS_DIRECTORIES.send(method, [type, dir])
-      CodeStatistics::TEST_TYPES << type if type.include? 'specs'
-    end
-  end
-  # rubocop:enable Rails/RakeEnvironment
-end