karafka 1.2.0 → 1.3.1

data/lib/karafka/persistence/topic.rb

@@ -1,29 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Persistence
-    # Local cache for routing topics
-    # We use it in order not to build string instances and remap incoming topic upon each
-    # message / message batches received
-    class Topic
-      # Thread.current scope under which we store topics data
-      PERSISTENCE_SCOPE = :topics
-
-      # @param group_id [String] group id for which we fetch a topic representation
-      # @param raw_topic_name [String] raw topic name (before remapping) for which we fetch a
-      #   topic representation
-      # @return [Karafka::Routing::Topic] remapped topic representation that can be used further
-      #   on when working with given parameters
-      def self.fetch(group_id, raw_topic_name)
-        Thread.current[PERSISTENCE_SCOPE] ||= Hash.new { |hash, key| hash[key] = {} }
-
-        Thread.current[PERSISTENCE_SCOPE][group_id][raw_topic_name] ||= begin
-          # We map from incoming topic name, as it might be namespaced, etc.
-          # @see topic_mapper internal docs
-          mapped_topic_name = Karafka::App.config.topic_mapper.incoming(raw_topic_name)
-          Routing::Router.find("#{group_id}_#{mapped_topic_name}")
-        end
-      end
-    end
-  end
-end

data/lib/karafka/schemas/config.rb

@@ -1,24 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Namespace for all the validation schemas that we use to check input
-  module Schemas
-    # Regexp for validating format of groups and topics
-    TOPIC_REGEXP = /\A(\w|\-|\.)+\z/
-
-    # Schema with validation rules for Karafka configuration details
-    # @note There are many more configuration options inside of the
-    #   Karafka::Setup::Config model, but we don't validate them here as they are
-    #   validated per each route (topic + consumer_group) because they can be overwritten,
-    #   so we validate all of that once all the routes are defined and ready
-    Config = Dry::Validation.Schema do
-      required(:client_id).filled(:str?, format?: Karafka::Schemas::TOPIC_REGEXP)
-      required(:shutdown_timeout) { none? | (int? & gteq?(0)) }
-      required(:consumer_mapper)
-      required(:topic_mapper)
-      required(:params_base_class).filled
-
-      optional(:backend).filled
-    end
-  end
-end

data/lib/karafka/schemas/consumer_group.rb

@@ -1,77 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Schemas
-    # Schema for single full route (consumer group + topics) validation.
-    ConsumerGroup = Dry::Validation.Schema do
-      # Valid uri schemas of Kafka broker url
-      # The ||= is due to the behavior of require_all that resolves dependencies
-      # but someetimes loads things twice
-      URI_SCHEMES ||= %w[kafka kafka+ssl].freeze
-
-      # Available sasl scram mechanism of authentication (plus nil)
-      SASL_SCRAM_MECHANISMS ||= %w[sha256 sha512].freeze
-
-      configure do
-        config.messages_file = File.join(
-          Karafka.gem_root, 'config', 'errors.yml'
-        )
-
-        # Uri validator to check if uri is in a Karafka acceptable format
-        # @param uri [String] uri we want to validate
-        # @return [Boolean] true if it is a valid uri, otherwise false
-        def broker_schema?(uri)
-          uri = URI.parse(uri)
-          URI_SCHEMES.include?(uri.scheme) && uri.port
-        rescue URI::InvalidURIError
-          false
-        end
-      end
-
-      required(:id).filled(:str?, format?: Karafka::Schemas::TOPIC_REGEXP)
-      required(:seed_brokers).filled { each(:broker_schema?) }
-      required(:session_timeout).filled { int? | float? }
-      required(:pause_timeout).filled { (int? | float?) & gteq?(0) }
-      required(:offset_commit_interval) { int? | float? }
-      required(:offset_commit_threshold).filled(:int?)
-      required(:offset_retention_time) { none?.not > int? }
-      required(:heartbeat_interval).filled { (int? | float?) & gteq?(0) }
-      required(:connect_timeout).filled { (int? | float?) & gt?(0) }
-      required(:socket_timeout).filled { (int? | float?) & gt?(0) }
-      required(:min_bytes).filled(:int?, gt?: 0)
-      required(:max_bytes).filled(:int?, gt?: 0)
-      required(:max_wait_time).filled { (int? | float?) & gteq?(0) }
-      required(:batch_fetching).filled(:bool?)
-      required(:topics).filled { each { schema(ConsumerGroupTopic) } }
-
-      # Max wait time cannot exceed socket_timeout - wouldn't make sense
-      rule(
-        max_wait_time_limit: %i[max_wait_time socket_timeout]
-      ) do |max_wait_time, socket_timeout|
-        socket_timeout.int? > max_wait_time.lteq?(value(:socket_timeout))
-      end
-
-      %i[
-        ssl_ca_cert
-        ssl_ca_cert_file_path
-        ssl_client_cert
-        ssl_client_cert_key
-        sasl_gssapi_principal
-        sasl_gssapi_keytab
-        sasl_plain_authzid
-        sasl_plain_username
-        sasl_plain_password
-        sasl_scram_username
-        sasl_scram_password
-      ].each do |encryption_attribute|
-        optional(encryption_attribute).maybe(:str?)
-      end
-
-      optional(:ssl_ca_certs_from_system).maybe(:bool?)
-
-      # It's not with other encryptions as it has some more rules
-      optional(:sasl_scram_mechanism)
-        .maybe(:str?, included_in?: Karafka::Schemas::SASL_SCRAM_MECHANISMS)
-    end
-  end
-end

data/lib/karafka/schemas/consumer_group_topic.rb

@@ -1,18 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Schemas
-    # Consumer group topic validation rules
-    ConsumerGroupTopic = Dry::Validation.Schema do
-      required(:id).filled(:str?, format?: Karafka::Schemas::TOPIC_REGEXP)
-      required(:name).filled(:str?, format?: Karafka::Schemas::TOPIC_REGEXP)
-      required(:backend).filled(included_in?: %i[inline sidekiq])
-      required(:consumer).filled
-      required(:parser).filled
-      required(:max_bytes_per_partition).filled(:int?, gteq?: 0)
-      required(:start_from_beginning).filled(:bool?)
-      required(:batch_consuming).filled(:bool?)
-      required(:persistent).filled(:bool?)
-    end
-  end
-end

data/lib/karafka/schemas/responder_usage.rb

@@ -1,39 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Schemas
-    # Validator to check responder topic usage
-    ResponderUsageTopic = Dry::Validation.Schema do
-      required(:name).filled(:str?, format?: Karafka::Schemas::TOPIC_REGEXP)
-      required(:required).filled(:bool?)
-      required(:multiple_usage).filled(:bool?)
-      required(:usage_count).filled(:int?, gteq?: 0)
-      required(:registered).filled(eql?: true)
-      required(:async).filled(:bool?)
-
-      rule(
-        required_usage: %i[required usage_count]
-      ) do |required, usage_count|
-        required.true? > usage_count.gteq?(1)
-      end
-
-      rule(
-        multiple_usage_permission: %i[multiple_usage usage_count]
-      ) do |multiple_usage, usage_count|
-        usage_count.gt?(1) > multiple_usage.true?
-      end
-
-      rule(
-        multiple_usage_block: %i[multiple_usage usage_count]
-      ) do |multiple_usage, usage_count|
-        multiple_usage.false? > usage_count.lteq?(1)
-      end
-    end
-
-    # Validator to check that everything in a responder flow matches responder rules
-    ResponderUsage = Dry::Validation.Schema do
-      required(:used_topics) { filled? > each { schema(ResponderUsageTopic) } }
-      required(:registered_topics) { filled? > each { schema(ResponderUsageTopic) } }
-    end
-  end
-end

data/lib/karafka/schemas/server_cli_options.rb

@@ -1,43 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Schemas
-    # Schema for validating correctness of the server cli command options
-    # We validate some basics + the list of consumer_groups on which we want to use, to make
-    # sure that all of them are defined, plus that a pidfile does not exist
-    ServerCliOptions = Dry::Validation.Schema do
-      configure do
-        option :consumer_groups
-
-        def self.messages
-          super.merge(
-            en: {
-              errors: {
-                consumer_groups_inclusion: 'Unknown consumer group.',
-                pid_existence: 'Pidfile already exists.'
-              }
-            }
-          )
-        end
-      end
-
-      optional(:pid).filled(:str?)
-      optional(:daemon).filled(:bool?)
-      optional(:consumer_groups).filled(:array?)
-
-      validate(consumer_groups_inclusion: :consumer_groups) do |consumer_groups|
-        # If there were no consumer_groups declared in the server cli, it means that we will
-        # run all of them and no need to validate them here at all
-        if consumer_groups.nil?
-          true
-        else
-          (consumer_groups - Karafka::Routing::Builder.instance.map(&:name)).empty?
-        end
-      end
-
-      validate(pid_existence: :pid) do |pid|
-        pid ? !File.exist?(pid) : true
-      end
-    end
-  end
-end

data/lib/karafka/setup/configurators/base.rb

@@ -1,29 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Setup
-    # Configurators module is used to enclose all the external dependencies configurations
-    # upon which Karafka depents
-    class Configurators
-      # Karafka has some components that it relies on (like Sidekiq)
-      # We need to configure all of them only when the framework was set up.
-      # Any class that descends from this one will be automatically invoked upon setup (after it)
-      # @note This should be used only for internal Karafka dependencies configuration
-      #   End users configuration should go to the after_init block
-      # @example Configure an Example class
-      #   class ExampleConfigurator < Base
-      #     def setup
-      #       ExampleClass.logger = Karafka.logger
-      #       ExampleClass.redis = config.redis
-      #     end
-      #   end
-      class Base
-        # @param _config [Karafka::Config] config instance
-        # This method needs to be implemented in a subclass
-        def self.setup(_config)
-          raise NotImplementedError
-        end
-      end
-    end
-  end
-end

data/lib/karafka/setup/configurators/params.rb

@@ -1,25 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Setup
-    class Configurators
-      # Karafka::Params::Params are dynamically built based on user defined parent class
-      # so we cannot just require it, we need to initialize it after user is done with
-      # the framework configuration. This is a configurator that does exactly that.
-      class Params < Base
-        # Builds up Karafka::Params::Params class with user defined parent class
-        # @param config [Karafka::Setup::Config] Config we can user to setup things
-        def self.setup(config)
-          return if defined? Karafka::Params::Params
-
-          Karafka::Params.const_set(
-            'Params',
-            Class
-              .new(config.params_base_class)
-              .tap { |klass| klass.include(Karafka::Params::Dsl) }
-          )
-        end
-      end
-    end
-  end
-end

data/lib/karafka/templates/karafka.rb.example

@@ -1,54 +0,0 @@
-# frozen_string_literal: true
-
-# Non Ruby on Rails setup
-ENV['RACK_ENV'] ||= 'development'
-ENV['KARAFKA_ENV'] ||= ENV['RACK_ENV']
-Bundler.require(:default, ENV['KARAFKA_ENV'])
-Karafka::Loader.load(Karafka::App.root)
-
-# Ruby on Rails setup
-# Remove whole non-Rails setup that is above and uncomment the 4 lines below
-# ENV['RAILS_ENV'] ||= 'development'
-# ENV['KARAFKA_ENV'] = ENV['RAILS_ENV']
-# require ::File.expand_path('../config/environment', __FILE__)
-# Rails.application.eager_load!
-
-class KarafkaApp < Karafka::App
-  setup do |config|
-    config.kafka.seed_brokers = %w[kafka://127.0.0.1:9092]
-    config.client_id = 'example_app'
-    config.backend = :inline
-    config.batch_fetching = true
-    # Uncomment this for Rails app integration
-    # config.logger = Rails.logger
-  end
-
-  after_init do |config|
-    # Put here all the things you want to do after the Karafka framework
-    # initialization
-  end
-
-  # Comment out this part if you are not using instrumentation and/or you are not
-  # interested in logging events for certain environments. Since instrumentation
-  # notifications add extra boilerplate, if you want to achieve max performance,
-  # listen to only what you really need for given environment.
-  Karafka.monitor.subscribe(Karafka::Instrumentation::Listener)
-
-  consumer_groups.draw do
-    # topic :example do
-    #   consumer ExampleConsumer
-    # end
-
-    # consumer_group :bigger_group do
-    #   topic :test do
-    #     consumer TestConsumer
-    #   end
-    #
-    #   topic :test2 do
-    #     consumer Test2Consumer
-    #   end
-    # end
-  end
-end
-
-KarafkaApp.boot!