karafka 1.4.0 → 2.0.10

data/config/errors.yml

@@ -1,39 +1,60 @@
 en:
-  dry_validation:
-    errors:
-      invalid_broker_schema: >
-        has an invalid format
-        Expected schema, host and port number
-        Example: kafka://127.0.0.1:9092 or kafka+ssl://127.0.0.1:9092
-      invalid_certificate: >
-        is not a valid certificate
-      invalid_certificate_from_path: >
-        is not a valid certificate
-      invalid_private_key: >
-        is not a valid private key
-      max_timeout_size_for_exponential: >
-        pause_timeout cannot be more than pause_max_timeout
-      max_wait_time_limit:
-        max_wait_time cannot be more than socket_timeout
-      topics_names_not_unique: >
-        all topic names within a single consumer group must be unique
-      ssl_client_cert_with_ssl_client_cert_key: >
-        Both ssl_client_cert and ssl_client_cert_key need to be provided
-      ssl_client_cert_key_with_ssl_client_cert: >
-        Both ssl_client_cert_key and ssl_client_cert need to be provided
-      ssl_client_cert_chain_with_ssl_client_cert: >
-        Both ssl_client_cert_chain and ssl_client_cert need to be provided
-      ssl_client_cert_chain_with_ssl_client_cert_key: >
-        Both ssl_client_cert_chain and ssl_client_cert_key need to be provided
-      ssl_client_cert_key_password_with_ssl_client_cert_key: >
-        Both ssl_client_cert_key_password and ssl_client_cert_key need to be provided
-      does_not_respond_to_token: >
-        needs to respond to a #token method
-      required_usage_count: >
-        Given topic must be used at least once
-      pid_already_exists: >
-        Pidfile already exists
-      consumer_groups_inclusion: >
-        Unknown consumer group
-      does_not_exist:
-        Given file does not exist or cannot be read
+  validations:
+    config:
+      missing: needs to be present
+      client_id_format: 'needs to be a string with a Kafka accepted format'
+      license.entity_format: needs to be a string
+      license.token_format: needs to be either false or a string
+      license.expires_on_format: needs to be a valid date
+      concurrency_format: needs to be an integer bigger than 0
+      consumer_mapper_format: needs to be present
+      consumer_persistence_format: needs to be either true or false
+      pause_timeout_format: needs to be an integer bigger than 0
+      pause_max_timeout_format: needs to be an integer bigger than 0
+      pause_with_exponential_backoff_format: needs to be either true or false
+      shutdown_timeout_format: needs to be an integer bigger than 0
+      max_wait_time_format: needs to be an integer bigger than 0
+      kafka_format: needs to be a filled hash
+      internal.status_format: needs to be present
+      internal.process_format: needs to be present
+      internal.routing.builder_format: needs to be present
+      internal.routing.subscription_groups_builder_format: needs to be present
+      key_must_be_a_symbol: All keys under the kafka settings scope need to be symbols
+      max_timeout_vs_pause_max_timeout: pause_timeout must be less or equal to pause_max_timeout
+      shutdown_timeout_vs_max_wait_time: shutdown_timeout must be more than max_wait_time
+
+    server_cli_options:
+      missing: needs to be present
+      consumer_groups_inclusion: Unknown consumer group
+
+    consumer_group_topic:
+      missing: needs to be present
+      name_format: 'needs to be a string with a Kafka accepted format'
+      deserializer_format: needs to be present
+      manual_offset_management_format: needs to be either true or false
+      consumer_format: needs to be present
+      id_format: 'needs to be a string with a Kafka accepted format'
+      initial_offset_format: needs to be either earliest or latest
+      subscription_group_format: must be nil or a non-empty string
+
+    consumer_group:
+      missing: needs to be present
+      topics_names_not_unique: all topic names within a single consumer group must be unique
+      id_format: 'needs to be a string with a Kafka accepted format'
+      topics_format: needs to be a non-empty array
+
+    job_options:
+      missing: needs to be present
+      dispatch_method_format: needs to be either :produce_async or :produce_sync
+      partitioner_format: 'needs to respond to #call'
+      partition_key_type_format: 'needs to be either :key or :partition_key'
+
+    test:
+      missing: needs to be present
+      id_format: needs to be a String
+
+    pro_consumer_group_topic:
+      consumer_format: needs to inherit from Karafka::Pro::BaseConsumer and not Karafka::Consumer
+      virtual_partitions.partitioner_respond_to_call: needs to be defined and needs to respond to `#call`
+      virtual_partitions.max_partitions_format: needs to be equl or more than 1
+      manual_offset_management_not_with_virtual_partitions: cannot be used together with Virtual Partitions

data/docker-compose.yml

@@ -1,17 +1,24 @@
 version: '2'
 services:
   zookeeper:
+    container_name: karafka_20_zookeeper
     image: wurstmeister/zookeeper
     ports:
-      - "2181:2181"
+      - '2181:2181'
   kafka:
-    image: wurstmeister/kafka:1.0.1
+    container_name: karafka_20_kafka
+    image: wurstmeister/kafka
     ports:
-      - "9092:9092"
+      - '9092:9092'
     environment:
       KAFKA_ADVERTISED_HOST_NAME: localhost
       KAFKA_ADVERTISED_PORT: 9092
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
+      KAFKA_CREATE_TOPICS:
+        "benchmarks_00_01:1:1,\
+         benchmarks_00_05:5:1,\
+         benchmarks_01_05:5:1,\
+         benchmarks_00_10:10:1"
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock

data/karafka.gemspec

@@ -5,39 +5,36 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 require 'karafka/version'
 
-# rubocop:disable Metrics/BlockLength
 Gem::Specification.new do |spec|
   spec.name        = 'karafka'
   spec.version     = ::Karafka::VERSION
   spec.platform    = Gem::Platform::RUBY
-  spec.authors     = ['Maciej Mensfeld', 'Pavlo Vavruk', 'Adam Gwozdowski']
-  spec.email       = %w[maciej@mensfeld.pl pavlo.vavruk@gmail.com adam99g@gmail.com]
-  spec.homepage    = 'https://github.com/karafka/karafka'
-  spec.summary     = 'Ruby based framework for working with Apache Kafka'
+  spec.authors     = ['Maciej Mensfeld']
+  spec.email       = %w[contact@karafka.io]
+  spec.homepage    = 'https://karafka.io'
+  spec.summary     = 'Efficient Kafka processing framework for Ruby and Rails'
   spec.description = 'Framework used to simplify Apache Kafka based Ruby applications development'
-  spec.license     = 'MIT'
-
-  spec.add_dependency 'dry-configurable', '~> 0.8'
-  spec.add_dependency 'dry-inflector', '~> 0.1'
-  spec.add_dependency 'dry-monitor', '~> 0.3'
-  spec.add_dependency 'dry-validation', '~> 1.2'
-  spec.add_dependency 'envlogic', '~> 1.1'
-  spec.add_dependency 'irb', '~> 1.0'
-  spec.add_dependency 'rake', '>= 11.3'
-  spec.add_dependency 'ruby-kafka', '>= 1.0.0'
+  spec.licenses    = ['LGPL-3.0', 'Commercial']
+
+  spec.add_dependency 'karafka-core', '>= 2.0.2', '< 3.0.0'
+  spec.add_dependency 'rdkafka', '>= 0.12'
   spec.add_dependency 'thor', '>= 0.20'
-  spec.add_dependency 'waterdrop', '~> 1.4.0'
-  spec.add_dependency 'zeitwerk', '~> 2.1'
+  spec.add_dependency 'waterdrop', '>= 2.4.1', '< 3.0.0'
+  spec.add_dependency 'zeitwerk', '~> 2.3'
 
-  spec.required_ruby_version = '>= 2.5.0'
+  spec.required_ruby_version = '>= 2.7.0'
 
   if $PROGRAM_NAME.end_with?('gem')
     spec.signing_key = File.expand_path('~/.ssh/gem-private_key.pem')
   end
 
-  spec.cert_chain    = %w[certs/mensfeld.pem]
+  spec.cert_chain    = %w[certs/cert_chain.pem]
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec)/}) }
-  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.executables   = %w[karafka]
   spec.require_paths = %w[lib]
+
+  spec.metadata = {
+    'source_code_uri' => 'https://github.com/karafka/karafka',
+    'rubygems_mfa_required' => 'true'
+  }
 end
-# rubocop:enable Metrics/BlockLength

data/lib/active_job/karafka.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+begin
+  require 'active_job'
+  require_relative 'queue_adapters/karafka_adapter'
+
+  module ActiveJob
+    # Namespace for usage simplification outside of Rails where Railtie will not kick in.
+    # That way a require 'active_job/karafka' should be enough to use it
+    module Karafka
+    end
+  end
+
+  # We extend routing builder by adding a simple wrapper for easier jobs topics defining
+  # This needs to be extended here as it is going to be used in karafka routes, hence doing that in
+  # the railtie initializer would be too late
+  ::Karafka::Routing::Builder.include ::Karafka::ActiveJob::Routing::Extensions
+  ::Karafka::Routing::Proxy.include ::Karafka::ActiveJob::Routing::Extensions
+rescue LoadError
+  # We extend ActiveJob stuff in the railtie
+end

data/lib/active_job/queue_adapters/karafka_adapter.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+# ActiveJob components to allow for jobs consumption with Karafka
+module ActiveJob
+  # ActiveJob queue adapters
+  module QueueAdapters
+    # Karafka adapter for enqueuing jobs
+    # This is here for ease of integration with ActiveJob.
+    class KarafkaAdapter
+      # Enqueues the job using the configured dispatcher
+      #
+      # @param job [Object] job that should be enqueued
+      def enqueue(job)
+        ::Karafka::App.config.internal.active_job.dispatcher.call(job)
+      end
+
+      # Raises info, that Karafka backend does not support scheduling jobs
+      #
+      # @param _job [Object] job we cannot enqueue
+      # @param _timestamp [Time] time when job should run
+      def enqueue_at(_job, _timestamp)
+        raise NotImplementedError, 'This queueing backend does not support scheduling jobs.'
+      end
+    end
+  end
+end

data/lib/karafka/active_job/consumer.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Karafka
+  module ActiveJob
+    # This is the consumer for ActiveJob that eats the messages enqueued with it one after another.
+    # It marks the offset after each message, so we make sure, none of the jobs is executed twice
+    class Consumer < ::Karafka::BaseConsumer
+      # Executes the ActiveJob logic
+      # @note ActiveJob does not support batches, so we just run one message after another
+      def consume
+        messages.each do |message|
+          break if Karafka::App.stopping?
+
+          ::ActiveJob::Base.execute(
+            # We technically speaking could set this as deserializer and reference it from the
+            # message instead of using the `#raw_payload`. This is not done on purpose to simplify
+            # the ActiveJob setup here
+            ::ActiveSupport::JSON.decode(message.raw_payload)
+          )
+
+          mark_as_consumed(message)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/active_job/dispatcher.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Karafka
+  module ActiveJob
+    # Dispatcher that sends the ActiveJob job to a proper topic based on the queue name
+    class Dispatcher
+      # Defaults for dispatching
+      # The can be updated by using `#karafka_options` on the job
+      DEFAULTS = {
+        dispatch_method: :produce_async
+      }.freeze
+
+      private_constant :DEFAULTS
+
+      # @param job [ActiveJob::Base] job
+      def call(job)
+        ::Karafka.producer.public_send(
+          fetch_option(job, :dispatch_method, DEFAULTS),
+          topic: job.queue_name,
+          payload: ::ActiveSupport::JSON.encode(job.serialize)
+        )
+      end
+
+      private
+
+      # @param job [ActiveJob::Base] job
+      # @param key [Symbol] key we want to fetch
+      # @param defaults [Hash]
+      # @return [Object] options we are interested in
+      def fetch_option(job, key, defaults)
+        job
+          .class
+          .karafka_options
+          .fetch(key, defaults.fetch(key))
+      end
+    end
+  end
+end

data/lib/karafka/active_job/job_extensions.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Karafka
+  module ActiveJob
+    # Allows for setting karafka specific options in ActiveJob jobs
+    module JobExtensions
+      class << self
+        # Defines all the needed accessors and sets defaults
+        # @param klass [ActiveJob::Base] active job base
+        def extended(klass)
+          klass.class_attribute :_karafka_options
+          klass._karafka_options = {}
+        end
+      end
+
+      # @param new_options [Hash] additional options that allow for jobs Karafka related options
+      #   customization
+      # @return [Hash] karafka options
+      def karafka_options(new_options = {})
+        return _karafka_options if new_options.empty?
+
+        # Make sure, that karafka options that someone wants to use are valid before assigning
+        # them
+        App.config.internal.active_job.job_options_contract.validate!(new_options)
+
+        new_options.each do |name, value|
+          _karafka_options[name] = value
+        end
+
+        _karafka_options
+      end
+    end
+  end
+end

data/lib/karafka/active_job/job_options_contract.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Karafka
+  module ActiveJob
+    # Contract for validating the options that can be altered with `#karafka_options` per job class
+    # @note We keep this in the `Karafka::ActiveJob` namespace instead of `Karafka::Contracts` as
+    #   we want to keep ActiveJob related Karafka components outside of the core Karafka code and
+    #   all in the same place
+    class JobOptionsContract < Contracts::Base
+      configure do |config|
+        config.error_messages = YAML.safe_load(
+          File.read(
+            File.join(Karafka.gem_root, 'config', 'errors.yml')
+          )
+        ).fetch('en').fetch('validations').fetch('job_options')
+      end
+
+      optional(:dispatch_method) { |val| %i[produce_async produce_sync].include?(val) }
+    end
+  end
+end

data/lib/karafka/active_job/routing/extensions.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Karafka
+  # ActiveJob related Karafka stuff
+  module ActiveJob
+    # Karafka routing ActiveJob related components
+    module Routing
+      # Routing extensions for ActiveJob
+      module Extensions
+        # This method simplifies routes definition for ActiveJob topics / queues by auto-injecting
+        # the consumer class
+        # @param name [String, Symbol] name of the topic where ActiveJobs jobs should go
+        # @param block [Proc] block that we can use for some extra configuration
+        def active_job_topic(name, &block)
+          topic(name) do
+            consumer App.config.internal.active_job.consumer_class
+
+            next unless block
+
+            instance_eval(&block)
+
+            target.tags << :active_job
+
+            # This is handled by our custom ActiveJob consumer
+            # Without this, default behaviour would cause messages to skip upon shutdown as the
+            # offset would be committed for the last message
+            manual_offset_management true
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/admin.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Simple admin actions that we can perform via Karafka on our Kafka cluster
+  #
+  # @note It always initializes a new admin instance as we want to ensure it is always closed
+  #   Since admin actions are not performed that often, that should be ok.
+  #
+  # @note It always uses the primary defined cluster and does not support multi-cluster work.
+  #   If you need this, just replace the cluster info for the time you use this
+  module Admin
+    class << self
+      # Creates Kafka topic with given settings
+      #
+      # @param name [String] topic name
+      # @param partitions [Integer] number of partitions we expect
+      # @param replication_factor [Integer] number of replicas
+      # @param topic_config [Hash] topic config details as described here:
+      #   https://kafka.apache.org/documentation/#topicconfigs
+      def create_topic(name, partitions, replication_factor, topic_config = {})
+        with_admin do |admin|
+          admin.create_topic(name, partitions, replication_factor, topic_config)
+
+          sleep(0.1) until topics_names.include?(name)
+        end
+      end
+
+      # Deleted a given topic
+      #
+      # @param name [String] topic name
+      def delete_topic(name)
+        with_admin do |admin|
+          admin.delete_topic(name)
+
+          sleep(0.1) while topics_names.include?(name)
+        end
+      end
+
+      # @return [Rdkafka::Metadata] cluster metadata info
+      def cluster_info
+        with_admin do |admin|
+          Rdkafka::Metadata.new(admin.instance_variable_get('@native_kafka'))
+        end
+      end
+
+      private
+
+      # @return [Array<String>] topics names
+      def topics_names
+        cluster_info.topics.map { |topic| topic.fetch(:topic_name) }
+      end
+
+      # Creates admin instance and yields it. After usage it closes the admin instance
+      def with_admin
+        admin = ::Rdkafka::Config.new(Karafka::App.config.kafka).admin
+        result = yield(admin)
+        result
+      ensure
+        admin&.close
+      end
+    end
+  end
+end

data/lib/karafka/app.rb

@@ -6,31 +6,24 @@ module Karafka
     extend Setup::Dsl
 
     class << self
-      # Sets up all the internal components and bootstrap whole app
-      # We need to know details about consumers in order to setup components,
-      # that's why we don't setup them after std setup is done
-      # @raise [Karafka::Errors::InvalidConfigurationError] raised when configuration
-      #   doesn't match with the config contract
-      def boot!
-        initialize!
-        Setup::Config.validate!
-        Setup::Config.setup_components
-        initialized!
-      end
-
-      # @return [Karafka::Routing::Builder] consumers builder instance
+      # @return [Karafka::Routing::Builder] consumers builder instance alias
       def consumer_groups
-        config.internal.routing_builder
+        config
+          .internal
+          .routing
+          .builder
       end
 
-      # Triggers reload of all cached Karafka app components, so we can use in-process
-      # in-development hot code reloading without Karafka process restart
-      def reload
-        Karafka::Persistence::Consumers.clear
-        Karafka::Persistence::Topics.clear
-        config.internal.routing_builder.reload
+      # @return [Array<Karafka::Routing::SubscriptionGroup>] active subscription groups
+      def subscription_groups
+        consumer_groups
+          .active
+          .flat_map(&:subscription_groups)
       end
 
+      # Just a nicer name for the consumer groups
+      alias routes consumer_groups
+
       Status.instance_methods(false).each do |delegated|
         define_method(delegated) do
           App.config.internal.status.send(delegated)
@@ -42,7 +35,9 @@ module Karafka
         root
         env
         logger
+        producer
         monitor
+        pro?
       ].each do |delegated|
         define_method(delegated) do
           Karafka.send(delegated)