karafka 2.0.0.beta5 → 2.0.0.rc3

data/lib/karafka/processing/coordinators_buffer.rb

@@ -2,7 +2,7 @@
 
 module Karafka
   module Processing
-    # Buffer used to build and store coordinators per topic partition
+    # Coordinators builder used to build coordinators per topic partition
     #
     # It provides direct pauses access for revocation
     #
@@ -34,17 +34,13 @@ module Karafka
       # @param topic [String] topic name
       # @param partition [Integer] partition number
       def revoke(topic, partition)
-        @pauses_manager.revoke(topic, partition)
+        return unless @coordinators[topic].key?(partition)
 
         # The fact that we delete here does not change the fact that the executor still holds the
         # reference to this coordinator. We delete it here, as we will no longer process any
         # new stuff with it and we may need a new coordinator if we regain this partition, but the
         # coordinator may still be in use
-        coordinator = @coordinators[topic].delete(partition)
-
-        return unless coordinator
-
-        coordinator.revoke
+        @coordinators[topic].delete(partition).revoke
       end
 
       # Clears coordinators and re-created the pauses manager

data/lib/karafka/processing/executor.rb

@@ -71,7 +71,7 @@ module Karafka
 
       # Runs consumer after consumption code
       def after_consume
-        consumer.on_after_consume if @consumer
+        consumer.on_after_consume
       end
 
       # Runs the controller `#revoked` method that should be triggered when a given consumer is

data/lib/karafka/processing/jobs_queue.rb

@@ -119,6 +119,17 @@ module Karafka
         @semaphores[group_id].pop while wait?(group_id)
       end
 
+      # - `processing` - number of jobs that are currently being processed (active work)
+      # - `enqueued` - number of jobs in the queue that are waiting to be picked up by a worker
+      #
+      # @return [Hash] hash with basic usage statistics of this queue.
+      def statistics
+        {
+          processing: size - @queue.size,
+          enqueued: @queue.size
+        }.freeze
+      end
+
       private
 
       # @param group_id [String] id of the group in which jobs we're interested.

data/lib/karafka/processing/partitioner.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Basic partitioner for work division
+    # It does not divide any work.
+    class Partitioner
+      # @param subscription_group [Karafka::Routing::SubscriptionGroup] subscription group
+      def initialize(subscription_group)
+        @subscription_group = subscription_group
+      end
+
+      # @param _topic [String] topic name
+      # @param messages [Array<Karafka::Messages::Message>] karafka messages
+      # @yieldparam [Integer] group id
+      # @yieldparam [Array<Karafka::Messages::Message>] karafka messages
+      def call(_topic, messages)
+        yield(0, messages)
+      end
+    end
+  end
+end

data/lib/karafka/processing/worker.rb

@@ -47,9 +47,11 @@ module Karafka
         job = @jobs_queue.pop
 
         if job
-          Karafka.monitor.instrument('worker.process', caller: self, job: job)
+          instrument_details = { caller: self, job: job, jobs_queue: @jobs_queue }
 
-          Karafka.monitor.instrument('worker.processed', caller: self, job: job) do
+          Karafka.monitor.instrument('worker.process', instrument_details)
+
+          Karafka.monitor.instrument('worker.processed', instrument_details) do
             job.before_call
 
             # If a job is marked as non blocking, we can run a tick in the job queue and if there

data/lib/karafka/setup/config.rb

@@ -12,7 +12,7 @@ module Karafka
     #   enough and will still keep the code simple
     # @see Karafka::Setup::Configurators::Base for more details about configurators api
     class Config
-      extend Dry::Configurable
+      extend ::WaterDrop::Configurable
 
       # Defaults for kafka settings, that will be overwritten only if not present already
       KAFKA_DEFAULTS = {
@@ -60,9 +60,9 @@ module Karafka
       # option [Boolean] should we leave offset management to the user
       setting :manual_offset_management, default: false
       # options max_messages [Integer] how many messages do we want to fetch from Kafka in one go
-      setting :max_messages, default: 1_000
+      setting :max_messages, default: 100
       # option [Integer] number of milliseconds we can wait while fetching data
-      setting :max_wait_time, default: 10_000
+      setting :max_wait_time, default: 1_000
       # option shutdown_timeout [Integer] the number of milliseconds after which Karafka no
       #   longer waits for the consumers to stop gracefully but instead we force terminate
       #   everything.
@@ -107,6 +107,8 @@ module Karafka
           setting :jobs_builder, default: Processing::JobsBuilder.new
           # option coordinator [Class] work coordinator we want to user for processing coordination
           setting :coordinator_class, default: Processing::Coordinator
+          # option partitioner_class [Class] partitioner we use against a batch of data
+          setting :partitioner_class, default: Processing::Partitioner
         end
 
         # Karafka components for ActiveJob
@@ -121,6 +123,10 @@ module Karafka
         end
       end
 
+      # This will load all the defaults that can be later overwritten.
+      # Thanks to that we have an initial state out of the box.
+      configure
+
       class << self
         # Configuring method
         # @param block [Proc] block we want to execute with the config instance

data/lib/karafka/templates/example_consumer.rb.erb

@@ -7,10 +7,10 @@ class ExampleConsumer < ApplicationConsumer
   end
 
   # Run anything upon partition being revoked
-  # def on_revoked
+  # def revoked
   # end
 
   # Define here any teardown things you want when Karafka server stops
-  # def on_shutdown
+  # def shutdown
   # end
 end

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.0.0.beta5'
+  VERSION = '2.0.0.rc3'
 end

data/lib/karafka.rb

@@ -12,8 +12,6 @@
   openssl
   base64
   date
-  dry-configurable
-  dry-validation
   dry/events/publisher
   dry/monitor/notifications
   zeitwerk
@@ -90,5 +88,7 @@ loader = Zeitwerk::Loader.for_gem
 loader.ignore(Karafka.gem_root.join('lib/active_job'))
 # Do not load pro components, this will be handled by license manager
 loader.ignore(Karafka.gem_root.join('lib/karafka/pro'))
+# Do not load vendors instrumentation components. Those need to be required manually if needed
+loader.ignore(Karafka.gem_root.join('lib/karafka/instrumentation/vendors'))
 loader.setup
 loader.eager_load

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.0.0.beta5
+  version: 2.0.0.rc3
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -34,22 +34,8 @@ cert_chain:
   R2P11bWoCtr70BsccVrN8jEhzwXngMyI2gVt750Y+dbTu1KgRqZKp/ECe7ZzPzXj
   pIy9vHxTANKYVyI4qj8OrFdEM5BQNu8oQpL0iQ==
   -----END CERTIFICATE-----
-date: 2022-07-05 00:00:00.000000000 Z
+date: 2022-07-26 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: dry-configurable
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.13'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.13'
 - !ruby/object:Gem::Dependency
   name: dry-monitor
   requirement: !ruby/object:Gem::Requirement
@@ -64,20 +50,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.5'
-- !ruby/object:Gem::Dependency
-  name: dry-validation
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.7'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.7'
 - !ruby/object:Gem::Dependency
   name: rdkafka
   requirement: !ruby/object:Gem::Requirement
@@ -112,7 +84,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.3.1
+        version: 2.3.3
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -122,7 +94,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.3.1
+        version: 2.3.3
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -212,6 +184,7 @@ files:
 - lib/karafka/env.rb
 - lib/karafka/errors.rb
 - lib/karafka/helpers/async.rb
+- lib/karafka/helpers/colorize.rb
 - lib/karafka/helpers/multi_delegator.rb
 - lib/karafka/instrumentation.rb
 - lib/karafka/instrumentation/callbacks/error.rb
@@ -220,6 +193,8 @@ files:
 - lib/karafka/instrumentation/logger_listener.rb
 - lib/karafka/instrumentation/monitor.rb
 - lib/karafka/instrumentation/proctitle_listener.rb
+- lib/karafka/instrumentation/vendors/datadog/dashboard.json
+- lib/karafka/instrumentation/vendors/datadog/listener.rb
 - lib/karafka/licenser.rb
 - lib/karafka/messages/batch_metadata.rb
 - lib/karafka/messages/builders/batch_metadata.rb
@@ -235,13 +210,18 @@ files:
 - lib/karafka/pro/active_job/dispatcher.rb
 - lib/karafka/pro/active_job/job_options_contract.rb
 - lib/karafka/pro/base_consumer.rb
+- lib/karafka/pro/contracts/base.rb
+- lib/karafka/pro/contracts/consumer_group.rb
+- lib/karafka/pro/contracts/consumer_group_topic.rb
 - lib/karafka/pro/loader.rb
 - lib/karafka/pro/performance_tracker.rb
 - lib/karafka/pro/processing/coordinator.rb
 - lib/karafka/pro/processing/jobs/consume_non_blocking.rb
 - lib/karafka/pro/processing/jobs_builder.rb
+- lib/karafka/pro/processing/partitioner.rb
 - lib/karafka/pro/processing/scheduler.rb
-- lib/karafka/pro/routing/extensions.rb
+- lib/karafka/pro/routing/builder_extensions.rb
+- lib/karafka/pro/routing/topic_extensions.rb
 - lib/karafka/process.rb
 - lib/karafka/processing/coordinator.rb
 - lib/karafka/processing/coordinators_buffer.rb
@@ -253,6 +233,7 @@ files:
 - lib/karafka/processing/jobs/shutdown.rb
 - lib/karafka/processing/jobs_builder.rb
 - lib/karafka/processing/jobs_queue.rb
+- lib/karafka/processing/partitioner.rb
 - lib/karafka/processing/result.rb
 - lib/karafka/processing/scheduler.rb
 - lib/karafka/processing/worker.rb
@@ -306,5 +287,5 @@ requirements: []
 rubygems_version: 3.3.7
 signing_key: 
 specification_version: 4
-summary: Ruby based framework for working with Apache Kafka
+summary: Efficient Kafka processing framework for Ruby and Rails
 test_files: []