karafka 1.4.13 → 2.0.0

metadata

@@ -1,12 +1,10 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 1.4.13
+  version: 2.0.0
 platform: ruby
 authors:
 - Maciej Mensfeld
-- Pavlo Vavruk
-- Adam Gwozdowski
 autorequire: 
 bindir: bin
 cert_chain:
@@ -36,139 +34,93 @@ cert_chain:
   R2P11bWoCtr70BsccVrN8jEhzwXngMyI2gVt750Y+dbTu1KgRqZKp/ECe7ZzPzXj
   pIy9vHxTANKYVyI4qj8OrFdEM5BQNu8oQpL0iQ==
   -----END CERTIFICATE-----
-date: 2022-02-19 00:00:00.000000000 Z
+date: 2022-08-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: dry-configurable
+  name: karafka-core
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.13'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.13'
-- !ruby/object:Gem::Dependency
-  name: dry-inflector
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.2'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.2'
-- !ruby/object:Gem::Dependency
-  name: dry-monitor
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.5'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.5'
-- !ruby/object:Gem::Dependency
-  name: dry-validation
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
+        version: 2.0.2
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '1.7'
+        version: 3.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.7'
-- !ruby/object:Gem::Dependency
-  name: envlogic
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.1'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
+        version: 2.0.2
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '1.1'
+        version: 3.0.0
 - !ruby/object:Gem::Dependency
-  name: ruby-kafka
+  name: rdkafka
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: '0.12'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: '0.12'
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '1.1'
+        version: '0.20'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '1.1'
+        version: '0.20'
 - !ruby/object:Gem::Dependency
   name: waterdrop
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.4.1
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '1.4'
+        version: 3.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.4.1
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '1.4'
+        version: 3.0.0
 - !ruby/object:Gem::Dependency
   name: zeitwerk
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.4'
+        version: '2.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.4'
+        version: '2.3'
 description: Framework used to simplify Apache Kafka based Ruby applications development
 email:
 - maciej@mensfeld.pl
-- pavlo.vavruk@gmail.com
-- adam99g@gmail.com
 executables:
 - karafka
 extensions: []
@@ -189,92 +141,138 @@ files:
 - CONTRIBUTING.md
 - Gemfile
 - Gemfile.lock
-- MIT-LICENCE
+- LICENSE
+- LICENSE-COMM
+- LICENSE-LGPL
 - README.md
+- bin/benchmarks
+- bin/create_token
+- bin/integrations
 - bin/karafka
+- bin/scenario
+- bin/stress_many
+- bin/stress_one
+- bin/wait_for_kafka
+- certs/karafka-pro.pem
 - certs/mensfeld.pem
 - config/errors.yml
 - docker-compose.yml
 - karafka.gemspec
+- lib/active_job/karafka.rb
+- lib/active_job/queue_adapters/karafka_adapter.rb
 - lib/karafka.rb
+- lib/karafka/active_job/consumer.rb
+- lib/karafka/active_job/dispatcher.rb
+- lib/karafka/active_job/job_extensions.rb
+- lib/karafka/active_job/job_options_contract.rb
+- lib/karafka/active_job/routing/extensions.rb
 - lib/karafka/app.rb
-- lib/karafka/assignment_strategies/round_robin.rb
-- lib/karafka/attributes_map.rb
-- lib/karafka/backends/inline.rb
 - lib/karafka/base_consumer.rb
-- lib/karafka/base_responder.rb
 - lib/karafka/cli.rb
 - lib/karafka/cli/base.rb
 - lib/karafka/cli/console.rb
-- lib/karafka/cli/flow.rb
 - lib/karafka/cli/info.rb
 - lib/karafka/cli/install.rb
-- lib/karafka/cli/missingno.rb
 - lib/karafka/cli/server.rb
-- lib/karafka/code_reloader.rb
-- lib/karafka/connection/api_adapter.rb
-- lib/karafka/connection/batch_delegator.rb
-- lib/karafka/connection/builder.rb
 - lib/karafka/connection/client.rb
 - lib/karafka/connection/listener.rb
-- lib/karafka/connection/message_delegator.rb
-- lib/karafka/consumers/batch_metadata.rb
-- lib/karafka/consumers/callbacks.rb
-- lib/karafka/consumers/includer.rb
-- lib/karafka/consumers/responders.rb
-- lib/karafka/consumers/single_params.rb
+- lib/karafka/connection/listeners_batch.rb
+- lib/karafka/connection/messages_buffer.rb
+- lib/karafka/connection/pauses_manager.rb
+- lib/karafka/connection/raw_messages_buffer.rb
+- lib/karafka/connection/rebalance_manager.rb
 - lib/karafka/contracts.rb
+- lib/karafka/contracts/base.rb
 - lib/karafka/contracts/config.rb
 - lib/karafka/contracts/consumer_group.rb
 - lib/karafka/contracts/consumer_group_topic.rb
-- lib/karafka/contracts/responder_usage.rb
 - lib/karafka/contracts/server_cli_options.rb
+- lib/karafka/env.rb
 - lib/karafka/errors.rb
-- lib/karafka/fetcher.rb
-- lib/karafka/helpers/class_matcher.rb
-- lib/karafka/helpers/config_retriever.rb
-- lib/karafka/helpers/inflector.rb
+- lib/karafka/helpers/async.rb
+- lib/karafka/helpers/colorize.rb
 - lib/karafka/helpers/multi_delegator.rb
+- lib/karafka/instrumentation.rb
+- lib/karafka/instrumentation/callbacks/error.rb
+- lib/karafka/instrumentation/callbacks/statistics.rb
 - lib/karafka/instrumentation/logger.rb
+- lib/karafka/instrumentation/logger_listener.rb
 - lib/karafka/instrumentation/monitor.rb
+- lib/karafka/instrumentation/notifications.rb
 - lib/karafka/instrumentation/proctitle_listener.rb
-- lib/karafka/instrumentation/stdout_listener.rb
-- lib/karafka/params/batch_metadata.rb
-- lib/karafka/params/builders/batch_metadata.rb
-- lib/karafka/params/builders/params.rb
-- lib/karafka/params/builders/params_batch.rb
-- lib/karafka/params/metadata.rb
-- lib/karafka/params/params.rb
-- lib/karafka/params/params_batch.rb
-- lib/karafka/patches/ruby_kafka.rb
-- lib/karafka/persistence/client.rb
-- lib/karafka/persistence/consumers.rb
-- lib/karafka/persistence/topics.rb
+- lib/karafka/instrumentation/vendors/datadog/dashboard.json
+- lib/karafka/instrumentation/vendors/datadog/listener.rb
+- lib/karafka/licenser.rb
+- lib/karafka/messages/batch_metadata.rb
+- lib/karafka/messages/builders/batch_metadata.rb
+- lib/karafka/messages/builders/message.rb
+- lib/karafka/messages/builders/messages.rb
+- lib/karafka/messages/message.rb
+- lib/karafka/messages/messages.rb
+- lib/karafka/messages/metadata.rb
+- lib/karafka/messages/seek.rb
+- lib/karafka/patches/rdkafka/consumer.rb
+- lib/karafka/pro.rb
+- lib/karafka/pro/active_job/consumer.rb
+- lib/karafka/pro/active_job/dispatcher.rb
+- lib/karafka/pro/active_job/job_options_contract.rb
+- lib/karafka/pro/base_consumer.rb
+- lib/karafka/pro/contracts/base.rb
+- lib/karafka/pro/contracts/consumer_group.rb
+- lib/karafka/pro/contracts/consumer_group_topic.rb
+- lib/karafka/pro/loader.rb
+- lib/karafka/pro/performance_tracker.rb
+- lib/karafka/pro/processing/coordinator.rb
+- lib/karafka/pro/processing/jobs/consume_non_blocking.rb
+- lib/karafka/pro/processing/jobs_builder.rb
+- lib/karafka/pro/processing/partitioner.rb
+- lib/karafka/pro/processing/scheduler.rb
+- lib/karafka/pro/routing/builder_extensions.rb
+- lib/karafka/pro/routing/topic_extensions.rb
 - lib/karafka/process.rb
-- lib/karafka/responders/builder.rb
-- lib/karafka/responders/topic.rb
+- lib/karafka/processing/coordinator.rb
+- lib/karafka/processing/coordinators_buffer.rb
+- lib/karafka/processing/executor.rb
+- lib/karafka/processing/executors_buffer.rb
+- lib/karafka/processing/jobs/base.rb
+- lib/karafka/processing/jobs/consume.rb
+- lib/karafka/processing/jobs/revoked.rb
+- lib/karafka/processing/jobs/shutdown.rb
+- lib/karafka/processing/jobs_builder.rb
+- lib/karafka/processing/jobs_queue.rb
+- lib/karafka/processing/partitioner.rb
+- lib/karafka/processing/result.rb
+- lib/karafka/processing/scheduler.rb
+- lib/karafka/processing/worker.rb
+- lib/karafka/processing/workers_batch.rb
+- lib/karafka/railtie.rb
 - lib/karafka/routing/builder.rb
 - lib/karafka/routing/consumer_group.rb
 - lib/karafka/routing/consumer_mapper.rb
 - lib/karafka/routing/proxy.rb
 - lib/karafka/routing/router.rb
+- lib/karafka/routing/subscription_group.rb
+- lib/karafka/routing/subscription_groups_builder.rb
 - lib/karafka/routing/topic.rb
-- lib/karafka/routing/topic_mapper.rb
+- lib/karafka/routing/topics.rb
+- lib/karafka/runner.rb
 - lib/karafka/serialization/json/deserializer.rb
-- lib/karafka/serialization/json/serializer.rb
 - lib/karafka/server.rb
 - lib/karafka/setup/config.rb
-- lib/karafka/setup/configurators/water_drop.rb
 - lib/karafka/setup/dsl.rb
 - lib/karafka/status.rb
 - lib/karafka/templates/application_consumer.rb.erb
-- lib/karafka/templates/application_responder.rb.erb
+- lib/karafka/templates/example_consumer.rb.erb
 - lib/karafka/templates/karafka.rb.erb
+- lib/karafka/time_trackers/base.rb
+- lib/karafka/time_trackers/pause.rb
+- lib/karafka/time_trackers/poll.rb
 - lib/karafka/version.rb
 - log/.gitkeep
 homepage: https://karafka.io
 licenses:
-- MIT
+- LGPL-3.0
+- Commercial
 metadata:
   source_code_uri: https://github.com/karafka/karafka
   rubygems_mfa_required: 'true'
@@ -286,15 +284,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.7'
+      version: 2.7.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.3
+rubygems_version: 3.3.7
 signing_key: 
 specification_version: 4
-summary: Ruby based framework for working with Apache Kafka
+summary: Efficient Kafka processing framework for Ruby and Rails
 test_files: []

data/MIT-LICENCE

@@ -1,18 +0,0 @@
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/lib/karafka/assignment_strategies/round_robin.rb

@@ -1,13 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Strategies for Kafka partitions assignments
-  module AssignmentStrategies
-    # Standard RoundRobin strategy
-    class RoundRobin < SimpleDelegator
-      def initialize
-        super(Kafka::RoundRobinAssignmentStrategy.new)
-      end
-    end
-  end
-end

data/lib/karafka/attributes_map.rb

@@ -1,63 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Both Karafka and Ruby-Kafka contain a lot of settings that can be applied on multiple
-  # levels. In Karafka that is on consumer group and on the topic level. In Ruby-Kafka it
-  # is on consumer, subscription and consumption levels. In order to maintain an order
-  # in managing those settings, this module was created. It contains details on what setting
-  # where should go and which layer (both on Karafka and Ruby-Kafka) is responsible for
-  # setting it and sending it forward
-  # @note Settings presented here cover all the settings that are being used across Karafka
-  module AttributesMap
-    class << self
-      # What settings should go where in ruby-kafka
-      # @return [Hash] hash with proper sections on what to proxy where in Ruby-Kafka
-      # @note All other settings will be passed to Kafka.new method invocation.
-      #   All elements in this hash are just edge cases
-      def api_adapter
-        {
-          consumer: %i[
-            session_timeout offset_commit_interval offset_commit_threshold
-            offset_retention_time heartbeat_interval fetcher_max_queue_size
-            assignment_strategy
-          ],
-          subscribe: %i[start_from_beginning max_bytes_per_partition],
-          consumption: %i[min_bytes max_bytes max_wait_time],
-          pause: %i[pause_timeout pause_max_timeout pause_exponential_backoff],
-          # All the options that are under kafka config namespace, but are not used
-          # directly with kafka api, but from the Karafka user perspective, they are
-          # still related to kafka. They should not be proxied anywhere
-          ignored: %i[reconnect_timeout automatically_mark_as_consumed]
-        }
-      end
-
-      # @return [Array<Symbol>] properties that can be set on a per topic level
-      def topic
-        (api_adapter[:subscribe] + %i[
-          backend
-          name
-          deserializer
-          responder
-          batch_consuming
-        ]).uniq
-      end
-
-      # @return [Array<Symbol>] properties that can be set on a per consumer group level
-      # @note Note that there are settings directly extracted from the config kafka namespace
-      #   I did this that way, so I won't have to repeat same setting keys over and over again
-      #   Thanks to this solution, if any new setting is available for ruby-kafka, we just need
-      #   to add it to our configuration class and it will be handled automatically.
-      def consumer_group
-        # @note We don't ignore the api_adapter[:ignored] values as they should be ignored
-        #   only when proxying details go ruby-kafka. We use ignored fields internally in karafka
-        ignored_settings = api_adapter[:subscribe]
-        defined_settings = api_adapter.values.flatten
-        karafka_settings = %i[batch_fetching]
-
-        dynamically_proxied = Karafka::Setup::Config.config.kafka.to_h.keys
-
-        (defined_settings + dynamically_proxied).uniq + karafka_settings - ignored_settings
-      end
-    end
-  end
-end

data/lib/karafka/backends/inline.rb

@@ -1,16 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Namespace for all different backends Karafka supports
-  module Backends
-    # Backend that just runs stuff asap without any scheduling
-    module Inline
-      private
-
-      # Executes consume code immediately (without enqueuing)
-      def process
-        Karafka.monitor.instrument('backends.inline.process', caller: self) { consume }
-      end
-    end
-  end
-end