karafka 1.4.9 → 2.0.0.alpha1

data/lib/karafka.rb

@@ -3,32 +3,48 @@
 %w[
   delegate
   English
+  rdkafka
   waterdrop
-  kafka
-  envlogic
   json
   thor
   forwardable
   fileutils
+  openssl
+  base64
+  date
   dry-configurable
   dry-validation
   dry/events/publisher
-  dry/inflector
   dry/monitor/notifications
-  dry/core/constants
   zeitwerk
 ].each(&method(:require))
 
-# Karafka library
+# Karafka framework main namespace
 module Karafka
-  extend Envlogic
-
   class << self
+    # @return [Karafka::Env] env instance that allows us to check environment
+    def env
+      @env ||= Env.new
+    end
+
+    # @param environment [String, Symbol] new environment that we want to set
+    # @return [Karafka::Env] env instance
+    # @example Assign new environment to Karafka::App
+    #   Karafka::App.env = :production
+    def env=(environment)
+      env.replace(environment.to_s)
+    end
+
     # @return [Logger] logger that we want to use. Will use ::Karafka::Logger by default
     def logger
       @logger ||= App.config.logger
     end
 
+    # @return [WaterDrop::Producer] waterdrop messages producer
+    def producer
+      @producer ||= App.config.producer
+    end
+
     # @return [::Karafka::Monitor] monitor that we want to use
     def monitor
       @monitor ||= App.config.monitor
@@ -64,9 +80,10 @@ module Karafka
   end
 end
 
-Zeitwerk::Loader
-  .for_gem
-  .tap(&:setup)
-  .tap(&:eager_load)
-
-Kafka::Consumer.prepend(Karafka::Patches::RubyKafka)
+loader = Zeitwerk::Loader.for_gem
+# Do not load Rails extensions by default, this will be handled by Railtie if they are needed
+loader.do_not_eager_load(Karafka.gem_root.join('lib/active_job'))
+# Do not load pro components, this will be handled by license manager
+loader.do_not_eager_load(Karafka.gem_root.join('lib/karafka/pro'))
+loader.setup
+loader.eager_load

metadata

@@ -1,12 +1,10 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 1.4.9
+  version: 2.0.0.alpha1
 platform: ruby
 authors:
 - Maciej Mensfeld
-- Pavlo Vavruk
-- Adam Gwozdowski
 autorequire: 
 bindir: bin
 cert_chain:
@@ -36,7 +34,7 @@ cert_chain:
   R2P11bWoCtr70BsccVrN8jEhzwXngMyI2gVt750Y+dbTu1KgRqZKp/ECe7ZzPzXj
   pIy9vHxTANKYVyI4qj8OrFdEM5BQNu8oQpL0iQ==
   -----END CERTIFICATE-----
-date: 2021-09-29 00:00:00.000000000 Z
+date: 2022-01-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dry-configurable
@@ -44,98 +42,56 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: '0.13'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.8'
-- !ruby/object:Gem::Dependency
-  name: dry-inflector
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.1'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.1'
+        version: '0.13'
 - !ruby/object:Gem::Dependency
   name: dry-monitor
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.3'
+        version: '0.5'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.3'
+        version: '0.5'
 - !ruby/object:Gem::Dependency
   name: dry-validation
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.2'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.2'
-- !ruby/object:Gem::Dependency
-  name: envlogic
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.1'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.1'
-- !ruby/object:Gem::Dependency
-  name: irb
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.7'
 - !ruby/object:Gem::Dependency
-  name: ruby-kafka
+  name: rdkafka
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: '0.10'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: '0.10'
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
@@ -154,35 +110,39 @@ dependencies:
   name: waterdrop
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 1.4.0
+        version: 2.1.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 1.4.0
+        version: 2.1.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
 - !ruby/object:Gem::Dependency
   name: zeitwerk
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: '2.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: '2.3'
 description: Framework used to simplify Apache Kafka based Ruby applications development
 email:
 - maciej@mensfeld.pl
-- pavlo.vavruk@gmail.com
-- adam99g@gmail.com
 executables:
 - karafka
 extensions: []
@@ -191,6 +151,7 @@ files:
 - ".coditsu/ci.yml"
 - ".console_irbrc"
 - ".diffend.yml"
+- ".github/FUNDING.yml"
 - ".github/ISSUE_TEMPLATE/bug_report.md"
 - ".github/ISSUE_TEMPLATE/feature_request.md"
 - ".github/workflows/ci.yml"
@@ -203,93 +164,102 @@ files:
 - CONTRIBUTING.md
 - Gemfile
 - Gemfile.lock
-- MIT-LICENCE
+- LICENSE
+- LICENSE-COMM
+- LICENSE-LGPL
 - README.md
+- bin/benchmarks
+- bin/create_token
+- bin/integrations
 - bin/karafka
+- bin/stress
+- certs/karafka-pro.pem
 - certs/mensfeld.pem
 - config/errors.yml
 - docker-compose.yml
 - karafka.gemspec
+- lib/active_job/consumer.rb
+- lib/active_job/karafka.rb
+- lib/active_job/queue_adapters/karafka_adapter.rb
+- lib/active_job/routing_extensions.rb
 - lib/karafka.rb
 - lib/karafka/app.rb
-- lib/karafka/assignment_strategies/round_robin.rb
-- lib/karafka/attributes_map.rb
-- lib/karafka/backends/inline.rb
 - lib/karafka/base_consumer.rb
-- lib/karafka/base_responder.rb
 - lib/karafka/cli.rb
 - lib/karafka/cli/base.rb
 - lib/karafka/cli/console.rb
-- lib/karafka/cli/flow.rb
 - lib/karafka/cli/info.rb
 - lib/karafka/cli/install.rb
-- lib/karafka/cli/missingno.rb
 - lib/karafka/cli/server.rb
-- lib/karafka/code_reloader.rb
-- lib/karafka/connection/api_adapter.rb
-- lib/karafka/connection/batch_delegator.rb
-- lib/karafka/connection/builder.rb
 - lib/karafka/connection/client.rb
 - lib/karafka/connection/listener.rb
-- lib/karafka/connection/message_delegator.rb
-- lib/karafka/consumers/batch_metadata.rb
-- lib/karafka/consumers/callbacks.rb
-- lib/karafka/consumers/includer.rb
-- lib/karafka/consumers/responders.rb
-- lib/karafka/consumers/single_params.rb
+- lib/karafka/connection/messages_buffer.rb
+- lib/karafka/connection/pauses_manager.rb
+- lib/karafka/connection/rebalance_manager.rb
 - lib/karafka/contracts.rb
 - lib/karafka/contracts/config.rb
 - lib/karafka/contracts/consumer_group.rb
 - lib/karafka/contracts/consumer_group_topic.rb
-- lib/karafka/contracts/responder_usage.rb
 - lib/karafka/contracts/server_cli_options.rb
+- lib/karafka/env.rb
 - lib/karafka/errors.rb
-- lib/karafka/fetcher.rb
-- lib/karafka/helpers/class_matcher.rb
-- lib/karafka/helpers/config_retriever.rb
-- lib/karafka/helpers/inflector.rb
 - lib/karafka/helpers/multi_delegator.rb
+- lib/karafka/instrumentation.rb
+- lib/karafka/instrumentation/callbacks/error.rb
+- lib/karafka/instrumentation/callbacks/statistics.rb
 - lib/karafka/instrumentation/logger.rb
 - lib/karafka/instrumentation/monitor.rb
 - lib/karafka/instrumentation/proctitle_listener.rb
 - lib/karafka/instrumentation/stdout_listener.rb
-- lib/karafka/params/batch_metadata.rb
-- lib/karafka/params/builders/batch_metadata.rb
-- lib/karafka/params/builders/params.rb
-- lib/karafka/params/builders/params_batch.rb
-- lib/karafka/params/metadata.rb
-- lib/karafka/params/params.rb
-- lib/karafka/params/params_batch.rb
-- lib/karafka/patches/ruby_kafka.rb
-- lib/karafka/persistence/client.rb
-- lib/karafka/persistence/consumers.rb
-- lib/karafka/persistence/topics.rb
+- lib/karafka/licenser.rb
+- lib/karafka/messages/batch_metadata.rb
+- lib/karafka/messages/builders/batch_metadata.rb
+- lib/karafka/messages/builders/message.rb
+- lib/karafka/messages/builders/messages.rb
+- lib/karafka/messages/message.rb
+- lib/karafka/messages/messages.rb
+- lib/karafka/messages/metadata.rb
+- lib/karafka/messages/seek.rb
+- lib/karafka/patches/rdkafka/consumer.rb
 - lib/karafka/process.rb
-- lib/karafka/responders/builder.rb
-- lib/karafka/responders/topic.rb
+- lib/karafka/processing/executor.rb
+- lib/karafka/processing/executors_buffer.rb
+- lib/karafka/processing/jobs/base.rb
+- lib/karafka/processing/jobs/consume.rb
+- lib/karafka/processing/jobs/revoked.rb
+- lib/karafka/processing/jobs/shutdown.rb
+- lib/karafka/processing/jobs_queue.rb
+- lib/karafka/processing/worker.rb
+- lib/karafka/processing/workers_batch.rb
+- lib/karafka/railtie.rb
 - lib/karafka/routing/builder.rb
 - lib/karafka/routing/consumer_group.rb
 - lib/karafka/routing/consumer_mapper.rb
 - lib/karafka/routing/proxy.rb
 - lib/karafka/routing/router.rb
+- lib/karafka/routing/subscription_group.rb
+- lib/karafka/routing/subscription_groups_builder.rb
 - lib/karafka/routing/topic.rb
-- lib/karafka/routing/topic_mapper.rb
+- lib/karafka/runner.rb
 - lib/karafka/serialization/json/deserializer.rb
-- lib/karafka/serialization/json/serializer.rb
 - lib/karafka/server.rb
 - lib/karafka/setup/config.rb
-- lib/karafka/setup/configurators/water_drop.rb
 - lib/karafka/setup/dsl.rb
 - lib/karafka/status.rb
 - lib/karafka/templates/application_consumer.rb.erb
-- lib/karafka/templates/application_responder.rb.erb
+- lib/karafka/templates/example_consumer.rb.erb
 - lib/karafka/templates/karafka.rb.erb
+- lib/karafka/time_trackers/base.rb
+- lib/karafka/time_trackers/pause.rb
+- lib/karafka/time_trackers/poll.rb
 - lib/karafka/version.rb
 - log/.gitkeep
-homepage: https://github.com/karafka/karafka
+homepage: https://karafka.io
 licenses:
-- MIT
-metadata: {}
+- LGPL-3.0
+metadata:
+  source_code_uri: https://github.com/karafka/karafka
+  rubygems_mfa_required: 'true'
 post_install_message: 
 rdoc_options: []
 require_paths:
@@ -301,11 +271,11 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: 2.6.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
-rubygems_version: 3.2.28
+rubygems_version: 3.3.4
 signing_key: 
 specification_version: 4
 summary: Ruby based framework for working with Apache Kafka

data/MIT-LICENCE

@@ -1,18 +0,0 @@
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/lib/karafka/assignment_strategies/round_robin.rb

@@ -1,13 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Strategies for Kafka partitions assignments
-  module AssignmentStrategies
-    # Standard RoundRobin strategy
-    class RoundRobin < SimpleDelegator
-      def initialize
-        super(Kafka::RoundRobinAssignmentStrategy.new)
-      end
-    end
-  end
-end

data/lib/karafka/attributes_map.rb

@@ -1,63 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Both Karafka and Ruby-Kafka contain a lot of settings that can be applied on multiple
-  # levels. In Karafka that is on consumer group and on the topic level. In Ruby-Kafka it
-  # is on consumer, subscription and consumption levels. In order to maintain an order
-  # in managing those settings, this module was created. It contains details on what setting
-  # where should go and which layer (both on Karafka and Ruby-Kafka) is responsible for
-  # setting it and sending it forward
-  # @note Settings presented here cover all the settings that are being used across Karafka
-  module AttributesMap
-    class << self
-      # What settings should go where in ruby-kafka
-      # @return [Hash] hash with proper sections on what to proxy where in Ruby-Kafka
-      # @note All other settings will be passed to Kafka.new method invocation.
-      #   All elements in this hash are just edge cases
-      def api_adapter
-        {
-          consumer: %i[
-            session_timeout offset_commit_interval offset_commit_threshold
-            offset_retention_time heartbeat_interval fetcher_max_queue_size
-            assignment_strategy
-          ],
-          subscribe: %i[start_from_beginning max_bytes_per_partition],
-          consumption: %i[min_bytes max_bytes max_wait_time],
-          pause: %i[pause_timeout pause_max_timeout pause_exponential_backoff],
-          # All the options that are under kafka config namespace, but are not used
-          # directly with kafka api, but from the Karafka user perspective, they are
-          # still related to kafka. They should not be proxied anywhere
-          ignored: %i[reconnect_timeout automatically_mark_as_consumed]
-        }
-      end
-
-      # @return [Array<Symbol>] properties that can be set on a per topic level
-      def topic
-        (api_adapter[:subscribe] + %i[
-          backend
-          name
-          deserializer
-          responder
-          batch_consuming
-        ]).uniq
-      end
-
-      # @return [Array<Symbol>] properties that can be set on a per consumer group level
-      # @note Note that there are settings directly extracted from the config kafka namespace
-      #   I did this that way, so I won't have to repeat same setting keys over and over again
-      #   Thanks to this solution, if any new setting is available for ruby-kafka, we just need
-      #   to add it to our configuration class and it will be handled automatically.
-      def consumer_group
-        # @note We don't ignore the api_adapter[:ignored] values as they should be ignored
-        #   only when proxying details go ruby-kafka. We use ignored fields internally in karafka
-        ignored_settings = api_adapter[:subscribe]
-        defined_settings = api_adapter.values.flatten
-        karafka_settings = %i[batch_fetching]
-
-        dynamically_proxied = Karafka::Setup::Config.config.kafka.to_h.keys
-
-        (defined_settings + dynamically_proxied).uniq + karafka_settings - ignored_settings
-      end
-    end
-  end
-end

data/lib/karafka/backends/inline.rb

@@ -1,16 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Namespace for all different backends Karafka supports
-  module Backends
-    # Backend that just runs stuff asap without any scheduling
-    module Inline
-      private
-
-      # Executes consume code immediately (without enqueuing)
-      def process
-        Karafka.monitor.instrument('backends.inline.process', caller: self) { consume }
-      end
-    end
-  end
-end