karafka 2.0.0 → 2.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 06f96b3ba14b7910d3cb23a90bdc135927f1483d5af4d079d59b3d6940d391b9
-  data.tar.gz: 2d936d4ddac360e229004c05f6ee51d004018e3c4cf4203067a99e6df93df92e
+  metadata.gz: 16983e52af6623dbbb0b30b6e65d7579fc8795e137c71f25d24f36e248f45248
+  data.tar.gz: 66087ddd0ac20272b573f81dbea84e2296919ea1b1813f1fec01b3880ad0e170
 SHA512:
-  metadata.gz: caeb9bcf1f0301f31442025176f3da631aa8ff9c21164e112b9ce3112372309d4b7fe59863cf7cf511e6a6bc4448f61448f3bc2024b101d4ae2668a358e5bbe9
-  data.tar.gz: cfc422fae74512c2142ef36f9d4b5c3024bf318d79dc83456e27553aebc177cec7a3772f6378160c8ecdf5a3b2626d862ba91d823fae6e30cbb593d4198474a9
+  metadata.gz: 7be4a093508e22c77fc5473cdedcafb2e2eb0338745a58da1a9272e0183ea72b20e631809d4a0f970703a13ed660965d5942593395b288f2db67a4070c3ab472
+  data.tar.gz: 64d3d95fbb84eb2bf272e5927a2ffe55c8389e15cc9fb71a12a79d583def6eefbcf13d8986af1464bad83e86a54d111cc682165946298eda5016afcafb41cc3b

data/.github/workflows/ci.yml

@@ -73,10 +73,6 @@ jobs:
           ruby-version: ${{matrix.ruby}}
           bundler-cache: true
 
-      - name: Ensure all needed Kafka topics are created and wait if not
-        run: |
-          bin/wait_for_kafka
-
       - name: Run all specs
         env:
           GITHUB_COVERAGE: ${{matrix.coverage}}
@@ -120,10 +116,6 @@ jobs:
           bundle config set without development
           bundle install
 
-      - name: Ensure all needed Kafka topics are created and wait if not
-        run: |
-          bin/wait_for_kafka
-
       - name: Run integration tests
         env:
           KARAFKA_PRO_LICENSE_TOKEN: ${{ secrets.KARAFKA_PRO_LICENSE_TOKEN }}

data/CHANGELOG.md

@@ -1,6 +1,21 @@
 # Karafka framework changelog
 
-## 2.0.0 (2022-08-5)
+## 2.0.3 (2022-08-09)
+- Update boot info on server startup.
+- Update `karafka info` with more descriptive Ruby version info.
+- Fix issue where when used with Rails in development, log would be too verbose.
+- Fix issue where Zeitwerk with Rails would not load Pro components despite license being present.
+
+## 2.0.2 (2022-08-07)
+- Bypass issue with Rails reload in development by releasing the connection (https://github.com/rails/rails/issues/44183).
+
+## 2.0.1 (2022-08-06)
+- Provide `Karafka::Admin` for creation and destruction of topics and fetching cluster info.
+- Update integration specs to always use one-time disposable topics.
+- Remove no longer needed `wait_for_kafka` script.
+- Add more integration specs for cover offset management upon errors.
+
+## 2.0.0 (2022-08-05)
 
 This changelog describes changes between `1.4` and `2.0`. Please refer to appropriate release notes for changes between particular `rc` releases.
 

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (2.0.0)
+    karafka (2.0.3)
       karafka-core (>= 2.0.2, < 3.0.0)
       rdkafka (>= 0.12)
       thor (>= 0.20)

data/README.md

@@ -4,8 +4,6 @@
 [![Gem Version](https://badge.fury.io/rb/karafka.svg)](http://badge.fury.io/rb/karafka)
 [![Join the chat at https://slack.karafka.io](https://raw.githubusercontent.com/karafka/misc/master/slack.svg)](https://slack.karafka.io)
 
-**Note**: All of the documentation here refers to Karafka `2.0.0` or higher. If you are looking for the documentation for Karafka `1.4`, please click [here](https://github.com/karafka/wiki/tree/1.4).
-
 ## About Karafka
 
 Karafka is a Ruby and Rails multi-threaded efficient Kafka processing framework that:

data/docker-compose.yml

@@ -16,36 +16,7 @@ services:
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
       KAFKA_CREATE_TOPICS:
-        "integrations_00_02:2:1,\
-         integrations_01_02:2:1,\
-         integrations_02_02:2:1,\
-         integrations_03_02:2:1,\
-         integrations_04_02:2:1,\
-         integrations_05_02:2:1,\
-         integrations_06_02:2:1,\
-         integrations_07_02:2:1,\
-         integrations_08_02:2:1,\
-         integrations_09_02:2:1,\
-         integrations_10_02:2:1,\
-         integrations_11_02:2:1,\
-         integrations_12_02:2:1,\
-         integrations_13_02:2:1,\
-         integrations_14_02:2:1,\
-         integrations_15_02:2:1,\
-         integrations_16_02:2:1,\
-         integrations_17_02:2:1,\
-         integrations_18_02:2:1,\
-         integrations_19_02:2:1,\
-         integrations_20_02:2:1,\
-         integrations_21_02:2:1,\
-         integrations_00_03:3:1,\
-         integrations_01_03:3:1,\
-         integrations_02_03:3:1,\
-         integrations_03_03:3:1,\
-         integrations_04_03:3:1,\
-         integrations_00_10:10:1,\
-         integrations_01_10:10:1,\
-         benchmarks_00_01:1:1,\
+        "benchmarks_00_01:1:1,\
          benchmarks_00_05:5:1,\
          benchmarks_01_05:5:1,\
          benchmarks_00_10:10:1"

data/lib/karafka/admin.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Simple admin actions that we can perform via Karafka on our Kafka cluster
+  #
+  # @note It always initializes a new admin instance as we want to ensure it is always closed
+  #   Since admin actions are not performed that often, that should be ok.
+  #
+  # @note It always uses the primary defined cluster and does not support multi-cluster work.
+  #   If you need this, just replace the cluster info for the time you use this
+  module Admin
+    class << self
+      # Creates Kafka topic with given settings
+      #
+      # @param name [String] topic name
+      # @param partitions [Integer] number of partitions we expect
+      # @param replication_factor [Integer] number of replicas
+      # @param topic_config [Hash] topic config details as described here:
+      #   https://kafka.apache.org/documentation/#topicconfigs
+      def create_topic(name, partitions, replication_factor, topic_config = {})
+        with_admin do |admin|
+          admin
+            .create_topic(name, partitions, replication_factor, topic_config)
+            .wait
+        end
+      end
+
+      # Deleted a given topic
+      #
+      # @param name [String] topic name
+      def delete_topic(name)
+        with_admin do |admin|
+          admin
+            .delete_topic(name)
+            .wait
+        end
+      end
+
+      # @return [Rdkafka::Metadata] cluster metadata info
+      def cluster_info
+        with_admin do |admin|
+          Rdkafka::Metadata.new(admin.instance_variable_get('@native_kafka'))
+        end
+      end
+
+      private
+
+      # Creates admin instance and yields it. After usage it closes the admin instance
+      def with_admin
+        admin = ::Rdkafka::Config.new(Karafka::App.config.kafka).admin
+        result = yield(admin)
+        admin.close
+        result
+      end
+    end
+  end
+end

data/lib/karafka/cli/info.rb

@@ -35,7 +35,7 @@ module Karafka
 
         [
           "Karafka version: #{Karafka::VERSION}#{postfix}",
-          "Ruby version: #{RUBY_VERSION}",
+          "Ruby version: #{RUBY_DESCRIPTION}",
           "Rdkafka version: #{::Rdkafka::VERSION}",
           "Subscription groups count: #{Karafka::App.subscription_groups.size}",
           "Workers count: #{Karafka::App.config.concurrency}",

data/lib/karafka/cli/server.rb

@@ -33,11 +33,11 @@ module Karafka
 
         if Karafka.pro?
           Karafka.logger.info(
-            green('Thank you for investing in the Karafka Pro subscription!')
+            green('Thank you for using Karafka Pro!')
           )
         else
           Karafka.logger.info(
-            red('You like Karafka? Please consider getting a Pro version!')
+            red('Upgrade to Karafka Pro for more features and support: https://karafka.io')
           )
         end
       end

data/lib/karafka/instrumentation/logger_listener.rb

@@ -96,7 +96,12 @@ module Karafka
       #
       # @param _event [Dry::Events::Event] event details including payload
       def on_app_running(_event)
-        info 'Running Karafka server'
+        info "Running in #{RUBY_DESCRIPTION}"
+        info "Running Karafka #{Karafka::VERSION} server"
+
+        return if Karafka.pro?
+
+        info 'See LICENSE and the LGPL-3.0 for licensing details.'
       end
 
       # Logs info that we're going to stop the Karafka server.

data/lib/karafka/instrumentation/notifications.rb

@@ -34,6 +34,7 @@ module Karafka
 
         worker.process
         worker.processed
+        worker.completed
 
         statistics.emitted
 

data/lib/karafka/processing/worker.rb

@@ -46,8 +46,9 @@ module Karafka
       def process
         job = @jobs_queue.pop
 
+        instrument_details = { caller: self, job: job, jobs_queue: @jobs_queue }
+
         if job
-          instrument_details = { caller: self, job: job, jobs_queue: @jobs_queue }
 
           Karafka.monitor.instrument('worker.process', instrument_details)
 
@@ -82,6 +83,9 @@ module Karafka
       ensure
         # job can be nil when the queue is being closed
         @jobs_queue.complete(job) if job
+
+        # Always publish info, that we completed all the work despite its result
+        Karafka.monitor.instrument('worker.completed', instrument_details)
       end
     end
   end

data/lib/karafka/railtie.rb

@@ -48,9 +48,13 @@ if rails
         next unless Rails.env.development?
         next unless ENV.key?('KARAFKA_CLI')
 
+        logger = ActiveSupport::Logger.new($stdout)
+        # Inherit the logger level from Rails, otherwise would always run with the debug level
+        logger.level = Rails.logger.level
+
         Rails.logger.extend(
           ActiveSupport::Logger.broadcast(
-            ActiveSupport::Logger.new($stdout)
+            logger
           )
         )
       end
@@ -77,6 +81,16 @@ if rails
 
           Rails.application.reloader.reload!
         end
+
+        ::Karafka::App.monitor.subscribe('worker.completed') do
+          # Skip in case someone is using Rails without ActiveRecord
+          next unless Object.const_defined?('ActiveRecord::Base')
+
+          # Always release the connection after processing is done. Otherwise thread may hang
+          # blocking the reload and further processing
+          # @see https://github.com/rails/rails/issues/44183
+          ActiveRecord::Base.connection_pool.release_connection
+        end
       end
 
       initializer 'karafka.require_karafka_boot_file' do |app|

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.0.0'
+  VERSION = '2.0.3'
 end

data/lib/karafka.rb

@@ -85,8 +85,16 @@ end
 loader = Zeitwerk::Loader.for_gem
 # Do not load Rails extensions by default, this will be handled by Railtie if they are needed
 loader.ignore(Karafka.gem_root.join('lib/active_job'))
-# Do not load pro components, this will be handled by license manager
-loader.ignore(Karafka.gem_root.join('lib/karafka/pro'))
+
+begin
+  require 'karafka-license'
+rescue LoadError
+  # Do not load pro components if we cannot load the license
+  # This is a preliminary check so autoload works as expected
+  # Later on the licenser will make sure to setup all the needed components anyhow
+  loader.ignore(Karafka.gem_root.join('lib/karafka/pro'))
+end
+
 # Do not load vendors instrumentation components. Those need to be required manually if needed
 loader.ignore(Karafka.gem_root.join('lib/karafka/instrumentation/vendors'))
 loader.setup

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 2.0.3
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -34,7 +34,7 @@ cert_chain:
   R2P11bWoCtr70BsccVrN8jEhzwXngMyI2gVt750Y+dbTu1KgRqZKp/ECe7ZzPzXj
   pIy9vHxTANKYVyI4qj8OrFdEM5BQNu8oQpL0iQ==
   -----END CERTIFICATE-----
-date: 2022-08-05 00:00:00.000000000 Z
+date: 2022-08-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -152,7 +152,6 @@ files:
 - bin/scenario
 - bin/stress_many
 - bin/stress_one
-- bin/wait_for_kafka
 - certs/karafka-pro.pem
 - certs/mensfeld.pem
 - config/errors.yml
@@ -166,6 +165,7 @@ files:
 - lib/karafka/active_job/job_extensions.rb
 - lib/karafka/active_job/job_options_contract.rb
 - lib/karafka/active_job/routing/extensions.rb
+- lib/karafka/admin.rb
 - lib/karafka/app.rb
 - lib/karafka/base_consumer.rb
 - lib/karafka/cli.rb

data/bin/wait_for_kafka

@@ -1,20 +0,0 @@
-#!/bin/bash
-
-# This script allows us to wait for Kafka docker to fully be ready
-# We consider it fully ready when all our topics that need to be created are created as expected
-
-KAFKA_NAME='karafka_20_kafka'
-ZOOKEEPER='zookeeper:2181'
-LIST_CMD="kafka-topics.sh --list --zookeeper $ZOOKEEPER"
-
-# Take the number of topics that we need to create prior to running anything
-TOPICS_COUNT=`cat docker-compose.yml | grep -E -i 'integrations_|benchmarks_' | wc -l`
-
-# And wait until all of them are created
-until (((`docker exec $KAFKA_NAME $LIST_CMD | wc -l`) >= $TOPICS_COUNT));
-do
-  echo "Waiting for Kafka to create all the needed topics..."
-  sleep 1
-done
-
-echo "All the needed topics created."