karafka 2.0.0 → 2.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 06f96b3ba14b7910d3cb23a90bdc135927f1483d5af4d079d59b3d6940d391b9
-  data.tar.gz: 2d936d4ddac360e229004c05f6ee51d004018e3c4cf4203067a99e6df93df92e
+  metadata.gz: 5adfb5654381a04d5b7111806bd5bbba70f75f02e7044b57af09b91b547dbd09
+  data.tar.gz: 9500da87035507a037218e9df5a11c8803e738c707b5d624c19559917939b689
 SHA512:
-  metadata.gz: caeb9bcf1f0301f31442025176f3da631aa8ff9c21164e112b9ce3112372309d4b7fe59863cf7cf511e6a6bc4448f61448f3bc2024b101d4ae2668a358e5bbe9
-  data.tar.gz: cfc422fae74512c2142ef36f9d4b5c3024bf318d79dc83456e27553aebc177cec7a3772f6378160c8ecdf5a3b2626d862ba91d823fae6e30cbb593d4198474a9
+  metadata.gz: 1acb20378ecdf95b87378297de714f6791f5abd3d411fbda4701231912e30033163c3b81a21f98a99d2ca247dbbe8cda72d81efc7b1d83a3aa019485dd2e8604
+  data.tar.gz: 2a8d352d68852da005c87b0494457abf3559ec3975deb1098da233e16e4e33fed517020733300b2364ad650cc3764d59deb7fff6efe63e216e4fdb057c3e4ed8

data/.github/workflows/ci.yml

@@ -73,10 +73,6 @@ jobs:
           ruby-version: ${{matrix.ruby}}
           bundler-cache: true
 
-      - name: Ensure all needed Kafka topics are created and wait if not
-        run: |
-          bin/wait_for_kafka
-
       - name: Run all specs
         env:
           GITHUB_COVERAGE: ${{matrix.coverage}}
@@ -120,10 +116,6 @@ jobs:
           bundle config set without development
           bundle install
 
-      - name: Ensure all needed Kafka topics are created and wait if not
-        run: |
-          bin/wait_for_kafka
-
       - name: Run integration tests
         env:
           KARAFKA_PRO_LICENSE_TOKEN: ${{ secrets.KARAFKA_PRO_LICENSE_TOKEN }}

data/CHANGELOG.md

@@ -1,6 +1,12 @@
 # Karafka framework changelog
 
-## 2.0.0 (2022-08-5)
+## 2.0.1 (2022-08-06)
+- Provide `Karafka::Admin` for creation and destruction of topics and fetching cluster info.
+- Update integration specs to always use one-time disposable topics.
+- Remove no longer needed `wait_for_kafka` script.
+- Add more integration specs for cover offset management upon errors.
+
+## 2.0.0 (2022-08-05)
 
 This changelog describes changes between `1.4` and `2.0`. Please refer to appropriate release notes for changes between particular `rc` releases.
 

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (2.0.0)
+    karafka (2.0.1)
       karafka-core (>= 2.0.2, < 3.0.0)
       rdkafka (>= 0.12)
       thor (>= 0.20)

data/README.md

@@ -4,8 +4,6 @@
 [![Gem Version](https://badge.fury.io/rb/karafka.svg)](http://badge.fury.io/rb/karafka)
 [![Join the chat at https://slack.karafka.io](https://raw.githubusercontent.com/karafka/misc/master/slack.svg)](https://slack.karafka.io)
 
-**Note**: All of the documentation here refers to Karafka `2.0.0` or higher. If you are looking for the documentation for Karafka `1.4`, please click [here](https://github.com/karafka/wiki/tree/1.4).
-
 ## About Karafka
 
 Karafka is a Ruby and Rails multi-threaded efficient Kafka processing framework that:

data/docker-compose.yml

@@ -16,36 +16,7 @@ services:
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
       KAFKA_CREATE_TOPICS:
-        "integrations_00_02:2:1,\
-         integrations_01_02:2:1,\
-         integrations_02_02:2:1,\
-         integrations_03_02:2:1,\
-         integrations_04_02:2:1,\
-         integrations_05_02:2:1,\
-         integrations_06_02:2:1,\
-         integrations_07_02:2:1,\
-         integrations_08_02:2:1,\
-         integrations_09_02:2:1,\
-         integrations_10_02:2:1,\
-         integrations_11_02:2:1,\
-         integrations_12_02:2:1,\
-         integrations_13_02:2:1,\
-         integrations_14_02:2:1,\
-         integrations_15_02:2:1,\
-         integrations_16_02:2:1,\
-         integrations_17_02:2:1,\
-         integrations_18_02:2:1,\
-         integrations_19_02:2:1,\
-         integrations_20_02:2:1,\
-         integrations_21_02:2:1,\
-         integrations_00_03:3:1,\
-         integrations_01_03:3:1,\
-         integrations_02_03:3:1,\
-         integrations_03_03:3:1,\
-         integrations_04_03:3:1,\
-         integrations_00_10:10:1,\
-         integrations_01_10:10:1,\
-         benchmarks_00_01:1:1,\
+        "benchmarks_00_01:1:1,\
          benchmarks_00_05:5:1,\
          benchmarks_01_05:5:1,\
          benchmarks_00_10:10:1"

data/lib/karafka/admin.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Simple admin actions that we can perform via Karafka on our Kafka cluster
+  #
+  # @note It always initializes a new admin instance as we want to ensure it is always closed
+  #   Since admin actions are not performed that often, that should be ok.
+  #
+  # @note It always uses the primary defined cluster and does not support multi-cluster work.
+  #   If you need this, just replace the cluster info for the time you use this
+  class Admin
+    class << self
+      # Creates Kafka topic with given settings
+      #
+      # @param name [String] topic name
+      # @param partitions [Integer] number of partitions we expect
+      # @param replication_factor [Integer] number of replicas
+      # @param topic_config [Hash] topic config details as described here:
+      #   https://kafka.apache.org/documentation/#topicconfigs
+      def create_topic(name, partitions, replication_factor, topic_config = {})
+        with_admin do |admin|
+          admin
+            .create_topic(name, partitions, replication_factor, topic_config)
+            .wait
+        end
+      end
+
+      # Deleted a given topic
+      #
+      # @param name [String] topic name
+      def delete_topic(name)
+        with_admin do |admin|
+          admin
+            .delete_topic(name)
+            .wait
+        end
+      end
+
+      # @return [Rdkafka::Metadata] cluster metadata info
+      def cluster_info
+        with_admin do |admin|
+          Rdkafka::Metadata.new(admin.instance_variable_get('@native_kafka'))
+        end
+      end
+
+      private
+
+      # Creates admin instance and yields it. After usage it closes the admin instance
+      def with_admin
+        admin = ::Rdkafka::Config.new(Karafka::App.config.kafka).admin
+        result = yield(admin)
+        admin.close
+        result
+      end
+    end
+  end
+end

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.0.0'
+  VERSION = '2.0.1'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 2.0.1
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -34,7 +34,7 @@ cert_chain:
   R2P11bWoCtr70BsccVrN8jEhzwXngMyI2gVt750Y+dbTu1KgRqZKp/ECe7ZzPzXj
   pIy9vHxTANKYVyI4qj8OrFdEM5BQNu8oQpL0iQ==
   -----END CERTIFICATE-----
-date: 2022-08-05 00:00:00.000000000 Z
+date: 2022-08-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -152,7 +152,6 @@ files:
 - bin/scenario
 - bin/stress_many
 - bin/stress_one
-- bin/wait_for_kafka
 - certs/karafka-pro.pem
 - certs/mensfeld.pem
 - config/errors.yml
@@ -166,6 +165,7 @@ files:
 - lib/karafka/active_job/job_extensions.rb
 - lib/karafka/active_job/job_options_contract.rb
 - lib/karafka/active_job/routing/extensions.rb
+- lib/karafka/admin.rb
 - lib/karafka/app.rb
 - lib/karafka/base_consumer.rb
 - lib/karafka/cli.rb

data/bin/wait_for_kafka

@@ -1,20 +0,0 @@
-#!/bin/bash
-
-# This script allows us to wait for Kafka docker to fully be ready
-# We consider it fully ready when all our topics that need to be created are created as expected
-
-KAFKA_NAME='karafka_20_kafka'
-ZOOKEEPER='zookeeper:2181'
-LIST_CMD="kafka-topics.sh --list --zookeeper $ZOOKEEPER"
-
-# Take the number of topics that we need to create prior to running anything
-TOPICS_COUNT=`cat docker-compose.yml | grep -E -i 'integrations_|benchmarks_' | wc -l`
-
-# And wait until all of them are created
-until (((`docker exec $KAFKA_NAME $LIST_CMD | wc -l`) >= $TOPICS_COUNT));
-do
-  echo "Waiting for Kafka to create all the needed topics..."
-  sleep 1
-done
-
-echo "All the needed topics created."