karafka 2.0.0.rc6 → 2.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 13fbbd2c70987d84f768acb1a01f02f18ca59f49d8f02a2c1103b6870dbcfb15
-  data.tar.gz: 223187a692ae8e3d5da3c028c135c20a3de0c05b26b7f67cfbae11b928b02773
+  metadata.gz: c2b67a18484b250b65788db213f615b7f1304ab33d6645bceeebdf8025b54bda
+  data.tar.gz: 27d4ece68701b9f34266ff834c9d0809690e06b108412f68916a39880a7a23f6
 SHA512:
-  metadata.gz: 381ddd6a4d3f9695ca5d657d4960213afb3c5b92b8b4e0f51b45f75d1de92f65a570ec903abb43d5cb2cf371a557dbbb01cb78ce33eb118902e6d5e77fdb6095
-  data.tar.gz: e4675ba160b9443240497f874e29bb7614b77bf06c4fa8e768aff42060de182059ac4a26b07841d251f5bb4d257ad8f3920d77ec551c8f1487a4eef3e6a6db32
+  metadata.gz: 682a0ff35d78e9eb5590bdb32bbc14bf47f18029003286d55d1f6f14425cd3d3ad6e8081a574f3f2146bc95d9ac692e9dc82622065e6dc6f092dc3dbe61652f3
+  data.tar.gz: 5a4e68ea639b9261172c055e6f22a741735262eb277acd18e465c862ead0242d0287b8872d6add290af0bba681f000a328f4c16d1ee31ca9075fe880552ddf9b

data/.github/workflows/ci.yml

@@ -73,10 +73,6 @@ jobs:
           ruby-version: ${{matrix.ruby}}
           bundler-cache: true
 
-      - name: Ensure all needed Kafka topics are created and wait if not
-        run: |
-          bin/wait_for_kafka
-
       - name: Run all specs
         env:
           GITHUB_COVERAGE: ${{matrix.coverage}}
@@ -120,10 +116,6 @@ jobs:
           bundle config set without development
           bundle install
 
-      - name: Ensure all needed Kafka topics are created and wait if not
-        run: |
-          bin/wait_for_kafka
-
       - name: Run integration tests
         env:
           KARAFKA_PRO_LICENSE_TOKEN: ${{ secrets.KARAFKA_PRO_LICENSE_TOKEN }}

data/CHANGELOG.md

@@ -1,7 +1,100 @@
 # Karafka framework changelog
 
+## 2.0.2 (2022-08-07)
+- Bypass issue with Rails reload in development by releasing the connection (https://github.com/rails/rails/issues/44183).
+
+## 2.0.1 (2022-08-06)
+- Provide `Karafka::Admin` for creation and destruction of topics and fetching cluster info.
+- Update integration specs to always use one-time disposable topics.
+- Remove no longer needed `wait_for_kafka` script.
+- Add more integration specs for cover offset management upon errors.
+
+## 2.0.0 (2022-08-05)
+
+This changelog describes changes between `1.4` and `2.0`. Please refer to appropriate release notes for changes between particular `rc` releases.
+
+Karafka 2.0 is a **major** rewrite that brings many new things to the table but also removes specific concepts that happened not to be as good as I initially thought when I created them.
+
+Please consider getting a Pro version if you want to **support** my work on the Karafka ecosystem!
+
+For anyone worried that I will start converting regular features into Pro: This will **not** happen. Anything free and fully OSS in Karafka 1.4 will **forever** remain free. Most additions and improvements to the ecosystem are to its free parts. Any feature that is introduced as a free and open one will not become paid.
+
+### Additions
+
+This section describes **new** things and concepts introduced with Karafka 2.0.
+
+Karafka 2.0:
+
+- Introduces multi-threaded support for [concurrent work](https://github.com/karafka/karafka/wiki/Concurrency-and-multithreading) consumption for separate partitions as well as for single partition work via [Virtual Partitions](https://github.com/karafka/karafka/wiki/Pro-Virtual-Partitions).
+- Introduces [Active Job adapter](https://github.com/karafka/karafka/wiki/Active-Job) for using Karafka as a jobs backend with Ruby on Rails Active Job.
+- Introduces fully automatic integration end-to-end [test suite](https://github.com/karafka/karafka/tree/master/spec/integrations) that checks any case I could imagine.
+- Introduces [Virtual Partitions](https://github.com/karafka/karafka/wiki/Pro-Virtual-Partitions) for ability to parallelize work of a single partition.
+- Introduces [Long-Running Jobs](https://github.com/karafka/karafka/wiki/Pro-Long-Running-Jobs) to allow for work that would otherwise exceed the `max.poll.interval.ms`.
+- Introduces the [Enhanced Scheduler](https://github.com/karafka/karafka/wiki/Pro-Enhanced-Scheduler) that uses a non-preemptive LJF (Longest Job First) algorithm instead of a a FIFO (First-In, First-Out) one.
+- Introduces [Enhanced Active Job adapter](https://github.com/karafka/karafka/wiki/Pro-Enhanced-Active-Job) that is optimized and allows for strong ordering of jobs and more.
+- Introduces seamless [Ruby on Rails integration](https://github.com/karafka/karafka/wiki/Integrating-with-Ruby-on-Rails-and-other-frameworks) via `Rails::Railte` without need for any extra configuration.
+- Provides `#revoked` [method](https://github.com/karafka/karafka/wiki/Consuming-messages#shutdown-and-partition-revocation-handlers) for taking actions upon topic revocation.
+- Emits underlying async errors emitted from `librdkafka` via the standardized `error.occurred` [monitor channel](https://github.com/karafka/karafka/wiki/Error-handling-and-back-off-policy#error-tracking).
+- Replaces `ruby-kafka` with `librdkafka` as an underlying driver.
+- Introduces official [EOL policies](https://github.com/karafka/karafka/wiki/Versions-Lifecycle-and-EOL).
+- Introduces [benchmarks](https://github.com/karafka/karafka/tree/master/spec/benchmarks) that can be used to profile Karafka.
+- Introduces a requirement that the end user code **needs** to be [thread-safe](https://github.com/karafka/karafka/wiki/FAQ#does-karafka-require-gems-to-be-thread-safe).
+- Introduces a [Pro subscription](https://github.com/karafka/karafka/wiki/Build-vs.-Buy) with a [commercial license](https://github.com/karafka/karafka/blob/master/LICENSE-COMM) to fund further ecosystem development.
+
+### Deletions
+
+This section describes things that are **no longer** part of the Karafka ecosystem.
+
+Karafka 2.0:
+
+- Removes topics mappers concept completely.
+- Removes pidfiles support.
+- Removes daemonization support.
+- Removes support for using `sidekiq-backend` due to introduction of [multi-threading](https://github.com/karafka/karafka/wiki/Concurrency-and-multithreading).
+- Removes the `Responders` concept in favour of WaterDrop producer usage.
+- Removes completely all the callbacks in favour of finalizer method `#shutdown`.
+- Removes single message consumption mode in favour of [documentation](https://github.com/karafka/karafka/wiki/Consuming-messages#one-at-a-time) on how to do it easily by yourself.
+
+### Changes
+
+This section describes things that were **changed** in Karafka but are still present.
+
+Karafka 2.0:
+
+- Uses only instrumentation that comes from Karafka. This applies also to notifications coming natively from `librdkafka`. They are now piped through Karafka prior to being dispatched.
+- Integrates WaterDrop `2.x` tightly with autoconfiguration inheritance and an option to redefine it.
+- Integrates with the `karafka-testing` gem for RSpec that also has been updated.
+- Updates `cli info` to reflect the `2.0` details.
+- Stops validating `kafka` configuration beyond minimum as the rest is handled by `librdkafka`.
+- No longer uses `dry-validation`.
+- No longer uses `dry-monitor`.
+- No longer uses `dry-configurable`.
+- Lowers general external dependencies three **heavily**.
+- Renames `Karafka::Params::BatchMetadata` to `Karafka::Messages::BatchMetadata`.
+- Renames `Karafka::Params::Params` to `Karafka::Messages::Message`.
+- Renames `#params_batch` in consumers to `#messages`.
+- Renames `Karafka::Params::Metadata` to `Karafka::Messages::Metadata`.
+- Renames `Karafka::Fetcher` to `Karafka::Runner` and align notifications key names.
+- Renames `StdoutListener` to `LoggerListener`.
+- Reorganizes [monitoring and logging](https://github.com/karafka/karafka/wiki/Monitoring-and-logging) to match new concepts.
+- Notifies on fatal worker processing errors.
+- Contains updated install templates for Rails and no-non Rails.
+- Changes how the routing style (`0.5`) behaves. It now builds a single consumer group instead of one per topic.
+- Introduces changes that will allow me to build full web-UI in the upcoming `2.1`.
+- Contains updated example apps.
+- Standardizes error hooks for all error reporting (`error.occurred`).
+- Changes license to `LGPL-3.0`.
+- Introduces a `karafka-core` dependency that contains common code used across the ecosystem.
+- Contains updated [wiki](https://github.com/karafka/karafka/wiki) on everything I could think of.
+
+### What's ahead
+
+Karafka 2.0 is just the beginning.
+
+There are several things in the plan already for 2.1 and beyond, including a web dashboard, at-rest encryption, transactions support, and more.
+
 ## 2.0.0.rc6 (2022-08-05)
-- Update licenser to use a gem based approach based on `karafka/pro/license`.
+- Update licenser to use a gem based approach based on `karafka-license`.
 - Do not mark intermediate jobs as consumed when Karafka runs Enhanced Active Job with Virtual Partitions.
 - Improve development experience by adding fast cluster state changes refresh (#944)
 - Improve the license loading.
@@ -59,7 +152,7 @@
 - Add more integration specs related to polling limits.
 - Remove auto-detection of re-assigned partitions upon rebalance as for too fast rebalances it could not be accurate enough. It would also mess up in case of rebalances that would happen right after a `#seek` was issued for a partition.
 - Optimize the removal of pre-buffered lost partitions data.
-- Always rune `#revoked` when rebalance with revocation happens.
+- Always run `#revoked` when rebalance with revocation happens.
 - Evict executors upon rebalance, to prevent race-conditions.
 - Align topics names for integration specs.
 

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (2.0.0.rc6)
+    karafka (2.0.2)
       karafka-core (>= 2.0.2, < 3.0.0)
       rdkafka (>= 0.12)
       thor (>= 0.20)

data/README.md

@@ -4,8 +4,6 @@
 [![Gem Version](https://badge.fury.io/rb/karafka.svg)](http://badge.fury.io/rb/karafka)
 [![Join the chat at https://slack.karafka.io](https://raw.githubusercontent.com/karafka/misc/master/slack.svg)](https://slack.karafka.io)
 
-**Note**: All of the documentation here refers to Karafka `2.0.0.rc5` or higher. If you are looking for the documentation for Karafka `1.4`, please click [here](https://github.com/karafka/wiki/tree/1.4).
-
 ## About Karafka
 
 Karafka is a Ruby and Rails multi-threaded efficient Kafka processing framework that:
@@ -55,7 +53,7 @@ We also maintain many [integration specs](https://github.com/karafka/karafka/tre
 1. Add and install Karafka:
 
 ```bash
-bundle add karafka -v 2.0.0.rc5
+bundle add karafka
 
 bundle exec karafka install
 ```

data/docker-compose.yml

@@ -16,36 +16,7 @@ services:
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
       KAFKA_CREATE_TOPICS:
-        "integrations_00_02:2:1,\
-         integrations_01_02:2:1,\
-         integrations_02_02:2:1,\
-         integrations_03_02:2:1,\
-         integrations_04_02:2:1,\
-         integrations_05_02:2:1,\
-         integrations_06_02:2:1,\
-         integrations_07_02:2:1,\
-         integrations_08_02:2:1,\
-         integrations_09_02:2:1,\
-         integrations_10_02:2:1,\
-         integrations_11_02:2:1,\
-         integrations_12_02:2:1,\
-         integrations_13_02:2:1,\
-         integrations_14_02:2:1,\
-         integrations_15_02:2:1,\
-         integrations_16_02:2:1,\
-         integrations_17_02:2:1,\
-         integrations_18_02:2:1,\
-         integrations_19_02:2:1,\
-         integrations_20_02:2:1,\
-         integrations_21_02:2:1,\
-         integrations_00_03:3:1,\
-         integrations_01_03:3:1,\
-         integrations_02_03:3:1,\
-         integrations_03_03:3:1,\
-         integrations_04_03:3:1,\
-         integrations_00_10:10:1,\
-         integrations_01_10:10:1,\
-         benchmarks_00_01:1:1,\
+        "benchmarks_00_01:1:1,\
          benchmarks_00_05:5:1,\
          benchmarks_01_05:5:1,\
          benchmarks_00_10:10:1"

data/lib/karafka/admin.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Simple admin actions that we can perform via Karafka on our Kafka cluster
+  #
+  # @note It always initializes a new admin instance as we want to ensure it is always closed
+  #   Since admin actions are not performed that often, that should be ok.
+  #
+  # @note It always uses the primary defined cluster and does not support multi-cluster work.
+  #   If you need this, just replace the cluster info for the time you use this
+  module Admin
+    class << self
+      # Creates Kafka topic with given settings
+      #
+      # @param name [String] topic name
+      # @param partitions [Integer] number of partitions we expect
+      # @param replication_factor [Integer] number of replicas
+      # @param topic_config [Hash] topic config details as described here:
+      #   https://kafka.apache.org/documentation/#topicconfigs
+      def create_topic(name, partitions, replication_factor, topic_config = {})
+        with_admin do |admin|
+          admin
+            .create_topic(name, partitions, replication_factor, topic_config)
+            .wait
+        end
+      end
+
+      # Deleted a given topic
+      #
+      # @param name [String] topic name
+      def delete_topic(name)
+        with_admin do |admin|
+          admin
+            .delete_topic(name)
+            .wait
+        end
+      end
+
+      # @return [Rdkafka::Metadata] cluster metadata info
+      def cluster_info
+        with_admin do |admin|
+          Rdkafka::Metadata.new(admin.instance_variable_get('@native_kafka'))
+        end
+      end
+
+      private
+
+      # Creates admin instance and yields it. After usage it closes the admin instance
+      def with_admin
+        admin = ::Rdkafka::Config.new(Karafka::App.config.kafka).admin
+        result = yield(admin)
+        admin.close
+        result
+      end
+    end
+  end
+end

data/lib/karafka/instrumentation/notifications.rb

@@ -34,6 +34,7 @@ module Karafka
 
         worker.process
         worker.processed
+        worker.completed
 
         statistics.emitted
 

data/lib/karafka/processing/worker.rb

@@ -46,8 +46,9 @@ module Karafka
       def process
         job = @jobs_queue.pop
 
+        instrument_details = { caller: self, job: job, jobs_queue: @jobs_queue }
+
         if job
-          instrument_details = { caller: self, job: job, jobs_queue: @jobs_queue }
 
           Karafka.monitor.instrument('worker.process', instrument_details)
 
@@ -82,6 +83,9 @@ module Karafka
       ensure
         # job can be nil when the queue is being closed
         @jobs_queue.complete(job) if job
+
+        # Always publish info, that we completed all the work despite its result
+        Karafka.monitor.instrument('worker.completed', instrument_details)
       end
     end
   end

data/lib/karafka/railtie.rb

@@ -77,6 +77,16 @@ if rails
 
           Rails.application.reloader.reload!
         end
+
+        ::Karafka::App.monitor.subscribe('worker.completed') do
+          # Skip in case someone is using Rails without ActiveRecord
+          next unless Object.const_defined?('ActiveRecord::Base')
+
+          # Always release the connection after processing is done. Otherwise thread may hang
+          # blocking the reload and further processing
+          # @see https://github.com/rails/rails/issues/44183
+          ActiveRecord::Base.connection_pool.release_connection
+        end
       end
 
       initializer 'karafka.require_karafka_boot_file' do |app|

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.0.0.rc6'
+  VERSION = '2.0.2'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.0.0.rc6
+  version: 2.0.2
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -34,7 +34,7 @@ cert_chain:
   R2P11bWoCtr70BsccVrN8jEhzwXngMyI2gVt750Y+dbTu1KgRqZKp/ECe7ZzPzXj
   pIy9vHxTANKYVyI4qj8OrFdEM5BQNu8oQpL0iQ==
   -----END CERTIFICATE-----
-date: 2022-08-05 00:00:00.000000000 Z
+date: 2022-08-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -152,7 +152,6 @@ files:
 - bin/scenario
 - bin/stress_many
 - bin/stress_one
-- bin/wait_for_kafka
 - certs/karafka-pro.pem
 - certs/mensfeld.pem
 - config/errors.yml
@@ -166,6 +165,7 @@ files:
 - lib/karafka/active_job/job_extensions.rb
 - lib/karafka/active_job/job_options_contract.rb
 - lib/karafka/active_job/routing/extensions.rb
+- lib/karafka/admin.rb
 - lib/karafka/app.rb
 - lib/karafka/base_consumer.rb
 - lib/karafka/cli.rb
@@ -287,9 +287,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: 2.7.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubygems_version: 3.3.7
 signing_key: 

data/bin/wait_for_kafka

@@ -1,20 +0,0 @@
-#!/bin/bash
-
-# This script allows us to wait for Kafka docker to fully be ready
-# We consider it fully ready when all our topics that need to be created are created as expected
-
-KAFKA_NAME='karafka_20_kafka'
-ZOOKEEPER='zookeeper:2181'
-LIST_CMD="kafka-topics.sh --list --zookeeper $ZOOKEEPER"
-
-# Take the number of topics that we need to create prior to running anything
-TOPICS_COUNT=`cat docker-compose.yml | grep -E -i 'integrations_|benchmarks_' | wc -l`
-
-# And wait until all of them are created
-until (((`docker exec $KAFKA_NAME $LIST_CMD | wc -l`) >= $TOPICS_COUNT));
-do
-  echo "Waiting for Kafka to create all the needed topics..."
-  sleep 1
-done
-
-echo "All the needed topics created."