karafka 2.0.0.beta2 → 2.0.0.beta5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7f75623e7d9cdcc4ba151ad551079275528c56bf66cd9c32ecc585756a8d505c
-  data.tar.gz: e0becf53133b579f581ddfdf947bbff21221fe69a8c73a0406174aecd0155f3a
+  metadata.gz: 2c8e680ffdf69f88899a715c84cc484e8f568f4a93da9284195f4bf55a283ee1
+  data.tar.gz: 974356226a10ba2c77de770351a47180716533021a89040bcdc1aae57f452121
 SHA512:
-  metadata.gz: 2ef2ac59f1ea60136abbaccf460206a0c2f6d4fe3124eda520f3a19568702acc774e0d9e02eae24cfe6bb6cb8ee8aa74602588aa818dc3537cd6bbc8409f159d
-  data.tar.gz: e29e964e777e2bd8a551458f591b92aea69a3ac2eafa1bc1d75bb42d7cd6bb904abf997724be4268ae1f9d429627d7b92d5c38e51817d36b0f27c6499a062af3
+  metadata.gz: 2427aaae1b1b07430df7c9f042d290bbae8380fb1f6ec7c26eecee92b8fe79e13ea9f3a99a36bf89b314ffba809c556618b22c0a87f0c0c83bb73cf8af72321b
+  data.tar.gz: 55e18448b5645acd38c4194967ea7df657c142d82a105699f7b204f222f8dfb2dbd14cce82b1f424ec177afb78049b3e7588642013674a3c2923a8848b6b87e7

data/.github/workflows/ci.yml

@@ -8,6 +8,10 @@ on:
   schedule:
     - cron:  '0 1 * * *'
 
+env:
+  BUNDLE_RETRY: 6
+  BUNDLE_JOBS: 4
+
 jobs:
   diffend:
     runs-on: ubuntu-latest
@@ -17,14 +21,16 @@ jobs:
       - uses: actions/checkout@v2
         with:
           fetch-depth: 0
+
       - name: Set up Ruby
         uses: ruby/setup-ruby@v1
         with:
           ruby-version: 3.1
-      - name: Install latest bundler
-        run: gem install bundler --no-document
+          bundler-cache: true
+
       - name: Install Diffend plugin
         run: bundle plugin install diffend
+
       - name: Bundle Secure
         run: bundle secure
 
@@ -57,25 +63,19 @@ jobs:
       - name: Install package dependencies
         run: "[ -e $APT_DEPS ] || sudo apt-get install -y --no-install-recommends $APT_DEPS"
 
-      - name: Run Kafka with docker-compose
+      - name: Start Kafka with docker-compose
         run: |
           docker-compose up -d
-          sleep 10
 
       - name: Set up Ruby
         uses: ruby/setup-ruby@v1
         with:
           ruby-version: ${{matrix.ruby}}
+          bundler-cache: true
 
-      - name: Install latest Bundler
+      - name: Ensure all needed Kafka topics are created and wait if not
         run: |
-          gem install bundler --no-document
-          bundle config set without 'tools benchmarks docs'
-
-      - name: Bundle install
-        run: |
-          bundle config set without development
-          bundle install --jobs 4 --retry 3
+          bin/wait_for_kafka
 
       - name: Run all specs
         env:
@@ -100,10 +100,9 @@ jobs:
       - name: Install package dependencies
         run: "[ -e $APT_DEPS ] || sudo apt-get install -y --no-install-recommends $APT_DEPS"
 
-      - name: Run Kafka with docker-compose
+      - name: Start Kafka with docker-compose
         run: |
           docker-compose up -d
-          sleep 5
 
       - name: Set up Ruby
         uses: ruby/setup-ruby@v1
@@ -119,7 +118,11 @@ jobs:
       - name: Bundle install
         run: |
           bundle config set without development
-          bundle install --jobs 4 --retry 3
+          bundle install
+
+      - name: Ensure all needed Kafka topics are created and wait if not
+        run: |
+          bin/wait_for_kafka
 
       - name: Run integration tests
         env:

data/CHANGELOG.md

@@ -1,5 +1,54 @@
 # Karafka framework changelog
 
+## 2.0.0-beta5 (2022-07-05)
+- Always resume processing of a revoked partition upon assignment.
+- Improve specs stability.
+- Fix a case where revocation job would be executed on partition for which we never did any work.
+- Introduce a jobs group coordinator for easier jobs management.
+- Improve stability of resuming paused partitions that were revoked and re-assigned.
+- Optimize reaction time on partition ownership changes.
+- Fix a bug where despite setting long max wait time, we would return messages prior to it while not reaching the desired max messages count.
+- Add more integration specs related to polling limits.
+- Remove auto-detection of re-assigned partitions upon rebalance as for too fast rebalances it could not be accurate enough. It would also mess up in case of rebalances that would happen right after a `#seek` was issued for a partition.
+- Optimize the removal of pre-buffered lost partitions data.
+- Always rune `#revoked` when rebalance with revocation happens.
+- Evict executors upon rebalance, to prevent race-conditions.
+- Align topics names for integration specs.
+
+## 2.0.0-beta4 (2022-06-20)
+- Rename job internal api methods from `#prepare` to `#before_call` and from `#teardown` to `#after_call` to abstract away jobs execution from any type of executors and consumers logic
+- Remove ability of running `before_consume` and `after_consume` completely. Those should be for internal usage only.
+- Reorganize how Pro consumer and Pro AJ consumers inherit.
+- Require WaterDrop `2.3.1`.
+- Add more integration specs for rebalancing and max poll exceeded.
+- Move `revoked?` state from PRO to regular Karafka.
+- Use return value of `mark_as_consumed!` and `mark_as_consumed` as indicator of partition ownership + use it to switch the ownership state.
+- Do not remove rebalance manager upon client reset and recovery. This will allow us to keep the notion of lost partitions, so we can run revocation jobs for blocking jobs that exceeded the max poll interval.
+- Run revocation jobs upon reaching max poll interval for blocking jobs.
+- Early exit `poll` operation upon partition lost or max poll exceeded event.
+- Always reset consumer instances on timeout exceeded.
+- Wait for Kafka to create all the needed topics before running specs in CI.
+
+## 2.0.0-beta3 (2022-06-14)
+- Jobs building responsibility extracted out of the listener code base.
+- Fix a case where specs supervisor would try to kill no longer running process (#868)
+- Fix an instable integration spec that could misbehave under load
+- Commit offsets prior to pausing partitions to ensure that the latest offset is always committed
+- Fix a case where consecutive CTRL+C (non-stop) would case an exception during forced shutdown
+- Add missing `consumer.prepared.error` into `LoggerListener`
+- Delegate partition resuming from the consumers to listeners threads.
+- Add support for Long Running Jobs (LRJ) for ActiveJob [PRO]
+- Add support for Long Running Jobs for consumers [PRO]
+- Allow `active_job_topic` to accept a block for extra topic related settings
+- Remove no longer needed logger threads
+- Auto-adapt number of processes for integration specs based on the number of CPUs
+- Introduce an integration spec runner that prints everything to stdout (better for development)
+- Introduce extra integration specs for various ActiveJob usage scenarios
+- Rename consumer method `#prepared` to `#prepare` to reflect better its use-case
+- For test and dev raise an error when expired license key is used (never for non dev)
+- Add worker related monitor events (`worker.process` and `worker.processed`)
+- Update `LoggerListener` to include more useful information about processing and polling messages
+
 ## 2.0.0-beta2 (2022-06-07)
 - Abstract away notion of topics groups (until now it was just an array)
 - Optimize how jobs queue is closed. Since we enqueue jobs only from the listeners, we can safely close jobs queue once listeners are done. By extracting this responsibility from listeners, we remove corner cases and race conditions. Note here: for non-blocking jobs we do wait for them to finish while running the `poll`. This ensures, that for async jobs that are long-living, we do not reach `max.poll.interval`.

data/Gemfile.lock

@@ -1,13 +1,13 @@
 PATH
   remote: .
   specs:
-    karafka (2.0.0.beta2)
+    karafka (2.0.0.beta5)
       dry-configurable (~> 0.13)
       dry-monitor (~> 0.5)
       dry-validation (~> 1.7)
       rdkafka (>= 0.10)
       thor (>= 0.20)
-      waterdrop (>= 2.3.0, < 3.0.0)
+      waterdrop (>= 2.3.1, < 3.0.0)
       zeitwerk (~> 2.3)
 
 GEM
@@ -45,7 +45,7 @@ GEM
       dry-configurable (~> 0.13, >= 0.13.0)
       dry-core (~> 0.5, >= 0.5)
       dry-events (~> 0.2)
-    dry-schema (1.9.1)
+    dry-schema (1.9.2)
       concurrent-ruby (~> 1.0)
       dry-configurable (~> 0.13, >= 0.13.0)
       dry-core (~> 0.5, >= 0.5)
@@ -58,12 +58,12 @@ GEM
       dry-core (~> 0.5, >= 0.5)
       dry-inflector (~> 0.1, >= 0.1.2)
       dry-logic (~> 1.0, >= 1.0.2)
-    dry-validation (1.8.0)
+    dry-validation (1.8.1)
       concurrent-ruby (~> 1.0)
       dry-container (~> 0.7, >= 0.7.1)
       dry-core (~> 0.5, >= 0.5)
       dry-initializer (~> 3.0)
-      dry-schema (~> 1.9, >= 1.9.1)
+      dry-schema (~> 1.8, >= 1.8.0)
     factory_bot (6.2.1)
       activesupport (>= 5.0.0)
     ffi (1.15.5)
@@ -74,7 +74,7 @@ GEM
     mini_portile2 (2.8.0)
     minitest (5.15.0)
     rake (13.0.6)
-    rdkafka (0.11.1)
+    rdkafka (0.12.0)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
@@ -100,14 +100,14 @@ GEM
     thor (1.2.1)
     tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
-    waterdrop (2.3.0)
+    waterdrop (2.3.1)
       concurrent-ruby (>= 1.1)
       dry-configurable (~> 0.13)
       dry-monitor (~> 0.5)
       dry-validation (~> 1.7)
       rdkafka (>= 0.10)
       zeitwerk (~> 2.3)
-    zeitwerk (2.5.4)
+    zeitwerk (2.6.0)
 
 PLATFORMS
   x86_64-linux

data/bin/benchmarks

@@ -39,8 +39,8 @@ if ENV['SEED']
 
   # We do not populate data of benchmarks_0_10 as we use it with life-stream data only
   %w[
-    benchmarks_0_01
-    benchmarks_0_05
+    benchmarks_00_01
+    benchmarks_00_05
   ].each do |topic_name|
     partitions_count = topic_name.split('_').last.to_i
 

data/bin/integrations

@@ -11,21 +11,24 @@ require 'open3'
 require 'fileutils'
 require 'pathname'
 require 'tmpdir'
+require 'etc'
 
 ROOT_PATH = Pathname.new(File.expand_path(File.join(File.dirname(__FILE__), '../')))
 
-# Raised from the parent process if any of the integration tests fails
-IntegrationTestError = Class.new(StandardError)
-
 # How many child processes with integration specs do we want to run in parallel
-# When the value is high, there's a problem with thread allocation on Github
-CONCURRENCY = 4
+# When the value is high, there's a problem with thread allocation on Github CI, tht is why
+# we limit it. Locally we can run a lot of those, as many of them have sleeps and do not use a lot
+# of CPU
+CONCURRENCY = ENV.key?('CI') ? 5 : Etc.nprocessors * 2
+
+# How may bytes do we want to keep from the stdout in the buffer for when we need to print it
+MAX_BUFFER_OUTPUT = 10_240
 
 # Abstraction around a single test scenario execution process
 class Scenario
   # How long a scenario can run before we kill it
   # This is a fail-safe just in case something would hang
-  MAX_RUN_TIME = 60 * 2
+  MAX_RUN_TIME = 3 * 60 # 3 minutes tops
 
   # There are rare cases where Karafka may force shutdown for some of the integration cases
   # This includes exactly those
@@ -73,15 +76,20 @@ class Scenario
     # If the thread is running too long, kill it
     if current_time - @started_at > MAX_RUN_TIME
       @wait_thr.kill
-      Process.kill('TERM', pid)
+
+      begin
+        Process.kill('TERM', pid)
+      # It may finish right after we want to kill it, that's why we ignore this
+      rescue Errno::ESRCH
+      end
     end
 
     # We read it so it won't grow as we use our default logger that prints to both test.log and
     # to stdout. Otherwise after reaching the buffer size, it would hang
     buffer = ''
-    @stdout.read_nonblock(10_240, buffer, exception: false)
+    @stdout.read_nonblock(MAX_BUFFER_OUTPUT, buffer, exception: false)
     @stdout_tail << buffer
-    @stdout_tail = @stdout_tail[-10_024..-1] || @stdout_tail
+    @stdout_tail = @stdout_tail[-MAX_BUFFER_OUTPUT..-1] || @stdout_tail
 
     !@wait_thr.alive?
   end
@@ -106,14 +114,19 @@ class Scenario
 
   # Prints a status report when scenario is finished and stdout if it failed
   def report
-    result = success? ? "\e[#{32}m#{'OK'}\e[0m" : "\e[#{31}m#{'FAILED'}\e[0m"
+    if success?
+      print "\e[#{32}m#{'.'}\e[0m"
+    else
+      buffer = ''
 
-    puts "#{result} #{name}"
+      @stderr.read_nonblock(MAX_BUFFER_OUTPUT, buffer, exception: false)
 
-    unless success?
+      puts
+      puts "\e[#{31}m#{'[FAILED]'}\e[0m #{name}"
       puts "Exit code: #{exit_code}"
       puts @stdout_tail
-      puts @stderr.read
+      puts buffer
+      puts
     end
   end
 
@@ -204,5 +217,21 @@ while finished_scenarios.size < scenarios.size
   sleep(0.1)
 end
 
-# Fail all if any of the tests does not have expected exit code
-raise IntegrationTestError unless finished_scenarios.all?(&:success?)
+failed_scenarios = finished_scenarios.reject(&:success?)
+
+# Report once more on the failed jobs
+# This will only list scenarios that failed without printing their stdout here.
+if failed_scenarios.empty?
+  puts
+else
+  puts "\nFailed scenarios:\n\n"
+
+  failed_scenarios.each do |scenario|
+    puts "\e[#{31}m#{'[FAILED]'}\e[0m #{scenario.name}"
+  end
+
+  puts
+
+  # Exit with 1 if not all scenarios were successful
+  exit 1
+end

data/bin/scenario

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+
+# Runner for non-parallel execution of a single scenario.
+# It prints all the info stdout, etc and basically replaces itself with the scenario execution.
+# It is useful when we work with a single spec and we need all the debug info
+
+raise 'This code needs to be executed WITHOUT bundle exec' if Kernel.const_defined?(:Bundler)
+
+require 'open3'
+require 'fileutils'
+require 'pathname'
+require 'tmpdir'
+require 'etc'
+
+ROOT_PATH = Pathname.new(File.expand_path(File.join(File.dirname(__FILE__), '../')))
+
+# Load all the specs
+specs = Dir[ROOT_PATH.join('spec/integrations/**/*.rb')]
+
+# If filters is provided, apply
+# Allows to provide several filters one after another and applies all of them
+ARGV.each do |filter|
+  specs.delete_if { |name| !name.include?(filter) }
+end
+
+raise ArgumentError, "No integration specs with filters: #{ARGV.join(', ')}" if specs.empty?
+raise ArgumentError, "Many specs found with filters: #{ARGV.join(', ')}" if specs.size != 1
+
+exec("bundle exec ruby -r #{ROOT_PATH}/spec/integrations_helper.rb #{specs[0]}")

data/bin/stress_one

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# Runs a single integration spec in an endless loop
+# This allows us to ensure (after long enough time) that the integration spec is stable and
+# that there are no anomalies when running it for a long period of time
+
+set -e
+
+while :
+do
+  reset
+  bin/scenario $1
+done

data/bin/wait_for_kafka

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# This script allows us to wait for Kafka docker to fully be ready
+# We consider it fully ready when all our topics that need to be created are created as expected
+
+KAFKA_NAME='karafka_20_kafka'
+ZOOKEEPER='zookeeper:2181'
+LIST_CMD="kafka-topics.sh --list --zookeeper $ZOOKEEPER"
+
+# Take the number of topics that we need to create prior to running anything
+TOPICS_COUNT=`cat docker-compose.yml | grep -E -i 'integrations_|benchmarks_' | wc -l`
+
+# And wait until all of them are created
+until (((`docker exec $KAFKA_NAME $LIST_CMD | wc -l`) >= $TOPICS_COUNT));
+do
+  echo "Waiting for Kafka to create all the needed topics..."
+  sleep 1
+done
+
+echo "All the needed topics created."

data/docker-compose.yml

@@ -1,10 +1,12 @@
 version: '2'
 services:
   zookeeper:
+    container_name: karafka_20_zookeeper
     image: wurstmeister/zookeeper
     ports:
       - '2181:2181'
   kafka:
+    container_name: karafka_20_kafka
     image: wurstmeister/kafka
     ports:
       - '9092:9092'
@@ -14,16 +16,31 @@ services:
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
       KAFKA_CREATE_TOPICS:
-        "integrations_0_02:2:1,\
-         integrations_1_02:2:1,\
-         integrations_2_02:2:1,\
-         integrations_0_03:3:1,\
-         integrations_1_03:3:1,\
-         integrations_2_03:3:1,\
-         integrations_0_10:10:1,\
-         integrations_1_10:10:1,\
-         benchmarks_0_01:1:1,\
-         benchmarks_0_05:5:1,\
-         benchmarks_0_10:10:1"
+        "integrations_00_02:2:1,\
+         integrations_01_02:2:1,\
+         integrations_02_02:2:1,\
+         integrations_03_02:2:1,\
+         integrations_04_02:2:1,\
+         integrations_05_02:2:1,\
+         integrations_06_02:2:1,\
+         integrations_07_02:2:1,\
+         integrations_08_02:2:1,\
+         integrations_09_02:2:1,\
+         integrations_10_02:2:1,\
+         integrations_11_02:2:1,\
+         integrations_12_02:2:1,\
+         integrations_13_02:2:1,\
+         integrations_14_02:2:1,\
+         integrations_15_02:2:1,\
+         integrations_16_02:2:1,\
+         integrations_00_03:3:1,\
+         integrations_01_03:3:1,\
+         integrations_02_03:3:1,\
+         integrations_03_03:3:1,\
+         integrations_00_10:10:1,\
+         integrations_01_10:10:1,\
+         benchmarks_00_01:1:1,\
+         benchmarks_00_05:5:1,\
+         benchmarks_00_10:10:1"
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock

data/karafka.gemspec

@@ -14,14 +14,14 @@ Gem::Specification.new do |spec|
   spec.homepage    = 'https://karafka.io'
   spec.summary     = 'Ruby based framework for working with Apache Kafka'
   spec.description = 'Framework used to simplify Apache Kafka based Ruby applications development'
-  spec.license     = 'LGPL-3.0'
+  spec.licenses    = ['LGPL-3.0', 'Commercial']
 
   spec.add_dependency 'dry-configurable', '~> 0.13'
   spec.add_dependency 'dry-monitor', '~> 0.5'
   spec.add_dependency 'dry-validation', '~> 1.7'
   spec.add_dependency 'rdkafka', '>= 0.10'
   spec.add_dependency 'thor', '>= 0.20'
-  spec.add_dependency 'waterdrop', '>= 2.3.0', '< 3.0.0'
+  spec.add_dependency 'waterdrop', '>= 2.3.1', '< 3.0.0'
   spec.add_dependency 'zeitwerk', '~> 2.3'
 
   spec.required_ruby_version = '>= 2.6.0'

data/lib/karafka/active_job/routing/extensions.rb

@@ -10,9 +10,19 @@ module Karafka
         # This method simplifies routes definition for ActiveJob topics / queues by auto-injecting
         # the consumer class
         # @param name [String, Symbol] name of the topic where ActiveJobs jobs should go
-        def active_job_topic(name)
+        # @param block [Proc] block that we can use for some extra configuration
+        def active_job_topic(name, &block)
           topic(name) do
-            consumer App.config.internal.active_job.consumer
+            consumer App.config.internal.active_job.consumer_class
+
+            next unless block
+
+            instance_eval(&block)
+
+            # This is handled by our custom ActiveJob consumer
+            # Without this, default behaviour would cause messages to skip upon shutdown as the
+            # offset would be committed for the last message
+            manual_offset_management true
           end
         end
       end

data/lib/karafka/app.rb

@@ -10,7 +10,8 @@ module Karafka
       def consumer_groups
         config
           .internal
-          .routing_builder
+          .routing
+          .builder
       end
 
       # @return [Array<Karafka::Routing::SubscriptionGroup>] active subscription groups