karafka 2.0.0.beta3 → 2.0.0.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 86b352cc7737bde8484567662d546165793496adb6a923e33aa7e4823e4a456f
-  data.tar.gz: 2a180ba7b177db9f4c67de387262d47ac3eb2580860a548992912cd89a049da7
+  metadata.gz: 506ffb9aef3309eae2ee26e3283b7bc83859b26f4fe41995dca9d8f5e7bf0533
+  data.tar.gz: 14b39f0597676207bf9f2bf10b06c51ba539c3aa01959dfb2378c2e23941d240
 SHA512:
-  metadata.gz: 207700c3e1fab4d3370de7eddae72453fc180c65d817f4d7c021929327e5464fcd8a9bb1610c641080f82a60bc038e4517dc7bc86c0aa4c83852dd9467441e75
-  data.tar.gz: ab94239255ff841e0728c6e6585d4145a9c4f42dfd02d8df062975bd28ad3caf2cef50f5488d30a1356ad49e2529f772379bc947ba5804a15f12e56989cf9a7a
+  metadata.gz: 4b2ad5ef4eff629abfc0088be0c400bd0f9420d24c05c1414541a4c167e7e6bf2b9735bf0e596358d8e3f3c2392bf2c2b4ba345cdb8f0226b54877ce111fd749
+  data.tar.gz: 93adbc64906ff4a03e67dee646a5fe696d825357753bb804c690036800c947edaa186b67e5da8db115a0cc4a9efa46e6b1e061a608e496255bbfdc6ddeb60c14

data/.github/workflows/ci.yml

@@ -8,6 +8,10 @@ on:
   schedule:
     - cron:  '0 1 * * *'
 
+env:
+  BUNDLE_RETRY: 6
+  BUNDLE_JOBS: 4
+
 jobs:
   diffend:
     runs-on: ubuntu-latest
@@ -17,14 +21,16 @@ jobs:
       - uses: actions/checkout@v2
         with:
           fetch-depth: 0
+
       - name: Set up Ruby
         uses: ruby/setup-ruby@v1
         with:
           ruby-version: 3.1
-      - name: Install latest bundler
-        run: gem install bundler --no-document
+          bundler-cache: true
+
       - name: Install Diffend plugin
         run: bundle plugin install diffend
+
       - name: Bundle Secure
         run: bundle secure
 
@@ -57,25 +63,19 @@ jobs:
       - name: Install package dependencies
         run: "[ -e $APT_DEPS ] || sudo apt-get install -y --no-install-recommends $APT_DEPS"
 
-      - name: Run Kafka with docker-compose
+      - name: Start Kafka with docker-compose
         run: |
           docker-compose up -d
-          sleep 10
 
       - name: Set up Ruby
         uses: ruby/setup-ruby@v1
         with:
           ruby-version: ${{matrix.ruby}}
+          bundler-cache: true
 
-      - name: Install latest Bundler
+      - name: Ensure all needed Kafka topics are created and wait if not
         run: |
-          gem install bundler --no-document
-          bundle config set without 'tools benchmarks docs'
-
-      - name: Bundle install
-        run: |
-          bundle config set without development
-          bundle install --jobs 4 --retry 3
+          bin/wait_for_kafka
 
       - name: Run all specs
         env:
@@ -100,10 +100,9 @@ jobs:
       - name: Install package dependencies
         run: "[ -e $APT_DEPS ] || sudo apt-get install -y --no-install-recommends $APT_DEPS"
 
-      - name: Run Kafka with docker-compose
+      - name: Start Kafka with docker-compose
         run: |
           docker-compose up -d
-          sleep 5
 
       - name: Set up Ruby
         uses: ruby/setup-ruby@v1
@@ -119,7 +118,11 @@ jobs:
       - name: Bundle install
         run: |
           bundle config set without development
-          bundle install --jobs 4 --retry 3
+          bundle install
+
+      - name: Ensure all needed Kafka topics are created and wait if not
+        run: |
+          bin/wait_for_kafka
 
       - name: Run integration tests
         env:

data/CHANGELOG.md

@@ -1,5 +1,42 @@
 # Karafka framework changelog
 
+## 2.0.0-rc1 (2022-07-08)
+- Extract consumption partitioner out of listener inline code.
+- Introduce virtual partitioner concept for parallel processing of data from a single topic partition.
+- Improve stability when there kafka internal errors occur while polling.
+- Fix a case where we would resume a LRJ partition upon rebalance where we would reclaim the partition while job was still running.
+- Do not revoke pauses for lost partitions. This will allow to un-pause reclaimed partitions when LRJ jobs are done.
+- Fail integrations by default (unless configured otherwise) if any errors occur during Karafka server execution.
+
+## 2.0.0-beta5 (2022-07-05)
+- Always resume processing of a revoked partition upon assignment.
+- Improve specs stability.
+- Fix a case where revocation job would be executed on partition for which we never did any work.
+- Introduce a jobs group coordinator for easier jobs management.
+- Improve stability of resuming paused partitions that were revoked and re-assigned.
+- Optimize reaction time on partition ownership changes.
+- Fix a bug where despite setting long max wait time, we would return messages prior to it while not reaching the desired max messages count.
+- Add more integration specs related to polling limits.
+- Remove auto-detection of re-assigned partitions upon rebalance as for too fast rebalances it could not be accurate enough. It would also mess up in case of rebalances that would happen right after a `#seek` was issued for a partition.
+- Optimize the removal of pre-buffered lost partitions data.
+- Always rune `#revoked` when rebalance with revocation happens.
+- Evict executors upon rebalance, to prevent race-conditions.
+- Align topics names for integration specs.
+
+## 2.0.0-beta4 (2022-06-20)
+- Rename job internal api methods from `#prepare` to `#before_call` and from `#teardown` to `#after_call` to abstract away jobs execution from any type of executors and consumers logic
+- Remove ability of running `before_consume` and `after_consume` completely. Those should be for internal usage only.
+- Reorganize how Pro consumer and Pro AJ consumers inherit.
+- Require WaterDrop `2.3.1`.
+- Add more integration specs for rebalancing and max poll exceeded.
+- Move `revoked?` state from PRO to regular Karafka.
+- Use return value of `mark_as_consumed!` and `mark_as_consumed` as indicator of partition ownership + use it to switch the ownership state.
+- Do not remove rebalance manager upon client reset and recovery. This will allow us to keep the notion of lost partitions, so we can run revocation jobs for blocking jobs that exceeded the max poll interval.
+- Run revocation jobs upon reaching max poll interval for blocking jobs.
+- Early exit `poll` operation upon partition lost or max poll exceeded event.
+- Always reset consumer instances on timeout exceeded.
+- Wait for Kafka to create all the needed topics before running specs in CI.
+
 ## 2.0.0-beta3 (2022-06-14)
 - Jobs building responsibility extracted out of the listener code base.
 - Fix a case where specs supervisor would try to kill no longer running process (#868)

data/CONTRIBUTING.md

@@ -34,8 +34,3 @@ By sending a pull request to the pro components, you are agreeing to transfer th
 
 If you have any questions, create an [issue](issue) (protip: do a quick search first to see if someone else didn't ask the same question before!).
 You can also reach us at hello@karafka.opencollective.com.
-
-## Credits
-
-Thank you to all the people who have already contributed to karafka!
-<a href="graphs/contributors"><img src="https://opencollective.com/karafka/contributors.svg?width=890" /></a>

data/Gemfile.lock

@@ -1,13 +1,13 @@
 PATH
   remote: .
   specs:
-    karafka (2.0.0.beta3)
+    karafka (2.0.0.rc1)
       dry-configurable (~> 0.13)
       dry-monitor (~> 0.5)
       dry-validation (~> 1.7)
       rdkafka (>= 0.10)
       thor (>= 0.20)
-      waterdrop (>= 2.3.0, < 3.0.0)
+      waterdrop (>= 2.3.1, < 3.0.0)
       zeitwerk (~> 2.3)
 
 GEM
@@ -45,7 +45,7 @@ GEM
       dry-configurable (~> 0.13, >= 0.13.0)
       dry-core (~> 0.5, >= 0.5)
       dry-events (~> 0.2)
-    dry-schema (1.9.2)
+    dry-schema (1.9.3)
       concurrent-ruby (~> 1.0)
       dry-configurable (~> 0.13, >= 0.13.0)
       dry-core (~> 0.5, >= 0.5)
@@ -74,7 +74,7 @@ GEM
     mini_portile2 (2.8.0)
     minitest (5.15.0)
     rake (13.0.6)
-    rdkafka (0.11.1)
+    rdkafka (0.12.0)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
@@ -100,14 +100,14 @@ GEM
     thor (1.2.1)
     tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
-    waterdrop (2.3.0)
+    waterdrop (2.3.1)
       concurrent-ruby (>= 1.1)
       dry-configurable (~> 0.13)
       dry-monitor (~> 0.5)
       dry-validation (~> 1.7)
       rdkafka (>= 0.10)
       zeitwerk (~> 2.3)
-    zeitwerk (2.5.4)
+    zeitwerk (2.6.0)
 
 PLATFORMS
   x86_64-linux

data/README.md

@@ -8,7 +8,7 @@
 
 ## About Karafka
 
-Karafka is a framework used to simplify Apache Kafka based Ruby and Ruby on Rails applications development.
+Karafka is a multi-threaded framework used to simplify Apache Kafka based Ruby and Ruby on Rails applications development.
 
 ```ruby
 # Define what topics you want to consume with which consumers in karafka.rb
@@ -45,7 +45,7 @@ We also maintain many [integration specs](https://github.com/karafka/karafka/tre
 
 ## Want to Upgrade? LGPL is not for you? Want to help?
 
-I also sell Karafka Pro subscription. It includes commercial-friendly license, priority support, architecture consultations and high throughput data processing-related features (under development).
+I also sell Karafka Pro subscription. It includes commercial-friendly license, priority support, architecture consultations and high throughput data processing-related features (virtual partitions, long running jobs and more).
 
 **20%** of the income will be distributed back to other OSS projects that Karafka uses under the hood.
 
@@ -56,11 +56,3 @@ Help me provide high-quality open-source software. Please see the Karafka [homep
 Karafka has [Wiki pages](https://github.com/karafka/karafka/wiki) for almost everything and a pretty decent [FAQ](https://github.com/karafka/karafka/wiki/FAQ). It covers the whole installation, setup and deployment along with other useful details on how to run Karafka.
 
 If you have any questions about using Karafka, feel free to join our [Slack](https://slack.karafka.io) channel.
-
-## Note on contributions
-
-First, thank you for considering contributing to the Karafka ecosystem! It's people like you that make the open source community such a great community!
-
-Each pull request must pass all the RSpec specs, integration tests and meet our quality requirements.
-
-Fork it, update and wait for the Github Actions results.

data/bin/benchmarks

@@ -39,8 +39,8 @@ if ENV['SEED']
 
   # We do not populate data of benchmarks_0_10 as we use it with life-stream data only
   %w[
-    benchmarks_0_01
-    benchmarks_0_05
+    benchmarks_00_01
+    benchmarks_00_05
   ].each do |topic_name|
     partitions_count = topic_name.split('_').last.to_i
 

data/bin/integrations

@@ -21,6 +21,9 @@ ROOT_PATH = Pathname.new(File.expand_path(File.join(File.dirname(__FILE__), '../
 # of CPU
 CONCURRENCY = ENV.key?('CI') ? 5 : Etc.nprocessors * 2
 
+# How may bytes do we want to keep from the stdout in the buffer for when we need to print it
+MAX_BUFFER_OUTPUT = 10_240
+
 # Abstraction around a single test scenario execution process
 class Scenario
   # How long a scenario can run before we kill it
@@ -84,9 +87,9 @@ class Scenario
     # We read it so it won't grow as we use our default logger that prints to both test.log and
     # to stdout. Otherwise after reaching the buffer size, it would hang
     buffer = ''
-    @stdout.read_nonblock(10_240, buffer, exception: false)
+    @stdout.read_nonblock(MAX_BUFFER_OUTPUT, buffer, exception: false)
     @stdout_tail << buffer
-    @stdout_tail = @stdout_tail[-10_024..-1] || @stdout_tail
+    @stdout_tail = @stdout_tail[-MAX_BUFFER_OUTPUT..-1] || @stdout_tail
 
     !@wait_thr.alive?
   end
@@ -114,11 +117,15 @@ class Scenario
     if success?
       print "\e[#{32}m#{'.'}\e[0m"
     else
+      buffer = ''
+
+      @stderr.read_nonblock(MAX_BUFFER_OUTPUT, buffer, exception: false)
+
       puts
       puts "\e[#{31}m#{'[FAILED]'}\e[0m #{name}"
       puts "Exit code: #{exit_code}"
       puts @stdout_tail
-      puts @stderr.read
+      puts buffer
       puts
     end
   end

data/bin/{stress → stress_many}

@@ -8,6 +8,6 @@ set -e
 
 while :
 do
-  reset
+  clear
   bin/integrations $1
 done

data/bin/stress_one

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# Runs a single integration spec in an endless loop
+# This allows us to ensure (after long enough time) that the integration spec is stable and
+# that there are no anomalies when running it for a long period of time
+
+set -e
+
+while :
+do
+  clear
+  bin/scenario $1
+done

data/bin/wait_for_kafka

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# This script allows us to wait for Kafka docker to fully be ready
+# We consider it fully ready when all our topics that need to be created are created as expected
+
+KAFKA_NAME='karafka_20_kafka'
+ZOOKEEPER='zookeeper:2181'
+LIST_CMD="kafka-topics.sh --list --zookeeper $ZOOKEEPER"
+
+# Take the number of topics that we need to create prior to running anything
+TOPICS_COUNT=`cat docker-compose.yml | grep -E -i 'integrations_|benchmarks_' | wc -l`
+
+# And wait until all of them are created
+until (((`docker exec $KAFKA_NAME $LIST_CMD | wc -l`) >= $TOPICS_COUNT));
+do
+  echo "Waiting for Kafka to create all the needed topics..."
+  sleep 1
+done
+
+echo "All the needed topics created."

data/docker-compose.yml

@@ -1,10 +1,12 @@
 version: '2'
 services:
   zookeeper:
+    container_name: karafka_20_zookeeper
     image: wurstmeister/zookeeper
     ports:
       - '2181:2181'
   kafka:
+    container_name: karafka_20_kafka
     image: wurstmeister/kafka
     ports:
       - '9092:9092'
@@ -14,18 +16,35 @@ services:
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
       KAFKA_CREATE_TOPICS:
-        "integrations_0_02:2:1,\
-         integrations_1_02:2:1,\
-         integrations_2_02:2:1,\
-         integrations_3_02:2:1,\
-         integrations_4_02:2:1,\
-         integrations_0_03:3:1,\
-         integrations_1_03:3:1,\
-         integrations_2_03:3:1,\
-         integrations_0_10:10:1,\
-         integrations_1_10:10:1,\
-         benchmarks_0_01:1:1,\
-         benchmarks_0_05:5:1,\
-         benchmarks_0_10:10:1"
+        "integrations_00_02:2:1,\
+         integrations_01_02:2:1,\
+         integrations_02_02:2:1,\
+         integrations_03_02:2:1,\
+         integrations_04_02:2:1,\
+         integrations_05_02:2:1,\
+         integrations_06_02:2:1,\
+         integrations_07_02:2:1,\
+         integrations_08_02:2:1,\
+         integrations_09_02:2:1,\
+         integrations_10_02:2:1,\
+         integrations_11_02:2:1,\
+         integrations_12_02:2:1,\
+         integrations_13_02:2:1,\
+         integrations_14_02:2:1,\
+         integrations_15_02:2:1,\
+         integrations_16_02:2:1,\
+         integrations_17_02:2:1,\
+         integrations_18_02:2:1,\
+         integrations_19_02:2:1,\
+         integrations_00_03:3:1,\
+         integrations_01_03:3:1,\
+         integrations_02_03:3:1,\
+         integrations_03_03:3:1,\
+         integrations_00_10:10:1,\
+         integrations_01_10:10:1,\
+         benchmarks_00_01:1:1,\
+         benchmarks_00_05:5:1,\
+         benchmarks_01_05:5:1,\
+         benchmarks_00_10:10:1"
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock

data/karafka.gemspec

@@ -21,7 +21,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'dry-validation', '~> 1.7'
   spec.add_dependency 'rdkafka', '>= 0.10'
   spec.add_dependency 'thor', '>= 0.20'
-  spec.add_dependency 'waterdrop', '>= 2.3.0', '< 3.0.0'
+  spec.add_dependency 'waterdrop', '>= 2.3.1', '< 3.0.0'
   spec.add_dependency 'zeitwerk', '~> 2.3'
 
   spec.required_ruby_version = '>= 2.6.0'

data/lib/karafka/active_job/routing/extensions.rb

@@ -13,7 +13,7 @@ module Karafka
         # @param block [Proc] block that we can use for some extra configuration
         def active_job_topic(name, &block)
           topic(name) do
-            consumer App.config.internal.active_job.consumer
+            consumer App.config.internal.active_job.consumer_class
 
             next unless block
 

data/lib/karafka/app.rb

@@ -10,7 +10,8 @@ module Karafka
       def consumer_groups
         config
           .internal
-          .routing_builder
+          .routing
+          .builder
       end
 
       # @return [Array<Karafka::Routing::SubscriptionGroup>] active subscription groups

data/lib/karafka/base_consumer.rb

@@ -10,8 +10,8 @@ module Karafka
     attr_accessor :messages
     # @return [Karafka::Connection::Client] kafka connection client
     attr_accessor :client
-    # @return [Karafka::TimeTrackers::Pause] current topic partition pause tracker
-    attr_accessor :pause_tracker
+    # @return [Karafka::Processing::Coordinator] coordinator
+    attr_accessor :coordinator
     # @return [Waterdrop::Producer] producer instance
     attr_accessor :producer
 
@@ -21,22 +21,7 @@ module Karafka
     # @note This should not be used by the end users as it is part of the lifecycle of things but
     #   not as part of the public api. This can act as a hook when creating non-blocking
     #   consumers and doing other advanced stuff
-    def on_prepare
-      Karafka.monitor.instrument('consumer.prepared', caller: self) do
-        prepare
-      end
-
-      true
-    rescue StandardError => e
-      Karafka.monitor.instrument(
-        'error.occurred',
-        error: e,
-        caller: self,
-        type: 'consumer.prepare.error'
-      )
-
-      false
-    end
+    def on_before_consume; end
 
     # Executes the default consumer flow.
     #
@@ -48,40 +33,52 @@ module Karafka
     def on_consume
       Karafka.monitor.instrument('consumer.consumed', caller: self) do
         consume
-
-        pause_tracker.reset
-
-        # Mark as consumed only if manual offset management is not on
-        next if topic.manual_offset_management
-
-        # We use the non-blocking one here. If someone needs the blocking one, can implement it
-        # with manual offset management
-        mark_as_consumed(messages.last)
       end
 
-      true
+      coordinator.consumption(self).success!
     rescue StandardError => e
+      coordinator.consumption(self).failure!
+
       Karafka.monitor.instrument(
         'error.occurred',
         error: e,
         caller: self,
         type: 'consumer.consume.error'
       )
+    ensure
+      # We need to decrease number of jobs that this coordinator coordinates as it has finished
+      coordinator.decrement
+    end
+
+    # @private
+    # @note This should not be used by the end users as it is part of the lifecycle of things but
+    #   not as part of the public api.
+    def on_after_consume
+      return if revoked?
+
+      if coordinator.success?
+        coordinator.pause_tracker.reset
 
-      pause(@seek_offset || messages.first.offset)
+        # Mark as consumed only if manual offset management is not on
+        return if topic.manual_offset_management?
 
-      false
+        # We use the non-blocking one here. If someone needs the blocking one, can implement it
+        # with manual offset management
+        mark_as_consumed(messages.last)
+      else
+        pause(@seek_offset || messages.first.offset)
+      end
     end
 
     # Trigger method for running on shutdown.
     #
     # @private
     def on_revoked
+      coordinator.revoke
+
       Karafka.monitor.instrument('consumer.revoked', caller: self) do
         revoked
       end
-
-      true
     rescue StandardError => e
       Karafka.monitor.instrument(
         'error.occurred',
@@ -89,8 +86,6 @@ module Karafka
         caller: self,
         type: 'consumer.revoked.error'
       )
-
-      false
     end
 
     # Trigger method for running on shutdown.
@@ -100,8 +95,6 @@ module Karafka
       Karafka.monitor.instrument('consumer.shutdown', caller: self) do
         shutdown
       end
-
-      true
     rescue StandardError => e
       Karafka.monitor.instrument(
         'error.occurred',
@@ -109,16 +102,10 @@ module Karafka
         caller: self,
         type: 'consumer.shutdown.error'
       )
-
-      false
     end
 
     private
 
-    # Method that gets called in the blocking flow allowing to setup any type of resources or to
-    # send additional commands to Kafka before the proper execution starts.
-    def prepare; end
-
     # Method that will perform business logic and on data received from Kafka (it will consume
     #   the data)
     # @note This method needs bo be implemented in a subclass. We stub it here as a failover if
@@ -138,21 +125,40 @@ module Karafka
     # Marks message as consumed in an async way.
     #
     # @param message [Messages::Message] last successfully processed message.
+    # @return [Boolean] true if we were able to mark the offset, false otherwise. False indicates
+    #   that we were not able and that we have lost the partition.
+    #
     # @note We keep track of this offset in case we would mark as consumed and got error when
     #   processing another message. In case like this we do not pause on the message we've already
     #   processed but rather at the next one. This applies to both sync and async versions of this
     #   method.
     def mark_as_consumed(message)
-      client.mark_as_consumed(message)
+      unless client.mark_as_consumed(message)
+        coordinator.revoke
+
+        return false
+      end
+
       @seek_offset = message.offset + 1
+
+      true
     end
 
     # Marks message as consumed in a sync way.
     #
     # @param message [Messages::Message] last successfully processed message.
+    # @return [Boolean] true if we were able to mark the offset, false otherwise. False indicates
+    #   that we were not able and that we have lost the partition.
     def mark_as_consumed!(message)
-      client.mark_as_consumed!(message)
+      unless client.mark_as_consumed!(message)
+        coordinator.revoke
+
+        return false
+      end
+
       @seek_offset = message.offset + 1
+
+      true
     end
 
     # Pauses processing on a given offset for the current topic partition
@@ -162,7 +168,7 @@ module Karafka
     # @param timeout [Integer, nil] how long in milliseconds do we want to pause or nil to use the
     #   default exponential pausing strategy defined for retries
     def pause(offset, timeout = nil)
-      timeout ? pause_tracker.pause(timeout) : pause_tracker.pause
+      timeout ? coordinator.pause_tracker.pause(timeout) : coordinator.pause_tracker.pause
 
       client.pause(
         messages.metadata.topic,
@@ -175,7 +181,7 @@ module Karafka
     def resume
       # This is sufficient to expire a partition pause, as with it will be resumed by the listener
       # thread before the next poll.
-      pause_tracker.expire
+      coordinator.pause_tracker.expire
     end
 
     # Seeks in the context of current topic and partition
@@ -190,5 +196,12 @@ module Karafka
         )
       )
     end
+
+    # @return [Boolean] true if partition was revoked from the current consumer
+    # @note We know that partition got revoked because when we try to mark message as consumed,
+    #   unless if is successful, it will return false
+    def revoked?
+      coordinator.revoked?
+    end
   end
 end