karafka 2.0.0.beta5 → 2.0.0.rc3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2c8e680ffdf69f88899a715c84cc484e8f568f4a93da9284195f4bf55a283ee1
-  data.tar.gz: 974356226a10ba2c77de770351a47180716533021a89040bcdc1aae57f452121
+  metadata.gz: b2a60c5c750ef681042f6e0a0d61032c2b9d9e8d48fa73ec480157c7bffb4cc7
+  data.tar.gz: c04074859408e77d236e86dcfdbe8502ba5322f6e7d8d1e236c9aa838b3e1a29
 SHA512:
-  metadata.gz: 2427aaae1b1b07430df7c9f042d290bbae8380fb1f6ec7c26eecee92b8fe79e13ea9f3a99a36bf89b314ffba809c556618b22c0a87f0c0c83bb73cf8af72321b
-  data.tar.gz: 55e18448b5645acd38c4194967ea7df657c142d82a105699f7b204f222f8dfb2dbd14cce82b1f424ec177afb78049b3e7588642013674a3c2923a8848b6b87e7
+  metadata.gz: 487dc8f5fb131dc8f25087ae026b40710c9859e7849b878ac42053737dd665a0147462fd6348e3e03eb87e0d6657c8f0038b02506f525ee221ee1f6d806bcf84
+  data.tar.gz: 23a68b2d97cd5441088521c949c2a0ec23befde2fa81c9b9178ea6786a582aea5e342697d08f9a3dd1dfe1401792bcbc3e936b91c87eb740ef66beccd69b39bf

data/CHANGELOG.md

@@ -1,5 +1,33 @@
 # Karafka framework changelog
 
+## 2.0.0.rc3 (2022-07-26)
+- Fix Pro partitioner hash function may not utilize all the threads (#907).
+- Improve virtual partitions messages distribution.
+- Add StatsD/DataDog optional monitoring listener + dashboard template.
+- Validate that Pro consumer is always used for Pro subscription.
+- Improve ActiveJob consumer shutdown behaviour.
+- Change default `max_wait_time` to 1 second.
+- Change default `max_messages` to 100 (#915).
+- Move logger listener polling reporting level to debug when no messages (#916).
+- Improve stability on aggressive rebalancing (multiple rebalances in a short period).
+- Improve specs stability.
+- Allow using `:key` and `:partition_key` for Enhanced Active Job partitioning.
+
+## 2.0.0.rc2 (2022-07-19)
+- Fix `example_consumer.rb.erb` `#shutdown` and `#revoked` signatures to correct once.
+- Improve the install user experience (print status and created files).
+- Change default `max_wait_time` from 10s to 5s.
+- Remove direct dependency on `dry-configurable` in favour of a home-brew.
+- Remove direct dependency on `dry-validation` in favour of a home-brew.
+
+## 2.0.0-rc1 (2022-07-08)
+- Extract consumption partitioner out of listener inline code.
+- Introduce virtual partitioner concept for parallel processing of data from a single topic partition.
+- Improve stability when there kafka internal errors occur while polling.
+- Fix a case where we would resume a LRJ partition upon rebalance where we would reclaim the partition while job was still running.
+- Do not revoke pauses for lost partitions. This will allow to un-pause reclaimed partitions when LRJ jobs are done.
+- Fail integrations by default (unless configured otherwise) if any errors occur during Karafka server execution.
+
 ## 2.0.0-beta5 (2022-07-05)
 - Always resume processing of a revoked partition upon assignment.
 - Improve specs stability.

data/CONTRIBUTING.md

@@ -34,8 +34,3 @@ By sending a pull request to the pro components, you are agreeing to transfer th
 
 If you have any questions, create an [issue](issue) (protip: do a quick search first to see if someone else didn't ask the same question before!).
 You can also reach us at hello@karafka.opencollective.com.
-
-## Credits
-
-Thank you to all the people who have already contributed to karafka!
-<a href="graphs/contributors"><img src="https://opencollective.com/karafka/contributors.svg?width=890" /></a>

data/Gemfile.lock

@@ -1,22 +1,20 @@
 PATH
   remote: .
   specs:
-    karafka (2.0.0.beta5)
-      dry-configurable (~> 0.13)
+    karafka (2.0.0.rc3)
       dry-monitor (~> 0.5)
-      dry-validation (~> 1.7)
       rdkafka (>= 0.10)
       thor (>= 0.20)
-      waterdrop (>= 2.3.1, < 3.0.0)
+      waterdrop (>= 2.3.3, < 3.0.0)
       zeitwerk (~> 2.3)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activejob (7.0.3)
-      activesupport (= 7.0.3)
+    activejob (7.0.3.1)
+      activesupport (= 7.0.3.1)
       globalid (>= 0.3.6)
-    activesupport (7.0.3)
+    activesupport (7.0.3.1)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -28,51 +26,25 @@ GEM
     dry-configurable (0.15.0)
       concurrent-ruby (~> 1.0)
       dry-core (~> 0.6)
-    dry-container (0.9.0)
-      concurrent-ruby (~> 1.0)
-      dry-configurable (~> 0.13, >= 0.13.0)
-    dry-core (0.7.1)
+    dry-core (0.8.0)
       concurrent-ruby (~> 1.0)
     dry-events (0.3.0)
       concurrent-ruby (~> 1.0)
       dry-core (~> 0.5, >= 0.5)
-    dry-inflector (0.2.1)
-    dry-initializer (3.1.1)
-    dry-logic (1.2.0)
-      concurrent-ruby (~> 1.0)
-      dry-core (~> 0.5, >= 0.5)
-    dry-monitor (0.5.0)
+    dry-monitor (0.6.1)
       dry-configurable (~> 0.13, >= 0.13.0)
       dry-core (~> 0.5, >= 0.5)
       dry-events (~> 0.2)
-    dry-schema (1.9.2)
-      concurrent-ruby (~> 1.0)
-      dry-configurable (~> 0.13, >= 0.13.0)
-      dry-core (~> 0.5, >= 0.5)
-      dry-initializer (~> 3.0)
-      dry-logic (~> 1.0)
-      dry-types (~> 1.5)
-    dry-types (1.5.1)
-      concurrent-ruby (~> 1.0)
-      dry-container (~> 0.3)
-      dry-core (~> 0.5, >= 0.5)
-      dry-inflector (~> 0.1, >= 0.1.2)
-      dry-logic (~> 1.0, >= 1.0.2)
-    dry-validation (1.8.1)
-      concurrent-ruby (~> 1.0)
-      dry-container (~> 0.7, >= 0.7.1)
-      dry-core (~> 0.5, >= 0.5)
-      dry-initializer (~> 3.0)
-      dry-schema (~> 1.8, >= 1.8.0)
+      zeitwerk (~> 2.5)
     factory_bot (6.2.1)
       activesupport (>= 5.0.0)
     ffi (1.15.5)
     globalid (1.0.0)
       activesupport (>= 5.0)
-    i18n (1.10.0)
+    i18n (1.12.0)
       concurrent-ruby (~> 1.0)
     mini_portile2 (2.8.0)
-    minitest (5.15.0)
+    minitest (5.16.2)
     rake (13.0.6)
     rdkafka (0.12.0)
       ffi (~> 1.15)
@@ -100,11 +72,9 @@ GEM
     thor (1.2.1)
     tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
-    waterdrop (2.3.1)
+    waterdrop (2.3.3)
       concurrent-ruby (>= 1.1)
-      dry-configurable (~> 0.13)
       dry-monitor (~> 0.5)
-      dry-validation (~> 1.7)
       rdkafka (>= 0.10)
       zeitwerk (~> 2.3)
     zeitwerk (2.6.0)
@@ -121,4 +91,4 @@ DEPENDENCIES
   simplecov
 
 BUNDLED WITH
-   2.3.11
+   2.3.15

data/LICENSE-COMM

@@ -30,7 +30,7 @@ The Open Source version of the Software (“LGPL Version”) is licensed under t
 
 4. Ownership. Notwithstanding anything to the contrary contained herein, except for the limited license rights expressly provided herein, Maciej Mensfeld and its suppliers have and will retain all rights, title and interest (including, without limitation, all patent, copyright, trademark, trade secret and other intellectual property rights) in and to the Software and all copies, modifications and derivative works thereof (including any changes which incorporate any of your ideas, feedback or suggestions). You acknowledge that you are obtaining only a limited license right to the Software, and that irrespective of any use of the words “purchase”, “sale” or like terms hereunder no ownership rights are being conveyed to you under this Agreement or otherwise.
 
-5. Fees and Payment. The Software license fees will be due and payable in full as set forth in the applicable invoice or at the time of purchase. If the Software does not function properly within two weeks of purchase, please contact us within those two weeks for a refund. You shall be responsible for all taxes, withholdings, duties and levies arising from the order (excluding taxes based on the net income of Maciej Mensfeld).
+5. Fees and Payment. The Software license fees will be due and payable in full as set forth in the applicable invoice or at the time of purchase. There are no refunds beyond the remedy refund.
 
 6. Support, Maintenance and Services. Subject to the terms and conditions of this Agreement, as set forth in your invoice, and as set forth on the Karafka Pro support page (https://github.com/karafka/karafka/wiki/Commercial-Support), support and maintenance services may be included with the purchase of your license subscription.
 

data/README.md

@@ -4,11 +4,18 @@
 [![Gem Version](https://badge.fury.io/rb/karafka.svg)](http://badge.fury.io/rb/karafka)
 [![Join the chat at https://slack.karafka.io](https://raw.githubusercontent.com/karafka/misc/master/slack.svg)](https://slack.karafka.io)
 
-**Note**: All of the documentation here refers to Karafka `2.0`. If you are looking for the documentation to Karafka `1.4` please click here (TBA).
+**Note**: All of the documentation here refers to Karafka `2.0.0.rc2` or higher. If you are looking for the documentation for Karafka `1.4`, please click [here](https://github.com/karafka/wiki/tree/1.4).
 
 ## About Karafka
 
-Karafka is a framework used to simplify Apache Kafka based Ruby and Ruby on Rails applications development.
+Karafka is a Ruby and Rails multi-threaded efficient Kafka processing framework that:
+
+- Supports parallel processing in [multiple threads](Concurrency-and-multithreading) (also for a single topic partition work)
+- Has [ActiveJob backend](Active-Job) support (including ordered jobs)
+- [Automatically integrates](Integrating-with-Ruby-on-Rails-and-other-frameworks#integrating-with-ruby-on-rails=) with Ruby on Rails
+- Supports in-development [code reloading](Auto-reload-of-code-changes-in-development)
+- Is powered by [librdkafka](https://github.com/edenhill/librdkafka) (the Apache Kafka C/C++ client library)
+- Has an out-of the box [StatsD/DataDog monitoring](Monitoring-and-logging) with a dashboard template.
 
 ```ruby
 # Define what topics you want to consume with which consumers in karafka.rb
@@ -28,13 +35,11 @@ class EventsConsumer < ApplicationConsumer
 end
 ```
 
-Karafka allows you to capture everything that happens in your systems in large scale, providing you with a seamless and stable core for consuming, processing and producing data, without having to focus on things that are not your business domain.
-
-Karafka **uses** threads to handle many messages at the same time in the same process. It does not require Rails but will integrate tightly with any Ruby on Rails applications to make event processing dead simple.
+Karafka **uses** threads to handle many messages simultaneously in the same process. It does not require Rails but will integrate tightly with any Ruby on Rails applications to make event processing dead simple.
 
 ## Getting started
 
-If you're completely new to the subject, you can start with our "Kafka on Rails" articles series, that will get you up and running with the terminology and basic ideas behind using Kafka:
+If you're entirely new to the subject, you can start with our "Kafka on Rails" articles series, which will get you up and running with the terminology and basic ideas behind using Kafka:
 
 - [Kafka on Rails: Using Kafka with Ruby on Rails – Part 1 – Kafka basics and its advantages](https://mensfeld.pl/2017/11/kafka-on-rails-using-kafka-with-ruby-on-rails-part-1-kafka-basics-and-its-advantages/)
 - [Kafka on Rails: Using Kafka with Ruby on Rails – Part 2 – Getting started with Ruby and Kafka](https://mensfeld.pl/2018/01/kafka-on-rails-using-kafka-with-ruby-on-rails-part-2-getting-started-with-ruby-and-kafka/)
@@ -43,9 +48,38 @@ If you want to get started with Kafka and Karafka as fast as possible, then the
 
 We also maintain many [integration specs](https://github.com/karafka/karafka/tree/master/spec/integrations) illustrating various use-cases and features of the framework.
 
+### TL;DR (1 minute from setup to publishing and consuming messages)
+
+**Prerequisites**: Kafka running. You can start it by following instructions from [here](https://github.com/karafka/karafka/wiki/Setting-up-Kafka).
+
+1. Add and install Karafka:
+
+```bash
+bundle add karafka -v 2.0.0.rc2
+
+bundle exec karafka install
+```
+
+2. Dispatch a message to the example topic using the Rails or Ruby console:
+
+```ruby
+Karafka.producer.produce_sync(topic: 'example', payload: { 'ping' => 'pong' }.to_json)
+```
+
+3. Run Karafka server and see the consumption magic happen:
+
+```bash
+bundle exec karafka server
+
+[7616dc24-505a-417f-b87b-6bf8fc2d98c5] Polled 1 message in 1000ms
+[dcf3a8d8-0bd9-433a-8f63-b70a0cdb0732] Consume job for ExampleConsumer on example started
+{"ping"=>"pong"}
+[dcf3a8d8-0bd9-433a-8f63-b70a0cdb0732] Consume job for ExampleConsumer on example finished in 0ms
+```
+
 ## Want to Upgrade? LGPL is not for you? Want to help?
 
-I also sell Karafka Pro subscription. It includes commercial-friendly license, priority support, architecture consultations and high throughput data processing-related features (under development).
+I also sell Karafka Pro subscriptions. It includes a commercial-friendly license, priority support, architecture consultations, and high throughput data processing-related features (virtual partitions, long-running jobs, and more).
 
 **20%** of the income will be distributed back to other OSS projects that Karafka uses under the hood.
 
@@ -53,14 +87,8 @@ Help me provide high-quality open-source software. Please see the Karafka [homep
 
 ## Support
 
-Karafka has [Wiki pages](https://github.com/karafka/karafka/wiki) for almost everything and a pretty decent [FAQ](https://github.com/karafka/karafka/wiki/FAQ). It covers the whole installation, setup and deployment along with other useful details on how to run Karafka.
-
-If you have any questions about using Karafka, feel free to join our [Slack](https://slack.karafka.io) channel.
-
-## Note on contributions
-
-First, thank you for considering contributing to the Karafka ecosystem! It's people like you that make the open source community such a great community!
+Karafka has [Wiki pages](https://github.com/karafka/karafka/wiki) for almost everything and a pretty decent [FAQ](https://github.com/karafka/karafka/wiki/FAQ). It covers the installation, setup, and deployment, along with other useful details on how to run Karafka.
 
-Each pull request must pass all the RSpec specs, integration tests and meet our quality requirements.
+If you have questions about using Karafka, feel free to join our [Slack](https://slack.karafka.io) channel.
 
-Fork it, update and wait for the Github Actions results.
+Karafka has priority support for technical and architectural questions that is part of the Karafka Pro subscription.

data/bin/stress_many

@@ -8,6 +8,6 @@ set -e
 
 while :
 do
-  reset
+  clear
   bin/integrations $1
 done

data/bin/stress_one

@@ -8,6 +8,6 @@ set -e
 
 while :
 do
-  reset
+  clear
   bin/scenario $1
 done

data/config/errors.yml

@@ -1,9 +1,56 @@
 en:
-  dry_validation:
-    errors:
+  validations:
+    config:
+      missing: needs to be present
+      client_id_format: 'needs to be a string with a Kafka accepted format'
+      license.entity_format: needs to be a string
+      license.token_format: needs to be either false or a string
+      license.expires_on_format: needs to be a valid date
+      concurrency_format: needs to be an integer bigger than 0
+      consumer_mapper_format: needs to be present
+      consumer_persistence_format: needs to be either true or false
+      pause_timeout_format: needs to be an integer bigger than 0
+      pause_max_timeout_format: needs to be an integer bigger than 0
+      pause_with_exponential_backoff_format: needs to be either true or false
+      shutdown_timeout_format: needs to be an integer bigger than 0
+      max_wait_time_format: needs to be an integer bigger than 0
+      kafka_format: needs to be a filled hash
+      internal.status_format: needs to be present
+      internal.process_format: needs to be present
+      internal.routing.builder_format: needs to be present
+      internal.routing.subscription_groups_builder_format: needs to be present
+      key_must_be_a_symbol: All keys under the kafka settings scope need to be symbols
       max_timeout_vs_pause_max_timeout: pause_timeout must be less or equal to pause_max_timeout
       shutdown_timeout_vs_max_wait_time: shutdown_timeout must be more than max_wait_time
-      topics_names_not_unique: all topic names within a single consumer group must be unique
-      required_usage_count: Given topic must be used at least once
+
+    server_cli_options:
+      missing: needs to be present
       consumer_groups_inclusion: Unknown consumer group
-      kafka_key_must_be_a_symbol: All keys under the kafka settings scope need to be symbols
+
+    consumer_group_topic:
+      missing: needs to be present
+      name_format: 'needs to be a string with a Kafka accepted format'
+      deserializer_format: needs to be present
+      manual_offset_management_format: needs to be either true or false
+      consumer_format: needs to be present
+      id_format: 'needs to be a string with a Kafka accepted format'
+      initial_offset_format: needs to be either earliest or latest
+
+    consumer_group:
+      missing: needs to be present
+      topics_names_not_unique: all topic names within a single consumer group must be unique
+      id_format: 'needs to be a string with a Kafka accepted format'
+      topics_format: needs to be a non-empty array
+
+    job_options:
+      missing: needs to be present
+      dispatch_method_format: needs to be either :produce_async or :produce_sync
+      partitioner_format: 'needs to respond to #call'
+      partition_key_type_format: 'needs to be either :key or :partition_key'
+
+    test:
+      missing: needs to be present
+      id_format: needs to be a String
+
+    pro_consumer_group_topic:
+      consumer_format: needs to inherit from Karafka::Pro::BaseConsumer and not Karafka::Consumer

data/docker-compose.yml

@@ -33,14 +33,21 @@ services:
          integrations_14_02:2:1,\
          integrations_15_02:2:1,\
          integrations_16_02:2:1,\
+         integrations_17_02:2:1,\
+         integrations_18_02:2:1,\
+         integrations_19_02:2:1,\
+         integrations_20_02:2:1,\
+         integrations_21_02:2:1,\
          integrations_00_03:3:1,\
          integrations_01_03:3:1,\
          integrations_02_03:3:1,\
          integrations_03_03:3:1,\
+         integrations_04_03:3:1,\
          integrations_00_10:10:1,\
          integrations_01_10:10:1,\
          benchmarks_00_01:1:1,\
          benchmarks_00_05:5:1,\
+         benchmarks_01_05:5:1,\
          benchmarks_00_10:10:1"
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock

data/karafka.gemspec

@@ -12,16 +12,14 @@ Gem::Specification.new do |spec|
   spec.authors     = ['Maciej Mensfeld']
   spec.email       = %w[maciej@mensfeld.pl]
   spec.homepage    = 'https://karafka.io'
-  spec.summary     = 'Ruby based framework for working with Apache Kafka'
+  spec.summary     = 'Efficient Kafka processing framework for Ruby and Rails '
   spec.description = 'Framework used to simplify Apache Kafka based Ruby applications development'
   spec.licenses    = ['LGPL-3.0', 'Commercial']
 
-  spec.add_dependency 'dry-configurable', '~> 0.13'
   spec.add_dependency 'dry-monitor', '~> 0.5'
-  spec.add_dependency 'dry-validation', '~> 1.7'
   spec.add_dependency 'rdkafka', '>= 0.10'
   spec.add_dependency 'thor', '>= 0.20'
-  spec.add_dependency 'waterdrop', '>= 2.3.1', '< 3.0.0'
+  spec.add_dependency 'waterdrop', '>= 2.3.3', '< 3.0.0'
   spec.add_dependency 'zeitwerk', '~> 2.3'
 
   spec.required_ruby_version = '>= 2.6.0'

data/lib/karafka/active_job/consumer.rb

@@ -9,6 +9,8 @@ module Karafka
       # @note ActiveJob does not support batches, so we just run one message after another
       def consume
         messages.each do |message|
+          break if Karafka::App.stopping?
+
           ::ActiveJob::Base.execute(
             # We technically speaking could set this as deserializer and reference it from the
             # message instead of using the `#raw_payload`. This is not done on purpose to simplify

data/lib/karafka/active_job/job_options_contract.rb

@@ -7,9 +7,15 @@ module Karafka
     #   we want to keep ActiveJob related Karafka components outside of the core Karafka code and
     #   all in the same place
     class JobOptionsContract < Contracts::Base
-      params do
-        optional(:dispatch_method).value(included_in?: %i[produce_async produce_sync])
+      configure do |config|
+        config.error_messages = YAML.safe_load(
+          File.read(
+            File.join(Karafka.gem_root, 'config', 'errors.yml')
+          )
+        ).fetch('en').fetch('validations').fetch('job_options')
       end
+
+      optional(:dispatch_method) { |val| %i[produce_async produce_sync].include?(val) }
     end
   end
 end

data/lib/karafka/base_consumer.rb

@@ -35,9 +35,9 @@ module Karafka
         consume
       end
 
-      @coordinator.consumption(self).success!
+      coordinator.consumption(self).success!
     rescue StandardError => e
-      @coordinator.consumption(self).failure!
+      coordinator.consumption(self).failure!
 
       Karafka.monitor.instrument(
         'error.occurred',
@@ -47,7 +47,7 @@ module Karafka
       )
     ensure
       # We need to decrease number of jobs that this coordinator coordinates as it has finished
-      @coordinator.decrement
+      coordinator.decrement
     end
 
     # @private
@@ -56,7 +56,7 @@ module Karafka
     def on_after_consume
       return if revoked?
 
-      if @coordinator.success?
+      if coordinator.success?
         coordinator.pause_tracker.reset
 
         # Mark as consumed only if manual offset management is not on
@@ -76,8 +76,6 @@ module Karafka
     def on_revoked
       coordinator.revoke
 
-      resume
-
       Karafka.monitor.instrument('consumer.revoked', caller: self) do
         revoked
       end

data/lib/karafka/cli/install.rb

@@ -7,6 +7,8 @@ module Karafka
   class Cli < Thor
     # Install Karafka Cli action
     class Install < Base
+      include Helpers::Colorize
+
       desc 'Install all required things for Karafka application in current directory'
 
       # Directories created by default
@@ -42,14 +44,25 @@ module Karafka
           FileUtils.mkdir_p Karafka.root.join(dir)
         end
 
+        puts
+        puts 'Installing Karafka framework...'
+        puts 'Ruby on Rails detected...' if rails?
+        puts
+
         INSTALL_FILES_MAP.each do |source, target|
-          target = Karafka.root.join(target)
+          pathed_target = Karafka.root.join(target)
 
           template = File.read(Karafka.core_root.join("templates/#{source}"))
           render = ::ERB.new(template, trim_mode: '-').result(binding)
 
-          File.open(target, 'w') { |file| file.write(render) }
+          File.open(pathed_target, 'w') { |file| file.write(render) }
+
+          puts "#{green('Created')} #{target}"
         end
+
+        puts
+        puts("Installation #{green('completed')}. Have fun!")
+        puts
       end
 
       # @return [Boolean] true if we have Rails loaded

data/lib/karafka/cli/server.rb

@@ -5,6 +5,8 @@ module Karafka
   class Cli < Thor
     # Server Karafka Cli action
     class Server < Base
+      include Helpers::Colorize
+
       desc 'Start the Karafka server (short-cut alias: "s")'
       option aliases: 's'
       option :consumer_groups, type: :array, default: nil, aliases: :g
@@ -31,11 +33,11 @@ module Karafka
 
         if Karafka.pro?
           Karafka.logger.info(
-            "\033[0;32mThank you for investing in the Karafka Pro subscription!\033[0m\n"
+            green('Thank you for investing in the Karafka Pro subscription!')
           )
         else
           Karafka.logger.info(
-            "\033[0;31mYou like Karafka? Please consider getting a Pro subscription!\033[0m\n"
+            red('You like Karafka? Please consider getting a Pro version!')
           )
         end
       end

data/lib/karafka/connection/client.rb

@@ -69,9 +69,6 @@ module Karafka
           # Put a message to the buffer if there is one
           @buffer << message if message
 
-          # Track time spent on all of the processing and polling
-          time_poll.checkpoint
-
           # Upon polling rebalance manager might have been updated.
           # If partition revocation happens, we need to remove messages from revoked partitions
           # as well as ensure we do not have duplicated due to the offset reset for partitions
@@ -82,6 +79,9 @@ module Karafka
             break
           end
 
+          # Track time spent on all of the processing and polling
+          time_poll.checkpoint
+
           # Finally once we've (potentially) removed revoked, etc, if no messages were returned
           # we can break.
           # Worth keeping in mind, that the rebalance manager might have been updated despite no
@@ -268,7 +268,7 @@ module Karafka
         true
       rescue Rdkafka::RdkafkaError => e
         return false if e.code == :assignment_lost
-        return false if e.code == :no_offset
+        return true if e.code == :no_offset
 
         raise e
       end
@@ -316,37 +316,40 @@ module Karafka
 
         time_poll.start
 
-        @kafka.poll(time_poll.remaining)
+        @kafka.poll(timeout)
       rescue ::Rdkafka::RdkafkaError => e
-        raise if time_poll.attempts > MAX_POLL_RETRIES
-        raise unless time_poll.retryable?
-
+        # We return nil, so we do not restart until running the whole loop
+        # This allows us to run revocation jobs and other things and we will pick up new work
+        # next time after dispatching all the things that are needed
+        #
+        # If we would retry here, the client reset would become transparent and we would not have
+        # a chance to take any actions
         case e.code
         when :max_poll_exceeded # -147
           reset
+          return nil
         when :transport # -195
           reset
+          return nil
         when :rebalance_in_progress # -27
           reset
+          return nil
         when :not_coordinator # 16
           reset
+          return nil
         when :network_exception # 13
           reset
+          return nil
         end
 
-        time_poll.checkpoint
-
+        raise if time_poll.attempts > MAX_POLL_RETRIES
         raise unless time_poll.retryable?
 
+        time_poll.checkpoint
         time_poll.backoff
 
-        # We return nil, so we do not restart until running the whole loop
-        # This allows us to run revocation jobs and other things and we will pick up new work
-        # next time after dispatching all the things that are needed
-        #
-        # If we would retry here, the client reset would become transparent and we would not have
-        # a chance to take any actions
-        nil
+        # On unknown errors we do our best to retry and handle them before raising
+        retry
       end
 
       # Builds a new rdkafka consumer instance based on the subscription group configuration