racecar 2.12.0 → 2.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a4e6b0c4d52637bd8c6bb2c5e3d69013889053973195a5d73ac5d5645047f985
-  data.tar.gz: 23425d552932665cafa03cf245d3d4ef879e053e9e9c60d1eb172455f405f3f3
+  metadata.gz: 994ab85c49335ccb0a2aee9d7f0c43f1bdc4ee7cd48e19cc01f2eba884f8ec1d
+  data.tar.gz: 680ccd2f1adaa3cfa46ca66086234667959b916da2e1c4f8748d2a47b82454f2
 SHA512:
-  metadata.gz: 62d52be41cb3821a5d6534fea43deea8d02dd8a8f03d3449249be0fcdcbf5c842cbb2be8aac7065b97fb515b16370ab93f114ab44dd85dce6a22091d9ac04de4
-  data.tar.gz: 006efc7649ddb4257a80e1f2400a62e9e9cba0fa5573d3947f47293d4d77b9b701d66ebf6d465cf91fbf209771e5699ddc9cd7df238ecddf7f54ffbab4eb0c15
+  metadata.gz: 1be971cb7d6cb591e0b3961e680057c49768dcf10dded68d4ec293e07ba32ad432d2ba2d3e4a57012a7f37831be655422fcad330efe5394e1a58bc56929fca59
+  data.tar.gz: 5a2b04c4cf2783f60b52acf38f0ed8e06a80a6b057d58d10d3419c84281edc719435fe6f420587bf358012adb917aa5e970b71d2d39645194bb1d024f0f39267

data/.github/CODEOWNERS

@@ -0,0 +1,4 @@
+# CODEOWNERS file
+# This file defines who should review code changes in this repository.
+
+* @zendesk/core-gem-owners

data/.github/workflows/ci.yml

@@ -2,9 +2,9 @@ name: CI
 
 on:
   pull_request:
-    branches: ["master"]
+    branches: ["main"]
   push:
-    branches: ["master"]
+    branches: ["main"]
 
 jobs:
   unit-specs:

data/.github/workflows/publish.yml

@@ -1,12 +1,26 @@
-name: Publish Gem
+name: Publish to RubyGems.org
 
 on:
   push:
-    tags: v*
+    branches: main
+    paths: lib/racecar/version.rb
+  workflow_dispatch:
 
 jobs:
-  call-workflow:
-    uses: zendesk/gw/.github/workflows/ruby-gem-publication.yml@main
-    secrets:
-      RUBY_GEMS_API_KEY: ${{ secrets.RUBY_GEMS_API_KEY }}
-      RUBY_GEMS_TOTP_DEVICE: ${{ secrets.RUBY_GEMS_TOTP_DEVICE }}
+  publish:
+    runs-on: ubuntu-latest
+    environment: rubygems-publish
+    if: github.repository_owner == 'zendesk'
+    permissions:
+      id-token: write
+      contents: write
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          bundler-cache: false
+          ruby-version: "3.4"
+      - name: Install dependencies
+        run: bundle install
+      - uses: rubygems/release-gem@v1

data/CHANGELOG.md

@@ -2,6 +2,15 @@
 
 ## Unreleased
 
+## 2.13.0
+
+* Fix `ConsumerSet#pause` raising `NameError` when `OpenStruct` is not loaded by
+  replacing the seek message with a `Struct`.
+* Commit stored offsets on a timer while idle to refresh their retention, so an
+  idle consumer group doesn't lose its position in the topics after
+  `offsets.retention.ms` expires. Enabled by default; set
+  `offset_commit_on_idle=false` to disable.
+
 ## 2.12.0
 
 * Add tests against Ruby 3.4

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    racecar (2.12.0)
+    racecar (2.13.0)
       king_konf (~> 1.0.0)
       rdkafka (>= 0.15.0)
 
@@ -45,7 +45,7 @@ GEM
       byebug (~> 11.0)
       pry (>= 0.13, < 0.15)
     rake (13.0.6)
-    rdkafka (0.18.0)
+    rdkafka (0.21.0)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)

data/README.md

@@ -349,6 +349,7 @@ end
 The consumers will checkpoint their positions from time to time in order to be able to recover from failures. This is called _committing offsets_, since it's done by tracking the offset reached in each partition being processed, and committing those offset numbers to the Kafka offset storage API. If you can tolerate more double-processing after a failure, you can increase the interval between commits in order to better performance. You can also do the opposite if you prefer less chance of double-processing.
 
 - `offset_commit_interval` – How often to save the consumer's position in Kafka. Default is every 10 seconds.
+- `offset_commit_on_idle` – Whether to keep committing the stored position on a timer even when no messages are being processed. Kafka retains committed offsets for only a finite period (`offsets.retention.ms`); without these keep-alive commits, an idle consumer group can lose its position and fall back to `auto.offset.reset` on the next restart, causing reprocessing or skipped messages. Default is `true`.
 
 #### Timeouts & intervals
 
@@ -418,7 +419,7 @@ Racecar supports [Datadog](https://www.datadoghq.com/) monitoring integration. I
 - `datadog_namespace` – The namespace to use for Datadog metrics.
 - `datadog_tags` – Tags that should always be set on Datadog metrics.
 
-Furthermore, there's a [standard Datadog dashboard configuration file](https://raw.githubusercontent.com/zendesk/racecar/master/extra/datadog-dashboard.json) that you can import to get started with a Racecar dashboard for all of your consumers.
+Furthermore, there's a [standard Datadog dashboard configuration file](https://raw.githubusercontent.com/zendesk/racecar/main/extra/datadog-dashboard.json) that you can import to get started with a Racecar dashboard for all of your consumers.
 
 #### Consumers Without Rails
 
@@ -729,6 +730,22 @@ There can be behavioural inconsistencies between running the specs on your machi
 
 Please note - your code directory is mounted as a volume, so you can make code changes without needing to rebuild
 
+### Releasing a new version
+
+A new version is published to RubyGems.org every time a change to `version.rb` is pushed to the `main` branch.
+In short, follow these steps:
+1. Update `version.rb`,
+2. run `bundle lock` to update `Gemfile.lock`,
+3. merge this change into `main`, and
+4. look at [the action](https://github.com/zendesk/racecar/actions/workflows/publish.yml) for output.
+
+To create a pre-release from a non-main branch:
+1. change the version in `version.rb` to something like `2.13.0.pre.1` or `3.0.0.beta.2`,
+2. push this change to your branch,
+3. go to [Actions → “Publish to RubyGems.org” on GitHub](https://github.com/zendesk/racecar/actions/workflows/publish.yml),
+4. click the “Run workflow” button,
+5. pick your branch from a dropdown.
+
 ## Contributing
 
 Bug reports and pull requests are welcome on [GitHub](https://github.com/zendesk/racecar). Feel free to [join our Slack team](https://ruby-kafka-slack.herokuapp.com/) and ask how best to contribute!

data/Rakefile

@@ -1,11 +1,9 @@
 # frozen_string_literal: true
 
+require "bundler/setup"
 require "bundler/gem_tasks"
 require "rspec/core/rake_task"
 
-# Pushing to rubygems is handled by a github workflow
-ENV["gem_push"] = "false"
-
 RSpec::Core::RakeTask.new(:spec)
 
 task :default => :spec

data/lib/racecar/config.rb

@@ -23,6 +23,9 @@ module Racecar
     desc "How frequently to commit offset positions"
     float :offset_commit_interval, default: 10
 
+    desc "Commit stored offsets on a timer even when no messages have been processed, refreshing their retention so an idle consumer group doesn't lose its position"
+    boolean :offset_commit_on_idle, default: true
+
     desc "How often to send a heartbeat message to Kafka"
     float :heartbeat_interval, default: 10
 

data/lib/racecar/consumer_set.rb

@@ -3,6 +3,8 @@
 module Racecar
   class ConsumerSet
     MAX_POLL_TRIES = 10
+    SeekMessage = Struct.new(:topic, :partition, :offset, keyword_init: true)
+    private_constant :SeekMessage
 
     def initialize(config, logger, instrumenter = NullInstrumenter)
       @config, @logger = config, logger
@@ -100,8 +102,8 @@ module Racecar
       end
 
       consumer.pause(filtered_tpl)
-      fake_msg = OpenStruct.new(topic: topic, partition: partition, offset: offset)
-      consumer.seek(fake_msg)
+      seek_message = SeekMessage.new(topic: topic, partition: partition, offset: offset)
+      consumer.seek(seek_message)
 
       @paused_tpls[topic][partition] = [consumer, filtered_tpl]
     end

data/lib/racecar/runner.rb

@@ -49,6 +49,7 @@ module Racecar
     def run
       install_signal_handlers
       @stop_requested = false
+      @last_keep_alive_commit_at = monotonic_time
 
       # Configure the consumer with a producer so it can produce messages and
       # with a consumer so that it can support advanced use-cases.
@@ -68,18 +69,22 @@ module Racecar
       loop do
         break if @stop_requested
         resume_paused_partitions
+        @offset_advanced_this_loop = false
 
         @instrumenter.instrument("start_main_loop", instrumentation_payload)
         @instrumenter.instrument("main_loop", instrumentation_payload) do
           case process_method
           when :batch then
-            msg_per_part = consumer.batch_poll(config.max_wait_time_ms).group_by(&:partition)
-            msg_per_part.each_value do |messages|
-              process_batch(messages)
+            messages = consumer.batch_poll(config.max_wait_time_ms)
+            msg_per_part = messages.group_by(&:partition)
+            msg_per_part.each_value do |batch|
+              process_batch(batch)
             end
+            maybe_keep_alive_commit(!@offset_advanced_this_loop)
           when :single then
             message = consumer.poll(config.max_wait_time_ms)
             process(message) if message
+            maybe_keep_alive_commit(!@offset_advanced_this_loop)
           end
         end
       end
@@ -185,6 +190,7 @@ module Racecar
             processor.process(Racecar::Message.new(message, retries_count: pause.pauses_count))
             processor.deliver!
             consumer.store_offset(message)
+            @offset_advanced_this_loop = true
           end
         rescue => e
           instrumentation_payload[:unrecoverable_delivery_error] = reset_producer_on_unrecoverable_delivery_errors(e)
@@ -217,6 +223,7 @@ module Racecar
             processor.process_batch(racecar_messages)
             processor.deliver!
             consumer.store_offset(messages.last)
+            @offset_advanced_this_loop = true
           end
         rescue => e
           instrumentation_payload[:unrecoverable_delivery_error] = reset_producer_on_unrecoverable_delivery_errors(e)
@@ -289,5 +296,38 @@ module Racecar
         end
       end
     end
+
+    # librdkafka's auto-commit only writes an OffsetCommit when the stored
+    # offset has changed, so a consumer that never advances its stored offset
+    # -- whether idle or stuck on failing messages -- never commits. Kafka
+    # retains committed offsets for only a finite period (offsets.retention.ms,
+    # default 7 days); once they expire the group loses its position and a
+    # restart falls back to auto.offset.reset, causing reprocessing or skipped
+    # messages. Re-committing the already stored position on a timer refreshes
+    # its retention. This only fires when the stored offset wasn't advanced
+    # this iteration, so it never duplicates the normal commit path, and it
+    # never advances past a stored offset, so at-least-once semantics are
+    # unchanged.
+    def maybe_keep_alive_commit(offset_stale)
+      return unless offset_stale
+      return unless config.offset_commit_on_idle
+      return if monotonic_time - @last_keep_alive_commit_at < config.offset_commit_interval
+
+      consumer.commit
+      @last_keep_alive_commit_at = monotonic_time
+    rescue Rdkafka::RdkafkaError => e
+      # Retry on the next idle iteration (the timer isn't advanced on failure),
+      # but throttle the warning so a persistent failure doesn't spam the log
+      # every iteration.
+      now = monotonic_time
+      unless @last_keep_alive_commit_error_at && now - @last_keep_alive_commit_error_at < config.offset_commit_interval
+        @last_keep_alive_commit_error_at = now
+        logger.warn "Keep-alive offset commit failed: #{e}"
+      end
+    end
+
+    def monotonic_time
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
   end
 end

data/lib/racecar/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Racecar
-  VERSION = "2.12.0"
+  VERSION = "2.13.0"
 end

metadata

@@ -1,15 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: racecar
 version: !ruby/object:Gem::Version
-  version: 2.12.0
+  version: 2.13.0
 platform: ruby
 authors:
 - Daniel Schierbeck
 - Benjamin Quorning
-autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-02-27 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: king_konf
@@ -149,7 +148,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description:
 email:
 - dschierbeck@zendesk.com
 - bquorning@zendesk.com
@@ -159,6 +157,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/CODEOWNERS"
 - ".github/dependabot.yml"
 - ".github/workflows/ci.yml"
 - ".github/workflows/publish.yml"
@@ -214,7 +213,6 @@ homepage: https://github.com/zendesk/racecar
 licenses:
 - Apache License Version 2.0
 metadata: {}
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -229,8 +227,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.22
-signing_key:
+rubygems_version: 3.6.9
 specification_version: 4
 summary: A framework for running Kafka consumers
 test_files: []