work_shaper 0.1.3.1rc5 → 0.1.3.1rc6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ee42adfc0a4002f31b442eec3ea46dd3b878dc8fabadccb010cfb27070142223
-  data.tar.gz: b8e268a9c867df76e210b279b3c8c2b62446b347c7ab02db1a381c3b5a80f7c6
+  metadata.gz: b86b6e151780b23abcdeb4c148775a3344f2b41a0da426f9ff0da148eca1ea5a
+  data.tar.gz: 4af867aee488107d5e934fe02c844a4c2f0b6548305bea935bc8b32b143c2808
 SHA512:
-  metadata.gz: 94b8e9d89e51fce2f374eabef54d89a43703b48dba1dd9979598342a26182e2eb739a1fc0f2afa842100c1ab8bc32f094f2904680243e053d34f3ab82d42aded
-  data.tar.gz: 2128b65b7c21b420b0e58c5765a9d557e466415fcfa32c80d4b9e9f407e2cb9de8b107f3e6deb81b61f086c0566bad256031aeb310f2676d004488e5af389bda
+  metadata.gz: 8cb3139f6e36a665bbcfad2291c11d6e785c4c41df11ab8cc69b35111a5b897da12fe13ac6fc2f5cfac55a6673314256b4bd2215171c722ba6b7c3299ab77c2a
+  data.tar.gz: 8342797e7b562d0fd8be90c802a8d83c4e480c81311e4474a9e78ce911d7f83bb15dbbb01c63d4a1394b5b40dc6e7a5223d499eb0a505ba2538a9e4836eacce9

data/lib/work_shaper/manager.rb

@@ -25,7 +25,6 @@ module WorkShaper
       @workers = {}
       @last_ack = {}
       @received_offsets = {}
-      @completed_offsets = {}
       @max_in_queue = max_in_queue
       @semaphore = Mutex.new
       @shutting_down = false
@@ -45,7 +44,7 @@ module WorkShaper
 
       @offset_manager = Thread.new do
         while true
-          @completed_offsets.each_key do |partition|
+          @received_offsets.each_key do |partition|
             offset_ack(partition)
           end
           sleep offset_commit_period_ms / 1000.0
@@ -77,7 +76,6 @@ module WorkShaper
               method(:offset_ack),
               @on_error,
               @last_ack,
-              @completed_offsets,
               @semaphore,
               @max_in_queue
             )
@@ -91,7 +89,7 @@ module WorkShaper
     # the consumer restarts.
     def flush(safe: true)
       sleep 5
-      @completed_offsets.each_key do |k|
+      @received_offsets.each_key do |k|
         safe ? offset_ack(k) : offset_ack_unsafe(k)
       end
     end
@@ -137,12 +135,11 @@ module WorkShaper
     end
 
     def offset_ack_unsafe(partition)
-      completed = @completed_offsets[partition].sort!
       received = @received_offsets[partition].sort!
 
       begin
-        offset = completed.first
-        while received.any? && received.first == offset
+        offset = received.first
+        while offset && offset.completed?
           # We observed Kafka sending the same message twice, even after
           # having committed the offset. Here we skip this offset if we
           # know it has already been committed.
@@ -172,7 +169,6 @@ module WorkShaper
             WorkShaper.logger.warn(
               { message: 'Failed to Ack Offset, likely re-balance',
                 offset: "#{partition}:#{offset}",
-                completed: @completed_offsets[partition].to_a[0..10].join(','),
                 received: @received_offsets[partition].to_a[0..10].join(',')
               })
           else
@@ -181,12 +177,10 @@ module WorkShaper
 
           @total_acked += 1
           WorkShaper.logger.debug "@total_acked: #{@total_acked}"
-          WorkShaper.logger.debug "completed: [#{completed.join(', ')}]"
           WorkShaper.logger.debug "received: [#{received.join(', ')}]"
-          completed.delete(offset)
           received.delete(offset)
 
-          offset = completed.first
+          offset = received.first
         end
       rescue => e
         WorkShaper.logger.error({ message: 'Error in offset_ack', error: e })

data/lib/work_shaper/offset_holder.rb

@@ -43,6 +43,10 @@ module WorkShaper
       @state = :completed
     end
 
+    def completed?
+      @state == :completed
+    end
+
     def to_i
       offset
     end

data/lib/work_shaper/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module WorkShaper
-  VERSION = "0.1.3.1rc5"
+  VERSION = "0.1.3.1rc6"
 end

data/lib/work_shaper/worker.rb

@@ -6,14 +6,13 @@ module WorkShaper
     # @param work [Lambda] Lambda that we will #call(message) to execute work.
     # @param on_done [Lambda] Lambda that we #call(partition, offset) when work is done.
     # @param on_error [Lambda] Lambda that we #call(exception) if an error is encountered.
-    def initialize(work, on_done, ack_handler, on_error, last_ack, offset_stack, semaphore, max_in_queue)
+    def initialize(work, on_done, ack_handler, on_error, last_ack, semaphore, max_in_queue)
       @jobs = []
       @work = work
       @on_done = on_done
       @ack_handler = ack_handler
       @on_error = on_error
       @last_ack = last_ack
-      @completed_offsets = offset_stack
       @semaphore = semaphore
       @max_in_queue = max_in_queue
       @thread_pool = Concurrent::FixedThreadPool.new(1, auto_terminate: false)
@@ -38,7 +37,6 @@ module WorkShaper
         @semaphore.synchronize do
           WorkShaper.logger.debug "Completed: #{partition}:#{offset}"
           offset_holder.complete!
-          (@completed_offsets[partition] ||= Array.new) << offset_holder
         end
       end
       # rubocop:enable Style/RescueStandardError

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: work_shaper
 version: !ruby/object:Gem::Version
-  version: 0.1.3.1rc5
+  version: 0.1.3.1rc6
 platform: ruby
 authors:
 - Jerry Fernholz
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-03-21 00:00:00.000000000 Z
+date: 2024-03-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby