work_shaper 0.1.3.1rc5 → 0.1.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ee42adfc0a4002f31b442eec3ea46dd3b878dc8fabadccb010cfb27070142223
-  data.tar.gz: b8e268a9c867df76e210b279b3c8c2b62446b347c7ab02db1a381c3b5a80f7c6
+  metadata.gz: d3ea26c32471d401d3544efae02d400d5317b83373487e9e7c11ea9259f678ec
+  data.tar.gz: c28379ed6fbe660f76f97950e81563524aca4e6808ef1b91c2b66920a31b775a
 SHA512:
-  metadata.gz: 94b8e9d89e51fce2f374eabef54d89a43703b48dba1dd9979598342a26182e2eb739a1fc0f2afa842100c1ab8bc32f094f2904680243e053d34f3ab82d42aded
-  data.tar.gz: 2128b65b7c21b420b0e58c5765a9d557e466415fcfa32c80d4b9e9f407e2cb9de8b107f3e6deb81b61f086c0566bad256031aeb310f2676d004488e5af389bda
+  metadata.gz: 913324499dcd1a241e116bc9f397199237ae32b9c2b5f6992b50e71790031649c73d3bb09862f427d3d210b97d0dff3dd9a382ad9ca87d92a8fe7e2b2ab8291d
+  data.tar.gz: fae26f3d854e512328334291f93a928a561ae363ccf2ba11b28f1f3a886385861aa4512c0a754d547ff6f4175481c0d4a1cef72aacbf6e1c6e80dac33300ecc3

data/lib/work_shaper/manager.rb

@@ -25,7 +25,6 @@ module WorkShaper
       @workers = {}
       @last_ack = {}
       @received_offsets = {}
-      @completed_offsets = {}
       @max_in_queue = max_in_queue
       @semaphore = Mutex.new
       @shutting_down = false
@@ -45,7 +44,7 @@ module WorkShaper
 
       @offset_manager = Thread.new do
         while true
-          @completed_offsets.each_key do |partition|
+          @received_offsets.each_key do |partition|
             offset_ack(partition)
           end
           sleep offset_commit_period_ms / 1000.0
@@ -77,7 +76,6 @@ module WorkShaper
               method(:offset_ack),
               @on_error,
               @last_ack,
-              @completed_offsets,
               @semaphore,
               @max_in_queue
             )
@@ -91,7 +89,7 @@ module WorkShaper
     # the consumer restarts.
     def flush(safe: true)
       sleep 5
-      @completed_offsets.each_key do |k|
+      @received_offsets.each_key do |k|
         safe ? offset_ack(k) : offset_ack_unsafe(k)
       end
     end
@@ -137,12 +135,11 @@ module WorkShaper
     end
 
     def offset_ack_unsafe(partition)
-      completed = @completed_offsets[partition].sort!
       received = @received_offsets[partition].sort!
 
       begin
-        offset = completed.first
-        while received.any? && received.first == offset
+        offset = received.first
+        while offset && offset.completed?
           # We observed Kafka sending the same message twice, even after
           # having committed the offset. Here we skip this offset if we
           # know it has already been committed.
@@ -172,7 +169,6 @@ module WorkShaper
             WorkShaper.logger.warn(
               { message: 'Failed to Ack Offset, likely re-balance',
                 offset: "#{partition}:#{offset}",
-                completed: @completed_offsets[partition].to_a[0..10].join(','),
                 received: @received_offsets[partition].to_a[0..10].join(',')
               })
           else
@@ -181,12 +177,10 @@ module WorkShaper
 
           @total_acked += 1
           WorkShaper.logger.debug "@total_acked: #{@total_acked}"
-          WorkShaper.logger.debug "completed: [#{completed.join(', ')}]"
           WorkShaper.logger.debug "received: [#{received.join(', ')}]"
-          completed.delete(offset)
           received.delete(offset)
 
-          offset = completed.first
+          offset = received.first
         end
       rescue => e
         WorkShaper.logger.error({ message: 'Error in offset_ack', error: e })
@@ -196,6 +190,8 @@ module WorkShaper
 
     def pause_on_overrun
       overrun = lambda do
+        # I think we can do this now with the refactor
+        # @received_offsets.values.count > @max_in_queue
         @total_enqueued.to_i - @total_acked.to_i > @max_in_queue
       end
 

data/lib/work_shaper/offset_holder.rb

@@ -43,6 +43,10 @@ module WorkShaper
       @state = :completed
     end
 
+    def completed?
+      @state == :completed
+    end
+
     def to_i
       offset
     end

data/lib/work_shaper/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module WorkShaper
-  VERSION = "0.1.3.1rc5"
+  VERSION = "0.1.3.1"
 end

data/lib/work_shaper/worker.rb

@@ -6,14 +6,13 @@ module WorkShaper
     # @param work [Lambda] Lambda that we will #call(message) to execute work.
     # @param on_done [Lambda] Lambda that we #call(partition, offset) when work is done.
     # @param on_error [Lambda] Lambda that we #call(exception) if an error is encountered.
-    def initialize(work, on_done, ack_handler, on_error, last_ack, offset_stack, semaphore, max_in_queue)
+    def initialize(work, on_done, ack_handler, on_error, last_ack, semaphore, max_in_queue)
       @jobs = []
       @work = work
       @on_done = on_done
       @ack_handler = ack_handler
       @on_error = on_error
       @last_ack = last_ack
-      @completed_offsets = offset_stack
       @semaphore = semaphore
       @max_in_queue = max_in_queue
       @thread_pool = Concurrent::FixedThreadPool.new(1, auto_terminate: false)
@@ -38,7 +37,6 @@ module WorkShaper
         @semaphore.synchronize do
           WorkShaper.logger.debug "Completed: #{partition}:#{offset}"
           offset_holder.complete!
-          (@completed_offsets[partition] ||= Array.new) << offset_holder
         end
       end
       # rubocop:enable Style/RescueStandardError

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: work_shaper
 version: !ruby/object:Gem::Version
-  version: 0.1.3.1rc5
+  version: 0.1.3.1
 platform: ruby
 authors:
 - Jerry Fernholz
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-03-21 00:00:00.000000000 Z
+date: 2024-03-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -63,9 +63,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: 2.6.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubygems_version: 3.4.10
 signing_key: