work_shaper 0.1.2.1 → 0.1.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5cd87f9d88041bc67137b36d582629575ec11bb992e3ff9de8611237cfdd56a3
-  data.tar.gz: 5c3ad4dee1e236c9c3ab59c15ccaf95716387c9ecaccbc7e45616d6465e8f350
+  metadata.gz: 9184d55b2d25f45b2f3fb832f9b990cca7d634dd590db1f6d3cf472d21361dcf
+  data.tar.gz: 7b0b8e413df089414635dd4362abe5f2d058403e0bba20370c9722fefe104e3e
 SHA512:
-  metadata.gz: fbf7d11372a4d8e69292a82729636fd8bad0bffd1bfa87eae270004648e9beb508ae9d446141cecf09798f306cc5af71a1acec7d9d1d81c09ae93fabf5aa70af
-  data.tar.gz: 25195a7ccafaa687d09967d9d9efa2f066edc9b89503b30d62d3e68877518a3c753065de67374de56acf05017731a586ef0a2516510815ebc9a23fa44aea5713
+  metadata.gz: 100431868af62d50e3e38a0d67c22f636660eb95db053788a785eb40ea13e27caf40e0a072924f16ea4129db76b20febeb98df4e8a116819442e40f5029eb9af
+  data.tar.gz: 1a075e88184f436172dee3b36e1088a8d9feba9ad43136f9e930c406fc3c095ba2dbe8943321d79077ddf7c3c570ca53d3286eee67d3d82ee8bff37f0f94891a

data/Gemfile

@@ -11,8 +11,6 @@ gem "rspec", "~> 3.0"
 
 gem "rubocop", "~> 1.21"
 
-gem "sorted_set", "~> 1.0"
-
 gem "logger", "~> 1.4"
 
 gem "concurrent-ruby", "~> 1.2"

data/lib/work_shaper/manager.rb

@@ -3,6 +3,8 @@ module WorkShaper
   # for each offset in monotonically increasing order (independent of the execution order), and gracefully
   # cleaning up when `#shutdown` is called.
   class Manager
+    attr_reader :total_acked, :total_enqueued
+
     # Several of the parameters here are Lambdas (not Proc). Note you can pass a method using
     # `method(:some_method)` or a lambda directly `->{ puts 'Hello'}`.
     #
@@ -29,6 +31,7 @@ module WorkShaper
       @shutdown = false
 
       @total_enqueued = 0
+      @total_acked = 0
 
       @heartbeat = Thread.new do
         while true
@@ -57,11 +60,12 @@ module WorkShaper
     def enqueue(sub_key, message, partition, offset)
       raise StandardError, 'Shutting down' if @shutdown
       pause_on_overrun
+      WorkShaper.logger.debug "Enqueue: #{sub_key}:#{partition}:#{offset}"
 
       worker = nil
       @semaphore.synchronize do
         @total_enqueued += 1
-        (@received_offsets[partition] ||= SortedSet.new) << offset
+        (@received_offsets[partition] ||= Array.new) << offset
 
         worker =
           @workers[sub_key] ||=
@@ -130,13 +134,11 @@ module WorkShaper
     end
 
     def offset_ack_unsafe(partition)
-      @total_acked ||= 0
-
       completed = @completed_offsets[partition]
       received = @received_offsets[partition]
 
-      offset = completed.first
-      while received.any? && received.first == offset
+      offset = completed.sort.first
+      while received.any? && received.sort.first == offset
         # We observed Kafka sending the same message twice, even after
         # having committed the offset. Here we skip this offset if we
         # know it has already been committed.
@@ -161,8 +163,11 @@ module WorkShaper
         end
 
         @total_acked += 1
-        completed.delete(offset)
-        received.delete(offset)
+        WorkShaper.logger.debug "@total_acked: #{@total_acked}"
+        WorkShaper.logger.debug "completed: [#{completed.join(', ')}]"
+        WorkShaper.logger.debug "received: [#{received.join(', ')}]"
+        completed.shift
+        received.shift
 
         offset = completed.first
       end
@@ -170,7 +175,11 @@ module WorkShaper
 
     def pause_on_overrun
       overrun = lambda do
+        completed = @completed_offsets.values.flatten.count
+        received = @received_offsets.values.flatten.count
+
         @total_enqueued.to_i - @total_acked.to_i > @max_in_queue
+        received - completed > @max_in_queue
       end
 
       # We have to be careful here to avoid a deadlock. Another thread may be waiting

data/lib/work_shaper/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module WorkShaper
-  VERSION = "0.1.2.1"
+  VERSION = "0.1.2.3"
 end

data/lib/work_shaper/worker.rb

@@ -33,7 +33,8 @@ module WorkShaper
         @on_error.call(e, message, partition, offset)
       ensure
         @semaphore.synchronize do
-          (@completed_offsets[partition] ||= SortedSet.new) << offset
+          WorkShaper.logger.debug "Completed: #{partition}:#{offset}"
+          (@completed_offsets[partition] ||= Array.new) << offset
         end
       end
       # rubocop:enable Style/RescueStandardError

data/work_shaper.gemspec

@@ -32,7 +32,6 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   # Uncomment to register a new dependency of your gem
-  spec.add_dependency "sorted_set", "~> 1.0"
   spec.add_dependency "concurrent-ruby", "~> 1.2"
 
   # For more information and examples about making a new gem, check out our

metadata

@@ -1,29 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: work_shaper
 version: !ruby/object:Gem::Version
-  version: 0.1.2.1
+  version: 0.1.2.3
 platform: ruby
 authors:
 - Jerry Fernholz
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-02-01 00:00:00.000000000 Z
+date: 2024-03-04 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: sorted_set
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
   requirement: !ruby/object:Gem::Requirement