work_shaper 0.1.2 → 0.1.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 357d650ad73cfb685d1fd05a2ed98a0b4136358dc232febe04be921f8954855c
-  data.tar.gz: 77bceccf96220ec5ba98e438fde84761dd3494878fae05005cccdad0121cb833
+  metadata.gz: 9184d55b2d25f45b2f3fb832f9b990cca7d634dd590db1f6d3cf472d21361dcf
+  data.tar.gz: 7b0b8e413df089414635dd4362abe5f2d058403e0bba20370c9722fefe104e3e
 SHA512:
-  metadata.gz: 25c75b8e97b6e320e57164a2128ab320b23efe22c81d3e3f64857e70f92ee3dd881133060b702109ec1b172120a446ffb8d214a0423a039eb95ac4a6d06b3de3
-  data.tar.gz: 72babc90aaef951eb50dcc3edea4bc71903497bc79aaf3c7708fc143649407a88d4df6240d5f3b232a60fc37f11446f758109d833bc65d0822a23578b2da7ee6
+  metadata.gz: 100431868af62d50e3e38a0d67c22f636660eb95db053788a785eb40ea13e27caf40e0a072924f16ea4129db76b20febeb98df4e8a116819442e40f5029eb9af
+  data.tar.gz: 1a075e88184f436172dee3b36e1088a8d9feba9ad43136f9e930c406fc3c095ba2dbe8943321d79077ddf7c3c570ca53d3286eee67d3d82ee8bff37f0f94891a

data/Gemfile

@@ -10,3 +10,7 @@ gem "rake", "~> 13.0"
 gem "rspec", "~> 3.0"
 
 gem "rubocop", "~> 1.21"
+
+gem "logger", "~> 1.4"
+
+gem "concurrent-ruby", "~> 1.2"

data/lib/work_shaper/manager.rb

@@ -3,6 +3,8 @@ module WorkShaper
   # for each offset in monotonically increasing order (independent of the execution order), and gracefully
   # cleaning up when `#shutdown` is called.
   class Manager
+    attr_reader :total_acked, :total_enqueued
+
     # Several of the parameters here are Lambdas (not Proc). Note you can pass a method using
     # `method(:some_method)` or a lambda directly `->{ puts 'Hello'}`.
     #
@@ -29,6 +31,7 @@ module WorkShaper
       @shutdown = false
 
       @total_enqueued = 0
+      @total_acked = 0
 
       @heartbeat = Thread.new do
         while true
@@ -57,11 +60,12 @@ module WorkShaper
     def enqueue(sub_key, message, partition, offset)
       raise StandardError, 'Shutting down' if @shutdown
       pause_on_overrun
+      WorkShaper.logger.debug "Enqueue: #{sub_key}:#{partition}:#{offset}"
 
       worker = nil
       @semaphore.synchronize do
         @total_enqueued += 1
-        (@received_offsets[partition] ||= SortedSet.new) << offset
+        (@received_offsets[partition] ||= Array.new) << offset
 
         worker =
           @workers[sub_key] ||=
@@ -98,7 +102,8 @@ module WorkShaper
             total_acked: @total_acked,
             in_flight: (@total_enqueued.to_i - @total_acked.to_i),
             last_acked_offsets: @last_ack,
-            worker_count: @workers.keys.count
+            worker_count: @workers.keys.count,
+            offset_mgr: @offset_manager.status
           })
         if detailed
           WorkShaper.logger.info(
@@ -129,13 +134,11 @@ module WorkShaper
     end
 
     def offset_ack_unsafe(partition)
-      @total_acked ||= 0
-
       completed = @completed_offsets[partition]
       received = @received_offsets[partition]
 
-      offset = completed.first
-      while received.any? && received.first == offset
+      offset = completed.sort.first
+      while received.any? && received.sort.first == offset
         # We observed Kafka sending the same message twice, even after
         # having committed the offset. Here we skip this offset if we
         # know it has already been committed.
@@ -160,8 +163,11 @@ module WorkShaper
         end
 
         @total_acked += 1
-        completed.delete(offset)
-        received.delete(offset)
+        WorkShaper.logger.debug "@total_acked: #{@total_acked}"
+        WorkShaper.logger.debug "completed: [#{completed.join(', ')}]"
+        WorkShaper.logger.debug "received: [#{received.join(', ')}]"
+        completed.shift
+        received.shift
 
         offset = completed.first
       end
@@ -169,7 +175,11 @@ module WorkShaper
 
     def pause_on_overrun
       overrun = lambda do
+        completed = @completed_offsets.values.flatten.count
+        received = @received_offsets.values.flatten.count
+
         @total_enqueued.to_i - @total_acked.to_i > @max_in_queue
+        received - completed > @max_in_queue
       end
 
       # We have to be careful here to avoid a deadlock. Another thread may be waiting

data/lib/work_shaper/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module WorkShaper
-  VERSION = "0.1.2"
+  VERSION = "0.1.2.3"
 end

data/lib/work_shaper/worker.rb

@@ -27,16 +27,15 @@ module WorkShaper
       @thread_pool.post do
         @work.call(message, partition, offset)
         @on_done.call(message, partition, offset)
-        @semaphore.synchronize do
-          (@completed_offsets[partition] ||= SortedSet.new) << offset
-        end
-        # @ack_handler.call(partition, offset)
       rescue => e
-        puts("Error processing #{partition}:#{offset} #{e}")
-        puts(e.backtrace.join("\n"))
-        # logger.error("Acking it anyways, why not?")
+        WorkShaper.logger.error("Error processing #{partition}:#{offset} #{e}")
+        WorkShaper.logger.error(e.backtrace.join(" > "))
         @on_error.call(e, message, partition, offset)
-        # @ack_handler.call(partition, offset)
+      ensure
+        @semaphore.synchronize do
+          WorkShaper.logger.debug "Completed: #{partition}:#{offset}"
+          (@completed_offsets[partition] ||= Array.new) << offset
+        end
       end
       # rubocop:enable Style/RescueStandardError
     end

data/lib/work_shaper.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+require 'logger'
+require 'sorted_set'
+require 'concurrent-ruby'
 require_relative "work_shaper/version"
 require_relative "work_shaper/manager"
 require_relative "work_shaper/worker"

data/work_shaper.gemspec

@@ -32,7 +32,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   # Uncomment to register a new dependency of your gem
-  spec.add_dependency "sorted_set", "~> 1.0"
+  spec.add_dependency "concurrent-ruby", "~> 1.2"
 
   # For more information and examples about making a new gem, check out our
   # guide at: https://bundler.io/guides/creating_gem.html

metadata

@@ -1,29 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: work_shaper
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.2.3
 platform: ruby
 authors:
 - Jerry Fernholz
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-01-05 00:00:00.000000000 Z
+date: 2024-03-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: sorted_set
+  name: concurrent-ruby
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.2'
 description: WorkShaper was built to parallelize the work needed to process Kafka
   messages.
 email: