work_shaper 0.1.2.1 → 0.1.2.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 5cd87f9d88041bc67137b36d582629575ec11bb992e3ff9de8611237cfdd56a3
4
- data.tar.gz: 5c3ad4dee1e236c9c3ab59c15ccaf95716387c9ecaccbc7e45616d6465e8f350
3
+ metadata.gz: 3486b765efce170ed99dd19dd25fe0b755e0c109c1f460ca2998f444cdf8557f
4
+ data.tar.gz: d9329ef84832dfbc0704a56f39ed82a8a51f17e3ebe93741938f4422cdb47797
5
5
  SHA512:
6
- metadata.gz: fbf7d11372a4d8e69292a82729636fd8bad0bffd1bfa87eae270004648e9beb508ae9d446141cecf09798f306cc5af71a1acec7d9d1d81c09ae93fabf5aa70af
7
- data.tar.gz: 25195a7ccafaa687d09967d9d9efa2f066edc9b89503b30d62d3e68877518a3c753065de67374de56acf05017731a586ef0a2516510815ebc9a23fa44aea5713
6
+ metadata.gz: 046bc8286d04fd48130621813262ea0537b8b68eec80fe6332658d46f09fe3ca4e5640d63f91b131e2f9774e286dbada92968d2e361feccabde139f91230ec41
7
+ data.tar.gz: 4851e8636d76ea5d5438f343527117dc6880d1e47a5de07c35d20ee469df474fbd103c5d73850dea6217c1ce3f5174c64acebe4faf2c27d2233aaa9031e7c7d1
data/Gemfile CHANGED
@@ -11,8 +11,6 @@ gem "rspec", "~> 3.0"
11
11
 
12
12
  gem "rubocop", "~> 1.21"
13
13
 
14
- gem "sorted_set", "~> 1.0"
15
-
16
14
  gem "logger", "~> 1.4"
17
15
 
18
16
  gem "concurrent-ruby", "~> 1.2"
@@ -3,6 +3,8 @@ module WorkShaper
3
3
  # for each offset in monotonically increasing order (independent of the execution order), and gracefully
4
4
  # cleaning up when `#shutdown` is called.
5
5
  class Manager
6
+ attr_reader :total_acked, :total_enqueued
7
+
6
8
  # Several of the parameters here are Lambdas (not Proc). Note you can pass a method using
7
9
  # `method(:some_method)` or a lambda directly `->{ puts 'Hello'}`.
8
10
  #
@@ -29,6 +31,7 @@ module WorkShaper
29
31
  @shutdown = false
30
32
 
31
33
  @total_enqueued = 0
34
+ @total_acked = 0
32
35
 
33
36
  @heartbeat = Thread.new do
34
37
  while true
@@ -57,11 +60,12 @@ module WorkShaper
57
60
  def enqueue(sub_key, message, partition, offset)
58
61
  raise StandardError, 'Shutting down' if @shutdown
59
62
  pause_on_overrun
63
+ WorkShaper.logger.debug "Enqueue: #{sub_key}:#{partition}:#{offset}"
60
64
 
61
65
  worker = nil
62
66
  @semaphore.synchronize do
63
67
  @total_enqueued += 1
64
- (@received_offsets[partition] ||= SortedSet.new) << offset
68
+ (@received_offsets[partition] ||= Array.new) << offset
65
69
 
66
70
  worker =
67
71
  @workers[sub_key] ||=
@@ -130,13 +134,11 @@ module WorkShaper
130
134
  end
131
135
 
132
136
  def offset_ack_unsafe(partition)
133
- @total_acked ||= 0
134
-
135
137
  completed = @completed_offsets[partition]
136
138
  received = @received_offsets[partition]
137
139
 
138
- offset = completed.first
139
- while received.any? && received.first == offset
140
+ offset = completed.sort.first
141
+ while received.any? && received.sort.first == offset
140
142
  # We observed Kafka sending the same message twice, even after
141
143
  # having committed the offset. Here we skip this offset if we
142
144
  # know it has already been committed.
@@ -161,8 +163,11 @@ module WorkShaper
161
163
  end
162
164
 
163
165
  @total_acked += 1
164
- completed.delete(offset)
165
- received.delete(offset)
166
+ WorkShaper.logger.debug "@total_acked: #{@total_acked}"
167
+ WorkShaper.logger.debug "completed: [#{completed.join(', ')}]"
168
+ WorkShaper.logger.debug "received: [#{received.join(', ')}]"
169
+ completed.shift
170
+ received.shift
166
171
 
167
172
  offset = completed.first
168
173
  end
@@ -170,7 +175,11 @@ module WorkShaper
170
175
 
171
176
  def pause_on_overrun
172
177
  overrun = lambda do
178
+ completed = @completed_offsets.values.flatten.count
179
+ received = @received_offsets.values.flatten.count
180
+
173
181
  @total_enqueued.to_i - @total_acked.to_i > @max_in_queue
182
+ received - completed > @max_in_queue
174
183
  end
175
184
 
176
185
  # We have to be careful here to avoid a deadlock. Another thread may be waiting
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module WorkShaper
4
- VERSION = "0.1.2.1"
4
+ VERSION = "0.1.2.5"
5
5
  end
@@ -33,7 +33,8 @@ module WorkShaper
33
33
  @on_error.call(e, message, partition, offset)
34
34
  ensure
35
35
  @semaphore.synchronize do
36
- (@completed_offsets[partition] ||= SortedSet.new) << offset
36
+ WorkShaper.logger.debug "Completed: #{partition}:#{offset}"
37
+ (@completed_offsets[partition] ||= Array.new) << offset
37
38
  end
38
39
  end
39
40
  # rubocop:enable Style/RescueStandardError
data/lib/work_shaper.rb CHANGED
@@ -1,7 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require 'logger'
4
- require 'sorted_set'
5
4
  require 'concurrent-ruby'
6
5
  require_relative "work_shaper/version"
7
6
  require_relative "work_shaper/manager"
data/work_shaper.gemspec CHANGED
@@ -32,7 +32,6 @@ Gem::Specification.new do |spec|
32
32
  spec.require_paths = ["lib"]
33
33
 
34
34
  # Uncomment to register a new dependency of your gem
35
- spec.add_dependency "sorted_set", "~> 1.0"
36
35
  spec.add_dependency "concurrent-ruby", "~> 1.2"
37
36
 
38
37
  # For more information and examples about making a new gem, check out our
metadata CHANGED
@@ -1,29 +1,15 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: work_shaper
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.2.1
4
+ version: 0.1.2.5
5
5
  platform: ruby
6
6
  authors:
7
7
  - Jerry Fernholz
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2024-02-01 00:00:00.000000000 Z
11
+ date: 2024-03-04 00:00:00.000000000 Z
12
12
  dependencies:
13
- - !ruby/object:Gem::Dependency
14
- name: sorted_set
15
- requirement: !ruby/object:Gem::Requirement
16
- requirements:
17
- - - "~>"
18
- - !ruby/object:Gem::Version
19
- version: '1.0'
20
- type: :runtime
21
- prerelease: false
22
- version_requirements: !ruby/object:Gem::Requirement
23
- requirements:
24
- - - "~>"
25
- - !ruby/object:Gem::Version
26
- version: '1.0'
27
13
  - !ruby/object:Gem::Dependency
28
14
  name: concurrent-ruby
29
15
  requirement: !ruby/object:Gem::Requirement