work_shaper 0.1.2.1 → 0.1.2.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +0 -2
- data/lib/work_shaper/manager.rb +16 -7
- data/lib/work_shaper/version.rb +1 -1
- data/lib/work_shaper/worker.rb +2 -1
- data/lib/work_shaper.rb +0 -1
- data/work_shaper.gemspec +0 -1
- metadata +2 -16
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 3486b765efce170ed99dd19dd25fe0b755e0c109c1f460ca2998f444cdf8557f
|
4
|
+
data.tar.gz: d9329ef84832dfbc0704a56f39ed82a8a51f17e3ebe93741938f4422cdb47797
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 046bc8286d04fd48130621813262ea0537b8b68eec80fe6332658d46f09fe3ca4e5640d63f91b131e2f9774e286dbada92968d2e361feccabde139f91230ec41
|
7
|
+
data.tar.gz: 4851e8636d76ea5d5438f343527117dc6880d1e47a5de07c35d20ee469df474fbd103c5d73850dea6217c1ce3f5174c64acebe4faf2c27d2233aaa9031e7c7d1
|
data/Gemfile
CHANGED
data/lib/work_shaper/manager.rb
CHANGED
@@ -3,6 +3,8 @@ module WorkShaper
|
|
3
3
|
# for each offset in monotonically increasing order (independent of the execution order), and gracefully
|
4
4
|
# cleaning up when `#shutdown` is called.
|
5
5
|
class Manager
|
6
|
+
attr_reader :total_acked, :total_enqueued
|
7
|
+
|
6
8
|
# Several of the parameters here are Lambdas (not Proc). Note you can pass a method using
|
7
9
|
# `method(:some_method)` or a lambda directly `->{ puts 'Hello'}`.
|
8
10
|
#
|
@@ -29,6 +31,7 @@ module WorkShaper
|
|
29
31
|
@shutdown = false
|
30
32
|
|
31
33
|
@total_enqueued = 0
|
34
|
+
@total_acked = 0
|
32
35
|
|
33
36
|
@heartbeat = Thread.new do
|
34
37
|
while true
|
@@ -57,11 +60,12 @@ module WorkShaper
|
|
57
60
|
def enqueue(sub_key, message, partition, offset)
|
58
61
|
raise StandardError, 'Shutting down' if @shutdown
|
59
62
|
pause_on_overrun
|
63
|
+
WorkShaper.logger.debug "Enqueue: #{sub_key}:#{partition}:#{offset}"
|
60
64
|
|
61
65
|
worker = nil
|
62
66
|
@semaphore.synchronize do
|
63
67
|
@total_enqueued += 1
|
64
|
-
(@received_offsets[partition] ||=
|
68
|
+
(@received_offsets[partition] ||= Array.new) << offset
|
65
69
|
|
66
70
|
worker =
|
67
71
|
@workers[sub_key] ||=
|
@@ -130,13 +134,11 @@ module WorkShaper
|
|
130
134
|
end
|
131
135
|
|
132
136
|
def offset_ack_unsafe(partition)
|
133
|
-
@total_acked ||= 0
|
134
|
-
|
135
137
|
completed = @completed_offsets[partition]
|
136
138
|
received = @received_offsets[partition]
|
137
139
|
|
138
|
-
offset = completed.first
|
139
|
-
while received.any? && received.first == offset
|
140
|
+
offset = completed.sort.first
|
141
|
+
while received.any? && received.sort.first == offset
|
140
142
|
# We observed Kafka sending the same message twice, even after
|
141
143
|
# having committed the offset. Here we skip this offset if we
|
142
144
|
# know it has already been committed.
|
@@ -161,8 +163,11 @@ module WorkShaper
|
|
161
163
|
end
|
162
164
|
|
163
165
|
@total_acked += 1
|
164
|
-
|
165
|
-
|
166
|
+
WorkShaper.logger.debug "@total_acked: #{@total_acked}"
|
167
|
+
WorkShaper.logger.debug "completed: [#{completed.join(', ')}]"
|
168
|
+
WorkShaper.logger.debug "received: [#{received.join(', ')}]"
|
169
|
+
completed.shift
|
170
|
+
received.shift
|
166
171
|
|
167
172
|
offset = completed.first
|
168
173
|
end
|
@@ -170,7 +175,11 @@ module WorkShaper
|
|
170
175
|
|
171
176
|
def pause_on_overrun
|
172
177
|
overrun = lambda do
|
178
|
+
completed = @completed_offsets.values.flatten.count
|
179
|
+
received = @received_offsets.values.flatten.count
|
180
|
+
|
173
181
|
@total_enqueued.to_i - @total_acked.to_i > @max_in_queue
|
182
|
+
received - completed > @max_in_queue
|
174
183
|
end
|
175
184
|
|
176
185
|
# We have to be careful here to avoid a deadlock. Another thread may be waiting
|
data/lib/work_shaper/version.rb
CHANGED
data/lib/work_shaper/worker.rb
CHANGED
@@ -33,7 +33,8 @@ module WorkShaper
|
|
33
33
|
@on_error.call(e, message, partition, offset)
|
34
34
|
ensure
|
35
35
|
@semaphore.synchronize do
|
36
|
-
|
36
|
+
WorkShaper.logger.debug "Completed: #{partition}:#{offset}"
|
37
|
+
(@completed_offsets[partition] ||= Array.new) << offset
|
37
38
|
end
|
38
39
|
end
|
39
40
|
# rubocop:enable Style/RescueStandardError
|
data/lib/work_shaper.rb
CHANGED
data/work_shaper.gemspec
CHANGED
@@ -32,7 +32,6 @@ Gem::Specification.new do |spec|
|
|
32
32
|
spec.require_paths = ["lib"]
|
33
33
|
|
34
34
|
# Uncomment to register a new dependency of your gem
|
35
|
-
spec.add_dependency "sorted_set", "~> 1.0"
|
36
35
|
spec.add_dependency "concurrent-ruby", "~> 1.2"
|
37
36
|
|
38
37
|
# For more information and examples about making a new gem, check out our
|
metadata
CHANGED
@@ -1,29 +1,15 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: work_shaper
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.2.
|
4
|
+
version: 0.1.2.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jerry Fernholz
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2024-
|
11
|
+
date: 2024-03-04 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
|
-
- !ruby/object:Gem::Dependency
|
14
|
-
name: sorted_set
|
15
|
-
requirement: !ruby/object:Gem::Requirement
|
16
|
-
requirements:
|
17
|
-
- - "~>"
|
18
|
-
- !ruby/object:Gem::Version
|
19
|
-
version: '1.0'
|
20
|
-
type: :runtime
|
21
|
-
prerelease: false
|
22
|
-
version_requirements: !ruby/object:Gem::Requirement
|
23
|
-
requirements:
|
24
|
-
- - "~>"
|
25
|
-
- !ruby/object:Gem::Version
|
26
|
-
version: '1.0'
|
27
13
|
- !ruby/object:Gem::Dependency
|
28
14
|
name: concurrent-ruby
|
29
15
|
requirement: !ruby/object:Gem::Requirement
|