work_shaper 0.1.2 → 0.1.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +4 -0
- data/lib/work_shaper/manager.rb +18 -8
- data/lib/work_shaper/version.rb +1 -1
- data/lib/work_shaper/worker.rb +7 -8
- data/lib/work_shaper.rb +3 -0
- data/work_shaper.gemspec +1 -1
- metadata +5 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9184d55b2d25f45b2f3fb832f9b990cca7d634dd590db1f6d3cf472d21361dcf
|
4
|
+
data.tar.gz: 7b0b8e413df089414635dd4362abe5f2d058403e0bba20370c9722fefe104e3e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 100431868af62d50e3e38a0d67c22f636660eb95db053788a785eb40ea13e27caf40e0a072924f16ea4129db76b20febeb98df4e8a116819442e40f5029eb9af
|
7
|
+
data.tar.gz: 1a075e88184f436172dee3b36e1088a8d9feba9ad43136f9e930c406fc3c095ba2dbe8943321d79077ddf7c3c570ca53d3286eee67d3d82ee8bff37f0f94891a
|
data/Gemfile
CHANGED
data/lib/work_shaper/manager.rb
CHANGED
@@ -3,6 +3,8 @@ module WorkShaper
|
|
3
3
|
# for each offset in monotonically increasing order (independent of the execution order), and gracefully
|
4
4
|
# cleaning up when `#shutdown` is called.
|
5
5
|
class Manager
|
6
|
+
attr_reader :total_acked, :total_enqueued
|
7
|
+
|
6
8
|
# Several of the parameters here are Lambdas (not Proc). Note you can pass a method using
|
7
9
|
# `method(:some_method)` or a lambda directly `->{ puts 'Hello'}`.
|
8
10
|
#
|
@@ -29,6 +31,7 @@ module WorkShaper
|
|
29
31
|
@shutdown = false
|
30
32
|
|
31
33
|
@total_enqueued = 0
|
34
|
+
@total_acked = 0
|
32
35
|
|
33
36
|
@heartbeat = Thread.new do
|
34
37
|
while true
|
@@ -57,11 +60,12 @@ module WorkShaper
|
|
57
60
|
def enqueue(sub_key, message, partition, offset)
|
58
61
|
raise StandardError, 'Shutting down' if @shutdown
|
59
62
|
pause_on_overrun
|
63
|
+
WorkShaper.logger.debug "Enqueue: #{sub_key}:#{partition}:#{offset}"
|
60
64
|
|
61
65
|
worker = nil
|
62
66
|
@semaphore.synchronize do
|
63
67
|
@total_enqueued += 1
|
64
|
-
(@received_offsets[partition] ||=
|
68
|
+
(@received_offsets[partition] ||= Array.new) << offset
|
65
69
|
|
66
70
|
worker =
|
67
71
|
@workers[sub_key] ||=
|
@@ -98,7 +102,8 @@ module WorkShaper
|
|
98
102
|
total_acked: @total_acked,
|
99
103
|
in_flight: (@total_enqueued.to_i - @total_acked.to_i),
|
100
104
|
last_acked_offsets: @last_ack,
|
101
|
-
worker_count: @workers.keys.count
|
105
|
+
worker_count: @workers.keys.count,
|
106
|
+
offset_mgr: @offset_manager.status
|
102
107
|
})
|
103
108
|
if detailed
|
104
109
|
WorkShaper.logger.info(
|
@@ -129,13 +134,11 @@ module WorkShaper
|
|
129
134
|
end
|
130
135
|
|
131
136
|
def offset_ack_unsafe(partition)
|
132
|
-
@total_acked ||= 0
|
133
|
-
|
134
137
|
completed = @completed_offsets[partition]
|
135
138
|
received = @received_offsets[partition]
|
136
139
|
|
137
|
-
offset = completed.first
|
138
|
-
while received.any? && received.first == offset
|
140
|
+
offset = completed.sort.first
|
141
|
+
while received.any? && received.sort.first == offset
|
139
142
|
# We observed Kafka sending the same message twice, even after
|
140
143
|
# having committed the offset. Here we skip this offset if we
|
141
144
|
# know it has already been committed.
|
@@ -160,8 +163,11 @@ module WorkShaper
|
|
160
163
|
end
|
161
164
|
|
162
165
|
@total_acked += 1
|
163
|
-
|
164
|
-
|
166
|
+
WorkShaper.logger.debug "@total_acked: #{@total_acked}"
|
167
|
+
WorkShaper.logger.debug "completed: [#{completed.join(', ')}]"
|
168
|
+
WorkShaper.logger.debug "received: [#{received.join(', ')}]"
|
169
|
+
completed.shift
|
170
|
+
received.shift
|
165
171
|
|
166
172
|
offset = completed.first
|
167
173
|
end
|
@@ -169,7 +175,11 @@ module WorkShaper
|
|
169
175
|
|
170
176
|
def pause_on_overrun
|
171
177
|
overrun = lambda do
|
178
|
+
completed = @completed_offsets.values.flatten.count
|
179
|
+
received = @received_offsets.values.flatten.count
|
180
|
+
|
172
181
|
@total_enqueued.to_i - @total_acked.to_i > @max_in_queue
|
182
|
+
received - completed > @max_in_queue
|
173
183
|
end
|
174
184
|
|
175
185
|
# We have to be careful here to avoid a deadlock. Another thread may be waiting
|
data/lib/work_shaper/version.rb
CHANGED
data/lib/work_shaper/worker.rb
CHANGED
@@ -27,16 +27,15 @@ module WorkShaper
|
|
27
27
|
@thread_pool.post do
|
28
28
|
@work.call(message, partition, offset)
|
29
29
|
@on_done.call(message, partition, offset)
|
30
|
-
@semaphore.synchronize do
|
31
|
-
(@completed_offsets[partition] ||= SortedSet.new) << offset
|
32
|
-
end
|
33
|
-
# @ack_handler.call(partition, offset)
|
34
30
|
rescue => e
|
35
|
-
|
36
|
-
|
37
|
-
# logger.error("Acking it anyways, why not?")
|
31
|
+
WorkShaper.logger.error("Error processing #{partition}:#{offset} #{e}")
|
32
|
+
WorkShaper.logger.error(e.backtrace.join(" > "))
|
38
33
|
@on_error.call(e, message, partition, offset)
|
39
|
-
|
34
|
+
ensure
|
35
|
+
@semaphore.synchronize do
|
36
|
+
WorkShaper.logger.debug "Completed: #{partition}:#{offset}"
|
37
|
+
(@completed_offsets[partition] ||= Array.new) << offset
|
38
|
+
end
|
40
39
|
end
|
41
40
|
# rubocop:enable Style/RescueStandardError
|
42
41
|
end
|
data/lib/work_shaper.rb
CHANGED
data/work_shaper.gemspec
CHANGED
@@ -32,7 +32,7 @@ Gem::Specification.new do |spec|
|
|
32
32
|
spec.require_paths = ["lib"]
|
33
33
|
|
34
34
|
# Uncomment to register a new dependency of your gem
|
35
|
-
spec.add_dependency "
|
35
|
+
spec.add_dependency "concurrent-ruby", "~> 1.2"
|
36
36
|
|
37
37
|
# For more information and examples about making a new gem, check out our
|
38
38
|
# guide at: https://bundler.io/guides/creating_gem.html
|
metadata
CHANGED
@@ -1,29 +1,29 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: work_shaper
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.2
|
4
|
+
version: 0.1.2.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jerry Fernholz
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2024-
|
11
|
+
date: 2024-03-04 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
name:
|
14
|
+
name: concurrent-ruby
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
17
|
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: '1.
|
19
|
+
version: '1.2'
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: '1.
|
26
|
+
version: '1.2'
|
27
27
|
description: WorkShaper was built to parallelize the work needed to process Kafka
|
28
28
|
messages.
|
29
29
|
email:
|