work_shaper 0.1.2 → 0.1.2.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 357d650ad73cfb685d1fd05a2ed98a0b4136358dc232febe04be921f8954855c
4
- data.tar.gz: 77bceccf96220ec5ba98e438fde84761dd3494878fae05005cccdad0121cb833
3
+ metadata.gz: 9184d55b2d25f45b2f3fb832f9b990cca7d634dd590db1f6d3cf472d21361dcf
4
+ data.tar.gz: 7b0b8e413df089414635dd4362abe5f2d058403e0bba20370c9722fefe104e3e
5
5
  SHA512:
6
- metadata.gz: 25c75b8e97b6e320e57164a2128ab320b23efe22c81d3e3f64857e70f92ee3dd881133060b702109ec1b172120a446ffb8d214a0423a039eb95ac4a6d06b3de3
7
- data.tar.gz: 72babc90aaef951eb50dcc3edea4bc71903497bc79aaf3c7708fc143649407a88d4df6240d5f3b232a60fc37f11446f758109d833bc65d0822a23578b2da7ee6
6
+ metadata.gz: 100431868af62d50e3e38a0d67c22f636660eb95db053788a785eb40ea13e27caf40e0a072924f16ea4129db76b20febeb98df4e8a116819442e40f5029eb9af
7
+ data.tar.gz: 1a075e88184f436172dee3b36e1088a8d9feba9ad43136f9e930c406fc3c095ba2dbe8943321d79077ddf7c3c570ca53d3286eee67d3d82ee8bff37f0f94891a
data/Gemfile CHANGED
@@ -10,3 +10,7 @@ gem "rake", "~> 13.0"
10
10
  gem "rspec", "~> 3.0"
11
11
 
12
12
  gem "rubocop", "~> 1.21"
13
+
14
+ gem "logger", "~> 1.4"
15
+
16
+ gem "concurrent-ruby", "~> 1.2"
@@ -3,6 +3,8 @@ module WorkShaper
3
3
  # for each offset in monotonically increasing order (independent of the execution order), and gracefully
4
4
  # cleaning up when `#shutdown` is called.
5
5
  class Manager
6
+ attr_reader :total_acked, :total_enqueued
7
+
6
8
  # Several of the parameters here are Lambdas (not Proc). Note you can pass a method using
7
9
  # `method(:some_method)` or a lambda directly `->{ puts 'Hello'}`.
8
10
  #
@@ -29,6 +31,7 @@ module WorkShaper
29
31
  @shutdown = false
30
32
 
31
33
  @total_enqueued = 0
34
+ @total_acked = 0
32
35
 
33
36
  @heartbeat = Thread.new do
34
37
  while true
@@ -57,11 +60,12 @@ module WorkShaper
57
60
  def enqueue(sub_key, message, partition, offset)
58
61
  raise StandardError, 'Shutting down' if @shutdown
59
62
  pause_on_overrun
63
+ WorkShaper.logger.debug "Enqueue: #{sub_key}:#{partition}:#{offset}"
60
64
 
61
65
  worker = nil
62
66
  @semaphore.synchronize do
63
67
  @total_enqueued += 1
64
- (@received_offsets[partition] ||= SortedSet.new) << offset
68
+ (@received_offsets[partition] ||= Array.new) << offset
65
69
 
66
70
  worker =
67
71
  @workers[sub_key] ||=
@@ -98,7 +102,8 @@ module WorkShaper
98
102
  total_acked: @total_acked,
99
103
  in_flight: (@total_enqueued.to_i - @total_acked.to_i),
100
104
  last_acked_offsets: @last_ack,
101
- worker_count: @workers.keys.count
105
+ worker_count: @workers.keys.count,
106
+ offset_mgr: @offset_manager.status
102
107
  })
103
108
  if detailed
104
109
  WorkShaper.logger.info(
@@ -129,13 +134,11 @@ module WorkShaper
129
134
  end
130
135
 
131
136
  def offset_ack_unsafe(partition)
132
- @total_acked ||= 0
133
-
134
137
  completed = @completed_offsets[partition]
135
138
  received = @received_offsets[partition]
136
139
 
137
- offset = completed.first
138
- while received.any? && received.first == offset
140
+ offset = completed.sort.first
141
+ while received.any? && received.sort.first == offset
139
142
  # We observed Kafka sending the same message twice, even after
140
143
  # having committed the offset. Here we skip this offset if we
141
144
  # know it has already been committed.
@@ -160,8 +163,11 @@ module WorkShaper
160
163
  end
161
164
 
162
165
  @total_acked += 1
163
- completed.delete(offset)
164
- received.delete(offset)
166
+ WorkShaper.logger.debug "@total_acked: #{@total_acked}"
167
+ WorkShaper.logger.debug "completed: [#{completed.join(', ')}]"
168
+ WorkShaper.logger.debug "received: [#{received.join(', ')}]"
169
+ completed.shift
170
+ received.shift
165
171
 
166
172
  offset = completed.first
167
173
  end
@@ -169,7 +175,11 @@ module WorkShaper
169
175
 
170
176
  def pause_on_overrun
171
177
  overrun = lambda do
178
+ completed = @completed_offsets.values.flatten.count
179
+ received = @received_offsets.values.flatten.count
180
+
172
181
  @total_enqueued.to_i - @total_acked.to_i > @max_in_queue
182
+ received - completed > @max_in_queue
173
183
  end
174
184
 
175
185
  # We have to be careful here to avoid a deadlock. Another thread may be waiting
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module WorkShaper
4
- VERSION = "0.1.2"
4
+ VERSION = "0.1.2.3"
5
5
  end
@@ -27,16 +27,15 @@ module WorkShaper
27
27
  @thread_pool.post do
28
28
  @work.call(message, partition, offset)
29
29
  @on_done.call(message, partition, offset)
30
- @semaphore.synchronize do
31
- (@completed_offsets[partition] ||= SortedSet.new) << offset
32
- end
33
- # @ack_handler.call(partition, offset)
34
30
  rescue => e
35
- puts("Error processing #{partition}:#{offset} #{e}")
36
- puts(e.backtrace.join("\n"))
37
- # logger.error("Acking it anyways, why not?")
31
+ WorkShaper.logger.error("Error processing #{partition}:#{offset} #{e}")
32
+ WorkShaper.logger.error(e.backtrace.join(" > "))
38
33
  @on_error.call(e, message, partition, offset)
39
- # @ack_handler.call(partition, offset)
34
+ ensure
35
+ @semaphore.synchronize do
36
+ WorkShaper.logger.debug "Completed: #{partition}:#{offset}"
37
+ (@completed_offsets[partition] ||= Array.new) << offset
38
+ end
40
39
  end
41
40
  # rubocop:enable Style/RescueStandardError
42
41
  end
data/lib/work_shaper.rb CHANGED
@@ -1,5 +1,8 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require 'logger'
4
+ require 'sorted_set'
5
+ require 'concurrent-ruby'
3
6
  require_relative "work_shaper/version"
4
7
  require_relative "work_shaper/manager"
5
8
  require_relative "work_shaper/worker"
data/work_shaper.gemspec CHANGED
@@ -32,7 +32,7 @@ Gem::Specification.new do |spec|
32
32
  spec.require_paths = ["lib"]
33
33
 
34
34
  # Uncomment to register a new dependency of your gem
35
- spec.add_dependency "sorted_set", "~> 1.0"
35
+ spec.add_dependency "concurrent-ruby", "~> 1.2"
36
36
 
37
37
  # For more information and examples about making a new gem, check out our
38
38
  # guide at: https://bundler.io/guides/creating_gem.html
metadata CHANGED
@@ -1,29 +1,29 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: work_shaper
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.2
4
+ version: 0.1.2.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Jerry Fernholz
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2024-01-05 00:00:00.000000000 Z
11
+ date: 2024-03-04 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
- name: sorted_set
14
+ name: concurrent-ruby
15
15
  requirement: !ruby/object:Gem::Requirement
16
16
  requirements:
17
17
  - - "~>"
18
18
  - !ruby/object:Gem::Version
19
- version: '1.0'
19
+ version: '1.2'
20
20
  type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
24
  - - "~>"
25
25
  - !ruby/object:Gem::Version
26
- version: '1.0'
26
+ version: '1.2'
27
27
  description: WorkShaper was built to parallelize the work needed to process Kafka
28
28
  messages.
29
29
  email: