work_shaper 0.1.3.1rc5 → 0.1.3.1

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: ee42adfc0a4002f31b442eec3ea46dd3b878dc8fabadccb010cfb27070142223
4
- data.tar.gz: b8e268a9c867df76e210b279b3c8c2b62446b347c7ab02db1a381c3b5a80f7c6
3
+ metadata.gz: d3ea26c32471d401d3544efae02d400d5317b83373487e9e7c11ea9259f678ec
4
+ data.tar.gz: c28379ed6fbe660f76f97950e81563524aca4e6808ef1b91c2b66920a31b775a
5
5
  SHA512:
6
- metadata.gz: 94b8e9d89e51fce2f374eabef54d89a43703b48dba1dd9979598342a26182e2eb739a1fc0f2afa842100c1ab8bc32f094f2904680243e053d34f3ab82d42aded
7
- data.tar.gz: 2128b65b7c21b420b0e58c5765a9d557e466415fcfa32c80d4b9e9f407e2cb9de8b107f3e6deb81b61f086c0566bad256031aeb310f2676d004488e5af389bda
6
+ metadata.gz: 913324499dcd1a241e116bc9f397199237ae32b9c2b5f6992b50e71790031649c73d3bb09862f427d3d210b97d0dff3dd9a382ad9ca87d92a8fe7e2b2ab8291d
7
+ data.tar.gz: fae26f3d854e512328334291f93a928a561ae363ccf2ba11b28f1f3a886385861aa4512c0a754d547ff6f4175481c0d4a1cef72aacbf6e1c6e80dac33300ecc3
@@ -25,7 +25,6 @@ module WorkShaper
25
25
  @workers = {}
26
26
  @last_ack = {}
27
27
  @received_offsets = {}
28
- @completed_offsets = {}
29
28
  @max_in_queue = max_in_queue
30
29
  @semaphore = Mutex.new
31
30
  @shutting_down = false
@@ -45,7 +44,7 @@ module WorkShaper
45
44
 
46
45
  @offset_manager = Thread.new do
47
46
  while true
48
- @completed_offsets.each_key do |partition|
47
+ @received_offsets.each_key do |partition|
49
48
  offset_ack(partition)
50
49
  end
51
50
  sleep offset_commit_period_ms / 1000.0
@@ -77,7 +76,6 @@ module WorkShaper
77
76
  method(:offset_ack),
78
77
  @on_error,
79
78
  @last_ack,
80
- @completed_offsets,
81
79
  @semaphore,
82
80
  @max_in_queue
83
81
  )
@@ -91,7 +89,7 @@ module WorkShaper
91
89
  # the consumer restarts.
92
90
  def flush(safe: true)
93
91
  sleep 5
94
- @completed_offsets.each_key do |k|
92
+ @received_offsets.each_key do |k|
95
93
  safe ? offset_ack(k) : offset_ack_unsafe(k)
96
94
  end
97
95
  end
@@ -137,12 +135,11 @@ module WorkShaper
137
135
  end
138
136
 
139
137
  def offset_ack_unsafe(partition)
140
- completed = @completed_offsets[partition].sort!
141
138
  received = @received_offsets[partition].sort!
142
139
 
143
140
  begin
144
- offset = completed.first
145
- while received.any? && received.first == offset
141
+ offset = received.first
142
+ while offset && offset.completed?
146
143
  # We observed Kafka sending the same message twice, even after
147
144
  # having committed the offset. Here we skip this offset if we
148
145
  # know it has already been committed.
@@ -172,7 +169,6 @@ module WorkShaper
172
169
  WorkShaper.logger.warn(
173
170
  { message: 'Failed to Ack Offset, likely re-balance',
174
171
  offset: "#{partition}:#{offset}",
175
- completed: @completed_offsets[partition].to_a[0..10].join(','),
176
172
  received: @received_offsets[partition].to_a[0..10].join(',')
177
173
  })
178
174
  else
@@ -181,12 +177,10 @@ module WorkShaper
181
177
 
182
178
  @total_acked += 1
183
179
  WorkShaper.logger.debug "@total_acked: #{@total_acked}"
184
- WorkShaper.logger.debug "completed: [#{completed.join(', ')}]"
185
180
  WorkShaper.logger.debug "received: [#{received.join(', ')}]"
186
- completed.delete(offset)
187
181
  received.delete(offset)
188
182
 
189
- offset = completed.first
183
+ offset = received.first
190
184
  end
191
185
  rescue => e
192
186
  WorkShaper.logger.error({ message: 'Error in offset_ack', error: e })
@@ -196,6 +190,8 @@ module WorkShaper
196
190
 
197
191
  def pause_on_overrun
198
192
  overrun = lambda do
193
+ # I think we can do this now with the refactor
194
+ # @received_offsets.values.count > @max_in_queue
199
195
  @total_enqueued.to_i - @total_acked.to_i > @max_in_queue
200
196
  end
201
197
 
@@ -43,6 +43,10 @@ module WorkShaper
43
43
  @state = :completed
44
44
  end
45
45
 
46
+ def completed?
47
+ @state == :completed
48
+ end
49
+
46
50
  def to_i
47
51
  offset
48
52
  end
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module WorkShaper
4
- VERSION = "0.1.3.1rc5"
4
+ VERSION = "0.1.3.1"
5
5
  end
@@ -6,14 +6,13 @@ module WorkShaper
6
6
  # @param work [Lambda] Lambda that we will #call(message) to execute work.
7
7
  # @param on_done [Lambda] Lambda that we #call(partition, offset) when work is done.
8
8
  # @param on_error [Lambda] Lambda that we #call(exception) if an error is encountered.
9
- def initialize(work, on_done, ack_handler, on_error, last_ack, offset_stack, semaphore, max_in_queue)
9
+ def initialize(work, on_done, ack_handler, on_error, last_ack, semaphore, max_in_queue)
10
10
  @jobs = []
11
11
  @work = work
12
12
  @on_done = on_done
13
13
  @ack_handler = ack_handler
14
14
  @on_error = on_error
15
15
  @last_ack = last_ack
16
- @completed_offsets = offset_stack
17
16
  @semaphore = semaphore
18
17
  @max_in_queue = max_in_queue
19
18
  @thread_pool = Concurrent::FixedThreadPool.new(1, auto_terminate: false)
@@ -38,7 +37,6 @@ module WorkShaper
38
37
  @semaphore.synchronize do
39
38
  WorkShaper.logger.debug "Completed: #{partition}:#{offset}"
40
39
  offset_holder.complete!
41
- (@completed_offsets[partition] ||= Array.new) << offset_holder
42
40
  end
43
41
  end
44
42
  # rubocop:enable Style/RescueStandardError
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: work_shaper
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.3.1rc5
4
+ version: 0.1.3.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Jerry Fernholz
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2024-03-21 00:00:00.000000000 Z
11
+ date: 2024-03-27 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: concurrent-ruby
@@ -63,9 +63,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
63
63
  version: 2.6.0
64
64
  required_rubygems_version: !ruby/object:Gem::Requirement
65
65
  requirements:
66
- - - ">"
66
+ - - ">="
67
67
  - !ruby/object:Gem::Version
68
- version: 1.3.1
68
+ version: '0'
69
69
  requirements: []
70
70
  rubygems_version: 3.4.10
71
71
  signing_key: