rdkafka 0.13.0.beta.7 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/rdkafka/consumer.rb +3 -1
- data/lib/rdkafka/error.rb +6 -0
- data/lib/rdkafka/native_kafka.rb +54 -22
- data/lib/rdkafka/version.rb +1 -1
- data/spec/rdkafka/consumer_spec.rb +23 -0
- metadata +4 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b4b1881445c53cdd213efd2e7ed3feb94a32a327fdf63ab35455d337f09e7320
|
4
|
+
data.tar.gz: 02276df9702539c951ab392b4c5e086f87b731f7d319aedafb7b1209bc444688
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7f0e11a17d03091c8e13308c6998131c142349ce520473de7fec5908f0bcf2dfd59542966c3a850008f0db39e96b763b66f7b937487ba11f2fbb444e00aff25c
|
7
|
+
data.tar.gz: d15c5f7cd5f4b9041da55e7e46bf05936c627517d5067a9449844e2721e752de852006f432e5fbe72549d77d8378545a671a969652b4abada8aa8182f4fa4985
|
data/lib/rdkafka/consumer.rb
CHANGED
@@ -27,9 +27,11 @@ module Rdkafka
|
|
27
27
|
def close
|
28
28
|
return if closed?
|
29
29
|
ObjectSpace.undefine_finalizer(self)
|
30
|
-
|
30
|
+
|
31
|
+
@native_kafka.synchronize do |inner|
|
31
32
|
Rdkafka::Bindings.rd_kafka_consumer_close(inner)
|
32
33
|
end
|
34
|
+
|
33
35
|
@native_kafka.close
|
34
36
|
end
|
35
37
|
|
data/lib/rdkafka/error.rb
CHANGED
data/lib/rdkafka/native_kafka.rb
CHANGED
@@ -10,6 +10,22 @@ module Rdkafka
|
|
10
10
|
@access_mutex = Mutex.new
|
11
11
|
# Lock around internal polling
|
12
12
|
@poll_mutex = Mutex.new
|
13
|
+
# Lock around decrementing the operations in progress counter
|
14
|
+
# We have two mutexes - one for increment (`@access_mutex`) and one for decrement mutex
|
15
|
+
# because they serve different purposes:
|
16
|
+
#
|
17
|
+
# - `@access_mutex` allows us to lock the execution and make sure that any operation within
|
18
|
+
# the `#synchronize` is the only one running and that there are no other running
|
19
|
+
# operations.
|
20
|
+
# - `@decrement_mutex` ensures, that our decrement operation is thread-safe for any Ruby
|
21
|
+
# implementation.
|
22
|
+
#
|
23
|
+
# We do not use the same mutex, because it could create a deadlock when an already
|
24
|
+
# incremented operation cannot decrement because `@access_lock` is now owned by a different
|
25
|
+
# thread in a synchronized mode and the synchronized mode is waiting on the decrement.
|
26
|
+
@decrement_mutex = Mutex.new
|
27
|
+
# counter for operations in progress using inner
|
28
|
+
@operations_in_progress = 0
|
13
29
|
|
14
30
|
if run_polling_thread
|
15
31
|
# Start thread to poll client for delivery callbacks,
|
@@ -35,10 +51,26 @@ module Rdkafka
|
|
35
51
|
end
|
36
52
|
|
37
53
|
def with_inner
|
38
|
-
|
54
|
+
if @access_mutex.owned?
|
55
|
+
@operations_in_progress += 1
|
56
|
+
else
|
57
|
+
@access_mutex.synchronize { @operations_in_progress += 1 }
|
58
|
+
end
|
59
|
+
|
60
|
+
@inner.nil? ? raise(ClosedInnerError) : yield(@inner)
|
61
|
+
ensure
|
62
|
+
@decrement_mutex.synchronize { @operations_in_progress -= 1 }
|
63
|
+
end
|
39
64
|
|
65
|
+
def synchronize(&block)
|
40
66
|
@access_mutex.synchronize do
|
41
|
-
|
67
|
+
# Wait for any commands using the inner to finish
|
68
|
+
# This can take a while on blocking operations like polling but is essential not to proceed
|
69
|
+
# with certain types of operations like resources destruction as it can cause the process
|
70
|
+
# to hang or crash
|
71
|
+
sleep(0.01) until @operations_in_progress.zero?
|
72
|
+
|
73
|
+
with_inner(&block)
|
42
74
|
end
|
43
75
|
end
|
44
76
|
|
@@ -53,31 +85,31 @@ module Rdkafka
|
|
53
85
|
def close(object_id=nil)
|
54
86
|
return if closed?
|
55
87
|
|
56
|
-
|
88
|
+
synchronize do
|
89
|
+
# Indicate to the outside world that we are closing
|
90
|
+
@closing = true
|
57
91
|
|
58
|
-
|
59
|
-
|
92
|
+
if @polling_thread
|
93
|
+
# Indicate to polling thread that we're closing
|
94
|
+
@polling_thread[:closing] = true
|
60
95
|
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
# this can be aborted in practice if this
|
67
|
-
# code runs from a finalizer.
|
68
|
-
@polling_thread.join
|
69
|
-
end
|
96
|
+
# Wait for the polling thread to finish up,
|
97
|
+
# this can be aborted in practice if this
|
98
|
+
# code runs from a finalizer.
|
99
|
+
@polling_thread.join
|
100
|
+
end
|
70
101
|
|
71
|
-
|
72
|
-
|
102
|
+
# Destroy the client after locking both mutexes
|
103
|
+
@poll_mutex.lock
|
73
104
|
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
105
|
+
# This check prevents a race condition, where we would enter the close in two threads
|
106
|
+
# and after unlocking the primary one that hold the lock but finished, ours would be unlocked
|
107
|
+
# and would continue to run, trying to destroy inner twice
|
108
|
+
return unless @inner
|
78
109
|
|
79
|
-
|
80
|
-
|
110
|
+
Rdkafka::Bindings.rd_kafka_destroy(@inner)
|
111
|
+
@inner = nil
|
112
|
+
end
|
81
113
|
end
|
82
114
|
end
|
83
115
|
end
|
data/lib/rdkafka/version.rb
CHANGED
@@ -286,6 +286,29 @@ describe Rdkafka::Consumer do
|
|
286
286
|
consumer.poll(100)
|
287
287
|
}.to raise_error(Rdkafka::ClosedConsumerError, /poll/)
|
288
288
|
end
|
289
|
+
|
290
|
+
context 'when there are outgoing operations in other threads' do
|
291
|
+
it 'should wait and not crash' do
|
292
|
+
times = []
|
293
|
+
|
294
|
+
# Run a long running poll
|
295
|
+
thread = Thread.new do
|
296
|
+
times << Time.now
|
297
|
+
consumer.subscribe("empty_test_topic")
|
298
|
+
times << Time.now
|
299
|
+
consumer.poll(1_000)
|
300
|
+
times << Time.now
|
301
|
+
end
|
302
|
+
|
303
|
+
# Make sure it starts before we close
|
304
|
+
sleep(0.1)
|
305
|
+
consumer.close
|
306
|
+
close_time = Time.now
|
307
|
+
thread.join
|
308
|
+
|
309
|
+
times.each { |op_time| expect(op_time).to be < close_time }
|
310
|
+
end
|
311
|
+
end
|
289
312
|
end
|
290
313
|
|
291
314
|
describe "#commit, #committed and #store_offset" do
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rdkafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.13.0
|
4
|
+
version: 0.13.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Thijs Cadier
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2023-
|
11
|
+
date: 2023-07-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: ffi
|
@@ -216,9 +216,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
216
216
|
version: '2.6'
|
217
217
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
218
218
|
requirements:
|
219
|
-
- - "
|
219
|
+
- - ">="
|
220
220
|
- !ruby/object:Gem::Version
|
221
|
-
version:
|
221
|
+
version: '0'
|
222
222
|
requirements: []
|
223
223
|
rubygems_version: 3.4.1
|
224
224
|
signing_key:
|