rafka 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +11 -1
- data/lib/rafka.rb +1 -26
- data/lib/rafka/consumer.rb +2 -44
- data/lib/rafka/producer.rb +3 -5
- data/lib/rafka/version.rb +1 -1
- data/test/consumer_test.rb +3 -0
- metadata +8 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 28f4bb6a0680cce617d517bf528b575cbb389c25
|
4
|
+
data.tar.gz: 0a17d24c5183175436d58c8e35dad93f752ec91b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3197247d5b810b717bb7038c04ad3fd39eb63b7319f48d41f8c5eaa73d30d632d23312c2b795bc122ff1626c6c80395810fb47042ebd238d542b47b2e1d47382
|
7
|
+
data.tar.gz: 6200c187acaf499eb24ccc66b29e7c3185efa5784a81a88c6d7a892854ed6245665d27ca5b9c50ccfe6ee13b53107724e7188a25c2b2a7dbf1962812b791a797
|
data/CHANGELOG.md
CHANGED
@@ -9,7 +9,17 @@ Breaking changes are prefixed with a "[BREAKING]" label.
|
|
9
9
|
|
10
10
|
|
11
11
|
|
12
|
-
## 0.
|
12
|
+
## 0.4.0 (2018-09-04)
|
13
|
+
|
14
|
+
### Changed
|
15
|
+
|
16
|
+
- [BREAKING] Depend on redis-rb 3.3.2 or later [[#15](https://github.com/skroutz/rafka-rb/pull/15)]
|
17
|
+
- [BREAKING] `reconnect_attempts` option now defaults to `1` (ie. no reconnects) [[#15](https://github.com/skroutz/rafka-rb/pull/15)]
|
18
|
+
|
19
|
+
|
20
|
+
|
21
|
+
|
22
|
+
|
13
23
|
|
14
24
|
### Added
|
15
25
|
|
data/lib/rafka.rb
CHANGED
@@ -11,8 +11,7 @@ require "rafka/producer"
|
|
11
11
|
module Rafka
|
12
12
|
REDIS_DEFAULTS = {
|
13
13
|
host: "localhost",
|
14
|
-
port: 6380
|
15
|
-
reconnect_attempts: 5
|
14
|
+
port: 6380
|
16
15
|
}.freeze
|
17
16
|
|
18
17
|
def self.wrap_errors
|
@@ -22,28 +21,4 @@ module Rafka
|
|
22
21
|
raise ConsumeError, e.message[5..-1] if e.message.start_with?("CONS ")
|
23
22
|
raise CommandError, e.message
|
24
23
|
end
|
25
|
-
|
26
|
-
# redis-rb until 3.2.1 didn't retry to connect on
|
27
|
-
# Redis::CannotConnectError (eg. when rafka is down) so we manually retry
|
28
|
-
# 5 times
|
29
|
-
#
|
30
|
-
# TODO(agis): get rid of this method when we go to 3.2.1 or later, because
|
31
|
-
# https://github.com/redis/redis-rb/pull/476/
|
32
|
-
def self.with_retry(times: REDIS_DEFAULTS[:reconnect_attempts], every_sec: 1)
|
33
|
-
attempts = 0
|
34
|
-
|
35
|
-
begin
|
36
|
-
yield
|
37
|
-
rescue Redis::CannotConnectError => e
|
38
|
-
if Gem::Version.new(Redis::VERSION) < Gem::Version.new("3.2.2")
|
39
|
-
if attempts < times
|
40
|
-
attempts += 1
|
41
|
-
sleep every_sec
|
42
|
-
retry
|
43
|
-
end
|
44
|
-
end
|
45
|
-
|
46
|
-
raise e
|
47
|
-
end
|
48
|
-
end
|
49
24
|
end
|
data/lib/rafka/consumer.rb
CHANGED
@@ -77,14 +77,6 @@ module Rafka
|
|
77
77
|
# @example Consume a message and commit offset if the block does not raise an exception
|
78
78
|
# consumer.consume { |msg| puts "I received #{msg.value}" }
|
79
79
|
def consume(timeout=5)
|
80
|
-
# redis-rb didn't automatically call `CLIENT SETNAME` until v3.2.2
|
81
|
-
# (https://github.com/redis/redis-rb/issues/510)
|
82
|
-
#
|
83
|
-
# TODO(agis): get rid of this when we drop support for 3.2.1 and before
|
84
|
-
if !@redis.client.connected? && Gem::Version.new(Redis::VERSION) < Gem::Version.new("3.2.2")
|
85
|
-
set_name!
|
86
|
-
end
|
87
|
-
|
88
80
|
raised = false
|
89
81
|
msg = consume_one(timeout)
|
90
82
|
|
@@ -145,14 +137,6 @@ module Rafka
|
|
145
137
|
raise ArgumentError, "one of batch_size or batching_max_sec must be greater than 0"
|
146
138
|
end
|
147
139
|
|
148
|
-
# redis-rb didn't automatically call `CLIENT SETNAME` until v3.2.2
|
149
|
-
# (https://github.com/redis/redis-rb/issues/510)
|
150
|
-
#
|
151
|
-
# TODO(agis): get rid of this when we drop support for 3.2.1 and before
|
152
|
-
if !@redis.client.connected? && Gem::Version.new(Redis::VERSION) < Gem::Version.new("3.2.2")
|
153
|
-
set_name!
|
154
|
-
end
|
155
|
-
|
156
140
|
raised = false
|
157
141
|
start_time = Time.now
|
158
142
|
msgs = []
|
@@ -246,12 +230,6 @@ module Rafka
|
|
246
230
|
tp
|
247
231
|
end
|
248
232
|
|
249
|
-
def set_name!
|
250
|
-
Rafka.wrap_errors do
|
251
|
-
@redis.client.call([:client, :setname, @redis.id])
|
252
|
-
end
|
253
|
-
end
|
254
|
-
|
255
233
|
# @param timeout [Fixnum]
|
256
234
|
#
|
257
235
|
# @raise [MalformedMessageError]
|
@@ -259,29 +237,9 @@ module Rafka
|
|
259
237
|
# @return [nil, Message]
|
260
238
|
def consume_one(timeout)
|
261
239
|
msg = nil
|
262
|
-
setname_attempts = 0
|
263
240
|
|
264
|
-
|
265
|
-
|
266
|
-
Rafka.with_retry(times: @redis_opts[:reconnect_attempts]) do
|
267
|
-
msg = @redis.blpop(@blpop_arg, timeout: timeout)
|
268
|
-
end
|
269
|
-
end
|
270
|
-
rescue ConsumeError => e
|
271
|
-
# redis-rb didn't automatically call `CLIENT SETNAME` until v3.2.2
|
272
|
-
# (https://github.com/redis/redis-rb/issues/510)
|
273
|
-
#
|
274
|
-
# this is in case the server restarts while we were performing a BLPOP
|
275
|
-
#
|
276
|
-
# TODO(agis): get rid of this when we drop support for 3.2.1 and before
|
277
|
-
if e.message =~ /Identify yourself/ && setname_attempts < 5
|
278
|
-
sleep 0.5
|
279
|
-
set_name!
|
280
|
-
setname_attempts += 1
|
281
|
-
retry
|
282
|
-
end
|
283
|
-
|
284
|
-
raise e
|
241
|
+
Rafka.wrap_errors do
|
242
|
+
msg = @redis.blpop(@blpop_arg, timeout: timeout)
|
285
243
|
end
|
286
244
|
|
287
245
|
msg = Message.new(msg) if msg
|
data/lib/rafka/producer.rb
CHANGED
@@ -40,11 +40,9 @@ module Rafka
|
|
40
40
|
# produce("greetings", "Hola", key: "abc")
|
41
41
|
def produce(topic, msg, key: nil)
|
42
42
|
Rafka.wrap_errors do
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
@redis.rpushx(redis_key, msg.to_s)
|
47
|
-
end
|
43
|
+
redis_key = "topics:#{topic}"
|
44
|
+
redis_key << ":#{key}" if key
|
45
|
+
@redis.rpushx(redis_key, msg.to_s)
|
48
46
|
end
|
49
47
|
end
|
50
48
|
|
data/lib/rafka/version.rb
CHANGED
data/test/consumer_test.rb
CHANGED
@@ -64,6 +64,9 @@ class ConsumerTest < Minitest::Test
|
|
64
64
|
cons = Rafka::Consumer.new(group: "foo", topic: "bar", librdkafka: {})
|
65
65
|
assert_equal cons.blpop_arg, "topics:bar"
|
66
66
|
|
67
|
+
cons = Rafka::Consumer.new(group: "foo", topic: "bar", librdkafka: nil)
|
68
|
+
assert_equal cons.blpop_arg, "topics:bar"
|
69
|
+
|
67
70
|
cons = Rafka::Consumer.new(group: "foo", topic: "bar")
|
68
71
|
assert_equal cons.blpop_arg, "topics:bar"
|
69
72
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Agis Anastasopoulos
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-
|
11
|
+
date: 2018-09-04 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: redis
|
@@ -17,6 +17,9 @@ dependencies:
|
|
17
17
|
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
19
|
version: '3'
|
20
|
+
- - ">="
|
21
|
+
- !ruby/object:Gem::Version
|
22
|
+
version: 3.3.2
|
20
23
|
type: :runtime
|
21
24
|
prerelease: false
|
22
25
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -24,6 +27,9 @@ dependencies:
|
|
24
27
|
- - "~>"
|
25
28
|
- !ruby/object:Gem::Version
|
26
29
|
version: '3'
|
30
|
+
- - ">="
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: 3.3.2
|
27
33
|
- !ruby/object:Gem::Dependency
|
28
34
|
name: minitest
|
29
35
|
requirement: !ruby/object:Gem::Requirement
|