deimos-ruby 1.9.1 → 1.10.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +25 -3
- data/Gemfile.lock +81 -76
- data/deimos-ruby.gemspec +2 -1
- data/docs/ARCHITECTURE.md +1 -5
- data/docs/CONFIGURATION.md +1 -0
- data/lib/deimos.rb +0 -1
- data/lib/deimos/active_record_consume/message_consumption.rb +5 -0
- data/lib/deimos/config/configuration.rb +13 -13
- data/lib/deimos/schema_backends/base.rb +2 -1
- data/lib/deimos/utils/db_producer.rb +2 -1
- data/lib/deimos/version.rb +1 -1
- data/spec/config/configuration_spec.rb +17 -17
- data/spec/schema_backends/base_spec.rb +4 -0
- data/spec/utils/db_producer_spec.rb +2 -0
- metadata +18 -8
- data/lib/deimos/config/configurable.rb +0 -278
- data/lib/deimos/monkey_patches/ruby_kafka_heartbeat.rb +0 -85
- data/spec/config/configurable_spec.rb +0 -136
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 4ef9a6d6960e0d8cf17946b13ba838646bc7bdbbad657131ebd36c7d41cacaf3
|
|
4
|
+
data.tar.gz: d37cdef98e76e9136b745c70ba2d4b8e4d72de2032299911b3f7e86e99bd45b7
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 1f794bd76a0f227a3ddeee6cb2957549076f805b3c68a586c2907af7829f5dded8764d8a35faf9ef214beff310743850ad1d3ca8963a03bcee6707f63791eaeb
|
|
7
|
+
data.tar.gz: aa129ceecf20171eb48de1fd24a06f4d5fa26470e80a05a51b0f541316969f5a724d7bf95c3f747043dc25e29d72ad119da94a63ba3dee74320ce6c621fde35e
|
data/CHANGELOG.md
CHANGED
|
@@ -5,13 +5,35 @@ All notable changes to this project will be documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
-
##
|
|
8
|
+
## 1.10.2 - 2021-07-20
|
|
9
9
|
|
|
10
|
-
|
|
10
|
+
- ### Fixes :wrench:
|
|
11
|
+
|
|
12
|
+
- Fixed issue where producers would stay in an error state after e.g. authorization failures for one topic that wouldn't apply to other topics.
|
|
13
|
+
|
|
14
|
+
## 1.10.1 - 2021-06-21
|
|
15
|
+
|
|
16
|
+
- ### Fixes :wrench:
|
|
17
|
+
|
|
18
|
+
- Fixed crash when trying to decode a nil payload (e.g. during instrumentation of `send_produce_error`.)
|
|
19
|
+
|
|
20
|
+
## 1.10.0 - 2021-03-22
|
|
21
|
+
|
|
22
|
+
- ### Roadmap :car:
|
|
23
|
+
|
|
24
|
+
- Extracted the configuration piece into a separate gem, [fig_tree](https://www.github.com/flipp-oss/fig_tree).
|
|
25
|
+
- Added a `save_record` method to ActiveRecordConsumer in case calling code wants to work with the record before saving.
|
|
26
|
+
|
|
27
|
+
- ### Fixes :wrench:
|
|
28
|
+
|
|
29
|
+
- Fixed a regression where the default values for consumer / Phobos listener configs were not correct (they were all nil). This is technically a breaking change, but it puts the configs back the way they were at version 1.4 and matches the documentation.
|
|
30
|
+
|
|
31
|
+
## 1.9.2 - 2021-01-29
|
|
11
32
|
|
|
12
33
|
- ### Fixes :wrench:
|
|
13
34
|
|
|
14
|
-
Fix for `uninitialized constant ActiveSupport::Autoload` in certain circumstances
|
|
35
|
+
- Fix for `uninitialized constant ActiveSupport::Autoload` in certain circumstances
|
|
36
|
+
- Removed unnecessary monkey patch which was crashing on newer versions of ruby-kafka
|
|
15
37
|
|
|
16
38
|
## 1.9.0 - 2021-01-28
|
|
17
39
|
|
data/Gemfile.lock
CHANGED
|
@@ -1,71 +1,72 @@
|
|
|
1
1
|
PATH
|
|
2
2
|
remote: .
|
|
3
3
|
specs:
|
|
4
|
-
deimos-ruby (1.
|
|
4
|
+
deimos-ruby (1.10.1)
|
|
5
5
|
avro_turf (~> 0.11)
|
|
6
|
+
fig_tree (~> 0.0.2)
|
|
6
7
|
phobos (>= 1.9, < 3.0)
|
|
7
8
|
ruby-kafka (< 2)
|
|
8
|
-
sigurd (
|
|
9
|
+
sigurd (~> 0.0.1)
|
|
9
10
|
|
|
10
11
|
GEM
|
|
11
12
|
remote: https://rubygems.org/
|
|
12
13
|
specs:
|
|
13
|
-
actioncable (6.1.
|
|
14
|
-
actionpack (= 6.1.
|
|
15
|
-
activesupport (= 6.1.
|
|
14
|
+
actioncable (6.1.3)
|
|
15
|
+
actionpack (= 6.1.3)
|
|
16
|
+
activesupport (= 6.1.3)
|
|
16
17
|
nio4r (~> 2.0)
|
|
17
18
|
websocket-driver (>= 0.6.1)
|
|
18
|
-
actionmailbox (6.1.
|
|
19
|
-
actionpack (= 6.1.
|
|
20
|
-
activejob (= 6.1.
|
|
21
|
-
activerecord (= 6.1.
|
|
22
|
-
activestorage (= 6.1.
|
|
23
|
-
activesupport (= 6.1.
|
|
19
|
+
actionmailbox (6.1.3)
|
|
20
|
+
actionpack (= 6.1.3)
|
|
21
|
+
activejob (= 6.1.3)
|
|
22
|
+
activerecord (= 6.1.3)
|
|
23
|
+
activestorage (= 6.1.3)
|
|
24
|
+
activesupport (= 6.1.3)
|
|
24
25
|
mail (>= 2.7.1)
|
|
25
|
-
actionmailer (6.1.
|
|
26
|
-
actionpack (= 6.1.
|
|
27
|
-
actionview (= 6.1.
|
|
28
|
-
activejob (= 6.1.
|
|
29
|
-
activesupport (= 6.1.
|
|
26
|
+
actionmailer (6.1.3)
|
|
27
|
+
actionpack (= 6.1.3)
|
|
28
|
+
actionview (= 6.1.3)
|
|
29
|
+
activejob (= 6.1.3)
|
|
30
|
+
activesupport (= 6.1.3)
|
|
30
31
|
mail (~> 2.5, >= 2.5.4)
|
|
31
32
|
rails-dom-testing (~> 2.0)
|
|
32
|
-
actionpack (6.1.
|
|
33
|
-
actionview (= 6.1.
|
|
34
|
-
activesupport (= 6.1.
|
|
33
|
+
actionpack (6.1.3)
|
|
34
|
+
actionview (= 6.1.3)
|
|
35
|
+
activesupport (= 6.1.3)
|
|
35
36
|
rack (~> 2.0, >= 2.0.9)
|
|
36
37
|
rack-test (>= 0.6.3)
|
|
37
38
|
rails-dom-testing (~> 2.0)
|
|
38
39
|
rails-html-sanitizer (~> 1.0, >= 1.2.0)
|
|
39
|
-
actiontext (6.1.
|
|
40
|
-
actionpack (= 6.1.
|
|
41
|
-
activerecord (= 6.1.
|
|
42
|
-
activestorage (= 6.1.
|
|
43
|
-
activesupport (= 6.1.
|
|
40
|
+
actiontext (6.1.3)
|
|
41
|
+
actionpack (= 6.1.3)
|
|
42
|
+
activerecord (= 6.1.3)
|
|
43
|
+
activestorage (= 6.1.3)
|
|
44
|
+
activesupport (= 6.1.3)
|
|
44
45
|
nokogiri (>= 1.8.5)
|
|
45
|
-
actionview (6.1.
|
|
46
|
-
activesupport (= 6.1.
|
|
46
|
+
actionview (6.1.3)
|
|
47
|
+
activesupport (= 6.1.3)
|
|
47
48
|
builder (~> 3.1)
|
|
48
49
|
erubi (~> 1.4)
|
|
49
50
|
rails-dom-testing (~> 2.0)
|
|
50
51
|
rails-html-sanitizer (~> 1.1, >= 1.2.0)
|
|
51
|
-
activejob (6.1.
|
|
52
|
-
activesupport (= 6.1.
|
|
52
|
+
activejob (6.1.3)
|
|
53
|
+
activesupport (= 6.1.3)
|
|
53
54
|
globalid (>= 0.3.6)
|
|
54
|
-
activemodel (6.1.
|
|
55
|
-
activesupport (= 6.1.
|
|
56
|
-
activerecord (6.1.
|
|
57
|
-
activemodel (= 6.1.
|
|
58
|
-
activesupport (= 6.1.
|
|
59
|
-
activerecord-import (1.0.
|
|
55
|
+
activemodel (6.1.3)
|
|
56
|
+
activesupport (= 6.1.3)
|
|
57
|
+
activerecord (6.1.3)
|
|
58
|
+
activemodel (= 6.1.3)
|
|
59
|
+
activesupport (= 6.1.3)
|
|
60
|
+
activerecord-import (1.0.8)
|
|
60
61
|
activerecord (>= 3.2)
|
|
61
|
-
activestorage (6.1.
|
|
62
|
-
actionpack (= 6.1.
|
|
63
|
-
activejob (= 6.1.
|
|
64
|
-
activerecord (= 6.1.
|
|
65
|
-
activesupport (= 6.1.
|
|
62
|
+
activestorage (6.1.3)
|
|
63
|
+
actionpack (= 6.1.3)
|
|
64
|
+
activejob (= 6.1.3)
|
|
65
|
+
activerecord (= 6.1.3)
|
|
66
|
+
activesupport (= 6.1.3)
|
|
66
67
|
marcel (~> 0.3.1)
|
|
67
68
|
mimemagic (~> 0.3.2)
|
|
68
|
-
activesupport (6.1.
|
|
69
|
+
activesupport (6.1.3)
|
|
69
70
|
concurrent-ruby (~> 1.0, >= 1.0.2)
|
|
70
71
|
i18n (>= 1.6, < 2)
|
|
71
72
|
minitest (>= 5.1)
|
|
@@ -83,17 +84,19 @@ GEM
|
|
|
83
84
|
concurrent-ruby-ext (1.1.8)
|
|
84
85
|
concurrent-ruby (= 1.1.8)
|
|
85
86
|
crass (1.0.6)
|
|
86
|
-
database_cleaner (1.
|
|
87
|
-
ddtrace (0.
|
|
87
|
+
database_cleaner (1.99.0)
|
|
88
|
+
ddtrace (0.46.0)
|
|
88
89
|
msgpack
|
|
89
90
|
diff-lcs (1.4.4)
|
|
90
91
|
digest-crc (0.6.3)
|
|
91
92
|
rake (>= 12.0.0, < 14.0.0)
|
|
92
|
-
dogstatsd-ruby (4.8.
|
|
93
|
+
dogstatsd-ruby (4.8.3)
|
|
93
94
|
erubi (1.10.0)
|
|
94
|
-
excon (0.
|
|
95
|
+
excon (0.81.0)
|
|
95
96
|
exponential-backoff (0.0.4)
|
|
96
|
-
ffi (1.
|
|
97
|
+
ffi (1.15.0)
|
|
98
|
+
fig_tree (0.0.2)
|
|
99
|
+
activesupport (>= 3.0.0)
|
|
97
100
|
formatador (0.2.5)
|
|
98
101
|
globalid (0.4.2)
|
|
99
102
|
activesupport (>= 4.2.0)
|
|
@@ -114,7 +117,7 @@ GEM
|
|
|
114
117
|
guard-rubocop (1.4.0)
|
|
115
118
|
guard (~> 2.0)
|
|
116
119
|
rubocop (< 2.0)
|
|
117
|
-
i18n (1.8.
|
|
120
|
+
i18n (1.8.9)
|
|
118
121
|
concurrent-ruby (~> 1.0)
|
|
119
122
|
listen (3.4.1)
|
|
120
123
|
rb-fsevent (~> 0.10, >= 0.10.3)
|
|
@@ -132,16 +135,18 @@ GEM
|
|
|
132
135
|
marcel (0.3.3)
|
|
133
136
|
mimemagic (~> 0.3.2)
|
|
134
137
|
method_source (1.0.0)
|
|
135
|
-
mimemagic (0.3.
|
|
138
|
+
mimemagic (0.3.10)
|
|
139
|
+
nokogiri (~> 1)
|
|
140
|
+
rake
|
|
136
141
|
mini_mime (1.0.2)
|
|
137
|
-
mini_portile2 (2.5.
|
|
138
|
-
minitest (5.14.
|
|
139
|
-
msgpack (1.4.
|
|
142
|
+
mini_portile2 (2.5.1)
|
|
143
|
+
minitest (5.14.4)
|
|
144
|
+
msgpack (1.4.2)
|
|
140
145
|
multi_json (1.15.0)
|
|
141
146
|
mysql2 (0.5.3)
|
|
142
147
|
nenv (0.3.0)
|
|
143
|
-
nio4r (2.5.
|
|
144
|
-
nokogiri (1.11.
|
|
148
|
+
nio4r (2.5.7)
|
|
149
|
+
nokogiri (1.11.5)
|
|
145
150
|
mini_portile2 (~> 2.5.0)
|
|
146
151
|
racc (~> 1.4)
|
|
147
152
|
notiffany (0.1.3)
|
|
@@ -159,36 +164,36 @@ GEM
|
|
|
159
164
|
logging
|
|
160
165
|
ruby-kafka
|
|
161
166
|
thor
|
|
162
|
-
pry (0.
|
|
167
|
+
pry (0.14.0)
|
|
163
168
|
coderay (~> 1.1)
|
|
164
169
|
method_source (~> 1.0)
|
|
165
170
|
racc (1.5.2)
|
|
166
171
|
rack (2.2.3)
|
|
167
172
|
rack-test (1.1.0)
|
|
168
173
|
rack (>= 1.0, < 3)
|
|
169
|
-
rails (6.1.
|
|
170
|
-
actioncable (= 6.1.
|
|
171
|
-
actionmailbox (= 6.1.
|
|
172
|
-
actionmailer (= 6.1.
|
|
173
|
-
actionpack (= 6.1.
|
|
174
|
-
actiontext (= 6.1.
|
|
175
|
-
actionview (= 6.1.
|
|
176
|
-
activejob (= 6.1.
|
|
177
|
-
activemodel (= 6.1.
|
|
178
|
-
activerecord (= 6.1.
|
|
179
|
-
activestorage (= 6.1.
|
|
180
|
-
activesupport (= 6.1.
|
|
174
|
+
rails (6.1.3)
|
|
175
|
+
actioncable (= 6.1.3)
|
|
176
|
+
actionmailbox (= 6.1.3)
|
|
177
|
+
actionmailer (= 6.1.3)
|
|
178
|
+
actionpack (= 6.1.3)
|
|
179
|
+
actiontext (= 6.1.3)
|
|
180
|
+
actionview (= 6.1.3)
|
|
181
|
+
activejob (= 6.1.3)
|
|
182
|
+
activemodel (= 6.1.3)
|
|
183
|
+
activerecord (= 6.1.3)
|
|
184
|
+
activestorage (= 6.1.3)
|
|
185
|
+
activesupport (= 6.1.3)
|
|
181
186
|
bundler (>= 1.15.0)
|
|
182
|
-
railties (= 6.1.
|
|
187
|
+
railties (= 6.1.3)
|
|
183
188
|
sprockets-rails (>= 2.0.0)
|
|
184
189
|
rails-dom-testing (2.0.3)
|
|
185
190
|
activesupport (>= 4.2.0)
|
|
186
191
|
nokogiri (>= 1.6)
|
|
187
192
|
rails-html-sanitizer (1.3.0)
|
|
188
193
|
loofah (~> 2.3)
|
|
189
|
-
railties (6.1.
|
|
190
|
-
actionpack (= 6.1.
|
|
191
|
-
activesupport (= 6.1.
|
|
194
|
+
railties (6.1.3)
|
|
195
|
+
actionpack (= 6.1.3)
|
|
196
|
+
activesupport (= 6.1.3)
|
|
192
197
|
method_source
|
|
193
198
|
rake (>= 0.8.7)
|
|
194
199
|
thor (~> 1.0)
|
|
@@ -197,8 +202,8 @@ GEM
|
|
|
197
202
|
rb-fsevent (0.10.4)
|
|
198
203
|
rb-inotify (0.10.1)
|
|
199
204
|
ffi (~> 1.0)
|
|
200
|
-
regexp_parser (2.
|
|
201
|
-
rexml (3.2.
|
|
205
|
+
regexp_parser (2.1.1)
|
|
206
|
+
rexml (3.2.5)
|
|
202
207
|
rspec (3.10.0)
|
|
203
208
|
rspec-core (~> 3.10.0)
|
|
204
209
|
rspec-expectations (~> 3.10.0)
|
|
@@ -208,10 +213,10 @@ GEM
|
|
|
208
213
|
rspec-expectations (3.10.1)
|
|
209
214
|
diff-lcs (>= 1.2.0, < 2.0)
|
|
210
215
|
rspec-support (~> 3.10.0)
|
|
211
|
-
rspec-mocks (3.10.
|
|
216
|
+
rspec-mocks (3.10.2)
|
|
212
217
|
diff-lcs (>= 1.2.0, < 2.0)
|
|
213
218
|
rspec-support (~> 3.10.0)
|
|
214
|
-
rspec-rails (4.
|
|
219
|
+
rspec-rails (4.1.1)
|
|
215
220
|
actionpack (>= 4.2)
|
|
216
221
|
activesupport (>= 4.2)
|
|
217
222
|
railties (>= 4.2)
|
|
@@ -219,7 +224,7 @@ GEM
|
|
|
219
224
|
rspec-expectations (~> 3.10)
|
|
220
225
|
rspec-mocks (~> 3.10)
|
|
221
226
|
rspec-support (~> 3.10)
|
|
222
|
-
rspec-support (3.10.
|
|
227
|
+
rspec-support (3.10.2)
|
|
223
228
|
rspec_junit_formatter (0.4.1)
|
|
224
229
|
rspec-core (>= 2, < 4, != 2.12.0)
|
|
225
230
|
rubocop (0.88.0)
|
|
@@ -239,7 +244,7 @@ GEM
|
|
|
239
244
|
digest-crc
|
|
240
245
|
ruby-progressbar (1.11.0)
|
|
241
246
|
shellany (0.0.1)
|
|
242
|
-
sigurd (0.0.
|
|
247
|
+
sigurd (0.0.3)
|
|
243
248
|
concurrent-ruby (~> 1)
|
|
244
249
|
exponential-backoff
|
|
245
250
|
sprockets (4.0.2)
|
|
@@ -284,4 +289,4 @@ DEPENDENCIES
|
|
|
284
289
|
sqlite3 (~> 1.3)
|
|
285
290
|
|
|
286
291
|
BUNDLED WITH
|
|
287
|
-
2.2.
|
|
292
|
+
2.2.17
|
data/deimos-ruby.gemspec
CHANGED
|
@@ -21,7 +21,8 @@ Gem::Specification.new do |spec|
|
|
|
21
21
|
spec.add_runtime_dependency('avro_turf', '~> 0.11')
|
|
22
22
|
spec.add_runtime_dependency('phobos', '>= 1.9', '< 3.0')
|
|
23
23
|
spec.add_runtime_dependency('ruby-kafka', '< 2')
|
|
24
|
-
spec.add_runtime_dependency('sigurd', '0.0.1')
|
|
24
|
+
spec.add_runtime_dependency('sigurd', '~> 0.0.1')
|
|
25
|
+
spec.add_runtime_dependency('fig_tree', '~> 0.0.2')
|
|
25
26
|
|
|
26
27
|
spec.add_development_dependency('activerecord-import')
|
|
27
28
|
spec.add_development_dependency('avro', '~> 1.9')
|
data/docs/ARCHITECTURE.md
CHANGED
|
@@ -74,11 +74,7 @@ must define methods such as:
|
|
|
74
74
|
|
|
75
75
|
### Configuration
|
|
76
76
|
|
|
77
|
-
Deimos
|
|
78
|
-
to provide a very succinct but powerful configuration format (including
|
|
79
|
-
default values, procs, print out as hash, reset, etc.). It also
|
|
80
|
-
allows for multiple blocks to define different objects of the same time
|
|
81
|
-
(like producers, consumers, pollers etc.).
|
|
77
|
+
Deimos uses the [https://www.github.com/flipp_oss/fig_tree](fig_tree) gem for configuration.
|
|
82
78
|
|
|
83
79
|
The configuration definition for Deimos is in `config/configuration.rb`. In
|
|
84
80
|
addition, there are methods in `config/phobos_config.rb` which translate to/from
|
data/docs/CONFIGURATION.md
CHANGED
|
@@ -91,6 +91,7 @@ delivery|`:batch`|The delivery mode for the consumer. Possible values: `:message
|
|
|
91
91
|
session_timeout|300|Number of seconds after which, if a client hasn't contacted the Kafka cluster, it will be kicked out of the group.
|
|
92
92
|
offset_commit_interval|10|Interval between offset commits, in seconds.
|
|
93
93
|
offset_commit_threshold|0|Number of messages that can be processed before their offsets are committed. If zero, offset commits are not triggered by message processing
|
|
94
|
+
offset_retention_time|nil|The time period that committed offsets will be retained, in seconds. Defaults to the broker setting.
|
|
94
95
|
heartbeat_interval|10|Interval between heartbeats; must be less than the session window.
|
|
95
96
|
backoff|`(1000..60_000)`|Range representing the minimum and maximum number of milliseconds to back off after a consumer error.
|
|
96
97
|
|
data/lib/deimos.rb
CHANGED
|
@@ -58,6 +58,11 @@ module Deimos
|
|
|
58
58
|
attrs.each do |k, v|
|
|
59
59
|
record.send("#{k}=", v)
|
|
60
60
|
end
|
|
61
|
+
save_record(record)
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
# @param record [ActiveRecord::Base]
|
|
65
|
+
def save_record(record)
|
|
61
66
|
record.created_at ||= Time.zone.now if record.respond_to?(:created_at)
|
|
62
67
|
record.updated_at = Time.zone.now if record.respond_to?(:updated_at)
|
|
63
68
|
record.save!
|
|
@@ -1,17 +1,17 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require 'fig_tree'
|
|
3
4
|
require_relative 'phobos_config'
|
|
4
|
-
require_relative 'configurable'
|
|
5
5
|
require_relative '../metrics/mock'
|
|
6
6
|
require_relative '../tracing/mock'
|
|
7
7
|
require 'active_support/core_ext/numeric'
|
|
8
8
|
|
|
9
9
|
# :nodoc:
|
|
10
10
|
module Deimos
|
|
11
|
-
include
|
|
11
|
+
include FigTree
|
|
12
12
|
|
|
13
13
|
# :nodoc:
|
|
14
|
-
class
|
|
14
|
+
class FigTree::ConfigStruct
|
|
15
15
|
include Deimos::PhobosConfig
|
|
16
16
|
end
|
|
17
17
|
|
|
@@ -60,7 +60,7 @@ module Deimos
|
|
|
60
60
|
end
|
|
61
61
|
end
|
|
62
62
|
|
|
63
|
-
# @param kafka_config [
|
|
63
|
+
# @param kafka_config [FigTree::ConfigStruct]
|
|
64
64
|
def self.configure_producer_or_consumer(kafka_config)
|
|
65
65
|
klass = kafka_config.class_name.constantize
|
|
66
66
|
klass.class_eval do
|
|
@@ -327,19 +327,19 @@ module Deimos
|
|
|
327
327
|
# These are the phobos "listener" configs. See CONFIGURATION.md for more
|
|
328
328
|
# info.
|
|
329
329
|
setting :group_id
|
|
330
|
-
setting :max_concurrency
|
|
331
|
-
setting :start_from_beginning
|
|
330
|
+
setting :max_concurrency, 1
|
|
331
|
+
setting :start_from_beginning, true
|
|
332
332
|
setting :max_bytes_per_partition, 500.kilobytes
|
|
333
|
-
setting :min_bytes
|
|
334
|
-
setting :max_wait_time
|
|
333
|
+
setting :min_bytes, 1
|
|
334
|
+
setting :max_wait_time, 5
|
|
335
335
|
setting :force_encoding
|
|
336
|
-
setting :delivery
|
|
336
|
+
setting :delivery, :batch
|
|
337
337
|
setting :backoff
|
|
338
|
-
setting :session_timeout
|
|
339
|
-
setting :offset_commit_interval
|
|
340
|
-
setting :offset_commit_threshold
|
|
338
|
+
setting :session_timeout, 300
|
|
339
|
+
setting :offset_commit_interval, 10
|
|
340
|
+
setting :offset_commit_threshold, 0
|
|
341
341
|
setting :offset_retention_time
|
|
342
|
-
setting :heartbeat_interval
|
|
342
|
+
setting :heartbeat_interval, 10
|
|
343
343
|
end
|
|
344
344
|
|
|
345
345
|
setting_object :db_poller do
|
|
@@ -40,8 +40,9 @@ module Deimos
|
|
|
40
40
|
# Decode a payload with a schema. Public method.
|
|
41
41
|
# @param payload [String]
|
|
42
42
|
# @param schema [Symbol|String]
|
|
43
|
-
# @return [Hash]
|
|
43
|
+
# @return [Hash,nil]
|
|
44
44
|
def decode(payload, schema: nil)
|
|
45
|
+
return nil if payload.nil?
|
|
45
46
|
decode_payload(payload, schema: schema || @schema)
|
|
46
47
|
end
|
|
47
48
|
|
|
@@ -60,7 +60,7 @@ module Deimos
|
|
|
60
60
|
end
|
|
61
61
|
|
|
62
62
|
# @param topic [String]
|
|
63
|
-
# @return [String] the topic that was locked, or nil if none were.
|
|
63
|
+
# @return [String, nil] the topic that was locked, or nil if none were.
|
|
64
64
|
def process_topic(topic)
|
|
65
65
|
# If the topic is already locked, another producer is currently
|
|
66
66
|
# working on it. Move on to the next one.
|
|
@@ -76,6 +76,7 @@ module Deimos
|
|
|
76
76
|
rescue StandardError => e
|
|
77
77
|
@logger.error("Error processing messages for topic #{@current_topic}: #{e.class.name}: #{e.message} #{e.backtrace.join("\n")}")
|
|
78
78
|
KafkaTopicInfo.register_error(@current_topic, @id)
|
|
79
|
+
shutdown_producer
|
|
79
80
|
end
|
|
80
81
|
|
|
81
82
|
# Process a single batch in a topic.
|
data/lib/deimos/version.rb
CHANGED
|
@@ -74,34 +74,34 @@ describe Deimos, 'configuration' do
|
|
|
74
74
|
{
|
|
75
75
|
topic: 'my_consume_topic',
|
|
76
76
|
group_id: 'my_group_id',
|
|
77
|
-
max_concurrency:
|
|
78
|
-
start_from_beginning:
|
|
77
|
+
max_concurrency: 1,
|
|
78
|
+
start_from_beginning: true,
|
|
79
79
|
max_bytes_per_partition: 524_288,
|
|
80
|
-
min_bytes:
|
|
81
|
-
max_wait_time:
|
|
80
|
+
min_bytes: 1,
|
|
81
|
+
max_wait_time: 5,
|
|
82
82
|
force_encoding: nil,
|
|
83
|
-
delivery:
|
|
84
|
-
session_timeout:
|
|
85
|
-
offset_commit_interval:
|
|
86
|
-
offset_commit_threshold:
|
|
83
|
+
delivery: 'batch',
|
|
84
|
+
session_timeout: 300,
|
|
85
|
+
offset_commit_interval: 10,
|
|
86
|
+
offset_commit_threshold: 0,
|
|
87
87
|
offset_retention_time: nil,
|
|
88
|
-
heartbeat_interval:
|
|
88
|
+
heartbeat_interval: 10,
|
|
89
89
|
handler: 'ConsumerTest::MyConsumer'
|
|
90
90
|
}, {
|
|
91
91
|
topic: 'my_batch_consume_topic',
|
|
92
92
|
group_id: 'my_batch_group_id',
|
|
93
|
-
max_concurrency:
|
|
94
|
-
start_from_beginning:
|
|
93
|
+
max_concurrency: 1,
|
|
94
|
+
start_from_beginning: true,
|
|
95
95
|
max_bytes_per_partition: 500.kilobytes,
|
|
96
|
-
min_bytes:
|
|
97
|
-
max_wait_time:
|
|
96
|
+
min_bytes: 1,
|
|
97
|
+
max_wait_time: 5,
|
|
98
98
|
force_encoding: nil,
|
|
99
99
|
delivery: 'inline_batch',
|
|
100
|
-
session_timeout:
|
|
101
|
-
offset_commit_interval:
|
|
102
|
-
offset_commit_threshold:
|
|
100
|
+
session_timeout: 300,
|
|
101
|
+
offset_commit_interval: 10,
|
|
102
|
+
offset_commit_threshold: 0,
|
|
103
103
|
offset_retention_time: nil,
|
|
104
|
-
heartbeat_interval:
|
|
104
|
+
heartbeat_interval: 10,
|
|
105
105
|
handler: 'ConsumerTest::MyBatchConsumer'
|
|
106
106
|
}
|
|
107
107
|
],
|
|
@@ -262,11 +262,13 @@ each_db_config(Deimos::Utils::DbProducer) do
|
|
|
262
262
|
end
|
|
263
263
|
|
|
264
264
|
it 'should register an error if it gets an error' do
|
|
265
|
+
allow(producer).to receive(:shutdown_producer)
|
|
265
266
|
expect(producer).to receive(:retrieve_messages).and_raise('OH NOES')
|
|
266
267
|
expect(Deimos::KafkaTopicInfo).to receive(:register_error).
|
|
267
268
|
with('my-topic', 'abc')
|
|
268
269
|
expect(producer).not_to receive(:produce_messages)
|
|
269
270
|
producer.process_topic('my-topic')
|
|
271
|
+
expect(producer).to have_received(:shutdown_producer)
|
|
270
272
|
end
|
|
271
273
|
|
|
272
274
|
it 'should move on if it gets a partial batch' do
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: deimos-ruby
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.
|
|
4
|
+
version: 1.10.2
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Daniel Orner
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2021-
|
|
11
|
+
date: 2021-07-20 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: avro_turf
|
|
@@ -62,16 +62,30 @@ dependencies:
|
|
|
62
62
|
name: sigurd
|
|
63
63
|
requirement: !ruby/object:Gem::Requirement
|
|
64
64
|
requirements:
|
|
65
|
-
- -
|
|
65
|
+
- - "~>"
|
|
66
66
|
- !ruby/object:Gem::Version
|
|
67
67
|
version: 0.0.1
|
|
68
68
|
type: :runtime
|
|
69
69
|
prerelease: false
|
|
70
70
|
version_requirements: !ruby/object:Gem::Requirement
|
|
71
71
|
requirements:
|
|
72
|
-
- -
|
|
72
|
+
- - "~>"
|
|
73
73
|
- !ruby/object:Gem::Version
|
|
74
74
|
version: 0.0.1
|
|
75
|
+
- !ruby/object:Gem::Dependency
|
|
76
|
+
name: fig_tree
|
|
77
|
+
requirement: !ruby/object:Gem::Requirement
|
|
78
|
+
requirements:
|
|
79
|
+
- - "~>"
|
|
80
|
+
- !ruby/object:Gem::Version
|
|
81
|
+
version: 0.0.2
|
|
82
|
+
type: :runtime
|
|
83
|
+
prerelease: false
|
|
84
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
85
|
+
requirements:
|
|
86
|
+
- - "~>"
|
|
87
|
+
- !ruby/object:Gem::Version
|
|
88
|
+
version: 0.0.2
|
|
75
89
|
- !ruby/object:Gem::Dependency
|
|
76
90
|
name: activerecord-import
|
|
77
91
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -370,7 +384,6 @@ files:
|
|
|
370
384
|
- lib/deimos/backends/kafka_async.rb
|
|
371
385
|
- lib/deimos/backends/test.rb
|
|
372
386
|
- lib/deimos/batch_consumer.rb
|
|
373
|
-
- lib/deimos/config/configurable.rb
|
|
374
387
|
- lib/deimos/config/configuration.rb
|
|
375
388
|
- lib/deimos/config/phobos_config.rb
|
|
376
389
|
- lib/deimos/consume/batch_consumption.rb
|
|
@@ -386,7 +399,6 @@ files:
|
|
|
386
399
|
- lib/deimos/metrics/provider.rb
|
|
387
400
|
- lib/deimos/monkey_patches/phobos_cli.rb
|
|
388
401
|
- lib/deimos/monkey_patches/phobos_producer.rb
|
|
389
|
-
- lib/deimos/monkey_patches/ruby_kafka_heartbeat.rb
|
|
390
402
|
- lib/deimos/poll_info.rb
|
|
391
403
|
- lib/deimos/producer.rb
|
|
392
404
|
- lib/deimos/railtie.rb
|
|
@@ -429,7 +441,6 @@ files:
|
|
|
429
441
|
- spec/backends/kafka_async_spec.rb
|
|
430
442
|
- spec/backends/kafka_spec.rb
|
|
431
443
|
- spec/batch_consumer_spec.rb
|
|
432
|
-
- spec/config/configurable_spec.rb
|
|
433
444
|
- spec/config/configuration_spec.rb
|
|
434
445
|
- spec/consumer_spec.rb
|
|
435
446
|
- spec/deimos_spec.rb
|
|
@@ -513,7 +524,6 @@ test_files:
|
|
|
513
524
|
- spec/backends/kafka_async_spec.rb
|
|
514
525
|
- spec/backends/kafka_spec.rb
|
|
515
526
|
- spec/batch_consumer_spec.rb
|
|
516
|
-
- spec/config/configurable_spec.rb
|
|
517
527
|
- spec/config/configuration_spec.rb
|
|
518
528
|
- spec/consumer_spec.rb
|
|
519
529
|
- spec/deimos_spec.rb
|
|
@@ -1,278 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
require 'active_support/concern'
|
|
4
|
-
require 'active_support/callbacks'
|
|
5
|
-
|
|
6
|
-
module Deimos
|
|
7
|
-
# Module to allow configuration. Loosely based off of the dry-configuration
|
|
8
|
-
# gem but with several advantages:
|
|
9
|
-
# - Works with Ruby 2.3.
|
|
10
|
-
# - More succinct syntax using method_missing so you do not need to write
|
|
11
|
-
# "config.whatever" and can just write "whatever".
|
|
12
|
-
# - Use nested blocks:
|
|
13
|
-
# Deimos.configure do
|
|
14
|
-
# config.kafka.ssl do
|
|
15
|
-
# enabled true
|
|
16
|
-
# ca_cert_file 'my_file'
|
|
17
|
-
# end
|
|
18
|
-
# config.kafka do
|
|
19
|
-
# ssl do
|
|
20
|
-
# enabled true
|
|
21
|
-
# end
|
|
22
|
-
# end
|
|
23
|
-
# end
|
|
24
|
-
# - Allows for arrays of configurations:
|
|
25
|
-
# Deimos.configure do |config|
|
|
26
|
-
# config.producer do
|
|
27
|
-
# class_name 'MyProducer'
|
|
28
|
-
# topic 'MyTopic'
|
|
29
|
-
# end
|
|
30
|
-
# end
|
|
31
|
-
# - Allows to call `configure` multiple times without crashing.
|
|
32
|
-
# - Allows to lazy-set default values by passing a proc as a default:
|
|
33
|
-
# Deimos.define_settings do |config|
|
|
34
|
-
# setting :my_val, default_proc: proc { MyDefault.calculated_value }
|
|
35
|
-
# end
|
|
36
|
-
# - Support for setting up and automatically calling deprecated configurations.
|
|
37
|
-
# - Support for configuration callbacks.
|
|
38
|
-
module Configurable
|
|
39
|
-
extend ActiveSupport::Concern
|
|
40
|
-
|
|
41
|
-
ConfigValue = Struct.new(:value, :default_value, :default_proc, :deprecation) do
|
|
42
|
-
|
|
43
|
-
# Reset value back to default.
|
|
44
|
-
def reset!
|
|
45
|
-
if self.value.is_a?(ConfigStruct)
|
|
46
|
-
self.value.reset!
|
|
47
|
-
else
|
|
48
|
-
self.value = self.default_value
|
|
49
|
-
end
|
|
50
|
-
end
|
|
51
|
-
|
|
52
|
-
# :nodoc:
|
|
53
|
-
def clone_and_reset
|
|
54
|
-
setting = ConfigValue.new(self.value, self.default_value,
|
|
55
|
-
self.default_proc, self.deprecation)
|
|
56
|
-
setting.reset!
|
|
57
|
-
setting
|
|
58
|
-
end
|
|
59
|
-
|
|
60
|
-
end
|
|
61
|
-
|
|
62
|
-
# Class that defines and keeps the configuration values.
|
|
63
|
-
class ConfigStruct
|
|
64
|
-
include ActiveSupport::Callbacks
|
|
65
|
-
|
|
66
|
-
define_callbacks :configure
|
|
67
|
-
|
|
68
|
-
# @param name [String]
|
|
69
|
-
def initialize(name)
|
|
70
|
-
@name = name
|
|
71
|
-
@settings = {}
|
|
72
|
-
@setting_objects = {}
|
|
73
|
-
@setting_templates = {}
|
|
74
|
-
end
|
|
75
|
-
|
|
76
|
-
# Reset config back to default values.
|
|
77
|
-
def reset!
|
|
78
|
-
@setting_objects = @setting_templates.map { |k, _| [k, []] }.to_h
|
|
79
|
-
@settings.values.each(&:reset!)
|
|
80
|
-
end
|
|
81
|
-
|
|
82
|
-
# Mark a configuration as deprecated and replaced with the new config.
|
|
83
|
-
# @param old_config [String]
|
|
84
|
-
# @param new_config [String]
|
|
85
|
-
def deprecate(old_config, new_config)
|
|
86
|
-
@settings[old_config.to_sym] ||= ConfigValue.new
|
|
87
|
-
@settings[old_config.to_sym].deprecation = new_config
|
|
88
|
-
end
|
|
89
|
-
|
|
90
|
-
# :nodoc:
|
|
91
|
-
def inspect
|
|
92
|
-
"#{@name}: #{@settings.inspect} #{@setting_objects.inspect}"
|
|
93
|
-
end
|
|
94
|
-
|
|
95
|
-
# @return [Hash]
|
|
96
|
-
def to_h
|
|
97
|
-
@settings.map { |k, v| [k, v.value] }.to_h
|
|
98
|
-
end
|
|
99
|
-
|
|
100
|
-
# :nodoc:
|
|
101
|
-
def clone_and_reset
|
|
102
|
-
new_config = self.clone
|
|
103
|
-
new_config.setting_objects = new_config.setting_objects.clone
|
|
104
|
-
new_config.settings = new_config.settings.map { |k, v| [k, v.clone_and_reset] }.to_h
|
|
105
|
-
new_config
|
|
106
|
-
end
|
|
107
|
-
|
|
108
|
-
# Define a setting template for an array of objects via a block:
|
|
109
|
-
# setting_object :producer do
|
|
110
|
-
# setting :topic
|
|
111
|
-
# setting :class_name
|
|
112
|
-
# end
|
|
113
|
-
# This will create the `producer` method to define these values as well
|
|
114
|
-
# as the `producer_objects` method to retrieve them.
|
|
115
|
-
# @param name [Symbol]
|
|
116
|
-
def setting_object(name, &block)
|
|
117
|
-
new_config = ConfigStruct.new("#{@name}.#{name}")
|
|
118
|
-
@setting_objects[name] = []
|
|
119
|
-
@setting_templates[name] = new_config
|
|
120
|
-
new_config.instance_eval(&block)
|
|
121
|
-
end
|
|
122
|
-
|
|
123
|
-
# Define a setting with the given name.
|
|
124
|
-
# @param name [Symbol]
|
|
125
|
-
# @param default_value [Object]
|
|
126
|
-
# @param default_proc [Proc]
|
|
127
|
-
def setting(name, default_value=nil, default_proc: nil, &block)
|
|
128
|
-
if block_given?
|
|
129
|
-
# Create a nested setting
|
|
130
|
-
setting_config = @settings[name]&.value || ConfigStruct.new("#{@name}.#{name}")
|
|
131
|
-
setting = ConfigValue.new
|
|
132
|
-
setting.value = setting_config
|
|
133
|
-
@settings[name] = setting
|
|
134
|
-
setting_config.instance_eval(&block)
|
|
135
|
-
else
|
|
136
|
-
setting = ConfigValue.new
|
|
137
|
-
setting.default_proc = default_proc
|
|
138
|
-
setting.default_value = default_value
|
|
139
|
-
setting.reset!
|
|
140
|
-
@settings[name] = setting
|
|
141
|
-
end
|
|
142
|
-
end
|
|
143
|
-
|
|
144
|
-
# :nodoc:
|
|
145
|
-
def respond_to_missing?(method, include_all=true)
|
|
146
|
-
method = method.to_s.sub(/=$/, '')
|
|
147
|
-
method.ends_with?('objects') ||
|
|
148
|
-
@setting_templates.key?(method.to_sym) ||
|
|
149
|
-
@settings.key?(method.to_sym) ||
|
|
150
|
-
super
|
|
151
|
-
end
|
|
152
|
-
|
|
153
|
-
# :nodoc:
|
|
154
|
-
def method_missing(method, *args, &block)
|
|
155
|
-
config_key = method.to_s.sub(/=$/, '').to_sym
|
|
156
|
-
|
|
157
|
-
# Return the list of setting objects with the given name
|
|
158
|
-
if config_key.to_s.end_with?('objects')
|
|
159
|
-
return _setting_object_method(config_key)
|
|
160
|
-
end
|
|
161
|
-
|
|
162
|
-
# Define a new setting object with the given name
|
|
163
|
-
if @setting_templates.key?(config_key) && block_given?
|
|
164
|
-
return _new_setting_object_method(config_key, &block)
|
|
165
|
-
end
|
|
166
|
-
|
|
167
|
-
setting = @settings[config_key]
|
|
168
|
-
|
|
169
|
-
if setting&.deprecation
|
|
170
|
-
return _deprecated_config_method(method, *args)
|
|
171
|
-
end
|
|
172
|
-
|
|
173
|
-
return super unless setting
|
|
174
|
-
|
|
175
|
-
if block_given?
|
|
176
|
-
return _block_config_method(config_key, &block)
|
|
177
|
-
end
|
|
178
|
-
|
|
179
|
-
_default_config_method(config_key, *args)
|
|
180
|
-
end
|
|
181
|
-
|
|
182
|
-
protected
|
|
183
|
-
|
|
184
|
-
# Only for the clone method
|
|
185
|
-
attr_accessor :settings, :setting_objects
|
|
186
|
-
|
|
187
|
-
private
|
|
188
|
-
|
|
189
|
-
def _deprecated_config_method(method, *args)
|
|
190
|
-
config_key = method.to_s.sub(/=$/, '').to_sym
|
|
191
|
-
new_config = @settings[config_key].deprecation
|
|
192
|
-
equals = method.to_s.end_with?('=') ? '=' : ''
|
|
193
|
-
ActiveSupport::Deprecation.warn("config.#{config_key}#{equals} is deprecated - use config.#{new_config}#{equals}")
|
|
194
|
-
obj = self
|
|
195
|
-
messages = new_config.split('.')
|
|
196
|
-
messages[0..-2].each do |message|
|
|
197
|
-
obj = obj.send(message)
|
|
198
|
-
end
|
|
199
|
-
if args.length.positive?
|
|
200
|
-
obj.send(messages[-1], args[0])
|
|
201
|
-
else
|
|
202
|
-
obj.send(messages[-1])
|
|
203
|
-
end
|
|
204
|
-
end
|
|
205
|
-
|
|
206
|
-
# Get or set a value.
|
|
207
|
-
def _default_config_method(config_key, *args)
|
|
208
|
-
if args.length.positive?
|
|
209
|
-
# Set the value
|
|
210
|
-
@settings[config_key].value = args[0]
|
|
211
|
-
else
|
|
212
|
-
# Get the value
|
|
213
|
-
setting = @settings[config_key]
|
|
214
|
-
if setting.default_proc && setting.value.nil?
|
|
215
|
-
setting.value = setting.default_proc.call
|
|
216
|
-
end
|
|
217
|
-
setting.value
|
|
218
|
-
end
|
|
219
|
-
end
|
|
220
|
-
|
|
221
|
-
# Define a new setting object and use the passed block to define values.
|
|
222
|
-
def _new_setting_object_method(config_key, &block)
|
|
223
|
-
new_config = @setting_templates[config_key].clone_and_reset
|
|
224
|
-
new_config.instance_eval(&block)
|
|
225
|
-
@setting_objects[config_key] << new_config
|
|
226
|
-
end
|
|
227
|
-
|
|
228
|
-
# Return a setting object.
|
|
229
|
-
def _setting_object_method(config_key)
|
|
230
|
-
key = config_key.to_s.sub(/_objects$/, '').to_sym
|
|
231
|
-
@setting_objects[key]
|
|
232
|
-
end
|
|
233
|
-
|
|
234
|
-
# Define new values inside a block.
|
|
235
|
-
def _block_config_method(config_key, &block)
|
|
236
|
-
unless @settings[config_key].value.is_a?(ConfigStruct)
|
|
237
|
-
raise "Block called for #{config_key} but it is not a nested config!"
|
|
238
|
-
end
|
|
239
|
-
|
|
240
|
-
@settings[config_key].value.instance_eval(&block)
|
|
241
|
-
end
|
|
242
|
-
end
|
|
243
|
-
|
|
244
|
-
# :nodoc:
|
|
245
|
-
module ClassMethods
|
|
246
|
-
# Define and redefine settings.
|
|
247
|
-
def define_settings(&block)
|
|
248
|
-
config.instance_eval(&block)
|
|
249
|
-
end
|
|
250
|
-
|
|
251
|
-
# Configure the settings with values.
|
|
252
|
-
def configure(&block)
|
|
253
|
-
if defined?(Rake) && defined?(Rake.application)
|
|
254
|
-
tasks = Rake.application.top_level_tasks
|
|
255
|
-
if tasks.any? { |t| %w(assets webpacker yarn).include?(t.split(':').first) }
|
|
256
|
-
puts 'Skipping Deimos configuration since we are in JS/CSS compilation'
|
|
257
|
-
return
|
|
258
|
-
end
|
|
259
|
-
end
|
|
260
|
-
config.run_callbacks(:configure) do
|
|
261
|
-
config.instance_eval(&block)
|
|
262
|
-
end
|
|
263
|
-
end
|
|
264
|
-
|
|
265
|
-
# @return [ConfigStruct]
|
|
266
|
-
def config
|
|
267
|
-
@config ||= ConfigStruct.new('config')
|
|
268
|
-
end
|
|
269
|
-
|
|
270
|
-
# Pass a block to run after configuration is done.
|
|
271
|
-
def after_configure(&block)
|
|
272
|
-
mod = self
|
|
273
|
-
config.class.set_callback(:configure, :after,
|
|
274
|
-
proc { mod.instance_eval(&block) })
|
|
275
|
-
end
|
|
276
|
-
end
|
|
277
|
-
end
|
|
278
|
-
end
|
|
@@ -1,85 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module Kafka
|
|
4
|
-
class Heartbeat
|
|
5
|
-
def initialize(group:, interval:, instrumenter:)
|
|
6
|
-
@group = group
|
|
7
|
-
@interval = interval
|
|
8
|
-
@last_heartbeat = Time.now
|
|
9
|
-
@instrumenter = instrumenter
|
|
10
|
-
end
|
|
11
|
-
|
|
12
|
-
def trigger!
|
|
13
|
-
@instrumenter.instrument('heartbeat.consumer',
|
|
14
|
-
group_id: @group.group_id,
|
|
15
|
-
topic_partitions: @group.assigned_partitions) do
|
|
16
|
-
@group.heartbeat
|
|
17
|
-
@last_heartbeat = Time.now
|
|
18
|
-
end
|
|
19
|
-
end
|
|
20
|
-
end
|
|
21
|
-
|
|
22
|
-
class Client
|
|
23
|
-
def consumer(
|
|
24
|
-
group_id:,
|
|
25
|
-
session_timeout: 30,
|
|
26
|
-
offset_commit_interval: 10,
|
|
27
|
-
offset_commit_threshold: 0,
|
|
28
|
-
heartbeat_interval: 10,
|
|
29
|
-
offset_retention_time: nil,
|
|
30
|
-
fetcher_max_queue_size: 100
|
|
31
|
-
)
|
|
32
|
-
cluster = initialize_cluster
|
|
33
|
-
|
|
34
|
-
instrumenter = DecoratingInstrumenter.new(@instrumenter,
|
|
35
|
-
group_id: group_id)
|
|
36
|
-
|
|
37
|
-
# The Kafka protocol expects the retention time to be in ms.
|
|
38
|
-
retention_time = (offset_retention_time && offset_retention_time * 1_000) || -1
|
|
39
|
-
|
|
40
|
-
group = ConsumerGroup.new(
|
|
41
|
-
cluster: cluster,
|
|
42
|
-
logger: @logger,
|
|
43
|
-
group_id: group_id,
|
|
44
|
-
session_timeout: session_timeout,
|
|
45
|
-
retention_time: retention_time,
|
|
46
|
-
instrumenter: instrumenter
|
|
47
|
-
)
|
|
48
|
-
|
|
49
|
-
fetcher = Fetcher.new(
|
|
50
|
-
cluster: initialize_cluster,
|
|
51
|
-
group: group,
|
|
52
|
-
logger: @logger,
|
|
53
|
-
instrumenter: instrumenter,
|
|
54
|
-
max_queue_size: fetcher_max_queue_size
|
|
55
|
-
)
|
|
56
|
-
|
|
57
|
-
offset_manager = OffsetManager.new(
|
|
58
|
-
cluster: cluster,
|
|
59
|
-
group: group,
|
|
60
|
-
fetcher: fetcher,
|
|
61
|
-
logger: @logger,
|
|
62
|
-
commit_interval: offset_commit_interval,
|
|
63
|
-
commit_threshold: offset_commit_threshold,
|
|
64
|
-
offset_retention_time: offset_retention_time
|
|
65
|
-
)
|
|
66
|
-
|
|
67
|
-
heartbeat = Heartbeat.new(
|
|
68
|
-
group: group,
|
|
69
|
-
interval: heartbeat_interval,
|
|
70
|
-
instrumenter: instrumenter
|
|
71
|
-
)
|
|
72
|
-
|
|
73
|
-
Consumer.new(
|
|
74
|
-
cluster: cluster,
|
|
75
|
-
logger: @logger,
|
|
76
|
-
instrumenter: instrumenter,
|
|
77
|
-
group: group,
|
|
78
|
-
offset_manager: offset_manager,
|
|
79
|
-
fetcher: fetcher,
|
|
80
|
-
session_timeout: session_timeout,
|
|
81
|
-
heartbeat: heartbeat
|
|
82
|
-
)
|
|
83
|
-
end
|
|
84
|
-
end
|
|
85
|
-
end
|
|
@@ -1,136 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
# :nodoc:
|
|
4
|
-
class MyConfig
|
|
5
|
-
include Deimos::Configurable
|
|
6
|
-
|
|
7
|
-
define_settings do
|
|
8
|
-
setting :set1
|
|
9
|
-
setting :set2, 'hi mom'
|
|
10
|
-
setting :group do
|
|
11
|
-
setting :set3, default_proc: proc { false }
|
|
12
|
-
setting :set5, (proc { 5 })
|
|
13
|
-
end
|
|
14
|
-
|
|
15
|
-
setting_object :listy do
|
|
16
|
-
setting :list1, 10
|
|
17
|
-
setting :list2, 5
|
|
18
|
-
end
|
|
19
|
-
end
|
|
20
|
-
end
|
|
21
|
-
|
|
22
|
-
describe Deimos::Configurable do
|
|
23
|
-
it 'should configure correctly with default values' do
|
|
24
|
-
expect(MyConfig.config.set1).to be_nil
|
|
25
|
-
expect(MyConfig.config.set2).to eq('hi mom')
|
|
26
|
-
expect(MyConfig.config.group.set3).to eq(false)
|
|
27
|
-
expect(MyConfig.config.listy_objects).to be_empty
|
|
28
|
-
expect { MyConfig.config.blah }.to raise_error(NameError)
|
|
29
|
-
expect { MyConfig.config.group.set4 }.to raise_error(NameError)
|
|
30
|
-
end
|
|
31
|
-
|
|
32
|
-
it 'should not call the proc until it has to' do
|
|
33
|
-
num_calls = 0
|
|
34
|
-
value_proc = proc do
|
|
35
|
-
num_calls += 1
|
|
36
|
-
num_calls
|
|
37
|
-
end
|
|
38
|
-
MyConfig.define_settings do
|
|
39
|
-
setting :set_with_proc, default_proc: value_proc
|
|
40
|
-
end
|
|
41
|
-
expect(num_calls).to eq(0)
|
|
42
|
-
expect(MyConfig.config.set_with_proc).to eq(1)
|
|
43
|
-
# calling twice should not call the proc again
|
|
44
|
-
expect(MyConfig.config.set_with_proc).to eq(1)
|
|
45
|
-
expect(num_calls).to eq(1)
|
|
46
|
-
end
|
|
47
|
-
|
|
48
|
-
it "should raise error when setting configs that don't exist" do
|
|
49
|
-
expect { MyConfig.configure { set15 'some_value' } }.to raise_error(NameError)
|
|
50
|
-
end
|
|
51
|
-
|
|
52
|
-
it 'should add values' do
|
|
53
|
-
MyConfig.configure do |config|
|
|
54
|
-
config.set1 = 5 # config.x syntax
|
|
55
|
-
set2 nil # method_missing syntax
|
|
56
|
-
config.group.set3 = true
|
|
57
|
-
end
|
|
58
|
-
|
|
59
|
-
# second configure should not blow anything away
|
|
60
|
-
MyConfig.configure do
|
|
61
|
-
listy do
|
|
62
|
-
list1 0
|
|
63
|
-
list2 1
|
|
64
|
-
end
|
|
65
|
-
listy do
|
|
66
|
-
list1 100
|
|
67
|
-
list2 200
|
|
68
|
-
end
|
|
69
|
-
end
|
|
70
|
-
|
|
71
|
-
expect(MyConfig.config.set1).to eq(5)
|
|
72
|
-
expect(MyConfig.config.set2).to be_nil
|
|
73
|
-
expect(MyConfig.config.group.set3).to eq(true)
|
|
74
|
-
expect(MyConfig.config.listy_objects.map(&:to_h)).
|
|
75
|
-
to eq([
|
|
76
|
-
{ list1: 0, list2: 1 },
|
|
77
|
-
{ list1: 100, list2: 200 }
|
|
78
|
-
])
|
|
79
|
-
|
|
80
|
-
# test reset!
|
|
81
|
-
MyConfig.config.reset!
|
|
82
|
-
expect(MyConfig.config.set1).to be_nil
|
|
83
|
-
expect(MyConfig.config.set2).to eq('hi mom')
|
|
84
|
-
expect(MyConfig.config.group.set3).to eq(false)
|
|
85
|
-
expect(MyConfig.config.listy_objects).to be_empty
|
|
86
|
-
end
|
|
87
|
-
|
|
88
|
-
it 'should add with block syntax' do
|
|
89
|
-
MyConfig.configure do
|
|
90
|
-
group do
|
|
91
|
-
set5(proc { 10 })
|
|
92
|
-
end
|
|
93
|
-
end
|
|
94
|
-
expect(MyConfig.config.group.set5.call).to eq(10)
|
|
95
|
-
end
|
|
96
|
-
|
|
97
|
-
it 'should add or redefine settings' do
|
|
98
|
-
MyConfig.define_settings do
|
|
99
|
-
setting :group do
|
|
100
|
-
setting :set6, 15
|
|
101
|
-
setting :set5, (proc { 15 })
|
|
102
|
-
end
|
|
103
|
-
setting_object :notey do
|
|
104
|
-
setting :note_title, 'some-title'
|
|
105
|
-
end
|
|
106
|
-
end
|
|
107
|
-
|
|
108
|
-
expect(MyConfig.config.group.set6).to eq(15)
|
|
109
|
-
expect(MyConfig.config.group.set5.call).to eq(15)
|
|
110
|
-
expect(MyConfig.config.listy_objects).to be_empty
|
|
111
|
-
expect(MyConfig.config.notey_objects).to be_empty
|
|
112
|
-
|
|
113
|
-
MyConfig.configure do
|
|
114
|
-
notey do
|
|
115
|
-
note_title 'hi mom'
|
|
116
|
-
end
|
|
117
|
-
listy do
|
|
118
|
-
list1 0
|
|
119
|
-
end
|
|
120
|
-
end
|
|
121
|
-
expect(MyConfig.config.notey_objects.size).to eq(1)
|
|
122
|
-
expect(MyConfig.config.notey_objects.first.note_title).to eq('hi mom')
|
|
123
|
-
expect(MyConfig.config.listy_objects.size).to eq(1)
|
|
124
|
-
expect(MyConfig.config.listy_objects.first.list1).to eq(0)
|
|
125
|
-
|
|
126
|
-
# This should not remove any keys
|
|
127
|
-
MyConfig.define_settings do
|
|
128
|
-
setting :group do
|
|
129
|
-
setting :set6, 20
|
|
130
|
-
end
|
|
131
|
-
end
|
|
132
|
-
expect(MyConfig.config.group.set6).to eq(20)
|
|
133
|
-
expect(MyConfig.config.group.set5.call).to eq(15)
|
|
134
|
-
end
|
|
135
|
-
|
|
136
|
-
end
|