deimos-ruby 1.9.1 → 1.10.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +25 -3
- data/Gemfile.lock +81 -76
- data/deimos-ruby.gemspec +2 -1
- data/docs/ARCHITECTURE.md +1 -5
- data/docs/CONFIGURATION.md +1 -0
- data/lib/deimos.rb +0 -1
- data/lib/deimos/active_record_consume/message_consumption.rb +5 -0
- data/lib/deimos/config/configuration.rb +13 -13
- data/lib/deimos/schema_backends/base.rb +2 -1
- data/lib/deimos/utils/db_producer.rb +2 -1
- data/lib/deimos/version.rb +1 -1
- data/spec/config/configuration_spec.rb +17 -17
- data/spec/schema_backends/base_spec.rb +4 -0
- data/spec/utils/db_producer_spec.rb +2 -0
- metadata +18 -8
- data/lib/deimos/config/configurable.rb +0 -278
- data/lib/deimos/monkey_patches/ruby_kafka_heartbeat.rb +0 -85
- data/spec/config/configurable_spec.rb +0 -136
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 4ef9a6d6960e0d8cf17946b13ba838646bc7bdbbad657131ebd36c7d41cacaf3
|
|
4
|
+
data.tar.gz: d37cdef98e76e9136b745c70ba2d4b8e4d72de2032299911b3f7e86e99bd45b7
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 1f794bd76a0f227a3ddeee6cb2957549076f805b3c68a586c2907af7829f5dded8764d8a35faf9ef214beff310743850ad1d3ca8963a03bcee6707f63791eaeb
|
|
7
|
+
data.tar.gz: aa129ceecf20171eb48de1fd24a06f4d5fa26470e80a05a51b0f541316969f5a724d7bf95c3f747043dc25e29d72ad119da94a63ba3dee74320ce6c621fde35e
|
data/CHANGELOG.md
CHANGED
|
@@ -5,13 +5,35 @@ All notable changes to this project will be documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
-
##
|
|
8
|
+
## 1.10.2 - 2021-07-20
|
|
9
9
|
|
|
10
|
-
|
|
10
|
+
- ### Fixes :wrench:
|
|
11
|
+
|
|
12
|
+
- Fixed issue where producers would stay in an error state after e.g. authorization failures for one topic that wouldn't apply to other topics.
|
|
13
|
+
|
|
14
|
+
## 1.10.1 - 2021-06-21
|
|
15
|
+
|
|
16
|
+
- ### Fixes :wrench:
|
|
17
|
+
|
|
18
|
+
- Fixed crash when trying to decode a nil payload (e.g. during instrumentation of `send_produce_error`.)
|
|
19
|
+
|
|
20
|
+
## 1.10.0 - 2021-03-22
|
|
21
|
+
|
|
22
|
+
- ### Roadmap :car:
|
|
23
|
+
|
|
24
|
+
- Extracted the configuration piece into a separate gem, [fig_tree](https://www.github.com/flipp-oss/fig_tree).
|
|
25
|
+
- Added a `save_record` method to ActiveRecordConsumer in case calling code wants to work with the record before saving.
|
|
26
|
+
|
|
27
|
+
- ### Fixes :wrench:
|
|
28
|
+
|
|
29
|
+
- Fixed a regression where the default values for consumer / Phobos listener configs were not correct (they were all nil). This is technically a breaking change, but it puts the configs back the way they were at version 1.4 and matches the documentation.
|
|
30
|
+
|
|
31
|
+
## 1.9.2 - 2021-01-29
|
|
11
32
|
|
|
12
33
|
- ### Fixes :wrench:
|
|
13
34
|
|
|
14
|
-
Fix for `uninitialized constant ActiveSupport::Autoload` in certain circumstances
|
|
35
|
+
- Fix for `uninitialized constant ActiveSupport::Autoload` in certain circumstances
|
|
36
|
+
- Removed unnecessary monkey patch which was crashing on newer versions of ruby-kafka
|
|
15
37
|
|
|
16
38
|
## 1.9.0 - 2021-01-28
|
|
17
39
|
|
data/Gemfile.lock
CHANGED
|
@@ -1,71 +1,72 @@
|
|
|
1
1
|
PATH
|
|
2
2
|
remote: .
|
|
3
3
|
specs:
|
|
4
|
-
deimos-ruby (1.
|
|
4
|
+
deimos-ruby (1.10.1)
|
|
5
5
|
avro_turf (~> 0.11)
|
|
6
|
+
fig_tree (~> 0.0.2)
|
|
6
7
|
phobos (>= 1.9, < 3.0)
|
|
7
8
|
ruby-kafka (< 2)
|
|
8
|
-
sigurd (
|
|
9
|
+
sigurd (~> 0.0.1)
|
|
9
10
|
|
|
10
11
|
GEM
|
|
11
12
|
remote: https://rubygems.org/
|
|
12
13
|
specs:
|
|
13
|
-
actioncable (6.1.
|
|
14
|
-
actionpack (= 6.1.
|
|
15
|
-
activesupport (= 6.1.
|
|
14
|
+
actioncable (6.1.3)
|
|
15
|
+
actionpack (= 6.1.3)
|
|
16
|
+
activesupport (= 6.1.3)
|
|
16
17
|
nio4r (~> 2.0)
|
|
17
18
|
websocket-driver (>= 0.6.1)
|
|
18
|
-
actionmailbox (6.1.
|
|
19
|
-
actionpack (= 6.1.
|
|
20
|
-
activejob (= 6.1.
|
|
21
|
-
activerecord (= 6.1.
|
|
22
|
-
activestorage (= 6.1.
|
|
23
|
-
activesupport (= 6.1.
|
|
19
|
+
actionmailbox (6.1.3)
|
|
20
|
+
actionpack (= 6.1.3)
|
|
21
|
+
activejob (= 6.1.3)
|
|
22
|
+
activerecord (= 6.1.3)
|
|
23
|
+
activestorage (= 6.1.3)
|
|
24
|
+
activesupport (= 6.1.3)
|
|
24
25
|
mail (>= 2.7.1)
|
|
25
|
-
actionmailer (6.1.
|
|
26
|
-
actionpack (= 6.1.
|
|
27
|
-
actionview (= 6.1.
|
|
28
|
-
activejob (= 6.1.
|
|
29
|
-
activesupport (= 6.1.
|
|
26
|
+
actionmailer (6.1.3)
|
|
27
|
+
actionpack (= 6.1.3)
|
|
28
|
+
actionview (= 6.1.3)
|
|
29
|
+
activejob (= 6.1.3)
|
|
30
|
+
activesupport (= 6.1.3)
|
|
30
31
|
mail (~> 2.5, >= 2.5.4)
|
|
31
32
|
rails-dom-testing (~> 2.0)
|
|
32
|
-
actionpack (6.1.
|
|
33
|
-
actionview (= 6.1.
|
|
34
|
-
activesupport (= 6.1.
|
|
33
|
+
actionpack (6.1.3)
|
|
34
|
+
actionview (= 6.1.3)
|
|
35
|
+
activesupport (= 6.1.3)
|
|
35
36
|
rack (~> 2.0, >= 2.0.9)
|
|
36
37
|
rack-test (>= 0.6.3)
|
|
37
38
|
rails-dom-testing (~> 2.0)
|
|
38
39
|
rails-html-sanitizer (~> 1.0, >= 1.2.0)
|
|
39
|
-
actiontext (6.1.
|
|
40
|
-
actionpack (= 6.1.
|
|
41
|
-
activerecord (= 6.1.
|
|
42
|
-
activestorage (= 6.1.
|
|
43
|
-
activesupport (= 6.1.
|
|
40
|
+
actiontext (6.1.3)
|
|
41
|
+
actionpack (= 6.1.3)
|
|
42
|
+
activerecord (= 6.1.3)
|
|
43
|
+
activestorage (= 6.1.3)
|
|
44
|
+
activesupport (= 6.1.3)
|
|
44
45
|
nokogiri (>= 1.8.5)
|
|
45
|
-
actionview (6.1.
|
|
46
|
-
activesupport (= 6.1.
|
|
46
|
+
actionview (6.1.3)
|
|
47
|
+
activesupport (= 6.1.3)
|
|
47
48
|
builder (~> 3.1)
|
|
48
49
|
erubi (~> 1.4)
|
|
49
50
|
rails-dom-testing (~> 2.0)
|
|
50
51
|
rails-html-sanitizer (~> 1.1, >= 1.2.0)
|
|
51
|
-
activejob (6.1.
|
|
52
|
-
activesupport (= 6.1.
|
|
52
|
+
activejob (6.1.3)
|
|
53
|
+
activesupport (= 6.1.3)
|
|
53
54
|
globalid (>= 0.3.6)
|
|
54
|
-
activemodel (6.1.
|
|
55
|
-
activesupport (= 6.1.
|
|
56
|
-
activerecord (6.1.
|
|
57
|
-
activemodel (= 6.1.
|
|
58
|
-
activesupport (= 6.1.
|
|
59
|
-
activerecord-import (1.0.
|
|
55
|
+
activemodel (6.1.3)
|
|
56
|
+
activesupport (= 6.1.3)
|
|
57
|
+
activerecord (6.1.3)
|
|
58
|
+
activemodel (= 6.1.3)
|
|
59
|
+
activesupport (= 6.1.3)
|
|
60
|
+
activerecord-import (1.0.8)
|
|
60
61
|
activerecord (>= 3.2)
|
|
61
|
-
activestorage (6.1.
|
|
62
|
-
actionpack (= 6.1.
|
|
63
|
-
activejob (= 6.1.
|
|
64
|
-
activerecord (= 6.1.
|
|
65
|
-
activesupport (= 6.1.
|
|
62
|
+
activestorage (6.1.3)
|
|
63
|
+
actionpack (= 6.1.3)
|
|
64
|
+
activejob (= 6.1.3)
|
|
65
|
+
activerecord (= 6.1.3)
|
|
66
|
+
activesupport (= 6.1.3)
|
|
66
67
|
marcel (~> 0.3.1)
|
|
67
68
|
mimemagic (~> 0.3.2)
|
|
68
|
-
activesupport (6.1.
|
|
69
|
+
activesupport (6.1.3)
|
|
69
70
|
concurrent-ruby (~> 1.0, >= 1.0.2)
|
|
70
71
|
i18n (>= 1.6, < 2)
|
|
71
72
|
minitest (>= 5.1)
|
|
@@ -83,17 +84,19 @@ GEM
|
|
|
83
84
|
concurrent-ruby-ext (1.1.8)
|
|
84
85
|
concurrent-ruby (= 1.1.8)
|
|
85
86
|
crass (1.0.6)
|
|
86
|
-
database_cleaner (1.
|
|
87
|
-
ddtrace (0.
|
|
87
|
+
database_cleaner (1.99.0)
|
|
88
|
+
ddtrace (0.46.0)
|
|
88
89
|
msgpack
|
|
89
90
|
diff-lcs (1.4.4)
|
|
90
91
|
digest-crc (0.6.3)
|
|
91
92
|
rake (>= 12.0.0, < 14.0.0)
|
|
92
|
-
dogstatsd-ruby (4.8.
|
|
93
|
+
dogstatsd-ruby (4.8.3)
|
|
93
94
|
erubi (1.10.0)
|
|
94
|
-
excon (0.
|
|
95
|
+
excon (0.81.0)
|
|
95
96
|
exponential-backoff (0.0.4)
|
|
96
|
-
ffi (1.
|
|
97
|
+
ffi (1.15.0)
|
|
98
|
+
fig_tree (0.0.2)
|
|
99
|
+
activesupport (>= 3.0.0)
|
|
97
100
|
formatador (0.2.5)
|
|
98
101
|
globalid (0.4.2)
|
|
99
102
|
activesupport (>= 4.2.0)
|
|
@@ -114,7 +117,7 @@ GEM
|
|
|
114
117
|
guard-rubocop (1.4.0)
|
|
115
118
|
guard (~> 2.0)
|
|
116
119
|
rubocop (< 2.0)
|
|
117
|
-
i18n (1.8.
|
|
120
|
+
i18n (1.8.9)
|
|
118
121
|
concurrent-ruby (~> 1.0)
|
|
119
122
|
listen (3.4.1)
|
|
120
123
|
rb-fsevent (~> 0.10, >= 0.10.3)
|
|
@@ -132,16 +135,18 @@ GEM
|
|
|
132
135
|
marcel (0.3.3)
|
|
133
136
|
mimemagic (~> 0.3.2)
|
|
134
137
|
method_source (1.0.0)
|
|
135
|
-
mimemagic (0.3.
|
|
138
|
+
mimemagic (0.3.10)
|
|
139
|
+
nokogiri (~> 1)
|
|
140
|
+
rake
|
|
136
141
|
mini_mime (1.0.2)
|
|
137
|
-
mini_portile2 (2.5.
|
|
138
|
-
minitest (5.14.
|
|
139
|
-
msgpack (1.4.
|
|
142
|
+
mini_portile2 (2.5.1)
|
|
143
|
+
minitest (5.14.4)
|
|
144
|
+
msgpack (1.4.2)
|
|
140
145
|
multi_json (1.15.0)
|
|
141
146
|
mysql2 (0.5.3)
|
|
142
147
|
nenv (0.3.0)
|
|
143
|
-
nio4r (2.5.
|
|
144
|
-
nokogiri (1.11.
|
|
148
|
+
nio4r (2.5.7)
|
|
149
|
+
nokogiri (1.11.5)
|
|
145
150
|
mini_portile2 (~> 2.5.0)
|
|
146
151
|
racc (~> 1.4)
|
|
147
152
|
notiffany (0.1.3)
|
|
@@ -159,36 +164,36 @@ GEM
|
|
|
159
164
|
logging
|
|
160
165
|
ruby-kafka
|
|
161
166
|
thor
|
|
162
|
-
pry (0.
|
|
167
|
+
pry (0.14.0)
|
|
163
168
|
coderay (~> 1.1)
|
|
164
169
|
method_source (~> 1.0)
|
|
165
170
|
racc (1.5.2)
|
|
166
171
|
rack (2.2.3)
|
|
167
172
|
rack-test (1.1.0)
|
|
168
173
|
rack (>= 1.0, < 3)
|
|
169
|
-
rails (6.1.
|
|
170
|
-
actioncable (= 6.1.
|
|
171
|
-
actionmailbox (= 6.1.
|
|
172
|
-
actionmailer (= 6.1.
|
|
173
|
-
actionpack (= 6.1.
|
|
174
|
-
actiontext (= 6.1.
|
|
175
|
-
actionview (= 6.1.
|
|
176
|
-
activejob (= 6.1.
|
|
177
|
-
activemodel (= 6.1.
|
|
178
|
-
activerecord (= 6.1.
|
|
179
|
-
activestorage (= 6.1.
|
|
180
|
-
activesupport (= 6.1.
|
|
174
|
+
rails (6.1.3)
|
|
175
|
+
actioncable (= 6.1.3)
|
|
176
|
+
actionmailbox (= 6.1.3)
|
|
177
|
+
actionmailer (= 6.1.3)
|
|
178
|
+
actionpack (= 6.1.3)
|
|
179
|
+
actiontext (= 6.1.3)
|
|
180
|
+
actionview (= 6.1.3)
|
|
181
|
+
activejob (= 6.1.3)
|
|
182
|
+
activemodel (= 6.1.3)
|
|
183
|
+
activerecord (= 6.1.3)
|
|
184
|
+
activestorage (= 6.1.3)
|
|
185
|
+
activesupport (= 6.1.3)
|
|
181
186
|
bundler (>= 1.15.0)
|
|
182
|
-
railties (= 6.1.
|
|
187
|
+
railties (= 6.1.3)
|
|
183
188
|
sprockets-rails (>= 2.0.0)
|
|
184
189
|
rails-dom-testing (2.0.3)
|
|
185
190
|
activesupport (>= 4.2.0)
|
|
186
191
|
nokogiri (>= 1.6)
|
|
187
192
|
rails-html-sanitizer (1.3.0)
|
|
188
193
|
loofah (~> 2.3)
|
|
189
|
-
railties (6.1.
|
|
190
|
-
actionpack (= 6.1.
|
|
191
|
-
activesupport (= 6.1.
|
|
194
|
+
railties (6.1.3)
|
|
195
|
+
actionpack (= 6.1.3)
|
|
196
|
+
activesupport (= 6.1.3)
|
|
192
197
|
method_source
|
|
193
198
|
rake (>= 0.8.7)
|
|
194
199
|
thor (~> 1.0)
|
|
@@ -197,8 +202,8 @@ GEM
|
|
|
197
202
|
rb-fsevent (0.10.4)
|
|
198
203
|
rb-inotify (0.10.1)
|
|
199
204
|
ffi (~> 1.0)
|
|
200
|
-
regexp_parser (2.
|
|
201
|
-
rexml (3.2.
|
|
205
|
+
regexp_parser (2.1.1)
|
|
206
|
+
rexml (3.2.5)
|
|
202
207
|
rspec (3.10.0)
|
|
203
208
|
rspec-core (~> 3.10.0)
|
|
204
209
|
rspec-expectations (~> 3.10.0)
|
|
@@ -208,10 +213,10 @@ GEM
|
|
|
208
213
|
rspec-expectations (3.10.1)
|
|
209
214
|
diff-lcs (>= 1.2.0, < 2.0)
|
|
210
215
|
rspec-support (~> 3.10.0)
|
|
211
|
-
rspec-mocks (3.10.
|
|
216
|
+
rspec-mocks (3.10.2)
|
|
212
217
|
diff-lcs (>= 1.2.0, < 2.0)
|
|
213
218
|
rspec-support (~> 3.10.0)
|
|
214
|
-
rspec-rails (4.
|
|
219
|
+
rspec-rails (4.1.1)
|
|
215
220
|
actionpack (>= 4.2)
|
|
216
221
|
activesupport (>= 4.2)
|
|
217
222
|
railties (>= 4.2)
|
|
@@ -219,7 +224,7 @@ GEM
|
|
|
219
224
|
rspec-expectations (~> 3.10)
|
|
220
225
|
rspec-mocks (~> 3.10)
|
|
221
226
|
rspec-support (~> 3.10)
|
|
222
|
-
rspec-support (3.10.
|
|
227
|
+
rspec-support (3.10.2)
|
|
223
228
|
rspec_junit_formatter (0.4.1)
|
|
224
229
|
rspec-core (>= 2, < 4, != 2.12.0)
|
|
225
230
|
rubocop (0.88.0)
|
|
@@ -239,7 +244,7 @@ GEM
|
|
|
239
244
|
digest-crc
|
|
240
245
|
ruby-progressbar (1.11.0)
|
|
241
246
|
shellany (0.0.1)
|
|
242
|
-
sigurd (0.0.
|
|
247
|
+
sigurd (0.0.3)
|
|
243
248
|
concurrent-ruby (~> 1)
|
|
244
249
|
exponential-backoff
|
|
245
250
|
sprockets (4.0.2)
|
|
@@ -284,4 +289,4 @@ DEPENDENCIES
|
|
|
284
289
|
sqlite3 (~> 1.3)
|
|
285
290
|
|
|
286
291
|
BUNDLED WITH
|
|
287
|
-
2.2.
|
|
292
|
+
2.2.17
|
data/deimos-ruby.gemspec
CHANGED
|
@@ -21,7 +21,8 @@ Gem::Specification.new do |spec|
|
|
|
21
21
|
spec.add_runtime_dependency('avro_turf', '~> 0.11')
|
|
22
22
|
spec.add_runtime_dependency('phobos', '>= 1.9', '< 3.0')
|
|
23
23
|
spec.add_runtime_dependency('ruby-kafka', '< 2')
|
|
24
|
-
spec.add_runtime_dependency('sigurd', '0.0.1')
|
|
24
|
+
spec.add_runtime_dependency('sigurd', '~> 0.0.1')
|
|
25
|
+
spec.add_runtime_dependency('fig_tree', '~> 0.0.2')
|
|
25
26
|
|
|
26
27
|
spec.add_development_dependency('activerecord-import')
|
|
27
28
|
spec.add_development_dependency('avro', '~> 1.9')
|
data/docs/ARCHITECTURE.md
CHANGED
|
@@ -74,11 +74,7 @@ must define methods such as:
|
|
|
74
74
|
|
|
75
75
|
### Configuration
|
|
76
76
|
|
|
77
|
-
Deimos
|
|
78
|
-
to provide a very succinct but powerful configuration format (including
|
|
79
|
-
default values, procs, print out as hash, reset, etc.). It also
|
|
80
|
-
allows for multiple blocks to define different objects of the same time
|
|
81
|
-
(like producers, consumers, pollers etc.).
|
|
77
|
+
Deimos uses the [https://www.github.com/flipp_oss/fig_tree](fig_tree) gem for configuration.
|
|
82
78
|
|
|
83
79
|
The configuration definition for Deimos is in `config/configuration.rb`. In
|
|
84
80
|
addition, there are methods in `config/phobos_config.rb` which translate to/from
|
data/docs/CONFIGURATION.md
CHANGED
|
@@ -91,6 +91,7 @@ delivery|`:batch`|The delivery mode for the consumer. Possible values: `:message
|
|
|
91
91
|
session_timeout|300|Number of seconds after which, if a client hasn't contacted the Kafka cluster, it will be kicked out of the group.
|
|
92
92
|
offset_commit_interval|10|Interval between offset commits, in seconds.
|
|
93
93
|
offset_commit_threshold|0|Number of messages that can be processed before their offsets are committed. If zero, offset commits are not triggered by message processing
|
|
94
|
+
offset_retention_time|nil|The time period that committed offsets will be retained, in seconds. Defaults to the broker setting.
|
|
94
95
|
heartbeat_interval|10|Interval between heartbeats; must be less than the session window.
|
|
95
96
|
backoff|`(1000..60_000)`|Range representing the minimum and maximum number of milliseconds to back off after a consumer error.
|
|
96
97
|
|
data/lib/deimos.rb
CHANGED
|
@@ -58,6 +58,11 @@ module Deimos
|
|
|
58
58
|
attrs.each do |k, v|
|
|
59
59
|
record.send("#{k}=", v)
|
|
60
60
|
end
|
|
61
|
+
save_record(record)
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
# @param record [ActiveRecord::Base]
|
|
65
|
+
def save_record(record)
|
|
61
66
|
record.created_at ||= Time.zone.now if record.respond_to?(:created_at)
|
|
62
67
|
record.updated_at = Time.zone.now if record.respond_to?(:updated_at)
|
|
63
68
|
record.save!
|
|
@@ -1,17 +1,17 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require 'fig_tree'
|
|
3
4
|
require_relative 'phobos_config'
|
|
4
|
-
require_relative 'configurable'
|
|
5
5
|
require_relative '../metrics/mock'
|
|
6
6
|
require_relative '../tracing/mock'
|
|
7
7
|
require 'active_support/core_ext/numeric'
|
|
8
8
|
|
|
9
9
|
# :nodoc:
|
|
10
10
|
module Deimos
|
|
11
|
-
include
|
|
11
|
+
include FigTree
|
|
12
12
|
|
|
13
13
|
# :nodoc:
|
|
14
|
-
class
|
|
14
|
+
class FigTree::ConfigStruct
|
|
15
15
|
include Deimos::PhobosConfig
|
|
16
16
|
end
|
|
17
17
|
|
|
@@ -60,7 +60,7 @@ module Deimos
|
|
|
60
60
|
end
|
|
61
61
|
end
|
|
62
62
|
|
|
63
|
-
# @param kafka_config [
|
|
63
|
+
# @param kafka_config [FigTree::ConfigStruct]
|
|
64
64
|
def self.configure_producer_or_consumer(kafka_config)
|
|
65
65
|
klass = kafka_config.class_name.constantize
|
|
66
66
|
klass.class_eval do
|
|
@@ -327,19 +327,19 @@ module Deimos
|
|
|
327
327
|
# These are the phobos "listener" configs. See CONFIGURATION.md for more
|
|
328
328
|
# info.
|
|
329
329
|
setting :group_id
|
|
330
|
-
setting :max_concurrency
|
|
331
|
-
setting :start_from_beginning
|
|
330
|
+
setting :max_concurrency, 1
|
|
331
|
+
setting :start_from_beginning, true
|
|
332
332
|
setting :max_bytes_per_partition, 500.kilobytes
|
|
333
|
-
setting :min_bytes
|
|
334
|
-
setting :max_wait_time
|
|
333
|
+
setting :min_bytes, 1
|
|
334
|
+
setting :max_wait_time, 5
|
|
335
335
|
setting :force_encoding
|
|
336
|
-
setting :delivery
|
|
336
|
+
setting :delivery, :batch
|
|
337
337
|
setting :backoff
|
|
338
|
-
setting :session_timeout
|
|
339
|
-
setting :offset_commit_interval
|
|
340
|
-
setting :offset_commit_threshold
|
|
338
|
+
setting :session_timeout, 300
|
|
339
|
+
setting :offset_commit_interval, 10
|
|
340
|
+
setting :offset_commit_threshold, 0
|
|
341
341
|
setting :offset_retention_time
|
|
342
|
-
setting :heartbeat_interval
|
|
342
|
+
setting :heartbeat_interval, 10
|
|
343
343
|
end
|
|
344
344
|
|
|
345
345
|
setting_object :db_poller do
|
|
@@ -40,8 +40,9 @@ module Deimos
|
|
|
40
40
|
# Decode a payload with a schema. Public method.
|
|
41
41
|
# @param payload [String]
|
|
42
42
|
# @param schema [Symbol|String]
|
|
43
|
-
# @return [Hash]
|
|
43
|
+
# @return [Hash,nil]
|
|
44
44
|
def decode(payload, schema: nil)
|
|
45
|
+
return nil if payload.nil?
|
|
45
46
|
decode_payload(payload, schema: schema || @schema)
|
|
46
47
|
end
|
|
47
48
|
|
|
@@ -60,7 +60,7 @@ module Deimos
|
|
|
60
60
|
end
|
|
61
61
|
|
|
62
62
|
# @param topic [String]
|
|
63
|
-
# @return [String] the topic that was locked, or nil if none were.
|
|
63
|
+
# @return [String, nil] the topic that was locked, or nil if none were.
|
|
64
64
|
def process_topic(topic)
|
|
65
65
|
# If the topic is already locked, another producer is currently
|
|
66
66
|
# working on it. Move on to the next one.
|
|
@@ -76,6 +76,7 @@ module Deimos
|
|
|
76
76
|
rescue StandardError => e
|
|
77
77
|
@logger.error("Error processing messages for topic #{@current_topic}: #{e.class.name}: #{e.message} #{e.backtrace.join("\n")}")
|
|
78
78
|
KafkaTopicInfo.register_error(@current_topic, @id)
|
|
79
|
+
shutdown_producer
|
|
79
80
|
end
|
|
80
81
|
|
|
81
82
|
# Process a single batch in a topic.
|
data/lib/deimos/version.rb
CHANGED
|
@@ -74,34 +74,34 @@ describe Deimos, 'configuration' do
|
|
|
74
74
|
{
|
|
75
75
|
topic: 'my_consume_topic',
|
|
76
76
|
group_id: 'my_group_id',
|
|
77
|
-
max_concurrency:
|
|
78
|
-
start_from_beginning:
|
|
77
|
+
max_concurrency: 1,
|
|
78
|
+
start_from_beginning: true,
|
|
79
79
|
max_bytes_per_partition: 524_288,
|
|
80
|
-
min_bytes:
|
|
81
|
-
max_wait_time:
|
|
80
|
+
min_bytes: 1,
|
|
81
|
+
max_wait_time: 5,
|
|
82
82
|
force_encoding: nil,
|
|
83
|
-
delivery:
|
|
84
|
-
session_timeout:
|
|
85
|
-
offset_commit_interval:
|
|
86
|
-
offset_commit_threshold:
|
|
83
|
+
delivery: 'batch',
|
|
84
|
+
session_timeout: 300,
|
|
85
|
+
offset_commit_interval: 10,
|
|
86
|
+
offset_commit_threshold: 0,
|
|
87
87
|
offset_retention_time: nil,
|
|
88
|
-
heartbeat_interval:
|
|
88
|
+
heartbeat_interval: 10,
|
|
89
89
|
handler: 'ConsumerTest::MyConsumer'
|
|
90
90
|
}, {
|
|
91
91
|
topic: 'my_batch_consume_topic',
|
|
92
92
|
group_id: 'my_batch_group_id',
|
|
93
|
-
max_concurrency:
|
|
94
|
-
start_from_beginning:
|
|
93
|
+
max_concurrency: 1,
|
|
94
|
+
start_from_beginning: true,
|
|
95
95
|
max_bytes_per_partition: 500.kilobytes,
|
|
96
|
-
min_bytes:
|
|
97
|
-
max_wait_time:
|
|
96
|
+
min_bytes: 1,
|
|
97
|
+
max_wait_time: 5,
|
|
98
98
|
force_encoding: nil,
|
|
99
99
|
delivery: 'inline_batch',
|
|
100
|
-
session_timeout:
|
|
101
|
-
offset_commit_interval:
|
|
102
|
-
offset_commit_threshold:
|
|
100
|
+
session_timeout: 300,
|
|
101
|
+
offset_commit_interval: 10,
|
|
102
|
+
offset_commit_threshold: 0,
|
|
103
103
|
offset_retention_time: nil,
|
|
104
|
-
heartbeat_interval:
|
|
104
|
+
heartbeat_interval: 10,
|
|
105
105
|
handler: 'ConsumerTest::MyBatchConsumer'
|
|
106
106
|
}
|
|
107
107
|
],
|
|
@@ -262,11 +262,13 @@ each_db_config(Deimos::Utils::DbProducer) do
|
|
|
262
262
|
end
|
|
263
263
|
|
|
264
264
|
it 'should register an error if it gets an error' do
|
|
265
|
+
allow(producer).to receive(:shutdown_producer)
|
|
265
266
|
expect(producer).to receive(:retrieve_messages).and_raise('OH NOES')
|
|
266
267
|
expect(Deimos::KafkaTopicInfo).to receive(:register_error).
|
|
267
268
|
with('my-topic', 'abc')
|
|
268
269
|
expect(producer).not_to receive(:produce_messages)
|
|
269
270
|
producer.process_topic('my-topic')
|
|
271
|
+
expect(producer).to have_received(:shutdown_producer)
|
|
270
272
|
end
|
|
271
273
|
|
|
272
274
|
it 'should move on if it gets a partial batch' do
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: deimos-ruby
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.
|
|
4
|
+
version: 1.10.2
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Daniel Orner
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2021-
|
|
11
|
+
date: 2021-07-20 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: avro_turf
|
|
@@ -62,16 +62,30 @@ dependencies:
|
|
|
62
62
|
name: sigurd
|
|
63
63
|
requirement: !ruby/object:Gem::Requirement
|
|
64
64
|
requirements:
|
|
65
|
-
- -
|
|
65
|
+
- - "~>"
|
|
66
66
|
- !ruby/object:Gem::Version
|
|
67
67
|
version: 0.0.1
|
|
68
68
|
type: :runtime
|
|
69
69
|
prerelease: false
|
|
70
70
|
version_requirements: !ruby/object:Gem::Requirement
|
|
71
71
|
requirements:
|
|
72
|
-
- -
|
|
72
|
+
- - "~>"
|
|
73
73
|
- !ruby/object:Gem::Version
|
|
74
74
|
version: 0.0.1
|
|
75
|
+
- !ruby/object:Gem::Dependency
|
|
76
|
+
name: fig_tree
|
|
77
|
+
requirement: !ruby/object:Gem::Requirement
|
|
78
|
+
requirements:
|
|
79
|
+
- - "~>"
|
|
80
|
+
- !ruby/object:Gem::Version
|
|
81
|
+
version: 0.0.2
|
|
82
|
+
type: :runtime
|
|
83
|
+
prerelease: false
|
|
84
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
85
|
+
requirements:
|
|
86
|
+
- - "~>"
|
|
87
|
+
- !ruby/object:Gem::Version
|
|
88
|
+
version: 0.0.2
|
|
75
89
|
- !ruby/object:Gem::Dependency
|
|
76
90
|
name: activerecord-import
|
|
77
91
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -370,7 +384,6 @@ files:
|
|
|
370
384
|
- lib/deimos/backends/kafka_async.rb
|
|
371
385
|
- lib/deimos/backends/test.rb
|
|
372
386
|
- lib/deimos/batch_consumer.rb
|
|
373
|
-
- lib/deimos/config/configurable.rb
|
|
374
387
|
- lib/deimos/config/configuration.rb
|
|
375
388
|
- lib/deimos/config/phobos_config.rb
|
|
376
389
|
- lib/deimos/consume/batch_consumption.rb
|
|
@@ -386,7 +399,6 @@ files:
|
|
|
386
399
|
- lib/deimos/metrics/provider.rb
|
|
387
400
|
- lib/deimos/monkey_patches/phobos_cli.rb
|
|
388
401
|
- lib/deimos/monkey_patches/phobos_producer.rb
|
|
389
|
-
- lib/deimos/monkey_patches/ruby_kafka_heartbeat.rb
|
|
390
402
|
- lib/deimos/poll_info.rb
|
|
391
403
|
- lib/deimos/producer.rb
|
|
392
404
|
- lib/deimos/railtie.rb
|
|
@@ -429,7 +441,6 @@ files:
|
|
|
429
441
|
- spec/backends/kafka_async_spec.rb
|
|
430
442
|
- spec/backends/kafka_spec.rb
|
|
431
443
|
- spec/batch_consumer_spec.rb
|
|
432
|
-
- spec/config/configurable_spec.rb
|
|
433
444
|
- spec/config/configuration_spec.rb
|
|
434
445
|
- spec/consumer_spec.rb
|
|
435
446
|
- spec/deimos_spec.rb
|
|
@@ -513,7 +524,6 @@ test_files:
|
|
|
513
524
|
- spec/backends/kafka_async_spec.rb
|
|
514
525
|
- spec/backends/kafka_spec.rb
|
|
515
526
|
- spec/batch_consumer_spec.rb
|
|
516
|
-
- spec/config/configurable_spec.rb
|
|
517
527
|
- spec/config/configuration_spec.rb
|
|
518
528
|
- spec/consumer_spec.rb
|
|
519
529
|
- spec/deimos_spec.rb
|
|
@@ -1,278 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
require 'active_support/concern'
|
|
4
|
-
require 'active_support/callbacks'
|
|
5
|
-
|
|
6
|
-
module Deimos
|
|
7
|
-
# Module to allow configuration. Loosely based off of the dry-configuration
|
|
8
|
-
# gem but with several advantages:
|
|
9
|
-
# - Works with Ruby 2.3.
|
|
10
|
-
# - More succinct syntax using method_missing so you do not need to write
|
|
11
|
-
# "config.whatever" and can just write "whatever".
|
|
12
|
-
# - Use nested blocks:
|
|
13
|
-
# Deimos.configure do
|
|
14
|
-
# config.kafka.ssl do
|
|
15
|
-
# enabled true
|
|
16
|
-
# ca_cert_file 'my_file'
|
|
17
|
-
# end
|
|
18
|
-
# config.kafka do
|
|
19
|
-
# ssl do
|
|
20
|
-
# enabled true
|
|
21
|
-
# end
|
|
22
|
-
# end
|
|
23
|
-
# end
|
|
24
|
-
# - Allows for arrays of configurations:
|
|
25
|
-
# Deimos.configure do |config|
|
|
26
|
-
# config.producer do
|
|
27
|
-
# class_name 'MyProducer'
|
|
28
|
-
# topic 'MyTopic'
|
|
29
|
-
# end
|
|
30
|
-
# end
|
|
31
|
-
# - Allows to call `configure` multiple times without crashing.
|
|
32
|
-
# - Allows to lazy-set default values by passing a proc as a default:
|
|
33
|
-
# Deimos.define_settings do |config|
|
|
34
|
-
# setting :my_val, default_proc: proc { MyDefault.calculated_value }
|
|
35
|
-
# end
|
|
36
|
-
# - Support for setting up and automatically calling deprecated configurations.
|
|
37
|
-
# - Support for configuration callbacks.
|
|
38
|
-
module Configurable
|
|
39
|
-
extend ActiveSupport::Concern
|
|
40
|
-
|
|
41
|
-
ConfigValue = Struct.new(:value, :default_value, :default_proc, :deprecation) do
|
|
42
|
-
|
|
43
|
-
# Reset value back to default.
|
|
44
|
-
def reset!
|
|
45
|
-
if self.value.is_a?(ConfigStruct)
|
|
46
|
-
self.value.reset!
|
|
47
|
-
else
|
|
48
|
-
self.value = self.default_value
|
|
49
|
-
end
|
|
50
|
-
end
|
|
51
|
-
|
|
52
|
-
# :nodoc:
|
|
53
|
-
def clone_and_reset
|
|
54
|
-
setting = ConfigValue.new(self.value, self.default_value,
|
|
55
|
-
self.default_proc, self.deprecation)
|
|
56
|
-
setting.reset!
|
|
57
|
-
setting
|
|
58
|
-
end
|
|
59
|
-
|
|
60
|
-
end
|
|
61
|
-
|
|
62
|
-
# Class that defines and keeps the configuration values.
|
|
63
|
-
class ConfigStruct
|
|
64
|
-
include ActiveSupport::Callbacks
|
|
65
|
-
|
|
66
|
-
define_callbacks :configure
|
|
67
|
-
|
|
68
|
-
# @param name [String]
|
|
69
|
-
def initialize(name)
|
|
70
|
-
@name = name
|
|
71
|
-
@settings = {}
|
|
72
|
-
@setting_objects = {}
|
|
73
|
-
@setting_templates = {}
|
|
74
|
-
end
|
|
75
|
-
|
|
76
|
-
# Reset config back to default values.
|
|
77
|
-
def reset!
|
|
78
|
-
@setting_objects = @setting_templates.map { |k, _| [k, []] }.to_h
|
|
79
|
-
@settings.values.each(&:reset!)
|
|
80
|
-
end
|
|
81
|
-
|
|
82
|
-
# Mark a configuration as deprecated and replaced with the new config.
|
|
83
|
-
# @param old_config [String]
|
|
84
|
-
# @param new_config [String]
|
|
85
|
-
def deprecate(old_config, new_config)
|
|
86
|
-
@settings[old_config.to_sym] ||= ConfigValue.new
|
|
87
|
-
@settings[old_config.to_sym].deprecation = new_config
|
|
88
|
-
end
|
|
89
|
-
|
|
90
|
-
# :nodoc:
|
|
91
|
-
def inspect
|
|
92
|
-
"#{@name}: #{@settings.inspect} #{@setting_objects.inspect}"
|
|
93
|
-
end
|
|
94
|
-
|
|
95
|
-
# @return [Hash]
|
|
96
|
-
def to_h
|
|
97
|
-
@settings.map { |k, v| [k, v.value] }.to_h
|
|
98
|
-
end
|
|
99
|
-
|
|
100
|
-
# :nodoc:
|
|
101
|
-
def clone_and_reset
|
|
102
|
-
new_config = self.clone
|
|
103
|
-
new_config.setting_objects = new_config.setting_objects.clone
|
|
104
|
-
new_config.settings = new_config.settings.map { |k, v| [k, v.clone_and_reset] }.to_h
|
|
105
|
-
new_config
|
|
106
|
-
end
|
|
107
|
-
|
|
108
|
-
# Define a setting template for an array of objects via a block:
|
|
109
|
-
# setting_object :producer do
|
|
110
|
-
# setting :topic
|
|
111
|
-
# setting :class_name
|
|
112
|
-
# end
|
|
113
|
-
# This will create the `producer` method to define these values as well
|
|
114
|
-
# as the `producer_objects` method to retrieve them.
|
|
115
|
-
# @param name [Symbol]
|
|
116
|
-
def setting_object(name, &block)
|
|
117
|
-
new_config = ConfigStruct.new("#{@name}.#{name}")
|
|
118
|
-
@setting_objects[name] = []
|
|
119
|
-
@setting_templates[name] = new_config
|
|
120
|
-
new_config.instance_eval(&block)
|
|
121
|
-
end
|
|
122
|
-
|
|
123
|
-
# Define a setting with the given name.
|
|
124
|
-
# @param name [Symbol]
|
|
125
|
-
# @param default_value [Object]
|
|
126
|
-
# @param default_proc [Proc]
|
|
127
|
-
def setting(name, default_value=nil, default_proc: nil, &block)
|
|
128
|
-
if block_given?
|
|
129
|
-
# Create a nested setting
|
|
130
|
-
setting_config = @settings[name]&.value || ConfigStruct.new("#{@name}.#{name}")
|
|
131
|
-
setting = ConfigValue.new
|
|
132
|
-
setting.value = setting_config
|
|
133
|
-
@settings[name] = setting
|
|
134
|
-
setting_config.instance_eval(&block)
|
|
135
|
-
else
|
|
136
|
-
setting = ConfigValue.new
|
|
137
|
-
setting.default_proc = default_proc
|
|
138
|
-
setting.default_value = default_value
|
|
139
|
-
setting.reset!
|
|
140
|
-
@settings[name] = setting
|
|
141
|
-
end
|
|
142
|
-
end
|
|
143
|
-
|
|
144
|
-
# :nodoc:
|
|
145
|
-
def respond_to_missing?(method, include_all=true)
|
|
146
|
-
method = method.to_s.sub(/=$/, '')
|
|
147
|
-
method.ends_with?('objects') ||
|
|
148
|
-
@setting_templates.key?(method.to_sym) ||
|
|
149
|
-
@settings.key?(method.to_sym) ||
|
|
150
|
-
super
|
|
151
|
-
end
|
|
152
|
-
|
|
153
|
-
# :nodoc:
|
|
154
|
-
def method_missing(method, *args, &block)
|
|
155
|
-
config_key = method.to_s.sub(/=$/, '').to_sym
|
|
156
|
-
|
|
157
|
-
# Return the list of setting objects with the given name
|
|
158
|
-
if config_key.to_s.end_with?('objects')
|
|
159
|
-
return _setting_object_method(config_key)
|
|
160
|
-
end
|
|
161
|
-
|
|
162
|
-
# Define a new setting object with the given name
|
|
163
|
-
if @setting_templates.key?(config_key) && block_given?
|
|
164
|
-
return _new_setting_object_method(config_key, &block)
|
|
165
|
-
end
|
|
166
|
-
|
|
167
|
-
setting = @settings[config_key]
|
|
168
|
-
|
|
169
|
-
if setting&.deprecation
|
|
170
|
-
return _deprecated_config_method(method, *args)
|
|
171
|
-
end
|
|
172
|
-
|
|
173
|
-
return super unless setting
|
|
174
|
-
|
|
175
|
-
if block_given?
|
|
176
|
-
return _block_config_method(config_key, &block)
|
|
177
|
-
end
|
|
178
|
-
|
|
179
|
-
_default_config_method(config_key, *args)
|
|
180
|
-
end
|
|
181
|
-
|
|
182
|
-
protected
|
|
183
|
-
|
|
184
|
-
# Only for the clone method
|
|
185
|
-
attr_accessor :settings, :setting_objects
|
|
186
|
-
|
|
187
|
-
private
|
|
188
|
-
|
|
189
|
-
def _deprecated_config_method(method, *args)
|
|
190
|
-
config_key = method.to_s.sub(/=$/, '').to_sym
|
|
191
|
-
new_config = @settings[config_key].deprecation
|
|
192
|
-
equals = method.to_s.end_with?('=') ? '=' : ''
|
|
193
|
-
ActiveSupport::Deprecation.warn("config.#{config_key}#{equals} is deprecated - use config.#{new_config}#{equals}")
|
|
194
|
-
obj = self
|
|
195
|
-
messages = new_config.split('.')
|
|
196
|
-
messages[0..-2].each do |message|
|
|
197
|
-
obj = obj.send(message)
|
|
198
|
-
end
|
|
199
|
-
if args.length.positive?
|
|
200
|
-
obj.send(messages[-1], args[0])
|
|
201
|
-
else
|
|
202
|
-
obj.send(messages[-1])
|
|
203
|
-
end
|
|
204
|
-
end
|
|
205
|
-
|
|
206
|
-
# Get or set a value.
|
|
207
|
-
def _default_config_method(config_key, *args)
|
|
208
|
-
if args.length.positive?
|
|
209
|
-
# Set the value
|
|
210
|
-
@settings[config_key].value = args[0]
|
|
211
|
-
else
|
|
212
|
-
# Get the value
|
|
213
|
-
setting = @settings[config_key]
|
|
214
|
-
if setting.default_proc && setting.value.nil?
|
|
215
|
-
setting.value = setting.default_proc.call
|
|
216
|
-
end
|
|
217
|
-
setting.value
|
|
218
|
-
end
|
|
219
|
-
end
|
|
220
|
-
|
|
221
|
-
# Define a new setting object and use the passed block to define values.
|
|
222
|
-
def _new_setting_object_method(config_key, &block)
|
|
223
|
-
new_config = @setting_templates[config_key].clone_and_reset
|
|
224
|
-
new_config.instance_eval(&block)
|
|
225
|
-
@setting_objects[config_key] << new_config
|
|
226
|
-
end
|
|
227
|
-
|
|
228
|
-
# Return a setting object.
|
|
229
|
-
def _setting_object_method(config_key)
|
|
230
|
-
key = config_key.to_s.sub(/_objects$/, '').to_sym
|
|
231
|
-
@setting_objects[key]
|
|
232
|
-
end
|
|
233
|
-
|
|
234
|
-
# Define new values inside a block.
|
|
235
|
-
def _block_config_method(config_key, &block)
|
|
236
|
-
unless @settings[config_key].value.is_a?(ConfigStruct)
|
|
237
|
-
raise "Block called for #{config_key} but it is not a nested config!"
|
|
238
|
-
end
|
|
239
|
-
|
|
240
|
-
@settings[config_key].value.instance_eval(&block)
|
|
241
|
-
end
|
|
242
|
-
end
|
|
243
|
-
|
|
244
|
-
# :nodoc:
|
|
245
|
-
module ClassMethods
|
|
246
|
-
# Define and redefine settings.
|
|
247
|
-
def define_settings(&block)
|
|
248
|
-
config.instance_eval(&block)
|
|
249
|
-
end
|
|
250
|
-
|
|
251
|
-
# Configure the settings with values.
|
|
252
|
-
def configure(&block)
|
|
253
|
-
if defined?(Rake) && defined?(Rake.application)
|
|
254
|
-
tasks = Rake.application.top_level_tasks
|
|
255
|
-
if tasks.any? { |t| %w(assets webpacker yarn).include?(t.split(':').first) }
|
|
256
|
-
puts 'Skipping Deimos configuration since we are in JS/CSS compilation'
|
|
257
|
-
return
|
|
258
|
-
end
|
|
259
|
-
end
|
|
260
|
-
config.run_callbacks(:configure) do
|
|
261
|
-
config.instance_eval(&block)
|
|
262
|
-
end
|
|
263
|
-
end
|
|
264
|
-
|
|
265
|
-
# @return [ConfigStruct]
|
|
266
|
-
def config
|
|
267
|
-
@config ||= ConfigStruct.new('config')
|
|
268
|
-
end
|
|
269
|
-
|
|
270
|
-
# Pass a block to run after configuration is done.
|
|
271
|
-
def after_configure(&block)
|
|
272
|
-
mod = self
|
|
273
|
-
config.class.set_callback(:configure, :after,
|
|
274
|
-
proc { mod.instance_eval(&block) })
|
|
275
|
-
end
|
|
276
|
-
end
|
|
277
|
-
end
|
|
278
|
-
end
|
|
@@ -1,85 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module Kafka
|
|
4
|
-
class Heartbeat
|
|
5
|
-
def initialize(group:, interval:, instrumenter:)
|
|
6
|
-
@group = group
|
|
7
|
-
@interval = interval
|
|
8
|
-
@last_heartbeat = Time.now
|
|
9
|
-
@instrumenter = instrumenter
|
|
10
|
-
end
|
|
11
|
-
|
|
12
|
-
def trigger!
|
|
13
|
-
@instrumenter.instrument('heartbeat.consumer',
|
|
14
|
-
group_id: @group.group_id,
|
|
15
|
-
topic_partitions: @group.assigned_partitions) do
|
|
16
|
-
@group.heartbeat
|
|
17
|
-
@last_heartbeat = Time.now
|
|
18
|
-
end
|
|
19
|
-
end
|
|
20
|
-
end
|
|
21
|
-
|
|
22
|
-
class Client
|
|
23
|
-
def consumer(
|
|
24
|
-
group_id:,
|
|
25
|
-
session_timeout: 30,
|
|
26
|
-
offset_commit_interval: 10,
|
|
27
|
-
offset_commit_threshold: 0,
|
|
28
|
-
heartbeat_interval: 10,
|
|
29
|
-
offset_retention_time: nil,
|
|
30
|
-
fetcher_max_queue_size: 100
|
|
31
|
-
)
|
|
32
|
-
cluster = initialize_cluster
|
|
33
|
-
|
|
34
|
-
instrumenter = DecoratingInstrumenter.new(@instrumenter,
|
|
35
|
-
group_id: group_id)
|
|
36
|
-
|
|
37
|
-
# The Kafka protocol expects the retention time to be in ms.
|
|
38
|
-
retention_time = (offset_retention_time && offset_retention_time * 1_000) || -1
|
|
39
|
-
|
|
40
|
-
group = ConsumerGroup.new(
|
|
41
|
-
cluster: cluster,
|
|
42
|
-
logger: @logger,
|
|
43
|
-
group_id: group_id,
|
|
44
|
-
session_timeout: session_timeout,
|
|
45
|
-
retention_time: retention_time,
|
|
46
|
-
instrumenter: instrumenter
|
|
47
|
-
)
|
|
48
|
-
|
|
49
|
-
fetcher = Fetcher.new(
|
|
50
|
-
cluster: initialize_cluster,
|
|
51
|
-
group: group,
|
|
52
|
-
logger: @logger,
|
|
53
|
-
instrumenter: instrumenter,
|
|
54
|
-
max_queue_size: fetcher_max_queue_size
|
|
55
|
-
)
|
|
56
|
-
|
|
57
|
-
offset_manager = OffsetManager.new(
|
|
58
|
-
cluster: cluster,
|
|
59
|
-
group: group,
|
|
60
|
-
fetcher: fetcher,
|
|
61
|
-
logger: @logger,
|
|
62
|
-
commit_interval: offset_commit_interval,
|
|
63
|
-
commit_threshold: offset_commit_threshold,
|
|
64
|
-
offset_retention_time: offset_retention_time
|
|
65
|
-
)
|
|
66
|
-
|
|
67
|
-
heartbeat = Heartbeat.new(
|
|
68
|
-
group: group,
|
|
69
|
-
interval: heartbeat_interval,
|
|
70
|
-
instrumenter: instrumenter
|
|
71
|
-
)
|
|
72
|
-
|
|
73
|
-
Consumer.new(
|
|
74
|
-
cluster: cluster,
|
|
75
|
-
logger: @logger,
|
|
76
|
-
instrumenter: instrumenter,
|
|
77
|
-
group: group,
|
|
78
|
-
offset_manager: offset_manager,
|
|
79
|
-
fetcher: fetcher,
|
|
80
|
-
session_timeout: session_timeout,
|
|
81
|
-
heartbeat: heartbeat
|
|
82
|
-
)
|
|
83
|
-
end
|
|
84
|
-
end
|
|
85
|
-
end
|
|
@@ -1,136 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
# :nodoc:
|
|
4
|
-
class MyConfig
|
|
5
|
-
include Deimos::Configurable
|
|
6
|
-
|
|
7
|
-
define_settings do
|
|
8
|
-
setting :set1
|
|
9
|
-
setting :set2, 'hi mom'
|
|
10
|
-
setting :group do
|
|
11
|
-
setting :set3, default_proc: proc { false }
|
|
12
|
-
setting :set5, (proc { 5 })
|
|
13
|
-
end
|
|
14
|
-
|
|
15
|
-
setting_object :listy do
|
|
16
|
-
setting :list1, 10
|
|
17
|
-
setting :list2, 5
|
|
18
|
-
end
|
|
19
|
-
end
|
|
20
|
-
end
|
|
21
|
-
|
|
22
|
-
describe Deimos::Configurable do
|
|
23
|
-
it 'should configure correctly with default values' do
|
|
24
|
-
expect(MyConfig.config.set1).to be_nil
|
|
25
|
-
expect(MyConfig.config.set2).to eq('hi mom')
|
|
26
|
-
expect(MyConfig.config.group.set3).to eq(false)
|
|
27
|
-
expect(MyConfig.config.listy_objects).to be_empty
|
|
28
|
-
expect { MyConfig.config.blah }.to raise_error(NameError)
|
|
29
|
-
expect { MyConfig.config.group.set4 }.to raise_error(NameError)
|
|
30
|
-
end
|
|
31
|
-
|
|
32
|
-
it 'should not call the proc until it has to' do
|
|
33
|
-
num_calls = 0
|
|
34
|
-
value_proc = proc do
|
|
35
|
-
num_calls += 1
|
|
36
|
-
num_calls
|
|
37
|
-
end
|
|
38
|
-
MyConfig.define_settings do
|
|
39
|
-
setting :set_with_proc, default_proc: value_proc
|
|
40
|
-
end
|
|
41
|
-
expect(num_calls).to eq(0)
|
|
42
|
-
expect(MyConfig.config.set_with_proc).to eq(1)
|
|
43
|
-
# calling twice should not call the proc again
|
|
44
|
-
expect(MyConfig.config.set_with_proc).to eq(1)
|
|
45
|
-
expect(num_calls).to eq(1)
|
|
46
|
-
end
|
|
47
|
-
|
|
48
|
-
it "should raise error when setting configs that don't exist" do
|
|
49
|
-
expect { MyConfig.configure { set15 'some_value' } }.to raise_error(NameError)
|
|
50
|
-
end
|
|
51
|
-
|
|
52
|
-
it 'should add values' do
|
|
53
|
-
MyConfig.configure do |config|
|
|
54
|
-
config.set1 = 5 # config.x syntax
|
|
55
|
-
set2 nil # method_missing syntax
|
|
56
|
-
config.group.set3 = true
|
|
57
|
-
end
|
|
58
|
-
|
|
59
|
-
# second configure should not blow anything away
|
|
60
|
-
MyConfig.configure do
|
|
61
|
-
listy do
|
|
62
|
-
list1 0
|
|
63
|
-
list2 1
|
|
64
|
-
end
|
|
65
|
-
listy do
|
|
66
|
-
list1 100
|
|
67
|
-
list2 200
|
|
68
|
-
end
|
|
69
|
-
end
|
|
70
|
-
|
|
71
|
-
expect(MyConfig.config.set1).to eq(5)
|
|
72
|
-
expect(MyConfig.config.set2).to be_nil
|
|
73
|
-
expect(MyConfig.config.group.set3).to eq(true)
|
|
74
|
-
expect(MyConfig.config.listy_objects.map(&:to_h)).
|
|
75
|
-
to eq([
|
|
76
|
-
{ list1: 0, list2: 1 },
|
|
77
|
-
{ list1: 100, list2: 200 }
|
|
78
|
-
])
|
|
79
|
-
|
|
80
|
-
# test reset!
|
|
81
|
-
MyConfig.config.reset!
|
|
82
|
-
expect(MyConfig.config.set1).to be_nil
|
|
83
|
-
expect(MyConfig.config.set2).to eq('hi mom')
|
|
84
|
-
expect(MyConfig.config.group.set3).to eq(false)
|
|
85
|
-
expect(MyConfig.config.listy_objects).to be_empty
|
|
86
|
-
end
|
|
87
|
-
|
|
88
|
-
it 'should add with block syntax' do
|
|
89
|
-
MyConfig.configure do
|
|
90
|
-
group do
|
|
91
|
-
set5(proc { 10 })
|
|
92
|
-
end
|
|
93
|
-
end
|
|
94
|
-
expect(MyConfig.config.group.set5.call).to eq(10)
|
|
95
|
-
end
|
|
96
|
-
|
|
97
|
-
it 'should add or redefine settings' do
|
|
98
|
-
MyConfig.define_settings do
|
|
99
|
-
setting :group do
|
|
100
|
-
setting :set6, 15
|
|
101
|
-
setting :set5, (proc { 15 })
|
|
102
|
-
end
|
|
103
|
-
setting_object :notey do
|
|
104
|
-
setting :note_title, 'some-title'
|
|
105
|
-
end
|
|
106
|
-
end
|
|
107
|
-
|
|
108
|
-
expect(MyConfig.config.group.set6).to eq(15)
|
|
109
|
-
expect(MyConfig.config.group.set5.call).to eq(15)
|
|
110
|
-
expect(MyConfig.config.listy_objects).to be_empty
|
|
111
|
-
expect(MyConfig.config.notey_objects).to be_empty
|
|
112
|
-
|
|
113
|
-
MyConfig.configure do
|
|
114
|
-
notey do
|
|
115
|
-
note_title 'hi mom'
|
|
116
|
-
end
|
|
117
|
-
listy do
|
|
118
|
-
list1 0
|
|
119
|
-
end
|
|
120
|
-
end
|
|
121
|
-
expect(MyConfig.config.notey_objects.size).to eq(1)
|
|
122
|
-
expect(MyConfig.config.notey_objects.first.note_title).to eq('hi mom')
|
|
123
|
-
expect(MyConfig.config.listy_objects.size).to eq(1)
|
|
124
|
-
expect(MyConfig.config.listy_objects.first.list1).to eq(0)
|
|
125
|
-
|
|
126
|
-
# This should not remove any keys
|
|
127
|
-
MyConfig.define_settings do
|
|
128
|
-
setting :group do
|
|
129
|
-
setting :set6, 20
|
|
130
|
-
end
|
|
131
|
-
end
|
|
132
|
-
expect(MyConfig.config.group.set6).to eq(20)
|
|
133
|
-
expect(MyConfig.config.group.set5.call).to eq(15)
|
|
134
|
-
end
|
|
135
|
-
|
|
136
|
-
end
|