deimos-ruby 1.9.2 → 1.10.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +10 -0
- data/Gemfile.lock +74 -71
- data/deimos-ruby.gemspec +2 -1
- data/docs/ARCHITECTURE.md +1 -5
- data/docs/CONFIGURATION.md +1 -0
- data/lib/deimos/active_record_consume/message_consumption.rb +5 -0
- data/lib/deimos/config/configuration.rb +13 -13
- data/lib/deimos/version.rb +1 -1
- data/spec/config/configuration_spec.rb +17 -17
- metadata +18 -7
- data/lib/deimos/config/configurable.rb +0 -278
- data/spec/config/configurable_spec.rb +0 -136
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a614febc556860a51e870e89beb90fde63201fffae3ba7ca78c53d423a65a7b6
|
4
|
+
data.tar.gz: e4676b33fbaf9c105ffd12b89a0090a86b8befd7cf0e09b39078a70e213f72dc
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 0d418e1d0b27008392da829980c6bdf754111a1f103965dce6882f97d3a02f0220d58756f6f1f0682df12944eac1a012516b0a3612c257342509b6e0b4c3da57
|
7
|
+
data.tar.gz: 1be3de2310297dc6b75a8f98b6c3f84483c6c401d6395dc6752cc57c342f8935911ba1ad11e8eb69e6bfcb19720e8540985c42976033401c39e137a3ab8fbc40
|
data/CHANGELOG.md
CHANGED
@@ -7,6 +7,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
7
7
|
|
8
8
|
## UNRELEASED
|
9
9
|
|
10
|
+
## 1.10.0 - 2021-03-22
|
11
|
+
|
12
|
+
- ### Roadmap :car:
|
13
|
+
|
14
|
+
- Extracted the configuration piece into a separate gem, [fig_tree](https://www.github.com/flipp-oss/fig_tree).
|
15
|
+
- Added a `save_record` method to ActiveRecordConsumer in case calling code wants to work with the record before saving.
|
16
|
+
|
17
|
+
- ### Fixes :wrench:
|
18
|
+
|
19
|
+
- Fixed a regression where the default values for consumer / Phobos listener configs were not correct (they were all nil). This is technically a breaking change, but it puts the configs back the way they were at version 1.4 and matches the documentation.
|
10
20
|
|
11
21
|
## 1.9.2 - 2021-01-29
|
12
22
|
|
data/Gemfile.lock
CHANGED
@@ -1,71 +1,72 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
deimos-ruby (1.9.
|
4
|
+
deimos-ruby (1.9.2)
|
5
5
|
avro_turf (~> 0.11)
|
6
|
+
fig_tree (~> 0.0.2)
|
6
7
|
phobos (>= 1.9, < 3.0)
|
7
8
|
ruby-kafka (< 2)
|
8
|
-
sigurd (
|
9
|
+
sigurd (~> 0.0.1)
|
9
10
|
|
10
11
|
GEM
|
11
12
|
remote: https://rubygems.org/
|
12
13
|
specs:
|
13
|
-
actioncable (6.1.
|
14
|
-
actionpack (= 6.1.
|
15
|
-
activesupport (= 6.1.
|
14
|
+
actioncable (6.1.3)
|
15
|
+
actionpack (= 6.1.3)
|
16
|
+
activesupport (= 6.1.3)
|
16
17
|
nio4r (~> 2.0)
|
17
18
|
websocket-driver (>= 0.6.1)
|
18
|
-
actionmailbox (6.1.
|
19
|
-
actionpack (= 6.1.
|
20
|
-
activejob (= 6.1.
|
21
|
-
activerecord (= 6.1.
|
22
|
-
activestorage (= 6.1.
|
23
|
-
activesupport (= 6.1.
|
19
|
+
actionmailbox (6.1.3)
|
20
|
+
actionpack (= 6.1.3)
|
21
|
+
activejob (= 6.1.3)
|
22
|
+
activerecord (= 6.1.3)
|
23
|
+
activestorage (= 6.1.3)
|
24
|
+
activesupport (= 6.1.3)
|
24
25
|
mail (>= 2.7.1)
|
25
|
-
actionmailer (6.1.
|
26
|
-
actionpack (= 6.1.
|
27
|
-
actionview (= 6.1.
|
28
|
-
activejob (= 6.1.
|
29
|
-
activesupport (= 6.1.
|
26
|
+
actionmailer (6.1.3)
|
27
|
+
actionpack (= 6.1.3)
|
28
|
+
actionview (= 6.1.3)
|
29
|
+
activejob (= 6.1.3)
|
30
|
+
activesupport (= 6.1.3)
|
30
31
|
mail (~> 2.5, >= 2.5.4)
|
31
32
|
rails-dom-testing (~> 2.0)
|
32
|
-
actionpack (6.1.
|
33
|
-
actionview (= 6.1.
|
34
|
-
activesupport (= 6.1.
|
33
|
+
actionpack (6.1.3)
|
34
|
+
actionview (= 6.1.3)
|
35
|
+
activesupport (= 6.1.3)
|
35
36
|
rack (~> 2.0, >= 2.0.9)
|
36
37
|
rack-test (>= 0.6.3)
|
37
38
|
rails-dom-testing (~> 2.0)
|
38
39
|
rails-html-sanitizer (~> 1.0, >= 1.2.0)
|
39
|
-
actiontext (6.1.
|
40
|
-
actionpack (= 6.1.
|
41
|
-
activerecord (= 6.1.
|
42
|
-
activestorage (= 6.1.
|
43
|
-
activesupport (= 6.1.
|
40
|
+
actiontext (6.1.3)
|
41
|
+
actionpack (= 6.1.3)
|
42
|
+
activerecord (= 6.1.3)
|
43
|
+
activestorage (= 6.1.3)
|
44
|
+
activesupport (= 6.1.3)
|
44
45
|
nokogiri (>= 1.8.5)
|
45
|
-
actionview (6.1.
|
46
|
-
activesupport (= 6.1.
|
46
|
+
actionview (6.1.3)
|
47
|
+
activesupport (= 6.1.3)
|
47
48
|
builder (~> 3.1)
|
48
49
|
erubi (~> 1.4)
|
49
50
|
rails-dom-testing (~> 2.0)
|
50
51
|
rails-html-sanitizer (~> 1.1, >= 1.2.0)
|
51
|
-
activejob (6.1.
|
52
|
-
activesupport (= 6.1.
|
52
|
+
activejob (6.1.3)
|
53
|
+
activesupport (= 6.1.3)
|
53
54
|
globalid (>= 0.3.6)
|
54
|
-
activemodel (6.1.
|
55
|
-
activesupport (= 6.1.
|
56
|
-
activerecord (6.1.
|
57
|
-
activemodel (= 6.1.
|
58
|
-
activesupport (= 6.1.
|
59
|
-
activerecord-import (1.0.
|
55
|
+
activemodel (6.1.3)
|
56
|
+
activesupport (= 6.1.3)
|
57
|
+
activerecord (6.1.3)
|
58
|
+
activemodel (= 6.1.3)
|
59
|
+
activesupport (= 6.1.3)
|
60
|
+
activerecord-import (1.0.8)
|
60
61
|
activerecord (>= 3.2)
|
61
|
-
activestorage (6.1.
|
62
|
-
actionpack (= 6.1.
|
63
|
-
activejob (= 6.1.
|
64
|
-
activerecord (= 6.1.
|
65
|
-
activesupport (= 6.1.
|
62
|
+
activestorage (6.1.3)
|
63
|
+
actionpack (= 6.1.3)
|
64
|
+
activejob (= 6.1.3)
|
65
|
+
activerecord (= 6.1.3)
|
66
|
+
activesupport (= 6.1.3)
|
66
67
|
marcel (~> 0.3.1)
|
67
68
|
mimemagic (~> 0.3.2)
|
68
|
-
activesupport (6.1.
|
69
|
+
activesupport (6.1.3)
|
69
70
|
concurrent-ruby (~> 1.0, >= 1.0.2)
|
70
71
|
i18n (>= 1.6, < 2)
|
71
72
|
minitest (>= 5.1)
|
@@ -83,17 +84,19 @@ GEM
|
|
83
84
|
concurrent-ruby-ext (1.1.8)
|
84
85
|
concurrent-ruby (= 1.1.8)
|
85
86
|
crass (1.0.6)
|
86
|
-
database_cleaner (1.
|
87
|
-
ddtrace (0.
|
87
|
+
database_cleaner (1.99.0)
|
88
|
+
ddtrace (0.46.0)
|
88
89
|
msgpack
|
89
90
|
diff-lcs (1.4.4)
|
90
91
|
digest-crc (0.6.3)
|
91
92
|
rake (>= 12.0.0, < 14.0.0)
|
92
|
-
dogstatsd-ruby (4.8.
|
93
|
+
dogstatsd-ruby (4.8.3)
|
93
94
|
erubi (1.10.0)
|
94
|
-
excon (0.
|
95
|
+
excon (0.79.0)
|
95
96
|
exponential-backoff (0.0.4)
|
96
|
-
ffi (1.
|
97
|
+
ffi (1.15.0)
|
98
|
+
fig_tree (0.0.2)
|
99
|
+
activesupport (>= 3.0.0)
|
97
100
|
formatador (0.2.5)
|
98
101
|
globalid (0.4.2)
|
99
102
|
activesupport (>= 4.2.0)
|
@@ -114,7 +117,7 @@ GEM
|
|
114
117
|
guard-rubocop (1.4.0)
|
115
118
|
guard (~> 2.0)
|
116
119
|
rubocop (< 2.0)
|
117
|
-
i18n (1.8.
|
120
|
+
i18n (1.8.9)
|
118
121
|
concurrent-ruby (~> 1.0)
|
119
122
|
listen (3.4.1)
|
120
123
|
rb-fsevent (~> 0.10, >= 0.10.3)
|
@@ -135,12 +138,12 @@ GEM
|
|
135
138
|
mimemagic (0.3.5)
|
136
139
|
mini_mime (1.0.2)
|
137
140
|
mini_portile2 (2.5.0)
|
138
|
-
minitest (5.14.
|
139
|
-
msgpack (1.4.
|
141
|
+
minitest (5.14.4)
|
142
|
+
msgpack (1.4.2)
|
140
143
|
multi_json (1.15.0)
|
141
144
|
mysql2 (0.5.3)
|
142
145
|
nenv (0.3.0)
|
143
|
-
nio4r (2.5.
|
146
|
+
nio4r (2.5.7)
|
144
147
|
nokogiri (1.11.1)
|
145
148
|
mini_portile2 (~> 2.5.0)
|
146
149
|
racc (~> 1.4)
|
@@ -159,36 +162,36 @@ GEM
|
|
159
162
|
logging
|
160
163
|
ruby-kafka
|
161
164
|
thor
|
162
|
-
pry (0.
|
165
|
+
pry (0.14.0)
|
163
166
|
coderay (~> 1.1)
|
164
167
|
method_source (~> 1.0)
|
165
168
|
racc (1.5.2)
|
166
169
|
rack (2.2.3)
|
167
170
|
rack-test (1.1.0)
|
168
171
|
rack (>= 1.0, < 3)
|
169
|
-
rails (6.1.
|
170
|
-
actioncable (= 6.1.
|
171
|
-
actionmailbox (= 6.1.
|
172
|
-
actionmailer (= 6.1.
|
173
|
-
actionpack (= 6.1.
|
174
|
-
actiontext (= 6.1.
|
175
|
-
actionview (= 6.1.
|
176
|
-
activejob (= 6.1.
|
177
|
-
activemodel (= 6.1.
|
178
|
-
activerecord (= 6.1.
|
179
|
-
activestorage (= 6.1.
|
180
|
-
activesupport (= 6.1.
|
172
|
+
rails (6.1.3)
|
173
|
+
actioncable (= 6.1.3)
|
174
|
+
actionmailbox (= 6.1.3)
|
175
|
+
actionmailer (= 6.1.3)
|
176
|
+
actionpack (= 6.1.3)
|
177
|
+
actiontext (= 6.1.3)
|
178
|
+
actionview (= 6.1.3)
|
179
|
+
activejob (= 6.1.3)
|
180
|
+
activemodel (= 6.1.3)
|
181
|
+
activerecord (= 6.1.3)
|
182
|
+
activestorage (= 6.1.3)
|
183
|
+
activesupport (= 6.1.3)
|
181
184
|
bundler (>= 1.15.0)
|
182
|
-
railties (= 6.1.
|
185
|
+
railties (= 6.1.3)
|
183
186
|
sprockets-rails (>= 2.0.0)
|
184
187
|
rails-dom-testing (2.0.3)
|
185
188
|
activesupport (>= 4.2.0)
|
186
189
|
nokogiri (>= 1.6)
|
187
190
|
rails-html-sanitizer (1.3.0)
|
188
191
|
loofah (~> 2.3)
|
189
|
-
railties (6.1.
|
190
|
-
actionpack (= 6.1.
|
191
|
-
activesupport (= 6.1.
|
192
|
+
railties (6.1.3)
|
193
|
+
actionpack (= 6.1.3)
|
194
|
+
activesupport (= 6.1.3)
|
192
195
|
method_source
|
193
196
|
rake (>= 0.8.7)
|
194
197
|
thor (~> 1.0)
|
@@ -197,7 +200,7 @@ GEM
|
|
197
200
|
rb-fsevent (0.10.4)
|
198
201
|
rb-inotify (0.10.1)
|
199
202
|
ffi (~> 1.0)
|
200
|
-
regexp_parser (2.
|
203
|
+
regexp_parser (2.1.1)
|
201
204
|
rexml (3.2.4)
|
202
205
|
rspec (3.10.0)
|
203
206
|
rspec-core (~> 3.10.0)
|
@@ -208,10 +211,10 @@ GEM
|
|
208
211
|
rspec-expectations (3.10.1)
|
209
212
|
diff-lcs (>= 1.2.0, < 2.0)
|
210
213
|
rspec-support (~> 3.10.0)
|
211
|
-
rspec-mocks (3.10.
|
214
|
+
rspec-mocks (3.10.2)
|
212
215
|
diff-lcs (>= 1.2.0, < 2.0)
|
213
216
|
rspec-support (~> 3.10.0)
|
214
|
-
rspec-rails (4.
|
217
|
+
rspec-rails (4.1.1)
|
215
218
|
actionpack (>= 4.2)
|
216
219
|
activesupport (>= 4.2)
|
217
220
|
railties (>= 4.2)
|
@@ -219,7 +222,7 @@ GEM
|
|
219
222
|
rspec-expectations (~> 3.10)
|
220
223
|
rspec-mocks (~> 3.10)
|
221
224
|
rspec-support (~> 3.10)
|
222
|
-
rspec-support (3.10.
|
225
|
+
rspec-support (3.10.2)
|
223
226
|
rspec_junit_formatter (0.4.1)
|
224
227
|
rspec-core (>= 2, < 4, != 2.12.0)
|
225
228
|
rubocop (0.88.0)
|
@@ -239,7 +242,7 @@ GEM
|
|
239
242
|
digest-crc
|
240
243
|
ruby-progressbar (1.11.0)
|
241
244
|
shellany (0.0.1)
|
242
|
-
sigurd (0.0.
|
245
|
+
sigurd (0.0.3)
|
243
246
|
concurrent-ruby (~> 1)
|
244
247
|
exponential-backoff
|
245
248
|
sprockets (4.0.2)
|
data/deimos-ruby.gemspec
CHANGED
@@ -21,7 +21,8 @@ Gem::Specification.new do |spec|
|
|
21
21
|
spec.add_runtime_dependency('avro_turf', '~> 0.11')
|
22
22
|
spec.add_runtime_dependency('phobos', '>= 1.9', '< 3.0')
|
23
23
|
spec.add_runtime_dependency('ruby-kafka', '< 2')
|
24
|
-
spec.add_runtime_dependency('sigurd', '0.0.1')
|
24
|
+
spec.add_runtime_dependency('sigurd', '~> 0.0.1')
|
25
|
+
spec.add_runtime_dependency('fig_tree', '~> 0.0.2')
|
25
26
|
|
26
27
|
spec.add_development_dependency('activerecord-import')
|
27
28
|
spec.add_development_dependency('avro', '~> 1.9')
|
data/docs/ARCHITECTURE.md
CHANGED
@@ -74,11 +74,7 @@ must define methods such as:
|
|
74
74
|
|
75
75
|
### Configuration
|
76
76
|
|
77
|
-
Deimos
|
78
|
-
to provide a very succinct but powerful configuration format (including
|
79
|
-
default values, procs, print out as hash, reset, etc.). It also
|
80
|
-
allows for multiple blocks to define different objects of the same time
|
81
|
-
(like producers, consumers, pollers etc.).
|
77
|
+
Deimos uses the [https://www.github.com/flipp_oss/fig_tree](fig_tree) gem for configuration.
|
82
78
|
|
83
79
|
The configuration definition for Deimos is in `config/configuration.rb`. In
|
84
80
|
addition, there are methods in `config/phobos_config.rb` which translate to/from
|
data/docs/CONFIGURATION.md
CHANGED
@@ -91,6 +91,7 @@ delivery|`:batch`|The delivery mode for the consumer. Possible values: `:message
|
|
91
91
|
session_timeout|300|Number of seconds after which, if a client hasn't contacted the Kafka cluster, it will be kicked out of the group.
|
92
92
|
offset_commit_interval|10|Interval between offset commits, in seconds.
|
93
93
|
offset_commit_threshold|0|Number of messages that can be processed before their offsets are committed. If zero, offset commits are not triggered by message processing
|
94
|
+
offset_retention_time|nil|The time period that committed offsets will be retained, in seconds. Defaults to the broker setting.
|
94
95
|
heartbeat_interval|10|Interval between heartbeats; must be less than the session window.
|
95
96
|
backoff|`(1000..60_000)`|Range representing the minimum and maximum number of milliseconds to back off after a consumer error.
|
96
97
|
|
@@ -58,6 +58,11 @@ module Deimos
|
|
58
58
|
attrs.each do |k, v|
|
59
59
|
record.send("#{k}=", v)
|
60
60
|
end
|
61
|
+
save_record(record)
|
62
|
+
end
|
63
|
+
|
64
|
+
# @param record [ActiveRecord::Base]
|
65
|
+
def save_record(record)
|
61
66
|
record.created_at ||= Time.zone.now if record.respond_to?(:created_at)
|
62
67
|
record.updated_at = Time.zone.now if record.respond_to?(:updated_at)
|
63
68
|
record.save!
|
@@ -1,17 +1,17 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require 'fig_tree'
|
3
4
|
require_relative 'phobos_config'
|
4
|
-
require_relative 'configurable'
|
5
5
|
require_relative '../metrics/mock'
|
6
6
|
require_relative '../tracing/mock'
|
7
7
|
require 'active_support/core_ext/numeric'
|
8
8
|
|
9
9
|
# :nodoc:
|
10
10
|
module Deimos
|
11
|
-
include
|
11
|
+
include FigTree
|
12
12
|
|
13
13
|
# :nodoc:
|
14
|
-
class
|
14
|
+
class FigTree::ConfigStruct
|
15
15
|
include Deimos::PhobosConfig
|
16
16
|
end
|
17
17
|
|
@@ -60,7 +60,7 @@ module Deimos
|
|
60
60
|
end
|
61
61
|
end
|
62
62
|
|
63
|
-
# @param kafka_config [
|
63
|
+
# @param kafka_config [FigTree::ConfigStruct]
|
64
64
|
def self.configure_producer_or_consumer(kafka_config)
|
65
65
|
klass = kafka_config.class_name.constantize
|
66
66
|
klass.class_eval do
|
@@ -327,19 +327,19 @@ module Deimos
|
|
327
327
|
# These are the phobos "listener" configs. See CONFIGURATION.md for more
|
328
328
|
# info.
|
329
329
|
setting :group_id
|
330
|
-
setting :max_concurrency
|
331
|
-
setting :start_from_beginning
|
330
|
+
setting :max_concurrency, 1
|
331
|
+
setting :start_from_beginning, true
|
332
332
|
setting :max_bytes_per_partition, 500.kilobytes
|
333
|
-
setting :min_bytes
|
334
|
-
setting :max_wait_time
|
333
|
+
setting :min_bytes, 1
|
334
|
+
setting :max_wait_time, 5
|
335
335
|
setting :force_encoding
|
336
|
-
setting :delivery
|
336
|
+
setting :delivery, :batch
|
337
337
|
setting :backoff
|
338
|
-
setting :session_timeout
|
339
|
-
setting :offset_commit_interval
|
340
|
-
setting :offset_commit_threshold
|
338
|
+
setting :session_timeout, 300
|
339
|
+
setting :offset_commit_interval, 10
|
340
|
+
setting :offset_commit_threshold, 0
|
341
341
|
setting :offset_retention_time
|
342
|
-
setting :heartbeat_interval
|
342
|
+
setting :heartbeat_interval, 10
|
343
343
|
end
|
344
344
|
|
345
345
|
setting_object :db_poller do
|
data/lib/deimos/version.rb
CHANGED
@@ -74,34 +74,34 @@ describe Deimos, 'configuration' do
|
|
74
74
|
{
|
75
75
|
topic: 'my_consume_topic',
|
76
76
|
group_id: 'my_group_id',
|
77
|
-
max_concurrency:
|
78
|
-
start_from_beginning:
|
77
|
+
max_concurrency: 1,
|
78
|
+
start_from_beginning: true,
|
79
79
|
max_bytes_per_partition: 524_288,
|
80
|
-
min_bytes:
|
81
|
-
max_wait_time:
|
80
|
+
min_bytes: 1,
|
81
|
+
max_wait_time: 5,
|
82
82
|
force_encoding: nil,
|
83
|
-
delivery:
|
84
|
-
session_timeout:
|
85
|
-
offset_commit_interval:
|
86
|
-
offset_commit_threshold:
|
83
|
+
delivery: 'batch',
|
84
|
+
session_timeout: 300,
|
85
|
+
offset_commit_interval: 10,
|
86
|
+
offset_commit_threshold: 0,
|
87
87
|
offset_retention_time: nil,
|
88
|
-
heartbeat_interval:
|
88
|
+
heartbeat_interval: 10,
|
89
89
|
handler: 'ConsumerTest::MyConsumer'
|
90
90
|
}, {
|
91
91
|
topic: 'my_batch_consume_topic',
|
92
92
|
group_id: 'my_batch_group_id',
|
93
|
-
max_concurrency:
|
94
|
-
start_from_beginning:
|
93
|
+
max_concurrency: 1,
|
94
|
+
start_from_beginning: true,
|
95
95
|
max_bytes_per_partition: 500.kilobytes,
|
96
|
-
min_bytes:
|
97
|
-
max_wait_time:
|
96
|
+
min_bytes: 1,
|
97
|
+
max_wait_time: 5,
|
98
98
|
force_encoding: nil,
|
99
99
|
delivery: 'inline_batch',
|
100
|
-
session_timeout:
|
101
|
-
offset_commit_interval:
|
102
|
-
offset_commit_threshold:
|
100
|
+
session_timeout: 300,
|
101
|
+
offset_commit_interval: 10,
|
102
|
+
offset_commit_threshold: 0,
|
103
103
|
offset_retention_time: nil,
|
104
|
-
heartbeat_interval:
|
104
|
+
heartbeat_interval: 10,
|
105
105
|
handler: 'ConsumerTest::MyBatchConsumer'
|
106
106
|
}
|
107
107
|
],
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: deimos-ruby
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.10.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Daniel Orner
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-
|
11
|
+
date: 2021-03-22 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: avro_turf
|
@@ -62,16 +62,30 @@ dependencies:
|
|
62
62
|
name: sigurd
|
63
63
|
requirement: !ruby/object:Gem::Requirement
|
64
64
|
requirements:
|
65
|
-
- -
|
65
|
+
- - "~>"
|
66
66
|
- !ruby/object:Gem::Version
|
67
67
|
version: 0.0.1
|
68
68
|
type: :runtime
|
69
69
|
prerelease: false
|
70
70
|
version_requirements: !ruby/object:Gem::Requirement
|
71
71
|
requirements:
|
72
|
-
- -
|
72
|
+
- - "~>"
|
73
73
|
- !ruby/object:Gem::Version
|
74
74
|
version: 0.0.1
|
75
|
+
- !ruby/object:Gem::Dependency
|
76
|
+
name: fig_tree
|
77
|
+
requirement: !ruby/object:Gem::Requirement
|
78
|
+
requirements:
|
79
|
+
- - "~>"
|
80
|
+
- !ruby/object:Gem::Version
|
81
|
+
version: 0.0.2
|
82
|
+
type: :runtime
|
83
|
+
prerelease: false
|
84
|
+
version_requirements: !ruby/object:Gem::Requirement
|
85
|
+
requirements:
|
86
|
+
- - "~>"
|
87
|
+
- !ruby/object:Gem::Version
|
88
|
+
version: 0.0.2
|
75
89
|
- !ruby/object:Gem::Dependency
|
76
90
|
name: activerecord-import
|
77
91
|
requirement: !ruby/object:Gem::Requirement
|
@@ -370,7 +384,6 @@ files:
|
|
370
384
|
- lib/deimos/backends/kafka_async.rb
|
371
385
|
- lib/deimos/backends/test.rb
|
372
386
|
- lib/deimos/batch_consumer.rb
|
373
|
-
- lib/deimos/config/configurable.rb
|
374
387
|
- lib/deimos/config/configuration.rb
|
375
388
|
- lib/deimos/config/phobos_config.rb
|
376
389
|
- lib/deimos/consume/batch_consumption.rb
|
@@ -428,7 +441,6 @@ files:
|
|
428
441
|
- spec/backends/kafka_async_spec.rb
|
429
442
|
- spec/backends/kafka_spec.rb
|
430
443
|
- spec/batch_consumer_spec.rb
|
431
|
-
- spec/config/configurable_spec.rb
|
432
444
|
- spec/config/configuration_spec.rb
|
433
445
|
- spec/consumer_spec.rb
|
434
446
|
- spec/deimos_spec.rb
|
@@ -512,7 +524,6 @@ test_files:
|
|
512
524
|
- spec/backends/kafka_async_spec.rb
|
513
525
|
- spec/backends/kafka_spec.rb
|
514
526
|
- spec/batch_consumer_spec.rb
|
515
|
-
- spec/config/configurable_spec.rb
|
516
527
|
- spec/config/configuration_spec.rb
|
517
528
|
- spec/consumer_spec.rb
|
518
529
|
- spec/deimos_spec.rb
|
@@ -1,278 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require 'active_support/concern'
|
4
|
-
require 'active_support/callbacks'
|
5
|
-
|
6
|
-
module Deimos
|
7
|
-
# Module to allow configuration. Loosely based off of the dry-configuration
|
8
|
-
# gem but with several advantages:
|
9
|
-
# - Works with Ruby 2.3.
|
10
|
-
# - More succinct syntax using method_missing so you do not need to write
|
11
|
-
# "config.whatever" and can just write "whatever".
|
12
|
-
# - Use nested blocks:
|
13
|
-
# Deimos.configure do
|
14
|
-
# config.kafka.ssl do
|
15
|
-
# enabled true
|
16
|
-
# ca_cert_file 'my_file'
|
17
|
-
# end
|
18
|
-
# config.kafka do
|
19
|
-
# ssl do
|
20
|
-
# enabled true
|
21
|
-
# end
|
22
|
-
# end
|
23
|
-
# end
|
24
|
-
# - Allows for arrays of configurations:
|
25
|
-
# Deimos.configure do |config|
|
26
|
-
# config.producer do
|
27
|
-
# class_name 'MyProducer'
|
28
|
-
# topic 'MyTopic'
|
29
|
-
# end
|
30
|
-
# end
|
31
|
-
# - Allows to call `configure` multiple times without crashing.
|
32
|
-
# - Allows to lazy-set default values by passing a proc as a default:
|
33
|
-
# Deimos.define_settings do |config|
|
34
|
-
# setting :my_val, default_proc: proc { MyDefault.calculated_value }
|
35
|
-
# end
|
36
|
-
# - Support for setting up and automatically calling deprecated configurations.
|
37
|
-
# - Support for configuration callbacks.
|
38
|
-
module Configurable
|
39
|
-
extend ActiveSupport::Concern
|
40
|
-
|
41
|
-
ConfigValue = Struct.new(:value, :default_value, :default_proc, :deprecation) do
|
42
|
-
|
43
|
-
# Reset value back to default.
|
44
|
-
def reset!
|
45
|
-
if self.value.is_a?(ConfigStruct)
|
46
|
-
self.value.reset!
|
47
|
-
else
|
48
|
-
self.value = self.default_value
|
49
|
-
end
|
50
|
-
end
|
51
|
-
|
52
|
-
# :nodoc:
|
53
|
-
def clone_and_reset
|
54
|
-
setting = ConfigValue.new(self.value, self.default_value,
|
55
|
-
self.default_proc, self.deprecation)
|
56
|
-
setting.reset!
|
57
|
-
setting
|
58
|
-
end
|
59
|
-
|
60
|
-
end
|
61
|
-
|
62
|
-
# Class that defines and keeps the configuration values.
|
63
|
-
class ConfigStruct
|
64
|
-
include ActiveSupport::Callbacks
|
65
|
-
|
66
|
-
define_callbacks :configure
|
67
|
-
|
68
|
-
# @param name [String]
|
69
|
-
def initialize(name)
|
70
|
-
@name = name
|
71
|
-
@settings = {}
|
72
|
-
@setting_objects = {}
|
73
|
-
@setting_templates = {}
|
74
|
-
end
|
75
|
-
|
76
|
-
# Reset config back to default values.
|
77
|
-
def reset!
|
78
|
-
@setting_objects = @setting_templates.map { |k, _| [k, []] }.to_h
|
79
|
-
@settings.values.each(&:reset!)
|
80
|
-
end
|
81
|
-
|
82
|
-
# Mark a configuration as deprecated and replaced with the new config.
|
83
|
-
# @param old_config [String]
|
84
|
-
# @param new_config [String]
|
85
|
-
def deprecate(old_config, new_config)
|
86
|
-
@settings[old_config.to_sym] ||= ConfigValue.new
|
87
|
-
@settings[old_config.to_sym].deprecation = new_config
|
88
|
-
end
|
89
|
-
|
90
|
-
# :nodoc:
|
91
|
-
def inspect
|
92
|
-
"#{@name}: #{@settings.inspect} #{@setting_objects.inspect}"
|
93
|
-
end
|
94
|
-
|
95
|
-
# @return [Hash]
|
96
|
-
def to_h
|
97
|
-
@settings.map { |k, v| [k, v.value] }.to_h
|
98
|
-
end
|
99
|
-
|
100
|
-
# :nodoc:
|
101
|
-
def clone_and_reset
|
102
|
-
new_config = self.clone
|
103
|
-
new_config.setting_objects = new_config.setting_objects.clone
|
104
|
-
new_config.settings = new_config.settings.map { |k, v| [k, v.clone_and_reset] }.to_h
|
105
|
-
new_config
|
106
|
-
end
|
107
|
-
|
108
|
-
# Define a setting template for an array of objects via a block:
|
109
|
-
# setting_object :producer do
|
110
|
-
# setting :topic
|
111
|
-
# setting :class_name
|
112
|
-
# end
|
113
|
-
# This will create the `producer` method to define these values as well
|
114
|
-
# as the `producer_objects` method to retrieve them.
|
115
|
-
# @param name [Symbol]
|
116
|
-
def setting_object(name, &block)
|
117
|
-
new_config = ConfigStruct.new("#{@name}.#{name}")
|
118
|
-
@setting_objects[name] = []
|
119
|
-
@setting_templates[name] = new_config
|
120
|
-
new_config.instance_eval(&block)
|
121
|
-
end
|
122
|
-
|
123
|
-
# Define a setting with the given name.
|
124
|
-
# @param name [Symbol]
|
125
|
-
# @param default_value [Object]
|
126
|
-
# @param default_proc [Proc]
|
127
|
-
def setting(name, default_value=nil, default_proc: nil, &block)
|
128
|
-
if block_given?
|
129
|
-
# Create a nested setting
|
130
|
-
setting_config = @settings[name]&.value || ConfigStruct.new("#{@name}.#{name}")
|
131
|
-
setting = ConfigValue.new
|
132
|
-
setting.value = setting_config
|
133
|
-
@settings[name] = setting
|
134
|
-
setting_config.instance_eval(&block)
|
135
|
-
else
|
136
|
-
setting = ConfigValue.new
|
137
|
-
setting.default_proc = default_proc
|
138
|
-
setting.default_value = default_value
|
139
|
-
setting.reset!
|
140
|
-
@settings[name] = setting
|
141
|
-
end
|
142
|
-
end
|
143
|
-
|
144
|
-
# :nodoc:
|
145
|
-
def respond_to_missing?(method, include_all=true)
|
146
|
-
method = method.to_s.sub(/=$/, '')
|
147
|
-
method.ends_with?('objects') ||
|
148
|
-
@setting_templates.key?(method.to_sym) ||
|
149
|
-
@settings.key?(method.to_sym) ||
|
150
|
-
super
|
151
|
-
end
|
152
|
-
|
153
|
-
# :nodoc:
|
154
|
-
def method_missing(method, *args, &block)
|
155
|
-
config_key = method.to_s.sub(/=$/, '').to_sym
|
156
|
-
|
157
|
-
# Return the list of setting objects with the given name
|
158
|
-
if config_key.to_s.end_with?('objects')
|
159
|
-
return _setting_object_method(config_key)
|
160
|
-
end
|
161
|
-
|
162
|
-
# Define a new setting object with the given name
|
163
|
-
if @setting_templates.key?(config_key) && block_given?
|
164
|
-
return _new_setting_object_method(config_key, &block)
|
165
|
-
end
|
166
|
-
|
167
|
-
setting = @settings[config_key]
|
168
|
-
|
169
|
-
if setting&.deprecation
|
170
|
-
return _deprecated_config_method(method, *args)
|
171
|
-
end
|
172
|
-
|
173
|
-
return super unless setting
|
174
|
-
|
175
|
-
if block_given?
|
176
|
-
return _block_config_method(config_key, &block)
|
177
|
-
end
|
178
|
-
|
179
|
-
_default_config_method(config_key, *args)
|
180
|
-
end
|
181
|
-
|
182
|
-
protected
|
183
|
-
|
184
|
-
# Only for the clone method
|
185
|
-
attr_accessor :settings, :setting_objects
|
186
|
-
|
187
|
-
private
|
188
|
-
|
189
|
-
def _deprecated_config_method(method, *args)
|
190
|
-
config_key = method.to_s.sub(/=$/, '').to_sym
|
191
|
-
new_config = @settings[config_key].deprecation
|
192
|
-
equals = method.to_s.end_with?('=') ? '=' : ''
|
193
|
-
ActiveSupport::Deprecation.warn("config.#{config_key}#{equals} is deprecated - use config.#{new_config}#{equals}")
|
194
|
-
obj = self
|
195
|
-
messages = new_config.split('.')
|
196
|
-
messages[0..-2].each do |message|
|
197
|
-
obj = obj.send(message)
|
198
|
-
end
|
199
|
-
if args.length.positive?
|
200
|
-
obj.send(messages[-1], args[0])
|
201
|
-
else
|
202
|
-
obj.send(messages[-1])
|
203
|
-
end
|
204
|
-
end
|
205
|
-
|
206
|
-
# Get or set a value.
|
207
|
-
def _default_config_method(config_key, *args)
|
208
|
-
if args.length.positive?
|
209
|
-
# Set the value
|
210
|
-
@settings[config_key].value = args[0]
|
211
|
-
else
|
212
|
-
# Get the value
|
213
|
-
setting = @settings[config_key]
|
214
|
-
if setting.default_proc && setting.value.nil?
|
215
|
-
setting.value = setting.default_proc.call
|
216
|
-
end
|
217
|
-
setting.value
|
218
|
-
end
|
219
|
-
end
|
220
|
-
|
221
|
-
# Define a new setting object and use the passed block to define values.
|
222
|
-
def _new_setting_object_method(config_key, &block)
|
223
|
-
new_config = @setting_templates[config_key].clone_and_reset
|
224
|
-
new_config.instance_eval(&block)
|
225
|
-
@setting_objects[config_key] << new_config
|
226
|
-
end
|
227
|
-
|
228
|
-
# Return a setting object.
|
229
|
-
def _setting_object_method(config_key)
|
230
|
-
key = config_key.to_s.sub(/_objects$/, '').to_sym
|
231
|
-
@setting_objects[key]
|
232
|
-
end
|
233
|
-
|
234
|
-
# Define new values inside a block.
|
235
|
-
def _block_config_method(config_key, &block)
|
236
|
-
unless @settings[config_key].value.is_a?(ConfigStruct)
|
237
|
-
raise "Block called for #{config_key} but it is not a nested config!"
|
238
|
-
end
|
239
|
-
|
240
|
-
@settings[config_key].value.instance_eval(&block)
|
241
|
-
end
|
242
|
-
end
|
243
|
-
|
244
|
-
# :nodoc:
|
245
|
-
module ClassMethods
|
246
|
-
# Define and redefine settings.
|
247
|
-
def define_settings(&block)
|
248
|
-
config.instance_eval(&block)
|
249
|
-
end
|
250
|
-
|
251
|
-
# Configure the settings with values.
|
252
|
-
def configure(&block)
|
253
|
-
if defined?(Rake) && defined?(Rake.application)
|
254
|
-
tasks = Rake.application.top_level_tasks
|
255
|
-
if tasks.any? { |t| %w(assets webpacker yarn).include?(t.split(':').first) }
|
256
|
-
puts 'Skipping Deimos configuration since we are in JS/CSS compilation'
|
257
|
-
return
|
258
|
-
end
|
259
|
-
end
|
260
|
-
config.run_callbacks(:configure) do
|
261
|
-
config.instance_eval(&block)
|
262
|
-
end
|
263
|
-
end
|
264
|
-
|
265
|
-
# @return [ConfigStruct]
|
266
|
-
def config
|
267
|
-
@config ||= ConfigStruct.new('config')
|
268
|
-
end
|
269
|
-
|
270
|
-
# Pass a block to run after configuration is done.
|
271
|
-
def after_configure(&block)
|
272
|
-
mod = self
|
273
|
-
config.class.set_callback(:configure, :after,
|
274
|
-
proc { mod.instance_eval(&block) })
|
275
|
-
end
|
276
|
-
end
|
277
|
-
end
|
278
|
-
end
|
@@ -1,136 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
# :nodoc:
|
4
|
-
class MyConfig
|
5
|
-
include Deimos::Configurable
|
6
|
-
|
7
|
-
define_settings do
|
8
|
-
setting :set1
|
9
|
-
setting :set2, 'hi mom'
|
10
|
-
setting :group do
|
11
|
-
setting :set3, default_proc: proc { false }
|
12
|
-
setting :set5, (proc { 5 })
|
13
|
-
end
|
14
|
-
|
15
|
-
setting_object :listy do
|
16
|
-
setting :list1, 10
|
17
|
-
setting :list2, 5
|
18
|
-
end
|
19
|
-
end
|
20
|
-
end
|
21
|
-
|
22
|
-
describe Deimos::Configurable do
|
23
|
-
it 'should configure correctly with default values' do
|
24
|
-
expect(MyConfig.config.set1).to be_nil
|
25
|
-
expect(MyConfig.config.set2).to eq('hi mom')
|
26
|
-
expect(MyConfig.config.group.set3).to eq(false)
|
27
|
-
expect(MyConfig.config.listy_objects).to be_empty
|
28
|
-
expect { MyConfig.config.blah }.to raise_error(NameError)
|
29
|
-
expect { MyConfig.config.group.set4 }.to raise_error(NameError)
|
30
|
-
end
|
31
|
-
|
32
|
-
it 'should not call the proc until it has to' do
|
33
|
-
num_calls = 0
|
34
|
-
value_proc = proc do
|
35
|
-
num_calls += 1
|
36
|
-
num_calls
|
37
|
-
end
|
38
|
-
MyConfig.define_settings do
|
39
|
-
setting :set_with_proc, default_proc: value_proc
|
40
|
-
end
|
41
|
-
expect(num_calls).to eq(0)
|
42
|
-
expect(MyConfig.config.set_with_proc).to eq(1)
|
43
|
-
# calling twice should not call the proc again
|
44
|
-
expect(MyConfig.config.set_with_proc).to eq(1)
|
45
|
-
expect(num_calls).to eq(1)
|
46
|
-
end
|
47
|
-
|
48
|
-
it "should raise error when setting configs that don't exist" do
|
49
|
-
expect { MyConfig.configure { set15 'some_value' } }.to raise_error(NameError)
|
50
|
-
end
|
51
|
-
|
52
|
-
it 'should add values' do
|
53
|
-
MyConfig.configure do |config|
|
54
|
-
config.set1 = 5 # config.x syntax
|
55
|
-
set2 nil # method_missing syntax
|
56
|
-
config.group.set3 = true
|
57
|
-
end
|
58
|
-
|
59
|
-
# second configure should not blow anything away
|
60
|
-
MyConfig.configure do
|
61
|
-
listy do
|
62
|
-
list1 0
|
63
|
-
list2 1
|
64
|
-
end
|
65
|
-
listy do
|
66
|
-
list1 100
|
67
|
-
list2 200
|
68
|
-
end
|
69
|
-
end
|
70
|
-
|
71
|
-
expect(MyConfig.config.set1).to eq(5)
|
72
|
-
expect(MyConfig.config.set2).to be_nil
|
73
|
-
expect(MyConfig.config.group.set3).to eq(true)
|
74
|
-
expect(MyConfig.config.listy_objects.map(&:to_h)).
|
75
|
-
to eq([
|
76
|
-
{ list1: 0, list2: 1 },
|
77
|
-
{ list1: 100, list2: 200 }
|
78
|
-
])
|
79
|
-
|
80
|
-
# test reset!
|
81
|
-
MyConfig.config.reset!
|
82
|
-
expect(MyConfig.config.set1).to be_nil
|
83
|
-
expect(MyConfig.config.set2).to eq('hi mom')
|
84
|
-
expect(MyConfig.config.group.set3).to eq(false)
|
85
|
-
expect(MyConfig.config.listy_objects).to be_empty
|
86
|
-
end
|
87
|
-
|
88
|
-
it 'should add with block syntax' do
|
89
|
-
MyConfig.configure do
|
90
|
-
group do
|
91
|
-
set5(proc { 10 })
|
92
|
-
end
|
93
|
-
end
|
94
|
-
expect(MyConfig.config.group.set5.call).to eq(10)
|
95
|
-
end
|
96
|
-
|
97
|
-
it 'should add or redefine settings' do
|
98
|
-
MyConfig.define_settings do
|
99
|
-
setting :group do
|
100
|
-
setting :set6, 15
|
101
|
-
setting :set5, (proc { 15 })
|
102
|
-
end
|
103
|
-
setting_object :notey do
|
104
|
-
setting :note_title, 'some-title'
|
105
|
-
end
|
106
|
-
end
|
107
|
-
|
108
|
-
expect(MyConfig.config.group.set6).to eq(15)
|
109
|
-
expect(MyConfig.config.group.set5.call).to eq(15)
|
110
|
-
expect(MyConfig.config.listy_objects).to be_empty
|
111
|
-
expect(MyConfig.config.notey_objects).to be_empty
|
112
|
-
|
113
|
-
MyConfig.configure do
|
114
|
-
notey do
|
115
|
-
note_title 'hi mom'
|
116
|
-
end
|
117
|
-
listy do
|
118
|
-
list1 0
|
119
|
-
end
|
120
|
-
end
|
121
|
-
expect(MyConfig.config.notey_objects.size).to eq(1)
|
122
|
-
expect(MyConfig.config.notey_objects.first.note_title).to eq('hi mom')
|
123
|
-
expect(MyConfig.config.listy_objects.size).to eq(1)
|
124
|
-
expect(MyConfig.config.listy_objects.first.list1).to eq(0)
|
125
|
-
|
126
|
-
# This should not remove any keys
|
127
|
-
MyConfig.define_settings do
|
128
|
-
setting :group do
|
129
|
-
setting :set6, 20
|
130
|
-
end
|
131
|
-
end
|
132
|
-
expect(MyConfig.config.group.set6).to eq(20)
|
133
|
-
expect(MyConfig.config.group.set5.call).to eq(15)
|
134
|
-
end
|
135
|
-
|
136
|
-
end
|