deimos-ruby 1.3.0.pre.beta5 → 1.4.0.pre.beta1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +2 -9
- data/Gemfile.lock +2 -88
- data/README.md +18 -135
- data/Rakefile +1 -1
- data/deimos-ruby.gemspec +0 -1
- data/docs/CONFIGURATION.md +200 -0
- data/lib/deimos/avro_data_coder.rb +4 -4
- data/lib/deimos/backends/db.rb +0 -5
- data/lib/deimos/base_consumer.rb +2 -2
- data/lib/deimos/config/configurable.rb +258 -0
- data/lib/deimos/config/configuration.rb +354 -0
- data/lib/deimos/config/phobos_config.rb +134 -0
- data/lib/deimos/kafka_message.rb +6 -0
- data/lib/deimos/producer.rb +5 -5
- data/lib/deimos/test_helpers.rb +2 -10
- data/lib/deimos/utils/db_producer.rb +0 -5
- data/lib/deimos/utils/lag_reporter.rb +4 -4
- data/lib/deimos/version.rb +1 -1
- data/lib/deimos.rb +2 -86
- data/lib/tasks/deimos.rake +1 -1
- data/spec/backends/db_spec.rb +0 -5
- data/spec/batch_consumer_spec.rb +1 -1
- data/spec/config/configurable_spec.rb +121 -0
- data/spec/config/configuration_spec.rb +229 -0
- data/spec/consumer_spec.rb +5 -5
- data/spec/deimos_spec.rb +14 -35
- data/spec/kafka_source_spec.rb +1 -1
- data/spec/producer_spec.rb +5 -5
- data/spec/spec_helper.rb +8 -20
- data/spec/updateable_schema_store_spec.rb +1 -1
- data/spec/utils/db_producer_spec.rb +0 -10
- data/spec/utils/lag_reporter_spec.rb +2 -2
- metadata +10 -19
- data/lib/deimos/configuration.rb +0 -124
- data/spec/rake_spec.rb +0 -19
@@ -0,0 +1,229 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
class MyConfigConsumer < Deimos::Consumer
|
4
|
+
end
|
5
|
+
describe Deimos, 'configuration' do
|
6
|
+
it 'should configure with deprecated fields' do
|
7
|
+
logger = Logger.new(nil)
|
8
|
+
Deimos.configure do
|
9
|
+
kafka_logger logger
|
10
|
+
reraise_consumer_errors true
|
11
|
+
schema_registry_url 'http://schema.registry'
|
12
|
+
seed_broker 'whatever'
|
13
|
+
schema_path 'some_path'
|
14
|
+
producer_schema_namespace 'namespace'
|
15
|
+
producer_topic_prefix 'prefix'
|
16
|
+
disable_producers true
|
17
|
+
ssl_enabled true
|
18
|
+
ssl_ca_cert 'cert'
|
19
|
+
ssl_client_cert 'cert'
|
20
|
+
ssl_client_cert_key 'key'
|
21
|
+
publish_backend 'db'
|
22
|
+
report_lag true
|
23
|
+
end
|
24
|
+
|
25
|
+
expect(Deimos.config.kafka.logger).to eq(logger)
|
26
|
+
expect(Deimos.config.consumers.reraise_errors).to eq(true)
|
27
|
+
expect(Deimos.config.schema.registry_url).to eq('http://schema.registry')
|
28
|
+
expect(Deimos.config.kafka.seed_brokers).to eq('whatever')
|
29
|
+
expect(Deimos.config.producers.schema_namespace).to eq('namespace')
|
30
|
+
expect(Deimos.config.producers.topic_prefix).to eq('prefix')
|
31
|
+
expect(Deimos.config.producers.disabled).to eq(true)
|
32
|
+
expect(Deimos.config.kafka.ssl.enabled).to eq(true)
|
33
|
+
expect(Deimos.config.kafka.ssl.ca_cert).to eq('cert')
|
34
|
+
expect(Deimos.config.kafka.ssl.client_cert).to eq('cert')
|
35
|
+
expect(Deimos.config.kafka.ssl.client_cert_key).to eq('key')
|
36
|
+
expect(Deimos.config.producers.backend).to eq('db')
|
37
|
+
expect(Deimos.config.consumers.report_lag).to eq(true)
|
38
|
+
end
|
39
|
+
|
40
|
+
it 'reads existing Phobos config YML files' do
|
41
|
+
Deimos.config.reset!
|
42
|
+
Deimos.configure { |c| c.phobos_config_file = File.join(File.dirname(__FILE__), '..', 'phobos.yml') }
|
43
|
+
expect(Deimos.config.phobos_config).to match(
|
44
|
+
logger: an_instance_of(Logger),
|
45
|
+
backoff: { min_ms: 1000, max_ms: 60_000 },
|
46
|
+
consumer: {
|
47
|
+
session_timeout: 300,
|
48
|
+
offset_commit_interval: 10,
|
49
|
+
offset_commit_threshold: 0,
|
50
|
+
heartbeat_interval: 10
|
51
|
+
},
|
52
|
+
custom_kafka_logger: an_instance_of(Logger),
|
53
|
+
custom_logger: an_instance_of(Logger),
|
54
|
+
kafka: {
|
55
|
+
client_id: 'phobos',
|
56
|
+
connect_timeout: 15,
|
57
|
+
socket_timeout: 15,
|
58
|
+
ssl_verify_hostname: true,
|
59
|
+
seed_brokers: ['localhost:9092']
|
60
|
+
},
|
61
|
+
listeners: [
|
62
|
+
{
|
63
|
+
topic: 'my_consume_topic',
|
64
|
+
group_id: 'my_group_id',
|
65
|
+
max_concurrency: nil,
|
66
|
+
start_from_beginning: nil,
|
67
|
+
max_bytes_per_partition: 524_288,
|
68
|
+
min_bytes: nil,
|
69
|
+
max_wait_time: nil,
|
70
|
+
force_encoding: nil,
|
71
|
+
delivery: nil,
|
72
|
+
session_timeout: nil,
|
73
|
+
offset_commit_interval: nil,
|
74
|
+
offset_commit_threshold: nil,
|
75
|
+
offset_retention_time: nil,
|
76
|
+
heartbeat_interval: nil,
|
77
|
+
handler: 'ConsumerTest::MyConsumer'
|
78
|
+
}, {
|
79
|
+
topic: 'my_batch_consume_topic',
|
80
|
+
group_id: 'my_batch_group_id',
|
81
|
+
max_concurrency: nil,
|
82
|
+
start_from_beginning: nil,
|
83
|
+
max_bytes_per_partition: nil,
|
84
|
+
min_bytes: nil,
|
85
|
+
max_wait_time: nil,
|
86
|
+
force_encoding: nil,
|
87
|
+
delivery: 'inline_batch',
|
88
|
+
session_timeout: nil,
|
89
|
+
offset_commit_interval: nil,
|
90
|
+
offset_commit_threshold: nil,
|
91
|
+
offset_retention_time: nil,
|
92
|
+
heartbeat_interval: nil,
|
93
|
+
handler: 'ConsumerTest::MyBatchConsumer'
|
94
|
+
}
|
95
|
+
],
|
96
|
+
producer: {
|
97
|
+
ack_timeout: 5,
|
98
|
+
required_acks: :all,
|
99
|
+
max_retries: 2,
|
100
|
+
retry_backoff: 1,
|
101
|
+
max_buffer_size: 10_000,
|
102
|
+
max_buffer_bytesize: 10_000_000,
|
103
|
+
compression_codec: nil,
|
104
|
+
compression_threshold: 1,
|
105
|
+
max_queue_size: 10_000,
|
106
|
+
delivery_threshold: 0,
|
107
|
+
delivery_interval: 0
|
108
|
+
}
|
109
|
+
)
|
110
|
+
end
|
111
|
+
|
112
|
+
specify '#phobos_config' do
|
113
|
+
logger1 = Logger.new(nil)
|
114
|
+
logger2 = Logger.new(nil)
|
115
|
+
Deimos.config.reset!
|
116
|
+
Deimos.configure do
|
117
|
+
phobos_logger logger1
|
118
|
+
kafka do
|
119
|
+
logger logger2
|
120
|
+
seed_brokers 'my-seed-brokers'
|
121
|
+
client_id 'phobos2'
|
122
|
+
connect_timeout 30
|
123
|
+
socket_timeout 30
|
124
|
+
ssl.enabled(true)
|
125
|
+
ssl.ca_cert('cert')
|
126
|
+
ssl.client_cert('cert')
|
127
|
+
ssl.client_cert_key('key')
|
128
|
+
ssl.verify_hostname(false)
|
129
|
+
end
|
130
|
+
consumers do
|
131
|
+
session_timeout 30
|
132
|
+
offset_commit_interval 5
|
133
|
+
offset_commit_threshold 0
|
134
|
+
heartbeat_interval 5
|
135
|
+
backoff 5..10
|
136
|
+
end
|
137
|
+
producers do
|
138
|
+
ack_timeout 3
|
139
|
+
required_acks 1
|
140
|
+
max_retries 1
|
141
|
+
retry_backoff 2
|
142
|
+
max_buffer_size 5
|
143
|
+
max_buffer_bytesize 5
|
144
|
+
compression_codec :snappy
|
145
|
+
compression_threshold 2
|
146
|
+
max_queue_size 10
|
147
|
+
delivery_threshold 1
|
148
|
+
delivery_interval 1
|
149
|
+
persistent_connections true
|
150
|
+
end
|
151
|
+
consumer do
|
152
|
+
class_name 'MyConfigConsumer'
|
153
|
+
schema 'blah'
|
154
|
+
topic 'blah'
|
155
|
+
group_id 'myconsumerid'
|
156
|
+
max_concurrency 1
|
157
|
+
start_from_beginning true
|
158
|
+
max_bytes_per_partition 10
|
159
|
+
min_bytes 5
|
160
|
+
max_wait_time 5
|
161
|
+
force_encoding true
|
162
|
+
delivery :message
|
163
|
+
backoff 100..200
|
164
|
+
session_timeout 10
|
165
|
+
offset_commit_interval 13
|
166
|
+
offset_commit_threshold 13
|
167
|
+
offset_retention_time 13
|
168
|
+
heartbeat_interval 13
|
169
|
+
end
|
170
|
+
end
|
171
|
+
|
172
|
+
expect(Deimos.config.phobos_config).
|
173
|
+
to match(
|
174
|
+
logger: an_instance_of(Logger),
|
175
|
+
backoff: { min_ms: 5, max_ms: 10 },
|
176
|
+
consumer: {
|
177
|
+
session_timeout: 30,
|
178
|
+
offset_commit_interval: 5,
|
179
|
+
offset_commit_threshold: 0,
|
180
|
+
heartbeat_interval: 5
|
181
|
+
},
|
182
|
+
custom_kafka_logger: logger2,
|
183
|
+
custom_logger: logger1,
|
184
|
+
kafka: {
|
185
|
+
client_id: 'phobos2',
|
186
|
+
connect_timeout: 30,
|
187
|
+
socket_timeout: 30,
|
188
|
+
ssl_ca_cert: 'cert',
|
189
|
+
ssl_client_cert: 'cert',
|
190
|
+
ssl_client_cert_key: 'key',
|
191
|
+
ssl_verify_hostname: false,
|
192
|
+
seed_brokers: ['my-seed-brokers']
|
193
|
+
},
|
194
|
+
listeners: [
|
195
|
+
{
|
196
|
+
topic: 'blah',
|
197
|
+
group_id: 'myconsumerid',
|
198
|
+
max_concurrency: 1,
|
199
|
+
start_from_beginning: true,
|
200
|
+
max_bytes_per_partition: 10,
|
201
|
+
min_bytes: 5,
|
202
|
+
max_wait_time: 5,
|
203
|
+
force_encoding: true,
|
204
|
+
delivery: 'message',
|
205
|
+
backoff: { min_ms: 100, max_ms: 200 },
|
206
|
+
session_timeout: 10,
|
207
|
+
offset_commit_interval: 13,
|
208
|
+
offset_commit_threshold: 13,
|
209
|
+
offset_retention_time: 13,
|
210
|
+
heartbeat_interval: 13,
|
211
|
+
handler: 'MyConfigConsumer'
|
212
|
+
}
|
213
|
+
],
|
214
|
+
producer: {
|
215
|
+
ack_timeout: 3,
|
216
|
+
required_acks: 1,
|
217
|
+
max_retries: 1,
|
218
|
+
retry_backoff: 2,
|
219
|
+
max_buffer_size: 5,
|
220
|
+
max_buffer_bytesize: 5,
|
221
|
+
compression_codec: :snappy,
|
222
|
+
compression_threshold: 2,
|
223
|
+
max_queue_size: 10,
|
224
|
+
delivery_threshold: 1,
|
225
|
+
delivery_interval: 1
|
226
|
+
}
|
227
|
+
)
|
228
|
+
end
|
229
|
+
end
|
data/spec/consumer_spec.rb
CHANGED
@@ -45,14 +45,14 @@ module ConsumerTest
|
|
45
45
|
end
|
46
46
|
|
47
47
|
it 'should fail if reraise is false but fatal_error is true' do
|
48
|
-
Deimos.configure { |config| config.
|
48
|
+
Deimos.configure { |config| config.consumers.reraise_errors = false }
|
49
49
|
test_consume_invalid_message(MyConsumer, 'fatal')
|
50
50
|
end
|
51
51
|
|
52
52
|
it 'should fail if fatal_error is true globally' do
|
53
53
|
Deimos.configure do |config|
|
54
|
-
config.fatal_error { true }
|
55
|
-
config.
|
54
|
+
config.consumers.fatal_error = proc { true }
|
55
|
+
config.consumers.reraise_errors = false
|
56
56
|
end
|
57
57
|
test_consume_invalid_message(MyConsumer, 'invalid' => 'key')
|
58
58
|
end
|
@@ -65,7 +65,7 @@ module ConsumerTest
|
|
65
65
|
end
|
66
66
|
|
67
67
|
it 'should not fail when before_consume fails without reraising errors' do
|
68
|
-
Deimos.configure { |config| config.
|
68
|
+
Deimos.configure { |config| config.consumers.reraise_errors = false }
|
69
69
|
expect {
|
70
70
|
test_consume_message(
|
71
71
|
MyConsumer,
|
@@ -77,7 +77,7 @@ module ConsumerTest
|
|
77
77
|
end
|
78
78
|
|
79
79
|
it 'should not fail when consume fails without reraising errors' do
|
80
|
-
Deimos.configure { |config| config.
|
80
|
+
Deimos.configure { |config| config.consumers.reraise_errors = false }
|
81
81
|
expect {
|
82
82
|
test_consume_message(
|
83
83
|
MyConsumer,
|
data/spec/deimos_spec.rb
CHANGED
@@ -57,26 +57,13 @@ describe Deimos do
|
|
57
57
|
expect(Deimos::VERSION).not_to be_nil
|
58
58
|
end
|
59
59
|
|
60
|
-
specify 'configure' do
|
61
|
-
expect(Phobos).to receive(:configure).with(phobos_configuration)
|
62
|
-
allow(described_class).to receive(:ssl_var_contents) { |key| key }
|
63
|
-
described_class.configure do |config|
|
64
|
-
config.phobos_config_file = config_path
|
65
|
-
config.seed_broker = 'my_seed_broker.com'
|
66
|
-
config.ssl_enabled = true
|
67
|
-
config.ssl_ca_cert = 'my_ssl_ca_cert'
|
68
|
-
config.ssl_client_cert = 'my_ssl_client_cert'
|
69
|
-
config.ssl_client_cert_key = 'my_ssl_client_cert_key'
|
70
|
-
end
|
71
|
-
end
|
72
|
-
|
73
60
|
it 'should error if required_acks is not all' do
|
74
61
|
expect {
|
75
62
|
described_class.configure do |config|
|
76
|
-
config.
|
63
|
+
config.producers.backend = :db
|
77
64
|
config.phobos_config_file = File.join(File.dirname(__FILE__), 'phobos.bad_db.yml')
|
78
65
|
end
|
79
|
-
}.to raise_error('Cannot set
|
66
|
+
}.to raise_error('Cannot set producers.backend to :db unless producers.required_acks is set to ":all"!')
|
80
67
|
end
|
81
68
|
|
82
69
|
describe '#start_db_backend!' do
|
@@ -94,7 +81,7 @@ describe Deimos do
|
|
94
81
|
signal_handler
|
95
82
|
end
|
96
83
|
described_class.configure do |config|
|
97
|
-
config.
|
84
|
+
config.producers.backend = :db
|
98
85
|
end
|
99
86
|
described_class.start_db_backend!(thread_count: 2)
|
100
87
|
end
|
@@ -102,7 +89,7 @@ describe Deimos do
|
|
102
89
|
it 'should not start if backend is not db' do
|
103
90
|
expect(Deimos::Utils::SignalHandler).not_to receive(:new)
|
104
91
|
described_class.configure do |config|
|
105
|
-
config.
|
92
|
+
config.producers.backend = :kafka
|
106
93
|
end
|
107
94
|
expect { described_class.start_db_backend!(thread_count: 2) }.
|
108
95
|
to raise_error('Publish backend is not set to :db, exiting')
|
@@ -111,7 +98,7 @@ describe Deimos do
|
|
111
98
|
it 'should not start if thread_count is nil' do
|
112
99
|
expect(Deimos::Utils::SignalHandler).not_to receive(:new)
|
113
100
|
described_class.configure do |config|
|
114
|
-
config.
|
101
|
+
config.producers.backend = :db
|
115
102
|
end
|
116
103
|
expect { described_class.start_db_backend!(thread_count: nil) }.
|
117
104
|
to raise_error('Thread count is not given or set to zero, exiting')
|
@@ -120,7 +107,7 @@ describe Deimos do
|
|
120
107
|
it 'should not start if thread_count is 0' do
|
121
108
|
expect(Deimos::Utils::SignalHandler).not_to receive(:new)
|
122
109
|
described_class.configure do |config|
|
123
|
-
config.
|
110
|
+
config.producers.backend = :db
|
124
111
|
end
|
125
112
|
expect { described_class.start_db_backend!(thread_count: 0) }.
|
126
113
|
to raise_error('Thread count is not given or set to zero, exiting')
|
@@ -128,11 +115,11 @@ describe Deimos do
|
|
128
115
|
|
129
116
|
describe 'delivery configuration' do
|
130
117
|
before(:each) do
|
131
|
-
described_class.config = nil
|
132
118
|
allow(YAML).to receive(:load).and_return(phobos_configuration)
|
133
119
|
end
|
134
120
|
|
135
121
|
it 'should not raise an error with properly configured handlers' do
|
122
|
+
path = config_path # for scope issues in the block below
|
136
123
|
# Add explicit consumers
|
137
124
|
phobos_configuration['listeners'] << { 'handler' => 'ConsumerTest::MyConsumer',
|
138
125
|
'delivery' => 'message' }
|
@@ -140,47 +127,39 @@ describe Deimos do
|
|
140
127
|
'delivery' => 'batch' }
|
141
128
|
|
142
129
|
expect {
|
143
|
-
described_class.configure { |c| c.phobos_config_file =
|
130
|
+
described_class.configure { |c| c.phobos_config_file = path }
|
144
131
|
}.not_to raise_error
|
145
132
|
end
|
146
133
|
|
147
134
|
it 'should raise an error if BatchConsumers do not have inline_batch delivery' do
|
135
|
+
path = config_path # for scope issues in the block below
|
148
136
|
phobos_configuration['listeners'] = [{ 'handler' => 'ConsumerTest::MyBatchConsumer',
|
149
137
|
'delivery' => 'message' }]
|
150
138
|
|
151
139
|
expect {
|
152
|
-
described_class.configure { |c| c.phobos_config_file =
|
140
|
+
described_class.configure { |c| c.phobos_config_file = path }
|
153
141
|
}.to raise_error('BatchConsumer ConsumerTest::MyBatchConsumer must have delivery set to `inline_batch`')
|
154
142
|
end
|
155
143
|
|
156
144
|
it 'should raise an error if Consumers do not have message or batch delivery' do
|
145
|
+
path = config_path # for scope issues in the block below
|
157
146
|
phobos_configuration['listeners'] = [{ 'handler' => 'ConsumerTest::MyConsumer',
|
158
147
|
'delivery' => 'inline_batch' }]
|
159
148
|
|
160
149
|
expect {
|
161
|
-
described_class.configure { |c| c.phobos_config_file =
|
150
|
+
described_class.configure { |c| c.phobos_config_file = path }
|
162
151
|
}.to raise_error('Non-batch Consumer ConsumerTest::MyConsumer must have delivery set to `message` or `batch`')
|
163
152
|
end
|
164
153
|
|
165
154
|
it 'should treat nil as `batch`' do
|
155
|
+
path = config_path # for scope issues in the block below
|
166
156
|
phobos_configuration['listeners'] = [{ 'handler' => 'ConsumerTest::MyConsumer' }]
|
167
157
|
|
168
158
|
expect {
|
169
|
-
described_class.configure { |c| c.phobos_config_file =
|
159
|
+
described_class.configure { |c| c.phobos_config_file = path }
|
170
160
|
}.not_to raise_error
|
171
161
|
end
|
172
162
|
|
173
|
-
it 'should ignore non-Deimos listeners' do
|
174
|
-
consumer_class = Class.new { include Phobos::Handler }
|
175
|
-
stub_const('ConsumerTest::MyOtherConsumer', consumer_class)
|
176
|
-
phobos_configuration['listeners'] = [{ 'handler' => 'ConsumerTest::MyOtherConsumer',
|
177
|
-
'topic' => 'my_consume_topic',
|
178
|
-
'group_id' => 'my_group_id' }]
|
179
|
-
|
180
|
-
expect {
|
181
|
-
described_class.configure { |c| c.phobos_config_file = config_path }
|
182
|
-
}.not_to raise_error
|
183
|
-
end
|
184
163
|
end
|
185
164
|
end
|
186
165
|
end
|
data/spec/kafka_source_spec.rb
CHANGED
@@ -145,7 +145,7 @@ module KafkaSourceSpec
|
|
145
145
|
context 'with DB backend' do
|
146
146
|
before(:each) do
|
147
147
|
Deimos.configure do |config|
|
148
|
-
config.
|
148
|
+
config.producers.backend = :db
|
149
149
|
end
|
150
150
|
setup_db(DB_OPTIONS.last) # sqlite
|
151
151
|
allow(Deimos::Producer).to receive(:produce_batch).and_call_original
|
data/spec/producer_spec.rb
CHANGED
@@ -125,7 +125,7 @@ module ProducerTest
|
|
125
125
|
|
126
126
|
it 'should not publish if publish disabled' do
|
127
127
|
expect(described_class).not_to receive(:produce_batch)
|
128
|
-
Deimos.configure { |c| c.
|
128
|
+
Deimos.configure { |c| c.producers.disabled = true }
|
129
129
|
MyProducer.publish_list(
|
130
130
|
[{ 'test_id' => 'foo', 'some_int' => 123 },
|
131
131
|
{ 'test_id' => 'bar', 'some_int' => 124 }]
|
@@ -158,7 +158,7 @@ module ProducerTest
|
|
158
158
|
end
|
159
159
|
|
160
160
|
it 'should produce to a prefixed topic' do
|
161
|
-
Deimos.configure { |c| c.
|
161
|
+
Deimos.configure { |c| c.producers.topic_prefix = 'prefix.' }
|
162
162
|
payload = { 'test_id' => 'foo', 'some_int' => 123 }
|
163
163
|
expect(described_class).to receive(:produce_batch).once do |_, messages|
|
164
164
|
expect(messages.size).to eq(1)
|
@@ -176,7 +176,7 @@ module ProducerTest
|
|
176
176
|
end
|
177
177
|
|
178
178
|
MyProducer.publish_list([payload])
|
179
|
-
Deimos.configure { |c| c.
|
179
|
+
Deimos.configure { |c| c.producers.topic_prefix = nil }
|
180
180
|
expect(described_class).to receive(:produce_batch).once do |_, messages|
|
181
181
|
expect(messages.size).to eq(1)
|
182
182
|
expect(messages[0].to_h).
|
@@ -377,7 +377,7 @@ module ProducerTest
|
|
377
377
|
end
|
378
378
|
|
379
379
|
it 'should return db if db is set' do
|
380
|
-
|
380
|
+
Deimos.configure { producers.backend = :db }
|
381
381
|
expect(described_class.determine_backend_class(true, false)).
|
382
382
|
to eq(Deimos::Backends::Db)
|
383
383
|
expect(described_class.determine_backend_class(false, false)).
|
@@ -385,7 +385,7 @@ module ProducerTest
|
|
385
385
|
end
|
386
386
|
|
387
387
|
it 'should return kafka if force_send is true' do
|
388
|
-
|
388
|
+
Deimos.configure { producers.backend = :db }
|
389
389
|
expect(described_class.determine_backend_class(true, true)).
|
390
390
|
to eq(Deimos::Backends::Kafka)
|
391
391
|
expect(described_class.determine_backend_class(false, true)).
|
data/spec/spec_helper.rb
CHANGED
@@ -144,33 +144,21 @@ RSpec.configure do |config|
|
|
144
144
|
Time.zone = 'EST'
|
145
145
|
ActiveRecord::Base.logger = Logger.new('/dev/null')
|
146
146
|
setup_db(DbConfigs::DB_OPTIONS.last)
|
147
|
+
end
|
148
|
+
|
149
|
+
config.before(:each) do |ex|
|
150
|
+
Deimos.config.reset!
|
147
151
|
Deimos.configure do |deimos_config|
|
148
152
|
deimos_config.phobos_config_file = File.join(File.dirname(__FILE__), 'phobos.yml')
|
149
|
-
deimos_config.
|
150
|
-
deimos_config.
|
151
|
-
deimos_config.
|
152
|
-
deimos_config.
|
153
|
+
deimos_config.schema.path = File.join(File.expand_path(__dir__), 'schemas')
|
154
|
+
deimos_config.consumers.reraise_errors = true
|
155
|
+
deimos_config.schema.registry_url = ENV['SCHEMA_REGISTRY'] || 'http://localhost:8081'
|
156
|
+
deimos_config.kafka.seed_brokers = ENV['KAFKA_SEED_BROKER'] || 'localhost:9092'
|
153
157
|
deimos_config.logger = Logger.new('/dev/null')
|
154
158
|
deimos_config.logger.level = Logger::INFO
|
155
|
-
|
156
|
-
# Use Mock Metrics and Tracing for rspecs
|
157
|
-
deimos_config.metrics = Deimos::Metrics::Mock.new
|
158
|
-
deimos_config.tracer = Deimos::Tracing::Mock.new
|
159
159
|
end
|
160
|
-
end
|
161
|
-
|
162
|
-
config.before(:each) do |ex|
|
163
160
|
stub_producers_and_consumers! unless ex.metadata[:integration]
|
164
|
-
|
165
|
-
@previous_config = Deimos.config.dup
|
166
|
-
@previous_phobos_config = Phobos.config.dup
|
167
161
|
end
|
168
|
-
|
169
|
-
config.after(:each) do
|
170
|
-
Deimos.config = @previous_config
|
171
|
-
Phobos.instance_variable_set(:@config, @previous_phobos_config)
|
172
|
-
end
|
173
|
-
|
174
162
|
end
|
175
163
|
|
176
164
|
RSpec.shared_context('with DB') do
|
@@ -3,7 +3,7 @@
|
|
3
3
|
describe AvroTurf::SchemaStore do
|
4
4
|
|
5
5
|
it 'should add an in-memory schema' do
|
6
|
-
schema_store = described_class.new(path: Deimos.config.
|
6
|
+
schema_store = described_class.new(path: Deimos.config.schema.path)
|
7
7
|
schema_store.load_schemas!
|
8
8
|
found_schema = schema_store.find('MySchema', 'com.my-namespace').as_json
|
9
9
|
expect(found_schema['name']).to eq('MySchema')
|
@@ -192,11 +192,6 @@ each_db_config(Deimos::Utils::DbProducer) do
|
|
192
192
|
topic: 'my-topic'
|
193
193
|
}
|
194
194
|
])
|
195
|
-
expect(Deimos.config.metrics).to receive(:increment).ordered.with(
|
196
|
-
'db_producer.process',
|
197
|
-
tags: %w(topic:my-topic),
|
198
|
-
by: 2
|
199
|
-
)
|
200
195
|
expect(producer).to receive(:retrieve_messages).ordered.
|
201
196
|
and_return(messages[2..3])
|
202
197
|
expect(producer).to receive(:produce_messages).ordered.with([
|
@@ -213,11 +208,6 @@ each_db_config(Deimos::Utils::DbProducer) do
|
|
213
208
|
topic: 'my-topic'
|
214
209
|
}
|
215
210
|
])
|
216
|
-
expect(Deimos.config.metrics).to receive(:increment).ordered.with(
|
217
|
-
'db_producer.process',
|
218
|
-
tags: %w(topic:my-topic),
|
219
|
-
by: 2
|
220
|
-
)
|
221
211
|
expect(producer).to receive(:retrieve_messages).ordered.
|
222
212
|
and_return([])
|
223
213
|
expect(Deimos::KafkaTopicInfo).to receive(:heartbeat).
|
@@ -6,12 +6,12 @@ describe Deimos::Utils::LagReporter do
|
|
6
6
|
kafka_client = instance_double(Kafka::Client)
|
7
7
|
allow(kafka_client).to receive(:last_offset_for).and_return(100)
|
8
8
|
allow(Phobos).to receive(:create_kafka_client).and_return(kafka_client)
|
9
|
-
Deimos.configure { |c| c.report_lag = true }
|
9
|
+
Deimos.configure { |c| c.consumers.report_lag = true }
|
10
10
|
end
|
11
11
|
|
12
12
|
after(:each) do
|
13
13
|
described_class.reset
|
14
|
-
Deimos.configure { |c| c.report_lag = false }
|
14
|
+
Deimos.configure { |c| c.consumers.report_lag = false }
|
15
15
|
end
|
16
16
|
|
17
17
|
it 'should not report lag before ready' do
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: deimos-ruby
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.4.0.pre.beta1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Daniel Orner
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2019-11-22 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: avro-patches
|
@@ -206,20 +206,6 @@ dependencies:
|
|
206
206
|
- - "~>"
|
207
207
|
- !ruby/object:Gem::Version
|
208
208
|
version: '1.1'
|
209
|
-
- !ruby/object:Gem::Dependency
|
210
|
-
name: rails
|
211
|
-
requirement: !ruby/object:Gem::Requirement
|
212
|
-
requirements:
|
213
|
-
- - "~>"
|
214
|
-
- !ruby/object:Gem::Version
|
215
|
-
version: '5.2'
|
216
|
-
type: :development
|
217
|
-
prerelease: false
|
218
|
-
version_requirements: !ruby/object:Gem::Requirement
|
219
|
-
requirements:
|
220
|
-
- - "~>"
|
221
|
-
- !ruby/object:Gem::Version
|
222
|
-
version: '5.2'
|
223
209
|
- !ruby/object:Gem::Dependency
|
224
210
|
name: rake
|
225
211
|
requirement: !ruby/object:Gem::Requirement
|
@@ -331,6 +317,7 @@ files:
|
|
331
317
|
- bin/deimos
|
332
318
|
- deimos-ruby.gemspec
|
333
319
|
- docker-compose.yml
|
320
|
+
- docs/CONFIGURATION.md
|
334
321
|
- docs/DATABASE_BACKEND.md
|
335
322
|
- docs/PULL_REQUEST_TEMPLATE.md
|
336
323
|
- lib/deimos.rb
|
@@ -344,7 +331,9 @@ files:
|
|
344
331
|
- lib/deimos/backends/kafka_async.rb
|
345
332
|
- lib/deimos/base_consumer.rb
|
346
333
|
- lib/deimos/batch_consumer.rb
|
347
|
-
- lib/deimos/
|
334
|
+
- lib/deimos/config/configurable.rb
|
335
|
+
- lib/deimos/config/configuration.rb
|
336
|
+
- lib/deimos/config/phobos_config.rb
|
348
337
|
- lib/deimos/consumer.rb
|
349
338
|
- lib/deimos/instrumentation.rb
|
350
339
|
- lib/deimos/kafka_message.rb
|
@@ -386,6 +375,8 @@ files:
|
|
386
375
|
- spec/backends/kafka_async_spec.rb
|
387
376
|
- spec/backends/kafka_spec.rb
|
388
377
|
- spec/batch_consumer_spec.rb
|
378
|
+
- spec/config/configurable_spec.rb
|
379
|
+
- spec/config/configuration_spec.rb
|
389
380
|
- spec/consumer_spec.rb
|
390
381
|
- spec/deimos_spec.rb
|
391
382
|
- spec/handlers/my_batch_consumer.rb
|
@@ -396,7 +387,6 @@ files:
|
|
396
387
|
- spec/phobos.yml
|
397
388
|
- spec/producer_spec.rb
|
398
389
|
- spec/publish_backend_spec.rb
|
399
|
-
- spec/rake_spec.rb
|
400
390
|
- spec/schemas/com/my-namespace/MySchema-key.avsc
|
401
391
|
- spec/schemas/com/my-namespace/MySchema.avsc
|
402
392
|
- spec/schemas/com/my-namespace/MySchemaWithBooleans.avsc
|
@@ -448,6 +438,8 @@ test_files:
|
|
448
438
|
- spec/backends/kafka_async_spec.rb
|
449
439
|
- spec/backends/kafka_spec.rb
|
450
440
|
- spec/batch_consumer_spec.rb
|
441
|
+
- spec/config/configurable_spec.rb
|
442
|
+
- spec/config/configuration_spec.rb
|
451
443
|
- spec/consumer_spec.rb
|
452
444
|
- spec/deimos_spec.rb
|
453
445
|
- spec/handlers/my_batch_consumer.rb
|
@@ -458,7 +450,6 @@ test_files:
|
|
458
450
|
- spec/phobos.yml
|
459
451
|
- spec/producer_spec.rb
|
460
452
|
- spec/publish_backend_spec.rb
|
461
|
-
- spec/rake_spec.rb
|
462
453
|
- spec/schemas/com/my-namespace/MySchema-key.avsc
|
463
454
|
- spec/schemas/com/my-namespace/MySchema.avsc
|
464
455
|
- spec/schemas/com/my-namespace/MySchemaWithBooleans.avsc
|