deimos-ruby 1.16.4 → 1.17.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +3 -3
- data/.rubocop.yml +20 -14
- data/.rubocop_todo.yml +364 -0
- data/.ruby-version +2 -1
- data/CHANGELOG.md +5 -0
- data/Gemfile +0 -1
- data/README.md +6 -2
- data/Steepfile +8 -0
- data/deimos-ruby.gemspec +3 -2
- data/docs/CONFIGURATION.md +2 -1
- data/lib/deimos/config/configuration.rb +2 -0
- data/lib/deimos/metrics/mock.rb +1 -1
- data/lib/deimos/schema_backends/mock.rb +1 -1
- data/lib/deimos/utils/db_poller.rb +49 -5
- data/lib/deimos/version.rb +1 -1
- data/lib/generators/deimos/schema_class_generator.rb +1 -1
- data/rbs_collection.lock.yaml +16 -16
- data/sig/defs.rbs +9 -1
- data/sig/fig_tree.rbs +1 -1
- data/spec/consumer_spec.rb +14 -14
- data/spec/generators/schema_class/my_schema_spec.rb +3 -3
- data/spec/generators/schema_class/my_schema_with_complex_types_spec.rb +1 -1
- data/spec/producer_spec.rb +1 -1
- data/spec/schemas/my_namespace/my_schema_with_complex_type.rb +3 -3
- data/spec/utils/db_poller_spec.rb +89 -46
- metadata +35 -22
- data/.gemfiles/avro_turf-0.gemfile +0 -3
- data/.gemfiles/avro_turf-1.gemfile +0 -3
- data/.ruby-gemset +0 -1
data/lib/deimos/version.rb
CHANGED
@@ -184,7 +184,7 @@ module Deimos
|
|
184
184
|
def generate_from_schema_files(found_schemas)
|
185
185
|
schema_store = AvroTurf::MutableSchemaStore.new(path: Deimos.config.schema.path)
|
186
186
|
schema_store.load_schemas!
|
187
|
-
schema_store.schemas.values.each do |schema|
|
187
|
+
schema_store.schemas.values.sort_by { |s| "#{s.namespace}#{s.name}" }.each do |schema|
|
188
188
|
name = "#{schema.namespace}.#{schema.name}"
|
189
189
|
next if found_schemas.include?(name)
|
190
190
|
|
data/rbs_collection.lock.yaml
CHANGED
@@ -11,7 +11,7 @@ gems:
|
|
11
11
|
source:
|
12
12
|
type: git
|
13
13
|
name: ruby/gem_rbs_collection
|
14
|
-
revision:
|
14
|
+
revision: 8e2fd75fc0e2bcbda9023e8c936fe5ba654b1436
|
15
15
|
remote: https://github.com/ruby/gem_rbs_collection.git
|
16
16
|
repo_dir: gems
|
17
17
|
- name: actionview
|
@@ -19,7 +19,7 @@ gems:
|
|
19
19
|
source:
|
20
20
|
type: git
|
21
21
|
name: ruby/gem_rbs_collection
|
22
|
-
revision:
|
22
|
+
revision: 8e2fd75fc0e2bcbda9023e8c936fe5ba654b1436
|
23
23
|
remote: https://github.com/ruby/gem_rbs_collection.git
|
24
24
|
repo_dir: gems
|
25
25
|
- name: activejob
|
@@ -27,7 +27,7 @@ gems:
|
|
27
27
|
source:
|
28
28
|
type: git
|
29
29
|
name: ruby/gem_rbs_collection
|
30
|
-
revision:
|
30
|
+
revision: 8e2fd75fc0e2bcbda9023e8c936fe5ba654b1436
|
31
31
|
remote: https://github.com/ruby/gem_rbs_collection.git
|
32
32
|
repo_dir: gems
|
33
33
|
- name: activemodel
|
@@ -35,7 +35,7 @@ gems:
|
|
35
35
|
source:
|
36
36
|
type: git
|
37
37
|
name: ruby/gem_rbs_collection
|
38
|
-
revision:
|
38
|
+
revision: 8e2fd75fc0e2bcbda9023e8c936fe5ba654b1436
|
39
39
|
remote: https://github.com/ruby/gem_rbs_collection.git
|
40
40
|
repo_dir: gems
|
41
41
|
- name: activerecord
|
@@ -43,7 +43,7 @@ gems:
|
|
43
43
|
source:
|
44
44
|
type: git
|
45
45
|
name: ruby/gem_rbs_collection
|
46
|
-
revision:
|
46
|
+
revision: 8e2fd75fc0e2bcbda9023e8c936fe5ba654b1436
|
47
47
|
remote: https://github.com/ruby/gem_rbs_collection.git
|
48
48
|
repo_dir: gems
|
49
49
|
- name: activestorage
|
@@ -51,7 +51,7 @@ gems:
|
|
51
51
|
source:
|
52
52
|
type: git
|
53
53
|
name: ruby/gem_rbs_collection
|
54
|
-
revision:
|
54
|
+
revision: 8e2fd75fc0e2bcbda9023e8c936fe5ba654b1436
|
55
55
|
remote: https://github.com/ruby/gem_rbs_collection.git
|
56
56
|
repo_dir: gems
|
57
57
|
- name: activesupport
|
@@ -59,7 +59,7 @@ gems:
|
|
59
59
|
source:
|
60
60
|
type: git
|
61
61
|
name: ruby/gem_rbs_collection
|
62
|
-
revision:
|
62
|
+
revision: 8e2fd75fc0e2bcbda9023e8c936fe5ba654b1436
|
63
63
|
remote: https://github.com/ruby/gem_rbs_collection.git
|
64
64
|
repo_dir: gems
|
65
65
|
- name: ast
|
@@ -67,11 +67,11 @@ gems:
|
|
67
67
|
source:
|
68
68
|
type: git
|
69
69
|
name: ruby/gem_rbs_collection
|
70
|
-
revision:
|
70
|
+
revision: 8e2fd75fc0e2bcbda9023e8c936fe5ba654b1436
|
71
71
|
remote: https://github.com/ruby/gem_rbs_collection.git
|
72
72
|
repo_dir: gems
|
73
73
|
- name: deimos-ruby
|
74
|
-
version: 1.16.
|
74
|
+
version: 1.16.4
|
75
75
|
source:
|
76
76
|
type: rubygems
|
77
77
|
- name: i18n
|
@@ -79,7 +79,7 @@ gems:
|
|
79
79
|
source:
|
80
80
|
type: git
|
81
81
|
name: ruby/gem_rbs_collection
|
82
|
-
revision:
|
82
|
+
revision: 8e2fd75fc0e2bcbda9023e8c936fe5ba654b1436
|
83
83
|
remote: https://github.com/ruby/gem_rbs_collection.git
|
84
84
|
repo_dir: gems
|
85
85
|
- name: listen
|
@@ -87,7 +87,7 @@ gems:
|
|
87
87
|
source:
|
88
88
|
type: git
|
89
89
|
name: ruby/gem_rbs_collection
|
90
|
-
revision:
|
90
|
+
revision: 8e2fd75fc0e2bcbda9023e8c936fe5ba654b1436
|
91
91
|
remote: https://github.com/ruby/gem_rbs_collection.git
|
92
92
|
repo_dir: gems
|
93
93
|
- name: minitest
|
@@ -99,7 +99,7 @@ gems:
|
|
99
99
|
source:
|
100
100
|
type: git
|
101
101
|
name: ruby/gem_rbs_collection
|
102
|
-
revision:
|
102
|
+
revision: 8e2fd75fc0e2bcbda9023e8c936fe5ba654b1436
|
103
103
|
remote: https://github.com/ruby/gem_rbs_collection.git
|
104
104
|
repo_dir: gems
|
105
105
|
- name: parallel
|
@@ -107,7 +107,7 @@ gems:
|
|
107
107
|
source:
|
108
108
|
type: git
|
109
109
|
name: ruby/gem_rbs_collection
|
110
|
-
revision:
|
110
|
+
revision: 8e2fd75fc0e2bcbda9023e8c936fe5ba654b1436
|
111
111
|
remote: https://github.com/ruby/gem_rbs_collection.git
|
112
112
|
repo_dir: gems
|
113
113
|
- name: phobos
|
@@ -119,7 +119,7 @@ gems:
|
|
119
119
|
source:
|
120
120
|
type: git
|
121
121
|
name: ruby/gem_rbs_collection
|
122
|
-
revision:
|
122
|
+
revision: 8e2fd75fc0e2bcbda9023e8c936fe5ba654b1436
|
123
123
|
remote: https://github.com/ruby/gem_rbs_collection.git
|
124
124
|
repo_dir: gems
|
125
125
|
- name: railties
|
@@ -127,7 +127,7 @@ gems:
|
|
127
127
|
source:
|
128
128
|
type: git
|
129
129
|
name: ruby/gem_rbs_collection
|
130
|
-
revision:
|
130
|
+
revision: 8e2fd75fc0e2bcbda9023e8c936fe5ba654b1436
|
131
131
|
remote: https://github.com/ruby/gem_rbs_collection.git
|
132
132
|
repo_dir: gems
|
133
133
|
- name: rainbow
|
@@ -135,7 +135,7 @@ gems:
|
|
135
135
|
source:
|
136
136
|
type: git
|
137
137
|
name: ruby/gem_rbs_collection
|
138
|
-
revision:
|
138
|
+
revision: 8e2fd75fc0e2bcbda9023e8c936fe5ba654b1436
|
139
139
|
remote: https://github.com/ruby/gem_rbs_collection.git
|
140
140
|
repo_dir: gems
|
141
141
|
- name: tempfile
|
data/sig/defs.rbs
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
module Deimos
|
3
3
|
include Deimos::Instrumentation
|
4
4
|
include FigTree
|
5
|
-
VERSION:
|
5
|
+
VERSION: untyped
|
6
6
|
|
7
7
|
def self.schema_backend_class: () -> singleton(Deimos::SchemaBackends::Base)
|
8
8
|
|
@@ -1144,6 +1144,9 @@ module Deimos
|
|
1144
1144
|
def schema_fields: () -> ::Array[String]
|
1145
1145
|
|
1146
1146
|
def self.initialize_from_value: (Object value) -> SchemaClass::Record
|
1147
|
+
|
1148
|
+
# Returns the value of attribute tombstone_key.
|
1149
|
+
attr_accessor tombstone_key: untyped
|
1147
1150
|
end
|
1148
1151
|
end
|
1149
1152
|
|
@@ -1589,6 +1592,11 @@ module Deimos
|
|
1589
1592
|
?column_name: Symbol,
|
1590
1593
|
min_id: Numeric
|
1591
1594
|
) -> ActiveRecord::Relation
|
1595
|
+
|
1596
|
+
# Post process records after publishing
|
1597
|
+
#
|
1598
|
+
# _@param_ `records`
|
1599
|
+
def self.post_process: (::Array[ActiveRecord::Base] _records) -> untyped
|
1592
1600
|
end
|
1593
1601
|
|
1594
1602
|
module Consume
|
data/sig/fig_tree.rbs
CHANGED
@@ -1,2 +1,2 @@
|
|
1
|
-
|
1
|
+
module FigTree
|
2
2
|
end
|
data/spec/consumer_spec.rb
CHANGED
@@ -32,8 +32,8 @@ module ConsumerTest
|
|
32
32
|
|
33
33
|
it 'should consume a message' do
|
34
34
|
test_consume_message(MyConsumer,
|
35
|
-
{'test_id' => 'foo',
|
36
|
-
'some_int' => 123}) do |payload, _metadata|
|
35
|
+
{ 'test_id' => 'foo',
|
36
|
+
'some_int' => 123 }) do |payload, _metadata|
|
37
37
|
expect(payload['test_id']).to eq('foo')
|
38
38
|
expect(payload['some_int']).to eq(123)
|
39
39
|
end
|
@@ -64,15 +64,15 @@ module ConsumerTest
|
|
64
64
|
|
65
65
|
it 'should consume a message on a topic' do
|
66
66
|
test_consume_message('my_consume_topic',
|
67
|
-
{'test_id' => 'foo',
|
68
|
-
'some_int' => 123}) do |payload, _metadata|
|
67
|
+
{ 'test_id' => 'foo',
|
68
|
+
'some_int' => 123 }) do |payload, _metadata|
|
69
69
|
expect(payload['test_id']).to eq('foo')
|
70
70
|
expect(payload['some_int']).to eq(123)
|
71
71
|
end
|
72
72
|
end
|
73
73
|
|
74
74
|
it 'should fail on invalid message' do
|
75
|
-
test_consume_invalid_message(MyConsumer, {'invalid' => 'key'})
|
75
|
+
test_consume_invalid_message(MyConsumer, { 'invalid' => 'key' })
|
76
76
|
end
|
77
77
|
|
78
78
|
it 'should fail if reraise is false but fatal_error is true' do
|
@@ -85,14 +85,14 @@ module ConsumerTest
|
|
85
85
|
config.consumers.fatal_error = proc { true }
|
86
86
|
config.consumers.reraise_errors = false
|
87
87
|
end
|
88
|
-
test_consume_invalid_message(MyConsumer, {'invalid' => 'key'})
|
88
|
+
test_consume_invalid_message(MyConsumer, { 'invalid' => 'key' })
|
89
89
|
end
|
90
90
|
|
91
91
|
it 'should fail on message with extra fields' do
|
92
92
|
test_consume_invalid_message(MyConsumer,
|
93
|
-
{'test_id' => 'foo',
|
93
|
+
{ 'test_id' => 'foo',
|
94
94
|
'some_int' => 123,
|
95
|
-
'extra_field' => 'field name'})
|
95
|
+
'extra_field' => 'field name' })
|
96
96
|
end
|
97
97
|
|
98
98
|
it 'should not fail when before_consume fails without reraising errors' do
|
@@ -189,10 +189,10 @@ module ConsumerTest
|
|
189
189
|
it 'should consume a message' do
|
190
190
|
expect(Deimos.config.metrics).to receive(:histogram).twice
|
191
191
|
test_consume_message('my_consume_topic',
|
192
|
-
{'test_id' => 'foo',
|
192
|
+
{ 'test_id' => 'foo',
|
193
193
|
'some_int' => 123,
|
194
194
|
'updated_at' => Time.now.to_i,
|
195
|
-
'timestamp' => 2.minutes.ago.to_s}) do |payload, _metadata|
|
195
|
+
'timestamp' => 2.minutes.ago.to_s }) do |payload, _metadata|
|
196
196
|
expect(payload['test_id']).to eq('foo')
|
197
197
|
end
|
198
198
|
end
|
@@ -200,17 +200,17 @@ module ConsumerTest
|
|
200
200
|
it 'should fail nicely when timestamp wrong format' do
|
201
201
|
expect(Deimos.config.metrics).to receive(:histogram).twice
|
202
202
|
test_consume_message('my_consume_topic',
|
203
|
-
{'test_id' => 'foo',
|
203
|
+
{ 'test_id' => 'foo',
|
204
204
|
'some_int' => 123,
|
205
205
|
'updated_at' => Time.now.to_i,
|
206
|
-
'timestamp' => 'dffdf'}) do |payload, _metadata|
|
206
|
+
'timestamp' => 'dffdf' }) do |payload, _metadata|
|
207
207
|
expect(payload['test_id']).to eq('foo')
|
208
208
|
end
|
209
209
|
test_consume_message('my_consume_topic',
|
210
|
-
{'test_id' => 'foo',
|
210
|
+
{ 'test_id' => 'foo',
|
211
211
|
'some_int' => 123,
|
212
212
|
'updated_at' => Time.now.to_i,
|
213
|
-
'timestamp' => ''}) do |payload, _metadata|
|
213
|
+
'timestamp' => '' }) do |payload, _metadata|
|
214
214
|
expect(payload['test_id']).to eq('foo')
|
215
215
|
end
|
216
216
|
end
|
@@ -2,15 +2,15 @@ RSpec.describe Schemas::MyNamespace::MySchema do
|
|
2
2
|
let(:key) { Schemas::MyNamespace::MySchemaKey.new(test_id: 123) }
|
3
3
|
|
4
4
|
it 'should produce a tombstone with a hash' do
|
5
|
-
result = described_class.tombstone({test_id: 123})
|
5
|
+
result = described_class.tombstone({ test_id: 123 })
|
6
6
|
expect(result.payload_key).to eq(key)
|
7
|
-
expect(result.to_h).to eq({ payload_key: { 'test_id' => 123}})
|
7
|
+
expect(result.to_h).to eq({ payload_key: { 'test_id' => 123 } })
|
8
8
|
end
|
9
9
|
|
10
10
|
it 'should work with a record' do
|
11
11
|
key = Schemas::MyNamespace::MySchemaKey.new(test_id: 123)
|
12
12
|
result = described_class.tombstone(key)
|
13
13
|
expect(result.payload_key).to eq(key)
|
14
|
-
expect(result.to_h).to eq({ payload_key: { 'test_id' => 123}})
|
14
|
+
expect(result.to_h).to eq({ payload_key: { 'test_id' => 123 } })
|
15
15
|
end
|
16
16
|
end
|
@@ -55,7 +55,7 @@ RSpec.describe Schemas::MyNamespace::MySchemaWithComplexType do
|
|
55
55
|
it 'should return a tombstone' do
|
56
56
|
record = described_class.tombstone('foo')
|
57
57
|
expect(record.tombstone_key).to eq('foo')
|
58
|
-
expect(record.to_h).to eq({payload_key: 'foo'})
|
58
|
+
expect(record.to_h).to eq({ payload_key: 'foo' })
|
59
59
|
end
|
60
60
|
end
|
61
61
|
|
data/spec/producer_spec.rb
CHANGED
@@ -79,7 +79,7 @@ module ProducerTest
|
|
79
79
|
expect(event.payload[:payloads]).to eq([{ 'invalid' => 'key' }])
|
80
80
|
end
|
81
81
|
expect(MyProducer.encoder).to receive(:validate).and_raise('OH NOES')
|
82
|
-
expect { MyProducer.publish({'invalid' => 'key', :payload_key => 'key'}) }.
|
82
|
+
expect { MyProducer.publish({ 'invalid' => 'key', :payload_key => 'key' }) }.
|
83
83
|
to raise_error('OH NOES')
|
84
84
|
Deimos.unsubscribe(subscriber)
|
85
85
|
end
|
@@ -9,7 +9,7 @@ module Schemas; module MyNamespace
|
|
9
9
|
### Secondary Schema Classes ###
|
10
10
|
# Autogenerated Schema for Record at com.my-namespace.ARecord
|
11
11
|
class ARecord < Deimos::SchemaClass::Record
|
12
|
-
|
12
|
+
|
13
13
|
### Attribute Accessors ###
|
14
14
|
# @return [String]
|
15
15
|
attr_accessor :a_record_field
|
@@ -150,8 +150,8 @@ module Schemas; module MyNamespace
|
|
150
150
|
test_string_array: ["test"],
|
151
151
|
test_int_array: [123],
|
152
152
|
test_optional_int: 123,
|
153
|
-
some_integer_map: {"abc"=>123},
|
154
|
-
some_record: {"a_record_field"=>"Test String"},
|
153
|
+
some_integer_map: { "abc"=>123 },
|
154
|
+
some_record: { "a_record_field"=>"Test String" },
|
155
155
|
some_optional_record: nil,
|
156
156
|
some_record_array: nil,
|
157
157
|
some_record_map: nil,
|
@@ -1,5 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
# rubocop:disable Layout/LineLength
|
4
|
+
|
3
5
|
# @param seconds [Integer]
|
4
6
|
# @return [Time]
|
5
7
|
def time_value(secs: 0, mins: 0)
|
@@ -156,16 +158,56 @@ each_db_config(Deimos::Utils::DbPoller) do
|
|
156
158
|
expect(poller.should_run?).to eq(false)
|
157
159
|
end
|
158
160
|
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
161
|
+
describe '#process_batch' do
|
162
|
+
let(:widgets) { (1..3).map { Widget.create!(test_id: 'some_id', some_int: 4) } }
|
163
|
+
|
164
|
+
before(:each) do
|
165
|
+
allow(Deimos.config.tracer).to receive(:start).and_return('a span')
|
166
|
+
allow(Deimos.config.tracer).to receive(:set_error)
|
167
|
+
allow(Deimos.config.tracer).to receive(:finish)
|
168
|
+
end
|
169
|
+
|
170
|
+
it 'should process the batch' do
|
171
|
+
travel_to time_value
|
172
|
+
widgets.last.update_attribute(:updated_at, time_value(mins: -30))
|
173
|
+
expect(MyProducer).to receive(:send_events).with(widgets)
|
174
|
+
poller.retrieve_poll_info
|
175
|
+
poller.process_batch(widgets)
|
176
|
+
info = Deimos::PollInfo.last
|
177
|
+
expect(info.last_sent.in_time_zone).to eq(time_value(mins: -30))
|
178
|
+
expect(info.last_sent_id).to eq(widgets.last.id)
|
179
|
+
end
|
180
|
+
|
181
|
+
it 'should create a span' do
|
182
|
+
poller.retrieve_poll_info
|
183
|
+
poller.process_batch_with_span(widgets)
|
184
|
+
expect(Deimos.config.tracer).to have_received(:finish).with('a span')
|
185
|
+
end
|
186
|
+
|
187
|
+
it 'should retry on Kafka error' do
|
188
|
+
called_once = false
|
189
|
+
allow(poller).to receive(:sleep)
|
190
|
+
allow(poller).to receive(:process_batch) do
|
191
|
+
unless called_once
|
192
|
+
called_once = true
|
193
|
+
raise Kafka::Error, 'OH NOES'
|
194
|
+
end
|
195
|
+
end
|
196
|
+
poller.retrieve_poll_info
|
197
|
+
poller.process_batch_with_span(widgets)
|
198
|
+
expect(poller).to have_received(:sleep).once.with(0.5)
|
199
|
+
expect(Deimos.config.tracer).to have_received(:finish).with('a span')
|
200
|
+
end
|
201
|
+
|
202
|
+
it 'should retry only once on other errors' do
|
203
|
+
error = RuntimeError.new('OH NOES')
|
204
|
+
allow(poller).to receive(:sleep)
|
205
|
+
allow(poller).to receive(:process_batch).and_raise(error)
|
206
|
+
poller.retrieve_poll_info
|
207
|
+
poller.process_batch_with_span(widgets)
|
208
|
+
expect(poller).to have_received(:sleep).once.with(0.5)
|
209
|
+
expect(Deimos.config.tracer).to have_received(:set_error).with('a span', error)
|
210
|
+
end
|
169
211
|
end
|
170
212
|
|
171
213
|
describe '#process_updates' do
|
@@ -234,6 +276,7 @@ each_db_config(Deimos::Utils::DbPoller) do
|
|
234
276
|
end
|
235
277
|
|
236
278
|
it 'should send events across multiple batches' do
|
279
|
+
allow(Deimos.config.logger).to receive(:info)
|
237
280
|
allow(MyProducer).to receive(:poll_query).and_call_original
|
238
281
|
expect(poller).to receive(:process_batch).ordered.
|
239
282
|
with([widgets[0], widgets[1], widgets[2]]).and_call_original
|
@@ -273,48 +316,48 @@ each_db_config(Deimos::Utils::DbPoller) do
|
|
273
316
|
time_to: time_value(secs: 120), # plus 122 seconds minus 2 seconds
|
274
317
|
column_name: :updated_at,
|
275
318
|
min_id: last_widget.id)
|
319
|
+
expect(Deimos.config.logger).to have_received(:info).
|
320
|
+
with('Poll my-topic-with-id complete at 2015-05-05 00:59:58 -0400 (7 messages, 3 successful batches, 0 batches errored}')
|
276
321
|
end
|
277
322
|
|
278
|
-
|
279
|
-
|
280
|
-
|
323
|
+
describe 'errors' do
|
324
|
+
before(:each) do
|
325
|
+
poller.config.retries = 0
|
326
|
+
allow(Deimos.config.logger).to receive(:info)
|
281
327
|
end
|
282
|
-
allow(MyProducer).to receive(:poll_query).and_call_original
|
283
|
-
expect(poller).to receive(:process_batch).ordered.
|
284
|
-
with([widgets[0], widgets[1], widgets[2]]).and_call_original
|
285
|
-
expect(poller).to receive(:process_batch).ordered.
|
286
|
-
with([widgets[3], widgets[4], widgets[5]]).and_raise('OH NOES')
|
287
|
-
|
288
|
-
expect { poller.process_updates }.to raise_exception('OH NOES')
|
289
|
-
|
290
|
-
expect(MyProducer).to have_received(:poll_query).
|
291
|
-
with(time_from: time_value(mins: -61),
|
292
|
-
time_to: time_value(secs: -2),
|
293
|
-
column_name: :updated_at,
|
294
|
-
min_id: 0)
|
295
|
-
|
296
|
-
info = Deimos::PollInfo.last
|
297
|
-
expect(info.last_sent.in_time_zone).to eq(time_value(mins: -61, secs: 30))
|
298
|
-
expect(info.last_sent_id).to eq(widgets[2].id)
|
299
328
|
|
300
|
-
|
301
|
-
|
302
|
-
|
303
|
-
with([widgets[3], widgets[4], widgets[5]]).and_call_original
|
304
|
-
expect(poller).to receive(:process_batch).with([widgets[6], last_widget]).
|
305
|
-
and_call_original
|
306
|
-
poller.process_updates
|
307
|
-
expect(MyProducer).to have_received(:poll_query).
|
308
|
-
with(time_from: time_value(mins: -61, secs: 30),
|
309
|
-
time_to: time_value(secs: 59),
|
310
|
-
column_name: :updated_at,
|
311
|
-
min_id: widgets[2].id)
|
329
|
+
after(:each) do
|
330
|
+
poller.config.retries = 1
|
331
|
+
end
|
312
332
|
|
313
|
-
|
314
|
-
|
333
|
+
it 'should recover correctly with errors and save the right ID' do
|
334
|
+
widgets.each do |w|
|
335
|
+
w.update_attribute(:updated_at, time_value(mins: -61, secs: 30))
|
336
|
+
end
|
337
|
+
allow(MyProducer).to receive(:poll_query).and_call_original
|
338
|
+
expect(poller).to receive(:process_batch).ordered.
|
339
|
+
with([widgets[0], widgets[1], widgets[2]]).and_call_original
|
340
|
+
expect(poller).to receive(:process_batch).ordered.
|
341
|
+
with([widgets[3], widgets[4], widgets[5]]).and_raise('OH NOES')
|
342
|
+
expect(poller).to receive(:process_batch).ordered.
|
343
|
+
with([widgets[6]]).and_call_original
|
344
|
+
|
345
|
+
poller.process_updates
|
346
|
+
|
347
|
+
expect(MyProducer).to have_received(:poll_query).
|
348
|
+
with(time_from: time_value(mins: -61),
|
349
|
+
time_to: time_value(secs: -2),
|
350
|
+
column_name: :updated_at,
|
351
|
+
min_id: 0)
|
352
|
+
|
353
|
+
info = Deimos::PollInfo.last
|
354
|
+
expect(info.last_sent.in_time_zone).to eq(time_value(mins: -61, secs: 30))
|
355
|
+
expect(info.last_sent_id).to eq(widgets[6].id)
|
356
|
+
expect(Deimos.config.logger).to have_received(:info).
|
357
|
+
with('Poll my-topic-with-id complete at 2015-05-05 00:59:58 -0400 (7 messages, 2 successful batches, 1 batches errored}')
|
358
|
+
end
|
315
359
|
end
|
316
|
-
|
317
360
|
end
|
318
|
-
|
319
361
|
end
|
320
362
|
end
|
363
|
+
# rubocop:enable Layout/LineLength
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: deimos-ruby
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.17.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Daniel Orner
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2022-
|
11
|
+
date: 2022-10-19 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: avro_turf
|
@@ -16,7 +16,7 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: '
|
19
|
+
version: '1.4'
|
20
20
|
- - "<"
|
21
21
|
- !ruby/object:Gem::Version
|
22
22
|
version: '2'
|
@@ -26,10 +26,24 @@ dependencies:
|
|
26
26
|
requirements:
|
27
27
|
- - ">="
|
28
28
|
- !ruby/object:Gem::Version
|
29
|
-
version: '
|
29
|
+
version: '1.4'
|
30
30
|
- - "<"
|
31
31
|
- !ruby/object:Gem::Version
|
32
32
|
version: '2'
|
33
|
+
- !ruby/object:Gem::Dependency
|
34
|
+
name: fig_tree
|
35
|
+
requirement: !ruby/object:Gem::Requirement
|
36
|
+
requirements:
|
37
|
+
- - "~>"
|
38
|
+
- !ruby/object:Gem::Version
|
39
|
+
version: 0.0.2
|
40
|
+
type: :runtime
|
41
|
+
prerelease: false
|
42
|
+
version_requirements: !ruby/object:Gem::Requirement
|
43
|
+
requirements:
|
44
|
+
- - "~>"
|
45
|
+
- !ruby/object:Gem::Version
|
46
|
+
version: 0.0.2
|
33
47
|
- !ruby/object:Gem::Dependency
|
34
48
|
name: phobos
|
35
49
|
requirement: !ruby/object:Gem::Requirement
|
@@ -84,20 +98,6 @@ dependencies:
|
|
84
98
|
- - "<"
|
85
99
|
- !ruby/object:Gem::Version
|
86
100
|
version: '1.0'
|
87
|
-
- !ruby/object:Gem::Dependency
|
88
|
-
name: fig_tree
|
89
|
-
requirement: !ruby/object:Gem::Requirement
|
90
|
-
requirements:
|
91
|
-
- - "~>"
|
92
|
-
- !ruby/object:Gem::Version
|
93
|
-
version: 0.0.2
|
94
|
-
type: :runtime
|
95
|
-
prerelease: false
|
96
|
-
version_requirements: !ruby/object:Gem::Requirement
|
97
|
-
requirements:
|
98
|
-
- - "~>"
|
99
|
-
- !ruby/object:Gem::Version
|
100
|
-
version: 0.0.2
|
101
101
|
- !ruby/object:Gem::Dependency
|
102
102
|
name: activerecord-import
|
103
103
|
requirement: !ruby/object:Gem::Requirement
|
@@ -364,6 +364,20 @@ dependencies:
|
|
364
364
|
- - "~>"
|
365
365
|
- !ruby/object:Gem::Version
|
366
366
|
version: '1.3'
|
367
|
+
- !ruby/object:Gem::Dependency
|
368
|
+
name: steep
|
369
|
+
requirement: !ruby/object:Gem::Requirement
|
370
|
+
requirements:
|
371
|
+
- - "~>"
|
372
|
+
- !ruby/object:Gem::Version
|
373
|
+
version: '1.0'
|
374
|
+
type: :development
|
375
|
+
prerelease: false
|
376
|
+
version_requirements: !ruby/object:Gem::Requirement
|
377
|
+
requirements:
|
378
|
+
- - "~>"
|
379
|
+
- !ruby/object:Gem::Version
|
380
|
+
version: '1.0'
|
367
381
|
description:
|
368
382
|
email:
|
369
383
|
- daniel.orner@wishabi.com
|
@@ -374,14 +388,12 @@ extensions: []
|
|
374
388
|
extra_rdoc_files: []
|
375
389
|
files:
|
376
390
|
- ".circleci/config.yml"
|
377
|
-
- ".gemfiles/avro_turf-0.gemfile"
|
378
|
-
- ".gemfiles/avro_turf-1.gemfile"
|
379
391
|
- ".github/workflows/ci.yml"
|
380
392
|
- ".gitignore"
|
381
393
|
- ".gitmodules"
|
382
394
|
- ".rspec"
|
383
395
|
- ".rubocop.yml"
|
384
|
-
- ".
|
396
|
+
- ".rubocop_todo.yml"
|
385
397
|
- ".ruby-version"
|
386
398
|
- CHANGELOG.md
|
387
399
|
- CODE_OF_CONDUCT.md
|
@@ -391,6 +403,7 @@ files:
|
|
391
403
|
- LICENSE.md
|
392
404
|
- README.md
|
393
405
|
- Rakefile
|
406
|
+
- Steepfile
|
394
407
|
- bin/console
|
395
408
|
- bin/deimos
|
396
409
|
- deimos-ruby.gemspec
|
@@ -594,7 +607,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
594
607
|
- !ruby/object:Gem::Version
|
595
608
|
version: '0'
|
596
609
|
requirements: []
|
597
|
-
rubygems_version: 3.3.
|
610
|
+
rubygems_version: 3.3.20
|
598
611
|
signing_key:
|
599
612
|
specification_version: 4
|
600
613
|
summary: Kafka libraries for Ruby.
|
data/.ruby-gemset
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
deimos
|