deimos-ruby 1.24.3 → 2.0.0.pre.alpha1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop_todo.yml +0 -17
- data/.tool-versions +1 -0
- data/CHANGELOG.md +1 -1
- data/README.md +287 -498
- data/deimos-ruby.gemspec +4 -4
- data/docs/CONFIGURATION.md +133 -227
- data/docs/UPGRADING.md +237 -0
- data/lib/deimos/active_record_consume/batch_consumption.rb +28 -29
- data/lib/deimos/active_record_consume/message_consumption.rb +15 -21
- data/lib/deimos/active_record_consumer.rb +36 -26
- data/lib/deimos/active_record_producer.rb +28 -9
- data/lib/deimos/backends/base.rb +4 -35
- data/lib/deimos/backends/kafka.rb +6 -22
- data/lib/deimos/backends/kafka_async.rb +6 -22
- data/lib/deimos/backends/{db.rb → outbox.rb} +13 -9
- data/lib/deimos/config/configuration.rb +116 -385
- data/lib/deimos/consume/batch_consumption.rb +24 -124
- data/lib/deimos/consume/message_consumption.rb +36 -63
- data/lib/deimos/consumer.rb +16 -75
- data/lib/deimos/ext/consumer_route.rb +35 -0
- data/lib/deimos/ext/producer_middleware.rb +94 -0
- data/lib/deimos/ext/producer_route.rb +22 -0
- data/lib/deimos/ext/redraw.rb +29 -0
- data/lib/deimos/ext/routing_defaults.rb +72 -0
- data/lib/deimos/ext/schema_route.rb +70 -0
- data/lib/deimos/kafka_message.rb +2 -2
- data/lib/deimos/kafka_source.rb +2 -7
- data/lib/deimos/kafka_topic_info.rb +1 -1
- data/lib/deimos/logging.rb +71 -0
- data/lib/deimos/message.rb +2 -11
- data/lib/deimos/metrics/datadog.rb +40 -1
- data/lib/deimos/metrics/provider.rb +4 -4
- data/lib/deimos/producer.rb +39 -116
- data/lib/deimos/railtie.rb +6 -0
- data/lib/deimos/schema_backends/avro_base.rb +21 -21
- data/lib/deimos/schema_backends/avro_schema_registry.rb +1 -2
- data/lib/deimos/schema_backends/avro_validation.rb +2 -2
- data/lib/deimos/schema_backends/base.rb +19 -12
- data/lib/deimos/schema_backends/mock.rb +6 -1
- data/lib/deimos/schema_backends/plain.rb +47 -0
- data/lib/deimos/schema_class/base.rb +2 -2
- data/lib/deimos/schema_class/enum.rb +1 -1
- data/lib/deimos/schema_class/record.rb +2 -2
- data/lib/deimos/test_helpers.rb +95 -320
- data/lib/deimos/tracing/provider.rb +6 -6
- data/lib/deimos/transcoder.rb +88 -0
- data/lib/deimos/utils/db_poller/base.rb +16 -14
- data/lib/deimos/utils/db_poller/state_based.rb +3 -3
- data/lib/deimos/utils/db_poller/time_based.rb +4 -4
- data/lib/deimos/utils/db_poller.rb +1 -1
- data/lib/deimos/utils/deadlock_retry.rb +1 -1
- data/lib/deimos/utils/{db_producer.rb → outbox_producer.rb} +16 -47
- data/lib/deimos/utils/schema_class.rb +0 -7
- data/lib/deimos/version.rb +1 -1
- data/lib/deimos.rb +79 -26
- data/lib/generators/deimos/{db_backend_generator.rb → outbox_backend_generator.rb} +4 -4
- data/lib/generators/deimos/schema_class_generator.rb +0 -1
- data/lib/generators/deimos/v2/templates/karafka.rb.tt +149 -0
- data/lib/generators/deimos/v2_generator.rb +193 -0
- data/lib/tasks/deimos.rake +5 -7
- data/spec/active_record_batch_consumer_association_spec.rb +22 -13
- data/spec/active_record_batch_consumer_spec.rb +84 -65
- data/spec/active_record_consume/batch_consumption_spec.rb +10 -10
- data/spec/active_record_consume/batch_slicer_spec.rb +12 -12
- data/spec/active_record_consumer_spec.rb +29 -13
- data/spec/active_record_producer_spec.rb +36 -26
- data/spec/backends/base_spec.rb +0 -23
- data/spec/backends/kafka_async_spec.rb +1 -3
- data/spec/backends/kafka_spec.rb +1 -3
- data/spec/backends/{db_spec.rb → outbox_spec.rb} +14 -20
- data/spec/batch_consumer_spec.rb +66 -116
- data/spec/consumer_spec.rb +53 -147
- data/spec/deimos_spec.rb +10 -126
- data/spec/kafka_source_spec.rb +19 -52
- data/spec/karafka/karafka.rb +69 -0
- data/spec/karafka_config/karafka_spec.rb +97 -0
- data/spec/logging_spec.rb +25 -0
- data/spec/message_spec.rb +9 -9
- data/spec/producer_spec.rb +112 -254
- data/spec/rake_spec.rb +1 -3
- data/spec/schema_backends/avro_validation_spec.rb +1 -1
- data/spec/schemas/com/my-namespace/MySchemaWithTitle.avsc +22 -0
- data/spec/snapshots/consumers-no-nest.snap +49 -0
- data/spec/snapshots/consumers.snap +49 -0
- data/spec/snapshots/consumers_and_producers-no-nest.snap +49 -0
- data/spec/snapshots/consumers_and_producers.snap +49 -0
- data/spec/snapshots/consumers_circular-no-nest.snap +49 -0
- data/spec/snapshots/consumers_circular.snap +49 -0
- data/spec/snapshots/consumers_complex_types-no-nest.snap +49 -0
- data/spec/snapshots/consumers_complex_types.snap +49 -0
- data/spec/snapshots/consumers_nested-no-nest.snap +49 -0
- data/spec/snapshots/consumers_nested.snap +49 -0
- data/spec/snapshots/namespace_folders.snap +49 -0
- data/spec/snapshots/namespace_map.snap +49 -0
- data/spec/snapshots/producers_with_key-no-nest.snap +49 -0
- data/spec/snapshots/producers_with_key.snap +49 -0
- data/spec/spec_helper.rb +61 -29
- data/spec/utils/db_poller_spec.rb +49 -39
- data/spec/utils/{db_producer_spec.rb → outbox_producer_spec.rb} +17 -184
- metadata +58 -67
- data/lib/deimos/batch_consumer.rb +0 -7
- data/lib/deimos/config/phobos_config.rb +0 -164
- data/lib/deimos/instrumentation.rb +0 -95
- data/lib/deimos/monkey_patches/phobos_cli.rb +0 -35
- data/lib/deimos/utils/inline_consumer.rb +0 -158
- data/lib/deimos/utils/lag_reporter.rb +0 -186
- data/lib/deimos/utils/schema_controller_mixin.rb +0 -129
- data/spec/config/configuration_spec.rb +0 -329
- data/spec/kafka_listener_spec.rb +0 -55
- data/spec/phobos.bad_db.yml +0 -73
- data/spec/phobos.yml +0 -77
- data/spec/utils/inline_consumer_spec.rb +0 -31
- data/spec/utils/lag_reporter_spec.rb +0 -76
- data/spec/utils/platform_schema_validation_spec.rb +0 -0
- data/spec/utils/schema_controller_mixin_spec.rb +0 -84
- /data/lib/generators/deimos/{db_backend → outbox_backend}/templates/migration +0 -0
- /data/lib/generators/deimos/{db_backend → outbox_backend}/templates/rails3_migration +0 -0
@@ -0,0 +1,97 @@
|
|
1
|
+
RSpec.describe 'Karafka configs' do
|
2
|
+
before(:each) do
|
3
|
+
KarafkaApp.routes.clear
|
4
|
+
$found_stuff = nil
|
5
|
+
end
|
6
|
+
|
7
|
+
let(:consumer_class) do
|
8
|
+
Class.new(Deimos::Consumer) do
|
9
|
+
def consume_message(message)
|
10
|
+
$found_stuff = message.payload
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
let(:producer_class) do
|
16
|
+
Class.new(Deimos::Producer) do
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
describe 'producers' do
|
21
|
+
before(:each) do
|
22
|
+
stub_const('MyProducer', producer_class)
|
23
|
+
end
|
24
|
+
|
25
|
+
it 'should work with key none' do
|
26
|
+
KarafkaApp.routes.draw do
|
27
|
+
topic 'MyTopic' do
|
28
|
+
producer_class MyProducer
|
29
|
+
schema 'MySchema'
|
30
|
+
namespace 'com.my-namespace'
|
31
|
+
key_config(none: true)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
producer_class.publish({test_id: "id1", some_int: 5})
|
35
|
+
expect('MyTopic').to have_sent({test_id: "id1", some_int: 5})
|
36
|
+
end
|
37
|
+
|
38
|
+
it 'should work with key plain' do
|
39
|
+
KarafkaApp.routes.draw do
|
40
|
+
topic 'MyTopic' do
|
41
|
+
producer_class MyProducer
|
42
|
+
schema 'MySchema'
|
43
|
+
namespace 'com.my-namespace'
|
44
|
+
key_config({plain: true})
|
45
|
+
end
|
46
|
+
end
|
47
|
+
producer_class.publish({test_id: "id1", some_int: 5, payload_key: 'key'})
|
48
|
+
expect('MyTopic').to have_sent({test_id: "id1", some_int: 5}, 'key')
|
49
|
+
end
|
50
|
+
|
51
|
+
it 'should work with key field' do
|
52
|
+
KarafkaApp.routes.draw do
|
53
|
+
topic 'MyTopic' do
|
54
|
+
producer_class MyProducer
|
55
|
+
schema 'MySchema'
|
56
|
+
namespace 'com.my-namespace'
|
57
|
+
key_config({field: :test_id})
|
58
|
+
end
|
59
|
+
end
|
60
|
+
producer_class.publish({test_id: "id1", some_int: 5})
|
61
|
+
expect('MyTopic').to have_sent({test_id: "id1", some_int: 5}, 'id1')
|
62
|
+
end
|
63
|
+
|
64
|
+
it 'should work with key schema' do
|
65
|
+
KarafkaApp.routes.draw do
|
66
|
+
topic 'MyTopic' do
|
67
|
+
producer_class MyProducer
|
68
|
+
schema 'MySchema'
|
69
|
+
namespace 'com.my-namespace'
|
70
|
+
key_config({schema: 'MySchema_key'})
|
71
|
+
end
|
72
|
+
end
|
73
|
+
producer_class.publish({test_id: "id1", some_int: 5, payload_key: {test_id: 'id3'}})
|
74
|
+
expect('MyTopic').to have_sent({test_id: "id1", some_int: 5}, { test_id: 'id3'})
|
75
|
+
end
|
76
|
+
|
77
|
+
end
|
78
|
+
|
79
|
+
it 'should be able to pick up a consumer' do
|
80
|
+
stub_const('MyConsumer', consumer_class)
|
81
|
+
KarafkaApp.routes.draw do
|
82
|
+
topic 'MyTopic' do
|
83
|
+
consumer MyConsumer
|
84
|
+
schema 'MySchema'
|
85
|
+
namespace 'com.my-namespace'
|
86
|
+
key_config({field: :test_id})
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
test_consume_message('MyTopic', {test_id: "id1", some_int: 5}, key: "id1")
|
91
|
+
expect($found_stuff).to eq({'test_id' => "id1", 'some_int' => 5})
|
92
|
+
$found_stuff = nil
|
93
|
+
test_consume_message(MyConsumer, {test_id: "id1", some_int: 5}, key: "id1")
|
94
|
+
expect($found_stuff).to eq({'test_id' => "id1", 'some_int' => 5})
|
95
|
+
end
|
96
|
+
|
97
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
RSpec.describe Deimos::Logging do
|
2
|
+
include_context 'with publish_backend'
|
3
|
+
describe '#messages_log_text' do
|
4
|
+
it 'should return whole payload (default behavior)' do
|
5
|
+
log_message = described_class.messages_log_text(:payloads, messages)
|
6
|
+
expect(log_message[:payloads].count).to eq(3)
|
7
|
+
expect(log_message[:payloads].first[:payload]).to eq({ some_int: 1, test_id: 'foo1' })
|
8
|
+
expect(log_message[:payloads].first[:key]).to eq('foo1')
|
9
|
+
end
|
10
|
+
|
11
|
+
it 'should return only keys of messages' do
|
12
|
+
log_message = described_class.messages_log_text(:keys, messages)
|
13
|
+
expect(log_message[:payload_keys].count).to eq(3)
|
14
|
+
expect(log_message[:payload_keys]).to be_a(Array)
|
15
|
+
expect(log_message[:payload_keys].first).to eq('foo1')
|
16
|
+
end
|
17
|
+
|
18
|
+
it 'should return only messages count' do
|
19
|
+
log_message = described_class.messages_log_text(:count, messages)
|
20
|
+
expect(log_message[:payloads_count]).to be_a(Integer)
|
21
|
+
expect(log_message[:payloads_count]).to eq(3)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
end
|
data/spec/message_spec.rb
CHANGED
@@ -2,37 +2,37 @@
|
|
2
2
|
|
3
3
|
RSpec.describe(Deimos::Message) do
|
4
4
|
it 'should detect tombstones' do
|
5
|
-
expect(described_class.new(nil,
|
5
|
+
expect(described_class.new(nil, key: 'key1')).
|
6
6
|
to be_tombstone
|
7
|
-
expect(described_class.new({ v: 'val1' },
|
7
|
+
expect(described_class.new({ v: 'val1' }, key: 'key1')).
|
8
8
|
not_to be_tombstone
|
9
|
-
expect(described_class.new({ v: '' },
|
9
|
+
expect(described_class.new({ v: '' }, key: 'key1')).
|
10
10
|
not_to be_tombstone
|
11
|
-
expect(described_class.new({ v: 'val1' },
|
11
|
+
expect(described_class.new({ v: 'val1' }, key: nil)).
|
12
12
|
not_to be_tombstone
|
13
13
|
end
|
14
14
|
|
15
15
|
it 'can support complex keys/values' do
|
16
|
-
expect { described_class.new({ a: 1, b: 2 },
|
16
|
+
expect { described_class.new({ a: 1, b: 2 }, key: { c: 3, d: 4 }) }.
|
17
17
|
not_to raise_exception
|
18
18
|
end
|
19
19
|
|
20
20
|
describe 'headers' do
|
21
21
|
it 'returns nil when not set' do
|
22
|
-
expect(described_class.new({ v: 'val1' },
|
22
|
+
expect(described_class.new({ v: 'val1' }, key: 'key1')).
|
23
23
|
to have_attributes(headers: nil)
|
24
24
|
end
|
25
25
|
|
26
26
|
it 'can set and get headers' do
|
27
|
-
expect(described_class.new({ v: 'val1' },
|
27
|
+
expect(described_class.new({ v: 'val1' }, key: 'key1', headers: { a: 1 })).
|
28
28
|
to have_attributes(headers: { a: 1 })
|
29
29
|
end
|
30
30
|
|
31
31
|
it 'includes headers when converting to Hash' do
|
32
|
-
expect(described_class.new({ v: 'val1' },
|
32
|
+
expect(described_class.new({ v: 'val1' }, key: 'key1', headers: { a: 1 }).to_h).
|
33
33
|
to include(headers: { a: 1 })
|
34
34
|
|
35
|
-
expect(described_class.new({ v: 'val1' },
|
35
|
+
expect(described_class.new({ v: 'val1' }, key: 'key1', headers: { a: 1 }).encoded_hash).
|
36
36
|
to include(headers: { a: 1 })
|
37
37
|
end
|
38
38
|
end
|