deimos-ruby 1.24.3 → 2.0.0.pre.alpha1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (118) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop_todo.yml +0 -17
  3. data/.tool-versions +1 -0
  4. data/CHANGELOG.md +1 -1
  5. data/README.md +287 -498
  6. data/deimos-ruby.gemspec +4 -4
  7. data/docs/CONFIGURATION.md +133 -227
  8. data/docs/UPGRADING.md +237 -0
  9. data/lib/deimos/active_record_consume/batch_consumption.rb +28 -29
  10. data/lib/deimos/active_record_consume/message_consumption.rb +15 -21
  11. data/lib/deimos/active_record_consumer.rb +36 -26
  12. data/lib/deimos/active_record_producer.rb +28 -9
  13. data/lib/deimos/backends/base.rb +4 -35
  14. data/lib/deimos/backends/kafka.rb +6 -22
  15. data/lib/deimos/backends/kafka_async.rb +6 -22
  16. data/lib/deimos/backends/{db.rb → outbox.rb} +13 -9
  17. data/lib/deimos/config/configuration.rb +116 -385
  18. data/lib/deimos/consume/batch_consumption.rb +24 -124
  19. data/lib/deimos/consume/message_consumption.rb +36 -63
  20. data/lib/deimos/consumer.rb +16 -75
  21. data/lib/deimos/ext/consumer_route.rb +35 -0
  22. data/lib/deimos/ext/producer_middleware.rb +94 -0
  23. data/lib/deimos/ext/producer_route.rb +22 -0
  24. data/lib/deimos/ext/redraw.rb +29 -0
  25. data/lib/deimos/ext/routing_defaults.rb +72 -0
  26. data/lib/deimos/ext/schema_route.rb +70 -0
  27. data/lib/deimos/kafka_message.rb +2 -2
  28. data/lib/deimos/kafka_source.rb +2 -7
  29. data/lib/deimos/kafka_topic_info.rb +1 -1
  30. data/lib/deimos/logging.rb +71 -0
  31. data/lib/deimos/message.rb +2 -11
  32. data/lib/deimos/metrics/datadog.rb +40 -1
  33. data/lib/deimos/metrics/provider.rb +4 -4
  34. data/lib/deimos/producer.rb +39 -116
  35. data/lib/deimos/railtie.rb +6 -0
  36. data/lib/deimos/schema_backends/avro_base.rb +21 -21
  37. data/lib/deimos/schema_backends/avro_schema_registry.rb +1 -2
  38. data/lib/deimos/schema_backends/avro_validation.rb +2 -2
  39. data/lib/deimos/schema_backends/base.rb +19 -12
  40. data/lib/deimos/schema_backends/mock.rb +6 -1
  41. data/lib/deimos/schema_backends/plain.rb +47 -0
  42. data/lib/deimos/schema_class/base.rb +2 -2
  43. data/lib/deimos/schema_class/enum.rb +1 -1
  44. data/lib/deimos/schema_class/record.rb +2 -2
  45. data/lib/deimos/test_helpers.rb +95 -320
  46. data/lib/deimos/tracing/provider.rb +6 -6
  47. data/lib/deimos/transcoder.rb +88 -0
  48. data/lib/deimos/utils/db_poller/base.rb +16 -14
  49. data/lib/deimos/utils/db_poller/state_based.rb +3 -3
  50. data/lib/deimos/utils/db_poller/time_based.rb +4 -4
  51. data/lib/deimos/utils/db_poller.rb +1 -1
  52. data/lib/deimos/utils/deadlock_retry.rb +1 -1
  53. data/lib/deimos/utils/{db_producer.rb → outbox_producer.rb} +16 -47
  54. data/lib/deimos/utils/schema_class.rb +0 -7
  55. data/lib/deimos/version.rb +1 -1
  56. data/lib/deimos.rb +79 -26
  57. data/lib/generators/deimos/{db_backend_generator.rb → outbox_backend_generator.rb} +4 -4
  58. data/lib/generators/deimos/schema_class_generator.rb +0 -1
  59. data/lib/generators/deimos/v2/templates/karafka.rb.tt +149 -0
  60. data/lib/generators/deimos/v2_generator.rb +193 -0
  61. data/lib/tasks/deimos.rake +5 -7
  62. data/spec/active_record_batch_consumer_association_spec.rb +22 -13
  63. data/spec/active_record_batch_consumer_spec.rb +84 -65
  64. data/spec/active_record_consume/batch_consumption_spec.rb +10 -10
  65. data/spec/active_record_consume/batch_slicer_spec.rb +12 -12
  66. data/spec/active_record_consumer_spec.rb +29 -13
  67. data/spec/active_record_producer_spec.rb +36 -26
  68. data/spec/backends/base_spec.rb +0 -23
  69. data/spec/backends/kafka_async_spec.rb +1 -3
  70. data/spec/backends/kafka_spec.rb +1 -3
  71. data/spec/backends/{db_spec.rb → outbox_spec.rb} +14 -20
  72. data/spec/batch_consumer_spec.rb +66 -116
  73. data/spec/consumer_spec.rb +53 -147
  74. data/spec/deimos_spec.rb +10 -126
  75. data/spec/kafka_source_spec.rb +19 -52
  76. data/spec/karafka/karafka.rb +69 -0
  77. data/spec/karafka_config/karafka_spec.rb +97 -0
  78. data/spec/logging_spec.rb +25 -0
  79. data/spec/message_spec.rb +9 -9
  80. data/spec/producer_spec.rb +112 -254
  81. data/spec/rake_spec.rb +1 -3
  82. data/spec/schema_backends/avro_validation_spec.rb +1 -1
  83. data/spec/schemas/com/my-namespace/MySchemaWithTitle.avsc +22 -0
  84. data/spec/snapshots/consumers-no-nest.snap +49 -0
  85. data/spec/snapshots/consumers.snap +49 -0
  86. data/spec/snapshots/consumers_and_producers-no-nest.snap +49 -0
  87. data/spec/snapshots/consumers_and_producers.snap +49 -0
  88. data/spec/snapshots/consumers_circular-no-nest.snap +49 -0
  89. data/spec/snapshots/consumers_circular.snap +49 -0
  90. data/spec/snapshots/consumers_complex_types-no-nest.snap +49 -0
  91. data/spec/snapshots/consumers_complex_types.snap +49 -0
  92. data/spec/snapshots/consumers_nested-no-nest.snap +49 -0
  93. data/spec/snapshots/consumers_nested.snap +49 -0
  94. data/spec/snapshots/namespace_folders.snap +49 -0
  95. data/spec/snapshots/namespace_map.snap +49 -0
  96. data/spec/snapshots/producers_with_key-no-nest.snap +49 -0
  97. data/spec/snapshots/producers_with_key.snap +49 -0
  98. data/spec/spec_helper.rb +61 -29
  99. data/spec/utils/db_poller_spec.rb +49 -39
  100. data/spec/utils/{db_producer_spec.rb → outbox_producer_spec.rb} +17 -184
  101. metadata +58 -67
  102. data/lib/deimos/batch_consumer.rb +0 -7
  103. data/lib/deimos/config/phobos_config.rb +0 -164
  104. data/lib/deimos/instrumentation.rb +0 -95
  105. data/lib/deimos/monkey_patches/phobos_cli.rb +0 -35
  106. data/lib/deimos/utils/inline_consumer.rb +0 -158
  107. data/lib/deimos/utils/lag_reporter.rb +0 -186
  108. data/lib/deimos/utils/schema_controller_mixin.rb +0 -129
  109. data/spec/config/configuration_spec.rb +0 -329
  110. data/spec/kafka_listener_spec.rb +0 -55
  111. data/spec/phobos.bad_db.yml +0 -73
  112. data/spec/phobos.yml +0 -77
  113. data/spec/utils/inline_consumer_spec.rb +0 -31
  114. data/spec/utils/lag_reporter_spec.rb +0 -76
  115. data/spec/utils/platform_schema_validation_spec.rb +0 -0
  116. data/spec/utils/schema_controller_mixin_spec.rb +0 -84
  117. /data/lib/generators/deimos/{db_backend → outbox_backend}/templates/migration +0 -0
  118. /data/lib/generators/deimos/{db_backend → outbox_backend}/templates/rails3_migration +0 -0
@@ -0,0 +1,97 @@
1
+ RSpec.describe 'Karafka configs' do
2
+ before(:each) do
3
+ KarafkaApp.routes.clear
4
+ $found_stuff = nil
5
+ end
6
+
7
+ let(:consumer_class) do
8
+ Class.new(Deimos::Consumer) do
9
+ def consume_message(message)
10
+ $found_stuff = message.payload
11
+ end
12
+ end
13
+ end
14
+
15
+ let(:producer_class) do
16
+ Class.new(Deimos::Producer) do
17
+ end
18
+ end
19
+
20
+ describe 'producers' do
21
+ before(:each) do
22
+ stub_const('MyProducer', producer_class)
23
+ end
24
+
25
+ it 'should work with key none' do
26
+ KarafkaApp.routes.draw do
27
+ topic 'MyTopic' do
28
+ producer_class MyProducer
29
+ schema 'MySchema'
30
+ namespace 'com.my-namespace'
31
+ key_config(none: true)
32
+ end
33
+ end
34
+ producer_class.publish({test_id: "id1", some_int: 5})
35
+ expect('MyTopic').to have_sent({test_id: "id1", some_int: 5})
36
+ end
37
+
38
+ it 'should work with key plain' do
39
+ KarafkaApp.routes.draw do
40
+ topic 'MyTopic' do
41
+ producer_class MyProducer
42
+ schema 'MySchema'
43
+ namespace 'com.my-namespace'
44
+ key_config({plain: true})
45
+ end
46
+ end
47
+ producer_class.publish({test_id: "id1", some_int: 5, payload_key: 'key'})
48
+ expect('MyTopic').to have_sent({test_id: "id1", some_int: 5}, 'key')
49
+ end
50
+
51
+ it 'should work with key field' do
52
+ KarafkaApp.routes.draw do
53
+ topic 'MyTopic' do
54
+ producer_class MyProducer
55
+ schema 'MySchema'
56
+ namespace 'com.my-namespace'
57
+ key_config({field: :test_id})
58
+ end
59
+ end
60
+ producer_class.publish({test_id: "id1", some_int: 5})
61
+ expect('MyTopic').to have_sent({test_id: "id1", some_int: 5}, 'id1')
62
+ end
63
+
64
+ it 'should work with key schema' do
65
+ KarafkaApp.routes.draw do
66
+ topic 'MyTopic' do
67
+ producer_class MyProducer
68
+ schema 'MySchema'
69
+ namespace 'com.my-namespace'
70
+ key_config({schema: 'MySchema_key'})
71
+ end
72
+ end
73
+ producer_class.publish({test_id: "id1", some_int: 5, payload_key: {test_id: 'id3'}})
74
+ expect('MyTopic').to have_sent({test_id: "id1", some_int: 5}, { test_id: 'id3'})
75
+ end
76
+
77
+ end
78
+
79
+ it 'should be able to pick up a consumer' do
80
+ stub_const('MyConsumer', consumer_class)
81
+ KarafkaApp.routes.draw do
82
+ topic 'MyTopic' do
83
+ consumer MyConsumer
84
+ schema 'MySchema'
85
+ namespace 'com.my-namespace'
86
+ key_config({field: :test_id})
87
+ end
88
+ end
89
+
90
+ test_consume_message('MyTopic', {test_id: "id1", some_int: 5}, key: "id1")
91
+ expect($found_stuff).to eq({'test_id' => "id1", 'some_int' => 5})
92
+ $found_stuff = nil
93
+ test_consume_message(MyConsumer, {test_id: "id1", some_int: 5}, key: "id1")
94
+ expect($found_stuff).to eq({'test_id' => "id1", 'some_int' => 5})
95
+ end
96
+
97
+ end
@@ -0,0 +1,25 @@
1
+ RSpec.describe Deimos::Logging do
2
+ include_context 'with publish_backend'
3
+ describe '#messages_log_text' do
4
+ it 'should return whole payload (default behavior)' do
5
+ log_message = described_class.messages_log_text(:payloads, messages)
6
+ expect(log_message[:payloads].count).to eq(3)
7
+ expect(log_message[:payloads].first[:payload]).to eq({ some_int: 1, test_id: 'foo1' })
8
+ expect(log_message[:payloads].first[:key]).to eq('foo1')
9
+ end
10
+
11
+ it 'should return only keys of messages' do
12
+ log_message = described_class.messages_log_text(:keys, messages)
13
+ expect(log_message[:payload_keys].count).to eq(3)
14
+ expect(log_message[:payload_keys]).to be_a(Array)
15
+ expect(log_message[:payload_keys].first).to eq('foo1')
16
+ end
17
+
18
+ it 'should return only messages count' do
19
+ log_message = described_class.messages_log_text(:count, messages)
20
+ expect(log_message[:payloads_count]).to be_a(Integer)
21
+ expect(log_message[:payloads_count]).to eq(3)
22
+ end
23
+ end
24
+
25
+ end
data/spec/message_spec.rb CHANGED
@@ -2,37 +2,37 @@
2
2
 
3
3
  RSpec.describe(Deimos::Message) do
4
4
  it 'should detect tombstones' do
5
- expect(described_class.new(nil, nil, key: 'key1')).
5
+ expect(described_class.new(nil, key: 'key1')).
6
6
  to be_tombstone
7
- expect(described_class.new({ v: 'val1' }, nil, key: 'key1')).
7
+ expect(described_class.new({ v: 'val1' }, key: 'key1')).
8
8
  not_to be_tombstone
9
- expect(described_class.new({ v: '' }, nil, key: 'key1')).
9
+ expect(described_class.new({ v: '' }, key: 'key1')).
10
10
  not_to be_tombstone
11
- expect(described_class.new({ v: 'val1' }, nil, key: nil)).
11
+ expect(described_class.new({ v: 'val1' }, key: nil)).
12
12
  not_to be_tombstone
13
13
  end
14
14
 
15
15
  it 'can support complex keys/values' do
16
- expect { described_class.new({ a: 1, b: 2 }, nil, key: { c: 3, d: 4 }) }.
16
+ expect { described_class.new({ a: 1, b: 2 }, key: { c: 3, d: 4 }) }.
17
17
  not_to raise_exception
18
18
  end
19
19
 
20
20
  describe 'headers' do
21
21
  it 'returns nil when not set' do
22
- expect(described_class.new({ v: 'val1' }, nil, key: 'key1')).
22
+ expect(described_class.new({ v: 'val1' }, key: 'key1')).
23
23
  to have_attributes(headers: nil)
24
24
  end
25
25
 
26
26
  it 'can set and get headers' do
27
- expect(described_class.new({ v: 'val1' }, nil, key: 'key1', headers: { a: 1 })).
27
+ expect(described_class.new({ v: 'val1' }, key: 'key1', headers: { a: 1 })).
28
28
  to have_attributes(headers: { a: 1 })
29
29
  end
30
30
 
31
31
  it 'includes headers when converting to Hash' do
32
- expect(described_class.new({ v: 'val1' }, nil, key: 'key1', headers: { a: 1 }).to_h).
32
+ expect(described_class.new({ v: 'val1' }, key: 'key1', headers: { a: 1 }).to_h).
33
33
  to include(headers: { a: 1 })
34
34
 
35
- expect(described_class.new({ v: 'val1' }, nil, key: 'key1', headers: { a: 1 }).encoded_hash).
35
+ expect(described_class.new({ v: 'val1' }, key: 'key1', headers: { a: 1 }).encoded_hash).
36
36
  to include(headers: { a: 1 })
37
37
  end
38
38
  end