deimos-ruby 1.24.3 → 2.0.0.pre.alpha1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (118) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop_todo.yml +0 -17
  3. data/.tool-versions +1 -0
  4. data/CHANGELOG.md +1 -1
  5. data/README.md +287 -498
  6. data/deimos-ruby.gemspec +4 -4
  7. data/docs/CONFIGURATION.md +133 -227
  8. data/docs/UPGRADING.md +237 -0
  9. data/lib/deimos/active_record_consume/batch_consumption.rb +28 -29
  10. data/lib/deimos/active_record_consume/message_consumption.rb +15 -21
  11. data/lib/deimos/active_record_consumer.rb +36 -26
  12. data/lib/deimos/active_record_producer.rb +28 -9
  13. data/lib/deimos/backends/base.rb +4 -35
  14. data/lib/deimos/backends/kafka.rb +6 -22
  15. data/lib/deimos/backends/kafka_async.rb +6 -22
  16. data/lib/deimos/backends/{db.rb → outbox.rb} +13 -9
  17. data/lib/deimos/config/configuration.rb +116 -385
  18. data/lib/deimos/consume/batch_consumption.rb +24 -124
  19. data/lib/deimos/consume/message_consumption.rb +36 -63
  20. data/lib/deimos/consumer.rb +16 -75
  21. data/lib/deimos/ext/consumer_route.rb +35 -0
  22. data/lib/deimos/ext/producer_middleware.rb +94 -0
  23. data/lib/deimos/ext/producer_route.rb +22 -0
  24. data/lib/deimos/ext/redraw.rb +29 -0
  25. data/lib/deimos/ext/routing_defaults.rb +72 -0
  26. data/lib/deimos/ext/schema_route.rb +70 -0
  27. data/lib/deimos/kafka_message.rb +2 -2
  28. data/lib/deimos/kafka_source.rb +2 -7
  29. data/lib/deimos/kafka_topic_info.rb +1 -1
  30. data/lib/deimos/logging.rb +71 -0
  31. data/lib/deimos/message.rb +2 -11
  32. data/lib/deimos/metrics/datadog.rb +40 -1
  33. data/lib/deimos/metrics/provider.rb +4 -4
  34. data/lib/deimos/producer.rb +39 -116
  35. data/lib/deimos/railtie.rb +6 -0
  36. data/lib/deimos/schema_backends/avro_base.rb +21 -21
  37. data/lib/deimos/schema_backends/avro_schema_registry.rb +1 -2
  38. data/lib/deimos/schema_backends/avro_validation.rb +2 -2
  39. data/lib/deimos/schema_backends/base.rb +19 -12
  40. data/lib/deimos/schema_backends/mock.rb +6 -1
  41. data/lib/deimos/schema_backends/plain.rb +47 -0
  42. data/lib/deimos/schema_class/base.rb +2 -2
  43. data/lib/deimos/schema_class/enum.rb +1 -1
  44. data/lib/deimos/schema_class/record.rb +2 -2
  45. data/lib/deimos/test_helpers.rb +95 -320
  46. data/lib/deimos/tracing/provider.rb +6 -6
  47. data/lib/deimos/transcoder.rb +88 -0
  48. data/lib/deimos/utils/db_poller/base.rb +16 -14
  49. data/lib/deimos/utils/db_poller/state_based.rb +3 -3
  50. data/lib/deimos/utils/db_poller/time_based.rb +4 -4
  51. data/lib/deimos/utils/db_poller.rb +1 -1
  52. data/lib/deimos/utils/deadlock_retry.rb +1 -1
  53. data/lib/deimos/utils/{db_producer.rb → outbox_producer.rb} +16 -47
  54. data/lib/deimos/utils/schema_class.rb +0 -7
  55. data/lib/deimos/version.rb +1 -1
  56. data/lib/deimos.rb +79 -26
  57. data/lib/generators/deimos/{db_backend_generator.rb → outbox_backend_generator.rb} +4 -4
  58. data/lib/generators/deimos/schema_class_generator.rb +0 -1
  59. data/lib/generators/deimos/v2/templates/karafka.rb.tt +149 -0
  60. data/lib/generators/deimos/v2_generator.rb +193 -0
  61. data/lib/tasks/deimos.rake +5 -7
  62. data/spec/active_record_batch_consumer_association_spec.rb +22 -13
  63. data/spec/active_record_batch_consumer_spec.rb +84 -65
  64. data/spec/active_record_consume/batch_consumption_spec.rb +10 -10
  65. data/spec/active_record_consume/batch_slicer_spec.rb +12 -12
  66. data/spec/active_record_consumer_spec.rb +29 -13
  67. data/spec/active_record_producer_spec.rb +36 -26
  68. data/spec/backends/base_spec.rb +0 -23
  69. data/spec/backends/kafka_async_spec.rb +1 -3
  70. data/spec/backends/kafka_spec.rb +1 -3
  71. data/spec/backends/{db_spec.rb → outbox_spec.rb} +14 -20
  72. data/spec/batch_consumer_spec.rb +66 -116
  73. data/spec/consumer_spec.rb +53 -147
  74. data/spec/deimos_spec.rb +10 -126
  75. data/spec/kafka_source_spec.rb +19 -52
  76. data/spec/karafka/karafka.rb +69 -0
  77. data/spec/karafka_config/karafka_spec.rb +97 -0
  78. data/spec/logging_spec.rb +25 -0
  79. data/spec/message_spec.rb +9 -9
  80. data/spec/producer_spec.rb +112 -254
  81. data/spec/rake_spec.rb +1 -3
  82. data/spec/schema_backends/avro_validation_spec.rb +1 -1
  83. data/spec/schemas/com/my-namespace/MySchemaWithTitle.avsc +22 -0
  84. data/spec/snapshots/consumers-no-nest.snap +49 -0
  85. data/spec/snapshots/consumers.snap +49 -0
  86. data/spec/snapshots/consumers_and_producers-no-nest.snap +49 -0
  87. data/spec/snapshots/consumers_and_producers.snap +49 -0
  88. data/spec/snapshots/consumers_circular-no-nest.snap +49 -0
  89. data/spec/snapshots/consumers_circular.snap +49 -0
  90. data/spec/snapshots/consumers_complex_types-no-nest.snap +49 -0
  91. data/spec/snapshots/consumers_complex_types.snap +49 -0
  92. data/spec/snapshots/consumers_nested-no-nest.snap +49 -0
  93. data/spec/snapshots/consumers_nested.snap +49 -0
  94. data/spec/snapshots/namespace_folders.snap +49 -0
  95. data/spec/snapshots/namespace_map.snap +49 -0
  96. data/spec/snapshots/producers_with_key-no-nest.snap +49 -0
  97. data/spec/snapshots/producers_with_key.snap +49 -0
  98. data/spec/spec_helper.rb +61 -29
  99. data/spec/utils/db_poller_spec.rb +49 -39
  100. data/spec/utils/{db_producer_spec.rb → outbox_producer_spec.rb} +17 -184
  101. metadata +58 -67
  102. data/lib/deimos/batch_consumer.rb +0 -7
  103. data/lib/deimos/config/phobos_config.rb +0 -164
  104. data/lib/deimos/instrumentation.rb +0 -95
  105. data/lib/deimos/monkey_patches/phobos_cli.rb +0 -35
  106. data/lib/deimos/utils/inline_consumer.rb +0 -158
  107. data/lib/deimos/utils/lag_reporter.rb +0 -186
  108. data/lib/deimos/utils/schema_controller_mixin.rb +0 -129
  109. data/spec/config/configuration_spec.rb +0 -329
  110. data/spec/kafka_listener_spec.rb +0 -55
  111. data/spec/phobos.bad_db.yml +0 -73
  112. data/spec/phobos.yml +0 -77
  113. data/spec/utils/inline_consumer_spec.rb +0 -31
  114. data/spec/utils/lag_reporter_spec.rb +0 -76
  115. data/spec/utils/platform_schema_validation_spec.rb +0 -0
  116. data/spec/utils/schema_controller_mixin_spec.rb +0 -84
  117. /data/lib/generators/deimos/{db_backend → outbox_backend}/templates/migration +0 -0
  118. /data/lib/generators/deimos/{db_backend → outbox_backend}/templates/rails3_migration +0 -0
@@ -0,0 +1,97 @@
1
+ RSpec.describe 'Karafka configs' do
2
+ before(:each) do
3
+ KarafkaApp.routes.clear
4
+ $found_stuff = nil
5
+ end
6
+
7
+ let(:consumer_class) do
8
+ Class.new(Deimos::Consumer) do
9
+ def consume_message(message)
10
+ $found_stuff = message.payload
11
+ end
12
+ end
13
+ end
14
+
15
+ let(:producer_class) do
16
+ Class.new(Deimos::Producer) do
17
+ end
18
+ end
19
+
20
+ describe 'producers' do
21
+ before(:each) do
22
+ stub_const('MyProducer', producer_class)
23
+ end
24
+
25
+ it 'should work with key none' do
26
+ KarafkaApp.routes.draw do
27
+ topic 'MyTopic' do
28
+ producer_class MyProducer
29
+ schema 'MySchema'
30
+ namespace 'com.my-namespace'
31
+ key_config(none: true)
32
+ end
33
+ end
34
+ producer_class.publish({test_id: "id1", some_int: 5})
35
+ expect('MyTopic').to have_sent({test_id: "id1", some_int: 5})
36
+ end
37
+
38
+ it 'should work with key plain' do
39
+ KarafkaApp.routes.draw do
40
+ topic 'MyTopic' do
41
+ producer_class MyProducer
42
+ schema 'MySchema'
43
+ namespace 'com.my-namespace'
44
+ key_config({plain: true})
45
+ end
46
+ end
47
+ producer_class.publish({test_id: "id1", some_int: 5, payload_key: 'key'})
48
+ expect('MyTopic').to have_sent({test_id: "id1", some_int: 5}, 'key')
49
+ end
50
+
51
+ it 'should work with key field' do
52
+ KarafkaApp.routes.draw do
53
+ topic 'MyTopic' do
54
+ producer_class MyProducer
55
+ schema 'MySchema'
56
+ namespace 'com.my-namespace'
57
+ key_config({field: :test_id})
58
+ end
59
+ end
60
+ producer_class.publish({test_id: "id1", some_int: 5})
61
+ expect('MyTopic').to have_sent({test_id: "id1", some_int: 5}, 'id1')
62
+ end
63
+
64
+ it 'should work with key schema' do
65
+ KarafkaApp.routes.draw do
66
+ topic 'MyTopic' do
67
+ producer_class MyProducer
68
+ schema 'MySchema'
69
+ namespace 'com.my-namespace'
70
+ key_config({schema: 'MySchema_key'})
71
+ end
72
+ end
73
+ producer_class.publish({test_id: "id1", some_int: 5, payload_key: {test_id: 'id3'}})
74
+ expect('MyTopic').to have_sent({test_id: "id1", some_int: 5}, { test_id: 'id3'})
75
+ end
76
+
77
+ end
78
+
79
+ it 'should be able to pick up a consumer' do
80
+ stub_const('MyConsumer', consumer_class)
81
+ KarafkaApp.routes.draw do
82
+ topic 'MyTopic' do
83
+ consumer MyConsumer
84
+ schema 'MySchema'
85
+ namespace 'com.my-namespace'
86
+ key_config({field: :test_id})
87
+ end
88
+ end
89
+
90
+ test_consume_message('MyTopic', {test_id: "id1", some_int: 5}, key: "id1")
91
+ expect($found_stuff).to eq({'test_id' => "id1", 'some_int' => 5})
92
+ $found_stuff = nil
93
+ test_consume_message(MyConsumer, {test_id: "id1", some_int: 5}, key: "id1")
94
+ expect($found_stuff).to eq({'test_id' => "id1", 'some_int' => 5})
95
+ end
96
+
97
+ end
@@ -0,0 +1,25 @@
1
+ RSpec.describe Deimos::Logging do
2
+ include_context 'with publish_backend'
3
+ describe '#messages_log_text' do
4
+ it 'should return whole payload (default behavior)' do
5
+ log_message = described_class.messages_log_text(:payloads, messages)
6
+ expect(log_message[:payloads].count).to eq(3)
7
+ expect(log_message[:payloads].first[:payload]).to eq({ some_int: 1, test_id: 'foo1' })
8
+ expect(log_message[:payloads].first[:key]).to eq('foo1')
9
+ end
10
+
11
+ it 'should return only keys of messages' do
12
+ log_message = described_class.messages_log_text(:keys, messages)
13
+ expect(log_message[:payload_keys].count).to eq(3)
14
+ expect(log_message[:payload_keys]).to be_a(Array)
15
+ expect(log_message[:payload_keys].first).to eq('foo1')
16
+ end
17
+
18
+ it 'should return only messages count' do
19
+ log_message = described_class.messages_log_text(:count, messages)
20
+ expect(log_message[:payloads_count]).to be_a(Integer)
21
+ expect(log_message[:payloads_count]).to eq(3)
22
+ end
23
+ end
24
+
25
+ end
data/spec/message_spec.rb CHANGED
@@ -2,37 +2,37 @@
2
2
 
3
3
  RSpec.describe(Deimos::Message) do
4
4
  it 'should detect tombstones' do
5
- expect(described_class.new(nil, nil, key: 'key1')).
5
+ expect(described_class.new(nil, key: 'key1')).
6
6
  to be_tombstone
7
- expect(described_class.new({ v: 'val1' }, nil, key: 'key1')).
7
+ expect(described_class.new({ v: 'val1' }, key: 'key1')).
8
8
  not_to be_tombstone
9
- expect(described_class.new({ v: '' }, nil, key: 'key1')).
9
+ expect(described_class.new({ v: '' }, key: 'key1')).
10
10
  not_to be_tombstone
11
- expect(described_class.new({ v: 'val1' }, nil, key: nil)).
11
+ expect(described_class.new({ v: 'val1' }, key: nil)).
12
12
  not_to be_tombstone
13
13
  end
14
14
 
15
15
  it 'can support complex keys/values' do
16
- expect { described_class.new({ a: 1, b: 2 }, nil, key: { c: 3, d: 4 }) }.
16
+ expect { described_class.new({ a: 1, b: 2 }, key: { c: 3, d: 4 }) }.
17
17
  not_to raise_exception
18
18
  end
19
19
 
20
20
  describe 'headers' do
21
21
  it 'returns nil when not set' do
22
- expect(described_class.new({ v: 'val1' }, nil, key: 'key1')).
22
+ expect(described_class.new({ v: 'val1' }, key: 'key1')).
23
23
  to have_attributes(headers: nil)
24
24
  end
25
25
 
26
26
  it 'can set and get headers' do
27
- expect(described_class.new({ v: 'val1' }, nil, key: 'key1', headers: { a: 1 })).
27
+ expect(described_class.new({ v: 'val1' }, key: 'key1', headers: { a: 1 })).
28
28
  to have_attributes(headers: { a: 1 })
29
29
  end
30
30
 
31
31
  it 'includes headers when converting to Hash' do
32
- expect(described_class.new({ v: 'val1' }, nil, key: 'key1', headers: { a: 1 }).to_h).
32
+ expect(described_class.new({ v: 'val1' }, key: 'key1', headers: { a: 1 }).to_h).
33
33
  to include(headers: { a: 1 })
34
34
 
35
- expect(described_class.new({ v: 'val1' }, nil, key: 'key1', headers: { a: 1 }).encoded_hash).
35
+ expect(described_class.new({ v: 'val1' }, key: 'key1', headers: { a: 1 }).encoded_hash).
36
36
  to include(headers: { a: 1 })
37
37
  end
38
38
  end