deimos-ruby 1.24.3 → 2.0.0.pre.alpha2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (117) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop_todo.yml +0 -17
  3. data/.tool-versions +1 -0
  4. data/CHANGELOG.md +1 -1
  5. data/README.md +287 -498
  6. data/deimos-ruby.gemspec +4 -4
  7. data/docs/CONFIGURATION.md +133 -227
  8. data/docs/UPGRADING.md +237 -0
  9. data/lib/deimos/active_record_consume/batch_consumption.rb +28 -29
  10. data/lib/deimos/active_record_consume/message_consumption.rb +15 -21
  11. data/lib/deimos/active_record_consumer.rb +36 -26
  12. data/lib/deimos/active_record_producer.rb +28 -9
  13. data/lib/deimos/backends/base.rb +4 -35
  14. data/lib/deimos/backends/kafka.rb +6 -22
  15. data/lib/deimos/backends/kafka_async.rb +6 -22
  16. data/lib/deimos/backends/{db.rb → outbox.rb} +13 -9
  17. data/lib/deimos/config/configuration.rb +116 -385
  18. data/lib/deimos/consume/batch_consumption.rb +24 -124
  19. data/lib/deimos/consume/message_consumption.rb +36 -63
  20. data/lib/deimos/consumer.rb +16 -75
  21. data/lib/deimos/ext/consumer_route.rb +35 -0
  22. data/lib/deimos/ext/producer_middleware.rb +94 -0
  23. data/lib/deimos/ext/producer_route.rb +22 -0
  24. data/lib/deimos/ext/routing_defaults.rb +72 -0
  25. data/lib/deimos/ext/schema_route.rb +70 -0
  26. data/lib/deimos/kafka_message.rb +2 -2
  27. data/lib/deimos/kafka_source.rb +2 -7
  28. data/lib/deimos/kafka_topic_info.rb +1 -1
  29. data/lib/deimos/logging.rb +71 -0
  30. data/lib/deimos/message.rb +2 -11
  31. data/lib/deimos/metrics/datadog.rb +40 -1
  32. data/lib/deimos/metrics/provider.rb +4 -4
  33. data/lib/deimos/producer.rb +39 -116
  34. data/lib/deimos/railtie.rb +6 -0
  35. data/lib/deimos/schema_backends/avro_base.rb +21 -21
  36. data/lib/deimos/schema_backends/avro_schema_registry.rb +1 -2
  37. data/lib/deimos/schema_backends/avro_validation.rb +2 -2
  38. data/lib/deimos/schema_backends/base.rb +19 -12
  39. data/lib/deimos/schema_backends/mock.rb +6 -1
  40. data/lib/deimos/schema_backends/plain.rb +47 -0
  41. data/lib/deimos/schema_class/base.rb +2 -2
  42. data/lib/deimos/schema_class/enum.rb +1 -1
  43. data/lib/deimos/schema_class/record.rb +2 -2
  44. data/lib/deimos/test_helpers.rb +95 -320
  45. data/lib/deimos/tracing/provider.rb +6 -6
  46. data/lib/deimos/transcoder.rb +88 -0
  47. data/lib/deimos/utils/db_poller/base.rb +16 -14
  48. data/lib/deimos/utils/db_poller/state_based.rb +3 -3
  49. data/lib/deimos/utils/db_poller/time_based.rb +4 -4
  50. data/lib/deimos/utils/db_poller.rb +1 -1
  51. data/lib/deimos/utils/deadlock_retry.rb +1 -1
  52. data/lib/deimos/utils/{db_producer.rb → outbox_producer.rb} +16 -47
  53. data/lib/deimos/utils/schema_class.rb +0 -7
  54. data/lib/deimos/version.rb +1 -1
  55. data/lib/deimos.rb +78 -26
  56. data/lib/generators/deimos/{db_backend_generator.rb → outbox_backend_generator.rb} +4 -4
  57. data/lib/generators/deimos/schema_class_generator.rb +0 -1
  58. data/lib/generators/deimos/v2/templates/karafka.rb.tt +149 -0
  59. data/lib/generators/deimos/v2_generator.rb +193 -0
  60. data/lib/tasks/deimos.rake +5 -7
  61. data/spec/active_record_batch_consumer_association_spec.rb +22 -13
  62. data/spec/active_record_batch_consumer_spec.rb +84 -65
  63. data/spec/active_record_consume/batch_consumption_spec.rb +10 -10
  64. data/spec/active_record_consume/batch_slicer_spec.rb +12 -12
  65. data/spec/active_record_consumer_spec.rb +29 -13
  66. data/spec/active_record_producer_spec.rb +36 -26
  67. data/spec/backends/base_spec.rb +0 -23
  68. data/spec/backends/kafka_async_spec.rb +1 -3
  69. data/spec/backends/kafka_spec.rb +1 -3
  70. data/spec/backends/{db_spec.rb → outbox_spec.rb} +14 -20
  71. data/spec/batch_consumer_spec.rb +66 -116
  72. data/spec/consumer_spec.rb +53 -147
  73. data/spec/deimos_spec.rb +10 -126
  74. data/spec/kafka_source_spec.rb +19 -52
  75. data/spec/karafka/karafka.rb +69 -0
  76. data/spec/karafka_config/karafka_spec.rb +97 -0
  77. data/spec/logging_spec.rb +25 -0
  78. data/spec/message_spec.rb +9 -9
  79. data/spec/producer_spec.rb +112 -254
  80. data/spec/rake_spec.rb +1 -3
  81. data/spec/schema_backends/avro_validation_spec.rb +1 -1
  82. data/spec/schemas/com/my-namespace/MySchemaWithTitle.avsc +22 -0
  83. data/spec/snapshots/consumers-no-nest.snap +49 -0
  84. data/spec/snapshots/consumers.snap +49 -0
  85. data/spec/snapshots/consumers_and_producers-no-nest.snap +49 -0
  86. data/spec/snapshots/consumers_and_producers.snap +49 -0
  87. data/spec/snapshots/consumers_circular-no-nest.snap +49 -0
  88. data/spec/snapshots/consumers_circular.snap +49 -0
  89. data/spec/snapshots/consumers_complex_types-no-nest.snap +49 -0
  90. data/spec/snapshots/consumers_complex_types.snap +49 -0
  91. data/spec/snapshots/consumers_nested-no-nest.snap +49 -0
  92. data/spec/snapshots/consumers_nested.snap +49 -0
  93. data/spec/snapshots/namespace_folders.snap +49 -0
  94. data/spec/snapshots/namespace_map.snap +49 -0
  95. data/spec/snapshots/producers_with_key-no-nest.snap +49 -0
  96. data/spec/snapshots/producers_with_key.snap +49 -0
  97. data/spec/spec_helper.rb +61 -29
  98. data/spec/utils/db_poller_spec.rb +49 -39
  99. data/spec/utils/{db_producer_spec.rb → outbox_producer_spec.rb} +17 -184
  100. metadata +57 -67
  101. data/lib/deimos/batch_consumer.rb +0 -7
  102. data/lib/deimos/config/phobos_config.rb +0 -164
  103. data/lib/deimos/instrumentation.rb +0 -95
  104. data/lib/deimos/monkey_patches/phobos_cli.rb +0 -35
  105. data/lib/deimos/utils/inline_consumer.rb +0 -158
  106. data/lib/deimos/utils/lag_reporter.rb +0 -186
  107. data/lib/deimos/utils/schema_controller_mixin.rb +0 -129
  108. data/spec/config/configuration_spec.rb +0 -329
  109. data/spec/kafka_listener_spec.rb +0 -55
  110. data/spec/phobos.bad_db.yml +0 -73
  111. data/spec/phobos.yml +0 -77
  112. data/spec/utils/inline_consumer_spec.rb +0 -31
  113. data/spec/utils/lag_reporter_spec.rb +0 -76
  114. data/spec/utils/platform_schema_validation_spec.rb +0 -0
  115. data/spec/utils/schema_controller_mixin_spec.rb +0 -84
  116. /data/lib/generators/deimos/{db_backend → outbox_backend}/templates/migration +0 -0
  117. /data/lib/generators/deimos/{db_backend → outbox_backend}/templates/rails3_migration +0 -0
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: de8ca3bf7233f799c3bf9169fa647dabebd8e08cd2b05ec00511c132a5f3920d
4
- data.tar.gz: 03177d9ac87184fe3ce6ef916bce0ad39e3b81a6576f60ba71c33b298f82dfd8
3
+ metadata.gz: c7563872db1d29eb0e6873eec799fe173ceb7484ec8e728063e49cacd3dc3083
4
+ data.tar.gz: d768d9026137b265887c3e0a063c9737974338de2a460c2c747440ba6d2ea94c
5
5
  SHA512:
6
- metadata.gz: 40ecdddd9c0b8f5f9675f8058f4a8c8adf1eb69a293e04870a46b9d8ae1972cdc70088611c898f0ab91e072f6627c4760bca5ca5399b698316edda8f3212f5aa
7
- data.tar.gz: 3f2fa463d5c777ce7e4dfc4cf610a7dd4ed3f6dba5b6dd459f1ca8fad037f0baa47278436c65c91dc90679f88e9d42a792b84ccbdc2cde4b58d1239d03b2fd4a
6
+ metadata.gz: 76c9cb88a3e130eedcdaa45e4ad5cdf9cbed0bdf3c61e68bde81358785f250429ad1c1ffbaaaa520e739f0e0ef3c16fd8a0f5422e706c3178267600456bf9930
7
+ data.tar.gz: b0a531fdbd9b1ec9276f44783418e8df5fc358c6c3c2e1e51f6265b6e664cdf4fb83ad33d631f2a3ebb465aba52724d8941a7e97fbf49632f9299bd9249d7928
data/.rubocop_todo.yml CHANGED
@@ -146,7 +146,6 @@ Lint/UselessAssignment:
146
146
  Metrics/AbcSize:
147
147
  Exclude:
148
148
  - 'lib/deimos/active_record_consume/message_consumption.rb'
149
- - 'lib/deimos/config/phobos_config.rb'
150
149
  - 'lib/deimos/instrumentation.rb'
151
150
  - 'lib/deimos/kafka_source.rb'
152
151
  - 'lib/deimos/kafka_topic_info.rb'
@@ -159,12 +158,6 @@ Metrics/AbcSize:
159
158
  - 'lib/deimos/utils/schema_controller_mixin.rb'
160
159
  - 'lib/generators/deimos/schema_class_generator.rb'
161
160
 
162
- # Offense count: 1
163
- # Configuration parameters: CountComments, Max, CountAsOne, ExcludedMethods.
164
- Metrics/MethodLength:
165
- Exclude:
166
- - 'lib/deimos/config/phobos_config.rb'
167
-
168
161
  # Offense count: 5
169
162
  # Configuration parameters: CountComments, Max, CountAsOne.
170
163
  Metrics/ModuleLength:
@@ -179,7 +172,6 @@ Metrics/ModuleLength:
179
172
  # Configuration parameters: IgnoredMethods, Max.
180
173
  Metrics/PerceivedComplexity:
181
174
  Exclude:
182
- - 'lib/deimos/config/phobos_config.rb'
183
175
  - 'lib/deimos/consume/batch_consumption.rb'
184
176
  - 'lib/deimos/kafka_source.rb'
185
177
  - 'lib/deimos/schema_backends/avro_schema_coercer.rb'
@@ -253,7 +245,6 @@ Style/FrozenStringLiteralComment:
253
245
  Style/GlobalStdStream:
254
246
  Exclude:
255
247
  - 'lib/deimos/config/configuration.rb'
256
- - 'lib/deimos/config/phobos_config.rb'
257
248
  - 'lib/deimos/metrics/mock.rb'
258
249
  - 'lib/deimos/test_helpers.rb'
259
250
  - 'lib/deimos/tracing/mock.rb'
@@ -329,14 +320,6 @@ Style/StringLiterals:
329
320
  - 'spec/schemas/my_namespace/my_schema_with_complex_type.rb'
330
321
  - 'spec/spec_helper.rb'
331
322
 
332
- # Offense count: 1
333
- # Cop supports --auto-correct.
334
- # Configuration parameters: EnforcedStyle, AllowSafeAssignment.
335
- # SupportedStyles: require_parentheses, require_no_parentheses, require_parentheses_when_complex
336
- Style/TernaryParentheses:
337
- Exclude:
338
- - 'lib/deimos/config/phobos_config.rb'
339
-
340
323
  # Offense count: 21
341
324
  # Cop supports --auto-correct.
342
325
  Style/TrailingBodyOnModule:
data/.tool-versions ADDED
@@ -0,0 +1 @@
1
+ ruby 3.2.2
data/CHANGELOG.md CHANGED
@@ -6,9 +6,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
6
6
  and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
7
7
 
8
8
  ## UNRELEASED
9
+ - Feature: Added `max_batch_size` config to producer to allow custom batch size for publishing.
9
10
 
10
11
  # 1.24.3 - 2024-05-13
11
-
12
12
  - Feature: Enable `producers.persistent_connections` phobos setting
13
13
  - Feature: Add consumer configuration, `save_associations_first` to save associated records of primary class prior to upserting primary records. Foreign key of associated records are assigned to the record class prior to saving the record class
14
14