deimos-ruby 1.24.3 → 2.0.0.pre.alpha1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (118) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop_todo.yml +0 -17
  3. data/.tool-versions +1 -0
  4. data/CHANGELOG.md +1 -1
  5. data/README.md +287 -498
  6. data/deimos-ruby.gemspec +4 -4
  7. data/docs/CONFIGURATION.md +133 -227
  8. data/docs/UPGRADING.md +237 -0
  9. data/lib/deimos/active_record_consume/batch_consumption.rb +28 -29
  10. data/lib/deimos/active_record_consume/message_consumption.rb +15 -21
  11. data/lib/deimos/active_record_consumer.rb +36 -26
  12. data/lib/deimos/active_record_producer.rb +28 -9
  13. data/lib/deimos/backends/base.rb +4 -35
  14. data/lib/deimos/backends/kafka.rb +6 -22
  15. data/lib/deimos/backends/kafka_async.rb +6 -22
  16. data/lib/deimos/backends/{db.rb → outbox.rb} +13 -9
  17. data/lib/deimos/config/configuration.rb +116 -385
  18. data/lib/deimos/consume/batch_consumption.rb +24 -124
  19. data/lib/deimos/consume/message_consumption.rb +36 -63
  20. data/lib/deimos/consumer.rb +16 -75
  21. data/lib/deimos/ext/consumer_route.rb +35 -0
  22. data/lib/deimos/ext/producer_middleware.rb +94 -0
  23. data/lib/deimos/ext/producer_route.rb +22 -0
  24. data/lib/deimos/ext/redraw.rb +29 -0
  25. data/lib/deimos/ext/routing_defaults.rb +72 -0
  26. data/lib/deimos/ext/schema_route.rb +70 -0
  27. data/lib/deimos/kafka_message.rb +2 -2
  28. data/lib/deimos/kafka_source.rb +2 -7
  29. data/lib/deimos/kafka_topic_info.rb +1 -1
  30. data/lib/deimos/logging.rb +71 -0
  31. data/lib/deimos/message.rb +2 -11
  32. data/lib/deimos/metrics/datadog.rb +40 -1
  33. data/lib/deimos/metrics/provider.rb +4 -4
  34. data/lib/deimos/producer.rb +39 -116
  35. data/lib/deimos/railtie.rb +6 -0
  36. data/lib/deimos/schema_backends/avro_base.rb +21 -21
  37. data/lib/deimos/schema_backends/avro_schema_registry.rb +1 -2
  38. data/lib/deimos/schema_backends/avro_validation.rb +2 -2
  39. data/lib/deimos/schema_backends/base.rb +19 -12
  40. data/lib/deimos/schema_backends/mock.rb +6 -1
  41. data/lib/deimos/schema_backends/plain.rb +47 -0
  42. data/lib/deimos/schema_class/base.rb +2 -2
  43. data/lib/deimos/schema_class/enum.rb +1 -1
  44. data/lib/deimos/schema_class/record.rb +2 -2
  45. data/lib/deimos/test_helpers.rb +95 -320
  46. data/lib/deimos/tracing/provider.rb +6 -6
  47. data/lib/deimos/transcoder.rb +88 -0
  48. data/lib/deimos/utils/db_poller/base.rb +16 -14
  49. data/lib/deimos/utils/db_poller/state_based.rb +3 -3
  50. data/lib/deimos/utils/db_poller/time_based.rb +4 -4
  51. data/lib/deimos/utils/db_poller.rb +1 -1
  52. data/lib/deimos/utils/deadlock_retry.rb +1 -1
  53. data/lib/deimos/utils/{db_producer.rb → outbox_producer.rb} +16 -47
  54. data/lib/deimos/utils/schema_class.rb +0 -7
  55. data/lib/deimos/version.rb +1 -1
  56. data/lib/deimos.rb +79 -26
  57. data/lib/generators/deimos/{db_backend_generator.rb → outbox_backend_generator.rb} +4 -4
  58. data/lib/generators/deimos/schema_class_generator.rb +0 -1
  59. data/lib/generators/deimos/v2/templates/karafka.rb.tt +149 -0
  60. data/lib/generators/deimos/v2_generator.rb +193 -0
  61. data/lib/tasks/deimos.rake +5 -7
  62. data/spec/active_record_batch_consumer_association_spec.rb +22 -13
  63. data/spec/active_record_batch_consumer_spec.rb +84 -65
  64. data/spec/active_record_consume/batch_consumption_spec.rb +10 -10
  65. data/spec/active_record_consume/batch_slicer_spec.rb +12 -12
  66. data/spec/active_record_consumer_spec.rb +29 -13
  67. data/spec/active_record_producer_spec.rb +36 -26
  68. data/spec/backends/base_spec.rb +0 -23
  69. data/spec/backends/kafka_async_spec.rb +1 -3
  70. data/spec/backends/kafka_spec.rb +1 -3
  71. data/spec/backends/{db_spec.rb → outbox_spec.rb} +14 -20
  72. data/spec/batch_consumer_spec.rb +66 -116
  73. data/spec/consumer_spec.rb +53 -147
  74. data/spec/deimos_spec.rb +10 -126
  75. data/spec/kafka_source_spec.rb +19 -52
  76. data/spec/karafka/karafka.rb +69 -0
  77. data/spec/karafka_config/karafka_spec.rb +97 -0
  78. data/spec/logging_spec.rb +25 -0
  79. data/spec/message_spec.rb +9 -9
  80. data/spec/producer_spec.rb +112 -254
  81. data/spec/rake_spec.rb +1 -3
  82. data/spec/schema_backends/avro_validation_spec.rb +1 -1
  83. data/spec/schemas/com/my-namespace/MySchemaWithTitle.avsc +22 -0
  84. data/spec/snapshots/consumers-no-nest.snap +49 -0
  85. data/spec/snapshots/consumers.snap +49 -0
  86. data/spec/snapshots/consumers_and_producers-no-nest.snap +49 -0
  87. data/spec/snapshots/consumers_and_producers.snap +49 -0
  88. data/spec/snapshots/consumers_circular-no-nest.snap +49 -0
  89. data/spec/snapshots/consumers_circular.snap +49 -0
  90. data/spec/snapshots/consumers_complex_types-no-nest.snap +49 -0
  91. data/spec/snapshots/consumers_complex_types.snap +49 -0
  92. data/spec/snapshots/consumers_nested-no-nest.snap +49 -0
  93. data/spec/snapshots/consumers_nested.snap +49 -0
  94. data/spec/snapshots/namespace_folders.snap +49 -0
  95. data/spec/snapshots/namespace_map.snap +49 -0
  96. data/spec/snapshots/producers_with_key-no-nest.snap +49 -0
  97. data/spec/snapshots/producers_with_key.snap +49 -0
  98. data/spec/spec_helper.rb +61 -29
  99. data/spec/utils/db_poller_spec.rb +49 -39
  100. data/spec/utils/{db_producer_spec.rb → outbox_producer_spec.rb} +17 -184
  101. metadata +58 -67
  102. data/lib/deimos/batch_consumer.rb +0 -7
  103. data/lib/deimos/config/phobos_config.rb +0 -164
  104. data/lib/deimos/instrumentation.rb +0 -95
  105. data/lib/deimos/monkey_patches/phobos_cli.rb +0 -35
  106. data/lib/deimos/utils/inline_consumer.rb +0 -158
  107. data/lib/deimos/utils/lag_reporter.rb +0 -186
  108. data/lib/deimos/utils/schema_controller_mixin.rb +0 -129
  109. data/spec/config/configuration_spec.rb +0 -329
  110. data/spec/kafka_listener_spec.rb +0 -55
  111. data/spec/phobos.bad_db.yml +0 -73
  112. data/spec/phobos.yml +0 -77
  113. data/spec/utils/inline_consumer_spec.rb +0 -31
  114. data/spec/utils/lag_reporter_spec.rb +0 -76
  115. data/spec/utils/platform_schema_validation_spec.rb +0 -0
  116. data/spec/utils/schema_controller_mixin_spec.rb +0 -84
  117. /data/lib/generators/deimos/{db_backend → outbox_backend}/templates/migration +0 -0
  118. /data/lib/generators/deimos/{db_backend → outbox_backend}/templates/rails3_migration +0 -0
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: de8ca3bf7233f799c3bf9169fa647dabebd8e08cd2b05ec00511c132a5f3920d
4
- data.tar.gz: 03177d9ac87184fe3ce6ef916bce0ad39e3b81a6576f60ba71c33b298f82dfd8
3
+ metadata.gz: 45c1138f93a466197d97834130389f2f17652221dee2ec8d909b8f238c85f554
4
+ data.tar.gz: 0b48e5070b44906ecf39e759fd3142d0b85805561cca43842407d5d823a0d381
5
5
  SHA512:
6
- metadata.gz: 40ecdddd9c0b8f5f9675f8058f4a8c8adf1eb69a293e04870a46b9d8ae1972cdc70088611c898f0ab91e072f6627c4760bca5ca5399b698316edda8f3212f5aa
7
- data.tar.gz: 3f2fa463d5c777ce7e4dfc4cf610a7dd4ed3f6dba5b6dd459f1ca8fad037f0baa47278436c65c91dc90679f88e9d42a792b84ccbdc2cde4b58d1239d03b2fd4a
6
+ metadata.gz: 88991e80cf0c4cfcd873058ed30de62b27d78ce0a29f84b66679b0c0b4799a0efb4a4fed647e54016b68f8e4bb282bdc1797ff52796b1ef615897107a156e263
7
+ data.tar.gz: 1b256132ce1385301c9b03cf0e3e2ca3c44da14c0a13063a6c911f6d782cb0a7832ec3a4913a5b3e420c663df99db62757108eafd76144d0d4351b9c47241190
data/.rubocop_todo.yml CHANGED
@@ -146,7 +146,6 @@ Lint/UselessAssignment:
146
146
  Metrics/AbcSize:
147
147
  Exclude:
148
148
  - 'lib/deimos/active_record_consume/message_consumption.rb'
149
- - 'lib/deimos/config/phobos_config.rb'
150
149
  - 'lib/deimos/instrumentation.rb'
151
150
  - 'lib/deimos/kafka_source.rb'
152
151
  - 'lib/deimos/kafka_topic_info.rb'
@@ -159,12 +158,6 @@ Metrics/AbcSize:
159
158
  - 'lib/deimos/utils/schema_controller_mixin.rb'
160
159
  - 'lib/generators/deimos/schema_class_generator.rb'
161
160
 
162
- # Offense count: 1
163
- # Configuration parameters: CountComments, Max, CountAsOne, ExcludedMethods.
164
- Metrics/MethodLength:
165
- Exclude:
166
- - 'lib/deimos/config/phobos_config.rb'
167
-
168
161
  # Offense count: 5
169
162
  # Configuration parameters: CountComments, Max, CountAsOne.
170
163
  Metrics/ModuleLength:
@@ -179,7 +172,6 @@ Metrics/ModuleLength:
179
172
  # Configuration parameters: IgnoredMethods, Max.
180
173
  Metrics/PerceivedComplexity:
181
174
  Exclude:
182
- - 'lib/deimos/config/phobos_config.rb'
183
175
  - 'lib/deimos/consume/batch_consumption.rb'
184
176
  - 'lib/deimos/kafka_source.rb'
185
177
  - 'lib/deimos/schema_backends/avro_schema_coercer.rb'
@@ -253,7 +245,6 @@ Style/FrozenStringLiteralComment:
253
245
  Style/GlobalStdStream:
254
246
  Exclude:
255
247
  - 'lib/deimos/config/configuration.rb'
256
- - 'lib/deimos/config/phobos_config.rb'
257
248
  - 'lib/deimos/metrics/mock.rb'
258
249
  - 'lib/deimos/test_helpers.rb'
259
250
  - 'lib/deimos/tracing/mock.rb'
@@ -329,14 +320,6 @@ Style/StringLiterals:
329
320
  - 'spec/schemas/my_namespace/my_schema_with_complex_type.rb'
330
321
  - 'spec/spec_helper.rb'
331
322
 
332
- # Offense count: 1
333
- # Cop supports --auto-correct.
334
- # Configuration parameters: EnforcedStyle, AllowSafeAssignment.
335
- # SupportedStyles: require_parentheses, require_no_parentheses, require_parentheses_when_complex
336
- Style/TernaryParentheses:
337
- Exclude:
338
- - 'lib/deimos/config/phobos_config.rb'
339
-
340
323
  # Offense count: 21
341
324
  # Cop supports --auto-correct.
342
325
  Style/TrailingBodyOnModule:
data/.tool-versions ADDED
@@ -0,0 +1 @@
1
+ ruby 3.2.2
data/CHANGELOG.md CHANGED
@@ -6,9 +6,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
6
6
  and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
7
7
 
8
8
  ## UNRELEASED
9
+ - Feature: Added `max_batch_size` config to producer to allow custom batch size for publishing.
9
10
 
10
11
  # 1.24.3 - 2024-05-13
11
-
12
12
  - Feature: Enable `producers.persistent_connections` phobos setting
13
13
  - Feature: Add consumer configuration, `save_associations_first` to save associated records of primary class prior to upserting primary records. Foreign key of associated records are assigned to the record class prior to saving the record class
14
14