deimos-ruby 1.24.2 → 2.0.0.pre.alpha1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop_todo.yml +0 -17
  3. data/.tool-versions +1 -0
  4. data/CHANGELOG.md +5 -0
  5. data/README.md +287 -498
  6. data/deimos-ruby.gemspec +4 -4
  7. data/docs/CONFIGURATION.md +133 -226
  8. data/docs/UPGRADING.md +237 -0
  9. data/lib/deimos/active_record_consume/batch_consumption.rb +29 -28
  10. data/lib/deimos/active_record_consume/mass_updater.rb +59 -4
  11. data/lib/deimos/active_record_consume/message_consumption.rb +15 -21
  12. data/lib/deimos/active_record_consumer.rb +36 -21
  13. data/lib/deimos/active_record_producer.rb +28 -9
  14. data/lib/deimos/backends/base.rb +4 -35
  15. data/lib/deimos/backends/kafka.rb +6 -22
  16. data/lib/deimos/backends/kafka_async.rb +6 -22
  17. data/lib/deimos/backends/{db.rb → outbox.rb} +13 -9
  18. data/lib/deimos/config/configuration.rb +116 -379
  19. data/lib/deimos/consume/batch_consumption.rb +24 -124
  20. data/lib/deimos/consume/message_consumption.rb +36 -63
  21. data/lib/deimos/consumer.rb +16 -75
  22. data/lib/deimos/ext/consumer_route.rb +35 -0
  23. data/lib/deimos/ext/producer_middleware.rb +94 -0
  24. data/lib/deimos/ext/producer_route.rb +22 -0
  25. data/lib/deimos/ext/redraw.rb +29 -0
  26. data/lib/deimos/ext/routing_defaults.rb +72 -0
  27. data/lib/deimos/ext/schema_route.rb +70 -0
  28. data/lib/deimos/kafka_message.rb +2 -2
  29. data/lib/deimos/kafka_source.rb +2 -7
  30. data/lib/deimos/kafka_topic_info.rb +1 -1
  31. data/lib/deimos/logging.rb +71 -0
  32. data/lib/deimos/message.rb +2 -11
  33. data/lib/deimos/metrics/datadog.rb +40 -1
  34. data/lib/deimos/metrics/provider.rb +4 -4
  35. data/lib/deimos/producer.rb +39 -116
  36. data/lib/deimos/railtie.rb +6 -0
  37. data/lib/deimos/schema_backends/avro_base.rb +21 -21
  38. data/lib/deimos/schema_backends/avro_schema_registry.rb +1 -2
  39. data/lib/deimos/schema_backends/avro_validation.rb +2 -2
  40. data/lib/deimos/schema_backends/base.rb +19 -12
  41. data/lib/deimos/schema_backends/mock.rb +6 -1
  42. data/lib/deimos/schema_backends/plain.rb +47 -0
  43. data/lib/deimos/schema_class/base.rb +2 -2
  44. data/lib/deimos/schema_class/enum.rb +1 -1
  45. data/lib/deimos/schema_class/record.rb +2 -2
  46. data/lib/deimos/test_helpers.rb +95 -320
  47. data/lib/deimos/tracing/provider.rb +6 -6
  48. data/lib/deimos/transcoder.rb +88 -0
  49. data/lib/deimos/utils/db_poller/base.rb +16 -14
  50. data/lib/deimos/utils/db_poller/state_based.rb +3 -3
  51. data/lib/deimos/utils/db_poller/time_based.rb +4 -4
  52. data/lib/deimos/utils/db_poller.rb +1 -1
  53. data/lib/deimos/utils/deadlock_retry.rb +1 -1
  54. data/lib/deimos/utils/{db_producer.rb → outbox_producer.rb} +16 -47
  55. data/lib/deimos/utils/schema_class.rb +0 -7
  56. data/lib/deimos/version.rb +1 -1
  57. data/lib/deimos.rb +79 -26
  58. data/lib/generators/deimos/{db_backend_generator.rb → outbox_backend_generator.rb} +4 -4
  59. data/lib/generators/deimos/schema_class_generator.rb +0 -1
  60. data/lib/generators/deimos/v2/templates/karafka.rb.tt +149 -0
  61. data/lib/generators/deimos/v2_generator.rb +193 -0
  62. data/lib/tasks/deimos.rake +5 -7
  63. data/spec/active_record_batch_consumer_association_spec.rb +22 -13
  64. data/spec/active_record_batch_consumer_spec.rb +84 -65
  65. data/spec/active_record_consume/batch_consumption_spec.rb +10 -10
  66. data/spec/active_record_consume/batch_slicer_spec.rb +12 -12
  67. data/spec/active_record_consume/mass_updater_spec.rb +137 -0
  68. data/spec/active_record_consumer_spec.rb +29 -13
  69. data/spec/active_record_producer_spec.rb +36 -26
  70. data/spec/backends/base_spec.rb +0 -23
  71. data/spec/backends/kafka_async_spec.rb +1 -3
  72. data/spec/backends/kafka_spec.rb +1 -3
  73. data/spec/backends/{db_spec.rb → outbox_spec.rb} +14 -20
  74. data/spec/batch_consumer_spec.rb +66 -116
  75. data/spec/consumer_spec.rb +53 -147
  76. data/spec/deimos_spec.rb +10 -126
  77. data/spec/kafka_source_spec.rb +19 -52
  78. data/spec/karafka/karafka.rb +69 -0
  79. data/spec/karafka_config/karafka_spec.rb +97 -0
  80. data/spec/logging_spec.rb +25 -0
  81. data/spec/message_spec.rb +9 -9
  82. data/spec/producer_spec.rb +112 -254
  83. data/spec/rake_spec.rb +1 -3
  84. data/spec/schema_backends/avro_validation_spec.rb +1 -1
  85. data/spec/schemas/com/my-namespace/MySchemaWithTitle.avsc +22 -0
  86. data/spec/snapshots/consumers-no-nest.snap +49 -0
  87. data/spec/snapshots/consumers.snap +49 -0
  88. data/spec/snapshots/consumers_and_producers-no-nest.snap +49 -0
  89. data/spec/snapshots/consumers_and_producers.snap +49 -0
  90. data/spec/snapshots/consumers_circular-no-nest.snap +49 -0
  91. data/spec/snapshots/consumers_circular.snap +49 -0
  92. data/spec/snapshots/consumers_complex_types-no-nest.snap +49 -0
  93. data/spec/snapshots/consumers_complex_types.snap +49 -0
  94. data/spec/snapshots/consumers_nested-no-nest.snap +49 -0
  95. data/spec/snapshots/consumers_nested.snap +49 -0
  96. data/spec/snapshots/namespace_folders.snap +49 -0
  97. data/spec/snapshots/namespace_map.snap +49 -0
  98. data/spec/snapshots/producers_with_key-no-nest.snap +49 -0
  99. data/spec/snapshots/producers_with_key.snap +49 -0
  100. data/spec/spec_helper.rb +61 -29
  101. data/spec/utils/db_poller_spec.rb +49 -39
  102. data/spec/utils/{db_producer_spec.rb → outbox_producer_spec.rb} +17 -184
  103. metadata +58 -67
  104. data/lib/deimos/batch_consumer.rb +0 -7
  105. data/lib/deimos/config/phobos_config.rb +0 -163
  106. data/lib/deimos/instrumentation.rb +0 -95
  107. data/lib/deimos/monkey_patches/phobos_cli.rb +0 -35
  108. data/lib/deimos/utils/inline_consumer.rb +0 -158
  109. data/lib/deimos/utils/lag_reporter.rb +0 -186
  110. data/lib/deimos/utils/schema_controller_mixin.rb +0 -129
  111. data/spec/config/configuration_spec.rb +0 -321
  112. data/spec/kafka_listener_spec.rb +0 -55
  113. data/spec/phobos.bad_db.yml +0 -73
  114. data/spec/phobos.yml +0 -77
  115. data/spec/utils/inline_consumer_spec.rb +0 -31
  116. data/spec/utils/lag_reporter_spec.rb +0 -76
  117. data/spec/utils/platform_schema_validation_spec.rb +0 -0
  118. data/spec/utils/schema_controller_mixin_spec.rb +0 -84
  119. /data/lib/generators/deimos/{db_backend → outbox_backend}/templates/migration +0 -0
  120. /data/lib/generators/deimos/{db_backend → outbox_backend}/templates/rails3_migration +0 -0
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: f2be475993ca3cec87d78682f7724b973f5aadb885612f98d5978169da2932bf
4
- data.tar.gz: 5ab0f052676184f800965bf68aebecac80583d1da2f38561a305e5f5b57a68c9
3
+ metadata.gz: 45c1138f93a466197d97834130389f2f17652221dee2ec8d909b8f238c85f554
4
+ data.tar.gz: 0b48e5070b44906ecf39e759fd3142d0b85805561cca43842407d5d823a0d381
5
5
  SHA512:
6
- metadata.gz: 918a363d9994537a221ccc6d00131c75a97f36c8cf89bdda47e0a6e6770b36fd1c3b07e5177d0ef284494020d90dd9081bc9d40ec41f4dc8bbe45e9c4165ea1f
7
- data.tar.gz: 60d316ef05ce427234e366e32f44addd7188adb45345b025bfb9fd21f339d0c579f317f1e55db20185b9f5f825af3226c27cb53d5e9336ec805200d2a9ce5bd9
6
+ metadata.gz: 88991e80cf0c4cfcd873058ed30de62b27d78ce0a29f84b66679b0c0b4799a0efb4a4fed647e54016b68f8e4bb282bdc1797ff52796b1ef615897107a156e263
7
+ data.tar.gz: 1b256132ce1385301c9b03cf0e3e2ca3c44da14c0a13063a6c911f6d782cb0a7832ec3a4913a5b3e420c663df99db62757108eafd76144d0d4351b9c47241190
data/.rubocop_todo.yml CHANGED
@@ -146,7 +146,6 @@ Lint/UselessAssignment:
146
146
  Metrics/AbcSize:
147
147
  Exclude:
148
148
  - 'lib/deimos/active_record_consume/message_consumption.rb'
149
- - 'lib/deimos/config/phobos_config.rb'
150
149
  - 'lib/deimos/instrumentation.rb'
151
150
  - 'lib/deimos/kafka_source.rb'
152
151
  - 'lib/deimos/kafka_topic_info.rb'
@@ -159,12 +158,6 @@ Metrics/AbcSize:
159
158
  - 'lib/deimos/utils/schema_controller_mixin.rb'
160
159
  - 'lib/generators/deimos/schema_class_generator.rb'
161
160
 
162
- # Offense count: 1
163
- # Configuration parameters: CountComments, Max, CountAsOne, ExcludedMethods.
164
- Metrics/MethodLength:
165
- Exclude:
166
- - 'lib/deimos/config/phobos_config.rb'
167
-
168
161
  # Offense count: 5
169
162
  # Configuration parameters: CountComments, Max, CountAsOne.
170
163
  Metrics/ModuleLength:
@@ -179,7 +172,6 @@ Metrics/ModuleLength:
179
172
  # Configuration parameters: IgnoredMethods, Max.
180
173
  Metrics/PerceivedComplexity:
181
174
  Exclude:
182
- - 'lib/deimos/config/phobos_config.rb'
183
175
  - 'lib/deimos/consume/batch_consumption.rb'
184
176
  - 'lib/deimos/kafka_source.rb'
185
177
  - 'lib/deimos/schema_backends/avro_schema_coercer.rb'
@@ -253,7 +245,6 @@ Style/FrozenStringLiteralComment:
253
245
  Style/GlobalStdStream:
254
246
  Exclude:
255
247
  - 'lib/deimos/config/configuration.rb'
256
- - 'lib/deimos/config/phobos_config.rb'
257
248
  - 'lib/deimos/metrics/mock.rb'
258
249
  - 'lib/deimos/test_helpers.rb'
259
250
  - 'lib/deimos/tracing/mock.rb'
@@ -329,14 +320,6 @@ Style/StringLiterals:
329
320
  - 'spec/schemas/my_namespace/my_schema_with_complex_type.rb'
330
321
  - 'spec/spec_helper.rb'
331
322
 
332
- # Offense count: 1
333
- # Cop supports --auto-correct.
334
- # Configuration parameters: EnforcedStyle, AllowSafeAssignment.
335
- # SupportedStyles: require_parentheses, require_no_parentheses, require_parentheses_when_complex
336
- Style/TernaryParentheses:
337
- Exclude:
338
- - 'lib/deimos/config/phobos_config.rb'
339
-
340
323
  # Offense count: 21
341
324
  # Cop supports --auto-correct.
342
325
  Style/TrailingBodyOnModule:
data/.tool-versions ADDED
@@ -0,0 +1 @@
1
+ ruby 3.2.2
data/CHANGELOG.md CHANGED
@@ -6,6 +6,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
6
6
  and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
7
7
 
8
8
  ## UNRELEASED
9
+ - Feature: Added `max_batch_size` config to producer to allow custom batch size for publishing.
10
+
11
+ # 1.24.3 - 2024-05-13
12
+ - Feature: Enable `producers.persistent_connections` phobos setting
13
+ - Feature: Add consumer configuration, `save_associations_first` to save associated records of primary class prior to upserting primary records. Foreign key of associated records are assigned to the record class prior to saving the record class
9
14
 
10
15
  # 1.24.2 - 2024-05-01
11
16
  - Fix: Deprecation notice with Rails 7.