deimos-ruby 1.24.2 → 2.0.0.pre.alpha1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (120) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop_todo.yml +0 -17
  3. data/.tool-versions +1 -0
  4. data/CHANGELOG.md +5 -0
  5. data/README.md +287 -498
  6. data/deimos-ruby.gemspec +4 -4
  7. data/docs/CONFIGURATION.md +133 -226
  8. data/docs/UPGRADING.md +237 -0
  9. data/lib/deimos/active_record_consume/batch_consumption.rb +29 -28
  10. data/lib/deimos/active_record_consume/mass_updater.rb +59 -4
  11. data/lib/deimos/active_record_consume/message_consumption.rb +15 -21
  12. data/lib/deimos/active_record_consumer.rb +36 -21
  13. data/lib/deimos/active_record_producer.rb +28 -9
  14. data/lib/deimos/backends/base.rb +4 -35
  15. data/lib/deimos/backends/kafka.rb +6 -22
  16. data/lib/deimos/backends/kafka_async.rb +6 -22
  17. data/lib/deimos/backends/{db.rb → outbox.rb} +13 -9
  18. data/lib/deimos/config/configuration.rb +116 -379
  19. data/lib/deimos/consume/batch_consumption.rb +24 -124
  20. data/lib/deimos/consume/message_consumption.rb +36 -63
  21. data/lib/deimos/consumer.rb +16 -75
  22. data/lib/deimos/ext/consumer_route.rb +35 -0
  23. data/lib/deimos/ext/producer_middleware.rb +94 -0
  24. data/lib/deimos/ext/producer_route.rb +22 -0
  25. data/lib/deimos/ext/redraw.rb +29 -0
  26. data/lib/deimos/ext/routing_defaults.rb +72 -0
  27. data/lib/deimos/ext/schema_route.rb +70 -0
  28. data/lib/deimos/kafka_message.rb +2 -2
  29. data/lib/deimos/kafka_source.rb +2 -7
  30. data/lib/deimos/kafka_topic_info.rb +1 -1
  31. data/lib/deimos/logging.rb +71 -0
  32. data/lib/deimos/message.rb +2 -11
  33. data/lib/deimos/metrics/datadog.rb +40 -1
  34. data/lib/deimos/metrics/provider.rb +4 -4
  35. data/lib/deimos/producer.rb +39 -116
  36. data/lib/deimos/railtie.rb +6 -0
  37. data/lib/deimos/schema_backends/avro_base.rb +21 -21
  38. data/lib/deimos/schema_backends/avro_schema_registry.rb +1 -2
  39. data/lib/deimos/schema_backends/avro_validation.rb +2 -2
  40. data/lib/deimos/schema_backends/base.rb +19 -12
  41. data/lib/deimos/schema_backends/mock.rb +6 -1
  42. data/lib/deimos/schema_backends/plain.rb +47 -0
  43. data/lib/deimos/schema_class/base.rb +2 -2
  44. data/lib/deimos/schema_class/enum.rb +1 -1
  45. data/lib/deimos/schema_class/record.rb +2 -2
  46. data/lib/deimos/test_helpers.rb +95 -320
  47. data/lib/deimos/tracing/provider.rb +6 -6
  48. data/lib/deimos/transcoder.rb +88 -0
  49. data/lib/deimos/utils/db_poller/base.rb +16 -14
  50. data/lib/deimos/utils/db_poller/state_based.rb +3 -3
  51. data/lib/deimos/utils/db_poller/time_based.rb +4 -4
  52. data/lib/deimos/utils/db_poller.rb +1 -1
  53. data/lib/deimos/utils/deadlock_retry.rb +1 -1
  54. data/lib/deimos/utils/{db_producer.rb → outbox_producer.rb} +16 -47
  55. data/lib/deimos/utils/schema_class.rb +0 -7
  56. data/lib/deimos/version.rb +1 -1
  57. data/lib/deimos.rb +79 -26
  58. data/lib/generators/deimos/{db_backend_generator.rb → outbox_backend_generator.rb} +4 -4
  59. data/lib/generators/deimos/schema_class_generator.rb +0 -1
  60. data/lib/generators/deimos/v2/templates/karafka.rb.tt +149 -0
  61. data/lib/generators/deimos/v2_generator.rb +193 -0
  62. data/lib/tasks/deimos.rake +5 -7
  63. data/spec/active_record_batch_consumer_association_spec.rb +22 -13
  64. data/spec/active_record_batch_consumer_spec.rb +84 -65
  65. data/spec/active_record_consume/batch_consumption_spec.rb +10 -10
  66. data/spec/active_record_consume/batch_slicer_spec.rb +12 -12
  67. data/spec/active_record_consume/mass_updater_spec.rb +137 -0
  68. data/spec/active_record_consumer_spec.rb +29 -13
  69. data/spec/active_record_producer_spec.rb +36 -26
  70. data/spec/backends/base_spec.rb +0 -23
  71. data/spec/backends/kafka_async_spec.rb +1 -3
  72. data/spec/backends/kafka_spec.rb +1 -3
  73. data/spec/backends/{db_spec.rb → outbox_spec.rb} +14 -20
  74. data/spec/batch_consumer_spec.rb +66 -116
  75. data/spec/consumer_spec.rb +53 -147
  76. data/spec/deimos_spec.rb +10 -126
  77. data/spec/kafka_source_spec.rb +19 -52
  78. data/spec/karafka/karafka.rb +69 -0
  79. data/spec/karafka_config/karafka_spec.rb +97 -0
  80. data/spec/logging_spec.rb +25 -0
  81. data/spec/message_spec.rb +9 -9
  82. data/spec/producer_spec.rb +112 -254
  83. data/spec/rake_spec.rb +1 -3
  84. data/spec/schema_backends/avro_validation_spec.rb +1 -1
  85. data/spec/schemas/com/my-namespace/MySchemaWithTitle.avsc +22 -0
  86. data/spec/snapshots/consumers-no-nest.snap +49 -0
  87. data/spec/snapshots/consumers.snap +49 -0
  88. data/spec/snapshots/consumers_and_producers-no-nest.snap +49 -0
  89. data/spec/snapshots/consumers_and_producers.snap +49 -0
  90. data/spec/snapshots/consumers_circular-no-nest.snap +49 -0
  91. data/spec/snapshots/consumers_circular.snap +49 -0
  92. data/spec/snapshots/consumers_complex_types-no-nest.snap +49 -0
  93. data/spec/snapshots/consumers_complex_types.snap +49 -0
  94. data/spec/snapshots/consumers_nested-no-nest.snap +49 -0
  95. data/spec/snapshots/consumers_nested.snap +49 -0
  96. data/spec/snapshots/namespace_folders.snap +49 -0
  97. data/spec/snapshots/namespace_map.snap +49 -0
  98. data/spec/snapshots/producers_with_key-no-nest.snap +49 -0
  99. data/spec/snapshots/producers_with_key.snap +49 -0
  100. data/spec/spec_helper.rb +61 -29
  101. data/spec/utils/db_poller_spec.rb +49 -39
  102. data/spec/utils/{db_producer_spec.rb → outbox_producer_spec.rb} +17 -184
  103. metadata +58 -67
  104. data/lib/deimos/batch_consumer.rb +0 -7
  105. data/lib/deimos/config/phobos_config.rb +0 -163
  106. data/lib/deimos/instrumentation.rb +0 -95
  107. data/lib/deimos/monkey_patches/phobos_cli.rb +0 -35
  108. data/lib/deimos/utils/inline_consumer.rb +0 -158
  109. data/lib/deimos/utils/lag_reporter.rb +0 -186
  110. data/lib/deimos/utils/schema_controller_mixin.rb +0 -129
  111. data/spec/config/configuration_spec.rb +0 -321
  112. data/spec/kafka_listener_spec.rb +0 -55
  113. data/spec/phobos.bad_db.yml +0 -73
  114. data/spec/phobos.yml +0 -77
  115. data/spec/utils/inline_consumer_spec.rb +0 -31
  116. data/spec/utils/lag_reporter_spec.rb +0 -76
  117. data/spec/utils/platform_schema_validation_spec.rb +0 -0
  118. data/spec/utils/schema_controller_mixin_spec.rb +0 -84
  119. /data/lib/generators/deimos/{db_backend → outbox_backend}/templates/migration +0 -0
  120. /data/lib/generators/deimos/{db_backend → outbox_backend}/templates/rails3_migration +0 -0
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: f2be475993ca3cec87d78682f7724b973f5aadb885612f98d5978169da2932bf
4
- data.tar.gz: 5ab0f052676184f800965bf68aebecac80583d1da2f38561a305e5f5b57a68c9
3
+ metadata.gz: 45c1138f93a466197d97834130389f2f17652221dee2ec8d909b8f238c85f554
4
+ data.tar.gz: 0b48e5070b44906ecf39e759fd3142d0b85805561cca43842407d5d823a0d381
5
5
  SHA512:
6
- metadata.gz: 918a363d9994537a221ccc6d00131c75a97f36c8cf89bdda47e0a6e6770b36fd1c3b07e5177d0ef284494020d90dd9081bc9d40ec41f4dc8bbe45e9c4165ea1f
7
- data.tar.gz: 60d316ef05ce427234e366e32f44addd7188adb45345b025bfb9fd21f339d0c579f317f1e55db20185b9f5f825af3226c27cb53d5e9336ec805200d2a9ce5bd9
6
+ metadata.gz: 88991e80cf0c4cfcd873058ed30de62b27d78ce0a29f84b66679b0c0b4799a0efb4a4fed647e54016b68f8e4bb282bdc1797ff52796b1ef615897107a156e263
7
+ data.tar.gz: 1b256132ce1385301c9b03cf0e3e2ca3c44da14c0a13063a6c911f6d782cb0a7832ec3a4913a5b3e420c663df99db62757108eafd76144d0d4351b9c47241190
data/.rubocop_todo.yml CHANGED
@@ -146,7 +146,6 @@ Lint/UselessAssignment:
146
146
  Metrics/AbcSize:
147
147
  Exclude:
148
148
  - 'lib/deimos/active_record_consume/message_consumption.rb'
149
- - 'lib/deimos/config/phobos_config.rb'
150
149
  - 'lib/deimos/instrumentation.rb'
151
150
  - 'lib/deimos/kafka_source.rb'
152
151
  - 'lib/deimos/kafka_topic_info.rb'
@@ -159,12 +158,6 @@ Metrics/AbcSize:
159
158
  - 'lib/deimos/utils/schema_controller_mixin.rb'
160
159
  - 'lib/generators/deimos/schema_class_generator.rb'
161
160
 
162
- # Offense count: 1
163
- # Configuration parameters: CountComments, Max, CountAsOne, ExcludedMethods.
164
- Metrics/MethodLength:
165
- Exclude:
166
- - 'lib/deimos/config/phobos_config.rb'
167
-
168
161
  # Offense count: 5
169
162
  # Configuration parameters: CountComments, Max, CountAsOne.
170
163
  Metrics/ModuleLength:
@@ -179,7 +172,6 @@ Metrics/ModuleLength:
179
172
  # Configuration parameters: IgnoredMethods, Max.
180
173
  Metrics/PerceivedComplexity:
181
174
  Exclude:
182
- - 'lib/deimos/config/phobos_config.rb'
183
175
  - 'lib/deimos/consume/batch_consumption.rb'
184
176
  - 'lib/deimos/kafka_source.rb'
185
177
  - 'lib/deimos/schema_backends/avro_schema_coercer.rb'
@@ -253,7 +245,6 @@ Style/FrozenStringLiteralComment:
253
245
  Style/GlobalStdStream:
254
246
  Exclude:
255
247
  - 'lib/deimos/config/configuration.rb'
256
- - 'lib/deimos/config/phobos_config.rb'
257
248
  - 'lib/deimos/metrics/mock.rb'
258
249
  - 'lib/deimos/test_helpers.rb'
259
250
  - 'lib/deimos/tracing/mock.rb'
@@ -329,14 +320,6 @@ Style/StringLiterals:
329
320
  - 'spec/schemas/my_namespace/my_schema_with_complex_type.rb'
330
321
  - 'spec/spec_helper.rb'
331
322
 
332
- # Offense count: 1
333
- # Cop supports --auto-correct.
334
- # Configuration parameters: EnforcedStyle, AllowSafeAssignment.
335
- # SupportedStyles: require_parentheses, require_no_parentheses, require_parentheses_when_complex
336
- Style/TernaryParentheses:
337
- Exclude:
338
- - 'lib/deimos/config/phobos_config.rb'
339
-
340
323
  # Offense count: 21
341
324
  # Cop supports --auto-correct.
342
325
  Style/TrailingBodyOnModule:
data/.tool-versions ADDED
@@ -0,0 +1 @@
1
+ ruby 3.2.2
data/CHANGELOG.md CHANGED
@@ -6,6 +6,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
6
6
  and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
7
7
 
8
8
  ## UNRELEASED
9
+ - Feature: Added `max_batch_size` config to producer to allow custom batch size for publishing.
10
+
11
+ # 1.24.3 - 2024-05-13
12
+ - Feature: Enable `producers.persistent_connections` phobos setting
13
+ - Feature: Add consumer configuration, `save_associations_first` to save associated records of primary class prior to upserting primary records. Foreign key of associated records are assigned to the record class prior to saving the record class
9
14
 
10
15
  # 1.24.2 - 2024-05-01
11
16
  - Fix: Deprecation notice with Rails 7.