deimos-ruby 2.0.0 → 2.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +1 -1
- data/CHANGELOG.md +3 -0
- data/CHANGELOG.md.orig +609 -0
- data/Gemfile +6 -0
- data/README.md +4 -2
- data/lib/deimos/ext/producer_middleware.rb +1 -0
- data/lib/deimos/kafka_message.rb +2 -1
- data/lib/deimos/version.rb +1 -1
- data/spec/generators/schema_class_generator_spec.rb +10 -1
- data/spec/utils/outbox_producer_spec.rb +12 -0
- metadata +3 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 4b183abdeabc75041a5d66ff5700061d23ab3f6a1613c753754ab007036f4e66
|
|
4
|
+
data.tar.gz: 49456497d55b2461a64ea0afdc1979f34e8ec1d5a654cddb23e2e3f20ffd9a2c
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: a6bf83750cc7530321f5df74d87aba53c1c08cd5d0cbfbaf57794043ee258de0e41965bfc09afc03c86ea3bb12baed0684d362ce29f506dba6d2cee4c87a70bb
|
|
7
|
+
data.tar.gz: 70cb1c5583cde7cb52775b65c597d91b05202f1f60ac5c5a3ae954532300aa4f4e88009f865cfa0df861e69ed3ad982e62fc19720777053eea4539d24e33bcc0
|
data/.github/workflows/ci.yml
CHANGED
data/CHANGELOG.md
CHANGED
|
@@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
|
|
8
8
|
## UNRELEASED
|
|
9
9
|
|
|
10
|
+
## 2.0.2 - 2025-03-10
|
|
11
|
+
- Fix: Outbox producer was trying to double-encode messages.
|
|
12
|
+
|
|
10
13
|
## 2.0.0 - 2025-03-03
|
|
11
14
|
- ***Full rewrite of Deimos to work with Karafka.*** Please see [Upgrading](./docs/UPGRADING.md) for full information.
|
|
12
15
|
|
data/CHANGELOG.md.orig
ADDED
|
@@ -0,0 +1,609 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project will be documented in this file.
|
|
4
|
+
|
|
5
|
+
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
|
|
6
|
+
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
|
+
|
|
8
|
+
## UNRELEASED
|
|
9
|
+
<<<<<<< HEAD
|
|
10
|
+
|
|
11
|
+
## 2.0.1 - 2025-03-10
|
|
12
|
+
- Fix: Added support to handle producing union type of multiple records & data types.
|
|
13
|
+
- Fix: Outbox producer was trying to double-encode messages.
|
|
14
|
+
=======
|
|
15
|
+
>>>>>>> parent of 91a65ea (Merge pull request #222 from flipp-oss/CIE-534)
|
|
16
|
+
|
|
17
|
+
## 2.0.0 - 2025-03-03
|
|
18
|
+
- ***Full rewrite of Deimos to work with Karafka.*** Please see [Upgrading](./docs/UPGRADING.md) for full information.
|
|
19
|
+
|
|
20
|
+
# 1.24.3 - 2024-05-13
|
|
21
|
+
- Feature: Enable `producers.persistent_connections` phobos setting
|
|
22
|
+
- Feature: Add consumer configuration, `save_associations_first` to save associated records of primary class prior to upserting primary records. Foreign key of associated records are assigned to the record class prior to saving the record class
|
|
23
|
+
|
|
24
|
+
# 1.24.2 - 2024-05-01
|
|
25
|
+
- Fix: Deprecation notice with Rails 7.
|
|
26
|
+
|
|
27
|
+
# 1.24.1 - 2024-03-26
|
|
28
|
+
- Fix: ActiveRecordConsumer batch consumption was not working with the trilogy adapter.
|
|
29
|
+
|
|
30
|
+
# 1.24.0 - 2024-02-26
|
|
31
|
+
|
|
32
|
+
***BREAKING CHANGE****
|
|
33
|
+
|
|
34
|
+
- Feature: Rename configuration option `generate_namespace_folders` to `use_full_namespace`. This will now use the entire namespace when generating schema classes
|
|
35
|
+
- Feature: Add configuration option `schema_namespace_map` to enable full control over the namespace for generated schema classes. Requires `use_full_namespace`
|
|
36
|
+
|
|
37
|
+
# 1.23.3 - 2024-01-25
|
|
38
|
+
- Feature: Add configuration to skip messages that are too large to publish via DB poller.
|
|
39
|
+
|
|
40
|
+
# 1.23.2 - 2024-01-22
|
|
41
|
+
- Fix: Send a `publish_error` metric for errors other than `DeliveryFailed`.
|
|
42
|
+
|
|
43
|
+
# 1.23.0 - 2024-01-09
|
|
44
|
+
|
|
45
|
+
- Fix: Fixed handler metric for status:received, status:success in batch consumption
|
|
46
|
+
- Feature: Allow pre processing of messages prior to bulk consumption
|
|
47
|
+
- Feature: Add global configuration for custom `bulk_import_id_generator` proc for all consumers
|
|
48
|
+
- Feature: Add individual configuration for custom `bulk_import_id_generator` proc per consumer
|
|
49
|
+
- Feature: Add global `replace_assocations` value for for all consumers
|
|
50
|
+
- Feature: Add individual `replace_assocations` value for for individual consumers
|
|
51
|
+
- Feature: `should_consume?` method accepts BatchRecord associations
|
|
52
|
+
- Feature: Reintroduce `filter_records` for bulk filtering of records prior to insertion
|
|
53
|
+
- Feature: Return valid and invalid records saved during consumption for further processing in `batch_consumption.valid_records` and `batch_consumption.invalid_records` ActiveSupport Notifications
|
|
54
|
+
|
|
55
|
+
# 1.22.5 - 2023-07-18
|
|
56
|
+
- Fix: Fixed buffer overflow crash with DB producer.
|
|
57
|
+
|
|
58
|
+
# 1.22.4 - 2023-07-05
|
|
59
|
+
- Feature: Add support for message headers.
|
|
60
|
+
|
|
61
|
+
# 1.22.3 - 2023-06-13
|
|
62
|
+
|
|
63
|
+
- Fix: Don't update last_sent to current time on every poll.
|
|
64
|
+
- Feature: Allow for infinite retries in DB poller.
|
|
65
|
+
|
|
66
|
+
# 1.22.2 - 2023-05-10
|
|
67
|
+
- Feature: Add `DEIMOS_TASK_NAME` env variable when running a task (consumer, DB poller, DB producer).
|
|
68
|
+
|
|
69
|
+
# 1.22.1 - 2023-05-01
|
|
70
|
+
|
|
71
|
+
- Fix: Bug introduced with 1.22 when associations are not used
|
|
72
|
+
|
|
73
|
+
# 1.22 - 2023-05-01
|
|
74
|
+
|
|
75
|
+
- Feature: Added `replace_associations` and changed default behavior for multi-table consuming. No longer relies on Rails direct associations and wonky magic for new vs. existing records.
|
|
76
|
+
- Fix: `bulk_import_id` is now handled by Deimos and does not need to be set by application code.
|
|
77
|
+
- ***BREAKING CHANGE*** Replaced `filter_records` with `should_consume?` on ActiveRecordConsumer.
|
|
78
|
+
- ***BREAKING CHANGE*** Replaced the behavior of `build_records` on ActiveRecordConsumer with a more powerful `record_attributes`.
|
|
79
|
+
- ***BREAKING CHANGE*** Removed the `association_list` config as it can now be inferred from the data.
|
|
80
|
+
|
|
81
|
+
# 1.21.1 - 2023-04-18
|
|
82
|
+
|
|
83
|
+
- Fix: Datadog tracing now works with Datadog 1.x
|
|
84
|
+
|
|
85
|
+
# 1.20.0 - 2023-04-17
|
|
86
|
+
|
|
87
|
+
- Feature: Updated the DB Poller logic to allow for inherited db poller classes to initialize producers and
|
|
88
|
+
be able to publish to multiple Kafka topics using the results from a single poll_query method
|
|
89
|
+
|
|
90
|
+
# 1.19.7 - 2023-04-17
|
|
91
|
+
|
|
92
|
+
- Fix: Update Datadog metrics backend so it doesn't crash on newer versions of dogstatsd-ruby
|
|
93
|
+
|
|
94
|
+
# 1.19.6 - 2023-04-04
|
|
95
|
+
|
|
96
|
+
- Fix: Fixing bug when having a not standard primary_key
|
|
97
|
+
|
|
98
|
+
# 1.19.5 - 2023-03-29
|
|
99
|
+
|
|
100
|
+
- Feature: add additional setting to Deimos.config - payload_log
|
|
101
|
+
|
|
102
|
+
# 1.19.4 - 2023-03-23
|
|
103
|
+
|
|
104
|
+
- Feature: Limit amount of transactions in a single database call with `max_db_batch_size`
|
|
105
|
+
|
|
106
|
+
# 1.19.3 - 2023-03-22
|
|
107
|
+
|
|
108
|
+
- Fix: Allow `have_sent` matcher to work with schema classes.
|
|
109
|
+
|
|
110
|
+
# 1.19.2 - 2023-03-20
|
|
111
|
+
|
|
112
|
+
- Fix for saving associations where the primary key is not `id`.
|
|
113
|
+
|
|
114
|
+
# 1.19.1 - 2023-03-17
|
|
115
|
+
|
|
116
|
+
- Fix for `rails g deimos:bulk_import_id` generator.
|
|
117
|
+
|
|
118
|
+
# 1.19.0 - 2023-02-21
|
|
119
|
+
|
|
120
|
+
- Feature: When consuming data in batch mode, this adds ability to save data to multiple tables with `association_list`.
|
|
121
|
+
- Fix bug with the `disable_producers` method where producers may remain disabled after the block
|
|
122
|
+
is done processing if an error occurs inside it.
|
|
123
|
+
|
|
124
|
+
# 1.18.2 - 2022-11-14
|
|
125
|
+
|
|
126
|
+
- Fixes bug related to wait between runs when no records are fetched by updating poll_info
|
|
127
|
+
|
|
128
|
+
# 1.18.1 - 2022-11-01
|
|
129
|
+
|
|
130
|
+
- Fix the primary_key detection way in `state_based` mode
|
|
131
|
+
|
|
132
|
+
# 1.18.0 - 2022-11-01
|
|
133
|
+
|
|
134
|
+
### Features :star:
|
|
135
|
+
|
|
136
|
+
- Add the `state_based` mode for DB pollers.
|
|
137
|
+
|
|
138
|
+
### Fixes :wrench:
|
|
139
|
+
- Fix the mock schema backend's `encode_key` method so it doesn't crash when used in application code.
|
|
140
|
+
|
|
141
|
+
# 1.17.1 - 2022-10-20
|
|
142
|
+
|
|
143
|
+
- Fix the log message for publishing messages so it uses the topic of the actual message instead of
|
|
144
|
+
the default for the producer.
|
|
145
|
+
- Use public sord gem instead of private fork.
|
|
146
|
+
|
|
147
|
+
# 1.17.0 - 2022-10-19
|
|
148
|
+
- Fix the linting step in the CI
|
|
149
|
+
- CHANGE: Add retries to DB poller and bypass "bad batches".
|
|
150
|
+
- Add tracing spans to DB poller production.
|
|
151
|
+
|
|
152
|
+
# 1.16.4 - 2022-09-09
|
|
153
|
+
|
|
154
|
+
- Now generates RBS types.
|
|
155
|
+
- Use `allocate` instead of `new` in `tombstone` to avoid issues with required fields in `initialize`.
|
|
156
|
+
|
|
157
|
+
# 1.16.3 - 2022-09-08
|
|
158
|
+
|
|
159
|
+
- Add the `tombstone` method to schema classes.
|
|
160
|
+
|
|
161
|
+
# 1.16.2 - 2022-09-07
|
|
162
|
+
|
|
163
|
+
- Added support of post_process method in ActiveRecordProducer
|
|
164
|
+
|
|
165
|
+
# 1.16.1 - 2022-08-03
|
|
166
|
+
|
|
167
|
+
- Fix issues with `enum` schema classes (e.g. equality not working, `to_s` not working)
|
|
168
|
+
- Changed assumption in the base schema class that it was a record (e.g. defining `to_h` instead of `as_json`). Moved record functionality to the record base class.
|
|
169
|
+
- Added the `active_span` and `set_tag` methods to the tracing classes.
|
|
170
|
+
- Added span tags for fields in SchemaControllerMixin.
|
|
171
|
+
- Updated SchemaControllerMixin so it works with generated schema classes.
|
|
172
|
+
- Fixed bug with previous release where the filename and constant names for `generate_namespace_folders` did not always match.
|
|
173
|
+
|
|
174
|
+
# 1.15.0 - 2022-07-20
|
|
175
|
+
|
|
176
|
+
- Update to `sigurd` 0.1.0 - DB producer should now exit when receiving a `SIGTERM` instead of throwing a `SignalException`
|
|
177
|
+
|
|
178
|
+
# 1.14.6 - 2022-06-21
|
|
179
|
+
|
|
180
|
+
- Add `generate_namespace_folders` to configuration; this will automatically generate subfolders to the `schemas` folder so that you can have different schemas with the same name but different namespaces generate separate classes.
|
|
181
|
+
|
|
182
|
+
# 1.14.5 - 2022-06-21
|
|
183
|
+
|
|
184
|
+
- Fix crash with the tracer when error happens in decoding a message during batch consuming
|
|
185
|
+
- Generate schema classes for all schemas, even without a consumer/producer set
|
|
186
|
+
|
|
187
|
+
# 1.14.4 - 2022-06-18
|
|
188
|
+
|
|
189
|
+
- Fix import in ActiveRecordConsumer on mysql
|
|
190
|
+
|
|
191
|
+
# 1.14.3 - 2022-06-17
|
|
192
|
+
|
|
193
|
+
- Fix issue with ActiveRecordConsumer double-decoding keys in batch mode.
|
|
194
|
+
|
|
195
|
+
# 1.14.2 - 2022-05-26
|
|
196
|
+
|
|
197
|
+
- Fix crash with `test_consume_message` when passing in an instance of a schema class instead of a hash.
|
|
198
|
+
|
|
199
|
+
# 1.14.1 - 2022-05-25
|
|
200
|
+
|
|
201
|
+
- Fix: When using key schemas, ActiveRecordConsumers were not finding the record by default.
|
|
202
|
+
|
|
203
|
+
# 1.14.0 - 2022-05-16
|
|
204
|
+
|
|
205
|
+
- **Breaking Change**: Nest sub-schemas by default into their parent schemas when generating classes.
|
|
206
|
+
- Add the `nest_child_schemas` option to essentially bring back the previous behavior in terms of code use (the actual classes will be separated out into different files).
|
|
207
|
+
|
|
208
|
+
# 1.13.3 - 2022-05-10
|
|
209
|
+
|
|
210
|
+
- Some cleanup on the active_record generator
|
|
211
|
+
- Fix crash with `test_consume_message` when using schema classes
|
|
212
|
+
- Add `[]=` and `merge` methods on the base schema class
|
|
213
|
+
|
|
214
|
+
# 1.13.2 - 2022-04-07
|
|
215
|
+
|
|
216
|
+
- Fix an issue with generating schema classes for schemas containing an enum with default value
|
|
217
|
+
|
|
218
|
+
# 1.13.1 - 2022-04-06
|
|
219
|
+
|
|
220
|
+
- Fix circular reference schema generation
|
|
221
|
+
|
|
222
|
+
# 1.13.0 - 2022-03-30
|
|
223
|
+
|
|
224
|
+
- Pass the Deimos logger to `AvroTurf::Messaging` for consistent logging
|
|
225
|
+
- Fix an issue with nullable enums not being included in the auto-generated schema class
|
|
226
|
+
|
|
227
|
+
# 1.12.6 - 2022-03-14
|
|
228
|
+
|
|
229
|
+
- Fix NameError when using Datadog Metrics
|
|
230
|
+
- Fix unwanted STDOUT output when loading the main `Deimos` module
|
|
231
|
+
|
|
232
|
+
# 1.12.5 - 2022-03-09
|
|
233
|
+
|
|
234
|
+
- Allow use of new avro_turf versions where child schemas are not listed with the top level schemas
|
|
235
|
+
- Add support for SASL authentication with brokers
|
|
236
|
+
- Add support for Basic auth with Schema Registry
|
|
237
|
+
|
|
238
|
+
# 1.12.4 - 2022-01-13
|
|
239
|
+
|
|
240
|
+
- Fix bug where schema controller mixin was using the schema name to register and not the namespaced schema name
|
|
241
|
+
|
|
242
|
+
# 1.12.3 - 2021-12-13
|
|
243
|
+
|
|
244
|
+
- Fix bug with previous release
|
|
245
|
+
|
|
246
|
+
# 1.12.2 - 2021-12-10
|
|
247
|
+
|
|
248
|
+
### Features :star:
|
|
249
|
+
|
|
250
|
+
- Added `Deimos.encode` and `Deimos.decode` for non-topic-related encoding and decoding.
|
|
251
|
+
|
|
252
|
+
# 1.12.1 - 2021-11-02
|
|
253
|
+
|
|
254
|
+
- ### Fixes :wrench:
|
|
255
|
+
- Fixed issue where Schema Class Consumer/Producer are using `Deimos::` instead of `Schema::` for instances of classes.
|
|
256
|
+
|
|
257
|
+
# 1.12.0 - 2021-11-01
|
|
258
|
+
|
|
259
|
+
### Features :star:
|
|
260
|
+
|
|
261
|
+
- Generate Schema classes from Avro Schemas
|
|
262
|
+
- Use Schema Classes in your consumer and producer
|
|
263
|
+
|
|
264
|
+
## 1.11.0 - 2021-08-27
|
|
265
|
+
|
|
266
|
+
- ### Fixes :wrench:
|
|
267
|
+
- Fixed issue where ActiveRecord batch consumption could fail when decoding keys.
|
|
268
|
+
|
|
269
|
+
- ### Roadmap :car:
|
|
270
|
+
- TestHelper does not automatically reset Deimos config before each test. [#120](https://github.com/flipp-oss/deimos/pull/120).
|
|
271
|
+
**Please note that this is a breaking change**
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
## 1.10.2 - 2021-07-20
|
|
275
|
+
|
|
276
|
+
- ### Fixes :wrench:
|
|
277
|
+
|
|
278
|
+
- Fixed issue where producers would stay in an error state after e.g. authorization failures for one topic that wouldn't apply to other topics.
|
|
279
|
+
|
|
280
|
+
## 1.10.1 - 2021-06-21
|
|
281
|
+
|
|
282
|
+
- ### Fixes :wrench:
|
|
283
|
+
|
|
284
|
+
- Fixed crash when trying to decode a nil payload (e.g. during instrumentation of `send_produce_error`.)
|
|
285
|
+
|
|
286
|
+
## 1.10.0 - 2021-03-22
|
|
287
|
+
|
|
288
|
+
- ### Roadmap :car:
|
|
289
|
+
|
|
290
|
+
- Extracted the configuration piece into a separate gem, [fig_tree](https://www.github.com/flipp-oss/fig_tree).
|
|
291
|
+
- Added a `save_record` method to ActiveRecordConsumer in case calling code wants to work with the record before saving.
|
|
292
|
+
|
|
293
|
+
- ### Fixes :wrench:
|
|
294
|
+
|
|
295
|
+
- Fixed a regression where the default values for consumer / Phobos listener configs were not correct (they were all nil). This is technically a breaking change, but it puts the configs back the way they were at version 1.4 and matches the documentation.
|
|
296
|
+
|
|
297
|
+
## 1.9.2 - 2021-01-29
|
|
298
|
+
|
|
299
|
+
- ### Fixes :wrench:
|
|
300
|
+
|
|
301
|
+
- Fix for `uninitialized constant ActiveSupport::Autoload` in certain circumstances
|
|
302
|
+
- Removed unnecessary monkey patch which was crashing on newer versions of ruby-kafka
|
|
303
|
+
|
|
304
|
+
## 1.9.0 - 2021-01-28
|
|
305
|
+
|
|
306
|
+
- ### Roadmap :car:
|
|
307
|
+
|
|
308
|
+
- Bumped the version of ruby-kafka to latest
|
|
309
|
+
|
|
310
|
+
- ### Fixes :wrench:
|
|
311
|
+
|
|
312
|
+
- Prevents DB Poller from reconnecting to DB if there is an open transaction
|
|
313
|
+
- Replaces `before` by `prepend_before` for more consistent test setups.
|
|
314
|
+
- Adds validation in the `kafka_producers` method (fixes [#90](https://github.com/flipp-oss/deimos/issues/90))
|
|
315
|
+
|
|
316
|
+
## 1.8.7 - 2021-01-14
|
|
317
|
+
|
|
318
|
+
- ### Roadmap :car:
|
|
319
|
+
- Update Phobos version to allow version 1.9 or 2.x.
|
|
320
|
+
|
|
321
|
+
## 1.8.6 - 2021-01-14
|
|
322
|
+
|
|
323
|
+
- ### Fixes :wrench:
|
|
324
|
+
- Fix for configuration bug with Ruby 3.0 (** instead of passing hash)
|
|
325
|
+
|
|
326
|
+
## 1.8.5 - 2021-01-13
|
|
327
|
+
|
|
328
|
+
- ### Fixes :wrench:
|
|
329
|
+
- Fixes for Rails 6.1 (remove usage of `update_attributes!`)
|
|
330
|
+
|
|
331
|
+
## 1.8.4 - 2020-12-02
|
|
332
|
+
|
|
333
|
+
### Features :star:
|
|
334
|
+
- Add overridable "process_message?" method to ActiveRecordConsumer to allow for skipping of saving/updating records
|
|
335
|
+
|
|
336
|
+
### Fixes :wrench:
|
|
337
|
+
|
|
338
|
+
- Do not apply type coercion to `timestamp-millis` and `timestamp-micros` logical types (fixes [#97](https://github.com/flipp-oss/deimos/issues/97))
|
|
339
|
+
|
|
340
|
+
## 1.8.3 - 2020-11-18
|
|
341
|
+
|
|
342
|
+
### Fixes :wrench:
|
|
343
|
+
- Do not resend already sent messages when splitting up batches
|
|
344
|
+
(fixes [#24](https://github.com/flipp-oss/deimos/issues/24))
|
|
345
|
+
- KafkaSource crashing on bulk-imports if import hooks are disabled
|
|
346
|
+
(fixes [#73](https://github.com/flipp-oss/deimos/issues/73))
|
|
347
|
+
- #96 Use string-safe encoding for partition keys
|
|
348
|
+
- Retry on offset seek failures in inline consumer
|
|
349
|
+
(fixes [#5](Inline consumer should use retries when seeking))
|
|
350
|
+
|
|
351
|
+
## 1.8.2 - 2020-09-25
|
|
352
|
+
|
|
353
|
+
### Features :star:
|
|
354
|
+
- Add "disabled" config field to consumers to allow disabling
|
|
355
|
+
individual consumers without having to comment out their
|
|
356
|
+
entries and possibly affecting unit tests.
|
|
357
|
+
|
|
358
|
+
### Fixes :wrench:
|
|
359
|
+
- Prepend topic_prefix while encoding messages
|
|
360
|
+
(fixes [#37](https://github.com/flipp-oss/deimos/issues/37))
|
|
361
|
+
- Raise error if producing without a topic
|
|
362
|
+
(fixes [#50](https://github.com/flipp-oss/deimos/issues/50))
|
|
363
|
+
- Don't try to load producers/consumers when running rake tasks involving webpacker or assets
|
|
364
|
+
|
|
365
|
+
## 1.8.2-beta2 - 2020-09-15
|
|
366
|
+
|
|
367
|
+
### Features :star:
|
|
368
|
+
|
|
369
|
+
- Add details on using schema backend directly in README.
|
|
370
|
+
- Default to the provided schema if topic is not provided when
|
|
371
|
+
encoding to `AvroSchemaRegistry`.
|
|
372
|
+
- Add mapping syntax for the `schema` call in `SchemaControllerMixin`.
|
|
373
|
+
|
|
374
|
+
## 1.8.2-beta1 - 2020-09-09
|
|
375
|
+
|
|
376
|
+
### Features :star:
|
|
377
|
+
|
|
378
|
+
- Added the ability to specify the topic for `publish`
|
|
379
|
+
and `publish_list` in a producer
|
|
380
|
+
|
|
381
|
+
## 1.8.1-beta9 - 2020-08-27
|
|
382
|
+
|
|
383
|
+
### Fixes :wrench:
|
|
384
|
+
- Moved the TestHelpers hook to `before(:suite)` to allow for
|
|
385
|
+
overriding e.g. in integration tests.
|
|
386
|
+
|
|
387
|
+
## 1.8.1-beta7 - 2020-08-25
|
|
388
|
+
|
|
389
|
+
### Fixes :wrench:
|
|
390
|
+
- Fix for crash when sending pending metrics with DB producer.
|
|
391
|
+
- Fix for compacting messages if all the keys are already unique
|
|
392
|
+
(fixes [#75](https://github.com/flipp-oss/deimos/issues/75))
|
|
393
|
+
|
|
394
|
+
## 1.8.1-beta6 - 2020-08-13
|
|
395
|
+
|
|
396
|
+
### Fixes :wrench:
|
|
397
|
+
|
|
398
|
+
- Fix for consuming nil payloads with Ruby 2.3.
|
|
399
|
+
|
|
400
|
+
## 1.8.1-beta5 - 2020-08-13
|
|
401
|
+
|
|
402
|
+
### Fixes :wrench:
|
|
403
|
+
- Fix regression bug which introduces backwards incompatibility
|
|
404
|
+
with ActiveRecordProducer's `record_attributes` method.
|
|
405
|
+
|
|
406
|
+
## 1.8.1-beta4 - 2020-08-12
|
|
407
|
+
|
|
408
|
+
### Fixes :wrench:
|
|
409
|
+
- Fix regression bug where arrays were not being encoded
|
|
410
|
+
|
|
411
|
+
## 1.8.1-beta3 - 2020-08-05
|
|
412
|
+
|
|
413
|
+
### Fixes :wrench:
|
|
414
|
+
- Simplify decoding messages and handle producer not found
|
|
415
|
+
- Consolidate types in sub-records recursively
|
|
416
|
+
(fixes [#72](https://github.com/flipp-oss/deimos/issues/72))
|
|
417
|
+
|
|
418
|
+
## 1.8.1-beta2 - 2020-07-28
|
|
419
|
+
|
|
420
|
+
### Features :star:
|
|
421
|
+
- Add `SchemaControllerMixin` to encode and decode schema-encoded
|
|
422
|
+
payloads in Rails controllers.
|
|
423
|
+
|
|
424
|
+
## 1.8.1-beta1 - 2020-07-22
|
|
425
|
+
|
|
426
|
+
### Fixes :wrench:
|
|
427
|
+
- Retry deleting messages without resending the batch if the
|
|
428
|
+
delete fails (fixes [#34](https://github.com/flipp-oss/deimos/issues/34))
|
|
429
|
+
- Delete messages in batches rather than all at once to
|
|
430
|
+
cut down on the chance of a deadlock.
|
|
431
|
+
|
|
432
|
+
### Features :star:
|
|
433
|
+
- Add `last_processed_at` to `kafka_topic_info` to ensure
|
|
434
|
+
that wait metrics are accurate in cases where records
|
|
435
|
+
get created with an old `created_at` time (e.g. for
|
|
436
|
+
long-running transactions).
|
|
437
|
+
- Add generator for ActiveRecord models and migrations (fixes [#6](https://github.com/flipp-oss/deimos/issues/6))
|
|
438
|
+
|
|
439
|
+
## 1.8.0-beta2 - 2020-07-08
|
|
440
|
+
|
|
441
|
+
### Fixes :wrench:
|
|
442
|
+
- Fix crash with batch consumption due to not having ActiveSupport::Concern
|
|
443
|
+
|
|
444
|
+
### Features :star:
|
|
445
|
+
- Add `first_offset` to the metadata sent via the batch
|
|
446
|
+
|
|
447
|
+
## 1.8.0-beta1 - 2020-07-06
|
|
448
|
+
### Features :star:
|
|
449
|
+
- Added `ActiveRecordConsumer` batch mode
|
|
450
|
+
|
|
451
|
+
### Fixes :wrench:
|
|
452
|
+
- Fixes `send_produce_error` to decode `failed_messages` with built-in decoder.
|
|
453
|
+
- Lag calculation can be incorrect if no messages are being consumed.
|
|
454
|
+
- Fixed bug where printing messages on a MessageSizeTooLarge
|
|
455
|
+
error didn't work.
|
|
456
|
+
|
|
457
|
+
### Roadmap
|
|
458
|
+
- Moved SignalHandler and Executor to the `sigurd` gem.
|
|
459
|
+
|
|
460
|
+
## 1.7.0-beta1 - 2020-05-12
|
|
461
|
+
### Features :star:
|
|
462
|
+
- Added the DB Poller feature / process.
|
|
463
|
+
|
|
464
|
+
## 1.6.4 - 2020-05-11
|
|
465
|
+
- Fixed the payload logging fix for errored messages as well.
|
|
466
|
+
|
|
467
|
+
## 1.6.3 - 2020-05-04
|
|
468
|
+
### Fixes :wrench:
|
|
469
|
+
- Fixed the payload logging fix.
|
|
470
|
+
|
|
471
|
+
## 1.6.2 - 2020-05-04
|
|
472
|
+
### Fixes :wrench:
|
|
473
|
+
- When saving records via `ActiveRecordConsumer`, update `updated_at` to today's time
|
|
474
|
+
even if nothing else was saved.
|
|
475
|
+
- When logging payloads and metadata, decode them first.
|
|
476
|
+
- Fixes bug in `KafkaSource` that crashes when importing a mix of existing and new records with the `:on_duplicate_key_update` option.
|
|
477
|
+
|
|
478
|
+
## [1.6.1] - 2020-04-20
|
|
479
|
+
### Fixes :wrench:
|
|
480
|
+
- Re-consuming a message after crashing would try to re-decode message keys.
|
|
481
|
+
|
|
482
|
+
# [1.6.0] - 2020-03-05
|
|
483
|
+
### Roadmap :car:
|
|
484
|
+
- Removed `was_message_sent?` method from `TestHelpers`.
|
|
485
|
+
|
|
486
|
+
# [1.6.0-beta1] - 2020-02-05
|
|
487
|
+
### Roadmap :car:
|
|
488
|
+
- Updated dependency for Phobos to 1.9.0-beta3. This ensures compatibility with
|
|
489
|
+
Phobos 2.0.
|
|
490
|
+
### Fixes :wrench:
|
|
491
|
+
- Fixed RSpec warning when using `test_consume_invalid_message`.
|
|
492
|
+
|
|
493
|
+
# [1.5.0-beta2] - 2020-01-17
|
|
494
|
+
### Roadmap :car:
|
|
495
|
+
- Added schema backends, which should simplify Avro encoding and make it
|
|
496
|
+
more flexible for unit tests and local development.
|
|
497
|
+
### Features :star:
|
|
498
|
+
- Add `:test` producer backend which replaces the existing TestHelpers
|
|
499
|
+
functionality of writing messages to an in-memory hash.
|
|
500
|
+
|
|
501
|
+
# [1.4.0-beta7] - 2019-12-16
|
|
502
|
+
### Fixes :wrench:
|
|
503
|
+
- Clone loggers when assigning to multiple levels.
|
|
504
|
+
|
|
505
|
+
# [1.4.0-beta6] - 2019-12-16
|
|
506
|
+
### Features :star:
|
|
507
|
+
- Added default for max_bytes_per_partition.
|
|
508
|
+
|
|
509
|
+
# [1.4.0-beta4] - 2019-11-26
|
|
510
|
+
### Features :star:
|
|
511
|
+
- Added `define_settings` to define settings without invoking callbacks.
|
|
512
|
+
|
|
513
|
+
# [1.4.0-beta2] - 2019-11-22
|
|
514
|
+
### Fixes :wrench:
|
|
515
|
+
- Settings with default_proc were being called immediately
|
|
516
|
+
instead of being lazy-evaluated.
|
|
517
|
+
|
|
518
|
+
# [1.4.0-beta1] - 2019-11-22
|
|
519
|
+
### Roadmap :car:
|
|
520
|
+
- Complete revamp of configuration method.
|
|
521
|
+
|
|
522
|
+
# [1.3.0-beta5] - 2020-01-14
|
|
523
|
+
### Features :star:
|
|
524
|
+
- Added `db_producer.insert` and `db_producer.process` metrics.
|
|
525
|
+
|
|
526
|
+
# [1.3.0-beta4] - 2019-12-02
|
|
527
|
+
### Fixes :wrench:
|
|
528
|
+
- Fixed bug where by running `rake deimos:start` without
|
|
529
|
+
specifying a producer backend would crash.
|
|
530
|
+
|
|
531
|
+
# [1.3.0-beta3] - 2019-11-26
|
|
532
|
+
### Fixes :wrench:
|
|
533
|
+
- Fixed bug in TestHelpers where key_decoder was not stubbed out.
|
|
534
|
+
|
|
535
|
+
# [1.3.0-beta2] - 2019-11-22
|
|
536
|
+
### Fixes :wrench:
|
|
537
|
+
- Fixed bug where consumers would require a key config in all cases
|
|
538
|
+
even though it's optional if they don't use keys.
|
|
539
|
+
|
|
540
|
+
# [1.3.0-beta1] - 2019-11-21
|
|
541
|
+
### Features :star:
|
|
542
|
+
- Added `fetch_record` and `assign_key` methods to ActiveRecordConsumer.
|
|
543
|
+
|
|
544
|
+
# [1.2.0-beta1] - 2019-09-12
|
|
545
|
+
### Features :star:
|
|
546
|
+
- Added `fatal_error` to both global config and consumer classes.
|
|
547
|
+
- Changed `pending_db_messages_max_wait` metric to send per topic.
|
|
548
|
+
- Added config to compact messages in the DB producer.
|
|
549
|
+
- Added config to log messages in the DB producer.
|
|
550
|
+
- Added config to provide a separate logger to the DB producer.
|
|
551
|
+
|
|
552
|
+
# [1.1.0-beta2] - 2019-09-11
|
|
553
|
+
### Fixes :wrench:
|
|
554
|
+
- Fixed bug where ActiveRecordConsumer was not using `unscoped` to update
|
|
555
|
+
via primary key and causing duplicate record errors.
|
|
556
|
+
|
|
557
|
+
# [1.1.0-beta1] - 2019-09-10
|
|
558
|
+
### Features :star:
|
|
559
|
+
- Added BatchConsumer.
|
|
560
|
+
|
|
561
|
+
## [1.0.0] - 2019-09-03
|
|
562
|
+
### Roadmap :car:
|
|
563
|
+
- Official release of Deimos 1.0!
|
|
564
|
+
|
|
565
|
+
## [1.0.0-beta26] - 2019-08-29
|
|
566
|
+
- Recover from Kafka::MessageSizeTooLarge in the DB producer.
|
|
567
|
+
- Shut down sync producers correctly when persistent_connections is true.
|
|
568
|
+
- Notify when messages fail to produce in the DB producer.
|
|
569
|
+
- Delete messages on failure and rely on notification.
|
|
570
|
+
|
|
571
|
+
## [1.0.0-beta25] - 2019-08-28
|
|
572
|
+
- Fix bug where crashing would cause producers to stay disabled
|
|
573
|
+
|
|
574
|
+
## [1.0.0-beta24] - 2019-08-26
|
|
575
|
+
- Reconnect DB backend if database goes away.
|
|
576
|
+
- Sleep only 5 seconds between attempts instead of using exponential backoff.
|
|
577
|
+
- Fix for null payload being Avro-encoded.
|
|
578
|
+
|
|
579
|
+
## [1.0.0-beta23] - 2019-08-22
|
|
580
|
+
- Fix bug where nil payloads were not being saved to the DB.
|
|
581
|
+
- Fix DB producer rake task looking at THREADS env var instead of THREAD_COUNT.
|
|
582
|
+
- Debug messages in the DB producer if debug logs are turned on.
|
|
583
|
+
- Changed logger in specs to info.
|
|
584
|
+
|
|
585
|
+
## [1.0.0-beta22] - 2019-08-09
|
|
586
|
+
- Add `pending_db_messages_max_wait` metric for the DB producer.
|
|
587
|
+
- Fix mock metrics to allow optional option hashes.
|
|
588
|
+
|
|
589
|
+
## [1.0.0-beta21] - 2019-08-08
|
|
590
|
+
- Handle Phobos `persistent_connections` setting in handling buffer overflows
|
|
591
|
+
|
|
592
|
+
## [1.0.0-beta20] - 2019-08-07
|
|
593
|
+
- Catch buffer overflows when producing via the DB producer and split the
|
|
594
|
+
batch up.
|
|
595
|
+
|
|
596
|
+
## [1.0.0-beta19] - 2019-08-06
|
|
597
|
+
- Fix for DB producer crashing on error in Rails 3.
|
|
598
|
+
|
|
599
|
+
## [1.0.0-beta18] - 2019-08-02
|
|
600
|
+
- Fixed crash when sending metrics in a couple of places.
|
|
601
|
+
|
|
602
|
+
## [1.0.0-beta17] - 2019-07-31
|
|
603
|
+
- Added `rails deimos:db_producer` rake task.
|
|
604
|
+
- Fixed the DB producer so it runs inline instead of on a separate thread.
|
|
605
|
+
Calling code should run it on a thread manually if that is the desired
|
|
606
|
+
behavior.
|
|
607
|
+
|
|
608
|
+
## [1.0.0-beta15] - 2019-07-08
|
|
609
|
+
- Initial release.
|
data/Gemfile
CHANGED
data/README.md
CHANGED
|
@@ -10,8 +10,10 @@ A Ruby framework for marrying Kafka, a schema definition like Avro, and/or Activ
|
|
|
10
10
|
a useful toolbox of goodies for Ruby-based Kafka development.
|
|
11
11
|
Built on [Karafka](https://karafka.io/).
|
|
12
12
|
|
|
13
|
-
[!IMPORTANT]
|
|
14
|
-
Deimos 2.x is a major rewrite from 1.x. Please see the [Upgrading Guide](./docs/UPGRADING.md) for information on the changes and how to upgrade.
|
|
13
|
+
> [!IMPORTANT]
|
|
14
|
+
> Deimos 2.x is a major rewrite from 1.x. Please see the [Upgrading Guide](./docs/UPGRADING.md) for information on the changes and how to upgrade.
|
|
15
|
+
|
|
16
|
+
Version 1.0 documentation can be found [here](https://github.com/flipp-oss/deimos/tree/v1).
|
|
15
17
|
|
|
16
18
|
<!--ts-->
|
|
17
19
|
* [Additional Documentation](#additional-documentation)
|
|
@@ -11,6 +11,7 @@ module Deimos
|
|
|
11
11
|
) do
|
|
12
12
|
config = Deimos.karafka_config_for(topic: message[:topic])
|
|
13
13
|
return message if config.nil? || config.schema.nil?
|
|
14
|
+
return message if message.delete(:already_encoded)
|
|
14
15
|
return if message[:payload] && !message[:payload].is_a?(Hash) && !message[:payload].is_a?(SchemaClass::Record)
|
|
15
16
|
|
|
16
17
|
m = Deimos::Message.new(message[:payload].to_h,
|
data/lib/deimos/kafka_message.rb
CHANGED
data/lib/deimos/version.rb
CHANGED
|
@@ -4,8 +4,17 @@ require 'generators/deimos/schema_class_generator'
|
|
|
4
4
|
require 'fileutils'
|
|
5
5
|
|
|
6
6
|
class MultiFileSerializer
|
|
7
|
+
def process_string(s)
|
|
8
|
+
# Ruby 3.4 changes how hashes are printed
|
|
9
|
+
if Gem::Version.new(RUBY_VERSION) > Gem::Version.new('3.4.0')
|
|
10
|
+
s.gsub(/{"(.*)" => /, '{"\1"=>')
|
|
11
|
+
else
|
|
12
|
+
s
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
end
|
|
7
16
|
def dump(value)
|
|
8
|
-
value.keys.sort.map { |k| "#{k}:\n#{value[k]}\n" }.join("\n")
|
|
17
|
+
value.keys.sort.map { |k| "#{k}:\n#{process_string(value[k])}\n" }.join("\n")
|
|
9
18
|
end
|
|
10
19
|
end
|
|
11
20
|
|
|
@@ -146,12 +146,14 @@ each_db_config(Deimos::Utils::OutboxProducer) do
|
|
|
146
146
|
expect(producer).to receive(:send_pending_metrics).twice
|
|
147
147
|
expect(producer).to receive(:produce_messages).ordered.with([
|
|
148
148
|
{
|
|
149
|
+
already_encoded: true,
|
|
149
150
|
payload: 'mess1',
|
|
150
151
|
key: nil,
|
|
151
152
|
partition_key: 'key1',
|
|
152
153
|
topic: 'my-topic'
|
|
153
154
|
},
|
|
154
155
|
{
|
|
156
|
+
already_encoded: true,
|
|
155
157
|
payload: 'mess2',
|
|
156
158
|
key: nil,
|
|
157
159
|
partition_key: 'key2',
|
|
@@ -167,12 +169,14 @@ each_db_config(Deimos::Utils::OutboxProducer) do
|
|
|
167
169
|
and_return(messages[2..3])
|
|
168
170
|
expect(producer).to receive(:produce_messages).ordered.with([
|
|
169
171
|
{
|
|
172
|
+
already_encoded: true,
|
|
170
173
|
payload: 'mess3',
|
|
171
174
|
partition_key: 'key3',
|
|
172
175
|
key: nil,
|
|
173
176
|
topic: 'my-topic'
|
|
174
177
|
},
|
|
175
178
|
{
|
|
179
|
+
already_encoded: true,
|
|
176
180
|
payload: 'mess4',
|
|
177
181
|
partition_key: 'key4',
|
|
178
182
|
key: nil,
|
|
@@ -357,12 +361,14 @@ each_db_config(Deimos::Utils::OutboxProducer) do
|
|
|
357
361
|
|
|
358
362
|
expect(producer).to have_received(:produce_messages).with([
|
|
359
363
|
{
|
|
364
|
+
already_encoded: true,
|
|
360
365
|
payload: 'mess1',
|
|
361
366
|
partition_key: 'key1',
|
|
362
367
|
key: nil,
|
|
363
368
|
topic: 'topic1'
|
|
364
369
|
},
|
|
365
370
|
{
|
|
371
|
+
already_encoded: true,
|
|
366
372
|
payload: 'mess2',
|
|
367
373
|
key: nil,
|
|
368
374
|
partition_key: 'key2',
|
|
@@ -371,12 +377,14 @@ each_db_config(Deimos::Utils::OutboxProducer) do
|
|
|
371
377
|
])
|
|
372
378
|
expect(producer).to have_received(:produce_messages).with([
|
|
373
379
|
{
|
|
380
|
+
already_encoded: true,
|
|
374
381
|
payload: 'mess3',
|
|
375
382
|
key: nil,
|
|
376
383
|
partition_key: 'key3',
|
|
377
384
|
topic: 'topic1'
|
|
378
385
|
},
|
|
379
386
|
{
|
|
387
|
+
already_encoded: true,
|
|
380
388
|
payload: 'mess4',
|
|
381
389
|
key: nil,
|
|
382
390
|
partition_key: 'key4',
|
|
@@ -385,12 +393,14 @@ each_db_config(Deimos::Utils::OutboxProducer) do
|
|
|
385
393
|
])
|
|
386
394
|
expect(producer).to have_received(:produce_messages).with([
|
|
387
395
|
{
|
|
396
|
+
already_encoded: true,
|
|
388
397
|
payload: 'mess1',
|
|
389
398
|
key: 'key1',
|
|
390
399
|
partition_key: 'key1',
|
|
391
400
|
topic: 'topic3'
|
|
392
401
|
},
|
|
393
402
|
{
|
|
403
|
+
already_encoded: true,
|
|
394
404
|
payload: 'mess2',
|
|
395
405
|
partition_key: 'key2',
|
|
396
406
|
key: 'key2',
|
|
@@ -399,12 +409,14 @@ each_db_config(Deimos::Utils::OutboxProducer) do
|
|
|
399
409
|
])
|
|
400
410
|
expect(producer).to have_received(:produce_messages).with([
|
|
401
411
|
{
|
|
412
|
+
already_encoded: true,
|
|
402
413
|
payload: 'mess3',
|
|
403
414
|
key: 'key3',
|
|
404
415
|
partition_key: 'key3',
|
|
405
416
|
topic: 'topic3'
|
|
406
417
|
},
|
|
407
418
|
{
|
|
419
|
+
already_encoded: true,
|
|
408
420
|
payload: 'mess4',
|
|
409
421
|
partition_key: 'key4',
|
|
410
422
|
key: 'key4',
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: deimos-ruby
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 2.0.
|
|
4
|
+
version: 2.0.2
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Daniel Orner
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2025-03-
|
|
11
|
+
date: 2025-03-10 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: avro_turf
|
|
@@ -418,6 +418,7 @@ files:
|
|
|
418
418
|
- ".ruby-version"
|
|
419
419
|
- ".tool-versions"
|
|
420
420
|
- CHANGELOG.md
|
|
421
|
+
- CHANGELOG.md.orig
|
|
421
422
|
- CODE_OF_CONDUCT.md
|
|
422
423
|
- Dockerfile
|
|
423
424
|
- Gemfile
|