deimos-ruby 2.5.3 → 2.6.0.pre.beta1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +2 -0
- data/README.md +20 -9
- data/deimos-ruby.gemspec +1 -0
- data/docs/CONFIGURATION.md +11 -4
- data/lib/deimos/active_record_consume/batch_consumption.rb +1 -1
- data/lib/deimos/active_record_consume/message_consumption.rb +2 -2
- data/lib/deimos/active_record_consumer.rb +1 -1
- data/lib/deimos/active_record_producer.rb +1 -1
- data/lib/deimos/config/configuration.rb +34 -11
- data/lib/deimos/ext/producer_middleware.rb +2 -2
- data/lib/deimos/logging.rb +9 -0
- data/lib/deimos/producer.rb +5 -5
- data/lib/deimos/schema_backends/avro_base.rb +7 -36
- data/lib/deimos/schema_class.rb +72 -0
- data/lib/deimos/transcoder.rb +6 -6
- data/lib/deimos/utils/schema_class.rb +14 -47
- data/lib/deimos/version.rb +1 -1
- data/lib/deimos.rb +3 -3
- data/lib/tasks/deimos.rake +16 -4
- data/regenerate_test_schema_classes.rb +12 -3
- data/spec/active_record_batch_consumer_association_spec.rb +1 -1
- data/spec/active_record_batch_consumer_spec.rb +1 -1
- data/spec/active_record_consume/batch_consumption_spec.rb +1 -1
- data/spec/active_record_consumer_spec.rb +3 -3
- data/spec/active_record_producer_spec.rb +1 -1
- data/spec/batch_consumer_spec.rb +2 -2
- data/spec/consumer_spec.rb +9 -9
- data/spec/schema_class_spec.rb +65 -0
- data/spec/schemas/my_namespace/generated.rb +4 -4
- data/spec/schemas/my_namespace/my_long_namespace_schema.rb +2 -2
- data/spec/schemas/my_namespace/my_nested_schema.rb +3 -3
- data/spec/schemas/my_namespace/my_schema.rb +2 -2
- data/spec/schemas/my_namespace/my_schema_compound_key.rb +2 -2
- data/spec/schemas/my_namespace/my_schema_id_key.rb +2 -2
- data/spec/schemas/my_namespace/my_schema_key.rb +2 -2
- data/spec/schemas/my_namespace/my_schema_with_boolean.rb +2 -2
- data/spec/schemas/my_namespace/my_schema_with_circular_reference.rb +3 -3
- data/spec/schemas/my_namespace/my_schema_with_complex_type.rb +6 -6
- data/spec/schemas/my_namespace/my_schema_with_date_time.rb +2 -2
- data/spec/schemas/my_namespace/my_schema_with_id.rb +2 -2
- data/spec/schemas/my_namespace/my_schema_with_title.rb +2 -2
- data/spec/schemas/my_namespace/my_schema_with_union_type.rb +6 -6
- data/spec/schemas/my_namespace/my_schema_with_unique_id.rb +2 -2
- data/spec/schemas/my_namespace/my_updated_schema.rb +1 -1
- data/spec/schemas/my_namespace/request/create_topic.rb +2 -2
- data/spec/schemas/my_namespace/request/index.rb +2 -2
- data/spec/schemas/my_namespace/request/update_request.rb +2 -2
- data/spec/schemas/my_namespace/response/create_topic.rb +2 -2
- data/spec/schemas/my_namespace/response/index.rb +2 -2
- data/spec/schemas/my_namespace/response/update_response.rb +2 -2
- data/spec/schemas/my_namespace/wibble.rb +2 -2
- data/spec/schemas/my_namespace/widget.rb +2 -2
- data/spec/schemas/my_namespace/widget_the_second.rb +2 -2
- data/spec/schemas/my_namespace/widget_the_third.rb +2 -2
- data/spec/spec_helper.rb +2 -2
- data/spec/utils/db_poller_spec.rb +1 -1
- metadata +18 -27
- data/lib/deimos/schema_class/base.rb +0 -62
- data/lib/deimos/schema_class/enum.rb +0 -49
- data/lib/deimos/schema_class/record.rb +0 -100
- data/lib/generators/deimos/schema_class/templates/schema_class.rb.tt +0 -8
- data/lib/generators/deimos/schema_class/templates/schema_enum.rb.tt +0 -13
- data/lib/generators/deimos/schema_class/templates/schema_record.rb.tt +0 -102
- data/lib/generators/deimos/schema_class_generator.rb +0 -369
- data/spec/generators/schema_class/my_schema_spec.rb +0 -16
- data/spec/generators/schema_class/my_schema_with_circular_reference_spec.rb +0 -98
- data/spec/generators/schema_class/my_schema_with_complex_types_spec.rb +0 -237
- data/spec/generators/schema_class_generator_spec.rb +0 -283
- data/spec/snapshots/consumers-no-nest.snap +0 -1740
- data/spec/snapshots/consumers.snap +0 -1720
- data/spec/snapshots/consumers_and_producers-no-nest.snap +0 -1740
- data/spec/snapshots/consumers_and_producers.snap +0 -1720
- data/spec/snapshots/consumers_circular-no-nest.snap +0 -1740
- data/spec/snapshots/consumers_circular.snap +0 -1720
- data/spec/snapshots/consumers_complex_types-no-nest.snap +0 -1740
- data/spec/snapshots/consumers_complex_types.snap +0 -1720
- data/spec/snapshots/consumers_nested-no-nest.snap +0 -1740
- data/spec/snapshots/consumers_nested.snap +0 -1720
- data/spec/snapshots/namespace_folders.snap +0 -1800
- data/spec/snapshots/namespace_map.snap +0 -1800
- data/spec/snapshots/producers_with_key-no-nest.snap +0 -1740
- data/spec/snapshots/producers_with_key.snap +0 -1720
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: ba96f4ae17cd71ddcda0499103bfaedc88d2757e5e6eb6034b3cffb9e9898034
|
|
4
|
+
data.tar.gz: f707d6ae52803d9e2cdca87b03c409ba5f47bc1ceaad243fb80b8bd91a2b4dff
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 6151699039457db7eecc81f33bdae4ab921f68220ba75db0b6ce935555a112903b280f942c35cd459bb298128cc4e3f4240f26c6538bbe94b0c0924ad0bd8958
|
|
7
|
+
data.tar.gz: d1882baeffc9e69998fd704559b7c21f83e966e365c4e96b60f073412b40c2b242a8749e0e9890379cdfc88d96e2687dcf5d9293015f3e8ee15608c78c5bc3fb
|
data/CHANGELOG.md
CHANGED
|
@@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
|
|
8
8
|
## Unreleased
|
|
9
9
|
|
|
10
|
+
- Major change: Extract Avro schema class generation out of Deimos and into `avro-gen-ruby`.
|
|
11
|
+
|
|
10
12
|
## 2.5.3 - 2026-04-28
|
|
11
13
|
|
|
12
14
|
- Fix: `setup_karafka` applies the merged kafka config to `Karafka.producer`, so kafka overrides set in later `Karafka::App.setup` calls take effect on `Karafka.producer` callers (Karafka::Web, DLQ, ActiveJob).
|
data/README.md
CHANGED
|
@@ -840,19 +840,23 @@ DB backend only when your rake task is running.
|
|
|
840
840
|
|
|
841
841
|
# Generated Schema Classes
|
|
842
842
|
|
|
843
|
-
Deimos
|
|
844
|
-
|
|
843
|
+
Deimos generates classes from Avro schemas using the
|
|
844
|
+
[avro-gen-ruby](https://github.com/flipp-oss/avro-gen-ruby) gem (namespace `AvroGen`).
|
|
845
|
+
These classes are documented with YARD to aid in IDE auto-complete, and will help to move
|
|
846
|
+
errors closer to the code.
|
|
845
847
|
|
|
846
|
-
Add the following configurations for schema class generation:
|
|
848
|
+
Add the following configurations for schema class generation:
|
|
847
849
|
|
|
848
850
|
```ruby
|
|
849
|
-
config.
|
|
851
|
+
config.avrogen.generated_class_path 'path/to/generated/classes' # Defaults to 'app/lib/schema_classes'
|
|
850
852
|
```
|
|
851
853
|
|
|
852
854
|
Run the following command to generate schema classes in your application. It will generate classes for every configured consumer or producer by `Deimos.configure`:
|
|
853
855
|
|
|
854
856
|
bundle exec rake deimos:generate_schema_classes
|
|
855
857
|
|
|
858
|
+
(The standalone `bundle exec rake avro:generate` task generates classes for every schema on disk, without the Kafka-aware key/tombstone handling.)
|
|
859
|
+
|
|
856
860
|
Add the following configurations to start using generated schema classes in your application's Consumers and Producers:
|
|
857
861
|
|
|
858
862
|
config.schema.use_schema_classes true
|
|
@@ -864,7 +868,14 @@ Note that if you have a schema in your repo but have not configured a producer o
|
|
|
864
868
|
|
|
865
869
|
One additional configuration option indicates whether nested records should be generated as top-level classes or would remain nested inside the generated class for its parent schema. The default is to nest them, as a flattened structure can have one sub-schema clobber another sub-schema defined in a different top-level schema.
|
|
866
870
|
|
|
867
|
-
config.
|
|
871
|
+
config.avrogen.nest_child_schemas = false # Flatten all classes into one directory
|
|
872
|
+
|
|
873
|
+
> **Note:** The schema-class generation settings moved from `config.schema.*` to
|
|
874
|
+
> `config.avrogen.*`. The old `config.schema.generated_class_path` /
|
|
875
|
+
> `nest_child_schemas` / `use_full_namespace` / `schema_namespace_map` settings still
|
|
876
|
+
> work but are deprecated. Generated classes now inherit from
|
|
877
|
+
> `AvroGen::SchemaClass::Record`/`Enum`; previously-generated files referencing
|
|
878
|
+
> `Deimos::SchemaClass::*` still load, and `bundle exec rake avro:upgrade` rewrites them.
|
|
868
879
|
|
|
869
880
|
You can generate a tombstone message (with only a key and no value) by calling the `YourSchemaClass.tombstone(key)` method. If you're using a `:field` key config, you can pass in just the key scalar value. If using a key schema, you can pass it in as a hash or as another schema class.
|
|
870
881
|
|
|
@@ -877,7 +888,7 @@ Examples of consumers would look like this:
|
|
|
877
888
|
```ruby
|
|
878
889
|
class MyConsumer < Deimos::Consumer
|
|
879
890
|
def consume_message(message)
|
|
880
|
-
# Same method as before but message.payload is now an instance of
|
|
891
|
+
# Same method as before but message.payload is now an instance of AvroGen::SchemaClass::Record
|
|
881
892
|
# rather than a hash.
|
|
882
893
|
# You can interact with the schema class instance in the following way:
|
|
883
894
|
do_something(message.payload.test_id, message.payload.some_int)
|
|
@@ -891,7 +902,7 @@ end
|
|
|
891
902
|
class MyActiveRecordConsumer < Deimos::ActiveRecordConsumer
|
|
892
903
|
record_class Widget
|
|
893
904
|
# Any method that expects a message payload as a hash will instead
|
|
894
|
-
# receive an instance of
|
|
905
|
+
# receive an instance of AvroGen::SchemaClass::Record.
|
|
895
906
|
def record_attributes(payload, key)
|
|
896
907
|
# You can interact with the schema class instance in the following way:
|
|
897
908
|
super.merge(:some_field => "some_value-#{payload.test_id}")
|
|
@@ -931,9 +942,9 @@ class MyActiveRecordProducer < Deimos::ActiveRecordProducer
|
|
|
931
942
|
record_class Widget
|
|
932
943
|
# @param attributes [Hash]
|
|
933
944
|
# @param _record [Widget]
|
|
934
|
-
# @return [
|
|
945
|
+
# @return [AvroGen::SchemaClass::Record]
|
|
935
946
|
def self.generate_payload(attributes, _record)
|
|
936
|
-
# This method converts your ActiveRecord into a
|
|
947
|
+
# This method converts your ActiveRecord into a AvroGen::SchemaClass::Record. You will be able to use super
|
|
937
948
|
# as an instance of Schemas::MySchema and set values that are not on your ActiveRecord schema.
|
|
938
949
|
res = super
|
|
939
950
|
res.some_value = "some_value-#{res.test_id}"
|
data/deimos-ruby.gemspec
CHANGED
|
@@ -17,6 +17,7 @@ Gem::Specification.new do |spec|
|
|
|
17
17
|
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
|
18
18
|
spec.require_paths = ['lib']
|
|
19
19
|
|
|
20
|
+
spec.add_dependency('avro-gen-ruby')
|
|
20
21
|
spec.add_dependency('benchmark', '~> 0.5')
|
|
21
22
|
spec.add_dependency('fig_tree', '~> 0.2.0')
|
|
22
23
|
spec.add_dependency('karafka', '~> 2.0')
|
data/docs/CONFIGURATION.md
CHANGED
|
@@ -60,10 +60,17 @@ things you need to reference into local variables before calling `configure`.
|
|
|
60
60
|
| schema.password | nil | Basic auth password. |
|
|
61
61
|
| schema.path | nil | Local path to find your schemas. |
|
|
62
62
|
| schema.use_schema_classes | false | Set this to true to use generated schema classes in your application. |
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
63
|
+
|
|
64
|
+
#### Schema Class Generation (`avrogen`)
|
|
65
|
+
|
|
66
|
+
Schema class generation is provided by the [avro-gen-ruby](https://github.com/flipp-oss/avro-gen-ruby) gem (namespace `AvroGen`). These settings are forwarded to `AvroGen.config`. The equivalent `schema.*` settings still work but are deprecated — using one prints a warning pointing to the `avrogen.*` setting, and you can migrate generated files with `rake avro:upgrade`.
|
|
67
|
+
|
|
68
|
+
| Config name | Default | Description |
|
|
69
|
+
|-------------------------------|--------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------|
|
|
70
|
+
| avrogen.generated_class_path | `app/lib/schema_classes` | Local path to generated schema classes. (was `schema.generated_class_path`) |
|
|
71
|
+
| avrogen.nest_child_schemas | true | Set to true to nest subschemas within the generated class for the parent schema. (was `schema.nest_child_schemas`) |
|
|
72
|
+
| avrogen.use_full_namespace | false | Set to true to generate folders for schemas matching the full namespace. (was `schema.use_full_namespace`) |
|
|
73
|
+
| avrogen.schema_namespace_map | {} | A map of namespace prefixes to base module name(s). Example: { 'com.mycompany.suborg' => ['SchemaClasses'] }. Requires `use_full_namespace` to be true. (was `schema.schema_namespace_map`) |
|
|
67
74
|
|
|
68
75
|
### Outbox Configuration
|
|
69
76
|
|
|
@@ -76,7 +76,7 @@ module Deimos
|
|
|
76
76
|
def record_key(key)
|
|
77
77
|
if key.nil?
|
|
78
78
|
{}
|
|
79
|
-
elsif key.is_a?(Hash) || key.is_a?(SchemaClass::Record)
|
|
79
|
+
elsif key.is_a?(Hash) || key.is_a?(AvroGen::SchemaClass::Record)
|
|
80
80
|
self.key_converter.convert(key)
|
|
81
81
|
elsif self.topic.key_config[:field].nil?
|
|
82
82
|
{ @klass.primary_key => key }
|
|
@@ -13,7 +13,7 @@ module Deimos
|
|
|
13
13
|
# Default is to use the primary key column and the value of the first
|
|
14
14
|
# field in the key.
|
|
15
15
|
# @param klass [Class<ActiveRecord::Base>]
|
|
16
|
-
# @param _payload [Hash,
|
|
16
|
+
# @param _payload [Hash,AvroGen::SchemaClass::Record]
|
|
17
17
|
# @param key [Object]
|
|
18
18
|
# @return [ActiveRecord::Base]
|
|
19
19
|
def fetch_record(klass, _payload, key)
|
|
@@ -23,7 +23,7 @@ module Deimos
|
|
|
23
23
|
|
|
24
24
|
# Assign a key to a new record.
|
|
25
25
|
# @param record [ActiveRecord::Base]
|
|
26
|
-
# @param _payload [Hash,
|
|
26
|
+
# @param _payload [Hash,AvroGen::SchemaClass::Record]
|
|
27
27
|
# @param key [Object]
|
|
28
28
|
# @return [void]
|
|
29
29
|
def assign_key(record, _payload, key)
|
|
@@ -90,7 +90,7 @@ module Deimos
|
|
|
90
90
|
|
|
91
91
|
# Override this method (with `super`) if you want to add/change the default
|
|
92
92
|
# attributes set to the new/existing record.
|
|
93
|
-
# @param payload [Hash,
|
|
93
|
+
# @param payload [Hash,AvroGen::SchemaClass::Record]
|
|
94
94
|
# @param _key [String]
|
|
95
95
|
# @return [Hash]
|
|
96
96
|
def record_attributes(payload, _key=nil)
|
|
@@ -81,7 +81,7 @@ module Deimos
|
|
|
81
81
|
return payload if self.karafka_config.use_schema_classes.nil? &&
|
|
82
82
|
!Deimos.config.schema.use_schema_classes
|
|
83
83
|
|
|
84
|
-
|
|
84
|
+
AvroGen::SchemaClass.instance(payload, encoder.schema, encoder.namespace)
|
|
85
85
|
end
|
|
86
86
|
|
|
87
87
|
# Deletion payload for a record by default, delegate to the
|
|
@@ -15,6 +15,7 @@ module Deimos
|
|
|
15
15
|
config.deserializers[:payload].try(:reset_backend)
|
|
16
16
|
config.deserializers[:key].try(:reset_backend)
|
|
17
17
|
end
|
|
18
|
+
Deimos::SchemaClass.sync_config!
|
|
18
19
|
if self.config.schema.use_schema_classes
|
|
19
20
|
load_generated_schema_classes
|
|
20
21
|
end
|
|
@@ -42,16 +43,16 @@ module Deimos
|
|
|
42
43
|
# Loads generated classes
|
|
43
44
|
# @return [void]
|
|
44
45
|
def load_generated_schema_classes
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
46
|
+
path = AvroGen.config.generated_class_path
|
|
47
|
+
if path.nil?
|
|
48
|
+
raise 'Cannot use schema classes without a generated class path. ' \
|
|
49
|
+
'Please set AvroGen.config.generated_class_path.'
|
|
48
50
|
end
|
|
49
51
|
|
|
50
|
-
Dir["./#{
|
|
52
|
+
Dir["./#{path}/**/*.rb"].
|
|
51
53
|
each { |f| require f }
|
|
52
54
|
rescue LoadError
|
|
53
|
-
raise 'Cannot load schema classes. Please regenerate classes with'
|
|
54
|
-
'rake deimos:generate_schema_models.'
|
|
55
|
+
raise 'Cannot load schema classes. Please regenerate classes with rake avro:generate.'
|
|
55
56
|
end
|
|
56
57
|
|
|
57
58
|
# Ensure everything is set up correctly for the DB backend.
|
|
@@ -173,7 +174,7 @@ module Deimos
|
|
|
173
174
|
# @return [String]
|
|
174
175
|
setting :path
|
|
175
176
|
|
|
176
|
-
#
|
|
177
|
+
# @deprecated Use config.avrogen.generated_class_path instead.
|
|
177
178
|
# @return [String]
|
|
178
179
|
setting :generated_class_path, 'app/lib/schema_classes'
|
|
179
180
|
|
|
@@ -181,6 +182,31 @@ module Deimos
|
|
|
181
182
|
# @return [Boolean]
|
|
182
183
|
setting :use_schema_classes
|
|
183
184
|
|
|
185
|
+
# @deprecated Use config.avrogen.nest_child_schemas instead.
|
|
186
|
+
# @return [Boolean]
|
|
187
|
+
setting :nest_child_schemas, true
|
|
188
|
+
|
|
189
|
+
# @deprecated Use config.avrogen.use_full_namespace instead.
|
|
190
|
+
# @return [Boolean]
|
|
191
|
+
setting :use_full_namespace, false
|
|
192
|
+
|
|
193
|
+
# @deprecated Use config.avrogen.schema_namespace_map instead.
|
|
194
|
+
# @return [Hash]
|
|
195
|
+
setting :schema_namespace_map, {}
|
|
196
|
+
|
|
197
|
+
# The base directory for generated protobuf key schemas.
|
|
198
|
+
setting :proto_schema_key_path, 'protos'
|
|
199
|
+
end
|
|
200
|
+
|
|
201
|
+
# Schema class generation settings, forwarded to the avro-gen-ruby gem
|
|
202
|
+
# (AvroGen.config). These previously lived under `schema` (still supported,
|
|
203
|
+
# but deprecated).
|
|
204
|
+
setting :avrogen do
|
|
205
|
+
|
|
206
|
+
# Local path for schema classes to be generated in.
|
|
207
|
+
# @return [String]
|
|
208
|
+
setting :generated_class_path, 'app/lib/schema_classes'
|
|
209
|
+
|
|
184
210
|
# Set to false to generate child schemas as their own files.
|
|
185
211
|
# @return [Boolean]
|
|
186
212
|
setting :nest_child_schemas, true
|
|
@@ -191,12 +217,9 @@ module Deimos
|
|
|
191
217
|
|
|
192
218
|
# Use this option to reduce nesting when using use_full_namespace.
|
|
193
219
|
# For example: { 'com.mycompany.suborg' => 'SchemaClasses' }
|
|
194
|
-
# would replace a
|
|
220
|
+
# would replace a prefix matching the given key with the module name SchemaClasses.
|
|
195
221
|
# @return [Hash]
|
|
196
222
|
setting :schema_namespace_map, {}
|
|
197
|
-
|
|
198
|
-
# The base directory for generated protobuf key schemas.
|
|
199
|
-
setting :proto_schema_key_path, 'protos'
|
|
200
223
|
end
|
|
201
224
|
|
|
202
225
|
# The configured metrics provider.
|
|
@@ -5,7 +5,7 @@ module Deimos
|
|
|
5
5
|
class << self
|
|
6
6
|
|
|
7
7
|
def allowed_classes
|
|
8
|
-
arr = [Hash, SchemaClass::Record]
|
|
8
|
+
arr = [Hash, AvroGen::SchemaClass::Record]
|
|
9
9
|
if defined?(Google::Protobuf)
|
|
10
10
|
arr.push(Google::Protobuf.const_get(:AbstractMessage))
|
|
11
11
|
end
|
|
@@ -26,7 +26,7 @@ module Deimos
|
|
|
26
26
|
self.allowed_classes.none? { |k| message[:payload].is_a?(k) }
|
|
27
27
|
|
|
28
28
|
payload = message[:payload]
|
|
29
|
-
payload = payload.to_h if payload.nil? || payload.is_a?(SchemaClass::Record)
|
|
29
|
+
payload = payload.to_h if payload.nil? || payload.is_a?(AvroGen::SchemaClass::Record)
|
|
30
30
|
m = Deimos::Message.new(payload,
|
|
31
31
|
headers: message[:headers],
|
|
32
32
|
partition_key: message[:partition_key])
|
data/lib/deimos/logging.rb
CHANGED
|
@@ -30,6 +30,15 @@ module Deimos
|
|
|
30
30
|
log_add(:warn, *args)
|
|
31
31
|
end
|
|
32
32
|
|
|
33
|
+
# Emit a deprecation warning at most once per unique message.
|
|
34
|
+
# @param msg [String]
|
|
35
|
+
def deprecate(msg)
|
|
36
|
+
@deprecations ||= Set.new
|
|
37
|
+
return unless @deprecations.add?(msg)
|
|
38
|
+
|
|
39
|
+
warn("DEPRECATION WARNING: #{msg}")
|
|
40
|
+
end
|
|
41
|
+
|
|
33
42
|
def metadata_log_text(metadata)
|
|
34
43
|
metadata.to_h.slice(:timestamp, :offset, :first_offset, :last_offset, :partition, :topic, :size)
|
|
35
44
|
end
|
data/lib/deimos/producer.rb
CHANGED
|
@@ -75,7 +75,7 @@ module Deimos
|
|
|
75
75
|
end
|
|
76
76
|
|
|
77
77
|
# Publish the payload to the topic.
|
|
78
|
-
# @param payload [Hash, SchemaClass::Record] with an optional payload_key hash key.
|
|
78
|
+
# @param payload [Hash, AvroGen::SchemaClass::Record] with an optional payload_key hash key.
|
|
79
79
|
# @param topic [String] if specifying the topic
|
|
80
80
|
# @param headers [Hash] if specifying headers
|
|
81
81
|
# @return [void]
|
|
@@ -99,7 +99,7 @@ module Deimos
|
|
|
99
99
|
end
|
|
100
100
|
|
|
101
101
|
# Publish a list of messages.
|
|
102
|
-
# @param payloads [Array<Hash, SchemaClass::Record>] with optional payload_key hash key.
|
|
102
|
+
# @param payloads [Array<Hash, AvroGen::SchemaClass::Record>] with optional payload_key hash key.
|
|
103
103
|
# @param sync [Boolean] if given, override the default setting of
|
|
104
104
|
# whether to publish synchronously.
|
|
105
105
|
# @param force_send [Boolean] if true, ignore the configured backend
|
|
@@ -112,7 +112,7 @@ module Deimos
|
|
|
112
112
|
|
|
113
113
|
messages = Array(payloads).map do |p|
|
|
114
114
|
payload = p
|
|
115
|
-
payload = payload.to_h if p.is_a?(SchemaClass::Record)
|
|
115
|
+
payload = payload.to_h if p.is_a?(AvroGen::SchemaClass::Record)
|
|
116
116
|
m = {
|
|
117
117
|
payload: payload,
|
|
118
118
|
headers: headers,
|
|
@@ -121,10 +121,10 @@ module Deimos
|
|
|
121
121
|
}
|
|
122
122
|
if payload.is_a?(Hash) && payload.key?(:key) && payload.key?(:message)
|
|
123
123
|
m[:key] = payload[:key]
|
|
124
|
-
m[:key] = m[:key].to_h if m[:key].nil? || m[:key].is_a?(SchemaClass::Record)
|
|
124
|
+
m[:key] = m[:key].to_h if m[:key].nil? || m[:key].is_a?(AvroGen::SchemaClass::Record)
|
|
125
125
|
m[:payload] = payload[:message]
|
|
126
126
|
m[:payload] = m[:payload].to_h if m[:payload].nil? ||
|
|
127
|
-
m[:payload].is_a?(SchemaClass::Record)
|
|
127
|
+
m[:payload].is_a?(AvroGen::SchemaClass::Record)
|
|
128
128
|
end
|
|
129
129
|
m
|
|
130
130
|
end
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
require_relative 'base'
|
|
4
4
|
require 'schema_registry_client'
|
|
5
|
+
require 'avro_gen/avro_parser'
|
|
5
6
|
require_relative 'avro_schema_coercer'
|
|
6
7
|
|
|
7
8
|
module Deimos
|
|
@@ -106,55 +107,25 @@ module Deimos
|
|
|
106
107
|
|
|
107
108
|
# @param schema [Avro::Schema::NamedSchema] A named schema
|
|
108
109
|
# @return [String]
|
|
110
|
+
# @deprecated Use AvroGen::AvroParser.schema_classname instead.
|
|
109
111
|
def self.schema_classname(schema)
|
|
110
|
-
schema
|
|
112
|
+
AvroGen::AvroParser.schema_classname(schema)
|
|
111
113
|
end
|
|
112
114
|
|
|
113
115
|
# Converts Avro::Schema::NamedSchema's to String form for generated YARD docs.
|
|
114
|
-
# Recursively handles the typing for Arrays, Maps and Unions.
|
|
115
116
|
# @param avro_schema [Avro::Schema::NamedSchema]
|
|
116
117
|
# @return [String] A string representation of the Type of this SchemaField
|
|
118
|
+
# @deprecated Use AvroGen::AvroParser.field_type instead.
|
|
117
119
|
def self.field_type(avro_schema)
|
|
118
|
-
|
|
119
|
-
when :string, :boolean
|
|
120
|
-
avro_schema.type_sym.to_s.titleize
|
|
121
|
-
when :int, :long
|
|
122
|
-
'Integer'
|
|
123
|
-
when :float, :double
|
|
124
|
-
'Float'
|
|
125
|
-
when :record, :enum
|
|
126
|
-
schema_classname(avro_schema)
|
|
127
|
-
when :array
|
|
128
|
-
arr_t = field_type(Deimos::SchemaField.new('n/a', avro_schema.items).type)
|
|
129
|
-
"Array<#{arr_t}>"
|
|
130
|
-
when :map
|
|
131
|
-
map_t = field_type(Deimos::SchemaField.new('n/a', avro_schema.values).type)
|
|
132
|
-
"Hash<String, #{map_t}>"
|
|
133
|
-
when :union
|
|
134
|
-
types = avro_schema.schemas.map do |t|
|
|
135
|
-
field_type(Deimos::SchemaField.new('n/a', t).type)
|
|
136
|
-
end
|
|
137
|
-
types.join(', ')
|
|
138
|
-
when :null
|
|
139
|
-
'nil'
|
|
140
|
-
end
|
|
120
|
+
AvroGen::AvroParser.field_type(avro_schema)
|
|
141
121
|
end
|
|
142
122
|
|
|
143
123
|
# Returns the base type of this schema. Decodes Arrays, Maps and Unions
|
|
144
124
|
# @param schema [Avro::Schema::NamedSchema]
|
|
145
125
|
# @return [Avro::Schema::NamedSchema]
|
|
126
|
+
# @deprecated Use AvroGen::AvroParser.schema_base_class instead.
|
|
146
127
|
def self.schema_base_class(schema)
|
|
147
|
-
|
|
148
|
-
when :array
|
|
149
|
-
schema_base_class(schema.items)
|
|
150
|
-
when :map
|
|
151
|
-
schema_base_class(schema.values)
|
|
152
|
-
when :union
|
|
153
|
-
schema.schemas.map(&method(:schema_base_class)).
|
|
154
|
-
reject { |s| s.type_sym == :null }.first
|
|
155
|
-
else
|
|
156
|
-
schema
|
|
157
|
-
end
|
|
128
|
+
AvroGen::AvroParser.schema_base_class(schema)
|
|
158
129
|
end
|
|
159
130
|
|
|
160
131
|
def generate_key_schema(field_name)
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'avro_gen'
|
|
4
|
+
|
|
5
|
+
module Deimos
|
|
6
|
+
# Backwards-compatible shim. Schema class generation moved to the avro-gen-ruby
|
|
7
|
+
# gem (namespace AvroGen). Previously-generated classes that reference
|
|
8
|
+
# `Deimos::SchemaClass::Record` / `Enum` / `Base` still resolve here, with a
|
|
9
|
+
# one-time deprecation warning, via const_missing.
|
|
10
|
+
module SchemaClass
|
|
11
|
+
DEPRECATED_CONSTANTS = {
|
|
12
|
+
Base: AvroGen::SchemaClass::Base,
|
|
13
|
+
Record: AvroGen::SchemaClass::Record,
|
|
14
|
+
Enum: AvroGen::SchemaClass::Enum
|
|
15
|
+
}.freeze
|
|
16
|
+
|
|
17
|
+
# The generation settings that live under Deimos.config.avrogen and are
|
|
18
|
+
# forwarded to AvroGen.config (same names on both sides).
|
|
19
|
+
GENERATION_SETTINGS = %i(
|
|
20
|
+
generated_class_path
|
|
21
|
+
nest_child_schemas
|
|
22
|
+
use_full_namespace
|
|
23
|
+
schema_namespace_map
|
|
24
|
+
).freeze
|
|
25
|
+
|
|
26
|
+
# @param name [Symbol]
|
|
27
|
+
def self.const_missing(name)
|
|
28
|
+
klass = DEPRECATED_CONSTANTS[name]
|
|
29
|
+
return super unless klass
|
|
30
|
+
|
|
31
|
+
Deimos::Logging.deprecate(
|
|
32
|
+
"Deimos::SchemaClass::#{name} is deprecated; use AvroGen::SchemaClass::#{name} instead. " \
|
|
33
|
+
'Run `rake avro:upgrade` to update generated classes.'
|
|
34
|
+
)
|
|
35
|
+
# Define the constant so the warning is only emitted once.
|
|
36
|
+
const_set(name, klass)
|
|
37
|
+
klass
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# Mirror the Deimos generation settings onto AvroGen.config. Deimos.config is
|
|
41
|
+
# the source of truth within Deimos, so this runs on every configure (keeping
|
|
42
|
+
# a reset in sync). The settings live under Deimos.config.avrogen; the legacy
|
|
43
|
+
# Deimos.config.schema.* equivalents still work but emit a deprecation warning.
|
|
44
|
+
# Standalone AvroGen users never trigger this and set AvroGen.config directly.
|
|
45
|
+
# @!visibility private
|
|
46
|
+
def self.sync_config!
|
|
47
|
+
# schema.path is shared with the Avro backends, so it stays under `schema`.
|
|
48
|
+
AvroGen.config.schema_path = Deimos.config.schema.path
|
|
49
|
+
# Refresh AvroGen's cached schema stores on (re)configuration so they don't
|
|
50
|
+
# serve stale schemas after the schema path or files change.
|
|
51
|
+
AvroGen::SchemaValidator.clear_store_cache!
|
|
52
|
+
|
|
53
|
+
GENERATION_SETTINGS.each do |key|
|
|
54
|
+
AvroGen.config.send("#{key}=", generation_setting(key))
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# Resolve a generation setting, preferring the legacy (deprecated)
|
|
59
|
+
# Deimos.config.schema.* location when it was explicitly set.
|
|
60
|
+
# @!visibility private
|
|
61
|
+
def self.generation_setting(key)
|
|
62
|
+
if Deimos.config.schema.default_value?(key)
|
|
63
|
+
Deimos.config.avrogen.send(key)
|
|
64
|
+
else
|
|
65
|
+
Deimos::Logging.deprecate(
|
|
66
|
+
"Deimos.config.schema.#{key} is deprecated; use Deimos.config.avrogen.#{key} instead."
|
|
67
|
+
)
|
|
68
|
+
Deimos.config.schema.send(key)
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
end
|
data/lib/deimos/transcoder.rb
CHANGED
|
@@ -58,9 +58,9 @@ module Deimos
|
|
|
58
58
|
|
|
59
59
|
schema_key = decoded_key.is_a?(Hash) ? decoded_key : { self.key_field => decoded_key }
|
|
60
60
|
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
61
|
+
AvroGen::SchemaClass.instance(schema_key,
|
|
62
|
+
"#{@schema}_key",
|
|
63
|
+
@namespace)
|
|
64
64
|
end
|
|
65
65
|
|
|
66
66
|
# @param payload [String]
|
|
@@ -71,9 +71,9 @@ module Deimos
|
|
|
71
71
|
decoded_payload = self.backend.decode(payload)
|
|
72
72
|
return decoded_payload if !@use_schema_classes || !decoded_payload.is_a?(Hash)
|
|
73
73
|
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
74
|
+
AvroGen::SchemaClass.instance(decoded_payload,
|
|
75
|
+
@schema,
|
|
76
|
+
@namespace)
|
|
77
77
|
end
|
|
78
78
|
|
|
79
79
|
# @param payload [Object]
|
|
@@ -1,60 +1,27 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require 'avro_gen'
|
|
4
|
+
|
|
3
5
|
module Deimos
|
|
4
6
|
module Utils
|
|
5
|
-
#
|
|
7
|
+
# Backwards-compatible shim. The schema class helpers moved to
|
|
8
|
+
# AvroGen::SchemaClass. This delegates with a deprecation warning.
|
|
6
9
|
module SchemaClass
|
|
7
10
|
class << self
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
if Deimos.config.schema.use_full_namespace
|
|
17
|
-
if Deimos.config.schema.schema_namespace_map.present?
|
|
18
|
-
namespace_keys = Deimos.config.schema.schema_namespace_map.keys.sort_by { |k| -k.length }
|
|
19
|
-
namespace_override = namespace_keys.find { |k| module_namespace.include?(k) }
|
|
20
|
-
end
|
|
21
|
-
|
|
22
|
-
if namespace_override.present?
|
|
23
|
-
# override default module
|
|
24
|
-
modules = Array(Deimos.config.schema.schema_namespace_map[namespace_override])
|
|
25
|
-
module_namespace = module_namespace.gsub(/#{namespace_override}\.?/, '')
|
|
26
|
-
end
|
|
27
|
-
|
|
28
|
-
namespace_folders = module_namespace.split('.').map { |f| f.underscore.camelize }
|
|
29
|
-
modules.concat(namespace_folders) if namespace_folders.any?
|
|
11
|
+
def method_missing(name, ...)
|
|
12
|
+
if AvroGen::SchemaClass.respond_to?(name)
|
|
13
|
+
Deimos::Logging.deprecate(
|
|
14
|
+
"Deimos::Utils::SchemaClass.#{name} is deprecated; use AvroGen::SchemaClass.#{name} instead."
|
|
15
|
+
)
|
|
16
|
+
AvroGen::SchemaClass.send(name, ...)
|
|
17
|
+
else
|
|
18
|
+
super
|
|
30
19
|
end
|
|
31
|
-
|
|
32
|
-
modules
|
|
33
20
|
end
|
|
34
21
|
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
# @param schema [String]
|
|
38
|
-
# @param namespace [String]
|
|
39
|
-
# @return [Deimos::SchemaClass::Record]
|
|
40
|
-
def instance(payload, schema, namespace='')
|
|
41
|
-
return payload if payload.is_a?(Deimos::SchemaClass::Base)
|
|
42
|
-
|
|
43
|
-
klass = klass(schema, namespace)
|
|
44
|
-
return payload if klass.nil? || payload.nil?
|
|
45
|
-
|
|
46
|
-
klass.new_from_message(**payload.symbolize_keys)
|
|
22
|
+
def respond_to_missing?(name, include_private=false)
|
|
23
|
+
AvroGen::SchemaClass.respond_to?(name, include_private) || super
|
|
47
24
|
end
|
|
48
|
-
|
|
49
|
-
# Determine and return the SchemaClass with the provided schema and namespace
|
|
50
|
-
# @param schema [String]
|
|
51
|
-
# @param namespace [String]
|
|
52
|
-
# @return [Deimos::SchemaClass]
|
|
53
|
-
def klass(schema, namespace)
|
|
54
|
-
constants = modules_for(namespace) + [schema.underscore.camelize.singularize]
|
|
55
|
-
constants.join('::').safe_constantize
|
|
56
|
-
end
|
|
57
|
-
|
|
58
25
|
end
|
|
59
26
|
end
|
|
60
27
|
end
|
data/lib/deimos/version.rb
CHANGED
data/lib/deimos.rb
CHANGED
|
@@ -17,9 +17,9 @@ require 'deimos/backends/kafka_async'
|
|
|
17
17
|
require 'deimos/backends/test'
|
|
18
18
|
|
|
19
19
|
require 'deimos/schema_backends/base'
|
|
20
|
+
require 'avro_gen'
|
|
21
|
+
require 'deimos/schema_class'
|
|
20
22
|
require 'deimos/utils/schema_class'
|
|
21
|
-
require 'deimos/schema_class/enum'
|
|
22
|
-
require 'deimos/schema_class/record'
|
|
23
23
|
|
|
24
24
|
require 'deimos/ext/schema_route'
|
|
25
25
|
require 'deimos/ext/consumer_route'
|
|
@@ -102,7 +102,7 @@ module Deimos
|
|
|
102
102
|
# Initialize an instance of the provided schema
|
|
103
103
|
# in the event the schema class is an override, the inherited
|
|
104
104
|
# schema and namespace will be applied
|
|
105
|
-
schema_class =
|
|
105
|
+
schema_class = AvroGen::SchemaClass.klass(schema, namespace)
|
|
106
106
|
if schema_class.nil?
|
|
107
107
|
schema_backend_class(backend: backend).
|
|
108
108
|
new(
|
data/lib/tasks/deimos.rake
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require '
|
|
3
|
+
require 'avro_gen/generator'
|
|
4
4
|
require 'optparse'
|
|
5
5
|
require 'deimos/schema_backends/proto_schema_registry'
|
|
6
6
|
|
|
@@ -32,10 +32,22 @@ namespace :deimos do
|
|
|
32
32
|
Deimos::Utils::DbPoller.start!
|
|
33
33
|
end
|
|
34
34
|
|
|
35
|
-
desc 'Run Schema Model Generator'
|
|
35
|
+
desc 'Run Schema Model Generator (deprecated: use rake avro:generate)'
|
|
36
36
|
task generate_schema_classes: :environment do
|
|
37
|
-
|
|
38
|
-
|
|
37
|
+
Deimos::Logging.deprecate(
|
|
38
|
+
'rake deimos:generate_schema_classes is deprecated; use rake avro:generate instead.'
|
|
39
|
+
)
|
|
40
|
+
Rails.logger.info('Running deimos:generate_schema_classes')
|
|
41
|
+
# Derive AvroGen configs from the configured Kafka topics so keyed records
|
|
42
|
+
# still get their tombstone/payload_key helpers, then delegate to AvroGen.
|
|
43
|
+
configs = Deimos.karafka_configs.filter_map do |config|
|
|
44
|
+
next if config.schema.nil?
|
|
45
|
+
|
|
46
|
+
{ schema: config.schema,
|
|
47
|
+
namespace: config.namespace,
|
|
48
|
+
key_config: config.key_config }
|
|
49
|
+
end
|
|
50
|
+
AvroGen::Generator.new.generate_from_configs(configs)
|
|
39
51
|
end
|
|
40
52
|
|
|
41
53
|
desc 'Output Protobuf key schemas'
|