deimos-ruby 2.5.3 → 2.6.0.pre.beta1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +2 -0
  3. data/README.md +20 -9
  4. data/deimos-ruby.gemspec +1 -0
  5. data/docs/CONFIGURATION.md +11 -4
  6. data/lib/deimos/active_record_consume/batch_consumption.rb +1 -1
  7. data/lib/deimos/active_record_consume/message_consumption.rb +2 -2
  8. data/lib/deimos/active_record_consumer.rb +1 -1
  9. data/lib/deimos/active_record_producer.rb +1 -1
  10. data/lib/deimos/config/configuration.rb +34 -11
  11. data/lib/deimos/ext/producer_middleware.rb +2 -2
  12. data/lib/deimos/logging.rb +9 -0
  13. data/lib/deimos/producer.rb +5 -5
  14. data/lib/deimos/schema_backends/avro_base.rb +7 -36
  15. data/lib/deimos/schema_class.rb +72 -0
  16. data/lib/deimos/transcoder.rb +6 -6
  17. data/lib/deimos/utils/schema_class.rb +14 -47
  18. data/lib/deimos/version.rb +1 -1
  19. data/lib/deimos.rb +3 -3
  20. data/lib/tasks/deimos.rake +16 -4
  21. data/regenerate_test_schema_classes.rb +12 -3
  22. data/spec/active_record_batch_consumer_association_spec.rb +1 -1
  23. data/spec/active_record_batch_consumer_spec.rb +1 -1
  24. data/spec/active_record_consume/batch_consumption_spec.rb +1 -1
  25. data/spec/active_record_consumer_spec.rb +3 -3
  26. data/spec/active_record_producer_spec.rb +1 -1
  27. data/spec/batch_consumer_spec.rb +2 -2
  28. data/spec/consumer_spec.rb +9 -9
  29. data/spec/schema_class_spec.rb +65 -0
  30. data/spec/schemas/my_namespace/generated.rb +4 -4
  31. data/spec/schemas/my_namespace/my_long_namespace_schema.rb +2 -2
  32. data/spec/schemas/my_namespace/my_nested_schema.rb +3 -3
  33. data/spec/schemas/my_namespace/my_schema.rb +2 -2
  34. data/spec/schemas/my_namespace/my_schema_compound_key.rb +2 -2
  35. data/spec/schemas/my_namespace/my_schema_id_key.rb +2 -2
  36. data/spec/schemas/my_namespace/my_schema_key.rb +2 -2
  37. data/spec/schemas/my_namespace/my_schema_with_boolean.rb +2 -2
  38. data/spec/schemas/my_namespace/my_schema_with_circular_reference.rb +3 -3
  39. data/spec/schemas/my_namespace/my_schema_with_complex_type.rb +6 -6
  40. data/spec/schemas/my_namespace/my_schema_with_date_time.rb +2 -2
  41. data/spec/schemas/my_namespace/my_schema_with_id.rb +2 -2
  42. data/spec/schemas/my_namespace/my_schema_with_title.rb +2 -2
  43. data/spec/schemas/my_namespace/my_schema_with_union_type.rb +6 -6
  44. data/spec/schemas/my_namespace/my_schema_with_unique_id.rb +2 -2
  45. data/spec/schemas/my_namespace/my_updated_schema.rb +1 -1
  46. data/spec/schemas/my_namespace/request/create_topic.rb +2 -2
  47. data/spec/schemas/my_namespace/request/index.rb +2 -2
  48. data/spec/schemas/my_namespace/request/update_request.rb +2 -2
  49. data/spec/schemas/my_namespace/response/create_topic.rb +2 -2
  50. data/spec/schemas/my_namespace/response/index.rb +2 -2
  51. data/spec/schemas/my_namespace/response/update_response.rb +2 -2
  52. data/spec/schemas/my_namespace/wibble.rb +2 -2
  53. data/spec/schemas/my_namespace/widget.rb +2 -2
  54. data/spec/schemas/my_namespace/widget_the_second.rb +2 -2
  55. data/spec/schemas/my_namespace/widget_the_third.rb +2 -2
  56. data/spec/spec_helper.rb +2 -2
  57. data/spec/utils/db_poller_spec.rb +1 -1
  58. metadata +18 -27
  59. data/lib/deimos/schema_class/base.rb +0 -62
  60. data/lib/deimos/schema_class/enum.rb +0 -49
  61. data/lib/deimos/schema_class/record.rb +0 -100
  62. data/lib/generators/deimos/schema_class/templates/schema_class.rb.tt +0 -8
  63. data/lib/generators/deimos/schema_class/templates/schema_enum.rb.tt +0 -13
  64. data/lib/generators/deimos/schema_class/templates/schema_record.rb.tt +0 -102
  65. data/lib/generators/deimos/schema_class_generator.rb +0 -369
  66. data/spec/generators/schema_class/my_schema_spec.rb +0 -16
  67. data/spec/generators/schema_class/my_schema_with_circular_reference_spec.rb +0 -98
  68. data/spec/generators/schema_class/my_schema_with_complex_types_spec.rb +0 -237
  69. data/spec/generators/schema_class_generator_spec.rb +0 -283
  70. data/spec/snapshots/consumers-no-nest.snap +0 -1740
  71. data/spec/snapshots/consumers.snap +0 -1720
  72. data/spec/snapshots/consumers_and_producers-no-nest.snap +0 -1740
  73. data/spec/snapshots/consumers_and_producers.snap +0 -1720
  74. data/spec/snapshots/consumers_circular-no-nest.snap +0 -1740
  75. data/spec/snapshots/consumers_circular.snap +0 -1720
  76. data/spec/snapshots/consumers_complex_types-no-nest.snap +0 -1740
  77. data/spec/snapshots/consumers_complex_types.snap +0 -1720
  78. data/spec/snapshots/consumers_nested-no-nest.snap +0 -1740
  79. data/spec/snapshots/consumers_nested.snap +0 -1720
  80. data/spec/snapshots/namespace_folders.snap +0 -1800
  81. data/spec/snapshots/namespace_map.snap +0 -1800
  82. data/spec/snapshots/producers_with_key-no-nest.snap +0 -1740
  83. data/spec/snapshots/producers_with_key.snap +0 -1720
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 634ebcbb254c467794eae6a3971200e2647472769eb59b6e8e89ed9453c3b203
4
- data.tar.gz: a72e4063b395f3a5388bc2988b3780e8fb9a8aa9052152f7c08b1f3469fc63f3
3
+ metadata.gz: ba96f4ae17cd71ddcda0499103bfaedc88d2757e5e6eb6034b3cffb9e9898034
4
+ data.tar.gz: f707d6ae52803d9e2cdca87b03c409ba5f47bc1ceaad243fb80b8bd91a2b4dff
5
5
  SHA512:
6
- metadata.gz: 1205344ccc7c9f97b5b0496e950d5a5e385654c0e1192ba8d6e1fa6f6c01fdd2a84c88f1980c587f2623717ca5eafd6562fc3fb713f3cd04b0d54d3382645385
7
- data.tar.gz: 15589b978f07ce93acfda561f8e8cb75f6a9ad1be25efd77942b5c04b9404237112c9668a1b6f295c94983848fcb18916b0c32a134daa208b8d7343ef845ee5b
6
+ metadata.gz: 6151699039457db7eecc81f33bdae4ab921f68220ba75db0b6ce935555a112903b280f942c35cd459bb298128cc4e3f4240f26c6538bbe94b0c0924ad0bd8958
7
+ data.tar.gz: d1882baeffc9e69998fd704559b7c21f83e966e365c4e96b60f073412b40c2b242a8749e0e9890379cdfc88d96e2687dcf5d9293015f3e8ee15608c78c5bc3fb
data/CHANGELOG.md CHANGED
@@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
7
7
 
8
8
  ## Unreleased
9
9
 
10
+ - Major change: Extract Avro schema class generation out of Deimos and into `avro-gen-ruby`.
11
+
10
12
  ## 2.5.3 - 2026-04-28
11
13
 
12
14
  - Fix: `setup_karafka` applies the merged kafka config to `Karafka.producer`, so kafka overrides set in later `Karafka::App.setup` calls take effect on `Karafka.producer` callers (Karafka::Web, DLQ, ActiveJob).
data/README.md CHANGED
@@ -840,19 +840,23 @@ DB backend only when your rake task is running.
840
840
 
841
841
  # Generated Schema Classes
842
842
 
843
- Deimos offers a way to generate classes from Avro schemas. These classes are documented
844
- with YARD to aid in IDE auto-complete, and will help to move errors closer to the code.
843
+ Deimos generates classes from Avro schemas using the
844
+ [avro-gen-ruby](https://github.com/flipp-oss/avro-gen-ruby) gem (namespace `AvroGen`).
845
+ These classes are documented with YARD to aid in IDE auto-complete, and will help to move
846
+ errors closer to the code.
845
847
 
846
- Add the following configurations for schema class generation:
848
+ Add the following configurations for schema class generation:
847
849
 
848
850
  ```ruby
849
- config.schema.generated_class_path 'path/to/generated/classes' # Defaults to 'app/lib/schema_classes'
851
+ config.avrogen.generated_class_path 'path/to/generated/classes' # Defaults to 'app/lib/schema_classes'
850
852
  ```
851
853
 
852
854
  Run the following command to generate schema classes in your application. It will generate classes for every configured consumer or producer by `Deimos.configure`:
853
855
 
854
856
  bundle exec rake deimos:generate_schema_classes
855
857
 
858
+ (The standalone `bundle exec rake avro:generate` task generates classes for every schema on disk, without the Kafka-aware key/tombstone handling.)
859
+
856
860
  Add the following configurations to start using generated schema classes in your application's Consumers and Producers:
857
861
 
858
862
  config.schema.use_schema_classes true
@@ -864,7 +868,14 @@ Note that if you have a schema in your repo but have not configured a producer o
864
868
 
865
869
  One additional configuration option indicates whether nested records should be generated as top-level classes or would remain nested inside the generated class for its parent schema. The default is to nest them, as a flattened structure can have one sub-schema clobber another sub-schema defined in a different top-level schema.
866
870
 
867
- config.schema.nest_child_schemas = false # Flatten all classes into one directory
871
+ config.avrogen.nest_child_schemas = false # Flatten all classes into one directory
872
+
873
+ > **Note:** The schema-class generation settings moved from `config.schema.*` to
874
+ > `config.avrogen.*`. The old `config.schema.generated_class_path` /
875
+ > `nest_child_schemas` / `use_full_namespace` / `schema_namespace_map` settings still
876
+ > work but are deprecated. Generated classes now inherit from
877
+ > `AvroGen::SchemaClass::Record`/`Enum`; previously-generated files referencing
878
+ > `Deimos::SchemaClass::*` still load, and `bundle exec rake avro:upgrade` rewrites them.
868
879
 
869
880
  You can generate a tombstone message (with only a key and no value) by calling the `YourSchemaClass.tombstone(key)` method. If you're using a `:field` key config, you can pass in just the key scalar value. If using a key schema, you can pass it in as a hash or as another schema class.
870
881
 
@@ -877,7 +888,7 @@ Examples of consumers would look like this:
877
888
  ```ruby
878
889
  class MyConsumer < Deimos::Consumer
879
890
  def consume_message(message)
880
- # Same method as before but message.payload is now an instance of Deimos::SchemaClass::Record
891
+ # Same method as before but message.payload is now an instance of AvroGen::SchemaClass::Record
881
892
  # rather than a hash.
882
893
  # You can interact with the schema class instance in the following way:
883
894
  do_something(message.payload.test_id, message.payload.some_int)
@@ -891,7 +902,7 @@ end
891
902
  class MyActiveRecordConsumer < Deimos::ActiveRecordConsumer
892
903
  record_class Widget
893
904
  # Any method that expects a message payload as a hash will instead
894
- # receive an instance of Deimos::SchemaClass::Record.
905
+ # receive an instance of AvroGen::SchemaClass::Record.
895
906
  def record_attributes(payload, key)
896
907
  # You can interact with the schema class instance in the following way:
897
908
  super.merge(:some_field => "some_value-#{payload.test_id}")
@@ -931,9 +942,9 @@ class MyActiveRecordProducer < Deimos::ActiveRecordProducer
931
942
  record_class Widget
932
943
  # @param attributes [Hash]
933
944
  # @param _record [Widget]
934
- # @return [Deimos::SchemaClass::Record]
945
+ # @return [AvroGen::SchemaClass::Record]
935
946
  def self.generate_payload(attributes, _record)
936
- # This method converts your ActiveRecord into a Deimos::SchemaClass::Record. You will be able to use super
947
+ # This method converts your ActiveRecord into a AvroGen::SchemaClass::Record. You will be able to use super
937
948
  # as an instance of Schemas::MySchema and set values that are not on your ActiveRecord schema.
938
949
  res = super
939
950
  res.some_value = "some_value-#{res.test_id}"
data/deimos-ruby.gemspec CHANGED
@@ -17,6 +17,7 @@ Gem::Specification.new do |spec|
17
17
  spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
18
18
  spec.require_paths = ['lib']
19
19
 
20
+ spec.add_dependency('avro-gen-ruby')
20
21
  spec.add_dependency('benchmark', '~> 0.5')
21
22
  spec.add_dependency('fig_tree', '~> 0.2.0')
22
23
  spec.add_dependency('karafka', '~> 2.0')
@@ -60,10 +60,17 @@ things you need to reference into local variables before calling `configure`.
60
60
  | schema.password | nil | Basic auth password. |
61
61
  | schema.path | nil | Local path to find your schemas. |
62
62
  | schema.use_schema_classes | false | Set this to true to use generated schema classes in your application. |
63
- | schema.generated_class_path | `app/lib/schema_classes` | Local path to generated schema classes. |
64
- | schema.nest_child_schemas | false | Set to true to nest subschemas within the generated class for the parent schema. |
65
- | schema.use_full_namespace | false | Set to true to generate folders for schemas matching the full namespace. |
66
- | schema.schema_namespace_map | {} | A map of namespace prefixes to base module name(s). Example: { 'com.mycompany.suborg' => ['SchemaClasses'] }. Requires `use_full_namespace` to be true. |
63
+
64
+ #### Schema Class Generation (`avrogen`)
65
+
66
+ Schema class generation is provided by the [avro-gen-ruby](https://github.com/flipp-oss/avro-gen-ruby) gem (namespace `AvroGen`). These settings are forwarded to `AvroGen.config`. The equivalent `schema.*` settings still work but are deprecated — using one prints a warning pointing to the `avrogen.*` setting, and you can migrate generated files with `rake avro:upgrade`.
67
+
68
+ | Config name | Default | Description |
69
+ |-------------------------------|--------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------|
70
+ | avrogen.generated_class_path | `app/lib/schema_classes` | Local path to generated schema classes. (was `schema.generated_class_path`) |
71
+ | avrogen.nest_child_schemas | true | Set to true to nest subschemas within the generated class for the parent schema. (was `schema.nest_child_schemas`) |
72
+ | avrogen.use_full_namespace | false | Set to true to generate folders for schemas matching the full namespace. (was `schema.use_full_namespace`) |
73
+ | avrogen.schema_namespace_map | {} | A map of namespace prefixes to base module name(s). Example: { 'com.mycompany.suborg' => ['SchemaClasses'] }. Requires `use_full_namespace` to be true. (was `schema.schema_namespace_map`) |
67
74
 
68
75
  ### Outbox Configuration
69
76
 
@@ -76,7 +76,7 @@ module Deimos
76
76
  def record_key(key)
77
77
  if key.nil?
78
78
  {}
79
- elsif key.is_a?(Hash) || key.is_a?(SchemaClass::Record)
79
+ elsif key.is_a?(Hash) || key.is_a?(AvroGen::SchemaClass::Record)
80
80
  self.key_converter.convert(key)
81
81
  elsif self.topic.key_config[:field].nil?
82
82
  { @klass.primary_key => key }
@@ -13,7 +13,7 @@ module Deimos
13
13
  # Default is to use the primary key column and the value of the first
14
14
  # field in the key.
15
15
  # @param klass [Class<ActiveRecord::Base>]
16
- # @param _payload [Hash,Deimos::SchemaClass::Record]
16
+ # @param _payload [Hash,AvroGen::SchemaClass::Record]
17
17
  # @param key [Object]
18
18
  # @return [ActiveRecord::Base]
19
19
  def fetch_record(klass, _payload, key)
@@ -23,7 +23,7 @@ module Deimos
23
23
 
24
24
  # Assign a key to a new record.
25
25
  # @param record [ActiveRecord::Base]
26
- # @param _payload [Hash,Deimos::SchemaClass::Record]
26
+ # @param _payload [Hash,AvroGen::SchemaClass::Record]
27
27
  # @param key [Object]
28
28
  # @return [void]
29
29
  def assign_key(record, _payload, key)
@@ -90,7 +90,7 @@ module Deimos
90
90
 
91
91
  # Override this method (with `super`) if you want to add/change the default
92
92
  # attributes set to the new/existing record.
93
- # @param payload [Hash,Deimos::SchemaClass::Record]
93
+ # @param payload [Hash,AvroGen::SchemaClass::Record]
94
94
  # @param _key [String]
95
95
  # @return [Hash]
96
96
  def record_attributes(payload, _key=nil)
@@ -81,7 +81,7 @@ module Deimos
81
81
  return payload if self.karafka_config.use_schema_classes.nil? &&
82
82
  !Deimos.config.schema.use_schema_classes
83
83
 
84
- Utils::SchemaClass.instance(payload, encoder.schema, encoder.namespace)
84
+ AvroGen::SchemaClass.instance(payload, encoder.schema, encoder.namespace)
85
85
  end
86
86
 
87
87
  # Deletion payload for a record by default, delegate to the
@@ -15,6 +15,7 @@ module Deimos
15
15
  config.deserializers[:payload].try(:reset_backend)
16
16
  config.deserializers[:key].try(:reset_backend)
17
17
  end
18
+ Deimos::SchemaClass.sync_config!
18
19
  if self.config.schema.use_schema_classes
19
20
  load_generated_schema_classes
20
21
  end
@@ -42,16 +43,16 @@ module Deimos
42
43
  # Loads generated classes
43
44
  # @return [void]
44
45
  def load_generated_schema_classes
45
- if Deimos.config.schema.generated_class_path.nil?
46
- raise 'Cannot use schema classes without schema.generated_class_path. ' \
47
- 'Please provide a directory.'
46
+ path = AvroGen.config.generated_class_path
47
+ if path.nil?
48
+ raise 'Cannot use schema classes without a generated class path. ' \
49
+ 'Please set AvroGen.config.generated_class_path.'
48
50
  end
49
51
 
50
- Dir["./#{Deimos.config.schema.generated_class_path}/**/*.rb"].
52
+ Dir["./#{path}/**/*.rb"].
51
53
  each { |f| require f }
52
54
  rescue LoadError
53
- raise 'Cannot load schema classes. Please regenerate classes with' \
54
- 'rake deimos:generate_schema_models.'
55
+ raise 'Cannot load schema classes. Please regenerate classes with rake avro:generate.'
55
56
  end
56
57
 
57
58
  # Ensure everything is set up correctly for the DB backend.
@@ -173,7 +174,7 @@ module Deimos
173
174
  # @return [String]
174
175
  setting :path
175
176
 
176
- # Local path for schema classes to be generated in.
177
+ # @deprecated Use config.avrogen.generated_class_path instead.
177
178
  # @return [String]
178
179
  setting :generated_class_path, 'app/lib/schema_classes'
179
180
 
@@ -181,6 +182,31 @@ module Deimos
181
182
  # @return [Boolean]
182
183
  setting :use_schema_classes
183
184
 
185
+ # @deprecated Use config.avrogen.nest_child_schemas instead.
186
+ # @return [Boolean]
187
+ setting :nest_child_schemas, true
188
+
189
+ # @deprecated Use config.avrogen.use_full_namespace instead.
190
+ # @return [Boolean]
191
+ setting :use_full_namespace, false
192
+
193
+ # @deprecated Use config.avrogen.schema_namespace_map instead.
194
+ # @return [Hash]
195
+ setting :schema_namespace_map, {}
196
+
197
+ # The base directory for generated protobuf key schemas.
198
+ setting :proto_schema_key_path, 'protos'
199
+ end
200
+
201
+ # Schema class generation settings, forwarded to the avro-gen-ruby gem
202
+ # (AvroGen.config). These previously lived under `schema` (still supported,
203
+ # but deprecated).
204
+ setting :avrogen do
205
+
206
+ # Local path for schema classes to be generated in.
207
+ # @return [String]
208
+ setting :generated_class_path, 'app/lib/schema_classes'
209
+
184
210
  # Set to false to generate child schemas as their own files.
185
211
  # @return [Boolean]
186
212
  setting :nest_child_schemas, true
@@ -191,12 +217,9 @@ module Deimos
191
217
 
192
218
  # Use this option to reduce nesting when using use_full_namespace.
193
219
  # For example: { 'com.mycompany.suborg' => 'SchemaClasses' }
194
- # would replace a prefixed with the given key with the module name SchemaClasses.
220
+ # would replace a prefix matching the given key with the module name SchemaClasses.
195
221
  # @return [Hash]
196
222
  setting :schema_namespace_map, {}
197
-
198
- # The base directory for generated protobuf key schemas.
199
- setting :proto_schema_key_path, 'protos'
200
223
  end
201
224
 
202
225
  # The configured metrics provider.
@@ -5,7 +5,7 @@ module Deimos
5
5
  class << self
6
6
 
7
7
  def allowed_classes
8
- arr = [Hash, SchemaClass::Record]
8
+ arr = [Hash, AvroGen::SchemaClass::Record]
9
9
  if defined?(Google::Protobuf)
10
10
  arr.push(Google::Protobuf.const_get(:AbstractMessage))
11
11
  end
@@ -26,7 +26,7 @@ module Deimos
26
26
  self.allowed_classes.none? { |k| message[:payload].is_a?(k) }
27
27
 
28
28
  payload = message[:payload]
29
- payload = payload.to_h if payload.nil? || payload.is_a?(SchemaClass::Record)
29
+ payload = payload.to_h if payload.nil? || payload.is_a?(AvroGen::SchemaClass::Record)
30
30
  m = Deimos::Message.new(payload,
31
31
  headers: message[:headers],
32
32
  partition_key: message[:partition_key])
@@ -30,6 +30,15 @@ module Deimos
30
30
  log_add(:warn, *args)
31
31
  end
32
32
 
33
+ # Emit a deprecation warning at most once per unique message.
34
+ # @param msg [String]
35
+ def deprecate(msg)
36
+ @deprecations ||= Set.new
37
+ return unless @deprecations.add?(msg)
38
+
39
+ warn("DEPRECATION WARNING: #{msg}")
40
+ end
41
+
33
42
  def metadata_log_text(metadata)
34
43
  metadata.to_h.slice(:timestamp, :offset, :first_offset, :last_offset, :partition, :topic, :size)
35
44
  end
@@ -75,7 +75,7 @@ module Deimos
75
75
  end
76
76
 
77
77
  # Publish the payload to the topic.
78
- # @param payload [Hash, SchemaClass::Record] with an optional payload_key hash key.
78
+ # @param payload [Hash, AvroGen::SchemaClass::Record] with an optional payload_key hash key.
79
79
  # @param topic [String] if specifying the topic
80
80
  # @param headers [Hash] if specifying headers
81
81
  # @return [void]
@@ -99,7 +99,7 @@ module Deimos
99
99
  end
100
100
 
101
101
  # Publish a list of messages.
102
- # @param payloads [Array<Hash, SchemaClass::Record>] with optional payload_key hash key.
102
+ # @param payloads [Array<Hash, AvroGen::SchemaClass::Record>] with optional payload_key hash key.
103
103
  # @param sync [Boolean] if given, override the default setting of
104
104
  # whether to publish synchronously.
105
105
  # @param force_send [Boolean] if true, ignore the configured backend
@@ -112,7 +112,7 @@ module Deimos
112
112
 
113
113
  messages = Array(payloads).map do |p|
114
114
  payload = p
115
- payload = payload.to_h if p.is_a?(SchemaClass::Record)
115
+ payload = payload.to_h if p.is_a?(AvroGen::SchemaClass::Record)
116
116
  m = {
117
117
  payload: payload,
118
118
  headers: headers,
@@ -121,10 +121,10 @@ module Deimos
121
121
  }
122
122
  if payload.is_a?(Hash) && payload.key?(:key) && payload.key?(:message)
123
123
  m[:key] = payload[:key]
124
- m[:key] = m[:key].to_h if m[:key].nil? || m[:key].is_a?(SchemaClass::Record)
124
+ m[:key] = m[:key].to_h if m[:key].nil? || m[:key].is_a?(AvroGen::SchemaClass::Record)
125
125
  m[:payload] = payload[:message]
126
126
  m[:payload] = m[:payload].to_h if m[:payload].nil? ||
127
- m[:payload].is_a?(SchemaClass::Record)
127
+ m[:payload].is_a?(AvroGen::SchemaClass::Record)
128
128
  end
129
129
  m
130
130
  end
@@ -2,6 +2,7 @@
2
2
 
3
3
  require_relative 'base'
4
4
  require 'schema_registry_client'
5
+ require 'avro_gen/avro_parser'
5
6
  require_relative 'avro_schema_coercer'
6
7
 
7
8
  module Deimos
@@ -106,55 +107,25 @@ module Deimos
106
107
 
107
108
  # @param schema [Avro::Schema::NamedSchema] A named schema
108
109
  # @return [String]
110
+ # @deprecated Use AvroGen::AvroParser.schema_classname instead.
109
111
  def self.schema_classname(schema)
110
- schema.name.underscore.camelize.singularize
112
+ AvroGen::AvroParser.schema_classname(schema)
111
113
  end
112
114
 
113
115
  # Converts Avro::Schema::NamedSchema's to String form for generated YARD docs.
114
- # Recursively handles the typing for Arrays, Maps and Unions.
115
116
  # @param avro_schema [Avro::Schema::NamedSchema]
116
117
  # @return [String] A string representation of the Type of this SchemaField
118
+ # @deprecated Use AvroGen::AvroParser.field_type instead.
117
119
  def self.field_type(avro_schema)
118
- case avro_schema.type_sym
119
- when :string, :boolean
120
- avro_schema.type_sym.to_s.titleize
121
- when :int, :long
122
- 'Integer'
123
- when :float, :double
124
- 'Float'
125
- when :record, :enum
126
- schema_classname(avro_schema)
127
- when :array
128
- arr_t = field_type(Deimos::SchemaField.new('n/a', avro_schema.items).type)
129
- "Array<#{arr_t}>"
130
- when :map
131
- map_t = field_type(Deimos::SchemaField.new('n/a', avro_schema.values).type)
132
- "Hash<String, #{map_t}>"
133
- when :union
134
- types = avro_schema.schemas.map do |t|
135
- field_type(Deimos::SchemaField.new('n/a', t).type)
136
- end
137
- types.join(', ')
138
- when :null
139
- 'nil'
140
- end
120
+ AvroGen::AvroParser.field_type(avro_schema)
141
121
  end
142
122
 
143
123
  # Returns the base type of this schema. Decodes Arrays, Maps and Unions
144
124
  # @param schema [Avro::Schema::NamedSchema]
145
125
  # @return [Avro::Schema::NamedSchema]
126
+ # @deprecated Use AvroGen::AvroParser.schema_base_class instead.
146
127
  def self.schema_base_class(schema)
147
- case schema.type_sym
148
- when :array
149
- schema_base_class(schema.items)
150
- when :map
151
- schema_base_class(schema.values)
152
- when :union
153
- schema.schemas.map(&method(:schema_base_class)).
154
- reject { |s| s.type_sym == :null }.first
155
- else
156
- schema
157
- end
128
+ AvroGen::AvroParser.schema_base_class(schema)
158
129
  end
159
130
 
160
131
  def generate_key_schema(field_name)
@@ -0,0 +1,72 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'avro_gen'
4
+
5
+ module Deimos
6
+ # Backwards-compatible shim. Schema class generation moved to the avro-gen-ruby
7
+ # gem (namespace AvroGen). Previously-generated classes that reference
8
+ # `Deimos::SchemaClass::Record` / `Enum` / `Base` still resolve here, with a
9
+ # one-time deprecation warning, via const_missing.
10
+ module SchemaClass
11
+ DEPRECATED_CONSTANTS = {
12
+ Base: AvroGen::SchemaClass::Base,
13
+ Record: AvroGen::SchemaClass::Record,
14
+ Enum: AvroGen::SchemaClass::Enum
15
+ }.freeze
16
+
17
+ # The generation settings that live under Deimos.config.avrogen and are
18
+ # forwarded to AvroGen.config (same names on both sides).
19
+ GENERATION_SETTINGS = %i(
20
+ generated_class_path
21
+ nest_child_schemas
22
+ use_full_namespace
23
+ schema_namespace_map
24
+ ).freeze
25
+
26
+ # @param name [Symbol]
27
+ def self.const_missing(name)
28
+ klass = DEPRECATED_CONSTANTS[name]
29
+ return super unless klass
30
+
31
+ Deimos::Logging.deprecate(
32
+ "Deimos::SchemaClass::#{name} is deprecated; use AvroGen::SchemaClass::#{name} instead. " \
33
+ 'Run `rake avro:upgrade` to update generated classes.'
34
+ )
35
+ # Define the constant so the warning is only emitted once.
36
+ const_set(name, klass)
37
+ klass
38
+ end
39
+
40
+ # Mirror the Deimos generation settings onto AvroGen.config. Deimos.config is
41
+ # the source of truth within Deimos, so this runs on every configure (keeping
42
+ # a reset in sync). The settings live under Deimos.config.avrogen; the legacy
43
+ # Deimos.config.schema.* equivalents still work but emit a deprecation warning.
44
+ # Standalone AvroGen users never trigger this and set AvroGen.config directly.
45
+ # @!visibility private
46
+ def self.sync_config!
47
+ # schema.path is shared with the Avro backends, so it stays under `schema`.
48
+ AvroGen.config.schema_path = Deimos.config.schema.path
49
+ # Refresh AvroGen's cached schema stores on (re)configuration so they don't
50
+ # serve stale schemas after the schema path or files change.
51
+ AvroGen::SchemaValidator.clear_store_cache!
52
+
53
+ GENERATION_SETTINGS.each do |key|
54
+ AvroGen.config.send("#{key}=", generation_setting(key))
55
+ end
56
+ end
57
+
58
+ # Resolve a generation setting, preferring the legacy (deprecated)
59
+ # Deimos.config.schema.* location when it was explicitly set.
60
+ # @!visibility private
61
+ def self.generation_setting(key)
62
+ if Deimos.config.schema.default_value?(key)
63
+ Deimos.config.avrogen.send(key)
64
+ else
65
+ Deimos::Logging.deprecate(
66
+ "Deimos.config.schema.#{key} is deprecated; use Deimos.config.avrogen.#{key} instead."
67
+ )
68
+ Deimos.config.schema.send(key)
69
+ end
70
+ end
71
+ end
72
+ end
@@ -58,9 +58,9 @@ module Deimos
58
58
 
59
59
  schema_key = decoded_key.is_a?(Hash) ? decoded_key : { self.key_field => decoded_key }
60
60
 
61
- Utils::SchemaClass.instance(schema_key,
62
- "#{@schema}_key",
63
- @namespace)
61
+ AvroGen::SchemaClass.instance(schema_key,
62
+ "#{@schema}_key",
63
+ @namespace)
64
64
  end
65
65
 
66
66
  # @param payload [String]
@@ -71,9 +71,9 @@ module Deimos
71
71
  decoded_payload = self.backend.decode(payload)
72
72
  return decoded_payload if !@use_schema_classes || !decoded_payload.is_a?(Hash)
73
73
 
74
- Utils::SchemaClass.instance(decoded_payload,
75
- @schema,
76
- @namespace)
74
+ AvroGen::SchemaClass.instance(decoded_payload,
75
+ @schema,
76
+ @namespace)
77
77
  end
78
78
 
79
79
  # @param payload [Object]
@@ -1,60 +1,27 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require 'avro_gen'
4
+
3
5
  module Deimos
4
6
  module Utils
5
- # Class used by SchemaClassGenerator and Consumer/Producer interfaces
7
+ # Backwards-compatible shim. The schema class helpers moved to
8
+ # AvroGen::SchemaClass. This delegates with a deprecation warning.
6
9
  module SchemaClass
7
10
  class << self
8
-
9
- # @param namespace [String]
10
- # @return [Array<String>]
11
- def modules_for(namespace)
12
- modules = ['Schemas']
13
- namespace_override = nil
14
- module_namespace = namespace
15
-
16
- if Deimos.config.schema.use_full_namespace
17
- if Deimos.config.schema.schema_namespace_map.present?
18
- namespace_keys = Deimos.config.schema.schema_namespace_map.keys.sort_by { |k| -k.length }
19
- namespace_override = namespace_keys.find { |k| module_namespace.include?(k) }
20
- end
21
-
22
- if namespace_override.present?
23
- # override default module
24
- modules = Array(Deimos.config.schema.schema_namespace_map[namespace_override])
25
- module_namespace = module_namespace.gsub(/#{namespace_override}\.?/, '')
26
- end
27
-
28
- namespace_folders = module_namespace.split('.').map { |f| f.underscore.camelize }
29
- modules.concat(namespace_folders) if namespace_folders.any?
11
+ def method_missing(name, ...)
12
+ if AvroGen::SchemaClass.respond_to?(name)
13
+ Deimos::Logging.deprecate(
14
+ "Deimos::Utils::SchemaClass.#{name} is deprecated; use AvroGen::SchemaClass.#{name} instead."
15
+ )
16
+ AvroGen::SchemaClass.send(name, ...)
17
+ else
18
+ super
30
19
  end
31
-
32
- modules
33
20
  end
34
21
 
35
- # Converts a raw payload into an instance of the Schema Class
36
- # @param payload [Hash, Deimos::SchemaClass::Base]
37
- # @param schema [String]
38
- # @param namespace [String]
39
- # @return [Deimos::SchemaClass::Record]
40
- def instance(payload, schema, namespace='')
41
- return payload if payload.is_a?(Deimos::SchemaClass::Base)
42
-
43
- klass = klass(schema, namespace)
44
- return payload if klass.nil? || payload.nil?
45
-
46
- klass.new_from_message(**payload.symbolize_keys)
22
+ def respond_to_missing?(name, include_private=false)
23
+ AvroGen::SchemaClass.respond_to?(name, include_private) || super
47
24
  end
48
-
49
- # Determine and return the SchemaClass with the provided schema and namespace
50
- # @param schema [String]
51
- # @param namespace [String]
52
- # @return [Deimos::SchemaClass]
53
- def klass(schema, namespace)
54
- constants = modules_for(namespace) + [schema.underscore.camelize.singularize]
55
- constants.join('::').safe_constantize
56
- end
57
-
58
25
  end
59
26
  end
60
27
  end
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Deimos
4
- VERSION = '2.5.3'
4
+ VERSION = '2.6.0-beta1'
5
5
  end
data/lib/deimos.rb CHANGED
@@ -17,9 +17,9 @@ require 'deimos/backends/kafka_async'
17
17
  require 'deimos/backends/test'
18
18
 
19
19
  require 'deimos/schema_backends/base'
20
+ require 'avro_gen'
21
+ require 'deimos/schema_class'
20
22
  require 'deimos/utils/schema_class'
21
- require 'deimos/schema_class/enum'
22
- require 'deimos/schema_class/record'
23
23
 
24
24
  require 'deimos/ext/schema_route'
25
25
  require 'deimos/ext/consumer_route'
@@ -102,7 +102,7 @@ module Deimos
102
102
  # Initialize an instance of the provided schema
103
103
  # in the event the schema class is an override, the inherited
104
104
  # schema and namespace will be applied
105
- schema_class = Utils::SchemaClass.klass(schema, namespace)
105
+ schema_class = AvroGen::SchemaClass.klass(schema, namespace)
106
106
  if schema_class.nil?
107
107
  schema_backend_class(backend: backend).
108
108
  new(
@@ -1,6 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'generators/deimos/schema_class_generator'
3
+ require 'avro_gen/generator'
4
4
  require 'optparse'
5
5
  require 'deimos/schema_backends/proto_schema_registry'
6
6
 
@@ -32,10 +32,22 @@ namespace :deimos do
32
32
  Deimos::Utils::DbPoller.start!
33
33
  end
34
34
 
35
- desc 'Run Schema Model Generator'
35
+ desc 'Run Schema Model Generator (deprecated: use rake avro:generate)'
36
36
  task generate_schema_classes: :environment do
37
- Rails.logger.info("Running deimos:generate_schema_classes")
38
- Deimos::Generators::SchemaClassGenerator.start
37
+ Deimos::Logging.deprecate(
38
+ 'rake deimos:generate_schema_classes is deprecated; use rake avro:generate instead.'
39
+ )
40
+ Rails.logger.info('Running deimos:generate_schema_classes')
41
+ # Derive AvroGen configs from the configured Kafka topics so keyed records
42
+ # still get their tombstone/payload_key helpers, then delegate to AvroGen.
43
+ configs = Deimos.karafka_configs.filter_map do |config|
44
+ next if config.schema.nil?
45
+
46
+ { schema: config.schema,
47
+ namespace: config.namespace,
48
+ key_config: config.key_config }
49
+ end
50
+ AvroGen::Generator.new.generate_from_configs(configs)
39
51
  end
40
52
 
41
53
  desc 'Output Protobuf key schemas'