deimos-ruby 1.11.2 → 1.12.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (50) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +9 -2
  3. data/Gemfile.lock +8 -8
  4. data/README.md +96 -0
  5. data/deimos-ruby.gemspec +1 -1
  6. data/docs/CONFIGURATION.md +4 -0
  7. data/lib/deimos/active_record_consume/batch_consumption.rb +7 -9
  8. data/lib/deimos/active_record_consume/message_consumption.rb +4 -3
  9. data/lib/deimos/active_record_consumer.rb +2 -2
  10. data/lib/deimos/active_record_producer.rb +3 -0
  11. data/lib/deimos/config/configuration.rb +29 -0
  12. data/lib/deimos/consume/batch_consumption.rb +2 -2
  13. data/lib/deimos/consume/message_consumption.rb +2 -2
  14. data/lib/deimos/consumer.rb +10 -0
  15. data/lib/deimos/producer.rb +4 -3
  16. data/lib/deimos/schema_backends/avro_base.rb +64 -1
  17. data/lib/deimos/schema_backends/avro_schema_registry.rb +1 -1
  18. data/lib/deimos/schema_backends/base.rb +18 -2
  19. data/lib/deimos/schema_class/base.rb +67 -0
  20. data/lib/deimos/schema_class/enum.rb +24 -0
  21. data/lib/deimos/schema_class/record.rb +59 -0
  22. data/lib/deimos/shared_config.rb +5 -0
  23. data/lib/deimos/test_helpers.rb +43 -7
  24. data/lib/deimos/utils/schema_class.rb +29 -0
  25. data/lib/deimos/version.rb +1 -1
  26. data/lib/deimos.rb +3 -0
  27. data/lib/generators/deimos/schema_class/templates/schema_class.rb.tt +15 -0
  28. data/lib/generators/deimos/schema_class/templates/schema_enum.rb.tt +21 -0
  29. data/lib/generators/deimos/schema_class/templates/schema_record.rb.tt +65 -0
  30. data/lib/generators/deimos/schema_class_generator.rb +247 -0
  31. data/lib/tasks/deimos.rake +8 -0
  32. data/spec/active_record_batch_consumer_spec.rb +120 -110
  33. data/spec/active_record_consumer_spec.rb +97 -88
  34. data/spec/active_record_producer_spec.rb +38 -27
  35. data/spec/batch_consumer_spec.rb +37 -28
  36. data/spec/config/configuration_spec.rb +10 -3
  37. data/spec/consumer_spec.rb +94 -83
  38. data/spec/generators/active_record_generator_spec.rb +1 -0
  39. data/spec/generators/schema_class/my_schema_with_complex_types_spec.rb +206 -0
  40. data/spec/generators/schema_class_generator_spec.rb +186 -0
  41. data/spec/producer_spec.rb +110 -0
  42. data/spec/schema_classes/generated.rb +156 -0
  43. data/spec/schema_classes/my_nested_schema.rb +114 -0
  44. data/spec/schema_classes/my_schema.rb +53 -0
  45. data/spec/schema_classes/my_schema_key.rb +35 -0
  46. data/spec/schema_classes/my_schema_with_complex_types.rb +172 -0
  47. data/spec/schemas/com/my-namespace/Generated.avsc +6 -0
  48. data/spec/schemas/com/my-namespace/MySchemaWithComplexTypes.avsc +95 -0
  49. data/spec/spec_helper.rb +6 -1
  50. metadata +28 -4
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: a5424d471910dffc66acead04734420173b69b41dae286fac03edb60825936fe
4
- data.tar.gz: fe424e2929984d971cb1f48cd88e6a1d78c83783c0b81fd8b8376571ebcf5634
3
+ metadata.gz: b7d6f4f732c60937fcc4e7c2834e59d7a225defbe930bfc37ac02729ddc7e092
4
+ data.tar.gz: e32bbddd1e6d01ebd824b8916fc40c2eac467966e4a154875e90e9f1ab50528e
5
5
  SHA512:
6
- metadata.gz: 9a5f53681cc8fdc79c39c2b42ae11c40f44246ffdf7bcfaeff6dac7b5b1e0cd21de4bc4a931615c1c4c944c0516dcc663a4638ad90d8e8ddf2a4ae9669b4fadc
7
- data.tar.gz: fc21c854ebebce35ab5ecf7a113ca24d7ba4c1a0a880dc225d53b3fdf99823f9a1b06a039159a1a0fe0b7acedd45eec29f3b84af09a847efb329b03eef9f46bd
6
+ metadata.gz: 1d1b3ba943c5ac49d87f5d4f77d22d88d0e5ff5467eef5e040acb2f5f7af0ae3f4d042d8156e5a8e413c394496a6092c7a691ddb840e366bb23617f48103bd62
7
+ data.tar.gz: ba8fe17bed6e62f86d87ae54b81a8a3cb57efe7fc06586166d1a59d92b87640e19f74213724f3b7404846975d77b0b016127fa8a846a4de270a1c8192f9a3344
data/CHANGELOG.md CHANGED
@@ -7,10 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
7
7
 
8
8
  ## UNRELEASED
9
9
 
10
- ## 1.11.2 - 2021-08-27
10
+ # 1.12.0 - 2021-11-01
11
+
12
+ ### Features :star:
13
+
14
+ - Generate Schema classes from Avro Schemas
15
+ - Use Schema Classes in your consumer and producer
16
+
17
+ ## 1.11.0 - 2021-08-27
11
18
 
12
19
  - ### Fixes :wrench:
13
- - Fixed ActiveRecord batch consumption on MySQL.
20
+ - Fixed issue where ActiveRecord batch consumption could fail when decoding keys.
14
21
 
15
22
  - ### Roadmap :car:
16
23
  - TestHelper does not automatically reset Deimos config before each test. [#120](https://github.com/flipp-oss/deimos/pull/120).
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- deimos-ruby (1.11.2)
4
+ deimos-ruby (1.11.0)
5
5
  avro_turf (~> 0.11)
6
6
  fig_tree (~> 0.0.2)
7
7
  phobos (>= 1.9, < 3.0)
@@ -88,11 +88,11 @@ GEM
88
88
  ddtrace (0.46.0)
89
89
  msgpack
90
90
  diff-lcs (1.4.4)
91
- digest-crc (0.6.3)
91
+ digest-crc (0.6.4)
92
92
  rake (>= 12.0.0, < 14.0.0)
93
93
  dogstatsd-ruby (4.8.3)
94
94
  erubi (1.10.0)
95
- excon (0.81.0)
95
+ excon (0.85.0)
96
96
  exponential-backoff (0.0.4)
97
97
  ffi (1.15.0)
98
98
  fig_tree (0.0.2)
@@ -153,10 +153,10 @@ GEM
153
153
  nenv (~> 0.1)
154
154
  shellany (~> 0.0)
155
155
  parallel (1.20.1)
156
- parser (3.0.0.0)
156
+ parser (3.0.1.1)
157
157
  ast (~> 2.4.1)
158
158
  pg (1.2.3)
159
- phobos (2.0.2)
159
+ phobos (2.1.0)
160
160
  activesupport (>= 3.0.0)
161
161
  concurrent-ruby (>= 1.0.2)
162
162
  concurrent-ruby-ext (>= 1.0.2)
@@ -227,7 +227,7 @@ GEM
227
227
  rspec-support (3.10.2)
228
228
  rspec_junit_formatter (0.4.1)
229
229
  rspec-core (>= 2, < 4, != 2.12.0)
230
- rubocop (0.88.0)
230
+ rubocop (0.89.0)
231
231
  parallel (~> 1.10)
232
232
  parser (>= 2.7.1.1)
233
233
  rainbow (>= 2.2.2, < 4.0)
@@ -240,7 +240,7 @@ GEM
240
240
  parser (>= 2.7.1.5)
241
241
  rubocop-rspec (1.42.0)
242
242
  rubocop (>= 0.87.0)
243
- ruby-kafka (1.3.0)
243
+ ruby-kafka (1.4.0)
244
244
  digest-crc
245
245
  ruby-progressbar (1.11.0)
246
246
  shellany (0.0.1)
@@ -284,7 +284,7 @@ DEPENDENCIES
284
284
  rspec (~> 3)
285
285
  rspec-rails (~> 4)
286
286
  rspec_junit_formatter (~> 0.3)
287
- rubocop (= 0.88.0)
287
+ rubocop (= 0.89.0)
288
288
  rubocop-rspec (= 1.42.0)
289
289
  sqlite3 (~> 1.3)
290
290
 
data/README.md CHANGED
@@ -27,6 +27,7 @@ Built on Phobos and hence Ruby-Kafka.
27
27
  * [Database Backend](#database-backend)
28
28
  * [Database Poller](#database-poller)
29
29
  * [Running Consumers](#running-consumers)
30
+ * [Generated Schema Classes](#generated-schema-classes)
30
31
  * [Metrics](#metrics)
31
32
  * [Testing](#testing)
32
33
  * [Test Helpers](#test-helpers)
@@ -795,6 +796,101 @@ which can be useful if you want to figure out if you're inside the task
795
796
  as opposed to running your Rails server or console. E.g. you could start your
796
797
  DB backend only when your rake task is running.
797
798
 
799
+ ## Generated Schema Classes
800
+
801
+ Deimos offers a way to generate classes from Avro schemas. These classes are documented
802
+ with YARD to aid in IDE auto-complete, and will help to move errors closer to the code.
803
+
804
+ Add the following configurations for schema class generation:
805
+
806
+ ```ruby
807
+ config.schema.generated_class_path 'path/to/generated/classes' # Defaults to 'app/lib/schema_classes'
808
+ ```
809
+
810
+ Run the following command to generate schema classes in your application. It will generate classes for every configured consumer or producer by `Deimos.configure`:
811
+
812
+ bundle exec rake deimos:generate_schema_classes
813
+
814
+ Add the following configurations to start using generated schema classes in your application's Consumers and Producers:
815
+
816
+ config.schema.use_schema_classes true
817
+
818
+ Additionally, you can enable or disable the usage of schema classes for a particular consumer or producer with the
819
+ `use_schema_classes` config. See [Configuration](./docs/CONFIGURATION.md#defining-producers).
820
+
821
+ ### Consumer
822
+
823
+ The consumer interface uses the `decode_message` method to turn JSON hash into the Schemas
824
+ generated Class and provides it to the `consume`/`consume_batch` methods for their use.
825
+
826
+ Examples of consumers would look like this:
827
+ ```ruby
828
+ class MyConsumer < Deimos::Consumer
829
+ def consume(payload, metadata)
830
+ # Same method as Phobos consumers but payload is now an instance of Deimos::SchemaClass::Record
831
+ # rather than a hash. metadata is still a hash that contains information like :key and :topic.
832
+ # You can interact with the schema class instance in the following way:
833
+ do_something(payload.test_id, payload.some_int)
834
+ # The original behaviour was as follows:
835
+ do_something(payload[:test_id], payload[:some_int])
836
+ end
837
+ end
838
+ ```
839
+
840
+ ```ruby
841
+ class MyActiveRecordConsumer < Deimos::ActiveRecordConsumer
842
+ record_class Widget
843
+ # Any method that expects a message payload as a hash will instead
844
+ # receive an instance of Deimos::SchemaClass::Record.
845
+ def record_attributes(payload, key)
846
+ # You can interact with the schema class instance in the following way:
847
+ super.merge(:some_field => "some_value-#{payload.test_id}")
848
+ # The original behaviour was as follows:
849
+ super.merge(:some_field => "some_value-#{payload[:test_id]}")
850
+ end
851
+ end
852
+ ```
853
+
854
+ ### Producer
855
+ Similarly to the consumer interface, the producer interface for using Schema Classes in your app
856
+ relies on the `publish`/`publish_list` methods to convert a _provided_ instance of a Schema Class
857
+ into a hash that can be used freely by the Kafka client.
858
+
859
+ Examples of producers would look like this:
860
+ ```ruby
861
+ class MyProducer < Deimos::Producer
862
+ class << self
863
+ # @param test_id [String]
864
+ # @param some_int [Integer]
865
+ def self.send_a_message(test_id, some_int)
866
+ # Instead of sending in a Hash object to the publish or publish_list method,
867
+ # you can initialize an instance of your schema class and send that in.
868
+ message = Schemas::MySchema.new(
869
+ test_id: test_id,
870
+ some_int: some_int
871
+ )
872
+ self.publish(message)
873
+ self.publish_list([message])
874
+ end
875
+ end
876
+ end
877
+ ```
878
+
879
+ ```ruby
880
+ class MyActiveRecordProducer < Deimos::ActiveRecordProducer
881
+ record_class Widget
882
+ # @param payload [Deimos::SchemaClass::Record]
883
+ # @param _record [Widget]
884
+ def self.generate_payload(attributes, _record)
885
+ # This method converts your ActiveRecord into a Deimos::SchemaClass::Record. You will be able to use super
886
+ # as an instance of Schemas::MySchema and set values that are not on your ActiveRecord schema.
887
+ res = super
888
+ res.some_value = "some_value-#{res.test_id}"
889
+ res
890
+ end
891
+ end
892
+ ```
893
+
798
894
  # Metrics
799
895
 
800
896
  Deimos includes some metrics reporting out the box. It ships with DataDog support, but you can add custom metric providers as well.
data/deimos-ruby.gemspec CHANGED
@@ -39,7 +39,7 @@ Gem::Specification.new do |spec|
39
39
  spec.add_development_dependency('rspec', '~> 3')
40
40
  spec.add_development_dependency('rspec_junit_formatter', '~>0.3')
41
41
  spec.add_development_dependency('rspec-rails', '~> 4')
42
- spec.add_development_dependency('rubocop', '0.88.0')
42
+ spec.add_development_dependency('rubocop', '0.89.0')
43
43
  spec.add_development_dependency('rubocop-rspec', '1.42.0')
44
44
  spec.add_development_dependency('sqlite3', '~> 1.3')
45
45
  end
@@ -44,6 +44,7 @@ topic|nil|Topic to produce to.
44
44
  schema|nil|Name of the schema to use to encode data before producing.
45
45
  namespace|nil|Namespace of the schema to use when finding it locally.
46
46
  key_config|nil|Configuration hash for message keys. See [Kafka Message Keys](../README.md#installation)
47
+ use_schema_classes|nil|Set to true or false to enable or disable using the producers schema classes. See [Generated Schema Classes](../README.md#generated-schema-classes)
47
48
 
48
49
  ## Defining Consumers
49
50
 
@@ -81,6 +82,7 @@ namespace|nil|Namespace of the schema to use when finding it locally.
81
82
  key_config|nil|Configuration hash for message keys. See [Kafka Message Keys](../README.md#installation)
82
83
  disabled|false|Set to true to skip starting an actual listener for this consumer on startup.
83
84
  group_id|nil|ID of the consumer group.
85
+ use_schema_classes|nil|Set to true or false to enable or disable using the consumers schema classes. See [Generated Schema Classes](../README.md#generated-schema-classes)
84
86
  max_concurrency|1|Number of threads created for this listener. Each thread will behave as an independent consumer. They don't share any state.
85
87
  start_from_beginning|true|Once the consumer group has checkpointed its progress in the topic's partitions, the consumers will always start from the checkpointed offsets, regardless of config. As such, this setting only applies when the consumer initially starts consuming from a topic
86
88
  max_bytes_per_partition|512.kilobytes|Maximum amount of data fetched from a single partition at a time.
@@ -176,6 +178,8 @@ Config name|Default|Description
176
178
  schema.backend|`:mock`|Backend representing the schema encoder/decoder. You can see a full list [here](../lib/deimos/schema_backends).
177
179
  schema.registry_url|`http://localhost:8081`|URL of the Confluent schema registry.
178
180
  schema.path|nil|Local path to find your schemas.
181
+ schema.use_schema_classes|false|Set this to true to use generated schema classes in your application.
182
+ schema.generated_schema_path|`app/lib/schema_classes`|Local path to generated schema classes.
179
183
 
180
184
  ## Database Producer Configuration
181
185
 
@@ -15,7 +15,7 @@ module Deimos
15
15
  # If two messages in a batch have the same key, we cannot process them
16
16
  # in the same operation as they would interfere with each other. Thus
17
17
  # they are split
18
- # @param payloads [Array<Hash>] Decoded payloads.
18
+ # @param payloads [Array<Hash|Deimos::SchemaClass::Record>] Decoded payloads
19
19
  # @param metadata [Hash] Information about batch, including keys.
20
20
  def consume_batch(payloads, metadata)
21
21
  messages = payloads.
@@ -44,12 +44,14 @@ module Deimos
44
44
  # @param key [String] The encoded key.
45
45
  # @return [Hash] The key attributes.
46
46
  def record_key(key)
47
- if key.nil?
47
+ decoded_key = decode_key(key)
48
+
49
+ if decoded_key.nil?
48
50
  {}
49
- elsif key.is_a?(Hash)
50
- @key_converter.convert(key)
51
+ elsif decoded_key.is_a?(Hash)
52
+ @key_converter.convert(decoded_key)
51
53
  else
52
- { @klass.primary_key => key }
54
+ { @klass.primary_key => decoded_key }
53
55
  end
54
56
  end
55
57
 
@@ -100,10 +102,6 @@ module Deimos
100
102
 
101
103
  options = if key_cols.empty?
102
104
  {} # Can't upsert with no key, just do regular insert
103
- elsif ActiveRecord::Base.connection.adapter_name =~ /mysql/i
104
- {
105
- on_duplicate_key_update: :all
106
- }
107
105
  else
108
106
  {
109
107
  on_duplicate_key_update: {
@@ -9,7 +9,7 @@ module Deimos
9
9
  # Default is to use the primary key column and the value of the first
10
10
  # field in the key.
11
11
  # @param klass [Class < ActiveRecord::Base]
12
- # @param _payload [Hash]
12
+ # @param _payload [Hash|Deimos::SchemaClass::Record]
13
13
  # @param key [Object]
14
14
  # @return [ActiveRecord::Base]
15
15
  def fetch_record(klass, _payload, key)
@@ -18,13 +18,14 @@ module Deimos
18
18
 
19
19
  # Assign a key to a new record.
20
20
  # @param record [ActiveRecord::Base]
21
- # @param _payload [Hash]
21
+ # @param _payload [Hash|Deimos::SchemaClass::Record]
22
22
  # @param key [Object]
23
23
  def assign_key(record, _payload, key)
24
24
  record[record.class.primary_key] = key
25
25
  end
26
26
 
27
- # :nodoc:
27
+ # @param payload [Hash|Deimos::SchemaClass::Record] Decoded payloads
28
+ # @param metadata [Hash] Information about batch, including keys.
28
29
  def consume(payload, metadata)
29
30
  unless self.process_message?(payload)
30
31
  Deimos.config.logger.debug(
@@ -50,14 +50,14 @@ module Deimos
50
50
 
51
51
  # Override this method (with `super`) if you want to add/change the default
52
52
  # attributes set to the new/existing record.
53
- # @param payload [Hash]
53
+ # @param payload [Hash|Deimos::SchemaClass::Record]
54
54
  # @param _key [String]
55
55
  def record_attributes(payload, _key=nil)
56
56
  @converter.convert(payload)
57
57
  end
58
58
 
59
59
  # Override this message to conditionally save records
60
- # @param payload [Hash] The kafka message as a hash
60
+ # @param _payload [Hash|Deimos::SchemaClass::Record] The kafka message
61
61
  # @return [Boolean] if true, record is created/update.
62
62
  # If false, record processing is skipped but message offset is still committed.
63
63
  def process_message?(_payload)
@@ -58,6 +58,9 @@ module Deimos
58
58
  payload.delete_if do |k, _|
59
59
  k.to_sym != :payload_key && !fields.map(&:name).include?(k)
60
60
  end
61
+ return payload unless Utils::SchemaClass.use?(config.to_h)
62
+
63
+ Utils::SchemaClass.instance(payload, config[:schema])
61
64
  end
62
65
 
63
66
  # Query to use when polling the database with the DbPoller. Add
@@ -18,6 +18,9 @@ module Deimos
18
18
  # :nodoc:
19
19
  after_configure do
20
20
  Phobos.configure(self.config.phobos_config)
21
+ if self.config.schema.use_schema_classes
22
+ load_generated_schema_classes
23
+ end
21
24
  self.config.producer_objects.each do |producer|
22
25
  configure_producer_or_consumer(producer)
23
26
  end
@@ -28,6 +31,17 @@ module Deimos
28
31
  validate_db_backend if self.config.producers.backend == :db
29
32
  end
30
33
 
34
+ # Loads generated classes
35
+ def self.load_generated_schema_classes
36
+ if Deimos.config.schema.generated_class_path.nil?
37
+ raise 'Cannot use schema classes without schema.generated_class_path. Please provide a directory.'
38
+ end
39
+
40
+ Dir["./#{Deimos.config.schema.generated_class_path}/**/*.rb"].sort.each { |f| require f }
41
+ rescue LoadError
42
+ raise 'Cannot load schema classes. Please regenerate classes with rake deimos:generate_schema_models.'
43
+ end
44
+
31
45
  # Ensure everything is set up correctly for the DB backend.
32
46
  def self.validate_db_backend
33
47
  begin
@@ -68,6 +82,7 @@ module Deimos
68
82
  schema(kafka_config.schema) if kafka_config.schema.present?
69
83
  namespace(kafka_config.namespace) if kafka_config.namespace.present?
70
84
  key_config(**kafka_config.key_config) if kafka_config.key_config.present?
85
+ schema_class_config(kafka_config.use_schema_classes) if kafka_config.use_schema_classes.present?
71
86
  end
72
87
  end
73
88
 
@@ -260,6 +275,14 @@ module Deimos
260
275
  # Local path to look for schemas in.
261
276
  # @return [String]
262
277
  setting :path
278
+
279
+ # Local path for schema classes to be generated in.
280
+ # @return [String]
281
+ setting :generated_class_path, 'app/lib/schema_classes'
282
+
283
+ # Set to true to use the generated schema classes in your application
284
+ # @return [Boolean]
285
+ setting :use_schema_classes, false
263
286
  end
264
287
 
265
288
  # The configured metrics provider.
@@ -301,6 +324,9 @@ module Deimos
301
324
  # Key configuration (see docs).
302
325
  # @return [Hash]
303
326
  setting :key_config
327
+ # Configure the usage of generated schema classes for this producer
328
+ # @return [Boolean]
329
+ setting :use_schema_classes
304
330
  end
305
331
 
306
332
  setting_object :consumer do
@@ -323,6 +349,9 @@ module Deimos
323
349
  # listener.
324
350
  # @return [Boolean]
325
351
  setting :disabled, false
352
+ # Configure the usage of generated schema classes for this consumer
353
+ # @return [Boolean]
354
+ setting :use_schema_classes
326
355
 
327
356
  # These are the phobos "listener" configs. See CONFIGURATION.md for more
328
357
  # info.
@@ -21,11 +21,11 @@ module Deimos
21
21
  metadata[:first_offset] = batch.first&.offset
22
22
 
23
23
  payloads = batch.map do |message|
24
- message.payload ? self.class.decoder.decode(message.payload) : nil
24
+ decode_message(message.payload)
25
25
  end
26
26
  _received_batch(payloads, metadata)
27
27
  _with_span do
28
- yield payloads, metadata
28
+ yield(payloads, metadata)
29
29
  end
30
30
  end
31
31
  _handle_batch_success(benchmark.real, payloads, metadata)
@@ -15,9 +15,9 @@ module Deimos
15
15
  benchmark = Benchmark.measure do
16
16
  _with_span do
17
17
  new_metadata[:key] = decode_key(metadata[:key]) if self.class.config[:key_configured]
18
- decoded_payload = payload ? self.class.decoder.decode(payload) : nil
18
+ decoded_payload = decode_message(payload)
19
19
  _received_message(decoded_payload, new_metadata)
20
- yield decoded_payload, new_metadata
20
+ yield(decoded_payload, new_metadata)
21
21
  end
22
22
  end
23
23
  _handle_success(benchmark.real, decoded_payload, new_metadata)
@@ -51,6 +51,16 @@ module Deimos
51
51
  end
52
52
  end
53
53
 
54
+ # Helper method to decode an encoded message.
55
+ # @param payload [Object]
56
+ # @return [Object] the decoded message.
57
+ def decode_message(payload)
58
+ decoded_payload = payload.nil? ? nil : self.class.decoder.decode(payload)
59
+ return decoded_payload unless Utils::SchemaClass.use?(self.class.config.to_h)
60
+
61
+ Utils::SchemaClass.instance(decoded_payload, self.class.config[:schema])
62
+ end
63
+
54
64
  private
55
65
 
56
66
  def _with_span
@@ -57,6 +57,7 @@ module Deimos
57
57
  MAX_BATCH_SIZE = 500
58
58
 
59
59
  class << self
60
+
60
61
  # @return [Hash]
61
62
  def config
62
63
  @config ||= {
@@ -86,14 +87,14 @@ module Deimos
86
87
  end
87
88
 
88
89
  # Publish the payload to the topic.
89
- # @param payload [Hash] with an optional payload_key hash key.
90
+ # @param payload [Hash|SchemaClass::Record] with an optional payload_key hash key.
90
91
  # @param topic [String] if specifying the topic
91
92
  def publish(payload, topic: self.topic)
92
93
  publish_list([payload], topic: topic)
93
94
  end
94
95
 
95
96
  # Publish a list of messages.
96
- # @param payloads [Hash|Array<Hash>] with optional payload_key hash key.
97
+ # @param payloads [Array<Hash|SchemaClass::Record>] with optional payload_key hash key.
97
98
  # @param sync [Boolean] if given, override the default setting of
98
99
  # whether to publish synchronously.
99
100
  # @param force_send [Boolean] if true, ignore the configured backend
@@ -113,7 +114,7 @@ module Deimos
113
114
  topic: topic,
114
115
  payloads: payloads
115
116
  ) do
116
- messages = Array(payloads).map { |p| Deimos::Message.new(p, self) }
117
+ messages = Array(payloads).map { |p| Deimos::Message.new(p.to_h, self) }
117
118
  messages.each { |m| _process_message(m, topic) }
118
119
  messages.in_groups_of(MAX_BATCH_SIZE, false) do |batch|
119
120
  self.produce_batch(backend_class, batch)
@@ -66,7 +66,7 @@ module Deimos
66
66
  def schema_fields
67
67
  avro_schema.fields.map do |field|
68
68
  enum_values = field.type.type == 'enum' ? field.type.symbols : []
69
- SchemaField.new(field.name, field.type, enum_values)
69
+ SchemaField.new(field.name, field.type, enum_values, field.default)
70
70
  end
71
71
  end
72
72
 
@@ -77,6 +77,12 @@ module Deimos
77
77
  fail_on_extra_fields: true)
78
78
  end
79
79
 
80
+ # @override
81
+ # @return [Avro::Schema]
82
+ def load_schema
83
+ avro_schema
84
+ end
85
+
80
86
  # @override
81
87
  def self.mock_backend
82
88
  :avro_validation
@@ -87,6 +93,59 @@ module Deimos
87
93
  'avro/binary'
88
94
  end
89
95
 
96
+ # @param schema [Avro::Schema::NamedSchema] A named schema
97
+ # @return [String]
98
+ def self.schema_classname(schema)
99
+ schema.name.underscore.camelize
100
+ end
101
+
102
+ # Converts Avro::Schema::NamedSchema's to String form for generated YARD docs.
103
+ # Recursively handles the typing for Arrays, Maps and Unions.
104
+ # @param avro_schema [Avro::Schema::NamedSchema]
105
+ # @return [String] A string representation of the Type of this SchemaField
106
+ def self.field_type(avro_schema)
107
+ case avro_schema.type_sym
108
+ when :string, :boolean
109
+ avro_schema.type_sym.to_s.titleize
110
+ when :int, :long
111
+ 'Integer'
112
+ when :float, :double
113
+ 'Float'
114
+ when :record, :enum
115
+ schema_classname(avro_schema)
116
+ when :array
117
+ arr_t = field_type(Deimos::SchemaField.new('n/a', avro_schema.items).type)
118
+ "Array<#{arr_t}>"
119
+ when :map
120
+ map_t = field_type(Deimos::SchemaField.new('n/a', avro_schema.values).type)
121
+ "Hash<String, #{map_t}>"
122
+ when :union
123
+ types = avro_schema.schemas.map do |t|
124
+ field_type(Deimos::SchemaField.new('n/a', t).type)
125
+ end
126
+ types.join(', ')
127
+ when :null
128
+ 'nil'
129
+ end
130
+ end
131
+
132
+ # Returns the base type of this schema. Decodes Arrays, Maps and Unions
133
+ # @param schema [Avro::Schema::NamedSchema]
134
+ # @return [Avro::Schema::NamedSchema]
135
+ def self.schema_base_class(schema)
136
+ case schema.type_sym
137
+ when :array
138
+ schema_base_class(schema.items)
139
+ when :map
140
+ schema_base_class(schema.values)
141
+ when :union
142
+ schema.schemas.map(&method(:schema_base_class)).
143
+ reject { |s| s.type_sym == :null }.first
144
+ else
145
+ schema
146
+ end
147
+ end
148
+
90
149
  private
91
150
 
92
151
  # @param schema [String]
@@ -135,6 +194,10 @@ module Deimos
135
194
  def _key_schema_name(schema)
136
195
  "#{schema}_key"
137
196
  end
197
+
198
+ def _schema_name
199
+ avro_schema.name
200
+ end
138
201
  end
139
202
  end
140
203
  end
@@ -10,7 +10,7 @@ module Deimos
10
10
  class AvroSchemaRegistry < AvroBase
11
11
  # @override
12
12
  def decode_payload(payload, schema:)
13
- avro_turf_messaging.decode(payload, schema_name: schema)
13
+ avro_turf_messaging.decode(payload.to_s, schema_name: schema)
14
14
  end
15
15
 
16
16
  # @override
@@ -3,15 +3,16 @@
3
3
  module Deimos
4
4
  # Represents a field in the schema.
5
5
  class SchemaField
6
- attr_accessor :name, :type, :enum_values
6
+ attr_accessor :name, :type, :enum_values, :default
7
7
 
8
8
  # @param name [String]
9
9
  # @param type [Object]
10
10
  # @param enum_values [Array<String>]
11
- def initialize(name, type, enum_values=[])
11
+ def initialize(name, type, enum_values=[], default=:no_default)
12
12
  @name = name
13
13
  @type = type
14
14
  @enum_values = enum_values
15
+ @default = default
15
16
  end
16
17
  end
17
18
 
@@ -43,6 +44,7 @@ module Deimos
43
44
  # @return [Hash,nil]
44
45
  def decode(payload, schema: nil)
45
46
  return nil if payload.nil?
47
+
46
48
  decode_payload(payload, schema: schema || @schema)
47
49
  end
48
50
 
@@ -78,6 +80,14 @@ module Deimos
78
80
  raise NotImplementedError
79
81
  end
80
82
 
83
+ # Converts your schema to String form for generated YARD docs.
84
+ # To be defined by subclass.
85
+ # @param schema [Object]
86
+ # @return [String] A string representation of the Type
87
+ def self.field_type(schema)
88
+ raise NotImplementedError
89
+ end
90
+
81
91
  # Encode a payload. To be defined by subclass.
82
92
  # @param payload [Hash]
83
93
  # @param schema [Symbol|String]
@@ -145,6 +155,12 @@ module Deimos
145
155
  def decode_key(_payload, _key_id)
146
156
  raise NotImplementedError
147
157
  end
158
+
159
+ # Forcefully loads the schema into memory.
160
+ # @return [Object] The schema that is of use.
161
+ def load_schema
162
+ raise NotImplementedError
163
+ end
148
164
  end
149
165
  end
150
166
  end