deimos-ruby 1.10.2 → 1.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +19 -0
  3. data/Gemfile.lock +8 -8
  4. data/README.md +150 -16
  5. data/deimos-ruby.gemspec +1 -1
  6. data/docs/CONFIGURATION.md +4 -0
  7. data/lib/deimos/active_record_consume/batch_consumption.rb +1 -1
  8. data/lib/deimos/active_record_consume/message_consumption.rb +4 -3
  9. data/lib/deimos/active_record_consumer.rb +2 -2
  10. data/lib/deimos/active_record_producer.rb +3 -0
  11. data/lib/deimos/config/configuration.rb +29 -0
  12. data/lib/deimos/consume/batch_consumption.rb +2 -2
  13. data/lib/deimos/consume/message_consumption.rb +2 -2
  14. data/lib/deimos/consumer.rb +10 -0
  15. data/lib/deimos/producer.rb +4 -3
  16. data/lib/deimos/schema_backends/avro_base.rb +64 -1
  17. data/lib/deimos/schema_backends/avro_schema_registry.rb +1 -1
  18. data/lib/deimos/schema_backends/base.rb +18 -2
  19. data/lib/deimos/schema_class/base.rb +67 -0
  20. data/lib/deimos/schema_class/enum.rb +24 -0
  21. data/lib/deimos/schema_class/record.rb +59 -0
  22. data/lib/deimos/shared_config.rb +5 -0
  23. data/lib/deimos/test_helpers.rb +70 -17
  24. data/lib/deimos/utils/schema_class.rb +29 -0
  25. data/lib/deimos/version.rb +1 -1
  26. data/lib/deimos.rb +3 -0
  27. data/lib/generators/deimos/schema_class/templates/schema_class.rb.tt +15 -0
  28. data/lib/generators/deimos/schema_class/templates/schema_enum.rb.tt +21 -0
  29. data/lib/generators/deimos/schema_class/templates/schema_record.rb.tt +65 -0
  30. data/lib/generators/deimos/schema_class_generator.rb +247 -0
  31. data/lib/tasks/deimos.rake +8 -0
  32. data/spec/active_record_batch_consumer_spec.rb +120 -110
  33. data/spec/active_record_consumer_spec.rb +97 -88
  34. data/spec/active_record_producer_spec.rb +38 -27
  35. data/spec/batch_consumer_spec.rb +37 -28
  36. data/spec/config/configuration_spec.rb +10 -3
  37. data/spec/consumer_spec.rb +94 -83
  38. data/spec/generators/active_record_generator_spec.rb +1 -0
  39. data/spec/generators/schema_class/my_schema_with_complex_types_spec.rb +206 -0
  40. data/spec/generators/schema_class_generator_spec.rb +186 -0
  41. data/spec/producer_spec.rb +110 -0
  42. data/spec/schema_classes/generated.rb +156 -0
  43. data/spec/schema_classes/my_nested_schema.rb +114 -0
  44. data/spec/schema_classes/my_schema.rb +53 -0
  45. data/spec/schema_classes/my_schema_key.rb +35 -0
  46. data/spec/schema_classes/my_schema_with_complex_types.rb +172 -0
  47. data/spec/schemas/com/my-namespace/Generated.avsc +6 -0
  48. data/spec/schemas/com/my-namespace/MySchemaWithComplexTypes.avsc +95 -0
  49. data/spec/spec_helper.rb +6 -1
  50. metadata +28 -4
@@ -3,15 +3,16 @@
3
3
  module Deimos
4
4
  # Represents a field in the schema.
5
5
  class SchemaField
6
- attr_accessor :name, :type, :enum_values
6
+ attr_accessor :name, :type, :enum_values, :default
7
7
 
8
8
  # @param name [String]
9
9
  # @param type [Object]
10
10
  # @param enum_values [Array<String>]
11
- def initialize(name, type, enum_values=[])
11
+ def initialize(name, type, enum_values=[], default=:no_default)
12
12
  @name = name
13
13
  @type = type
14
14
  @enum_values = enum_values
15
+ @default = default
15
16
  end
16
17
  end
17
18
 
@@ -43,6 +44,7 @@ module Deimos
43
44
  # @return [Hash,nil]
44
45
  def decode(payload, schema: nil)
45
46
  return nil if payload.nil?
47
+
46
48
  decode_payload(payload, schema: schema || @schema)
47
49
  end
48
50
 
@@ -78,6 +80,14 @@ module Deimos
78
80
  raise NotImplementedError
79
81
  end
80
82
 
83
+ # Converts your schema to String form for generated YARD docs.
84
+ # To be defined by subclass.
85
+ # @param schema [Object]
86
+ # @return [String] A string representation of the Type
87
+ def self.field_type(schema)
88
+ raise NotImplementedError
89
+ end
90
+
81
91
  # Encode a payload. To be defined by subclass.
82
92
  # @param payload [Hash]
83
93
  # @param schema [Symbol|String]
@@ -145,6 +155,12 @@ module Deimos
145
155
  def decode_key(_payload, _key_id)
146
156
  raise NotImplementedError
147
157
  end
158
+
159
+ # Forcefully loads the schema into memory.
160
+ # @return [Object] The schema that is of use.
161
+ def load_schema
162
+ raise NotImplementedError
163
+ end
148
164
  end
149
165
  end
150
166
  end
@@ -0,0 +1,67 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'json'
4
+
5
+ module Deimos
6
+ module SchemaClass
7
+ # Base Class for Schema Classes generated from Avro.
8
+ class Base
9
+
10
+ # :nodoc:
11
+ def initialize(*_args)
12
+ end
13
+
14
+ # Converts the object to a string that represents a JSON object
15
+ # @return [String] a JSON string
16
+ def to_json(*_args)
17
+ to_h.to_json
18
+ end
19
+
20
+ # Converts the object to a hash which can be used for debugging.
21
+ # @return [Hash] a hash representation of the payload
22
+ def as_json(_opts={})
23
+ JSON.parse(to_json)
24
+ end
25
+
26
+ # Converts the object attributes to a hash which can be used for Kafka
27
+ # @return [Hash] the payload as a hash.
28
+ def to_h
29
+ raise NotImplementedError
30
+ end
31
+
32
+ # :nodoc:
33
+ def ==(other)
34
+ comparison = other
35
+ if other.class == self.class
36
+ comparison = other.state
37
+ end
38
+
39
+ comparison == self.state
40
+ end
41
+
42
+ # :nodoc:
43
+ def to_s
44
+ klass = self.class
45
+ "#{klass}(#{self.as_json.symbolize_keys.to_s[1..-2]})"
46
+ end
47
+
48
+ # Initializes this class from a given value
49
+ # @param value [Object]
50
+ def self.initialize_from_value(value)
51
+ raise NotImplementedError
52
+ end
53
+
54
+ protected
55
+
56
+ # :nodoc:
57
+ def state
58
+ as_json
59
+ end
60
+
61
+ # :nodoc:
62
+ def hash
63
+ state.hash
64
+ end
65
+ end
66
+ end
67
+ end
@@ -0,0 +1,24 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative 'base'
4
+ require 'json'
5
+
6
+ module Deimos
7
+ module SchemaClass
8
+ # Base Class for Enum Classes generated from Avro.
9
+ class Enum < Base
10
+ # Returns all the valid symbols for this enum.
11
+ # @return [Array<String>]
12
+ def symbols
13
+ raise NotImplementedError
14
+ end
15
+
16
+ # :nodoc:
17
+ def self.initialize_from_value(value)
18
+ return nil if value.nil?
19
+
20
+ value.is_a?(self) ? value : self.new(value)
21
+ end
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,59 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative 'base'
4
+ require 'json'
5
+
6
+ module Deimos
7
+ module SchemaClass
8
+ # Base Class of Record Classes generated from Avro.
9
+ class Record < Base
10
+ # Element access method as if this Object were a hash
11
+ # @param key[String||Symbol]
12
+ # @return [Object] The value of the attribute if exists, nil otherwise
13
+ def [](key)
14
+ self.try(key.to_sym)
15
+ end
16
+
17
+ # :nodoc
18
+ def with_indifferent_access
19
+ self
20
+ end
21
+
22
+ # Returns the schema name of the inheriting class.
23
+ # @return [String]
24
+ def schema
25
+ raise NotImplementedError
26
+ end
27
+
28
+ # Returns the namespace for the schema of the inheriting class.
29
+ # @return [String]
30
+ def namespace
31
+ raise NotImplementedError
32
+ end
33
+
34
+ # Returns the full schema name of the inheriting class.
35
+ # @return [String]
36
+ def full_schema
37
+ "#{namespace}.#{schema}"
38
+ end
39
+
40
+ # Returns the schema validator from the schema backend
41
+ # @return [Deimos::SchemaBackends::Base]
42
+ def validator
43
+ Deimos.schema_backend(schema: schema, namespace: namespace)
44
+ end
45
+
46
+ # @return [Array<String>] an array of fields names in the schema.
47
+ def schema_fields
48
+ validator.schema_fields.map(&:name)
49
+ end
50
+
51
+ # :nodoc:
52
+ def self.initialize_from_value(value)
53
+ return nil if value.nil?
54
+
55
+ value.is_a?(self) ? value : self.new(**value.symbolize_keys)
56
+ end
57
+ end
58
+ end
59
+ end
@@ -58,6 +58,11 @@ module Deimos
58
58
  config[:key_field] = field&.to_s
59
59
  config[:key_schema] = schema
60
60
  end
61
+
62
+ # @param enabled [Boolean]
63
+ def schema_class_config(use_schema_classes)
64
+ config[:use_schema_classes] = use_schema_classes
65
+ end
61
66
  end
62
67
  end
63
68
  end
@@ -18,22 +18,38 @@ module Deimos
18
18
  def sent_messages
19
19
  Deimos::Backends::Test.sent_messages
20
20
  end
21
- end
22
21
 
23
- included do
22
+ # Set the config to the right settings for a unit test
23
+ def unit_test!
24
+ Deimos.configure do |deimos_config|
25
+ deimos_config.logger = Logger.new(STDOUT)
26
+ deimos_config.consumers.reraise_errors = true
27
+ deimos_config.kafka.seed_brokers ||= ['test_broker']
28
+ deimos_config.schema.backend = Deimos.schema_backend_class.mock_backend
29
+ deimos_config.producers.backend = :test
30
+ end
31
+ end
24
32
 
25
- RSpec.configure do |config|
33
+ # Kafka test config with avro schema registry
34
+ def full_integration_test!
35
+ Deimos.configure do |deimos_config|
36
+ deimos_config.producers.backend = :kafka
37
+ deimos_config.schema.backend = :avro_schema_registry
38
+ end
39
+ end
26
40
 
27
- config.before(:suite) do
28
- Deimos.configure do |d_config|
29
- d_config.logger = Logger.new(STDOUT)
30
- d_config.consumers.reraise_errors = true
31
- d_config.kafka.seed_brokers ||= ['test_broker']
32
- d_config.schema.backend = Deimos.schema_backend_class.mock_backend
33
- d_config.producers.backend = :test
34
- end
41
+ # Set the config to the right settings for a kafka test
42
+ def kafka_test!
43
+ Deimos.configure do |deimos_config|
44
+ deimos_config.producers.backend = :kafka
45
+ deimos_config.schema.backend = :avro_validation
35
46
  end
47
+ end
48
+ end
36
49
 
50
+ included do
51
+
52
+ RSpec.configure do |config|
37
53
  config.prepend_before(:each) do
38
54
  client = double('client').as_null_object
39
55
  allow(client).to receive(:time) do |*_args, &block|
@@ -193,9 +209,11 @@ module Deimos
193
209
  'value' => payload)
194
210
 
195
211
  unless skip_expectation
196
- expectation = expect(handler).to receive(:consume).
197
- with(payload, anything, &block)
198
- expectation.and_call_original if call_original
212
+ _handler_expectation(:consume,
213
+ payload,
214
+ handler,
215
+ call_original,
216
+ &block)
199
217
  end
200
218
  Phobos::Actions::ProcessMessage.new(
201
219
  listener: listener,
@@ -260,9 +278,11 @@ module Deimos
260
278
  'partition' => 1,
261
279
  'offset_lag' => 0)
262
280
  unless skip_expectation
263
- expectation = expect(handler).to receive(:consume_batch).
264
- with(payloads, anything, &block)
265
- expectation.and_call_original if call_original
281
+ _handler_expectation(:consume_batch,
282
+ payloads,
283
+ handler,
284
+ call_original,
285
+ &block)
266
286
  end
267
287
  action = Phobos::Actions::ProcessBatchInline.new(
268
288
  listener: listener,
@@ -339,5 +359,38 @@ module Deimos
339
359
 
340
360
  handler.handler.constantize
341
361
  end
362
+
363
+ # Test that a given handler will execute a `method` on an `input` correctly,
364
+ # If a block is given, that block will be executed when `method` is called.
365
+ # Otherwise it will just confirm that `method` is called at all.
366
+ # @param method [Symbol]
367
+ # @param input [Object]
368
+ # @param handler [Deimos::Consumer]
369
+ # @param call_original [Boolean]
370
+ def _handler_expectation(method,
371
+ input,
372
+ handler,
373
+ call_original,
374
+ &block)
375
+ schema_class = handler.class.config[:schema]
376
+ expected = input.dup
377
+
378
+ config = handler.class.config
379
+ use_schema_classes = config[:use_schema_classes]
380
+ use_schema_classes = use_schema_classes.present? ? use_schema_classes : Deimos.config.schema.use_schema_classes
381
+
382
+ if use_schema_classes && schema_class.present?
383
+ expected = if input.is_a?(Array)
384
+ input.map do |payload|
385
+ Utils::SchemaClass.instance(payload, schema_class)
386
+ end
387
+ else
388
+ Utils::SchemaClass.instance(input, schema_class)
389
+ end
390
+ end
391
+
392
+ expectation = expect(handler).to receive(method).with(expected, anything, &block)
393
+ expectation.and_call_original if call_original
394
+ end
342
395
  end
343
396
  end
@@ -0,0 +1,29 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Deimos
4
+ module Utils
5
+ # Class used by SchemaClassGenerator and Consumer/Producer interfaces
6
+ module SchemaClass
7
+ class << self
8
+ # Converts a raw payload into an instance of the Schema Class
9
+ # @param payload [Hash]
10
+ # @param schema [String]
11
+ # @return [Deimos::SchemaClass::Record]
12
+ def instance(payload, schema)
13
+ klass = "Deimos::#{schema.underscore.camelize}".safe_constantize
14
+ return payload if klass.nil? || payload.nil?
15
+
16
+ klass.new(**payload.symbolize_keys)
17
+ end
18
+
19
+ # @param config [Hash] Producer or Consumer config
20
+ # @return [Boolean]
21
+ def use?(config)
22
+ use_schema_classes = config[:use_schema_classes]
23
+ use_schema_classes.present? ? use_schema_classes : Deimos.config.schema.use_schema_classes
24
+ end
25
+
26
+ end
27
+ end
28
+ end
29
+ end
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Deimos
4
- VERSION = '1.10.2'
4
+ VERSION = '1.12.0'
5
5
  end
data/lib/deimos.rb CHANGED
@@ -19,6 +19,9 @@ require 'deimos/backends/kafka_async'
19
19
  require 'deimos/backends/test'
20
20
 
21
21
  require 'deimos/schema_backends/base'
22
+ require 'deimos/utils/schema_class'
23
+ require 'deimos/schema_class/enum'
24
+ require 'deimos/schema_class/record'
22
25
 
23
26
  require 'deimos/monkey_patches/phobos_producer'
24
27
  require 'deimos/monkey_patches/phobos_cli'
@@ -0,0 +1,15 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This file is autogenerated by Deimos, Do NOT modify
4
+ module Schemas
5
+ <% if @sub_schema_templates.present? && @sub_schema_templates.any? -%>
6
+ ### Secondary Schema Classes ###
7
+ <% end -%>
8
+ <%- @sub_schema_templates.each do |schema_template| -%>
9
+ <%=- schema_template %>
10
+
11
+ <%- end -%>
12
+ ### Primary Schema Class ###
13
+ <%=- @main_class_definition -%>
14
+
15
+ end
@@ -0,0 +1,21 @@
1
+ # Autogenerated Schema for Enum at <%= @current_schema.namespace %>.<%= @current_schema.name %>
2
+ class <%= Deimos::SchemaBackends::AvroBase.schema_classname(@current_schema) %> < Deimos::SchemaClass::Enum
3
+ # @return ['<%= @current_schema.symbols.join("', '") %>']
4
+ attr_accessor :<%= @current_schema.name.underscore %>
5
+
6
+ # :nodoc:
7
+ def initialize(<%= @current_schema.name.underscore %>)
8
+ super
9
+ self.<%= @current_schema.name.underscore %> = <%= @current_schema.name.underscore %>
10
+ end
11
+
12
+ # @override
13
+ def symbols
14
+ %w(<%= @current_schema.symbols.join(' ') %>)
15
+ end
16
+
17
+ # @override
18
+ def to_h
19
+ @<%= @current_schema.name.underscore %>
20
+ end
21
+ end
@@ -0,0 +1,65 @@
1
+ # Autogenerated Schema for Record at <%= @current_schema.namespace %>.<%= @current_schema.name %>
2
+ class <%= Deimos::SchemaBackends::AvroBase.schema_classname(@current_schema) %> < Deimos::SchemaClass::Record
3
+ <%- if @field_assignments.select{ |h| h[:is_schema_class] }.any? -%>
4
+ ### Attribute Readers ###
5
+ <%- @field_assignments.select{ |h| h[:is_schema_class] }.each do |method_definition| -%>
6
+ # @return [<%= method_definition[:deimos_type] %>]
7
+ attr_reader :<%= method_definition[:field].name %>
8
+ <%- end -%>
9
+
10
+ <% end -%>
11
+ <%- if @field_assignments.select{ |h| !h[:is_schema_class] }.any? -%>
12
+ ### Attribute Accessors ###
13
+ <%- @field_assignments.select{ |h| !h[:is_schema_class] }.each do |method_definition| -%>
14
+ # @param <%= method_definition[:method_argument] %> [<%= method_definition[:deimos_type] %>]
15
+ attr_accessor :<%= method_definition[:field].name %>
16
+ <%- end -%>
17
+
18
+ <% end -%>
19
+ <%- if @field_assignments.select{ |h| h[:is_schema_class] }.any? -%>
20
+ ### Attribute Writers ###
21
+ <%- @field_assignments.select{ |h| h[:is_schema_class] }.each do |method_definition| -%>
22
+ # @param <%= method_definition[:method_argument] %> [<%= method_definition[:deimos_type] %>]
23
+ def <%= method_definition[:field].name %>=(<%= method_definition[:method_argument] %>)
24
+ <%- if method_definition[:field_type] == :array -%>
25
+ @<%= method_definition[:field].name %> = values.map do |value|
26
+ <%= method_definition[:field_initialization] %>
27
+ end
28
+ <%- elsif method_definition[:field_type] == :map -%>
29
+ @<%= method_definition[:field].name %> = values.transform_values do |value|
30
+ <%= method_definition[:field_initialization] %>
31
+ end
32
+ <%- else -%>
33
+ @<%= method_definition[:field].name %> = <%= method_definition[:field_initialization] %>
34
+ <%- end -%>
35
+ end
36
+
37
+ <%- end -%>
38
+ <% end -%>
39
+ # @override
40
+ <%= @initialization_definition %>
41
+ super
42
+ <%- @fields.each do |field| -%>
43
+ self.<%= field.name %> = <%= field.name %>
44
+ <% end -%>
45
+ end
46
+
47
+ # @override
48
+ def schema
49
+ '<%= @current_schema.name %>'
50
+ end
51
+
52
+ # @override
53
+ def namespace
54
+ '<%= @current_schema.namespace %>'
55
+ end
56
+
57
+ # @override
58
+ def to_h
59
+ {
60
+ <%- @fields.each do |field| -%>
61
+ <%= field_to_h(field) %>
62
+ <% end -%>
63
+ }
64
+ end
65
+ end