deimos-ruby 1.10.2 → 1.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +19 -0
- data/Gemfile.lock +8 -8
- data/README.md +150 -16
- data/deimos-ruby.gemspec +1 -1
- data/docs/CONFIGURATION.md +4 -0
- data/lib/deimos/active_record_consume/batch_consumption.rb +1 -1
- data/lib/deimos/active_record_consume/message_consumption.rb +4 -3
- data/lib/deimos/active_record_consumer.rb +2 -2
- data/lib/deimos/active_record_producer.rb +3 -0
- data/lib/deimos/config/configuration.rb +29 -0
- data/lib/deimos/consume/batch_consumption.rb +2 -2
- data/lib/deimos/consume/message_consumption.rb +2 -2
- data/lib/deimos/consumer.rb +10 -0
- data/lib/deimos/producer.rb +4 -3
- data/lib/deimos/schema_backends/avro_base.rb +64 -1
- data/lib/deimos/schema_backends/avro_schema_registry.rb +1 -1
- data/lib/deimos/schema_backends/base.rb +18 -2
- data/lib/deimos/schema_class/base.rb +67 -0
- data/lib/deimos/schema_class/enum.rb +24 -0
- data/lib/deimos/schema_class/record.rb +59 -0
- data/lib/deimos/shared_config.rb +5 -0
- data/lib/deimos/test_helpers.rb +70 -17
- data/lib/deimos/utils/schema_class.rb +29 -0
- data/lib/deimos/version.rb +1 -1
- data/lib/deimos.rb +3 -0
- data/lib/generators/deimos/schema_class/templates/schema_class.rb.tt +15 -0
- data/lib/generators/deimos/schema_class/templates/schema_enum.rb.tt +21 -0
- data/lib/generators/deimos/schema_class/templates/schema_record.rb.tt +65 -0
- data/lib/generators/deimos/schema_class_generator.rb +247 -0
- data/lib/tasks/deimos.rake +8 -0
- data/spec/active_record_batch_consumer_spec.rb +120 -110
- data/spec/active_record_consumer_spec.rb +97 -88
- data/spec/active_record_producer_spec.rb +38 -27
- data/spec/batch_consumer_spec.rb +37 -28
- data/spec/config/configuration_spec.rb +10 -3
- data/spec/consumer_spec.rb +94 -83
- data/spec/generators/active_record_generator_spec.rb +1 -0
- data/spec/generators/schema_class/my_schema_with_complex_types_spec.rb +206 -0
- data/spec/generators/schema_class_generator_spec.rb +186 -0
- data/spec/producer_spec.rb +110 -0
- data/spec/schema_classes/generated.rb +156 -0
- data/spec/schema_classes/my_nested_schema.rb +114 -0
- data/spec/schema_classes/my_schema.rb +53 -0
- data/spec/schema_classes/my_schema_key.rb +35 -0
- data/spec/schema_classes/my_schema_with_complex_types.rb +172 -0
- data/spec/schemas/com/my-namespace/Generated.avsc +6 -0
- data/spec/schemas/com/my-namespace/MySchemaWithComplexTypes.avsc +95 -0
- data/spec/spec_helper.rb +6 -1
- metadata +28 -4
@@ -3,15 +3,16 @@
|
|
3
3
|
module Deimos
|
4
4
|
# Represents a field in the schema.
|
5
5
|
class SchemaField
|
6
|
-
attr_accessor :name, :type, :enum_values
|
6
|
+
attr_accessor :name, :type, :enum_values, :default
|
7
7
|
|
8
8
|
# @param name [String]
|
9
9
|
# @param type [Object]
|
10
10
|
# @param enum_values [Array<String>]
|
11
|
-
def initialize(name, type, enum_values=[])
|
11
|
+
def initialize(name, type, enum_values=[], default=:no_default)
|
12
12
|
@name = name
|
13
13
|
@type = type
|
14
14
|
@enum_values = enum_values
|
15
|
+
@default = default
|
15
16
|
end
|
16
17
|
end
|
17
18
|
|
@@ -43,6 +44,7 @@ module Deimos
|
|
43
44
|
# @return [Hash,nil]
|
44
45
|
def decode(payload, schema: nil)
|
45
46
|
return nil if payload.nil?
|
47
|
+
|
46
48
|
decode_payload(payload, schema: schema || @schema)
|
47
49
|
end
|
48
50
|
|
@@ -78,6 +80,14 @@ module Deimos
|
|
78
80
|
raise NotImplementedError
|
79
81
|
end
|
80
82
|
|
83
|
+
# Converts your schema to String form for generated YARD docs.
|
84
|
+
# To be defined by subclass.
|
85
|
+
# @param schema [Object]
|
86
|
+
# @return [String] A string representation of the Type
|
87
|
+
def self.field_type(schema)
|
88
|
+
raise NotImplementedError
|
89
|
+
end
|
90
|
+
|
81
91
|
# Encode a payload. To be defined by subclass.
|
82
92
|
# @param payload [Hash]
|
83
93
|
# @param schema [Symbol|String]
|
@@ -145,6 +155,12 @@ module Deimos
|
|
145
155
|
def decode_key(_payload, _key_id)
|
146
156
|
raise NotImplementedError
|
147
157
|
end
|
158
|
+
|
159
|
+
# Forcefully loads the schema into memory.
|
160
|
+
# @return [Object] The schema that is of use.
|
161
|
+
def load_schema
|
162
|
+
raise NotImplementedError
|
163
|
+
end
|
148
164
|
end
|
149
165
|
end
|
150
166
|
end
|
@@ -0,0 +1,67 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'json'
|
4
|
+
|
5
|
+
module Deimos
|
6
|
+
module SchemaClass
|
7
|
+
# Base Class for Schema Classes generated from Avro.
|
8
|
+
class Base
|
9
|
+
|
10
|
+
# :nodoc:
|
11
|
+
def initialize(*_args)
|
12
|
+
end
|
13
|
+
|
14
|
+
# Converts the object to a string that represents a JSON object
|
15
|
+
# @return [String] a JSON string
|
16
|
+
def to_json(*_args)
|
17
|
+
to_h.to_json
|
18
|
+
end
|
19
|
+
|
20
|
+
# Converts the object to a hash which can be used for debugging.
|
21
|
+
# @return [Hash] a hash representation of the payload
|
22
|
+
def as_json(_opts={})
|
23
|
+
JSON.parse(to_json)
|
24
|
+
end
|
25
|
+
|
26
|
+
# Converts the object attributes to a hash which can be used for Kafka
|
27
|
+
# @return [Hash] the payload as a hash.
|
28
|
+
def to_h
|
29
|
+
raise NotImplementedError
|
30
|
+
end
|
31
|
+
|
32
|
+
# :nodoc:
|
33
|
+
def ==(other)
|
34
|
+
comparison = other
|
35
|
+
if other.class == self.class
|
36
|
+
comparison = other.state
|
37
|
+
end
|
38
|
+
|
39
|
+
comparison == self.state
|
40
|
+
end
|
41
|
+
|
42
|
+
# :nodoc:
|
43
|
+
def to_s
|
44
|
+
klass = self.class
|
45
|
+
"#{klass}(#{self.as_json.symbolize_keys.to_s[1..-2]})"
|
46
|
+
end
|
47
|
+
|
48
|
+
# Initializes this class from a given value
|
49
|
+
# @param value [Object]
|
50
|
+
def self.initialize_from_value(value)
|
51
|
+
raise NotImplementedError
|
52
|
+
end
|
53
|
+
|
54
|
+
protected
|
55
|
+
|
56
|
+
# :nodoc:
|
57
|
+
def state
|
58
|
+
as_json
|
59
|
+
end
|
60
|
+
|
61
|
+
# :nodoc:
|
62
|
+
def hash
|
63
|
+
state.hash
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative 'base'
|
4
|
+
require 'json'
|
5
|
+
|
6
|
+
module Deimos
|
7
|
+
module SchemaClass
|
8
|
+
# Base Class for Enum Classes generated from Avro.
|
9
|
+
class Enum < Base
|
10
|
+
# Returns all the valid symbols for this enum.
|
11
|
+
# @return [Array<String>]
|
12
|
+
def symbols
|
13
|
+
raise NotImplementedError
|
14
|
+
end
|
15
|
+
|
16
|
+
# :nodoc:
|
17
|
+
def self.initialize_from_value(value)
|
18
|
+
return nil if value.nil?
|
19
|
+
|
20
|
+
value.is_a?(self) ? value : self.new(value)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,59 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative 'base'
|
4
|
+
require 'json'
|
5
|
+
|
6
|
+
module Deimos
|
7
|
+
module SchemaClass
|
8
|
+
# Base Class of Record Classes generated from Avro.
|
9
|
+
class Record < Base
|
10
|
+
# Element access method as if this Object were a hash
|
11
|
+
# @param key[String||Symbol]
|
12
|
+
# @return [Object] The value of the attribute if exists, nil otherwise
|
13
|
+
def [](key)
|
14
|
+
self.try(key.to_sym)
|
15
|
+
end
|
16
|
+
|
17
|
+
# :nodoc
|
18
|
+
def with_indifferent_access
|
19
|
+
self
|
20
|
+
end
|
21
|
+
|
22
|
+
# Returns the schema name of the inheriting class.
|
23
|
+
# @return [String]
|
24
|
+
def schema
|
25
|
+
raise NotImplementedError
|
26
|
+
end
|
27
|
+
|
28
|
+
# Returns the namespace for the schema of the inheriting class.
|
29
|
+
# @return [String]
|
30
|
+
def namespace
|
31
|
+
raise NotImplementedError
|
32
|
+
end
|
33
|
+
|
34
|
+
# Returns the full schema name of the inheriting class.
|
35
|
+
# @return [String]
|
36
|
+
def full_schema
|
37
|
+
"#{namespace}.#{schema}"
|
38
|
+
end
|
39
|
+
|
40
|
+
# Returns the schema validator from the schema backend
|
41
|
+
# @return [Deimos::SchemaBackends::Base]
|
42
|
+
def validator
|
43
|
+
Deimos.schema_backend(schema: schema, namespace: namespace)
|
44
|
+
end
|
45
|
+
|
46
|
+
# @return [Array<String>] an array of fields names in the schema.
|
47
|
+
def schema_fields
|
48
|
+
validator.schema_fields.map(&:name)
|
49
|
+
end
|
50
|
+
|
51
|
+
# :nodoc:
|
52
|
+
def self.initialize_from_value(value)
|
53
|
+
return nil if value.nil?
|
54
|
+
|
55
|
+
value.is_a?(self) ? value : self.new(**value.symbolize_keys)
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
data/lib/deimos/shared_config.rb
CHANGED
data/lib/deimos/test_helpers.rb
CHANGED
@@ -18,22 +18,38 @@ module Deimos
|
|
18
18
|
def sent_messages
|
19
19
|
Deimos::Backends::Test.sent_messages
|
20
20
|
end
|
21
|
-
end
|
22
21
|
|
23
|
-
|
22
|
+
# Set the config to the right settings for a unit test
|
23
|
+
def unit_test!
|
24
|
+
Deimos.configure do |deimos_config|
|
25
|
+
deimos_config.logger = Logger.new(STDOUT)
|
26
|
+
deimos_config.consumers.reraise_errors = true
|
27
|
+
deimos_config.kafka.seed_brokers ||= ['test_broker']
|
28
|
+
deimos_config.schema.backend = Deimos.schema_backend_class.mock_backend
|
29
|
+
deimos_config.producers.backend = :test
|
30
|
+
end
|
31
|
+
end
|
24
32
|
|
25
|
-
|
33
|
+
# Kafka test config with avro schema registry
|
34
|
+
def full_integration_test!
|
35
|
+
Deimos.configure do |deimos_config|
|
36
|
+
deimos_config.producers.backend = :kafka
|
37
|
+
deimos_config.schema.backend = :avro_schema_registry
|
38
|
+
end
|
39
|
+
end
|
26
40
|
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
d_config.schema.backend = Deimos.schema_backend_class.mock_backend
|
33
|
-
d_config.producers.backend = :test
|
34
|
-
end
|
41
|
+
# Set the config to the right settings for a kafka test
|
42
|
+
def kafka_test!
|
43
|
+
Deimos.configure do |deimos_config|
|
44
|
+
deimos_config.producers.backend = :kafka
|
45
|
+
deimos_config.schema.backend = :avro_validation
|
35
46
|
end
|
47
|
+
end
|
48
|
+
end
|
36
49
|
|
50
|
+
included do
|
51
|
+
|
52
|
+
RSpec.configure do |config|
|
37
53
|
config.prepend_before(:each) do
|
38
54
|
client = double('client').as_null_object
|
39
55
|
allow(client).to receive(:time) do |*_args, &block|
|
@@ -193,9 +209,11 @@ module Deimos
|
|
193
209
|
'value' => payload)
|
194
210
|
|
195
211
|
unless skip_expectation
|
196
|
-
|
197
|
-
|
198
|
-
|
212
|
+
_handler_expectation(:consume,
|
213
|
+
payload,
|
214
|
+
handler,
|
215
|
+
call_original,
|
216
|
+
&block)
|
199
217
|
end
|
200
218
|
Phobos::Actions::ProcessMessage.new(
|
201
219
|
listener: listener,
|
@@ -260,9 +278,11 @@ module Deimos
|
|
260
278
|
'partition' => 1,
|
261
279
|
'offset_lag' => 0)
|
262
280
|
unless skip_expectation
|
263
|
-
|
264
|
-
|
265
|
-
|
281
|
+
_handler_expectation(:consume_batch,
|
282
|
+
payloads,
|
283
|
+
handler,
|
284
|
+
call_original,
|
285
|
+
&block)
|
266
286
|
end
|
267
287
|
action = Phobos::Actions::ProcessBatchInline.new(
|
268
288
|
listener: listener,
|
@@ -339,5 +359,38 @@ module Deimos
|
|
339
359
|
|
340
360
|
handler.handler.constantize
|
341
361
|
end
|
362
|
+
|
363
|
+
# Test that a given handler will execute a `method` on an `input` correctly,
|
364
|
+
# If a block is given, that block will be executed when `method` is called.
|
365
|
+
# Otherwise it will just confirm that `method` is called at all.
|
366
|
+
# @param method [Symbol]
|
367
|
+
# @param input [Object]
|
368
|
+
# @param handler [Deimos::Consumer]
|
369
|
+
# @param call_original [Boolean]
|
370
|
+
def _handler_expectation(method,
|
371
|
+
input,
|
372
|
+
handler,
|
373
|
+
call_original,
|
374
|
+
&block)
|
375
|
+
schema_class = handler.class.config[:schema]
|
376
|
+
expected = input.dup
|
377
|
+
|
378
|
+
config = handler.class.config
|
379
|
+
use_schema_classes = config[:use_schema_classes]
|
380
|
+
use_schema_classes = use_schema_classes.present? ? use_schema_classes : Deimos.config.schema.use_schema_classes
|
381
|
+
|
382
|
+
if use_schema_classes && schema_class.present?
|
383
|
+
expected = if input.is_a?(Array)
|
384
|
+
input.map do |payload|
|
385
|
+
Utils::SchemaClass.instance(payload, schema_class)
|
386
|
+
end
|
387
|
+
else
|
388
|
+
Utils::SchemaClass.instance(input, schema_class)
|
389
|
+
end
|
390
|
+
end
|
391
|
+
|
392
|
+
expectation = expect(handler).to receive(method).with(expected, anything, &block)
|
393
|
+
expectation.and_call_original if call_original
|
394
|
+
end
|
342
395
|
end
|
343
396
|
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Deimos
|
4
|
+
module Utils
|
5
|
+
# Class used by SchemaClassGenerator and Consumer/Producer interfaces
|
6
|
+
module SchemaClass
|
7
|
+
class << self
|
8
|
+
# Converts a raw payload into an instance of the Schema Class
|
9
|
+
# @param payload [Hash]
|
10
|
+
# @param schema [String]
|
11
|
+
# @return [Deimos::SchemaClass::Record]
|
12
|
+
def instance(payload, schema)
|
13
|
+
klass = "Deimos::#{schema.underscore.camelize}".safe_constantize
|
14
|
+
return payload if klass.nil? || payload.nil?
|
15
|
+
|
16
|
+
klass.new(**payload.symbolize_keys)
|
17
|
+
end
|
18
|
+
|
19
|
+
# @param config [Hash] Producer or Consumer config
|
20
|
+
# @return [Boolean]
|
21
|
+
def use?(config)
|
22
|
+
use_schema_classes = config[:use_schema_classes]
|
23
|
+
use_schema_classes.present? ? use_schema_classes : Deimos.config.schema.use_schema_classes
|
24
|
+
end
|
25
|
+
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
data/lib/deimos/version.rb
CHANGED
data/lib/deimos.rb
CHANGED
@@ -19,6 +19,9 @@ require 'deimos/backends/kafka_async'
|
|
19
19
|
require 'deimos/backends/test'
|
20
20
|
|
21
21
|
require 'deimos/schema_backends/base'
|
22
|
+
require 'deimos/utils/schema_class'
|
23
|
+
require 'deimos/schema_class/enum'
|
24
|
+
require 'deimos/schema_class/record'
|
22
25
|
|
23
26
|
require 'deimos/monkey_patches/phobos_producer'
|
24
27
|
require 'deimos/monkey_patches/phobos_cli'
|
@@ -0,0 +1,15 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# This file is autogenerated by Deimos, Do NOT modify
|
4
|
+
module Schemas
|
5
|
+
<% if @sub_schema_templates.present? && @sub_schema_templates.any? -%>
|
6
|
+
### Secondary Schema Classes ###
|
7
|
+
<% end -%>
|
8
|
+
<%- @sub_schema_templates.each do |schema_template| -%>
|
9
|
+
<%=- schema_template %>
|
10
|
+
|
11
|
+
<%- end -%>
|
12
|
+
### Primary Schema Class ###
|
13
|
+
<%=- @main_class_definition -%>
|
14
|
+
|
15
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
# Autogenerated Schema for Enum at <%= @current_schema.namespace %>.<%= @current_schema.name %>
|
2
|
+
class <%= Deimos::SchemaBackends::AvroBase.schema_classname(@current_schema) %> < Deimos::SchemaClass::Enum
|
3
|
+
# @return ['<%= @current_schema.symbols.join("', '") %>']
|
4
|
+
attr_accessor :<%= @current_schema.name.underscore %>
|
5
|
+
|
6
|
+
# :nodoc:
|
7
|
+
def initialize(<%= @current_schema.name.underscore %>)
|
8
|
+
super
|
9
|
+
self.<%= @current_schema.name.underscore %> = <%= @current_schema.name.underscore %>
|
10
|
+
end
|
11
|
+
|
12
|
+
# @override
|
13
|
+
def symbols
|
14
|
+
%w(<%= @current_schema.symbols.join(' ') %>)
|
15
|
+
end
|
16
|
+
|
17
|
+
# @override
|
18
|
+
def to_h
|
19
|
+
@<%= @current_schema.name.underscore %>
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,65 @@
|
|
1
|
+
# Autogenerated Schema for Record at <%= @current_schema.namespace %>.<%= @current_schema.name %>
|
2
|
+
class <%= Deimos::SchemaBackends::AvroBase.schema_classname(@current_schema) %> < Deimos::SchemaClass::Record
|
3
|
+
<%- if @field_assignments.select{ |h| h[:is_schema_class] }.any? -%>
|
4
|
+
### Attribute Readers ###
|
5
|
+
<%- @field_assignments.select{ |h| h[:is_schema_class] }.each do |method_definition| -%>
|
6
|
+
# @return [<%= method_definition[:deimos_type] %>]
|
7
|
+
attr_reader :<%= method_definition[:field].name %>
|
8
|
+
<%- end -%>
|
9
|
+
|
10
|
+
<% end -%>
|
11
|
+
<%- if @field_assignments.select{ |h| !h[:is_schema_class] }.any? -%>
|
12
|
+
### Attribute Accessors ###
|
13
|
+
<%- @field_assignments.select{ |h| !h[:is_schema_class] }.each do |method_definition| -%>
|
14
|
+
# @param <%= method_definition[:method_argument] %> [<%= method_definition[:deimos_type] %>]
|
15
|
+
attr_accessor :<%= method_definition[:field].name %>
|
16
|
+
<%- end -%>
|
17
|
+
|
18
|
+
<% end -%>
|
19
|
+
<%- if @field_assignments.select{ |h| h[:is_schema_class] }.any? -%>
|
20
|
+
### Attribute Writers ###
|
21
|
+
<%- @field_assignments.select{ |h| h[:is_schema_class] }.each do |method_definition| -%>
|
22
|
+
# @param <%= method_definition[:method_argument] %> [<%= method_definition[:deimos_type] %>]
|
23
|
+
def <%= method_definition[:field].name %>=(<%= method_definition[:method_argument] %>)
|
24
|
+
<%- if method_definition[:field_type] == :array -%>
|
25
|
+
@<%= method_definition[:field].name %> = values.map do |value|
|
26
|
+
<%= method_definition[:field_initialization] %>
|
27
|
+
end
|
28
|
+
<%- elsif method_definition[:field_type] == :map -%>
|
29
|
+
@<%= method_definition[:field].name %> = values.transform_values do |value|
|
30
|
+
<%= method_definition[:field_initialization] %>
|
31
|
+
end
|
32
|
+
<%- else -%>
|
33
|
+
@<%= method_definition[:field].name %> = <%= method_definition[:field_initialization] %>
|
34
|
+
<%- end -%>
|
35
|
+
end
|
36
|
+
|
37
|
+
<%- end -%>
|
38
|
+
<% end -%>
|
39
|
+
# @override
|
40
|
+
<%= @initialization_definition %>
|
41
|
+
super
|
42
|
+
<%- @fields.each do |field| -%>
|
43
|
+
self.<%= field.name %> = <%= field.name %>
|
44
|
+
<% end -%>
|
45
|
+
end
|
46
|
+
|
47
|
+
# @override
|
48
|
+
def schema
|
49
|
+
'<%= @current_schema.name %>'
|
50
|
+
end
|
51
|
+
|
52
|
+
# @override
|
53
|
+
def namespace
|
54
|
+
'<%= @current_schema.namespace %>'
|
55
|
+
end
|
56
|
+
|
57
|
+
# @override
|
58
|
+
def to_h
|
59
|
+
{
|
60
|
+
<%- @fields.each do |field| -%>
|
61
|
+
<%= field_to_h(field) %>
|
62
|
+
<% end -%>
|
63
|
+
}
|
64
|
+
end
|
65
|
+
end
|