deimos-ruby 1.11.2 → 1.12.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +9 -2
- data/Gemfile.lock +8 -8
- data/README.md +96 -0
- data/deimos-ruby.gemspec +1 -1
- data/docs/CONFIGURATION.md +4 -0
- data/lib/deimos/active_record_consume/batch_consumption.rb +7 -9
- data/lib/deimos/active_record_consume/message_consumption.rb +4 -3
- data/lib/deimos/active_record_consumer.rb +2 -2
- data/lib/deimos/active_record_producer.rb +3 -0
- data/lib/deimos/config/configuration.rb +29 -0
- data/lib/deimos/consume/batch_consumption.rb +2 -2
- data/lib/deimos/consume/message_consumption.rb +2 -2
- data/lib/deimos/consumer.rb +10 -0
- data/lib/deimos/producer.rb +4 -3
- data/lib/deimos/schema_backends/avro_base.rb +64 -1
- data/lib/deimos/schema_backends/avro_schema_registry.rb +1 -1
- data/lib/deimos/schema_backends/base.rb +18 -2
- data/lib/deimos/schema_class/base.rb +67 -0
- data/lib/deimos/schema_class/enum.rb +24 -0
- data/lib/deimos/schema_class/record.rb +59 -0
- data/lib/deimos/shared_config.rb +5 -0
- data/lib/deimos/test_helpers.rb +43 -7
- data/lib/deimos/utils/schema_class.rb +29 -0
- data/lib/deimos/version.rb +1 -1
- data/lib/deimos.rb +3 -0
- data/lib/generators/deimos/schema_class/templates/schema_class.rb.tt +15 -0
- data/lib/generators/deimos/schema_class/templates/schema_enum.rb.tt +21 -0
- data/lib/generators/deimos/schema_class/templates/schema_record.rb.tt +65 -0
- data/lib/generators/deimos/schema_class_generator.rb +247 -0
- data/lib/tasks/deimos.rake +8 -0
- data/spec/active_record_batch_consumer_spec.rb +120 -110
- data/spec/active_record_consumer_spec.rb +97 -88
- data/spec/active_record_producer_spec.rb +38 -27
- data/spec/batch_consumer_spec.rb +37 -28
- data/spec/config/configuration_spec.rb +10 -3
- data/spec/consumer_spec.rb +94 -83
- data/spec/generators/active_record_generator_spec.rb +1 -0
- data/spec/generators/schema_class/my_schema_with_complex_types_spec.rb +206 -0
- data/spec/generators/schema_class_generator_spec.rb +186 -0
- data/spec/producer_spec.rb +110 -0
- data/spec/schema_classes/generated.rb +156 -0
- data/spec/schema_classes/my_nested_schema.rb +114 -0
- data/spec/schema_classes/my_schema.rb +53 -0
- data/spec/schema_classes/my_schema_key.rb +35 -0
- data/spec/schema_classes/my_schema_with_complex_types.rb +172 -0
- data/spec/schemas/com/my-namespace/Generated.avsc +6 -0
- data/spec/schemas/com/my-namespace/MySchemaWithComplexTypes.avsc +95 -0
- data/spec/spec_helper.rb +6 -1
- metadata +28 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b7d6f4f732c60937fcc4e7c2834e59d7a225defbe930bfc37ac02729ddc7e092
|
4
|
+
data.tar.gz: e32bbddd1e6d01ebd824b8916fc40c2eac467966e4a154875e90e9f1ab50528e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1d1b3ba943c5ac49d87f5d4f77d22d88d0e5ff5467eef5e040acb2f5f7af0ae3f4d042d8156e5a8e413c394496a6092c7a691ddb840e366bb23617f48103bd62
|
7
|
+
data.tar.gz: ba8fe17bed6e62f86d87ae54b81a8a3cb57efe7fc06586166d1a59d92b87640e19f74213724f3b7404846975d77b0b016127fa8a846a4de270a1c8192f9a3344
|
data/CHANGELOG.md
CHANGED
@@ -7,10 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
7
7
|
|
8
8
|
## UNRELEASED
|
9
9
|
|
10
|
-
|
10
|
+
# 1.12.0 - 2021-11-01
|
11
|
+
|
12
|
+
### Features :star:
|
13
|
+
|
14
|
+
- Generate Schema classes from Avro Schemas
|
15
|
+
- Use Schema Classes in your consumer and producer
|
16
|
+
|
17
|
+
## 1.11.0 - 2021-08-27
|
11
18
|
|
12
19
|
- ### Fixes :wrench:
|
13
|
-
- Fixed ActiveRecord batch consumption
|
20
|
+
- Fixed issue where ActiveRecord batch consumption could fail when decoding keys.
|
14
21
|
|
15
22
|
- ### Roadmap :car:
|
16
23
|
- TestHelper does not automatically reset Deimos config before each test. [#120](https://github.com/flipp-oss/deimos/pull/120).
|
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
deimos-ruby (1.11.
|
4
|
+
deimos-ruby (1.11.0)
|
5
5
|
avro_turf (~> 0.11)
|
6
6
|
fig_tree (~> 0.0.2)
|
7
7
|
phobos (>= 1.9, < 3.0)
|
@@ -88,11 +88,11 @@ GEM
|
|
88
88
|
ddtrace (0.46.0)
|
89
89
|
msgpack
|
90
90
|
diff-lcs (1.4.4)
|
91
|
-
digest-crc (0.6.
|
91
|
+
digest-crc (0.6.4)
|
92
92
|
rake (>= 12.0.0, < 14.0.0)
|
93
93
|
dogstatsd-ruby (4.8.3)
|
94
94
|
erubi (1.10.0)
|
95
|
-
excon (0.
|
95
|
+
excon (0.85.0)
|
96
96
|
exponential-backoff (0.0.4)
|
97
97
|
ffi (1.15.0)
|
98
98
|
fig_tree (0.0.2)
|
@@ -153,10 +153,10 @@ GEM
|
|
153
153
|
nenv (~> 0.1)
|
154
154
|
shellany (~> 0.0)
|
155
155
|
parallel (1.20.1)
|
156
|
-
parser (3.0.
|
156
|
+
parser (3.0.1.1)
|
157
157
|
ast (~> 2.4.1)
|
158
158
|
pg (1.2.3)
|
159
|
-
phobos (2.0
|
159
|
+
phobos (2.1.0)
|
160
160
|
activesupport (>= 3.0.0)
|
161
161
|
concurrent-ruby (>= 1.0.2)
|
162
162
|
concurrent-ruby-ext (>= 1.0.2)
|
@@ -227,7 +227,7 @@ GEM
|
|
227
227
|
rspec-support (3.10.2)
|
228
228
|
rspec_junit_formatter (0.4.1)
|
229
229
|
rspec-core (>= 2, < 4, != 2.12.0)
|
230
|
-
rubocop (0.
|
230
|
+
rubocop (0.89.0)
|
231
231
|
parallel (~> 1.10)
|
232
232
|
parser (>= 2.7.1.1)
|
233
233
|
rainbow (>= 2.2.2, < 4.0)
|
@@ -240,7 +240,7 @@ GEM
|
|
240
240
|
parser (>= 2.7.1.5)
|
241
241
|
rubocop-rspec (1.42.0)
|
242
242
|
rubocop (>= 0.87.0)
|
243
|
-
ruby-kafka (1.
|
243
|
+
ruby-kafka (1.4.0)
|
244
244
|
digest-crc
|
245
245
|
ruby-progressbar (1.11.0)
|
246
246
|
shellany (0.0.1)
|
@@ -284,7 +284,7 @@ DEPENDENCIES
|
|
284
284
|
rspec (~> 3)
|
285
285
|
rspec-rails (~> 4)
|
286
286
|
rspec_junit_formatter (~> 0.3)
|
287
|
-
rubocop (= 0.
|
287
|
+
rubocop (= 0.89.0)
|
288
288
|
rubocop-rspec (= 1.42.0)
|
289
289
|
sqlite3 (~> 1.3)
|
290
290
|
|
data/README.md
CHANGED
@@ -27,6 +27,7 @@ Built on Phobos and hence Ruby-Kafka.
|
|
27
27
|
* [Database Backend](#database-backend)
|
28
28
|
* [Database Poller](#database-poller)
|
29
29
|
* [Running Consumers](#running-consumers)
|
30
|
+
* [Generated Schema Classes](#generated-schema-classes)
|
30
31
|
* [Metrics](#metrics)
|
31
32
|
* [Testing](#testing)
|
32
33
|
* [Test Helpers](#test-helpers)
|
@@ -795,6 +796,101 @@ which can be useful if you want to figure out if you're inside the task
|
|
795
796
|
as opposed to running your Rails server or console. E.g. you could start your
|
796
797
|
DB backend only when your rake task is running.
|
797
798
|
|
799
|
+
## Generated Schema Classes
|
800
|
+
|
801
|
+
Deimos offers a way to generate classes from Avro schemas. These classes are documented
|
802
|
+
with YARD to aid in IDE auto-complete, and will help to move errors closer to the code.
|
803
|
+
|
804
|
+
Add the following configurations for schema class generation:
|
805
|
+
|
806
|
+
```ruby
|
807
|
+
config.schema.generated_class_path 'path/to/generated/classes' # Defaults to 'app/lib/schema_classes'
|
808
|
+
```
|
809
|
+
|
810
|
+
Run the following command to generate schema classes in your application. It will generate classes for every configured consumer or producer by `Deimos.configure`:
|
811
|
+
|
812
|
+
bundle exec rake deimos:generate_schema_classes
|
813
|
+
|
814
|
+
Add the following configurations to start using generated schema classes in your application's Consumers and Producers:
|
815
|
+
|
816
|
+
config.schema.use_schema_classes true
|
817
|
+
|
818
|
+
Additionally, you can enable or disable the usage of schema classes for a particular consumer or producer with the
|
819
|
+
`use_schema_classes` config. See [Configuration](./docs/CONFIGURATION.md#defining-producers).
|
820
|
+
|
821
|
+
### Consumer
|
822
|
+
|
823
|
+
The consumer interface uses the `decode_message` method to turn JSON hash into the Schemas
|
824
|
+
generated Class and provides it to the `consume`/`consume_batch` methods for their use.
|
825
|
+
|
826
|
+
Examples of consumers would look like this:
|
827
|
+
```ruby
|
828
|
+
class MyConsumer < Deimos::Consumer
|
829
|
+
def consume(payload, metadata)
|
830
|
+
# Same method as Phobos consumers but payload is now an instance of Deimos::SchemaClass::Record
|
831
|
+
# rather than a hash. metadata is still a hash that contains information like :key and :topic.
|
832
|
+
# You can interact with the schema class instance in the following way:
|
833
|
+
do_something(payload.test_id, payload.some_int)
|
834
|
+
# The original behaviour was as follows:
|
835
|
+
do_something(payload[:test_id], payload[:some_int])
|
836
|
+
end
|
837
|
+
end
|
838
|
+
```
|
839
|
+
|
840
|
+
```ruby
|
841
|
+
class MyActiveRecordConsumer < Deimos::ActiveRecordConsumer
|
842
|
+
record_class Widget
|
843
|
+
# Any method that expects a message payload as a hash will instead
|
844
|
+
# receive an instance of Deimos::SchemaClass::Record.
|
845
|
+
def record_attributes(payload, key)
|
846
|
+
# You can interact with the schema class instance in the following way:
|
847
|
+
super.merge(:some_field => "some_value-#{payload.test_id}")
|
848
|
+
# The original behaviour was as follows:
|
849
|
+
super.merge(:some_field => "some_value-#{payload[:test_id]}")
|
850
|
+
end
|
851
|
+
end
|
852
|
+
```
|
853
|
+
|
854
|
+
### Producer
|
855
|
+
Similarly to the consumer interface, the producer interface for using Schema Classes in your app
|
856
|
+
relies on the `publish`/`publish_list` methods to convert a _provided_ instance of a Schema Class
|
857
|
+
into a hash that can be used freely by the Kafka client.
|
858
|
+
|
859
|
+
Examples of producers would look like this:
|
860
|
+
```ruby
|
861
|
+
class MyProducer < Deimos::Producer
|
862
|
+
class << self
|
863
|
+
# @param test_id [String]
|
864
|
+
# @param some_int [Integer]
|
865
|
+
def self.send_a_message(test_id, some_int)
|
866
|
+
# Instead of sending in a Hash object to the publish or publish_list method,
|
867
|
+
# you can initialize an instance of your schema class and send that in.
|
868
|
+
message = Schemas::MySchema.new(
|
869
|
+
test_id: test_id,
|
870
|
+
some_int: some_int
|
871
|
+
)
|
872
|
+
self.publish(message)
|
873
|
+
self.publish_list([message])
|
874
|
+
end
|
875
|
+
end
|
876
|
+
end
|
877
|
+
```
|
878
|
+
|
879
|
+
```ruby
|
880
|
+
class MyActiveRecordProducer < Deimos::ActiveRecordProducer
|
881
|
+
record_class Widget
|
882
|
+
# @param payload [Deimos::SchemaClass::Record]
|
883
|
+
# @param _record [Widget]
|
884
|
+
def self.generate_payload(attributes, _record)
|
885
|
+
# This method converts your ActiveRecord into a Deimos::SchemaClass::Record. You will be able to use super
|
886
|
+
# as an instance of Schemas::MySchema and set values that are not on your ActiveRecord schema.
|
887
|
+
res = super
|
888
|
+
res.some_value = "some_value-#{res.test_id}"
|
889
|
+
res
|
890
|
+
end
|
891
|
+
end
|
892
|
+
```
|
893
|
+
|
798
894
|
# Metrics
|
799
895
|
|
800
896
|
Deimos includes some metrics reporting out the box. It ships with DataDog support, but you can add custom metric providers as well.
|
data/deimos-ruby.gemspec
CHANGED
@@ -39,7 +39,7 @@ Gem::Specification.new do |spec|
|
|
39
39
|
spec.add_development_dependency('rspec', '~> 3')
|
40
40
|
spec.add_development_dependency('rspec_junit_formatter', '~>0.3')
|
41
41
|
spec.add_development_dependency('rspec-rails', '~> 4')
|
42
|
-
spec.add_development_dependency('rubocop', '0.
|
42
|
+
spec.add_development_dependency('rubocop', '0.89.0')
|
43
43
|
spec.add_development_dependency('rubocop-rspec', '1.42.0')
|
44
44
|
spec.add_development_dependency('sqlite3', '~> 1.3')
|
45
45
|
end
|
data/docs/CONFIGURATION.md
CHANGED
@@ -44,6 +44,7 @@ topic|nil|Topic to produce to.
|
|
44
44
|
schema|nil|Name of the schema to use to encode data before producing.
|
45
45
|
namespace|nil|Namespace of the schema to use when finding it locally.
|
46
46
|
key_config|nil|Configuration hash for message keys. See [Kafka Message Keys](../README.md#installation)
|
47
|
+
use_schema_classes|nil|Set to true or false to enable or disable using the producers schema classes. See [Generated Schema Classes](../README.md#generated-schema-classes)
|
47
48
|
|
48
49
|
## Defining Consumers
|
49
50
|
|
@@ -81,6 +82,7 @@ namespace|nil|Namespace of the schema to use when finding it locally.
|
|
81
82
|
key_config|nil|Configuration hash for message keys. See [Kafka Message Keys](../README.md#installation)
|
82
83
|
disabled|false|Set to true to skip starting an actual listener for this consumer on startup.
|
83
84
|
group_id|nil|ID of the consumer group.
|
85
|
+
use_schema_classes|nil|Set to true or false to enable or disable using the consumers schema classes. See [Generated Schema Classes](../README.md#generated-schema-classes)
|
84
86
|
max_concurrency|1|Number of threads created for this listener. Each thread will behave as an independent consumer. They don't share any state.
|
85
87
|
start_from_beginning|true|Once the consumer group has checkpointed its progress in the topic's partitions, the consumers will always start from the checkpointed offsets, regardless of config. As such, this setting only applies when the consumer initially starts consuming from a topic
|
86
88
|
max_bytes_per_partition|512.kilobytes|Maximum amount of data fetched from a single partition at a time.
|
@@ -176,6 +178,8 @@ Config name|Default|Description
|
|
176
178
|
schema.backend|`:mock`|Backend representing the schema encoder/decoder. You can see a full list [here](../lib/deimos/schema_backends).
|
177
179
|
schema.registry_url|`http://localhost:8081`|URL of the Confluent schema registry.
|
178
180
|
schema.path|nil|Local path to find your schemas.
|
181
|
+
schema.use_schema_classes|false|Set this to true to use generated schema classes in your application.
|
182
|
+
schema.generated_schema_path|`app/lib/schema_classes`|Local path to generated schema classes.
|
179
183
|
|
180
184
|
## Database Producer Configuration
|
181
185
|
|
@@ -15,7 +15,7 @@ module Deimos
|
|
15
15
|
# If two messages in a batch have the same key, we cannot process them
|
16
16
|
# in the same operation as they would interfere with each other. Thus
|
17
17
|
# they are split
|
18
|
-
# @param payloads [Array<Hash>] Decoded payloads
|
18
|
+
# @param payloads [Array<Hash|Deimos::SchemaClass::Record>] Decoded payloads
|
19
19
|
# @param metadata [Hash] Information about batch, including keys.
|
20
20
|
def consume_batch(payloads, metadata)
|
21
21
|
messages = payloads.
|
@@ -44,12 +44,14 @@ module Deimos
|
|
44
44
|
# @param key [String] The encoded key.
|
45
45
|
# @return [Hash] The key attributes.
|
46
46
|
def record_key(key)
|
47
|
-
|
47
|
+
decoded_key = decode_key(key)
|
48
|
+
|
49
|
+
if decoded_key.nil?
|
48
50
|
{}
|
49
|
-
elsif
|
50
|
-
@key_converter.convert(
|
51
|
+
elsif decoded_key.is_a?(Hash)
|
52
|
+
@key_converter.convert(decoded_key)
|
51
53
|
else
|
52
|
-
{ @klass.primary_key =>
|
54
|
+
{ @klass.primary_key => decoded_key }
|
53
55
|
end
|
54
56
|
end
|
55
57
|
|
@@ -100,10 +102,6 @@ module Deimos
|
|
100
102
|
|
101
103
|
options = if key_cols.empty?
|
102
104
|
{} # Can't upsert with no key, just do regular insert
|
103
|
-
elsif ActiveRecord::Base.connection.adapter_name =~ /mysql/i
|
104
|
-
{
|
105
|
-
on_duplicate_key_update: :all
|
106
|
-
}
|
107
105
|
else
|
108
106
|
{
|
109
107
|
on_duplicate_key_update: {
|
@@ -9,7 +9,7 @@ module Deimos
|
|
9
9
|
# Default is to use the primary key column and the value of the first
|
10
10
|
# field in the key.
|
11
11
|
# @param klass [Class < ActiveRecord::Base]
|
12
|
-
# @param _payload [Hash]
|
12
|
+
# @param _payload [Hash|Deimos::SchemaClass::Record]
|
13
13
|
# @param key [Object]
|
14
14
|
# @return [ActiveRecord::Base]
|
15
15
|
def fetch_record(klass, _payload, key)
|
@@ -18,13 +18,14 @@ module Deimos
|
|
18
18
|
|
19
19
|
# Assign a key to a new record.
|
20
20
|
# @param record [ActiveRecord::Base]
|
21
|
-
# @param _payload [Hash]
|
21
|
+
# @param _payload [Hash|Deimos::SchemaClass::Record]
|
22
22
|
# @param key [Object]
|
23
23
|
def assign_key(record, _payload, key)
|
24
24
|
record[record.class.primary_key] = key
|
25
25
|
end
|
26
26
|
|
27
|
-
#
|
27
|
+
# @param payload [Hash|Deimos::SchemaClass::Record] Decoded payloads
|
28
|
+
# @param metadata [Hash] Information about batch, including keys.
|
28
29
|
def consume(payload, metadata)
|
29
30
|
unless self.process_message?(payload)
|
30
31
|
Deimos.config.logger.debug(
|
@@ -50,14 +50,14 @@ module Deimos
|
|
50
50
|
|
51
51
|
# Override this method (with `super`) if you want to add/change the default
|
52
52
|
# attributes set to the new/existing record.
|
53
|
-
# @param payload [Hash]
|
53
|
+
# @param payload [Hash|Deimos::SchemaClass::Record]
|
54
54
|
# @param _key [String]
|
55
55
|
def record_attributes(payload, _key=nil)
|
56
56
|
@converter.convert(payload)
|
57
57
|
end
|
58
58
|
|
59
59
|
# Override this message to conditionally save records
|
60
|
-
# @param
|
60
|
+
# @param _payload [Hash|Deimos::SchemaClass::Record] The kafka message
|
61
61
|
# @return [Boolean] if true, record is created/update.
|
62
62
|
# If false, record processing is skipped but message offset is still committed.
|
63
63
|
def process_message?(_payload)
|
@@ -58,6 +58,9 @@ module Deimos
|
|
58
58
|
payload.delete_if do |k, _|
|
59
59
|
k.to_sym != :payload_key && !fields.map(&:name).include?(k)
|
60
60
|
end
|
61
|
+
return payload unless Utils::SchemaClass.use?(config.to_h)
|
62
|
+
|
63
|
+
Utils::SchemaClass.instance(payload, config[:schema])
|
61
64
|
end
|
62
65
|
|
63
66
|
# Query to use when polling the database with the DbPoller. Add
|
@@ -18,6 +18,9 @@ module Deimos
|
|
18
18
|
# :nodoc:
|
19
19
|
after_configure do
|
20
20
|
Phobos.configure(self.config.phobos_config)
|
21
|
+
if self.config.schema.use_schema_classes
|
22
|
+
load_generated_schema_classes
|
23
|
+
end
|
21
24
|
self.config.producer_objects.each do |producer|
|
22
25
|
configure_producer_or_consumer(producer)
|
23
26
|
end
|
@@ -28,6 +31,17 @@ module Deimos
|
|
28
31
|
validate_db_backend if self.config.producers.backend == :db
|
29
32
|
end
|
30
33
|
|
34
|
+
# Loads generated classes
|
35
|
+
def self.load_generated_schema_classes
|
36
|
+
if Deimos.config.schema.generated_class_path.nil?
|
37
|
+
raise 'Cannot use schema classes without schema.generated_class_path. Please provide a directory.'
|
38
|
+
end
|
39
|
+
|
40
|
+
Dir["./#{Deimos.config.schema.generated_class_path}/**/*.rb"].sort.each { |f| require f }
|
41
|
+
rescue LoadError
|
42
|
+
raise 'Cannot load schema classes. Please regenerate classes with rake deimos:generate_schema_models.'
|
43
|
+
end
|
44
|
+
|
31
45
|
# Ensure everything is set up correctly for the DB backend.
|
32
46
|
def self.validate_db_backend
|
33
47
|
begin
|
@@ -68,6 +82,7 @@ module Deimos
|
|
68
82
|
schema(kafka_config.schema) if kafka_config.schema.present?
|
69
83
|
namespace(kafka_config.namespace) if kafka_config.namespace.present?
|
70
84
|
key_config(**kafka_config.key_config) if kafka_config.key_config.present?
|
85
|
+
schema_class_config(kafka_config.use_schema_classes) if kafka_config.use_schema_classes.present?
|
71
86
|
end
|
72
87
|
end
|
73
88
|
|
@@ -260,6 +275,14 @@ module Deimos
|
|
260
275
|
# Local path to look for schemas in.
|
261
276
|
# @return [String]
|
262
277
|
setting :path
|
278
|
+
|
279
|
+
# Local path for schema classes to be generated in.
|
280
|
+
# @return [String]
|
281
|
+
setting :generated_class_path, 'app/lib/schema_classes'
|
282
|
+
|
283
|
+
# Set to true to use the generated schema classes in your application
|
284
|
+
# @return [Boolean]
|
285
|
+
setting :use_schema_classes, false
|
263
286
|
end
|
264
287
|
|
265
288
|
# The configured metrics provider.
|
@@ -301,6 +324,9 @@ module Deimos
|
|
301
324
|
# Key configuration (see docs).
|
302
325
|
# @return [Hash]
|
303
326
|
setting :key_config
|
327
|
+
# Configure the usage of generated schema classes for this producer
|
328
|
+
# @return [Boolean]
|
329
|
+
setting :use_schema_classes
|
304
330
|
end
|
305
331
|
|
306
332
|
setting_object :consumer do
|
@@ -323,6 +349,9 @@ module Deimos
|
|
323
349
|
# listener.
|
324
350
|
# @return [Boolean]
|
325
351
|
setting :disabled, false
|
352
|
+
# Configure the usage of generated schema classes for this consumer
|
353
|
+
# @return [Boolean]
|
354
|
+
setting :use_schema_classes
|
326
355
|
|
327
356
|
# These are the phobos "listener" configs. See CONFIGURATION.md for more
|
328
357
|
# info.
|
@@ -21,11 +21,11 @@ module Deimos
|
|
21
21
|
metadata[:first_offset] = batch.first&.offset
|
22
22
|
|
23
23
|
payloads = batch.map do |message|
|
24
|
-
|
24
|
+
decode_message(message.payload)
|
25
25
|
end
|
26
26
|
_received_batch(payloads, metadata)
|
27
27
|
_with_span do
|
28
|
-
yield
|
28
|
+
yield(payloads, metadata)
|
29
29
|
end
|
30
30
|
end
|
31
31
|
_handle_batch_success(benchmark.real, payloads, metadata)
|
@@ -15,9 +15,9 @@ module Deimos
|
|
15
15
|
benchmark = Benchmark.measure do
|
16
16
|
_with_span do
|
17
17
|
new_metadata[:key] = decode_key(metadata[:key]) if self.class.config[:key_configured]
|
18
|
-
decoded_payload =
|
18
|
+
decoded_payload = decode_message(payload)
|
19
19
|
_received_message(decoded_payload, new_metadata)
|
20
|
-
yield
|
20
|
+
yield(decoded_payload, new_metadata)
|
21
21
|
end
|
22
22
|
end
|
23
23
|
_handle_success(benchmark.real, decoded_payload, new_metadata)
|
data/lib/deimos/consumer.rb
CHANGED
@@ -51,6 +51,16 @@ module Deimos
|
|
51
51
|
end
|
52
52
|
end
|
53
53
|
|
54
|
+
# Helper method to decode an encoded message.
|
55
|
+
# @param payload [Object]
|
56
|
+
# @return [Object] the decoded message.
|
57
|
+
def decode_message(payload)
|
58
|
+
decoded_payload = payload.nil? ? nil : self.class.decoder.decode(payload)
|
59
|
+
return decoded_payload unless Utils::SchemaClass.use?(self.class.config.to_h)
|
60
|
+
|
61
|
+
Utils::SchemaClass.instance(decoded_payload, self.class.config[:schema])
|
62
|
+
end
|
63
|
+
|
54
64
|
private
|
55
65
|
|
56
66
|
def _with_span
|
data/lib/deimos/producer.rb
CHANGED
@@ -57,6 +57,7 @@ module Deimos
|
|
57
57
|
MAX_BATCH_SIZE = 500
|
58
58
|
|
59
59
|
class << self
|
60
|
+
|
60
61
|
# @return [Hash]
|
61
62
|
def config
|
62
63
|
@config ||= {
|
@@ -86,14 +87,14 @@ module Deimos
|
|
86
87
|
end
|
87
88
|
|
88
89
|
# Publish the payload to the topic.
|
89
|
-
# @param payload [Hash] with an optional payload_key hash key.
|
90
|
+
# @param payload [Hash|SchemaClass::Record] with an optional payload_key hash key.
|
90
91
|
# @param topic [String] if specifying the topic
|
91
92
|
def publish(payload, topic: self.topic)
|
92
93
|
publish_list([payload], topic: topic)
|
93
94
|
end
|
94
95
|
|
95
96
|
# Publish a list of messages.
|
96
|
-
# @param payloads [
|
97
|
+
# @param payloads [Array<Hash|SchemaClass::Record>] with optional payload_key hash key.
|
97
98
|
# @param sync [Boolean] if given, override the default setting of
|
98
99
|
# whether to publish synchronously.
|
99
100
|
# @param force_send [Boolean] if true, ignore the configured backend
|
@@ -113,7 +114,7 @@ module Deimos
|
|
113
114
|
topic: topic,
|
114
115
|
payloads: payloads
|
115
116
|
) do
|
116
|
-
messages = Array(payloads).map { |p| Deimos::Message.new(p, self) }
|
117
|
+
messages = Array(payloads).map { |p| Deimos::Message.new(p.to_h, self) }
|
117
118
|
messages.each { |m| _process_message(m, topic) }
|
118
119
|
messages.in_groups_of(MAX_BATCH_SIZE, false) do |batch|
|
119
120
|
self.produce_batch(backend_class, batch)
|
@@ -66,7 +66,7 @@ module Deimos
|
|
66
66
|
def schema_fields
|
67
67
|
avro_schema.fields.map do |field|
|
68
68
|
enum_values = field.type.type == 'enum' ? field.type.symbols : []
|
69
|
-
SchemaField.new(field.name, field.type, enum_values)
|
69
|
+
SchemaField.new(field.name, field.type, enum_values, field.default)
|
70
70
|
end
|
71
71
|
end
|
72
72
|
|
@@ -77,6 +77,12 @@ module Deimos
|
|
77
77
|
fail_on_extra_fields: true)
|
78
78
|
end
|
79
79
|
|
80
|
+
# @override
|
81
|
+
# @return [Avro::Schema]
|
82
|
+
def load_schema
|
83
|
+
avro_schema
|
84
|
+
end
|
85
|
+
|
80
86
|
# @override
|
81
87
|
def self.mock_backend
|
82
88
|
:avro_validation
|
@@ -87,6 +93,59 @@ module Deimos
|
|
87
93
|
'avro/binary'
|
88
94
|
end
|
89
95
|
|
96
|
+
# @param schema [Avro::Schema::NamedSchema] A named schema
|
97
|
+
# @return [String]
|
98
|
+
def self.schema_classname(schema)
|
99
|
+
schema.name.underscore.camelize
|
100
|
+
end
|
101
|
+
|
102
|
+
# Converts Avro::Schema::NamedSchema's to String form for generated YARD docs.
|
103
|
+
# Recursively handles the typing for Arrays, Maps and Unions.
|
104
|
+
# @param avro_schema [Avro::Schema::NamedSchema]
|
105
|
+
# @return [String] A string representation of the Type of this SchemaField
|
106
|
+
def self.field_type(avro_schema)
|
107
|
+
case avro_schema.type_sym
|
108
|
+
when :string, :boolean
|
109
|
+
avro_schema.type_sym.to_s.titleize
|
110
|
+
when :int, :long
|
111
|
+
'Integer'
|
112
|
+
when :float, :double
|
113
|
+
'Float'
|
114
|
+
when :record, :enum
|
115
|
+
schema_classname(avro_schema)
|
116
|
+
when :array
|
117
|
+
arr_t = field_type(Deimos::SchemaField.new('n/a', avro_schema.items).type)
|
118
|
+
"Array<#{arr_t}>"
|
119
|
+
when :map
|
120
|
+
map_t = field_type(Deimos::SchemaField.new('n/a', avro_schema.values).type)
|
121
|
+
"Hash<String, #{map_t}>"
|
122
|
+
when :union
|
123
|
+
types = avro_schema.schemas.map do |t|
|
124
|
+
field_type(Deimos::SchemaField.new('n/a', t).type)
|
125
|
+
end
|
126
|
+
types.join(', ')
|
127
|
+
when :null
|
128
|
+
'nil'
|
129
|
+
end
|
130
|
+
end
|
131
|
+
|
132
|
+
# Returns the base type of this schema. Decodes Arrays, Maps and Unions
|
133
|
+
# @param schema [Avro::Schema::NamedSchema]
|
134
|
+
# @return [Avro::Schema::NamedSchema]
|
135
|
+
def self.schema_base_class(schema)
|
136
|
+
case schema.type_sym
|
137
|
+
when :array
|
138
|
+
schema_base_class(schema.items)
|
139
|
+
when :map
|
140
|
+
schema_base_class(schema.values)
|
141
|
+
when :union
|
142
|
+
schema.schemas.map(&method(:schema_base_class)).
|
143
|
+
reject { |s| s.type_sym == :null }.first
|
144
|
+
else
|
145
|
+
schema
|
146
|
+
end
|
147
|
+
end
|
148
|
+
|
90
149
|
private
|
91
150
|
|
92
151
|
# @param schema [String]
|
@@ -135,6 +194,10 @@ module Deimos
|
|
135
194
|
def _key_schema_name(schema)
|
136
195
|
"#{schema}_key"
|
137
196
|
end
|
197
|
+
|
198
|
+
def _schema_name
|
199
|
+
avro_schema.name
|
200
|
+
end
|
138
201
|
end
|
139
202
|
end
|
140
203
|
end
|
@@ -3,15 +3,16 @@
|
|
3
3
|
module Deimos
|
4
4
|
# Represents a field in the schema.
|
5
5
|
class SchemaField
|
6
|
-
attr_accessor :name, :type, :enum_values
|
6
|
+
attr_accessor :name, :type, :enum_values, :default
|
7
7
|
|
8
8
|
# @param name [String]
|
9
9
|
# @param type [Object]
|
10
10
|
# @param enum_values [Array<String>]
|
11
|
-
def initialize(name, type, enum_values=[])
|
11
|
+
def initialize(name, type, enum_values=[], default=:no_default)
|
12
12
|
@name = name
|
13
13
|
@type = type
|
14
14
|
@enum_values = enum_values
|
15
|
+
@default = default
|
15
16
|
end
|
16
17
|
end
|
17
18
|
|
@@ -43,6 +44,7 @@ module Deimos
|
|
43
44
|
# @return [Hash,nil]
|
44
45
|
def decode(payload, schema: nil)
|
45
46
|
return nil if payload.nil?
|
47
|
+
|
46
48
|
decode_payload(payload, schema: schema || @schema)
|
47
49
|
end
|
48
50
|
|
@@ -78,6 +80,14 @@ module Deimos
|
|
78
80
|
raise NotImplementedError
|
79
81
|
end
|
80
82
|
|
83
|
+
# Converts your schema to String form for generated YARD docs.
|
84
|
+
# To be defined by subclass.
|
85
|
+
# @param schema [Object]
|
86
|
+
# @return [String] A string representation of the Type
|
87
|
+
def self.field_type(schema)
|
88
|
+
raise NotImplementedError
|
89
|
+
end
|
90
|
+
|
81
91
|
# Encode a payload. To be defined by subclass.
|
82
92
|
# @param payload [Hash]
|
83
93
|
# @param schema [Symbol|String]
|
@@ -145,6 +155,12 @@ module Deimos
|
|
145
155
|
def decode_key(_payload, _key_id)
|
146
156
|
raise NotImplementedError
|
147
157
|
end
|
158
|
+
|
159
|
+
# Forcefully loads the schema into memory.
|
160
|
+
# @return [Object] The schema that is of use.
|
161
|
+
def load_schema
|
162
|
+
raise NotImplementedError
|
163
|
+
end
|
148
164
|
end
|
149
165
|
end
|
150
166
|
end
|