deimos-ruby 2.2.0 → 2.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +33 -30
- data/CHANGELOG.md +10 -0
- data/Gemfile +0 -6
- data/deimos-ruby.gemspec +15 -11
- data/karafka.rb +7 -4
- data/lib/deimos/active_record_consume/batch_consumption.rb +7 -7
- data/lib/deimos/active_record_consume/batch_record.rb +2 -2
- data/lib/deimos/active_record_consume/message_consumption.rb +6 -5
- data/lib/deimos/active_record_consume/schema_model_converter.rb +2 -2
- data/lib/deimos/active_record_consumer.rb +1 -0
- data/lib/deimos/active_record_producer.rb +4 -2
- data/lib/deimos/backends/base.rb +1 -3
- data/lib/deimos/backends/outbox.rb +1 -1
- data/lib/deimos/config/configuration.rb +88 -75
- data/lib/deimos/consume/batch_consumption.rb +5 -5
- data/lib/deimos/consume/message_consumption.rb +3 -3
- data/lib/deimos/ext/consumer_route.rb +3 -3
- data/lib/deimos/ext/producer_metrics_listener.rb +2 -2
- data/lib/deimos/ext/producer_middleware.rb +19 -15
- data/lib/deimos/ext/producer_route.rb +4 -2
- data/lib/deimos/ext/routing_defaults.rb +9 -7
- data/lib/deimos/ext/schema_route.rb +22 -15
- data/lib/deimos/kafka_message.rb +1 -1
- data/lib/deimos/kafka_source.rb +36 -31
- data/lib/deimos/kafka_topic_info.rb +1 -1
- data/lib/deimos/logging.rb +20 -19
- data/lib/deimos/message.rb +1 -1
- data/lib/deimos/metrics/minimal_datadog_listener.rb +19 -6
- data/lib/deimos/metrics/provider.rb +4 -4
- data/lib/deimos/producer.rb +3 -1
- data/lib/deimos/railtie.rb +1 -1
- data/lib/deimos/schema_backends/avro_base.rb +1 -1
- data/lib/deimos/schema_backends/avro_schema_coercer.rb +46 -27
- data/lib/deimos/schema_backends/avro_schema_registry.rb +8 -8
- data/lib/deimos/schema_backends/base.rb +9 -9
- data/lib/deimos/schema_backends/plain.rb +1 -1
- data/lib/deimos/schema_backends/proto_base.rb +7 -5
- data/lib/deimos/schema_backends/proto_local.rb +0 -2
- data/lib/deimos/schema_backends/proto_schema_registry.rb +0 -2
- data/lib/deimos/schema_class/base.rb +1 -1
- data/lib/deimos/schema_class/record.rb +3 -3
- data/lib/deimos/test_helpers.rb +31 -26
- data/lib/deimos/tracing/provider.rb +5 -5
- data/lib/deimos/transcoder.rb +6 -2
- data/lib/deimos/utils/db_poller/base.rb +3 -3
- data/lib/deimos/utils/deadlock_retry.rb +2 -2
- data/lib/deimos/utils/outbox_producer.rb +14 -14
- data/lib/deimos/version.rb +1 -1
- data/lib/deimos.rb +4 -4
- data/lib/generators/deimos/active_record_generator.rb +2 -1
- data/lib/generators/deimos/db_poller_generator.rb +1 -0
- data/lib/generators/deimos/outbox_backend_generator.rb +1 -0
- data/lib/generators/deimos/schema_class_generator.rb +3 -2
- data/lib/generators/deimos/v2_generator.rb +184 -155
- data/spec/active_record_batch_consumer_association_spec.rb +6 -2
- data/spec/active_record_batch_consumer_spec.rb +83 -106
- data/spec/active_record_consume/batch_consumption_spec.rb +27 -28
- data/spec/active_record_consume/batch_slicer_spec.rb +4 -12
- data/spec/active_record_consume/mass_updater_spec.rb +42 -46
- data/spec/active_record_consume/schema_model_converter_spec.rb +1 -1
- data/spec/active_record_consumer_spec.rb +7 -5
- data/spec/active_record_producer_spec.rb +83 -73
- data/spec/backends/outbox_spec.rb +1 -1
- data/spec/batch_consumer_spec.rb +20 -20
- data/spec/consumer_spec.rb +23 -12
- data/spec/gen/sample/v1/sample_pb.rb +3 -3
- data/spec/generators/active_record_generator_spec.rb +4 -4
- data/spec/generators/schema_class/my_schema_with_circular_reference_spec.rb +2 -1
- data/spec/generators/schema_class/my_schema_with_complex_types_spec.rb +9 -2
- data/spec/generators/schema_class_generator_spec.rb +5 -5
- data/spec/kafka_source_spec.rb +13 -6
- data/spec/kafka_topic_info_spec.rb +7 -7
- data/spec/karafka/karafka.rb +6 -5
- data/spec/karafka_config/karafka_spec.rb +22 -19
- data/spec/logging_spec.rb +2 -0
- data/spec/producer_spec.rb +25 -20
- data/spec/schema_backends/avro_base_shared.rb +8 -8
- data/spec/schema_backends/avro_local_spec.rb +5 -6
- data/spec/schema_backends/avro_schema_registry_spec.rb +5 -6
- data/spec/schema_backends/proto_schema_registry_spec.rb +9 -12
- data/spec/schemas/my_namespace/generated.rb +1 -2
- data/spec/schemas/my_namespace/my_schema_with_complex_type.rb +5 -8
- data/spec/schemas/my_namespace/my_schema_with_union_type.rb +22 -23
- data/spec/spec_helper.rb +13 -17
- data/spec/utils/db_poller_spec.rb +5 -5
- data/spec/utils/deadlock_retry_spec.rb +1 -4
- data/spec/utils/outbox_producer_spec.rb +36 -24
- metadata +68 -161
- data/.ruby-version +0 -1
data/lib/deimos/logging.rb
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
module Deimos
|
|
2
4
|
module Logging
|
|
3
5
|
class << self
|
|
@@ -33,34 +35,33 @@ module Deimos
|
|
|
33
35
|
end
|
|
34
36
|
|
|
35
37
|
def _payloads(messages)
|
|
36
|
-
|
|
37
38
|
end
|
|
38
39
|
|
|
39
|
-
def payload(
|
|
40
|
-
return nil if
|
|
40
|
+
def payload(msg)
|
|
41
|
+
return nil if msg.nil?
|
|
41
42
|
|
|
42
|
-
if
|
|
43
|
-
|
|
44
|
-
elsif
|
|
45
|
-
|
|
43
|
+
if msg.respond_to?(:payload)
|
|
44
|
+
msg.payload
|
|
45
|
+
elsif msg[:label]
|
|
46
|
+
msg.dig(:label, :original_payload)
|
|
46
47
|
else
|
|
47
|
-
|
|
48
|
+
msg[:payload]
|
|
48
49
|
end
|
|
49
50
|
end
|
|
50
51
|
|
|
51
|
-
def key(
|
|
52
|
-
return nil if
|
|
52
|
+
def key(msg)
|
|
53
|
+
return nil if msg.nil?
|
|
53
54
|
|
|
54
|
-
if
|
|
55
|
-
|
|
56
|
-
elsif
|
|
57
|
-
if
|
|
58
|
-
|
|
59
|
-
elsif
|
|
60
|
-
|
|
55
|
+
if msg.respond_to?(:payload) && msg.payload
|
|
56
|
+
msg.key || msg.payload['message_id']
|
|
57
|
+
elsif msg.respond_to?(:[])
|
|
58
|
+
if msg[:label]
|
|
59
|
+
msg.dig(:label, :original_key)
|
|
60
|
+
elsif msg[:payload].is_a?(String)
|
|
61
|
+
msg[:key] || msg[:payload_key]
|
|
61
62
|
else
|
|
62
|
-
payload =
|
|
63
|
-
|
|
63
|
+
payload = msg[:payload]&.with_indifferent_access
|
|
64
|
+
msg[:key] || msg[:payload_key] || payload[:payload_key] || payload[:message_id]
|
|
64
65
|
end
|
|
65
66
|
end
|
|
66
67
|
end
|
data/lib/deimos/message.rb
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
require 'karafka/instrumentation/vendors/datadog/metrics_listener'
|
|
2
4
|
require 'waterdrop/instrumentation/vendors/datadog/metrics_listener'
|
|
3
5
|
|
|
@@ -5,12 +7,23 @@ module Deimos
|
|
|
5
7
|
module Metrics
|
|
6
8
|
class MinimalDatadogListener < ::Karafka::Instrumentation::Vendors::Datadog::MetricsListener
|
|
7
9
|
# override existing listener so we don't emit metrics we don't care about
|
|
8
|
-
def on_connection_listener_fetch_loop_received(_event)
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
def
|
|
12
|
-
|
|
13
|
-
|
|
10
|
+
def on_connection_listener_fetch_loop_received(_event)
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
def on_consumer_revoked(_event)
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def on_consumer_shutdown(_event)
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def on_consumer_ticked(_event)
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def on_worker_process(_event)
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def on_worker_processed(_event)
|
|
26
|
+
end
|
|
14
27
|
end
|
|
15
28
|
end
|
|
16
29
|
end
|
|
@@ -8,7 +8,7 @@ module Deimos
|
|
|
8
8
|
# @param metric_name [String] The name of the counter metric
|
|
9
9
|
# @param options [Hash] Any additional options, e.g. :tags
|
|
10
10
|
# @return [void]
|
|
11
|
-
def increment(
|
|
11
|
+
def increment(_metric_name, _options={})
|
|
12
12
|
raise MissingImplementationError
|
|
13
13
|
end
|
|
14
14
|
|
|
@@ -17,7 +17,7 @@ module Deimos
|
|
|
17
17
|
# @param count [Integer]
|
|
18
18
|
# @param options [Hash] Any additional options, e.g. :tags
|
|
19
19
|
# @return [void]
|
|
20
|
-
def gauge(
|
|
20
|
+
def gauge(_metric_name, _count, _options={})
|
|
21
21
|
raise MissingImplementationError
|
|
22
22
|
end
|
|
23
23
|
|
|
@@ -26,7 +26,7 @@ module Deimos
|
|
|
26
26
|
# @param count [Integer]
|
|
27
27
|
# @param options [Hash] Any additional options, e.g. :tags
|
|
28
28
|
# @return [void]
|
|
29
|
-
def histogram(
|
|
29
|
+
def histogram(_metric_name, _count, _options={})
|
|
30
30
|
raise MissingImplementationError
|
|
31
31
|
end
|
|
32
32
|
|
|
@@ -34,7 +34,7 @@ module Deimos
|
|
|
34
34
|
# @param metric_name [String] The name of the metric
|
|
35
35
|
# @param options [Hash] Any additional options, e.g. :tags
|
|
36
36
|
# @return [void]
|
|
37
|
-
def time(
|
|
37
|
+
def time(_metric_name, _options={})
|
|
38
38
|
raise MissingImplementationError
|
|
39
39
|
end
|
|
40
40
|
end
|
data/lib/deimos/producer.rb
CHANGED
|
@@ -80,7 +80,7 @@ module Deimos
|
|
|
80
80
|
# @param headers [Hash] if specifying headers
|
|
81
81
|
# @return [void]
|
|
82
82
|
def publish(payload, topic: self.topic, headers: nil)
|
|
83
|
-
produce([{payload: payload, topic: topic, headers: headers}])
|
|
83
|
+
produce([{ payload: payload, topic: topic, headers: headers }])
|
|
84
84
|
end
|
|
85
85
|
|
|
86
86
|
# Produce a list of messages in WaterDrop message hash format.
|
|
@@ -141,6 +141,7 @@ module Deimos
|
|
|
141
141
|
karafka_config&.name
|
|
142
142
|
end
|
|
143
143
|
|
|
144
|
+
# rubocop:disable Style/OptionalBooleanParameter
|
|
144
145
|
# @param sync [Boolean]
|
|
145
146
|
# @param force_send [Boolean]
|
|
146
147
|
# @return [Class<Deimos::Backends::Base>]
|
|
@@ -157,6 +158,7 @@ module Deimos
|
|
|
157
158
|
end
|
|
158
159
|
"Deimos::Backends::#{backend.to_s.classify}".constantize
|
|
159
160
|
end
|
|
161
|
+
# rubocop:enable Style/OptionalBooleanParameter
|
|
160
162
|
|
|
161
163
|
# Send a batch to the backend.
|
|
162
164
|
# @param backend [Class<Deimos::Backends::Base>]
|
data/lib/deimos/railtie.rb
CHANGED
|
@@ -74,45 +74,64 @@ module Deimos
|
|
|
74
74
|
val.keys.zip(record).to_h
|
|
75
75
|
end
|
|
76
76
|
|
|
77
|
+
# @param type [Avro::Schema::RecordSchema]
|
|
78
|
+
# @param val [Object]
|
|
79
|
+
# @return [Integer]
|
|
80
|
+
def coerce_int(type, val)
|
|
81
|
+
int_classes = [Time, ActiveSupport::TimeWithZone]
|
|
82
|
+
if %w(timestamp-millis timestamp-micros).include?(type.logical_type)
|
|
83
|
+
val
|
|
84
|
+
elsif val.is_a?(Integer) ||
|
|
85
|
+
_is_integer_string?(val) ||
|
|
86
|
+
int_classes.any? { |klass| val.is_a?(klass) }
|
|
87
|
+
val.to_i
|
|
88
|
+
else # rubocop:disable Lint/DuplicateBranch
|
|
89
|
+
val # this will fail
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
# @param val [Object]
|
|
94
|
+
# @return [Float]
|
|
95
|
+
def coerce_float(val)
|
|
96
|
+
if val.is_a?(Numeric) || _is_float_string?(val)
|
|
97
|
+
val.to_f
|
|
98
|
+
else
|
|
99
|
+
val # this will fail
|
|
100
|
+
end
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# @param val [Object]
|
|
104
|
+
# @return [String]
|
|
105
|
+
def coerce_string(val)
|
|
106
|
+
if val.respond_to?(:to_str) || _is_to_s_defined?(val)
|
|
107
|
+
val.to_s
|
|
108
|
+
else
|
|
109
|
+
val # this will fail
|
|
110
|
+
end
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
# @param val [Object]
|
|
114
|
+
# @return [Boolean]
|
|
115
|
+
def coerce_boolean(val) # rubocop:disable Naming/PredicateMethod
|
|
116
|
+
!(val.nil? || val == false)
|
|
117
|
+
end
|
|
118
|
+
|
|
77
119
|
# Coerce values in a payload to match the schema.
|
|
78
120
|
# @param type [Avro::Schema]
|
|
79
121
|
# @param val [Object]
|
|
80
122
|
# @return [Object]
|
|
81
123
|
def coerce_type(type, val)
|
|
82
|
-
int_classes = [Time, ActiveSupport::TimeWithZone]
|
|
83
124
|
field_type = type.type.to_sym
|
|
84
125
|
|
|
85
126
|
case field_type
|
|
86
127
|
when :int, :long
|
|
87
|
-
|
|
88
|
-
val
|
|
89
|
-
elsif val.is_a?(Integer) ||
|
|
90
|
-
_is_integer_string?(val) ||
|
|
91
|
-
int_classes.any? { |klass| val.is_a?(klass) }
|
|
92
|
-
val.to_i
|
|
93
|
-
else
|
|
94
|
-
val # this will fail
|
|
95
|
-
end
|
|
128
|
+
coerce_int(type, val)
|
|
96
129
|
when :float, :double
|
|
97
|
-
|
|
98
|
-
val.to_f
|
|
99
|
-
else
|
|
100
|
-
val # this will fail
|
|
101
|
-
end
|
|
130
|
+
coerce_float(val)
|
|
102
131
|
when :string
|
|
103
|
-
|
|
104
|
-
val.to_s
|
|
105
|
-
elsif _is_to_s_defined?(val)
|
|
106
|
-
val.to_s
|
|
107
|
-
else
|
|
108
|
-
val # this will fail
|
|
109
|
-
end
|
|
132
|
+
coerce_string(val)
|
|
110
133
|
when :boolean
|
|
111
|
-
|
|
112
|
-
false
|
|
113
|
-
else
|
|
114
|
-
true
|
|
115
|
-
end
|
|
134
|
+
coerce_boolean(val)
|
|
116
135
|
when :union
|
|
117
136
|
coerce_union(type, val)
|
|
118
137
|
when :record
|
|
@@ -22,14 +22,14 @@ module Deimos
|
|
|
22
22
|
# @return [AvroTurf::Messaging]
|
|
23
23
|
def avro_turf_messaging
|
|
24
24
|
@avro_turf_messaging ||= AvroTurf::Messaging.new(
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
25
|
+
schema_store: @schema_store,
|
|
26
|
+
registry_url: Deimos.config.schema.registry_url,
|
|
27
|
+
schemas_path: Deimos.config.schema.path,
|
|
28
|
+
user: Deimos.config.schema.user,
|
|
29
|
+
password: Deimos.config.schema.password,
|
|
30
|
+
namespace: @namespace,
|
|
31
|
+
logger: Karafka.logger
|
|
32
|
+
)
|
|
33
33
|
end
|
|
34
34
|
end
|
|
35
35
|
end
|
|
@@ -107,7 +107,7 @@ module Deimos
|
|
|
107
107
|
# To be defined by subclass.
|
|
108
108
|
# @param schema [Object]
|
|
109
109
|
# @return [String] A string representation of the Type
|
|
110
|
-
def self.field_type(
|
|
110
|
+
def self.field_type(_schema)
|
|
111
111
|
raise MissingImplementationError
|
|
112
112
|
end
|
|
113
113
|
|
|
@@ -116,7 +116,7 @@ module Deimos
|
|
|
116
116
|
# @param schema [String,Symbol]
|
|
117
117
|
# @param topic [String]
|
|
118
118
|
# @return [String]
|
|
119
|
-
def encode_payload(
|
|
119
|
+
def encode_payload(_payload, schema:, topic: nil)
|
|
120
120
|
raise MissingImplementationError
|
|
121
121
|
end
|
|
122
122
|
|
|
@@ -124,7 +124,7 @@ module Deimos
|
|
|
124
124
|
# @param payload [String]
|
|
125
125
|
# @param schema [String,Symbol]
|
|
126
126
|
# @return [Hash]
|
|
127
|
-
def decode_payload(
|
|
127
|
+
def decode_payload(_payload, schema:)
|
|
128
128
|
raise MissingImplementationError
|
|
129
129
|
end
|
|
130
130
|
|
|
@@ -132,7 +132,7 @@ module Deimos
|
|
|
132
132
|
# @param payload [Hash]
|
|
133
133
|
# @param schema [String,Symbol]
|
|
134
134
|
# @return [void]
|
|
135
|
-
def validate(
|
|
135
|
+
def validate(_payload, schema:)
|
|
136
136
|
raise MissingImplementationError
|
|
137
137
|
end
|
|
138
138
|
|
|
@@ -148,7 +148,7 @@ module Deimos
|
|
|
148
148
|
# @param field [SchemaField]
|
|
149
149
|
# @param value [Object]
|
|
150
150
|
# @return [Object]
|
|
151
|
-
def coerce_field(
|
|
151
|
+
def coerce_field(_field, _value)
|
|
152
152
|
raise MissingImplementationError
|
|
153
153
|
end
|
|
154
154
|
|
|
@@ -159,14 +159,14 @@ module Deimos
|
|
|
159
159
|
# `:enum` is also recognized.
|
|
160
160
|
# @param field [SchemaField]
|
|
161
161
|
# @return [Symbol]
|
|
162
|
-
def sql_type(
|
|
162
|
+
def sql_type(_field)
|
|
163
163
|
raise MissingImplementationError
|
|
164
164
|
end
|
|
165
165
|
|
|
166
166
|
# Generate a key schema from the given value schema and key ID. This
|
|
167
167
|
# is used when encoding or decoding keys from an existing value schema.
|
|
168
168
|
# @param field_name [Symbol]
|
|
169
|
-
def generate_key_schema(
|
|
169
|
+
def generate_key_schema(_field_name)
|
|
170
170
|
raise MissingImplementationError
|
|
171
171
|
end
|
|
172
172
|
|
|
@@ -175,7 +175,7 @@ module Deimos
|
|
|
175
175
|
# @param key_id [String,Symbol] the field name of the key.
|
|
176
176
|
# @param topic [String]
|
|
177
177
|
# @return [String]
|
|
178
|
-
def encode_key(
|
|
178
|
+
def encode_key(_key, _key_id, topic: nil)
|
|
179
179
|
raise MissingImplementationError
|
|
180
180
|
end
|
|
181
181
|
|
|
@@ -183,7 +183,7 @@ module Deimos
|
|
|
183
183
|
# @param payload [Hash] the message itself.
|
|
184
184
|
# @param key_id [String,Symbol] the field in the message to decode.
|
|
185
185
|
# @return [String]
|
|
186
|
-
def decode_key(
|
|
186
|
+
def decode_key(_payload, _key_id)
|
|
187
187
|
raise MissingImplementationError
|
|
188
188
|
end
|
|
189
189
|
|
|
@@ -23,7 +23,7 @@ module Deimos
|
|
|
23
23
|
bytes: :string,
|
|
24
24
|
float: :float,
|
|
25
25
|
message: :record
|
|
26
|
-
}
|
|
26
|
+
}.freeze
|
|
27
27
|
def proto_schema(schema=@schema)
|
|
28
28
|
Google::Protobuf::DescriptorPool.generated_pool.lookup(schema)
|
|
29
29
|
end
|
|
@@ -39,7 +39,11 @@ module Deimos
|
|
|
39
39
|
|
|
40
40
|
# @override
|
|
41
41
|
def decode_key(payload, key_id)
|
|
42
|
-
val =
|
|
42
|
+
val = begin
|
|
43
|
+
JSON.parse(payload)
|
|
44
|
+
rescue StandardError
|
|
45
|
+
payload
|
|
46
|
+
end
|
|
43
47
|
key_id ? val[key_id.to_s] : val
|
|
44
48
|
end
|
|
45
49
|
|
|
@@ -81,12 +85,10 @@ module Deimos
|
|
|
81
85
|
:mock
|
|
82
86
|
end
|
|
83
87
|
|
|
84
|
-
def generate_key_schema(
|
|
88
|
+
def generate_key_schema(_field_name)
|
|
85
89
|
raise 'Protobuf cannot generate key schemas! Please use field_config :plain'
|
|
86
90
|
end
|
|
87
91
|
|
|
88
|
-
private
|
|
89
|
-
|
|
90
92
|
end
|
|
91
93
|
end
|
|
92
94
|
end
|
|
@@ -78,9 +78,8 @@ module Deimos
|
|
|
78
78
|
# @return [SchemaClass::Record]
|
|
79
79
|
def self.new_from_message(**kwargs)
|
|
80
80
|
record = self.new
|
|
81
|
-
attrs = kwargs.select { |k,
|
|
82
|
-
|
|
83
|
-
record
|
|
81
|
+
attrs = kwargs.select { |k, _v| record.respond_to?("#{k}=") }
|
|
82
|
+
self.new(_from_message: true, **attrs)
|
|
84
83
|
end
|
|
85
84
|
|
|
86
85
|
# @return [SchemaClass::Record]
|
|
@@ -89,6 +88,7 @@ module Deimos
|
|
|
89
88
|
return nil if value.nil?
|
|
90
89
|
|
|
91
90
|
return value if value.is_a?(self)
|
|
91
|
+
|
|
92
92
|
if from_message
|
|
93
93
|
self.new_from_message(**value&.symbolize_keys || {})
|
|
94
94
|
else
|
data/lib/deimos/test_helpers.rb
CHANGED
|
@@ -12,6 +12,7 @@ module Deimos
|
|
|
12
12
|
# and add methods to use to test encoding/decoding.
|
|
13
13
|
module TestHelpers
|
|
14
14
|
extend ActiveSupport::Concern
|
|
15
|
+
|
|
15
16
|
def self.included(base)
|
|
16
17
|
super
|
|
17
18
|
base.include Karafka::Testing::RSpec::Helpers
|
|
@@ -38,7 +39,8 @@ module Deimos
|
|
|
38
39
|
# @return [void]
|
|
39
40
|
def unit_test!
|
|
40
41
|
Deimos.config.schema.backend = :avro_validation
|
|
41
|
-
warn
|
|
42
|
+
warn("unit_test! is deprecated and can be replaced by setting Deimos's schema backend " \
|
|
43
|
+
'to `:avro_validation`. All other test behavior is provided by Karafka.')
|
|
42
44
|
end
|
|
43
45
|
end
|
|
44
46
|
|
|
@@ -60,16 +62,16 @@ module Deimos
|
|
|
60
62
|
end
|
|
61
63
|
end
|
|
62
64
|
|
|
63
|
-
def self.normalize_message(
|
|
64
|
-
return nil if
|
|
65
|
+
def self.normalize_message(message)
|
|
66
|
+
return nil if message.nil?
|
|
65
67
|
|
|
66
|
-
if
|
|
67
|
-
|
|
68
|
+
if message.respond_to?(:to_h)
|
|
69
|
+
message = message.to_h
|
|
68
70
|
end
|
|
69
|
-
if
|
|
70
|
-
|
|
71
|
+
if message.respond_to?(:with_indifferent_access)
|
|
72
|
+
message = message.with_indifferent_access
|
|
71
73
|
end
|
|
72
|
-
|
|
74
|
+
message
|
|
73
75
|
end
|
|
74
76
|
|
|
75
77
|
# @!visibility private
|
|
@@ -80,11 +82,12 @@ module Deimos
|
|
|
80
82
|
min_hash_diff = nil
|
|
81
83
|
message = Deimos::TestHelpers.normalize_message(message)
|
|
82
84
|
if messages.any?
|
|
83
|
-
message_string = messages.map { |m| m[:payload].inspect}.join("\n")
|
|
85
|
+
message_string = messages.map { |m| m[:payload].inspect }.join("\n")
|
|
84
86
|
min_hash_diff = messages.min_by { |m| _hash_diff(m, message)&.keys&.size }
|
|
85
87
|
diff = RSpec::Expectations.differ.diff_as_object(message, min_hash_diff[:payload])
|
|
86
88
|
end
|
|
87
|
-
|
|
89
|
+
printed_message = message.try(:to_h) || message
|
|
90
|
+
str = "Expected #{topic} #{'not ' if was_negated}to have sent #{printed_message}"
|
|
88
91
|
str += " with key #{key}" if key
|
|
89
92
|
str += " with partition key #{partition_key}" if partition_key
|
|
90
93
|
str += "\nClosest message received: #{min_hash_diff}" if min_hash_diff
|
|
@@ -101,7 +104,9 @@ module Deimos
|
|
|
101
104
|
return m[:payload] == message if message.is_a?(String)
|
|
102
105
|
|
|
103
106
|
message.delete(:payload_key) if message.respond_to?(:[]) && message[:payload_key].nil?
|
|
104
|
-
|
|
107
|
+
if m.respond_to?(:[]) && m[:payload].respond_to?(:[]) && m[:payload][:payload_key].nil?
|
|
108
|
+
m[:payload].delete(:payload_key)
|
|
109
|
+
end
|
|
105
110
|
hash_matcher.send(:match, message, m[:payload]) &&
|
|
106
111
|
topic == m[:topic] &&
|
|
107
112
|
(key.present? ? message_key == m[:key] : true) &&
|
|
@@ -128,7 +133,8 @@ module Deimos
|
|
|
128
133
|
# particular messages were sent or not sent after a point in time.
|
|
129
134
|
# @return [void]
|
|
130
135
|
def clear_kafka_messages!
|
|
131
|
-
puts
|
|
136
|
+
puts '[Deprecated] clear_kafka_messages! can be replaced with' \
|
|
137
|
+
'`karafka.produced_messages.clear`'
|
|
132
138
|
karafka.produced_messages.clear
|
|
133
139
|
end
|
|
134
140
|
|
|
@@ -150,7 +156,8 @@ module Deimos
|
|
|
150
156
|
partition_key: nil,
|
|
151
157
|
&block)
|
|
152
158
|
unless call_original.nil?
|
|
153
|
-
puts
|
|
159
|
+
puts 'test_consume_message(call_original: true) is deprecated and will be removed' \
|
|
160
|
+
'in the future. You can remove the call_original parameter.'
|
|
154
161
|
end
|
|
155
162
|
test_consume_batch(handler_class_or_topic,
|
|
156
163
|
[payload],
|
|
@@ -177,36 +184,34 @@ module Deimos
|
|
|
177
184
|
keys: [],
|
|
178
185
|
call_original: nil,
|
|
179
186
|
single: false,
|
|
180
|
-
partition_keys: []
|
|
187
|
+
partition_keys: [])
|
|
181
188
|
unless call_original.nil?
|
|
182
|
-
puts
|
|
189
|
+
puts 'test_consume_batch(call_original: true) is deprecated and will be removed' \
|
|
190
|
+
'in the future. You can remove the call_original parameter.'
|
|
183
191
|
end
|
|
184
192
|
karafka.consumer_messages.clear
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
consumer = karafka.consumer_for(topic_name)
|
|
190
|
-
else
|
|
191
|
-
topic_name = Deimos.topic_for_consumer(handler_class_or_topic)
|
|
192
|
-
consumer = karafka.consumer_for(topic_name)
|
|
193
|
+
topic_name = if handler_class_or_topic.is_a?(String)
|
|
194
|
+
handler_class_or_topic
|
|
195
|
+
else
|
|
196
|
+
Deimos.topic_for_consumer(handler_class_or_topic)
|
|
193
197
|
end
|
|
198
|
+
consumer = karafka.consumer_for(topic_name)
|
|
194
199
|
|
|
195
200
|
Deimos.karafka_config_for(topic: topic_name).each_message(single)
|
|
196
201
|
|
|
197
202
|
# don't record messages sent with test_consume_batch
|
|
198
203
|
original_messages = karafka.produced_messages.dup
|
|
199
204
|
payloads.each_with_index do |payload, i|
|
|
200
|
-
karafka.produce(payload, {key: keys[i], partition_key: partition_keys[i], topic: consumer.topic.name})
|
|
205
|
+
karafka.produce(payload, { key: keys[i], partition_key: partition_keys[i], topic: consumer.topic.name })
|
|
201
206
|
end
|
|
202
207
|
if block_given?
|
|
203
208
|
if single
|
|
204
209
|
allow(consumer).to receive(:consume_message) do
|
|
205
|
-
yield
|
|
210
|
+
yield(consumer.messages.first.payload, consumer.messages.first.as_json['metadata'])
|
|
206
211
|
end
|
|
207
212
|
else
|
|
208
213
|
allow(consumer).to receive(:consume_batch) do
|
|
209
|
-
yield
|
|
214
|
+
yield(consumer.messages)
|
|
210
215
|
end
|
|
211
216
|
end
|
|
212
217
|
end
|
|
@@ -8,14 +8,14 @@ module Deimos
|
|
|
8
8
|
# @param span_name [String] The name of the span/trace
|
|
9
9
|
# @param options [Hash] Options for the span
|
|
10
10
|
# @return [Object] The span object
|
|
11
|
-
def start(
|
|
11
|
+
def start(_span_name, _options={})
|
|
12
12
|
raise MissingImplementationError
|
|
13
13
|
end
|
|
14
14
|
|
|
15
15
|
# Finishes the trace on the span object.
|
|
16
16
|
# @param span [Object] The span to finish trace on
|
|
17
17
|
# @return [void]
|
|
18
|
-
def finish(
|
|
18
|
+
def finish(_span)
|
|
19
19
|
raise MissingImplementationError
|
|
20
20
|
end
|
|
21
21
|
|
|
@@ -23,7 +23,7 @@ module Deimos
|
|
|
23
23
|
# @param span [Object] The span to set error on
|
|
24
24
|
# @param exception [Exception] The exception that occurred
|
|
25
25
|
# @return [void]
|
|
26
|
-
def set_error(
|
|
26
|
+
def set_error(_span, _exception)
|
|
27
27
|
raise MissingImplementationError
|
|
28
28
|
end
|
|
29
29
|
|
|
@@ -38,13 +38,13 @@ module Deimos
|
|
|
38
38
|
# @param value [String]
|
|
39
39
|
# @param span [Object]
|
|
40
40
|
# @return [void]
|
|
41
|
-
def set_tag(
|
|
41
|
+
def set_tag(_tag, _value, _span=nil)
|
|
42
42
|
raise MissingImplementationError
|
|
43
43
|
end
|
|
44
44
|
|
|
45
45
|
# Get a tag from a span with the specified tag.
|
|
46
46
|
# @param tag [String]
|
|
47
|
-
def get_tag(
|
|
47
|
+
def get_tag(_tag)
|
|
48
48
|
raise MissingImplementationError
|
|
49
49
|
end
|
|
50
50
|
|
data/lib/deimos/transcoder.rb
CHANGED
|
@@ -1,7 +1,11 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
module Deimos
|
|
2
4
|
class Transcoder
|
|
3
5
|
|
|
4
|
-
attr_accessor :key_field, :
|
|
6
|
+
attr_accessor :key_field, :backend_type
|
|
7
|
+
# @param backend [Class<Deimos::SchemaBackends::Base>]
|
|
8
|
+
attr_writer :backend
|
|
5
9
|
|
|
6
10
|
# @param schema [String]
|
|
7
11
|
# @param namespace [String]
|
|
@@ -18,7 +22,7 @@ module Deimos
|
|
|
18
22
|
@topic = topic
|
|
19
23
|
end
|
|
20
24
|
|
|
21
|
-
# @return [Class
|
|
25
|
+
# @return [Class<Deimos::SchemaBackends::Base>]
|
|
22
26
|
def backend
|
|
23
27
|
@backend ||= Deimos.schema_backend(schema: @schema,
|
|
24
28
|
namespace: @namespace,
|