deimos-ruby 1.24.2 → 2.0.0.pre.alpha1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop_todo.yml +0 -17
- data/.tool-versions +1 -0
- data/CHANGELOG.md +5 -0
- data/README.md +287 -498
- data/deimos-ruby.gemspec +4 -4
- data/docs/CONFIGURATION.md +133 -226
- data/docs/UPGRADING.md +237 -0
- data/lib/deimos/active_record_consume/batch_consumption.rb +29 -28
- data/lib/deimos/active_record_consume/mass_updater.rb +59 -4
- data/lib/deimos/active_record_consume/message_consumption.rb +15 -21
- data/lib/deimos/active_record_consumer.rb +36 -21
- data/lib/deimos/active_record_producer.rb +28 -9
- data/lib/deimos/backends/base.rb +4 -35
- data/lib/deimos/backends/kafka.rb +6 -22
- data/lib/deimos/backends/kafka_async.rb +6 -22
- data/lib/deimos/backends/{db.rb → outbox.rb} +13 -9
- data/lib/deimos/config/configuration.rb +116 -379
- data/lib/deimos/consume/batch_consumption.rb +24 -124
- data/lib/deimos/consume/message_consumption.rb +36 -63
- data/lib/deimos/consumer.rb +16 -75
- data/lib/deimos/ext/consumer_route.rb +35 -0
- data/lib/deimos/ext/producer_middleware.rb +94 -0
- data/lib/deimos/ext/producer_route.rb +22 -0
- data/lib/deimos/ext/redraw.rb +29 -0
- data/lib/deimos/ext/routing_defaults.rb +72 -0
- data/lib/deimos/ext/schema_route.rb +70 -0
- data/lib/deimos/kafka_message.rb +2 -2
- data/lib/deimos/kafka_source.rb +2 -7
- data/lib/deimos/kafka_topic_info.rb +1 -1
- data/lib/deimos/logging.rb +71 -0
- data/lib/deimos/message.rb +2 -11
- data/lib/deimos/metrics/datadog.rb +40 -1
- data/lib/deimos/metrics/provider.rb +4 -4
- data/lib/deimos/producer.rb +39 -116
- data/lib/deimos/railtie.rb +6 -0
- data/lib/deimos/schema_backends/avro_base.rb +21 -21
- data/lib/deimos/schema_backends/avro_schema_registry.rb +1 -2
- data/lib/deimos/schema_backends/avro_validation.rb +2 -2
- data/lib/deimos/schema_backends/base.rb +19 -12
- data/lib/deimos/schema_backends/mock.rb +6 -1
- data/lib/deimos/schema_backends/plain.rb +47 -0
- data/lib/deimos/schema_class/base.rb +2 -2
- data/lib/deimos/schema_class/enum.rb +1 -1
- data/lib/deimos/schema_class/record.rb +2 -2
- data/lib/deimos/test_helpers.rb +95 -320
- data/lib/deimos/tracing/provider.rb +6 -6
- data/lib/deimos/transcoder.rb +88 -0
- data/lib/deimos/utils/db_poller/base.rb +16 -14
- data/lib/deimos/utils/db_poller/state_based.rb +3 -3
- data/lib/deimos/utils/db_poller/time_based.rb +4 -4
- data/lib/deimos/utils/db_poller.rb +1 -1
- data/lib/deimos/utils/deadlock_retry.rb +1 -1
- data/lib/deimos/utils/{db_producer.rb → outbox_producer.rb} +16 -47
- data/lib/deimos/utils/schema_class.rb +0 -7
- data/lib/deimos/version.rb +1 -1
- data/lib/deimos.rb +79 -26
- data/lib/generators/deimos/{db_backend_generator.rb → outbox_backend_generator.rb} +4 -4
- data/lib/generators/deimos/schema_class_generator.rb +0 -1
- data/lib/generators/deimos/v2/templates/karafka.rb.tt +149 -0
- data/lib/generators/deimos/v2_generator.rb +193 -0
- data/lib/tasks/deimos.rake +5 -7
- data/spec/active_record_batch_consumer_association_spec.rb +22 -13
- data/spec/active_record_batch_consumer_spec.rb +84 -65
- data/spec/active_record_consume/batch_consumption_spec.rb +10 -10
- data/spec/active_record_consume/batch_slicer_spec.rb +12 -12
- data/spec/active_record_consume/mass_updater_spec.rb +137 -0
- data/spec/active_record_consumer_spec.rb +29 -13
- data/spec/active_record_producer_spec.rb +36 -26
- data/spec/backends/base_spec.rb +0 -23
- data/spec/backends/kafka_async_spec.rb +1 -3
- data/spec/backends/kafka_spec.rb +1 -3
- data/spec/backends/{db_spec.rb → outbox_spec.rb} +14 -20
- data/spec/batch_consumer_spec.rb +66 -116
- data/spec/consumer_spec.rb +53 -147
- data/spec/deimos_spec.rb +10 -126
- data/spec/kafka_source_spec.rb +19 -52
- data/spec/karafka/karafka.rb +69 -0
- data/spec/karafka_config/karafka_spec.rb +97 -0
- data/spec/logging_spec.rb +25 -0
- data/spec/message_spec.rb +9 -9
- data/spec/producer_spec.rb +112 -254
- data/spec/rake_spec.rb +1 -3
- data/spec/schema_backends/avro_validation_spec.rb +1 -1
- data/spec/schemas/com/my-namespace/MySchemaWithTitle.avsc +22 -0
- data/spec/snapshots/consumers-no-nest.snap +49 -0
- data/spec/snapshots/consumers.snap +49 -0
- data/spec/snapshots/consumers_and_producers-no-nest.snap +49 -0
- data/spec/snapshots/consumers_and_producers.snap +49 -0
- data/spec/snapshots/consumers_circular-no-nest.snap +49 -0
- data/spec/snapshots/consumers_circular.snap +49 -0
- data/spec/snapshots/consumers_complex_types-no-nest.snap +49 -0
- data/spec/snapshots/consumers_complex_types.snap +49 -0
- data/spec/snapshots/consumers_nested-no-nest.snap +49 -0
- data/spec/snapshots/consumers_nested.snap +49 -0
- data/spec/snapshots/namespace_folders.snap +49 -0
- data/spec/snapshots/namespace_map.snap +49 -0
- data/spec/snapshots/producers_with_key-no-nest.snap +49 -0
- data/spec/snapshots/producers_with_key.snap +49 -0
- data/spec/spec_helper.rb +61 -29
- data/spec/utils/db_poller_spec.rb +49 -39
- data/spec/utils/{db_producer_spec.rb → outbox_producer_spec.rb} +17 -184
- metadata +58 -67
- data/lib/deimos/batch_consumer.rb +0 -7
- data/lib/deimos/config/phobos_config.rb +0 -163
- data/lib/deimos/instrumentation.rb +0 -95
- data/lib/deimos/monkey_patches/phobos_cli.rb +0 -35
- data/lib/deimos/utils/inline_consumer.rb +0 -158
- data/lib/deimos/utils/lag_reporter.rb +0 -186
- data/lib/deimos/utils/schema_controller_mixin.rb +0 -129
- data/spec/config/configuration_spec.rb +0 -321
- data/spec/kafka_listener_spec.rb +0 -55
- data/spec/phobos.bad_db.yml +0 -73
- data/spec/phobos.yml +0 -77
- data/spec/utils/inline_consumer_spec.rb +0 -31
- data/spec/utils/lag_reporter_spec.rb +0 -76
- data/spec/utils/platform_schema_validation_spec.rb +0 -0
- data/spec/utils/schema_controller_mixin_spec.rb +0 -84
- /data/lib/generators/deimos/{db_backend → outbox_backend}/templates/migration +0 -0
- /data/lib/generators/deimos/{db_backend → outbox_backend}/templates/rails3_migration +0 -0
@@ -79,7 +79,7 @@ module Deimos
|
|
79
79
|
# Indicate a class which should act as a mocked version of this backend.
|
80
80
|
# This class should perform all validations but not actually do any
|
81
81
|
# encoding.
|
82
|
-
# Note that the "mock" version
|
82
|
+
# Note that the "mock" version should return
|
83
83
|
# its own symbol when this is called, since it may be called multiple
|
84
84
|
# times depending on the order of RSpec helpers.
|
85
85
|
# @return [Symbol]
|
@@ -90,7 +90,7 @@ module Deimos
|
|
90
90
|
# The content type to use when encoding / decoding requests over HTTP via ActionController.
|
91
91
|
# @return [String]
|
92
92
|
def self.content_type
|
93
|
-
raise
|
93
|
+
raise MissingImplementationError
|
94
94
|
end
|
95
95
|
|
96
96
|
# Converts your schema to String form for generated YARD docs.
|
@@ -98,7 +98,7 @@ module Deimos
|
|
98
98
|
# @param schema [Object]
|
99
99
|
# @return [String] A string representation of the Type
|
100
100
|
def self.field_type(schema)
|
101
|
-
raise
|
101
|
+
raise MissingImplementationError
|
102
102
|
end
|
103
103
|
|
104
104
|
# Encode a payload. To be defined by subclass.
|
@@ -107,7 +107,7 @@ module Deimos
|
|
107
107
|
# @param topic [String]
|
108
108
|
# @return [String]
|
109
109
|
def encode_payload(payload, schema:, topic: nil)
|
110
|
-
raise
|
110
|
+
raise MissingImplementationError
|
111
111
|
end
|
112
112
|
|
113
113
|
# Decode a payload. To be defined by subclass.
|
@@ -115,7 +115,7 @@ module Deimos
|
|
115
115
|
# @param schema [String,Symbol]
|
116
116
|
# @return [Hash]
|
117
117
|
def decode_payload(payload, schema:)
|
118
|
-
raise
|
118
|
+
raise MissingImplementationError
|
119
119
|
end
|
120
120
|
|
121
121
|
# Validate that a payload matches the schema. To be defined by subclass.
|
@@ -123,13 +123,13 @@ module Deimos
|
|
123
123
|
# @param schema [String,Symbol]
|
124
124
|
# @return [void]
|
125
125
|
def validate(payload, schema:)
|
126
|
-
raise
|
126
|
+
raise MissingImplementationError
|
127
127
|
end
|
128
128
|
|
129
129
|
# List of field names belonging to the schema. To be defined by subclass.
|
130
130
|
# @return [Array<SchemaField>]
|
131
131
|
def schema_fields
|
132
|
-
raise
|
132
|
+
raise MissingImplementationError
|
133
133
|
end
|
134
134
|
|
135
135
|
# Given a value and a field definition (as defined by whatever the
|
@@ -139,7 +139,7 @@ module Deimos
|
|
139
139
|
# @param value [Object]
|
140
140
|
# @return [Object]
|
141
141
|
def coerce_field(field, value)
|
142
|
-
raise
|
142
|
+
raise MissingImplementationError
|
143
143
|
end
|
144
144
|
|
145
145
|
# Given a field definition, return the SQL type that might be used in
|
@@ -150,7 +150,14 @@ module Deimos
|
|
150
150
|
# @param field [SchemaField]
|
151
151
|
# @return [Symbol]
|
152
152
|
def sql_type(field)
|
153
|
-
raise
|
153
|
+
raise MissingImplementationError
|
154
|
+
end
|
155
|
+
|
156
|
+
# Generate a key schema from the given value schema and key ID. This
|
157
|
+
# is used when encoding or decoding keys from an existing value schema.
|
158
|
+
# @param field_name [Symbol]
|
159
|
+
def generate_key_schema(field_name)
|
160
|
+
raise MissingImplementationError
|
154
161
|
end
|
155
162
|
|
156
163
|
# Encode a message key. To be defined by subclass.
|
@@ -159,7 +166,7 @@ module Deimos
|
|
159
166
|
# @param topic [String]
|
160
167
|
# @return [String]
|
161
168
|
def encode_key(key, key_id, topic: nil)
|
162
|
-
raise
|
169
|
+
raise MissingImplementationError
|
163
170
|
end
|
164
171
|
|
165
172
|
# Decode a message key. To be defined by subclass.
|
@@ -167,13 +174,13 @@ module Deimos
|
|
167
174
|
# @param key_id [String,Symbol] the field in the message to decode.
|
168
175
|
# @return [String]
|
169
176
|
def decode_key(payload, key_id)
|
170
|
-
raise
|
177
|
+
raise MissingImplementationError
|
171
178
|
end
|
172
179
|
|
173
180
|
# Forcefully loads the schema into memory.
|
174
181
|
# @return [Object] The schema that is of use.
|
175
182
|
def load_schema
|
176
|
-
raise
|
183
|
+
raise MissingImplementationError
|
177
184
|
end
|
178
185
|
end
|
179
186
|
end
|
@@ -4,6 +4,11 @@ module Deimos
|
|
4
4
|
module SchemaBackends
|
5
5
|
# Mock implementation of a schema backend that does no encoding or validation.
|
6
6
|
class Mock < Base
|
7
|
+
|
8
|
+
# @override
|
9
|
+
def generate_key_schema(field_name)
|
10
|
+
end
|
11
|
+
|
7
12
|
# @override
|
8
13
|
def decode_payload(payload, schema:)
|
9
14
|
payload.is_a?(String) ? 'payload-decoded' : payload.map { |k, v| [k, "decoded-#{v}"] }
|
@@ -11,7 +16,7 @@ module Deimos
|
|
11
16
|
|
12
17
|
# @override
|
13
18
|
def encode_payload(payload, schema:, topic: nil)
|
14
|
-
payload.is_a?(String) ? 'payload-encoded' : payload.map { |k, v| [k, "encoded-#{v}"] }
|
19
|
+
payload.is_a?(String) ? 'payload-encoded' : payload.map { |k, v| [k, "encoded-#{v}"] }.to_json
|
15
20
|
end
|
16
21
|
|
17
22
|
# @override
|
@@ -0,0 +1,47 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Deimos
|
4
|
+
module SchemaBackends
|
5
|
+
# Schema backend that passes through as a basic string.
|
6
|
+
class Plain < Base
|
7
|
+
|
8
|
+
# @override
|
9
|
+
def generate_key_schema(field_name)
|
10
|
+
end
|
11
|
+
|
12
|
+
# @override
|
13
|
+
def decode_payload(payload, schema:)
|
14
|
+
payload
|
15
|
+
end
|
16
|
+
|
17
|
+
# @override
|
18
|
+
def encode_payload(payload, schema:, topic: nil)
|
19
|
+
payload.to_s
|
20
|
+
end
|
21
|
+
|
22
|
+
# @override
|
23
|
+
def validate(payload, schema:)
|
24
|
+
end
|
25
|
+
|
26
|
+
# @override
|
27
|
+
def schema_fields
|
28
|
+
[]
|
29
|
+
end
|
30
|
+
|
31
|
+
# @override
|
32
|
+
def coerce_field(_field, value)
|
33
|
+
value
|
34
|
+
end
|
35
|
+
|
36
|
+
# @override
|
37
|
+
def encode_key(key_id, key, topic: nil)
|
38
|
+
key
|
39
|
+
end
|
40
|
+
|
41
|
+
# @override
|
42
|
+
def decode_key(payload, key_id)
|
43
|
+
payload[key_id]
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
@@ -15,7 +15,7 @@ module Deimos
|
|
15
15
|
# @param _opts [Hash]
|
16
16
|
# @return [Hash] a hash representation of the payload
|
17
17
|
def as_json(_opts={})
|
18
|
-
raise
|
18
|
+
raise MissingImplementationError
|
19
19
|
end
|
20
20
|
|
21
21
|
# @param key [String,Symbol]
|
@@ -46,7 +46,7 @@ module Deimos
|
|
46
46
|
# @param value [Object]
|
47
47
|
# @return [SchemaClass::Base]
|
48
48
|
def self.initialize_from_value(value)
|
49
|
-
raise
|
49
|
+
raise MissingImplementationError
|
50
50
|
end
|
51
51
|
|
52
52
|
protected
|
@@ -46,13 +46,13 @@ module Deimos
|
|
46
46
|
# Returns the schema name of the inheriting class.
|
47
47
|
# @return [String]
|
48
48
|
def schema
|
49
|
-
raise
|
49
|
+
raise MissingImplementationError
|
50
50
|
end
|
51
51
|
|
52
52
|
# Returns the namespace for the schema of the inheriting class.
|
53
53
|
# @return [String]
|
54
54
|
def namespace
|
55
|
-
raise
|
55
|
+
raise MissingImplementationError
|
56
56
|
end
|
57
57
|
|
58
58
|
# Returns the full schema name of the inheriting class.
|