deimos-ruby 1.24.3 → 2.0.0.pre.alpha1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop_todo.yml +0 -17
- data/.tool-versions +1 -0
- data/CHANGELOG.md +1 -1
- data/README.md +287 -498
- data/deimos-ruby.gemspec +4 -4
- data/docs/CONFIGURATION.md +133 -227
- data/docs/UPGRADING.md +237 -0
- data/lib/deimos/active_record_consume/batch_consumption.rb +28 -29
- data/lib/deimos/active_record_consume/message_consumption.rb +15 -21
- data/lib/deimos/active_record_consumer.rb +36 -26
- data/lib/deimos/active_record_producer.rb +28 -9
- data/lib/deimos/backends/base.rb +4 -35
- data/lib/deimos/backends/kafka.rb +6 -22
- data/lib/deimos/backends/kafka_async.rb +6 -22
- data/lib/deimos/backends/{db.rb → outbox.rb} +13 -9
- data/lib/deimos/config/configuration.rb +116 -385
- data/lib/deimos/consume/batch_consumption.rb +24 -124
- data/lib/deimos/consume/message_consumption.rb +36 -63
- data/lib/deimos/consumer.rb +16 -75
- data/lib/deimos/ext/consumer_route.rb +35 -0
- data/lib/deimos/ext/producer_middleware.rb +94 -0
- data/lib/deimos/ext/producer_route.rb +22 -0
- data/lib/deimos/ext/redraw.rb +29 -0
- data/lib/deimos/ext/routing_defaults.rb +72 -0
- data/lib/deimos/ext/schema_route.rb +70 -0
- data/lib/deimos/kafka_message.rb +2 -2
- data/lib/deimos/kafka_source.rb +2 -7
- data/lib/deimos/kafka_topic_info.rb +1 -1
- data/lib/deimos/logging.rb +71 -0
- data/lib/deimos/message.rb +2 -11
- data/lib/deimos/metrics/datadog.rb +40 -1
- data/lib/deimos/metrics/provider.rb +4 -4
- data/lib/deimos/producer.rb +39 -116
- data/lib/deimos/railtie.rb +6 -0
- data/lib/deimos/schema_backends/avro_base.rb +21 -21
- data/lib/deimos/schema_backends/avro_schema_registry.rb +1 -2
- data/lib/deimos/schema_backends/avro_validation.rb +2 -2
- data/lib/deimos/schema_backends/base.rb +19 -12
- data/lib/deimos/schema_backends/mock.rb +6 -1
- data/lib/deimos/schema_backends/plain.rb +47 -0
- data/lib/deimos/schema_class/base.rb +2 -2
- data/lib/deimos/schema_class/enum.rb +1 -1
- data/lib/deimos/schema_class/record.rb +2 -2
- data/lib/deimos/test_helpers.rb +95 -320
- data/lib/deimos/tracing/provider.rb +6 -6
- data/lib/deimos/transcoder.rb +88 -0
- data/lib/deimos/utils/db_poller/base.rb +16 -14
- data/lib/deimos/utils/db_poller/state_based.rb +3 -3
- data/lib/deimos/utils/db_poller/time_based.rb +4 -4
- data/lib/deimos/utils/db_poller.rb +1 -1
- data/lib/deimos/utils/deadlock_retry.rb +1 -1
- data/lib/deimos/utils/{db_producer.rb → outbox_producer.rb} +16 -47
- data/lib/deimos/utils/schema_class.rb +0 -7
- data/lib/deimos/version.rb +1 -1
- data/lib/deimos.rb +79 -26
- data/lib/generators/deimos/{db_backend_generator.rb → outbox_backend_generator.rb} +4 -4
- data/lib/generators/deimos/schema_class_generator.rb +0 -1
- data/lib/generators/deimos/v2/templates/karafka.rb.tt +149 -0
- data/lib/generators/deimos/v2_generator.rb +193 -0
- data/lib/tasks/deimos.rake +5 -7
- data/spec/active_record_batch_consumer_association_spec.rb +22 -13
- data/spec/active_record_batch_consumer_spec.rb +84 -65
- data/spec/active_record_consume/batch_consumption_spec.rb +10 -10
- data/spec/active_record_consume/batch_slicer_spec.rb +12 -12
- data/spec/active_record_consumer_spec.rb +29 -13
- data/spec/active_record_producer_spec.rb +36 -26
- data/spec/backends/base_spec.rb +0 -23
- data/spec/backends/kafka_async_spec.rb +1 -3
- data/spec/backends/kafka_spec.rb +1 -3
- data/spec/backends/{db_spec.rb → outbox_spec.rb} +14 -20
- data/spec/batch_consumer_spec.rb +66 -116
- data/spec/consumer_spec.rb +53 -147
- data/spec/deimos_spec.rb +10 -126
- data/spec/kafka_source_spec.rb +19 -52
- data/spec/karafka/karafka.rb +69 -0
- data/spec/karafka_config/karafka_spec.rb +97 -0
- data/spec/logging_spec.rb +25 -0
- data/spec/message_spec.rb +9 -9
- data/spec/producer_spec.rb +112 -254
- data/spec/rake_spec.rb +1 -3
- data/spec/schema_backends/avro_validation_spec.rb +1 -1
- data/spec/schemas/com/my-namespace/MySchemaWithTitle.avsc +22 -0
- data/spec/snapshots/consumers-no-nest.snap +49 -0
- data/spec/snapshots/consumers.snap +49 -0
- data/spec/snapshots/consumers_and_producers-no-nest.snap +49 -0
- data/spec/snapshots/consumers_and_producers.snap +49 -0
- data/spec/snapshots/consumers_circular-no-nest.snap +49 -0
- data/spec/snapshots/consumers_circular.snap +49 -0
- data/spec/snapshots/consumers_complex_types-no-nest.snap +49 -0
- data/spec/snapshots/consumers_complex_types.snap +49 -0
- data/spec/snapshots/consumers_nested-no-nest.snap +49 -0
- data/spec/snapshots/consumers_nested.snap +49 -0
- data/spec/snapshots/namespace_folders.snap +49 -0
- data/spec/snapshots/namespace_map.snap +49 -0
- data/spec/snapshots/producers_with_key-no-nest.snap +49 -0
- data/spec/snapshots/producers_with_key.snap +49 -0
- data/spec/spec_helper.rb +61 -29
- data/spec/utils/db_poller_spec.rb +49 -39
- data/spec/utils/{db_producer_spec.rb → outbox_producer_spec.rb} +17 -184
- metadata +58 -67
- data/lib/deimos/batch_consumer.rb +0 -7
- data/lib/deimos/config/phobos_config.rb +0 -164
- data/lib/deimos/instrumentation.rb +0 -95
- data/lib/deimos/monkey_patches/phobos_cli.rb +0 -35
- data/lib/deimos/utils/inline_consumer.rb +0 -158
- data/lib/deimos/utils/lag_reporter.rb +0 -186
- data/lib/deimos/utils/schema_controller_mixin.rb +0 -129
- data/spec/config/configuration_spec.rb +0 -329
- data/spec/kafka_listener_spec.rb +0 -55
- data/spec/phobos.bad_db.yml +0 -73
- data/spec/phobos.yml +0 -77
- data/spec/utils/inline_consumer_spec.rb +0 -31
- data/spec/utils/lag_reporter_spec.rb +0 -76
- data/spec/utils/platform_schema_validation_spec.rb +0 -0
- data/spec/utils/schema_controller_mixin_spec.rb +0 -84
- /data/lib/generators/deimos/{db_backend → outbox_backend}/templates/migration +0 -0
- /data/lib/generators/deimos/{db_backend → outbox_backend}/templates/rails3_migration +0 -0
@@ -79,7 +79,7 @@ module Deimos
|
|
79
79
|
# Indicate a class which should act as a mocked version of this backend.
|
80
80
|
# This class should perform all validations but not actually do any
|
81
81
|
# encoding.
|
82
|
-
# Note that the "mock" version
|
82
|
+
# Note that the "mock" version should return
|
83
83
|
# its own symbol when this is called, since it may be called multiple
|
84
84
|
# times depending on the order of RSpec helpers.
|
85
85
|
# @return [Symbol]
|
@@ -90,7 +90,7 @@ module Deimos
|
|
90
90
|
# The content type to use when encoding / decoding requests over HTTP via ActionController.
|
91
91
|
# @return [String]
|
92
92
|
def self.content_type
|
93
|
-
raise
|
93
|
+
raise MissingImplementationError
|
94
94
|
end
|
95
95
|
|
96
96
|
# Converts your schema to String form for generated YARD docs.
|
@@ -98,7 +98,7 @@ module Deimos
|
|
98
98
|
# @param schema [Object]
|
99
99
|
# @return [String] A string representation of the Type
|
100
100
|
def self.field_type(schema)
|
101
|
-
raise
|
101
|
+
raise MissingImplementationError
|
102
102
|
end
|
103
103
|
|
104
104
|
# Encode a payload. To be defined by subclass.
|
@@ -107,7 +107,7 @@ module Deimos
|
|
107
107
|
# @param topic [String]
|
108
108
|
# @return [String]
|
109
109
|
def encode_payload(payload, schema:, topic: nil)
|
110
|
-
raise
|
110
|
+
raise MissingImplementationError
|
111
111
|
end
|
112
112
|
|
113
113
|
# Decode a payload. To be defined by subclass.
|
@@ -115,7 +115,7 @@ module Deimos
|
|
115
115
|
# @param schema [String,Symbol]
|
116
116
|
# @return [Hash]
|
117
117
|
def decode_payload(payload, schema:)
|
118
|
-
raise
|
118
|
+
raise MissingImplementationError
|
119
119
|
end
|
120
120
|
|
121
121
|
# Validate that a payload matches the schema. To be defined by subclass.
|
@@ -123,13 +123,13 @@ module Deimos
|
|
123
123
|
# @param schema [String,Symbol]
|
124
124
|
# @return [void]
|
125
125
|
def validate(payload, schema:)
|
126
|
-
raise
|
126
|
+
raise MissingImplementationError
|
127
127
|
end
|
128
128
|
|
129
129
|
# List of field names belonging to the schema. To be defined by subclass.
|
130
130
|
# @return [Array<SchemaField>]
|
131
131
|
def schema_fields
|
132
|
-
raise
|
132
|
+
raise MissingImplementationError
|
133
133
|
end
|
134
134
|
|
135
135
|
# Given a value and a field definition (as defined by whatever the
|
@@ -139,7 +139,7 @@ module Deimos
|
|
139
139
|
# @param value [Object]
|
140
140
|
# @return [Object]
|
141
141
|
def coerce_field(field, value)
|
142
|
-
raise
|
142
|
+
raise MissingImplementationError
|
143
143
|
end
|
144
144
|
|
145
145
|
# Given a field definition, return the SQL type that might be used in
|
@@ -150,7 +150,14 @@ module Deimos
|
|
150
150
|
# @param field [SchemaField]
|
151
151
|
# @return [Symbol]
|
152
152
|
def sql_type(field)
|
153
|
-
raise
|
153
|
+
raise MissingImplementationError
|
154
|
+
end
|
155
|
+
|
156
|
+
# Generate a key schema from the given value schema and key ID. This
|
157
|
+
# is used when encoding or decoding keys from an existing value schema.
|
158
|
+
# @param field_name [Symbol]
|
159
|
+
def generate_key_schema(field_name)
|
160
|
+
raise MissingImplementationError
|
154
161
|
end
|
155
162
|
|
156
163
|
# Encode a message key. To be defined by subclass.
|
@@ -159,7 +166,7 @@ module Deimos
|
|
159
166
|
# @param topic [String]
|
160
167
|
# @return [String]
|
161
168
|
def encode_key(key, key_id, topic: nil)
|
162
|
-
raise
|
169
|
+
raise MissingImplementationError
|
163
170
|
end
|
164
171
|
|
165
172
|
# Decode a message key. To be defined by subclass.
|
@@ -167,13 +174,13 @@ module Deimos
|
|
167
174
|
# @param key_id [String,Symbol] the field in the message to decode.
|
168
175
|
# @return [String]
|
169
176
|
def decode_key(payload, key_id)
|
170
|
-
raise
|
177
|
+
raise MissingImplementationError
|
171
178
|
end
|
172
179
|
|
173
180
|
# Forcefully loads the schema into memory.
|
174
181
|
# @return [Object] The schema that is of use.
|
175
182
|
def load_schema
|
176
|
-
raise
|
183
|
+
raise MissingImplementationError
|
177
184
|
end
|
178
185
|
end
|
179
186
|
end
|
@@ -4,6 +4,11 @@ module Deimos
|
|
4
4
|
module SchemaBackends
|
5
5
|
# Mock implementation of a schema backend that does no encoding or validation.
|
6
6
|
class Mock < Base
|
7
|
+
|
8
|
+
# @override
|
9
|
+
def generate_key_schema(field_name)
|
10
|
+
end
|
11
|
+
|
7
12
|
# @override
|
8
13
|
def decode_payload(payload, schema:)
|
9
14
|
payload.is_a?(String) ? 'payload-decoded' : payload.map { |k, v| [k, "decoded-#{v}"] }
|
@@ -11,7 +16,7 @@ module Deimos
|
|
11
16
|
|
12
17
|
# @override
|
13
18
|
def encode_payload(payload, schema:, topic: nil)
|
14
|
-
payload.is_a?(String) ? 'payload-encoded' : payload.map { |k, v| [k, "encoded-#{v}"] }
|
19
|
+
payload.is_a?(String) ? 'payload-encoded' : payload.map { |k, v| [k, "encoded-#{v}"] }.to_json
|
15
20
|
end
|
16
21
|
|
17
22
|
# @override
|
@@ -0,0 +1,47 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Deimos
|
4
|
+
module SchemaBackends
|
5
|
+
# Schema backend that passes through as a basic string.
|
6
|
+
class Plain < Base
|
7
|
+
|
8
|
+
# @override
|
9
|
+
def generate_key_schema(field_name)
|
10
|
+
end
|
11
|
+
|
12
|
+
# @override
|
13
|
+
def decode_payload(payload, schema:)
|
14
|
+
payload
|
15
|
+
end
|
16
|
+
|
17
|
+
# @override
|
18
|
+
def encode_payload(payload, schema:, topic: nil)
|
19
|
+
payload.to_s
|
20
|
+
end
|
21
|
+
|
22
|
+
# @override
|
23
|
+
def validate(payload, schema:)
|
24
|
+
end
|
25
|
+
|
26
|
+
# @override
|
27
|
+
def schema_fields
|
28
|
+
[]
|
29
|
+
end
|
30
|
+
|
31
|
+
# @override
|
32
|
+
def coerce_field(_field, value)
|
33
|
+
value
|
34
|
+
end
|
35
|
+
|
36
|
+
# @override
|
37
|
+
def encode_key(key_id, key, topic: nil)
|
38
|
+
key
|
39
|
+
end
|
40
|
+
|
41
|
+
# @override
|
42
|
+
def decode_key(payload, key_id)
|
43
|
+
payload[key_id]
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
@@ -15,7 +15,7 @@ module Deimos
|
|
15
15
|
# @param _opts [Hash]
|
16
16
|
# @return [Hash] a hash representation of the payload
|
17
17
|
def as_json(_opts={})
|
18
|
-
raise
|
18
|
+
raise MissingImplementationError
|
19
19
|
end
|
20
20
|
|
21
21
|
# @param key [String,Symbol]
|
@@ -46,7 +46,7 @@ module Deimos
|
|
46
46
|
# @param value [Object]
|
47
47
|
# @return [SchemaClass::Base]
|
48
48
|
def self.initialize_from_value(value)
|
49
|
-
raise
|
49
|
+
raise MissingImplementationError
|
50
50
|
end
|
51
51
|
|
52
52
|
protected
|
@@ -46,13 +46,13 @@ module Deimos
|
|
46
46
|
# Returns the schema name of the inheriting class.
|
47
47
|
# @return [String]
|
48
48
|
def schema
|
49
|
-
raise
|
49
|
+
raise MissingImplementationError
|
50
50
|
end
|
51
51
|
|
52
52
|
# Returns the namespace for the schema of the inheriting class.
|
53
53
|
# @return [String]
|
54
54
|
def namespace
|
55
|
-
raise
|
55
|
+
raise MissingImplementationError
|
56
56
|
end
|
57
57
|
|
58
58
|
# Returns the full schema name of the inheriting class.
|