deimos-ruby 1.24.2 → 2.0.0.pre.alpha1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop_todo.yml +0 -17
- data/.tool-versions +1 -0
- data/CHANGELOG.md +5 -0
- data/README.md +287 -498
- data/deimos-ruby.gemspec +4 -4
- data/docs/CONFIGURATION.md +133 -226
- data/docs/UPGRADING.md +237 -0
- data/lib/deimos/active_record_consume/batch_consumption.rb +29 -28
- data/lib/deimos/active_record_consume/mass_updater.rb +59 -4
- data/lib/deimos/active_record_consume/message_consumption.rb +15 -21
- data/lib/deimos/active_record_consumer.rb +36 -21
- data/lib/deimos/active_record_producer.rb +28 -9
- data/lib/deimos/backends/base.rb +4 -35
- data/lib/deimos/backends/kafka.rb +6 -22
- data/lib/deimos/backends/kafka_async.rb +6 -22
- data/lib/deimos/backends/{db.rb → outbox.rb} +13 -9
- data/lib/deimos/config/configuration.rb +116 -379
- data/lib/deimos/consume/batch_consumption.rb +24 -124
- data/lib/deimos/consume/message_consumption.rb +36 -63
- data/lib/deimos/consumer.rb +16 -75
- data/lib/deimos/ext/consumer_route.rb +35 -0
- data/lib/deimos/ext/producer_middleware.rb +94 -0
- data/lib/deimos/ext/producer_route.rb +22 -0
- data/lib/deimos/ext/redraw.rb +29 -0
- data/lib/deimos/ext/routing_defaults.rb +72 -0
- data/lib/deimos/ext/schema_route.rb +70 -0
- data/lib/deimos/kafka_message.rb +2 -2
- data/lib/deimos/kafka_source.rb +2 -7
- data/lib/deimos/kafka_topic_info.rb +1 -1
- data/lib/deimos/logging.rb +71 -0
- data/lib/deimos/message.rb +2 -11
- data/lib/deimos/metrics/datadog.rb +40 -1
- data/lib/deimos/metrics/provider.rb +4 -4
- data/lib/deimos/producer.rb +39 -116
- data/lib/deimos/railtie.rb +6 -0
- data/lib/deimos/schema_backends/avro_base.rb +21 -21
- data/lib/deimos/schema_backends/avro_schema_registry.rb +1 -2
- data/lib/deimos/schema_backends/avro_validation.rb +2 -2
- data/lib/deimos/schema_backends/base.rb +19 -12
- data/lib/deimos/schema_backends/mock.rb +6 -1
- data/lib/deimos/schema_backends/plain.rb +47 -0
- data/lib/deimos/schema_class/base.rb +2 -2
- data/lib/deimos/schema_class/enum.rb +1 -1
- data/lib/deimos/schema_class/record.rb +2 -2
- data/lib/deimos/test_helpers.rb +95 -320
- data/lib/deimos/tracing/provider.rb +6 -6
- data/lib/deimos/transcoder.rb +88 -0
- data/lib/deimos/utils/db_poller/base.rb +16 -14
- data/lib/deimos/utils/db_poller/state_based.rb +3 -3
- data/lib/deimos/utils/db_poller/time_based.rb +4 -4
- data/lib/deimos/utils/db_poller.rb +1 -1
- data/lib/deimos/utils/deadlock_retry.rb +1 -1
- data/lib/deimos/utils/{db_producer.rb → outbox_producer.rb} +16 -47
- data/lib/deimos/utils/schema_class.rb +0 -7
- data/lib/deimos/version.rb +1 -1
- data/lib/deimos.rb +79 -26
- data/lib/generators/deimos/{db_backend_generator.rb → outbox_backend_generator.rb} +4 -4
- data/lib/generators/deimos/schema_class_generator.rb +0 -1
- data/lib/generators/deimos/v2/templates/karafka.rb.tt +149 -0
- data/lib/generators/deimos/v2_generator.rb +193 -0
- data/lib/tasks/deimos.rake +5 -7
- data/spec/active_record_batch_consumer_association_spec.rb +22 -13
- data/spec/active_record_batch_consumer_spec.rb +84 -65
- data/spec/active_record_consume/batch_consumption_spec.rb +10 -10
- data/spec/active_record_consume/batch_slicer_spec.rb +12 -12
- data/spec/active_record_consume/mass_updater_spec.rb +137 -0
- data/spec/active_record_consumer_spec.rb +29 -13
- data/spec/active_record_producer_spec.rb +36 -26
- data/spec/backends/base_spec.rb +0 -23
- data/spec/backends/kafka_async_spec.rb +1 -3
- data/spec/backends/kafka_spec.rb +1 -3
- data/spec/backends/{db_spec.rb → outbox_spec.rb} +14 -20
- data/spec/batch_consumer_spec.rb +66 -116
- data/spec/consumer_spec.rb +53 -147
- data/spec/deimos_spec.rb +10 -126
- data/spec/kafka_source_spec.rb +19 -52
- data/spec/karafka/karafka.rb +69 -0
- data/spec/karafka_config/karafka_spec.rb +97 -0
- data/spec/logging_spec.rb +25 -0
- data/spec/message_spec.rb +9 -9
- data/spec/producer_spec.rb +112 -254
- data/spec/rake_spec.rb +1 -3
- data/spec/schema_backends/avro_validation_spec.rb +1 -1
- data/spec/schemas/com/my-namespace/MySchemaWithTitle.avsc +22 -0
- data/spec/snapshots/consumers-no-nest.snap +49 -0
- data/spec/snapshots/consumers.snap +49 -0
- data/spec/snapshots/consumers_and_producers-no-nest.snap +49 -0
- data/spec/snapshots/consumers_and_producers.snap +49 -0
- data/spec/snapshots/consumers_circular-no-nest.snap +49 -0
- data/spec/snapshots/consumers_circular.snap +49 -0
- data/spec/snapshots/consumers_complex_types-no-nest.snap +49 -0
- data/spec/snapshots/consumers_complex_types.snap +49 -0
- data/spec/snapshots/consumers_nested-no-nest.snap +49 -0
- data/spec/snapshots/consumers_nested.snap +49 -0
- data/spec/snapshots/namespace_folders.snap +49 -0
- data/spec/snapshots/namespace_map.snap +49 -0
- data/spec/snapshots/producers_with_key-no-nest.snap +49 -0
- data/spec/snapshots/producers_with_key.snap +49 -0
- data/spec/spec_helper.rb +61 -29
- data/spec/utils/db_poller_spec.rb +49 -39
- data/spec/utils/{db_producer_spec.rb → outbox_producer_spec.rb} +17 -184
- metadata +58 -67
- data/lib/deimos/batch_consumer.rb +0 -7
- data/lib/deimos/config/phobos_config.rb +0 -163
- data/lib/deimos/instrumentation.rb +0 -95
- data/lib/deimos/monkey_patches/phobos_cli.rb +0 -35
- data/lib/deimos/utils/inline_consumer.rb +0 -158
- data/lib/deimos/utils/lag_reporter.rb +0 -186
- data/lib/deimos/utils/schema_controller_mixin.rb +0 -129
- data/spec/config/configuration_spec.rb +0 -321
- data/spec/kafka_listener_spec.rb +0 -55
- data/spec/phobos.bad_db.yml +0 -73
- data/spec/phobos.yml +0 -77
- data/spec/utils/inline_consumer_spec.rb +0 -31
- data/spec/utils/lag_reporter_spec.rb +0 -76
- data/spec/utils/platform_schema_validation_spec.rb +0 -0
- data/spec/utils/schema_controller_mixin_spec.rb +0 -84
- /data/lib/generators/deimos/{db_backend → outbox_backend}/templates/migration +0 -0
- /data/lib/generators/deimos/{db_backend → outbox_backend}/templates/rails3_migration +0 -0
@@ -18,9 +18,11 @@ module Deimos
|
|
18
18
|
# a record object, refetch the record to pass into the `generate_payload`
|
19
19
|
# method.
|
20
20
|
# @return [void]
|
21
|
-
def record_class(klass, refetch: true)
|
22
|
-
|
23
|
-
|
21
|
+
def record_class(klass=nil, refetch: true)
|
22
|
+
return @record_class if klass.nil?
|
23
|
+
|
24
|
+
@record_class = klass
|
25
|
+
@refetch_record = refetch
|
24
26
|
end
|
25
27
|
|
26
28
|
# @param record [ActiveRecord::Base]
|
@@ -34,14 +36,14 @@ module Deimos
|
|
34
36
|
# @param force_send [Boolean]
|
35
37
|
# @return [void]
|
36
38
|
def send_events(records, force_send: false)
|
37
|
-
primary_key =
|
39
|
+
primary_key = @record_class&.primary_key
|
38
40
|
messages = records.map do |record|
|
39
41
|
if record.respond_to?(:attributes)
|
40
42
|
attrs = record.attributes.with_indifferent_access
|
41
43
|
else
|
42
44
|
attrs = record.with_indifferent_access
|
43
|
-
if
|
44
|
-
record =
|
45
|
+
if @refetch_record && attrs[primary_key]
|
46
|
+
record = @record_class.find(attrs[primary_key])
|
45
47
|
end
|
46
48
|
end
|
47
49
|
generate_payload(attrs, record).with_indifferent_access
|
@@ -50,6 +52,15 @@ module Deimos
|
|
50
52
|
self.post_process(records)
|
51
53
|
end
|
52
54
|
|
55
|
+
def config
|
56
|
+
Deimos.karafka_configs.find { |t| t.producer_class == self }
|
57
|
+
end
|
58
|
+
|
59
|
+
def encoder
|
60
|
+
raise "No schema or namespace configured for #{self.name}" if config.nil?
|
61
|
+
config.deserializers[:payload].backend
|
62
|
+
end
|
63
|
+
|
53
64
|
# Generate the payload, given a list of attributes or a record..
|
54
65
|
# Can be overridden or added to by subclasses.
|
55
66
|
# @param attributes [Hash]
|
@@ -62,9 +73,9 @@ module Deimos
|
|
62
73
|
payload.delete_if do |k, _|
|
63
74
|
k.to_sym != :payload_key && !fields.map(&:name).include?(k)
|
64
75
|
end
|
65
|
-
return payload unless
|
76
|
+
return payload unless self.config.use_schema_classes
|
66
77
|
|
67
|
-
Utils::SchemaClass.instance(payload,
|
78
|
+
Utils::SchemaClass.instance(payload, encoder.schema, encoder.namespace)
|
68
79
|
end
|
69
80
|
|
70
81
|
# Query to use when polling the database with the DbPoller. Add
|
@@ -76,7 +87,7 @@ module Deimos
|
|
76
87
|
# than this value).
|
77
88
|
# @return [ActiveRecord::Relation]
|
78
89
|
def poll_query(time_from:, time_to:, column_name: :updated_at, min_id:)
|
79
|
-
klass =
|
90
|
+
klass = @record_class
|
80
91
|
table = ActiveRecord::Base.connection.quote_table_name(klass.table_name)
|
81
92
|
column = ActiveRecord::Base.connection.quote_column_name(column_name)
|
82
93
|
primary = ActiveRecord::Base.connection.quote_column_name(klass.primary_key)
|
@@ -95,6 +106,14 @@ module Deimos
|
|
95
106
|
def post_process(_records)
|
96
107
|
end
|
97
108
|
|
109
|
+
# Override this in active record producers to add
|
110
|
+
# non-schema fields to check for updates
|
111
|
+
# @param _record [ActiveRecord::Base]
|
112
|
+
# @return [Array<String>] fields to check for updates
|
113
|
+
def watched_attributes(_record)
|
114
|
+
self.encoder.schema_fields.map(&:name)
|
115
|
+
end
|
116
|
+
|
98
117
|
end
|
99
118
|
end
|
100
119
|
end
|
data/lib/deimos/backends/base.rb
CHANGED
@@ -6,10 +6,11 @@ module Deimos
|
|
6
6
|
class Base
|
7
7
|
class << self
|
8
8
|
# @param producer_class [Class<Deimos::Producer>]
|
9
|
-
# @param messages [Array<
|
9
|
+
# @param messages [Array<Hash>]
|
10
10
|
# @return [void]
|
11
11
|
def publish(producer_class:, messages:)
|
12
|
-
Deimos.
|
12
|
+
message = ::Deimos::Logging.messages_log_text(producer_class.karafka_config.payload_log, messages)
|
13
|
+
Deimos::Logging.log_info({message: 'Publishing Messages:'}.merge(message))
|
13
14
|
execute(producer_class: producer_class, messages: messages)
|
14
15
|
end
|
15
16
|
|
@@ -17,43 +18,11 @@ module Deimos
|
|
17
18
|
# @param messages [Array<Deimos::Message>]
|
18
19
|
# @return [void]
|
19
20
|
def execute(producer_class:, messages:)
|
20
|
-
raise
|
21
|
+
raise MissingImplementationError
|
21
22
|
end
|
22
23
|
|
23
24
|
private
|
24
25
|
|
25
|
-
def log_message(messages)
|
26
|
-
log_message = {
|
27
|
-
message: 'Publishing messages',
|
28
|
-
topic: messages.first&.topic
|
29
|
-
}
|
30
|
-
|
31
|
-
case Deimos.config.payload_log
|
32
|
-
when :keys
|
33
|
-
log_message.merge!(
|
34
|
-
payload_keys: messages.map(&:key)
|
35
|
-
)
|
36
|
-
when :count
|
37
|
-
log_message.merge!(
|
38
|
-
payloads_count: messages.count
|
39
|
-
)
|
40
|
-
when :headers
|
41
|
-
log_message.merge!(
|
42
|
-
payload_headers: messages.map(&:headers)
|
43
|
-
)
|
44
|
-
else
|
45
|
-
log_message.merge!(
|
46
|
-
payloads: messages.map do |message|
|
47
|
-
{
|
48
|
-
payload: message.payload,
|
49
|
-
key: message.key
|
50
|
-
}
|
51
|
-
end
|
52
|
-
)
|
53
|
-
end
|
54
|
-
|
55
|
-
log_message
|
56
|
-
end
|
57
26
|
end
|
58
27
|
end
|
59
28
|
end
|
@@ -4,30 +4,14 @@ module Deimos
|
|
4
4
|
module Backends
|
5
5
|
# Default backend to produce to Kafka.
|
6
6
|
class Kafka < Base
|
7
|
-
include Phobos::Producer
|
8
|
-
|
9
|
-
# Shut down the producer if necessary.
|
10
|
-
# @return [void]
|
11
|
-
def self.shutdown_producer
|
12
|
-
producer.sync_producer_shutdown if producer.respond_to?(:sync_producer_shutdown)
|
13
|
-
producer.kafka_client&.close
|
14
|
-
end
|
15
|
-
|
16
7
|
# :nodoc:
|
17
8
|
def self.execute(producer_class:, messages:)
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
)
|
24
|
-
producer.publish_list(messages.map(&:encoded_hash))
|
25
|
-
Deimos.config.metrics&.increment(
|
26
|
-
'publish',
|
27
|
-
tags: %W(status:success topic:#{producer_class.topic}),
|
28
|
-
by: messages.size
|
29
|
-
)
|
30
|
-
end
|
9
|
+
Karafka.producer.produce_many_sync(messages)
|
10
|
+
Deimos.config.metrics&.increment(
|
11
|
+
'publish',
|
12
|
+
tags: %W(status:success topic:#{messages.first[:topic]}),
|
13
|
+
by: messages.size
|
14
|
+
)
|
31
15
|
end
|
32
16
|
end
|
33
17
|
end
|
@@ -4,30 +4,14 @@ module Deimos
|
|
4
4
|
module Backends
|
5
5
|
# Backend which produces to Kafka via an async producer.
|
6
6
|
class KafkaAsync < Base
|
7
|
-
include Phobos::Producer
|
8
|
-
|
9
|
-
# Shut down the producer cleanly.
|
10
|
-
# @return [void]
|
11
|
-
def self.shutdown_producer
|
12
|
-
producer.async_producer_shutdown
|
13
|
-
producer.kafka_client&.close
|
14
|
-
end
|
15
|
-
|
16
7
|
# :nodoc:
|
17
8
|
def self.execute(producer_class:, messages:)
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
)
|
24
|
-
producer.async_publish_list(messages.map(&:encoded_hash))
|
25
|
-
Deimos.config.metrics&.increment(
|
26
|
-
'publish',
|
27
|
-
tags: %W(status:success topic:#{producer_class.topic}),
|
28
|
-
by: messages.size
|
29
|
-
)
|
30
|
-
end
|
9
|
+
Karafka.producer.produce_many_async(messages)
|
10
|
+
Deimos.config.metrics&.increment(
|
11
|
+
'publish',
|
12
|
+
tags: %W(status:success topic:#{messages.first[:topic]}),
|
13
|
+
by: messages.size
|
14
|
+
)
|
31
15
|
end
|
32
16
|
end
|
33
17
|
end
|
@@ -6,22 +6,23 @@ module Deimos
|
|
6
6
|
module Backends
|
7
7
|
# Backend which saves messages to the database instead of immediately
|
8
8
|
# sending them.
|
9
|
-
class
|
9
|
+
class Outbox < Base
|
10
10
|
class << self
|
11
11
|
# :nodoc:
|
12
12
|
def execute(producer_class:, messages:)
|
13
13
|
records = messages.map do |m|
|
14
|
+
Deimos::ProducerMiddleware.call(m)
|
14
15
|
message = Deimos::KafkaMessage.new(
|
15
|
-
message: m
|
16
|
-
topic: m
|
16
|
+
message: m[:payload] ? m[:payload].to_s.b : nil,
|
17
|
+
topic: m[:topic],
|
17
18
|
partition_key: partition_key_for(m)
|
18
19
|
)
|
19
|
-
message.key = m.
|
20
|
+
message.key = m[:key].to_s.b if m[:key]
|
20
21
|
message
|
21
22
|
end
|
22
23
|
Deimos::KafkaMessage.import(records)
|
23
24
|
Deimos.config.metrics&.increment(
|
24
|
-
'
|
25
|
+
'outbox.insert',
|
25
26
|
tags: %W(topic:#{producer_class.topic}),
|
26
27
|
by: records.size
|
27
28
|
)
|
@@ -30,10 +31,13 @@ module Deimos
|
|
30
31
|
# @param message [Deimos::Message]
|
31
32
|
# @return [String] the partition key to use for this message
|
32
33
|
def partition_key_for(message)
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
34
|
+
if message[:partition_key].present?
|
35
|
+
message[:partition_key]
|
36
|
+
elsif message[:key].present?
|
37
|
+
message[:key].to_s.b
|
38
|
+
else
|
39
|
+
nil
|
40
|
+
end
|
37
41
|
end
|
38
42
|
end
|
39
43
|
end
|