deimos-ruby 1.16.3 → 1.16.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/CHANGELOG.md +5 -0
- data/Gemfile +6 -0
- data/README.md +5 -0
- data/lib/deimos/active_record_consume/batch_consumption.rb +7 -2
- data/lib/deimos/active_record_consume/batch_slicer.rb +2 -0
- data/lib/deimos/active_record_consume/message_consumption.rb +8 -4
- data/lib/deimos/active_record_consumer.rb +7 -4
- data/lib/deimos/active_record_producer.rb +3 -0
- data/lib/deimos/backends/base.rb +4 -2
- data/lib/deimos/backends/kafka.rb +1 -0
- data/lib/deimos/backends/kafka_async.rb +1 -0
- data/lib/deimos/config/configuration.rb +4 -0
- data/lib/deimos/config/phobos_config.rb +2 -1
- data/lib/deimos/consume/batch_consumption.rb +8 -1
- data/lib/deimos/consume/message_consumption.rb +4 -1
- data/lib/deimos/instrumentation.rb +11 -4
- data/lib/deimos/kafka_message.rb +1 -0
- data/lib/deimos/kafka_source.rb +5 -0
- data/lib/deimos/kafka_topic_info.rb +4 -0
- data/lib/deimos/message.rb +19 -2
- data/lib/deimos/metrics/datadog.rb +2 -1
- data/lib/deimos/metrics/mock.rb +2 -2
- data/lib/deimos/metrics/provider.rb +6 -0
- data/lib/deimos/monkey_patches/phobos_cli.rb +1 -1
- data/lib/deimos/monkey_patches/phobos_producer.rb +1 -0
- data/lib/deimos/producer.rb +12 -6
- data/lib/deimos/schema_backends/base.rb +31 -17
- data/lib/deimos/schema_backends/mock.rb +2 -2
- data/lib/deimos/schema_class/base.rb +9 -5
- data/lib/deimos/schema_class/enum.rb +4 -2
- data/lib/deimos/schema_class/record.rb +5 -5
- data/lib/deimos/shared_config.rb +6 -2
- data/lib/deimos/test_helpers.rb +21 -4
- data/lib/deimos/tracing/datadog.rb +1 -1
- data/lib/deimos/tracing/mock.rb +4 -3
- data/lib/deimos/tracing/provider.rb +5 -0
- data/lib/deimos/utils/db_poller.rb +9 -1
- data/lib/deimos/utils/db_producer.rb +14 -2
- data/lib/deimos/utils/deadlock_retry.rb +3 -0
- data/lib/deimos/utils/inline_consumer.rb +14 -6
- data/lib/deimos/utils/lag_reporter.rb +11 -0
- data/lib/deimos/utils/schema_controller_mixin.rb +8 -0
- data/lib/deimos/version.rb +1 -1
- data/lib/deimos.rb +3 -2
- data/lib/generators/deimos/active_record_generator.rb +1 -1
- data/lib/generators/deimos/db_backend_generator.rb +1 -0
- data/lib/generators/deimos/db_poller_generator.rb +1 -0
- data/lib/generators/deimos/schema_class/templates/schema_record.rb.tt +1 -1
- data/lib/generators/deimos/schema_class_generator.rb +12 -3
- data/rbs_collection.lock.yaml +176 -0
- data/rbs_collection.yaml +15 -0
- data/sig/avro.rbs +14 -0
- data/sig/defs.rbs +1859 -0
- data/sig/fig_tree.rbs +2 -0
- data/spec/snapshots/consumers-no-nest.snap +1 -1
- data/spec/snapshots/consumers.snap +1 -1
- data/spec/snapshots/consumers_and_producers-no-nest.snap +3 -3
- data/spec/snapshots/consumers_and_producers.snap +3 -3
- data/spec/snapshots/consumers_circular-no-nest.snap +1 -1
- data/spec/snapshots/consumers_circular.snap +1 -1
- data/spec/snapshots/consumers_complex_types-no-nest.snap +1 -1
- data/spec/snapshots/consumers_complex_types.snap +1 -1
- data/spec/snapshots/consumers_nested-no-nest.snap +1 -1
- data/spec/snapshots/consumers_nested.snap +1 -1
- data/spec/snapshots/namespace_folders.snap +3 -3
- data/spec/snapshots/producers_with_key-no-nest.snap +1 -1
- data/spec/snapshots/producers_with_key.snap +1 -1
- metadata +7 -2
data/lib/deimos/producer.rb
CHANGED
@@ -11,7 +11,8 @@ module Deimos
|
|
11
11
|
# Run a block without allowing any messages to be produced to Kafka.
|
12
12
|
# Optionally add a list of producer classes to limit the disabling to those
|
13
13
|
# classes.
|
14
|
-
# @param producer_classes [Array<Class
|
14
|
+
# @param producer_classes [Array<Class>, Class]
|
15
|
+
# @return [void]
|
15
16
|
def disable_producers(*producer_classes, &block)
|
16
17
|
if producer_classes.any?
|
17
18
|
_disable_producer_classes(producer_classes, &block)
|
@@ -31,7 +32,7 @@ module Deimos
|
|
31
32
|
end
|
32
33
|
end
|
33
34
|
|
34
|
-
#
|
35
|
+
# @!visibility private
|
35
36
|
def _disable_producer_classes(producer_classes)
|
36
37
|
Thread.current[:frk_disabled_producers] ||= Set.new
|
37
38
|
producers_to_disable = producer_classes -
|
@@ -43,6 +44,7 @@ module Deimos
|
|
43
44
|
|
44
45
|
# Are producers disabled? If a class is passed in, check only that class.
|
45
46
|
# Otherwise check if the global disable flag is set.
|
47
|
+
# @param producer_class [Class]
|
46
48
|
# @return [Boolean]
|
47
49
|
def producers_disabled?(producer_class=nil)
|
48
50
|
Thread.current[:frk_disable_all_producers] ||
|
@@ -54,6 +56,7 @@ module Deimos
|
|
54
56
|
class Producer
|
55
57
|
include SharedConfig
|
56
58
|
|
59
|
+
# @return [Integer]
|
57
60
|
MAX_BATCH_SIZE = 500
|
58
61
|
|
59
62
|
class << self
|
@@ -87,19 +90,21 @@ module Deimos
|
|
87
90
|
end
|
88
91
|
|
89
92
|
# Publish the payload to the topic.
|
90
|
-
# @param payload [Hash
|
93
|
+
# @param payload [Hash, SchemaClass::Record] with an optional payload_key hash key.
|
91
94
|
# @param topic [String] if specifying the topic
|
95
|
+
# @return [void]
|
92
96
|
def publish(payload, topic: self.topic)
|
93
97
|
publish_list([payload], topic: topic)
|
94
98
|
end
|
95
99
|
|
96
100
|
# Publish a list of messages.
|
97
|
-
# @param payloads [Array<Hash
|
101
|
+
# @param payloads [Array<Hash, SchemaClass::Record>] with optional payload_key hash key.
|
98
102
|
# @param sync [Boolean] if given, override the default setting of
|
99
103
|
# whether to publish synchronously.
|
100
104
|
# @param force_send [Boolean] if true, ignore the configured backend
|
101
105
|
# and send immediately to Kafka.
|
102
106
|
# @param topic [String] if specifying the topic
|
107
|
+
# @return [void]
|
103
108
|
def publish_list(payloads, sync: nil, force_send: false, topic: self.topic)
|
104
109
|
return if Deimos.config.kafka.seed_brokers.blank? ||
|
105
110
|
Deimos.config.producers.disabled ||
|
@@ -124,7 +129,7 @@ module Deimos
|
|
124
129
|
|
125
130
|
# @param sync [Boolean]
|
126
131
|
# @param force_send [Boolean]
|
127
|
-
# @return [Class
|
132
|
+
# @return [Class<Deimos::Backends::Base>]
|
128
133
|
def determine_backend_class(sync, force_send)
|
129
134
|
backend = if force_send
|
130
135
|
:kafka
|
@@ -140,8 +145,9 @@ module Deimos
|
|
140
145
|
end
|
141
146
|
|
142
147
|
# Send a batch to the backend.
|
143
|
-
# @param backend [Class
|
148
|
+
# @param backend [Class<Deimos::Backends::Base>]
|
144
149
|
# @param batch [Array<Deimos::Message>]
|
150
|
+
# @return [void]
|
145
151
|
def produce_batch(backend, batch)
|
146
152
|
backend.publish(producer_class: self, messages: batch)
|
147
153
|
end
|
@@ -3,11 +3,19 @@
|
|
3
3
|
module Deimos
|
4
4
|
# Represents a field in the schema.
|
5
5
|
class SchemaField
|
6
|
-
|
6
|
+
# @return [String]
|
7
|
+
attr_accessor :name
|
8
|
+
# @return [String]
|
9
|
+
attr_accessor :type
|
10
|
+
# @return [Array<String>]
|
11
|
+
attr_accessor :enum_values
|
12
|
+
# @return [Object]
|
13
|
+
attr_accessor :default
|
7
14
|
|
8
15
|
# @param name [String]
|
9
16
|
# @param type [Object]
|
10
17
|
# @param enum_values [Array<String>]
|
18
|
+
# @param default [Object]
|
11
19
|
def initialize(name, type, enum_values=[], default=:no_default)
|
12
20
|
@name = name
|
13
21
|
@type = type
|
@@ -19,9 +27,14 @@ module Deimos
|
|
19
27
|
module SchemaBackends
|
20
28
|
# Base class for encoding / decoding.
|
21
29
|
class Base
|
22
|
-
|
30
|
+
# @return [String]
|
31
|
+
attr_accessor :schema
|
32
|
+
# @return [String]
|
33
|
+
attr_accessor :namespace
|
34
|
+
# @return [String]
|
35
|
+
attr_accessor :key_schema
|
23
36
|
|
24
|
-
# @param schema [String
|
37
|
+
# @param schema [String,Symbol]
|
25
38
|
# @param namespace [String]
|
26
39
|
def initialize(schema:, namespace: nil)
|
27
40
|
@schema = schema
|
@@ -30,7 +43,7 @@ module Deimos
|
|
30
43
|
|
31
44
|
# Encode a payload with a schema. Public method.
|
32
45
|
# @param payload [Hash]
|
33
|
-
# @param schema [Symbol
|
46
|
+
# @param schema [String,Symbol]
|
34
47
|
# @param topic [String]
|
35
48
|
# @return [String]
|
36
49
|
def encode(payload, schema: nil, topic: nil)
|
@@ -40,7 +53,7 @@ module Deimos
|
|
40
53
|
|
41
54
|
# Decode a payload with a schema. Public method.
|
42
55
|
# @param payload [String]
|
43
|
-
# @param schema [Symbol
|
56
|
+
# @param schema [String,Symbol]
|
44
57
|
# @return [Hash,nil]
|
45
58
|
def decode(payload, schema: nil)
|
46
59
|
return nil if payload.nil?
|
@@ -90,25 +103,26 @@ module Deimos
|
|
90
103
|
|
91
104
|
# Encode a payload. To be defined by subclass.
|
92
105
|
# @param payload [Hash]
|
93
|
-
# @param schema [Symbol
|
106
|
+
# @param schema [String,Symbol]
|
94
107
|
# @param topic [String]
|
95
108
|
# @return [String]
|
96
|
-
def encode_payload(
|
109
|
+
def encode_payload(payload, schema:, topic: nil)
|
97
110
|
raise NotImplementedError
|
98
111
|
end
|
99
112
|
|
100
113
|
# Decode a payload. To be defined by subclass.
|
101
114
|
# @param payload [String]
|
102
|
-
# @param schema [String
|
115
|
+
# @param schema [String,Symbol]
|
103
116
|
# @return [Hash]
|
104
|
-
def decode_payload(
|
117
|
+
def decode_payload(payload, schema:)
|
105
118
|
raise NotImplementedError
|
106
119
|
end
|
107
120
|
|
108
121
|
# Validate that a payload matches the schema. To be defined by subclass.
|
109
122
|
# @param payload [Hash]
|
110
|
-
# @param schema [String
|
111
|
-
|
123
|
+
# @param schema [String,Symbol]
|
124
|
+
# @return [void]
|
125
|
+
def validate(payload, schema:)
|
112
126
|
raise NotImplementedError
|
113
127
|
end
|
114
128
|
|
@@ -124,7 +138,7 @@ module Deimos
|
|
124
138
|
# @param field [SchemaField]
|
125
139
|
# @param value [Object]
|
126
140
|
# @return [Object]
|
127
|
-
def coerce_field(
|
141
|
+
def coerce_field(field, value)
|
128
142
|
raise NotImplementedError
|
129
143
|
end
|
130
144
|
|
@@ -140,19 +154,19 @@ module Deimos
|
|
140
154
|
end
|
141
155
|
|
142
156
|
# Encode a message key. To be defined by subclass.
|
143
|
-
# @param key [String
|
144
|
-
# @param key_id [Symbol
|
157
|
+
# @param key [String,Hash] the value to use as the key.
|
158
|
+
# @param key_id [String,Symbol] the field name of the key.
|
145
159
|
# @param topic [String]
|
146
160
|
# @return [String]
|
147
|
-
def encode_key(
|
161
|
+
def encode_key(key, key_id, topic: nil)
|
148
162
|
raise NotImplementedError
|
149
163
|
end
|
150
164
|
|
151
165
|
# Decode a message key. To be defined by subclass.
|
152
166
|
# @param payload [Hash] the message itself.
|
153
|
-
# @param key_id [Symbol
|
167
|
+
# @param key_id [String,Symbol] the field in the message to decode.
|
154
168
|
# @return [String]
|
155
|
-
def decode_key(
|
169
|
+
def decode_key(payload, key_id)
|
156
170
|
raise NotImplementedError
|
157
171
|
end
|
158
172
|
|
@@ -15,7 +15,7 @@ module Deimos
|
|
15
15
|
end
|
16
16
|
|
17
17
|
# @override
|
18
|
-
def validate(
|
18
|
+
def validate(payload, schema:)
|
19
19
|
end
|
20
20
|
|
21
21
|
# @override
|
@@ -24,7 +24,7 @@ module Deimos
|
|
24
24
|
end
|
25
25
|
|
26
26
|
# @override
|
27
|
-
def coerce_field(
|
27
|
+
def coerce_field(field, value)
|
28
28
|
value
|
29
29
|
end
|
30
30
|
|
@@ -7,23 +7,26 @@ module Deimos
|
|
7
7
|
# Base Class for Schema Classes generated from Avro.
|
8
8
|
class Base
|
9
9
|
|
10
|
-
#
|
10
|
+
# @param _args [Array<Object>]
|
11
11
|
def initialize(*_args)
|
12
12
|
end
|
13
13
|
|
14
14
|
# Converts the object to a hash which can be used for debugging or comparing objects.
|
15
|
+
# @param _opts [Hash]
|
15
16
|
# @return [Hash] a hash representation of the payload
|
16
17
|
def as_json(_opts={})
|
17
18
|
raise NotImplementedError
|
18
19
|
end
|
19
20
|
|
20
|
-
# @param key [String
|
21
|
+
# @param key [String,Symbol]
|
21
22
|
# @param val [Object]
|
23
|
+
# @return [void]
|
22
24
|
def []=(key, val)
|
23
25
|
self.send("#{key}=", val)
|
24
26
|
end
|
25
27
|
|
26
|
-
#
|
28
|
+
# @param other [SchemaClass::Base]
|
29
|
+
# @return [Boolean]
|
27
30
|
def ==(other)
|
28
31
|
comparison = other
|
29
32
|
if other.class == self.class
|
@@ -33,7 +36,7 @@ module Deimos
|
|
33
36
|
comparison == self.as_json
|
34
37
|
end
|
35
38
|
|
36
|
-
#
|
39
|
+
# @return [String]
|
37
40
|
def inspect
|
38
41
|
klass = self.class
|
39
42
|
"#{klass}(#{self.as_json})"
|
@@ -41,13 +44,14 @@ module Deimos
|
|
41
44
|
|
42
45
|
# Initializes this class from a given value
|
43
46
|
# @param value [Object]
|
47
|
+
# @return [SchemaClass::Base]
|
44
48
|
def self.initialize_from_value(value)
|
45
49
|
raise NotImplementedError
|
46
50
|
end
|
47
51
|
|
48
52
|
protected
|
49
53
|
|
50
|
-
#
|
54
|
+
# @return [Integer]
|
51
55
|
def hash
|
52
56
|
as_json.hash
|
53
57
|
end
|
@@ -8,9 +8,11 @@ module Deimos
|
|
8
8
|
# Base Class for Enum Classes generated from Avro.
|
9
9
|
class Enum < Base
|
10
10
|
|
11
|
+
# @return [String]
|
11
12
|
attr_accessor :value
|
12
13
|
|
13
14
|
# @param other [Deimos::SchemaClass::Enum]
|
15
|
+
# @return [Boolean]
|
14
16
|
def ==(other)
|
15
17
|
other.is_a?(self.class) ? other.value == @value : other == @value
|
16
18
|
end
|
@@ -31,12 +33,12 @@ module Deimos
|
|
31
33
|
raise NotImplementedError
|
32
34
|
end
|
33
35
|
|
34
|
-
#
|
36
|
+
# @return [String]
|
35
37
|
def as_json(_opts={})
|
36
38
|
@value
|
37
39
|
end
|
38
40
|
|
39
|
-
#
|
41
|
+
# @return [SchemaClass::Enum]
|
40
42
|
def self.initialize_from_value(value)
|
41
43
|
return nil if value.nil?
|
42
44
|
|
@@ -21,8 +21,8 @@ module Deimos
|
|
21
21
|
end
|
22
22
|
|
23
23
|
# Merge a hash or an identical schema object with this one and return a new object.
|
24
|
-
# @param other_hash [Hash
|
25
|
-
# @return [
|
24
|
+
# @param other_hash [Hash,SchemaClass::Base]
|
25
|
+
# @return [SchemaClass::Base]
|
26
26
|
def merge(other_hash)
|
27
27
|
obj = self.class.new(**self.to_h.symbolize_keys)
|
28
28
|
other_hash.to_h.each do |k, v|
|
@@ -32,13 +32,13 @@ module Deimos
|
|
32
32
|
end
|
33
33
|
|
34
34
|
# Element access method as if this Object were a hash
|
35
|
-
# @param key[String
|
35
|
+
# @param key[String,Symbol]
|
36
36
|
# @return [Object] The value of the attribute if exists, nil otherwise
|
37
37
|
def [](key)
|
38
38
|
self.try(key.to_sym)
|
39
39
|
end
|
40
40
|
|
41
|
-
#
|
41
|
+
# @return [SchemaClass::Record]
|
42
42
|
def with_indifferent_access
|
43
43
|
self
|
44
44
|
end
|
@@ -72,7 +72,7 @@ module Deimos
|
|
72
72
|
validator.schema_fields.map(&:name)
|
73
73
|
end
|
74
74
|
|
75
|
-
#
|
75
|
+
# @return [SchemaClass::Record]
|
76
76
|
def self.initialize_from_value(value)
|
77
77
|
return nil if value.nil?
|
78
78
|
|
data/lib/deimos/shared_config.rb
CHANGED
@@ -35,12 +35,14 @@ module Deimos
|
|
35
35
|
|
36
36
|
# Set the schema.
|
37
37
|
# @param schema [String]
|
38
|
+
# @return [void]
|
38
39
|
def schema(schema)
|
39
40
|
config[:schema] = schema
|
40
41
|
end
|
41
42
|
|
42
43
|
# Set the namespace.
|
43
44
|
# @param namespace [String]
|
45
|
+
# @return [void]
|
44
46
|
def namespace(namespace)
|
45
47
|
config[:namespace] = namespace
|
46
48
|
end
|
@@ -48,9 +50,10 @@ module Deimos
|
|
48
50
|
# Set key configuration.
|
49
51
|
# @param field [Symbol] the name of a field to use in the value schema as
|
50
52
|
# a generated key schema
|
51
|
-
# @param schema [String
|
53
|
+
# @param schema [String, Symbol] the name of a schema to use for the key
|
52
54
|
# @param plain [Boolean] if true, do not encode keys at all
|
53
55
|
# @param none [Boolean] if true, do not use keys at all
|
56
|
+
# @return [void]
|
54
57
|
def key_config(plain: nil, field: nil, schema: nil, none: nil)
|
55
58
|
config[:key_configured] = true
|
56
59
|
config[:no_keys] = none
|
@@ -59,7 +62,8 @@ module Deimos
|
|
59
62
|
config[:key_schema] = schema
|
60
63
|
end
|
61
64
|
|
62
|
-
# @param
|
65
|
+
# @param use_schema_classes [Boolean]
|
66
|
+
# @return [void]
|
63
67
|
def schema_class_config(use_schema_classes)
|
64
68
|
config[:use_schema_classes] = use_schema_classes
|
65
69
|
end
|
data/lib/deimos/test_helpers.rb
CHANGED
@@ -20,6 +20,7 @@ module Deimos
|
|
20
20
|
end
|
21
21
|
|
22
22
|
# Set the config to the right settings for a unit test
|
23
|
+
# @return [void]
|
23
24
|
def unit_test!
|
24
25
|
Deimos.configure do |deimos_config|
|
25
26
|
deimos_config.logger = Logger.new(STDOUT)
|
@@ -31,6 +32,7 @@ module Deimos
|
|
31
32
|
end
|
32
33
|
|
33
34
|
# Kafka test config with avro schema registry
|
35
|
+
# @return [void]
|
34
36
|
def full_integration_test!
|
35
37
|
Deimos.configure do |deimos_config|
|
36
38
|
deimos_config.producers.backend = :kafka
|
@@ -39,6 +41,7 @@ module Deimos
|
|
39
41
|
end
|
40
42
|
|
41
43
|
# Set the config to the right settings for a kafka test
|
44
|
+
# @return [void]
|
42
45
|
def kafka_test!
|
43
46
|
Deimos.configure do |deimos_config|
|
44
47
|
deimos_config.producers.backend = :kafka
|
@@ -62,21 +65,25 @@ module Deimos
|
|
62
65
|
end
|
63
66
|
|
64
67
|
# @deprecated
|
68
|
+
# @!visibility private
|
65
69
|
def stub_producers_and_consumers!
|
66
70
|
warn('stub_producers_and_consumers! is no longer necessary and this method will be removed in 3.0')
|
67
71
|
end
|
68
72
|
|
69
73
|
# @deprecated
|
74
|
+
# @!visibility private
|
70
75
|
def stub_producer(_klass)
|
71
76
|
warn('Stubbing producers is no longer necessary and this method will be removed in 3.0')
|
72
77
|
end
|
73
78
|
|
74
79
|
# @deprecated
|
80
|
+
# @!visibility private
|
75
81
|
def stub_consumer(_klass)
|
76
82
|
warn('Stubbing consumers is no longer necessary and this method will be removed in 3.0')
|
77
83
|
end
|
78
84
|
|
79
85
|
# @deprecated
|
86
|
+
# @!visibility private
|
80
87
|
def stub_batch_consumer(_klass)
|
81
88
|
warn('Stubbing batch consumers is no longer necessary and this method will be removed in 3.0')
|
82
89
|
end
|
@@ -84,6 +91,7 @@ module Deimos
|
|
84
91
|
# get the difference of 2 hashes.
|
85
92
|
# @param hash1 [Hash]
|
86
93
|
# @param hash2 [Hash]
|
94
|
+
# @!visibility private
|
87
95
|
def _hash_diff(hash1, hash2)
|
88
96
|
if hash1.nil? || !hash1.is_a?(Hash)
|
89
97
|
hash2
|
@@ -96,7 +104,7 @@ module Deimos
|
|
96
104
|
end
|
97
105
|
end
|
98
106
|
|
99
|
-
#
|
107
|
+
# @!visibility private
|
100
108
|
def _frk_failure_message(topic, message, key=nil, partition_key=nil, was_negated=false)
|
101
109
|
messages = Deimos::Backends::Test.sent_messages.
|
102
110
|
select { |m| m[:topic] == topic }.
|
@@ -162,6 +170,7 @@ module Deimos
|
|
162
170
|
|
163
171
|
# Clear all sent messages - e.g. if we want to check that
|
164
172
|
# particular messages were sent or not sent after a point in time.
|
173
|
+
# @return [void]
|
165
174
|
def clear_kafka_messages!
|
166
175
|
Deimos::Backends::Test.sent_messages.clear
|
167
176
|
end
|
@@ -170,15 +179,16 @@ module Deimos
|
|
170
179
|
# that the schema is correct. If
|
171
180
|
# a block is given, that block will be executed when `consume` is called.
|
172
181
|
# Otherwise it will just confirm that `consume` is called at all.
|
173
|
-
# @param handler_class_or_topic [Class
|
182
|
+
# @param handler_class_or_topic [Class, String] Class which inherits from
|
174
183
|
# Deimos::Consumer or the topic as a string
|
175
184
|
# @param payload [Hash] the payload to consume
|
176
185
|
# @param call_original [Boolean] if true, allow the consume handler
|
177
186
|
# to continue as normal. Not compatible with a block.
|
178
|
-
# @param
|
187
|
+
# @param skip_expectation [Boolean] Set to true to not place any
|
179
188
|
# expectations on the consumer. Primarily used internally to Deimos.
|
180
189
|
# @param key [Object] the key to use.
|
181
190
|
# @param partition_key [Object] the partition key to use.
|
191
|
+
# @return [void]
|
182
192
|
def test_consume_message(handler_class_or_topic,
|
183
193
|
payload,
|
184
194
|
call_original: false,
|
@@ -225,6 +235,7 @@ module Deimos
|
|
225
235
|
# Check to see that a given message will fail due to validation errors.
|
226
236
|
# @param handler_class [Class]
|
227
237
|
# @param payload [Hash]
|
238
|
+
# @return [void]
|
228
239
|
def test_consume_invalid_message(handler_class, payload)
|
229
240
|
expect {
|
230
241
|
handler_class.decoder.validate(payload,
|
@@ -236,9 +247,14 @@ module Deimos
|
|
236
247
|
# i.e. that the schema is correct. If
|
237
248
|
# a block is given, that block will be executed when `consume` is called.
|
238
249
|
# Otherwise it will just confirm that `consume` is called at all.
|
239
|
-
# @param handler_class_or_topic [Class
|
250
|
+
# @param handler_class_or_topic [Class, String] Class which inherits from
|
240
251
|
# Deimos::Consumer or the topic as a string
|
241
252
|
# @param payloads [Array<Hash>] the payload to consume
|
253
|
+
# @param keys [Array<Hash,String>]
|
254
|
+
# @param partition_keys [Array<Integer>]
|
255
|
+
# @param call_original [Boolean]
|
256
|
+
# @param skip_expectation [Boolean]
|
257
|
+
# @return [void]
|
242
258
|
def test_consume_batch(handler_class_or_topic,
|
243
259
|
payloads,
|
244
260
|
keys: [],
|
@@ -297,6 +313,7 @@ module Deimos
|
|
297
313
|
# Check to see that a given message will fail due to validation errors.
|
298
314
|
# @param handler_class [Class]
|
299
315
|
# @param payloads [Array<Hash>]
|
316
|
+
# @return [void]
|
300
317
|
def test_consume_batch_invalid_message(handler_class, payloads)
|
301
318
|
topic_name = 'my-topic'
|
302
319
|
handler = handler_class.new
|
data/lib/deimos/tracing/mock.rb
CHANGED
@@ -6,13 +6,14 @@ module Deimos
|
|
6
6
|
module Tracing
|
7
7
|
# Class that mocks out tracing functionality
|
8
8
|
class Mock < Tracing::Provider
|
9
|
-
#
|
9
|
+
# @param logger [Logger]
|
10
10
|
def initialize(logger=nil)
|
11
11
|
@logger = logger || Logger.new(STDOUT)
|
12
12
|
@logger.info('MockTracingProvider initialized')
|
13
13
|
end
|
14
14
|
|
15
|
-
#
|
15
|
+
# @param span_name [String]
|
16
|
+
# @param _options [Hash]
|
16
17
|
def start(span_name, _options={})
|
17
18
|
@logger.info("Mock span '#{span_name}' started")
|
18
19
|
{
|
@@ -35,7 +36,7 @@ module Deimos
|
|
35
36
|
end
|
36
37
|
|
37
38
|
# :nodoc:
|
38
|
-
def set_tag(
|
39
|
+
def set_tag(tag, value, span=nil)
|
39
40
|
nil
|
40
41
|
end
|
41
42
|
|
@@ -14,6 +14,7 @@ module Deimos
|
|
14
14
|
|
15
15
|
# Finishes the trace on the span object.
|
16
16
|
# @param span [Object] The span to finish trace on
|
17
|
+
# @return [void]
|
17
18
|
def finish(span)
|
18
19
|
raise NotImplementedError
|
19
20
|
end
|
@@ -21,11 +22,13 @@ module Deimos
|
|
21
22
|
# Set an error on the span.
|
22
23
|
# @param span [Object] The span to set error on
|
23
24
|
# @param exception [Exception] The exception that occurred
|
25
|
+
# @return [void]
|
24
26
|
def set_error(span, exception)
|
25
27
|
raise NotImplementedError
|
26
28
|
end
|
27
29
|
|
28
30
|
# Get the currently activated span.
|
31
|
+
# @return [Object]
|
29
32
|
def active_span
|
30
33
|
raise NotImplementedError
|
31
34
|
end
|
@@ -33,6 +36,8 @@ module Deimos
|
|
33
36
|
# Set a tag to a span. Use the currently active span if not given.
|
34
37
|
# @param tag [String]
|
35
38
|
# @param value [String]
|
39
|
+
# @param span [Object]
|
40
|
+
# @return [void]
|
36
41
|
def set_tag(tag, value, span=nil)
|
37
42
|
raise NotImplementedError
|
38
43
|
end
|
@@ -7,12 +7,15 @@ module Deimos
|
|
7
7
|
module Utils
|
8
8
|
# Class which continually polls the database and sends Kafka messages.
|
9
9
|
class DbPoller
|
10
|
+
# @return [Integer]
|
10
11
|
BATCH_SIZE = 1000
|
11
12
|
|
12
13
|
# Needed for Executor so it can identify the worker
|
14
|
+
# @return [Integer]
|
13
15
|
attr_reader :id
|
14
16
|
|
15
17
|
# Begin the DB Poller process.
|
18
|
+
# @return [void]
|
16
19
|
def self.start!
|
17
20
|
if Deimos.config.db_poller_objects.empty?
|
18
21
|
raise('No pollers configured!')
|
@@ -28,7 +31,7 @@ module Deimos
|
|
28
31
|
signal_handler.run!
|
29
32
|
end
|
30
33
|
|
31
|
-
# @param config [
|
34
|
+
# @param config [FigTree::ConfigStruct]
|
32
35
|
def initialize(config)
|
33
36
|
@config = config
|
34
37
|
@id = SecureRandom.hex
|
@@ -47,6 +50,7 @@ module Deimos
|
|
47
50
|
# time we ran
|
48
51
|
# 2) On a loop, process all the recent updates between the last time
|
49
52
|
# we ran and now.
|
53
|
+
# @return [void]
|
50
54
|
def start
|
51
55
|
# Don't send asynchronously
|
52
56
|
if Deimos.config.producers.backend == :kafka_async
|
@@ -66,6 +70,7 @@ module Deimos
|
|
66
70
|
end
|
67
71
|
|
68
72
|
# Grab the PollInfo or create if it doesn't exist.
|
73
|
+
# @return [void]
|
69
74
|
def retrieve_poll_info
|
70
75
|
ActiveRecord::Base.connection.reconnect! unless ActiveRecord::Base.connection.open_transactions.positive?
|
71
76
|
new_time = @config.start_from_beginning ? Time.new(0) : Time.zone.now
|
@@ -76,6 +81,7 @@ module Deimos
|
|
76
81
|
end
|
77
82
|
|
78
83
|
# Stop the poll.
|
84
|
+
# @return [void]
|
79
85
|
def stop
|
80
86
|
Deimos.config.logger.info('Received signal to stop')
|
81
87
|
@signal_to_stop = true
|
@@ -95,6 +101,7 @@ module Deimos
|
|
95
101
|
end
|
96
102
|
|
97
103
|
# Send messages for updated data.
|
104
|
+
# @return [void]
|
98
105
|
def process_updates
|
99
106
|
return unless should_run?
|
100
107
|
|
@@ -135,6 +142,7 @@ module Deimos
|
|
135
142
|
end
|
136
143
|
|
137
144
|
# @param batch [Array<ActiveRecord::Base>]
|
145
|
+
# @return [void]
|
138
146
|
def process_batch(batch)
|
139
147
|
record = batch.last
|
140
148
|
id_method = record.class.primary_key
|