deimos-ruby 1.24.2 → 2.0.0.pre.alpha1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop_todo.yml +0 -17
  3. data/.tool-versions +1 -0
  4. data/CHANGELOG.md +5 -0
  5. data/README.md +287 -498
  6. data/deimos-ruby.gemspec +4 -4
  7. data/docs/CONFIGURATION.md +133 -226
  8. data/docs/UPGRADING.md +237 -0
  9. data/lib/deimos/active_record_consume/batch_consumption.rb +29 -28
  10. data/lib/deimos/active_record_consume/mass_updater.rb +59 -4
  11. data/lib/deimos/active_record_consume/message_consumption.rb +15 -21
  12. data/lib/deimos/active_record_consumer.rb +36 -21
  13. data/lib/deimos/active_record_producer.rb +28 -9
  14. data/lib/deimos/backends/base.rb +4 -35
  15. data/lib/deimos/backends/kafka.rb +6 -22
  16. data/lib/deimos/backends/kafka_async.rb +6 -22
  17. data/lib/deimos/backends/{db.rb → outbox.rb} +13 -9
  18. data/lib/deimos/config/configuration.rb +116 -379
  19. data/lib/deimos/consume/batch_consumption.rb +24 -124
  20. data/lib/deimos/consume/message_consumption.rb +36 -63
  21. data/lib/deimos/consumer.rb +16 -75
  22. data/lib/deimos/ext/consumer_route.rb +35 -0
  23. data/lib/deimos/ext/producer_middleware.rb +94 -0
  24. data/lib/deimos/ext/producer_route.rb +22 -0
  25. data/lib/deimos/ext/redraw.rb +29 -0
  26. data/lib/deimos/ext/routing_defaults.rb +72 -0
  27. data/lib/deimos/ext/schema_route.rb +70 -0
  28. data/lib/deimos/kafka_message.rb +2 -2
  29. data/lib/deimos/kafka_source.rb +2 -7
  30. data/lib/deimos/kafka_topic_info.rb +1 -1
  31. data/lib/deimos/logging.rb +71 -0
  32. data/lib/deimos/message.rb +2 -11
  33. data/lib/deimos/metrics/datadog.rb +40 -1
  34. data/lib/deimos/metrics/provider.rb +4 -4
  35. data/lib/deimos/producer.rb +39 -116
  36. data/lib/deimos/railtie.rb +6 -0
  37. data/lib/deimos/schema_backends/avro_base.rb +21 -21
  38. data/lib/deimos/schema_backends/avro_schema_registry.rb +1 -2
  39. data/lib/deimos/schema_backends/avro_validation.rb +2 -2
  40. data/lib/deimos/schema_backends/base.rb +19 -12
  41. data/lib/deimos/schema_backends/mock.rb +6 -1
  42. data/lib/deimos/schema_backends/plain.rb +47 -0
  43. data/lib/deimos/schema_class/base.rb +2 -2
  44. data/lib/deimos/schema_class/enum.rb +1 -1
  45. data/lib/deimos/schema_class/record.rb +2 -2
  46. data/lib/deimos/test_helpers.rb +95 -320
  47. data/lib/deimos/tracing/provider.rb +6 -6
  48. data/lib/deimos/transcoder.rb +88 -0
  49. data/lib/deimos/utils/db_poller/base.rb +16 -14
  50. data/lib/deimos/utils/db_poller/state_based.rb +3 -3
  51. data/lib/deimos/utils/db_poller/time_based.rb +4 -4
  52. data/lib/deimos/utils/db_poller.rb +1 -1
  53. data/lib/deimos/utils/deadlock_retry.rb +1 -1
  54. data/lib/deimos/utils/{db_producer.rb → outbox_producer.rb} +16 -47
  55. data/lib/deimos/utils/schema_class.rb +0 -7
  56. data/lib/deimos/version.rb +1 -1
  57. data/lib/deimos.rb +79 -26
  58. data/lib/generators/deimos/{db_backend_generator.rb → outbox_backend_generator.rb} +4 -4
  59. data/lib/generators/deimos/schema_class_generator.rb +0 -1
  60. data/lib/generators/deimos/v2/templates/karafka.rb.tt +149 -0
  61. data/lib/generators/deimos/v2_generator.rb +193 -0
  62. data/lib/tasks/deimos.rake +5 -7
  63. data/spec/active_record_batch_consumer_association_spec.rb +22 -13
  64. data/spec/active_record_batch_consumer_spec.rb +84 -65
  65. data/spec/active_record_consume/batch_consumption_spec.rb +10 -10
  66. data/spec/active_record_consume/batch_slicer_spec.rb +12 -12
  67. data/spec/active_record_consume/mass_updater_spec.rb +137 -0
  68. data/spec/active_record_consumer_spec.rb +29 -13
  69. data/spec/active_record_producer_spec.rb +36 -26
  70. data/spec/backends/base_spec.rb +0 -23
  71. data/spec/backends/kafka_async_spec.rb +1 -3
  72. data/spec/backends/kafka_spec.rb +1 -3
  73. data/spec/backends/{db_spec.rb → outbox_spec.rb} +14 -20
  74. data/spec/batch_consumer_spec.rb +66 -116
  75. data/spec/consumer_spec.rb +53 -147
  76. data/spec/deimos_spec.rb +10 -126
  77. data/spec/kafka_source_spec.rb +19 -52
  78. data/spec/karafka/karafka.rb +69 -0
  79. data/spec/karafka_config/karafka_spec.rb +97 -0
  80. data/spec/logging_spec.rb +25 -0
  81. data/spec/message_spec.rb +9 -9
  82. data/spec/producer_spec.rb +112 -254
  83. data/spec/rake_spec.rb +1 -3
  84. data/spec/schema_backends/avro_validation_spec.rb +1 -1
  85. data/spec/schemas/com/my-namespace/MySchemaWithTitle.avsc +22 -0
  86. data/spec/snapshots/consumers-no-nest.snap +49 -0
  87. data/spec/snapshots/consumers.snap +49 -0
  88. data/spec/snapshots/consumers_and_producers-no-nest.snap +49 -0
  89. data/spec/snapshots/consumers_and_producers.snap +49 -0
  90. data/spec/snapshots/consumers_circular-no-nest.snap +49 -0
  91. data/spec/snapshots/consumers_circular.snap +49 -0
  92. data/spec/snapshots/consumers_complex_types-no-nest.snap +49 -0
  93. data/spec/snapshots/consumers_complex_types.snap +49 -0
  94. data/spec/snapshots/consumers_nested-no-nest.snap +49 -0
  95. data/spec/snapshots/consumers_nested.snap +49 -0
  96. data/spec/snapshots/namespace_folders.snap +49 -0
  97. data/spec/snapshots/namespace_map.snap +49 -0
  98. data/spec/snapshots/producers_with_key-no-nest.snap +49 -0
  99. data/spec/snapshots/producers_with_key.snap +49 -0
  100. data/spec/spec_helper.rb +61 -29
  101. data/spec/utils/db_poller_spec.rb +49 -39
  102. data/spec/utils/{db_producer_spec.rb → outbox_producer_spec.rb} +17 -184
  103. metadata +58 -67
  104. data/lib/deimos/batch_consumer.rb +0 -7
  105. data/lib/deimos/config/phobos_config.rb +0 -163
  106. data/lib/deimos/instrumentation.rb +0 -95
  107. data/lib/deimos/monkey_patches/phobos_cli.rb +0 -35
  108. data/lib/deimos/utils/inline_consumer.rb +0 -158
  109. data/lib/deimos/utils/lag_reporter.rb +0 -186
  110. data/lib/deimos/utils/schema_controller_mixin.rb +0 -129
  111. data/spec/config/configuration_spec.rb +0 -321
  112. data/spec/kafka_listener_spec.rb +0 -55
  113. data/spec/phobos.bad_db.yml +0 -73
  114. data/spec/phobos.yml +0 -77
  115. data/spec/utils/inline_consumer_spec.rb +0 -31
  116. data/spec/utils/lag_reporter_spec.rb +0 -76
  117. data/spec/utils/platform_schema_validation_spec.rb +0 -0
  118. data/spec/utils/schema_controller_mixin_spec.rb +0 -84
  119. /data/lib/generators/deimos/{db_backend → outbox_backend}/templates/migration +0 -0
  120. /data/lib/generators/deimos/{db_backend → outbox_backend}/templates/rails3_migration +0 -0
@@ -79,7 +79,7 @@ module Deimos
79
79
  # Indicate a class which should act as a mocked version of this backend.
80
80
  # This class should perform all validations but not actually do any
81
81
  # encoding.
82
- # Note that the "mock" version (e.g. avro_validation) should return
82
+ # Note that the "mock" version should return
83
83
  # its own symbol when this is called, since it may be called multiple
84
84
  # times depending on the order of RSpec helpers.
85
85
  # @return [Symbol]
@@ -90,7 +90,7 @@ module Deimos
90
90
  # The content type to use when encoding / decoding requests over HTTP via ActionController.
91
91
  # @return [String]
92
92
  def self.content_type
93
- raise NotImplementedError
93
+ raise MissingImplementationError
94
94
  end
95
95
 
96
96
  # Converts your schema to String form for generated YARD docs.
@@ -98,7 +98,7 @@ module Deimos
98
98
  # @param schema [Object]
99
99
  # @return [String] A string representation of the Type
100
100
  def self.field_type(schema)
101
- raise NotImplementedError
101
+ raise MissingImplementationError
102
102
  end
103
103
 
104
104
  # Encode a payload. To be defined by subclass.
@@ -107,7 +107,7 @@ module Deimos
107
107
  # @param topic [String]
108
108
  # @return [String]
109
109
  def encode_payload(payload, schema:, topic: nil)
110
- raise NotImplementedError
110
+ raise MissingImplementationError
111
111
  end
112
112
 
113
113
  # Decode a payload. To be defined by subclass.
@@ -115,7 +115,7 @@ module Deimos
115
115
  # @param schema [String,Symbol]
116
116
  # @return [Hash]
117
117
  def decode_payload(payload, schema:)
118
- raise NotImplementedError
118
+ raise MissingImplementationError
119
119
  end
120
120
 
121
121
  # Validate that a payload matches the schema. To be defined by subclass.
@@ -123,13 +123,13 @@ module Deimos
123
123
  # @param schema [String,Symbol]
124
124
  # @return [void]
125
125
  def validate(payload, schema:)
126
- raise NotImplementedError
126
+ raise MissingImplementationError
127
127
  end
128
128
 
129
129
  # List of field names belonging to the schema. To be defined by subclass.
130
130
  # @return [Array<SchemaField>]
131
131
  def schema_fields
132
- raise NotImplementedError
132
+ raise MissingImplementationError
133
133
  end
134
134
 
135
135
  # Given a value and a field definition (as defined by whatever the
@@ -139,7 +139,7 @@ module Deimos
139
139
  # @param value [Object]
140
140
  # @return [Object]
141
141
  def coerce_field(field, value)
142
- raise NotImplementedError
142
+ raise MissingImplementationError
143
143
  end
144
144
 
145
145
  # Given a field definition, return the SQL type that might be used in
@@ -150,7 +150,14 @@ module Deimos
150
150
  # @param field [SchemaField]
151
151
  # @return [Symbol]
152
152
  def sql_type(field)
153
- raise NotImplementedError
153
+ raise MissingImplementationError
154
+ end
155
+
156
+ # Generate a key schema from the given value schema and key ID. This
157
+ # is used when encoding or decoding keys from an existing value schema.
158
+ # @param field_name [Symbol]
159
+ def generate_key_schema(field_name)
160
+ raise MissingImplementationError
154
161
  end
155
162
 
156
163
  # Encode a message key. To be defined by subclass.
@@ -159,7 +166,7 @@ module Deimos
159
166
  # @param topic [String]
160
167
  # @return [String]
161
168
  def encode_key(key, key_id, topic: nil)
162
- raise NotImplementedError
169
+ raise MissingImplementationError
163
170
  end
164
171
 
165
172
  # Decode a message key. To be defined by subclass.
@@ -167,13 +174,13 @@ module Deimos
167
174
  # @param key_id [String,Symbol] the field in the message to decode.
168
175
  # @return [String]
169
176
  def decode_key(payload, key_id)
170
- raise NotImplementedError
177
+ raise MissingImplementationError
171
178
  end
172
179
 
173
180
  # Forcefully loads the schema into memory.
174
181
  # @return [Object] The schema that is of use.
175
182
  def load_schema
176
- raise NotImplementedError
183
+ raise MissingImplementationError
177
184
  end
178
185
  end
179
186
  end
@@ -4,6 +4,11 @@ module Deimos
4
4
  module SchemaBackends
5
5
  # Mock implementation of a schema backend that does no encoding or validation.
6
6
  class Mock < Base
7
+
8
+ # @override
9
+ def generate_key_schema(field_name)
10
+ end
11
+
7
12
  # @override
8
13
  def decode_payload(payload, schema:)
9
14
  payload.is_a?(String) ? 'payload-decoded' : payload.map { |k, v| [k, "decoded-#{v}"] }
@@ -11,7 +16,7 @@ module Deimos
11
16
 
12
17
  # @override
13
18
  def encode_payload(payload, schema:, topic: nil)
14
- payload.is_a?(String) ? 'payload-encoded' : payload.map { |k, v| [k, "encoded-#{v}"] }
19
+ payload.is_a?(String) ? 'payload-encoded' : payload.map { |k, v| [k, "encoded-#{v}"] }.to_json
15
20
  end
16
21
 
17
22
  # @override
@@ -0,0 +1,47 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Deimos
4
+ module SchemaBackends
5
+ # Schema backend that passes through as a basic string.
6
+ class Plain < Base
7
+
8
+ # @override
9
+ def generate_key_schema(field_name)
10
+ end
11
+
12
+ # @override
13
+ def decode_payload(payload, schema:)
14
+ payload
15
+ end
16
+
17
+ # @override
18
+ def encode_payload(payload, schema:, topic: nil)
19
+ payload.to_s
20
+ end
21
+
22
+ # @override
23
+ def validate(payload, schema:)
24
+ end
25
+
26
+ # @override
27
+ def schema_fields
28
+ []
29
+ end
30
+
31
+ # @override
32
+ def coerce_field(_field, value)
33
+ value
34
+ end
35
+
36
+ # @override
37
+ def encode_key(key_id, key, topic: nil)
38
+ key
39
+ end
40
+
41
+ # @override
42
+ def decode_key(payload, key_id)
43
+ payload[key_id]
44
+ end
45
+ end
46
+ end
47
+ end
@@ -15,7 +15,7 @@ module Deimos
15
15
  # @param _opts [Hash]
16
16
  # @return [Hash] a hash representation of the payload
17
17
  def as_json(_opts={})
18
- raise NotImplementedError
18
+ raise MissingImplementationError
19
19
  end
20
20
 
21
21
  # @param key [String,Symbol]
@@ -46,7 +46,7 @@ module Deimos
46
46
  # @param value [Object]
47
47
  # @return [SchemaClass::Base]
48
48
  def self.initialize_from_value(value)
49
- raise NotImplementedError
49
+ raise MissingImplementationError
50
50
  end
51
51
 
52
52
  protected
@@ -30,7 +30,7 @@ module Deimos
30
30
  # Returns all the valid symbols for this enum.
31
31
  # @return [Array<String>]
32
32
  def symbols
33
- raise NotImplementedError
33
+ raise MissingImplementationError
34
34
  end
35
35
 
36
36
  # @return [String]
@@ -46,13 +46,13 @@ module Deimos
46
46
  # Returns the schema name of the inheriting class.
47
47
  # @return [String]
48
48
  def schema
49
- raise NotImplementedError
49
+ raise MissingImplementationError
50
50
  end
51
51
 
52
52
  # Returns the namespace for the schema of the inheriting class.
53
53
  # @return [String]
54
54
  def namespace
55
- raise NotImplementedError
55
+ raise MissingImplementationError
56
56
  end
57
57
 
58
58
  # Returns the full schema name of the inheriting class.