deimos-ruby 1.24.3 → 2.0.0.pre.alpha1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (118) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop_todo.yml +0 -17
  3. data/.tool-versions +1 -0
  4. data/CHANGELOG.md +1 -1
  5. data/README.md +287 -498
  6. data/deimos-ruby.gemspec +4 -4
  7. data/docs/CONFIGURATION.md +133 -227
  8. data/docs/UPGRADING.md +237 -0
  9. data/lib/deimos/active_record_consume/batch_consumption.rb +28 -29
  10. data/lib/deimos/active_record_consume/message_consumption.rb +15 -21
  11. data/lib/deimos/active_record_consumer.rb +36 -26
  12. data/lib/deimos/active_record_producer.rb +28 -9
  13. data/lib/deimos/backends/base.rb +4 -35
  14. data/lib/deimos/backends/kafka.rb +6 -22
  15. data/lib/deimos/backends/kafka_async.rb +6 -22
  16. data/lib/deimos/backends/{db.rb → outbox.rb} +13 -9
  17. data/lib/deimos/config/configuration.rb +116 -385
  18. data/lib/deimos/consume/batch_consumption.rb +24 -124
  19. data/lib/deimos/consume/message_consumption.rb +36 -63
  20. data/lib/deimos/consumer.rb +16 -75
  21. data/lib/deimos/ext/consumer_route.rb +35 -0
  22. data/lib/deimos/ext/producer_middleware.rb +94 -0
  23. data/lib/deimos/ext/producer_route.rb +22 -0
  24. data/lib/deimos/ext/redraw.rb +29 -0
  25. data/lib/deimos/ext/routing_defaults.rb +72 -0
  26. data/lib/deimos/ext/schema_route.rb +70 -0
  27. data/lib/deimos/kafka_message.rb +2 -2
  28. data/lib/deimos/kafka_source.rb +2 -7
  29. data/lib/deimos/kafka_topic_info.rb +1 -1
  30. data/lib/deimos/logging.rb +71 -0
  31. data/lib/deimos/message.rb +2 -11
  32. data/lib/deimos/metrics/datadog.rb +40 -1
  33. data/lib/deimos/metrics/provider.rb +4 -4
  34. data/lib/deimos/producer.rb +39 -116
  35. data/lib/deimos/railtie.rb +6 -0
  36. data/lib/deimos/schema_backends/avro_base.rb +21 -21
  37. data/lib/deimos/schema_backends/avro_schema_registry.rb +1 -2
  38. data/lib/deimos/schema_backends/avro_validation.rb +2 -2
  39. data/lib/deimos/schema_backends/base.rb +19 -12
  40. data/lib/deimos/schema_backends/mock.rb +6 -1
  41. data/lib/deimos/schema_backends/plain.rb +47 -0
  42. data/lib/deimos/schema_class/base.rb +2 -2
  43. data/lib/deimos/schema_class/enum.rb +1 -1
  44. data/lib/deimos/schema_class/record.rb +2 -2
  45. data/lib/deimos/test_helpers.rb +95 -320
  46. data/lib/deimos/tracing/provider.rb +6 -6
  47. data/lib/deimos/transcoder.rb +88 -0
  48. data/lib/deimos/utils/db_poller/base.rb +16 -14
  49. data/lib/deimos/utils/db_poller/state_based.rb +3 -3
  50. data/lib/deimos/utils/db_poller/time_based.rb +4 -4
  51. data/lib/deimos/utils/db_poller.rb +1 -1
  52. data/lib/deimos/utils/deadlock_retry.rb +1 -1
  53. data/lib/deimos/utils/{db_producer.rb → outbox_producer.rb} +16 -47
  54. data/lib/deimos/utils/schema_class.rb +0 -7
  55. data/lib/deimos/version.rb +1 -1
  56. data/lib/deimos.rb +79 -26
  57. data/lib/generators/deimos/{db_backend_generator.rb → outbox_backend_generator.rb} +4 -4
  58. data/lib/generators/deimos/schema_class_generator.rb +0 -1
  59. data/lib/generators/deimos/v2/templates/karafka.rb.tt +149 -0
  60. data/lib/generators/deimos/v2_generator.rb +193 -0
  61. data/lib/tasks/deimos.rake +5 -7
  62. data/spec/active_record_batch_consumer_association_spec.rb +22 -13
  63. data/spec/active_record_batch_consumer_spec.rb +84 -65
  64. data/spec/active_record_consume/batch_consumption_spec.rb +10 -10
  65. data/spec/active_record_consume/batch_slicer_spec.rb +12 -12
  66. data/spec/active_record_consumer_spec.rb +29 -13
  67. data/spec/active_record_producer_spec.rb +36 -26
  68. data/spec/backends/base_spec.rb +0 -23
  69. data/spec/backends/kafka_async_spec.rb +1 -3
  70. data/spec/backends/kafka_spec.rb +1 -3
  71. data/spec/backends/{db_spec.rb → outbox_spec.rb} +14 -20
  72. data/spec/batch_consumer_spec.rb +66 -116
  73. data/spec/consumer_spec.rb +53 -147
  74. data/spec/deimos_spec.rb +10 -126
  75. data/spec/kafka_source_spec.rb +19 -52
  76. data/spec/karafka/karafka.rb +69 -0
  77. data/spec/karafka_config/karafka_spec.rb +97 -0
  78. data/spec/logging_spec.rb +25 -0
  79. data/spec/message_spec.rb +9 -9
  80. data/spec/producer_spec.rb +112 -254
  81. data/spec/rake_spec.rb +1 -3
  82. data/spec/schema_backends/avro_validation_spec.rb +1 -1
  83. data/spec/schemas/com/my-namespace/MySchemaWithTitle.avsc +22 -0
  84. data/spec/snapshots/consumers-no-nest.snap +49 -0
  85. data/spec/snapshots/consumers.snap +49 -0
  86. data/spec/snapshots/consumers_and_producers-no-nest.snap +49 -0
  87. data/spec/snapshots/consumers_and_producers.snap +49 -0
  88. data/spec/snapshots/consumers_circular-no-nest.snap +49 -0
  89. data/spec/snapshots/consumers_circular.snap +49 -0
  90. data/spec/snapshots/consumers_complex_types-no-nest.snap +49 -0
  91. data/spec/snapshots/consumers_complex_types.snap +49 -0
  92. data/spec/snapshots/consumers_nested-no-nest.snap +49 -0
  93. data/spec/snapshots/consumers_nested.snap +49 -0
  94. data/spec/snapshots/namespace_folders.snap +49 -0
  95. data/spec/snapshots/namespace_map.snap +49 -0
  96. data/spec/snapshots/producers_with_key-no-nest.snap +49 -0
  97. data/spec/snapshots/producers_with_key.snap +49 -0
  98. data/spec/spec_helper.rb +61 -29
  99. data/spec/utils/db_poller_spec.rb +49 -39
  100. data/spec/utils/{db_producer_spec.rb → outbox_producer_spec.rb} +17 -184
  101. metadata +58 -67
  102. data/lib/deimos/batch_consumer.rb +0 -7
  103. data/lib/deimos/config/phobos_config.rb +0 -164
  104. data/lib/deimos/instrumentation.rb +0 -95
  105. data/lib/deimos/monkey_patches/phobos_cli.rb +0 -35
  106. data/lib/deimos/utils/inline_consumer.rb +0 -158
  107. data/lib/deimos/utils/lag_reporter.rb +0 -186
  108. data/lib/deimos/utils/schema_controller_mixin.rb +0 -129
  109. data/spec/config/configuration_spec.rb +0 -329
  110. data/spec/kafka_listener_spec.rb +0 -55
  111. data/spec/phobos.bad_db.yml +0 -73
  112. data/spec/phobos.yml +0 -77
  113. data/spec/utils/inline_consumer_spec.rb +0 -31
  114. data/spec/utils/lag_reporter_spec.rb +0 -76
  115. data/spec/utils/platform_schema_validation_spec.rb +0 -0
  116. data/spec/utils/schema_controller_mixin_spec.rb +0 -84
  117. /data/lib/generators/deimos/{db_backend → outbox_backend}/templates/migration +0 -0
  118. /data/lib/generators/deimos/{db_backend → outbox_backend}/templates/rails3_migration +0 -0
@@ -79,7 +79,7 @@ module Deimos
79
79
  # Indicate a class which should act as a mocked version of this backend.
80
80
  # This class should perform all validations but not actually do any
81
81
  # encoding.
82
- # Note that the "mock" version (e.g. avro_validation) should return
82
+ # Note that the "mock" version should return
83
83
  # its own symbol when this is called, since it may be called multiple
84
84
  # times depending on the order of RSpec helpers.
85
85
  # @return [Symbol]
@@ -90,7 +90,7 @@ module Deimos
90
90
  # The content type to use when encoding / decoding requests over HTTP via ActionController.
91
91
  # @return [String]
92
92
  def self.content_type
93
- raise NotImplementedError
93
+ raise MissingImplementationError
94
94
  end
95
95
 
96
96
  # Converts your schema to String form for generated YARD docs.
@@ -98,7 +98,7 @@ module Deimos
98
98
  # @param schema [Object]
99
99
  # @return [String] A string representation of the Type
100
100
  def self.field_type(schema)
101
- raise NotImplementedError
101
+ raise MissingImplementationError
102
102
  end
103
103
 
104
104
  # Encode a payload. To be defined by subclass.
@@ -107,7 +107,7 @@ module Deimos
107
107
  # @param topic [String]
108
108
  # @return [String]
109
109
  def encode_payload(payload, schema:, topic: nil)
110
- raise NotImplementedError
110
+ raise MissingImplementationError
111
111
  end
112
112
 
113
113
  # Decode a payload. To be defined by subclass.
@@ -115,7 +115,7 @@ module Deimos
115
115
  # @param schema [String,Symbol]
116
116
  # @return [Hash]
117
117
  def decode_payload(payload, schema:)
118
- raise NotImplementedError
118
+ raise MissingImplementationError
119
119
  end
120
120
 
121
121
  # Validate that a payload matches the schema. To be defined by subclass.
@@ -123,13 +123,13 @@ module Deimos
123
123
  # @param schema [String,Symbol]
124
124
  # @return [void]
125
125
  def validate(payload, schema:)
126
- raise NotImplementedError
126
+ raise MissingImplementationError
127
127
  end
128
128
 
129
129
  # List of field names belonging to the schema. To be defined by subclass.
130
130
  # @return [Array<SchemaField>]
131
131
  def schema_fields
132
- raise NotImplementedError
132
+ raise MissingImplementationError
133
133
  end
134
134
 
135
135
  # Given a value and a field definition (as defined by whatever the
@@ -139,7 +139,7 @@ module Deimos
139
139
  # @param value [Object]
140
140
  # @return [Object]
141
141
  def coerce_field(field, value)
142
- raise NotImplementedError
142
+ raise MissingImplementationError
143
143
  end
144
144
 
145
145
  # Given a field definition, return the SQL type that might be used in
@@ -150,7 +150,14 @@ module Deimos
150
150
  # @param field [SchemaField]
151
151
  # @return [Symbol]
152
152
  def sql_type(field)
153
- raise NotImplementedError
153
+ raise MissingImplementationError
154
+ end
155
+
156
+ # Generate a key schema from the given value schema and key ID. This
157
+ # is used when encoding or decoding keys from an existing value schema.
158
+ # @param field_name [Symbol]
159
+ def generate_key_schema(field_name)
160
+ raise MissingImplementationError
154
161
  end
155
162
 
156
163
  # Encode a message key. To be defined by subclass.
@@ -159,7 +166,7 @@ module Deimos
159
166
  # @param topic [String]
160
167
  # @return [String]
161
168
  def encode_key(key, key_id, topic: nil)
162
- raise NotImplementedError
169
+ raise MissingImplementationError
163
170
  end
164
171
 
165
172
  # Decode a message key. To be defined by subclass.
@@ -167,13 +174,13 @@ module Deimos
167
174
  # @param key_id [String,Symbol] the field in the message to decode.
168
175
  # @return [String]
169
176
  def decode_key(payload, key_id)
170
- raise NotImplementedError
177
+ raise MissingImplementationError
171
178
  end
172
179
 
173
180
  # Forcefully loads the schema into memory.
174
181
  # @return [Object] The schema that is of use.
175
182
  def load_schema
176
- raise NotImplementedError
183
+ raise MissingImplementationError
177
184
  end
178
185
  end
179
186
  end
@@ -4,6 +4,11 @@ module Deimos
4
4
  module SchemaBackends
5
5
  # Mock implementation of a schema backend that does no encoding or validation.
6
6
  class Mock < Base
7
+
8
+ # @override
9
+ def generate_key_schema(field_name)
10
+ end
11
+
7
12
  # @override
8
13
  def decode_payload(payload, schema:)
9
14
  payload.is_a?(String) ? 'payload-decoded' : payload.map { |k, v| [k, "decoded-#{v}"] }
@@ -11,7 +16,7 @@ module Deimos
11
16
 
12
17
  # @override
13
18
  def encode_payload(payload, schema:, topic: nil)
14
- payload.is_a?(String) ? 'payload-encoded' : payload.map { |k, v| [k, "encoded-#{v}"] }
19
+ payload.is_a?(String) ? 'payload-encoded' : payload.map { |k, v| [k, "encoded-#{v}"] }.to_json
15
20
  end
16
21
 
17
22
  # @override
@@ -0,0 +1,47 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Deimos
4
+ module SchemaBackends
5
+ # Schema backend that passes through as a basic string.
6
+ class Plain < Base
7
+
8
+ # @override
9
+ def generate_key_schema(field_name)
10
+ end
11
+
12
+ # @override
13
+ def decode_payload(payload, schema:)
14
+ payload
15
+ end
16
+
17
+ # @override
18
+ def encode_payload(payload, schema:, topic: nil)
19
+ payload.to_s
20
+ end
21
+
22
+ # @override
23
+ def validate(payload, schema:)
24
+ end
25
+
26
+ # @override
27
+ def schema_fields
28
+ []
29
+ end
30
+
31
+ # @override
32
+ def coerce_field(_field, value)
33
+ value
34
+ end
35
+
36
+ # @override
37
+ def encode_key(key_id, key, topic: nil)
38
+ key
39
+ end
40
+
41
+ # @override
42
+ def decode_key(payload, key_id)
43
+ payload[key_id]
44
+ end
45
+ end
46
+ end
47
+ end
@@ -15,7 +15,7 @@ module Deimos
15
15
  # @param _opts [Hash]
16
16
  # @return [Hash] a hash representation of the payload
17
17
  def as_json(_opts={})
18
- raise NotImplementedError
18
+ raise MissingImplementationError
19
19
  end
20
20
 
21
21
  # @param key [String,Symbol]
@@ -46,7 +46,7 @@ module Deimos
46
46
  # @param value [Object]
47
47
  # @return [SchemaClass::Base]
48
48
  def self.initialize_from_value(value)
49
- raise NotImplementedError
49
+ raise MissingImplementationError
50
50
  end
51
51
 
52
52
  protected
@@ -30,7 +30,7 @@ module Deimos
30
30
  # Returns all the valid symbols for this enum.
31
31
  # @return [Array<String>]
32
32
  def symbols
33
- raise NotImplementedError
33
+ raise MissingImplementationError
34
34
  end
35
35
 
36
36
  # @return [String]
@@ -46,13 +46,13 @@ module Deimos
46
46
  # Returns the schema name of the inheriting class.
47
47
  # @return [String]
48
48
  def schema
49
- raise NotImplementedError
49
+ raise MissingImplementationError
50
50
  end
51
51
 
52
52
  # Returns the namespace for the schema of the inheriting class.
53
53
  # @return [String]
54
54
  def namespace
55
- raise NotImplementedError
55
+ raise MissingImplementationError
56
56
  end
57
57
 
58
58
  # Returns the full schema name of the inheriting class.