deimos-ruby 1.24.2 → 2.0.0.pre.alpha1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (120) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop_todo.yml +0 -17
  3. data/.tool-versions +1 -0
  4. data/CHANGELOG.md +5 -0
  5. data/README.md +287 -498
  6. data/deimos-ruby.gemspec +4 -4
  7. data/docs/CONFIGURATION.md +133 -226
  8. data/docs/UPGRADING.md +237 -0
  9. data/lib/deimos/active_record_consume/batch_consumption.rb +29 -28
  10. data/lib/deimos/active_record_consume/mass_updater.rb +59 -4
  11. data/lib/deimos/active_record_consume/message_consumption.rb +15 -21
  12. data/lib/deimos/active_record_consumer.rb +36 -21
  13. data/lib/deimos/active_record_producer.rb +28 -9
  14. data/lib/deimos/backends/base.rb +4 -35
  15. data/lib/deimos/backends/kafka.rb +6 -22
  16. data/lib/deimos/backends/kafka_async.rb +6 -22
  17. data/lib/deimos/backends/{db.rb → outbox.rb} +13 -9
  18. data/lib/deimos/config/configuration.rb +116 -379
  19. data/lib/deimos/consume/batch_consumption.rb +24 -124
  20. data/lib/deimos/consume/message_consumption.rb +36 -63
  21. data/lib/deimos/consumer.rb +16 -75
  22. data/lib/deimos/ext/consumer_route.rb +35 -0
  23. data/lib/deimos/ext/producer_middleware.rb +94 -0
  24. data/lib/deimos/ext/producer_route.rb +22 -0
  25. data/lib/deimos/ext/redraw.rb +29 -0
  26. data/lib/deimos/ext/routing_defaults.rb +72 -0
  27. data/lib/deimos/ext/schema_route.rb +70 -0
  28. data/lib/deimos/kafka_message.rb +2 -2
  29. data/lib/deimos/kafka_source.rb +2 -7
  30. data/lib/deimos/kafka_topic_info.rb +1 -1
  31. data/lib/deimos/logging.rb +71 -0
  32. data/lib/deimos/message.rb +2 -11
  33. data/lib/deimos/metrics/datadog.rb +40 -1
  34. data/lib/deimos/metrics/provider.rb +4 -4
  35. data/lib/deimos/producer.rb +39 -116
  36. data/lib/deimos/railtie.rb +6 -0
  37. data/lib/deimos/schema_backends/avro_base.rb +21 -21
  38. data/lib/deimos/schema_backends/avro_schema_registry.rb +1 -2
  39. data/lib/deimos/schema_backends/avro_validation.rb +2 -2
  40. data/lib/deimos/schema_backends/base.rb +19 -12
  41. data/lib/deimos/schema_backends/mock.rb +6 -1
  42. data/lib/deimos/schema_backends/plain.rb +47 -0
  43. data/lib/deimos/schema_class/base.rb +2 -2
  44. data/lib/deimos/schema_class/enum.rb +1 -1
  45. data/lib/deimos/schema_class/record.rb +2 -2
  46. data/lib/deimos/test_helpers.rb +95 -320
  47. data/lib/deimos/tracing/provider.rb +6 -6
  48. data/lib/deimos/transcoder.rb +88 -0
  49. data/lib/deimos/utils/db_poller/base.rb +16 -14
  50. data/lib/deimos/utils/db_poller/state_based.rb +3 -3
  51. data/lib/deimos/utils/db_poller/time_based.rb +4 -4
  52. data/lib/deimos/utils/db_poller.rb +1 -1
  53. data/lib/deimos/utils/deadlock_retry.rb +1 -1
  54. data/lib/deimos/utils/{db_producer.rb → outbox_producer.rb} +16 -47
  55. data/lib/deimos/utils/schema_class.rb +0 -7
  56. data/lib/deimos/version.rb +1 -1
  57. data/lib/deimos.rb +79 -26
  58. data/lib/generators/deimos/{db_backend_generator.rb → outbox_backend_generator.rb} +4 -4
  59. data/lib/generators/deimos/schema_class_generator.rb +0 -1
  60. data/lib/generators/deimos/v2/templates/karafka.rb.tt +149 -0
  61. data/lib/generators/deimos/v2_generator.rb +193 -0
  62. data/lib/tasks/deimos.rake +5 -7
  63. data/spec/active_record_batch_consumer_association_spec.rb +22 -13
  64. data/spec/active_record_batch_consumer_spec.rb +84 -65
  65. data/spec/active_record_consume/batch_consumption_spec.rb +10 -10
  66. data/spec/active_record_consume/batch_slicer_spec.rb +12 -12
  67. data/spec/active_record_consume/mass_updater_spec.rb +137 -0
  68. data/spec/active_record_consumer_spec.rb +29 -13
  69. data/spec/active_record_producer_spec.rb +36 -26
  70. data/spec/backends/base_spec.rb +0 -23
  71. data/spec/backends/kafka_async_spec.rb +1 -3
  72. data/spec/backends/kafka_spec.rb +1 -3
  73. data/spec/backends/{db_spec.rb → outbox_spec.rb} +14 -20
  74. data/spec/batch_consumer_spec.rb +66 -116
  75. data/spec/consumer_spec.rb +53 -147
  76. data/spec/deimos_spec.rb +10 -126
  77. data/spec/kafka_source_spec.rb +19 -52
  78. data/spec/karafka/karafka.rb +69 -0
  79. data/spec/karafka_config/karafka_spec.rb +97 -0
  80. data/spec/logging_spec.rb +25 -0
  81. data/spec/message_spec.rb +9 -9
  82. data/spec/producer_spec.rb +112 -254
  83. data/spec/rake_spec.rb +1 -3
  84. data/spec/schema_backends/avro_validation_spec.rb +1 -1
  85. data/spec/schemas/com/my-namespace/MySchemaWithTitle.avsc +22 -0
  86. data/spec/snapshots/consumers-no-nest.snap +49 -0
  87. data/spec/snapshots/consumers.snap +49 -0
  88. data/spec/snapshots/consumers_and_producers-no-nest.snap +49 -0
  89. data/spec/snapshots/consumers_and_producers.snap +49 -0
  90. data/spec/snapshots/consumers_circular-no-nest.snap +49 -0
  91. data/spec/snapshots/consumers_circular.snap +49 -0
  92. data/spec/snapshots/consumers_complex_types-no-nest.snap +49 -0
  93. data/spec/snapshots/consumers_complex_types.snap +49 -0
  94. data/spec/snapshots/consumers_nested-no-nest.snap +49 -0
  95. data/spec/snapshots/consumers_nested.snap +49 -0
  96. data/spec/snapshots/namespace_folders.snap +49 -0
  97. data/spec/snapshots/namespace_map.snap +49 -0
  98. data/spec/snapshots/producers_with_key-no-nest.snap +49 -0
  99. data/spec/snapshots/producers_with_key.snap +49 -0
  100. data/spec/spec_helper.rb +61 -29
  101. data/spec/utils/db_poller_spec.rb +49 -39
  102. data/spec/utils/{db_producer_spec.rb → outbox_producer_spec.rb} +17 -184
  103. metadata +58 -67
  104. data/lib/deimos/batch_consumer.rb +0 -7
  105. data/lib/deimos/config/phobos_config.rb +0 -163
  106. data/lib/deimos/instrumentation.rb +0 -95
  107. data/lib/deimos/monkey_patches/phobos_cli.rb +0 -35
  108. data/lib/deimos/utils/inline_consumer.rb +0 -158
  109. data/lib/deimos/utils/lag_reporter.rb +0 -186
  110. data/lib/deimos/utils/schema_controller_mixin.rb +0 -129
  111. data/spec/config/configuration_spec.rb +0 -321
  112. data/spec/kafka_listener_spec.rb +0 -55
  113. data/spec/phobos.bad_db.yml +0 -73
  114. data/spec/phobos.yml +0 -77
  115. data/spec/utils/inline_consumer_spec.rb +0 -31
  116. data/spec/utils/lag_reporter_spec.rb +0 -76
  117. data/spec/utils/platform_schema_validation_spec.rb +0 -0
  118. data/spec/utils/schema_controller_mixin_spec.rb +0 -84
  119. /data/lib/generators/deimos/{db_backend → outbox_backend}/templates/migration +0 -0
  120. /data/lib/generators/deimos/{db_backend → outbox_backend}/templates/rails3_migration +0 -0
@@ -18,9 +18,11 @@ module Deimos
18
18
  # a record object, refetch the record to pass into the `generate_payload`
19
19
  # method.
20
20
  # @return [void]
21
- def record_class(klass, refetch: true)
22
- config[:record_class] = klass
23
- config[:refetch_record] = refetch
21
+ def record_class(klass=nil, refetch: true)
22
+ return @record_class if klass.nil?
23
+
24
+ @record_class = klass
25
+ @refetch_record = refetch
24
26
  end
25
27
 
26
28
  # @param record [ActiveRecord::Base]
@@ -34,14 +36,14 @@ module Deimos
34
36
  # @param force_send [Boolean]
35
37
  # @return [void]
36
38
  def send_events(records, force_send: false)
37
- primary_key = config[:record_class]&.primary_key
39
+ primary_key = @record_class&.primary_key
38
40
  messages = records.map do |record|
39
41
  if record.respond_to?(:attributes)
40
42
  attrs = record.attributes.with_indifferent_access
41
43
  else
42
44
  attrs = record.with_indifferent_access
43
- if config[:refetch_record] && attrs[primary_key]
44
- record = config[:record_class].find(attrs[primary_key])
45
+ if @refetch_record && attrs[primary_key]
46
+ record = @record_class.find(attrs[primary_key])
45
47
  end
46
48
  end
47
49
  generate_payload(attrs, record).with_indifferent_access
@@ -50,6 +52,15 @@ module Deimos
50
52
  self.post_process(records)
51
53
  end
52
54
 
55
+ def config
56
+ Deimos.karafka_configs.find { |t| t.producer_class == self }
57
+ end
58
+
59
+ def encoder
60
+ raise "No schema or namespace configured for #{self.name}" if config.nil?
61
+ config.deserializers[:payload].backend
62
+ end
63
+
53
64
  # Generate the payload, given a list of attributes or a record..
54
65
  # Can be overridden or added to by subclasses.
55
66
  # @param attributes [Hash]
@@ -62,9 +73,9 @@ module Deimos
62
73
  payload.delete_if do |k, _|
63
74
  k.to_sym != :payload_key && !fields.map(&:name).include?(k)
64
75
  end
65
- return payload unless Utils::SchemaClass.use?(config.to_h)
76
+ return payload unless self.config.use_schema_classes
66
77
 
67
- Utils::SchemaClass.instance(payload, config[:schema], config[:namespace])
78
+ Utils::SchemaClass.instance(payload, encoder.schema, encoder.namespace)
68
79
  end
69
80
 
70
81
  # Query to use when polling the database with the DbPoller. Add
@@ -76,7 +87,7 @@ module Deimos
76
87
  # than this value).
77
88
  # @return [ActiveRecord::Relation]
78
89
  def poll_query(time_from:, time_to:, column_name: :updated_at, min_id:)
79
- klass = config[:record_class]
90
+ klass = @record_class
80
91
  table = ActiveRecord::Base.connection.quote_table_name(klass.table_name)
81
92
  column = ActiveRecord::Base.connection.quote_column_name(column_name)
82
93
  primary = ActiveRecord::Base.connection.quote_column_name(klass.primary_key)
@@ -95,6 +106,14 @@ module Deimos
95
106
  def post_process(_records)
96
107
  end
97
108
 
109
+ # Override this in active record producers to add
110
+ # non-schema fields to check for updates
111
+ # @param _record [ActiveRecord::Base]
112
+ # @return [Array<String>] fields to check for updates
113
+ def watched_attributes(_record)
114
+ self.encoder.schema_fields.map(&:name)
115
+ end
116
+
98
117
  end
99
118
  end
100
119
  end
@@ -6,10 +6,11 @@ module Deimos
6
6
  class Base
7
7
  class << self
8
8
  # @param producer_class [Class<Deimos::Producer>]
9
- # @param messages [Array<Deimos::Message>]
9
+ # @param messages [Array<Hash>]
10
10
  # @return [void]
11
11
  def publish(producer_class:, messages:)
12
- Deimos.config.logger.info(log_message(messages))
12
+ message = ::Deimos::Logging.messages_log_text(producer_class.karafka_config.payload_log, messages)
13
+ Deimos::Logging.log_info({message: 'Publishing Messages:'}.merge(message))
13
14
  execute(producer_class: producer_class, messages: messages)
14
15
  end
15
16
 
@@ -17,43 +18,11 @@ module Deimos
17
18
  # @param messages [Array<Deimos::Message>]
18
19
  # @return [void]
19
20
  def execute(producer_class:, messages:)
20
- raise NotImplementedError
21
+ raise MissingImplementationError
21
22
  end
22
23
 
23
24
  private
24
25
 
25
- def log_message(messages)
26
- log_message = {
27
- message: 'Publishing messages',
28
- topic: messages.first&.topic
29
- }
30
-
31
- case Deimos.config.payload_log
32
- when :keys
33
- log_message.merge!(
34
- payload_keys: messages.map(&:key)
35
- )
36
- when :count
37
- log_message.merge!(
38
- payloads_count: messages.count
39
- )
40
- when :headers
41
- log_message.merge!(
42
- payload_headers: messages.map(&:headers)
43
- )
44
- else
45
- log_message.merge!(
46
- payloads: messages.map do |message|
47
- {
48
- payload: message.payload,
49
- key: message.key
50
- }
51
- end
52
- )
53
- end
54
-
55
- log_message
56
- end
57
26
  end
58
27
  end
59
28
  end
@@ -4,30 +4,14 @@ module Deimos
4
4
  module Backends
5
5
  # Default backend to produce to Kafka.
6
6
  class Kafka < Base
7
- include Phobos::Producer
8
-
9
- # Shut down the producer if necessary.
10
- # @return [void]
11
- def self.shutdown_producer
12
- producer.sync_producer_shutdown if producer.respond_to?(:sync_producer_shutdown)
13
- producer.kafka_client&.close
14
- end
15
-
16
7
  # :nodoc:
17
8
  def self.execute(producer_class:, messages:)
18
- Deimos.instrument(
19
- 'produce',
20
- producer: producer_class,
21
- topic: producer_class.topic,
22
- payloads: messages.map(&:payload)
23
- ) do
24
- producer.publish_list(messages.map(&:encoded_hash))
25
- Deimos.config.metrics&.increment(
26
- 'publish',
27
- tags: %W(status:success topic:#{producer_class.topic}),
28
- by: messages.size
29
- )
30
- end
9
+ Karafka.producer.produce_many_sync(messages)
10
+ Deimos.config.metrics&.increment(
11
+ 'publish',
12
+ tags: %W(status:success topic:#{messages.first[:topic]}),
13
+ by: messages.size
14
+ )
31
15
  end
32
16
  end
33
17
  end
@@ -4,30 +4,14 @@ module Deimos
4
4
  module Backends
5
5
  # Backend which produces to Kafka via an async producer.
6
6
  class KafkaAsync < Base
7
- include Phobos::Producer
8
-
9
- # Shut down the producer cleanly.
10
- # @return [void]
11
- def self.shutdown_producer
12
- producer.async_producer_shutdown
13
- producer.kafka_client&.close
14
- end
15
-
16
7
  # :nodoc:
17
8
  def self.execute(producer_class:, messages:)
18
- Deimos.instrument(
19
- 'produce',
20
- producer: producer_class,
21
- topic: producer_class.topic,
22
- payloads: messages.map(&:payload)
23
- ) do
24
- producer.async_publish_list(messages.map(&:encoded_hash))
25
- Deimos.config.metrics&.increment(
26
- 'publish',
27
- tags: %W(status:success topic:#{producer_class.topic}),
28
- by: messages.size
29
- )
30
- end
9
+ Karafka.producer.produce_many_async(messages)
10
+ Deimos.config.metrics&.increment(
11
+ 'publish',
12
+ tags: %W(status:success topic:#{messages.first[:topic]}),
13
+ by: messages.size
14
+ )
31
15
  end
32
16
  end
33
17
  end
@@ -6,22 +6,23 @@ module Deimos
6
6
  module Backends
7
7
  # Backend which saves messages to the database instead of immediately
8
8
  # sending them.
9
- class Db < Base
9
+ class Outbox < Base
10
10
  class << self
11
11
  # :nodoc:
12
12
  def execute(producer_class:, messages:)
13
13
  records = messages.map do |m|
14
+ Deimos::ProducerMiddleware.call(m)
14
15
  message = Deimos::KafkaMessage.new(
15
- message: m.encoded_payload ? m.encoded_payload.to_s.b : nil,
16
- topic: m.topic,
16
+ message: m[:payload] ? m[:payload].to_s.b : nil,
17
+ topic: m[:topic],
17
18
  partition_key: partition_key_for(m)
18
19
  )
19
- message.key = m.encoded_key.to_s.b unless producer_class.config[:no_keys]
20
+ message.key = m[:key].to_s.b if m[:key]
20
21
  message
21
22
  end
22
23
  Deimos::KafkaMessage.import(records)
23
24
  Deimos.config.metrics&.increment(
24
- 'db_producer.insert',
25
+ 'outbox.insert',
25
26
  tags: %W(topic:#{producer_class.topic}),
26
27
  by: records.size
27
28
  )
@@ -30,10 +31,13 @@ module Deimos
30
31
  # @param message [Deimos::Message]
31
32
  # @return [String] the partition key to use for this message
32
33
  def partition_key_for(message)
33
- return message.partition_key if message.partition_key.present?
34
- return message.key unless message.key.is_a?(Hash)
35
-
36
- message.key.to_yaml
34
+ if message[:partition_key].present?
35
+ message[:partition_key]
36
+ elsif message[:key].present?
37
+ message[:key].to_s.b
38
+ else
39
+ nil
40
+ end
37
41
  end
38
42
  end
39
43
  end