deimos-ruby 1.0.0.pre.beta22

data/Rakefile

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+begin
+  require 'rspec/core/rake_task'
+
+  RSpec::Core::RakeTask.new(:spec)
+  task(default: :spec)
+rescue LoadError # rubocop:disable Lint/HandleExceptions
+  # no rspec available
+end
+
+import('./lib/tasks/phobos.rake')

data/bin/deimos

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'deimos'

data/deimos-kafka.gemspec

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+lib = File.expand_path('lib', __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'deimos/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'deimos-ruby'
+  spec.version       = Deimos::VERSION
+  spec.authors       = ['Daniel Orner']
+  spec.email         = ['daniel.orner@wishabi.com']
+  spec.summary       = 'Kafka libraries for Ruby.'
+  spec.homepage      = ''
+  spec.license       = 'Apache-2.0'
+
+  spec.files         = `git ls-files`.split($INPUT_RECORD_SEPARATOR)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+
+  spec.add_runtime_dependency('avro-patches', '~> 0.3')
+  spec.add_runtime_dependency('avro_turf', '~> 0.8')
+  spec.add_runtime_dependency('phobos', '~> 1.8')
+  spec.add_runtime_dependency('ruby-kafka', '~> 0.7')
+
+  spec.add_development_dependency('activerecord', '~> 5.2')
+  spec.add_development_dependency('activerecord-import')
+  spec.add_development_dependency('bundler', '~> 1')
+  spec.add_development_dependency('ddtrace', '~> 0.11')
+  spec.add_development_dependency('dogstatsd-ruby', '~> 4.2')
+  spec.add_development_dependency('guard', '~> 2')
+  spec.add_development_dependency('guard-rspec', '~> 4')
+  spec.add_development_dependency('guard-rubocop', '~> 1')
+  spec.add_development_dependency('mysql2', '~> 0.5')
+  spec.add_development_dependency('pg', '~> 1.1')
+  spec.add_development_dependency('rake', '~> 10')
+  spec.add_development_dependency('rspec', '~> 3')
+  spec.add_development_dependency('rspec_junit_formatter', '~>0.3')
+  spec.add_development_dependency('rubocop', '~> 0.72')
+  spec.add_development_dependency('rubocop-rspec', '~> 1.27')
+  spec.add_development_dependency('sqlite3', '~> 1.3')
+end

data/docker-compose.yml

@@ -0,0 +1,71 @@
+version: '3.6'
+services:
+  mysql:
+    image: mysql:5.7
+    expose:
+      - 3306
+    environment:
+      - MYSQL_ALLOW_EMPTY_PASSWORD=yes
+      - MYSQL_DATABASE=test
+      - TZ=America/Toronto
+
+  postgres:
+    image: postgres:11.1
+    expose:
+      - 5432
+    environment:
+      POSTGRES_PASSWORD: root
+
+  test:
+    volumes:
+      - .:/var/app
+    depends_on:
+      - kafka-broker
+      - mysql
+      - postgres
+    build: .
+    environment:
+      - "DEFAULT_TIMEOUT=${DEFAULT_TIMEOUT}"
+      - MYSQL_HOST=mysql
+      - PG_HOST=postgres
+      - SCHEMA_REGISTRY=http://schema-registry:8081
+      - KAFKA_SEED_BROKER=kafka-broker:9092
+    command: dockerize -wait tcp://mysql:3306 -wait tcp://postgres:5432 -timeout 1m rspec
+
+  zookeeper:
+    image: wurstmeister/zookeeper:latest
+    ports:
+     - 2181:2181
+
+  schema-registry:
+    image: confluentinc/cp-schema-registry
+    hostname: schema-registry
+    depends_on:
+      - zookeeper
+      - kafka-broker
+    ports:
+      - "8081:8081"
+    environment:
+      SCHEMA_REGISTRY_HOST_NAME: schema-registry
+      SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181'
+
+  kafka-broker:
+    image: confluentinc/cp-enterprise-kafka
+    hostname: kafka-broker
+    depends_on:
+      - zookeeper
+    ports:
+      - "9092:9092"
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
+      KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://kafka-broker:9092'
+      KAFKA_METRIC_REPORTERS: io.confluent.metrics.reporter.ConfluentMetricsReporter
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
+      CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: kafka-broker:9092
+      CONFLUENT_METRICS_REPORTER_ZOOKEEPER_CONNECT: zookeeper:2181
+      CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1
+      CONFLUENT_METRICS_ENABLE: 'true'
+      CONFLUENT_SUPPORT_CUSTOMER_ID: 'anonymous'
+

data/docs/DATABASE_BACKEND.md

@@ -0,0 +1,147 @@
+# Database Backend Design
+
+Kafka is a messaging protocol, while databases are transactional and relational. 
+Marrying the two (e.g. by using Kafka to publish changes to a database table)
+is not a simple task. This document describes the problem and the current
+implementation. There will be references to microservices architecture as that
+informs some of the discussion.
+
+## A Pure Solution
+
+The purest solution is to use Kafka as the "source of truth" by first publishing
+all messages synchronously to Kafka, and then using a consumer to read these 
+messages back into the database. If there are any errors sending to Kafka,
+the thread will crash and no data would be written. If there are errors
+reading the data back into the database, the data remains in Kafka and can
+be re-read at any time.
+
+There are several real-world problems with this pure solution:
+
+1. The solution assumes that creating a consumer is a simple task, which is 
+   definitely not the case. Depending on how many topics are being produced to,
+   a separate consumer thread per topic (which is how Phobos works) is overkill.
+   The other option is to introduce an entirely new consumer service to handle
+   consuming the topics we've already produced, which is even more overkill.
+2.  For CRUD interfaces or any other UI that saves data to the database, we do 
+    not want to use an asynchronous method to ensure that data is published to 
+    Kafka before saving, and then being able to serve that data back to the user
+    in a single API call, which is a common use case. 
+    This could involve a large amount of added complexity and may force the user 
+    to wait unnecessarily.
+3.  We want to make use of database transactions - i.e. if an error happens 
+    saving one record, the others should roll back. Once a message is written to 
+    Kafka, it can't be "rolled back" easily. Kafka transactions do exist but 
+    they are not widely supported in Kafka clients, and we still would be 
+    faced with the fact that one transaction (Kafka or database) could finish 
+    and then the process could be killed before the other one could be finished.
+4.  We want to make use of auto-increment database IDs - we can't do this if we 
+    write to Kafka first.
+5.  Kafka is an external dependency. If either the DB **or** Kafka goes down, 
+    our app becomes unusable.
+
+Using tools like Kafka Connect and Debezium are not ideal because:
+
+1. They are tied very closely to the internal relational schema, which is not 
+   ideal, especially for legacy systems. It makes it nearly impossible to make 
+   internal changes.
+2. They are separate services and connectors must be created for each 
+   microservice separately, which is a large overhead.
+
+## Database Backend Solution
+
+We will be using the database itself as the source of our Kafka messages. 
+We will first write our messages to a database table and then asynchronously 
+send those messages to Kafka. This solves our problems:
+
+1.  The database is the (interim) source of truth. The Kafka message log is 
+    essentially the changelog, which we can tail and send out. If our producing 
+    thread errors out, a new one will simply pick up where it left off. 
+    This ensures eventual consistency.
+2.  Because we are only using the database in the main application thread, we do 
+    not need to wait for Kafka production to continue and can return immediately.
+3.  Because we are only saving to the database, we can use transactions normally 
+    - if a transaction fails, it will roll back along with any Kafka messages we 
+    intended to send.
+4.  Records are saved normally and messages are created after that, all as part 
+    of the transaction, so we can use database IDs as usual.
+5.  We remove Kafka entirely as a dependency for normal work - the Kafka sending 
+    piece is a separate thread.
+    
+The one downside to this is a slight delay (generally less than 1 second) 
+between the message being written to the database and sent to Kafka - in most 
+cases this is an acceptable limitation.
+
+### The Implementation
+
+The database backend consists of three tables:
+
+* `kafka_messages` - this keeps track of the messages that were "published",
+  including the payload, topic, key and partition key. These messages
+  are *raw data* - all processing, including Avro encoding, must happen
+  upstream before they are inserted.
+* `kafka_topic_info` - this table is essentially a lock table used to ensure
+  that only one producer thread is ever "working" on a topic at a time.
+
+The backend code structure is such that when a producer calls `publish_list`,
+it delegates that logic to the configured backend. A backend of `kafka`
+or `kafka_async` will use existing Phobos logic. A backend of `db` will use
+the database backend instead.
+
+### "Publishing" A Message
+
+When `publish_list` is called when the database backend is configured, 
+Deimos will instead save the message to the `kafka_messages` table.
+
+### Sending Messages to Kafka
+
+The database executor is started by calling `Deimos.start_db_backend!`
+with a specified number of threads. These threads will continually scan the
+two messages tables and send the messages to Kafka. 
+
+The algorithm for sending the messages makes use of the `kafka_topic_info` table as a lock table. There is also an `error` boolean column which is used to track when a topic has errored out. When this happens, the topic is marked as errored and will not be picked up for the next minute, after which it will be treated as any other topic. The full algorithm is as follows:
+
+* Create a UUID for the thread - this is created once on thread start.
+* Find all unique topics in the `kafka_messages` table.
+* For each topic:
+  * Create an entry in `kafka_topic_info` for this topic if it doesn't exist.
+  * Run the following query: 
+  
+      ```sql
+            UPDATE kafka_topic_info 
+            SET locked_by=#{uuid}, locked_at=NOW(), error=0
+            WHERE (locked_by IS NULL AND error=0) OR locked_at < #{1.minute.ago} 
+            LIMIT 1
+      ```
+     * If the lock was unsuccessful, move on to the next topic in the list
+  * Find the first 1000 messages in `kafka_messages` for that topic, ordered by ID (insertion order)
+  * Send the messages synchronously to Kafka, with all brokers acking the message.
+  * Delete the records from the DB
+  * Update the `locked_at` timestamp in `kafka_topic_info` to `NOW()` to ensure liveness in case a particular batch took longer than expected to send.
+  * If the current batch is 1000 messages, repeat with the next batch of
+    messages until it returns less than 1000
+  * When all batches are sent:
+      * Unlock the topic by updating the `kafka_topic_info` for this topic, setting `locked_by=NULL, locked_at=NULL, error=0, retries=0`
+      * Move on to the next topic
+   * If there are errors sending a batch:
+     * Update the `kafka_topic_info` for this topic to have `locked_by=NULL, locked_at=NULL, error=1, retries=retries+1` - this will effectively keep it
+       locked for the next minute
+     * Move on to the next topic.
+* When all topics are done, or if there are no topics, sleep for 0.5 seconds and begin again.
+
+### Class / Method Design
+
+The algorithm is split up into the following classes:
+
+* Backends::Db - this is the class that saves the message to the database.
+* KafkaMessage: This is an ActiveRecord class that handle saving the messages to the database and querying them.
+* KafkaTopicInfo: This is an ActiveRecord class that handles locking, unlocking and heartbeating.
+* Utils::SignalHandler: This is the equivalent of Phobos's Runner class and
+  handles the KILL, INT and TERM signals to gracefully shut down the threads.
+* Utils::Executor is the equivalent of Phobos's Executor class and handles
+  the thread pool of producer threads.
+* Utils::DbProducer is the producer thread itself which implements most of the
+  algorithm listed above.
+
+### Caveats
+
+There is one disadvantage of this pattern, which is that it is possible for events to be sent multiple times if the thread which sent the messages dies before being able to delete it from the database. In general this is an acceptable effect, since Kafka only guarantees at-least-once delivery in any case.

data/docs/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,34 @@
+# Pull Request Template
+
+## Description
+
+Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. List any dependencies that are required for this change.
+
+Fixes # (issue)
+
+## Type of change
+
+Please delete options that are not relevant.
+
+- [ ] Bug fix (non-breaking change which fixes an issue)
+- [ ] New feature (non-breaking change which adds functionality)
+- [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected)
+- [ ] This change requires a documentation update
+
+## How Has This Been Tested?
+
+Please describe the tests that you ran to verify your changes. Provide instructions so we can reproduce. Please also list any relevant details for your test configuration
+
+- [ ] Test A
+- [ ] Test B
+
+## Checklist:
+
+- [ ] My code follows the style guidelines of this project
+- [ ] I have performed a self-review of my own code
+- [ ] I have commented my code, particularly in hard-to-understand areas
+- [ ] I have made corresponding changes to the documentation
+- [ ] My changes generate no new warnings
+- [ ] I have added tests that prove my fix is effective or that my feature works
+- [ ] New and existing unit tests pass locally with my changes
+- [ ] Any dependent changes have been merged and published in downstream modules

data/lib/deimos/active_record_consumer.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+require 'deimos/consumer'
+
+module Deimos
+  # Consumer that automatically saves the payload into the database.
+  class ActiveRecordConsumer < Consumer
+    class << self
+      # param klass [Class < ActiveRecord::Base] the class used to save to the
+      # database.
+      def record_class(klass)
+        config[:record_class] = klass
+      end
+    end
+
+    # :nodoc:
+    def consume(payload, metadata)
+      key = metadata.with_indifferent_access[:key]
+      klass = self.class.config[:record_class]
+      record = klass.where(klass.primary_key => key).first
+      if payload.nil?
+        destroy_record(record)
+        return
+      end
+      record ||= klass.new
+      attrs = record_attributes(payload.with_indifferent_access)
+      # don't use attributes= - bypass Rails < 5 attr_protected
+      attrs.each do |k, v|
+        record.send("#{k}=", v)
+      end
+      record[klass.primary_key] = key
+      record.created_at ||= Time.zone.now if record.respond_to?(:created_at)
+      record.updated_at ||= Time.zone.now if record.respond_to?(:updated_at)
+      record.save!
+    end
+
+    # Destroy a record that received a null payload. Override if you need
+    # to do something other than a straight destroy (e.g. mark as archived).
+    # @param record [ActiveRecord::Base]
+    def destroy_record(record)
+      record&.destroy
+    end
+
+    # Override this method (with `super`) if you want to add/change the default
+    # attributes set to the new/existing record.
+    # @param payload [Hash]
+    def record_attributes(payload)
+      klass = self.class.config[:record_class]
+      attributes = {}
+      schema = self.class.decoder.avro_schema
+      schema.fields.each do |field|
+        column = klass.columns.find { |c| c.name == field.name }
+        next if column.nil?
+        next if %w(updated_at created_at).include?(field.name)
+
+        attributes[field.name] = _coerce_field(field, column, payload[field.name])
+      end
+      attributes
+    end
+
+  private
+
+    # @param field [Avro::Schema]
+    # @param column [ActiveRecord::ConnectionAdapters::Column]
+    # @param val [Object]
+    def _coerce_field(field, column, val)
+      return nil if val.nil?
+
+      field_type = field.type.type.to_sym
+      if field_type == :union
+        union_types = field.type.schemas.map { |s| s.type.to_sym }
+        field_type = union_types.find { |t| t != :null }
+      end
+      if column.type == :datetime && %i(int long).include?(field_type)
+        return Time.zone.strptime(val.to_s, '%s')
+      end
+
+      val
+    end
+  end
+end

data/lib/deimos/active_record_producer.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+require 'deimos/producer'
+
+module Deimos
+  # Class which automatically produces a record when given an ActiveRecord
+  # instance or a list of them. Just call `send_events` on a list of records
+  # and they will be auto-published. You can override `generate_payload`
+  # to make changes to the payload before it's published.
+  #
+  # You can also call this with a list of hashes representing attributes.
+  # This is common when using activerecord-import.
+  class ActiveRecordProducer < Producer
+    class << self
+      # Indicate the class this producer is working on.
+      # @param klass [Class]
+      # @param refetch [Boolean] if true, and we are given a hash instead of
+      # a record object, refetch the record to pass into the `generate_payload`
+      # method.
+      def record_class(klass, refetch: true)
+        config[:record_class] = klass
+        config[:refetch_record] = refetch
+      end
+
+      # @param record [ActiveRecord::Base]
+      # @param force_send [Boolean]
+      def send_event(record, force_send: false)
+        send_events([record], force_send: force_send)
+      end
+
+      # @param records [Array<ActiveRecord::Base>]
+      # @param force_send [Boolean]
+      def send_events(records, force_send: false)
+        primary_key = config[:record_class]&.primary_key
+        messages = records.map do |record|
+          if record.respond_to?(:attributes)
+            attrs = record.attributes.with_indifferent_access
+          else
+            attrs = record.with_indifferent_access
+            if config[:refetch_record] && attrs[primary_key]
+              record = config[:record_class].find(attrs[primary_key])
+            end
+          end
+          generate_payload(attrs, record).with_indifferent_access
+        end
+        self.publish_list(messages, force_send: force_send)
+      end
+
+      # Generate the payload, given a list of attributes or a record..
+      # Can be overridden or added to by subclasses.
+      # @param attributes [Hash]
+      # @param _record [ActiveRecord::Base] May be nil if refetch_record
+      # is not set.
+      # @return [Hash]
+      def generate_payload(attributes, _record)
+        schema = self.encoder.avro_schema
+        payload = attributes.stringify_keys
+        payload.delete_if do |k, _|
+          k.to_sym != :payload_key && !schema.fields.find { |f| f.name == k }
+        end
+      end
+    end
+  end
+end

data/lib/deimos/avro_data_coder.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+module Deimos
+  # Base class for the encoder / decoder classes.
+  class AvroDataCoder
+    attr_accessor :schema, :namespace, :config, :schema_store
+
+    # @param schema [String]
+    # @param namespace [String]
+    # @param schema_store [AvroTurf::SchemaStore]
+    def initialize(schema:, namespace:, schema_store: nil)
+      @schema = schema
+      @namespace = namespace
+      @schema_store = schema_store ||
+                      AvroTurf::SchemaStore.new(path: Deimos.config.schema_path)
+    end
+
+    # @param schema [String]
+    # @return [Avro::Schema]
+    def avro_schema(schema=nil)
+      schema ||= @schema
+      @schema_store.find(schema, @namespace)
+    end
+
+  private
+
+    # @return [AvroTurf]
+    def avro_turf
+      @avro_turf ||= AvroTurf.new(
+        schemas_path: Deimos.config.schema_path,
+        schema_store: @schema_store
+      )
+      @avro_turf
+    end
+
+    # @return [AvroTurf::Messaging]
+    def avro_turf_messaging
+      @avro_turf_messaging ||= AvroTurf::Messaging.new(
+        schema_store: @schema_store,
+        registry_url: Deimos.config.schema_registry_url,
+        schemas_path: Deimos.config.schema_path,
+        namespace: @namespace
+      )
+    end
+
+    # Generate a key schema from the given value schema and key ID. This
+    # is used when encoding or decoding keys from an existing value schema.
+    # @param key_id [Symbol]
+    # @return [Hash]
+    def _generate_key_schema(key_id)
+      return @key_schema if @key_schema
+
+      value_schema = @schema_store.find(@schema, @namespace)
+      key_field = value_schema.fields.find { |f| f.name == key_id.to_s }
+      name = _key_schema_name(@schema)
+      @key_schema = {
+        'type' => 'record',
+        'name' => name,
+        'namespace' => @namespace,
+        'doc' => "Key for #{@namespace}.#{@schema}",
+        'fields' => [
+          {
+            'name' => key_id,
+            'type' => key_field.type.type_sym.to_s
+          }
+        ]
+      }
+      @schema_store.add_schema(@key_schema)
+      @key_schema
+    end
+
+    # @param value_schema [Hash]
+    # @return [String]
+    def _field_name_from_schema(value_schema)
+      raise "Schema #{@schema} not found!" if value_schema.nil?
+      if value_schema['fields'].nil? || value_schema['fields'].empty?
+        raise "Schema #{@schema} has no fields!"
+      end
+
+      value_schema['fields'][0]['name']
+    end
+
+    # @param schema [String]
+    # @return [String]
+    def _key_schema_name(schema)
+      "#{schema.gsub('-value', '')}_key"
+    end
+  end
+end

data/lib/deimos/avro_data_decoder.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require 'avro_turf/messaging'
+require 'deimos/avro_data_coder'
+
+module Deimos
+  # Service Object to decode avro messages
+  class AvroDataDecoder < AvroDataCoder
+    # Decode some data.
+    # @param payload [Hash|String]
+    # @param schema [String]
+    # @return [Hash]
+    def decode(payload, schema: nil)
+      schema ||= @schema
+      avro_turf_messaging.decode(payload, schema_name: schema)
+    end
+
+    # Decode against a local schema.
+    # @param payload [Hash]
+    # @param schema [String]
+    # @return [Hash]
+    def decode_local(payload, schema: nil)
+      schema ||= @schema
+      avro_turf.decode(payload, schema_name: schema, namespace: @namespace)
+    end
+
+    # @param payload [String] the encoded key.
+    # @param key_id [String|Symbol]
+    # @return [Object] the decoded key (int/long/string).
+    def decode_key(payload, key_id)
+      key_schema = _generate_key_schema(key_id)
+      field_name = _field_name_from_schema(key_schema)
+      decode(payload, schema: key_schema['name'])[field_name]
+    end
+  end
+end

data/lib/deimos/avro_data_encoder.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require 'avro_turf/messaging'
+require 'deimos/avro_data_coder'
+
+module Deimos
+  # Service Object to decode Avro messages.
+  class AvroDataEncoder < AvroDataCoder
+    # @param payload [Hash]
+    # @param schema [String]
+    # @return [String]
+    def encode_local(payload, schema: nil)
+      schema ||= @schema
+      Avro::SchemaValidator.validate!(avro_schema(schema), payload,
+                                      recursive: true,
+                                      fail_on_extra_fields: true)
+      avro_turf.encode(payload, schema_name: schema, namespace: @namespace)
+    rescue Avro::IO::AvroTypeError
+      # throw a more detailed error
+      value_schema = @schema_store.find(schema, @namespace)
+      Avro::SchemaValidator.validate!(value_schema, payload)
+    end
+
+    # @param payload [Hash]
+    # @param schema [String]
+    # @param topic [String]
+    # @return [String]
+    def encode(payload, schema: nil, topic: nil)
+      schema ||= @schema
+      Avro::SchemaValidator.validate!(avro_schema(schema), payload,
+                                      recursive: true,
+                                      fail_on_extra_fields: true)
+      avro_turf_messaging.encode(payload, schema_name: schema, subject: topic)
+    rescue Avro::IO::AvroTypeError
+      # throw a more detailed error
+      schema = @schema_store.find(@schema, @namespace)
+      Avro::SchemaValidator.validate!(schema, payload)
+    end
+
+    # @param key_id [Symbol|String]
+    # @param key [Object]
+    # @param topic [String]
+    # @return [String] the encoded key.
+    def encode_key(key_id, key, topic=nil)
+      key_schema = _generate_key_schema(key_id)
+      field_name = _field_name_from_schema(key_schema)
+      payload = { field_name => key }
+      encode(payload, schema: key_schema['name'], topic: topic)
+    end
+  end
+end

data/lib/deimos/backends/db.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require 'deimos/kafka_message'
+
+module Deimos
+  module Backends
+    # Backend which saves messages to the database instead of immediately
+    # sending them.
+    class Db < Deimos::PublishBackend
+      class << self
+        # :nodoc:
+        def execute(producer_class:, messages:)
+          records = messages.map do |m|
+            message = Deimos::KafkaMessage.new(
+              message: m.encoded_payload.to_s.b,
+              topic: m.topic,
+              partition_key: m.partition_key || m.key
+            )
+            message.key = m.encoded_key.to_s.b unless producer_class.config[:no_keys]
+            message
+          end
+          Deimos::KafkaMessage.import(records)
+        end
+      end
+    end
+  end
+end