deimos-ruby 1.4.0.pre.beta7 → 1.5.0.pre.beta2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2c3894cd6181a4213507473bee8a572421f0b3d645a3b9c9fea042b699634376
-  data.tar.gz: 980ec0edfda96e7b51822b2209c4f91c8c36286c78910636c6a904d168e9fa66
+  metadata.gz: c2f20063d68540f1d48e8a87b210f42b146156120b0cc183bd9ce9b3aacee848
+  data.tar.gz: 23a7dac7a19d8defae93a5e675b614541d771b4c46746f12177dbd184ff37d67
 SHA512:
-  metadata.gz: 0e5abba9957ae55c5c2894da01852f20e24682d7ff0a863c8a825e629fcd70937b56a6238d0d8a73df087c1f4be46ea2a9d4ed1568ee116ada6c0bdc4f74c75e
-  data.tar.gz: affd7b4a83f851f4b34e170787aa4b9e7a483cd58e9572df5b280d5b50021b3fac696ac6eea6cde36ef0263a10e82eded22eb59b2cbcb3c6dd881bd0c75f6e67
+  metadata.gz: d3e9e905a67b22e953c9b8066d13adcff0f7c219676791f30e05ba34e007913c7615996278cf1aa52655a3f272d007baefd08f9d2612cbcc7d4453d656872197
+  data.tar.gz: 7978987a6ba5e56c2d9f5fd46582b17ed4ae85cac8b94fd67a65bac13dd60a0d56491eb91f060b030324e29f94b00158fdd0b87125ba4acd2b0ccfe887a009cf

data/.rubocop.yml

@@ -291,6 +291,9 @@ RSpec/HookArgument:
 RSpec/ItBehavesLike:
   EnforcedStyle: it_should_behave_like
 
+RSpec/LeakyConstantDeclaration:
+  Enabled: false
+
 RSpec/MessageChain:
   Severity: refactor
 

data/CHANGELOG.md

@@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
+# [1.5.0-beta2] - 2020-01-17
+- Added schema backends, which should simplify Avro encoding and make it
+  more flexible for unit tests and local development.
+- Add `:test` producer backend which replaces the existing TestHelpers
+  functionality of writing messages to an in-memory hash.
+
 # [1.4.0-beta7] - 2019-12-16
 - Clone loggers when assigning to multiple levels.
 
@@ -23,6 +29,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 # [1.4.0-beta1] - 2019-11-22
 - Complete revamp of configuration method.
 
+# [1.3.0-beta5] - 2020-01-14
+- Added `db_producer.insert` and `db_producer.process` metrics.
+
+# [1.3.0-beta4] - 2019-12-02
+- Fixed bug where by running `rake deimos:start` without
+  specifying a producer backend would crash.
+
 # [1.3.0-beta3] - 2019-11-26
 - Fixed bug in TestHelpers where key_decoder was not stubbed out.
 

data/Gemfile.lock

@@ -1,51 +1,83 @@
 PATH
   remote: .
   specs:
-    deimos-ruby (1.4.0.pre.beta7)
-      avro-patches (~> 0.3)
-      avro_turf (~> 0.8)
+    deimos-ruby (1.5.0.pre.beta2)
+      avro_turf (~> 0.11)
       phobos (~> 1.8.2.pre.beta2)
       ruby-kafka (~> 0.7)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activemodel (5.2.0)
-      activesupport (= 5.2.0)
-    activerecord (5.2.0)
-      activemodel (= 5.2.0)
-      activesupport (= 5.2.0)
+    actioncable (5.2.4)
+      actionpack (= 5.2.4)
+      nio4r (~> 2.0)
+      websocket-driver (>= 0.6.1)
+    actionmailer (5.2.4)
+      actionpack (= 5.2.4)
+      actionview (= 5.2.4)
+      activejob (= 5.2.4)
+      mail (~> 2.5, >= 2.5.4)
+      rails-dom-testing (~> 2.0)
+    actionpack (5.2.4)
+      actionview (= 5.2.4)
+      activesupport (= 5.2.4)
+      rack (~> 2.0)
+      rack-test (>= 0.6.3)
+      rails-dom-testing (~> 2.0)
+      rails-html-sanitizer (~> 1.0, >= 1.0.2)
+    actionview (5.2.4)
+      activesupport (= 5.2.4)
+      builder (~> 3.1)
+      erubi (~> 1.4)
+      rails-dom-testing (~> 2.0)
+      rails-html-sanitizer (~> 1.0, >= 1.0.3)
+    activejob (5.2.4)
+      activesupport (= 5.2.4)
+      globalid (>= 0.3.6)
+    activemodel (5.2.4)
+      activesupport (= 5.2.4)
+    activerecord (5.2.4)
+      activemodel (= 5.2.4)
+      activesupport (= 5.2.4)
       arel (>= 9.0)
-    activerecord-import (0.28.1)
+    activerecord-import (1.0.3)
       activerecord (>= 3.2)
-    activesupport (5.2.0)
+    activestorage (5.2.4)
+      actionpack (= 5.2.4)
+      activerecord (= 5.2.4)
+      marcel (~> 0.3.1)
+    activesupport (5.2.4)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
     arel (9.0.0)
     ast (2.4.0)
-    avro (1.8.2)
+    avro (1.9.1)
       multi_json
-    avro-patches (0.4.1)
-      avro (= 1.8.2)
     avro_turf (0.11.0)
       avro (>= 1.7.7, < 1.10)
       excon (~> 0.45)
+    builder (3.2.4)
     coderay (1.1.2)
-    concurrent-ruby (1.1.3)
-    concurrent-ruby-ext (1.1.3)
-      concurrent-ruby (= 1.1.3)
-    ddtrace (0.22.0)
+    concurrent-ruby (1.1.5)
+    concurrent-ruby-ext (1.1.5)
+      concurrent-ruby (= 1.1.5)
+    crass (1.0.5)
+    ddtrace (0.30.0)
       msgpack
     diff-lcs (1.3)
     digest-crc (0.4.1)
-    dogstatsd-ruby (4.2.0)
-    excon (0.71.0)
+    dogstatsd-ruby (4.5.0)
+    erubi (1.9.0)
+    excon (0.71.1)
     exponential-backoff (0.0.4)
-    ffi (1.11.1)
+    ffi (1.11.3)
     formatador (0.2.5)
-    guard (2.14.2)
+    globalid (0.4.2)
+      activesupport (>= 4.2.0)
+    guard (2.16.1)
       formatador (>= 0.2.4)
       listen (>= 2.7, < 4.0)
       lumberjack (>= 1.0.12, < 2.0)
@@ -62,31 +94,43 @@ GEM
     guard-rubocop (1.3.0)
       guard (~> 2.0)
       rubocop (~> 0.20)
-    i18n (1.1.1)
+    i18n (1.7.0)
       concurrent-ruby (~> 1.0)
-    jaro_winkler (1.5.3)
-    listen (3.1.5)
-      rb-fsevent (~> 0.9, >= 0.9.4)
-      rb-inotify (~> 0.9, >= 0.9.7)
-      ruby_dep (~> 1.2)
+    jaro_winkler (1.5.4)
+    listen (3.2.1)
+      rb-fsevent (~> 0.10, >= 0.10.3)
+      rb-inotify (~> 0.9, >= 0.9.10)
     little-plugger (1.1.4)
     logging (2.2.2)
       little-plugger (~> 1.1)
       multi_json (~> 1.10)
-    lumberjack (1.0.12)
-    method_source (0.9.0)
-    minitest (5.11.3)
-    msgpack (1.2.10)
+    loofah (2.4.0)
+      crass (~> 1.0.2)
+      nokogiri (>= 1.5.9)
+    lumberjack (1.0.13)
+    mail (2.7.1)
+      mini_mime (>= 0.1.1)
+    marcel (0.3.3)
+      mimemagic (~> 0.3.2)
+    method_source (0.9.2)
+    mimemagic (0.3.3)
+    mini_mime (1.0.2)
+    mini_portile2 (2.4.0)
+    minitest (5.13.0)
+    msgpack (1.3.1)
     multi_json (1.14.1)
-    mysql2 (0.5.2)
+    mysql2 (0.5.3)
     nenv (0.3.0)
-    notiffany (0.1.1)
+    nio4r (2.5.2)
+    nokogiri (1.10.7)
+      mini_portile2 (~> 2.4.0)
+    notiffany (0.1.3)
       nenv (~> 0.1)
       shellany (~> 0.0)
-    parallel (1.17.0)
-    parser (2.6.3.0)
+    parallel (1.19.1)
+    parser (2.6.5.0)
       ast (~> 2.4.0)
-    pg (1.1.3)
+    pg (1.1.4)
     phobos (1.8.2)
       activesupport (>= 3.0.0)
       concurrent-ruby (>= 1.0.2)
@@ -95,49 +139,85 @@ GEM
       logging
       ruby-kafka
       thor
-    pry (0.11.3)
+    pry (0.12.2)
       coderay (~> 1.1.0)
       method_source (~> 0.9.0)
+    rack (2.0.8)
+    rack-test (1.1.0)
+      rack (>= 1.0, < 3)
+    rails (5.2.4)
+      actioncable (= 5.2.4)
+      actionmailer (= 5.2.4)
+      actionpack (= 5.2.4)
+      actionview (= 5.2.4)
+      activejob (= 5.2.4)
+      activemodel (= 5.2.4)
+      activerecord (= 5.2.4)
+      activestorage (= 5.2.4)
+      activesupport (= 5.2.4)
+      bundler (>= 1.3.0)
+      railties (= 5.2.4)
+      sprockets-rails (>= 2.0.0)
+    rails-dom-testing (2.0.3)
+      activesupport (>= 4.2.0)
+      nokogiri (>= 1.6)
+    rails-html-sanitizer (1.3.0)
+      loofah (~> 2.3)
+    railties (5.2.4)
+      actionpack (= 5.2.4)
+      activesupport (= 5.2.4)
+      method_source
+      rake (>= 0.8.7)
+      thor (>= 0.19.0, < 2.0)
     rainbow (3.0.0)
     rake (10.5.0)
-    rb-fsevent (0.10.2)
-    rb-inotify (0.9.10)
-      ffi (>= 0.5.0, < 2)
-    rspec (3.7.0)
-      rspec-core (~> 3.7.0)
-      rspec-expectations (~> 3.7.0)
-      rspec-mocks (~> 3.7.0)
-    rspec-core (3.7.1)
-      rspec-support (~> 3.7.0)
-    rspec-expectations (3.7.0)
+    rb-fsevent (0.10.3)
+    rb-inotify (0.10.0)
+      ffi (~> 1.0)
+    rspec (3.9.0)
+      rspec-core (~> 3.9.0)
+      rspec-expectations (~> 3.9.0)
+      rspec-mocks (~> 3.9.0)
+    rspec-core (3.9.0)
+      rspec-support (~> 3.9.0)
+    rspec-expectations (3.9.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.7.0)
-    rspec-mocks (3.7.0)
+      rspec-support (~> 3.9.0)
+    rspec-mocks (3.9.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.7.0)
-    rspec-support (3.7.1)
-    rspec_junit_formatter (0.3.0)
+      rspec-support (~> 3.9.0)
+    rspec-support (3.9.0)
+    rspec_junit_formatter (0.4.1)
       rspec-core (>= 2, < 4, != 2.12.0)
-    rubocop (0.72.0)
+    rubocop (0.77.0)
       jaro_winkler (~> 1.5.1)
       parallel (~> 1.10)
       parser (>= 2.6)
       rainbow (>= 2.2.2, < 4.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 1.7)
-    rubocop-rspec (1.32.0)
-      rubocop (>= 0.60.0)
+    rubocop-rspec (1.37.1)
+      rubocop (>= 0.68.1)
     ruby-kafka (0.7.10)
       digest-crc
     ruby-progressbar (1.10.1)
-    ruby_dep (1.5.0)
     shellany (0.0.1)
-    sqlite3 (1.3.13)
-    thor (0.20.3)
+    sprockets (4.0.0)
+      concurrent-ruby (~> 1.0)
+      rack (> 1, < 3)
+    sprockets-rails (3.2.1)
+      actionpack (>= 4.0)
+      activesupport (>= 4.0)
+      sprockets (>= 3.0.0)
+    sqlite3 (1.4.1)
+    thor (1.0.1)
     thread_safe (0.3.6)
     tzinfo (1.2.5)
       thread_safe (~> 0.1)
     unicode-display_width (1.6.0)
+    websocket-driver (0.7.1)
+      websocket-extensions (>= 0.1.0)
+    websocket-extensions (0.1.4)
 
 PLATFORMS
   ruby
@@ -145,6 +225,7 @@ PLATFORMS
 DEPENDENCIES
   activerecord (~> 5.2)
   activerecord-import
+  avro (~> 1.9)
   bundler (~> 1)
   ddtrace (~> 0.11)
   deimos-ruby!
@@ -154,6 +235,7 @@ DEPENDENCIES
   guard-rubocop (~> 1)
   mysql2 (~> 0.5)
   pg (~> 1.1)
+  rails (~> 5.2)
   rake (~> 10)
   rspec (~> 3)
   rspec_junit_formatter (~> 0.3)

data/README.md

@@ -6,7 +6,7 @@
   <img src="https://img.shields.io/codeclimate/maintainability/flipp-oss/deimos.svg"/>
 </p>
 
-A Ruby framework for marrying Kafka, Avro, and/or ActiveRecord and provide
+A Ruby framework for marrying Kafka, a schema definition like Avro, and/or ActiveRecord and provide
 a useful toolbox of goodies for Ruby-based Kafka development.
 Built on Phobos and hence Ruby-Kafka.
 
@@ -14,6 +14,7 @@ Built on Phobos and hence Ruby-Kafka.
    * [Installation](#installation)
    * [Versioning](#versioning)
    * [Configuration](#configuration)
+   * [Schemas](#schemas)
    * [Producers](#producers)
         * [Auto-added Fields](#auto-added-fields)
         * [Coerced Values](#coerced-values)
@@ -60,6 +61,27 @@ gem 'deimos-ruby', '~> 1.1'
 
 For a full configuration reference, please see [the configuration docs ](docs/CONFIGURATION.md).
 
+# Schemas
+
+Deimos was originally written only supporting Avro encoding via a schema registry.
+This has since been expanded to a plugin architecture allowing messages to be
+encoded and decoded via any schema specification you wish. 
+
+Currently we have the following possible schema backends:
+* Avro Local (use pure Avro)
+* Avro Schema Registry (use the Confluent Schema Registry)
+* Avro Validation (validate using an Avro schema but leave decoded - this is useful
+  for unit testing and development)
+* Mock (no actual encoding/decoding).
+
+Note that to use Avro-encoding, you must include the [avro_turf](https://github.com/dasch/avro_turf) gem in your
+Gemfile.
+
+Other possible schemas could include [Protobuf](https://developers.google.com/protocol-buffers), [JSONSchema](https://json-schema.org/), etc. Feel free to
+contribute!
+
+To create a new schema backend, please see the existing examples [here](lib/deimos/schema_backends).
+
 # Producers
 
 Producers will look like this:
@@ -137,7 +159,7 @@ produced by Phobos and RubyKafka):
   * topic
   * exception_object
   * payloads - the unencoded payloads
-* `encode_messages` - sent when messages are being Avro-encoded.
+* `encode_messages` - sent when messages are being schema-encoded.
   * producer - the class that produced the message
   * topic
   * payloads - the unencoded payloads
@@ -165,8 +187,8 @@ Similarly:
 ### Kafka Message Keys
 
 Topics representing events rather than domain data don't need keys. However,
-best practice for domain messages is to Avro-encode message keys 
-with a separate Avro schema. 
+best practice for domain messages is to schema-encode message keys 
+with a separate schema.
 
 This enforced by requiring producers to define a `key_config` directive. If
 any message comes in with a key, the producer will error out if `key_config` is
@@ -179,7 +201,7 @@ There are three possible configurations to use:
   all your messages in a topic need to have a key, or they all need to have
   no key. This is a good choice for events that aren't keyed - you can still
   set a partition key.
-* `key_config plain: true` - this indicates that you are not using an Avro-encoded
+* `key_config plain: true` - this indicates that you are not using an encoded
   key. Use this for legacy topics - new topics should not use this setting.
 * `key_config schema: 'MyKeySchema-key'` - this tells the producer to look for
   an existing key schema named `MyKeySchema-key` in the schema registry and to
@@ -234,8 +256,8 @@ like this:
 ```
 
 If you publish a payload `{ "test_id" => "123", "some_int" => 123 }`, this
-will be turned into a key that looks like `{ "test_id" => "123"}` and encoded
-via Avro before being sent to Kafka. 
+will be turned into a key that looks like `{ "test_id" => "123"}` and schema-encoded
+before being sent to Kafka. 
 
 If you are using `plain` or `schema` as your config, you will need to have a
 special `payload_key` key to your payload hash. This will be extracted and
@@ -261,7 +283,7 @@ class MyConsumer < Deimos::Consumer
 
   def consume(payload, metadata)
     # Same method as Phobos consumers.
-    # payload is an Avro-decoded hash.
+    # payload is an schema-decoded hash.
     # metadata is a hash that contains information like :key and :topic.
     # In general, your key should be included in the payload itself. However,
     # if you need to access it separately from the payload, you can use
@@ -311,7 +333,7 @@ this sample:
 class MyBatchConsumer < Deimos::BatchConsumer
 
   def consume_batch(payloads, metadata)
-    # payloads is an array of Avro-decoded hashes.
+    # payloads is an array of schema-decoded hashes.
     # metadata is a hash that contains information like :keys and :topic.
     # Keys are automatically decoded and available as an array with
     # the same cardinality as the payloads. If you need to iterate
@@ -581,6 +603,11 @@ The following metrics are reported:
   with the database backend. Tagged with the topic that is waiting.
   Will send a value of 0 with no topics tagged if there are no messages
   waiting.
+* `db_producer.insert` - the number of messages inserted into the database
+  for publishing. Tagged with `topic:{topic_name}`
+* `db_producer.process` - the number of DB messages processed. Note that this
+  is *not* the same as the number of messages *published* if those messages
+  are compacted. Tagged with `topic:{topic_name}`
 
 ### Configuring Metrics Providers
 
@@ -604,7 +631,7 @@ Also see [deimos.rb](lib/deimos.rb) under `Configure metrics` to see how the met
 Deimos also includes some tracing for kafka consumers. It ships with 
 DataDog support, but you can add custom tracing providers as well.
 
-Trace spans are used for when incoming messages are avro decoded, and a 
+Trace spans are used for when incoming messages are schema-decoded, and a 
 separate span for message consume logic.
 
 ### Configuring Tracing Providers
@@ -749,7 +776,7 @@ be
 }
 ```
 
-Both payload and key will be Avro-decoded as necessary according to the
+Both payload and key will be schema-decoded as necessary according to the
 key config.
 
 You can also just pass an existing producer or consumer class into the method,

data/Rakefile

@@ -6,8 +6,8 @@ begin
 
   RSpec::Core::RakeTask.new(:spec)
   task(default: :spec)
-rescue LoadError # rubocop:disable Lint/HandleExceptions
+rescue LoadError # rubocop:disable Lint/SuppressedException
   # no rspec available
 end
 
-import('./lib/tasks/phobos.rake')
+import('./lib/tasks/deimos.rake')

data/deimos-ruby.gemspec

@@ -18,13 +18,13 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ['lib']
 
-  spec.add_runtime_dependency('avro-patches', '~> 0.3')
-  spec.add_runtime_dependency('avro_turf', '~> 0.8')
+  spec.add_runtime_dependency('avro_turf', '~> 0.11')
   spec.add_runtime_dependency('phobos', '~> 1.8.2.pre.beta2')
   spec.add_runtime_dependency('ruby-kafka', '~> 0.7')
 
   spec.add_development_dependency('activerecord', '~> 5.2')
   spec.add_development_dependency('activerecord-import')
+  spec.add_development_dependency('avro', '~> 1.9')
   spec.add_development_dependency('bundler', '~> 1')
   spec.add_development_dependency('ddtrace', '~> 0.11')
   spec.add_development_dependency('dogstatsd-ruby', '~> 4.2')
@@ -33,6 +33,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency('guard-rubocop', '~> 1')
   spec.add_development_dependency('mysql2', '~> 0.5')
   spec.add_development_dependency('pg', '~> 1.1')
+  spec.add_development_dependency('rails', '~> 5.2')
   spec.add_development_dependency('rake', '~> 10')
   spec.add_development_dependency('rspec', '~> 3')
   spec.add_development_dependency('rspec_junit_formatter', '~>0.3')

data/docs/CONFIGURATION.md

@@ -144,6 +144,7 @@ producers.backend|`:kafka_async`|Currently can be set to `:db`, `:kafka`, or `:k
 
 Config name|Default|Description
 -----------|-------|-----------
+schema.backend|`:mock`|Backend representing the schema encoder/decoder. You can see a full list [here](../lib/deimos/schema_backends).
 schema.registry_url|`http://localhost:8081`|URL of the Confluent schema registry.
 schema.path|nil|Local path to find your schemas.
 

data/docs/DATABASE_BACKEND.md

@@ -77,7 +77,7 @@ The database backend consists of three tables:
 
 * `kafka_messages` - this keeps track of the messages that were "published",
   including the payload, topic, key and partition key. These messages
-  are *raw data* - all processing, including Avro encoding, must happen
+  are *raw data* - all processing, including schema-encoding, must happen
   upstream before they are inserted.
 * `kafka_topic_info` - this table is essentially a lock table used to ensure
   that only one producer thread is ever "working" on a topic at a time.

data/lib/deimos/active_record_consumer.rb

@@ -68,32 +68,31 @@ module Deimos
     def record_attributes(payload)
       klass = self.class.config[:record_class]
       attributes = {}
-      schema = self.class.decoder.avro_schema
-      schema.fields.each do |field|
+      self.class.decoder.schema_fields.each do |field|
         column = klass.columns.find { |c| c.name == field.name }
         next if column.nil?
         next if %w(updated_at created_at).include?(field.name)
 
-        attributes[field.name] = _coerce_field(field, column, payload[field.name])
+        attributes[field.name] = _coerce_field(column, payload[field.name])
       end
       attributes
     end
 
   private
 
-    # @param field [Avro::Schema]
     # @param column [ActiveRecord::ConnectionAdapters::Column]
     # @param val [Object]
-    def _coerce_field(field, column, val)
+    def _coerce_field(column, val)
       return nil if val.nil?
 
-      field_type = field.type.type.to_sym
-      if field_type == :union
-        union_types = field.type.schemas.map { |s| s.type.to_sym }
-        field_type = union_types.find { |t| t != :null }
-      end
-      if column.type == :datetime && %i(int long).include?(field_type)
-        return Time.zone.strptime(val.to_s, '%s')
+      if column.type == :datetime
+        int_val = begin
+                    val.is_a?(Integer) ? val : (val.is_a?(String) && Integer(val))
+                  rescue StandardError
+                    nil
+                  end
+
+        return Time.zone.at(int_val) if int_val
       end
 
       val

data/lib/deimos/active_record_producer.rb

@@ -53,10 +53,10 @@ module Deimos
       # is not set.
       # @return [Hash]
       def generate_payload(attributes, _record)
-        schema = self.encoder.avro_schema
+        fields = self.encoder.schema_fields
         payload = attributes.stringify_keys
         payload.delete_if do |k, _|
-          k.to_sym != :payload_key && !schema.fields.find { |f| f.name == k }
+          k.to_sym != :payload_key && !fields.map(&:name).include?(k)
         end
       end
     end

data/lib/deimos/backends/base.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Deimos
+  module Backends
+    # Abstract class for all publish backends.
+    class Base
+      class << self
+        # @param producer_class [Class < Deimos::Producer]
+        # @param messages [Array<Deimos::Message>]
+        def publish(producer_class:, messages:)
+          Deimos.config.logger.info(
+            message: 'Publishing messages',
+            topic: producer_class.topic,
+            payloads: messages.map do |message|
+              {
+                payload: message.payload,
+                key: message.key
+              }
+            end
+          )
+          execute(producer_class: producer_class, messages: messages)
+        end
+
+        # @param producer_class [Class < Deimos::Producer]
+        # @param messages [Array<Deimos::Message>]
+        def execute(producer_class:, messages:)
+          raise NotImplementedError
+        end
+      end
+    end
+  end
+end

data/lib/deimos/backends/db.rb

@@ -6,7 +6,7 @@ module Deimos
   module Backends
     # Backend which saves messages to the database instead of immediately
     # sending them.
-    class Db < Deimos::PublishBackend
+    class Db < Base
       class << self
         # :nodoc:
         def execute(producer_class:, messages:)
@@ -20,6 +20,11 @@ module Deimos
             message
           end
           Deimos::KafkaMessage.import(records)
+          Deimos.config.metrics&.increment(
+            'db_producer.insert',
+            tags: %W(topic:#{producer_class.topic}),
+            by: records.size
+          )
         end
       end
     end

data/lib/deimos/backends/kafka.rb

@@ -3,7 +3,7 @@
 module Deimos
   module Backends
     # Default backend to produce to Kafka.
-    class Kafka < Deimos::PublishBackend
+    class Kafka < Base
       include Phobos::Producer
 
       # Shut down the producer if necessary.

data/lib/deimos/backends/kafka_async.rb

@@ -3,7 +3,7 @@
 module Deimos
   module Backends
     # Backend which produces to Kafka via an async producer.
-    class KafkaAsync < Deimos::PublishBackend
+    class KafkaAsync < Base
       include Phobos::Producer
 
       # Shut down the producer cleanly.

data/lib/deimos/backends/test.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Deimos
+  module Backends
+    # Backend which saves messages to an in-memory hash.
+    class Test < Deimos::Backends::Base
+      class << self
+        # @return [Array<Hash>]
+        def sent_messages
+          @sent_messages ||= []
+        end
+      end
+
+      # @override
+      def self.execute(producer_class:, messages:)
+        self.sent_messages.concat(messages.map(&:to_h))
+      end
+    end
+  end
+end