deimos-ruby 1.7.0.pre.beta1 → 1.8.1.pre.beta3

data/deimos-ruby.gemspec

@@ -21,10 +21,11 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency('avro_turf', '~> 0.11')
   spec.add_runtime_dependency('phobos', '~> 1.9')
   spec.add_runtime_dependency('ruby-kafka', '~> 0.7')
+  spec.add_runtime_dependency('sigurd', '0.0.1')
 
-  spec.add_development_dependency('activerecord', '~> 5.2')
   spec.add_development_dependency('activerecord-import')
   spec.add_development_dependency('avro', '~> 1.9')
+  spec.add_development_dependency('database_cleaner', '~> 1.7')
   spec.add_development_dependency('ddtrace', '~> 0.11')
   spec.add_development_dependency('dogstatsd-ruby', '~> 4.2')
   spec.add_development_dependency('guard', '~> 2')
@@ -32,10 +33,11 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency('guard-rubocop', '~> 1')
   spec.add_development_dependency('mysql2', '~> 0.5')
   spec.add_development_dependency('pg', '~> 1.1')
-  spec.add_development_dependency('rails', '~> 5.2', '>= 5.2.4.2')
+  spec.add_development_dependency('rails', '~> 6')
   spec.add_development_dependency('rake', '~> 13')
   spec.add_development_dependency('rspec', '~> 3')
   spec.add_development_dependency('rspec_junit_formatter', '~>0.3')
+  spec.add_development_dependency('rspec-rails', '~> 4')
   spec.add_development_dependency('rubocop', '~> 0.72')
   spec.add_development_dependency('rubocop-rspec', '~> 1.27')
   spec.add_development_dependency('sqlite3', '~> 1.3')

data/docs/ARCHITECTURE.md

@@ -0,0 +1,144 @@
+# Deimos Architecture
+
+Deimos is the third of three libraries that add functionality on top of each 
+other:
+
+* [RubyKafka](https://github.com/zendesk/ruby-kafka) is the low-level Kafka
+  client, providing API's for producers, consumers and the client as a whole.
+* [Phobos](https://github.com/phobos/phobos) is a lightweight wrapper on top
+  of RubyKafka that provides threaded consumers, a simpler way to write
+  producers, and lifecycle management.
+* [Deimos](https://github.com/flipp-oss/deimos/) is a full-featured framework
+  using Phobos as its base which provides schema integration (e.g. Avro), 
+  database integration, metrics, tracing, test helpers and other utilities.
+  
+## Folder structure
+
+As of May 12, 2020, the following are the important files to understand in how
+Deimos fits together:
+* `lib/generators`: Generators to generate database migrations, e.g.
+   for the DB Poller and DB Producer features.
+* `lib/tasks`: Rake tasks for starting consumers, DB Pollers, etc.
+* `lib/deimos`: Main Deimos code.
+* `lib/deimos/deimos.rb`: The bootstrap / startup code for Deimos. Also provides
+   some global convenience methods and (for legacy purposes) the way to 
+   start the DB Producer.
+* `lib/deimos/backends`: The different plug-in producer backends - e.g. produce
+   directly to Kafka, use the DB backend, etc.
+* `lib/deimos/schema_backends`: The different plug-in schema handlers, such
+  as the various flavors of Avro (with/without schema registry etc.)
+* `lib/deimos/metrics`: The different plug-in metrics providers, e.g. Datadog.
+* `lib/deimos/tracing`: The different plug-in tracing providers, e.g. Datadog.
+* `lib/deimos/utils`: Utility classes for things not directly related to
+   producing and consuming, such as the DB Poller, DB Producer, lag reporter, etc.
+* `lib/deimos/config`: Classes related to configuring Deimos.
+* `lib/deimos/monkey_patches`: Monkey patches to existing libraries. These
+   should be removed in a future update.
+
+## Features
+
+### Producers and Consumers
+
+Both producers and consumers include the `SharedConfig` module, which 
+standardizes configuration like schema settings, topic, keys, etc.
+
+Consumers come in two flavors: `Consumer` and `BatchConsumer`. Both include
+`BaseConsumer` for shared functionality.
+
+While producing messages go to Kafka by default, literally anything else
+can happen when your producer calls `produce`, by swapping out the producer
+_backend_. This is just a file that needs to inherit from `Deimos::Backends::Base`
+and must implement a single method, `execute`.
+
+Producers have a complex workflow while processing the payload to publish. This
+is aided by the `Deimos::Message` class (not to be confused with the 
+`KafkaMessage` class, which is an ActiveRecord used by the DB Producer feature,
+below).
+
+### Schemas
+
+Schema backends are used to encode and decode payloads into different formats
+such as Avro. These are integrated with producers and consumers, as well
+as test helpers. These are a bit more involved than producer backends, and
+must define methods such as:
+* `encode` a payload or key (when encoding a key, for Avro a key schema
+  may be auto-generated)
+* `decode` a payload or key
+* `validate` that a payload is correct for encoding
+* `coerce` a payload into the given schema (e.g. turn ints into strings)
+* Get a list of `schema_fields` in the configured schema, used when interacting
+  with ActiveRecord
+* Define a `mock` backend when the given backend is used. This is used
+  during testing. Typically mock backends will validate values but not
+  actually encode/decode them.
+  
+### Configuration
+
+Deimos has its own `Configurable` module that makes heavy use of `method_missing`
+to provide a very succinct but powerful configuration format (including
+default values, procs, print out as hash, reset, etc.). It also
+allows for multiple blocks to define different objects of the same time
+(like producers, consumers, pollers etc.).
+
+The configuration definition for Deimos is in `config/configuration.rb`. In
+addition, there are methods in `config/phobos_config.rb` which translate to/from
+the Phobos configuration format and support the old `phobos.yml` method
+of configuration.
+
+### Metrics and Tracing
+
+These are simpler than other plugins and must implement the expected methods
+(`increment`, `gauge`, `histogram` and `time` for metrics, and `start`, `finish`
+and `set_error` for tracing). These are used primarily in producers and consumers.
+
+### ActiveRecord Integration
+
+Deimos provides an `ActiveRecordConsumer` and `ActiveRecordProducer`. These are
+relatively lightweight ways to save data into a database or read it off 
+the database as part of app logic. It uses things like the `coerce` method
+of the schema backends to manage the differences between the given payload
+and the configured schema for the topic.
+
+### Database Backend / Database Producer
+
+This feature (which provides better performance and transaction guarantees)
+is powered by two components:
+* The `db` _publish backend_, which saves messages to the database rather
+  than to Kafka;
+* The `DbProducer` utility, which runs as a separate process, pulls data
+  from the database and sends it to Kafka.
+
+There are a set of utility classes that power the producer, which are largely
+copied from Phobos:
+* `Executor` takes a set of "runnable" things (which implement a `start` and `stop`
+  method) puts them in a thread pool and runs them all concurrently. It
+  manages starting and stopping all threads when necessary.
+* `SignalHandler` wraps the Executor and handles SIGINT and SIGTERM signals
+  to stop the executor gracefully.
+  
+In the case of this feature, the `DbProducer` is the runnable object - it
+can run several threads at once.
+
+On the database side, the `ActiveRecord` models that power this feature are:
+* `KafkaMessage`: The actual message, saved to the database. This message
+  is already encoded by the producer, so only has to be sent.
+* `KafkaTopicInfo`: Used for locking topics so only one producer can work
+  on it at once.
+  
+A Rake task (defined in `deimos.rake`) can be used to start the producer.
+  
+### Database Poller
+
+This feature (which periodically polls the database to send Kafka messages)
+primarily uses other aspects of Deimos and hence is relatively small in size.
+The `DbPoller` class acts as a "runnable" and is used by an Executor (above).
+The `PollInfo` class is saved to the database to keep track of where each
+poller is up to.
+
+A Rake task (defined in `deimos.rake`) can be used to start the pollers.
+
+### Other Utilities
+
+The `utils` folder also contains the `LagReporter` (which sends metrics on
+lag) and the `InlineConsumer`, which can read data from a topic and directly
+pass it into a handler or save it to memory.

data/docs/CONFIGURATION.md

@@ -58,6 +58,10 @@ Deimos.configure do
     namespace 'my.namespace'
     key_config field: :id
 
+    # Setting to :inline_batch will invoke consume_batch instead of consume
+    # for each batch of messages.
+    delivery :batch
+
   # If config.schema.path is app/schemas, assumes there is a file in
   # app/schemas/my/namespace/MyTopicSchema.avsc
   end

data/lib/deimos.rb

@@ -23,14 +23,15 @@ require 'deimos/monkey_patches/phobos_producer'
 require 'deimos/monkey_patches/phobos_cli'
 
 require 'deimos/railtie' if defined?(Rails)
+require 'deimos/utils/schema_controller_mixin' if defined?(ActionController)
 
 if defined?(ActiveRecord)
   require 'deimos/kafka_source'
   require 'deimos/kafka_topic_info'
   require 'deimos/backends/db'
-  require 'deimos/utils/signal_handler.rb'
-  require 'deimos/utils/executor.rb'
-  require 'deimos/utils/db_producer.rb'
+  require 'sigurd/signal_handler'
+  require 'sigurd/executor'
+  require 'deimos/utils/db_producer'
   require 'deimos/utils/db_poller'
 end
 
@@ -72,10 +73,10 @@ module Deimos
         Deimos::Utils::DbProducer.
           new(self.config.db_producer.logger || self.config.logger)
       end
-      executor = Deimos::Utils::Executor.new(producers,
-                                             sleep_seconds: 5,
-                                             logger: self.config.logger)
-      signal_handler = Deimos::Utils::SignalHandler.new(executor)
+      executor = Sigurd::Executor.new(producers,
+                                      sleep_seconds: 5,
+                                      logger: self.config.logger)
+      signal_handler = Sigurd::SignalHandler.new(executor)
       signal_handler.run!
     end
   end

data/lib/deimos/active_record_consume/batch_consumption.rb

@@ -0,0 +1,159 @@
+# frozen_string_literal: true
+
+require 'deimos/active_record_consume/batch_slicer'
+require 'deimos/utils/deadlock_retry'
+require 'deimos/message'
+
+module Deimos
+  module ActiveRecordConsume
+    # Methods for consuming batches of messages and saving them to the database
+    # in bulk ActiveRecord operations.
+    module BatchConsumption
+      # Handle a batch of Kafka messages. Batches are split into "slices",
+      # which are groups of independent messages that can be processed together
+      # in a single database operation.
+      # If two messages in a batch have the same key, we cannot process them
+      # in the same operation as they would interfere with each other. Thus
+      # they are split
+      # @param payloads [Array<Hash>] Decoded payloads.
+      # @param metadata [Hash] Information about batch, including keys.
+      def consume_batch(payloads, metadata)
+        messages = payloads.
+          zip(metadata[:keys]).
+          map { |p, k| Deimos::Message.new(p, nil, key: k) }
+
+        tags = %W(topic:#{metadata[:topic]})
+
+        Deimos.instrument('ar_consumer.consume_batch', tags) do
+          # The entire batch should be treated as one transaction so that if
+          # any message fails, the whole thing is rolled back or retried
+          # if there is deadlock
+          Deimos::Utils::DeadlockRetry.wrap(tags) do
+            if @compacted || self.class.config[:no_keys]
+              update_database(compact_messages(messages))
+            else
+              uncompacted_update(messages)
+            end
+          end
+        end
+      end
+
+      # Get unique key for the ActiveRecord instance from the incoming key.
+      # Override this method (with super) to customize the set of attributes that
+      # uniquely identifies each record in the database.
+      # @param key [String] The encoded key.
+      # @return [Hash] The key attributes.
+      def record_key(key)
+        decoded_key = decode_key(key)
+
+        if decoded_key.nil?
+          {}
+        elsif decoded_key.is_a?(Hash)
+          @key_converter.convert(decoded_key)
+        else
+          { @klass.primary_key => decoded_key }
+        end
+      end
+
+    protected
+
+      # Perform database operations for a batch of messages without compaction.
+      # All messages are split into slices containing only unique keys, and
+      # each slice is handles as its own batch.
+      # @param messages [Array<Message>] List of messages.
+      def uncompacted_update(messages)
+        BatchSlicer.
+          slice(messages).
+          each(&method(:update_database))
+      end
+
+      # Perform database operations for a group of messages.
+      # All messages with payloads are passed to upsert_records.
+      # All tombstones messages are passed to remove_records.
+      # @param messages [Array<Message>] List of messages.
+      def update_database(messages)
+        # Find all upserted records (i.e. that have a payload) and all
+        # deleted record (no payload)
+        removed, upserted = messages.partition(&:tombstone?)
+
+        upsert_records(upserted) if upserted.any?
+        remove_records(removed) if removed.any?
+      end
+
+      # Upsert any non-deleted records
+      # @param messages [Array<Message>] List of messages for a group of
+      # records to either be updated or inserted.
+      def upsert_records(messages)
+        key_cols = key_columns(messages)
+
+        # Create payloads with payload + key attributes
+        upserts = messages.map do |m|
+          record_attributes(m.payload, m.key)&.
+            merge(record_key(m.key))
+        end
+
+        # If overridden record_attributes indicated no record, skip
+        upserts.compact!
+
+        options = if key_cols.empty?
+                    {} # Can't upsert with no key, just do regular insert
+                  else
+                    {
+                      on_duplicate_key_update: {
+                        # conflict_target must explicitly list the columns for
+                        # Postgres and SQLite. Not required for MySQL, but this
+                        # ensures consistent behaviour.
+                        conflict_target: key_cols,
+                        columns: :all
+                      }
+                    }
+                  end
+
+        @klass.import!(upserts, options)
+      end
+
+      # Delete any records with a tombstone.
+      # @param messages [Array<Message>] List of messages for a group of
+      # deleted records.
+      def remove_records(messages)
+        clause = deleted_query(messages)
+
+        clause.delete_all
+      end
+
+      # Create an ActiveRecord relation that matches all of the passed
+      # records. Used for bulk deletion.
+      # @param records [Array<Message>] List of messages.
+      # @return ActiveRecord::Relation Matching relation.
+      def deleted_query(records)
+        keys = records.
+          map { |m| record_key(m.key)[@klass.primary_key] }.
+          reject(&:nil?)
+
+        @klass.unscoped.where(@klass.primary_key => keys)
+      end
+
+      # Get the set of attribute names that uniquely identify messages in the
+      # batch. Requires at least one record.
+      # @param records [Array<Message>] Non-empty list of messages.
+      # @return [Array<String>] List of attribute names.
+      # @raise If records is empty.
+      def key_columns(records)
+        raise 'Cannot determine key from empty batch' if records.empty?
+
+        first_key = records.first.key
+        record_key(first_key).keys
+      end
+
+      # Compact a batch of messages, taking only the last message for each
+      # unique key.
+      # @param batch [Array<Message>] Batch of messages.
+      # @return [Array<Message>] Compacted batch.
+      def compact_messages(batch)
+        return batch unless batch.first&.key.present?
+
+        batch.reverse.uniq(&:key).reverse!
+      end
+    end
+  end
+end

data/lib/deimos/active_record_consume/batch_slicer.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Deimos
+  module ActiveRecordConsume
+    # Helper class for breaking down batches into independent groups for
+    # processing
+    class BatchSlicer
+      # Split the batch into a series of independent slices. Each slice contains
+      # messages that can be processed in any order (i.e. they have distinct
+      # keys). Messages with the same key will be separated into different
+      # slices that maintain the correct order.
+      # E.g. Given messages A1, A2, B1, C1, C2, C3, they will be sliced as:
+      # [[A1, B1, C1], [A2, C2], [C3]]
+      def self.slice(messages)
+        ops = messages.group_by(&:key)
+
+        # Find maximum depth
+        depth = ops.values.map(&:length).max || 0
+
+        # Generate slices for each depth
+        depth.times.map do |i|
+          ops.values.map { |arr| arr.dig(i) }.compact
+        end
+      end
+    end
+  end
+end

data/lib/deimos/active_record_consume/message_consumption.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Deimos
+  module ActiveRecordConsume
+    # Methods for consuming individual messages and saving them to the database
+    # as ActiveRecord instances.
+    module MessageConsumption
+      # Find the record specified by the given payload and key.
+      # Default is to use the primary key column and the value of the first
+      # field in the key.
+      # @param klass [Class < ActiveRecord::Base]
+      # @param _payload [Hash]
+      # @param key [Object]
+      # @return [ActiveRecord::Base]
+      def fetch_record(klass, _payload, key)
+        klass.unscoped.where(klass.primary_key => key).first
+      end
+
+      # Assign a key to a new record.
+      # @param record [ActiveRecord::Base]
+      # @param _payload [Hash]
+      # @param key [Object]
+      def assign_key(record, _payload, key)
+        record[record.class.primary_key] = key
+      end
+
+      # :nodoc:
+      def consume(payload, metadata)
+        key = metadata.with_indifferent_access[:key]
+        klass = self.class.config[:record_class]
+        record = fetch_record(klass, (payload || {}).with_indifferent_access, key)
+        if payload.nil?
+          destroy_record(record)
+          return
+        end
+        if record.blank?
+          record = klass.new
+          assign_key(record, payload, key)
+        end
+        attrs = record_attributes(payload.with_indifferent_access, key)
+        # don't use attributes= - bypass Rails < 5 attr_protected
+        attrs.each do |k, v|
+          record.send("#{k}=", v)
+        end
+        record.created_at ||= Time.zone.now if record.respond_to?(:created_at)
+        record.updated_at = Time.zone.now if record.respond_to?(:updated_at)
+        record.save!
+      end
+
+      # Destroy a record that received a null payload. Override if you need
+      # to do something other than a straight destroy (e.g. mark as archived).
+      # @param record [ActiveRecord::Base]
+      def destroy_record(record)
+        record&.destroy
+      end
+    end
+  end
+end