deimos-ruby 1.7.0.pre.beta1 → 1.8.0.pre.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ab8ac284db2c98dac5624caf5bf75118ad89fb9ec6e1f3109f15373f2bf4c8be
-  data.tar.gz: 7d26a7d8d163ab4783638c9393bea2d1a7c8f364a7eed2ea4cb699cbcbafd244
+  metadata.gz: 82c1b89b49622cb0d47a7a7aa56e076d908785906581f9f0772f09679acd6895
+  data.tar.gz: eeadf1d1a63db1407a81ef1f993be1b8d421e8e2624b9f42c324a2dad776bd1c
 SHA512:
-  metadata.gz: 2610223a8d8c2546dad4037e4d2b1845e77372b304f4c072c269316bceed4a6626bd36d3541561d91e1c68a06ed891c051feb16a1800763cf805061c05cadc58
-  data.tar.gz: 3916fa546b45182b987b0409d51bcce96c5a1beee449722861819f76377258e7ccb5700f50d2c0d8d1791986ed4f5d19b7c989cdcdbe77d1289e8531eb24ba15
+  metadata.gz: 6a85cd1d407c11d4becb6d6d11636623b96c706325f8bbca52caf4bf0cdce079df65432cb076d01503a4d4d977c8a662c22622e27193aeafc87ffbdc8d934045
+  data.tar.gz: '08f9b75f3057071f41effaf870f8d0927dca85d647726ba833371001c470a7a062ee74ac63b5acc2f284493d3a32373ffb6c3ca96f9428ffb762c83cfd804c42'

data/CHANGELOG.md

@@ -7,6 +7,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
+## 1.8.0-beta1 - 2020-07-06
+### Features :star:
+- Added `ActiveRecordConsumer` batch mode
+
+### Fixes :wrench:
+- Lag calculation can be incorrect if no messages are being consumed.
+- Fixed bug where printing messages on a MessageSizeTooLarge
+  error didn't work.
+
+### Roadmap
+- Moved SignalHandler and Executor to the `sigurd` gem.
+
+## 1.7.0-beta1 - 2020-05-12
 ### Features :star:
 - Added the DB Poller feature / process.
 

data/Gemfile.lock

@@ -5,6 +5,7 @@ PATH
       avro_turf (~> 0.11)
       phobos (~> 1.9)
       ruby-kafka (~> 0.7)
+      sigurd (= 0.0.1)
 
 GEM
   remote: https://rubygems.org/
@@ -65,6 +66,7 @@ GEM
     concurrent-ruby-ext (1.1.6)
       concurrent-ruby (= 1.1.6)
     crass (1.0.6)
+    database_cleaner (1.8.5)
     ddtrace (0.35.1)
       msgpack
     diff-lcs (1.3)
@@ -142,7 +144,7 @@ GEM
     pry (0.13.1)
       coderay (~> 1.1)
       method_source (~> 1.0)
-    rack (2.2.2)
+    rack (2.2.3)
     rack-test (1.1.0)
       rack (>= 1.0, < 3)
     rails (5.2.4.2)
@@ -204,6 +206,9 @@ GEM
       digest-crc
     ruby-progressbar (1.10.1)
     shellany (0.0.1)
+    sigurd (0.0.1)
+      concurrent-ruby (~> 1)
+      exponential-backoff
     sprockets (4.0.0)
       concurrent-ruby (~> 1.0)
       rack (> 1, < 3)
@@ -219,7 +224,7 @@ GEM
     unicode-display_width (1.7.0)
     websocket-driver (0.7.1)
       websocket-extensions (>= 0.1.0)
-    websocket-extensions (0.1.4)
+    websocket-extensions (0.1.5)
 
 PLATFORMS
   ruby
@@ -228,6 +233,7 @@ DEPENDENCIES
   activerecord (~> 5.2)
   activerecord-import
   avro (~> 1.9)
+  database_cleaner (~> 1.7)
   ddtrace (~> 0.11)
   deimos-ruby!
   dogstatsd-ruby (~> 4.2)

data/README.md

@@ -314,24 +314,14 @@ messages as an array and then process them together. This can improve
 consumer throughput, depending on the use case. Batch consumers behave like
 other consumers in regards to key and payload decoding, etc.
 
-To enable batch consumption, ensure that the `delivery` property is set to `inline_batch`. For example:
+To enable batch consumption, ensure that the `delivery` property of your
+consumer is set to `inline_batch`.
 
-```ruby
-Deimos.configure do
-  consumer do
-    class_name 'Consumers::MyBatchConsumer'
-    topic 'my_batched_topic'
-    group_id 'my_group_id'
-    delivery :inline_batch
-  end
-end
-```
-
-Batch consumers must inherit from the Deimos::BatchConsumer class as in
-this sample:
+Batch consumers will invoke the `consume_batch` method instead of `consume`
+as in this example:
 
 ```ruby
-class MyBatchConsumer < Deimos::BatchConsumer
+class MyBatchConsumer < Deimos::Consumer
 
   def consume_batch(payloads, metadata)
     # payloads is an array of schema-decoded hashes.
@@ -533,12 +523,14 @@ class MyConsumer < Deimos::ActiveRecordConsumer
 
   # Optional override of the way to fetch records based on payload and
   # key. Default is to use the key to search the primary key of the table.
+  # Only used in non-batch mode.
   def fetch_record(klass, payload, key)
     super
   end
 
   # Optional override on how to set primary key for new records. 
   # Default is to set the class's primary key to the message's decoded key. 
+  # Only used in non-batch mode.
   def assign_key(record, payload, key)
     super
   end
@@ -546,6 +538,7 @@ class MyConsumer < Deimos::ActiveRecordConsumer
   # Optional override of the default behavior, which is to call `destroy`
   # on the record - e.g. you can replace this with "archiving" the record
   # in some way. 
+  # Only used in non-batch mode.
   def destroy_record(record)
     super
   end
@@ -555,6 +548,64 @@ class MyConsumer < Deimos::ActiveRecordConsumer
   def record_attributes(payload)
     super.merge(:some_field => 'some_value')
   end
+
+  # Optional override to change the attributes used for identifying records
+  def record_key(payload)
+    super
+  end
+end
+```
+
+#### Batch Consumers
+
+Deimos also provides a batch consumption mode for `ActiveRecordConsumer` which
+processes groups of messages at once using the ActiveRecord backend. 
+
+Batch ActiveRecord consumers make use of the
+[activerecord-import](https://github.com/zdennis/activerecord-import) to insert
+or update multiple records in bulk SQL statements. This reduces processing
+time at the cost of skipping ActiveRecord callbacks for individual records.
+Deleted records (tombstones) are grouped into `delete_all` calls and thus also
+skip `destroy` callbacks.
+
+Batch consumption is used when the `delivery` setting for your consumer is set to `inline_batch`.
+
+**Note**: Currently, batch consumption only supports only primary keys as identifiers out of the box. See
+[the specs](spec/active_record_batch_consumer_spec.rb) for an example of how to use compound keys.
+
+By default, batches will be compacted before processing, i.e. only the last
+message for each unique key in a batch will actually be processed. To change
+this behaviour, call `compacted false` inside of your consumer definition.
+
+A sample batch consumer would look as follows:
+
+```ruby
+class MyConsumer < Deimos::ActiveRecordConsumer
+  schema 'MySchema'
+  key_config field: 'my_field'
+  record_class Widget
+
+  # Controls whether the batch is compacted before consuming.
+  # If true, only the last message for each unique key in a batch will be
+  # processed.
+  # If false, messages will be grouped into "slices" of independent keys
+  # and each slice will be imported separately.
+  #
+  # compacted false
+
+
+  # Optional override of the default behavior, which is to call `delete_all`
+  # on the associated records - e.g. you can replace this with setting a deleted
+  # flag on the record. 
+  def remove_records(records)
+    super
+  end
+ 
+  # Optional override to change the attributes of the record before they
+  # are saved.
+  def record_attributes(payload)
+    super.merge(:some_field => 'some_value')
+  end
 end
 ```
 
@@ -853,6 +904,9 @@ Deimos::Utils::InlineConsumer.get_messages_for(
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/flipp-oss/deimos .
 
+We have more information on the [internal architecture](docs/ARCHITECTURE.md) of Deimos
+for contributors!
+
 ### Linting
 
 Deimos uses Rubocop to lint the code. Please run Rubocop on your code 

data/deimos-ruby.gemspec

@@ -21,10 +21,12 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency('avro_turf', '~> 0.11')
   spec.add_runtime_dependency('phobos', '~> 1.9')
   spec.add_runtime_dependency('ruby-kafka', '~> 0.7')
+  spec.add_runtime_dependency('sigurd', '0.0.1')
 
   spec.add_development_dependency('activerecord', '~> 5.2')
   spec.add_development_dependency('activerecord-import')
   spec.add_development_dependency('avro', '~> 1.9')
+  spec.add_development_dependency('database_cleaner', '~> 1.7')
   spec.add_development_dependency('ddtrace', '~> 0.11')
   spec.add_development_dependency('dogstatsd-ruby', '~> 4.2')
   spec.add_development_dependency('guard', '~> 2')

data/docs/ARCHITECTURE.md

@@ -0,0 +1,144 @@
+# Deimos Architecture
+
+Deimos is the third of three libraries that add functionality on top of each 
+other:
+
+* [RubyKafka](https://github.com/zendesk/ruby-kafka) is the low-level Kafka
+  client, providing API's for producers, consumers and the client as a whole.
+* [Phobos](https://github.com/phobos/phobos) is a lightweight wrapper on top
+  of RubyKafka that provides threaded consumers, a simpler way to write
+  producers, and lifecycle management.
+* [Deimos](https://github.com/flipp-oss/deimos/) is a full-featured framework
+  using Phobos as its base which provides schema integration (e.g. Avro), 
+  database integration, metrics, tracing, test helpers and other utilities.
+  
+## Folder structure
+
+As of May 12, 2020, the following are the important files to understand in how
+Deimos fits together:
+* `lib/generators`: Generators to generate database migrations, e.g.
+   for the DB Poller and DB Producer features.
+* `lib/tasks`: Rake tasks for starting consumers, DB Pollers, etc.
+* `lib/deimos`: Main Deimos code.
+* `lib/deimos/deimos.rb`: The bootstrap / startup code for Deimos. Also provides
+   some global convenience methods and (for legacy purposes) the way to 
+   start the DB Producer.
+* `lib/deimos/backends`: The different plug-in producer backends - e.g. produce
+   directly to Kafka, use the DB backend, etc.
+* `lib/deimos/schema_backends`: The different plug-in schema handlers, such
+  as the various flavors of Avro (with/without schema registry etc.)
+* `lib/deimos/metrics`: The different plug-in metrics providers, e.g. Datadog.
+* `lib/deimos/tracing`: The different plug-in tracing providers, e.g. Datadog.
+* `lib/deimos/utils`: Utility classes for things not directly related to
+   producing and consuming, such as the DB Poller, DB Producer, lag reporter, etc.
+* `lib/deimos/config`: Classes related to configuring Deimos.
+* `lib/deimos/monkey_patches`: Monkey patches to existing libraries. These
+   should be removed in a future update.
+
+## Features
+
+### Producers and Consumers
+
+Both producers and consumers include the `SharedConfig` module, which 
+standardizes configuration like schema settings, topic, keys, etc.
+
+Consumers come in two flavors: `Consumer` and `BatchConsumer`. Both include
+`BaseConsumer` for shared functionality.
+
+While producing messages go to Kafka by default, literally anything else
+can happen when your producer calls `produce`, by swapping out the producer
+_backend_. This is just a file that needs to inherit from `Deimos::Backends::Base`
+and must implement a single method, `execute`.
+
+Producers have a complex workflow while processing the payload to publish. This
+is aided by the `Deimos::Message` class (not to be confused with the 
+`KafkaMessage` class, which is an ActiveRecord used by the DB Producer feature,
+below).
+
+### Schemas
+
+Schema backends are used to encode and decode payloads into different formats
+such as Avro. These are integrated with producers and consumers, as well
+as test helpers. These are a bit more involved than producer backends, and
+must define methods such as:
+* `encode` a payload or key (when encoding a key, for Avro a key schema
+  may be auto-generated)
+* `decode` a payload or key
+* `validate` that a payload is correct for encoding
+* `coerce` a payload into the given schema (e.g. turn ints into strings)
+* Get a list of `schema_fields` in the configured schema, used when interacting
+  with ActiveRecord
+* Define a `mock` backend when the given backend is used. This is used
+  during testing. Typically mock backends will validate values but not
+  actually encode/decode them.
+  
+### Configuration
+
+Deimos has its own `Configurable` module that makes heavy use of `method_missing`
+to provide a very succinct but powerful configuration format (including
+default values, procs, print out as hash, reset, etc.). It also
+allows for multiple blocks to define different objects of the same time
+(like producers, consumers, pollers etc.).
+
+The configuration definition for Deimos is in `config/configuration.rb`. In
+addition, there are methods in `config/phobos_config.rb` which translate to/from
+the Phobos configuration format and support the old `phobos.yml` method
+of configuration.
+
+### Metrics and Tracing
+
+These are simpler than other plugins and must implement the expected methods
+(`increment`, `gauge`, `histogram` and `time` for metrics, and `start`, `finish`
+and `set_error` for tracing). These are used primarily in producers and consumers.
+
+### ActiveRecord Integration
+
+Deimos provides an `ActiveRecordConsumer` and `ActiveRecordProducer`. These are
+relatively lightweight ways to save data into a database or read it off 
+the database as part of app logic. It uses things like the `coerce` method
+of the schema backends to manage the differences between the given payload
+and the configured schema for the topic.
+
+### Database Backend / Database Producer
+
+This feature (which provides better performance and transaction guarantees)
+is powered by two components:
+* The `db` _publish backend_, which saves messages to the database rather
+  than to Kafka;
+* The `DbProducer` utility, which runs as a separate process, pulls data
+  from the database and sends it to Kafka.
+
+There are a set of utility classes that power the producer, which are largely
+copied from Phobos:
+* `Executor` takes a set of "runnable" things (which implement a `start` and `stop`
+  method) puts them in a thread pool and runs them all concurrently. It
+  manages starting and stopping all threads when necessary.
+* `SignalHandler` wraps the Executor and handles SIGINT and SIGTERM signals
+  to stop the executor gracefully.
+  
+In the case of this feature, the `DbProducer` is the runnable object - it
+can run several threads at once.
+
+On the database side, the `ActiveRecord` models that power this feature are:
+* `KafkaMessage`: The actual message, saved to the database. This message
+  is already encoded by the producer, so only has to be sent.
+* `KafkaTopicInfo`: Used for locking topics so only one producer can work
+  on it at once.
+  
+A Rake task (defined in `deimos.rake`) can be used to start the producer.
+  
+### Database Poller
+
+This feature (which periodically polls the database to send Kafka messages)
+primarily uses other aspects of Deimos and hence is relatively small in size.
+The `DbPoller` class acts as a "runnable" and is used by an Executor (above).
+The `PollInfo` class is saved to the database to keep track of where each
+poller is up to.
+
+A Rake task (defined in `deimos.rake`) can be used to start the pollers.
+
+### Other Utilities
+
+The `utils` folder also contains the `LagReporter` (which sends metrics on
+lag) and the `InlineConsumer`, which can read data from a topic and directly
+pass it into a handler or save it to memory.

data/docs/CONFIGURATION.md

@@ -58,6 +58,10 @@ Deimos.configure do
     namespace 'my.namespace'
     key_config field: :id
 
+    # Setting to :inline_batch will invoke consume_batch instead of consume
+    # for each batch of messages.
+    delivery :batch
+
   # If config.schema.path is app/schemas, assumes there is a file in
   # app/schemas/my/namespace/MyTopicSchema.avsc
   end

data/lib/deimos.rb

@@ -28,8 +28,8 @@ if defined?(ActiveRecord)
   require 'deimos/kafka_source'
   require 'deimos/kafka_topic_info'
   require 'deimos/backends/db'
-  require 'deimos/utils/signal_handler.rb'
-  require 'deimos/utils/executor.rb'
+  require 'sigurd/signal_handler.rb'
+  require 'sigurd/executor.rb'
   require 'deimos/utils/db_producer.rb'
   require 'deimos/utils/db_poller'
 end
@@ -72,10 +72,10 @@ module Deimos
         Deimos::Utils::DbProducer.
           new(self.config.db_producer.logger || self.config.logger)
       end
-      executor = Deimos::Utils::Executor.new(producers,
-                                             sleep_seconds: 5,
-                                             logger: self.config.logger)
-      signal_handler = Deimos::Utils::SignalHandler.new(executor)
+      executor = Sigurd::Executor.new(producers,
+                                      sleep_seconds: 5,
+                                      logger: self.config.logger)
+      signal_handler = Sigurd::SignalHandler.new(executor)
       signal_handler.run!
     end
   end

data/lib/deimos/active_record_consume/batch_consumption.rb

@@ -0,0 +1,159 @@
+# frozen_string_literal: true
+
+require 'deimos/active_record_consume/batch_slicer'
+require 'deimos/utils/deadlock_retry'
+require 'deimos/message'
+
+module Deimos
+  module ActiveRecordConsume
+    # Methods for consuming batches of messages and saving them to the database
+    # in bulk ActiveRecord operations.
+    module BatchConsumption
+      # Handle a batch of Kafka messages. Batches are split into "slices",
+      # which are groups of independent messages that can be processed together
+      # in a single database operation.
+      # If two messages in a batch have the same key, we cannot process them
+      # in the same operation as they would interfere with each other. Thus
+      # they are split
+      # @param payloads [Array<Hash>] Decoded payloads.
+      # @param metadata [Hash] Information about batch, including keys.
+      def consume_batch(payloads, metadata)
+        messages = payloads.
+          zip(metadata[:keys]).
+          map { |p, k| Deimos::Message.new(p, nil, key: k) }
+
+        tags = %W(topic:#{metadata[:topic]})
+
+        Deimos.instrument('ar_consumer.consume_batch', tags) do
+          # The entire batch should be treated as one transaction so that if
+          # any message fails, the whole thing is rolled back or retried
+          # if there is deadlock
+          Deimos::Utils::DeadlockRetry.wrap(tags) do
+            if @compacted || self.class.config[:no_keys]
+              update_database(compact_messages(messages))
+            else
+              uncompacted_update(messages)
+            end
+          end
+        end
+      end
+
+      # Get unique key for the ActiveRecord instance from the incoming key.
+      # Override this method (with super) to customize the set of attributes that
+      # uniquely identifies each record in the database.
+      # @param key [String] The encoded key.
+      # @return [Hash] The key attributes.
+      def record_key(key)
+        decoded_key = decode_key(key)
+
+        if decoded_key.nil?
+          {}
+        elsif decoded_key.is_a?(Hash)
+          @key_converter.convert(decoded_key)
+        else
+          { @klass.primary_key => decoded_key }
+        end
+      end
+
+    protected
+
+      # Perform database operations for a batch of messages without compaction.
+      # All messages are split into slices containing only unique keys, and
+      # each slice is handles as its own batch.
+      # @param messages [Array<Message>] List of messages.
+      def uncompacted_update(messages)
+        BatchSlicer.
+          slice(messages).
+          each(&method(:update_database))
+      end
+
+      # Perform database operations for a group of messages.
+      # All messages with payloads are passed to upsert_records.
+      # All tombstones messages are passed to remove_records.
+      # @param messages [Array<Message>] List of messages.
+      def update_database(messages)
+        # Find all upserted records (i.e. that have a payload) and all
+        # deleted record (no payload)
+        removed, upserted = messages.partition(&:tombstone?)
+
+        upsert_records(upserted) if upserted.any?
+        remove_records(removed) if removed.any?
+      end
+
+      # Upsert any non-deleted records
+      # @param messages [Array<Message>] List of messages for a group of
+      # records to either be updated or inserted.
+      def upsert_records(messages)
+        key_cols = key_columns(messages)
+
+        # Create payloads with payload + key attributes
+        upserts = messages.map do |m|
+          record_attributes(m.payload, m.key)&.
+            merge(record_key(m.key))
+        end
+
+        # If overridden record_attributes indicated no record, skip
+        upserts.compact!
+
+        options = if key_cols.empty?
+                    {} # Can't upsert with no key, just do regular insert
+                  else
+                    {
+                      on_duplicate_key_update: {
+                        # conflict_target must explicitly list the columns for
+                        # Postgres and SQLite. Not required for MySQL, but this
+                        # ensures consistent behaviour.
+                        conflict_target: key_cols,
+                        columns: :all
+                      }
+                    }
+                  end
+
+        @klass.import!(upserts, options)
+      end
+
+      # Delete any records with a tombstone.
+      # @param messages [Array<Message>] List of messages for a group of
+      # deleted records.
+      def remove_records(messages)
+        clause = deleted_query(messages)
+
+        clause.delete_all
+      end
+
+      # Create an ActiveRecord relation that matches all of the passed
+      # records. Used for bulk deletion.
+      # @param records [Array<Message>] List of messages.
+      # @return ActiveRecord::Relation Matching relation.
+      def deleted_query(records)
+        keys = records.
+          map { |m| record_key(m.key)[@klass.primary_key] }.
+          reject(&:nil?)
+
+        @klass.unscoped.where(@klass.primary_key => keys)
+      end
+
+      # Get the set of attribute names that uniquely identify messages in the
+      # batch. Requires at least one record.
+      # @param records [Array<Message>] Non-empty list of messages.
+      # @return [Array<String>] List of attribute names.
+      # @raise If records is empty.
+      def key_columns(records)
+        raise 'Cannot determine key from empty batch' if records.empty?
+
+        first_key = records.first.key
+        record_key(first_key).keys
+      end
+
+      # Compact a batch of messages, taking only the last message for each
+      # unique key.
+      # @param batch [Array<Message>] Batch of messages.
+      # @return [Array<Message>] Compacted batch.
+      def compact_messages(batch)
+        return batch unless batch.first&.key.present?
+
+        batch.reverse.uniq(&:key).reverse!
+      end
+    end
+  end
+end