deimos-ruby 1.6.2 → 1.8.0.pre.beta2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 53aab4ddcc19808cf7149ed226de9b7ddda07d12ed4237c157cce75780379c6c
-  data.tar.gz: 74ebf879894b88fc4ce5082d4f8b82f8f79a0ef581bf368f25423a755d71d635
+  metadata.gz: 5c184b8f182ecfa304684f4e8e522e93c43c12fe50b34300326c2e1ef206b6d1
+  data.tar.gz: 5861974e036c92470a4a801bed477e5cebefa679cc9996a5d6f0258eaa484150
 SHA512:
-  metadata.gz: 4c0fa67f0b958ddc3083c278c5c195f896b134f822168df245e906e89f5f5414f063175e2ed1a444891a210d28e7a0d96836806b6ba2efd6f55bf46d929450c1
-  data.tar.gz: 9fc31899551ec75dc0ca1e7ac02cc3f4ed429ec8eb25f42057024d2dc9cde5bef70eabafcf8276e7a8a4ae2b7a88b4efff5efe047715612bd5f5480ce1f52576
+  metadata.gz: f19a799d333485b6461e2d8cdeaec5643ec6ddf8f6a2ce1725f3c39269cc193cc5b1bc1e81266b6a33af87ec5d09b3ac2f857b48da61c3baded00a98d6935624
+  data.tar.gz: a08f6519a635e76944b3b7c2f08d5369b6f327e60e968391fcd5e09e703e8ed526d35d8e124f91cbff8c885fbf8d620f90e0a99b48e4289868a0c11925b43d51

data/.circleci/config.yml

@@ -20,6 +20,9 @@ jobs:
       # Bundle install dependencies in /tmp/
       # so Dockerfile does not copy them since
       # its base image is different than CircleCI
+      - run:
+          name: Install bundler
+          command: gem install bundler:2.1.4
       - run:
           name: Bundle install
           command: bundle install --path vendor/bundle --jobs=4 --retry=3
@@ -40,6 +43,9 @@ jobs:
     steps:
       - attach_workspace:
           at: ~/workspace
+      - run:
+          name: Install bundler
+          command: gem install bundler:2.1.4
       - run:
            name: Point bundle to vendor/bundle
            command: bundle --path vendor/bundle
@@ -50,6 +56,9 @@ jobs:
     steps:
       - attach_workspace:
           at: ~/workspace
+      - run:
+          name: Install bundler
+          command: gem install bundler:2.1.4
       - run:
            name: Point bundle to vendor/bundle
            command: bundle --path vendor/bundle

data/.rubocop.yml

@@ -1,10 +1,11 @@
 require: rubocop-rspec
 
 AllCops:
-  TargetRubyVersion: 2.3
+  TargetRubyVersion: 2.4
   Exclude:
     - lib/deimos/monkey_patches/*.rb
     - vendor/**/*
+  NewCops: enable
 
 # class Plumbus
 # private
@@ -34,6 +35,12 @@ Layout/DotPosition:
 Layout/EmptyLinesAroundBlockBody:
   Enabled: false
 
+Layout/LineLength:
+  Max: 100
+  Severity: refactor
+  Exclude:
+  - 'spec/**/*'
+
 # foo = if expression
 #         'bar'
 #       end
@@ -82,12 +89,6 @@ Metrics/CyclomaticComplexity:
   Severity: refactor
   Max: 20
 
-Metrics/LineLength:
-  Max: 100
-  Severity: refactor
-  Exclude:
-  - 'spec/**/*'
-
 Metrics/MethodLength:
   Severity: refactor
   Max: 30
@@ -123,12 +124,6 @@ Style/BlockDelimiters:
 # some_method(x, y, {a: 1, b: 2})
 # some_method(x, y, {a: 1, b: 2}, a: 1, b: 2)
 
-# good
-# some_method(x, y, a: 1, b: 2)
-# some_method(x, y, {a: 1, b: 2}, {a: 1, b: 2})
-Style/BracesAroundHashParameters:
-  EnforcedStyle: context_dependent
-
 # Enable both this:
 #  MyModule::MyClass
 # and this:
@@ -179,6 +174,13 @@ Style/GuardClause:
 Style/HashSyntax:
   EnforcedStyle: ruby19_no_mixed_keys
 
+# We are still unofficially targeting Ruby 2.3
+Style/HashTransformKeys:
+  Enabled: false
+
+Style/HashTransformValues:
+  Enabled: false
+
 Style/IfUnlessModifier:
   Enabled: false
 

data/.ruby-version

@@ -1 +1 @@
-2.5.1
+2.5.3

data/CHANGELOG.md

@@ -7,6 +7,37 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
+## 1.8.0-beta2 - 2020-07-08
+
+### Fixes :wrench:
+- Fix crash with batch consumption due to not having ActiveSupport::Concern
+
+### Features :star:
+- Add `first_offset` to the metadata sent via the batch
+
+## 1.8.0-beta1 - 2020-07-06
+### Features :star:
+- Added `ActiveRecordConsumer` batch mode
+
+### Fixes :wrench:
+- Lag calculation can be incorrect if no messages are being consumed.
+- Fixed bug where printing messages on a MessageSizeTooLarge
+  error didn't work.
+
+### Roadmap
+- Moved SignalHandler and Executor to the `sigurd` gem.
+
+## 1.7.0-beta1 - 2020-05-12
+### Features :star:
+- Added the DB Poller feature / process.
+
+## 1.6.4 - 2020-05-11
+- Fixed the payload logging fix for errored messages as well.
+
+## 1.6.3 - 2020-05-04
+### Fixes :wrench:
+- Fixed the payload logging fix.
+
 ## 1.6.2 - 2020-05-04
 ### Fixes :wrench:
 - When saving records via `ActiveRecordConsumer`, update `updated_at` to today's time

data/Gemfile.lock

@@ -1,10 +1,11 @@
 PATH
   remote: .
   specs:
-    deimos-ruby (1.6.1)
+    deimos-ruby (1.8.0.pre.beta1)
       avro_turf (~> 0.11)
       phobos (~> 1.9)
       ruby-kafka (~> 0.7)
+      sigurd (= 0.0.1)
 
 GEM
   remote: https://rubygems.org/
@@ -41,7 +42,7 @@ GEM
       activemodel (= 5.2.4.2)
       activesupport (= 5.2.4.2)
       arel (>= 9.0)
-    activerecord-import (1.0.3)
+    activerecord-import (1.0.4)
       activerecord (>= 3.2)
     activestorage (5.2.4.2)
       actionpack (= 5.2.4.2)
@@ -54,30 +55,31 @@ GEM
       tzinfo (~> 1.1)
     arel (9.0.0)
     ast (2.4.0)
-    avro (1.9.1)
+    avro (1.9.2)
       multi_json
     avro_turf (0.11.0)
       avro (>= 1.7.7, < 1.10)
       excon (~> 0.45)
     builder (3.2.4)
     coderay (1.1.2)
-    concurrent-ruby (1.1.5)
-    concurrent-ruby-ext (1.1.5)
-      concurrent-ruby (= 1.1.5)
+    concurrent-ruby (1.1.6)
+    concurrent-ruby-ext (1.1.6)
+      concurrent-ruby (= 1.1.6)
     crass (1.0.6)
-    ddtrace (0.30.0)
+    database_cleaner (1.8.5)
+    ddtrace (0.35.1)
       msgpack
     diff-lcs (1.3)
     digest-crc (0.5.1)
-    dogstatsd-ruby (4.5.0)
+    dogstatsd-ruby (4.8.0)
     erubi (1.9.0)
     excon (0.73.0)
     exponential-backoff (0.0.4)
-    ffi (1.11.3)
+    ffi (1.12.2)
     formatador (0.2.5)
     globalid (0.4.2)
       activesupport (>= 4.2.0)
-    guard (2.16.1)
+    guard (2.16.2)
       formatador (>= 0.2.4)
       listen (>= 2.7, < 4.0)
       lumberjack (>= 1.0.12, < 2.0)
@@ -107,17 +109,17 @@ GEM
     loofah (2.5.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
-    lumberjack (1.0.13)
+    lumberjack (1.2.4)
     mail (2.7.1)
       mini_mime (>= 0.1.1)
     marcel (0.3.3)
       mimemagic (~> 0.3.2)
-    method_source (0.9.2)
-    mimemagic (0.3.4)
+    method_source (1.0.0)
+    mimemagic (0.3.5)
     mini_mime (1.0.2)
     mini_portile2 (2.4.0)
     minitest (5.14.0)
-    msgpack (1.3.1)
+    msgpack (1.3.3)
     multi_json (1.14.1)
     mysql2 (0.5.3)
     nenv (0.3.0)
@@ -128,9 +130,9 @@ GEM
       nenv (~> 0.1)
       shellany (~> 0.0)
     parallel (1.19.1)
-    parser (2.6.5.0)
+    parser (2.7.1.2)
       ast (~> 2.4.0)
-    pg (1.1.4)
+    pg (1.2.3)
     phobos (1.9.0)
       activesupport (>= 3.0.0)
       concurrent-ruby (>= 1.0.2)
@@ -139,10 +141,10 @@ GEM
       logging
       ruby-kafka
       thor
-    pry (0.12.2)
-      coderay (~> 1.1.0)
-      method_source (~> 0.9.0)
-    rack (2.2.2)
+    pry (0.13.1)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    rack (2.2.3)
     rack-test (1.1.0)
       rack (>= 1.0, < 3)
     rails (5.2.4.2)
@@ -171,37 +173,42 @@ GEM
       thor (>= 0.19.0, < 2.0)
     rainbow (3.0.0)
     rake (13.0.1)
-    rb-fsevent (0.10.3)
-    rb-inotify (0.10.0)
+    rb-fsevent (0.10.4)
+    rb-inotify (0.10.1)
       ffi (~> 1.0)
+    rexml (3.2.4)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
       rspec-expectations (~> 3.9.0)
       rspec-mocks (~> 3.9.0)
-    rspec-core (3.9.0)
-      rspec-support (~> 3.9.0)
-    rspec-expectations (3.9.0)
+    rspec-core (3.9.2)
+      rspec-support (~> 3.9.3)
+    rspec-expectations (3.9.1)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.9.0)
-    rspec-mocks (3.9.0)
+    rspec-mocks (3.9.1)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.9.0)
-    rspec-support (3.9.0)
+    rspec-support (3.9.3)
     rspec_junit_formatter (0.4.1)
       rspec-core (>= 2, < 4, != 2.12.0)
-    rubocop (0.77.0)
+    rubocop (0.82.0)
       jaro_winkler (~> 1.5.1)
       parallel (~> 1.10)
-      parser (>= 2.6)
+      parser (>= 2.7.0.1)
       rainbow (>= 2.2.2, < 4.0)
+      rexml
       ruby-progressbar (~> 1.7)
-      unicode-display_width (>= 1.4.0, < 1.7)
-    rubocop-rspec (1.37.1)
+      unicode-display_width (>= 1.4.0, < 2.0)
+    rubocop-rspec (1.39.0)
       rubocop (>= 0.68.1)
     ruby-kafka (0.7.10)
       digest-crc
     ruby-progressbar (1.10.1)
     shellany (0.0.1)
+    sigurd (0.0.1)
+      concurrent-ruby (~> 1)
+      exponential-backoff
     sprockets (4.0.0)
       concurrent-ruby (~> 1.0)
       rack (> 1, < 3)
@@ -209,15 +216,15 @@ GEM
       actionpack (>= 4.0)
       activesupport (>= 4.0)
       sprockets (>= 3.0.0)
-    sqlite3 (1.4.1)
+    sqlite3 (1.4.2)
     thor (1.0.1)
     thread_safe (0.3.6)
     tzinfo (1.2.7)
       thread_safe (~> 0.1)
-    unicode-display_width (1.6.0)
+    unicode-display_width (1.7.0)
     websocket-driver (0.7.1)
       websocket-extensions (>= 0.1.0)
-    websocket-extensions (0.1.4)
+    websocket-extensions (0.1.5)
 
 PLATFORMS
   ruby
@@ -226,7 +233,7 @@ DEPENDENCIES
   activerecord (~> 5.2)
   activerecord-import
   avro (~> 1.9)
-  bundler (~> 1)
+  database_cleaner (~> 1.7)
   ddtrace (~> 0.11)
   deimos-ruby!
   dogstatsd-ruby (~> 4.2)
@@ -244,4 +251,4 @@ DEPENDENCIES
   sqlite3 (~> 1.3)
 
 BUNDLED WITH
-   1.17.3
+   2.1.4

data/README.md

@@ -23,6 +23,7 @@ Built on Phobos and hence Ruby-Kafka.
    * [Consumers](#consumers)
    * [Rails Integration](#rails-integration)
    * [Database Backend](#database-backend)
+   * [Database Poller](#database-poller)
    * [Running Consumers](#running-consumers)
    * [Metrics](#metrics)
    * [Testing](#testing)
@@ -313,28 +314,19 @@ messages as an array and then process them together. This can improve
 consumer throughput, depending on the use case. Batch consumers behave like
 other consumers in regards to key and payload decoding, etc.
 
-To enable batch consumption, ensure that the `delivery` property is set to `inline_batch`. For example:
+To enable batch consumption, ensure that the `delivery` property of your
+consumer is set to `inline_batch`.
 
-```ruby
-Deimos.configure do
-  consumer do
-    class_name 'Consumers::MyBatchConsumer'
-    topic 'my_batched_topic'
-    group_id 'my_group_id'
-    delivery :inline_batch
-  end
-end
-```
-
-Batch consumers must inherit from the Deimos::BatchConsumer class as in
-this sample:
+Batch consumers will invoke the `consume_batch` method instead of `consume`
+as in this example:
 
 ```ruby
-class MyBatchConsumer < Deimos::BatchConsumer
+class MyBatchConsumer < Deimos::Consumer
 
   def consume_batch(payloads, metadata)
     # payloads is an array of schema-decoded hashes.
-    # metadata is a hash that contains information like :keys and :topic.
+    # metadata is a hash that contains information like :keys, :topic, 
+    # and :first_offset.
     # Keys are automatically decoded and available as an array with
     # the same cardinality as the payloads. If you need to iterate
     # over payloads and keys together, you can use something like this:
@@ -532,12 +524,14 @@ class MyConsumer < Deimos::ActiveRecordConsumer
 
   # Optional override of the way to fetch records based on payload and
   # key. Default is to use the key to search the primary key of the table.
+  # Only used in non-batch mode.
   def fetch_record(klass, payload, key)
     super
   end
 
   # Optional override on how to set primary key for new records. 
   # Default is to set the class's primary key to the message's decoded key. 
+  # Only used in non-batch mode.
   def assign_key(record, payload, key)
     super
   end
@@ -545,6 +539,7 @@ class MyConsumer < Deimos::ActiveRecordConsumer
   # Optional override of the default behavior, which is to call `destroy`
   # on the record - e.g. you can replace this with "archiving" the record
   # in some way. 
+  # Only used in non-batch mode.
   def destroy_record(record)
     super
   end
@@ -554,9 +549,136 @@ class MyConsumer < Deimos::ActiveRecordConsumer
   def record_attributes(payload)
     super.merge(:some_field => 'some_value')
   end
+
+  # Optional override to change the attributes used for identifying records
+  def record_key(payload)
+    super
+  end
 end
 ```
 
+#### Batch Consumers
+
+Deimos also provides a batch consumption mode for `ActiveRecordConsumer` which
+processes groups of messages at once using the ActiveRecord backend. 
+
+Batch ActiveRecord consumers make use of the
+[activerecord-import](https://github.com/zdennis/activerecord-import) to insert
+or update multiple records in bulk SQL statements. This reduces processing
+time at the cost of skipping ActiveRecord callbacks for individual records.
+Deleted records (tombstones) are grouped into `delete_all` calls and thus also
+skip `destroy` callbacks.
+
+Batch consumption is used when the `delivery` setting for your consumer is set to `inline_batch`.
+
+**Note**: Currently, batch consumption only supports only primary keys as identifiers out of the box. See
+[the specs](spec/active_record_batch_consumer_spec.rb) for an example of how to use compound keys.
+
+By default, batches will be compacted before processing, i.e. only the last
+message for each unique key in a batch will actually be processed. To change
+this behaviour, call `compacted false` inside of your consumer definition.
+
+A sample batch consumer would look as follows:
+
+```ruby
+class MyConsumer < Deimos::ActiveRecordConsumer
+  schema 'MySchema'
+  key_config field: 'my_field'
+  record_class Widget
+
+  # Controls whether the batch is compacted before consuming.
+  # If true, only the last message for each unique key in a batch will be
+  # processed.
+  # If false, messages will be grouped into "slices" of independent keys
+  # and each slice will be imported separately.
+  #
+  # compacted false
+
+
+  # Optional override of the default behavior, which is to call `delete_all`
+  # on the associated records - e.g. you can replace this with setting a deleted
+  # flag on the record. 
+  def remove_records(records)
+    super
+  end
+ 
+  # Optional override to change the attributes of the record before they
+  # are saved.
+  def record_attributes(payload)
+    super.merge(:some_field => 'some_value')
+  end
+end
+```
+
+## Database Poller
+
+Another method of fetching updates from the database to Kafka is by polling
+the database (a process popularized by [Kafka Connect](https://docs.confluent.io/current/connect/index.html)).
+Deimos provides a database poller, which allows you the same pattern but
+with all the flexibility of real Ruby code, and the added advantage of having
+a single consistent framework to talk to Kafka.
+
+One of the disadvantages of polling the database is that it can't detect deletions.
+You can get over this by configuring a mixin to send messages *only* on deletion,
+and use the poller to handle all other updates. You can reuse the same producer
+for both cases to handle joins, changes/mappings, business logic, etc.
+
+To enable the poller, generate the migration:
+
+```ruby
+rails g deimos:db_poller
+```
+
+Run the migration:
+
+```ruby
+rails db:migrate
+```
+
+Add the following configuration:
+
+```ruby
+Deimos.configure do
+  db_poller do
+    producer_class 'MyProducer' # an ActiveRecordProducer
+  end
+  db_poller do
+    producer_class 'MyOtherProducer'
+    run_every 2.minutes
+    delay 5.seconds # to allow for transactions to finish
+    full_table true # if set, dump the entire table every run; use for small tables
+  end
+end
+```
+
+All the information around connecting and querying the database lives in the
+producer itself, so you don't need to write any additional code. You can 
+define one additional method on the producer:
+
+```ruby
+class MyProducer < Deimos::ActiveRecordProducer
+  ...
+  def poll_query(time_from:, time_to:, column_name:, min_id:)
+    # Default is to use the timestamp `column_name` to find all records
+    # between time_from and time_to, or records where `updated_at` is equal to
+    # `time_from` but its ID is greater than `min_id`. This is called
+    # successively as the DB is polled to ensure even if a batch ends in the
+    # middle of a timestamp, we won't miss any records.
+    # You can override or change this behavior if necessary.
+  end
+end
+```
+
+To run the DB poller:
+
+    rake deimos:db_poller
+
+Note that the DB poller creates one thread per configured poller, and is
+currently designed *not* to be scaled out - i.e. it assumes you will only
+have one process running at a time. If a particular poll takes longer than
+the poll interval (i.e. interval is set at 1 minute but it takes 75 seconds)
+the next poll will begin immediately following the first one completing.
+
 ## Running consumers
 
 Deimos includes a rake task. Once it's in your gemfile, just run
@@ -783,6 +905,9 @@ Deimos::Utils::InlineConsumer.get_messages_for(
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/flipp-oss/deimos .
 
+We have more information on the [internal architecture](docs/ARCHITECTURE.md) of Deimos
+for contributors!
+
 ### Linting
 
 Deimos uses Rubocop to lint the code. Please run Rubocop on your code