deimos-ruby 1.7.0.pre.beta1 → 1.8.1.pre.beta3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ab8ac284db2c98dac5624caf5bf75118ad89fb9ec6e1f3109f15373f2bf4c8be
-  data.tar.gz: 7d26a7d8d163ab4783638c9393bea2d1a7c8f364a7eed2ea4cb699cbcbafd244
+  metadata.gz: 380e0a86786dc4b308858186e5f97a102d13e0d0ac72ca172e941d42b6dfb81a
+  data.tar.gz: ca99315cf3ee096160c24f15d194ce73d77d95b1e7c27b3454e2a5e4165ef156
 SHA512:
-  metadata.gz: 2610223a8d8c2546dad4037e4d2b1845e77372b304f4c072c269316bceed4a6626bd36d3541561d91e1c68a06ed891c051feb16a1800763cf805061c05cadc58
-  data.tar.gz: 3916fa546b45182b987b0409d51bcce96c5a1beee449722861819f76377258e7ccb5700f50d2c0d8d1791986ed4f5d19b7c989cdcdbe77d1289e8531eb24ba15
+  metadata.gz: 8aa602000e702a9ce3c271b0796f4ea864fdc0acaf0852fdf7bdc3bf51e0eaf2df3265420e2558f37e3c174a5a555d43920323bb3a1539753b8821ae00ad6f32
+  data.tar.gz: c505ef8b587d2a405f9773a3c0f5a178f27811a071a4579ecfbe46c9f75759f1375b773949a384cdd465638ca7e95c53e6e34d8a1b7dd27e5128e03a90c0c3b1

data/.rubocop.yml

@@ -5,6 +5,7 @@ AllCops:
   Exclude:
     - lib/deimos/monkey_patches/*.rb
     - vendor/**/*
+    - Guardfile
   NewCops: enable
 
 # class Plumbus
@@ -36,7 +37,7 @@ Layout/EmptyLinesAroundBlockBody:
   Enabled: false
 
 Layout/LineLength:
-  Max: 100
+  Max: 120
   Severity: refactor
   Exclude:
   - 'spec/**/*'
@@ -77,13 +78,14 @@ Lint/UnusedMethodArgument:
 
 Metrics/AbcSize:
   Severity: refactor
-  Max: 20
+  Max: 25
 
 Metrics/BlockLength:
-  Severity: refactor
+  Enabled: false
 
 Metrics/ClassLength:
   Severity: refactor
+  Max: 500
 
 Metrics/CyclomaticComplexity:
   Severity: refactor
@@ -91,10 +93,11 @@ Metrics/CyclomaticComplexity:
 
 Metrics/MethodLength:
   Severity: refactor
-  Max: 30
+  Max: 50
 
 Metrics/ModuleLength:
   Severity: refactor
+  Max: 200
 
 Metrics/ParameterLists:
   Max: 5
@@ -102,6 +105,7 @@ Metrics/ParameterLists:
 
 Metrics/PerceivedComplexity:
   Severity: refactor
+  Max: 10
 
 # Use alias_method instead of alias
 Style/Alias:

data/CHANGELOG.md

@@ -7,6 +7,56 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
+## 1.8.1-beta3 - 2020-08-05
+
+### Fixes :wrench:
+- Simplify decoding messages and handle producer not found
+- Consolidate types in sub-records recursively
+  (fixes [#72](https://github.com/flipp-oss/deimos/issues/72))
+
+## 1.8.1-beta2 - 2020-07-28
+
+### Features :star:
+- Add `SchemaControllerMixin` to encode and decode schema-encoded
+  payloads in Rails controllers.
+
+## 1.8.1-beta1 - 2020-07-22
+
+### Fixes :wrench:
+- Retry deleting messages without resending the batch if the
+  delete fails (fixes [#34](https://github.com/flipp-oss/deimos/issues/34))
+- Delete messages in batches rather than all at once to
+  cut down on the chance of a deadlock.
+
+### Features :star:
+- Add `last_processed_at` to `kafka_topic_info` to ensure
+  that wait metrics are accurate in cases where records
+  get created with an old `created_at` time (e.g. for
+  long-running transactions).
+- Add generator for ActiveRecord models and migrations (fixes [#6](https://github.com/flipp-oss/deimos/issues/6))
+
+## 1.8.0-beta2 - 2020-07-08
+
+### Fixes :wrench:
+- Fix crash with batch consumption due to not having ActiveSupport::Concern
+
+### Features :star:
+- Add `first_offset` to the metadata sent via the batch
+
+## 1.8.0-beta1 - 2020-07-06
+### Features :star:
+- Added `ActiveRecordConsumer` batch mode
+
+### Fixes :wrench:
+- Fixes `send_produce_error` to decode `failed_messages` with built-in decoder. 
+- Lag calculation can be incorrect if no messages are being consumed.
+- Fixed bug where printing messages on a MessageSizeTooLarge
+  error didn't work.
+
+### Roadmap
+- Moved SignalHandler and Executor to the `sigurd` gem.
+
+## 1.7.0-beta1 - 2020-05-12
 ### Features :star:
 - Added the DB Poller feature / process.
 

data/Gemfile.lock

@@ -1,79 +1,95 @@
 PATH
   remote: .
   specs:
-    deimos-ruby (1.7.0.pre.beta1)
+    deimos-ruby (1.8.1.pre.beta3)
       avro_turf (~> 0.11)
       phobos (~> 1.9)
       ruby-kafka (~> 0.7)
+      sigurd (= 0.0.1)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    actioncable (5.2.4.2)
-      actionpack (= 5.2.4.2)
+    actioncable (6.0.3.2)
+      actionpack (= 6.0.3.2)
       nio4r (~> 2.0)
       websocket-driver (>= 0.6.1)
-    actionmailer (5.2.4.2)
-      actionpack (= 5.2.4.2)
-      actionview (= 5.2.4.2)
-      activejob (= 5.2.4.2)
+    actionmailbox (6.0.3.2)
+      actionpack (= 6.0.3.2)
+      activejob (= 6.0.3.2)
+      activerecord (= 6.0.3.2)
+      activestorage (= 6.0.3.2)
+      activesupport (= 6.0.3.2)
+      mail (>= 2.7.1)
+    actionmailer (6.0.3.2)
+      actionpack (= 6.0.3.2)
+      actionview (= 6.0.3.2)
+      activejob (= 6.0.3.2)
       mail (~> 2.5, >= 2.5.4)
       rails-dom-testing (~> 2.0)
-    actionpack (5.2.4.2)
-      actionview (= 5.2.4.2)
-      activesupport (= 5.2.4.2)
+    actionpack (6.0.3.2)
+      actionview (= 6.0.3.2)
+      activesupport (= 6.0.3.2)
       rack (~> 2.0, >= 2.0.8)
       rack-test (>= 0.6.3)
       rails-dom-testing (~> 2.0)
-      rails-html-sanitizer (~> 1.0, >= 1.0.2)
-    actionview (5.2.4.2)
-      activesupport (= 5.2.4.2)
+      rails-html-sanitizer (~> 1.0, >= 1.2.0)
+    actiontext (6.0.3.2)
+      actionpack (= 6.0.3.2)
+      activerecord (= 6.0.3.2)
+      activestorage (= 6.0.3.2)
+      activesupport (= 6.0.3.2)
+      nokogiri (>= 1.8.5)
+    actionview (6.0.3.2)
+      activesupport (= 6.0.3.2)
       builder (~> 3.1)
       erubi (~> 1.4)
       rails-dom-testing (~> 2.0)
-      rails-html-sanitizer (~> 1.0, >= 1.0.3)
-    activejob (5.2.4.2)
-      activesupport (= 5.2.4.2)
+      rails-html-sanitizer (~> 1.1, >= 1.2.0)
+    activejob (6.0.3.2)
+      activesupport (= 6.0.3.2)
       globalid (>= 0.3.6)
-    activemodel (5.2.4.2)
-      activesupport (= 5.2.4.2)
-    activerecord (5.2.4.2)
-      activemodel (= 5.2.4.2)
-      activesupport (= 5.2.4.2)
-      arel (>= 9.0)
-    activerecord-import (1.0.4)
+    activemodel (6.0.3.2)
+      activesupport (= 6.0.3.2)
+    activerecord (6.0.3.2)
+      activemodel (= 6.0.3.2)
+      activesupport (= 6.0.3.2)
+    activerecord-import (1.0.5)
       activerecord (>= 3.2)
-    activestorage (5.2.4.2)
-      actionpack (= 5.2.4.2)
-      activerecord (= 5.2.4.2)
+    activestorage (6.0.3.2)
+      actionpack (= 6.0.3.2)
+      activejob (= 6.0.3.2)
+      activerecord (= 6.0.3.2)
       marcel (~> 0.3.1)
-    activesupport (5.2.4.2)
+    activesupport (6.0.3.2)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
-    arel (9.0.0)
-    ast (2.4.0)
+      zeitwerk (~> 2.2, >= 2.2.2)
+    ast (2.4.1)
     avro (1.9.2)
       multi_json
     avro_turf (0.11.0)
       avro (>= 1.7.7, < 1.10)
       excon (~> 0.45)
     builder (3.2.4)
-    coderay (1.1.2)
+    coderay (1.1.3)
     concurrent-ruby (1.1.6)
     concurrent-ruby-ext (1.1.6)
       concurrent-ruby (= 1.1.6)
     crass (1.0.6)
-    ddtrace (0.35.1)
+    database_cleaner (1.8.5)
+    ddtrace (0.37.0)
       msgpack
-    diff-lcs (1.3)
-    digest-crc (0.5.1)
-    dogstatsd-ruby (4.8.0)
+    diff-lcs (1.4.4)
+    digest-crc (0.6.1)
+      rake (~> 13.0)
+    dogstatsd-ruby (4.8.1)
     erubi (1.9.0)
-    excon (0.73.0)
+    excon (0.76.0)
     exponential-backoff (0.0.4)
-    ffi (1.12.2)
+    ffi (1.13.1)
     formatador (0.2.5)
     globalid (0.4.2)
       activesupport (>= 4.2.0)
@@ -94,20 +110,19 @@ GEM
     guard-rubocop (1.3.0)
       guard (~> 2.0)
       rubocop (~> 0.20)
-    i18n (1.8.2)
+    i18n (1.8.4)
       concurrent-ruby (~> 1.0)
-    jaro_winkler (1.5.4)
     listen (3.2.1)
       rb-fsevent (~> 0.10, >= 0.10.3)
       rb-inotify (~> 0.9, >= 0.9.10)
     little-plugger (1.1.4)
-    logging (2.2.2)
+    logging (2.3.0)
       little-plugger (~> 1.1)
-      multi_json (~> 1.10)
-    loofah (2.5.0)
+      multi_json (~> 1.14)
+    loofah (2.6.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
-    lumberjack (1.2.4)
+    lumberjack (1.2.6)
     mail (2.7.1)
       mini_mime (>= 0.1.1)
     marcel (0.3.3)
@@ -116,20 +131,20 @@ GEM
     mimemagic (0.3.5)
     mini_mime (1.0.2)
     mini_portile2 (2.4.0)
-    minitest (5.14.0)
+    minitest (5.14.1)
     msgpack (1.3.3)
-    multi_json (1.14.1)
+    multi_json (1.15.0)
     mysql2 (0.5.3)
     nenv (0.3.0)
     nio4r (2.5.2)
-    nokogiri (1.10.9)
+    nokogiri (1.10.10)
       mini_portile2 (~> 2.4.0)
     notiffany (0.1.3)
       nenv (~> 0.1)
       shellany (~> 0.0)
-    parallel (1.19.1)
-    parser (2.7.1.2)
-      ast (~> 2.4.0)
+    parallel (1.19.2)
+    parser (2.7.1.4)
+      ast (~> 2.4.1)
     pg (1.2.3)
     phobos (1.9.0)
       activesupport (>= 3.0.0)
@@ -142,38 +157,41 @@ GEM
     pry (0.13.1)
       coderay (~> 1.1)
       method_source (~> 1.0)
-    rack (2.2.2)
+    rack (2.2.3)
     rack-test (1.1.0)
       rack (>= 1.0, < 3)
-    rails (5.2.4.2)
-      actioncable (= 5.2.4.2)
-      actionmailer (= 5.2.4.2)
-      actionpack (= 5.2.4.2)
-      actionview (= 5.2.4.2)
-      activejob (= 5.2.4.2)
-      activemodel (= 5.2.4.2)
-      activerecord (= 5.2.4.2)
-      activestorage (= 5.2.4.2)
-      activesupport (= 5.2.4.2)
+    rails (6.0.3.2)
+      actioncable (= 6.0.3.2)
+      actionmailbox (= 6.0.3.2)
+      actionmailer (= 6.0.3.2)
+      actionpack (= 6.0.3.2)
+      actiontext (= 6.0.3.2)
+      actionview (= 6.0.3.2)
+      activejob (= 6.0.3.2)
+      activemodel (= 6.0.3.2)
+      activerecord (= 6.0.3.2)
+      activestorage (= 6.0.3.2)
+      activesupport (= 6.0.3.2)
       bundler (>= 1.3.0)
-      railties (= 5.2.4.2)
+      railties (= 6.0.3.2)
       sprockets-rails (>= 2.0.0)
     rails-dom-testing (2.0.3)
       activesupport (>= 4.2.0)
       nokogiri (>= 1.6)
     rails-html-sanitizer (1.3.0)
       loofah (~> 2.3)
-    railties (5.2.4.2)
-      actionpack (= 5.2.4.2)
-      activesupport (= 5.2.4.2)
+    railties (6.0.3.2)
+      actionpack (= 6.0.3.2)
+      activesupport (= 6.0.3.2)
       method_source
       rake (>= 0.8.7)
-      thor (>= 0.19.0, < 2.0)
+      thor (>= 0.20.3, < 2.0)
     rainbow (3.0.0)
     rake (13.0.1)
     rb-fsevent (0.10.4)
     rb-inotify (0.10.1)
       ffi (~> 1.0)
+    regexp_parser (1.7.1)
     rexml (3.2.4)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
@@ -181,30 +199,44 @@ GEM
       rspec-mocks (~> 3.9.0)
     rspec-core (3.9.2)
       rspec-support (~> 3.9.3)
-    rspec-expectations (3.9.1)
+    rspec-expectations (3.9.2)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.9.0)
     rspec-mocks (3.9.1)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.9.0)
+    rspec-rails (4.0.1)
+      actionpack (>= 4.2)
+      activesupport (>= 4.2)
+      railties (>= 4.2)
+      rspec-core (~> 3.9)
+      rspec-expectations (~> 3.9)
+      rspec-mocks (~> 3.9)
+      rspec-support (~> 3.9)
     rspec-support (3.9.3)
     rspec_junit_formatter (0.4.1)
       rspec-core (>= 2, < 4, != 2.12.0)
-    rubocop (0.82.0)
-      jaro_winkler (~> 1.5.1)
+    rubocop (0.88.0)
       parallel (~> 1.10)
-      parser (>= 2.7.0.1)
+      parser (>= 2.7.1.1)
       rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.7)
       rexml
+      rubocop-ast (>= 0.1.0, < 1.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 2.0)
-    rubocop-rspec (1.39.0)
-      rubocop (>= 0.68.1)
+    rubocop-ast (0.2.0)
+      parser (>= 2.7.0.1)
+    rubocop-rspec (1.42.0)
+      rubocop (>= 0.87.0)
     ruby-kafka (0.7.10)
       digest-crc
     ruby-progressbar (1.10.1)
     shellany (0.0.1)
-    sprockets (4.0.0)
+    sigurd (0.0.1)
+      concurrent-ruby (~> 1)
+      exponential-backoff
+    sprockets (4.0.2)
       concurrent-ruby (~> 1.0)
       rack (> 1, < 3)
     sprockets-rails (3.2.1)
@@ -217,17 +249,18 @@ GEM
     tzinfo (1.2.7)
       thread_safe (~> 0.1)
     unicode-display_width (1.7.0)
-    websocket-driver (0.7.1)
+    websocket-driver (0.7.3)
       websocket-extensions (>= 0.1.0)
-    websocket-extensions (0.1.4)
+    websocket-extensions (0.1.5)
+    zeitwerk (2.4.0)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  activerecord (~> 5.2)
   activerecord-import
   avro (~> 1.9)
+  database_cleaner (~> 1.7)
   ddtrace (~> 0.11)
   deimos-ruby!
   dogstatsd-ruby (~> 4.2)
@@ -236,9 +269,10 @@ DEPENDENCIES
   guard-rubocop (~> 1)
   mysql2 (~> 0.5)
   pg (~> 1.1)
-  rails (~> 5.2, >= 5.2.4.2)
+  rails (~> 6)
   rake (~> 13)
   rspec (~> 3)
+  rspec-rails (~> 4)
   rspec_junit_formatter (~> 0.3)
   rubocop (~> 0.72)
   rubocop-rspec (~> 1.27)

data/README.md

@@ -22,6 +22,7 @@ Built on Phobos and hence Ruby-Kafka.
         * [Kafka Message Keys](#kafka-message-keys)
    * [Consumers](#consumers)
    * [Rails Integration](#rails-integration)
+        * [Controller Mixin](#controller-mixin)
    * [Database Backend](#database-backend)
    * [Database Poller](#database-poller)
    * [Running Consumers](#running-consumers)
@@ -314,28 +315,19 @@ messages as an array and then process them together. This can improve
 consumer throughput, depending on the use case. Batch consumers behave like
 other consumers in regards to key and payload decoding, etc.
 
-To enable batch consumption, ensure that the `delivery` property is set to `inline_batch`. For example:
+To enable batch consumption, ensure that the `delivery` property of your
+consumer is set to `inline_batch`.
 
-```ruby
-Deimos.configure do
-  consumer do
-    class_name 'Consumers::MyBatchConsumer'
-    topic 'my_batched_topic'
-    group_id 'my_group_id'
-    delivery :inline_batch
-  end
-end
-```
-
-Batch consumers must inherit from the Deimos::BatchConsumer class as in
-this sample:
+Batch consumers will invoke the `consume_batch` method instead of `consume`
+as in this example:
 
 ```ruby
-class MyBatchConsumer < Deimos::BatchConsumer
+class MyBatchConsumer < Deimos::Consumer
 
   def consume_batch(payloads, metadata)
     # payloads is an array of schema-decoded hashes.
-    # metadata is a hash that contains information like :keys and :topic.
+    # metadata is a hash that contains information like :keys, :topic, 
+    # and :first_offset.
     # Keys are automatically decoded and available as an array with
     # the same cardinality as the payloads. If you need to iterate
     # over payloads and keys together, you can use something like this:
@@ -456,6 +448,58 @@ class Widget < ActiveRecord::Base
 end
 ```
 
+### Controller Mixin
+
+Deimos comes with a mixin for `ActionController` which automatically encodes and decodes schema
+payloads. There are some advantages to encoding your data in e.g. Avro rather than straight JSON,
+particularly if your service is talking to another backend service rather than the front-end
+browser:
+
+* It enforces a contract between services. Solutions like [OpenAPI](https://swagger.io/specification/) 
+  do this as well, but in order for the client to know the contract, usually some kind of code 
+  generation has to happen. Using schemas ensures both sides know the contract without having to change code.
+  In addition, OpenAPI is now a huge and confusing format, and using simpler schema formats
+  can be beneficial.
+* Using Avro or Protobuf ensures both forwards and backwards compatibility,
+  which reduces the need for versioning since both sides can simply ignore fields they aren't aware
+  of.
+* Encoding and decoding using Avro or Protobuf is generally faster than straight JSON, and
+  results in smaller payloads and therefore less network traffic.
+
+To use the mixin, add the following to your `WhateverController`:
+
+```ruby
+class WhateverController < ApplicationController
+  include Deimos::Utils::SchemaControllerMixin
+
+  request_namespace 'my.namespace.requests'
+  response_namespace 'my.namespace.responses'
+  
+  # Add a "schemas" line for all routes that should encode/decode schemas.
+  # Default is to match the schema name to the route name.
+  schemas :index
+  # will look for: my.namespace.requests.Index.avsc
+  #                my.namespace.responses.Index.avsc 
+  
+  # If all routes use the default, you can add them all at once
+  schemas :index, :show, :update
+
+  # Different schemas can be specified as well
+  schemas :index, :show, request: 'IndexRequest', response: 'IndexResponse'
+
+  # To access the encoded data, use the `payload` helper method, and to render it back,
+  # use the `render_schema` method.
+  
+  def index
+    response = { 'response_id' => payload['request_id'] + 'hi mom' }
+    render_schema(response)
+  end
+end
+```
+
+To make use of this feature, your requests and responses need to have the correct content type.
+For Avro content, this is the `avro/binary` content type.
+
 # Database Backend
 
 Deimos provides a way to allow Kafka messages to be created inside a
@@ -533,12 +577,14 @@ class MyConsumer < Deimos::ActiveRecordConsumer
 
   # Optional override of the way to fetch records based on payload and
   # key. Default is to use the key to search the primary key of the table.
+  # Only used in non-batch mode.
   def fetch_record(klass, payload, key)
     super
   end
 
   # Optional override on how to set primary key for new records. 
   # Default is to set the class's primary key to the message's decoded key. 
+  # Only used in non-batch mode.
   def assign_key(record, payload, key)
     super
   end
@@ -546,6 +592,7 @@ class MyConsumer < Deimos::ActiveRecordConsumer
   # Optional override of the default behavior, which is to call `destroy`
   # on the record - e.g. you can replace this with "archiving" the record
   # in some way. 
+  # Only used in non-batch mode.
   def destroy_record(record)
     super
   end
@@ -555,6 +602,87 @@ class MyConsumer < Deimos::ActiveRecordConsumer
   def record_attributes(payload)
     super.merge(:some_field => 'some_value')
   end
+
+  # Optional override to change the attributes used for identifying records
+  def record_key(payload)
+    super
+  end
+end
+```
+
+#### Generating Tables and Models
+
+Deimos provides a generator that takes an existing schema and generates a 
+database table based on its fields. By default, any complex sub-types (such as 
+records or arrays) are turned into JSON (if supported) or string columns.
+
+Before running this migration, you must first copy the schema into your repo
+in the correct path (in the example above, you would need to have a file
+`{SCHEMA_ROOT}/com/my-namespace/MySchema.avsc`).
+
+To generate a model and migration, run the following:
+
+    rails g deimos:active_record TABLE_NAME FULL_SCHEMA_NAME
+    
+Example:
+
+    rails g deimos:active_record my_table com.my-namespace.MySchema
+    
+...would generate:
+
+    db/migrate/1234_create_my_table.rb
+    app/models/my_table.rb
+
+#### Batch Consumers
+
+Deimos also provides a batch consumption mode for `ActiveRecordConsumer` which
+processes groups of messages at once using the ActiveRecord backend. 
+
+Batch ActiveRecord consumers make use of the
+[activerecord-import](https://github.com/zdennis/activerecord-import) to insert
+or update multiple records in bulk SQL statements. This reduces processing
+time at the cost of skipping ActiveRecord callbacks for individual records.
+Deleted records (tombstones) are grouped into `delete_all` calls and thus also
+skip `destroy` callbacks.
+
+Batch consumption is used when the `delivery` setting for your consumer is set to `inline_batch`.
+
+**Note**: Currently, batch consumption only supports only primary keys as identifiers out of the box. See
+[the specs](spec/active_record_batch_consumer_spec.rb) for an example of how to use compound keys.
+
+By default, batches will be compacted before processing, i.e. only the last
+message for each unique key in a batch will actually be processed. To change
+this behaviour, call `compacted false` inside of your consumer definition.
+
+A sample batch consumer would look as follows:
+
+```ruby
+class MyConsumer < Deimos::ActiveRecordConsumer
+  schema 'MySchema'
+  key_config field: 'my_field'
+  record_class Widget
+
+  # Controls whether the batch is compacted before consuming.
+  # If true, only the last message for each unique key in a batch will be
+  # processed.
+  # If false, messages will be grouped into "slices" of independent keys
+  # and each slice will be imported separately.
+  #
+  # compacted false
+
+
+  # Optional override of the default behavior, which is to call `delete_all`
+  # on the associated records - e.g. you can replace this with setting a deleted
+  # flag on the record. 
+  def remove_records(records)
+    super
+  end
+ 
+  # Optional override to change the attributes of the record before they
+  # are saved.
+  def record_attributes(payload)
+    super.merge(:some_field => 'some_value')
+  end
 end
 ```
 
@@ -853,6 +981,9 @@ Deimos::Utils::InlineConsumer.get_messages_for(
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/flipp-oss/deimos .
 
+We have more information on the [internal architecture](docs/ARCHITECTURE.md) of Deimos
+for contributors!
+
 ### Linting
 
 Deimos uses Rubocop to lint the code. Please run Rubocop on your code