deimos-ruby 1.8.0.pre.beta1 → 1.8.1.pre.beta4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 82c1b89b49622cb0d47a7a7aa56e076d908785906581f9f0772f09679acd6895
-  data.tar.gz: eeadf1d1a63db1407a81ef1f993be1b8d421e8e2624b9f42c324a2dad776bd1c
+  metadata.gz: bb7b72866d42e8a74dfe9e441b5e1436c81cb16684d2985840980c010416e8af
+  data.tar.gz: 97b806b1b52807ea487a0da20099450d005bf71ac4cd88fbc5eb50aae66726bb
 SHA512:
-  metadata.gz: 6a85cd1d407c11d4becb6d6d11636623b96c706325f8bbca52caf4bf0cdce079df65432cb076d01503a4d4d977c8a662c22622e27193aeafc87ffbdc8d934045
-  data.tar.gz: '08f9b75f3057071f41effaf870f8d0927dca85d647726ba833371001c470a7a062ee74ac63b5acc2f284493d3a32373ffb6c3ca96f9428ffb762c83cfd804c42'
+  metadata.gz: 0d98923fd57076e391a9e8e1c2a6afb83fbde7769d646fc88d599921d9b0d55a6b1c7bd0df7f3344c71c66915766484bd0c9c19cfa72fef7f9299ab41cf66232
+  data.tar.gz: c537564c70e3d297ca76000cd683f6c0891c18dc3b54d95b9aa6f3ea4b390a59fd2992d070112afc1e562c0bc59e0a4b523cb7bd4d71efafe2b03083df5273a6

data/.rubocop.yml

@@ -5,6 +5,7 @@ AllCops:
   Exclude:
     - lib/deimos/monkey_patches/*.rb
     - vendor/**/*
+    - Guardfile
   NewCops: enable
 
 # class Plumbus
@@ -36,7 +37,7 @@ Layout/EmptyLinesAroundBlockBody:
   Enabled: false
 
 Layout/LineLength:
-  Max: 100
+  Max: 120
   Severity: refactor
   Exclude:
   - 'spec/**/*'
@@ -77,13 +78,14 @@ Lint/UnusedMethodArgument:
 
 Metrics/AbcSize:
   Severity: refactor
-  Max: 20
+  Max: 25
 
 Metrics/BlockLength:
-  Severity: refactor
+  Enabled: false
 
 Metrics/ClassLength:
   Severity: refactor
+  Max: 500
 
 Metrics/CyclomaticComplexity:
   Severity: refactor
@@ -91,10 +93,11 @@ Metrics/CyclomaticComplexity:
 
 Metrics/MethodLength:
   Severity: refactor
-  Max: 30
+  Max: 50
 
 Metrics/ModuleLength:
   Severity: refactor
+  Max: 200
 
 Metrics/ParameterLists:
   Max: 5
@@ -102,6 +105,7 @@ Metrics/ParameterLists:
 
 Metrics/PerceivedComplexity:
   Severity: refactor
+  Max: 10
 
 # Use alias_method instead of alias
 Style/Alias:

data/CHANGELOG.md

@@ -7,11 +7,53 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
+## 1.8.1-beta4 - 2020-08-12
+
+### Fixes :wrench:
+- Fix regression bug where arrays were not being encoded
+
+## 1.8.1-beta3 - 2020-08-05
+
+### Fixes :wrench:
+- Simplify decoding messages and handle producer not found
+- Consolidate types in sub-records recursively
+  (fixes [#72](https://github.com/flipp-oss/deimos/issues/72))
+
+## 1.8.1-beta2 - 2020-07-28
+
+### Features :star:
+- Add `SchemaControllerMixin` to encode and decode schema-encoded
+  payloads in Rails controllers.
+
+## 1.8.1-beta1 - 2020-07-22
+
+### Fixes :wrench:
+- Retry deleting messages without resending the batch if the
+  delete fails (fixes [#34](https://github.com/flipp-oss/deimos/issues/34))
+- Delete messages in batches rather than all at once to
+  cut down on the chance of a deadlock.
+
+### Features :star:
+- Add `last_processed_at` to `kafka_topic_info` to ensure
+  that wait metrics are accurate in cases where records
+  get created with an old `created_at` time (e.g. for
+  long-running transactions).
+- Add generator for ActiveRecord models and migrations (fixes [#6](https://github.com/flipp-oss/deimos/issues/6))
+
+## 1.8.0-beta2 - 2020-07-08
+
+### Fixes :wrench:
+- Fix crash with batch consumption due to not having ActiveSupport::Concern
+
+### Features :star:
+- Add `first_offset` to the metadata sent via the batch
+
 ## 1.8.0-beta1 - 2020-07-06
 ### Features :star:
 - Added `ActiveRecordConsumer` batch mode
 
 ### Fixes :wrench:
+- Fixes `send_produce_error` to decode `failed_messages` with built-in decoder. 
 - Lag calculation can be incorrect if no messages are being consumed.
 - Fixed bug where printing messages on a MessageSizeTooLarge
   error didn't work.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    deimos-ruby (1.7.0.pre.beta1)
+    deimos-ruby (1.8.1.pre.beta3)
       avro_turf (~> 0.11)
       phobos (~> 1.9)
       ruby-kafka (~> 0.7)
@@ -10,72 +10,86 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    actioncable (5.2.4.2)
-      actionpack (= 5.2.4.2)
+    actioncable (6.0.3.2)
+      actionpack (= 6.0.3.2)
       nio4r (~> 2.0)
       websocket-driver (>= 0.6.1)
-    actionmailer (5.2.4.2)
-      actionpack (= 5.2.4.2)
-      actionview (= 5.2.4.2)
-      activejob (= 5.2.4.2)
+    actionmailbox (6.0.3.2)
+      actionpack (= 6.0.3.2)
+      activejob (= 6.0.3.2)
+      activerecord (= 6.0.3.2)
+      activestorage (= 6.0.3.2)
+      activesupport (= 6.0.3.2)
+      mail (>= 2.7.1)
+    actionmailer (6.0.3.2)
+      actionpack (= 6.0.3.2)
+      actionview (= 6.0.3.2)
+      activejob (= 6.0.3.2)
       mail (~> 2.5, >= 2.5.4)
       rails-dom-testing (~> 2.0)
-    actionpack (5.2.4.2)
-      actionview (= 5.2.4.2)
-      activesupport (= 5.2.4.2)
+    actionpack (6.0.3.2)
+      actionview (= 6.0.3.2)
+      activesupport (= 6.0.3.2)
       rack (~> 2.0, >= 2.0.8)
       rack-test (>= 0.6.3)
       rails-dom-testing (~> 2.0)
-      rails-html-sanitizer (~> 1.0, >= 1.0.2)
-    actionview (5.2.4.2)
-      activesupport (= 5.2.4.2)
+      rails-html-sanitizer (~> 1.0, >= 1.2.0)
+    actiontext (6.0.3.2)
+      actionpack (= 6.0.3.2)
+      activerecord (= 6.0.3.2)
+      activestorage (= 6.0.3.2)
+      activesupport (= 6.0.3.2)
+      nokogiri (>= 1.8.5)
+    actionview (6.0.3.2)
+      activesupport (= 6.0.3.2)
       builder (~> 3.1)
       erubi (~> 1.4)
       rails-dom-testing (~> 2.0)
-      rails-html-sanitizer (~> 1.0, >= 1.0.3)
-    activejob (5.2.4.2)
-      activesupport (= 5.2.4.2)
+      rails-html-sanitizer (~> 1.1, >= 1.2.0)
+    activejob (6.0.3.2)
+      activesupport (= 6.0.3.2)
       globalid (>= 0.3.6)
-    activemodel (5.2.4.2)
-      activesupport (= 5.2.4.2)
-    activerecord (5.2.4.2)
-      activemodel (= 5.2.4.2)
-      activesupport (= 5.2.4.2)
-      arel (>= 9.0)
-    activerecord-import (1.0.4)
+    activemodel (6.0.3.2)
+      activesupport (= 6.0.3.2)
+    activerecord (6.0.3.2)
+      activemodel (= 6.0.3.2)
+      activesupport (= 6.0.3.2)
+    activerecord-import (1.0.5)
       activerecord (>= 3.2)
-    activestorage (5.2.4.2)
-      actionpack (= 5.2.4.2)
-      activerecord (= 5.2.4.2)
+    activestorage (6.0.3.2)
+      actionpack (= 6.0.3.2)
+      activejob (= 6.0.3.2)
+      activerecord (= 6.0.3.2)
       marcel (~> 0.3.1)
-    activesupport (5.2.4.2)
+    activesupport (6.0.3.2)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
-    arel (9.0.0)
-    ast (2.4.0)
+      zeitwerk (~> 2.2, >= 2.2.2)
+    ast (2.4.1)
     avro (1.9.2)
       multi_json
     avro_turf (0.11.0)
       avro (>= 1.7.7, < 1.10)
       excon (~> 0.45)
     builder (3.2.4)
-    coderay (1.1.2)
+    coderay (1.1.3)
     concurrent-ruby (1.1.6)
     concurrent-ruby-ext (1.1.6)
       concurrent-ruby (= 1.1.6)
     crass (1.0.6)
     database_cleaner (1.8.5)
-    ddtrace (0.35.1)
+    ddtrace (0.37.0)
       msgpack
-    diff-lcs (1.3)
-    digest-crc (0.5.1)
-    dogstatsd-ruby (4.8.0)
+    diff-lcs (1.4.4)
+    digest-crc (0.6.1)
+      rake (~> 13.0)
+    dogstatsd-ruby (4.8.1)
     erubi (1.9.0)
-    excon (0.73.0)
+    excon (0.76.0)
     exponential-backoff (0.0.4)
-    ffi (1.12.2)
+    ffi (1.13.1)
     formatador (0.2.5)
     globalid (0.4.2)
       activesupport (>= 4.2.0)
@@ -96,20 +110,19 @@ GEM
     guard-rubocop (1.3.0)
       guard (~> 2.0)
       rubocop (~> 0.20)
-    i18n (1.8.2)
+    i18n (1.8.4)
       concurrent-ruby (~> 1.0)
-    jaro_winkler (1.5.4)
     listen (3.2.1)
       rb-fsevent (~> 0.10, >= 0.10.3)
       rb-inotify (~> 0.9, >= 0.9.10)
     little-plugger (1.1.4)
-    logging (2.2.2)
+    logging (2.3.0)
       little-plugger (~> 1.1)
-      multi_json (~> 1.10)
-    loofah (2.5.0)
+      multi_json (~> 1.14)
+    loofah (2.6.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
-    lumberjack (1.2.4)
+    lumberjack (1.2.6)
     mail (2.7.1)
       mini_mime (>= 0.1.1)
     marcel (0.3.3)
@@ -118,20 +131,20 @@ GEM
     mimemagic (0.3.5)
     mini_mime (1.0.2)
     mini_portile2 (2.4.0)
-    minitest (5.14.0)
+    minitest (5.14.1)
     msgpack (1.3.3)
-    multi_json (1.14.1)
+    multi_json (1.15.0)
     mysql2 (0.5.3)
     nenv (0.3.0)
     nio4r (2.5.2)
-    nokogiri (1.10.9)
+    nokogiri (1.10.10)
       mini_portile2 (~> 2.4.0)
     notiffany (0.1.3)
       nenv (~> 0.1)
       shellany (~> 0.0)
-    parallel (1.19.1)
-    parser (2.7.1.2)
-      ast (~> 2.4.0)
+    parallel (1.19.2)
+    parser (2.7.1.4)
+      ast (~> 2.4.1)
     pg (1.2.3)
     phobos (1.9.0)
       activesupport (>= 3.0.0)
@@ -147,35 +160,38 @@ GEM
     rack (2.2.3)
     rack-test (1.1.0)
       rack (>= 1.0, < 3)
-    rails (5.2.4.2)
-      actioncable (= 5.2.4.2)
-      actionmailer (= 5.2.4.2)
-      actionpack (= 5.2.4.2)
-      actionview (= 5.2.4.2)
-      activejob (= 5.2.4.2)
-      activemodel (= 5.2.4.2)
-      activerecord (= 5.2.4.2)
-      activestorage (= 5.2.4.2)
-      activesupport (= 5.2.4.2)
+    rails (6.0.3.2)
+      actioncable (= 6.0.3.2)
+      actionmailbox (= 6.0.3.2)
+      actionmailer (= 6.0.3.2)
+      actionpack (= 6.0.3.2)
+      actiontext (= 6.0.3.2)
+      actionview (= 6.0.3.2)
+      activejob (= 6.0.3.2)
+      activemodel (= 6.0.3.2)
+      activerecord (= 6.0.3.2)
+      activestorage (= 6.0.3.2)
+      activesupport (= 6.0.3.2)
       bundler (>= 1.3.0)
-      railties (= 5.2.4.2)
+      railties (= 6.0.3.2)
       sprockets-rails (>= 2.0.0)
     rails-dom-testing (2.0.3)
       activesupport (>= 4.2.0)
       nokogiri (>= 1.6)
     rails-html-sanitizer (1.3.0)
       loofah (~> 2.3)
-    railties (5.2.4.2)
-      actionpack (= 5.2.4.2)
-      activesupport (= 5.2.4.2)
+    railties (6.0.3.2)
+      actionpack (= 6.0.3.2)
+      activesupport (= 6.0.3.2)
       method_source
       rake (>= 0.8.7)
-      thor (>= 0.19.0, < 2.0)
+      thor (>= 0.20.3, < 2.0)
     rainbow (3.0.0)
     rake (13.0.1)
     rb-fsevent (0.10.4)
     rb-inotify (0.10.1)
       ffi (~> 1.0)
+    regexp_parser (1.7.1)
     rexml (3.2.4)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
@@ -183,25 +199,36 @@ GEM
       rspec-mocks (~> 3.9.0)
     rspec-core (3.9.2)
       rspec-support (~> 3.9.3)
-    rspec-expectations (3.9.1)
+    rspec-expectations (3.9.2)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.9.0)
     rspec-mocks (3.9.1)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.9.0)
+    rspec-rails (4.0.1)
+      actionpack (>= 4.2)
+      activesupport (>= 4.2)
+      railties (>= 4.2)
+      rspec-core (~> 3.9)
+      rspec-expectations (~> 3.9)
+      rspec-mocks (~> 3.9)
+      rspec-support (~> 3.9)
     rspec-support (3.9.3)
     rspec_junit_formatter (0.4.1)
       rspec-core (>= 2, < 4, != 2.12.0)
-    rubocop (0.82.0)
-      jaro_winkler (~> 1.5.1)
+    rubocop (0.88.0)
       parallel (~> 1.10)
-      parser (>= 2.7.0.1)
+      parser (>= 2.7.1.1)
       rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.7)
       rexml
+      rubocop-ast (>= 0.1.0, < 1.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 2.0)
-    rubocop-rspec (1.39.0)
-      rubocop (>= 0.68.1)
+    rubocop-ast (0.2.0)
+      parser (>= 2.7.0.1)
+    rubocop-rspec (1.42.0)
+      rubocop (>= 0.87.0)
     ruby-kafka (0.7.10)
       digest-crc
     ruby-progressbar (1.10.1)
@@ -209,7 +236,7 @@ GEM
     sigurd (0.0.1)
       concurrent-ruby (~> 1)
       exponential-backoff
-    sprockets (4.0.0)
+    sprockets (4.0.2)
       concurrent-ruby (~> 1.0)
       rack (> 1, < 3)
     sprockets-rails (3.2.1)
@@ -222,15 +249,15 @@ GEM
     tzinfo (1.2.7)
       thread_safe (~> 0.1)
     unicode-display_width (1.7.0)
-    websocket-driver (0.7.1)
+    websocket-driver (0.7.3)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.5)
+    zeitwerk (2.4.0)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  activerecord (~> 5.2)
   activerecord-import
   avro (~> 1.9)
   database_cleaner (~> 1.7)
@@ -242,9 +269,10 @@ DEPENDENCIES
   guard-rubocop (~> 1)
   mysql2 (~> 0.5)
   pg (~> 1.1)
-  rails (~> 5.2, >= 5.2.4.2)
+  rails (~> 6)
   rake (~> 13)
   rspec (~> 3)
+  rspec-rails (~> 4)
   rspec_junit_formatter (~> 0.3)
   rubocop (~> 0.72)
   rubocop-rspec (~> 1.27)

data/README.md

@@ -22,6 +22,7 @@ Built on Phobos and hence Ruby-Kafka.
         * [Kafka Message Keys](#kafka-message-keys)
    * [Consumers](#consumers)
    * [Rails Integration](#rails-integration)
+        * [Controller Mixin](#controller-mixin)
    * [Database Backend](#database-backend)
    * [Database Poller](#database-poller)
    * [Running Consumers](#running-consumers)
@@ -325,7 +326,8 @@ class MyBatchConsumer < Deimos::Consumer
 
   def consume_batch(payloads, metadata)
     # payloads is an array of schema-decoded hashes.
-    # metadata is a hash that contains information like :keys and :topic.
+    # metadata is a hash that contains information like :keys, :topic, 
+    # and :first_offset.
     # Keys are automatically decoded and available as an array with
     # the same cardinality as the payloads. If you need to iterate
     # over payloads and keys together, you can use something like this:
@@ -446,6 +448,58 @@ class Widget < ActiveRecord::Base
 end
 ```
 
+### Controller Mixin
+
+Deimos comes with a mixin for `ActionController` which automatically encodes and decodes schema
+payloads. There are some advantages to encoding your data in e.g. Avro rather than straight JSON,
+particularly if your service is talking to another backend service rather than the front-end
+browser:
+
+* It enforces a contract between services. Solutions like [OpenAPI](https://swagger.io/specification/) 
+  do this as well, but in order for the client to know the contract, usually some kind of code 
+  generation has to happen. Using schemas ensures both sides know the contract without having to change code.
+  In addition, OpenAPI is now a huge and confusing format, and using simpler schema formats
+  can be beneficial.
+* Using Avro or Protobuf ensures both forwards and backwards compatibility,
+  which reduces the need for versioning since both sides can simply ignore fields they aren't aware
+  of.
+* Encoding and decoding using Avro or Protobuf is generally faster than straight JSON, and
+  results in smaller payloads and therefore less network traffic.
+
+To use the mixin, add the following to your `WhateverController`:
+
+```ruby
+class WhateverController < ApplicationController
+  include Deimos::Utils::SchemaControllerMixin
+
+  request_namespace 'my.namespace.requests'
+  response_namespace 'my.namespace.responses'
+  
+  # Add a "schemas" line for all routes that should encode/decode schemas.
+  # Default is to match the schema name to the route name.
+  schemas :index
+  # will look for: my.namespace.requests.Index.avsc
+  #                my.namespace.responses.Index.avsc 
+  
+  # If all routes use the default, you can add them all at once
+  schemas :index, :show, :update
+
+  # Different schemas can be specified as well
+  schemas :index, :show, request: 'IndexRequest', response: 'IndexResponse'
+
+  # To access the encoded data, use the `payload` helper method, and to render it back,
+  # use the `render_schema` method.
+  
+  def index
+    response = { 'response_id' => payload['request_id'] + 'hi mom' }
+    render_schema(response)
+  end
+end
+```
+
+To make use of this feature, your requests and responses need to have the correct content type.
+For Avro content, this is the `avro/binary` content type.
+
 # Database Backend
 
 Deimos provides a way to allow Kafka messages to be created inside a
@@ -556,6 +610,29 @@ class MyConsumer < Deimos::ActiveRecordConsumer
 end
 ```
 
+#### Generating Tables and Models
+
+Deimos provides a generator that takes an existing schema and generates a 
+database table based on its fields. By default, any complex sub-types (such as 
+records or arrays) are turned into JSON (if supported) or string columns.
+
+Before running this migration, you must first copy the schema into your repo
+in the correct path (in the example above, you would need to have a file
+`{SCHEMA_ROOT}/com/my-namespace/MySchema.avsc`).
+
+To generate a model and migration, run the following:
+
+    rails g deimos:active_record TABLE_NAME FULL_SCHEMA_NAME
+    
+Example:
+
+    rails g deimos:active_record my_table com.my-namespace.MySchema
+    
+...would generate:
+
+    db/migrate/1234_create_my_table.rb
+    app/models/my_table.rb
+
 #### Batch Consumers
 
 Deimos also provides a batch consumption mode for `ActiveRecordConsumer` which