deimos-ruby 1.8.0.pre.beta2 → 1.8.1.pre.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5c184b8f182ecfa304684f4e8e522e93c43c12fe50b34300326c2e1ef206b6d1
-  data.tar.gz: 5861974e036c92470a4a801bed477e5cebefa679cc9996a5d6f0258eaa484150
+  metadata.gz: 335328a0ca5fb9147405485a66944ebe0d05af552ec09ec9605d76a7e7da7e9a
+  data.tar.gz: 438a9a91bda52be144d63867a4ae42dd1901cc7472f61232044aa934efebb464
 SHA512:
-  metadata.gz: f19a799d333485b6461e2d8cdeaec5643ec6ddf8f6a2ce1725f3c39269cc193cc5b1bc1e81266b6a33af87ec5d09b3ac2f857b48da61c3baded00a98d6935624
-  data.tar.gz: a08f6519a635e76944b3b7c2f08d5369b6f327e60e968391fcd5e09e703e8ed526d35d8e124f91cbff8c885fbf8d620f90e0a99b48e4289868a0c11925b43d51
+  metadata.gz: 3e01c5c9db93cb6e0f56dafa2f1b301c30b9cc11c397ca35f77b80cfe62bd9600eacb8774516ee1c81337193687944a8c7bec3cba400df6ea8b1c33af7e82c27
+  data.tar.gz: 9de8cea35db671c43123d096653a6423ccfcaad8a2b8e9478f787e879bd7fd50a626604b8626f79b5f8654c84130fded58385d6f75fe44e59e1bfb5a284e1096

data/.rubocop.yml

@@ -5,6 +5,7 @@ AllCops:
   Exclude:
     - lib/deimos/monkey_patches/*.rb
     - vendor/**/*
+    - Guardfile
   NewCops: enable
 
 # class Plumbus
@@ -36,7 +37,7 @@ Layout/EmptyLinesAroundBlockBody:
   Enabled: false
 
 Layout/LineLength:
-  Max: 100
+  Max: 120
   Severity: refactor
   Exclude:
   - 'spec/**/*'
@@ -77,13 +78,14 @@ Lint/UnusedMethodArgument:
 
 Metrics/AbcSize:
   Severity: refactor
-  Max: 20
+  Max: 25
 
 Metrics/BlockLength:
-  Severity: refactor
+  Enabled: false
 
 Metrics/ClassLength:
   Severity: refactor
+  Max: 500
 
 Metrics/CyclomaticComplexity:
   Severity: refactor
@@ -91,10 +93,11 @@ Metrics/CyclomaticComplexity:
 
 Metrics/MethodLength:
   Severity: refactor
-  Max: 30
+  Max: 50
 
 Metrics/ModuleLength:
   Severity: refactor
+  Max: 200
 
 Metrics/ParameterLists:
   Max: 5
@@ -102,6 +105,7 @@ Metrics/ParameterLists:
 
 Metrics/PerceivedComplexity:
   Severity: refactor
+  Max: 10
 
 # Use alias_method instead of alias
 Style/Alias:

data/CHANGELOG.md

@@ -7,6 +7,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
+## 1.8.1-beta1 - 2020-07-22
+
+### Fixes :wrench:
+- Retry deleting messages without resending the batch if the
+  delete fails (fixes [#34](https://github.com/flipp-oss/deimos/issues/34))
+- Delete messages in batches rather than all at once to
+  cut down on the chance of a deadlock.
+
+### Features :star:
+- Add `last_processed_at` to `kafka_topic_info` to ensure
+  that wait metrics are accurate in cases where records
+  get created with an old `created_at` time (e.g. for
+  long-running transactions).
+- Add generator for ActiveRecord models and migrations (fixes [#6](https://github.com/flipp-oss/deimos/issues/6))
+
 ## 1.8.0-beta2 - 2020-07-08
 
 ### Fixes :wrench:

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    deimos-ruby (1.8.0.pre.beta1)
+    deimos-ruby (1.8.1.pre.beta1)
       avro_turf (~> 0.11)
       phobos (~> 1.9)
       ruby-kafka (~> 0.7)
@@ -10,72 +10,86 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    actioncable (5.2.4.2)
-      actionpack (= 5.2.4.2)
+    actioncable (6.0.3.2)
+      actionpack (= 6.0.3.2)
       nio4r (~> 2.0)
       websocket-driver (>= 0.6.1)
-    actionmailer (5.2.4.2)
-      actionpack (= 5.2.4.2)
-      actionview (= 5.2.4.2)
-      activejob (= 5.2.4.2)
+    actionmailbox (6.0.3.2)
+      actionpack (= 6.0.3.2)
+      activejob (= 6.0.3.2)
+      activerecord (= 6.0.3.2)
+      activestorage (= 6.0.3.2)
+      activesupport (= 6.0.3.2)
+      mail (>= 2.7.1)
+    actionmailer (6.0.3.2)
+      actionpack (= 6.0.3.2)
+      actionview (= 6.0.3.2)
+      activejob (= 6.0.3.2)
       mail (~> 2.5, >= 2.5.4)
       rails-dom-testing (~> 2.0)
-    actionpack (5.2.4.2)
-      actionview (= 5.2.4.2)
-      activesupport (= 5.2.4.2)
+    actionpack (6.0.3.2)
+      actionview (= 6.0.3.2)
+      activesupport (= 6.0.3.2)
       rack (~> 2.0, >= 2.0.8)
       rack-test (>= 0.6.3)
       rails-dom-testing (~> 2.0)
-      rails-html-sanitizer (~> 1.0, >= 1.0.2)
-    actionview (5.2.4.2)
-      activesupport (= 5.2.4.2)
+      rails-html-sanitizer (~> 1.0, >= 1.2.0)
+    actiontext (6.0.3.2)
+      actionpack (= 6.0.3.2)
+      activerecord (= 6.0.3.2)
+      activestorage (= 6.0.3.2)
+      activesupport (= 6.0.3.2)
+      nokogiri (>= 1.8.5)
+    actionview (6.0.3.2)
+      activesupport (= 6.0.3.2)
       builder (~> 3.1)
       erubi (~> 1.4)
       rails-dom-testing (~> 2.0)
-      rails-html-sanitizer (~> 1.0, >= 1.0.3)
-    activejob (5.2.4.2)
-      activesupport (= 5.2.4.2)
+      rails-html-sanitizer (~> 1.1, >= 1.2.0)
+    activejob (6.0.3.2)
+      activesupport (= 6.0.3.2)
       globalid (>= 0.3.6)
-    activemodel (5.2.4.2)
-      activesupport (= 5.2.4.2)
-    activerecord (5.2.4.2)
-      activemodel (= 5.2.4.2)
-      activesupport (= 5.2.4.2)
-      arel (>= 9.0)
-    activerecord-import (1.0.4)
+    activemodel (6.0.3.2)
+      activesupport (= 6.0.3.2)
+    activerecord (6.0.3.2)
+      activemodel (= 6.0.3.2)
+      activesupport (= 6.0.3.2)
+    activerecord-import (1.0.5)
       activerecord (>= 3.2)
-    activestorage (5.2.4.2)
-      actionpack (= 5.2.4.2)
-      activerecord (= 5.2.4.2)
+    activestorage (6.0.3.2)
+      actionpack (= 6.0.3.2)
+      activejob (= 6.0.3.2)
+      activerecord (= 6.0.3.2)
       marcel (~> 0.3.1)
-    activesupport (5.2.4.2)
+    activesupport (6.0.3.2)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
-    arel (9.0.0)
-    ast (2.4.0)
+      zeitwerk (~> 2.2, >= 2.2.2)
+    ast (2.4.1)
     avro (1.9.2)
       multi_json
     avro_turf (0.11.0)
       avro (>= 1.7.7, < 1.10)
       excon (~> 0.45)
     builder (3.2.4)
-    coderay (1.1.2)
+    coderay (1.1.3)
     concurrent-ruby (1.1.6)
     concurrent-ruby-ext (1.1.6)
       concurrent-ruby (= 1.1.6)
     crass (1.0.6)
     database_cleaner (1.8.5)
-    ddtrace (0.35.1)
+    ddtrace (0.37.0)
       msgpack
-    diff-lcs (1.3)
-    digest-crc (0.5.1)
-    dogstatsd-ruby (4.8.0)
+    diff-lcs (1.4.4)
+    digest-crc (0.6.1)
+      rake (~> 13.0)
+    dogstatsd-ruby (4.8.1)
     erubi (1.9.0)
-    excon (0.73.0)
+    excon (0.75.0)
     exponential-backoff (0.0.4)
-    ffi (1.12.2)
+    ffi (1.13.1)
     formatador (0.2.5)
     globalid (0.4.2)
       activesupport (>= 4.2.0)
@@ -96,20 +110,19 @@ GEM
     guard-rubocop (1.3.0)
       guard (~> 2.0)
       rubocop (~> 0.20)
-    i18n (1.8.2)
+    i18n (1.8.4)
       concurrent-ruby (~> 1.0)
-    jaro_winkler (1.5.4)
     listen (3.2.1)
       rb-fsevent (~> 0.10, >= 0.10.3)
       rb-inotify (~> 0.9, >= 0.9.10)
     little-plugger (1.1.4)
-    logging (2.2.2)
+    logging (2.3.0)
       little-plugger (~> 1.1)
-      multi_json (~> 1.10)
-    loofah (2.5.0)
+      multi_json (~> 1.14)
+    loofah (2.6.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
-    lumberjack (1.2.4)
+    lumberjack (1.2.6)
     mail (2.7.1)
       mini_mime (>= 0.1.1)
     marcel (0.3.3)
@@ -118,20 +131,20 @@ GEM
     mimemagic (0.3.5)
     mini_mime (1.0.2)
     mini_portile2 (2.4.0)
-    minitest (5.14.0)
+    minitest (5.14.1)
     msgpack (1.3.3)
-    multi_json (1.14.1)
+    multi_json (1.15.0)
     mysql2 (0.5.3)
     nenv (0.3.0)
     nio4r (2.5.2)
-    nokogiri (1.10.9)
+    nokogiri (1.10.10)
       mini_portile2 (~> 2.4.0)
     notiffany (0.1.3)
       nenv (~> 0.1)
       shellany (~> 0.0)
-    parallel (1.19.1)
-    parser (2.7.1.2)
-      ast (~> 2.4.0)
+    parallel (1.19.2)
+    parser (2.7.1.4)
+      ast (~> 2.4.1)
     pg (1.2.3)
     phobos (1.9.0)
       activesupport (>= 3.0.0)
@@ -147,35 +160,38 @@ GEM
     rack (2.2.3)
     rack-test (1.1.0)
       rack (>= 1.0, < 3)
-    rails (5.2.4.2)
-      actioncable (= 5.2.4.2)
-      actionmailer (= 5.2.4.2)
-      actionpack (= 5.2.4.2)
-      actionview (= 5.2.4.2)
-      activejob (= 5.2.4.2)
-      activemodel (= 5.2.4.2)
-      activerecord (= 5.2.4.2)
-      activestorage (= 5.2.4.2)
-      activesupport (= 5.2.4.2)
+    rails (6.0.3.2)
+      actioncable (= 6.0.3.2)
+      actionmailbox (= 6.0.3.2)
+      actionmailer (= 6.0.3.2)
+      actionpack (= 6.0.3.2)
+      actiontext (= 6.0.3.2)
+      actionview (= 6.0.3.2)
+      activejob (= 6.0.3.2)
+      activemodel (= 6.0.3.2)
+      activerecord (= 6.0.3.2)
+      activestorage (= 6.0.3.2)
+      activesupport (= 6.0.3.2)
       bundler (>= 1.3.0)
-      railties (= 5.2.4.2)
+      railties (= 6.0.3.2)
       sprockets-rails (>= 2.0.0)
     rails-dom-testing (2.0.3)
       activesupport (>= 4.2.0)
       nokogiri (>= 1.6)
     rails-html-sanitizer (1.3.0)
       loofah (~> 2.3)
-    railties (5.2.4.2)
-      actionpack (= 5.2.4.2)
-      activesupport (= 5.2.4.2)
+    railties (6.0.3.2)
+      actionpack (= 6.0.3.2)
+      activesupport (= 6.0.3.2)
       method_source
       rake (>= 0.8.7)
-      thor (>= 0.19.0, < 2.0)
+      thor (>= 0.20.3, < 2.0)
     rainbow (3.0.0)
     rake (13.0.1)
     rb-fsevent (0.10.4)
     rb-inotify (0.10.1)
       ffi (~> 1.0)
+    regexp_parser (1.7.1)
     rexml (3.2.4)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
@@ -183,7 +199,7 @@ GEM
       rspec-mocks (~> 3.9.0)
     rspec-core (3.9.2)
       rspec-support (~> 3.9.3)
-    rspec-expectations (3.9.1)
+    rspec-expectations (3.9.2)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.9.0)
     rspec-mocks (3.9.1)
@@ -192,16 +208,19 @@ GEM
     rspec-support (3.9.3)
     rspec_junit_formatter (0.4.1)
       rspec-core (>= 2, < 4, != 2.12.0)
-    rubocop (0.82.0)
-      jaro_winkler (~> 1.5.1)
+    rubocop (0.88.0)
       parallel (~> 1.10)
-      parser (>= 2.7.0.1)
+      parser (>= 2.7.1.1)
       rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.7)
       rexml
+      rubocop-ast (>= 0.1.0, < 1.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 2.0)
-    rubocop-rspec (1.39.0)
-      rubocop (>= 0.68.1)
+    rubocop-ast (0.2.0)
+      parser (>= 2.7.0.1)
+    rubocop-rspec (1.42.0)
+      rubocop (>= 0.87.0)
     ruby-kafka (0.7.10)
       digest-crc
     ruby-progressbar (1.10.1)
@@ -209,7 +228,7 @@ GEM
     sigurd (0.0.1)
       concurrent-ruby (~> 1)
       exponential-backoff
-    sprockets (4.0.0)
+    sprockets (4.0.2)
       concurrent-ruby (~> 1.0)
       rack (> 1, < 3)
     sprockets-rails (3.2.1)
@@ -222,15 +241,16 @@ GEM
     tzinfo (1.2.7)
       thread_safe (~> 0.1)
     unicode-display_width (1.7.0)
-    websocket-driver (0.7.1)
+    websocket-driver (0.7.3)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.5)
+    zeitwerk (2.4.0)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  activerecord (~> 5.2)
+  activerecord (~> 6)
   activerecord-import
   avro (~> 1.9)
   database_cleaner (~> 1.7)
@@ -242,7 +262,7 @@ DEPENDENCIES
   guard-rubocop (~> 1)
   mysql2 (~> 0.5)
   pg (~> 1.1)
-  rails (~> 5.2, >= 5.2.4.2)
+  rails (~> 6)
   rake (~> 13)
   rspec (~> 3)
   rspec_junit_formatter (~> 0.3)

data/README.md

@@ -557,6 +557,29 @@ class MyConsumer < Deimos::ActiveRecordConsumer
 end
 ```
 
+#### Generating Tables and Models
+
+Deimos provides a generator that takes an existing schema and generates a 
+database table based on its fields. By default, any complex sub-types (such as 
+records or arrays) are turned into JSON (if supported) or string columns.
+
+Before running this migration, you must first copy the schema into your repo
+in the correct path (in the example above, you would need to have a file
+`{SCHEMA_ROOT}/com/my-namespace/MySchema.avsc`).
+
+To generate a model and migration, run the following:
+
+    rails g deimos:active_record TABLE_NAME FULL_SCHEMA_NAME
+    
+Example:
+
+    rails g deimos:active_record my_table com.my-namespace.MySchema
+    
+...would generate:
+
+    db/migrate/1234_create_my_table.rb
+    app/models/my_table.rb
+
 #### Batch Consumers
 
 Deimos also provides a batch consumption mode for `ActiveRecordConsumer` which

data/deimos-ruby.gemspec

@@ -23,7 +23,7 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency('ruby-kafka', '~> 0.7')
   spec.add_runtime_dependency('sigurd', '0.0.1')
 
-  spec.add_development_dependency('activerecord', '~> 5.2')
+  spec.add_development_dependency('activerecord', '~> 6')
   spec.add_development_dependency('activerecord-import')
   spec.add_development_dependency('avro', '~> 1.9')
   spec.add_development_dependency('database_cleaner', '~> 1.7')
@@ -34,7 +34,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency('guard-rubocop', '~> 1')
   spec.add_development_dependency('mysql2', '~> 0.5')
   spec.add_development_dependency('pg', '~> 1.1')
-  spec.add_development_dependency('rails', '~> 5.2', '>= 5.2.4.2')
+  spec.add_development_dependency('rails', '~> 6')
   spec.add_development_dependency('rake', '~> 13')
   spec.add_development_dependency('rspec', '~> 3')
   spec.add_development_dependency('rspec_junit_formatter', '~>0.3')

data/lib/deimos.rb

@@ -28,9 +28,9 @@ if defined?(ActiveRecord)
   require 'deimos/kafka_source'
   require 'deimos/kafka_topic_info'
   require 'deimos/backends/db'
-  require 'sigurd/signal_handler.rb'
-  require 'sigurd/executor.rb'
-  require 'deimos/utils/db_producer.rb'
+  require 'sigurd/signal_handler'
+  require 'sigurd/executor'
+  require 'deimos/utils/db_producer'
   require 'deimos/utils/db_poller'
 end
 

data/lib/deimos/kafka_topic_info.rb

@@ -13,7 +13,7 @@ module Deimos
       def lock(topic, lock_id)
         # Try to create it - it's fine if it already exists
         begin
-          self.create(topic: topic)
+          self.create(topic: topic, last_processed_at: Time.zone.now)
         rescue ActiveRecord::RecordNotUnique
           # continue on
         end
@@ -52,7 +52,26 @@ module Deimos
       # @param lock_id [String]
       def clear_lock(topic, lock_id)
         self.where(topic: topic, locked_by: lock_id).
-          update_all(locked_by: nil, locked_at: nil, error: false, retries: 0)
+          update_all(locked_by: nil,
+                     locked_at: nil,
+                     error: false,
+                     retries: 0,
+                     last_processed_at: Time.zone.now)
+      end
+
+      # Update all topics that aren't currently locked and have no messages
+      # waiting. It's OK if some messages get inserted in the middle of this
+      # because the point is that at least within a few milliseconds of each
+      # other, it wasn't locked and had no messages, meaning the topic
+      # was in a good state.
+      # @param except_topics [Array<String>] the list of topics we've just
+      # realized had messages in them, meaning all other topics were empty.
+      def ping_empty_topics(except_topics)
+        records = KafkaTopicInfo.where(locked_by: nil).
+          where('topic not in(?)', except_topics)
+        records.each do |info|
+          info.update_attribute(:last_processed_at, Time.zone.now)
+        end
       end
 
       # The producer calls this if it gets an error sending messages. This

data/lib/deimos/schema_backends/avro_base.rb

@@ -33,6 +33,30 @@ module Deimos
         decode(payload, schema: @key_schema['name'])[field_name]
       end
 
+      # :nodoc:
+      def sql_type(field)
+        type = field.type.type
+        return type if %w(array map record).include?(type)
+
+        if type == :union
+          non_null = field.type.schemas.reject { |f| f.type == :null }
+          if non_null.size > 1
+            warn("WARNING: #{field.name} has more than one non-null type. Picking the first for the SQL type.")
+          end
+          return non_null.first.type
+        end
+        return type.to_sym if %w(float boolean).include?(type)
+        return :integer if type == 'int'
+        return :bigint if type == 'long'
+
+        if type == 'double'
+          warn('Avro `double` type turns into SQL `float` type. Please ensure you have the correct `limit` set.')
+          return :float
+        end
+
+        :string
+      end
+
       # @override
       def coerce_field(field, value)
         AvroSchemaCoercer.new(avro_schema).coerce_type(field.type, value)
@@ -40,7 +64,10 @@ module Deimos
 
       # @override
       def schema_fields
-        avro_schema.fields.map { |field| SchemaField.new(field.name, field.type) }
+        avro_schema.fields.map do |field|
+          enum_values = field.type.type == 'enum' ? field.type.symbols : []
+          SchemaField.new(field.name, field.type, enum_values)
+        end
       end
 
       # @override

data/lib/deimos/schema_backends/base.rb

@@ -3,13 +3,15 @@
 module Deimos
   # Represents a field in the schema.
   class SchemaField
-    attr_accessor :name, :type
+    attr_accessor :name, :type, :enum_values
 
     # @param name [String]
     # @param type [Object]
-    def initialize(name, type)
+    # @param enum_values [Array<String>]
+    def initialize(name, type, enum_values=[])
       @name = name
       @type = type
+      @enum_values = enum_values
     end
   end
 
@@ -109,6 +111,17 @@ module Deimos
         raise NotImplementedError
       end
 
+      # Given a field definition, return the SQL type that might be used in
+      # ActiveRecord table creation - e.g. for Avro, a `long` type would
+      # return `:bigint`. There are also special values that need to be returned:
+      # `:array`, `:map` and `:record`, for types representing those structures.
+      # `:enum` is also recognized.
+      # @param field [SchemaField]
+      # @return [Symbol]
+      def sql_type(field)
+        raise NotImplementedError
+      end
+
       # Encode a message key. To be defined by subclass.
       # @param key [String|Hash] the value to use as the key.
       # @param key_id [Symbol|String] the field name of the key.

data/lib/deimos/utils/db_producer.rb

@@ -9,6 +9,8 @@ module Deimos
       attr_accessor :id, :current_topic
 
       BATCH_SIZE = 1000
+      DELETE_BATCH_SIZE = 10
+      MAX_DELETE_ATTEMPTS = 3
 
       # @param logger [Logger]
       def initialize(logger=Logger.new(STDOUT))
@@ -48,6 +50,7 @@ module Deimos
         topics = retrieve_topics
         @logger.info("Found topics: #{topics}")
         topics.each(&method(:process_topic))
+        KafkaTopicInfo.ping_empty_topics(topics)
         sleep(0.5)
       end
 
@@ -87,13 +90,13 @@ module Deimos
           begin
             produce_messages(compacted_messages.map(&:phobos_message))
           rescue Kafka::BufferOverflow, Kafka::MessageSizeTooLarge, Kafka::RecordListTooLarge
+            delete_messages(messages)
             @logger.error('Message batch too large, deleting...')
             @logger.error(Deimos::KafkaMessage.decoded(messages))
-            Deimos::KafkaMessage.where(id: messages.map(&:id)).delete_all
             raise
           end
         end
-        Deimos::KafkaMessage.where(id: messages.map(&:id)).delete_all
+        delete_messages(messages)
         Deimos.config.metrics&.increment(
           'db_producer.process',
           tags: %W(topic:#{@current_topic}),
@@ -106,6 +109,27 @@ module Deimos
         true
       end
 
+      # @param messages [Array<Deimos::KafkaMessage>]
+      def delete_messages(messages)
+        attempts = 1
+        begin
+          messages.in_groups_of(DELETE_BATCH_SIZE, false).each do |batch|
+            Deimos::KafkaMessage.where(topic: batch.first.topic,
+                                       id: batch.map(&:id)).
+              delete_all
+          end
+        rescue StandardError => e
+          if (e.message =~ /Lock wait/i || e.message =~ /Lost connection/i) &&
+             attempts <= MAX_DELETE_ATTEMPTS
+            attempts += 1
+            ActiveRecord::Base.connection.verify!
+            sleep(1)
+            retry
+          end
+          raise
+        end
+      end
+
       # @return [Array<Deimos::KafkaMessage>]
       def retrieve_messages
         KafkaMessage.where(topic: @current_topic).order(:id).limit(BATCH_SIZE)
@@ -126,19 +150,33 @@ module Deimos
         metrics = Deimos.config.metrics
         return unless metrics
 
+        topics = KafkaTopicInfo.select(%w(topic last_processed_at))
         messages = Deimos::KafkaMessage.
           select('count(*) as num_messages, min(created_at) as earliest, topic').
-          group(:topic)
-        if messages.none?
-          metrics.gauge('pending_db_messages_max_wait', 0)
-        end
-        messages.each do |record|
-          earliest = record.earliest
-          # SQLite gives a string here
-          earliest = Time.zone.parse(earliest) if earliest.is_a?(String)
-
-          time_diff = Time.zone.now - earliest
-          metrics.gauge('pending_db_messages_max_wait', time_diff,
+          group(:topic).
+          index_by(&:topic)
+        topics.each do |record|
+          message_record = messages[record.topic]
+          # We want to record the last time we saw any activity, meaning either
+          # the oldest message, or the last time we processed, whichever comes
+          # last.
+          if message_record
+            record_earliest = record.earliest
+            # SQLite gives a string here
+            if record_earliest.is_a?(String)
+              record_earliest = Time.zone.parse(record_earliest)
+            end
+
+            earliest = [record.last_processed_at, record_earliest].max
+            time_diff = Time.zone.now - earliest
+            metrics.gauge('pending_db_messages_max_wait', time_diff,
+                          tags: ["topic:#{record.topic}"])
+          else
+            # no messages waiting
+            metrics.gauge('pending_db_messages_max_wait', 0,
+                          tags: ["topic:#{record.topic}"])
+          end
+          metrics.gauge('pending_db_messages_count', message_record&.num_messages || 0,
                         tags: ["topic:#{record.topic}"])
         end
       end
@@ -174,11 +212,11 @@ module Deimos
           end
 
           @logger.error("Got error #{e.class.name} when publishing #{batch.size} in groups of #{batch_size}, retrying...")
-          if batch_size < 10
-            batch_size = 1
-          else
-            batch_size /= 10
-          end
+          batch_size = if batch_size < 10
+                         1
+                       else
+                         (batch_size / 10).to_i
+                       end
           shutdown_producer
           retry
         end
@@ -187,7 +225,7 @@ module Deimos
       # @param batch [Array<Deimos::KafkaMessage>]
       # @return [Array<Deimos::KafkaMessage>]
       def compact_messages(batch)
-        return batch unless batch.first&.key.present?
+        return batch if batch.first&.key.blank?
 
         topic = batch.first.topic
         return batch if config.compact_topics != :all &&

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '1.8.0-beta2'
+  VERSION = '1.8.1-beta1'
 end

data/lib/generators/deimos/active_record/templates/migration.rb.tt

@@ -0,0 +1,28 @@
+class <%= migration_class_name %> < ActiveRecord::Migration<%= migration_version %>
+  def up
+    if table_exists?(:<%= table_name %>)
+      warn "<%= table_name %> already exists, exiting"
+      return
+    end
+    create_table :<%= table_name %> do |t|
+      <%- fields.each do |key| -%>
+      <%- next if %w(id message_id timestamp).include?(key.name) -%>
+      <%- sql_type = schema_base.sql_type(key)
+         if %w(record array map).include?(sql_type)
+           conn = ActiveRecord::Base.connection
+           sql_type = conn.respond_to?(:supports_json?) && conn.supports_json? ? :json : :string
+         end
+      -%>
+      t.<%= sql_type %> :<%= key.name %>
+      <%- end -%>
+    end
+
+    # TODO add indexes as necessary
+  end
+
+  def down
+    return unless table_exists?(:<%= table_name %>)
+    drop_table :<%= table_name %>
+  end
+
+end

data/lib/generators/deimos/active_record/templates/model.rb.tt

@@ -0,0 +1,5 @@
+class <%= table_name.classify %> < ApplicationRecord
+<%- fields.select { |f| f.enum_values.any? }.each do |field| -%>
+  enum <%= field.name %>: {<%= field.enum_values.map { |v| "#{v}: '#{v}'"}.join(', ') %>}
+<% end -%>
+end

data/lib/generators/deimos/active_record_generator.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require 'rails/generators'
+require 'rails/generators/active_record/migration'
+require 'rails/version'
+
+# Generates a new consumer.
+module Deimos
+  module Generators
+    # Generator for ActiveRecord model and migration.
+    class ActiveRecordGenerator < Rails::Generators::Base
+      include Rails::Generators::Migration
+      if Rails.version < '4'
+        extend(ActiveRecord::Generators::Migration)
+      else
+        include ActiveRecord::Generators::Migration
+      end
+      source_root File.expand_path('active_record/templates', __dir__)
+
+      argument :table_name, desc: 'The table to create.', required: true
+      argument :full_schema, desc: 'The fully qualified schema name.', required: true
+
+      no_commands do
+
+        # @return [String]
+        def db_migrate_path
+          if defined?(Rails.application) && Rails.application
+            paths = Rails.application.config.paths['db/migrate']
+            paths.respond_to?(:to_ary) ? paths.to_ary.first : paths.to_a.first
+          else
+            'db/migrate'
+          end
+        end
+
+        # @return [String]
+        def migration_version
+          "[#{ActiveRecord::Migration.current_version}]"
+        rescue StandardError
+          ''
+        end
+
+        # @return [String]
+        def table_class
+          self.table_name.classify
+        end
+
+        # @return [String]
+        def schema
+          last_dot = self.full_schema.rindex('.')
+          self.full_schema[last_dot + 1..-1]
+        end
+
+        # @return [String]
+        def namespace
+          last_dot = self.full_schema.rindex('.')
+          self.full_schema[0...last_dot]
+        end
+
+        # @return [Deimos::SchemaBackends::Base]
+        def schema_base
+          @schema_base ||= Deimos.schema_backend_class.new(schema: schema, namespace: namespace)
+        end
+
+        # @return [Array<SchemaField>]
+        def fields
+          schema_base.schema_fields
+        end
+
+      end
+
+      desc 'Generate migration for a table based on an existing schema.'
+      # :nodoc:
+      def generate
+        migration_template('migration.rb', "db/migrate/create_#{table_name.underscore}.rb")
+        template('model.rb', "app/models/#{table_name.underscore}.rb")
+      end
+    end
+  end
+end

data/lib/generators/deimos/db_backend/templates/migration

@@ -16,6 +16,7 @@ class <%= migration_class_name %> < ActiveRecord::Migration<%= migration_version
       t.datetime :locked_at
       t.boolean :error, null: false, default: false
       t.integer :retries, null: false, default: 0
+      t.datetime :last_processed_at
     end
     add_index :kafka_topic_info, :topic, unique: true
     add_index :kafka_topic_info, [:locked_by, :error]

data/lib/generators/deimos/db_backend/templates/rails3_migration

@@ -16,6 +16,7 @@ class <%= migration_class_name %> < ActiveRecord::Migration<%= migration_version
       t.datetime :locked_at
       t.boolean :error, null: false, default: false
       t.integer :retries, null: false, default: 0
+      t.datetime :last_processed_at
     end
     add_index :kafka_topic_info, :topic, unique: true
     add_index :kafka_topic_info, [:locked_by, :error]

data/spec/generators/active_record_generator_spec.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require 'generators/deimos/active_record_generator'
+
+RSpec.describe Deimos::Generators::ActiveRecordGenerator do
+
+  after(:each) do
+    FileUtils.rm_rf('db') if File.exist?('db')
+    FileUtils.rm_rf('app') if File.exist?('app')
+  end
+
+  it 'should generate a migration' do
+    expect(Dir['db/migrate/*.rb']).to be_empty
+    expect(Dir['app/models/*.rb']).to be_empty
+    described_class.start(['generated_table', 'com.my-namespace.Generated'])
+    files = Dir['db/migrate/*.rb']
+    expect(files.length).to eq(1)
+    results = <<~MIGRATION
+      class CreateGeneratedTable < ActiveRecord::Migration[6.0]
+        def up
+          if table_exists?(:generated_table)
+            warn "generated_table already exists, exiting"
+            return
+          end
+          create_table :generated_table do |t|
+            t.string :a_string
+            t.integer :a_int
+            t.bigint :a_long
+            t.float :a_float
+            t.float :a_double
+            t.string :an_enum
+            t.json :an_array
+            t.json :a_map
+            t.json :a_record
+          end
+      
+          # TODO add indexes as necessary
+        end
+      
+        def down
+          return unless table_exists?(:generated_table)
+          drop_table :generated_table
+        end
+      
+      end
+    MIGRATION
+    expect(File.read(files[0])).to eq(results)
+    model = <<~MODEL
+      class GeneratedTable < ApplicationRecord
+        enum an_enum: {sym1: 'sym1', sym2: 'sym2'}
+      end
+    MODEL
+    expect(File.read('app/models/generated_table.rb')).to eq(model)
+  end
+
+end

data/spec/kafka_topic_info_spec.rb

@@ -37,22 +37,45 @@ each_db_config(Deimos::KafkaTopicInfo) do
   end
 
   specify '#clear_lock' do
-    described_class.create!(topic: 'my-topic', locked_by: 'abc',
-                            locked_at: 10.seconds.ago, error: true, retries: 1)
-    described_class.create!(topic: 'my-topic2', locked_by: 'def',
-                            locked_at: 10.seconds.ago, error: true, retries: 1)
-    described_class.clear_lock('my-topic', 'abc')
-    expect(described_class.count).to eq(2)
-    record = described_class.first
-    expect(record.locked_by).to eq(nil)
-    expect(record.locked_at).to eq(nil)
-    expect(record.error).to eq(false)
-    expect(record.retries).to eq(0)
-    record = described_class.last
-    expect(record.locked_by).not_to eq(nil)
-    expect(record.locked_at).not_to eq(nil)
-    expect(record.error).not_to eq(false)
-    expect(record.retries).not_to eq(0)
+    freeze_time do
+      Deimos::KafkaTopicInfo.create!(topic: 'my-topic', locked_by: 'abc',
+                                     locked_at: 10.seconds.ago, error: true, retries: 1,
+                                     last_processed_at: 20.seconds.ago)
+      Deimos::KafkaTopicInfo.create!(topic: 'my-topic2', locked_by: 'def',
+                                     locked_at: 10.seconds.ago, error: true, retries: 1,
+                                     last_processed_at: 20.seconds.ago)
+      Deimos::KafkaTopicInfo.clear_lock('my-topic', 'abc')
+      expect(Deimos::KafkaTopicInfo.count).to eq(2)
+      record = Deimos::KafkaTopicInfo.first
+      expect(record.locked_by).to eq(nil)
+      expect(record.locked_at).to eq(nil)
+      expect(record.error).to eq(false)
+      expect(record.retries).to eq(0)
+      expect(record.last_processed_at.to_s).to eq(Time.zone.now.to_s)
+      record = Deimos::KafkaTopicInfo.last
+      expect(record.locked_by).not_to eq(nil)
+      expect(record.locked_at).not_to eq(nil)
+      expect(record.error).not_to eq(false)
+      expect(record.retries).not_to eq(0)
+      expect(record.last_processed_at.to_s).to eq(20.seconds.ago.to_s)
+    end
+  end
+
+  specify '#ping_empty_topics' do
+    freeze_time do
+      old_time = 1.hour.ago.to_s
+      t1 = Deimos::KafkaTopicInfo.create!(topic: 'topic1', last_processed_at: old_time)
+      t2 = Deimos::KafkaTopicInfo.create!(topic: 'topic2', last_processed_at: old_time)
+      t3 = Deimos::KafkaTopicInfo.create!(topic: 'topic3', last_processed_at: old_time,
+                                          locked_by: 'me', locked_at: 1.minute.ago)
+
+      expect(Deimos::KafkaTopicInfo.count).to eq(3)
+      Deimos::KafkaTopicInfo.all.each { |t| expect(t.last_processed_at.to_s).to eq(old_time) }
+      Deimos::KafkaTopicInfo.ping_empty_topics(%w(topic1))
+      expect(t1.reload.last_processed_at.to_s).to eq(old_time) # was passed as an exception
+      expect(t2.reload.last_processed_at.to_s).to eq(Time.zone.now.to_s)
+      expect(t3.reload.last_processed_at.to_s).to eq(old_time) # is locked
+    end
   end
 
   specify '#register_error' do

data/spec/schemas/com/my-namespace/Generated.avsc

@@ -0,0 +1,71 @@
+{
+  "namespace": "com.my-namespace",
+  "name": "Generated",
+  "type": "record",
+  "doc": "Test schema",
+  "fields": [
+    {
+      "name": "a_string",
+      "type": "string"
+    },
+    {
+      "name": "a_int",
+      "type": "int"
+    },
+    {
+      "name": "a_long",
+      "type": "long"
+    },
+    {
+      "name": "a_float",
+      "type": "float"
+    },
+    {
+      "name": "a_double",
+      "type": "double"
+    },
+    {
+      "name": "an_enum",
+      "type": {
+        "type": "enum",
+        "name": "AnEnum",
+        "symbols": ["sym1", "sym2"]
+      }
+    },
+    {
+      "name": "an_array",
+      "type": {
+        "type": "array",
+        "items": "int"
+      }
+    },
+    {
+      "name": "a_map",
+      "type": {
+        "type": "map",
+        "values": "string"
+      }
+    },
+    {
+      "name": "timestamp",
+      "type": "string"
+    },
+    {
+      "name": "message_id",
+      "type": "string"
+    },
+    {
+      "name": "a_record",
+      "type": {
+        "type": "record",
+        "name": "ARecord",
+        "fields": [
+          {
+            "name": "a_record_field",
+            "type": "string"
+          }
+        ]
+      }
+    }
+  ]
+}

data/spec/spec_helper.rb

@@ -32,6 +32,7 @@ module TestRunners
   # Test runner
   class TestRunner
     attr_accessor :id, :started, :stopped, :should_error
+
     # :nodoc:
     def initialize(id=nil)
       @id = id

data/spec/utils/db_producer_spec.rb

@@ -16,11 +16,13 @@ each_db_config(Deimos::Utils::DbProducer) do
 
   before(:each) do
     stub_const('Deimos::Utils::DbProducer::BATCH_SIZE', 2)
+    stub_const('Deimos::Utils::DbProducer::DELETE_BATCH_SIZE', 1)
   end
 
   specify '#process_next_messages' do
     expect(producer).to receive(:retrieve_topics).and_return(%w(topic1 topic2))
     expect(producer).to receive(:process_topic).twice
+    expect(Deimos::KafkaTopicInfo).to receive(:ping_empty_topics).with(%w(topic1 topic2))
     expect(producer).to receive(:sleep).with(0.5)
     producer.process_next_messages
   end
@@ -40,6 +42,9 @@ each_db_config(Deimos::Utils::DbProducer) do
       Deimos::KafkaMessage.create!(topic: 'topic1',
                                    message: 'blah',
                                    key: "key#{i}")
+      Deimos::KafkaMessage.create!(topic: 'topic2',
+                                   message: 'blah',
+                                   key: "key#{i}")
     end
     stub_const('Deimos::Utils::DbProducer::BATCH_SIZE', 5)
     producer.current_topic = 'topic1'
@@ -280,6 +285,12 @@ each_db_config(Deimos::Utils::DbProducer) do
           message: "mess#{i}",
           partition_key: "key#{i}"
         )
+        Deimos::KafkaMessage.create!(
+          id: i,
+          topic: 'my-topic2',
+          message: "mess#{i}",
+          partition_key: "key#{i}"
+        )
       end
 
       expect(Deimos::KafkaTopicInfo).to receive(:lock).
@@ -288,9 +299,60 @@ each_db_config(Deimos::Utils::DbProducer) do
       expect(producer).to receive(:retrieve_messages).and_return(messages)
       expect(Deimos::KafkaTopicInfo).to receive(:register_error)
 
+      expect(Deimos::KafkaMessage.count).to eq(8)
+      producer.process_topic('my-topic')
       expect(Deimos::KafkaMessage.count).to eq(4)
+    end
+
+    it 'should retry deletes and not re-publish' do
+      messages = (1..4).map do |i|
+        Deimos::KafkaMessage.create!(
+          id: i,
+          topic: 'my-topic',
+          message: "mess#{i}",
+          partition_key: "key#{i}"
+        )
+      end
+      (5..8).each do |i|
+        Deimos::KafkaMessage.create!(
+          id: i,
+          topic: 'my-topic2',
+          message: "mess#{i}",
+          partition_key: "key#{i}"
+        )
+      end
+
+      raise_error = true
+      expect(Deimos::KafkaMessage).to receive(:where).exactly(5).times.and_wrap_original do |m, *args|
+        if raise_error
+          raise_error = false
+          raise 'Lock wait timeout'
+        end
+        m.call(*args)
+      end
+
+      expect(Deimos::KafkaTopicInfo).to receive(:lock).
+        with('my-topic', 'abc').and_return(true)
+      expect(producer).to receive(:retrieve_messages).ordered.and_return(messages)
+      expect(producer).to receive(:retrieve_messages).ordered.and_return([])
+      expect(phobos_producer).to receive(:publish_list).once.with(messages.map(&:phobos_message))
+
+      expect(Deimos::KafkaMessage.count).to eq(8)
       producer.process_topic('my-topic')
-      expect(Deimos::KafkaMessage.count).to eq(0)
+      expect(Deimos::KafkaMessage.count).to eq(4)
+    end
+
+    it 'should re-raise misc errors on delete' do
+      messages = (1..3).map do |i|
+        Deimos::KafkaMessage.create!(
+          id: i,
+          topic: 'my-topic',
+          message: "mess#{i}",
+          partition_key: "key#{i}"
+        )
+      end
+      expect(Deimos::KafkaMessage).to receive(:where).once.and_raise('OH NOES')
+      expect { producer.delete_messages(messages) }.to raise_exception('OH NOES')
     end
 
   end
@@ -309,21 +371,34 @@ each_db_config(Deimos::Utils::DbProducer) do
           Deimos::KafkaMessage.create!(topic: "topic#{i}", message: nil,
                                        created_at: (1 + i).minute.ago)
         end
+        Deimos::KafkaTopicInfo.create!(topic: 'topic1',
+                                       last_processed_at: 6.minutes.ago)
+        Deimos::KafkaTopicInfo.create!(topic: 'topic2',
+                                       last_processed_at: 3.minutes.ago)
+        Deimos::KafkaTopicInfo.create!(topic: 'topic3',
+                                       last_processed_at: 5.minutes.ago)
         allow(Deimos.config.metrics).to receive(:gauge)
         producer.send_pending_metrics
-        expect(Deimos.config.metrics).to have_received(:gauge).twice
+        expect(Deimos.config.metrics).to have_received(:gauge).exactly(6).times
+        # topic1 has the earliest message 4 minutes ago and last processed 6
+        # minutes ago, so the most amount of time we've seen nothing is 4 minutes
         expect(Deimos.config.metrics).to have_received(:gauge).
           with('pending_db_messages_max_wait', 4.minutes.to_i, tags: ['topic:topic1'])
+        # topic2 has earliest message 5 minutes ago and last processed 3 minutes
+        # ago, so we should give it 3 minutes
+        expect(Deimos.config.metrics).to have_received(:gauge).
+          with('pending_db_messages_max_wait', 3.minutes.to_i, tags: ['topic:topic2'])
+        # topic3 has no messages, so should get 0
         expect(Deimos.config.metrics).to have_received(:gauge).
-          with('pending_db_messages_max_wait', 5.minutes.to_i, tags: ['topic:topic2'])
+          with('pending_db_messages_max_wait', 0, tags: ['topic:topic3'])
+        expect(Deimos.config.metrics).to have_received(:gauge).
+          with('pending_db_messages_count', 3, tags: ['topic:topic1'])
+        expect(Deimos.config.metrics).to have_received(:gauge).
+          with('pending_db_messages_count', 3, tags: ['topic:topic2'])
+        expect(Deimos.config.metrics).to have_received(:gauge).
+          with('pending_db_messages_count', 0, tags: ['topic:topic3'])
       end
     end
-
-    it 'should send 0 if no messages' do
-      expect(Deimos.config.metrics).to receive(:gauge).
-        with('pending_db_messages_max_wait', 0)
-      producer.send_pending_metrics
-    end
   end
 
   example 'Full integration test' do
@@ -403,5 +478,4 @@ each_db_config(Deimos::Utils::DbProducer) do
                                                                 }
                                                               ])
   end
-
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deimos-ruby
 version: !ruby/object:Gem::Version
-  version: 1.8.0.pre.beta2
+  version: 1.8.1.pre.beta1
 platform: ruby
 authors:
 - Daniel Orner
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-07-08 00:00:00.000000000 Z
+date: 2020-07-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro_turf
@@ -72,14 +72,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.2'
+        version: '6'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.2'
+        version: '6'
 - !ruby/object:Gem::Dependency
   name: activerecord-import
   requirement: !ruby/object:Gem::Requirement
@@ -226,20 +226,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.2'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 5.2.4.2
+        version: '6'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.2'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 5.2.4.2
+        version: '6'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -406,6 +400,9 @@ files:
 - lib/deimos/utils/inline_consumer.rb
 - lib/deimos/utils/lag_reporter.rb
 - lib/deimos/version.rb
+- lib/generators/deimos/active_record/templates/migration.rb.tt
+- lib/generators/deimos/active_record/templates/model.rb.tt
+- lib/generators/deimos/active_record_generator.rb
 - lib/generators/deimos/db_backend/templates/migration
 - lib/generators/deimos/db_backend/templates/rails3_migration
 - lib/generators/deimos/db_backend_generator.rb
@@ -427,6 +424,7 @@ files:
 - spec/config/configuration_spec.rb
 - spec/consumer_spec.rb
 - spec/deimos_spec.rb
+- spec/generators/active_record_generator_spec.rb
 - spec/handlers/my_batch_consumer.rb
 - spec/handlers/my_consumer.rb
 - spec/kafka_source_spec.rb
@@ -441,6 +439,7 @@ files:
 - spec/schema_backends/avro_schema_registry_spec.rb
 - spec/schema_backends/avro_validation_spec.rb
 - spec/schema_backends/base_spec.rb
+- spec/schemas/com/my-namespace/Generated.avsc
 - spec/schemas/com/my-namespace/MySchema-key.avsc
 - spec/schemas/com/my-namespace/MySchema.avsc
 - spec/schemas/com/my-namespace/MySchemaCompound-key.avsc
@@ -499,6 +498,7 @@ test_files:
 - spec/config/configuration_spec.rb
 - spec/consumer_spec.rb
 - spec/deimos_spec.rb
+- spec/generators/active_record_generator_spec.rb
 - spec/handlers/my_batch_consumer.rb
 - spec/handlers/my_consumer.rb
 - spec/kafka_source_spec.rb
@@ -513,6 +513,7 @@ test_files:
 - spec/schema_backends/avro_schema_registry_spec.rb
 - spec/schema_backends/avro_validation_spec.rb
 - spec/schema_backends/base_spec.rb
+- spec/schemas/com/my-namespace/Generated.avsc
 - spec/schemas/com/my-namespace/MySchema-key.avsc
 - spec/schemas/com/my-namespace/MySchema.avsc
 - spec/schemas/com/my-namespace/MySchemaCompound-key.avsc