deimos-ruby 1.0.0.pre.beta22 → 1.0.0.pre.beta23

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 907d55a8c7e56e87ff98674d7b07f2946e6d1543aeeffaf36da7fe15e5556cc2
-  data.tar.gz: f92ce4eb4ebb455db022e66924885e2c0144c2485f10a152d187781b13d69645
+  metadata.gz: a7de207dcf4a09b8881881cfbb6af9c007c48083e091344ed3401e4a6fb944fb
+  data.tar.gz: 662ecac0adcd71cd8bf4bb44fa70436a88844d4ef65c76f8a2f341c964ae2cd8
 SHA512:
-  metadata.gz: a947fe880133b5f93a076a618b882309b689160d3baa8aa91f0b12deadec1ffe326138900d97cbda82db92843b200af86b7ac40b20e3aaa7afce2bf24a25a80c
-  data.tar.gz: d9fac9b0fe096bd6a69bcebb6c2daa0ae121feb593ded2192a5c7c5316b7631555c9d88634095ef019ee6e71300ad80e2733d5114c90f450d048664b970bd2aa
+  metadata.gz: 4508f6fb52f68a89b57cdda8bf46d1069d2359c3fafd9e12e099601ed5b474a36739958a56fa4c796584eec73ae1eac3e8e31aab121d139a9a8ee6dac7797fc2
+  data.tar.gz: b2b31653474653660fd0e4fac2af4843954c39b1525604fb7ff9fa39aa14f6e982627ad1da48c655077f4ad3143253ee923043229b36f6cf44d03135fb4c873a

data/CHANGELOG.md

@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.0.0-beta23] - 2019-08-22
+- Fix bug where nil payloads were not being saved to the DB.
+- Fix DB producer rake task looking at THREADS env var instead of THREAD_COUNT.
+- Debug messages in the DB producer if debug logs are turned on.
+- Changed logger in specs to info.
+
 ## [1.0.0-beta22] - 2019-08-09
 - Add `pending_db_messages_max_wait` metric for the DB producer.
 - Fix mock metrics to allow optional option hashes.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    deimos-ruby (1.0.0.pre.beta22)
+    deimos-ruby (1.0.0.pre.beta23)
       avro-patches (~> 0.3)
       avro_turf (~> 0.8)
       phobos (~> 1.8)

data/lib/deimos/kafka_message.rb

@@ -5,7 +5,7 @@ module Deimos
   class KafkaMessage < ActiveRecord::Base
     self.table_name = 'kafka_messages'
 
-    validates_presence_of :message, :topic
+    validates_presence_of :topic
 
     # Ensure it gets turned into a string, e.g. for testing purposes. It
     # should already be a string.
@@ -23,5 +23,6 @@ module Deimos
         topic: self.topic
       }
     end
+
   end
 end

data/lib/deimos/kafka_source.rb

@@ -93,14 +93,13 @@ module Deimos
 
         # This will contain an array of hashes, where each hash is the actual
         # attribute hash that created the object.
-        ids =
-          ids = if results.is_a?(Array)
-                  results[1]
-                elsif results.respond_to?(:ids)
-                  results.ids
-                else
-                  []
-                end
+        ids = if results.is_a?(Array)
+                results[1]
+              elsif results.respond_to?(:ids)
+                results.ids
+              else
+                []
+              end
         if ids.blank?
           # re-fill IDs based on what was just entered into the DB.
           if self.connection.adapter_name.downcase =~ /sqlite/

data/lib/deimos/kafka_topic_info.rb

@@ -66,7 +66,7 @@ module Deimos
                       locked_at: Time.zone.now,
                       error: true,
                       retries: record.retries + 1 }
-        if Rails::VERSION::MAJOR >= 4
+        if ActiveRecord::VERSION::MAJOR >= 4
           record.update!(attr_hash)
         else
           record.update_attributes!(attr_hash)

data/lib/deimos/utils/db_producer.rb

@@ -62,6 +62,22 @@ module Deimos
         messages = retrieve_messages
 
         while messages.any?
+          @logger.debug do
+            producer = Deimos::Producer.descendants.find { |c| c.topic == topic }
+            decoded_messages = if producer
+                                 consumer = Class.new(Deimos::Consumer)
+                                 consumer.config.merge!(producer.config)
+                                 messages.map do |message|
+                                   {
+                                     :key => message[:key].present? ? consumer.new.decode_key(message[:key]) : nil,
+                                     :message => consumer.decoder.decode(message[:payload])
+                                   }
+                                 end
+                               else
+                                 messages
+                               end
+            "DB producer: Topic #{topic} Producing messages: #{decoded_messages}"
+          end
           produce_messages(messages.map(&:phobos_message))
           messages.first.class.where(id: messages.map(&:id)).delete_all
           break if messages.size < BATCH_SIZE

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '1.0.0-beta22'
+  VERSION = '1.0.0-beta23'
 end

data/lib/tasks/deimos.rake

@@ -20,7 +20,7 @@ namespace :deimos do
     ENV['DEIMOS_RAKE_TASK'] = 'true'
     STDOUT.sync = true
     Rails.logger.info('Running deimos:db_producer rake task.')
-    thread_count = ENV['THREADS'].presence || 1
+    thread_count = ENV['THREAD_COUNT'].to_i.zero? ? 1 : ENV['THREAD_COUNT'].to_i
     Deimos.start_db_backend!(thread_count: thread_count)
   end
 

data/spec/backends/db_spec.rb

@@ -23,6 +23,13 @@ each_db_config(Deimos::Backends::Db) do
       'key' => 'foo3'
     )
   end
+
+  it 'should add nil messages' do
+    described_class.publish(producer_class: MyProducer,
+                            messages: [build_message(nil, 'my-topic', "foo1")])
+    expect(Deimos::KafkaMessage.count).to eq(1)
+  end
+
   it 'should add to non-keyed messages' do
     described_class.publish(producer_class: MyNoKeyProducer,
                             messages: messages)

data/spec/spec_helper.rb

@@ -7,6 +7,7 @@ require 'deimos/metrics/mock'
 require 'deimos/tracing/mock'
 require 'deimos/test_helpers'
 require 'active_support/testing/time_helpers'
+require 'activerecord-import'
 
 # Helpers for Executor/DbProducer
 module TestRunners
@@ -141,17 +142,18 @@ RSpec.configure do |config|
     Time.zone = 'EST'
     ActiveRecord::Base.logger = Logger.new('/dev/null')
     setup_db(DbConfigs::DB_OPTIONS.last)
-    Deimos.configure do |fr_config|
-      fr_config.phobos_config_file = File.join(File.dirname(__FILE__), 'phobos.yml')
-      fr_config.schema_path = File.join(File.expand_path(__dir__), 'schemas')
-      fr_config.reraise_consumer_errors = true
-      fr_config.schema_registry_url = ENV['SCHEMA_REGISTRY'] || 'http://localhost:8081'
-      fr_config.seed_broker = ENV['KAFKA_SEED_BROKER'] || 'localhost:9092'
-      fr_config.logger = Logger.new('/dev/null')
+    Deimos.configure do |deimos_config|
+      deimos_config.phobos_config_file = File.join(File.dirname(__FILE__), 'phobos.yml')
+      deimos_config.schema_path = File.join(File.expand_path(__dir__), 'schemas')
+      deimos_config.reraise_consumer_errors = true
+      deimos_config.schema_registry_url = ENV['SCHEMA_REGISTRY'] || 'http://localhost:8081'
+      deimos_config.seed_broker = ENV['KAFKA_SEED_BROKER'] || 'localhost:9092'
+      deimos_config.logger = Logger.new('/dev/null')
+      deimos_config.logger.level = Logger::INFO
 
       # Use Mock Metrics and Tracing for rspecs
-      fr_config.metrics = Deimos::Metrics::Mock.new
-      fr_config.tracer = Deimos::Tracing::Mock.new
+      deimos_config.metrics = Deimos::Metrics::Mock.new
+      deimos_config.tracer = Deimos::Tracing::Mock.new
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deimos-ruby
 version: !ruby/object:Gem::Version
-  version: 1.0.0.pre.beta22
+  version: 1.0.0.pre.beta23
 platform: ruby
 authors:
 - Daniel Orner
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-08-09 00:00:00.000000000 Z
+date: 2019-08-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro-patches
@@ -315,7 +315,7 @@ files:
 - README.md
 - Rakefile
 - bin/deimos
-- deimos-kafka.gemspec
+- deimos-ruby.gemspec
 - docker-compose.yml
 - docs/DATABASE_BACKEND.md
 - docs/PULL_REQUEST_TEMPLATE.md