deimos-kafka 1.0.0.pre.beta16 → 1.0.0.pre.beta17

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2e0cb253905c806b2609cbce7566dc2fa987b9f2f49e91adcc38525cfe1a6a56
-  data.tar.gz: cf80441d26aaf4e0a69aa96dc344ccd895474e7c2a4d1c2ae5f1ff2ac6d25a95
+  metadata.gz: 30b4706911af70b7e4662afdecf5b547583b74dbdebce492a7e57a9b1c9d504e
+  data.tar.gz: d5583aefa2e976497d8190398f0b04b2d07e33f6ff8f5ac79089bffa850b53f9
 SHA512:
-  metadata.gz: 9483035eddaa55c3add6291f60c89c701c8c0b4fa220e6757890cae9c4e45282d20d1737247f2dbb15ee907d678529f6c0e0e0a00037bd504ae7cf8529a18e65
-  data.tar.gz: da780f7e8a19e5a1e700d14cc3c80f2a659d905b0f9d1f064ccafef40dc4aec5e10b3b0eb50fd66843aea7bde3077f0fe3226425785fe21661110d67feb4be29
+  metadata.gz: 329ad07cc57cc869c1f9a843d74f70a428c7706882ce5bcabe8006afbc0e557f5931649ea4871973d21b5edd8d5e892dcf82ae5e0231a21e8b6e8021c2da187c
+  data.tar.gz: dc46360e6061b586b127888ac784b946fde6e9e5efe73173b8963aa1345fd65b58fe174e4fc69db37d9093ef17549c436087e1d35a73df555135c1d430f19a3b

data/CHANGELOG.md

@@ -5,5 +5,11 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.0.0-beta17] - 2019-07-31
+- Added `rails deimos:db_producer` rake task.
+- Fixed the DB producer so it runs inline instead of on a separate thread.
+  Calling code should run it on a thread manually if that is the desired
+  behavior.
+
 ## [1.0.0-beta15] - 2019-07-08
 - Initial release.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    deimos-kafka (1.0.0.pre.beta15)
+    deimos-kafka (1.0.0.pre.beta17)
       avro-patches (~> 0.3)
       avro_turf (~> 0.8)
       phobos (~> 1.8)

data/README.md

@@ -455,6 +455,8 @@ message is first validated, encoded, and saved in the database, and then sent
 on a separate thread. This means if you have to roll back your transaction,
 it also rolls back your Kafka messages.
 
+This is also known as the [Transactional Outbox pattern](https://microservices.io/patterns/data/transactional-outbox.html).
+
 To enable this, first generate the migration to create the relevant tables:
 
     rails g deimos:db_backend
@@ -468,12 +470,17 @@ of immediately sending to Kafka. Now, you just need to call
 
     Deimos.start_db_backend!
     
+If using Rails, you can use a Rake task to do this:
+
+    rails deimos:db_producer
+    
 This creates one or more threads dedicated to scanning and publishing these 
 messages by using the `kafka_topics` table in a manner similar to 
 [Delayed Job](https://github.com/collectiveidea/delayed_job).
 You can pass in a number of threads to the method:
 
-    Deimos.start_db_backend!(thread_count: 2)
+    Deimos.start_db_backend!(thread_count: 2) # OR
+    THREAD_COUNT=5 rails deimos:db_producer
 
 If you want to force a message to send immediately, just call the `publish_list`
 method with `force_send: true`. You can also pass `force_send` into any of the

data/deimos-kafka.gemspec

@@ -11,7 +11,7 @@ Gem::Specification.new do |spec|
   spec.email         = ['daniel.orner@wishabi.com']
   spec.summary       = 'Kafka libraries for Ruby.'
   spec.homepage      = ''
-  spec.license       = 'Apache 2.0'
+  spec.license       = 'Apache-2.0'
 
   spec.files         = `git ls-files`.split($INPUT_RECORD_SEPARATOR)
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }

data/lib/deimos/configuration.rb

@@ -83,6 +83,7 @@ module Deimos
     def phobos_config_changed?(other_config)
       phobos_keys = %w(seed_broker phobos_config_file ssl_ca_cert ssl_client_cert ssl_client_cert_key)
       return true if phobos_keys.any? { |key| self.send(key) != other_config.send(key) }
+
       other_config.logger != self.logger
     end
   end

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '1.0.0-beta16'
+  VERSION = '1.0.0-beta17'
 end

data/lib/deimos.rb

@@ -78,9 +78,13 @@ module Deimos
     # Start the DB producers to send Kafka messages.
     # @param thread_count [Integer] the number of threads to start.
     def start_db_backend!(thread_count: 1)
-      return if self.config.publish_backend != :db ||
-                thread_count.nil? ||
-                thread_count.zero?
+      if self.config.publish_backend != :db
+        raise("Publish backend is not set to :db, exiting")
+      end
+
+      if thread_count.nil? || thread_count.zero?
+        raise("Thread count is not given or set to zero, exiting")
+      end
 
       producers = (1..thread_count).map do
         Deimos::Utils::DbProducer.new(self.config.logger)
@@ -88,12 +92,7 @@ module Deimos
       executor = Deimos::Utils::Executor.new(producers,
                                              self.config.logger)
       signal_handler = Deimos::Utils::SignalHandler.new(executor)
-      run_db_backend_in_thread(signal_handler)
-    end
-
-    # @param signal_handler [Deimos::Utils::SignalHandler]
-    def run_db_backend_in_thread(signal_handler)
-      Thread.new { signal_handler.run! }
+      signal_handler.run!
     end
 
     # @param phobos_config [Hash]

data/lib/tasks/deimos.rake

@@ -14,4 +14,14 @@ namespace :deimos do
     Rails.logger.info('Running deimos:start rake task.')
     Phobos::CLI::Commands.start(%w(start --skip_config))
   end
+
+  desc 'Starts the Deimos database producer'
+  task db_producer: :environment do
+    ENV['DEIMOS_RAKE_TASK'] = 'true'
+    STDOUT.sync = true
+    Rails.logger.info("Running deimos:db_producer rake task.")
+    thread_count = ENV['THREADS'].presence || 1
+    Deimos.start_db_backend!(thread_count: thread_count)
+  end
+
 end

data/spec/deimos_spec.rb

@@ -73,7 +73,7 @@ describe Deimos do
 
   describe '#start_db_backend!' do
     before(:each) do
-      allow(described_class).to receive(:run_db_backend_in_thread)
+      allow(described_class).to receive(:run_db_backend)
     end
 
     it 'should start if backend is db and num_producer_threads is > 0' do
@@ -94,7 +94,8 @@ describe Deimos do
       described_class.configure do |config|
         config.publish_backend = :kafka
       end
-      described_class.start_db_backend!(thread_count: 2)
+      expect { described_class.start_db_backend!(thread_count: 2) }.
+        to raise_error('Publish backend is not set to :db, exiting')
     end
 
     it 'should not start if num_producer_threads is nil' do
@@ -102,7 +103,8 @@ describe Deimos do
       described_class.configure do |config|
         config.publish_backend = :db
       end
-      described_class.start_db_backend!(thread_count: nil)
+      expect { described_class.start_db_backend!(thread_count: nil) }.
+        to raise_error('Thread count is not given or set to zero, exiting')
     end
 
     it 'should not start if num_producer_threads is 0' do
@@ -110,7 +112,8 @@ describe Deimos do
       described_class.configure do |config|
         config.publish_backend = :db
       end
-      described_class.start_db_backend!(thread_count: 0)
+      expect { described_class.start_db_backend!(thread_count: 0) }.
+        to raise_error('Thread count is not given or set to zero, exiting')
     end
 
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deimos-kafka
 version: !ruby/object:Gem::Version
-  version: 1.0.0.pre.beta16
+  version: 1.0.0.pre.beta17
 platform: ruby
 authors:
 - Daniel Orner
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-07-09 00:00:00.000000000 Z
+date: 2019-07-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro-patches
@@ -398,7 +398,7 @@ files:
 - support/flipp-logo.png
 homepage: ''
 licenses:
-- Apache 2.0
+- Apache-2.0
 metadata: {}
 post_install_message: 
 rdoc_options: []