ruby_event_store-outbox 0.0.16 → 0.0.17

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '008b96c4b43e3cbfc23d0aa8c9ebb160f0adbfcfc82a976967f1c56338346cc0'
-  data.tar.gz: e65157c0638ecec0a4967a251e820979d18eb92e55212f107be95c23d5a4315b
+  metadata.gz: 2dbabda53c709b7e11850772272b736226ed33e441392e9f7a1b7b5e3ead47b7
+  data.tar.gz: 02eec2972f701e1ecc09bd65c38cecafef53d4dc45f95f69a869457289a1edcc
 SHA512:
-  metadata.gz: f0181744b8c621bc1f0b20d6d4a65ac779db7ba48a80c9f1a02f39208762aa82e5579e745f20cd35f2b54992876065f803e04e24c1c239abe22641f927327070
-  data.tar.gz: 453ce565584436b1b7dd145c738a0d50e95821cb8b8f5d9c00f0d70d2a8ea88e585402de534388459d04853d293c0bcbf7027285d31bbd00aeaef09471cf666d
+  metadata.gz: aec65f023d21c3a86204b31b22ae1b21157f46c39869a22e522efe7b13e320cd3d4f75ef139c6e421d40b203a3d33bcb368cf16cb98552ff803380a36469290a
+  data.tar.gz: 6fa822fb40b763717f31297944423bcb708400ba3bfad9852cde4edc99f79308ad3bb837ffc593d2cae6942961735a6d5c6c9bf25088bafbaea9b48f2d278842

data/README.md

@@ -14,6 +14,12 @@ Add to your gemfile in application:
 gem "ruby_event_store-outbox"
 ```
 
+Generate and execute the migration adding necessary tables. If it's needed, change the type of the `payload` column to `mediumbinary` or `longbinary`.
+
+```
+bin/rails generate ruby_event_store:outbox:migration
+```
+
 In your event store configuration, as a dispatcher use `RubyEventStore::ImmediateAsyncDispatcher` with `RubyEventStore::Outbox::SidekiqScheduler`, for example:
 
 ```ruby
@@ -38,14 +44,18 @@ end
 Run following process in any way you prefer:
 
 ```
-res_outbox --database-url="mysql2://root@0.0.0.0:3306/my_database" --redis-url="redis://localhost:6379/0" --log-level=info --split-keys=sidekiq_queue1,sidekiq_queue2
+res_outbox \
+  --database-url="mysql2://root@0.0.0.0:3306/my_database" \
+  --redis-url="redis://localhost:6379/0" \
+  --log-level=info \
+  --split-keys=sidekiq_queue1,sidekiq_queue2
 ```
 
 It is possible to run as many instances as you prefer, but it does not make sense to run more instances than there are different split keys (sidekiq queues), as one process is operating at one moment only one split key.
 
 ### Metrics
 
-It is possible for the outbox process to send metrics to InfluxDB. In order to do that, specify a `--metrics-url` parameter, for example:
+It is possible for the outbox process to send metrics to InfluxDB (this requires influxdb gem in version at least 0.8.1). In order to do that, specify a `--metrics-url` parameter, for example:
 
 ```
 res_outbox --database-url="mysql2://root@0.0.0.0:3306/my_database" \
@@ -59,3 +69,11 @@ res_outbox --database-url="mysql2://root@0.0.0.0:3306/my_database" \
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/RailsEventStore/rails_event_store.
+
+## Releasing
+
+1. Bump version
+2. `make build`
+3. `make push`
+4. `make docker-build`
+5. `make docker-push`

data/lib/ruby_event_store/outbox/cli.rb

@@ -1,51 +1,66 @@
 require "optparse"
-require "ruby_event_store/outbox/version"
-require "ruby_event_store/outbox/consumer"
-require "ruby_event_store/outbox/metrics"
+require_relative "version"
+require_relative "consumer"
+require_relative "metrics"
 
 module RubyEventStore
   module Outbox
     class CLI
-      Options = Struct.new(:database_url, :redis_url, :log_level, :split_keys, :message_format, :batch_size, :metrics_url, :cleanup_strategy)
+      DEFAULTS = {
+        database_url: nil,
+        redis_url: nil,
+        log_level: :warn,
+        split_keys: nil,
+        message_format: 'sidekiq5',
+        batch_size: 100,
+        metrics_url: nil,
+        cleanup_strategy: :none,
+        sleep_on_empty: 0.5
+      }
+      Options = Struct.new(*DEFAULTS.keys)
 
       class Parser
         def self.parse(argv)
-          options = Options.new(nil, nil, :warn, nil, nil, 100, nil, :none)
+          options = Options.new(*DEFAULTS.values)
           OptionParser.new do |option_parser|
             option_parser.banner = "Usage: res_outbox [options]"
 
-            option_parser.on("--database-url DATABASE_URL", "Database where outbox table is stored") do |database_url|
+            option_parser.on("--database-url=DATABASE_URL", "Database where outbox table is stored") do |database_url|
               options.database_url = database_url
             end
 
-            option_parser.on("--redis-url REDIS_URL", "URL to redis database") do |redis_url|
+            option_parser.on("--redis-url=REDIS_URL", "URL to redis database") do |redis_url|
               options.redis_url = redis_url
             end
 
-            option_parser.on("--log-level LOG_LEVEL", [:fatal, :error, :warn, :info, :debug], "Logging level, one of: fatal, error, warn, info, debug") do |log_level|
+            option_parser.on("--log-level=LOG_LEVEL", [:fatal, :error, :warn, :info, :debug], "Logging level, one of: fatal, error, warn, info, debug. Default: warn") do |log_level|
               options.log_level = log_level.to_sym
             end
 
-            option_parser.on("--message-format FORMAT", ["sidekiq5"], "Message format, supported: sidekiq5") do |message_format|
+            option_parser.on("--message-format=FORMAT", ["sidekiq5"], "Message format, supported: sidekiq5. Default: sidekiq5") do |message_format|
               options.message_format = message_format
             end
 
-            option_parser.on("--split-keys=split_keys", Array, "Split keys which should be handled, all if not specified") do |split_keys|
+            option_parser.on("--split-keys=SPLIT_KEYS", Array, "Split keys which should be handled, all if not specified") do |split_keys|
               options.split_keys = split_keys if !split_keys.empty?
             end
 
-            option_parser.on("--batch-size BATCH_SIZE", Integer, "Amount of records fetched in one fetch. Bigger value means more duplicated messages when network problems occur.") do |batch_size|
+            option_parser.on("--batch-size=BATCH_SIZE", Integer, "Amount of records fetched in one fetch. Bigger value means more duplicated messages when network problems occur. Default: 100") do |batch_size|
               options.batch_size = batch_size
             end
 
-            option_parser.on("--metrics-url METRICS_URL", "URI to metrics collector") do |metrics_url|
+            option_parser.on("--metrics-url=METRICS_URL", "URI to metrics collector, optional") do |metrics_url|
               options.metrics_url = metrics_url
             end
 
-            option_parser.on("--cleanup CLEANUP_STRATEGY", "A strategy for cleaning old records. One of: none or iso8601 duration format how old enqueued records should be removed") do |cleanup_strategy|
+            option_parser.on("--cleanup=STRATEGY", "A strategy for cleaning old records. One of: none or iso8601 duration format how old enqueued records should be removed. Default: none") do |cleanup_strategy|
               options.cleanup_strategy = cleanup_strategy
             end
 
+            option_parser.on("--sleep-on-empty=SLEEP_TIME", Float, "How long to sleep before next check when there was nothing to do. Default: 0.5") do |sleep_on_empty|
+              options.sleep_on_empty = sleep_on_empty
+            end
+
             option_parser.on_tail("--version", "Show version") do
               puts VERSION
               exit
@@ -72,6 +87,7 @@ module RubyEventStore
           database_url: options.database_url,
           redis_url: options.redis_url,
           cleanup: options.cleanup_strategy,
+          sleep_on_empty: options.sleep_on_empty,
         )
         metrics = Metrics.from_url(options.metrics_url)
         outbox_consumer = RubyEventStore::Outbox::Consumer.new(

data/lib/ruby_event_store/outbox/consumer.rb

@@ -1,17 +1,16 @@
 require "logger"
 require "redis"
 require "active_record"
-require "ruby_event_store/outbox/repository"
-require "ruby_event_store/outbox/sidekiq5_format"
-require "ruby_event_store/outbox/sidekiq_processor"
-require "ruby_event_store/outbox/fetch_specification"
-require "ruby_event_store/outbox/cleanup_strategies/none"
-require "ruby_event_store/outbox/cleanup_strategies/clean_old_enqueued"
+require_relative "repository"
+require_relative "sidekiq5_format"
+require_relative "sidekiq_processor"
+require_relative "fetch_specification"
+require_relative "cleanup_strategies/none"
+require_relative "cleanup_strategies/clean_old_enqueued"
 
 module RubyEventStore
   module Outbox
     class Consumer
-      SLEEP_TIME_WHEN_NOTHING_TO_DO = 0.5
       MAXIMUM_BATCH_FETCHES_IN_ONE_LOCK = 10
 
       class Configuration
@@ -21,7 +20,8 @@ module RubyEventStore
           batch_size:,
           database_url:,
           redis_url:,
-          cleanup:
+          cleanup:,
+          sleep_on_empty:
         )
           @split_keys = split_keys
           @message_format = message_format
@@ -29,6 +29,7 @@ module RubyEventStore
           @database_url = database_url
           @redis_url = redis_url
           @cleanup = cleanup
+          @sleep_on_empty = sleep_on_empty
           freeze
         end
 
@@ -39,11 +40,12 @@ module RubyEventStore
             batch_size: overriden_options.fetch(:batch_size, batch_size),
             database_url: overriden_options.fetch(:database_url, database_url),
             redis_url: overriden_options.fetch(:redis_url, redis_url),
-            cleanup: overriden_options.fetch(:cleanup, cleanup)
+            cleanup: overriden_options.fetch(:cleanup, cleanup),
+            sleep_on_empty: overriden_options.fetch(:sleep_on_empty, sleep_on_empty)
           )
         end
 
-        attr_reader :split_keys, :message_format, :batch_size, :database_url, :redis_url, :cleanup
+        attr_reader :split_keys, :message_format, :batch_size, :database_url, :redis_url, :cleanup, :sleep_on_empty
       end
 
       def initialize(consumer_uuid, configuration, clock: Time, logger:, metrics:)
@@ -52,6 +54,7 @@ module RubyEventStore
         @logger = logger
         @metrics = metrics
         @batch_size = configuration.batch_size
+        @sleep_on_empty = configuration.sleep_on_empty
         @consumer_uuid = consumer_uuid
 
         raise "Unknown format" if configuration.message_format != SIDEKIQ5_FORMAT
@@ -79,7 +82,7 @@ module RubyEventStore
           was_something_changed = one_loop
           if !was_something_changed
             STDOUT.flush
-            sleep SLEEP_TIME_WHEN_NOTHING_TO_DO
+            sleep sleep_on_empty
           end
         end
         logger.info "Gracefully shutting down"
@@ -153,7 +156,7 @@ module RubyEventStore
       end
 
       private
-      attr_reader :split_keys, :logger, :batch_size, :metrics, :processor, :consumer_uuid, :repository, :cleanup_strategy
+      attr_reader :split_keys, :logger, :batch_size, :metrics, :processor, :consumer_uuid, :repository, :cleanup_strategy, :sleep_on_empty
 
       def obtain_lock_for_process(fetch_specification)
         result = repository.obtain_lock_for_process(fetch_specification, consumer_uuid, clock: @clock)

data/lib/ruby_event_store/outbox/metrics/influx.rb

@@ -6,14 +6,11 @@ module RubyEventStore
       class Influx
         def initialize(url)
           uri = URI.parse(url)
-          params = CGI.parse(uri.query || "")
           options = {
             url: url,
             async: true,
             time_precision: 'ns',
           }
-          options[:username] = params.fetch("username").first if params.key?("username")
-          options[:password] = params.fetch("password").first if params.key?("password")
           @influxdb_client = InfluxDB::Client.new(**options)
         end
 

data/lib/ruby_event_store/outbox/metrics.rb

@@ -3,10 +3,10 @@ module RubyEventStore
     module Metrics
       def self.from_url(metrics_url)
         if metrics_url.nil?
-          require "ruby_event_store/outbox/metrics/null"
+          require_relative "metrics/null"
           Null.new
         else
-          require "ruby_event_store/outbox/metrics/influx"
+          require_relative "metrics/influx"
           Influx.new(metrics_url)
         end
       end

data/lib/ruby_event_store/outbox/sidekiq_processor.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require "ruby_event_store/outbox/sidekiq5_format"
+require_relative "sidekiq5_format"
 
 module RubyEventStore
   module Outbox

data/lib/ruby_event_store/outbox/sidekiq_producer.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 require 'sidekiq'
-require "ruby_event_store/outbox/repository"
+require_relative "repository"
 
 module RubyEventStore
   module Outbox

data/lib/ruby_event_store/outbox/sidekiq_scheduler.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require "ruby_event_store/outbox/sidekiq_producer"
+require_relative "sidekiq_producer"
 
 module RubyEventStore
   module Outbox

data/lib/ruby_event_store/outbox/version.rb

@@ -2,6 +2,6 @@
 
 module RubyEventStore
   module Outbox
-    VERSION = "0.0.16"
+    VERSION = "0.0.17"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby_event_store-outbox
 version: !ruby/object:Gem::Version
-  version: 0.0.16
+  version: 0.0.17
 platform: ruby
 authors:
 - Arkency
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-01-04 00:00:00.000000000 Z
+date: 2021-09-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby_event_store
@@ -39,12 +39,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '5.2'
 description: 
-email:
-- dev@arkency.com
+email: dev@arkency.com
 executables:
 - res_outbox
 extensions: []
-extra_rdoc_files: []
+extra_rdoc_files:
+- README.md
 files:
 - README.md
 - bin/res_outbox
@@ -55,14 +55,12 @@ files:
 - lib/ruby_event_store/outbox/cleanup_strategies/none.rb
 - lib/ruby_event_store/outbox/cli.rb
 - lib/ruby_event_store/outbox/consumer.rb
-- lib/ruby_event_store/outbox/consumer_process.rb
 - lib/ruby_event_store/outbox/fetch_specification.rb
 - lib/ruby_event_store/outbox/metrics.rb
 - lib/ruby_event_store/outbox/metrics/influx.rb
 - lib/ruby_event_store/outbox/metrics/null.rb
 - lib/ruby_event_store/outbox/repository.rb
 - lib/ruby_event_store/outbox/sidekiq5_format.rb
-- lib/ruby_event_store/outbox/sidekiq_message_handler.rb
 - lib/ruby_event_store/outbox/sidekiq_processor.rb
 - lib/ruby_event_store/outbox/sidekiq_producer.rb
 - lib/ruby_event_store/outbox/sidekiq_scheduler.rb
@@ -71,8 +69,7 @@ homepage: https://railseventstore.org
 licenses:
 - MIT
 metadata:
-  homepage_uri: https://railseventstore.org/
-  changelog_uri: https://github.com/RailsEventStore/rails_event_store/releases
+  homepage_uri: https://railseventstore.org
   source_code_uri: https://github.com/RailsEventStore/rails_event_store
   bug_tracker_uri: https://github.com/RailsEventStore/rails_event_store/issues
 post_install_message: 
@@ -83,14 +80,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '2.6'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.0.6
 signing_key: 
 specification_version: 4
 summary: Active Record based outbox for Ruby Event Store

data/lib/ruby_event_store/outbox/consumer_process.rb

@@ -1,6 +0,0 @@
-module RubyEventStore
-  module Outbox
-    class ConsumerProcess
-    end
-  end
-end

data/lib/ruby_event_store/outbox/sidekiq_message_handler.rb

@@ -1,19 +0,0 @@
-require "sidekiq"
-
-module RubyEventStore
-  module Outbox
-    class SidekiqMessageHandler
-      def initialize
-        @sidekiq = Sidekiq::Client.new(Sidekiq.redis_pool)
-      end
-
-      def init
-      end
-
-      def handle_one_record(now, record)
-        hash_payload = JSON.parse(record.payload)
-        @sidekiq.__send__(:raw_push, [hash_payload])
-      end
-    end
-  end
-end