ruby_event_store-outbox 0.0.16 → 0.0.20

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '008b96c4b43e3cbfc23d0aa8c9ebb160f0adbfcfc82a976967f1c56338346cc0'
-  data.tar.gz: e65157c0638ecec0a4967a251e820979d18eb92e55212f107be95c23d5a4315b
+  metadata.gz: 151cc4dc1f5abead45b181c09dd52ade1850944b1f240cf222778ab7891ca146
+  data.tar.gz: 95ff963b0957cd62a9456764f6f87f3e3dac8e4484ebd886c8ac3c5434db3026
 SHA512:
-  metadata.gz: f0181744b8c621bc1f0b20d6d4a65ac779db7ba48a80c9f1a02f39208762aa82e5579e745f20cd35f2b54992876065f803e04e24c1c239abe22641f927327070
-  data.tar.gz: 453ce565584436b1b7dd145c738a0d50e95821cb8b8f5d9c00f0d70d2a8ea88e585402de534388459d04853d293c0bcbf7027285d31bbd00aeaef09471cf666d
+  metadata.gz: a894a74e5a4db50a5f9e3d030ea97ea725bbf0098f79d986d8ab7ead14dcfcf8b8daca4187d87e16a836ca40039bd6d950fa8dc55ca47aa8d3977d2bf69b078e
+  data.tar.gz: 5a251df0856e1dadae4868871fb3a428fd4b9ccb8f4d17dfcc9d623559c9c4ea5cfb58ec566589ed3358724e6063f9f40a43d62d15b3bde5bc6814bd73ee0758

data/README.md

@@ -14,6 +14,12 @@ Add to your gemfile in application:
 gem "ruby_event_store-outbox"
 ```
 
+Generate and execute the migration adding necessary tables. If it's needed, change the type of the `payload` column to `mediumbinary` or `longbinary`.
+
+```
+bin/rails generate ruby_event_store:outbox:migration
+```
+
 In your event store configuration, as a dispatcher use `RubyEventStore::ImmediateAsyncDispatcher` with `RubyEventStore::Outbox::SidekiqScheduler`, for example:
 
 ```ruby
@@ -38,14 +44,18 @@ end
 Run following process in any way you prefer:
 
 ```
-res_outbox --database-url="mysql2://root@0.0.0.0:3306/my_database" --redis-url="redis://localhost:6379/0" --log-level=info --split-keys=sidekiq_queue1,sidekiq_queue2
+res_outbox \
+  --database-url="mysql2://root@0.0.0.0:3306/my_database" \
+  --redis-url="redis://localhost:6379/0" \
+  --log-level=info \
+  --split-keys=sidekiq_queue1,sidekiq_queue2
 ```
 
 It is possible to run as many instances as you prefer, but it does not make sense to run more instances than there are different split keys (sidekiq queues), as one process is operating at one moment only one split key.
 
 ### Metrics
 
-It is possible for the outbox process to send metrics to InfluxDB. In order to do that, specify a `--metrics-url` parameter, for example:
+It is possible for the outbox process to send metrics to InfluxDB (this requires influxdb gem in version at least 0.8.1). In order to do that, specify a `--metrics-url` parameter, for example:
 
 ```
 res_outbox --database-url="mysql2://root@0.0.0.0:3306/my_database" \
@@ -59,3 +69,11 @@ res_outbox --database-url="mysql2://root@0.0.0.0:3306/my_database" \
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/RailsEventStore/rails_event_store.
+
+## Releasing
+
+1. Bump version
+2. `make build`
+3. `make push`
+4. `make docker-build`
+5. `make docker-push`

data/lib/ruby_event_store/outbox/cleanup_strategies/none.rb

@@ -6,6 +6,7 @@ module RubyEventStore
         end
 
         def call(_fetch_specification)
+          :ok
         end
       end
     end

data/lib/ruby_event_store/outbox/cli.rb

@@ -1,51 +1,66 @@
 require "optparse"
-require "ruby_event_store/outbox/version"
-require "ruby_event_store/outbox/consumer"
-require "ruby_event_store/outbox/metrics"
+require_relative "version"
+require_relative "consumer"
+require_relative "metrics"
 
 module RubyEventStore
   module Outbox
     class CLI
-      Options = Struct.new(:database_url, :redis_url, :log_level, :split_keys, :message_format, :batch_size, :metrics_url, :cleanup_strategy)
+      DEFAULTS = {
+        database_url: nil,
+        redis_url: nil,
+        log_level: :warn,
+        split_keys: nil,
+        message_format: 'sidekiq5',
+        batch_size: 100,
+        metrics_url: nil,
+        cleanup_strategy: :none,
+        sleep_on_empty: 0.5
+      }
+      Options = Struct.new(*DEFAULTS.keys)
 
       class Parser
         def self.parse(argv)
-          options = Options.new(nil, nil, :warn, nil, nil, 100, nil, :none)
+          options = Options.new(*DEFAULTS.values)
           OptionParser.new do |option_parser|
             option_parser.banner = "Usage: res_outbox [options]"
 
-            option_parser.on("--database-url DATABASE_URL", "Database where outbox table is stored") do |database_url|
+            option_parser.on("--database-url=DATABASE_URL", "Database where outbox table is stored") do |database_url|
               options.database_url = database_url
             end
 
-            option_parser.on("--redis-url REDIS_URL", "URL to redis database") do |redis_url|
+            option_parser.on("--redis-url=REDIS_URL", "URL to redis database") do |redis_url|
               options.redis_url = redis_url
             end
 
-            option_parser.on("--log-level LOG_LEVEL", [:fatal, :error, :warn, :info, :debug], "Logging level, one of: fatal, error, warn, info, debug") do |log_level|
+            option_parser.on("--log-level=LOG_LEVEL", [:fatal, :error, :warn, :info, :debug], "Logging level, one of: fatal, error, warn, info, debug. Default: warn") do |log_level|
               options.log_level = log_level.to_sym
             end
 
-            option_parser.on("--message-format FORMAT", ["sidekiq5"], "Message format, supported: sidekiq5") do |message_format|
+            option_parser.on("--message-format=FORMAT", ["sidekiq5"], "Message format, supported: sidekiq5. Default: sidekiq5") do |message_format|
               options.message_format = message_format
             end
 
-            option_parser.on("--split-keys=split_keys", Array, "Split keys which should be handled, all if not specified") do |split_keys|
+            option_parser.on("--split-keys=SPLIT_KEYS", Array, "Split keys which should be handled, all if not specified") do |split_keys|
               options.split_keys = split_keys if !split_keys.empty?
             end
 
-            option_parser.on("--batch-size BATCH_SIZE", Integer, "Amount of records fetched in one fetch. Bigger value means more duplicated messages when network problems occur.") do |batch_size|
+            option_parser.on("--batch-size=BATCH_SIZE", Integer, "Amount of records fetched in one fetch. Bigger value means more duplicated messages when network problems occur. Default: 100") do |batch_size|
               options.batch_size = batch_size
             end
 
-            option_parser.on("--metrics-url METRICS_URL", "URI to metrics collector") do |metrics_url|
+            option_parser.on("--metrics-url=METRICS_URL", "URI to metrics collector, optional") do |metrics_url|
               options.metrics_url = metrics_url
             end
 
-            option_parser.on("--cleanup CLEANUP_STRATEGY", "A strategy for cleaning old records. One of: none or iso8601 duration format how old enqueued records should be removed") do |cleanup_strategy|
+            option_parser.on("--cleanup=STRATEGY", "A strategy for cleaning old records. One of: none or iso8601 duration format how old enqueued records should be removed. Default: none") do |cleanup_strategy|
               options.cleanup_strategy = cleanup_strategy
             end
 
+            option_parser.on("--sleep-on-empty=SLEEP_TIME", Float, "How long to sleep before next check when there was nothing to do. Default: 0.5") do |sleep_on_empty|
+              options.sleep_on_empty = sleep_on_empty
+            end
+
             option_parser.on_tail("--version", "Show version") do
               puts VERSION
               exit
@@ -72,6 +87,7 @@ module RubyEventStore
           database_url: options.database_url,
           redis_url: options.redis_url,
           cleanup: options.cleanup_strategy,
+          sleep_on_empty: options.sleep_on_empty,
         )
         metrics = Metrics.from_url(options.metrics_url)
         outbox_consumer = RubyEventStore::Outbox::Consumer.new(

data/lib/ruby_event_store/outbox/consumer.rb

@@ -1,17 +1,16 @@
 require "logger"
 require "redis"
 require "active_record"
-require "ruby_event_store/outbox/repository"
-require "ruby_event_store/outbox/sidekiq5_format"
-require "ruby_event_store/outbox/sidekiq_processor"
-require "ruby_event_store/outbox/fetch_specification"
-require "ruby_event_store/outbox/cleanup_strategies/none"
-require "ruby_event_store/outbox/cleanup_strategies/clean_old_enqueued"
+require_relative "repository"
+require_relative "sidekiq5_format"
+require_relative "sidekiq_processor"
+require_relative "fetch_specification"
+require_relative "cleanup_strategies/none"
+require_relative "cleanup_strategies/clean_old_enqueued"
 
 module RubyEventStore
   module Outbox
     class Consumer
-      SLEEP_TIME_WHEN_NOTHING_TO_DO = 0.5
       MAXIMUM_BATCH_FETCHES_IN_ONE_LOCK = 10
 
       class Configuration
@@ -21,7 +20,8 @@ module RubyEventStore
           batch_size:,
           database_url:,
           redis_url:,
-          cleanup:
+          cleanup:,
+          sleep_on_empty:
         )
           @split_keys = split_keys
           @message_format = message_format
@@ -29,6 +29,7 @@ module RubyEventStore
           @database_url = database_url
           @redis_url = redis_url
           @cleanup = cleanup
+          @sleep_on_empty = sleep_on_empty
           freeze
         end
 
@@ -39,11 +40,12 @@ module RubyEventStore
             batch_size: overriden_options.fetch(:batch_size, batch_size),
             database_url: overriden_options.fetch(:database_url, database_url),
             redis_url: overriden_options.fetch(:redis_url, redis_url),
-            cleanup: overriden_options.fetch(:cleanup, cleanup)
+            cleanup: overriden_options.fetch(:cleanup, cleanup),
+            sleep_on_empty: overriden_options.fetch(:sleep_on_empty, sleep_on_empty)
           )
         end
 
-        attr_reader :split_keys, :message_format, :batch_size, :database_url, :redis_url, :cleanup
+        attr_reader :split_keys, :message_format, :batch_size, :database_url, :redis_url, :cleanup, :sleep_on_empty
       end
 
       def initialize(consumer_uuid, configuration, clock: Time, logger:, metrics:)
@@ -52,6 +54,7 @@ module RubyEventStore
         @logger = logger
         @metrics = metrics
         @batch_size = configuration.batch_size
+        @sleep_on_empty = configuration.sleep_on_empty
         @consumer_uuid = consumer_uuid
 
         raise "Unknown format" if configuration.message_format != SIDEKIQ5_FORMAT
@@ -79,7 +82,7 @@ module RubyEventStore
           was_something_changed = one_loop
           if !was_something_changed
             STDOUT.flush
-            sleep SLEEP_TIME_WHEN_NOTHING_TO_DO
+            sleep sleep_on_empty
           end
         end
         logger.info "Gracefully shutting down"
@@ -145,7 +148,7 @@ module RubyEventStore
 
         release_lock_for_process(fetch_specification)
 
-        cleanup_strategy.call(fetch_specification)
+        cleanup(fetch_specification)
 
         processor.after_batch
 
@@ -153,7 +156,7 @@ module RubyEventStore
       end
 
       private
-      attr_reader :split_keys, :logger, :batch_size, :metrics, :processor, :consumer_uuid, :repository, :cleanup_strategy
+      attr_reader :split_keys, :logger, :batch_size, :metrics, :processor, :consumer_uuid, :repository, :cleanup_strategy, :sleep_on_empty
 
       def obtain_lock_for_process(fetch_specification)
         result = repository.obtain_lock_for_process(fetch_specification, consumer_uuid, clock: @clock)
@@ -161,24 +164,23 @@ module RubyEventStore
         when :deadlocked
           logger.warn "Obtaining lock for split_key '#{fetch_specification.split_key}' failed (deadlock)"
           metrics.write_operation_result("obtain", "deadlocked")
-          return false
+          false
         when :lock_timeout
           logger.warn "Obtaining lock for split_key '#{fetch_specification.split_key}' failed (lock timeout)"
           metrics.write_operation_result("obtain", "lock_timeout")
-          return false
+          false
         when :taken
           logger.debug "Obtaining lock for split_key '#{fetch_specification.split_key}' unsuccessful (taken)"
           metrics.write_operation_result("obtain", "taken")
-          return false
+          false
         else
-          return result
+          result
         end
       end
 
       def release_lock_for_process(fetch_specification)
         result = repository.release_lock_for_process(fetch_specification, consumer_uuid)
         case result
-        when :ok
         when :deadlocked
           logger.warn "Releasing lock for split_key '#{fetch_specification.split_key}' failed (deadlock)"
           metrics.write_operation_result("release", "deadlocked")
@@ -188,8 +190,6 @@ module RubyEventStore
         when :not_taken_by_this_process
           logger.debug "Releasing lock for split_key '#{fetch_specification.split_key}' failed (not taken by this process)"
           metrics.write_operation_result("release", "not_taken_by_this_process")
-        else
-          raise "Unexpected result #{result}"
         end
       end
 
@@ -215,6 +215,18 @@ module RubyEventStore
         end
       end
 
+      def cleanup(fetch_specification)
+        result = cleanup_strategy.call(fetch_specification)
+        case result
+        when :deadlocked
+          logger.warn "Cleanup for split_key '#{fetch_specification.split_key}' failed (deadlock)"
+          metrics.write_operation_result("cleanup", "deadlocked")
+        when :lock_timeout
+          logger.warn "Cleanup for split_key '#{fetch_specification.split_key}' failed (lock timeout)"
+          metrics.write_operation_result("cleanup", "lock_timeout")
+        end
+      end
+
       def prepare_traps
         Signal.trap("INT") do
           initiate_graceful_shutdown

data/lib/ruby_event_store/outbox/legacy_sidekiq_scheduler.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require_relative "sidekiq_producer"
+
+module RubyEventStore
+  module Outbox
+    class LegacySidekiqScheduler
+      def initialize
+        @sidekiq_producer = SidekiqProducer.new
+      end
+
+      def call(klass, serialized_record)
+        sidekiq_producer.call(klass, [serialized_record])
+      end
+
+      def verify(subscriber)
+        Class === subscriber && subscriber.respond_to?(:through_outbox?) && subscriber.through_outbox?
+      end
+
+      private
+      attr_reader :sidekiq_producer
+    end
+  end
+end

data/lib/ruby_event_store/outbox/metrics/influx.rb

@@ -6,14 +6,11 @@ module RubyEventStore
       class Influx
         def initialize(url)
           uri = URI.parse(url)
-          params = CGI.parse(uri.query || "")
           options = {
             url: url,
             async: true,
             time_precision: 'ns',
           }
-          options[:username] = params.fetch("username").first if params.key?("username")
-          options[:password] = params.fetch("password").first if params.key?("password")
           @influxdb_client = InfluxDB::Client.new(**options)
         end
 

data/lib/ruby_event_store/outbox/metrics/test.rb

@@ -0,0 +1,24 @@
+require 'influxdb'
+
+module RubyEventStore
+  module Outbox
+    module Metrics
+      class Test
+        def initialize
+          @operation_results = []
+          @queue_stats = []
+        end
+
+        def write_operation_result(operation, result)
+          @operation_results << { operation: operation, result: result }
+        end
+
+        def write_point_queue(enqueued: 0, failed: 0, remaining: 0, format: nil, split_key: nil)
+          @queue_stats << { enqueued: enqueued, failed: failed, remaining: remaining, format: format, split_key: split_key}
+        end
+
+        attr_reader :operation_results, :queue_stats
+      end
+    end
+  end
+end

data/lib/ruby_event_store/outbox/metrics.rb

@@ -3,10 +3,10 @@ module RubyEventStore
     module Metrics
       def self.from_url(metrics_url)
         if metrics_url.nil?
-          require "ruby_event_store/outbox/metrics/null"
+          require_relative "metrics/null"
           Null.new
         else
-          require "ruby_event_store/outbox/metrics/influx"
+          require_relative "metrics/influx"
           Influx.new(metrics_url)
         end
       end

data/lib/ruby_event_store/outbox/repository.rb

@@ -36,7 +36,7 @@ module RubyEventStore
           transaction do
             l = get_lock_record(fetch_specification)
 
-            if l.recently_locked?
+            if l.recently_locked?(clock: clock)
               :taken
             else
               l.update!(
@@ -89,8 +89,8 @@ module RubyEventStore
           locked_by.eql?(process_uuid)
         end
 
-        def recently_locked?
-          locked_by && locked_at > RECENTLY_LOCKED_DURATION.ago
+        def recently_locked?(clock:)
+          locked_by && locked_at > RECENTLY_LOCKED_DURATION.ago(clock.now)
         end
 
         def fetch_specification
@@ -147,6 +147,11 @@ module RubyEventStore
           .for_fetch_specification(fetch_specification)
           .where("enqueued_at < ?", duration.ago)
           .delete_all
+        :ok
+      rescue ActiveRecord::Deadlocked
+        :deadlocked
+      rescue ActiveRecord::LockWaitTimeout
+        :lock_timeout
       end
     end
   end

data/lib/ruby_event_store/outbox/sidekiq_processor.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require "ruby_event_store/outbox/sidekiq5_format"
+require_relative "sidekiq5_format"
 
 module RubyEventStore
   module Outbox

data/lib/ruby_event_store/outbox/sidekiq_producer.rb

@@ -1,7 +1,8 @@
 # frozen_string_literal: true
 
 require 'sidekiq'
-require "ruby_event_store/outbox/repository"
+require_relative 'sidekiq5_format'
+require_relative "repository"
 
 module RubyEventStore
   module Outbox
@@ -10,7 +11,7 @@ module RubyEventStore
         sidekiq_client = Sidekiq::Client.new(Sidekiq.redis_pool)
         item = {
           'class' => klass,
-          'args' => args,
+          'args' => args.map(&:to_h),
         }
         normalized_item = sidekiq_client.__send__(:normalize_item, item)
         payload = sidekiq_client.__send__(:process_single, normalized_item.fetch('class'), normalized_item)

data/lib/ruby_event_store/outbox/sidekiq_scheduler.rb

@@ -1,16 +1,17 @@
 # frozen_string_literal: true
 
-require "ruby_event_store/outbox/sidekiq_producer"
+require_relative "sidekiq_producer"
 
 module RubyEventStore
   module Outbox
     class SidekiqScheduler
-      def initialize
+      def initialize(serializer: YAML)
+        @serializer = serializer
         @sidekiq_producer = SidekiqProducer.new
       end
 
-      def call(klass, serialized_record)
-        sidekiq_producer.call(klass, [serialized_record.to_h])
+      def call(klass, record)
+        sidekiq_producer.call(klass, [record.serialize(serializer)])
       end
 
       def verify(subscriber)
@@ -18,7 +19,7 @@ module RubyEventStore
       end
 
       private
-      attr_reader :sidekiq_producer
+      attr_reader :serializer, :sidekiq_producer
     end
   end
 end

data/lib/ruby_event_store/outbox/version.rb

@@ -2,6 +2,6 @@
 
 module RubyEventStore
   module Outbox
-    VERSION = "0.0.16"
+    VERSION = "0.0.20"
   end
 end

data/lib/ruby_event_store/outbox.rb

@@ -8,5 +8,5 @@ end
 require_relative 'outbox/fetch_specification'
 require_relative 'outbox/repository'
 require_relative 'outbox/sidekiq_scheduler'
-require_relative 'outbox/consumer'
+require_relative 'outbox/legacy_sidekiq_scheduler'
 require_relative 'outbox/version'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby_event_store-outbox
 version: !ruby/object:Gem::Version
-  version: 0.0.16
+  version: 0.0.20
 platform: ruby
 authors:
 - Arkency
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-01-04 00:00:00.000000000 Z
+date: 2021-12-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby_event_store
@@ -38,13 +38,13 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '5.2'
-description: 
-email:
-- dev@arkency.com
+description:
+email: dev@arkency.com
 executables:
 - res_outbox
 extensions: []
-extra_rdoc_files: []
+extra_rdoc_files:
+- README.md
 files:
 - README.md
 - bin/res_outbox
@@ -55,14 +55,14 @@ files:
 - lib/ruby_event_store/outbox/cleanup_strategies/none.rb
 - lib/ruby_event_store/outbox/cli.rb
 - lib/ruby_event_store/outbox/consumer.rb
-- lib/ruby_event_store/outbox/consumer_process.rb
 - lib/ruby_event_store/outbox/fetch_specification.rb
+- lib/ruby_event_store/outbox/legacy_sidekiq_scheduler.rb
 - lib/ruby_event_store/outbox/metrics.rb
 - lib/ruby_event_store/outbox/metrics/influx.rb
 - lib/ruby_event_store/outbox/metrics/null.rb
+- lib/ruby_event_store/outbox/metrics/test.rb
 - lib/ruby_event_store/outbox/repository.rb
 - lib/ruby_event_store/outbox/sidekiq5_format.rb
-- lib/ruby_event_store/outbox/sidekiq_message_handler.rb
 - lib/ruby_event_store/outbox/sidekiq_processor.rb
 - lib/ruby_event_store/outbox/sidekiq_producer.rb
 - lib/ruby_event_store/outbox/sidekiq_scheduler.rb
@@ -71,11 +71,11 @@ homepage: https://railseventstore.org
 licenses:
 - MIT
 metadata:
-  homepage_uri: https://railseventstore.org/
-  changelog_uri: https://github.com/RailsEventStore/rails_event_store/releases
+  homepage_uri: https://railseventstore.org
   source_code_uri: https://github.com/RailsEventStore/rails_event_store
   bug_tracker_uri: https://github.com/RailsEventStore/rails_event_store/issues
-post_install_message: 
+  rubygems_mfa_required: 'true'
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -83,15 +83,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '2.6'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.2.22
+signing_key:
 specification_version: 4
 summary: Active Record based outbox for Ruby Event Store
 test_files: []

data/lib/ruby_event_store/outbox/consumer_process.rb

@@ -1,6 +0,0 @@
-module RubyEventStore
-  module Outbox
-    class ConsumerProcess
-    end
-  end
-end

data/lib/ruby_event_store/outbox/sidekiq_message_handler.rb

@@ -1,19 +0,0 @@
-require "sidekiq"
-
-module RubyEventStore
-  module Outbox
-    class SidekiqMessageHandler
-      def initialize
-        @sidekiq = Sidekiq::Client.new(Sidekiq.redis_pool)
-      end
-
-      def init
-      end
-
-      def handle_one_record(now, record)
-        hash_payload = JSON.parse(record.payload)
-        @sidekiq.__send__(:raw_push, [hash_payload])
-      end
-    end
-  end
-end