ruby_event_store-outbox 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: '08a5fc74b45a962065d79e01a6347919882bd97903ed30d5c722069d3e7ee617'
+  data.tar.gz: 4238ad6ce88f09744ee98feb3b513ec6e3903db92579494109f0575098f23be5
+SHA512:
+  metadata.gz: 7939074f5b2fc85d9f4795103e0aaf38a59ca8e10b27f746a265b611fa4eda84a23806b1009845ce918dc61669bce4f5887d71c0139f152e4b04a9e7102a1fd9
+  data.tar.gz: 7c6f7168c15b2c81cc462e63dd528ef1415b690eae8b3f49344bd2fdb8681338721ef2a2c0b511d75c265c1590031d799d268417ab69694714624fe0246e1e8a

data/README.md

@@ -0,0 +1,44 @@
+# Ruby Event Store Outbox
+
+Very much work in progress.
+
+
+## Installation (app)
+
+Add to your gemfile in application:
+
+```ruby
+gem "ruby_event_store-outbox"
+```
+
+In your event store configuration, as a dispatcher use `RubyEventStore::ImmediateAsyncDispatcher` with `RubyEventStore::Outbox::SidekiqScheduler`, for example:
+
+```ruby
+
+RailsEventStore::Client.new(
+  dispatcher: RailsEventStore::ImmediateAsyncDispatcher.new(scheduler: RubyEventStore::Outbox::SidekiqScheduler.new),
+  ...
+)
+```
+
+Additionally, your handler's `through_outbox?` method should return `true`, for example:
+
+```ruby
+class SomeHandler
+  def self.through_outbox?; true; end
+end
+```
+
+
+## Installation (outbox process)
+
+Run following process in any way you prefer:
+
+```
+res_outbox --database-url="mysql2://root@0.0.0.0:3306/my_database" --redis-url="redis://localhost:6379/0" --log-level=info
+```
+
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/RailsEventStore/rails_event_store.

data/bin/res_outbox

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+require "ruby_event_store/outbox/cli"
+
+cli = RubyEventStore::Outbox::CLI.new
+cli.run(ARGV)

data/lib/generators/ruby_event_store/outbox/migration_generator.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+begin
+  require 'rails/generators'
+rescue LoadError
+end
+
+module RubyEventStore
+  module Outbox
+    class MigrationGenerator < Rails::Generators::Base
+      source_root File.expand_path(File.join(File.dirname(__FILE__), './templates'))
+
+      def create_migration
+        template "create_event_store_outbox_template.rb", "db/migrate/#{timestamp}_create_event_store_outbox.rb"
+      end
+
+      private
+
+      def rails_version
+        Rails::VERSION::STRING
+      end
+
+      def migration_version
+        return nil if Gem::Version.new(rails_version) < Gem::Version.new("5.0.0")
+        "[4.2]"
+      end
+
+      def timestamp
+        Time.now.strftime("%Y%m%d%H%M%S")
+      end
+    end
+  end
+end if defined?(Rails::Generators::Base)

data/lib/generators/ruby_event_store/outbox/templates/create_event_store_outbox_template.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+class CreateEventStoreOutbox < ActiveRecord::Migration<%= migration_version %>
+  def change
+    create_table(:event_store_outbox, force: false) do |t|
+      t.string :split_key, null: true
+      t.string :format, null: false
+      t.binary :payload, null: false
+      t.datetime :created_at, null: false
+      t.datetime :enqueued_at, null: true
+    end
+    add_index :event_store_outbox, [:format, :enqueued_at, :split_key], name: "index_event_store_outbox_for_pool"
+    add_index :event_store_outbox, [:created_at, :enqueued_at], name: "index_event_store_outbox_for_clear"
+  end
+end

data/lib/ruby_event_store/outbox.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module RubyEventStore
+  module Outbox
+  end
+end
+
+require_relative 'outbox/record'
+require_relative 'outbox/sidekiq_scheduler'
+require_relative 'outbox/consumer'
+require_relative 'outbox/version'

data/lib/ruby_event_store/outbox/cli.rb

@@ -0,0 +1,60 @@
+require "optparse"
+require "ruby_event_store/outbox/version"
+require "ruby_event_store/outbox/consumer"
+
+module RubyEventStore
+  module Outbox
+    class CLI
+      Options = Struct.new(:database_url, :redis_url, :log_level, :split_keys, :message_format)
+
+      class Parser
+        def self.parse(argv)
+          options = Options.new(nil, nil, :warn, nil, nil)
+          OptionParser.new do |option_parser|
+            option_parser.banner = "Usage: res_outbox [options]"
+
+            option_parser.on("--database-url DATABASE_URL", "Database where outbox table is stored") do |database_url|
+              options.database_url = database_url
+            end
+
+            option_parser.on("--redis-url REDIS_URL", "URL to redis database") do |redis_url|
+              options.redis_url = redis_url
+            end
+
+            option_parser.on("--log-level LOG_LEVEL", [:fatal, :error, :warn, :info, :debug], "Logging level, one of: fatal, error, warn, info, debug") do |log_level|
+              options.log_level = log_level.to_sym
+            end
+
+            option_parser.on("--message-format FORMAT", ["sidekiq5"], "Message format, supported: sidekiq5") do |message_format|
+              options.message_format = message_format
+            end
+
+            option_parser.on("--split-keys=split_keys", Array, "Split keys which should be handled, all if not specified") do |split_keys|
+              options.split_keys = split_keys if !split_keys.empty?
+            end
+
+            option_parser.on_tail("--version", "Show version") do
+              puts VERSION
+              exit
+            end
+          end.parse(argv)
+          return options
+        end
+      end
+
+      def run(argv)
+        options = Parser.parse(argv)
+        logger = Logger.new(STDOUT, level: options.log_level, progname: "RES-Outbox")
+        outbox_consumer = RubyEventStore::Outbox::Consumer.new(
+          options.message_format,
+          options.split_keys,
+          database_url: options.database_url,
+          redis_url: options.redis_url,
+          logger: logger,
+        )
+        outbox_consumer.init
+        outbox_consumer.run
+      end
+    end
+  end
+end

data/lib/ruby_event_store/outbox/consumer.rb

@@ -0,0 +1,94 @@
+require "logger"
+require "redis"
+require "active_record"
+require "ruby_event_store/outbox/record"
+require "ruby_event_store/outbox/sidekiq5_format"
+
+module RubyEventStore
+  module Outbox
+    class Consumer
+      SLEEP_TIME_WHEN_NOTHING_TO_DO = 0.1
+
+      def initialize(format, split_keys, database_url:, redis_url:, clock: Time, logger:)
+        @split_keys = split_keys
+        @clock = clock
+        @redis = Redis.new(url: redis_url)
+        @logger = logger
+        ActiveRecord::Base.establish_connection(database_url) unless ActiveRecord::Base.connected?
+
+        raise "Unknown format" if format != SIDEKIQ5_FORMAT
+        @message_format = SIDEKIQ5_FORMAT
+
+        @gracefully_shutting_down = false
+        prepare_traps
+      end
+
+      def init
+        @redis.sadd("queues", split_keys)
+        logger.info("Initiated RubyEventStore::Outbox v#{VERSION}")
+        logger.info("Handling split keys: #{split_keys ? split_keys.join(", ") : "(all of them)"}")
+      end
+
+      def run
+        while !@gracefully_shutting_down do
+          was_something_changed = one_loop
+          sleep SLEEP_TIME_WHEN_NOTHING_TO_DO if !was_something_changed
+        end
+        logger.info "Gracefully shutting down"
+      end
+
+      def one_loop
+        Record.transaction do
+          records_scope = Record.lock.where(format: message_format, enqueued_at: nil)
+          records_scope = records_scope.where(split_key: split_keys) if !split_keys.nil?
+          records = records_scope.order("id ASC").limit(100)
+          return false if records.empty?
+
+          now = @clock.now.utc
+          failed_record_ids = []
+          records.each do |record|
+            begin
+              handle_one_record(now, record)
+            rescue => e
+              failed_record_ids << record.id
+              e.full_message.split($/).each {|line| logger.error(line) }
+            end
+          end
+
+          update_scope = if failed_record_ids.empty?
+            records
+          else
+            records.where("id NOT IN (?)", failed_record_ids)
+          end
+          update_scope.update_all(enqueued_at: now)
+
+          logger.info "Sent #{records.size} messages from outbox table"
+          return true
+        end
+      end
+
+      private
+      attr_reader :split_keys, :logger, :message_format
+
+      def handle_one_record(now, record)
+        hash_payload = JSON.parse(record.payload)
+        @redis.lpush("queue:#{hash_payload.fetch("queue")}", JSON.generate(JSON.parse(record.payload).merge({
+          "enqueued_at" => now.to_f,
+        })))
+      end
+
+      def prepare_traps
+        Signal.trap("INT") do
+          initiate_graceful_shutdown
+        end
+        Signal.trap("TERM") do
+          initiate_graceful_shutdown
+        end
+      end
+
+      def initiate_graceful_shutdown
+        @gracefully_shutting_down = true
+      end
+    end
+  end
+end

data/lib/ruby_event_store/outbox/record.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require 'active_record'
+
+module RubyEventStore
+  module Outbox
+    class Record < ::ActiveRecord::Base
+      self.primary_key = :id
+      self.table_name = 'event_store_outbox'
+
+      def hash_payload
+        JSON.parse(payload).deep_symbolize_keys
+      end
+    end
+  end
+end

data/lib/ruby_event_store/outbox/sidekiq5_format.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module RubyEventStore
+  module Outbox
+    SIDEKIQ5_FORMAT = "sidekiq5"
+  end
+end

data/lib/ruby_event_store/outbox/sidekiq_scheduler.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require 'sidekiq'
+require "ruby_event_store/outbox/sidekiq5_format"
+
+module RubyEventStore
+  module Outbox
+    class SidekiqScheduler
+      def call(klass, serialized_event)
+        sidekiq_client = Sidekiq::Client.new(Sidekiq.redis_pool)
+        item = {
+          'class' => klass,
+          'args' => [serialized_event.to_h],
+        }
+        normalized_item = sidekiq_client.__send__(:normalize_item, item)
+        payload = sidekiq_client.__send__(:process_single, normalized_item.fetch('class'), normalized_item)
+        if payload
+          Record.create!(
+            format: SIDEKIQ5_FORMAT,
+            split_key: payload.fetch('queue'),
+            payload: payload.to_json
+          )
+        end
+      end
+
+      def verify(subscriber)
+        Class === subscriber && subscriber.respond_to?(:through_outbox?) && subscriber.through_outbox?
+      end
+    end
+  end
+end

data/lib/ruby_event_store/outbox/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module RubyEventStore
+  module Outbox
+    VERSION = "0.0.2"
+  end
+end

metadata

@@ -0,0 +1,87 @@
+--- !ruby/object:Gem::Specification
+name: ruby_event_store-outbox
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+platform: ruby
+authors:
+- Arkency
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-06-12 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: ruby_event_store
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.0'
+description: 
+email:
+- dev@arkency.com
+executables:
+- res_outbox
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- bin/res_outbox
+- lib/generators/ruby_event_store/outbox/migration_generator.rb
+- lib/generators/ruby_event_store/outbox/templates/create_event_store_outbox_template.rb
+- lib/ruby_event_store/outbox.rb
+- lib/ruby_event_store/outbox/cli.rb
+- lib/ruby_event_store/outbox/consumer.rb
+- lib/ruby_event_store/outbox/record.rb
+- lib/ruby_event_store/outbox/sidekiq5_format.rb
+- lib/ruby_event_store/outbox/sidekiq_scheduler.rb
+- lib/ruby_event_store/outbox/version.rb
+homepage: https://railseventstore.org
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://railseventstore.org/
+  changelog_uri: https://github.com/RailsEventStore/rails_event_store/releases
+  source_code_uri: https://github.com/RailsEventStore/rails_event_store
+  bug_tracker_uri: https://github.com/RailsEventStore/rails_event_store/issues
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key: 
+specification_version: 4
+summary: Active Record based outbox for Ruby Event Store
+test_files: []