pg_eventstore 0.3.0 → 0.5.0

data/lib/pg_eventstore/subscriptions/basic_runner.rb

@@ -0,0 +1,220 @@
+# frozen_string_literal: true
+
+module PgEventstore
+  # Implements simple background jobs runner. The job execution is done via declaring a callback on the specific
+  # action. The implementation also allows you to hook into different places of life cycle of the runner by defining
+  # callbacks on various actions. Here is the list of available actions:
+  # - :before_runner_started. Happens before the runner's state switches from "initial"/"stopped" to "running" and
+  #   runner's thread is started. It is also fired when the runner is restoring - right after :before_runner_restored
+  #   action.
+  # - :after_runner_stopped. Happens after runner's state got switched from "running"/"dead" to "stopped" and runner's
+  #   thread is terminated.
+  # - :before_runner_restored. Happens before runner's state gets switched from "dead" to "running" and runner's
+  #   thread is started.
+  # - :process_async. Happens each @run_interval seconds within runner's thread.
+  # - :after_runner_died. Happens when runner's state switches to "dead" because of exception inside runner's thread.
+  #   Callback function must be able to accept one argument - the exception which caused the runner to die will be
+  #   passed.
+  # - :change_state. It happens each time the runner changes the state. Callback function must be able to accept one
+  #   argument - current state will be passed.
+  #
+  # Example of BasicRunner usage:
+  #   class MyAwesomeRunner
+  #     extend Forwardable
+  #
+  #     def_delegators :@basic_runner, :start, :stop, :wait_for_finish, :stop_async, :restore
+  #
+  #     def initialize
+  #       @basic_runner = PgEventstore::BasicRunner.new(1, 2)
+  #       @jobs_performed = 0
+  #       attach_runner_callbacks
+  #     end
+  #
+  #     private
+  #
+  #     def attach_runner_callbacks
+  #       @basic_runner.define_callback(:change_state, :after, method(:state_changed))
+  #       @basic_runner.define_callback(:process_async, :before, method(:process_action))
+  #       @basic_runner.define_callback(:process_async, :after, method(:count_jobs))
+  #       @basic_runner.define_callback(:before_runner_started, :before, method(:before_runner_started))
+  #       @basic_runner.define_callback(:after_runner_stopped, :before, method(:after_runner_stopped))
+  #       @basic_runner.define_callback(:after_runner_died, :before, method(:after_runner_died))
+  #     end
+  #
+  #     def process_action
+  #       raise "What's the point? I can not handle this any more!" if @jobs_performed >= 3
+  #       puts "Doing some heavy lifting job"
+  #       sleep 2 # Simulate long running job
+  #     end
+  #
+  #     def count_jobs
+  #       @jobs_performed += 1
+  #     end
+  #
+  #     # @param state [String]
+  #     def state_changed(state)
+  #       puts "New state is #{state.inspect}"
+  #     end
+  #
+  #     def before_runner_started
+  #       puts "Doing some preparations..."
+  #     end
+  #
+  #     def after_runner_stopped
+  #       puts "You job is not processing any more. Total jobs performed: #{@jobs_performed}. Bye-bye!"
+  #     end
+  #
+  #     def after_runner_died(error)
+  #       puts "Error occurred: #{error.inspect}"
+  #     end
+  #   end
+  #
+  #   runner = MyAwesomeRunner.new
+  #   runner.start # to start your background runner to process the job, defined by #process_action method
+  #   runner.stop # to stop the runner
+  #
+  # See {PgEventstore::RunnerState} for the list of available states
+  # See {PgEventstore::CallbacksExtension} and {PgEventstore::Callbacks} for more info about how to use callbacks
+  class BasicRunner
+    extend Forwardable
+    include Extensions::CallbacksExtension
+
+    def_delegators :@state, :initial?, :running?, :halting?, :stopped?, :dead?
+
+    # @param run_interval [Integer, Float] seconds. Determines how often to run async task. Async task is determined by
+    #   :after_runner_stopped callback
+    # @param async_shutdown_time [Integer, Float] seconds. Determines how long to wait for the async shutdown to wait
+    #   for the runner to finish.
+    def initialize(run_interval, async_shutdown_time)
+      @run_interval = run_interval
+      @async_shutdown_time = async_shutdown_time
+      @state = RunnerState.new
+      @mutex = Thread::Mutex.new
+      delegate_change_state_cbx
+    end
+
+    # Start asynchronous runner. If the runner is dead - please use #restore to restart it.
+    # @return [self]
+    def start
+      synchronize do
+        return self unless @state.initial? || @state.stopped?
+
+        callbacks.run_callbacks(:before_runner_started)
+        _start
+      end
+      self
+    end
+
+    # Stop asynchronous runner. This operation is immediate and it won't be waiting for current job to finish - it will
+    # instantly halt it. If you care about the result of your async job - use #stop_async instead.
+    # @return [self]
+    def stop
+      synchronize do
+        return self unless @state.running? || @state.dead?
+
+        @runner&.exit
+        @runner = nil
+        @state.stopped!
+        callbacks.run_callbacks(:after_runner_stopped)
+      end
+      self
+    end
+
+    # Asynchronously stop asynchronous runner. This operation spawns another thread to gracefully stop the runner. It
+    # will wait up to @async_shutdown_time seconds before force-stopping the runner.
+    # @return [self]
+    def stop_async
+      synchronize do
+        return self unless @state.running? || @state.dead?
+
+        @state.halting!
+        Thread.new do
+          stopping_at = Time.now.utc
+          halt = false
+          loop do
+            synchronize do
+              # Give the runner up to @async_shutdown_time seconds for graceful shutdown
+              @runner&.exit if Time.now.utc - stopping_at > @async_shutdown_time
+
+              unless @runner&.alive?
+                @state.stopped!
+                @runner = nil
+                callbacks.run_callbacks(:after_runner_stopped)
+                halt = true
+              end
+            end
+            break if halt
+            sleep 0.1
+          end
+        end
+        self
+      end
+    end
+
+    # Restores the runner after its death.
+    # @return [self]
+    def restore
+      synchronize do
+        return self unless @state.dead?
+
+        callbacks.run_callbacks(:before_runner_restored)
+        _start
+      end
+      self
+    end
+
+    # Wait until the runner switches the state to either "stopped" or "dead". This operation is synchronous.
+    # @return [self]
+    def wait_for_finish
+      loop do
+        continue = synchronize do
+          @state.halting? || @state.running?
+        end
+        break unless continue
+
+        sleep 0.1
+      end
+      self
+    end
+
+    # @return [String]
+    def state
+      @state.to_s
+    end
+
+    private
+
+    def synchronize
+      @mutex.synchronize { yield }
+    end
+
+    # @return [void]
+    def _start
+      @state.running!
+      @runner = Thread.new do
+        loop do
+          Thread.current.exit unless @state.running?
+          sleep @run_interval
+
+          callbacks.run_callbacks(:process_async)
+        end
+      rescue => error
+        synchronize do
+          raise unless @state.halting? || @state.running?
+
+          @state.dead!
+          callbacks.run_callbacks(:after_runner_died, error)
+        end
+      end
+    end
+
+    # Delegates :change_state action to the runner
+    def delegate_change_state_cbx
+      @state.define_callback(:change_state, :before, method(:change_state))
+    end
+
+    def change_state(...)
+      callbacks.run_callbacks(:change_state, ...)
+    end
+  end
+end

data/lib/pg_eventstore/subscriptions/command_handlers/subscription_feeder_commands.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module PgEventstore
+  module CommandHandlers
+    class SubscriptionFeederCommands
+      AVAILABLE_COMMANDS = %w[StopAll StartAll].freeze
+
+      # @param config_name [Symbol]
+      # @param subscription_feeder [PgEventstore::SubscriptionFeeder]
+      def initialize(config_name, subscription_feeder)
+        @config_name = config_name
+        @subscription_feeder = subscription_feeder
+      end
+
+      # Look up commands for the given SubscriptionFeeder and execute them
+      # @return [void]
+      def process
+        queries.find_commands(@subscription_feeder.id).each do |command|
+          unless AVAILABLE_COMMANDS.include?(command[:name])
+            PgEventstore.logger&.warn(
+              "#{self.class.name}: Don't know how to handle #{command[:name].inspect}. Details: #{command.inspect}."
+            )
+            next
+          end
+          send(Utils.underscore_str(command[:name]))
+        ensure
+          queries.delete(command[:id])
+        end
+      end
+
+      private
+
+      # @return [PgEventstore::SubscriptionsSetCommandQueries]
+      def queries
+        SubscriptionsSetCommandQueries.new(connection)
+      end
+
+      # @return [PgEventstore::Connection]
+      def connection
+        PgEventstore.connection(@config_name)
+      end
+
+      def stop_all
+        @subscription_feeder.stop_all
+      end
+
+      def start_all
+        @subscription_feeder.start_all
+      end
+    end
+  end
+end

data/lib/pg_eventstore/subscriptions/command_handlers/subscription_runners_commands.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module PgEventstore
+  module CommandHandlers
+    class SubscriptionRunnersCommands
+      AVAILABLE_COMMANDS = %w[StopRunner RestoreRunner StartRunner].freeze
+
+      # @param config_name [Symbol]
+      # @param runners [Array<PgEventstore::SubscriptionRunner>]
+      def initialize(config_name, runners)
+        @config_name = config_name
+        @runners = runners
+      end
+
+      # Look up commands for all given SubscriptionRunner-s and execute them
+      # @return [void]
+      def process
+        queries.find_commands(@runners.map(&:id)).each do |command|
+          unless AVAILABLE_COMMANDS.include?(command[:name])
+            PgEventstore.logger&.warn(
+              "#{self.class.name}: Don't know how to handle #{command[:name].inspect}. Details: #{command.inspect}."
+            )
+            next
+          end
+          send(Utils.underscore_str(command[:name]), command[:subscription_id])
+        ensure
+          queries.delete(command[:id])
+        end
+      end
+
+      private
+
+      # @return [PgEventstore::SubscriptionCommandQueries]
+      def queries
+        SubscriptionCommandQueries.new(connection)
+      end
+
+      # @return [PgEventstore::Connection]
+      def connection
+        PgEventstore.connection(@config_name)
+      end
+
+      # @param subscription_id [Integer]
+      # @return [PgEventstore::SubscriptionRunner, nil]
+      def find_subscription_runner(subscription_id)
+        @runners.find { |runner| runner.id == subscription_id }
+      end
+
+      # @param subscription_id [Integer]
+      # @return [void]
+      def start_runner(subscription_id)
+        find_subscription_runner(subscription_id)&.start
+      end
+
+      # @param subscription_id [Integer]
+      # @return [void]
+      def restore_runner(subscription_id)
+        find_subscription_runner(subscription_id)&.restore
+      end
+
+      # @param subscription_id [Integer]
+      # @return [void]
+      def stop_runner(subscription_id)
+        find_subscription_runner(subscription_id)&.stop_async
+      end
+    end
+  end
+end

data/lib/pg_eventstore/subscriptions/commands_handler.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require_relative 'command_handlers/subscription_feeder_commands'
+require_relative 'command_handlers/subscription_runners_commands'
+
+module PgEventstore
+  # This class implements the runner which processes remote commands in the background. This allows you to remotely
+  # control such actions as stop, start and restart of your Subscriptions.
+  # @!visibility private
+  class CommandsHandler
+    extend Forwardable
+
+    RESTART_DELAY = 5 # seconds
+
+    def_delegators :@basic_runner, :start, :stop, :state, :stop_async, :wait_for_finish
+
+    # @param config_name [Symbol]
+    # @param subscription_feeder [PgEventstore::SUbscriptionFeeder]
+    # @param runners [Array<PgEventstore::SubscriptionRunner>]
+    def initialize(config_name, subscription_feeder, runners)
+      @config_name = config_name
+      @subscription_feeder = subscription_feeder
+      @runners = runners
+      @basic_runner = BasicRunner.new(1, 0)
+      attach_runner_callbacks
+    end
+
+    private
+
+    def attach_runner_callbacks
+      @basic_runner.define_callback(:process_async, :before, method(:process_async))
+      @basic_runner.define_callback(:after_runner_died, :before, method(:after_runner_died))
+    end
+
+    def process_async
+      subscription_feeder_commands.process
+      subscription_runners_commands.process
+    end
+
+    # @param error [StandardError]
+    # @return [void]
+    def after_runner_died(error)
+      PgEventstore.logger&.error "#{self.class.name}: Error occurred: #{error.message}"
+      PgEventstore.logger&.error "#{self.class.name}: Backtrace: #{error.backtrace&.join("\n")}"
+      PgEventstore.logger&.error "#{self.class.name}: Trying to auto-repair in #{RESTART_DELAY} seconds..."
+      Thread.new do
+        sleep RESTART_DELAY
+        @basic_runner.restore
+      end
+    end
+
+    # @return [PgEventstore::CommandHandlers::SubscriptionFeederCommands]
+    def subscription_feeder_commands
+      CommandHandlers::SubscriptionFeederCommands.new(@config_name, @subscription_feeder)
+    end
+
+    # @return [PgEventstore::CommandHandlers::SubscriptionRunnersCommands]
+    def subscription_runners_commands
+      CommandHandlers::SubscriptionRunnersCommands.new(@config_name, @runners)
+    end
+  end
+end

data/lib/pg_eventstore/subscriptions/events_processor.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module PgEventstore
+  # This class actually processes events.
+  # @!visibility private
+  class EventsProcessor
+    include Extensions::CallbacksExtension
+    extend Forwardable
+
+    def_delegators :@basic_runner, :state, :start, :stop, :wait_for_finish, :stop_async, :restore, :running?
+
+    # @param handler [#call]
+    def initialize(handler)
+      @handler = handler
+      @raw_events = []
+      @basic_runner = BasicRunner.new(0, 5)
+      attach_runner_callbacks
+    end
+
+    # @param raw_events [Array<Hash>]
+    # @return [void]
+    def feed(raw_events)
+      callbacks.run_callbacks(:feed, raw_events.last&.dig('global_position'))
+      @raw_events.push(*raw_events)
+    end
+
+    # Number of unprocessed events which are currently in a queue
+    # @return [Integer]
+    def events_left_in_chunk
+      @raw_events.size
+    end
+
+    private
+
+    # @param raw_event [Hash]
+    # @return [void]
+    def process_event(raw_event)
+      callbacks.run_callbacks(:process, raw_event['global_position']) do
+        @handler.call(raw_event)
+      end
+    end
+
+    def attach_runner_callbacks
+      @basic_runner.define_callback(:process_async, :before, method(:process_async))
+      @basic_runner.define_callback(:after_runner_died, :before, method(:after_runner_died))
+      @basic_runner.define_callback(:before_runner_restored, :before, method(:before_runner_restored))
+      @basic_runner.define_callback(:change_state, :before, method(:change_state))
+    end
+
+    def process_async
+      raw_event = @raw_events.shift
+      return sleep 0.5 if raw_event.nil?
+
+      process_event(raw_event)
+    rescue
+      @raw_events.unshift(raw_event)
+      raise
+    end
+
+    def after_runner_died(...)
+      callbacks.run_callbacks(:error, ...)
+    end
+
+    def before_runner_restored
+      callbacks.run_callbacks(:restart)
+    end
+
+    def change_state(...)
+      callbacks.run_callbacks(:change_state, ...)
+    end
+  end
+end

data/lib/pg_eventstore/subscriptions/runner_state.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module PgEventstore
+  # Implements different states of a runner.
+  # @!visibility private
+  class RunnerState
+    include Extensions::CallbacksExtension
+
+    STATES = %i(initial running halting stopped dead).to_h { [_1, _1.to_s.freeze] }.freeze
+
+    def initialize
+      initial!
+    end
+
+    STATES.each do |state, value|
+      # Checks whether a runner is in appropriate state
+      # @return [Boolean]
+      define_method "#{state}?" do
+        @state == value
+      end
+
+      # Sets the state.
+      # @return [String]
+      define_method "#{state}!" do
+        set_state(value)
+      end
+    end
+
+    # @return [String] string representation of the state
+    def to_s
+      @state
+    end
+
+    private
+
+    # @param state [String]
+    # @return [String]
+    def set_state(state)
+      old_state = @state
+      @state = state
+      callbacks.run_callbacks(:change_state, @state) unless old_state == @state
+      @state
+    end
+  end
+end

data/lib/pg_eventstore/subscriptions/subscription.rb

@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+
+module PgEventstore
+  # Defines ruby's representation of subscriptions record.
+  class Subscription
+    include Extensions::UsingConnectionExtension
+    include Extensions::OptionsExtension
+
+    # @!attribute id
+    #   @return [Integer]
+    attribute(:id)
+    # @!attribute set
+    #   @return [String] Subscription's set. Subscription should have unique pair of set and name.
+    attribute(:set)
+    # @!attribute name
+    #   @return [String] Subscription's name. Subscription should have unique pair of set and name.
+    attribute(:name)
+    # @!attribute total_processed_events
+    #   @return [Integer] total number of events, processed by this subscription
+    attribute(:total_processed_events)
+    # @!attribute options
+    #   @return [Hash] subscription's options to be used to query events. See {SubscriptionManager#subscribe} for the
+    #     list of available options
+    attribute(:options)
+    # @!attribute current_position
+    #   @return [Integer, nil] current Subscription's position. It is updated automatically each time an event is processed
+    attribute(:current_position)
+    # @!attribute state
+    #   @return [String, nil] current Subscription's state. It is updated automatically during Subscription's life cycle.
+    #     See {RunnerState::STATES} for possible values.
+    attribute(:state)
+    # @!attribute average_event_processing_time
+    #   @return [Float, nil] a speed of the subscription. Divide 1 by this value to determine how much events are
+    #     processed by the Subscription per second.
+    attribute(:average_event_processing_time)
+    # @!attribute restart_count
+    #   @return [Integer] the number of Subscription's restarts after its failure
+    attribute(:restart_count)
+    # @!attribute max_restarts_number
+    #   @return [Integer] maximum number of times the Subscription can be restarted
+    attribute(:max_restarts_number)
+    # @!attribute time_between_restarts
+    #   @return [Integer] interval in seconds between retries of failed Subscription
+    attribute(:time_between_restarts)
+    # @!attribute last_restarted_at
+    #   @return [Time, nil] last time the Subscription was restarted
+    attribute(:last_restarted_at)
+    # @!attribute last_error
+    #   @return [Hash{'class' => String, 'message' => String, 'backtrace' => Array<String>}, nil] the information about
+    #     last error caused when processing events by the Subscription.
+    attribute(:last_error)
+    # @!attribute last_error_occurred_at
+    #   @return [Time, nil] the time when the last error occurred
+    attribute(:last_error_occurred_at)
+    # @!attribute chunk_query_interval
+    #   @return [Integer] determines how often to pull events for the given Subscription in seconds
+    attribute(:chunk_query_interval)
+    # @!attribute chunk_query_interval
+    #   @return [Time] shows the time when last time events were fed to the event's processor
+    attribute(:last_chunk_fed_at)
+    # @!attribute last_chunk_greatest_position
+    #   @return [Integer, nil] shows the greatest global_position of the last event in the last chunk fed to the event's
+    #     processor
+    attribute(:last_chunk_greatest_position)
+    # @!attribute locked_by
+    #   @return [String, nil] UUIDv4. The id of subscription manager which obtained the lock of the Subscription. _nil_
+    #     value means that the Subscription isn't locked yet by any subscription manager.
+    attribute(:locked_by)
+    # @!attribute created_at
+    #   @return [Time]
+    attribute(:created_at)
+    # @!attribute updated_at
+    #   @return [Time]
+    attribute(:updated_at)
+
+    def options=(val)
+      @options = Utils.deep_transform_keys(val, &:to_sym)
+    end
+
+    # @param attrs [Hash]
+    # @return [Hash]
+    def update(attrs)
+      assign_attributes(subscription_queries.update(id, attrs))
+    end
+
+    # @param attrs [Hash]
+    # @return [Hash]
+    def assign_attributes(attrs)
+      attrs.each do |attr, value|
+        public_send("#{attr}=", value)
+      end
+    end
+
+    # Locks the Subscription by the given lock id
+    # @return [PgEventstore::Subscription]
+    def lock!(lock_id, force = false)
+      self.id = subscription_queries.find_or_create_by(set: set, name: name)[:id]
+      self.locked_by = subscription_queries.lock!(id, lock_id, force)
+      reset_runtime_attributes
+      self
+    end
+
+    # Unlocks the Subscription.
+    # @return [void]
+    def unlock!
+      subscription_queries.unlock!(id, locked_by)
+      self.locked_by = nil
+    end
+
+    # Dup the current object without assigned connection
+    # @return [PgEventstore::Subscription]
+    def dup
+      Subscription.new(**Utils.deep_dup(options_hash))
+    end
+
+    # @return [PgEventstore::Subscription]
+    def reload
+      assign_attributes(subscription_queries.find!(id))
+      self
+    end
+
+    private
+
+    def reset_runtime_attributes
+      update(
+        options: options,
+        restart_count: 0,
+        last_restarted_at: nil,
+        max_restarts_number: max_restarts_number,
+        chunk_query_interval: chunk_query_interval,
+        last_chunk_fed_at: Time.at(0).utc,
+        last_chunk_greatest_position: nil,
+        state: RunnerState::STATES[:initial]
+      )
+    end
+
+    def subscription_queries
+      SubscriptionQueries.new(self.class.connection)
+    end
+  end
+end