pg_eventstore 0.3.0 → 0.5.0

data/lib/pg_eventstore/subscriptions/subscription_feeder.rb

@@ -0,0 +1,171 @@
+# frozen_string_literal: true
+
+module PgEventstore
+  # This class is responsible for starting/stopping all SubscriptionRunners. The background runner of it is responsible
+  # for events pulling and feeding those SubscriptionRunners.
+  # @!visibility private
+  class SubscriptionFeeder
+    extend Forwardable
+
+    def_delegators :subscriptions_set, :id
+    def_delegators :@basic_runner, :start, :stop, :restore, :state, :wait_for_finish, :stop_async
+
+    # @param config_name [Symbol]
+    # @param set_name [String]
+    # @param max_retries [Integer] max number of retries of failed SubscriptionsSet
+    # @param retries_interval [Integer] a delay between retries of failed SubscriptionsSet
+    def initialize(config_name:, set_name:, max_retries:, retries_interval:)
+      @config_name = config_name
+      @runners = []
+      @set_name = set_name
+      @max_retries = max_retries
+      @retries_interval = retries_interval
+      @commands_handler = CommandsHandler.new(@config_name, self, @runners)
+      @basic_runner = BasicRunner.new(1, 0)
+      @force_lock = false
+      attach_runner_callbacks
+    end
+
+    # Adds SubscriptionRunner to the set
+    # @param runner [PgEventstore::SubscriptionRunner]
+    def add(runner)
+      assert_proper_state!
+      @runners.push(runner)
+      runner
+    end
+
+    # Starts all SubscriptionRunners. This is only available if SubscriptionFeeder's runner is alive.
+    # @return [void]
+    def start_all
+      return self unless @basic_runner.running?
+
+      @runners.each(&:start)
+      self
+    end
+
+    # Stops all SubscriptionRunners asynchronous. This is only available if SubscriptionFeeder's runner is alive.
+    # @return [void]
+    def stop_all
+      return self unless @basic_runner.running?
+
+      @runners.each(&:stop_async)
+      self
+    end
+
+    # Sets the force_lock flash to true. If set - all related Subscriptions will ignore their lock state and will be
+    # locked by the new SubscriptionsSet.
+    def force_lock!
+      @force_lock = true
+    end
+
+    # Produces a copy of currently running Subscriptions. This is needed, because original Subscriptions objects are
+    # dangerous to use - users may incidentally break their state.
+    # @return [Array<PgEventstore::Subscription>]
+    def read_only_subscriptions
+      @runners.map(&:subscription).map(&:dup)
+    end
+
+    # Produces a copy of current SubscriptionsSet. This is needed, because original SubscriptionsSet object is
+    # dangerous to use - users may incidentally break its state.
+    # @return [PgEventstore::SubscriptionsSet, nil]
+    def read_only_subscriptions_set
+      @subscriptions_set&.dup
+    end
+
+    private
+
+    # Locks all Subscriptions behind the current SubscriptionsSet
+    # @return [void]
+    def lock_all
+      @runners.each { |runner| runner.lock!(subscriptions_set.id, @force_lock) }
+    end
+
+    # @return [void]
+    def unlock_all
+      @runners.each(&:unlock!)
+    end
+
+    # @return [PgEventstore::SubscriptionsSet]
+    def subscriptions_set
+      @subscriptions_set ||= SubscriptionsSet.using_connection(@config_name).
+        create(name: @set_name, max_restarts_number: @max_retries, time_between_restarts: @retries_interval)
+    end
+
+    # @return [PgEventstore::SubscriptionRunnersFeeder]
+    def feeder
+      SubscriptionRunnersFeeder.new(@config_name)
+    end
+
+    # @return [void]
+    def attach_runner_callbacks
+      @basic_runner.define_callback(:change_state, :after, method(:update_subscriptions_set_state))
+      @basic_runner.define_callback(:before_runner_started, :before, method(:before_runner_started))
+      @basic_runner.define_callback(:after_runner_died, :before, method(:after_runner_died))
+      @basic_runner.define_callback(:after_runner_died, :after, method(:restart_runner))
+      @basic_runner.define_callback(:process_async, :before, method(:process_async))
+      @basic_runner.define_callback(:after_runner_stopped, :before, method(:after_runner_stopped))
+      @basic_runner.define_callback(:before_runner_restored, :after, method(:update_runner_restarts))
+    end
+
+    # @return [void]
+    def before_runner_started
+      lock_all
+      @runners.each(&:start)
+      @commands_handler.start
+    end
+
+    # @param error [StandardError]
+    # @return [void]
+    def after_runner_died(error)
+      subscriptions_set.update(last_error: Utils.error_info(error), last_error_occurred_at: Time.now.utc)
+    end
+
+    # @param _error [StandardError]
+    # @return [void]
+    def restart_runner(_error)
+      return if subscriptions_set.restart_count >= subscriptions_set.max_restarts_number
+
+      Thread.new do
+        sleep subscriptions_set.time_between_restarts
+        restore
+      end
+    end
+
+    # @return [void]
+    def update_runner_restarts
+      subscriptions_set.update(last_restarted_at: Time.now.utc, restart_count: subscriptions_set.restart_count + 1)
+    end
+
+    # @return [void]
+    def process_async
+      feeder.feed(@runners)
+    end
+
+    # @return [void]
+    def after_runner_stopped
+      @commands_handler.stop
+      @subscriptions_set&.delete
+      @subscriptions_set = nil
+      @runners.each(&:stop_async).each(&:wait_for_finish)
+      unlock_all
+    end
+
+    # @return [void]
+    def update_subscriptions_set_state(state)
+      subscriptions_set.update(state: state)
+    end
+
+    # This method helps to ensure that no Subscription is added after SubscriptionFeeder's runner is working
+    # @return [void]
+    # @raise [RuntimeError]
+    def assert_proper_state!
+      return if @basic_runner.initial? || @basic_runner.stopped?
+
+      error_message = <<~TEXT
+        Could not add subscription - #{subscriptions_set.name}##{subscriptions_set.id} must be either in the initial \
+        or in the stopped state, but it is in the #{@basic_runner.state} state now.
+      TEXT
+      raise error_message
+    end
+  end
+end

data/lib/pg_eventstore/subscriptions/subscription_handler_performance.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require 'benchmark'
+
+module PgEventstore
+  # This class measures the performance of Subscription's handler and returns the average time required to process an
+  # event.
+  # @!visibility private
+  class SubscriptionHandlerPerformance
+    extend Forwardable
+
+    TIMINGS_TO_KEEP = 100
+
+    def_delegators :@lock, :synchronize
+
+    def initialize
+      @lock = Thread::Mutex.new
+      @timings = []
+    end
+
+    # Yields the given block to measure its execution time
+    # @return [Object] the result of yielded block
+    def track_exec_time
+      result = nil
+      time = Benchmark.realtime { result = yield }
+      synchronize do
+        @timings.shift if @timings.size == TIMINGS_TO_KEEP
+        @timings.push(time)
+      end
+      result
+    end
+
+    # The average time required to process an event.
+    # @return [Float]
+    def average_event_processing_time
+      synchronize { @timings.size.zero? ? 0 : @timings.sum / @timings.size }
+    end
+  end
+end

data/lib/pg_eventstore/subscriptions/subscription_runner.rb

@@ -0,0 +1,125 @@
+# frozen_string_literal: true
+
+require 'forwardable'
+
+module PgEventstore
+  # This class connects Subscription and EventsProcessor. Its public API is directed on locking/unlocking related
+  # Subscription, starting/stopping/restarting EventsProcessor and calculating options(starting position, number of
+  # events to fetch, etc) for the events pulling query.
+  # @!visibility private
+  class SubscriptionRunner
+    extend Forwardable
+
+    MAX_EVENTS_PER_CHUNK = 1_000
+    INITIAL_EVENTS_PER_CHUNK = 10
+
+    attr_reader :subscription
+
+    def_delegators :@events_processor, :start, :stop, :stop_async, :feed, :wait_for_finish, :restore, :state, :running?
+    def_delegators :@subscription, :lock!, :unlock!, :id
+
+    # @param stats [PgEventstore::SubscriptionHandlerPerformance]
+    # @param events_processor [PgEventstore::EventsProcessor]
+    # @param subscription [PgEventstore::Subscription]
+    # @param restart_terminator [#call, nil]
+    def initialize(stats:, events_processor:, subscription:, restart_terminator: nil)
+      @stats = stats
+      @events_processor = events_processor
+      @subscription = subscription
+      @restart_terminator = restart_terminator
+
+      attach_callbacks
+    end
+
+    # @return [Hash]
+    def next_chunk_query_opts
+      @subscription.options.merge(from_position: next_chunk_global_position, max_count: estimate_events_number)
+    end
+
+    # @return [Boolean]
+    def time_to_feed?
+      @subscription.last_chunk_fed_at + @subscription.chunk_query_interval <= Time.now.utc
+    end
+
+    private
+
+    # @return [Integer]
+    def next_chunk_global_position
+      (
+        @subscription.last_chunk_greatest_position || @subscription.current_position ||
+          @subscription.options[:from_position] || 0
+      ) + 1
+    end
+
+    # @return [Integer]
+    def estimate_events_number
+      return INITIAL_EVENTS_PER_CHUNK if @stats.average_event_processing_time.zero?
+
+      events_per_chunk = @subscription.chunk_query_interval / @stats.average_event_processing_time
+      [events_per_chunk, MAX_EVENTS_PER_CHUNK].min - @events_processor.events_left_in_chunk
+    end
+
+    # @return [void]
+    def attach_callbacks
+      @events_processor.define_callback(:process, :around, method(:track_exec_time))
+      @events_processor.define_callback(:process, :after, method(:update_subscription_stats))
+      @events_processor.define_callback(:error, :after, method(:update_subscription_error))
+      @events_processor.define_callback(:error, :after, method(:restart_subscription))
+      @events_processor.define_callback(:feed, :after, method(:update_subscription_chunk_stats))
+      @events_processor.define_callback(:restart, :after, method(:update_subscription_restarts))
+      @events_processor.define_callback(:change_state, :after, method(:update_subscription_state))
+    end
+
+    # @param action [Proc]
+    # @return [void]
+    def track_exec_time(action, *)
+      @stats.track_exec_time { action.call }
+    end
+
+    # @param current_position [Integer]
+    # @return [void]
+    def update_subscription_stats(current_position)
+      @subscription.update(
+        average_event_processing_time: @stats.average_event_processing_time,
+        current_position: current_position,
+        total_processed_events: @subscription.total_processed_events + 1
+      )
+    end
+
+    # @param state [String]
+    # @return [void]
+    def update_subscription_state(state)
+      @subscription.update(state: state)
+    end
+
+    # @return [void]
+    def update_subscription_restarts
+      @subscription.update(last_restarted_at: Time.now.utc, restart_count: @subscription.restart_count + 1)
+    end
+
+    # @param error [StandardError]
+    # @return [void]
+    def update_subscription_error(error)
+      @subscription.update(last_error: Utils.error_info(error), last_error_occurred_at: Time.now.utc)
+    end
+
+    # @param global_position [Integer, nil]
+    # @return [void]
+    def update_subscription_chunk_stats(global_position)
+      global_position ||= @subscription.last_chunk_greatest_position
+      @subscription.update(last_chunk_fed_at: Time.now.utc, last_chunk_greatest_position: global_position)
+    end
+
+    # @param _error [StandardError]
+    # @return [void]
+    def restart_subscription(_error)
+      return if @restart_terminator&.call(@subscription.dup)
+      return if @subscription.restart_count >= @subscription.max_restarts_number
+
+      Thread.new do
+        sleep @subscription.time_between_restarts
+        restore
+      end
+    end
+  end
+end

data/lib/pg_eventstore/subscriptions/subscription_runners_feeder.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module PgEventstore
+  # This class pulls events from db and feeds given SubscriptionRunners
+  # @!visibility private
+  class SubscriptionRunnersFeeder
+    # @param config_name [Symbol]
+    def initialize(config_name)
+      @config_name = config_name
+    end
+
+    # @param runners [Array<PgEventstore::SubscriptionRunner>]
+    # @return [void]
+    def feed(runners)
+      runners = runners.select(&:running?).select(&:time_to_feed?)
+      return if runners.empty?
+
+      runners_query_options = runners.map { |runner| [runner.id, runner.next_chunk_query_opts] }
+      raw_events = subscription_queries.subscriptions_events(runners_query_options)
+      grouped_events = raw_events.group_by { |attrs| attrs['runner_id'] }
+      runners.each do |runner|
+        runner.feed(grouped_events[runner.id]) if grouped_events[runner.id]
+      end
+    end
+
+    private
+
+    # @return [PgEventstore::Connection]
+    def connection
+      PgEventstore.connection(@config_name)
+    end
+
+    # @return [PgEventstore::SubscriptionQueries]
+    def subscription_queries
+      SubscriptionQueries.new(connection)
+    end
+  end
+end

data/lib/pg_eventstore/subscriptions/subscriptions_manager.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+require 'forwardable'
+require_relative 'basic_runner'
+require_relative 'subscription'
+require_relative 'events_processor'
+require_relative 'subscription_handler_performance'
+require_relative 'subscription_runner'
+require_relative 'runner_state'
+require_relative 'subscriptions_set'
+require_relative 'subscription_runners_feeder'
+require_relative 'subscription_feeder'
+require_relative 'commands_handler'
+
+module PgEventstore
+  # The public Subscriptions API, available to the user.
+  class SubscriptionsManager
+    extend Forwardable
+
+    attr_reader :config
+    private :config
+
+    def_delegators :@subscription_feeder, :start, :stop, :force_lock!
+
+    # @param config [PgEventstore::Config]
+    # @param set_name [String]
+    # @param max_retries [Integer, nil] max number of retries of failed SubscriptionsSet
+    # @param retries_interval [Integer, nil] a delay between retries of failed SubscriptionsSet
+    def initialize(config:, set_name:, max_retries: nil, retries_interval: nil)
+      @config = config
+      @set_name = set_name
+      @subscription_feeder = SubscriptionFeeder.new(
+        config_name: config.name,
+        set_name: set_name,
+        max_retries: max_retries || config.subscriptions_set_max_retries,
+        retries_interval: retries_interval || config.subscriptions_set_retries_interval
+      )
+    end
+
+    # @param subscription_name [String] subscription's name
+    # @param handler [#call] subscription's handler
+    # @param options [Hash] request options
+    # @option options [Integer, Symbol] :from_position a starting subscription position
+    # @option options [Boolean] :resolve_link_tos When using projections to create new events you
+    #   can set whether the generated events are pointers to existing events. Setting this option to true tells
+    #   PgEventstore to return the original event instead a link event.
+    # @option options [Hash] :filter provide it to filter events. It works the same way as a :filter option of
+    #   {PgEventstore::Client#read} method. Filtering by both - event types and streams are available.
+    # @param middlewares [Array<Symbol>, nil] provide a list of middleware names to override a config's middlewares
+    # @param pull_interval [Integer] an interval in seconds to determine how often to query new events of the given
+    #   subscription.
+    # @param max_retries [Integer] max number of retries of failed Subscription
+    # @param retries_interval [Integer] a delay between retries of failed Subscription
+    # @param restart_terminator [#call, nil] a callable object which, when called - accepts PgEventstore::Subscription
+    #   object to determine whether restarts should be stopped(true - stops restarts, false - continues restarts)
+    # @return [void]
+    def subscribe(subscription_name, handler:, options: {}, middlewares: nil,
+                  pull_interval: config.subscription_pull_interval,
+                  max_retries: config.subscription_max_retries,
+                  retries_interval: config.subscription_retries_interval,
+                  restart_terminator: config.subscription_restart_terminator)
+      subscription = Subscription.using_connection(config.name).new(
+        set: @set_name, name: subscription_name, options: options, chunk_query_interval: pull_interval,
+        max_restarts_number: max_retries, time_between_restarts: retries_interval
+      )
+      runner = SubscriptionRunner.new(
+        stats: SubscriptionHandlerPerformance.new,
+        events_processor: EventsProcessor.new(create_event_handler(middlewares, handler)),
+        subscription: subscription,
+        restart_terminator: restart_terminator
+      )
+
+      @subscription_feeder.add(runner)
+      true
+    end
+
+    # @return [Array<PgEventstore::Subscription>]
+    def subscriptions
+      @subscription_feeder.read_only_subscriptions
+    end
+
+    # @return [PgEventstore::SubscriptionsSet, nil]
+    def subscriptions_set
+      @subscription_feeder.read_only_subscriptions_set
+    end
+
+    private
+
+    # @param middlewares [Array<Symbol>, nil]
+    # @param handler [#call]
+    # @return [Proc]
+    def create_event_handler(middlewares, handler)
+      deserializer = EventDeserializer.new(select_middlewares(middlewares), config.event_class_resolver)
+      ->(raw_event) { handler.call(deserializer.deserialize(raw_event)) }
+    end
+
+    # @param middlewares [Array, nil]
+    # @return [Array<Object<#serialize, #deserialize>>]
+    def select_middlewares(middlewares = nil)
+      return config.middlewares.values unless middlewares
+
+      config.middlewares.slice(*middlewares).values
+    end
+  end
+end

data/lib/pg_eventstore/subscriptions/subscriptions_set.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+module PgEventstore
+  # Defines ruby's representation of subscriptions_set record.
+  class SubscriptionsSet
+    include Extensions::UsingConnectionExtension
+    include Extensions::OptionsExtension
+
+    class << self
+      # @param attrs [Hash]
+      # @return [PgEventstore::SubscriptionsSet]
+      def create(attrs)
+        new(**subscriptions_set_queries.create(attrs))
+      end
+
+      private
+
+      # @return [PgEventstore::SubscriptionsSetQueries]
+      def subscriptions_set_queries
+        SubscriptionsSetQueries.new(connection)
+      end
+    end
+
+    # @!attribute id
+    #   @return [String] UUIDv4. It is used to lock the Subscription by updating Subscription#locked_by attribute
+    attribute(:id)
+    # @!attribute name
+    #   @return [String] name of the set
+    attribute(:name)
+    # @!attribute state
+    #   @return [String]
+    attribute(:state)
+    # @!attribute restart_count
+    #   @return [Integer] the number of SubscriptionsSet's restarts after its failure
+    attribute(:restart_count)
+    # @!attribute max_restarts_number
+    #   @return [Integer] maximum number of times the SubscriptionsSet can be restarted
+    attribute(:max_restarts_number)
+    # @!attribute time_between_restarts
+    #   @return [Integer] interval in seconds between retries of failed SubscriptionsSet
+    attribute(:time_between_restarts)
+    # @!attribute last_restarted_at
+    #   @return [Time, nil] last time the SubscriptionsSet was restarted
+    attribute(:last_restarted_at)
+    # @!attribute last_error
+    #   @return [Hash{'class' => String, 'message' => String, 'backtrace' => Array<String>}, nil] the information about
+    #     last error caused when pulling Subscriptions events.
+    attribute(:last_error)
+    # @!attribute last_error_occurred_at
+    #   @return [Time, nil] the time when the last error occurred
+    attribute(:last_error_occurred_at)
+    # @!attribute created_at
+    #   @return [Time]
+    attribute(:created_at)
+    # @!attribute updated_at
+    #   @return [Time]
+    attribute(:updated_at)
+
+    # @param attrs [Hash]
+    # @return [Hash]
+    def assign_attributes(attrs)
+      attrs.each do |attr, value|
+        public_send("#{attr}=", value)
+      end
+    end
+
+    # @param attrs [Hash]
+    # @return [Hash]
+    def update(attrs)
+      assign_attributes(subscriptions_set_queries.update(id, attrs))
+    end
+
+    # @return [void]
+    def delete
+      subscriptions_set_queries.delete(id)
+    end
+
+    # Dup the current object without assigned connection
+    # @return [PgEventstore::SubscriptionsSet]
+    def dup
+      SubscriptionsSet.new(**Utils.deep_dup(options_hash))
+    end
+
+    # @return [PgEventstore::SubscriptionsSet]
+    def reload
+      assign_attributes(subscriptions_set_queries.find!(id))
+      self
+    end
+
+    private
+
+    # @return [PgEventstore::SubscriptionsSetQueries]
+    def subscriptions_set_queries
+      SubscriptionsSetQueries.new(self.class.connection)
+    end
+  end
+end

data/lib/pg_eventstore/tasks/setup.rake

@@ -32,7 +32,7 @@ namespace :pg_eventstore do
         conn.exec('SELECT number FROM migrations ORDER BY number DESC LIMIT 1').to_a.dig(0, 'number') || -1
 
       Dir.chdir migration_files_root do
-        Dir["*.{sql,rb}"].sort.each do |f_name|
+        Dir["*.{sql,rb}"].sort_by { |f_name| f_name.split('_').first.to_i }.each do |f_name|
           number = File.basename(f_name).split('_')[0].to_i
           next if latest_migration >= number
 
@@ -59,6 +59,10 @@ namespace :pg_eventstore do
         DROP TABLE IF EXISTS public.streams;
         DROP TABLE IF EXISTS public.event_types;
         DROP TABLE IF EXISTS public.migrations;
+        DROP TABLE IF EXISTS public.subscriptions_set;
+        DROP TABLE IF EXISTS public.subscriptions;
+        DROP TABLE IF EXISTS public.subscription_commands;
+        DROP TABLE IF EXISTS public.subscriptions_set_commands;
         DROP EXTENSION IF EXISTS "uuid-ossp";
         DROP EXTENSION IF EXISTS pgcrypto;
       SQL

data/lib/pg_eventstore/utils.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module PgEventstore
+  class Utils
+    class << self
+      # Deep transforms keys of a given Hash
+      # @param object [Object]
+      # @return [Object] a hash with transformed keys
+      def deep_transform_keys(object, &block)
+        case object
+        when Hash
+          object.each_with_object({}) do |(key, value), result|
+            result[yield(key)] = deep_transform_keys(value, &block)
+          end
+        when Array
+          object.map { |e| deep_transform_keys(e, &block) }
+        else
+          object
+        end
+      end
+
+      # Deep dup Array or Hash
+      # @param object [Object]
+      # @return [Object]
+      def deep_dup(object)
+        case object
+        when Hash
+          object.each_with_object({}) do |(key, value), result|
+            result[deep_dup(key)] = deep_dup(value)
+          end
+        when Array
+          object.map { |e| deep_dup(e) }
+        else
+          object.dup
+        end
+      end
+
+      # Converts array to the string containing SQL positional variables
+      # @param array [Array]
+      # @return [String] positional variables, based on array size. Example: "$1, $2, $3"
+      def positional_vars(array)
+        array.size.times.map { |t| "$#{t + 1}" }.join(', ')
+      end
+
+      # Transforms exception instance into a hash
+      # @param error [StandardError]
+      # @return [Hash]
+      def error_info(error)
+        {
+          class: error.class,
+          message: error.message,
+          backtrace: error.backtrace
+        }
+      end
+
+      # @param str [String]
+      # @return [String]
+      def underscore_str(str)
+        str = str.dup
+        str[0] = str[0].downcase
+        str.gsub!(/[A-Z]/) { |letter| '_' + letter.downcase }
+        str
+      end
+    end
+  end
+end

data/lib/pg_eventstore/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PgEventstore
-  VERSION = "0.3.0"
+  VERSION = "0.5.0"
 end