pg_eventstore 1.11.0 → 1.13.0

data/lib/pg_eventstore/query_builders/events_filtering.rb

@@ -3,7 +3,11 @@
 module PgEventstore
   module QueryBuilders
     # @!visibility private
-    class EventsFiltering
+    class EventsFiltering < BasicFiltering
+      # @return [String]
+      TABLE_NAME = 'events'
+      private_constant :TABLE_NAME
+
       # @return [Integer]
       DEFAULT_LIMIT = 1_000
       # @return [Hash<String => String, Symbol => String>]
@@ -26,6 +30,7 @@ module PgEventstore
         # @return [PgEventstore::EventsFiltering]
         def events_filtering(stream, options)
           return all_stream_filtering(options) if stream.all_stream?
+
           if stream.system? && Stream::KNOWN_SYSTEM_STREAMS.include?(stream.context)
             return system_stream_filtering(stream, options)
           end
@@ -43,11 +48,9 @@ module PgEventstore
         # @return [PgEventstore::QueryBuilders::EventsFiltering]
         def all_stream_filtering(options)
           event_filter = new
-          options in { filter: { event_types: Array => event_types } }
-          event_filter.add_event_types(event_types)
+          event_filter.add_event_types(extract_event_types_filter(options))
           event_filter.add_limit(options[:max_count])
-          options in { filter: { streams: Array => streams } }
-          streams&.each { |attrs| event_filter.add_stream_attrs(**attrs) }
+          extract_streams_filter(options).each { |attrs| event_filter.add_stream_attrs(**attrs) }
           event_filter.add_global_position(options[:from_position], options[:direction])
           event_filter.add_all_stream_direction(options[:direction])
           event_filter
@@ -58,8 +61,7 @@ module PgEventstore
         # @return [PgEventstore::QueryBuilders::EventsFiltering]
         def specific_stream_filtering(stream, options)
           event_filter = new
-          options in { filter: { event_types: Array => event_types } }
-          event_filter.add_event_types(event_types)
+          event_filter.add_event_types(extract_event_types_filter(options))
           event_filter.add_limit(options[:max_count])
           event_filter.add_stream_attrs(**stream.to_hash)
           event_filter.add_revision(options[:from_revision], options[:direction])
@@ -75,14 +77,39 @@ module PgEventstore
             event_filter.set_source(stream.context)
           end
         end
+
+        # @param options [Hash]
+        # @return [Array<String>]
+        def extract_event_types_filter(options)
+          options in { filter: { event_types: Array => event_types } }
+          event_types&.select! do
+            _1.is_a?(String)
+          end
+          event_types || []
+        end
+
+        # @param options [Hash]
+        # @return [Array<Hash[Symbol, String]>]
+        def extract_streams_filter(options)
+          options in { filter: { streams: Array => streams } }
+          streams = streams&.map do
+            _1 in { context: String | NilClass => context }
+            _1 in { stream_name: String | NilClass => stream_name }
+            _1 in { stream_id: String | NilClass => stream_id }
+            { context: context, stream_name: stream_name, stream_id: stream_id }
+          end
+          streams || []
+        end
       end
 
       def initialize
-        @sql_builder =
-          SQLBuilder.new.
-            select('events.*').
-            from('events').
-            limit(DEFAULT_LIMIT)
+        super
+        @sql_builder.limit(DEFAULT_LIMIT)
+      end
+
+      # @return [String]
+      def to_table_name
+        TABLE_NAME
       end
 
       # @param context [String, nil]
@@ -95,21 +122,20 @@ module PgEventstore
 
         stream_attrs.compact!
         sql = stream_attrs.map do |attr, _|
-          "events.#{attr} = ?"
+          "#{to_table_name}.#{attr} = ?"
         end.join(" AND ")
         @sql_builder.where_or(sql, *stream_attrs.values)
       end
 
-      # @param event_types [Array<String>, nil]
+      # @param event_types [Array<String>]
       # @return [void]
       def add_event_types(event_types)
-        return if event_types.nil?
         return if event_types.empty?
 
         sql = event_types.size.times.map do
           "?"
         end.join(", ")
-        @sql_builder.where("events.type IN (#{sql})", *event_types)
+        @sql_builder.where("#{to_table_name}.type IN (#{sql})", *event_types)
       end
 
       # @param revision [Integer, nil]
@@ -118,7 +144,7 @@ module PgEventstore
       def add_revision(revision, direction)
         return unless revision
 
-        @sql_builder.where("events.stream_revision #{direction_operator(direction)} ?", revision)
+        @sql_builder.where("#{to_table_name}.stream_revision #{direction_operator(direction)} ?", revision)
       end
 
       # @param position [Integer, nil]
@@ -127,19 +153,19 @@ module PgEventstore
       def add_global_position(position, direction)
         return unless position
 
-        @sql_builder.where("events.global_position #{direction_operator(direction)} ?", position)
+        @sql_builder.where("#{to_table_name}.global_position #{direction_operator(direction)} ?", position)
       end
 
       # @param direction [String, Symbol, nil]
       # @return [void]
       def add_stream_direction(direction)
-        @sql_builder.order("events.stream_revision #{SQL_DIRECTIONS[direction]}")
+        @sql_builder.order("#{to_table_name}.stream_revision #{SQL_DIRECTIONS[direction]}")
       end
 
       # @param direction [String, Symbol, nil]
       # @return [void]
       def add_all_stream_direction(direction)
-        @sql_builder.order("events.global_position #{SQL_DIRECTIONS[direction]}")
+        @sql_builder.order("#{to_table_name}.global_position #{SQL_DIRECTIONS[direction]}")
       end
 
       # @param limit [Integer, nil]
@@ -150,20 +176,10 @@ module PgEventstore
         @sql_builder.limit(limit)
       end
 
-      # @return [PgEventstore::SQLBuilder]
-      def to_sql_builder
-        @sql_builder
-      end
-
-      # @return [Array]
-      def to_exec_params
-        @sql_builder.to_exec_params
-      end
-
       # @param table_name [String] system stream view name
       # @return [void]
       def set_source(table_name)
-        @sql_builder.from(%{ "#{PG::Connection.escape(table_name)}" events })
+        @sql_builder.from(%{ "#{PG::Connection.escape(table_name)}" #{to_table_name} })
       end
 
       private

data/lib/pg_eventstore/query_builders/partitions_filtering.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module PgEventstore
+  module QueryBuilders
+    # @!visibility private
+    class PartitionsFiltering < BasicFiltering
+      # @return [String]
+      TABLE_NAME = 'partitions'
+      private_constant :TABLE_NAME
+
+      class << self
+        # @param options [Hash]
+        # @return [Array<String>]
+        def extract_event_types_filter(options)
+          options in { filter: { event_types: Array => event_types } }
+          event_types&.select! do
+            _1.is_a?(String)
+          end
+          event_types || []
+        end
+
+        # @param options [Hash]
+        # @return [Array<Hash[Symbol, String]>]
+        def extract_streams_filter(options)
+          options in { filter: { streams: Array => streams } }
+          streams = streams&.map do
+            _1 in { context: String | NilClass => context }
+            _1 in { stream_name: String | NilClass => stream_name }
+            { context: context, stream_name: stream_name }
+          end
+          streams || []
+        end
+      end
+
+      # @return [String]
+      def to_table_name
+        TABLE_NAME
+      end
+
+      # @param context [String, nil]
+      # @param stream_name [String, nil]
+      # @return [void]
+      def add_stream_attrs(context: nil, stream_name: nil)
+        stream_attrs = { context: context, stream_name: stream_name }
+        return unless correct_stream_filter?(stream_attrs)
+
+        stream_attrs.compact!
+        sql = stream_attrs.map do |attr, _|
+          "#{to_table_name}.#{attr} = ?"
+        end.join(" AND ")
+        @sql_builder.where_or(sql, *stream_attrs.values)
+      end
+
+      # @param event_types [Array<String>]
+      # @return [void]
+      def add_event_types(event_types)
+        return if event_types.empty?
+
+        @sql_builder.where("#{to_table_name}.event_type = ANY(?::varchar[])", event_types)
+      end
+
+      # @return [void]
+      def with_event_types
+        @sql_builder.where('event_type IS NOT NULL')
+      end
+
+      private
+
+      # @param stream_attrs [Hash]
+      # @return [Boolean]
+      def correct_stream_filter?(stream_attrs)
+        result = (stream_attrs in { context: String, stream_name: String } | { context: String, stream_name: nil })
+        return true if result
+
+        PgEventstore&.logger&.debug(<<~TEXT)
+          Ignoring unsupported stream filter format for grouped read #{stream_attrs.compact.inspect}. \
+          See docs/reading_events.md docs for supported formats.
+        TEXT
+        false
+      end
+    end
+  end
+end

data/lib/pg_eventstore/sql_builder.rb

@@ -4,6 +4,16 @@ module PgEventstore
   # Deadly simple SQL builder
   # @!visibility private
   class SQLBuilder
+    class << self
+      # @param builders [Array<PgEventstore::SQLBuilder>]
+      # @return [PgEventstore::SQLBuilder]
+      def union_builders(builders)
+        builders[1..].each_with_object(builders[0]) do |builder, first_builder|
+          first_builder.union(builder)
+        end
+      end
+    end
+
     def initialize
       @select_values = []
       @from_value = nil

data/lib/pg_eventstore/subscriptions/basic_runner.rb

@@ -1,9 +1,9 @@
 # frozen_string_literal: true
 
 module PgEventstore
-  # Implements simple background jobs runner. The job execution is done via declaring a callback on the specific
-  # action. The implementation also allows you to hook into different places of life cycle of the runner by defining
-  # callbacks on various actions. Here is the list of available actions:
+  # Implements simple background job runner. A job execution is done via declaring a callback on a specific action. The
+  # implementation also allows you to hook into different places of life cycle of the runner by defining callbacks on
+  # various actions. Here is the list of available actions:
   # - :before_runner_started. Happens before the runner's state switches from "initial"/"stopped" to "running" and
   #   runner's thread is started. It is also fired when the runner is restoring - right after :before_runner_restored
   #   action.
@@ -24,12 +24,41 @@ module PgEventstore
   #
   #     def_delegators :@basic_runner, :start, :stop, :wait_for_finish, :stop_async, :restore
   #
+  #     class SimpleRecoveryStrategy
+  #       include PgEventstore::RunnerRecoveryStrategy
+  #
+  #       def initialize(restore_func)
+  #         @attempts_count = 0
+  #         @restore_func = restore_func
+  #       end
+  #
+  #       def recovers?(error)
+  #         error.message.include?("I can not handle this any more!")
+  #       end
+  #
+  #       def recover(error)
+  #         (@attempts_count < 3).tap do |res|
+  #           @attempts_count += 1
+  #           @restore_func.call if res
+  #         end
+  #       end
+  #     end
+  #
   #     def initialize
-  #       @basic_runner = PgEventstore::BasicRunner.new(1, 2)
+  #       @basic_runner = PgEventstore::BasicRunner.new(
+  #         run_interval: 1, async_shutdown_time: 2, recovery_strategies: recovery_strategies
+  #       )
   #       @jobs_performed = 0
+  #       @jobs_limit = 3
   #       attach_runner_callbacks
   #     end
   #
+  #     protected
+  #
+  #     def work_harder
+  #       @jobs_limit += 3
+  #     end
+  #
   #     private
   #
   #     def attach_runner_callbacks
@@ -42,7 +71,7 @@ module PgEventstore
   #     end
   #
   #     def process_action
-  #       raise "What's the point? I can not handle this any more!" if @jobs_performed >= 3
+  #       raise "What's the point? I can not handle this any more!" if @jobs_performed >= @jobs_limit
   #       puts "Doing some heavy lifting job"
   #       sleep 2 # Simulate long running job
   #     end
@@ -67,6 +96,10 @@ module PgEventstore
   #     def after_runner_died(error)
   #       puts "Error occurred: #{error.inspect}"
   #     end
+  #
+  #     def recovery_strategies
+  #       [SimpleRecoveryStrategy.new(method(:work_harder))]
+  #     end
   #   end
   #
   #   runner = MyAwesomeRunner.new
@@ -85,11 +118,13 @@ module PgEventstore
     #   :after_runner_stopped callback
     # @param async_shutdown_time [Integer, Float] seconds. Determines how long to wait before force-shutdown the runner.
     #   It is only meaningful for the #stop_async
-    def initialize(run_interval, async_shutdown_time)
+    def initialize(run_interval:, async_shutdown_time:, recovery_strategies: [])
       @run_interval = run_interval
       @async_shutdown_time = async_shutdown_time
+      @recovery_strategies = recovery_strategies
       @state = RunnerState.new
       @mutex = Thread::Mutex.new
+      @runner = nil
       delegate_change_state_cbx
     end
 
@@ -127,36 +162,40 @@ module PgEventstore
       synchronize do
         return self unless @state.running? || @state.dead?
 
-        @state.halting!
-        Thread.new do
-          stopping_at = Time.now.utc
-          halt = false
-          loop do
-            synchronize do
-              # Give the runner up to @async_shutdown_time seconds for graceful shutdown
-              @runner&.exit if Time.now.utc - stopping_at > @async_shutdown_time
+        begin
+          @state.halting!
+        ensure
+          Thread.new do
+            stopping_at = Time.now.utc
+            halt = false
+            loop do
+              synchronize do
+                # Give the runner up to @async_shutdown_time seconds for graceful shutdown
+                @runner&.exit if Time.now.utc - stopping_at > @async_shutdown_time
+
+                unless @runner&.alive?
+                  @state.stopped!
+                  callbacks.run_callbacks(:after_runner_stopped)
+                end
+              ensure
+                next if @runner&.alive?
 
-              unless @runner&.alive?
-                @state.stopped!
                 @runner = nil
-                callbacks.run_callbacks(:after_runner_stopped)
                 halt = true
               end
+              break if halt
+              sleep 0.1
             end
-            break if halt
-            sleep 0.1
           end
         end
-        self
       end
+      self
     end
 
     # Restores the runner after its death.
     # @return [self]
     def restore
-      synchronize do
-        return self unless @state.dead?
-
+      within_state(:dead) do
         callbacks.run_callbacks(:before_runner_restored)
         _start
       end
@@ -183,7 +222,7 @@ module PgEventstore
     end
 
     # @param state [Symbol]
-    # @return [Object] a result of evaluating of passed block
+    # @return [Object, nil] a result of evaluating of passed block
     def within_state(state, &blk)
       synchronize do
         return unless @state.public_send("#{RunnerState::STATES.fetch(state)}?")
@@ -192,6 +231,20 @@ module PgEventstore
       end
     end
 
+    protected
+
+    # @param error [StandardError]
+    # @param strategy [PgEventstore::RunnerRecoveryStrategy]
+    # @param current_runner_id [Integer]
+    # @return [Thread]
+    def async_recover(error, strategy, current_runner_id)
+      Thread.new do
+        init_recovery_ripper(current_runner_id)
+        Thread.current.exit unless strategy.recover(error)
+        recoverable { restore }
+      end
+    end
+
     private
 
     def synchronize
@@ -201,19 +254,15 @@ module PgEventstore
     # @return [void]
     def _start
       @state.running!
+    ensure
       @runner = Thread.new do
-        loop do
-          Thread.current.exit unless @state.running?
-          sleep @run_interval
-
-          callbacks.run_callbacks(:process_async)
-        end
-      rescue => error
-        synchronize do
-          raise unless @state.halting? || @state.running?
+        recoverable do
+          loop do
+            Thread.current.exit unless @state.running?
+            sleep @run_interval
 
-          @state.dead!
-          callbacks.run_callbacks(:after_runner_died, error)
+            callbacks.run_callbacks(:process_async)
+          end
         end
       end
     end
@@ -228,5 +277,43 @@ module PgEventstore
     def change_state(...)
       callbacks.run_callbacks(:change_state, ...)
     end
+
+    # @param error [StandardError]
+    # @return [PgEventstore::RunnerRecoveryStrategy, nil]
+    def suitable_strategy(error)
+      @recovery_strategies.find { _1.recovers?(error) }
+    end
+
+    # @return [void]
+    def recoverable
+      yield
+    rescue => error
+      synchronize do
+        raise unless @state.halting? || @state.running?
+
+        recovery_strategy = suitable_strategy(error)
+        @state.dead!
+        callbacks.run_callbacks(:after_runner_died, error)
+      ensure
+        async_recover(error, recovery_strategy, @runner.__id__) if recovery_strategy
+      end
+    end
+
+    # @param current_runner_id [Integer]
+    # @return [Thread]
+    def init_recovery_ripper(current_runner_id)
+      recovery_job = Thread.current
+      Thread.new do
+        loop do
+          synchronize do
+            recovery_job.exit unless @state.dead?
+            recovery_job.exit unless current_runner_id == @runner.__id__
+          end
+          break unless recovery_job.alive?
+
+          sleep 1
+        end
+      end
+    end
   end
 end

data/lib/pg_eventstore/subscriptions/callback_handlers/commands_handler_handlers.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 module PgEventstore
+  # @!visibility private
   class CommandsHandlerHandlers
     include Extensions::CallbackHandlersExtension
 
@@ -19,20 +20,6 @@ module PgEventstore
       def process_runners_commands(config_name, runners, subscription_feeder)
         CommandHandlers::SubscriptionRunnersCommands.new(config_name, runners, subscription_feeder.id).process
       end
-
-      # @param basic_runner [PgEventstore::BasicRunner]
-      # @param restart_delay [Integer]
-      # @param error [StandardError]
-      # @return [void]
-      def restore_runner(basic_runner, restart_delay, error)
-        PgEventstore.logger&.error "PgEventstore::CommandsHandler: Error occurred: #{error.message}"
-        PgEventstore.logger&.error "PgEventstore::CommandsHandler: Backtrace: #{error.backtrace&.join("\n")}"
-        PgEventstore.logger&.error "PgEventstore::CommandsHandler: Trying to auto-repair in #{restart_delay} seconds..."
-        Thread.new do
-          sleep restart_delay
-          basic_runner.restore
-        end
-      end
     end
   end
 end

data/lib/pg_eventstore/subscriptions/callback_handlers/events_processor_handlers.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 module PgEventstore
+  # @!visibility private
   class EventsProcessorHandlers
     include Extensions::CallbackHandlersExtension
 
@@ -15,10 +16,10 @@ module PgEventstore
 
         callbacks.run_callbacks(:process, Utils.original_global_position(raw_event)) do
           handler.call(raw_event)
+        rescue => exception
+          raw_events.unshift(raw_event)
+          raise Utils.wrap_exception(exception, global_position: Utils.original_global_position(raw_event))
         end
-      rescue => exception
-        raw_events.unshift(raw_event)
-        raise Utils.wrap_exception(exception, global_position: Utils.original_global_position(raw_event))
       end
 
       # @param callbacks [PgEventstore::Callbacks]

data/lib/pg_eventstore/subscriptions/callback_handlers/subscription_feeder_handlers.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 module PgEventstore
+  # @!visibility private
   class SubscriptionFeederHandlers
     include Extensions::CallbackHandlersExtension
 
@@ -39,20 +40,6 @@ module PgEventstore
         )
       end
 
-      # @param subscriptions_set_lifecycle [PgEventstore::SubscriptionsSetLifecycle]
-      # @param basic_runner [PgEventstore::BasicRunner]
-      # @param _error [StandardError]
-      # @return [void]
-      def restart_runner(subscriptions_set_lifecycle, basic_runner, _error)
-        subscriptions_set = subscriptions_set_lifecycle.persisted_subscriptions_set
-        return if subscriptions_set.restart_count >= subscriptions_set.max_restarts_number
-
-        Thread.new do
-          sleep subscriptions_set.time_between_restarts
-          basic_runner.restore
-        end
-      end
-
       # @param subscriptions_set_lifecycle [PgEventstore::SubscriptionsSetLifecycle]
       # @return [void]
       def ping_subscriptions_set(subscriptions_set_lifecycle)

data/lib/pg_eventstore/subscriptions/callback_handlers/subscription_runner_handlers.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 module PgEventstore
+  # @!visibility private
   class SubscriptionRunnerHandlers
     include Extensions::CallbackHandlersExtension
 
@@ -32,25 +33,6 @@ module PgEventstore
         subscription.update(last_error: Utils.error_info(error), last_error_occurred_at: Time.now.utc)
       end
 
-      # @param subscription [PgEventstore::Subscription]
-      # @param restart_terminator [#call, nil]
-      # @param failed_subscription_notifier [#call, nil]
-      # @param events_processor [PgEventstore::EventsProcessor]
-      # @param error [PgEventstore::WrappedException]
-      # @return [void]
-      def restart_events_processor(subscription, restart_terminator, failed_subscription_notifier, events_processor,
-                                   error)
-        return if restart_terminator&.call(subscription.dup)
-        if subscription.restart_count >= subscription.max_restarts_number
-          return failed_subscription_notifier&.call(subscription.dup, Utils.unwrap_exception(error))
-        end
-
-        Thread.new do
-          sleep subscription.time_between_restarts
-          events_processor.restore
-        end
-      end
-
       # @param subscription [PgEventstore::Subscription]
       # @param global_position [Integer]
       # @return [void]

data/lib/pg_eventstore/subscriptions/command_handlers/subscription_feeder_commands.rb

@@ -2,6 +2,7 @@
 
 module PgEventstore
   module CommandHandlers
+    # @!visibility private
     class SubscriptionFeederCommands
       # @param config_name [Symbol]
       # @param subscription_feeder [PgEventstore::SubscriptionFeeder]

data/lib/pg_eventstore/subscriptions/command_handlers/subscription_runners_commands.rb

@@ -2,6 +2,7 @@
 
 module PgEventstore
   module CommandHandlers
+    # @!visibility private
     class SubscriptionRunnersCommands
       # @param config_name [Symbol]
       # @param runners [Array<PgEventstore::SubscriptionRunner>]

data/lib/pg_eventstore/subscriptions/commands_handler.rb

@@ -10,8 +10,6 @@ module PgEventstore
   class CommandsHandler
     extend Forwardable
 
-    # @return [Integer] the delay in seconds between runner restarts
-    RESTART_DELAY = 5
     # @return [Integer] seconds, how often to check for new commands
     PULL_INTERVAL = 1
 
@@ -24,7 +22,11 @@ module PgEventstore
       @config_name = config_name
       @subscription_feeder = subscription_feeder
       @runners = runners
-      @basic_runner = BasicRunner.new(PULL_INTERVAL, 0)
+      @basic_runner = BasicRunner.new(
+        run_interval: PULL_INTERVAL,
+        async_shutdown_time: 0,
+        recovery_strategies: [RunnerRecoveryStrategies::RestoreConnection.new(config_name)]
+      )
       attach_runner_callbacks
     end
 
@@ -39,11 +41,6 @@ module PgEventstore
         :process_async, :before,
         CommandsHandlerHandlers.setup_handler(:process_runners_commands, @config_name, @runners, @subscription_feeder)
       )
-
-      @basic_runner.define_callback(
-        :after_runner_died, :before,
-        CommandsHandlerHandlers.setup_handler(:restore_runner, @basic_runner, RESTART_DELAY)
-      )
     end
   end
 end