pg_eventstore 1.13.3 → 1.13.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 12b4fa45d100b1c19c06d3ec5066b1ba3016059d6465517f814f435427b68581
-  data.tar.gz: 86b78cbbe11da08a8d4fd1abeca95af36089ae0837abda4fd830d34dbf97badc
+  metadata.gz: 94443dc3aa36701d5e126573a5babdd4585932d803b766b06b94c861f6b4ef5a
+  data.tar.gz: 86fcca21a3a2d3da35f3c760542b23cc97ae93ce97bae7b9baf011a957a9ffea
 SHA512:
-  metadata.gz: 80f670af45edd8684746bb055c18c9174ed85e11ccfc4dede1cb129e83e1e8059be7ad9a8cd5880323c33595e8ad40dd8f81c7e4cce4c320bd60df29011349a7
-  data.tar.gz: 504bf0e5dcc20e134c7c2d3ef7ddfcdcf1af4b8c07567f251f00a0326f7441e0f4bab52b8a82d9abe90eadd27fc8e07b47a9b7f5eba53cc092bc2d32bef54caf
+  metadata.gz: 4fb5ed37edce4c557082ba60cfa18f7209f795fa29b98dff9f848bb83b0b1cec889f4a7ca7f77bef5c0d9c6b4065d53765dcfcdf50dffe2628c3e3ffb57d26ad
+  data.tar.gz: 0bc33722f13c57c11888ac205873ac93a3610ef5218e8dccf7c3e64d671852757e9d755ffe17ea8176b238dbe1722df47db522b5252ddceed52e14534c164c45

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 ## [Unreleased]
 
+## [1.13.4]
+
+- Fix subscriptions potentially skipping events when multiple events are appended in concurrent transactions
+
 ## [1.13.3]
 
 - Reduce subscription delays for newly published events
@@ -118,7 +122,7 @@ subscriptions_manager.subscribe(
 ```
 
 ## [1.1.5]
-- Review the way to handle SubscriptionAlreadyLockedError error. This removes noise when attempting to lock an already locked subscription. 
+- Review the way to handle SubscriptionAlreadyLockedError error. This removes noise when attempting to lock an already locked subscription.
 
 ## [1.1.4]
 - Add rbs signatures
@@ -167,7 +171,7 @@ subscriptions_manager.subscribe(
 
 ## [0.10.2] - 2024-03-13
 
-- Review the approach to resolve link events 
+- Review the approach to resolve link events
 - Fix subscriptions restart interval option not being processed correctly
 
 ## [0.10.1] - 2024-03-12
@@ -230,7 +234,7 @@ subscriptions_manager.subscribe(
 
 ## [0.3.0] - 2024-01-24
 
-- Log SQL queries when `PgEvenstore.logger` is set and it is in `:debug` mode 
+- Log SQL queries when `PgEvenstore.logger` is set and it is in `:debug` mode
 
 ## [0.2.6] - 2023-12-20
 

data/lib/pg_eventstore/event.rb

@@ -6,6 +6,8 @@ module PgEventstore
 
     # @return [String] a type of link event
     LINK_TYPE = '$>'
+    # @return [String]
+    PRIMARY_TABLE_NAME = 'events'
 
     # @!attribute id
     #   @return [String] UUIDv4 string

data/lib/pg_eventstore/queries/event_queries.rb

@@ -28,7 +28,8 @@ module PgEventstore
     def event_exists?(event)
       return false if event.id.nil? || event.stream.nil?
 
-      sql_builder = SQLBuilder.new.select('1 as exists').from('events').where('id = ?', event.id).limit(1)
+      sql_builder = SQLBuilder.new.select('1 as exists').from(Event::PRIMARY_TABLE_NAME).where('id = ?', event.id)
+      sql_builder.limit(1)
       sql_builder.where(
         'context = ? and stream_name = ? and type = ?', event.stream.context, event.stream.stream_name, event.type
       )
@@ -42,7 +43,7 @@ module PgEventstore
     # @param events [Array<PgEventstore::Event>]
     # @return [Array<String>]
     def ids_from_db(events)
-      sql_builder = SQLBuilder.new.from('events').select('id')
+      sql_builder = SQLBuilder.new.from(Event::PRIMARY_TABLE_NAME).select('id')
       partition_attrs = events.map { |event| [event.stream&.context, event.stream&.stream_name, event.type] }.uniq
       partition_attrs.each do |context, stream_name, event_type|
         sql_builder.where_or('context = ? and stream_name = ? and type = ?', context, stream_name, event_type)
@@ -57,7 +58,7 @@ module PgEventstore
     # @param stream [PgEventstore::Stream]
     # @return [Integer, nil]
     def stream_revision(stream)
-      sql_builder = SQLBuilder.new.from('events').select('stream_revision')
+      sql_builder = SQLBuilder.new.from(Event::PRIMARY_TABLE_NAME).select('stream_revision')
       sql_builder.where('context = ? and stream_name = ? and stream_id = ?', *stream.to_a)
       sql_builder.order('stream_revision DESC').limit(1)
       connection.with do |conn|

data/lib/pg_eventstore/queries/partition_queries.rb

@@ -178,14 +178,42 @@ module PgEventstore
 
     # @param stream_filters [Array<Hash[Symbol, String]>]
     # @param event_filters [Array<String>]
+    # @param scope [Symbol] what kind of partition we want to receive. Available options are :event_type, :context,
+    #   :stream_name and :auto. In :auto mode the scope will be calculated based on stream_filters and event_filters.
     # @return [Array<PgEventstore::Partition>]
-    def partitions(stream_filters, event_filters)
-      partitions_filter = QueryBuilders::PartitionsFiltering.new
-      stream_filters.each { |attrs| partitions_filter.add_stream_attrs(**attrs) }
-      partitions_filter.add_event_types(event_filters)
-      partitions_filter.with_event_types
+    def partitions(stream_filters, event_filters, scope: :event_type)
+      stream_filters = stream_filters.select { QueryBuilders::PartitionsFiltering.correct_stream_filter?(_1) }
+      sql_builder =
+        if event_filters.any?
+          # When event type filters are present - they apply constraints to any stream filter. Thus, we can't look up
+          # partitions by stream attributes separately.
+          filter = QueryBuilders::PartitionsFiltering.new
+          stream_filters.each { |attrs| filter.add_stream_attrs(**attrs) }
+          filter.add_event_types(event_filters)
+          set_partitions_scope(filter, stream_filters, event_filters, scope)
+        else
+          # When event type filters are absent - we can look up partitions by context and context/stream_name
+          # separately, thus potentially producing one-to-one mapping of filter-to-partition with :auto scope. For
+          # example, let's say we have stream attributes filter like
+          # [{ context: 'FooCtx', stream_name: 'Bar'}, { context: 'BarCtx' }], then we would be able to look up
+          # partitions by the exact match, returning only two of them according to the provided filters - stream
+          # partition for first filter and context partition for second filter.
+          builders = stream_filters.map do |attrs|
+            filter = QueryBuilders::PartitionsFiltering.new
+            filter.add_stream_attrs(**attrs)
+            set_partitions_scope(filter, [attrs], event_filters, scope)
+          end
+
+          sql_builder = SQLBuilder.union_builders(builders) if builders.any?
+          sql_builder ||
+            begin
+              builder = QueryBuilders::PartitionsFiltering.new
+              set_partitions_scope(builder, stream_filters, event_filters, scope)
+            end
+        end
+
       connection.with do |conn|
-        conn.exec_params(*partitions_filter.to_exec_params)
+        conn.exec_params(*sql_builder.to_exec_params)
       end.map(&method(:deserialize))
     end
 
@@ -210,6 +238,43 @@ module PgEventstore
 
     private
 
+    # @param partitions_filter [PgEventstore::QueryBuilders::PartitionsFiltering]
+    # @param stream_filters [Array<Hash[Symbol, String]>]
+    # @param event_filters [Array<String>]
+    # @param scope [Symbol]
+    # @return [PgEventstore::SQLBuilder]
+    def set_partitions_scope(partitions_filter, stream_filters, event_filters, scope)
+      case scope
+      when :event_type
+        partitions_filter.with_event_types
+      when :stream_name
+        filter = QueryBuilders::PartitionsFiltering.new
+        filter.without_event_types
+        filter.with_stream_names
+        builder = filter.to_sql_builder
+        builder.where(
+          '(context, stream_name) in ?',
+          partitions_filter.to_sql_builder.unselect.select('context, stream_name').group('context, stream_name')
+        )
+      when :context
+        filter = QueryBuilders::PartitionsFiltering.new
+        filter.without_event_types
+        filter.without_stream_names
+        builder = filter.to_sql_builder
+        builder.where('context in ?', partitions_filter.to_sql_builder.unselect.select('context').group('context'))
+      when :auto
+        if event_filters.any?
+          set_partitions_scope(partitions_filter, stream_filters, event_filters, :event_type)
+        elsif stream_filters.any? { _1[:stream_name] }
+          set_partitions_scope(partitions_filter, stream_filters, event_filters, :stream_name)
+        else
+          set_partitions_scope(partitions_filter, stream_filters, event_filters, :context)
+        end
+      else
+        raise NotImplementedError, "Don't know how to handle #{scope.inspect} scope!"
+      end
+    end
+
     # @param attrs [Hash]
     # @return [PgEventstore::Partition]
     def deserialize(attrs)

data/lib/pg_eventstore/query_builders/events_filtering.rb

@@ -4,10 +4,6 @@ module PgEventstore
   module QueryBuilders
     # @!visibility private
     class EventsFiltering < BasicFiltering
-      # @return [String]
-      TABLE_NAME = 'events'
-      private_constant :TABLE_NAME
-
       # @return [Integer]
       DEFAULT_LIMIT = 1_000
       # @return [Hash<String => String, Symbol => String>]
@@ -107,7 +103,7 @@ module PgEventstore
 
       # @return [String]
       def to_table_name
-        TABLE_NAME
+        Event::PRIMARY_TABLE_NAME
       end
 
       # @param context [String, nil]

data/lib/pg_eventstore/query_builders/partitions_filtering.rb

@@ -28,6 +28,19 @@ module PgEventstore
           end
           streams || []
         end
+
+        # @param stream_attrs [Hash]
+        # @return [Boolean]
+        def correct_stream_filter?(stream_attrs)
+          result = (stream_attrs in { context: String, stream_name: String } | { context: String })
+          return true if result
+
+          PgEventstore.logger&.debug(<<~TEXT)
+            Ignoring unsupported stream filter format for grouped read #{stream_attrs.compact.inspect}. \
+            See docs/reading_events.md docs for supported formats.
+          TEXT
+          false
+        end
       end
 
       # @return [String]
@@ -37,10 +50,10 @@ module PgEventstore
 
       # @param context [String, nil]
       # @param stream_name [String, nil]
-      # @return [void]
+      # @return [PgEventstore::SQLBuilder]
       def add_stream_attrs(context: nil, stream_name: nil)
         stream_attrs = { context: context, stream_name: stream_name }
-        return unless correct_stream_filter?(stream_attrs)
+        return @sql_builder unless self.class.correct_stream_filter?(stream_attrs)
 
         stream_attrs.compact!
         sql = stream_attrs.map do |attr, _|
@@ -50,31 +63,28 @@ module PgEventstore
       end
 
       # @param event_types [Array<String>]
-      # @return [void]
+      # @return [PgEventstore::SQLBuilder]
       def add_event_types(event_types)
-        return if event_types.empty?
+        return @sql_builder if event_types.empty?
 
         @sql_builder.where("#{to_table_name}.event_type = ANY(?::varchar[])", event_types)
       end
 
-      # @return [void]
+      # @return [PgEventstore::SQLBuilder]
       def with_event_types
         @sql_builder.where('event_type IS NOT NULL')
       end
 
-      private
+      def with_stream_names
+        @sql_builder.where('stream_name IS NOT NULL')
+      end
 
-      # @param stream_attrs [Hash]
-      # @return [Boolean]
-      def correct_stream_filter?(stream_attrs)
-        result = (stream_attrs in { context: String, stream_name: String } | { context: String, stream_name: nil })
-        return true if result
+      def without_event_types
+        @sql_builder.where('event_type IS NULL')
+      end
 
-        PgEventstore.logger&.debug(<<~TEXT)
-          Ignoring unsupported stream filter format for grouped read #{stream_attrs.compact.inspect}. \
-          See docs/reading_events.md docs for supported formats.
-        TEXT
-        false
+      def without_stream_names
+        @sql_builder.where('stream_name IS NULL')
       end
     end
   end

data/lib/pg_eventstore/sql_builder.rb

@@ -64,7 +64,7 @@ module PgEventstore
       self
     end
 
-    # @param table_name [String]
+    # @param table_name [String | SQLBuilder]
     # @return [self]
     def from(table_name)
       @from_value = table_name
@@ -132,7 +132,7 @@ module PgEventstore
       self
     end
 
-    # @return [Array<String, Array<Object>>]
+    # @return [[String, Array<_>]]
     def to_exec_params
       @positional_values.clear
       @positional_values_size = 0
@@ -141,19 +141,27 @@ module PgEventstore
 
     protected
 
-    # @return [Array<String, Array<Object>>]
+    # @return [[String, Array<_>]]
     def _to_exec_params
       return [single_query_sql, @positional_values] if @union_values.empty?
 
       [union_query_sql, @positional_values]
     end
 
+    # @return [String]
+    def from_sql
+      return @from_value if @from_value.is_a?(String)
+
+      sql = merge(@from_value)
+      "(#{sql}) #{@from_value.from_sql}"
+    end
+
     private
 
     # @return [String]
     def single_query_sql
       where_sql = [where_sql('OR'), where_sql('AND')].reject(&:empty?).map { |sql| "(#{sql})" }.join(' AND ')
-      sql = "SELECT #{select_sql} FROM #{@from_value}"
+      sql = "SELECT #{select_sql} FROM #{from_sql}"
       sql += " #{join_sql}" unless @join_values.empty?
       sql += " WHERE #{where_sql}" unless where_sql.empty?
       sql += " GROUP BY #{@group_values.join(', ')}" unless @group_values.empty?
@@ -168,11 +176,7 @@ module PgEventstore
       sql = single_query_sql
       union_parts = ["(#{sql})"]
       union_parts += @union_values.map do |builder|
-        builder.positional_values_size = @positional_values_size
-        builder_sql, values = builder._to_exec_params
-        @positional_values.push(*values)
-        @positional_values_size += values.size
-        "(#{builder_sql})"
+        "(#{merge(builder)})"
       end
       union_parts.join(' UNION ALL ')
     end
@@ -200,14 +204,28 @@ module PgEventstore
       @order_values.join(', ')
     end
 
+    # @param builder [PgEventstore::SQLBuilder]
+    # @return [String]
+    def merge(builder)
+      builder.positional_values_size = @positional_values_size
+      sql_query, positional_values = builder._to_exec_params
+      @positional_values.push(*positional_values)
+      @positional_values_size += positional_values.size
+      sql_query
+    end
+
     # Replaces "?" signs in the given string with positional variables and memorize positional values they refer to.
     # @param sql [String]
     # @return [String]
     def extract_positional_args(sql, *arguments)
       sql.gsub('?').each_with_index do |_, index|
-        @positional_values.push(arguments[index])
-        @positional_values_size += 1
-        "$#{@positional_values_size}"
+        if arguments[index].is_a?(SQLBuilder)
+          "(#{merge(arguments[index])})"
+        else
+          @positional_values.push(arguments[index])
+          @positional_values_size += 1
+          "$#{@positional_values_size}"
+        end
       end
     end
   end

data/lib/pg_eventstore/subscriptions/callback_handlers/subscription_runner_handlers.rb

@@ -52,6 +52,15 @@ module PgEventstore
       def update_subscription_state(subscription, state)
         subscription.update(state: state)
       end
+
+      # @param subscription_position_evaluation [PgEventstore::SubscriptionPositionEvaluation]
+      # @param state [String]
+      # @return [void]
+      def stop_position_evaluation(subscription_position_evaluation, state)
+        return if state == 'running'
+
+        subscription_position_evaluation.stop_evaluation
+      end
     end
   end
 end

data/lib/pg_eventstore/subscriptions/queries/service_queries.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module PgEventstore
+  # @!visibility private
+  class ServiceQueries
+    # @param connection [PgEventstore::Connection]
+    def initialize(connection)
+      @connection = connection
+    end
+
+    # @param relation_oids [Array<Integer>]
+    # @return [Array<String>]
+    def relation_transaction_ids(relation_oids)
+      result = @connection.with do |conn|
+        # Look up transactions that change table's content
+        conn.exec_params(
+          <<~SQL,
+            SELECT virtualtransaction AS trx_id FROM pg_locks
+              WHERE relation = ANY($1::oid[]) AND mode = 'RowExclusiveLock'
+          SQL
+          [relation_oids]
+        )
+      end
+      result.map { _1['trx_id'] }
+    end
+
+    # @param relation_ids [Array<Integer>]
+    # @param transaction_ids [Array<String>]
+    # @return [Boolean]
+    def transactions_in_progress?(relation_ids:, transaction_ids:)
+      result = @connection.with do |conn|
+        conn.exec_params(
+          <<~SQL,
+            SELECT 1 as one FROM pg_locks
+              WHERE virtualtransaction = ANY($1) AND relation = ANY($2::oid[]) AND mode = 'RowExclusiveLock'
+              LIMIT 1
+          SQL
+          [transaction_ids, relation_ids]
+        )
+      end
+      result.any?
+    end
+
+    # @param table_names [Array<String>] existing table names
+    # @return [Integer]
+    def max_global_position(table_names)
+      return 0 if table_names.empty?
+
+      partition_builds = table_names.map do |table_name|
+        SQLBuilder.new.select("MAX(#{table_name}.global_position) AS global_position").from(table_name)
+      end
+      sql, positional_values = SQLBuilder.union_builders(partition_builds).to_exec_params
+      result = @connection.with do |conn|
+        conn.exec_params("SELECT MAX(global_position) AS max_pos FROM (#{sql}) pos", positional_values)
+      end
+      result.first['max_pos'] || 0
+    end
+
+    # @param table_names [Array<String>]
+    # @return [Hash<String, Integer>]
+    def relation_ids_by_names(table_names)
+      result = @connection.with do |conn|
+        conn.exec_params(
+          <<~SQL,
+            SELECT relname, oid FROM pg_class WHERE relname = ANY($1::varchar[])
+          SQL
+          [table_names]
+        )
+      end
+      result.each_with_object({}) { |attrs, res| res[attrs['relname']] = attrs['oid'] }
+    end
+  end
+end

data/lib/pg_eventstore/subscriptions/queries/subscription_queries.rb

@@ -177,6 +177,7 @@ module PgEventstore
     # @return [PgEventstore::SQLBuilder]
     def query_builder(id, options)
       builder = PgEventstore::QueryBuilders::EventsFiltering.subscriptions_events_filtering(options).to_sql_builder
+      builder.where('global_position <= ?', options[:to_position]) if options[:to_position]
       builder.select("#{id} as runner_id")
     end
 

data/lib/pg_eventstore/subscriptions/subscription.rb

@@ -6,6 +6,16 @@ module PgEventstore
     include Extensions::UsingConnectionExtension
     include Extensions::OptionsExtension
 
+    # Determines the minimal allowed value of events pull frequency of the particular subscription. You can find similar
+    # constant - SubscriptionFeeder::EVENTS_PULL_INTERVAL. Unlike it - this one is responsible to detect whether the
+    # subscription should be included in the subscriptions list to query next chunk of events. Thus, this setting only
+    # determines whether it is time to make a request, but how frequent would be the actual request - determines
+    # SubscriptionFeeder::EVENTS_PULL_INTERVAL.
+    # @see PgEventstore::SubscriptionFeeder::EVENTS_PULL_INTERVAL
+    # @return [Float]
+    MIN_EVENTS_PULL_INTERVAL = 0.2
+    private_constant :MIN_EVENTS_PULL_INTERVAL
+
     # @return [Time]
     DEFAULT_TIMESTAMP = Time.at(0).utc.freeze
 
@@ -167,7 +177,7 @@ module PgEventstore
         restart_count: 0,
         last_restarted_at: nil,
         max_restarts_number: max_restarts_number,
-        chunk_query_interval: chunk_query_interval,
+        chunk_query_interval: [chunk_query_interval, MIN_EVENTS_PULL_INTERVAL].max,
         last_chunk_fed_at: DEFAULT_TIMESTAMP,
         last_chunk_greatest_position: nil,
         last_error: nil,

data/lib/pg_eventstore/subscriptions/subscription_feeder.rb

@@ -7,6 +7,12 @@ module PgEventstore
   class SubscriptionFeeder
     extend Forwardable
 
+    # Determines how often to fetch events from the event store.
+    # @see PgEventstore::Subscription::MIN_EVENTS_PULL_INTERVAL
+    # @return [Float]
+    EVENTS_PULL_INTERVAL = 0.1 # seconds
+    private_constant :EVENTS_PULL_INTERVAL
+
     attr_reader :config_name
 
     def_delegators :@basic_runner, :start, :stop, :restore, :state, :wait_for_finish, :stop_async, :running?
@@ -17,7 +23,7 @@ module PgEventstore
     def initialize(config_name:, subscriptions_set_lifecycle:, subscriptions_lifecycle:)
       @config_name = config_name
       @basic_runner = BasicRunner.new(
-        run_interval: 0.2,
+        run_interval: EVENTS_PULL_INTERVAL,
         async_shutdown_time: 0,
         recovery_strategies: recovery_strategies(config_name, subscriptions_set_lifecycle)
       )

data/lib/pg_eventstore/subscriptions/subscription_position_evaluation.rb

@@ -0,0 +1,195 @@
+# frozen_string_literal: true
+
+module PgEventstore
+  # When subscription pulls events at the edge of the events list and several events arrives concurrently - there is a
+  # chance some events will never be picked. Example:
+  # - event1 has been assigned global_position#9
+  # - event2 has been assigned global_position#10
+  # - event1 and event2 are currently in concurrent transactions, but those transactions does not block each other
+  # - transaction holding event2 commits first
+  # - a subscription picks event2 and sets next position to 11
+  # - transaction holding event1 commits
+  # Illustration:
+  #   Time → → → → → → → → → → → → → → → → → → → → → → → → → → → → → → → → → → → → → → → → → → → → → → → → → → → → → → →
+  #
+  #   T1:  BEGIN
+  #          INSERT INTO events(...);
+  #        --------------------global_position#9------------------->
+  #       COMMIT
+  #   T2:       BEGIN
+  #               INSERT INTO events(...);
+  #             --------------global_position#10-------->
+  #             COMMIT
+  #   Query events:                                       SELECT * FROM event WHERE global_position >= 8
+  #                                                       --------------Picks #8, #10, but never #9-------->
+  #
+  # To solve this problem we can:
+  # 1. pause events fetching for the subscription
+  # 2. fetch latest global position that matches subscription's filter
+  # 3. wait for all currently running transactions that affects on the subscription to finish
+  # 4. unpause events fetching and use this position as a right hand limiter, because we can now confidently say there
+  #   are no uncommited events that would be lost otherwise
+  # This class is responsible for the step 2 and step 3, and persists the last safe position to be used in step 4.
+  # @!visibility private
+  class SubscriptionPositionEvaluation
+    # Determines how often to check the list of currently running transactions
+    # @return [Float]
+    TRANSACTIONS_STATUS_REFRESH_INTERVAL = 0.05 # seconds
+    private_constant :TRANSACTIONS_STATUS_REFRESH_INTERVAL
+
+    # @param config_name [Symbol]
+    # @param filter_options [Hash]
+    def initialize(config_name:, filter_options:)
+      @config_name = config_name
+      @stream_filters = QueryBuilders::PartitionsFiltering.extract_streams_filter(filter: filter_options)
+      @event_type_filters = QueryBuilders::PartitionsFiltering.extract_event_types_filter(filter: filter_options)
+      @position_to_evaluate = nil
+      @position_is_safe = nil
+      @last_safe_position = nil
+      @runner = nil
+      @relation_ids_cache = {}
+      @mutex = Mutex.new
+    end
+
+    # @param position_to_evaluate [Integer]
+    # @return [self]
+    def evaluate(position_to_evaluate)
+      unless self.position_to_evaluate == position_to_evaluate
+        stop_evaluation
+        self.position_to_evaluate = position_to_evaluate
+        calculate_safe_position
+      end
+      self
+    end
+
+    # @return [Boolean]
+    def safe?
+      @mutex.synchronize { @position_is_safe } || false
+    end
+
+    # @return [Integer, nil]
+    def last_safe_position
+      @mutex.synchronize { @last_safe_position }
+    end
+
+    # @return [Thread, nil] a runner who is being stopped if any
+    def stop_evaluation
+      _stop_evaluation(@runner)
+    end
+
+    private
+
+    # @param runner [Thread, nil]
+    # @return [Thread, nil]
+    def _stop_evaluation(runner)
+      runner&.exit
+      self.position_is_safe = nil
+      self.last_safe_position = nil
+      self.position_to_evaluate = nil
+    end
+
+    # @return [Integer, nil]
+    def position_to_evaluate
+      @mutex.synchronize { @position_to_evaluate }
+    end
+
+    # @return [Boolean, nil]
+    def position_is_safe
+      @mutex.synchronize { @position_is_safe }
+    end
+
+    # @param val [Integer, nil]
+    # @return [Integer, nil]
+    def last_safe_position=(val)
+      @mutex.synchronize { @last_safe_position = val }
+    end
+
+    # @param val [Integer, nil]
+    # @return [Integer, nil]
+    def position_to_evaluate=(val)
+      @mutex.synchronize { @position_to_evaluate = val }
+    end
+
+    # @param val [Boolean, nil]
+    # @return [Boolean, nil]
+    def position_is_safe=(val)
+      @mutex.synchronize { @position_is_safe = val }
+    end
+
+    # @return [Array<String>]
+    def affected_tables
+      if @stream_filters.empty? && @event_type_filters.empty?
+        [Event::PRIMARY_TABLE_NAME]
+      else
+        partition_queries.partitions(@stream_filters, @event_type_filters, scope: :auto).map(&:table_name)
+      end
+    end
+
+    # @return [void]
+    def calculate_safe_position
+      @runner = Thread.new do
+        tables_to_track = affected_tables
+        update_relation_ids_cache(tables_to_track)
+        safe_position = service_queries.max_global_position(tables_to_track)
+        trx_ids = transaction_queries.transaction do
+          service_queries.relation_transaction_ids(@relation_ids_cache.values)
+        end
+
+        loop do
+          transactions_in_progress = service_queries.transactions_in_progress?(
+            relation_ids: @relation_ids_cache.values, transaction_ids: trx_ids
+          )
+          break unless transactions_in_progress
+
+          sleep TRANSACTIONS_STATUS_REFRESH_INTERVAL
+        end
+        @mutex.synchronize do
+          if safe_position >= @position_to_evaluate
+            # We progressed forward. New position can be persisted
+            @last_safe_position = safe_position
+            @position_is_safe = true
+          else
+            # safe_position is less than the current position to fetch the events from. This means that no new events
+            # are present at this point. We will need to re-evaluate the safe position during next attempts. Until that
+            # we can't progress.
+            @position_to_evaluate = nil
+          end
+        end
+      rescue
+        # Clean up the state immediately in case of error
+        _stop_evaluation(nil)
+      end
+    end
+
+    # @param tables_to_track [Array<String>]
+    # @return [void]
+    def update_relation_ids_cache(tables_to_track)
+      missing_relation_ids = tables_to_track - @relation_ids_cache.keys
+      deleted_relations = @relation_ids_cache.keys - tables_to_track
+      @relation_ids_cache = @relation_ids_cache.except(*deleted_relations)
+      return if missing_relation_ids.empty?
+
+      @relation_ids_cache.merge!(service_queries.relation_ids_by_names(missing_relation_ids))
+    end
+
+    # @return [PgEventstore::PartitionQueries]
+    def partition_queries
+      PartitionQueries.new(connection)
+    end
+
+    # @return [PgEventstore::TransactionQueries]
+    def transaction_queries
+      TransactionQueries.new(connection)
+    end
+
+    # @return [PgEventstore::ServiceQueries]
+    def service_queries
+      ServiceQueries.new(connection)
+    end
+
+    # @return [PgEventstore::Connection]
+    def connection
+      PgEventstore.connection(@config_name)
+    end
+  end
+end

data/lib/pg_eventstore/subscriptions/subscription_runner.rb

@@ -28,17 +28,23 @@ module PgEventstore
     # @param stats [PgEventstore::SubscriptionHandlerPerformance]
     # @param events_processor [PgEventstore::EventsProcessor]
     # @param subscription [PgEventstore::Subscription]
-    def initialize(stats:, events_processor:, subscription:)
+    # @param position_evaluation [PgEventstore::SubscriptionPositionEvaluation]
+    def initialize(stats:, events_processor:, subscription:, position_evaluation:)
       @stats = stats
       @events_processor = events_processor
       @subscription = subscription
+      @position_evaluation = position_evaluation
 
       attach_callbacks
     end
 
     # @return [Hash]
     def next_chunk_query_opts
-      @subscription.options.merge(from_position: next_chunk_global_position, max_count: estimate_events_number)
+      @subscription.options.merge(
+        from_position: next_chunk_global_position,
+        max_count: estimate_events_number,
+        to_position: @position_evaluation.last_safe_position
+      )
     end
 
     # @return [Boolean]
@@ -46,6 +52,11 @@ module PgEventstore
       @subscription.last_chunk_fed_at + @subscription.chunk_query_interval <= Time.now.utc
     end
 
+    # @return [Boolean]
+    def next_chunk_safe?
+      estimate_events_number == 0 ? false : @position_evaluation.evaluate(next_chunk_global_position).safe?
+    end
+
     private
 
     # @return [Integer]
@@ -97,6 +108,11 @@ module PgEventstore
         :change_state, :after,
         SubscriptionRunnerHandlers.setup_handler(:update_subscription_state, @subscription)
       )
+      # Prevent dangling position evaluation runner when subscription changes the state to something except 'running'
+      @events_processor.define_callback(
+        :change_state, :after,
+        SubscriptionRunnerHandlers.setup_handler(:stop_position_evaluation, @position_evaluation)
+      )
     end
   end
 end

data/lib/pg_eventstore/subscriptions/subscription_runners_feeder.rb

@@ -12,7 +12,7 @@ module PgEventstore
     # @param runners [Array<PgEventstore::SubscriptionRunner>]
     # @return [void]
     def feed(runners)
-      runners = runners.select(&:running?).select(&:time_to_feed?)
+      runners = runners.select(&:running?).select(&:time_to_feed?).select(&:next_chunk_safe?)
       return if runners.empty?
 
       runners_query_options = runners.to_h { |runner| [runner.id, runner.next_chunk_query_opts] }

data/lib/pg_eventstore/subscriptions/subscriptions_manager.rb

@@ -10,6 +10,7 @@ require_relative 'synchronized_array'
 require_relative 'events_processor'
 require_relative 'subscription_handler_performance'
 require_relative 'subscription_runner'
+require_relative 'subscription_position_evaluation'
 require_relative 'subscriptions_set'
 require_relative 'subscription_runners_feeder'
 require_relative 'subscriptions_set_lifecycle'
@@ -27,6 +28,7 @@ require_relative 'queries/subscription_command_queries'
 require_relative 'queries/subscription_queries'
 require_relative 'queries/subscriptions_set_command_queries'
 require_relative 'queries/subscriptions_set_queries'
+require_relative 'queries/service_queries'
 require_relative 'commands_handler'
 
 module PgEventstore
@@ -109,7 +111,11 @@ module PgEventstore
           graceful_shutdown_timeout: graceful_shutdown_timeout,
           recovery_strategies: recovery_strategies(subscription, restart_terminator, failed_subscription_notifier)
         ),
-        subscription: subscription
+        subscription: subscription,
+        position_evaluation: SubscriptionPositionEvaluation.new(
+          config_name: config.name,
+          filter_options: options[:filter] || {}
+        )
       )
 
       @subscriptions_lifecycle.runners.push(runner)
@@ -186,5 +192,10 @@ module PgEventstore
         ),
       ]
     end
+
+    # @return [PgEventstore::Connection]
+    def connection
+      PgEventstore.connection(config_name)
+    end
   end
 end

data/lib/pg_eventstore/version.rb

@@ -2,5 +2,5 @@
 
 module PgEventstore
   # @return [String]
-  VERSION = '1.13.3'
+  VERSION = '1.13.4'
 end

data/lib/pg_eventstore/web/paginator/events_collection.rb

@@ -58,7 +58,7 @@ module PgEventstore
             options.merge(from_position: from_position, max_count: per_page, direction: order == :asc ? :desc : :asc)
           ).to_sql_builder.unselect.select('global_position').offset(1)
           sql, params = sql_builder.to_exec_params
-          sql = "SELECT * FROM (#{sql}) events ORDER BY global_position #{order} LIMIT 1"
+          sql = "SELECT * FROM (#{sql}) #{Event::PRIMARY_TABLE_NAME} ORDER BY global_position #{order} LIMIT 1"
           connection.with do |conn|
             conn.exec_params(sql, params)
           end.to_a.dig(0, 'global_position')

data/lib/pg_eventstore/web/paginator/stream_ids_collection.rb

@@ -11,7 +11,7 @@ module PgEventstore
         def collection
           @collection ||=
             begin
-              sql_builder = SQLBuilder.new.select('stream_id').from('events')
+              sql_builder = SQLBuilder.new.select('stream_id').from(Event::PRIMARY_TABLE_NAME)
               sql_builder.where('context = ? and stream_name = ?', options[:context], options[:stream_name])
               sql_builder.where('stream_id like ?', "#{options[:query]}%")
               sql_builder.where("stream_id #{direction_operator} ?", starting_id) if starting_id
@@ -27,7 +27,7 @@ module PgEventstore
           return unless collection.size == per_page
 
           starting_id = collection.first['stream_id']
-          sql_builder = SQLBuilder.new.select('stream_id').from('events')
+          sql_builder = SQLBuilder.new.select('stream_id').from(Event::PRIMARY_TABLE_NAME)
           sql_builder.where("stream_id #{direction_operator} ?", starting_id)
           sql_builder.where('stream_id like ?', "#{options[:query]}%")
           sql_builder.where('context = ? and stream_name = ?', options[:context], options[:stream_name])

data/sig/pg_eventstore/event.rbs

@@ -1,7 +1,9 @@
 module PgEventstore
   class Event
-    include PgEventstore::Extensions::OptionsExtension
+    include Extensions::OptionsExtension
+
     LINK_TYPE: String
+    PRIMARY_TABLE_NAME: String
 
     def ==: ((Object | PgEventstore::Event) other) -> bool
 

data/sig/pg_eventstore/queries/partition_queries.rbs

@@ -39,7 +39,8 @@ module PgEventstore
     # _@return_ — partition attributes
     def context_partition: (PgEventstore::Stream stream) -> ::Hash[untyped, untyped]?
 
-    def partitions: (Array[Hash[Symbol, String | nil]] stream_filters, Array[String] event_filters)-> Array[Partition]
+    def partitions: (Array[Hash[Symbol, String | nil]] stream_filters, Array[String] event_filters,
+        ?scope: Symbol) -> Array[Partition]
 
     # _@param_ `stream`
     #
@@ -76,5 +77,8 @@ module PgEventstore
     private
 
     def deserialize: (Hash[untyped, untyped] attrs)-> Partition
+
+    def set_partitions_scope: (QueryBuilders::PartitionsFiltering partitions_filter, Array[untyped] stream_filters,
+        Array[untyped] event_filters, Symbol scope) -> SQLBuilder
   end
 end

data/sig/pg_eventstore/query_builders/partitions_filtering.rbs

@@ -7,15 +7,19 @@ module PgEventstore
 
       def self.extract_streams_filter: (Hash[untyped, untyped] options) -> Array[Hash[untyped, untyped]]
 
-      def add_event_types: (::Array[String] event_types) -> void
+      def self.correct_stream_filter?: (::Hash[untyped, untyped] stream_attrs) -> bool
 
-      def add_stream_attrs: (?context: String?, ?stream_name: String?) -> void
+      def add_event_types: (::Array[String] event_types) -> SQLBuilder
 
-      def with_event_types: -> void
+      def add_stream_attrs: (?context: String?, ?stream_name: String?) -> SQLBuilder
 
-      private
+      def with_event_types: -> SQLBuilder
 
-      def correct_stream_filter?: (::Hash[untyped, untyped] stream_attrs) -> bool
+      def with_stream_names: -> SQLBuilder
+
+      def without_event_types: -> SQLBuilder
+
+      def without_stream_names: -> SQLBuilder
     end
   end
 end

data/sig/pg_eventstore/sql_builder.rbs

@@ -2,6 +2,8 @@ module PgEventstore
   class SQLBuilder
     def initialize: () -> void
 
+    def from_sql: -> String
+
     # _@param_ `sql`
     def select: (String sql) -> self
 
@@ -18,7 +20,7 @@ module PgEventstore
     def where_or: (String sql, *Object arguments) -> self
 
     # _@param_ `table_name`
-    def from: (String table_name) -> self
+    def from: (String | SQLBuilder table_name) -> self
 
     # _@param_ `sql`
     #
@@ -53,7 +55,7 @@ module PgEventstore
     # _@param_ `val`
     def positional_values_size=: (Integer val) -> Integer
 
-    def _to_exec_params: () -> ::Array[(String | ::Array[untyped])]
+    def _to_exec_params: () -> [String, ::Array[untyped]]
 
     def single_query_sql: () -> String
 
@@ -70,5 +72,9 @@ module PgEventstore
 
     # _@param_ `sql`
     def extract_positional_args: (String sql, *untyped arguments) -> String
+
+    private
+
+    def merge: (SQLBuilder builder)-> String
   end
 end

data/sig/pg_eventstore/subscriptions/callback_handlers/subscription_runner_handlers.rbs

@@ -1,5 +1,8 @@
 module PgEventstore
   class SubscriptionRunnerHandlers
+    def self.stop_position_evaluation: (SubscriptionPositionEvaluation subscription_position_evaluation,
+        String state) -> void
+
     def self.track_exec_time: (PgEventstore::SubscriptionHandlerPerformance stats, ^() -> void, Integer _current_position) -> void
 
     def self.update_subscription_stats: (PgEventstore::Subscription subscription, PgEventstore::SubscriptionHandlerPerformance stats, Integer current_position) -> void

data/sig/pg_eventstore/subscriptions/queries/service_queries.rbs

@@ -0,0 +1,15 @@
+module PgEventstore
+  class ServiceQueries
+    @connection: Connection
+
+    def initialize: (Connection connection) -> void
+
+    def max_global_position: (Array[String] table_names) -> Integer
+
+    def relation_ids_by_names: (Array[String] table_names) -> Hash[String, Integer]
+
+    def relation_transaction_ids: (Array[Integer] relation_oids) -> Array[String]
+
+    def transactions_in_progress?: (relation_ids: Array[Integer], transaction_ids: Array[String]) -> bool
+  end
+end

data/sig/pg_eventstore/subscriptions/subscription.rbs

@@ -2,6 +2,8 @@ module PgEventstore
   class Subscription
     DEFAULT_TIMESTAMP: Time
 
+    MIN_EVENTS_PULL_INTERVAL: Float
+
     include PgEventstore::Extensions::UsingConnectionExtension
     include PgEventstore::Extensions::OptionsExtension
 

data/sig/pg_eventstore/subscriptions/subscription_feeder.rbs

@@ -1,5 +1,7 @@
 module PgEventstore
   class SubscriptionFeeder
+    EVENTS_PULL_INTERVAL: Float
+
     extend Forwardable
 
     @basic_runner: PgEventstore::BasicRunner

data/sig/pg_eventstore/subscriptions/subscription_position_evaluation.rbs

@@ -0,0 +1,53 @@
+module PgEventstore
+  class SubscriptionPositionEvaluation
+    TRANSACTIONS_STATUS_REFRESH_INTERVAL: Float
+
+    @config_name: Symbol
+    @event_type_filters: Array[String]
+    @last_safe_position: Integer?
+    @mutex: Mutex
+    @position_is_safe: bool?
+    @position_to_evaluate: Integer?
+    @relation_ids_cache: Hash[String, Integer]
+    @runner: Thread?
+    @stream_filters: Array[Hash[untyped, untyped]]
+
+    def initialize: (config_name: Symbol, filter_options: Hash[untyped, untyped]) -> void
+
+    def evaluate: (Integer position_to_evaluate) -> self
+
+    def last_safe_position: -> Integer?
+
+    def safe?: -> bool
+
+    def stop_evaluation: -> void
+
+    private
+
+    def _stop_evaluation: (Thread? runner)-> void
+
+    def affected_tables: -> Array[String]
+
+    def calculate_safe_position: -> void
+
+    def connection: -> Connection
+
+    def last_safe_position=: (Integer? val) -> Integer?
+
+    def partition_queries: -> PartitionQueries
+
+    def position_is_safe: -> bool?
+
+    def position_is_safe=: (bool? val) -> bool?
+
+    def position_to_evaluate: -> Integer?
+
+    def position_to_evaluate=: (Integer? val) -> Integer?
+
+    def service_queries: -> ServiceQueries
+
+    def transaction_queries: -> TransactionQueries
+
+    def update_relation_ids_cache: (Array[String] tables_to_track) -> void
+  end
+end

data/sig/pg_eventstore/subscriptions/subscription_runner.rbs

@@ -12,12 +12,13 @@ module PgEventstore
         stats: SubscriptionHandlerPerformance,
         events_processor: EventsProcessor,
         subscription: Subscription,
-        ?restart_terminator: _RestartTerminator?,
-        ?failed_subscription_notifier: _FailedSubscriptionNotifier?
+        position_evaluation: SubscriptionPositionEvaluation,
       ) -> void
 
     def next_chunk_query_opts: () -> ::Hash[untyped, untyped]
 
+    def next_chunk_safe?: -> bool
+
     def time_to_feed?: () -> bool
 
     def next_chunk_global_position: () -> Integer

data/sig/pg_eventstore/subscriptions/subscriptions_manager.rbs

@@ -77,6 +77,8 @@ module PgEventstore
 
     private
 
+    def connection: -> Connection
+
     def recovery_strategies: (
         Subscription subscription,
         _RestartTerminator? restart_terminator,

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pg_eventstore
 version: !ruby/object:Gem::Version
-  version: 1.13.3
+  version: 1.13.4
 platform: ruby
 authors:
 - Ivan Dzyzenko
@@ -167,6 +167,7 @@ files:
 - lib/pg_eventstore/subscriptions/events_processor.rb
 - lib/pg_eventstore/subscriptions/extensions/base_command_extension.rb
 - lib/pg_eventstore/subscriptions/extensions/command_class_lookup_extension.rb
+- lib/pg_eventstore/subscriptions/queries/service_queries.rb
 - lib/pg_eventstore/subscriptions/queries/subscription_command_queries.rb
 - lib/pg_eventstore/subscriptions/queries/subscription_queries.rb
 - lib/pg_eventstore/subscriptions/queries/subscriptions_set_command_queries.rb
@@ -187,6 +188,7 @@ files:
 - lib/pg_eventstore/subscriptions/subscription_feeder_commands/stop.rb
 - lib/pg_eventstore/subscriptions/subscription_feeder_commands/stop_all.rb
 - lib/pg_eventstore/subscriptions/subscription_handler_performance.rb
+- lib/pg_eventstore/subscriptions/subscription_position_evaluation.rb
 - lib/pg_eventstore/subscriptions/subscription_runner.rb
 - lib/pg_eventstore/subscriptions/subscription_runner_commands.rb
 - lib/pg_eventstore/subscriptions/subscription_runner_commands/base.rb
@@ -332,6 +334,7 @@ files:
 - sig/pg_eventstore/subscriptions/events_processor.rbs
 - sig/pg_eventstore/subscriptions/extensions/base_command_extension.rbs
 - sig/pg_eventstore/subscriptions/extensions/command_class_lookup_extension.rbs
+- sig/pg_eventstore/subscriptions/queries/service_queries.rbs
 - sig/pg_eventstore/subscriptions/queries/subscription_command_queries.rbs
 - sig/pg_eventstore/subscriptions/queries/subscription_queries.rbs
 - sig/pg_eventstore/subscriptions/queries/subscriptions_set_command_queries.rbs
@@ -351,6 +354,7 @@ files:
 - sig/pg_eventstore/subscriptions/subscription_feeder_commands/stop.rbs
 - sig/pg_eventstore/subscriptions/subscription_feeder_commands/stop_all.rbs
 - sig/pg_eventstore/subscriptions/subscription_handler_performance.rbs
+- sig/pg_eventstore/subscriptions/subscription_position_evaluation.rbs
 - sig/pg_eventstore/subscriptions/subscription_runner.rbs
 - sig/pg_eventstore/subscriptions/subscription_runner_commands.rbs
 - sig/pg_eventstore/subscriptions/subscription_runner_commands/base.rbs