pg_eventstore 2.0.0 → 3.0.0

data/lib/pg_eventstore/subscriptions/queries/event_subscription_position_queries.rb

@@ -0,0 +1,177 @@
+# frozen_string_literal: true
+
+module PgEventstore
+  # @!visibility private
+  class EventSubscriptionPositionQueries
+    # Max number of events_global_index records to update
+    # @return [Integer]
+    MAX_INDEX_RECORDS_TO_UPDATE_SUBSCRIPTION_POSITION = 100_000
+    # @return [Integer]
+    ASSIGN_SUBSCRIPTION_POSITIONS_LOCK_ID = 462_315_339_855_922
+    # @return [Integer]
+    REINDEX_PERIOD = 24 * 60 * 60 # 1 day
+    # Do not reindex if index size is under this value
+    # @return [Integer]
+    UNPROCESSED_POSITIONS_INDEX_SIZE_THRESHOLD = 500 * 1_024 * 1_024 # 500 MB
+    # @return [String]
+    UNPROCESSED_POSITIONS_INDEX_NAME = 'idx_event_subscription_positions_unprocessed_gposition'
+    # @return [Integer]
+    UNPROCESSED_POSITIONS_LOCK_EXPIRES_IN = 5 * 60 # 5 minutes
+
+    # @!attribute connection
+    #   @return [PgEventstore::Connection]
+    attr_reader :connection
+    private :connection
+
+    # @param connection [PgEventstore::Connection]
+    def initialize(connection)
+      @connection = connection
+    end
+
+    # @param raw_events [Array<Hash>]
+    # @return [void]
+    def create_unprocessed_positions(raw_events)
+      values = raw_events.map { "(#{_1['global_position']})" }.join(', ')
+      connection.with do |conn|
+        conn.exec(<<~SQL)
+          INSERT INTO event_subscription_positions_unprocessed ("global_position") VALUES #{values};
+        SQL
+      end
+    end
+
+    # @param events [Array<PgEventstore::Event>]
+    # @return [Hash<Integer, Integer>]
+    def subscription_positions_from_db(events)
+      filtering = QueryBuilders::EventSubscriptionPositionsFiltering.new
+      filtering.by_global_positions(events.map(&:global_position))
+      connection.with do |conn|
+        conn.exec_params(*filtering.to_exec_params).to_h do |attrs|
+          [attrs['global_position'], attrs['subscription_position']]
+        end
+      end
+    end
+
+    # @return [Integer, nil]
+    def max_subscription_position
+      filtering = QueryBuilders::EventSubscriptionPositionsFiltering.new
+      filtering.max_subscription_position
+      connection.with do |conn|
+        conn.exec_params(*filtering.to_exec_params).first['max_subscription_position']
+      end
+    end
+
+    # @return [Integer, nil] number of updated records
+    def assign_subscription_position
+      transaction_queries.transaction(:read_committed) do
+        connection.with do |conn|
+          locked = conn.exec_params(
+            'select pg_try_advisory_xact_lock($1::bigint) as locked',
+            [ASSIGN_SUBSCRIPTION_POSITIONS_LOCK_ID]
+          ).first['locked']
+          return unless locked
+
+          new_positions = conn.exec_params(<<~SQL, [MAX_INDEX_RECORDS_TO_UPDATE_SUBSCRIPTION_POSITION])
+            select global_position
+            from event_subscription_positions_unprocessed
+            order by global_position
+            limit $1
+          SQL
+          return 0 if new_positions.ntuples == 0
+
+          affected_positions = new_positions.map { _1['global_position'] }
+          new_positions_values = affected_positions.map { "(#{_1})" }.join(', ')
+
+          conn.exec(<<~SQL)
+            insert into event_subscription_positions (global_position) values #{new_positions_values}
+          SQL
+          affected_positions = new_positions.map { _1['global_position'] }
+          conn.exec(<<~SQL).cmd_tuples
+            delete from event_subscription_positions_unprocessed
+            where global_position in (#{affected_positions.join(',')})
+          SQL
+        end
+      end
+    end
+
+    # There is no concept of periodic maintenance tasks yet. Simply hardcode it for now and abstract when needed.
+    # @return [Time, nil]
+    def reindex_unprocessed_positions
+      task_name = 'reindex_unprocessed_positions'
+      task = connection.with do |conn|
+        conn.exec_params(
+          'select * from maintenance_tasks where task_name = $1',
+          [task_name]
+        ).first
+      end
+      if task && task['performed_at'] && task['performed_at'] + REINDEX_PERIOD > Time.now
+        return task['performed_at'] + REINDEX_PERIOD
+      end
+
+      if task
+        locked =
+          if task['locked_at']
+            if task['locked_at'] + UNPROCESSED_POSITIONS_LOCK_EXPIRES_IN > Time.now
+              # the task is locked by someone else. Retry later
+              return task['locked_at'] + UNPROCESSED_POSITIONS_LOCK_EXPIRES_IN
+            end
+
+            connection.with do |conn|
+              conn.exec_params(
+                'update maintenance_tasks set locked_at = $1 where task_name = $2 and locked_at = $3',
+                [Time.now.utc, task_name, task['locked_at']]
+              )
+            end.cmd_tuples == 1
+          else
+            connection.with do |conn|
+              conn.exec_params(
+                'update maintenance_tasks set locked_at = $1 where task_name = $2 and locked_at is null',
+                [Time.now.utc, task_name]
+              )
+            end.cmd_tuples == 1
+          end
+        return unless locked
+      else
+        connection.with do |conn|
+          conn.exec_params(
+            'insert into maintenance_tasks (task_name, locked_at) values ($1, $2) returning *',
+            [task_name, Time.now.utc]
+          ).first
+        end
+      end
+
+      index_name = UNPROCESSED_POSITIONS_INDEX_NAME
+      if index_size(index_name) > UNPROCESSED_POSITIONS_INDEX_SIZE_THRESHOLD
+        connection.with do |conn|
+          conn.exec("reindex index concurrently #{index_name}")
+        end
+      end
+      connection.with do |conn|
+        conn.exec_params(
+          'update maintenance_tasks set performed_at = $1, locked_at = null where task_name = $2',
+          [Time.now.utc, task_name]
+        )
+      end
+      Time.now.utc + REINDEX_PERIOD
+    rescue PG::UniqueViolation
+      # concurrent process has created the record before us
+    end
+
+    # @param index_name [String]
+    # @return [Integer]
+    def index_size(index_name)
+      connection.with do |conn|
+        conn.exec_params(
+          'select pg_relation_size(c.oid) as size from pg_class c where c.relname = $1',
+          [index_name]
+        ).first['size'] || 0
+      end
+    end
+
+    private
+
+    # @return [PgEventstore::TransactionQueries]
+    def transaction_queries
+      TransactionQueries.new(connection)
+    end
+  end
+end

data/lib/pg_eventstore/subscriptions/queries/subscription_queries.rb

@@ -9,8 +9,10 @@ module PgEventstore
     private :connection
 
     # @param connection [PgEventstore::Connection]
-    def initialize(connection)
+    # @param query_strategy [PgEventstore::QueryStrategy]
+    def initialize(connection, query_strategy)
       @connection = connection
+      @query_strategy = query_strategy
     end
 
     # @param attrs [Hash]
@@ -25,9 +27,7 @@ module PgEventstore
     # @return [Hash, nil]
     def find_by(attrs)
       builder = find_by_attrs_builder(attrs).limit(1)
-      pg_result = connection.with do |conn|
-        conn.exec_params(*builder.to_exec_params)
-      end
+      pg_result = @query_strategy.exec_params(*builder.to_exec_params)
       return if pg_result.ntuples == 0
 
       deserialize(pg_result.to_a.first)
@@ -37,9 +37,7 @@ module PgEventstore
     # @return [Array<Hash>]
     def find_all(attrs)
       builder = find_by_attrs_builder(attrs)
-      pg_result = connection.with do |conn|
-        conn.exec_params(*builder.to_exec_params)
-      end
+      pg_result = @query_strategy.exec_params(*builder.to_exec_params)
       return [] if pg_result.ntuples == 0
 
       pg_result.map(&method(:deserialize))
@@ -50,9 +48,7 @@ module PgEventstore
     def set_collection(state = nil)
       builder = SQLBuilder.new.from('subscriptions').select('set').group('set').order('set ASC')
       builder.where('state = ?', state) if state
-      raw_subscriptions = connection.with do |conn|
-        conn.exec_params(*builder.to_exec_params)
-      end
+      raw_subscriptions = @query_strategy.exec_params(*builder.to_exec_params)
       raw_subscriptions.map { |attrs| attrs['set'] }
     end
 
@@ -71,12 +67,67 @@ module PgEventstore
           VALUES (#{Utils.positional_vars(attrs.values)})
           RETURNING *
       SQL
-      pg_result = connection.with do |conn|
-        conn.exec_params(sql, attrs.values)
-      end
+      pg_result = @query_strategy.exec_params(sql, attrs.values)
       deserialize(pg_result.to_a.first)
     end
 
+    # @param id [Integer] Subscription#id
+    # @param options [Hash]
+    # @option options [Hash] :filter
+    # @param locked_by [Integer] SubscriptionSet#id
+    # @return [void]
+    def create_or_replace_table_function(id, options, locked_by)
+      filter_collection = QueryBuilders::Filters::Collection.from_options(options)
+      builder = QueryBuilders::IndexBasedEventsFiltering.sql_builder_for_subscriptions(filter_collection.collection)
+      function_name = QueryBuilders::SubscriptionEventsFiltering.new(id).to_table_name
+      transaction_queries.transaction(:read_committed) do
+        connection.with do |conn|
+          attrs = conn.exec_params('select * from subscriptions where id = $1 for update', [id]).to_a.first
+          unless attrs['locked_by'] == locked_by
+            # Subscription is force-locked by someone else. We have to roll back such transaction
+            raise(WrongLockIdError.new(attrs['set'], attrs['name'], attrs['locked_by']))
+          end
+
+          compiled = conn.compile(*builder.to_exec_params)
+          compiled = <<~SQL
+            create or replace function #{function_name}(
+              from_position bigint,
+              to_position bigint,
+              max_count int
+            )
+            returns table (
+              global_position bigint,
+              event_type_partition_id bigint,
+              subscription_position bigint
+            )
+            language plpgsql
+            stable
+            parallel safe
+            as $$
+            declare
+              from_gpos bigint;
+              to_gpos bigint;
+            begin
+              with candidates as (
+                select esp.global_position
+                from event_subscription_positions esp
+                where esp.subscription_position >= from_position
+                  and esp.subscription_position <= to_position
+              )
+              /* + 0 here is to keep PostgreSQL from trying to optimize the query by picking wrong index */
+              select coalesce(min(candidates.global_position + 0), 0), coalesce(max(candidates.global_position + 0), 0)
+              into from_gpos, to_gpos
+              from candidates;
+
+              return query #{compiled};
+            end;
+            $$;
+          SQL
+          conn.exec(compiled)
+        end
+      end
+    end
+
     # @param id [Integer]
     # @param attrs [Hash]
     # @param locked_by [Integer, nil]
@@ -91,9 +142,7 @@ module PgEventstore
       sql =
         "UPDATE subscriptions SET #{attrs_sql} WHERE id = $#{attrs.keys.size + 1} RETURNING *"
       updated_attrs = transaction_queries.transaction(:read_committed) do
-        pg_result = connection.with do |conn|
-          conn.exec_params(sql, [*attrs.values, id])
-        end
+        pg_result = @query_strategy.exec_params(sql, [*attrs.values, id])
         raise(RecordNotFound.new('subscriptions', id)) if pg_result.ntuples == 0
 
         updated_attrs = pg_result.to_a.first
@@ -112,34 +161,16 @@ module PgEventstore
     # @param subscriptions_ids [Array<Integer>] Array of Subscription#id
     # @return [Hash<Integer => Time>]
     def ping_all(subscriptions_set_id, subscriptions_ids)
-      pg_result = connection.with do |conn|
-        sql = <<~SQL
-          UPDATE subscriptions SET updated_at = $1 WHERE locked_by = $2 AND id = ANY($3::int[])
-            RETURNING id, updated_at
-        SQL
-        conn.exec_params(sql, [Time.now.utc, subscriptions_set_id, subscriptions_ids])
-      end
+      sql = <<~SQL
+        UPDATE subscriptions SET updated_at = $1 WHERE locked_by = $2 AND id = ANY($3::int[])
+          RETURNING id, updated_at
+      SQL
+      pg_result = @query_strategy.exec_params(sql, [Time.now.utc, subscriptions_set_id, subscriptions_ids])
       pg_result.to_h do |attrs|
         [attrs['id'], attrs['updated_at']]
       end
     end
 
-    # @param query_options [Hash{Integer => Hash}] runner_id/query options association
-    # @return [Hash{Integer => Array<Hash>}] runner_id/events association
-    def subscriptions_events(query_options)
-      return {} if query_options.empty?
-
-      final_builder = SQLBuilder.union_builders(query_options.map { |id, opts| query_builder(id, opts) })
-      raw_events = connection.with do |conn|
-        conn.exec_params(*final_builder.to_exec_params)
-      end.to_a
-      raw_events.group_by { _1['runner_id'] }.to_h do |runner_id, runner_raw_events|
-        next [runner_id, runner_raw_events] unless query_options[runner_id][:resolve_link_tos]
-
-        [runner_id, links_resolver.resolve(runner_raw_events)]
-      end
-    end
-
     # @param id [Integer] subscription's id
     # @param lock_id [Integer] id of the subscriptions set which reserves the subscription
     # @param force [Boolean] whether to lock the subscription despite on #locked_by value
@@ -155,9 +186,7 @@ module PgEventstore
           raise SubscriptionAlreadyLockedError.new(attrs[:set], attrs[:name], attrs[:locked_by])
         end
 
-        connection.with do |conn|
-          conn.exec_params('UPDATE subscriptions SET locked_by = $1 WHERE id = $2', [lock_id, id])
-        end
+        @query_strategy.exec_params('UPDATE subscriptions SET locked_by = $1 WHERE id = $2', [lock_id, id])
       end
       lock_id
     end
@@ -165,22 +194,17 @@ module PgEventstore
     # @param id [Integer]
     # @return [void]
     def delete(id)
-      connection.with do |conn|
-        conn.exec_params('DELETE FROM subscriptions WHERE id = $1', [id])
+      function_name = QueryBuilders::SubscriptionEventsFiltering.new(id).to_table_name
+      transaction_queries.transaction(:read_committed) do
+        connection.with do |conn|
+          conn.exec_params('delete from subscriptions where id = $1', [id])
+          conn.exec("drop function if exists #{function_name}")
+        end
       end
     end
 
     private
 
-    # @param id [Integer] runner id
-    # @param options [Hash] query options
-    # @return [PgEventstore::SQLBuilder]
-    def query_builder(id, options)
-      builder = PgEventstore::QueryBuilders::EventsFiltering.subscriptions_events_filtering(options).to_sql_builder
-      builder.where('global_position <= ?', options[:to_position]) if options[:to_position]
-      builder.select("#{id} as runner_id")
-    end
-
     # @return [PgEventstore::TransactionQueries]
     def transaction_queries
       TransactionQueries.new(connection)
@@ -188,7 +212,7 @@ module PgEventstore
 
     # @return [PgEventstore::LinksResolver]
     def links_resolver
-      LinksResolver.new(connection)
+      LinksResolver.new(connection, @query_strategy)
     end
 
     # @param hash [Hash]

data/lib/pg_eventstore/subscriptions/replica_subscription_handler.rb

@@ -0,0 +1,140 @@
+# frozen_string_literal: true
+
+module PgEventstore
+  class ReplicaSubscriptionHandler
+    def initialize(config_name, replica_config_name)
+      @replica_config_name = replica_config_name
+      @config_name = config_name
+    end
+
+    # @param indexes [Array<PgEventstore::EventGlobalIndex::SubscriptionRepr>]
+    # @return [void]
+    def call(indexes)
+      indexes = reject_already_processed(indexes)
+      runner = AsyncRunner.new
+      objects_to_migrate = {}
+      runner.async do
+        objects_to_migrate[:raw_events] = source_replica_queries.load_events(indexes)
+      end
+      runner.async do
+        objects_to_migrate[:markers_index] = source_replica_queries.load_event_markers_index(indexes)
+        objects_to_migrate[:markers] = source_replica_queries.load_markers(objects_to_migrate[:markers_index])
+      end
+      runner.async do
+        objects_to_migrate[:events_index] = source_replica_queries.load_events_global_index(indexes)
+        objects_to_migrate[:streams_index] = source_replica_queries.load_streams_global_index(
+          objects_to_migrate[:events_index]
+        )
+        objects_to_migrate[:partitions] = source_replica_queries.load_partitions(objects_to_migrate[:events_index])
+      end
+      runner.run
+
+      sql = []
+      sql << records_to_sql(
+        QueryBuilders::PartitionsFiltering::TABLE_NAME,
+        Partition.options.map(&:name),
+        objects_to_migrate[:partitions].map(&:options_hash),
+        on_conflict: 'on conflict do nothing'
+      )
+      sql << records_to_sql(
+        Event::PRIMARY_TABLE_NAME,
+        RawEvent.options.map(&:name),
+        objects_to_migrate[:raw_events].map(&:options_hash)
+      )
+      sql << records_to_sql(
+        QueryBuilders::EventsGlobalIndexFiltering::PRIMARY_TABLE_NAME,
+        EventGlobalIndex.options.map(&:name),
+        objects_to_migrate[:events_index].map(&:options_hash)
+      )
+      sql << records_to_sql(
+        QueryBuilders::StreamsGlobalIndexFiltering::PRIMARY_TABLE_NAME,
+        StreamGlobalIndex.options.map(&:name),
+        objects_to_migrate[:streams_index].map(&:options_hash),
+        on_conflict: 'on conflict (id) do update set stream_revision = EXCLUDED.stream_revision'
+      )
+      sql << records_to_sql(
+        QueryBuilders::EventSubscriptionPositionsFiltering::PRIMARY_TABLE_NAME,
+        %i[global_position subscription_position],
+        indexes.map(&:to_subscription_position_attrs)
+      )
+      if objects_to_migrate[:markers_index].any?
+        sql << records_to_sql(
+          QueryBuilders::EventMarkersIndexFiltering::PRIMARY_TABLE_NAME,
+          EventMarkerIndex.options.map(&:name),
+          objects_to_migrate[:markers_index].map(&:options_hash)
+        )
+      end
+      if objects_to_migrate[:markers].any?
+        sql << records_to_sql(
+          QueryBuilders::EventMarkersFiltering::PRIMARY_TABLE_NAME,
+          EventMarker.options.map(&:name),
+          objects_to_migrate[:markers].map(&:options_hash),
+          on_conflict: 'on conflict do nothing'
+        )
+      end
+      sql = sql.join("\n")
+      replica_transaction_queries.transaction(:read_committed) do
+        replica_connection.with do |conn|
+          conn.exec(sql)
+        end
+      end
+    end
+
+    private
+
+    # @param indexes [Array<PgEventstore::EventGlobalIndex::SubscriptionRepr>]
+    # @return [Array<PgEventstore::EventGlobalIndex::SubscriptionRepr>]
+    def reject_already_processed(indexes)
+      existing_positions = destination_replica_queries.load_subscription_positions(indexes)
+      return indexes if existing_positions.empty?
+
+      indexes.reject do |index|
+        existing_positions.include?(index.subscription_position)
+      end
+    end
+
+    # @param table_name [String]
+    # @param attribute_names [Array<Symbol>]
+    # @param attributes_collection [Array<Hash<Symbol, Object>>]
+    # @param on_conflict [String, nil]
+    # @return [String]
+    def records_to_sql(table_name, attribute_names, attributes_collection, on_conflict: nil)
+      sql_values = attributes_collection.map do |attributes|
+        values = attribute_names.map do |attribute_name|
+          connection.with do |conn|
+            conn.prepared_value(attributes[attribute_name])
+          end
+        end
+        "(#{values.join(', ')})"
+      end
+      sql_values = sql_values.join(', ')
+      attributes = attribute_names.join(', ')
+      "insert into #{table_name} (#{attributes}) values #{sql_values} #{on_conflict if on_conflict};"
+    end
+
+    # @return [PgEventstore::Connection]
+    def connection
+      PgEventstore.connection(@config_name)
+    end
+
+    # @return [PgEventstore::Connection]
+    def replica_connection
+      PgEventstore.connection(@replica_config_name)
+    end
+
+    # @return [PgEventstore::TransactionQueries]
+    def replica_transaction_queries
+      TransactionQueries.new(replica_connection)
+    end
+
+    # @return [PgEventstore::ReplicaQueries]
+    def source_replica_queries
+      ReplicaQueries.new(connection, QueryStrategy::Async.new(connection))
+    end
+
+    # @return [PgEventstore::ReplicaQueries]
+    def destination_replica_queries
+      ReplicaQueries.new(replica_connection, QueryStrategy::Foreground.new(replica_connection))
+    end
+  end
+end

data/lib/pg_eventstore/subscriptions/replica_subscription_runner.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module PgEventstore
+  # @!visibility private
+  # We need to differentiate between replica subscription runner and regular subscription runner. They both have the
+  # same functional though
+  class ReplicaSubscriptionRunner < SubscriptionRunner
+  end
+end

data/lib/pg_eventstore/subscriptions/subscription.rb

@@ -102,7 +102,7 @@ module PgEventstore
 
       # @return [PgEventstore::SubscriptionQueries]
       def subscription_queries
-        SubscriptionQueries.new(connection)
+        SubscriptionQueries.new(connection, QueryStrategy::Foreground.new(connection))
       end
     end
 
@@ -185,6 +185,7 @@ module PgEventstore
         time_between_restarts:,
         state: RunnerState::STATES[:initial]
       )
+      subscription_queries.create_or_replace_table_function(id, options, locked_by)
       reload
     end
 

data/lib/pg_eventstore/subscriptions/subscription_feed_strategies/collection.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module PgEventstore
+  module SubscriptionFeedStrategy
+    # @!visibility private
+    class Collection < Array
+      # This number is used to determine how many subscriptions we fetch per SQL query.
+      # @return [Integer]
+      DEFAULT_SUBSCRIPTIONS_NUM_PER_QUERY = 10
+
+      class << self
+        # @param runners [Array<PgEventstore::SubscriptionRunner>]
+        # @param connection [PgEventstore::Connection]
+        # @param query_strategy [PgEventstore::QueryStrategy]
+        # @param subscriptions_per_query [Integer]
+        # @return [PgEventstore::SubscriptionFeedStrategy::Collection]
+        def create(runners, connection, query_strategy, subscriptions_per_query: DEFAULT_SUBSCRIPTIONS_NUM_PER_QUERY)
+          instance = new
+          replica_runners = runners.grep(ReplicaSubscriptionRunner)
+          common_runners = runners - replica_runners
+          common_runners.each_slice(subscriptions_per_query).each do |runners_slice|
+            strategy = IndexReadStrategy.new(connection, query_strategy)
+            strategy.add(*runners_slice)
+            instance.push(strategy)
+          end
+          replica_runners.each_slice(subscriptions_per_query) do |runners_slice|
+            strategy = ReplicationStrategy.new(connection, query_strategy)
+            strategy.add(*runners_slice)
+            instance.push(strategy)
+          end
+          instance
+        end
+      end
+    end
+  end
+end

data/lib/pg_eventstore/subscriptions/subscription_feed_strategies/index_read_strategy.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+module PgEventstore
+  module SubscriptionFeedStrategy
+    # @!visibility private
+    class IndexReadStrategy
+      include SubscriptionFeedStrategy
+
+      # Allow subscriptions to scan through up to this amount of events per a single query. This allows to make query
+      # plan more predictable. Downside: let's say subscription1 targets "Foo" event type, but between
+      # SubscriptionRunnersFeeder#feed runs more than this amount of events other than "Foo" event type are published -
+      # it will require at least one more loop to pick that event.
+      # @return [Integer]
+      INDEX_LOOK_UP_DISTANCE = 100_000
+
+      # @param connection [PgEventstore::Connection]
+      # @param query_strategy [PgEventstore::QueryStrategy]
+      def initialize(connection, query_strategy)
+        @connection = connection
+        @query_strategy = query_strategy
+        @runners = []
+      end
+
+      # @param runners [Array<PgEventstore::SubscriptionRunner>]
+      # @return [Array<PgEventstore::SubscriptionRunner>]
+      def add(*runners)
+        @runners.push(*runners)
+      end
+
+      # @return [Integer]
+      def size
+        @runners.size
+      end
+
+      # @return [Boolean]
+      def any?
+        @runners.any?
+      end
+
+      # @return [void]
+      def feed
+        runners_query_options = @runners.to_h do |runner|
+          next_chunk_query_opts = runner.next_chunk_query_opts
+          next_chunk_query_opts[:to_position] =
+            [next_chunk_query_opts[:from_position] + INDEX_LOOK_UP_DISTANCE, safe_position].min
+          [runner.id, next_chunk_query_opts]
+        end
+        grouped_indexes = events_global_index_queries.fetch_indexes_for_subscriptions(runners_query_options)
+        @runners.each do |runner|
+          if grouped_indexes[runner.id]
+            chunk = Chunks::SubscriptionEventsIndexChunk.new(
+              grouped_indexes[runner.id],
+              @connection,
+              QueryStrategy::Foreground.new(@connection),
+              runners_query_options[runner.id][:resolve_link_tos]
+            )
+            runner.feed(chunk)
+          else
+            runner.feed(Chunks::SubscriptionCheckpointChunk.new(runners_query_options[runner.id][:to_position]))
+          end
+        end
+      end
+
+      private
+
+      # @return [Integer]
+      def safe_position
+        event_subscription_position_queries.max_subscription_position || 0
+      end
+
+      # @return [PgEventstore::EventsGlobalIndexQueries]
+      def events_global_index_queries
+        EventsGlobalIndexQueries.new(@connection, @query_strategy)
+      end
+
+      # @return [PgEventstore::EventSubscriptionPositionQueries]
+      def event_subscription_position_queries
+        EventSubscriptionPositionQueries.new(@connection)
+      end
+    end
+  end
+end