pg_eventstore 1.13.4 → 3.0.0

data/lib/pg_eventstore/subscriptions/subscription_feed_strategies/collection.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module PgEventstore
+  module SubscriptionFeedStrategy
+    # @!visibility private
+    class Collection < Array
+      # This number is used to determine how many subscriptions we fetch per SQL query.
+      # @return [Integer]
+      DEFAULT_SUBSCRIPTIONS_NUM_PER_QUERY = 10
+
+      class << self
+        # @param runners [Array<PgEventstore::SubscriptionRunner>]
+        # @param connection [PgEventstore::Connection]
+        # @param query_strategy [PgEventstore::QueryStrategy]
+        # @param subscriptions_per_query [Integer]
+        # @return [PgEventstore::SubscriptionFeedStrategy::Collection]
+        def create(runners, connection, query_strategy, subscriptions_per_query: DEFAULT_SUBSCRIPTIONS_NUM_PER_QUERY)
+          instance = new
+          replica_runners = runners.grep(ReplicaSubscriptionRunner)
+          common_runners = runners - replica_runners
+          common_runners.each_slice(subscriptions_per_query).each do |runners_slice|
+            strategy = IndexReadStrategy.new(connection, query_strategy)
+            strategy.add(*runners_slice)
+            instance.push(strategy)
+          end
+          replica_runners.each_slice(subscriptions_per_query) do |runners_slice|
+            strategy = ReplicationStrategy.new(connection, query_strategy)
+            strategy.add(*runners_slice)
+            instance.push(strategy)
+          end
+          instance
+        end
+      end
+    end
+  end
+end

data/lib/pg_eventstore/subscriptions/subscription_feed_strategies/index_read_strategy.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+module PgEventstore
+  module SubscriptionFeedStrategy
+    # @!visibility private
+    class IndexReadStrategy
+      include SubscriptionFeedStrategy
+
+      # Allow subscriptions to scan through up to this amount of events per a single query. This allows to make query
+      # plan more predictable. Downside: let's say subscription1 targets "Foo" event type, but between
+      # SubscriptionRunnersFeeder#feed runs more than this amount of events other than "Foo" event type are published -
+      # it will require at least one more loop to pick that event.
+      # @return [Integer]
+      INDEX_LOOK_UP_DISTANCE = 100_000
+
+      # @param connection [PgEventstore::Connection]
+      # @param query_strategy [PgEventstore::QueryStrategy]
+      def initialize(connection, query_strategy)
+        @connection = connection
+        @query_strategy = query_strategy
+        @runners = []
+      end
+
+      # @param runners [Array<PgEventstore::SubscriptionRunner>]
+      # @return [Array<PgEventstore::SubscriptionRunner>]
+      def add(*runners)
+        @runners.push(*runners)
+      end
+
+      # @return [Integer]
+      def size
+        @runners.size
+      end
+
+      # @return [Boolean]
+      def any?
+        @runners.any?
+      end
+
+      # @return [void]
+      def feed
+        runners_query_options = @runners.to_h do |runner|
+          next_chunk_query_opts = runner.next_chunk_query_opts
+          next_chunk_query_opts[:to_position] =
+            [next_chunk_query_opts[:from_position] + INDEX_LOOK_UP_DISTANCE, safe_position].min
+          [runner.id, next_chunk_query_opts]
+        end
+        grouped_indexes = events_global_index_queries.fetch_indexes_for_subscriptions(runners_query_options)
+        @runners.each do |runner|
+          if grouped_indexes[runner.id]
+            chunk = Chunks::SubscriptionEventsIndexChunk.new(
+              grouped_indexes[runner.id],
+              @connection,
+              QueryStrategy::Foreground.new(@connection),
+              runners_query_options[runner.id][:resolve_link_tos]
+            )
+            runner.feed(chunk)
+          else
+            runner.feed(Chunks::SubscriptionCheckpointChunk.new(runners_query_options[runner.id][:to_position]))
+          end
+        end
+      end
+
+      private
+
+      # @return [Integer]
+      def safe_position
+        event_subscription_position_queries.max_subscription_position || 0
+      end
+
+      # @return [PgEventstore::EventsGlobalIndexQueries]
+      def events_global_index_queries
+        EventsGlobalIndexQueries.new(@connection, @query_strategy)
+      end
+
+      # @return [PgEventstore::EventSubscriptionPositionQueries]
+      def event_subscription_position_queries
+        EventSubscriptionPositionQueries.new(@connection)
+      end
+    end
+  end
+end

data/lib/pg_eventstore/subscriptions/subscription_feed_strategies/replication_strategy.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module PgEventstore
+  module SubscriptionFeedStrategy
+    # @!visibility private
+    class ReplicationStrategy
+      include SubscriptionFeedStrategy
+
+      # Allow subscriptions to scan through up to this amount of events per a single query. This allows to make query
+      # plan more predictable.
+      # @return [Integer]
+      INDEX_LOOK_UP_DISTANCE = 100_000
+
+      # @param connection [PgEventstore::Connection]
+      # @param query_strategy [PgEventstore::QueryStrategy]
+      def initialize(connection, query_strategy)
+        @connection = connection
+        @query_strategy = query_strategy
+        @runners = []
+      end
+
+      # @param runners [Array<PgEventstore::ReplicaSubscriptionRunner>]
+      # @return [Array<PgEventstore::ReplicaSubscriptionRunner>]
+      def add(*runners)
+        @runners.push(*runners)
+      end
+
+      # @return [Integer]
+      def size
+        @runners.size
+      end
+
+      # @return [Boolean]
+      def any?
+        @runners.any?
+      end
+
+      # @return [void]
+      def feed
+        runners_query_options = @runners.to_h do |runner|
+          next_chunk_query_opts = runner.next_chunk_query_opts
+          next_chunk_query_opts[:to_position] =
+            [next_chunk_query_opts[:from_position] + INDEX_LOOK_UP_DISTANCE, safe_position].min
+          [runner.id, next_chunk_query_opts]
+        end
+        grouped_indexes = events_global_index_queries.fetch_indexes_for_subscriptions(runners_query_options)
+        @runners.each do |runner|
+          if grouped_indexes[runner.id]
+            chunk = Chunks::ReplicaEventsIndexChunk.new(grouped_indexes[runner.id])
+            runner.feed(chunk)
+          else
+            runner.feed(Chunks::SubscriptionCheckpointChunk.new(runners_query_options[runner.id][:to_position]))
+          end
+        end
+      end
+
+      private
+
+      # @return [Integer]
+      def safe_position
+        event_subscription_position_queries.max_subscription_position || 0
+      end
+
+      # @return [PgEventstore::EventsGlobalIndexQueries]
+      def events_global_index_queries
+        EventsGlobalIndexQueries.new(@connection, @query_strategy)
+      end
+
+      # @return [PgEventstore::EventSubscriptionPositionQueries]
+      def event_subscription_position_queries
+        EventSubscriptionPositionQueries.new(@connection)
+      end
+    end
+  end
+end

data/lib/pg_eventstore/subscriptions/subscription_feed_strategy.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module PgEventstore
+  # @!visibility private
+  module SubscriptionFeedStrategy
+    # @param runners [PgEventstore::SubscriptionRunner, Array<PgEventstore::SubscriptionRunner>]
+    # @return [Array<PgEventstore::SubscriptionRunner>]
+    def add(*runners)
+      raise NotImplementedError
+    end
+
+    # @return [Integer]
+    def size
+      raise NotImplementedError
+    end
+
+    # @return [Boolean]
+    def any?
+      raise NotImplementedError
+    end
+
+    # @return [void]
+    def feed
+      raise NotImplementedError
+    end
+  end
+end
+
+require_relative 'subscription_feed_strategies/collection'
+require_relative 'subscription_feed_strategies/index_read_strategy'
+require_relative 'subscription_feed_strategies/replication_strategy'

data/lib/pg_eventstore/subscriptions/subscription_feeder.rb

@@ -10,7 +10,7 @@ module PgEventstore
     # Determines how often to fetch events from the event store.
     # @see PgEventstore::Subscription::MIN_EVENTS_PULL_INTERVAL
     # @return [Float]
-    EVENTS_PULL_INTERVAL = 0.1 # seconds
+    EVENTS_PULL_INTERVAL = 0.2 # seconds
     private_constant :EVENTS_PULL_INTERVAL
 
     attr_reader :config_name
@@ -30,6 +30,9 @@ module PgEventstore
       @subscriptions_set_lifecycle = subscriptions_set_lifecycle
       @subscriptions_lifecycle = subscriptions_lifecycle
       @commands_handler = CommandsHandler.new(@config_name, self, @subscriptions_lifecycle.runners)
+      if requires_subscription_position_assignment?
+        @events_subscription_position_worker = EventsSubscriptionPositionWorker.new(@config_name)
+      end
       attach_runner_callbacks
     end
 
@@ -77,6 +80,14 @@ module PgEventstore
         :before_runner_started, :before,
         SubscriptionFeederHandlers.setup_handler(:start_cmds_handler, @commands_handler)
       )
+      if requires_subscription_position_assignment?
+        @basic_runner.define_callback(
+          :before_runner_started, :before,
+          SubscriptionFeederHandlers.setup_handler(
+            :start_events_subscription_position_worker, @events_subscription_position_worker
+          )
+        )
+      end
 
       @basic_runner.define_callback(
         :after_runner_died, :before,
@@ -108,6 +119,14 @@ module PgEventstore
         :after_runner_stopped, :before,
         SubscriptionFeederHandlers.setup_handler(:stop_commands_handler, @commands_handler)
       )
+      if requires_subscription_position_assignment?
+        @basic_runner.define_callback(
+          :after_runner_stopped, :before,
+          SubscriptionFeederHandlers.setup_handler(
+            :stop_events_subscription_position_worker, @events_subscription_position_worker
+          )
+        )
+      end
 
       @basic_runner.define_callback(
         :before_runner_restored, :after,
@@ -122,9 +141,14 @@ module PgEventstore
       [
         RunnerRecoveryStrategies::RestoreConnection.new(config_name),
         RunnerRecoveryStrategies::RestoreSubscriptionFeeder.new(
-          subscriptions_set_lifecycle: subscriptions_set_lifecycle
+          subscriptions_set_lifecycle:
         ),
       ]
     end
+
+    # @return [Boolean]
+    def requires_subscription_position_assignment?
+      Config::NodeRole.writable.include?(PgEventstore.config(@config_name).eventstore_role)
+    end
   end
 end

data/lib/pg_eventstore/subscriptions/subscription_handler_performance.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require 'benchmark'
-
 module PgEventstore
   # This class measures the performance of Subscription's handler and returns the average time required to process an
   # event.
@@ -10,7 +8,7 @@ module PgEventstore
     extend Forwardable
 
     # @return [Integer] the number of measurements to keep
-    TIMINGS_TO_KEEP = 100
+    TIMINGS_TO_KEEP = 10
 
     def_delegators :@lock, :synchronize
 
@@ -20,10 +18,12 @@ module PgEventstore
     end
 
     # Yields the given block to measure its execution time
+    # @param events_number [Integer] number of events processed by the given block
     # @return [Object] the result of yielded block
-    def track_exec_time
+    def track_exec_time(events_number)
       result = nil
-      time = Benchmark.realtime { result = yield }
+      time = Utils.benchmark { result = yield }
+      time /= events_number
       synchronize do
         @timings.shift if @timings.size == TIMINGS_TO_KEEP
         @timings.push(time)

data/lib/pg_eventstore/subscriptions/subscription_runner.rb

@@ -11,50 +11,48 @@ module PgEventstore
     extend Forwardable
 
     # @return [Integer]
-    MAX_EVENTS_PER_CHUNK = 1_000
+    DEFAULT_MAX_EVENTS_PER_CHUNK = 1_000
     # @return [Integer]
     MIN_EVENTS_PER_CHUNK = 10
     # @return [Integer]
-    INITIAL_EVENTS_PER_CHUNK = 10
+    DEFAULT_INITIAL_EVENTS_PER_CHUNK = 10
 
     # @!attribute subscription
     #   @return [PgEventstore::Subscription]
     attr_reader :subscription
 
     def_delegators :@events_processor, :start, :stop, :stop_async, :feed, :wait_for_finish, :restore, :state, :running?,
-                   :clear_chunk, :within_state
+                   :clear_events_repository, :within_state
     def_delegators :@subscription, :lock!, :id
 
     # @param stats [PgEventstore::SubscriptionHandlerPerformance]
     # @param events_processor [PgEventstore::EventsProcessor]
     # @param subscription [PgEventstore::Subscription]
-    # @param position_evaluation [PgEventstore::SubscriptionPositionEvaluation]
-    def initialize(stats:, events_processor:, subscription:, position_evaluation:)
+    # @param max_events_per_chunk [Integer]
+    def initialize(stats:, events_processor:, subscription:,
+                   initial_events_per_chunk: DEFAULT_INITIAL_EVENTS_PER_CHUNK,
+                   max_events_per_chunk: DEFAULT_MAX_EVENTS_PER_CHUNK)
       @stats = stats
       @events_processor = events_processor
       @subscription = subscription
-      @position_evaluation = position_evaluation
+      @max_events_per_chunk = max_events_per_chunk
+      @initial_events_per_chunk = initial_events_per_chunk
 
       attach_callbacks
     end
 
     # @return [Hash]
     def next_chunk_query_opts
-      @subscription.options.merge(
+      {
         from_position: next_chunk_global_position,
         max_count: estimate_events_number,
-        to_position: @position_evaluation.last_safe_position
-      )
+        resolve_link_tos: @subscription.options[:resolve_link_tos] || false,
+      }
     end
 
     # @return [Boolean]
     def time_to_feed?
-      @subscription.last_chunk_fed_at + @subscription.chunk_query_interval <= Time.now.utc
-    end
-
-    # @return [Boolean]
-    def next_chunk_safe?
-      estimate_events_number == 0 ? false : @position_evaluation.evaluate(next_chunk_global_position).safe?
+      estimate_events_number > 0 && @subscription.last_chunk_fed_at + @subscription.chunk_query_interval <= Time.now.utc
     end
 
     private
@@ -69,10 +67,10 @@ module PgEventstore
 
     # @return [Integer]
     def estimate_events_number
-      return INITIAL_EVENTS_PER_CHUNK if @stats.average_event_processing_time == 0
+      return @initial_events_per_chunk if @stats.average_event_processing_time == 0
 
       events_per_chunk = (@subscription.chunk_query_interval / @stats.average_event_processing_time).round
-      events_to_fetch = [events_per_chunk, MAX_EVENTS_PER_CHUNK].min - @events_processor.events_left_in_chunk
+      events_to_fetch = [events_per_chunk, @max_events_per_chunk].min - @events_processor.events_left_in_repo
       return 0 if events_to_fetch < 0 # We still have a lot of events in the chunk - no need to fetch more
 
       [events_to_fetch, MIN_EVENTS_PER_CHUNK].max
@@ -99,6 +97,11 @@ module PgEventstore
         SubscriptionRunnerHandlers.setup_handler(:update_subscription_chunk_stats, @subscription)
       )
 
+      @events_processor.define_callback(
+        :checkpoint, :after,
+        SubscriptionRunnerHandlers.setup_handler(:checkpoint, @subscription)
+      )
+
       @events_processor.define_callback(
         :restart, :after,
         SubscriptionRunnerHandlers.setup_handler(:update_subscription_restarts, @subscription)
@@ -108,11 +111,6 @@ module PgEventstore
         :change_state, :after,
         SubscriptionRunnerHandlers.setup_handler(:update_subscription_state, @subscription)
       )
-      # Prevent dangling position evaluation runner when subscription changes the state to something except 'running'
-      @events_processor.define_callback(
-        :change_state, :after,
-        SubscriptionRunnerHandlers.setup_handler(:stop_position_evaluation, @position_evaluation)
-      )
     end
   end
 end

data/lib/pg_eventstore/subscriptions/subscription_runner_commands/reset_position.rb

@@ -16,7 +16,7 @@ module PgEventstore
       # @return [void]
       def exec_cmd(subscription_runner)
         subscription_runner.within_state(:stopped) do
-          subscription_runner.clear_chunk
+          subscription_runner.clear_events_repository
           subscription_runner.subscription.update(
             last_chunk_greatest_position: nil, current_position: data['position'], total_processed_events: 0
           )

data/lib/pg_eventstore/subscriptions/subscription_runners_feeder.rb

@@ -12,15 +12,19 @@ module PgEventstore
     # @param runners [Array<PgEventstore::SubscriptionRunner>]
     # @return [void]
     def feed(runners)
-      runners = runners.select(&:running?).select(&:time_to_feed?).select(&:next_chunk_safe?)
+      runners = runners.select(&:running?).select(&:time_to_feed?)
       return if runners.empty?
 
-      runners_query_options = runners.to_h { |runner| [runner.id, runner.next_chunk_query_opts] }
-      grouped_events = subscription_queries.subscriptions_events(runners_query_options)
-
-      runners.each do |runner|
-        runner.feed(grouped_events[runner.id]) if grouped_events[runner.id]
+      feed_strategies_collection = SubscriptionFeedStrategy::Collection.create(
+        runners,
+        connection,
+        QueryStrategy::Async.new(connection)
+      )
+      query_runner = AsyncRunner.new
+      feed_strategies_collection.each do |strategy|
+        query_runner.async { strategy.feed }
       end
+      query_runner.run
     end
 
     private
@@ -29,10 +33,5 @@ module PgEventstore
     def connection
       PgEventstore.connection(@config_name)
     end
-
-    # @return [PgEventstore::SubscriptionQueries]
-    def subscription_queries
-      SubscriptionQueries.new(connection)
-    end
   end
 end

data/lib/pg_eventstore/subscriptions/subscriptions_lifecycle.rb

@@ -56,18 +56,6 @@ module PgEventstore
       @runners.map(&:subscription)
     end
 
-    # Sets the force_lock flag to true. If set - all related Subscriptions will ignore their lock state and will be
-    # locked by the new SubscriptionsSet.
-    # @return [void]
-    def force_lock!
-      message = <<~TEXT
-        Usage of #force_lock! is deprecated and will be removed in v2. Please pass :force_lock keyword argument when \
-        initializing SubscriptionsManager. Example: PgEventstore.subscriptions_manager(force_lock: true)
-      TEXT
-      Utils.deprecation_warning(message)
-      @force_lock = true
-    end
-
     # @return [Boolean]
     def force_locked?
       @force_lock