pg_eventstore 1.13.2 → 1.13.4

data/lib/pg_eventstore/subscriptions/subscription_position_evaluation.rb

@@ -0,0 +1,195 @@
+# frozen_string_literal: true
+
+module PgEventstore
+  # When subscription pulls events at the edge of the events list and several events arrives concurrently - there is a
+  # chance some events will never be picked. Example:
+  # - event1 has been assigned global_position#9
+  # - event2 has been assigned global_position#10
+  # - event1 and event2 are currently in concurrent transactions, but those transactions does not block each other
+  # - transaction holding event2 commits first
+  # - a subscription picks event2 and sets next position to 11
+  # - transaction holding event1 commits
+  # Illustration:
+  #   Time → → → → → → → → → → → → → → → → → → → → → → → → → → → → → → → → → → → → → → → → → → → → → → → → → → → → → → →
+  #
+  #   T1:  BEGIN
+  #          INSERT INTO events(...);
+  #        --------------------global_position#9------------------->
+  #       COMMIT
+  #   T2:       BEGIN
+  #               INSERT INTO events(...);
+  #             --------------global_position#10-------->
+  #             COMMIT
+  #   Query events:                                       SELECT * FROM event WHERE global_position >= 8
+  #                                                       --------------Picks #8, #10, but never #9-------->
+  #
+  # To solve this problem we can:
+  # 1. pause events fetching for the subscription
+  # 2. fetch latest global position that matches subscription's filter
+  # 3. wait for all currently running transactions that affects on the subscription to finish
+  # 4. unpause events fetching and use this position as a right hand limiter, because we can now confidently say there
+  #   are no uncommited events that would be lost otherwise
+  # This class is responsible for the step 2 and step 3, and persists the last safe position to be used in step 4.
+  # @!visibility private
+  class SubscriptionPositionEvaluation
+    # Determines how often to check the list of currently running transactions
+    # @return [Float]
+    TRANSACTIONS_STATUS_REFRESH_INTERVAL = 0.05 # seconds
+    private_constant :TRANSACTIONS_STATUS_REFRESH_INTERVAL
+
+    # @param config_name [Symbol]
+    # @param filter_options [Hash]
+    def initialize(config_name:, filter_options:)
+      @config_name = config_name
+      @stream_filters = QueryBuilders::PartitionsFiltering.extract_streams_filter(filter: filter_options)
+      @event_type_filters = QueryBuilders::PartitionsFiltering.extract_event_types_filter(filter: filter_options)
+      @position_to_evaluate = nil
+      @position_is_safe = nil
+      @last_safe_position = nil
+      @runner = nil
+      @relation_ids_cache = {}
+      @mutex = Mutex.new
+    end
+
+    # @param position_to_evaluate [Integer]
+    # @return [self]
+    def evaluate(position_to_evaluate)
+      unless self.position_to_evaluate == position_to_evaluate
+        stop_evaluation
+        self.position_to_evaluate = position_to_evaluate
+        calculate_safe_position
+      end
+      self
+    end
+
+    # @return [Boolean]
+    def safe?
+      @mutex.synchronize { @position_is_safe } || false
+    end
+
+    # @return [Integer, nil]
+    def last_safe_position
+      @mutex.synchronize { @last_safe_position }
+    end
+
+    # @return [Thread, nil] a runner who is being stopped if any
+    def stop_evaluation
+      _stop_evaluation(@runner)
+    end
+
+    private
+
+    # @param runner [Thread, nil]
+    # @return [Thread, nil]
+    def _stop_evaluation(runner)
+      runner&.exit
+      self.position_is_safe = nil
+      self.last_safe_position = nil
+      self.position_to_evaluate = nil
+    end
+
+    # @return [Integer, nil]
+    def position_to_evaluate
+      @mutex.synchronize { @position_to_evaluate }
+    end
+
+    # @return [Boolean, nil]
+    def position_is_safe
+      @mutex.synchronize { @position_is_safe }
+    end
+
+    # @param val [Integer, nil]
+    # @return [Integer, nil]
+    def last_safe_position=(val)
+      @mutex.synchronize { @last_safe_position = val }
+    end
+
+    # @param val [Integer, nil]
+    # @return [Integer, nil]
+    def position_to_evaluate=(val)
+      @mutex.synchronize { @position_to_evaluate = val }
+    end
+
+    # @param val [Boolean, nil]
+    # @return [Boolean, nil]
+    def position_is_safe=(val)
+      @mutex.synchronize { @position_is_safe = val }
+    end
+
+    # @return [Array<String>]
+    def affected_tables
+      if @stream_filters.empty? && @event_type_filters.empty?
+        [Event::PRIMARY_TABLE_NAME]
+      else
+        partition_queries.partitions(@stream_filters, @event_type_filters, scope: :auto).map(&:table_name)
+      end
+    end
+
+    # @return [void]
+    def calculate_safe_position
+      @runner = Thread.new do
+        tables_to_track = affected_tables
+        update_relation_ids_cache(tables_to_track)
+        safe_position = service_queries.max_global_position(tables_to_track)
+        trx_ids = transaction_queries.transaction do
+          service_queries.relation_transaction_ids(@relation_ids_cache.values)
+        end
+
+        loop do
+          transactions_in_progress = service_queries.transactions_in_progress?(
+            relation_ids: @relation_ids_cache.values, transaction_ids: trx_ids
+          )
+          break unless transactions_in_progress
+
+          sleep TRANSACTIONS_STATUS_REFRESH_INTERVAL
+        end
+        @mutex.synchronize do
+          if safe_position >= @position_to_evaluate
+            # We progressed forward. New position can be persisted
+            @last_safe_position = safe_position
+            @position_is_safe = true
+          else
+            # safe_position is less than the current position to fetch the events from. This means that no new events
+            # are present at this point. We will need to re-evaluate the safe position during next attempts. Until that
+            # we can't progress.
+            @position_to_evaluate = nil
+          end
+        end
+      rescue
+        # Clean up the state immediately in case of error
+        _stop_evaluation(nil)
+      end
+    end
+
+    # @param tables_to_track [Array<String>]
+    # @return [void]
+    def update_relation_ids_cache(tables_to_track)
+      missing_relation_ids = tables_to_track - @relation_ids_cache.keys
+      deleted_relations = @relation_ids_cache.keys - tables_to_track
+      @relation_ids_cache = @relation_ids_cache.except(*deleted_relations)
+      return if missing_relation_ids.empty?
+
+      @relation_ids_cache.merge!(service_queries.relation_ids_by_names(missing_relation_ids))
+    end
+
+    # @return [PgEventstore::PartitionQueries]
+    def partition_queries
+      PartitionQueries.new(connection)
+    end
+
+    # @return [PgEventstore::TransactionQueries]
+    def transaction_queries
+      TransactionQueries.new(connection)
+    end
+
+    # @return [PgEventstore::ServiceQueries]
+    def service_queries
+      ServiceQueries.new(connection)
+    end
+
+    # @return [PgEventstore::Connection]
+    def connection
+      PgEventstore.connection(@config_name)
+    end
+  end
+end

data/lib/pg_eventstore/subscriptions/subscription_runner.rb

@@ -28,17 +28,23 @@ module PgEventstore
     # @param stats [PgEventstore::SubscriptionHandlerPerformance]
     # @param events_processor [PgEventstore::EventsProcessor]
     # @param subscription [PgEventstore::Subscription]
-    def initialize(stats:, events_processor:, subscription:)
+    # @param position_evaluation [PgEventstore::SubscriptionPositionEvaluation]
+    def initialize(stats:, events_processor:, subscription:, position_evaluation:)
       @stats = stats
       @events_processor = events_processor
       @subscription = subscription
+      @position_evaluation = position_evaluation
 
       attach_callbacks
     end
 
     # @return [Hash]
     def next_chunk_query_opts
-      @subscription.options.merge(from_position: next_chunk_global_position, max_count: estimate_events_number)
+      @subscription.options.merge(
+        from_position: next_chunk_global_position,
+        max_count: estimate_events_number,
+        to_position: @position_evaluation.last_safe_position
+      )
     end
 
     # @return [Boolean]
@@ -46,6 +52,11 @@ module PgEventstore
       @subscription.last_chunk_fed_at + @subscription.chunk_query_interval <= Time.now.utc
     end
 
+    # @return [Boolean]
+    def next_chunk_safe?
+      estimate_events_number == 0 ? false : @position_evaluation.evaluate(next_chunk_global_position).safe?
+    end
+
     private
 
     # @return [Integer]
@@ -97,6 +108,11 @@ module PgEventstore
         :change_state, :after,
         SubscriptionRunnerHandlers.setup_handler(:update_subscription_state, @subscription)
       )
+      # Prevent dangling position evaluation runner when subscription changes the state to something except 'running'
+      @events_processor.define_callback(
+        :change_state, :after,
+        SubscriptionRunnerHandlers.setup_handler(:stop_position_evaluation, @position_evaluation)
+      )
     end
   end
 end

data/lib/pg_eventstore/subscriptions/subscription_runners_feeder.rb

@@ -12,7 +12,7 @@ module PgEventstore
     # @param runners [Array<PgEventstore::SubscriptionRunner>]
     # @return [void]
     def feed(runners)
-      runners = runners.select(&:running?).select(&:time_to_feed?)
+      runners = runners.select(&:running?).select(&:time_to_feed?).select(&:next_chunk_safe?)
       return if runners.empty?
 
       runners_query_options = runners.to_h { |runner| [runner.id, runner.next_chunk_query_opts] }

data/lib/pg_eventstore/subscriptions/subscriptions_manager.rb

@@ -6,9 +6,11 @@ require_relative 'basic_runner'
 require_relative 'runner_recovery_strategy'
 require_relative 'runner_recovery_strategies'
 require_relative 'subscription'
+require_relative 'synchronized_array'
 require_relative 'events_processor'
 require_relative 'subscription_handler_performance'
 require_relative 'subscription_runner'
+require_relative 'subscription_position_evaluation'
 require_relative 'subscriptions_set'
 require_relative 'subscription_runners_feeder'
 require_relative 'subscriptions_set_lifecycle'
@@ -26,6 +28,7 @@ require_relative 'queries/subscription_command_queries'
 require_relative 'queries/subscription_queries'
 require_relative 'queries/subscriptions_set_command_queries'
 require_relative 'queries/subscriptions_set_queries'
+require_relative 'queries/service_queries'
 require_relative 'commands_handler'
 
 module PgEventstore
@@ -108,7 +111,11 @@ module PgEventstore
           graceful_shutdown_timeout: graceful_shutdown_timeout,
           recovery_strategies: recovery_strategies(subscription, restart_terminator, failed_subscription_notifier)
         ),
-        subscription: subscription
+        subscription: subscription,
+        position_evaluation: SubscriptionPositionEvaluation.new(
+          config_name: config.name,
+          filter_options: options[:filter] || {}
+        )
       )
 
       @subscriptions_lifecycle.runners.push(runner)
@@ -185,5 +192,10 @@ module PgEventstore
         ),
       ]
     end
+
+    # @return [PgEventstore::Connection]
+    def connection
+      PgEventstore.connection(config_name)
+    end
   end
 end

data/lib/pg_eventstore/subscriptions/synchronized_array.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require 'monitor'
+
+module PgEventstore
+  # @!visibility private
+  class SynchronizedArray < Array
+    include MonitorMixin
+
+    alias old_shift shift
+    alias old_unshift unshift
+    alias old_push push
+    alias old_clear clear
+    alias old_size size
+    alias old_empty? empty?
+
+    def shift(...)
+      synchronize { old_shift(...) }
+    end
+
+    def unshift(...)
+      synchronize { old_unshift(...) }
+    end
+
+    def push(...)
+      synchronize { old_push(...) }
+    end
+
+    def clear(...)
+      synchronize { old_clear(...) }
+    end
+
+    def size(...)
+      synchronize { old_size(...) }
+    end
+
+    def empty?(...)
+      synchronize { old_empty?(...) }
+    end
+  end
+end

data/lib/pg_eventstore/version.rb

@@ -2,5 +2,5 @@
 
 module PgEventstore
   # @return [String]
-  VERSION = '1.13.2'
+  VERSION = '1.13.4'
 end

data/lib/pg_eventstore/web/paginator/events_collection.rb

@@ -58,7 +58,7 @@ module PgEventstore
             options.merge(from_position: from_position, max_count: per_page, direction: order == :asc ? :desc : :asc)
           ).to_sql_builder.unselect.select('global_position').offset(1)
           sql, params = sql_builder.to_exec_params
-          sql = "SELECT * FROM (#{sql}) events ORDER BY global_position #{order} LIMIT 1"
+          sql = "SELECT * FROM (#{sql}) #{Event::PRIMARY_TABLE_NAME} ORDER BY global_position #{order} LIMIT 1"
           connection.with do |conn|
             conn.exec_params(sql, params)
           end.to_a.dig(0, 'global_position')

data/lib/pg_eventstore/web/paginator/stream_ids_collection.rb

@@ -11,7 +11,7 @@ module PgEventstore
         def collection
           @collection ||=
             begin
-              sql_builder = SQLBuilder.new.select('stream_id').from('events')
+              sql_builder = SQLBuilder.new.select('stream_id').from(Event::PRIMARY_TABLE_NAME)
               sql_builder.where('context = ? and stream_name = ?', options[:context], options[:stream_name])
               sql_builder.where('stream_id like ?', "#{options[:query]}%")
               sql_builder.where("stream_id #{direction_operator} ?", starting_id) if starting_id
@@ -27,7 +27,7 @@ module PgEventstore
           return unless collection.size == per_page
 
           starting_id = collection.first['stream_id']
-          sql_builder = SQLBuilder.new.select('stream_id').from('events')
+          sql_builder = SQLBuilder.new.select('stream_id').from(Event::PRIMARY_TABLE_NAME)
           sql_builder.where("stream_id #{direction_operator} ?", starting_id)
           sql_builder.where('stream_id like ?', "#{options[:query]}%")
           sql_builder.where('context = ? and stream_name = ?', options[:context], options[:stream_name])

data/rbs_collection.lock.yaml

@@ -6,9 +6,17 @@ gems:
   source:
     type: git
     name: ruby/gem_rbs_collection
-    revision: ac8b33799c494ea5b22e7dc850d53f99dc4849e9
+    revision: 0cb6351d218704700cf4df797ff8966b3a418fcd
     remote: https://github.com/ruby/gem_rbs_collection.git
     repo_dir: gems
+- name: forwardable
+  version: '0'
+  source:
+    type: stdlib
+- name: monitor
+  version: '0'
+  source:
+    type: stdlib
 - name: timeout
   version: '0'
   source:

data/rbs_collection.yaml

@@ -16,6 +16,8 @@ path: .gem_rbs_collection
 gems:
   - name: sinatra
     ignore: true
+  - name: monitor
+  - name: forwardable
 
 # gems:
 #   # If you want to avoid installing rbs files for gems, you can specify them here.

data/sig/manifest.yaml

@@ -0,0 +1,3 @@
+dependencies:
+  - name: monitor
+  - name: forwardable

data/sig/pg_eventstore/event.rbs

@@ -1,7 +1,9 @@
 module PgEventstore
   class Event
-    include PgEventstore::Extensions::OptionsExtension
+    include Extensions::OptionsExtension
+
     LINK_TYPE: String
+    PRIMARY_TABLE_NAME: String
 
     def ==: ((Object | PgEventstore::Event) other) -> bool
 

data/sig/pg_eventstore/queries/partition_queries.rbs

@@ -39,7 +39,8 @@ module PgEventstore
     # _@return_ — partition attributes
     def context_partition: (PgEventstore::Stream stream) -> ::Hash[untyped, untyped]?
 
-    def partitions: (Array[Hash[Symbol, String | nil]] stream_filters, Array[String] event_filters)-> Array[Partition]
+    def partitions: (Array[Hash[Symbol, String | nil]] stream_filters, Array[String] event_filters,
+        ?scope: Symbol) -> Array[Partition]
 
     # _@param_ `stream`
     #
@@ -76,5 +77,8 @@ module PgEventstore
     private
 
     def deserialize: (Hash[untyped, untyped] attrs)-> Partition
+
+    def set_partitions_scope: (QueryBuilders::PartitionsFiltering partitions_filter, Array[untyped] stream_filters,
+        Array[untyped] event_filters, Symbol scope) -> SQLBuilder
   end
 end

data/sig/pg_eventstore/query_builders/partitions_filtering.rbs

@@ -7,15 +7,19 @@ module PgEventstore
 
       def self.extract_streams_filter: (Hash[untyped, untyped] options) -> Array[Hash[untyped, untyped]]
 
-      def add_event_types: (::Array[String] event_types) -> void
+      def self.correct_stream_filter?: (::Hash[untyped, untyped] stream_attrs) -> bool
 
-      def add_stream_attrs: (?context: String?, ?stream_name: String?) -> void
+      def add_event_types: (::Array[String] event_types) -> SQLBuilder
 
-      def with_event_types: -> void
+      def add_stream_attrs: (?context: String?, ?stream_name: String?) -> SQLBuilder
 
-      private
+      def with_event_types: -> SQLBuilder
 
-      def correct_stream_filter?: (::Hash[untyped, untyped] stream_attrs) -> bool
+      def with_stream_names: -> SQLBuilder
+
+      def without_event_types: -> SQLBuilder
+
+      def without_stream_names: -> SQLBuilder
     end
   end
 end

data/sig/pg_eventstore/sql_builder.rbs

@@ -2,6 +2,8 @@ module PgEventstore
   class SQLBuilder
     def initialize: () -> void
 
+    def from_sql: -> String
+
     # _@param_ `sql`
     def select: (String sql) -> self
 
@@ -18,7 +20,7 @@ module PgEventstore
     def where_or: (String sql, *Object arguments) -> self
 
     # _@param_ `table_name`
-    def from: (String table_name) -> self
+    def from: (String | SQLBuilder table_name) -> self
 
     # _@param_ `sql`
     #
@@ -53,7 +55,7 @@ module PgEventstore
     # _@param_ `val`
     def positional_values_size=: (Integer val) -> Integer
 
-    def _to_exec_params: () -> ::Array[(String | ::Array[untyped])]
+    def _to_exec_params: () -> [String, ::Array[untyped]]
 
     def single_query_sql: () -> String
 
@@ -70,5 +72,9 @@ module PgEventstore
 
     # _@param_ `sql`
     def extract_positional_args: (String sql, *untyped arguments) -> String
+
+    private
+
+    def merge: (SQLBuilder builder)-> String
   end
 end

data/sig/pg_eventstore/subscriptions/callback_handlers/events_processor_handlers.rbs

@@ -1,6 +1,7 @@
 module PgEventstore
   class EventsProcessorHandlers
-    def self.process_event: (PgEventstore::Callbacks callbacks, _RawEventHandler handler, Array[Hash[untyped, untyped]] raw_events) -> void
+    def self.process_event: (PgEventstore::Callbacks callbacks, _RawEventHandler handler, SynchronizedArray raw_events,
+        MonitorMixin::ConditionVariable raw_events_cond) -> void
 
     def self.after_runner_died: (PgEventstore::Callbacks callbacks, StandardError error) -> void
 

data/sig/pg_eventstore/subscriptions/callback_handlers/subscription_runner_handlers.rbs

@@ -1,5 +1,8 @@
 module PgEventstore
   class SubscriptionRunnerHandlers
+    def self.stop_position_evaluation: (SubscriptionPositionEvaluation subscription_position_evaluation,
+        String state) -> void
+
     def self.track_exec_time: (PgEventstore::SubscriptionHandlerPerformance stats, ^() -> void, Integer _current_position) -> void
 
     def self.update_subscription_stats: (PgEventstore::Subscription subscription, PgEventstore::SubscriptionHandlerPerformance stats, Integer current_position) -> void

data/sig/pg_eventstore/subscriptions/queries/service_queries.rbs

@@ -0,0 +1,15 @@
+module PgEventstore
+  class ServiceQueries
+    @connection: Connection
+
+    def initialize: (Connection connection) -> void
+
+    def max_global_position: (Array[String] table_names) -> Integer
+
+    def relation_ids_by_names: (Array[String] table_names) -> Hash[String, Integer]
+
+    def relation_transaction_ids: (Array[Integer] relation_oids) -> Array[String]
+
+    def transactions_in_progress?: (relation_ids: Array[Integer], transaction_ids: Array[String]) -> bool
+  end
+end

data/sig/pg_eventstore/subscriptions/subscription.rbs

@@ -2,6 +2,8 @@ module PgEventstore
   class Subscription
     DEFAULT_TIMESTAMP: Time
 
+    MIN_EVENTS_PULL_INTERVAL: Float
+
     include PgEventstore::Extensions::UsingConnectionExtension
     include PgEventstore::Extensions::OptionsExtension
 

data/sig/pg_eventstore/subscriptions/subscription_feeder.rbs

@@ -1,5 +1,7 @@
 module PgEventstore
   class SubscriptionFeeder
+    EVENTS_PULL_INTERVAL: Float
+
     extend Forwardable
 
     @basic_runner: PgEventstore::BasicRunner

data/sig/pg_eventstore/subscriptions/subscription_position_evaluation.rbs

@@ -0,0 +1,53 @@
+module PgEventstore
+  class SubscriptionPositionEvaluation
+    TRANSACTIONS_STATUS_REFRESH_INTERVAL: Float
+
+    @config_name: Symbol
+    @event_type_filters: Array[String]
+    @last_safe_position: Integer?
+    @mutex: Mutex
+    @position_is_safe: bool?
+    @position_to_evaluate: Integer?
+    @relation_ids_cache: Hash[String, Integer]
+    @runner: Thread?
+    @stream_filters: Array[Hash[untyped, untyped]]
+
+    def initialize: (config_name: Symbol, filter_options: Hash[untyped, untyped]) -> void
+
+    def evaluate: (Integer position_to_evaluate) -> self
+
+    def last_safe_position: -> Integer?
+
+    def safe?: -> bool
+
+    def stop_evaluation: -> void
+
+    private
+
+    def _stop_evaluation: (Thread? runner)-> void
+
+    def affected_tables: -> Array[String]
+
+    def calculate_safe_position: -> void
+
+    def connection: -> Connection
+
+    def last_safe_position=: (Integer? val) -> Integer?
+
+    def partition_queries: -> PartitionQueries
+
+    def position_is_safe: -> bool?
+
+    def position_is_safe=: (bool? val) -> bool?
+
+    def position_to_evaluate: -> Integer?
+
+    def position_to_evaluate=: (Integer? val) -> Integer?
+
+    def service_queries: -> ServiceQueries
+
+    def transaction_queries: -> TransactionQueries
+
+    def update_relation_ids_cache: (Array[String] tables_to_track) -> void
+  end
+end

data/sig/pg_eventstore/subscriptions/subscription_runner.rbs

@@ -12,12 +12,13 @@ module PgEventstore
         stats: SubscriptionHandlerPerformance,
         events_processor: EventsProcessor,
         subscription: Subscription,
-        ?restart_terminator: _RestartTerminator?,
-        ?failed_subscription_notifier: _FailedSubscriptionNotifier?
+        position_evaluation: SubscriptionPositionEvaluation,
       ) -> void
 
     def next_chunk_query_opts: () -> ::Hash[untyped, untyped]
 
+    def next_chunk_safe?: -> bool
+
     def time_to_feed?: () -> bool
 
     def next_chunk_global_position: () -> Integer

data/sig/pg_eventstore/subscriptions/subscriptions_manager.rbs

@@ -77,6 +77,8 @@ module PgEventstore
 
     private
 
+    def connection: -> Connection
+
     def recovery_strategies: (
         Subscription subscription,
         _RestartTerminator? restart_terminator,

data/sig/pg_eventstore/subscriptions/synchronized_array.rbs

@@ -0,0 +1,4 @@
+module PgEventstore
+  class SynchronizedArray < Array[Hash[untyped, untyped]]
+  end
+end