pg_eventstore 1.13.2 → 1.13.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5411bc50e9e33cb1e75d41e3e0955ea68b2f4964a431a039a773727e14575840
-  data.tar.gz: cb0d9c313a28ea13be313bd6c7f7760513b85ad3f4a660ab66588ef2fc1a6441
+  metadata.gz: 94443dc3aa36701d5e126573a5babdd4585932d803b766b06b94c861f6b4ef5a
+  data.tar.gz: 86fcca21a3a2d3da35f3c760542b23cc97ae93ce97bae7b9baf011a957a9ffea
 SHA512:
-  metadata.gz: 733b57829ff8883f24dd205b1aeb98ff39c2cf33c522ebcd11e9f1b984ee71844edf8cf907928a2cea729732f569b84b432461e8db264bf6d68e79d1f2b9a27b
-  data.tar.gz: fc6bfb9d2acbd4dc8cc3e9f981d5280aecdc3ecd8527a24bf7704309c9b2a52aff08f416e77da2279682ee474ab2c0ecbe014690fcff6969eda7b6199fd404e3
+  metadata.gz: 4fb5ed37edce4c557082ba60cfa18f7209f795fa29b98dff9f848bb83b0b1cec889f4a7ca7f77bef5c0d9c6b4065d53765dcfcdf50dffe2628c3e3ffb57d26ad
+  data.tar.gz: 0bc33722f13c57c11888ac205873ac93a3610ef5218e8dccf7c3e64d671852757e9d755ffe17ea8176b238dbe1722df47db522b5252ddceed52e14534c164c45

data/CHANGELOG.md

@@ -1,5 +1,13 @@
 ## [Unreleased]
 
+## [1.13.4]
+
+- Fix subscriptions potentially skipping events when multiple events are appended in concurrent transactions
+
+## [1.13.3]
+
+- Reduce subscription delays for newly published events
+
 ## [1.13.2]
 
 - Fix a bug that prevents correct processing of CLI commands using public API
@@ -114,7 +122,7 @@ subscriptions_manager.subscribe(
 ```
 
 ## [1.1.5]
-- Review the way to handle SubscriptionAlreadyLockedError error. This removes noise when attempting to lock an already locked subscription. 
+- Review the way to handle SubscriptionAlreadyLockedError error. This removes noise when attempting to lock an already locked subscription.
 
 ## [1.1.4]
 - Add rbs signatures
@@ -163,7 +171,7 @@ subscriptions_manager.subscribe(
 
 ## [0.10.2] - 2024-03-13
 
-- Review the approach to resolve link events 
+- Review the approach to resolve link events
 - Fix subscriptions restart interval option not being processed correctly
 
 ## [0.10.1] - 2024-03-12
@@ -226,7 +234,7 @@ subscriptions_manager.subscribe(
 
 ## [0.3.0] - 2024-01-24
 
-- Log SQL queries when `PgEvenstore.logger` is set and it is in `:debug` mode 
+- Log SQL queries when `PgEvenstore.logger` is set and it is in `:debug` mode
 
 ## [0.2.6] - 2023-12-20
 

data/lib/pg_eventstore/event.rb

@@ -6,6 +6,8 @@ module PgEventstore
 
     # @return [String] a type of link event
     LINK_TYPE = '$>'
+    # @return [String]
+    PRIMARY_TABLE_NAME = 'events'
 
     # @!attribute id
     #   @return [String] UUIDv4 string

data/lib/pg_eventstore/queries/event_queries.rb

@@ -28,7 +28,8 @@ module PgEventstore
     def event_exists?(event)
       return false if event.id.nil? || event.stream.nil?
 
-      sql_builder = SQLBuilder.new.select('1 as exists').from('events').where('id = ?', event.id).limit(1)
+      sql_builder = SQLBuilder.new.select('1 as exists').from(Event::PRIMARY_TABLE_NAME).where('id = ?', event.id)
+      sql_builder.limit(1)
       sql_builder.where(
         'context = ? and stream_name = ? and type = ?', event.stream.context, event.stream.stream_name, event.type
       )
@@ -42,7 +43,7 @@ module PgEventstore
     # @param events [Array<PgEventstore::Event>]
     # @return [Array<String>]
     def ids_from_db(events)
-      sql_builder = SQLBuilder.new.from('events').select('id')
+      sql_builder = SQLBuilder.new.from(Event::PRIMARY_TABLE_NAME).select('id')
       partition_attrs = events.map { |event| [event.stream&.context, event.stream&.stream_name, event.type] }.uniq
       partition_attrs.each do |context, stream_name, event_type|
         sql_builder.where_or('context = ? and stream_name = ? and type = ?', context, stream_name, event_type)
@@ -57,7 +58,7 @@ module PgEventstore
     # @param stream [PgEventstore::Stream]
     # @return [Integer, nil]
     def stream_revision(stream)
-      sql_builder = SQLBuilder.new.from('events').select('stream_revision')
+      sql_builder = SQLBuilder.new.from(Event::PRIMARY_TABLE_NAME).select('stream_revision')
       sql_builder.where('context = ? and stream_name = ? and stream_id = ?', *stream.to_a)
       sql_builder.order('stream_revision DESC').limit(1)
       connection.with do |conn|

data/lib/pg_eventstore/queries/partition_queries.rb

@@ -178,14 +178,42 @@ module PgEventstore
 
     # @param stream_filters [Array<Hash[Symbol, String]>]
     # @param event_filters [Array<String>]
+    # @param scope [Symbol] what kind of partition we want to receive. Available options are :event_type, :context,
+    #   :stream_name and :auto. In :auto mode the scope will be calculated based on stream_filters and event_filters.
     # @return [Array<PgEventstore::Partition>]
-    def partitions(stream_filters, event_filters)
-      partitions_filter = QueryBuilders::PartitionsFiltering.new
-      stream_filters.each { |attrs| partitions_filter.add_stream_attrs(**attrs) }
-      partitions_filter.add_event_types(event_filters)
-      partitions_filter.with_event_types
+    def partitions(stream_filters, event_filters, scope: :event_type)
+      stream_filters = stream_filters.select { QueryBuilders::PartitionsFiltering.correct_stream_filter?(_1) }
+      sql_builder =
+        if event_filters.any?
+          # When event type filters are present - they apply constraints to any stream filter. Thus, we can't look up
+          # partitions by stream attributes separately.
+          filter = QueryBuilders::PartitionsFiltering.new
+          stream_filters.each { |attrs| filter.add_stream_attrs(**attrs) }
+          filter.add_event_types(event_filters)
+          set_partitions_scope(filter, stream_filters, event_filters, scope)
+        else
+          # When event type filters are absent - we can look up partitions by context and context/stream_name
+          # separately, thus potentially producing one-to-one mapping of filter-to-partition with :auto scope. For
+          # example, let's say we have stream attributes filter like
+          # [{ context: 'FooCtx', stream_name: 'Bar'}, { context: 'BarCtx' }], then we would be able to look up
+          # partitions by the exact match, returning only two of them according to the provided filters - stream
+          # partition for first filter and context partition for second filter.
+          builders = stream_filters.map do |attrs|
+            filter = QueryBuilders::PartitionsFiltering.new
+            filter.add_stream_attrs(**attrs)
+            set_partitions_scope(filter, [attrs], event_filters, scope)
+          end
+
+          sql_builder = SQLBuilder.union_builders(builders) if builders.any?
+          sql_builder ||
+            begin
+              builder = QueryBuilders::PartitionsFiltering.new
+              set_partitions_scope(builder, stream_filters, event_filters, scope)
+            end
+        end
+
       connection.with do |conn|
-        conn.exec_params(*partitions_filter.to_exec_params)
+        conn.exec_params(*sql_builder.to_exec_params)
       end.map(&method(:deserialize))
     end
 
@@ -210,6 +238,43 @@ module PgEventstore
 
     private
 
+    # @param partitions_filter [PgEventstore::QueryBuilders::PartitionsFiltering]
+    # @param stream_filters [Array<Hash[Symbol, String]>]
+    # @param event_filters [Array<String>]
+    # @param scope [Symbol]
+    # @return [PgEventstore::SQLBuilder]
+    def set_partitions_scope(partitions_filter, stream_filters, event_filters, scope)
+      case scope
+      when :event_type
+        partitions_filter.with_event_types
+      when :stream_name
+        filter = QueryBuilders::PartitionsFiltering.new
+        filter.without_event_types
+        filter.with_stream_names
+        builder = filter.to_sql_builder
+        builder.where(
+          '(context, stream_name) in ?',
+          partitions_filter.to_sql_builder.unselect.select('context, stream_name').group('context, stream_name')
+        )
+      when :context
+        filter = QueryBuilders::PartitionsFiltering.new
+        filter.without_event_types
+        filter.without_stream_names
+        builder = filter.to_sql_builder
+        builder.where('context in ?', partitions_filter.to_sql_builder.unselect.select('context').group('context'))
+      when :auto
+        if event_filters.any?
+          set_partitions_scope(partitions_filter, stream_filters, event_filters, :event_type)
+        elsif stream_filters.any? { _1[:stream_name] }
+          set_partitions_scope(partitions_filter, stream_filters, event_filters, :stream_name)
+        else
+          set_partitions_scope(partitions_filter, stream_filters, event_filters, :context)
+        end
+      else
+        raise NotImplementedError, "Don't know how to handle #{scope.inspect} scope!"
+      end
+    end
+
     # @param attrs [Hash]
     # @return [PgEventstore::Partition]
     def deserialize(attrs)

data/lib/pg_eventstore/query_builders/events_filtering.rb

@@ -4,10 +4,6 @@ module PgEventstore
   module QueryBuilders
     # @!visibility private
     class EventsFiltering < BasicFiltering
-      # @return [String]
-      TABLE_NAME = 'events'
-      private_constant :TABLE_NAME
-
       # @return [Integer]
       DEFAULT_LIMIT = 1_000
       # @return [Hash<String => String, Symbol => String>]
@@ -107,7 +103,7 @@ module PgEventstore
 
       # @return [String]
       def to_table_name
-        TABLE_NAME
+        Event::PRIMARY_TABLE_NAME
       end
 
       # @param context [String, nil]

data/lib/pg_eventstore/query_builders/partitions_filtering.rb

@@ -28,6 +28,19 @@ module PgEventstore
           end
           streams || []
         end
+
+        # @param stream_attrs [Hash]
+        # @return [Boolean]
+        def correct_stream_filter?(stream_attrs)
+          result = (stream_attrs in { context: String, stream_name: String } | { context: String })
+          return true if result
+
+          PgEventstore.logger&.debug(<<~TEXT)
+            Ignoring unsupported stream filter format for grouped read #{stream_attrs.compact.inspect}. \
+            See docs/reading_events.md docs for supported formats.
+          TEXT
+          false
+        end
       end
 
       # @return [String]
@@ -37,10 +50,10 @@ module PgEventstore
 
       # @param context [String, nil]
       # @param stream_name [String, nil]
-      # @return [void]
+      # @return [PgEventstore::SQLBuilder]
       def add_stream_attrs(context: nil, stream_name: nil)
         stream_attrs = { context: context, stream_name: stream_name }
-        return unless correct_stream_filter?(stream_attrs)
+        return @sql_builder unless self.class.correct_stream_filter?(stream_attrs)
 
         stream_attrs.compact!
         sql = stream_attrs.map do |attr, _|
@@ -50,31 +63,28 @@ module PgEventstore
       end
 
       # @param event_types [Array<String>]
-      # @return [void]
+      # @return [PgEventstore::SQLBuilder]
       def add_event_types(event_types)
-        return if event_types.empty?
+        return @sql_builder if event_types.empty?
 
         @sql_builder.where("#{to_table_name}.event_type = ANY(?::varchar[])", event_types)
       end
 
-      # @return [void]
+      # @return [PgEventstore::SQLBuilder]
       def with_event_types
         @sql_builder.where('event_type IS NOT NULL')
       end
 
-      private
+      def with_stream_names
+        @sql_builder.where('stream_name IS NOT NULL')
+      end
 
-      # @param stream_attrs [Hash]
-      # @return [Boolean]
-      def correct_stream_filter?(stream_attrs)
-        result = (stream_attrs in { context: String, stream_name: String } | { context: String, stream_name: nil })
-        return true if result
+      def without_event_types
+        @sql_builder.where('event_type IS NULL')
+      end
 
-        PgEventstore.logger&.debug(<<~TEXT)
-          Ignoring unsupported stream filter format for grouped read #{stream_attrs.compact.inspect}. \
-          See docs/reading_events.md docs for supported formats.
-        TEXT
-        false
+      def without_stream_names
+        @sql_builder.where('stream_name IS NULL')
       end
     end
   end

data/lib/pg_eventstore/sql_builder.rb

@@ -64,7 +64,7 @@ module PgEventstore
       self
     end
 
-    # @param table_name [String]
+    # @param table_name [String | SQLBuilder]
     # @return [self]
     def from(table_name)
       @from_value = table_name
@@ -132,7 +132,7 @@ module PgEventstore
       self
     end
 
-    # @return [Array<String, Array<Object>>]
+    # @return [[String, Array<_>]]
     def to_exec_params
       @positional_values.clear
       @positional_values_size = 0
@@ -141,19 +141,27 @@ module PgEventstore
 
     protected
 
-    # @return [Array<String, Array<Object>>]
+    # @return [[String, Array<_>]]
     def _to_exec_params
       return [single_query_sql, @positional_values] if @union_values.empty?
 
       [union_query_sql, @positional_values]
     end
 
+    # @return [String]
+    def from_sql
+      return @from_value if @from_value.is_a?(String)
+
+      sql = merge(@from_value)
+      "(#{sql}) #{@from_value.from_sql}"
+    end
+
     private
 
     # @return [String]
     def single_query_sql
       where_sql = [where_sql('OR'), where_sql('AND')].reject(&:empty?).map { |sql| "(#{sql})" }.join(' AND ')
-      sql = "SELECT #{select_sql} FROM #{@from_value}"
+      sql = "SELECT #{select_sql} FROM #{from_sql}"
       sql += " #{join_sql}" unless @join_values.empty?
       sql += " WHERE #{where_sql}" unless where_sql.empty?
       sql += " GROUP BY #{@group_values.join(', ')}" unless @group_values.empty?
@@ -168,11 +176,7 @@ module PgEventstore
       sql = single_query_sql
       union_parts = ["(#{sql})"]
       union_parts += @union_values.map do |builder|
-        builder.positional_values_size = @positional_values_size
-        builder_sql, values = builder._to_exec_params
-        @positional_values.push(*values)
-        @positional_values_size += values.size
-        "(#{builder_sql})"
+        "(#{merge(builder)})"
       end
       union_parts.join(' UNION ALL ')
     end
@@ -200,14 +204,28 @@ module PgEventstore
       @order_values.join(', ')
     end
 
+    # @param builder [PgEventstore::SQLBuilder]
+    # @return [String]
+    def merge(builder)
+      builder.positional_values_size = @positional_values_size
+      sql_query, positional_values = builder._to_exec_params
+      @positional_values.push(*positional_values)
+      @positional_values_size += positional_values.size
+      sql_query
+    end
+
     # Replaces "?" signs in the given string with positional variables and memorize positional values they refer to.
     # @param sql [String]
     # @return [String]
     def extract_positional_args(sql, *arguments)
       sql.gsub('?').each_with_index do |_, index|
-        @positional_values.push(arguments[index])
-        @positional_values_size += 1
-        "$#{@positional_values_size}"
+        if arguments[index].is_a?(SQLBuilder)
+          "(#{merge(arguments[index])})"
+        else
+          @positional_values.push(arguments[index])
+          @positional_values_size += 1
+          "$#{@positional_values_size}"
+        end
       end
     end
   end

data/lib/pg_eventstore/subscriptions/callback_handlers/events_processor_handlers.rb

@@ -9,10 +9,15 @@ module PgEventstore
       # @param callbacks [PgEventstore::Callbacks]
       # @param handler [#call]
       # @param raw_events [Array<Hash>]
+      # @param raw_events_cond [MonitorMixin::ConditionVariable]
       # @return [void]
-      def process_event(callbacks, handler, raw_events)
-        raw_event = raw_events.shift
-        return sleep 0.5 if raw_event.nil?
+      def process_event(callbacks, handler, raw_events, raw_events_cond)
+        raw_event = nil
+        raw_events.synchronize do
+          raw_events_cond.wait(0.5) if raw_events.empty?
+          raw_event = raw_events.shift
+        end
+        return if raw_event.nil?
 
         callbacks.run_callbacks(:process, Utils.original_global_position(raw_event)) do
           handler.call(raw_event)

data/lib/pg_eventstore/subscriptions/callback_handlers/subscription_runner_handlers.rb

@@ -52,6 +52,15 @@ module PgEventstore
       def update_subscription_state(subscription, state)
         subscription.update(state: state)
       end
+
+      # @param subscription_position_evaluation [PgEventstore::SubscriptionPositionEvaluation]
+      # @param state [String]
+      # @return [void]
+      def stop_position_evaluation(subscription_position_evaluation, state)
+        return if state == 'running'
+
+        subscription_position_evaluation.stop_evaluation
+      end
     end
   end
 end

data/lib/pg_eventstore/subscriptions/events_processor.rb

@@ -16,7 +16,8 @@ module PgEventstore
     # @param recovery_strategies [Array<PgEventstore::RunnerRecoveryStrategy>]
     def initialize(handler, graceful_shutdown_timeout:, recovery_strategies: [])
       @handler = handler
-      @raw_events = []
+      @raw_events = SynchronizedArray.new
+      @raw_events_cond = @raw_events.new_cond
       @basic_runner = BasicRunner.new(
         run_interval: 0,
         async_shutdown_time: graceful_shutdown_timeout,
@@ -32,7 +33,10 @@ module PgEventstore
 
       within_state(:running) do
         callbacks.run_callbacks(:feed, Utils.original_global_position(raw_events.last))
-        @raw_events.push(*raw_events)
+        @raw_events.synchronize do
+          @raw_events.push(*raw_events)
+          @raw_events_cond.broadcast
+        end
       end
     end
 
@@ -52,7 +56,7 @@ module PgEventstore
     def attach_runner_callbacks
       @basic_runner.define_callback(
         :process_async, :before,
-        EventsProcessorHandlers.setup_handler(:process_event, @callbacks, @handler, @raw_events)
+        EventsProcessorHandlers.setup_handler(:process_event, @callbacks, @handler, @raw_events, @raw_events_cond)
       )
 
       @basic_runner.define_callback(

data/lib/pg_eventstore/subscriptions/queries/service_queries.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module PgEventstore
+  # @!visibility private
+  class ServiceQueries
+    # @param connection [PgEventstore::Connection]
+    def initialize(connection)
+      @connection = connection
+    end
+
+    # @param relation_oids [Array<Integer>]
+    # @return [Array<String>]
+    def relation_transaction_ids(relation_oids)
+      result = @connection.with do |conn|
+        # Look up transactions that change table's content
+        conn.exec_params(
+          <<~SQL,
+            SELECT virtualtransaction AS trx_id FROM pg_locks
+              WHERE relation = ANY($1::oid[]) AND mode = 'RowExclusiveLock'
+          SQL
+          [relation_oids]
+        )
+      end
+      result.map { _1['trx_id'] }
+    end
+
+    # @param relation_ids [Array<Integer>]
+    # @param transaction_ids [Array<String>]
+    # @return [Boolean]
+    def transactions_in_progress?(relation_ids:, transaction_ids:)
+      result = @connection.with do |conn|
+        conn.exec_params(
+          <<~SQL,
+            SELECT 1 as one FROM pg_locks
+              WHERE virtualtransaction = ANY($1) AND relation = ANY($2::oid[]) AND mode = 'RowExclusiveLock'
+              LIMIT 1
+          SQL
+          [transaction_ids, relation_ids]
+        )
+      end
+      result.any?
+    end
+
+    # @param table_names [Array<String>] existing table names
+    # @return [Integer]
+    def max_global_position(table_names)
+      return 0 if table_names.empty?
+
+      partition_builds = table_names.map do |table_name|
+        SQLBuilder.new.select("MAX(#{table_name}.global_position) AS global_position").from(table_name)
+      end
+      sql, positional_values = SQLBuilder.union_builders(partition_builds).to_exec_params
+      result = @connection.with do |conn|
+        conn.exec_params("SELECT MAX(global_position) AS max_pos FROM (#{sql}) pos", positional_values)
+      end
+      result.first['max_pos'] || 0
+    end
+
+    # @param table_names [Array<String>]
+    # @return [Hash<String, Integer>]
+    def relation_ids_by_names(table_names)
+      result = @connection.with do |conn|
+        conn.exec_params(
+          <<~SQL,
+            SELECT relname, oid FROM pg_class WHERE relname = ANY($1::varchar[])
+          SQL
+          [table_names]
+        )
+      end
+      result.each_with_object({}) { |attrs, res| res[attrs['relname']] = attrs['oid'] }
+    end
+  end
+end

data/lib/pg_eventstore/subscriptions/queries/subscription_queries.rb

@@ -177,6 +177,7 @@ module PgEventstore
     # @return [PgEventstore::SQLBuilder]
     def query_builder(id, options)
       builder = PgEventstore::QueryBuilders::EventsFiltering.subscriptions_events_filtering(options).to_sql_builder
+      builder.where('global_position <= ?', options[:to_position]) if options[:to_position]
       builder.select("#{id} as runner_id")
     end
 

data/lib/pg_eventstore/subscriptions/subscription.rb

@@ -6,6 +6,16 @@ module PgEventstore
     include Extensions::UsingConnectionExtension
     include Extensions::OptionsExtension
 
+    # Determines the minimal allowed value of events pull frequency of the particular subscription. You can find similar
+    # constant - SubscriptionFeeder::EVENTS_PULL_INTERVAL. Unlike it - this one is responsible to detect whether the
+    # subscription should be included in the subscriptions list to query next chunk of events. Thus, this setting only
+    # determines whether it is time to make a request, but how frequent would be the actual request - determines
+    # SubscriptionFeeder::EVENTS_PULL_INTERVAL.
+    # @see PgEventstore::SubscriptionFeeder::EVENTS_PULL_INTERVAL
+    # @return [Float]
+    MIN_EVENTS_PULL_INTERVAL = 0.2
+    private_constant :MIN_EVENTS_PULL_INTERVAL
+
     # @return [Time]
     DEFAULT_TIMESTAMP = Time.at(0).utc.freeze
 
@@ -167,7 +177,7 @@ module PgEventstore
         restart_count: 0,
         last_restarted_at: nil,
         max_restarts_number: max_restarts_number,
-        chunk_query_interval: chunk_query_interval,
+        chunk_query_interval: [chunk_query_interval, MIN_EVENTS_PULL_INTERVAL].max,
         last_chunk_fed_at: DEFAULT_TIMESTAMP,
         last_chunk_greatest_position: nil,
         last_error: nil,

data/lib/pg_eventstore/subscriptions/subscription_feeder.rb

@@ -7,6 +7,12 @@ module PgEventstore
   class SubscriptionFeeder
     extend Forwardable
 
+    # Determines how often to fetch events from the event store.
+    # @see PgEventstore::Subscription::MIN_EVENTS_PULL_INTERVAL
+    # @return [Float]
+    EVENTS_PULL_INTERVAL = 0.1 # seconds
+    private_constant :EVENTS_PULL_INTERVAL
+
     attr_reader :config_name
 
     def_delegators :@basic_runner, :start, :stop, :restore, :state, :wait_for_finish, :stop_async, :running?
@@ -17,7 +23,7 @@ module PgEventstore
     def initialize(config_name:, subscriptions_set_lifecycle:, subscriptions_lifecycle:)
       @config_name = config_name
       @basic_runner = BasicRunner.new(
-        run_interval: 0.2,
+        run_interval: EVENTS_PULL_INTERVAL,
         async_shutdown_time: 0,
         recovery_strategies: recovery_strategies(config_name, subscriptions_set_lifecycle)
       )