pg_eventstore 2.0.0 → 3.0.0

data/lib/pg_eventstore/query_builders/filters/marker_filter_row.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module PgEventstore
+  module QueryBuilders
+    module Filters
+      # @!visibility private
+      class MarkerFilterRow
+        include Extensions::OptionsExtension
+        include Extensions::OptionsDefaults
+
+        # @!attribute stream_filter
+        #   @return [PgEventstore::QueryBuilders::Filters::StreamFilter, nil]
+        attribute(:stream_filter)
+        # @!attribute stream_filter
+        #   @return [PgEventstore::QueryBuilders::Filters::MarkerFilter]
+        attribute(:marker_filter)
+
+        # @return [Array<PgEventstore::QueryBuilders::Filters::MarkerFilterRow>]
+        def flatten
+          return [dup] if marker_filter.markers.size == 1
+
+          marker_filter.markers.map do |marker|
+            marker_filter = MarkerFilter.new(event_type: self.marker_filter.event_type, markers: [marker])
+            self.class.new(stream_filter:, marker_filter:)
+          end
+        end
+
+        # Event type is ambiguous in case we can't tell which partition it belongs to. We **can** tell the exact
+        # partition of the given event type only when we have context and stream name as well.
+        # @return [Boolean]
+        def ambiguous_event_type?
+          return true if !stream_filter.nil? && !stream_filter.stream? && marker_filter.event_type.nil?
+
+          (stream_filter.nil? || stream_filter.context?) && !marker_filter.event_type.nil?
+        end
+
+        # @return [PgEventstore::QueryBuilders::Filters::FilterRow]
+        def to_filter_row
+          if marker_filter.event_type
+            event_type_filter = EventTypeFilter.new(event_type: marker_filter.event_type, prefix: false)
+            FilterRow.new(stream_filter:, event_type_filters: [event_type_filter])
+          else
+            FilterRow.new(stream_filter:, event_type_filters: [])
+          end
+        end
+      end
+    end
+  end
+end

data/lib/pg_eventstore/query_builders/filters/stream_filter.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module PgEventstore
+  module QueryBuilders
+    module Filters
+      # @!visibility private
+      class StreamFilter
+        include Extensions::OptionsExtension
+        include Extensions::OptionsDefaults
+
+        # @!attribute context
+        #   @return [String]
+        attribute(:context)
+        # @!attribute stream_name
+        #   @return [String, nil]
+        attribute(:stream_name)
+        # @!attribute stream_id
+        #   @return [String, nil]
+        attribute(:stream_id)
+
+        # @return [Boolean]
+        def context?
+          stream_name.nil? && stream_id.nil?
+        end
+
+        # @return [Boolean]
+        def stream_name?
+          !stream_name.nil? && stream_id.nil?
+        end
+
+        # @return [Boolean]
+        def stream?
+          !stream_name.nil? && !stream_id.nil?
+        end
+
+        # @return [Hash]
+        def to_partition_h
+          { context:, stream_name: }.compact
+        end
+
+        # @return [Hash]
+        def to_h
+          { context:, stream_name:, stream_id: }.compact
+        end
+      end
+    end
+  end
+end

data/lib/pg_eventstore/query_builders/index_based_events_filtering.rb

@@ -0,0 +1,260 @@
+# frozen_string_literal: true
+
+module PgEventstore
+  module QueryBuilders
+    class IndexBasedEventsFiltering
+      include BasicFiltering
+
+      # @return [Integer]
+      DEFAULT_LIMIT = 1_000
+      # @return [String]
+      ABSTRACT_TABLE_NAME = 'events_index'
+
+      class << self
+        # @param stream [PgEventstore::Stream]
+        # @param expected_revision [PgEventstore::Commands::RevisionCheck::ExpectedRevision::EventTypeRevision]
+        # @return [PgEventstore::SQLBuilder]
+        def sql_builder_for_event_type_revision_validation(stream, expected_revision)
+          cursor = QueryBuilders::ReadCursor::StreamCursor.from_stream_and_options(
+            stream, { direction: :desc, max_count: 1 }
+          )
+          filters_collection = Filters::Collection.from_stream_and_options(
+            stream, { filter: { event_types: [expected_revision.event_type] } }
+          )
+          rows = filters_collection.collection
+          Utils.assert!(rows.size == 1, filters_collection.inspect)
+
+          EventsGlobalIndexFiltering.for_revision_validation_per_type(
+            rows.first, cursor, expected_revision.sequence_number
+          )
+        end
+
+        # @param stream [PgEventstore::Stream]
+        # @param expected_revision [PgEventstore::Commands::RevisionCheck::ExpectedRevision::EventTypeRevisionWithMarkers]
+        # @return [PgEventstore::SQLBuilder]
+        def sql_builder_for_event_type_with_markers_revision_validation(stream, expected_revision)
+          cursor = QueryBuilders::ReadCursor::StreamCursor.from_stream_and_options(
+            stream, { direction: :desc, max_count: 1 }
+          )
+          filters_collection = Filters::Collection.from_stream_and_options(
+            stream,
+            {
+              filter: {
+                event_types: [{ type: expected_revision.event_type, markers: expected_revision.markers }],
+              },
+            }
+          )
+          rows = filters_collection.collection
+          Utils.assert!(rows.size == 1, filters_collection.inspect)
+
+          builders = rows.first.flatten.map do |flattened|
+            EventMarkersIndexFiltering.for_revision_validation_per_type(
+              flattened, cursor, expected_revision.sequence_number
+            )
+          end
+          final = SQLBuilder.new.select('*')
+          final.from(SQLBuilder.union_builders(builders, mode: :union_distinct))
+          final.order('stream_revision desc')
+          final.limit(1)
+        end
+
+        # @param stream [PgEventstore::Stream]
+        # @param expected_revision [PgEventstore::Commands::RevisionCheck::ExpectedRevision::MarkersRevision]
+        # @return [PgEventstore::SQLBuilder]
+        def sql_builder_for_markers_revision_validation(stream, expected_revision)
+          cursor = QueryBuilders::ReadCursor::StreamCursor.from_stream_and_options(
+            stream, { direction: :desc, max_count: 1 }
+          )
+          filters_collection = Filters::Collection.from_stream_and_options(
+            stream, { filter: { event_types: [{ markers: expected_revision.markers }] } }
+          )
+          rows = filters_collection.collection
+          Utils.assert!(rows.size == 1, filters_collection.inspect)
+
+          builders = rows.first.flatten.map do |flattened|
+            EventMarkersIndexFiltering.for_revision_validation_per_type(
+              flattened, cursor, expected_revision.sequence_number
+            )
+          end
+          final = SQLBuilder.new.select('*')
+          final.from(SQLBuilder.union_builders(builders, mode: :union_distinct))
+          final.order('stream_revision desc')
+          final.limit(1)
+        end
+
+        # @param filter_rows [Array<PgEventstore::QueryBuilders::Filters::FilterRow,
+        #   PgEventstore::QueryBuilders::Filters::MarkerFilterRow>]
+        # @param cursor [PgEventstore::QueryBuilders::ReadCursor::StreamCursor]
+        # @return [PgEventstore::SQLBuilder]
+        def sql_builder_for_read_grouped(filter_rows, cursor)
+          cursor = cursor.dup
+          cursor.max_count = 1
+          has_markers = false
+          builders = filter_rows.flat_map do |filter_row|
+            case filter_row
+            when Filters::FilterRow
+              filter_row.flatten.map do |flattened|
+                EventsGlobalIndexFiltering.for_read_common(flattened, cursor)
+              end
+            when Filters::MarkerFilterRow
+              has_markers ||= true
+              filter_row.flatten.map do |flattened|
+                EventMarkersIndexFiltering.for_read_common(flattened, cursor)
+              end
+            else
+              Utils.missing_implementation!(filter_row)
+            end
+          end
+          union_builders(builders, cursor, mode: has_markers ? :union_distinct : :union_all, requires_limit: false)
+        end
+
+        # @param filter_rows [Array<PgEventstore::QueryBuilders::Filters::FilterRow,
+        #   PgEventstore::QueryBuilders::Filters::MarkerFilterRow>]
+        # @param cursor [PgEventstore::QueryBuilders::ReadCursor::StreamCursor]
+        # @return [PgEventstore::SQLBuilder]
+        def sql_builder_for_read_common(filter_rows, cursor)
+          return EventsGlobalIndexFiltering.default_filtering(cursor).to_sql_builder if filter_rows.empty?
+
+          has_markers = false
+          builders = filter_rows.flat_map do |filter_row|
+            case filter_row
+            when Filters::FilterRow
+              filter_row.flatten.map do |flattened|
+                EventsGlobalIndexFiltering.for_read_common(flattened, cursor)
+              end
+            when Filters::MarkerFilterRow
+              has_markers ||= true
+              filter_row.flatten.map do |flattened|
+                EventMarkersIndexFiltering.for_read_common(flattened, cursor)
+              end
+            else
+              Utils.missing_implementation!(filter_row)
+            end
+          end
+          union_builders(builders, cursor, mode: has_markers ? :union_distinct : :union_all)
+        end
+
+        # @param filter_rows [Array<PgEventstore::QueryBuilders::Filters::FilterRow,
+        #   PgEventstore::QueryBuilders::Filters::MarkerFilterRow>]
+        # @return [PgEventstore::SQLBuilder]
+        def sql_builder_for_estimate_count(filter_rows)
+          cursor = ReadCursor::StreamCursor.from_options({})
+          if filter_rows.empty?
+            return EventsGlobalIndexFiltering.default_filtering(cursor).to_sql_builder.remove_limit.remove_order
+          end
+
+          has_markers = false
+          builders = filter_rows.flat_map do |filter_row|
+            case filter_row
+            when Filters::FilterRow
+              filter_row.flatten.map do |flattened|
+                EventsGlobalIndexFiltering.for_read_common(flattened, cursor)
+              end
+            when Filters::MarkerFilterRow
+              has_markers ||= true
+              filter_row.flatten.map do |flattened|
+                EventMarkersIndexFiltering.for_read_common(flattened, cursor)
+              end
+            else
+              Utils.missing_implementation!(filter_row)
+            end
+          end
+          builders.each { _1.remove_limit.remove_order }
+          top_builder = SQLBuilder.new.select('*')
+          top_builder.from(SQLBuilder.union_builders(builders, mode: has_markers ? :union_distinct : :union_all))
+          top_builder
+        end
+
+        # @param filter_rows [Array<PgEventstore::QueryBuilders::Filters::FilterRow,
+        #   PgEventstore::QueryBuilders::Filters::MarkerFilterRow>]
+        # @return [PgEventstore::SQLBuilder]
+        def sql_builder_for_subscriptions(filter_rows)
+          if filter_rows.empty?
+            sql_builder = EventsGlobalIndexFiltering.new.tap(&:for_subscription).to_sql_builder
+            return finalize_subscription_builders([sql_builder])
+          end
+
+          builders = []
+          grouped = filter_rows.group_by(&:class)
+          grouped.each_value do |rows|
+            case rows.first
+            when Filters::FilterRow
+              filtering = EventsGlobalIndexFiltering.new
+              filtering.for_subscription
+              rows.each(&filtering.method(:add_filter_row))
+              builders.push(filtering.to_sql_builder)
+            when Filters::MarkerFilterRow
+              filtering = EventMarkersIndexFiltering.new
+              filtering.for_subscription
+              rows.each(&filtering.method(:add_marker_filter_row))
+              builders.push(filtering.to_sql_builder)
+            else
+              Utils.missing_implementation!(rows.first)
+            end
+          end
+
+          finalize_subscription_builders(builders)
+        end
+
+        private
+
+        # @param builders [Array<PgEventstore::SQLBuilder>]
+        # @return [PgEventstore::SQLBuilder]
+        def finalize_subscription_builders(builders)
+          return builders.first if builders.size == 1
+
+          final = SQLBuilder.new.select(
+            'events_idx.global_position, events_idx.event_type_partition_id, events_idx.subscription_position'
+          )
+          final.from(SQLBuilder.union_builders(builders, mode: :union_distinct), table_alias: 'events_idx')
+          final.order('events_idx.subscription_position')
+          final.limit('max_count')
+        end
+
+        # @param builders [Array<PgEventstore::SQLBuilder>]
+        # @param cursor [PgEventstore::QueryBuilders::ReadCursor::StreamCursor]
+        # @param mode [Symbol]
+        # @param requires_limit [Boolean]
+        # @return [PgEventstore::SQLBuilder]
+        def union_builders(builders, cursor, mode:, requires_limit: true)
+          return builders.first if builders.size == 1
+
+          union_builder = SQLBuilder.union_builders(builders, mode:)
+          top_filtering = new
+          # Union builder always has fixed columns in SELECT (global_position and event_type_partition_id). Thus,
+          # define SystemStreamOptions that is resolved to the sorting by global_position instead of possible
+          # sorting by stream_revision which may not be present among the columns list
+          cursor = ReadCursor::StreamCursor.from_options(direction: cursor.direction, max_count: cursor.max_count)
+          top_filtering.add_global_position_direction(cursor.direction)
+          top_filtering.add_limit(cursor.max_count) if requires_limit
+          top_filtering.to_sql_builder.from(union_builder)
+        end
+      end
+
+      def initialize
+        @sql_builder = SQLBuilder.new
+        @sql_builder.select('global_position, event_type_partition_id')
+      end
+
+      def to_table_name
+        ''
+      end
+
+      def to_sql_builder
+        @sql_builder
+      end
+
+      # @param direction [String, Symbol, nil]
+      # @return [void]
+      def add_global_position_direction(direction)
+        @sql_builder.order("global_position #{SQL_DIRECTIONS[direction]}")
+      end
+
+      # @param limit [Integer, nil]
+      # @return [void]
+      def add_limit(limit)
+        @sql_builder.limit(limit || DEFAULT_LIMIT)
+      end
+    end
+  end
+end

data/lib/pg_eventstore/query_builders/partitions_filtering.rb

@@ -3,88 +3,212 @@
 module PgEventstore
   module QueryBuilders
     # @!visibility private
-    class PartitionsFiltering < BasicFiltering
+    class PartitionsFiltering
+      include BasicFiltering
+
       # @return [String]
       TABLE_NAME = 'partitions'
-      private_constant :TABLE_NAME
 
       class << self
-        # @param options [Hash]
-        # @return [Array<String>]
-        def extract_event_types_filter(options)
-          options in { filter: { event_types: Array => event_types } }
-          event_types = event_types&.select { _1.is_a?(String) }
-          event_types || []
-        end
+        # @param filters_collection [PgEventstore::QueryBuilders::Filters::Collection]
+        # @param scope [Symbol] what kind of partition we want to receive. Available options are :event_type, :context,
+        #   :stream_name and :auto. In :auto mode the scope will be calculated based on stream_filters and event_filters
+        # @return [PgEventstore::SQLBuilder]
+        def assemble_sql_builder(filters_collection, scope: :event_type)
+          if filters_collection.has_event_types?
+            # When event type filters are present - they apply constraints as is. Thus, we can't look up partitions by
+            # stream attributes separately.
+            filtering = new
+            filters_collection.collection.each(&filtering.method(:add_filter_row))
+            set_partitions_scope(filtering, has_stream_name_filter?(filters_collection), true, scope)
+          else
+            # When event type filters are absent - we can look up partitions by context and context/stream_name
+            # separately, thus potentially producing one-to-one mapping of filter-to-partition with :auto scope. For
+            # example, let's say we have stream attributes filter like
+            # [{ context: 'FooCtx', stream_name: 'Bar'}, { context: 'BarCtx' }], then we would be able to look up
+            # partitions by the exact match, returning only two of them according to the provided filters - stream
+            # partition for first filter and context partition for second filter.
+            builders = filters_collection.collection.map do |filter_row|
+              filtering = new
+              filtering.add_filter_row(filter_row)
+              set_partitions_scope(filtering, filter_row.stream_filter.stream_name?, false, scope)
+            end
 
-        # @param options [Hash]
-        # @return [Array<Hash[Symbol, String]>]
-        def extract_streams_filter(options)
-          options in { filter: { streams: Array => streams } }
-          streams = streams&.map do |stream_attrs|
-            stream_attrs in { context: String | NilClass => context }
-            stream_attrs in { stream_name: String | NilClass => stream_name }
-            { context:, stream_name: }
+            sql_builder = SQLBuilder.union_builders(builders) if builders.any?
+            sql_builder ||
+              begin
+                filtering = new
+                set_partitions_scope(filtering, has_stream_name_filter?(filters_collection), false, scope)
+              end
           end
-          streams || []
         end
 
-        # @param stream_attrs [Hash]
+        # @param filter_row [PgEventstore::QueryBuilders::Filters::FilterRow]
+        # @param scope [Symbol]
+        # @return [PgEventstore::SQLBuilder]
+        def from_filter_row(filter_row, scope: :event_type)
+          filtering = new
+          filtering.add_filter_row(filter_row)
+          has_stream_name_filter = filter_row.stream_filter&.stream_name? || false
+          has_event_type_filters = filter_row.event_type_filters.any?
+          set_partitions_scope(filtering, has_stream_name_filter, has_event_type_filters, scope)
+        end
+
+        private
+
+        # @param filters_collection [PgEventstore::QueryBuilders::Filters::Collection]
         # @return [Boolean]
-        def correct_stream_filter?(stream_attrs)
-          result = (stream_attrs in { context: String, stream_name: String } | { context: String })
-          return true if result
-
-          PgEventstore.logger&.debug(<<~TEXT)
-            Ignoring unsupported stream filter format for grouped read #{stream_attrs.compact.inspect}. \
-            See docs/reading_events.md docs for supported formats.
-          TEXT
-          false
+        # rubocop:disable Naming/PredicatePrefix
+        def has_stream_name_filter?(filters_collection)
+          filters_collection.collection.any? { |filter_row| filter_row.stream_filter&.stream_name? }
+        end
+        # rubocop:enable Naming/PredicatePrefix
+
+        # @param partitions_filtering [PgEventstore::QueryBuilders::PartitionsFiltering]
+        # @param has_stream_name_filters [Boolean]
+        # @param has_event_type_filters [Boolean]
+        # @param scope [Symbol]
+        # @return [PgEventstore::SQLBuilder]
+        def set_partitions_scope(partitions_filtering, has_stream_name_filters, has_event_type_filters, scope)
+          case scope
+          when :event_type
+            partitions_filtering.with_event_types
+            partitions_filtering.order_by_event_type
+            partitions_filtering.to_sql_builder
+          when :stream_name
+            filter = QueryBuilders::PartitionsFiltering.new
+            filter.without_event_types
+            filter.order_by_stream_name
+            table_name = filter.to_table_name
+            builder = filter.to_sql_builder
+            builder.where(
+              "(#{table_name}.context, #{table_name}.stream_name) in ?",
+              partitions_filtering.
+                to_sql_builder.
+                unselect.
+                select(<<~SQL)
+                  distinct on (#{table_name}.context, #{table_name}.stream_name)
+                    #{table_name}.context, #{table_name}.stream_name
+                SQL
+            )
+          when :context
+            filter = QueryBuilders::PartitionsFiltering.new
+            filter.without_event_types
+            filter.without_stream_names
+            filter.order_by_context
+            builder = filter.to_sql_builder
+            builder.where(
+              "#{filter.to_table_name}.context in ?",
+              partitions_filtering.
+                to_sql_builder.
+                unselect.
+                select("distinct on (#{filter.to_table_name}.context) #{filter.to_table_name}.context")
+            )
+          when :auto
+            if has_event_type_filters
+              set_partitions_scope(partitions_filtering, has_stream_name_filters, has_event_type_filters, :event_type)
+            elsif has_stream_name_filters
+              set_partitions_scope(partitions_filtering, has_stream_name_filters, has_event_type_filters, :stream_name)
+            else
+              set_partitions_scope(partitions_filtering, has_stream_name_filters, has_event_type_filters, :context)
+            end
+          else
+            Utils.missing_implementation!(scope)
+          end
         end
       end
 
+      def initialize
+        @sql_builder = SQLBuilder.new.select("#{to_table_name}.*").from(to_table_name)
+        @has_event_types = []
+        @has_stream_names = []
+        @wants_event_type = false
+        @wants_stream_name = false
+      end
+
+      def to_sql_builder
+        if @wants_event_type && (@has_event_types.empty? || !@has_event_types.all?)
+          @sql_builder.where("#{to_table_name}.event_type IS NOT NULL")
+        end
+        if @wants_stream_name && (@has_stream_names.empty? || !@has_stream_names.all?)
+          @sql_builder.where("#{to_table_name}.stream_name IS NOT NULL")
+        end
+        @sql_builder
+      end
+
       # @return [String]
       def to_table_name
         TABLE_NAME
       end
 
-      # @param context [String, nil]
-      # @param stream_name [String, nil]
-      # @return [PgEventstore::SQLBuilder]
-      def add_stream_attrs(context: nil, stream_name: nil)
-        stream_attrs = { context:, stream_name: }
-        return @sql_builder unless self.class.correct_stream_filter?(stream_attrs)
-
-        stream_attrs.compact!
-        sql = stream_attrs.map do |attr, _|
-          "#{to_table_name}.#{attr} = ?"
-        end.join(' AND ')
-        @sql_builder.where_or(sql, *stream_attrs.values)
-      end
+      # @param filter_row [PgEventstore::QueryBuilders::Filters::FilterRow]
+      # @return [void]
+      def add_filter_row(filter_row)
+        stream_filter = filter_row.stream_filter
+        event_type_filters = filter_row.event_type_filters
+        @has_stream_names.push(stream_filter&.stream_name? || false)
+        @has_event_types.push(event_type_filters.any?)
 
-      # @param event_types [Array<String>]
-      # @return [PgEventstore::SQLBuilder]
-      def add_event_types(event_types)
-        return @sql_builder if event_types.empty?
+        stream_parts_filters = []
+        event_types_filters = []
+        stream_filter&.to_partition_h&.each do |attr, value|
+          stream_parts_filters.push(["#{to_table_name}.#{attr} = ?", value])
+        end
+        event_type_filters.each do |event_type_filter|
+          comparison_part = event_type_filter.prefix? ? 'like' : '='
+          event_types_filters.push(
+            ["#{to_table_name}.event_type #{comparison_part} ?", event_type_filter.to_sql_value]
+          )
+        end
 
-        @sql_builder.where("#{to_table_name}.event_type = ANY(?::varchar[])", event_types)
+        stream_parts_filters_query, stream_parts_filters_values = stream_parts_filters.transpose
+        event_types_query, event_types_values = event_types_filters.transpose
+        sql = ''
+        sql += stream_parts_filters_query.join(' and ') if stream_parts_filters_query&.any?
+        sql += ' and ' if sql != '' && event_types_query&.any?
+        sql += "(#{event_types_query.join(' or ')})" if event_types_query&.any?
+        @sql_builder.where_or(sql, *stream_parts_filters_values, *event_types_values)
+        nil
       end
 
-      # @return [PgEventstore::SQLBuilder]
+      # @return [void]
       def with_event_types
-        @sql_builder.where('event_type IS NOT NULL')
+        @wants_event_type = true
       end
 
+      # @return [void]
       def with_stream_names
-        @sql_builder.where('stream_name IS NOT NULL')
+        @wants_stream_name = true
       end
 
+      # @return [void]
       def without_event_types
-        @sql_builder.where('event_type IS NULL')
+        @sql_builder.where("#{to_table_name}.event_type IS NULL")
+        nil
       end
 
+      # @return [void]
       def without_stream_names
-        @sql_builder.where('stream_name IS NULL')
+        @sql_builder.where("#{to_table_name}.stream_name IS NULL")
+        nil
+      end
+
+      # @return [void]
+      def order_by_event_type
+        @sql_builder.order("#{to_table_name}.event_type asc, #{to_table_name}.id asc")
+        nil
+      end
+
+      # @return [void]
+      def order_by_stream_name
+        @sql_builder.order("#{to_table_name}.context asc, #{to_table_name}.stream_name asc")
+        nil
+      end
+
+      # @return [void]
+      def order_by_context
+        @sql_builder.order("#{to_table_name}.context asc")
+        nil
       end
     end
   end