pg_eventstore 2.0.0 → 3.0.0

data/lib/pg_eventstore/queries/replica_queries.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+module PgEventstore
+  # @!visibility private
+  class ReplicaQueries
+    MAX_PARTITIONS_TO_RESOLVE_PER_CALL = 100
+
+    # @!attribute connection
+    #   @return [PgEventstore::Connection]
+    attr_reader :connection
+    private :connection
+
+    # @param connection [PgEventstore::Connection]
+    # @param query_strategy [PgEventstore::QueryStrategy]
+    def initialize(connection, query_strategy)
+      @connection = connection
+      @query_strategy = query_strategy
+    end
+
+    # @param indexes [Array<PgEventstore::EventGlobalIndex::SubscriptionRepr>]
+    # @return [Array<Integer>]
+    def load_subscription_positions(indexes)
+      params = [indexes.first.subscription_position, indexes.last.subscription_position]
+      exists = @query_strategy.exec_params(<<~SQL, params).first&.[]('one')
+        select 1 as one from event_subscription_positions where subscription_position between $1 and $2 limit 1
+      SQL
+      return [] unless exists
+
+      @query_strategy.exec_params(<<~SQL, [indexes.map(&:subscription_position)]).map { _1['subscription_position'] }
+        select subscription_position from event_subscription_positions where subscription_position = any($1::bigint[])
+      SQL
+    end
+
+    # @param indexes [Array<PgEventstore::EventGlobalIndex::SubscriptionRepr>]
+    # @return [Array<PgEventstore::RawEvent>]
+    def load_events(indexes)
+      indexes = indexes.dup
+      raw_events = []
+      loop do
+        range = Utils.range_to_slice(indexes.map(&:event_type_partition_id), MAX_PARTITIONS_TO_RESOLVE_PER_CALL)
+        indexes_to_resolve = indexes.slice!(range)
+        break if indexes_to_resolve.empty?
+
+        raw_events.push(*events_global_index_queries.resolve_indexes(indexes_to_resolve, resolve_link_tos: false))
+      end
+      raw_events.map { RawEvent.new(**_1.transform_keys(&:to_sym)) }
+    end
+
+    # @param indexes [Array<PgEventstore::EventGlobalIndex::SubscriptionRepr>]
+    # @return [Array<PgEventstore::EventGlobalIndex>]
+    def load_events_global_index(indexes)
+      result = @query_strategy.exec_params(<<~SQL, [indexes.map(&:global_position)])
+        select * from events_global_index where global_position = any($1::bigint[])
+      SQL
+      result.map { EventGlobalIndex.new(**_1.transform_keys(&:to_sym)) }
+    end
+
+    # @param indexes [Array<PgEventstore::EventGlobalIndex>]
+    # @return [Array<PgEventstore::StreamGlobalIndex>]
+    def load_streams_global_index(indexes)
+      result = @query_strategy.exec_params(<<~SQL, [indexes.map(&:streams_global_index_id)])
+        select * from streams_global_index where id = any($1::bigint[])
+      SQL
+      stream_revisions_map = indexes.group_by(&:streams_global_index_id).to_h do |stream_id, events_idx|
+        [stream_id, events_idx.max_by(&:stream_revision).stream_revision]
+      end
+      result.map do |attrs|
+        attrs = attrs.transform_keys(&:to_sym)
+        attrs[:stream_revision] = stream_revisions_map[attrs[:id]]
+        StreamGlobalIndex.new(**attrs)
+      end
+    end
+
+    # @param indexes [Array<PgEventstore::EventGlobalIndex::SubscriptionRepr>]
+    # @return [Array<PgEventstore::EventMarkerIndex>]
+    def load_event_markers_index(indexes)
+      result = @query_strategy.exec_params(<<~SQL, [indexes.map(&:global_position)])
+        select * from event_markers_index where global_position = any($1::bigint[])
+      SQL
+      result.map { EventMarkerIndex.new(**_1.transform_keys(&:to_sym)) }
+    end
+
+    # @param indexes [Array<PgEventstore::EventMarkerIndex>]
+    # @return [Array<PgEventstore::EventMarker>]
+    def load_markers(indexes)
+      result = @query_strategy.exec_params(<<~SQL, [indexes.map(&:marker_id).uniq])
+        select * from event_markers where id = any($1::bigint[])
+      SQL
+      result.map { EventMarker.new(**_1.transform_keys(&:to_sym)) }
+    end
+
+    # @param indexes [Array<PgEventstore::EventGlobalIndex>]
+    # @return [Array<PgEventstore::Partition>]
+    def load_partitions(indexes)
+      ids = indexes.map(&:context_partition_id).uniq + indexes.map(&:stream_name_partition_id).uniq +
+            indexes.map(&:event_type_partition_id).uniq
+      result = @query_strategy.exec_params(<<~SQL, [ids])
+        select * from partitions where id = any($1::bigint[]) order by id
+      SQL
+      result.map { Partition.new(**_1.transform_keys(&:to_sym)) }
+    end
+
+    private
+
+    # @return [PgEventstore::EventsGlobalIndexQueries]
+    def events_global_index_queries
+      EventsGlobalIndexQueries.new(connection, @query_strategy)
+    end
+  end
+end

data/lib/pg_eventstore/queries/streams_global_index_queries.rb

@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+module PgEventstore
+  # @!visibility private
+  class StreamsGlobalIndexQueries
+    # @!attribute connection
+    #   @return [PgEventstore::Connection]
+    attr_reader :connection
+    private :connection
+
+    # @param connection [PgEventstore::Connection]
+    # @param query_strategy [PgEventstore::QueryStrategy]
+    def initialize(connection, query_strategy)
+      @connection = connection
+      @query_strategy = query_strategy
+    end
+
+    # @param stream [PgEventstore::Stream]
+    # @return [PgEventstore::StreamGlobalIndex]
+    def find_or_create_by(stream)
+      transaction_queries.transaction do
+        find_by(stream) || create(stream)
+      end
+    end
+
+    # @param stream [PgEventstore::Stream]
+    # @return [PgEventstore::StreamGlobalIndex, nil]
+    def find_by(stream)
+      idx_partitions_filter = QueryBuilders::Filters::Collection.from_options({ filter: { streams: [stream.to_hash] } })
+      filtering = QueryBuilders::StreamsGlobalIndexFiltering.new
+      idx_partitions_filter.collection.each(&filtering.method(:add_filter_row))
+      deserialize_one(@query_strategy.exec_params(*filtering.to_exec_params))
+    end
+
+    # @param stream [PgEventstore::Stream]
+    # @return [PgEventstore::StreamGlobalIndex]
+    def find_by!(stream)
+      find_by(stream) ||
+        raise(RecordNotFound.new(QueryBuilders::StreamsGlobalIndexFiltering::PRIMARY_TABLE_NAME, stream))
+    end
+
+    # @param id [Integer]
+    # @param attrs [Hash]
+    # @return [void]
+    def update(id, **attrs)
+      attrs_sql = attrs.keys.map.with_index(2) do |attr, index|
+        "#{attr} = $#{index}"
+      end.join(', ')
+      @query_strategy.exec_params(<<~SQL, [id, *attrs.values])
+        UPDATE streams_global_index SET #{attrs_sql} WHERE id = $1
+      SQL
+    end
+
+    # @param stream [PgEventstore::Stream]
+    # @return [PgEventstore::StreamGlobalIndex]
+    def create(stream)
+      filter_collection = QueryBuilders::Filters::Collection.from_options(
+        { filter: { streams: [{ context: stream.context, stream_name: stream.stream_name }] } }
+      )
+      partitions_sql_builder = QueryBuilders::PartitionsFiltering.assemble_sql_builder(
+        filter_collection,
+        scope: :stream_name
+      )
+      partitions_sql_builder.unselect.select('id')
+      sql, positional_args = partitions_sql_builder.to_exec_params
+      stream_revision = Stream::NON_EXISTING_STREAM_REVISION
+      starting_position = StreamGlobalIndex::INITIAL_STARTING_POSITION
+      stream_id = PG::Connection.escape(stream.stream_id)
+      value = "((#{sql}), '#{stream_id}', #{stream_revision}, #{starting_position})"
+
+      res = @query_strategy.exec_params(<<~SQL, positional_args)
+        INSERT INTO streams_global_index ("partition_id", "stream_id", "stream_revision", "starting_position")
+        VALUES #{value} RETURNING *
+      SQL
+      deserialize_one(res)
+    end
+
+    # @param id [Integer]
+    # @return [Boolean]
+    def delete(id)
+      res = @query_strategy.exec_params('delete from streams_global_index where id = $1', [id])
+      res.cmd_tuples == 1
+    end
+
+    # @param stream [PgEventstore::Stream]
+    # @return [Boolean]
+    def stream_exists?(stream)
+      idx_partitions_filter = QueryBuilders::Filters::Collection.from_options({ filter: { streams: [stream.to_hash] } })
+      filtering = QueryBuilders::StreamsGlobalIndexFiltering.new
+      idx_partitions_filter.collection.each(&filtering.method(:add_filter_row))
+      builder = filtering.to_sql_builder.unselect.select('1 as exists')
+      @query_strategy.exec_params(*builder.to_exec_params).ntuples == 1
+    end
+
+    # @param stream [PgEventstore::Stream]
+    # @return [Integer, nil]
+    def stream_revision(stream)
+      idx_partitions_filter = QueryBuilders::Filters::Collection.from_options({ filter: { streams: [stream.to_hash] } })
+      filtering = QueryBuilders::StreamsGlobalIndexFiltering.new
+      idx_partitions_filter.collection.each(&filtering.method(:add_filter_row))
+      builder = filtering.to_sql_builder.unselect.select('stream_revision')
+      @query_strategy.exec_params(*builder.to_exec_params).first&.dig('stream_revision')
+    end
+
+    # @param options [Hash]
+    # @option options [Integer, nil] :from_position
+    # @option options [Integer, nil] :max_count
+    # @option options [String, Symbol, nil] :direction
+    # @return [Array<PgEventstore::StreamGlobalIndex>]
+    def streams_global_index(options)
+      sql_builder = QueryBuilders::StreamsGlobalIndexFiltering.sql_builder_for_basic_pagination(options)
+      deserialize_many(@query_strategy.exec_params(*sql_builder.to_exec_params))
+    end
+
+    # @param indexes [Array<PgEventstore::StreamGlobalIndex>]
+    # @return [Array<PgEventstore::Stream>]
+    def resolve_indexes(indexes)
+      return [] if indexes.empty?
+
+      partition_ids = indexes.map(&:partition_id).uniq
+      partitions = partition_queries.find_by_ids(partition_ids).to_h { [_1['id'], _1] }
+      indexes.map do |stream_global_index|
+        partition = partitions[stream_global_index.partition_id]
+        Stream.new(
+          context: partition['context'],
+          stream_name: partition['stream_name'],
+          stream_id: stream_global_index.stream_id,
+          stream_revision: stream_global_index.stream_revision,
+          starting_position: stream_global_index.starting_position
+        )
+      end
+    end
+
+    private
+
+    # @return [PgEventstore::TransactionQueries]
+    def transaction_queries
+      TransactionQueries.new(connection)
+    end
+
+    # @param pg_result [PG::Result]
+    # @return [PgEventstore::StreamGlobalIndex, nil]
+    def deserialize_one(pg_result)
+      first_result = pg_result.first
+      return unless first_result
+
+      StreamGlobalIndex.new(**first_result.transform_keys(&:to_sym))
+    end
+
+    # @param pg_result [PG::Result]
+    # @return [Array<PgEventstore::StreamGlobalIndex>]
+    def deserialize_many(pg_result)
+      pg_result.map do |attrs|
+        StreamGlobalIndex.new(**attrs.transform_keys(&:to_sym))
+      end
+    end
+
+    # @return [PgEventstore::PartitionQueries]
+    def partition_queries
+      PartitionQueries.new(connection)
+    end
+  end
+end

data/lib/pg_eventstore/queries/transaction_queries.rb

@@ -11,6 +11,8 @@ module PgEventstore
     }.tap do |h|
       h.default = h[:serializable]
     end.freeze
+    # @return [Integer]
+    MAX_NUMBER_OF_UNIQUE_CONSTRAINTS_ERROR_RETRIES = 5
 
     # @!attribute connection
     #   @return [PgEventstore::Connection]
@@ -41,27 +43,67 @@ module PgEventstore
     # @param pg_connection [PG::Connection]
     # @return [void]
     def pg_transaction(level, read_only, pg_connection, &)
-      pg_connection.transaction do
-        if read_only
-          pg_connection.exec("SET TRANSACTION ISOLATION LEVEL #{level} READ ONLY")
-        else
-          pg_connection.exec("SET TRANSACTION ISOLATION LEVEL #{level}")
+      retries_count = 0
+      begin
+        _transaction(pg_connection) do
+          if read_only
+            pg_connection.exec("SET TRANSACTION ISOLATION LEVEL #{level} READ ONLY")
+          else
+            pg_connection.exec("SET TRANSACTION ISOLATION LEVEL #{level}")
+          end
+          yield
         end
-        yield
+      rescue PG::TRSerializationFailure, PG::TRDeadlockDetected
+        retry
+      rescue PG::UniqueViolation
+        retries_count += 1
+        raise if retries_count >= MAX_NUMBER_OF_UNIQUE_CONSTRAINTS_ERROR_RETRIES
+
+        retry
       end
-    rescue PG::TRSerializationFailure, PG::TRDeadlockDetected
-      retry
     rescue MissingPartitions => error
       error.event_types.each do |event_type|
         transaction do
           partition_queries.create_partitions(error.stream, event_type)
         end
-      rescue PG::UniqueViolation
-        retry
       end
       retry
     end
 
+    # TODO: next pg gem release (presumably v1.6.4) should already include the fix of PG::Connection#transaction.
+    #       Changes are already in master branch
+    #       https://github.com/ged/ruby-pg/commit/c3d2aabd0a19d20bb1f2b2fa2e5b30f03b043aff
+    #       Once it is released - change pg gem requirement in gemspec to '>= 1.6.4' and remove this implementation.
+    # @param conn [PG::Connection]
+    # rubocop:disable Lint/RescueException
+    def _transaction(conn)
+      rollback = false
+      conn.exec('BEGIN')
+      yield
+    rescue PG::RollbackTransaction
+      rollback = true
+      perform_rollback(conn)
+    rescue Exception
+      rollback = true
+      perform_rollback(conn)
+      raise
+    ensure
+      unless rollback
+        if Thread.current.status == 'aborting'
+          perform_rollback(conn)
+        else
+          conn.exec('COMMIT')
+        end
+      end
+    end
+    # rubocop:enable Lint/RescueException
+
+    def perform_rollback(conn)
+      conn.cancel if conn.transaction_status == PG::PQTRANS_ACTIVE
+      conn.block
+      conn.exec('ROLLBACK')
+    end
+
     # @return [PgEventstore::PartitionQueries]
     def partition_queries
       PartitionQueries.new(connection)

data/lib/pg_eventstore/queries.rb

@@ -1,14 +1,28 @@
 # frozen_string_literal: true
 
 require_relative 'sql_builder'
+require_relative 'query_builders/filters/collection'
+require_relative 'query_builders/read_cursor/stream_cursor'
 require_relative 'query_builders/basic_filtering'
+require_relative 'query_builders/subscription_events_filtering'
 require_relative 'query_builders/events_filtering'
 require_relative 'query_builders/partitions_filtering'
+require_relative 'query_builders/events_global_index_filtering'
+require_relative 'query_builders/streams_global_index_filtering'
+require_relative 'query_builders/event_subscription_positions_filtering'
+require_relative 'query_builders/event_markers_filtering'
+require_relative 'query_builders/event_markers_index_filtering'
+require_relative 'query_builders/index_based_events_filtering'
 require_relative 'queries/transaction_queries'
 require_relative 'queries/event_queries'
 require_relative 'queries/partition_queries'
 require_relative 'queries/links_resolver'
 require_relative 'queries/maintenance_queries'
+require_relative 'queries/events_global_index_queries'
+require_relative 'queries/streams_global_index_queries'
+require_relative 'queries/event_marker_queries'
+require_relative 'queries/index_filtering_queries'
+require_relative 'queries/replica_queries'
 
 module PgEventstore
   # @!visibility private
@@ -24,9 +38,23 @@ module PgEventstore
     # @!attribute transactions
     #   @return [PgEventstore::TransactionQueries, nil]
     attribute(:transactions)
-
     # @!attribute maintenance
     #   @return [PgEventstore::MaintenanceQueries, nil]
     attribute(:maintenance)
+    # @!attribute maintenance
+    #   @return [PgEventstore::EventsGlobalIndexQueries, nil]
+    attribute(:events_global_index)
+    # @!attribute maintenance
+    #   @return [PgEventstore::StreamsGlobalIndexQueries, nil]
+    attribute(:streams_global_index)
+    # @!attribute event_subscription_positions
+    #   @return [PgEventstore::EventSubscriptionPositionQueries, nil]
+    attribute(:event_subscription_positions)
+    # @!attribute event_subscription_positions
+    #   @return [PgEventstore::EventMarkerQueries, nil]
+    attribute(:event_markers)
+    # @!attribute event_subscription_positions
+    #   @return [PgEventstore::IndexFilteringQueries, nil]
+    attribute(:index_filtering)
   end
 end

data/lib/pg_eventstore/query_builders/basic_filtering.rb

@@ -3,9 +3,42 @@
 module PgEventstore
   module QueryBuilders
     # @!visibility private
-    class BasicFiltering
-      def initialize
-        @sql_builder = SQLBuilder.new.select("#{to_table_name}.*").from(to_table_name)
+    module BasicFiltering
+      # @return [Hash<String => String, Symbol => String>]
+      SQL_DIRECTIONS = {
+        'asc' => 'ASC',
+        'desc' => 'DESC',
+        :asc => 'ASC',
+        :desc => 'DESC',
+        'Forwards' => 'ASC',
+        'Backwards' => 'DESC',
+      }.tap do |directions|
+        directions.default = 'ASC'
+      end.freeze
+
+      module ClassMethods
+        # @param order [String, Symbol, nil]
+        # @return [Symbol]
+        def reverse_order(order)
+          SQL_DIRECTIONS[order] == 'ASC' ? :desc : :asc
+        end
+
+        # @param order [String, Symbol, nil]
+        # @return [Boolean]
+        def ascending?(order)
+          SQL_DIRECTIONS[order] == 'ASC'
+        end
+
+        # @param order [String, Symbol, nil]
+        # @return [Boolean]
+        def descending?(order)
+          SQL_DIRECTIONS[order] == 'DESC'
+        end
+      end
+
+      def self.included(base)
+        super
+        base.extend(ClassMethods)
       end
 
       # @return [String]
@@ -15,12 +48,12 @@ module PgEventstore
 
       # @return [PgEventstore::SQLBuilder]
       def to_sql_builder
-        @sql_builder
+        raise NotImplementedError
       end
 
       # @return [Array]
       def to_exec_params
-        @sql_builder.to_exec_params
+        to_sql_builder.to_exec_params
       end
     end
   end

data/lib/pg_eventstore/query_builders/event_markers_filtering.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module PgEventstore
+  module QueryBuilders
+    # @!visibility private
+    class EventMarkersFiltering
+      include BasicFiltering
+
+      PRIMARY_TABLE_NAME = 'event_markers'
+
+      class << self
+        # @param names [Array<String>]
+        # @return [PgEventstore::SQLBuilder]
+        def sql_builder_by_names(names)
+          instance = new
+          instance.add_names(names)
+          instance.to_sql_builder
+        end
+      end
+
+      def initialize
+        @sql_builder = SQLBuilder.new.select("#{to_table_name}.id, #{to_table_name}.name").from(to_table_name)
+      end
+
+      def to_sql_builder
+        @sql_builder
+      end
+
+      # @return [String]
+      def to_table_name
+        PRIMARY_TABLE_NAME
+      end
+
+      # @param names [Array<String>]
+      # @return [void]
+      def add_names(names)
+        @sql_builder.where_or("#{to_table_name}.name = any(?::varchar[])", names)
+      end
+
+      # @param name [String]
+      # @return [void]
+      def add_name(name)
+        @sql_builder.where_or("#{to_table_name}.name = ?", name)
+      end
+    end
+  end
+end