pg_eventstore 2.0.0 → 3.0.0

data/lib/pg_eventstore/middleware/event_tracing.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module PgEventstore
+  module Middleware
+    class EventTracing
+      include Middleware
+
+      # @return [String]
+      CAUSATION_ID_KEY = "#{Event::SYSTEM_SYMBOL}CaId".freeze
+      # @return [String]
+      CORRELATION_ID_KEY = "#{Event::SYSTEM_SYMBOL}CoId".freeze
+
+      class << self
+        # @param marker [String]
+        # @return [PgEventstore::FeatureMarker]
+        def causation_marker(marker)
+          FeatureMarker.new(marker:, purpose: :causation_id, description: 'Causation ID')
+        end
+
+        # @param marker [String]
+        # @return [PgEventstore::FeatureMarker]
+        def correlation_marker(marker)
+          FeatureMarker.new(marker:, purpose: :correlation_id, description: 'Correlation ID')
+        end
+      end
+
+      def serialize(event)
+        if event.caused_by
+          event.causation_id = event.caused_by.id.dup
+          event.metadata[CAUSATION_ID_KEY] = event.causation_id
+          event.feature_markers.push(self.class.causation_marker(event.causation_id))
+        end
+
+        event.correlation_id = event.caused_by&.correlation_id&.dup || SecureRandom.uuid_v7
+        event.metadata[CORRELATION_ID_KEY] = event.correlation_id
+        event.feature_markers.push(self.class.correlation_marker(event.correlation_id))
+      end
+
+      # rubocop:disable Style/GuardClause
+      def deserialize(event)
+        if event.metadata[CAUSATION_ID_KEY]
+          event.causation_id = event.metadata[CAUSATION_ID_KEY]
+          event.feature_markers.push(self.class.causation_marker(event.causation_id))
+        end
+        if event.metadata[CORRELATION_ID_KEY]
+          event.correlation_id = event.metadata[CORRELATION_ID_KEY]
+          event.feature_markers.push(self.class.correlation_marker(event.correlation_id))
+        end
+      end
+      # rubocop:enable Style/GuardClause
+    end
+  end
+end

data/lib/pg_eventstore/partition.rb

@@ -19,5 +19,11 @@ module PgEventstore
     # @!attribute table_name
     #   @return [String]
     option(:table_name)
+    # @!attribute parent_context_partition_id
+    #   @return [Integer, nil]
+    option(:parent_context_partition_id)
+    # @!attribute parent_stream_name_partition_id
+    #   @return [Integer, nil]
+    option(:parent_stream_name_partition_id)
   end
 end

data/lib/pg_eventstore/pg_connection.rb

@@ -3,6 +3,17 @@
 module PgEventstore
   # @!visibility private
   class PgConnection < PG::Connection
+    PLACEHOLDER_RE = /
+      '(?:''|[^'])*'                                             | # string literal
+      "(?:""|[^"])*"                                             | # quoted identifier
+      --[^\n]*                                                   | # line comment
+      \/\*.*?\*\/                                                | # block comment
+      \$\$.*?\$\$                                                | # dollar-quoted string. E.g. $$ $1 $$
+      \$(?<__dq_tag>[A-Za-z_][A-Za-z_0-9]*)\$.*?\$\k<__dq_tag>\$ | # named dollar-quoted string. E.g. $foo$ $1 $foo$
+      (?<placeholder>\$(?:[1-9]\d*))                               # placeholder we are interested in
+    /mx
+    private_constant :PLACEHOLDER_RE
+
     def exec(sql)
       log(sql, [])
       super
@@ -13,37 +24,82 @@ module PgEventstore
       super
     end
 
-    private
+    def send_query_params(sql, params)
+      log(sql, params)
+      super
+    end
 
-    def log(sql, params)
-      return unless PgEventstore.logger&.debug?
+    def send_query(sql)
+      log(sql, [])
+      super
+    end
+
+    # This is a partial copy-paste of https://github.com/ged/ruby-pg/pull/726. Because maintainers think it is too risky
+    # to accept those changes because of possible sql injections in the long run - I copy-pasted it here and simplified
+    # for pg_eventstore use cases.
+    # Why is it safe for pg_eventstore? The main concern there is that if new string literal would be introduced in some
+    # future PostgreSQL version - the implementation would become vulnerable to sql injections in case that new string
+    # literal is used in queries along with #compile. Because pg_eventstore uses only two forms of string literals
+    # (those are $$ my string $$ and 'my string') - we are totally safe. Also, there is only one place so far where this
+    # method is used(create subscription table function) - all other places use exec/exec_params. To conclude: careful
+    # usage + very low risk of future exposure to sql injections makes the usage of this method acceptable for
+    # pg_eventstore.
+    #
+    # Compiles your prepared SQL statement and the given positional arguments into plain SQL string.
+    #
+    # The resulting SQL string can be used with +conn.exec+ like the prepared SQL statement and parameters with
+    # +conn.exec_params+. +conn.exec_params+ is usually preferred because it's faster and safer.
+    #
+    # Example:
+    # 	res = conn.compile('SELECT $1 AS a, $2 AS b, $3 AS c', [1, 2, nil])
+    # 	# => "SELECT '1' AS a, '2' AS b, NULL AS c"
+    # @param sql [String]
+    # @param params [Array]
+    # @return [String]
+    def compile(sql, params)
+      return sql if params.empty?
 
-      unless params&.empty?
-        sql = sql.gsub(/\$\d+/).each do |matched|
-          value = params[matched[1..].to_i - 1]
-          value = encode_value(value)
-          normalize_value(value)
-        end
+      sql.gsub(PLACEHOLDER_RE).each do |matched|
+        placeholder = Regexp.last_match[:placeholder]
+        # Do not replace non-positional args string and pass it as is
+        next matched unless placeholder
+
+        prepared_value(params[placeholder[1..].to_i - 1])
       end
-      PgEventstore.logger.debug(sql)
     end
 
+    # Converts the given object into its PostgreSQL representation
+    def prepared_value(value)
+      value = encode_value(value)
+      normalize_value(value)
+    end
+
+    private
+
     def encode_value(value)
       encoder = type_map_for_queries[value.class]
       return type_map_for_queries.send(encoder, value).encode(value) if encoder.is_a?(Symbol)
+      # format == 1 stands for binary format
+      return value if encoder.nil? || encoder.format == 1
 
-      type_map_for_queries[value.class]&.encode(value) || value
+      encoder.encode(value)
     end
 
     def normalize_value(value)
       case value
-      when String
-        "'#{value}'"
+      when TrueClass, FalseClass
+        value.to_s
       when NilClass
         'NULL'
       else
-        value
+        "'#{escape(value.to_s)}'"
       end
     end
+
+    def log(sql, params)
+      return unless PgEventstore.logger&.debug?
+
+      PgEventstore.logger.debug(compile(sql, params))
+    end
   end
 end

data/lib/pg_eventstore/queries/event_marker_queries.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module PgEventstore
+  # @!visibility private
+  class EventMarkerQueries
+    # @!attribute connection
+    #   @return [PgEventstore::Connection]
+    attr_reader :connection
+    private :connection
+
+    # @param connection [PgEventstore::Connection]
+    # @param query_strategy [PgEventstore::QueryStrategy]
+    def initialize(connection, query_strategy)
+      @connection = connection
+      @query_strategy = query_strategy
+    end
+
+    # @param names [Array<String>]
+    # @return [Array<PgEventstore::EventMarker>]
+    def find_or_create_by(names)
+      builder = QueryBuilders::EventMarkersFiltering.sql_builder_by_names(names)
+      existing = @query_strategy.exec_params(*builder.to_exec_params)
+      existing = deserialize(existing)
+      rest = names - existing.map(&:name)
+      return existing if rest.empty?
+
+      values = rest.map do |name|
+        "('#{PG::Connection.escape(name)}')"
+      end.join(',')
+      table_name = QueryBuilders::EventMarkersFiltering::PRIMARY_TABLE_NAME
+      rest = @query_strategy.exec(<<~SQL)
+        insert into #{table_name} ("name") values #{values} returning *
+      SQL
+      existing + deserialize(rest)
+    end
+
+    # @param stream_idx_id [Integer]
+    # @param write_api_events_index [Array<PgEventstore::EventGlobalIndex::WriteApiRepr>]
+    # @param revision_to_marker_ids_map [Hash<Integer, Set<Integer>>]
+    # @return [void]
+    def create_indexes(stream_idx_id, write_api_events_index, revision_to_marker_ids_map)
+      offset = write_api_events_index.first.stream_revision
+      values = revision_to_marker_ids_map.flat_map do |stream_revision, marker_ids|
+        event_index = write_api_events_index[stream_revision - offset]
+        marker_ids.map do |marker_id|
+          values = [
+            marker_id,
+            stream_idx_id,
+            event_index.global_position,
+            event_index.stream_revision,
+            event_index.event_type_partition_id,
+          ]
+          "(#{values.join(',')})"
+        end
+      end
+      values = values.join(', ')
+      table_name = QueryBuilders::EventMarkersIndexFiltering::PRIMARY_TABLE_NAME
+      @query_strategy.exec(<<~SQL)
+        insert into #{table_name}
+        ("marker_id", "streams_global_index_id", "global_position", "stream_revision", "event_type_partition_id")
+        values
+        #{values}
+      SQL
+    end
+
+    private
+
+    # @param pg_result [PG::Result]
+    # @return [Array<PgEventstore::EventMarker>]
+    def deserialize(pg_result)
+      pg_result.map { EventMarker.new(**_1.transform_keys(&:to_sym)) }
+    end
+  end
+end

data/lib/pg_eventstore/queries/event_queries.rb

@@ -6,82 +6,16 @@ module PgEventstore
     # @!attribute connection
     #   @return [PgEventstore::Connection]
     attr_reader :connection
-    # @!attribute serializer
-    #   @return [PgEventstore::EventSerializer]
-    attr_reader :serializer
-    # @!attribute deserializer
-    #   @return [PgEventstore::EventDeserializer]
-    attr_reader :deserializer
-    private :connection, :serializer, :deserializer
+    private :connection
 
     # @param connection [PgEventstore::Connection]
-    # @param serializer [PgEventstore::EventSerializer]
-    # @param deserializer [PgEventstore::EventDeserializer]
-    def initialize(connection, serializer, deserializer)
+    def initialize(connection)
       @connection = connection
-      @serializer = serializer
-      @deserializer = deserializer
-    end
-
-    # @param event [PgEventstore::Event]
-    # @return [Boolean]
-    def event_exists?(event)
-      return false if event.id.nil? || event.stream.nil?
-
-      sql_builder = SQLBuilder.new.select('1 as exists').from(Event::PRIMARY_TABLE_NAME).where('id = ?', event.id)
-      sql_builder.limit(1)
-      sql_builder.where(
-        'context = ? and stream_name = ? and type = ?', event.stream.context, event.stream.stream_name, event.type
-      )
-      connection.with do |conn|
-        conn.exec_params(*sql_builder.to_exec_params)
-      end.to_a.dig(0, 'exists') == 1
-    end
-
-    # Takes an array of potentially persisted events and loads their ids from db. Those ids can be later used to check
-    # whether events are actually existing events.
-    # @param events [Array<PgEventstore::Event>]
-    # @return [Array<Integer>]
-    def global_positions_from_db(events)
-      sql_builder = SQLBuilder.new.from(Event::PRIMARY_TABLE_NAME).select('global_position')
-      partition_attrs = events.map { |event| [event.stream&.context, event.stream&.stream_name, event.type] }.uniq
-      partition_attrs.each do |context, stream_name, event_type|
-        sql_builder.where_or('context = ? and stream_name = ? and type = ?', context, stream_name, event_type)
-      end
-      sql_builder.where('global_position = ANY(?::bigint[])', events.map(&:global_position))
-      raw_events = PgEventstore.connection.with do |conn|
-        conn.exec_params(*sql_builder.to_exec_params)
-      end.to_a
-      raw_events.map { |attrs| attrs['global_position'] }
-    end
-
-    # @param stream [PgEventstore::Stream]
-    # @return [Integer, nil]
-    def stream_revision(stream)
-      sql_builder = SQLBuilder.new.from(Event::PRIMARY_TABLE_NAME).select('stream_revision')
-      sql_builder.where('context = ? and stream_name = ? and stream_id = ?', *stream.to_a)
-      sql_builder.order('stream_revision DESC').limit(1)
-      connection.with do |conn|
-        conn.exec_params(*sql_builder.to_exec_params)
-      end.to_a.dig(0, 'stream_revision')
-    end
-
-    # @see PgEventstore::Client#read for more info
-    # @param stream [PgEventstore::Stream]
-    # @param options [Hash]
-    # @return [Array<PgEventstore::Event>]
-    def stream_events(stream, options)
-      exec_params = QueryBuilders::EventsFiltering.events_filtering(stream, options).to_exec_params
-      raw_events = connection.with do |conn|
-        conn.exec_params(*exec_params)
-      end.to_a
-      raw_events = links_resolver.resolve(raw_events) if options[:resolve_link_tos]
-      deserializer.deserialize_many(raw_events)
     end
 
     # @param stream [PgEventstore::Stream]
     # @param events [Array<PgEventstore::Event>]
-    # @return [Array<PgEventstore::Event>]
+    # @return [Array<Hash>]
     def insert(stream, events)
       sql_rows_for_insert, values = prepared_statements(stream, events)
       columns = %w[id data metadata stream_revision link_global_position link_partition_id type context stream_name stream_id]
@@ -92,30 +26,11 @@ module PgEventstore
           RETURNING *
       SQL
 
-      raw_events = connection.with do |conn|
+      result = connection.with do |conn|
         conn.exec_params(sql, values)
       end
-      raw_events.map do |raw_event|
-        deserializer.without_middlewares.deserialize(raw_event)
-      end
-    end
-
-    # @param stream [PgEventstore::Stream]
-    # @param options_by_event_type [Array<Hash>] a set of options per an event type
-    # @param options [Hash]
-    # @option options [Boolean] :resolve_link_tos
-    # @return [Array<PgEventstore::Event>]
-    def grouped_events(stream, options_by_event_type, **options)
-      builders = options_by_event_type.map do |filter|
-        QueryBuilders::EventsFiltering.events_filtering(stream, filter)
-      end
-      final_builder = SQLBuilder.union_builders(builders.map(&:to_sql_builder))
-
-      raw_events = connection.with do |conn|
-        conn.exec_params(*final_builder.to_exec_params)
-      end.to_a
-      raw_events = links_resolver.resolve(raw_events) if options[:resolve_link_tos]
-      deserializer.deserialize_many(raw_events)
+      # "returning" statement has no guarantees about the order in which rows are returned. Thus, sort them explicitly
+      result.sort_by { _1['stream_revision'] }
     end
 
     private
@@ -127,7 +42,6 @@ module PgEventstore
       positional_counter = 1
       values = []
       sql_rows_for_insert = events.map do |event|
-        event = serializer.serialize(event)
         attributes = event.options_hash.slice(
           :id, :data, :metadata, :stream_revision, :link_global_position, :link_partition_id, :type
         )
@@ -150,7 +64,7 @@ module PgEventstore
 
     # @return [PgEventstore::LinksResolver]
     def links_resolver
-      LinksResolver.new(connection)
+      LinksResolver.new(connection, QueryStrategy::Foreground.new(connection))
     end
   end
 end

data/lib/pg_eventstore/queries/events_global_index_queries.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+module PgEventstore
+  # @!visibility private
+  class EventsGlobalIndexQueries
+    # @!attribute connection
+    #   @return [PgEventstore::Connection]
+    attr_reader :connection
+    private :connection
+
+    # @param connection [PgEventstore::Connection]
+    # @param query_strategy [PgEventstore::QueryStrategy]
+    def initialize(connection, query_strategy)
+      @connection = connection
+      @query_strategy = query_strategy
+    end
+
+    # @param raw_events [Array<Hash>]
+    # @param affected_partitions [Array<PgEventstore::Partition>]
+    # @param stream_index_id [Integer]
+    # @return [Array<PgEventstore::EventGlobalIndex::WriteApiRepr>]
+    def index_events(raw_events, affected_partitions, stream_index_id)
+      partitions = affected_partitions.to_h { [_1.event_type, _1] }
+      values = raw_events.map do |event_attrs|
+        values = [
+          event_attrs['global_position'],
+          event_attrs['stream_revision'],
+          partitions[event_attrs['type']].parent_context_partition_id,
+          partitions[event_attrs['type']].parent_stream_name_partition_id,
+          partitions[event_attrs['type']].id,
+          stream_index_id,
+        ]
+        "(#{values.join(', ')})"
+      end
+      values = values.join(',')
+
+      result = @query_strategy.exec(<<~SQL)
+        INSERT INTO events_global_index
+          ("global_position", "stream_revision", "context_partition_id", "stream_name_partition_id",
+           "event_type_partition_id", "streams_global_index_id")
+        VALUES #{values}
+        RETURNING global_position, stream_revision, event_type_partition_id
+      SQL
+      # "returning" statement has no guarantees about the order in which rows are returned. Thus, sort them explicitly
+      deserialize(result, repr: EventGlobalIndex::ReprType::WRITE_API).sort_by(&:stream_revision)
+    end
+
+    # @param grouped_opts [Hash]
+    # @return [Hash<Integer, Array<PgEventstore::SubscriptionRepr>>]
+    def fetch_indexes_for_subscriptions(grouped_opts)
+      builders = grouped_opts.map do |subscription_id, opts|
+        filter = QueryBuilders::SubscriptionEventsFiltering.build(subscription_id, opts)
+        filter.to_sql_builder.select("#{subscription_id} as subscription_id")
+      end
+      final_builder = SQLBuilder.union_builders(builders)
+      res = @query_strategy.exec_params(*final_builder.to_exec_params)
+      res = res.group_by { _1['subscription_id'] }
+      res.to_h do |sid, indexes|
+        [
+          sid,
+          indexes.map do |attrs|
+            EventGlobalIndex::SubscriptionRepr.new(**attrs.except('subscription_id').transform_keys(&:to_sym))
+          end,
+        ]
+      end
+    end
+
+    # @param indexes [Array<EventGlobalIndex::ReadApiRepr>, Array<EventGlobalIndex::SubscriptionRepr>]
+    # @param resolve_link_tos [Boolean]
+    # @param direction [String, Symbol, nil]
+    # @return [Array<Hash>]
+    def resolve_indexes(indexes, resolve_link_tos:, direction: nil)
+      indexes = indexes.group_by(&:event_type_partition_id)
+      partitions = partition_queries.find_by_ids(indexes.keys).to_h { [_1['id'], _1] }
+      builders = indexes.map do |partition_id, idxs|
+        partition = partitions[partition_id]
+        events_filtering = QueryBuilders::EventsFiltering.new
+        events_filtering.add_stream_attrs(context: partition['context'], stream_name: partition['stream_name'])
+        events_filtering.add_event_types([partition['event_type']])
+        builder = events_filtering.to_sql_builder
+        builder.where('global_position = ANY(?)', idxs.map(&:global_position))
+      end
+      main_builder = SQLBuilder.union_builders(builders)
+      with_sorted_events = SQLBuilder.new.select('*').from(main_builder)
+      if direction
+        with_sorted_events.order("global_position #{QueryBuilders::BasicFiltering::SQL_DIRECTIONS[direction]}")
+      end
+      raw_events = @query_strategy.exec_params(*with_sorted_events.to_exec_params).to_a
+      resolve_link_tos ? links_resolver.resolve(raw_events) : raw_events
+    end
+
+    # @return [Integer, nil]
+    def max_global_position
+      builder = SQLBuilder.new.from(QueryBuilders::EventsGlobalIndexFiltering::PRIMARY_TABLE_NAME)
+      builder.select('max(global_position) as max_global_position')
+      @query_strategy.exec_params(*builder.to_exec_params).first['max_global_position']
+    end
+
+    # Takes an array of potentially persisted events and loads their ids from db. Those ids can be later used to check
+    # whether events are actually existing events.
+    # @param events [Array<PgEventstore::Event>]
+    # @return [Array<Integer>]
+    def global_positions_from_db(events)
+      builder = SQLBuilder.new.from(QueryBuilders::EventsGlobalIndexFiltering::PRIMARY_TABLE_NAME)
+      builder.where('global_position = ANY(?::bigint[])', events.map(&:global_position))
+      @query_strategy.exec_params(*builder.to_exec_params).map { _1['global_position'] }
+    end
+
+    private
+
+    # @return [PgEventstore::PartitionQueries]
+    def partition_queries
+      PartitionQueries.new(connection)
+    end
+
+    # @return [PgEventstore::LinksResolver]
+    def links_resolver
+      LinksResolver.new(connection, @query_strategy)
+    end
+
+    # @param repr [Symbol, nil]
+    # @return [Array<PgEventstore::EventGlobalIndex>, Array<PgEventstore::EventGlobalIndex::SubscriptionRepr>,
+    #   Array<PgEventstore::EventGlobalIndex::ReadApiRepr>, Array<PgEventstore::EventGlobalIndex::WriteApiRepr>,
+    #   Array<PgEventstore::EventGlobalIndex::RevisionCheckRepr>]
+    def deserialize(pg_result, repr: nil)
+      pg_result.map do |attrs|
+        EventGlobalIndex.create_representation(attrs.transform_keys(&:to_sym), repr:)
+      end
+    end
+  end
+end