pg_eventstore 0.9.0 → 0.10.2

data/lib/pg_eventstore/queries/partition_queries.rb

@@ -0,0 +1,184 @@
+# frozen_string_literal: true
+
+module PgEventstore
+  # @!visibility private
+  class PartitionQueries
+    attr_reader :connection
+    private :connection
+
+    # @param connection [PgEventstore::Connection]
+    def initialize(connection)
+      @connection = connection
+    end
+
+    # @param stream [PgEventstore::Stream]
+    # @return [Hash] partition attributes
+    def create_context_partition(stream)
+      attributes = { context: stream.context, table_name: context_partition_name(stream) }
+
+      loop do
+        break unless partition_name_taken?(attributes[:table_name])
+
+        attributes[:table_name] = attributes[:table_name].next
+      end
+
+      partition_sql = <<~SQL
+        INSERT INTO partitions (#{attributes.keys.join(', ')})
+          VALUES (#{Utils.positional_vars(attributes.values)}) RETURNING *
+      SQL
+      partition = connection.with do |conn|
+        conn.exec_params(partition_sql, [*attributes.values])
+      end.to_a.first
+      connection.with do |conn|
+        conn.exec(<<~SQL)
+          CREATE TABLE #{attributes[:table_name]} PARTITION OF events
+            FOR VALUES IN('#{conn.escape_string(stream.context)}') PARTITION BY LIST (stream_name)
+        SQL
+      end
+      partition
+    end
+
+    # @param stream [PgEventstore::Stream]
+    # @param context_partition_name [String]
+    # @return [Hash] partition attributes
+    def create_stream_name_partition(stream, context_partition_name)
+      attributes = {
+        context: stream.context, stream_name: stream.stream_name, table_name: stream_name_partition_name(stream)
+      }
+
+      loop do
+        break unless partition_name_taken?(attributes[:table_name])
+
+        attributes[:table_name] = attributes[:table_name].next
+      end
+
+      partition_sql = <<~SQL
+        INSERT INTO partitions (#{attributes.keys.join(', ')}) 
+          VALUES (#{Utils.positional_vars(attributes.values)}) RETURNING *
+      SQL
+      partition = connection.with do |conn|
+        conn.exec_params(partition_sql, [*attributes.values])
+      end.to_a.first
+      connection.with do |conn|
+        conn.exec(<<~SQL)
+          CREATE TABLE #{attributes[:table_name]} PARTITION OF #{context_partition_name}
+            FOR VALUES IN('#{conn.escape_string(stream.stream_name)}') PARTITION BY LIST (type)
+        SQL
+      end
+      partition
+    end
+
+    # @param stream [PgEventstore::Stream]
+    # @param stream_name_partition_name [String]
+    # @return [Hash] partition attributes
+    def create_event_type_partition(stream, event_type, stream_name_partition_name)
+      attributes = {
+        context: stream.context, stream_name: stream.stream_name, event_type: event_type,
+        table_name: event_type_partition_name(stream, event_type)
+      }
+
+      loop do
+        break unless partition_name_taken?(attributes[:table_name])
+
+        attributes[:table_name] = attributes[:table_name].next
+      end
+
+      partition_sql = <<~SQL
+        INSERT INTO partitions (#{attributes.keys.join(', ')}) 
+          VALUES (#{Utils.positional_vars(attributes.values)}) RETURNING *
+      SQL
+      partition = connection.with do |conn|
+        conn.exec_params(partition_sql, [*attributes.values])
+      end.to_a.first
+      connection.with do |conn|
+        conn.exec(<<~SQL)
+          CREATE TABLE #{attributes[:table_name]} PARTITION OF #{stream_name_partition_name}
+            FOR VALUES IN('#{conn.escape_string(event_type)}')
+        SQL
+      end
+      partition
+    end
+
+    # @param stream [PgEventstore::Stream]
+    # @param event_type [String]
+    # @return [Boolean]
+    def partition_required?(stream, event_type)
+      event_type_partition(stream, event_type).nil?
+    end
+
+    # @param stream [PgEventstore::Stream]
+    # @param event_type [String]
+    # @return [void]
+    def create_partitions(stream, event_type)
+      return unless partition_required?(stream, event_type)
+
+      context_partition = context_partition(stream) || create_context_partition(stream)
+      stream_name_partition = stream_name_partition(stream) ||
+        create_stream_name_partition(stream, context_partition['table_name'])
+
+      create_event_type_partition(stream, event_type, stream_name_partition['table_name'])
+    end
+
+    # @param stream [PgEventstore::Stream]
+    # @return [Hash, nil] partition attributes
+    def context_partition(stream)
+      connection.with do |conn|
+        conn.exec_params(
+          'select * from partitions where context = $1 and stream_name is null and event_type is null',
+          [stream.context]
+        )
+      end.first
+    end
+
+    # @param stream [PgEventstore::Stream]
+    # @return [Hash, nil] partition attributes
+    def stream_name_partition(stream)
+      connection.with do |conn|
+        conn.exec_params(
+          <<~SQL,
+            select * from partitions where context = $1 and stream_name = $2 and event_type is null
+          SQL
+          [stream.context, stream.stream_name]
+        )
+      end.first
+    end
+
+    # @param stream [PgEventstore::Stream]
+    # @param event_type [String]
+    # @return [Hash, nil] partition attributes
+    def event_type_partition(stream, event_type)
+      connection.with do |conn|
+        conn.exec_params(
+          <<~SQL,
+            select * from partitions where context = $1 and stream_name = $2 and event_type = $3
+          SQL
+          [stream.context, stream.stream_name, event_type]
+        )
+      end.first
+    end
+
+    # @param table_name [String]
+    # @return [Boolean]
+    def partition_name_taken?(table_name)
+      connection.with do |conn|
+        conn.exec_params('select 1 as exists from partitions where table_name = $1', [table_name])
+      end.to_a.dig(0, 'exists') == 1
+    end
+
+    # @param stream [PgEventstore::Stream]
+    # @return [String]
+    def context_partition_name(stream)
+      "contexts_#{Digest::MD5.hexdigest(stream.context)[0..5]}"
+    end
+
+    # @param stream [PgEventstore::Stream]
+    # @return [String]
+    def stream_name_partition_name(stream)
+      "stream_names_#{Digest::MD5.hexdigest("#{stream.context}-#{stream.stream_name}")[0..5]}"
+    end
+
+    def event_type_partition_name(stream, event_type)
+      "event_types_#{Digest::MD5.hexdigest("#{stream.context}-#{stream.stream_name}-#{event_type}")[0..5]}"
+    end
+  end
+end

data/lib/pg_eventstore/queries/subscription_queries.rb

@@ -73,16 +73,20 @@ module PgEventstore
       deserialize(pg_result.to_a.first)
     end
 
-    # @param query_options [Array<Array<Integer, Hash>>] array of runner ids and query options
-    # @return [Array<Hash>] array of raw events
+    # @param query_options [Hash{Integer => Hash}] runner_id/query options association
+    # @return [Hash{Integer => Hash}] runner_id/events association
     def subscriptions_events(query_options)
-      return [] if query_options.empty?
+      return {} if query_options.empty?
 
       final_builder = union_builders(query_options.map { |id, opts| query_builder(id, opts) })
       raw_events = connection.with do |conn|
         conn.exec_params(*final_builder.to_exec_params)
       end.to_a
-      preloader.preload_related_objects(raw_events)
+      raw_events.group_by { _1['runner_id'] }.to_h do |runner_id, runner_raw_events|
+        next [runner_id, runner_raw_events] unless query_options[runner_id][:resolve_link_tos]
+
+        [runner_id, links_resolver.resolve(runner_raw_events)]
+      end
     end
 
     # @param id [Integer] subscription's id
@@ -128,9 +132,7 @@ module PgEventstore
     # @param options [Hash] query options
     # @return [PgEventstore::SQLBuilder]
     def query_builder(id, options)
-      builder = PgEventstore::QueryBuilders::EventsFiltering.subscriptions_events_filtering(
-        event_type_queries.include_event_types_ids(options)
-      ).to_sql_builder
+      builder = PgEventstore::QueryBuilders::EventsFiltering.subscriptions_events_filtering(options).to_sql_builder
       builder.select("#{id} as runner_id")
     end
 
@@ -147,14 +149,9 @@ module PgEventstore
       TransactionQueries.new(connection)
     end
 
-    # @return [PgEventstore::EventTypeQueries]
-    def event_type_queries
-      EventTypeQueries.new(connection)
-    end
-
-    # @return [PgEventstore::Preloader]
-    def preloader
-      Preloader.new(connection)
+    # @return [PgEventstore::LinksResolver]
+    def links_resolver
+      LinksResolver.new(connection)
     end
 
     # @param hash [Hash]

data/lib/pg_eventstore/queries/transaction_queries.rb

@@ -36,6 +36,19 @@ module PgEventstore
       end
     rescue PG::TRSerializationFailure, PG::TRDeadlockDetected
       retry
+    rescue MissingPartitions => error
+      error.event_types.each do |event_type|
+        transaction do
+          partition_queries.create_partitions(error.stream, event_type)
+        end
+      rescue PG::UniqueViolation
+        retry
+      end
+      retry
+    end
+
+    def partition_queries
+      PartitionQueries.new(connection)
     end
   end
 end

data/lib/pg_eventstore/queries.rb

@@ -4,13 +4,12 @@ require_relative 'sql_builder'
 require_relative 'query_builders/events_filtering_query'
 require_relative 'queries/transaction_queries'
 require_relative 'queries/event_queries'
-require_relative 'queries/stream_queries'
-require_relative 'queries/event_type_queries'
+require_relative 'queries/partition_queries'
 require_relative 'queries/subscription_queries'
 require_relative 'queries/subscriptions_set_queries'
 require_relative 'queries/subscription_command_queries'
 require_relative 'queries/subscriptions_set_command_queries'
-require_relative 'queries/preloader'
+require_relative 'queries/links_resolver'
 
 module PgEventstore
   # @!visibility private
@@ -20,9 +19,9 @@ module PgEventstore
     # @!attribute events
     #   @return [PgEventstore::EventQueries, nil]
     attribute(:events)
-    # @!attribute streams
-    #   @return [PgEventstore::StreamQueries, nil]
-    attribute(:streams)
+    # @!attribute partitions
+    #   @return [PgEventstore::PartitionQueries, nil]
+    attribute(:partitions)
     # @!attribute transactions
     #   @return [PgEventstore::TransactionQueries, nil]
     attribute(:transactions)

data/lib/pg_eventstore/query_builders/events_filtering_query.rb

@@ -28,10 +28,9 @@ module PgEventstore
         # @return [PgEventstore::QueryBuilders::EventsFiltering]
         def all_stream_filtering(options)
           event_filter = new
-          options in { filter: { event_type_ids: Array => event_type_ids } }
+          options in { filter: { event_types: Array => event_type_ids } }
           event_filter.add_event_types(event_type_ids)
           event_filter.add_limit(options[:max_count])
-          event_filter.resolve_links(options[:resolve_link_tos])
           options in { filter: { streams: Array => streams } }
           streams&.each { |attrs| event_filter.add_stream_attrs(**attrs) }
           event_filter.add_global_position(options[:from_position], options[:direction])
@@ -44,11 +43,10 @@ module PgEventstore
         # @return [PgEventstore::QueryBuilders::EventsFiltering]
         def specific_stream_filtering(stream, options)
           event_filter = new
-          options in { filter: { event_type_ids: Array => event_type_ids } }
+          options in { filter: { event_types: Array => event_type_ids } }
           event_filter.add_event_types(event_type_ids)
           event_filter.add_limit(options[:max_count])
-          event_filter.resolve_links(options[:resolve_link_tos])
-          event_filter.add_stream(stream)
+          event_filter.add_stream_attrs(**stream.to_hash)
           event_filter.add_revision(options[:from_revision], options[:direction])
           event_filter.add_stream_direction(options[:direction])
           event_filter
@@ -60,8 +58,6 @@ module PgEventstore
           SQLBuilder.new.
             select('events.*').
             from('events').
-            join('JOIN streams ON streams.id = events.stream_id').
-            join('JOIN event_types ON event_types.id = events.event_type_id').
             limit(DEFAULT_LIMIT)
       end
 
@@ -75,27 +71,21 @@ module PgEventstore
 
         stream_attrs.compact!
         sql = stream_attrs.map do |attr, _|
-          "streams.#{attr} = ?"
+          "events.#{attr} = ?"
         end.join(" AND ")
         @sql_builder.where_or(sql, *stream_attrs.values)
       end
 
-      # @param stream [PgEventstore::Stream]
+      # @param event_types [Array<String>, nil]
       # @return [void]
-      def add_stream(stream)
-        @sql_builder.where("streams.id = ?", stream.id)
-      end
-
-      # @param event_type_ids [Array<Integer>, nil]
-      # @return [void]
-      def add_event_types(event_type_ids)
-        return if event_type_ids.nil?
-        return if event_type_ids.empty?
+      def add_event_types(event_types)
+        return if event_types.nil?
+        return if event_types.empty?
 
-        sql = event_type_ids.size.times.map do
+        sql = event_types.size.times.map do
           "?"
         end.join(", ")
-        @sql_builder.where("events.event_type_id IN (#{sql})", *event_type_ids)
+        @sql_builder.where("events.type IN (#{sql})", *event_types)
       end
 
       # @param revision [Integer, nil]
@@ -136,17 +126,6 @@ module PgEventstore
         @sql_builder.limit(limit)
       end
 
-      # @param should_resolve [Boolean]
-      # @return [void]
-      def resolve_links(should_resolve)
-        return unless should_resolve
-
-        @sql_builder.
-          unselect.
-          select("(COALESCE(original_events.*, events.*)).*").
-          join("LEFT JOIN events original_events ON original_events.id = events.link_id")
-      end
-
       # @return [PgEventstore::SQLBuilder]
       def to_sql_builder
         @sql_builder

data/lib/pg_eventstore/rspec/test_helpers.rb

@@ -4,10 +4,25 @@ module PgEventstore
   module TestHelpers
     class << self
       def clean_up_db
+        clean_up_data
+        clean_up_partitions
+      end
+
+      def clean_up_partitions
+        PgEventstore.connection.with do |conn|
+          # Dropping parent partition also drops all child partitions
+          conn.exec("select tablename from pg_tables where tablename like 'contexts_%'").each do |attrs|
+            conn.exec("drop table #{attrs['tablename']}")
+          end
+        end
+      end
+
+      def clean_up_data
         tables_to_purge = PgEventstore.connection.with do |conn|
           conn.exec(<<~SQL)
             SELECT tablename 
-            FROM pg_catalog.pg_tables WHERE schemaname NOT IN ('pg_catalog', 'information_schema') AND tablename != 'migrations'                
+            FROM pg_catalog.pg_tables 
+            WHERE schemaname NOT IN ('pg_catalog', 'information_schema') AND tablename != 'migrations'
           SQL
         end.map { |attrs| attrs['tablename'] }
         tables_to_purge.each do |table_name|

data/lib/pg_eventstore/sql_builder.rb

@@ -9,6 +9,7 @@ module PgEventstore
       @from_value = nil
       @where_values = { 'AND' => [], 'OR' => [] }
       @join_values = []
+      @group_values = []
       @order_values = []
       @limit_value = nil
       @offset_value = nil
@@ -68,6 +69,11 @@ module PgEventstore
       self
     end
 
+    def remove_order
+      @order_values.clear
+      self
+    end
+
     # @param limit [Integer]
     # @return self
     def limit(limit)
@@ -75,6 +81,11 @@ module PgEventstore
       self
     end
 
+    def remove_limit
+      @limit_value = nil
+      self
+    end
+
     # @param offset [Integer]
     # @return self
     def offset(offset)
@@ -89,10 +100,22 @@ module PgEventstore
       self
     end
 
-    def to_exec_params
-      return [single_query_sql, @positional_values] if @union_values.empty?
+    # @param sql [String]
+    # @return self
+    def group(sql)
+      @group_values.push(sql)
+      self
+    end
 
-      [union_query_sql, @positional_values]
+    def remove_group
+      @group_values.clear
+      self
+    end
+
+    def to_exec_params
+      @positional_values.clear
+      @positional_values_size = 0
+      _to_exec_params
     end
 
     protected
@@ -106,6 +129,12 @@ module PgEventstore
       @positional_values_size = val
     end
 
+    def _to_exec_params
+      return [single_query_sql, @positional_values] if @union_values.empty?
+
+      [union_query_sql, @positional_values]
+    end
+
     private
 
     # @return [String]
@@ -114,6 +143,7 @@ module PgEventstore
       sql = "SELECT #{select_sql} FROM #{@from_value}"
       sql += " #{join_sql}" unless @join_values.empty?
       sql += " WHERE #{where_sql}" unless where_sql.empty?
+      sql += " GROUP BY #{@group_values.join(', ')}" unless @group_values.empty?
       sql += " ORDER BY #{order_sql}" unless @order_values.empty?
       sql += " LIMIT #{@limit_value}" if @limit_value
       sql += " OFFSET #{@offset_value}" if @offset_value
@@ -126,7 +156,7 @@ module PgEventstore
       union_parts = ["(#{sql})"]
       union_parts += @union_values.map do |builder|
         builder.positional_values_size = @positional_values_size
-        builder_sql, values = builder.to_exec_params
+        builder_sql, values = builder._to_exec_params
         @positional_values.push(*values)
         @positional_values_size += values.size
         "(#{builder_sql})"

data/lib/pg_eventstore/stream.rb

@@ -5,7 +5,7 @@ require 'digest/md5'
 module PgEventstore
   class Stream
     SYSTEM_STREAM_PREFIX = '$'
-    INITIAL_STREAM_REVISION = -1 # this is the default value of streams.stream_revision column
+    NON_EXISTING_STREAM_REVISION = -1
 
     class << self
       # Produces "all" stream instance. "all" stream does not represent any specific stream. Instead, it indicates that
@@ -18,20 +18,15 @@ module PgEventstore
       end
     end
 
-    attr_reader :context, :stream_name, :stream_id, :id
-    attr_accessor :stream_revision
+    attr_reader :context, :stream_name, :stream_id
 
     # @param context [String]
     # @param stream_name [String]
     # @param stream_id [String]
-    # @param id [Integer, nil] internal stream's id, read only
-    # @param stream_revision [Integer, nil] current stream revision, read only
-    def initialize(context:, stream_name:, stream_id:, id: nil, stream_revision: nil)
+    def initialize(context:, stream_name:, stream_id:)
       @context = context
       @stream_name = stream_name
       @stream_id = stream_id
-      @id = id
-      @stream_revision = stream_revision
     end
 
     # @return [Boolean]

data/lib/pg_eventstore/subscriptions/events_processor.rb

@@ -20,7 +20,7 @@ module PgEventstore
     # @param raw_events [Array<Hash>]
     # @return [void]
     def feed(raw_events)
-      callbacks.run_callbacks(:feed, raw_events.last&.dig('global_position'))
+      callbacks.run_callbacks(:feed, global_position(raw_events.last))
       @raw_events.push(*raw_events)
     end
 
@@ -35,7 +35,7 @@ module PgEventstore
     # @param raw_event [Hash]
     # @return [void]
     def process_event(raw_event)
-      callbacks.run_callbacks(:process, raw_event['global_position']) do
+      callbacks.run_callbacks(:process, global_position(raw_event)) do
         @handler.call(raw_event)
       end
     end
@@ -68,5 +68,13 @@ module PgEventstore
     def change_state(...)
       callbacks.run_callbacks(:change_state, ...)
     end
+
+    # @param raw_event [Hash, nil]
+    # @return [Integer, nil]
+    def global_position(raw_event)
+      return unless raw_event
+
+      raw_event['link'] ? raw_event['link']['global_position'] : raw_event['global_position']
+    end
   end
 end

data/lib/pg_eventstore/subscriptions/subscription.rb

@@ -130,6 +130,7 @@ module PgEventstore
         chunk_query_interval: chunk_query_interval,
         last_chunk_fed_at: Time.at(0).utc,
         last_chunk_greatest_position: nil,
+        time_between_restarts: time_between_restarts,
         state: RunnerState::STATES[:initial]
       )
     end

data/lib/pg_eventstore/subscriptions/subscription_feeder.rb

@@ -21,7 +21,7 @@ module PgEventstore
       @max_retries = max_retries
       @retries_interval = retries_interval
       @commands_handler = CommandsHandler.new(@config_name, self, @runners)
-      @basic_runner = BasicRunner.new(1, 0)
+      @basic_runner = BasicRunner.new(0.2, 0)
       @force_lock = false
       attach_runner_callbacks
     end

data/lib/pg_eventstore/subscriptions/subscription_runners_feeder.rb

@@ -15,9 +15,8 @@ module PgEventstore
       runners = runners.select(&:running?).select(&:time_to_feed?)
       return if runners.empty?
 
-      runners_query_options = runners.map { |runner| [runner.id, runner.next_chunk_query_opts] }
-      raw_events = subscription_queries.subscriptions_events(runners_query_options)
-      grouped_events = raw_events.group_by { |attrs| attrs['runner_id'] }
+      runners_query_options = runners.to_h { |runner| [runner.id, runner.next_chunk_query_opts] }
+      grouped_events = subscription_queries.subscriptions_events(runners_query_options)
       runners.each do |runner|
         runner.feed(grouped_events[runner.id]) if grouped_events[runner.id]
       end

data/lib/pg_eventstore/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PgEventstore
-  VERSION = "0.9.0"
+  VERSION = "0.10.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pg_eventstore
 version: !ruby/object:Gem::Version
-  version: 0.9.0
+  version: 0.10.2
 platform: ruby
 authors:
 - Ivan Dzyzenko
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-02-23 00:00:00.000000000 Z
+date: 2024-03-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
@@ -50,15 +50,15 @@ files:
 - LICENSE.txt
 - README.md
 - db/migrations/0_create_extensions.sql
-- db/migrations/1_create_streams.sql
-- db/migrations/2_create_event_types.sql
-- db/migrations/3_create_events.sql
-- db/migrations/4_create_subscriptions.sql
-- db/migrations/5_create_subscription_commands.sql
-- db/migrations/6_create_subscriptions_set_commands.sql
+- db/migrations/1_create_events.sql
+- db/migrations/2_create_subscriptions.sql
+- db/migrations/3_create_subscription_commands.sql
+- db/migrations/4_create_subscriptions_set_commands.sql
+- db/migrations/5_partitions.sql
 - docs/appending_events.md
 - docs/configuration.md
 - docs/events_and_streams.md
+- docs/how_it_works.md
 - docs/linking_events.md
 - docs/multiple_commands.md
 - docs/reading_events.md
@@ -91,9 +91,8 @@ files:
 - lib/pg_eventstore/pg_connection.rb
 - lib/pg_eventstore/queries.rb
 - lib/pg_eventstore/queries/event_queries.rb
-- lib/pg_eventstore/queries/event_type_queries.rb
-- lib/pg_eventstore/queries/preloader.rb
-- lib/pg_eventstore/queries/stream_queries.rb
+- lib/pg_eventstore/queries/links_resolver.rb
+- lib/pg_eventstore/queries/partition_queries.rb
 - lib/pg_eventstore/queries/subscription_command_queries.rb
 - lib/pg_eventstore/queries/subscription_queries.rb
 - lib/pg_eventstore/queries/subscriptions_set_command_queries.rb

data/db/migrations/1_create_streams.sql

@@ -1,13 +0,0 @@
-CREATE TABLE public.streams
-(
-    id              bigserial                         NOT NULL,
-    context         character varying COLLATE "POSIX" NOT NULL,
-    stream_name     character varying COLLATE "POSIX" NOT NULL,
-    stream_id       character varying COLLATE "POSIX" NOT NULL,
-    stream_revision integer DEFAULT '-1'::integer     NOT NULL
-);
-
-ALTER TABLE ONLY public.streams
-    ADD CONSTRAINT streams_pkey PRIMARY KEY (id);
-
-CREATE UNIQUE INDEX idx_streams_context_and_stream_name_and_stream_id ON public.streams USING btree (context, stream_name, stream_id);

data/db/migrations/2_create_event_types.sql

@@ -1,10 +0,0 @@
-CREATE TABLE public.event_types
-(
-    id   bigserial                         NOT NULL,
-    type character varying COLLATE "POSIX" NOT NULL
-);
-
-ALTER TABLE ONLY public.event_types
-    ADD CONSTRAINT event_types_pkey PRIMARY KEY (id);
-
-CREATE UNIQUE INDEX idx_event_types_type ON public.event_types USING btree (type);