pg_eventstore 1.13.4 → 3.0.0

data/db/migrations/16_populate_streams_global_index.rb

@@ -0,0 +1,150 @@
+# frozen_string_literal: true
+
+CONCURRENCY = ENV['CONCURRENCY']&.to_i || 10
+
+PgEventstore.configure(name: :_eventstore_db_connection) do |config|
+  config.connection_pool_size = CONCURRENCY * 10
+end
+
+partitions = PgEventstore.connection(:_eventstore_db_connection).with do |conn|
+  conn.exec('select * from partitions where event_type is null')
+end.to_a
+
+stream_name_partitions = partitions.reject { _1['stream_name'].nil? }.group_by { _1['context'] }
+stream_name_partitions = stream_name_partitions.to_h do |context, context_parts|
+  context_parts = context_parts.to_h { |p| [p['stream_name'], p['id']] }
+  [context, context_parts]
+end
+
+total_streams = PgEventstore.connection(:_eventstore_db_connection).with do |conn|
+  conn.exec('explain select * from "$streams"')
+end.first['QUERY PLAN'][/rows=\d+/].sub('rows=', '').to_i
+
+tables = PgEventstore.connection(:_eventstore_db_connection).with do |conn|
+  conn.exec('select table_name from partitions where event_type is not null')
+end.to_a.map { _1['table_name'] }
+
+PgEventstore.connection(:_eventstore_db_connection).with do |conn|
+  conn.exec('delete from streams_global_index')
+end
+
+puts "Indexing streams. Approximate number of streams to process: #{total_streams}. Concurrency is #{CONCURRENCY} concurrent writers."
+puts <<~TEXT
+  This migration consumes CONCURRENCY x10 connections. Make sure you have plenty of them. Migrations are idempotent, \
+  so you can break, adjust settings if needed and then start over.
+TEXT
+puts
+
+processed = 0
+processed_was = 0
+time = Time.now
+lock = Thread::Mutex.new
+sub_partitions_count_cache = {}
+threads = CONCURRENCY.times.map do |t|
+  Thread.new do
+    tables.each do |table|
+      global_position = 0
+      loop do
+        events = PgEventstore.connection(:_eventstore_db_connection).with do |conn|
+          conn.exec_params(<<~SQL, [global_position, CONCURRENCY, t])
+            select context, stream_name, stream_id, global_position
+            from "#{table}"
+            where global_position > $1 and global_position % $2 = $3 and stream_revision = 0
+            group by context, stream_name, stream_id, global_position
+            order by global_position asc
+            limit 100
+          SQL
+        end.to_a
+        break if events.empty?
+
+        global_position = events.last['global_position']
+
+        # retrieve stream revision
+        events.each_slice(10) do |sliced_events|
+          query_runner = PgEventstore::AsyncRunner.new
+          sliced_events.each do |event|
+            query_runner.async do
+              query_strategy = PgEventstore::QueryStrategy::Async.new(PgEventstore.connection(:_eventstore_db_connection))
+              context = event['context']
+              stream_name = event['stream_name']
+              stream_id = event['stream_id']
+              sub_partitions_count_cache[[context, stream_name]] ||= query_strategy.exec_params(
+                'select count(*) as c_all from partitions where context = $1 and stream_name = $2',
+                [context, stream_name]
+              ).first['c_all']
+              if sub_partitions_count_cache[[context, stream_name]] > 50
+                sub_partitions = query_strategy.exec_params(<<~SQL, [context, stream_name]).map { _1['table_name'] }
+                  select table_name from partitions where context = $1 and stream_name = $2 and event_type is not null
+                SQL
+                max_revisions = []
+                sub_partitions.each_slice(50) do |table_names|
+                  q_parts = table_names.map do |table_name|
+                    <<~SQL
+                      select max(stream_revision) as stream_revision
+                      from #{table_name}
+                      where context = $1 and stream_name = $2 and stream_id = $3
+                    SQL
+                  end
+                  q_parts = q_parts.map { "(#{_1})" }.join(' union all ')
+                  q = "select coalesce(max(stream_revision), -1) as stream_revision from (#{q_parts})"
+                  max_revisions.push(
+                    query_strategy.exec_params(q, [context, stream_name, stream_id]).first['stream_revision']
+                  )
+                end
+                event['stream_revision'] = max_revisions.max
+              else
+                q = <<~SQL
+                  select max(stream_revision) as stream_revision
+                  from events
+                  where context = $1 and stream_name = $2 and stream_id = $3
+                SQL
+                attrs = query_strategy.exec_params(q, [context, stream_name, stream_id]).to_a.first
+                event['stream_revision'] = attrs['stream_revision']
+              end
+            end
+          end
+          query_runner.run
+        end
+
+        values = events.map do |event|
+          partition_id = stream_name_partitions[event['context']][event['stream_name']]
+          stream_id = PG::Connection.escape(event['stream_id'])
+          stream_revision = event['stream_revision']
+          starting_position = event['global_position']
+          "(#{partition_id}, '#{stream_id}', #{stream_revision}, #{starting_position})"
+        end.join(',')
+
+        PgEventstore.connection(:_eventstore_db_connection).with do |conn|
+          conn.exec(<<~SQL)
+            INSERT INTO streams_global_index
+              ("partition_id", "stream_id", "stream_revision", "starting_position")
+            VALUES #{values}
+          SQL
+        end
+
+        lock.synchronize do
+          processed += events.size
+          next if Time.now - time < 2
+
+          time_was = time
+          time = Time.now
+
+          performance_info = <<~TEXT.strip
+            Processed: #{processed}. Left: #{total_streams - processed} (approximately). \
+            Performance: #{((processed - processed_was) / (time - time_was)).round(2)} streams/second.
+          TEXT
+          processed_was = processed
+          print "#{performance_info}               \r"
+        end
+      end
+    end
+  end
+end
+threads.each(&:join)
+
+PgEventstore.connection(:_eventstore_db_connection).with do |conn|
+  puts 'Running cluster on streams_global_index. This may take some time.'
+  conn.exec('CLUSTER streams_global_index USING idx_streams_global_index_on_starting_position')
+  puts 'Running vacuum on streams_global_index. This may take some time.'
+  conn.exec('VACUUM (ANALYZE) streams_global_index')
+end

data/db/migrations/17_create_events_global_index.sql

@@ -0,0 +1,62 @@
+CREATE TABLE public.events_global_index
+(
+    global_position          bigint NOT NULL,
+    stream_revision          bigint NOT NULL,
+    context_partition_id     bigint NOT NULL,
+    stream_name_partition_id bigint NOT NULL,
+    event_type_partition_id  bigint NOT NULL,
+    streams_global_index_id  bigint NOT NULL
+);
+
+CREATE INDEX idx_events_idx_on_global_position ON public.events_global_index
+    USING btree (global_position) INCLUDE (event_type_partition_id);
+
+-- (context, global_position)
+CREATE INDEX idx_events_idx_on_ctx_part_id_N_position ON public.events_global_index
+    USING btree (context_partition_id, global_position) INCLUDE (event_type_partition_id);
+
+-- (context, stream_name, global_position)
+CREATE INDEX idx_events_idx_on_stream_name_part_id_N_position ON public.events_global_index
+    USING btree (stream_name_partition_id, global_position) INCLUDE (event_type_partition_id);
+
+-- (context, stream_name, event_type, global_position)
+CREATE INDEX idx_events_idx_on_e_type_part_id_N_position ON public.events_global_index
+    USING btree (event_type_partition_id, global_position);
+
+-- (context, stream_name, stream_id, global_position)
+-- Including stream_revision keeps us from creating another index to cover event markers queries
+CREATE INDEX idx_events_idx_on_streams_idx_id_N_position ON public.events_global_index
+    USING btree (streams_global_index_id, global_position) INCLUDE (event_type_partition_id, stream_revision);
+
+-- (context, stream_name, stream_id, stream_revision)
+CREATE INDEX idx_events_idx_on_streams_idx_id_N_revision ON public.events_global_index
+    USING btree (streams_global_index_id, stream_revision) INCLUDE (event_type_partition_id, global_position);
+
+-- (context, stream_name, event_type, stream_id, global_position)
+-- Including stream_revision keeps us from creating another index to cover event markers queries
+CREATE INDEX idx_events_idx_on_streams_idx_id_N_e_type_part_id_n_position ON public.events_global_index
+    USING btree (streams_global_index_id, event_type_partition_id, global_position) INCLUDE (stream_revision);
+
+-- (context, stream_name, event_type, stream_id, stream_revision)
+CREATE INDEX idx_events_idx_on_streams_idx_id_N_e_type_part_id_n_revision ON public.events_global_index
+    USING btree (streams_global_index_id, event_type_partition_id, stream_revision) INCLUDE (global_position);
+
+CREATE TABLE public.event_subscription_positions
+(
+    global_position       bigint    NOT NULL,
+    subscription_position bigserial NOT NULL
+);
+
+CREATE TABLE public.event_subscription_positions_unprocessed
+(
+    global_position bigint NOT NULL
+);
+
+CREATE INDEX idx_event_subscription_positions_sposition_N_gposition ON public.event_subscription_positions
+    USING btree (subscription_position, global_position);
+
+CREATE INDEX idx_event_subscription_positions_gposition_N_sposition ON public.event_subscription_positions
+    USING btree (global_position, subscription_position);
+
+CREATE INDEX idx_event_subscription_positions_unprocessed_gposition ON public.event_subscription_positions_unprocessed
+    USING btree (global_position);

data/db/migrations/18_populate_events_global_index.rb

@@ -0,0 +1,124 @@
+# frozen_string_literal: true
+
+CONCURRENCY = ENV['CONCURRENCY']&.to_i || 10
+
+PgEventstore.configure(name: :_eventstore_db_connection) do |config|
+  config.connection_pool_size = CONCURRENCY
+end
+
+partitions = PgEventstore.connection(:_eventstore_db_connection).with do |conn|
+  conn.exec('select * from partitions')
+end.to_a
+context_partitions = partitions.select { _1['stream_name'].nil? }.to_h { [_1['context'], _1['id']] }
+event_type_partitions = partitions.reject { _1['stream_name'].nil? }.group_by { _1['context'] }
+event_type_partitions = event_type_partitions.map do |context, context_parts|
+  context_parts = context_parts.group_by { |p| p['stream_name'] }
+  context_parts = context_parts.map do |stream_name, parts|
+    stream_name_part = parts.find { _1['event_type'].nil? }
+    event_type_parts = parts - [stream_name_part]
+    event_type_parts = event_type_parts.map do |part|
+      [part['event_type'], part]
+    end
+    [stream_name, { stream_name_part:, event_types: event_type_parts.to_h }]
+  end
+  [context, context_parts.to_h]
+end.to_h
+
+total_events = PgEventstore.connection(:_eventstore_db_connection).with do |conn|
+  conn.exec('explain select * from events')
+end.first['QUERY PLAN'][/rows=\d+/].sub('rows=', '').to_i
+
+PgEventstore.connection(:_eventstore_db_connection).with do |conn|
+  conn.exec('delete from events_global_index')
+  conn.exec('delete from event_subscription_positions_unprocessed')
+end
+
+puts "Indexing events. Events to process: #{total_events} (approximately). Concurrency is #{CONCURRENCY} concurrent writers."
+puts
+processed = 0
+processed_was = 0
+time = Time.now
+lock = Thread::Mutex.new
+threads = CONCURRENCY.times.map do |t|
+  Thread.new do
+    event_type_partitions.each_value do |stream_name_parts|
+      stream_name_parts.each_value do |partitions_info|
+        event_type_parts = partitions_info[:event_types]
+        stream_name_part = partitions_info[:stream_name_part]
+        event_type_parts.each_value do |event_type_part|
+          global_position = 0
+          loop do
+            events = PgEventstore.connection(:_eventstore_db_connection).with do |conn|
+              conn.exec_params(<<~SQL, [global_position, CONCURRENCY, t])
+                select global_position, context, stream_name, type, stream_id, stream_revision
+                from #{event_type_part['table_name']}
+                where global_position > $1 and global_position % $2 = $3
+                order by global_position asc
+                limit 1_000
+              SQL
+            end.to_a
+            break if events.empty?
+
+            global_position = events.last['global_position']
+            stream_builders = events.map do |event|
+              sql_builder = PgEventstore::SQLBuilder.new.from('streams_global_index')
+              sql_builder.select(%( id, #{event['global_position']} as event_global_position ))
+              sql_builder.where(
+                'stream_id = ? and partition_id = ?',
+                event['stream_id'], stream_name_part['id']
+              )
+            end
+            final_streams_builder = PgEventstore::SQLBuilder.union_builders(stream_builders)
+            stream_ids = PgEventstore.connection(:_eventstore_db_connection).with do |conn|
+              conn.exec_params(*final_streams_builder.to_exec_params)
+            end.to_a
+            stream_ids = stream_ids.to_h { [_1['event_global_position'], _1['id']] }
+
+            events_idx_values = []
+            global_positions = []
+            events.each do |event|
+              stream_idx_id = stream_ids[event['global_position']]
+              ctx_partition_id = context_partitions[event['context']]
+              stream_name_partition_id = stream_name_part['id']
+              event_type_partition_id = event_type_part['id']
+              events_idx_values << "(#{event['global_position']}, #{ctx_partition_id}, #{stream_name_partition_id}, #{event_type_partition_id}, #{stream_idx_id}, #{event['stream_revision']})"
+              global_positions << "(#{event['global_position']})"
+            end
+
+            PgEventstore.connection(:_eventstore_db_connection).with do |conn|
+              conn.exec(<<~SQL)
+                INSERT INTO events_global_index ("global_position", "context_partition_id", "stream_name_partition_id", "event_type_partition_id", "streams_global_index_id", "stream_revision") VALUES #{events_idx_values.join(',')};
+                INSERT INTO event_subscription_positions_unprocessed ("global_position") VALUES #{global_positions.join(',')};
+              SQL
+            end
+
+            lock.synchronize do
+              processed += events.size
+              next if Time.now - time < 2
+
+              time_was = time
+              time = Time.now
+
+              performance_info = <<~TEXT.strip
+                Processed: #{processed}. Left: #{total_events - processed}(approximately). \
+                Performance: #{((processed - processed_was) / (time - time_was)).round(2)} events/second.
+              TEXT
+              processed_was = processed
+              print "#{performance_info}               \r"
+            end
+          end
+        end
+      end
+    end
+  end
+end
+threads.each(&:join)
+
+puts
+
+PgEventstore.connection(:_eventstore_db_connection).with do |conn|
+  puts 'Running cluster on events_global_index. This may take some time.'
+  conn.exec('CLUSTER events_global_index USING idx_events_idx_on_global_position')
+  puts 'Running vacuum on events_global_index. This may take some time.'
+  conn.exec('VACUUM (ANALYZE) events_global_index')
+end

data/db/migrations/19_populate_events_global_index_p2.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+event_subscription_position_queries = PgEventstore::EventSubscriptionPositionQueries.new(
+  PgEventstore.connection(:_eventstore_db_connection)
+)
+
+total_count = PgEventstore.connection(:_eventstore_db_connection).with do |conn|
+  conn.exec('select count(*) as c_all from event_subscription_positions_unprocessed')
+end.first['c_all'] || 0
+processed = 0
+puts "Assigning event subscription positions. Total count: #{total_count} events."
+
+loop do
+  updated_num = event_subscription_position_queries.assign_subscription_position
+  break if updated_num == 0
+
+  processed += updated_num
+  print "Processed: #{processed}. Left: #{total_count - processed}                    \r"
+end
+
+puts
+PgEventstore.connection(:_eventstore_db_connection).with do |conn|
+  puts 'Running vacuum on event_subscription_positions_unprocessed. This may take some time.'
+  conn.exec('VACUUM (ANALYZE) event_subscription_positions_unprocessed')
+end

data/db/migrations/20_migrate_subscriptions_current_position.sql

@@ -0,0 +1,19 @@
+update subscriptions
+set current_position = (select subscription_position
+                        from event_subscription_positions
+                        where global_position <= subscriptions.current_position
+                        order by global_position desc
+                        limit 1)
+where current_position is not null;
+
+update subscriptions
+set options = jsonb_set(
+    options,
+    '{from_position}',
+    to_jsonb(coalesce((select subscription_position
+                       from event_subscription_positions
+                       where global_position <= (subscriptions.options ->> 'from_position')::bigint
+                       order by global_position desc
+                       limit 1), 0))
+)
+where options ? 'from_position';

data/db/migrations/21_create_partitions_stats.sql

@@ -0,0 +1,2 @@
+CREATE STATISTICS partition_parts_dep (dependencies) on context, stream_name, event_type FROM public.partitions;
+ANALYZE public.partitions;

data/db/migrations/22_drop_streams_sql_view.sql

@@ -0,0 +1,2 @@
+DROP INDEX IF EXISTS idx_events_0_stream_revision_global_position;
+DROP VIEW IF EXISTS "$streams";

data/db/migrations/23_adjust_partitions_indexes.sql

@@ -0,0 +1,5 @@
+DROP INDEX idx_partitions_by_event_type;
+
+CREATE INDEX idx_partitions_by_event_type_and_id ON public.partitions USING btree (event_type, id);
+CREATE INDEX idx_partitions_by_context_and_stream_name_and_event_type_and_id ON public.partitions
+    USING btree (context, stream_name, event_type, id);

data/db/migrations/24_drop_legacy_events_indexes.sql

@@ -0,0 +1,3 @@
+DROP INDEX IF EXISTS idx_events_stream_id_and_stream_revision;
+DROP INDEX IF EXISTS idx_events_stream_id_and_global_position;
+ALTER TABLE public.events DROP CONSTRAINT IF EXISTS events_pkey;

data/db/migrations/25_add_parent_partitions_info_to_partitions.sql

@@ -0,0 +1,16 @@
+ALTER TABLE public.partitions
+    ADD COLUMN parent_stream_name_partition_id bigint;
+ALTER TABLE public.partitions
+    ADD COLUMN parent_context_partition_id bigint;
+
+UPDATE partitions p
+SET parent_context_partition_id = (SELECT id
+                                   FROM partitions
+                                   WHERE context = p.context AND stream_name IS NULL AND event_type IS NULL)
+WHERE stream_name IS NOT NULL;
+
+UPDATE partitions p
+SET parent_stream_name_partition_id = (SELECT id
+                                       FROM partitions
+                                       WHERE context = p.context AND stream_name = p.stream_name AND event_type IS NULL)
+WHERE event_type IS NOT NULL;

data/db/migrations/26_change_events_stream_revision_type.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+puts <<~TEXT
+  Changing stream_revision from int to bigint. In order to do so we need to detach each event type partition, change \
+  stream_revision column type and attach the partition again to the partent table.
+TEXT
+
+filters = PgEventstore::QueryBuilders::Filters::Collection.new
+partition_queries = PgEventstore::PartitionQueries.new(PgEventstore.connection(:_eventstore_db_connection))
+
+event_type_partitions = partition_queries.partitions(filters)
+
+PgEventstore.connection(:_eventstore_db_connection).with do |conn|
+  event_type_partitions.each do |partition|
+    begin
+      partition_queries.detach_event_type_partition(partition)
+    rescue PG::Error
+      # the partition was already detached in previous runs. Just proceed to the next step
+    end
+
+    puts "Starting changing stream_revision from int to bigint for #{partition.event_type.inspect} event type partition."
+    conn.exec("ALTER TABLE #{partition.table_name} ALTER COLUMN stream_revision TYPE bigint")
+  end
+
+  conn.exec('ALTER TABLE events ALTER COLUMN stream_revision TYPE bigint')
+
+  event_type_partitions.each do |partition|
+    partition_queries.attach_event_type_partition(partition)
+  end
+end
+
+puts 'Running vacuum on events.'
+PgEventstore.connection(:_eventstore_db_connection).with do |conn|
+  conn.exec('VACUUM (ANALYZE) events')
+end

data/db/migrations/27_drop_events_horizon.sql

@@ -0,0 +1,3 @@
+DROP TRIGGER log_events_horizon ON events;
+DROP FUNCTION log_events_horizon;
+DROP TABLE events_horizon;

data/db/migrations/28_create_maintenance_tasks.sql

@@ -0,0 +1,8 @@
+CREATE TABLE public.maintenance_tasks
+(
+    task_name    character varying COLLATE "POSIX" NOT NULL,
+    locked_at    timestamp without time zone,
+    performed_at timestamp without time zone
+);
+
+CREATE UNIQUE INDEX idx_maintenance_tasks_task_name ON maintenance_tasks USING btree (task_name);

data/db/migrations/29_create_event_markers.sql

@@ -0,0 +1,35 @@
+CREATE TABLE public.event_markers
+(
+    id   bigserial NOT NULL,
+    name text      NOT NULL
+);
+
+CREATE UNIQUE INDEX idx_event_markers_on_name ON public.event_markers
+    USING btree (name) INCLUDE (id);
+
+CREATE INDEX idx_event_markers_on_id ON public.event_markers
+    USING btree (id) INCLUDE (name);
+
+CREATE TABLE public.event_markers_index
+(
+    marker_id               bigint NOT NULL,
+    streams_global_index_id bigint NOT NULL,
+    event_type_partition_id bigint NOT NULL,
+    global_position         bigint NOT NULL,
+    stream_revision         bigint NOT NULL
+);
+
+CREATE INDEX idx_event_markers_index_on_marker_N_stream_N_rev ON public.event_markers_index
+    USING btree (marker_id, streams_global_index_id, stream_revision) INCLUDE (global_position, event_type_partition_id);
+
+CREATE INDEX idx_event_markers_index_on_marker_N_stream_N_partition_N_rev ON public.event_markers_index
+    USING btree (marker_id, streams_global_index_id, event_type_partition_id, stream_revision) INCLUDE (global_position);
+
+CREATE INDEX idx_event_markers_index_on_marker_N_pos ON public.event_markers_index
+    USING btree (marker_id, global_position) INCLUDE (event_type_partition_id);
+
+CREATE INDEX idx_event_markers_index_on_marker_N_partition_N_pos ON public.event_markers_index
+    USING btree (marker_id, event_type_partition_id, global_position);
+
+CREATE INDEX idx_event_markers_index_on_pos ON public.event_markers_index
+    USING btree (global_position);

data/db/migrations/30_improve_partitions_search_in_admin_ui.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+PgEventstore.connection(:_eventstore_db_connection).with do |conn|
+  conn.transaction do
+    conn.exec(<<~SQL)
+      CREATE EXTENSION IF NOT EXISTS pg_trgm;
+    SQL
+    conn.exec(<<~SQL)
+      CREATE INDEX idx_partitions_context_and_stream_name_search ON public.partitions
+        USING GIN ((context || '#{PgEventstore::Event::SYSTEM_SYMBOL}' || stream_name) gin_trgm_ops)
+        WHERE stream_name IS NOT NULL AND event_type IS NULL;
+    SQL
+    conn.exec(<<~SQL)
+      CREATE INDEX idx_partitions_context_search ON public.partitions USING GIN (context gin_trgm_ops)
+        WHERE stream_name IS NULL and event_type IS NULL;
+      CREATE INDEX idx_partitions_event_type_search ON public.partitions USING GIN (event_type gin_trgm_ops)
+        WHERE event_type IS NOT NULL;
+
+      COMMENT ON INDEX idx_partitions_context_search IS 'Admin web UI search support.';
+      COMMENT ON INDEX idx_partitions_context_and_stream_name_search IS 'Admin web UI search support.';
+      COMMENT ON INDEX idx_partitions_event_type_search IS 'Admin web UI search support.';
+    SQL
+  end
+end

data/db/migrations/31_drop_events_id_default_value.sql

@@ -0,0 +1 @@
+ALTER TABLE public.events ALTER COLUMN id DROP DEFAULT;

data/db/migrations/32_create_parittions_triggers.sql

@@ -0,0 +1,101 @@
+CREATE FUNCTION public.create_context_partition_table()
+RETURNS trigger
+LANGUAGE plpgsql
+AS $$
+BEGIN
+    EXECUTE format(
+        'CREATE TABLE public.%I PARTITION OF public.events FOR VALUES IN (%L) PARTITION BY LIST (stream_name)',
+        NEW.table_name,
+        NEW.context
+    );
+
+    EXECUTE format(
+        'COMMENT ON TABLE public.%I IS %L',
+        NEW.table_name,
+        format('''%s'' context partition', NEW.context)
+    );
+
+    RETURN NEW;
+END;
+$$;
+
+CREATE FUNCTION public.create_stream_name_partition_table()
+RETURNS trigger
+LANGUAGE plpgsql
+AS $$
+DECLARE
+    context_partition_name public.partitions.table_name%TYPE;
+BEGIN
+    SELECT table_name
+    INTO STRICT context_partition_name
+    FROM public.partitions
+    WHERE id = NEW.parent_context_partition_id;
+
+    EXECUTE format(
+        'CREATE TABLE public.%I PARTITION OF public.%I FOR VALUES IN (%L) PARTITION BY LIST (type)',
+        NEW.table_name,
+        context_partition_name,
+        NEW.stream_name
+    );
+
+    EXECUTE format(
+        'COMMENT ON TABLE public.%I IS %L',
+        NEW.table_name,
+        format('''%s'' context and ''%s'' stream name partition', NEW.context, NEW.stream_name)
+    );
+
+    RETURN NEW;
+END;
+$$;
+
+CREATE FUNCTION public.create_event_type_partition_table()
+RETURNS trigger
+LANGUAGE plpgsql
+AS $$
+DECLARE
+    stream_name_partition_name public.partitions.table_name%TYPE;
+BEGIN
+    SELECT table_name
+    INTO STRICT stream_name_partition_name
+    FROM public.partitions
+    WHERE id = NEW.parent_stream_name_partition_id;
+
+    EXECUTE format(
+        'CREATE TABLE public.%I PARTITION OF public.%I FOR VALUES IN (%L)',
+        NEW.table_name,
+        stream_name_partition_name,
+        NEW.event_type
+    );
+
+    EXECUTE format(
+        'COMMENT ON TABLE public.%I IS %L',
+        NEW.table_name,
+        format(
+            '''%s'' context and ''%s'' stream name and ''%s'' event type partition',
+            NEW.context,
+            NEW.stream_name,
+            NEW.event_type
+        )
+    );
+
+    RETURN NEW;
+END;
+$$;
+
+CREATE TRIGGER create_context_partition_table
+AFTER INSERT ON public.partitions
+FOR EACH ROW
+WHEN (NEW.stream_name IS NULL AND NEW.event_type IS NULL)
+EXECUTE FUNCTION public.create_context_partition_table();
+
+CREATE TRIGGER create_stream_name_partition_table
+AFTER INSERT ON public.partitions
+FOR EACH ROW
+WHEN (NEW.stream_name IS NOT NULL AND NEW.event_type IS NULL)
+EXECUTE FUNCTION public.create_stream_name_partition_table();
+
+CREATE TRIGGER create_event_type_partition_table
+AFTER INSERT ON public.partitions
+FOR EACH ROW
+WHEN (NEW.event_type IS NOT NULL)
+EXECUTE FUNCTION public.create_event_type_partition_table();

data/db/migrations/9_create_events_horizon.sql

@@ -0,0 +1,21 @@
+CREATE UNLOGGED TABLE events_horizon
+(
+    global_position bigint not null,
+    xact_id         xid8 not null default pg_current_xact_id()
+);
+CREATE INDEX idx_xact_id_and_created_at_and_global_position ON events_horizon USING btree(xact_id, global_position);
+COMMENT ON TABLE events_horizon IS 'Internal use only. Data is limited to the PostgreSQL cluster in which it was created. DO NOT INCLUDE ITS DATA INTO YOUR DUMP.';
+
+CREATE OR REPLACE FUNCTION log_events_horizon()
+    RETURNS TRIGGER AS $$
+BEGIN
+    INSERT INTO events_horizon(global_position)
+    VALUES (NEW.global_position);
+    RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE TRIGGER log_events_horizon
+    AFTER INSERT ON events
+    FOR EACH ROW
+EXECUTE FUNCTION log_events_horizon();