pg_eventstore 2.0.0 → 3.0.0

data/db/migrations/17_create_events_global_index.sql

@@ -0,0 +1,62 @@
+CREATE TABLE public.events_global_index
+(
+    global_position          bigint NOT NULL,
+    stream_revision          bigint NOT NULL,
+    context_partition_id     bigint NOT NULL,
+    stream_name_partition_id bigint NOT NULL,
+    event_type_partition_id  bigint NOT NULL,
+    streams_global_index_id  bigint NOT NULL
+);
+
+CREATE INDEX idx_events_idx_on_global_position ON public.events_global_index
+    USING btree (global_position) INCLUDE (event_type_partition_id);
+
+-- (context, global_position)
+CREATE INDEX idx_events_idx_on_ctx_part_id_N_position ON public.events_global_index
+    USING btree (context_partition_id, global_position) INCLUDE (event_type_partition_id);
+
+-- (context, stream_name, global_position)
+CREATE INDEX idx_events_idx_on_stream_name_part_id_N_position ON public.events_global_index
+    USING btree (stream_name_partition_id, global_position) INCLUDE (event_type_partition_id);
+
+-- (context, stream_name, event_type, global_position)
+CREATE INDEX idx_events_idx_on_e_type_part_id_N_position ON public.events_global_index
+    USING btree (event_type_partition_id, global_position);
+
+-- (context, stream_name, stream_id, global_position)
+-- Including stream_revision keeps us from creating another index to cover event markers queries
+CREATE INDEX idx_events_idx_on_streams_idx_id_N_position ON public.events_global_index
+    USING btree (streams_global_index_id, global_position) INCLUDE (event_type_partition_id, stream_revision);
+
+-- (context, stream_name, stream_id, stream_revision)
+CREATE INDEX idx_events_idx_on_streams_idx_id_N_revision ON public.events_global_index
+    USING btree (streams_global_index_id, stream_revision) INCLUDE (event_type_partition_id, global_position);
+
+-- (context, stream_name, event_type, stream_id, global_position)
+-- Including stream_revision keeps us from creating another index to cover event markers queries
+CREATE INDEX idx_events_idx_on_streams_idx_id_N_e_type_part_id_n_position ON public.events_global_index
+    USING btree (streams_global_index_id, event_type_partition_id, global_position) INCLUDE (stream_revision);
+
+-- (context, stream_name, event_type, stream_id, stream_revision)
+CREATE INDEX idx_events_idx_on_streams_idx_id_N_e_type_part_id_n_revision ON public.events_global_index
+    USING btree (streams_global_index_id, event_type_partition_id, stream_revision) INCLUDE (global_position);
+
+CREATE TABLE public.event_subscription_positions
+(
+    global_position       bigint    NOT NULL,
+    subscription_position bigserial NOT NULL
+);
+
+CREATE TABLE public.event_subscription_positions_unprocessed
+(
+    global_position bigint NOT NULL
+);
+
+CREATE INDEX idx_event_subscription_positions_sposition_N_gposition ON public.event_subscription_positions
+    USING btree (subscription_position, global_position);
+
+CREATE INDEX idx_event_subscription_positions_gposition_N_sposition ON public.event_subscription_positions
+    USING btree (global_position, subscription_position);
+
+CREATE INDEX idx_event_subscription_positions_unprocessed_gposition ON public.event_subscription_positions_unprocessed
+    USING btree (global_position);

data/db/migrations/18_populate_events_global_index.rb

@@ -0,0 +1,124 @@
+# frozen_string_literal: true
+
+CONCURRENCY = ENV['CONCURRENCY']&.to_i || 10
+
+PgEventstore.configure(name: :_eventstore_db_connection) do |config|
+  config.connection_pool_size = CONCURRENCY
+end
+
+partitions = PgEventstore.connection(:_eventstore_db_connection).with do |conn|
+  conn.exec('select * from partitions')
+end.to_a
+context_partitions = partitions.select { _1['stream_name'].nil? }.to_h { [_1['context'], _1['id']] }
+event_type_partitions = partitions.reject { _1['stream_name'].nil? }.group_by { _1['context'] }
+event_type_partitions = event_type_partitions.map do |context, context_parts|
+  context_parts = context_parts.group_by { |p| p['stream_name'] }
+  context_parts = context_parts.map do |stream_name, parts|
+    stream_name_part = parts.find { _1['event_type'].nil? }
+    event_type_parts = parts - [stream_name_part]
+    event_type_parts = event_type_parts.map do |part|
+      [part['event_type'], part]
+    end
+    [stream_name, { stream_name_part:, event_types: event_type_parts.to_h }]
+  end
+  [context, context_parts.to_h]
+end.to_h
+
+total_events = PgEventstore.connection(:_eventstore_db_connection).with do |conn|
+  conn.exec('explain select * from events')
+end.first['QUERY PLAN'][/rows=\d+/].sub('rows=', '').to_i
+
+PgEventstore.connection(:_eventstore_db_connection).with do |conn|
+  conn.exec('delete from events_global_index')
+  conn.exec('delete from event_subscription_positions_unprocessed')
+end
+
+puts "Indexing events. Events to process: #{total_events} (approximately). Concurrency is #{CONCURRENCY} concurrent writers."
+puts
+processed = 0
+processed_was = 0
+time = Time.now
+lock = Thread::Mutex.new
+threads = CONCURRENCY.times.map do |t|
+  Thread.new do
+    event_type_partitions.each_value do |stream_name_parts|
+      stream_name_parts.each_value do |partitions_info|
+        event_type_parts = partitions_info[:event_types]
+        stream_name_part = partitions_info[:stream_name_part]
+        event_type_parts.each_value do |event_type_part|
+          global_position = 0
+          loop do
+            events = PgEventstore.connection(:_eventstore_db_connection).with do |conn|
+              conn.exec_params(<<~SQL, [global_position, CONCURRENCY, t])
+                select global_position, context, stream_name, type, stream_id, stream_revision
+                from #{event_type_part['table_name']}
+                where global_position > $1 and global_position % $2 = $3
+                order by global_position asc
+                limit 1_000
+              SQL
+            end.to_a
+            break if events.empty?
+
+            global_position = events.last['global_position']
+            stream_builders = events.map do |event|
+              sql_builder = PgEventstore::SQLBuilder.new.from('streams_global_index')
+              sql_builder.select(%( id, #{event['global_position']} as event_global_position ))
+              sql_builder.where(
+                'stream_id = ? and partition_id = ?',
+                event['stream_id'], stream_name_part['id']
+              )
+            end
+            final_streams_builder = PgEventstore::SQLBuilder.union_builders(stream_builders)
+            stream_ids = PgEventstore.connection(:_eventstore_db_connection).with do |conn|
+              conn.exec_params(*final_streams_builder.to_exec_params)
+            end.to_a
+            stream_ids = stream_ids.to_h { [_1['event_global_position'], _1['id']] }
+
+            events_idx_values = []
+            global_positions = []
+            events.each do |event|
+              stream_idx_id = stream_ids[event['global_position']]
+              ctx_partition_id = context_partitions[event['context']]
+              stream_name_partition_id = stream_name_part['id']
+              event_type_partition_id = event_type_part['id']
+              events_idx_values << "(#{event['global_position']}, #{ctx_partition_id}, #{stream_name_partition_id}, #{event_type_partition_id}, #{stream_idx_id}, #{event['stream_revision']})"
+              global_positions << "(#{event['global_position']})"
+            end
+
+            PgEventstore.connection(:_eventstore_db_connection).with do |conn|
+              conn.exec(<<~SQL)
+                INSERT INTO events_global_index ("global_position", "context_partition_id", "stream_name_partition_id", "event_type_partition_id", "streams_global_index_id", "stream_revision") VALUES #{events_idx_values.join(',')};
+                INSERT INTO event_subscription_positions_unprocessed ("global_position") VALUES #{global_positions.join(',')};
+              SQL
+            end
+
+            lock.synchronize do
+              processed += events.size
+              next if Time.now - time < 2
+
+              time_was = time
+              time = Time.now
+
+              performance_info = <<~TEXT.strip
+                Processed: #{processed}. Left: #{total_events - processed}(approximately). \
+                Performance: #{((processed - processed_was) / (time - time_was)).round(2)} events/second.
+              TEXT
+              processed_was = processed
+              print "#{performance_info}               \r"
+            end
+          end
+        end
+      end
+    end
+  end
+end
+threads.each(&:join)
+
+puts
+
+PgEventstore.connection(:_eventstore_db_connection).with do |conn|
+  puts 'Running cluster on events_global_index. This may take some time.'
+  conn.exec('CLUSTER events_global_index USING idx_events_idx_on_global_position')
+  puts 'Running vacuum on events_global_index. This may take some time.'
+  conn.exec('VACUUM (ANALYZE) events_global_index')
+end

data/db/migrations/19_populate_events_global_index_p2.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+event_subscription_position_queries = PgEventstore::EventSubscriptionPositionQueries.new(
+  PgEventstore.connection(:_eventstore_db_connection)
+)
+
+total_count = PgEventstore.connection(:_eventstore_db_connection).with do |conn|
+  conn.exec('select count(*) as c_all from event_subscription_positions_unprocessed')
+end.first['c_all'] || 0
+processed = 0
+puts "Assigning event subscription positions. Total count: #{total_count} events."
+
+loop do
+  updated_num = event_subscription_position_queries.assign_subscription_position
+  break if updated_num == 0
+
+  processed += updated_num
+  print "Processed: #{processed}. Left: #{total_count - processed}                    \r"
+end
+
+puts
+PgEventstore.connection(:_eventstore_db_connection).with do |conn|
+  puts 'Running vacuum on event_subscription_positions_unprocessed. This may take some time.'
+  conn.exec('VACUUM (ANALYZE) event_subscription_positions_unprocessed')
+end

data/db/migrations/20_migrate_subscriptions_current_position.sql

@@ -0,0 +1,19 @@
+update subscriptions
+set current_position = (select subscription_position
+                        from event_subscription_positions
+                        where global_position <= subscriptions.current_position
+                        order by global_position desc
+                        limit 1)
+where current_position is not null;
+
+update subscriptions
+set options = jsonb_set(
+    options,
+    '{from_position}',
+    to_jsonb(coalesce((select subscription_position
+                       from event_subscription_positions
+                       where global_position <= (subscriptions.options ->> 'from_position')::bigint
+                       order by global_position desc
+                       limit 1), 0))
+)
+where options ? 'from_position';

data/db/migrations/21_create_partitions_stats.sql

@@ -0,0 +1,2 @@
+CREATE STATISTICS partition_parts_dep (dependencies) on context, stream_name, event_type FROM public.partitions;
+ANALYZE public.partitions;

data/db/migrations/22_drop_streams_sql_view.sql

@@ -0,0 +1,2 @@
+DROP INDEX IF EXISTS idx_events_0_stream_revision_global_position;
+DROP VIEW IF EXISTS "$streams";

data/db/migrations/23_adjust_partitions_indexes.sql

@@ -0,0 +1,5 @@
+DROP INDEX idx_partitions_by_event_type;
+
+CREATE INDEX idx_partitions_by_event_type_and_id ON public.partitions USING btree (event_type, id);
+CREATE INDEX idx_partitions_by_context_and_stream_name_and_event_type_and_id ON public.partitions
+    USING btree (context, stream_name, event_type, id);

data/db/migrations/24_drop_legacy_events_indexes.sql

@@ -0,0 +1,3 @@
+DROP INDEX IF EXISTS idx_events_stream_id_and_stream_revision;
+DROP INDEX IF EXISTS idx_events_stream_id_and_global_position;
+ALTER TABLE public.events DROP CONSTRAINT IF EXISTS events_pkey;

data/db/migrations/25_add_parent_partitions_info_to_partitions.sql

@@ -0,0 +1,16 @@
+ALTER TABLE public.partitions
+    ADD COLUMN parent_stream_name_partition_id bigint;
+ALTER TABLE public.partitions
+    ADD COLUMN parent_context_partition_id bigint;
+
+UPDATE partitions p
+SET parent_context_partition_id = (SELECT id
+                                   FROM partitions
+                                   WHERE context = p.context AND stream_name IS NULL AND event_type IS NULL)
+WHERE stream_name IS NOT NULL;
+
+UPDATE partitions p
+SET parent_stream_name_partition_id = (SELECT id
+                                       FROM partitions
+                                       WHERE context = p.context AND stream_name = p.stream_name AND event_type IS NULL)
+WHERE event_type IS NOT NULL;

data/db/migrations/26_change_events_stream_revision_type.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+puts <<~TEXT
+  Changing stream_revision from int to bigint. In order to do so we need to detach each event type partition, change \
+  stream_revision column type and attach the partition again to the partent table.
+TEXT
+
+filters = PgEventstore::QueryBuilders::Filters::Collection.new
+partition_queries = PgEventstore::PartitionQueries.new(PgEventstore.connection(:_eventstore_db_connection))
+
+event_type_partitions = partition_queries.partitions(filters)
+
+PgEventstore.connection(:_eventstore_db_connection).with do |conn|
+  event_type_partitions.each do |partition|
+    begin
+      partition_queries.detach_event_type_partition(partition)
+    rescue PG::Error
+      # the partition was already detached in previous runs. Just proceed to the next step
+    end
+
+    puts "Starting changing stream_revision from int to bigint for #{partition.event_type.inspect} event type partition."
+    conn.exec("ALTER TABLE #{partition.table_name} ALTER COLUMN stream_revision TYPE bigint")
+  end
+
+  conn.exec('ALTER TABLE events ALTER COLUMN stream_revision TYPE bigint')
+
+  event_type_partitions.each do |partition|
+    partition_queries.attach_event_type_partition(partition)
+  end
+end
+
+puts 'Running vacuum on events.'
+PgEventstore.connection(:_eventstore_db_connection).with do |conn|
+  conn.exec('VACUUM (ANALYZE) events')
+end

data/db/migrations/27_drop_events_horizon.sql

@@ -0,0 +1,3 @@
+DROP TRIGGER log_events_horizon ON events;
+DROP FUNCTION log_events_horizon;
+DROP TABLE events_horizon;

data/db/migrations/28_create_maintenance_tasks.sql

@@ -0,0 +1,8 @@
+CREATE TABLE public.maintenance_tasks
+(
+    task_name    character varying COLLATE "POSIX" NOT NULL,
+    locked_at    timestamp without time zone,
+    performed_at timestamp without time zone
+);
+
+CREATE UNIQUE INDEX idx_maintenance_tasks_task_name ON maintenance_tasks USING btree (task_name);

data/db/migrations/29_create_event_markers.sql

@@ -0,0 +1,35 @@
+CREATE TABLE public.event_markers
+(
+    id   bigserial NOT NULL,
+    name text      NOT NULL
+);
+
+CREATE UNIQUE INDEX idx_event_markers_on_name ON public.event_markers
+    USING btree (name) INCLUDE (id);
+
+CREATE INDEX idx_event_markers_on_id ON public.event_markers
+    USING btree (id) INCLUDE (name);
+
+CREATE TABLE public.event_markers_index
+(
+    marker_id               bigint NOT NULL,
+    streams_global_index_id bigint NOT NULL,
+    event_type_partition_id bigint NOT NULL,
+    global_position         bigint NOT NULL,
+    stream_revision         bigint NOT NULL
+);
+
+CREATE INDEX idx_event_markers_index_on_marker_N_stream_N_rev ON public.event_markers_index
+    USING btree (marker_id, streams_global_index_id, stream_revision) INCLUDE (global_position, event_type_partition_id);
+
+CREATE INDEX idx_event_markers_index_on_marker_N_stream_N_partition_N_rev ON public.event_markers_index
+    USING btree (marker_id, streams_global_index_id, event_type_partition_id, stream_revision) INCLUDE (global_position);
+
+CREATE INDEX idx_event_markers_index_on_marker_N_pos ON public.event_markers_index
+    USING btree (marker_id, global_position) INCLUDE (event_type_partition_id);
+
+CREATE INDEX idx_event_markers_index_on_marker_N_partition_N_pos ON public.event_markers_index
+    USING btree (marker_id, event_type_partition_id, global_position);
+
+CREATE INDEX idx_event_markers_index_on_pos ON public.event_markers_index
+    USING btree (global_position);

data/db/migrations/30_improve_partitions_search_in_admin_ui.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+PgEventstore.connection(:_eventstore_db_connection).with do |conn|
+  conn.transaction do
+    conn.exec(<<~SQL)
+      CREATE EXTENSION IF NOT EXISTS pg_trgm;
+    SQL
+    conn.exec(<<~SQL)
+      CREATE INDEX idx_partitions_context_and_stream_name_search ON public.partitions
+        USING GIN ((context || '#{PgEventstore::Event::SYSTEM_SYMBOL}' || stream_name) gin_trgm_ops)
+        WHERE stream_name IS NOT NULL AND event_type IS NULL;
+    SQL
+    conn.exec(<<~SQL)
+      CREATE INDEX idx_partitions_context_search ON public.partitions USING GIN (context gin_trgm_ops)
+        WHERE stream_name IS NULL and event_type IS NULL;
+      CREATE INDEX idx_partitions_event_type_search ON public.partitions USING GIN (event_type gin_trgm_ops)
+        WHERE event_type IS NOT NULL;
+
+      COMMENT ON INDEX idx_partitions_context_search IS 'Admin web UI search support.';
+      COMMENT ON INDEX idx_partitions_context_and_stream_name_search IS 'Admin web UI search support.';
+      COMMENT ON INDEX idx_partitions_event_type_search IS 'Admin web UI search support.';
+    SQL
+  end
+end

data/db/migrations/31_drop_events_id_default_value.sql

@@ -0,0 +1 @@
+ALTER TABLE public.events ALTER COLUMN id DROP DEFAULT;

data/db/migrations/32_create_parittions_triggers.sql

@@ -0,0 +1,101 @@
+CREATE FUNCTION public.create_context_partition_table()
+RETURNS trigger
+LANGUAGE plpgsql
+AS $$
+BEGIN
+    EXECUTE format(
+        'CREATE TABLE public.%I PARTITION OF public.events FOR VALUES IN (%L) PARTITION BY LIST (stream_name)',
+        NEW.table_name,
+        NEW.context
+    );
+
+    EXECUTE format(
+        'COMMENT ON TABLE public.%I IS %L',
+        NEW.table_name,
+        format('''%s'' context partition', NEW.context)
+    );
+
+    RETURN NEW;
+END;
+$$;
+
+CREATE FUNCTION public.create_stream_name_partition_table()
+RETURNS trigger
+LANGUAGE plpgsql
+AS $$
+DECLARE
+    context_partition_name public.partitions.table_name%TYPE;
+BEGIN
+    SELECT table_name
+    INTO STRICT context_partition_name
+    FROM public.partitions
+    WHERE id = NEW.parent_context_partition_id;
+
+    EXECUTE format(
+        'CREATE TABLE public.%I PARTITION OF public.%I FOR VALUES IN (%L) PARTITION BY LIST (type)',
+        NEW.table_name,
+        context_partition_name,
+        NEW.stream_name
+    );
+
+    EXECUTE format(
+        'COMMENT ON TABLE public.%I IS %L',
+        NEW.table_name,
+        format('''%s'' context and ''%s'' stream name partition', NEW.context, NEW.stream_name)
+    );
+
+    RETURN NEW;
+END;
+$$;
+
+CREATE FUNCTION public.create_event_type_partition_table()
+RETURNS trigger
+LANGUAGE plpgsql
+AS $$
+DECLARE
+    stream_name_partition_name public.partitions.table_name%TYPE;
+BEGIN
+    SELECT table_name
+    INTO STRICT stream_name_partition_name
+    FROM public.partitions
+    WHERE id = NEW.parent_stream_name_partition_id;
+
+    EXECUTE format(
+        'CREATE TABLE public.%I PARTITION OF public.%I FOR VALUES IN (%L)',
+        NEW.table_name,
+        stream_name_partition_name,
+        NEW.event_type
+    );
+
+    EXECUTE format(
+        'COMMENT ON TABLE public.%I IS %L',
+        NEW.table_name,
+        format(
+            '''%s'' context and ''%s'' stream name and ''%s'' event type partition',
+            NEW.context,
+            NEW.stream_name,
+            NEW.event_type
+        )
+    );
+
+    RETURN NEW;
+END;
+$$;
+
+CREATE TRIGGER create_context_partition_table
+AFTER INSERT ON public.partitions
+FOR EACH ROW
+WHEN (NEW.stream_name IS NULL AND NEW.event_type IS NULL)
+EXECUTE FUNCTION public.create_context_partition_table();
+
+CREATE TRIGGER create_stream_name_partition_table
+AFTER INSERT ON public.partitions
+FOR EACH ROW
+WHEN (NEW.stream_name IS NOT NULL AND NEW.event_type IS NULL)
+EXECUTE FUNCTION public.create_stream_name_partition_table();
+
+CREATE TRIGGER create_event_type_partition_table
+AFTER INSERT ON public.partitions
+FOR EACH ROW
+WHEN (NEW.event_type IS NOT NULL)
+EXECUTE FUNCTION public.create_event_type_partition_table();