pg_eventstore 2.0.0 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a4799e200314277c53e9991483c429267213a9243a93c0ea85e7a23fa8c9b1ad
-  data.tar.gz: 38c68fd540c903d19ac316aca849007f3b4c27ab94dc09cfb2e71a9b9acdd734
+  metadata.gz: 4425f2f4cffd24b06156465726034e4c8da383fed7f0576837851a65390068e0
+  data.tar.gz: bb809d708eaea98368d54bae14a8d35f36265a7a5ad66c4b8fd9b5dc57293163
 SHA512:
-  metadata.gz: 347354eb5327bf7a38df0682710f9b5a676488910ed3d0e7e967c9bcccd1d4e4b784c87e178dc6f4b5fcea2c8f681bfa0209618cf58f76dd5b36db1577d656f0
-  data.tar.gz: a567b9bfee1112e16d4f8b96b130e0f01c12b1f5349429c1ac873f4f3b4daa36628239b041eeecdc0f69a9807210d36bb30352dcb05b4100ef14eda37f1c6133
+  metadata.gz: e86a98610d16b95eeb4bffa5a3b0d74055ac7901a30f20d297aee1b37786e06f94ff483c135f99df255cd23f50b3cac4d9360c6338ccbdab7d58d14b05fc5578
+  data.tar.gz: cdfe9c6093814265d91905f1405ff39bbd347a4e844c75961f3b3db56eb0fb4f32537b4cd26a0bb7c46b82abb21d25cb2aa7831623466cd3e2a3d5d50caf4a42

data/.rubocop.yml

@@ -27,7 +27,7 @@ RSpec/IndexedLet:
   Enabled: false
 
 RSpec/NestedGroups:
-  Max: 6
+  Enabled: false
 
 RSpec/LetSetup:
   Enabled: false
@@ -124,3 +124,12 @@ Gemspec/RequiredRubyVersion:
 
 Gemspec/RequireMFA:
   Enabled: false
+
+Style/StructInheritance:
+  Enabled: false
+
+Lint/NonLocalExitFromIterator:
+  Enabled: false
+
+Style/RegexpLiteral:
+  Enabled: false

data/CHANGELOG.md

@@ -1,17 +1,141 @@
 ## [Unreleased]
 
+## [3.0.0]
+
+- New feature: pg_eventstore replication
+- **Breaking change**: drop support of Ruby v3.2. The gem now requires Ruby v3.3+
+- New feature: event tracing. You can now configure pg_eventstore to trace causation dependencies of events.
+  Read more [here](docs/event_tracing.md)
+- **Breaking change**: when event is persisted using `#append_to_stream` - it now gets deserialized via each registered
+  middleware by default. Previously this wasn't the case - the deserialization phase was skipped. Such behavior was
+  creating ambiguity about assumptions when deserialization happens. If you need old behavior - you have to create
+  another middleware class which skips deserialization when publishing events and use it instead. Example:
+Let's say you have this middleware configured
+```ruby
+class MyMiddleware
+  include PgEventstore::Middleware
+
+  def serialize(event)
+    # do something with event before persisting it
+  end
+
+  def deserialize(event)
+    # do something with event after reading it from db
+  end
+end
+
+PgEventstore.configure do |config|
+  config.middleware = { my_middleware: MyMiddleware.new }
+end
+```
+
+Now define another middleware that has empty `#deserialize` method
+
+```ruby
+class MyMiddlewareWithoutDeserialize < MyMiddleware
+  def deserialize(event)
+  end
+end
+
+PgEventstore.configure do |config|
+  config.middleware = {
+    my_middleware: MyMiddleware.new,
+    my_middleware_wo_deserialize: MyMiddlewareWithoutDeserialize.new
+  }
+end
+```
+and use only it when publishing events:
+
+```ruby
+PgEventstore.client.append_to_stream(stream, event, middlewares: [:my_middleware_wo_deserialize])
+```
+
+Alternatively, if you don't rely on `#multiple`, you can create another config specially for write operations:
+
+```ruby
+# this is your default
+PgEventstore.configure do |config|
+  config.middleware = { my_middleware: MyMiddleware.new }
+end
+
+PgEventstore.configure(name: :write) do |config|
+  config.middleware = { my_middleware: MyMiddlewareWithoutDeserialize.new }
+end
+```
+
+and use it for write operations:
+
+```ruby
+PgEventstore.client(:write).append_to_stream(stream, event)
+```
+
+- **Breaking change**: `Event#id` uniqueness is no longer guaranteed. It was dropped because there is not much usage of
+  it internally. The default value was moved from the database(it was `gen_random_uuid()`) to the application level and
+  is `SecureRandom.uuid_v7` now.
+- **Breaking change**: event types, stream attributes, event metadata keys, markers that start from `▒`(`"\u2592"` 
+Unicode character) character are now reserved by pg_eventstore. It is less likely you have any, but if you do - you have
+to adjust your implementation to no longer rely on it.
+- New feature: event markers. You can now mark an event and later use those markers to read events, build a projection
+using subscriptions or validate stream revision scoped to an event type and specific markers when publishing events as 
+a part of Dynamic Consistency Boundaries. Find more in [docs](docs/appending_events.md#event-markers)
+- New config option `config.events_subscription_position_update_interval`. See more
+  in [Configuration](docs/configuration.md) docs
+- **Breaking change**: `:from_position` option of `SubscriptionManager#subscribe`, "Current position" column in admin
+  web UI, "Reset position" form in admin web UI now refers to the event subscription position instead
+  `Event#global_position`. See more information in related
+  docs [Subscription position](docs/subscriptions.md#subscription-position)
+- **Breaking change**: Drop `pg_cron` dependency
+- Remove deprecated `SubscriptionsManager#force_lock!`
+- New feature: explicit support of Dynamic Consistency Boundaries when publishing events. You can now provide "event
+  type"-to-"expected stream revision" hash map into `:expected_revision` option. It allows to validate against stream
+  revision at the specific event type instead validating the whole stream. Example:
+  `PgEventstore.client.append_to_stream(stream, events, options: { expected_revision: { 'Foo' => 10, 'Bar' => 20 } })`
+- **Breaking change**: `WrongExpectedRevisionError#new` now requires `:verdict` keyword argument.
+  `Client#append_to_stream` now also may raise `WrongExpectedTypesRevisionError` error in case `:expected_revision`
+  option is "event type"-to-"expected stream revision" hash map
+- Remove any practical limit of `Event#stream_revision` by changing its type to `bigint`. Stream size can be practically
+  unreachable now
+- Fix issue when incomplete transaction block may be interrupted half way and commited successfully. The issue is
+  related to incorrect transaction termination flow during non-graceful thread shutdown(e.g. via `Thread.exit`)
+- New feature: `Client#stream_revision` allows to quickly get stream's revision. Example:
+  `PgEventstore.client.stream_revision(stream) #=> number, greater than or equal to 0 or -1 if stream does not exist`
+- **Breaking change**: `Client#read_grouped` now requires to provide `event_types` filter option explicitly. Example:
+  `PgEventstore.client.read_grouped(stream, options: { filter: { event_types: ['Foo', 'Bar'] } })`
+- New feature: Subscription event types filter now support prefixes. So, if you have subscriptions like
+  `manager.subscribe('MyAwesomeSubscription', handler: my_handler, options: { filter: { event_types: ['Foo1', 'Foo2', 'Foo3'] } })`,
+  you can replace its filter as follows: `{ event_types: [{ prefix: 'Foo' }] }`. **Read API does not support this
+  feature yet**
+- New feature: Read API now supports `:to_position`(and `:to_revision` when reading from specific stream) option. Use it
+  if you want to limit your result alongside `:max_count` option. Example:
+  `PgEventstore.client.read(PgEventstore::Stream.all_stream, options: { from_position: 100, to_position: 200 })`
+- **Breaking change**: `Middleware#serialize` is now called outside SQL transaction that inserts an event tuple.
+  Previously it was happening right before an `INSERT` statement
+- **Breaking change**: Database structure and the implementation of `#append_to_stream` was changed in a way it is not
+  compatible with v2. Public API stays the same
+- New feature: allow subscription to process events in batches. You can now provide `in_batches: true` to `#subscribe`
+  to yield an array of events. See [docs](docs/subscriptions.md#processing-events-in-batches) for more info
+- Introduce `PgEventstore::BasicConfig` and `PgEventstore::Extensions::ActsAsConfigurable` to allow to easily create
+  configurable objects
+
 ## [2.0.0]
 
 - **Breaking change**: `pg_eventstore` now requires [pg_cron](https://github.com/citusdata/pg_cron) extension
 - **Breaking change**: `pg_eventstore` now requires PostgreSQL v16+
 - Greatly decreased the number of connections, used by `pg_eventstore` subscriptions
 - **Breaking change**: drop support of Ruby v3.0 and v3.1. The gem now requires Ruby v3.2+
-- **Breaking change**: `PgEventstore::Extensions::OptionsExtension::Options` class is no longer a child of `Set` class - it has independent implementation now
+- **Breaking change**: `PgEventstore::Extensions::OptionsExtension::Options` class is no longer a child of `Set` class -
+  it has independent implementation now
 - Add support of Ruby v4.0
-- `Client#multiple` method now accepts `read_only` keyword argument. When it is set to true - transaction is run in read-only mode
-- **Breaking change**: rework links implementation. This change boosts performance, but affects the database structure, so your previous database dumps become incompatible with this change. `PgEventstore::Event#link_id` was replaced by `PgEventstore::Event#link_global_position`
+- `Client#multiple` method now accepts `read_only` keyword argument. When it is set to true - transaction is run in
+  read-only mode
+- **Breaking change**: rework links implementation. This change boosts performance, but affects the database structure,
+  so your previous database dumps become incompatible with this change. `PgEventstore::Event#link_id` was replaced by
+  `PgEventstore::Event#link_global_position`
 
-Changes above require you to run migrations - `bundle exec rake pg_eventstore:migrate`. One of the migrations also migrates existing data using several concurrent workers(threads). You can adjust the number of workers using `CONCURRENCY` environment variable. Default number of concurrent workers is `10`. **Migrations require a downtime - no reads/writes should be performed during the time of the migrations, so plan your maintenance downtime accordingly.**
+Changes above require you to run migrations - `bundle exec rake pg_eventstore:migrate`. One of the migrations also
+migrates existing data using several concurrent workers(threads). You can adjust the number of workers using
+`CONCURRENCY` environment variable. Default number of concurrent workers is `10`. **Migrations require a downtime - no
+reads/writes should be performed during the time of the migrations, so plan your maintenance downtime accordingly.**
 
 ## [1.13.4]
 
@@ -31,7 +155,8 @@ Changes above require you to run migrations - `bundle exec rake pg_eventstore:mi
 
 ## [1.13.0]
 
-- Introduce automatic subscriptions recovery from connection errors. This way if a subscription process loses the connection to the database - it will be trying to reconnect until the connection is restored.
+- Introduce automatic subscriptions recovery from connection errors. This way if a subscription process loses the
+  connection to the database - it will be trying to reconnect until the connection is restored.
 - Resolve ambiguity in usage of `PgEventstore.config` method. It now returns the frozen object.
 
 ## [1.12.0]
@@ -40,12 +165,17 @@ Changes above require you to run migrations - `bundle exec rake pg_eventstore:mi
 
 ## [1.11.0]
 
-- Add a global position that caused an error to the subscription's error JSON info. This will help you understand what event caused your subscription to fail.
+- Add a global position that caused an error to the subscription's error JSON info. This will help you understand what
+  event caused your subscription to fail.
 - Improve long payloads in JSON preview in admin web UI in the way it does not moves content out of the visible area.
-- Admin UI: adjust events filtering and displaying of stream context, stream name, stream id and event type when values of them contain empty strings or non-displayable characters
+- Admin UI: adjust events filtering and displaying of stream context, stream name, stream id and event type when values
+  of them contain empty strings or non-displayable characters
 
 ## [1.10.0]
-- Admin UI: Adjust `SubscriptionSet` "Stop"/"Delete" buttons appearance. Now if `SubscriptionsSet` is not alive anymore(the related process is dead or does not exist anymore) - "Delete" button is shown. If `SubscriptionSet` is alive - "Stop" button is shown
+
+- Admin UI: Adjust `SubscriptionSet` "Stop"/"Delete" buttons appearance. Now if `SubscriptionsSet` is not alive anymore(
+  the related process is dead or does not exist anymore) - "Delete" button is shown. If `SubscriptionSet` is alive - "
+  Stop" button is shown
 - Admin IU: fixed several potential XSS vulnerabilities
 - Admin IU: Add "Copy to clipboard" button near stream id that copies ruby stream definition
 - Admin UI: allow deletion of streams with empty attribute values
@@ -57,23 +187,33 @@ Changes above require you to run migrations - `bundle exec rake pg_eventstore:mi
 - Add "Delete event" and "Delete stream" buttons into admin UI
 
 ## [1.8.0]
-- Introduce default config for admin web UI. Now if you define `:admin_web_ui` config - it will be preferred over default config
+
+- Introduce default config for admin web UI. Now if you define `:admin_web_ui` config - it will be preferred over
+  default config
 - Fix pagination of events in admin UI
 - Improve partial index for `$streams` system stream
 
 ## [1.7.0]
+
 - Implement reading from `"$streams"` system stream
 - Disable Host authorization introduced in sinatra v4.1
 
 ## [1.6.0]
-- Introduce subscriptions CLI. Type `pg-eventstore subscriptions --help` to see available commands. The main purpose of it is to provide the single way to start/stop subscription processes. Check [Subscriptions](docs/subscriptions.md#creating-a-subscription) docs about the new way to start and keep running a subscriptions process.
+
+- Introduce subscriptions CLI. Type `pg-eventstore subscriptions --help` to see available commands. The main purpose of
+  it is to provide the single way to start/stop subscription processes.
+  Check [Subscriptions](docs/subscriptions.md#creating-a-subscription) docs about the new way to start and keep running
+  a subscriptions process.
 
 ## [1.5.0]
+
 - Add ability to toggle link events in the admin UI
 - Mark linked events in the admin UI with "link" icon
 
 ## [1.4.0]
-- Add an ability to configure subscription graceful shutdown timeout globally and per subscription. Default value is 15 seconds. Previously it was hardcoded to 5 seconds. Examples:
+
+- Add an ability to configure subscription graceful shutdown timeout globally and per subscription. Default value is 15
+  seconds. Previously it was hardcoded to 5 seconds. Examples:
 
 ```ruby
 # Set it globally, for all subscriptions
@@ -90,18 +230,24 @@ subscriptions_manager.subscribe(
 ```
 
 ## [1.3.4]
-- Fix `NoMethodError` error in `Client#read_paginated` when stream does not exist or when there are no events matching the given filter
+
+- Fix `NoMethodError` error in `Client#read_paginated` when stream does not exist or when there are no events matching
+  the given filter
 
 ## [1.3.3]
+
 - Adjust default value of `subscription_max_retries` setting
 
 ## [1.3.2]
+
 - Fix UI when switching subscription status
 
 ## [1.3.1]
+
 - Swap "Search" button and "Add filter" button on Dashboard page
 
 ## [1.3.0]
+
 - Add ability to filter subscriptions by state in admin UI
 - Reset error-related subscription's attributes on subscription restore
 - Reset total processed events number when user changes subscription's position
@@ -109,9 +255,12 @@ subscriptions_manager.subscribe(
 - Relax sinatra version requirement to v3+
 
 ## [1.2.0]
+
 - Implement `failed_subscription_notifier` subscription hook.
 
-Now you are able to define a function that is called when subscription fails and no longer can be automatically restarted because it hit max number of retries. You can define the hook globally in the config and per subscription. Examples:
+Now you are able to define a function that is called when subscription fails and no longer can be automatically
+restarted because it hit max number of retries. You can define the hook globally in the config and per subscription.
+Examples:
 
 ```ruby
 PgEventstore.configure do |config|
@@ -135,36 +284,48 @@ subscriptions_manager.subscribe(
 ```
 
 ## [1.1.5]
-- Review the way to handle SubscriptionAlreadyLockedError error. This removes noise when attempting to lock an already locked subscription.
+
+- Review the way to handle SubscriptionAlreadyLockedError error. This removes noise when attempting to lock an already
+  locked subscription.
 
 ## [1.1.4]
+
 - Add rbs signatures
 
 ## [1.1.3]
+
 - Fix issue with assets caching between different gem's versions
 
 ## [1.1.2]
+
 - Improve web app compatibility with rails
 
 ## [1.1.1]
+
 - Allow case insensitive search by context, stream name and event type in admin UI
 
 ## [1.1.0]
+
 - Add "Reset position" button on Subscriptions Admin UI page
 
-**Note** This release includes a migration to support new functional. Please don't forget to run `rake pg_eventstore:migrate` to apply latest db changes.
+**Note** This release includes a migration to support new functional. Please don't forget to run
+`rake pg_eventstore:migrate` to apply latest db changes.
 
 ## [1.0.4]
+
 - Fix bug which caused slow Subscriptions to stop processing new events
 - Optimize Subscriptions update queries
 
 ## [1.0.3]
+
 - Do no update `Subscription#last_chunk_fed_at` if the chunk is empty
 
 ## [1.0.2]
+
 - UI: Fix opening of SubscriptionsSet tab of non-existing SubscriptionsSet
 
 ## [1.0.1]
+
 - Adjust assets urls to correctly act when mounting sinatra app under non-root url
 
 ## [1.0.0]
@@ -202,7 +363,8 @@ subscriptions_manager.subscribe(
 
 ## [0.8.0] - 2024-02-20
 
-- Allow float values for `subscription_pull_interval`. The default value of it was also set to `1.0`(it was `2` previously)
+- Allow float values for `subscription_pull_interval`. The default value of it was also set to `1.0`(it was `2`
+  previously)
 
 ## [0.7.2] - 2024-02-14
 
@@ -214,7 +376,8 @@ subscriptions_manager.subscribe(
 
 ## [0.7.0] - 2024-02-09
 
-- Refactor `pg_eventstore:create` and `pg_eventstore:drop` rake tasks. They now actually create/drop the database. You will have to execute `delete from migrations where number > 6` query before deploying this version.
+- Refactor `pg_eventstore:create` and `pg_eventstore:drop` rake tasks. They now actually create/drop the database. You
+  will have to execute `delete from migrations where number > 6` query before deploying this version.
 - Drop legacy migrations
 
 ## [0.6.0] - 2024-02-08
@@ -259,9 +422,11 @@ subscriptions_manager.subscribe(
 
 ## [0.2.4] - 2023-12-20
 
-Due to performance issues under certain circumstances, searching by event type was giving bad performance. I decided to extract `type` column from `events` table into separated table. **No breaking changes in public API though.**
+Due to performance issues under certain circumstances, searching by event type was giving bad performance. I decided to
+extract `type` column from `events` table into separated table. **No breaking changes in public API though.**
 
-**Warning** The migrations this version has, requires you to shut down applications that use `pg_eventstore` and only then run `rake pg_eventstore:migrate`.
+**Warning** The migrations this version has, requires you to shut down applications that use `pg_eventstore` and only
+then run `rake pg_eventstore:migrate`.
 
 ## [0.2.3] - 2023-12-18
 
@@ -273,7 +438,8 @@ Due to performance issues under certain circumstances, searching by event type w
 
 ## [0.2.1] - 2023-12-14
 
-Under certain circumstances `PG::TRSerializationFailure` exception wasn't retried. Adjust connection's states list to fix that.
+Under certain circumstances `PG::TRSerializationFailure` exception wasn't retried. Adjust connection's states list to
+fix that.
 
 ## [0.2.0] - 2023-12-14
 

data/README.md

@@ -4,11 +4,21 @@ Implements database and API to store and read events in event sourced systems.
 
 ## Requirements
 
-- `pg_eventstore` requires a PostgreSQL v16+ with [pg_cron](https://github.com/citusdata/pg_cron) extension installed.
-- `pg_evenstore` requires a separate detabase. However, it is recommended that you spin it up on a separate PostgreSQL instance in a production environment.
+- `pg_eventstore` requires a PostgreSQL v16+.
+- `pg_evenstore` requires a separate database. However, it is recommended that you spin it up on a separate PostgreSQL instance in a production environment.
 - `pg_eventstore` requires `default_transaction_isolation` server config option to be set to `'read committed'` (default behavior). Having this value set to move strict isolation level may result in unexpected behavior.
 - It is recommended to use a connection pooler (for example [PgBouncer](https://www.pgbouncer.org/)) in `transaction` pool mode to lower the load on a database.
-- `pg_eventstore` requires ruby v3+. The development of this gem is targeted at [current](https://endoflife.date/ruby) ruby versions.
+- `pg_eventstore` requires ruby v3.3+. The development of this gem is targeted at [current](https://endoflife.date/ruby) ruby versions.
+
+### Migrating to v3
+
+If you are migrating from v2 - please don't forget to delete cron jobs and `pg_cron` extension after migration to v3.
+You can remove cron jobs as follows:
+
+```sql
+SELECT cron.unschedule('prune_eventstore_events_horizon');
+SELECT cron.unschedule('delete-job-run-details');
+```
 
 ## Installation
 
@@ -47,26 +57,22 @@ Documentation chapters:
 - [Appending events](docs/appending_events.md)
 - [Linking events](docs/linking_events.md)
 - [Reading events](docs/reading_events.md)
+- [Reading streams](docs/reading_streams.md)
+- [Tracing events](docs/event_tracing.md)
 - [Subscriptions](docs/subscriptions.md)
 - [Maintenance functions](docs/maintenance.md)
 - [Writing middlewares](docs/writing_middleware.md)
 - [How to make multiple commands atomic](docs/multiple_commands.md)
 - [Admin UI](docs/admin_ui.md)
+- [Replication](docs/replication.md)
+
+To help your AI assistant to better recognize the capabilities of this gem - there is a crafted [instructions](docs/AGENTS.fragment.md)
+you can include into your `AGENTS.md`.
 
 ## CLI
 
 The gem is shipped with its own CLI. Use `pg-eventstore --help` to find out its capabilities.
 
-## Maintenance
-
-You may want to backup your eventstore database. It is important to mention that you don't want to dump/restore records of `events_horizon` table. `events_horizon` table is used to supply subscriptions functionality and contains temporary data which is scoped to the PostgreSQL cluster they were created in. **Thus, it is even may be harmful if you restore records from this table into a new PostgreSQL cluster. Simply exclude that table's data when performing backups.** Example:
-
-```bash
-pg_dump --exclude-table-data=events_horizon eventstore -U postgres > eventstore.sql
-```
-
-Also, it is important you create and migrate new database via provided rake commands - they include an important setup of `pg_cron` jobs as well. **Even if you would like to restore your db backup on clean PostgreSQL instance - please initialize pg_eventstore via built-in tools first.**
-
 ## RSpec
 
 ### Clean up test db
@@ -117,14 +123,18 @@ end
 After checking out the repo, run:
 - `bundle` to install dependencies
 - `docker compose up` to start dev/test services
+- `bundle exec rake compile` to compile native extension 
 - `bin/setup_db` to create/re-create development and test databases, tables and related objects
 - `bundle exec rbs collection install` to install external rbs definitions
 
-Then, run `bin/rspec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+Then, run `bin/rspec` to run the tests(or just `rspec` if you want to temporary skip rbs checks). Optional env variables for testing:
+- `DEBUG=1 bin/rspec`. Enable nicely formatted and highlighted SQL output into stdout.
+
+You can also run `bin/console` for an interactive prompt that will allow you to experiment.
 
 To install this gem onto your local machine, run `bundle exec rake install`.
 
-To run admin UI web server - run `puma` in your terminal. By default it will start web server on `http://0.0.0.0:9292`.
+To run admin UI web server - run `puma` in your terminal. By default, it will start web server on `http://0.0.0.0:9292`.
 
 ### Benchmarks
 

data/db/migrations/10_setup_pg_cron.rb

@@ -1,23 +1 @@
-# frozen_string_literal: true
-
-PgEventstore.connection(:_postgres_db_connection).with do |conn|
-  conn.exec(<<~SQL)
-    CREATE EXTENSION IF NOT EXISTS pg_cron
-  SQL
-  conn.exec_params(<<~SQL, ["prune_#{PgEventstore::MigrationHelpers.db_name}_events_horizon", PgEventstore::MigrationHelpers.db_name])
-    SELECT cron.schedule_in_database(
-      $1,
-      '*/10 * * * *',
-      $$DELETE FROM events_horizon WHERE xact_id <= (SELECT xact_id FROM events_horizon ORDER BY xact_id DESC OFFSET 100 LIMIT 1)$$,
-      $2
-     )
-  SQL
-  # Store information about finished cron jobs for 1 day
-  conn.exec(<<~SQL)
-    SELECT cron.schedule(
-      'delete-job-run-details',
-      '0 12 * * *',
-      $$DELETE FROM cron.job_run_details WHERE end_time < now() - interval '1 day'$$
-    );
-  SQL
-end
+# pg_cron was removed. This file is kept for informative purpose.

data/db/migrations/15_create_streams_global_index.sql

@@ -0,0 +1,17 @@
+CREATE TABLE public.streams_global_index
+(
+    id                bigserial                         NOT NULL,
+    partition_id      bigint                            NOT NULL,
+    stream_id         character varying COLLATE "POSIX" NOT NULL,
+    stream_revision   bigint                            NOT NULL,
+    starting_position bigint                            NOT NULL
+);
+
+ALTER TABLE ONLY public.streams_global_index
+    ADD CONSTRAINT streams_global_index_pkey PRIMARY KEY (id);
+
+CREATE UNIQUE INDEX idx_streams_global_index_on_stream_id_and_partition_id ON public.streams_global_index
+    USING btree (stream_id, partition_id) INCLUDE (id);
+
+CREATE INDEX idx_streams_global_index_on_starting_position ON public.streams_global_index
+    USING btree (starting_position);

data/db/migrations/16_populate_streams_global_index.rb

@@ -0,0 +1,150 @@
+# frozen_string_literal: true
+
+CONCURRENCY = ENV['CONCURRENCY']&.to_i || 10
+
+PgEventstore.configure(name: :_eventstore_db_connection) do |config|
+  config.connection_pool_size = CONCURRENCY * 10
+end
+
+partitions = PgEventstore.connection(:_eventstore_db_connection).with do |conn|
+  conn.exec('select * from partitions where event_type is null')
+end.to_a
+
+stream_name_partitions = partitions.reject { _1['stream_name'].nil? }.group_by { _1['context'] }
+stream_name_partitions = stream_name_partitions.to_h do |context, context_parts|
+  context_parts = context_parts.to_h { |p| [p['stream_name'], p['id']] }
+  [context, context_parts]
+end
+
+total_streams = PgEventstore.connection(:_eventstore_db_connection).with do |conn|
+  conn.exec('explain select * from "$streams"')
+end.first['QUERY PLAN'][/rows=\d+/].sub('rows=', '').to_i
+
+tables = PgEventstore.connection(:_eventstore_db_connection).with do |conn|
+  conn.exec('select table_name from partitions where event_type is not null')
+end.to_a.map { _1['table_name'] }
+
+PgEventstore.connection(:_eventstore_db_connection).with do |conn|
+  conn.exec('delete from streams_global_index')
+end
+
+puts "Indexing streams. Approximate number of streams to process: #{total_streams}. Concurrency is #{CONCURRENCY} concurrent writers."
+puts <<~TEXT
+  This migration consumes CONCURRENCY x10 connections. Make sure you have plenty of them. Migrations are idempotent, \
+  so you can break, adjust settings if needed and then start over.
+TEXT
+puts
+
+processed = 0
+processed_was = 0
+time = Time.now
+lock = Thread::Mutex.new
+sub_partitions_count_cache = {}
+threads = CONCURRENCY.times.map do |t|
+  Thread.new do
+    tables.each do |table|
+      global_position = 0
+      loop do
+        events = PgEventstore.connection(:_eventstore_db_connection).with do |conn|
+          conn.exec_params(<<~SQL, [global_position, CONCURRENCY, t])
+            select context, stream_name, stream_id, global_position
+            from "#{table}"
+            where global_position > $1 and global_position % $2 = $3 and stream_revision = 0
+            group by context, stream_name, stream_id, global_position
+            order by global_position asc
+            limit 100
+          SQL
+        end.to_a
+        break if events.empty?
+
+        global_position = events.last['global_position']
+
+        # retrieve stream revision
+        events.each_slice(10) do |sliced_events|
+          query_runner = PgEventstore::AsyncRunner.new
+          sliced_events.each do |event|
+            query_runner.async do
+              query_strategy = PgEventstore::QueryStrategy::Async.new(PgEventstore.connection(:_eventstore_db_connection))
+              context = event['context']
+              stream_name = event['stream_name']
+              stream_id = event['stream_id']
+              sub_partitions_count_cache[[context, stream_name]] ||= query_strategy.exec_params(
+                'select count(*) as c_all from partitions where context = $1 and stream_name = $2',
+                [context, stream_name]
+              ).first['c_all']
+              if sub_partitions_count_cache[[context, stream_name]] > 50
+                sub_partitions = query_strategy.exec_params(<<~SQL, [context, stream_name]).map { _1['table_name'] }
+                  select table_name from partitions where context = $1 and stream_name = $2 and event_type is not null
+                SQL
+                max_revisions = []
+                sub_partitions.each_slice(50) do |table_names|
+                  q_parts = table_names.map do |table_name|
+                    <<~SQL
+                      select max(stream_revision) as stream_revision
+                      from #{table_name}
+                      where context = $1 and stream_name = $2 and stream_id = $3
+                    SQL
+                  end
+                  q_parts = q_parts.map { "(#{_1})" }.join(' union all ')
+                  q = "select coalesce(max(stream_revision), -1) as stream_revision from (#{q_parts})"
+                  max_revisions.push(
+                    query_strategy.exec_params(q, [context, stream_name, stream_id]).first['stream_revision']
+                  )
+                end
+                event['stream_revision'] = max_revisions.max
+              else
+                q = <<~SQL
+                  select max(stream_revision) as stream_revision
+                  from events
+                  where context = $1 and stream_name = $2 and stream_id = $3
+                SQL
+                attrs = query_strategy.exec_params(q, [context, stream_name, stream_id]).to_a.first
+                event['stream_revision'] = attrs['stream_revision']
+              end
+            end
+          end
+          query_runner.run
+        end
+
+        values = events.map do |event|
+          partition_id = stream_name_partitions[event['context']][event['stream_name']]
+          stream_id = PG::Connection.escape(event['stream_id'])
+          stream_revision = event['stream_revision']
+          starting_position = event['global_position']
+          "(#{partition_id}, '#{stream_id}', #{stream_revision}, #{starting_position})"
+        end.join(',')
+
+        PgEventstore.connection(:_eventstore_db_connection).with do |conn|
+          conn.exec(<<~SQL)
+            INSERT INTO streams_global_index
+              ("partition_id", "stream_id", "stream_revision", "starting_position")
+            VALUES #{values}
+          SQL
+        end
+
+        lock.synchronize do
+          processed += events.size
+          next if Time.now - time < 2
+
+          time_was = time
+          time = Time.now
+
+          performance_info = <<~TEXT.strip
+            Processed: #{processed}. Left: #{total_streams - processed} (approximately). \
+            Performance: #{((processed - processed_was) / (time - time_was)).round(2)} streams/second.
+          TEXT
+          processed_was = processed
+          print "#{performance_info}               \r"
+        end
+      end
+    end
+  end
+end
+threads.each(&:join)
+
+PgEventstore.connection(:_eventstore_db_connection).with do |conn|
+  puts 'Running cluster on streams_global_index. This may take some time.'
+  conn.exec('CLUSTER streams_global_index USING idx_streams_global_index_on_starting_position')
+  puts 'Running vacuum on streams_global_index. This may take some time.'
+  conn.exec('VACUUM (ANALYZE) streams_global_index')
+end