pg_eventstore 1.13.4 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 94443dc3aa36701d5e126573a5babdd4585932d803b766b06b94c861f6b4ef5a
-  data.tar.gz: 86fcca21a3a2d3da35f3c760542b23cc97ae93ce97bae7b9baf011a957a9ffea
+  metadata.gz: 4425f2f4cffd24b06156465726034e4c8da383fed7f0576837851a65390068e0
+  data.tar.gz: bb809d708eaea98368d54bae14a8d35f36265a7a5ad66c4b8fd9b5dc57293163
 SHA512:
-  metadata.gz: 4fb5ed37edce4c557082ba60cfa18f7209f795fa29b98dff9f848bb83b0b1cec889f4a7ca7f77bef5c0d9c6b4065d53765dcfcdf50dffe2628c3e3ffb57d26ad
-  data.tar.gz: 0bc33722f13c57c11888ac205873ac93a3610ef5218e8dccf7c3e64d671852757e9d755ffe17ea8176b238dbe1722df47db522b5252ddceed52e14534c164c45
+  metadata.gz: e86a98610d16b95eeb4bffa5a3b0d74055ac7901a30f20d297aee1b37786e06f94ff483c135f99df255cd23f50b3cac4d9360c6338ccbdab7d58d14b05fc5578
+  data.tar.gz: cdfe9c6093814265d91905f1405ff39bbd347a4e844c75961f3b3db56eb0fb4f32537b4cd26a0bb7c46b82abb21d25cb2aa7831623466cd3e2a3d5d50caf4a42

data/.rubocop.yml

@@ -2,7 +2,7 @@ plugins:
   - rubocop-rspec
 
 AllCops:
-  TargetRubyVersion: '3.0'
+  TargetRubyVersion: '3.2'
   NewCops: enable
   Exclude:
     - 'benchmark/*'
@@ -27,7 +27,7 @@ RSpec/IndexedLet:
   Enabled: false
 
 RSpec/NestedGroups:
-  Max: 6
+  Enabled: false
 
 RSpec/LetSetup:
   Enabled: false
@@ -124,3 +124,12 @@ Gemspec/RequiredRubyVersion:
 
 Gemspec/RequireMFA:
   Enabled: false
+
+Style/StructInheritance:
+  Enabled: false
+
+Lint/NonLocalExitFromIterator:
+  Enabled: false
+
+Style/RegexpLiteral:
+  Enabled: false

data/CHANGELOG.md

@@ -1,5 +1,142 @@
 ## [Unreleased]
 
+## [3.0.0]
+
+- New feature: pg_eventstore replication
+- **Breaking change**: drop support of Ruby v3.2. The gem now requires Ruby v3.3+
+- New feature: event tracing. You can now configure pg_eventstore to trace causation dependencies of events.
+  Read more [here](docs/event_tracing.md)
+- **Breaking change**: when event is persisted using `#append_to_stream` - it now gets deserialized via each registered
+  middleware by default. Previously this wasn't the case - the deserialization phase was skipped. Such behavior was
+  creating ambiguity about assumptions when deserialization happens. If you need old behavior - you have to create
+  another middleware class which skips deserialization when publishing events and use it instead. Example:
+Let's say you have this middleware configured
+```ruby
+class MyMiddleware
+  include PgEventstore::Middleware
+
+  def serialize(event)
+    # do something with event before persisting it
+  end
+
+  def deserialize(event)
+    # do something with event after reading it from db
+  end
+end
+
+PgEventstore.configure do |config|
+  config.middleware = { my_middleware: MyMiddleware.new }
+end
+```
+
+Now define another middleware that has empty `#deserialize` method
+
+```ruby
+class MyMiddlewareWithoutDeserialize < MyMiddleware
+  def deserialize(event)
+  end
+end
+
+PgEventstore.configure do |config|
+  config.middleware = {
+    my_middleware: MyMiddleware.new,
+    my_middleware_wo_deserialize: MyMiddlewareWithoutDeserialize.new
+  }
+end
+```
+and use only it when publishing events:
+
+```ruby
+PgEventstore.client.append_to_stream(stream, event, middlewares: [:my_middleware_wo_deserialize])
+```
+
+Alternatively, if you don't rely on `#multiple`, you can create another config specially for write operations:
+
+```ruby
+# this is your default
+PgEventstore.configure do |config|
+  config.middleware = { my_middleware: MyMiddleware.new }
+end
+
+PgEventstore.configure(name: :write) do |config|
+  config.middleware = { my_middleware: MyMiddlewareWithoutDeserialize.new }
+end
+```
+
+and use it for write operations:
+
+```ruby
+PgEventstore.client(:write).append_to_stream(stream, event)
+```
+
+- **Breaking change**: `Event#id` uniqueness is no longer guaranteed. It was dropped because there is not much usage of
+  it internally. The default value was moved from the database(it was `gen_random_uuid()`) to the application level and
+  is `SecureRandom.uuid_v7` now.
+- **Breaking change**: event types, stream attributes, event metadata keys, markers that start from `▒`(`"\u2592"` 
+Unicode character) character are now reserved by pg_eventstore. It is less likely you have any, but if you do - you have
+to adjust your implementation to no longer rely on it.
+- New feature: event markers. You can now mark an event and later use those markers to read events, build a projection
+using subscriptions or validate stream revision scoped to an event type and specific markers when publishing events as 
+a part of Dynamic Consistency Boundaries. Find more in [docs](docs/appending_events.md#event-markers)
+- New config option `config.events_subscription_position_update_interval`. See more
+  in [Configuration](docs/configuration.md) docs
+- **Breaking change**: `:from_position` option of `SubscriptionManager#subscribe`, "Current position" column in admin
+  web UI, "Reset position" form in admin web UI now refers to the event subscription position instead
+  `Event#global_position`. See more information in related
+  docs [Subscription position](docs/subscriptions.md#subscription-position)
+- **Breaking change**: Drop `pg_cron` dependency
+- Remove deprecated `SubscriptionsManager#force_lock!`
+- New feature: explicit support of Dynamic Consistency Boundaries when publishing events. You can now provide "event
+  type"-to-"expected stream revision" hash map into `:expected_revision` option. It allows to validate against stream
+  revision at the specific event type instead validating the whole stream. Example:
+  `PgEventstore.client.append_to_stream(stream, events, options: { expected_revision: { 'Foo' => 10, 'Bar' => 20 } })`
+- **Breaking change**: `WrongExpectedRevisionError#new` now requires `:verdict` keyword argument.
+  `Client#append_to_stream` now also may raise `WrongExpectedTypesRevisionError` error in case `:expected_revision`
+  option is "event type"-to-"expected stream revision" hash map
+- Remove any practical limit of `Event#stream_revision` by changing its type to `bigint`. Stream size can be practically
+  unreachable now
+- Fix issue when incomplete transaction block may be interrupted half way and commited successfully. The issue is
+  related to incorrect transaction termination flow during non-graceful thread shutdown(e.g. via `Thread.exit`)
+- New feature: `Client#stream_revision` allows to quickly get stream's revision. Example:
+  `PgEventstore.client.stream_revision(stream) #=> number, greater than or equal to 0 or -1 if stream does not exist`
+- **Breaking change**: `Client#read_grouped` now requires to provide `event_types` filter option explicitly. Example:
+  `PgEventstore.client.read_grouped(stream, options: { filter: { event_types: ['Foo', 'Bar'] } })`
+- New feature: Subscription event types filter now support prefixes. So, if you have subscriptions like
+  `manager.subscribe('MyAwesomeSubscription', handler: my_handler, options: { filter: { event_types: ['Foo1', 'Foo2', 'Foo3'] } })`,
+  you can replace its filter as follows: `{ event_types: [{ prefix: 'Foo' }] }`. **Read API does not support this
+  feature yet**
+- New feature: Read API now supports `:to_position`(and `:to_revision` when reading from specific stream) option. Use it
+  if you want to limit your result alongside `:max_count` option. Example:
+  `PgEventstore.client.read(PgEventstore::Stream.all_stream, options: { from_position: 100, to_position: 200 })`
+- **Breaking change**: `Middleware#serialize` is now called outside SQL transaction that inserts an event tuple.
+  Previously it was happening right before an `INSERT` statement
+- **Breaking change**: Database structure and the implementation of `#append_to_stream` was changed in a way it is not
+  compatible with v2. Public API stays the same
+- New feature: allow subscription to process events in batches. You can now provide `in_batches: true` to `#subscribe`
+  to yield an array of events. See [docs](docs/subscriptions.md#processing-events-in-batches) for more info
+- Introduce `PgEventstore::BasicConfig` and `PgEventstore::Extensions::ActsAsConfigurable` to allow to easily create
+  configurable objects
+
+## [2.0.0]
+
+- **Breaking change**: `pg_eventstore` now requires [pg_cron](https://github.com/citusdata/pg_cron) extension
+- **Breaking change**: `pg_eventstore` now requires PostgreSQL v16+
+- Greatly decreased the number of connections, used by `pg_eventstore` subscriptions
+- **Breaking change**: drop support of Ruby v3.0 and v3.1. The gem now requires Ruby v3.2+
+- **Breaking change**: `PgEventstore::Extensions::OptionsExtension::Options` class is no longer a child of `Set` class -
+  it has independent implementation now
+- Add support of Ruby v4.0
+- `Client#multiple` method now accepts `read_only` keyword argument. When it is set to true - transaction is run in
+  read-only mode
+- **Breaking change**: rework links implementation. This change boosts performance, but affects the database structure,
+  so your previous database dumps become incompatible with this change. `PgEventstore::Event#link_id` was replaced by
+  `PgEventstore::Event#link_global_position`
+
+Changes above require you to run migrations - `bundle exec rake pg_eventstore:migrate`. One of the migrations also
+migrates existing data using several concurrent workers(threads). You can adjust the number of workers using
+`CONCURRENCY` environment variable. Default number of concurrent workers is `10`. **Migrations require a downtime - no
+reads/writes should be performed during the time of the migrations, so plan your maintenance downtime accordingly.**
+
 ## [1.13.4]
 
 - Fix subscriptions potentially skipping events when multiple events are appended in concurrent transactions
@@ -18,7 +155,8 @@
 
 ## [1.13.0]
 
-- Introduce automatic subscriptions recovery from connection errors. This way if a subscription process loses the connection to the database - it will be trying to reconnect until the connection is restored.
+- Introduce automatic subscriptions recovery from connection errors. This way if a subscription process loses the
+  connection to the database - it will be trying to reconnect until the connection is restored.
 - Resolve ambiguity in usage of `PgEventstore.config` method. It now returns the frozen object.
 
 ## [1.12.0]
@@ -27,12 +165,17 @@
 
 ## [1.11.0]
 
-- Add a global position that caused an error to the subscription's error JSON info. This will help you understand what event caused your subscription to fail.
+- Add a global position that caused an error to the subscription's error JSON info. This will help you understand what
+  event caused your subscription to fail.
 - Improve long payloads in JSON preview in admin web UI in the way it does not moves content out of the visible area.
-- Admin UI: adjust events filtering and displaying of stream context, stream name, stream id and event type when values of them contain empty strings or non-displayable characters
+- Admin UI: adjust events filtering and displaying of stream context, stream name, stream id and event type when values
+  of them contain empty strings or non-displayable characters
 
 ## [1.10.0]
-- Admin UI: Adjust `SubscriptionSet` "Stop"/"Delete" buttons appearance. Now if `SubscriptionsSet` is not alive anymore(the related process is dead or does not exist anymore) - "Delete" button is shown. If `SubscriptionSet` is alive - "Stop" button is shown
+
+- Admin UI: Adjust `SubscriptionSet` "Stop"/"Delete" buttons appearance. Now if `SubscriptionsSet` is not alive anymore(
+  the related process is dead or does not exist anymore) - "Delete" button is shown. If `SubscriptionSet` is alive - "
+  Stop" button is shown
 - Admin IU: fixed several potential XSS vulnerabilities
 - Admin IU: Add "Copy to clipboard" button near stream id that copies ruby stream definition
 - Admin UI: allow deletion of streams with empty attribute values
@@ -44,23 +187,33 @@
 - Add "Delete event" and "Delete stream" buttons into admin UI
 
 ## [1.8.0]
-- Introduce default config for admin web UI. Now if you define `:admin_web_ui` config - it will be preferred over default config
+
+- Introduce default config for admin web UI. Now if you define `:admin_web_ui` config - it will be preferred over
+  default config
 - Fix pagination of events in admin UI
 - Improve partial index for `$streams` system stream
 
 ## [1.7.0]
+
 - Implement reading from `"$streams"` system stream
 - Disable Host authorization introduced in sinatra v4.1
 
 ## [1.6.0]
-- Introduce subscriptions CLI. Type `pg-eventstore subscriptions --help` to see available commands. The main purpose of it is to provide the single way to start/stop subscription processes. Check [Subscriptions](docs/subscriptions.md#creating-a-subscription) docs about the new way to start and keep running a subscriptions process.
+
+- Introduce subscriptions CLI. Type `pg-eventstore subscriptions --help` to see available commands. The main purpose of
+  it is to provide the single way to start/stop subscription processes.
+  Check [Subscriptions](docs/subscriptions.md#creating-a-subscription) docs about the new way to start and keep running
+  a subscriptions process.
 
 ## [1.5.0]
+
 - Add ability to toggle link events in the admin UI
 - Mark linked events in the admin UI with "link" icon
 
 ## [1.4.0]
-- Add an ability to configure subscription graceful shutdown timeout globally and per subscription. Default value is 15 seconds. Previously it was hardcoded to 5 seconds. Examples:
+
+- Add an ability to configure subscription graceful shutdown timeout globally and per subscription. Default value is 15
+  seconds. Previously it was hardcoded to 5 seconds. Examples:
 
 ```ruby
 # Set it globally, for all subscriptions
@@ -77,18 +230,24 @@ subscriptions_manager.subscribe(
 ```
 
 ## [1.3.4]
-- Fix `NoMethodError` error in `Client#read_paginated` when stream does not exist or when there are no events matching the given filter
+
+- Fix `NoMethodError` error in `Client#read_paginated` when stream does not exist or when there are no events matching
+  the given filter
 
 ## [1.3.3]
+
 - Adjust default value of `subscription_max_retries` setting
 
 ## [1.3.2]
+
 - Fix UI when switching subscription status
 
 ## [1.3.1]
+
 - Swap "Search" button and "Add filter" button on Dashboard page
 
 ## [1.3.0]
+
 - Add ability to filter subscriptions by state in admin UI
 - Reset error-related subscription's attributes on subscription restore
 - Reset total processed events number when user changes subscription's position
@@ -96,9 +255,12 @@ subscriptions_manager.subscribe(
 - Relax sinatra version requirement to v3+
 
 ## [1.2.0]
+
 - Implement `failed_subscription_notifier` subscription hook.
 
-Now you are able to define a function that is called when subscription fails and no longer can be automatically restarted because it hit max number of retries. You can define the hook globally in the config and per subscription. Examples:
+Now you are able to define a function that is called when subscription fails and no longer can be automatically
+restarted because it hit max number of retries. You can define the hook globally in the config and per subscription.
+Examples:
 
 ```ruby
 PgEventstore.configure do |config|
@@ -122,36 +284,48 @@ subscriptions_manager.subscribe(
 ```
 
 ## [1.1.5]
-- Review the way to handle SubscriptionAlreadyLockedError error. This removes noise when attempting to lock an already locked subscription.
+
+- Review the way to handle SubscriptionAlreadyLockedError error. This removes noise when attempting to lock an already
+  locked subscription.
 
 ## [1.1.4]
+
 - Add rbs signatures
 
 ## [1.1.3]
+
 - Fix issue with assets caching between different gem's versions
 
 ## [1.1.2]
+
 - Improve web app compatibility with rails
 
 ## [1.1.1]
+
 - Allow case insensitive search by context, stream name and event type in admin UI
 
 ## [1.1.0]
+
 - Add "Reset position" button on Subscriptions Admin UI page
 
-**Note** This release includes a migration to support new functional. Please don't forget to run `rake pg_eventstore:migrate` to apply latest db changes.
+**Note** This release includes a migration to support new functional. Please don't forget to run
+`rake pg_eventstore:migrate` to apply latest db changes.
 
 ## [1.0.4]
+
 - Fix bug which caused slow Subscriptions to stop processing new events
 - Optimize Subscriptions update queries
 
 ## [1.0.3]
+
 - Do no update `Subscription#last_chunk_fed_at` if the chunk is empty
 
 ## [1.0.2]
+
 - UI: Fix opening of SubscriptionsSet tab of non-existing SubscriptionsSet
 
 ## [1.0.1]
+
 - Adjust assets urls to correctly act when mounting sinatra app under non-root url
 
 ## [1.0.0]
@@ -189,7 +363,8 @@ subscriptions_manager.subscribe(
 
 ## [0.8.0] - 2024-02-20
 
-- Allow float values for `subscription_pull_interval`. The default value of it was also set to `1.0`(it was `2` previously)
+- Allow float values for `subscription_pull_interval`. The default value of it was also set to `1.0`(it was `2`
+  previously)
 
 ## [0.7.2] - 2024-02-14
 
@@ -201,7 +376,8 @@ subscriptions_manager.subscribe(
 
 ## [0.7.0] - 2024-02-09
 
-- Refactor `pg_eventstore:create` and `pg_eventstore:drop` rake tasks. They now actually create/drop the database. You will have to execute `delete from migrations where number > 6` query before deploying this version.
+- Refactor `pg_eventstore:create` and `pg_eventstore:drop` rake tasks. They now actually create/drop the database. You
+  will have to execute `delete from migrations where number > 6` query before deploying this version.
 - Drop legacy migrations
 
 ## [0.6.0] - 2024-02-08
@@ -246,9 +422,11 @@ subscriptions_manager.subscribe(
 
 ## [0.2.4] - 2023-12-20
 
-Due to performance issues under certain circumstances, searching by event type was giving bad performance. I decided to extract `type` column from `events` table into separated table. **No breaking changes in public API though.**
+Due to performance issues under certain circumstances, searching by event type was giving bad performance. I decided to
+extract `type` column from `events` table into separated table. **No breaking changes in public API though.**
 
-**Warning** The migrations this version has, requires you to shut down applications that use `pg_eventstore` and only then run `rake pg_eventstore:migrate`.
+**Warning** The migrations this version has, requires you to shut down applications that use `pg_eventstore` and only
+then run `rake pg_eventstore:migrate`.
 
 ## [0.2.3] - 2023-12-18
 
@@ -260,7 +438,8 @@ Due to performance issues under certain circumstances, searching by event type w
 
 ## [0.2.1] - 2023-12-14
 
-Under certain circumstances `PG::TRSerializationFailure` exception wasn't retried. Adjust connection's states list to fix that.
+Under certain circumstances `PG::TRSerializationFailure` exception wasn't retried. Adjust connection's states list to
+fix that.
 
 ## [0.2.0] - 2023-12-14
 

data/README.md

@@ -4,27 +4,40 @@ Implements database and API to store and read events in event sourced systems.
 
 ## Requirements
 
-- `pg_eventstore` requires a PostgreSQL database with jsonb data type support (which means you need to have v9.2+). However it is recommended to use a non [EOL](https://www.postgresql.org/support/versioning/) PostgreSQL version, because the development of this gem is targeted at current PostgreSQL versions. 
-- It is recommended you to have the default value set for `default_transaction_isolation` PostgreSQL config setting(`"read committed"`) as the implementation relies on it. All other transaction isolation levels(`"repeatable read"` and `"serializable"`) may cause unexpected serialization errors.
+- `pg_eventstore` requires a PostgreSQL v16+.
+- `pg_evenstore` requires a separate database. However, it is recommended that you spin it up on a separate PostgreSQL instance in a production environment.
+- `pg_eventstore` requires `default_transaction_isolation` server config option to be set to `'read committed'` (default behavior). Having this value set to move strict isolation level may result in unexpected behavior.
 - It is recommended to use a connection pooler (for example [PgBouncer](https://www.pgbouncer.org/)) in `transaction` pool mode to lower the load on a database.
-- `pg_eventstore` requires ruby v3+. The development of this gem is targeted at [current](https://endoflife.date/ruby) ruby versions.
+- `pg_eventstore` requires ruby v3.3+. The development of this gem is targeted at [current](https://endoflife.date/ruby) ruby versions.
+
+### Migrating to v3
+
+If you are migrating from v2 - please don't forget to delete cron jobs and `pg_cron` extension after migration to v3.
+You can remove cron jobs as follows:
+
+```sql
+SELECT cron.unschedule('prune_eventstore_events_horizon');
+SELECT cron.unschedule('delete-job-run-details');
+```
 
 ## Installation
 
 Install the gem and add to the application's Gemfile by executing:
-
-    $ bundle add pg_eventstore
+```bash
+bundle add pg_eventstore
+```
 
 If bundler is not being used to manage dependencies, install the gem by executing:
-
-    $ gem install pg_eventstore
+```bash
+gem install pg_eventstore
+```
 
 ## Usage
 
 Before start using the gem - you have to create the database. Please include this line into your `Rakefile`:
 
 ```ruby
-load "pg_eventstore/tasks/setup.rake"
+load 'pg_eventstore/tasks/setup.rake'
 ```
 
 This will include necessary rake tasks. You can now run 
@@ -44,11 +57,17 @@ Documentation chapters:
 - [Appending events](docs/appending_events.md)
 - [Linking events](docs/linking_events.md)
 - [Reading events](docs/reading_events.md)
+- [Reading streams](docs/reading_streams.md)
+- [Tracing events](docs/event_tracing.md)
 - [Subscriptions](docs/subscriptions.md)
 - [Maintenance functions](docs/maintenance.md)
 - [Writing middlewares](docs/writing_middleware.md)
 - [How to make multiple commands atomic](docs/multiple_commands.md)
 - [Admin UI](docs/admin_ui.md)
+- [Replication](docs/replication.md)
+
+To help your AI assistant to better recognize the capabilities of this gem - there is a crafted [instructions](docs/AGENTS.fragment.md)
+you can include into your `AGENTS.md`.
 
 ## CLI
 
@@ -104,14 +123,18 @@ end
 After checking out the repo, run:
 - `bundle` to install dependencies
 - `docker compose up` to start dev/test services
+- `bundle exec rake compile` to compile native extension 
 - `bin/setup_db` to create/re-create development and test databases, tables and related objects
 - `bundle exec rbs collection install` to install external rbs definitions
 
-Then, run `bin/rspec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+Then, run `bin/rspec` to run the tests(or just `rspec` if you want to temporary skip rbs checks). Optional env variables for testing:
+- `DEBUG=1 bin/rspec`. Enable nicely formatted and highlighted SQL output into stdout.
+
+You can also run `bin/console` for an interactive prompt that will allow you to experiment.
 
 To install this gem onto your local machine, run `bundle exec rake install`.
 
-To run admin UI web server - run `puma` in your terminal. By default it will start web server on `http://0.0.0.0:9292`.
+To run admin UI web server - run `puma` in your terminal. By default, it will start web server on `http://0.0.0.0:9292`.
 
 ### Benchmarks
 

data/db/migrations/10_setup_pg_cron.rb

@@ -0,0 +1 @@
+# pg_cron was removed. This file is kept for informative purpose.

data/db/migrations/11_add_events_link_global_position.sql

@@ -0,0 +1 @@
+ALTER TABLE public.events ADD COLUMN link_global_position bigint;

data/db/migrations/12_migrate_legacy_links.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+CONCURRENCY = ENV['CONCURRENCY']&.to_i || 10
+
+PgEventstore.configure(name: :_eventstore_db_connection) do |config|
+  config.connection_pool_size = CONCURRENCY
+end
+
+partitions = PgEventstore.connection(:_eventstore_db_connection).with do |conn|
+  conn.exec('select id, table_name from partitions')
+end
+partitions = partitions.to_h { |attrs| [attrs['id'], attrs['table_name']] }
+
+total_links = PgEventstore.connection(:_eventstore_db_connection).with do |conn|
+  conn.exec_params(
+    'select count(*) all_count from events where events.type = $1 and link_global_position is null',
+    [PgEventstore::Event::LINK_TYPE]
+  )
+end.first['all_count']
+
+puts "Migrating legacy links. Links to process: #{total_links}. Concurrency is #{CONCURRENCY} concurrent writers."
+processed = 0
+processed_was = 0
+time = Time.now
+lock = Thread::Mutex.new
+threads = CONCURRENCY.times.map do |t|
+  Thread.new do
+    loop do
+      link_events = PgEventstore.connection(:_eventstore_db_connection).with do |conn|
+        conn.exec_params(<<~SQL, [PgEventstore::Event::LINK_TYPE, CONCURRENCY, t])
+          select * from events
+            where events.type = $1 and events.link_global_position is null and global_position % $2 = $3
+            limit 1_000
+        SQL
+      end.to_a
+      break if link_events.empty?
+
+      lock.synchronize { processed += link_events.size }
+      link_events = link_events.to_h { [_1['global_position'], _1] }
+      builders = link_events.values.map do |event|
+        builder = PgEventstore::SQLBuilder.new
+        builder.select("global_position, #{event['global_position']} as link_event_global_position")
+        builder.from(partitions[event['link_partition_id']]).where('id = ?', event['link_id'])
+      end
+      final_builder = PgEventstore::SQLBuilder.union_builders(builders)
+
+      positions_map = PgEventstore.connection(:_eventstore_db_connection).with do |conn|
+        conn.exec_params(*final_builder.to_exec_params)
+      end.to_a
+
+      update_queries = positions_map.map do |attrs|
+        <<~SQL
+          UPDATE events SET link_global_position = #{attrs['global_position']}
+            WHERE global_position = #{attrs['link_event_global_position']};
+        SQL
+      end
+
+      PgEventstore.connection(:_eventstore_db_connection).with do |conn|
+        conn.exec(update_queries.join("\n"))
+      end
+
+      # Only log from the first thread to prevent messages spam
+      next unless t == 0
+
+      lock.synchronize do
+        time_was = time
+        time = Time.now
+
+        performance_info = <<~TEXT.strip
+          Processed: #{processed}. Left: #{total_links - processed}. \
+          Performance: #{((processed - processed_was) / (time - time_was)).round(2)} events/second.
+        TEXT
+        processed_was = processed
+        print "#{performance_info}               \r"
+      end
+    end
+  end
+end
+threads.each(&:join)
+
+PgEventstore.connection(:_eventstore_db_connection).with do |conn|
+  conn.exec('VACUUM (ANALYZE) events;')
+end

data/db/migrations/13_remove_events_link_id.sql

@@ -0,0 +1,6 @@
+-- We need to drop view which uses link_id and re-create it after we remove link_id column
+DROP VIEW "$streams";
+
+ALTER TABLE public.events DROP COLUMN link_id;
+
+CREATE VIEW "$streams" AS SELECT * FROM events WHERE stream_revision = 0;

data/db/migrations/14_remove_ids_events_id_index.sql

@@ -0,0 +1 @@
+DROP INDEX idx_events_id;

data/db/migrations/15_create_streams_global_index.sql

@@ -0,0 +1,17 @@
+CREATE TABLE public.streams_global_index
+(
+    id                bigserial                         NOT NULL,
+    partition_id      bigint                            NOT NULL,
+    stream_id         character varying COLLATE "POSIX" NOT NULL,
+    stream_revision   bigint                            NOT NULL,
+    starting_position bigint                            NOT NULL
+);
+
+ALTER TABLE ONLY public.streams_global_index
+    ADD CONSTRAINT streams_global_index_pkey PRIMARY KEY (id);
+
+CREATE UNIQUE INDEX idx_streams_global_index_on_stream_id_and_partition_id ON public.streams_global_index
+    USING btree (stream_id, partition_id) INCLUDE (id);
+
+CREATE INDEX idx_streams_global_index_on_starting_position ON public.streams_global_index
+    USING btree (starting_position);