@relaybus/postgres 0.4.0 → 0.4.1

package/README.md

@@ -102,7 +102,8 @@ Where the budget lives depends on who owns the pool:
 - **A pool or executor you pass in**: the budget is transaction-scoped (`SET LOCAL` semantics), set with `BEGIN` by
   `PgExecutor` and by its own statement for the other adapters. Session settings of your pool are never changed.
 
-**Prisma** is integration-tested with a generated Prisma Client 7.10 (and 6.19 in CI) on the `@prisma/adapter-pg`
+**Prisma** 5, 6 and 7 are supported (optional peer `@prisma/client` `^5.0.0 || ^6.0.0 || ^7.0.0`). It is
+integration-tested with a generated Prisma Client 7.10 (and 6.19 and 5.22 in CI) on the `@prisma/adapter-pg`
 driver adapter: the full event store contract, publishing in `prisma.$transaction`, rollback and concurrent
 transactions. The adapter is typed structurally, so the package has no Prisma dependency. Prisma interactive
 transactions default to a 5 s timeout, so pass `createPrismaExecutor(prisma, { timeout, maxWait })` if handlers run
@@ -155,14 +156,14 @@ import { createPostgresSchemaSql, createPostgresSchemaStatements } from '@relayb
 const migration = createPostgresSchemaSql({ schema: 'app', tablePrefix: 'bus_' });
 ```
 
-| Table                  | Purpose                                                                                                                                                                                                                                                                            |
-| ---------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `events`               | One row per event: `id` (**text**, so any caller id works), `type`, `source`, `key`, `sequence`, `payload jsonb`, `headers jsonb`, correlation and causation ids, `occurred_at`, `created_at`, and `handlers`, the handlers that ever got a delivery (it outlives the deliveries). |
-| `deliveries`           | One row per (event, handler) that is not finished: `lane`, `status` (`pending` or `dead`), `attempts`, `available_at`, `last_error jsonb`, key and sequence copied from the event, timestamps. `unique (event_id, handler)`, foreign key to events with `ON DELETE CASCADE`.       |
-| `lanes`                | 256 rows, one per lane: `epoch` (the fencing token), `lease_owner`, `lease_expires_at` and `in_flight` (ids of the deliveries the owner claimed). `fillfactor = 50` and no secondary index, so its updates are heap-only (HOT).                                                    |
-| `event_completions`    | One row per event that had a delivery complete, with the time of its last completion. Retention of completed work (`retention.completed`) is measured from it, however old the event is; events that never had a delivery complete are measured from their creation.               |
-| `key_sequences`        | One counter row per ordering key.                                                                                                                                                                                                                                                  |
-| `completed_deliveries` | Only with `completionHistory`: one insert-only row per completed delivery.                                                                                                                                                                                                         |
+| Table                  | Purpose                                                                                                                                                                                                                                                                          |
+| ---------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `events`               | One row per event: `id` (**text**, so any caller id works), `type`, `source`, `key`, `sequence`, `payload json`, `headers json`, correlation and causation ids, `occurred_at`, `created_at`, and `handlers`, the handlers that ever got a delivery (it outlives the deliveries). |
+| `deliveries`           | One row per (event, handler) that is not finished: `lane`, `status` (`pending` or `dead`), `attempts`, `available_at`, `last_error jsonb`, key and sequence copied from the event, timestamps. `unique (event_id, handler)`, foreign key to events with `ON DELETE CASCADE`.     |
+| `lanes`                | 256 rows, one per lane: `epoch` (the fencing token), `lease_owner`, `lease_expires_at` and `in_flight` (ids of the deliveries the owner claimed). `fillfactor = 50` and no secondary index, so its updates are heap-only (HOT).                                                  |
+| `event_completions`    | One row per event that had a delivery complete, with the time of its last completion. Retention of completed work (`retention.completed`) is measured from it, however old the event is; events that never had a delivery complete are measured from their creation.             |
+| `key_sequences`        | One counter row per ordering key.                                                                                                                                                                                                                                                |
+| `completed_deliveries` | Only with `completionHistory`: one insert-only row per completed delivery.                                                                                                                                                                                                       |
 
 A delivery row is inserted once and deleted once when it completes. Claiming, renewing a lease and completing never
 update it: they update the lane row (claim, renewal) or delete the delivery (completion), so the indexed columns of a
@@ -257,10 +258,12 @@ tool, run it as one transaction with the workers stopped. Running 0.1.0 and the
 - **Key counters.** `key_sequences` keeps one row per key ever published and rows are never deleted; the table grows
   with the number of distinct keys (a bigint counter each). Pick keys with bounded cardinality, or prune counters of keys
   you know are retired once nothing for them is pending.
-- **Stored JSON.** Payloads and headers are stored as `jsonb`: object keys come back **reordered** (shorter keys first,
-  then bytewise) and duplicate keys collapse, values are otherwise identical. A string containing `\u0000` (NUL) or a
-  lone UTF-16 surrogate is rejected by PostgreSQL and fails the publish with a database error; strip or encode such
-  text before publishing.
+- **Stored JSON.** Payloads and headers are stored as `json`, the text the producer serialized: an event read back
+  serializes byte for byte as it was published, key order included. (Before 0.4.1 they were `jsonb`, which re-orders
+  object keys; `setup()` converts the columns, and rows stored before the upgrade keep the order `jsonb` gave them. See
+  [Upgrading](https://berkcansavur.github.io/relaybus/upgrading).) A string containing `\u0000` (NUL) or a lone UTF-16
+  surrogate is rejected by PostgreSQL and fails the publish with a database error; strip or encode such text before
+  publishing.
 - **Sequence gaps.** Republishing an existing event id consumes a sequence number. Gaps are normal; only order matters.
 - **Vacuum.** Completing a delivery deletes its row, which leaves a dead tuple until autovacuum runs. The tables are
   created with `autovacuum_vacuum_scale_factor = 0.02`; for sustained throughput above a few thousand deliveries per
@@ -315,8 +318,8 @@ tool, run it as one transaction with the workers stopped. Running 0.1.0 and the
   `createPostgresSchemaSql()`.
 - `permission denied for schema`: the role needs `CREATE` on the schema for `setup()`; with `setupStore: false` it needs
   only DML rights.
-- `invalid input syntax for type json` / `\u0000 cannot be converted to text`: payloads with NUL characters are not valid
-  `jsonb`.
+- `invalid input syntax for type json` / `unsupported Unicode escape sequence`: the payload holds a NUL character or a
+  lone UTF-16 surrogate, which PostgreSQL rejects.
 - Foreign key error on append: the only known cause is republishing an event id in the same instant its last delivery
   is purged. The publishing transaction rolls back and can simply be retried.
 - A key looks stuck: a pending or processing head blocks it. Inspect `listDeliveries({ status: 'processing' })` and

package/dist/index.cjs

@@ -230,6 +230,7 @@ function resolvePostgresNames(options = {}) {
     schema,
     quotedSchema,
     events: table("events"),
+    eventsTableName: name("events"),
     deliveries: table("deliveries"),
     keySequences: table("key_sequences"),
     lanes: table("lanes"),
@@ -256,8 +257,17 @@ function laneExpression(keyExpression, eventIdExpression) {
 
 // src/schema.ts
 var SCHEMA_MARKER_DIGITS = 32;
+var PAYLOAD_COLUMNS = `  payload json NOT NULL,
+  headers json NOT NULL DEFAULT '{}'::json,`;
+var JSONB_PAYLOAD_COLUMNS = `  payload jsonb NOT NULL,
+  headers jsonb NOT NULL DEFAULT '{}'::jsonb,`;
 function createPostgresSchemaStatements(options = {}) {
   const names = resolvePostgresNames(options);
+  const statements = schemaBodyStatements(names);
+  statements.push(`COMMENT ON TABLE ${names.lanes} IS '${version040Marker(names, statements)}'`);
+  return statements;
+}
+function schemaBodyStatements(names) {
   const statements = [...eventAndDeliveryStatements(names), ...laneAndCompletionStatements(names)];
   if (names.completionHistory) {
     statements.push(...completionHistoryStatements(names));
@@ -265,7 +275,6 @@ function createPostgresSchemaStatements(options = {}) {
   if (names.schema !== "public") {
     statements.unshift(`CREATE SCHEMA IF NOT EXISTS ${names.quotedSchema}`);
   }
-  statements.push(`COMMENT ON TABLE ${names.lanes} IS '${postgresSchemaMarker(statements)}'`);
   return statements;
 }
 function eventAndDeliveryStatements(names) {
@@ -279,14 +288,14 @@ function eventAndKeySequenceStatements(names) {
   source text NOT NULL,
   key text,
   sequence bigint,
-  payload jsonb NOT NULL,
-  headers jsonb NOT NULL DEFAULT '{}'::jsonb,
+${PAYLOAD_COLUMNS}
   correlation_id text,
   causation_id text,
   occurred_at timestamptz NOT NULL,
   created_at timestamptz NOT NULL,
   handlers text[] NOT NULL DEFAULT '{}'
 )`,
+    upgradePayloadToJson(names),
     `CREATE INDEX IF NOT EXISTS ${names.indexes.eventCreated} ON ${names.events} (created_at, id)`,
     `CREATE TABLE IF NOT EXISTS ${names.keySequences} (
   key text PRIMARY KEY,
@@ -382,6 +391,33 @@ BEGIN
 END
 $relaybus_upgrade$`;
 }
+function upgradePayloadToJson(names) {
+  return `DO $relaybus_payload_json$
+BEGIN
+  IF EXISTS (
+    SELECT 1 FROM pg_attribute
+    WHERE attrelid = to_regclass('${names.events}') AND attname IN ('payload', 'headers')
+      AND atttypid = 'jsonb'::regtype AND NOT attisdropped
+  ) THEN
+    ALTER TABLE ${names.events}
+      ALTER COLUMN headers DROP DEFAULT,
+      ALTER COLUMN payload TYPE json USING payload::json,
+      ALTER COLUMN headers TYPE json USING headers::json,
+      ALTER COLUMN headers SET DEFAULT '{}'::json;
+  END IF;
+END
+$relaybus_payload_json$`;
+}
+function version040SchemaMarker(options = {}) {
+  const names = resolvePostgresNames(options);
+  return version040Marker(names, schemaBodyStatements(names));
+}
+function version040Marker(names, statements) {
+  const conversion = upgradePayloadToJson(names);
+  return postgresSchemaMarker(
+    statements.filter((statement) => statement !== conversion).map((statement) => statement.replace(PAYLOAD_COLUMNS, JSONB_PAYLOAD_COLUMNS))
+  );
+}
 function createPostgresSchemaSql(options = {}) {
   return `${createPostgresSchemaStatements(options).join(";\n\n")};
 `;
@@ -1446,6 +1482,67 @@ var PostgresMaintenance = class {
     });
   }
 };
+
+// src/postgres-setup.ts
+var PostgresSetup = class {
+  #state;
+  /** The conversion of `jsonb` payload columns. */
+  #payloadUpgrade;
+  constructor(state, options) {
+    this.#state = state;
+    this.#payloadUpgrade = upgradePayloadToJson(resolvePostgresNames(options));
+  }
+  /** Implements {@link PostgresEventStore.setup}. */
+  async run() {
+    if (isCurrent(await this.#schemaState())) {
+      return;
+    }
+    await this.#underSetupLock(async (transaction) => {
+      await this.#run(this.#payloadUpgrade, transaction);
+    });
+    await this.#underSetupLock(async (transaction, schema) => {
+      if (schema.marked) {
+        await this.#run(this.#payloadUpgrade, transaction);
+        return;
+      }
+      for (const statement of this.#state.schemaStatements) {
+        await this.#run(statement, transaction);
+      }
+    });
+  }
+  /**
+   * Runs `work` in a transaction under the setup lock, unless the schema became current. The
+   * payload conversion goes first in a transaction of its own, so it holds no other table lock
+   * while it waits for the event table (a completion locks deliveries before the event rows).
+   */
+  async #underSetupLock(work) {
+    await this.#state.executor.transaction(async (transaction) => {
+      await this.#state.executor.query(
+        this.#state.statements.setup.setupLock,
+        [this.#state.setupLockKey],
+        transaction
+      );
+      const schema = await this.#schemaState(transaction);
+      if (!isCurrent(schema)) {
+        await work(transaction, schema);
+      }
+    });
+  }
+  async #run(statement, transaction) {
+    await this.#state.executor.query(statement, [], transaction);
+  }
+  async #schemaState(transaction) {
+    const rows = await this.#state.executor.query(
+      this.#state.statements.setup.schemaCurrent,
+      [this.#state.schemaMarker],
+      transaction
+    );
+    return { marked: rows[0]["marked"] === true, payloadJson: rows[0]["payload_json"] === true };
+  }
+};
+function isCurrent(schema) {
+  return schema.marked && schema.payloadJson;
+}
 var WAKE_UPS = ["notify", "poll"];
 var DEFAULT_PURGE_BATCH_SIZE = 1e3;
 function resolveStoreSettings(options) {
@@ -1520,8 +1617,8 @@ WHERE (SELECT count(*) FROM woken) >= 0`;
       reserved.sequence - count(*) OVER (PARTITION BY incoming.key)
         + row_number() OVER (PARTITION BY incoming.key ORDER BY incoming.ordinal)
     END AS sequence
-  FROM jsonb_to_recordset($1::jsonb) AS incoming (
-    ordinal int, id text, type text, source text, key text, payload jsonb, headers jsonb,
+  FROM json_to_recordset($1::json) AS incoming (
+    ordinal int, id text, type text, source text, key text, payload json, headers json,
     correlation_id text, causation_id text, occurred_at timestamptz, handlers text[])
   LEFT JOIN reserved ON reserved.key = incoming.key
 )`;
@@ -1532,7 +1629,7 @@ WHERE (SELECT count(*) FROM woken) >= 0`;
   INSERT INTO ${this.#names.events}
     (id, type, source, key, sequence, payload, headers, correlation_id, causation_id, occurred_at, created_at, handlers)
   SELECT incoming.id, incoming.type, incoming.source, incoming.key, incoming.sequence,
-    COALESCE(incoming.payload, 'null'::jsonb), COALESCE(incoming.headers, '{}'::jsonb),
+    COALESCE(incoming.payload, 'null'::json), COALESCE(incoming.headers, '{}'::json),
     incoming.correlation_id, incoming.causation_id, incoming.occurred_at, $2::timestamptz,
     COALESCE(incoming.handlers, '{}')
   FROM incoming
@@ -1585,11 +1682,11 @@ WHERE (SELECT count(*) FROM woken) >= 0`;
     return `INSERT INTO ${this.#names.events}
   (id, type, source, key, sequence, payload, headers, correlation_id, causation_id, occurred_at, created_at, handlers)
 SELECT incoming.id, incoming.type, incoming.source, incoming.key, incoming.sequence,
-  COALESCE(incoming.payload, 'null'::jsonb), COALESCE(incoming.headers, '{}'::jsonb),
+  COALESCE(incoming.payload, 'null'::json), COALESCE(incoming.headers, '{}'::json),
   incoming.correlation_id, incoming.causation_id, incoming.occurred_at, $2::timestamptz,
   COALESCE(incoming.handlers, '{}')
-FROM jsonb_to_recordset($1::jsonb) AS incoming (
-  id text, type text, source text, key text, sequence bigint, payload jsonb, headers jsonb,
+FROM json_to_recordset($1::json) AS incoming (
+  id text, type text, source text, key text, sequence bigint, payload json, headers json,
   correlation_id text, causation_id text, occurred_at timestamptz, handlers text[])
 ORDER BY incoming.id
 ON CONFLICT (id) DO NOTHING
@@ -2839,8 +2936,10 @@ var SetupStatements = class {
     return `relaybus:${this.#names.schema}`;
   }
   /**
-   * Whether the lane table carries `marker` as its comment, that is whether the schema statements
-   * that produce that marker already ran. Takes no table lock. Parameters: `[marker]`.
+   * The state of the schema, taking no table lock. Parameters: `[marker]`. Columns: `marked`, whether
+   * the lane table carries `marker` as its comment (the schema statements that produce it already
+   * ran), and `payload_json`, whether the payload and headers columns of the event table are both
+   * `json` (the conversion of the `jsonb` columns of schemas before 0.4.1 ran).
    */
   get schemaCurrent() {
     return `SELECT EXISTS (
@@ -2849,7 +2948,14 @@ var SetupStatements = class {
   JOIN pg_catalog.pg_description AS d
     ON d.objoid = c.oid AND d.classoid = 'pg_catalog.pg_class'::regclass AND d.objsubid = 0
   WHERE n.nspname = '${this.#names.schema}' AND c.relname = '${this.#names.lanesTableName}' AND d.description = $1::text
-) AS current`;
+) AS marked, (
+  SELECT count(*) FROM pg_catalog.pg_attribute AS a
+  JOIN pg_catalog.pg_class AS c ON c.oid = a.attrelid
+  JOIN pg_catalog.pg_namespace AS n ON n.oid = c.relnamespace
+  WHERE n.nspname = '${this.#names.schema}' AND c.relname = '${this.#names.eventsTableName}'
+    AND a.attname IN ('payload', 'headers') AND a.atttypid = 'pg_catalog.json'::pg_catalog.regtype
+    AND NOT a.attisdropped
+) = 2 AS payload_json`;
   }
   /**
    * Takes the setup lock. Returns a boolean column rather than the `void` of the lock function, which
@@ -2887,6 +2993,7 @@ var PostgresEventStore = class {
   #claims;
   #completion;
   #maintenance;
+  #setup;
   #owned = new OwnedConnections();
   constructor(options) {
     const { purgeBatchSize, wakeUp, queryOptions } = resolveStoreSettings(options);
@@ -2900,7 +3007,7 @@ var PostgresEventStore = class {
       statements,
       schemaStatements,
       setupLockKey: statements.setup.setupLockKey,
-      schemaMarker: postgresSchemaMarker(schemaStatements.slice(0, -1)),
+      schemaMarker: version040SchemaMarker(options),
       purgeBatchSize,
       completionHistory: names.completionHistory,
       logger,
@@ -2924,42 +3031,20 @@ var PostgresEventStore = class {
     this.#claims = new PostgresClaims(state);
     this.#completion = new PostgresCompletion(state);
     this.#maintenance = new PostgresMaintenance(state);
+    this.#setup = new PostgresSetup(state, options);
   }
   /**
    * Creates the schema, tables and indexes if they do not exist. Safe to call
    * from many processes at once: a transaction-scoped advisory lock serializes
    * the DDL, so racing `CREATE TABLE IF NOT EXISTS` statements cannot collide.
-   * When the lane table already carries the marker of the current statements
-   * ({@link postgresSchemaMarker}) nothing runs and no table is locked, so instances
+   * When the lane table already carries the schema marker ({@link version040SchemaMarker})
+   * and the payload columns are `json`, nothing runs and no table is locked, so instances
    * that start next to running dispatchers cannot deadlock with their claims.
    * Skip it (`setupStore: false`) when a migration tool applies
    * `createPostgresSchemaSql()` instead.
    */
   async setup() {
-    if (await this.#schemaIsCurrent()) {
-      return;
-    }
-    await this.#state.executor.transaction(async (transaction) => {
-      await this.#state.executor.query(
-        this.#state.statements.setup.setupLock,
-        [this.#state.setupLockKey],
-        transaction
-      );
-      if (await this.#schemaIsCurrent(transaction)) {
-        return;
-      }
-      for (const statement of this.#state.schemaStatements) {
-        await this.#state.executor.query(statement, [], transaction);
-      }
-    });
-  }
-  async #schemaIsCurrent(transaction) {
-    const rows = await this.#state.executor.query(
-      this.#state.statements.setup.schemaCurrent,
-      [this.#state.schemaMarker],
-      transaction
-    );
-    return rows[0]["current"] === true;
+    await this.#setup.run();
   }
   /**
    * Runs `work` in a transaction of the executor. With `options.timeoutMs` (from `bus.transaction`)