@nylorun/runtime 0.12.0-beta → 0.14.0-beta

package/dist/store/postgres/migrations/006_drop_executors.d.ts

@@ -0,0 +1,11 @@
+import type { Migration } from "./index.js";
+/**
+ * Protocol 3 removes executors: Actions are only delivered to Action endpoints.
+ *
+ * An Action an executor had claimed becomes `delivering` with a deadline that has already
+ * passed, so the next sweep applies the delivery loss rule to it (`delivery.ts` `loseAction`):
+ * a tool becomes `uncertain` with its `action.uncertain` event, and a hook, `fn` or `verify` is
+ * delivered again. Open Actions lose the executor-only `claimId` and `leaseExpiresAt`; settled
+ * ones are never delivered again and keep them.
+ */
+export declare const dropExecutors: Migration;

package/dist/store/postgres/migrations/006_drop_executors.js

@@ -0,0 +1,32 @@
+/**
+ * Protocol 3 removes executors: Actions are only delivered to Action endpoints.
+ *
+ * An Action an executor had claimed becomes `delivering` with a deadline that has already
+ * passed, so the next sweep applies the delivery loss rule to it (`delivery.ts` `loseAction`):
+ * a tool becomes `uncertain` with its `action.uncertain` event, and a hook, `fn` or `verify` is
+ * delivered again. Open Actions lose the executor-only `claimId` and `leaseExpiresAt`; settled
+ * ones are never delivered again and keep them.
+ */
+export const dropExecutors = {
+    version: 6,
+    name: "drop_executors",
+    up: (s) => `
+    DROP INDEX ${s}.actions_lease;
+    ALTER TABLE ${s}.actions DROP COLUMN lease_expires_at;
+
+    UPDATE ${s}.actions
+      SET body = (
+        ${s}.doc(body) - 'claimId' - 'leaseExpiresAt'
+          || jsonb_build_object(
+            'status', 'delivering',
+            'deadlineAt', to_char(now() AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')
+          )
+      )::json
+      WHERE status = 'claimed';
+    UPDATE ${s}.actions
+      SET body = (${s}.doc(body) - 'claimId' - 'leaseExpiresAt')::json
+      WHERE status IN ('pending', 'delivering');
+
+    DROP TABLE ${s}.executors;
+  `,
+};

package/dist/store/postgres/migrations/007_event_time.d.ts

@@ -0,0 +1,7 @@
+import type { Migration } from "./index.js";
+/**
+ * Protocol 4 puts events on the `nylorun.event/2` envelope, where `createdAt` is `time`. The
+ * outbox's `created_at` (its age, for relay lag) is generated from either, so rows written
+ * before the upgrade keep theirs.
+ */
+export declare const eventTime: Migration;

package/dist/store/postgres/migrations/007_event_time.js

@@ -0,0 +1,14 @@
+/**
+ * Protocol 4 puts events on the `nylorun.event/2` envelope, where `createdAt` is `time`. The
+ * outbox's `created_at` (its age, for relay lag) is generated from either, so rows written
+ * before the upgrade keep theirs.
+ */
+export const eventTime = {
+    version: 7,
+    name: "event_time",
+    up: (s) => `
+    ALTER TABLE ${s}.outbox DROP COLUMN created_at;
+    ALTER TABLE ${s}.outbox ADD COLUMN created_at text COLLATE "C"
+      GENERATED ALWAYS AS (coalesce(${s}.doc(body)->>'time', ${s}.doc(body)->>'createdAt')) STORED;
+  `,
+};

package/dist/store/postgres/migrations/008_durable_streams.d.ts

@@ -0,0 +1,13 @@
+import type { Migration } from "./index.js";
+/**
+ * Durable Streams v1 (Durable Streams §18.1): session events move from the Tenant's outbox to
+ * the shared record (`nylorun_streams`), and stream names lose their incarnation.
+ *
+ * **A fresh start for session data.** A Tenant that has sessions loses them, with their
+ * commands, checkpoints, effects, Actions, links, outbox and turn buckets, as a
+ * sessions-scope reset does; settings, agents, Action endpoints, keys, policy and vaults stay.
+ * It moves to basin generation 1, and generation 0 (the basin holding the old streams) is
+ * retired, so the Runtime deletes it when the Tenant opens. A Tenant without sessions stays at
+ * generation 0: its basin holds no session stream it could collide with.
+ */
+export declare const durableStreams: Migration;

package/dist/store/postgres/migrations/008_durable_streams.js

@@ -0,0 +1,35 @@
+/**
+ * Durable Streams v1 (Durable Streams §18.1): session events move from the Tenant's outbox to
+ * the shared record (`nylorun_streams`), and stream names lose their incarnation.
+ *
+ * **A fresh start for session data.** A Tenant that has sessions loses them, with their
+ * commands, checkpoints, effects, Actions, links, outbox and turn buckets, as a
+ * sessions-scope reset does; settings, agents, Action endpoints, keys, policy and vaults stay.
+ * It moves to basin generation 1, and generation 0 (the basin holding the old streams) is
+ * retired, so the Runtime deletes it when the Tenant opens. A Tenant without sessions stays at
+ * generation 0: its basin holds no session stream it could collide with.
+ */
+export const durableStreams = {
+    version: 8,
+    name: "durable_streams",
+    up: (s) => `
+    ALTER TABLE ${s}.tenant
+      ADD COLUMN basin_generation integer NOT NULL DEFAULT 0,
+      ADD COLUMN retired_generations integer[] NOT NULL DEFAULT '{}';
+    UPDATE ${s}.tenant SET basin_generation = 1, retired_generations = '{0}'
+      WHERE EXISTS (SELECT 1 FROM ${s}.sessions);
+
+    DELETE FROM ${s}.links;
+    DELETE FROM ${s}.actions;
+    DELETE FROM ${s}.effects;
+    DELETE FROM ${s}.checkpoints;
+    DELETE FROM ${s}.commands;
+    DELETE FROM ${s}.sessions;
+    DELETE FROM ${s}.subject_usage;
+    DROP TABLE ${s}.outbox;
+
+    ALTER TABLE ${s}.sessions
+      DROP COLUMN stream_incarnation,
+      DROP COLUMN next_event_seq;
+  `,
+};

package/dist/store/postgres/migrations/index.d.ts

@@ -24,7 +24,11 @@ export declare const MIGRATIONS: readonly Migration[];
 export declare const POSTGRES_SCHEMA_VERSION: number;
 /** Checks that versions run 1, 2, 3, … Throws otherwise. */
 export declare function assertMigrations(migrations: readonly Migration[]): void;
-/** Takes the schema's migration lock until the transaction ends. */
+/**
+ * Takes the schema's migration lock until the transaction ends. Waiting for it, and the
+ * migrations or deletion done under it, may outlast the pool's statement timeout: it is lifted
+ * for the transaction, since a migration that times out would quarantine the Tenant.
+ */
 export declare function lockSchema(tx: TransactionSql, schema: string): Promise<void>;
 export declare function schemaExists(sql: Sql | TransactionSql, schema: string): Promise<boolean>;
 /**

package/dist/store/postgres/migrations/index.js

@@ -2,7 +2,22 @@ import { quarantine } from "../../../tenant/quarantine.js";
 import { quoteIdentifier, tenantIdFromSchema } from "../names.js";
 import { initial } from "./001_initial.js";
 import { sessionOwner } from "./002_session_owner.js";
-export const MIGRATIONS = [initial, sessionOwner];
+import { subjectTokens } from "./003_subject_tokens.js";
+import { publishableKeys } from "./004_publishable_keys.js";
+import { actionEndpoints } from "./005_action_endpoints.js";
+import { dropExecutors } from "./006_drop_executors.js";
+import { eventTime } from "./007_event_time.js";
+import { durableStreams } from "./008_durable_streams.js";
+export const MIGRATIONS = [
+    initial,
+    sessionOwner,
+    subjectTokens,
+    publishableKeys,
+    actionEndpoints,
+    dropExecutors,
+    eventTime,
+    durableStreams,
+];
 /** The schema version this Runtime writes and expects. */
 export const POSTGRES_SCHEMA_VERSION = MIGRATIONS.at(-1).version;
 /** Checks that versions run 1, 2, 3, … Throws otherwise. */
@@ -12,8 +27,13 @@ export function assertMigrations(migrations) {
             throw new Error(`Migration ${migration.name} has version ${migration.version}, expected ${index + 1}`);
     });
 }
-/** Takes the schema's migration lock until the transaction ends. */
+/**
+ * Takes the schema's migration lock until the transaction ends. Waiting for it, and the
+ * migrations or deletion done under it, may outlast the pool's statement timeout: it is lifted
+ * for the transaction, since a migration that times out would quarantine the Tenant.
+ */
 export async function lockSchema(tx, schema) {
+    await tx `SET LOCAL statement_timeout = 0`;
     await tx `SELECT pg_advisory_xact_lock(hashtextextended(${`nylorun.schema:${schema}`}, 0))`;
 }
 export async function schemaExists(sql, schema) {

package/dist/store/postgres/migrations/shared/index.d.ts

@@ -0,0 +1,31 @@
+/**
+ * Forward-only migrations of the shared `nylorun_streams` schema (Durable Streams §6): the
+ * record of every Tenant's session events, the log heads, the relay's state and the
+ * publication the stream relay reads. Unlike Tenant schemas, there is one for the database;
+ * the Host migrates it at startup, before any Tenant opens, under an advisory lock.
+ *
+ * Adding a migration: append to `STREAMS_MIGRATIONS` with the next version. Never edit or
+ * reorder an applied one.
+ */
+import type { Sql } from "postgres";
+import { type Migration } from "../index.js";
+export declare const STREAMS_SCHEMA = "nylorun_streams";
+/** The publication the stream relay subscribes to. */
+export declare const STREAMS_PUBLICATION = "nylorun_stream_relay";
+export declare const STREAMS_MIGRATIONS: readonly Migration[];
+/** The shared schema version this Runtime writes and expects. */
+export declare const STREAMS_SCHEMA_VERSION: number;
+/**
+ * Creates `nylorun_streams` when missing and applies every missing migration, in one
+ * transaction under the schema's migration lock. Throws when the schema is newer than this
+ * Runtime.
+ */
+export declare function migrateStreamsSchema(sql: Sql, migrations?: readonly Migration[]): Promise<{
+    from: number;
+    to: number;
+}>;
+/**
+ * Throws, naming the setting, when Postgres cannot run the stream relay: logical decoding
+ * needs `wal_level = logical` (a restart) and a role allowed to replicate.
+ */
+export declare function assertLogicalReplication(sql: Sql): Promise<void>;

package/dist/store/postgres/migrations/shared/index.js

@@ -0,0 +1,92 @@
+import { assertMigrations, lockSchema } from "../index.js";
+export const STREAMS_SCHEMA = "nylorun_streams";
+/** The publication the stream relay subscribes to. */
+export const STREAMS_PUBLICATION = "nylorun_stream_relay";
+export const STREAMS_MIGRATIONS = [
+    {
+        version: 1,
+        name: "record",
+        up: (s) => `
+      CREATE TABLE ${s}.session_log_heads (
+        tenant_id text COLLATE "C" NOT NULL,
+        session_id text COLLATE "C" NOT NULL,
+        -- The basin generation the session's events go to.
+        generation integer NOT NULL,
+        -- The next seq.
+        head bigint NOT NULL DEFAULT 0,
+        PRIMARY KEY (tenant_id, session_id)
+      );
+      CREATE TABLE ${s}.session_events (
+        tenant_id text COLLATE "C" NOT NULL,
+        session_id text COLLATE "C" NOT NULL,
+        seq bigint NOT NULL,
+        generation integer NOT NULL,
+        type text NOT NULL,
+        -- The nylorun.event/2 envelope, as written.
+        body json NOT NULL,
+        committed_at timestamptz NOT NULL DEFAULT now(),
+        PRIMARY KEY (tenant_id, session_id, seq)
+      );
+      -- One row per replication slot: whether the record still has to be reconciled with S2
+      -- after the slot was created (a crash during reconciliation leaves it pending).
+      CREATE TABLE ${s}.relay_slots (
+        slot_name text PRIMARY KEY,
+        reconcile_pending boolean NOT NULL
+      );
+      CREATE PUBLICATION ${STREAMS_PUBLICATION}
+        FOR TABLE ${s}.session_events WITH (publish = 'insert');
+    `,
+    },
+];
+/** The shared schema version this Runtime writes and expects. */
+export const STREAMS_SCHEMA_VERSION = STREAMS_MIGRATIONS.at(-1).version;
+/**
+ * Creates `nylorun_streams` when missing and applies every missing migration, in one
+ * transaction under the schema's migration lock. Throws when the schema is newer than this
+ * Runtime.
+ */
+export async function migrateStreamsSchema(sql, migrations = STREAMS_MIGRATIONS) {
+    assertMigrations(migrations);
+    const result = await sql.begin(async (tx) => {
+        await lockSchema(tx, STREAMS_SCHEMA);
+        return { result: await migrateInTx(tx, migrations) };
+    });
+    return result.result;
+}
+async function migrateInTx(tx, migrations) {
+    const s = STREAMS_SCHEMA;
+    await tx.unsafe(`
+    CREATE SCHEMA IF NOT EXISTS ${s};
+    CREATE TABLE IF NOT EXISTS ${s}.schema_version (
+      version integer PRIMARY KEY,
+      name text NOT NULL,
+      applied_at timestamptz NOT NULL DEFAULT now()
+    );`);
+    const [row] = await tx `
+    SELECT max(version)::int AS version FROM ${tx(`${s}.schema_version`)}`;
+    const from = row?.version ?? 0;
+    const latest = migrations.length;
+    if (from > latest)
+        throw new Error(`The ${s} schema is at version ${from}, newer than this Runtime's ${latest}: upgrade the Runtime`);
+    for (const migration of migrations.slice(from)) {
+        await tx.unsafe(migration.up(s));
+        await tx `INSERT INTO ${tx(`${s}.schema_version`)} (version, name)
+             VALUES (${migration.version}, ${migration.name})`;
+    }
+    return { from, to: latest };
+}
+/**
+ * Throws, naming the setting, when Postgres cannot run the stream relay: logical decoding
+ * needs `wal_level = logical` (a restart) and a role allowed to replicate.
+ */
+export async function assertLogicalReplication(sql) {
+    const [row] = await sql `
+    SELECT current_setting('wal_level') AS wal_level,
+           (SELECT rolreplication OR rolsuper FROM pg_roles WHERE rolname = current_user) AS replicates`;
+    if (row?.wal_level !== "logical")
+        throw new Error(`Postgres has wal_level = ${row?.wal_level ?? "unknown"}; the stream relay needs logical replication. ` +
+            "Set wal_level = logical and restart Postgres (`nylorun start` does this for the local stack; " +
+            "on a managed Postgres, turn on its logical replication option). See DEPLOYMENT.md.");
+    if (!row.replicates)
+        throw new Error("The Runtime's Postgres role cannot replicate: grant it REPLICATION (ALTER ROLE … REPLICATION). See DEPLOYMENT.md.");
+}

package/dist/store/postgres/record.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Reads the shared record back for the stream relay (Durable Streams §7.2–7.3): a session's
+ * rows to refill a gap in S2, every log head to reconcile after a fresh slot, and a Tenant's
+ * current basin generation to tell obsolete rows apart. Reads every Tenant's rows by design.
+ */
+import type { Sql } from "postgres";
+import type { RecordReader } from "../../streams/relay/types.js";
+/** Every Tenant's record (the stream relay), or one Tenant's (`tenantId`, its Session Store). */
+export declare function createPostgresRecordReader(sql: Sql, options?: {
+    tenantId?: string;
+}): RecordReader;

package/dist/store/postgres/record.js

@@ -0,0 +1,64 @@
+import { STREAMS_SCHEMA } from "./migrations/shared/index.js";
+import { quoteIdentifier, tenantSchemaName } from "./names.js";
+/** Every Tenant's record (the stream relay), or one Tenant's (`tenantId`, its Session Store). */
+export function createPostgresRecordReader(sql, options = {}) {
+    const only = options.tenantId;
+    const events = sql(`${STREAMS_SCHEMA}.session_events`);
+    const heads = sql(`${STREAMS_SCHEMA}.session_log_heads`);
+    return {
+        async readRange(tenantId, sessionId, from, to) {
+            const rows = await sql `
+        SELECT seq, generation, body FROM ${events}
+        WHERE tenant_id = ${tenantId} AND session_id = ${sessionId}
+          AND seq >= ${from} AND seq < ${to}
+        ORDER BY seq`;
+            return rows.map((row) => ({
+                tenantId,
+                sessionId,
+                seq: Number(row.seq),
+                generation: row.generation,
+                body: row.body,
+            }));
+        },
+        async heads(after, limit) {
+            const rows = await sql `
+        SELECT tenant_id, session_id, generation, head FROM ${heads}
+        WHERE head > 0
+          ${only === undefined ? sql `` : sql `AND tenant_id = ${only}`}
+          ${after ? sql `AND (tenant_id, session_id) > (${after.tenantId}, ${after.sessionId})` : sql ``}
+        ORDER BY tenant_id, session_id
+        LIMIT ${limit}`;
+            return rows.map((row) => ({
+                tenantId: row.tenant_id,
+                sessionId: row.session_id,
+                generation: row.generation,
+                head: Number(row.head),
+            }));
+        },
+        async generation(tenantId) {
+            let schema;
+            try {
+                schema = tenantSchemaName(tenantId);
+            }
+            catch {
+                return undefined;
+            }
+            const [exists] = await sql `
+        SELECT to_regclass(${`${quoteIdentifier(schema)}.tenant`}) IS NOT NULL AS exists`;
+            if (!exists?.exists)
+                return undefined;
+            try {
+                const [row] = await sql `
+          SELECT basin_generation FROM ${sql(`${schema}.tenant`)}`;
+                // A schema without its Tenant row (store tests) is at generation 0.
+                return row ? Number(row.basin_generation) : 0;
+            }
+            catch (error) {
+                // Deleted between the two reads.
+                if (error.code === "42P01")
+                    return undefined;
+                throw error;
+            }
+        },
+    };
+}