@nylorun/runtime 0.12.0-beta → 0.14.0-beta

package/dist/adapters/execution/restate.js

@@ -1,7 +1,7 @@
 /**
  * Durable Session Execution on Restate (architecture §12.3).
  *
- * The only module that imports the Restate SDK. It serves three virtual
+ * The only module that imports the Restate SDK. It serves four virtual
  * objects on the Worker endpoint, all named with `servicePrefix`:
  *
  * - `<prefix>NylorunSession`, key `<tenantId>:<sessionId>`. Its exclusive
@@ -17,9 +17,14 @@
  * - `<prefix>NylorunTimer`, key `<tenantId>:<timer key>`. `set` records the
  *   latest time and sends a delayed `fire`; a `fire` for an older time is a
  *   no-op, which is how setting a key again replaces it.
+ * - `<prefix>NylorunAction`, key `<tenantId>:<actionId>`. Its exclusive
+ *   `deliver` handler calls `WorkerHandlers.deliver`. Like `advance`, it keeps
+ *   no state and makes no `ctx.run`: its only journal entry is the delayed
+ *   self-send after a `retry` result, so an endpoint that is down never spends
+ *   the retry budget, which is for infrastructure errors.
  *
- * Wakes, timers and sweep arming go through the ingress as one-way sends, so
- * an API node can call them without ever calling `start`.
+ * Wakes, deliveries, timers and sweep arming go through the ingress as one-way
+ * sends, so an API node can call them without ever calling `start`.
  */
 import { createServer } from "node:http2";
 import * as restate from "@restatedev/restate-sdk";
@@ -49,6 +54,7 @@ export class RestateExecution {
             session: `${prefix}NylorunSession`,
             tenant: `${prefix}NylorunTenant`,
             timer: `${prefix}NylorunTimer`,
+            action: `${prefix}NylorunAction`,
         };
         this.ingressUrl = options.ingressUrl.replace(/\/+$/, "");
         this.adminUrl = options.adminUrl.replace(/\/+$/, "");
@@ -63,6 +69,9 @@ export class RestateExecution {
             throw new Error(`Unknown wake reason: ${String(wake.reason)}`);
         await this.send(this.names.session, sessionKey(tenantId, sessionId), "advance", { reason: wake.reason }, wake.dedupeKey);
     }
+    async deliver(tenantId, actionId) {
+        await this.send(this.names.action, sessionKey(tenantId, actionId), "deliver", {});
+    }
     async timer(tenantId, key, at) {
         await this.send(this.names.timer, sessionKey(tenantId, key), "set", {
             at: at.getTime(),
@@ -161,7 +170,7 @@ export class RestateExecution {
             handlers: {
                 advance: async (ctx, _wake) => {
                     const { tenantId, sessionId } = parseSessionKey(ctx.key);
-                    const result = await this.runAdvance(ctx.request().attemptCompletedSignal, (handlers, signal) => handlers.advance(tenantId, sessionId, signal));
+                    const result = await this.runAborted(ctx.request().attemptCompletedSignal, (handlers, signal) => handlers.advance(tenantId, sessionId, signal));
                     if (result.status === "busy")
                         ctx
                             .objectSendClient({ name: names.session }, ctx.key)
@@ -232,10 +241,31 @@ export class RestateExecution {
             },
             options: serviceOptions,
         });
-        return [session, tenant, timer];
+        const action = restate.object({
+            name: names.action,
+            handlers: {
+                deliver: async (ctx, _input) => {
+                    const { tenantId, sessionId: actionId } = parseSessionKey(ctx.key);
+                    const result = await this.runAborted(ctx.request().attemptCompletedSignal, (handlers, signal) => {
+                        if (!handlers.deliver)
+                            throw new Error("WorkerHandlers.deliver is required for deliveries");
+                        return handlers.deliver(tenantId, actionId, signal);
+                    });
+                    if (result.status === "retry")
+                        ctx
+                            .objectSendClient({ name: names.action }, ctx.key)
+                            .deliver({}, restate.rpc.sendOpts({ delay: Math.max(0, result.retryAfterMs) }));
+                },
+            },
+            options: serviceOptions,
+        });
+        return [session, tenant, timer, action];
     }
-    /** Runs `advance` with a signal aborted by `stop` or by the end of the Restate attempt. */
-    async runAdvance(attemptDone, fn) {
+    /**
+     * Runs a Worker handler with a signal aborted by `stop` or by the end of the Restate
+     * attempt (`advance` and `deliver`).
+     */
+    async runAborted(attemptDone, fn) {
         const controller = new AbortController();
         const onAttemptDone = () => controller.abort(new Error("Restate attempt ended"));
         if (attemptDone.aborted)
@@ -335,7 +365,7 @@ export async function listStuckInvocations(options) {
     const prefix = options.servicePrefix ?? "";
     if (!/^[A-Za-z0-9_]*$/.test(prefix))
         throw new Error(`Invalid Restate service prefix: ${prefix}`);
-    const services = ["NylorunSession", "NylorunTenant", "NylorunTimer"].map((name) => `'${prefix}${name}'`);
+    const services = ["NylorunSession", "NylorunTenant", "NylorunTimer", "NylorunAction"].map((name) => `'${prefix}${name}'`);
     const clauses = [
         `target_service_name IN (${services.join(", ")})`,
         `status IN ('paused', 'backing-off')`,

package/dist/adapters/replication/pgoutput.d.ts

@@ -0,0 +1,16 @@
+import type { ChangeSource } from "../../streams/relay/types.js";
+export interface PgoutputSourceOptions {
+    /** A Postgres URL whose role may replicate (`REPLICATION`) and read `nylorun_streams`. */
+    connectionString: string;
+    /** Default `nylorun_stream_relay`. */
+    slot?: string;
+    /** Default `nylorun_stream_relay`. */
+    publication?: string;
+    /** The schema of the record tables. Default `nylorun_streams`. */
+    schema?: string;
+    /** Wait between attempts to take the slot. Default 5 s. */
+    retryMs?: number;
+    log?: (message: string, fields?: Record<string, unknown>) => void;
+}
+export declare const DEFAULT_RELAY_SLOT = "nylorun_stream_relay";
+export declare function createPgoutputSource(options: PgoutputSourceOptions): ChangeSource;

package/dist/adapters/replication/pgoutput.js

@@ -0,0 +1,169 @@
+/**
+ * The stream relay's change source on Postgres logical replication (Durable Streams §7.1,
+ * §7.4, §7.6): a persistent `pgoutput` slot on the `nylorun_stream_relay` publication, read
+ * with `pg-logical-replication`. The only file importing it (with `pg`, which it needs).
+ *
+ * - **One reader per slot.** Postgres lets one connection hold a slot, so every Runtime
+ *   process can run a source and exactly one is active; the others retry every few seconds
+ *   and take over when it goes away. The slot is the leader election.
+ * - **Acknowledged only on request.** The library acknowledges the last position it
+ *   *received* on a timer; that is turned off (`timeoutSeconds: 0`) and keepalives are
+ *   answered by the relay, which acknowledges only what S2 has.
+ * - **Always from the confirmed position.** Each attempt is a new connection that starts at
+ *   `0/0`, which Postgres reads as the slot's confirmed position, so a reconnect replays
+ *   whatever was received but not acknowledged.
+ * - **Fresh slots.** A missing slot is created; a slot Postgres invalidated
+ *   (`max_slot_wal_keep_size`) is dropped and created again. Either way the relay must
+ *   reconcile the record with S2; that is recorded in `relay_slots` until `reconciled()`, so a
+ *   crash during reconciliation, or another process winning the slot, does not lose it.
+ */
+import pg from "pg";
+import { LogicalReplicationService, PgoutputPlugin } from "pg-logical-replication";
+export const DEFAULT_RELAY_SLOT = "nylorun_stream_relay";
+export function createPgoutputSource(options) {
+    const slot = options.slot ?? DEFAULT_RELAY_SLOT;
+    const publication = options.publication ?? "nylorun_stream_relay";
+    const schema = options.schema ?? "nylorun_streams";
+    const retryMs = options.retryMs ?? 5000;
+    const log = options.log ?? (() => { });
+    const pool = new pg.Pool({
+        connectionString: options.connectionString,
+        max: 2,
+        application_name: "nylorun-stream-relay",
+    });
+    let service;
+    let stopped = false;
+    let running;
+    let wake;
+    /** Creates the slot when missing or lost; true when the record must be reconciled. */
+    async function prepareSlot() {
+        const client = await pool.connect();
+        try {
+            const { rows } = await client.query("SELECT wal_status, active FROM pg_replication_slots WHERE slot_name = $1", [slot]);
+            const existing = rows[0];
+            if (existing?.wal_status === "lost" && !existing.active) {
+                log("replication slot was invalidated; creating it again", { slot });
+                await client.query("SELECT pg_drop_replication_slot($1)", [slot]);
+            }
+            if (!existing || existing.wal_status === "lost") {
+                try {
+                    await client.query("SELECT pg_create_logical_replication_slot($1, 'pgoutput')", [slot]);
+                    log("replication slot created", { slot });
+                    await client.query(`INSERT INTO ${schema}.relay_slots (slot_name, reconcile_pending) VALUES ($1, true)
+             ON CONFLICT (slot_name) DO UPDATE SET reconcile_pending = true`, [slot]);
+                }
+                catch (error) {
+                    // Another process created it first; its row says whether to reconcile.
+                    if (error.code !== "42710")
+                        throw error;
+                }
+            }
+            const pending = await client.query(`SELECT reconcile_pending FROM ${schema}.relay_slots WHERE slot_name = $1`, [slot]);
+            // A slot created before this table knew it: reconcile once to be sure.
+            return pending.rows[0]?.reconcile_pending ?? true;
+        }
+        finally {
+            client.release();
+        }
+    }
+    async function attempt(handlers) {
+        const fresh = await prepareSlot();
+        // `stop()` during `prepareSlot` had no service to stop: do not start one it cannot.
+        if (stopped)
+            return;
+        const current = new LogicalReplicationService({ connectionString: options.connectionString, application_name: "nylorun-stream-relay" }, { acknowledge: { auto: false, timeoutSeconds: 0 }, flowControl: { enabled: false } });
+        service = current;
+        let decoding;
+        current.on("start", () => handlers.onActive({ fresh }));
+        current.on("data", (_lsn, message) => {
+            switch (message.tag) {
+                case "begin":
+                    decoding = { rows: [] };
+                    break;
+                case "insert":
+                    if (decoding &&
+                        message.relation?.schema === schema &&
+                        message.relation?.name === "session_events")
+                        decoding.rows.push(rowOf(message.new));
+                    break;
+                case "commit": {
+                    const tx = { endLsn: message.commitEndLsn, rows: decoding?.rows ?? [] };
+                    decoding = undefined;
+                    // A transaction with no record rows still moves the slot once acknowledged.
+                    handlers.onTx(tx);
+                    break;
+                }
+            }
+        });
+        current.on("heartbeat", (lsn, _timestamp, shouldRespond) => {
+            // Mid-transaction, the server's position may be past rows not yet delivered.
+            if (!shouldRespond || decoding)
+                return;
+            const position = handlers.onKeepalive(lsn);
+            if (position)
+                void current.acknowledge(position).catch(() => { });
+        });
+        current.on("error", () => { });
+        const plugin = new PgoutputPlugin({ protoVersion: 1, publicationNames: [publication] });
+        // Resolves when replication ends, rejects when it fails; `0/0` = the slot's confirmed position.
+        await current.subscribe(plugin, slot, "0/00000000");
+    }
+    async function run(handlers) {
+        while (!stopped) {
+            let failure;
+            try {
+                await attempt(handlers);
+            }
+            catch (error) {
+                failure = error;
+            }
+            if (stopped)
+                break;
+            const busy = failure?.code === "55006";
+            // Another process holds the slot: it is the relay. Anything else: report and retry.
+            handlers.onInactive(busy ? undefined : failure ?? new Error("replication ended"));
+            await new Promise((resolve) => {
+                wake = resolve;
+                setTimeout(resolve, retryMs).unref?.();
+            });
+            wake = undefined;
+        }
+    }
+    return {
+        start(handlers) {
+            if (running)
+                throw new Error("The change source is already started");
+            running = run(handlers);
+        },
+        acknowledge(lsn) {
+            void service?.acknowledge(lsn).catch(() => { });
+        },
+        async reconciled() {
+            await pool.query(`UPDATE ${schema}.relay_slots SET reconcile_pending = false WHERE slot_name = $1`, [
+                slot,
+            ]);
+        },
+        async lag() {
+            const { rows } = await pool.query(`SELECT pg_wal_lsn_diff(pg_current_wal_lsn(), confirmed_flush_lsn)::text AS lag
+         FROM pg_replication_slots WHERE slot_name = $1`, [slot]);
+            return rows[0]?.lag == null ? undefined : Number(rows[0].lag);
+        },
+        async stop() {
+            stopped = true;
+            wake?.();
+            await service?.stop().catch(() => undefined);
+            await running?.catch(() => undefined);
+            await pool.end().catch(() => undefined);
+        },
+    };
+}
+/** A decoded `session_events` row (`pg` parses `json`; `bigint` arrives as text). */
+function rowOf(values) {
+    return {
+        tenantId: values.tenant_id,
+        sessionId: values.session_id,
+        seq: Number(values.seq),
+        generation: Number(values.generation),
+        body: typeof values.body === "string" ? JSON.parse(values.body) : values.body,
+    };
+}

package/dist/adapters/streams/s2.js

@@ -5,12 +5,12 @@
  *
  * ## Layout
  *
- * - One basin per Tenant, named by `tenantBasinName` (`streams/basin.ts`):
- *   `<basinPrefix>tn-<ulid>` for a Tenant id `tn_<ulid>`.
- * - The basin is created with `createStreamOnAppend`, so `sessions/<id>/<incarnation>`,
- *   `tenant/work` and `tenant/control` come into existence on their first
- *   append. Its default stream config sets infinite retention, which session
- *   streams keep; the work and control streams are created with a one-day
+ * - One basin per Tenant basin generation, named by `tenantBasinName` (`streams/basin.ts`):
+ *   `<basinPrefix>tn-<ulid>` for a Tenant id `tn_<ulid>`, `…-<g base36>` for generation g.
+ * - The basin is created with `createStreamOnAppend` and `createStreamOnRead`, so
+ *   `sessions/<id>` and `tenant/control` come into existence on first use. Its
+ *   default stream config sets infinite retention, which session streams
+ *   keep; the control stream is created with a one-day
  *   age-based retention instead (signals are latency hints, not history).
  * - Bodies are JSON text in string records. Record timestamps are S2 arrival
  *   times.
@@ -27,7 +27,7 @@
  *   explicit `{ infinite: {} }` retention to keep session history; this adapter
  *   sets it as the basin default, and s2-lite honours it (a new stream reports
  *   `infinite`). A `streamConfig` sent with an append that creates the stream
- *   is honoured too (used for the work and control streams). `deleteOnEmpty`
+ *   is honoured too (used for the control stream). `deleteOnEmpty`
  *   defaults to disabled (`minAgeSecs: 0`).
  * - **Stream deletion** is immediate for our purposes: right after `DELETE`,
  *   `checkTail` and reads return 404 `stream_not_found` (new read sessions
@@ -58,9 +58,9 @@
  *   use the SDK's HTTP/2 (s2s) transport, which s2-lite serves over cleartext.
  */
 import { AppendInput, AppendRecord, RangeNotSatisfiableError, S2, S2Error, SeqNumMismatchError, } from "@s2-dev/streamstore";
-import { tenantBasinName, validateBasinPrefix } from "../../streams/basin.js";
-import { CONTROL_STREAM, WORK_STREAM, } from "../../streams/types.js";
-/** Retention of the work and control signal streams. */
+import { parseBasin, tenantBasinName, validateBasinPrefix } from "../../streams/basin.js";
+import { CONTROL_STREAM, } from "../../streams/types.js";
+/** Retention of the control signal stream. */
 const SIGNAL_RETENTION_SECS = 24 * 60 * 60;
 /** Records per unary read while reading history. */
 const HISTORY_PAGE = 1000;
@@ -99,7 +99,7 @@ class S2Streams {
             return AppendRecord.string({ body });
         }), {
             ...(options.matchSeq !== undefined ? { matchSeqNum: options.matchSeq } : {}),
-            ...(stream === WORK_STREAM || stream === CONTROL_STREAM
+            ...(stream === CONTROL_STREAM
                 ? { streamConfig: { retentionPolicy: { ageSecs: SIGNAL_RETENTION_SECS } } }
                 : {}),
         });
@@ -155,6 +155,9 @@ class S2Streams {
                     basin,
                     config: {
                         createStreamOnAppend: true,
+                        // A reader that arrives before a session's first event creates the
+                        // stream and follows it live, instead of polling until it exists.
+                        createStreamOnRead: true,
                         defaultStreamConfig: { retentionPolicy: { infinite: {} } },
                     },
                 }, { signal: this.closing.signal });
@@ -171,15 +174,24 @@ class S2Streams {
             }
         }
     }
-    async deleteTenant(tenantId) {
+    async deleteTenant(tenantId, options = {}) {
         this.checkOpen();
-        try {
-            await this.s2.basins.delete({ basin: this.basinName(tenantId) }, { signal: this.closing.signal });
-        }
-        catch (error) {
-            if (!isMissing(error))
-                throw error;
+        const basins = [this.basinName(tenantId)];
+        // Every later generation's basin too: `<basin>-<g base36>`.
+        if (options.allGenerations && parseBasin(tenantId).generation === 0) {
+            const later = new RegExp(`^${escapeRegExp(basins[0])}-[0-9a-z]{1,2}$`);
+            for await (const info of this.s2.basins.listAll({ prefix: `${basins[0]}-` }, { signal: this.closing.signal }))
+                if (later.test(info.name) && info.deletedAt == null)
+                    basins.push(info.name);
         }
+        for (const basin of basins)
+            try {
+                await this.s2.basins.delete({ basin }, { signal: this.closing.signal });
+            }
+            catch (error) {
+                if (!isMissing(error))
+                    throw error;
+            }
     }
     async deleteStream(tenantId, stream) {
         this.checkOpen();
@@ -256,7 +268,9 @@ class S2Streams {
     /**
      * Follows `stream` from `fromSeq` with S2 read sessions, reopening a session
      * from the next sequence whenever one ends or fails, until aborted, closed,
-     * or the Tenant's basin is gone. A missing stream is polled until it exists.
+     * or the Tenant's basin is gone. Basins create a stream on read, so a reader
+     * that arrives before the first append waits on the live session instead of
+     * polling.
      */
     async *readLive(tenantId, stream, fromSeq, signal) {
         const aborted = anySignal(signal, this.closing.signal);
@@ -288,7 +302,7 @@ class S2Streams {
                     if ((isMissing(error) || isCode(error, "stream_deletion_pending")) &&
                         (await this.basinGone(tenantId, aborted)))
                         return;
-                    // Stream not created yet, or S2 unreachable: retry below.
+                    // S2 unreachable, or a stream being deleted: retry below.
                 }
                 await sleep(delay, aborted);
                 delay = Math.min(delay * 2, RETRY_MAX_MS);
@@ -343,6 +357,9 @@ function isTransactionConflict(error) {
 function isTenantGone(error) {
     return isCode(error, "basin_not_found") || isCode(error, "basin_deletion_pending");
 }
+function escapeRegExp(text) {
+    return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
 function anySignal(...signals) {
     return AbortSignal.any(signals.filter((s) => s !== undefined));
 }