@nylorun/runtime 0.9.0-beta → 0.11.0-beta

package/dist/adapters/execution/restate.js

@@ -0,0 +1,442 @@
+/**
+ * Durable Session Execution on Restate (architecture §12.3).
+ *
+ * The only module that imports the Restate SDK. It serves three virtual
+ * objects on the Worker endpoint, all named with `servicePrefix`:
+ *
+ * - `<prefix>NylorunSession`, key `<tenantId>:<sessionId>`. Its exclusive
+ *   `advance` handler calls `WorkerHandlers.advance`. It makes no `ctx.run`,
+ *   keeps no state and uses no awakeables: the only journal entry it can write
+ *   is the delayed self-send after a `busy` result. Restate's one invocation
+ *   per key at a time is the "one advance per key" guarantee.
+ * - `<prefix>NylorunTenant`, key `<tenantId>`. `arm`, `sweep` and `disarm`
+ *   keep one self-re-arming sweep chain per Tenant. The chain's generation is
+ *   kept in the object's state, so arming is idempotent and a disarmed or
+ *   replaced chain dies at its next link. The state is rebuildable: Workers
+ *   re-arm every Tenant at startup (§14.8).
+ * - `<prefix>NylorunTimer`, key `<tenantId>:<timer key>`. `set` records the
+ *   latest time and sends a delayed `fire`; a `fire` for an older time is a
+ *   no-op, which is how setting a key again replaces it.
+ *
+ * Wakes, timers and sweep arming go through the ingress as one-way sends, so
+ * an API node can call them without ever calling `start`.
+ */
+import { createServer } from "node:http2";
+import * as restate from "@restatedev/restate-sdk";
+import { WAKE_REASONS, parseSessionKey, sessionKey, } from "../../execution/types.js";
+const HOUR_MS = 60 * 60 * 1000;
+export function createRestateExecution(options) {
+    return new RestateExecution(options);
+}
+export class RestateExecution {
+    options;
+    handlers;
+    stopping = false;
+    server;
+    sessions = new Set();
+    controllers = new Set();
+    inflight = new Set();
+    names;
+    ingressUrl;
+    adminUrl;
+    sweepIntervalMs;
+    constructor(options) {
+        this.options = options;
+        const prefix = options.servicePrefix ?? "";
+        if (!/^[A-Za-z0-9_]*$/.test(prefix))
+            throw new Error(`Invalid Restate service prefix: ${prefix}`);
+        this.names = {
+            session: `${prefix}NylorunSession`,
+            tenant: `${prefix}NylorunTenant`,
+            timer: `${prefix}NylorunTimer`,
+        };
+        this.ingressUrl = options.ingressUrl.replace(/\/+$/, "");
+        this.adminUrl = options.adminUrl.replace(/\/+$/, "");
+        this.sweepIntervalMs = options.sweepIntervalMs ?? 5000;
+    }
+    /** Service names this execution registers, for diagnostics and tests. */
+    get serviceNames() {
+        return { ...this.names };
+    }
+    async wake(tenantId, sessionId, wake) {
+        if (!WAKE_REASONS.includes(wake.reason))
+            throw new Error(`Unknown wake reason: ${String(wake.reason)}`);
+        await this.send(this.names.session, sessionKey(tenantId, sessionId), "advance", { reason: wake.reason }, wake.dedupeKey);
+    }
+    async timer(tenantId, key, at) {
+        await this.send(this.names.timer, sessionKey(tenantId, key), "set", {
+            at: at.getTime(),
+        });
+    }
+    async armSweep(tenantId) {
+        await this.send(this.names.tenant, tenantKey(tenantId), "arm", {});
+    }
+    async disarmSweep(tenantId) {
+        await this.send(this.names.tenant, tenantKey(tenantId), "disarm", {});
+    }
+    /**
+     * Readiness: the admin API (`/health`), which `start` registers with, and the
+     * ingress (`/restate/health`), which every wake, timer and sweep goes
+     * through, both answer.
+     */
+    async probe(signal) {
+        await Promise.all([`${this.adminUrl}/health`, `${this.ingressUrl}/restate/health`].map(async (url) => {
+            const response = await fetch(url, { signal });
+            await response.body?.cancel();
+            if (!response.ok)
+                throw new Error(`Restate health ${response.status} at ${url}`);
+        }));
+    }
+    /** This Runtime's paused or backing-off invocations for `tenantId` (Tenant status). */
+    stuckInvocations(tenantId) {
+        return listStuckInvocations({
+            adminUrl: this.adminUrl,
+            servicePrefix: this.options.servicePrefix ?? "",
+            tenantId,
+        });
+    }
+    async start(handlers) {
+        if (this.handlers)
+            throw new Error("DurableExecution already started");
+        const { workerListen, workerAdvertisedUrl } = this.options;
+        if (!workerListen || !workerAdvertisedUrl)
+            throw new Error("Restate execution needs workerListen and workerAdvertisedUrl to start");
+        this.handlers = handlers;
+        this.stopping = false;
+        const server = createServer(restate.createEndpointHandler({
+            services: this.definitions(),
+            ...(this.options.identityKeys?.length
+                ? { identityKeys: this.options.identityKeys }
+                : {}),
+            ...(this.options.logger ? { logger: sdkLogger(this.options.logger) } : {}),
+        }));
+        server.on("session", (session) => {
+            this.sessions.add(session);
+            session.once("close", () => this.sessions.delete(session));
+        });
+        await new Promise((resolve, reject) => {
+            server.once("error", reject);
+            server.listen(workerListen.port, workerListen.host, () => {
+                server.off("error", reject);
+                resolve();
+            });
+        });
+        this.server = server;
+        try {
+            await this.register(workerAdvertisedUrl);
+        }
+        catch (error) {
+            await this.stop();
+            throw error;
+        }
+    }
+    async stop() {
+        this.stopping = true;
+        for (const controller of this.controllers)
+            controller.abort(new Error("Worker stopping"));
+        while (this.inflight.size > 0)
+            await Promise.allSettled(this.inflight);
+        this.handlers = undefined;
+        const server = this.server;
+        this.server = undefined;
+        if (!server)
+            return;
+        // Let Restate read the responses already written, then drop the
+        // connections it keeps open so `close` can finish.
+        const closed = new Promise((resolve) => server.close(() => resolve()));
+        for (const session of this.sessions)
+            session.close();
+        const timeout = setTimeout(() => {
+            for (const session of this.sessions)
+                session.destroy();
+        }, 2000);
+        await closed;
+        clearTimeout(timeout);
+    }
+    definitions() {
+        const names = this.names;
+        const serviceOptions = this.serviceOptions();
+        const session = restate.object({
+            name: names.session,
+            handlers: {
+                advance: async (ctx, _wake) => {
+                    const { tenantId, sessionId } = parseSessionKey(ctx.key);
+                    const result = await this.runAdvance(ctx.request().attemptCompletedSignal, (handlers, signal) => handlers.advance(tenantId, sessionId, signal));
+                    if (result.status === "busy")
+                        ctx
+                            .objectSendClient({ name: names.session }, ctx.key)
+                            .advance({ reason: "recover" }, restate.rpc.sendOpts({ delay: Math.max(0, result.retryAfterMs) }));
+                },
+            },
+            options: {
+                ...serviceOptions,
+                idempotencyRetention: Math.max(this.options.dedupeRetentionMs ?? 24 * HOUR_MS, 24 * HOUR_MS),
+            },
+        });
+        const tenant = restate.object({
+            name: names.tenant,
+            handlers: {
+                arm: async (ctx) => {
+                    const now = await ctx.date.now();
+                    const state = await ctx.get("sweep");
+                    // An armed chain that beat recently is alive; a stale one was lost
+                    // (killed or purged invocation) and is replaced.
+                    if (state && now - state.beatAt < this.staleSweepMs())
+                        return;
+                    const generation = (state?.generation ?? 0) + 1;
+                    ctx.set("sweep", { generation, beatAt: now });
+                    ctx
+                        .objectSendClient({ name: names.tenant }, ctx.key)
+                        .sweep({ generation });
+                },
+                sweep: async (ctx, input) => {
+                    const state = await ctx.get("sweep");
+                    if (!state || state.generation !== input.generation)
+                        return;
+                    await this.runTracked((handlers) => handlers.sweep(ctx.key));
+                    ctx.set("sweep", {
+                        generation: input.generation,
+                        beatAt: await ctx.date.now(),
+                    });
+                    ctx
+                        .objectSendClient({ name: names.tenant }, ctx.key)
+                        .sweep({ generation: input.generation }, restate.rpc.sendOpts({ delay: this.sweepIntervalMs }));
+                },
+                disarm: async (ctx) => {
+                    ctx.clear("sweep");
+                },
+            },
+            options: serviceOptions,
+        });
+        const timer = restate.object({
+            name: names.timer,
+            handlers: {
+                set: async (ctx, input) => {
+                    ctx.set("at", input.at);
+                    const now = await ctx.date.now();
+                    ctx
+                        .objectSendClient({ name: names.timer }, ctx.key)
+                        .fire(input, restate.rpc.sendOpts({ delay: Math.max(0, input.at - now) }));
+                },
+                fire: async (ctx, input) => {
+                    if ((await ctx.get("at")) !== input.at)
+                        return;
+                    const { tenantId, sessionId: key } = parseSessionKey(ctx.key);
+                    await this.runTracked((handlers) => {
+                        if (!handlers.fire)
+                            throw new Error("WorkerHandlers.fire is required for timers");
+                        return handlers.fire(tenantId, key);
+                    });
+                    ctx.clear("at");
+                },
+            },
+            options: serviceOptions,
+        });
+        return [session, tenant, timer];
+    }
+    /** Runs `advance` with a signal aborted by `stop` or by the end of the Restate attempt. */
+    async runAdvance(attemptDone, fn) {
+        const controller = new AbortController();
+        const onAttemptDone = () => controller.abort(new Error("Restate attempt ended"));
+        if (attemptDone.aborted)
+            onAttemptDone();
+        else
+            attemptDone.addEventListener("abort", onAttemptDone, { once: true });
+        this.controllers.add(controller);
+        try {
+            return await this.runTracked((handlers) => fn(handlers, controller.signal));
+        }
+        finally {
+            this.controllers.delete(controller);
+            attemptDone.removeEventListener("abort", onAttemptDone);
+        }
+    }
+    /** Runs a Worker handler so that `stop` waits for it. Refuses once stopping; Restate retries. */
+    runTracked(fn) {
+        const handlers = this.handlers;
+        if (!handlers || this.stopping)
+            return Promise.reject(new Error("Worker is stopping"));
+        const run = fn(handlers);
+        this.inflight.add(run);
+        void run.then(() => this.inflight.delete(run), () => this.inflight.delete(run));
+        return run;
+    }
+    staleSweepMs() {
+        return Math.max(this.sweepIntervalMs * 3, 60_000);
+    }
+    serviceOptions() {
+        const timeouts = this.options.timeouts ?? {};
+        const retry = this.options.retry ?? {};
+        return {
+            inactivityTimeout: timeouts.inactivityMs ?? HOUR_MS,
+            abortTimeout: timeouts.abortMs ?? HOUR_MS,
+            retryPolicy: {
+                initialInterval: retry.initialIntervalMs ?? 100,
+                maxInterval: retry.maxIntervalMs ?? 30_000,
+                exponentiationFactor: retry.exponentiationFactor ?? 2,
+                maxAttempts: retry.maxAttempts ?? 70,
+                onMaxAttempts: "pause",
+            },
+        };
+    }
+    /** One-way send through the ingress. Retries while the ingress is unreachable or overloaded. */
+    async send(service, key, handler, body, idempotencyKey) {
+        const url = `${this.ingressUrl}/${service}/${encodeURIComponent(key)}/${handler}/send`;
+        const headers = { "content-type": "application/json" };
+        if (idempotencyKey !== undefined)
+            headers["idempotency-key"] = idempotencyKey;
+        await withRetry(`Restate send to ${service}/${handler}`, 10_000, async () => {
+            const response = await fetch(url, {
+                method: "POST",
+                headers,
+                body: JSON.stringify(body),
+                signal: AbortSignal.timeout(10_000),
+            });
+            if (response.ok) {
+                await response.body?.cancel();
+                return;
+            }
+            const text = await response.text().catch(() => "");
+            const error = new Error(`Restate ingress ${response.status} for ${service}/${handler}: ${text}`);
+            // A just-registered service can take a moment to reach the ingress.
+            throw response.status >= 500 || response.status === 404 || response.status === 429
+                ? error
+                : new PermanentError(error.message);
+        });
+    }
+    /** Registers the Worker endpoint with the admin API, retrying until Restate is up. */
+    async register(uri) {
+        const force = this.options.forceRegistration ?? true;
+        await withRetry("Restate deployment registration", this.options.registrationTimeoutMs ?? 60_000, async () => {
+            const response = await fetch(`${this.adminUrl}/deployments`, {
+                method: "POST",
+                headers: { "content-type": "application/json" },
+                body: JSON.stringify({ uri, force }),
+                signal: AbortSignal.timeout(30_000),
+            });
+            if (response.ok) {
+                await response.body?.cancel();
+                return;
+            }
+            const text = await response.text().catch(() => "");
+            const message = `Restate admin ${response.status} registering ${uri}: ${text}`;
+            // Discovery failures (Restate cannot reach the endpoint yet) come back
+            // as 4xx/5xx; keep trying until the deadline.
+            throw new Error(message);
+        });
+    }
+}
+/**
+ * Lists invocations of this Runtime's services that need an operator: paused
+ * after exhausting retries, or backing off after a failure. Reads Restate's
+ * SQL introspection (`POST /query` on the admin API).
+ */
+export async function listStuckInvocations(options) {
+    const prefix = options.servicePrefix ?? "";
+    if (!/^[A-Za-z0-9_]*$/.test(prefix))
+        throw new Error(`Invalid Restate service prefix: ${prefix}`);
+    const services = ["NylorunSession", "NylorunTenant", "NylorunTimer"].map((name) => `'${prefix}${name}'`);
+    const clauses = [
+        `target_service_name IN (${services.join(", ")})`,
+        `status IN ('paused', 'backing-off')`,
+    ];
+    if (options.tenantId !== undefined) {
+        const tenant = sqlString(options.tenantId);
+        clauses.push(`(target_service_key = ${tenant} OR starts_with(target_service_key, ${sqlString(`${options.tenantId}:`)}))`);
+    }
+    const limit = Math.max(1, Math.min(options.limit ?? 100, 1000));
+    const rows = await query(options.adminUrl, "SELECT id, status, target_service_name, target_handler_name, target_service_key, " +
+        "retry_count, last_failure, modified_at FROM sys_invocation " +
+        `WHERE ${clauses.join(" AND ")} ORDER BY modified_at LIMIT ${limit}`);
+    // Restate 1.7 clears `last_failure` and `retry_count` when it pauses an
+    // invocation; the failure is kept on the journal's `Paused` event.
+    const paused = rows.filter((row) => row.status === "paused").map((row) => String(row.id));
+    const pausedFailures = new Map();
+    if (paused.length > 0) {
+        const events = await query(options.adminUrl, "SELECT id, appended_at, event_json FROM sys_journal_events " +
+            `WHERE event_type = 'Paused' AND id IN (${paused.map(sqlString).join(", ")}) ` +
+            "ORDER BY appended_at");
+        for (const event of events) {
+            const message = pausedFailure(event.event_json);
+            if (message !== undefined)
+                pausedFailures.set(String(event.id), message);
+        }
+    }
+    return rows.map((row) => {
+        const id = String(row.id);
+        const key = String(row.target_service_key ?? "");
+        const colon = key.indexOf(":");
+        const lastFailure = row.last_failure == null ? pausedFailures.get(id) : String(row.last_failure);
+        const modifiedAt = row.modified_at == null ? undefined : String(row.modified_at);
+        return {
+            id,
+            status: String(row.status),
+            service: String(row.target_service_name).slice(prefix.length),
+            handler: String(row.target_handler_name),
+            key,
+            ...(key ? { tenantId: colon > 0 ? key.slice(0, colon) : key } : {}),
+            retryCount: Number(row.retry_count ?? 0),
+            ...(lastFailure !== undefined ? { lastFailure } : {}),
+            ...(modifiedAt !== undefined ? { modifiedAt } : {}),
+        };
+    });
+}
+async function query(adminUrl, sql) {
+    const response = await fetch(`${adminUrl.replace(/\/+$/, "")}/query`, {
+        method: "POST",
+        headers: { "content-type": "application/json", accept: "application/json" },
+        body: JSON.stringify({ query: sql }),
+        signal: AbortSignal.timeout(10_000),
+    });
+    if (!response.ok)
+        throw new Error(`Restate query ${response.status}: ${await response.text().catch(() => "")}`);
+    return (await response.json()).rows;
+}
+function pausedFailure(eventJson) {
+    try {
+        const event = JSON.parse(String(eventJson));
+        const failure = event.last_failure;
+        if (!failure?.error_message)
+            return undefined;
+        return failure.error_code === undefined
+            ? failure.error_message
+            : `[${failure.error_code}] ${failure.error_message}`;
+    }
+    catch {
+        return undefined;
+    }
+}
+function tenantKey(tenantId) {
+    if (!tenantId || tenantId.includes(":"))
+        throw new Error(`Invalid tenant id: ${tenantId}`);
+    return tenantId;
+}
+function sqlString(value) {
+    return `'${value.replaceAll("'", "''")}'`;
+}
+class PermanentError extends Error {
+}
+async function withRetry(what, timeoutMs, attempt) {
+    const deadline = Date.now() + timeoutMs;
+    let delay = 100;
+    for (;;) {
+        try {
+            await attempt();
+            return;
+        }
+        catch (error) {
+            if (error instanceof PermanentError || Date.now() + delay > deadline)
+                throw new Error(`${what} failed: ${error.message}`, {
+                    cause: error,
+                });
+            await new Promise((resolve) => setTimeout(resolve, delay));
+            delay = Math.min(delay * 2, 2000);
+        }
+    }
+}
+function sdkLogger(log) {
+    return (meta, message, ...rest) => {
+        const parts = [message, ...rest].map((part) => part instanceof Error ? part.stack ?? part.message : String(part));
+        log(meta.level, parts.join(" "));
+    };
+}

package/dist/adapters/media.d.ts

@@ -19,6 +19,6 @@ export declare class MediaStore {
     private readMetadata;
     private directory;
 }
-export declare function localMedia(options?: {
-    root?: string;
+export declare function localMedia(options: {
+    root: string;
 }): MediaStore;

package/dist/adapters/media.js

@@ -106,6 +106,6 @@ function omitCreatedAt(asset) {
 function safe(value) {
     return value !== "." && value !== ".." && /^[a-zA-Z0-9._-]+$/u.test(value);
 }
-export function localMedia(options = {}) {
-    return new MediaStore(options.root ?? join(process.cwd(), ".data", "media"));
+export function localMedia(options) {
+    return new MediaStore(options.root);
 }

package/dist/adapters/observe.d.ts

@@ -2,7 +2,8 @@
 export declare function jsonlObserver(options: {
     readonly agentId: string;
     readonly sessionId: string;
-    readonly root?: string;
+    readonly root: string;
+    readonly env?: Readonly<Record<string, string | undefined>>;
 }): {
     (event: {
         readonly type: string;

package/dist/adapters/observe.js

@@ -4,10 +4,9 @@ import { scrub } from "../redact.js";
 import { projectSecrets } from "../model/settings.js";
 /** Local JSONL observer. Writes raw engine events beside session durability files. */
 export function jsonlObserver(options) {
-    const root = options.root ?? join(process.cwd(), ".data", "sessions");
-    const directory = join(root, safe(options.agentId), safe(options.sessionId));
+    const directory = join(options.root, safe(options.agentId), safe(options.sessionId));
     const file = join(directory, "observe.jsonl");
-    const secrets = projectSecrets();
+    const secrets = projectSecrets(options.root, options.env ?? {});
     return (event) => {
         const line = `${JSON.stringify(scrub(event, secrets))}\n`;
         void mkdir(directory, { recursive: true })

package/dist/adapters/sandbox/virtual.js

@@ -36,7 +36,7 @@ export function virtualBackend(options) {
         },
         unmet(spec) {
             if (spec.network.suffixes.length > 0)
-                return "the virtual backend cannot allow wildcard hosts; list exact host names in network.allow or run with microsandbox";
+                return "the virtual backend cannot allow wildcard hosts; list exact host names in network.allow";
             return undefined;
         },
         async open(spec) {

package/dist/adapters/streams/s2.d.ts

@@ -0,0 +1,18 @@
+import { type DurableStreams } from "../../streams/types.js";
+export interface S2StreamsOptions {
+    /**
+     * Base URL serving both S2's account and basin APIs, e.g. `http://s2:80` for
+     * s2-lite. Omit to use the S2 service's default endpoints.
+     */
+    endpoint?: string;
+    /** Access token. s2-lite needs none. */
+    token?: string;
+    /** Prepended to every basin name (see `streams/basin.ts`). */
+    basinPrefix?: string;
+    /**
+     * How long `ensureTenant` waits for a previous deletion of the same basin to
+     * finish. Defaults to two minutes.
+     */
+    basinDeletionWaitMs?: number;
+}
+export declare function createS2Streams(options?: S2StreamsOptions): DurableStreams;