@nylorun/runtime 0.13.0-beta → 0.15.0-beta

package/dist/tenant/module.d.ts

@@ -1,38 +1,27 @@
-import { isQuarantineError, quarantine, TenantBusyError, TenantConflictError } from "./quarantine.js";
-import { TenantNotFoundError, TenantUnavailableError, type Logger, type OpenTenantRuntime, type TenantConfig, type TenantModule, type TenantStore } from "./types.js";
+import type { StreamRelayStatus } from "@nylorun/core/contracts";
+import { TenantUnavailableError, type Logger, type TenantHandle, type TenantModule, type TenantOpener } from "./types.js";
 export interface CreateTenantModuleOptions {
-    store: TenantStore;
-    /** Unused: the store derives paths. Accepted so callers can pass the store's options. */
-    hostRoot?: string;
-    /** Unused: the store opens Tenants. Accepted so callers can pass the store's options. */
-    openRuntime?: OpenTenantRuntime;
-    /** Unused; see `openRuntime`. */
-    configFor?: (tenantId: string) => TenantConfig;
+    /** Opens the Host's Tenant (`store-pg.ts`). */
+    open: TenantOpener;
     logger: Logger;
-    /** Bound on one Tenant open; a slower open quarantines it with `open-timeout`. */
+    /** Bound on opening the Tenant; a slower open fails with `open-timeout`. */
     openTimeoutMs?: number;
     /**
-     * Runs after a Tenant is created in the store, before it opens: creates what lives outside
-     * it (its Durable Streams basin). A failure is logged; opening the Tenant repairs it.
+     * Runs once the Tenant is open, at start or on a later retry: the Host starts what needs the
+     * Tenant's id (its stream relay). A failure is logged; the Tenant stays open.
      */
-    onCreated?: (tenantId: string) => Promise<void>;
-    /**
-     * Runs after a Tenant is removed from the store: removes what lives outside it (its
-     * Durable Streams basin, its armed sweep). A failure is logged; the Tenant stays deleted.
-     */
-    onDeleted?: (tenantId: string) => Promise<void>;
+    onOpen?: (handle: TenantHandle) => void | Promise<void>;
+    /** This process's stream relay, for the Host aggregate (a Host with S2 runs one). */
+    relayStatus?: () => Promise<StreamRelayStatus>;
 }
 /**
- * Deep Tenant module (§8): discovery, create, resolve, list, delete and summarize over an
- * injected `TenantStore`.
+ * Deep Tenant module (§8): the Host's one Tenant over an injected opener (tenancy.md §5).
  *
- * Handles open on demand and are cached (architecture §8.2). The first call that needs a
- * Tenant (`resolve`, `status`, `worker`, `delete`) opens it once, even when several ask at
- * the same time; a failed open is cached as a quarantine, and a missing Tenant is not
- * cached. A failure outside the Tenant (`TenantUnavailableError`: the database is down, its
- * sweep could not be armed) is thrown to the caller and not cached either, so the next use
- * tries again. A Tenant being deleted resolves as not found, so a late request or Worker
- * call cannot reopen it while its storage is removed.
+ * `start` opens it once, even when requests or Worker calls ask at the same time. An open that
+ * fails for a reason in the Tenant (a `TenantOpenError`, or a timeout) is kept as the Tenant's
+ * cause: the Host is not ready, `/v1/admin/status` names it, requests get the opaque 404.
+ * One that fails outside the Tenant (`TenantUnavailableError`) is retried in the background
+ * with backoff, and by the next request, which gets 503 meanwhile.
  */
 export declare function createTenantModule(options: CreateTenantModuleOptions): TenantModule;
-export { TenantBusyError, TenantConflictError, TenantNotFoundError, TenantUnavailableError, isQuarantineError, quarantine, };
+export { TenantUnavailableError };

package/dist/tenant/module.js

@@ -1,322 +1,176 @@
-import { isTenantId } from "@nylorun/core/compatibility";
-import { POSTGRES_SCHEMA_VERSION } from "../store/postgres/migrations/index.js";
-import { envelopeNow } from "./envelope.js";
+import { openError, TenantOpenError } from "./cause.js";
 import { TimeoutError, withTimeout } from "./pool.js";
-import { asQuarantine, isQuarantineError, quarantine, TenantBusyError, TenantConflictError, } from "./quarantine.js";
-import { TenantNotFoundError, TenantUnavailableError, } from "./types.js";
+import { TenantUnavailableError, } from "./types.js";
 const OPEN_TIMEOUT_MS = 30_000;
-function hasLiveWork(summary) {
-    return (summary.runningSessions > 0 ||
-        summary.connectedExecutors > 0 ||
-        summary.pendingActions > 0);
-}
-function toAdmin(id, entry, envelope) {
-    if (entry?.kind === "open") {
-        return {
-            id,
-            name: entry.handle.envelope.name,
-            state: "open",
-            envelope: entry.handle.envelope,
-        };
-    }
-    return {
-        id,
-        name: envelope?.name ?? null,
-        state: entry ? "quarantined" : "open",
-        envelope,
-    };
-}
-/** The quarantine an open failure carries, from either quarantine error type. */
-function quarantineOf(error) {
-    const carried = error?.quarantine;
-    if (carried && typeof carried === "object" && "code" in carried)
-        return carried;
-    return asQuarantine(error);
-}
+/** Retries of an open that failed outside the Tenant: from 1 s, doubling, at most 30 s apart. */
+const RETRY_FIRST_MS = 1_000;
+const RETRY_MAX_MS = 30_000;
 /**
- * Deep Tenant module (§8): discovery, create, resolve, list, delete and summarize over an
- * injected `TenantStore`.
+ * Deep Tenant module (§8): the Host's one Tenant over an injected opener (tenancy.md §5).
  *
- * Handles open on demand and are cached (architecture §8.2). The first call that needs a
- * Tenant (`resolve`, `status`, `worker`, `delete`) opens it once, even when several ask at
- * the same time; a failed open is cached as a quarantine, and a missing Tenant is not
- * cached. A failure outside the Tenant (`TenantUnavailableError`: the database is down, its
- * sweep could not be armed) is thrown to the caller and not cached either, so the next use
- * tries again. A Tenant being deleted resolves as not found, so a late request or Worker
- * call cannot reopen it while its storage is removed.
+ * `start` opens it once, even when requests or Worker calls ask at the same time. An open that
+ * fails for a reason in the Tenant (a `TenantOpenError`, or a timeout) is kept as the Tenant's
+ * cause: the Host is not ready, `/v1/admin/status` names it, requests get the opaque 404.
+ * One that fails outside the Tenant (`TenantUnavailableError`) is retried in the background
+ * with backoff, and by the next request, which gets 503 meanwhile.
  */
 export function createTenantModule(options) {
-    const { store, logger, openTimeoutMs = OPEN_TIMEOUT_MS } = options;
-    const entries = new Map();
-    const opening = new Map();
-    const deleting = new Set();
-    let started = false;
-    let closed = false;
-    function rememberQuarantine(id, q) {
-        const entry = { kind: "quarantined", quarantine: q };
-        entries.set(id, entry);
-        logger.warn("tenant quarantined", {
-            tenantId: id,
-            code: q.code,
-            repair: q.repair,
-        });
-        return entry;
-    }
-    async function openEntry(id) {
+    const { logger, openTimeoutMs = OPEN_TIMEOUT_MS } = options;
+    let state = { kind: "idle" };
+    let opening;
+    let retry;
+    let nextDelayMs = RETRY_FIRST_MS;
+    const current = () => state;
+    async function openOnce() {
         let handle;
         try {
-            handle = await withTimeout(store.open(id), openTimeoutMs, (late) => {
+            handle = await withTimeout(options.open(), openTimeoutMs, (late) => {
                 void late.then((h) => h.close()).catch(() => undefined);
             });
         }
         catch (error) {
-            if (error instanceof TenantNotFoundError)
-                return undefined;
-            if (closed || deleting.has(id))
-                return undefined;
+            if (current().kind === "closed")
+                return;
             if (error instanceof TenantUnavailableError) {
-                // Not the Tenant's fault: not cached, so the next use opens it again.
+                state = { kind: "unavailable", error };
                 logger.warn("tenant unavailable", {
-                    tenantId: id,
                     message: error.cause instanceof Error ? error.cause.message : String(error.cause),
+                    retryMs: nextDelayMs,
                 });
-                throw error;
+                scheduleRetry();
+                return;
             }
-            if (error instanceof TimeoutError)
-                return rememberQuarantine(id, quarantine("open-timeout", `open timed out after ${openTimeoutMs}ms`, {
-                    tenantId: id,
-                }).toQuarantine());
-            return rememberQuarantine(id, quarantineOf(error) ??
-                quarantine("open-failed", error instanceof Error ? error.message : "open failed", { tenantId: id }).toQuarantine());
+            const failure = error instanceof TenantOpenError
+                ? error
+                : error instanceof TimeoutError
+                    ? openError("open-timeout", `Opening the Tenant timed out after ${openTimeoutMs}ms`)
+                    : openError("open-failed", error instanceof Error ? error.message : "Opening the Tenant failed");
+            const failed = {
+                kind: "failed",
+                cause: failure.toCause(),
+                envelope: failure.envelope ?? null,
+            };
+            state = failed;
+            logger.error("tenant not opened", {
+                ...(failed.envelope ? { tenantId: failed.envelope.id } : {}),
+                code: failed.cause.code,
+                message: failed.cause.message,
+                repair: failed.cause.repair,
+            });
+            return;
         }
-        if (closed || deleting.has(id)) {
+        if (current().kind === "closed") {
             await handle.close().catch(() => undefined);
-            return undefined;
+            return;
         }
-        const entry = { kind: "open", handle };
-        entries.set(id, entry);
-        return entry;
-    }
-    /** The cached entry, or the result of opening the Tenant once. */
-    async function load(id) {
-        if (deleting.has(id))
-            return undefined;
-        const cached = entries.get(id);
-        if (cached)
-            return cached;
-        if (closed)
-            return undefined;
-        let pending = opening.get(id);
-        if (!pending) {
-            pending = openEntry(id).finally(() => opening.delete(id));
-            opening.set(id, pending);
+        state = { kind: "open", handle };
+        nextDelayMs = RETRY_FIRST_MS;
+        logger.info("tenant opened", { tenantId: handle.envelope.id });
+        try {
+            await options.onOpen?.(handle);
         }
-        return pending;
+        catch (error) {
+            logger.warn("tenant open step failed", {
+                tenantId: handle.envelope.id,
+                message: error instanceof Error ? error.message : String(error),
+            });
+        }
+    }
+    /** Opens the Tenant unless it is open, failed or closed; one open at a time. */
+    function load() {
+        const { kind } = current();
+        if (kind !== "idle" && kind !== "unavailable")
+            return Promise.resolve();
+        clearTimeout(retry);
+        retry = undefined;
+        opening ??= openOnce().finally(() => {
+            opening = undefined;
+        });
+        return opening;
+    }
+    function scheduleRetry() {
+        if (retry)
+            return;
+        const delayMs = nextDelayMs;
+        nextDelayMs = Math.min(delayMs * 2, RETRY_MAX_MS);
+        retry = setTimeout(() => {
+            retry = undefined;
+            void load();
+        }, delayMs);
+        retry.unref();
     }
-    const openEntries = () => [...entries.values()].filter((e) => e.kind === "open");
     const module = {
-        get started() {
-            return started;
+        get ready() {
+            return current().kind === "open";
         },
         async start() {
-            if (closed)
+            if (current().kind === "closed")
                 throw new Error("Tenant module is closed");
-            const ids = await store.enumerate();
-            started = true;
-            logger.info("tenant module started", { tenants: ids.length });
+            await load();
         },
-        async resolve(id) {
-            if (!isTenantId(id))
-                return { kind: "not-found" };
-            const entry = await load(id);
-            if (!entry)
-                return { kind: "not-found" };
-            if (entry.kind === "open")
-                return { kind: "open", handle: entry.handle };
-            return { kind: "quarantined", quarantine: entry.quarantine };
+        async resolve() {
+            await load();
+            const now = current();
+            switch (now.kind) {
+                case "open":
+                    return { kind: "open", handle: now.handle };
+                case "failed":
+                    return { kind: "unavailable", cause: now.cause };
+                case "unavailable":
+                    throw now.error;
+                default:
+                    return { kind: "unavailable" };
+            }
         },
         async worker(id) {
-            if (!isTenantId(id))
-                return undefined;
-            const entry = await load(id);
-            return entry?.kind === "open" ? entry.handle.worker : undefined;
+            await load();
+            const now = current();
+            return now.kind === "open" && now.handle.envelope.id === id
+                ? now.handle.worker
+                : undefined;
         },
-        async create(input) {
-            if (!isTenantId(input.tenantId)) {
-                throw new Error(`Invalid tenant id: ${input.tenantId}`);
-            }
-            const bootstrap = {
-                principalId: input.principalId,
-                credentialHash: input.credentialHash,
-                idempotencyKey: input.idempotencyKey,
-                ...(input.studioCredentialHash
-                    ? { studioCredentialHash: input.studioCredentialHash }
-                    : {}),
-                ...(input.derivedPrincipals?.length
-                    ? { derivedPrincipals: input.derivedPrincipals }
-                    : {}),
-            };
-            // The store records its own schema version.
-            const envelope = envelopeNow({
-                id: input.tenantId,
-                name: input.name,
-                schemaVersion: POSTGRES_SCHEMA_VERSION,
-            });
-            let outcome;
-            try {
-                outcome = await store.create(envelope, bootstrap);
-            }
-            catch (error) {
-                await store.removePartial(input.tenantId).catch(() => undefined);
-                throw error;
-            }
-            if (outcome === "exists") {
-                const matches = await store.bootstrapMatches(input.tenantId, bootstrap);
-                if (!matches)
-                    throw new TenantConflictError();
-                const existing = await store.readEnvelope(input.tenantId);
-                await load(input.tenantId).catch(() => undefined);
-                return { envelope: existing, created: false };
-            }
-            await options.onCreated?.(input.tenantId).catch((error) => logger.warn("tenant create step failed; open repairs it", {
-                tenantId: input.tenantId,
-                message: error instanceof Error ? error.message : String(error),
-            }));
-            const opened = await load(input.tenantId).catch(() => undefined);
-            if (opened?.kind === "open") {
-                return { envelope: opened.handle.envelope, created: true };
-            }
-            // Created but failed to open: report the stored envelope.
-            return {
-                envelope: await store
-                    .readEnvelope(input.tenantId)
-                    .catch(() => envelope),
-                created: true,
-            };
-        },
-        async list() {
-            const ids = new Set(await store.enumerate());
-            for (const id of entries.keys())
-                ids.add(id);
-            const result = [];
-            for (const id of [...ids].sort()) {
-                if (!isTenantId(id) || deleting.has(id))
-                    continue;
-                let entry = entries.get(id);
-                if (entry?.kind === "open") {
-                    result.push(toAdmin(id, entry, null));
-                    continue;
-                }
-                let envelope = null;
-                try {
-                    envelope = await store.readEnvelope(id);
-                    if (envelope.id !== id) {
-                        entry = {
-                            kind: "quarantined",
-                            quarantine: quarantine("envelope-invalid", `Tenant envelope id ${envelope.id} does not match ${id}`, { tenantId: id }).toQuarantine(),
-                        };
-                        envelope = null;
-                    }
+        tenant() {
+            const now = current();
+            switch (now.kind) {
+                case "open": {
+                    const { envelope } = now.handle;
+                    return { id: envelope.id, name: envelope.name, state: "open", envelope };
                 }
-                catch (error) {
-                    // Deleted since it was enumerated.
-                    if (error instanceof TenantNotFoundError)
-                        continue;
-                    if (error instanceof TenantUnavailableError)
-                        throw error;
-                    // Listed as quarantined; only opening it caches a quarantine.
-                    entry ??= {
-                        kind: "quarantined",
-                        quarantine: quarantineOf(error) ??
-                            quarantine("envelope-invalid", "Tenant envelope is unreadable", {
-                                tenantId: id,
-                            }).toQuarantine(),
+                case "failed":
+                    return {
+                        id: now.envelope?.id ?? null,
+                        name: now.envelope?.name ?? null,
+                        state: "unavailable",
+                        envelope: now.envelope,
+                        cause: now.cause,
                     };
-                }
-                result.push(toAdmin(id, entry, envelope));
-            }
-            return result;
-        },
-        async status(id) {
-            if (!isTenantId(id))
-                return undefined;
-            const entry = await load(id);
-            if (!entry)
-                return undefined;
-            if (entry.kind === "open")
-                return toAdmin(id, entry, null);
-            const envelope = await store.readEnvelope(id).catch(() => null);
-            return { ...toAdmin(id, entry, envelope), quarantine: entry.quarantine };
-        },
-        async delete(id, activeWork) {
-            if (!isTenantId(id))
-                throw new Error(`Invalid tenant id: ${id}`);
-            if (deleting.has(id))
-                throw new TenantBusyError("Tenant is being deleted");
-            const entry = await load(id);
-            if (!entry)
-                throw new TenantNotFoundError(id);
-            if (entry.kind === "open") {
-                const summary = await entry.handle.summary();
-                if (activeWork === "refuse" && hasLiveWork(summary)) {
-                    throw new TenantBusyError();
-                }
-                if (activeWork === "drain" || activeWork === "cancel") {
-                    await entry.handle.drain(activeWork);
-                }
-            }
-            deleting.add(id);
-            try {
-                entries.delete(id);
-                // Sandbox prefix cleanup is owned by TenantHandle.close().
-                if (entry.kind === "open")
-                    await entry.handle.close();
-                await store.trash(id, new Date());
-            }
-            finally {
-                deleting.delete(id);
+                default:
+                    return { id: null, name: null, state: "unavailable", envelope: null };
             }
-            logger.info("tenant deleted", { tenantId: id, activeWork });
-            await options.onDeleted?.(id).catch((error) => logger.warn("tenant delete cleanup failed", {
-                tenantId: id,
-                message: error instanceof Error ? error.message : String(error),
-            }));
         },
         async summarize() {
-            let runningSessions = 0;
-            let connectedExecutors = 0;
-            let pendingActions = 0;
-            let uncertainEffects = 0;
-            let outboxDepth;
-            let relayLagMs;
-            for (const s of await Promise.all(openEntries().map((e) => e.handle.summary()))) {
-                runningSessions += s.runningSessions;
-                connectedExecutors += s.connectedExecutors;
-                pendingActions += s.pendingActions;
-                uncertainEffects += s.uncertainEffects;
-                if (s.outboxDepth !== undefined)
-                    outboxDepth = (outboxDepth ?? 0) + s.outboxDepth;
-                if (s.relayLagMs !== undefined)
-                    relayLagMs = Math.max(relayLagMs ?? 0, s.relayLagMs);
-            }
+            const now = current();
+            const summary = now.kind === "open" ? await now.handle.summary() : undefined;
+            const relay = await options.relayStatus?.().catch(() => undefined);
             return {
-                runningSessions,
-                connectedExecutors,
-                pendingActions,
-                uncertainEffects,
-                ...(outboxDepth !== undefined ? { outboxDepth } : {}),
-                ...(relayLagMs !== undefined ? { relayLagMs } : {}),
+                runningSessions: summary?.runningSessions ?? 0,
+                inFlightDeliveries: summary?.inFlightDeliveries ?? 0,
+                pendingActions: summary?.pendingActions ?? 0,
+                uncertainEffects: summary?.uncertainEffects ?? 0,
+                ...(relay ? { relay } : {}),
             };
         },
         async close() {
-            closed = true;
-            started = false;
-            await Promise.allSettled([...opening.values()]);
-            const open = openEntries();
-            entries.clear();
-            await Promise.all(open.map((e) => e.handle.close().catch(() => undefined)));
+            const previous = current();
+            state = { kind: "closed" };
+            clearTimeout(retry);
+            retry = undefined;
+            // An open in progress sees `closed` and closes what it opened.
+            await opening?.catch(() => undefined);
+            if (previous.kind === "open")
+                await previous.handle.close().catch(() => undefined);
         },
     };
     return module;
 }
-export { TenantBusyError, TenantConflictError, TenantNotFoundError, TenantUnavailableError, isQuarantineError, quarantine, };
+export { TenantUnavailableError };

package/dist/tenant/outbound.d.ts

@@ -0,0 +1,44 @@
+import { type IncomingHttpHeaders } from "node:http";
+/** How the Host lets the Runtime call developer URLs (Host settings, `TenantConfig.delivery`). */
+export interface OutboundPolicy {
+    /** `http` URLs as well as `https`. Default true (OSS). */
+    allowHttp?: boolean;
+    /** Private, loopback and link-local addresses. Default `allow` (OSS); Cloud refuses. */
+    privateAddresses?: "allow" | "refuse";
+    /**
+     * `docker-host`: the Runtime runs in the local stack's container, so `localhost`,
+     * `127.0.0.1` and `[::1]` mean the machine that runs Docker (`host.docker.internal`).
+     */
+    loopback?: "docker-host";
+}
+export type OutboundResult = {
+    kind: "response";
+    status: number;
+    headers: IncomingHttpHeaders;
+    body: Buffer;
+}
+/** The answer was larger than `maxResponseBytes`. */
+ | {
+    kind: "too_large";
+    status: number;
+} | {
+    kind: "not_sent";
+    code: string;
+    message: string;
+} | {
+    kind: "lost";
+    code: string;
+    message: string;
+};
+/** The largest answer read from an endpoint. */
+export declare const MAX_RESPONSE_BYTES: number;
+/** True when `address` is private, loopback, link-local or unspecified (IPv4-mapped included). */
+export declare function isPrivateAddress(address: string): boolean;
+/** Why the Host refuses `url`, or `undefined` when it may be called. */
+export declare function refusal(url: URL, policy: OutboundPolicy): string | undefined;
+/** POSTs `body` to `url` under `policy`. Never throws; `signal` aborts the request. */
+export declare function post(target: string, body: string, headers: Record<string, string>, options: {
+    policy: OutboundPolicy;
+    signal: AbortSignal;
+    maxResponseBytes?: number;
+}): Promise<OutboundResult>;