@nylorun/runtime 0.15.2-beta → 0.17.0-beta

package/dist/tenant/runtime.js

@@ -5,8 +5,10 @@ import { tenantModelGate } from "../gates/in-process.js";
 import { createKekFile, readVaultKek } from "../vault/kek.js";
 import { SigningKeys } from "./signing-keys.js";
 import { VaultService } from "../vault/service.js";
+import { CredentialSources } from "../vault/sources.js";
 import { McpPool } from "../mcp/pool.js";
 import { SandboxManager } from "../sandbox/manager.js";
+import { storeSandboxRecords } from "../sandbox/records.js";
 import { defaultSandboxBackends } from "../sandbox/select.js";
 import { MemoryExecution } from "../execution/memory.js";
 import { openError } from "./cause.js";
@@ -20,8 +22,18 @@ import { deliverAction } from "./delivery.js";
 import { TenantWorkers, WORKER_ID, } from "./worker.js";
 import { authorize } from "./effects.js";
 import { inProcessToolGate } from "../gates/tool-gate.js";
+import { guardedFetch } from "./outbound.js";
 import { inProcessKeys } from "../keys/keys.js";
 import { tenantApi } from "../api/http/app.js";
+import { createFsBlobStore } from "../blob/index.js";
+import { createHarnessApiServer, } from "../harness-api/server.js";
+import { harnessStatusOf } from "./status.js";
+import { startInProcessHarness, startLoopbackHarness, } from "../harness-api/in-process.js";
+import { localWorkspace, remoteWorkspace, withPodWorkspaces, } from "../harness-api/workspace.js";
+import { hostAuthority } from "../sandbox/join.js";
+import { reconcileSandbox } from "../sandbox/pods/reconcile.js";
+import { sandboxWorkspaceReader } from "../artifacts/workspace.js";
+import { operatorKeys } from "./operator-keys.js";
 /** Default ownership lease of an advance; the heartbeat renews it every third. */
 const DEFAULT_OWNER_LEASE_MS = 30_000;
 function validateConfig(config) {
@@ -45,16 +57,24 @@ export class TenantRuntime {
     worker;
     detach;
     closeGraceMs;
+    harness;
+    hosts;
     constructor(ctx, envelope, 
     /** This Tenant's Worker handlers, as registered with the execution. */
     worker, detach, 
     /** How long close waits for running advances: the execution's advance grace period. */
-    closeGraceMs) {
+    closeGraceMs, harness, hosts) {
         this.ctx = ctx;
         this.envelope = envelope;
         this.worker = worker;
         this.detach = detach;
         this.closeGraceMs = closeGraceMs;
+        this.harness = harness;
+        this.hosts = hosts;
+    }
+    /** The Tenant's side of a sandbox pod's join (F7.2). */
+    hostAuthority() {
+        return this.ctx.closing || this.ctx.closed ? undefined : this.hosts;
     }
     static async open(config, hooks) {
         const envelope = hooks.envelope;
@@ -69,6 +89,7 @@ export class TenantRuntime {
         const store = hooks.store;
         let wired;
         let detach;
+        let harness;
         try {
             // With the keys service (F4.2) the key lives in the gateway: this process never reads,
             // creates or holds it, and the gateway reports a missing key on its readiness.
@@ -101,38 +122,75 @@ export class TenantRuntime {
             const vault = new VaultService({
                 store: opened,
                 kek: ensureKek,
-                fetch: config.vaultFetch ?? globalThis.fetch,
+                // OAuth refresh and connect follow the Host's address policy (F9 C2).
+                fetch: config.vaultFetch ?? guardedFetch(config.delivery ?? {}),
+            });
+            const credentials = new CredentialSources({
+                vault,
+                ...(config.resolver ? { resolver: config.resolver } : {}),
             });
             // `ctx` is assigned below; these callbacks only run once the Tenant is open.
             let ctx;
             const toolGate = hooks.toolGate ?? inProcessToolGate(config.delivery ?? {});
             const signingKeys = new SigningKeys({ tenantId: config.tenantId, kek: ensureKek });
             const keys = hooks.keys ?? inProcessKeys({ store: opened, vault, signingKeys, kek: ensureKek });
-            const mcp = new McpPool({
-                pluginData: paths.pluginData,
-                childEnv: config.childEnv,
-                authorize: (sessionId, request) => authorize(ctx, sessionId, request),
-                ...(toolGate.openMcp ? { openRemote: (server) => toolGate.openMcp(server) } : {}),
-            });
+            const harnessMode = hooks.harness ?? "memory";
+            const inProcess = harnessMode === "memory" || harnessMode === "json";
+            const preference = seededBackend ?? config.sandbox.backend;
+            // With a harness in this process, the Tenant's MCP pool and SandboxManager are its
+            // executors; otherwise the harness holds both (F6.2).
+            const mcp = inProcess
+                ? new McpPool({
+                    pluginData: paths.pluginData,
+                    childEnv: config.childEnv,
+                    authorize: (sessionId, request) => authorize(ctx, sessionId, request),
+                    ...(toolGate.openMcp ? { openRemote: (server) => toolGate.openMcp(server) } : {}),
+                })
+                : undefined;
             const ephemeral = config.mode === "ephemeral";
-            const sandbox = new SandboxManager({
-                scope: config.tenantId,
-                store: opened,
-                backends: config.sandbox.backends ??
-                    defaultSandboxBackends({ root: paths.sandboxes }),
-                preference: seededBackend ?? config.sandbox.backend,
-                ephemeral,
-                emit: async (sessionId, turnId, type, payload) => {
-                    if (ctx.closed)
-                        return;
-                    try {
-                        await opened.tx((t) => t.event(sessionId, turnId, type, payload));
-                    }
-                    catch {
-                        /* the session is gone (reset) or the store closed */
-                    }
-                },
-            });
+            const manager = inProcess
+                ? new SandboxManager({
+                    scope: config.tenantId,
+                    records: storeSandboxRecords(opened),
+                    backends: config.sandbox.backends ??
+                        defaultSandboxBackends({ root: paths.sandboxes }),
+                    preference,
+                    ephemeral,
+                    emit: async (sessionId, turnId, type, payload) => {
+                        if (ctx.closed)
+                            return;
+                        try {
+                            await opened.tx((t) => t.event(sessionId, turnId, type, payload));
+                        }
+                        catch {
+                            /* the session is gone (reset) or the store closed */
+                        }
+                    },
+                })
+                : undefined;
+            let harnessServer;
+            const workspaces = manager
+                ? localWorkspace(manager)
+                : remoteWorkspace({
+                    get server() {
+                        return harnessServer;
+                    },
+                    store: opened,
+                    logger: config.logger,
+                    preference,
+                });
+            // A pod sandbox's workspace is in its pod, served by its engine (F7.2).
+            const sandbox = hooks.pods
+                ? withPodWorkspaces(workspaces, {
+                    get server() {
+                        return harnessServer;
+                    },
+                    store: opened,
+                    logger: config.logger,
+                    preference,
+                    isPod: async (sandboxId) => (await opened.tx((t) => t.sandboxResource(sandboxId)))?.kind === "pod",
+                })
+                : workspaces;
             const useVaultModel = config.model.kind === "vault";
             let modelProvider;
             if (hooks.modelProvider)
@@ -143,6 +201,7 @@ export class TenantRuntime {
                 modelProvider = toolFixtureModel();
             else
                 modelProvider = scriptedModel();
+            const blobs = hooks.blobs ?? createFsBlobStore({ root: paths.blobs });
             // With the gates service the loop never reads a model credential: only a Runtime
             // without one (embedding, the ephemeral Runtime, tests) reads it here.
             const modelGate = hooks.modelGate ??
@@ -151,40 +210,48 @@ export class TenantRuntime {
                     kek: ensureKek,
                     root: paths.home,
                     logger: config.logger,
+                    blobs,
                     ...(config.modelCall ? { settings: config.modelCall } : {}),
                 });
             const sessionStreams = createSessionStreams();
+            const workers = hooks.execution?.workers ?? new TenantWorkers();
             const local = hooks.execution
                 ? undefined
                 : new MemoryExecution({
                     sweepIntervalMs: config.sweepIntervalMs ?? Math.min(config.leaseMs ?? 30_000, 5000),
+                    // Close's `detach` step waits no longer for running work than for its advances.
+                    stopGraceMs: workers.graceMs,
                     onError: (error) => config.logger.error("tenant execution failed", {
                         message: error instanceof Error ? error.message : String(error),
                     }),
                 });
-            const { execution, workers } = hooks.execution ?? {
-                execution: local,
-                workers: new TenantWorkers(),
-            };
+            const execution = hooks.execution?.execution ?? local;
             const sweepHooks = new Set();
             ctx = {
                 config,
                 envelope,
                 store: opened,
                 vault,
-                mcp,
+                credentials,
+                ...(mcp ? { mcp } : {}),
                 sandbox,
                 flowLimits,
                 modelProvider,
                 useVaultModel,
                 modelGate,
                 toolGate,
+                get harness() {
+                    return harnessServer;
+                },
                 closing: false,
                 closed: false,
                 work: createWorkState(),
                 sessionStreams,
                 signingKeys,
                 keys,
+                blobs,
+                workspaces: sandboxWorkspaceReader(sandbox),
+                ...(hooks.runGrants ? { runGrants: hooks.runGrants } : {}),
                 workerId: hooks.workerId ?? WORKER_ID,
                 ownerLeaseMs: config.ownerLeaseMs ?? DEFAULT_OWNER_LEASE_MS,
                 wake: async (sessionId, wake) => {
@@ -198,6 +265,12 @@ export class TenantRuntime {
                         return;
                     await execution.deliver(config.tenantId, actionId);
                 },
+                ...(hooks.pods ? { pods: hooks.pods } : {}),
+                sandboxSignal: async (sandboxId, signal) => {
+                    if (ctx.closing || ctx.closed || !execution.sandbox)
+                        return;
+                    await execution.sandbox(config.tenantId, sandboxId, signal);
+                },
                 ...(hooks.execution?.stuckInvocations
                     ? {
                         stuckInvocations: () => hooks.execution.stuckInvocations(config.tenantId),
@@ -209,6 +282,16 @@ export class TenantRuntime {
                     return () => sweepHooks.delete(hook);
                 },
             };
+            harnessServer = createHarnessApiServer(ctx, { sandboxPreference: preference });
+            if (mcp && manager)
+                harness = await startInProcessHarness(ctx, { mcp, sandbox: manager }, {
+                    json: harnessMode === "json",
+                    ...(hooks.harnessTap ? { tap: hooks.harnessTap } : {}),
+                });
+            else if (harnessMode === "ws")
+                harness = await startLoopbackHarness(ctx, {
+                    ...(config.sandbox.backends ? { sandboxBackends: config.sandbox.backends } : {}),
+                });
             wired = await wireStreams(ctx, {
                 store: opened,
                 streams: hooks.streams ?? new MemoryStreams(),
@@ -223,6 +306,7 @@ export class TenantRuntime {
                 advance: (sessionId, signal) => advance(ctx, sessionId, signal),
                 deliver: (actionId, signal) => deliverAction(ctx, actionId, signal),
                 sweep: () => sweep(ctx),
+                sandbox: (sandboxId, trigger) => reconcileSandbox(ctx, sandboxId, trigger),
             };
             const unregister = workers.register(config.tenantId, worker);
             detach = async () => {
@@ -232,10 +316,11 @@ export class TenantRuntime {
             if (local)
                 await local.start(workers.handlers);
             await execution.armSweep(config.tenantId);
-            return new TenantRuntime(ctx, envelope, worker, detach, workers.graceMs);
+            return new TenantRuntime(ctx, envelope, worker, detach, workers.graceMs, harness, hooks.pods ? hostAuthority(ctx) : undefined);
         }
         catch (error) {
             await detach?.().catch(() => undefined);
+            await harness?.stop().catch(() => undefined);
             await wired?.close().catch(() => undefined);
             await store.close().catch(() => undefined);
             throw error;
@@ -256,6 +341,7 @@ export class TenantRuntime {
             inFlightDeliveries: [...this.ctx.work.deliveries.values()].reduce((n, set) => n + set.size, 0),
             pendingActions: counts.pendingActions,
             uncertainEffects: counts.uncertainEffects,
+            harness: harnessStatusOf(this.ctx),
         };
     }
     drain(activeWork, timeoutMs = 30_000) {
@@ -279,6 +365,13 @@ export class TenantRuntime {
     onSweep(hook) {
         return this.ctx.onSweep(hook);
     }
+    attachHarness(channel, peer) {
+        return this.ctx.harness.attach(channel, peer);
+    }
+    /** The Tenant's operator keys (F9 I1), over its store. */
+    operatorKeys() {
+        return operatorKeys(this.ctx.store);
+    }
     async close() {
         const ctx = this.ctx;
         ctx.closing = true;
@@ -289,10 +382,18 @@ export class TenantRuntime {
         // error is rethrown at the end.
         const steps = [
             ["detach", () => this.detach()],
-            ["mcp", () => ctx.mcp.close()],
+            ["mcp", () => ctx.mcp?.close()],
             ["session streams", () => endAllStreams(ctx.sessionStreams)],
             // Bounded: an advance that ignores its abort is abandoned; its lease lapses (§11.4).
             ["idle", () => waitForIdle(ctx, Math.max(0, idleBy - Date.now()))],
+            [
+                "harness",
+                async () => {
+                    ctx.harness.close();
+                    // Advances still running were abandoned at "idle": their runs are left behind too.
+                    await this.harness?.stop(Math.max(0, idleBy - Date.now()));
+                },
+            ],
             ["sandbox", () => ctx.sandbox.close()],
             [
                 "streams",

package/dist/tenant/sandboxes.d.ts

@@ -0,0 +1,119 @@
+/**
+ * Sandboxes as a resource (blueprint D39, F7.1; Host feature `sandboxes`): `PUT`, `GET` and
+ * `DELETE /v1/sandboxes/{id}`, the list by label, sessions attaching with `sandbox: { id }`, and
+ * the checks every turn start makes on the sandbox its session is attached to.
+ *
+ * - **No scope.** The Runtime decides nothing about who shares a sandbox. It stores the
+ *   sandbox's id, kind, spec and labels, and the session body names the sandbox it is attached
+ *   to (`Session.sandboxId`); deleting a session (a sessions reset) only detaches it.
+ * - **Access.** An application key reaches every sandbox, and so does an application key acting
+ *   for a subject (`Nylorun-Subject`), which needs `sandboxes:write` to change one. A token
+ *   caller (a trusted issuer's token) reaches only the ids its grants match, and changes them
+ *   only with `sandboxes:write`; any other id is the 404 of a missing one.
+ * - **Serial turns.** One turn at a time per sandbox: a session whose sandbox another attached
+ *   session's turn holds is refused (`409 sandbox_busy`), as the session sandbox route is
+ *   refused while its session's turn runs. Commands within the workspace keep queueing in the
+ *   Sandbox Manager.
+ * - **Kinds.** `virtual`: its workspace is the virtual backend's, keyed by the sandbox id
+ *   (`SandboxManager.sandboxKeyOf`) and created on the first tool call. `pod` (F7.2, Host feature
+ *   `sandbox-pods`): an agent-sandbox pod on the Tenant's cluster, created at once, whose engine
+ *   runs the turns of the sessions attached to it. Its lifecycle (`sandbox/pods/lifecycle.ts`)
+ *   runs in the `Sandbox` object: this module records what is wanted (`desired`, `rev`) and asks
+ *   for a reconcile after the commit (`ctx.sandboxSignal`). Without sandbox pods kind `pod` is
+ *   `409 sandbox_unavailable`. `POST .../stop` suspends a pod (the volume is kept; the next turn
+ *   starts it), `POST .../reset` gives it a new pod on a new volume. An expired pod refuses
+ *   turns (`sandbox_expired`) until a `PUT` sets a longer `lifecycle.ttl`; a lost one
+ *   (`sandbox_lost`) until a reset. Deleting a pod sandbox keeps its row (`desired: deleted`,
+ *   invisible here) until its Sandbox is gone.
+ * - **Lifecycle events** go to the sandbox's own stream in the record (`Tx.sandboxEvent`,
+ *   `record/sandbox.ts`); the session's log records `sandbox.attached` too.
+ */
+import { type PutSandboxRequest, type SandboxView } from "@nylorun/core/contracts";
+import type { SandboxResource, Tx } from "../store/types.js";
+import type { AuthScope, Session, TenantContext } from "./context.js";
+/** The sandbox grants that limit `scope`, or undefined when it reaches every sandbox. */
+export declare function sandboxGrantsOf(scope: AuthScope): readonly string[] | undefined;
+/** A sandbox resource, unless it is gone or being deleted. */
+export declare function liveSandbox(t: Tx, id: string, options?: {
+    lock?: boolean;
+}): Promise<SandboxResource | undefined>;
+/**
+ * `PUT /v1/sandboxes/{id}`: creates the sandbox (within the Tenant's limit on their number), or
+ * finds the one with this id, so "get or create" is one call. The spec is fixed once it exists;
+ * labels, when sent, replace its labels. On an existing pod sandbox it also starts a stopped or
+ * failed pod again, and a new `lifecycle.ttl` sets a new expiry (reviving an expired one).
+ */
+export declare function putSandbox(ctx: TenantContext, id: string, body: PutSandboxRequest, scope: AuthScope): Promise<SandboxView>;
+/** `GET /v1/sandboxes/{id}`. */
+export declare function getSandbox(ctx: TenantContext, id: string, scope: AuthScope): Promise<SandboxView>;
+/** `GET /v1/sandboxes?label=k=v`: every sandbox with all the labels, that the caller reaches. */
+export declare function listSandboxes(ctx: TenantContext, labels: Record<string, string>, scope: AuthScope): Promise<{
+    sandboxes: SandboxView[];
+}>;
+/**
+ * `DELETE /v1/sandboxes/{id}`: deletes the sandbox and its workspace. Refused while a session
+ * attached to it has an active turn. Sessions stay attached by id: their next turn is refused
+ * (`sandbox_unavailable`) unless a sandbox with this id exists again. A pod sandbox's row stays
+ * (`desired: deleted`) until its Sandbox is gone.
+ */
+export declare function deleteSandbox(ctx: TenantContext, id: string, scope: AuthScope): Promise<{
+    id: string;
+    deleted: boolean;
+}>;
+/**
+ * `POST /v1/sandboxes/{id}/stop`: suspends a pod sandbox. Its volume is kept, and the next turn
+ * of a session attached to it (or a `PUT`) starts it again. Refused during a turn.
+ */
+export declare function stopSandbox(ctx: TenantContext, id: string, scope: AuthScope): Promise<SandboxView>;
+/**
+ * `POST /v1/sandboxes/{id}/reset`: a new pod on a new, empty volume (the next volume
+ * generation), with a new join token; the old Sandbox and volume are deleted. Leaves `lost` and
+ * `failed`; an expired sandbox stays expired. Refused during a turn, and while an earlier
+ * reset's Sandbox is still being deleted.
+ */
+export declare function resetSandbox(ctx: TenantContext, id: string, scope: AuthScope): Promise<SandboxView>;
+/** `GET /v1/sandboxes/{id}/events`: the sandbox's lifecycle stream. */
+export declare function sandboxEventsOf(ctx: TenantContext, id: string, scope: AuthScope, fromSeq: number | undefined): Promise<{
+    events: {
+        schema: "nylorun.sandbox-event/1";
+        eventId: string;
+        tenantId: string;
+        sandboxId: string;
+        seq: number;
+        time: string;
+        type: "sandbox.attached" | "sandbox.created" | "sandbox.deleted" | "sandbox.detached" | "sandbox.expired" | "sandbox.failed" | "sandbox.lost" | "sandbox.relaunched" | "sandbox.reset" | "sandbox.running" | "sandbox.suspended";
+        payload: Record<string, unknown>;
+    }[];
+}>;
+/**
+ * Attaches a new session to sandbox `id` (`PutSessionRequest.sandbox = { id }`): the sandbox
+ * must exist and be reachable with `grants`. Returns its spec, which the session pins. Runs in
+ * the session's transaction, after its session row lock.
+ */
+export declare function attachSandbox(t: Tx, id: string, grants: readonly string[] | undefined): Promise<SandboxResource>;
+/** Records the attachment on the session's log and on the sandbox's stream. */
+export declare function recordAttachment(t: Tx, sessionId: string, sandboxId: string): Promise<void>;
+/**
+ * Checks a turn start on a session attached to a sandbox, in the command's transaction after
+ * the session row lock: the sandbox still exists, a token caller's grants reach it, and
+ * no other session attached to it has a turn running. Holds the sandbox row's lock until the
+ * transaction ends, so two turn starts on one sandbox are decided one after the other.
+ *
+ * A pod sandbox must also be usable: not lost (`sandbox_lost`) or expired (`sandbox_expired`),
+ * on a Runtime with sandbox pods (`sandbox_unavailable`). A stopped or failed pod is asked to
+ * run again, after the commit.
+ */
+export declare function checkSandboxTurn(t: Tx, session: Session, scope: AuthScope, ctx?: TenantContext): Promise<void>;
+/** A sessions reset detaches every session from its sandbox: each sandbox's stream says so. */
+export declare function detachAllSessions(t: Tx): Promise<void>;
+/**
+ * Marks the end of a turn's segment on pod sandbox `id`: idle counts from now, and the idle
+ * timer is armed (D34). Best effort.
+ */
+export declare function touchPodSandbox(ctx: TenantContext, id: string): Promise<void>;
+/**
+ * Re-arms the idle timer of every running pod sandbox after the Tenant's settings change, so a
+ * shorter `limits.idle` applies to pods already idle (D36: the settings are read whenever a
+ * timer is set). Best effort.
+ */
+export declare function rearmPodTimers(ctx: TenantContext): Promise<void>;