@capybara-hq/kernel 0.1.0-rc.21 → 0.1.0-rc.23

package/dist/sandbox/sandbox.service.js

@@ -8,9 +8,10 @@ var __param = (this && this.__param) || function (paramIndex, decorator) {
     return function (target, key) { decorator(target, key, paramIndex); }
 };
 import { randomUUID } from "node:crypto";
-import { isFileSystemGoneError, isSandboxGoneError, rootsOf, } from "@capybara-hq/plugin-sdk";
+import { compareTiers, isFileSystemGoneError, isSandboxGoneError, rootsOf, } from "@capybara-hq/plugin-sdk";
 import { Inject, Injectable } from "@nestjs/common";
 import { sql, } from "kysely";
+import { SESSION_MODE_SCRIPT } from "../background/launch.js";
 import { BackgroundTaskStore } from "../background/store.js";
 import { DEFAULT_MAX_COMMAND_SECONDS, DEFAULT_MAX_RUN_SECONDS, } from "../config/schema.js";
 import { ApiError } from "../errors.js";
@@ -18,10 +19,12 @@ import { readSandboxLimitOverride } from "../identity/settings.js";
 import { KernelConfigService } from "../infra/config/kernel-config.service.js";
 import { KernelDb } from "../infra/database/kernel-db.js";
 import { KERNEL_LOGGER, REDIS } from "../infra/tokens.js";
-import { effectiveSandboxEgress, effectiveSandboxLimits, readOrgSettings, } from "../orgs/settings.service.js";
+import { effectiveSandboxLimits, readOrgSettings } from "../orgs/settings.service.js";
+import { appendMissingGrant, TIER_OPEN_GRANT_KEY, UNATTENDED_OPEN_TIER_ERROR, unattendedTierOf, } from "../schedules/unattended.js";
 import { canonicalJson } from "../tools/canonical-json.js";
 import { SandboxLeases, SandboxLockLostError, sandboxBusy, } from "./leases.js";
 import { LOCAL_PROVIDER, providerForDevice, SandboxProviders, } from "./providers.js";
+import { appliedOf, effectiveSandboxPolicy, instanceLimitsJson, limitsForTier, readDeploymentSandboxPolicy, readInstanceLimits, sameApplied, } from "./tiers.js";
 // Moved to leases.ts (phase C); re-exported for existing importers
 export { SANDBOX_LOCK_PREFIX, SANDBOX_LOCK_TTL_MS, SandboxLockLostError, sandboxBusy, } from "./leases.js";
 /** Margin (ms) of a holder record's TTL beyond the longest command: by then the in-sandbox limit has ended its commands. */
@@ -82,19 +85,20 @@ function tagged(impl, tag, actor, before) {
  * Sandbox config problems to tell operators at startup (English log lines):
  * - `runtime.maxRunSeconds` below `sandbox.limits.maxCommandSeconds`: long commands are cut by the
  *   run time limit (Ruling M20);
- * - `sandbox.limits.egress` is open but the current provider plugin has no `allowOpenEgress`: every
- *   instance creation is refused.
+ * - `sandbox.tiers.max` is open but the current provider plugin sets `openEgress: "off"`: every
+ *   creation of an open-tier instance is refused.
  * With both time limits at their defaults (900 < 7200, a known trade-off) it is only info; warn
  * when an operator changed one and they still disagree.
  */
 export function sandboxConfigWarnings(cfg, providerPlugin) {
     const out = [];
-    if (cfg.sandbox.limits.egress === "open" && providerPlugin !== undefined) {
+    if (cfg.sandbox.tiers.max === "open" && providerPlugin !== undefined) {
+        // absent means the plugin's default ("proxy"), which creates open instances
         const pc = cfg.extensionConfig[providerPlugin];
-        if (pc?.allowOpenEgress !== true) {
+        if (pc?.openEgress === "off") {
             out.push({
                 level: "warn",
-                message: `sandbox.limits.egress is "open" but extensionConfig."${providerPlugin}".allowOpenEgress is not true: the ${providerPlugin} plugin refuses to create sandboxes with open egress; set allowOpenEgress to true (trusted environments only) or use "allowlist" or "none"`,
+                message: `sandbox.tiers.max is "open" but extensionConfig."${providerPlugin}".openEgress is "off": the ${providerPlugin} plugin refuses to create cloud devices in the open tier; set openEgress to "proxy" (or "direct" for trusted deployments only), or lower sandbox.tiers.max to "dev"`,
             });
         }
     }
@@ -269,23 +273,125 @@ let SandboxService = class SandboxService {
         return this.config.get().sandbox.maxConcurrentCommands;
     }
     /**
-     * The org's effective sandbox limits (spec §8.4): the deployment's `sandbox.limits`, egress
-     * narrowed by the org setting `sandboxEgress` (Ruling M19), cpu / memory per the super admin's
-     * org override (C19 / Ruling M16). Instances are created with them; phase 1 devices use
-     * them unchanged.
+     * The effective sandbox policy of a device (sandbox dev tiers spec §4.2): the platform's and
+     * the org's tiers and network with the device's own tier (`devices.tier`, read fresh; a local
+     * device and a missing one have none). Without `deviceId`: a device that follows the org's
+     * default.
      */
-    async orgLimits(orgId) {
+    async devicePolicy(device) {
+        const [deployment, settings, tier] = await Promise.all([
+            readDeploymentSandboxPolicy(this.db.db, this.config),
+            readOrgSettings(this.db.db, device.orgId),
+            device.deviceId === undefined ? null : this.deviceTier(device.orgId, device.deviceId),
+        ]);
+        return effectiveSandboxPolicy(deployment, settings, tier);
+    }
+    /** The device's own tier (`devices.tier`); null when it follows the org or is not there. */
+    async deviceTier(orgId, deviceId) {
+        const row = await this.db
+            .selectFrom("devices")
+            .select("tier")
+            .where("id", "=", deviceId)
+            .where("org_id", "=", orgId)
+            .executeTakeFirst();
+        return row?.tier ?? null;
+    }
+    /**
+     * The device's effective sandbox limits (spec §8.4; sandbox dev tiers spec §4.2): the
+     * deployment's `sandbox.limits`, cpu / memory per the super admin's org override (C19 / Ruling
+     * M16), and the network of the effective tier ({@link devicePolicy}): egress narrowed by the
+     * org setting `sandboxEgress` (Ruling M19), widened by the tier. `opts.policy`: a policy the
+     * caller already read; `opts.tier`: limits for that tier instead (a local device's ceiling, an
+     * unattended run's cap). Cloud instances are created with them.
+     */
+    async deviceLimits(device, opts = {}) {
         const cfg = this.config.get().sandbox;
-        const egress = effectiveSandboxEgress(cfg.limits, (await readOrgSettings(this.db.db, orgId)).sandboxEgress);
-        const sized = effectiveSandboxLimits({ ...cfg.limits, ...egress }, await readSandboxLimitOverride(this.db.db, orgId));
+        const [policy, override] = await Promise.all([
+            opts.policy ?? this.devicePolicy(device),
+            readSandboxLimitOverride(this.db.db, device.orgId),
+        ]);
+        const tier = opts.tier ?? policy.tier;
+        const net = limitsForTier(policy, tier);
+        const sized = effectiveSandboxLimits({ ...cfg.limits, ...net, tier, devHosts: policy.devHosts }, override);
         return {
             cpu: sized.cpu,
             memoryMb: sized.memoryMb,
             diskMb: sized.diskMb,
             egress: sized.egress,
             allowHosts: sized.allowHosts,
+            tier: sized.tier,
+            devHosts: [...policy.devHosts],
+        };
+    }
+    /**
+     * The limits of the org's ceiling (its highest tier, {@link deviceLimits} at `ceiling`): what a
+     * local device's exec may reach at most, so what a network grant or approval is checked
+     * against.
+     */
+    async ceilingLimits(orgId) {
+        const policy = await this.devicePolicy({ orgId });
+        return await this.deviceLimits({ orgId }, { policy, tier: policy.ceiling });
+    }
+    /**
+     * The limits state of a device's instance for its page (sandbox dev tiers spec §7.1, §11.1),
+     * read without a lease: `applied` is what the instance or file system was made with or updated
+     * to (null: unknown, or no instance yet), `effective` what it should run with now.
+     * `pendingNextUse`: a cloud instance exists and differs; its next use applies the new limits
+     * (files kept). `pendingFilesReset`: the provider fixes limits at its file system
+     * (`limitsApplyAt: "fileSystem"`) and cannot `update` them, so only a file reset applies them.
+     * A local device (its machine applies the tier at each command) is never pending, and its
+     * `applied` is null. `opts.policy`: the device's effective policy when the caller already read
+     * it ({@link devicePolicy}).
+     */
+    async limitsState(device, opts = {}) {
+        const [policy, row, p] = await Promise.all([
+            opts.policy ?? this.devicePolicy(device),
+            this.db
+                .selectFrom("device_instances")
+                .select(["provider", "status", "handle", "fs_status", "instance_limits"])
+                .where("device_id", "=", device.deviceId)
+                .where("org_id", "=", device.orgId)
+                .executeTakeFirst(),
+            this.providerOf(device.deviceId, device.orgId),
+        ]);
+        const effective = { tier: policy.tier, egress: policy.egress };
+        if (p === undefined || p.plugin === LOCAL_PROVIDER || row === undefined) {
+            return { applied: null, effective, pendingNextUse: false, pendingFilesReset: false };
+        }
+        const rec = readInstanceLimits(row.instance_limits);
+        const differs = row.provider === p.plugin && !sameApplied(rec.applied, effective);
+        const fsBound = differs && row.fs_status === "ready" && this.fsBound(p, rec, effective);
+        const hasInstance = row.handle !== null && (row.status === "running" || row.status === "suspended");
+        return {
+            applied: rec.applied,
+            effective,
+            pendingNextUse: differs && !fsBound && hasInstance,
+            pendingFilesReset: fsBound,
         };
     }
+    /**
+     * Whether a provider whose limits are fixed at its file system cannot apply `effective`
+     * without a file reset: it has no `update`, or its `update` already answered `recreate` for it.
+     */
+    fsBound(p, rec, effective) {
+        if (p.impl.limitsApplyAt !== "fileSystem")
+            return false;
+        return p.impl.update === undefined || sameApplied(rec.unapplied, effective);
+    }
+    /**
+     * Whether the running instance of `row` must get `want` before use (spec §7.1): a cloud
+     * instance whose recorded tier or egress mode differs, unless only a file reset could apply it
+     * ({@link fsBound}). `want` null: a local device (the tier travels with each command).
+     */
+    mustApply(row, p, want) {
+        if (want === null)
+            return false;
+        const rec = readInstanceLimits(row.instance_limits);
+        const target = appliedOf(want);
+        if (sameApplied(rec.applied, target))
+            return false;
+        return !this.fsBound(p, rec, target);
+    }
     /**
      * The device's instance state as shown to people (read only, no lease): `running` /
      * `suspended` and the last use. No instance, a destroyed one, or one stuck in `creating` (being
@@ -411,7 +517,10 @@ let SandboxService = class SandboxService {
         const p = await this.require(ctx.deviceId, ctx.orgId);
         const env = ctx.deviceId;
         const holder = ctx.holderConversationId;
-        let e = await this.seenBy(env, holder, await this.enter(ctx, p, signal, null));
+        // decides the shared fast path only (read again before each shared lease is used): the
+        // exclusive path reads it again under the lease
+        const want = () => this.wantedLimits(ctx, p);
+        let e = await this.seenBy(env, holder, await this.enter(ctx, p, signal, null, want));
         try {
             try {
                 return await this.runAs(e, ctx, p, fn, signal);
@@ -425,7 +534,7 @@ let SandboxService = class SandboxService {
                 const gone = { handle: e.handle, createdAt: e.createdAt };
                 const prev = e;
                 await this.leave(env, prev);
-                e = await this.seenBy(env, holder, await this.enter(ctx, p, signal, gone));
+                e = await this.seenBy(env, holder, await this.enter(ctx, p, signal, gone, want));
                 // a reset seen on the first try still has to be told (it covers the restart)
                 const info = prev.info.filesReset
                     ? { fresh: true, restarted: false, filesReset: true }
@@ -444,6 +553,58 @@ let SandboxService = class SandboxService {
             await this.leave(env, e);
         }
     }
+    /**
+     * The limits a cloud device's instance must run with for this use (sandbox dev tiers spec
+     * §7.1), read fresh; null for a local device (its machine gets the tier with each command).
+     * `row`: the instance row read under the exclusive lease (read here when absent).
+     *
+     * A scheduled run without a person on a cloud device that may run in the open tier, without the
+     * task's `tier:open` grant, is refused (`device.tier_not_granted`, recorded in the task run's
+     * `missing_grants`): the tier belongs to the shared instance, so it is not switched under the
+     * device's attended sessions (spec §11.4). The tier it may run in is the effective one, or for a
+     * provider whose limits are fixed at its file system the higher of that and the recorded one
+     * (a NULL record there counts as open: nothing says the file system is narrower).
+     */
+    async wantedLimits(ctx, p, row) {
+        if (p.plugin === LOCAL_PROVIDER)
+            return null;
+        const policy = await this.devicePolicy(ctx);
+        const runId = ctx.actor.runId;
+        if (runId !== undefined && (await this.mayRunOpen(ctx, p, policy.tier, row))) {
+            const u = await unattendedTierOf(this.db.db, runId, ctx.deviceId);
+            if (u !== null && !u.openGranted) {
+                await appendMissingGrant(this.db.db, u.taskRunId, TIER_OPEN_GRANT_KEY).catch((err) => this.log.warn("cannot record a missing task grant", { taskRunId: u.taskRunId, err }));
+                this.log.warn("scheduled run refused on a cloud device in the open tier", {
+                    deviceId: ctx.deviceId,
+                    taskRunId: u.taskRunId,
+                });
+                throw new ApiError("device.tier_not_granted", UNATTENDED_OPEN_TIER_ERROR);
+            }
+        }
+        return await this.deviceLimits(ctx, { policy });
+    }
+    /**
+     * Whether the device's instance may run in the open tier during this use (see
+     * {@link wantedLimits}): the effective tier is open, or the provider fixes limits at its file
+     * system and the recorded tier is open or unknown.
+     */
+    async mayRunOpen(ctx, p, tier, row) {
+        if (tier === "open")
+            return true;
+        if (p.impl.limitsApplyAt !== "fileSystem")
+            return false;
+        const r = row ??
+            (await this.db
+                .selectFrom("device_instances")
+                .selectAll()
+                .where("device_id", "=", ctx.deviceId)
+                .where("org_id", "=", ctx.orgId)
+                .executeTakeFirst());
+        if (r === undefined || r.fs_status !== "ready" || r.provider !== p.plugin)
+            return false;
+        const applied = readInstanceLimits(r.instance_limits).applied;
+        return applied === null || compareTiers(applied.tier, "open") >= 0;
+    }
     /**
      * Ruling R24: an operation on the held instance `e` found its file system gone. Records
      * `fs_status = 'lost'` while the row still records that instance on its file system (the
@@ -516,25 +677,43 @@ let SandboxService = class SandboxService {
      * Becomes a holder: loops until it has a shared lease on a running instance, or `sandbox.busy`.
      * An instance recorded `running` (with the current provider, not the `gone` instance) is used
      * under a shared lease; otherwise the exclusive lease (with intent) makes it run and is
-     * downgraded. One token serves both, so the waiter passes its own intent.
+     * downgraded. One token serves both, so the waiter passes its own intent. `wantNow` reads the
+     * limits this use wants ({@link wantedLimits}), again after each wait, so a tier narrowed
+     * during the wait is never used with the old, wider limits.
      */
-    async enter(ctx, p, signal, gone) {
+    async enter(ctx, p, signal, gone, wantNow) {
         const env = ctx.deviceId;
         const deadline = Date.now() + this.config.get().sandbox.lockWaitSeconds * 1000;
         const token = randomUUID();
         let intent = false;
+        let want = await wantNow();
         try {
-            for (;;) {
+            for (let waited = false;; waited = true) {
                 if (signal.aborted)
                     throw abortReason(signal);
+                if (waited)
+                    want = await wantNow();
                 const row = await this.readRow(ctx);
-                if (this.usable(row, p, gone)) {
+                // an instance whose tier or egress must change goes through the exclusive lease
+                const ready = (r) => this.usable(r, p, gone) && !this.mustApply(r, p, want);
+                if (ready(row)) {
                     const lease = await this.leases.tryShared(env, this.cap, token);
                     if (lease) {
                         if (intent)
                             await this.clearIntent(env, token);
                         intent = false;
-                        const e = await this.asShared(ctx, p, lease, (r) => this.usable(r, p, gone), signal);
+                        // after a wait, the limits again under the lease: a tier narrowed just before the
+                        // holder we waited for let go is applied now (asShared checks `ready` with them)
+                        if (waited) {
+                            try {
+                                want = await wantNow();
+                            }
+                            catch (err) {
+                                await lease.release();
+                                throw err;
+                            }
+                        }
+                        const e = await this.asShared(ctx, p, lease, ready, signal);
                         if (e)
                             return e;
                     }
@@ -898,6 +1077,8 @@ let SandboxService = class SandboxService {
                 sweep_failed_at: null,
                 // conversations that used the old files are told they were reset (seenBy)
                 fs_resets: sql `fs_resets + 1`,
+                // the next file system and instance record their own limits
+                instance_limits: null,
                 ...prev,
             });
             this.log.info("device files reset", { deviceId });
@@ -919,7 +1100,7 @@ let SandboxService = class SandboxService {
     /**
      * Under the exclusive lease, right after a resume or a (re)build: kills the sessions of the
      * background tasks marked lost while their process may live on (`kill_pending`), in one exec of
-     * `pkill -KILL -s <sid>`, then clears their flags. Tasks of another instance have no process
+     * `pkill -KILL -s <sid>` (`-g` on macOS), then clears their flags. Tasks of another instance have no process
      * here and are only cleared. A gone instance and an abort are thrown; another failure is warned
      * and leaves the flags of this instance's tasks for the next resume.
      */
@@ -931,10 +1112,12 @@ let SandboxService = class SandboxService {
         let cleared = rows;
         if (mine.length > 0) {
             // pkill exits 1 when the session has no process left: only a higher status is a failure
-            const command = `rc=0; for sid in ${mine.map((r) => r.sid).join(" ")}; do pkill -KILL -s "$sid"; c=$?; [ "$c" -le 1 ] || rc=$c; done; exit $rc`;
+            const command = `${SESSION_MODE_SCRIPT}\nrc=0; for sid in ${mine.map((r) => r.sid).join(" ")}; do tk_kill KILL "$sid"; c=$?; [ "$c" -le 1 ] || rc=$c; done; exit $rc`;
             try {
                 const r = await p.impl.exec(handle, {
                     command,
+                    // tk_kill's control files are relative to the workspace root (macOS devices)
+                    cwd: rootsOf(handle).workspace,
                     timeoutMs: PENDING_KILL_TIMEOUT_MS,
                     maxOutputBytes: 4096,
                     signal,
@@ -1099,19 +1282,42 @@ let SandboxService = class SandboxService {
             // provider may have made it by name, and nothing else would ever destroy it
             await this.dropUnrecordedFiles(id, row.provider, signal);
         }
+        // read under the lease, after the fence moved: a value read before the wait may be stale
+        // (another replica may have changed the tier meanwhile)
+        const want = await this.wantedLimits(ctx, p, row);
         const handle = row.handle;
         const createdAt = row.created_at.getTime();
         const fsResets = row.fs_resets;
         if (handle && isInstance(row, gone) && row.status === "running") {
-            const created = await this.create(ctx, p, held, row, fence, signal);
+            const created = await this.create(ctx, p, held, row, fence, signal, want);
             return { ...created, fsResets, info: { fresh: true, restarted: true }, fence };
         }
         if (handle && row.status === "running") {
+            if (want !== null && this.mustApply(row, p, want)) {
+                const created = await this.applyLimits(ctx, p, held, row, handle, fence, signal, want);
+                if (created)
+                    return { ...created, fsResets, info: { fresh: true, restarted: true }, fence };
+            }
             return { handle, createdAt, fsResets, info: { fresh: false, restarted: false }, fence };
         }
         if (handle && row.status === "suspended") {
             const r = await p.impl.resume(handle, signal);
             if (r === "resumed") {
+                // the new limits first: the row says running only once the instance runs with them
+                if (want !== null && this.mustApply(row, p, want)) {
+                    let created;
+                    try {
+                        created = await this.applyLimits(ctx, p, held, row, handle, fence, signal, want);
+                    }
+                    catch (err) {
+                        // the row still says suspended; the next use resumes and kills what is left
+                        await this.bestEffort(id, "kill the commands left after a failed limits change", (s) => p.impl.killAll(handle, s));
+                        throw err;
+                    }
+                    if (created) {
+                        return { ...created, fsResets, info: { fresh: true, restarted: true }, fence };
+                    }
+                }
                 await this.fenced(held, id, fence, {
                     status: "running",
                     running_since: sql `now()`,
@@ -1131,7 +1337,7 @@ let SandboxService = class SandboxService {
                 deviceId: id,
             });
         }
-        const created = await this.create(ctx, p, held, row, fence, signal);
+        const created = await this.create(ctx, p, held, row, fence, signal, want);
         // had_instance only ever becomes true: any earlier instance (even if a rebuild failed since)
         // makes this one a restart on the kept files
         return {
@@ -1148,7 +1354,7 @@ let SandboxService = class SandboxService {
      * throws `device.files_lost`. When the instance handle cannot be written (lease lost), the
      * new instance is destroyed (best effort) before {@link SandboxLockLostError} is thrown.
      */
-    async create(ctx, p, held, row, fence, signal) {
+    async create(ctx, p, held, row, fence, signal, want) {
         const id = ctx.deviceId;
         const name = deviceResourceName(id);
         await this.fenced(held, id, fence, {
@@ -1162,9 +1368,13 @@ let SandboxService = class SandboxService {
             destroyed_at: null,
             sweep_failed_at: null,
         });
-        // the org's effective limits apply at creation only, so a changed setting reaches only
-        // instances created afterwards
-        const limits = await this.orgLimits(ctx.orgId);
+        // the device's effective limits apply at creation (a changed tier or egress mode is applied
+        // on use, see applyLimits; other changes reach only instances created afterwards)
+        const limits = want ?? (await this.deviceLimits(ctx));
+        // the record says what is in force: a provider whose limits are fixed at its file system
+        // records them with the file system, any other with the instance (spec §7.1)
+        const record = instanceLimitsJson(appliedOf(limits));
+        const atFileSystem = p.impl.limitsApplyAt === "fileSystem";
         const cfg = this.config.get().sandbox;
         let fs = row.fs_status === "ready" && (row.provider === null || row.provider === p.plugin)
             ? row.fs_handle
@@ -1177,6 +1387,7 @@ let SandboxService = class SandboxService {
                 provider: p.plugin,
                 fs_handle: JSON.stringify(fs),
                 fs_status: "ready",
+                ...(atFileSystem && { instance_limits: record }),
             });
             this.log.info("device file system created", { deviceId: id, provider: p.plugin });
         }
@@ -1191,8 +1402,13 @@ let SandboxService = class SandboxService {
             });
         }
         catch (err) {
-            if (!isFileSystemGoneError(err))
+            if (!isFileSystemGoneError(err)) {
+                if (limits.egress === "open" && !signal.aborted) {
+                    // spec §5.3: a provider set to refuse open egress (`openEgress: "off"`) fails here
+                    this.log.warn("the sandbox provider did not create an instance in the open tier; check its openEgress setting or lower sandbox.tiers.max", { deviceId: id, provider: p.plugin, err });
+                }
                 throw err;
+            }
             this.log.warn("device file system is gone at the provider; its files are lost", {
                 deviceId: id,
                 provider: p.plugin,
@@ -1209,6 +1425,7 @@ let SandboxService = class SandboxService {
                 created_at: sql `now()`,
                 running_since: sql `now()`,
                 last_used_at: sql `now()`,
+                ...(!atFileSystem && { instance_limits: record }),
             });
         }
         catch (e) {
@@ -1226,6 +1443,110 @@ let SandboxService = class SandboxService {
         this.log.info("sandbox created", { deviceId: id, provider: p.plugin });
         return { handle, createdAt: createdAt.getTime() };
     }
+    /**
+     * Under the exclusive lease, on the running instance `handle` of `row` whose tier or egress
+     * mode differs from `want` (sandbox dev tiers spec §7.1): the provider's `update` when it has
+     * one; `updated` records `want`. Otherwise (`recreate`, no `update`, or the instance found gone
+     * by `update`) the instance is destroyed and created again on the kept files, which records
+     * `want` and is returned (the caller reports a restart). A provider whose limits are fixed at
+     * its file system is not recreated (that would not apply them): `want` is recorded as
+     * `unapplied` and the instance keeps running as is until a file reset.
+     *
+     * An `update` that fails otherwise never records `want`: a provider whose limits apply at
+     * instance create is recreated (warned); after an abort, or for a provider bound to its file
+     * system, the instance is suspended ({@link quarantine}) and the error thrown, so the next use
+     * resumes it and tries again.
+     */
+    async applyLimits(ctx, p, held, row, handle, fence, signal, want) {
+        const id = ctx.deviceId;
+        const target = appliedOf(want);
+        const was = readInstanceLimits(row.instance_limits).applied;
+        const facts = { deviceId: id, provider: p.plugin, from: was, to: target };
+        let gone = false;
+        if (p.impl.update) {
+            try {
+                if ((await p.impl.update(handle, want, signal)) === "updated") {
+                    await this.fenced(held, id, fence, { instance_limits: instanceLimitsJson(target) });
+                    this.log.info("sandbox instance limits updated", facts);
+                    return null;
+                }
+            }
+            catch (err) {
+                if (isSandboxGoneError(err)) {
+                    gone = true;
+                }
+                else if (signal.aborted || p.impl.limitsApplyAt === "fileSystem") {
+                    // the instance may run with half-applied limits and cannot be recreated here: stop it,
+                    // so the next use resumes it and applies them again
+                    await this.quarantine(held, id, handle, p, fence);
+                    throw err;
+                }
+                else {
+                    // a new instance gets the limits at create: recreate rather than trust a failed update
+                    this.log.warn("sandbox limits update failed; recreating the instance", {
+                        ...facts,
+                        err,
+                    });
+                }
+            }
+        }
+        if (p.impl.limitsApplyAt === "fileSystem" && !gone) {
+            await this.fenced(held, id, fence, { instance_limits: instanceLimitsJson(was, target) });
+            this.log.info("sandbox limits changed; they apply when the device's files are reset", facts);
+            return null;
+        }
+        this.log.info("sandbox limits changed; recreating the instance on the kept files", facts);
+        if (!gone) {
+            try {
+                await p.impl.destroy(handle, signal);
+            }
+            catch (err) {
+                // `create` replaces an instance of the same name anyway
+                if (signal.aborted)
+                    throw err;
+                this.log.warn("cannot destroy the sandbox instance before recreating it", {
+                    deviceId: id,
+                    err,
+                });
+            }
+        }
+        return await this.create(ctx, p, held, row, fence, signal, want);
+    }
+    /**
+     * After a failed `update` that cannot be repaired by a recreate: best effort, with fresh short
+     * signals (the use's own may be aborted), suspends the instance (or kills its commands) and
+     * records it suspended (fenced: a lost lease leaves the record to the new holder).
+     */
+    async quarantine(held, id, handle, p, fence) {
+        const suspended = await this.bestEffort(id, "suspend after a failed limits update", (s) => p.impl.suspend(handle, s));
+        if (!suspended) {
+            await this.bestEffort(id, "kill commands after a failed limits update", (s) => p.impl.killAll(handle, s));
+        }
+        try {
+            await this.fenced(held, id, fence, {
+                status: "suspended",
+                suspended_at: sql `now()`,
+                running_since: null,
+            });
+        }
+        catch (err) {
+            this.log.warn("cannot record the sandbox suspended after a failed limits update", {
+                deviceId: id,
+                err,
+            });
+        }
+    }
+    /** Runs `call` with its own short signal; a failure is warned and returns false. */
+    async bestEffort(deviceId, what, call) {
+        try {
+            await call(AbortSignal.timeout(PENDING_KILL_TIMEOUT_MS));
+            return true;
+        }
+        catch (err) {
+            this.log.warn(`cannot ${what}`, { deviceId, err });
+            return false;
+        }
+    }
     /**
      * Destroys the file system another provider may have made under the device's name without
      * the kernel recording it (`destroyFileSystemByName`; not there is success). That provider not

package/dist/sandbox/tiers.d.ts

@@ -0,0 +1,98 @@
+import { type SandboxTier } from "@capybara-hq/plugin-sdk";
+import type { KernelConfig } from "../config/schema.js";
+import type { KernelSchemaDb } from "../db/kernel-schema.js";
+import type { OrgSettings } from "../orgs/settings.service.js";
+import { type PlatformTiers } from "./platform-tiers.js";
+/**
+ * Sandbox tiers (sandbox dev tiers spec §4): who may reach which tier, and the network each tier
+ * gets. One function, {@link effectiveSandboxPolicy}, combines the platform's ceiling and default
+ * (`sandbox.tiers`), the org's (`sandboxTier`) and the device's own choice.
+ */
+/**
+ * What the policy reads of the platform: the `sandbox.limits` network fields and the effective
+ * platform tiers (`readPlatformTiers`: the startup config over the admin console's choice).
+ */
+export interface DeploymentSandboxPolicy {
+    limits: Pick<KernelConfig["sandbox"]["limits"], "egress" | "allowHosts" | "devHosts">;
+    tiers: PlatformTiers;
+}
+/**
+ * The platform side of the policy, read live: the `sandbox.limits` network fields of the config
+ * and the effective platform tiers (in the caller's transaction when there is one).
+ */
+export declare function readDeploymentSandboxPolicy(db: KernelSchemaDb, config: {
+    get(): Pick<KernelConfig, "sandbox">;
+}): Promise<DeploymentSandboxPolicy>;
+/** The org settings the policy reads. */
+export type OrgSandboxSettings = Pick<OrgSettings, "sandboxEgress" | "sandboxTier">;
+/**
+ * The org's base list `L` (spec §3): `none` when the org's or the platform's egress is `none`,
+ * else an allowlist of the org's hosts (the platform's `allowHosts` when the org gives none).
+ * Network requests and approvals add to this list; tiers widen it (§4.2).
+ */
+export interface BaseSandboxEgress {
+    egress: "none" | "allowlist";
+    allowHosts: string[];
+}
+/** The network of one tier: what a cloud instance is created with, or a local exec's ceiling. */
+export interface SandboxNetwork {
+    egress: "none" | "allowlist" | "open";
+    allowHosts: string[];
+}
+/** The effective sandbox policy of one device (or of a new device, with no tier of its own). */
+export interface EffectiveSandboxPolicy extends SandboxNetwork {
+    /** min(device tier ?? the org's default, the org's ceiling, the platform's ceiling). */
+    tier: SandboxTier;
+    /** min(org max, platform max): the highest tier a device of the org may reach. */
+    ceiling: SandboxTier;
+    /** The org's default tier (the platform's when the org sets none), at most {@link ceiling}. */
+    defaultTier: SandboxTier;
+    /** Which ceiling cut the chosen tier (the device's, else the org's default), null when none did. */
+    cappedBy: "org" | "platform" | null;
+    /** The base list `L` the tiers widen. */
+    base: BaseSandboxEgress;
+    /** The dev hosts `D` (`sandbox.limits.devHosts`). */
+    devHosts: string[];
+    /** Whether a local device may run unsandboxed: platform and org allow it and the ceiling is open. */
+    allowUnsandboxed: boolean;
+}
+/** The org's base list `L`: see {@link BaseSandboxEgress}. */
+export declare function baseSandboxEgress(deployment: Pick<DeploymentSandboxPolicy["limits"], "egress" | "allowHosts">, org: OrgSettings["sandboxEgress"]): BaseSandboxEgress;
+/**
+ * The network of `tier` over the base list (spec §4.2): `none` stays `none` in every tier; an
+ * allowlist is `L` in strict, `L ∪ D` in dev and open egress in open.
+ */
+export declare function networkForTier(base: BaseSandboxEgress, devHosts: readonly string[], tier: SandboxTier): SandboxNetwork;
+/** The network of `tier` under `policy` (e.g. the ceiling's for a local device's exec args). */
+export declare function limitsForTier(policy: EffectiveSandboxPolicy, tier: SandboxTier): SandboxNetwork;
+/**
+ * The effective sandbox policy (spec §4.2). `deviceTier` is the device's own choice (null =
+ * follow the org's default). The org's ceiling is its `sandboxTier.max` (absent = the platform's)
+ * cut to the platform's; its default is `sandboxTier.default ?? sandbox.tiers.default`, cut to the
+ * ceiling. Stored values above the platform (the platform lowered its max later) are cut, never
+ * honoured.
+ */
+export declare function effectiveSandboxPolicy(deployment: DeploymentSandboxPolicy, org: OrgSandboxSettings, deviceTier: SandboxTier | null): EffectiveSandboxPolicy;
+/**
+ * What decides whether a running cloud instance must change (spec §7.1): its tier and egress
+ * mode only. `allowHosts`, cpu, memory and disk reach instances created afterwards.
+ */
+export interface AppliedLimits {
+    tier: SandboxTier;
+    egress: SandboxNetwork["egress"];
+}
+/** The {@link AppliedLimits} of `limits`. */
+export declare const appliedOf: (limits: AppliedLimits) => AppliedLimits;
+/** Whether `a` and `b` are the same tier and egress mode (null, unknown, is never the same). */
+export declare const sameApplied: (a: AppliedLimits | null | undefined, b: AppliedLimits) => boolean;
+/**
+ * `device_instances.instance_limits` read leniently: `applied` is what the instance runs with
+ * (null: unknown, e.g. an instance made before migration 0117, which differs from every effective
+ * value); `unapplied` is a newer value the provider could not apply without a file reset.
+ */
+export declare function readInstanceLimits(v: unknown): {
+    applied: AppliedLimits | null;
+    unapplied: AppliedLimits | null;
+};
+/** The JSON written to `device_instances.instance_limits`. */
+export declare function instanceLimitsJson(applied: AppliedLimits | null, unapplied?: AppliedLimits): string | null;