@sema-agent/server 7.130.0 → 7.132.0

package/dist/plugins/remote-env-k8s.d.ts

@@ -1,11 +1,11 @@
 import { type SpaceReclaimer, type WorkspaceSpacePolicy, type CacheReclaimReason } from "../workspace-space.js";
 import { type SpaceCommitFence } from "../workspace-space-commit.js";
-import { type WorkspaceCacheCapability } from "../workspace-cache.js";
+import { type CacheMissMemory, type WorkspaceCacheCapability } from "../workspace-cache.js";
 import { type SandboxEgressGap, type SandboxIpv6 } from "../sandbox-egress-policy.js";
 import { FileError, ExecutionError, RemoteExecutionError, type ExecutionEnv, type RemoteExecutionEnv, type WorkspaceHandle, type FileInfo, type Result, type SymlinkChain, type OutputChunk, type ExecStreamOptions, type RemoteConnectConfig, type SnapshotId, type SessionToken, type VmLifecycleOptions, type ExecutionEnvFactory, type BackgroundShellCapability, type BackgroundShellId, type BackgroundShellError, type BackgroundSpawnOptions, type BackgroundPoll } from "@sema-agent/core";
 import type { PodSpecPatch } from "./k8s-exec-protocol.js";
 type ExecOpts = Parameters<ExecutionEnv["exec"]>[1];
-export { buildBgRunnerScript, buildBgLauncherScript, buildBgPollScript, buildBgKillScript, buildBgDisposeScript, buildDetachCapableExec, parseBgPollOutput, type ParsedBgPoll, } from "./k8s-bg-scripts.js";
+export { buildBgRunnerScript, buildBgLauncherScript, buildBgPollScript, buildBgKillScript, buildBgDisposeScript, buildDetachCapableExec, buildKillAllScript, parseBgPollOutput, type ParsedBgPoll, } from "./k8s-bg-scripts.js";
 export interface K8sEnvConfig {
     /** API server URL. Default: in-cluster (`https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT`). */
     apiUrl?: string;
@@ -112,6 +112,10 @@ export interface K8sEnvConfig {
      * ({@link SpaceOccupant}). Absent ⇒ no claim request, the pod as before.
      */
     cache?: WorkspaceCacheCapability;
+    /** The deployment's memory of cache volumes that could not be had ({@link CacheMissMemory}; the per-run binding hands every run's env the same one, only with `cache`):
+     *  a space whose volume did not bind / did not mount is not asked for again for one data bound (`dataTimeoutMs` — the wait a run was willing to give it; no knob of its own).
+     *  Absent (a direct construction) ⇒ no memory: every env asks for itself. */
+    cacheMisses?: CacheMissMemory;
     /** Operator log seat (the e2b adapter's shape): `workspace_snapshot_failed` / `workspace_snapshot_prune_failed` /
      *  `workspace_space_unavailable` / `workspace_space_unreadable_occupant` / `workspace_commit_abandoned` / `workspace_latest_dangling` /
      *  `workspace_restore_failed` / `workspace_cache_fresh` / `workspace_cache_unavailable`. */
@@ -183,8 +187,8 @@ export interface K8sEnvDeps {
  *  - absent: `unread: "incarnation_unknown"` — there is NO space for this run: the store answered that it has no such session
  *    (deleted / never registered), or this replica holds no commit fence to key one by (see `occupantOf`) ⇒ a lane keyed by it
  *    does not open (no restore, no save, no volume) and says so in one line;
- *  - faulted: `fault` — the read did not answer (it threw, or answered something that is not a token): the code of the store's
- *    error (never its text), `incarnation_read_failed` without one. A space we could not read is not "no space" ⇒ a lane keyed by
+ *  - faulted: `fault` — the read did not answer (it threw, did not come within the lane's store wall `rpcTimeoutMs` — `incarnation_read_timeout`,
+ *    or answered something that is not a token): the code of the store's error (never its text), `incarnation_read_failed` without one. A space we could not read is not "no space" ⇒ a lane keyed by
  *    it refuses this env (`RemoteK8sExecutionEnv.provision`, the one pod seat): the run never starts on a workspace it believes is its own.
  * Never a fallback to the session id.
  */
@@ -210,6 +214,22 @@ export interface CacheReclaimFailure {
     readonly reason: Exclude<CacheReclaimReason, "occupant_unreadable">;
     readonly detail: string;
 }
+/** The control-plane wall of an env built from `config` (`rpcTimeoutMs`) — the one reading: the env's own requests, the commit-fence calls and the
+ *  boot binding's occupant read all run under THIS number (the session store's by-id read is one more store call of the same connect). */
+export declare function rpcTimeoutMsOf(config: K8sEnvConfig): number;
+/**
+ * One call to a store this adapter does not own — the run store's commit-fence rows ({@link RemoteK8sExecutionEnv.fenceCall}), the session store's
+ * by-id occupant read (`boot/execution-env.ts`) — under a wall-clock bound, and, when given, the caller's abort. A store that never answers never
+ * hangs the caller: the bound rejects with `onTimeout()`, the abort with `abort.error()`. The call itself is not cancellable — a late answer is dropped.
+ */
+export declare function boundedStoreCall<T>(op: () => Promise<T>, bound: {
+    readonly ms: number;
+    readonly onTimeout: () => Error;
+    readonly abort?: {
+        readonly signal: AbortSignal;
+        readonly error: () => Error;
+    };
+}): Promise<T>;
 export { execSocketTlsOptions } from "./k8s-exec-protocol.js";
 /** `Error`-or-event → message. Bun's ws shim emits browser-style ErrorEvents (NOT instanceof Error) whose
  *  String() is "[object ErrorEvent]" — that masked the real TLS failure for a whole drill. */
@@ -241,14 +261,20 @@ export declare class RemoteK8sExecutionEnv implements RemoteExecutionEnv, Backgr
     private destroyed;
     /** Set after a successful resumeVM so workspaceHandle() carries the snapshot it was restored from. */
     private handleSnapshotId?;
-    /** In-flight exec count — `suspendVM` refuses while a command runs (council #4 suspend×in-flight). */
-    private inFlightExecs;
-    /** Memoized in-flight connect (lazy-connect; prevents concurrent double pod-create — the E2B race lesson). */
-    private connectPromise?;
-    /** The space restore's refusal: `latest` names an archive that is there and did not come back. A fact of this env until `destroy` —
-     *  written once, by {@link restoreSnapshot} at the restore's end; read by {@link connect} alone (the lazy and the explicit connect alike),
-     *  which answers it to every later connect: the same refusal, zero pods, zero store requests, no second line. */
-    private restoreRefusal?;
+    /** The foreground commands in flight (`exec` / `execStream`), each by the abort controller its runExec listens to — ONE record: `suspendVM` refuses while any
+     *  runs (council #4 suspend×in-flight); `destroy` waits for them to end before the run-end snapshot and, past its wait, aborts them
+     *  ({@link quiesce}). Every entry is removed by {@link endExec}, whichever way its command ended. */
+    private readonly inFlight;
+    /** Resolvers of {@link whenIdle}: woken when the last command in flight ends. */
+    private idleWaiters;
+    /** This env's one connect in flight (see {@link connect}): the explicit connect, the lazy connect and `reconnect` that arrive while it runs
+     *  share it — one pod creation, one restore (the E2B race lesson). Cleared the moment it settles, whatever the outcome. */
+    private connectInFlight?;
+    /** The env's refusal — a fact of this env until `destroy`, written once ({@link holdRefusal}) and answered as it stands to every later connect (and to
+     *  every later pod creation: {@link provision}): the same refusal, zero pods, zero store requests, no second line. Two causes write it:
+     *  the space restore (`latest` names an archive that is there and did not come back — {@link restoreSnapshot}) and the pod's admission (the cluster created
+     *  the pod and it ended `Failed` before it ran — {@link createPod}). Neither goes away by asking again: the same request answers the same. */
+    private connectRefusal?;
     /** core `ExecutionEnv.homeDir`([ref]):本沙箱/目标机上执行用户的 home。见配置同名字段。 */
     readonly homeDir?: string | undefined;
     /** core 7.49.0 #1554:shell 子进程真拿到的 `CDPATH` 条目 —— 本车道的命令跑在 pod 里(镜像与 pod spec 的 env),本类不读那份 env ⇒ `"unknown"`(core 契约:说不出就说 unknown,
@@ -273,6 +299,7 @@ export declare class RemoteK8sExecutionEnv implements RemoteExecutionEnv, Backgr
     private cacheUnavailableSaid;
     private readonly logger?;
     private readonly onWorkspaceSnapshot?;
+    private readonly cacheMisses?;
     constructor(config: K8sEnvConfig, deps?: K8sEnvDeps);
     /** 166-T2 — ExecutionEnv 寿命声明:pod `activeDeadlineSeconds = ceil(timeoutMs/1000)` 是平台真硬死线
      *  (k8s 到点杀 pod,无 keepalive 续期机制)⇒ 恒声明,core 在死线−60s 处 suspend(基建合格)或响亮
@@ -296,10 +323,16 @@ export declare class RemoteK8sExecutionEnv implements RemoteExecutionEnv, Backgr
      * Persistence off / no store ⇒ a fresh pod, exactly as before. Persistence on but no such session (the occupant absent) ⇒ a
      * fresh pod and one `workspace_space_unavailable` line (no restore by any other key). The occupant read FAULTED ⇒ the same
      * loud `connect_failed` as a failed `latest` read, no pod ({@link provision} refuses it — every entry that would create one).
-     * One restore per env: a restore that ended refused ({@link restoreRefusal}) is answered as it stands to every later connect — the
-     * previous commit is not waited on again, `latest` is not read again, no pod is created.
+     * One connect in flight per env: a connect that arrives while another is running — the explicit one, the lazy one of the first operation,
+     * `reconnect` — gets THAT connect's answer (one pod creation, one restore, one refusal line), not a walk of its own; the share is for a connect in flight
+     * only (a connect that failed is not remembered by it — how long a failure stands is {@link connectRefusal}'s). The abort signal of the connect that
+     * started is the one in force.
+     * One refusal per env: a refusal this env holds ({@link connectRefusal} — a restore that did not come back, a pod the cluster refused at admission) is answered
+     * as it stands to every later connect on every lane — the previous commit is not waited on again, `latest` is not read again, no pod is created.
      */
     connect(config?: RemoteConnectConfig): Promise<ConnectResult>;
+    /** The one walk of a connect ({@link connect} runs it once at a time). */
+    private establish;
     /** This env holds a workspace its `destroy()` will commit ⇒ arm its seat on the space's commit fence (from here the
      *  run's terminal write fences the space; `workspace-space-commit.ts`). Each write of this run's row sets its deadline to
      *  now + the upload's own wall-clock bound (`dataTimeoutMs`). No-op unless the space lane is open. */
@@ -318,6 +351,16 @@ export declare class RemoteK8sExecutionEnv implements RemoteExecutionEnv, Backgr
     /** The space lane is on but this run's session has no occupant (no such session — {@link SpaceOccupant}) — said once per env
      *  (at connect, or at a destroy with a pod). `reason` = the reading's word. */
     private sayUnavailable;
+    /** Hold `error` as this env's refusal ({@link connectRefusal}) and answer it — the ONE writer of the slot (the restore and the pod's admission both write through it). */
+    private holdRefusal;
+    /**
+     * The cluster created this env's pod and it ended `Failed` (or `Succeeded`) before it ever ran — the kubelet refused it at admission
+     * (`SysctlForbidden` when `K8S_SANDBOX_IPV6=disabled` meets a kubelet that does not allow the two sysctls, a node-affinity miss, no capacity on the
+     * node …): the env's refusal ({@link holdRefusal}), in the one `connect_failed` form with the kubelet's reason in the sentence — so the operation that meets
+     * it, and every later one, says what the cluster said, instead of a new pod per operation. One `sandbox_pod_refused{pod, phase, reason, detail}` line per env
+     * (`detail` = the kubelet's own text, cut). A retry of the run builds a fresh env, which asks the cluster afresh.
+     */
+    private refusePod;
     /**
      * A lane that is on is keyed by an occupant read that FAULTED (`unreadable`) ⇒ this env creates no pod: the refusal, in the one
      * space-read failure form ({@link spaceReadFailed} — a failed `latest` / commit-row read says the same), naming the knob of every
@@ -335,16 +378,17 @@ export declare class RemoteK8sExecutionEnv implements RemoteExecutionEnv, Backgr
     /**
      * Lazy-connect on first use — 🔴 core NEVER calls `connect()` (verified: the Runner only calls `resumeVM` on
      * the resume path), so an adapter that demands an explicit connect can never work through the production
-     * hand-tool path (lazy-connect belongs in the adapter). Memoized so concurrent first ops share ONE
-     * pod create; `destroyed` is terminal. A failed connect clears the memo — how long a failure stands is {@link connect}'s, not
-     * this memo's: a failure of that attempt (a pod that could not be created, a space read that failed) ⇒ the next op connects
-     * afresh; the space restore's refusal is the env's ({@link restoreRefusal}) ⇒ the next op's connect answers that same refusal at once.
+     * hand-tool path (lazy-connect belongs in the adapter). `destroyed` is terminal. Concurrent first ops share ONE pod create — through
+     * {@link connect}'s one connect in flight, the same seat the explicit connect shares. How long a failure stands is {@link connect}'s: a failure of that
+     * attempt (a pod that could not be created, a space read that failed) ⇒ the next op connects afresh; the env's refusal ({@link connectRefusal}) ⇒ the
+     * next op's connect answers that same refusal at once.
      */
     private ensureConnected;
     /**
      * One pod creation attempt. With the cache volume open: first the space's claim ({@link attachCache} — found, or created),
      * then the pod carrying it; a pod that only waits on a claim that does not bind within the data bound is deleted and ONE pod
-     * without the volume is created in its place (the run starts; the miss is counted). Every other failure is the attempt's.
+     * without the volume is created in its place (the run starts; the miss is counted and remembered for the space — {@link K8sEnvConfig.cacheMisses}:
+     * the next runs inside one data bound go without it at once). Every other failure is the attempt's.
      */
     private connectOnce;
     /** The cache volume for this pod creation, or `undefined` (off, or not to be had — counted and said there). */
@@ -505,7 +549,7 @@ export declare class RemoteK8sExecutionEnv implements RemoteExecutionEnv, Backgr
      *  - space: the archive `latest` names is THERE and did not come back ({@link RestoreFailureClass}): the run is refused all the
      *    same — never an empty workspace (the user's changes are still in the bucket) — with exactly one `workspace_restore_failed`
      *    line (its `attempts`; an attempt that failed before one that succeeded is no line) and a refusal naming the two ways out,
-     *    held for the env ({@link restoreRefusal}).
+     *    held for the env ({@link connectRefusal}).
      * A pod that could not be created is that attempt's failure, not the archive's: returned as it is — no line, nothing held, the
      * next connect tries again. An abort during the pause ends the restore `aborted` (not the archive's either).
      */
@@ -536,16 +580,38 @@ export declare class RemoteK8sExecutionEnv implements RemoteExecutionEnv, Backgr
     private deletePod;
     /**
      * Delete the sandbox pod. Idempotent, never throws. Space lane open and a pod exists ⇒ FIRST the run-end space
-     * snapshot ({@link persistWorkspace}); its failure is loud and never blocks the delete (the pod lifetime is the
+     * snapshot ({@link persistWorkspace}) — after the pod's processes are confirmed stopped, the commands still in flight having ended or been cut first ({@link quiesce}); from the
+     * moment destroy begins no new command is admitted. Its failure is loud and never blocks the delete (the pod lifetime is the
      * leak guard). A run that never connected has no workspace to save — `latest` stays where it was. Space lane
      * unavailable (no such session) ⇒ nothing is saved (the once-per-env line says so); unreadable ⇒ there is no pod
      * ({@link provision} refused every creation).
      */
     destroy(): Promise<void>;
     cleanup(): Promise<void>;
+    /** A foreground command begins: register its abort controller (the one `runExec` listens to through the caller's signal composed with it). */
+    private beginExec;
+    /** A foreground command ended, however it ended: unregister it; the last one out wakes {@link whenIdle}. */
+    private endExec;
+    /** Resolves `true` once no command is in flight (at once when none is), `false` when `ms` passed first. */
+    private whenIdle;
+    /**
+     * The run is over and its workspace is about to be saved: the tar reads the tree only AFTER the pod's processes are CONFIRMED stopped
+     * ({@link killPodProcesses}: the script exits 0 only when nothing it should have killed is alive) — whether or not a command is in flight at this
+     * moment. The in-flight registry is not evidence of a quiet pod: a command ends however it ends (its own exit, a cancel, a wall) and drops out of
+     * the registry, yet ending a command does not end what it started (closing the exec socket does not stop the process). So the one exit without a
+     * stop is "no command and no pod" — nothing runs and there is no tree to read. Commands still in flight are first given one control-plane wall
+     * (`rpcTimeoutMs`) to end — a cancelled or timed-out run's commands were told to stop and end within moments, so the wait is normally short, and
+     * there is nothing to wait for when none is in flight. A wait that ran out also aborts the exec sockets still open and says one
+     * `workspace_snapshot_commands_cut{inFlight, waitedMs, killed}` line. Processes that could not be confirmed stopped ⇒ refused (`command_in_flight`, as a
+     * suspend is): a workspace that may still be changing is never saved as `latest`. Answers how long it took (the commit's data bound pays for it).
+     */
+    private quiesce;
+    /** Kill every live process of the pod but its init, its keeper and the killing shell, and confirm they are gone ({@link buildKillAllScript} exits 0 only then); `false` = not confirmed (the exec is bounded by `rpcTimeoutMs` plus the script's own bounded re-checks; `runExec` answers a Result and does not reject). */
+    private killPodProcesses;
     /**
      * Run-end space snapshot, under the space's commit fence (the run store's row, `workspace-space-commit.ts`): the fence
-     * first (this run's row moves to its upload state; lost ⇒ no upload, loud) → the one tar upload ({@link uploadSnapshot},
+     * first (this run's row moves to its upload state; lost ⇒ no upload, loud) → the pod's processes are stopped and confirmed, the commands still in flight
+     * first ended or cut ({@link quiesce}; the wait is part of the commit's one data bound) → the one tar upload ({@link uploadSnapshot},
      * lane `space` — it also releases the pod) → the fence again (this run's row moves to its publishing state; lost ⇒
      * `latest` is not published, loud) → move `latest` → keep N → the row is deleted (always, saved or failed).
      * A lost fence, an upload or a pointer failure ⇒ one `workspace_snapshot_failed` line + the `failed` outcome; `latest`
@@ -673,13 +739,16 @@ export declare class RemoteK8sExecutionEnv implements RemoteExecutionEnv, Backgr
  * managed-retention sweep). For each ended space it reclaims everything the space left behind, over this deployment's k8s config (it
  * never creates a pod): its build-cache volume (the claims labelled by the space key; only when the cache volume is configured),
  * its snapshot objects — the `space/` partition (archives + `latest`) and the park lane's objects in `<prefix>/<sessionId>/` that carry the ended occupant's mark (only when a
- * snapshot store is configured). Each part is independent: one part failing never skips another. A failed part ⇒ one operator line
- * (`workspace_cache_reclaim_failed{reason}` for the volume, `workspace_space_reclaim_failed{stage}` for the objects) + `recordFailOpen`; never throws.
+ * snapshot store is configured). Each part is independent: one part failing never skips another. A failed part ⇒ `recordFailOpen` per space + ONE operator line per
+ * CLASS of failure at the end of the call (`workspace_cache_reclaim_failed{reason}` for the volume, `workspace_space_reclaim_failed{lane, stage}` for the objects): the line names the
+ * first space of the call that met it and carries `count` = how many spaces met it (an RBAC drift on a sweep of hundreds of spaces is one line, not hundreds); never throws.
  * ONE call = ONE absolute deadline (the snapshot bound, `dataTimeoutMs`; no knob of its own), built at the entry and nowhere else: every request of the call — each
  * space's cache-volume delete and every LIST / DELETE of its objects — runs under it. A session delete calls with its one space; the managed-retention sweep calls
  * once per domain per round with the spaces that round ended there. When the deadline passes the request in flight is cut and every space not yet done is still
- * walked but sends nothing: its volume says `reason: "budget"`, its objects `stage: "budget"` (the volume stays for deletion by label, the objects for the
- * bucket's lifecycle rule). A caller that must not wait for the seat (the session-delete answer) starts it and does not await it.
+ * walked but sends nothing: its volume says `reason: "budget"`, its objects `stage: "budget"` — the budget classes' `count` is how many spaces were left (the volume stays for deletion by label,
+ * the objects for the bucket's lifecycle rule). ONE queue, ONE bound: at most {@link SPACE_RECLAIM_CONCURRENCY} spaces are worked on at once across ALL calls of this reclaimer (the
+ * sweep's spaces and every session delete go through the same gate, so a burst of deletes cannot start a burst of reclaims); a space waiting for its turn is waiting under its call's
+ * deadline. A caller that must not wait for the seat (the session-delete answer) starts it and does not await it.
  */
 export declare function createSpaceReclaimer(config: K8sEnvConfig, deps?: K8sEnvDeps): SpaceReclaimer;
 /**