@jr2/orchestrator 0.5.1 → 0.7.0

package/src/sandbox-kube.ts

@@ -43,8 +43,10 @@
 // containers:
 //
 //   volume        runtime     the kit's Harness image, as an `image` volume → /opt/jr2
-//   volume        conversations  an emptyDir → /conversations, the Harness container's alone
+//   volume        conversations  an emptyDir → /run/jr2/conversations, the Harness container's alone
 //                             (ADR-0031: a Harness restart inside the pod keeps its conversations)
+//   volume        shell       an emptyDir → /run/jr2/shell, the Harness container's alone: the
+//                             socket of a person's shell (`jr2 workspace shell`, ADR-0076)
 //   initContainer preflight   the USER'S image + that volume → ADR-0037's probe, the thing that
 //                             proves a registry ref, whose first appearance is this provision
 //   container     harness     the Sandbox Image, command overridden, /work + /opt/jr2 mounted
@@ -83,11 +85,14 @@ import {
   custodianBootstrap,
   custodianComposition,
   custodianScript,
+  CREDENTIAL_KEYS,
+  heldStandInKey,
   CUSTODIAN_BOOTSTRAP_KEY,
   CUSTODIAN_SCRIPT_KEY,
   type CustodianComposition,
 } from "./custodian.ts";
-import { readHeldManifest, workspaceHeldPod, type HeldManifest } from "./held-secrets.ts";
+import { heldStandIn, readHeldManifest, workspaceHeldPod, type HeldManifest } from "./held-secrets.ts";
+import { markHarnessEnv } from "./harness-env.ts";
 import {
   effectorManifestKey,
   readSandboxEffectorManifest,
@@ -102,6 +107,8 @@ import {
   CONVERSATIONS_DIR,
   CONVERSATIONS_DIR_ENV,
   CONVERSATIONS_VOLUME,
+  SHELL_DIR,
+  SHELL_VOLUME,
   HELD_MOUNT,
   CA_CONFIGMAP,
   heldManifestKey,
@@ -112,6 +119,15 @@ import {
   PRIORITY_CLASS_SANDBOX,
   REPOS_MOUNT,
   sandboxCustodianConfigMap,
+  ANNOTATION_HOLD,
+  ANNOTATION_KEEPALIVE,
+  ANNOTATION_WORKSPACE_PATH,
+  HOLD_HELD,
+  HOLD_RELEASED,
+  LABEL_SANDBOX_RUN,
+  LABEL_SANDBOX_WORKFLOW,
+  LABEL_SANDBOX_WORKSPACE,
+  workspaceLabel,
 } from "./names.ts";
 import {
   CONFIGMAPS,
@@ -133,10 +149,14 @@ import {
   type SandboxObject,
   type SandboxWatch,
 } from "./sandbox-watch.ts";
-import { effectorKey, harnessToken, harnessTokenDigest, sandboxToken } from "./tokens.ts";
+import { controlStandIn, controlToken, effectorKey, harnessToken, harnessTokenDigest, sandboxToken } from "./tokens.ts";
 import type { AttachError, AttachRequest, AttachResponse } from "./wire.ts";
 import type { Continuity, MemoryKill, PlacingWait, ProvisionedRepo, SandboxLoss, SandboxPort } from "./workspace.ts";
 
+/** A Secret's `data`: each value base64'd. */
+const base64Values = (values: Record<string, string>): Record<string, string> =>
+  Object.fromEntries(Object.entries(values).map(([k, v]) => [k, Buffer.from(v).toString("base64")]));
+
 /** Where jr2's runtime lands in every container that gets it (ADR-0037). `/opt/jr2` and not `/app`
  * because a stranger's base may already use `/app`, and one layout must serve both the stock
  * Harness image and an arbitrary Sandbox Image. It is a PUBLISHED surface: `bin/` beside `lib/`
@@ -156,11 +176,21 @@ const RUNTIME_VOLUME_MOUNT = { name: RUNTIME_VOLUME, mountPath: RUNTIME_MOUNT, s
 /** The primary container's command (ADR-0037). Absolute, so it never depends on the image's
  * `WORKDIR`, and identical to the stock Harness image's own `CMD` — one runtime, two placements.
  * `tini` is PID 1 (ADR-0061): the Agent's processes are the Harness's children, and a tree the
- * memory guard kills must leave no zombies behind. */
+ * memory guard kills must leave no zombies behind. The Harness holds the Stand-ins in its memory,
+ * and below Landlock ABI 6 a confined child can still signal it (ADR-0076): `effector nocore`
+ * starts it with no core file, so a dumping signal writes nothing a child reads, and
+ * `--disable-sigusr1` keeps SIGUSR1 from opening Node's inspector, which would run a child's code
+ * in the Harness. `--harness-env` gives the Harness process only the names jr2 composed, less each
+ * name that loads code from a path, and the domain the whole start env (`harness-env.ts`). */
 const HARNESS_COMMAND = [
   `${RUNTIME_MOUNT}/bin/tini`,
   "--",
+  `${RUNTIME_MOUNT}/bin/effector`,
+  "nocore",
+  "--harness-env",
+  "--",
   `${RUNTIME_MOUNT}/bin/node`,
+  "--disable-sigusr1",
   `${RUNTIME_MOUNT}/src/main.ts`,
 ];
 
@@ -753,6 +783,7 @@ export function kubeSandbox(opts: KubeSandboxOptions = {}): SandboxPort {
       name: string;
       runId: string;
       workflow: string;
+      path: string;
       image?: string;
       user?: string;
       workGroup?: number;
@@ -785,8 +816,16 @@ export function kubeSandbox(opts: KubeSandboxOptions = {}): SandboxPort {
       metadata: {
         name: req.name,
         namespace: ns,
-        // The run↔workspace link `jr2 ls` groups by (ADR-0009/0012) — readable without the host.
-        labels: { "jr2.dev/run": req.runId, "jr2.dev/workflow": req.workflow },
+        // The run↔workspace link the `jr2 workspace` verbs read (ADR-0009/0012), without the host:
+        // the run, the Workflow and a selector form of the path as labels, the exact path as an
+        // annotation. The hold mark is `held` from this first write (ADR-0021); the Lease writes it
+        // again with each renewal, and `released` when it stops for good.
+        labels: {
+          [LABEL_SANDBOX_RUN]: req.runId,
+          [LABEL_SANDBOX_WORKFLOW]: req.workflow,
+          ...(req.path ? { [LABEL_SANDBOX_WORKSPACE]: workspaceLabel(req.path) } : {}),
+        },
+        annotations: { ...(req.path ? { [ANNOTATION_WORKSPACE_PATH]: req.path } : {}), [ANNOTATION_HOLD]: HOLD_HELD },
       },
       spec: {
         // The Sandbox Image, unmodified (ADR-0037) — the user's tools, its own USER and HOME, and
@@ -834,8 +873,14 @@ export function kubeSandbox(opts: KubeSandboxOptions = {}): SandboxPort {
         // Never empty: the Custodian's address is unconditional. The seat's own vars (the fallback
         // `HOME`) come FIRST, so the instance's `harness.env` can still override them the way it
         // overrides anything the image set.
-        env: [...seat.env, ...harnessEnv(req.name, custodian, cpuHintEnv())],
-        ...(cfg().envFrom?.length ? { envFrom: cfg().envFrom } : {}),
+        // Marked (`harness-env.ts`, ADR-0076): the Harness's start tells jr2's names from the image's.
+        ...(() => {
+          const marked = markHarnessEnv(
+            [...seat.env, ...harnessEnv(req.name, custodian, cpuHintEnv())],
+            cfg().envFrom ?? [],
+          );
+          return { env: marked.env, ...(marked.envFrom.length ? { envFrom: marked.envFrom } : {}) };
+        })(),
         // What the AGENT gets: an address on its own loopback and a Stand-in for every credential,
         // and no credential anywhere. The Custodian beside it holds them (ADR-0013, ADR-0059).
         sidecars,
@@ -865,6 +910,9 @@ export function kubeSandbox(opts: KubeSandboxOptions = {}): SandboxPort {
           // that Workspace is `Lost` (ADR-0021) — and mounted below into the Harness container
           // ALONE: the User Container must not read an Agent's history.
           { name: CONVERSATIONS_VOLUME, emptyDir: {} },
+          // The shell socket's folder (ADR-0076): the Harness listens there for `jr2 workspace
+          // shell`, in the Harness container ALONE, where no process that it confines makes a file.
+          { name: SHELL_VOLUME, emptyDir: {} },
           // Only for ADR-0037's fallback seat: uid 1000 on a stranger's base has no home at all.
           ...seat.homeVolume,
           // The Custodian's (its values, leaves and config), and the trust ConfigMap (ADR-0020).
@@ -879,6 +927,7 @@ export function kubeSandbox(opts: KubeSandboxOptions = {}): SandboxPort {
           { name: "work", mountPath: WORK_ROOT },
           RUNTIME_VOLUME_MOUNT,
           { name: CONVERSATIONS_VOLUME, mountPath: CONVERSATIONS_DIR },
+          { name: SHELL_VOLUME, mountPath: SHELL_DIR },
           ...seat.homeMount,
           ...custodian.harnessMounts,
         ],
@@ -902,7 +951,7 @@ export function kubeSandbox(opts: KubeSandboxOptions = {}): SandboxPort {
    * `data`, not `stringData`: a server-side apply owns the fields it names, and `stringData` is
    * write-only — it never reads back as the field applied.
    */
-  const applyTokenSecret = async (name: string, owner: KubeObject): Promise<void> => {
+  const applyTokenSecret = async (name: string, owner: KubeObject, manifest: HeldManifest): Promise<void> => {
     if (!opts.signingKey) throw new Error("kubeSandbox: a Custodian needs a signingKey to mint its Sandbox token");
     const uid = owner.metadata?.uid;
     if (!uid) throw new Error(`Sandbox "${name}": the CR's apply answered with no uid to own its token Secret`);
@@ -925,7 +974,19 @@ export function kubeSandbox(opts: KubeSandboxOptions = {}): SandboxPort {
         ],
       },
       type: "Opaque",
-      data: { JR2_SANDBOX_TOKEN: Buffer.from(sandboxToken(opts.signingKey, name)).toString("base64") },
+      // The pod's credentials (ADR-0076, C1, C1b): its fetch token and its control token, which the
+      // Custodian alone mounts; its secret control Stand-in; and one secret Stand-in per held secret
+      // the pod's manifest names, which the Custodian (for its swap) and the Harness container (as
+      // files under `/run/jr2/`) mount. Every one a signed name, so a re-provision re-applies the
+      // same values. Never a ConfigMap, the CR, an env, an event or a log line.
+      data: base64Values({
+        [CREDENTIAL_KEYS.fetchToken]: sandboxToken(opts.signingKey, name),
+        [CREDENTIAL_KEYS.controlToken]: controlToken(opts.signingKey, name),
+        [CREDENTIAL_KEYS.controlStandIn]: controlStandIn(opts.signingKey, name),
+        ...Object.fromEntries(
+          manifest.secrets.map((s) => [heldStandInKey(s.name), heldStandIn(opts.signingKey!, name, s.name)]),
+        ),
+      }),
     });
   };
 
@@ -1030,7 +1091,7 @@ export function kubeSandbox(opts: KubeSandboxOptions = {}): SandboxPort {
       },
       data: {
         [CUSTODIAN_BOOTSTRAP_KEY]: JSON.stringify(custodianBootstrap(manifest, orchestrator)),
-        [CUSTODIAN_SCRIPT_KEY]: custodianScript(manifest),
+        [CUSTODIAN_SCRIPT_KEY]: custodianScript(manifest, { fetch: true }),
       },
     });
   };
@@ -1257,8 +1318,9 @@ export function kubeSandbox(opts: KubeSandboxOptions = {}): SandboxPort {
         image: refs.custodian,
         pod,
         config: sandboxCustodianConfigMap(req.name),
-        token: { secret: secretName(req.name), key: "JR2_SANDBOX_TOKEN" },
+        credentials: { secret: secretName(req.name) },
         sandbox: req.name,
+        ...(req.secrets?.length ? { granted: req.secrets } : {}),
         caBundle: cfg().caBundle === true,
         resources: split.custodian,
       });
@@ -1287,7 +1349,7 @@ export function kubeSandbox(opts: KubeSandboxOptions = {}): SandboxPort {
       sandboxes();
       const applied = await kube().apply(SANDBOXES, ns, cr).catch(noOperator(req.name, "Sandbox"));
       // After the CR: its uid owns both, so they go when it goes.
-      await applyTokenSecret(req.name, applied);
+      await applyTokenSecret(req.name, applied, manifest);
       await applyCustodianConfig(req.name, applied, manifest);
       if (effectorManifest) {
         await applyEffectorKey(req.name, applied);
@@ -1403,24 +1465,44 @@ export function kubeSandbox(opts: KubeSandboxOptions = {}): SandboxPort {
           const { identity, key } = repoIdentity(r.url);
           return { slot: r.slot, url: r.url, identity, key, ...(r.ref !== undefined ? { ref: r.ref } : {}) };
         }),
-        branch: req.spec.branch,
-        ...(req.spec.reviewSha ? { reviewSha: req.spec.reviewSha } : {}),
       };
-      const { repos, review } = await postAttach(req.name, endpoint, body);
+      const { repos, bases } = await postAttach(req.name, endpoint, body);
       const stale = staleSlots(req.repos, staleByName.get(req.name));
-      return { repos, ...(review ? { review } : {}), ...(stale ? { stale } : {}) };
+      return { repos, bases, ...(stale ? { stale } : {}) };
     },
 
     leaseIntervalMs,
 
     async renew(name) {
-      // A write, and only that (ADR-0021): the stamp the operator's idle GC reads. What the
-      // workspace IS comes from the watch, never from this answer.
+      // A write, and only that (ADR-0021): the stamp the operator's idle GC reads, and the hold
+      // mark, written again so a restored run's Lease marks the Sandbox `held`. What the workspace
+      // IS comes from the watch, never from this answer.
       await kube().patch(SANDBOXES, ns, name, {
-        metadata: { annotations: { "jr2.dev/keepalive": new Date().toISOString() } },
+        metadata: { annotations: { [ANNOTATION_KEEPALIVE]: new Date().toISOString(), [ANNOTATION_HOLD]: HOLD_HELD } },
       });
     },
 
+    async releaseHold(name) {
+      // The Lease stopped for good (ADR-0021): no run holds this Sandbox now, and `jr2 workspace rm`
+      // may remove it. A write, and only that.
+      await kube().patch(SANDBOXES, ns, name, { metadata: { annotations: { [ANNOTATION_HOLD]: HOLD_RELEASED } } });
+    },
+
+    async releaseRun(runId) {
+      // A restore ended this run without hosting it (ADR-0021): no Lease runs, so each of its
+      // Sandboxes gets the mark here. The list is the cluster's, not the watch's cache, which may
+      // not have listed yet at boot.
+      const list = await kube().list<SandboxObject>(SANDBOXES, ns, { labelSelector: `${LABEL_SANDBOX_RUN}=${runId}` });
+      await Promise.all(
+        list.items.flatMap((sb) => {
+          const name = sb.metadata?.name;
+          return name
+            ? [kube().patch(SANDBOXES, ns, name, { metadata: { annotations: { [ANNOTATION_HOLD]: HOLD_RELEASED } } })]
+            : [];
+        }),
+      );
+    },
+
     continuity(name, listener) {
       // The watch's word on this name (ADR-0063): the first list, then every event. A dropped
       // watch says nothing, so this says nothing either — unknown is never loss.
@@ -1458,7 +1540,20 @@ export function kubeSandbox(opts: KubeSandboxOptions = {}): SandboxPort {
     async destroy(name) {
       staleByName.delete(name);
       // The token Secret is the CR's child from birth, so deleting the CR reaps it (ADR-0001).
-      await kube().delete(SANDBOXES, ns, name);
+      await kube().delete(SANDBOXES, ns, name, { foreground: true });
+      // A retry of the same invoke id takes the same name, and the pod takes it too: a new Sandbox
+      // must not find the old pod on its way out, and read `Lost` (ADR-0075). So the wait is
+      // for the pod as well: foreground keeps the CR until the pod is gone. Bounded: a stuck finalizer
+      // is the operator's to report, and the run does not wait on it for ever.
+      for (let i = 0; i < 120; i++) {
+        if (
+          (await kube()
+            .get<SandboxObject>(SANDBOXES, ns, name)
+            .catch(() => undefined)) === undefined
+        )
+          return;
+        await new Promise((resolve) => setTimeout(resolve, 500));
+      }
     },
   };
 }