@jr2/cli 0.4.1 → 0.6.0

package/src/kube.ts

@@ -5,6 +5,7 @@
 // one, shelling out to the same `kubectl` a human debugging the cluster would use.
 
 import { execFile, spawn } from "node:child_process";
+import { setTimeout as sleep } from "node:timers/promises";
 import { promisify } from "node:util";
 
 const exec = promisify(execFile);
@@ -31,7 +32,7 @@ function unreachable(e: unknown): string {
 
 // The fixed in-namespace object names live with the orchestrator (its entrypoint consumes them
 // too); re-exported here for the CLI's own modules.
-export { INSTANCE_SECRET, ORCHESTRATOR_PORT, ORCHESTRATOR_SERVICE } from "@jr2/orchestrator";
+export { INSTANCE_SECRET, ORCHESTRATOR_PORT, ORCHESTRATOR_SERVICE } from "@jr2/orchestrator/internal";
 
 export type KubePort = {
   /** The current kubectl context, or undefined when there is none configured. */
@@ -101,8 +102,23 @@ export type KubeAdmin = {
   /** `kubectl apply -f -` of a multi-doc YAML or JSON manifest string. `serverSide` is for an
    * object too large for client-side apply's `last-applied-configuration` annotation (256 KiB) —
    * the trust bundles, which carry every root certificate (ADR-0059); jr2 is its only writer, so it
-   * takes the fields it owns (`--force-conflicts`). */
-  apply(opts: { manifest: string; context?: string; serverSide?: boolean }): Promise<void>;
+   * takes the fields it owns (`--force-conflicts`).
+   *
+   * A Secret is ALWAYS applied server-side, whatever `serverSide` says (`splitSecrets`): a
+   * client-side apply copies the whole object, its values too, into that annotation, and a value
+   * there is in plaintext for every reader of the object's metadata. On a Secret that an older
+   * `jr2 up` applied client-side, the first server-side apply migrates: kubectl (1.26 and later)
+   * moves the client-side manager's fields to `jr2`, so that apply removes the annotation and each
+   * key that it does not name. A migration that fails is only a warning on kubectl's stderr. So a
+   * fallback removes the annotation from each Secret after the apply; when the fallback fails, it is
+   * one line to `warn` (default: stderr), and the apply goes on. The values go on stdin, never in
+   * argv. */
+  apply(opts: {
+    manifest: string;
+    context?: string;
+    serverSide?: boolean;
+    warn?: (line: string) => void;
+  }): Promise<void>;
   /** `kubectl label --overwrite`. */
   label(opts: {
     kind: string;
@@ -145,20 +161,224 @@ export type KubeAdmin = {
   /** `kubectl create --dry-run=server -o json -f -`: the object as admission would store it,
    * stored nowhere. THROWS on a refusal, with kubectl's reason. */
   dryRunCreate(opts: { manifest: string; context?: string }): Promise<{ spec?: { resources?: unknown } }>;
-  /** Run a one-shot node script IN the cluster (`kubectl run --rm`) and return its output — the
-   * provider-preflight seam (ADR-0019: reachability must be probed from where pods live).
-   * `caPem` makes the probe trust a private CA the same way the Harness does (ADR-0020):
-   * `NODE_EXTRA_CA_CERTS`, which must be process env at node start — it cannot be set from
-   * inside the script. */
-  runOneShot(opts: {
+  /** Wait until one pod has run to its end, and read its logs — the provider preflight's seam
+   * (ADR-0019, ADR-0059). Its pod has a Custodian beside the check container, which `kubectl run` cannot
+   * make, so `jr2 up` applies it and waits here. Two bounds: `startSeconds` until the check
+   * container runs (the image pulls and the Custodian's start), then `runSeconds` for the check. It
+   * stops early on a pod that cannot start (`podProgress`). Never throws. */
+  awaitPodEnd(opts: {
     namespace: string;
     name: string;
-    script: string;
-    caPem?: string;
+    container: string;
+    startSeconds: number;
+    runSeconds: number;
     context?: string;
-  }): Promise<string>;
+    /** Called with each new reason the pod gives for not running yet (`Unschedulable: …`). */
+    onPending?: (line: string) => void;
+    /** Called with the check container's log on each poll while it runs. */
+    onLog?: (log: string) => void;
+  }): Promise<PodEnd>;
+};
+
+/** How one pod ended (`awaitPodEnd`). `phase` is the pod's own, or `stuck` (it cannot start:
+ * `stuck` says why) or `timeout` (`started` says whether the check ran). `output` is the check
+ * container's whole log; `custodian` the Custodian's, its previous run's first when it restarted.
+ * `reasons` is each container's waiting or ended reason as the kubelet says it. */
+export type PodEnd = {
+  phase: "Succeeded" | "Failed" | "stuck" | "timeout";
+  started: boolean;
+  stuck?: string;
+  output: string;
+  custodian: string;
+  reasons: string[];
 };
 
+/** A container's status as the kubelet reports it: the part `podProgress` reads. */
+type CheckContainerStatus = {
+  name?: string;
+  restartCount?: number;
+  state?: Record<string, { reason?: string; message?: string }>;
+  /** Why the run before the current one ended: where an OOMKilled sidecar says so. */
+  lastState?: { terminated?: { reason?: string; exitCode?: number } };
+};
+
+/** The pod as `podProgress` reads it. */
+export type CheckPodObject = {
+  status?: {
+    phase?: string;
+    conditions?: Array<{ type?: string; status?: string; reason?: string; message?: string }>;
+    initContainerStatuses?: CheckContainerStatus[];
+    containerStatuses?: CheckContainerStatus[];
+  };
+};
+
+/** Why a pod that has not ended is not running yet, for a person who waits on it: the scheduler's
+ * reason, else the first container's waiting reason, else its phase. Undefined when it gives none,
+ * and for a pod that runs with no container waiting. */
+export function podPending(pod: CheckPodObject | undefined): string | undefined {
+  const unscheduled = pod?.status?.conditions?.find((c) => c.type === "PodScheduled" && c.status === "False");
+  if (unscheduled)
+    return `${unscheduled.reason ?? "not scheduled"}${unscheduled.message ? `: ${unscheduled.message}` : ""}`;
+  const statuses = [...(pod?.status?.initContainerStatuses ?? []), ...(pod?.status?.containerStatuses ?? [])];
+  for (const c of statuses) {
+    const waiting = c.state?.waiting;
+    if (waiting?.reason) return `${c.name ?? "?"}: ${waiting.reason}`;
+  }
+  // A pod that runs, with no container waiting, is not waiting to start: its check is on its way.
+  return pod?.status?.phase === "Running" ? undefined : pod?.status?.phase;
+}
+
+/** A volume a pod cannot mount because a Secret, a ConfigMap or a key it names is missing: the
+ * kubelet keeps the pod in ContainerCreating and says so in a FailedMount event alone — `secret "x"
+ * not found`, `references non-existent secret key: y`, `configmap references non-existent config
+ * key`. The first such event can be a race with the objects that the same apply wrote, so it counts
+ * only when the kubelet says it again: its `count` or `lastTimestamp` went up since an earlier
+ * poll. `seen` keeps each message's last value from poll to poll. */
+export function failedMount(
+  events: ReadonlyArray<{ reason?: string; message?: string; count?: number; lastTimestamp?: string }>,
+  seen: Map<string, string>,
+): string | undefined {
+  let found: string | undefined;
+  for (const e of events) {
+    if (e.reason !== "FailedMount" || !/not found|non-existent|couldn't find key|does not exist/i.test(e.message ?? ""))
+      continue;
+    const message = e.message ?? "";
+    const mark = `${e.count ?? 1}|${e.lastTimestamp ?? ""}`;
+    const before = seen.get(message);
+    if (before !== undefined && before !== mark) found ??= `FailedMount: ${message}`;
+    seen.set(message, mark);
+  }
+  return found;
+}
+
+/** The waiting reasons from which a pod does not come back by itself (ADR-0019): an image no node
+ * can pull, a Secret or ConfigMap key it names that is missing, a container that crashes at start. */
+export const TERMINAL_WAITING = new Set(["InvalidImageName", "CreateContainerConfigError", "CrashLoopBackOff"]);
+
+/** A pull that ends with one of these comes back by itself (a slow registry, a timeout, a rate
+ * limit); a pull whose message says that the image or the right to it is not there does not. */
+const PULL_WAITING = new Set(["ErrImagePull", "ImagePullBackOff"]);
+const PULL_FINAL = /not found|manifest unknown|unauthorized|denied/i;
+
+/**
+ * Where a check pod is (`awaitPodEnd`): ended (`phase`), unable to start (`stuck`, the reason), or
+ * still on its way, with whether `container` runs yet. A restart of `sidecar` is `stuck`: the
+ * Custodian restarts only when it refused to start, and the next start refuses alike. Pure, for its
+ * test.
+ */
+export function podProgress(
+  pod: CheckPodObject | undefined,
+  container: string,
+  sidecar: string,
+): { phase?: "Succeeded" | "Failed"; stuck?: string; running: boolean; reasons: string[] } {
+  const statuses = [...(pod?.status?.initContainerStatuses ?? []), ...(pod?.status?.containerStatuses ?? [])];
+  const reasons = statuses.flatMap((c) => [
+    ...Object.entries(c.state ?? {}).flatMap(([state, s]) =>
+      s.reason || s.message ? [`${c.name ?? "?"}: ${state} ${s.reason ?? ""} ${s.message ?? ""}`.trim()] : [],
+    ),
+    ...(c.lastState?.terminated?.reason ? [`${c.name ?? "?"}: last ended ${c.lastState.terminated.reason}`] : []),
+  ]);
+  const running = statuses.some((c) => c.name === container && c.state?.running !== undefined);
+  const phase = pod?.status?.phase;
+  if (phase === "Succeeded" || phase === "Failed") return { phase, running, reasons };
+  for (const c of statuses) {
+    const waiting = c.state?.waiting?.reason;
+    const pullFinal =
+      waiting !== undefined && PULL_WAITING.has(waiting) && PULL_FINAL.test(c.state?.waiting?.message ?? "");
+    if (waiting && (TERMINAL_WAITING.has(waiting) || pullFinal)) {
+      const last = c.lastState?.terminated?.reason;
+      return { stuck: `${c.name ?? "?"}: ${waiting}${last ? ` (last ended ${last})` : ""}`, running, reasons };
+    }
+    if (c.name === sidecar && (c.restartCount ?? 0) > 0) {
+      const last = c.lastState?.terminated?.reason;
+      return { stuck: `${sidecar}: restarted${last ? ` (last ended ${last})` : ""}`, running, reasons };
+    }
+  }
+  return { running, reasons };
+}
+
+/** What `waitPodEnd` reads the pod and the clock through: kubectl and the wall clock in
+ * `kubectlAdmin`, a script and a fake clock in its test. */
+export type PodEndPorts = {
+  pod(): Promise<CheckPodObject | undefined>;
+  /** The pod's events: where a FailedMount is told. */
+  events(): Promise<Array<{ reason?: string; message?: string; count?: number; lastTimestamp?: string }>>;
+  /** Each new reason the pod gives for not running yet. */
+  pending?(line: string): void;
+  /** The container's log so far, on each poll while it runs: its results as they come. */
+  log?(output: string): void;
+  logs(container: string, opts: { tailLines: number; previous?: boolean }): Promise<string>;
+  sleep(ms: number): Promise<unknown>;
+  now(): number;
+};
+
+/**
+ * The wait of `awaitPodEnd`: poll the pod every 2 s until it ends (`podProgress`), cannot start
+ * (a terminal waiting reason, a Custodian restart, a `FailedMount` event), or runs out of time —
+ * `startSeconds` until `container` runs, then `runSeconds` from then on. While it waits to start,
+ * each new reason it gives goes to `pending`. Then it reads the logs: the check container's whole log, for
+ * every model's line, read again when a pod that ended answers nothing; and the Custodian's last
+ * 50 lines, its run before first when it restarted.
+ */
+export async function waitPodEnd(
+  ports: PodEndPorts,
+  opts: { container: string; startSeconds: number; runSeconds: number },
+): Promise<PodEnd> {
+  const startBy = ports.now() + opts.startSeconds * 1000;
+  let runBy: number | undefined;
+  let seen: ReturnType<typeof podProgress>;
+  let end: PodEnd["phase"];
+  let restarted = false;
+  let said: string | undefined;
+  const mounts = new Map<string, string>();
+  for (;;) {
+    const pod = await ports.pod();
+    seen = podProgress(pod, opts.container, "custodian");
+    restarted ||= seen.stuck?.startsWith("custodian: restarted") === true;
+    if (seen.running && runBy === undefined) runBy = ports.now() + opts.runSeconds * 1000;
+    if (seen.phase) {
+      end = seen.phase;
+      break;
+    }
+    if (!seen.stuck && !seen.running) {
+      const mount = failedMount(await ports.events(), mounts);
+      if (mount) seen = { ...seen, stuck: mount };
+    }
+    if (seen.stuck) {
+      end = "stuck";
+      break;
+    }
+    if (ports.now() >= (runBy ?? startBy)) {
+      end = "timeout";
+      break;
+    }
+    if (seen.running && ports.log) ports.log(await ports.logs(opts.container, { tailLines: -1 }));
+    const pending = seen.running ? undefined : podPending(pod);
+    if (pending && pending !== said) ports.pending?.(pending);
+    said = pending;
+    await ports.sleep(2000);
+  }
+  // The whole log: one line per model. A check that ended always wrote one, so an empty read is a
+  // read that failed (kubectl answers nothing then): read again before calling it empty.
+  let output = await ports.logs(opts.container, { tailLines: -1 });
+  for (let n = 0; n < 3 && !output.trim() && (end === "Succeeded" || end === "Failed"); n++) {
+    await ports.sleep(1000);
+    output = await ports.logs(opts.container, { tailLines: -1 });
+  }
+  const custodian = [
+    ...(restarted ? [await ports.logs("custodian", { tailLines: 50, previous: true })] : []),
+    await ports.logs("custodian", { tailLines: 50 }),
+  ].join("\n");
+  return {
+    phase: end,
+    started: runBy !== undefined,
+    ...(seen.stuck ? { stuck: seen.stuck } : {}),
+    output,
+    custodian,
+    reasons: seen.reasons,
+  };
+}
+
 const nsArgs = (namespace?: string): string[] => (namespace ? ["--namespace", namespace] : []);
 
 /** The workload kinds `jr2 up` rolls out and waits on. */
@@ -270,9 +490,30 @@ export const kubectlAdmin: KubeAdmin = {
     return (JSON.parse(stdout) as { items?: never[] }).items ?? [];
   },
 
-  async apply({ manifest, context, serverSide }) {
-    const mode = serverSide ? ["--server-side", "--force-conflicts", "--field-manager=jr2"] : [];
-    await execStdin(["kubectl", ...ctxArgs(context), "apply", ...mode, "-f", "-"], manifest);
+  async apply({ manifest, context, serverSide, warn = (line) => process.stderr.write(`${line}\n`) }) {
+    const { secrets, identities, rest } = splitSecrets(manifest);
+    if (secrets !== undefined) {
+      await execStdin(["kubectl", ...ctxArgs(context), "apply", ...SERVER_SIDE, "-f", "-"], secrets);
+      // The fallback for a migration that failed. Removing an annotation that is not there is no
+      // fault, so this changes nothing once it is gone.
+      try {
+        await execStdin(
+          ["kubectl", ...ctxArgs(context), "annotate", "--overwrite", "-f", "-", `${LAST_APPLIED}-`],
+          identities!,
+        );
+      } catch (err) {
+        const names = (JSON.parse(identities!) as { items: Array<{ metadata: { name: string } }> }).items
+          .map((o) => o.metadata.name)
+          .join(", ");
+        warn(
+          `warning: the ${LAST_APPLIED} annotation was not removed from Secret ${names} ` +
+            `(${(err as Error).message}) — the next \`jr2 up\` tries again`,
+        );
+      }
+    }
+    if (rest !== undefined) {
+      await execStdin(["kubectl", ...ctxArgs(context), "apply", ...(serverSide ? SERVER_SIDE : []), "-f", "-"], rest);
+    }
   },
 
   async label({ kind, name, namespace, labels, context }) {
@@ -339,59 +580,32 @@ export const kubectlAdmin: KubeAdmin = {
     return JSON.parse(out) as { spec?: { resources?: unknown } };
   },
 
-  async runOneShot({ namespace, name, script, caPem, context }) {
-    // With a CA bundle: write the PEM to a file BEFORE node starts, because NODE_EXTRA_CA_CERTS
-    // is only honored as process-start env. The PEM rides an env var base64'd (multiline values
-    // and `--env` don't mix); `"$1"` keeps the script out of shell parsing entirely (execFile
-    // passes argv verbatim, no host shell either).
-    const command = caPem
-      ? ["sh", "-ec", 'echo "$JR2_CA_B64" | base64 -d > /tmp/jr2-ca.crt && exec node -e "$1"', "sh", script]
-      : ["node", "-e", script];
-    try {
-      const { stdout } = await exec(
-        "kubectl",
-        [
-          ...ctxArgs(context),
-          ...nsArgs(namespace),
-          "run",
-          name,
-          "--rm",
-          "--attach",
-          "--restart=Never",
-          "--quiet",
-          "--image=node:24-slim",
-          ...(caPem
-            ? [`--env=JR2_CA_B64=${Buffer.from(caPem).toString("base64")}`, "--env=NODE_EXTRA_CA_CERTS=/tmp/jr2-ca.crt"]
-            : []),
-          "--command",
-          "--",
-          ...command,
-        ],
-        { timeout: 180_000 },
-      );
-      return stdout;
-    } catch (err) {
-      throw oneShotFailure(err);
-    }
+  async awaitPodEnd({ namespace, name, container, startSeconds, runSeconds, context, onPending, onLog }) {
+    const ctx = context ? { context } : {};
+    return waitPodEnd(
+      {
+        pod: () => kubectlAdmin.getJson<CheckPodObject>({ kind: "pod", name, namespace, ...ctx }),
+        // A read that fails is no event: the wait goes on, and its bounds still hold.
+        events: () =>
+          kubectlAdmin
+            .listJson<{ reason?: string; message?: string; count?: number; lastTimestamp?: string }>({
+              kind: "event",
+              fieldSelector: `involvedObject.name=${name}`,
+              namespace,
+              ...ctx,
+            })
+            .catch(() => []),
+        ...(onPending ? { pending: onPending } : {}),
+        ...(onLog ? { log: onLog } : {}),
+        logs: (c, opts) => kubectlAdmin.logs({ namespace, pod: name, container: c, ...opts, ...ctx }),
+        sleep,
+        now: Date.now,
+      },
+      { container, startSeconds, runSeconds },
+    );
   },
 };
 
-/**
- * A one-shot probe's failure, made readable. `kubectl run --attach` streams the POD's own output
- * on ITS stdout, and kubectl writes only its verdict ("pod … terminated (Error)") to stderr — so
- * the one line that says WHY (a stack, a DNS error, an HTTP status) is exactly what execFile's
- * error message drops, since that message carries stderr alone. The provider preflight (ADR-0019)
- * exists to be the cheapest diagnosis available; without the pod's own words it can only report
- * that something in the cluster exited non-zero, and the caller is back to reproducing the probe
- * by hand. So the attached output leads, and the kubectl verdict follows it.
- */
-export function oneShotFailure(err: unknown): Error {
-  const attached = typeof (err as { stdout?: unknown })?.stdout === "string" ? (err as { stdout: string }).stdout : "";
-  const base = err instanceof Error ? err.message : String(err);
-  const trimmed = attached.trim();
-  return trimmed ? new Error(`${trimmed}\n${base}`) : err instanceof Error ? err : new Error(base);
-}
-
 /** What a rollout wait needs to know to go looking: the workload that did not come up (a Deployment
  * unless `kind` says otherwise), and the label selector its pods carry (each converge layer already
  * knows its own — it verifies the running image through the same selector). */
@@ -478,8 +692,7 @@ type EventObject = {
  * A rollout's failure, made readable (ADR-0046). `kubectl rollout status` reports its verdict and
  * nothing else — `error: timed out waiting for the condition` — so the fact that says WHY (an
  * `exec format error`, a pull the kubelet gave up on, a Secret that is not there) lives in the
- * pods, which the caller then goes and reads by hand. This is `oneShotFailure()` one layer up: the
- * evidence leads, kubectl's verdict follows, and `jr2 up`'s three rollout waits share the one pair
+ * pods, which the caller then goes and reads by hand. So the evidence leads, kubectl's verdict follows, and `jr2 up`'s three rollout waits share the one pair
  * of eyes.
  *
  * Every read here is best-effort by construction. It runs only after a failure has already
@@ -774,6 +987,60 @@ function resolveRef(ref?: string): string {
   return `docker.io/${ref.includes("/") ? ref : `library/${ref}`}`;
 }
 
+/** A server-side apply as jr2's own field manager. jr2 is the only writer of what it applies, so it
+ * takes the fields it names (`--force-conflicts`) from an older client-side apply. */
+const SERVER_SIDE = ["--server-side", "--force-conflicts", "--field-manager=jr2"];
+
+/** The annotation in which a client-side apply keeps a copy of the whole object it applied. */
+export const LAST_APPLIED = "kubectl.kubernetes.io/last-applied-configuration";
+
+/**
+ * A manifest, split for `apply`: its Secrets as one List to apply server-side, the names of those
+ * Secrets alone (for the annotation's removal), and the other objects as they were given. A
+ * Secret's `stringData` goes into `data` (base64): a server-side apply owns the fields it names,
+ * and `stringData` is write-only, so a key that a later apply leaves out would stay in `data`. A
+ * manifest that is not JSON (the operator's YAML, which holds no Secret) is all `rest`. Pure, for
+ * its test.
+ */
+export function splitSecrets(manifest: string): { secrets?: string; identities?: string; rest?: string } {
+  let parsed: { kind?: string; items?: Array<Record<string, unknown>> } & Record<string, unknown>;
+  try {
+    parsed = JSON.parse(manifest);
+  } catch {
+    return { rest: manifest };
+  }
+  const items = parsed.kind === "List" ? (parsed.items ?? []) : [parsed];
+  const secrets = items.filter((o) => o.kind === "Secret");
+  if (!secrets.length) return { rest: manifest };
+  const others = items.filter((o) => o.kind !== "Secret");
+  const list = (objects: unknown[]) => JSON.stringify({ apiVersion: "v1", kind: "List", items: objects });
+  return {
+    secrets: list(secrets.map(serverSideSecret)),
+    identities: list(
+      secrets.map((o) => {
+        const m = o.metadata as { name: string; namespace?: string };
+        return {
+          apiVersion: "v1",
+          kind: "Secret",
+          metadata: { name: m.name, ...(m.namespace ? { namespace: m.namespace } : {}) },
+        };
+      }),
+    ),
+    ...(others.length ? { rest: list(others) } : {}),
+  };
+}
+
+/** A Secret with its `stringData` in `data`, as Kubernetes stores it: a key in both takes the
+ * `stringData` value, as the API server merges them. */
+function serverSideSecret(o: Record<string, unknown>): Record<string, unknown> {
+  const { stringData, ...secret } = o as { stringData?: Record<string, string> } & Record<string, unknown>;
+  if (stringData === undefined) return secret;
+  const encoded = Object.fromEntries(
+    Object.entries(stringData).map(([k, v]) => [k, Buffer.from(v, "utf8").toString("base64")]),
+  );
+  return { ...secret, data: { ...((secret.data as Record<string, string> | undefined) ?? {}), ...encoded } };
+}
+
 /** Run a command feeding `stdin`, surfacing stderr in the thrown error (kubectl's messages are the
  * useful part of an apply failure). */
 function execStdin(cmd: string[], stdin: string): Promise<void> {

package/src/landlock.ts

@@ -0,0 +1,129 @@
+// jr2 requires Landlock on every Sandbox node (ADR-0077). The Repo cache agent is the detector: its
+// DaemonSet has one pod on each Sandbox node, by the Sandboxes' own placement (ADR-0052), and on a
+// node without Landlock, or with an ABI below 3, the agent exits at start with a termination
+// message that names Landlock, the node and the probe's reason, which states the ABI it found
+// (`operator/internal/repocache/landlock.go`). jr2's minimum is ABI 3: under ABI 1 no confined
+// process can move a file into another folder, and under ABI 2 a confined process can truncate a
+// file of jr2's that it may not write. That pod never gets Ready, so the cache agent's
+// rollout never completes.
+//
+// `jr2 up` reads the DaemonSet's pods within the rollout wait it already makes, and refuses to
+// converge while any of them reports no Landlock. It names each node, the reason, and the fix: keep
+// the node out of the Sandbox nodes with `sandbox.nodeSelector` or a taint, or enable Landlock. There
+// is no check pod, no node label and no flag to skip it. A run after the fix converges.
+
+import { setTimeout as sleep } from "node:timers/promises";
+import type { KubeAdmin, RolloutTarget } from "./kube.ts";
+
+/** The start of the cache agent's termination message on a node without Landlock — the Go side's
+ * `LandlockMessagePrefix`. The two change together. */
+export const LANDLOCK_MESSAGE_PREFIX = "no Landlock on node ";
+
+/** How often `awaitCacheAgent` reads the DaemonSet's pods while the rollout wait runs. */
+const POLL_MS = 3000;
+
+type Terminated = { message?: string };
+type ContainerStatus = {
+  state?: { terminated?: Terminated };
+  lastState?: { terminated?: Terminated };
+};
+/** A cache agent pod, read for the node it runs on and its containers' termination messages. */
+export type CacheAgentPod = {
+  metadata: { name: string; deletionTimestamp?: string };
+  spec?: { nodeName?: string };
+  status?: { containerStatuses?: ContainerStatus[] };
+};
+
+/** One Sandbox node whose cache agent reports no Landlock, and the probe's reason. */
+export type NoLandlock = { node: string; reason: string };
+
+// The node, then the reason up to the sentence the agent ends its message with. Not anchored: with
+// `FallbackToLogsOnError` the message can be the agent's log line, which quotes it.
+const MESSAGE = /no Landlock on node ([^\s:]+): (.+?) — jr2 needs Landlock/;
+
+/** The Sandbox nodes whose cache agent reports no Landlock, each once, in node order. */
+export function nodesWithoutLandlock(pods: readonly CacheAgentPod[]): NoLandlock[] {
+  const found = new Map<string, string>();
+  for (const pod of pods) {
+    if (pod.metadata.deletionTimestamp) continue;
+    for (const s of pod.status?.containerStatuses ?? []) {
+      const message = s.state?.terminated?.message ?? s.lastState?.terminated?.message ?? "";
+      const match = MESSAGE.exec(message);
+      if (!match) continue;
+      const node = pod.spec?.nodeName ?? match[1]!;
+      if (!found.has(node)) found.set(node, match[2]!);
+    }
+  }
+  return [...found].sort(([a], [b]) => a.localeCompare(b)).map(([node, reason]) => ({ node, reason }));
+}
+
+/** The refusal (ADR-0077, ADR-0019): each node, its reason, and the fix. */
+export function landlockRefusal(nodes: readonly NoLandlock[]): Error {
+  return new Error(
+    [
+      `repo cache: ${nodes.length === 1 ? "a Sandbox node has" : `${nodes.length} Sandbox nodes have`} no Landlock, ` +
+        "and jr2 needs it on every Sandbox node (ADR-0077):",
+      ...nodes.map((n) => `  ${n.node}: ${n.reason}`),
+      "Keep each such node out of the Sandbox nodes with sandbox.nodeSelector or a taint (ADR-0052), or enable " +
+        "Landlock ABI 3 or later in its kernel (Linux 6.2 or later, or a kernel with a backport, with landlock in " +
+        "its lsm= list). Then run `jr2 up` again.",
+    ].join("\n"),
+  );
+}
+
+/**
+ * Wait for the cache agent's rollout, and refuse as soon as one of its pods reports no Landlock:
+ * a pod that exits at start never gets Ready, so the wait alone would only time out. `rollout` is
+ * the converge's own rollout wait (with its own diagnosis on failure); while it runs, the
+ * DaemonSet's pods are read every few seconds, one list call each. A failed rollout is read once
+ * more, so a Landlock refusal wins over the generic diagnosis.
+ */
+export async function awaitCacheAgent(
+  kube: KubeAdmin,
+  target: RolloutTarget,
+  rollout: () => Promise<void>,
+  opts: { pollMs?: number } = {},
+): Promise<void> {
+  const read = async (): Promise<NoLandlock[]> => {
+    try {
+      return nodesWithoutLandlock(
+        await kube.listJson<CacheAgentPod>({
+          kind: "pod",
+          selector: target.selector,
+          namespace: target.namespace,
+          ...(target.context ? { context: target.context } : {}),
+        }),
+      );
+    } catch {
+      // A read that fails is no evidence: the rollout wait still decides.
+      return [];
+    }
+  };
+  const stop = new AbortController();
+  const waited = rollout().then(
+    () => ({ ok: true as const }),
+    (err: unknown) => ({ ok: false as const, err }),
+  );
+  const polled = (async (): Promise<NoLandlock[] | undefined> => {
+    for (;;) {
+      try {
+        await sleep(opts.pollMs ?? POLL_MS, undefined, { signal: stop.signal });
+      } catch {
+        return undefined;
+      }
+      const nodes = await read();
+      if (nodes.length > 0) return nodes;
+    }
+  })();
+  const first = await Promise.race([waited, polled.then((nodes) => ({ nodes }))]);
+  stop.abort();
+  if ("nodes" in first) {
+    if (first.nodes) throw landlockRefusal(first.nodes);
+    // The poll stops only when aborted, so the rollout settled first.
+  }
+  const result = await waited;
+  if (result.ok) return;
+  const nodes = await read();
+  if (nodes.length > 0) throw landlockRefusal(nodes);
+  throw result.err;
+}

package/src/nodes.ts

@@ -7,7 +7,13 @@
 // at the moment of the read. The set moves (a pool autoscales, a node is cordoned), which is why an
 // empty set is reported and never refused.
 
-import { cpuMillis, memoryBytes, type ResolvedSize, type SandboxPlacement, type Toleration } from "@jr2/orchestrator";
+import {
+  cpuMillis,
+  memoryBytes,
+  type ResolvedSize,
+  type SandboxPlacement,
+  type Toleration,
+} from "@jr2/orchestrator/internal";
 
 /** A cluster node, read for the facts that decide placement and platform (ADR-0045/0052). */
 export type NodeObject = {

package/src/output.ts

@@ -24,7 +24,7 @@ export type Io = {
   kubeAdmin?: KubeAdmin;
   /** Override the image build port; default → pnpm + docker + kind subprocesses. */
   build?: BuildPort;
-  /** Override `jr2 up`'s Instance typecheck (ADR-0050); default → the Instance's own `tsc`. */
+  /** Override the Instance typecheck of `jr2 up` and `jr2 check` (ADR-0050); default → the Instance's own `tsc`. */
   typecheck?: TypecheckPort;
   /** Where kit-checkout detection starts walking up from (ADR-0038); default → the CLI's own
    * module directory, which is the whole signal: a checkout resolves the kit sources, an npm

package/src/repo-sweep.ts

@@ -11,7 +11,7 @@
 // no Repos — the one read that may answer "none" — while any other failure throws and the caller
 // sweeps nothing.
 
-import { ANNOTATION_REPO_LAST_ATTACHED, LABEL_REPO_BOUND } from "@jr2/orchestrator";
+import { ANNOTATION_REPO_LAST_ATTACHED, LABEL_REPO_BOUND } from "@jr2/orchestrator/internal";
 import { LABEL_INSTANCE } from "./deploy.ts";
 import { isMissingResourceType, type KubeAdmin } from "./kube.ts";
 import { activity, type Io } from "./output.ts";

package/src/sweep.ts

@@ -24,12 +24,13 @@
 // all: a cluster with no Sandbox CRD holds no Sandbox CRs, so that read's "none" is complete (see
 // `listSandboxes`).
 //
-// Accepted, and written down so nobody adds a name filter to "fix" it (ADR-0039): the host keep set
-// only sees the CURRENT context's roots, so a second checkout converging to a different cluster can
-// have its host kit generation swept. The rebuild is BuildKit-cached seconds. Re-introducing a name
-// filter would resurrect the exact primitive this ADR deletes.
+// Written down so nobody adds a name filter to "fix" it (ADR-0039): the host keep set only sees the
+// CURRENT context's roots, so a converge to a different cluster — a second checkout's, or this
+// one's — can lose a host image it built and has not used yet. `jr2 up` answers that itself: a tag
+// it counted on and finds missing is rebuilt (BuildKit-cached seconds) and the use retried.
+// Re-introducing a name filter would resurrect the exact primitive this ADR deletes.
 
-import { IMAGES_CONFIGMAP, IMAGES_KEY } from "@jr2/orchestrator";
+import { IMAGES_CONFIGMAP, IMAGES_KEY } from "@jr2/orchestrator/internal";
 import { formatBytes, mergeSweeps, sweepHost, sweepNodes, type BuildPort, type SweepResult } from "./build.ts";
 import { LABEL_INSTANCE, OPERATOR_NAMESPACE } from "./deploy.ts";
 import { isMissingResourceType, type KubeAdmin } from "./kube.ts";