@jr2/cli 0.4.1 → 0.6.0

package/src/commands/up.ts

@@ -1,11 +1,13 @@
-// `jr2 up [--yes] [--force] [-n <ns>] [--context <ctx>]` (ADR-0019): idempotently converge the target
-// namespace to this instance — every layer, loudly narrated, safe to re-run. Layers in order:
-// typecheck → the Machine walk → staging the Assembly (ADR-0069) → the Kubernetes floor (ADR-0037)
-// → ownership → image resolution → operator → Kit images (the Harness, the Orchestrator) → Sandbox
-// Images → Secret (+ preflight of referenced Secrets) → apply + rollout → the cache agent
-// (ADR-0051: a DaemonSet converged when a registered Machine composes a Sandbox, deleted when none
-// does) → Instance Harness (ADR-0031: converged by convention when a carried definition declares
-// `workspace: "none"`, deleted when none does) → Instance Effector → publishing the Assembly to the
+// `jr2 up [--yes] [--force] [--skip-provider-preflight] [-n <ns>] [--context <ctx>]` (ADR-0019): idempotently converge the target
+// namespace to this instance — every layer, loudly narrated, safe to re-run. Layers in order: the
+// AGENTS.md block (ADR-0072: a file in the Instance folder, not the cluster) → typecheck → the
+// Machine walk → staging the Assembly (ADR-0069) → the Kubernetes floor (ADR-0037) → ownership →
+// image resolution → Kit images (the operator, the Harness, the Orchestrator) → Sandbox Images →
+// Secret (+ preflight of referenced Secrets) → the provider preflight (ADR-0019, ADR-0059: through a
+// Custodian, in pods that read their own objects) → operator → git ssh keys → apply → rollout → the cache
+// agent (ADR-0051: a DaemonSet converged when a registered Machine composes a Sandbox, deleted
+// when none does) → Instance Harness (ADR-0031: converged by convention when a carried definition
+// declares `workspace: "none"`, deleted when none does) → Instance Effector → publishing the Assembly to the
 // running Orchestrator (ADR-0069) → a report of live workspaces still on an older image.
 //
 // The Assembly is staged EARLY and published LAST (ADR-0069). Early, so its refusals (a second copy
@@ -15,7 +17,7 @@
 // version the Assembly names. A Machine edit therefore rolls nothing: it is a new Assembly, not a
 // new pod template. The Orchestrator creates a Repo resource per
 // Repo the walk found bound, at boot and at each publish, and the cache agent clones it onto a node on first need
-// (ADR-0051); a configured custom provider is preflighted from inside the cluster. A bound ssh url
+// (ADR-0051); every provider a carried model names is preflighted from inside the cluster. A bound ssh url
 // asks where its deploy key comes from (ADR-0047), and a converge that GENERATED one ends by saying
 // so — the key is dead until a human registers it.
 //
@@ -25,7 +27,9 @@
 // the one converge-time refusal the compiler cannot make (ADR-0051): a registered Machine whose
 // Repo Slot is still open has nothing bound where the run would need a url, and a `workflows/`
 // export has no type to hang that on. Nothing is built, and nothing on the cluster is touched,
-// before the compiler has agreed the folder is coherent and every slot is bound.
+// before the compiler has agreed the folder is coherent and every slot is bound. The two are one
+// function, `checkInstance` (instance-check.ts), which `jr2 check` runs alone with no cluster, so
+// an Assistant that may not run `jr2 up` gets the same refusals from the same code.
 //
 // Images (ADR-0038, as amended by ADR-0045): `jr2 up` builds every image it deploys, and every tag is
 // a content address of (its own inputs × the platform set it was built for) — `<hash>-<arch>`, with
@@ -55,11 +59,11 @@
 // (`--yes` for CI); objects labeled as a DIFFERENT instance → refuse.
 
 import { createHash, createHmac, randomBytes } from "node:crypto";
+import { setTimeout as sleepMs } from "node:timers/promises";
 import { open, readdir, readFile } from "node:fs/promises";
 import { homedir } from "node:os";
 import { basename, join, relative } from "node:path";
 import { fileURLToPath } from "node:url";
-import { parseArgs } from "node:util";
 import {
   custodianBootstrap,
   custodianRef,
@@ -75,18 +79,12 @@ import {
   imageContextDigest,
   isSshUrl,
   loadConfig,
-  loadCharts,
-  loadWorkflows,
-  chartFileRefusals,
-  machineBodies,
+  doorCompat,
   matchCredential,
-  openPartRefusals,
   packAssembly,
   configMapDigest,
   MOUNTED_CONFIGMAPS,
   partsOf,
-  placementWalkOf,
-  refuseEffectSecrets,
   LABEL_EFFECTOR_POD,
   LABEL_GIT_PUSH,
   LABEL_GIT_CLONE,
@@ -102,19 +100,19 @@ import {
   harnessTokenDigest,
   formatCpu,
   formatMemory,
-  resolveSize,
-  sandboxToken,
-  seatLine,
-  serializeMachine,
-  splitSize,
+  type CarriedEffect,
   type CarriedImage,
   type CarriedRepo,
   type HeldManifest,
   type HeldPodPlan,
   type ImageRefs,
+  MODEL_KEYS,
+  PROVIDER_KEY_NAME,
+  type PlacementWalk,
+  type SandboxPlacement,
+  workspaceHeldPod,
   type JR2Config,
-  type ResolvedSize,
-} from "@jr2/orchestrator";
+} from "@jr2/orchestrator/internal";
 import { stageAssembly, stagedCopyOf } from "../assembly.ts";
 import { caFits, fingerprint, issueCa, issueLeaf, leafFits, trustBundles, type Pem } from "../held-pki.ts";
 import {
@@ -122,6 +120,7 @@ import {
   buildSandboxImage,
   choosePlatforms,
   detectKitCheckout,
+  isMissingHostImage,
   kitImageBuild,
   kitImageRefs,
   normalizeRef,
@@ -130,6 +129,7 @@ import {
   sandboxImageTag,
   KIT_IMAGE_HOME,
   type BuildPort,
+  type ImageFiles,
   type KitImageName,
   type KitImageRefs,
 } from "../build.ts";
@@ -141,6 +141,8 @@ import {
   INSTANCE_HARNESS_PVC,
   INSTANCE_HARNESS_SERVICE,
   instanceHarnessObjects,
+  INSTANCE_HARNESS_CREDENTIALS,
+  podCredentials,
   instanceEffectorObjects,
   effectorDigest,
   storedValueDigest,
@@ -148,6 +150,7 @@ import {
   effectorGraceSeconds,
   INSTANCE_EFFECTOR,
   instanceObjects,
+  KIT_POD_SIZES,
   KIT_VERSION,
   LABEL_INSTANCE,
   LABEL_VERSION,
@@ -164,8 +167,15 @@ import {
   type HeldObjects,
   type CustodianFiles,
   type HeldPodObjects,
+  checkPodObjects,
+  LABEL_PROVIDER_PREFLIGHT,
+  checkPodObjectNames,
+  PROVIDER_PREFLIGHT,
+  CHECK_CONTAINER,
 } from "../deploy.ts";
+import { syncAgentsMd } from "../agents-md.ts";
 import { resolveRoot } from "../instance.ts";
+import { resolvePackage } from "../kit-version.ts";
 import { effectorImagePaths, effectorImageRefusal, STOCK_HARNESS_IMAGE, type PushForm } from "../effector-image.ts";
 import {
   kubeFloorRefusal,
@@ -177,27 +187,30 @@ import {
   ORCHESTRATOR_SERVICE,
   type KubeAdmin,
   type KubeObject,
+  type PodEnd,
   type RolloutTarget,
 } from "../kube.ts";
+import { awaitCacheAgent } from "../landlock.ts";
 import { anyNodeHolds, buildNodes, sandboxNodes, type NodeObject } from "../nodes.ts";
 import { activity, chooseOrBail, confirmOrBail, promptLine, readSecretInput, type Io } from "../output.ts";
 import { kindCluster, sweepImages } from "../sweep.ts";
-import { tscTypecheck } from "../typecheck.ts";
+import { chartFilesIn, checkInstance } from "../instance-check.ts";
+import { parseCommand } from "../command-table.ts";
+
+/** How many times one converge rebuilds a host image a sweep took before it uses it (ADR-0039).
+ * A bound, because a tag that is gone again after two rebuilds is not a race to wait out: something
+ * on the host removes it as fast as it is built, and docker's own error is the honest report. */
+const HOST_REBUILDS = 2;
 
 export async function up(args: string[], io: Io): Promise<number> {
-  const { values } = parseArgs({
-    args,
-    allowPositionals: true,
-    strict: false,
-    options: {
-      yes: { type: "boolean" },
-      force: { type: "boolean" },
-      namespace: { type: "string", short: "n" },
-      context: { type: "string" },
-    },
-  });
+  const { values } = parseCommand("up", args);
 
   const root = resolveRoot(io.cwd);
+  // --- the AGENTS.md block (ADR-0072): before anything can refuse, because it describes the Kit the
+  // Instance resolves, not the cluster, so a converge that fails still leaves the index current.
+  const kit = resolvePackage("@jr2/orchestrator", root);
+  if (kit) await syncAgentsMd(io, root, kit.root, kit.version);
+
   const config = (await loadConfig(root)) ?? {};
   const name = config.name ?? basename(root);
   const namespace = (values.namespace as string | undefined) ?? name;
@@ -210,181 +223,16 @@ export async function up(args: string[], io: Io): Promise<number> {
 
   activity(io, `jr2 up — instance "${name}" → context ${context} / namespace ${namespace}`);
 
-  // --- typecheck (ADR-0050): the compiler agrees the folder is coherent, before anything is spent -
+  // --- the Instance check (ADR-0050): the typecheck and the Machine walk, shared with `jr2 check` -
   // A REFUSAL, not a warning, and it comes before the namespace apply as well as before the builds:
-  // the names a Machine carries (an Agent slot, a composed Machine, a Repo Slot, a `customize()` of
-  // any of them) are typed since ADR-0049/0051, so the answer here is the same answer the author's
-  // editor gives — and a converge that shipped a Machine the compiler rejects would surface it as
-  // an invoke-time failure mid-run, minutes and three image builds later.
-  activity(io, "typecheck: tsc --noEmit (the instance's own compiler)");
-  const checked = await (io.typecheck ?? tscTypecheck)(root);
-  if (!checked.ok) {
-    activity(io, "refusing: this instance does not typecheck — nothing was built and nothing was applied");
-    for (const line of checked.output.split("\n")) activity(io, `  ${line}`);
-    return 1;
-  }
-
-  // --- the Machine walk (ADR-0049, ADR-0051) -----------------------------------------------------
-  // A Machine carries its parts, so this converge LOADS the instance's registered Workflows — the
-  // same discovery and module contract the Orchestrator boots with — and walks them for everything
-  // the later layers need: the Agent definitions whose models are preflighted (ADR-0018), any
-  // `workspace: "none"`, which converges the Instance Harness (ADR-0031), every `file:` docker
-  // context a `workspace()` names, which is built below (ADR-0037), every bound Repo, whose ssh
-  // key is asked for here and whose resource the Orchestrator creates at boot (ADR-0051), and
-  // whether any Machine composes a Sandbox at all — the data-plane switch. It runs right after the
-  // gate and before ownership because it holds the one REFUSAL the compiler cannot make: a part
-  // left Open on a registered Machine — a Repo Slot with no url, an Agent with no model
-  // (ADR-0054). Nothing about Agents is published, because the definition rides each Turn.
-  const workflows = await loadWorkflows(root);
-  const carried = partsOf(workflows.map((w) => w.machine));
+  // the names a Machine carries are typed since ADR-0049/0051, so the compiler's answer is the
+  // author's editor's answer; the walk then refuses what the compiler cannot see (an Open part, a
+  // Chart file, a dispatch, an Effect secret, a Size). `jr2 check` runs the same function, so an
+  // Assistant that may not run `jr2 up` gets the answer of this converge.
+  const checked = await checkInstance(io, root, config);
+  if (!checked) return 1;
+  const { workflows, carried, placementWalk, sizes, chartFiles } = checked;
   const { agents, images: contexts } = carried;
-  // The Machine is named by where it sits under the registered root — the slot chain a
-  // `customize()` walks — and the fix line nests the same way, so it pastes into
-  // `workflows/<name>.ts` with the imported Machine and the url filled in.
-  // Every Open part is refused in ONE listing, in walk order within each kind: a composer fixing a
-  // packaged Machine fixes all of them in the same edit, and a converge that refused the Repo Slot,
-  // then the Agent one run later, would cost them two rounds to learn one thing. The listing is the
-  // door's own (ADR-0069): the Orchestrator refuses a publish with the same lines, so the host walk
-  // and the door's walk cannot drift.
-  const open = openPartRefusals(workflows);
-  if (open.length) {
-    for (const line of open) activity(io, `refusing: ${line}`);
-    return 1;
-  }
-  // Each Chart file is admitted HERE by the door's own check (ADR-0070), so a Chart file the
-  // Orchestrator would refuse is refused at the same command as `tsc`'s errors, before any
-  // cluster work, with the lines the publish would answer.
-  const chartRefusals = chartFileRefusals(
-    await loadCharts(root, (line) => activity(io, `warning: ${line}`)),
-    workflows,
-  );
-  if (chartRefusals.length) {
-    for (const line of chartRefusals) activity(io, `refusing: ${line}`);
-    return 1;
-  }
-  // What each registered Machine runs outside itself (ADR-0067, ADR-0068): every Effect slot, with
-  // the plain data it carries — a command's argv, its timeout, the Repo Slot it runs in; an http
-  // Effect's method, its fixed target and the secrets it names — read, never run.
-  for (const w of workflows) {
-    const effects = partsOf([w.machine]).effects;
-    if (!effects.length) continue;
-    activity(io, `effects: workflow "${w.name}" — what it runs outside itself (ADR-0067)`);
-    for (const e of effects) {
-      const where = e.path?.length ? ` (on ${e.path.map((k) => `"${k}"`).join(" → ")})` : "";
-      const slot = e.slot !== undefined ? `, in Repo Slot "${e.slot}"` : "";
-      if (e.kind === "http") {
-        const names = e.secrets.length ? `, secrets ${e.secrets.map((n) => `"${n}"`).join(", ")}` : "";
-        // Its body decides where it runs (ADR-0067): a Worktree file needs its Workspace's Sandbox Effector.
-        const where2 = e.file
-          ? `in the Sandbox Effector, a Worktree file as its body${slot}`
-          : "in the Instance Effector";
-        activity(
-          io,
-          `  ${e.name}${where}: http ${e.method} ${e.target ? targetText({ target: e.target }) : "<open>"} — ${e.timeoutMs / 1000}s, ` +
-            `${where2}${names}${e.idempotent ? ", idempotent" : ""}`,
-        );
-        continue;
-      }
-      if (e.kind === "push") {
-        const opts = (["pin", "sign", "squash", "force"] as const).filter((k) => e[k]);
-        activity(
-          io,
-          `  ${e.name}${where}: push${opts.length ? ` (${opts.join(", ")})` : ""} — ${e.timeoutMs / 1000}s, ` +
-            `in the Sandbox Effector${slot}`,
-        );
-        continue;
-      }
-      if (!("argv" in e)) continue;
-      const names = e.secrets?.length ? `, secrets ${e.secrets.map((n) => `"${n}"`).join(", ")}` : "";
-      const seat = e.secrets?.length ? `the Sandbox Effector${e.pin ? ", pinned" : ""}` : "the Harness";
-      activity(
-        io,
-        `  ${e.name}${where}: ${e.kind} ${JSON.stringify(e.argv)} — ${e.timeoutMs / 1000}s, in ${seat}${slot}${names}`,
-      );
-    }
-  }
-  // An Effect that names a secret the `secrets` list does not hold, and a literal Effect secret too
-  // short to scrub (ADR-0068), are refused HERE, off the walk and the config alone — before anything is
-  // built or applied (ADR-0050, ADR-0068). A secret read from the user's Secret is placed below,
-  // once the cluster is reached.
-  // What names each secret, read off the walk (ADR-0059, ADR-0068): every Effect, every Workspace's
-  // pod — its `workspace()`'s `secrets` and the models of the Agents whose Turns run there — and the
-  // models of the `"none"` Agents the Instance Harness hosts.
-  // An Open body that allows "none" needs the Instance Harness, holding a key for each model it
-  // allows (ADR-0070), as a "none" Agent does. The door's walk says both, so `jr2 up` and the
-  // publish door place the same set.
-  const placementWalk = placementWalkOf(workflows);
-  refuseEffectSecrets(config, placementWalk);
-  // Who may send what, per state (ADR-0015): the audience rule's answer for each registered
-  // Machine and every Machine it composes, as its doc carries it — so an author reads who holds a
-  // pick before a run finds out. Then what the build found legal and suspect: a warning, never a
-  // refusal (the refusals threw when the Machine loaded).
-  for (const w of workflows) {
-    const bodies = machineBodies(serializeMachine(w.name, w.machine));
-    const seats = bodies.flatMap((body) => body.seats.map(seatLine));
-    if (seats.length) {
-      activity(io, `seats: workflow "${w.name}" — who may send what, per state (ADR-0015)`);
-      for (const line of seats) activity(io, `  ${line}`);
-    }
-    for (const body of bodies) {
-      for (const warning of body.warnings) activity(io, `warning: workflow "${w.name}": ${warning}  (ADR-0015)`);
-    }
-  }
-  // The Size (ADR-0060): every `workspace()` a registered Machine composes, resolved down its chain
-  // (the Machine's, then `sandbox.resources`, then the kit's) and split inside the pod, exactly as
-  // the provision will. A split that leaves the Harness below its floor is refused HERE, naming the
-  // Machine and the number to change, rather than at the first provision of a run. The resolved
-  // Sizes are kept for the Sandbox-node warning below.
-  const sizes: Array<{ workflow: string; size: ResolvedSize }> = [];
-  for (const w of workflows) {
-    for (const { path, resources, user } of partsOf([w.machine]).sizes) {
-      try {
-        const size = resolveSize(resources, config.sandbox?.resources);
-        splitSize(size, user);
-        sizes.push({ workflow: w.name, size });
-      } catch (e) {
-        const where =
-          path !== undefined && path.length
-            ? ` (on the Machine composed as ${path.map((k) => `"${k}"`).join(" → ")})`
-            : "";
-        const fix =
-          path === undefined
-            ? ""
-            : `; a composer retunes it where the Machine is registered: export const machine = ${sizeLine(path)}`;
-        activity(io, `refusing: workflow "${w.name}"${where}: ${(e as Error).message}${fix}`);
-        return 1;
-      }
-    }
-  }
-  const agentNames = [...new Set(agents.map((a) => a.name))].join(", ") || "(none)";
-  activity(io, `agents: ${agentNames} (carried by ${workflows.length} workflow machine(s))`);
-  // Each Chart wrapper (ADR-0070): what its Open body lends a Chart, how many models a Chart's own
-  // Agents may name (each gets its key placed below), and the Chart files the Assembly carries for
-  // it. The walk reads the Open body's declaration, so the keys and the Instance Harness a Chart
-  // could need are placed before any run.
-  const chartFiles = await chartFilesIn(root);
-  for (const w of workflows) {
-    const bodies = partsOf([w.machine]).openBodies.filter((b) => b.path !== undefined && b.path.length === 0);
-    if (!bodies.length) continue;
-    const lends = [
-      ...new Set(
-        bodies.flatMap(({ declaration: { lends } }) => [
-          ...(lends.agents ?? []),
-          ...(lends.effects ?? []),
-          ...(lends.machines ?? []),
-          ...(lends.guards ?? []),
-          ...(lends.actions ?? []),
-          ...(lends.delays ?? []),
-        ]),
-      ),
-    ];
-    const models = new Set(bodies.flatMap((b) => b.declaration.allows.models));
-    const files = chartFiles.filter((n) => n.startsWith(`${w.name}/`)).length;
-    activity(
-      io,
-      `chart wrapper "${w.name}": lends ${lends.join(", ") || "nothing"}; allows ${models.size} model(s); ${files} Chart file(s)`,
-    );
-  }
 
   // --- the Assembly (ADR-0069): staged here, published last ---------------------------------------
   // Staged BEFORE any cluster spend, so its refusals cost nothing: a second copy of a Kit package
@@ -686,6 +534,26 @@ export async function up(args: string[], io: Io): Promise<number> {
   const hostBuilt = async (ref: string): Promise<boolean> =>
     values.force !== true && (await hostHeld()).has(normalizeRef(ref));
 
+  // What this converge counts on finding on the host — its own build, or the host-held skip above —
+  // can be gone by the time it is used: another converge's sweep keeps only the roots of ITS context
+  // (ADR-0039), and a ref whose map is not written yet is in none of them. The tag is a content
+  // address and BuildKit's cache is a store no sweep touches, so a ref found missing at its use is
+  // built again (seconds) and the use retried, at most HOST_REBUILDS times. Only a singleton build
+  // lives on the daemon: a multi-platform one is in the registry (ADR-0045), where no sweep reaches.
+  const builders = new Map<string, () => Promise<void>>();
+  const onHost = async <T>(ref: string, use: () => Promise<T>): Promise<T> => {
+    for (let rebuilt = 0; ; rebuilt++) {
+      try {
+        return await use();
+      } catch (err) {
+        const rebuild = builders.get(ref);
+        if (!rebuild || platforms.length > 1 || rebuilt >= HOST_REBUILDS || !isMissingHostImage(err)) throw err;
+        activity(io, `  ${ref}: gone from the host daemon (another converge's sweep took it) — rebuilding`);
+        await rebuild();
+      }
+    }
+  };
+
   // The binfmt preflight (ADR-0045), paid at most once and only when a build is really about to be
   // spent: a steady-state converge must still spend no docker at all, and a foreign-arch build must
   // fail BEFORE minutes of docker rather than mid-`RUN` with `exec format error`.
@@ -706,15 +574,19 @@ export async function up(args: string[], io: Io): Promise<number> {
       return ref;
     }
     assertDeliverable();
+    const buildRef = async (): Promise<void> => {
+      await ensureEmulation();
+      await build.build(kitImageBuild(kitRoot, which, ref, platforms));
+    };
+    builders.set(ref, buildRef);
     if (await hostBuilt(ref)) {
       activity(io, `${which} image: ${ref} (host-built — build skipped, delivering; --force to rebuild)`);
-      await deliver(ref);
+      await onHost(ref, () => deliver(ref));
       return ref;
     }
     activity(io, `${which} image: building ${ref}${values.force === true ? " (--force)" : ""}`);
-    await ensureEmulation();
-    await build.build(kitImageBuild(kitRoot, which, ref, platforms));
-    await deliver(ref);
+    await buildRef();
+    await onHost(ref, () => deliver(ref));
     return ref;
   };
 
@@ -736,55 +608,8 @@ export async function up(args: string[], io: Io): Promise<number> {
   const operatorImage =
     config.operator?.manage === false && !carried.composesSandbox ? undefined : await ensureKitImage("operator");
   if (operatorImage !== undefined) converged.operator = operatorImage;
-  if (config.operator?.manage === false) {
-    activity(
-      io,
-      "operator: skipped (operator.manage: false — run the controller loop yourself)" +
-        (operatorImage !== undefined ? `; its image ${operatorImage} still runs as the cache agent` : ""),
-    );
-  } else {
-    const image = operatorImage!;
-    const existing = await kube.getJson({
-      kind: "deployment",
-      name: OPERATOR_DEPLOYMENT,
-      namespace: OPERATOR_NAMESPACE,
-      ...ctx,
-    });
-    const deployed = existing?.metadata.labels?.[LABEL_VERSION];
-    if (deployed && compareVersions(deployed, KIT_VERSION) > 0) {
-      activity(io, `operator: leaving v${deployed} (newer than this kit's v${KIT_VERSION} — never downgraded)`);
-    } else {
-      activity(io, `operator: applying v${KIT_VERSION} (${image})${deployed ? ` over v${deployed}` : ""}`);
-      await kube.apply({
-        manifest: await operatorManifest(image, priorityClassNames(config.priorityClasses).control),
-        ...ctx,
-      });
-      await kube.label({
-        kind: "deployment",
-        name: OPERATOR_DEPLOYMENT,
-        namespace: OPERATOR_NAMESPACE,
-        labels: { [LABEL_VERSION]: KIT_VERSION },
-        ...ctx,
-      });
-      await awaitRollout(kube, {
-        name: OPERATOR_DEPLOYMENT,
-        namespace: OPERATOR_NAMESPACE,
-        selector: OPERATOR_SELECTOR,
-        ...ctx,
-      });
-      // Sound for every layer now that EVERY tag is a content address (ADR-0038): a source edit
-      // moves the ref, which moves the pod template, which rolls. The old hole — kit dev pinning a
-      // static `:local` tag, so a rebuild left the template identical and nothing rolled — closed
-      // with the `images` block that created it.
-      await verifyRunningImage(io, kube, {
-        layer: "operator",
-        namespace: OPERATOR_NAMESPACE,
-        selector: OPERATOR_SELECTOR,
-        image,
-        ...ctx,
-      });
-    }
-  }
+  // Applied after the provider preflight (below): the operator is the cluster's, and a converge
+  // whose provider check fails must not have changed it.
 
   // --- the kit's own runtime images (ADR-0038) ---------------------------------------------------
   // Built here rather than lazily beside its consumers. The Harness is the injection source
@@ -818,6 +643,16 @@ export async function up(args: string[], io: Io): Promise<number> {
       const digest = contextDigests.get(image.dir)!;
       const key = image.reserved ?? digest;
       const ref = sandboxImageTag(name, image.name, digest, { platforms, registry });
+      const buildRef = async (): Promise<void> => {
+        // ONE build of the user's own Dockerfile, straight to its content tag — no wrap, no
+        // intermediate tag, so two converges of one checkout share no mutable image name and
+        // the `@kind` tier's `--parallel` rides on it (ADR-0037).
+        await ensureEmulation();
+        await buildSandboxImage(build, { dir: image.dir, tag: ref, instance: name, platforms });
+      };
+      // Known on every path, the record's skip included: the Effector check reads this ref off the
+      // host daemon later in this converge, whatever decided its build.
+      builders.set(ref, buildRef);
       // A record is only worth skipping on when it is COMPLETE: the ref AND the seat the image
       // declared. A record from a kit that predates `sandboxUser` names the right image and
       // silently drops the fallback fact, which surfaces as a pod that will not start — so it reads
@@ -838,11 +673,7 @@ export async function up(args: string[], io: Io): Promise<number> {
           activity(io, `sandbox image "${image.name}": ${ref} (host-built — build skipped; --force to rebuild)`);
         } else {
           activity(io, `sandbox image "${image.name}": building ${ref}`);
-          // ONE build of the user's own Dockerfile, straight to its content tag — no wrap, no
-          // intermediate tag, so two converges of one checkout share no mutable image name and
-          // the `@kind` tier's `--parallel` rides on it (ADR-0037).
-          await ensureEmulation();
-          await buildSandboxImage(build, { dir: image.dir, tag: ref, instance: name, platforms });
+          await buildRef();
         }
         // The image's own `USER`, read while the image is certainly on this daemon (it was just
         // built, or the host holds it). Recorded because a provision cannot inspect an image, and
@@ -852,10 +683,10 @@ export async function up(args: string[], io: Io): Promise<number> {
         // (ADR-0005), which owes no floor — a distinction only the provision can make (ADR-0031).
         // `platforms` says where to look, not which variant to read (ADR-0045): a singleton build is
         // on this daemon, a multi-platform one is the manifest list buildx just pushed.
-        const user = await build.imageUser(ref, platforms);
+        const user = await onHost(ref, () => build.imageUser(ref, platforms));
         converged.sandboxUser[key] = user;
         activity(io, `  ${user ? `USER ${user}` : "no USER declared — the pod's uid-1000 fallback applies"}`);
-        await deliver(ref);
+        await onHost(ref, () => deliver(ref));
       }
       converged.sandbox[key] = ref;
     }
@@ -885,12 +716,10 @@ export async function up(args: string[], io: Io): Promise<number> {
     JR2_INSTANCE_TOKEN: instanceToken,
     JR2_PUBLISH_TOKEN: publishToken,
     JR2_SIGNING_KEY: signingKey,
-    // The Instance Harness Custodian's credential (ADR-0013/0031): a sandbox-style token signed
-    // for the placement's name — it may deliver only to Turns hosted THERE (tokens.ts), so the
-    // Instance token never enters that pod. Derived from the kept key, so re-runs converge to
-    // the same value; live Custodians keep verifying.
-    JR2_INSTANCE_HARNESS_TOKEN: sandboxToken(Buffer.from(signingKey, "base64"), INSTANCE_HARNESS_SERVICE),
   };
+  // The Instance Harness Custodian's credentials are not here (ADR-0076): its control token and
+  // its secret Stand-ins ride its own Secret (`INSTANCE_HARNESS_CREDENTIALS`), whose Stand-ins its
+  // Harness container mounts — so it must never be this one.
   // Git tokens (ADR-0051, ADR-0068): each entry's token, from `.env`, goes into that entry's own clone
   // Secret (`jr2-git-<hash>`, Flux's key names), which the cache agent reads — never into this
   // Secret, the Orchestrator's env. The Orchestrator names the Secret on a Repo and holds no git token.
@@ -985,6 +814,89 @@ export async function up(args: string[], io: Io): Promise<number> {
     caBundle: caPem,
   });
 
+  // --- provider preflight (ADR-0019, ADR-0059): through a Custodian, FROM INSIDE the cluster -----
+  // BEFORE anything the Instance or the cluster reads is written — the operator, a deploy key, the
+  // trust, the held objects, the instance List: each check pod has its own copies of what it reads,
+  // from the values on this host. So a check that fails leaves the Instance, its Assembly and the
+  // operator as they were. Only the namespace, the PriorityClasses and the delivered images come
+  // first, because the check pod needs them.
+  const checkPods = checkPodsOf(config, placementWalk, held.objects.pods);
+  const custom = config.harness?.provider;
+  if (custom && !checkPods.some((p) => p.providers.some((v) => v.id === custom.id))) {
+    activity(
+      io,
+      `provider: ${custom.baseUrl} configured, but no carried Agent names a "${custom.id}/…" model — skipped`,
+    );
+  }
+  await preflightProviders(io, kube, {
+    checkPods,
+    skip: values["skip-provider-preflight"] === true,
+    name,
+    namespace,
+    ctx,
+    harnessImage: refs.harness,
+    custodianImage,
+    harness: config.harness,
+    caBundle: caPem !== undefined,
+    values: Object.assign({}, ...held.objects.pods.map((p) => p.values)) as Record<string, string>,
+    leaves: held.objects.leaves,
+    trust: trustData(caPem, held.bundles),
+    harnessEnvData,
+    priorityClassName: priorityClassNames(config.priorityClasses).sandbox,
+    signingKey: Buffer.from(signingKey, "base64"),
+  });
+
+  // --- operator (per-cluster, shared): after the provider preflight ------------------------------
+  if (config.operator?.manage === false) {
+    activity(
+      io,
+      "operator: skipped (operator.manage: false — run the controller loop yourself)" +
+        (operatorImage !== undefined ? `; its image ${operatorImage} still runs as the cache agent` : ""),
+    );
+  } else {
+    const image = operatorImage!;
+    const existing = await kube.getJson({
+      kind: "deployment",
+      name: OPERATOR_DEPLOYMENT,
+      namespace: OPERATOR_NAMESPACE,
+      ...ctx,
+    });
+    const deployed = existing?.metadata.labels?.[LABEL_VERSION];
+    if (deployed && compareVersions(deployed, KIT_VERSION) > 0) {
+      activity(io, `operator: leaving v${deployed} (newer than this kit's v${KIT_VERSION} — never downgraded)`);
+    } else {
+      activity(io, `operator: applying v${KIT_VERSION} (${image})${deployed ? ` over v${deployed}` : ""}`);
+      await kube.apply({
+        manifest: await operatorManifest(image, priorityClassNames(config.priorityClasses).control),
+        ...ctx,
+      });
+      await kube.label({
+        kind: "deployment",
+        name: OPERATOR_DEPLOYMENT,
+        namespace: OPERATOR_NAMESPACE,
+        labels: { [LABEL_VERSION]: KIT_VERSION },
+        ...ctx,
+      });
+      await awaitRollout(kube, {
+        name: OPERATOR_DEPLOYMENT,
+        namespace: OPERATOR_NAMESPACE,
+        selector: OPERATOR_SELECTOR,
+        ...ctx,
+      });
+      // Sound for every layer now that EVERY tag is a content address (ADR-0038): a source edit
+      // moves the ref, which moves the pod template, which rolls. The old hole — kit dev pinning a
+      // static `:local` tag, so a rebuild left the template identical and nothing rolled — closed
+      // with the `images` block that created it.
+      await verifyRunningImage(io, kube, {
+        layer: "operator",
+        namespace: OPERATOR_NAMESPACE,
+        selector: OPERATOR_SELECTOR,
+        image,
+        ...ctx,
+      });
+    }
+  }
+
   // --- git over ssh: where the key comes from (ADR-0019, ADR-0047) -------------------------------
   // Runs BEFORE the apply, because a missing key source is a converge that must not start, and the
   // notice it hands back is owed to the END of the converge — a generated key is dead until a human
@@ -994,22 +906,27 @@ export async function up(args: string[], io: Io): Promise<number> {
   // --- what each Sandbox Effector's image owes its pushes (ADR-0068) ------------------------
   // Before anything is applied: an image that lacks a push form's tools, or the passwd entry ssh
   // needs, is refused here with the fix, not at the first push in a pod.
-  await checkEffectorImages(build, config, placements.sandbox, converged, platforms, contextDigests);
+  await checkEffectorImages(
+    (ref, paths) => onHost(ref, () => build.imageFiles(ref, platforms, paths)),
+    config,
+    placements.sandbox,
+    converged,
+    contextDigests,
+  );
 
   // --- push credentials (ADR-0051): one Secret per git.credentials entry, never per pod ----
   // After the deploy keys exist (a generated one included), before anything else is applied: a
   // static Binding whose entry cannot push is refused here.
   const gitPush = await resolveGitPush(io, kube, config, placements.sandbox, namespace, ctx);
 
-  // --- provider preflight (ADR-0019): probe the endpoint FROM INSIDE the cluster ----------------
-  await preflightProvider(io, kube, config, carried.models, namespace, ctx, caPem);
+  // --- dispatch preflight (ADR-0071): each foreign target's door, from the host ------------------
+  await preflightDispatch(io, carried.effects);
 
   // --- apply + rollout ---------------------------------------------------------------------------
   // The trust bundles first, and server-side: every pod that mounts them must find them, and they
   // carry every root certificate — past what client-side apply's annotation can hold.
   const trust = trustObject({ name, namespace, caBundle: caPem, bundles: held.bundles });
   if (trust) await kube.apply({ manifest: trust, serverSide: true, ...ctx });
-  activity(io, `orchestrator: applying (image ${refs.orchestrator})`);
   const instanceList = instanceObjects({
     name,
     namespace,
@@ -1025,7 +942,8 @@ export async function up(args: string[], io: Io): Promise<number> {
     gitPush,
     gitClone,
   });
-  await kube.apply({ manifest: instanceList, ...ctx });
+  activity(io, `orchestrator: applying (image ${refs.orchestrator})`);
+  await kube.apply({ manifest: instanceList, warn: (line) => activity(io, line), ...ctx });
   // An entry whose clone Secret this converge did not write loses it: a dropped entry, a token
   // unset, or nothing left that clones.
   const cloneSecrets = new Set(gitClone.map((s) => s.name));
@@ -1074,6 +992,7 @@ export async function up(args: string[], io: Io): Promise<number> {
   if (!holdsAny(held.objects)) {
     await kube.deleteObject({ kind: "secret", name: HELD_TLS_SECRET, namespace, ...ctx });
   }
+
   activity(io, "orchestrator: waiting for rollout");
   await awaitRollout(kube, {
     name: ORCHESTRATOR_SERVICE,
@@ -1097,14 +1016,17 @@ export async function up(args: string[], io: Io): Promise<number> {
   // over from a Machine that dropped its `workspace()` is deleted with its RBAC: the layer
   // converges toward the Machines like the Instance Harness does.
   if (carried.composesSandbox) {
+    // The agent is also the Landlock detector (ADR-0077): on a Sandbox node without Landlock it
+    // exits at start with the reason, and the converge refuses, naming each such node.
     activity(io, "repo cache: waiting for the cache agent on every Sandbox node");
-    await awaitRollout(kube, {
+    const cacheTarget: RolloutTarget = {
       kind: "daemonset",
       name: REPO_CACHE,
       namespace,
       selector: `app=${REPO_CACHE}`,
       ...ctx,
-    });
+    };
+    await awaitCacheAgent(kube, cacheTarget, () => awaitRollout(kube, cacheTarget));
     await verifyRunningImage(io, kube, {
       layer: "repo cache",
       namespace,
@@ -1160,6 +1082,13 @@ export async function up(args: string[], io: Io): Promise<number> {
         held: instance.manifest,
         custodian: instance.custodian,
         heldDigest: heldDigest(Buffer.from(signingKey, "base64"), held.objects, instance, valueDigest),
+        // Its control token and secret Stand-ins (ADR-0076, ADR-0059): minted here, as the pod's
+        // maker, from the kept key — so a re-run converges to the same values.
+        credentials: podCredentials(Buffer.from(signingKey, "base64"), {
+          placement: INSTANCE_HARNESS_SERVICE,
+          heldPod: INSTANCE_HARNESS_HELD_POD,
+          manifest: instance.manifest,
+        }),
         harness: config.harness,
         caBundle: caPem !== undefined,
         // The wire's gate (ADR-0058): the digest of the bearer the Orchestrator derives for this
@@ -1194,8 +1123,10 @@ export async function up(args: string[], io: Io): Promise<number> {
     await kube.deleteObject({ kind: "persistentvolumeclaim", name: INSTANCE_HARNESS_PVC, namespace, ...ctx });
     await kube.deleteObject({ kind: "service", name: INSTANCE_HARNESS_SERVICE, namespace, ...ctx });
     await kube.deleteObject({ kind: "service", name: INSTANCE_HARNESS_PODS_SERVICE, namespace, ...ctx });
-    // Its Custodian's files go with the pod they were rendered for (ADR-0059).
+    // Its Custodian's files and its credentials go with the pod they were made for (ADR-0059,
+    // ADR-0076).
     await kube.deleteObject({ kind: "configmap", name: INSTANCE_CUSTODIAN_CONFIGMAP, namespace, ...ctx });
+    await kube.deleteObject({ kind: "secret", name: INSTANCE_HARNESS_CREDENTIALS, namespace, ...ctx });
   }
 
   // --- Instance Effector (ADR-0068): converged when an Effect is placed there, absent otherwise ---
@@ -1223,6 +1154,7 @@ export async function up(args: string[], io: Io): Promise<number> {
         caBundle: caPem !== undefined,
         ...(config.priorityClasses ? { priorityClasses: config.priorityClasses } : {}),
       }),
+      warn: (line) => activity(io, line),
       ...ctx,
     });
     await awaitRollout(kube, {
@@ -1561,6 +1493,8 @@ async function resolveHeld(
     models: opts.models,
   });
   for (const warning of warnings) activity(io, `warning: ${warning}`);
+  const r14 = envFromHeldRefusal(config, manifest, opts.envFromKeys);
+  if (r14) throw new Error(r14);
   // Each pod's own: the bindings of the names it holds and their literal values. Its Custodian's
   // files are rendered by whoever makes the pod — here for the Instance Harness alone; the
   // Orchestrator at provision for a Sandbox, from the same manifest (ADR-0059).
@@ -1577,7 +1511,7 @@ async function resolveHeld(
   const orchestrator = { host: `${ORCHESTRATOR_SERVICE}.${namespace}.svc`, port: ORCHESTRATOR_PORT };
   const instance: CustodianFiles | undefined = instancePod && {
     bootstrap: JSON.stringify(custodianBootstrap(instancePod.manifest, orchestrator)),
-    script: custodianScript(instancePod.manifest),
+    script: custodianScript(instancePod.manifest, { fetch: false }),
   };
   if (manifest.secrets.length === 0) return { objects: { pods, ...(instance && { instance }), leaves: {} } };
   for (const s of manifest.secrets) {
@@ -1612,6 +1546,36 @@ async function resolveHeld(
   };
 }
 
+/**
+ * R14 (ADR-0059): a held secret's name that is also a key of a Secret `harness.envFrom` loads. The
+ * Stand-ins are files, not env (ADR-0076), so no Stand-in hides that key: the Secret's real value
+ * would reach the Harness container's env under the held secret's own name, and the Harness uses a
+ * held secret but never holds it. The key is compared as the kubelet writes it, `prefix` first.
+ * The message names the Secret, the key and the held secret, and never a value. Exported for its
+ * test.
+ */
+export function envFromHeldRefusal(
+  config: JR2Config,
+  manifest: HeldManifest,
+  envFromKeys: ReadonlyMap<string, readonly string[]>,
+): string | undefined {
+  const held = new Set(manifest.secrets.map((s) => s.name));
+  for (const [i, ref] of (config.harness?.envFrom ?? []).entries()) {
+    const secret = ref.secretRef?.name;
+    if (secret === undefined) continue;
+    for (const key of envFromKeys.get(`secret/${secret}`) ?? []) {
+      const name = (ref.prefix ?? "") + key;
+      if (!held.has(name)) continue;
+      return (
+        `harness.envFrom[${i}] loads Secret "${secret}", whose key "${key}" is the env var ${name} — and ${name} is ` +
+        `a held secret. Its Stand-in is a file, so nothing hides that key, and its value would reach the ` +
+        `Harness's env. Drop the key from Secret "${secret}", or rename the held secret (ADR-0059, R14)`
+      );
+    }
+  }
+  return undefined;
+}
+
 /** The Instance Harness's held objects and its Custodian's files — the walk always places it when
  * it converges. */
 function instanceHeld(held: HeldObjects): HeldPodObjects & { custodian: CustodianFiles } {
@@ -1815,6 +1779,8 @@ async function resolveGitPush(
     const data: Record<string, string> = {};
     let sshMissing: string | undefined;
     let signMissing: string | undefined;
+    // One warning per entry and cause: an absent Secret is missing every key it was to give.
+    const warned = new Set<string>();
     for (const source of plan.sources) {
       if ("literal" in source) {
         data[source.key] = Buffer.from(source.literal).toString("base64");
@@ -1833,6 +1799,8 @@ async function resolveGitPush(
           : `the Secret "${source.from.secret}" holds no key ${source.from.key}`;
       if (source.key.startsWith("sign.")) signMissing ??= why;
       else sshMissing ??= why;
+      if (warned.has(why)) continue;
+      warned.add(why);
       // Not a refusal: the Secret may be written later (a scenario writes it, a human creates it).
       // A push under the entry that needs it is refused at provision until `jr2 up` runs again.
       activity(
@@ -1896,11 +1864,11 @@ async function resolveGitPush(
  * is not read, as its `USER` is not.
  */
 async function checkEffectorImages(
-  build: BuildPort,
+  /** {@link BuildPort.imageFiles} for a ref this converge resolved — rebuilt if a sweep took it. */
+  imageFiles: (ref: string, paths: readonly string[]) => Promise<ImageFiles>,
   config: JR2Config,
   sandbox: readonly SandboxEffectorPlan[],
   converged: ConvergedImages,
-  platforms: readonly string[],
   /** Each `file:` context's key in the map: the digest of its staged copy (ADR-0069). */
   contextDigests: ReadonlyMap<string, string>,
 ): Promise<void> {
@@ -1940,7 +1908,7 @@ async function checkEffectorImages(
       if (refusal) throw new Error(refusal);
       continue;
     }
-    const files = await build.imageFiles(ref, platforms, effectorImagePaths(forms.map((f) => f.form)));
+    const files = await imageFiles(ref, effectorImagePaths(forms.map((f) => f.form)));
     const refusal = effectorImageRefusal({
       label: plan.label,
       ref,
@@ -2022,7 +1990,7 @@ async function ensureGitSsh(
 
     if (source.kind === "generate") {
       const { privateKey, publicKey } = await (io.sshKeygen ?? sshKeygen)();
-      await applyGitSshSecret(kube, namespace, ctx, secret, privateKey, publicKey);
+      await applyGitSshSecret(io, kube, namespace, ctx, secret, privateKey, publicKey);
       activity(io, `generated deploy key ${sshFingerprint(publicKey)} — register this PUBLIC key (read access):`);
       activity(io, `  ${publicKey.trim()}`);
       // The pause sits exactly where the user must act anyway (ADR-0047): the printed key is useless
@@ -2053,7 +2021,7 @@ async function ensureGitSsh(
     } catch (err) {
       throw new Error(`refusing the key from ${where}: ${err instanceof Error ? err.message : err}`);
     }
-    await applyGitSshSecret(kube, namespace, ctx, secret, privateKey, publicKey);
+    await applyGitSshSecret(io, kube, namespace, ctx, secret, privateKey, publicKey);
     // The FINGERPRINT, never the material — not the private half, and not the public one either:
     // this key is the user's own, and its public half identifies them wherever it is registered.
     activity(io, `git ssh: "${secret}" applied from ${where} — ${sshFingerprint(publicKey)}`);
@@ -2151,6 +2119,7 @@ async function readSuppliedKey(path: string): Promise<string> {
  * `identity.pub` rides along so the cluster can say which key it holds without holding it up to a
  * human. Trailing newline enforced — ssh refuses a key file that lacks one. */
 async function applyGitSshSecret(
+  io: Io,
   kube: KubeAdmin,
   namespace: string,
   ctx: { context?: string },
@@ -2166,6 +2135,7 @@ async function applyGitSshSecret(
       type: "Opaque",
       stringData: { identity: newlineTerminated(privateKey), "identity.pub": newlineTerminated(publicKey) },
     }),
+    warn: (line) => activity(io, line),
     ...ctx,
   });
 }
@@ -2272,92 +2242,671 @@ async function readCaBundle(root: string, config: JR2Config): Promise<string | u
 }
 
 /**
- * The custom-provider preflight (ADR-0019): from a pod, list models AND demand one trivial
- * tool-call completion — so an unreachable endpoint (localhost never works from a pod) or a vLLM
- * missing `--enable-auto-tool-choice` fails HERE, at converge time, not as an `agent.fault`
- * mid-run. Reachability itself stays the user's concern; this makes the answer immediate.
+ * The foreign dispatch preflight (ADR-0071): for each dispatch whose target is another
+ * Orchestrator, `GET <origin>/workflows/<name>`, and the served door compared with the slot's
+ * input schema (`doorCompat`, one direction). The route is open band (ADR-0014), so no token is
+ * sent: the token is the Effector's to hold, and the host would otherwise carry it over a wire the
+ * target may not secure. Every failure — no answer, a miss, a mismatch, a schema not compared — is a
+ * WARNING, never a refusal: the other Instance may be down or publish later, and refusing here
+ * would couple two Instances' publish order. A run of the dispatch answers what the door then says,
+ * the token's refusal included.
  */
-async function preflightProvider(
+async function preflightDispatch(io: Io, effects: readonly CarriedEffect[]): Promise<void> {
+  const fetchImpl = io.fetch ?? globalThis.fetch;
+  const seen = new Set<string>();
+  for (const e of effects) {
+    if (e.kind !== "dispatch" || e.target === undefined) continue;
+    const url = targetText({ target: e.target });
+    const once = JSON.stringify([url, e.workflow, e.secrets, e.input ?? null]);
+    if (seen.has(once)) continue;
+    seen.add(once);
+    const warn = (why: string) => activity(io, `warning: dispatch "${e.workflow}" at ${url}: ${why} (ADR-0071)`);
+    let res: Response;
+    try {
+      // No redirect is followed: the Effector's dispatch client follows none, so a door that
+      // redirects is one it never reaches.
+      res = await fetchImpl(`${url}/workflows/${encodeURIComponent(e.workflow)}`, {
+        headers: { accept: "application/json" },
+        redirect: "manual",
+        signal: AbortSignal.timeout(10_000),
+      });
+    } catch (err) {
+      warn(
+        `no answer from this host (${(err as Error).message}) — the other Instance may be down, or reached only ` +
+          'from inside its cluster; while no door answers, the dispatch settles "unreachable"',
+      );
+      continue;
+    }
+    if ((res.status >= 300 && res.status < 400) || res.type === "opaqueredirect") {
+      warn(
+        `the door redirected (HTTP ${res.status}); the Effector follows no redirect, so a run of the dispatch is refused there`,
+      );
+      continue;
+    }
+    if (res.status === 404) {
+      warn(`no Workflow "${e.workflow}" there — a run of the dispatch is refused with the door's 404 until it has one`);
+      continue;
+    }
+    if (!res.ok) {
+      warn(`the door answered HTTP ${res.status}`);
+      continue;
+    }
+    if (e.input === undefined) {
+      activity(io, `dispatch "${e.workflow}" at ${url}: the door answers; the slot states no input schema to compare`);
+      continue;
+    }
+    // A Workflow detail carries its door as `input` (null: it declares none). Any other answer — a
+    // page an ingress serves, another shape at another Kit version — is no door: not compared,
+    // never compared as `{}`, which would call every slot field stripped.
+    const detail: unknown = await res.json().catch(() => undefined);
+    if (typeof detail !== "object" || detail === null || Array.isArray(detail) || !Object.hasOwn(detail, "input")) {
+      warn("the door's answer is not a Workflow detail (no `input`); not compared");
+      continue;
+    }
+    const compared = doorCompat(e.input, (detail as { input: unknown }).input);
+    if (compared.ok === true) {
+      activity(io, `dispatch "${e.workflow}" at ${url}: the door takes the slot's input (ADR-0071)`);
+    } else if (compared.ok === "not-compared") {
+      warn(`its input schema and the door are not compared — ${compared.why}`);
+    } else {
+      for (const fault of compared.faults) warn(`the door does not take the slot's input: ${fault}`);
+    }
+  }
+}
+
+/** One provider of one check pod (ADR-0019, ADR-0059). */
+export type CheckedProvider = {
+  /** The provider id, the specifier's part before the first `/`. */
+  id: string;
+  /** Every carried model of it that a Harness pod of this placement runs: a typo, a retired model
+   * or a model the key has no access to is per model. */
+  models: string[];
+  /** The held secrets that are its keys in the check pod's Custodian; none when no pod of this
+   * placement holds one, and the check then shows what a Turn there meets. */
+  keys: string[];
+  /** Where the calls go: the keys' bound hosts, else the provider's host as jr2 knows it. */
+  hosts: string[];
+  /** `harness.catalog.<id>.baseUrl`, when it moves the provider. */
+  gateway?: string;
+  /** The custom provider (`harness.provider`, ADR-0018): its key is optional, and its causes are
+   * its own. */
+  custom?: true;
+};
+
+/** One check pod: the providers that the Harness pods of one placement call. */
+export type CheckPod = {
+  /** For a human: `the Sandbox nodes`, `the Instance Harness`, or both. */
+  where: string;
+  /** The node predicate of those pods (ADR-0052); none for the Instance Harness. */
+  placement?: SandboxPlacement;
+  /** The bindings of the checked keys: each a part of a held pod's manifest. */
+  manifest: HeldManifest;
+  providers: CheckedProvider[];
+};
+
+/** One Harness pod as the walk places it: its held pod key, its class, the models it runs, and
+ * the bindings it holds (none: its HTTPS goes direct, with no proxy). */
+type Seat = { pod: string; sandbox: boolean; models: readonly string[]; held: HeldManifest["secrets"] };
+
+/**
+ * Which providers the preflight checks, and in which pod (ADR-0019, ADR-0059). Every carried model
+ * that a Harness pod runs is checked, in a pod with that pod's placement: a Workspace's model on the
+ * Sandbox nodes, a `"none"` Agent's where the Instance Harness runs, so a node class whose egress is
+ * closed fails here. The two are one check pod when their placements are the same. The custom provider
+ * (`harness.provider`) goes the same way: its key is a held secret too (`JR2_PROVIDER_API_KEY`), so
+ * the key never enters a pod spec, an argv or a script.
+ *
+ * In one placement, a provider's keys are the ones the first pod there that runs one of its models
+ * holds (`MODEL_KEYS`, claimed by the provider). The check pod holds that pod's bindings of them and
+ * reads their values where that pod reads them. A provider no pod there holds a key for is checked
+ * all the same, with no key: a Turn there meets the same, and pi says so (`not configured`); a
+ * signing credential in `harness.env` reaches the check as it reaches a Turn. Pure, for its test.
+ */
+export function checkPodsOf(config: JR2Config, walk: PlacementWalk, pods: readonly HeldPodObjects[]): CheckPod[] {
+  const custom = config.harness?.provider;
+  const heldOf = (pod: string) => pods.find((p) => p.pod === pod)?.manifest.secrets ?? [];
+  const seats: Seat[] = [
+    ...walk.workspaces.map((ws) => {
+      const pod = workspaceHeldPod({ workflow: ws.workflow, route: ws.pod.route });
+      return { pod, sandbox: true, models: ws.pod.models, held: heldOf(pod) };
+    }),
+    ...(walk.instanceHarness
+      ? [
+          {
+            pod: INSTANCE_HARNESS_HELD_POD,
+            sandbox: false,
+            models: walk.instanceHarness.models,
+            held: heldOf(INSTANCE_HARNESS_HELD_POD),
+          },
+        ]
+      : []),
+  ];
+  const sandbox = config.sandbox?.nodeSelector || config.sandbox?.tolerations?.length ? config.sandbox : undefined;
+  const placementOf = (seat: Seat) => (seat.sandbox ? sandbox : undefined);
+  // A check pod stands for one class of Harness pod: its nodes, and whether it holds a secret. A pod
+  // that holds one sends its HTTPS through its Custodian (a tunnel for an unbound host); a pod that
+  // holds none sends it direct. So the two are two check pods, even on the same nodes.
+  const keyOf = (seat: Seat) => {
+    const placement = placementOf(seat);
+    return JSON.stringify({
+      nodeSelector: placement?.nodeSelector ?? {},
+      tolerations: placement?.tolerations ?? [],
+      holds: seat.held.length > 0,
+    });
+  };
+  const groups = new Map<string, { seats: Seat[]; placement?: SandboxPlacement }>();
+  for (const seat of seats) {
+    const key = keyOf(seat);
+    const group = groups.get(key) ?? { seats: [], ...(placementOf(seat) ? { placement: placementOf(seat) } : {}) };
+    group.seats.push(seat);
+    groups.set(key, group);
+  }
+  const checkPods: CheckPod[] = [];
+  for (const group of groups.values()) {
+    const models: string[] = [];
+    for (const seat of group.seats) for (const m of seat.models) if (!models.includes(m)) models.push(m);
+    const checkPod: CheckPod = {
+      where: [
+        ...(group.seats.some((s) => s.sandbox) ? ["the Sandbox nodes"] : []),
+        ...(group.seats.some((s) => !s.sandbox) ? ["the Instance Harness"] : []),
+      ].join(" and "),
+      ...(group.placement ? { placement: group.placement } : {}),
+      // A class that holds a secret: the first pod's own held set, so the check pod's env (the proxy, the
+      // Stand-ins) is that of a pod of the class; the keys the class's other pods hold join it below.
+      manifest: { version: 1, secrets: [...group.seats[0]!.held] },
+      providers: [],
+    };
+    for (const model of models) {
+      const slash = model.indexOf("/");
+      const id = slash > 0 ? model.slice(0, slash) : model;
+      const known = checkPod.providers.find((p) => p.id === id);
+      if (known) {
+        known.models.push(model);
+        continue;
+      }
+      const names =
+        id === custom?.id
+          ? [PROVIDER_KEY_NAME]
+          : Object.keys(MODEL_KEYS).filter((name) => MODEL_KEYS[name]!.providers.includes(id));
+      const runs = group.seats.filter((s) => s.models.some((m) => m.startsWith(`${id}/`)));
+      const holder = runs
+        .map((s) => pods.find((p) => p.pod === s.pod))
+        .find((p) => p?.manifest.secrets.some((b) => names.includes(b.name)));
+      const bindings = holder?.manifest.secrets.filter((b) => names.includes(b.name)) ?? [];
+      for (const b of bindings) {
+        if (checkPod.manifest.secrets.some((h) => h.name === b.name)) continue;
+        checkPod.manifest.secrets.push(b);
+      }
+      const gateway = config.harness?.catalog?.[id]?.baseUrl;
+      const fallback =
+        id === custom?.id
+          ? [new URL(custom.baseUrl).host]
+          : gateway
+            ? [new URL(gateway).host]
+            : names.flatMap((n) => (MODEL_KEYS[n]!.host ? [MODEL_KEYS[n]!.host!] : [])).slice(0, 1);
+      checkPod.providers.push({
+        id,
+        ...(id === custom?.id ? { custom: true } : {}),
+        models: [model],
+        keys: bindings.map((b) => b.name),
+        hosts: bindings.length ? [...new Set(bindings.flatMap((b) => b.hosts.map(hostText)))] : fallback,
+        ...(gateway ? { gateway } : {}),
+      });
+    }
+    if (checkPod.providers.length) checkPods.push(checkPod);
+  }
+  return checkPods;
+}
+
+const hostText = (h: { host: string; port: number }): string => (h.port === 443 ? h.host : `${h.host}:${h.port}`);
+
+/** How long a check pod may take to start: two cold image pulls (the Harness and the Custodian) and
+ * Envoy's start. The check's own bound starts when its container runs. */
+const CHECK_START_SECONDS = 300;
+/** Each model's bound in the pod: its own budget in the check (`PREFLIGHT_MODEL_BUDGET_SECONDS` in
+ * `@jr2/harness`'s `preflight-line.ts`, which a test holds this to), so one slow model never uses
+ * the time of the others. Exported for that test. */
+export const CHECK_MODEL_SECONDS = 360;
+const CHECK_RUN_MARGIN_SECONDS = 30;
+/** A leftover a run left with no pod, older than this, is a dead run's, not a live one's. */
+const CHECK_LEFTOVER_SECONDS = 3600;
+
+/**
+ * The provider preflight (ADR-0019, ADR-0059): each check pod of `checkPodsOf`, one at a time — a
+ * one-shot Harness-shaped pod, a Custodian that holds the providers' keys and the stock Harness
+ * image's `provider-preflight.ts` — that demands one tool call of each model, on the path a Turn takes. So no
+ * egress to the provider's host, a missing, wrong or empty key, a model no endpoint serves, and a
+ * gateway that does not answer each fail HERE. Every check pod runs, a pod that cannot be applied
+ * included, and every failure is in one error. A provider still busy after the retries, and a
+ * model that answered but did not call a tool it was not forced to, are warnings (ADR-0064).
+ *
+ * Each pod reads its own objects (`checkPodObjects`), written from `values`, `leaves`, `trust`
+ * and `harnessEnvData` — what the apply would write — and named for this run (`<run>-<n>`), so two
+ * converges in one namespace never read or delete each other's. The ones of an earlier run of this
+ * Instance that did not end go first (`removeCheckLeftovers`). A failed cleanup is a warning and
+ * never hides the check's own answer.
+ */
+async function preflightProviders(
   io: Io,
   kube: KubeAdmin,
-  config: JR2Config,
-  carriedModels: readonly string[],
-  namespace: string,
-  ctx: { context?: string },
-  caPem?: string,
+  opts: {
+    checkPods: CheckPod[];
+    /** `--skip-provider-preflight`: no check pod runs, and one warning line says so. */
+    skip: boolean;
+    name: string;
+    namespace: string;
+    ctx: { context?: string };
+    harnessImage: string;
+    custodianImage: string;
+    harness?: JR2Config["harness"];
+    caBundle: boolean;
+    values: Record<string, string>;
+    leaves: Record<string, Pem>;
+    trust: Record<string, string>;
+    harnessEnvData: Record<string, string>;
+    priorityClassName: string;
+    /** The signing key, for each check pod's own Stand-ins (ADR-0019, ADR-0059). */
+    signingKey: Buffer;
+  },
 ): Promise<void> {
-  const provider = config.harness?.provider;
-  if (!provider) return;
-  // The DEFINITIONS name the models (ADR-0018) and the Machines carry the definitions (ADR-0049),
-  // so probe the ones that will actually run, not one instance-wide default.
-  // "vllm/Qwen/Qwen3-32B" → the endpoint's model id is everything after the provider prefix; a
-  // definition on a different provider is not this endpoint's business. A workflow's per-turn dial cannot be probed here — invoke `input` is a
-  // function, so it is not statically recoverable; it is checked at admission instead.
-  // An Open model is nobody's model yet (ADR-0054), so there is nothing to probe for it. The walk
-  // already refused the converge above, so this filter is the type's, not the flow's.
-  const prefix = `${provider.id}/`;
-  // An Open body's allowed models are probed too (ADR-0070): a Chart's own Agent may name one.
-  const models = [...new Set(carriedModels.filter((m) => m.startsWith(prefix)).map((m) => m.slice(prefix.length)))];
-  if (models.length === 0) {
+  const { namespace, ctx } = opts;
+  // First, and with no check to run too: the leftovers of an earlier run can hold a key.
+  await removeCheckLeftovers(io, kube, opts.name, namespace, ctx);
+  if (opts.skip) {
+    // Never silent: a converge that checked nothing must not read as one that checked. A model that
+    // two check pods would check is named once.
+    const models = [...new Set(opts.checkPods.flatMap((p) => p.providers.flatMap((v) => v.models)))];
     activity(
       io,
-      `provider: ${provider.baseUrl} configured, but no carried Agent names a "${provider.id}/…" model — skipped`,
+      "warning: provider: preflight skipped (--skip-provider-preflight) — " +
+        (models.length ? `these models were not checked: ${models.join(", ")}` : "no model was checked"),
     );
     return;
   }
-  for (const model of models) {
-    activity(io, `provider: probing ${provider.baseUrl} from inside the cluster (model ${model})`);
-    const script = providerProbeScript(provider.baseUrl, model, provider.apiKey);
+  if (!opts.checkPods.length) return;
+  const run = randomBytes(4).toString("hex");
+  const orchestrator = { host: `${ORCHESTRATOR_SERVICE}.${namespace}.svc`, port: ORCHESTRATOR_PORT };
+  const faults: string[] = [];
+  for (const [n, checkPod] of opts.checkPods.entries()) {
+    const object = `${PROVIDER_PREFLIGHT}-${run}-${n}`;
+    const models = checkPod.providers.flatMap((p) => p.models);
+    /** The models whose result is already said, while the pod ran, and what it was: a final log
+     * read that comes back empty must not turn a result already said into "not checked". */
+    const said = new Set<string>();
+    const live = new Map<string, CheckVerdict>();
+    let lastLive = "";
+    for (const p of checkPod.providers) {
+      // No held key is not no credential: a credential that pi reads from `harness.env` (an AWS
+      // role for Bedrock, say) reaches the check as it reaches a Turn.
+      const why = p.custom
+        ? "harness.provider has no apiKey"
+        : `no Harness pod there holds a key of "${p.id}"; a credential in harness.env still reaches it`;
+      const holds = p.keys.length
+        ? `through a Custodian that holds ${p.keys.join(", ")} for ${p.hosts.join(", ")}`
+        : checkPod.manifest.secrets.length
+          ? `with no held key (${why}), through the Custodian's tunnel, as those pods call it`
+          : `with no held key (${why}), direct, as a pod that holds no secret calls it`;
+      activity(io, `provider: checking ${p.models.join(", ")} on ${checkPod.where}, ${holds}`);
+    }
+    activity(
+      io,
+      `provider: the check pod has up to ${CHECK_START_SECONDS / 60} min to start, then up to ` +
+        `${CHECK_MODEL_SECONDS / 60} min for ${models.length === 1 ? "its model" : `each of its ${models.length} models`}`,
+    );
+    const runSeconds = CHECK_MODEL_SECONDS * models.length + CHECK_RUN_MARGIN_SECONDS;
+    // Built before the apply: a fault in jr2 here is jr2's, never the cluster's refusal.
+    const manifest = checkPodObjects({
+      name: opts.name,
+      namespace,
+      object,
+      run,
+      harnessImage: opts.harnessImage,
+      custodianImage: opts.custodianImage,
+      held: checkPod.manifest,
+      values: opts.values,
+      leaves: opts.leaves,
+      trust: opts.trust,
+      custodian: {
+        bootstrap: JSON.stringify(custodianBootstrap(checkPod.manifest, orchestrator)),
+        script: custodianScript(checkPod.manifest, { fetch: false }),
+      },
+      // Its own credentials (ADR-0019, ADR-0076): a control token the Orchestrator never signed,
+      // and secret Stand-ins minted for this pod alone.
+      credentials: podCredentials(opts.signingKey, {
+        placement: object,
+        heldPod: object,
+        manifest: checkPod.manifest,
+        controlToken: "unsigned",
+      }),
+      harnessEnvData: opts.harnessEnvData,
+      models,
+      ...(checkPod.placement ? { placement: checkPod.placement } : {}),
+      priorityClassName: opts.priorityClassName,
+      ...(opts.harness ? { harness: opts.harness } : {}),
+      caBundle: opts.caBundle,
+      deadlineSeconds: CHECK_START_SECONDS + runSeconds + 60,
+    });
+    let end: PodEnd | undefined;
+    let refused: string | undefined;
     try {
-      // caPem: the probe trusts the instance's CA bundle exactly like the Harness will (ADR-0020) —
-      // a preflight that fails where the Harness would succeed is a broken promise, and vice versa.
-      await kube.runOneShot({
-        namespace,
-        name: `jr2-provider-preflight-${Date.now() % 100000}`,
-        script,
-        caPem,
-        ...ctx,
-      });
-      activity(io, `provider: ${model} reachable, and it completed a tool call`);
+      // Server-side: the trust bundles carry every root, past client-side apply's annotation.
+      await applyFirstContact(kube, { manifest, serverSide: true, warn: (line) => activity(io, line), ...ctx });
     } catch (err) {
-      throw new Error(
-        `provider preflight failed for model "${model}" against ${provider.baseUrl} (from inside the cluster): ` +
-          `${err instanceof Error ? err.message : err}\n` +
-          `  - localhost never works from a pod; use a LAN address\n` +
-          `  - the model id must be exactly what the endpoint serves (vLLM: GET /v1/models)\n` +
-          `  - vLLM needs --enable-auto-tool-choice and a matching --tool-call-parser\n` +
-          `  - finish_reason "length" means the model ran out of tokens before the call (a thinking model reasons first)`,
+      // A quota, a Pod Security level or a webhook that refuses the pod: this check pod's fault alone.
+      refused = `the cluster refused the check pod's objects: ${errText(err)}`;
+    }
+    try {
+      if (refused === undefined) {
+        end = await kube.awaitPodEnd({
+          namespace,
+          name: object,
+          container: CHECK_CONTAINER,
+          startSeconds: CHECK_START_SECONDS,
+          runSeconds,
+          onPending: (line) => activity(io, `provider: the check pod is not running yet — ${line}`),
+          // Each model's result as it arrives: a check of several slow models is not silence.
+          onLog: (log: string) => {
+            lastLive = log;
+            for (const [model, verdict] of checkVerdicts(log)) {
+              live.set(model, verdict);
+              if (said.has(model) || verdict.kind === "fault") continue;
+              said.add(model);
+              sayVerdict(io, model, verdict);
+            }
+          },
+          ...ctx,
+        });
+      }
+    } finally {
+      await removeCheckPod(io, kube, object, namespace, ctx);
+    }
+    // The last log read while the pod ran stands in for an empty final read.
+    const output = end?.output || lastLive;
+    const verdicts = checkVerdicts(output);
+    const started = checkStarts(output);
+    for (const p of checkPod.providers) {
+      for (const model of p.models) {
+        const verdict = verdicts.get(model) ?? live.get(model);
+        if (verdict && verdict.kind !== "fault") {
+          if (!said.has(model)) sayVerdict(io, model, verdict);
+          continue;
+        }
+        const pod = () => podFault(end!, checkPod, opts);
+        const why =
+          verdict?.reason ??
+          refused ??
+          (started.has(model)
+            ? `the check pod stopped while it checked this model: ${pod()}`
+            : `not checked: the check pod stopped before it reached this model: ${pod()}`);
+        faults.push(providerFault(p, model, checkPod.where, why, checkPod.manifest.secrets.length > 0));
+      }
+    }
+  }
+  if (faults.length) throw new Error(faults.join("\n"));
+}
+
+/** How long the first apply in a new namespace waits for its `default` ServiceAccount: the
+ * controller makes it a moment after the namespace, and until then a pod is refused. */
+const FIRST_CONTACT_TRIES = 10;
+const FIRST_CONTACT_PAUSE_MS = 1000;
+
+/** `kube.apply`, tried again while the namespace has no `default` ServiceAccount yet. Any other
+ * refusal is the answer. */
+async function applyFirstContact(kube: KubeAdmin, opts: Parameters<KubeAdmin["apply"]>[0]): Promise<void> {
+  for (let n = 1; ; n++) {
+    try {
+      return await kube.apply(opts);
+    } catch (err) {
+      if (n >= FIRST_CONTACT_TRIES || !/serviceaccount "default" not found/i.test(errText(err))) throw err;
+      await sleepMs(FIRST_CONTACT_PAUSE_MS);
+    }
+  }
+}
+
+/** The trust bundles a check pod reads (`<object>-ca`), as `trustObject` writes them to `jr2-ca`. */
+function trustData(
+  caBundle: string | undefined,
+  bundles: { extra: string; bundle: string; upstream: string } | undefined,
+) {
+  return {
+    ...(caBundle !== undefined ? { "ca.crt": caBundle } : {}),
+    ...(bundles ? { "extra.crt": bundles.extra, "bundle.crt": bundles.bundle, "upstream.crt": bundles.upstream } : {}),
+  };
+}
+
+/** How long after it ended an earlier run's check pod is left alone: that run may still be reading
+ * its log. Exported for its test. */
+export const CHECK_ENDED_MARGIN_SECONDS = 600;
+
+/**
+ * The leftovers of earlier runs of THIS Instance (`preflightProviders`), by label: a check pod
+ * past its `activeDeadlineSeconds` and `CHECK_ENDED_MARGIN_SECONDS`, in any phase; a check pod that
+ * ended more than `CHECK_ENDED_MARGIN_SECONDS` ago; each with its objects; and each object whose
+ * pod is gone and that is older than any check pod lives. A live pod, a pod that ended a short time
+ * ago and another Instance's objects stay. Every failure here is a warning.
+ */
+async function removeCheckLeftovers(
+  io: Io,
+  kube: KubeAdmin,
+  name: string,
+  namespace: string,
+  ctx: { context?: string },
+) {
+  const selector = `${LABEL_PROVIDER_PREFLIGHT},${LABEL_INSTANCE}=${name}`;
+  const since = (at: string | undefined) => (Date.now() - Date.parse(at ?? "")) / 1000;
+  try {
+    type CheckObject = KubeObject & {
+      metadata: { creationTimestamp?: string };
+      spec?: { activeDeadlineSeconds?: number };
+      status?: {
+        phase?: string;
+        containerStatuses?: Array<{ name?: string; state?: { terminated?: { finishedAt?: string } } }>;
+      };
+    };
+    const pods = await kube.listJson<CheckObject>({ kind: "pod", selector, namespace, ...ctx });
+    const named = new Set(pods.map((p) => p.metadata.name));
+    for (const p of pods) {
+      // Past its deadline and the margin, in any phase: a pod no node took has no start time, so
+      // the kubelet never applies `activeDeadlineSeconds` to it, and its Secret holds a key.
+      const deadline = (p.spec?.activeDeadlineSeconds ?? 0) + CHECK_ENDED_MARGIN_SECONDS;
+      if (since(p.metadata.creationTimestamp) > deadline) {
+        await removeCheckPod(io, kube, p.metadata.name, namespace, ctx);
+        continue;
+      }
+      if (!["Succeeded", "Failed"].includes(p.status?.phase ?? "")) continue;
+      // The check container, or `probe` in a pod that an older `jr2 up` made: the label finds both.
+      const finished = p.status?.containerStatuses?.find((c) => c.name === CHECK_CONTAINER || c.name === "probe")?.state
+        ?.terminated?.finishedAt;
+      if (!(since(finished ?? p.metadata.creationTimestamp) > CHECK_ENDED_MARGIN_SECONDS)) continue;
+      await removeCheckPod(io, kube, p.metadata.name, namespace, ctx);
+    }
+    for (const kind of ["configmap", "secret", "networkpolicy"]) {
+      for (const o of await kube.listJson<CheckObject>({ kind, selector, namespace, ...ctx })) {
+        const owner = o.metadata.name.replace(/-(ca|tls|env)$/, "");
+        if (named.has(owner) || !(since(o.metadata.creationTimestamp) > CHECK_LEFTOVER_SECONDS)) continue;
+        await kube.deleteObject({ kind, name: o.metadata.name, namespace, wait: false, ...ctx });
+      }
+    }
+  } catch (err) {
+    activity(io, `warning: provider: the leftovers of an earlier preflight were not removed (${errText(err)})`);
+  }
+}
+
+/** One check pod's objects, deleted; a failure is a warning, so it hides no check's answer. */
+async function removeCheckPod(io: Io, kube: KubeAdmin, object: string, namespace: string, ctx: { context?: string }) {
+  for (const { kind, name } of checkPodObjectNames(object)) {
+    try {
+      await kube.deleteObject({ kind, name, namespace, wait: false, ...ctx });
+    } catch (err) {
+      activity(
+        io,
+        `warning: provider: ${kind} ${name} was not deleted (${errText(err)}) — the next \`jr2 up\` removes it`,
       );
     }
   }
 }
 
+const errText = (err: unknown): string => (err instanceof Error ? err.message : String(err));
+
+/** A fault in which the provider refused the key or its rights: a 401 or a 403, in the words the
+ * apis use. The provider answered, so the key is the likely cause, and the egress is not. */
+export function refusedKey(fault: string): boolean {
+  return /\b40[13]\b|authentication|unauthori[sz]ed|permission|forbidden|invalid[ _-]?(x-)?api[ _-]?key/i.test(fault);
+}
+
+/** One model's result line (`preflight-line.ts` in `@jr2/harness`). */
+type CheckVerdict = { kind: "ok" } | { kind: "fault" | "busy" | "warn"; reason: string };
+
+/** One model's result that does not fail the converge, said as it comes. */
+function sayVerdict(
+  io: Io,
+  model: string,
+  verdict: { kind: "ok" } | { kind: "busy" | "warn" | "fault"; reason: string },
+) {
+  if (verdict.kind === "ok") activity(io, `provider: ${model} reachable, and it completed a tool call`);
+  else if (verdict.kind === "busy") {
+    activity(io, `warning: provider: ${model} answered with the key, but is busy — ${verdict.reason} (ADR-0064)`);
+  } else if (verdict.kind === "warn")
+    activity(io, `warning: provider: ${model} answered with the key — ${verdict.reason}`);
+}
+
+/** The models whose check began (`start` lines, `preflight-line.ts`): a model with a start and no
+ * verdict was on when the pod stopped; a model with neither was never reached. */
+export function checkStarts(output: string): Set<string> {
+  const started = new Set<string>();
+  for (const line of output.split("\n")) {
+    const m = /^jr2\.preflight model=(\S+) start$/.exec(line.trim());
+    if (m) started.add(m[1]!);
+  }
+  return started;
+}
+
+/** Each model's verdict, off the check container's output lines (`preflight-line.ts` in `@jr2/harness`, held
+ * to this parser by a test). A line that is not a result is not a verdict. */
+export function checkVerdicts(output: string): Map<string, CheckVerdict> {
+  const verdicts = new Map<string, CheckVerdict>();
+  for (const line of output.split("\n")) {
+    const m = /^jr2\.preflight model=(\S+) (?:(ok)|(fault|busy|warn)=(.*))$/.exec(line.trim());
+    if (!m) continue;
+    verdicts.set(m[1]!, m[2] ? { kind: "ok" } : { kind: m[3] as "fault" | "busy" | "warn", reason: m[4]! });
+  }
+  return verdicts;
+}
+
 /**
- * The in-cluster probe: GET /models, then one chat completion that must answer with tool_calls. The budget
- * is generous because a thinking model (Qwen3, DeepSeek-R1) reasons BEFORE it calls the tool, and that
- * reasoning counts against max_tokens — at 64 the completion ended with finish_reason "length" and an
- * empty tool_calls, which read as "the endpoint cannot call tools" when it could.
+ * Why a check pod said nothing of a model, from how it ended (`PodEnd`): the hint follows the reason
+ * the kubelet gave, and the Custodian's own refusal line is quoted when it has one. Exported for its
+ * test.
  */
-function providerProbeScript(baseUrl: string, model: string, apiKey?: string): string {
-  const base = JSON.stringify(baseUrl.replace(/\/+$/, ""));
-  const headers = apiKey
-    ? `{ "content-type": "application/json", authorization: "Bearer " + ${JSON.stringify(apiKey)} }`
-    : `{ "content-type": "application/json" }`;
+export function podFault(
+  end: PodEnd,
+  checkPod: Pick<CheckPod, "where">,
+  images: { harnessImage: string; custodianImage: string },
+): string {
+  const refusal = /jr2\.custodian refuse secret=\S+ reason=[\w-]+/.exec(end.custodian)?.[0];
+  const said = end.reasons.length ? ` (${end.reasons.join("; ")})` : "";
+  const stuck = end.stuck ?? "";
+  if (/ErrImagePull|ImagePullBackOff|InvalidImageName/.test(stuck)) {
+    return (
+      `the check pod cannot start: ${stuck}${said} — a node on ${checkPod.where} cannot pull ` +
+      `${stuck.startsWith("custodian") ? images.custodianImage : images.harnessImage}; check \`registry\` or ` +
+      "`kitRegistry`, and that those nodes can pull from it"
+    );
+  }
+  if (/CreateContainerConfigError/.test(stuck)) {
+    // The env of the check container: what `harness.env` and `harness.envFrom` name.
+    return (
+      `the check pod cannot start: ${stuck}${said} — a Secret, a ConfigMap or a key that the Harness env names ` +
+      "is missing (a `harness.envFrom` Secret, or a `harness.env` valueFrom)"
+    );
+  }
+  if (stuck.startsWith("FailedMount")) {
+    // A volume of the Custodian: a held secret's `valueFrom` Secret, or its key.
+    return (
+      `the check pod cannot start: ${stuck} — a Secret or a key that a \`secrets\` valueFrom names is missing ` +
+      "in the Instance's namespace"
+    );
+  }
+  const oom = [stuck, ...end.reasons].find((r) => /OOMKilled/.test(r));
+  if (oom) {
+    const custodian = oom.startsWith("custodian");
+    return (
+      `the check pod ran out of memory: ${oom}${said} — ` +
+      (custodian
+        ? `its Custodian has the Instance Harness Custodian's limit (${KIT_POD_SIZES.instanceCustodian.limits.memory})`
+        : `the check container's limit is ${KIT_POD_SIZES.checkContainer.limits.memory}`) +
+      "; a jr2 defect, not a fault of the provider"
+    );
+  }
+  if (refusal || stuck.startsWith("custodian")) {
+    return (
+      `the check pod's Custodian did not start: ${refusal ?? stuck}${said} — a Custodian stops at start on a held ` +
+      "value that is missing, empty or not printable ASCII"
+    );
+  }
+  if (end.phase === "timeout") {
+    return end.started
+      ? `the check did not end in time${said}`
+      : `the check pod did not start in ${CHECK_START_SECONDS}s${said} — no node on ${checkPod.where} fits it, or its image pulls are slow`;
+  }
+  if (!end.output.trim()) {
+    return `the check pod ended ${end.phase}${said}, and jr2 up could not read its log — run \`jr2 up\` again`;
+  }
+  const tail = end.output.trim().split("\n").filter(Boolean).slice(-5).join("; ");
+  return `the check pod ended ${end.phase} before it checked this model${said} — it said: ${tail}`;
+}
+
+/** One model's preflight failure: the model, the host, and the likely causes. */
+function providerFault(p: CheckedProvider, model: string, where: string, fault: string, custodian: boolean): string {
+  const hosts = p.hosts.join(", ") || `the host of "${p.id}"`;
+  // A model that loads on first use (Ollama, a large vLLM model) can take longer than the bound.
+  // Only where jr2 cannot know the endpoint: the custom provider, or a gateway.
+  const slow =
+    (p.custom || p.gateway) && /no answer in \d+ s|no verdict in \d+ s|time ran out/.test(fault)
+      ? "  - a model that loads on first use can take longer than the check waits: run `jr2 up` again once it has loaded\n"
+      : "";
+  if (p.custom) {
+    // The custom provider's own causes (ADR-0018): an endpoint jr2 knows nothing about but its url.
+    return (
+      `provider preflight failed for model "${model}" against ${hosts} (from a pod on ${where}, as a Turn calls it): ` +
+      `${fault}\n` +
+      slow +
+      // A refused key: the endpoint answered, so the key comes first and the reach is not it.
+      (refusedKey(fault)
+        ? (p.keys.length
+            ? `  - the held secret ${p.keys.join(" or ")} may be wrong (harness.provider.apiKey)\n`
+            : "  - the endpoint wants a key: set harness.provider.apiKey\n") +
+          "  - the model id must be exactly what the endpoint serves (vLLM: GET /v1/models)"
+        : "  - localhost never works from a pod; use a LAN address\n" +
+          "  - the model id must be exactly what the endpoint serves (vLLM: GET /v1/models)\n" +
+          "  - vLLM needs --enable-auto-tool-choice and a matching --tool-call-parser\n" +
+          '  - stop reason "length" means the model ran out of tokens before the call (a thinking model reasons first)' +
+          (p.keys.length ? `\n  - the held secret ${p.keys.join(" or ")} may be wrong (harness.provider.apiKey)` : ""))
+    );
+  }
+  const key = p.keys.length
+    ? `  - the held secret ${p.keys.join(" or ")} may be empty, wrong, or without access to this model (\`secrets\` in jr2.config.ts)\n`
+    : `  - no Harness pod on ${where} holds a key of "${p.id}": add its key to \`secrets\`` +
+      (Object.keys(MODEL_KEYS).some((n) => MODEL_KEYS[n]!.providers.includes(p.id))
+        ? ` (${Object.keys(MODEL_KEYS)
+            .filter((n) => MODEL_KEYS[n]!.providers.includes(p.id))
+            .join(" or ")}), or set in harness.env the credential that pi reads for it\n`
+        : `, or set in harness.env the credential that pi reads for it; jr2 knows no key name of "${p.id}" ` +
+          "(when pi does not know the id either, the line above says that it resolves to nothing)\n");
+  // The likely cause first: a provider that refused the key answered, so the egress is not it.
+  const egress = `  - the pods may have no egress to ${hosts} (a NetworkPolicy, a firewall, no route out of the cluster)\n`;
+  const causes = refusedKey(fault) ? key : egress + key;
   return (
-    `const h = ${headers};` +
-    `const m = await fetch(${base} + "/models", { headers: h });` +
-    `if (!m.ok) throw new Error("GET /models: HTTP " + m.status);` +
-    `const c = await fetch(${base} + "/chat/completions", { method: "POST", headers: h, body: JSON.stringify({` +
-    ` model: ${JSON.stringify(model)}, max_tokens: 1024,` +
-    ` messages: [{ role: "user", content: "Call the ping tool." }],` +
-    ` tools: [{ type: "function", function: { name: "ping", description: "reply with a ping", parameters: { type: "object", properties: {} } } }]` +
-    ` }) });` +
-    `const j = await c.json();` +
-    `if (!c.ok) throw new Error("POST /chat/completions: HTTP " + c.status + " " + JSON.stringify(j).slice(0, 300));` +
-    `const choice = j.choices?.[0];` +
-    `const calls = choice?.message?.tool_calls;` +
-    `if (!calls?.length) throw new Error("completion carried no tool_calls (finish_reason " + JSON.stringify(choice?.finish_reason ?? null) + ")");` +
-    `console.log("PROVIDER OK");`
+    `provider preflight failed for model "${model}" toward ${hosts} (from a pod on ${where}, ` +
+    `${custodian ? "through a Custodian" : "direct"}, ` +
+    `as a Turn calls it): ${fault}\n` +
+    slow +
+    causes +
+    (p.gateway
+      ? `  - harness.catalog.${p.id}.baseUrl (${p.gateway}) may not answer, or may not serve "${model.slice(p.id.length + 1)}" ` +
+        "(a LiteLLM model_name alias)\n"
+      : "") +
+    "  - a private CA on that path must be in harness.caBundle"
   );
 }
 
@@ -2371,31 +2920,3 @@ function noteDeferred(io: Io, repos: CarriedRepo[]): void {
     );
   }
 }
-
-/**
- * The `customize` line that retunes a composed Machine's Size (ADR-0060), nested through `actors`
- * the way `customize()` accepts — the Size twin of `customizeLine`, so the refusal pastes.
- */
-function sizeLine(path: readonly string[]): string {
-  const binds = '{ resources: { limits: { memory: "<memory>" } } }';
-  return `customize(<import>, ${path.reduceRight((parts, key) => `{ actors: { ${key}: ${parts} } }`, binds)})`;
-}
-
-/**
- * The Chart files under an Assembly root (ADR-0070), as Workflow names `<wrapper>/<name>`: each
- * `workflows/<wrapper>/<name>.chart.json`, one level below the flat `workflows/*.ts`, in a directory
- * named after its wrapper. The layout the Orchestrator's discovery reads (charts.ts); a directory
- * whose name starts with `_` or `.` is not a wrapper's.
- */
-async function chartFilesIn(root: string): Promise<string[]> {
-  const out: string[] = [];
-  const dir = join(root, "workflows");
-  const entries = await readdir(dir, { withFileTypes: true }).catch(() => []);
-  for (const entry of entries) {
-    if (!entry.isDirectory() || entry.name.startsWith("_") || entry.name.startsWith(".")) continue;
-    const files = await readdir(join(dir, entry.name)).catch(() => [] as string[]);
-    for (const f of files)
-      if (f.endsWith(".chart.json")) out.push(`${entry.name}/${f.slice(0, -".chart.json".length)}`);
-  }
-  return out.sort();
-}