@jr2/orchestrator 0.3.0 → 0.4.1

package/src/repo-fetch.ts

@@ -17,8 +17,8 @@
 // would count a fetch that began BEFORE the ask. The same coalescing happens one hop earlier, in
 // this process: asks that raise the same bar for one Sandbox and key share one mark and one wait,
 // and past a cap on how many waits may be open the answer is the cache — the caller is inside an
-// untrusted pod (ADR-0013), and a `git fetch` loop must not cost this process a kubectl per
-// second per iteration.
+// untrusted pod (ADR-0013), and a `git fetch` loop must not cost this process a write per
+// iteration.
 //
 // Freshness degrades, absence does not (ADR-0051/0053): a remote fetch that failed, or one that
 // outran the budget, is answered as `stale` with what the cache holds — never as an error. The
@@ -26,13 +26,15 @@
 // still succeeds. Only a Repo the Sandbox does not mount is a refusal: the Sandbox token's scope
 // is the caches its own pod mounts (ADR-0013).
 //
-// Drives the Sandbox CR through `kubectl` exactly as the provision's Ready wait does
-// (sandbox-kubectl.ts) — same process seam, injectable, so the mapping is unit-testable without a
-// cluster; the kind e2e tier exercises the real thing.
+// The mark is a merge patch through the Orchestrator's own client, and the wait is the one
+// Sandbox watch (ADR-0063): the watch event that carries the landing answers the ask. Both are
+// injectable, so the mapping is unit-testable against a fake API server; the kind e2e tier
+// exercises the real thing.
 
+import { SANDBOXES, kubeClient, type KubeClient } from "./kube-client.ts";
 import { askedAnnotation } from "./names.ts";
 import { repoKeyOfIdentity } from "./repo-identity.ts";
-import { defaultKubectlExec, type KubectlExec } from "./sandbox-kubectl.ts";
+import { watchSandboxes, type SandboxObject, type SandboxRepoStatus, type SandboxWatch } from "./sandbox-watch.ts";
 
 /** What one ask settles as. `fetched` is the landing's timestamp; `stale` is git's own words (or
  * this port's, when the budget ran out) beside the time the cache's objects are as of — `null`
@@ -54,54 +56,49 @@ export class UnmountedRepoError extends Error {}
 const CACHE_BUDGET_MS = 60_000;
 /** How many asks this Orchestrator will hold open at once, counting one per Sandbox and key. The
  * caller is inside a pod (ADR-0013: untrusted), an Agent that loops on `git fetch` is a mode this
- * codebase has already seen, and every open ask costs a `kubectl` per second in the one process
+ * codebase has already seen, and every open ask costs a write and a waiter in the one process
  * that serves every run in the instance. Past the cap an ask is answered the way a failed one is —
  * the cache, said out loud — so a pod that floods degrades its own fetches and nobody else's. */
 const MAX_IN_FLIGHT = 16;
 /** Watch slack on top of it: the agent's verdict has to reach the Repo CR, the operator has to
- * compute the Sandbox entry from it, and this port has to read it. The ADR's "nothing else owns a
+ * compute the Sandbox entry from it, and the watch has to carry it here. The ADR's "nothing else owns a
  * timeout" holds — this is the agent's budget plus the round trip, not a second policy. */
 const WATCH_SLACK_MS = 15_000;
 
-export type KubectlRepoFetchesOptions = {
+export type KubeRepoFetchesOptions = {
   /** The instance's namespace — the Sandboxes are here. */
   namespace: string;
-  /** kubectl `--context` override. Default: the current context (ADR-0009). */
-  context?: string;
-  /** Process seam, injectable for tests. Defaults shell to the `kubectl` on PATH. */
-  exec?: KubectlExec;
-  /** The clock the ask is stamped from, and the budget measured with. Injectable for tests. */
+  /** The Kubernetes client (ADR-0063), shared with the Sandbox port. Default: the in-cluster one. */
+  client?: KubeClient;
+  /** The one Sandbox watch, shared with the Sandbox port. Default: one started on first use. */
+  watch?: SandboxWatch;
+  /** The clock the ask is stamped from. Injectable for tests. */
   now?: () => Date;
-  /** How often the Sandbox's status is re-read while waiting. Default 1s. */
-  pollMs?: number;
   /** The whole wait. Default: the cache agent's on-demand budget plus watch slack. */
   budgetMs?: number;
   /** How many asks may be open at once. Default {@link MAX_IN_FLIGHT}. */
   maxInFlight?: number;
 };
 
-export function kubectlRepoFetches(opts: KubectlRepoFetchesOptions): RepoFetches {
-  const exec = opts.exec ?? defaultKubectlExec;
+export function kubeRepoFetches(opts: KubeRepoFetchesOptions): RepoFetches {
   const now = opts.now ?? (() => new Date());
-  const pollMs = opts.pollMs ?? 1_000;
   const budgetMs = opts.budgetMs ?? CACHE_BUDGET_MS + WATCH_SLACK_MS;
   const maxInFlight = opts.maxInFlight ?? MAX_IN_FLIGHT;
-  const base = ["--namespace", opts.namespace, ...(opts.context ? ["--context", opts.context] : [])];
+  let client: KubeClient | undefined = opts.client;
+  const kube = () => (client ??= kubeClient());
+  let watch: SandboxWatch | undefined = opts.watch;
+  const sandboxes = () => (watch ??= watchSandboxes(kube(), { namespace: opts.namespace }));
   // The asks this port is holding open, one per Sandbox and key, each with the second its mark was
   // raised to. The mark is a COALESCER on the node; this is the same coalescing one hop earlier,
   // and it is exact rather than approximate: every stamp that can answer an ask is kept at the
   // second (askedAt below), so two asks that raise the same bar CANNOT get different answers.
-  // Sharing one wait between them spares the CR a second write and this process a second poll.
+  // Sharing one wait between them spares the CR a second write and this process a second waiter.
   const openAsks = new Map<string, { until: number; answer: Promise<FetchAnswer> }>();
 
-  const get = async (sandbox: string): Promise<SandboxItem | undefined> => {
-    try {
-      const { stdout } = await exec(["get", "sandbox", sandbox, ...base, "-o", "json"]);
-      return JSON.parse(stdout) as SandboxItem;
-    } catch (err) {
-      if (isNotFound(err)) return undefined;
-      throw err;
-    }
+  /** The Sandbox as the watch holds it; the API server itself for one the watch has not seen (or
+   * before its first list has landed). */
+  const get = async (sandbox: string): Promise<SandboxObject | undefined> => {
+    return sandboxes().get(sandbox) ?? (await kube().get<SandboxObject>(SANDBOXES, opts.namespace, sandbox));
   };
 
   return {
@@ -144,38 +141,37 @@ export function kubectlRepoFetches(opts: KubectlRepoFetchesOptions): RepoFetches
   /** The mark and the wait on it, for one Sandbox and key. Every ask that shares the mark's second
    * shares this one promise. */
   async function wait(sandbox: string, key: string, asked: string): Promise<FetchAnswer> {
-    // The mark. ONE call writes it and prints the object as it stands afterwards (the lease's
-    // trick, sandbox-kubectl.ts), so the first look at the status costs no second round trip —
-    // and a fetch that already landed since an earlier ask answers immediately.
-    const { stdout } = await exec([
-      "annotate",
-      "sandbox",
-      sandbox,
-      ...base,
-      `${askedAnnotation(key)}=${asked}`,
-      "--overwrite",
-      "-o",
-      "json",
-    ]);
-    const deadline = now().getTime() + budgetMs;
-    let item = JSON.parse(stdout) as SandboxItem;
-    for (;;) {
-      const answer = verdict(item, key, asked);
-      if (answer) return answer;
-      if (now().getTime() >= deadline) {
-        // The budget is the cache agent's, and this is what running past it looks like from
-        // here: the agent may still be at it (its next interval will land), so what the caller
-        // gets is the cache, said out loud.
-        return { stale: "timed out waiting for the node cache", asOf: fetchedOf(item, key) };
-      }
-      await sleep(pollMs);
-      const next = await get(sandbox);
-      // The Sandbox went while we waited: the pod that asked is gone, so there is nothing left
-      // to answer for. The caller is inside that pod, so this is nearly unreachable — and a
-      // refusal beats inventing a verdict on a resource that no longer exists.
-      if (!next) throw new UnmountedRepoError(`Sandbox "${sandbox}" is gone`);
-      item = next;
-    }
+    // The mark. The merge patch answers the object as it stands afterwards, so a fetch that
+    // already landed since an earlier ask answers at once, with no event to wait for.
+    const marked = await kube().patch<SandboxObject>(SANDBOXES, opts.namespace, sandbox, {
+      metadata: { annotations: { [askedAnnotation(key)]: asked } },
+    });
+    const first = verdict(marked, key, asked);
+    if (first) return first;
+    // Then the watch: the event that carries the landing is the answer (ADR-0063). The budget is
+    // the cache agent's, and running past it looks like this from here: the agent may still be at
+    // it (its next interval will land), so what the caller gets is the cache, said out loud.
+    return new Promise<FetchAnswer>((resolve, reject) => {
+      let last: SandboxObject = marked;
+      const done = (fn: () => void) => {
+        clearTimeout(timer);
+        unsubscribe();
+        fn();
+      };
+      const timer = setTimeout(
+        () => done(() => resolve({ stale: "timed out waiting for the node cache", asOf: fetchedOf(last, key) })),
+        budgetMs,
+      );
+      const unsubscribe = sandboxes().subscribe(sandbox, (item) => {
+        // The Sandbox went while we waited: the pod that asked is gone, so there is nothing left
+        // to answer for. The caller is inside that pod, so this is nearly unreachable — and a
+        // refusal beats inventing a verdict on a resource that no longer exists.
+        if (!item) return done(() => reject(new UnmountedRepoError(`Sandbox "${sandbox}" is gone`)));
+        last = item;
+        const answer = verdict(item, key, asked);
+        if (answer) done(() => resolve(answer));
+      });
+    });
   }
 }
 
@@ -194,7 +190,7 @@ export function kubectlRepoFetches(opts: KubectlRepoFetchesOptions): RepoFetches
  *
  * `error` is the other verdict, and the operator scopes it to an attempt made for that same ask.
  */
-function verdict(item: SandboxItem, key: string, asked: string): FetchAnswer | undefined {
+function verdict(item: SandboxObject, key: string, asked: string): FetchAnswer | undefined {
   const entry = entryOf(item, key);
   if (!entry) return undefined;
   const entryAsked = at(entry.asked);
@@ -221,11 +217,11 @@ function askedAt(stamp: string): number {
  * fetch at all, which is what the program prints as "an unknown time". This is the `asOf` of a
  * stale answer: the fetch it names did NOT answer the ask, and saying when the objects are from is
  * the whole of what a degraded answer can offer (ADR-0053). */
-function fetchedOf(item: SandboxItem, key: string): string | null {
+function fetchedOf(item: SandboxObject, key: string): string | null {
   return entryOf(item, key)?.fetched ?? null;
 }
 
-function entryOf(item: SandboxItem, key: string): SandboxRepoStatus | undefined {
+function entryOf(item: SandboxObject, key: string): SandboxRepoStatus | undefined {
   return (item.status?.repos ?? []).find((r) => r.key === key);
 }
 
@@ -236,24 +232,3 @@ function at(stamp: string | undefined): number | undefined {
   const ms = Date.parse(stamp);
   return Number.isNaN(ms) ? undefined : ms;
 }
-
-/** A `Sandbox` resource as `kubectl get -o json` prints it — the fields this port reads. */
-type SandboxItem = {
-  spec?: { repos?: Array<{ key?: string }> };
-  status?: { repos?: SandboxRepoStatus[] };
-};
-
-/** The operator's standing per-key Repo entry on a Sandbox (ADR-0053). */
-type SandboxRepoStatus = {
-  key: string;
-  asked?: string;
-  fetched?: string;
-  attempted?: string;
-  error?: string;
-};
-
-function isNotFound(err: unknown): boolean {
-  return err instanceof Error && /NotFound|not found/i.test(err.message);
-}
-
-const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));

package/src/repos.ts

@@ -6,9 +6,9 @@
 // on every node that needs the repository. What comes back is that agent's per-node status, which
 // is what `jr2 status` reports and what a provision waits on through the Sandbox's `Ready`.
 //
-// The port's kubectl implementation (`kubectlRepos`) drives the CRD the same way the Sandbox port
-// does (sandbox-kubectl.ts): shelling to `kubectl`, honoring the current kube context, with the
-// process seam injectable so the mapping is unit-testable without a cluster.
+// The port's implementation (`kubeRepos`) drives the CRD the same way the Sandbox port does
+// (sandbox-kube.ts): through the Orchestrator's own Kubernetes client (kube-client.ts, ADR-0063),
+// injectable so the mapping is unit-testable against a fake API server.
 //
 // TWO SPELLINGS, ONE RESOURCE: the resource is named by the cache key, and its `spec.url` and
 // `secretRef` are ONE statement — the boot's. The walk collapses every Machine binding an
@@ -18,7 +18,7 @@
 // eviction clock — never the url. So a Machine that binds over ssh what another bound over https
 // borrows the cache the boot stated, and its runs never flip the resource between the two (each
 // flip is a generation the cache agent re-points origin and refetches on). The push url is each
-// Binding's own (attachScript), so nothing about the run is wrong; only the cache's transport is
+// Binding's own (the Harness's attach, ADR-0063), so nothing about the run is wrong; only the cache's transport is
 // shared.
 //
 // A resource NOTHING binds has no boot to restate it, so the provision is its one writer: an
@@ -33,7 +33,7 @@
 
 import { credentialSecretFor, matchCredential, type GitCredential } from "./config.ts";
 import { ANNOTATION_REPO_IDENTITY, ANNOTATION_REPO_LAST_ATTACHED, LABEL_REPO_BOUND } from "./names.ts";
-import { defaultKubectlExec, type KubectlExec } from "./sandbox-kubectl.ts";
+import { REPOS, isAlreadyExists, kubeClient, KubeError, type KubeClient } from "./kube-client.ts";
 
 /** One node's view of a Repo, as the cache agent reports it on the resource's status. */
 export type RepoNodeState = {
@@ -75,8 +75,8 @@ export interface RepoResources {
    * that moved the url or the credential reaches the cache. The only caller that writes a BOUND
    * resource's spec, and never the eviction clock: a boot is not an attach. Resolves the
    * `git.credentials` entry for the identity into the `secretRef` the cache agent reads: an https
-   * entry's token materializes as a Secret in Flux's shape, an ssh entry names its deploy-key
-   * Secret and the Orchestrator never reads it.
+   * entry names the clone Secret `jr2 up` wrote from its token, an ssh entry names its deploy-key
+   * Secret. The Orchestrator writes and reads neither (ADR-0068).
    */
   bind(repo: { url: string; identity: string; key: string }): Promise<void>;
   /**
@@ -95,63 +95,48 @@ export interface RepoResources {
   list(): Promise<RepoStatus[]>;
 }
 
-/** The CRD's fully qualified plural — unambiguous to kubectl whatever else calls itself a repo. */
-const REPO_RESOURCE = "repos.core.jr2.dev";
-
-export type KubectlReposOptions = {
+export type KubeReposOptions = {
   /** The instance's namespace — the Repo resources live beside the Sandboxes that name them. */
   namespace: string;
-  /** kubectl `--context` override. Default: the current context (ADR-0009). */
-  context?: string;
-  /** The instance's `git.credentials`, matched by prefix on the identity (config.ts). */
-  credentials: readonly GitCredential[];
-  /** Where a token entry's env var is read from (deployed: `process.env`, which the Instance
-   * Secret's `envFrom` populated at `jr2 up`). Unset → the resource carries no `secretRef` and the
-   * clone is anonymous. */
-  env: Record<string, string | undefined>;
+  /** The instance's `git.credentials`, matched by prefix on the identity (config.ts). A function
+   * is read on every call: the config that applies is the current Assembly's, and a publish
+   * swaps it without a new port (ADR-0069). */
+  credentials: readonly GitCredential[] | (() => readonly GitCredential[]);
+  /** The clone Secrets `jr2 up` wrote (deployed: `git-clone.json` in `jr2-held`), by name. A token
+   * entry whose Secret is not among them — its variable was unset at `jr2 up` — gives the resource
+   * no `secretRef`, and the clone is anonymous. Default: none. */
+  cloneSecrets?: () => Promise<ReadonlySet<string>>;
   /** CR `spec.refreshInterval` — how often the cache agent fetches a warm cache. Default `5m`. */
   refreshInterval?: string;
-  /** Process seam, injectable for tests. Defaults shell to the `kubectl` on PATH. */
-  exec?: KubectlExec;
+  /** The Kubernetes client (ADR-0063), shared with the Sandbox port. Default: the in-cluster one. */
+  client?: KubeClient;
   /** The clock `last-attached` is stamped from. Injectable for tests. */
   now?: () => Date;
 };
 
-export function kubectlRepos(opts: KubectlReposOptions): RepoResources {
-  const exec = opts.exec ?? defaultKubectlExec;
+export function kubeRepos(opts: KubeReposOptions): RepoResources {
   const now = opts.now ?? (() => new Date());
-  const base = ["--namespace", opts.namespace, ...(opts.context ? ["--context", opts.context] : [])];
-
-  /**
-   * The Secret the cache agent spends for an https url, in Flux's shape (`username`/`password`),
-   * so a Flux or Argo user reuses the Secret they have. Its name is derived from the entry's
-   * `match`, so a redeploy finds its own and two entries never share one. Applied (create-or-
-   * update) on every ensure: a token rotated by `jr2 up` reaches the Secret at the next boot.
-   */
-  const applyTokenSecret = async (name: string, password: string): Promise<void> => {
-    const secret = {
-      apiVersion: "v1",
-      kind: "Secret",
-      metadata: { name, namespace: opts.namespace, labels: { "app.kubernetes.io/managed-by": "jr2" } },
-      type: "Opaque",
-      stringData: { username: "x-access-token", password },
-    };
-    await exec(["apply", ...base, "-f", "-"], { input: JSON.stringify(secret) });
-  };
+  const ns = opts.namespace;
+  // Built on first use: a boot never needs the cluster to construct this port (ADR-0048).
+  let client: KubeClient | undefined = opts.client;
+  const kube = () => (client ??= kubeClient());
 
   /**
    * The `secretRef` for one url: the entry the identity matches, then the url's scheme picks
-   * which of its fields applies (config.ts). A token entry whose env var is unset writes no ref —
-   * the clone is anonymous, and `jr2 status` will show git's refusal if the host wanted one.
+   * which of its fields applies (config.ts). It only names the Secret (ADR-0068): `jr2 up` wrote the
+   * token's, in Flux's shape, and the cache agent reads it. A token entry whose Secret `jr2 up` did
+   * not write — its variable was unset — names none: the clone is anonymous, and `jr2 status` will
+   * show git's refusal if the host wanted one.
    */
   const secretRefFor = async (url: string, identity: string): Promise<{ name: string } | undefined> => {
-    const cred = credentialSecretFor(url, matchCredential(identity, opts.credentials));
+    const cred = credentialSecretFor(
+      url,
+      matchCredential(identity, typeof opts.credentials === "function" ? opts.credentials() : opts.credentials),
+    );
     if (cred === undefined) return undefined;
     if (cred.kind === "ssh") return { name: cred.secret };
-    const value = opts.env[cred.env];
-    if (value === undefined || value === "") return undefined;
-    await applyTokenSecret(cred.secret, value);
-    return { name: cred.secret };
+    const written = (await opts.cloneSecrets?.()) ?? new Set<string>();
+    return written.has(cred.secret) ? { name: cred.secret } : undefined;
   };
 
   /** The resource as one statement writes it: `create` needs the whole, `bind` its spec again.
@@ -168,7 +153,7 @@ export function kubectlRepos(opts: KubectlReposOptions): RepoResources {
         kind: "Repo",
         metadata: {
           name: repo.key,
-          namespace: opts.namespace,
+          namespace: ns,
           ...(repo.bound ? { labels: { [LABEL_REPO_BOUND]: "true" } } : {}),
           annotations: {
             [ANNOTATION_REPO_IDENTITY]: repo.identity,
@@ -185,9 +170,9 @@ export function kubectlRepos(opts: KubectlReposOptions): RepoResources {
   };
 
   /** Create, never apply — an existing resource keeps its spec. True when this call created it. */
-  const create = async (cr: object): Promise<boolean> => {
+  const create = async (cr: RepoItem & { metadata: { name: string } }): Promise<boolean> => {
     try {
-      await exec(["create", ...base, "-f", "-"], { input: JSON.stringify(cr) });
+      await kube().create(REPOS, ns, cr);
       return true;
     } catch (err) {
       if (!isAlreadyExists(err)) throw err;
@@ -196,13 +181,14 @@ export function kubectlRepos(opts: KubectlReposOptions): RepoResources {
   };
 
   const patch = async (key: string, body: object): Promise<void> => {
-    await exec(["patch", REPO_RESOURCE, key, ...base, "--type", "merge", "-p", JSON.stringify(body)]);
+    await kube().patch(REPOS, ns, key, body);
   };
 
   /** One resource as it stands — what an unbound `ensure` restates its credential against. */
   const get = async (key: string): Promise<RepoItem> => {
-    const { stdout } = await exec(["get", REPO_RESOURCE, key, ...base, "-o", "json"]);
-    return JSON.parse(stdout) as RepoItem;
+    const item = await kube().get<RepoItem>(REPOS, ns, key);
+    if (!item) throw new Error(`Repo "${key}" vanished between its create and its read`);
+    return item;
   };
 
   return {
@@ -246,9 +232,9 @@ export function kubectlRepos(opts: KubectlReposOptions): RepoResources {
 
     async reconcileBound(keys) {
       const keep = new Set(keys);
-      let stdout: string;
+      let items: RepoItem[];
       try {
-        ({ stdout } = await exec(["get", REPO_RESOURCE, ...base, "-l", `${LABEL_REPO_BOUND}=true`, "-o", "json"]));
+        ({ items } = await kube().list<RepoItem>(REPOS, ns, { labelSelector: `${LABEL_REPO_BOUND}=true` }));
       } catch (err) {
         // A cluster with no `repos.core.jr2.dev` resource type holds no Repos, so "nothing to
         // unlabel" is the complete answer — the one read that may answer none. An instance that
@@ -258,23 +244,34 @@ export function kubectlRepos(opts: KubectlReposOptions): RepoResources {
         if (!isMissingResourceType(err)) throw err;
         return;
       }
-      for (const item of itemsOf(stdout)) {
+      for (const item of items) {
         const name = item.metadata?.name;
         if (name === undefined || keep.has(name)) continue;
-        // `<label>-` is kubectl's spelling for "remove the label".
-        await exec(["label", REPO_RESOURCE, name, ...base, `${LABEL_REPO_BOUND}-`]);
+        // A merge patch's `null` removes the label.
+        await patch(name, { metadata: { labels: { [LABEL_REPO_BOUND]: null } } });
       }
     },
 
     async list() {
-      const { stdout } = await exec(["get", REPO_RESOURCE, ...base, "-o", "json"]);
-      return itemsOf(stdout).map(repoStatusOf);
+      let items: RepoItem[];
+      try {
+        ({ items } = await kube().list<RepoItem>(REPOS, ns));
+      } catch (err) {
+        // The data plane is wired on every deployed boot (ADR-0069), so `GET /repos` asks here on
+        // a cluster with no operator too. No Repo type holds no Repos: an empty list, the same
+        // answer `reconcileBound` gives. Any other refusal still throws.
+        if (!isMissingResourceType(err)) throw err;
+        return [];
+      }
+      return items.map(repoStatusOf);
     },
   };
 }
 
-/** A `Repo` resource as `kubectl get -o json` prints it — the fields this port reads. */
+/** A `Repo` resource as the API server answers it — the fields this port reads. */
 type RepoItem = {
+  apiVersion?: string;
+  kind?: string;
   metadata?: { name?: string; labels?: Record<string, string>; annotations?: Record<string, string> };
   spec?: { url?: string };
   status?: {
@@ -290,11 +287,6 @@ type RepoItem = {
   };
 };
 
-function itemsOf(stdout: string): RepoItem[] {
-  const parsed = JSON.parse(stdout) as { items?: RepoItem[] };
-  return parsed.items ?? [];
-}
-
 /** The resource → what `GET /repos` reports: the Orchestrator's own metadata read back, and the
  * cache agent's per-node entries verbatim. Absent optional fields stay absent, not `undefined`
  * keys, so the JSON a client sees is exactly the type. */
@@ -319,12 +311,9 @@ export function repoStatusOf(item: RepoItem): RepoStatus {
   };
 }
 
-function isAlreadyExists(err: unknown): boolean {
-  return err instanceof Error && /AlreadyExists|already exists/i.test(err.message);
-}
-
-/** kubectl's words for "this cluster has no such CRD" — the Repo type is the operator's to install
- * (ADR-0051), and an instance may be deployed where nothing installed it. */
+/** The API server's answer for a kind it does not serve: 404 on the collection itself — the Repo
+ * type is the operator's to install (ADR-0051), and an instance may be deployed where nothing
+ * installed it. (An object that is absent is a 404 on the OBJECT, which a list never asks for.) */
 function isMissingResourceType(err: unknown): boolean {
-  return /doesn't have a resource type/i.test(err instanceof Error ? err.message : String(err));
+  return err instanceof KubeError && err.status === 404;
 }

package/src/rest.ts

@@ -0,0 +1,133 @@
+// Rest (ADR-0024, ADR-0029): the condition of an Agent's state under which a pick is answered.
+//
+//   A pick that moves the workflow to a state inside the Agent's state is answered when the
+//   workflow is at rest there: no state below the Agent's state is running something (an Effect, a
+//   Gate, another Agent, a child Machine), and no pending timer will move the workflow to another
+//   state inside the Agent's state.
+//
+// It is read from the Machine, never kept: the snapshot (which states are active, which children
+// run) and the timer table (`timers.ts`, the same table a restore re-arms). So a restored run gives
+// the same answer as the run before the restart. Read it AFTER the mailbox drains — a microtask
+// after a snapshot, or in a request of its own — never inside a subscriber: an event already queued
+// (a `sendTo(self)` with no delay) moves the Machine before the next tick.
+//
+// Everything below is computed over the SET of active states below the Agent's state. Today that set
+// is one path (a parallel region below an Agent's state is refused at load, ADR-0015), and nothing
+// here assumes it.
+//
+// What the rule does not know: what an Effect is. It knows what waiting is — a live invoked child, a
+// timer that holds. A settled child with no `onDone` is rest; a state named `rest` means nothing.
+// Known limit: a delayed `sendTo` to a child whose reply then moves the Machine is not counted.
+
+import type { AnyActorRef, AnyMachineSnapshot, AnyStateMachine } from "xstate";
+import { seatOf, type PartySeat } from "./party.ts";
+import { scheduledFor } from "./timers.ts";
+
+type Node = AnyStateMachine["root"];
+
+/** Is `node` active in `snapshot`? Read off the state value, a public shape. */
+function isActive(snapshot: AnyMachineSnapshot, node: Node): boolean {
+  let value: unknown = snapshot.value;
+  for (const key of node.path) {
+    if (typeof value === "string") return value === key && node.path.at(-1) === key;
+    if (!value || typeof value !== "object" || !(key in value)) return false;
+    value = (value as Record<string, unknown>)[key];
+  }
+  return true;
+}
+
+/** Every active state strictly below `s`: a set, which is one path today and may be more. */
+export function activeBelow(snapshot: AnyMachineSnapshot, s: Node): Node[] {
+  const out: Node[] = [];
+  const visit = (node: Node, value: unknown): void => {
+    if (node.type === "parallel") {
+      for (const child of Object.values(node.states)) {
+        out.push(child);
+        visit(child, value && typeof value === "object" ? (value as Record<string, unknown>)[child.key] : undefined);
+      }
+      return;
+    }
+    if (typeof value === "string") {
+      const child = node.states[value];
+      if (child) {
+        out.push(child);
+        visit(child, undefined);
+      }
+    } else if (value && typeof value === "object") {
+      for (const [key, sub] of Object.entries(value)) {
+        const child = node.states[key];
+        if (!child) continue;
+        out.push(child);
+        visit(child, sub);
+      }
+    }
+  };
+  let value: unknown = snapshot.value;
+  for (const key of s.path)
+    value = value && typeof value === "object" ? (value as Record<string, unknown>)[key] : undefined;
+  visit(s, value);
+  return out;
+}
+
+const isInside = (node: Node, s: Node): boolean => {
+  for (let n: Node | undefined = node; n; n = n.parent) if (n === s) return true;
+  return false;
+};
+
+/**
+ * The pending timers that hold Rest: a timer whose event, delivered to this Machine, is handled by a
+ * transition of an active state at or below `s` whose target is inside `s` (or `s` itself, from `s`,
+ * without re-entering it — re-entering ends the Turn). Guards are not run: a guarded handler holds
+ * until the timer fires, and then nothing is pending. A timer whose handler leaves `s` (a Turn
+ * timeout) does not hold: when it fires, the Turn ends (ADR-0024).
+ */
+export function holdingTimers(machine: AnyActorRef, s: Node): string[] {
+  const snapshot = machine.getSnapshot() as AnyMachineSnapshot;
+  const active = [s, ...activeBelow(snapshot, s)];
+  return scheduledFor(machine)
+    .filter((t) =>
+      active.some((node) =>
+        (node.transitions.get(t.event.type) ?? []).some((tr) =>
+          ((tr.target ?? []) as Node[]).some((target) =>
+            target === s ? node === s && !tr.reenter : isInside(target, s),
+          ),
+        ),
+      ),
+    )
+    .map((t) => t.id);
+}
+
+/** The invoke ids of the live children of active states strictly below `s`. The Agent's own invoke
+ * is on `s` and does not count. */
+export function runningBelow(machine: AnyActorRef, s: Node): string[] {
+  const snapshot = machine.getSnapshot() as AnyMachineSnapshot;
+  const children = snapshot.children as Record<string, AnyActorRef | undefined>;
+  const out: string[] = [];
+  for (const node of activeBelow(snapshot, s))
+    for (const inv of node.invoke) if (children[inv.id]?.getSnapshot().status === "active") out.push(inv.id);
+  return out;
+}
+
+/** Is the Machine at Rest in `s`? Nothing below it runs, and no pending timer holds. */
+export function atRest(machine: AnyActorRef, s: Node): boolean {
+  return runningBelow(machine, s).length === 0 && holdingTimers(machine, s).length === 0;
+}
+
+/**
+ * Where an Agent's registration stands: `rest`, `busy` (its state is active and not at Rest), or
+ * `gone` (its state is no longer active: the Turn is over, ADR-0024). A registration with no seat to
+ * read (a plain `setup()` tree, a bare test entry) reads `rest` — today's behavior, never a false
+ * hold.
+ */
+export type RestReading = "rest" | "busy" | "gone";
+
+export function restOf(reg: PartySeat): RestReading {
+  if (!reg.invoker || !reg.invokeId) return "rest";
+  const snapshot = reg.invoker.getSnapshot() as AnyMachineSnapshot | undefined;
+  if (!snapshot || !(snapshot.machine as AnyStateMachine | undefined)?.root) return "rest";
+  if (snapshot.status !== "active") return "gone";
+  const seat = seatOf(snapshot, reg.invokeId);
+  if (!seat) return "rest";
+  if (!isActive(snapshot, seat)) return "gone";
+  return atRest(reg.invoker, seat) ? "rest" : "busy";
+}