@cohortapp/agent-sdk 2.3.1 → 2.4.0

package/lib/subagents/resolve.mjs

@@ -0,0 +1,455 @@
+/**
+ * lib/subagents/resolve.mjs — three-layer, total, per-slug resolution (§3).
+ *
+ * THE MODEL IN ONE SENTENCE: for every slug, exactly one layer wins and layers
+ * NEVER merge — LOCAL > WORKSPACE > SDK — and the winner is written to
+ * `agents/<slug>/agent.md` with its provenance stamped into the frontmatter.
+ *
+ * WHY NO MERGING. The obvious alternative (section-level `mode:"extend"` over an
+ * informal `## Mandate / ## Responsibilities / …` skeleton) is brittle exactly
+ * where it matters: the shipped agents and the generated ones already disagree
+ * about that skeleton, so a heading rename upstream silently drops an operator's
+ * override. Total precedence is legible — you can always answer "why does this
+ * file say that" with one word. "Extend" is spelled `subagent.fork`: copy the
+ * body, own it.
+ *
+ * WHY RESOLUTION IS LOCAL-FIRST AND NEVER NETWORKS. `detect` and `verify` run
+ * under `--headless`, in `doctor`, and in CI. The lock + `agents/` + the SDK
+ * manifest are sufficient for all three layers (layer 1 via the offline cache),
+ * so a dead hq degrades the roster, it never blocks the agent. hq is not on the
+ * runtime hot path at all.
+ *
+ * THE LOCAL RULE, SPELLED OUT. A file is a LOCAL EDIT when it exists and either:
+ *   (a) the lock has an entry for the slug and sha256(file) ≠ the fileHash we
+ *       recorded when we wrote it — i.e. something changed it after us; or
+ *   (b) the lock has NO entry and sha256(file) matches no sha the SDK manifest
+ *       has ever shipped — i.e. it did not come from us and it did not come from
+ *       the package, so a human wrote it.
+ * Everything else is MANAGED and may be silently re-materialised. That is the
+ * whole reason `create`'s copy list and `UPGRADE_PATHS` can stay untouched.
+ *
+ * Note (a) and (b) also give the safe default in every degraded case: no lock and
+ * no manifest ⇒ everything reads as a local edit ⇒ nothing is overwritten.
+ *
+ * Node builtins only. ESM.
+ *
+ * @module lib/subagents/resolve
+ */
+
+"use strict";
+
+import { existsSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+import { writeFileAtomic } from "../fs-atomic.mjs";
+import { listAgentDirs, manifestShas, readManifest, readSdkAgent } from "./manifest.mjs";
+import { latestCached, readLock } from "./lock.mjs";
+import {
+  KNOWN_TOKENS,
+  hashBody,
+  hashFile,
+  parseAgentMd,
+  stampProvenance,
+  unsupportedTokens,
+  validateAgentMd,
+} from "./schema.mjs";
+
+/** Layer names, in precedence order (first wins). */
+export const LAYERS = Object.freeze(["local", "workspace", "sdk"]);
+
+/** @param {string} agentRoot @param {string} slug @returns {string} */
+export function agentFilePath(agentRoot, slug) {
+  return join(agentRoot, "agents", String(slug), "agent.md");
+}
+
+/**
+ * Read every on-disk agent.md in the agent repo.
+ * @param {string} agentRoot
+ * @returns {{files:Map<string,{slug:string, path:string, text:string, fileHash:string}>, empty:string[]}}
+ */
+export function readLocalAgents(agentRoot) {
+  const { slugs, empty } = listAgentDirs(agentRoot);
+  const files = new Map();
+  for (const slug of slugs) {
+    const p = agentFilePath(agentRoot, slug);
+    let text;
+    try { text = readFileSync(p, "utf8"); } catch { continue; }
+    files.set(slug, { slug, path: p, text, fileHash: hashFile(text) });
+  }
+  return { files, empty };
+}
+
+/**
+ * Classify an on-disk file as a local edit or a managed copy.
+ * @param {{fileHash:string, lockEntry?:object, shas:Set<string>}} o
+ * @returns {{localEdits:boolean, reason:string}}
+ */
+export function classifyLocal(o = {}) {
+  const { fileHash, lockEntry, shas } = o;
+
+  // `fileHash` in the lock means "bytes WE materialised". It is only ever
+  // written after a successful write, so a match is proof the file is managed.
+  if (lockEntry && lockEntry.fileHash) {
+    if (lockEntry.fileHash === fileHash) return { localEdits: false, reason: "matches lock fileHash" };
+    return { localEdits: true, reason: "differs from the bytes recorded in the lock" };
+  }
+
+  // A file that has been restored to canonical bytes is managed again — checked
+  // BEFORE the sticky flag so reverting an edit re-adopts the file rather than
+  // marking it dirty forever.
+  if (shas && shas.has(fileHash)) return { localEdits: false, reason: "matches an SDK manifest sha" };
+
+  // STICKY: a previously-recorded local edit stays a local edit.
+  //
+  // Without this, a refusal was protective for exactly one run and destructive
+  // on the next: `materialise` correctly refused to overwrite a human's file,
+  // but the caller then recorded that file's OWN hash as `fileHash` — i.e. as
+  // bytes we had written — so the next run matched it, classified the file as
+  // managed, and overwrote the human's work with no `--force` and no warning.
+  // Reproduced end to end: run 1 refuses, run 2 silently destroys the edit.
+  //
+  // The flag is now authoritative and the hash of an unmanaged file is never
+  // recorded as ours (see the persist site in `materialiseAll`).
+  if (lockEntry && lockEntry.localEdits) {
+    return { localEdits: true, reason: "lock records an unresolved local edit" };
+  }
+
+  return { localEdits: true, reason: lockEntry ? "lock entry carries no fileHash" : "no lock entry and no SDK sha match" };
+}
+
+/**
+ * Build the WORKSPACE layer's per-slug body source from a `subagent.resolve`
+ * payload and/or the offline cache.
+ *
+ * `pins` is whatever the client last got from hq — either live or from the
+ * cache. Each entry is `{slug, definitionId, version, contentHash, frontmatter,
+ * body}`. When an entry carries no body (e.g. it came from the cheap
+ * `subagent.roster` read), we look for the body in the local cache; if it is not
+ * there either, the slug simply has no workspace layer this run and resolution
+ * falls through to SDK. That is the degraded-but-correct behaviour §4 demands.
+ *
+ * @param {{agentRoot:string, pins?:Array<object>}} o
+ * @returns {Map<string, {slug:string, definitionId:string|null, version:number|null, contentHash:string, body:string, frontmatter:object, fromCache:boolean}>}
+ */
+export function buildWorkspaceLayer(o = {}) {
+  const out = new Map();
+  for (const p of Array.isArray(o.pins) ? o.pins : []) {
+    if (!p || !p.slug) continue;
+    let body = typeof p.body === "string" ? p.body : "";
+    let frontmatter = p.frontmatter && typeof p.frontmatter === "object" ? { ...p.frontmatter } : null;
+    let fromCache = false;
+    if (!body && o.agentRoot) {
+      const cached = latestCached(o.agentRoot, p.slug);
+      if (cached) {
+        const parsed = parseAgentMd(cached.text);
+        body = parsed.body;
+        if (!frontmatter) frontmatter = parsed.frontmatter;
+        fromCache = true;
+      }
+    }
+    if (!body) continue;
+    out.set(String(p.slug), {
+      slug: String(p.slug),
+      definitionId: p.definitionId ? String(p.definitionId) : null,
+      version: Number.isFinite(p.version) ? p.version : null,
+      contentHash: p.contentHash ? String(p.contentHash) : hashBody(body),
+      body,
+      frontmatter: frontmatter || {},
+      fromCache,
+    });
+  }
+  return out;
+}
+
+/**
+ * Resolve every slug in `(SDK manifest) ∪ (workspace pins) ∪ (on-disk agents/*)`.
+ *
+ * Pure over its injected inputs (the only I/O is reading files). Never networks.
+ *
+ * @param {object} o
+ * @param {string} o.agentRoot
+ * @param {string} o.maestroRoot
+ * @param {Array<object>} [o.pins] the workspace layer (from client/cache)
+ * @param {object} [o.manifest] injectable (defaults to reading the SDK manifest)
+ * @param {object} [o.lock] injectable (defaults to reading the lock)
+ * @param {boolean} [o.force] DEMOTE the local layer: a locally-edited file is
+ *   still REPORTED in `localEdits`, but it no longer wins, so `pull --force`
+ *   actually re-resolves to workspace/SDK. Without this the local layer absorbs
+ *   the slug before precedence is ever consulted and `--force` would only ever
+ *   rewrite a file from itself. `--force` is the ONLY thing that does this.
+ * @returns {{entries:Array<object>, bySlug:Map<string,object>, localEdits:string[], unresolved:string[], emptyDirs:string[], degraded:string[]}}
+ */
+export function resolveSubagents(o = {}) {
+  const agentRoot = o.agentRoot;
+  const maestroRoot = o.maestroRoot;
+  const manifest = o.manifest || readManifest(maestroRoot);
+  const lock = o.lock || readLock(agentRoot);
+  const shas = manifestShas(manifest);
+  const { files, empty } = readLocalAgents(agentRoot);
+  const workspace = buildWorkspaceLayer({ agentRoot, pins: o.pins });
+
+  const slugs = new Set([...Object.keys(manifest.agents || {}), ...workspace.keys(), ...files.keys()]);
+  const entries = [];
+  const localEdits = [];
+  const unresolved = [];
+  const degraded = [];
+  if (workspace.size && [...workspace.values()].some((w) => w.fromCache)) degraded.push("workspace-from-cache");
+
+  for (const slug of [...slugs].sort()) {
+    const file = files.get(slug) || null;
+    const lockEntry = lock.agents ? lock.agents[slug] : null;
+
+    // ── Layer 2: LOCAL ────────────────────────────────────────────────────
+    // `localEdited` is computed once and reused by the fallback branch below, so
+    // an overridden (--force) file is still reported as edited even though it
+    // lost. Classification and precedence are deliberately separate questions.
+    let localEdited = false;
+    if (file) {
+      const cls = classifyLocal({ fileHash: file.fileHash, lockEntry, shas });
+      localEdited = cls.localEdits;
+      if (cls.localEdits) localEdits.push(slug);
+      if (cls.localEdits && !o.force) {
+        const parsed = parseAgentMd(file.text);
+        entries.push({
+          slug,
+          layer: "local",
+          source: `file://${file.path}`,
+          version: null,
+          definitionId: null,
+          contentHash: hashBody(parsed.body),
+          fileHash: file.fileHash,
+          localEdits: true,
+          reason: cls.reason,
+          frontmatter: parsed.frontmatter,
+          body: parsed.body,
+          path: file.path,
+          onDisk: true,
+        });
+        continue;
+      }
+    }
+
+    // ── Layer 1: WORKSPACE ────────────────────────────────────────────────
+    const w = workspace.get(slug);
+    if (w) {
+      entries.push({
+        slug,
+        layer: "workspace",
+        source: w.definitionId ? `cohort://${o.orgSlug || "workspace"}/${slug}` : `cohort://workspace/${slug}`,
+        version: w.version,
+        definitionId: w.definitionId,
+        contentHash: w.contentHash,
+        fileHash: file ? file.fileHash : "",
+        localEdits: false,
+        reason: w.fromCache ? "workspace pin (offline cache)" : "workspace pin",
+        frontmatter: w.frontmatter,
+        body: w.body,
+        path: agentFilePath(agentRoot, slug),
+        onDisk: !!file,
+        fromCache: w.fromCache,
+      });
+      continue;
+    }
+
+    // ── Layer 0: SDK ──────────────────────────────────────────────────────
+    if (manifest.agents && manifest.agents[slug]) {
+      const sdk = readSdkAgent(maestroRoot, slug);
+      if (sdk) {
+        entries.push({
+          slug,
+          layer: "sdk",
+          source: `sdk://${manifest.sdkVersion || "unknown"}/${slug}`,
+          version: manifest.sdkVersion || null,
+          definitionId: null,
+          contentHash: hashBody(sdk.body),
+          fileHash: file ? file.fileHash : "",
+          localEdits: false,
+          reason: "shipped in the SDK manifest",
+          frontmatter: sdk.frontmatter,
+          body: sdk.body,
+          path: agentFilePath(agentRoot, slug),
+          onDisk: !!file,
+        });
+        continue;
+      }
+    }
+
+    // A managed on-disk file whose layer has disappeared (yanked pin, dropped
+    // from the manifest). It still RESOLVES — the bytes are right there — but it
+    // is orphaned, and saying so is more useful than pretending it is SDK.
+    // Under --force a locally-edited file also lands here when no higher layer
+    // offers the slug: there is nothing to override it WITH, so it keeps the
+    // file and stays flagged as an edit rather than being silently blanked.
+    if (file) {
+      const parsed = parseAgentMd(file.text);
+      entries.push({
+        slug,
+        layer: "local",
+        source: `file://${file.path}`,
+        version: lockEntry ? lockEntry.version : null,
+        definitionId: lockEntry ? lockEntry.definitionId : null,
+        contentHash: hashBody(parsed.body),
+        fileHash: file.fileHash,
+        localEdits: localEdited,
+        orphaned: !localEdited,
+        reason: localEdited
+          ? "locally edited, and no other layer offers this slug"
+          : "managed file whose source layer no longer offers this slug",
+        frontmatter: parsed.frontmatter,
+        body: parsed.body,
+        path: file.path,
+        onDisk: true,
+      });
+      continue;
+    }
+
+    unresolved.push(slug);
+  }
+
+  const bySlug = new Map(entries.map((e) => [e.slug, e]));
+  return { entries, bySlug, localEdits, unresolved, emptyDirs: empty, degraded };
+}
+
+/**
+ * Render a resolved entry to the EXACT bytes that belong on disk.
+ *
+ * This is deliberately one function shared by `materialise` and by
+ * `subagents diff`. When `diff` re-derived the bytes itself it skipped
+ * validation — so it compared the raw layer frontmatter (`model: sonnet`, no
+ * `tokens:`) against the normalised bytes materialise writes (`model:
+ * claude-sonnet-4-6`, `tokens: []`) and reported EVERY managed file as differing
+ * from the layer it had just been written from. One renderer, one answer.
+ *
+ * Refusals, in order, and why each one exists:
+ *   1. Body fails the schema — the registry never puts prose on disk it could not
+ *      validate; a bad publish upstream must not become a bad file downstream.
+ *   2. Declared tokens the local renderer does not know — hq does not vendor
+ *      KNOWN_TOKENS, so the fetching agent is the only place this can be caught.
+ *      Refusing here makes `scripts/ci/check-unresolved-tokens.mjs` structurally
+ *      unable to fail after a fetch.
+ *
+ * @param {object} entry a resolveSubagents entry
+ * @param {{known?:Set<string>}} [o]
+ * @returns {{ok:boolean, text:string, reason:string, errors?:string[]}}
+ */
+export function renderEntry(entry, o = {}) {
+  const known = o.known || KNOWN_TOKENS;
+  const check = validateAgentMd({ frontmatter: entry.frontmatter, body: entry.body, slug: entry.slug, known });
+  if (!check.ok) return { ok: false, text: "", reason: "failed schema validation", errors: check.errors };
+
+  const bad = unsupportedTokens(check.normalised.frontmatter.tokens, known);
+  if (bad.length) {
+    return { ok: false, text: "", reason: `declares token(s) this agent cannot render: ${bad.join(", ")}`, errors: bad };
+  }
+
+  return {
+    ok: true,
+    reason: "",
+    text: stampProvenance({
+      frontmatter: check.normalised.frontmatter,
+      body: entry.body,
+      layer: entry.layer,
+      source: entry.source,
+      version: entry.version,
+      contentHash: entry.contentHash,
+    }),
+  };
+}
+
+/**
+ * Materialise a resolved entry to `agents/<slug>/agent.md`.
+ *
+ * The first refusal is precedence, the rest are {@link renderEntry}'s:
+ *   1. LOCAL without `--force` — the file IS the winner; rewriting it would be
+ *      the clobber lib/setup/enroll-from-cohort.mjs:1-31 exists to forbid.
+ *
+ * Tokens are NOT rendered: `{{agent.*}}` survives to disk and lib/render.mjs
+ * resolves it at load, which is what keeps one shipped body valid for every agent.
+ *
+ * @param {{agentRoot:string, entry:object, force?:boolean, known?:Set<string>, orgSlug?:string}} o
+ * @returns {{written:boolean, path:string, reason:string, fileHash?:string, errors?:string[]}}
+ */
+export function materialise(o = {}) {
+  const { agentRoot, entry } = o;
+  const path = agentFilePath(agentRoot, entry.slug);
+
+  if (entry.layer === "local" && !o.force) {
+    return { written: false, path, reason: entry.orphaned ? "orphaned local file left untouched" : "local edit wins (use --force to overwrite)" };
+  }
+
+  const rendered = renderEntry(entry, { known: o.known });
+  if (!rendered.ok) return { written: false, path, reason: rendered.reason, errors: rendered.errors };
+  const text = rendered.text;
+
+  // Idempotence: if the bytes already on disk are byte-identical, do not write.
+  // Not just an optimisation — a no-op write would churn mtime on every `doctor`
+  // run and make "when did this last change" useless.
+  if (existsSync(path)) {
+    try {
+      if (readFileSync(path, "utf8") === text) {
+        return { written: false, path, reason: "already up to date", fileHash: hashFile(text) };
+      }
+    } catch { /* fall through and write */ }
+  }
+
+  try {
+    writeFileAtomic(path, text);
+  } catch (e) {
+    return { written: false, path, reason: `write failed: ${e && e.message ? e.message : e}` };
+  }
+  return { written: true, path, reason: `materialised from ${entry.layer}`, fileHash: hashFile(text) };
+}
+
+/**
+ * Materialise every resolved entry and return the lock patch describing what
+ * ended up on disk. The caller writes the lock (so a caller can dry-run).
+ *
+ * @param {{agentRoot:string, resolved:object, force?:boolean, known?:Set<string>, only?:string[]}} o
+ * @returns {{results:Array<object>, lockAgents:Record<string,object>, written:number, refused:Array<object>}}
+ */
+export function materialiseAll(o = {}) {
+  const results = [];
+  const lockAgents = {};
+  const refused = [];
+  let written = 0;
+  const only = Array.isArray(o.only) && o.only.length ? new Set(o.only) : null;
+
+  for (const entry of (o.resolved && o.resolved.entries) || []) {
+    if (only && !only.has(entry.slug)) continue;
+    const r = materialise({ agentRoot: o.agentRoot, entry, force: o.force, known: o.known });
+    results.push({ slug: entry.slug, layer: entry.layer, ...r });
+    if (r.written) written++;
+    if (!r.written && r.errors) refused.push({ slug: entry.slug, reason: r.reason, errors: r.errors });
+
+    // `fileHash` means ONE thing: the bytes WE materialised. It is the proof
+    // `classifyLocal` uses to call a file managed, so recording anything else
+    // there is a licence to overwrite.
+    //
+    // It previously fell back to `entry.fileHash` — the hash of whatever was on
+    // disk — including on a REFUSAL. So refusing to overwrite a human's file
+    // recorded that human's bytes as ours, and the next run overwrote them with
+    // no `--force` and no warning. The refusal protected the file for exactly
+    // one run and then destroyed it.
+    //
+    // Now: only a real write (or a confirmed byte-identical no-op) records a
+    // fileHash. A refusal records none, and `localEdits` carries the state
+    // forward instead.
+    const materialised = r.written || r.reason === "already up to date";
+    const fileHash = materialised ? r.fileHash || "" : "";
+    lockAgents[entry.slug] = {
+      layer: entry.layer,
+      definitionId: entry.definitionId || null,
+      version: entry.version === undefined ? null : entry.version,
+      contentHash: entry.contentHash || "",
+      fileHash,
+      source: entry.source || "",
+      // A successful materialise RESOLVES the edit (we just replaced the file
+      // with canonical bytes, by force or because nothing was protecting it).
+      // A refusal leaves it unresolved, and `classifyLocal` now treats that as
+      // authoritative on the next run.
+      localEdits: materialised ? false : !!entry.localEdits,
+      materialisedAt: r.written ? new Date().toISOString() : (r.reason === "already up to date" ? new Date().toISOString() : ""),
+    };
+  }
+
+  return { results, lockAgents, written, refused };
+}