full-reverse 0.0.0-stage → 1.0.0

package/skills/full-reverse/scripts/lib/workspace.mjs

@@ -0,0 +1,861 @@
+// Workspace operations: init, atomic publication, evidence/external registration,
+// digests, reconcile and build descriptions. Nothing here executes a target.
+import { randomBytes } from "node:crypto";
+import fsp from "node:fs/promises";
+import path from "node:path";
+import { scanFile } from "./jsonscan.mjs";
+import { DEFAULT_MAX_PARSE_BYTES, inspectFile } from "./rea.mjs";
+import { workspaceScopeDigest } from "./scope.mjs";
+import { validateSchema } from "./schemas.mjs";
+import {
+  CliError,
+  EXIT,
+  atomicReplace,
+  canonicalDigest,
+  hashFile,
+  nowIso,
+  openWorkspace,
+  publishExclusive,
+  readJson,
+  resolveInWorkspace,
+  slugify,
+  stableStringify,
+  toInt,
+  usage,
+  withLock,
+} from "./util.mjs";
+
+export const INDEX_PATH = "evidence/index.json";
+const LAYOUT_DIRS = ["evidence/records", "evidence/external", "evidence/diagnostics", "evidence/.staging", "specs"];
+const KIND_DIR = {
+  "canonical-record": "evidence/records",
+  "canonical-bundle": "evidence/records",
+  "derived-view": "evidence/records",
+  external: "evidence/external",
+  diagnostic: "evidence/diagnostics",
+  "interrupted-output": "evidence/diagnostics",
+};
+export const kindDir = (kind) => KIND_DIR[kind];
+export const EVIDENCE_DIRS = ["evidence/records", "evidence/external", "evidence/diagnostics"];
+const MUTABLE_TARGETS = {
+  "investigation.json": "investigation",
+  "inventory.json": "inventory",
+  "claims.json": "claims",
+  "review.json": "review",
+};
+
+/** Directories that make up an empty workspace skeleton (and their ancestors). */
+const SKELETON_DIRS = new Set(["evidence", ...LAYOUT_DIRS]);
+
+async function readIndexFile(ws) {
+  const target = await resolveInWorkspace(ws.abs, INDEX_PATH);
+  if (!target.exists) throw new CliError(`evidence/index.json is missing; run init-workspace first`, { code: "index_missing" });
+  const index = await readJson(target.abs);
+  const result = await validateSchema("evidence-index", index);
+  if (!result.valid) throw new CliError("evidence/index.json does not match its schema", { code: "index_invalid", details: result.errors });
+  return index;
+}
+
+async function writeIndexFile(ws, index) {
+  index.updated_at = nowIso();
+  const result = await validateSchema("evidence-index", index);
+  if (!result.valid) throw new CliError("refusing to write an invalid evidence index", { code: "index_invalid", details: result.errors });
+  await atomicReplace((await resolveInWorkspace(ws.abs, INDEX_PATH)).abs, stableStringify(index));
+}
+
+async function readInvestigationMaybe(ws) {
+  const target = await resolveInWorkspace(ws.abs, "investigation.json");
+  if (!target.exists) return null;
+  try {
+    return await readJson(target.abs);
+  } catch {
+    throw new CliError("investigation.json is not valid JSON", { code: "investigation_invalid" });
+  }
+}
+
+/** Bind intrinsic digests to a build: every digest must equal the build's exact identity digest. */
+export function matchBuild(builds, digests) {
+  if (!Array.isArray(builds) || digests.length === 0) return null;
+  return builds.find((build) => digests.every((digest) => digest === build.identity?.sha256)) ?? null;
+}
+
+function targetFor(investigation, binding, declaredBuildId) {
+  const base = { binding_source: binding.source, binding_digests: binding.digests, identity_state: binding.identity_state ?? null };
+  const builds = investigation?.builds ?? [];
+  if (binding.digests.length > 0) {
+    const match = matchBuild(builds, binding.digests);
+    if (declaredBuildId) {
+      if (!match || match.build_id !== declaredBuildId) {
+        throw new CliError(
+          `--build-id ${declaredBuildId} does not match the record's own target digest ${binding.digests.join(",")}; refusing cross-target publication`,
+          { code: "cross_target" },
+        );
+      }
+    }
+    return match
+      ? { association: "intrinsic", build_id: match.build_id, ...base }
+      : { association: "unbound", build_id: null, ...base };
+  }
+  if (declaredBuildId) {
+    if (!builds.some((build) => build.build_id === declaredBuildId)) {
+      throw new CliError(`--build-id ${declaredBuildId} is not a build in investigation.json`, { code: "unknown_build" });
+    }
+    return { association: "declared", build_id: declaredBuildId, ...base };
+  }
+  return { association: "unbound", build_id: null, ...base };
+}
+
+// --- init-workspace -----------------------------------------------------------
+
+/**
+ * True when the directory holds nothing but (recursively empty) layout
+ * directories, i.e. an interrupted earlier init that never wrote the index.
+ */
+async function isEmptySkeleton(abs, rel = "") {
+  for (const entry of await fsp.readdir(path.join(abs, rel), { withFileTypes: true })) {
+    const childRel = rel ? `${rel}/${entry.name}` : entry.name;
+    if (!entry.isDirectory() || !SKELETON_DIRS.has(childRel) || !(await isEmptySkeleton(abs, childRel))) return false;
+  }
+  return true;
+}
+
+export async function initWorkspace(flags) {
+  if (!flags.workspace) throw usage("--workspace DIR is required");
+  const investigationId = flags["investigation-id"] ?? `inv_${new Date().toISOString().slice(0, 10).replace(/-/g, "")}-${randomBytes(3).toString("hex")}`;
+  if (!/^inv_[a-z0-9][a-z0-9_-]{2,62}$/.test(investigationId)) throw usage("--investigation-id must match ^inv_[a-z0-9][a-z0-9_-]{2,62}$");
+  const requested = path.resolve(flags.workspace);
+
+  // Phase 1: decide whether the directory may be initialized. Nothing is written here.
+  const rootStat = await fsp.stat(requested).catch((error) => {
+    if (error.code === "ENOENT") return null;
+    throw error;
+  });
+  let ws = null;
+  let hasIndex = false;
+  if (rootStat) {
+    if (!rootStat.isDirectory()) throw new CliError(`workspace is not a directory: ${requested}`, { code: "workspace_missing" });
+    ws = await openWorkspace(requested);
+    for (const dir of LAYOUT_DIRS) {
+      const layout = await resolveInWorkspace(ws.abs, dir);
+      if (layout.exists && !layout.stat.isDirectory()) throw new CliError(`${dir} exists and is not a directory`, { code: "workspace_exists" });
+    }
+    const indexTarget = await resolveInWorkspace(ws.abs, INDEX_PATH);
+    if (indexTarget.exists) {
+      hasIndex = true;
+      const index = await readIndexFile(ws);
+      if (index.investigation_id !== investigationId) {
+        throw new CliError(`workspace already belongs to ${index.investigation_id}`, { code: "workspace_exists" });
+      }
+    } else if (!(await isEmptySkeleton(ws.abs))) {
+      throw new CliError(
+        `${ws.abs} is not empty and is not an investigation workspace; init-workspace only initializes an empty directory or re-opens a workspace with the same --investigation-id. Nothing was written`,
+        { code: "workspace_not_empty" },
+      );
+    }
+  }
+  const existingInvestigation = ws ? await readInvestigationMaybe(ws) : null;
+  if (existingInvestigation && existingInvestigation.investigation_id !== investigationId) {
+    throw new CliError("investigation.json belongs to a different investigation", { code: "workspace_exists" });
+  }
+
+  // Phase 2: writes.
+  if (!ws) {
+    await fsp.mkdir(requested, { recursive: true });
+    ws = await openWorkspace(requested);
+  }
+  for (const dir of LAYOUT_DIRS) await fsp.mkdir(path.join(ws.abs, dir), { recursive: true });
+  const created = [];
+  if (!hasIndex) {
+    await writeIndexFile(ws, { schema_version: "full-reverse.evidence-index/1", investigation_id: investigationId, updated_at: nowIso(), entries: [] });
+    created.push(INDEX_PATH);
+  }
+  if (!existingInvestigation && flags.objective) {
+    const investigation = {
+      schema_version: "full-reverse.investigation/1",
+      investigation_id: investigationId,
+      created_at: nowIso(),
+      objective: flags.objective,
+      non_goals: [],
+      workspace: { path: ws.abs, allowed_output_paths: [ws.abs] },
+      builds: [],
+      questions: [],
+      authorizations: [],
+      tools: [],
+      phase: "scope",
+      steps: [],
+      lifecycle: { last_resumed_at: null, review_sha256: null, updated_at: nowIso() },
+    };
+    await atomicReplace(path.join(ws.abs, "investigation.json"), stableStringify(investigation));
+    created.push("investigation.json");
+  }
+  return { workspace: ws.abs, investigation_id: investigationId, created, directories: LAYOUT_DIRS, staging_dir: path.join(ws.abs, "evidence/.staging") };
+}
+
+// --- publish (mutable files) --------------------------------------------------
+
+async function readStdin(limit) {
+  const chunks = [];
+  let size = 0;
+  for await (const chunk of process.stdin) {
+    size += chunk.length;
+    if (size > limit) throw new CliError("stdin exceeds the 64 MiB publish limit", { code: "too_large" });
+    chunks.push(chunk);
+  }
+  return Buffer.concat(chunks);
+}
+
+export async function publishFile(flags) {
+  const ws = await openWorkspace(flags.workspace);
+  const target = flags.target;
+  if (!target) throw usage("--target REL is required");
+  if (!flags.source) throw usage("--source FILE|- is required");
+  const isSpec = /^specs\/.+\.md$/.test(target);
+  const schemaName = MUTABLE_TARGETS[target] ?? null;
+  if (!schemaName && !isSpec) {
+    throw new CliError(
+      `publish only writes ${Object.keys(MUTABLE_TARGETS).join(", ")} and specs/**/*.md; evidence/** is written by publish-evidence/register-external and evidence/index.json by the helper`,
+      { exit: EXIT.USAGE, code: "target_not_allowed" },
+    );
+  }
+  if (flags.schema && flags.schema !== schemaName) throw usage(`--schema ${flags.schema} does not match target ${target}`);
+  const resolved = await resolveInWorkspace(ws.abs, target);
+  const data = flags.source === "-" ? await readStdin(64 * 1024 * 1024) : await fsp.readFile(flags.source);
+  let report = { schema: schemaName ?? "none", valid: true };
+  if (schemaName) {
+    let doc;
+    try {
+      doc = JSON.parse(data.toString("utf8"));
+    } catch (error) {
+      throw new CliError(`source is not valid JSON: ${error.message}`, { code: "invalid_json" });
+    }
+    const result = await validateSchema(schemaName, doc);
+    if (!result.valid) {
+      throw new CliError(`${target} does not match schema ${schemaName}; nothing was written`, { code: "schema_invalid", details: result.errors });
+    }
+    const index = await readIndexFile(ws);
+    if (doc.investigation_id !== index.investigation_id) {
+      throw new CliError(`investigation_id ${doc.investigation_id} does not match the workspace (${index.investigation_id})`, { code: "investigation_mismatch" });
+    }
+  } else if (data.includes(0)) {
+    throw new CliError("spec files must be text", { code: "invalid_text" });
+  }
+  await fsp.mkdir(path.dirname(resolved.abs), { recursive: true });
+  await resolveInWorkspace(ws.abs, target); // re-check after mkdir
+  await atomicReplace(resolved.abs, data);
+  const { sha256, bytes } = await hashFile(resolved.abs);
+  return { published: target, sha256, bytes, replaced: resolved.exists, ...report };
+}
+
+// --- publish-evidence -----------------------------------------------------------
+
+function parseCommandJson(value) {
+  if (value === undefined) return null;
+  let text = value;
+  if (!value.trim().startsWith("[")) {
+    // treat as file path
+    throw usage("--command-json must be a JSON array of strings (inline) or use @FILE");
+  }
+  let parsed;
+  try {
+    parsed = JSON.parse(text);
+  } catch (error) {
+    throw usage(`--command-json is not valid JSON: ${error.message}`);
+  }
+  if (!Array.isArray(parsed) || !parsed.every((item) => typeof item === "string")) throw usage("--command-json must be an array of strings");
+  return parsed;
+}
+
+async function commandFromFlag(value) {
+  if (value === undefined) return null;
+  if (value.startsWith("@")) return parseCommandJson(await fsp.readFile(value.slice(1), "utf8"));
+  return parseCommandJson(value);
+}
+
+function extensionFor(contentType) {
+  if (["rea-record", "rea-bundle", "rea-error", "json"].includes(contentType)) return "json";
+  if (contentType === "jsonl") return "jsonl";
+  if (contentType === "binary") return "bin";
+  return "txt";
+}
+
+/**
+ * Validate a caller-supplied source file and canonicalize its location. The
+ * final component must be a regular file (never a symlink/FIFO/device); the
+ * parent directory is realpath-ed so a workspace reached through a symlink
+ * compares equal to the canonical root that openWorkspace returns.
+ */
+async function stagedSource(ws, stagedArg, name) {
+  if (!stagedArg) throw usage(`--${name} FILE is required`);
+  const lexical = path.resolve(stagedArg);
+  const stat = await fsp.lstat(lexical).catch(() => null);
+  if (!stat) throw new CliError(`staged file does not exist: ${lexical}`, { code: "missing" });
+  if (!stat.isFile()) throw new CliError(`staged path must be a regular file (no symlinks, FIFOs or devices): ${lexical}`, { code: "not_regular_file" });
+  const abs = path.join(await fsp.realpath(path.dirname(lexical)), path.basename(lexical));
+  const relToWs = path.relative(ws.abs, abs).split(path.sep).join("/");
+  const insideWorkspace = relToWs !== "" && relToWs !== ".." && !relToWs.startsWith("../") && !path.isAbsolute(relToWs);
+  if (insideWorkspace) {
+    for (const dir of EVIDENCE_DIRS) {
+      if (relToWs === dir || relToWs.startsWith(`${dir}/`)) throw new CliError("staged file is already inside published evidence", { code: "already_published" });
+    }
+  }
+  return { abs, inStaging: insideWorkspace && relToWs.startsWith("evidence/.staging/") };
+}
+
+// --- publication identity -------------------------------------------------------------
+//
+// One index entry == one file == one semantic publication. The identity of a
+// publication is the digest of everything that gives the bytes meaning (kind,
+// status, invocation outcome, command, step, stderr bytes, target, producer,
+// note, external metadata); only where/when it is stored is excluded. The same
+// bytes published with a different outcome or metadata therefore become a new
+// entry with its own file, while an exact retry maps to the existing entry.
+// Blob identity stays bytes-based: ext_id is ext_<sha256>; record evidence_ids
+// are intrinsic; entries/files are what disambiguate provenance.
+
+export function semanticDigest(entry) {
+  const { entry_id: _id, file: _file, published_at: _at, ...rest } = entry;
+  const body = { ...rest, invocation: { ...rest.invocation, stderr_file: null } };
+  if (body.external) body.external = { ...body.external, registered_at: null, file: { ...body.external.file, path: null } };
+  return canonicalDigest(body);
+}
+
+/** Attach the digest-derived entry_id and file name to a draft entry. */
+function identify(draft, { dir, slug, ext }) {
+  const digest = semanticDigest(draft);
+  const short = digest.slice(0, 16);
+  const { kind, ...rest } = draft;
+  const file = `${dir}/${short}-${slug}.${ext}`;
+  const entry = { entry_id: `ei_${short}`, kind, file, ...rest };
+  if (entry.external) entry.external = { ...entry.external, file: { ...entry.external.file, path: file } };
+  return { digest, entry };
+}
+
+async function verifyIndexedFile(ws, entry) {
+  const resolved = await resolveInWorkspace(ws.abs, entry.file, { allowMissing: false });
+  const live = await hashFile(resolved.abs);
+  if (live.sha256 !== entry.sha256 || live.bytes !== entry.bytes) {
+    throw new CliError(`${entry.file} no longer matches its indexed sha256/bytes; the saved evidence was modified`, { code: "evidence_modified" });
+  }
+}
+
+/** Copy the staged snapshot to its exclusive destination; an existing destination must hold identical bytes. */
+async function placeFile(ws, stagedAbs, file, sha256) {
+  const probe = await resolveInWorkspace(ws.abs, file);
+  await fsp.mkdir(path.dirname(probe.abs), { recursive: true });
+  const dest = await resolveInWorkspace(ws.abs, file); // re-check after mkdir
+  const outcome = await publishExclusive(stagedAbs, dest.abs, { expectedSha256: sha256 });
+  if (outcome === "exists") {
+    const live = await hashFile(dest.abs);
+    if (live.sha256 !== sha256) throw new CliError(`destination ${file} exists with different content`, { code: "destination_conflict", exit: EXIT.IO });
+  }
+}
+
+/**
+ * Add an entry to the in-memory index, or recognize an exact retry. An existing
+ * entry with the same id must have the same semantics and a verified live file;
+ * an index that was edited or a truncated-id collision fails closed.
+ */
+async function ensureEntry(ws, index, entry, digest, stagedAbs) {
+  const existing = index.entries.find((candidate) => candidate.entry_id === entry.entry_id);
+  if (existing) {
+    // The slug is a cosmetic part of the file name and not of the identity: an exact retry
+    // under another --name reuses the existing file, provided the file lives where this
+    // identity puts it (same directory, digest-derived prefix) and the entry is consistent.
+    const prefix = `${entry.entry_id.slice(3)}-`;
+    const sameHome = path.posix.dirname(existing.file) === path.posix.dirname(entry.file) && path.posix.basename(existing.file).startsWith(prefix);
+    const sameExternalPath = !existing.external || existing.external.file.path === existing.file;
+    if (!sameHome || !sameExternalPath || semanticDigest(existing) !== digest) {
+      throw new CliError(`entry ${entry.entry_id} is already indexed with different content or metadata; refusing to merge publications`, { code: "entry_conflict", exit: EXIT.IO });
+    }
+    await verifyIndexedFile(ws, existing);
+    return { entry: existing, created: false };
+  }
+  if (index.entries.some((candidate) => candidate.file === entry.file)) {
+    throw new CliError(`${entry.file} is already indexed by another entry`, { code: "entry_conflict", exit: EXIT.IO });
+  }
+  await placeFile(ws, stagedAbs, entry.file, entry.sha256);
+  const stored = { ...entry, published_at: nowIso() };
+  index.entries.push(stored);
+  return { entry: stored, created: true };
+}
+
+function companionDraft(inspected) {
+  return {
+    kind: "diagnostic",
+    sha256: inspected.sha256,
+    bytes: inspected.bytes,
+    status: "complete",
+    content_type: inspected.content_type === "empty" ? "empty" : inspected.content_type === "binary" ? "binary" : "text",
+    producer: { name: null, version: null, provider: null },
+    operation: null,
+    predicate_type: null,
+    records: [],
+    records_truncated: false,
+    id_authentication: { status: "not-applicable", reason: null },
+    invocation: { source: "unknown", command: null, exit_code: null, stderr_file: null, stderr_sha256: null, step_id: null },
+    target: { association: "unbound", build_id: null, binding_source: "none", binding_digests: [], identity_state: null },
+    external: null,
+    note: "stderr companion; shared by every publication whose stderr has these bytes",
+  };
+}
+
+/**
+ * Producer metadata: what the saved bytes say about themselves wins. --producer-name and
+ * --producer-version only fill what the content does not declare; a flag that contradicts
+ * intrinsic metadata is refused instead of silently replacing it (it would enter the
+ * publication identity and nothing downstream re-checks it).
+ */
+function producerFor(inspected, flags) {
+  const intrinsic = { name: inspected.content_type.startsWith("rea-") ? "rea" : null, version: inspected.producer_version ?? null };
+  const resolved = {};
+  for (const field of ["name", "version"]) {
+    const flag = flags[`producer-${field}`];
+    if (flag !== undefined && (typeof flag !== "string" || flag === "")) throw usage(`--producer-${field} needs a non-empty value`);
+    if (flag !== undefined && intrinsic[field] !== null && flag !== intrinsic[field]) {
+      throw usage(`--producer-${field} ${flag} contradicts the producer ${field} recorded in the content (${intrinsic[field]}); omit the flag`);
+    }
+    resolved[field] = intrinsic[field] ?? flag ?? null;
+  }
+  return { name: resolved.name, version: resolved.version, provider: inspected.provider };
+}
+
+export async function publishEvidence(flags) {
+  const ws = await openWorkspace(flags.workspace);
+  const maxParse = toInt(flags["max-parse-bytes"], "max-parse-bytes", { min: 1 }) ?? DEFAULT_MAX_PARSE_BYTES;
+  const staged = await stagedSource(ws, flags.staged, "staged");
+  const exitCode = flags["exit-code"] === undefined ? null : toInt(flags["exit-code"], "exit-code", { min: 0 });
+  const interrupted = flags.interrupted === true;
+  const command = await commandFromFlag(flags["command-json"]);
+  const source = flags.source ?? (command ? "cli" : "unknown");
+  if (!["cli", "session", "export", "unknown"].includes(source)) throw usage("--source must be cli|session|export|unknown");
+  const stepId = flags["step-id"] ?? null;
+  if (stepId !== null && !/^step_[a-z0-9][a-z0-9_-]{0,62}$/.test(stepId)) throw usage("--step-id must match ^step_[a-z0-9][a-z0-9_-]{0,62}$");
+  const declaredBuild = flags["build-id"] ?? null;
+  if (declaredBuild !== null && !/^bld_[a-z0-9][a-z0-9_-]{0,62}$/.test(declaredBuild)) throw usage("--build-id must match ^bld_[a-z0-9][a-z0-9_-]{0,62}$");
+  const inspected = await inspectFile(staged.abs, { maxParseBytes: maxParse });
+  const investigation = await readInvestigationMaybe(ws);
+
+  // Classify by content and caller-reported outcome.
+  let kind;
+  let status;
+  const notes = [];
+  if (interrupted) {
+    kind = "interrupted-output";
+    status = "interrupted";
+  } else if (inspected.kind_hint && (exitCode === null || exitCode === 0)) {
+    kind = inspected.kind_hint;
+    status = "complete";
+  } else {
+    kind = "diagnostic";
+    status =
+      (exitCode !== null && exitCode !== 0) ||
+      inspected.content_type === "rea-error" ||
+      inspected.id_authentication.status === "failed" ||
+      inspected.structure === "incomplete" ||
+      inspected.jsonl_ok === false ||
+      inspected.content_type === "empty"
+        ? "error"
+        : "complete";
+    if (inspected.kind_hint && exitCode) notes.push(`valid REA output saved as diagnostic because exit code was ${exitCode}`);
+    if (!inspected.kind_hint && (inspected.content_type === "rea-record" || inspected.content_type === "rea-bundle")) {
+      notes.push(`not publishable as canonical evidence: ${inspected.reasons.join(", ") || inspected.id_authentication.reason}`);
+    }
+  }
+  const canonical = kind === "canonical-record" || kind === "canonical-bundle" || kind === "derived-view";
+  const target = canonical
+    ? targetFor(investigation, inspected.binding, declaredBuild)
+    : { association: "unbound", build_id: null, binding_source: "none", binding_digests: [], identity_state: null };
+  if (!canonical && declaredBuild) notes.push("--build-id ignored for non-canonical output");
+  if (canonical && target.association === "unbound" && inspected.binding.digests.length > 0) {
+    notes.push("record target digest matches no build in investigation.json; it is unbound until the build exists");
+  }
+
+  // The stderr companion is its own entry whose identity is its bytes plus the
+  // companion role; it is shared by every publication with equal stderr bytes
+  // and never collides with a main output that happens to have the same bytes.
+  let companion = null;
+  if (flags["stderr-file"]) {
+    const support = await stagedSource(ws, flags["stderr-file"], "stderr-file");
+    const supportInspected = await inspectFile(support.abs, { shallow: true });
+    companion = { support, ...identify(companionDraft(supportInspected), { dir: kindDir("diagnostic"), slug: "stderr", ext: "txt" }) };
+  }
+
+  const slug = slugify(flags.name ?? inspected.operation ?? (kind === "diagnostic" ? "diagnostic" : "output"));
+  const main = identify(
+    {
+      kind,
+      sha256: inspected.sha256,
+      bytes: inspected.bytes,
+      status,
+      content_type: inspected.content_type,
+      producer: producerFor(inspected, flags),
+      operation: inspected.operation,
+      predicate_type: inspected.predicate_type,
+      records: canonical ? inspected.records : [],
+      records_truncated: canonical ? inspected.records_truncated : false,
+      id_authentication: canonical ? inspected.id_authentication : { status: "not-applicable", reason: null },
+      invocation: {
+        source,
+        command,
+        exit_code: exitCode,
+        stderr_file: companion?.entry.file ?? null,
+        stderr_sha256: companion?.entry.sha256 ?? null,
+        step_id: stepId,
+      },
+      target,
+      external: null,
+      note: [flags.note, ...notes].filter(Boolean).join("; "),
+    },
+    { dir: kindDir(kind), slug, ext: extensionFor(inspected.content_type) },
+  );
+
+  const result = await withLock(path.join(ws.abs, "evidence", ".index.lock"), async () => {
+    const index = await readIndexFile(ws);
+    // stderr is always preserved, even when the main output turns out to be an exact retry.
+    const stderr = companion ? await ensureEntry(ws, index, companion.entry, companion.digest, companion.support.abs) : null;
+    const published = await ensureEntry(ws, index, main.entry, main.digest, staged.abs);
+    if (stderr?.created || published.created) await writeIndexFile(ws, index);
+    if (staged.inStaging) await fsp.rm(staged.abs, { force: true });
+    if (companion?.support.inStaging) await fsp.rm(companion.support.abs, { force: true });
+    return { already_published: !published.created, entry: published.entry, stderr_entry: stderr?.entry ?? null, stderr_reused: stderr ? !stderr.created : null };
+  });
+  return {
+    ...result,
+    staged_retained: !staged.inStaging,
+    summary: {
+      entry_id: result.entry.entry_id,
+      kind: result.entry.kind,
+      status: result.entry.status,
+      file: result.entry.file,
+      sha256: result.entry.sha256,
+      evidence_ids: result.entry.records.map((record) => record.evidence_id),
+      binding_digests: result.entry.target.binding_digests,
+      association: result.entry.target.association,
+      build_id: result.entry.target.build_id,
+      id_authentication: result.entry.id_authentication,
+      stderr_entry_id: result.stderr_entry?.entry_id ?? null,
+    },
+  };
+}
+
+// --- register-external ------------------------------------------------------------
+
+export async function registerExternal(flags) {
+  const ws = await openWorkspace(flags.workspace);
+  const staged = await stagedSource(ws, flags.staged, "staged");
+  if (!flags.metadata) throw usage("--metadata META.json is required");
+  let meta;
+  try {
+    meta = await readJson(flags.metadata);
+  } catch (error) {
+    if (error instanceof SyntaxError) throw usage(`--metadata is not valid JSON: ${error.message}`);
+    throw error;
+  }
+  if (meta === null || typeof meta !== "object" || Array.isArray(meta)) throw usage("--metadata must be a JSON object");
+  const inspected = await inspectFile(staged.abs, { shallow: true });
+  const investigation = await readInvestigationMaybe(ws);
+  const stepId = flags["step-id"] ?? null;
+  if (stepId !== null && !/^step_[a-z0-9][a-z0-9_-]{0,62}$/.test(stepId)) throw usage("--step-id must match ^step_[a-z0-9][a-z0-9_-]{0,62}$");
+  const slug = slugify(flags.name ?? meta.observation_kind ?? "external");
+  const originalName = path.basename(staged.abs);
+  const ext = path.extname(originalName).replace(/[^.a-zA-Z0-9]/g, "").replace(/^\.+/, "").slice(0, 10).toLowerCase() || "bin";
+  const external = {
+    ext_id: `ext_${inspected.sha256}`,
+    file: { path: "evidence/external/pending", sha256: inspected.sha256, bytes: inspected.bytes },
+    media_type: meta.media_type ?? "application/octet-stream",
+    observation_kind: meta.observation_kind,
+    description: meta.description,
+    acquisition: meta.acquisition,
+    sensitivity: meta.sensitivity,
+    target_association: meta.target_association,
+    limitations: meta.limitations ?? [],
+    registered_at: nowIso(),
+  };
+  const check = await validateSchema("external-evidence", external);
+  if (!check.valid) throw new CliError("metadata does not form a valid external-evidence record", { code: "schema_invalid", details: check.errors });
+  const association = external.target_association;
+  if (association.status === "declared") {
+    if (!association.build_id || !(investigation?.builds ?? []).some((b) => b.build_id === association.build_id)) {
+      throw new CliError("target_association.build_id must name a build in investigation.json", { code: "unknown_build" });
+    }
+  } else if (association.build_id !== null) {
+    throw new CliError("target_association.build_id must be null unless status is declared", { code: "schema_invalid" });
+  }
+  // Same bytes with different provenance/metadata are different publications:
+  // ext_id stays bytes-based, the entry and file carry the provenance.
+  const registration = identify(
+    {
+      kind: "external",
+      sha256: inspected.sha256,
+      bytes: inspected.bytes,
+      status: "complete",
+      content_type: ["binary", "empty", "text", "json", "jsonl"].includes(inspected.content_type) ? inspected.content_type : "json",
+      producer: { name: null, version: null, provider: null },
+      operation: null,
+      predicate_type: null,
+      records: [],
+      records_truncated: false,
+      id_authentication: { status: "not-applicable", reason: null },
+      invocation: { source: "external", command: null, exit_code: null, stderr_file: null, stderr_sha256: null, step_id: stepId },
+      target: {
+        association: association.status === "declared" ? "declared" : "unbound",
+        build_id: association.build_id,
+        binding_source: "none",
+        binding_digests: [],
+        identity_state: null,
+      },
+      external,
+      note: flags.note ?? "",
+    },
+    { dir: kindDir("external"), slug, ext },
+  );
+  const result = await withLock(path.join(ws.abs, "evidence", ".index.lock"), async () => {
+    const index = await readIndexFile(ws);
+    const registered = await ensureEntry(ws, index, registration.entry, registration.digest, staged.abs);
+    if (registered.created) await writeIndexFile(ws, index);
+    if (staged.inStaging) await fsp.rm(staged.abs, { force: true });
+    return { already_registered: !registered.created, entry: registered.entry };
+  });
+  return { ...result, staged_retained: !staged.inStaging, summary: { ext_id: external.ext_id, entry_id: result.entry.entry_id, file: result.entry.file, sha256: inspected.sha256 } };
+}
+
+// --- digests ------------------------------------------------------------------------
+
+export async function walkFiles(rootAbs, relBase, { skipDirs = [] } = {}) {
+  const out = [];
+  const visit = async (abs, rel) => {
+    let entries;
+    try {
+      entries = await fsp.readdir(abs, { withFileTypes: true });
+    } catch (error) {
+      if (error.code === "ENOENT") return;
+      throw error;
+    }
+    entries.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
+    for (const entry of entries) {
+      const childRel = rel ? `${rel}/${entry.name}` : entry.name;
+      if (skipDirs.includes(childRel)) continue;
+      if (entry.isSymbolicLink()) out.push({ rel: childRel, symlink: true });
+      else if (entry.isDirectory()) await visit(path.join(abs, entry.name), childRel);
+      else if (entry.isFile()) out.push({ rel: childRel, symlink: false });
+    }
+  };
+  await visit(path.join(rootAbs, relBase), relBase);
+  return out;
+}
+
+export async function digests(flags, { paths = [] } = {}) {
+  const ws = await openWorkspace(flags.workspace);
+  const wanted = paths.length
+    ? paths
+    : ["investigation.json", "inventory.json", INDEX_PATH, "claims.json", "review.json"];
+  const files = [];
+  const addFile = async (rel) => {
+    const resolved = await resolveInWorkspace(ws.abs, rel);
+    if (!resolved.exists) return files.push({ path: rel, exists: false, sha256: null, bytes: null });
+    if (!resolved.stat.isFile()) throw new CliError(`not a regular file: ${rel}`, { code: "not_regular_file" });
+    const { sha256, bytes } = await hashFile(resolved.abs);
+    files.push({ path: rel, exists: true, sha256, bytes });
+  };
+  for (const rel of wanted) await addFile(rel);
+  let specs = [];
+  if (!paths.length) {
+    const specFiles = (await walkFiles(ws.abs, "specs")).filter((f) => !f.symlink && f.rel.endsWith(".md"));
+    for (const file of specFiles) await addFile(file.rel);
+    specs = specFiles.map((f) => f.rel);
+  }
+  const byPath = Object.fromEntries(files.map((f) => [f.path, f]));
+  const digestOf = (rel) => (byPath[rel]?.exists ? { path: rel, sha256: byPath[rel].sha256 } : null);
+  return {
+    files,
+    reviewed_inputs_template: {
+      inventory: digestOf("inventory.json"),
+      evidence_index: digestOf(INDEX_PATH),
+      claims: digestOf("claims.json"),
+      specs: specs.map(digestOf),
+      scope: await workspaceScopeDigest(ws.abs),
+    },
+    claims_inputs_template: { inventory: digestOf("inventory.json"), evidence_index: digestOf(INDEX_PATH) },
+  };
+}
+
+// --- reconcile ------------------------------------------------------------------------
+
+export async function reconcile(flags) {
+  const ws = await openWorkspace(flags.workspace);
+  const index = await readIndexFile(ws);
+  const findings = [];
+  const indexed = new Map(index.entries.map((entry) => [entry.file, entry]));
+  for (const entry of index.entries) {
+    const resolved = await resolveInWorkspace(ws.abs, entry.file).catch((error) => ({ error }));
+    if (resolved.error) {
+      findings.push({ severity: "error", code: resolved.error.code ?? "unsafe_path", file: entry.file, message: resolved.error.message });
+      continue;
+    }
+    if (!resolved.exists) {
+      findings.push({ severity: "error", code: "index_target_missing", file: entry.file, entry_id: entry.entry_id, message: "indexed file is missing" });
+      continue;
+    }
+    if (flags["no-hash"] !== true) {
+      const live = await hashFile(resolved.abs);
+      if (live.sha256 !== entry.sha256 || live.bytes !== entry.bytes) {
+        findings.push({ severity: "error", code: "index_hash_mismatch", file: entry.file, entry_id: entry.entry_id, message: "file content differs from the indexed digest" });
+      }
+    }
+  }
+  for (const dir of EVIDENCE_DIRS) {
+    for (const file of await walkFiles(ws.abs, dir)) {
+      if (file.symlink) findings.push({ severity: "error", code: "symlink_refused", file: file.rel, message: "symbolic link inside evidence" });
+      else if (!indexed.has(file.rel)) {
+        findings.push({ severity: "warning", code: "orphan_evidence_file", file: file.rel, message: "published file has no index entry; not adopted. Re-run publish-evidence with a staged copy only after review" });
+      }
+    }
+  }
+  for (const file of await walkFiles(ws.abs, "evidence/.staging")) {
+    findings.push({ severity: "warning", code: "staged_file", file: file.rel, message: "unpublished staged output (possible interrupted run); inspect before publishing or discarding" });
+  }
+  const lock = await resolveInWorkspace(ws.abs, "evidence/.index.lock");
+  if (lock.exists) findings.push({ severity: "warning", code: "stale_index_lock", file: "evidence/.index.lock", message: "index lock present; confirm no helper is running before removing" });
+  const everything = await walkFiles(ws.abs, "", { skipDirs: ["evidence/.staging"] });
+  for (const file of everything) {
+    if (/(^|\/)\.[^/]*\.tmp-\d+-[0-9a-f]{8}$/.test(file.rel)) {
+      findings.push({ severity: "warning", code: "interrupted_write_temp", file: file.rel, message: "leftover temporary file from an interrupted atomic write" });
+    }
+  }
+  const failed = findings.some((f) => f.severity === "error");
+  return { outcome: failed ? "fail" : "pass", findings, entries: index.entries.length };
+}
+
+// --- authenticated entries ------------------------------------------------------------
+
+/**
+ * Re-derive everything an index entry claims about a canonical file from the bytes on disk:
+ * entry id and file prefix from the semantic digest, SHA-256/size, content classification,
+ * semantic evidence_id authentication of every record, record summaries and the intrinsic
+ * target digests. The index is a locator, never the authority.
+ */
+async function authenticateCanonicalEntry(ws, entry, maxParseBytes) {
+  const fail = (code, message) => new CliError(`${entry.entry_id}: ${message}`, { code });
+  const short = semanticDigest(entry).slice(0, 16);
+  if (entry.entry_id !== `ei_${short}` || !path.posix.basename(entry.file).startsWith(`${short}-`)) {
+    throw fail("entry_identity_mismatch", "entry_id/file do not derive from the entry's semantic content; the index was edited");
+  }
+  const resolved = await resolveInWorkspace(ws.abs, entry.file, { allowMissing: false });
+  const live = await inspectFile(resolved.abs, { maxParseBytes });
+  if (live.sha256 !== entry.sha256 || live.bytes !== entry.bytes) {
+    throw fail("evidence_modified", `${entry.file} no longer matches its indexed sha256/bytes; the saved evidence was modified`);
+  }
+  if (live.kind_hint !== entry.kind) {
+    throw fail("entry_unauthenticated", `saved content classifies as ${live.kind_hint ?? live.content_type}, not ${entry.kind}${live.reasons.length ? ` (${live.reasons.join(", ")})` : ""}`);
+  }
+  if (live.id_authentication.status !== "verified") {
+    throw fail(
+      "entry_unauthenticated",
+      `evidence_id authentication is ${live.id_authentication.status}: ${live.id_authentication.reason}${live.parse === "header-only" ? `; raise --max-parse-bytes above ${live.bytes}` : ""}`,
+    );
+  }
+  const sameRecords =
+    live.records_truncated ||
+    (live.records.length === entry.records.length &&
+      live.records.every((record, i) => {
+        const other = entry.records[i];
+        return (
+          other &&
+          record.evidence_id === other.evidence_id &&
+          record.pointer === other.pointer &&
+          record.operation === other.operation &&
+          record.confidence === other.confidence &&
+          record.authority === other.authority &&
+          JSON.stringify(record.binding_digests) === JSON.stringify(other.binding_digests) &&
+          record.parent_evidence_id === other.parent_evidence_id
+        );
+      }));
+  if (!sameRecords) throw fail("entry_binding_mismatch", "index record summaries differ from the saved records");
+  const liveDigests = [...live.binding.digests].sort();
+  const indexDigests = [...entry.target.binding_digests].sort();
+  if (JSON.stringify(liveDigests) !== JSON.stringify(indexDigests) || live.binding.source !== entry.target.binding_source) {
+    throw fail("entry_binding_mismatch", "index target digests differ from the saved record's intrinsic target digests");
+  }
+  return { abs: resolved.abs, live };
+}
+
+// --- describe-build ---------------------------------------------------------------------
+
+export async function describeBuild(flags) {
+  const buildId = flags["build-id"];
+  if (!buildId || !/^bld_[a-z0-9][a-z0-9_-]{0,62}$/.test(buildId)) throw usage("--build-id bld_<slug> is required");
+  if (flags.file) {
+    const abs = path.resolve(flags.file);
+    const stat = await fsp.stat(abs).catch(() => null);
+    if (!stat?.isFile()) throw new CliError("--file must be a regular file; for directories/apps use --entry from a published REA record", { code: "not_regular_file" });
+    const { sha256, bytes } = await hashFile(abs);
+    return {
+      build: {
+        build_id: buildId,
+        kind: flags.kind ?? "file",
+        name: flags.name ?? path.basename(abs),
+        identity: { method: "file-sha256", sha256, bytes, coverage: "complete", rea_format: null, note: "Full streamed SHA-256 of the file." },
+        local_path: abs,
+        parent_build_id: flags.parent ?? null,
+      },
+    };
+  }
+  if (flags.entry) {
+    const ws = await openWorkspace(flags.workspace);
+    const index = await readIndexFile(ws);
+    const entry = index.entries.find((e) => e.entry_id === flags.entry);
+    if (!entry) throw new CliError(`no such entry: ${flags.entry}`, { code: "unknown_entry" });
+    if (!["canonical-record", "canonical-bundle", "derived-view"].includes(entry.kind)) throw usage("--entry must be a canonical record");
+    const maxParse = toInt(flags["max-parse-bytes"], "max-parse-bytes", { min: 1 }) ?? DEFAULT_MAX_PARSE_BYTES;
+    const { abs, live } = await authenticateCanonicalEntry(ws, entry, maxParse);
+    const digests = [...new Set(live.binding.digests)];
+    if (digests.length !== 1) throw new CliError(`entry binds ${digests.length} distinct target digests; select a single-target record`, { code: "ambiguous_target" });
+    const scan = await scanFile(abs, ["/subject", "/records/*/subject"]);
+    const subjectText = [...scan.captures.values()].map((c) => c.text).find((t) => t && t !== "null");
+    const subject = subjectText ? JSON.parse(subjectText) : null;
+    if (!subject) {
+      throw new CliError("record has no subject (for example a process capture); describe the executable with --file instead", { code: "no_subject" });
+    }
+    const isDirectoryLike = subject.format === "directory";
+    let bytes = null;
+    let verifiedLocal = false;
+    if (!isDirectoryLike && typeof subject.local_path === "string") {
+      const stat = await fsp.stat(subject.local_path).catch(() => null);
+      if (stat?.isFile()) {
+        const live = await hashFile(subject.local_path);
+        if (live.sha256 !== digests[0]) {
+          throw new CliError("the file now at the recorded local_path no longer matches the record's subject digest", { code: "identity_changed" });
+        }
+        bytes = live.bytes;
+        verifiedLocal = true;
+      }
+    }
+    return {
+      build: {
+        build_id: buildId,
+        kind: flags.kind ?? (isDirectoryLike ? "directory" : "file"),
+        name: flags.name ?? subject.name ?? buildId,
+        identity: isDirectoryLike
+          ? {
+              method: "rea-subject-digest",
+              sha256: digests[0],
+              bytes: null,
+              coverage: "unknown",
+              rea_format: subject.format,
+              note: "Digest REA reported as the subject of the directory/bundle analysis. Manifest coverage is not established by this helper; set coverage from the record's own coverage fields and do not call it a whole-directory fingerprint unless complete.",
+            }
+          : {
+              method: "file-sha256",
+              sha256: digests[0],
+              bytes,
+              coverage: "complete",
+              rea_format: subject.format ?? null,
+              note: verifiedLocal ? "File SHA-256 reported by REA and re-verified against the local file." : "File SHA-256 reported by REA as the subject digest; local file was not re-hashed.",
+            },
+        local_path: typeof subject.local_path === "string" ? subject.local_path : null,
+        parent_build_id: flags.parent ?? null,
+      },
+    };
+  }
+  throw usage("describe-build needs --file PATH or --workspace D --entry ei_...");
+}