@agent-delivery-harness/cli 0.7.4 → 0.8.1

package/src/scoped-diagnostics.ts

@@ -5,7 +5,7 @@ import { CheckSnapshotError } from "./check-snapshot.ts";
 import { projectScopedAttempt } from "./scoped-observations.ts";
 
 const phases = ["snapshot-setup", "pre-command-verification", "command", "post-command-verification", "output-capture", "complete"] as const;
-const failureCodes = ["check_snapshot_interrupted", "check_snapshot_timeout", "check_snapshot_unavailable", "check_snapshot_escape", "check_snapshot_drift", "check_snapshot_cleanup_failed", "check_dependency_source_overlap", "check_dependency_failed", "check_command_failed", "check_output_missing", "check_attempt_superseded", "check_artifact_unavailable", "check_evidence_rejected"] as const;
+const failureCodes = ["check_snapshot_interrupted", "check_snapshot_timeout", "check_snapshot_unavailable", "check_snapshot_escape", "check_snapshot_drift", "check_snapshot_cleanup_failed", "check_dependency_source_overlap", "check_dependency_failed", "check_command_failed", "check_output_missing", "check_attempt_superseded", "check_artifact_unavailable", "check_evidence_rejected", "check_cancelled"] as const;
 const executionCodes = ["ENOENT", "EACCES", "EPERM", "ABORT_ERR", "ERR_CHILD_PROCESS_STDIO_MAXBUFFER", "SIGKILL", "SIGTERM", "execution_failed"] as const;
 export type ScopedDiagnosticPhase = typeof phases[number];
 export type ScopedDiagnosticFailureCode = typeof failureCodes[number];
@@ -18,6 +18,21 @@ export interface ScopedDependencyDiagnostic {
   readonly executionErrorCode?: ScopedDiagnosticExecutionCode;
   readonly command: ScopedDiagnosticCommand;
 }
+const setupPhases = ["materialization", "baseline-inventory", "dependencies", "post-dependency-inventory", "warm-base-wait", "clone", "pre-command-verification"] as const;
+/** How this attempt got the tree its command ran in, and what that cost up to the command.
+ * `executed`: this attempt ran the setup (a tree it built, or the warm base it then cloned);
+ * `reused`: the setup had already run (a clone of a warm base another check built, or the
+ * worker's tree retained from its previous check). `clone` names the copy-on-write method, or says
+ * why each worker built its own tree: cloning is unavailable here, or the setup wrote its own path
+ * into the tree (`path-bound`), or there is one worker (`not-used`). */
+export interface ScopedSetupDiagnostic {
+  readonly action: "executed" | "reused";
+  readonly snapshot: "built" | "cloned" | "retained";
+  readonly clone: "clonefile" | "reflink" | "unavailable" | "path-bound" | "not-used";
+  /** `index`: Git's index and untracked listing; `inventory`: every immutable byte hashed. */
+  readonly verification: "index" | "inventory";
+  readonly phases: readonly { readonly phase: typeof setupPhases[number]; readonly durationMs: number }[];
+}
 export interface RecordedScopedAttemptDiagnostic {
   readonly availability: "available";
   readonly phase: ScopedDiagnosticPhase;
@@ -26,6 +41,8 @@ export interface RecordedScopedAttemptDiagnostic {
   readonly command: ScopedDiagnosticCommand;
   /** Present only when this attempt actually ran dependency setup; never borrowed from a reused snapshot. */
   readonly dependency?: ScopedDependencyDiagnostic;
+  /** Present once this attempt began getting its tree, so a failed setup keeps its phases; absent on attempts recorded before setup was measured. */
+  readonly setup?: ScopedSetupDiagnostic;
 }
 export type ScopedAttemptDiagnostic = RecordedScopedAttemptDiagnostic
   | { readonly availability: "unavailable"; readonly reason: "legacy" | "running" };
@@ -43,8 +60,55 @@ export function scopedDiagnosticFailure(error: unknown, executionErrorCode?: str
   return { code: error.code as ScopedDiagnosticFailureCode,
     ...((executionCodes as readonly string[]).includes(executionErrorCode ?? "") ? { executionErrorCode: executionErrorCode as ScopedDiagnosticExecutionCode } : {}) };
 }
+/** Shortest credential prefix masked where later output interrupts it. Any prefix
+ * ending the text is masked; mid-text, a shorter run is indistinguishable from
+ * ordinary output sharing a credential's leading characters. */
+export const INTERRUPTED_CREDENTIAL_PREFIX = 8;
+/** Masks every occurrence of each credential, overlapping ones included, and the
+ * credential prefixes a writer that exits, is killed or is clipped mid-write
+ * leaves behind: at the end of the text, or interrupted by whatever writes next
+ * to the same stream (a wrapping shell's own output). Apply it to each captured
+ * stream before joining or bounding them. */
 export function redactScopedOutput(text: string, secrets: readonly string[]): string {
-  return [...secrets].filter(Boolean).sort((a, b) => b.length - a.length).reduce((value, secret) => value.split(secret).join("[REDACTED]"), text);
+  // Ascending start order makes a union of overlapping spans one append-or-extend step.
+  const union = (spans: [number, number][], [start, end]: [number, number]) => {
+    const last = spans.at(-1);
+    if (last && last[1] >= start) last[1] = Math.max(last[1], end); else spans.push([start, end]);
+    return spans;
+  };
+  // A byte clip inside a multi-byte character decodes to U+FFFD after the prefix it cut. Counted
+  // backwards: an anchored /\uFFFD+$/ backtracks quadratically over a run that does not end the text.
+  let tail = text.length;
+  while (tail > 0 && text.charCodeAt(tail - 1) === 0xfffd) tail--;
+  const spans: [number, number][] = [];
+  for (const secret of new Set(secrets.filter(Boolean))) {
+    // Knuth-Morris-Pratt: linear in the output however self-similar the credential.
+    const border = [0];
+    for (let i = 1, k = 0; i < secret.length; i++) {
+      while (k > 0 && secret.charCodeAt(i) !== secret.charCodeAt(k)) k = border[k - 1]!;
+      border.push(k += secret.charCodeAt(i) === secret.charCodeAt(k) ? 1 : 0);
+    }
+    // The candidate match's start never moves backwards, so spans arrive in ascending start order.
+    const own: [number, number][] = [];
+    let matched = 0, atTail = 0;
+    for (let i = 0; i < text.length; i++) {
+      if (i === tail) atTail = matched;
+      if (matched >= INTERRUPTED_CREDENTIAL_PREFIX && text.charCodeAt(i) !== secret.charCodeAt(matched)) union(own, [i - matched, i]);
+      while (matched > 0 && text.charCodeAt(i) !== secret.charCodeAt(matched)) matched = border[matched - 1]!;
+      if (text.charCodeAt(i) === secret.charCodeAt(matched)) matched++;
+      if (matched === secret.length) { union(own, [i + 1 - matched, i + 1]); matched = border[matched - 1]!; }
+    }
+    if (tail === text.length) atTail = matched;
+    for (const span of own) spans.push(span);
+    // The matcher's state at the tail is the longest credential prefix ending the text.
+    if (atTail > 0) spans.push([tail - atTail, text.length]);
+  }
+  const merged = spans.sort((a, b) => a[0] - b[0]).reduce(union, []);
+  return merged.map(([start], i) => text.slice(i ? merged[i - 1]![1] : 0, start) + "[REDACTED]").join("") + text.slice(merged.at(-1)?.[1] ?? 0);
+}
+/** Each stream is redacted on its own: one can end inside a credential the join would hide mid-text. */
+export function redactScopedStreams(result: Pick<ExecOutcome, "stdout" | "stderr">, secrets: readonly string[]): string {
+  return `${redactScopedOutput(result.stdout, secrets)}\n${redactScopedOutput(result.stderr, secrets)}`;
 }
 export function scopedCommandDiagnostic(result: ExecOutcome, secrets: readonly string[]): ScopedDiagnosticCommand {
   // execFile can clip a fully emitted credential before full-value redaction.
@@ -52,7 +116,7 @@ export function scopedCommandDiagnostic(result: ExecOutcome, secrets: readonly s
   if (result.errorCode === "ERR_CHILD_PROCESS_STDIO_MAXBUFFER") return { unavailable: "not-completed" };
   const numericExit = result.errorCode === undefined || result.errorCode === String(result.code);
   if (!numericExit && result.stdout.length + result.stderr.length === 0) return { unavailable: ["ENOENT", "EACCES", "EPERM"].includes(result.errorCode!) ? "not-started" : "not-completed" };
-  const redacted = redactScopedOutput(`${result.stdout}\n${result.stderr}`, secrets);
+  const redacted = redactScopedStreams(result, secrets);
   return { exitCode: numericExit ? result.code : null, outputTail: redacted.slice(-4000), truncated: redacted.length > 4000 };
 }
 
@@ -91,8 +155,22 @@ function projectDiagnostic(value: unknown): RecordedScopedAttemptDiagnostic {
     dependency = { durationMs: value["durationMs"], command: commandDiagnostic(value["command"]),
       ...("executionErrorCode" in value ? { executionErrorCode: value["executionErrorCode"] as ScopedDiagnosticExecutionCode } : {}) };
   }
+  let setup: ScopedSetupDiagnostic | undefined;
+  if ("setup" in row) {
+    const value = object(row["setup"]);
+    const one = <T extends string>(field: string, allowed: readonly T[]): T => (allowed as readonly unknown[]).includes(value[field]) ? value[field] as T : corrupt();
+    const phasesValue = value["phases"];
+    if (!Array.isArray(phasesValue) || phasesValue.length > 16) return corrupt();
+    setup = { action: one("action", ["executed", "reused"]), snapshot: one("snapshot", ["built", "cloned", "retained"]),
+      clone: one("clone", ["clonefile", "reflink", "unavailable", "path-bound", "not-used"]), verification: one("verification", ["index", "inventory"]),
+      phases: phasesValue.map(entry => {
+        const phase = object(entry);
+        if (!(setupPhases as readonly unknown[]).includes(phase["phase"]) || typeof phase["durationMs"] !== "number" || !Number.isSafeInteger(phase["durationMs"]) || phase["durationMs"] < 0) return corrupt();
+        return { phase: phase["phase"] as typeof setupPhases[number], durationMs: phase["durationMs"] };
+      }) };
+  }
   return { availability: "available", phase: row["phase"] as ScopedDiagnosticPhase, failure: safeFailure, command: commandDiagnostic(row["command"]),
-    ...(dependency === undefined ? {} : { dependency }) };
+    ...(dependency === undefined ? {} : { dependency }), ...(setup === undefined ? {} : { setup }) };
 }
 
 /** Read at most 100 explicit attempts. No logs from legacy payloads, no writes,

package/src/scoped-installed-data.ts

@@ -0,0 +1,8 @@
+/** CLI error projection over the shared immutable source binding. */
+import { captureInstalledDataBinding as capture, InstalledDataSourceError } from "@agent-delivery-harness/kernel";
+import { CheckSnapshotError } from "./check-snapshot.ts";
+export type { InstalledDataSourceTree } from "@agent-delivery-harness/kernel";
+export async function captureInstalledDataBinding(...args: Parameters<typeof capture>) {
+  try { return await capture(...args); }
+  catch (error) { if (error instanceof InstalledDataSourceError) throw new CheckSnapshotError(error.code, error.message); throw error; }
+}

package/src/snapshot-inventory-worker.ts

@@ -4,35 +4,177 @@ export const snapshotInventoryWorker = String.raw`
 import { lstat, readFile, readdir, readlink, realpath } from 'node:fs/promises';
 import { createHash } from 'node:crypto';
 import path from 'node:path';
-const { root, outputs, excludeDependencies, forbidGit } = JSON.parse(process.env.DELIVERY_SNAPSHOT_REQUEST);
+const { root, outputs, excludeDependencies, forbidGit, includeSource, absentInputs = [], selfReferences = [] } = JSON.parse(process.env.DELIVERY_SNAPSHOT_REQUEST);
+// A setup that writes this tree's own absolute path into a file binds the tree to where it is.
+const references = selfReferences.map(reference => Buffer.from(reference));
+let selfReferenced = false;
 const entries = [];
+const sourceEntries = [];
+const record = (relative, tuple) => {
+  entries.push(tuple);
+  if (includeSource && !relative.split('/').includes('node_modules')) sourceEntries.push(tuple);
+};
 const inside = target => {
   const relative = path.relative(root, target);
   return relative !== '..' && !relative.startsWith('..' + path.sep) && !path.isAbsolute(relative);
 };
 const escape = () => { throw { code: 'check_snapshot_escape' }; };
-const walk = async relative => {
-  if (relative === '.git' || (excludeDependencies && relative.split('/').includes('node_modules')) ||
-      outputs.some(o => o.endsWith('/') ? relative === o.slice(0, -1) || relative.startsWith(o) : relative === o)) return;
-  const absolute = path.join(root, relative), stat = await lstat(absolute);
-  if (stat.isSymbolicLink()) {
-    const target = await readlink(absolute);
-    if (path.isAbsolute(target) || !inside(path.resolve(path.dirname(absolute), target))) escape();
-    try { if (!inside(await realpath(absolute))) escape(); }
-    catch (error) { if (error.code !== 'ENOENT') throw error; }
-    entries.push([relative, 'link', target]);
-  } else if (stat.isDirectory()) {
-    for (const name of (await readdir(absolute)).sort()) await walk(relative ? relative + '/' + name : name);
-  } else if (stat.isFile()) {
-    entries.push([relative, stat.mode & 0o111, createHash('sha256').update(await readFile(absolute)).digest('hex')]);
-  } else escape();
+// Eight workers bound filesystem operations, not just initial directory reads.
+// Outstanding includes running directories: an empty queue is not completion
+// while one of them may still discover children. Every started visit settles
+// before an error is reported and the owner may clean up the private tree.
+const drainQueue = async (queue, visit) => {
+  let cursor = 0, outstanding = queue.length, failed = false, failure;
+  const waiting = [];
+  const wake = () => { for (const resume of waiting.splice(0)) resume(); };
+  const worker = async () => {
+    while (true) {
+      while (!failed && cursor === queue.length && outstanding > 0) await new Promise(resolve => waiting.push(resolve));
+      if (failed || outstanding === 0) return;
+      const slot = queue[cursor++];
+      try {
+        const children = await visit(slot);
+        if (!failed && children?.length) {
+          outstanding += children.length;
+          for (const child of children) queue.push(child);
+        }
+      } catch (error) { if (!failed) { failed = true; failure = error; } }
+      finally { outstanding--; wake(); }
+    }
+  };
+  await Promise.all(Array.from({ length: 8 }, worker));
+  if (failed) throw failure;
+};
+const boundedMap = async (values, visit) => {
+  const slots = values.map(value => ({ value }));
+  await drainQueue([...slots], async slot => { slot.result = await visit(slot.value); });
+  return slots.map(slot => slot.result);
+};
+
+// Existing directory coordinates come only from the actual walk, with exact
+// bigint filesystem identity. Requested spelling and realpath are not canonical.
+const directories = new Map();
+const identity = stat => stat.dev.toString() + ':' + stat.ino.toString();
+const resolveMissing = async target => {
+  if (!inside(target)) escape();
+  let resolved = root, key = root, links = 0;
+  const pending = path.relative(root, target).split(path.sep);
+  while (pending.length) {
+    const segment = pending.shift();
+    if (!segment || segment === '.') continue;
+    if (segment === '..') { resolved = path.dirname(resolved); key = path.dirname(key); }
+    else {
+      const next = path.join(resolved, segment);
+      const info = await lstat(next, { bigint: true }).catch(error => { if (error.code === 'ENOENT') return null; throw error; });
+      if (info?.isSymbolicLink()) {
+        if (++links > 32) escape();
+        const link = await readlink(next);
+        if (path.isAbsolute(link)) escape();
+        pending.unshift(...link.split(path.sep));
+      } else if (info?.isDirectory()) {
+        resolved = directories.get(identity(info));
+        if (resolved === undefined) escape();
+        key = resolved;
+      } else {
+        resolved = next;
+        // No object exists to establish Unicode equivalence. The finite port
+        // explicitly refuses non-ASCII missing suffixes rather than guessing a
+        // filesystem folding table. Existing distinct directories stay distinct.
+        if (info === null && /[^\x00-\x7f]/.test(segment)) throw { code: 'check_snapshot_drift' };
+        key = info === null ? path.join(key, segment.toLowerCase()) : resolved;
+      }
+    }
+    if (!inside(resolved)) escape();
+  }
+  return { absolute: resolved, key };
+};
+// Retain the already-walked links so output authority can follow directory
+// referents without recursively following links or weakening inventory digests.
+const aliases = [];
+let protectedNames;
+const protectOutputAlias = resolved => {
+  const name = path.relative(root, resolved.key);
+  for (const a of protectedNames) {
+    if (name === '' || name === a || name.startsWith(a + '/') || a.startsWith(name + '/')) throw { code: 'check_snapshot_drift' };
+  }
+};
+const protectOutputReachability = async () => {
+  protectedNames = await boundedMap(absentInputs, async absent => path.relative(root, (await resolveMissing(path.join(root, absent))).key));
+  const reachable = [];
+  for (const output of outputs) reachable.push(await resolveMissing(path.join(root, output.replace(/\/$/, ''))));
+  for (const target of reachable) protectOutputAlias(target);
+  // Each collected link is expanded at most once. Its fully resolved target
+  // is writable whenever its lexical location is below an output referent.
+  const pending = new Set(aliases);
+  for (let index = 0; index < reachable.length; index++) {
+    const directory = reachable[index].absolute;
+    for (const alias of pending) {
+      const relative = path.relative(directory, alias.absolute);
+      if (relative === '..' || relative.startsWith('..' + path.sep) || path.isAbsolute(relative)) continue;
+      pending.delete(alias);
+      const resolved = await resolveMissing(alias.absolute);
+      protectOutputAlias(resolved);
+      reachable.push(resolved);
+    }
+  }
+};
+const walk = async () => {
+  const tree = { relative: '', mutable: false };
+  await drainQueue([tree], async slot => {
+    const { relative } = slot;
+    if (relative === '.git' || (excludeDependencies && relative.split('/').includes('node_modules'))) return;
+    // Mutable paths are excluded from the digest, never from the link/type walk.
+    // Carry the exclusion through directories, including an exact output path.
+    const mutable = slot.mutable || outputs.some(o => o.endsWith('/') ? relative === o.slice(0, -1) || relative.startsWith(o) : relative === o);
+    const absolute = path.join(root, relative), stat = await lstat(absolute, { bigint: true });
+    if (stat.isSymbolicLink()) {
+      const target = await readlink(absolute);
+      if (path.isAbsolute(target) || !inside(path.resolve(path.dirname(absolute), target))) escape();
+      try { if (!inside(await realpath(absolute))) escape(); }
+      catch (error) { if (error.code !== 'ENOENT') throw error; }
+      if (absentInputs.length) slot.alias = { absolute };
+      if (!mutable) slot.tuple = [relative, 'link', target];
+    } else if (stat.isDirectory()) {
+      slot.directory = { identity: identity(stat), absolute };
+      slot.children = (await readdir(absolute)).sort().map(name => ({ relative: relative ? relative + '/' + name : name, mutable }));
+      return slot.children;
+    } else if (stat.isFile()) {
+      if (!mutable) {
+        const bytes = await readFile(absolute);
+        if (!selfReferenced && references.some(reference => bytes.includes(reference))) selfReferenced = true;
+        slot.tuple = [relative, Number(stat.mode & 0o111n), createHash('sha256').update(bytes).digest('hex')];
+      }
+    } else escape();
+  });
+  // Completion order grants no directory authority and cannot change digest
+  // bytes. Flatten exact sorted-child DFS, including a/child before a.c;
+  // globally sorting relative paths would silently change the old identity.
+  const ordered = [tree];
+  while (ordered.length) {
+    const slot = ordered.pop();
+    if (slot.directory) directories.set(slot.directory.identity, slot.directory.absolute);
+    if (slot.alias) aliases.push(slot.alias);
+    if (slot.tuple) record(slot.relative, slot.tuple);
+    if (slot.children) for (let index = slot.children.length - 1; index >= 0; index--) ordered.push(slot.children[index]);
+  }
 };
 try {
+  for (const absent of absentInputs) {
+    let relative = '';
+    for (const part of absent.split('/')) {
+      relative = relative ? relative + '/' + part : part;
+      const info = await lstat(path.join(root, relative)).catch(error => { if (error.code === 'ENOENT') return null; throw error; });
+      if (info === null) break;
+      if (info.isSymbolicLink() || relative === absent || !info.isDirectory()) throw { code: 'check_snapshot_drift' };
+    }
+  }
   if (forbidGit && await lstat(path.join(root, '.git')).then(() => true, error => { if (error.code === 'ENOENT') return false; throw error; })) throw { code: 'check_snapshot_drift' };
-  await walk('');
+  await walk();
+  if (absentInputs.length) await protectOutputReachability();
   // Tuples contain only arrays, strings and permission integers: JSON.stringify
   // has exactly the same bytes as canonical JSON for this restricted shape.
-  process.stdout.write(JSON.stringify({ digest: createHash('sha256').update(JSON.stringify(entries)).digest('hex') }));
+  const digest = tuples => createHash('sha256').update(JSON.stringify(tuples)).digest('hex');
+  process.stdout.write(JSON.stringify({ digest: digest(entries), ...(includeSource ? { sourceDigest: digest(sourceEntries) } : {}), ...(references.length ? { selfReferenced } : {}) }));
 } catch (error) {
   process.stdout.write(JSON.stringify({ code: ['check_snapshot_escape', 'check_snapshot_drift'].includes(error.code) ? error.code : 'check_snapshot_unavailable' }));
   process.exitCode = 1;