@staix/agent-research 0.0.0-stage → 0.2.1

package/src/console/registry.ts

@@ -0,0 +1,97 @@
+import { chmodSync, closeSync, existsSync, fstatSync, lstatSync, mkdirSync, openSync, realpathSync, unlinkSync } from "node:fs";
+import { dirname, isAbsolute, join, resolve } from "node:path";
+import { assertWritable, openStudy, plain, readTextBounded, writeAtomic } from "../study.ts";
+
+export interface StudyRegistry { v: 1; studies: string[] }
+export const STUDY_REGISTRY_BYTES = 1024 * 1024, MAX_REGISTERED_STUDIES = 200;
+function fail(message: string): never { throw new Error(`study registry: ${message}; restore the bounded v1 registry or correct AGENTHUB_HOME and retry ares console add/remove/list`); }
+function absolute(value: string, label: string): string {
+  if (!isAbsolute(value) || value.includes("\0")) fail(`${label} must be an absolute path`);
+  return resolve(value);
+}
+function paths(home: string) {
+  const base = absolute(home, "family home"), dir = join(base, "research"), file = join(dir, "studies.json");
+  return { base, dir, file, lock: `${file}.lock` };
+}
+/** Root-owned system ancestors are allowed; the mutable family/registry paths must belong to this user. */
+function guard(home: string, file: string): void {
+  const uid = process.getuid!(), { base, dir } = paths(home), ancestors: string[] = [];
+  for (let current = resolve(file); ; current = dirname(current)) { ancestors.unshift(current); if (current === dirname(current)) break; }
+  for (const current of ancestors) {
+    let info;
+    try { info = lstatSync(current); } catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") continue; throw error; }
+    if (info.isSymbolicLink()) fail("links are refused in registry paths");
+    if (current === file) {
+      if (!info.isFile()) fail("registry or lock is not a regular file");
+      if (info.uid !== uid) fail("registry or lock is owned by another user");
+    } else {
+      if (!info.isDirectory()) fail("registry parent is not a directory");
+      if (info.uid !== uid && (current === base || current === dir || info.uid !== 0)) fail("registry parent is owned by another user");
+    }
+  }
+}
+function validate(raw: unknown): StudyRegistry {
+  if (!plain(raw) || raw.v !== 1) fail("unsupported registry version or malformed object; keep studies.json unchanged and use a compatible ares version");
+  if (!Array.isArray(raw.studies) || raw.studies.length > MAX_REGISTERED_STUDIES) fail("studies must be an array of at most 200 directories");
+  if (raw.studies.some(value => typeof value !== "string" || !isAbsolute(value) || value.includes("\0") || value !== resolve(value))) fail("stored study directories must be absolute normalized paths");
+  if (new Set(raw.studies).size !== raw.studies.length) fail("duplicate study directory in registry");
+  return { ...raw, v: 1, studies: [...raw.studies] } as StudyRegistry;
+}
+export function readRegistry(home: string): StudyRegistry {
+  const { file } = paths(home); guard(home, file);
+  let content: string;
+  try { content = readTextBounded(file, STUDY_REGISTRY_BYTES, { noFollow: true, uid: process.getuid!() }); }
+  catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return { v: 1, studies: [] }; fail("studies.json is unreadable, non-regular or over 1 MiB"); }
+  let raw: unknown; try { raw = JSON.parse(content); } catch { fail("studies.json is not valid JSON"); }
+  return validate(raw);
+}
+function update(home: string, change: (current: StudyRegistry) => StudyRegistry): StudyRegistry {
+  const { dir, file, lock } = paths(home); guard(home, file); guard(home, lock);
+  const busy = () => fail(`registry is locked at ${lock}; wait for the active registry command, or remove this stale lock only after confirming no registry command is running`);
+  if (existsSync(lock)) busy();
+  mkdirSync(dir, { recursive: true, mode: 0o700 }); guard(home, file); chmodSync(dir, 0o700);
+  let fd: number;
+  try { fd = openSync(lock, "wx", 0o600); } catch (error) { if ((error as NodeJS.ErrnoException).code === "EEXIST") busy(); throw error; }
+  let owner: { dev: number; ino: number } | undefined;
+  try {
+    // Capture the created path before descriptor inspection so a failed fstat still has guarded cleanup.
+    try { owner = lstatSync(lock); } catch (error) {
+      try { owner = fstatSync(fd); } catch { /* Unknown identity must leave a stale lock for manual recovery. */ }
+      throw error;
+    }
+    owner = fstatSync(fd);
+    const current = readRegistry(home), next = change(current);
+    if (next === current) return current;
+    const content = JSON.stringify(next, null, 2) + "\n"; assertWritable(content, STUDY_REGISTRY_BYTES, "study registry");
+    guard(home, file); writeAtomic(file, content, STUDY_REGISTRY_BYTES, 0o600); return next;
+  } finally {
+    try { closeSync(fd); } finally {
+      if (owner) {
+        try { const current = lstatSync(lock); if (current.dev === owner.dev && current.ino === owner.ino) unlinkSync(lock); }
+        catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; }
+      }
+    }
+  }
+}
+function canonical(dir: string, mustExist: boolean): string {
+  if (dir.includes("\0")) fail("study directory contains NUL");
+  const full = resolve(dir);
+  try { return realpathSync(full); } catch (error) { if (!mustExist && (error as NodeJS.ErrnoException).code === "ENOENT") return full; fail("study directory is unreadable; select a readable study for add or use its listed absolute directory for remove"); }
+}
+export function addStudy(home: string, dir: string): StudyRegistry {
+  const target = canonical(dir, true); openStudy(target);
+  const current = readRegistry(home); if (current.studies.includes(target)) return current;
+  if (current.studies.length >= MAX_REGISTERED_STUDIES) fail("200-study limit reached; remove an entry before adding another study");
+  return update(home, current => {
+    if (current.studies.includes(target)) return current;
+    if (current.studies.length >= MAX_REGISTERED_STUDIES) fail("200-study limit reached; remove an entry before adding another study");
+    return { ...current, studies: [...current.studies, target] };
+  });
+}
+export function removeStudy(home: string, dir: string): StudyRegistry {
+  const listed = resolve(dir), current = readRegistry(home);
+  // A moved or unreadable registered directory remains removable by its exact listed path.
+  const target = current.studies.includes(listed) ? listed : canonical(dir, false);
+  if (!current.studies.includes(target)) return current;
+  return update(home, current => current.studies.includes(target) ? { ...current, studies: current.studies.filter(value => value !== target) } : current);
+}

package/src/copilot.ts

@@ -0,0 +1,91 @@
+import { lstatSync, opendirSync } from "node:fs";
+import { join } from "node:path";
+import { ACCEPT_ARTIFACTS, resolveAutonomy, type AcceptArtifact } from "./autonomy.ts";
+import { canonical } from "./plan/state.ts";
+import { gateState, openStudy, readBytesBounded, readLedger, readTextBounded, sha256 } from "./study.ts";
+
+export const COPILOT_HYPOTHESES_BYTES = 2 * 1024 * 1024;
+export const COPILOT_PAPER_BYTES = 64 * 1024 * 1024;
+export const COPILOT_PAPER_FILES = 500, COPILOT_PAPER_DIRECTORIES = 1000, COPILOT_PAPER_DEPTH = 64;
+export class CopilotAcceptError extends Error {}
+class DraftReadError extends CopilotAcceptError {}
+function refuse(artifact: AcceptArtifact, reason: string): never {
+  throw new DraftReadError(`copilot draft ${artifact}: ${reason}; restore the last valid bounded regular draft files and ledger, then run ares gate list and ares gate approve <dir> accept:${artifact} before retrying`);
+}
+function draftLedger(dir: string, artifact: AcceptArtifact) {
+  try { return readLedger(dir); }
+  catch (error) {
+    if (error instanceof Error && /^ledger\.jsonl line [0-9]+: unsupported schema version/.test(error.message)) throw new DraftReadError(`copilot draft ${artifact}: unsupported ledger version; keep its bytes unchanged and use a compatible ares version before ares gate approve <dir> accept:${artifact}`);
+    refuse(artifact, "acceptance ledger is unreadable, malformed or over-bound");
+  }
+}
+function metadata(file: string) {
+  try { return lstatSync(file); }
+  catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined; throw error; }
+}
+function paperTree(dir: string): [string, string][] {
+  const root = join(dir, "paper"), entries: [string, string][] = [];
+  const rootStat = metadata(root);
+  if (rootStat === undefined) return entries;
+  if (rootStat.isSymbolicLink() || !rootStat.isDirectory()) refuse("paper", "paper source root must be a directory without symbolic links");
+  const derived = metadata(join(root, "build"));
+  let bytes = 0, directories = 0;
+  function walk(relative: string, depth: number): void {
+    // paper/build is derived output; manuscript sources include all other notes and assets.
+    if (relative === "build") return;
+    const file = relative ? join(root, relative) : root, stat = lstatSync(file);
+    // Respect the filesystem's case aliases without excluding a distinct source directory on case-sensitive hosts.
+    if (depth === 1 && relative.toLowerCase() === "build" && derived && derived.dev === stat.dev && derived.ino === stat.ino) return;
+    if (stat.isSymbolicLink()) refuse("paper", "symbolic links refused");
+    if (stat.isDirectory()) {
+      if (depth > COPILOT_PAPER_DEPTH || ++directories > COPILOT_PAPER_DIRECTORIES) refuse("paper", `source tree exceeds ${COPILOT_PAPER_DIRECTORIES} directories or ${COPILOT_PAPER_DEPTH} levels`);
+      const directory = opendirSync(file);
+      try { for (let entry = directory.readSync(); entry; entry = directory.readSync()) walk(relative ? `${relative}/${entry.name}` : entry.name, depth + 1); }
+      finally { directory.closeSync(); }
+    } else {
+      if (!relative || !stat.isFile()) refuse("paper", "regular files within the paper directory required");
+      if (entries.length >= COPILOT_PAPER_FILES) refuse("paper", `file count exceeds ${COPILOT_PAPER_FILES}`);
+      if (stat.size > COPILOT_PAPER_BYTES - bytes) refuse("paper", `aggregate bytes exceed ${COPILOT_PAPER_BYTES}`);
+      const content = readBytesBounded(file, COPILOT_PAPER_BYTES - bytes, { noFollow: true });
+      bytes += content.byteLength; entries.push([relative, sha256(content)]);
+    }
+  }
+  walk("", 0);
+  return entries.sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0);
+}
+
+/** Only active, present drafts have accept bindings. Irrelevant and acceptance events are excluded. */
+export function copilotBinding(dirArg: string, artifact: AcceptArtifact): string | undefined {
+  if (!ACCEPT_ARTIFACTS.includes(artifact)) throw new Error("copilot draft: unknown artifact; select hypotheses, paper or reviews and retry ares gate list");
+  const { dir, study } = openStudy(dirArg);
+  if (resolveAutonomy(study.autonomy, `accept:${artifact}`) !== "copilot") return undefined;
+  try {
+    if (artifact === "paper") {
+      const files = paperTree(dir);
+      return files.length ? sha256(canonical(files)) : undefined;
+    }
+    const events = draftLedger(dir, artifact).filter(event => artifact === "hypotheses" ? event.type === "idea.added" || event.type === "idea.decided" : event.type === "review.added" || event.type === "review.resolved");
+    if (artifact === "reviews") return events.length ? sha256(canonical(events)) : undefined;
+    const file = join(dir, "hypotheses.md"), stat = metadata(file);
+    let hypotheses: string | undefined;
+    if (stat !== undefined) {
+      if (stat.isSymbolicLink() || !stat.isFile()) refuse(artifact, "regular hypotheses file required; symbolic links refused");
+      if (stat.size > COPILOT_HYPOTHESES_BYTES) refuse(artifact, `hypotheses source exceeds ${COPILOT_HYPOTHESES_BYTES} bytes`);
+      hypotheses = readTextBounded(file, COPILOT_HYPOTHESES_BYTES, { noFollow: true });
+    }
+    return hypotheses !== undefined || events.length ? sha256(canonical([hypotheses === undefined ? null : sha256(hypotheses), events])) : undefined;
+  } catch (error) {
+    if (error instanceof DraftReadError) throw error;
+    refuse(artifact, "artifact or ledger is unreadable, non-regular, invalid UTF-8 or over-bound");
+  }
+}
+
+/** Acceptance records the existing gate actor; neither agents nor people receive an implicit approval. */
+export function requireCopilotAccept(dir: string, artifact: AcceptArtifact): void {
+  const binding = copilotBinding(dir, artifact);
+  if (binding === undefined) return;
+  const events = draftLedger(dir, artifact);
+  const gate = `accept:${artifact}` as const, state = gateState(gate, events, () => binding);
+  if (state.state === "approved") return;
+  throw new CopilotAcceptError(`copilot draft ${artifact}: ${state.state} acceptance; run ares gate approve <dir> ${gate} for the current draft before retrying`);
+}

package/src/exp/ablate.ts

@@ -0,0 +1,140 @@
+import { existsSync } from "node:fs";
+import { join } from "node:path";
+import { assertWritable, boundedText, commit, onlyKeys, openStudy, plain, readLedger, readTextBounded, requireExperiment, sha256, type Actor, type LedgerEvent } from "../study.ts";
+import { listAttempts, type AttemptStatus } from "./attempts.ts";
+import { readProtocol } from "./protocol.ts";
+import { readFinalSeeds, finalSeedsHash } from "./final.ts";
+import { metricValue, pairSamples, readClosedInvestigation, type Sample, type PairedSamples } from "./investigate.ts";
+
+export const MAX_ABLATIONS = 200, ABLATE_BYTES = 1_048_576;
+export interface Ablation { id: string; commit: string; removes: string }
+type Exclusion = "other-protocol" | "before-selection" | "missing-start-event" | "undeclared-seed" | "reserved-seed" | "undeclared-ablation" | "later-success";
+type AttemptsByStatus = Record<AttemptStatus, string[]>;
+export interface AblateRow extends Ablation { attempts: AttemptsByStatus; perSeed: Sample[]; paired: PairedSamples; reason: "no-data" | "reference-seeds" | "no-shared-seeds" | "paired" }
+export interface AblateResult {
+  v: 1; run: "ablate"; investigateSha256: string; inputsSha256: string; protocolHash: string;
+  metric: { name: string; unit: string; direction: "max" | "min" };
+  reference: { commit: string; attempts: AttemptsByStatus; perSeed: Sample[] };
+  ablations: AblateRow[];
+  totals: { attempts: number; byStatus: Record<AttemptStatus, number>; countedSuccesses: number; excluded: number };
+  excluded: { id: string; commit: string; seed: number; status: AttemptStatus; reasons: Exclusion[] }[];
+}
+function bad(reason: string): never { throw new Error(`ablate: ${reason}; inspect ares investigate show and ares attempt list, then correct the inputs and retry ablate`); }
+function canonical(raw: unknown): string {
+  if (Array.isArray(raw)) return `[${raw.map(canonical).join(",")}]`;
+  if (plain(raw)) return `{${Object.keys(raw).sort().map(k => `${JSON.stringify(k)}:${canonical(raw[k])}`).join(",")}}`;
+  return JSON.stringify(raw) ?? "null";
+}
+function declaration(raw: unknown, strict: boolean): Ablation {
+  if (!plain(raw)) bad("declaration object required");
+  if (strict) onlyKeys(raw, ["id", "commit", "removes"], "ablate declaration");
+  if (typeof raw.id !== "string" || !/^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$/.test(raw.id)) bad("id must be a run name of 1..64 characters");
+  if (typeof raw.commit !== "string" || !/^[a-fA-F0-9]{40}$/.test(raw.commit)) bad("commit must be a full 40-hex SHA");
+  const removes = boundedText(raw.removes, "ablate removes", 2000);
+  if (Buffer.byteLength(removes) > 2000) bad("removes exceeds 2000 bytes");
+  return { id: raw.id, commit: raw.commit.toLowerCase(), removes };
+}
+export function foldAblations(events: LedgerEvent[]): Ablation[] {
+  const rows: Ablation[] = [];
+  for (const e of events) if (e.type === "ablation.added") {
+    const row = declaration(e.data.ablation, false);
+    if (rows.length >= MAX_ABLATIONS || rows.some(old => old.id === row.id || old.commit === row.commit)) bad("journal has duplicate ablation id/commit or exceeds 200 declarations; restore the last valid ledger");
+    rows.push(row);
+  }
+  return rows;
+}
+function context(dirArg: string) {
+  const { dir, study } = openStudy(dirArg); requireExperiment(study);
+  const closed = readClosedInvestigation(dir), protocol = readProtocol(dir)!;
+  if (!closed.result.winner) bad("investigation has no winner; finish and close an investigation with a winner first");
+  const events = readLedger(dir), declarations = foldAblations(events);
+  return { dir, study, closed, protocol, events, declarations, winner: closed.result.winner.commit, baseline: closed.result.baseline.commit };
+}
+/** Caller holds the study lock; declarations never dispatch work or rewrite earlier journal entries. */
+export function addAblation(dirArg: string, raw: unknown, actor: Actor): Ablation {
+  try {
+    const input = context(dirArg), row = declaration(raw, true);
+    if (input.declarations.length >= MAX_ABLATIONS) bad("200 declarations reached; start a new study for more ablations");
+    if (input.declarations.some(old => old.id === row.id)) bad(`duplicate id ${row.id}; choose a unique id`);
+    if (input.declarations.some(old => old.commit === row.commit)) bad(`duplicate commit for ${row.id}; choose a unique frozen ablation`);
+    if (row.commit === input.winner || row.commit === input.baseline) bad("commit is the winner or selection baseline; use a distinct ablation commit");
+    commit(input.dir, [], [{ actor, type: "ablation.added", data: { ablation: row } }]); return row;
+  } catch (error) {
+    const message = error instanceof Error ? error.message : "ablate: invalid declaration";
+    throw new Error(message.includes("retry ablate") ? message : `${message}; correct the file and run ares ablate add --file PATH again`);
+  }
+}
+function empty(): AttemptsByStatus { return { running: [], succeeded: [], failed: [], cancelled: [], unknown: [] }; }
+function calculate(input: ReturnType<typeof context>): { result: AblateResult; running: boolean } {
+  const { dir, study, closed, protocol, events, declarations, winner } = input;
+  const attempts = listAttempts(dir, { run: "ablate" }).attempts, reserved = readFinalSeeds(dir, protocol)?.seeds ?? [];
+  const cutoff = events.findLastIndex(e => e.type === "selection.set"), starts = new Map<string, { index: number; data: Record<string, unknown> }>();
+  events.forEach((e, index) => { if (e.type === "attempt.started" && typeof e.data.id === "string") { if (starts.has(e.data.id)) bad(`duplicate start receipt ${e.data.id}`); starts.set(e.data.id, { index, data: e.data }); } });
+  const ordered = [...attempts].sort((a, b) => (starts.get(a.id)?.index ?? -1) - (starts.get(b.id)?.index ?? -1) || a.id.localeCompare(b.id));
+  const reference = { commit: winner, attempts: empty(), perSeed: [] as Sample[] };
+  const rows = declarations.map(d => ({ ...d, attempts: empty(), perSeed: [] as Sample[] }));
+  const byStatus = { running: 0, succeeded: 0, failed: 0, cancelled: 0, unknown: 0 }, excluded: AblateResult["excluded"] = [];
+  for (const a of ordered) {
+    if (a.studyId !== study.id || typeof a.commit !== "string" || !/^[a-f0-9]{40}$/.test(a.commit) || !Number.isSafeInteger(a.seed)) bad(`attempt ${a.id} has invalid study/commit/seed metadata`);
+    byStatus[a.status]++;
+    const row = a.commit === winner ? reference : rows.find(r => r.commit === a.commit);
+    if (row) row.attempts[a.status].push(a.id);
+    const reasons: Exclusion[] = [], start = starts.get(a.id);
+    if (!row) reasons.push("undeclared-ablation");
+    if (a.protocolHash !== closed.result.protocolHash) reasons.push("other-protocol");
+    if (!start) reasons.push("missing-start-event");
+    else {
+      if (start.data.run !== a.run || start.data.commit !== a.commit || start.data.seed !== a.seed) bad(`attempt ${a.id} start receipt identity mismatch`);
+      if (start.index <= cutoff) reasons.push("before-selection");
+    }
+    if (!protocol.seeds.includes(a.seed)) reasons.push("undeclared-seed");
+    if (reserved.includes(a.seed)) reasons.push("reserved-seed");
+    if (!reasons.length && a.status === "succeeded" && row) {
+      if (row.perSeed.some(s => s.seed === a.seed)) reasons.push("later-success");
+      else row.perSeed.push({ seed: a.seed, attempt: a.id, value: metricValue(a, closed.result.metric) });
+    }
+    if (reasons.length) excluded.push({ id: a.id, commit: a.commit, seed: a.seed, status: a.status, reasons });
+  }
+  reference.perSeed.sort((a, b) => a.seed - b.seed);
+  const ablations: AblateRow[] = rows.map(row => {
+    row.perSeed.sort((a, b) => a.seed - b.seed);
+    const paired = pairSamples(row.perSeed, reference.perSeed, closed.result.metric.direction, row.commit);
+    const reason = !row.perSeed.length ? "no-data" : !reference.perSeed.length ? "reference-seeds" : !paired.pairedSeeds.length ? "no-shared-seeds" : "paired";
+    return { ...row, paired, reason };
+  });
+  const result: AblateResult = {
+    v: 1, run: "ablate", investigateSha256: closed.sha256, protocolHash: closed.result.protocolHash, metric: closed.result.metric,
+    inputsSha256: sha256(canonical({ investigate: closed.sha256, declarations, finalSeeds: finalSeedsHash(dir, protocol) ?? null, attempts: ordered.map(a => ({ record: sha256(canonical(a)), start: starts.get(a.id)?.index ?? null })) })),
+    reference, ablations, totals: { attempts: attempts.length, byStatus, countedSuccesses: reference.perSeed.length + rows.reduce((n, r) => n + r.perSeed.length, 0), excluded: excluded.length }, excluded,
+  };
+  assertWritable(JSON.stringify(result, null, 2) + "\n", ABLATE_BYTES, "ablate result; archive this study and use a new bounded run");
+  return { result, running: attempts.some(a => a.status === "running") };
+}
+export function ablateResultPath(dir: string): string { return join(dir, "runs", "ablate", "ablate.json"); }
+function previous(file: string): string | undefined {
+  if (!existsSync(file)) return undefined;
+  let text: string;
+  try { text = readTextBounded(file, ABLATE_BYTES); } catch { bad("result is unreadable or oversized; restore the last valid bounded ablate.json"); }
+  let value: unknown; try { value = JSON.parse(text); } catch { bad("result JSON is malformed; restore the last valid ablate.json"); }
+  if (!plain(value) || value.v !== 1) throw new Error("ablate result: unsupported version; use an ares version that supports it, keeping ablate.json unchanged");
+  return text;
+}
+export function showAblate(dirArg: string): AblateResult { const input = context(dirArg); previous(ablateResultPath(input.dir)); return calculate(input).result; }
+/** Caller holds the study lock; result and receipt share one commit. */
+export function closeAblate(dirArg: string, actor: Actor): AblateResult {
+  const input = context(dirArg), file = ablateResultPath(input.dir), old = previous(file), next = calculate(input);
+  if (next.running) throw new Error("ablate close: an ablate attempt is running; run ares attempt reconcile before closing");
+  const text = JSON.stringify(next.result, null, 2) + "\n";
+  if (old !== text) commit(input.dir, [[file, text, ABLATE_BYTES]], [{ actor, type: "ablate.closed", data: { sha256: sha256(text), investigateSha256: input.closed.sha256 } }]);
+  return next.result;
+}
+
+export function readClosedAblate(dirArg: string): { result: AblateResult; sha256: string } {
+  const input = context(dirArg), text = previous(ablateResultPath(input.dir));
+  if (text === undefined) throw new Error("ablate result: not closed; run ares ablate close before final approval");
+  const hash = sha256(text), receipt = input.events.findLast(e => e.type === "ablate.closed");
+  if (!receipt || receipt.data.sha256 !== hash) throw new Error("ablate result: bytes disagree with the latest close receipt; run ares ablate close again");
+  const next = calculate(input);
+  if (next.running || text !== JSON.stringify(next.result, null, 2) + "\n") throw new Error("ablate result: stale inputs or running work; reconcile attempts and run ares ablate close again");
+  return { result: next.result, sha256: hash };
+}

package/src/exp/artifacts.ts

@@ -0,0 +1,229 @@
+import { createHash, type Hash } from "node:crypto";
+import { runOnExecutor, type Executor, type ProcessRunner } from "./executor.ts";
+
+export const ARTIFACT_FILES = 500, ARTIFACT_BYTES = 256 * 1024 * 1024;
+export const ARTIFACT_CAPTURE_BYTES = 65_536;
+export const ARTIFACT_HEADERS = 2048, ARTIFACT_METADATA_BYTES = 65_536;
+export const ARTIFACT_WIRE_BYTES = ARTIFACT_BYTES + 16 * 1024 * 1024;
+export type ArtifactErrorCode = "invalid-artifact" | "artifact-limit" | "artifact-read";
+export class ArtifactError extends Error {
+  constructor(readonly code: ArtifactErrorCode, cause: string) {
+    super(`artifact archive: ${cause}; ${code === "artifact-limit" ? "reduce artifact file count, bytes or metadata within the declared limits" : code === "artifact-read" ? "check the executor host and archive command" : "recreate the archive using regular files/directories, valid tar headers and safe relative paths"}, then rerun ares attempt reconcile <dir>`);
+    this.name = "ArtifactError";
+  }
+}
+export interface ArtifactFile { path: string; sha256: string; bytes: number }
+export interface ArtifactArchive { files: ArtifactFile[]; captured?: Uint8Array }
+export interface ArtifactArchiveOptions {
+  capture?: string; captureBytes?: number; maxBytes?: number; maxFiles?: number;
+}
+function invalid(cause: string): never { throw new ArtifactError("invalid-artifact", cause); }
+function limit(cause: string): never { throw new ArtifactError("artifact-limit", cause); }
+function utf8(bytes: Uint8Array): string {
+  try { return new TextDecoder("utf-8", { fatal: true }).decode(bytes); }
+  catch { return invalid("invalid UTF-8 metadata"); }
+}
+function field(header: Buffer, start: number, length: number, strict = true): string {
+  const bytes = header.subarray(start, start + length), end = bytes.indexOf(0);
+  const content = end === -1 ? bytes : bytes.subarray(0, end);
+  return strict ? utf8(content) : content.toString("utf8");
+}
+function octal(bytes: Uint8Array): number {
+  // GNU base-256 numbers are accepted only when positive and exactly representable.
+  if ((bytes[0]! & 128) !== 0) {
+    if ((bytes[0]! & 64) !== 0) return invalid("negative numeric header field");
+    let value = BigInt(bytes[0]! & 127);
+    for (const byte of bytes.subarray(1)) value = value * 256n + BigInt(byte);
+    if (value > BigInt(Number.MAX_SAFE_INTEGER)) return limit("numeric header field exceeds safe bounds");
+    return Number(value);
+  }
+  if (bytes.some(byte => byte !== 0 && byte !== 32 && (byte < 48 || byte > 55))) return invalid("malformed octal header field");
+  const text = Buffer.from(bytes).toString("ascii").replaceAll("\0", " ").trim();
+  if (!/^[0-7]+$/.test(text)) return invalid("malformed octal header field");
+  const value = Number.parseInt(text, 8);
+  if (!Number.isSafeInteger(value)) return limit("numeric header field exceeds safe bounds");
+  return value;
+}
+function decimal(value: string): number {
+  if (!/^[0-9]+$/.test(value)) return invalid("malformed PAX size field");
+  const number = Number(value);
+  if (!Number.isSafeInteger(number)) return limit("PAX size exceeds safe bounds");
+  return number;
+}
+function relative(value: string, directory = false): string {
+  if (!value || Buffer.byteLength(value) > 4096 || value.startsWith("/") || /^[A-Za-z]:/.test(value) || value.includes("\\") || /\p{Cc}/u.test(value)) return invalid("unsafe or oversized artifact path");
+  const parts = value.split("/");
+  if (parts.includes("..") || (!directory && value.endsWith("/"))) return invalid("traversal or ambiguous artifact path");
+  const normalized = parts.filter(part => part !== "" && part !== ".").join("/");
+  if (!normalized && !directory) return invalid("empty artifact file path");
+  return normalized;
+}
+function bound(value: number, max: number, what: string): number {
+  if (!Number.isSafeInteger(value) || value < 0 || value > max) return limit(`${what} must be an integer in 0..${max}`);
+  return value;
+}
+const ZERO_BLOCK = Buffer.alloc(65_536);
+function allZero(bytes: Uint8Array): boolean {
+  const buffer = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+  for (let offset = 0; offset < buffer.length; offset += ZERO_BLOCK.length) {
+    const part = buffer.subarray(offset, offset + ZERO_BLOCK.length);
+    if (!part.equals(ZERO_BLOCK.subarray(0, part.length))) return false;
+  }
+  return true;
+}
+type Pax = { path?: string | null; size?: string | null };
+interface FileBody { kind: "file"; path: string; size: number; remaining: number; hash: Hash; capture?: Buffer; offset: number }
+interface MetadataBody { kind: "metadata"; type: "x" | "g" | "L"; remaining: number; data: Buffer; offset: number }
+
+class TarArtifacts {
+  readonly files: ArtifactFile[] = [];
+  captured?: Uint8Array;
+  private readonly header = Buffer.alloc(512);
+  private headerBytes = 0;
+  private body?: FileBody | MetadataBody;
+  private padding = 0;
+  private zeros = 0;
+  private ended = false;
+  private wireBytes = 0;
+  private headers = 0;
+  private totalBytes = 0;
+  private fileCount = 0;
+  private global: Pax = {};
+  private next: Pax = {};
+  private pendingMetadata = false;
+  private longName?: string;
+  private seen = new Map<string, "file" | "directory">();
+  constructor(private capture: string | undefined, private captureBytes: number, private maxBytes: number, private maxFiles: number) {}
+  push(chunk: Uint8Array): void {
+    if (!(chunk instanceof Uint8Array)) return invalid("non-binary archive stream");
+    this.wireBytes += chunk.byteLength;
+    if (this.wireBytes > this.maxBytes + ARTIFACT_WIRE_BYTES - ARTIFACT_BYTES) return limit("archive stream exceeds its payload and metadata overhead limits");
+    let offset = 0;
+    while (offset < chunk.byteLength) {
+      if (this.ended) { if (!allZero(chunk.subarray(offset))) return invalid("nonzero data after tar end markers"); return; }
+      if (this.padding && !this.body) {
+        const count = Math.min(this.padding, chunk.byteLength - offset);
+        if (chunk.subarray(offset, offset + count).some(byte => byte !== 0)) return invalid("nonzero tar entry padding");
+        this.padding -= count; offset += count; continue;
+      }
+      if (this.body) {
+        const body = this.body, count = Math.min(body.remaining, chunk.byteLength - offset), part = chunk.subarray(offset, offset + count);
+        if (body.kind === "metadata") body.data.set(part, body.offset);
+        else {
+          body.hash.update(part);
+          body.capture?.set(part, body.offset);
+        }
+        body.offset += count; body.remaining -= count; offset += count;
+        if (body.remaining === 0) this.completeBody();
+        continue;
+      }
+      const count = Math.min(512 - this.headerBytes, chunk.byteLength - offset);
+      this.header.set(chunk.subarray(offset, offset + count), this.headerBytes); this.headerBytes += count; offset += count;
+      if (this.headerBytes === 512) { this.headerBytes = 0; this.readHeader(); }
+    }
+  }
+  private readHeader(): void {
+    const header = this.header;
+    if (header.every(byte => byte === 0)) { if (++this.zeros === 2) this.ended = true; return; }
+    if (this.zeros) return invalid("entry after an incomplete tar end marker");
+    if (++this.headers > ARTIFACT_HEADERS) return limit(`archive exceeds ${ARTIFACT_HEADERS} headers`);
+    const expected = octal(header.subarray(148, 156));
+    let checksum = 0;
+    for (let i = 0; i < 512; i++) checksum += i >= 148 && i < 156 ? 32 : header[i]!;
+    if (checksum !== expected) return invalid("tar header checksum mismatch");
+    const magic = field(header, 257, 6);
+    if (magic !== "" && magic !== "ustar" && magic !== "ustar ") return invalid("unsupported tar header format");
+    const type = header[156] === 0 ? "0" : String.fromCharCode(header[156]!);
+    if (!["0", "5", "x", "g", "L"].includes(type)) return invalid("link or special tar entry is forbidden");
+    if (field(header, 157, 100) !== "") return invalid("link target on a regular tar entry is forbidden");
+    const hasPathOverride = ({ ...this.global, ...this.next }.path ?? this.longName) !== undefined;
+    const prefix = magic === "ustar" ? field(header, 345, 155, !hasPathOverride) : "", name = field(header, 0, 100, !hasPathOverride);
+    const headerPath = prefix ? `${prefix}/${name}` : name;
+    // Metadata entry paths never become artifacts, but still must be safe paths.
+    relative(headerPath, type === "5" || hasPathOverride);
+    const headerSize = octal(header.subarray(124, 136));
+    if (type === "x" || type === "g" || type === "L") {
+      if (headerSize > ARTIFACT_METADATA_BYTES) return limit(`tar metadata exceeds ${ARTIFACT_METADATA_BYTES} bytes`);
+      this.body = { kind: "metadata", type, remaining: headerSize, data: Buffer.alloc(headerSize), offset: 0 };
+      this.padding = (512 - headerSize % 512) % 512;
+    } else {
+      const overrides = { ...this.global, ...this.next }, path = relative(overrides.path ?? this.longName ?? headerPath, type === "5"), size = overrides.size === undefined || overrides.size === null ? headerSize : decimal(overrides.size);
+      this.next = {}; this.longName = undefined; this.pendingMetadata = false;
+      const parts = path.split("/");
+      for (let count = 1; count < parts.length; count++) if (this.seen.get(parts.slice(0, count).join("/")) === "file") return invalid("artifact file is used as a parent directory");
+      if (type === "5") {
+        if (size !== 0) return invalid("directory tar entry has a payload");
+        if (this.seen.get(path) === "file") return invalid("artifact path changes entry type");
+        this.seen.set(path, "directory"); return;
+      }
+      if (this.seen.has(path)) return invalid("duplicate artifact file path");
+      if ([...this.seen.keys()].some(existing => existing.startsWith(`${path}/`))) return invalid("artifact file replaces a parent directory");
+      this.seen.set(path, "file");
+      if (++this.fileCount > this.maxFiles) return limit(`archive exceeds ${this.maxFiles} regular files`);
+      if (size > this.maxBytes - this.totalBytes) return limit(`regular file bytes exceed ${this.maxBytes}`);
+      this.totalBytes += size;
+      if (path === this.capture && size > this.captureBytes) return limit(`selected capture exceeds ${this.captureBytes} bytes`);
+      this.body = { kind: "file", path, size, remaining: size, hash: createHash("sha256"), ...(path === this.capture ? { capture: Buffer.alloc(size) } : {}), offset: 0 };
+      this.padding = (512 - size % 512) % 512;
+    }
+    if (this.body.remaining === 0) this.completeBody();
+  }
+  private completeBody(): void {
+    const body = this.body!; this.body = undefined;
+    if (body.kind === "file") {
+      this.files.push({ path: body.path, sha256: body.hash.digest("hex"), bytes: body.size });
+      if (body.capture) this.captured = body.capture;
+    } else if (body.type === "L") {
+      let end = body.data.length;
+      while (end && body.data[end - 1] === 0) end--;
+      this.longName = utf8(body.data.subarray(0, end)); relative(this.longName, true); this.pendingMetadata = true;
+    } else {
+      const values: Pax = {}, seen = new Set<string>();
+      for (let offset = 0; offset < body.data.length;) {
+        const separator = body.data.indexOf(32, offset);
+        if (separator === -1) return invalid("malformed PAX record length");
+        const lengthText = body.data.subarray(offset, separator).toString("utf8");
+        if (!/^[1-9][0-9]*$/.test(lengthText)) return invalid("malformed PAX record length");
+        const length = Number(lengthText), end = offset + length;
+        if (!Number.isSafeInteger(length) || end > body.data.length || end <= separator + 1 || body.data[end - 1] !== 10) return invalid("truncated or malformed PAX record");
+        const record = utf8(body.data.subarray(separator + 1, end - 1)), equals = record.indexOf("=");
+        if (equals < 1) return invalid("malformed PAX key/value record");
+        const key = record.slice(0, equals), value = record.slice(equals + 1);
+        if (seen.has(key)) return invalid("duplicate PAX metadata key"); seen.add(key);
+        if (key === "linkpath" || key.startsWith("GNU.sparse.") || key === "SCHILY.filetype" || key === "SCHILY.realsize") return invalid("link, sparse or special PAX metadata is forbidden");
+        if (key === "path") { if (value) relative(value, true); values.path = value || null; }
+        if (key === "size") { if (value) decimal(value); values.size = value || null; }
+        offset = end;
+      }
+      if (body.type === "g") {
+        for (const key of ["path", "size"] as const) if (values[key] !== undefined) { if (values[key] === null) delete this.global[key]; else this.global[key] = values[key]; }
+      } else { this.next = { ...this.next, ...values }; this.pendingMetadata = true; }
+    }
+  }
+  finish(): ArtifactArchive {
+    if (this.body || this.padding || this.headerBytes || !this.ended || this.pendingMetadata) return invalid("truncated archive, missing end markers or dangling metadata");
+    return { files: this.files, ...(this.captured !== undefined ? { captured: this.captured } : {}) };
+  }
+}
+
+/** Hash regular artifacts, retaining no payload except an explicitly selected bounded capture. */
+export async function readArtifactArchive(executor: Executor, command: string, args: string[], runner: ProcessRunner, options: ArtifactArchiveOptions = {}): Promise<ArtifactArchive> {
+  const capture = options.capture === undefined ? undefined : relative(options.capture), captureBytes = bound(options.captureBytes ?? ARTIFACT_CAPTURE_BYTES, ARTIFACT_CAPTURE_BYTES, "captureBytes");
+  const maxBytes = bound(options.maxBytes ?? ARTIFACT_BYTES, ARTIFACT_BYTES, "maxBytes"), maxFiles = bound(options.maxFiles ?? ARTIFACT_FILES, ARTIFACT_FILES, "maxFiles");
+  const parser = new TarArtifacts(capture, captureBytes, maxBytes, maxFiles), controller = new AbortController();
+  let parseError: ArtifactError | undefined;
+  try {
+    const result = await runOnExecutor(executor, command, args, runner, { signal: controller.signal, timeoutMs: 300_000,
+      stdoutSink: chunk => {
+        try { parser.push(chunk); }
+        catch (error) { parseError = error instanceof ArtifactError ? error : new ArtifactError("invalid-artifact", "malformed archive stream"); controller.abort(); throw parseError; }
+      } });
+    if (result.exitCode !== 0 || result.timedOut || result.truncated) throw new ArtifactError("artifact-read", "executor archive read failed");
+    return parser.finish();
+  } catch (error) {
+    controller.abort();
+    if (parseError) throw parseError;
+    if (error instanceof ArtifactError) throw error;
+    throw new ArtifactError("artifact-read", "executor archive read failed");
+  }
+}