@piagent/platform 1.2.16 → 1.3.0-ie.8

package/evals/long-horizon-v1/runner.mjs

@@ -0,0 +1,355 @@
+import assert from "node:assert/strict";
+import crypto from "node:crypto";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { spawn, spawnSync } from "node:child_process";
+import { fileURLToPath } from "node:url";
+
+import { buildContextEfficiencyReport } from "../../packages/piagent-core/extensions/context-engine.js";
+import { recordCompletionAudit, recordTaskStartCheckpoint } from "../../packages/piagent-core/extensions/task-runtime-audit.js";
+import { inspectTaskContinuationBudget, reserveTaskContinuation } from "../../packages/piagent-core/runtime/recovery/continuation-budget.ts";
+import { buildHandoffProjection, readHandoffProjection, writeHandoffProjection } from "../../packages/piagent-core/runtime/recovery/handoff-projection.ts";
+import { inspectTaskResumeState } from "../../packages/piagent-core/runtime/recovery/resume-state.ts";
+import { createBoundTaskAuthority } from "../../packages/piagent-core/runtime/policy/task-authority-runtime.ts";
+import { bindSessionTask, workingTreeSnapshot, writeTaskContract } from "../../packages/piagent-core/extensions/task-state.js";
+import { readTaskJournal } from "../../packages/piagent-core/extensions/task-journal.js";
+import { workingTreeEvidenceDigest } from "../../packages/piagent-core/extensions/working-tree-digest.js";
+
+const laneRoot = path.dirname(fileURLToPath(import.meta.url));
+const repositoryRoot = path.resolve(laneRoot, "../..");
+const lane = JSON.parse(fs.readFileSync(path.join(laneRoot, "lane.json"), "utf8"));
+const argumentsList = process.argv.slice(2);
+const option = (name) => {
+  const index = argumentsList.indexOf(name);
+  return index >= 0 ? argumentsList[index + 1] : undefined;
+};
+const calibrationFast = argumentsList.includes("--calibration-fast");
+const outputPath = path.resolve(option("--output") ?? path.join(process.cwd(), "long-horizon-report.json"));
+const totalUnits = calibrationFast ? 9 : lane.totalUnits;
+const crashAfterUnit = calibrationFast ? 3 : lane.hardCrashAfterUnit;
+const handoffAfterUnit = calibrationFast ? 6 : lane.handoffAfterUnit;
+const compactionUnits = calibrationFast ? [2, 4, 6, 8] : lane.compactionUnits;
+const tickMilliseconds = calibrationFast ? Number(option("--tick-ms") ?? 2) : lane.tickMilliseconds;
+const evidenceClass = calibrationFast ? "calibration-fast" : "provider-free-long-horizon";
+if (!calibrationFast && option("--tick-ms") !== undefined) throw new Error("wall-clock evidence cannot override tick duration");
+
+const temporaryRoot = fs.mkdtempSync(path.join(os.tmpdir(), "piagent-long-horizon-"));
+const workspace = path.join(temporaryRoot, "project");
+const startedAtMs = Date.now();
+const startedAt = new Date(startedAtMs).toISOString();
+const fixture = path.resolve(laneRoot, lane.fixture);
+fs.cpSync(fixture, workspace, { recursive: true });
+spawnSync("git", ["init", "-q", workspace], { stdio: "inherit" });
+spawnSync("git", ["-C", workspace, "config", "user.email", "long-horizon@example.invalid"], { stdio: "inherit" });
+spawnSync("git", ["-C", workspace, "config", "user.name", "Piagent Long Horizon"], { stdio: "inherit" });
+spawnSync("git", ["-C", workspace, "add", "."], { stdio: "inherit" });
+spawnSync("git", ["-C", workspace, "commit", "-qm", "fixture"], { stdio: "inherit" });
+
+const taskId = "long-horizon-90";
+const taskRunId = "long-horizon-90-run-1";
+const sessionId = "long-horizon-session";
+const sessionName = "LONG-HORIZON-90 repository audit";
+const createdAt = startedAt;
+const portableVerifier = path.join(workspace, ".pi", "piagent-state", "long-horizon", "verify.mjs");
+fs.mkdirSync(path.dirname(portableVerifier), { recursive: true, mode: 0o700 });
+fs.copyFileSync(path.join(laneRoot, "verify.mjs"), portableVerifier);
+fs.chmodSync(portableVerifier, 0o600);
+let task = {
+  schemaVersion: 2,
+  taskRunId,
+  taskId,
+  sessionId,
+  sessionName,
+  changeMode: "source-change",
+  attempt: 1,
+  maxAttempts: 3,
+  previousAttempts: [],
+  summary: "Complete a durable repository inventory across three 30-minute logical stages and survive a hard process death.",
+  riskLane: "normal",
+  expectedOutput: "A verified 90-unit inventory report completed from durable resume state.",
+  acceptanceCriteria: [
+    "All configured units are present and bind to current source bytes.",
+    "A hard process death resumes from the last durable checkpoint without duplicate units.",
+    "Compaction keeps context proxy and private state growth within declared ceilings.",
+    "Exactly one system continuation is consumable and a second is refused.",
+    "The final verifier is stable on the completed current tree."
+  ],
+  scope: ["artifacts/long-horizon/**"],
+  outOfScope: ["Existing source, package, vendor, configuration, and dependency files."],
+  protectedPaths: [],
+  requiredContext: [],
+  contextManifest: [],
+  memoryCitations: [],
+  mcpCapabilities: [],
+  verifyGroup: "source",
+  verifyCommands: [`node .pi/piagent-state/long-horizon/verify.mjs . ${totalUnits}`],
+  workPlan: [
+    { id: "discovery", title: "Inventory the first repository segment.", role: "parent", mode: "single-writer", status: "in-progress" },
+    { id: "validation", title: "Resume and validate the second segment.", role: "parent", mode: "single-writer", status: "pending", dependsOn: ["discovery"] },
+    { id: "synthesis", title: "Resume from handoff and synthesize the final report.", role: "parent", mode: "single-writer", status: "pending", dependsOn: ["validation"] }
+  ],
+  reviewLenses: ["correctness", "tests", "scope"],
+  workingTreeDigestAlgorithm: "wt-content-v2",
+  orchestration: {
+    mode: "solo-first", subagents: "not-used", reason: "Deterministic one-writer lifecycle lane.", fieldGuidePath: ".pi/memory/MEMORY.md",
+    modelRoles: { planner: "not-used", worker: "not-used", reviewer: "not-used", watchdog: "not-used" }
+  },
+  baselineChangedFiles: [],
+  baselineFileDigests: workingTreeSnapshot(workspace),
+  observedChangedFiles: [],
+  finalWorkingTreeFiles: [],
+  finalFileDigests: {},
+  changedFiles: [],
+  verifyEvidence: [],
+  trace: { outcome: "pending" },
+  createdAt,
+  updatedAt: createdAt
+};
+task.authoritySnapshot = createBoundTaskAuthority({ taskId, taskRunId, createdAt });
+task = writeTaskContract(workspace, task);
+bindSessionTask(workspace, sessionId, sessionName, task);
+recordTaskStartCheckpoint({ cwd: workspace, ui: { notify() {} } }, task, "discovery", "automatic");
+
+const runtimeBase = {
+  laneId: lane.id,
+  taskId,
+  taskRunId,
+  sessionId,
+  totalUnits,
+  logicalDurationMinutes: lane.logicalDurationMinutes,
+  compactionUnits,
+  tickMilliseconds
+};
+const workerPath = path.join(laneRoot, "worker.mjs");
+const statePath = path.join(workspace, ".pi", "piagent-state", "long-horizon", "state.json");
+
+function state() {
+  try { return JSON.parse(fs.readFileSync(statePath, "utf8")); }
+  catch (error) { if (error?.code === "ENOENT") return { currentUnit: 0 }; throw error; }
+}
+
+function startWorker(stopAfterUnit) {
+  const runtime = Buffer.from(JSON.stringify({ ...runtimeBase, stopAfterUnit })).toString("base64url");
+  const child = spawn(process.execPath, [workerPath, workspace, runtime], { stdio: ["ignore", "pipe", "pipe"] });
+  let stdout = "", stderr = "";
+  child.stdout.on("data", (chunk) => { stdout += chunk; });
+  child.stderr.on("data", (chunk) => { stderr += chunk; });
+  return { child, result: new Promise((resolve) => child.once("exit", (code, signal) => resolve({ code, signal, stdout, stderr }))) };
+}
+
+async function waitForUnit(unit) {
+  const remaining = Math.max(1, unit - Number(state().currentUnit ?? 0));
+  const deadline = Date.now() + Math.max(30_000, tickMilliseconds * (remaining + 2));
+  while (Date.now() < deadline) {
+    if (state().currentUnit >= unit) return;
+    await new Promise((resolve) => setTimeout(resolve, Math.min(100, Math.max(5, tickMilliseconds / 20))));
+  }
+  throw new Error(`timed out waiting for long-horizon unit ${unit}`);
+}
+
+function updatePendingTask(completed, active) {
+  const snapshot = workingTreeSnapshot(workspace);
+  const files = Object.keys(snapshot).sort();
+  task = writeTaskContract(workspace, {
+    ...task,
+    workPlan: task.workPlan.map((step) => ({ ...step, status: step.id === completed ? "done" : step.id === active ? "in-progress" : step.status })),
+    observedChangedFiles: files,
+    updatedAt: new Date().toISOString()
+  });
+  bindSessionTask(workspace, sessionId, sessionName, task);
+}
+
+function treeBytes(directory) {
+  let bytes = 0;
+  const visit = (current) => {
+    for (const entry of fs.readdirSync(current, { withFileTypes: true })) {
+      const absolute = path.join(current, entry.name);
+      if (entry.isDirectory()) visit(absolute);
+      else if (entry.isFile()) bytes += fs.statSync(absolute).size;
+    }
+  };
+  visit(directory);
+  return bytes;
+}
+
+function persistHandoff(reason) {
+  const projection = buildHandoffProjection(workspace, task, {
+    gate: { decision: "fail", missing: [reason], missingVerifyCommands: task.verifyCommands, currentWorkingTreeDigest: workingTreeEvidenceDigest(workingTreeSnapshot(workspace)) },
+    recovery: null
+  });
+  writeHandoffProjection(workspace, projection);
+  return readHandoffProjection(workspace, taskRunId);
+}
+
+const first = startWorker(undefined);
+await waitForUnit(crashAfterUnit);
+first.child.kill("SIGKILL");
+const firstResult = await first.result;
+assert.equal(firstResult.signal, "SIGKILL", firstResult.stderr);
+assert.equal(state().currentUnit, crashAfterUnit);
+updatePendingTask("discovery", "validation");
+const crashDigest = workingTreeEvidenceDigest(workingTreeSnapshot(workspace));
+const firstContinuation = reserveTaskContinuation(workspace, task, {
+  capabilityId: "CAP-12",
+  classification: "infrastructure-retry",
+  action: "retry",
+  currentWorkingTreeDigest: crashDigest,
+  reasonCodes: ["process-killed"],
+  recordedAt: new Date().toISOString()
+});
+assert.equal(firstContinuation.allowed, true);
+const secondContinuation = reserveTaskContinuation(workspace, task, {
+  capabilityId: "CAP-12",
+  classification: "infrastructure-retry",
+  action: "retry",
+  currentWorkingTreeDigest: crashDigest,
+  reasonCodes: ["second-process-killed"],
+  recordedAt: new Date().toISOString()
+});
+assert.equal(secondContinuation.allowed, false);
+assert.equal(secondContinuation.reason, "global-budget-exhausted");
+const crashHandoff = persistHandoff("hard process death at a durable checkpoint");
+assert.equal(crashHandoff?.identity.taskRunId, taskRunId);
+const crashResume = inspectTaskResumeState(workspace, task, sessionId);
+assert.equal(crashResume.enforcementSafe, true, crashResume.reason);
+
+const second = startWorker(handoffAfterUnit);
+const secondResult = await second.result;
+assert.equal(secondResult.code, 75, secondResult.stderr);
+assert.equal(state().currentUnit, handoffAfterUnit);
+updatePendingTask("validation", "synthesis");
+const plannedHandoff = persistHandoff("planned process handoff after validation stage");
+assert.equal(plannedHandoff?.identity.taskRunId, taskRunId);
+
+const third = startWorker(undefined);
+const thirdResult = await third.result;
+assert.equal(thirdResult.code, 0, thirdResult.stderr);
+assert.equal(state().currentUnit, totalUnits);
+const beforeVerify = workingTreeEvidenceDigest(workingTreeSnapshot(workspace));
+const verifier = spawnSync(process.execPath, [portableVerifier, workspace, String(totalUnits)], { encoding: "utf8" });
+assert.equal(verifier.status, 0, verifier.stderr);
+const afterVerify = workingTreeEvidenceDigest(workingTreeSnapshot(workspace));
+assert.equal(afterVerify, beforeVerify);
+const finalSnapshot = workingTreeSnapshot(workspace);
+const finalFiles = Object.keys(finalSnapshot).sort();
+const completedAt = new Date().toISOString();
+task = writeTaskContract(workspace, {
+  ...task,
+  workPlan: task.workPlan.map((step) => ({ ...step, status: "done" })),
+  observedChangedFiles: finalFiles,
+  finalWorkingTreeFiles: finalFiles,
+  finalFileDigests: finalSnapshot,
+  changedFiles: finalFiles,
+  verifyEvidence: [{
+    command: task.verifyCommands[0], exitCode: 0, summary: verifier.stdout.trim(), recordedAt: completedAt, observedAt: completedAt,
+    observed: true, matchedProfileCommand: true, isError: false, preWorkingTreeDigest: beforeVerify, workingTreeDigest: afterVerify
+  }],
+  trace: { outcome: "completed", recordedAt: completedAt },
+  updatedAt: completedAt
+});
+recordCompletionAudit({ cwd: workspace, ui: { notify() {} } }, task, { outcome: "completed", phase: "review", idempotencyAt: completedAt });
+const terminalResume = inspectTaskResumeState(workspace, task, sessionId);
+assert.equal(terminalResume.decision, "terminal");
+
+const finalState = state();
+const telemetry = fs.readFileSync(path.join(workspace, ".pi", "piagent-state", "long-horizon", "telemetry.jsonl"), "utf8").trim().split("\n").map((line) => JSON.parse(line));
+const journal = readTaskJournal(workspace, { taskRunId });
+const continuation = inspectTaskContinuationBudget(workspace, task);
+const contextEfficiency = buildContextEfficiencyReport(workspace);
+const finalDurableStateBytes = treeBytes(path.join(workspace, ".pi", "piagent-state"));
+const stateSamples = telemetry.map((entry) => ({
+  unit: entry.unit,
+  processStart: entry.processStart,
+  resumedFromUnit: entry.resumedFromUnit,
+  compactions: entry.compactions,
+  contextProxyTokens: entry.contextProxyTokens,
+  durableStateBytes: entry.durableStateBytes
+}));
+const peakDurableStateBytes = Math.max(finalState.peakDurableStateBytes, finalDurableStateBytes);
+const peakContextSample = stateSamples.reduce((peak, sample) => sample.contextProxyTokens > peak.contextProxyTokens ? sample : peak, stateSamples[0]);
+const endedAtMs = Date.now();
+const wallClockMilliseconds = endedAtMs - startedAtMs;
+const completedFromResume = finalState.processStarts === 3
+  && finalState.resumedUnits.includes(crashAfterUnit)
+  && finalState.resumedUnits.includes(handoffAfterUnit)
+  && terminalResume.decision === "terminal";
+const wallClockQualified = wallClockMilliseconds >= lane.minimumWallClockMinutes * 60_000;
+const report = {
+  schemaVersion: 1,
+  laneId: lane.id,
+  workItem: lane.workItem,
+  evidenceClass,
+  startedAt,
+  completedAt,
+  logicalDurationMinutes: lane.logicalDurationMinutes,
+  wallClockMilliseconds,
+  wallClockMinutes: Number((wallClockMilliseconds / 60_000).toFixed(4)),
+  minimumWallClockMinutes: lane.minimumWallClockMinutes,
+  wallClockQualified,
+  providerUsed: false,
+  completedFromResume,
+  lifecycle: {
+    totalUnits,
+    hardCrashAfterUnit: crashAfterUnit,
+    handoffAfterUnit,
+    processStarts: finalState.processStarts,
+    hardCrashes: 1,
+    resumedUnits: finalState.resumedUnits,
+    compactions: finalState.compactions,
+    handoffReadback: Boolean(crashHandoff && plannedHandoff),
+    journalEvents: journal.events.length,
+    journalCorruptions: journal.corruptions,
+    terminalDecision: terminalResume.decision
+  },
+  context: {
+    source: "deterministic-provider-free-proxy",
+    observations: telemetry.length,
+    peakProxyTokens: finalState.peakContextProxyTokens,
+    peakObservedAtUnit: peakContextSample.unit,
+    ceilingTokens: lane.contextProxyCeilingTokens,
+    withinCeiling: finalState.peakContextProxyTokens <= lane.contextProxyCeilingTokens,
+    efficiency: contextEfficiency.metrics
+  },
+  stateGrowth: {
+    observations: telemetry.length,
+    initialBytes: stateSamples[0].durableStateBytes,
+    finalBytes: finalDurableStateBytes,
+    growthBytes: finalDurableStateBytes - stateSamples[0].durableStateBytes,
+    peakBytes: peakDurableStateBytes,
+    ceilingBytes: lane.durableStateCeilingBytes,
+    withinCeiling: peakDurableStateBytes <= lane.durableStateCeilingBytes,
+    checkpointSequenceComplete: telemetry.length === totalUnits && telemetry.every((entry, index) => entry.unit === index + 1),
+    samples: stateSamples
+  },
+  continuation: {
+    firstAllowed: firstContinuation.allowed,
+    secondAllowed: secondContinuation.allowed,
+    secondReason: secondContinuation.reason,
+    consumed: continuation.consumed,
+    maximum: continuation.maximum,
+    enforcementSafe: continuation.enforcementSafe
+  },
+  verification: {
+    command: task.verifyCommands[0],
+    exitCode: verifier.status,
+    preWorkingTreeDigest: beforeVerify,
+    workingTreeDigest: afterVerify,
+    stableCurrentTree: beforeVerify === afterVerify,
+    changedFiles: finalFiles,
+    finalFileDigestCount: Object.keys(task.finalFileDigests).length
+  },
+  claimBoundary: lane.claimBoundary
+};
+assert.equal(report.completedFromResume, true);
+assert.equal(report.context.withinCeiling, true);
+assert.equal(report.stateGrowth.withinCeiling, true);
+assert.equal(report.stateGrowth.checkpointSequenceComplete, true);
+assert.equal(report.continuation.consumed, 1);
+assert.equal(report.continuation.maximum, 1);
+assert.equal(report.verification.stableCurrentTree, true);
+if (!calibrationFast) assert.equal(report.wallClockQualified, true, `wall clock ${report.wallClockMinutes}m is below ${lane.minimumWallClockMinutes}m`);
+fs.mkdirSync(path.dirname(outputPath), { recursive: true });
+fs.writeFileSync(outputPath, `${JSON.stringify(report, null, 2)}\n`, { mode: 0o600 });
+fs.rmSync(temporaryRoot, { recursive: true, force: true });
+process.stdout.write(`${JSON.stringify(report)}\n`);

package/evals/long-horizon-v1/verify.mjs

@@ -0,0 +1,28 @@
+import assert from "node:assert/strict";
+import crypto from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+
+const [workspaceArgument, expectedArgument] = process.argv.slice(2);
+const workspace = path.resolve(workspaceArgument);
+const expected = Number(expectedArgument);
+assert.equal(Number.isSafeInteger(expected) && expected > 0, true);
+const root = path.join(workspace, "artifacts", "long-horizon");
+const unitRoot = path.join(root, "units");
+const files = fs.readdirSync(unitRoot).filter((file) => file.endsWith(".json")).sort();
+assert.equal(files.length, expected);
+const aggregate = crypto.createHash("sha256");
+for (let index = 0; index < files.length; index += 1) {
+  const raw = fs.readFileSync(path.join(unitRoot, files[index]));
+  const unit = JSON.parse(raw);
+  assert.equal(unit.unit, index + 1);
+  assert.equal(unit.logicalMinute, index + 1);
+  assert.match(unit.sourcePath, /\S/);
+  assert.match(unit.sourceDigest, /^[a-f0-9]{64}$/);
+  assert.equal(crypto.createHash("sha256").update(fs.readFileSync(path.join(workspace, unit.sourcePath))).digest("hex"), unit.sourceDigest);
+  aggregate.update(raw);
+}
+const report = JSON.parse(fs.readFileSync(path.join(root, "report.json"), "utf8"));
+assert.equal(report.completedUnits, expected);
+assert.equal(report.aggregateDigest, aggregate.digest("hex"));
+process.stdout.write(`${JSON.stringify({ passed: true, completedUnits: expected, aggregateDigest: report.aggregateDigest })}\n`);

package/evals/long-horizon-v1/worker.mjs

@@ -0,0 +1,177 @@
+import crypto from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+
+import { appendContextTelemetry, estimateContextTokens } from "../../packages/piagent-core/extensions/context-engine.js";
+import { ensurePrivateStateDirectory, resolveLocalStatePath } from "../../packages/piagent-core/extensions/local-state-path.js";
+import { appendTaskJournalEvent } from "../../packages/piagent-core/extensions/task-journal.js";
+import { workingTreeSnapshot } from "../../packages/piagent-core/extensions/task-state.js";
+import { workingTreeEvidenceDigest } from "../../packages/piagent-core/extensions/working-tree-digest.js";
+
+const [workspaceArgument, runtimeArgument] = process.argv.slice(2);
+if (!workspaceArgument || !runtimeArgument) throw new Error("worker requires workspace and runtime specification");
+const workspace = path.resolve(workspaceArgument);
+const runtime = JSON.parse(Buffer.from(runtimeArgument, "base64url").toString("utf8"));
+const privateRoot = ensurePrivateStateDirectory(workspace, path.join(workspace, ".pi", "piagent-state", "long-horizon"), "Long-horizon state");
+const statePath = resolveLocalStatePath(workspace, path.join(privateRoot, "state.json"), { label: "Long-horizon state" });
+const workingSetPath = resolveLocalStatePath(workspace, path.join(privateRoot, "working-set.json"), { label: "Long-horizon working set" });
+const telemetryPath = resolveLocalStatePath(workspace, path.join(privateRoot, "telemetry.jsonl"), { label: "Long-horizon telemetry" });
+const artifactRoot = path.join(workspace, "artifacts", "long-horizon");
+const unitRoot = path.join(artifactRoot, "units");
+fs.mkdirSync(unitRoot, { recursive: true });
+
+function processAlive(pid) {
+  if (!Number.isInteger(pid) || pid <= 0) return false;
+  try { process.kill(pid, 0); return true; }
+  catch (error) { return error?.code === "EPERM"; }
+}
+
+function removeDeadWriterTemps(directory) {
+  for (const name of fs.readdirSync(directory)) {
+    const match = name.match(/\.(\d+)\.tmp$/);
+    if (!match || processAlive(Number.parseInt(match[1], 10))) continue;
+    const target = path.join(directory, name);
+    try { if (fs.lstatSync(target).isFile()) fs.rmSync(target); } catch (error) { if (error?.code !== "ENOENT") throw error; }
+  }
+}
+
+removeDeadWriterTemps(privateRoot);
+removeDeadWriterTemps(unitRoot);
+
+function readJson(file, fallback) {
+  try { return JSON.parse(fs.readFileSync(file, "utf8")); }
+  catch (error) { if (error?.code === "ENOENT") return fallback; throw error; }
+}
+
+function writeAtomic(file, value, mode = 0o600) {
+  const temporary = `${file}.${process.pid}.tmp`;
+  fs.writeFileSync(temporary, `${JSON.stringify(value)}\n`, { mode });
+  fs.renameSync(temporary, file);
+  fs.chmodSync(file, mode);
+}
+
+function projectSources() {
+  const values = [];
+  const visit = (directory) => {
+    for (const entry of fs.readdirSync(directory, { withFileTypes: true }).sort((left, right) => Buffer.from(left.name).compare(Buffer.from(right.name)))) {
+      if ([".git", ".pi", "artifacts"].includes(entry.name)) continue;
+      const absolute = path.join(directory, entry.name);
+      if (entry.isDirectory()) visit(absolute);
+      else if (entry.isFile()) values.push(path.relative(workspace, absolute).split(path.sep).join("/"));
+    }
+  };
+  visit(workspace);
+  return values;
+}
+
+function treeBytes(root) {
+  let bytes = 0;
+  const visit = (directory) => {
+    for (const entry of fs.readdirSync(directory, { withFileTypes: true })) {
+      const absolute = path.join(directory, entry.name);
+      if (entry.isDirectory()) visit(absolute);
+      else if (entry.isFile()) bytes += fs.statSync(absolute).size;
+    }
+  };
+  visit(root);
+  return bytes;
+}
+
+function appendTelemetry(value) {
+  fs.appendFileSync(telemetryPath, `${JSON.stringify(value)}\n`, { mode: 0o600 });
+  fs.chmodSync(telemetryPath, 0o600);
+}
+
+function sleep(milliseconds) {
+  return new Promise((resolve) => setTimeout(resolve, milliseconds));
+}
+
+const sources = projectSources();
+if (sources.length < runtime.totalUnits) throw new Error(`fixture has only ${sources.length} source files for ${runtime.totalUnits} units`);
+let state = readJson(statePath, {
+  schemaVersion: 1,
+  laneId: runtime.laneId,
+  taskId: runtime.taskId,
+  taskRunId: runtime.taskRunId,
+  currentUnit: 0,
+  processStarts: 0,
+  compactions: 0,
+  peakContextProxyTokens: 0,
+  peakDurableStateBytes: 0,
+  resumedUnits: []
+});
+if (state.laneId !== runtime.laneId || state.taskId !== runtime.taskId || state.taskRunId !== runtime.taskRunId) throw new Error("long-horizon state identity mismatch");
+const startingUnit = state.currentUnit;
+state.processStarts += 1;
+if (startingUnit > 0) state.resumedUnits.push(startingUnit);
+writeAtomic(statePath, state);
+let workingSet = readJson(workingSetPath, []);
+
+for (let unit = startingUnit + 1; unit <= runtime.totalUnits; unit += 1) {
+  const sourcePath = sources[(unit - 1) % sources.length];
+  const source = fs.readFileSync(path.join(workspace, sourcePath));
+  const sourceDigest = crypto.createHash("sha256").update(source).digest("hex");
+  const unitRecord = { schemaVersion: 1, unit, logicalMinute: unit, sourcePath, sourceDigest };
+  writeAtomic(path.join(unitRoot, `${String(unit).padStart(3, "0")}.json`), unitRecord, 0o644);
+  workingSet.push({ unit, sourcePath, sourceDigest, observation: `validated-${unit}-${sourceDigest.slice(0, 16)}` });
+  const beforeCompactionTokens = estimateContextTokens(JSON.stringify(workingSet));
+  state.peakContextProxyTokens = Math.max(state.peakContextProxyTokens, beforeCompactionTokens);
+  if (runtime.compactionUnits.includes(unit)) {
+    workingSet = [{ unit, compactedThrough: unit, retainedDigest: crypto.createHash("sha256").update(JSON.stringify(workingSet)).digest("hex") }];
+    state.compactions += 1;
+    appendContextTelemetry(workspace, { event: "session_compact", taskRunId: runtime.taskRunId, unit, reason: "long-horizon-boundary", fromExtension: true });
+  }
+  writeAtomic(workingSetPath, workingSet);
+  const currentWorkingTreeDigest = workingTreeEvidenceDigest(workingTreeSnapshot(workspace));
+  appendTaskJournalEvent(workspace, {
+    eventType: "long-horizon-progress",
+    taskId: runtime.taskId,
+    taskRunId: runtime.taskRunId,
+    sessionId: runtime.sessionId,
+    idempotencyKey: `long-horizon:${unit}`,
+    data: { unit, logicalMinute: unit, sourceDigest, currentWorkingTreeDigest }
+  }, { recordedAt: new Date().toISOString() });
+  appendContextTelemetry(workspace, {
+    event: "agent_prompt",
+    taskRunId: runtime.taskRunId,
+    unit,
+    activeTools: 4,
+    systemPromptTokens: 1006,
+    toolSchemaTokens: 872,
+    contextProxyTokens: beforeCompactionTokens,
+    source: "deterministic-provider-free-proxy"
+  });
+  const durableStateBytes = treeBytes(path.join(workspace, ".pi", "piagent-state"));
+  state.currentUnit = unit;
+  state.peakDurableStateBytes = Math.max(state.peakDurableStateBytes, durableStateBytes);
+  state.lastWorkingTreeDigest = currentWorkingTreeDigest;
+  state.updatedAt = new Date().toISOString();
+  appendTelemetry({
+    schemaVersion: 1,
+    recordedAt: state.updatedAt,
+    processId: process.pid,
+    processStart: state.processStarts,
+    unit,
+    resumedFromUnit: startingUnit,
+    contextProxyTokens: beforeCompactionTokens,
+    durableStateBytes,
+    compactions: state.compactions,
+    currentWorkingTreeDigest
+  });
+  writeAtomic(statePath, state);
+  if (unit === runtime.stopAfterUnit && unit < runtime.totalUnits) process.exit(75);
+  if (unit < runtime.totalUnits) await sleep(runtime.tickMilliseconds);
+}
+
+const unitFiles = fs.readdirSync(unitRoot).filter((file) => file.endsWith(".json")).sort();
+const aggregate = crypto.createHash("sha256");
+for (const file of unitFiles) aggregate.update(fs.readFileSync(path.join(unitRoot, file)));
+writeAtomic(path.join(artifactRoot, "report.json"), {
+  schemaVersion: 1,
+  laneId: runtime.laneId,
+  taskRunId: runtime.taskRunId,
+  completedUnits: unitFiles.length,
+  logicalDurationMinutes: runtime.logicalDurationMinutes,
+  aggregateDigest: aggregate.digest("hex")
+}, 0o644);
+process.exit(0);

package/evals/private-holdout-v1/CUSTODIAN_RUNBOOK.md

@@ -0,0 +1,79 @@
+# E3 custodian runbook
+
+This runbook is for an independent holdout custodian. It is intentionally safe
+to ship with the public candidate: it contains no private task, repository,
+grader, reviewer identity, score, credential, or locator.
+
+## Trust boundary
+
+The candidate author cannot act as custodian, reviewer, adjudicator, or release
+auditor. The readiness CLI validates a closed receipt, but it is not a
+self-authenticating proof of who created that receipt. The independent release
+auditor must verify the receipt origin and append-only custody record in the
+controlled environment before `CF-FS4-05` can pass. Candidate authors cannot
+self-attest E3.
+
+Never copy the private suite, source repositories, prompts, graders, reference
+solutions, mutations, access log, raw reviewer sheets, or private paths into the
+candidate repository, benchmark output, public tracker, or model context.
+
+## Inputs held outside the candidate-author environment
+
+- a private suite with at least six task-family lineages from at least six
+  repository lineages;
+- the append-only access log and independent custodian identity;
+- hidden graders, reference solutions, declared mutations and their reports;
+- the blinded reviewer roster, sealed first-pass score sheets, disagreement log
+  and independent adjudication record;
+- the exact public candidate package or commit being evaluated.
+
+## Procedure
+
+1. Record the exact candidate identity and the SHA-256 commitments for
+   `access-policy.v1.json`, `human-rubric.v1.json`,
+   `public-exposure.v1.json`, and `real-task-taxonomy.v1.json`.
+2. Compare private family and repository lineage against every entry in the
+   public exposure file. A rename, fork, vendor copy, generated variant, or
+   shared fixture lineage is overlap, not disjointness.
+3. Verify that the candidate author had no private prompt, grader, repository,
+   reviewer, or raw-score access and that the benchmark operator had only an
+   execute-only interface. Seal the append-only access-log commitment before
+   private execution.
+4. Run every reference solution and declared mutation in the controlled
+   environment. Every reference must pass and every mutation must be killed.
+5. Select at least 12 items across at least four task families after candidate
+   freeze. Have at least two blinded reviewers independently score every item.
+   Preserve both initial scores, record every disagreement, and use an
+   independent adjudicator. No unresolved disagreement is permitted.
+6. Create a schema-v2 assurance receipt containing only the closed values
+   allowed by `benchmark-assurance-evidence.schema.json`. Do not add raw text,
+   paths, locators, reviewer identities, outputs, credentials, or private
+   metadata.
+7. In the controlled environment, run:
+
+   ```text
+   node scripts/private-holdout-readiness.mjs --evidence <secure-receipt-path>
+   ```
+
+8. Independently read back the candidate identity, receipt SHA-256, custody
+   origin, access validity window, disjointness counts, reference/mutation
+   totals, sample counts, agreement arithmetic, and zero-unresolved status.
+9. Export only the CLI's redacted JSON output plus the auditor's external
+   custody attestation. Do not export the input path or private artifacts.
+
+## Stop conditions
+
+Stop without rerunning the candidate if any author access, lineage overlap,
+surviving mutation, failed reference, invalid/expired access receipt, missing
+double score, unresolved disagreement, arithmetic mismatch, digest mismatch,
+candidate drift, or private-field leak is observed. A failed receipt cannot be
+repaired by changing the private task or grader after observing candidate
+output; create a new versioned holdout and a new evaluation attempt instead.
+
+## Output boundary
+
+`ready: true` means only that the sealed E3 readiness receipt is structurally
+and cryptographically hash-bound to the current public boundary. It does not by
+itself grant a quality, token, latency, generalization, production-stability, or
+release claim. Those claims require the later exact-RC FS5-FS7 evidence and an
+independent GO decision.

package/evals/private-holdout-v1/README.md

@@ -0,0 +1,28 @@
+# E3 private holdout custody boundary
+
+This directory contains only the public, provider-free boundary for `CF-FS4-04`.
+It does not contain a holdout suite, private repository, prompt, grader, oracle,
+reference solution, reviewer identity, raw score sheet, or private locator.
+
+The operational handoff is documented in `CUSTODIAN_RUNBOOK.md`. The readiness
+CLI checks the closed receipt, while the independent FS4-05 auditor must verify
+its custody origin outside this repository; authors cannot self-attest E3.
+
+The external custodian must:
+
+1. compare every private task-family lineage and repository lineage with
+   `public-exposure.v1.json` inside the controlled environment;
+2. keep candidate authors outside the custodian, reviewer, and adjudicator roles;
+3. expose an execute-only interface to the benchmark operator after an exact RC
+   is frozen;
+4. validate every reference solution and declared mutation;
+5. sample at least 12 items from at least four families, double-score all of
+   them with at least two blinded reviewers, preserve first-pass scores, and
+   independently adjudicate every disagreement;
+6. export only a schema-v2 `benchmark-assurance-evidence` receipt whose fields
+   are closed enums, counts, timestamps, booleans, and SHA-256 commitments.
+
+Schema-v1 assurance evidence remains parseable as historical metadata, but it
+cannot unlock a private-holdout/generalization claim. The release benchmark must
+still use the exact frozen RC, explicit provider approval, Piagent versus Codex
+CLI parity, and the later FS5-FS7 gates.