@piagent/platform 1.6.1 → 1.8.0

package/scripts/benchmark-runner-support.mjs

@@ -6,8 +6,14 @@ import readline from "node:readline/promises";
 import { fileURLToPath, pathToFileURL } from "node:url";
 
 import { productionStageResumeWindow } from "../packages/piagent-core/benchmark/benchmark-stage-diagnostic.js";
+import { productionV3FatalMeasurementEvidence } from "../packages/piagent-core/benchmark/benchmark-claim-restrictions.js";
+import { writeBenchmarkAbort } from "../packages/piagent-core/benchmark/benchmark-forensics.js";
+import { assertBenchmarkMeasurementOptions } from "../packages/piagent-core/benchmark/benchmark-cli.js";
 import { benchmarkLedgerCheckpoint, inspectBenchmarkLedger } from "../packages/piagent-core/benchmark/benchmark-ledger.js";
 import { acquireBenchmarkRunLock } from "../packages/piagent-core/benchmark/benchmark-run-lock.js";
+import { recoverBenchmarkObservationCheckpoints } from "../packages/piagent-core/benchmark/benchmark-resume-recovery.js";
+import { benchmarkSurfaceLabel } from "../packages/piagent-core/benchmark/benchmark-core.js";
+import { codexModelName, codexThinkingEffort } from "../packages/piagent-core/benchmark/benchmark-codex.js";
 
 export function fail(message, code = 2) {
   const error = new Error(message);
@@ -15,19 +21,57 @@ export function fail(message, code = 2) {
   throw error;
 }
 
+export function applyBenchmarkResumeOptions(options, resumeState) {
+  if (!resumeState) return;
+  const manifest = resumeState.manifest;
+  if (manifest.measurementOnly !== undefined && typeof manifest.measurementOnly !== "boolean") {
+    fail("Cannot resume benchmark: manifest measurementOnly must be a boolean", 1);
+  }
+  const measurementOnly = manifest.measurementOnly === true;
+  if (options.measurementOnly === true && !measurementOnly) {
+    fail("Cannot enable --measurement-only on an existing release manifest; start a new measurement-only run", 1);
+  }
+  assertBenchmarkMeasurementOptions({ ...options, measurementOnly });
+  assertBenchmarkMeasurementOptions({ ...manifest, measurementOnly });
+  options.measurementOnly = measurementOnly;
+  options.suite = manifest.suite?.source ?? manifest.suite?.manifestPath ?? manifest.suite?.id ?? options.suite;
+  options.surfaces = manifest.surfaces;
+  options.model = manifest.model ?? undefined;
+  options.thinking = manifest.thinking ?? undefined;
+  options.serviceTier = manifest.serviceTier ?? undefined;
+  options.codexMode = manifest.codexMode ?? "controlled";
+  options.codexBaseline = manifest.codexBaseline ?? "controlled-custom";
+  options.piagentTreatment = manifest.piagentTreatment ?? "release-defaults";
+  options.allowPiAuthWriteback = manifest.allowPiAuthWriteback === true;
+  options.seed = manifest.rootSeed;
+  options.repeats = manifest.repeats;
+  options.scenarioIds = manifest.scenarioIds ?? undefined;
+  options.timeoutSeconds = manifest.timeoutSeconds;
+  options.infrastructureRetries = manifest.infrastructureRetries;
+  options.retryDelaySeconds = manifest.retryDelaySeconds;
+  options.stopAfterFailedPair = manifest.stopAfterFailedPair === true;
+  options.campaignStopPolicy = manifest.campaignStopPolicy;
+  options.output = resumeState.runRoot;
+}
+
 export function frozenRuntimeCommandsForFinalization(manifest, surfaces) {
   const commands = manifest?.runtimeCommands;
   const validIdentity = (identity) => identity?.schemaVersion === 1
     && identity.kind === "regular"
     && typeof identity.resolvedPath === "string"
     && /^[a-f0-9]{64}$/.test(String(identity.contentDigest ?? ""));
-  const required = ["pi", "node", "git", "bash", ...(surfaces.includes("codex-cli") ? ["codex"] : [])];
+  const required = ["pi", "node", "git", "bash", ...(surfaces.includes("codex-cli") ? ["codex"] : []),
+    ...(manifest?.registeredMeasurement ? ["docker"] : [])];
   if (!commands || typeof commands !== "object" || required.some((label) => !validIdentity(commands[label]))) {
     fail("Cannot finalize production benchmark provider-free: frozen runtime command identities are missing or invalid", 1);
   }
   if (!surfaces.includes("codex-cli") && commands.codex !== null) {
     fail("Cannot finalize production benchmark provider-free: frozen Codex command identity does not match the surface list", 1);
   }
+  if (surfaces.includes("codex-cli") && manifest?.codexBaseline === "stock"
+    && commands.codex?.installationClosure?.kind !== "stock-codex-installation-v1") {
+    fail("Cannot finalize production benchmark provider-free: frozen stock Codex installation closure is missing", 1);
+  }
   return commands;
 }
 
@@ -41,6 +85,15 @@ export function privateDirectory(target) {
   return target;
 }
 
+export function registeredBenchmarkCustodyRoot(runRoot) {
+  const root = fs.realpathSync(runRoot);
+  const custodyRoot = fs.realpathSync(privateDirectory(path.join(root, "scoped-custody")));
+  if (custodyRoot !== path.join(root, "scoped-custody")) {
+    fail("Registered benchmark custody root must be a canonical direct child of the run root", 1);
+  }
+  return custodyRoot;
+}
+
 export function defaultOutputRoot(bootstrapMetadata) {
   if (bootstrapMetadata?.defaultOutputRoot) return bootstrapMetadata.defaultOutputRoot;
   const agentRoot = process.env.PI_CODING_AGENT_DIR || path.join(os.homedir(), ".pi", "agent");
@@ -79,8 +132,8 @@ function shellCommandArgument(value) {
   return `'${text.replaceAll("'", `'"'"'`)}'`;
 }
 
-export function benchmarkResumeCommand({ runRoot, productionSpendControlled, completedRuns, stageControl, stageBoundaries }) {
-  const base = `piagent-benchmark --resume ${shellCommandArgument(runRoot)}`;
+export function benchmarkResumeCommand({ runRoot, productionSpendControlled, completedRuns, stageControl, stageBoundaries, verificationRequired = false }) {
+  const base = `piagent-benchmark --resume ${shellCommandArgument(runRoot)}${verificationRequired ? " --approve-verification" : ""}`;
   if (!productionSpendControlled) return `${base} --yes`;
   const window = productionStageResumeWindow(stageControl, { completedRuns, stageBoundaries });
   return window?.remainingSessions > 0
@@ -111,16 +164,25 @@ function resolveResumeRunRoot(input) {
 export function loadResumeState(input) {
   const runRoot = resolveResumeRunRoot(input);
   const releaseRunLock = acquireBenchmarkRunLock(runRoot, "resume-pending");
+  let manifest;
+  let productionV3Finalized = false;
   try {
     const manifestPath = path.join(runRoot, "run-manifest.json");
     if (!fs.existsSync(manifestPath)) {
       fail(`Cannot resume ${runRoot}: missing run-manifest.json. This run was created before resume metadata was written, so its root seed cannot be recovered safely. Start a new run with --max-sessions or --max-runtime-minutes to make it resumable.`, 1);
     }
-    const manifest = readJsonFile(manifestPath, "benchmark resume manifest");
+    manifest = readJsonFile(manifestPath, "benchmark resume manifest");
+    productionV3Finalized = manifest?.suite?.id === "production-v3"
+      && (fs.existsSync(path.join(runRoot, "report.json"))
+        || ["claim-passed", "no-claim"].includes(manifest.campaignEvidence?.status));
+    if (productionV3Finalized) {
+      fail(`Cannot resume ${runRoot}: the production-v3 run already has a finalized report or campaign outcome`, 1);
+    }
     if (fs.existsSync(path.join(runRoot, "stopped.json"))) fail(`Cannot resume ${runRoot}: the paired release stop is terminal`, 1);
     if (manifest?.schemaVersion !== 1 || typeof manifest.runId !== "string") {
       fail(`Cannot resume ${runRoot}: run-manifest.json has an unsupported shape`, 1);
     }
+    const observationCheckpoints = recoverBenchmarkObservationCheckpoints({ runRoot, manifest });
     const ledger = benchmarkLedgerCheckpoint(
       manifest.ledger,
       inspectBenchmarkLedger(path.join(runRoot, "runs.jsonl")),
@@ -138,9 +200,17 @@ export function loadResumeState(input) {
       recoveredLedgerSuffix: ledger.recovered,
       pendingRecord,
       measuredReady,
+      observationCheckpoints,
       releaseRunLock
     };
   } catch (error) {
+    if (!productionV3Finalized && manifest?.suite?.id === "production-v3" && manifest.measurementOnly !== true) {
+      writeBenchmarkAbort(runRoot, {
+        runId: manifest.runId,
+        completedRuns: Number.isSafeInteger(manifest.ledger?.records) ? manifest.ledger.records : 0,
+        expectedRuns: Array.isArray(manifest.order) ? manifest.order.length : 108
+      }, error, { ledger: manifest.ledger ?? null, ...productionV3FatalMeasurementEvidence(error) });
+    }
     releaseRunLock();
     throw error;
   }
@@ -163,6 +233,68 @@ export function formatDuration(ms) {
   return `${seconds}s`;
 }
 
+export function benchmarkExecutionPlan({
+  packageVersion, suite, declaredScenarioCount, suiteDigest, options, comparison,
+  fullOrder, resumeState, pendingOrder, order, lifecycles, rootSeedDigest
+}) {
+  const plan = [
+    "Piagent automatic benchmark",
+    `  platform:  v${packageVersion}`,
+    `  suite:     ${suite.id} (${suite.scenarios.length}${suite.scenarios.length !== declaredScenarioCount ? `/${declaredScenarioCount}` : ""} scenarios)`,
+    ...(options.measurementOnly === true ? [options.failedAttemptsOnly
+      ? "  mode:      measurement-only · selected failed attempts · no full-matrix or release claim"
+      : "  mode:      measurement-only · full 108-session observation · no release claim"] : []),
+    ...(options.measurementOnly !== true && options.campaignStopPolicy === "measurement-invalidating-only"
+      ? ["  mode:      complete measurement · retain valid agent failures · adjudicate the release claim at S108"]
+      : []),
+    `  claim:     ${suite.assurance?.claimTier ?? "unavailable"} · family-disjoint=${suite.assurance?.familyDisjointSplit === true} · generalization ${suite.assurance?.familyDisjointSplit === true ? "bounded" : "unavailable"}`,
+    `  digest:    ${suiteDigest.slice(0, 16)}`,
+    `  surfaces:  ${options.surfaces.join(", ")}`,
+    `  compare:   ${benchmarkSurfaceLabel(comparison.candidateSurface)} vs ${benchmarkSurfaceLabel(comparison.baselineSurface)}`,
+    `  repeats:   ${options.repeats}`,
+    `  retries:   ${options.infrastructureRetries} infrastructure-only · ${options.retryDelaySeconds}s backoff`,
+    `  sessions:  ${fullOrder.length}`,
+    ...(resumeState ? [`  completed: ${resumeState.completedRuns.length}`, `  remaining: ${pendingOrder.length}`] : []),
+    ...(options.maxSessions !== undefined ? [`  chunk:    up to ${order.length}/${pendingOrder.length} remaining sessions`] : []),
+    ...(options.maxRuntimeMinutes !== undefined ? [`  budget:   ${options.maxRuntimeMinutes} minute runtime chunk`] : []),
+    ...(options.stopAfterFailedPair ? ["  stop:     terminal after a completed pair falls below the outcome floor"] : []),
+    `  model:     ${options.model ?? "Pi default"}`,
+    `  thinking:  ${options.thinking ?? "Pi default"}`,
+    `  tier:      ${options.serviceTier ?? "default"}`,
+    `  treatment: ${options.piagentTreatment}`,
+    `  lifecycle: ${lifecycles.join(", ")}`,
+    ...(options.replaySource ? [`  replay:    ${options.replaySource.runId ?? "prior-report"} · ${options.replayRuns.length} sessions`] : []),
+    ...(resumeState ? [`  resume:    ${resumeState.manifest.runId}`] : []),
+    `  variants:  ${suite.scenarios.some((scenario) => scenario.variantGenerator) ? `generated · seed ${rootSeedDigest.slice(0, 16)}` : "static"}`,
+    `  ordering:  ${options.failedAttemptsOnly ? "original failed-attempt ledger order" : suite.schemaVersion === 2 ? "seeded paired blocks" : "paired alternating"}`,
+    `  timeout:   ${options.timeoutSeconds}s per session`,
+    "  grading:   hidden verifier + scope + output safety + Pi task evidence"
+  ].join("\n");
+  const codexPlan = options.surfaces.includes("codex-cli")
+    ? `\n  codex:     ${options.codexBaseline} baseline · ${options.codexMode} config · model ${codexModelName(options.model)} · effort ${codexThinkingEffort(options.thinking)} · tier ${options.serviceTier ?? "default"}${options.codexMode === "controlled" ? " · isolated home" : ""}`
+    : "";
+  const nativeWarning = options.surfaces.includes("codex-cli") && options.codexMode === "native"
+    ? "\nNative Codex mode loads the operator's global AGENTS.md, configuration, rules, hooks, MCP servers, and plugins."
+    : "";
+  return { plan, codexPlan, nativeWarning };
+}
+
+export function bindBenchmarkTerminationSignals({ interrupted, interrupt, terminateAll }) {
+  const handlers = new Map();
+  for (const signal of ["SIGINT", "SIGTERM", "SIGHUP"]) {
+    const handler = () => {
+      if (interrupted()) return;
+      interrupt(signal);
+      terminateAll(signal);
+    };
+    handlers.set(signal, handler);
+    process.on(signal, handler);
+  }
+  return () => {
+    for (const [signal, handler] of handlers) process.off(signal, handler);
+  };
+}
+
 export function executionOrder(suite, repeats, surfaces, rootSeed) {
   const order = [];
   for (let repeat = 1; repeat <= repeats; repeat += 1) {

package/scripts/benchmark-runner.mjs

@@ -9,6 +9,12 @@ import {
   createBenchmarkExecutionSnapshot
 } from "../packages/piagent-core/benchmark/benchmark-bootstrap.js";
 import { assertBenchmarkLaunchEnvironmentSafe } from "../packages/piagent-core/benchmark/benchmark-runtime.js";
+import fs from "node:fs";
+import os from "node:os";
+import { parseBenchmarkArgs } from "../packages/piagent-core/benchmark/benchmark-cli.js";
+import { readBenchmarkBudgetControl } from "./benchmark-budget-runtime.mjs";
+import { startBenchmarkBudgetLaunch } from "./benchmark-budget-launcher.mjs";
+import { assertBenchmarkBudgetParentReceipts, finalizeBenchmarkBudgetPublication } from "./benchmark-budget-publication.mjs";
 
 const liveRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
 
@@ -25,57 +31,128 @@ function terminate(child, signal) {
   }
 }
 
-function runChild(script, argv, env, cleanup) {
+export function runBenchmarkChild(script, argv, env, cleanup, budgetLaunch) {
   return new Promise((resolve, reject) => {
     const loader = path.join(path.dirname(script), "register-typescript-loader.mjs");
     const child = spawn(process.execPath, ["--disable-warning=ExperimentalWarning", "--import", loader, script, ...argv], {
       cwd: process.cwd(),
       env,
       detached: process.platform !== "win32",
-      stdio: "inherit"
+      stdio: budgetLaunch ? ["inherit", "inherit", "inherit", "ipc"] : "inherit"
     });
     const handlers = new Map();
     for (const signal of ["SIGINT", "SIGTERM", "SIGHUP"]) {
-      const handler = () => terminate(child, signal);
+      const handler = () => { if (budgetLaunch) budgetLaunch.terminate(signal); else terminate(child, signal); };
       handlers.set(signal, handler);
       process.on(signal, handler);
     }
-    const finish = () => {
+    const finish = (cleanupAllowed = true) => {
       for (const [signal, handler] of handlers) process.off(signal, handler);
-      cleanup?.();
+      if (cleanupAllowed) cleanup?.();
     };
     let spawnError;
     child.once("error", (error) => { spawnError = error; });
-    child.once("close", (code, signal) => {
-      try { finish(); }
+    child.once("close", async (code, signal) => {
+      try {
+        let drained, cleanupAllowed = false;
+        try { drained = await budgetLaunch?.drain(); cleanupAllowed = !drained || drained.cleanupConfirmed === true; }
+        catch (error) { error.preserveBenchmarkSnapshot = true; throw error; }
+        finally { finish(cleanupAllowed); }
+        if (drained && !drained.cleanupConfirmed) {
+          throw Object.assign(new Error("Benchmark child process cleanup could not be confirmed; private snapshot retained"),
+            { preserveBenchmarkSnapshot: true });
+        }
+      }
       catch (error) { reject(error); return; }
       if (spawnError) { reject(spawnError); return; }
       if (signal) {
-        process.kill(process.pid, signal);
+        resolve(128 + (os.constants.signals[signal] ?? 1));
         return;
       }
       resolve(code ?? 1);
     });
+    try { budgetLaunch?.supervise(child); }
+    catch (error) { spawnError = error; terminate(child, "SIGTERM"); }
   });
 }
 
+// No cleanup/finalizer error may replace an earlier child error. A budgeted
+// report remains provisional unless every outer step and the durable receipt
+// succeed; a valid quality-failure exit code is not itself an outer failure.
+export async function completeBenchmarkOuterStage({ snapshot, preserveSnapshot = false, launch, control,
+  childExitCode, primaryError, cleanupSnapshot = cleanupBenchmarkExecutionSnapshot,
+  finalizePublication = finalizeBenchmarkBudgetPublication,
+  writeAccounting = accounting => process.stdout.write(`Budget stage accounting: ${JSON.stringify(accounting)}\n`) }) {
+  const outerErrorCodes = primaryError ? ["child-execution-error"] : [];
+  const remember = (error, code) => { primaryError ??= error; outerErrorCodes.push(code); };
+  let cleanupSucceeded = !snapshot, accounting = null, publication = null;
+  if (snapshot && !preserveSnapshot) {
+    try { await cleanupSnapshot(snapshot.temporaryRoot, snapshot.runtimeParent, snapshot.metadata.piAgentHome); cleanupSucceeded = true; }
+    catch (error) { remember(error, "snapshot-cleanup-failed"); }
+  } else if (snapshot) {
+    remember(new Error("Private benchmark snapshot retained because process cleanup is unconfirmed"), "snapshot-cleanup-unconfirmed");
+  }
+  if (launch) {
+    try {
+      accounting = launch.finish();
+      if (!accounting.launcherSucceeded) remember(Object.assign(
+        new Error(`Benchmark launcher stopped: ${accounting.launcherErrors.join(", ")}`), { exitCode: 1 }), "launcher-failed");
+    } catch (error) { remember(error, "budget-finalizer-failed"); }
+    if (accounting) {
+      try { await writeAccounting(accounting); } catch (error) { remember(error, "accounting-output-failed"); }
+    }
+    try {
+      publication = await finalizePublication({ control, stageId: JSON.parse(launch.context).stageId,
+        launcherReceipt: accounting, childExitCode: childExitCode ?? null,
+        outerSucceeded: !primaryError && cleanupSucceeded && accounting?.launcherSucceeded === true,
+        outerErrorCodes: [...outerErrorCodes] });
+      if (publication?.closureAllowed !== true) remember(Object.assign(
+        new Error("Benchmark parent receipt did not authorize stage closure"), { exitCode: 1 }), "budget-publication-denied");
+    } catch (error) { remember(error, "budget-publication-failed"); }
+  }
+  if (primaryError) {
+    primaryError.benchmarkOuterErrorCodes = outerErrorCodes;
+    throw primaryError;
+  }
+  return { code: childExitCode ?? 1, accounting, publication };
+}
+
 async function main() {
   const argv = process.argv.slice(2);
   if (legacyInvocation(argv) || argv.includes("--help") || argv.includes("-h")) {
-    return runChild(path.join(liveRoot, "scripts", "benchmark-runner-core.mjs"), argv, process.env);
+    return runBenchmarkChild(path.join(liveRoot, "scripts", "benchmark-runner-core.mjs"), argv, process.env);
   }
   assertBenchmarkLaunchEnvironmentSafe();
-  const snapshot = createBenchmarkExecutionSnapshot({ liveRoot, argv, cwd: process.cwd() });
-  const script = path.join(snapshot.candidateRoot, "scripts", "benchmark-runner-core.mjs");
-  return runChild(
-    script,
-    argv,
-    benchmarkBootstrapEnvironment(snapshot.metadata),
-    () => cleanupBenchmarkExecutionSnapshot(snapshot.temporaryRoot, snapshot.runtimeParent, snapshot.metadata.piAgentHome)
-  );
+  const options = parseBenchmarkArgs(argv);
+  const resumeManifestPath = options.resume ? path.join(options.resume, "run-manifest.json") : null;
+  if (resumeManifestPath && !fs.existsSync(resumeManifestPath)) {
+    throw new Error("Cannot resume benchmark: missing run-manifest.json; the root seed cannot be recovered safely");
+  }
+  const resumeManifest = options.resume
+    ? JSON.parse(fs.readFileSync(resumeManifestPath, "utf8")) : undefined;
+  const control = readBenchmarkBudgetControl({ options, resumeManifest, sourceRoot: liveRoot });
+  if (control && !options.dryRun && !options.preflightOnly) assertBenchmarkBudgetParentReceipts(control);
+  const launch = !options.dryRun && !options.preflightOnly
+    ? startBenchmarkBudgetLaunch(control, { resume: Boolean(options.resume) }) : null;
+  let snapshot, preserveSnapshot = false, childExitCode, primaryError;
+  try {
+    launch?.assertReady();
+    snapshot = createBenchmarkExecutionSnapshot({ liveRoot, argv, cwd: process.cwd() });
+    const script = path.join(snapshot.candidateRoot, "scripts", "benchmark-runner-core.mjs");
+    const environment = benchmarkBootstrapEnvironment(snapshot.metadata);
+    if (launch) environment[launch.environmentKey] = launch.context;
+    launch?.assertReady();
+    childExitCode = await runBenchmarkChild(script, argv, environment, undefined, launch);
+  } catch (error) {
+    preserveSnapshot = error.preserveBenchmarkSnapshot === true;
+    primaryError = error;
+  }
+  const completion = await completeBenchmarkOuterStage({ snapshot, preserveSnapshot, launch, control, childExitCode, primaryError });
+  return completion.code;
 }
 
-main()
+if (process.argv[1] && fs.existsSync(process.argv[1])
+  && fs.realpathSync(process.argv[1]) === fs.realpathSync(fileURLToPath(import.meta.url))) main()
   .then((code) => { process.exitCode = code; })
   .catch((error) => {
     console.error(`FAIL: ${error.message}`);