@piagent/platform 1.2.17 → 1.3.0-ie.9

package/scripts/benchmark-session.mjs

@@ -0,0 +1,412 @@
+import crypto from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+
+import {
+  aggregateSessionUsage,
+  createCodexExecJsonlCollector,
+  evaluateWorkflowEvidence
+} from "../packages/piagent-core/benchmark/benchmark-core.js";
+import { codexExecArgs } from "../packages/piagent-core/benchmark/benchmark-codex.js";
+import {
+  benchmarkEnvironment,
+  benchmarkGitEnvironment,
+  codexProcessEnvironment,
+  piagentProcessEnvironment
+} from "../packages/piagent-core/benchmark/benchmark-runtime.js";
+import {
+  acceptedTaskStartTraceCount,
+  benchmarkOperationalEvidence,
+  classifyPreUsageFailure,
+  failureReason,
+  safeInfrastructureDiagnostic,
+  terminalPiSessionError,
+  writePrivate,
+  writePrivateAtomic
+} from "../packages/piagent-core/benchmark/benchmark-forensics.js";
+import { readContextTelemetry } from "../packages/piagent-core/extensions/context-engine.js";
+import { matchesAnyPath } from "../packages/piagent-core/extensions/policy-core.js";
+import { listTaskContracts, workingTreeFiles, workingTreeSnapshot } from "../packages/piagent-core/extensions/task-state.js";
+import { benchmarkTreeIdentity } from "../packages/piagent-core/benchmark/benchmark-tree-identity.js";
+import { summarizeSession, walkJsonl } from "./pi-usage-history.mjs";
+
+const coldStartRuntimeManagedPaths = [
+  ".pi/project-context.md",
+  ".pi/context-index.json",
+  ".pi/piagent-state/project-onboarding.json",
+  ".pi/piagent-state/context-engine.json"
+];
+
+function fail(message) {
+  const error = new Error(message);
+  error.exitCode = 1;
+  throw error;
+}
+
+function privateDirectory(target) {
+  fs.mkdirSync(target, { recursive: true, mode: 0o700 });
+  try { fs.chmodSync(target, 0o700); } catch { /* Non-POSIX filesystem. */ }
+  return target;
+}
+
+function makeFixtureWritable(root) {
+  const pending = [root];
+  while (pending.length > 0) {
+    const current = pending.pop();
+    const stat = fs.lstatSync(current);
+    if (stat.isSymbolicLink()) continue;
+    if (stat.isDirectory()) {
+      fs.chmodSync(current, stat.mode | 0o700);
+      for (const name of fs.readdirSync(current)) pending.push(path.join(current, name));
+    } else if (stat.isFile()) {
+      fs.chmodSync(current, stat.mode | 0o600);
+    }
+  }
+}
+
+function graderEnvironment(scenarioId) {
+  const env = {};
+  for (const key of ["HOME", "PATH", "TMPDIR", "TMP", "TEMP", "LANG", "LC_ALL", "TZ", "SystemRoot"]) {
+    if (process.env[key]) env[key] = process.env[key];
+  }
+  return { ...env, NO_COLOR: "1", PIAGENT_BENCHMARK_SCENARIO: scenarioId };
+}
+
+function inside(parent, target) {
+  const relative = path.relative(parent, target);
+  return relative !== ".." && !relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative);
+}
+
+function safeWorkspaceFile(workspace, relativePath) {
+  const target = path.resolve(workspace, relativePath);
+  if (!inside(workspace, target)) fail(`Benchmark setup file escapes the workspace: ${relativePath}`);
+  return target;
+}
+
+async function initializeGit(runCommand, gitCommand, workspace, setupFiles) {
+  const commands = [
+    ["init", "-q"],
+    ["config", "user.email", "benchmark@piagent.local"],
+    ["config", "user.name", "Piagent Benchmark"],
+    ["add", "-A"]
+  ];
+  for (const args of commands) {
+    const result = await runCommand(gitCommand, args, { cwd: workspace, timeoutMs: 30_000, env: benchmarkGitEnvironment() });
+    if (result.code !== 0) fail(`Git fixture setup failed: git ${args.join(" ")}`);
+  }
+  for (const file of Object.keys(setupFiles ?? {})) {
+    const result = await runCommand(gitCommand, ["add", "-f", "--", file], { cwd: workspace, timeoutMs: 30_000, env: benchmarkGitEnvironment() });
+    if (result.code !== 0) fail(`Git could not track benchmark setup file ${file}`);
+  }
+  const commit = await runCommand(gitCommand, ["commit", "-qm", "benchmark fixture"], { cwd: workspace, timeoutMs: 30_000, env: benchmarkGitEnvironment() });
+  if (commit.code !== 0) fail("Git could not commit the benchmark fixture");
+}
+
+function applySetupFiles(workspace, setupFiles) {
+  for (const [relativePath, content] of Object.entries(setupFiles ?? {})) {
+    const target = safeWorkspaceFile(workspace, relativePath);
+    privateDirectory(path.dirname(target));
+    writePrivate(target, content);
+  }
+}
+
+function variantSeed(rootSeed, suiteDigest, scenarioId, repeat) {
+  return crypto.createHmac("sha256", rootSeed).update(`${suiteDigest}\0${scenarioId}\0${repeat}`).digest("hex");
+}
+
+function generatedStringArray(value, field) {
+  if (value === undefined) return [];
+  if (!Array.isArray(value) || value.length > 20 || value.some((item) => typeof item !== "string" || !item || item.length > 1_000)) {
+    fail(`Benchmark variant ${field} must contain at most 20 non-empty strings of at most 1000 characters`);
+  }
+  return [...new Set(value)];
+}
+
+async function generateVariant({ runCommand, nodeCommand, generator, workspace, oraclePath, seed, scenario, timeoutSeconds }) {
+  const result = await runCommand(nodeCommand, [generator, workspace, oraclePath, seed, scenario.id], {
+    cwd: path.dirname(generator), timeoutMs: Math.min(timeoutSeconds, 120) * 1_000, env: graderEnvironment(scenario.id)
+  });
+  if (result.timedOut) fail(`Benchmark variant generator timed out for ${scenario.id}`);
+  if (result.code !== 0) fail(`Benchmark variant generator failed for ${scenario.id}: ${result.stderr.trim() || result.stdout.trim()}`);
+  let oracle;
+  try { oracle = JSON.parse(fs.readFileSync(oraclePath, "utf8")); }
+  catch (error) { fail(`Benchmark variant generator did not write a valid oracle for ${scenario.id}: ${error.message}`); }
+  if (!oracle || typeof oracle !== "object" || Array.isArray(oracle) || oracle.schemaVersion !== 1 || !oracle.graderData || typeof oracle.graderData !== "object" || Array.isArray(oracle.graderData)) {
+    fail(`Benchmark variant oracle is invalid for ${scenario.id}`);
+  }
+  const serialized = JSON.stringify(oracle);
+  if (Buffer.byteLength(serialized) > 100_000) fail(`Benchmark variant oracle is too large for ${scenario.id}`);
+  writePrivate(oraclePath, `${JSON.stringify(oracle)}\n`);
+  return {
+    oraclePath,
+    oracleDigest: crypto.createHash("sha256").update(serialized).digest("hex"),
+    seedDigest: crypto.createHash("sha256").update(seed).digest("hex"),
+    requiredOutputSubstrings: generatedStringArray(oracle.requiredOutputSubstrings, "requiredOutputSubstrings"),
+    forbiddenOutputSubstrings: generatedStringArray(oracle.forbiddenOutputSubstrings, "forbiddenOutputSubstrings")
+  };
+}
+
+function writableCopyEnvironment(workspace) {
+  const systemCopy = ["/bin/cp", "/usr/bin/cp"].find((candidate) => fs.existsSync(candidate));
+  if (!systemCopy) fail("Piagent fixture initialization cannot locate the system copy command");
+  const shimRoot = privateDirectory(path.join(path.dirname(workspace), "bootstrap-bin"));
+  const shim = path.join(shimRoot, "cp");
+  fs.writeFileSync(shim, `#!/usr/bin/env node
+import fs from "node:fs";
+import { spawnSync } from "node:child_process";
+const args = process.argv.slice(2);
+const result = spawnSync(process.env.BENCHMARK_SYSTEM_CP, args, { stdio: "inherit" });
+if (result.error) throw result.error;
+if ((result.status ?? 1) !== 0) process.exit(result.status ?? 1);
+const target = args.at(-1);
+if (target && fs.existsSync(target)) {
+  const stat = fs.lstatSync(target);
+  if (!stat.isSymbolicLink()) fs.chmodSync(target, stat.mode | (stat.isDirectory() ? 0o700 : 0o600));
+}
+`, { mode: 0o700 });
+  return benchmarkEnvironment({
+    PIAGENT_NO_UPDATE_CHECK: "1",
+    BENCHMARK_SYSTEM_CP: systemCopy,
+    PATH: `${shimRoot}${path.delimiter}${process.env.PATH ?? ""}`
+  });
+}
+
+async function initializeTreatment(runCommand, systemCommands, packageRoot, workspace, profile) {
+  const result = await runCommand(systemCommands.bash, [
+    path.join(packageRoot, "scripts", "init-project.sh"), workspace, "--profile", profile, "--package-source", packageRoot
+  ], { cwd: packageRoot, timeoutMs: 60_000, env: writableCopyEnvironment(workspace) });
+  if (result.code !== 0) fail(`Piagent fixture initialization failed: ${result.stderr.trim() || result.stdout.trim()}`);
+}
+
+function prepareTreatmentBaseline(workspace, profileName, scenario) {
+  const profilePath = path.join(workspace, ".pi", "piagent-profile.json");
+  const contextPath = path.join(workspace, ".pi", "project-context.md");
+  const indexPath = path.join(workspace, ".pi", "context-index.json");
+  let profile;
+  let existingIndex;
+  try {
+    profile = JSON.parse(fs.readFileSync(profilePath, "utf8"));
+    existingIndex = JSON.parse(fs.readFileSync(indexPath, "utf8"));
+  } catch (error) {
+    fail(`Piagent benchmark baseline could not read initialized context: ${error.message}`);
+  }
+  const recordedAt = new Date().toISOString();
+  const projectId = typeof profile.projectId === "string" && profile.projectId.trim() ? profile.projectId.trim() : "piagent-benchmark-project";
+  const profileMode = typeof profile.mode === "string" && profile.mode.trim() ? profile.mode.trim() : profileName;
+  const sourceScope = (scenario.allowedChanges ?? []).join(", ") || "read-only";
+  const summary = `Pre-onboarded synthetic benchmark fixture for ${scenario.id}.`;
+  fs.writeFileSync(contextPath, [
+    "# Project Context", "", "## Status", "", `- Generated: ${recordedAt}`, `- Profile: ${profileMode}`,
+    "- Source: deterministic benchmark baseline", "- Verification: use the configured source verifier",
+    `- Task scope: ${sourceScope}`, "", "## Project", "",
+    "- Small synthetic Node.js fixture used to compare Raw Pi and Piagent.",
+    "- Source files and focused tests are authoritative.",
+    "- Keep implementation changes within the task scope supplied by the user.", ""
+  ].join("\n"));
+  const profileNode = `profile:${profileMode}`;
+  const contextNode = "context:.pi/project-context.md";
+  const index = {
+    schemaVersion: 1, projectId, profileMode, source: "onboarding-record", summary, generatedAt: recordedAt, updatedAt: recordedAt,
+    policy: { ...(existingIndex.policy ?? {}), ...(profile.contextIndex ?? {}) },
+    nodes: [
+      { id: profileNode, kind: "profile", label: profileMode, summary: "Active benchmark profile.", path: ".pi/piagent-profile.json", tags: ["profile", "benchmark"], citations: [{ path: ".pi/piagent-profile.json", reason: "Active benchmark profile" }], updatedAt: recordedAt },
+      { id: contextNode, kind: "context", label: ".pi/project-context.md", summary, path: ".pi/project-context.md", tags: ["snapshot", "benchmark"], citations: [{ path: "package.json", reason: "Synthetic project manifest" }], updatedAt: recordedAt }
+    ],
+    edges: [{ from: profileNode, to: contextNode, kind: "documented_by", reason: "Benchmark profile uses the prepared context." }],
+    citations: [{ path: "package.json", reason: "Synthetic project manifest" }], warnings: []
+  };
+  fs.writeFileSync(indexPath, `${JSON.stringify(index, null, 2)}\n`);
+  const stateRoot = privateDirectory(path.join(workspace, ".pi", "piagent-state"));
+  writePrivate(path.join(stateRoot, "project-onboarding.json"), `${JSON.stringify({
+    schemaVersion: 1, projectId, profileMode, contextFile: ".pi/project-context.md", summary, model: "benchmark-setup",
+    sourceFiles: [{ path: "package.json", reason: "Synthetic project manifest" }],
+    updateTriggers: ["fixture source or benchmark profile changes"],
+    notes: "Prepared outside measured model execution so task runs represent steady-state usage.", recordedAt
+  }, null, 2)}\n`);
+}
+
+async function prepareTreatmentContextEngine(runCommand, nodeCommand, packageRoot, workspace) {
+  const result = await runCommand(nodeCommand, [
+    path.join(packageRoot, "scripts", "context-engine.mjs"), "rebuild", "--project", workspace, "--json"
+  ], { cwd: packageRoot, timeoutMs: 60_000, env: benchmarkEnvironment({ PIAGENT_NO_UPDATE_CHECK: "1" }) });
+  if (result.code !== 0) fail(`Piagent benchmark context preparation failed: ${result.stderr.trim() || result.stdout.trim()}`);
+}
+
+function forbiddenSessionHits(sessionFiles, candidates) {
+  const hits = new Set();
+  for (const file of sessionFiles) {
+    for (const line of fs.readFileSync(file, "utf8").split("\n")) {
+      if (!line) continue;
+      let entry;
+      try { entry = JSON.parse(line); } catch { continue; }
+      if (entry.type !== "message" || !entry.message) continue;
+      const content = entry.message.content;
+      if (typeof content === "string") {
+        for (const value of candidates) if (content.includes(value)) hits.add(value);
+      } else if (Array.isArray(content)) {
+        for (const block of content) if (block?.type === "text" && typeof block.text === "string") {
+          for (const value of candidates) if (block.text.includes(value)) hits.add(value);
+        }
+      }
+    }
+  }
+  return [...hits];
+}
+
+function inspectForbiddenValue(value, candidates, hits) {
+  if (typeof value === "string") {
+    for (const candidate of candidates) if (value.includes(candidate)) hits.add(candidate);
+  } else if (Array.isArray(value)) {
+    for (const item of value) inspectForbiddenValue(item, candidates, hits);
+  } else if (value && typeof value === "object") {
+    for (const item of Object.values(value)) inspectForbiddenValue(item, candidates, hits);
+  }
+}
+
+function parseGraderResult(stdout) {
+  const lines = stdout.trim().split("\n").filter(Boolean);
+  let value;
+  try { value = JSON.parse(lines.at(-1) ?? ""); } catch { fail("Benchmark grader did not return a JSON object"); }
+  if (!value || typeof value !== "object" || Array.isArray(value) || typeof value.passed !== "boolean" || !Array.isArray(value.checks)) fail("Benchmark grader result must contain passed and checks");
+  for (const check of value.checks) if (!check || typeof check.id !== "string" || typeof check.passed !== "boolean") fail("Benchmark grader returned an invalid check");
+  return {
+    passed: value.passed,
+    score: Number.isFinite(value.score) ? Math.max(0, Math.min(10, value.score)) : value.passed ? 10 : 0,
+    checks: value.checks.map((check) => ({ id: check.id, passed: check.passed, detail: typeof check.detail === "string" ? check.detail.slice(0, 500) : undefined }))
+  };
+}
+
+async function gradeWorkspace(runCommand, nodeCommand, grader, workspace, scenario, timeoutSeconds, oraclePath) {
+  const result = await runCommand(nodeCommand, oraclePath ? [grader, workspace, oraclePath] : [grader, workspace], {
+    cwd: path.dirname(grader), timeoutMs: Math.min(timeoutSeconds, 120) * 1_000, env: graderEnvironment(scenario.id)
+  });
+  if (result.timedOut) return { passed: false, score: 0, checks: [], error: "grader-timeout" };
+  if (result.code !== 0) return { passed: false, score: 0, checks: [], error: `grader-exit-${result.code}` };
+  try { return parseGraderResult(result.stdout); } catch (error) { return { passed: false, score: 0, checks: [], error: error.message }; }
+}
+
+function sessionSummaries(sessionDir) {
+  return walkJsonl(sessionDir).map((file) => summarizeSession(file, {})).filter(Boolean);
+}
+
+export async function runBenchmarkSession({ packageRoot, runCommand, resolveSuiteEntry, interrupted, persistCompletedRecord, suite, suiteRoot, scenario, surface, repeat, orderIndex, infrastructureAttempt = 1, runId, runRoot, options, piCommand, codexCommand, codexDisabledFeatures, codexRuntime, piRuntimeHome, systemCommands, suiteDigest, configurationDigest, rootSeed }) {
+  if (surface !== "codex-cli" && !piRuntimeHome?.path) fail("Pi benchmark session is missing its controlled writable runtime home");
+  const attemptSuffix = infrastructureAttempt > 1 ? `-infra-${infrastructureAttempt}` : "";
+  const key = `${String(repeat).padStart(2, "0")}-${scenario.id}-${surface}${attemptSuffix}`;
+  const workspaceRoot = privateDirectory(path.join(runRoot, "workspaces", key));
+  const workspace = path.join(workspaceRoot, "project");
+  const sessions = privateDirectory(path.join(workspaceRoot, "sessions"));
+  fs.cpSync(resolveSuiteEntry(suiteRoot, scenario.fixture, "fixture"), workspace, { recursive: true, errorOnExist: true });
+  makeFixtureWritable(workspace);
+  applySetupFiles(workspace, scenario.setupFiles);
+  const profile = scenario.profile ?? suite.profile;
+  const lifecycle = scenario.lifecycle ?? "steady-state";
+  let variant = { oraclePath: null, oracleDigest: null, seedDigest: null, requiredOutputSubstrings: [], forbiddenOutputSubstrings: [] };
+  if (scenario.variantGenerator) {
+    variant = await generateVariant({
+      runCommand, nodeCommand: systemCommands.node, generator: resolveSuiteEntry(suiteRoot, scenario.variantGenerator, "variant generator"), workspace,
+      oraclePath: path.join(workspaceRoot, "oracle.json"), seed: variantSeed(rootSeed, suiteDigest, scenario.id, repeat),
+      scenario, timeoutSeconds: options.timeoutSeconds
+    });
+  }
+  const fixtureDigest = benchmarkTreeIdentity(workspace, { rejectSymlinks: true }).contentDigest;
+  const forbiddenOutputSubstrings = [...new Set([...(scenario.forbiddenOutputSubstrings ?? []), ...variant.forbiddenOutputSubstrings])];
+  const requiredOutputSubstrings = [...new Set([...(scenario.requiredOutputSubstrings ?? []), ...variant.requiredOutputSubstrings])];
+  if (surface === "piagent") {
+    await initializeTreatment(runCommand, systemCommands, packageRoot, workspace, profile);
+    if (lifecycle === "steady-state") {
+      prepareTreatmentBaseline(workspace, profile, scenario);
+      await prepareTreatmentContextEngine(runCommand, systemCommands.node, packageRoot, workspace);
+    }
+  }
+  await initializeGit(runCommand, systemCommands.git, workspace, scenario.setupFiles);
+  const prompt = fs.readFileSync(resolveSuiteEntry(suiteRoot, scenario.prompt, "prompt"), "utf8").trim();
+  const graderPath = resolveSuiteEntry(suiteRoot, scenario.grader, "grader");
+  const sessionId = crypto.randomUUID();
+  const attemptId = crypto.randomUUID();
+  const piArgs = ["--print", "--mode", "json", "--session-dir", sessions, "--session-id", sessionId, "--name", `BENCH ${scenario.id} ${surface} r${repeat}`, "--approve", "--no-skills", "--no-prompt-templates", "--no-extensions", "--no-context-files"];
+  if (surface === "piagent") {
+    const projectInstructions = path.join(workspace, "AGENTS.md");
+    if (!fs.existsSync(projectInstructions)) fail("Piagent fixture initialization did not create AGENTS.md");
+    piArgs.push("--append-system-prompt", projectInstructions, "--extension", path.join(packageRoot, "packages", "piagent-core", "extensions", "piagent-guard.ts"), "--skill", path.join(packageRoot, "packages", "piagent-core", "skills"));
+  }
+  if (options.model) piArgs.push("--model", options.model);
+  if (options.thinking) piArgs.push("--thinking", options.thinking);
+  piArgs.push(prompt);
+  const command = surface === "codex-cli" ? codexCommand : piCommand;
+  const args = surface === "codex-cli" ? codexExecArgs({ workspace, options, disabledFeatures: codexDisabledFeatures }) : piArgs;
+  const codexForbiddenHits = new Set();
+  const codexCollector = surface === "codex-cli" ? createCodexExecJsonlCollector({ model: options.model, thinkingLevel: options.thinking, onEvent: (event) => inspectForbiddenValue(event, forbiddenOutputSubstrings, codexForbiddenHits) }) : undefined;
+  const environment = {
+    PIAGENT_NO_UPDATE_CHECK: "1", PIAGENT_BENCHMARK_RUN_ID: runId, PIAGENT_BENCHMARK_SCENARIO: scenario.id,
+    PIAGENT_BENCHMARK_SURFACE: surface, PIAGENT_BENCHMARK_SESSION_ID: sessionId, PIAGENT_BENCHMARK_PROFILE: profile,
+    PIAGENT_BENCHMARK_LIFECYCLE: lifecycle, NO_COLOR: "1"
+  };
+  const inflightPath = path.join(workspaceRoot, "inflight.json");
+  writePrivateAtomic(inflightPath, `${JSON.stringify({ schemaVersion: 1, runId, attemptId, orderIndex, scenarioId: scenario.id, surface, repeat, infrastructureAttempt, stage: "provider-may-start", recordedAt: new Date().toISOString() }, null, 2)}\n`);
+  const agent = await runCommand(command, args, {
+    cwd: workspace, input: surface === "codex-cli" ? prompt : undefined, timeoutMs: options.timeoutSeconds * 1_000,
+    forbiddenSubstrings: forbiddenOutputSubstrings, requiredSubstrings: requiredOutputSubstrings,
+    onStdoutChunk: codexCollector ? (chunk) => codexCollector.write(chunk) : undefined,
+    env: surface === "codex-cli" ? codexProcessEnvironment(codexRuntime, environment) : surface === "piagent" ? piagentProcessEnvironment(options.piagentTreatment, { ...environment, PI_CODING_AGENT_DIR: piRuntimeHome.path }) : benchmarkEnvironment({ ...environment, PI_CODING_AGENT_DIR: piRuntimeHome.path })
+  });
+  const sessionFiles = surface === "codex-cli" ? [] : walkJsonl(sessions);
+  let usage;
+  let codexDiagnostics = [];
+  if (surface === "codex-cli") {
+    try { usage = codexCollector.finish(); }
+    catch (error) { if (agent.code === 0 && !agent.timedOut) throw error; usage = aggregateSessionUsage([]); }
+    codexDiagnostics = codexCollector.diagnostics();
+  } else usage = aggregateSessionUsage(sessionSummaries(sessions));
+  const piTerminalError = surface === "codex-cli" ? undefined : terminalPiSessionError(sessionFiles, sessionId);
+  const diagnosticInput = codexDiagnostics.length > 0
+    ? JSON.stringify(codexDiagnostics)
+    : piTerminalError ?? `${agent.stderr ?? ""}\n${agent.stdout ?? ""}`;
+  const preUsageFailure = classifyPreUsageFailure(agent, usage, diagnosticInput, { terminalProviderError: Boolean(piTerminalError) });
+  writePrivateAtomic(inflightPath, `${JSON.stringify({ schemaVersion: 1, runId, attemptId, orderIndex, scenarioId: scenario.id, surface, repeat, infrastructureAttempt, stage: "provider-returned", usage, recordedAt: new Date().toISOString() }, null, 2)}\n`);
+  const forbiddenHits = [...new Set([...(agent.forbiddenHits ?? []), ...(surface === "codex-cli" ? [...codexForbiddenHits] : forbiddenSessionHits(sessionFiles, forbiddenOutputSubstrings))])];
+  const requiredHits = new Set(agent.requiredHits ?? []);
+  if (surface !== "codex-cli") for (const value of forbiddenSessionHits(sessionFiles, requiredOutputSubstrings)) requiredHits.add(value);
+  const missingRequired = requiredOutputSubstrings.filter((value) => !requiredHits.has(value));
+  const allChangedFiles = workingTreeFiles(workspace);
+  const runtimeManagedChanges = surface === "piagent" && lifecycle === "cold-start" ? allChangedFiles.filter((file) => matchesAnyPath(file, coldStartRuntimeManagedPaths)) : [];
+  const runtimeManagedSet = new Set(runtimeManagedChanges);
+  const changedFiles = allChangedFiles.filter((file) => !runtimeManagedSet.has(file));
+  let workflow = null;
+  if (surface === "piagent" && scenario.kind !== "safety-refusal") {
+    const task = listTaskContracts(workspace).find((item) => item.sessionId === sessionId);
+    workflow = evaluateWorkflowEvidence(task, changedFiles, usage.toolNames, { scenarioKind: scenario.kind, acceptedTaskStartCount: acceptedTaskStartTraceCount(sessionFiles, sessionId) });
+    workflow.operational = benchmarkOperationalEvidence(readContextTelemetry(workspace, { limit: 5_000 }));
+  }
+  const beforeGrade = workingTreeSnapshot(workspace);
+  const outsideScope = changedFiles.filter((file) => !matchesAnyPath(file, scenario.allowedChanges));
+  const grade = preUsageFailure
+    ? { passed: false, score: 0, checks: [], error: preUsageFailure.failure }
+    : interrupted()
+    ? { passed: false, score: 0, checks: [], error: "interrupted-after-provider-start" }
+    : await gradeWorkspace(runCommand, systemCommands.node, graderPath, workspace, scenario, options.timeoutSeconds, variant.oraclePath);
+  const graderIntegrity = { passed: JSON.stringify(beforeGrade) === JSON.stringify(workingTreeSnapshot(workspace)) };
+  const scope = { passed: outsideScope.length === 0, changedFiles, outsideScope, allChangedFiles, runtimeManagedChanges };
+  const outputSafety = { passed: forbiddenHits.length === 0, forbiddenHits: forbiddenHits.map((value) => crypto.createHash("sha256").update(value).digest("hex")) };
+  const outputEvidence = { passed: missingRequired.length === 0, requiredCount: requiredOutputSubstrings.length, observedCount: requiredOutputSubstrings.length - missingRequired.length, missingHashes: missingRequired.map((value) => crypto.createHash("sha256").update(value).digest("hex")) };
+  const resolved = agent.code === 0 && !agent.timedOut && grade.passed && graderIntegrity.passed && scope.passed && outputSafety.passed && outputEvidence.passed;
+  const abortSuite = Boolean(preUsageFailure);
+  const record = {
+    schemaVersion: 1, runId, attemptId, configurationDigest, orderIndex, scenarioId: scenario.id, scenarioTitle: scenario.title, scenarioKind: scenario.kind,
+    category: scenario.category ?? "unspecified", difficulty: scenario.difficulty ?? "unspecified", profile, lifecycle,
+    surface, repeat, infrastructureAttempt, sessionId, providerSessionId: usage.providerSessionId ?? null, abortSuite,
+    infrastructureFailure: preUsageFailure?.failure, infrastructureClass: preUsageFailure?.class,
+    infrastructureRetryable: preUsageFailure?.retryable,
+    usageStatus: preUsageFailure?.usageStatus ?? "measured",
+    infrastructureDiagnostic: abortSuite ? safeInfrastructureDiagnostic(diagnosticInput, [...forbiddenOutputSubstrings, piRuntimeHome?.path].filter(Boolean)) : undefined,
+    infrastructureDiagnosticSource: abortSuite ? (codexDiagnostics.length > 0 ? "codex-error-events" : piTerminalError ? "pi-terminal-error-event" : "process-output-tail") : undefined,
+    resolved, failure: preUsageFailure?.failure ?? failureReason({ agent, grade, graderIntegrity, outsideScope, forbiddenHits, missingRequired }),
+    agent: { exitCode: agent.code, signal: agent.signal, timedOut: agent.timedOut, stdoutHash: agent.stdoutHash ?? crypto.createHash("sha256").update(agent.stdout).digest("hex"), stderrHash: crypto.createHash("sha256").update(agent.stderr ?? "").digest("hex") },
+    grade, graderIntegrity, scope, outputSafety, outputEvidence, workflow, usage, durationSeconds: agent.durationSeconds,
+    promptHash: crypto.createHash("sha256").update(prompt).digest("hex"),
+    variant: scenario.variantGenerator ? { generated: true, seedDigest: variant.seedDigest, oracleDigest: variant.oracleDigest, fixtureDigest } : { generated: false, fixtureDigest }
+  };
+  const workflowFailed = surface === "piagent" && (workflow?.checks ?? []).some((check) => check.passed === false);
+  if (!record.abortSuite && !interrupted()) persistCompletedRecord(record);
+  return { record, workspaceRoot, key, workflowFailed, inflightPath };
+}

package/scripts/build-docs-site.mjs

@@ -43,6 +43,12 @@ const NAV = [
         title: "Step by step cho thành viên mới",
         lead: "Hai lệnh để cài, rồi mở repo, chọn profile, bật quyền phù hợp, chạy task, và verify bằng bằng chứng thật trước khi handoff."
       },
+      {
+        slug: "whats-new",
+        nav: "Có gì mới",
+        title: "Luồng task an toàn hơn, ít thao tác tay hơn",
+        lead: "Product stabilization bổ sung runtime truth, trajectory, recovery, helper policy, activity inspector và completion receipt fail-closed mà không phá workflow cũ."
+      },
       {
         slug: "scope",
         nav: "Phạm vi platform",
@@ -70,13 +76,13 @@ const NAV = [
         slug: "commands",
         nav: "Command catalog",
         title: "Command ngắn, rõ, ít token burn",
-        lead: "Ưu tiên dùng slash command vì mỗi command đã gói sẵn policy và flow. Thành viên không cần nhớ lại quy trình trong prompt."
+        lead: "Slash command gói sẵn policy và flow; một inspector menu duy nhất cho diff, command, safety và context. Thành viên không cần nhớ nhiều command rời."
       },
       {
         slug: "workflows",
         nav: "Workflows",
-        title: "Các luồng chính",
-        lead: "Chọn workflow theo intent thay vì mô tả lại policy trong prompt. Cách này giảm context lãng phí và giữ hành vi nhất quán."
+        title: "Workflow thay đổi thế nào sau update",
+        lead: "Workflow launcher cũ vẫn giữ nguyên. Runtime lifecycle và live inspector chạy tự động; command mới chỉ giúp quan sát preflight, diff, tiến độ, receipt, hiệu suất và orchestration policy."
       }
     ]
   },
@@ -163,6 +169,11 @@ const EN_PAGE_COPY = {
     title: "Step by step for new team members",
     lead: "Install with two commands, open a repository, select its profile and permission mode, run a task, and verify real evidence before handoff."
   },
+  "whats-new": {
+    nav: "What’s new",
+    title: "Safer task flow with less manual coordination",
+    lead: "Product stabilization adds runtime truth, trajectories, recovery, helper policy, an activity inspector, and fail-closed completion receipts without breaking existing workflows."
+  },
   scope: {
     nav: "Platform scope",
     title: "What the platform owns",
@@ -181,12 +192,12 @@ const EN_PAGE_COPY = {
   commands: {
     nav: "Command catalog",
     title: "Short commands with predictable behavior",
-    lead: "Slash commands package policy and workflow directly, avoiding model-driven command discovery and unnecessary token use."
+    lead: "Slash commands package policy and workflow directly, with one inspector menu for diffs, commands, safety, and context instead of many one-off commands."
   },
   workflows: {
     nav: "Workflows",
-    title: "Core workflows",
-    lead: "Select a workflow by intent instead of repeating policy in every prompt. This keeps behavior consistent and context compact."
+    title: "How workflows change after the update",
+    lead: "Existing workflow launchers stay unchanged. The runtime lifecycle and live inspector update automatically, while commands expose preflight, diffs, progress, receipts, efficiency, and orchestration policy."
   },
   mcp: {
     nav: "MCP overview",

package/scripts/check-runtime-advisories.mjs

@@ -4,35 +4,9 @@
 //
 // The rule is "no moderate, high, or critical advisory in the pinned Pi host
 // and add-on tree".
-// What this adds is a way to accept one named advisory when there is provably
-// nothing to do about it, without lowering the bar for everything else.
-//
-// An accepted advisory is not forgotten:
-//   - it must still be present, so the entry disappears the moment upstream
-//     ships a fix rather than lingering as scar tissue;
-//   - it expires on a date, so accepting it can never become permanent by
-//     inattention.
-// Both of those fail the audit. A gate that quietly stops checking is worse
-// than no gate, because it reads as a pass.
-
-const ACCEPTED = [
-  {
-    id: "GHSA-mh99-v99m-4gvg",
-    package: "brace-expansion",
-    reviewBy: "2026-08-25",
-    reason: [
-      "Denial of service only: brace expansion can exhaust memory and crash the",
-      "process. No privilege escalation and no data exposure.",
-      "Nothing can fix it from here. The Pi host publishes an npm-shrinkwrap.json",
-      "that pins brace-expansion 5.0.7, and a published shrinkwrap takes",
-      "precedence over consumer overrides, so the resolution cannot be changed",
-      "by this repository. Every released Pi host carries a high brace-expansion",
-      "advisory: 0.81.1 and 0.82.0 carry this one, and 0.80.x carry the previous",
-      "GHSA-3jxr-9vmj-r5cp, so pinning an older host only trades one for another.",
-      "The fix is a regenerated shrinkwrap in a Pi release."
-    ].join(" ")
-  }
-];
+// There is deliberately no advisory allowlist. A supported host release must
+// audit clean at these severities; changing that policy requires changing this
+// gate and its tests explicitly.
 
 function readStdin() {
   return new Promise((resolve, reject) => {
@@ -53,12 +27,6 @@ function advisoryIdsFor(vulnerability) {
     .filter(Boolean);
 }
 
-function today() {
-  // Date-only comparison in UTC keeps the expiry independent of the runner's
-  // timezone.
-  return new Date().toISOString().slice(0, 10);
-}
-
 const raw = await readStdin();
 let report;
 try {
@@ -76,7 +44,6 @@ if (report.auditReportVersion !== 2) {
 }
 
 const blocking = [];
-const seen = new Set();
 for (const [name, vulnerability] of Object.entries(report.vulnerabilities ?? {})) {
   if (!["moderate", "high", "critical"].includes(vulnerability.severity)) continue;
   const ids = advisoryIdsFor(vulnerability);
@@ -85,23 +52,13 @@ for (const [name, vulnerability] of Object.entries(report.vulnerabilities ?? {})
     continue;
   }
   for (const id of ids) {
-    seen.add(id);
-    const accepted = ACCEPTED.find((entry) => entry.id === id && entry.package === name);
-    if (!accepted) blocking.push({ name, severity: vulnerability.severity, id });
+    blocking.push({ name, severity: vulnerability.severity, id });
   }
 }
 
-const failures = [];
-for (const entry of blocking) {
-  failures.push(`${entry.severity} advisory ${entry.id} in ${entry.name} is not accepted`);
-}
-for (const entry of ACCEPTED) {
-  if (!seen.has(entry.id)) {
-    failures.push(`accepted advisory ${entry.id} is no longer reported; remove it from ACCEPTED in ${"scripts/check-runtime-advisories.mjs"}`);
-  } else if (today() > entry.reviewBy) {
-    failures.push(`accepted advisory ${entry.id} passed its ${entry.reviewBy} review date; confirm it still applies or remove it`);
-  }
-}
+const failures = blocking.map(
+  (entry) => `${entry.severity} advisory ${entry.id} in ${entry.name} blocks the supported runtime`
+);
 
 if (failures.length > 0) {
   console.error("FAIL: runtime advisory policy");
@@ -110,6 +67,4 @@ if (failures.length > 0) {
 }
 
 const counts = report.metadata?.vulnerabilities ?? {};
-const acceptedList = ACCEPTED.map((entry) => `${entry.id} (review by ${entry.reviewBy})`).join(", ");
-console.log(`PASS: no unaccepted moderate, high, or critical advisory (${counts.moderate ?? 0} moderate, ${counts.high ?? 0} high, ${counts.critical ?? 0} critical)`);
-if (acceptedList) console.log(`Accepted, with reasons recorded in scripts/check-runtime-advisories.mjs: ${acceptedList}`);
+console.log(`PASS: no moderate, high, or critical advisory (${counts.moderate ?? 0} moderate, ${counts.high ?? 0} high, ${counts.critical ?? 0} critical)`);

package/scripts/configure-model-scope.sh

@@ -100,9 +100,9 @@ const codexModels = [
   "openai-codex/gpt-5.4-mini:minimal",
   "openai-codex/gpt-5.4:xhigh",
   "openai-codex/gpt-5.5:high",
-  "openai-codex/gpt-5.6-luna:xhigh",
-  "openai-codex/gpt-5.6-sol:xhigh",
-  "openai-codex/gpt-5.6-terra:xhigh"
+  "openai-codex/gpt-5.6-luna:medium",
+  "openai-codex/gpt-5.6-terra:high",
+  "openai-codex/gpt-5.6-sol:high"
 ];
 
 const claudeModels = [

package/scripts/evaluate-fs-release-transition.mjs

@@ -0,0 +1,27 @@
+#!/usr/bin/env node
+import crypto from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { evaluateFsReleaseTransition } from "../packages/piagent-core/benchmark/fs-release-transition.js";
+
+const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
+const inputPath = path.join(root, "evals", "fs-release-transition.v1.json");
+const inputBytes = fs.readFileSync(inputPath);
+const input = JSON.parse(inputBytes);
+const result = evaluateFsReleaseTransition(input);
+const evidencePath = path.join(root, input.fs5Closure.risk.evidencePath);
+const evidenceSha256 = fs.existsSync(evidencePath) ? crypto.createHash("sha256").update(fs.readFileSync(evidencePath)).digest("hex") : null;
+if (evidenceSha256 !== input.fs5Closure.risk.evidenceSha256) {
+  result.status = "failed";
+  result.errors.push("FS5 terminal evidence is missing or has drifted");
+  result.rcAssemblyAllowed = false;
+  result.nextWorkItem = null;
+}
+process.stdout.write(`${JSON.stringify({
+  transitionVersion: input.transitionVersion,
+  transitionDigest: crypto.createHash("sha256").update(inputBytes).digest("hex"),
+  evidenceSha256,
+  ...result
+}, null, 2)}\n`);
+if (result.status !== "passed") process.exitCode = 1;