@piagent/platform 1.5.5 → 1.6.1

package/packages/piagent-core/benchmark/benchmark-bootstrap.js

@@ -13,9 +13,14 @@ import {
   recoverBenchmarkPiCredentialWriteback
 } from "./benchmark-pi-home.js";
 import { benchmarkTreeIdentity } from "./benchmark-tree-identity.js";
+import { inspectBenchmarkLedger } from "./benchmark-ledger.js";
+import { completedBenchmarkRecord } from "./benchmark-record-validation.js";
 import { benchmarkGitEnvironment, benchmarkHostEnvironment } from "./benchmark-runtime.js";
+import {
+  canonicalBuiltInBenchmarkSuiteId,
+  isReservedBenchmarkSuiteId
+} from "./benchmark-suite-identity.js";
 
-const builtInSuites = new Set(["core-v1", "capability-v1", "e2-framework-v1", "deep-logic-v1", "production-v1"]);
 const metadataVariable = "PIAGENT_BENCHMARK_BOOTSTRAP_METADATA";
 
 function fail(message) {
@@ -37,6 +42,67 @@ function runRootFromResume(input, cwd) {
   return stat.isDirectory() ? target : path.dirname(target);
 }
 
+function sameLedgerBinding(left, right) {
+  return ["schemaVersion", "algorithm", "digest", "records", "bytes"]
+    .every((field) => left?.[field] === right?.[field]);
+}
+
+function recordMatchesManifestOrder(record, index, manifest) {
+  const expected = manifest.order[index];
+  return completedBenchmarkRecord(record)
+    && record.runId === manifest.runId
+    && record.configurationDigest === manifest.configurationDigest
+    && record.orderIndex === index + 1
+    && record.scenarioId === expected?.scenarioId
+    && record.surface === expected?.surface
+    && record.repeat === expected?.repeat;
+}
+
+function ledgerMatchesManifestPrefix(records, manifest) {
+  const sessions = new Set();
+  const keys = new Set();
+  for (const [index, record] of records.entries()) {
+    const key = `${record?.scenarioId}\0${record?.surface}\0${record?.repeat}`;
+    if (!recordMatchesManifestOrder(record, index, manifest) || sessions.has(record.sessionId) || keys.has(key)) return false;
+    sessions.add(record.sessionId);
+    keys.add(key);
+  }
+  return true;
+}
+
+export function productionResumeHasCompleteLedger(argv, cwd) {
+  const resume = optionValue(argv, "--resume");
+  if (!resume) return false;
+  const runRoot = runRootFromResume(resume, cwd);
+  const manifestPath = path.join(runRoot, "run-manifest.json");
+  if (!fs.existsSync(manifestPath)) return false;
+  const manifest = jsonFile(manifestPath, "benchmark resume manifest");
+  if (manifest?.stageControl?.policy !== "production-v1-provider-spend-v1"
+    || !Array.isArray(manifest.order) || manifest.order.length === 0) return false;
+  const ledgerPath = path.join(runRoot, "runs.jsonl");
+  const ledger = inspectBenchmarkLedger(ledgerPath);
+  if (!ledgerMatchesManifestPrefix(ledger.records, manifest)) return false;
+  if (ledger.records.length === manifest.order.length) return true;
+  if (ledger.records.length !== manifest.order.length - 1) return false;
+
+  const pendingPath = path.join(runRoot, "pending-record.json");
+  if (!fs.existsSync(pendingPath)) return false;
+  const pending = jsonFile(pendingPath, "benchmark pending record");
+  const finalIndex = manifest.order.length - 1;
+  if (pending?.schemaVersion !== 2
+    || pending.postSessionGuard?.matched !== true
+    || !String(pending.postSessionGuard.stage ?? "").startsWith("after-session:")
+    || !sameLedgerBinding(pending.previousLedger, ledger.binding)
+    || !recordMatchesManifestOrder(pending.record, finalIndex, manifest)) return false;
+
+  const measuredPath = path.join(runRoot, "measured-record-ready.json");
+  if (!fs.existsSync(measuredPath)) return true;
+  const measured = jsonFile(measuredPath, "measured benchmark record");
+  return measured?.schemaVersion === 1
+    && sameLedgerBinding(measured.previousLedger, ledger.binding)
+    && JSON.stringify(measured.record) === JSON.stringify(pending.record);
+}
+
 function jsonFile(file, label) {
   try { return JSON.parse(fs.readFileSync(file, "utf8")); }
   catch (error) { fail(`Cannot read ${label} ${file}: ${error.message}`); }
@@ -98,15 +164,22 @@ function snapshotReplay(argv, cwd, temporaryRoot) {
   };
 }
 
-function resolveSuiteManifest(source, cwd) {
-  if (builtInSuites.has(source)) return { origin: source, manifest: null };
+function resolveSuiteManifest(source, cwd, liveRoot) {
+  if (isReservedBenchmarkSuiteId(source)) return { origin: source, manifest: null, builtInId: source };
   const target = path.resolve(cwd, source);
   let stat;
   try { stat = fs.statSync(target); }
   catch (error) { fail(`Cannot snapshot benchmark suite ${target}: ${error.message}`); }
   const manifest = stat.isDirectory() ? path.join(target, "suite.json") : target;
   if (!fs.statSync(manifest).isFile()) fail(`Benchmark suite manifest is not a file: ${manifest}`);
-  return { origin: fs.realpathSync(manifest), manifest: fs.realpathSync(manifest) };
+  const resolvedManifest = fs.realpathSync(manifest);
+  const builtInId = canonicalBuiltInBenchmarkSuiteId(liveRoot, resolvedManifest);
+  if (builtInId) return { origin: builtInId, manifest: null, builtInId };
+  const suite = jsonFile(resolvedManifest, "benchmark suite manifest");
+  if (isReservedBenchmarkSuiteId(suite?.id)) {
+    fail(`Benchmark suite id ${suite.id} is reserved for its canonical built-in suite`);
+  }
+  return { origin: resolvedManifest, manifest: resolvedManifest, builtInId: null };
 }
 
 function inside(root, target) {
@@ -273,12 +346,12 @@ function requestedPiProvider(model, configRoot, credentials) {
   return credentials.length === 1 ? credentials[0].providerId : null;
 }
 
-function snapshotPiAgentHome(temporaryRoot, runtimeParent, argv, cwd, replay) {
+function snapshotPiAgentHome(temporaryRoot, runtimeParent, argv, cwd, replay, { providerFreeFinalization = false } = {}) {
   const sourceRoot = path.resolve(process.env.PI_CODING_AGENT_DIR || path.join(os.homedir(), ".pi", "agent"));
   const configRoot = path.join(temporaryRoot, "pi-agent-config");
   fs.mkdirSync(configRoot, { recursive: true, mode: 0o700 });
   const copied = [];
-  for (const name of ["auth.json", "models.json"]) {
+  for (const name of providerFreeFinalization ? [] : ["auth.json", "models.json"]) {
     const source = path.join(sourceRoot, name);
     const configTarget = path.join(configRoot, name);
     try { copyBenchmarkPiConfigFile(source, configTarget, 0o400); }
@@ -395,10 +468,13 @@ export function createBenchmarkExecutionSnapshot({ liveRoot, argv, cwd }) {
     }
     const candidateIndex = writeBoundJson(path.join(temporaryRoot, "candidate-index.json"), candidate.index);
     const replay = snapshotReplay(argv, cwd, temporaryRoot);
-    const piAgentHome = snapshotPiAgentHome(temporaryRoot, runtimeParent, argv, cwd, replay);
-    const codexCredential = requestsCodexSurface(argv, cwd, replay) ? snapshotCodexCredential(temporaryRoot) : null;
-    const suite = resolveSuiteManifest(requestedSuite(argv, cwd, replay), cwd);
+    const suite = resolveSuiteManifest(requestedSuite(argv, cwd, replay), cwd, root);
     validateOutputIsolation(argv, cwd, root, suite.manifest);
+    const providerFreeFinalization = productionResumeHasCompleteLedger(argv, cwd);
+    const piAgentHome = snapshotPiAgentHome(temporaryRoot, runtimeParent, argv, cwd, replay, { providerFreeFinalization });
+    const codexCredential = !providerFreeFinalization && requestsCodexSurface(argv, cwd, replay)
+      ? snapshotCodexCredential(temporaryRoot)
+      : null;
     let suiteSnapshot = null;
     let suiteRoot;
     if (suite.manifest) {
@@ -421,6 +497,7 @@ export function createBenchmarkExecutionSnapshot({ liveRoot, argv, cwd }) {
       candidateProvenance: candidate.provenance,
       candidateIndex,
       runtimeDependencies: runtimeDependencies(candidateRoot, root),
+      providerFreeFinalization,
       piAgentHome,
       codexCredential,
       replay: replay ? {
@@ -432,6 +509,7 @@ export function createBenchmarkExecutionSnapshot({ liveRoot, argv, cwd }) {
       suite: {
         origin: suite.origin,
         snapshot: suiteSnapshot,
+        builtInId: suite.builtInId,
         identity: benchmarkTreeIdentity(suiteRoot, { rejectSymlinks: true })
       }
     };
@@ -485,6 +563,7 @@ export function benchmarkBootstrapMetadata(env = process.env) {
     || typeof value.candidateIndex?.path !== "string"
     || !/^[a-f0-9]{64}$/.test(String(value.candidateIndex?.digest ?? ""))
     || typeof value.runtimeDependencies?.digest !== "string"
+    || typeof value.providerFreeFinalization !== "boolean"
     || typeof value.piAgentHome?.configRoot !== "string"
     || typeof value.piAgentHome?.runtimeParent !== "string"
     || typeof value.piAgentHome?.vaultRoot !== "string"
@@ -503,6 +582,9 @@ export function benchmarkBootstrapMetadata(env = process.env) {
       || typeof value.codexCredential?.privateIdentity?.contentDigest !== "string"
     ))
     || typeof value.suite?.origin !== "string"
+    || !(value.suite?.builtInId === null || isReservedBenchmarkSuiteId(value.suite?.builtInId))
+    || (value.suite?.builtInId === null && typeof value.suite?.snapshot !== "string")
+    || (value.suite?.builtInId !== null && (value.suite?.origin !== value.suite?.builtInId || value.suite?.snapshot !== null))
     || typeof value.suite?.identity?.contentDigest !== "string"
     || (value.replay !== null && (
       typeof value.replay?.snapshot !== "string"

package/packages/piagent-core/benchmark/benchmark-claim-restrictions.js

@@ -1,21 +1,50 @@
-function diagnostic(report, { tier, purpose, reason, limitation, flag }) {
-  report[flag] = true;
-  report.comparison.purpose = purpose;
+const RESTRICTION_VERDICTS = new Set([
+  "token-claim-withheld",
+  "diagnostic-native-codex",
+  "diagnostic-replay"
+]);
+
+function restrictionCanReplaceVerdict(status, includeObservational) {
+  return typeof status === "string"
+    && (status.endsWith("-more-efficient")
+      || RESTRICTION_VERDICTS.has(status)
+      || (includeObservational && status === "observational-efficiency-only"));
+}
+
+function withholdTokenClaim(report, { reason, limitation, verdictStatus, includeObservational = false }) {
   report.comparison.tokenClaimAllowed = false;
   report.comparison.tokenClaimUnavailableReason = reason;
+  report.comparison.claimEligibility = {
+    ...(report.comparison.claimEligibility ?? {}),
+    tokenClaimScope: "unavailable",
+    limitations: [...new Set([...(report.comparison.claimEligibility?.limitations ?? []), limitation])]
+  };
+  report.verdict ??= {};
+  if (restrictionCanReplaceVerdict(report.verdict.status, includeObservational)) {
+    report.verdict.status = verdictStatus;
+  }
+  report.verdict.claimRestrictionReason = reason;
+}
+
+function diagnostic(report, { tier, purpose, reason, limitation, flag, verdictStatus }) {
+  report[flag] = true;
+  report.comparison.purpose = purpose;
+  withholdTokenClaim(report, { reason, limitation, verdictStatus, includeObservational: true });
   report.comparison.claimEligibility = {
     ...(report.comparison.claimEligibility ?? {}),
     achievedTier: tier,
     generalizationClaimAllowed: false,
-    comparisonPurpose: purpose,
-    limitations: [...new Set([...(report.comparison.claimEligibility?.limitations ?? []), limitation])]
+    comparisonPurpose: purpose
   };
 }
 
 export function applyBenchmarkClaimRestrictions(report, { tokenReason, replaySource, codexMode, surfaces }) {
   if (tokenReason) {
-    report.comparison.tokenClaimAllowed = false;
-    report.comparison.tokenClaimUnavailableReason = tokenReason;
+    withholdTokenClaim(report, {
+      reason: tokenReason,
+      limitation: `token-claim-withheld:${tokenReason}`,
+      verdictStatus: "token-claim-withheld"
+    });
   }
   if (surfaces.includes("codex-cli") && codexMode === "native") {
     diagnostic(report, {
@@ -23,6 +52,7 @@ export function applyBenchmarkClaimRestrictions(report, { tokenReason, replaySou
       purpose: "diagnostic-native-codex",
       reason: "native-codex-operator-configuration-is-not-frozen",
       limitation: "native-codex-results-cannot-support-release-token-or-generalization-claims",
+      verdictStatus: "diagnostic-native-codex",
       flag: "nativeCodexDiagnosticOnly"
     });
   }
@@ -32,6 +62,7 @@ export function applyBenchmarkClaimRestrictions(report, { tokenReason, replaySou
       purpose: "diagnostic-replay",
       reason: "replay-results-are-diagnostic-only",
       limitation: "replay-results-cannot-support-release-token-or-generalization-claims",
+      verdictStatus: "diagnostic-replay",
       flag: "replayDiagnosticOnly"
     });
   }

package/packages/piagent-core/benchmark/benchmark-cli.js

@@ -11,16 +11,16 @@ export const benchmarkUsage = `Usage:
   piagent-benchmark <project-path> --record ...   Legacy recorder
   piagent-benchmark <project-path> --init         Legacy scenario notes
 
-Runs a clean, paired baseline vs Piagent benchmark and grades it automatically.
-With no options it uses the built-in core-v1 suite, the current Pi default
-model/thinking setting, and the suite's repeat count.
+Runs a clean, paired Codex CLI vs Piagent benchmark and grades it automatically.
+With no options it uses the built-in core-v1 suite, GPT-5.6 Luna at medium
+thinking on both surfaces, and the suite's repeat count.
 
 Options:
   --suite <id|path>            core-v1, capability-v1, e2-framework-v1, deep-logic-v1, production-v1, or suite.json path.
-  --deep                       Alias for --suite deep-logic-v1 (6 large scenarios, current model by default).
+  --deep                       Alias for --suite deep-logic-v1 (7 large scenarios, locked to Luna/medium).
   --capability                 Alias for --suite capability-v1.
   --production                 Alias for --suite production-v1.
-  --surfaces <a,b>             raw-pi,piagent (default) or piagent,codex-cli.
+  --surfaces <a,b>             piagent,codex-cli (default); raw-pi is diagnostic ablation only.
   --model <provider/model>     Pin one model identity for both surfaces.
   --thinking <level>           off|minimal|low|medium|high|xhigh|max.
   --codex-mode <mode>          controlled isolated home (default) or native user configuration.
@@ -47,7 +47,7 @@ Options:
 
 Outputs report.json, report.html, summary.txt, and runs.jsonl with mode 0600.
 No token/result value is entered manually; usage comes from Pi session JSONL
-or Codex exec JSONL. Codex CLI comparisons require --model and --thinking.
+or Codex exec JSONL. Override --model and --thinking together for another parity target.
 `;
 
 function fail(message, code = 2) {
@@ -72,7 +72,7 @@ function positiveInteger(raw, name, minimum, maximum) {
 export function parseBenchmarkArgs(argv) {
   const options = {
     suite: "core-v1",
-    surfaces: ["raw-pi", "piagent"],
+    surfaces: ["piagent", "codex-cli"],
     model: undefined,
     thinking: undefined,
     codexMode: "controlled",