@piagent/platform 1.5.4 → 1.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (230) hide show
  1. package/CHANGELOG.md +136 -0
  2. package/README.md +9 -8
  3. package/README.vi.md +9 -5
  4. package/adapters/web-frontend/profile.json +2 -4
  5. package/architecture/layers.json +18 -3
  6. package/benchmarks/deep-logic-v1/SPECIALIST_PROTOCOL.vi.md +122 -0
  7. package/benchmarks/deep-logic-v1/grade.mjs +272 -10
  8. package/benchmarks/deep-logic-v1/project/apps/admin/src/billing-summary.js +3 -0
  9. package/benchmarks/deep-logic-v1/project/benchmark-contract/public-smoke.test.js +166 -0
  10. package/benchmarks/deep-logic-v1/project/package.json +1 -1
  11. package/benchmarks/deep-logic-v1/project/packages/billing/src/close-period.js +3 -0
  12. package/benchmarks/deep-logic-v1/project/packages/billing/src/plan-timeline.js +3 -0
  13. package/benchmarks/deep-logic-v1/project/test/smoke.test.js +7 -1
  14. package/benchmarks/deep-logic-v1/prompts/budgeted-context-graph.md +2 -0
  15. package/benchmarks/deep-logic-v1/prompts/fair-dependency-scheduler.md +2 -0
  16. package/benchmarks/deep-logic-v1/prompts/layered-policy-resolution.md +2 -0
  17. package/benchmarks/deep-logic-v1/prompts/resumable-stream-assembly.md +3 -1
  18. package/benchmarks/deep-logic-v1/prompts/revision-event-reconciliation.md +2 -0
  19. package/benchmarks/deep-logic-v1/prompts/temporal-usage-billing-close.md +123 -0
  20. package/benchmarks/deep-logic-v1/prompts/transactional-config-merge.md +2 -0
  21. package/benchmarks/deep-logic-v1/suite.json +51 -3
  22. package/benchmarks/deep-logic-v1/variant.mjs +22 -1
  23. package/benchmarks/production-v1/spend-control.v1.json +68 -0
  24. package/benchmarks/production-v1/suite.json +42 -3
  25. package/catalog/capabilities.json +6 -6
  26. package/docs/command-reference-vietnamese.md +6 -6
  27. package/docs/context-engine.md +7 -1
  28. package/docs/distribution-standard.md +7 -7
  29. package/docs/en/architecture.md +1 -0
  30. package/docs/languages.json +4 -4
  31. package/docs/model-options.md +8 -5
  32. package/docs/operator-manual-vietnamese.md +13 -11
  33. package/docs/project-adapters.md +2 -2
  34. package/docs/publishing-for-teams.md +9 -0
  35. package/docs/quality-benchmark.md +271 -62
  36. package/docs/release-install-policy.md +10 -9
  37. package/docs/runtime-policy-design.md +4 -4
  38. package/docs/runtime-quality-baseline.md +13 -6
  39. package/docs/security-threat-model.md +2 -2
  40. package/docs/task-lifecycle-vietnamese.md +2 -2
  41. package/docs/team-onboarding.md +2 -2
  42. package/docs/usage-observability.md +7 -2
  43. package/docs/vi/architecture.md +1 -0
  44. package/docs-site/architecture.html +1 -1
  45. package/docs-site/benchmark.html +1 -1
  46. package/docs-site/commands.html +1 -1
  47. package/docs-site/context-engine.html +1 -1
  48. package/docs-site/en/architecture.html +1 -1
  49. package/docs-site/en/benchmark.html +1 -1
  50. package/docs-site/en/commands.html +1 -1
  51. package/docs-site/en/context-engine.html +1 -1
  52. package/docs-site/en/index.html +1 -1
  53. package/docs-site/en/mcp-approval.html +1 -1
  54. package/docs-site/en/mcp-auth.html +1 -1
  55. package/docs-site/en/mcp-servers.html +1 -1
  56. package/docs-site/en/mcp.html +1 -1
  57. package/docs-site/en/permissions.html +1 -1
  58. package/docs-site/en/profiles.html +1 -1
  59. package/docs-site/en/quickstart.html +1 -1
  60. package/docs-site/en/runtime.html +1 -1
  61. package/docs-site/en/scope.html +1 -1
  62. package/docs-site/en/security.html +1 -1
  63. package/docs-site/en/team.html +1 -1
  64. package/docs-site/en/whats-new.html +1 -1
  65. package/docs-site/en/workflows.html +1 -1
  66. package/docs-site/index.html +1 -1
  67. package/docs-site/mcp-approval.html +1 -1
  68. package/docs-site/mcp-auth.html +1 -1
  69. package/docs-site/mcp-servers.html +1 -1
  70. package/docs-site/mcp.html +1 -1
  71. package/docs-site/permissions.html +1 -1
  72. package/docs-site/profiles.html +1 -1
  73. package/docs-site/quickstart.html +1 -1
  74. package/docs-site/runtime.html +1 -1
  75. package/docs-site/scope.html +1 -1
  76. package/docs-site/security.html +1 -1
  77. package/docs-site/team.html +1 -1
  78. package/docs-site/whats-new.html +1 -1
  79. package/docs-site/workflows.html +1 -1
  80. package/evals/fixtures/benchmark-assurance-evidence.invalid.json +1 -1
  81. package/evals/fixtures/benchmark-assurance-evidence.valid.json +1 -1
  82. package/evals/fixtures/piagent-webui/session-live-state-v1.invalid.json +18 -0
  83. package/evals/fixtures/piagent-webui/session-live-state-v1.valid.json +27 -0
  84. package/evals/fs4-readiness-matrix.v1.json +2 -2
  85. package/evals/fs5-causal-arm.v1.json +2 -2
  86. package/evals/fs5-pilot-protocol.v1.json +2 -2
  87. package/evals/fs5-pilot-protocol.v4.json +6 -5
  88. package/evals/fs5-pilot-protocol.v5.json +7 -6
  89. package/evals/private-holdout-v1/public-exposure.v1.json +1 -1
  90. package/package.json +3 -1
  91. package/packages/piagent-core/benchmark/benchmark-assurance.js +4 -1
  92. package/packages/piagent-core/benchmark/benchmark-bootstrap.js +91 -9
  93. package/packages/piagent-core/benchmark/benchmark-claim-restrictions.js +38 -7
  94. package/packages/piagent-core/benchmark/benchmark-cli.js +7 -7
  95. package/packages/piagent-core/benchmark/benchmark-codex.js +3 -1
  96. package/packages/piagent-core/benchmark/benchmark-comparison.js +271 -2
  97. package/packages/piagent-core/benchmark/benchmark-core.js +732 -661
  98. package/packages/piagent-core/benchmark/benchmark-execution-guard.js +7 -5
  99. package/packages/piagent-core/benchmark/benchmark-forensics.js +439 -4
  100. package/packages/piagent-core/benchmark/benchmark-host-readiness.js +639 -0
  101. package/packages/piagent-core/benchmark/benchmark-matrix.js +22 -4
  102. package/packages/piagent-core/benchmark/benchmark-normalized-cost.js +317 -0
  103. package/packages/piagent-core/benchmark/benchmark-preflight.js +3 -1
  104. package/packages/piagent-core/benchmark/benchmark-process.js +6 -5
  105. package/packages/piagent-core/benchmark/benchmark-provider-wire.js +126 -0
  106. package/packages/piagent-core/benchmark/benchmark-record-validation.js +183 -0
  107. package/packages/piagent-core/benchmark/benchmark-report.js +116 -12
  108. package/packages/piagent-core/benchmark/benchmark-resume-recovery.js +14 -0
  109. package/packages/piagent-core/benchmark/benchmark-runner-policy.js +26 -0
  110. package/packages/piagent-core/benchmark/benchmark-stage-diagnostic.js +782 -0
  111. package/packages/piagent-core/benchmark/benchmark-statistics.js +64 -0
  112. package/packages/piagent-core/benchmark/benchmark-stop-policy.js +8 -5
  113. package/packages/piagent-core/benchmark/benchmark-suite-identity.js +31 -0
  114. package/packages/piagent-core/benchmark/benchmark-suite-runtime.js +13 -9
  115. package/packages/piagent-core/benchmark/benchmark-suite.js +128 -2
  116. package/packages/piagent-core/benchmark/benchmark-summary-support.js +376 -0
  117. package/packages/piagent-core/benchmark/benchmark-timing-diagnostics.js +971 -0
  118. package/packages/piagent-core/benchmark/benchmark-usage.js +311 -0
  119. package/packages/piagent-core/benchmark/benchmark-workflow.js +166 -0
  120. package/packages/piagent-core/docs/context-efficiency-improvement-plan.md +1080 -0
  121. package/packages/piagent-core/extensions/acceptance-contract-semantics.js +83 -69
  122. package/packages/piagent-core/extensions/acceptance-error-classes.js +160 -0
  123. package/packages/piagent-core/extensions/acceptance-executable-evidence.js +12 -9
  124. package/packages/piagent-core/extensions/acceptance-receipt.js +34 -11
  125. package/packages/piagent-core/extensions/acceptance-state-machine-evidence.js +518 -0
  126. package/packages/piagent-core/extensions/acceptance-test-binding-evidence.js +60 -0
  127. package/packages/piagent-core/extensions/context-efficiency-metrics.js +302 -0
  128. package/packages/piagent-core/extensions/context-efficiency-report.js +167 -0
  129. package/packages/piagent-core/extensions/context-engine.js +148 -145
  130. package/packages/piagent-core/extensions/context-evidence.js +48 -0
  131. package/packages/piagent-core/extensions/context-import-links.js +296 -0
  132. package/packages/piagent-core/extensions/context-telemetry-integrity.js +65 -0
  133. package/packages/piagent-core/extensions/criterion-context-pack.js +422 -7
  134. package/packages/piagent-core/extensions/criterion-graph.js +78 -8
  135. package/packages/piagent-core/extensions/guard-types.ts +3 -0
  136. package/packages/piagent-core/extensions/javascript-regex-evidence.js +32 -0
  137. package/packages/piagent-core/extensions/piagent-guard.ts +250 -43
  138. package/packages/piagent-core/extensions/policy-core.js +1 -1
  139. package/packages/piagent-core/extensions/task-contract-view.js +17 -3
  140. package/packages/piagent-core/extensions/task-lifecycle.js +7 -2
  141. package/packages/piagent-core/extensions/task-state.js +40 -134
  142. package/packages/piagent-core/extensions/verification-intelligence.js +3 -1
  143. package/packages/piagent-core/extensions/workspace-evidence-roots.js +157 -0
  144. package/packages/piagent-core/package.json +1 -1
  145. package/packages/piagent-core/policies/base-policy.json +3 -0
  146. package/packages/piagent-core/runtime/context/context-delivery.ts +103 -0
  147. package/packages/piagent-core/runtime/context/context-delta-shadow.ts +53 -0
  148. package/packages/piagent-core/runtime/context/context-efficiency-format.ts +47 -0
  149. package/packages/piagent-core/runtime/context/context-evidence-qualification.ts +32 -0
  150. package/packages/piagent-core/runtime/context/prefix-telemetry.ts +47 -0
  151. package/packages/piagent-core/runtime/context/repository-memory-hints.ts +27 -0
  152. package/packages/piagent-core/runtime/hooks/agent-prompt-telemetry.ts +56 -0
  153. package/packages/piagent-core/runtime/hooks/agent-start-hook.ts +162 -102
  154. package/packages/piagent-core/runtime/hooks/completion-hook.ts +5 -6
  155. package/packages/piagent-core/runtime/hooks/input-hook.ts +11 -1
  156. package/packages/piagent-core/runtime/hooks/session-hooks.ts +22 -2
  157. package/packages/piagent-core/runtime/hooks/session-start-hook.ts +7 -2
  158. package/packages/piagent-core/runtime/hooks/tool-result-content-guards.ts +95 -0
  159. package/packages/piagent-core/runtime/hooks/tool-result-hook.ts +84 -102
  160. package/packages/piagent-core/runtime/inspection/activity-event-adapter.ts +9 -3
  161. package/packages/piagent-core/runtime/inspection/activity-recovery.ts +104 -0
  162. package/packages/piagent-core/runtime/inspection/approval-broker.ts +29 -5
  163. package/packages/piagent-core/runtime/inspection/task-compaction-history.ts +1 -1
  164. package/packages/piagent-core/runtime/inspection/task-handoff-history.ts +1 -1
  165. package/packages/piagent-core/runtime/inspection/tool-failure-classification.ts +53 -0
  166. package/packages/piagent-core/runtime/inspection/webui-snapshot.ts +34 -6
  167. package/packages/piagent-core/runtime/model/authenticated-catalog.ts +17 -9
  168. package/packages/piagent-core/runtime/model/openai-codex-reasoning.ts +83 -0
  169. package/packages/piagent-core/runtime/model/provider-wire-fingerprint.ts +238 -0
  170. package/packages/piagent-core/runtime/orchestration/role-policy.ts +1 -1
  171. package/packages/piagent-core/runtime/product/activity-inspector.ts +7 -2
  172. package/packages/piagent-core/runtime/product/efficiency-metrics.ts +4 -2
  173. package/packages/piagent-core/runtime/quality/exact-output-contract.ts +2 -1
  174. package/packages/piagent-core/runtime/quality/performance-assurance-signals.ts +23 -0
  175. package/packages/piagent-core/runtime/quality/performance-assurance.ts +6 -2
  176. package/packages/piagent-core/runtime/recovery/edit-recovery-context.ts +113 -0
  177. package/packages/piagent-core/runtime/recovery/edit-recovery-delivery.ts +42 -0
  178. package/packages/piagent-core/runtime/recovery/semantic-repair-handshake.ts +55 -2
  179. package/packages/piagent-core/runtime/recovery/semantic-repair-runtime.ts +55 -2
  180. package/packages/piagent-core/runtime/registration/context-commands.ts +66 -5
  181. package/packages/piagent-core/runtime/registration/policy-tools.ts +46 -11
  182. package/packages/piagent-core/runtime/registration/task-completion-tools.ts +49 -16
  183. package/packages/piagent-core/runtime/registration/task-start-guidance.ts +163 -0
  184. package/packages/piagent-core/runtime/registration/task-start-retry-helpers.ts +37 -0
  185. package/packages/piagent-core/runtime/registration/task-start-tool.ts +67 -62
  186. package/packages/piagent-core/runtime/session/runtime-state.ts +154 -2
  187. package/packages/piagent-core/runtime/session/system-prompt.ts +4 -4
  188. package/packages/piagent-core/runtime/tools/apply-patch-tool.ts +500 -0
  189. package/packages/piagent-core/runtime/trajectory/trajectory-runtime.ts +2 -1
  190. package/packages/piagent-core/runtime/workflows/task-intake.ts +43 -2
  191. package/packages/piagent-core/security/sensitive-data.js +13 -218
  192. package/packages/piagent-core/security/sensitive-project-file.js +42 -0
  193. package/packages/piagent-core/security/sensitive-source-expression.js +330 -0
  194. package/packages/piagent-core/security/sensitive-source-formats.js +394 -0
  195. package/packages/piagent-core/security/sensitive-text.js +234 -0
  196. package/packages/piagent-core/subagents/piagent-worker.md +1 -1
  197. package/packages/piagent-webui/dist/client/assets/{MarkdownMessage-rD9WJKxs.js → MarkdownMessage-CxeZqTte.js} +1 -1
  198. package/packages/piagent-webui/dist/client/assets/index-DynIMe8I.js +176 -0
  199. package/packages/piagent-webui/dist/client/index.html +1 -1
  200. package/packages/piagent-webui/gateway/gateway-events.ts +109 -5
  201. package/packages/piagent-webui/gateway/gateway-service.ts +3 -0
  202. package/packages/piagent-webui/gateway/gateway-session-stream.ts +93 -7
  203. package/packages/piagent-webui/gateway/session-command-controller.ts +18 -3
  204. package/packages/piagent-webui/gateway/session-effective-options.ts +47 -0
  205. package/packages/piagent-webui/gateway/session-live-state.ts +21 -0
  206. package/packages/piagent-webui/gateway/session-operation-start.ts +5 -1
  207. package/packages/piagent-webui/gateway/session-operation-watchdog.ts +158 -0
  208. package/packages/piagent-webui/gateway/session-runtime-factory.ts +45 -0
  209. package/packages/piagent-webui/gateway/session-runtime-supervisor.ts +139 -120
  210. package/packages/piagent-webui/server/loopback-server.ts +8 -0
  211. package/packages/piagent-webui/server/transcript-projection.ts +22 -4
  212. package/packages/piagent-webui/shared/text-visibility.ts +10 -0
  213. package/schemas/piagent-webui/catalog-v1.json +9 -0
  214. package/schemas/piagent-webui/gateway-protocol-v1.schema.json +53 -3
  215. package/schemas/piagent-webui/session-live-state-v1.schema.json +76 -0
  216. package/schemas/task-contract.schema.json +12 -1
  217. package/scripts/benchmark-runner-core.mjs +355 -359
  218. package/scripts/benchmark-runner-finalization.mjs +309 -0
  219. package/scripts/benchmark-runner-host-readiness.mjs +94 -0
  220. package/scripts/benchmark-runner-support.mjs +200 -0
  221. package/scripts/benchmark-session.mjs +95 -41
  222. package/scripts/configure-model-scope.sh +4 -3
  223. package/scripts/helper-evaluation.mjs +1 -1
  224. package/scripts/install-global.sh +2 -2
  225. package/scripts/pi-usage-history.mjs +37 -10
  226. package/scripts/piagent-dashboard.mjs +13 -1
  227. package/scripts/setup.sh +2 -2
  228. package/scripts/verify-local.sh +5 -1
  229. package/templates/global/settings.json +2 -1
  230. package/packages/piagent-webui/dist/client/assets/index-BZRFw1u6.js +0 -174
@@ -1,26 +1,17 @@
1
1
  #!/usr/bin/env node
2
2
  import crypto from "node:crypto";
3
3
  import fs from "node:fs";
4
- import os from "node:os";
5
4
  import path from "node:path";
6
- import readline from "node:readline/promises";
7
- import { fileURLToPath, pathToFileURL } from "node:url";
5
+ import { fileURLToPath } from "node:url";
8
6
 
9
- import {
10
- aggregateSessionUsage, benchmarkSurfaceLabel,
11
- renderBenchmarkHtml,
12
- renderBenchmarkText,
13
- summarizeBenchmark
14
- } from "../packages/piagent-core/benchmark/benchmark-core.js";
7
+ import { aggregateSessionUsage, benchmarkSurfaceLabel } from "../packages/piagent-core/benchmark/benchmark-core.js";
15
8
  import { benchmarkUsage, parseBenchmarkArgs } from "../packages/piagent-core/benchmark/benchmark-cli.js";
16
9
  import { codexModelName, codexThinkingEffort } from "../packages/piagent-core/benchmark/benchmark-codex.js";
17
- import { benchmarkTrustChecklist } from "../packages/piagent-core/benchmark/benchmark-matrix.js";
18
- import { applyBenchmarkClaimRestrictions } from "../packages/piagent-core/benchmark/benchmark-claim-restrictions.js";
19
10
  import { benchmarkEnvironment, benchmarkEnvironmentPolicy, comparisonSurfaces, createCodexRuntime, piagentTreatment } from "../packages/piagent-core/benchmark/benchmark-runtime.js";
20
11
  import { assertBenchmarkPiCredentialReady, assertBenchmarkPiCredentialWritebackPolicy, cleanupBenchmarkPiRuntimeHome, createBenchmarkPiRuntimeHome, resetBenchmarkPiRuntimeEphemeralState, withBenchmarkPiCredentialWriteback } from "../packages/piagent-core/benchmark/benchmark-pi-home.js";
21
12
  import { benchmarkPreflight, benchmarkPreflightReceipt } from "../packages/piagent-core/benchmark/benchmark-preflight.js";
13
+ import { applyBenchmarkExecutionDefaults } from "../packages/piagent-core/benchmark/benchmark-runner-policy.js";
22
14
  import {
23
- cleanupUnretainedWorkspaces,
24
15
  appendPrivateJsonl,
25
16
  createBenchmarkCandidateGuard,
26
17
  loadReplayFailurePlan,
@@ -28,7 +19,6 @@ import {
28
19
  safeInfrastructureDiagnostic,
29
20
  writeBenchmarkAbort,
30
21
  writeBenchmarkRunManifest,
31
- writePrivate,
32
22
  writePrivateAtomic
33
23
  } from "../packages/piagent-core/benchmark/benchmark-forensics.js";
34
24
  import { benchmarkBootstrapCandidateIndex, benchmarkBootstrapMetadata } from "../packages/piagent-core/benchmark/benchmark-bootstrap.js";
@@ -37,8 +27,16 @@ import { benchmarkCommandIdentity } from "../packages/piagent-core/benchmark/ben
37
27
  import { benchmarkTreeIdentity } from "../packages/piagent-core/benchmark/benchmark-tree-identity.js";
38
28
  import { completedBenchmarkRecord, expectedBenchmarkRecord } from "../packages/piagent-core/benchmark/benchmark-record-validation.js";
39
29
  import { pairedOutcomeFloorStop } from "../packages/piagent-core/benchmark/benchmark-stop-policy.js";
30
+ import {
31
+ approveProductionStageControl,
32
+ buildBenchmarkStageDiagnostic,
33
+ createProductionStageControl,
34
+ durablePairedOutcomeFloorStop,
35
+ productionSpendControlValidationErrors,
36
+ productionStageResumeDisposition
37
+ } from "../packages/piagent-core/benchmark/benchmark-stage-diagnostic.js";
40
38
  import { loadBenchmarkAssuranceEvidence, loadBenchmarkSuite, resolveBenchmarkSuiteEntry, validateBenchmarkSuiteFiles } from "../packages/piagent-core/benchmark/benchmark-suite-runtime.js";
41
- import { appendBenchmarkLedger, assertBenchmarkLedgerBinding, benchmarkLedgerCheckpoint, emptyBenchmarkLedgerBinding, inspectBenchmarkLedger, validateBenchmarkLedgerPrefix } from "../packages/piagent-core/benchmark/benchmark-ledger.js";
39
+ import { appendBenchmarkLedger, assertBenchmarkLedgerBinding, emptyBenchmarkLedgerBinding, inspectBenchmarkLedger, validateBenchmarkLedgerPrefix } from "../packages/piagent-core/benchmark/benchmark-ledger.js";
42
40
  import { acquireBenchmarkRunLock } from "../packages/piagent-core/benchmark/benchmark-run-lock.js";
43
41
  import { createBenchmarkProcessController } from "../packages/piagent-core/benchmark/benchmark-process.js";
44
42
  import {
@@ -49,7 +47,33 @@ import {
49
47
  recoverPendingBenchmarkRecord,
50
48
  stageMeasuredBenchmarkRecord
51
49
  } from "../packages/piagent-core/benchmark/benchmark-resume-recovery.js";
50
+ import { finalizeBenchmarkRun } from "./benchmark-runner-finalization.mjs";
51
+ import {
52
+ benchmarkRunKey,
53
+ confirmPlan,
54
+ createRunId,
55
+ defaultOutputRoot,
56
+ ensureEmptyOutput,
57
+ executionOrder,
58
+ fail,
59
+ formatDuration,
60
+ frozenRuntimeCommandsForFinalization,
61
+ invokedAsEntrypoint,
62
+ isLegacyInvocation,
63
+ loadResumeState,
64
+ pairedChunk,
65
+ privateDirectory,
66
+ readJsonFile,
67
+ samePairedBlock,
68
+ sameStringList
69
+ } from "./benchmark-runner-support.mjs";
52
70
  import { runBenchmarkSession } from "./benchmark-session.mjs";
71
+ import {
72
+ assertBenchmarkHostReadinessStartReady,
73
+ assertExistingBenchmarkHostReadiness,
74
+ benchmarkHostReadinessPolicyDigest,
75
+ collectReadyBenchmarkHostReadiness
76
+ } from "./benchmark-runner-host-readiness.mjs";
53
77
  export { parseBenchmarkArgs };
54
78
  const packageRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
55
79
  const packageManifest = JSON.parse(fs.readFileSync(path.join(packageRoot, "package.json"), "utf8"));
@@ -58,11 +82,6 @@ const bootstrapCandidateIndex = bootstrapMetadata ? benchmarkBootstrapCandidateI
58
82
  let interruptedSignal;
59
83
  const processController = createBenchmarkProcessController(() => Boolean(interruptedSignal));
60
84
  const runCommand = processController.run;
61
- function fail(message, code = 2) {
62
- const error = new Error(message);
63
- error.exitCode = code;
64
- throw error;
65
- }
66
85
 
67
86
  function installSignalForwarding() {
68
87
  const handlers = new Map();
@@ -80,154 +99,6 @@ function installSignalForwarding() {
80
99
  };
81
100
  }
82
101
 
83
- function isLegacyInvocation(argv) {
84
- return argv.includes("--record") || argv.includes("--init");
85
- }
86
-
87
- function privateDirectory(target) {
88
- fs.mkdirSync(target, { recursive: true, mode: 0o700 });
89
- try { fs.chmodSync(target, 0o700); } catch { /* Non-POSIX filesystem. */ }
90
- return target;
91
- }
92
-
93
- function defaultOutputRoot() {
94
- if (bootstrapMetadata?.defaultOutputRoot) return bootstrapMetadata.defaultOutputRoot;
95
- const agentRoot = process.env.PI_CODING_AGENT_DIR || path.join(os.homedir(), ".pi", "agent");
96
- return path.join(agentRoot, "benchmarks", "piagent");
97
- }
98
-
99
- function createRunId(suiteId) {
100
- const timestamp = new Date().toISOString().replace(/[-:]/g, "").replace(/\.\d{3}Z$/, "Z");
101
- return `${suiteId}-${timestamp}-${crypto.randomBytes(3).toString("hex")}`;
102
- }
103
-
104
- function ensureEmptyOutput(target) {
105
- if (fs.existsSync(target) && fs.readdirSync(target).length > 0) fail(`Output directory is not empty: ${target}`, 1);
106
- return privateDirectory(target);
107
- }
108
-
109
- function benchmarkRunKey(value) {
110
- return `${value.scenarioId ?? value.scenario?.id}\0${value.surface}\0${value.repeat}`;
111
- }
112
-
113
- function samePairedBlock(left, right) {
114
- return left?.scenario?.id === right?.scenario?.id && left?.repeat === right?.repeat;
115
- }
116
-
117
- function pairedChunk(order, maximum) {
118
- if (maximum === undefined || order.length <= maximum) return order;
119
- let length = maximum;
120
- while (length > 0 && samePairedBlock(order[length - 1], order[length])) length -= 1;
121
- if (length === 0) fail(`--max-sessions ${maximum} would split the first paired benchmark block; increase the chunk size`, 1);
122
- return order.slice(0, length);
123
- }
124
-
125
- function readJsonFile(file, label) {
126
- try {
127
- return JSON.parse(fs.readFileSync(file, "utf8"));
128
- } catch (error) {
129
- fail(`Cannot read ${label} ${file}: ${error.message}`, 1);
130
- }
131
- }
132
-
133
- function resolveResumeRunRoot(input) {
134
- const target = path.resolve(input);
135
- try {
136
- const stat = fs.statSync(target);
137
- if (stat.isDirectory()) return target;
138
- if (stat.isFile()) return path.dirname(target);
139
- } catch (error) {
140
- fail(`Cannot resume benchmark; path does not exist: ${target}`, 1);
141
- }
142
- fail(`Cannot resume benchmark; path is not a file or directory: ${target}`, 1);
143
- }
144
-
145
- function loadResumeState(input) {
146
- const runRoot = resolveResumeRunRoot(input);
147
- const releaseRunLock = acquireBenchmarkRunLock(runRoot, "resume-pending");
148
- try {
149
- const manifestPath = path.join(runRoot, "run-manifest.json");
150
- if (!fs.existsSync(manifestPath)) {
151
- fail(`Cannot resume ${runRoot}: missing run-manifest.json. This run was created before resume metadata was written, so its root seed cannot be recovered safely. Start a new run with --max-sessions or --max-runtime-minutes to make it resumable.`, 1);
152
- }
153
- const manifest = readJsonFile(manifestPath, "benchmark resume manifest");
154
- if (fs.existsSync(path.join(runRoot, "stopped.json"))) fail(`Cannot resume ${runRoot}: the paired release stop is terminal`, 1);
155
- if (manifest?.schemaVersion !== 1 || typeof manifest.runId !== "string") {
156
- fail(`Cannot resume ${runRoot}: run-manifest.json has an unsupported shape`, 1);
157
- }
158
- const ledger = benchmarkLedgerCheckpoint(
159
- manifest.ledger,
160
- inspectBenchmarkLedger(path.join(runRoot, "runs.jsonl")),
161
- "benchmark resume ledger"
162
- );
163
- const pendingPath = path.join(runRoot, "pending-record.json");
164
- const pendingRecord = fs.existsSync(pendingPath) ? readJsonFile(pendingPath, "benchmark pending record") : null;
165
- const measuredPath = path.join(runRoot, "measured-record-ready.json");
166
- const measuredReady = fs.existsSync(measuredPath) ? readJsonFile(measuredPath, "measured benchmark record") : null;
167
- return {
168
- runRoot,
169
- manifest,
170
- completedRuns: ledger.records,
171
- ledgerBinding: ledger.binding,
172
- recoveredLedgerSuffix: ledger.recovered,
173
- pendingRecord,
174
- measuredReady,
175
- releaseRunLock
176
- };
177
- } catch (error) {
178
- releaseRunLock();
179
- throw error;
180
- }
181
- }
182
-
183
- function sameStringList(left, right) {
184
- return Array.isArray(left)
185
- && Array.isArray(right)
186
- && left.length === right.length
187
- && left.every((value, index) => value === right[index]);
188
- }
189
-
190
- function formatDuration(ms) {
191
- const totalSeconds = Math.max(0, Math.round(ms / 1_000));
192
- const hours = Math.floor(totalSeconds / 3600);
193
- const minutes = Math.floor((totalSeconds % 3600) / 60);
194
- const seconds = totalSeconds % 60;
195
- if (hours > 0) return `${hours}h${String(minutes).padStart(2, "0")}m`;
196
- if (minutes > 0) return `${minutes}m${String(seconds).padStart(2, "0")}s`;
197
- return `${seconds}s`;
198
- }
199
-
200
- function executionOrder(suite, repeats, surfaces, rootSeed) {
201
- const order = [];
202
- for (let repeat = 1; repeat <= repeats; repeat += 1) {
203
- const scenarios = suite.schemaVersion === 2
204
- ? [...suite.scenarios].sort((left, right) => {
205
- const rank = (scenario) => crypto.createHmac("sha256", rootSeed).update(`order\0${repeat}\0${scenario.id}`).digest("hex");
206
- return rank(left).localeCompare(rank(right));
207
- })
208
- : suite.scenarios;
209
- for (const [index, scenario] of scenarios.entries()) {
210
- const reverse = suite.schemaVersion === 2
211
- ? (crypto.createHmac("sha256", rootSeed).update(`surface\0${repeat}\0${scenario.id}`).digest()[0] & 1) === 1
212
- : (repeat + index) % 2 !== 0;
213
- const ordered = reverse ? [...surfaces].reverse() : surfaces;
214
- for (const surface of ordered) order.push({ scenario, surface, repeat });
215
- }
216
- }
217
- return order;
218
- }
219
-
220
- async function confirmPlan(message) {
221
- if (!process.stdin.isTTY || !process.stdout.isTTY) fail("Refusing to start billed model runs without --yes in a non-interactive terminal", 1);
222
- const terminal = readline.createInterface({ input: process.stdin, output: process.stdout });
223
- try {
224
- const answer = await terminal.question(`${message}\nContinue? [y/N] `);
225
- return /^(?:y|yes)$/i.test(answer.trim());
226
- } finally {
227
- terminal.close();
228
- }
229
- }
230
-
231
102
  async function runLegacy(argv) {
232
103
  const result = await runCommand("bash", [path.join(packageRoot, "scripts", "quality-benchmark.sh"), ...argv], { cwd: process.cwd(), inherit: true });
233
104
  process.exitCode = result.code;
@@ -264,6 +135,7 @@ async function main() {
264
135
  options.timeoutSeconds = manifest.timeoutSeconds;
265
136
  options.infrastructureRetries = manifest.infrastructureRetries;
266
137
  options.retryDelaySeconds = manifest.retryDelaySeconds;
138
+ options.stopAfterFailedPair = manifest.stopAfterFailedPair === true;
267
139
  options.output = resumeState.runRoot;
268
140
  }
269
141
  if (options.replayFailures) {
@@ -283,14 +155,43 @@ async function main() {
283
155
  evidenceComplete: bootstrapMetadata?.replay?.evidenceComplete ?? replay.source.evidenceComplete
284
156
  };
285
157
  }
286
- if (bootstrapMetadata?.suite?.snapshot) options.suite = bootstrapMetadata.suite.snapshot;
158
+ if (bootstrapMetadata?.suite?.builtInId) options.suite = bootstrapMetadata.suite.builtInId;
159
+ else if (bootstrapMetadata?.suite?.snapshot) options.suite = bootstrapMetadata.suite.snapshot;
287
160
  if (options.help) {
288
161
  process.stdout.write(benchmarkUsage);
289
162
  return;
290
163
  }
291
164
  piagentTreatment(options.piagentTreatment);
292
- const { suite, manifestPath, suiteRoot } = loadBenchmarkSuite(options.suite, packageRoot);
165
+ const { suite, manifestPath, suiteRoot, builtInId } = loadBenchmarkSuite(options.suite, packageRoot);
293
166
  validateBenchmarkSuiteFiles(suite, suiteRoot);
167
+ if (bootstrapMetadata && bootstrapMetadata.suite.builtInId !== builtInId) {
168
+ fail("Frozen benchmark suite origin no longer matches its canonical built-in identity", 1);
169
+ }
170
+ const canonicalProductionSuite = builtInId === "production-v1";
171
+ const productionSpendControlPath = path.join(suiteRoot, "spend-control.v1.json");
172
+ const productionSpendControl = canonicalProductionSuite || fs.existsSync(productionSpendControlPath)
173
+ ? readJsonFile(productionSpendControlPath, "production spend-control contract")
174
+ : null;
175
+ const productionExpectedSessions = productionSpendControl
176
+ && Number.isSafeInteger(productionSpendControl.execution?.repeats)
177
+ && Array.isArray(productionSpendControl.execution?.surfaces)
178
+ ? suite.scenarios.length * productionSpendControl.execution.repeats * productionSpendControl.execution.surfaces.length
179
+ : undefined;
180
+ const productionSpendControlErrors = productionSpendControl
181
+ ? productionSpendControlValidationErrors(productionSpendControl, {
182
+ suiteId: suite.id,
183
+ expectedSessions: productionExpectedSessions,
184
+ requireHostReadiness: canonicalProductionSuite
185
+ && suite.releaseGate?.requireHostReadinessForClaim === true
186
+ })
187
+ : [];
188
+ if (productionSpendControlErrors.length > 0) {
189
+ fail(`Production spend-control contract is invalid (${productionSpendControlErrors.join(", ")})`, 1);
190
+ }
191
+ const productionAllStageBoundaries = productionSpendControl
192
+ ? productionSpendControl.stages.map((stage) => stage.cumulativeSessions)
193
+ : [];
194
+ const productionStageBoundaries = productionAllStageBoundaries.slice(1);
294
195
  const assuranceEvidence = loadBenchmarkAssuranceEvidence(suite, suiteRoot);
295
196
  const declaredScenarioCount = suite.scenarios.length;
296
197
  if (options.scenarioIds) {
@@ -305,12 +206,24 @@ async function main() {
305
206
  if (missing.length) fail(`Replay report references unknown scenario: ${missing.join(", ")}`, 1);
306
207
  suite.scenarios = [...new Set(options.replayRuns.map((run) => run.scenarioId))].map((id) => byId.get(id));
307
208
  }
308
- options.repeats = options.repeats ?? suite.defaultRepeats;
309
- options.infrastructureRetries = options.infrastructureRetries ?? (suite.schemaVersion === 2 ? 2 : 0);
310
- options.retryDelaySeconds = options.retryDelaySeconds ?? (suite.schemaVersion === 2 ? 60 : 0);
311
- options.timeoutSeconds = options.timeoutSeconds ?? suite.timeoutSeconds;
209
+ applyBenchmarkExecutionDefaults(options, suite);
210
+ const productionFullMatrixRequested = Boolean(productionSpendControl)
211
+ && !options.replayRuns
212
+ && suite.scenarios.length === declaredScenarioCount;
213
+ if (productionFullMatrixRequested) {
214
+ const spendExecution = productionSpendControl.execution;
215
+ if (options.seed === undefined) options.seed = productionSpendControl.rootSeed;
216
+ if (options.seed !== productionSpendControl.rootSeed) fail("Production spend control requires its frozen root seed", 1);
217
+ if (!sameStringList(options.surfaces, spendExecution.surfaces)) fail("Production spend control requires its frozen surface order", 1);
218
+ if ((options.model ?? null) !== spendExecution.model) fail("Production spend control requires its frozen model", 1);
219
+ if ((options.thinking ?? null) !== spendExecution.thinking) fail("Production spend control requires its frozen thinking level", 1);
220
+ if (options.repeats !== spendExecution.repeats) fail("Production spend control requires its frozen repeat count", 1);
221
+ if (options.infrastructureRetries !== spendExecution.infrastructureRetries) fail("Production spend control requires zero infrastructure retries", 1);
222
+ }
312
223
  if (options.stopAfterFailedPair && !Number.isFinite(suite.releaseGate?.minimumOutcomeScoreExclusive)) fail("--stop-after-failed-pair requires a suite outcome floor", 1);
313
224
  if (options.surfaces.includes("codex-cli")) {
225
+ options.model = options.model ?? "openai-codex/gpt-5.6-luna";
226
+ options.thinking = options.thinking ?? "medium";
314
227
  codexModelName(options.model);
315
228
  codexThinkingEffort(options.thinking);
316
229
  }
@@ -334,6 +247,24 @@ async function main() {
334
247
  repeat: run.repeat
335
248
  }))
336
249
  : executionOrder(suite, options.repeats, options.surfaces, rootSeed);
250
+ const productionSpendControlled = productionFullMatrixRequested;
251
+ const productionHostReadinessRequired = productionSpendControlled
252
+ && canonicalProductionSuite
253
+ && suite.releaseGate?.requireHostReadinessForClaim === true;
254
+ const productionHostReadinessPolicy = productionHostReadinessRequired
255
+ ? productionSpendControl.hostReadiness
256
+ : null;
257
+ const productionHostReadinessPolicyDigest = productionHostReadinessRequired
258
+ ? benchmarkHostReadinessPolicyDigest(productionHostReadinessPolicy)
259
+ : null;
260
+ const productionReleaseClaimRun = productionSpendControlled
261
+ && suite.schemaVersion === 2
262
+ && suite.releaseGate?.requireEfficiencyClaim === true
263
+ && suite.releaseGate?.requireFullSuiteForClaim === true;
264
+ if (productionSpendControlled && productionStageBoundaries.at(-1) !== fullOrder.length) {
265
+ fail("Production spend control final stage does not match the frozen full execution order", 1);
266
+ }
267
+ let deferredProductionStageApproval;
337
268
  if (resumeState) {
338
269
  const manifest = resumeState.manifest;
339
270
  if (manifest.suiteDigest !== suiteDigest) fail("Cannot resume benchmark: suite files changed since the original run", 1);
@@ -381,7 +312,104 @@ async function main() {
381
312
  });
382
313
  throw provenanceError;
383
314
  }
315
+ const recoveredTerminalStop = durablePairedOutcomeFloorStop({
316
+ enabled: options.stopAfterFailedPair,
317
+ suite,
318
+ runs: resumeState.completedRuns,
319
+ fullOrder
320
+ });
321
+ if (recoveredTerminalStop) {
322
+ for (const marker of ["paused.json", "stage-diagnostic.json", "interrupted.json", "aborted.json"]) {
323
+ fs.rmSync(path.join(resumeState.runRoot, marker), { force: true });
324
+ }
325
+ writePrivateAtomic(path.join(resumeState.runRoot, "stopped.json"), `${JSON.stringify({
326
+ ...recoveredTerminalStop,
327
+ runId: manifest.runId,
328
+ completedRuns: resumeState.completedRuns.length,
329
+ expectedRuns: fullOrder.length,
330
+ stoppedAt: new Date().toISOString(),
331
+ resumeAllowed: false,
332
+ recoveredFromAcceptedLedger: true,
333
+ ledger: resumeState.ledgerBinding,
334
+ provenanceStamp: candidateGuard.stamp("resume-terminal-floor")
335
+ }, null, 2)}\n`);
336
+ fail("Cannot resume benchmark: the accepted ledger already contains a terminal paired outcome-floor failure", 1);
337
+ }
338
+ if (productionSpendControlled) {
339
+ if (productionHostReadinessRequired) {
340
+ if (manifest.hostReadinessPolicyDigest !== productionHostReadinessPolicyDigest
341
+ && resumeState.completedRuns.length < fullOrder.length) {
342
+ fail("Cannot resume production benchmark: host-readiness policy binding is missing or changed. Start a new staged run.", 1);
343
+ }
344
+ if (resumeState.completedRuns.length < fullOrder.length) {
345
+ assertExistingBenchmarkHostReadiness({
346
+ policy: productionHostReadinessPolicy,
347
+ receipts: manifest.hostReadinessReceipts,
348
+ completedRuns: resumeState.completedRuns.length,
349
+ authorizedThroughRuns: manifest.stageControl?.authorizedThroughRuns,
350
+ runId: manifest.runId,
351
+ configurationDigest: manifest.configurationDigest,
352
+ stageBoundaries: productionAllStageBoundaries
353
+ });
354
+ }
355
+ }
356
+ const stageDisposition = productionStageResumeDisposition(manifest.stageControl, {
357
+ completedRuns: resumeState.completedRuns.length,
358
+ ledger: resumeState.ledgerBinding,
359
+ stageBoundaries: productionAllStageBoundaries
360
+ });
361
+ if (!stageDisposition.passed) {
362
+ fail(`Cannot resume production benchmark: invalid durable stage-control state (${stageDisposition.errors.join(", ")}). Start a new staged run.`, 1);
363
+ }
364
+ if (stageDisposition.requiresStageGate && resumeState.completedRuns.length < fullOrder.length) {
365
+ const recoveredBoundaryReason = `max-sessions:recovered:${resumeState.completedRuns.length}`;
366
+ const resumeStageDiagnostic = buildBenchmarkStageDiagnostic({
367
+ runId: manifest.runId,
368
+ reason: manifest.stageControl.pendingBoundary?.reason ?? recoveredBoundaryReason,
369
+ runs: resumeState.completedRuns,
370
+ fullOrder,
371
+ candidateSurface: comparison.candidateSurface,
372
+ baselineSurface: comparison.baselineSurface,
373
+ requestedModel: options.model,
374
+ requestedThinking: options.thinking,
375
+ suite,
376
+ manifest,
377
+ hostReadinessPolicy: productionHostReadinessPolicy,
378
+ hostReadinessStageBoundaries: productionAllStageBoundaries
379
+ });
380
+ writePrivateAtomic(path.join(resumeState.runRoot, "stage-diagnostic.json"), `${JSON.stringify(resumeStageDiagnostic, null, 2)}\n`);
381
+ if (!resumeStageDiagnostic.stageAdvanceAllowed) {
382
+ fail(`Cannot resume production benchmark: the provider-free stage gate blocked further paid sessions (${resumeStageDiagnostic.blockingReasons.join(", ")}). Fix the candidate and start a new staged run.`, 1);
383
+ }
384
+ const currentWindow = manifest.stageControl.authorizedThroughRuns;
385
+ const nextAuthorizedThroughRuns = resumeState.completedRuns.length < currentWindow
386
+ ? currentWindow
387
+ : productionStageBoundaries.find((boundary) => boundary > resumeState.completedRuns.length);
388
+ if (!Number.isSafeInteger(nextAuthorizedThroughRuns)) {
389
+ fail("Cannot resume production benchmark: no bounded spend-control window remains", 1);
390
+ }
391
+ const requiredResumeSessions = nextAuthorizedThroughRuns - resumeState.completedRuns.length;
392
+ if (!options.dryRun && options.maxSessions !== requiredResumeSessions) {
393
+ fail(`Production spend control requires --max-sessions ${requiredResumeSessions} for the next authorized window; durable stage state was not changed.`, 1);
394
+ }
395
+ if (!options.dryRun) {
396
+ deferredProductionStageApproval = {
397
+ completedRuns: resumeState.completedRuns.length,
398
+ authorizedThroughRuns: nextAuthorizedThroughRuns
399
+ };
400
+ }
401
+ } else if (stageDisposition.requiresStageGate) {
402
+ // A crash after the final accepted record must not change the verdict by
403
+ // applying an early-stage heuristic. The complete report gates the same
404
+ // frozen ledger without starting another provider session.
405
+ manifest.ledger = resumeState.ledgerBinding;
406
+ writeBenchmarkRunManifest(resumeState.runRoot, manifest);
407
+ }
408
+ }
384
409
  }
410
+ const productionFinalizationOnly = productionSpendControlled
411
+ && Boolean(resumeState)
412
+ && resumeState.completedRuns.length === fullOrder.length;
385
413
  const pendingOrder = resumeState
386
414
  ? fullOrder.filter((item) => !resumeState.completedKeys.has(benchmarkRunKey(item)))
387
415
  : fullOrder;
@@ -416,24 +444,64 @@ async function main() {
416
444
  const codexPlan = options.surfaces.includes("codex-cli")
417
445
  ? `\n codex: ${options.codexMode} mode · model ${codexModelName(options.model)} · effort ${codexThinkingEffort(options.thinking)}${options.codexMode === "controlled" ? " · isolated home" : ""}`
418
446
  : "";
447
+ const nativeWarning = options.surfaces.includes("codex-cli") && options.codexMode === "native"
448
+ ? "\nNative Codex mode loads the operator's global AGENTS.md, configuration, rules, hooks, MCP servers, and plugins."
449
+ : "";
419
450
  if (options.dryRun) {
420
451
  process.stdout.write(`${plan}${codexPlan}\n manifest: ${manifestPath}\nDRY RUN: no model session started.\n`);
421
452
  return;
422
453
  }
454
+ if (productionSpendControlled) {
455
+ if (options.stopAfterFailedPair !== productionSpendControl.execution.stopAfterFailedPair) {
456
+ fail("Production spend control requires --stop-after-failed-pair before any provider session", 1);
457
+ }
458
+ const completedRuns = resumeState?.completedRuns.length ?? 0;
459
+ const authorizedThroughRuns = deferredProductionStageApproval?.authorizedThroughRuns
460
+ ?? resumeState?.manifest.stageControl?.authorizedThroughRuns
461
+ ?? productionStageBoundaries[0];
462
+ const requiredChunkSessions = authorizedThroughRuns - completedRuns;
463
+ if (!options.preflightOnly && !productionFinalizationOnly
464
+ && (!Number.isSafeInteger(requiredChunkSessions) || requiredChunkSessions <= 0
465
+ || options.maxSessions !== requiredChunkSessions)) {
466
+ fail(`Production spend control requires --max-sessions ${requiredChunkSessions} for the current authorized window; refusing an unbounded or oversized paid chunk.`, 1);
467
+ }
468
+ }
423
469
  if (!bootstrapMetadata) fail("Modern billed benchmarks must start through scripts/benchmark-runner.mjs so execution assets are frozen", 1);
470
+ if (productionReleaseClaimRun && bootstrapMetadata.sourceIdentity.dirty !== false) {
471
+ fail("Production benchmark requires a clean release source before auth, tool preflight, or any provider session", 1);
472
+ }
424
473
  if (options.replayFailures && bootstrapMetadata.replay?.evidenceComplete !== true) {
425
474
  fail("Billed replay requires the original run-manifest.json and runs.jsonl beside the source report", 1);
426
475
  }
427
- const runtimeCommands = {
428
- pi: benchmarkCommandIdentity(piCommand, { cwd: bootstrapMetadata?.originalCwd ?? process.cwd() }),
429
- codex: options.surfaces.includes("codex-cli")
430
- ? benchmarkCommandIdentity(codexCommand, { cwd: bootstrapMetadata?.originalCwd ?? process.cwd() })
431
- : null,
432
- node: benchmarkCommandIdentity(process.execPath),
433
- git: benchmarkCommandIdentity("git"),
434
- bash: benchmarkCommandIdentity("bash")
435
- };
476
+ if (!options.preflightOnly) {
477
+ if (productionFinalizationOnly) {
478
+ process.stdout.write(`${plan}${codexPlan}\n mode: provider-free finalization of the complete frozen ledger\n`);
479
+ } else if (options.yes) {
480
+ process.stdout.write(`${plan}${codexPlan}${nativeWarning}\n`);
481
+ } else if (!(await confirmPlan(`${plan}${codexPlan}${nativeWarning}\nThis may use paid model quota.`))) {
482
+ process.stdout.write("Benchmark cancelled; no model session started.\n");
483
+ return;
484
+ }
485
+ }
486
+ const runId = resumeState?.manifest.runId ?? createRunId(suite.id);
487
+ const runtimeCommands = productionFinalizationOnly
488
+ ? frozenRuntimeCommandsForFinalization(resumeState.manifest, options.surfaces)
489
+ : {
490
+ pi: benchmarkCommandIdentity(piCommand, { cwd: bootstrapMetadata?.originalCwd ?? process.cwd() }),
491
+ codex: options.surfaces.includes("codex-cli")
492
+ ? benchmarkCommandIdentity(codexCommand, { cwd: bootstrapMetadata?.originalCwd ?? process.cwd() })
493
+ : null,
494
+ node: benchmarkCommandIdentity(process.execPath),
495
+ git: benchmarkCommandIdentity("git"),
496
+ bash: benchmarkCommandIdentity("bash")
497
+ };
436
498
  const environmentPolicy = benchmarkEnvironmentPolicy();
499
+ const configurationPiAgentHome = productionFinalizationOnly
500
+ ? resumeState.manifest.piAgentHome?.identity
501
+ : bootstrapMetadata.piAgentHome.identity;
502
+ const configurationCodexCredential = productionFinalizationOnly
503
+ ? resumeState.manifest.codexCredentialIdentity ?? null
504
+ : bootstrapMetadata.codexCredential?.identity ?? null;
437
505
  const configuration = {
438
506
  schemaVersion: 1,
439
507
  source: bootstrapMetadata.sourceIdentity,
@@ -442,8 +510,9 @@ async function main() {
442
510
  runtimeDependencyDigest: bootstrapMetadata.runtimeDependencies?.digest ?? null,
443
511
  runtimeCommands,
444
512
  environmentPolicy,
445
- piAgentHome: bootstrapMetadata.piAgentHome.identity,
446
- codexCredential: bootstrapMetadata.codexCredential?.identity ?? null,
513
+ ...(productionHostReadinessRequired ? { hostReadinessPolicyDigest: productionHostReadinessPolicyDigest } : {}),
514
+ piAgentHome: configurationPiAgentHome,
515
+ codexCredential: configurationCodexCredential,
447
516
  rootSeedDigest,
448
517
  surfaces: options.surfaces,
449
518
  model: options.model ?? null,
@@ -475,48 +544,81 @@ async function main() {
475
544
  completedKeys: resumeState.completedKeys
476
545
  });
477
546
  }
547
+ let hostReadinessReceipt = null;
548
+ const currentAuthorizedThroughRuns = resumeState?.manifest.stageControl?.authorizedThroughRuns
549
+ ?? productionStageBoundaries[0];
550
+ const nextAuthorizedThroughRuns = deferredProductionStageApproval?.authorizedThroughRuns
551
+ ?? currentAuthorizedThroughRuns;
552
+ if (productionHostReadinessRequired && !productionFinalizationOnly) {
553
+ hostReadinessReceipt = await collectReadyBenchmarkHostReadiness({
554
+ policy: productionHostReadinessPolicy,
555
+ completedRuns: resumeState?.completedRuns.length ?? 0,
556
+ authorizedThroughRuns: nextAuthorizedThroughRuns,
557
+ runId,
558
+ configurationDigest,
559
+ stageBoundaries: productionAllStageBoundaries,
560
+ existingReceipts: resumeState?.manifest.hostReadinessReceipts ?? []
561
+ });
562
+ }
563
+ const assertHostReadinessStartReady = () => assertBenchmarkHostReadinessStartReady(hostReadinessReceipt, {
564
+ policy: productionHostReadinessPolicy,
565
+ runId,
566
+ configurationDigest
567
+ });
478
568
  const executionGuard = createBenchmarkExecutionGuard({
479
569
  candidateGuard,
480
570
  suiteRoot,
481
571
  suiteIdentity: bootstrapMetadata?.suite?.identity ?? suiteIdentity,
482
- piAgentHome: bootstrapMetadata?.piAgentHome,
483
- codexCredential: bootstrapMetadata?.codexCredential,
572
+ piAgentHome: productionFinalizationOnly ? null : bootstrapMetadata?.piAgentHome,
573
+ codexCredential: productionFinalizationOnly ? null : bootstrapMetadata?.codexCredential,
484
574
  runtimeDependencies: bootstrapMetadata?.runtimeDependencies,
485
- commands: runtimeCommands
575
+ commands: runtimeCommands,
576
+ verifyCommandAssets: !productionFinalizationOnly
486
577
  });
487
- assertBenchmarkPiCredentialReady(bootstrapMetadata.piAgentHome.credentialReadiness, options.model);
488
- assertBenchmarkPiCredentialWritebackPolicy(bootstrapMetadata.piAgentHome);
489
- piRuntimeHome = createBenchmarkPiRuntimeHome(bootstrapMetadata.piAgentHome);
490
- const preflightAssetError = executionGuard.check("before-preflight", [piRuntimeHome]);
491
- if (preflightAssetError) throw preflightAssetError;
492
- codexRuntime = createCodexRuntime(options);
493
578
  let runtime;
494
- try { runtime = await withBenchmarkPiCredentialWriteback(bootstrapMetadata.piAgentHome, piRuntimeHome, () => benchmarkPreflight({ runCommand, packageRoot, piCommand, piEnvironment: benchmarkEnvironment({ PI_CODING_AGENT_DIR: piRuntimeHome.path }), codexCommand, gitCommand: runtimeCommands.git.resolvedPath, surfaces: options.surfaces, codexMode: options.codexMode, codexRuntime })); }
495
- catch (error) { preservePiRuntime ||= error.code === "BENCHMARK_PI_CREDENTIAL_RECONCILIATION_FAILED"; throw error; }
496
- const postPreflightAssetError = executionGuard.check("after-preflight", [piRuntimeHome]);
497
- if (postPreflightAssetError) throw postPreflightAssetError;
498
- resetBenchmarkPiRuntimeEphemeralState(piRuntimeHome);
579
+ let codexCredentialBridge;
580
+ if (productionFinalizationOnly) {
581
+ runtime = resumeState.manifest.preflightRuntime;
582
+ if (!runtime || typeof runtime.gitVersion !== "string" || typeof runtime.piVersion !== "string"
583
+ || !Array.isArray(runtime.codexDisabledFeatures)
584
+ || (options.surfaces.includes("codex-cli") && typeof runtime.codexVersion !== "string")) {
585
+ fail("Cannot finalize production benchmark provider-free: frozen preflight runtime evidence is missing or invalid", 1);
586
+ }
587
+ codexCredentialBridge = resumeState.manifest.codexCredentialBridge ?? null;
588
+ const finalizationAssetError = executionGuard.check("provider-free-finalization");
589
+ if (finalizationAssetError) throw finalizationAssetError;
590
+ } else {
591
+ assertBenchmarkPiCredentialReady(bootstrapMetadata.piAgentHome.credentialReadiness, options.model);
592
+ assertBenchmarkPiCredentialWritebackPolicy(bootstrapMetadata.piAgentHome);
593
+ piRuntimeHome = createBenchmarkPiRuntimeHome(bootstrapMetadata.piAgentHome);
594
+ const preflightAssetError = executionGuard.check("before-preflight", [piRuntimeHome]);
595
+ if (preflightAssetError) throw preflightAssetError;
596
+ codexRuntime = createCodexRuntime(options);
597
+ try { runtime = await withBenchmarkPiCredentialWriteback(bootstrapMetadata.piAgentHome, piRuntimeHome, () => benchmarkPreflight({ runCommand, packageRoot, piCommand, piEnvironment: benchmarkEnvironment({ PI_CODING_AGENT_DIR: piRuntimeHome.path }), codexCommand, gitCommand: runtimeCommands.git.resolvedPath, surfaces: options.surfaces, codexMode: options.codexMode, codexRuntime })); }
598
+ catch (error) { preservePiRuntime ||= error.code === "BENCHMARK_PI_CREDENTIAL_RECONCILIATION_FAILED"; throw error; }
599
+ const postPreflightAssetError = executionGuard.check("after-preflight", [piRuntimeHome]);
600
+ if (postPreflightAssetError) throw postPreflightAssetError;
601
+ resetBenchmarkPiRuntimeEphemeralState(piRuntimeHome);
602
+ codexCredentialBridge = codexRuntime.credentialBridge;
603
+ if (resumeState && resumeState.manifest.preflightRuntime !== undefined
604
+ && JSON.stringify(resumeState.manifest.preflightRuntime) !== JSON.stringify(runtime)) {
605
+ fail("Cannot resume benchmark: provider-free preflight runtime evidence changed since the original run", 1);
606
+ }
607
+ if (resumeState && resumeState.manifest.codexCredentialBridge !== undefined
608
+ && resumeState.manifest.codexCredentialBridge !== codexCredentialBridge) {
609
+ fail("Cannot resume benchmark: Codex credential bridge changed since the original run", 1);
610
+ }
611
+ }
612
+ assertHostReadinessStartReady();
499
613
  const source = bootstrapMetadata?.sourceIdentity;
500
614
  if (!source) fail("Modern benchmark is missing its frozen Git source identity", 1);
501
615
  if (options.preflightOnly) {
502
- const receipt = benchmarkPreflightReceipt({ packageVersion: packageManifest.version, source, candidateProvenance: candidateGuard.report(), suite, suiteDigest, runtimeDependencies: bootstrapMetadata.runtimeDependencies, runtimeCommands, environmentPolicy, configurationDigest, rootSeedDigest, options, runtime });
616
+ const receipt = benchmarkPreflightReceipt({ packageVersion: packageManifest.version, source, candidateProvenance: candidateGuard.report(), suite, suiteDigest, runtimeDependencies: bootstrapMetadata.runtimeDependencies, runtimeCommands, environmentPolicy, configurationDigest, rootSeedDigest, options, runtime, hostReadinessPolicyDigest: productionHostReadinessPolicyDigest, hostReadiness: hostReadinessReceipt });
503
617
  process.stdout.write(options.json ? `${JSON.stringify(receipt, null, 2)}\n` : `${plan}${codexPlan}\nPREFLIGHT READY: no model session started.\n${JSON.stringify(receipt, null, 2)}\n`);
504
618
  return;
505
619
  }
506
- const nativeWarning = options.surfaces.includes("codex-cli") && options.codexMode === "native"
507
- ? "\nNative Codex mode loads the operator's global AGENTS.md, configuration, rules, hooks, MCP servers, and plugins."
508
- : "";
509
- if (options.yes) {
510
- process.stdout.write(`${plan}${codexPlan}${nativeWarning}\n`);
511
- } else {
512
- if (!(await confirmPlan(`${plan}${codexPlan}${nativeWarning}\nThis may use paid model quota.`))) {
513
- process.stdout.write("Benchmark cancelled; no model session started.\n");
514
- return;
515
- }
516
- }
517
620
  candidateGuard.freeze();
518
- const runId = resumeState?.manifest.runId ?? createRunId(suite.id);
519
- const output = options.output ?? path.join(defaultOutputRoot(), runId);
621
+ const output = options.output ?? path.join(defaultOutputRoot(bootstrapMetadata), runId);
520
622
  const runRoot = resumeState ? privateDirectory(output) : ensureEmptyOutput(output);
521
623
  releaseRunLock ??= acquireBenchmarkRunLock(runRoot, runId);
522
624
  privateDirectory(path.join(runRoot, "workspaces"));
@@ -535,13 +637,17 @@ async function main() {
535
637
  suite: {
536
638
  id: suite.id,
537
639
  source: bootstrapMetadata?.suite?.origin ?? manifestPath,
538
- manifestPath: bootstrapMetadata?.suite?.snapshot ? bootstrapMetadata.suite.origin : null
640
+ manifestPath: bootstrapMetadata?.suite?.snapshot ? bootstrapMetadata.suite.origin : null,
641
+ builtInId
539
642
  },
540
643
  suiteDigest,
541
644
  suiteIdentity,
542
645
  candidateProvenance: candidateGuard.provenance,
543
646
  runtimeDependencies: bootstrapMetadata?.runtimeDependencies ?? null,
544
647
  runtimeCommands,
648
+ preflightRuntime: runtime,
649
+ codexCredentialIdentity: bootstrapMetadata.codexCredential?.identity ?? null,
650
+ codexCredentialBridge,
545
651
  configurationDigest,
546
652
  environmentPolicy,
547
653
  piAgentHome: {
@@ -569,13 +675,37 @@ async function main() {
569
675
  retryDelaySeconds: options.retryDelaySeconds,
570
676
  stopAfterFailedPair: options.stopAfterFailedPair,
571
677
  scenarioIds: options.scenarioIds ?? null,
678
+ ...(productionSpendControlled ? {
679
+ stageControl: createProductionStageControl({
680
+ authorizedThroughRuns: productionStageBoundaries[0],
681
+ generatedAt: startedAt
682
+ }),
683
+ ...(productionHostReadinessRequired ? {
684
+ hostReadinessPolicyDigest: productionHostReadinessPolicyDigest,
685
+ hostReadinessReceipts: []
686
+ } : {})
687
+ } : {}),
572
688
  order: fullOrder.map((item) => ({
573
689
  scenarioId: item.scenario.id,
574
690
  surface: item.surface,
575
691
  repeat: item.repeat
576
692
  }))
577
693
  };
694
+ assertHostReadinessStartReady();
695
+ if (productionSpendControlled && deferredProductionStageApproval) {
696
+ manifest.stageControl = approveProductionStageControl(manifest.stageControl, deferredProductionStageApproval);
697
+ }
698
+ if (productionHostReadinessRequired && hostReadinessReceipt) {
699
+ manifest.hostReadinessPolicyDigest = productionHostReadinessPolicyDigest;
700
+ manifest.hostReadinessReceipts = [...(manifest.hostReadinessReceipts ?? []), hostReadinessReceipt];
701
+ }
578
702
  writeBenchmarkRunManifest(runRoot, manifest);
703
+ if (resumeState) {
704
+ fs.rmSync(path.join(runRoot, "stage-diagnostic.json"), { force: true });
705
+ if (productionSpendControlled && manifest.stageControl?.state === "window-authorized") {
706
+ fs.rmSync(path.join(runRoot, "paused.json"), { force: true });
707
+ }
708
+ }
579
709
  const fullIndexByKey = new Map(fullOrder.map((item, index) => [benchmarkRunKey(item), index + 1]));
580
710
  const wallStartedAt = Date.now();
581
711
  const runtimeDeadline = options.maxRuntimeMinutes === undefined
@@ -785,147 +915,18 @@ async function main() {
785
915
  finalizationReceipt = executionGuard.receipt("finalization", [piRuntimeHome]);
786
916
  fatalRunError = finalizationReceipt.error;
787
917
  }
788
- if (interruptedSignal) {
789
- const provenanceStamp = executionGuard.stamp("interrupted", [piRuntimeHome]);
790
- piRuntimeHome = undefined;
791
- recoverOrphanedBenchmarkAttempts({ runRoot, manifest, fullOrder, completedKeys: new Set(runs.map(benchmarkRunKey)) });
792
- cleanupUnretainedWorkspaces(runRoot, options.keepWorkspaces);
793
- writePrivateAtomic(path.join(runRoot, "interrupted.json"), `${JSON.stringify({ schemaVersion: 1, runId, signal: interruptedSignal, completedRuns: runs.filter(completedBenchmarkRecord).length, expectedRuns: fullOrder.length, interruptedAt: new Date().toISOString(), resumeCommand: `piagent-benchmark --resume ${runRoot} --yes`, ledger: ledgerBinding, provenanceStamp }, null, 2)}\n`);
794
- process.stderr.write(`Benchmark interrupted by ${interruptedSignal}. Partial ledger: ${ledgerPath}\n`);
795
- process.exitCode = interruptedSignal === "SIGINT" ? 130 : interruptedSignal === "SIGHUP" ? 129 : 143;
796
- return;
797
- }
798
- if (fatalRunError) {
799
- const provenanceStamp = finalizationReceipt?.stamp ?? fatalExecutionReceipt?.stamp ?? executionGuard.stamp("fatal", piRuntimeHome ? [piRuntimeHome] : []);
800
- if (!preservePiRuntime) cleanupBenchmarkPiRuntimeHome(bootstrapMetadata.piAgentHome, piRuntimeHome);
801
- piRuntimeHome = undefined;
802
- recoverOrphanedBenchmarkAttempts({ runRoot, manifest, fullOrder, completedKeys: new Set(runs.map(benchmarkRunKey)) });
803
- cleanupUnretainedWorkspaces(runRoot, options.keepWorkspaces);
804
- const provenanceFailure = writeBenchmarkAbort(runRoot, { runId, completedRuns: runs.filter(completedBenchmarkRecord).length, expectedRuns: fullOrder.length }, fatalRunError, {
805
- ledger: ledgerBinding,
806
- provenanceStamp
807
- });
808
- process.stderr.write(provenanceFailure
809
- ? `Benchmark aborted because candidate provenance changed. Partial ledger: ${ledgerPath}\n`
810
- : `Benchmark aborted after an infrastructure error. Partial ledger: ${ledgerPath}\n`);
811
- process.exitCode = 1;
812
- return;
813
- }
814
- if (terminalStop) {
815
- cleanupBenchmarkPiRuntimeHome(bootstrapMetadata.piAgentHome, piRuntimeHome); piRuntimeHome = undefined;
816
- cleanupUnretainedWorkspaces(runRoot, options.keepWorkspaces);
817
- for (const marker of ["paused.json", "interrupted.json", "aborted.json"]) fs.rmSync(path.join(runRoot, marker), { force: true });
818
- writePrivateAtomic(path.join(runRoot, "stopped.json"), `${JSON.stringify({ ...terminalStop, runId, completedRuns: runs.filter(completedBenchmarkRecord).length, expectedRuns: fullOrder.length, stoppedAt: new Date().toISOString(), resumeAllowed: false, ledger: ledgerBinding, provenanceStamp: finalizationReceipt.stamp }, null, 2)}\n`);
819
- process.stderr.write(`Benchmark terminal-stopped after paired outcome-floor failure. Partial ledger: ${ledgerPath}\n`);
820
- process.exitCode = 1;
821
- return;
822
- }
823
- const completedRuns = runs.filter(completedBenchmarkRecord);
824
- if (completedRuns.length < fullOrder.length) {
825
- piRuntimeHome = undefined;
826
- cleanupUnretainedWorkspaces(runRoot, options.keepWorkspaces);
827
- const paused = {
828
- schemaVersion: 1,
829
- runId,
830
- reason: pauseReason ?? "partial-run",
831
- completedRuns: completedRuns.length,
832
- expectedRuns: fullOrder.length,
833
- remainingRuns: fullOrder.length - completedRuns.length,
834
- pausedAt: new Date().toISOString(),
835
- resumeCommand: `piagent-benchmark --resume ${runRoot} --yes`,
836
- ledger: ledgerBinding,
837
- provenanceStamp: executionGuard.stamp("paused")
838
- };
839
- writePrivateAtomic(path.join(runRoot, "paused.json"), `${JSON.stringify(paused, null, 2)}\n`);
840
- process.stdout.write(`Benchmark paused after ${completedRuns.length}/${fullOrder.length} completed sessions (${paused.reason}).\nResume: ${paused.resumeCommand}\nPartial ledger: ${ledgerPath}\n`);
841
- return;
842
- }
843
- cleanupBenchmarkPiRuntimeHome(bootstrapMetadata.piAgentHome, piRuntimeHome); piRuntimeHome = undefined;
844
- const report = summarizeBenchmark({
845
- suite,
846
- runId,
847
- startedAt,
848
- completedAt: new Date().toISOString(),
849
- repeats: options.repeats,
850
- environment: {
851
- platformVersion: packageManifest.version,
852
- suiteDigest,
853
- variantRootSeed: suite.scenarios.some((scenario) => scenario.variantGenerator) ? rootSeed : null,
854
- variantRootSeedDigest: suite.scenarios.some((scenario) => scenario.variantGenerator) ? rootSeedDigest : null,
855
- executionOrder: suite.schemaVersion === 2 ? "seeded-paired-block-randomized" : "paired-alternating",
856
- replaySource: options.replaySource ?? null,
857
- profile: suite.profile,
858
- requestedModel: options.model ?? null,
859
- requestedThinking: options.thinking ?? null,
860
- piagentTreatment: piagentTreatment(options.piagentTreatment),
861
- treatmentBaseline: lifecycles.length === 1 && lifecycles[0] === "steady-state"
862
- ? options.surfaces.includes("codex-cli")
863
- ? "piagent-initialized-and-onboarded; codex-clean-fixture"
864
- : "initialized-and-onboarded"
865
- : "scenario-defined-mixed-lifecycle",
866
- timeoutSeconds: options.timeoutSeconds,
867
- nodeVersion: process.version,
868
- piVersion: runtime.piVersion,
869
- codexVersion: runtime.codexVersion ?? null,
870
- codexMode: options.surfaces.includes("codex-cli") ? options.codexMode : null,
871
- codexAuth: runtime.codexAuth ?? null,
872
- codexIsolation: options.surfaces.includes("codex-cli")
873
- ? options.codexMode === "controlled" ? "per-session-temporary-home" : "operator-home"
874
- : null,
875
- codexCredentialBridge: options.surfaces.includes("codex-cli") ? codexRuntime.credentialBridge : null,
876
- codexGlobalInstructions: options.surfaces.includes("codex-cli")
877
- ? options.codexMode === "controlled" ? "excluded" : "operator-home"
878
- : null,
879
- piGlobalInstructions: "excluded",
880
- piAgentHome: manifest.piAgentHome,
881
- usageIntegrity: manifest.tokenClaimsUnavailableReason ?? "measured",
882
- codexDisabledFeatures: runtime.codexDisabledFeatures,
883
- surfaces: options.surfaces,
884
- scenarioSelection: options.scenarioIds ?? null,
885
- surfaceModels: options.surfaces.includes("codex-cli") ? {
886
- piagent: options.model,
887
- "codex-cli": codexModelName(options.model)
888
- } : { "raw-pi": options.model ?? null, piagent: options.model ?? null },
889
- modelParityEvidence: options.surfaces.includes("codex-cli") ? "command-line-pinned" : "session-reported",
890
- gitVersion: runtime.gitVersion,
891
- source,
892
- candidateProvenance: candidateGuard.report(),
893
- suiteIdentity,
894
- runtimeCommands,
895
- configurationDigest,
896
- environmentPolicy,
897
- runtimeDependencies: bootstrapMetadata?.runtimeDependencies ?? null,
898
- assuranceEvidence
899
- },
900
- runs: completedRuns,
901
- ...comparison
918
+ finalizeBenchmarkRun({
919
+ assuranceEvidence, bootstrapMetadata, candidateGuard, canonicalProductionSuite,
920
+ codexCredentialBridge, comparison, configurationDigest, declaredScenarioCount,
921
+ detachPiRuntimeHome: () => { piRuntimeHome = undefined; },
922
+ environmentPolicy, executionGuard, fatalExecutionReceipt, fatalRunError,
923
+ finalizationReceipt, fullOrder, interruptedSignal, ledgerBinding, ledgerPath,
924
+ lifecycles, manifest, options, packageVersion: packageManifest.version,
925
+ pauseReason, piRuntimeHome, preservePiRuntime,
926
+ productionAllStageBoundaries, productionHostReadinessPolicy, productionSpendControlled, rootSeed, rootSeedDigest,
927
+ runId, runRoot, runs, runtime, runtimeCommands, source, startedAt, suite,
928
+ suiteDigest, suiteIdentity, terminalStop
902
929
  });
903
- report.ledger = ledgerBinding;
904
- applyBenchmarkClaimRestrictions(report, { tokenReason: manifest.tokenClaimsUnavailableReason, replaySource: options.replaySource, codexMode: options.codexMode, surfaces: options.surfaces });
905
- report.trustChecklist = benchmarkTrustChecklist(report);
906
- const text = renderBenchmarkText(report);
907
- const reportLedger = inspectBenchmarkLedger(ledgerPath);
908
- assertBenchmarkLedgerBinding(ledgerBinding, reportLedger.binding, "benchmark report ledger");
909
- validateBenchmarkLedgerPrefix(reportLedger.records, fullOrder, (record, index, expected) => expectedBenchmarkRecord(record, index, expected, runId, suite, configurationDigest));
910
- writePrivate(path.join(runRoot, "report.html"), renderBenchmarkHtml(report));
911
- writePrivate(path.join(runRoot, "summary.txt"), text);
912
- cleanupUnretainedWorkspaces(runRoot, options.keepWorkspaces);
913
- const prepublishReceipt = executionGuard.receipt("prepublish");
914
- const prepublishError = prepublishReceipt.error;
915
- if (prepublishError) { writeBenchmarkAbort(runRoot, { runId, completedRuns: completedRuns.length, expectedRuns: fullOrder.length }, prepublishError, { ledger: ledgerBinding, provenanceStamp: prepublishReceipt.stamp }); throw prepublishError; }
916
- for (const marker of ["paused.json", "interrupted.json", "aborted.json", "stopped.json"]) fs.rmSync(path.join(runRoot, marker), { force: true });
917
- writePrivateAtomic(path.join(runRoot, "report.json"), `${JSON.stringify(report, null, 2)}\n`);
918
- process.stdout.write(options.json ? `${JSON.stringify(report, null, 2)}\n` : text);
919
- process.stdout.write(`Reports: ${runRoot}\n`);
920
- if (
921
- report.comparison.qualityGate === false
922
- || report.comparison.safetyGate === false
923
- || report.comparison.reliabilityGate === false
924
- || report.comparison.qualityNonInferior === false
925
- || report.comparison.workflowGate === false
926
- || report.comparison.categoryGate === false
927
- || report.comparison.suiteGate?.passed === false
928
- ) process.exitCode = 1;
929
930
  } finally {
930
931
  releaseRunLock?.();
931
932
  codexRuntime?.cleanup();
@@ -933,12 +934,7 @@ async function main() {
933
934
  }
934
935
  }
935
936
 
936
- function invokedAsEntrypoint() {
937
- try { return fs.realpathSync(fileURLToPath(import.meta.url)) === fs.realpathSync(process.argv[1] || ""); }
938
- catch { return import.meta.url === pathToFileURL(process.argv[1] || "").href; }
939
- }
940
-
941
- if (invokedAsEntrypoint()) {
937
+ if (invokedAsEntrypoint(import.meta.url, process.argv[1])) {
942
938
  main().catch((error) => {
943
939
  console.error(`FAIL: ${error.message}`);
944
940
  process.exit(error.exitCode ?? 1);