@piagent/platform 1.5.5 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +117 -0
- package/README.md +9 -8
- package/README.vi.md +9 -5
- package/adapters/web-frontend/profile.json +2 -4
- package/architecture/layers.json +18 -3
- package/benchmarks/deep-logic-v1/SPECIALIST_PROTOCOL.vi.md +122 -0
- package/benchmarks/deep-logic-v1/grade.mjs +272 -10
- package/benchmarks/deep-logic-v1/project/apps/admin/src/billing-summary.js +3 -0
- package/benchmarks/deep-logic-v1/project/benchmark-contract/public-smoke.test.js +166 -0
- package/benchmarks/deep-logic-v1/project/package.json +1 -1
- package/benchmarks/deep-logic-v1/project/packages/billing/src/close-period.js +3 -0
- package/benchmarks/deep-logic-v1/project/packages/billing/src/plan-timeline.js +3 -0
- package/benchmarks/deep-logic-v1/project/test/smoke.test.js +7 -1
- package/benchmarks/deep-logic-v1/prompts/budgeted-context-graph.md +2 -0
- package/benchmarks/deep-logic-v1/prompts/fair-dependency-scheduler.md +2 -0
- package/benchmarks/deep-logic-v1/prompts/layered-policy-resolution.md +2 -0
- package/benchmarks/deep-logic-v1/prompts/resumable-stream-assembly.md +3 -1
- package/benchmarks/deep-logic-v1/prompts/revision-event-reconciliation.md +2 -0
- package/benchmarks/deep-logic-v1/prompts/temporal-usage-billing-close.md +123 -0
- package/benchmarks/deep-logic-v1/prompts/transactional-config-merge.md +2 -0
- package/benchmarks/deep-logic-v1/suite.json +51 -3
- package/benchmarks/deep-logic-v1/variant.mjs +22 -1
- package/benchmarks/production-v1/spend-control.v1.json +68 -0
- package/benchmarks/production-v1/suite.json +42 -3
- package/catalog/capabilities.json +6 -6
- package/docs/command-reference-vietnamese.md +6 -6
- package/docs/context-engine.md +7 -1
- package/docs/distribution-standard.md +7 -7
- package/docs/en/architecture.md +1 -0
- package/docs/languages.json +4 -4
- package/docs/model-options.md +8 -5
- package/docs/operator-manual-vietnamese.md +13 -11
- package/docs/project-adapters.md +2 -2
- package/docs/publishing-for-teams.md +5 -0
- package/docs/quality-benchmark.md +271 -62
- package/docs/release-install-policy.md +10 -9
- package/docs/runtime-policy-design.md +4 -4
- package/docs/runtime-quality-baseline.md +13 -6
- package/docs/security-threat-model.md +2 -2
- package/docs/task-lifecycle-vietnamese.md +2 -2
- package/docs/team-onboarding.md +2 -2
- package/docs/usage-observability.md +7 -2
- package/docs/vi/architecture.md +1 -0
- package/docs-site/architecture.html +1 -1
- package/docs-site/benchmark.html +1 -1
- package/docs-site/commands.html +1 -1
- package/docs-site/context-engine.html +1 -1
- package/docs-site/en/architecture.html +1 -1
- package/docs-site/en/benchmark.html +1 -1
- package/docs-site/en/commands.html +1 -1
- package/docs-site/en/context-engine.html +1 -1
- package/docs-site/en/index.html +1 -1
- package/docs-site/en/mcp-approval.html +1 -1
- package/docs-site/en/mcp-auth.html +1 -1
- package/docs-site/en/mcp-servers.html +1 -1
- package/docs-site/en/mcp.html +1 -1
- package/docs-site/en/permissions.html +1 -1
- package/docs-site/en/profiles.html +1 -1
- package/docs-site/en/quickstart.html +1 -1
- package/docs-site/en/runtime.html +1 -1
- package/docs-site/en/scope.html +1 -1
- package/docs-site/en/security.html +1 -1
- package/docs-site/en/team.html +1 -1
- package/docs-site/en/whats-new.html +1 -1
- package/docs-site/en/workflows.html +1 -1
- package/docs-site/index.html +1 -1
- package/docs-site/mcp-approval.html +1 -1
- package/docs-site/mcp-auth.html +1 -1
- package/docs-site/mcp-servers.html +1 -1
- package/docs-site/mcp.html +1 -1
- package/docs-site/permissions.html +1 -1
- package/docs-site/profiles.html +1 -1
- package/docs-site/quickstart.html +1 -1
- package/docs-site/runtime.html +1 -1
- package/docs-site/scope.html +1 -1
- package/docs-site/security.html +1 -1
- package/docs-site/team.html +1 -1
- package/docs-site/whats-new.html +1 -1
- package/docs-site/workflows.html +1 -1
- package/evals/fixtures/benchmark-assurance-evidence.invalid.json +1 -1
- package/evals/fixtures/benchmark-assurance-evidence.valid.json +1 -1
- package/evals/fixtures/piagent-webui/session-live-state-v1.invalid.json +18 -0
- package/evals/fixtures/piagent-webui/session-live-state-v1.valid.json +27 -0
- package/evals/fs4-readiness-matrix.v1.json +2 -2
- package/evals/fs5-causal-arm.v1.json +2 -2
- package/evals/fs5-pilot-protocol.v1.json +2 -2
- package/evals/fs5-pilot-protocol.v4.json +6 -5
- package/evals/fs5-pilot-protocol.v5.json +7 -6
- package/evals/private-holdout-v1/public-exposure.v1.json +1 -1
- package/package.json +3 -1
- package/packages/piagent-core/benchmark/benchmark-assurance.js +4 -1
- package/packages/piagent-core/benchmark/benchmark-bootstrap.js +91 -9
- package/packages/piagent-core/benchmark/benchmark-claim-restrictions.js +38 -7
- package/packages/piagent-core/benchmark/benchmark-cli.js +7 -7
- package/packages/piagent-core/benchmark/benchmark-codex.js +3 -1
- package/packages/piagent-core/benchmark/benchmark-comparison.js +271 -2
- package/packages/piagent-core/benchmark/benchmark-core.js +732 -661
- package/packages/piagent-core/benchmark/benchmark-execution-guard.js +7 -5
- package/packages/piagent-core/benchmark/benchmark-forensics.js +439 -4
- package/packages/piagent-core/benchmark/benchmark-host-readiness.js +639 -0
- package/packages/piagent-core/benchmark/benchmark-matrix.js +22 -4
- package/packages/piagent-core/benchmark/benchmark-normalized-cost.js +317 -0
- package/packages/piagent-core/benchmark/benchmark-preflight.js +3 -1
- package/packages/piagent-core/benchmark/benchmark-process.js +6 -5
- package/packages/piagent-core/benchmark/benchmark-provider-wire.js +126 -0
- package/packages/piagent-core/benchmark/benchmark-record-validation.js +183 -0
- package/packages/piagent-core/benchmark/benchmark-report.js +116 -12
- package/packages/piagent-core/benchmark/benchmark-resume-recovery.js +14 -0
- package/packages/piagent-core/benchmark/benchmark-runner-policy.js +26 -0
- package/packages/piagent-core/benchmark/benchmark-stage-diagnostic.js +782 -0
- package/packages/piagent-core/benchmark/benchmark-statistics.js +64 -0
- package/packages/piagent-core/benchmark/benchmark-stop-policy.js +8 -5
- package/packages/piagent-core/benchmark/benchmark-suite-identity.js +31 -0
- package/packages/piagent-core/benchmark/benchmark-suite-runtime.js +13 -9
- package/packages/piagent-core/benchmark/benchmark-suite.js +128 -2
- package/packages/piagent-core/benchmark/benchmark-summary-support.js +376 -0
- package/packages/piagent-core/benchmark/benchmark-timing-diagnostics.js +971 -0
- package/packages/piagent-core/benchmark/benchmark-usage.js +311 -0
- package/packages/piagent-core/benchmark/benchmark-workflow.js +166 -0
- package/packages/piagent-core/docs/context-efficiency-improvement-plan.md +1080 -0
- package/packages/piagent-core/extensions/acceptance-contract-semantics.js +83 -69
- package/packages/piagent-core/extensions/acceptance-error-classes.js +160 -0
- package/packages/piagent-core/extensions/acceptance-executable-evidence.js +12 -9
- package/packages/piagent-core/extensions/acceptance-receipt.js +34 -11
- package/packages/piagent-core/extensions/acceptance-state-machine-evidence.js +518 -0
- package/packages/piagent-core/extensions/acceptance-test-binding-evidence.js +60 -0
- package/packages/piagent-core/extensions/context-efficiency-metrics.js +302 -0
- package/packages/piagent-core/extensions/context-efficiency-report.js +167 -0
- package/packages/piagent-core/extensions/context-engine.js +148 -145
- package/packages/piagent-core/extensions/context-evidence.js +48 -0
- package/packages/piagent-core/extensions/context-import-links.js +296 -0
- package/packages/piagent-core/extensions/context-telemetry-integrity.js +65 -0
- package/packages/piagent-core/extensions/criterion-context-pack.js +422 -7
- package/packages/piagent-core/extensions/criterion-graph.js +78 -8
- package/packages/piagent-core/extensions/guard-types.ts +2 -0
- package/packages/piagent-core/extensions/javascript-regex-evidence.js +32 -0
- package/packages/piagent-core/extensions/piagent-guard.ts +233 -41
- package/packages/piagent-core/extensions/task-contract-view.js +17 -3
- package/packages/piagent-core/extensions/task-lifecycle.js +7 -2
- package/packages/piagent-core/extensions/task-state.js +40 -134
- package/packages/piagent-core/extensions/verification-intelligence.js +3 -1
- package/packages/piagent-core/extensions/workspace-evidence-roots.js +157 -0
- package/packages/piagent-core/package.json +1 -1
- package/packages/piagent-core/policies/base-policy.json +2 -0
- package/packages/piagent-core/runtime/context/context-delivery.ts +103 -0
- package/packages/piagent-core/runtime/context/context-delta-shadow.ts +53 -0
- package/packages/piagent-core/runtime/context/context-efficiency-format.ts +47 -0
- package/packages/piagent-core/runtime/context/context-evidence-qualification.ts +32 -0
- package/packages/piagent-core/runtime/context/prefix-telemetry.ts +47 -0
- package/packages/piagent-core/runtime/context/repository-memory-hints.ts +27 -0
- package/packages/piagent-core/runtime/hooks/agent-prompt-telemetry.ts +56 -0
- package/packages/piagent-core/runtime/hooks/agent-start-hook.ts +162 -102
- package/packages/piagent-core/runtime/hooks/completion-hook.ts +5 -6
- package/packages/piagent-core/runtime/hooks/input-hook.ts +11 -1
- package/packages/piagent-core/runtime/hooks/session-hooks.ts +22 -2
- package/packages/piagent-core/runtime/hooks/session-start-hook.ts +7 -2
- package/packages/piagent-core/runtime/hooks/tool-result-content-guards.ts +95 -0
- package/packages/piagent-core/runtime/hooks/tool-result-hook.ts +84 -102
- package/packages/piagent-core/runtime/inspection/activity-event-adapter.ts +4 -2
- package/packages/piagent-core/runtime/inspection/activity-recovery.ts +104 -0
- package/packages/piagent-core/runtime/inspection/task-compaction-history.ts +1 -1
- package/packages/piagent-core/runtime/inspection/task-handoff-history.ts +1 -1
- package/packages/piagent-core/runtime/inspection/tool-failure-classification.ts +53 -0
- package/packages/piagent-core/runtime/inspection/webui-snapshot.ts +34 -6
- package/packages/piagent-core/runtime/model/authenticated-catalog.ts +17 -9
- package/packages/piagent-core/runtime/model/openai-codex-reasoning.ts +83 -0
- package/packages/piagent-core/runtime/model/provider-wire-fingerprint.ts +238 -0
- package/packages/piagent-core/runtime/orchestration/role-policy.ts +1 -1
- package/packages/piagent-core/runtime/product/activity-inspector.ts +7 -2
- package/packages/piagent-core/runtime/product/efficiency-metrics.ts +4 -2
- package/packages/piagent-core/runtime/quality/exact-output-contract.ts +2 -1
- package/packages/piagent-core/runtime/quality/performance-assurance-signals.ts +23 -0
- package/packages/piagent-core/runtime/quality/performance-assurance.ts +6 -2
- package/packages/piagent-core/runtime/recovery/edit-recovery-context.ts +113 -0
- package/packages/piagent-core/runtime/recovery/edit-recovery-delivery.ts +42 -0
- package/packages/piagent-core/runtime/recovery/semantic-repair-handshake.ts +55 -2
- package/packages/piagent-core/runtime/recovery/semantic-repair-runtime.ts +55 -2
- package/packages/piagent-core/runtime/registration/context-commands.ts +66 -5
- package/packages/piagent-core/runtime/registration/policy-tools.ts +46 -11
- package/packages/piagent-core/runtime/registration/task-completion-tools.ts +49 -16
- package/packages/piagent-core/runtime/registration/task-start-guidance.ts +163 -0
- package/packages/piagent-core/runtime/registration/task-start-retry-helpers.ts +37 -0
- package/packages/piagent-core/runtime/registration/task-start-tool.ts +67 -62
- package/packages/piagent-core/runtime/session/runtime-state.ts +154 -2
- package/packages/piagent-core/runtime/session/system-prompt.ts +4 -4
- package/packages/piagent-core/runtime/tools/apply-patch-tool.ts +500 -0
- package/packages/piagent-core/runtime/trajectory/trajectory-runtime.ts +2 -1
- package/packages/piagent-core/runtime/workflows/task-intake.ts +43 -2
- package/packages/piagent-core/security/sensitive-data.js +13 -218
- package/packages/piagent-core/security/sensitive-project-file.js +42 -0
- package/packages/piagent-core/security/sensitive-source-expression.js +330 -0
- package/packages/piagent-core/security/sensitive-source-formats.js +394 -0
- package/packages/piagent-core/security/sensitive-text.js +234 -0
- package/packages/piagent-core/subagents/piagent-worker.md +1 -1
- package/packages/piagent-webui/dist/client/assets/{MarkdownMessage-rD9WJKxs.js → MarkdownMessage-CxeZqTte.js} +1 -1
- package/packages/piagent-webui/dist/client/assets/index-DynIMe8I.js +176 -0
- package/packages/piagent-webui/dist/client/index.html +1 -1
- package/packages/piagent-webui/gateway/gateway-events.ts +109 -5
- package/packages/piagent-webui/gateway/gateway-service.ts +3 -0
- package/packages/piagent-webui/gateway/gateway-session-stream.ts +93 -7
- package/packages/piagent-webui/gateway/session-command-controller.ts +18 -3
- package/packages/piagent-webui/gateway/session-effective-options.ts +47 -0
- package/packages/piagent-webui/gateway/session-live-state.ts +21 -0
- package/packages/piagent-webui/gateway/session-operation-start.ts +5 -1
- package/packages/piagent-webui/gateway/session-operation-watchdog.ts +158 -0
- package/packages/piagent-webui/gateway/session-runtime-factory.ts +45 -0
- package/packages/piagent-webui/gateway/session-runtime-supervisor.ts +139 -124
- package/packages/piagent-webui/server/loopback-server.ts +8 -0
- package/packages/piagent-webui/server/transcript-projection.ts +22 -4
- package/packages/piagent-webui/shared/text-visibility.ts +10 -0
- package/schemas/piagent-webui/catalog-v1.json +9 -0
- package/schemas/piagent-webui/gateway-protocol-v1.schema.json +53 -3
- package/schemas/piagent-webui/session-live-state-v1.schema.json +76 -0
- package/schemas/task-contract.schema.json +12 -1
- package/scripts/benchmark-runner-core.mjs +355 -359
- package/scripts/benchmark-runner-finalization.mjs +309 -0
- package/scripts/benchmark-runner-host-readiness.mjs +94 -0
- package/scripts/benchmark-runner-support.mjs +200 -0
- package/scripts/benchmark-session.mjs +95 -41
- package/scripts/configure-model-scope.sh +4 -3
- package/scripts/helper-evaluation.mjs +1 -1
- package/scripts/install-global.sh +2 -2
- package/scripts/pi-usage-history.mjs +37 -10
- package/scripts/setup.sh +2 -2
- package/scripts/verify-local.sh +5 -1
- package/templates/global/settings.json +2 -1
- package/packages/piagent-webui/dist/client/assets/index-BZRFw1u6.js +0 -174
|
@@ -1,26 +1,17 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import crypto from "node:crypto";
|
|
3
3
|
import fs from "node:fs";
|
|
4
|
-
import os from "node:os";
|
|
5
4
|
import path from "node:path";
|
|
6
|
-
import
|
|
7
|
-
import { fileURLToPath, pathToFileURL } from "node:url";
|
|
5
|
+
import { fileURLToPath } from "node:url";
|
|
8
6
|
|
|
9
|
-
import {
|
|
10
|
-
aggregateSessionUsage, benchmarkSurfaceLabel,
|
|
11
|
-
renderBenchmarkHtml,
|
|
12
|
-
renderBenchmarkText,
|
|
13
|
-
summarizeBenchmark
|
|
14
|
-
} from "../packages/piagent-core/benchmark/benchmark-core.js";
|
|
7
|
+
import { aggregateSessionUsage, benchmarkSurfaceLabel } from "../packages/piagent-core/benchmark/benchmark-core.js";
|
|
15
8
|
import { benchmarkUsage, parseBenchmarkArgs } from "../packages/piagent-core/benchmark/benchmark-cli.js";
|
|
16
9
|
import { codexModelName, codexThinkingEffort } from "../packages/piagent-core/benchmark/benchmark-codex.js";
|
|
17
|
-
import { benchmarkTrustChecklist } from "../packages/piagent-core/benchmark/benchmark-matrix.js";
|
|
18
|
-
import { applyBenchmarkClaimRestrictions } from "../packages/piagent-core/benchmark/benchmark-claim-restrictions.js";
|
|
19
10
|
import { benchmarkEnvironment, benchmarkEnvironmentPolicy, comparisonSurfaces, createCodexRuntime, piagentTreatment } from "../packages/piagent-core/benchmark/benchmark-runtime.js";
|
|
20
11
|
import { assertBenchmarkPiCredentialReady, assertBenchmarkPiCredentialWritebackPolicy, cleanupBenchmarkPiRuntimeHome, createBenchmarkPiRuntimeHome, resetBenchmarkPiRuntimeEphemeralState, withBenchmarkPiCredentialWriteback } from "../packages/piagent-core/benchmark/benchmark-pi-home.js";
|
|
21
12
|
import { benchmarkPreflight, benchmarkPreflightReceipt } from "../packages/piagent-core/benchmark/benchmark-preflight.js";
|
|
13
|
+
import { applyBenchmarkExecutionDefaults } from "../packages/piagent-core/benchmark/benchmark-runner-policy.js";
|
|
22
14
|
import {
|
|
23
|
-
cleanupUnretainedWorkspaces,
|
|
24
15
|
appendPrivateJsonl,
|
|
25
16
|
createBenchmarkCandidateGuard,
|
|
26
17
|
loadReplayFailurePlan,
|
|
@@ -28,7 +19,6 @@ import {
|
|
|
28
19
|
safeInfrastructureDiagnostic,
|
|
29
20
|
writeBenchmarkAbort,
|
|
30
21
|
writeBenchmarkRunManifest,
|
|
31
|
-
writePrivate,
|
|
32
22
|
writePrivateAtomic
|
|
33
23
|
} from "../packages/piagent-core/benchmark/benchmark-forensics.js";
|
|
34
24
|
import { benchmarkBootstrapCandidateIndex, benchmarkBootstrapMetadata } from "../packages/piagent-core/benchmark/benchmark-bootstrap.js";
|
|
@@ -37,8 +27,16 @@ import { benchmarkCommandIdentity } from "../packages/piagent-core/benchmark/ben
|
|
|
37
27
|
import { benchmarkTreeIdentity } from "../packages/piagent-core/benchmark/benchmark-tree-identity.js";
|
|
38
28
|
import { completedBenchmarkRecord, expectedBenchmarkRecord } from "../packages/piagent-core/benchmark/benchmark-record-validation.js";
|
|
39
29
|
import { pairedOutcomeFloorStop } from "../packages/piagent-core/benchmark/benchmark-stop-policy.js";
|
|
30
|
+
import {
|
|
31
|
+
approveProductionStageControl,
|
|
32
|
+
buildBenchmarkStageDiagnostic,
|
|
33
|
+
createProductionStageControl,
|
|
34
|
+
durablePairedOutcomeFloorStop,
|
|
35
|
+
productionSpendControlValidationErrors,
|
|
36
|
+
productionStageResumeDisposition
|
|
37
|
+
} from "../packages/piagent-core/benchmark/benchmark-stage-diagnostic.js";
|
|
40
38
|
import { loadBenchmarkAssuranceEvidence, loadBenchmarkSuite, resolveBenchmarkSuiteEntry, validateBenchmarkSuiteFiles } from "../packages/piagent-core/benchmark/benchmark-suite-runtime.js";
|
|
41
|
-
import { appendBenchmarkLedger, assertBenchmarkLedgerBinding,
|
|
39
|
+
import { appendBenchmarkLedger, assertBenchmarkLedgerBinding, emptyBenchmarkLedgerBinding, inspectBenchmarkLedger, validateBenchmarkLedgerPrefix } from "../packages/piagent-core/benchmark/benchmark-ledger.js";
|
|
42
40
|
import { acquireBenchmarkRunLock } from "../packages/piagent-core/benchmark/benchmark-run-lock.js";
|
|
43
41
|
import { createBenchmarkProcessController } from "../packages/piagent-core/benchmark/benchmark-process.js";
|
|
44
42
|
import {
|
|
@@ -49,7 +47,33 @@ import {
|
|
|
49
47
|
recoverPendingBenchmarkRecord,
|
|
50
48
|
stageMeasuredBenchmarkRecord
|
|
51
49
|
} from "../packages/piagent-core/benchmark/benchmark-resume-recovery.js";
|
|
50
|
+
import { finalizeBenchmarkRun } from "./benchmark-runner-finalization.mjs";
|
|
51
|
+
import {
|
|
52
|
+
benchmarkRunKey,
|
|
53
|
+
confirmPlan,
|
|
54
|
+
createRunId,
|
|
55
|
+
defaultOutputRoot,
|
|
56
|
+
ensureEmptyOutput,
|
|
57
|
+
executionOrder,
|
|
58
|
+
fail,
|
|
59
|
+
formatDuration,
|
|
60
|
+
frozenRuntimeCommandsForFinalization,
|
|
61
|
+
invokedAsEntrypoint,
|
|
62
|
+
isLegacyInvocation,
|
|
63
|
+
loadResumeState,
|
|
64
|
+
pairedChunk,
|
|
65
|
+
privateDirectory,
|
|
66
|
+
readJsonFile,
|
|
67
|
+
samePairedBlock,
|
|
68
|
+
sameStringList
|
|
69
|
+
} from "./benchmark-runner-support.mjs";
|
|
52
70
|
import { runBenchmarkSession } from "./benchmark-session.mjs";
|
|
71
|
+
import {
|
|
72
|
+
assertBenchmarkHostReadinessStartReady,
|
|
73
|
+
assertExistingBenchmarkHostReadiness,
|
|
74
|
+
benchmarkHostReadinessPolicyDigest,
|
|
75
|
+
collectReadyBenchmarkHostReadiness
|
|
76
|
+
} from "./benchmark-runner-host-readiness.mjs";
|
|
53
77
|
export { parseBenchmarkArgs };
|
|
54
78
|
const packageRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
|
55
79
|
const packageManifest = JSON.parse(fs.readFileSync(path.join(packageRoot, "package.json"), "utf8"));
|
|
@@ -58,11 +82,6 @@ const bootstrapCandidateIndex = bootstrapMetadata ? benchmarkBootstrapCandidateI
|
|
|
58
82
|
let interruptedSignal;
|
|
59
83
|
const processController = createBenchmarkProcessController(() => Boolean(interruptedSignal));
|
|
60
84
|
const runCommand = processController.run;
|
|
61
|
-
function fail(message, code = 2) {
|
|
62
|
-
const error = new Error(message);
|
|
63
|
-
error.exitCode = code;
|
|
64
|
-
throw error;
|
|
65
|
-
}
|
|
66
85
|
|
|
67
86
|
function installSignalForwarding() {
|
|
68
87
|
const handlers = new Map();
|
|
@@ -80,154 +99,6 @@ function installSignalForwarding() {
|
|
|
80
99
|
};
|
|
81
100
|
}
|
|
82
101
|
|
|
83
|
-
function isLegacyInvocation(argv) {
|
|
84
|
-
return argv.includes("--record") || argv.includes("--init");
|
|
85
|
-
}
|
|
86
|
-
|
|
87
|
-
function privateDirectory(target) {
|
|
88
|
-
fs.mkdirSync(target, { recursive: true, mode: 0o700 });
|
|
89
|
-
try { fs.chmodSync(target, 0o700); } catch { /* Non-POSIX filesystem. */ }
|
|
90
|
-
return target;
|
|
91
|
-
}
|
|
92
|
-
|
|
93
|
-
function defaultOutputRoot() {
|
|
94
|
-
if (bootstrapMetadata?.defaultOutputRoot) return bootstrapMetadata.defaultOutputRoot;
|
|
95
|
-
const agentRoot = process.env.PI_CODING_AGENT_DIR || path.join(os.homedir(), ".pi", "agent");
|
|
96
|
-
return path.join(agentRoot, "benchmarks", "piagent");
|
|
97
|
-
}
|
|
98
|
-
|
|
99
|
-
function createRunId(suiteId) {
|
|
100
|
-
const timestamp = new Date().toISOString().replace(/[-:]/g, "").replace(/\.\d{3}Z$/, "Z");
|
|
101
|
-
return `${suiteId}-${timestamp}-${crypto.randomBytes(3).toString("hex")}`;
|
|
102
|
-
}
|
|
103
|
-
|
|
104
|
-
function ensureEmptyOutput(target) {
|
|
105
|
-
if (fs.existsSync(target) && fs.readdirSync(target).length > 0) fail(`Output directory is not empty: ${target}`, 1);
|
|
106
|
-
return privateDirectory(target);
|
|
107
|
-
}
|
|
108
|
-
|
|
109
|
-
function benchmarkRunKey(value) {
|
|
110
|
-
return `${value.scenarioId ?? value.scenario?.id}\0${value.surface}\0${value.repeat}`;
|
|
111
|
-
}
|
|
112
|
-
|
|
113
|
-
function samePairedBlock(left, right) {
|
|
114
|
-
return left?.scenario?.id === right?.scenario?.id && left?.repeat === right?.repeat;
|
|
115
|
-
}
|
|
116
|
-
|
|
117
|
-
function pairedChunk(order, maximum) {
|
|
118
|
-
if (maximum === undefined || order.length <= maximum) return order;
|
|
119
|
-
let length = maximum;
|
|
120
|
-
while (length > 0 && samePairedBlock(order[length - 1], order[length])) length -= 1;
|
|
121
|
-
if (length === 0) fail(`--max-sessions ${maximum} would split the first paired benchmark block; increase the chunk size`, 1);
|
|
122
|
-
return order.slice(0, length);
|
|
123
|
-
}
|
|
124
|
-
|
|
125
|
-
function readJsonFile(file, label) {
|
|
126
|
-
try {
|
|
127
|
-
return JSON.parse(fs.readFileSync(file, "utf8"));
|
|
128
|
-
} catch (error) {
|
|
129
|
-
fail(`Cannot read ${label} ${file}: ${error.message}`, 1);
|
|
130
|
-
}
|
|
131
|
-
}
|
|
132
|
-
|
|
133
|
-
function resolveResumeRunRoot(input) {
|
|
134
|
-
const target = path.resolve(input);
|
|
135
|
-
try {
|
|
136
|
-
const stat = fs.statSync(target);
|
|
137
|
-
if (stat.isDirectory()) return target;
|
|
138
|
-
if (stat.isFile()) return path.dirname(target);
|
|
139
|
-
} catch (error) {
|
|
140
|
-
fail(`Cannot resume benchmark; path does not exist: ${target}`, 1);
|
|
141
|
-
}
|
|
142
|
-
fail(`Cannot resume benchmark; path is not a file or directory: ${target}`, 1);
|
|
143
|
-
}
|
|
144
|
-
|
|
145
|
-
function loadResumeState(input) {
|
|
146
|
-
const runRoot = resolveResumeRunRoot(input);
|
|
147
|
-
const releaseRunLock = acquireBenchmarkRunLock(runRoot, "resume-pending");
|
|
148
|
-
try {
|
|
149
|
-
const manifestPath = path.join(runRoot, "run-manifest.json");
|
|
150
|
-
if (!fs.existsSync(manifestPath)) {
|
|
151
|
-
fail(`Cannot resume ${runRoot}: missing run-manifest.json. This run was created before resume metadata was written, so its root seed cannot be recovered safely. Start a new run with --max-sessions or --max-runtime-minutes to make it resumable.`, 1);
|
|
152
|
-
}
|
|
153
|
-
const manifest = readJsonFile(manifestPath, "benchmark resume manifest");
|
|
154
|
-
if (fs.existsSync(path.join(runRoot, "stopped.json"))) fail(`Cannot resume ${runRoot}: the paired release stop is terminal`, 1);
|
|
155
|
-
if (manifest?.schemaVersion !== 1 || typeof manifest.runId !== "string") {
|
|
156
|
-
fail(`Cannot resume ${runRoot}: run-manifest.json has an unsupported shape`, 1);
|
|
157
|
-
}
|
|
158
|
-
const ledger = benchmarkLedgerCheckpoint(
|
|
159
|
-
manifest.ledger,
|
|
160
|
-
inspectBenchmarkLedger(path.join(runRoot, "runs.jsonl")),
|
|
161
|
-
"benchmark resume ledger"
|
|
162
|
-
);
|
|
163
|
-
const pendingPath = path.join(runRoot, "pending-record.json");
|
|
164
|
-
const pendingRecord = fs.existsSync(pendingPath) ? readJsonFile(pendingPath, "benchmark pending record") : null;
|
|
165
|
-
const measuredPath = path.join(runRoot, "measured-record-ready.json");
|
|
166
|
-
const measuredReady = fs.existsSync(measuredPath) ? readJsonFile(measuredPath, "measured benchmark record") : null;
|
|
167
|
-
return {
|
|
168
|
-
runRoot,
|
|
169
|
-
manifest,
|
|
170
|
-
completedRuns: ledger.records,
|
|
171
|
-
ledgerBinding: ledger.binding,
|
|
172
|
-
recoveredLedgerSuffix: ledger.recovered,
|
|
173
|
-
pendingRecord,
|
|
174
|
-
measuredReady,
|
|
175
|
-
releaseRunLock
|
|
176
|
-
};
|
|
177
|
-
} catch (error) {
|
|
178
|
-
releaseRunLock();
|
|
179
|
-
throw error;
|
|
180
|
-
}
|
|
181
|
-
}
|
|
182
|
-
|
|
183
|
-
function sameStringList(left, right) {
|
|
184
|
-
return Array.isArray(left)
|
|
185
|
-
&& Array.isArray(right)
|
|
186
|
-
&& left.length === right.length
|
|
187
|
-
&& left.every((value, index) => value === right[index]);
|
|
188
|
-
}
|
|
189
|
-
|
|
190
|
-
function formatDuration(ms) {
|
|
191
|
-
const totalSeconds = Math.max(0, Math.round(ms / 1_000));
|
|
192
|
-
const hours = Math.floor(totalSeconds / 3600);
|
|
193
|
-
const minutes = Math.floor((totalSeconds % 3600) / 60);
|
|
194
|
-
const seconds = totalSeconds % 60;
|
|
195
|
-
if (hours > 0) return `${hours}h${String(minutes).padStart(2, "0")}m`;
|
|
196
|
-
if (minutes > 0) return `${minutes}m${String(seconds).padStart(2, "0")}s`;
|
|
197
|
-
return `${seconds}s`;
|
|
198
|
-
}
|
|
199
|
-
|
|
200
|
-
function executionOrder(suite, repeats, surfaces, rootSeed) {
|
|
201
|
-
const order = [];
|
|
202
|
-
for (let repeat = 1; repeat <= repeats; repeat += 1) {
|
|
203
|
-
const scenarios = suite.schemaVersion === 2
|
|
204
|
-
? [...suite.scenarios].sort((left, right) => {
|
|
205
|
-
const rank = (scenario) => crypto.createHmac("sha256", rootSeed).update(`order\0${repeat}\0${scenario.id}`).digest("hex");
|
|
206
|
-
return rank(left).localeCompare(rank(right));
|
|
207
|
-
})
|
|
208
|
-
: suite.scenarios;
|
|
209
|
-
for (const [index, scenario] of scenarios.entries()) {
|
|
210
|
-
const reverse = suite.schemaVersion === 2
|
|
211
|
-
? (crypto.createHmac("sha256", rootSeed).update(`surface\0${repeat}\0${scenario.id}`).digest()[0] & 1) === 1
|
|
212
|
-
: (repeat + index) % 2 !== 0;
|
|
213
|
-
const ordered = reverse ? [...surfaces].reverse() : surfaces;
|
|
214
|
-
for (const surface of ordered) order.push({ scenario, surface, repeat });
|
|
215
|
-
}
|
|
216
|
-
}
|
|
217
|
-
return order;
|
|
218
|
-
}
|
|
219
|
-
|
|
220
|
-
async function confirmPlan(message) {
|
|
221
|
-
if (!process.stdin.isTTY || !process.stdout.isTTY) fail("Refusing to start billed model runs without --yes in a non-interactive terminal", 1);
|
|
222
|
-
const terminal = readline.createInterface({ input: process.stdin, output: process.stdout });
|
|
223
|
-
try {
|
|
224
|
-
const answer = await terminal.question(`${message}\nContinue? [y/N] `);
|
|
225
|
-
return /^(?:y|yes)$/i.test(answer.trim());
|
|
226
|
-
} finally {
|
|
227
|
-
terminal.close();
|
|
228
|
-
}
|
|
229
|
-
}
|
|
230
|
-
|
|
231
102
|
async function runLegacy(argv) {
|
|
232
103
|
const result = await runCommand("bash", [path.join(packageRoot, "scripts", "quality-benchmark.sh"), ...argv], { cwd: process.cwd(), inherit: true });
|
|
233
104
|
process.exitCode = result.code;
|
|
@@ -264,6 +135,7 @@ async function main() {
|
|
|
264
135
|
options.timeoutSeconds = manifest.timeoutSeconds;
|
|
265
136
|
options.infrastructureRetries = manifest.infrastructureRetries;
|
|
266
137
|
options.retryDelaySeconds = manifest.retryDelaySeconds;
|
|
138
|
+
options.stopAfterFailedPair = manifest.stopAfterFailedPair === true;
|
|
267
139
|
options.output = resumeState.runRoot;
|
|
268
140
|
}
|
|
269
141
|
if (options.replayFailures) {
|
|
@@ -283,14 +155,43 @@ async function main() {
|
|
|
283
155
|
evidenceComplete: bootstrapMetadata?.replay?.evidenceComplete ?? replay.source.evidenceComplete
|
|
284
156
|
};
|
|
285
157
|
}
|
|
286
|
-
if (bootstrapMetadata?.suite?.
|
|
158
|
+
if (bootstrapMetadata?.suite?.builtInId) options.suite = bootstrapMetadata.suite.builtInId;
|
|
159
|
+
else if (bootstrapMetadata?.suite?.snapshot) options.suite = bootstrapMetadata.suite.snapshot;
|
|
287
160
|
if (options.help) {
|
|
288
161
|
process.stdout.write(benchmarkUsage);
|
|
289
162
|
return;
|
|
290
163
|
}
|
|
291
164
|
piagentTreatment(options.piagentTreatment);
|
|
292
|
-
const { suite, manifestPath, suiteRoot } = loadBenchmarkSuite(options.suite, packageRoot);
|
|
165
|
+
const { suite, manifestPath, suiteRoot, builtInId } = loadBenchmarkSuite(options.suite, packageRoot);
|
|
293
166
|
validateBenchmarkSuiteFiles(suite, suiteRoot);
|
|
167
|
+
if (bootstrapMetadata && bootstrapMetadata.suite.builtInId !== builtInId) {
|
|
168
|
+
fail("Frozen benchmark suite origin no longer matches its canonical built-in identity", 1);
|
|
169
|
+
}
|
|
170
|
+
const canonicalProductionSuite = builtInId === "production-v1";
|
|
171
|
+
const productionSpendControlPath = path.join(suiteRoot, "spend-control.v1.json");
|
|
172
|
+
const productionSpendControl = canonicalProductionSuite || fs.existsSync(productionSpendControlPath)
|
|
173
|
+
? readJsonFile(productionSpendControlPath, "production spend-control contract")
|
|
174
|
+
: null;
|
|
175
|
+
const productionExpectedSessions = productionSpendControl
|
|
176
|
+
&& Number.isSafeInteger(productionSpendControl.execution?.repeats)
|
|
177
|
+
&& Array.isArray(productionSpendControl.execution?.surfaces)
|
|
178
|
+
? suite.scenarios.length * productionSpendControl.execution.repeats * productionSpendControl.execution.surfaces.length
|
|
179
|
+
: undefined;
|
|
180
|
+
const productionSpendControlErrors = productionSpendControl
|
|
181
|
+
? productionSpendControlValidationErrors(productionSpendControl, {
|
|
182
|
+
suiteId: suite.id,
|
|
183
|
+
expectedSessions: productionExpectedSessions,
|
|
184
|
+
requireHostReadiness: canonicalProductionSuite
|
|
185
|
+
&& suite.releaseGate?.requireHostReadinessForClaim === true
|
|
186
|
+
})
|
|
187
|
+
: [];
|
|
188
|
+
if (productionSpendControlErrors.length > 0) {
|
|
189
|
+
fail(`Production spend-control contract is invalid (${productionSpendControlErrors.join(", ")})`, 1);
|
|
190
|
+
}
|
|
191
|
+
const productionAllStageBoundaries = productionSpendControl
|
|
192
|
+
? productionSpendControl.stages.map((stage) => stage.cumulativeSessions)
|
|
193
|
+
: [];
|
|
194
|
+
const productionStageBoundaries = productionAllStageBoundaries.slice(1);
|
|
294
195
|
const assuranceEvidence = loadBenchmarkAssuranceEvidence(suite, suiteRoot);
|
|
295
196
|
const declaredScenarioCount = suite.scenarios.length;
|
|
296
197
|
if (options.scenarioIds) {
|
|
@@ -305,12 +206,24 @@ async function main() {
|
|
|
305
206
|
if (missing.length) fail(`Replay report references unknown scenario: ${missing.join(", ")}`, 1);
|
|
306
207
|
suite.scenarios = [...new Set(options.replayRuns.map((run) => run.scenarioId))].map((id) => byId.get(id));
|
|
307
208
|
}
|
|
308
|
-
options
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
209
|
+
applyBenchmarkExecutionDefaults(options, suite);
|
|
210
|
+
const productionFullMatrixRequested = Boolean(productionSpendControl)
|
|
211
|
+
&& !options.replayRuns
|
|
212
|
+
&& suite.scenarios.length === declaredScenarioCount;
|
|
213
|
+
if (productionFullMatrixRequested) {
|
|
214
|
+
const spendExecution = productionSpendControl.execution;
|
|
215
|
+
if (options.seed === undefined) options.seed = productionSpendControl.rootSeed;
|
|
216
|
+
if (options.seed !== productionSpendControl.rootSeed) fail("Production spend control requires its frozen root seed", 1);
|
|
217
|
+
if (!sameStringList(options.surfaces, spendExecution.surfaces)) fail("Production spend control requires its frozen surface order", 1);
|
|
218
|
+
if ((options.model ?? null) !== spendExecution.model) fail("Production spend control requires its frozen model", 1);
|
|
219
|
+
if ((options.thinking ?? null) !== spendExecution.thinking) fail("Production spend control requires its frozen thinking level", 1);
|
|
220
|
+
if (options.repeats !== spendExecution.repeats) fail("Production spend control requires its frozen repeat count", 1);
|
|
221
|
+
if (options.infrastructureRetries !== spendExecution.infrastructureRetries) fail("Production spend control requires zero infrastructure retries", 1);
|
|
222
|
+
}
|
|
312
223
|
if (options.stopAfterFailedPair && !Number.isFinite(suite.releaseGate?.minimumOutcomeScoreExclusive)) fail("--stop-after-failed-pair requires a suite outcome floor", 1);
|
|
313
224
|
if (options.surfaces.includes("codex-cli")) {
|
|
225
|
+
options.model = options.model ?? "openai-codex/gpt-5.6-luna";
|
|
226
|
+
options.thinking = options.thinking ?? "medium";
|
|
314
227
|
codexModelName(options.model);
|
|
315
228
|
codexThinkingEffort(options.thinking);
|
|
316
229
|
}
|
|
@@ -334,6 +247,24 @@ async function main() {
|
|
|
334
247
|
repeat: run.repeat
|
|
335
248
|
}))
|
|
336
249
|
: executionOrder(suite, options.repeats, options.surfaces, rootSeed);
|
|
250
|
+
const productionSpendControlled = productionFullMatrixRequested;
|
|
251
|
+
const productionHostReadinessRequired = productionSpendControlled
|
|
252
|
+
&& canonicalProductionSuite
|
|
253
|
+
&& suite.releaseGate?.requireHostReadinessForClaim === true;
|
|
254
|
+
const productionHostReadinessPolicy = productionHostReadinessRequired
|
|
255
|
+
? productionSpendControl.hostReadiness
|
|
256
|
+
: null;
|
|
257
|
+
const productionHostReadinessPolicyDigest = productionHostReadinessRequired
|
|
258
|
+
? benchmarkHostReadinessPolicyDigest(productionHostReadinessPolicy)
|
|
259
|
+
: null;
|
|
260
|
+
const productionReleaseClaimRun = productionSpendControlled
|
|
261
|
+
&& suite.schemaVersion === 2
|
|
262
|
+
&& suite.releaseGate?.requireEfficiencyClaim === true
|
|
263
|
+
&& suite.releaseGate?.requireFullSuiteForClaim === true;
|
|
264
|
+
if (productionSpendControlled && productionStageBoundaries.at(-1) !== fullOrder.length) {
|
|
265
|
+
fail("Production spend control final stage does not match the frozen full execution order", 1);
|
|
266
|
+
}
|
|
267
|
+
let deferredProductionStageApproval;
|
|
337
268
|
if (resumeState) {
|
|
338
269
|
const manifest = resumeState.manifest;
|
|
339
270
|
if (manifest.suiteDigest !== suiteDigest) fail("Cannot resume benchmark: suite files changed since the original run", 1);
|
|
@@ -381,7 +312,104 @@ async function main() {
|
|
|
381
312
|
});
|
|
382
313
|
throw provenanceError;
|
|
383
314
|
}
|
|
315
|
+
const recoveredTerminalStop = durablePairedOutcomeFloorStop({
|
|
316
|
+
enabled: options.stopAfterFailedPair,
|
|
317
|
+
suite,
|
|
318
|
+
runs: resumeState.completedRuns,
|
|
319
|
+
fullOrder
|
|
320
|
+
});
|
|
321
|
+
if (recoveredTerminalStop) {
|
|
322
|
+
for (const marker of ["paused.json", "stage-diagnostic.json", "interrupted.json", "aborted.json"]) {
|
|
323
|
+
fs.rmSync(path.join(resumeState.runRoot, marker), { force: true });
|
|
324
|
+
}
|
|
325
|
+
writePrivateAtomic(path.join(resumeState.runRoot, "stopped.json"), `${JSON.stringify({
|
|
326
|
+
...recoveredTerminalStop,
|
|
327
|
+
runId: manifest.runId,
|
|
328
|
+
completedRuns: resumeState.completedRuns.length,
|
|
329
|
+
expectedRuns: fullOrder.length,
|
|
330
|
+
stoppedAt: new Date().toISOString(),
|
|
331
|
+
resumeAllowed: false,
|
|
332
|
+
recoveredFromAcceptedLedger: true,
|
|
333
|
+
ledger: resumeState.ledgerBinding,
|
|
334
|
+
provenanceStamp: candidateGuard.stamp("resume-terminal-floor")
|
|
335
|
+
}, null, 2)}\n`);
|
|
336
|
+
fail("Cannot resume benchmark: the accepted ledger already contains a terminal paired outcome-floor failure", 1);
|
|
337
|
+
}
|
|
338
|
+
if (productionSpendControlled) {
|
|
339
|
+
if (productionHostReadinessRequired) {
|
|
340
|
+
if (manifest.hostReadinessPolicyDigest !== productionHostReadinessPolicyDigest
|
|
341
|
+
&& resumeState.completedRuns.length < fullOrder.length) {
|
|
342
|
+
fail("Cannot resume production benchmark: host-readiness policy binding is missing or changed. Start a new staged run.", 1);
|
|
343
|
+
}
|
|
344
|
+
if (resumeState.completedRuns.length < fullOrder.length) {
|
|
345
|
+
assertExistingBenchmarkHostReadiness({
|
|
346
|
+
policy: productionHostReadinessPolicy,
|
|
347
|
+
receipts: manifest.hostReadinessReceipts,
|
|
348
|
+
completedRuns: resumeState.completedRuns.length,
|
|
349
|
+
authorizedThroughRuns: manifest.stageControl?.authorizedThroughRuns,
|
|
350
|
+
runId: manifest.runId,
|
|
351
|
+
configurationDigest: manifest.configurationDigest,
|
|
352
|
+
stageBoundaries: productionAllStageBoundaries
|
|
353
|
+
});
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
const stageDisposition = productionStageResumeDisposition(manifest.stageControl, {
|
|
357
|
+
completedRuns: resumeState.completedRuns.length,
|
|
358
|
+
ledger: resumeState.ledgerBinding,
|
|
359
|
+
stageBoundaries: productionAllStageBoundaries
|
|
360
|
+
});
|
|
361
|
+
if (!stageDisposition.passed) {
|
|
362
|
+
fail(`Cannot resume production benchmark: invalid durable stage-control state (${stageDisposition.errors.join(", ")}). Start a new staged run.`, 1);
|
|
363
|
+
}
|
|
364
|
+
if (stageDisposition.requiresStageGate && resumeState.completedRuns.length < fullOrder.length) {
|
|
365
|
+
const recoveredBoundaryReason = `max-sessions:recovered:${resumeState.completedRuns.length}`;
|
|
366
|
+
const resumeStageDiagnostic = buildBenchmarkStageDiagnostic({
|
|
367
|
+
runId: manifest.runId,
|
|
368
|
+
reason: manifest.stageControl.pendingBoundary?.reason ?? recoveredBoundaryReason,
|
|
369
|
+
runs: resumeState.completedRuns,
|
|
370
|
+
fullOrder,
|
|
371
|
+
candidateSurface: comparison.candidateSurface,
|
|
372
|
+
baselineSurface: comparison.baselineSurface,
|
|
373
|
+
requestedModel: options.model,
|
|
374
|
+
requestedThinking: options.thinking,
|
|
375
|
+
suite,
|
|
376
|
+
manifest,
|
|
377
|
+
hostReadinessPolicy: productionHostReadinessPolicy,
|
|
378
|
+
hostReadinessStageBoundaries: productionAllStageBoundaries
|
|
379
|
+
});
|
|
380
|
+
writePrivateAtomic(path.join(resumeState.runRoot, "stage-diagnostic.json"), `${JSON.stringify(resumeStageDiagnostic, null, 2)}\n`);
|
|
381
|
+
if (!resumeStageDiagnostic.stageAdvanceAllowed) {
|
|
382
|
+
fail(`Cannot resume production benchmark: the provider-free stage gate blocked further paid sessions (${resumeStageDiagnostic.blockingReasons.join(", ")}). Fix the candidate and start a new staged run.`, 1);
|
|
383
|
+
}
|
|
384
|
+
const currentWindow = manifest.stageControl.authorizedThroughRuns;
|
|
385
|
+
const nextAuthorizedThroughRuns = resumeState.completedRuns.length < currentWindow
|
|
386
|
+
? currentWindow
|
|
387
|
+
: productionStageBoundaries.find((boundary) => boundary > resumeState.completedRuns.length);
|
|
388
|
+
if (!Number.isSafeInteger(nextAuthorizedThroughRuns)) {
|
|
389
|
+
fail("Cannot resume production benchmark: no bounded spend-control window remains", 1);
|
|
390
|
+
}
|
|
391
|
+
const requiredResumeSessions = nextAuthorizedThroughRuns - resumeState.completedRuns.length;
|
|
392
|
+
if (!options.dryRun && options.maxSessions !== requiredResumeSessions) {
|
|
393
|
+
fail(`Production spend control requires --max-sessions ${requiredResumeSessions} for the next authorized window; durable stage state was not changed.`, 1);
|
|
394
|
+
}
|
|
395
|
+
if (!options.dryRun) {
|
|
396
|
+
deferredProductionStageApproval = {
|
|
397
|
+
completedRuns: resumeState.completedRuns.length,
|
|
398
|
+
authorizedThroughRuns: nextAuthorizedThroughRuns
|
|
399
|
+
};
|
|
400
|
+
}
|
|
401
|
+
} else if (stageDisposition.requiresStageGate) {
|
|
402
|
+
// A crash after the final accepted record must not change the verdict by
|
|
403
|
+
// applying an early-stage heuristic. The complete report gates the same
|
|
404
|
+
// frozen ledger without starting another provider session.
|
|
405
|
+
manifest.ledger = resumeState.ledgerBinding;
|
|
406
|
+
writeBenchmarkRunManifest(resumeState.runRoot, manifest);
|
|
407
|
+
}
|
|
408
|
+
}
|
|
384
409
|
}
|
|
410
|
+
const productionFinalizationOnly = productionSpendControlled
|
|
411
|
+
&& Boolean(resumeState)
|
|
412
|
+
&& resumeState.completedRuns.length === fullOrder.length;
|
|
385
413
|
const pendingOrder = resumeState
|
|
386
414
|
? fullOrder.filter((item) => !resumeState.completedKeys.has(benchmarkRunKey(item)))
|
|
387
415
|
: fullOrder;
|
|
@@ -416,24 +444,64 @@ async function main() {
|
|
|
416
444
|
const codexPlan = options.surfaces.includes("codex-cli")
|
|
417
445
|
? `\n codex: ${options.codexMode} mode · model ${codexModelName(options.model)} · effort ${codexThinkingEffort(options.thinking)}${options.codexMode === "controlled" ? " · isolated home" : ""}`
|
|
418
446
|
: "";
|
|
447
|
+
const nativeWarning = options.surfaces.includes("codex-cli") && options.codexMode === "native"
|
|
448
|
+
? "\nNative Codex mode loads the operator's global AGENTS.md, configuration, rules, hooks, MCP servers, and plugins."
|
|
449
|
+
: "";
|
|
419
450
|
if (options.dryRun) {
|
|
420
451
|
process.stdout.write(`${plan}${codexPlan}\n manifest: ${manifestPath}\nDRY RUN: no model session started.\n`);
|
|
421
452
|
return;
|
|
422
453
|
}
|
|
454
|
+
if (productionSpendControlled) {
|
|
455
|
+
if (options.stopAfterFailedPair !== productionSpendControl.execution.stopAfterFailedPair) {
|
|
456
|
+
fail("Production spend control requires --stop-after-failed-pair before any provider session", 1);
|
|
457
|
+
}
|
|
458
|
+
const completedRuns = resumeState?.completedRuns.length ?? 0;
|
|
459
|
+
const authorizedThroughRuns = deferredProductionStageApproval?.authorizedThroughRuns
|
|
460
|
+
?? resumeState?.manifest.stageControl?.authorizedThroughRuns
|
|
461
|
+
?? productionStageBoundaries[0];
|
|
462
|
+
const requiredChunkSessions = authorizedThroughRuns - completedRuns;
|
|
463
|
+
if (!options.preflightOnly && !productionFinalizationOnly
|
|
464
|
+
&& (!Number.isSafeInteger(requiredChunkSessions) || requiredChunkSessions <= 0
|
|
465
|
+
|| options.maxSessions !== requiredChunkSessions)) {
|
|
466
|
+
fail(`Production spend control requires --max-sessions ${requiredChunkSessions} for the current authorized window; refusing an unbounded or oversized paid chunk.`, 1);
|
|
467
|
+
}
|
|
468
|
+
}
|
|
423
469
|
if (!bootstrapMetadata) fail("Modern billed benchmarks must start through scripts/benchmark-runner.mjs so execution assets are frozen", 1);
|
|
470
|
+
if (productionReleaseClaimRun && bootstrapMetadata.sourceIdentity.dirty !== false) {
|
|
471
|
+
fail("Production benchmark requires a clean release source before auth, tool preflight, or any provider session", 1);
|
|
472
|
+
}
|
|
424
473
|
if (options.replayFailures && bootstrapMetadata.replay?.evidenceComplete !== true) {
|
|
425
474
|
fail("Billed replay requires the original run-manifest.json and runs.jsonl beside the source report", 1);
|
|
426
475
|
}
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
476
|
+
if (!options.preflightOnly) {
|
|
477
|
+
if (productionFinalizationOnly) {
|
|
478
|
+
process.stdout.write(`${plan}${codexPlan}\n mode: provider-free finalization of the complete frozen ledger\n`);
|
|
479
|
+
} else if (options.yes) {
|
|
480
|
+
process.stdout.write(`${plan}${codexPlan}${nativeWarning}\n`);
|
|
481
|
+
} else if (!(await confirmPlan(`${plan}${codexPlan}${nativeWarning}\nThis may use paid model quota.`))) {
|
|
482
|
+
process.stdout.write("Benchmark cancelled; no model session started.\n");
|
|
483
|
+
return;
|
|
484
|
+
}
|
|
485
|
+
}
|
|
486
|
+
const runId = resumeState?.manifest.runId ?? createRunId(suite.id);
|
|
487
|
+
const runtimeCommands = productionFinalizationOnly
|
|
488
|
+
? frozenRuntimeCommandsForFinalization(resumeState.manifest, options.surfaces)
|
|
489
|
+
: {
|
|
490
|
+
pi: benchmarkCommandIdentity(piCommand, { cwd: bootstrapMetadata?.originalCwd ?? process.cwd() }),
|
|
491
|
+
codex: options.surfaces.includes("codex-cli")
|
|
492
|
+
? benchmarkCommandIdentity(codexCommand, { cwd: bootstrapMetadata?.originalCwd ?? process.cwd() })
|
|
493
|
+
: null,
|
|
494
|
+
node: benchmarkCommandIdentity(process.execPath),
|
|
495
|
+
git: benchmarkCommandIdentity("git"),
|
|
496
|
+
bash: benchmarkCommandIdentity("bash")
|
|
497
|
+
};
|
|
436
498
|
const environmentPolicy = benchmarkEnvironmentPolicy();
|
|
499
|
+
const configurationPiAgentHome = productionFinalizationOnly
|
|
500
|
+
? resumeState.manifest.piAgentHome?.identity
|
|
501
|
+
: bootstrapMetadata.piAgentHome.identity;
|
|
502
|
+
const configurationCodexCredential = productionFinalizationOnly
|
|
503
|
+
? resumeState.manifest.codexCredentialIdentity ?? null
|
|
504
|
+
: bootstrapMetadata.codexCredential?.identity ?? null;
|
|
437
505
|
const configuration = {
|
|
438
506
|
schemaVersion: 1,
|
|
439
507
|
source: bootstrapMetadata.sourceIdentity,
|
|
@@ -442,8 +510,9 @@ async function main() {
|
|
|
442
510
|
runtimeDependencyDigest: bootstrapMetadata.runtimeDependencies?.digest ?? null,
|
|
443
511
|
runtimeCommands,
|
|
444
512
|
environmentPolicy,
|
|
445
|
-
|
|
446
|
-
|
|
513
|
+
...(productionHostReadinessRequired ? { hostReadinessPolicyDigest: productionHostReadinessPolicyDigest } : {}),
|
|
514
|
+
piAgentHome: configurationPiAgentHome,
|
|
515
|
+
codexCredential: configurationCodexCredential,
|
|
447
516
|
rootSeedDigest,
|
|
448
517
|
surfaces: options.surfaces,
|
|
449
518
|
model: options.model ?? null,
|
|
@@ -475,48 +544,81 @@ async function main() {
|
|
|
475
544
|
completedKeys: resumeState.completedKeys
|
|
476
545
|
});
|
|
477
546
|
}
|
|
547
|
+
let hostReadinessReceipt = null;
|
|
548
|
+
const currentAuthorizedThroughRuns = resumeState?.manifest.stageControl?.authorizedThroughRuns
|
|
549
|
+
?? productionStageBoundaries[0];
|
|
550
|
+
const nextAuthorizedThroughRuns = deferredProductionStageApproval?.authorizedThroughRuns
|
|
551
|
+
?? currentAuthorizedThroughRuns;
|
|
552
|
+
if (productionHostReadinessRequired && !productionFinalizationOnly) {
|
|
553
|
+
hostReadinessReceipt = await collectReadyBenchmarkHostReadiness({
|
|
554
|
+
policy: productionHostReadinessPolicy,
|
|
555
|
+
completedRuns: resumeState?.completedRuns.length ?? 0,
|
|
556
|
+
authorizedThroughRuns: nextAuthorizedThroughRuns,
|
|
557
|
+
runId,
|
|
558
|
+
configurationDigest,
|
|
559
|
+
stageBoundaries: productionAllStageBoundaries,
|
|
560
|
+
existingReceipts: resumeState?.manifest.hostReadinessReceipts ?? []
|
|
561
|
+
});
|
|
562
|
+
}
|
|
563
|
+
const assertHostReadinessStartReady = () => assertBenchmarkHostReadinessStartReady(hostReadinessReceipt, {
|
|
564
|
+
policy: productionHostReadinessPolicy,
|
|
565
|
+
runId,
|
|
566
|
+
configurationDigest
|
|
567
|
+
});
|
|
478
568
|
const executionGuard = createBenchmarkExecutionGuard({
|
|
479
569
|
candidateGuard,
|
|
480
570
|
suiteRoot,
|
|
481
571
|
suiteIdentity: bootstrapMetadata?.suite?.identity ?? suiteIdentity,
|
|
482
|
-
piAgentHome: bootstrapMetadata?.piAgentHome,
|
|
483
|
-
codexCredential: bootstrapMetadata?.codexCredential,
|
|
572
|
+
piAgentHome: productionFinalizationOnly ? null : bootstrapMetadata?.piAgentHome,
|
|
573
|
+
codexCredential: productionFinalizationOnly ? null : bootstrapMetadata?.codexCredential,
|
|
484
574
|
runtimeDependencies: bootstrapMetadata?.runtimeDependencies,
|
|
485
|
-
commands: runtimeCommands
|
|
575
|
+
commands: runtimeCommands,
|
|
576
|
+
verifyCommandAssets: !productionFinalizationOnly
|
|
486
577
|
});
|
|
487
|
-
assertBenchmarkPiCredentialReady(bootstrapMetadata.piAgentHome.credentialReadiness, options.model);
|
|
488
|
-
assertBenchmarkPiCredentialWritebackPolicy(bootstrapMetadata.piAgentHome);
|
|
489
|
-
piRuntimeHome = createBenchmarkPiRuntimeHome(bootstrapMetadata.piAgentHome);
|
|
490
|
-
const preflightAssetError = executionGuard.check("before-preflight", [piRuntimeHome]);
|
|
491
|
-
if (preflightAssetError) throw preflightAssetError;
|
|
492
|
-
codexRuntime = createCodexRuntime(options);
|
|
493
578
|
let runtime;
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
579
|
+
let codexCredentialBridge;
|
|
580
|
+
if (productionFinalizationOnly) {
|
|
581
|
+
runtime = resumeState.manifest.preflightRuntime;
|
|
582
|
+
if (!runtime || typeof runtime.gitVersion !== "string" || typeof runtime.piVersion !== "string"
|
|
583
|
+
|| !Array.isArray(runtime.codexDisabledFeatures)
|
|
584
|
+
|| (options.surfaces.includes("codex-cli") && typeof runtime.codexVersion !== "string")) {
|
|
585
|
+
fail("Cannot finalize production benchmark provider-free: frozen preflight runtime evidence is missing or invalid", 1);
|
|
586
|
+
}
|
|
587
|
+
codexCredentialBridge = resumeState.manifest.codexCredentialBridge ?? null;
|
|
588
|
+
const finalizationAssetError = executionGuard.check("provider-free-finalization");
|
|
589
|
+
if (finalizationAssetError) throw finalizationAssetError;
|
|
590
|
+
} else {
|
|
591
|
+
assertBenchmarkPiCredentialReady(bootstrapMetadata.piAgentHome.credentialReadiness, options.model);
|
|
592
|
+
assertBenchmarkPiCredentialWritebackPolicy(bootstrapMetadata.piAgentHome);
|
|
593
|
+
piRuntimeHome = createBenchmarkPiRuntimeHome(bootstrapMetadata.piAgentHome);
|
|
594
|
+
const preflightAssetError = executionGuard.check("before-preflight", [piRuntimeHome]);
|
|
595
|
+
if (preflightAssetError) throw preflightAssetError;
|
|
596
|
+
codexRuntime = createCodexRuntime(options);
|
|
597
|
+
try { runtime = await withBenchmarkPiCredentialWriteback(bootstrapMetadata.piAgentHome, piRuntimeHome, () => benchmarkPreflight({ runCommand, packageRoot, piCommand, piEnvironment: benchmarkEnvironment({ PI_CODING_AGENT_DIR: piRuntimeHome.path }), codexCommand, gitCommand: runtimeCommands.git.resolvedPath, surfaces: options.surfaces, codexMode: options.codexMode, codexRuntime })); }
|
|
598
|
+
catch (error) { preservePiRuntime ||= error.code === "BENCHMARK_PI_CREDENTIAL_RECONCILIATION_FAILED"; throw error; }
|
|
599
|
+
const postPreflightAssetError = executionGuard.check("after-preflight", [piRuntimeHome]);
|
|
600
|
+
if (postPreflightAssetError) throw postPreflightAssetError;
|
|
601
|
+
resetBenchmarkPiRuntimeEphemeralState(piRuntimeHome);
|
|
602
|
+
codexCredentialBridge = codexRuntime.credentialBridge;
|
|
603
|
+
if (resumeState && resumeState.manifest.preflightRuntime !== undefined
|
|
604
|
+
&& JSON.stringify(resumeState.manifest.preflightRuntime) !== JSON.stringify(runtime)) {
|
|
605
|
+
fail("Cannot resume benchmark: provider-free preflight runtime evidence changed since the original run", 1);
|
|
606
|
+
}
|
|
607
|
+
if (resumeState && resumeState.manifest.codexCredentialBridge !== undefined
|
|
608
|
+
&& resumeState.manifest.codexCredentialBridge !== codexCredentialBridge) {
|
|
609
|
+
fail("Cannot resume benchmark: Codex credential bridge changed since the original run", 1);
|
|
610
|
+
}
|
|
611
|
+
}
|
|
612
|
+
assertHostReadinessStartReady();
|
|
499
613
|
const source = bootstrapMetadata?.sourceIdentity;
|
|
500
614
|
if (!source) fail("Modern benchmark is missing its frozen Git source identity", 1);
|
|
501
615
|
if (options.preflightOnly) {
|
|
502
|
-
const receipt = benchmarkPreflightReceipt({ packageVersion: packageManifest.version, source, candidateProvenance: candidateGuard.report(), suite, suiteDigest, runtimeDependencies: bootstrapMetadata.runtimeDependencies, runtimeCommands, environmentPolicy, configurationDigest, rootSeedDigest, options, runtime });
|
|
616
|
+
const receipt = benchmarkPreflightReceipt({ packageVersion: packageManifest.version, source, candidateProvenance: candidateGuard.report(), suite, suiteDigest, runtimeDependencies: bootstrapMetadata.runtimeDependencies, runtimeCommands, environmentPolicy, configurationDigest, rootSeedDigest, options, runtime, hostReadinessPolicyDigest: productionHostReadinessPolicyDigest, hostReadiness: hostReadinessReceipt });
|
|
503
617
|
process.stdout.write(options.json ? `${JSON.stringify(receipt, null, 2)}\n` : `${plan}${codexPlan}\nPREFLIGHT READY: no model session started.\n${JSON.stringify(receipt, null, 2)}\n`);
|
|
504
618
|
return;
|
|
505
619
|
}
|
|
506
|
-
const nativeWarning = options.surfaces.includes("codex-cli") && options.codexMode === "native"
|
|
507
|
-
? "\nNative Codex mode loads the operator's global AGENTS.md, configuration, rules, hooks, MCP servers, and plugins."
|
|
508
|
-
: "";
|
|
509
|
-
if (options.yes) {
|
|
510
|
-
process.stdout.write(`${plan}${codexPlan}${nativeWarning}\n`);
|
|
511
|
-
} else {
|
|
512
|
-
if (!(await confirmPlan(`${plan}${codexPlan}${nativeWarning}\nThis may use paid model quota.`))) {
|
|
513
|
-
process.stdout.write("Benchmark cancelled; no model session started.\n");
|
|
514
|
-
return;
|
|
515
|
-
}
|
|
516
|
-
}
|
|
517
620
|
candidateGuard.freeze();
|
|
518
|
-
const
|
|
519
|
-
const output = options.output ?? path.join(defaultOutputRoot(), runId);
|
|
621
|
+
const output = options.output ?? path.join(defaultOutputRoot(bootstrapMetadata), runId);
|
|
520
622
|
const runRoot = resumeState ? privateDirectory(output) : ensureEmptyOutput(output);
|
|
521
623
|
releaseRunLock ??= acquireBenchmarkRunLock(runRoot, runId);
|
|
522
624
|
privateDirectory(path.join(runRoot, "workspaces"));
|
|
@@ -535,13 +637,17 @@ async function main() {
|
|
|
535
637
|
suite: {
|
|
536
638
|
id: suite.id,
|
|
537
639
|
source: bootstrapMetadata?.suite?.origin ?? manifestPath,
|
|
538
|
-
manifestPath: bootstrapMetadata?.suite?.snapshot ? bootstrapMetadata.suite.origin : null
|
|
640
|
+
manifestPath: bootstrapMetadata?.suite?.snapshot ? bootstrapMetadata.suite.origin : null,
|
|
641
|
+
builtInId
|
|
539
642
|
},
|
|
540
643
|
suiteDigest,
|
|
541
644
|
suiteIdentity,
|
|
542
645
|
candidateProvenance: candidateGuard.provenance,
|
|
543
646
|
runtimeDependencies: bootstrapMetadata?.runtimeDependencies ?? null,
|
|
544
647
|
runtimeCommands,
|
|
648
|
+
preflightRuntime: runtime,
|
|
649
|
+
codexCredentialIdentity: bootstrapMetadata.codexCredential?.identity ?? null,
|
|
650
|
+
codexCredentialBridge,
|
|
545
651
|
configurationDigest,
|
|
546
652
|
environmentPolicy,
|
|
547
653
|
piAgentHome: {
|
|
@@ -569,13 +675,37 @@ async function main() {
|
|
|
569
675
|
retryDelaySeconds: options.retryDelaySeconds,
|
|
570
676
|
stopAfterFailedPair: options.stopAfterFailedPair,
|
|
571
677
|
scenarioIds: options.scenarioIds ?? null,
|
|
678
|
+
...(productionSpendControlled ? {
|
|
679
|
+
stageControl: createProductionStageControl({
|
|
680
|
+
authorizedThroughRuns: productionStageBoundaries[0],
|
|
681
|
+
generatedAt: startedAt
|
|
682
|
+
}),
|
|
683
|
+
...(productionHostReadinessRequired ? {
|
|
684
|
+
hostReadinessPolicyDigest: productionHostReadinessPolicyDigest,
|
|
685
|
+
hostReadinessReceipts: []
|
|
686
|
+
} : {})
|
|
687
|
+
} : {}),
|
|
572
688
|
order: fullOrder.map((item) => ({
|
|
573
689
|
scenarioId: item.scenario.id,
|
|
574
690
|
surface: item.surface,
|
|
575
691
|
repeat: item.repeat
|
|
576
692
|
}))
|
|
577
693
|
};
|
|
694
|
+
assertHostReadinessStartReady();
|
|
695
|
+
if (productionSpendControlled && deferredProductionStageApproval) {
|
|
696
|
+
manifest.stageControl = approveProductionStageControl(manifest.stageControl, deferredProductionStageApproval);
|
|
697
|
+
}
|
|
698
|
+
if (productionHostReadinessRequired && hostReadinessReceipt) {
|
|
699
|
+
manifest.hostReadinessPolicyDigest = productionHostReadinessPolicyDigest;
|
|
700
|
+
manifest.hostReadinessReceipts = [...(manifest.hostReadinessReceipts ?? []), hostReadinessReceipt];
|
|
701
|
+
}
|
|
578
702
|
writeBenchmarkRunManifest(runRoot, manifest);
|
|
703
|
+
if (resumeState) {
|
|
704
|
+
fs.rmSync(path.join(runRoot, "stage-diagnostic.json"), { force: true });
|
|
705
|
+
if (productionSpendControlled && manifest.stageControl?.state === "window-authorized") {
|
|
706
|
+
fs.rmSync(path.join(runRoot, "paused.json"), { force: true });
|
|
707
|
+
}
|
|
708
|
+
}
|
|
579
709
|
const fullIndexByKey = new Map(fullOrder.map((item, index) => [benchmarkRunKey(item), index + 1]));
|
|
580
710
|
const wallStartedAt = Date.now();
|
|
581
711
|
const runtimeDeadline = options.maxRuntimeMinutes === undefined
|
|
@@ -785,147 +915,18 @@ async function main() {
|
|
|
785
915
|
finalizationReceipt = executionGuard.receipt("finalization", [piRuntimeHome]);
|
|
786
916
|
fatalRunError = finalizationReceipt.error;
|
|
787
917
|
}
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
const provenanceStamp = finalizationReceipt?.stamp ?? fatalExecutionReceipt?.stamp ?? executionGuard.stamp("fatal", piRuntimeHome ? [piRuntimeHome] : []);
|
|
800
|
-
if (!preservePiRuntime) cleanupBenchmarkPiRuntimeHome(bootstrapMetadata.piAgentHome, piRuntimeHome);
|
|
801
|
-
piRuntimeHome = undefined;
|
|
802
|
-
recoverOrphanedBenchmarkAttempts({ runRoot, manifest, fullOrder, completedKeys: new Set(runs.map(benchmarkRunKey)) });
|
|
803
|
-
cleanupUnretainedWorkspaces(runRoot, options.keepWorkspaces);
|
|
804
|
-
const provenanceFailure = writeBenchmarkAbort(runRoot, { runId, completedRuns: runs.filter(completedBenchmarkRecord).length, expectedRuns: fullOrder.length }, fatalRunError, {
|
|
805
|
-
ledger: ledgerBinding,
|
|
806
|
-
provenanceStamp
|
|
807
|
-
});
|
|
808
|
-
process.stderr.write(provenanceFailure
|
|
809
|
-
? `Benchmark aborted because candidate provenance changed. Partial ledger: ${ledgerPath}\n`
|
|
810
|
-
: `Benchmark aborted after an infrastructure error. Partial ledger: ${ledgerPath}\n`);
|
|
811
|
-
process.exitCode = 1;
|
|
812
|
-
return;
|
|
813
|
-
}
|
|
814
|
-
if (terminalStop) {
|
|
815
|
-
cleanupBenchmarkPiRuntimeHome(bootstrapMetadata.piAgentHome, piRuntimeHome); piRuntimeHome = undefined;
|
|
816
|
-
cleanupUnretainedWorkspaces(runRoot, options.keepWorkspaces);
|
|
817
|
-
for (const marker of ["paused.json", "interrupted.json", "aborted.json"]) fs.rmSync(path.join(runRoot, marker), { force: true });
|
|
818
|
-
writePrivateAtomic(path.join(runRoot, "stopped.json"), `${JSON.stringify({ ...terminalStop, runId, completedRuns: runs.filter(completedBenchmarkRecord).length, expectedRuns: fullOrder.length, stoppedAt: new Date().toISOString(), resumeAllowed: false, ledger: ledgerBinding, provenanceStamp: finalizationReceipt.stamp }, null, 2)}\n`);
|
|
819
|
-
process.stderr.write(`Benchmark terminal-stopped after paired outcome-floor failure. Partial ledger: ${ledgerPath}\n`);
|
|
820
|
-
process.exitCode = 1;
|
|
821
|
-
return;
|
|
822
|
-
}
|
|
823
|
-
const completedRuns = runs.filter(completedBenchmarkRecord);
|
|
824
|
-
if (completedRuns.length < fullOrder.length) {
|
|
825
|
-
piRuntimeHome = undefined;
|
|
826
|
-
cleanupUnretainedWorkspaces(runRoot, options.keepWorkspaces);
|
|
827
|
-
const paused = {
|
|
828
|
-
schemaVersion: 1,
|
|
829
|
-
runId,
|
|
830
|
-
reason: pauseReason ?? "partial-run",
|
|
831
|
-
completedRuns: completedRuns.length,
|
|
832
|
-
expectedRuns: fullOrder.length,
|
|
833
|
-
remainingRuns: fullOrder.length - completedRuns.length,
|
|
834
|
-
pausedAt: new Date().toISOString(),
|
|
835
|
-
resumeCommand: `piagent-benchmark --resume ${runRoot} --yes`,
|
|
836
|
-
ledger: ledgerBinding,
|
|
837
|
-
provenanceStamp: executionGuard.stamp("paused")
|
|
838
|
-
};
|
|
839
|
-
writePrivateAtomic(path.join(runRoot, "paused.json"), `${JSON.stringify(paused, null, 2)}\n`);
|
|
840
|
-
process.stdout.write(`Benchmark paused after ${completedRuns.length}/${fullOrder.length} completed sessions (${paused.reason}).\nResume: ${paused.resumeCommand}\nPartial ledger: ${ledgerPath}\n`);
|
|
841
|
-
return;
|
|
842
|
-
}
|
|
843
|
-
cleanupBenchmarkPiRuntimeHome(bootstrapMetadata.piAgentHome, piRuntimeHome); piRuntimeHome = undefined;
|
|
844
|
-
const report = summarizeBenchmark({
|
|
845
|
-
suite,
|
|
846
|
-
runId,
|
|
847
|
-
startedAt,
|
|
848
|
-
completedAt: new Date().toISOString(),
|
|
849
|
-
repeats: options.repeats,
|
|
850
|
-
environment: {
|
|
851
|
-
platformVersion: packageManifest.version,
|
|
852
|
-
suiteDigest,
|
|
853
|
-
variantRootSeed: suite.scenarios.some((scenario) => scenario.variantGenerator) ? rootSeed : null,
|
|
854
|
-
variantRootSeedDigest: suite.scenarios.some((scenario) => scenario.variantGenerator) ? rootSeedDigest : null,
|
|
855
|
-
executionOrder: suite.schemaVersion === 2 ? "seeded-paired-block-randomized" : "paired-alternating",
|
|
856
|
-
replaySource: options.replaySource ?? null,
|
|
857
|
-
profile: suite.profile,
|
|
858
|
-
requestedModel: options.model ?? null,
|
|
859
|
-
requestedThinking: options.thinking ?? null,
|
|
860
|
-
piagentTreatment: piagentTreatment(options.piagentTreatment),
|
|
861
|
-
treatmentBaseline: lifecycles.length === 1 && lifecycles[0] === "steady-state"
|
|
862
|
-
? options.surfaces.includes("codex-cli")
|
|
863
|
-
? "piagent-initialized-and-onboarded; codex-clean-fixture"
|
|
864
|
-
: "initialized-and-onboarded"
|
|
865
|
-
: "scenario-defined-mixed-lifecycle",
|
|
866
|
-
timeoutSeconds: options.timeoutSeconds,
|
|
867
|
-
nodeVersion: process.version,
|
|
868
|
-
piVersion: runtime.piVersion,
|
|
869
|
-
codexVersion: runtime.codexVersion ?? null,
|
|
870
|
-
codexMode: options.surfaces.includes("codex-cli") ? options.codexMode : null,
|
|
871
|
-
codexAuth: runtime.codexAuth ?? null,
|
|
872
|
-
codexIsolation: options.surfaces.includes("codex-cli")
|
|
873
|
-
? options.codexMode === "controlled" ? "per-session-temporary-home" : "operator-home"
|
|
874
|
-
: null,
|
|
875
|
-
codexCredentialBridge: options.surfaces.includes("codex-cli") ? codexRuntime.credentialBridge : null,
|
|
876
|
-
codexGlobalInstructions: options.surfaces.includes("codex-cli")
|
|
877
|
-
? options.codexMode === "controlled" ? "excluded" : "operator-home"
|
|
878
|
-
: null,
|
|
879
|
-
piGlobalInstructions: "excluded",
|
|
880
|
-
piAgentHome: manifest.piAgentHome,
|
|
881
|
-
usageIntegrity: manifest.tokenClaimsUnavailableReason ?? "measured",
|
|
882
|
-
codexDisabledFeatures: runtime.codexDisabledFeatures,
|
|
883
|
-
surfaces: options.surfaces,
|
|
884
|
-
scenarioSelection: options.scenarioIds ?? null,
|
|
885
|
-
surfaceModels: options.surfaces.includes("codex-cli") ? {
|
|
886
|
-
piagent: options.model,
|
|
887
|
-
"codex-cli": codexModelName(options.model)
|
|
888
|
-
} : { "raw-pi": options.model ?? null, piagent: options.model ?? null },
|
|
889
|
-
modelParityEvidence: options.surfaces.includes("codex-cli") ? "command-line-pinned" : "session-reported",
|
|
890
|
-
gitVersion: runtime.gitVersion,
|
|
891
|
-
source,
|
|
892
|
-
candidateProvenance: candidateGuard.report(),
|
|
893
|
-
suiteIdentity,
|
|
894
|
-
runtimeCommands,
|
|
895
|
-
configurationDigest,
|
|
896
|
-
environmentPolicy,
|
|
897
|
-
runtimeDependencies: bootstrapMetadata?.runtimeDependencies ?? null,
|
|
898
|
-
assuranceEvidence
|
|
899
|
-
},
|
|
900
|
-
runs: completedRuns,
|
|
901
|
-
...comparison
|
|
918
|
+
finalizeBenchmarkRun({
|
|
919
|
+
assuranceEvidence, bootstrapMetadata, candidateGuard, canonicalProductionSuite,
|
|
920
|
+
codexCredentialBridge, comparison, configurationDigest, declaredScenarioCount,
|
|
921
|
+
detachPiRuntimeHome: () => { piRuntimeHome = undefined; },
|
|
922
|
+
environmentPolicy, executionGuard, fatalExecutionReceipt, fatalRunError,
|
|
923
|
+
finalizationReceipt, fullOrder, interruptedSignal, ledgerBinding, ledgerPath,
|
|
924
|
+
lifecycles, manifest, options, packageVersion: packageManifest.version,
|
|
925
|
+
pauseReason, piRuntimeHome, preservePiRuntime,
|
|
926
|
+
productionAllStageBoundaries, productionHostReadinessPolicy, productionSpendControlled, rootSeed, rootSeedDigest,
|
|
927
|
+
runId, runRoot, runs, runtime, runtimeCommands, source, startedAt, suite,
|
|
928
|
+
suiteDigest, suiteIdentity, terminalStop
|
|
902
929
|
});
|
|
903
|
-
report.ledger = ledgerBinding;
|
|
904
|
-
applyBenchmarkClaimRestrictions(report, { tokenReason: manifest.tokenClaimsUnavailableReason, replaySource: options.replaySource, codexMode: options.codexMode, surfaces: options.surfaces });
|
|
905
|
-
report.trustChecklist = benchmarkTrustChecklist(report);
|
|
906
|
-
const text = renderBenchmarkText(report);
|
|
907
|
-
const reportLedger = inspectBenchmarkLedger(ledgerPath);
|
|
908
|
-
assertBenchmarkLedgerBinding(ledgerBinding, reportLedger.binding, "benchmark report ledger");
|
|
909
|
-
validateBenchmarkLedgerPrefix(reportLedger.records, fullOrder, (record, index, expected) => expectedBenchmarkRecord(record, index, expected, runId, suite, configurationDigest));
|
|
910
|
-
writePrivate(path.join(runRoot, "report.html"), renderBenchmarkHtml(report));
|
|
911
|
-
writePrivate(path.join(runRoot, "summary.txt"), text);
|
|
912
|
-
cleanupUnretainedWorkspaces(runRoot, options.keepWorkspaces);
|
|
913
|
-
const prepublishReceipt = executionGuard.receipt("prepublish");
|
|
914
|
-
const prepublishError = prepublishReceipt.error;
|
|
915
|
-
if (prepublishError) { writeBenchmarkAbort(runRoot, { runId, completedRuns: completedRuns.length, expectedRuns: fullOrder.length }, prepublishError, { ledger: ledgerBinding, provenanceStamp: prepublishReceipt.stamp }); throw prepublishError; }
|
|
916
|
-
for (const marker of ["paused.json", "interrupted.json", "aborted.json", "stopped.json"]) fs.rmSync(path.join(runRoot, marker), { force: true });
|
|
917
|
-
writePrivateAtomic(path.join(runRoot, "report.json"), `${JSON.stringify(report, null, 2)}\n`);
|
|
918
|
-
process.stdout.write(options.json ? `${JSON.stringify(report, null, 2)}\n` : text);
|
|
919
|
-
process.stdout.write(`Reports: ${runRoot}\n`);
|
|
920
|
-
if (
|
|
921
|
-
report.comparison.qualityGate === false
|
|
922
|
-
|| report.comparison.safetyGate === false
|
|
923
|
-
|| report.comparison.reliabilityGate === false
|
|
924
|
-
|| report.comparison.qualityNonInferior === false
|
|
925
|
-
|| report.comparison.workflowGate === false
|
|
926
|
-
|| report.comparison.categoryGate === false
|
|
927
|
-
|| report.comparison.suiteGate?.passed === false
|
|
928
|
-
) process.exitCode = 1;
|
|
929
930
|
} finally {
|
|
930
931
|
releaseRunLock?.();
|
|
931
932
|
codexRuntime?.cleanup();
|
|
@@ -933,12 +934,7 @@ async function main() {
|
|
|
933
934
|
}
|
|
934
935
|
}
|
|
935
936
|
|
|
936
|
-
|
|
937
|
-
try { return fs.realpathSync(fileURLToPath(import.meta.url)) === fs.realpathSync(process.argv[1] || ""); }
|
|
938
|
-
catch { return import.meta.url === pathToFileURL(process.argv[1] || "").href; }
|
|
939
|
-
}
|
|
940
|
-
|
|
941
|
-
if (invokedAsEntrypoint()) {
|
|
937
|
+
if (invokedAsEntrypoint(import.meta.url, process.argv[1])) {
|
|
942
938
|
main().catch((error) => {
|
|
943
939
|
console.error(`FAIL: ${error.message}`);
|
|
944
940
|
process.exit(error.exitCode ?? 1);
|