@piagent/platform 1.5.5 → 1.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +166 -0
- package/README.md +25 -21
- package/README.vi.md +34 -5
- package/adapters/web-frontend/profile.json +2 -4
- package/architecture/layers.json +18 -3
- package/benchmarks/deep-logic-v1/SPECIALIST_PROTOCOL.vi.md +122 -0
- package/benchmarks/deep-logic-v1/grade.mjs +272 -10
- package/benchmarks/deep-logic-v1/project/apps/admin/src/billing-summary.js +3 -0
- package/benchmarks/deep-logic-v1/project/benchmark-contract/public-smoke.test.js +166 -0
- package/benchmarks/deep-logic-v1/project/package.json +1 -1
- package/benchmarks/deep-logic-v1/project/packages/billing/src/close-period.js +3 -0
- package/benchmarks/deep-logic-v1/project/packages/billing/src/plan-timeline.js +3 -0
- package/benchmarks/deep-logic-v1/project/test/smoke.test.js +7 -1
- package/benchmarks/deep-logic-v1/prompts/budgeted-context-graph.md +2 -0
- package/benchmarks/deep-logic-v1/prompts/fair-dependency-scheduler.md +2 -0
- package/benchmarks/deep-logic-v1/prompts/layered-policy-resolution.md +2 -0
- package/benchmarks/deep-logic-v1/prompts/resumable-stream-assembly.md +3 -1
- package/benchmarks/deep-logic-v1/prompts/revision-event-reconciliation.md +2 -0
- package/benchmarks/deep-logic-v1/prompts/temporal-usage-billing-close.md +123 -0
- package/benchmarks/deep-logic-v1/prompts/transactional-config-merge.md +2 -0
- package/benchmarks/deep-logic-v1/suite.json +51 -3
- package/benchmarks/deep-logic-v1/variant.mjs +22 -1
- package/benchmarks/production-v1/spend-control.v1.json +92 -0
- package/benchmarks/production-v1/suite.json +46 -3
- package/catalog/capabilities.json +18 -18
- package/docs/authority-manifest.md +1 -1
- package/docs/auto-delegation-policy.md +38 -73
- package/docs/command-reference-vietnamese.md +27 -35
- package/docs/context-engine.md +12 -4
- package/docs/context-window-policy.md +37 -1
- package/docs/distribution-standard.md +7 -7
- package/docs/en/architecture.md +27 -13
- package/docs/languages.json +4 -4
- package/docs/model-options.md +8 -5
- package/docs/operator-manual-vietnamese.md +27 -45
- package/docs/project-adapters.md +2 -2
- package/docs/publishing-for-teams.md +10 -0
- package/docs/quality-benchmark.md +427 -62
- package/docs/release-install-policy.md +10 -9
- package/docs/runtime-harness-standard.md +3 -3
- package/docs/runtime-policy-design.md +4 -4
- package/docs/runtime-quality-baseline.md +13 -6
- package/docs/security-threat-model.md +2 -2
- package/docs/subagent-orchestration-capabilities.md +19 -51
- package/docs/subagents-and-multiagent.md +21 -47
- package/docs/task-implementation-contract.md +3 -3
- package/docs/task-lifecycle-vietnamese.md +3 -3
- package/docs/team-onboarding.md +2 -2
- package/docs/usage-observability.md +7 -2
- package/docs/vi/architecture.md +27 -13
- package/docs-site/architecture.html +1 -1
- package/docs-site/benchmark.html +89 -95
- package/docs-site/commands.html +1 -1
- package/docs-site/content/benchmark.html +88 -94
- package/docs-site/content/en/benchmark.html +134 -90
- package/docs-site/content/en/index.html +19 -19
- package/docs-site/content/en/runtime.html +4 -4
- package/docs-site/content/en/whats-new.html +12 -1
- package/docs-site/content/en/workflows.html +2 -2
- package/docs-site/content/index.html +19 -19
- package/docs-site/content/runtime.html +4 -4
- package/docs-site/content/whats-new.html +12 -1
- package/docs-site/content/workflows.html +2 -2
- package/docs-site/context-engine.html +1 -1
- package/docs-site/en/architecture.html +1 -1
- package/docs-site/en/benchmark.html +135 -91
- package/docs-site/en/commands.html +1 -1
- package/docs-site/en/context-engine.html +1 -1
- package/docs-site/en/index.html +20 -20
- package/docs-site/en/mcp-approval.html +1 -1
- package/docs-site/en/mcp-auth.html +1 -1
- package/docs-site/en/mcp-servers.html +1 -1
- package/docs-site/en/mcp.html +1 -1
- package/docs-site/en/permissions.html +1 -1
- package/docs-site/en/profiles.html +1 -1
- package/docs-site/en/quickstart.html +1 -1
- package/docs-site/en/runtime.html +5 -5
- package/docs-site/en/scope.html +1 -1
- package/docs-site/en/security.html +1 -1
- package/docs-site/en/team.html +1 -1
- package/docs-site/en/whats-new.html +13 -2
- package/docs-site/en/workflows.html +3 -3
- package/docs-site/index.html +20 -20
- package/docs-site/mcp-approval.html +1 -1
- package/docs-site/mcp-auth.html +1 -1
- package/docs-site/mcp-servers.html +1 -1
- package/docs-site/mcp.html +1 -1
- package/docs-site/permissions.html +1 -1
- package/docs-site/profiles.html +1 -1
- package/docs-site/quickstart.html +1 -1
- package/docs-site/runtime.html +5 -5
- package/docs-site/scope.html +1 -1
- package/docs-site/security.html +1 -1
- package/docs-site/team.html +1 -1
- package/docs-site/whats-new.html +13 -2
- package/docs-site/workflows.html +3 -3
- package/evals/fixtures/benchmark-assurance-evidence.invalid.json +1 -1
- package/evals/fixtures/benchmark-assurance-evidence.valid.json +1 -1
- package/evals/fixtures/codex-relative-efficiency-v1.invalid.json +4 -0
- package/evals/fixtures/codex-relative-efficiency-v1.valid.json +94 -0
- package/evals/fixtures/helper-request.invalid.json +1 -1
- package/evals/fixtures/helper-request.valid.json +1 -1
- package/evals/fixtures/piagent-webui/session-live-state-v1.invalid.json +18 -0
- package/evals/fixtures/piagent-webui/session-live-state-v1.valid.json +27 -0
- package/evals/fixtures/piagent-webui/subagent-tree-v1.valid.json +7 -0
- package/evals/fixtures/task-authority-snapshot.valid.json +4 -4
- package/evals/fs4-readiness-matrix.v1.json +2 -2
- package/evals/fs5-causal-arm.v1.json +3 -3
- package/evals/fs5-pilot-protocol.v1.json +3 -3
- package/evals/fs5-pilot-protocol.v2.json +8 -8
- package/evals/fs5-pilot-protocol.v3.json +8 -8
- package/evals/fs5-pilot-protocol.v4.json +7 -6
- package/evals/fs5-pilot-protocol.v5.json +9 -7
- package/evals/private-holdout-v1/public-exposure.v1.json +1 -1
- package/evals/runtime-conformance-v1/README.md +43 -0
- package/evals/runtime-conformance-v1/evaluate-paid-canary.mjs +198 -0
- package/evals/runtime-conformance-v1/lane.json +57 -0
- package/evals/runtime-conformance-v1/paid-canary.json +31 -0
- package/evals/runtime-conformance-v1/runner.mjs +456 -0
- package/package.json +3 -1
- package/packages/piagent-core/README.md +13 -4
- package/packages/piagent-core/benchmark/benchmark-assurance.js +4 -1
- package/packages/piagent-core/benchmark/benchmark-bootstrap.js +91 -9
- package/packages/piagent-core/benchmark/benchmark-claim-restrictions.js +38 -7
- package/packages/piagent-core/benchmark/benchmark-cli.js +7 -7
- package/packages/piagent-core/benchmark/benchmark-codex-relative-efficiency.js +698 -0
- package/packages/piagent-core/benchmark/benchmark-codex.js +3 -1
- package/packages/piagent-core/benchmark/benchmark-comparison.js +271 -2
- package/packages/piagent-core/benchmark/benchmark-core.js +736 -664
- package/packages/piagent-core/benchmark/benchmark-execution-guard.js +7 -5
- package/packages/piagent-core/benchmark/benchmark-forensics.js +441 -4
- package/packages/piagent-core/benchmark/benchmark-host-readiness.js +639 -0
- package/packages/piagent-core/benchmark/benchmark-matrix.js +22 -4
- package/packages/piagent-core/benchmark/benchmark-normalized-cost.js +317 -0
- package/packages/piagent-core/benchmark/benchmark-preflight.js +5 -1
- package/packages/piagent-core/benchmark/benchmark-process.js +6 -5
- package/packages/piagent-core/benchmark/benchmark-provider-free-evidence.js +345 -0
- package/packages/piagent-core/benchmark/benchmark-provider-wire.js +126 -0
- package/packages/piagent-core/benchmark/benchmark-record-validation.js +252 -0
- package/packages/piagent-core/benchmark/benchmark-report.js +130 -12
- package/packages/piagent-core/benchmark/benchmark-resume-recovery.js +14 -0
- package/packages/piagent-core/benchmark/benchmark-runner-policy.js +26 -0
- package/packages/piagent-core/benchmark/benchmark-runtime-causal.js +244 -0
- package/packages/piagent-core/benchmark/benchmark-stage-diagnostic.js +979 -0
- package/packages/piagent-core/benchmark/benchmark-statistics.js +64 -0
- package/packages/piagent-core/benchmark/benchmark-stop-policy.js +8 -5
- package/packages/piagent-core/benchmark/benchmark-suite-identity.js +31 -0
- package/packages/piagent-core/benchmark/benchmark-suite-runtime.js +13 -9
- package/packages/piagent-core/benchmark/benchmark-suite.js +149 -2
- package/packages/piagent-core/benchmark/benchmark-summary-support.js +380 -0
- package/packages/piagent-core/benchmark/benchmark-timing-diagnostics.js +971 -0
- package/packages/piagent-core/benchmark/benchmark-usage.js +322 -0
- package/packages/piagent-core/benchmark/benchmark-workflow.js +166 -0
- package/packages/piagent-core/capabilities/runtime-integrity.js +3 -0
- package/packages/piagent-core/docs/context-efficiency-improvement-plan.md +1080 -0
- package/packages/piagent-core/extensions/acceptance-behavior-proof.js +806 -0
- package/packages/piagent-core/extensions/acceptance-contract-semantics.js +83 -69
- package/packages/piagent-core/extensions/acceptance-error-classes.js +160 -0
- package/packages/piagent-core/extensions/acceptance-executable-evidence.js +12 -9
- package/packages/piagent-core/extensions/acceptance-receipt.js +123 -59
- package/packages/piagent-core/extensions/acceptance-state-machine-evidence.js +518 -0
- package/packages/piagent-core/extensions/acceptance-test-binding-evidence.js +60 -0
- package/packages/piagent-core/extensions/context-efficiency-metrics.js +302 -0
- package/packages/piagent-core/extensions/context-efficiency-report.js +167 -0
- package/packages/piagent-core/extensions/context-engine.js +149 -156
- package/packages/piagent-core/extensions/context-evidence.js +48 -0
- package/packages/piagent-core/extensions/context-import-links.js +296 -0
- package/packages/piagent-core/extensions/context-index-policy.js +4 -0
- package/packages/piagent-core/extensions/context-query-paths.js +57 -0
- package/packages/piagent-core/extensions/context-telemetry-integrity.js +65 -0
- package/packages/piagent-core/extensions/criterion-context-pack.js +435 -7
- package/packages/piagent-core/extensions/criterion-graph.js +82 -9
- package/packages/piagent-core/extensions/fallback-base-policy.ts +101 -0
- package/packages/piagent-core/extensions/guard-types.ts +6 -0
- package/packages/piagent-core/extensions/javascript-regex-evidence.js +32 -0
- package/packages/piagent-core/extensions/piagent-guard.ts +420 -260
- package/packages/piagent-core/extensions/task-contract-view.js +17 -3
- package/packages/piagent-core/extensions/task-lifecycle.js +39 -2
- package/packages/piagent-core/extensions/task-state.js +109 -142
- package/packages/piagent-core/extensions/verification-intelligence.js +52 -13
- package/packages/piagent-core/extensions/workspace-evidence-roots.js +193 -0
- package/packages/piagent-core/package.json +1 -1
- package/packages/piagent-core/policies/base-policy.json +44 -2
- package/packages/piagent-core/policy/authority-manifest.v1.json +3 -3
- package/packages/piagent-core/prompts/be-to-fe.md +2 -8
- package/packages/piagent-core/prompts/plan.md +1 -1
- package/packages/piagent-core/prompts/platform-improve.md +2 -6
- package/packages/piagent-core/prompts/review.md +2 -7
- package/packages/piagent-core/prompts/scout.md +1 -1
- package/packages/piagent-core/prompts/task.md +4 -3
- package/packages/piagent-core/runtime/context/context-delivery.ts +103 -0
- package/packages/piagent-core/runtime/context/context-delta-shadow.ts +53 -0
- package/packages/piagent-core/runtime/context/context-efficiency-format.ts +47 -0
- package/packages/piagent-core/runtime/context/context-evidence-qualification.ts +32 -0
- package/packages/piagent-core/runtime/context/prefix-telemetry.ts +47 -0
- package/packages/piagent-core/runtime/context/repository-memory-hints.ts +27 -0
- package/packages/piagent-core/runtime/hooks/agent-prompt-telemetry.ts +56 -0
- package/packages/piagent-core/runtime/hooks/agent-start-hook.ts +162 -102
- package/packages/piagent-core/runtime/hooks/completion-hook.ts +12 -12
- package/packages/piagent-core/runtime/hooks/input-hook.ts +17 -1
- package/packages/piagent-core/runtime/hooks/session-hooks.ts +22 -2
- package/packages/piagent-core/runtime/hooks/session-start-hook.ts +107 -3
- package/packages/piagent-core/runtime/hooks/tool-result-content-guards.ts +95 -0
- package/packages/piagent-core/runtime/hooks/tool-result-hook.ts +134 -166
- package/packages/piagent-core/runtime/hooks/tool-result-value-helpers.ts +66 -0
- package/packages/piagent-core/runtime/inspection/activity-event-adapter.ts +4 -2
- package/packages/piagent-core/runtime/inspection/activity-recovery.ts +104 -0
- package/packages/piagent-core/runtime/inspection/bounded-emission-guard.ts +286 -0
- package/packages/piagent-core/runtime/inspection/runtime-event-store.ts +37 -7
- package/packages/piagent-core/runtime/inspection/task-compaction-history.ts +1 -1
- package/packages/piagent-core/runtime/inspection/task-handoff-history.ts +1 -1
- package/packages/piagent-core/runtime/inspection/task-subagent-tree.ts +27 -4
- package/packages/piagent-core/runtime/inspection/tool-failure-classification.ts +70 -0
- package/packages/piagent-core/runtime/inspection/webui-snapshot.ts +73 -9
- package/packages/piagent-core/runtime/model/authenticated-catalog.ts +17 -9
- package/packages/piagent-core/runtime/model/openai-codex-reasoning.ts +83 -0
- package/packages/piagent-core/runtime/model/provider-wire-fingerprint.ts +238 -0
- package/packages/piagent-core/runtime/orchestration/helper-lifecycle.ts +16 -12
- package/packages/piagent-core/runtime/orchestration/minimal-delegation-policy.ts +71 -0
- package/packages/piagent-core/runtime/orchestration/owned-work-budget.ts +28 -20
- package/packages/piagent-core/runtime/orchestration/role-policy.ts +27 -9
- package/packages/piagent-core/runtime/orchestration/subagent-tool-policy.ts +329 -0
- package/packages/piagent-core/runtime/policy/block-remedy.ts +0 -4
- package/packages/piagent-core/runtime/product/activity-inspector.ts +7 -2
- package/packages/piagent-core/runtime/product/efficiency-metrics.ts +7 -2
- package/packages/piagent-core/runtime/quality/edit-freshness-guard.ts +102 -0
- package/packages/piagent-core/runtime/quality/exact-output-contract.ts +22 -11
- package/packages/piagent-core/runtime/quality/performance-assurance-signals.ts +23 -0
- package/packages/piagent-core/runtime/quality/performance-assurance.ts +9 -5
- package/packages/piagent-core/runtime/recovery/continuation-budget.ts +1 -1
- package/packages/piagent-core/runtime/recovery/edit-recovery-context.ts +113 -0
- package/packages/piagent-core/runtime/recovery/edit-recovery-delivery.ts +42 -0
- package/packages/piagent-core/runtime/recovery/handoff-projection.ts +4 -1
- package/packages/piagent-core/runtime/recovery/pristine-task-policy-refresh.ts +91 -0
- package/packages/piagent-core/runtime/recovery/recovery-policy.ts +7 -4
- package/packages/piagent-core/runtime/recovery/resume-state.ts +112 -14
- package/packages/piagent-core/runtime/recovery/semantic-repair-handshake.ts +60 -13
- package/packages/piagent-core/runtime/recovery/semantic-repair-runtime.ts +55 -2
- package/packages/piagent-core/runtime/recovery/task-evidence-coverage-refresh.ts +217 -0
- package/packages/piagent-core/runtime/registration/context-commands.ts +82 -7
- package/packages/piagent-core/runtime/registration/knowledge-tools.ts +9 -5
- package/packages/piagent-core/runtime/registration/operator-request-intake.ts +49 -0
- package/packages/piagent-core/runtime/registration/policy-tools.ts +50 -15
- package/packages/piagent-core/runtime/registration/session-commands.ts +1 -1
- package/packages/piagent-core/runtime/registration/task-completion-tools.ts +61 -25
- package/packages/piagent-core/runtime/registration/task-evidence-tools.ts +21 -12
- package/packages/piagent-core/runtime/registration/task-start-guidance.ts +169 -0
- package/packages/piagent-core/runtime/registration/task-start-manifest.ts +54 -0
- package/packages/piagent-core/runtime/registration/task-start-retry-helpers.ts +37 -0
- package/packages/piagent-core/runtime/registration/task-start-tool.ts +108 -103
- package/packages/piagent-core/runtime/runtime-limits.ts +16 -2
- package/packages/piagent-core/runtime/session/adaptive-context-analysis.ts +461 -0
- package/packages/piagent-core/runtime/session/adaptive-context-governor.ts +464 -0
- package/packages/piagent-core/runtime/session/adaptive-context-ledger.ts +265 -0
- package/packages/piagent-core/runtime/session/message-signals.ts +7 -3
- package/packages/piagent-core/runtime/session/operator-request-carry.ts +44 -0
- package/packages/piagent-core/runtime/session/runtime-state.ts +161 -5
- package/packages/piagent-core/runtime/session/source-checkout-read-grants.ts +35 -0
- package/packages/piagent-core/runtime/session/system-prompt.ts +76 -14
- package/packages/piagent-core/runtime/session/task-contract-presentation.ts +39 -0
- package/packages/piagent-core/runtime/tools/apply-patch-tool.ts +500 -0
- package/packages/piagent-core/runtime/tools/phase-tools.ts +8 -8
- package/packages/piagent-core/runtime/tools/tool-groups.ts +12 -2
- package/packages/piagent-core/runtime/trajectory/trajectory-runtime.ts +2 -1
- package/packages/piagent-core/runtime/verification/exact-verifier-reuse.ts +101 -0
- package/packages/piagent-core/runtime/workflows/runtime-intake-compaction.ts +164 -0
- package/packages/piagent-core/runtime/workflows/task-intake.ts +207 -38
- package/packages/piagent-core/security/sensitive-data.js +13 -218
- package/packages/piagent-core/security/sensitive-project-file.js +42 -0
- package/packages/piagent-core/security/sensitive-source-expression.js +330 -0
- package/packages/piagent-core/security/sensitive-source-formats.js +394 -0
- package/packages/piagent-core/security/sensitive-text.js +234 -0
- package/packages/piagent-core/subagents/piagent-oracle.md +5 -3
- package/packages/piagent-core/subagents/piagent-planner.md +6 -4
- package/packages/piagent-core/subagents/piagent-reviewer.md +5 -3
- package/packages/piagent-core/subagents/piagent-scout.md +4 -2
- package/packages/piagent-core/subagents/piagent-worker.md +7 -5
- package/packages/piagent-webui/dist/client/assets/{MarkdownMessage-rD9WJKxs.js → MarkdownMessage-B4XehTCU.js} +1 -1
- package/packages/piagent-webui/dist/client/assets/index-CgOEyRPG.js +176 -0
- package/packages/piagent-webui/dist/client/index.html +1 -1
- package/packages/piagent-webui/extension/piagent-webui.ts +3 -1
- package/packages/piagent-webui/extension/same-session-bridge.ts +1 -1
- package/packages/piagent-webui/extension/session-stream-adapter.ts +33 -10
- package/packages/piagent-webui/gateway/gateway-events.ts +109 -5
- package/packages/piagent-webui/gateway/gateway-service.ts +3 -0
- package/packages/piagent-webui/gateway/gateway-session-stream.ts +122 -20
- package/packages/piagent-webui/gateway/session-command-controller.ts +18 -3
- package/packages/piagent-webui/gateway/session-effective-options.ts +47 -0
- package/packages/piagent-webui/gateway/session-live-state.ts +21 -0
- package/packages/piagent-webui/gateway/session-operation-lifecycle.ts +237 -0
- package/packages/piagent-webui/gateway/session-operation-retry-control.ts +38 -0
- package/packages/piagent-webui/gateway/session-operation-start.ts +5 -1
- package/packages/piagent-webui/gateway/session-operation-watchdog.ts +165 -0
- package/packages/piagent-webui/gateway/session-runtime-factory.ts +45 -0
- package/packages/piagent-webui/gateway/session-runtime-supervisor.ts +141 -125
- package/packages/piagent-webui/ownership/session-lease-store.ts +5 -2
- package/packages/piagent-webui/server/core-inspection-provider.ts +2 -1
- package/packages/piagent-webui/server/loopback-server.ts +8 -0
- package/packages/piagent-webui/server/transcript-projection.ts +113 -9
- package/packages/piagent-webui/shared/text-visibility.ts +10 -0
- package/schemas/codex-relative-efficiency-v1.schema.json +155 -0
- package/schemas/piagent-webui/catalog-v1.json +9 -0
- package/schemas/piagent-webui/gateway-protocol-v1.schema.json +53 -3
- package/schemas/piagent-webui/runtime-event-v2.schema.json +15 -5
- package/schemas/piagent-webui/session-live-state-v1.schema.json +76 -0
- package/schemas/piagent-webui/subagent-tree-v1.schema.json +11 -1
- package/schemas/project-profile.schema.json +1 -1
- package/schemas/task-contract.schema.json +27 -1
- package/scripts/benchmark-runner-core.mjs +367 -363
- package/scripts/benchmark-runner-finalization.mjs +313 -0
- package/scripts/benchmark-runner-host-readiness.mjs +94 -0
- package/scripts/benchmark-runner-support.mjs +200 -0
- package/scripts/benchmark-session.mjs +95 -41
- package/scripts/configure-model-scope.sh +4 -3
- package/scripts/configure-subagents.sh +42 -34
- package/scripts/helper-evaluation.mjs +2 -2
- package/scripts/install-global.sh +2 -2
- package/scripts/pi-usage-history.mjs +37 -10
- package/scripts/setup.sh +2 -2
- package/scripts/team-doctor.sh +1 -1
- package/scripts/verify-local.sh +10 -1
- package/templates/global/settings.json +2 -1
- package/packages/piagent-webui/dist/client/assets/index-BZRFw1u6.js +0 -174
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
export const BENCHMARK_SCOPE_BANDS = [
|
|
2
2
|
{ id: "core", suite: "core-v1", scenarios: 4, availability: "runnable", purpose: "Fast regression check for task quality, scope, safety, and exact usage." },
|
|
3
|
+
{ id: "deep-logic", suite: "deep-logic-v1", scenarios: 7, availability: "runnable", claimTier: "capability", purpose: "Luna-medium-locked specialist regression across seven generated families, including exact temporal usage billing; full-suite only, not a generalization claim." },
|
|
3
4
|
{ id: "production", suite: "production-v1", scenarios: 18, availability: "runnable", claimTier: "public-regression", purpose: "Broader public regression matrix across archetypes, lifecycle states, and verifier profiles; not a generalization or production-stability claim." },
|
|
4
5
|
{ id: "capability", suite: "capability-v1", scenarios: 6, availability: "runnable", claimTier: "capability", purpose: "Unsaturated multi-file and multi-component tasks for hill-climbing; not a release gate." },
|
|
5
6
|
{ id: "e2-framework", suite: "e2-framework-v1", scenarios: 4, availability: "runnable", claimTier: "capability", purpose: "Taxonomy-bound real Hono, SQLite, and workspace tasks with reference, mutation, alternative-valid, scope, and grader-sensitivity calibration; not a release gate." },
|
|
@@ -25,6 +26,8 @@ export function requiresLongHorizonEvidence({ recoveryChange = false, contextCha
|
|
|
25
26
|
export function benchmarkTrustChecklist(report = {}) {
|
|
26
27
|
const comparison = report.comparison ?? {};
|
|
27
28
|
const hasPairedUsage = Number(comparison.pairedUsageRuns ?? 0) > 0;
|
|
29
|
+
const successfulPairEfficiencyRequired = comparison.successfulPairEfficiencyRole !== "diagnostic";
|
|
30
|
+
const failureAwareEfficiencyRequired = comparison.failureAwareEfficiencyRole !== "diagnostic";
|
|
28
31
|
const qualityPassed = comparison.qualityGate === true;
|
|
29
32
|
const safetyPassed = comparison.safetyGate === true;
|
|
30
33
|
const reliabilityPassed = comparison.reliabilityGate === true;
|
|
@@ -35,7 +38,13 @@ export function benchmarkTrustChecklist(report = {}) {
|
|
|
35
38
|
const outcomeEvidencePassed = comparison.outcomeEvidenceGate !== false;
|
|
36
39
|
const pairedRegressionPassed = comparison.pairedRegressionGate !== false;
|
|
37
40
|
const failureAwareEfficiencyPassed = comparison.failureAwareEfficiencyGate !== false;
|
|
41
|
+
const primaryEfficiencyEvidencePassed = comparison.primaryEfficiencyEvidenceGate === true;
|
|
42
|
+
const primaryEfficiencyBandCoveragePassed = comparison.primaryEfficiencyBandCoverageGate !== false;
|
|
43
|
+
const primaryEfficiencyPassed = comparison.primaryEfficiencyGate === true;
|
|
38
44
|
const comparisonProtocolPassed = comparison.comparisonProtocolGate?.passed === true;
|
|
45
|
+
const fullSuitePassed = comparison.fullSuiteGate !== false;
|
|
46
|
+
const performancePassed = comparison.performanceGate !== false;
|
|
47
|
+
const stabilityPassed = comparison.stabilityGate !== false;
|
|
39
48
|
return {
|
|
40
49
|
hasPairedUsage,
|
|
41
50
|
hasQualityGate: typeof comparison.qualityGate === "boolean",
|
|
@@ -48,21 +57,30 @@ export function benchmarkTrustChecklist(report = {}) {
|
|
|
48
57
|
hasOutcomeEvidenceGate: typeof comparison.outcomeEvidenceGate === "boolean",
|
|
49
58
|
hasPairedRegressionGate: typeof comparison.pairedRegressionGate === "boolean",
|
|
50
59
|
hasFailureAwareEfficiencyGate: typeof comparison.failureAwareEfficiencyGate === "boolean",
|
|
60
|
+
hasPrimaryEfficiencyEvidenceGate: typeof comparison.primaryEfficiencyEvidenceGate === "boolean",
|
|
61
|
+
hasPrimaryEfficiencyBandCoverageGate: typeof comparison.primaryEfficiencyBandCoverageGate === "boolean",
|
|
62
|
+
hasPrimaryEfficiencyGate: typeof comparison.primaryEfficiencyGate === "boolean",
|
|
51
63
|
hasComparisonProtocolGate: typeof comparison.comparisonProtocolGate?.passed === "boolean",
|
|
52
64
|
achievedClaimTier: comparison.claimEligibility?.achievedTier ?? "unavailable",
|
|
53
65
|
generalizationClaimAllowed: comparison.claimEligibility?.generalizationClaimAllowed === true,
|
|
54
66
|
tokenSavingClaimAllowed: comparison.tokenClaimAllowed === true
|
|
55
|
-
&& hasPairedUsage
|
|
67
|
+
&& (!successfulPairEfficiencyRequired || hasPairedUsage)
|
|
56
68
|
&& qualityPassed
|
|
57
69
|
&& safetyPassed
|
|
58
70
|
&& reliabilityPassed
|
|
59
71
|
&& workflowPassed
|
|
60
72
|
&& qualityNonInferior
|
|
61
|
-
&& efficiencyEvidencePassed
|
|
62
|
-
&& efficiencyBandCoveragePassed
|
|
73
|
+
&& (!successfulPairEfficiencyRequired || efficiencyEvidencePassed)
|
|
74
|
+
&& (!successfulPairEfficiencyRequired || efficiencyBandCoveragePassed)
|
|
63
75
|
&& outcomeEvidencePassed
|
|
64
76
|
&& pairedRegressionPassed
|
|
65
|
-
&& failureAwareEfficiencyPassed
|
|
77
|
+
&& (!failureAwareEfficiencyRequired || failureAwareEfficiencyPassed)
|
|
78
|
+
&& primaryEfficiencyEvidencePassed
|
|
79
|
+
&& primaryEfficiencyBandCoveragePassed
|
|
80
|
+
&& primaryEfficiencyPassed
|
|
81
|
+
&& fullSuitePassed
|
|
82
|
+
&& performancePassed
|
|
83
|
+
&& stabilityPassed
|
|
66
84
|
&& comparisonProtocolPassed
|
|
67
85
|
};
|
|
68
86
|
}
|
|
@@ -0,0 +1,317 @@
|
|
|
1
|
+
import {
|
|
2
|
+
atMostWithinFloatingPrecision,
|
|
3
|
+
geometricMean,
|
|
4
|
+
geometricMeanConfidence95,
|
|
5
|
+
geometricMeanConfidence95Raw,
|
|
6
|
+
rounded
|
|
7
|
+
} from "./benchmark-statistics.js";
|
|
8
|
+
|
|
9
|
+
const SNAPSHOT_FIELDS = new Set([
|
|
10
|
+
"schemaVersion", "id", "model", "currency", "unitTokens", "rates",
|
|
11
|
+
"cacheWrite", "longContext", "source"
|
|
12
|
+
]);
|
|
13
|
+
const RATE_FIELDS = new Set(["freshInput", "cachedInput", "output"]);
|
|
14
|
+
const CACHE_WRITE_FIELDS = new Set(["basis", "multiplier"]);
|
|
15
|
+
const LONG_CONTEXT_FIELDS = new Set(["thresholdInputTokens", "condition", "inputMultiplier", "outputMultiplier"]);
|
|
16
|
+
const SOURCE_FIELDS = new Set(["url", "retrievedAt"]);
|
|
17
|
+
const USAGE_BUCKETS = Object.freeze(["input", "output", "cacheRead", "cacheWrite"]);
|
|
18
|
+
const BAND_DIMENSIONS = Object.freeze({
|
|
19
|
+
categories: "category",
|
|
20
|
+
profiles: "profile",
|
|
21
|
+
lifecycles: "lifecycle",
|
|
22
|
+
difficulties: "difficulty"
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
function plainObject(value) {
|
|
26
|
+
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function unsupportedFields(value, allowed, label, errors) {
|
|
30
|
+
if (!plainObject(value)) return;
|
|
31
|
+
for (const field of Object.keys(value)) {
|
|
32
|
+
if (!allowed.has(field)) errors.push(`${label} has unsupported field ${field}`);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function positiveFinite(value) {
|
|
37
|
+
return Number.isFinite(value) && value > 0;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/** Validate the immutable API-equivalent pricing input used by benchmark reports. */
|
|
41
|
+
export function benchmarkPricingSnapshotValidationErrors(snapshot) {
|
|
42
|
+
if (!plainObject(snapshot)) return ["pricingSnapshot must be an object"];
|
|
43
|
+
const errors = [];
|
|
44
|
+
unsupportedFields(snapshot, SNAPSHOT_FIELDS, "pricingSnapshot", errors);
|
|
45
|
+
if (snapshot.schemaVersion !== 1) errors.push("pricingSnapshot.schemaVersion must be 1");
|
|
46
|
+
if (typeof snapshot.id !== "string" || !/^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(snapshot.id)) {
|
|
47
|
+
errors.push("pricingSnapshot.id must use lowercase kebab-case");
|
|
48
|
+
}
|
|
49
|
+
if (typeof snapshot.model !== "string" || !/^[^/\s]+\/[^/\s]+$/.test(snapshot.model)) {
|
|
50
|
+
errors.push("pricingSnapshot.model must use provider/model form");
|
|
51
|
+
}
|
|
52
|
+
if (snapshot.currency !== "USD") errors.push("pricingSnapshot.currency must be USD");
|
|
53
|
+
if (snapshot.unitTokens !== 1_000_000) errors.push("pricingSnapshot.unitTokens must be 1000000");
|
|
54
|
+
|
|
55
|
+
if (!plainObject(snapshot.rates)) errors.push("pricingSnapshot.rates must be an object");
|
|
56
|
+
else {
|
|
57
|
+
unsupportedFields(snapshot.rates, RATE_FIELDS, "pricingSnapshot.rates", errors);
|
|
58
|
+
for (const field of RATE_FIELDS) {
|
|
59
|
+
if (!positiveFinite(snapshot.rates[field])) errors.push(`pricingSnapshot.rates.${field} must be greater than 0`);
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
if (!plainObject(snapshot.cacheWrite)) errors.push("pricingSnapshot.cacheWrite must be an object");
|
|
64
|
+
else {
|
|
65
|
+
unsupportedFields(snapshot.cacheWrite, CACHE_WRITE_FIELDS, "pricingSnapshot.cacheWrite", errors);
|
|
66
|
+
if (snapshot.cacheWrite.basis !== "fresh-input") errors.push("pricingSnapshot.cacheWrite.basis must be fresh-input");
|
|
67
|
+
if (!positiveFinite(snapshot.cacheWrite.multiplier)) errors.push("pricingSnapshot.cacheWrite.multiplier must be greater than 0");
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
if (!plainObject(snapshot.longContext)) errors.push("pricingSnapshot.longContext must be an object");
|
|
71
|
+
else {
|
|
72
|
+
unsupportedFields(snapshot.longContext, LONG_CONTEXT_FIELDS, "pricingSnapshot.longContext", errors);
|
|
73
|
+
if (!Number.isSafeInteger(snapshot.longContext.thresholdInputTokens) || snapshot.longContext.thresholdInputTokens <= 0) {
|
|
74
|
+
errors.push("pricingSnapshot.longContext.thresholdInputTokens must be a positive safe integer");
|
|
75
|
+
}
|
|
76
|
+
if (snapshot.longContext.condition !== "per-request-input-greater-than") {
|
|
77
|
+
errors.push("pricingSnapshot.longContext.condition must be per-request-input-greater-than");
|
|
78
|
+
}
|
|
79
|
+
for (const field of ["inputMultiplier", "outputMultiplier"]) {
|
|
80
|
+
if (!positiveFinite(snapshot.longContext[field])) errors.push(`pricingSnapshot.longContext.${field} must be greater than 0`);
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
if (!plainObject(snapshot.source)) errors.push("pricingSnapshot.source must be an object");
|
|
85
|
+
else {
|
|
86
|
+
unsupportedFields(snapshot.source, SOURCE_FIELDS, "pricingSnapshot.source", errors);
|
|
87
|
+
if (typeof snapshot.source.url !== "string" || !snapshot.source.url.startsWith("https://developers.openai.com/")) {
|
|
88
|
+
errors.push("pricingSnapshot.source.url must be an official developers.openai.com HTTPS URL");
|
|
89
|
+
}
|
|
90
|
+
if (typeof snapshot.source.retrievedAt !== "string" || !/^\d{4}-\d{2}-\d{2}$/.test(snapshot.source.retrievedAt)) {
|
|
91
|
+
errors.push("pricingSnapshot.source.retrievedAt must use YYYY-MM-DD");
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
return errors;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
function exactUsageIssue(usage, snapshot) {
|
|
98
|
+
if (!plainObject(usage) || usage.usageCompleteness !== "exact") return "usage-not-exact";
|
|
99
|
+
if (!Number.isSafeInteger(usage.sessions) || usage.sessions <= 0) return "usage-session-count-not-exact";
|
|
100
|
+
for (const field of USAGE_BUCKETS) {
|
|
101
|
+
if (!Number.isSafeInteger(usage[field]) || usage[field] < 0) return `usage-${field}-not-exact`;
|
|
102
|
+
}
|
|
103
|
+
if (!Number.isSafeInteger(usage.fresh) || usage.fresh !== usage.input + usage.output) return "usage-fresh-invariant-failed";
|
|
104
|
+
if (!Number.isSafeInteger(usage.total)
|
|
105
|
+
|| usage.total !== usage.input + usage.cacheRead + usage.cacheWrite + usage.output) {
|
|
106
|
+
return "usage-total-invariant-failed";
|
|
107
|
+
}
|
|
108
|
+
if (usage.model !== snapshot.model) return "pricing-model-mismatch";
|
|
109
|
+
return null;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* Convert exact token buckets to an API-equivalent amount without treating it as
|
|
114
|
+
* OAuth/provider-billed cost. Aggregate prompt usage at or below the threshold
|
|
115
|
+
* is a valid upper bound for every constituent request. Above it, aggregate-only
|
|
116
|
+
* evidence cannot identify which per-request multiplier applies, so we fail closed.
|
|
117
|
+
*/
|
|
118
|
+
export function normalizeBenchmarkUsageCost(usage, snapshot) {
|
|
119
|
+
const snapshotErrors = benchmarkPricingSnapshotValidationErrors(snapshot);
|
|
120
|
+
if (snapshotErrors.length > 0) {
|
|
121
|
+
return { status: "unavailable", reason: "pricing-snapshot-invalid", errors: snapshotErrors };
|
|
122
|
+
}
|
|
123
|
+
const usageIssue = exactUsageIssue(usage, snapshot);
|
|
124
|
+
if (usageIssue) return { status: "unavailable", reason: usageIssue, errors: [] };
|
|
125
|
+
const aggregatePromptTokens = usage.input + usage.cacheRead + usage.cacheWrite;
|
|
126
|
+
if (aggregatePromptTokens > snapshot.longContext.thresholdInputTokens) {
|
|
127
|
+
return {
|
|
128
|
+
status: "unavailable",
|
|
129
|
+
reason: "long-context-per-request-usage-unavailable",
|
|
130
|
+
errors: [],
|
|
131
|
+
aggregatePromptTokens,
|
|
132
|
+
thresholdInputTokens: snapshot.longContext.thresholdInputTokens,
|
|
133
|
+
pricingApplicability: "unknown"
|
|
134
|
+
};
|
|
135
|
+
}
|
|
136
|
+
const cacheWriteRate = snapshot.rates.freshInput * snapshot.cacheWrite.multiplier;
|
|
137
|
+
const amount = (
|
|
138
|
+
(usage.input * snapshot.rates.freshInput)
|
|
139
|
+
+ (usage.cacheRead * snapshot.rates.cachedInput)
|
|
140
|
+
+ (usage.cacheWrite * cacheWriteRate)
|
|
141
|
+
+ (usage.output * snapshot.rates.output)
|
|
142
|
+
) / snapshot.unitTokens;
|
|
143
|
+
return {
|
|
144
|
+
status: "measured",
|
|
145
|
+
source: "versioned-api-equivalent-text-token-pricing",
|
|
146
|
+
pricingSnapshotId: snapshot.id,
|
|
147
|
+
pricingApplicability: "standard-context-proven-by-aggregate-upper-bound",
|
|
148
|
+
currency: snapshot.currency,
|
|
149
|
+
amount,
|
|
150
|
+
amountUsd: rounded(amount, 9),
|
|
151
|
+
aggregatePromptTokens,
|
|
152
|
+
thresholdInputTokens: snapshot.longContext.thresholdInputTokens,
|
|
153
|
+
buckets: Object.fromEntries(USAGE_BUCKETS.map((field) => [field, usage[field]]))
|
|
154
|
+
};
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
function pairRecord(pair, snapshot) {
|
|
158
|
+
const baseline = normalizeBenchmarkUsageCost(pair.baseline?.usage, snapshot);
|
|
159
|
+
const candidate = normalizeBenchmarkUsageCost(pair.candidate?.usage, snapshot);
|
|
160
|
+
const issues = [];
|
|
161
|
+
if (baseline.status !== "measured") issues.push(`baseline:${baseline.reason}`);
|
|
162
|
+
if (candidate.status !== "measured") issues.push(`candidate:${candidate.reason}`);
|
|
163
|
+
if (pair.baseline?.usage?.thinkingLevel !== pair.candidate?.usage?.thinkingLevel) issues.push("thinking-mismatch");
|
|
164
|
+
const comparable = issues.length === 0 && baseline.amount > 0 && candidate.amount > 0;
|
|
165
|
+
if (issues.length === 0 && !comparable) issues.push("non-positive-normalized-cost");
|
|
166
|
+
return {
|
|
167
|
+
scenarioId: pair.candidate?.scenarioId ?? pair.baseline?.scenarioId ?? null,
|
|
168
|
+
repeat: pair.candidate?.repeat ?? pair.baseline?.repeat ?? null,
|
|
169
|
+
resolved: pair.baseline?.resolved === true && pair.candidate?.resolved === true,
|
|
170
|
+
comparable,
|
|
171
|
+
issues,
|
|
172
|
+
baseline,
|
|
173
|
+
candidate,
|
|
174
|
+
ratio: comparable ? candidate.amount / baseline.amount : null
|
|
175
|
+
};
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
function bandReport(suite, familyRatios) {
|
|
179
|
+
return Object.fromEntries(Object.entries(BAND_DIMENSIONS).map(([dimension, field]) => {
|
|
180
|
+
const bands = new Map();
|
|
181
|
+
for (const scenario of suite.scenarios) {
|
|
182
|
+
const name = scenario[field] ?? "unspecified";
|
|
183
|
+
const item = bands.get(name) ?? { expectedScenarioFamilies: 0, ratios: [], missingScenarioIds: [] };
|
|
184
|
+
item.expectedScenarioFamilies += 1;
|
|
185
|
+
const family = familyRatios.find((candidate) => candidate.scenarioId === scenario.id);
|
|
186
|
+
if (Number.isFinite(family?.ratio)) item.ratios.push(family.ratio);
|
|
187
|
+
else item.missingScenarioIds.push(scenario.id);
|
|
188
|
+
bands.set(name, item);
|
|
189
|
+
}
|
|
190
|
+
return [dimension, Object.fromEntries([...bands.entries()].sort(([left], [right]) => left.localeCompare(right)).map(([name, item]) => [name, {
|
|
191
|
+
expectedScenarioFamilies: item.expectedScenarioFamilies,
|
|
192
|
+
comparableScenarioFamilies: item.ratios.length,
|
|
193
|
+
complete: item.missingScenarioIds.length === 0,
|
|
194
|
+
ratio: item.missingScenarioIds.length === 0 ? geometricMean(item.ratios) : null,
|
|
195
|
+
missingScenarioIds: item.missingScenarioIds
|
|
196
|
+
}]))];
|
|
197
|
+
}));
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
/** Build the independent normalized-cost evidence and family-clustered estimand. */
|
|
201
|
+
export function normalizedCostComparison({ suite, allPairs, repeats }) {
|
|
202
|
+
const snapshot = suite.pricingSnapshot;
|
|
203
|
+
const snapshotErrors = benchmarkPricingSnapshotValidationErrors(snapshot);
|
|
204
|
+
const records = allPairs.map((pair) => pairRecord(pair, snapshot));
|
|
205
|
+
const resolvedComparable = records.filter((record) => record.resolved && record.comparable);
|
|
206
|
+
const recordsByScenario = new Map();
|
|
207
|
+
for (const record of resolvedComparable) {
|
|
208
|
+
const family = recordsByScenario.get(record.scenarioId) ?? [];
|
|
209
|
+
family.push(record);
|
|
210
|
+
recordsByScenario.set(record.scenarioId, family);
|
|
211
|
+
}
|
|
212
|
+
const familyRatios = suite.scenarios.map((scenario) => {
|
|
213
|
+
const records = recordsByScenario.get(scenario.id) ?? [];
|
|
214
|
+
const observedRepeats = new Set(records.map((record) => record.repeat));
|
|
215
|
+
const complete = Number.isInteger(repeats) && repeats > 0
|
|
216
|
+
&& records.length === repeats && observedRepeats.size === repeats;
|
|
217
|
+
return {
|
|
218
|
+
scenarioId: scenario.id,
|
|
219
|
+
pairs: records.length,
|
|
220
|
+
complete,
|
|
221
|
+
ratio: complete ? geometricMean(records.map((record) => record.ratio)) : null,
|
|
222
|
+
issues: complete ? [] : ["missing-comparable-normalized-cost"]
|
|
223
|
+
};
|
|
224
|
+
});
|
|
225
|
+
const completeFamilies = familyRatios.filter((family) => family.complete);
|
|
226
|
+
const ratio = completeFamilies.length === suite.scenarios.length
|
|
227
|
+
? geometricMean(completeFamilies.map((family) => family.ratio))
|
|
228
|
+
: null;
|
|
229
|
+
const confidence95 = completeFamilies.length === suite.scenarios.length
|
|
230
|
+
? geometricMeanConfidence95(completeFamilies.map((family) => family.ratio))
|
|
231
|
+
: null;
|
|
232
|
+
const confidence95Raw = completeFamilies.length === suite.scenarios.length
|
|
233
|
+
? geometricMeanConfidence95Raw(completeFamilies.map((family) => family.ratio))
|
|
234
|
+
: null;
|
|
235
|
+
const applicabilityFailures = records
|
|
236
|
+
.filter((record) => !record.comparable)
|
|
237
|
+
.map((record) => ({ scenarioId: record.scenarioId, repeat: record.repeat, issues: record.issues }));
|
|
238
|
+
const expectedPairs = suite.scenarios.length * repeats;
|
|
239
|
+
const exactAndApplicable = snapshotErrors.length === 0
|
|
240
|
+
&& records.length === expectedPairs
|
|
241
|
+
&& records.length > 0
|
|
242
|
+
&& applicabilityFailures.length === 0;
|
|
243
|
+
const complete = exactAndApplicable && completeFamilies.length === suite.scenarios.length;
|
|
244
|
+
const amounts = records.filter((record) => record.comparable);
|
|
245
|
+
return {
|
|
246
|
+
schemaVersion: 1,
|
|
247
|
+
source: "versioned-api-equivalent-text-token-pricing",
|
|
248
|
+
billedCost: false,
|
|
249
|
+
scope: "model-text-token-input-cache-output-only",
|
|
250
|
+
pricingSnapshot: snapshotErrors.length === 0 ? {
|
|
251
|
+
id: snapshot.id,
|
|
252
|
+
model: snapshot.model,
|
|
253
|
+
currency: snapshot.currency,
|
|
254
|
+
source: snapshot.source
|
|
255
|
+
} : null,
|
|
256
|
+
snapshotErrors,
|
|
257
|
+
expectedPairs,
|
|
258
|
+
observedPairs: allPairs.length,
|
|
259
|
+
comparablePairs: records.filter((record) => record.comparable).length,
|
|
260
|
+
resolvedComparablePairs: resolvedComparable.length,
|
|
261
|
+
exactUsageAndPricingApplicability: exactAndApplicable,
|
|
262
|
+
applicabilityFailures,
|
|
263
|
+
complete,
|
|
264
|
+
expectedScenarioFamilies: suite.scenarios.length,
|
|
265
|
+
completeScenarioFamilies: completeFamilies.length,
|
|
266
|
+
ratio: complete ? ratio : null,
|
|
267
|
+
ratioConfidence95: complete ? confidence95 : null,
|
|
268
|
+
ratioConfidence95Raw: complete ? confidence95Raw : null,
|
|
269
|
+
deltaPercent: complete ? (ratio - 1) * 100 : null,
|
|
270
|
+
familyRatios,
|
|
271
|
+
bands: bandReport(suite, familyRatios),
|
|
272
|
+
totals: {
|
|
273
|
+
baselineUsd: rounded(amounts.reduce((sum, record) => sum + record.baseline.amount, 0), 9),
|
|
274
|
+
candidateUsd: rounded(amounts.reduce((sum, record) => sum + record.candidate.amount, 0), 9)
|
|
275
|
+
}
|
|
276
|
+
};
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
export function normalizedCostGate(comparison, thresholds) {
|
|
280
|
+
const evidenceGate = comparison.complete === true;
|
|
281
|
+
const applicabilityGate = comparison.exactUsageAndPricingApplicability === true;
|
|
282
|
+
const confidenceGate = evidenceGate
|
|
283
|
+
&& Number.isFinite(comparison.ratioConfidence95Raw?.upper)
|
|
284
|
+
&& atMostWithinFloatingPrecision(comparison.ratioConfidence95Raw.upper, thresholds.maximumRatioUpper95);
|
|
285
|
+
const bandFailures = [];
|
|
286
|
+
for (const [dimension, bands] of Object.entries(comparison.bands ?? {})) {
|
|
287
|
+
for (const [name, band] of Object.entries(bands)) {
|
|
288
|
+
if (!Number.isFinite(band.ratio) || !atMostWithinFloatingPrecision(band.ratio, thresholds.maximumBandRatio)) {
|
|
289
|
+
bandFailures.push({
|
|
290
|
+
dimension,
|
|
291
|
+
name,
|
|
292
|
+
ratio: Number.isFinite(band.ratio) ? band.ratio : null,
|
|
293
|
+
reason: Number.isFinite(band.ratio) ? "ratio-above-limit" : "missing-comparable-normalized-cost",
|
|
294
|
+
missingScenarioIds: band.missingScenarioIds ?? []
|
|
295
|
+
});
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
const familyFailures = (comparison.familyRatios ?? [])
|
|
300
|
+
.filter((family) => !Number.isFinite(family.ratio)
|
|
301
|
+
|| !atMostWithinFloatingPrecision(family.ratio, thresholds.maximumFamilyRatio))
|
|
302
|
+
.map((family) => ({
|
|
303
|
+
scenarioId: family.scenarioId,
|
|
304
|
+
ratio: Number.isFinite(family.ratio) ? family.ratio : null,
|
|
305
|
+
reason: Number.isFinite(family.ratio) ? "ratio-above-limit" : "missing-comparable-normalized-cost"
|
|
306
|
+
}));
|
|
307
|
+
return {
|
|
308
|
+
evidenceGate,
|
|
309
|
+
applicabilityGate,
|
|
310
|
+
confidenceGate,
|
|
311
|
+
bandGate: bandFailures.length === 0,
|
|
312
|
+
familyGate: familyFailures.length === 0,
|
|
313
|
+
bandFailures,
|
|
314
|
+
familyFailures,
|
|
315
|
+
passed: evidenceGate && applicabilityGate && confidenceGate && bandFailures.length === 0 && familyFailures.length === 0
|
|
316
|
+
};
|
|
317
|
+
}
|
|
@@ -59,7 +59,8 @@ function publicCommandIdentity(value) {
|
|
|
59
59
|
export function benchmarkPreflightReceipt({
|
|
60
60
|
packageVersion, source, candidateProvenance, suite, suiteDigest,
|
|
61
61
|
runtimeDependencies, runtimeCommands, environmentPolicy, configurationDigest,
|
|
62
|
-
rootSeedDigest, options, runtime
|
|
62
|
+
rootSeedDigest, options, runtime, hostReadinessPolicyDigest = null, hostReadiness = null,
|
|
63
|
+
providerFreeEvidence = null
|
|
63
64
|
}) {
|
|
64
65
|
return {
|
|
65
66
|
schemaVersion: 1,
|
|
@@ -72,6 +73,7 @@ export function benchmarkPreflightReceipt({
|
|
|
72
73
|
suite: { id: suite.id, contentDigest: suiteDigest, scenarioCount: suite.scenarios.length },
|
|
73
74
|
configuration: {
|
|
74
75
|
contentDigest: configurationDigest,
|
|
76
|
+
...(hostReadinessPolicyDigest ? { hostReadinessPolicyDigest } : {}),
|
|
75
77
|
runtimeDependencyDigest: runtimeDependencies?.digest ?? null,
|
|
76
78
|
environmentPolicyDigest: environmentPolicy.digest,
|
|
77
79
|
rootSeedDigest,
|
|
@@ -87,6 +89,8 @@ export function benchmarkPreflightReceipt({
|
|
|
87
89
|
maxSessions: options.maxSessions ?? null,
|
|
88
90
|
stopAfterFailedPair: options.stopAfterFailedPair
|
|
89
91
|
},
|
|
92
|
+
...(hostReadiness ? { hostReadiness } : {}),
|
|
93
|
+
...(providerFreeEvidence ? { providerFreeEvidence } : {}),
|
|
90
94
|
runtime: {
|
|
91
95
|
gitVersion: runtime.gitVersion,
|
|
92
96
|
piVersion: runtime.piVersion,
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import crypto from "node:crypto";
|
|
2
2
|
import { spawn } from "node:child_process";
|
|
3
|
+
import { performance } from "node:perf_hooks";
|
|
3
4
|
|
|
4
5
|
const outputLimit = 4 * 1024 * 1024;
|
|
5
6
|
|
|
@@ -43,7 +44,7 @@ export function createBenchmarkProcessController(interrupted) {
|
|
|
43
44
|
reject(new Error("Benchmark was interrupted before the child process started"));
|
|
44
45
|
return;
|
|
45
46
|
}
|
|
46
|
-
const started =
|
|
47
|
+
const started = performance.now();
|
|
47
48
|
const child = spawn(command, args, {
|
|
48
49
|
cwd: options.cwd,
|
|
49
50
|
env: options.env ?? process.env,
|
|
@@ -80,7 +81,7 @@ export function createBenchmarkProcessController(interrupted) {
|
|
|
80
81
|
if (!options.inherit) {
|
|
81
82
|
child.stdout.on("data", (chunk) => {
|
|
82
83
|
stdoutDigest.update(chunk);
|
|
83
|
-
options.onStdoutChunk?.(chunk);
|
|
84
|
+
options.onStdoutChunk?.(chunk, { observedAtSeconds: (performance.now() - started) / 1000 });
|
|
84
85
|
inspect("stdout", chunk);
|
|
85
86
|
stdout = append(stdout, chunk);
|
|
86
87
|
});
|
|
@@ -116,7 +117,7 @@ export function createBenchmarkProcessController(interrupted) {
|
|
|
116
117
|
if (settled) return;
|
|
117
118
|
settled = true;
|
|
118
119
|
cleanup();
|
|
119
|
-
resolve({ code: code ?? 1, signal, timedOut, stdout, stdoutHash: options.inherit ? undefined : stdoutDigest.digest("hex"), stderr, forbiddenHits: [...forbiddenHits], requiredHits: [...requiredHits], durationSeconds: (
|
|
120
|
+
resolve({ code: code ?? 1, signal, timedOut, stdout, stdoutHash: options.inherit ? undefined : stdoutDigest.digest("hex"), stderr, forbiddenHits: [...forbiddenHits], requiredHits: [...requiredHits], durationSeconds: (performance.now() - started) / 1000 });
|
|
120
121
|
};
|
|
121
122
|
if (!groupAlive(child)) {
|
|
122
123
|
finish();
|
|
@@ -125,10 +126,10 @@ export function createBenchmarkProcessController(interrupted) {
|
|
|
125
126
|
// A detached descendant can close its inherited stdio before exiting. Keep
|
|
126
127
|
// the process group owned until TERM has had time to work and the bounded
|
|
127
128
|
// KILL escalation has run, rather than treating stream close as cleanup.
|
|
128
|
-
const deadline =
|
|
129
|
+
const deadline = performance.now() + 1_000;
|
|
129
130
|
const drain = () => {
|
|
130
131
|
if (!groupAlive(child)) finish();
|
|
131
|
-
else if (
|
|
132
|
+
else if (performance.now() >= deadline) {
|
|
132
133
|
settled = true;
|
|
133
134
|
cleanup();
|
|
134
135
|
reject(new Error(`Benchmark child process group ${child.pid} survived bounded SIGKILL cleanup`));
|