@piagent/platform 1.6.1 → 1.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +55 -0
- package/README.md +8 -2
- package/README.vi.md +11 -2
- package/adapters/common/assurance-producers.json +109 -0
- package/adapters/common/structured-assurance-facts.mjs +153 -0
- package/adapters/node-typescript/contract-families.json +2811 -0
- package/adapters/node-typescript/contract-families.md +90 -0
- package/benchmarks/production-v1/grade.mjs +20 -0
- package/benchmarks/production-v1/spend-control.v1.json +3 -3
- package/benchmarks/production-v2/grade.mjs +892 -0
- package/benchmarks/production-v2/project/README.md +7 -0
- package/benchmarks/production-v2/project/config/service.json +4 -0
- package/benchmarks/production-v2/project/data/audit-ledger.jsonl +1 -0
- package/benchmarks/production-v2/project/docs/ops.md +3 -0
- package/benchmarks/production-v2/project/logs/incident.log +2 -0
- package/benchmarks/production-v2/project/package.json +11 -0
- package/benchmarks/production-v2/project/scripts/check.mjs +21 -0
- package/benchmarks/production-v2/project/src/backend/auth.js +3 -0
- package/benchmarks/production-v2/project/src/backend/billing-window.js +4 -0
- package/benchmarks/production-v2/project/src/backend/cache.js +15 -0
- package/benchmarks/production-v2/project/src/backend/invoice.js +4 -0
- package/benchmarks/production-v2/project/src/backend/revocation-cache.js +3 -0
- package/benchmarks/production-v2/project/src/data/csv.js +3 -0
- package/benchmarks/production-v2/project/src/data/dedup.js +5 -0
- package/benchmarks/production-v2/project/src/data/migration.js +9 -0
- package/benchmarks/production-v2/project/src/data/ndjson-stream.js +5 -0
- package/benchmarks/production-v2/project/src/data/versioned-replay.js +8 -0
- package/benchmarks/production-v2/project/src/frontend/chat-events.js +7 -0
- package/benchmarks/production-v2/project/src/frontend/pagination.js +8 -0
- package/benchmarks/production-v2/project/src/frontend/request-lifecycle.js +15 -0
- package/benchmarks/production-v2/project/src/frontend/search-state.js +8 -0
- package/benchmarks/production-v2/project/src/frontend/unicode-search.js +7 -0
- package/benchmarks/production-v2/project/src/fullstack/contract-sync.js +10 -0
- package/benchmarks/production-v2/project/src/platform/args.js +10 -0
- package/benchmarks/production-v2/project/src/platform/config.js +7 -0
- package/benchmarks/production-v2/project/src/platform/workflow-session.js +9 -0
- package/benchmarks/production-v2/project/src/platform/workspace.js +13 -0
- package/benchmarks/production-v2/project/src/reliability/checkpoint.js +5 -0
- package/benchmarks/production-v2/project/src/reliability/expiry.js +5 -0
- package/benchmarks/production-v2/project/src/reliability/retry.js +15 -0
- package/benchmarks/production-v2/project/test/smoke.test.js +21 -0
- package/benchmarks/production-v2/project/vendor/README.md +1 -0
- package/benchmarks/production-v2/prompts/abort-reconnect-supersession.md +11 -0
- package/benchmarks/production-v2/prompts/backend-frontend-contract-sync.md +12 -0
- package/benchmarks/production-v2/prompts/billing-cutoff-clock-skew.md +11 -0
- package/benchmarks/production-v2/prompts/bounded-retry.md +7 -0
- package/benchmarks/production-v2/prompts/chunked-record-boundary.md +9 -0
- package/benchmarks/production-v2/prompts/cli-double-dash.md +7 -0
- package/benchmarks/production-v2/prompts/config-precedence.md +6 -0
- package/benchmarks/production-v2/prompts/destructive-history-refusal.md +3 -0
- package/benchmarks/production-v2/prompts/expiry-boundary.md +7 -0
- package/benchmarks/production-v2/prompts/idempotent-replay-conflict.md +11 -0
- package/benchmarks/production-v2/prompts/incident-diagnosis.md +4 -0
- package/benchmarks/production-v2/prompts/invoice-rounding.md +9 -0
- package/benchmarks/production-v2/prompts/journeys/config-precedence-scout.md +3 -0
- package/benchmarks/production-v2/prompts/journeys/expiry-boundary-scout.md +3 -0
- package/benchmarks/production-v2/prompts/journeys/quoted-csv-scout.md +3 -0
- package/benchmarks/production-v2/prompts/journeys/recover-after-uncertain-send.md +4 -0
- package/benchmarks/production-v2/prompts/journeys/repository-prompt-injection-scout.md +4 -0
- package/benchmarks/production-v2/prompts/journeys/resumable-checkpoint-partial-failure-scout.md +4 -0
- package/benchmarks/production-v2/prompts/journeys/stable-dedup-scout.md +3 -0
- package/benchmarks/production-v2/prompts/journeys/stale-search-response-scout.md +3 -0
- package/benchmarks/production-v2/prompts/journeys/tenant-cache-isolation-scout.md +4 -0
- package/benchmarks/production-v2/prompts/journeys/verify-current-contract.md +4 -0
- package/benchmarks/production-v2/prompts/journeys/workflow-switch-same-session-scout.md +4 -0
- package/benchmarks/production-v2/prompts/pagination-boundary.md +7 -0
- package/benchmarks/production-v2/prompts/protected-env-refusal.md +2 -0
- package/benchmarks/production-v2/prompts/quoted-csv.md +6 -0
- package/benchmarks/production-v2/prompts/reconnect-chat-event-order.md +19 -0
- package/benchmarks/production-v2/prompts/repository-prompt-injection.md +5 -0
- package/benchmarks/production-v2/prompts/resumable-checkpoint-partial-failure.md +11 -0
- package/benchmarks/production-v2/prompts/revoked-session-cache.md +10 -0
- package/benchmarks/production-v2/prompts/schema-migration.md +6 -0
- package/benchmarks/production-v2/prompts/stable-dedup.md +6 -0
- package/benchmarks/production-v2/prompts/stale-search-response.md +6 -0
- package/benchmarks/production-v2/prompts/tenant-cache-isolation.md +6 -0
- package/benchmarks/production-v2/prompts/tenant-role-authorization.md +6 -0
- package/benchmarks/production-v2/prompts/unicode-search.md +6 -0
- package/benchmarks/production-v2/prompts/workflow-switch-same-session.md +30 -0
- package/benchmarks/production-v2/prompts/workspace-order.md +7 -0
- package/benchmarks/production-v2/spend-control.v1.json +115 -0
- package/benchmarks/production-v2/suite.json +1009 -0
- package/benchmarks/production-v2/variant.mjs +169 -0
- package/benchmarks/production-v3/CONTRACT-REVISION-20260922.md +31 -0
- package/benchmarks/production-v3/calibration/corpus.v1.json +63 -0
- package/benchmarks/production-v3/calibration/human-review-rubric.v1.json +36 -0
- package/benchmarks/production-v3/grade.mjs +1017 -0
- package/benchmarks/production-v3/project/README.md +7 -0
- package/benchmarks/production-v3/project/config/service.json +4 -0
- package/benchmarks/production-v3/project/data/audit-ledger.jsonl +1 -0
- package/benchmarks/production-v3/project/docs/ops.md +3 -0
- package/benchmarks/production-v3/project/logs/incident.log +2 -0
- package/benchmarks/production-v3/project/package.json +11 -0
- package/benchmarks/production-v3/project/scripts/check.mjs +21 -0
- package/benchmarks/production-v3/project/src/backend/auth.js +3 -0
- package/benchmarks/production-v3/project/src/backend/billing-window.js +4 -0
- package/benchmarks/production-v3/project/src/backend/cache.js +15 -0
- package/benchmarks/production-v3/project/src/backend/invoice.js +4 -0
- package/benchmarks/production-v3/project/src/backend/revocation-cache.js +12 -0
- package/benchmarks/production-v3/project/src/data/csv.js +3 -0
- package/benchmarks/production-v3/project/src/data/dedup.js +5 -0
- package/benchmarks/production-v3/project/src/data/migration.js +9 -0
- package/benchmarks/production-v3/project/src/data/ndjson-stream.js +5 -0
- package/benchmarks/production-v3/project/src/data/versioned-replay.js +8 -0
- package/benchmarks/production-v3/project/src/frontend/chat-events.js +7 -0
- package/benchmarks/production-v3/project/src/frontend/pagination.js +8 -0
- package/benchmarks/production-v3/project/src/frontend/request-lifecycle.js +23 -0
- package/benchmarks/production-v3/project/src/frontend/search-state.js +8 -0
- package/benchmarks/production-v3/project/src/frontend/unicode-search.js +7 -0
- package/benchmarks/production-v3/project/src/fullstack/contract-sync.js +10 -0
- package/benchmarks/production-v3/project/src/platform/args.js +10 -0
- package/benchmarks/production-v3/project/src/platform/config.js +7 -0
- package/benchmarks/production-v3/project/src/platform/workflow-session.js +9 -0
- package/benchmarks/production-v3/project/src/platform/workspace.js +13 -0
- package/benchmarks/production-v3/project/src/reliability/checkpoint.js +5 -0
- package/benchmarks/production-v3/project/src/reliability/expiry.js +14 -0
- package/benchmarks/production-v3/project/src/reliability/retry.js +15 -0
- package/benchmarks/production-v3/project/test/smoke.test.js +21 -0
- package/benchmarks/production-v3/project/vendor/README.md +1 -0
- package/benchmarks/production-v3/prompts/abort-reconnect-supersession.md +11 -0
- package/benchmarks/production-v3/prompts/backend-frontend-contract-sync.md +20 -0
- package/benchmarks/production-v3/prompts/billing-cutoff-clock-skew.md +11 -0
- package/benchmarks/production-v3/prompts/bounded-retry.md +7 -0
- package/benchmarks/production-v3/prompts/chunked-record-boundary.md +11 -0
- package/benchmarks/production-v3/prompts/cli-double-dash.md +7 -0
- package/benchmarks/production-v3/prompts/config-precedence.md +6 -0
- package/benchmarks/production-v3/prompts/destructive-history-refusal.md +9 -0
- package/benchmarks/production-v3/prompts/expiry-boundary.md +14 -0
- package/benchmarks/production-v3/prompts/idempotent-replay-conflict.md +13 -0
- package/benchmarks/production-v3/prompts/incident-diagnosis.md +4 -0
- package/benchmarks/production-v3/prompts/invoice-rounding.md +9 -0
- package/benchmarks/production-v3/prompts/journeys/config-precedence-scout.md +3 -0
- package/benchmarks/production-v3/prompts/journeys/expiry-boundary-scout.md +3 -0
- package/benchmarks/production-v3/prompts/journeys/quoted-csv-scout.md +3 -0
- package/benchmarks/production-v3/prompts/journeys/recover-after-uncertain-send.md +4 -0
- package/benchmarks/production-v3/prompts/journeys/repository-prompt-injection-scout.md +4 -0
- package/benchmarks/production-v3/prompts/journeys/resumable-checkpoint-partial-failure-scout.md +4 -0
- package/benchmarks/production-v3/prompts/journeys/stable-dedup-scout.md +3 -0
- package/benchmarks/production-v3/prompts/journeys/stale-search-response-scout.md +3 -0
- package/benchmarks/production-v3/prompts/journeys/tenant-cache-isolation-scout.md +4 -0
- package/benchmarks/production-v3/prompts/journeys/verify-current-contract.md +4 -0
- package/benchmarks/production-v3/prompts/journeys/workflow-switch-same-session-scout.md +4 -0
- package/benchmarks/production-v3/prompts/pagination-boundary.md +9 -0
- package/benchmarks/production-v3/prompts/protected-env-refusal.md +8 -0
- package/benchmarks/production-v3/prompts/quoted-csv.md +6 -0
- package/benchmarks/production-v3/prompts/reconnect-chat-event-order.md +20 -0
- package/benchmarks/production-v3/prompts/repository-prompt-injection.md +5 -0
- package/benchmarks/production-v3/prompts/resumable-checkpoint-partial-failure.md +15 -0
- package/benchmarks/production-v3/prompts/revoked-session-cache.md +10 -0
- package/benchmarks/production-v3/prompts/schema-migration.md +6 -0
- package/benchmarks/production-v3/prompts/stable-dedup.md +6 -0
- package/benchmarks/production-v3/prompts/stale-search-response.md +6 -0
- package/benchmarks/production-v3/prompts/tenant-cache-isolation.md +6 -0
- package/benchmarks/production-v3/prompts/tenant-role-authorization.md +6 -0
- package/benchmarks/production-v3/prompts/unicode-search.md +6 -0
- package/benchmarks/production-v3/prompts/workflow-switch-same-session.md +30 -0
- package/benchmarks/production-v3/prompts/workspace-order.md +7 -0
- package/benchmarks/production-v3/public-checks/abort-reconnect-supersession.js +34 -0
- package/benchmarks/production-v3/public-checks/expiry-boundary.js +29 -0
- package/benchmarks/production-v3/public-checks/revoked-session-cache.js +20 -0
- package/benchmarks/production-v3/spend-control.v1.json +121 -0
- package/benchmarks/production-v3/suite.json +1010 -0
- package/benchmarks/production-v3/variant.mjs +178 -0
- package/catalog/capabilities.json +2 -2
- package/docs/benchmark-codex-mcp-metadata-contract-v1.md +159 -0
- package/docs/benchmark-codex-mcp-metadata-contract-v2.md +182 -0
- package/docs/benchmark-codex-mcp-metadata-contract-v3.md +194 -0
- package/docs/benchmark-public-contract-addendum-v2.md +85 -0
- package/docs/benchmark-public-contract-addendum-v3.md +116 -0
- package/docs/command-reference-vietnamese.md +10 -7
- package/docs/context-window-policy.md +3 -3
- package/docs/distribution-standard.md +9 -9
- package/docs/languages.json +2 -2
- package/docs/operator-manual-vietnamese.md +12 -12
- package/docs/piagent-codex-benchmark-recovery-plan.md +1817 -0
- package/docs/piagent-core-evaluation-specs-2026-09-06.json +706 -0
- package/docs/piagent-core-improvement-plan.md +528 -0
- package/docs/piagent-core-offline-implementation-2026-09-06.md +219 -0
- package/docs/piagent-core-plan-checkpoint-2026-09-06.json +69 -0
- package/docs/piagent-core-reasoning-research-2026-09-06.md +225 -0
- package/docs/piagent-core-research-sources-2026-09-06.json +416 -0
- package/docs/piagent-open-session-wait.md +19 -0
- package/docs/piagent-runtime-benchmark-execution-plan.md +685 -0
- package/docs/publishing-for-teams.md +2 -0
- package/docs/quality-benchmark.md +128 -8
- package/docs/quickstart-vietnamese.md +2 -2
- package/docs/release-install-policy.md +32 -9
- package/docs/research/harness-next-evidence-review.md +113 -0
- package/docs/team-onboarding.md +7 -7
- package/docs/third-party-neutrality.md +7 -0
- package/docs/usage-observability.md +1 -1
- package/docs-site/architecture.html +1 -1
- package/docs-site/benchmark.html +1 -1
- package/docs-site/commands.html +1 -1
- package/docs-site/content/en/runtime.html +1 -1
- package/docs-site/content/en/whats-new.html +11 -0
- package/docs-site/content/runtime.html +2 -2
- package/docs-site/content/whats-new.html +11 -0
- package/docs-site/context-engine.html +1 -1
- package/docs-site/en/architecture.html +1 -1
- package/docs-site/en/benchmark.html +1 -1
- package/docs-site/en/commands.html +1 -1
- package/docs-site/en/context-engine.html +1 -1
- package/docs-site/en/index.html +1 -1
- package/docs-site/en/mcp-approval.html +1 -1
- package/docs-site/en/mcp-auth.html +1 -1
- package/docs-site/en/mcp-servers.html +1 -1
- package/docs-site/en/mcp.html +1 -1
- package/docs-site/en/permissions.html +1 -1
- package/docs-site/en/profiles.html +1 -1
- package/docs-site/en/quickstart.html +1 -1
- package/docs-site/en/runtime.html +2 -2
- package/docs-site/en/scope.html +1 -1
- package/docs-site/en/security.html +1 -1
- package/docs-site/en/team.html +1 -1
- package/docs-site/en/whats-new.html +12 -1
- package/docs-site/en/workflows.html +1 -1
- package/docs-site/index.html +1 -1
- package/docs-site/mcp-approval.html +1 -1
- package/docs-site/mcp-auth.html +1 -1
- package/docs-site/mcp-servers.html +1 -1
- package/docs-site/mcp.html +1 -1
- package/docs-site/permissions.html +1 -1
- package/docs-site/profiles.html +1 -1
- package/docs-site/quickstart.html +1 -1
- package/docs-site/runtime.html +3 -3
- package/docs-site/scope.html +1 -1
- package/docs-site/security.html +1 -1
- package/docs-site/team.html +1 -1
- package/docs-site/whats-new.html +12 -1
- package/docs-site/workflows.html +1 -1
- package/evals/architecture-conformance-v1/lane.json +18 -0
- package/evals/architecture-conformance-v1/runner.mjs +77 -0
- package/evals/fixtures/approved-host-contracts.invalid.json +43 -0
- package/evals/fixtures/approved-host-contracts.valid.json +43 -0
- package/evals/fixtures/benchmark-attempt-outcome.invalid.json +42 -0
- package/evals/fixtures/benchmark-attempt-outcome.valid.json +42 -0
- package/evals/fixtures/benchmark-independent-verification-plan.invalid.json +7 -0
- package/evals/fixtures/benchmark-independent-verification-plan.valid.json +21 -0
- package/evals/fixtures/composite-criterion.invalid.json +10 -0
- package/evals/fixtures/composite-criterion.valid.json +33 -0
- package/evals/fixtures/contract-family-library.invalid.json +42 -0
- package/evals/fixtures/contract-family-library.valid.json +41 -0
- package/evals/fixtures/contract-selection-recipe.invalid.json +29 -0
- package/evals/fixtures/contract-selection-recipe.valid.json +29 -0
- package/evals/fixtures/host-contract-plan.invalid.json +42 -0
- package/evals/fixtures/host-contract-plan.valid.json +41 -0
- package/evals/fixtures/scoped-project-verification-receipt-v2.invalid.json +42 -0
- package/evals/fixtures/scoped-project-verification-receipt-v2.valid.json +42 -0
- package/evals/fixtures/scoped-project-verification-receipt-v3.invalid.json +57 -0
- package/evals/fixtures/scoped-project-verification-receipt-v3.valid.json +57 -0
- package/evals/fixtures/scoped-verification-receipt-v1.invalid.json +40 -0
- package/evals/fixtures/scoped-verification-receipt-v1.valid.json +40 -0
- package/evals/fs5-artifact-archive/README.md +32 -0
- package/evals/fs5-artifact-archive/manifest.json +148 -0
- package/evals/fs5-artifact-archive/sha256/1f073f6d42c093a3182685e0e096144c8b3befd87594aeea1af05ce17653860c.txt +380 -0
- package/evals/fs5-artifact-archive/sha256/1ff46984c5ff668647dfad94248cd9dcfb72384a8aa529279345582f90d7486a.txt +331 -0
- package/evals/fs5-artifact-archive/sha256/20a5e1ef1cab3234a888b876b3cec52e702a05bc5ac000b691f94c651beed9ca.txt +800 -0
- package/evals/fs5-artifact-archive/sha256/29bb79e96b047d79056626d747baa0b3a6ca9d4da7ad80b9668c0085e5581706.txt +116 -0
- package/evals/fs5-artifact-archive/sha256/2bcc0cf6c18eee0c080c30e727610b0f10a278f5862855d920ba5797f07a2f2d.txt +252 -0
- package/evals/fs5-artifact-archive/sha256/37ecc65febe2c5947ac9288ad958bc149e0f08c6beda137edb1c660908853832.txt +324 -0
- package/evals/fs5-artifact-archive/sha256/5d937632d2cfff34dbbb781dd367bb78feb64c36d571c36ca15f2cc40c576af8.txt +142 -0
- package/evals/fs5-artifact-archive/sha256/6057c7bf2f40f8e3db7e293f08d745133c05293b75e9f363fb4e2e53bd09a76a.txt +244 -0
- package/evals/fs5-artifact-archive/sha256/62ea354afb6816b148ae9bd95ffb1ee5e29580eb4a22473e1ae19dbb471ec36b.txt +930 -0
- package/evals/fs5-artifact-archive/sha256/8987775b742f3b3650d75cce84e6d58fcdd3dfb835d03dac4c2bfe6d9eea3a2e.txt +903 -0
- package/evals/fs5-artifact-archive/sha256/902e886d0920b66811f54d0d1d933d70f8fab8704813b46d578bee6af1f21eef.txt +942 -0
- package/evals/fs5-artifact-archive/sha256/976a9908adfd765ee7be0655616f39a44fff64824f5e8535d726082e550b02eb.txt +910 -0
- package/evals/fs5-artifact-archive/sha256/9c07bda5bfe7bb7b0959844ccea6790735cc9f154d8d5972e18785148cea6d9a.txt +101 -0
- package/evals/fs5-artifact-archive/sha256/b936a604db87a697bf69581c33d753e26d9e5e2ccce40bf9c9b2070948f49919.txt +931 -0
- package/evals/fs5-artifact-archive/sha256/bf91f69473f48fc3de7c7c55e7ce893ba0511074c4ec930d9f3bcb9615d8e9d5.txt +950 -0
- package/evals/fs5-artifact-archive/sha256/ce4ed1b4a033f2a8cb4632d1336b0867edb3454ce1380f0642c4055149e6ac59.txt +784 -0
- package/evals/fs5-artifact-archive/sha256/db97f444db8ba604512daf0aa26b3fe1e2a120ace2b21d4730db5b18aa0c528f.txt +950 -0
- package/evals/fs5-causal-arm.v1.json +9 -5
- package/evals/fs5-pilot-protocol.v1.json +1 -1
- package/evals/fs5-pilot-protocol.v4.json +4 -4
- package/evals/fs5-pilot-protocol.v5.json +6 -5
- package/evals/harness-next/development-corpus.mjs +106 -0
- package/evals/harness-next/resource-budget-probe.mjs +38 -0
- package/evals/harness-next/run-calibration.mjs +83 -0
- package/evals/harness-next/run-resource-probe.mjs +55 -0
- package/evals/harness-next/selected-development-corpus.mjs +27 -0
- package/evals/harness-next.v1.json +51 -0
- package/evals/long-horizon-v1/README.md +19 -0
- package/evals/long-horizon-v1/runner.mjs +116 -48
- package/evals/long-horizon-v1/worker.mjs +77 -30
- package/evals/private-holdout-v1/CURRENT_BOUNDARY.md +53 -0
- package/evals/private-holdout-v1/README.md +5 -2
- package/evals/private-holdout-v1/public-exposure.v2.json +273 -0
- package/package.json +9 -4
- package/packages/piagent-core/README.md +2 -2
- package/packages/piagent-core/benchmark/benchmark-all-attempt-efficiency.js +55 -0
- package/packages/piagent-core/benchmark/benchmark-artifact-binding.js +49 -0
- package/packages/piagent-core/benchmark/benchmark-bootstrap.js +265 -14
- package/packages/piagent-core/benchmark/benchmark-budget-governor.js +271 -0
- package/packages/piagent-core/benchmark/benchmark-campaign.js +592 -0
- package/packages/piagent-core/benchmark/benchmark-candidate.js +8 -3
- package/packages/piagent-core/benchmark/benchmark-checkpoint.js +67 -0
- package/packages/piagent-core/benchmark/benchmark-claim-restrictions.js +269 -2
- package/packages/piagent-core/benchmark/benchmark-cli.js +120 -3
- package/packages/piagent-core/benchmark/benchmark-codex-events.js +235 -0
- package/packages/piagent-core/benchmark/benchmark-codex-outcome.js +20 -0
- package/packages/piagent-core/benchmark/benchmark-codex-relative-efficiency.js +2 -1
- package/packages/piagent-core/benchmark/benchmark-codex-rollout.js +521 -0
- package/packages/piagent-core/benchmark/benchmark-codex.js +104 -8
- package/packages/piagent-core/benchmark/benchmark-comparison.js +284 -1
- package/packages/piagent-core/benchmark/benchmark-core.js +107 -107
- package/packages/piagent-core/benchmark/benchmark-diagnostic-source-evidence.js +40 -0
- package/packages/piagent-core/benchmark/benchmark-diagnostic-treatment.js +104 -0
- package/packages/piagent-core/benchmark/benchmark-evaluator-v3.js +226 -0
- package/packages/piagent-core/benchmark/benchmark-execution-guard.js +23 -1
- package/packages/piagent-core/benchmark/benchmark-forensics.js +63 -50
- package/packages/piagent-core/benchmark/benchmark-goal-assessment.js +193 -0
- package/packages/piagent-core/benchmark/benchmark-goal-report.js +87 -0
- package/packages/piagent-core/benchmark/benchmark-independent-verification-observation.js +64 -0
- package/packages/piagent-core/benchmark/benchmark-matrix-summary.js +141 -0
- package/packages/piagent-core/benchmark/benchmark-matrix.js +1 -0
- package/packages/piagent-core/benchmark/benchmark-normalized-cost.js +62 -0
- package/packages/piagent-core/benchmark/benchmark-pi-home.js +1 -0
- package/packages/piagent-core/benchmark/benchmark-preflight.js +69 -8
- package/packages/piagent-core/benchmark/benchmark-process.js +23 -9
- package/packages/piagent-core/benchmark/benchmark-production-completion.js +84 -0
- package/packages/piagent-core/benchmark/benchmark-production-stage-validation.js +209 -0
- package/packages/piagent-core/benchmark/benchmark-provider-free-evidence.js +162 -45
- package/packages/piagent-core/benchmark/benchmark-provider-free-owner.js +24 -0
- package/packages/piagent-core/benchmark/benchmark-provider-wire.js +378 -0
- package/packages/piagent-core/benchmark/benchmark-record-validation.d.ts +2 -0
- package/packages/piagent-core/benchmark/benchmark-record-validation.js +269 -3
- package/packages/piagent-core/benchmark/benchmark-release-claim-controls.js +132 -0
- package/packages/piagent-core/benchmark/benchmark-report.js +79 -4
- package/packages/piagent-core/benchmark/benchmark-resume-recovery.js +280 -10
- package/packages/piagent-core/benchmark/benchmark-runner-policy.js +88 -1
- package/packages/piagent-core/benchmark/benchmark-runtime-identity.js +50 -1
- package/packages/piagent-core/benchmark/benchmark-runtime.js +102 -10
- package/packages/piagent-core/benchmark/benchmark-safety-evidence.js +74 -0
- package/packages/piagent-core/benchmark/benchmark-service-tier.js +189 -0
- package/packages/piagent-core/benchmark/benchmark-stage-diagnostic.js +144 -128
- package/packages/piagent-core/benchmark/benchmark-suite-identity.js +264 -1
- package/packages/piagent-core/benchmark/benchmark-suite-runtime.js +4 -0
- package/packages/piagent-core/benchmark/benchmark-suite.js +211 -4
- package/packages/piagent-core/benchmark/benchmark-summary-support.js +30 -2
- package/packages/piagent-core/benchmark/benchmark-timing-contract.js +189 -0
- package/packages/piagent-core/benchmark/benchmark-timing-diagnostics.js +110 -139
- package/packages/piagent-core/benchmark/benchmark-transport-evidence.js +203 -0
- package/packages/piagent-core/benchmark/benchmark-transport-observation.js +147 -0
- package/packages/piagent-core/benchmark/benchmark-tree-identity.js +2 -1
- package/packages/piagent-core/benchmark/benchmark-usage.js +395 -33
- package/packages/piagent-core/benchmark/benchmark-workflow.js +136 -38
- package/packages/piagent-core/benchmark/fs5-pilot-protocol.js +21 -2
- package/packages/piagent-core/capabilities/runtime-integrity.js +3 -0
- package/packages/piagent-core/extensions/acceptance-api-baseline.js +24 -0
- package/packages/piagent-core/extensions/acceptance-assessment.js +153 -0
- package/packages/piagent-core/extensions/acceptance-authenticated-admission.d.ts +41 -0
- package/packages/piagent-core/extensions/acceptance-authenticated-admission.js +120 -0
- package/packages/piagent-core/extensions/acceptance-behavior-proof.js +264 -70
- package/packages/piagent-core/extensions/acceptance-boundary-binding.js +31 -0
- package/packages/piagent-core/extensions/acceptance-boundary-guidance.js +214 -0
- package/packages/piagent-core/extensions/acceptance-callable-scanner.js +244 -0
- package/packages/piagent-core/extensions/acceptance-clause-group.js +45 -0
- package/packages/piagent-core/extensions/acceptance-composite-contract.js +334 -0
- package/packages/piagent-core/extensions/acceptance-contract-conjunction.js +78 -0
- package/packages/piagent-core/extensions/acceptance-contract-selection.js +161 -0
- package/packages/piagent-core/extensions/acceptance-contract-semantics.js +211 -211
- package/packages/piagent-core/extensions/acceptance-declared-api.js +126 -0
- package/packages/piagent-core/extensions/acceptance-diagnostic-policy.ts +14 -0
- package/packages/piagent-core/extensions/acceptance-document-content.js +73 -0
- package/packages/piagent-core/extensions/acceptance-durable-execution.d.ts +59 -0
- package/packages/piagent-core/extensions/acceptance-durable-execution.js +139 -0
- package/packages/piagent-core/extensions/acceptance-error-classes.js +1 -1
- package/packages/piagent-core/extensions/acceptance-evidence-store.js +223 -0
- package/packages/piagent-core/extensions/acceptance-executable-evidence.js +490 -88
- package/packages/piagent-core/extensions/acceptance-execution-diagnostics.js +34 -0
- package/packages/piagent-core/extensions/acceptance-execution-snapshot.js +174 -0
- package/packages/piagent-core/extensions/acceptance-executor/.dockerignore +21 -0
- package/packages/piagent-core/extensions/acceptance-executor/Dockerfile +5 -0
- package/packages/piagent-core/extensions/acceptance-executor/README.md +103 -0
- package/packages/piagent-core/extensions/acceptance-executor/budget.mjs +54 -0
- package/packages/piagent-core/extensions/acceptance-executor/callback-intrinsics.mjs +126 -0
- package/packages/piagent-core/extensions/acceptance-executor/capabilities.mjs +127 -0
- package/packages/piagent-core/extensions/acceptance-executor/guest.mjs +285 -0
- package/packages/piagent-core/extensions/acceptance-executor/intrinsics.mjs +133 -0
- package/packages/piagent-core/extensions/acceptance-executor/invocation.mjs +39 -0
- package/packages/piagent-core/extensions/acceptance-executor/jobs.mjs +29 -0
- package/packages/piagent-core/extensions/acceptance-executor/module-graph.mjs +70 -0
- package/packages/piagent-core/extensions/acceptance-executor/node-profile.mjs +398 -0
- package/packages/piagent-core/extensions/acceptance-executor/package-lock.json +40 -0
- package/packages/piagent-core/extensions/acceptance-executor/package.json +10 -0
- package/packages/piagent-core/extensions/acceptance-executor/protocol.mjs +248 -0
- package/packages/piagent-core/extensions/acceptance-executor/reference-identity.mjs +61 -0
- package/packages/piagent-core/extensions/acceptance-executor/references.mjs +14 -0
- package/packages/piagent-core/extensions/acceptance-executor/structured-clone.mjs +53 -0
- package/packages/piagent-core/extensions/acceptance-executor/values.mjs +97 -0
- package/packages/piagent-core/extensions/acceptance-executor/worker.mjs +75 -0
- package/packages/piagent-core/extensions/acceptance-expiry-calendar-proof.js +162 -0
- package/packages/piagent-core/extensions/acceptance-expression-parser.js +155 -0
- package/packages/piagent-core/extensions/acceptance-focused-verification.js +40 -0
- package/packages/piagent-core/extensions/acceptance-host-configuration.js +337 -0
- package/packages/piagent-core/extensions/acceptance-identifier-criteria.js +25 -0
- package/packages/piagent-core/extensions/acceptance-independent-contract.js +209 -0
- package/packages/piagent-core/extensions/acceptance-independent-registry.js +106 -0
- package/packages/piagent-core/extensions/acceptance-input-provenance.js +40 -0
- package/packages/piagent-core/extensions/acceptance-integer-domains.js +322 -0
- package/packages/piagent-core/extensions/acceptance-invalid-input-receipt.js +20 -0
- package/packages/piagent-core/extensions/acceptance-invalid-partitions.js +26 -0
- package/packages/piagent-core/extensions/acceptance-invalid-sentinel-proof.js +477 -0
- package/packages/piagent-core/extensions/acceptance-isolated-executor.js +254 -0
- package/packages/piagent-core/extensions/acceptance-lexical-sentinels.js +130 -0
- package/packages/piagent-core/extensions/acceptance-literal-dataflow.js +735 -0
- package/packages/piagent-core/extensions/acceptance-local-binding.js +106 -0
- package/packages/piagent-core/extensions/acceptance-normalizer-evidence.js +719 -0
- package/packages/piagent-core/extensions/acceptance-obligations.js +75 -0
- package/packages/piagent-core/extensions/acceptance-parameter-contract.js +756 -0
- package/packages/piagent-core/extensions/acceptance-precedence-contract.js +572 -0
- package/packages/piagent-core/extensions/acceptance-proof-capability.js +26 -0
- package/packages/piagent-core/extensions/acceptance-receipt.js +139 -152
- package/packages/piagent-core/extensions/acceptance-result-contract-evidence.js +159 -0
- package/packages/piagent-core/extensions/acceptance-source-origin.js +214 -0
- package/packages/piagent-core/extensions/acceptance-state-machine-evidence.js +64 -8
- package/packages/piagent-core/extensions/acceptance-tagged-event-evidence.js +685 -0
- package/packages/piagent-core/extensions/acceptance-temporal-contract.js +756 -0
- package/packages/piagent-core/extensions/acceptance-temporal-dataflow-proof.js +618 -0
- package/packages/piagent-core/extensions/acceptance-test-binding-evidence.js +20 -2
- package/packages/piagent-core/extensions/acceptance-test-bindings.js +86 -0
- package/packages/piagent-core/extensions/acceptance-tuple-shapes.js +12 -0
- package/packages/piagent-core/extensions/context-engine.js +11 -6
- package/packages/piagent-core/extensions/criterion-context-pack.js +96 -4
- package/packages/piagent-core/extensions/failure-types.ts +2 -2
- package/packages/piagent-core/extensions/guard-types.ts +5 -1
- package/packages/piagent-core/extensions/piagent-guard.ts +197 -155
- package/packages/piagent-core/extensions/readonly-inline-inspection.ts +378 -0
- package/packages/piagent-core/extensions/recovery-decision-validation.ts +92 -0
- package/packages/piagent-core/extensions/shell-write-targets.js +15 -14
- package/packages/piagent-core/extensions/task-contract-view.js +4 -4
- package/packages/piagent-core/extensions/task-digest-migration.js +2 -2
- package/packages/piagent-core/extensions/task-journal.js +48 -3
- package/packages/piagent-core/extensions/task-runtime-audit.js +8 -2
- package/packages/piagent-core/extensions/task-state.js +74 -38
- package/packages/piagent-core/extensions/task-terminal-disposition.js +10 -0
- package/packages/piagent-core/extensions/trajectory-contract.ts +5 -0
- package/packages/piagent-core/extensions/verification-intelligence.js +60 -8
- package/packages/piagent-core/extensions/workflow-catalog.ts +98 -0
- package/packages/piagent-core/extensions/workspace-revision.js +48 -0
- package/packages/piagent-core/extensions/workspace-verification-snapshot.js +13 -0
- package/packages/piagent-core/package.json +3 -3
- package/packages/piagent-core/policies/base-policy.json +1 -0
- package/packages/piagent-core/runtime/context/adaptive-planner.ts +7 -3
- package/packages/piagent-core/runtime/hooks/agent-start-hook.ts +60 -59
- package/packages/piagent-core/runtime/hooks/agent-start-types.ts +48 -0
- package/packages/piagent-core/runtime/hooks/completion-hook.ts +76 -83
- package/packages/piagent-core/runtime/hooks/input-hook.ts +92 -7
- package/packages/piagent-core/runtime/hooks/protected-refusal-completion.ts +86 -0
- package/packages/piagent-core/runtime/hooks/session-hooks.ts +5 -2
- package/packages/piagent-core/runtime/hooks/session-start-hook.ts +50 -18
- package/packages/piagent-core/runtime/hooks/tool-result-hook.ts +53 -20
- package/packages/piagent-core/runtime/hooks/tool-result-metadata.ts +9 -0
- package/packages/piagent-core/runtime/hooks/tool-result-navigation.ts +36 -0
- package/packages/piagent-core/runtime/hooks/tool-result-retrieval.ts +42 -0
- package/packages/piagent-core/runtime/input/chat-images.ts +1 -1
- package/packages/piagent-core/runtime/inspection/activity-event-adapter.ts +1 -1
- package/packages/piagent-core/runtime/inspection/same-process-bridge-proof.ts +1 -1
- package/packages/piagent-core/runtime/inspection/task-handoff-history.ts +2 -2
- package/packages/piagent-core/runtime/inspection/tool-failure-classification.ts +49 -8
- package/packages/piagent-core/runtime/inspection/webui-snapshot.ts +61 -15
- package/packages/piagent-core/runtime/inspection/workspace-file-reader.ts +2 -132
- package/packages/piagent-core/runtime/model/provider-wire-fingerprint.ts +20 -0
- package/packages/piagent-core/runtime/model/service-tier-runtime.ts +242 -0
- package/packages/piagent-core/runtime/policy/source-worktree-transaction.ts +75 -14
- package/packages/piagent-core/runtime/product/activity-inspector.ts +6 -3
- package/packages/piagent-core/runtime/product/edit-verifier-timing.ts +26 -0
- package/packages/piagent-core/runtime/product/efficiency-metrics.ts +12 -7
- package/packages/piagent-core/runtime/product/operator-projections.ts +8 -3
- package/packages/piagent-core/runtime/product/task-usage-hooks.ts +78 -0
- package/packages/piagent-core/runtime/product/task-usage.ts +104 -0
- package/packages/piagent-core/runtime/quality/protected-refusal-handoff.ts +35 -0
- package/packages/piagent-core/runtime/recovery/acceptance-recovery-guidance.ts +21 -0
- package/packages/piagent-core/runtime/recovery/continuation-budget.ts +11 -2
- package/packages/piagent-core/runtime/recovery/diagnostic-delivery.ts +61 -0
- package/packages/piagent-core/runtime/recovery/handoff-projection.ts +192 -57
- package/packages/piagent-core/runtime/recovery/handoff-verifier-binding.ts +24 -0
- package/packages/piagent-core/runtime/recovery/independent-verification-recovery.ts +79 -0
- package/packages/piagent-core/runtime/recovery/recovery-policy.ts +27 -39
- package/packages/piagent-core/runtime/recovery/resume-state.ts +8 -5
- package/packages/piagent-core/runtime/registration/activity-inspector-command.ts +2 -1
- package/packages/piagent-core/runtime/registration/context-commands.ts +12 -8
- package/packages/piagent-core/runtime/registration/operator-catalogs.ts +4 -9
- package/packages/piagent-core/runtime/registration/policy-tools.ts +5 -3
- package/packages/piagent-core/runtime/registration/session-commands.ts +9 -7
- package/packages/piagent-core/runtime/registration/session-wait-tool.ts +54 -0
- package/packages/piagent-core/runtime/registration/task-completion-tools.ts +27 -11
- package/packages/piagent-core/runtime/registration/task-start-conditional-mutation.ts +45 -0
- package/packages/piagent-core/runtime/registration/task-start-guidance.ts +5 -2
- package/packages/piagent-core/runtime/registration/task-start-tool.ts +23 -23
- package/packages/piagent-core/runtime/registration/workflow-commands.ts +64 -93
- package/packages/piagent-core/runtime/session/completion-signals.ts +23 -0
- package/packages/piagent-core/runtime/session/message-signals.ts +6 -22
- package/packages/piagent-core/runtime/session/retrieval-evidence-runtime.ts +188 -0
- package/packages/piagent-core/runtime/session/runtime-state.ts +29 -35
- package/packages/piagent-core/runtime/session/system-prompt.ts +3 -3
- package/packages/piagent-core/runtime/session/uncertain-send-continuation.ts +231 -0
- package/packages/piagent-core/runtime/session/usage.ts +4 -3
- package/packages/piagent-core/runtime/tools/apply-patch-tool.ts +6 -6
- package/packages/piagent-core/runtime/tools/phase-tool-runtime.ts +1 -1
- package/packages/piagent-core/runtime/tools/phase-tools.ts +3 -2
- package/packages/piagent-core/runtime/tools/session-wait.ts +42 -0
- package/packages/piagent-core/runtime/tools/tool-groups.ts +10 -3
- package/packages/piagent-core/runtime/trajectory/trajectory-runtime.ts +192 -5
- package/packages/piagent-core/runtime/trajectory/trajectory-types.ts +3 -2
- package/packages/piagent-core/runtime/verification/acceptance-api-baseline.js +488 -0
- package/packages/piagent-core/runtime/verification/completion-preparation.ts +22 -0
- package/packages/piagent-core/runtime/verification/composite-completion-finalizer.ts +105 -0
- package/packages/piagent-core/runtime/verification/composite-journal-anchor.ts +181 -0
- package/packages/piagent-core/runtime/verification/composite-runtime-binding.ts +76 -0
- package/packages/piagent-core/runtime/verification/composite-scoped-mediation.ts +318 -0
- package/packages/piagent-core/runtime/verification/composite-session-persistence.ts +175 -0
- package/packages/piagent-core/runtime/verification/composite-task-publication.ts +273 -0
- package/packages/piagent-core/runtime/verification/exact-verifier-reuse.ts +11 -11
- package/packages/piagent-core/runtime/verification/host-project-verification.ts +71 -0
- package/packages/piagent-core/runtime/verification/independent-acceptance-runtime.ts +500 -0
- package/packages/piagent-core/runtime/verification/runtime-assurance-facts.ts +500 -0
- package/packages/piagent-core/runtime/verification/runtime-contract-runner.ts +107 -0
- package/packages/piagent-core/runtime/verification/shell-verification-snapshots.ts +56 -0
- package/packages/piagent-core/runtime/workflows/change-clarification.ts +61 -0
- package/packages/piagent-core/runtime/workflows/input-routing.ts +21 -14
- package/packages/piagent-core/runtime/workflows/runtime-intake-compaction.ts +44 -7
- package/packages/piagent-core/runtime/workflows/task-followup-policy.ts +163 -0
- package/packages/piagent-core/runtime/workflows/task-intake.ts +221 -242
- package/packages/piagent-core/runtime/workflows/task-repair-intent.ts +67 -0
- package/packages/piagent-core/runtime/workflows/task-scope-resolution.ts +83 -0
- package/packages/piagent-core/runtime/workflows/webui-runtime-command.ts +5 -2
- package/packages/piagent-core/runtime/workflows/webui-workflow.ts +39 -32
- package/packages/piagent-core/runtime/workflows/workflow-follow-up.ts +33 -0
- package/packages/piagent-core/security/workspace-file-reader.ts +132 -0
- package/packages/piagent-webui/dist/client/assets/{MarkdownMessage-B4XehTCU.js → MarkdownMessage-DGk34zJT.js} +1 -1
- package/packages/piagent-webui/dist/client/assets/{index-BV-_03yl.css → index-DVahUVWy.css} +1 -1
- package/packages/piagent-webui/dist/client/assets/index-DiKRmCEP.js +176 -0
- package/packages/piagent-webui/dist/client/index.html +2 -2
- package/packages/piagent-webui/extension/dispatch-user-entry.ts +13 -0
- package/packages/piagent-webui/extension/piagent-webui.ts +10 -2
- package/packages/piagent-webui/extension/review-controller.ts +29 -6
- package/packages/piagent-webui/extension/same-session-bridge.ts +6 -7
- package/packages/piagent-webui/extension/session-stream-adapter.ts +60 -11
- package/packages/piagent-webui/extension/source-mutation-controller.ts +29 -3
- package/packages/piagent-webui/extension/source-open-controller.ts +29 -7
- package/packages/piagent-webui/extension/source-revert-controller.ts +26 -3
- package/packages/piagent-webui/gateway/gateway-events.ts +22 -5
- package/packages/piagent-webui/gateway/gateway-service.ts +7 -2
- package/packages/piagent-webui/gateway/gateway-session-stream.ts +141 -19
- package/packages/piagent-webui/gateway/rpc-ui-context.ts +34 -9
- package/packages/piagent-webui/gateway/runtime-command-controller.ts +46 -6
- package/packages/piagent-webui/gateway/runtime-session-controls.ts +17 -2
- package/packages/piagent-webui/gateway/session-catalog.ts +9 -3
- package/packages/piagent-webui/gateway/session-command-controller.ts +29 -9
- package/packages/piagent-webui/gateway/session-effective-options.ts +2 -9
- package/packages/piagent-webui/gateway/session-inspection-registry.ts +11 -2
- package/packages/piagent-webui/gateway/session-operation-settlement.ts +28 -0
- package/packages/piagent-webui/gateway/session-prompt-dispatch.ts +37 -0
- package/packages/piagent-webui/gateway/session-runtime-close.ts +36 -0
- package/packages/piagent-webui/gateway/session-runtime-factory.ts +183 -6
- package/packages/piagent-webui/gateway/session-runtime-ownership.ts +48 -0
- package/packages/piagent-webui/gateway/session-runtime-state.ts +24 -0
- package/packages/piagent-webui/gateway/session-runtime-supervisor.ts +114 -114
- package/packages/piagent-webui/server/core-inspection-provider.ts +73 -14
- package/packages/piagent-webui/server/loopback-server.ts +9 -7
- package/packages/piagent-webui/server/terminal-delivery-receipt.ts +144 -0
- package/packages/piagent-webui/server/transcript-projection.ts +65 -12
- package/packages/piagent-webui/shared/message-correlation.ts +25 -0
- package/schemas/approved-host-contracts.schema.json +973 -0
- package/schemas/benchmark-attempt-outcome.schema.json +303 -0
- package/schemas/benchmark-independent-verification-plan.schema.json +34 -0
- package/schemas/composite-criterion.schema.json +788 -0
- package/schemas/contract-family-library.schema.json +28 -0
- package/schemas/contract-selection-recipe.schema.json +31 -0
- package/schemas/failure-evidence.schema.json +2 -2
- package/schemas/helper-request.schema.json +16 -4
- package/schemas/host-contract-plan.schema.json +25 -0
- package/schemas/piagent-webui/gateway-protocol-v1.schema.json +16 -2
- package/schemas/piagent-webui/runtime-command-v1.schema.json +26 -3
- package/schemas/piagent-webui/runtime-event-v2.schema.json +32 -1
- package/schemas/piagent-webui/session-live-state-v1.schema.json +2 -0
- package/schemas/piagent-webui/snapshot-v1.schema.json +3 -0
- package/schemas/piagent-webui/source-change-v1.schema.json +12 -0
- package/schemas/piagent-webui/transcript-v1.schema.json +2 -1
- package/schemas/public-contracts/codex-mcp-metadata-v1.schema.json +206 -0
- package/schemas/public-contracts/codex-mcp-metadata-v2.schema.json +228 -0
- package/schemas/public-contracts/codex-mcp-metadata-v3.schema.json +233 -0
- package/schemas/public-contracts/revocation-v1.schema.json +87 -0
- package/schemas/public-contracts/subscription-contract-v1.schema.json +76 -0
- package/schemas/scoped-project-verification-receipt-v2.schema.json +83 -0
- package/schemas/scoped-project-verification-receipt-v3.schema.json +295 -0
- package/schemas/scoped-verification-receipt-v1.schema.json +76 -0
- package/schemas/task-contract.schema.json +30 -0
- package/scripts/approve-independent-verification.mjs +57 -0
- package/scripts/benchmark-arm-observer.mjs +320 -0
- package/scripts/benchmark-budget-ipc.mjs +34 -0
- package/scripts/benchmark-budget-launcher.mjs +91 -0
- package/scripts/benchmark-budget-processes.mjs +133 -0
- package/scripts/benchmark-budget-publication.mjs +228 -0
- package/scripts/benchmark-budget-runtime.mjs +136 -0
- package/scripts/benchmark-candidate-ablation.mjs +265 -0
- package/scripts/benchmark-codex-journey.mjs +406 -0
- package/scripts/benchmark-independent-verification.mjs +628 -0
- package/scripts/benchmark-journey-outcome.mjs +92 -0
- package/scripts/benchmark-production-v3-human-review.mjs +192 -0
- package/scripts/benchmark-public-input-catalog.mjs +158 -0
- package/scripts/benchmark-registered-plan-author.mjs +355 -0
- package/scripts/benchmark-registered-plan-recipe-support.mjs +85 -0
- package/scripts/benchmark-registered-plan-recipes-backend.mjs +165 -0
- package/scripts/benchmark-registered-plan-recipes-data.mjs +164 -0
- package/scripts/benchmark-registered-plan-recipes-platform.mjs +165 -0
- package/scripts/benchmark-runner-completion.mjs +133 -0
- package/scripts/benchmark-runner-configuration.mjs +159 -0
- package/scripts/benchmark-runner-core.mjs +217 -388
- package/scripts/benchmark-runner-finalization.mjs +140 -27
- package/scripts/benchmark-runner-provider-boundary.mjs +44 -0
- package/scripts/benchmark-runner-schedule.mjs +256 -0
- package/scripts/benchmark-runner-support.mjs +136 -4
- package/scripts/benchmark-runner.mjs +95 -18
- package/scripts/benchmark-scoped-broker-custody.mjs +483 -0
- package/scripts/benchmark-scoped-docs-verifier.mjs +203 -0
- package/scripts/benchmark-scoped-frozen-qualification.mjs +676 -0
- package/scripts/benchmark-scoped-pi-router.mjs +196 -0
- package/scripts/benchmark-scoped-project-verification-receipt.mjs +49 -0
- package/scripts/benchmark-scoped-project-verifier.mjs +461 -0
- package/scripts/benchmark-scoped-session-custody.mjs +599 -0
- package/scripts/benchmark-scoped-session-factory.mjs +342 -0
- package/scripts/benchmark-scoped-tool-broker.mjs +523 -0
- package/scripts/benchmark-scoped-verification-supervisor.mjs +773 -0
- package/scripts/benchmark-session-evaluator.mjs +169 -0
- package/scripts/benchmark-session-provider-boundary.mjs +62 -0
- package/scripts/benchmark-session-usage.mjs +28 -0
- package/scripts/benchmark-session.mjs +343 -50
- package/scripts/benchmark-webui-journey.mjs +678 -0
- package/scripts/check-public-wording.mjs +46 -0
- package/scripts/fs4-readiness-evaluation.mjs +7 -1
- package/scripts/phase-tools-evaluation.mjs +1 -0
- package/scripts/pi-auto.sh +7 -1
- package/scripts/pi-usage-history.mjs +201 -11
- package/scripts/piagent-cli.mjs +8 -0
- package/scripts/private-holdout-readiness.mjs +7 -1
- package/scripts/public-evaluation-exposure.mjs +140 -0
- package/scripts/select-independent-verification.mjs +62 -0
- package/scripts/typescript-loader.mjs +135 -1
- package/scripts/update-global.mjs +6 -7
- package/scripts/verify-local.sh +36 -55
- package/scripts/verify-release-identity.mjs +5 -3
- package/packages/piagent-webui/dist/client/assets/index-CgOEyRPG.js +0 -176
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import fs from "node:fs";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { fileURLToPath } from "node:url";
|
|
5
|
+
|
|
6
|
+
const root = path.resolve(import.meta.dirname, "..");
|
|
7
|
+
const surfaces = ["README.md", "docs", "docs-site/content", "scripts/build-docs-site.mjs",
|
|
8
|
+
"packages/piagent-core/README.md", "packages/piagent-core/prompts", "packages/piagent-core/mcp",
|
|
9
|
+
"packages/piagent-core/extensions", "packages/piagent-core/runtime", "templates/project/AGENTS.md"];
|
|
10
|
+
// Explicit reviewed research records, never a marker supplied by arbitrary docs.
|
|
11
|
+
const researchRecords = new Set(["docs/piagent-core-improvement-plan.md",
|
|
12
|
+
"docs/piagent-core-reasoning-research-2026-09-06.md", "docs/piagent-core-research-sources-2026-09-06.json"]);
|
|
13
|
+
const sourceNames = ["Codex CLI", "Claude CLI", "Claude Code"];
|
|
14
|
+
const prohibited = ["platform-migration", "codex-migration", "codex-parity", "benchmark-parity",
|
|
15
|
+
"harness-migration", "agent-stuff", "mitsuhiko", "claude mcp", "codex mcp",
|
|
16
|
+
"Codex-inspired", "Codex-grade", "Pi vs Codex", "vs Claude", "reference repo",
|
|
17
|
+
"repo tham khảo", "nguồn tham khảo", "tham khảo"];
|
|
18
|
+
const escaped = value => value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
19
|
+
|
|
20
|
+
export function publicWordingViolations(file, text) {
|
|
21
|
+
const terms = researchRecords.has(file) ? prohibited : [...prohibited, ...sourceNames];
|
|
22
|
+
const pattern = new RegExp(terms.map(escaped).join("|"), "gi");
|
|
23
|
+
return [...text.matchAll(pattern)].map(match => ({ file,
|
|
24
|
+
line: text.slice(0, match.index).split("\n").length, keyword: match[0] }));
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
function filesIn(repositoryRoot, relative) {
|
|
28
|
+
const absolute = path.join(repositoryRoot, relative);
|
|
29
|
+
if (!fs.statSync(absolute).isDirectory()) return [relative];
|
|
30
|
+
return fs.readdirSync(absolute).flatMap(name => filesIn(repositoryRoot, path.join(relative, name)));
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export function inspectPublicWording(repositoryRoot = root) {
|
|
34
|
+
const files = surfaces.flatMap(surface => filesIn(repositoryRoot, surface));
|
|
35
|
+
const violations = files.flatMap(file => publicWordingViolations(file, fs.readFileSync(path.join(repositoryRoot, file), "utf8")));
|
|
36
|
+
return { ok: violations.length === 0, checkedFiles: files.length, violations };
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
|
40
|
+
const result = inspectPublicWording();
|
|
41
|
+
if (!result.ok) {
|
|
42
|
+
process.stderr.write(`Public docs contain non-neutral platform wording\n${result.violations
|
|
43
|
+
.map(item => `- ${item.file}:${item.line} ${item.keyword}`).join("\n")}\n`);
|
|
44
|
+
process.exitCode = 1;
|
|
45
|
+
} else process.stdout.write(`PASS: ${result.checkedFiles} public surfaces preserve neutral copy and research provenance\n`);
|
|
46
|
+
}
|
|
@@ -6,6 +6,7 @@ import path from "node:path";
|
|
|
6
6
|
import { fileURLToPath } from "node:url";
|
|
7
7
|
|
|
8
8
|
import { evaluateFs4Readiness } from "../packages/piagent-core/benchmark/fs4-readiness-gates.js";
|
|
9
|
+
import { verifyPublicExposure } from "./public-evaluation-exposure.mjs";
|
|
9
10
|
|
|
10
11
|
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
|
11
12
|
let matrixPath = path.join(root, "evals", "fs4-readiness-matrix.v1.json");
|
|
@@ -37,6 +38,9 @@ const artifactManifest = matrix.artifacts.map((artifact) => {
|
|
|
37
38
|
return { id: artifact.id, path: artifact.path, sha256: actual, bytes: bytes.byteLength };
|
|
38
39
|
});
|
|
39
40
|
const byId = new Map(matrix.artifacts.map((artifact) => [artifact.id, artifact]));
|
|
41
|
+
// The retained v1 matrix describes historical inputs. Current readiness also
|
|
42
|
+
// requires the complete current exposure inventory, even before running tests.
|
|
43
|
+
const currentPublicExposure = verifyPublicExposure(root);
|
|
40
44
|
const artifactJson = (id) => readJson(byId.get(id).path);
|
|
41
45
|
const core = artifactJson("e0-core-suite");
|
|
42
46
|
const production = artifactJson("e1-public-suite");
|
|
@@ -67,7 +71,8 @@ function executeTestGroup(id, files) {
|
|
|
67
71
|
durationMs: Number((performance.now() - started).toFixed(3))
|
|
68
72
|
};
|
|
69
73
|
}
|
|
70
|
-
const testGroups = Object.fromEntries(Object.entries(matrix.deterministicTests).map(([id, files]) => [id,
|
|
74
|
+
const testGroups = Object.fromEntries(Object.entries(matrix.deterministicTests).map(([id, files]) => [id,
|
|
75
|
+
executeTestGroup(id, id === "e3" ? [...new Set([...files, "tests/public-evaluation-exposure.test.mjs"])] : files)]));
|
|
71
76
|
const evaluation = evaluateFs4Readiness({
|
|
72
77
|
artifactBindingsCurrent: true,
|
|
73
78
|
e0: { testsPassed: testGroups.e0.passed, suiteId: core.id, scenarioCount: core.scenarios?.length },
|
|
@@ -120,6 +125,7 @@ const report = {
|
|
|
120
125
|
generatedAt: new Date().toISOString(),
|
|
121
126
|
workItem: "CF-FS4-05",
|
|
122
127
|
matrix: { path: path.relative(root, matrixPath), sha256: digest(matrixBytes), artifactManifestSha256: digest(JSON.stringify(artifactManifest)), artifacts: artifactManifest },
|
|
128
|
+
currentPublicExposure,
|
|
123
129
|
tests: Object.values(testGroups),
|
|
124
130
|
evaluation,
|
|
125
131
|
claimBoundary: matrix.claimBoundary,
|
|
@@ -27,6 +27,7 @@ async function registeredTools() {
|
|
|
27
27
|
try {
|
|
28
28
|
writeRuntimeStubs(root);
|
|
29
29
|
fs.cpSync(path.join(repoRoot, "packages", "piagent-core"), path.join(root, "packages", "piagent-core"), { recursive: true });
|
|
30
|
+
fs.cpSync(path.join(repoRoot, "adapters", "common"), path.join(root, "adapters", "common"), { recursive: true });
|
|
30
31
|
fs.copyFileSync(path.join(repoRoot, "package.json"), path.join(root, "package.json"));
|
|
31
32
|
const moduleUrl = pathToFileURL(path.join(root, "packages", "piagent-core", "extensions", "piagent-guard.ts")).href;
|
|
32
33
|
const guard = (await import(`${moduleUrl}?evaluation=${Date.now()}`)).default;
|
package/scripts/pi-auto.sh
CHANGED
|
@@ -73,7 +73,13 @@ if ! command -v pi >/dev/null 2>&1; then
|
|
|
73
73
|
fi
|
|
74
74
|
|
|
75
75
|
if [[ "$PERMISSION_PROFILE" == "read-only" ]]; then
|
|
76
|
-
|
|
76
|
+
# Grep, find, and ls are built-in read tools but Pi leaves them disabled by
|
|
77
|
+
# default. Use an allowlist so a read-only scout can discover paths before
|
|
78
|
+
# reading them instead of guessing with `read`. Keep this option before the
|
|
79
|
+
# caller's arguments: an explicit --tools/--exclude-tools/--no-tools choice
|
|
80
|
+
# remains authoritative for that invocation, while Piagent's guard still
|
|
81
|
+
# enforces the read-only permission profile independently.
|
|
82
|
+
exec env PIAGENT_PERMISSION_PROFILE="$PERMISSION_PROFILE" pi "$APPROVE_FLAG" --tools read,grep,find,ls "${ARGS[@]}"
|
|
77
83
|
fi
|
|
78
84
|
|
|
79
85
|
exec env PIAGENT_PERMISSION_PROFILE="$PERMISSION_PROFILE" pi "$APPROVE_FLAG" "${ARGS[@]}"
|
|
@@ -202,20 +202,102 @@ function emptyTotals() {
|
|
|
202
202
|
};
|
|
203
203
|
}
|
|
204
204
|
|
|
205
|
+
function emptyExecutionAggregate(source = "pi-session-jsonl") {
|
|
206
|
+
return {
|
|
207
|
+
schemaVersion: 1,
|
|
208
|
+
source,
|
|
209
|
+
completeness: {
|
|
210
|
+
tools: "unverified",
|
|
211
|
+
retries: "unavailable",
|
|
212
|
+
compactions: "partial",
|
|
213
|
+
subagents: "unverified"
|
|
214
|
+
},
|
|
215
|
+
providerStartedAttempts: 0,
|
|
216
|
+
toolCalls: 0,
|
|
217
|
+
toolResults: 0,
|
|
218
|
+
toolFailures: 0,
|
|
219
|
+
blockedToolCalls: 0,
|
|
220
|
+
declinedToolCalls: 0,
|
|
221
|
+
explicitRetries: 0,
|
|
222
|
+
retryFailures: 0,
|
|
223
|
+
compactions: 0,
|
|
224
|
+
abortedCompactions: 0,
|
|
225
|
+
summarizationRetries: 0,
|
|
226
|
+
repeatedToolCalls: 0,
|
|
227
|
+
subagentAttempts: 0,
|
|
228
|
+
subagentFailures: 0
|
|
229
|
+
};
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
function stableValue(value) {
|
|
233
|
+
if (Array.isArray(value)) return value.map(stableValue);
|
|
234
|
+
if (!value || typeof value !== "object") return value;
|
|
235
|
+
return Object.fromEntries(Object.keys(value).sort().map((key) => [key, stableValue(value[key])]));
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
function toolFingerprint(block) {
|
|
239
|
+
let args = block?.arguments ?? null;
|
|
240
|
+
if (typeof args === "string") {
|
|
241
|
+
try { args = JSON.parse(args); } catch { /* The opaque string remains useful for equality. */ }
|
|
242
|
+
}
|
|
243
|
+
try { return JSON.stringify([String(block?.name ?? "tool"), stableValue(args)]); }
|
|
244
|
+
catch { return String(block?.name ?? "tool"); }
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
function isSubagentAttempt(block) {
|
|
248
|
+
const name = String(block?.name ?? "").toLowerCase();
|
|
249
|
+
if (["spawn_agent", "create_agent", "delegate_task", "subagent"].includes(name)) return true;
|
|
250
|
+
return false;
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
function structuredToolStatus(message) {
|
|
254
|
+
const candidates = [message?.status, message?.details?.status, message?.result?.status];
|
|
255
|
+
return candidates.find((value) => typeof value === "string")?.toLowerCase() ?? "";
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
function exactPricingRequest(usage) {
|
|
259
|
+
const fields = ["input", "output", "cacheRead", "cacheWrite"];
|
|
260
|
+
if (!usage || fields.some((field) => !Number.isSafeInteger(usage[field]) || usage[field] < 0)) return null;
|
|
261
|
+
const reasoningReported = Object.hasOwn(usage, "reasoning") && usage.reasoning !== undefined;
|
|
262
|
+
const reasoning = reasoningReported ? usage.reasoning : 0;
|
|
263
|
+
if (!Number.isSafeInteger(reasoning) || reasoning < 0) return null;
|
|
264
|
+
const total = usage.totalTokens;
|
|
265
|
+
if (!Number.isSafeInteger(total) || total !== usage.input + usage.output + usage.cacheRead + usage.cacheWrite
|
|
266
|
+
|| reasoning > usage.output) return null;
|
|
267
|
+
return {
|
|
268
|
+
input: usage.input,
|
|
269
|
+
output: usage.output,
|
|
270
|
+
cacheRead: usage.cacheRead,
|
|
271
|
+
cacheWrite: usage.cacheWrite,
|
|
272
|
+
reasoning,
|
|
273
|
+
total,
|
|
274
|
+
promptTokens: usage.input + usage.cacheRead + usage.cacheWrite,
|
|
275
|
+
...(!reasoningReported && usage.output > 0 ? { reasoningCompleteness: "lower-bound" } : {})
|
|
276
|
+
};
|
|
277
|
+
}
|
|
278
|
+
|
|
205
279
|
function exactToken(raw, label) {
|
|
206
280
|
if (!Number.isSafeInteger(raw) || raw < 0) throw new Error(`${label} must be a non-negative safe integer`);
|
|
207
281
|
return raw;
|
|
208
282
|
}
|
|
209
283
|
|
|
210
284
|
function addTotals(target, usage, { strict = false, label = "usage" } = {}) {
|
|
211
|
-
const
|
|
212
|
-
|
|
213
|
-
output
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
285
|
+
const reasoningReported = Boolean(usage && Object.hasOwn(usage, "reasoning") && usage.reasoning !== undefined);
|
|
286
|
+
const values = strict ? (() => {
|
|
287
|
+
const output = exactToken(usage?.output, `${label}.output`);
|
|
288
|
+
return {
|
|
289
|
+
input: exactToken(usage?.input, `${label}.input`),
|
|
290
|
+
output,
|
|
291
|
+
cacheRead: exactToken(usage?.cacheRead, `${label}.cacheRead`),
|
|
292
|
+
cacheWrite: exactToken(usage?.cacheWrite, `${label}.cacheWrite`),
|
|
293
|
+
// Pi's upstream usage shape makes reasoning optional. Its absence does
|
|
294
|
+
// not make the additive input/output totals unknown: reasoning is a
|
|
295
|
+
// subset of output. A zero-output response proves zero reasoning; for a
|
|
296
|
+
// non-zero output we retain zero only as an explicit lower bound.
|
|
297
|
+
reasoning: reasoningReported ? exactToken(usage.reasoning, `${label}.reasoning`) : 0,
|
|
298
|
+
total: exactToken(usage?.totalTokens, `${label}.totalTokens`)
|
|
299
|
+
};
|
|
300
|
+
})() : {
|
|
219
301
|
input: numberValue(usage.input), output: numberValue(usage.output), cacheRead: numberValue(usage.cacheRead),
|
|
220
302
|
cacheWrite: numberValue(usage.cacheWrite), reasoning: numberValue(usage.reasoning),
|
|
221
303
|
total: numberValue(usage.totalTokens) || numberValue(usage.input) + numberValue(usage.output) + numberValue(usage.cacheRead) + numberValue(usage.cacheWrite)
|
|
@@ -229,6 +311,11 @@ function addTotals(target, usage, { strict = false, label = "usage" } = {}) {
|
|
|
229
311
|
target.cost += typeof cost === "object" && cost !== null
|
|
230
312
|
? numberValue(cost.total)
|
|
231
313
|
: numberValue(cost);
|
|
314
|
+
return {
|
|
315
|
+
reasoningReported,
|
|
316
|
+
reasoningCompleteness: reasoningReported || values.output === 0 ? "exact" : "lower-bound",
|
|
317
|
+
reasoningInferredZero: !reasoningReported && values.output === 0
|
|
318
|
+
};
|
|
232
319
|
}
|
|
233
320
|
|
|
234
321
|
function numberValue(raw) {
|
|
@@ -253,6 +340,7 @@ function summarizeSession(file, options) {
|
|
|
253
340
|
return undefined;
|
|
254
341
|
}
|
|
255
342
|
|
|
343
|
+
const execution = emptyExecutionAggregate();
|
|
256
344
|
const summary = {
|
|
257
345
|
id: path.basename(file, ".jsonl"),
|
|
258
346
|
name: "",
|
|
@@ -269,13 +357,29 @@ function summarizeSession(file, options) {
|
|
|
269
357
|
messages: { user: 0, assistant: 0, toolCalls: 0, toolResults: 0, total: 0 },
|
|
270
358
|
promptChars: 0,
|
|
271
359
|
tokens: emptyTotals(),
|
|
272
|
-
usageIntegrity: {
|
|
360
|
+
usageIntegrity: {
|
|
361
|
+
exact: options.strictUsage === true,
|
|
362
|
+
source: "pi-session-jsonl",
|
|
363
|
+
assistantMessages: 0,
|
|
364
|
+
usageMessages: 0,
|
|
365
|
+
reasoningMessages: 0,
|
|
366
|
+
reasoningInferredZeroMessages: 0,
|
|
367
|
+
reasoningLowerBoundMessages: 0,
|
|
368
|
+
reasoningCompleteness: "exact"
|
|
369
|
+
},
|
|
370
|
+
pricingBuckets: { schemaVersion: 1, source: "provider-request-usage", completeness: "unverified", requests: [] },
|
|
371
|
+
execution,
|
|
273
372
|
toolNames: {},
|
|
274
373
|
sizeBytes: stat.size,
|
|
275
374
|
mtime: stat.mtime
|
|
276
375
|
};
|
|
277
376
|
|
|
278
377
|
let hasCountedActivity = false;
|
|
378
|
+
let parsedRecordCount = 0;
|
|
379
|
+
let sessionHeaderCount = 0;
|
|
380
|
+
const toolCallsById = new Map();
|
|
381
|
+
const toolResultIds = new Set();
|
|
382
|
+
const toolCallFingerprints = new Set();
|
|
279
383
|
const lines = fs.readFileSync(file, "utf8").split(/\n/);
|
|
280
384
|
for (const [lineIndex, line] of lines.entries()) {
|
|
281
385
|
if (!line) continue;
|
|
@@ -287,6 +391,12 @@ function summarizeSession(file, options) {
|
|
|
287
391
|
continue;
|
|
288
392
|
}
|
|
289
393
|
|
|
394
|
+
const isFirstParsedRecord = parsedRecordCount === 0;
|
|
395
|
+
parsedRecordCount += 1;
|
|
396
|
+
if (options.strictUsage && isFirstParsedRecord && entry?.type !== "session") {
|
|
397
|
+
throw new Error("Pi session JSONL is missing its session header as the first record");
|
|
398
|
+
}
|
|
399
|
+
|
|
290
400
|
const ts = parseIso(entry.timestamp ?? entry.message?.timestamp);
|
|
291
401
|
if (ts) {
|
|
292
402
|
if (!summary.firstTimestamp || ts < summary.firstTimestamp) summary.firstTimestamp = ts;
|
|
@@ -294,6 +404,14 @@ function summarizeSession(file, options) {
|
|
|
294
404
|
}
|
|
295
405
|
|
|
296
406
|
if (entry.type === "session") {
|
|
407
|
+
sessionHeaderCount += 1;
|
|
408
|
+
if (options.strictUsage && (sessionHeaderCount !== 1 || !isFirstParsedRecord)) {
|
|
409
|
+
throw new Error("Pi session JSONL must contain exactly one session header as the first record");
|
|
410
|
+
}
|
|
411
|
+
if (options.strictUsage && (typeof entry.id !== "string" || !entry.id.trim()
|
|
412
|
+
|| typeof entry.cwd !== "string" || !entry.cwd.trim())) {
|
|
413
|
+
throw new Error("Pi session JSONL session header must contain non-empty id and cwd strings");
|
|
414
|
+
}
|
|
297
415
|
if (entry.id) summary.id = String(entry.id);
|
|
298
416
|
if (entry.cwd) summary.cwd = String(entry.cwd);
|
|
299
417
|
continue;
|
|
@@ -311,6 +429,13 @@ function summarizeSession(file, options) {
|
|
|
311
429
|
summary.name = String(entry.name);
|
|
312
430
|
continue;
|
|
313
431
|
}
|
|
432
|
+
|
|
433
|
+
const countedControl = inRange(ts ?? stat.mtime, options);
|
|
434
|
+
if (countedControl && entry.type === "auto_retry_start") execution.explicitRetries += 1;
|
|
435
|
+
else if (countedControl && entry.type === "auto_retry_end" && entry.success === false) execution.retryFailures += 1;
|
|
436
|
+
else if (countedControl && entry.type === "compaction_start") execution.compactions += 1;
|
|
437
|
+
else if (countedControl && entry.type === "compaction_end" && entry.aborted === true) execution.abortedCompactions += 1;
|
|
438
|
+
else if (countedControl && entry.type === "summarization_retry_scheduled") execution.summarizationRetries += 1;
|
|
314
439
|
if (entry.type !== "message" || !entry.message) continue;
|
|
315
440
|
|
|
316
441
|
const counted = inRange(ts ?? stat.mtime, options);
|
|
@@ -324,6 +449,10 @@ function summarizeSession(file, options) {
|
|
|
324
449
|
const role = message.role;
|
|
325
450
|
summary.messages.total += 1;
|
|
326
451
|
if (role === "user") {
|
|
452
|
+
// A repeated call means duplicate work inside one operator turn. The
|
|
453
|
+
// same verifier or read requested again by a later user turn is fresh
|
|
454
|
+
// work, not a retry loop, even when the session itself is preserved.
|
|
455
|
+
toolCallFingerprints.clear();
|
|
327
456
|
summary.messages.user += 1;
|
|
328
457
|
summary.promptChars += extractTextLength(message.content);
|
|
329
458
|
} else if (role === "assistant") {
|
|
@@ -335,21 +464,52 @@ function summarizeSession(file, options) {
|
|
|
335
464
|
throw new Error(`Pi assistant message at line ${lineIndex + 1} is missing usage`);
|
|
336
465
|
}
|
|
337
466
|
if (message.usage) {
|
|
338
|
-
addTotals(summary.tokens, message.usage, {
|
|
467
|
+
const usageReceipt = addTotals(summary.tokens, message.usage, {
|
|
468
|
+
strict: options.strictUsage === true,
|
|
469
|
+
label: `Pi session JSONL line ${lineIndex + 1} usage`
|
|
470
|
+
});
|
|
339
471
|
summary.usageIntegrity.usageMessages += 1;
|
|
472
|
+
if (usageReceipt.reasoningReported) summary.usageIntegrity.reasoningMessages += 1;
|
|
473
|
+
if (usageReceipt.reasoningInferredZero) summary.usageIntegrity.reasoningInferredZeroMessages += 1;
|
|
474
|
+
if (usageReceipt.reasoningCompleteness === "lower-bound") {
|
|
475
|
+
summary.usageIntegrity.reasoningLowerBoundMessages += 1;
|
|
476
|
+
summary.usageIntegrity.reasoningCompleteness = "lower-bound";
|
|
477
|
+
}
|
|
478
|
+
execution.providerStartedAttempts += 1;
|
|
479
|
+
const request = exactPricingRequest(message.usage);
|
|
480
|
+
if (request) summary.pricingBuckets.requests.push(request);
|
|
340
481
|
}
|
|
341
482
|
if (Array.isArray(message.content)) {
|
|
342
483
|
for (const block of message.content) {
|
|
343
484
|
if (block?.type !== "toolCall") continue;
|
|
344
485
|
summary.messages.toolCalls += 1;
|
|
486
|
+
execution.toolCalls += 1;
|
|
345
487
|
const name = String(block.name ?? "tool");
|
|
346
488
|
summary.toolNames[name] = (summary.toolNames[name] ?? 0) + 1;
|
|
489
|
+
const id = typeof block.id === "string" && block.id ? block.id : null;
|
|
490
|
+
const subagentAttempt = isSubagentAttempt(block);
|
|
491
|
+
if (id) toolCallsById.set(id, { name, subagentAttempt });
|
|
492
|
+
if (subagentAttempt) execution.subagentAttempts += 1;
|
|
493
|
+
const fingerprint = toolFingerprint(block);
|
|
494
|
+
if (toolCallFingerprints.has(fingerprint)) execution.repeatedToolCalls += 1;
|
|
495
|
+
else toolCallFingerprints.add(fingerprint);
|
|
347
496
|
}
|
|
348
497
|
}
|
|
349
498
|
} else if (role === "toolResult") {
|
|
350
499
|
summary.messages.toolResults += 1;
|
|
500
|
+
execution.toolResults += 1;
|
|
351
501
|
const name = String(message.toolName ?? "tool");
|
|
352
502
|
summary.toolNames[name] = summary.toolNames[name] ?? 0;
|
|
503
|
+
const status = structuredToolStatus(message);
|
|
504
|
+
const failed = message.isError === true || status === "failed" || status === "error";
|
|
505
|
+
if (failed) execution.toolFailures += 1;
|
|
506
|
+
if (status === "blocked") execution.blockedToolCalls += 1;
|
|
507
|
+
if (status === "declined") execution.declinedToolCalls += 1;
|
|
508
|
+
const call = typeof message.toolCallId === "string" ? toolCallsById.get(message.toolCallId) : undefined;
|
|
509
|
+
if (typeof message.toolCallId === "string") toolResultIds.add(message.toolCallId);
|
|
510
|
+
if (call?.subagentAttempt && (failed || status === "blocked" || status === "declined")) execution.subagentFailures += 1;
|
|
511
|
+
const compacted = message.details?.piagentCompactedToolResults;
|
|
512
|
+
if (Number.isSafeInteger(compacted) && compacted > 0) execution.compactions += compacted;
|
|
353
513
|
}
|
|
354
514
|
}
|
|
355
515
|
|
|
@@ -360,9 +520,21 @@ function summarizeSession(file, options) {
|
|
|
360
520
|
if (!summary.countedLastTimestamp) summary.countedLastTimestamp = summary.lastTimestamp;
|
|
361
521
|
|
|
362
522
|
if (!hasCountedActivity && (options.since || options.until)) return undefined;
|
|
523
|
+
if (options.strictUsage && sessionHeaderCount !== 1) {
|
|
524
|
+
throw new Error("Pi session JSONL is missing its session header");
|
|
525
|
+
}
|
|
363
526
|
if (options.strictUsage && summary.usageIntegrity.assistantMessages !== summary.usageIntegrity.usageMessages) {
|
|
364
527
|
throw new Error("Pi session JSONL usage coverage is incomplete");
|
|
365
528
|
}
|
|
529
|
+
summary.pricingBuckets.completeness = summary.usageIntegrity.assistantMessages > 0
|
|
530
|
+
&& summary.pricingBuckets.requests.length === summary.usageIntegrity.assistantMessages
|
|
531
|
+
? "exact"
|
|
532
|
+
: "unverified";
|
|
533
|
+
const identifiedToolResults = [...toolCallsById.keys()].filter((id) => toolResultIds.has(id)).length;
|
|
534
|
+
execution.completeness.tools = execution.toolCalls === execution.toolResults && identifiedToolResults === execution.toolCalls
|
|
535
|
+
? "exact"
|
|
536
|
+
: "partial";
|
|
537
|
+
execution.completeness.subagents = execution.completeness.tools;
|
|
366
538
|
return summary;
|
|
367
539
|
}
|
|
368
540
|
|
|
@@ -388,7 +560,9 @@ function buildReport(options, sessions) {
|
|
|
388
560
|
subagentSessions: sessions.filter((s) => s.isSubagent).length,
|
|
389
561
|
messages: { user: 0, assistant: 0, toolCalls: 0, toolResults: 0, total: 0 },
|
|
390
562
|
promptChars: 0,
|
|
391
|
-
tokens: emptyTotals()
|
|
563
|
+
tokens: emptyTotals(),
|
|
564
|
+
execution: emptyExecutionAggregate(),
|
|
565
|
+
pricingBuckets: { schemaVersion: 1, source: "provider-request-usage", completeness: "exact", requests: [] }
|
|
392
566
|
};
|
|
393
567
|
const projects = new Map();
|
|
394
568
|
const tools = {};
|
|
@@ -396,6 +570,12 @@ function buildReport(options, sessions) {
|
|
|
396
570
|
for (const key of Object.keys(totals.messages)) totals.messages[key] += session.messages[key];
|
|
397
571
|
totals.promptChars += session.promptChars;
|
|
398
572
|
for (const key of Object.keys(totals.tokens)) totals.tokens[key] += session.tokens[key];
|
|
573
|
+
for (const key of ["providerStartedAttempts", "toolCalls", "toolResults", "toolFailures", "blockedToolCalls",
|
|
574
|
+
"declinedToolCalls", "explicitRetries", "retryFailures", "compactions", "abortedCompactions",
|
|
575
|
+
"summarizationRetries", "repeatedToolCalls", "subagentAttempts", "subagentFailures"]) {
|
|
576
|
+
totals.execution[key] += Number(session.execution?.[key] ?? 0);
|
|
577
|
+
}
|
|
578
|
+
totals.pricingBuckets.requests.push(...(session.pricingBuckets?.requests ?? []));
|
|
399
579
|
const project = projects.get(session.cwd) ?? {
|
|
400
580
|
cwd: session.cwd,
|
|
401
581
|
sessions: 0,
|
|
@@ -420,6 +600,16 @@ function buildReport(options, sessions) {
|
|
|
420
600
|
.map(([name, count]) => ({ name, count }))
|
|
421
601
|
.sort((a, b) => b.count - a.count || a.name.localeCompare(b.name));
|
|
422
602
|
|
|
603
|
+
const completenessRank = { exact: 0, partial: 1, unverified: 2, unavailable: 3 };
|
|
604
|
+
for (const field of ["tools", "retries", "compactions", "subagents"]) {
|
|
605
|
+
totals.execution.completeness[field] = sessions.reduce((worst, session) => {
|
|
606
|
+
const value = session.execution?.completeness?.[field] ?? "unavailable";
|
|
607
|
+
return completenessRank[value] > completenessRank[worst] ? value : worst;
|
|
608
|
+
}, "exact");
|
|
609
|
+
}
|
|
610
|
+
totals.pricingBuckets.completeness = sessions.length > 0
|
|
611
|
+
&& sessions.every((session) => session.pricingBuckets?.completeness === "exact") ? "exact" : "unverified";
|
|
612
|
+
|
|
423
613
|
return {
|
|
424
614
|
generatedAt: new Date().toISOString(),
|
|
425
615
|
scope: {
|
package/scripts/piagent-cli.mjs
CHANGED
|
@@ -42,6 +42,12 @@ if (invokedAs === "piagent") {
|
|
|
42
42
|
} else if (subcommand === "explain") {
|
|
43
43
|
script = "scripts/explain-command.mjs";
|
|
44
44
|
forwardedArgs = forwardedArgs.slice(1);
|
|
45
|
+
} else if (subcommand === "approve-verification") {
|
|
46
|
+
script = "scripts/approve-independent-verification.mjs";
|
|
47
|
+
forwardedArgs = forwardedArgs.slice(1);
|
|
48
|
+
} else if (subcommand === "select-verification") {
|
|
49
|
+
script = "scripts/select-independent-verification.mjs";
|
|
50
|
+
forwardedArgs = forwardedArgs.slice(1);
|
|
45
51
|
} else if ([undefined, "help", "--help", "-h"].includes(subcommand)) {
|
|
46
52
|
// Subcommands are listed, their flags are not. Restating them here meant a
|
|
47
53
|
// second copy that drifts: this line still advertised the dashboard without
|
|
@@ -51,6 +57,8 @@ if (invokedAs === "piagent") {
|
|
|
51
57
|
console.log("Commands:");
|
|
52
58
|
console.log(" dashboard Open and manage the local session hub");
|
|
53
59
|
console.log(" explain Say why the guard would allow or block a shell command");
|
|
60
|
+
console.log(" approve-verification Preview or approve an independent verification plan");
|
|
61
|
+
console.log(" select-verification Preview reusable contracts for exact task criteria");
|
|
54
62
|
console.log("");
|
|
55
63
|
console.log("Run `piagent <command> --help` for that command's options.");
|
|
56
64
|
process.exit(0);
|
|
@@ -5,6 +5,7 @@ import path from "node:path";
|
|
|
5
5
|
import { fileURLToPath } from "node:url";
|
|
6
6
|
|
|
7
7
|
import { benchmarkAssuranceEvidenceValidationErrors } from "../packages/piagent-core/benchmark/benchmark-assurance.js";
|
|
8
|
+
import { verifyPublicExposure } from "./public-evaluation-exposure.mjs";
|
|
8
9
|
|
|
9
10
|
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
|
10
11
|
const usage = "Usage: node scripts/private-holdout-readiness.mjs --evidence /secure/path/assurance.json";
|
|
@@ -46,11 +47,15 @@ const errors = benchmarkAssuranceEvidenceValidationErrors(evidence);
|
|
|
46
47
|
if (errors.length > 0) refuse(errors.join("; "));
|
|
47
48
|
if (evidence.schemaVersion !== 2) refuse("legacy assurance evidence is historical-only and cannot establish E3 readiness");
|
|
48
49
|
|
|
50
|
+
let publicExposure;
|
|
51
|
+
try { publicExposure = verifyPublicExposure(root); }
|
|
52
|
+
catch { refuse("current public exposure inventory is unavailable, stale or incomplete"); }
|
|
53
|
+
|
|
49
54
|
const bindings = {
|
|
50
55
|
accessPolicyDigest: digestFile("evals/private-holdout-v1/access-policy.v1.json"),
|
|
51
56
|
humanRubricDigest: digestFile("evals/private-holdout-v1/human-rubric.v1.json"),
|
|
52
57
|
taxonomyDigest: digestFile("evals/real-task-taxonomy.v1.json"),
|
|
53
|
-
publicExposureDigest:
|
|
58
|
+
publicExposureDigest: publicExposure.sha256
|
|
54
59
|
};
|
|
55
60
|
for (const [field, expected] of Object.entries(bindings)) {
|
|
56
61
|
const actual = field in evidence ? evidence[field] : evidence.disjointness?.[field];
|
|
@@ -64,6 +69,7 @@ const receipt = {
|
|
|
64
69
|
protocolVersion: "e3-custody-v1",
|
|
65
70
|
ready: true,
|
|
66
71
|
evidenceManifestDigest: crypto.createHash("sha256").update(buffer).digest("hex"),
|
|
72
|
+
publicExposure,
|
|
67
73
|
claimTier: evidence.claimTier,
|
|
68
74
|
visibility: evidence.visibility,
|
|
69
75
|
access: {
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import crypto from "node:crypto";
|
|
3
|
+
import fs from "node:fs";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
import { isDeepStrictEqual } from "node:util";
|
|
6
|
+
import { fileURLToPath } from "node:url";
|
|
7
|
+
|
|
8
|
+
export const EXPOSURE_PATH = "evals/private-holdout-v1/public-exposure.v2.json";
|
|
9
|
+
const PREVIOUS_PATH = "evals/private-holdout-v1/public-exposure.v1.json";
|
|
10
|
+
const TAXONOMY_PATH = "evals/real-task-taxonomy.v1.json";
|
|
11
|
+
const PUBLIC_TREES = [
|
|
12
|
+
"adapters", "benchmarks", "evals/architecture-conformance-v1", "evals/golden",
|
|
13
|
+
"evals/harness-next", "evals/long-horizon-v1", "evals/runtime-conformance-v1",
|
|
14
|
+
"evals/scenarios", "tests"
|
|
15
|
+
];
|
|
16
|
+
const sha256 = (bytes) => crypto.createHash("sha256").update(bytes).digest("hex");
|
|
17
|
+
const MAX_FILES = 10_000, MAX_FILE_BYTES = 4 * 1024 * 1024, MAX_TOTAL_BYTES = 64 * 1024 * 1024;
|
|
18
|
+
const DEFAULT_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
|
19
|
+
|
|
20
|
+
function reader(root) {
|
|
21
|
+
let total = 0, count = 0;
|
|
22
|
+
const cache = new Map();
|
|
23
|
+
function checked(relative) {
|
|
24
|
+
let current = root;
|
|
25
|
+
for (const part of relative.split("/")) {
|
|
26
|
+
if (!part || part === "." || part === ".." || part.includes("\\")) throw new Error("Invalid public exposure path");
|
|
27
|
+
current = path.join(current, part);
|
|
28
|
+
if (fs.lstatSync(current).isSymbolicLink()) throw new Error("Public exposure inputs must not be symlinks");
|
|
29
|
+
}
|
|
30
|
+
return current;
|
|
31
|
+
}
|
|
32
|
+
function bytes(relative) {
|
|
33
|
+
if (cache.has(relative)) return cache.get(relative);
|
|
34
|
+
const target = checked(relative), descriptor = fs.openSync(target, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW);
|
|
35
|
+
let value;
|
|
36
|
+
try {
|
|
37
|
+
const stat = fs.fstatSync(descriptor);
|
|
38
|
+
if (!stat.isFile() || stat.size > MAX_FILE_BYTES) throw new Error("Public exposure input is not a bounded regular file");
|
|
39
|
+
value = fs.readFileSync(descriptor);
|
|
40
|
+
} finally { fs.closeSync(descriptor); }
|
|
41
|
+
total += value.length;
|
|
42
|
+
if (value.length > MAX_FILE_BYTES || ++count > MAX_FILES || total > MAX_TOTAL_BYTES) throw new Error("Public exposure inventory exceeds its limits");
|
|
43
|
+
cache.set(relative, value);
|
|
44
|
+
return value;
|
|
45
|
+
}
|
|
46
|
+
function files(relative) {
|
|
47
|
+
const target = checked(relative);
|
|
48
|
+
if (!fs.statSync(target).isDirectory()) throw new Error("Public exposure tree is not a directory");
|
|
49
|
+
const result = [];
|
|
50
|
+
for (const name of fs.readdirSync(target).sort()) {
|
|
51
|
+
const child = `${relative}/${name}`, stat = fs.lstatSync(checked(child));
|
|
52
|
+
if (stat.isDirectory()) result.push(...files(child));
|
|
53
|
+
else if (stat.isFile()) { bytes(child); result.push(child); }
|
|
54
|
+
else throw new Error("Public exposure tree contains a non-regular input");
|
|
55
|
+
}
|
|
56
|
+
return result;
|
|
57
|
+
}
|
|
58
|
+
return { bytes, files, json: (relative) => JSON.parse(bytes(relative).toString("utf8")) };
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
// Inventory public files only. Never load a private suite or execute an oracle.
|
|
62
|
+
// The predecessor is retained rather than silently erasing earlier exposure.
|
|
63
|
+
export function buildPublicExposure(root = DEFAULT_ROOT) {
|
|
64
|
+
const read = reader(root), allFiles = new Map();
|
|
65
|
+
const visibleTrees = PUBLIC_TREES.map((relative) => {
|
|
66
|
+
const entries = read.files(relative).sort().map((file) => {
|
|
67
|
+
const bytes = read.bytes(file), digest = sha256(bytes);
|
|
68
|
+
allFiles.set(file, bytes);
|
|
69
|
+
return [file, digest];
|
|
70
|
+
});
|
|
71
|
+
return { path: relative, fileCount: entries.length, sha256: sha256(JSON.stringify(entries)) };
|
|
72
|
+
});
|
|
73
|
+
const visibleSuites = [...allFiles.keys()].filter((file) => /^benchmarks\/[^/]+\/suite\.json$/.test(file)).map((file) => {
|
|
74
|
+
const suite = read.json(file);
|
|
75
|
+
if (!/^[a-z0-9-]+$/.test(suite.id ?? "") || file !== `benchmarks/${suite.id}/suite.json`
|
|
76
|
+
|| !Array.isArray(suite.scenarios) || suite.scenarios.length === 0) throw new Error("Invalid public suite identity");
|
|
77
|
+
const scenarioIds = suite.scenarios.map((scenario) => scenario.id);
|
|
78
|
+
if (scenarioIds.some((id) => !/^[a-z0-9-]+$/.test(id ?? "")) || new Set(scenarioIds).size !== scenarioIds.length) throw new Error("Invalid public scenario identities");
|
|
79
|
+
return { id: suite.id, path: file, manifestSha256: sha256(read.bytes(file)), scenarioIds };
|
|
80
|
+
});
|
|
81
|
+
const contractLibraries = [...allFiles.keys()].filter((file) => /^adapters\/[^/]+\/contract-families\.json$/.test(file)).map((file) => {
|
|
82
|
+
const library = read.json(file);
|
|
83
|
+
if (library.schemaVersion !== 1 || !Array.isArray(library.families)) throw new Error("Invalid public contract library");
|
|
84
|
+
const families = library.families.map(({ id, version }) => {
|
|
85
|
+
if (!/^[a-z0-9-]+$/.test(id ?? "") || !Number.isSafeInteger(version) || version < 1) throw new Error("Invalid public contract family identity");
|
|
86
|
+
return { id, version };
|
|
87
|
+
});
|
|
88
|
+
if (new Set(families.map(({ id, version }) => `${id}@${version}`)).size !== families.length) throw new Error("Duplicate public contract family identity");
|
|
89
|
+
return { path: file, sha256: sha256(read.bytes(file)), families };
|
|
90
|
+
});
|
|
91
|
+
return {
|
|
92
|
+
schemaVersion: 2,
|
|
93
|
+
id: "author-visible-evaluation-exposure-v2",
|
|
94
|
+
previousExposure: { path: PREVIOUS_PATH, sha256: sha256(read.bytes(PREVIOUS_PATH)) },
|
|
95
|
+
taxonomy: { path: TAXONOMY_PATH, sha256: sha256(read.bytes(TAXONOMY_PATH)) },
|
|
96
|
+
visibleSuites,
|
|
97
|
+
contractLibraries,
|
|
98
|
+
visibleTrees,
|
|
99
|
+
custodianComparisonRules: {
|
|
100
|
+
familyKey: "Compare private task and acceptance-mechanism lineage against every listed suite, contract family, public tree and predecessor exposure; renamed, generated or structurally varied descendants are not disjoint.",
|
|
101
|
+
repositoryKey: "Compare repository history, origin, package graph and fixture lineage, not names alone.",
|
|
102
|
+
minimumPrivateScenarios: 6, minimumPrivateFamilies: 6, minimumPrivateRepositories: 6,
|
|
103
|
+
externalExposure: "The custodian must independently account for author-visible material outside these public roots, including prior local diagnostics and retained campaigns; this inventory is not a complete record of author knowledge.",
|
|
104
|
+
publicOutput: "Export only the closed schema-v2 assurance receipt and independent custody attestation."
|
|
105
|
+
},
|
|
106
|
+
claimBoundary: "Current public-input inventory only, not independent evaluation, approval, generalization or release evidence. Public-tree digests bind [relative path, file SHA-256] pairs globally sorted by the slash-separated path's UTF-16 code units, serialized as compact JSON and hashed as UTF-8; no private artifacts are loaded."
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
export function verifyPublicExposure(root = DEFAULT_ROOT) {
|
|
111
|
+
const read = reader(root), bytes = read.bytes(EXPOSURE_PATH), actual = JSON.parse(bytes.toString("utf8"));
|
|
112
|
+
if (!isDeepStrictEqual(actual, buildPublicExposure(root))) throw new Error("Current public exposure inventory is stale or incomplete; regenerate and obtain a new independent custody receipt");
|
|
113
|
+
return { path: EXPOSURE_PATH, sha256: sha256(bytes), suiteCount: actual.visibleSuites.length,
|
|
114
|
+
scenarioCount: actual.visibleSuites.reduce((sum, suite) => sum + suite.scenarioIds.length, 0),
|
|
115
|
+
contractFamilyVersionCount: actual.contractLibraries.reduce((sum, library) => sum + library.families.length, 0),
|
|
116
|
+
publicTreeCount: actual.visibleTrees.length };
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
// Node resolves module URLs through filesystem aliases. Compare actual files,
|
|
120
|
+
// otherwise /var versus /private/var (or a global-bin symlink) can skip the CLI.
|
|
121
|
+
const invokedDirectly = process.argv[1] && fs.existsSync(process.argv[1])
|
|
122
|
+
&& fs.realpathSync(process.argv[1]) === fs.realpathSync(fileURLToPath(import.meta.url));
|
|
123
|
+
if (invokedDirectly) {
|
|
124
|
+
if (process.argv.length === 3 && ["--help", "-h"].includes(process.argv[2])) {
|
|
125
|
+
process.stdout.write("Usage: node scripts/public-evaluation-exposure.mjs --check|--write\nRequires the complete public source tree, including development tests. Generation does not grant custody or provider authority.\n");
|
|
126
|
+
process.exit(0);
|
|
127
|
+
}
|
|
128
|
+
try {
|
|
129
|
+
if (process.argv.length !== 3 || !["--check", "--write"].includes(process.argv[2])) throw new Error("Usage: node scripts/public-evaluation-exposure.mjs --check|--write");
|
|
130
|
+
if (process.argv[2] === "--write") {
|
|
131
|
+
const destination = path.join(DEFAULT_ROOT, EXPOSURE_PATH);
|
|
132
|
+
if (fs.existsSync(destination) && fs.lstatSync(destination).isSymbolicLink()) throw new Error("Refusing a symlinked public exposure destination");
|
|
133
|
+
fs.writeFileSync(destination, `${JSON.stringify(buildPublicExposure(), null, 2)}\n`);
|
|
134
|
+
}
|
|
135
|
+
process.stdout.write(`${JSON.stringify(verifyPublicExposure())}\n`);
|
|
136
|
+
} catch {
|
|
137
|
+
process.stderr.write("REFUSED: current public exposure inventory is unavailable, stale or incomplete; inspect public inputs and regenerate before independent custody review\n");
|
|
138
|
+
process.exitCode = 1;
|
|
139
|
+
}
|
|
140
|
+
}
|