@piagent/platform 1.5.5 → 1.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +166 -0
- package/README.md +25 -21
- package/README.vi.md +34 -5
- package/adapters/web-frontend/profile.json +2 -4
- package/architecture/layers.json +18 -3
- package/benchmarks/deep-logic-v1/SPECIALIST_PROTOCOL.vi.md +122 -0
- package/benchmarks/deep-logic-v1/grade.mjs +272 -10
- package/benchmarks/deep-logic-v1/project/apps/admin/src/billing-summary.js +3 -0
- package/benchmarks/deep-logic-v1/project/benchmark-contract/public-smoke.test.js +166 -0
- package/benchmarks/deep-logic-v1/project/package.json +1 -1
- package/benchmarks/deep-logic-v1/project/packages/billing/src/close-period.js +3 -0
- package/benchmarks/deep-logic-v1/project/packages/billing/src/plan-timeline.js +3 -0
- package/benchmarks/deep-logic-v1/project/test/smoke.test.js +7 -1
- package/benchmarks/deep-logic-v1/prompts/budgeted-context-graph.md +2 -0
- package/benchmarks/deep-logic-v1/prompts/fair-dependency-scheduler.md +2 -0
- package/benchmarks/deep-logic-v1/prompts/layered-policy-resolution.md +2 -0
- package/benchmarks/deep-logic-v1/prompts/resumable-stream-assembly.md +3 -1
- package/benchmarks/deep-logic-v1/prompts/revision-event-reconciliation.md +2 -0
- package/benchmarks/deep-logic-v1/prompts/temporal-usage-billing-close.md +123 -0
- package/benchmarks/deep-logic-v1/prompts/transactional-config-merge.md +2 -0
- package/benchmarks/deep-logic-v1/suite.json +51 -3
- package/benchmarks/deep-logic-v1/variant.mjs +22 -1
- package/benchmarks/production-v1/spend-control.v1.json +92 -0
- package/benchmarks/production-v1/suite.json +46 -3
- package/catalog/capabilities.json +18 -18
- package/docs/authority-manifest.md +1 -1
- package/docs/auto-delegation-policy.md +38 -73
- package/docs/command-reference-vietnamese.md +27 -35
- package/docs/context-engine.md +12 -4
- package/docs/context-window-policy.md +37 -1
- package/docs/distribution-standard.md +7 -7
- package/docs/en/architecture.md +27 -13
- package/docs/languages.json +4 -4
- package/docs/model-options.md +8 -5
- package/docs/operator-manual-vietnamese.md +27 -45
- package/docs/project-adapters.md +2 -2
- package/docs/publishing-for-teams.md +10 -0
- package/docs/quality-benchmark.md +427 -62
- package/docs/release-install-policy.md +10 -9
- package/docs/runtime-harness-standard.md +3 -3
- package/docs/runtime-policy-design.md +4 -4
- package/docs/runtime-quality-baseline.md +13 -6
- package/docs/security-threat-model.md +2 -2
- package/docs/subagent-orchestration-capabilities.md +19 -51
- package/docs/subagents-and-multiagent.md +21 -47
- package/docs/task-implementation-contract.md +3 -3
- package/docs/task-lifecycle-vietnamese.md +3 -3
- package/docs/team-onboarding.md +2 -2
- package/docs/usage-observability.md +7 -2
- package/docs/vi/architecture.md +27 -13
- package/docs-site/architecture.html +1 -1
- package/docs-site/benchmark.html +89 -95
- package/docs-site/commands.html +1 -1
- package/docs-site/content/benchmark.html +88 -94
- package/docs-site/content/en/benchmark.html +134 -90
- package/docs-site/content/en/index.html +19 -19
- package/docs-site/content/en/runtime.html +4 -4
- package/docs-site/content/en/whats-new.html +12 -1
- package/docs-site/content/en/workflows.html +2 -2
- package/docs-site/content/index.html +19 -19
- package/docs-site/content/runtime.html +4 -4
- package/docs-site/content/whats-new.html +12 -1
- package/docs-site/content/workflows.html +2 -2
- package/docs-site/context-engine.html +1 -1
- package/docs-site/en/architecture.html +1 -1
- package/docs-site/en/benchmark.html +135 -91
- package/docs-site/en/commands.html +1 -1
- package/docs-site/en/context-engine.html +1 -1
- package/docs-site/en/index.html +20 -20
- package/docs-site/en/mcp-approval.html +1 -1
- package/docs-site/en/mcp-auth.html +1 -1
- package/docs-site/en/mcp-servers.html +1 -1
- package/docs-site/en/mcp.html +1 -1
- package/docs-site/en/permissions.html +1 -1
- package/docs-site/en/profiles.html +1 -1
- package/docs-site/en/quickstart.html +1 -1
- package/docs-site/en/runtime.html +5 -5
- package/docs-site/en/scope.html +1 -1
- package/docs-site/en/security.html +1 -1
- package/docs-site/en/team.html +1 -1
- package/docs-site/en/whats-new.html +13 -2
- package/docs-site/en/workflows.html +3 -3
- package/docs-site/index.html +20 -20
- package/docs-site/mcp-approval.html +1 -1
- package/docs-site/mcp-auth.html +1 -1
- package/docs-site/mcp-servers.html +1 -1
- package/docs-site/mcp.html +1 -1
- package/docs-site/permissions.html +1 -1
- package/docs-site/profiles.html +1 -1
- package/docs-site/quickstart.html +1 -1
- package/docs-site/runtime.html +5 -5
- package/docs-site/scope.html +1 -1
- package/docs-site/security.html +1 -1
- package/docs-site/team.html +1 -1
- package/docs-site/whats-new.html +13 -2
- package/docs-site/workflows.html +3 -3
- package/evals/fixtures/benchmark-assurance-evidence.invalid.json +1 -1
- package/evals/fixtures/benchmark-assurance-evidence.valid.json +1 -1
- package/evals/fixtures/codex-relative-efficiency-v1.invalid.json +4 -0
- package/evals/fixtures/codex-relative-efficiency-v1.valid.json +94 -0
- package/evals/fixtures/helper-request.invalid.json +1 -1
- package/evals/fixtures/helper-request.valid.json +1 -1
- package/evals/fixtures/piagent-webui/session-live-state-v1.invalid.json +18 -0
- package/evals/fixtures/piagent-webui/session-live-state-v1.valid.json +27 -0
- package/evals/fixtures/piagent-webui/subagent-tree-v1.valid.json +7 -0
- package/evals/fixtures/task-authority-snapshot.valid.json +4 -4
- package/evals/fs4-readiness-matrix.v1.json +2 -2
- package/evals/fs5-causal-arm.v1.json +3 -3
- package/evals/fs5-pilot-protocol.v1.json +3 -3
- package/evals/fs5-pilot-protocol.v2.json +8 -8
- package/evals/fs5-pilot-protocol.v3.json +8 -8
- package/evals/fs5-pilot-protocol.v4.json +7 -6
- package/evals/fs5-pilot-protocol.v5.json +9 -7
- package/evals/private-holdout-v1/public-exposure.v1.json +1 -1
- package/evals/runtime-conformance-v1/README.md +43 -0
- package/evals/runtime-conformance-v1/evaluate-paid-canary.mjs +198 -0
- package/evals/runtime-conformance-v1/lane.json +57 -0
- package/evals/runtime-conformance-v1/paid-canary.json +31 -0
- package/evals/runtime-conformance-v1/runner.mjs +456 -0
- package/package.json +3 -1
- package/packages/piagent-core/README.md +13 -4
- package/packages/piagent-core/benchmark/benchmark-assurance.js +4 -1
- package/packages/piagent-core/benchmark/benchmark-bootstrap.js +91 -9
- package/packages/piagent-core/benchmark/benchmark-claim-restrictions.js +38 -7
- package/packages/piagent-core/benchmark/benchmark-cli.js +7 -7
- package/packages/piagent-core/benchmark/benchmark-codex-relative-efficiency.js +698 -0
- package/packages/piagent-core/benchmark/benchmark-codex.js +3 -1
- package/packages/piagent-core/benchmark/benchmark-comparison.js +271 -2
- package/packages/piagent-core/benchmark/benchmark-core.js +736 -664
- package/packages/piagent-core/benchmark/benchmark-execution-guard.js +7 -5
- package/packages/piagent-core/benchmark/benchmark-forensics.js +441 -4
- package/packages/piagent-core/benchmark/benchmark-host-readiness.js +639 -0
- package/packages/piagent-core/benchmark/benchmark-matrix.js +22 -4
- package/packages/piagent-core/benchmark/benchmark-normalized-cost.js +317 -0
- package/packages/piagent-core/benchmark/benchmark-preflight.js +5 -1
- package/packages/piagent-core/benchmark/benchmark-process.js +6 -5
- package/packages/piagent-core/benchmark/benchmark-provider-free-evidence.js +345 -0
- package/packages/piagent-core/benchmark/benchmark-provider-wire.js +126 -0
- package/packages/piagent-core/benchmark/benchmark-record-validation.js +252 -0
- package/packages/piagent-core/benchmark/benchmark-report.js +130 -12
- package/packages/piagent-core/benchmark/benchmark-resume-recovery.js +14 -0
- package/packages/piagent-core/benchmark/benchmark-runner-policy.js +26 -0
- package/packages/piagent-core/benchmark/benchmark-runtime-causal.js +244 -0
- package/packages/piagent-core/benchmark/benchmark-stage-diagnostic.js +979 -0
- package/packages/piagent-core/benchmark/benchmark-statistics.js +64 -0
- package/packages/piagent-core/benchmark/benchmark-stop-policy.js +8 -5
- package/packages/piagent-core/benchmark/benchmark-suite-identity.js +31 -0
- package/packages/piagent-core/benchmark/benchmark-suite-runtime.js +13 -9
- package/packages/piagent-core/benchmark/benchmark-suite.js +149 -2
- package/packages/piagent-core/benchmark/benchmark-summary-support.js +380 -0
- package/packages/piagent-core/benchmark/benchmark-timing-diagnostics.js +971 -0
- package/packages/piagent-core/benchmark/benchmark-usage.js +322 -0
- package/packages/piagent-core/benchmark/benchmark-workflow.js +166 -0
- package/packages/piagent-core/capabilities/runtime-integrity.js +3 -0
- package/packages/piagent-core/docs/context-efficiency-improvement-plan.md +1080 -0
- package/packages/piagent-core/extensions/acceptance-behavior-proof.js +806 -0
- package/packages/piagent-core/extensions/acceptance-contract-semantics.js +83 -69
- package/packages/piagent-core/extensions/acceptance-error-classes.js +160 -0
- package/packages/piagent-core/extensions/acceptance-executable-evidence.js +12 -9
- package/packages/piagent-core/extensions/acceptance-receipt.js +123 -59
- package/packages/piagent-core/extensions/acceptance-state-machine-evidence.js +518 -0
- package/packages/piagent-core/extensions/acceptance-test-binding-evidence.js +60 -0
- package/packages/piagent-core/extensions/context-efficiency-metrics.js +302 -0
- package/packages/piagent-core/extensions/context-efficiency-report.js +167 -0
- package/packages/piagent-core/extensions/context-engine.js +149 -156
- package/packages/piagent-core/extensions/context-evidence.js +48 -0
- package/packages/piagent-core/extensions/context-import-links.js +296 -0
- package/packages/piagent-core/extensions/context-index-policy.js +4 -0
- package/packages/piagent-core/extensions/context-query-paths.js +57 -0
- package/packages/piagent-core/extensions/context-telemetry-integrity.js +65 -0
- package/packages/piagent-core/extensions/criterion-context-pack.js +435 -7
- package/packages/piagent-core/extensions/criterion-graph.js +82 -9
- package/packages/piagent-core/extensions/fallback-base-policy.ts +101 -0
- package/packages/piagent-core/extensions/guard-types.ts +6 -0
- package/packages/piagent-core/extensions/javascript-regex-evidence.js +32 -0
- package/packages/piagent-core/extensions/piagent-guard.ts +420 -260
- package/packages/piagent-core/extensions/task-contract-view.js +17 -3
- package/packages/piagent-core/extensions/task-lifecycle.js +39 -2
- package/packages/piagent-core/extensions/task-state.js +109 -142
- package/packages/piagent-core/extensions/verification-intelligence.js +52 -13
- package/packages/piagent-core/extensions/workspace-evidence-roots.js +193 -0
- package/packages/piagent-core/package.json +1 -1
- package/packages/piagent-core/policies/base-policy.json +44 -2
- package/packages/piagent-core/policy/authority-manifest.v1.json +3 -3
- package/packages/piagent-core/prompts/be-to-fe.md +2 -8
- package/packages/piagent-core/prompts/plan.md +1 -1
- package/packages/piagent-core/prompts/platform-improve.md +2 -6
- package/packages/piagent-core/prompts/review.md +2 -7
- package/packages/piagent-core/prompts/scout.md +1 -1
- package/packages/piagent-core/prompts/task.md +4 -3
- package/packages/piagent-core/runtime/context/context-delivery.ts +103 -0
- package/packages/piagent-core/runtime/context/context-delta-shadow.ts +53 -0
- package/packages/piagent-core/runtime/context/context-efficiency-format.ts +47 -0
- package/packages/piagent-core/runtime/context/context-evidence-qualification.ts +32 -0
- package/packages/piagent-core/runtime/context/prefix-telemetry.ts +47 -0
- package/packages/piagent-core/runtime/context/repository-memory-hints.ts +27 -0
- package/packages/piagent-core/runtime/hooks/agent-prompt-telemetry.ts +56 -0
- package/packages/piagent-core/runtime/hooks/agent-start-hook.ts +162 -102
- package/packages/piagent-core/runtime/hooks/completion-hook.ts +12 -12
- package/packages/piagent-core/runtime/hooks/input-hook.ts +17 -1
- package/packages/piagent-core/runtime/hooks/session-hooks.ts +22 -2
- package/packages/piagent-core/runtime/hooks/session-start-hook.ts +107 -3
- package/packages/piagent-core/runtime/hooks/tool-result-content-guards.ts +95 -0
- package/packages/piagent-core/runtime/hooks/tool-result-hook.ts +134 -166
- package/packages/piagent-core/runtime/hooks/tool-result-value-helpers.ts +66 -0
- package/packages/piagent-core/runtime/inspection/activity-event-adapter.ts +4 -2
- package/packages/piagent-core/runtime/inspection/activity-recovery.ts +104 -0
- package/packages/piagent-core/runtime/inspection/bounded-emission-guard.ts +286 -0
- package/packages/piagent-core/runtime/inspection/runtime-event-store.ts +37 -7
- package/packages/piagent-core/runtime/inspection/task-compaction-history.ts +1 -1
- package/packages/piagent-core/runtime/inspection/task-handoff-history.ts +1 -1
- package/packages/piagent-core/runtime/inspection/task-subagent-tree.ts +27 -4
- package/packages/piagent-core/runtime/inspection/tool-failure-classification.ts +70 -0
- package/packages/piagent-core/runtime/inspection/webui-snapshot.ts +73 -9
- package/packages/piagent-core/runtime/model/authenticated-catalog.ts +17 -9
- package/packages/piagent-core/runtime/model/openai-codex-reasoning.ts +83 -0
- package/packages/piagent-core/runtime/model/provider-wire-fingerprint.ts +238 -0
- package/packages/piagent-core/runtime/orchestration/helper-lifecycle.ts +16 -12
- package/packages/piagent-core/runtime/orchestration/minimal-delegation-policy.ts +71 -0
- package/packages/piagent-core/runtime/orchestration/owned-work-budget.ts +28 -20
- package/packages/piagent-core/runtime/orchestration/role-policy.ts +27 -9
- package/packages/piagent-core/runtime/orchestration/subagent-tool-policy.ts +329 -0
- package/packages/piagent-core/runtime/policy/block-remedy.ts +0 -4
- package/packages/piagent-core/runtime/product/activity-inspector.ts +7 -2
- package/packages/piagent-core/runtime/product/efficiency-metrics.ts +7 -2
- package/packages/piagent-core/runtime/quality/edit-freshness-guard.ts +102 -0
- package/packages/piagent-core/runtime/quality/exact-output-contract.ts +22 -11
- package/packages/piagent-core/runtime/quality/performance-assurance-signals.ts +23 -0
- package/packages/piagent-core/runtime/quality/performance-assurance.ts +9 -5
- package/packages/piagent-core/runtime/recovery/continuation-budget.ts +1 -1
- package/packages/piagent-core/runtime/recovery/edit-recovery-context.ts +113 -0
- package/packages/piagent-core/runtime/recovery/edit-recovery-delivery.ts +42 -0
- package/packages/piagent-core/runtime/recovery/handoff-projection.ts +4 -1
- package/packages/piagent-core/runtime/recovery/pristine-task-policy-refresh.ts +91 -0
- package/packages/piagent-core/runtime/recovery/recovery-policy.ts +7 -4
- package/packages/piagent-core/runtime/recovery/resume-state.ts +112 -14
- package/packages/piagent-core/runtime/recovery/semantic-repair-handshake.ts +60 -13
- package/packages/piagent-core/runtime/recovery/semantic-repair-runtime.ts +55 -2
- package/packages/piagent-core/runtime/recovery/task-evidence-coverage-refresh.ts +217 -0
- package/packages/piagent-core/runtime/registration/context-commands.ts +82 -7
- package/packages/piagent-core/runtime/registration/knowledge-tools.ts +9 -5
- package/packages/piagent-core/runtime/registration/operator-request-intake.ts +49 -0
- package/packages/piagent-core/runtime/registration/policy-tools.ts +50 -15
- package/packages/piagent-core/runtime/registration/session-commands.ts +1 -1
- package/packages/piagent-core/runtime/registration/task-completion-tools.ts +61 -25
- package/packages/piagent-core/runtime/registration/task-evidence-tools.ts +21 -12
- package/packages/piagent-core/runtime/registration/task-start-guidance.ts +169 -0
- package/packages/piagent-core/runtime/registration/task-start-manifest.ts +54 -0
- package/packages/piagent-core/runtime/registration/task-start-retry-helpers.ts +37 -0
- package/packages/piagent-core/runtime/registration/task-start-tool.ts +108 -103
- package/packages/piagent-core/runtime/runtime-limits.ts +16 -2
- package/packages/piagent-core/runtime/session/adaptive-context-analysis.ts +461 -0
- package/packages/piagent-core/runtime/session/adaptive-context-governor.ts +464 -0
- package/packages/piagent-core/runtime/session/adaptive-context-ledger.ts +265 -0
- package/packages/piagent-core/runtime/session/message-signals.ts +7 -3
- package/packages/piagent-core/runtime/session/operator-request-carry.ts +44 -0
- package/packages/piagent-core/runtime/session/runtime-state.ts +161 -5
- package/packages/piagent-core/runtime/session/source-checkout-read-grants.ts +35 -0
- package/packages/piagent-core/runtime/session/system-prompt.ts +76 -14
- package/packages/piagent-core/runtime/session/task-contract-presentation.ts +39 -0
- package/packages/piagent-core/runtime/tools/apply-patch-tool.ts +500 -0
- package/packages/piagent-core/runtime/tools/phase-tools.ts +8 -8
- package/packages/piagent-core/runtime/tools/tool-groups.ts +12 -2
- package/packages/piagent-core/runtime/trajectory/trajectory-runtime.ts +2 -1
- package/packages/piagent-core/runtime/verification/exact-verifier-reuse.ts +101 -0
- package/packages/piagent-core/runtime/workflows/runtime-intake-compaction.ts +164 -0
- package/packages/piagent-core/runtime/workflows/task-intake.ts +207 -38
- package/packages/piagent-core/security/sensitive-data.js +13 -218
- package/packages/piagent-core/security/sensitive-project-file.js +42 -0
- package/packages/piagent-core/security/sensitive-source-expression.js +330 -0
- package/packages/piagent-core/security/sensitive-source-formats.js +394 -0
- package/packages/piagent-core/security/sensitive-text.js +234 -0
- package/packages/piagent-core/subagents/piagent-oracle.md +5 -3
- package/packages/piagent-core/subagents/piagent-planner.md +6 -4
- package/packages/piagent-core/subagents/piagent-reviewer.md +5 -3
- package/packages/piagent-core/subagents/piagent-scout.md +4 -2
- package/packages/piagent-core/subagents/piagent-worker.md +7 -5
- package/packages/piagent-webui/dist/client/assets/{MarkdownMessage-rD9WJKxs.js → MarkdownMessage-B4XehTCU.js} +1 -1
- package/packages/piagent-webui/dist/client/assets/index-CgOEyRPG.js +176 -0
- package/packages/piagent-webui/dist/client/index.html +1 -1
- package/packages/piagent-webui/extension/piagent-webui.ts +3 -1
- package/packages/piagent-webui/extension/same-session-bridge.ts +1 -1
- package/packages/piagent-webui/extension/session-stream-adapter.ts +33 -10
- package/packages/piagent-webui/gateway/gateway-events.ts +109 -5
- package/packages/piagent-webui/gateway/gateway-service.ts +3 -0
- package/packages/piagent-webui/gateway/gateway-session-stream.ts +122 -20
- package/packages/piagent-webui/gateway/session-command-controller.ts +18 -3
- package/packages/piagent-webui/gateway/session-effective-options.ts +47 -0
- package/packages/piagent-webui/gateway/session-live-state.ts +21 -0
- package/packages/piagent-webui/gateway/session-operation-lifecycle.ts +237 -0
- package/packages/piagent-webui/gateway/session-operation-retry-control.ts +38 -0
- package/packages/piagent-webui/gateway/session-operation-start.ts +5 -1
- package/packages/piagent-webui/gateway/session-operation-watchdog.ts +165 -0
- package/packages/piagent-webui/gateway/session-runtime-factory.ts +45 -0
- package/packages/piagent-webui/gateway/session-runtime-supervisor.ts +141 -125
- package/packages/piagent-webui/ownership/session-lease-store.ts +5 -2
- package/packages/piagent-webui/server/core-inspection-provider.ts +2 -1
- package/packages/piagent-webui/server/loopback-server.ts +8 -0
- package/packages/piagent-webui/server/transcript-projection.ts +113 -9
- package/packages/piagent-webui/shared/text-visibility.ts +10 -0
- package/schemas/codex-relative-efficiency-v1.schema.json +155 -0
- package/schemas/piagent-webui/catalog-v1.json +9 -0
- package/schemas/piagent-webui/gateway-protocol-v1.schema.json +53 -3
- package/schemas/piagent-webui/runtime-event-v2.schema.json +15 -5
- package/schemas/piagent-webui/session-live-state-v1.schema.json +76 -0
- package/schemas/piagent-webui/subagent-tree-v1.schema.json +11 -1
- package/schemas/project-profile.schema.json +1 -1
- package/schemas/task-contract.schema.json +27 -1
- package/scripts/benchmark-runner-core.mjs +367 -363
- package/scripts/benchmark-runner-finalization.mjs +313 -0
- package/scripts/benchmark-runner-host-readiness.mjs +94 -0
- package/scripts/benchmark-runner-support.mjs +200 -0
- package/scripts/benchmark-session.mjs +95 -41
- package/scripts/configure-model-scope.sh +4 -3
- package/scripts/configure-subagents.sh +42 -34
- package/scripts/helper-evaluation.mjs +2 -2
- package/scripts/install-global.sh +2 -2
- package/scripts/pi-usage-history.mjs +37 -10
- package/scripts/setup.sh +2 -2
- package/scripts/team-doctor.sh +1 -1
- package/scripts/verify-local.sh +10 -1
- package/templates/global/settings.json +2 -1
- package/packages/piagent-webui/dist/client/assets/index-BZRFw1u6.js +0 -174
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import crypto from "node:crypto";
|
|
2
2
|
import fs from "node:fs";
|
|
3
|
+
import os from "node:os";
|
|
3
4
|
import path from "node:path";
|
|
4
5
|
|
|
5
6
|
import {
|
|
@@ -16,6 +17,7 @@ import {
|
|
|
16
17
|
} from "../packages/piagent-core/benchmark/benchmark-runtime.js";
|
|
17
18
|
import {
|
|
18
19
|
acceptedTaskStartTraceCount,
|
|
20
|
+
benchmarkCausalContextReceipt,
|
|
19
21
|
benchmarkOperationalEvidence,
|
|
20
22
|
classifyPreUsageFailure,
|
|
21
23
|
failureReason,
|
|
@@ -24,10 +26,12 @@ import {
|
|
|
24
26
|
writePrivate,
|
|
25
27
|
writePrivateAtomic
|
|
26
28
|
} from "../packages/piagent-core/benchmark/benchmark-forensics.js";
|
|
27
|
-
import {
|
|
29
|
+
import { inspectContextTelemetry } from "../packages/piagent-core/extensions/context-engine.js";
|
|
28
30
|
import { matchesAnyPath } from "../packages/piagent-core/extensions/policy-core.js";
|
|
29
31
|
import { listTaskContracts, workingTreeFiles, workingTreeSnapshot } from "../packages/piagent-core/extensions/task-state.js";
|
|
30
32
|
import { benchmarkTreeIdentity } from "../packages/piagent-core/benchmark/benchmark-tree-identity.js";
|
|
33
|
+
import { buildBenchmarkProviderWireEvidence } from "../packages/piagent-core/benchmark/benchmark-provider-wire.js";
|
|
34
|
+
import { createDeferredBenchmarkTimingCollector } from "../packages/piagent-core/benchmark/benchmark-timing-diagnostics.js";
|
|
31
35
|
import { summarizeSession, walkJsonl } from "./pi-usage-history.mjs";
|
|
32
36
|
|
|
33
37
|
const coldStartRuntimeManagedPaths = [
|
|
@@ -49,6 +53,12 @@ function privateDirectory(target) {
|
|
|
49
53
|
return target;
|
|
50
54
|
}
|
|
51
55
|
|
|
56
|
+
function privateTemporaryDirectory(prefix) {
|
|
57
|
+
const target = fs.mkdtempSync(path.join(os.tmpdir(), prefix));
|
|
58
|
+
try { fs.chmodSync(target, 0o700); } catch { /* Non-POSIX filesystem. */ }
|
|
59
|
+
return target;
|
|
60
|
+
}
|
|
61
|
+
|
|
52
62
|
function makeFixtureWritable(root) {
|
|
53
63
|
const pending = [root];
|
|
54
64
|
while (pending.length > 0) {
|
|
@@ -122,28 +132,33 @@ function generatedStringArray(value, field) {
|
|
|
122
132
|
return [...new Set(value)];
|
|
123
133
|
}
|
|
124
134
|
|
|
125
|
-
async function generateVariant({ runCommand, nodeCommand, generator, workspace,
|
|
126
|
-
const
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
135
|
+
async function generateVariant({ runCommand, nodeCommand, generator, workspace, seed, scenario, timeoutSeconds }) {
|
|
136
|
+
const temporaryRoot = privateTemporaryDirectory("piagent-benchmark-oracle-input-");
|
|
137
|
+
const oraclePath = path.join(temporaryRoot, "oracle.json");
|
|
138
|
+
try {
|
|
139
|
+
const result = await runCommand(nodeCommand, [generator, workspace, oraclePath, seed, scenario.id], {
|
|
140
|
+
cwd: path.dirname(generator), timeoutMs: Math.min(timeoutSeconds, 120) * 1_000, env: graderEnvironment(scenario.id)
|
|
141
|
+
});
|
|
142
|
+
if (result.timedOut) fail(`Benchmark variant generator timed out for ${scenario.id}`);
|
|
143
|
+
if (result.code !== 0) fail(`Benchmark variant generator failed for ${scenario.id}: ${result.stderr.trim() || result.stdout.trim()}`);
|
|
144
|
+
let oracle;
|
|
145
|
+
try { oracle = JSON.parse(fs.readFileSync(oraclePath, "utf8")); }
|
|
146
|
+
catch (error) { fail(`Benchmark variant generator did not write a valid oracle for ${scenario.id}: ${error.message}`); }
|
|
147
|
+
if (!oracle || typeof oracle !== "object" || Array.isArray(oracle) || oracle.schemaVersion !== 1 || !oracle.graderData || typeof oracle.graderData !== "object" || Array.isArray(oracle.graderData)) {
|
|
148
|
+
fail(`Benchmark variant oracle is invalid for ${scenario.id}`);
|
|
149
|
+
}
|
|
150
|
+
const serialized = JSON.stringify(oracle);
|
|
151
|
+
if (Buffer.byteLength(serialized) > 100_000) fail(`Benchmark variant oracle is too large for ${scenario.id}`);
|
|
152
|
+
return {
|
|
153
|
+
oracleSerialized: `${serialized}\n`,
|
|
154
|
+
oracleDigest: crypto.createHash("sha256").update(serialized).digest("hex"),
|
|
155
|
+
seedDigest: crypto.createHash("sha256").update(seed).digest("hex"),
|
|
156
|
+
requiredOutputSubstrings: generatedStringArray(oracle.requiredOutputSubstrings, "requiredOutputSubstrings"),
|
|
157
|
+
forbiddenOutputSubstrings: generatedStringArray(oracle.forbiddenOutputSubstrings, "forbiddenOutputSubstrings")
|
|
158
|
+
};
|
|
159
|
+
} finally {
|
|
160
|
+
fs.rmSync(temporaryRoot, { recursive: true, force: true });
|
|
136
161
|
}
|
|
137
|
-
const serialized = JSON.stringify(oracle);
|
|
138
|
-
if (Buffer.byteLength(serialized) > 100_000) fail(`Benchmark variant oracle is too large for ${scenario.id}`);
|
|
139
|
-
writePrivate(oraclePath, `${JSON.stringify(oracle)}\n`);
|
|
140
|
-
return {
|
|
141
|
-
oraclePath,
|
|
142
|
-
oracleDigest: crypto.createHash("sha256").update(serialized).digest("hex"),
|
|
143
|
-
seedDigest: crypto.createHash("sha256").update(seed).digest("hex"),
|
|
144
|
-
requiredOutputSubstrings: generatedStringArray(oracle.requiredOutputSubstrings, "requiredOutputSubstrings"),
|
|
145
|
-
forbiddenOutputSubstrings: generatedStringArray(oracle.forbiddenOutputSubstrings, "forbiddenOutputSubstrings")
|
|
146
|
-
};
|
|
147
162
|
}
|
|
148
163
|
|
|
149
164
|
function writableCopyEnvironment(workspace) {
|
|
@@ -276,17 +291,24 @@ function parseGraderResult(stdout) {
|
|
|
276
291
|
};
|
|
277
292
|
}
|
|
278
293
|
|
|
279
|
-
async function gradeWorkspace(runCommand, nodeCommand, grader, workspace, scenario, timeoutSeconds,
|
|
280
|
-
const
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
294
|
+
async function gradeWorkspace(runCommand, nodeCommand, grader, workspace, scenario, timeoutSeconds, oracleSerialized) {
|
|
295
|
+
const temporaryRoot = oracleSerialized ? privateTemporaryDirectory("piagent-benchmark-oracle-grader-") : null;
|
|
296
|
+
try {
|
|
297
|
+
const oraclePath = temporaryRoot ? path.join(temporaryRoot, "oracle.json") : null;
|
|
298
|
+
if (oraclePath) writePrivate(oraclePath, oracleSerialized);
|
|
299
|
+
const result = await runCommand(nodeCommand, oraclePath ? [grader, workspace, oraclePath] : [grader, workspace], {
|
|
300
|
+
cwd: path.dirname(grader), timeoutMs: Math.min(timeoutSeconds, 120) * 1_000, env: graderEnvironment(scenario.id)
|
|
301
|
+
});
|
|
302
|
+
if (result.timedOut) return { passed: false, score: 0, checks: [], error: "grader-timeout" };
|
|
303
|
+
if (result.code !== 0) return { passed: false, score: 0, checks: [], error: `grader-exit-${result.code}` };
|
|
304
|
+
try { return parseGraderResult(result.stdout); } catch (error) { return { passed: false, score: 0, checks: [], error: error.message }; }
|
|
305
|
+
} finally {
|
|
306
|
+
if (temporaryRoot) fs.rmSync(temporaryRoot, { recursive: true, force: true });
|
|
307
|
+
}
|
|
286
308
|
}
|
|
287
309
|
|
|
288
310
|
function sessionSummaries(sessionDir) {
|
|
289
|
-
return walkJsonl(sessionDir).map((file) => summarizeSession(file, {})).filter(Boolean);
|
|
311
|
+
return walkJsonl(sessionDir).map((file) => summarizeSession(file, { strictUsage: true })).filter(Boolean);
|
|
290
312
|
}
|
|
291
313
|
|
|
292
314
|
export async function runBenchmarkSession({ packageRoot, runCommand, resolveSuiteEntry, interrupted, persistCompletedRecord, suite, suiteRoot, scenario, surface, repeat, orderIndex, infrastructureAttempt = 1, runId, runRoot, options, piCommand, codexCommand, codexDisabledFeatures, codexRuntime, piRuntimeHome, systemCommands, suiteDigest, configurationDigest, rootSeed }) {
|
|
@@ -301,11 +323,11 @@ export async function runBenchmarkSession({ packageRoot, runCommand, resolveSuit
|
|
|
301
323
|
applySetupFiles(workspace, scenario.setupFiles);
|
|
302
324
|
const profile = scenario.profile ?? suite.profile;
|
|
303
325
|
const lifecycle = scenario.lifecycle ?? "steady-state";
|
|
304
|
-
let variant = {
|
|
326
|
+
let variant = { oracleSerialized: null, oracleDigest: null, seedDigest: null, requiredOutputSubstrings: [], forbiddenOutputSubstrings: [] };
|
|
305
327
|
if (scenario.variantGenerator) {
|
|
306
328
|
variant = await generateVariant({
|
|
307
329
|
runCommand, nodeCommand: systemCommands.node, generator: resolveSuiteEntry(suiteRoot, scenario.variantGenerator, "variant generator"), workspace,
|
|
308
|
-
|
|
330
|
+
seed: variantSeed(rootSeed, suiteDigest, scenario.id, repeat),
|
|
309
331
|
scenario, timeoutSeconds: options.timeoutSeconds
|
|
310
332
|
});
|
|
311
333
|
}
|
|
@@ -337,6 +359,7 @@ export async function runBenchmarkSession({ packageRoot, runCommand, resolveSuit
|
|
|
337
359
|
const args = surface === "codex-cli" ? codexExecArgs({ workspace, options, disabledFeatures: codexDisabledFeatures }) : piArgs;
|
|
338
360
|
const codexForbiddenHits = new Set();
|
|
339
361
|
const codexCollector = surface === "codex-cli" ? createCodexExecJsonlCollector({ model: options.model, thinkingLevel: options.thinking, onEvent: (event) => inspectForbiddenValue(event, forbiddenOutputSubstrings, codexForbiddenHits) }) : undefined;
|
|
362
|
+
const timingCollector = createDeferredBenchmarkTimingCollector({ surface });
|
|
340
363
|
const environment = {
|
|
341
364
|
PIAGENT_NO_UPDATE_CHECK: "1", PIAGENT_BENCHMARK_RUN_ID: runId, PIAGENT_BENCHMARK_SCENARIO: scenario.id,
|
|
342
365
|
PIAGENT_BENCHMARK_SURFACE: surface, PIAGENT_BENCHMARK_SESSION_ID: sessionId, PIAGENT_BENCHMARK_PROFILE: profile,
|
|
@@ -344,12 +367,22 @@ export async function runBenchmarkSession({ packageRoot, runCommand, resolveSuit
|
|
|
344
367
|
};
|
|
345
368
|
const inflightPath = path.join(workspaceRoot, "inflight.json");
|
|
346
369
|
writePrivateAtomic(inflightPath, `${JSON.stringify({ schemaVersion: 1, runId, attemptId, orderIndex, scenarioId: scenario.id, surface, repeat, infrastructureAttempt, stage: "provider-may-start", recordedAt: new Date().toISOString() }, null, 2)}\n`);
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
370
|
+
let agent;
|
|
371
|
+
try {
|
|
372
|
+
agent = await runCommand(command, args, {
|
|
373
|
+
cwd: workspace, input: surface === "codex-cli" ? prompt : undefined, timeoutMs: options.timeoutSeconds * 1_000,
|
|
374
|
+
forbiddenSubstrings: forbiddenOutputSubstrings, requiredSubstrings: requiredOutputSubstrings,
|
|
375
|
+
onStdoutChunk: (chunk, observation) => {
|
|
376
|
+
timingCollector.write(chunk, observation?.observedAtSeconds);
|
|
377
|
+
codexCollector?.write(chunk);
|
|
378
|
+
},
|
|
379
|
+
env: surface === "codex-cli" ? codexProcessEnvironment(codexRuntime, environment) : surface === "piagent" ? piagentProcessEnvironment(options.piagentTreatment, { ...environment, PI_CODING_AGENT_DIR: piRuntimeHome.path }) : benchmarkEnvironment({ ...environment, PI_CODING_AGENT_DIR: piRuntimeHome.path })
|
|
380
|
+
});
|
|
381
|
+
} catch (error) {
|
|
382
|
+
timingCollector.discard();
|
|
383
|
+
throw error;
|
|
384
|
+
}
|
|
385
|
+
const timingDiagnostics = timingCollector.finish(agent.durationSeconds);
|
|
353
386
|
const sessionFiles = surface === "codex-cli" ? [] : walkJsonl(sessions);
|
|
354
387
|
let usage;
|
|
355
388
|
let codexDiagnostics = [];
|
|
@@ -372,11 +405,31 @@ export async function runBenchmarkSession({ packageRoot, runCommand, resolveSuit
|
|
|
372
405
|
const runtimeManagedChanges = surface === "piagent" && lifecycle === "cold-start" ? allChangedFiles.filter((file) => matchesAnyPath(file, coldStartRuntimeManagedPaths)) : [];
|
|
373
406
|
const runtimeManagedSet = new Set(runtimeManagedChanges);
|
|
374
407
|
const changedFiles = allChangedFiles.filter((file) => !runtimeManagedSet.has(file));
|
|
408
|
+
const telemetryLimit = 50_000;
|
|
409
|
+
const contextTelemetryInspection = surface === "piagent"
|
|
410
|
+
? inspectContextTelemetry(workspace, { limit: telemetryLimit })
|
|
411
|
+
: { records: [], exists: false, integrityFailures: 0, recoverableTailBytes: 0, inputTruncated: false };
|
|
412
|
+
const contextTelemetry = contextTelemetryInspection.records;
|
|
413
|
+
const causalContextReceipt = benchmarkCausalContextReceipt(contextTelemetry, {
|
|
414
|
+
surface,
|
|
415
|
+
sessionId,
|
|
416
|
+
criterionExpected: scenario.kind !== "safety-refusal",
|
|
417
|
+
telemetryExists: contextTelemetryInspection.exists,
|
|
418
|
+
telemetryTruncated: contextTelemetryInspection.inputTruncated,
|
|
419
|
+
telemetryIntegrityFailures: contextTelemetryInspection.integrityFailures,
|
|
420
|
+
recoverableTailBytes: contextTelemetryInspection.recoverableTailBytes
|
|
421
|
+
});
|
|
422
|
+
const providerWireEvidence = surface === "piagent" ? buildBenchmarkProviderWireEvidence({
|
|
423
|
+
events: contextTelemetry,
|
|
424
|
+
requestedModel: options.model,
|
|
425
|
+
requestedThinking: options.thinking,
|
|
426
|
+
telemetryTruncated: contextTelemetryInspection.inputTruncated
|
|
427
|
+
}) : null;
|
|
375
428
|
let workflow = null;
|
|
376
429
|
if (surface === "piagent" && scenario.kind !== "safety-refusal") {
|
|
377
430
|
const task = listTaskContracts(workspace).find((item) => item.sessionId === sessionId);
|
|
378
431
|
workflow = evaluateWorkflowEvidence(task, changedFiles, usage.toolNames, { scenarioKind: scenario.kind, acceptedTaskStartCount: acceptedTaskStartTraceCount(sessionFiles, sessionId) });
|
|
379
|
-
workflow.operational = benchmarkOperationalEvidence(
|
|
432
|
+
workflow.operational = benchmarkOperationalEvidence(contextTelemetry);
|
|
380
433
|
}
|
|
381
434
|
const beforeGrade = workingTreeSnapshot(workspace);
|
|
382
435
|
const outsideScope = changedFiles.filter((file) => !matchesAnyPath(file, scenario.allowedChanges));
|
|
@@ -384,7 +437,7 @@ export async function runBenchmarkSession({ packageRoot, runCommand, resolveSuit
|
|
|
384
437
|
? { passed: false, score: 0, checks: [], error: preUsageFailure.failure }
|
|
385
438
|
: interrupted()
|
|
386
439
|
? { passed: false, score: 0, checks: [], error: "interrupted-after-provider-start" }
|
|
387
|
-
: await gradeWorkspace(runCommand, systemCommands.node, graderPath, workspace, scenario, options.timeoutSeconds, variant.
|
|
440
|
+
: await gradeWorkspace(runCommand, systemCommands.node, graderPath, workspace, scenario, options.timeoutSeconds, variant.oracleSerialized);
|
|
388
441
|
const graderIntegrity = { passed: JSON.stringify(beforeGrade) === JSON.stringify(workingTreeSnapshot(workspace)) };
|
|
389
442
|
const scope = { passed: outsideScope.length === 0, changedFiles, outsideScope, allChangedFiles, runtimeManagedChanges };
|
|
390
443
|
const outputSafety = { passed: forbiddenHits.length === 0, forbiddenHits: forbiddenHits.map((value) => crypto.createHash("sha256").update(value).digest("hex")) };
|
|
@@ -402,7 +455,8 @@ export async function runBenchmarkSession({ packageRoot, runCommand, resolveSuit
|
|
|
402
455
|
infrastructureDiagnosticSource: abortSuite ? (codexDiagnostics.length > 0 ? "codex-error-events" : piTerminalError ? "pi-terminal-error-event" : "process-output-tail") : undefined,
|
|
403
456
|
resolved, failure: preUsageFailure?.failure ?? failureReason({ agent, grade, graderIntegrity, outsideScope, forbiddenHits, missingRequired }),
|
|
404
457
|
agent: { exitCode: agent.code, signal: agent.signal, timedOut: agent.timedOut, stdoutHash: agent.stdoutHash ?? crypto.createHash("sha256").update(agent.stdout).digest("hex"), stderrHash: crypto.createHash("sha256").update(agent.stderr ?? "").digest("hex") },
|
|
405
|
-
grade, graderIntegrity, scope, outputSafety, outputEvidence, workflow,
|
|
458
|
+
grade, graderIntegrity, scope, outputSafety, outputEvidence, workflow, providerWireEvidence, causalContextReceipt, usage,
|
|
459
|
+
durationSeconds: agent.durationSeconds, timingDiagnostics,
|
|
406
460
|
promptHash: crypto.createHash("sha256").update(prompt).digest("hex"),
|
|
407
461
|
variant: scenario.variantGenerator ? { generated: true, seedDigest: variant.seedDigest, oracleDigest: variant.oracleDigest, fixtureDigest } : { generated: false, fixtureDigest }
|
|
408
462
|
};
|
|
@@ -9,7 +9,7 @@ Usage:
|
|
|
9
9
|
Options:
|
|
10
10
|
--preset <full|codex|claude> Model families to enable (default: full = both)
|
|
11
11
|
--default-model <provider/model[:thinking]>
|
|
12
|
-
Default model (default: openai-codex/gpt-5.
|
|
12
|
+
Default model (default: openai-codex/gpt-5.6-sol:high)
|
|
13
13
|
--settings <path> Pi settings file (default: ~/.pi/agent/settings.json)
|
|
14
14
|
--dry-run Print resulting settings JSON without writing
|
|
15
15
|
-h, --help
|
|
@@ -30,7 +30,7 @@ USAGE
|
|
|
30
30
|
}
|
|
31
31
|
|
|
32
32
|
PRESET="full"
|
|
33
|
-
DEFAULT_MODEL="openai-codex/gpt-5.
|
|
33
|
+
DEFAULT_MODEL="openai-codex/gpt-5.6-sol:high"
|
|
34
34
|
SETTINGS_PATH="${PI_CODING_AGENT_DIR:-"${HOME}/.pi/agent"}/settings.json"
|
|
35
35
|
DRY_RUN=false
|
|
36
36
|
|
|
@@ -102,7 +102,8 @@ const codexModels = [
|
|
|
102
102
|
"openai-codex/gpt-5.5:high",
|
|
103
103
|
"openai-codex/gpt-5.6-luna:medium",
|
|
104
104
|
"openai-codex/gpt-5.6-terra:high",
|
|
105
|
-
"openai-codex/gpt-5.6-sol:high"
|
|
105
|
+
"openai-codex/gpt-5.6-sol:high",
|
|
106
|
+
"openai-codex/*"
|
|
106
107
|
];
|
|
107
108
|
|
|
108
109
|
const claudeModels = [
|
|
@@ -7,7 +7,7 @@ Usage:
|
|
|
7
7
|
scripts/configure-subagents.sh [options]
|
|
8
8
|
|
|
9
9
|
Purpose:
|
|
10
|
-
Configure pi-subagents with
|
|
10
|
+
Configure pi-subagents with Piagent's parent-direct, one-read-only-helper ceiling.
|
|
11
11
|
|
|
12
12
|
Options:
|
|
13
13
|
--preset <minimal|safe|async|parallel>
|
|
@@ -27,7 +27,7 @@ After installing/configuring:
|
|
|
27
27
|
pi
|
|
28
28
|
/subagents-doctor
|
|
29
29
|
/subagents-models
|
|
30
|
-
|
|
30
|
+
# Piagent stays parent-direct unless runtime evidence proves >=30% net token saving.
|
|
31
31
|
USAGE
|
|
32
32
|
}
|
|
33
33
|
|
|
@@ -103,16 +103,16 @@ const presets = {
|
|
|
103
103
|
asyncByDefault: false,
|
|
104
104
|
asyncWidget: true,
|
|
105
105
|
forceTopLevelAsync: false,
|
|
106
|
-
waitTool: { enabled:
|
|
107
|
-
intercomBridge: { mode: "
|
|
108
|
-
globalConcurrencyLimit:
|
|
109
|
-
maxSubagentSpawnsPerSession:
|
|
106
|
+
waitTool: { enabled: false },
|
|
107
|
+
intercomBridge: { mode: "off", resultDelivery: false },
|
|
108
|
+
globalConcurrencyLimit: 1,
|
|
109
|
+
maxSubagentSpawnsPerSession: 1,
|
|
110
110
|
maxSubagentDepth: 1,
|
|
111
111
|
defaultSessionDir: "~/.pi/agent/sessions/subagent",
|
|
112
112
|
singleRunOutputBaseDir: "~/.pi/agent/subagent-outputs",
|
|
113
113
|
worktreeBaseDir: "~/.pi/agent/worktrees/pi-subagents",
|
|
114
|
-
scheduledRuns: { enabled: false, maxPending:
|
|
115
|
-
parallel: { maxTasks:
|
|
114
|
+
scheduledRuns: { enabled: false, maxPending: 1, maxLatenessMs: 300000 },
|
|
115
|
+
parallel: { maxTasks: 1, concurrency: 1 },
|
|
116
116
|
completionBatch: { enabled: true, debounceMs: 150, maxWaitMs: 1000 }
|
|
117
117
|
},
|
|
118
118
|
safe: {
|
|
@@ -120,33 +120,33 @@ const presets = {
|
|
|
120
120
|
asyncByDefault: false,
|
|
121
121
|
asyncWidget: true,
|
|
122
122
|
forceTopLevelAsync: false,
|
|
123
|
-
waitTool: { enabled:
|
|
124
|
-
intercomBridge: { mode: "
|
|
125
|
-
globalConcurrencyLimit:
|
|
126
|
-
maxSubagentSpawnsPerSession:
|
|
123
|
+
waitTool: { enabled: false },
|
|
124
|
+
intercomBridge: { mode: "off", resultDelivery: false },
|
|
125
|
+
globalConcurrencyLimit: 1,
|
|
126
|
+
maxSubagentSpawnsPerSession: 1,
|
|
127
127
|
maxSubagentDepth: 1,
|
|
128
128
|
defaultSessionDir: "~/.pi/agent/sessions/subagent",
|
|
129
129
|
singleRunOutputBaseDir: "~/.pi/agent/subagent-outputs",
|
|
130
130
|
worktreeBaseDir: "~/.pi/agent/worktrees/pi-subagents",
|
|
131
|
-
scheduledRuns: { enabled: false, maxPending:
|
|
132
|
-
parallel: { maxTasks:
|
|
131
|
+
scheduledRuns: { enabled: false, maxPending: 1, maxLatenessMs: 300000 },
|
|
132
|
+
parallel: { maxTasks: 1, concurrency: 1 },
|
|
133
133
|
completionBatch: { enabled: true, debounceMs: 150, maxWaitMs: 1000, stragglerDebounceMs: 75, stragglerMaxWaitMs: 400, stragglerWindowMs: 2000 }
|
|
134
134
|
},
|
|
135
135
|
async: {
|
|
136
136
|
toolDescriptionMode: "compact",
|
|
137
|
-
asyncByDefault:
|
|
137
|
+
asyncByDefault: false,
|
|
138
138
|
asyncWidget: true,
|
|
139
139
|
forceTopLevelAsync: false,
|
|
140
|
-
waitTool: { enabled:
|
|
141
|
-
intercomBridge: { mode: "
|
|
142
|
-
globalConcurrencyLimit:
|
|
143
|
-
maxSubagentSpawnsPerSession:
|
|
140
|
+
waitTool: { enabled: false },
|
|
141
|
+
intercomBridge: { mode: "off", resultDelivery: false },
|
|
142
|
+
globalConcurrencyLimit: 1,
|
|
143
|
+
maxSubagentSpawnsPerSession: 1,
|
|
144
144
|
maxSubagentDepth: 1,
|
|
145
145
|
defaultSessionDir: "~/.pi/agent/sessions/subagent",
|
|
146
146
|
singleRunOutputBaseDir: "~/.pi/agent/subagent-outputs",
|
|
147
147
|
worktreeBaseDir: "~/.pi/agent/worktrees/pi-subagents",
|
|
148
|
-
scheduledRuns: { enabled: false, maxPending:
|
|
149
|
-
parallel: { maxTasks:
|
|
148
|
+
scheduledRuns: { enabled: false, maxPending: 1, maxLatenessMs: 300000 },
|
|
149
|
+
parallel: { maxTasks: 1, concurrency: 1 },
|
|
150
150
|
completionBatch: { enabled: true, debounceMs: 150, maxWaitMs: 1000, stragglerDebounceMs: 75, stragglerMaxWaitMs: 400, stragglerWindowMs: 2000 }
|
|
151
151
|
},
|
|
152
152
|
parallel: {
|
|
@@ -154,16 +154,16 @@ const presets = {
|
|
|
154
154
|
asyncByDefault: false,
|
|
155
155
|
asyncWidget: true,
|
|
156
156
|
forceTopLevelAsync: false,
|
|
157
|
-
waitTool: { enabled:
|
|
158
|
-
intercomBridge: { mode: "
|
|
159
|
-
globalConcurrencyLimit:
|
|
160
|
-
maxSubagentSpawnsPerSession:
|
|
157
|
+
waitTool: { enabled: false },
|
|
158
|
+
intercomBridge: { mode: "off", resultDelivery: false },
|
|
159
|
+
globalConcurrencyLimit: 1,
|
|
160
|
+
maxSubagentSpawnsPerSession: 1,
|
|
161
161
|
maxSubagentDepth: 1,
|
|
162
162
|
defaultSessionDir: "~/.pi/agent/sessions/subagent",
|
|
163
163
|
singleRunOutputBaseDir: "~/.pi/agent/subagent-outputs",
|
|
164
164
|
worktreeBaseDir: "~/.pi/agent/worktrees/pi-subagents",
|
|
165
|
-
scheduledRuns: { enabled: false, maxPending:
|
|
166
|
-
parallel: { maxTasks:
|
|
165
|
+
scheduledRuns: { enabled: false, maxPending: 1, maxLatenessMs: 300000 },
|
|
166
|
+
parallel: { maxTasks: 1, concurrency: 1 },
|
|
167
167
|
completionBatch: { enabled: true, debounceMs: 150, maxWaitMs: 1000, stragglerDebounceMs: 75, stragglerMaxWaitMs: 400, stragglerWindowMs: 2000 }
|
|
168
168
|
}
|
|
169
169
|
};
|
|
@@ -223,10 +223,20 @@ const mergedConfig = mergeObject(existingConfig && typeof existingConfig === "ob
|
|
|
223
223
|
|
|
224
224
|
const existingSettings = readJson(settingsPath, {});
|
|
225
225
|
const mergedSettings = existingSettings && typeof existingSettings === "object" && !Array.isArray(existingSettings) ? existingSettings : {};
|
|
226
|
+
mergedSettings.subagents = mergedSettings.subagents && typeof mergedSettings.subagents === "object" && !Array.isArray(mergedSettings.subagents)
|
|
227
|
+
? mergedSettings.subagents
|
|
228
|
+
: {};
|
|
229
|
+
mergedSettings.subagents.disableBuiltins = true;
|
|
230
|
+
mergedSettings.subagents.agentOverrides = mergeObject(
|
|
231
|
+
mergedSettings.subagents.agentOverrides && typeof mergedSettings.subagents.agentOverrides === "object" && !Array.isArray(mergedSettings.subagents.agentOverrides)
|
|
232
|
+
? mergedSettings.subagents.agentOverrides
|
|
233
|
+
: {},
|
|
234
|
+
{
|
|
235
|
+
worker: { disabled: true },
|
|
236
|
+
"piagent-worker": { disabled: true }
|
|
237
|
+
}
|
|
238
|
+
);
|
|
226
239
|
if (modelScopes[modelScopeName]) {
|
|
227
|
-
mergedSettings.subagents = mergedSettings.subagents && typeof mergedSettings.subagents === "object" && !Array.isArray(mergedSettings.subagents)
|
|
228
|
-
? mergedSettings.subagents
|
|
229
|
-
: {};
|
|
230
240
|
mergedSettings.subagents.modelScope = modelScopes[modelScopeName];
|
|
231
241
|
}
|
|
232
242
|
|
|
@@ -245,10 +255,8 @@ if (dryRun) {
|
|
|
245
255
|
} else {
|
|
246
256
|
fs.mkdirSync(path.dirname(configPath), { recursive: true });
|
|
247
257
|
fs.writeFileSync(configPath, `${JSON.stringify(mergedConfig, null, 2)}\n`);
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
fs.writeFileSync(settingsPath, `${JSON.stringify(mergedSettings, null, 2)}\n`);
|
|
251
|
-
}
|
|
258
|
+
fs.mkdirSync(path.dirname(settingsPath), { recursive: true });
|
|
259
|
+
fs.writeFileSync(settingsPath, `${JSON.stringify(mergedSettings, null, 2)}\n`);
|
|
252
260
|
console.log(JSON.stringify(report, null, 2));
|
|
253
261
|
}
|
|
254
262
|
NODE
|
|
@@ -32,7 +32,7 @@ function helperRequest(role, objective, run) {
|
|
|
32
32
|
return createHelperRequest({
|
|
33
33
|
policy, objective, taskId: "helper-evaluation", taskRunId: run, sessionId: "private-helper-session",
|
|
34
34
|
parentReadScope: ["src/**"], parentWriteScope: ["src/**"],
|
|
35
|
-
parentAllowedTools: ["read", "grep", "find", "ls", "bash", "edit", "write", "contact_supervisor"],
|
|
35
|
+
parentAllowedTools: ["read", "grep", "find", "ls", "bash", "edit", "write", "apply_patch", "contact_supervisor"],
|
|
36
36
|
requestedWriteScope: role === "worker" ? ["src/**"] : [], singleWriterOwnership: role === "worker" ? "evaluation-writer" : null
|
|
37
37
|
});
|
|
38
38
|
}
|
|
@@ -62,7 +62,7 @@ async function lifecycleProbes() {
|
|
|
62
62
|
return {
|
|
63
63
|
duplicateWork: duplicate.decision === "duplicate" ? 0 : 1,
|
|
64
64
|
budgetViolations: overBudget.disposition === "budget-exceeded" && overBudget.outputDigest === null ? 0 : 1,
|
|
65
|
-
writerInvariantViolations: firstWriter.decision === "
|
|
65
|
+
writerInvariantViolations: firstWriter.decision === "blocked" && secondWriter.decision === "blocked" ? 0 : 1,
|
|
66
66
|
automaticWorkerDelegations: defaultRolePolicy("worker").enabledByDefault ? 1 : 0,
|
|
67
67
|
cancellationLatencyMs: Number((performance.now() - cancelStarted).toFixed(3)),
|
|
68
68
|
timeoutCovered: timedOut.disposition === "timeout" && timedOut.outputDigest === null,
|
|
@@ -28,7 +28,7 @@ Options:
|
|
|
28
28
|
--no-model-scope Leave the user's enabledModels untouched.
|
|
29
29
|
--default-model <provider/model[:thinking]>
|
|
30
30
|
Default model written to Pi settings
|
|
31
|
-
(default: openai-codex/gpt-5.
|
|
31
|
+
(default: openai-codex/gpt-5.6-sol:high).
|
|
32
32
|
-h, --help Print this help and exit.
|
|
33
33
|
|
|
34
34
|
Purpose:
|
|
@@ -91,7 +91,7 @@ WITH_WEB_ACCESS=true
|
|
|
91
91
|
WITH_HERDR=false
|
|
92
92
|
CONFIGURE_MODEL_SCOPE=true
|
|
93
93
|
MODEL_SCOPE_PRESET="full"
|
|
94
|
-
DEFAULT_MODEL="openai-codex/gpt-5.
|
|
94
|
+
DEFAULT_MODEL="openai-codex/gpt-5.6-sol:high"
|
|
95
95
|
DRY_RUN=false
|
|
96
96
|
FLOATING_PACKAGE_SOURCE=false
|
|
97
97
|
RESOLVED_CHANNEL_LABEL="custom"
|
|
@@ -202,14 +202,29 @@ function emptyTotals() {
|
|
|
202
202
|
};
|
|
203
203
|
}
|
|
204
204
|
|
|
205
|
-
function
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
205
|
+
function exactToken(raw, label) {
|
|
206
|
+
if (!Number.isSafeInteger(raw) || raw < 0) throw new Error(`${label} must be a non-negative safe integer`);
|
|
207
|
+
return raw;
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
function addTotals(target, usage, { strict = false, label = "usage" } = {}) {
|
|
211
|
+
const values = strict ? {
|
|
212
|
+
input: exactToken(usage?.input, `${label}.input`),
|
|
213
|
+
output: exactToken(usage?.output, `${label}.output`),
|
|
214
|
+
cacheRead: exactToken(usage?.cacheRead, `${label}.cacheRead`),
|
|
215
|
+
cacheWrite: exactToken(usage?.cacheWrite, `${label}.cacheWrite`),
|
|
216
|
+
reasoning: exactToken(usage?.reasoning, `${label}.reasoning`),
|
|
217
|
+
total: exactToken(usage?.totalTokens, `${label}.totalTokens`)
|
|
218
|
+
} : {
|
|
219
|
+
input: numberValue(usage.input), output: numberValue(usage.output), cacheRead: numberValue(usage.cacheRead),
|
|
220
|
+
cacheWrite: numberValue(usage.cacheWrite), reasoning: numberValue(usage.reasoning),
|
|
221
|
+
total: numberValue(usage.totalTokens) || numberValue(usage.input) + numberValue(usage.output) + numberValue(usage.cacheRead) + numberValue(usage.cacheWrite)
|
|
222
|
+
};
|
|
223
|
+
if (strict && values.total !== values.input + values.output + values.cacheRead + values.cacheWrite) {
|
|
224
|
+
throw new Error(`${label}.totalTokens does not equal the reported token categories`);
|
|
225
|
+
}
|
|
226
|
+
if (strict && values.reasoning > values.output) throw new Error(`${label}.reasoning exceeds output`);
|
|
227
|
+
for (const field of ["input", "output", "cacheRead", "cacheWrite", "reasoning", "total"]) target[field] += values[field];
|
|
213
228
|
const cost = usage.cost;
|
|
214
229
|
target.cost += typeof cost === "object" && cost !== null
|
|
215
230
|
? numberValue(cost.total)
|
|
@@ -254,6 +269,7 @@ function summarizeSession(file, options) {
|
|
|
254
269
|
messages: { user: 0, assistant: 0, toolCalls: 0, toolResults: 0, total: 0 },
|
|
255
270
|
promptChars: 0,
|
|
256
271
|
tokens: emptyTotals(),
|
|
272
|
+
usageIntegrity: { exact: options.strictUsage === true, source: "pi-session-jsonl", assistantMessages: 0, usageMessages: 0 },
|
|
257
273
|
toolNames: {},
|
|
258
274
|
sizeBytes: stat.size,
|
|
259
275
|
mtime: stat.mtime
|
|
@@ -261,12 +277,13 @@ function summarizeSession(file, options) {
|
|
|
261
277
|
|
|
262
278
|
let hasCountedActivity = false;
|
|
263
279
|
const lines = fs.readFileSync(file, "utf8").split(/\n/);
|
|
264
|
-
for (const line of lines) {
|
|
280
|
+
for (const [lineIndex, line] of lines.entries()) {
|
|
265
281
|
if (!line) continue;
|
|
266
282
|
let entry;
|
|
267
283
|
try {
|
|
268
284
|
entry = JSON.parse(line);
|
|
269
285
|
} catch {
|
|
286
|
+
if (options.strictUsage) throw new Error(`Pi session JSONL line ${lineIndex + 1} is not valid JSON`);
|
|
270
287
|
continue;
|
|
271
288
|
}
|
|
272
289
|
|
|
@@ -311,9 +328,16 @@ function summarizeSession(file, options) {
|
|
|
311
328
|
summary.promptChars += extractTextLength(message.content);
|
|
312
329
|
} else if (role === "assistant") {
|
|
313
330
|
summary.messages.assistant += 1;
|
|
331
|
+
summary.usageIntegrity.assistantMessages += 1;
|
|
314
332
|
if (message.provider && !summary.provider) summary.provider = String(message.provider);
|
|
315
333
|
if (message.model && !summary.modelId) summary.modelId = String(message.model);
|
|
316
|
-
if (message.usage
|
|
334
|
+
if (options.strictUsage && (!message.usage || typeof message.usage !== "object" || Array.isArray(message.usage))) {
|
|
335
|
+
throw new Error(`Pi assistant message at line ${lineIndex + 1} is missing usage`);
|
|
336
|
+
}
|
|
337
|
+
if (message.usage) {
|
|
338
|
+
addTotals(summary.tokens, message.usage, { strict: options.strictUsage === true, label: `Pi session JSONL line ${lineIndex + 1} usage` });
|
|
339
|
+
summary.usageIntegrity.usageMessages += 1;
|
|
340
|
+
}
|
|
317
341
|
if (Array.isArray(message.content)) {
|
|
318
342
|
for (const block of message.content) {
|
|
319
343
|
if (block?.type !== "toolCall") continue;
|
|
@@ -336,6 +360,9 @@ function summarizeSession(file, options) {
|
|
|
336
360
|
if (!summary.countedLastTimestamp) summary.countedLastTimestamp = summary.lastTimestamp;
|
|
337
361
|
|
|
338
362
|
if (!hasCountedActivity && (options.since || options.until)) return undefined;
|
|
363
|
+
if (options.strictUsage && summary.usageIntegrity.assistantMessages !== summary.usageIntegrity.usageMessages) {
|
|
364
|
+
throw new Error("Pi session JSONL usage coverage is incomplete");
|
|
365
|
+
}
|
|
339
366
|
return summary;
|
|
340
367
|
}
|
|
341
368
|
|
package/scripts/setup.sh
CHANGED
|
@@ -36,7 +36,7 @@ Options:
|
|
|
36
36
|
--model-scope <full|codex|claude> model families to enable
|
|
37
37
|
Configure Pi enabledModels for selector/cycling (default: full)
|
|
38
38
|
--default-model <provider/model[:thinking]>
|
|
39
|
-
Configure default provider/model/thinking (default: openai-codex/gpt-5.
|
|
39
|
+
Configure default provider/model/thinking (default: openai-codex/gpt-5.6-sol:high)
|
|
40
40
|
--no-model-scope Skip global model-scope configuration
|
|
41
41
|
--install-pi / --no-install-pi Install/skip Pi CLI with npm if `pi` is missing (default: install)
|
|
42
42
|
--force-profile Replace existing project .pi/piagent-profile.json
|
|
@@ -78,7 +78,7 @@ WITH_WEB_ACCESS=true
|
|
|
78
78
|
WITH_HERDR=true
|
|
79
79
|
CONFIGURE_MODEL_SCOPE=true
|
|
80
80
|
MODEL_SCOPE_PRESET="full"
|
|
81
|
-
DEFAULT_MODEL="openai-codex/gpt-5.
|
|
81
|
+
DEFAULT_MODEL="openai-codex/gpt-5.6-sol:high"
|
|
82
82
|
AUTO_INSTALL_PI=true
|
|
83
83
|
FORCE_PROFILE=false
|
|
84
84
|
FORCE_SETTINGS=false
|
package/scripts/team-doctor.sh
CHANGED
|
@@ -506,7 +506,7 @@ const productReadiness = {
|
|
|
506
506
|
subagents: { packageAvailable: piHasSubagents, roleDefinitions: rootPackage.pi?.subagents?.agents ?? [] },
|
|
507
507
|
executionBackend: projectProfile?.executionBackend?.type ?? "host",
|
|
508
508
|
executionBoundary: "host execution is not a sandbox",
|
|
509
|
-
featureModes: { solver: process.env.PIAGENT_SOLVER_MODE ?? "shadow", phaseTools: process.env.PIAGENT_PHASE_TOOLS ?? "shadow", recovery: process.env.PIAGENT_AUTO_RECOVERY ?? "on", helpers: process.env.PIAGENT_HELPERS_MODE ?? "
|
|
509
|
+
featureModes: { solver: process.env.PIAGENT_SOLVER_MODE ?? "shadow", phaseTools: process.env.PIAGENT_PHASE_TOOLS ?? "shadow", recovery: process.env.PIAGENT_AUTO_RECOVERY ?? "on", helpers: process.env.PIAGENT_HELPERS_MODE ?? "off" },
|
|
510
510
|
migrationRecovery: { taskStateExists: fs.existsSync(path.join(stateRoot, "tasks")), handoffStateExists: fs.existsSync(path.join(stateRoot, "handoffs")), action: "run piagent-doctor --json and inspect /piagent-status before deleting state" },
|
|
511
511
|
supportedPlatform: runtime.status,
|
|
512
512
|
rollbackTarget: { packageVersion: rootPackage.version ?? null, instruction: "reinstall the last known-good exact version after operator confirmation; project state is preserved" }
|