@piagent/platform 1.5.5 → 1.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +166 -0
- package/README.md +25 -21
- package/README.vi.md +34 -5
- package/adapters/web-frontend/profile.json +2 -4
- package/architecture/layers.json +18 -3
- package/benchmarks/deep-logic-v1/SPECIALIST_PROTOCOL.vi.md +122 -0
- package/benchmarks/deep-logic-v1/grade.mjs +272 -10
- package/benchmarks/deep-logic-v1/project/apps/admin/src/billing-summary.js +3 -0
- package/benchmarks/deep-logic-v1/project/benchmark-contract/public-smoke.test.js +166 -0
- package/benchmarks/deep-logic-v1/project/package.json +1 -1
- package/benchmarks/deep-logic-v1/project/packages/billing/src/close-period.js +3 -0
- package/benchmarks/deep-logic-v1/project/packages/billing/src/plan-timeline.js +3 -0
- package/benchmarks/deep-logic-v1/project/test/smoke.test.js +7 -1
- package/benchmarks/deep-logic-v1/prompts/budgeted-context-graph.md +2 -0
- package/benchmarks/deep-logic-v1/prompts/fair-dependency-scheduler.md +2 -0
- package/benchmarks/deep-logic-v1/prompts/layered-policy-resolution.md +2 -0
- package/benchmarks/deep-logic-v1/prompts/resumable-stream-assembly.md +3 -1
- package/benchmarks/deep-logic-v1/prompts/revision-event-reconciliation.md +2 -0
- package/benchmarks/deep-logic-v1/prompts/temporal-usage-billing-close.md +123 -0
- package/benchmarks/deep-logic-v1/prompts/transactional-config-merge.md +2 -0
- package/benchmarks/deep-logic-v1/suite.json +51 -3
- package/benchmarks/deep-logic-v1/variant.mjs +22 -1
- package/benchmarks/production-v1/spend-control.v1.json +92 -0
- package/benchmarks/production-v1/suite.json +46 -3
- package/catalog/capabilities.json +18 -18
- package/docs/authority-manifest.md +1 -1
- package/docs/auto-delegation-policy.md +38 -73
- package/docs/command-reference-vietnamese.md +27 -35
- package/docs/context-engine.md +12 -4
- package/docs/context-window-policy.md +37 -1
- package/docs/distribution-standard.md +7 -7
- package/docs/en/architecture.md +27 -13
- package/docs/languages.json +4 -4
- package/docs/model-options.md +8 -5
- package/docs/operator-manual-vietnamese.md +27 -45
- package/docs/project-adapters.md +2 -2
- package/docs/publishing-for-teams.md +10 -0
- package/docs/quality-benchmark.md +427 -62
- package/docs/release-install-policy.md +10 -9
- package/docs/runtime-harness-standard.md +3 -3
- package/docs/runtime-policy-design.md +4 -4
- package/docs/runtime-quality-baseline.md +13 -6
- package/docs/security-threat-model.md +2 -2
- package/docs/subagent-orchestration-capabilities.md +19 -51
- package/docs/subagents-and-multiagent.md +21 -47
- package/docs/task-implementation-contract.md +3 -3
- package/docs/task-lifecycle-vietnamese.md +3 -3
- package/docs/team-onboarding.md +2 -2
- package/docs/usage-observability.md +7 -2
- package/docs/vi/architecture.md +27 -13
- package/docs-site/architecture.html +1 -1
- package/docs-site/benchmark.html +89 -95
- package/docs-site/commands.html +1 -1
- package/docs-site/content/benchmark.html +88 -94
- package/docs-site/content/en/benchmark.html +134 -90
- package/docs-site/content/en/index.html +19 -19
- package/docs-site/content/en/runtime.html +4 -4
- package/docs-site/content/en/whats-new.html +12 -1
- package/docs-site/content/en/workflows.html +2 -2
- package/docs-site/content/index.html +19 -19
- package/docs-site/content/runtime.html +4 -4
- package/docs-site/content/whats-new.html +12 -1
- package/docs-site/content/workflows.html +2 -2
- package/docs-site/context-engine.html +1 -1
- package/docs-site/en/architecture.html +1 -1
- package/docs-site/en/benchmark.html +135 -91
- package/docs-site/en/commands.html +1 -1
- package/docs-site/en/context-engine.html +1 -1
- package/docs-site/en/index.html +20 -20
- package/docs-site/en/mcp-approval.html +1 -1
- package/docs-site/en/mcp-auth.html +1 -1
- package/docs-site/en/mcp-servers.html +1 -1
- package/docs-site/en/mcp.html +1 -1
- package/docs-site/en/permissions.html +1 -1
- package/docs-site/en/profiles.html +1 -1
- package/docs-site/en/quickstart.html +1 -1
- package/docs-site/en/runtime.html +5 -5
- package/docs-site/en/scope.html +1 -1
- package/docs-site/en/security.html +1 -1
- package/docs-site/en/team.html +1 -1
- package/docs-site/en/whats-new.html +13 -2
- package/docs-site/en/workflows.html +3 -3
- package/docs-site/index.html +20 -20
- package/docs-site/mcp-approval.html +1 -1
- package/docs-site/mcp-auth.html +1 -1
- package/docs-site/mcp-servers.html +1 -1
- package/docs-site/mcp.html +1 -1
- package/docs-site/permissions.html +1 -1
- package/docs-site/profiles.html +1 -1
- package/docs-site/quickstart.html +1 -1
- package/docs-site/runtime.html +5 -5
- package/docs-site/scope.html +1 -1
- package/docs-site/security.html +1 -1
- package/docs-site/team.html +1 -1
- package/docs-site/whats-new.html +13 -2
- package/docs-site/workflows.html +3 -3
- package/evals/fixtures/benchmark-assurance-evidence.invalid.json +1 -1
- package/evals/fixtures/benchmark-assurance-evidence.valid.json +1 -1
- package/evals/fixtures/codex-relative-efficiency-v1.invalid.json +4 -0
- package/evals/fixtures/codex-relative-efficiency-v1.valid.json +94 -0
- package/evals/fixtures/helper-request.invalid.json +1 -1
- package/evals/fixtures/helper-request.valid.json +1 -1
- package/evals/fixtures/piagent-webui/session-live-state-v1.invalid.json +18 -0
- package/evals/fixtures/piagent-webui/session-live-state-v1.valid.json +27 -0
- package/evals/fixtures/piagent-webui/subagent-tree-v1.valid.json +7 -0
- package/evals/fixtures/task-authority-snapshot.valid.json +4 -4
- package/evals/fs4-readiness-matrix.v1.json +2 -2
- package/evals/fs5-causal-arm.v1.json +3 -3
- package/evals/fs5-pilot-protocol.v1.json +3 -3
- package/evals/fs5-pilot-protocol.v2.json +8 -8
- package/evals/fs5-pilot-protocol.v3.json +8 -8
- package/evals/fs5-pilot-protocol.v4.json +7 -6
- package/evals/fs5-pilot-protocol.v5.json +9 -7
- package/evals/private-holdout-v1/public-exposure.v1.json +1 -1
- package/evals/runtime-conformance-v1/README.md +43 -0
- package/evals/runtime-conformance-v1/evaluate-paid-canary.mjs +198 -0
- package/evals/runtime-conformance-v1/lane.json +57 -0
- package/evals/runtime-conformance-v1/paid-canary.json +31 -0
- package/evals/runtime-conformance-v1/runner.mjs +456 -0
- package/package.json +3 -1
- package/packages/piagent-core/README.md +13 -4
- package/packages/piagent-core/benchmark/benchmark-assurance.js +4 -1
- package/packages/piagent-core/benchmark/benchmark-bootstrap.js +91 -9
- package/packages/piagent-core/benchmark/benchmark-claim-restrictions.js +38 -7
- package/packages/piagent-core/benchmark/benchmark-cli.js +7 -7
- package/packages/piagent-core/benchmark/benchmark-codex-relative-efficiency.js +698 -0
- package/packages/piagent-core/benchmark/benchmark-codex.js +3 -1
- package/packages/piagent-core/benchmark/benchmark-comparison.js +271 -2
- package/packages/piagent-core/benchmark/benchmark-core.js +736 -664
- package/packages/piagent-core/benchmark/benchmark-execution-guard.js +7 -5
- package/packages/piagent-core/benchmark/benchmark-forensics.js +441 -4
- package/packages/piagent-core/benchmark/benchmark-host-readiness.js +639 -0
- package/packages/piagent-core/benchmark/benchmark-matrix.js +22 -4
- package/packages/piagent-core/benchmark/benchmark-normalized-cost.js +317 -0
- package/packages/piagent-core/benchmark/benchmark-preflight.js +5 -1
- package/packages/piagent-core/benchmark/benchmark-process.js +6 -5
- package/packages/piagent-core/benchmark/benchmark-provider-free-evidence.js +345 -0
- package/packages/piagent-core/benchmark/benchmark-provider-wire.js +126 -0
- package/packages/piagent-core/benchmark/benchmark-record-validation.js +252 -0
- package/packages/piagent-core/benchmark/benchmark-report.js +130 -12
- package/packages/piagent-core/benchmark/benchmark-resume-recovery.js +14 -0
- package/packages/piagent-core/benchmark/benchmark-runner-policy.js +26 -0
- package/packages/piagent-core/benchmark/benchmark-runtime-causal.js +244 -0
- package/packages/piagent-core/benchmark/benchmark-stage-diagnostic.js +979 -0
- package/packages/piagent-core/benchmark/benchmark-statistics.js +64 -0
- package/packages/piagent-core/benchmark/benchmark-stop-policy.js +8 -5
- package/packages/piagent-core/benchmark/benchmark-suite-identity.js +31 -0
- package/packages/piagent-core/benchmark/benchmark-suite-runtime.js +13 -9
- package/packages/piagent-core/benchmark/benchmark-suite.js +149 -2
- package/packages/piagent-core/benchmark/benchmark-summary-support.js +380 -0
- package/packages/piagent-core/benchmark/benchmark-timing-diagnostics.js +971 -0
- package/packages/piagent-core/benchmark/benchmark-usage.js +322 -0
- package/packages/piagent-core/benchmark/benchmark-workflow.js +166 -0
- package/packages/piagent-core/capabilities/runtime-integrity.js +3 -0
- package/packages/piagent-core/docs/context-efficiency-improvement-plan.md +1080 -0
- package/packages/piagent-core/extensions/acceptance-behavior-proof.js +806 -0
- package/packages/piagent-core/extensions/acceptance-contract-semantics.js +83 -69
- package/packages/piagent-core/extensions/acceptance-error-classes.js +160 -0
- package/packages/piagent-core/extensions/acceptance-executable-evidence.js +12 -9
- package/packages/piagent-core/extensions/acceptance-receipt.js +123 -59
- package/packages/piagent-core/extensions/acceptance-state-machine-evidence.js +518 -0
- package/packages/piagent-core/extensions/acceptance-test-binding-evidence.js +60 -0
- package/packages/piagent-core/extensions/context-efficiency-metrics.js +302 -0
- package/packages/piagent-core/extensions/context-efficiency-report.js +167 -0
- package/packages/piagent-core/extensions/context-engine.js +149 -156
- package/packages/piagent-core/extensions/context-evidence.js +48 -0
- package/packages/piagent-core/extensions/context-import-links.js +296 -0
- package/packages/piagent-core/extensions/context-index-policy.js +4 -0
- package/packages/piagent-core/extensions/context-query-paths.js +57 -0
- package/packages/piagent-core/extensions/context-telemetry-integrity.js +65 -0
- package/packages/piagent-core/extensions/criterion-context-pack.js +435 -7
- package/packages/piagent-core/extensions/criterion-graph.js +82 -9
- package/packages/piagent-core/extensions/fallback-base-policy.ts +101 -0
- package/packages/piagent-core/extensions/guard-types.ts +6 -0
- package/packages/piagent-core/extensions/javascript-regex-evidence.js +32 -0
- package/packages/piagent-core/extensions/piagent-guard.ts +420 -260
- package/packages/piagent-core/extensions/task-contract-view.js +17 -3
- package/packages/piagent-core/extensions/task-lifecycle.js +39 -2
- package/packages/piagent-core/extensions/task-state.js +109 -142
- package/packages/piagent-core/extensions/verification-intelligence.js +52 -13
- package/packages/piagent-core/extensions/workspace-evidence-roots.js +193 -0
- package/packages/piagent-core/package.json +1 -1
- package/packages/piagent-core/policies/base-policy.json +44 -2
- package/packages/piagent-core/policy/authority-manifest.v1.json +3 -3
- package/packages/piagent-core/prompts/be-to-fe.md +2 -8
- package/packages/piagent-core/prompts/plan.md +1 -1
- package/packages/piagent-core/prompts/platform-improve.md +2 -6
- package/packages/piagent-core/prompts/review.md +2 -7
- package/packages/piagent-core/prompts/scout.md +1 -1
- package/packages/piagent-core/prompts/task.md +4 -3
- package/packages/piagent-core/runtime/context/context-delivery.ts +103 -0
- package/packages/piagent-core/runtime/context/context-delta-shadow.ts +53 -0
- package/packages/piagent-core/runtime/context/context-efficiency-format.ts +47 -0
- package/packages/piagent-core/runtime/context/context-evidence-qualification.ts +32 -0
- package/packages/piagent-core/runtime/context/prefix-telemetry.ts +47 -0
- package/packages/piagent-core/runtime/context/repository-memory-hints.ts +27 -0
- package/packages/piagent-core/runtime/hooks/agent-prompt-telemetry.ts +56 -0
- package/packages/piagent-core/runtime/hooks/agent-start-hook.ts +162 -102
- package/packages/piagent-core/runtime/hooks/completion-hook.ts +12 -12
- package/packages/piagent-core/runtime/hooks/input-hook.ts +17 -1
- package/packages/piagent-core/runtime/hooks/session-hooks.ts +22 -2
- package/packages/piagent-core/runtime/hooks/session-start-hook.ts +107 -3
- package/packages/piagent-core/runtime/hooks/tool-result-content-guards.ts +95 -0
- package/packages/piagent-core/runtime/hooks/tool-result-hook.ts +134 -166
- package/packages/piagent-core/runtime/hooks/tool-result-value-helpers.ts +66 -0
- package/packages/piagent-core/runtime/inspection/activity-event-adapter.ts +4 -2
- package/packages/piagent-core/runtime/inspection/activity-recovery.ts +104 -0
- package/packages/piagent-core/runtime/inspection/bounded-emission-guard.ts +286 -0
- package/packages/piagent-core/runtime/inspection/runtime-event-store.ts +37 -7
- package/packages/piagent-core/runtime/inspection/task-compaction-history.ts +1 -1
- package/packages/piagent-core/runtime/inspection/task-handoff-history.ts +1 -1
- package/packages/piagent-core/runtime/inspection/task-subagent-tree.ts +27 -4
- package/packages/piagent-core/runtime/inspection/tool-failure-classification.ts +70 -0
- package/packages/piagent-core/runtime/inspection/webui-snapshot.ts +73 -9
- package/packages/piagent-core/runtime/model/authenticated-catalog.ts +17 -9
- package/packages/piagent-core/runtime/model/openai-codex-reasoning.ts +83 -0
- package/packages/piagent-core/runtime/model/provider-wire-fingerprint.ts +238 -0
- package/packages/piagent-core/runtime/orchestration/helper-lifecycle.ts +16 -12
- package/packages/piagent-core/runtime/orchestration/minimal-delegation-policy.ts +71 -0
- package/packages/piagent-core/runtime/orchestration/owned-work-budget.ts +28 -20
- package/packages/piagent-core/runtime/orchestration/role-policy.ts +27 -9
- package/packages/piagent-core/runtime/orchestration/subagent-tool-policy.ts +329 -0
- package/packages/piagent-core/runtime/policy/block-remedy.ts +0 -4
- package/packages/piagent-core/runtime/product/activity-inspector.ts +7 -2
- package/packages/piagent-core/runtime/product/efficiency-metrics.ts +7 -2
- package/packages/piagent-core/runtime/quality/edit-freshness-guard.ts +102 -0
- package/packages/piagent-core/runtime/quality/exact-output-contract.ts +22 -11
- package/packages/piagent-core/runtime/quality/performance-assurance-signals.ts +23 -0
- package/packages/piagent-core/runtime/quality/performance-assurance.ts +9 -5
- package/packages/piagent-core/runtime/recovery/continuation-budget.ts +1 -1
- package/packages/piagent-core/runtime/recovery/edit-recovery-context.ts +113 -0
- package/packages/piagent-core/runtime/recovery/edit-recovery-delivery.ts +42 -0
- package/packages/piagent-core/runtime/recovery/handoff-projection.ts +4 -1
- package/packages/piagent-core/runtime/recovery/pristine-task-policy-refresh.ts +91 -0
- package/packages/piagent-core/runtime/recovery/recovery-policy.ts +7 -4
- package/packages/piagent-core/runtime/recovery/resume-state.ts +112 -14
- package/packages/piagent-core/runtime/recovery/semantic-repair-handshake.ts +60 -13
- package/packages/piagent-core/runtime/recovery/semantic-repair-runtime.ts +55 -2
- package/packages/piagent-core/runtime/recovery/task-evidence-coverage-refresh.ts +217 -0
- package/packages/piagent-core/runtime/registration/context-commands.ts +82 -7
- package/packages/piagent-core/runtime/registration/knowledge-tools.ts +9 -5
- package/packages/piagent-core/runtime/registration/operator-request-intake.ts +49 -0
- package/packages/piagent-core/runtime/registration/policy-tools.ts +50 -15
- package/packages/piagent-core/runtime/registration/session-commands.ts +1 -1
- package/packages/piagent-core/runtime/registration/task-completion-tools.ts +61 -25
- package/packages/piagent-core/runtime/registration/task-evidence-tools.ts +21 -12
- package/packages/piagent-core/runtime/registration/task-start-guidance.ts +169 -0
- package/packages/piagent-core/runtime/registration/task-start-manifest.ts +54 -0
- package/packages/piagent-core/runtime/registration/task-start-retry-helpers.ts +37 -0
- package/packages/piagent-core/runtime/registration/task-start-tool.ts +108 -103
- package/packages/piagent-core/runtime/runtime-limits.ts +16 -2
- package/packages/piagent-core/runtime/session/adaptive-context-analysis.ts +461 -0
- package/packages/piagent-core/runtime/session/adaptive-context-governor.ts +464 -0
- package/packages/piagent-core/runtime/session/adaptive-context-ledger.ts +265 -0
- package/packages/piagent-core/runtime/session/message-signals.ts +7 -3
- package/packages/piagent-core/runtime/session/operator-request-carry.ts +44 -0
- package/packages/piagent-core/runtime/session/runtime-state.ts +161 -5
- package/packages/piagent-core/runtime/session/source-checkout-read-grants.ts +35 -0
- package/packages/piagent-core/runtime/session/system-prompt.ts +76 -14
- package/packages/piagent-core/runtime/session/task-contract-presentation.ts +39 -0
- package/packages/piagent-core/runtime/tools/apply-patch-tool.ts +500 -0
- package/packages/piagent-core/runtime/tools/phase-tools.ts +8 -8
- package/packages/piagent-core/runtime/tools/tool-groups.ts +12 -2
- package/packages/piagent-core/runtime/trajectory/trajectory-runtime.ts +2 -1
- package/packages/piagent-core/runtime/verification/exact-verifier-reuse.ts +101 -0
- package/packages/piagent-core/runtime/workflows/runtime-intake-compaction.ts +164 -0
- package/packages/piagent-core/runtime/workflows/task-intake.ts +207 -38
- package/packages/piagent-core/security/sensitive-data.js +13 -218
- package/packages/piagent-core/security/sensitive-project-file.js +42 -0
- package/packages/piagent-core/security/sensitive-source-expression.js +330 -0
- package/packages/piagent-core/security/sensitive-source-formats.js +394 -0
- package/packages/piagent-core/security/sensitive-text.js +234 -0
- package/packages/piagent-core/subagents/piagent-oracle.md +5 -3
- package/packages/piagent-core/subagents/piagent-planner.md +6 -4
- package/packages/piagent-core/subagents/piagent-reviewer.md +5 -3
- package/packages/piagent-core/subagents/piagent-scout.md +4 -2
- package/packages/piagent-core/subagents/piagent-worker.md +7 -5
- package/packages/piagent-webui/dist/client/assets/{MarkdownMessage-rD9WJKxs.js → MarkdownMessage-B4XehTCU.js} +1 -1
- package/packages/piagent-webui/dist/client/assets/index-CgOEyRPG.js +176 -0
- package/packages/piagent-webui/dist/client/index.html +1 -1
- package/packages/piagent-webui/extension/piagent-webui.ts +3 -1
- package/packages/piagent-webui/extension/same-session-bridge.ts +1 -1
- package/packages/piagent-webui/extension/session-stream-adapter.ts +33 -10
- package/packages/piagent-webui/gateway/gateway-events.ts +109 -5
- package/packages/piagent-webui/gateway/gateway-service.ts +3 -0
- package/packages/piagent-webui/gateway/gateway-session-stream.ts +122 -20
- package/packages/piagent-webui/gateway/session-command-controller.ts +18 -3
- package/packages/piagent-webui/gateway/session-effective-options.ts +47 -0
- package/packages/piagent-webui/gateway/session-live-state.ts +21 -0
- package/packages/piagent-webui/gateway/session-operation-lifecycle.ts +237 -0
- package/packages/piagent-webui/gateway/session-operation-retry-control.ts +38 -0
- package/packages/piagent-webui/gateway/session-operation-start.ts +5 -1
- package/packages/piagent-webui/gateway/session-operation-watchdog.ts +165 -0
- package/packages/piagent-webui/gateway/session-runtime-factory.ts +45 -0
- package/packages/piagent-webui/gateway/session-runtime-supervisor.ts +141 -125
- package/packages/piagent-webui/ownership/session-lease-store.ts +5 -2
- package/packages/piagent-webui/server/core-inspection-provider.ts +2 -1
- package/packages/piagent-webui/server/loopback-server.ts +8 -0
- package/packages/piagent-webui/server/transcript-projection.ts +113 -9
- package/packages/piagent-webui/shared/text-visibility.ts +10 -0
- package/schemas/codex-relative-efficiency-v1.schema.json +155 -0
- package/schemas/piagent-webui/catalog-v1.json +9 -0
- package/schemas/piagent-webui/gateway-protocol-v1.schema.json +53 -3
- package/schemas/piagent-webui/runtime-event-v2.schema.json +15 -5
- package/schemas/piagent-webui/session-live-state-v1.schema.json +76 -0
- package/schemas/piagent-webui/subagent-tree-v1.schema.json +11 -1
- package/schemas/project-profile.schema.json +1 -1
- package/schemas/task-contract.schema.json +27 -1
- package/scripts/benchmark-runner-core.mjs +367 -363
- package/scripts/benchmark-runner-finalization.mjs +313 -0
- package/scripts/benchmark-runner-host-readiness.mjs +94 -0
- package/scripts/benchmark-runner-support.mjs +200 -0
- package/scripts/benchmark-session.mjs +95 -41
- package/scripts/configure-model-scope.sh +4 -3
- package/scripts/configure-subagents.sh +42 -34
- package/scripts/helper-evaluation.mjs +2 -2
- package/scripts/install-global.sh +2 -2
- package/scripts/pi-usage-history.mjs +37 -10
- package/scripts/setup.sh +2 -2
- package/scripts/team-doctor.sh +1 -1
- package/scripts/verify-local.sh +10 -1
- package/templates/global/settings.json +2 -1
- package/packages/piagent-webui/dist/client/assets/index-BZRFw1u6.js +0 -174
|
@@ -0,0 +1,456 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import fs from "node:fs";
|
|
3
|
+
import os from "node:os";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
import { fileURLToPath } from "node:url";
|
|
6
|
+
|
|
7
|
+
import {
|
|
8
|
+
auditToolProtocol,
|
|
9
|
+
minimumContextSavingsTokens,
|
|
10
|
+
projectAdaptiveContext
|
|
11
|
+
} from "../../packages/piagent-core/runtime/session/adaptive-context-governor.ts";
|
|
12
|
+
import {
|
|
13
|
+
BoundedEmissionGuard,
|
|
14
|
+
runtimeEventEmissionCandidate
|
|
15
|
+
} from "../../packages/piagent-core/runtime/inspection/bounded-emission-guard.ts";
|
|
16
|
+
import { EditFreshnessGuard } from "../../packages/piagent-core/runtime/quality/edit-freshness-guard.ts";
|
|
17
|
+
import {
|
|
18
|
+
SessionOperationLifecycle,
|
|
19
|
+
sessionOperationRetryPolicy
|
|
20
|
+
} from "../../packages/piagent-webui/gateway/session-operation-lifecycle.ts";
|
|
21
|
+
|
|
22
|
+
const laneRoot = path.dirname(fileURLToPath(import.meta.url));
|
|
23
|
+
const lane = JSON.parse(fs.readFileSync(path.join(laneRoot, "lane.json"), "utf8"));
|
|
24
|
+
const argumentsList = process.argv.slice(2);
|
|
25
|
+
const option = (name) => {
|
|
26
|
+
const index = argumentsList.indexOf(name);
|
|
27
|
+
return index >= 0 ? argumentsList[index + 1] : undefined;
|
|
28
|
+
};
|
|
29
|
+
const outputArgument = option("--output");
|
|
30
|
+
if (argumentsList.includes("--output") && !outputArgument) throw new Error("--output requires a path");
|
|
31
|
+
const outputPath = outputArgument ? path.resolve(outputArgument) : null;
|
|
32
|
+
const startedAtMs = Date.now();
|
|
33
|
+
let timestamp = 1;
|
|
34
|
+
|
|
35
|
+
function user(text) {
|
|
36
|
+
return { role: "user", content: text, timestamp: timestamp++ };
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function toolRound(id, name, args, output) {
|
|
40
|
+
return [
|
|
41
|
+
{
|
|
42
|
+
role: "assistant",
|
|
43
|
+
content: [{ type: "toolCall", id, name, arguments: args }],
|
|
44
|
+
timestamp: timestamp++
|
|
45
|
+
},
|
|
46
|
+
{
|
|
47
|
+
role: "toolResult",
|
|
48
|
+
toolCallId: id,
|
|
49
|
+
toolName: name,
|
|
50
|
+
content: [{ type: "text", text: output }],
|
|
51
|
+
isError: false,
|
|
52
|
+
timestamp: timestamp++
|
|
53
|
+
}
|
|
54
|
+
];
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
function task() {
|
|
58
|
+
return {
|
|
59
|
+
taskId: "runtime-conformance",
|
|
60
|
+
taskRunId: "runtime-conformance-run-1",
|
|
61
|
+
summary: "Implement a frontend contract without changing backend behavior.",
|
|
62
|
+
riskLane: "normal",
|
|
63
|
+
acceptanceCriteria: ["Preserve behavior", "Pass focused verification"],
|
|
64
|
+
scope: ["src/**"],
|
|
65
|
+
outOfScope: ["backend mutation"],
|
|
66
|
+
changedFiles: ["src/page.tsx"],
|
|
67
|
+
verifyCommands: ["node --test"],
|
|
68
|
+
trace: { outcome: "pending" }
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
function boundedCandidate(text, updateKey, reference) {
|
|
73
|
+
return {
|
|
74
|
+
channel: "reviewer",
|
|
75
|
+
identityKey: "runtime-conformance-reviewer",
|
|
76
|
+
updateKey,
|
|
77
|
+
text,
|
|
78
|
+
actionable: true,
|
|
79
|
+
reference
|
|
80
|
+
};
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
function finalMessageCandidate() {
|
|
84
|
+
return runtimeEventEmissionCandidate({
|
|
85
|
+
kind: "message.completed",
|
|
86
|
+
sessionRef: "runtime-conformance-session",
|
|
87
|
+
taskRunId: "runtime-conformance-run-1",
|
|
88
|
+
agentOperationId: "runtime-conformance-operation",
|
|
89
|
+
turnIndex: 1,
|
|
90
|
+
toolCallId: null,
|
|
91
|
+
revision: { runtimeRevision: "runtime-revision-1" },
|
|
92
|
+
payload: { role: "assistant", textPreview: "Final response remains visible." }
|
|
93
|
+
}, "message-final");
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
const results = [];
|
|
97
|
+
const metrics = {
|
|
98
|
+
context: {
|
|
99
|
+
projections: 0,
|
|
100
|
+
noops: 0,
|
|
101
|
+
estimatedSavingsTokens: 0,
|
|
102
|
+
governorProviderCalls: 0,
|
|
103
|
+
toolProtocolOrphans: 0
|
|
104
|
+
},
|
|
105
|
+
operations: {
|
|
106
|
+
pristineRetriesAllowed: 0,
|
|
107
|
+
unsafeRetriesAllowed: 0,
|
|
108
|
+
unsafeRetriesAborted: 0,
|
|
109
|
+
terminalSettlements: 0,
|
|
110
|
+
duplicateTerminalSettlements: 0,
|
|
111
|
+
lateEventsIgnored: 0,
|
|
112
|
+
compactionRetryBudgetConsumed: 0
|
|
113
|
+
},
|
|
114
|
+
emissions: {
|
|
115
|
+
accepted: 0,
|
|
116
|
+
suppressed: 0,
|
|
117
|
+
retained: 0,
|
|
118
|
+
finalMessagesSuppressed: 0
|
|
119
|
+
},
|
|
120
|
+
edits: {
|
|
121
|
+
staleEvaluations: 0,
|
|
122
|
+
staleMutationsAllowed: 0,
|
|
123
|
+
freshRereadRecoveries: 0
|
|
124
|
+
}
|
|
125
|
+
};
|
|
126
|
+
|
|
127
|
+
function runCase(id, area, execute) {
|
|
128
|
+
const caseStartedAt = Date.now();
|
|
129
|
+
try {
|
|
130
|
+
const evidence = execute() ?? {};
|
|
131
|
+
results.push({ id, area, status: "passed", durationMilliseconds: Date.now() - caseStartedAt, evidence });
|
|
132
|
+
} catch (error) {
|
|
133
|
+
results.push({
|
|
134
|
+
id,
|
|
135
|
+
area,
|
|
136
|
+
status: "failed",
|
|
137
|
+
durationMilliseconds: Date.now() - caseStartedAt,
|
|
138
|
+
error: error instanceof Error ? error.message : String(error)
|
|
139
|
+
});
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
runCase("context-small-passthrough", "adaptive-context-governor", () => {
|
|
144
|
+
const messages = [user("Implement src/cart.ts"), ...toolRound("small-read", "read", { path: "src/cart.ts" }, "export const cart = true;")];
|
|
145
|
+
const projection = projectAdaptiveContext(messages, {
|
|
146
|
+
reportedTokens: 9_000,
|
|
147
|
+
contextWindow: 400_000,
|
|
148
|
+
task: task()
|
|
149
|
+
});
|
|
150
|
+
assert.equal(projection.decision.action, "passthrough");
|
|
151
|
+
assert.equal(projection.messages, messages);
|
|
152
|
+
assert.equal(projection.decision.accounting.billedTraffic.governorProviderCalls, 0);
|
|
153
|
+
metrics.context.governorProviderCalls += projection.decision.accounting.billedTraffic.governorProviderCalls;
|
|
154
|
+
return { action: projection.decision.action, originalMessages: messages.length };
|
|
155
|
+
});
|
|
156
|
+
|
|
157
|
+
runCase("context-insufficient-savings-noop", "adaptive-context-governor", () => {
|
|
158
|
+
const messages = [user("Continue the same implementation")];
|
|
159
|
+
for (let index = 0; index < 8; index += 1) {
|
|
160
|
+
messages.push(...toolRound(`noop-${index}`, "read", { path: `src/noop-${index}.ts` }, "x".repeat(6_000)));
|
|
161
|
+
}
|
|
162
|
+
const projection = projectAdaptiveContext(messages, {
|
|
163
|
+
reportedTokens: 250_000,
|
|
164
|
+
contextWindow: 400_000,
|
|
165
|
+
task: task()
|
|
166
|
+
});
|
|
167
|
+
assert.equal(projection.decision.action, "passthrough");
|
|
168
|
+
assert.equal(projection.decision.fallback, "insufficient-savings");
|
|
169
|
+
assert.equal(projection.messages, messages);
|
|
170
|
+
assert.ok(projection.decision.estimatedSavingsTokens < projection.decision.minimumSavingsTokens);
|
|
171
|
+
assert.equal(
|
|
172
|
+
projection.decision.minimumSavingsTokens,
|
|
173
|
+
minimumContextSavingsTokens(projection.decision.originalMessageTokens)
|
|
174
|
+
);
|
|
175
|
+
assert.equal(projection.decision.accounting.billedTraffic.governorProviderCalls, 0);
|
|
176
|
+
metrics.context.noops += 1;
|
|
177
|
+
metrics.context.governorProviderCalls += projection.decision.accounting.billedTraffic.governorProviderCalls;
|
|
178
|
+
return {
|
|
179
|
+
action: projection.decision.action,
|
|
180
|
+
fallback: projection.decision.fallback,
|
|
181
|
+
candidateSavingsTokens: projection.decision.estimatedSavingsTokens,
|
|
182
|
+
minimumSavingsTokens: projection.decision.minimumSavingsTokens
|
|
183
|
+
};
|
|
184
|
+
});
|
|
185
|
+
|
|
186
|
+
runCase("context-projection-protocol-safe", "adaptive-context-governor", () => {
|
|
187
|
+
const messages = [user("Scout the backend subscription contract")];
|
|
188
|
+
for (let index = 0; index < 14; index += 1) {
|
|
189
|
+
messages.push(...toolRound(
|
|
190
|
+
`project-${index}`,
|
|
191
|
+
"read",
|
|
192
|
+
{ path: `backend/src/module-${index}.ts` },
|
|
193
|
+
`source-${index}\n${"x".repeat(5_000)}`
|
|
194
|
+
));
|
|
195
|
+
}
|
|
196
|
+
messages.push(user("Proceed to implement the frontend subscription mapping"));
|
|
197
|
+
const projection = projectAdaptiveContext(messages, {
|
|
198
|
+
reportedTokens: 92_000,
|
|
199
|
+
contextWindow: 400_000,
|
|
200
|
+
task: task()
|
|
201
|
+
});
|
|
202
|
+
const protocol = auditToolProtocol(projection.messages);
|
|
203
|
+
assert.equal(projection.decision.action, "project");
|
|
204
|
+
assert.equal(projection.decision.reason, "semantic-boundary");
|
|
205
|
+
assert.equal(protocol.intact, true);
|
|
206
|
+
assert.equal(protocol.orphanCallIds.length + protocol.orphanResultIds.length, 0);
|
|
207
|
+
assert.ok(projection.decision.estimatedSavingsTokens >= projection.decision.minimumSavingsTokens);
|
|
208
|
+
assert.match(String(projection.messages.at(-1)?.content ?? ""), /implement the frontend subscription mapping/i);
|
|
209
|
+
assert.equal(projection.decision.accounting.billedTraffic.governorProviderCalls, 0);
|
|
210
|
+
metrics.context.projections += 1;
|
|
211
|
+
metrics.context.estimatedSavingsTokens += projection.decision.estimatedSavingsTokens;
|
|
212
|
+
metrics.context.governorProviderCalls += projection.decision.accounting.billedTraffic.governorProviderCalls;
|
|
213
|
+
metrics.context.toolProtocolOrphans += protocol.orphanCallIds.length + protocol.orphanResultIds.length;
|
|
214
|
+
return {
|
|
215
|
+
action: projection.decision.action,
|
|
216
|
+
reason: projection.decision.reason,
|
|
217
|
+
originalMessageTokens: projection.decision.originalMessageTokens,
|
|
218
|
+
projectedMessageTokens: projection.decision.projectedMessageTokens,
|
|
219
|
+
estimatedSavingsTokens: projection.decision.estimatedSavingsTokens,
|
|
220
|
+
savingsRatio: projection.decision.savingsRatio,
|
|
221
|
+
toolProtocolIntact: protocol.intact
|
|
222
|
+
};
|
|
223
|
+
});
|
|
224
|
+
|
|
225
|
+
runCase("context-orphan-protocol-fail-closed", "adaptive-context-governor", () => {
|
|
226
|
+
const messages = [user("Continue implementation")];
|
|
227
|
+
for (let index = 0; index < 8; index += 1) {
|
|
228
|
+
messages.push(...toolRound(`valid-${index}`, "read", { path: `src/valid-${index}.ts` }, "x".repeat(6_000)));
|
|
229
|
+
}
|
|
230
|
+
messages.push({
|
|
231
|
+
role: "toolResult",
|
|
232
|
+
toolCallId: "missing-call",
|
|
233
|
+
toolName: "read",
|
|
234
|
+
content: [{ type: "text", text: "orphan result" }],
|
|
235
|
+
timestamp: timestamp++
|
|
236
|
+
});
|
|
237
|
+
messages.push({
|
|
238
|
+
role: "assistant",
|
|
239
|
+
content: [{ type: "toolCall", id: "missing-result", name: "read", arguments: { path: "src/missing.ts" } }],
|
|
240
|
+
timestamp: timestamp++
|
|
241
|
+
});
|
|
242
|
+
const sourceProtocol = auditToolProtocol(messages);
|
|
243
|
+
const projection = projectAdaptiveContext(messages, {
|
|
244
|
+
reportedTokens: 250_000,
|
|
245
|
+
contextWindow: 400_000,
|
|
246
|
+
task: task()
|
|
247
|
+
});
|
|
248
|
+
assert.equal(sourceProtocol.intact, false);
|
|
249
|
+
assert.equal(projection.decision.action, "passthrough");
|
|
250
|
+
assert.equal(projection.decision.fallback, "unsafe-tool-protocol");
|
|
251
|
+
assert.equal(projection.messages, messages);
|
|
252
|
+
assert.ok(projection.decision.reasonCodes.includes("unsafe-tool-protocol"));
|
|
253
|
+
assert.equal(projection.decision.accounting.billedTraffic.governorProviderCalls, 0);
|
|
254
|
+
metrics.context.noops += 1;
|
|
255
|
+
metrics.context.governorProviderCalls += projection.decision.accounting.billedTraffic.governorProviderCalls;
|
|
256
|
+
return {
|
|
257
|
+
sourceOrphanCalls: sourceProtocol.orphanCallIds.length,
|
|
258
|
+
sourceOrphanResults: sourceProtocol.orphanResultIds.length,
|
|
259
|
+
projected: false,
|
|
260
|
+
fallback: projection.decision.fallback
|
|
261
|
+
};
|
|
262
|
+
});
|
|
263
|
+
|
|
264
|
+
runCase("operation-pristine-retry-bounded", "webui-operation-lifecycle", () => {
|
|
265
|
+
assert.deepEqual(sessionOperationRetryPolicy(), { maximumAttempts: 1, maximumDelayMs: 8_000 });
|
|
266
|
+
const lifecycle = new SessionOperationLifecycle({ operationRef: "operation-pristine" });
|
|
267
|
+
lifecycle.observe({ type: "agent_start", operationRef: "operation-pristine" });
|
|
268
|
+
const first = lifecycle.observe({
|
|
269
|
+
type: "auto_retry_start",
|
|
270
|
+
operationRef: "operation-pristine",
|
|
271
|
+
attempt: 1,
|
|
272
|
+
delayMs: 100
|
|
273
|
+
});
|
|
274
|
+
assert.equal(first.retry, "allowed");
|
|
275
|
+
metrics.operations.pristineRetriesAllowed += 1;
|
|
276
|
+
lifecycle.observe({ type: "auto_retry_end", operationRef: "operation-pristine", attempt: 1, success: false });
|
|
277
|
+
const second = lifecycle.observe({
|
|
278
|
+
type: "auto_retry_start",
|
|
279
|
+
operationRef: "operation-pristine",
|
|
280
|
+
attempt: 2,
|
|
281
|
+
delayMs: 100
|
|
282
|
+
});
|
|
283
|
+
assert.equal(second.retry, "abort");
|
|
284
|
+
assert.equal(second.reasonCode, "automatic-retry-attempt-limit");
|
|
285
|
+
return { maximumAttempts: lifecycle.retryPolicy.maximumAttempts, firstRetry: first.retry, secondRetry: second.retry };
|
|
286
|
+
});
|
|
287
|
+
|
|
288
|
+
runCase("operation-replay-unsafe-and-exactly-once", "webui-operation-lifecycle", () => {
|
|
289
|
+
const visible = new SessionOperationLifecycle({ operationRef: "operation-visible" });
|
|
290
|
+
visible.observe({ type: "agent_start" });
|
|
291
|
+
visible.observe({
|
|
292
|
+
type: "message_update",
|
|
293
|
+
message: { role: "assistant", content: [{ type: "text", text: "Visible output" }] },
|
|
294
|
+
assistantMessageEvent: { type: "text_delta", delta: "Visible output" }
|
|
295
|
+
});
|
|
296
|
+
const visibleRetry = visible.observe({ type: "auto_retry_start", attempt: 1, delayMs: 0 });
|
|
297
|
+
assert.equal(visibleRetry.retry, "abort");
|
|
298
|
+
assert.equal(visibleRetry.reasonCode, "automatic-retry-replay-unsafe");
|
|
299
|
+
|
|
300
|
+
const tool = new SessionOperationLifecycle({ operationRef: "operation-tool" });
|
|
301
|
+
tool.observe({ type: "agent_start" });
|
|
302
|
+
tool.observe({ type: "tool_execution_start", toolCallId: "write-1", toolName: "write" });
|
|
303
|
+
const toolRetry = tool.observe({ type: "auto_retry_start", attempt: 1, delayMs: 0 });
|
|
304
|
+
assert.equal(toolRetry.retry, "abort");
|
|
305
|
+
assert.equal(toolRetry.reasonCode, "automatic-retry-replay-unsafe");
|
|
306
|
+
metrics.operations.unsafeRetriesAborted += 2;
|
|
307
|
+
|
|
308
|
+
assert.equal(tool.markTerminal(), true);
|
|
309
|
+
metrics.operations.terminalSettlements += 1;
|
|
310
|
+
const duplicateTerminal = tool.markTerminal();
|
|
311
|
+
assert.equal(duplicateTerminal, false);
|
|
312
|
+
if (duplicateTerminal) metrics.operations.duplicateTerminalSettlements += 1;
|
|
313
|
+
const late = tool.observe({ type: "message_update", assistantMessageEvent: { type: "text_delta", delta: "late" } });
|
|
314
|
+
assert.equal(late.accepted, false);
|
|
315
|
+
assert.equal(late.reasonCode, "operation-event-after-settlement");
|
|
316
|
+
metrics.operations.lateEventsIgnored += 1;
|
|
317
|
+
return {
|
|
318
|
+
visibleRetry: visibleRetry.retry,
|
|
319
|
+
toolRetry: toolRetry.retry,
|
|
320
|
+
terminalSettlements: 1,
|
|
321
|
+
duplicateTerminalSettlements: 0,
|
|
322
|
+
lateEvent: late.reasonCode
|
|
323
|
+
};
|
|
324
|
+
});
|
|
325
|
+
|
|
326
|
+
runCase("operation-compaction-isolated", "webui-operation-lifecycle", () => {
|
|
327
|
+
const lifecycle = new SessionOperationLifecycle({ operationRef: "operation-compaction" });
|
|
328
|
+
lifecycle.observe({ type: "agent_start" });
|
|
329
|
+
const start = lifecycle.observe({ type: "compaction_start", reason: "overflow" });
|
|
330
|
+
assert.equal(start.phase, "compaction");
|
|
331
|
+
const summarization = lifecycle.observe({ type: "summarization_retry_scheduled", attempt: 1, delayMs: 5 });
|
|
332
|
+
assert.equal(summarization.accepted, true);
|
|
333
|
+
assert.equal(summarization.retry, "none");
|
|
334
|
+
const end = lifecycle.observe({ type: "compaction_end", reason: "overflow" });
|
|
335
|
+
assert.equal(end.phase, "running");
|
|
336
|
+
const providerRetry = lifecycle.observe({ type: "auto_retry_start", attempt: 1, delayMs: 5 });
|
|
337
|
+
assert.equal(providerRetry.retry, "allowed");
|
|
338
|
+
metrics.operations.pristineRetriesAllowed += 1;
|
|
339
|
+
return { compactionPhase: start.phase, summarizationRetry: summarization.retry, providerRetry: providerRetry.retry };
|
|
340
|
+
});
|
|
341
|
+
|
|
342
|
+
runCase("emission-bounded-final-preserved", "bounded-emission-guard", () => {
|
|
343
|
+
const guard = new BoundedEmissionGuard({ capacity: 2 });
|
|
344
|
+
const filler = guard.inspect(boundedCandidate("No issue; continue.", "update-0", "review-0"));
|
|
345
|
+
assert.equal(filler.reason, "filler");
|
|
346
|
+
|
|
347
|
+
const first = boundedCandidate("Potential race in queue.", "update-1", "review-1");
|
|
348
|
+
assert.equal(guard.inspect(first).accepted, true);
|
|
349
|
+
guard.recordAccepted(first);
|
|
350
|
+
assert.equal(guard.inspect(boundedCandidate("Potential race in queue.", "update-2", "review-2")).reason, "duplicate-exact");
|
|
351
|
+
assert.equal(guard.inspect(boundedCandidate("potential RACE in queue!", "update-3", "review-3")).reason, "duplicate-normalized");
|
|
352
|
+
|
|
353
|
+
const second = boundedCandidate("Mutation lacks a verifier.", "update-4", "review-4");
|
|
354
|
+
assert.equal(guard.inspect(second).accepted, true);
|
|
355
|
+
guard.recordAccepted(second);
|
|
356
|
+
const third = boundedCandidate("Late activity remains bounded.", "update-5", "review-5");
|
|
357
|
+
assert.equal(guard.inspect(third).accepted, true);
|
|
358
|
+
guard.recordAccepted(third);
|
|
359
|
+
|
|
360
|
+
const telemetry = guard.telemetry();
|
|
361
|
+
assert.equal(telemetry.retained, 2);
|
|
362
|
+
assert.equal(finalMessageCandidate(), null, "final assistant messages must bypass suppression");
|
|
363
|
+
metrics.emissions.accepted += telemetry.accepted;
|
|
364
|
+
metrics.emissions.suppressed += telemetry.suppressed;
|
|
365
|
+
metrics.emissions.retained = telemetry.retained;
|
|
366
|
+
return {
|
|
367
|
+
capacity: telemetry.capacity,
|
|
368
|
+
retained: telemetry.retained,
|
|
369
|
+
accepted: telemetry.accepted,
|
|
370
|
+
suppressed: telemetry.suppressed,
|
|
371
|
+
suppressedByReason: telemetry.suppressedByReason,
|
|
372
|
+
finalMessageBypassesGuard: true
|
|
373
|
+
};
|
|
374
|
+
});
|
|
375
|
+
|
|
376
|
+
runCase("edit-stale-blocked-until-reread", "edit-freshness-guard", () => {
|
|
377
|
+
const cwd = fs.mkdtempSync(path.join(os.tmpdir(), "piagent-runtime-conformance-edit-"));
|
|
378
|
+
try {
|
|
379
|
+
fs.mkdirSync(path.join(cwd, "src"), { recursive: true });
|
|
380
|
+
const target = path.join(cwd, "src", "value.ts");
|
|
381
|
+
fs.writeFileSync(target, "export const value = 1;\n");
|
|
382
|
+
const ctx = { cwd, sessionManager: { getSessionId: () => "runtime-conformance-session" } };
|
|
383
|
+
const guard = new EditFreshnessGuard("enforce");
|
|
384
|
+
assert.deepEqual(guard.observe(ctx, "runtime-conformance-run-1", ["src/value.ts"], "read"), ["src/value.ts"]);
|
|
385
|
+
assert.equal(guard.evaluate(ctx, "runtime-conformance-run-1", ["src/value.ts"]).decision, "current");
|
|
386
|
+
|
|
387
|
+
fs.writeFileSync(target, "// concurrent change\nexport const value = 1;\n");
|
|
388
|
+
const beforeRejectedMutation = fs.readFileSync(target, "utf8");
|
|
389
|
+
const stale = guard.evaluate(ctx, "runtime-conformance-run-1", ["src/value.ts"]);
|
|
390
|
+
assert.equal(stale.decision, "stale");
|
|
391
|
+
assert.equal(stale.enforce, true);
|
|
392
|
+
assert.deepEqual(stale.stalePaths, ["src/value.ts"]);
|
|
393
|
+
assert.equal(fs.readFileSync(target, "utf8"), beforeRejectedMutation, "evaluation must not mutate stale content");
|
|
394
|
+
metrics.edits.staleEvaluations += 1;
|
|
395
|
+
|
|
396
|
+
guard.observe(ctx, "runtime-conformance-run-1", ["src/value.ts"], "read");
|
|
397
|
+
const recovered = guard.evaluate(ctx, "runtime-conformance-run-1", ["src/value.ts"]);
|
|
398
|
+
assert.equal(recovered.decision, "current");
|
|
399
|
+
metrics.edits.freshRereadRecoveries += 1;
|
|
400
|
+
return { staleDecision: stale.decision, stalePaths: stale.stalePaths, recoveredDecision: recovered.decision };
|
|
401
|
+
} finally {
|
|
402
|
+
fs.rmSync(cwd, { recursive: true, force: true });
|
|
403
|
+
}
|
|
404
|
+
});
|
|
405
|
+
|
|
406
|
+
const completedAtMs = Date.now();
|
|
407
|
+
const failed = results.filter((result) => result.status !== "passed");
|
|
408
|
+
const configuredCaseIds = lane.cases.map((item) => item.id);
|
|
409
|
+
const observedCaseIds = results.map((item) => item.id);
|
|
410
|
+
const configuredCasesMatched = JSON.stringify(configuredCaseIds) === JSON.stringify(observedCaseIds);
|
|
411
|
+
const gates = {
|
|
412
|
+
allCasesPass: failed.length === 0 && configuredCasesMatched,
|
|
413
|
+
providerCalls: metrics.context.governorProviderCalls === lane.releaseGates.providerCalls,
|
|
414
|
+
modelInputTokens: lane.releaseGates.modelInputTokens === 0,
|
|
415
|
+
modelOutputTokens: lane.releaseGates.modelOutputTokens === 0,
|
|
416
|
+
toolProtocolOrphans: metrics.context.toolProtocolOrphans === lane.releaseGates.toolProtocolOrphans,
|
|
417
|
+
unsafeRetriesAllowed: metrics.operations.unsafeRetriesAllowed === lane.releaseGates.unsafeRetriesAllowed,
|
|
418
|
+
duplicateTerminalSettlements: metrics.operations.duplicateTerminalSettlements === lane.releaseGates.duplicateTerminalSettlements,
|
|
419
|
+
finalMessagesSuppressed: metrics.emissions.finalMessagesSuppressed === lane.releaseGates.finalMessagesSuppressed,
|
|
420
|
+
staleMutationsAllowed: metrics.edits.staleMutationsAllowed === lane.releaseGates.staleMutationsAllowed
|
|
421
|
+
};
|
|
422
|
+
const passed = failed.length === 0 && configuredCasesMatched && Object.values(gates).every(Boolean);
|
|
423
|
+
const report = {
|
|
424
|
+
schemaVersion: 1,
|
|
425
|
+
laneId: lane.id,
|
|
426
|
+
evidenceClass: "provider-free-runtime-conformance",
|
|
427
|
+
startedAt: new Date(startedAtMs).toISOString(),
|
|
428
|
+
completedAt: new Date(completedAtMs).toISOString(),
|
|
429
|
+
wallClockMilliseconds: completedAtMs - startedAtMs,
|
|
430
|
+
provider: {
|
|
431
|
+
required: lane.providerRequired,
|
|
432
|
+
used: false,
|
|
433
|
+
calls: 0,
|
|
434
|
+
modelInputTokens: 0,
|
|
435
|
+
modelOutputTokens: 0
|
|
436
|
+
},
|
|
437
|
+
summary: {
|
|
438
|
+
configuredCases: lane.cases.length,
|
|
439
|
+
executedCases: results.length,
|
|
440
|
+
passedCases: results.length - failed.length,
|
|
441
|
+
failedCases: failed.length,
|
|
442
|
+
passed
|
|
443
|
+
},
|
|
444
|
+
metrics,
|
|
445
|
+
gates,
|
|
446
|
+
cases: results,
|
|
447
|
+
claimBoundary: lane.claimBoundary
|
|
448
|
+
};
|
|
449
|
+
|
|
450
|
+
const serialized = `${JSON.stringify(report, null, 2)}\n`;
|
|
451
|
+
if (outputPath) {
|
|
452
|
+
fs.mkdirSync(path.dirname(outputPath), { recursive: true });
|
|
453
|
+
fs.writeFileSync(outputPath, serialized);
|
|
454
|
+
}
|
|
455
|
+
process.stdout.write(serialized);
|
|
456
|
+
if (!passed) process.exitCode = 1;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@piagent/platform",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.6.1",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"publishConfig": {
|
|
6
6
|
"access": "public",
|
|
@@ -59,6 +59,7 @@
|
|
|
59
59
|
"packages/piagent-webui/extension/",
|
|
60
60
|
"packages/piagent-webui/gateway/",
|
|
61
61
|
"packages/piagent-webui/ownership/",
|
|
62
|
+
"packages/piagent-webui/shared/",
|
|
62
63
|
"packages/piagent-webui/dist/client/",
|
|
63
64
|
"schemas/",
|
|
64
65
|
"scripts/",
|
|
@@ -109,6 +110,7 @@
|
|
|
109
110
|
"piagent": {
|
|
110
111
|
"runtimeIntegrityEntrypoints": [
|
|
111
112
|
"packages/piagent-webui/server/sidecar-main.ts",
|
|
113
|
+
"scripts/benchmark-runner-core.mjs",
|
|
112
114
|
"scripts/register-typescript-loader.mjs",
|
|
113
115
|
"scripts/typescript-loader.mjs"
|
|
114
116
|
]
|
|
@@ -22,8 +22,11 @@ Architecture boundaries and non-growth file budgets are enforced by `npm run arc
|
|
|
22
22
|
The root package exposes `piagent-benchmark` for an automatic paired Piagent
|
|
23
23
|
steady-state benchmark against Raw Pi or the `codex-cli` surface. Treatment
|
|
24
24
|
onboarding/context preparation happens before measured model usage;
|
|
25
|
-
post-baseline mutations remain
|
|
26
|
-
|
|
25
|
+
post-baseline mutations remain auditable. Task scope is an advisory retrieval
|
|
26
|
+
and review focus rather than filesystem authority; Codex can follow repository
|
|
27
|
+
evidence across packages and child repositories. Protected/read-only paths,
|
|
28
|
+
destructive commands, and external actions remain governed. Codex runs through a
|
|
29
|
+
strict streaming JSONL adapter and defaults to controlled ephemeral execution with an
|
|
27
30
|
isolated temporary Codex home. Run
|
|
28
31
|
`piagent-benchmark --dry-run` before starting billed model sessions; see
|
|
29
32
|
`docs/quality-benchmark.md` for score and verdict rules.
|
|
@@ -130,10 +133,15 @@ When `pi-subagents` is installed, this package exposes:
|
|
|
130
133
|
|
|
131
134
|
- `piagent-scout`
|
|
132
135
|
- `piagent-planner`
|
|
133
|
-
- `piagent-worker`
|
|
136
|
+
- `piagent-worker` (compatibility metadata; disabled by Piagent config)
|
|
134
137
|
- `piagent-reviewer`
|
|
135
138
|
- `piagent-oracle`
|
|
136
139
|
|
|
140
|
+
The parent model works directly. Helpers default to `off`; explicit opt-in still
|
|
141
|
+
allows only one fresh read-only helper after a runtime-owned estimate proves at
|
|
142
|
+
least 30% projected net token saving. Worker, parallel, nested, and retry paths
|
|
143
|
+
remain disabled.
|
|
144
|
+
|
|
137
145
|
## Install
|
|
138
146
|
|
|
139
147
|
```bash
|
|
@@ -197,7 +205,8 @@ rebuild and database vacuum before the new policy is considered clean.
|
|
|
197
205
|
Passing source final gates require an observed exit `0` result for every
|
|
198
206
|
meaningful entry in `task.verifyCommands`. Other observed commands are traceable
|
|
199
207
|
but advisory. Declared changed files must also differ from the task baseline,
|
|
200
|
-
match observed tool results
|
|
208
|
+
and match observed tool results. Changes beyond `task.scope` are retained as an
|
|
209
|
+
auditable focus expansion and do not block source mutation or completion.
|
|
201
210
|
|
|
202
211
|
Raw path-like tool access to protected paths is blocked before execution. This includes Pi built-ins (`read`, `write`, `edit`, `grep`, `find`, `ls`) and custom/MCP tools with nested path-like strings, arrays, or `file://` URIs. Path-like strings are percent-decoded once, and input nesting above `MAX_TOOL_INPUT_INSPECTION_DEPTH=32` fails closed. Known content fields such as `content`, `query`, `pattern`, `text`, and `command` are excluded from generic extraction. `grep.glob` and `find.pattern` are checked when they explicitly target protected paths, while broad `grep`, `find`, and `ls` results are filtered so protected content lines or path metadata are redacted before reaching the model.
|
|
203
212
|
|
|
@@ -181,12 +181,15 @@ export function benchmarkClaimEligibility({ suite, environment = {}, baselineSur
|
|
|
181
181
|
if (baselineSurface === "codex-cli") limitations.push("codex-comparison-is-external-and-non-causal");
|
|
182
182
|
if ((suite.defaultRepeats ?? 0) <= 3) limitations.push("three-repeats-do-not-support-an-always-claim");
|
|
183
183
|
if (assurance.generatedVariants === true && assurance.familyDisjointSplit !== true) limitations.push("generated-value-variants-are-not-independent-task-families");
|
|
184
|
+
const tokenClaimScope = suite.releaseGate?.primaryEfficiencyEstimand === "fixed-workload-family-ratio"
|
|
185
|
+
? "bounded-to-predeclared-fixed-workload-families"
|
|
186
|
+
: "bounded-to-observed-comparable-pairs-and-failure-aware-effort";
|
|
184
187
|
return {
|
|
185
188
|
declaredTier, achievedTier, comparisonPurpose,
|
|
186
189
|
causalAttributionAllowed: comparisonPurpose === "causal-harness-ablation" && protocolPassed,
|
|
187
190
|
generalizationClaimAllowed: privateHoldoutEligible,
|
|
188
191
|
productionStabilityClaimAllowed: productionShadowEligible,
|
|
189
|
-
tokenClaimScope: tokenClaimAllowed ?
|
|
192
|
+
tokenClaimScope: tokenClaimAllowed ? tokenClaimScope : "unavailable",
|
|
190
193
|
privateHoldoutChecks: privateChecks,
|
|
191
194
|
limitations
|
|
192
195
|
};
|