@velum-labs/routekit-eval-setup 1.3.2 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/authoring-responses-request.d.ts +5 -0
- package/dist/adapters/authoring-responses-request.js +38 -0
- package/dist/adapters/evaluation-evidence-freshness.d.ts +7 -0
- package/dist/adapters/evaluation-evidence-freshness.js +76 -0
- package/dist/adapters/git-task-history.d.ts +67 -0
- package/dist/adapters/git-task-history.js +171 -0
- package/dist/adapters/integrated-repository-history.d.ts +21 -0
- package/dist/adapters/integrated-repository-history.js +175 -0
- package/dist/adapters/repository-command-diagnostic.d.ts +8 -0
- package/dist/adapters/repository-command-diagnostic.js +46 -0
- package/dist/adapters/repository-command-evidence.d.ts +13 -0
- package/dist/adapters/repository-command-evidence.js +102 -0
- package/dist/adapters/repository-command-runner.d.ts +238 -0
- package/dist/adapters/repository-command-runner.js +1483 -0
- package/dist/adapters/repository-import-context.d.ts +47 -0
- package/dist/adapters/repository-import-context.js +469 -0
- package/dist/adapters/repository-node-test-reporter.d.ts +3 -0
- package/dist/adapters/repository-node-test-reporter.js +27 -0
- package/dist/adapters/repository-review-evidence.d.ts +39 -0
- package/dist/adapters/repository-review-evidence.js +632 -0
- package/dist/adapters/repository-seed-selection.d.ts +7 -0
- package/dist/adapters/repository-seed-selection.js +79 -0
- package/dist/adapters/repository-solution-edits.d.ts +49 -0
- package/dist/adapters/repository-solution-edits.js +136 -0
- package/dist/adapters/repository-vitest-phase-adapter.d.ts +8 -0
- package/dist/adapters/repository-vitest-phase-adapter.js +310 -0
- package/dist/adapters/repository-vitest-reporter.d.ts +24 -0
- package/dist/adapters/repository-vitest-reporter.js +314 -0
- package/dist/adapters/strict-authoring-schema.d.ts +5 -0
- package/dist/adapters/strict-authoring-schema.js +158 -0
- package/dist/adapters/test-discovery.d.ts +30 -0
- package/dist/adapters/test-discovery.js +124 -0
- package/dist/adapters/typescript-repository-index.d.ts +51 -0
- package/dist/adapters/typescript-repository-index.js +226 -0
- package/dist/agentic-capabilities-protocol.d.ts +1373 -0
- package/dist/agentic-capabilities-protocol.js +786 -0
- package/dist/case-checkpoint-store.d.ts +29 -0
- package/dist/case-checkpoint-store.js +133 -0
- package/dist/case-pipeline-protocol-v2.d.ts +184 -0
- package/dist/case-pipeline-protocol-v2.js +193 -0
- package/dist/case-pipeline-protocol.d.ts +2626 -0
- package/dist/case-pipeline-protocol.js +371 -0
- package/dist/effect-api.d.ts +74 -10
- package/dist/effect-api.js +56 -6
- package/dist/errors.d.ts +31 -0
- package/dist/errors.js +10 -0
- package/dist/eval-capability-execution-envelope.d.ts +64 -0
- package/dist/eval-capability-execution-envelope.js +98 -0
- package/dist/eval-capability-policy.d.ts +90 -0
- package/dist/eval-capability-policy.js +107 -0
- package/dist/eval-event-log.d.ts +140 -0
- package/dist/eval-event-log.js +220 -0
- package/dist/evaluation-authoring-policy.d.ts +18 -0
- package/dist/evaluation-authoring-policy.js +19 -0
- package/dist/evaluation-authoring-validation.d.ts +22 -0
- package/dist/evaluation-authoring-validation.js +72 -0
- package/dist/evaluation-evidence.d.ts +20 -0
- package/dist/evaluation-evidence.js +319 -0
- package/dist/evaluation-grader-calibration-protocol.d.ts +108 -0
- package/dist/evaluation-grader-calibration-protocol.js +80 -0
- package/dist/evaluation-grader-calibration.d.ts +18 -0
- package/dist/evaluation-grader-calibration.js +334 -0
- package/dist/evaluation-grading-policy.d.ts +24 -0
- package/dist/evaluation-grading-policy.js +54 -0
- package/dist/evaluation-proposal-policy.d.ts +4 -0
- package/dist/evaluation-proposal-policy.js +91 -0
- package/dist/evaluation-source-retrieval.d.ts +68 -0
- package/dist/evaluation-source-retrieval.js +513 -0
- package/dist/evaluation-structure-policy.d.ts +29 -0
- package/dist/evaluation-structure-policy.js +138 -0
- package/dist/index.d.ts +124 -17
- package/dist/index.js +69 -11
- package/dist/inspection.js +2 -3
- package/dist/project-artifacts.d.ts +7 -2
- package/dist/project-artifacts.js +49 -136
- package/dist/project-authoring.d.ts +66 -5
- package/dist/project-authoring.js +783 -109
- package/dist/project-contracts.d.ts +419 -84
- package/dist/project-contracts.js +160 -52
- package/dist/project-store.js +2 -1
- package/dist/project-workflow.d.ts +5 -4
- package/dist/project-workflow.js +154 -35
- package/dist/repository-adversary-protocol.d.ts +64 -0
- package/dist/repository-adversary-protocol.js +105 -0
- package/dist/repository-behavior-protocol.d.ts +188 -0
- package/dist/repository-behavior-protocol.js +202 -0
- package/dist/repository-benchmark-protocol.d.ts +487 -0
- package/dist/repository-benchmark-protocol.js +96 -0
- package/dist/repository-execution-protocol.d.ts +150 -0
- package/dist/repository-execution-protocol.js +38 -0
- package/dist/repository-fixture-instructions.d.ts +3 -0
- package/dist/repository-fixture-instructions.js +91 -0
- package/dist/repository-fixture-protocol.d.ts +79 -0
- package/dist/repository-fixture-protocol.js +79 -0
- package/dist/repository-foundry-plan-protocol.d.ts +118 -0
- package/dist/repository-foundry-plan-protocol.js +296 -0
- package/dist/repository-foundry-progress-protocol.d.ts +52 -0
- package/dist/repository-foundry-progress-protocol.js +52 -0
- package/dist/repository-improvement-protocol.d.ts +100 -0
- package/dist/repository-improvement-protocol.js +106 -0
- package/dist/repository-language-model-protocol.d.ts +43 -0
- package/dist/repository-language-model-protocol.js +146 -0
- package/dist/repository-oracle-coverage-protocol.d.ts +18 -0
- package/dist/repository-oracle-coverage-protocol.js +39 -0
- package/dist/repository-oracle-execution-binding.d.ts +27 -0
- package/dist/repository-oracle-execution-binding.js +59 -0
- package/dist/repository-oracle-protocol.d.ts +230 -0
- package/dist/repository-oracle-protocol.js +156 -0
- package/dist/repository-oracle-scope-policy.d.ts +22 -0
- package/dist/repository-oracle-scope-policy.js +92 -0
- package/dist/repository-quality-policy.d.ts +15 -0
- package/dist/repository-quality-policy.js +357 -0
- package/dist/repository-routing-benchmark-protocol.d.ts +176 -0
- package/dist/repository-routing-benchmark-protocol.js +103 -0
- package/dist/repository-routing-model-protocol.d.ts +36 -0
- package/dist/repository-routing-model-protocol.js +89 -0
- package/dist/repository-routing-plan-protocol.d.ts +112 -0
- package/dist/repository-routing-plan-protocol.js +58 -0
- package/dist/repository-routing-quality-policy.d.ts +9 -0
- package/dist/repository-routing-quality-policy.js +191 -0
- package/dist/repository-seed-qualification-progress-protocol.d.ts +205 -0
- package/dist/repository-seed-qualification-progress-protocol.js +28 -0
- package/dist/repository-semantic-calibration-protocol.d.ts +768 -0
- package/dist/repository-semantic-calibration-protocol.js +276 -0
- package/dist/repository-semantic-calibration.d.ts +163 -0
- package/dist/repository-semantic-calibration.js +581 -0
- package/dist/repository-specification-contract-facts-protocol.d.ts +224 -0
- package/dist/repository-specification-contract-facts-protocol.js +276 -0
- package/dist/repository-specification-critique-protocol.d.ts +189 -0
- package/dist/repository-specification-critique-protocol.js +103 -0
- package/dist/repository-task-family-protocol.d.ts +24 -0
- package/dist/repository-task-family-protocol.js +37 -0
- package/dist/repository-task-seed-protocol.d.ts +384 -0
- package/dist/repository-task-seed-protocol.js +236 -0
- package/dist/repository-trajectory-protocol.d.ts +20 -0
- package/dist/repository-trajectory-protocol.js +42 -0
- package/dist/service.js +1 -1
- package/dist/services/adversary/service.d.ts +64 -0
- package/dist/services/adversary/service.js +330 -0
- package/dist/services/benchmark-compiler/service.d.ts +450 -0
- package/dist/services/benchmark-compiler/service.js +9 -0
- package/dist/services/budgeted-model/service.d.ts +118 -0
- package/dist/services/budgeted-model/service.js +460 -0
- package/dist/services/case-authoring/service.d.ts +163 -0
- package/dist/services/case-authoring/service.js +1456 -0
- package/dist/services/case-finalization/service.d.ts +283 -0
- package/dist/services/case-finalization/service.js +370 -0
- package/dist/services/case-generation/service.d.ts +619 -0
- package/dist/services/case-generation/service.js +2628 -0
- package/dist/services/case-pipeline/service.d.ts +31 -0
- package/dist/services/case-pipeline/service.js +485 -0
- package/dist/services/case-pipeline-v2/service.d.ts +70 -0
- package/dist/services/case-pipeline-v2/service.js +477 -0
- package/dist/services/command-observability/service.d.ts +13 -0
- package/dist/services/command-observability/service.js +3 -0
- package/dist/services/dimension-labeling/service.d.ts +77 -0
- package/dist/services/dimension-labeling/service.js +188 -0
- package/dist/services/eval-candidate/service.d.ts +208 -0
- package/dist/services/eval-candidate/service.js +64 -0
- package/dist/services/eval-capabilities/service.d.ts +183 -0
- package/dist/services/eval-capabilities/service.js +1433 -0
- package/dist/services/eval-environment/service.d.ts +173 -0
- package/dist/services/eval-environment/service.js +127 -0
- package/dist/services/evidence-reconstruction/service.d.ts +36 -0
- package/dist/services/evidence-reconstruction/service.js +145 -0
- package/dist/services/fixture-builder/service.d.ts +62 -0
- package/dist/services/fixture-builder/service.js +36 -0
- package/dist/services/fixture-validation/service.d.ts +75 -0
- package/dist/services/fixture-validation/service.js +295 -0
- package/dist/services/foundry/service.d.ts +831 -0
- package/dist/services/foundry/service.js +442 -0
- package/dist/services/foundry-progress/service.d.ts +62 -0
- package/dist/services/foundry-progress/service.js +149 -0
- package/dist/services/foundry-v2/service.d.ts +54 -0
- package/dist/services/foundry-v2/service.js +28 -0
- package/dist/services/grounded-authoring/service.d.ts +126 -0
- package/dist/services/grounded-authoring/service.js +822 -0
- package/dist/services/historical-case/service.d.ts +722 -0
- package/dist/services/historical-case/service.js +177 -0
- package/dist/services/improvement-loop/service.d.ts +59 -0
- package/dist/services/improvement-loop/service.js +176 -0
- package/dist/services/language-model/service.d.ts +52 -0
- package/dist/services/language-model/service.js +194 -0
- package/dist/services/oracle-builder/service.d.ts +146 -0
- package/dist/services/oracle-builder/service.js +513 -0
- package/dist/services/oracle-coverage/service.d.ts +28 -0
- package/dist/services/oracle-coverage/service.js +50 -0
- package/dist/services/oracle-coverage-witness/service.d.ts +130 -0
- package/dist/services/oracle-coverage-witness/service.js +538 -0
- package/dist/services/pipeline-challenge/service.d.ts +551 -0
- package/dist/services/pipeline-challenge/service.js +427 -0
- package/dist/services/pipeline-controls/service.d.ts +130 -0
- package/dist/services/pipeline-controls/service.js +483 -0
- package/dist/services/pipeline-oracle/service.d.ts +8 -0
- package/dist/services/pipeline-oracle/service.js +256 -0
- package/dist/services/pipeline-seed/service.d.ts +298 -0
- package/dist/services/pipeline-seed/service.js +428 -0
- package/dist/services/pipeline-spec/service.d.ts +103 -0
- package/dist/services/pipeline-spec/service.js +619 -0
- package/dist/services/pipeline-tournament/service.d.ts +258 -0
- package/dist/services/pipeline-tournament/service.js +476 -0
- package/dist/services/quality-gate/service.d.ts +233 -0
- package/dist/services/quality-gate/service.js +136 -0
- package/dist/services/repository-bundle/service.d.ts +33 -0
- package/dist/services/repository-bundle/service.js +114 -0
- package/dist/services/repository-model/service.d.ts +105 -0
- package/dist/services/repository-model/service.js +250 -0
- package/dist/services/repository-public-artifact/service.d.ts +133 -0
- package/dist/services/repository-public-artifact/service.js +330 -0
- package/dist/services/routing-benchmark/service.d.ts +362 -0
- package/dist/services/routing-benchmark/service.js +96 -0
- package/dist/services/specification-critic/service.d.ts +92 -0
- package/dist/services/specification-critic/service.js +172 -0
- package/dist/services/task-family/service.d.ts +40 -0
- package/dist/services/task-family/service.js +55 -0
- package/dist/services/task-seed/service.d.ts +906 -0
- package/dist/services/task-seed/service.js +1406 -0
- package/dist/services/task-specification/service.d.ts +27 -0
- package/dist/services/task-specification/service.js +40 -0
- package/dist/services/trajectory-policy/service.d.ts +110 -0
- package/dist/services/trajectory-policy/service.js +216 -0
- package/dist/test/agentic-capabilities-protocol.test.d.ts +1 -0
- package/dist/test/agentic-capabilities-protocol.test.js +570 -0
- package/dist/test/agentic-capabilities.test.d.ts +1 -0
- package/dist/test/agentic-capabilities.test.js +1461 -0
- package/dist/test/agentic-environment.test.d.ts +1 -0
- package/dist/test/agentic-environment.test.js +213 -0
- package/dist/test/case-pipeline-foundation.test.d.ts +1 -0
- package/dist/test/case-pipeline-foundation.test.js +535 -0
- package/dist/test/case-pipeline-protocol-v2.test.d.ts +1 -0
- package/dist/test/case-pipeline-protocol-v2.test.js +124 -0
- package/dist/test/case-pipeline-v2.test.d.ts +1 -0
- package/dist/test/case-pipeline-v2.test.js +286 -0
- package/dist/test/case-pipeline.test.d.ts +1 -0
- package/dist/test/case-pipeline.test.js +851 -0
- package/dist/test/eval-capability-policy.test.d.ts +1 -0
- package/dist/test/eval-capability-policy.test.js +50 -0
- package/dist/test/eval-event-log.test.d.ts +1 -0
- package/dist/test/eval-event-log.test.js +125 -0
- package/dist/test/evaluation-evidence-freshness.test.d.ts +1 -0
- package/dist/test/evaluation-evidence-freshness.test.js +44 -0
- package/dist/test/evaluation-evidence.test.d.ts +1 -0
- package/dist/test/evaluation-evidence.test.js +230 -0
- package/dist/test/evaluation-grader-calibration.test.d.ts +1 -0
- package/dist/test/evaluation-grader-calibration.test.js +373 -0
- package/dist/test/evaluation-proposal-digest.test.d.ts +1 -0
- package/dist/test/evaluation-proposal-digest.test.js +187 -0
- package/dist/test/evaluation-source-retrieval.test.d.ts +1 -0
- package/dist/test/evaluation-source-retrieval.test.js +237 -0
- package/dist/test/evaluation-structure-policy.test.d.ts +1 -0
- package/dist/test/evaluation-structure-policy.test.js +196 -0
- package/dist/test/fixtures/repository-resource-panel.d.ts +39 -0
- package/dist/test/fixtures/repository-resource-panel.js +111 -0
- package/dist/test/fixtures/vitest-boundary-panel.d.ts +84 -0
- package/dist/test/fixtures/vitest-boundary-panel.js +120 -0
- package/dist/test/fixtures/vitest-phase-panel.d.ts +135 -0
- package/dist/test/fixtures/vitest-phase-panel.js +213 -0
- package/dist/test/fixtures/vitest-reporter-results.d.ts +76 -0
- package/dist/test/fixtures/vitest-reporter-results.js +94 -0
- package/dist/test/grounded-authoring.test.d.ts +1 -0
- package/dist/test/grounded-authoring.test.js +565 -0
- package/dist/test/integrated-repository-history.test.d.ts +1 -0
- package/dist/test/integrated-repository-history.test.js +227 -0
- package/dist/test/project-authoring.test.js +593 -43
- package/dist/test/project-workflow.test.js +419 -40
- package/dist/test/repository-authoring-artifacts.test.d.ts +1 -0
- package/dist/test/repository-authoring-artifacts.test.js +185 -0
- package/dist/test/repository-bundle.test.d.ts +1 -0
- package/dist/test/repository-bundle.test.js +52 -0
- package/dist/test/repository-case-generation.test.d.ts +1 -0
- package/dist/test/repository-case-generation.test.js +3465 -0
- package/dist/test/repository-command-diagnostic.test.d.ts +1 -0
- package/dist/test/repository-command-diagnostic.test.js +55 -0
- package/dist/test/repository-command-signals.test.d.ts +1 -0
- package/dist/test/repository-command-signals.test.js +124 -0
- package/dist/test/repository-fixture-scope-coverage.test.d.ts +1 -0
- package/dist/test/repository-fixture-scope-coverage.test.js +127 -0
- package/dist/test/repository-fixture-validation.test.d.ts +1 -0
- package/dist/test/repository-fixture-validation.test.js +362 -0
- package/dist/test/repository-foundry-progress.test.d.ts +1 -0
- package/dist/test/repository-foundry-progress.test.js +110 -0
- package/dist/test/repository-foundry-quality.test.d.ts +1 -0
- package/dist/test/repository-foundry-quality.test.js +1138 -0
- package/dist/test/repository-import-context.test.d.ts +1 -0
- package/dist/test/repository-import-context.test.js +354 -0
- package/dist/test/repository-model-authoring.test.d.ts +1 -0
- package/dist/test/repository-model-authoring.test.js +544 -0
- package/dist/test/repository-model.test.d.ts +1 -0
- package/dist/test/repository-model.test.js +2195 -0
- package/dist/test/repository-node-test-reporter.test.d.ts +1 -0
- package/dist/test/repository-node-test-reporter.test.js +104 -0
- package/dist/test/repository-oracle-concurrency.test.d.ts +1 -0
- package/dist/test/repository-oracle-concurrency.test.js +542 -0
- package/dist/test/repository-oracle-coverage-witness.test.d.ts +1 -0
- package/dist/test/repository-oracle-coverage-witness.test.js +511 -0
- package/dist/test/repository-oracle-coverage.test.d.ts +1 -0
- package/dist/test/repository-oracle-coverage.test.js +168 -0
- package/dist/test/repository-oracle-evidence.test.d.ts +1 -0
- package/dist/test/repository-oracle-evidence.test.js +185 -0
- package/dist/test/repository-oracle-plan.test.d.ts +1 -0
- package/dist/test/repository-oracle-plan.test.js +176 -0
- package/dist/test/repository-overlay-isolation.test.d.ts +1 -0
- package/dist/test/repository-overlay-isolation.test.js +85 -0
- package/dist/test/repository-preparation-cache.test.d.ts +1 -0
- package/dist/test/repository-preparation-cache.test.js +414 -0
- package/dist/test/repository-public-artifact.test.d.ts +1 -0
- package/dist/test/repository-public-artifact.test.js +273 -0
- package/dist/test/repository-qualification-diagnostics.test.d.ts +1 -0
- package/dist/test/repository-qualification-diagnostics.test.js +524 -0
- package/dist/test/repository-reference-authoring.test.d.ts +1 -0
- package/dist/test/repository-reference-authoring.test.js +1633 -0
- package/dist/test/repository-review-evidence-v2.test.d.ts +1 -0
- package/dist/test/repository-review-evidence-v2.test.js +183 -0
- package/dist/test/repository-review-evidence.test.d.ts +1 -0
- package/dist/test/repository-review-evidence.test.js +124 -0
- package/dist/test/repository-seed-exclusions.test.d.ts +1 -0
- package/dist/test/repository-seed-exclusions.test.js +96 -0
- package/dist/test/repository-seed-selection.test.d.ts +1 -0
- package/dist/test/repository-seed-selection.test.js +504 -0
- package/dist/test/repository-semantic-calibration.test.d.ts +1 -0
- package/dist/test/repository-semantic-calibration.test.js +688 -0
- package/dist/test/repository-solution-edits.test.d.ts +1 -0
- package/dist/test/repository-solution-edits.test.js +377 -0
- package/dist/test/repository-specification-budget.test.d.ts +1 -0
- package/dist/test/repository-specification-budget.test.js +171 -0
- package/dist/test/repository-specification-contract-checkpoint.test.d.ts +1 -0
- package/dist/test/repository-specification-contract-checkpoint.test.js +228 -0
- package/dist/test/repository-specification-contract-facts.test.d.ts +1 -0
- package/dist/test/repository-specification-contract-facts.test.js +177 -0
- package/dist/test/repository-trajectory-authoring.test.d.ts +1 -0
- package/dist/test/repository-trajectory-authoring.test.js +176 -0
- package/dist/test/repository-valid-control-plan.test.d.ts +1 -0
- package/dist/test/repository-valid-control-plan.test.js +45 -0
- package/dist/test/repository-vitest-phase.test.d.ts +1 -0
- package/dist/test/repository-vitest-phase.test.js +848 -0
- package/dist/test/repository-vitest-reporter.test.d.ts +1 -0
- package/dist/test/repository-vitest-reporter.test.js +158 -0
- package/dist/test/repository-workspace-build.test.d.ts +1 -0
- package/dist/test/repository-workspace-build.test.js +160 -0
- package/dist/test/strict-authoring-schema.test.d.ts +1 -0
- package/dist/test/strict-authoring-schema.test.js +169 -0
- package/package.json +48 -6
|
@@ -0,0 +1,570 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import test from "node:test";
|
|
3
|
+
import { Schema } from "effect";
|
|
4
|
+
import { AgenticCapabilityRequestV1, agenticDigestV1, agenticCapabilityRequestJsonSchemaV1, agenticSemanticOperationDigestV1, assertAgenticArtifactPrerequisitesV1, assertAgenticCandidateTransitionV1, assertAgenticEvidenceInputsV1, decodeAgenticAcceptedArtifactV1, decodeAgenticCandidateV1, decodeAgenticCapabilityRequestV1, decodeAgenticCapabilityResultV1, decodeAgenticEnvironmentPlanV1, decodeAgenticQualityScorecardV1, decodeAgenticScientificProgressReceiptV1, decodeAgenticTrustedInvocationV1, makeAgenticProgressFingerprintV1 } from "../agentic-capabilities-protocol.js";
|
|
5
|
+
const sha = "a".repeat(64);
|
|
6
|
+
const otherSha = "b".repeat(64);
|
|
7
|
+
const at = "2026-09-15T12:00:00.000Z";
|
|
8
|
+
const inputs = {
|
|
9
|
+
sourceDigest: sha,
|
|
10
|
+
environmentDigest: sha,
|
|
11
|
+
specificationDigest: sha,
|
|
12
|
+
oracleDigest: sha,
|
|
13
|
+
controlsDigest: sha,
|
|
14
|
+
policyDigest: sha,
|
|
15
|
+
modelPlanDigest: sha
|
|
16
|
+
};
|
|
17
|
+
const candidate = {
|
|
18
|
+
version: 1,
|
|
19
|
+
candidate: "candidate",
|
|
20
|
+
revision: "revision",
|
|
21
|
+
coverageCell: "cell",
|
|
22
|
+
sourceDigest: sha,
|
|
23
|
+
proposal: "proposal",
|
|
24
|
+
createdAt: at
|
|
25
|
+
};
|
|
26
|
+
const artifact = (handle, kind, extra = {}) => decodeAgenticAcceptedArtifactV1({
|
|
27
|
+
handle,
|
|
28
|
+
candidate: "candidate",
|
|
29
|
+
revision: "revision",
|
|
30
|
+
kind,
|
|
31
|
+
contentDigest: sha,
|
|
32
|
+
state: "reference_qualified",
|
|
33
|
+
dependencies: inputs,
|
|
34
|
+
gradeExecuted: true,
|
|
35
|
+
...extra
|
|
36
|
+
});
|
|
37
|
+
const qualified = decodeAgenticCandidateV1({
|
|
38
|
+
...candidate,
|
|
39
|
+
state: "reference_qualified",
|
|
40
|
+
environment: "environment",
|
|
41
|
+
referenceQualification: "reference"
|
|
42
|
+
});
|
|
43
|
+
const frozen = decodeAgenticCandidateV1({
|
|
44
|
+
...qualified,
|
|
45
|
+
state: "frozen",
|
|
46
|
+
specification: "spec",
|
|
47
|
+
oracle: "oracle",
|
|
48
|
+
controls: ["valid", "wrong"],
|
|
49
|
+
tournament: "tournament",
|
|
50
|
+
inputs,
|
|
51
|
+
frozenAt: at
|
|
52
|
+
});
|
|
53
|
+
const admitted = decodeAgenticCandidateV1({
|
|
54
|
+
...frozen,
|
|
55
|
+
state: "admitted",
|
|
56
|
+
challenge: "challenge",
|
|
57
|
+
admission: "admission"
|
|
58
|
+
});
|
|
59
|
+
const records = [
|
|
60
|
+
artifact("environment", "environment"),
|
|
61
|
+
artifact("reference", "reference_qualification"),
|
|
62
|
+
artifact("spec", "specification"),
|
|
63
|
+
artifact("oracle", "oracle"),
|
|
64
|
+
artifact("valid", "valid_controls"),
|
|
65
|
+
artifact("wrong", "wrong_controls"),
|
|
66
|
+
artifact("tournament", "tournament"),
|
|
67
|
+
artifact("challenge", "held_out_challenge", { fresh: true }),
|
|
68
|
+
artifact("admission", "admission")
|
|
69
|
+
];
|
|
70
|
+
test("canonical tool guidance explains cumulative controls even when schema annotations are absent", () => {
|
|
71
|
+
const rendered = agenticCapabilityRequestJsonSchemaV1({
|
|
72
|
+
anyOf: [{
|
|
73
|
+
properties: {
|
|
74
|
+
tool: { enum: ["evaluate_oracle"] },
|
|
75
|
+
args: { properties: { controls: { type: "array", description: "" } } }
|
|
76
|
+
}
|
|
77
|
+
}]
|
|
78
|
+
});
|
|
79
|
+
const description = rendered.anyOf[0].properties.args.properties.controls.description;
|
|
80
|
+
assert.match(description, /one-element list.*latest cumulative controls checkpoint/u);
|
|
81
|
+
assert.match(description, /supersedes earlier snapshots/u);
|
|
82
|
+
assert.match(description, /do not combine old and new handles/u);
|
|
83
|
+
});
|
|
84
|
+
test("source-first drafts require no invented seed or execution evidence", () => {
|
|
85
|
+
assert.equal(decodeAgenticCandidateV1({ ...candidate, state: "draft" }).state, "draft");
|
|
86
|
+
assert.deepEqual(decodeAgenticCapabilityRequestV1({ tool: "inspect_repository", args: {} }), {
|
|
87
|
+
tool: "inspect_repository",
|
|
88
|
+
args: {}
|
|
89
|
+
});
|
|
90
|
+
assert.deepEqual(decodeAgenticCapabilityRequestV1({
|
|
91
|
+
tool: "draft_specification",
|
|
92
|
+
args: { candidate: "candidate" }
|
|
93
|
+
}), { tool: "draft_specification", args: { candidate: "candidate" } });
|
|
94
|
+
assert.throws(() => decodeAgenticCandidateV1({ ...candidate, state: "reference_qualified" }));
|
|
95
|
+
assert.throws(() => decodeAgenticCandidateV1({ ...candidate, state: "admitted", admission: "model-says-approved" }));
|
|
96
|
+
});
|
|
97
|
+
test("every tool decodes its normative domain arguments but not trusted authority", () => {
|
|
98
|
+
const requests = [
|
|
99
|
+
{ tool: "list_candidates", args: { coverageCell: "cell", limit: 50 } },
|
|
100
|
+
{
|
|
101
|
+
tool: "read_source",
|
|
102
|
+
args: { source: "source", path: "src/main.ts", offset: 0, length: 120_000 }
|
|
103
|
+
},
|
|
104
|
+
{ tool: "search_source", args: { source: "source", query: "export" } },
|
|
105
|
+
{
|
|
106
|
+
tool: "plan_environment",
|
|
107
|
+
args: {
|
|
108
|
+
candidate: "candidate",
|
|
109
|
+
recipeIds: ["test"],
|
|
110
|
+
executionMode: "source",
|
|
111
|
+
targetRoots: ["src"],
|
|
112
|
+
requiredArtifacts: [],
|
|
113
|
+
justification: "The actual test recipe runs TypeScript source."
|
|
114
|
+
}
|
|
115
|
+
},
|
|
116
|
+
{
|
|
117
|
+
tool: "probe_fixture",
|
|
118
|
+
args: { candidate: "candidate", environment: "environment", fixtureDraft: "fixture" }
|
|
119
|
+
},
|
|
120
|
+
{ tool: "qualify_reference", args: { candidate: "candidate", environment: "environment" } },
|
|
121
|
+
{ tool: "author_oracle", args: { candidate: "candidate", specification: "spec" } },
|
|
122
|
+
{
|
|
123
|
+
tool: "generate_controls",
|
|
124
|
+
args: {
|
|
125
|
+
candidate: "candidate",
|
|
126
|
+
specification: "spec",
|
|
127
|
+
environment: "environment",
|
|
128
|
+
kind: "wrong"
|
|
129
|
+
}
|
|
130
|
+
},
|
|
131
|
+
{
|
|
132
|
+
tool: "evaluate_oracle",
|
|
133
|
+
args: { candidate: "candidate", oracle: "oracle", controls: ["controls"] }
|
|
134
|
+
},
|
|
135
|
+
{
|
|
136
|
+
tool: "repair_oracle",
|
|
137
|
+
args: { candidate: "candidate", oracle: "oracle", findings: "findings" }
|
|
138
|
+
},
|
|
139
|
+
{ tool: "freeze_candidate", args: { candidate: "candidate", revision: "revision" } },
|
|
140
|
+
{ tool: "run_held_out_challenge", args: { frozenCandidate: "frozen" } },
|
|
141
|
+
{
|
|
142
|
+
tool: "request_admission",
|
|
143
|
+
args: { frozenCandidate: "frozen", qualification: "qualification" }
|
|
144
|
+
},
|
|
145
|
+
{ tool: "get_progress", args: {} },
|
|
146
|
+
{ tool: "read_diagnostics", args: { operation: "operation", maximumBytes: 65_536 } },
|
|
147
|
+
{
|
|
148
|
+
tool: "reject_candidate",
|
|
149
|
+
args: { candidate: "candidate", reason: "Not discriminating", evidence: [] }
|
|
150
|
+
},
|
|
151
|
+
{ tool: "finish_campaign", args: { reason: "Coverage complete" } }
|
|
152
|
+
];
|
|
153
|
+
for (const request of requests) {
|
|
154
|
+
assert.equal(decodeAgenticCapabilityRequestV1(request).tool, request.tool);
|
|
155
|
+
assert.throws(() => decodeAgenticCapabilityRequestV1({ ...request, orgId: "other-org" }));
|
|
156
|
+
assert.throws(() => Schema.decodeUnknownSync(AgenticCapabilityRequestV1)({
|
|
157
|
+
...request,
|
|
158
|
+
args: { ...request.args, reservation: "mine" }
|
|
159
|
+
}));
|
|
160
|
+
}
|
|
161
|
+
});
|
|
162
|
+
test("tool arguments enforce UTF-8 bytes, bounded arrays and normalized source paths", () => {
|
|
163
|
+
assert.throws(() => decodeAgenticCapabilityRequestV1({
|
|
164
|
+
tool: "search_source",
|
|
165
|
+
args: { source: "source", query: "🧪".repeat(129) }
|
|
166
|
+
}));
|
|
167
|
+
assert.throws(() => decodeAgenticCapabilityRequestV1({
|
|
168
|
+
tool: "list_candidates",
|
|
169
|
+
args: { coverageCell: "cell", limit: 51 }
|
|
170
|
+
}));
|
|
171
|
+
for (const path of [
|
|
172
|
+
"../secret",
|
|
173
|
+
"/etc/passwd",
|
|
174
|
+
"C:/file",
|
|
175
|
+
"a/../file",
|
|
176
|
+
"a\\file",
|
|
177
|
+
"a//file",
|
|
178
|
+
"a/./file",
|
|
179
|
+
"a\u0000file"
|
|
180
|
+
]) {
|
|
181
|
+
assert.throws(() => decodeAgenticCapabilityRequestV1({
|
|
182
|
+
tool: "read_source",
|
|
183
|
+
args: { source: "source", path, offset: 0, length: 10 }
|
|
184
|
+
}));
|
|
185
|
+
}
|
|
186
|
+
assert.throws(() => decodeAgenticCapabilityRequestV1({
|
|
187
|
+
tool: "evaluate_oracle",
|
|
188
|
+
args: { candidate: "c", oracle: "o", controls: Array.from({ length: 33 }, () => "control") }
|
|
189
|
+
}));
|
|
190
|
+
assert.throws(() => decodeAgenticCapabilityRequestV1({
|
|
191
|
+
tool: "evaluate_oracle",
|
|
192
|
+
args: { candidate: "c", oracle: "o", controls: [] }
|
|
193
|
+
}));
|
|
194
|
+
assert.throws(() => decodeAgenticCapabilityRequestV1({
|
|
195
|
+
tool: "plan_environment",
|
|
196
|
+
args: {
|
|
197
|
+
candidate: "candidate",
|
|
198
|
+
recipeIds: [],
|
|
199
|
+
executionMode: "source",
|
|
200
|
+
targetRoots: ["."],
|
|
201
|
+
requiredArtifacts: [],
|
|
202
|
+
justification: "A deliberately invalid empty recipe selection."
|
|
203
|
+
}
|
|
204
|
+
}));
|
|
205
|
+
assert.throws(() => decodeAgenticCapabilityRequestV1({
|
|
206
|
+
tool: "plan_environment",
|
|
207
|
+
args: {
|
|
208
|
+
candidate: "candidate",
|
|
209
|
+
recipeIds: ["test"],
|
|
210
|
+
executionMode: "source",
|
|
211
|
+
targetRoots: [],
|
|
212
|
+
requiredArtifacts: [],
|
|
213
|
+
justification: "A deliberately invalid empty target-root selection."
|
|
214
|
+
}
|
|
215
|
+
}));
|
|
216
|
+
});
|
|
217
|
+
test("trusted invocation is strict and semantic identity survives transcript tool-call changes", () => {
|
|
218
|
+
const request = decodeAgenticCapabilityRequestV1({ tool: "inspect_repository", args: {} });
|
|
219
|
+
const invocation = {
|
|
220
|
+
orgId: "org",
|
|
221
|
+
campaignId: "campaign",
|
|
222
|
+
executionId: "execution",
|
|
223
|
+
epoch: 1,
|
|
224
|
+
operationId: "operation",
|
|
225
|
+
inputDigest: sha,
|
|
226
|
+
actorRole: "eval-orchestrator",
|
|
227
|
+
modelBinding: "model",
|
|
228
|
+
reservation: "reservation",
|
|
229
|
+
deadlineAt: at,
|
|
230
|
+
toolBundleDigest: sha,
|
|
231
|
+
request
|
|
232
|
+
};
|
|
233
|
+
assert.equal(decodeAgenticTrustedInvocationV1(invocation).epoch, 1);
|
|
234
|
+
assert.deepEqual(decodeAgenticTrustedInvocationV1({
|
|
235
|
+
...invocation,
|
|
236
|
+
maximumModelCalls: 12,
|
|
237
|
+
maximumWallTimeMs: 420_000
|
|
238
|
+
}), { ...invocation, maximumModelCalls: 12, maximumWallTimeMs: 420_000 });
|
|
239
|
+
assert.throws(() => decodeAgenticTrustedInvocationV1({ ...invocation, maximumModelCalls: 0 }));
|
|
240
|
+
assert.throws(() => decodeAgenticTrustedInvocationV1({ ...invocation, maximumWallTimeMs: 0 }));
|
|
241
|
+
assert.throws(() => decodeAgenticTrustedInvocationV1({ ...invocation, epoch: 0 }));
|
|
242
|
+
assert.throws(() => decodeAgenticTrustedInvocationV1({
|
|
243
|
+
...invocation,
|
|
244
|
+
traceparent: `00-${"0".repeat(32)}-${"1".repeat(16)}-01`
|
|
245
|
+
}));
|
|
246
|
+
const first = agenticSemanticOperationDigestV1(invocation);
|
|
247
|
+
assert.equal(first, agenticSemanticOperationDigestV1({ ...invocation, request: structuredClone(request) }));
|
|
248
|
+
assert.notEqual(first, agenticSemanticOperationDigestV1({ ...invocation, campaignId: "different" }));
|
|
249
|
+
assert.equal(agenticDigestV1({ a: 1, b: 2 }), agenticDigestV1({ b: 2, a: 1 }));
|
|
250
|
+
});
|
|
251
|
+
test("environment plan is lazy and specific to the real repository recipe", () => {
|
|
252
|
+
const plan = {
|
|
253
|
+
version: 1,
|
|
254
|
+
planId: "env",
|
|
255
|
+
candidate: "candidate",
|
|
256
|
+
revision: "revision",
|
|
257
|
+
sourceDigest: sha,
|
|
258
|
+
commitSha: "c".repeat(40),
|
|
259
|
+
recipeIds: ["unit"],
|
|
260
|
+
executionMode: "source",
|
|
261
|
+
targetRoots: ["src"],
|
|
262
|
+
dependencyClosure: [],
|
|
263
|
+
requiredArtifacts: [],
|
|
264
|
+
lockfileDigest: null,
|
|
265
|
+
packageManager: null,
|
|
266
|
+
runtime: { name: "node", version: "24.17.0", platform: "linux", architecture: "x64" },
|
|
267
|
+
commands: [
|
|
268
|
+
{
|
|
269
|
+
id: "probe",
|
|
270
|
+
purpose: "probe",
|
|
271
|
+
executable: "node",
|
|
272
|
+
args: ["--test", "src/example.test.ts"],
|
|
273
|
+
cwd: null,
|
|
274
|
+
timeoutMs: 120_000
|
|
275
|
+
}
|
|
276
|
+
],
|
|
277
|
+
expectedCost: { wallTimeMs: 1000, spendUsd: "0" },
|
|
278
|
+
justification: "Run source-native test recipe without build.",
|
|
279
|
+
justificationEvidence: ["recipe"]
|
|
280
|
+
};
|
|
281
|
+
assert.equal(decodeAgenticEnvironmentPlanV1(plan).commands[0]?.purpose, "probe");
|
|
282
|
+
assert.deepEqual(decodeAgenticEnvironmentPlanV1({
|
|
283
|
+
...plan,
|
|
284
|
+
targetRoots: ["."],
|
|
285
|
+
dependencyClosure: [".", "packages/shared"]
|
|
286
|
+
}).targetRoots, ["."]);
|
|
287
|
+
const rootWorkspacePlan = {
|
|
288
|
+
...plan,
|
|
289
|
+
targetRoots: ["."],
|
|
290
|
+
dependencyClosure: Array.from({ length: 36 }, (_, index) => `packages/p${index}`),
|
|
291
|
+
commands: Array.from({ length: 38 }, (_, index) => ({
|
|
292
|
+
...plan.commands[0],
|
|
293
|
+
id: `command-${index}`,
|
|
294
|
+
cwd: `packages/p${index % 36}`
|
|
295
|
+
}))
|
|
296
|
+
};
|
|
297
|
+
assert.equal(decodeAgenticEnvironmentPlanV1(rootWorkspacePlan).commands.length, 38);
|
|
298
|
+
assert.throws(() => decodeAgenticEnvironmentPlanV1({
|
|
299
|
+
...rootWorkspacePlan,
|
|
300
|
+
dependencyClosure: Array.from({ length: 65 }, (_, index) => `packages/p${index}`)
|
|
301
|
+
}));
|
|
302
|
+
assert.throws(() => decodeAgenticEnvironmentPlanV1({
|
|
303
|
+
...rootWorkspacePlan,
|
|
304
|
+
commands: Array.from({ length: 65 }, (_, index) => ({
|
|
305
|
+
...plan.commands[0],
|
|
306
|
+
id: `command-${index}`
|
|
307
|
+
}))
|
|
308
|
+
}));
|
|
309
|
+
const rootRequest = decodeAgenticCapabilityRequestV1({
|
|
310
|
+
tool: "plan_environment",
|
|
311
|
+
args: {
|
|
312
|
+
candidate: "candidate",
|
|
313
|
+
recipeIds: ["unit"],
|
|
314
|
+
executionMode: "source",
|
|
315
|
+
targetRoots: ["."],
|
|
316
|
+
requiredArtifacts: [],
|
|
317
|
+
justification: "The selected package is the repository root."
|
|
318
|
+
}
|
|
319
|
+
});
|
|
320
|
+
assert.ok(rootRequest.tool === "plan_environment");
|
|
321
|
+
assert.deepEqual(rootRequest.args.targetRoots, ["."]);
|
|
322
|
+
for (const invalid of ["..", "./src", "packages/../shared", "/tmp", "C:\\repo"]) {
|
|
323
|
+
assert.throws(() => decodeAgenticEnvironmentPlanV1({ ...plan, targetRoots: [invalid] }));
|
|
324
|
+
}
|
|
325
|
+
assert.throws(() => decodeAgenticEnvironmentPlanV1({ ...plan, requiredArtifacts: ["."] }));
|
|
326
|
+
assert.throws(() => decodeAgenticEnvironmentPlanV1({
|
|
327
|
+
...plan,
|
|
328
|
+
commands: [{ ...plan.commands[0], timeoutMs: 600_001 }]
|
|
329
|
+
}));
|
|
330
|
+
assert.throws(() => decodeAgenticEnvironmentPlanV1({
|
|
331
|
+
...plan,
|
|
332
|
+
runtime: { ...plan.runtime, approvalPolicy: "none" }
|
|
333
|
+
}));
|
|
334
|
+
});
|
|
335
|
+
test("environment failure is not a grade; useful diagnostic results remain bounded", () => {
|
|
336
|
+
const result = {
|
|
337
|
+
outcome: "environment_blocked",
|
|
338
|
+
operation: "op",
|
|
339
|
+
inputDigest: sha,
|
|
340
|
+
accepted: [],
|
|
341
|
+
evidenceSummary: "Package installation failed; no oracle ran.",
|
|
342
|
+
gradeExecuted: false,
|
|
343
|
+
diagnostics: "full-log",
|
|
344
|
+
usage: { receivedCalls: 0, spendKnownUsd: "0", spendUnknown: false },
|
|
345
|
+
allowedNext: ["plan_environment", "read_diagnostics"]
|
|
346
|
+
};
|
|
347
|
+
assert.equal(decodeAgenticCapabilityResultV1(result).gradeExecuted, false);
|
|
348
|
+
assert.throws(() => decodeAgenticCapabilityResultV1({ ...result, gradeExecuted: true }));
|
|
349
|
+
assert.throws(() => decodeAgenticCapabilityResultV1({ ...result, gradeExecuted: undefined }));
|
|
350
|
+
assert.throws(() => decodeAgenticCapabilityResultV1({ ...result, evidenceSummary: "🧪".repeat(2049) }));
|
|
351
|
+
assert.throws(() => decodeAgenticCapabilityResultV1({ ...result, usage: { ...result.usage, spendKnownUsd: "NaN" } }));
|
|
352
|
+
});
|
|
353
|
+
test("accepted handles are revision bound; changed ancestors invalidate downstream evidence", () => {
|
|
354
|
+
const evidence = artifact("challenge", "held_out_challenge", { fresh: true });
|
|
355
|
+
assert.doesNotThrow(() => assertAgenticEvidenceInputsV1(evidence, inputs));
|
|
356
|
+
assert.throws(() => assertAgenticEvidenceInputsV1({ ...evidence, dependencies: {} }, inputs));
|
|
357
|
+
for (const key of Object.keys(inputs))
|
|
358
|
+
assert.throws(() => assertAgenticEvidenceInputsV1(evidence, { ...inputs, [key]: otherSha }));
|
|
359
|
+
const request = {
|
|
360
|
+
candidate: "candidate",
|
|
361
|
+
revision: "revision",
|
|
362
|
+
handles: ["challenge"],
|
|
363
|
+
accepted: [evidence],
|
|
364
|
+
inputs,
|
|
365
|
+
requireGrade: true,
|
|
366
|
+
requireFresh: true
|
|
367
|
+
};
|
|
368
|
+
assert.doesNotThrow(() => assertAgenticArtifactPrerequisitesV1(request));
|
|
369
|
+
assert.throws(() => assertAgenticArtifactPrerequisitesV1({ ...request, revision: "other-revision" }));
|
|
370
|
+
assert.throws(() => assertAgenticArtifactPrerequisitesV1({ ...request, accepted: [] }));
|
|
371
|
+
assert.throws(() => assertAgenticArtifactPrerequisitesV1({ ...request, accepted: [{ ...evidence, fresh: false }] }));
|
|
372
|
+
assert.throws(() => assertAgenticArtifactPrerequisitesV1({
|
|
373
|
+
...request,
|
|
374
|
+
accepted: [{ ...evidence, gradeExecuted: false }]
|
|
375
|
+
}));
|
|
376
|
+
assert.throws(() => assertAgenticArtifactPrerequisitesV1({
|
|
377
|
+
...request,
|
|
378
|
+
accepted: [{ ...evidence, state: "superseded" }]
|
|
379
|
+
}));
|
|
380
|
+
});
|
|
381
|
+
test("draft to admitted is impossible; real accepted scientific records enable monotonic qualification", () => {
|
|
382
|
+
const draft = decodeAgenticCandidateV1({ ...candidate, state: "draft" });
|
|
383
|
+
assert.throws(() => assertAgenticCandidateTransitionV1(draft, admitted, records, inputs));
|
|
384
|
+
assert.throws(() => assertAgenticCandidateTransitionV1(draft, qualified, [], inputs));
|
|
385
|
+
assert.doesNotThrow(() => assertAgenticCandidateTransitionV1(draft, qualified, records, inputs));
|
|
386
|
+
assert.doesNotThrow(() => assertAgenticCandidateTransitionV1(qualified, frozen, records));
|
|
387
|
+
assert.doesNotThrow(() => assertAgenticCandidateTransitionV1(frozen, admitted, records));
|
|
388
|
+
assert.doesNotThrow(() => assertAgenticCandidateTransitionV1(admitted, admitted, records));
|
|
389
|
+
assert.throws(() => assertAgenticCandidateTransitionV1(admitted, draft, records));
|
|
390
|
+
assert.throws(() => assertAgenticCandidateTransitionV1(qualified, frozen, records.map((entry) => entry.handle === "valid" ? { ...entry, kind: "wrong_controls" } : entry)));
|
|
391
|
+
assert.throws(() => assertAgenticCandidateTransitionV1(qualified, frozen, records.map((entry) => (entry.handle === "spec" ? { ...entry, state: "draft" } : entry))));
|
|
392
|
+
assert.throws(() => assertAgenticCandidateTransitionV1(frozen, admitted, records.map((entry) => (entry.handle === "challenge" ? { ...entry, fresh: false } : entry))));
|
|
393
|
+
assert.throws(() => assertAgenticCandidateTransitionV1(frozen, { ...admitted, oracle: "another-oracle" }, [...records, artifact("another-oracle", "oracle")]));
|
|
394
|
+
});
|
|
395
|
+
test("quality scorecards are measured criteria, not an agent-provided quality number", () => {
|
|
396
|
+
// For wrong/held-out controls, passed means the discrimination criterion passed, not that a wrong solution was accepted.
|
|
397
|
+
const measured = { passed: 2, failed: 0, missing: 0, evidence: ["observation"] };
|
|
398
|
+
const score = {
|
|
399
|
+
version: 1,
|
|
400
|
+
candidate: "candidate",
|
|
401
|
+
revision: "revision",
|
|
402
|
+
reference: measured,
|
|
403
|
+
validControls: measured,
|
|
404
|
+
wrongControls: measured,
|
|
405
|
+
heldOut: measured,
|
|
406
|
+
coverage: [{ cell: "cell", observed: true, evidence: ["coverage"] }],
|
|
407
|
+
disposition: "admitted",
|
|
408
|
+
reasons: [],
|
|
409
|
+
missingEvidence: [],
|
|
410
|
+
usage: { elapsedMs: 1000, receivedCalls: 4, spendKnownUsd: "0.1", spendUnknown: false },
|
|
411
|
+
admission: "admission"
|
|
412
|
+
};
|
|
413
|
+
assert.equal(decodeAgenticQualityScorecardV1(score).disposition, "admitted");
|
|
414
|
+
assert.throws(() => decodeAgenticQualityScorecardV1({ ...score, quality: 0.99 }));
|
|
415
|
+
assert.throws(() => decodeAgenticQualityScorecardV1({ ...score, admission: null }));
|
|
416
|
+
assert.throws(() => decodeAgenticQualityScorecardV1({ ...score, heldOut: { ...measured, missing: 1 } }));
|
|
417
|
+
assert.throws(() => decodeAgenticQualityScorecardV1({ ...score, missingEvidence: ["held-out"] }));
|
|
418
|
+
});
|
|
419
|
+
test("progress fingerprints depend only on accepted scientific state", () => {
|
|
420
|
+
const scientific = {
|
|
421
|
+
acceptedEvidence: [
|
|
422
|
+
{ handle: "specification", contentDigest: sha, kind: "specification" },
|
|
423
|
+
{ handle: "reference", contentDigest: otherSha, kind: "reference_qualification" }
|
|
424
|
+
],
|
|
425
|
+
completedGates: ["reference_qualified", "specification_accepted"],
|
|
426
|
+
environmentDigest: sha
|
|
427
|
+
};
|
|
428
|
+
const first = makeAgenticProgressFingerprintV1(scientific);
|
|
429
|
+
const reordered = makeAgenticProgressFingerprintV1({
|
|
430
|
+
...scientific,
|
|
431
|
+
acceptedEvidence: [...scientific.acceptedEvidence].reverse(),
|
|
432
|
+
completedGates: [...scientific.completedGates].reverse()
|
|
433
|
+
});
|
|
434
|
+
assert.deepEqual(first, reordered);
|
|
435
|
+
// Candidate IDs, revisions, phase labels, timestamps, prose and tool-call IDs
|
|
436
|
+
// belong on surrounding receipts/events and are not accepted fingerprint inputs.
|
|
437
|
+
const activityOnlyChanges = {
|
|
438
|
+
candidateId: "another-candidate",
|
|
439
|
+
candidateRevision: "revision-999",
|
|
440
|
+
phase: "another-phase",
|
|
441
|
+
recordedAt: "2099-01-01T00:00:00.000Z",
|
|
442
|
+
modelProse: "I made progress",
|
|
443
|
+
toolCallId: "call-999"
|
|
444
|
+
};
|
|
445
|
+
assert.deepEqual(makeAgenticProgressFingerprintV1({
|
|
446
|
+
...scientific,
|
|
447
|
+
...activityOnlyChanges,
|
|
448
|
+
acceptedEvidence: scientific.acceptedEvidence.map((entry, index) => ({
|
|
449
|
+
...entry,
|
|
450
|
+
handle: `different-handle-${index}`
|
|
451
|
+
}))
|
|
452
|
+
}), first);
|
|
453
|
+
assert.notDeepEqual(makeAgenticProgressFingerprintV1({
|
|
454
|
+
...scientific,
|
|
455
|
+
acceptedEvidence: [
|
|
456
|
+
...scientific.acceptedEvidence,
|
|
457
|
+
{ handle: "oracle", contentDigest: sha, kind: "oracle" }
|
|
458
|
+
]
|
|
459
|
+
}), first);
|
|
460
|
+
});
|
|
461
|
+
test("continuation evidence changes progress without completing a scientific gate", () => {
|
|
462
|
+
const base = {
|
|
463
|
+
acceptedEvidence: [{ handle: "frozen", contentDigest: sha, kind: "frozen_candidate" }],
|
|
464
|
+
completedGates: ["candidate_frozen"],
|
|
465
|
+
environmentDigest: sha
|
|
466
|
+
};
|
|
467
|
+
const before = makeAgenticProgressFingerprintV1(base);
|
|
468
|
+
const after = makeAgenticProgressFingerprintV1({
|
|
469
|
+
...base,
|
|
470
|
+
acceptedEvidence: [
|
|
471
|
+
...base.acceptedEvidence,
|
|
472
|
+
{ handle: "partial", contentDigest: otherSha, kind: "continuation_progress" }
|
|
473
|
+
]
|
|
474
|
+
});
|
|
475
|
+
assert.equal(after.acceptedEvidenceDigest, before.acceptedEvidenceDigest);
|
|
476
|
+
assert.equal(after.completedGateDigest, before.completedGateDigest);
|
|
477
|
+
assert.notEqual(after.continuationProgressDigest, before.continuationProgressDigest);
|
|
478
|
+
assert.deepEqual(makeAgenticProgressFingerprintV1(base), before);
|
|
479
|
+
});
|
|
480
|
+
test("progress receipts attribute revisions, phase, consumption and downstream reserve", () => {
|
|
481
|
+
const fingerprint = makeAgenticProgressFingerprintV1({
|
|
482
|
+
acceptedEvidence: [{ handle: "specification", contentDigest: sha, kind: "specification" }],
|
|
483
|
+
completedGates: ["specification_accepted"],
|
|
484
|
+
environmentDigest: null
|
|
485
|
+
});
|
|
486
|
+
const receipt = decodeAgenticScientificProgressReceiptV1({
|
|
487
|
+
version: 1,
|
|
488
|
+
receiptId: "progress-1",
|
|
489
|
+
operationId: "operation-1",
|
|
490
|
+
capability: "draft_specification",
|
|
491
|
+
scientificRole: "specification-author",
|
|
492
|
+
phase: "drafting",
|
|
493
|
+
inputRevision: "revision-1",
|
|
494
|
+
outputRevision: "revision-2",
|
|
495
|
+
acceptedEvidenceDigest: fingerprint.acceptedEvidenceDigest,
|
|
496
|
+
previousFingerprint: null,
|
|
497
|
+
resultingFingerprint: fingerprint,
|
|
498
|
+
acceptedEvidenceAdded: ["specification"],
|
|
499
|
+
acceptedEvidenceRemoved: [],
|
|
500
|
+
gatesCompleted: ["specification_accepted"],
|
|
501
|
+
gatesInvalidated: [],
|
|
502
|
+
sameFingerprintCount: 0,
|
|
503
|
+
consumption: {
|
|
504
|
+
wallTimeMs: 1000,
|
|
505
|
+
receivedCalls: 1,
|
|
506
|
+
spendKnownUsd: "0.01",
|
|
507
|
+
spendUnknown: false
|
|
508
|
+
},
|
|
509
|
+
remaining: {
|
|
510
|
+
phaseCalls: 2,
|
|
511
|
+
phaseSpendUsd: "0.25",
|
|
512
|
+
phaseWallTimeMs: 30_000,
|
|
513
|
+
phaseAttempts: 1,
|
|
514
|
+
downstreamCalls: 8,
|
|
515
|
+
downstreamSpendUsd: "1.5",
|
|
516
|
+
downstreamWallTimeMs: 300_000,
|
|
517
|
+
downstreamAttempts: 6
|
|
518
|
+
},
|
|
519
|
+
recordedAt: at
|
|
520
|
+
});
|
|
521
|
+
assert.equal(receipt.sameFingerprintCount, 0);
|
|
522
|
+
assert.equal(receipt.remaining.downstreamCalls, 8);
|
|
523
|
+
const result = decodeAgenticCapabilityResultV1({
|
|
524
|
+
outcome: "progress",
|
|
525
|
+
operation: "operation-1",
|
|
526
|
+
inputDigest: sha,
|
|
527
|
+
candidateRevision: "revision-2",
|
|
528
|
+
accepted: ["specification"],
|
|
529
|
+
evidenceSummary: "Accepted a specification revision.",
|
|
530
|
+
progressReceipt: receipt,
|
|
531
|
+
usage: { receivedCalls: 1, spendKnownUsd: "0.01", spendUnknown: false },
|
|
532
|
+
allowedNext: ["plan_environment"]
|
|
533
|
+
});
|
|
534
|
+
assert.equal(result.progressReceipt?.receiptId, "progress-1");
|
|
535
|
+
const reconciled = decodeAgenticCapabilityResultV1({
|
|
536
|
+
...result,
|
|
537
|
+
reconciliation: {
|
|
538
|
+
kind: "scientific_progress_stale",
|
|
539
|
+
latestProgressReceipt: "progress-authoritative",
|
|
540
|
+
recoveryAction: "get_progress"
|
|
541
|
+
}
|
|
542
|
+
});
|
|
543
|
+
assert.equal(reconciled.reconciliation?.kind, "scientific_progress_stale");
|
|
544
|
+
assert.equal(reconciled.reconciliation?.latestProgressReceipt, "progress-authoritative");
|
|
545
|
+
assert.throws(() => decodeAgenticCapabilityResultV1({
|
|
546
|
+
...result,
|
|
547
|
+
reconciliation: {
|
|
548
|
+
kind: "scientific_progress_stale",
|
|
549
|
+
latestProgressReceipt: "progress-authoritative",
|
|
550
|
+
recoveryAction: "retry_same_action"
|
|
551
|
+
}
|
|
552
|
+
}));
|
|
553
|
+
assert.throws(() => decodeAgenticScientificProgressReceiptV1({
|
|
554
|
+
...receipt,
|
|
555
|
+
acceptedEvidenceDigest: otherSha
|
|
556
|
+
}));
|
|
557
|
+
assert.throws(() => decodeAgenticCapabilityResultV1({
|
|
558
|
+
...result,
|
|
559
|
+
progressReceipt: { ...receipt, acceptedEvidenceDigest: otherSha }
|
|
560
|
+
}));
|
|
561
|
+
const largeHandles = Array.from({ length: 128 }, (_, index) => `${String(index).padStart(3, "0")}-${"x".repeat(245)}`);
|
|
562
|
+
assert.throws(() => decodeAgenticCapabilityResultV1({
|
|
563
|
+
...result,
|
|
564
|
+
progressReceipt: {
|
|
565
|
+
...receipt,
|
|
566
|
+
acceptedEvidenceAdded: largeHandles,
|
|
567
|
+
acceptedEvidenceRemoved: largeHandles
|
|
568
|
+
}
|
|
569
|
+
}));
|
|
570
|
+
});
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|