@velum-labs/routekit-eval-setup 1.3.2 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/authoring-responses-request.d.ts +5 -0
- package/dist/adapters/authoring-responses-request.js +38 -0
- package/dist/adapters/evaluation-evidence-freshness.d.ts +7 -0
- package/dist/adapters/evaluation-evidence-freshness.js +76 -0
- package/dist/adapters/git-task-history.d.ts +67 -0
- package/dist/adapters/git-task-history.js +171 -0
- package/dist/adapters/integrated-repository-history.d.ts +21 -0
- package/dist/adapters/integrated-repository-history.js +175 -0
- package/dist/adapters/repository-command-diagnostic.d.ts +8 -0
- package/dist/adapters/repository-command-diagnostic.js +46 -0
- package/dist/adapters/repository-command-evidence.d.ts +13 -0
- package/dist/adapters/repository-command-evidence.js +102 -0
- package/dist/adapters/repository-command-runner.d.ts +238 -0
- package/dist/adapters/repository-command-runner.js +1483 -0
- package/dist/adapters/repository-import-context.d.ts +47 -0
- package/dist/adapters/repository-import-context.js +469 -0
- package/dist/adapters/repository-node-test-reporter.d.ts +3 -0
- package/dist/adapters/repository-node-test-reporter.js +27 -0
- package/dist/adapters/repository-review-evidence.d.ts +39 -0
- package/dist/adapters/repository-review-evidence.js +632 -0
- package/dist/adapters/repository-seed-selection.d.ts +7 -0
- package/dist/adapters/repository-seed-selection.js +79 -0
- package/dist/adapters/repository-solution-edits.d.ts +49 -0
- package/dist/adapters/repository-solution-edits.js +136 -0
- package/dist/adapters/repository-vitest-phase-adapter.d.ts +8 -0
- package/dist/adapters/repository-vitest-phase-adapter.js +310 -0
- package/dist/adapters/repository-vitest-reporter.d.ts +24 -0
- package/dist/adapters/repository-vitest-reporter.js +314 -0
- package/dist/adapters/strict-authoring-schema.d.ts +5 -0
- package/dist/adapters/strict-authoring-schema.js +158 -0
- package/dist/adapters/test-discovery.d.ts +30 -0
- package/dist/adapters/test-discovery.js +124 -0
- package/dist/adapters/typescript-repository-index.d.ts +51 -0
- package/dist/adapters/typescript-repository-index.js +226 -0
- package/dist/agentic-capabilities-protocol.d.ts +1373 -0
- package/dist/agentic-capabilities-protocol.js +786 -0
- package/dist/case-checkpoint-store.d.ts +29 -0
- package/dist/case-checkpoint-store.js +133 -0
- package/dist/case-pipeline-protocol-v2.d.ts +184 -0
- package/dist/case-pipeline-protocol-v2.js +193 -0
- package/dist/case-pipeline-protocol.d.ts +2626 -0
- package/dist/case-pipeline-protocol.js +371 -0
- package/dist/effect-api.d.ts +74 -10
- package/dist/effect-api.js +56 -6
- package/dist/errors.d.ts +31 -0
- package/dist/errors.js +10 -0
- package/dist/eval-capability-execution-envelope.d.ts +64 -0
- package/dist/eval-capability-execution-envelope.js +98 -0
- package/dist/eval-capability-policy.d.ts +90 -0
- package/dist/eval-capability-policy.js +107 -0
- package/dist/eval-event-log.d.ts +140 -0
- package/dist/eval-event-log.js +220 -0
- package/dist/evaluation-authoring-policy.d.ts +18 -0
- package/dist/evaluation-authoring-policy.js +19 -0
- package/dist/evaluation-authoring-validation.d.ts +22 -0
- package/dist/evaluation-authoring-validation.js +72 -0
- package/dist/evaluation-evidence.d.ts +20 -0
- package/dist/evaluation-evidence.js +319 -0
- package/dist/evaluation-grader-calibration-protocol.d.ts +108 -0
- package/dist/evaluation-grader-calibration-protocol.js +80 -0
- package/dist/evaluation-grader-calibration.d.ts +18 -0
- package/dist/evaluation-grader-calibration.js +334 -0
- package/dist/evaluation-grading-policy.d.ts +24 -0
- package/dist/evaluation-grading-policy.js +54 -0
- package/dist/evaluation-proposal-policy.d.ts +4 -0
- package/dist/evaluation-proposal-policy.js +91 -0
- package/dist/evaluation-source-retrieval.d.ts +68 -0
- package/dist/evaluation-source-retrieval.js +513 -0
- package/dist/evaluation-structure-policy.d.ts +29 -0
- package/dist/evaluation-structure-policy.js +138 -0
- package/dist/index.d.ts +124 -17
- package/dist/index.js +69 -11
- package/dist/inspection.js +2 -3
- package/dist/project-artifacts.d.ts +7 -2
- package/dist/project-artifacts.js +49 -136
- package/dist/project-authoring.d.ts +66 -5
- package/dist/project-authoring.js +783 -109
- package/dist/project-contracts.d.ts +419 -84
- package/dist/project-contracts.js +160 -52
- package/dist/project-store.js +2 -1
- package/dist/project-workflow.d.ts +5 -4
- package/dist/project-workflow.js +154 -35
- package/dist/repository-adversary-protocol.d.ts +64 -0
- package/dist/repository-adversary-protocol.js +105 -0
- package/dist/repository-behavior-protocol.d.ts +188 -0
- package/dist/repository-behavior-protocol.js +202 -0
- package/dist/repository-benchmark-protocol.d.ts +487 -0
- package/dist/repository-benchmark-protocol.js +96 -0
- package/dist/repository-execution-protocol.d.ts +150 -0
- package/dist/repository-execution-protocol.js +38 -0
- package/dist/repository-fixture-instructions.d.ts +3 -0
- package/dist/repository-fixture-instructions.js +91 -0
- package/dist/repository-fixture-protocol.d.ts +79 -0
- package/dist/repository-fixture-protocol.js +79 -0
- package/dist/repository-foundry-plan-protocol.d.ts +118 -0
- package/dist/repository-foundry-plan-protocol.js +296 -0
- package/dist/repository-foundry-progress-protocol.d.ts +52 -0
- package/dist/repository-foundry-progress-protocol.js +52 -0
- package/dist/repository-improvement-protocol.d.ts +100 -0
- package/dist/repository-improvement-protocol.js +106 -0
- package/dist/repository-language-model-protocol.d.ts +43 -0
- package/dist/repository-language-model-protocol.js +146 -0
- package/dist/repository-oracle-coverage-protocol.d.ts +18 -0
- package/dist/repository-oracle-coverage-protocol.js +39 -0
- package/dist/repository-oracle-execution-binding.d.ts +27 -0
- package/dist/repository-oracle-execution-binding.js +59 -0
- package/dist/repository-oracle-protocol.d.ts +230 -0
- package/dist/repository-oracle-protocol.js +156 -0
- package/dist/repository-oracle-scope-policy.d.ts +22 -0
- package/dist/repository-oracle-scope-policy.js +92 -0
- package/dist/repository-quality-policy.d.ts +15 -0
- package/dist/repository-quality-policy.js +357 -0
- package/dist/repository-routing-benchmark-protocol.d.ts +176 -0
- package/dist/repository-routing-benchmark-protocol.js +103 -0
- package/dist/repository-routing-model-protocol.d.ts +36 -0
- package/dist/repository-routing-model-protocol.js +89 -0
- package/dist/repository-routing-plan-protocol.d.ts +112 -0
- package/dist/repository-routing-plan-protocol.js +58 -0
- package/dist/repository-routing-quality-policy.d.ts +9 -0
- package/dist/repository-routing-quality-policy.js +191 -0
- package/dist/repository-seed-qualification-progress-protocol.d.ts +205 -0
- package/dist/repository-seed-qualification-progress-protocol.js +28 -0
- package/dist/repository-semantic-calibration-protocol.d.ts +768 -0
- package/dist/repository-semantic-calibration-protocol.js +276 -0
- package/dist/repository-semantic-calibration.d.ts +163 -0
- package/dist/repository-semantic-calibration.js +581 -0
- package/dist/repository-specification-contract-facts-protocol.d.ts +224 -0
- package/dist/repository-specification-contract-facts-protocol.js +276 -0
- package/dist/repository-specification-critique-protocol.d.ts +189 -0
- package/dist/repository-specification-critique-protocol.js +103 -0
- package/dist/repository-task-family-protocol.d.ts +24 -0
- package/dist/repository-task-family-protocol.js +37 -0
- package/dist/repository-task-seed-protocol.d.ts +384 -0
- package/dist/repository-task-seed-protocol.js +236 -0
- package/dist/repository-trajectory-protocol.d.ts +20 -0
- package/dist/repository-trajectory-protocol.js +42 -0
- package/dist/service.js +1 -1
- package/dist/services/adversary/service.d.ts +64 -0
- package/dist/services/adversary/service.js +330 -0
- package/dist/services/benchmark-compiler/service.d.ts +450 -0
- package/dist/services/benchmark-compiler/service.js +9 -0
- package/dist/services/budgeted-model/service.d.ts +118 -0
- package/dist/services/budgeted-model/service.js +460 -0
- package/dist/services/case-authoring/service.d.ts +163 -0
- package/dist/services/case-authoring/service.js +1456 -0
- package/dist/services/case-finalization/service.d.ts +283 -0
- package/dist/services/case-finalization/service.js +370 -0
- package/dist/services/case-generation/service.d.ts +619 -0
- package/dist/services/case-generation/service.js +2628 -0
- package/dist/services/case-pipeline/service.d.ts +31 -0
- package/dist/services/case-pipeline/service.js +485 -0
- package/dist/services/case-pipeline-v2/service.d.ts +70 -0
- package/dist/services/case-pipeline-v2/service.js +477 -0
- package/dist/services/command-observability/service.d.ts +13 -0
- package/dist/services/command-observability/service.js +3 -0
- package/dist/services/dimension-labeling/service.d.ts +77 -0
- package/dist/services/dimension-labeling/service.js +188 -0
- package/dist/services/eval-candidate/service.d.ts +208 -0
- package/dist/services/eval-candidate/service.js +64 -0
- package/dist/services/eval-capabilities/service.d.ts +183 -0
- package/dist/services/eval-capabilities/service.js +1433 -0
- package/dist/services/eval-environment/service.d.ts +173 -0
- package/dist/services/eval-environment/service.js +127 -0
- package/dist/services/evidence-reconstruction/service.d.ts +36 -0
- package/dist/services/evidence-reconstruction/service.js +145 -0
- package/dist/services/fixture-builder/service.d.ts +62 -0
- package/dist/services/fixture-builder/service.js +36 -0
- package/dist/services/fixture-validation/service.d.ts +75 -0
- package/dist/services/fixture-validation/service.js +295 -0
- package/dist/services/foundry/service.d.ts +831 -0
- package/dist/services/foundry/service.js +442 -0
- package/dist/services/foundry-progress/service.d.ts +62 -0
- package/dist/services/foundry-progress/service.js +149 -0
- package/dist/services/foundry-v2/service.d.ts +54 -0
- package/dist/services/foundry-v2/service.js +28 -0
- package/dist/services/grounded-authoring/service.d.ts +126 -0
- package/dist/services/grounded-authoring/service.js +822 -0
- package/dist/services/historical-case/service.d.ts +722 -0
- package/dist/services/historical-case/service.js +177 -0
- package/dist/services/improvement-loop/service.d.ts +59 -0
- package/dist/services/improvement-loop/service.js +176 -0
- package/dist/services/language-model/service.d.ts +52 -0
- package/dist/services/language-model/service.js +194 -0
- package/dist/services/oracle-builder/service.d.ts +146 -0
- package/dist/services/oracle-builder/service.js +513 -0
- package/dist/services/oracle-coverage/service.d.ts +28 -0
- package/dist/services/oracle-coverage/service.js +50 -0
- package/dist/services/oracle-coverage-witness/service.d.ts +130 -0
- package/dist/services/oracle-coverage-witness/service.js +538 -0
- package/dist/services/pipeline-challenge/service.d.ts +551 -0
- package/dist/services/pipeline-challenge/service.js +427 -0
- package/dist/services/pipeline-controls/service.d.ts +130 -0
- package/dist/services/pipeline-controls/service.js +483 -0
- package/dist/services/pipeline-oracle/service.d.ts +8 -0
- package/dist/services/pipeline-oracle/service.js +256 -0
- package/dist/services/pipeline-seed/service.d.ts +298 -0
- package/dist/services/pipeline-seed/service.js +428 -0
- package/dist/services/pipeline-spec/service.d.ts +103 -0
- package/dist/services/pipeline-spec/service.js +619 -0
- package/dist/services/pipeline-tournament/service.d.ts +258 -0
- package/dist/services/pipeline-tournament/service.js +476 -0
- package/dist/services/quality-gate/service.d.ts +233 -0
- package/dist/services/quality-gate/service.js +136 -0
- package/dist/services/repository-bundle/service.d.ts +33 -0
- package/dist/services/repository-bundle/service.js +114 -0
- package/dist/services/repository-model/service.d.ts +105 -0
- package/dist/services/repository-model/service.js +250 -0
- package/dist/services/repository-public-artifact/service.d.ts +133 -0
- package/dist/services/repository-public-artifact/service.js +330 -0
- package/dist/services/routing-benchmark/service.d.ts +362 -0
- package/dist/services/routing-benchmark/service.js +96 -0
- package/dist/services/specification-critic/service.d.ts +92 -0
- package/dist/services/specification-critic/service.js +172 -0
- package/dist/services/task-family/service.d.ts +40 -0
- package/dist/services/task-family/service.js +55 -0
- package/dist/services/task-seed/service.d.ts +906 -0
- package/dist/services/task-seed/service.js +1406 -0
- package/dist/services/task-specification/service.d.ts +27 -0
- package/dist/services/task-specification/service.js +40 -0
- package/dist/services/trajectory-policy/service.d.ts +110 -0
- package/dist/services/trajectory-policy/service.js +216 -0
- package/dist/test/agentic-capabilities-protocol.test.d.ts +1 -0
- package/dist/test/agentic-capabilities-protocol.test.js +570 -0
- package/dist/test/agentic-capabilities.test.d.ts +1 -0
- package/dist/test/agentic-capabilities.test.js +1461 -0
- package/dist/test/agentic-environment.test.d.ts +1 -0
- package/dist/test/agentic-environment.test.js +213 -0
- package/dist/test/case-pipeline-foundation.test.d.ts +1 -0
- package/dist/test/case-pipeline-foundation.test.js +535 -0
- package/dist/test/case-pipeline-protocol-v2.test.d.ts +1 -0
- package/dist/test/case-pipeline-protocol-v2.test.js +124 -0
- package/dist/test/case-pipeline-v2.test.d.ts +1 -0
- package/dist/test/case-pipeline-v2.test.js +286 -0
- package/dist/test/case-pipeline.test.d.ts +1 -0
- package/dist/test/case-pipeline.test.js +851 -0
- package/dist/test/eval-capability-policy.test.d.ts +1 -0
- package/dist/test/eval-capability-policy.test.js +50 -0
- package/dist/test/eval-event-log.test.d.ts +1 -0
- package/dist/test/eval-event-log.test.js +125 -0
- package/dist/test/evaluation-evidence-freshness.test.d.ts +1 -0
- package/dist/test/evaluation-evidence-freshness.test.js +44 -0
- package/dist/test/evaluation-evidence.test.d.ts +1 -0
- package/dist/test/evaluation-evidence.test.js +230 -0
- package/dist/test/evaluation-grader-calibration.test.d.ts +1 -0
- package/dist/test/evaluation-grader-calibration.test.js +373 -0
- package/dist/test/evaluation-proposal-digest.test.d.ts +1 -0
- package/dist/test/evaluation-proposal-digest.test.js +187 -0
- package/dist/test/evaluation-source-retrieval.test.d.ts +1 -0
- package/dist/test/evaluation-source-retrieval.test.js +237 -0
- package/dist/test/evaluation-structure-policy.test.d.ts +1 -0
- package/dist/test/evaluation-structure-policy.test.js +196 -0
- package/dist/test/fixtures/repository-resource-panel.d.ts +39 -0
- package/dist/test/fixtures/repository-resource-panel.js +111 -0
- package/dist/test/fixtures/vitest-boundary-panel.d.ts +84 -0
- package/dist/test/fixtures/vitest-boundary-panel.js +120 -0
- package/dist/test/fixtures/vitest-phase-panel.d.ts +135 -0
- package/dist/test/fixtures/vitest-phase-panel.js +213 -0
- package/dist/test/fixtures/vitest-reporter-results.d.ts +76 -0
- package/dist/test/fixtures/vitest-reporter-results.js +94 -0
- package/dist/test/grounded-authoring.test.d.ts +1 -0
- package/dist/test/grounded-authoring.test.js +565 -0
- package/dist/test/integrated-repository-history.test.d.ts +1 -0
- package/dist/test/integrated-repository-history.test.js +227 -0
- package/dist/test/project-authoring.test.js +593 -43
- package/dist/test/project-workflow.test.js +419 -40
- package/dist/test/repository-authoring-artifacts.test.d.ts +1 -0
- package/dist/test/repository-authoring-artifacts.test.js +185 -0
- package/dist/test/repository-bundle.test.d.ts +1 -0
- package/dist/test/repository-bundle.test.js +52 -0
- package/dist/test/repository-case-generation.test.d.ts +1 -0
- package/dist/test/repository-case-generation.test.js +3465 -0
- package/dist/test/repository-command-diagnostic.test.d.ts +1 -0
- package/dist/test/repository-command-diagnostic.test.js +55 -0
- package/dist/test/repository-command-signals.test.d.ts +1 -0
- package/dist/test/repository-command-signals.test.js +124 -0
- package/dist/test/repository-fixture-scope-coverage.test.d.ts +1 -0
- package/dist/test/repository-fixture-scope-coverage.test.js +127 -0
- package/dist/test/repository-fixture-validation.test.d.ts +1 -0
- package/dist/test/repository-fixture-validation.test.js +362 -0
- package/dist/test/repository-foundry-progress.test.d.ts +1 -0
- package/dist/test/repository-foundry-progress.test.js +110 -0
- package/dist/test/repository-foundry-quality.test.d.ts +1 -0
- package/dist/test/repository-foundry-quality.test.js +1138 -0
- package/dist/test/repository-import-context.test.d.ts +1 -0
- package/dist/test/repository-import-context.test.js +354 -0
- package/dist/test/repository-model-authoring.test.d.ts +1 -0
- package/dist/test/repository-model-authoring.test.js +544 -0
- package/dist/test/repository-model.test.d.ts +1 -0
- package/dist/test/repository-model.test.js +2195 -0
- package/dist/test/repository-node-test-reporter.test.d.ts +1 -0
- package/dist/test/repository-node-test-reporter.test.js +104 -0
- package/dist/test/repository-oracle-concurrency.test.d.ts +1 -0
- package/dist/test/repository-oracle-concurrency.test.js +542 -0
- package/dist/test/repository-oracle-coverage-witness.test.d.ts +1 -0
- package/dist/test/repository-oracle-coverage-witness.test.js +511 -0
- package/dist/test/repository-oracle-coverage.test.d.ts +1 -0
- package/dist/test/repository-oracle-coverage.test.js +168 -0
- package/dist/test/repository-oracle-evidence.test.d.ts +1 -0
- package/dist/test/repository-oracle-evidence.test.js +185 -0
- package/dist/test/repository-oracle-plan.test.d.ts +1 -0
- package/dist/test/repository-oracle-plan.test.js +176 -0
- package/dist/test/repository-overlay-isolation.test.d.ts +1 -0
- package/dist/test/repository-overlay-isolation.test.js +85 -0
- package/dist/test/repository-preparation-cache.test.d.ts +1 -0
- package/dist/test/repository-preparation-cache.test.js +414 -0
- package/dist/test/repository-public-artifact.test.d.ts +1 -0
- package/dist/test/repository-public-artifact.test.js +273 -0
- package/dist/test/repository-qualification-diagnostics.test.d.ts +1 -0
- package/dist/test/repository-qualification-diagnostics.test.js +524 -0
- package/dist/test/repository-reference-authoring.test.d.ts +1 -0
- package/dist/test/repository-reference-authoring.test.js +1633 -0
- package/dist/test/repository-review-evidence-v2.test.d.ts +1 -0
- package/dist/test/repository-review-evidence-v2.test.js +183 -0
- package/dist/test/repository-review-evidence.test.d.ts +1 -0
- package/dist/test/repository-review-evidence.test.js +124 -0
- package/dist/test/repository-seed-exclusions.test.d.ts +1 -0
- package/dist/test/repository-seed-exclusions.test.js +96 -0
- package/dist/test/repository-seed-selection.test.d.ts +1 -0
- package/dist/test/repository-seed-selection.test.js +504 -0
- package/dist/test/repository-semantic-calibration.test.d.ts +1 -0
- package/dist/test/repository-semantic-calibration.test.js +688 -0
- package/dist/test/repository-solution-edits.test.d.ts +1 -0
- package/dist/test/repository-solution-edits.test.js +377 -0
- package/dist/test/repository-specification-budget.test.d.ts +1 -0
- package/dist/test/repository-specification-budget.test.js +171 -0
- package/dist/test/repository-specification-contract-checkpoint.test.d.ts +1 -0
- package/dist/test/repository-specification-contract-checkpoint.test.js +228 -0
- package/dist/test/repository-specification-contract-facts.test.d.ts +1 -0
- package/dist/test/repository-specification-contract-facts.test.js +177 -0
- package/dist/test/repository-trajectory-authoring.test.d.ts +1 -0
- package/dist/test/repository-trajectory-authoring.test.js +176 -0
- package/dist/test/repository-valid-control-plan.test.d.ts +1 -0
- package/dist/test/repository-valid-control-plan.test.js +45 -0
- package/dist/test/repository-vitest-phase.test.d.ts +1 -0
- package/dist/test/repository-vitest-phase.test.js +848 -0
- package/dist/test/repository-vitest-reporter.test.d.ts +1 -0
- package/dist/test/repository-vitest-reporter.test.js +158 -0
- package/dist/test/repository-workspace-build.test.d.ts +1 -0
- package/dist/test/repository-workspace-build.test.js +160 -0
- package/dist/test/strict-authoring-schema.test.d.ts +1 -0
- package/dist/test/strict-authoring-schema.test.js +169 -0
- package/package.json +48 -6
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,377 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { execFile } from "node:child_process";
|
|
3
|
+
import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
|
|
4
|
+
import { tmpdir } from "node:os";
|
|
5
|
+
import { join } from "node:path";
|
|
6
|
+
import { test } from "node:test";
|
|
7
|
+
import { promisify } from "node:util";
|
|
8
|
+
import { Effect, Layer, Schema } from "effect";
|
|
9
|
+
import { loadPinnedSolutionEditSourcesV1, materializeSolutionFileOverridesV1, RepositorySolutionFileOutputV1 } from "../adapters/repository-solution-edits.js";
|
|
10
|
+
import { strictAuthoringSchema } from "../adapters/strict-authoring-schema.js";
|
|
11
|
+
import { EvalAuthoringTransport } from "../project-authoring.js";
|
|
12
|
+
import { repositoryFoundryModelPlanV1 } from "../repository-language-model-protocol.js";
|
|
13
|
+
import { generateRepositorySolutionProposalV1 } from "../services/case-generation/service.js";
|
|
14
|
+
import { RepositoryFoundryProgressLive } from "../services/foundry-progress/service.js";
|
|
15
|
+
import { RepositoryFoundryLanguageModelLive } from "../services/language-model/service.js";
|
|
16
|
+
const path = "src/wire.js";
|
|
17
|
+
const apply = (content, edits, visibleExcerpt = content) => materializeSolutionFileOverridesV1({
|
|
18
|
+
fileOverrides: [{ path, edits }],
|
|
19
|
+
sources: [{ path, content, visibleExcerpt }],
|
|
20
|
+
allowedPaths: new Set([path])
|
|
21
|
+
});
|
|
22
|
+
test("exact edits preserve all untouched bytes and apply original ranges without chaining", () => {
|
|
23
|
+
const base = "first=A; second=B;\r\n// preserved π suffix\r\n";
|
|
24
|
+
const result = apply(base, [
|
|
25
|
+
{ search: "first=A", replace: "first=B; inserted=B" },
|
|
26
|
+
{ search: "second=B", replace: "second=C" }
|
|
27
|
+
], "first=A; second=B;\r\n[truncated]");
|
|
28
|
+
assert.equal(result.fileOverrides[0]?.content, "first=B; inserted=B; second=C;\r\n// preserved π suffix\r\n");
|
|
29
|
+
assert.deepEqual(result.provenance[0]?.ranges, [
|
|
30
|
+
{ start: 0, end: 7 },
|
|
31
|
+
{ start: 9, end: 17 }
|
|
32
|
+
]);
|
|
33
|
+
assert.match(result.provenance[0].baseSha256, /^[a-f0-9]{64}$/u);
|
|
34
|
+
assert.notEqual(result.provenance[0]?.baseSha256, result.provenance[0]?.resultSha256);
|
|
35
|
+
assert.throws(() => apply("original", [
|
|
36
|
+
{ search: "original", replace: "new text" },
|
|
37
|
+
{ search: "new text", replace: "chained" }
|
|
38
|
+
]), /match once/u);
|
|
39
|
+
});
|
|
40
|
+
test("exact edits permit anchored insertions, deletion and adjacent nonoverlapping edits", () => {
|
|
41
|
+
assert.equal(apply("left middle right", [{ search: "middle ", replace: "" }]).fileOverrides[0]?.content, "left right");
|
|
42
|
+
assert.equal(apply("left right", [{ search: "left ", replace: "left inserted " }]).fileOverrides[0]?.content, "left inserted right");
|
|
43
|
+
assert.equal(apply("AB", [
|
|
44
|
+
{ search: "A", replace: "C" },
|
|
45
|
+
{ search: "B", replace: "D" }
|
|
46
|
+
]).fileOverrides[0]?.content, "CD");
|
|
47
|
+
});
|
|
48
|
+
test("exact edits reject absent, repeated, overlapping and unseen search targets", () => {
|
|
49
|
+
for (const [base, edits, visible] of [
|
|
50
|
+
["abc", [{ search: "missing", replace: "x" }], "abc"],
|
|
51
|
+
["aa aa", [{ search: "aa", replace: "x" }], "aa aa"],
|
|
52
|
+
["aaa", [{ search: "aa", replace: "x" }], "aaa"],
|
|
53
|
+
["unique tail unique", [{ search: "unique", replace: "x" }], "unique [truncated]"],
|
|
54
|
+
["visible hidden", [{ search: "hidden", replace: "x" }], "visible [truncated]"],
|
|
55
|
+
[
|
|
56
|
+
"abcde",
|
|
57
|
+
[
|
|
58
|
+
{ search: "abc", replace: "x" },
|
|
59
|
+
{ search: "bcd", replace: "y" }
|
|
60
|
+
],
|
|
61
|
+
"abcde"
|
|
62
|
+
],
|
|
63
|
+
[
|
|
64
|
+
"abc",
|
|
65
|
+
[
|
|
66
|
+
{ search: "abc", replace: "x" },
|
|
67
|
+
{ search: "abc", replace: "y" }
|
|
68
|
+
],
|
|
69
|
+
"abc"
|
|
70
|
+
]
|
|
71
|
+
]) {
|
|
72
|
+
assert.throws(() => apply(base, edits, visible), /match once|overlap/u);
|
|
73
|
+
}
|
|
74
|
+
assert.throws(() => apply("abc", [{ search: "", replace: "x" }]), /nonempty/u);
|
|
75
|
+
assert.throws(() => apply("abc", [{ search: "abc", replace: "abc" }]), /change/u);
|
|
76
|
+
assert.throws(() => apply("abc", []), /between one and 32/u);
|
|
77
|
+
assert.throws(() => apply("abc", [{ search: "abc", replace: "" }]), /empty/u);
|
|
78
|
+
assert.throws(() => apply("abc", [{ search: "abc", replace: "x".repeat(256_001) }]), /file size bound/u);
|
|
79
|
+
});
|
|
80
|
+
test("materialization enforces exact path allowlists and retains legacy full-file compatibility", () => {
|
|
81
|
+
const sources = [{ path, content: "base", visibleExcerpt: "base" }];
|
|
82
|
+
for (const denied of [
|
|
83
|
+
"../src/wire.js",
|
|
84
|
+
"/src/wire.js",
|
|
85
|
+
"src/../wire.js",
|
|
86
|
+
"src\\wire.js",
|
|
87
|
+
"src//wire.js",
|
|
88
|
+
"src/other.js",
|
|
89
|
+
"src/wire.js\0"
|
|
90
|
+
]) {
|
|
91
|
+
assert.throws(() => materializeSolutionFileOverridesV1({
|
|
92
|
+
fileOverrides: [{ path: denied, edits: [{ search: "base", replace: "next" }] }],
|
|
93
|
+
sources,
|
|
94
|
+
allowedPaths: new Set([path])
|
|
95
|
+
}), /unsafe|unreviewed/u);
|
|
96
|
+
}
|
|
97
|
+
assert.throws(() => materializeSolutionFileOverridesV1({
|
|
98
|
+
fileOverrides: [
|
|
99
|
+
{ path, content: "one" },
|
|
100
|
+
{ path, content: "two" }
|
|
101
|
+
],
|
|
102
|
+
sources,
|
|
103
|
+
allowedPaths: new Set([path])
|
|
104
|
+
}), /duplicate/u);
|
|
105
|
+
assert.throws(() => materializeSolutionFileOverridesV1({
|
|
106
|
+
fileOverrides: [{ path, edits: [{ search: "base", replace: "next" }] }],
|
|
107
|
+
sources: [],
|
|
108
|
+
allowedPaths: new Set([path])
|
|
109
|
+
}), /complete pinned/u);
|
|
110
|
+
const legacy = [{ path, content: "legacy full-file output" }];
|
|
111
|
+
assert.deepEqual(materializeSolutionFileOverridesV1({
|
|
112
|
+
fileOverrides: legacy,
|
|
113
|
+
sources,
|
|
114
|
+
allowedPaths: new Set([path])
|
|
115
|
+
}).fileOverrides, legacy);
|
|
116
|
+
});
|
|
117
|
+
test("the model-facing nested union is strict and accepts both compact and legacy fixtures", () => {
|
|
118
|
+
const schema = Schema.Struct({ fileOverrides: Schema.Array(RepositorySolutionFileOutputV1) });
|
|
119
|
+
const document = Schema.toJsonSchemaDocument(schema);
|
|
120
|
+
const strict = strictAuthoringSchema({ ...document.schema, $defs: document.definitions });
|
|
121
|
+
const visit = (value) => {
|
|
122
|
+
if (typeof value !== "object" || value === null)
|
|
123
|
+
return;
|
|
124
|
+
const object = value;
|
|
125
|
+
if (object.type === "object") {
|
|
126
|
+
assert.equal(object.additionalProperties, false);
|
|
127
|
+
assert.deepEqual([...object.required].sort(), Object.keys(object.properties).sort());
|
|
128
|
+
}
|
|
129
|
+
for (const entry of Object.values(object))
|
|
130
|
+
visit(entry);
|
|
131
|
+
};
|
|
132
|
+
visit(strict.jsonSchema);
|
|
133
|
+
for (const file of [
|
|
134
|
+
{ path, content: "legacy" },
|
|
135
|
+
{ path, edits: [{ search: "base", replace: "next" }] }
|
|
136
|
+
]) {
|
|
137
|
+
const output = { fileOverrides: [file] };
|
|
138
|
+
assert.deepEqual(Schema.decodeUnknownSync(schema)(strict.normalizeResponse(output)), output);
|
|
139
|
+
}
|
|
140
|
+
});
|
|
141
|
+
test("ten variants of a real 20KB source fixture require over 95 percent less model output", () => {
|
|
142
|
+
const base = "export const wire = value => value;\n" + "// unchanged context\n".repeat(1_024);
|
|
143
|
+
assert.ok(Buffer.byteLength(base) >= 20_000);
|
|
144
|
+
const raw = Array.from({ length: 10 }, (_, index) => ({
|
|
145
|
+
path,
|
|
146
|
+
edits: [{ search: "value => value", replace: `value => value + ${index + 1}` }]
|
|
147
|
+
}));
|
|
148
|
+
const full = raw.map(({ edits }) => apply(base, edits, base.slice(0, 100) + "\n[truncated]").fileOverrides[0]);
|
|
149
|
+
assert.ok(Buffer.byteLength(JSON.stringify(raw)) < Buffer.byteLength(JSON.stringify(full)) * 0.05);
|
|
150
|
+
for (const entry of full)
|
|
151
|
+
assert.ok(entry?.content.endsWith("// unchanged context\n".repeat(1_024)));
|
|
152
|
+
});
|
|
153
|
+
test("edit bases come only from reviewed existing paths in the pinned initial tree", async () => {
|
|
154
|
+
const root = await mkdtemp(join(tmpdir(), "routekit-edit-base-"));
|
|
155
|
+
const git = async (...args) => (await promisify(execFile)("git", ["-C", root, ...args], { encoding: "utf8" })).stdout.trim();
|
|
156
|
+
try {
|
|
157
|
+
await git("init", "-q");
|
|
158
|
+
await git("config", "user.name", "Foundry Test");
|
|
159
|
+
await git("config", "user.email", "foundry@example.test");
|
|
160
|
+
await mkdir(join(root, "src"));
|
|
161
|
+
await writeFile(join(root, path), "initial visible prefix\nPRIVATE UNSEEN INITIAL SUFFIX\n");
|
|
162
|
+
await writeFile(join(root, "src/denied.js"), "DENIED SOURCE\n");
|
|
163
|
+
await git("add", ".");
|
|
164
|
+
await git("commit", "-qm", "initial");
|
|
165
|
+
const initialCommit = await git("rev-parse", "HEAD");
|
|
166
|
+
await writeFile(join(root, path), "REFERENCE POST-CHANGE SOURCE\n");
|
|
167
|
+
await git("commit", "-qam", "reference");
|
|
168
|
+
await writeFile(join(root, path), "MUTABLE WORKTREE SOURCE\n");
|
|
169
|
+
const sources = await Effect.runPromise(loadPinnedSolutionEditSourcesV1({
|
|
170
|
+
repositoryRoot: root,
|
|
171
|
+
initialCommit,
|
|
172
|
+
visibleSources: [
|
|
173
|
+
{ path, content: "initial visible" },
|
|
174
|
+
{ path, content: "initial visible prefix\n[truncated]" },
|
|
175
|
+
{ path: "src/denied.js", content: "DENIED SOURCE" },
|
|
176
|
+
{ path: "src/missing.js", content: "invented" }
|
|
177
|
+
],
|
|
178
|
+
allowedPaths: new Set([path, "src/missing.js"])
|
|
179
|
+
}));
|
|
180
|
+
assert.equal(sources.length, 1);
|
|
181
|
+
assert.equal(sources[0]?.visibleExcerpt, "initial visible prefix\n[truncated]");
|
|
182
|
+
assert.equal(sources[0]?.additionalVisibleExcerpts, undefined);
|
|
183
|
+
assert.equal(sources[0]?.content, "initial visible prefix\nPRIVATE UNSEEN INITIAL SUFFIX\n");
|
|
184
|
+
assert.doesNotMatch(JSON.stringify(sources), /REFERENCE|WORKTREE|DENIED SOURCE/u);
|
|
185
|
+
const result = materializeSolutionFileOverridesV1({
|
|
186
|
+
fileOverrides: [
|
|
187
|
+
{ path, edits: [{ search: "initial visible prefix", replace: "updated visible prefix" }] }
|
|
188
|
+
],
|
|
189
|
+
sources,
|
|
190
|
+
allowedPaths: new Set([path])
|
|
191
|
+
});
|
|
192
|
+
assert.equal(result.fileOverrides[0]?.content, "updated visible prefix\nPRIVATE UNSEEN INITIAL SUFFIX\n");
|
|
193
|
+
const separate = await Effect.runPromise(loadPinnedSolutionEditSourcesV1({
|
|
194
|
+
repositoryRoot: root,
|
|
195
|
+
initialCommit,
|
|
196
|
+
visibleSources: [
|
|
197
|
+
{ path, content: "initial visible prefix\n" },
|
|
198
|
+
{ path, content: "INITIAL SUFFIX\n" }
|
|
199
|
+
],
|
|
200
|
+
allowedPaths: new Set([path])
|
|
201
|
+
}));
|
|
202
|
+
assert.equal(separate.length, 1);
|
|
203
|
+
assert.deepEqual(separate[0]?.additionalVisibleExcerpts, ["INITIAL SUFFIX\n"]);
|
|
204
|
+
assert.equal(materializeSolutionFileOverridesV1({
|
|
205
|
+
fileOverrides: [{ path, edits: [{ search: "INITIAL SUFFIX", replace: "SUPPLIED SUFFIX" }] }],
|
|
206
|
+
sources: separate,
|
|
207
|
+
allowedPaths: new Set([path])
|
|
208
|
+
}).fileOverrides[0]?.content, "initial visible prefix\nPRIVATE UNSEEN SUPPLIED SUFFIX\n");
|
|
209
|
+
assert.throws(() => materializeSolutionFileOverridesV1({
|
|
210
|
+
fileOverrides: [{ path, edits: [{ search: "prefix\nPRIVATE UNSEEN INITIAL", replace: "invented bridge" }] }],
|
|
211
|
+
sources: separate,
|
|
212
|
+
allowedPaths: new Set([path])
|
|
213
|
+
}), /match once/u);
|
|
214
|
+
const adjacent = await Effect.runPromise(loadPinnedSolutionEditSourcesV1({
|
|
215
|
+
repositoryRoot: root,
|
|
216
|
+
initialCommit,
|
|
217
|
+
visibleSources: [
|
|
218
|
+
{ path, content: "initial visible " },
|
|
219
|
+
{ path, content: "prefix\n" }
|
|
220
|
+
],
|
|
221
|
+
allowedPaths: new Set([path])
|
|
222
|
+
}));
|
|
223
|
+
assert.throws(() => materializeSolutionFileOverridesV1({
|
|
224
|
+
fileOverrides: [{ path, edits: [{ search: "visible prefix", replace: "joined snippets" }] }],
|
|
225
|
+
sources: adjacent,
|
|
226
|
+
allowedPaths: new Set([path])
|
|
227
|
+
}), /match once/u, "adjacent snippets cannot invent a larger visible anchor by concatenation");
|
|
228
|
+
}
|
|
229
|
+
finally {
|
|
230
|
+
await rm(root, { recursive: true, force: true });
|
|
231
|
+
}
|
|
232
|
+
});
|
|
233
|
+
test("all solution roles retain raw edits and separate host provenance without leaking full bases into model inputs", async () => {
|
|
234
|
+
for (const role of [
|
|
235
|
+
"valid-solution-generator-a",
|
|
236
|
+
"valid-solution-generator-b",
|
|
237
|
+
"adversary",
|
|
238
|
+
"held-out-adversary"
|
|
239
|
+
]) {
|
|
240
|
+
for (const repair of [false, true]) {
|
|
241
|
+
const operationId = `${role}${repair ? ":repair-1" : ""}`;
|
|
242
|
+
const kind = role.startsWith("valid-") ? "independent-valid" : "mutation";
|
|
243
|
+
const raw = {
|
|
244
|
+
solutions: [
|
|
245
|
+
{
|
|
246
|
+
id: "candidate",
|
|
247
|
+
family: "wire",
|
|
248
|
+
kind,
|
|
249
|
+
fileOverrides: [
|
|
250
|
+
{ path, edits: [{ search: "visible original", replace: "visible changed" }] }
|
|
251
|
+
]
|
|
252
|
+
}
|
|
253
|
+
]
|
|
254
|
+
};
|
|
255
|
+
const rawText = JSON.stringify(raw);
|
|
256
|
+
const artifacts = [];
|
|
257
|
+
const events = [];
|
|
258
|
+
const transport = Layer.succeed(EvalAuthoringTransport, EvalAuthoringTransport.of({
|
|
259
|
+
complete: (request) => {
|
|
260
|
+
assert.match(request.instructions, /ORIGINAL pre-change source/u);
|
|
261
|
+
assert.doesNotMatch(request.input, /HOST ONLY SUFFIX|REFERENCE_SECRET|ORACLE_SECRET/u);
|
|
262
|
+
return Effect.succeed(rawText);
|
|
263
|
+
}
|
|
264
|
+
}));
|
|
265
|
+
const result = await Effect.runPromise(generateRepositorySolutionProposalV1({
|
|
266
|
+
request: {
|
|
267
|
+
plan: repositoryFoundryModelPlanV1({ primaryModel: "codex/gpt-6-astra" }),
|
|
268
|
+
role,
|
|
269
|
+
operationId,
|
|
270
|
+
instructions: "Solve from visible context only",
|
|
271
|
+
schemaName: "solution",
|
|
272
|
+
maximumOutputTokens: 512,
|
|
273
|
+
input: { preChangeSources: [{ path, content: "visible original\n[truncated]" }] }
|
|
274
|
+
},
|
|
275
|
+
initialCommit: "1".repeat(40),
|
|
276
|
+
editSources: [
|
|
277
|
+
{
|
|
278
|
+
path,
|
|
279
|
+
content: "visible original\nHOST ONLY SUFFIX\n",
|
|
280
|
+
visibleExcerpt: "visible original\n[truncated]"
|
|
281
|
+
}
|
|
282
|
+
],
|
|
283
|
+
allowedSolutionPaths: new Set([path]),
|
|
284
|
+
allowedRewardHackingPaths: new Set([path])
|
|
285
|
+
}).pipe(Effect.provide(Layer.merge(RepositoryFoundryLanguageModelLive.pipe(Layer.provide(transport)), RepositoryFoundryProgressLive((event) => Effect.sync(() => {
|
|
286
|
+
events.push(event);
|
|
287
|
+
}), (artifact) => Effect.sync(() => {
|
|
288
|
+
artifacts.push(artifact);
|
|
289
|
+
}))))));
|
|
290
|
+
assert.equal(result.call.operationId, operationId);
|
|
291
|
+
assert.deepEqual(result.rawProposal, raw);
|
|
292
|
+
assert.deepEqual(result.value.solutions[0]?.fileOverrides, [
|
|
293
|
+
{ path, content: "visible changed\nHOST ONLY SUFFIX\n" }
|
|
294
|
+
]);
|
|
295
|
+
assert.equal(artifacts[0]?.response.text, rawText);
|
|
296
|
+
assert.deepEqual(artifacts.find((entry) => entry.operationId === operationId && entry.validation === "validated")?.value, raw);
|
|
297
|
+
const materialized = artifacts.find((entry) => entry.operationId === `${operationId}:materialized`);
|
|
298
|
+
assert.equal(materialized?.role, "host-exact-edit-materializer");
|
|
299
|
+
assert.equal(materialized?.request.maximumOutputTokens, 0);
|
|
300
|
+
assert.deepEqual(materialized?.request.input, {
|
|
301
|
+
sourceModelOperationId: operationId,
|
|
302
|
+
initialCommit: "1".repeat(40)
|
|
303
|
+
});
|
|
304
|
+
assert.match(JSON.stringify(materialized?.value), /baseSha256|resultSha256/u);
|
|
305
|
+
assert.doesNotMatch(JSON.stringify(events), /HOST ONLY SUFFIX|visible original|visible changed/u);
|
|
306
|
+
assert.equal(events.filter((event) => event.kind === "model-call" && event.phase === "start").length, 1);
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
});
|
|
310
|
+
test("visible protected manifests remain editable only by explicitly classified adversarial reward hacks", async () => {
|
|
311
|
+
const manifest = '{"scripts":{"test":"node --test"}}';
|
|
312
|
+
for (const role of ["valid-solution-generator-a", "valid-solution-generator-b", "adversary", "held-out-adversary"]) {
|
|
313
|
+
for (const kind of ["mutation", "reward-hacking"]) {
|
|
314
|
+
const raw = { solutions: [{ id: "manifest-control", family: "test-command", kind,
|
|
315
|
+
fileOverrides: [{ path: "package.json", edits: [{ search: "node --test", replace: "node --version" }] }]
|
|
316
|
+
}] };
|
|
317
|
+
const transport = Layer.succeed(EvalAuthoringTransport, EvalAuthoringTransport.of({
|
|
318
|
+
complete: () => Effect.succeed(JSON.stringify(raw))
|
|
319
|
+
}));
|
|
320
|
+
const run = generateRepositorySolutionProposalV1({
|
|
321
|
+
request: {
|
|
322
|
+
plan: repositoryFoundryModelPlanV1({ primaryModel: "codex/gpt-6-astra" }),
|
|
323
|
+
role, operationId: `${role}:${kind}`, instructions: "Use only reviewed paths.",
|
|
324
|
+
schemaName: "protected_manifest_regression", maximumOutputTokens: 512,
|
|
325
|
+
input: { repositoryEnvironment: { sources: [{ path: "package.json", content: manifest }] } }
|
|
326
|
+
},
|
|
327
|
+
initialCommit: "1".repeat(40),
|
|
328
|
+
editSources: [{ path: "package.json", content: manifest, visibleExcerpt: manifest }],
|
|
329
|
+
allowedSolutionPaths: new Set([path]),
|
|
330
|
+
allowedRewardHackingPaths: new Set([path, "package.json"])
|
|
331
|
+
}).pipe(Effect.provide(RepositoryFoundryLanguageModelLive.pipe(Layer.provide(transport))));
|
|
332
|
+
if ((role === "adversary" || role === "held-out-adversary") && kind === "reward-hacking") {
|
|
333
|
+
const result = await Effect.runPromise(run);
|
|
334
|
+
assert.equal(result.value.solutions[0]?.fileOverrides[0]?.content, manifest.replace("node --test", "node --version"));
|
|
335
|
+
}
|
|
336
|
+
else {
|
|
337
|
+
await assert.rejects(Effect.runPromise(run), /unreviewed path/u);
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
});
|
|
342
|
+
test("compact proposals cannot expand beyond the 32MiB host materialization bound", async () => {
|
|
343
|
+
const content = "VALUE=0;\n" + "x".repeat(255_900);
|
|
344
|
+
assert.ok(Buffer.byteLength(content) <= 256_000);
|
|
345
|
+
const raw = {
|
|
346
|
+
solutions: Array.from({ length: 134 }, (_, index) => ({
|
|
347
|
+
id: `case-${index}`,
|
|
348
|
+
family: `family-${index}`,
|
|
349
|
+
kind: "mutation",
|
|
350
|
+
fileOverrides: [{ path, edits: [{ search: "VALUE=0", replace: "VALUE=1" }] }]
|
|
351
|
+
}))
|
|
352
|
+
};
|
|
353
|
+
assert.ok(Buffer.byteLength(JSON.stringify(raw)) < 32_000);
|
|
354
|
+
const artifacts = [];
|
|
355
|
+
const transport = Layer.succeed(EvalAuthoringTransport, EvalAuthoringTransport.of({
|
|
356
|
+
complete: () => Effect.succeed(JSON.stringify(raw))
|
|
357
|
+
}));
|
|
358
|
+
await assert.rejects(Effect.runPromise(generateRepositorySolutionProposalV1({
|
|
359
|
+
request: {
|
|
360
|
+
plan: repositoryFoundryModelPlanV1({ primaryModel: "codex/gpt-6-astra" }),
|
|
361
|
+
role: "adversary",
|
|
362
|
+
operationId: "bounded-expansion",
|
|
363
|
+
instructions: "Use exact edits",
|
|
364
|
+
schemaName: "solutions",
|
|
365
|
+
maximumOutputTokens: 32_768,
|
|
366
|
+
input: { preChangeSources: [{ path, content: "VALUE=0;\n[truncated]" }] }
|
|
367
|
+
},
|
|
368
|
+
initialCommit: "1".repeat(40),
|
|
369
|
+
editSources: [{ path, content, visibleExcerpt: "VALUE=0;\n[truncated]" }],
|
|
370
|
+
allowedSolutionPaths: new Set([path]),
|
|
371
|
+
allowedRewardHackingPaths: new Set([path])
|
|
372
|
+
}).pipe(Effect.provide(Layer.merge(RepositoryFoundryLanguageModelLive.pipe(Layer.provide(transport)), RepositoryFoundryProgressLive(undefined, (artifact) => Effect.sync(() => {
|
|
373
|
+
artifacts.push(artifact);
|
|
374
|
+
})))))), /32MiB total source bound/u);
|
|
375
|
+
assert.ok(artifacts.some((entry) => entry.operationId === "bounded-expansion" && entry.validation === "validated"));
|
|
376
|
+
assert.ok(!artifacts.some((entry) => entry.role === "host-exact-edit-materializer"));
|
|
377
|
+
});
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { test } from "node:test";
|
|
3
|
+
import { assertRepositoryFoundryGenerationPlanV1, repositoryFoundryGenerationBudgetV1, repositoryFoundryGenerationPlanDigestV1 } from "../repository-foundry-plan-protocol.js";
|
|
4
|
+
import { repositoryFoundryModelPlanV1 } from "../repository-language-model-protocol.js";
|
|
5
|
+
const plan = (revisions, specificationContractFactsVersion, reviewInputVersion) => {
|
|
6
|
+
const unsigned = {
|
|
7
|
+
version: 1,
|
|
8
|
+
repository: { root: "/repo", requestedRef: "HEAD", commit: "commit", tree: "tree" },
|
|
9
|
+
caseId: "legacy-case",
|
|
10
|
+
...(reviewInputVersion === undefined ? {} : { reviewInputVersion }),
|
|
11
|
+
...(specificationContractFactsVersion === undefined ? {} : { specificationContractFactsVersion }),
|
|
12
|
+
seedId: "seed",
|
|
13
|
+
initialCommit: "initial",
|
|
14
|
+
referenceCommit: "reference",
|
|
15
|
+
modelPlan: repositoryFoundryModelPlanV1({ primaryModel: "codex/gpt-6-astra" }),
|
|
16
|
+
qualification: { batches: 2, repetitionsPerBatch: 2, oracleRepetitions: 2 },
|
|
17
|
+
budget: repositoryFoundryGenerationBudgetV1(2, undefined, revisions, undefined, undefined, reviewInputVersion, specificationContractFactsVersion)
|
|
18
|
+
};
|
|
19
|
+
return { ...unsigned, planDigest: repositoryFoundryGenerationPlanDigestV1(unsigned) };
|
|
20
|
+
};
|
|
21
|
+
test("absent specification revisions preserve the pre-change immutable plan digest and budget", () => {
|
|
22
|
+
const legacy = plan();
|
|
23
|
+
assert.equal(legacy.planDigest, "357e58c2853e4379791e3d1c5b9475993a65c371684c8f6a6f4c69400f1d2994");
|
|
24
|
+
assert.deepEqual(legacy.budget, {
|
|
25
|
+
plannedModelCalls: 35,
|
|
26
|
+
maximumTransientRetries: 3,
|
|
27
|
+
maximumHillClimbAttempts: 2,
|
|
28
|
+
requiredTotalInputTokens: 17_920_000,
|
|
29
|
+
requiredTotalOutputTokens: 1_671_168,
|
|
30
|
+
perCallOutputTokens: 65_536,
|
|
31
|
+
wallTimeMs: 7_200_000
|
|
32
|
+
});
|
|
33
|
+
assertRepositoryFoundryGenerationPlanV1(legacy);
|
|
34
|
+
assert.equal(Object.hasOwn(legacy, "specificationContractFactsVersion"), false);
|
|
35
|
+
assert.equal(JSON.stringify(repositoryFoundryGenerationBudgetV1(2, undefined, undefined, undefined, undefined, undefined, undefined)), JSON.stringify(legacy.budget), "an absent appended marker preserves the exact legacy budget bytes");
|
|
36
|
+
});
|
|
37
|
+
test("contract-fact revisions reserve one extra analyst call without expanding the retry reserve", () => {
|
|
38
|
+
for (const reviewInputVersion of [undefined, 2]) {
|
|
39
|
+
for (const revisions of [0, 1, 2]) {
|
|
40
|
+
const legacy = plan(revisions, undefined, reviewInputVersion);
|
|
41
|
+
const current = plan(revisions, 1, reviewInputVersion);
|
|
42
|
+
assertRepositoryFoundryGenerationPlanV1(legacy);
|
|
43
|
+
assertRepositoryFoundryGenerationPlanV1(current);
|
|
44
|
+
assert.equal(current.specificationContractFactsVersion, 1);
|
|
45
|
+
assert.notEqual(current.planDigest, legacy.planDigest, "even zero revisions pins the new protocol");
|
|
46
|
+
assert.deepEqual(current.budget, {
|
|
47
|
+
...legacy.budget,
|
|
48
|
+
plannedModelCalls: legacy.budget.plannedModelCalls + revisions,
|
|
49
|
+
requiredTotalInputTokens: legacy.budget.requiredTotalInputTokens + revisions * 512_000,
|
|
50
|
+
requiredTotalOutputTokens: legacy.budget.requiredTotalOutputTokens + revisions * 32_768
|
|
51
|
+
});
|
|
52
|
+
assert.equal(current.budget.maximumTransientRetries, 3);
|
|
53
|
+
assert.equal(current.budget.perCallOutputTokens, legacy.budget.perCallOutputTokens);
|
|
54
|
+
assert.equal(Object.hasOwn(current.budget, "specificationContractFactsVersion"), false);
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
});
|
|
58
|
+
test("contract-fact plans require an explicit revision allowance and cannot borrow an old immutable budget", () => {
|
|
59
|
+
assert.throws(() => repositoryFoundryGenerationBudgetV1(2, undefined, undefined, undefined, undefined, undefined, 1), /explicit specification revision allowance/u);
|
|
60
|
+
for (const marker of [0, 2, -1, 0.5]) {
|
|
61
|
+
assert.throws(() => repositoryFoundryGenerationBudgetV1(2, undefined, 0, undefined, undefined, undefined, marker), /version 1/u);
|
|
62
|
+
assert.throws(() => assertRepositoryFoundryGenerationPlanV1({
|
|
63
|
+
...plan(0, 1), specificationContractFactsVersion: marker
|
|
64
|
+
}));
|
|
65
|
+
}
|
|
66
|
+
const { planDigest: _legacyDigest, ...legacy } = plan();
|
|
67
|
+
const missingAllowance = { ...legacy, specificationContractFactsVersion: 1 };
|
|
68
|
+
assert.throws(() => assertRepositoryFoundryGenerationPlanV1({
|
|
69
|
+
...missingAllowance, planDigest: repositoryFoundryGenerationPlanDigestV1(missingAllowance)
|
|
70
|
+
}), /explicit specification revision allowance/u);
|
|
71
|
+
for (const field of ["plannedModelCalls", "requiredTotalInputTokens", "requiredTotalOutputTokens"]) {
|
|
72
|
+
const { planDigest: _digest, ...current } = plan(2, 1);
|
|
73
|
+
const unmarked = plan(2);
|
|
74
|
+
const underreserved = {
|
|
75
|
+
...current,
|
|
76
|
+
budget: { ...current.budget, [field]: unmarked.budget[field] }
|
|
77
|
+
};
|
|
78
|
+
assert.throws(() => assertRepositoryFoundryGenerationPlanV1({
|
|
79
|
+
...underreserved, planDigest: repositoryFoundryGenerationPlanDigestV1(underreserved)
|
|
80
|
+
}), /immutable budget/u);
|
|
81
|
+
}
|
|
82
|
+
assert.throws(() => assertRepositoryFoundryGenerationPlanV1({
|
|
83
|
+
...plan(0), specificationContractFactsVersion: 1
|
|
84
|
+
}), /digest/u, "a saved zero-revision plan cannot acquire a marker without a new digest");
|
|
85
|
+
});
|
|
86
|
+
test("each pinned specification revision reserves one writer and two independent critics", () => {
|
|
87
|
+
const legacy = plan();
|
|
88
|
+
for (const revisions of [0, 1, 2]) {
|
|
89
|
+
const current = plan(revisions);
|
|
90
|
+
assert.equal(current.budget.maximumSpecificationRevisions, revisions);
|
|
91
|
+
assert.equal(current.budget.plannedModelCalls, 35 + revisions * 3);
|
|
92
|
+
assert.equal(current.budget.requiredTotalInputTokens, 17_920_000 + revisions * 1_536_000);
|
|
93
|
+
assert.equal(current.budget.requiredTotalOutputTokens, 1_671_168 + revisions * 98_304);
|
|
94
|
+
assert.equal(current.budget.maximumTransientRetries, legacy.budget.maximumTransientRetries);
|
|
95
|
+
assert.equal(current.budget.perCallOutputTokens, legacy.budget.perCallOutputTokens);
|
|
96
|
+
assert.notEqual(current.planDigest, legacy.planDigest);
|
|
97
|
+
assertRepositoryFoundryGenerationPlanV1(current);
|
|
98
|
+
}
|
|
99
|
+
});
|
|
100
|
+
test("invalid revision bounds and underreserved immutable plans fail closed", () => {
|
|
101
|
+
for (const revisions of [-1, 0.5, 3, Number.NaN, Number.POSITIVE_INFINITY]) {
|
|
102
|
+
assert.throws(() => repositoryFoundryGenerationBudgetV1(2, undefined, revisions), /integer from 0 to 2/u);
|
|
103
|
+
const { planDigest: _digest, ...legacy } = plan();
|
|
104
|
+
const invalid = {
|
|
105
|
+
...legacy,
|
|
106
|
+
budget: { ...legacy.budget, maximumSpecificationRevisions: revisions }
|
|
107
|
+
};
|
|
108
|
+
assert.throws(() => assertRepositoryFoundryGenerationPlanV1({
|
|
109
|
+
...invalid,
|
|
110
|
+
planDigest: repositoryFoundryGenerationPlanDigestV1(invalid)
|
|
111
|
+
}));
|
|
112
|
+
}
|
|
113
|
+
for (const field of [
|
|
114
|
+
"plannedModelCalls",
|
|
115
|
+
"requiredTotalInputTokens",
|
|
116
|
+
"requiredTotalOutputTokens"
|
|
117
|
+
]) {
|
|
118
|
+
const { planDigest: _digest, ...current } = plan(2);
|
|
119
|
+
const underreserved = {
|
|
120
|
+
...current,
|
|
121
|
+
budget: { ...current.budget, [field]: current.budget[field] - 1 }
|
|
122
|
+
};
|
|
123
|
+
assert.throws(() => assertRepositoryFoundryGenerationPlanV1({
|
|
124
|
+
...underreserved,
|
|
125
|
+
planDigest: repositoryFoundryGenerationPlanDigestV1(underreserved)
|
|
126
|
+
}), /immutable budget/u);
|
|
127
|
+
}
|
|
128
|
+
const current = plan(2);
|
|
129
|
+
assert.throws(() => assertRepositoryFoundryGenerationPlanV1({
|
|
130
|
+
...current,
|
|
131
|
+
budget: repositoryFoundryGenerationBudgetV1(2, undefined, 1)
|
|
132
|
+
}), /digest/u);
|
|
133
|
+
});
|
|
134
|
+
test("independent fixture repair allowance and dependency context are immutable without changing legacy plans", () => {
|
|
135
|
+
const { planDigest: _digest, ...legacy } = plan(2);
|
|
136
|
+
for (const repairs of [0, 1, 2, 3]) {
|
|
137
|
+
const unsigned = {
|
|
138
|
+
version: legacy.version,
|
|
139
|
+
repository: legacy.repository,
|
|
140
|
+
caseId: legacy.caseId,
|
|
141
|
+
dependencyContextVersion: 1,
|
|
142
|
+
seedId: legacy.seedId,
|
|
143
|
+
initialCommit: legacy.initialCommit,
|
|
144
|
+
referenceCommit: legacy.referenceCommit,
|
|
145
|
+
modelPlan: legacy.modelPlan,
|
|
146
|
+
qualification: legacy.qualification,
|
|
147
|
+
budget: repositoryFoundryGenerationBudgetV1(2, undefined, 2, undefined, repairs)
|
|
148
|
+
};
|
|
149
|
+
const current = { ...unsigned, planDigest: repositoryFoundryGenerationPlanDigestV1(unsigned) };
|
|
150
|
+
assertRepositoryFoundryGenerationPlanV1(current);
|
|
151
|
+
assert.equal(current.budget.maximumHillClimbAttempts, 2);
|
|
152
|
+
assert.equal(current.budget.maximumFixtureRepairAttempts, repairs);
|
|
153
|
+
assert.equal(current.budget.plannedModelCalls, legacy.budget.plannedModelCalls + repairs);
|
|
154
|
+
assert.equal(current.budget.requiredTotalOutputTokens, legacy.budget.requiredTotalOutputTokens + repairs * 65_536);
|
|
155
|
+
assert.throws(() => assertRepositoryFoundryGenerationPlanV1({
|
|
156
|
+
...current,
|
|
157
|
+
dependencyContextVersion: undefined
|
|
158
|
+
}));
|
|
159
|
+
const underreserved = {
|
|
160
|
+
...unsigned,
|
|
161
|
+
budget: { ...unsigned.budget, plannedModelCalls: unsigned.budget.plannedModelCalls - 1 }
|
|
162
|
+
};
|
|
163
|
+
assert.throws(() => assertRepositoryFoundryGenerationPlanV1({
|
|
164
|
+
...underreserved,
|
|
165
|
+
planDigest: repositoryFoundryGenerationPlanDigestV1(underreserved)
|
|
166
|
+
}));
|
|
167
|
+
}
|
|
168
|
+
for (const repairs of [-1, 0.5, 4, Number.NaN, Number.POSITIVE_INFINITY]) {
|
|
169
|
+
assert.throws(() => repositoryFoundryGenerationBudgetV1(2, undefined, 2, undefined, repairs), /fixture repairs must be an integer/u);
|
|
170
|
+
}
|
|
171
|
+
});
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|