@velum-labs/routekit-eval-setup 1.3.2 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/authoring-responses-request.d.ts +5 -0
- package/dist/adapters/authoring-responses-request.js +38 -0
- package/dist/adapters/evaluation-evidence-freshness.d.ts +7 -0
- package/dist/adapters/evaluation-evidence-freshness.js +76 -0
- package/dist/adapters/git-task-history.d.ts +67 -0
- package/dist/adapters/git-task-history.js +171 -0
- package/dist/adapters/integrated-repository-history.d.ts +21 -0
- package/dist/adapters/integrated-repository-history.js +175 -0
- package/dist/adapters/repository-command-diagnostic.d.ts +8 -0
- package/dist/adapters/repository-command-diagnostic.js +46 -0
- package/dist/adapters/repository-command-evidence.d.ts +13 -0
- package/dist/adapters/repository-command-evidence.js +102 -0
- package/dist/adapters/repository-command-runner.d.ts +238 -0
- package/dist/adapters/repository-command-runner.js +1483 -0
- package/dist/adapters/repository-import-context.d.ts +47 -0
- package/dist/adapters/repository-import-context.js +469 -0
- package/dist/adapters/repository-node-test-reporter.d.ts +3 -0
- package/dist/adapters/repository-node-test-reporter.js +27 -0
- package/dist/adapters/repository-review-evidence.d.ts +39 -0
- package/dist/adapters/repository-review-evidence.js +632 -0
- package/dist/adapters/repository-seed-selection.d.ts +7 -0
- package/dist/adapters/repository-seed-selection.js +79 -0
- package/dist/adapters/repository-solution-edits.d.ts +49 -0
- package/dist/adapters/repository-solution-edits.js +136 -0
- package/dist/adapters/repository-vitest-phase-adapter.d.ts +8 -0
- package/dist/adapters/repository-vitest-phase-adapter.js +310 -0
- package/dist/adapters/repository-vitest-reporter.d.ts +24 -0
- package/dist/adapters/repository-vitest-reporter.js +314 -0
- package/dist/adapters/strict-authoring-schema.d.ts +5 -0
- package/dist/adapters/strict-authoring-schema.js +158 -0
- package/dist/adapters/test-discovery.d.ts +30 -0
- package/dist/adapters/test-discovery.js +124 -0
- package/dist/adapters/typescript-repository-index.d.ts +51 -0
- package/dist/adapters/typescript-repository-index.js +226 -0
- package/dist/agentic-capabilities-protocol.d.ts +1373 -0
- package/dist/agentic-capabilities-protocol.js +786 -0
- package/dist/case-checkpoint-store.d.ts +29 -0
- package/dist/case-checkpoint-store.js +133 -0
- package/dist/case-pipeline-protocol-v2.d.ts +184 -0
- package/dist/case-pipeline-protocol-v2.js +193 -0
- package/dist/case-pipeline-protocol.d.ts +2626 -0
- package/dist/case-pipeline-protocol.js +371 -0
- package/dist/effect-api.d.ts +74 -10
- package/dist/effect-api.js +56 -6
- package/dist/errors.d.ts +31 -0
- package/dist/errors.js +10 -0
- package/dist/eval-capability-execution-envelope.d.ts +64 -0
- package/dist/eval-capability-execution-envelope.js +98 -0
- package/dist/eval-capability-policy.d.ts +90 -0
- package/dist/eval-capability-policy.js +107 -0
- package/dist/eval-event-log.d.ts +140 -0
- package/dist/eval-event-log.js +220 -0
- package/dist/evaluation-authoring-policy.d.ts +18 -0
- package/dist/evaluation-authoring-policy.js +19 -0
- package/dist/evaluation-authoring-validation.d.ts +22 -0
- package/dist/evaluation-authoring-validation.js +72 -0
- package/dist/evaluation-evidence.d.ts +20 -0
- package/dist/evaluation-evidence.js +319 -0
- package/dist/evaluation-grader-calibration-protocol.d.ts +108 -0
- package/dist/evaluation-grader-calibration-protocol.js +80 -0
- package/dist/evaluation-grader-calibration.d.ts +18 -0
- package/dist/evaluation-grader-calibration.js +334 -0
- package/dist/evaluation-grading-policy.d.ts +24 -0
- package/dist/evaluation-grading-policy.js +54 -0
- package/dist/evaluation-proposal-policy.d.ts +4 -0
- package/dist/evaluation-proposal-policy.js +91 -0
- package/dist/evaluation-source-retrieval.d.ts +68 -0
- package/dist/evaluation-source-retrieval.js +513 -0
- package/dist/evaluation-structure-policy.d.ts +29 -0
- package/dist/evaluation-structure-policy.js +138 -0
- package/dist/index.d.ts +124 -17
- package/dist/index.js +69 -11
- package/dist/inspection.js +2 -3
- package/dist/project-artifacts.d.ts +7 -2
- package/dist/project-artifacts.js +49 -136
- package/dist/project-authoring.d.ts +66 -5
- package/dist/project-authoring.js +783 -109
- package/dist/project-contracts.d.ts +419 -84
- package/dist/project-contracts.js +160 -52
- package/dist/project-store.js +2 -1
- package/dist/project-workflow.d.ts +5 -4
- package/dist/project-workflow.js +154 -35
- package/dist/repository-adversary-protocol.d.ts +64 -0
- package/dist/repository-adversary-protocol.js +105 -0
- package/dist/repository-behavior-protocol.d.ts +188 -0
- package/dist/repository-behavior-protocol.js +202 -0
- package/dist/repository-benchmark-protocol.d.ts +487 -0
- package/dist/repository-benchmark-protocol.js +96 -0
- package/dist/repository-execution-protocol.d.ts +150 -0
- package/dist/repository-execution-protocol.js +38 -0
- package/dist/repository-fixture-instructions.d.ts +3 -0
- package/dist/repository-fixture-instructions.js +91 -0
- package/dist/repository-fixture-protocol.d.ts +79 -0
- package/dist/repository-fixture-protocol.js +79 -0
- package/dist/repository-foundry-plan-protocol.d.ts +118 -0
- package/dist/repository-foundry-plan-protocol.js +296 -0
- package/dist/repository-foundry-progress-protocol.d.ts +52 -0
- package/dist/repository-foundry-progress-protocol.js +52 -0
- package/dist/repository-improvement-protocol.d.ts +100 -0
- package/dist/repository-improvement-protocol.js +106 -0
- package/dist/repository-language-model-protocol.d.ts +43 -0
- package/dist/repository-language-model-protocol.js +146 -0
- package/dist/repository-oracle-coverage-protocol.d.ts +18 -0
- package/dist/repository-oracle-coverage-protocol.js +39 -0
- package/dist/repository-oracle-execution-binding.d.ts +27 -0
- package/dist/repository-oracle-execution-binding.js +59 -0
- package/dist/repository-oracle-protocol.d.ts +230 -0
- package/dist/repository-oracle-protocol.js +156 -0
- package/dist/repository-oracle-scope-policy.d.ts +22 -0
- package/dist/repository-oracle-scope-policy.js +92 -0
- package/dist/repository-quality-policy.d.ts +15 -0
- package/dist/repository-quality-policy.js +357 -0
- package/dist/repository-routing-benchmark-protocol.d.ts +176 -0
- package/dist/repository-routing-benchmark-protocol.js +103 -0
- package/dist/repository-routing-model-protocol.d.ts +36 -0
- package/dist/repository-routing-model-protocol.js +89 -0
- package/dist/repository-routing-plan-protocol.d.ts +112 -0
- package/dist/repository-routing-plan-protocol.js +58 -0
- package/dist/repository-routing-quality-policy.d.ts +9 -0
- package/dist/repository-routing-quality-policy.js +191 -0
- package/dist/repository-seed-qualification-progress-protocol.d.ts +205 -0
- package/dist/repository-seed-qualification-progress-protocol.js +28 -0
- package/dist/repository-semantic-calibration-protocol.d.ts +768 -0
- package/dist/repository-semantic-calibration-protocol.js +276 -0
- package/dist/repository-semantic-calibration.d.ts +163 -0
- package/dist/repository-semantic-calibration.js +581 -0
- package/dist/repository-specification-contract-facts-protocol.d.ts +224 -0
- package/dist/repository-specification-contract-facts-protocol.js +276 -0
- package/dist/repository-specification-critique-protocol.d.ts +189 -0
- package/dist/repository-specification-critique-protocol.js +103 -0
- package/dist/repository-task-family-protocol.d.ts +24 -0
- package/dist/repository-task-family-protocol.js +37 -0
- package/dist/repository-task-seed-protocol.d.ts +384 -0
- package/dist/repository-task-seed-protocol.js +236 -0
- package/dist/repository-trajectory-protocol.d.ts +20 -0
- package/dist/repository-trajectory-protocol.js +42 -0
- package/dist/service.js +1 -1
- package/dist/services/adversary/service.d.ts +64 -0
- package/dist/services/adversary/service.js +330 -0
- package/dist/services/benchmark-compiler/service.d.ts +450 -0
- package/dist/services/benchmark-compiler/service.js +9 -0
- package/dist/services/budgeted-model/service.d.ts +118 -0
- package/dist/services/budgeted-model/service.js +460 -0
- package/dist/services/case-authoring/service.d.ts +163 -0
- package/dist/services/case-authoring/service.js +1456 -0
- package/dist/services/case-finalization/service.d.ts +283 -0
- package/dist/services/case-finalization/service.js +370 -0
- package/dist/services/case-generation/service.d.ts +619 -0
- package/dist/services/case-generation/service.js +2628 -0
- package/dist/services/case-pipeline/service.d.ts +31 -0
- package/dist/services/case-pipeline/service.js +485 -0
- package/dist/services/case-pipeline-v2/service.d.ts +70 -0
- package/dist/services/case-pipeline-v2/service.js +477 -0
- package/dist/services/command-observability/service.d.ts +13 -0
- package/dist/services/command-observability/service.js +3 -0
- package/dist/services/dimension-labeling/service.d.ts +77 -0
- package/dist/services/dimension-labeling/service.js +188 -0
- package/dist/services/eval-candidate/service.d.ts +208 -0
- package/dist/services/eval-candidate/service.js +64 -0
- package/dist/services/eval-capabilities/service.d.ts +183 -0
- package/dist/services/eval-capabilities/service.js +1433 -0
- package/dist/services/eval-environment/service.d.ts +173 -0
- package/dist/services/eval-environment/service.js +127 -0
- package/dist/services/evidence-reconstruction/service.d.ts +36 -0
- package/dist/services/evidence-reconstruction/service.js +145 -0
- package/dist/services/fixture-builder/service.d.ts +62 -0
- package/dist/services/fixture-builder/service.js +36 -0
- package/dist/services/fixture-validation/service.d.ts +75 -0
- package/dist/services/fixture-validation/service.js +295 -0
- package/dist/services/foundry/service.d.ts +831 -0
- package/dist/services/foundry/service.js +442 -0
- package/dist/services/foundry-progress/service.d.ts +62 -0
- package/dist/services/foundry-progress/service.js +149 -0
- package/dist/services/foundry-v2/service.d.ts +54 -0
- package/dist/services/foundry-v2/service.js +28 -0
- package/dist/services/grounded-authoring/service.d.ts +126 -0
- package/dist/services/grounded-authoring/service.js +822 -0
- package/dist/services/historical-case/service.d.ts +722 -0
- package/dist/services/historical-case/service.js +177 -0
- package/dist/services/improvement-loop/service.d.ts +59 -0
- package/dist/services/improvement-loop/service.js +176 -0
- package/dist/services/language-model/service.d.ts +52 -0
- package/dist/services/language-model/service.js +194 -0
- package/dist/services/oracle-builder/service.d.ts +146 -0
- package/dist/services/oracle-builder/service.js +513 -0
- package/dist/services/oracle-coverage/service.d.ts +28 -0
- package/dist/services/oracle-coverage/service.js +50 -0
- package/dist/services/oracle-coverage-witness/service.d.ts +130 -0
- package/dist/services/oracle-coverage-witness/service.js +538 -0
- package/dist/services/pipeline-challenge/service.d.ts +551 -0
- package/dist/services/pipeline-challenge/service.js +427 -0
- package/dist/services/pipeline-controls/service.d.ts +130 -0
- package/dist/services/pipeline-controls/service.js +483 -0
- package/dist/services/pipeline-oracle/service.d.ts +8 -0
- package/dist/services/pipeline-oracle/service.js +256 -0
- package/dist/services/pipeline-seed/service.d.ts +298 -0
- package/dist/services/pipeline-seed/service.js +428 -0
- package/dist/services/pipeline-spec/service.d.ts +103 -0
- package/dist/services/pipeline-spec/service.js +619 -0
- package/dist/services/pipeline-tournament/service.d.ts +258 -0
- package/dist/services/pipeline-tournament/service.js +476 -0
- package/dist/services/quality-gate/service.d.ts +233 -0
- package/dist/services/quality-gate/service.js +136 -0
- package/dist/services/repository-bundle/service.d.ts +33 -0
- package/dist/services/repository-bundle/service.js +114 -0
- package/dist/services/repository-model/service.d.ts +105 -0
- package/dist/services/repository-model/service.js +250 -0
- package/dist/services/repository-public-artifact/service.d.ts +133 -0
- package/dist/services/repository-public-artifact/service.js +330 -0
- package/dist/services/routing-benchmark/service.d.ts +362 -0
- package/dist/services/routing-benchmark/service.js +96 -0
- package/dist/services/specification-critic/service.d.ts +92 -0
- package/dist/services/specification-critic/service.js +172 -0
- package/dist/services/task-family/service.d.ts +40 -0
- package/dist/services/task-family/service.js +55 -0
- package/dist/services/task-seed/service.d.ts +906 -0
- package/dist/services/task-seed/service.js +1406 -0
- package/dist/services/task-specification/service.d.ts +27 -0
- package/dist/services/task-specification/service.js +40 -0
- package/dist/services/trajectory-policy/service.d.ts +110 -0
- package/dist/services/trajectory-policy/service.js +216 -0
- package/dist/test/agentic-capabilities-protocol.test.d.ts +1 -0
- package/dist/test/agentic-capabilities-protocol.test.js +570 -0
- package/dist/test/agentic-capabilities.test.d.ts +1 -0
- package/dist/test/agentic-capabilities.test.js +1461 -0
- package/dist/test/agentic-environment.test.d.ts +1 -0
- package/dist/test/agentic-environment.test.js +213 -0
- package/dist/test/case-pipeline-foundation.test.d.ts +1 -0
- package/dist/test/case-pipeline-foundation.test.js +535 -0
- package/dist/test/case-pipeline-protocol-v2.test.d.ts +1 -0
- package/dist/test/case-pipeline-protocol-v2.test.js +124 -0
- package/dist/test/case-pipeline-v2.test.d.ts +1 -0
- package/dist/test/case-pipeline-v2.test.js +286 -0
- package/dist/test/case-pipeline.test.d.ts +1 -0
- package/dist/test/case-pipeline.test.js +851 -0
- package/dist/test/eval-capability-policy.test.d.ts +1 -0
- package/dist/test/eval-capability-policy.test.js +50 -0
- package/dist/test/eval-event-log.test.d.ts +1 -0
- package/dist/test/eval-event-log.test.js +125 -0
- package/dist/test/evaluation-evidence-freshness.test.d.ts +1 -0
- package/dist/test/evaluation-evidence-freshness.test.js +44 -0
- package/dist/test/evaluation-evidence.test.d.ts +1 -0
- package/dist/test/evaluation-evidence.test.js +230 -0
- package/dist/test/evaluation-grader-calibration.test.d.ts +1 -0
- package/dist/test/evaluation-grader-calibration.test.js +373 -0
- package/dist/test/evaluation-proposal-digest.test.d.ts +1 -0
- package/dist/test/evaluation-proposal-digest.test.js +187 -0
- package/dist/test/evaluation-source-retrieval.test.d.ts +1 -0
- package/dist/test/evaluation-source-retrieval.test.js +237 -0
- package/dist/test/evaluation-structure-policy.test.d.ts +1 -0
- package/dist/test/evaluation-structure-policy.test.js +196 -0
- package/dist/test/fixtures/repository-resource-panel.d.ts +39 -0
- package/dist/test/fixtures/repository-resource-panel.js +111 -0
- package/dist/test/fixtures/vitest-boundary-panel.d.ts +84 -0
- package/dist/test/fixtures/vitest-boundary-panel.js +120 -0
- package/dist/test/fixtures/vitest-phase-panel.d.ts +135 -0
- package/dist/test/fixtures/vitest-phase-panel.js +213 -0
- package/dist/test/fixtures/vitest-reporter-results.d.ts +76 -0
- package/dist/test/fixtures/vitest-reporter-results.js +94 -0
- package/dist/test/grounded-authoring.test.d.ts +1 -0
- package/dist/test/grounded-authoring.test.js +565 -0
- package/dist/test/integrated-repository-history.test.d.ts +1 -0
- package/dist/test/integrated-repository-history.test.js +227 -0
- package/dist/test/project-authoring.test.js +593 -43
- package/dist/test/project-workflow.test.js +419 -40
- package/dist/test/repository-authoring-artifacts.test.d.ts +1 -0
- package/dist/test/repository-authoring-artifacts.test.js +185 -0
- package/dist/test/repository-bundle.test.d.ts +1 -0
- package/dist/test/repository-bundle.test.js +52 -0
- package/dist/test/repository-case-generation.test.d.ts +1 -0
- package/dist/test/repository-case-generation.test.js +3465 -0
- package/dist/test/repository-command-diagnostic.test.d.ts +1 -0
- package/dist/test/repository-command-diagnostic.test.js +55 -0
- package/dist/test/repository-command-signals.test.d.ts +1 -0
- package/dist/test/repository-command-signals.test.js +124 -0
- package/dist/test/repository-fixture-scope-coverage.test.d.ts +1 -0
- package/dist/test/repository-fixture-scope-coverage.test.js +127 -0
- package/dist/test/repository-fixture-validation.test.d.ts +1 -0
- package/dist/test/repository-fixture-validation.test.js +362 -0
- package/dist/test/repository-foundry-progress.test.d.ts +1 -0
- package/dist/test/repository-foundry-progress.test.js +110 -0
- package/dist/test/repository-foundry-quality.test.d.ts +1 -0
- package/dist/test/repository-foundry-quality.test.js +1138 -0
- package/dist/test/repository-import-context.test.d.ts +1 -0
- package/dist/test/repository-import-context.test.js +354 -0
- package/dist/test/repository-model-authoring.test.d.ts +1 -0
- package/dist/test/repository-model-authoring.test.js +544 -0
- package/dist/test/repository-model.test.d.ts +1 -0
- package/dist/test/repository-model.test.js +2195 -0
- package/dist/test/repository-node-test-reporter.test.d.ts +1 -0
- package/dist/test/repository-node-test-reporter.test.js +104 -0
- package/dist/test/repository-oracle-concurrency.test.d.ts +1 -0
- package/dist/test/repository-oracle-concurrency.test.js +542 -0
- package/dist/test/repository-oracle-coverage-witness.test.d.ts +1 -0
- package/dist/test/repository-oracle-coverage-witness.test.js +511 -0
- package/dist/test/repository-oracle-coverage.test.d.ts +1 -0
- package/dist/test/repository-oracle-coverage.test.js +168 -0
- package/dist/test/repository-oracle-evidence.test.d.ts +1 -0
- package/dist/test/repository-oracle-evidence.test.js +185 -0
- package/dist/test/repository-oracle-plan.test.d.ts +1 -0
- package/dist/test/repository-oracle-plan.test.js +176 -0
- package/dist/test/repository-overlay-isolation.test.d.ts +1 -0
- package/dist/test/repository-overlay-isolation.test.js +85 -0
- package/dist/test/repository-preparation-cache.test.d.ts +1 -0
- package/dist/test/repository-preparation-cache.test.js +414 -0
- package/dist/test/repository-public-artifact.test.d.ts +1 -0
- package/dist/test/repository-public-artifact.test.js +273 -0
- package/dist/test/repository-qualification-diagnostics.test.d.ts +1 -0
- package/dist/test/repository-qualification-diagnostics.test.js +524 -0
- package/dist/test/repository-reference-authoring.test.d.ts +1 -0
- package/dist/test/repository-reference-authoring.test.js +1633 -0
- package/dist/test/repository-review-evidence-v2.test.d.ts +1 -0
- package/dist/test/repository-review-evidence-v2.test.js +183 -0
- package/dist/test/repository-review-evidence.test.d.ts +1 -0
- package/dist/test/repository-review-evidence.test.js +124 -0
- package/dist/test/repository-seed-exclusions.test.d.ts +1 -0
- package/dist/test/repository-seed-exclusions.test.js +96 -0
- package/dist/test/repository-seed-selection.test.d.ts +1 -0
- package/dist/test/repository-seed-selection.test.js +504 -0
- package/dist/test/repository-semantic-calibration.test.d.ts +1 -0
- package/dist/test/repository-semantic-calibration.test.js +688 -0
- package/dist/test/repository-solution-edits.test.d.ts +1 -0
- package/dist/test/repository-solution-edits.test.js +377 -0
- package/dist/test/repository-specification-budget.test.d.ts +1 -0
- package/dist/test/repository-specification-budget.test.js +171 -0
- package/dist/test/repository-specification-contract-checkpoint.test.d.ts +1 -0
- package/dist/test/repository-specification-contract-checkpoint.test.js +228 -0
- package/dist/test/repository-specification-contract-facts.test.d.ts +1 -0
- package/dist/test/repository-specification-contract-facts.test.js +177 -0
- package/dist/test/repository-trajectory-authoring.test.d.ts +1 -0
- package/dist/test/repository-trajectory-authoring.test.js +176 -0
- package/dist/test/repository-valid-control-plan.test.d.ts +1 -0
- package/dist/test/repository-valid-control-plan.test.js +45 -0
- package/dist/test/repository-vitest-phase.test.d.ts +1 -0
- package/dist/test/repository-vitest-phase.test.js +848 -0
- package/dist/test/repository-vitest-reporter.test.d.ts +1 -0
- package/dist/test/repository-vitest-reporter.test.js +158 -0
- package/dist/test/repository-workspace-build.test.d.ts +1 -0
- package/dist/test/repository-workspace-build.test.js +160 -0
- package/dist/test/strict-authoring-schema.test.d.ts +1 -0
- package/dist/test/strict-authoring-schema.test.js +169 -0
- package/package.json +48 -6
|
@@ -0,0 +1,2195 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { execFile } from "node:child_process";
|
|
3
|
+
import { mkdir, mkdtemp, readFile, realpath, rm, writeFile } from "node:fs/promises";
|
|
4
|
+
import net from "node:net";
|
|
5
|
+
import os from "node:os";
|
|
6
|
+
import path from "node:path";
|
|
7
|
+
import { after, test } from "node:test";
|
|
8
|
+
import { promisify } from "node:util";
|
|
9
|
+
import { Effect } from "effect";
|
|
10
|
+
import { captureGitTreeSnapshotV1 } from "../adapters/git-task-history.js";
|
|
11
|
+
import { repositoryCommandTestEvidenceV1 } from "../adapters/repository-command-evidence.js";
|
|
12
|
+
import { inspectRepositorySolutionPathsV1, RepositoryCommandExecutionPort, replayRepositoryCommandV1 } from "../adapters/repository-command-runner.js";
|
|
13
|
+
import { indexTypeScriptRepositoryV1 } from "../adapters/typescript-repository-index.js";
|
|
14
|
+
import { RepositoryFoundryError } from "../errors.js";
|
|
15
|
+
import { evaluateRepositoryOracleAdequacyV1 } from "../repository-quality-policy.js";
|
|
16
|
+
import { generateDeterministicAdversariesV1, repositoryIntegrationMutationFamilyV1 } from "../services/adversary/service.js";
|
|
17
|
+
import { compileRepositoryBenchmarkV1 } from "../services/benchmark-compiler/service.js";
|
|
18
|
+
import { compileReviewedHiddenFixtureSuiteV1 } from "../services/fixture-builder/service.js";
|
|
19
|
+
import { RepositoryFoundryProgressLive } from "../services/foundry-progress/service.js";
|
|
20
|
+
import { compileReviewedHistoricalCaseV1 } from "../services/historical-case/service.js";
|
|
21
|
+
import { RepositoryCommandObservability } from "../services/command-observability/service.js";
|
|
22
|
+
import { buildHistoricalHiddenOracleV1 } from "../services/oracle-builder/service.js";
|
|
23
|
+
import { evaluateRepositoryBenchmarkDraftV1 } from "../services/quality-gate/service.js";
|
|
24
|
+
import { buildRepositoryBehaviorMapV1 } from "../services/repository-model/service.js";
|
|
25
|
+
import { critiqueRepositoryTaskSpecificationV1 } from "../services/specification-critic/service.js";
|
|
26
|
+
import { deriveRepositoryTaskFamiliesV1 } from "../services/task-family/service.js";
|
|
27
|
+
import { deriveHistoricalSeedEnvironmentV1, extractHistoricalTaskSeedsV1, qualifyHistoricalTaskSeedAcrossBatchesV1, qualifyHistoricalTaskSeedV1 } from "../services/task-seed/service.js";
|
|
28
|
+
import { constructRepositoryAgentTaskV1 } from "../services/task-specification/service.js";
|
|
29
|
+
const execFilePromise = promisify(execFile);
|
|
30
|
+
const roots = [];
|
|
31
|
+
const independentSpecificationReviews = [
|
|
32
|
+
{
|
|
33
|
+
reviewerId: "independent-solver-a",
|
|
34
|
+
reviewerKind: "independent-solver",
|
|
35
|
+
independentlyProduced: true,
|
|
36
|
+
verdict: "sufficient",
|
|
37
|
+
detail: "The request identifies the affected behavior, checkout, and constraints without prescribing an implementation.",
|
|
38
|
+
findings: []
|
|
39
|
+
},
|
|
40
|
+
{
|
|
41
|
+
reviewerId: "independent-solver-b",
|
|
42
|
+
reviewerKind: "model-critic",
|
|
43
|
+
independentlyProduced: true,
|
|
44
|
+
verdict: "sufficient",
|
|
45
|
+
detail: "The pre-task repository contains enough context to reproduce and repair the requested behavior.",
|
|
46
|
+
findings: []
|
|
47
|
+
}
|
|
48
|
+
];
|
|
49
|
+
after(async () => Promise.all(roots.map((root) => rm(root, { recursive: true, force: true }))));
|
|
50
|
+
const git = (root, ...args) => execFilePromise("git", ["-C", root, ...args], {
|
|
51
|
+
encoding: "utf8",
|
|
52
|
+
windowsHide: true
|
|
53
|
+
});
|
|
54
|
+
const commit = async (root, subject, occurredAt) => {
|
|
55
|
+
await git(root, "add", ".");
|
|
56
|
+
await execFilePromise("git", ["-C", root, "commit", "-q", "-m", subject], {
|
|
57
|
+
encoding: "utf8",
|
|
58
|
+
windowsHide: true,
|
|
59
|
+
env: {
|
|
60
|
+
...process.env,
|
|
61
|
+
GIT_AUTHOR_DATE: occurredAt,
|
|
62
|
+
GIT_COMMITTER_DATE: occurredAt
|
|
63
|
+
}
|
|
64
|
+
});
|
|
65
|
+
return (await git(root, "rev-parse", "HEAD")).stdout.trim();
|
|
66
|
+
};
|
|
67
|
+
async function repository() {
|
|
68
|
+
const root = await mkdtemp(path.join(os.tmpdir(), "routekit-foundry-repository-"));
|
|
69
|
+
roots.push(root);
|
|
70
|
+
await git(root, "init", "-q");
|
|
71
|
+
await git(root, "config", "user.name", "RouteKit Foundry Test");
|
|
72
|
+
await git(root, "config", "user.email", "foundry@example.test");
|
|
73
|
+
await mkdir(path.join(root, "packages", "stream", "src"), {
|
|
74
|
+
recursive: true
|
|
75
|
+
});
|
|
76
|
+
await writeFile(path.join(root, "package.json"), `${JSON.stringify({
|
|
77
|
+
name: "fixture-root",
|
|
78
|
+
private: true,
|
|
79
|
+
scripts: { check: "echo root-check" }
|
|
80
|
+
}, null, 2)}\n`);
|
|
81
|
+
await writeFile(path.join(root, "packages", "stream", "package.json"), `${JSON.stringify({
|
|
82
|
+
name: "@fixture/stream",
|
|
83
|
+
type: "module",
|
|
84
|
+
exports: { ".": "./dist/index.js" },
|
|
85
|
+
scripts: {
|
|
86
|
+
build: "node --check src/stream-protocol.js",
|
|
87
|
+
test: "node --test src/*.test.js"
|
|
88
|
+
}
|
|
89
|
+
}, null, 2)}\n`);
|
|
90
|
+
await writeFile(path.join(root, "packages", "stream", "src", "index.js"), ["export { translateEvents } from './stream-protocol.js';", ""].join("\n"));
|
|
91
|
+
await writeFile(path.join(root, "packages", "stream", "src", "stream-protocol.js"), ["export function translateEvents(events) {", " return events;", "}", ""].join("\n"));
|
|
92
|
+
await writeFile(path.join(root, "packages", "stream", "src", "stream-protocol.test.js"), [
|
|
93
|
+
"import { test } from 'node:test';",
|
|
94
|
+
"import { strict as assert } from 'node:assert';",
|
|
95
|
+
"import { translateEvents } from './stream-protocol.js';",
|
|
96
|
+
"test('preserves an empty event stream', () => {",
|
|
97
|
+
" assert.deepEqual(translateEvents([]), []);",
|
|
98
|
+
"});",
|
|
99
|
+
""
|
|
100
|
+
].join("\n"));
|
|
101
|
+
await commit(root, "initial stream protocol", "2026-01-01T00:00:00Z");
|
|
102
|
+
await writeFile(path.join(root, "packages", "stream", "src", "stream-protocol.js"), [
|
|
103
|
+
"export function translateEvents(events) {",
|
|
104
|
+
" const terminal = events.find((event) => event.type === 'completed');",
|
|
105
|
+
" return terminal === undefined ? events : [...events.filter((event) => event.type !== 'completed'), terminal];",
|
|
106
|
+
"}",
|
|
107
|
+
""
|
|
108
|
+
].join("\n"));
|
|
109
|
+
await writeFile(path.join(root, "packages", "stream", "src", "stream-protocol.test.js"), [
|
|
110
|
+
"import { test } from 'node:test';",
|
|
111
|
+
"import { strict as assert } from 'node:assert';",
|
|
112
|
+
"import { translateEvents } from './stream-protocol.js';",
|
|
113
|
+
"test('keeps terminal event last', () => {",
|
|
114
|
+
" assert.equal(translateEvents([{ type: 'completed' }, { type: 'output', value: 'late' }]).at(-1)?.type, 'completed');",
|
|
115
|
+
"});",
|
|
116
|
+
"test('preserves output before completion', () => {",
|
|
117
|
+
" assert.deepEqual(translateEvents([{ type: 'completed' }, { type: 'output', value: 'late' }]), [{ type: 'output', value: 'late' }, { type: 'completed' }]);",
|
|
118
|
+
"});",
|
|
119
|
+
""
|
|
120
|
+
].join("\n"));
|
|
121
|
+
await commit(root, "fix terminal event ordering", "2026-02-01T00:00:00Z");
|
|
122
|
+
await writeFile(path.join(root, "packages", "stream", "src", "stream-protocol.js"), [
|
|
123
|
+
"export function translateEvents(events) {",
|
|
124
|
+
" const terminal = events.find((event) => event.type === 'completed');",
|
|
125
|
+
" const output = events.filter((event) => event.type === 'output');",
|
|
126
|
+
" return terminal === undefined ? output : [...output, terminal];",
|
|
127
|
+
"}",
|
|
128
|
+
""
|
|
129
|
+
].join("\n"));
|
|
130
|
+
await writeFile(path.join(root, "packages", "stream", "src", "stream-protocol.test.js"), [
|
|
131
|
+
"import { test } from 'node:test';",
|
|
132
|
+
"import { strict as assert } from 'node:assert';",
|
|
133
|
+
"import { translateEvents } from './stream-protocol.js';",
|
|
134
|
+
"test('keeps exactly one terminal event last', () => {",
|
|
135
|
+
" assert.deepEqual(translateEvents([{ type: 'completed' }, { type: 'completed' }]), [{ type: 'completed' }]);",
|
|
136
|
+
"});",
|
|
137
|
+
""
|
|
138
|
+
].join("\n"));
|
|
139
|
+
await commit(root, "repair duplicate terminal events", "2026-03-01T00:00:00Z");
|
|
140
|
+
await writeFile(path.join(root, "packages", "stream", "src", "stream-protocol.js"), [
|
|
141
|
+
"export function translateEvents(events) {",
|
|
142
|
+
" const terminal = events.find((event) => event.type === 'completed');",
|
|
143
|
+
" const nonTerminal = events.filter((event) => event.type !== 'completed');",
|
|
144
|
+
" return terminal === undefined ? nonTerminal : [...nonTerminal, terminal];",
|
|
145
|
+
"}",
|
|
146
|
+
""
|
|
147
|
+
].join("\n"));
|
|
148
|
+
await writeFile(path.join(root, "packages", "stream", "src", "stream-protocol.test.js"), [
|
|
149
|
+
"import { test } from 'node:test';",
|
|
150
|
+
"import { strict as assert } from 'node:assert';",
|
|
151
|
+
"import { translateEvents } from './stream-protocol.js';",
|
|
152
|
+
"test('keeps exactly one terminal event last', () => {",
|
|
153
|
+
" assert.deepEqual(translateEvents([{ type: 'completed' }, { type: 'completed' }]), [{ type: 'completed' }]);",
|
|
154
|
+
"});",
|
|
155
|
+
"test('preserves usage without a terminal event', () => {",
|
|
156
|
+
" assert.deepEqual(translateEvents([{ type: 'usage', tokens: 3 }]), [{ type: 'usage', tokens: 3 }]);",
|
|
157
|
+
"});",
|
|
158
|
+
""
|
|
159
|
+
].join("\n"));
|
|
160
|
+
await commit(root, "feat(stream): preserve usage summaries", "2026-04-01T00:00:00Z");
|
|
161
|
+
return root;
|
|
162
|
+
}
|
|
163
|
+
async function repositoryWithCrossPackageCochange() {
|
|
164
|
+
const root = await mkdtemp(path.join(os.tmpdir(), "routekit-foundry-cochange-"));
|
|
165
|
+
roots.push(root);
|
|
166
|
+
await git(root, "init", "-q");
|
|
167
|
+
await git(root, "config", "user.name", "RouteKit Foundry Test");
|
|
168
|
+
await git(root, "config", "user.email", "foundry@example.test");
|
|
169
|
+
for (const name of ["accounts", "gateway"]) {
|
|
170
|
+
await mkdir(path.join(root, "packages", name, "src"), {
|
|
171
|
+
recursive: true
|
|
172
|
+
});
|
|
173
|
+
await writeFile(path.join(root, "packages", name, "package.json"), `${JSON.stringify({
|
|
174
|
+
name: `@fixture/${name}`,
|
|
175
|
+
type: "module",
|
|
176
|
+
exports: { ".": "./src/index.js" },
|
|
177
|
+
scripts: { test: "node --test src/index.test.js" }
|
|
178
|
+
}, null, 2)}\n`);
|
|
179
|
+
await writeFile(path.join(root, "packages", name, "src", "index.js"), `export const revision = 0;\n`);
|
|
180
|
+
await writeFile(path.join(root, "packages", name, "src", "index.test.js"), [
|
|
181
|
+
"import { test } from 'node:test';",
|
|
182
|
+
"import { strict as assert } from 'node:assert';",
|
|
183
|
+
"import { revision } from './index.js';",
|
|
184
|
+
"test('revision', () => assert.equal(revision, 0));",
|
|
185
|
+
""
|
|
186
|
+
].join("\n"));
|
|
187
|
+
}
|
|
188
|
+
await commit(root, "initial account gateway boundary", "2026-01-01T00:00:00Z");
|
|
189
|
+
for (const change of [
|
|
190
|
+
{
|
|
191
|
+
revision: 1,
|
|
192
|
+
subject: "fix(accounts): surface discovery failures",
|
|
193
|
+
occurredAt: "2026-02-01T00:00:00Z"
|
|
194
|
+
},
|
|
195
|
+
{
|
|
196
|
+
revision: 2,
|
|
197
|
+
subject: "fix(router): preserve healthy provider models",
|
|
198
|
+
occurredAt: "2026-03-01T00:00:00Z"
|
|
199
|
+
}
|
|
200
|
+
]) {
|
|
201
|
+
for (const name of ["accounts", "gateway"]) {
|
|
202
|
+
await writeFile(path.join(root, "packages", name, "src", "index.js"), `export const revision = ${String(change.revision)};\n`);
|
|
203
|
+
await writeFile(path.join(root, "packages", name, "src", "index.test.js"), [
|
|
204
|
+
"import { test } from 'node:test';",
|
|
205
|
+
"import { strict as assert } from 'node:assert';",
|
|
206
|
+
"import { revision } from './index.js';",
|
|
207
|
+
`test('revision', () => assert.equal(revision, ${String(change.revision)}));`,
|
|
208
|
+
""
|
|
209
|
+
].join("\n"));
|
|
210
|
+
}
|
|
211
|
+
await commit(root, change.subject, change.occurredAt);
|
|
212
|
+
}
|
|
213
|
+
return root;
|
|
214
|
+
}
|
|
215
|
+
async function repositoryWithDeclaredWorkspaceBoundary() {
|
|
216
|
+
const root = await mkdtemp(path.join(os.tmpdir(), "routekit-foundry-workspace-boundary-"));
|
|
217
|
+
roots.push(root);
|
|
218
|
+
await git(root, "init", "-q");
|
|
219
|
+
await git(root, "config", "user.name", "RouteKit Foundry Test");
|
|
220
|
+
await git(root, "config", "user.email", "foundry@example.test");
|
|
221
|
+
await mkdir(path.join(root, "packages", "owned", "src"), {
|
|
222
|
+
recursive: true
|
|
223
|
+
});
|
|
224
|
+
await mkdir(path.join(root, ".repos", "vendored", "src"), {
|
|
225
|
+
recursive: true
|
|
226
|
+
});
|
|
227
|
+
await writeFile(path.join(root, "package.json"), `${JSON.stringify({
|
|
228
|
+
name: "fixture-workspace-root",
|
|
229
|
+
private: true,
|
|
230
|
+
packageManager: "pnpm@11.15.1"
|
|
231
|
+
}, null, 2)}\n`);
|
|
232
|
+
await writeFile(path.join(root, "pnpm-workspace.yaml"), "packages:\n - packages/*\n");
|
|
233
|
+
for (const fixture of [
|
|
234
|
+
{
|
|
235
|
+
root: path.join(root, "packages", "owned"),
|
|
236
|
+
name: "@fixture/owned",
|
|
237
|
+
value: 0
|
|
238
|
+
},
|
|
239
|
+
{
|
|
240
|
+
root: path.join(root, ".repos", "vendored"),
|
|
241
|
+
name: "@fixture/vendored",
|
|
242
|
+
value: 0
|
|
243
|
+
}
|
|
244
|
+
]) {
|
|
245
|
+
await writeFile(path.join(fixture.root, "package.json"), `${JSON.stringify({
|
|
246
|
+
name: fixture.name,
|
|
247
|
+
type: "module",
|
|
248
|
+
exports: {
|
|
249
|
+
".": "./src/index.js"
|
|
250
|
+
},
|
|
251
|
+
scripts: {
|
|
252
|
+
test: fixture.name.includes("owned") ? "vp test run" : "node --test src/index.test.js"
|
|
253
|
+
}
|
|
254
|
+
}, null, 2)}\n`);
|
|
255
|
+
await writeFile(path.join(fixture.root, "src", "index.js"), `export const value = ${String(fixture.value)};\n`);
|
|
256
|
+
await writeFile(path.join(fixture.root, "src", "wire-schema.ts"), `export const ${fixture.name.includes("owned") ? "Owned" : "Vendored"}Schema = {};\n`);
|
|
257
|
+
await writeFile(path.join(fixture.root, "src", "index.test.js"), [
|
|
258
|
+
"import { test } from 'node:test';",
|
|
259
|
+
"import { strict as assert } from 'node:assert';",
|
|
260
|
+
"import { value } from './index.js';",
|
|
261
|
+
`test('value', () => assert.equal(value, ${String(fixture.value)}));`,
|
|
262
|
+
""
|
|
263
|
+
].join("\n"));
|
|
264
|
+
}
|
|
265
|
+
await writeFile(path.join(root, ".repos", "vendored", "tsconfig.json"), `${JSON.stringify({
|
|
266
|
+
references: [{ path: "../missing-project" }]
|
|
267
|
+
}, null, 2)}\n`);
|
|
268
|
+
await commit(root, "initial declared workspace", "2026-01-01T00:00:00Z");
|
|
269
|
+
await writeFile(path.join(root, ".repos", "vendored", "src", "index.js"), "export const value = 1;\n");
|
|
270
|
+
await writeFile(path.join(root, ".repos", "vendored", "src", "index.test.js"), [
|
|
271
|
+
"import { test } from 'node:test';",
|
|
272
|
+
"import { strict as assert } from 'node:assert';",
|
|
273
|
+
"import { value } from './index.js';",
|
|
274
|
+
"test('value', () => assert.equal(value, 1));",
|
|
275
|
+
""
|
|
276
|
+
].join("\n"));
|
|
277
|
+
await commit(root, "fix(vendor): update embedded source snapshot", "2026-02-01T00:00:00Z");
|
|
278
|
+
await writeFile(path.join(root, "packages", "owned", "src", "index.js"), "export const value = 1;\n");
|
|
279
|
+
await writeFile(path.join(root, "packages", "owned", "src", "index.test.js"), [
|
|
280
|
+
"import { test } from 'node:test';",
|
|
281
|
+
"import { strict as assert } from 'node:assert';",
|
|
282
|
+
"import { value } from './index.js';",
|
|
283
|
+
"test('value', () => assert.equal(value, 1));",
|
|
284
|
+
""
|
|
285
|
+
].join("\n"));
|
|
286
|
+
await commit(root, "fix(owned): preserve workspace behavior", "2026-03-01T00:00:00Z");
|
|
287
|
+
return root;
|
|
288
|
+
}
|
|
289
|
+
async function repositoryWithPnpmBuildGraph(projectReferenceMode = "valid") {
|
|
290
|
+
const root = await mkdtemp(path.join(os.tmpdir(), "routekit-foundry-pnpm-graph-"));
|
|
291
|
+
roots.push(root);
|
|
292
|
+
await git(root, "init", "-q");
|
|
293
|
+
await git(root, "config", "user.name", "RouteKit Foundry Test");
|
|
294
|
+
await git(root, "config", "user.email", "foundry@example.test");
|
|
295
|
+
await mkdir(path.join(root, "packages", "core", "src"), {
|
|
296
|
+
recursive: true
|
|
297
|
+
});
|
|
298
|
+
await mkdir(path.join(root, "packages", "app", "src"), {
|
|
299
|
+
recursive: true
|
|
300
|
+
});
|
|
301
|
+
await mkdir(path.join(root, "packages", "setup", "src"), {
|
|
302
|
+
recursive: true
|
|
303
|
+
});
|
|
304
|
+
await mkdir(path.join(root, "packages", "non-ts", "src"), {
|
|
305
|
+
recursive: true
|
|
306
|
+
});
|
|
307
|
+
await writeFile(path.join(root, "package.json"), `${JSON.stringify({
|
|
308
|
+
name: "fixture-pnpm-root",
|
|
309
|
+
private: true,
|
|
310
|
+
packageManager: "pnpm@11.15.1"
|
|
311
|
+
}, null, 2)}\n`);
|
|
312
|
+
await writeFile(path.join(root, "pnpm-workspace.yaml"), 'packages:\n - "packages/*"\n');
|
|
313
|
+
await writeFile(path.join(root, "packages", "core", "package.json"), `${JSON.stringify({
|
|
314
|
+
name: "@fixture/core",
|
|
315
|
+
type: "module",
|
|
316
|
+
scripts: {
|
|
317
|
+
build: "node --check src/index.js",
|
|
318
|
+
test: "node --test src/index.test.js"
|
|
319
|
+
}
|
|
320
|
+
}, null, 2)}\n`);
|
|
321
|
+
await writeFile(path.join(root, "packages", "core", "src", "index.js"), "export const stable = 1;\n");
|
|
322
|
+
await writeFile(path.join(root, "packages", "core", "src", "index.test.js"), [
|
|
323
|
+
"import { test } from 'node:test';",
|
|
324
|
+
"import { strict as assert } from 'node:assert';",
|
|
325
|
+
"import { stable } from './index.js';",
|
|
326
|
+
"test('stable', () => assert.equal(stable, 1));",
|
|
327
|
+
""
|
|
328
|
+
].join("\n"));
|
|
329
|
+
await writeFile(path.join(root, "packages", "core", "tsconfig.json"), `${JSON.stringify({ compilerOptions: { composite: true } }, null, 2)}\n`);
|
|
330
|
+
await writeFile(path.join(root, "packages", "setup", "package.json"), `${JSON.stringify({
|
|
331
|
+
name: "@fixture/setup",
|
|
332
|
+
type: "module",
|
|
333
|
+
dependencies: {
|
|
334
|
+
"@fixture/core": "workspace:*"
|
|
335
|
+
},
|
|
336
|
+
scripts: {
|
|
337
|
+
build: "node --check src/index.js"
|
|
338
|
+
}
|
|
339
|
+
}, null, 2)}\n`);
|
|
340
|
+
await writeFile(path.join(root, "packages", "setup", "src", "index.js"), "export const prepared = true;\n");
|
|
341
|
+
await writeFile(path.join(root, "packages", "setup", "tsconfig.json"), `${projectReferenceMode === "cycle"
|
|
342
|
+
? [
|
|
343
|
+
"{",
|
|
344
|
+
" // A cycle must be rejected rather than silently truncated.",
|
|
345
|
+
' "references": [',
|
|
346
|
+
' { "path": "../app", },',
|
|
347
|
+
" ],",
|
|
348
|
+
"}",
|
|
349
|
+
""
|
|
350
|
+
].join("\n")
|
|
351
|
+
: JSON.stringify({
|
|
352
|
+
compilerOptions: { composite: true },
|
|
353
|
+
references: [{ path: "../core" }]
|
|
354
|
+
}, null, 2)}\n`);
|
|
355
|
+
await writeFile(path.join(root, "packages", "non-ts", "package.json"), `${JSON.stringify({
|
|
356
|
+
name: "@fixture/non-ts",
|
|
357
|
+
type: "module",
|
|
358
|
+
scripts: {
|
|
359
|
+
build: "node --check src/index.js"
|
|
360
|
+
}
|
|
361
|
+
}, null, 2)}\n`);
|
|
362
|
+
await writeFile(path.join(root, "packages", "non-ts", "src", "index.js"), "export const portable = true;\n");
|
|
363
|
+
await writeFile(path.join(root, "packages", "app", "package.json"), `${JSON.stringify({
|
|
364
|
+
name: "@fixture/app",
|
|
365
|
+
type: "module",
|
|
366
|
+
dependencies: {
|
|
367
|
+
"@fixture/core": "workspace:*",
|
|
368
|
+
"@fixture/non-ts": "workspace:*"
|
|
369
|
+
},
|
|
370
|
+
scripts: {
|
|
371
|
+
build: "node --check src/index.js",
|
|
372
|
+
test: "node --test src/index.test.js"
|
|
373
|
+
}
|
|
374
|
+
}, null, 2)}\n`);
|
|
375
|
+
await writeFile(path.join(root, "packages", "app", "src", "index.js"), "export const value = 1;\n");
|
|
376
|
+
await writeFile(path.join(root, "packages", "app", "src", "index.test.js"), [
|
|
377
|
+
"import { test } from 'node:test';",
|
|
378
|
+
"import { strict as assert } from 'node:assert';",
|
|
379
|
+
"import { value } from './index.js';",
|
|
380
|
+
"test('value', () => assert.equal(value, 1));",
|
|
381
|
+
""
|
|
382
|
+
].join("\n"));
|
|
383
|
+
await writeFile(path.join(root, "packages", "app", "tsconfig.json"), `${projectReferenceMode === "escaping"
|
|
384
|
+
? JSON.stringify({ references: [{ path: "../../../outside" }] }, null, 2)
|
|
385
|
+
: projectReferenceMode === "malformed"
|
|
386
|
+
? JSON.stringify({ references: [{ path: 42 }] }, null, 2)
|
|
387
|
+
: [
|
|
388
|
+
"{",
|
|
389
|
+
" // This dependency is intentionally absent from package.json.",
|
|
390
|
+
' "references": [',
|
|
391
|
+
' { "path": "../setup", },',
|
|
392
|
+
" ],",
|
|
393
|
+
"}",
|
|
394
|
+
""
|
|
395
|
+
].join("\n")}\n`);
|
|
396
|
+
await execFilePromise("corepack", ["pnpm@11.15.1", "install", "--lockfile-only", "--offline", "--ignore-scripts"], {
|
|
397
|
+
cwd: root,
|
|
398
|
+
encoding: "utf8",
|
|
399
|
+
windowsHide: true
|
|
400
|
+
});
|
|
401
|
+
await commit(root, "initial app behavior", "2026-01-01T00:00:00Z");
|
|
402
|
+
await writeFile(path.join(root, "packages", "app", "src", "index.js"), "export const value = 2;\n");
|
|
403
|
+
await writeFile(path.join(root, "packages", "core", "src", "index.js"), "export const stable = 2;\n");
|
|
404
|
+
await writeFile(path.join(root, "packages", "core", "src", "index.test.js"), [
|
|
405
|
+
"import { test } from 'node:test';",
|
|
406
|
+
"import { strict as assert } from 'node:assert';",
|
|
407
|
+
"import { stable } from './index.js';",
|
|
408
|
+
"test('stable', () => assert.equal(stable, 2));",
|
|
409
|
+
""
|
|
410
|
+
].join("\n"));
|
|
411
|
+
await writeFile(path.join(root, "packages", "app", "src", "index.test.js"), [
|
|
412
|
+
"import { test } from 'node:test';",
|
|
413
|
+
"import { strict as assert } from 'node:assert';",
|
|
414
|
+
"import { value } from './index.js';",
|
|
415
|
+
"test('value', () => assert.equal(value, 2));",
|
|
416
|
+
""
|
|
417
|
+
].join("\n"));
|
|
418
|
+
await commit(root, "fix(app): preserve the new value", "2026-02-01T00:00:00Z");
|
|
419
|
+
return root;
|
|
420
|
+
}
|
|
421
|
+
async function repositoryWithNewRegressionTest() {
|
|
422
|
+
const root = await mkdtemp(path.join(os.tmpdir(), "routekit-foundry-new-regression-test-"));
|
|
423
|
+
roots.push(root);
|
|
424
|
+
await git(root, "init", "-q");
|
|
425
|
+
await git(root, "config", "user.name", "RouteKit Foundry Test");
|
|
426
|
+
await git(root, "config", "user.email", "foundry@example.test");
|
|
427
|
+
await mkdir(path.join(root, "packages", "normalizer", "src"), {
|
|
428
|
+
recursive: true
|
|
429
|
+
});
|
|
430
|
+
await writeFile(path.join(root, "package.json"), `${JSON.stringify({
|
|
431
|
+
name: "fixture-new-regression-root",
|
|
432
|
+
private: true,
|
|
433
|
+
packageManager: "pnpm@11.15.1"
|
|
434
|
+
}, null, 2)}\n`);
|
|
435
|
+
await writeFile(path.join(root, "packages", "normalizer", "package.json"), `${JSON.stringify({
|
|
436
|
+
name: "@fixture/normalizer",
|
|
437
|
+
type: "module",
|
|
438
|
+
scripts: {
|
|
439
|
+
test: "node --test"
|
|
440
|
+
}
|
|
441
|
+
}, null, 2)}\n`);
|
|
442
|
+
await writeFile(path.join(root, "packages", "normalizer", "src", "index.js"), ["export function normalize(value) {", " return value;", "}", ""].join("\n"));
|
|
443
|
+
await writeFile(path.join(root, "packages", "normalizer", "src", "index.test.js"), [
|
|
444
|
+
"import { test } from 'node:test';",
|
|
445
|
+
"import { strict as assert } from 'node:assert';",
|
|
446
|
+
"import { normalize } from './index.js';",
|
|
447
|
+
"test('preserves an already-normalized value', () => {",
|
|
448
|
+
" assert.equal(normalize('value'), 'value');",
|
|
449
|
+
"});",
|
|
450
|
+
""
|
|
451
|
+
].join("\n"));
|
|
452
|
+
await commit(root, "initial normalizer behavior", "2026-01-01T00:00:00Z");
|
|
453
|
+
await writeFile(path.join(root, "packages", "normalizer", "src", "index.js"), ["export function normalize(value) {", " return value.trim();", "}", ""].join("\n"));
|
|
454
|
+
await writeFile(path.join(root, "packages", "normalizer", "src", "whitespace-regression.test.js"), [
|
|
455
|
+
"import { test } from 'node:test';",
|
|
456
|
+
"import { strict as assert } from 'node:assert';",
|
|
457
|
+
"import { normalize } from './index.js';",
|
|
458
|
+
"test('trims surrounding whitespace', () => {",
|
|
459
|
+
" assert.equal(normalize(' value '), 'value');",
|
|
460
|
+
"});",
|
|
461
|
+
""
|
|
462
|
+
].join("\n"));
|
|
463
|
+
await commit(root, "fix(normalizer): trim surrounding whitespace", "2026-02-01T00:00:00Z");
|
|
464
|
+
return root;
|
|
465
|
+
}
|
|
466
|
+
async function repositoryWithCompiledNodeTestRunner() {
|
|
467
|
+
const root = await mkdtemp(path.join(os.tmpdir(), "routekit-foundry-compiled-node-test-"));
|
|
468
|
+
roots.push(root);
|
|
469
|
+
await git(root, "init", "-q");
|
|
470
|
+
await git(root, "config", "user.name", "RouteKit Foundry Test");
|
|
471
|
+
await git(root, "config", "user.email", "foundry@example.test");
|
|
472
|
+
await mkdir(path.join(root, "packages", "compiled", "src", "test"), {
|
|
473
|
+
recursive: true
|
|
474
|
+
});
|
|
475
|
+
await writeFile(path.join(root, "package.json"), `${JSON.stringify({
|
|
476
|
+
name: "fixture-compiled-root",
|
|
477
|
+
private: true,
|
|
478
|
+
packageManager: "pnpm@11.15.1"
|
|
479
|
+
}, null, 2)}\n`);
|
|
480
|
+
await writeFile(path.join(root, "pnpm-workspace.yaml"), "packages:\n - packages/*\n");
|
|
481
|
+
await writeFile(path.join(root, "packages", "compiled", "package.json"), `${JSON.stringify({
|
|
482
|
+
name: "@fixture/compiled",
|
|
483
|
+
private: true,
|
|
484
|
+
type: "module",
|
|
485
|
+
exports: "./dist/index.js",
|
|
486
|
+
scripts: {
|
|
487
|
+
build: "node build.mjs",
|
|
488
|
+
test: 'node --test "dist/test/*.test.js"'
|
|
489
|
+
}
|
|
490
|
+
}, null, 2)}\n`);
|
|
491
|
+
await writeFile(path.join(root, "packages", "compiled", "build.mjs"), [
|
|
492
|
+
'import { cp, mkdir, rm, rename } from "node:fs/promises";',
|
|
493
|
+
'await rm("dist", { recursive: true, force: true });',
|
|
494
|
+
'await mkdir("dist", { recursive: true });',
|
|
495
|
+
'await cp("src", "dist", { recursive: true });',
|
|
496
|
+
'await rename("dist/index.ts", "dist/index.js");',
|
|
497
|
+
'await rename("dist/test/value.test.ts", "dist/test/value.test.js");',
|
|
498
|
+
""
|
|
499
|
+
].join("\n"));
|
|
500
|
+
await writeFile(path.join(root, "packages", "compiled", "src", "index.ts"), "export const value = 1;\n");
|
|
501
|
+
await writeFile(path.join(root, "packages", "compiled", "src", "test", "value.test.ts"), [
|
|
502
|
+
'import assert from "node:assert/strict";',
|
|
503
|
+
'import test from "node:test";',
|
|
504
|
+
'import { value } from "../index.js";',
|
|
505
|
+
'test("value is one", () => assert.equal(value, 1));',
|
|
506
|
+
""
|
|
507
|
+
].join("\n"));
|
|
508
|
+
await execFilePromise("corepack", ["pnpm@11.15.1", "install", "--lockfile-only", "--offline", "--ignore-scripts"], {
|
|
509
|
+
cwd: root,
|
|
510
|
+
encoding: "utf8",
|
|
511
|
+
windowsHide: true
|
|
512
|
+
});
|
|
513
|
+
await commit(root, "initial compiled value", "2026-01-01T00:00:00Z");
|
|
514
|
+
await writeFile(path.join(root, "packages", "compiled", "src", "index.ts"), "export const value = 2;\n");
|
|
515
|
+
await writeFile(path.join(root, "packages", "compiled", "src", "test", "value.test.ts"), [
|
|
516
|
+
'import assert from "node:assert/strict";',
|
|
517
|
+
'import test from "node:test";',
|
|
518
|
+
'import { value } from "../index.js";',
|
|
519
|
+
'test("value is two", () => assert.equal(value, 2));',
|
|
520
|
+
""
|
|
521
|
+
].join("\n"));
|
|
522
|
+
const referenceCommit = await commit(root, "fix(compiled): preserve the new value", "2026-02-01T00:00:00Z");
|
|
523
|
+
return { root, referenceCommit };
|
|
524
|
+
}
|
|
525
|
+
async function repositoryWithRootTestRunner() {
|
|
526
|
+
const root = await mkdtemp(path.join(os.tmpdir(), "routekit-foundry-root-test-runner-"));
|
|
527
|
+
roots.push(root);
|
|
528
|
+
await git(root, "init", "-q");
|
|
529
|
+
await git(root, "config", "user.name", "RouteKit Foundry Test");
|
|
530
|
+
await git(root, "config", "user.email", "foundry@example.test");
|
|
531
|
+
await mkdir(path.join(root, "apps", "api", "src"), { recursive: true });
|
|
532
|
+
await mkdir(path.join(root, "apps", "api", "test"), { recursive: true });
|
|
533
|
+
await writeFile(path.join(root, "pnpm-workspace.yaml"), ["packages:", " - apps/*", ""].join("\n"));
|
|
534
|
+
await writeFile(path.join(root, "package.json"), `${JSON.stringify({
|
|
535
|
+
name: "fixture-root-test-runner",
|
|
536
|
+
private: true,
|
|
537
|
+
packageManager: "pnpm@11.15.1",
|
|
538
|
+
scripts: { test: "vitest run" }
|
|
539
|
+
}, null, 2)}\n`);
|
|
540
|
+
await writeFile(path.join(root, "apps", "api", "package.json"), `${JSON.stringify({
|
|
541
|
+
name: "@fixture/api",
|
|
542
|
+
private: true,
|
|
543
|
+
type: "module"
|
|
544
|
+
}, null, 2)}\n`);
|
|
545
|
+
await writeFile(path.join(root, "apps", "api", "src", "event.ts"), "export const acceptsScreenShare = (type) => type === 'screen_share.started';\n");
|
|
546
|
+
await writeFile(path.join(root, "apps", "api", "test", "event.test.ts"), [
|
|
547
|
+
"import { describe, expect, it } from 'vitest';",
|
|
548
|
+
"import { acceptsScreenShare } from '../src/event.js';",
|
|
549
|
+
"describe('events', () => {",
|
|
550
|
+
" it('rejects unknown events', () => expect(acceptsScreenShare('unknown')).toBe(false));",
|
|
551
|
+
"});",
|
|
552
|
+
""
|
|
553
|
+
].join("\n"));
|
|
554
|
+
await commit(root, "initial realtime event behavior", "2026-01-01T00:00:00Z");
|
|
555
|
+
await writeFile(path.join(root, "apps", "api", "src", "event.ts"), [
|
|
556
|
+
"export const acceptsScreenShare = (type) =>",
|
|
557
|
+
" type === 'screen_share.started' || type === 'screen_share.stopped';",
|
|
558
|
+
""
|
|
559
|
+
].join("\n"));
|
|
560
|
+
await writeFile(path.join(root, "apps", "api", "test", "event.test.ts"), [
|
|
561
|
+
"import { describe, expect, it } from 'vitest';",
|
|
562
|
+
"import { acceptsScreenShare } from '../src/event.js';",
|
|
563
|
+
"describe('events', () => {",
|
|
564
|
+
" it('accepts screen-share lifecycle events', () => {",
|
|
565
|
+
" expect(acceptsScreenShare('screen_share.started')).toBe(true);",
|
|
566
|
+
" expect(acceptsScreenShare('screen_share.stopped')).toBe(true);",
|
|
567
|
+
" });",
|
|
568
|
+
"});",
|
|
569
|
+
""
|
|
570
|
+
].join("\n"));
|
|
571
|
+
const referenceCommit = await commit(root, "accept screen share realtime events", "2026-02-01T00:00:00Z");
|
|
572
|
+
return { root, referenceCommit };
|
|
573
|
+
}
|
|
574
|
+
async function repositoryWithSolutionBranches() {
|
|
575
|
+
const root = await repository();
|
|
576
|
+
const mainBranch = (await git(root, "branch", "--show-current")).stdout.trim();
|
|
577
|
+
const commits = (await git(root, "rev-list", "--reverse", "HEAD")).stdout.trim().split(/\r?\n/u);
|
|
578
|
+
const initialCommit = commits[0];
|
|
579
|
+
const referenceCommit = commits[1];
|
|
580
|
+
await git(root, "checkout", "-q", "-b", "alternate-valid", initialCommit);
|
|
581
|
+
await writeFile(path.join(root, "packages", "stream", "src", "stream-protocol.js"), [
|
|
582
|
+
"export function translateEvents(events) {",
|
|
583
|
+
" const output = [];",
|
|
584
|
+
" let terminal;",
|
|
585
|
+
" for (const event of events) {",
|
|
586
|
+
" if (event.type === 'completed') terminal ??= event;",
|
|
587
|
+
" else output.push(event);",
|
|
588
|
+
" }",
|
|
589
|
+
" return terminal === undefined ? output : output.concat(terminal);",
|
|
590
|
+
"}",
|
|
591
|
+
""
|
|
592
|
+
].join("\n"));
|
|
593
|
+
const alternateCommit = await commit(root, "implement alternate terminal ordering", "2026-02-02T00:00:00Z");
|
|
594
|
+
await git(root, "checkout", "-q", mainBranch);
|
|
595
|
+
await git(root, "checkout", "-q", "-b", "behavior-preserving-valid", referenceCommit);
|
|
596
|
+
await writeFile(path.join(root, "packages", "stream", "src", "stream-protocol.js"), [
|
|
597
|
+
"export function translateEvents(events) {",
|
|
598
|
+
" const terminalIndex = events.findIndex((event) => event.type === 'completed');",
|
|
599
|
+
" if (terminalIndex < 0) return events.slice();",
|
|
600
|
+
" const terminal = events[terminalIndex];",
|
|
601
|
+
" return events.filter((_, index) => index !== terminalIndex && events[index].type !== 'completed').concat(terminal);",
|
|
602
|
+
"}",
|
|
603
|
+
""
|
|
604
|
+
].join("\n"));
|
|
605
|
+
const behaviorPreservingCommit = await commit(root, "refactor terminal ordering without changing behavior", "2026-02-02T12:00:00Z");
|
|
606
|
+
await git(root, "checkout", "-q", mainBranch);
|
|
607
|
+
await git(root, "checkout", "-q", "-b", "plausible-mutant", initialCommit);
|
|
608
|
+
await writeFile(path.join(root, "packages", "stream", "src", "stream-protocol.js"), [
|
|
609
|
+
"export function translateEvents(events) {",
|
|
610
|
+
" const terminal = events.find((event) => event.type === 'completed');",
|
|
611
|
+
" return terminal === undefined ? events : [terminal];",
|
|
612
|
+
"}",
|
|
613
|
+
""
|
|
614
|
+
].join("\n"));
|
|
615
|
+
const mutantCommit = await commit(root, "fix ordering by dropping output", "2026-02-03T00:00:00Z");
|
|
616
|
+
await git(root, "checkout", "-q", mainBranch);
|
|
617
|
+
await git(root, "checkout", "-q", "-b", "unterminated-drop-mutant", referenceCommit);
|
|
618
|
+
await writeFile(path.join(root, "packages", "stream", "src", "stream-protocol.js"), [
|
|
619
|
+
"export function translateEvents(events) {",
|
|
620
|
+
" const terminal = events.find((event) => event.type === 'completed');",
|
|
621
|
+
" return terminal === undefined ? [] : [...events.filter((event) => event.type !== 'completed'), terminal];",
|
|
622
|
+
"}",
|
|
623
|
+
""
|
|
624
|
+
].join("\n"));
|
|
625
|
+
const unterminatedDropCommit = await commit(root, "drop streams without terminal events", "2026-02-04T00:00:00Z");
|
|
626
|
+
await git(root, "checkout", "-q", mainBranch);
|
|
627
|
+
return {
|
|
628
|
+
root,
|
|
629
|
+
alternateCommit,
|
|
630
|
+
behaviorPreservingCommit,
|
|
631
|
+
mutantCommit,
|
|
632
|
+
unterminatedDropCommit
|
|
633
|
+
};
|
|
634
|
+
}
|
|
635
|
+
async function repositoryWithStructuralMigrationBranches() {
|
|
636
|
+
const root = await mkdtemp(path.join(os.tmpdir(), "routekit-structural-foundry-"));
|
|
637
|
+
roots.push(root);
|
|
638
|
+
await git(root, "init", "-q");
|
|
639
|
+
await git(root, "config", "user.name", "RouteKit Structural Test");
|
|
640
|
+
await git(root, "config", "user.email", "structural@example.test");
|
|
641
|
+
await mkdir(path.join(root, "src", "legacy"), { recursive: true });
|
|
642
|
+
await writeFile(path.join(root, "package.json"), `${JSON.stringify({ name: "structural-fixture", private: true, type: "module" }, null, 2)}\n`);
|
|
643
|
+
const legacySource = [
|
|
644
|
+
"export function normalizeName(value) {",
|
|
645
|
+
" return value.trim();",
|
|
646
|
+
"}",
|
|
647
|
+
""
|
|
648
|
+
].join("\n");
|
|
649
|
+
const migratedSource = [
|
|
650
|
+
"export function normalizeName(value) {",
|
|
651
|
+
" return value.trim().toLowerCase();",
|
|
652
|
+
"}",
|
|
653
|
+
""
|
|
654
|
+
].join("\n");
|
|
655
|
+
await writeFile(path.join(root, "src", "legacy", "formatter.js"), legacySource);
|
|
656
|
+
await writeFile(path.join(root, "src", "index.js"), "export { normalizeName } from './legacy/formatter.js';\n");
|
|
657
|
+
await writeFile(path.join(root, "src", "formatter.test.js"), [
|
|
658
|
+
"import { test } from 'node:test';",
|
|
659
|
+
"import { strict as assert } from 'node:assert';",
|
|
660
|
+
"import { normalizeName } from './index.js';",
|
|
661
|
+
"test('trims names', () => {",
|
|
662
|
+
" assert.equal(normalizeName(' RouteKit '), 'RouteKit');",
|
|
663
|
+
"});",
|
|
664
|
+
""
|
|
665
|
+
].join("\n"));
|
|
666
|
+
const initialCommit = await commit(root, "initial legacy formatter", "2026-05-01T00:00:00Z");
|
|
667
|
+
await mkdir(path.join(root, "src", "services"), { recursive: true });
|
|
668
|
+
await writeFile(path.join(root, "src", "services", "formatter.js"), migratedSource);
|
|
669
|
+
await writeFile(path.join(root, "src", "index.js"), "export { normalizeName } from './services/formatter.js';\n");
|
|
670
|
+
await rm(path.join(root, "src", "legacy", "formatter.js"));
|
|
671
|
+
await writeFile(path.join(root, "src", "formatter.test.js"), [
|
|
672
|
+
"import { test } from 'node:test';",
|
|
673
|
+
"import { strict as assert } from 'node:assert';",
|
|
674
|
+
"import { normalizeName } from './index.js';",
|
|
675
|
+
"test('normalizes names through the public facade', () => {",
|
|
676
|
+
" assert.equal(normalizeName(' RouteKit '), 'routekit');",
|
|
677
|
+
"});",
|
|
678
|
+
""
|
|
679
|
+
].join("\n"));
|
|
680
|
+
const referenceCommit = await commit(root, "feat: migrate formatter service and normalize case", "2026-05-02T00:00:00Z");
|
|
681
|
+
const mainBranch = (await git(root, "branch", "--show-current")).stdout.trim();
|
|
682
|
+
const branchFromInitial = async (branch, mutate, subject, date) => {
|
|
683
|
+
await git(root, "checkout", "-q", mainBranch);
|
|
684
|
+
await git(root, "checkout", "-q", "-b", branch, initialCommit);
|
|
685
|
+
await mutate();
|
|
686
|
+
return commit(root, subject, date);
|
|
687
|
+
};
|
|
688
|
+
const alternateCommit = await branchFromInitial("structural-alternate", async () => {
|
|
689
|
+
await mkdir(path.join(root, "src", "services"), { recursive: true });
|
|
690
|
+
await writeFile(path.join(root, "src", "services", "formatter.js"), [
|
|
691
|
+
"export const normalizeName = (value) =>",
|
|
692
|
+
" value.toLocaleLowerCase('en-US').trim();",
|
|
693
|
+
""
|
|
694
|
+
].join("\n"));
|
|
695
|
+
await writeFile(path.join(root, "src", "index.js"), "export { normalizeName } from './services/formatter.js';\n");
|
|
696
|
+
await rm(path.join(root, "src", "legacy", "formatter.js"));
|
|
697
|
+
}, "implement independent formatter migration", "2026-05-03T00:00:00Z");
|
|
698
|
+
const behaviorPreservingCommit = await branchFromInitial("structural-preserving", async () => {
|
|
699
|
+
await mkdir(path.join(root, "src", "internal"), { recursive: true });
|
|
700
|
+
await mkdir(path.join(root, "src", "services"), { recursive: true });
|
|
701
|
+
await writeFile(path.join(root, "src", "internal", "normalize-name.js"), migratedSource);
|
|
702
|
+
await writeFile(path.join(root, "src", "services", "formatter.js"), "export { normalizeName } from '../internal/normalize-name.js';\n");
|
|
703
|
+
await writeFile(path.join(root, "src", "index.js"), "export { normalizeName } from './services/formatter.js';\n");
|
|
704
|
+
await rm(path.join(root, "src", "legacy", "formatter.js"));
|
|
705
|
+
}, "refactor formatter implementation behind service boundary", "2026-05-04T00:00:00Z");
|
|
706
|
+
const partialMigrationCommit = await branchFromInitial("structural-partial", async () => {
|
|
707
|
+
await mkdir(path.join(root, "src", "services"), { recursive: true });
|
|
708
|
+
await writeFile(path.join(root, "src", "services", "formatter.js"), migratedSource);
|
|
709
|
+
await writeFile(path.join(root, "src", "index.js"), "export { normalizeName } from './services/formatter.js';\n");
|
|
710
|
+
}, "partially migrate formatter but retain legacy module", "2026-05-05T00:00:00Z");
|
|
711
|
+
const behaviorOnlyCommit = await branchFromInitial("structural-behavior-only", async () => {
|
|
712
|
+
await writeFile(path.join(root, "src", "legacy", "formatter.js"), migratedSource);
|
|
713
|
+
}, "normalize names without completing migration", "2026-05-06T00:00:00Z");
|
|
714
|
+
const rewardHackingCommit = await branchFromInitial("structural-test-weakening", async () => {
|
|
715
|
+
await writeFile(path.join(root, "src", "formatter.test.js"), ["import { test } from 'node:test';", "test('migration is complete', () => {});", ""].join("\n"));
|
|
716
|
+
}, "weaken migration test without implementing task", "2026-05-07T00:00:00Z");
|
|
717
|
+
await git(root, "checkout", "-q", mainBranch);
|
|
718
|
+
return {
|
|
719
|
+
root,
|
|
720
|
+
initialCommit,
|
|
721
|
+
referenceCommit,
|
|
722
|
+
alternateCommit,
|
|
723
|
+
behaviorPreservingCommit,
|
|
724
|
+
partialMigrationCommit,
|
|
725
|
+
behaviorOnlyCommit,
|
|
726
|
+
rewardHackingCommit
|
|
727
|
+
};
|
|
728
|
+
}
|
|
729
|
+
test("command repetitions and structural inspections use independent clean checkouts", async () => {
|
|
730
|
+
const { root, initialCommit } = await repositoryWithStructuralMigrationBranches();
|
|
731
|
+
const results = await Effect.runPromise(replayRepositoryCommandV1({
|
|
732
|
+
repositoryRoot: root,
|
|
733
|
+
commit: initialCommit,
|
|
734
|
+
recipe: {
|
|
735
|
+
id: "command-clean-repetition",
|
|
736
|
+
kind: "test",
|
|
737
|
+
cwd: ".",
|
|
738
|
+
executable: "node",
|
|
739
|
+
args: [
|
|
740
|
+
"--eval",
|
|
741
|
+
[
|
|
742
|
+
"const fs = require('node:fs');",
|
|
743
|
+
"if (fs.existsSync('.repetition-state')) process.exit(19);",
|
|
744
|
+
"fs.writeFileSync('.repetition-state', 'created');"
|
|
745
|
+
].join(" ")
|
|
746
|
+
],
|
|
747
|
+
timeoutMs: 30_000
|
|
748
|
+
},
|
|
749
|
+
repetitions: 2
|
|
750
|
+
}));
|
|
751
|
+
assert.deepEqual(results.map((result) => result.exitCode), [0, 0]);
|
|
752
|
+
const inspections = await Effect.runPromise(inspectRepositorySolutionPathsV1({
|
|
753
|
+
repositoryRoot: root,
|
|
754
|
+
commit: initialCommit,
|
|
755
|
+
paths: ["src/index.js", "src/legacy", "src/services/formatter.js"],
|
|
756
|
+
repetitions: 2
|
|
757
|
+
}));
|
|
758
|
+
assert.deepEqual(inspections.map((entry) => [entry.repetition, entry.path, entry.kind]), [
|
|
759
|
+
[1, "src/index.js", "file"],
|
|
760
|
+
[1, "src/legacy", "directory"],
|
|
761
|
+
[1, "src/services/formatter.js", "missing"],
|
|
762
|
+
[2, "src/index.js", "file"],
|
|
763
|
+
[2, "src/legacy", "directory"],
|
|
764
|
+
[2, "src/services/formatter.js", "missing"]
|
|
765
|
+
]);
|
|
766
|
+
});
|
|
767
|
+
test("command replay exposes only a synthetic single-commit repository and isolated process state", async () => {
|
|
768
|
+
const { root, initialCommit, referenceCommit } = await repositoryWithStructuralMigrationBranches();
|
|
769
|
+
const results = await Effect.runPromise(replayRepositoryCommandV1({
|
|
770
|
+
repositoryRoot: root,
|
|
771
|
+
commit: initialCommit,
|
|
772
|
+
recipe: {
|
|
773
|
+
id: "command-prove-candidate-isolation",
|
|
774
|
+
kind: "test",
|
|
775
|
+
cwd: ".",
|
|
776
|
+
executable: "node",
|
|
777
|
+
args: [
|
|
778
|
+
"--eval",
|
|
779
|
+
[
|
|
780
|
+
"const cp = require('node:child_process');",
|
|
781
|
+
"const fs = require('node:fs');",
|
|
782
|
+
"const count = cp.execFileSync('git', ['rev-list', '--all', '--count'], { encoding: 'utf8' }).trim();",
|
|
783
|
+
"if (count !== '1') process.exit(41);",
|
|
784
|
+
"const remotes = cp.execFileSync('git', ['remote'], { encoding: 'utf8' }).trim();",
|
|
785
|
+
"if (remotes !== '') process.exit(42);",
|
|
786
|
+
`const hidden = cp.spawnSync('git', ['cat-file', '-e', '${referenceCommit}^{commit}']);`,
|
|
787
|
+
"if (hidden.status === 0) process.exit(43);",
|
|
788
|
+
"if (!process.env.HOME?.endsWith('/home')) process.exit(44);",
|
|
789
|
+
"if (!process.env.TMPDIR?.endsWith('/tmp')) process.exit(45);",
|
|
790
|
+
"fs.writeFileSync(process.env.TMPDIR + '/candidate-write', 'isolated');"
|
|
791
|
+
].join(" ")
|
|
792
|
+
],
|
|
793
|
+
timeoutMs: 30_000
|
|
794
|
+
},
|
|
795
|
+
repetitions: 2
|
|
796
|
+
}));
|
|
797
|
+
assert.ok(results.every((result) => result.exitCode === 0 && !result.timedOut && result.gradeExecuted), JSON.stringify(results, null, 2));
|
|
798
|
+
});
|
|
799
|
+
test("trusted preparation may materialize dependencies while candidate commands remain network isolated", async () => {
|
|
800
|
+
const { root, initialCommit } = await repositoryWithStructuralMigrationBranches();
|
|
801
|
+
const server = net.createServer((socket) => socket.end("ready"));
|
|
802
|
+
await new Promise((resolve, reject) => {
|
|
803
|
+
server.once("error", reject);
|
|
804
|
+
server.listen(0, "127.0.0.1", () => resolve());
|
|
805
|
+
});
|
|
806
|
+
try {
|
|
807
|
+
const address = server.address();
|
|
808
|
+
assert.ok(address && typeof address !== "string");
|
|
809
|
+
const connectionProbe = [
|
|
810
|
+
"const net = require('node:net');",
|
|
811
|
+
`const socket = net.connect({ host: '127.0.0.1', port: ${String(address.port)} });`,
|
|
812
|
+
"socket.once('data', () => process.exit(0));",
|
|
813
|
+
"socket.once('error', () => process.exit(41));",
|
|
814
|
+
"setTimeout(() => process.exit(42), 2_000).unref();"
|
|
815
|
+
].join("\n");
|
|
816
|
+
const trusted = await Effect.runPromise(replayRepositoryCommandV1({
|
|
817
|
+
repositoryRoot: root,
|
|
818
|
+
commit: initialCommit,
|
|
819
|
+
trustedPreparationRecipes: [
|
|
820
|
+
{
|
|
821
|
+
id: "command-trusted-network-materialization",
|
|
822
|
+
kind: "custom",
|
|
823
|
+
cwd: ".",
|
|
824
|
+
executable: "node",
|
|
825
|
+
args: ["--eval", connectionProbe],
|
|
826
|
+
timeoutMs: 5_000
|
|
827
|
+
}
|
|
828
|
+
],
|
|
829
|
+
recipe: {
|
|
830
|
+
id: "command-candidate-offline-control",
|
|
831
|
+
kind: "test",
|
|
832
|
+
cwd: ".",
|
|
833
|
+
executable: "node",
|
|
834
|
+
args: ["--eval", "process.exit(0)"],
|
|
835
|
+
timeoutMs: 5_000
|
|
836
|
+
},
|
|
837
|
+
repetitions: 1
|
|
838
|
+
}));
|
|
839
|
+
assert.equal(trusted[0]?.trustedPreparationResults[0]?.exitCode, 0, JSON.stringify(trusted, null, 2));
|
|
840
|
+
assert.equal(trusted[0]?.exitCode, 0);
|
|
841
|
+
if (process.platform === "darwin") {
|
|
842
|
+
const candidate = await Effect.runPromise(replayRepositoryCommandV1({
|
|
843
|
+
repositoryRoot: root,
|
|
844
|
+
commit: initialCommit,
|
|
845
|
+
recipe: {
|
|
846
|
+
id: "command-candidate-network-denied",
|
|
847
|
+
kind: "test",
|
|
848
|
+
cwd: ".",
|
|
849
|
+
executable: "node",
|
|
850
|
+
args: ["--eval", connectionProbe],
|
|
851
|
+
timeoutMs: 5_000
|
|
852
|
+
},
|
|
853
|
+
repetitions: 1
|
|
854
|
+
}));
|
|
855
|
+
assert.notEqual(candidate[0]?.exitCode, 0);
|
|
856
|
+
}
|
|
857
|
+
}
|
|
858
|
+
finally {
|
|
859
|
+
await new Promise((resolve, reject) => server.close((error) => (error === undefined ? resolve() : reject(error))));
|
|
860
|
+
}
|
|
861
|
+
});
|
|
862
|
+
test("command replay separates trusted preparation, candidate validation, and grading", async () => {
|
|
863
|
+
const { root, initialCommit } = await repositoryWithStructuralMigrationBranches();
|
|
864
|
+
const trustedPreparationRecipe = {
|
|
865
|
+
id: "command-trusted-preparation",
|
|
866
|
+
kind: "custom",
|
|
867
|
+
cwd: ".",
|
|
868
|
+
executable: "node",
|
|
869
|
+
args: ["--eval", "require('node:fs').writeFileSync('.trusted-prepared', 'yes')"],
|
|
870
|
+
timeoutMs: 30_000
|
|
871
|
+
};
|
|
872
|
+
const candidateValidationRecipe = {
|
|
873
|
+
id: "command-candidate-validation",
|
|
874
|
+
kind: "build",
|
|
875
|
+
cwd: ".",
|
|
876
|
+
executable: "node",
|
|
877
|
+
args: [
|
|
878
|
+
"--eval",
|
|
879
|
+
[
|
|
880
|
+
"const fs = require('node:fs');",
|
|
881
|
+
"if (fs.readFileSync('.trusted-prepared', 'utf8') !== 'yes') process.exit(21);",
|
|
882
|
+
"if (fs.readFileSync('src/index.js', 'utf8') !== 'candidate') process.exit(22);"
|
|
883
|
+
].join(" ")
|
|
884
|
+
],
|
|
885
|
+
timeoutMs: 30_000
|
|
886
|
+
};
|
|
887
|
+
const gradeRecipe = {
|
|
888
|
+
id: "command-grade",
|
|
889
|
+
kind: "test",
|
|
890
|
+
cwd: ".",
|
|
891
|
+
executable: "node",
|
|
892
|
+
args: [
|
|
893
|
+
"--eval",
|
|
894
|
+
[
|
|
895
|
+
"const fs = require('node:fs');",
|
|
896
|
+
"if (!fs.existsSync('.trusted-prepared')) process.exit(31);",
|
|
897
|
+
"if (fs.readFileSync('src/index.js', 'utf8') !== 'candidate') process.exit(32);"
|
|
898
|
+
].join(" ")
|
|
899
|
+
],
|
|
900
|
+
timeoutMs: 30_000
|
|
901
|
+
};
|
|
902
|
+
const successful = await Effect.runPromise(replayRepositoryCommandV1({
|
|
903
|
+
repositoryRoot: root,
|
|
904
|
+
commit: initialCommit,
|
|
905
|
+
trustedPreparationRecipes: [trustedPreparationRecipe],
|
|
906
|
+
candidateValidationRecipes: [candidateValidationRecipe],
|
|
907
|
+
fileOverrides: [{ path: "src/index.js", content: "candidate" }],
|
|
908
|
+
recipe: gradeRecipe,
|
|
909
|
+
repetitions: 2
|
|
910
|
+
}));
|
|
911
|
+
assert.ok(successful.every((result) => result.stage === "grade" &&
|
|
912
|
+
result.gradeExecuted &&
|
|
913
|
+
result.exitCode === 0 &&
|
|
914
|
+
result.trustedPreparationResults.length === 1 &&
|
|
915
|
+
result.trustedPreparationResults[0]?.stage === "trusted-preparation" &&
|
|
916
|
+
result.candidateValidationResults.length === 1 &&
|
|
917
|
+
result.candidateValidationResults[0]?.stage === "candidate-validation"));
|
|
918
|
+
const preparationFailure = await Effect.runPromise(replayRepositoryCommandV1({
|
|
919
|
+
repositoryRoot: root,
|
|
920
|
+
commit: initialCommit,
|
|
921
|
+
trustedPreparationRecipes: [
|
|
922
|
+
{
|
|
923
|
+
...trustedPreparationRecipe,
|
|
924
|
+
id: "command-trusted-preparation-fails",
|
|
925
|
+
args: ["--eval", "process.exit(41)"]
|
|
926
|
+
}
|
|
927
|
+
],
|
|
928
|
+
candidateValidationRecipes: [candidateValidationRecipe],
|
|
929
|
+
recipe: gradeRecipe,
|
|
930
|
+
repetitions: 1
|
|
931
|
+
}));
|
|
932
|
+
assert.equal(preparationFailure[0]?.stage, "trusted-preparation");
|
|
933
|
+
assert.equal(preparationFailure[0]?.gradeExecuted, false);
|
|
934
|
+
assert.equal(preparationFailure[0]?.exitCode, 41);
|
|
935
|
+
assert.deepEqual(preparationFailure[0]?.candidateValidationResults, []);
|
|
936
|
+
const validationFailure = await Effect.runPromise(replayRepositoryCommandV1({
|
|
937
|
+
repositoryRoot: root,
|
|
938
|
+
commit: initialCommit,
|
|
939
|
+
trustedPreparationRecipes: [trustedPreparationRecipe],
|
|
940
|
+
candidateValidationRecipes: [
|
|
941
|
+
{
|
|
942
|
+
...candidateValidationRecipe,
|
|
943
|
+
id: "command-candidate-validation-fails",
|
|
944
|
+
args: ["--eval", "process.exit(51)"]
|
|
945
|
+
}
|
|
946
|
+
],
|
|
947
|
+
recipe: gradeRecipe,
|
|
948
|
+
repetitions: 1
|
|
949
|
+
}));
|
|
950
|
+
assert.equal(validationFailure[0]?.stage, "candidate-validation");
|
|
951
|
+
assert.equal(validationFailure[0]?.gradeExecuted, false);
|
|
952
|
+
assert.equal(validationFailure[0]?.exitCode, 51);
|
|
953
|
+
});
|
|
954
|
+
test("cross-file adversary taxonomy identifies coupled repository failure families", () => {
|
|
955
|
+
assert.deepEqual([
|
|
956
|
+
"src/protocol/request-schema.ts",
|
|
957
|
+
"src/adapters/request-adapter.ts",
|
|
958
|
+
"src/storage/config-reader.ts",
|
|
959
|
+
"src/storage/config-writer.ts",
|
|
960
|
+
"src/cli/commands/serve.ts",
|
|
961
|
+
"src/services/router/service.ts",
|
|
962
|
+
"src/index.ts",
|
|
963
|
+
"src/migrations/legacy-reader.ts"
|
|
964
|
+
].map((repositoryPath) => [
|
|
965
|
+
repositoryPath,
|
|
966
|
+
repositoryIntegrationMutationFamilyV1(repositoryPath)
|
|
967
|
+
]), [
|
|
968
|
+
["src/protocol/request-schema.ts", "schema-protocol-change-omission"],
|
|
969
|
+
["src/adapters/request-adapter.ts", "adapter-change-omission"],
|
|
970
|
+
["src/storage/config-reader.ts", "reader-change-omission"],
|
|
971
|
+
["src/storage/config-writer.ts", "writer-change-omission"],
|
|
972
|
+
["src/cli/commands/serve.ts", "cli-forwarding-change-omission"],
|
|
973
|
+
["src/services/router/service.ts", "service-change-omission"],
|
|
974
|
+
["src/index.ts", "facade-export-omission"],
|
|
975
|
+
["src/migrations/legacy-reader.ts", "reader-change-omission"]
|
|
976
|
+
]);
|
|
977
|
+
});
|
|
978
|
+
test("structural oracle rejects incomplete migrations without crediting adversaries that cannot execute", async () => {
|
|
979
|
+
const fixture = await repositoryWithStructuralMigrationBranches();
|
|
980
|
+
const recipe = {
|
|
981
|
+
id: "command-test-formatter-migration",
|
|
982
|
+
kind: "test",
|
|
983
|
+
cwd: ".",
|
|
984
|
+
executable: "node",
|
|
985
|
+
args: ["--test", "src/formatter.test.js"],
|
|
986
|
+
timeoutMs: 30_000
|
|
987
|
+
};
|
|
988
|
+
const seed = {
|
|
989
|
+
version: 1,
|
|
990
|
+
id: "seed-formatter-migration",
|
|
991
|
+
status: "qualified",
|
|
992
|
+
source: {
|
|
993
|
+
kind: "historical-feature",
|
|
994
|
+
changeEpisodeId: "episode-formatter-migration"
|
|
995
|
+
},
|
|
996
|
+
summary: "Migrate the formatter service and normalize name casing.",
|
|
997
|
+
capabilityEvidence: [
|
|
998
|
+
{
|
|
999
|
+
kind: "history-episode",
|
|
1000
|
+
id: "episode-formatter-migration"
|
|
1001
|
+
},
|
|
1002
|
+
{
|
|
1003
|
+
kind: "test",
|
|
1004
|
+
id: "formatter-migration-test",
|
|
1005
|
+
path: "src/formatter.test.js"
|
|
1006
|
+
},
|
|
1007
|
+
{
|
|
1008
|
+
kind: "symbol",
|
|
1009
|
+
id: "formatter-public-facade",
|
|
1010
|
+
path: "src/index.js"
|
|
1011
|
+
},
|
|
1012
|
+
{
|
|
1013
|
+
kind: "symbol",
|
|
1014
|
+
id: "formatter-service",
|
|
1015
|
+
path: "src/services/formatter.js"
|
|
1016
|
+
},
|
|
1017
|
+
{
|
|
1018
|
+
kind: "symbol",
|
|
1019
|
+
id: "formatter-legacy-module",
|
|
1020
|
+
path: "src/legacy/formatter.js"
|
|
1021
|
+
}
|
|
1022
|
+
],
|
|
1023
|
+
initialState: { commit: fixture.initialCommit },
|
|
1024
|
+
targetBehavior: [
|
|
1025
|
+
{
|
|
1026
|
+
id: "formatter-public-behavior",
|
|
1027
|
+
description: "The public formatter normalizes whitespace and casing.",
|
|
1028
|
+
critical: true,
|
|
1029
|
+
evidence: [
|
|
1030
|
+
{
|
|
1031
|
+
kind: "test",
|
|
1032
|
+
id: "formatter-migration-test",
|
|
1033
|
+
path: "src/formatter.test.js"
|
|
1034
|
+
}
|
|
1035
|
+
]
|
|
1036
|
+
},
|
|
1037
|
+
{
|
|
1038
|
+
id: "formatter-service-migration",
|
|
1039
|
+
description: "The formatter is owned by the service path and the legacy module is removed.",
|
|
1040
|
+
critical: true,
|
|
1041
|
+
evidence: [
|
|
1042
|
+
{
|
|
1043
|
+
kind: "history-episode",
|
|
1044
|
+
id: "episode-formatter-migration"
|
|
1045
|
+
}
|
|
1046
|
+
]
|
|
1047
|
+
}
|
|
1048
|
+
],
|
|
1049
|
+
referenceCommit: fixture.referenceCommit,
|
|
1050
|
+
baselineObservations: [
|
|
1051
|
+
{
|
|
1052
|
+
id: "formatter-baseline-1",
|
|
1053
|
+
kind: "test-command",
|
|
1054
|
+
subjectId: `${recipe.id}-baseline-control`,
|
|
1055
|
+
outcome: "pass",
|
|
1056
|
+
detail: "The historical checkout passes before hidden evidence is overlaid."
|
|
1057
|
+
}
|
|
1058
|
+
],
|
|
1059
|
+
preChangeObservations: [
|
|
1060
|
+
{
|
|
1061
|
+
id: "formatter-pre-1",
|
|
1062
|
+
kind: "test-command",
|
|
1063
|
+
subjectId: recipe.id,
|
|
1064
|
+
outcome: "fail",
|
|
1065
|
+
detail: "The reviewed post-change behavior fails before the feature."
|
|
1066
|
+
},
|
|
1067
|
+
{
|
|
1068
|
+
id: "formatter-pre-2",
|
|
1069
|
+
kind: "test-command",
|
|
1070
|
+
subjectId: recipe.id,
|
|
1071
|
+
outcome: "fail",
|
|
1072
|
+
detail: "The reviewed post-change behavior fails on a clean replay."
|
|
1073
|
+
}
|
|
1074
|
+
],
|
|
1075
|
+
postChangeObservations: [
|
|
1076
|
+
{
|
|
1077
|
+
id: "formatter-post-1",
|
|
1078
|
+
kind: "test-command",
|
|
1079
|
+
subjectId: recipe.id,
|
|
1080
|
+
outcome: "pass",
|
|
1081
|
+
detail: "The feature and migration pass."
|
|
1082
|
+
},
|
|
1083
|
+
{
|
|
1084
|
+
id: "formatter-post-2",
|
|
1085
|
+
kind: "test-command",
|
|
1086
|
+
subjectId: recipe.id,
|
|
1087
|
+
outcome: "pass",
|
|
1088
|
+
detail: "The feature and migration pass on a clean replay."
|
|
1089
|
+
}
|
|
1090
|
+
],
|
|
1091
|
+
candidateTestIds: ["formatter-migration-test"],
|
|
1092
|
+
environment: {
|
|
1093
|
+
protectedControlPaths: [],
|
|
1094
|
+
trustedPreparationRecipes: [],
|
|
1095
|
+
candidateValidationRecipes: [
|
|
1096
|
+
{
|
|
1097
|
+
id: "command-build-formatter-migration",
|
|
1098
|
+
kind: "build",
|
|
1099
|
+
cwd: ".",
|
|
1100
|
+
executable: "node",
|
|
1101
|
+
args: ["--check", "src/index.js"],
|
|
1102
|
+
timeoutMs: 30_000
|
|
1103
|
+
}
|
|
1104
|
+
],
|
|
1105
|
+
baselineGradeRecipes: [
|
|
1106
|
+
{
|
|
1107
|
+
...recipe,
|
|
1108
|
+
id: `${recipe.id}-baseline-control`
|
|
1109
|
+
}
|
|
1110
|
+
],
|
|
1111
|
+
gradeRecipes: [recipe]
|
|
1112
|
+
},
|
|
1113
|
+
confidence: "high",
|
|
1114
|
+
risks: [],
|
|
1115
|
+
rejectionReasons: []
|
|
1116
|
+
};
|
|
1117
|
+
const integrationAdversaries = await Effect.runPromise(generateDeterministicAdversariesV1({
|
|
1118
|
+
repositoryRoot: fixture.root,
|
|
1119
|
+
seed,
|
|
1120
|
+
validationRecipe: {
|
|
1121
|
+
id: "command-check-formatter-migration",
|
|
1122
|
+
kind: "check",
|
|
1123
|
+
cwd: ".",
|
|
1124
|
+
executable: "node",
|
|
1125
|
+
args: ["--check", "src/index.js"],
|
|
1126
|
+
timeoutMs: 30_000
|
|
1127
|
+
},
|
|
1128
|
+
repetitions: 2
|
|
1129
|
+
}));
|
|
1130
|
+
assert.deepEqual(integrationAdversaries.report.candidates.map((candidate) => candidate.operator), ["cross-file-omission", "cross-file-omission", "cross-file-omission"]);
|
|
1131
|
+
assert.deepEqual(integrationAdversaries.report.generationCoverage, {
|
|
1132
|
+
implementationPathCount: 3,
|
|
1133
|
+
changedPathCount: 3,
|
|
1134
|
+
generatedCrossFileCandidateCount: 3,
|
|
1135
|
+
omittedCrossFileCandidateCount: 0
|
|
1136
|
+
});
|
|
1137
|
+
assert.deepEqual(integrationAdversaries.report.candidates.map((candidate) => candidate.family), ["facade-export-omission", "new-path-deletion", "removed-path-resurrection"]);
|
|
1138
|
+
assert.ok(integrationAdversaries.report.candidates.every((candidate) => candidate.compilationStatus === "passed" && candidate.relatedPaths.length === 2));
|
|
1139
|
+
const hidden = await Effect.runPromise(buildHistoricalHiddenOracleV1({
|
|
1140
|
+
repositoryRoot: fixture.root,
|
|
1141
|
+
caseId: "case-formatter-migration",
|
|
1142
|
+
seed,
|
|
1143
|
+
repetitions: 2,
|
|
1144
|
+
additionalSolutions: [
|
|
1145
|
+
{
|
|
1146
|
+
id: "independent-migration",
|
|
1147
|
+
kind: "independent-valid",
|
|
1148
|
+
family: "independent-valid",
|
|
1149
|
+
expectedClass: "valid",
|
|
1150
|
+
commit: fixture.alternateCommit
|
|
1151
|
+
},
|
|
1152
|
+
{
|
|
1153
|
+
id: "behavior-preserving-migration",
|
|
1154
|
+
kind: "behavior-preserving-valid",
|
|
1155
|
+
family: "behavior-preserving",
|
|
1156
|
+
expectedClass: "valid",
|
|
1157
|
+
commit: fixture.behaviorPreservingCommit
|
|
1158
|
+
},
|
|
1159
|
+
{
|
|
1160
|
+
id: "partial-migration",
|
|
1161
|
+
kind: "model-near-miss",
|
|
1162
|
+
family: "partial-migration",
|
|
1163
|
+
expectedClass: "wrong",
|
|
1164
|
+
commit: fixture.partialMigrationCommit
|
|
1165
|
+
},
|
|
1166
|
+
{
|
|
1167
|
+
id: "behavior-only-workaround",
|
|
1168
|
+
kind: "model-near-miss",
|
|
1169
|
+
family: "behavior-only-workaround",
|
|
1170
|
+
expectedClass: "wrong",
|
|
1171
|
+
commit: fixture.behaviorOnlyCommit
|
|
1172
|
+
},
|
|
1173
|
+
{
|
|
1174
|
+
id: "test-only-weakening",
|
|
1175
|
+
kind: "reward-hacking",
|
|
1176
|
+
family: "reward-hacking",
|
|
1177
|
+
expectedClass: "wrong",
|
|
1178
|
+
commit: fixture.rewardHackingCommit
|
|
1179
|
+
},
|
|
1180
|
+
...integrationAdversaries.solutions
|
|
1181
|
+
],
|
|
1182
|
+
structuralClauses: [
|
|
1183
|
+
{
|
|
1184
|
+
id: "formatter-service-architecture",
|
|
1185
|
+
kind: "architecture",
|
|
1186
|
+
critical: true,
|
|
1187
|
+
description: "The public facade and formatter service are regular files.",
|
|
1188
|
+
ruleId: "formatter-service-owned",
|
|
1189
|
+
expectedPaths: ["src/index.js", "src/services/formatter.js"]
|
|
1190
|
+
},
|
|
1191
|
+
{
|
|
1192
|
+
id: "formatter-migration-complete",
|
|
1193
|
+
kind: "transformation-completeness",
|
|
1194
|
+
critical: true,
|
|
1195
|
+
description: "The new formatter service exists and the legacy formatter is absent.",
|
|
1196
|
+
ruleId: "formatter-legacy-removed",
|
|
1197
|
+
forbiddenLegacyPaths: ["src/legacy/formatter.js"],
|
|
1198
|
+
requiredNewPaths: ["src/services/formatter.js"]
|
|
1199
|
+
}
|
|
1200
|
+
]
|
|
1201
|
+
}));
|
|
1202
|
+
const adequacy = evaluateRepositoryOracleAdequacyV1(hidden);
|
|
1203
|
+
const nonExecutableIds = new Set(integrationAdversaries.solutions
|
|
1204
|
+
.filter((solution) => ["facade-export-omission", "new-path-deletion"].includes(solution.family))
|
|
1205
|
+
.map((solution) => solution.id));
|
|
1206
|
+
assert.equal(nonExecutableIds.size, 2);
|
|
1207
|
+
assert.equal(adequacy.admitted, false);
|
|
1208
|
+
assert.deepEqual(adequacy.falseAcceptedSolutionIds, []);
|
|
1209
|
+
assert.deepEqual(adequacy.falseRejectedSolutionIds, []);
|
|
1210
|
+
assert.deepEqual(adequacy.unstableSolutionIds, [...nonExecutableIds].sort());
|
|
1211
|
+
assert.deepEqual(adequacy.insufficientObservationPairs, []);
|
|
1212
|
+
assert.ok(adequacy.mutationFamilyKills
|
|
1213
|
+
.filter((family) => !["facade-export-omission", "new-path-deletion"].includes(family.family))
|
|
1214
|
+
.every((family) => family.killed === family.total));
|
|
1215
|
+
assert.ok(adequacy.mutationFamilyKills
|
|
1216
|
+
.filter((family) => ["facade-export-omission", "new-path-deletion"].includes(family.family))
|
|
1217
|
+
.every((family) => family.killed === 0));
|
|
1218
|
+
assert.equal(evaluateRepositoryOracleAdequacyV1({
|
|
1219
|
+
...hidden,
|
|
1220
|
+
solutions: hidden.solutions.filter((solution) => !nonExecutableIds.has(solution.id)),
|
|
1221
|
+
observations: hidden.observations.filter((observation) => !nonExecutableIds.has(observation.solutionId))
|
|
1222
|
+
}).admitted, true);
|
|
1223
|
+
const removedPathResurrection = integrationAdversaries.solutions.find((solution) => solution.family === "removed-path-resurrection");
|
|
1224
|
+
assert.ok(removedPathResurrection);
|
|
1225
|
+
assert.deepEqual(hidden.observations
|
|
1226
|
+
.filter((observation) => observation.solutionId === removedPathResurrection.id &&
|
|
1227
|
+
observation.clauseId !== "candidate-policy-protected-paths")
|
|
1228
|
+
.map((observation) => [observation.clauseId, observation.outcome]), [
|
|
1229
|
+
["validation-command-build-formatter-migration", "pass"],
|
|
1230
|
+
["validation-command-build-formatter-migration", "pass"],
|
|
1231
|
+
["test-command-test-formatter-migration", "pass"],
|
|
1232
|
+
["test-command-test-formatter-migration", "pass"],
|
|
1233
|
+
["formatter-service-architecture", "pass"],
|
|
1234
|
+
["formatter-service-architecture", "pass"],
|
|
1235
|
+
["formatter-migration-complete", "fail"],
|
|
1236
|
+
["formatter-migration-complete", "fail"]
|
|
1237
|
+
]);
|
|
1238
|
+
assert.equal(hidden.clauses.length, 5);
|
|
1239
|
+
assert.deepEqual(hidden.observations
|
|
1240
|
+
.filter((observation) => observation.solutionId === "partial-migration" &&
|
|
1241
|
+
observation.clauseId !== "candidate-policy-protected-paths")
|
|
1242
|
+
.map((observation) => [observation.clauseId, observation.outcome]), [
|
|
1243
|
+
["validation-command-build-formatter-migration", "pass"],
|
|
1244
|
+
["validation-command-build-formatter-migration", "pass"],
|
|
1245
|
+
["test-command-test-formatter-migration", "pass"],
|
|
1246
|
+
["test-command-test-formatter-migration", "pass"],
|
|
1247
|
+
["formatter-service-architecture", "pass"],
|
|
1248
|
+
["formatter-service-architecture", "pass"],
|
|
1249
|
+
["formatter-migration-complete", "fail"],
|
|
1250
|
+
["formatter-migration-complete", "fail"]
|
|
1251
|
+
]);
|
|
1252
|
+
assert.deepEqual(hidden.observations
|
|
1253
|
+
.filter((observation) => observation.solutionId === "behavior-only-workaround" &&
|
|
1254
|
+
observation.clauseId !== "candidate-policy-protected-paths")
|
|
1255
|
+
.map((observation) => [observation.clauseId, observation.outcome]), [
|
|
1256
|
+
["validation-command-build-formatter-migration", "pass"],
|
|
1257
|
+
["validation-command-build-formatter-migration", "pass"],
|
|
1258
|
+
["test-command-test-formatter-migration", "pass"],
|
|
1259
|
+
["test-command-test-formatter-migration", "pass"],
|
|
1260
|
+
["formatter-service-architecture", "fail"],
|
|
1261
|
+
["formatter-service-architecture", "fail"],
|
|
1262
|
+
["formatter-migration-complete", "fail"],
|
|
1263
|
+
["formatter-migration-complete", "fail"]
|
|
1264
|
+
]);
|
|
1265
|
+
assert.ok(hidden.knownLimitations.every((limitation) => !limitation.includes("only hard clause")));
|
|
1266
|
+
});
|
|
1267
|
+
test("repository model maps packages, public surfaces, tests, history, and recurring work", async () => {
|
|
1268
|
+
const root = await repository();
|
|
1269
|
+
const map = await Effect.runPromise(buildRepositoryBehaviorMapV1({
|
|
1270
|
+
repositoryRoot: root,
|
|
1271
|
+
requestedRef: "HEAD"
|
|
1272
|
+
}));
|
|
1273
|
+
assert.equal(map.repository.root, await realpath(root));
|
|
1274
|
+
assert.ok(map.packages.some((entry) => entry.name === "@fixture/stream"));
|
|
1275
|
+
assert.ok(map.publicSurfaces.some((entry) => entry.packageId === "package-fixture-stream" &&
|
|
1276
|
+
entry.path === "packages/stream/src/index.js"));
|
|
1277
|
+
assert.ok(map.protocols.some((entry) => entry.path === "packages/stream/src/stream-protocol.js" && entry.name === "translateEvents"));
|
|
1278
|
+
const testNode = map.tests.find((entry) => entry.path === "packages/stream/src/stream-protocol.test.js");
|
|
1279
|
+
assert.ok(testNode);
|
|
1280
|
+
assert.ok(map.testBindings.some((binding) => binding.testId === testNode.id && binding.signals.includes("direct-import")));
|
|
1281
|
+
assert.equal(map.historyEpisodes.length, 4);
|
|
1282
|
+
assert.equal(map.historyEpisodes.filter((episode) => episode.seedEligible).length, 3);
|
|
1283
|
+
assert.equal(map.workloadSignals.length, 1);
|
|
1284
|
+
assert.equal(map.workloadSignals[0]?.frequency, 2);
|
|
1285
|
+
assert.deepEqual(map.workloadSignals[0]?.pathPatterns, ["packages/stream"]);
|
|
1286
|
+
});
|
|
1287
|
+
test("TypeScript repository indexing keeps sanitized public-surface and protocol ids unique", async () => {
|
|
1288
|
+
const root = await mkdtemp(path.join(os.tmpdir(), "routekit-foundry-index-ids-"));
|
|
1289
|
+
roots.push(root);
|
|
1290
|
+
await git(root, "init", "-q");
|
|
1291
|
+
await git(root, "config", "user.name", "RouteKit Foundry Test");
|
|
1292
|
+
await git(root, "config", "user.email", "foundry@example.test");
|
|
1293
|
+
await mkdir(path.join(root, "src"), { recursive: true });
|
|
1294
|
+
await writeFile(path.join(root, "package.json"), `${JSON.stringify({
|
|
1295
|
+
name: "@fixture/index-ids",
|
|
1296
|
+
exports: {
|
|
1297
|
+
"./v4/mini": "./src/index.ts",
|
|
1298
|
+
"./v4-mini": "./src/index.ts"
|
|
1299
|
+
}
|
|
1300
|
+
}, null, 2)}\n`);
|
|
1301
|
+
await writeFile(path.join(root, "src", "index.ts"), ["export type ZodType = string;", "export interface ZodType {}", ""].join("\n"));
|
|
1302
|
+
await commit(root, "initial index fixture", "2026-01-01T00:00:00Z");
|
|
1303
|
+
const snapshot = await Effect.runPromise(captureGitTreeSnapshotV1({
|
|
1304
|
+
repositoryRoot: root,
|
|
1305
|
+
requestedRef: "HEAD"
|
|
1306
|
+
}));
|
|
1307
|
+
const index = await Effect.runPromise(indexTypeScriptRepositoryV1(snapshot));
|
|
1308
|
+
assert.equal(new Set(index.publicSurfaces.map((entry) => entry.id)).size, index.publicSurfaces.length);
|
|
1309
|
+
assert.equal(new Set(index.protocols.map((entry) => entry.id)).size, index.protocols.length);
|
|
1310
|
+
});
|
|
1311
|
+
test("root test commands cover nested workspace tests when the owning package has no test script", async () => {
|
|
1312
|
+
const fixture = await repositoryWithRootTestRunner();
|
|
1313
|
+
const map = await Effect.runPromise(buildRepositoryBehaviorMapV1({
|
|
1314
|
+
repositoryRoot: fixture.root,
|
|
1315
|
+
requestedRef: "HEAD"
|
|
1316
|
+
}));
|
|
1317
|
+
const testNode = map.tests.find((entry) => entry.path === "apps/api/test/event.test.ts");
|
|
1318
|
+
assert.ok(testNode);
|
|
1319
|
+
assert.deepEqual(testNode.commandIds, ["command-fixture-root-test-runner-test"]);
|
|
1320
|
+
const seeds = await Effect.runPromise(extractHistoricalTaskSeedsV1(map));
|
|
1321
|
+
const seed = seeds.find((entry) => entry.referenceCommit === fixture.referenceCommit);
|
|
1322
|
+
assert.ok(seed);
|
|
1323
|
+
assert.deepEqual(seed.environment.gradeRecipes[0]?.args, [
|
|
1324
|
+
"pnpm@11.15.1",
|
|
1325
|
+
"exec",
|
|
1326
|
+
"vitest",
|
|
1327
|
+
"run",
|
|
1328
|
+
"apps/api/test/event.test.ts"
|
|
1329
|
+
]);
|
|
1330
|
+
});
|
|
1331
|
+
test("repository model detects recurring cross-package work even when commit scopes differ", async () => {
|
|
1332
|
+
const root = await repositoryWithCrossPackageCochange();
|
|
1333
|
+
const map = await Effect.runPromise(buildRepositoryBehaviorMapV1({
|
|
1334
|
+
repositoryRoot: root,
|
|
1335
|
+
requestedRef: "HEAD"
|
|
1336
|
+
}));
|
|
1337
|
+
const signal = map.workloadSignals.find((entry) => entry.kind === "recurring-cochange" &&
|
|
1338
|
+
JSON.stringify(entry.pathPatterns) ===
|
|
1339
|
+
JSON.stringify(["packages/accounts", "packages/gateway"]));
|
|
1340
|
+
assert.ok(signal);
|
|
1341
|
+
assert.equal(signal.frequency, 2);
|
|
1342
|
+
assert.equal(signal.confidence, "medium");
|
|
1343
|
+
assert.equal(signal.importance, "normal");
|
|
1344
|
+
assert.deepEqual(signal.episodeIds
|
|
1345
|
+
.map((id) => map.historyEpisodes.find((episode) => episode.id === id)?.subject)
|
|
1346
|
+
.sort(), ["fix(accounts): surface discovery failures", "fix(router): preserve healthy provider models"]);
|
|
1347
|
+
});
|
|
1348
|
+
test("repository model excludes package-like trees outside the declared pnpm workspace", async () => {
|
|
1349
|
+
const root = await repositoryWithDeclaredWorkspaceBoundary();
|
|
1350
|
+
const map = await Effect.runPromise(buildRepositoryBehaviorMapV1({
|
|
1351
|
+
repositoryRoot: root,
|
|
1352
|
+
requestedRef: "HEAD"
|
|
1353
|
+
}));
|
|
1354
|
+
assert.deepEqual(map.packages.map((entry) => entry.path), ["packages/owned", "."]);
|
|
1355
|
+
assert.ok(map.publicSurfaces.every((entry) => !entry.path.startsWith(".repos/")));
|
|
1356
|
+
assert.ok(map.protocols.every((entry) => !entry.path.startsWith(".repos/")));
|
|
1357
|
+
assert.ok(map.tests.every((entry) => !entry.path.startsWith(".repos/")));
|
|
1358
|
+
const vendoredEpisode = map.historyEpisodes.find((entry) => entry.subject === "fix(vendor): update embedded source snapshot");
|
|
1359
|
+
assert.ok(vendoredEpisode);
|
|
1360
|
+
assert.deepEqual(vendoredEpisode.changedPaths, []);
|
|
1361
|
+
assert.equal(vendoredEpisode.seedEligible, false);
|
|
1362
|
+
assert.equal(map.historyEpisodes.find((entry) => entry.subject === "fix(owned): preserve workspace behavior")
|
|
1363
|
+
?.seedEligible, true);
|
|
1364
|
+
assert.deepEqual(map.blindSpots.find((entry) => entry.id === "blind-excluded-package-roots")?.paths, [".repos/vendored"]);
|
|
1365
|
+
const seeds = await Effect.runPromise(extractHistoricalTaskSeedsV1(map));
|
|
1366
|
+
assert.deepEqual(seeds.map((seed) => seed.summary), ["fix(owned): preserve workspace behavior"]);
|
|
1367
|
+
assert.deepEqual(seeds[0]?.environment.gradeRecipes[0]?.args, [
|
|
1368
|
+
"pnpm@11.15.1",
|
|
1369
|
+
"exec",
|
|
1370
|
+
"vp",
|
|
1371
|
+
"test",
|
|
1372
|
+
"run",
|
|
1373
|
+
"src/index.test.js"
|
|
1374
|
+
]);
|
|
1375
|
+
});
|
|
1376
|
+
test("historical seed extraction preserves behavior delta but remains unqualified before replay", async () => {
|
|
1377
|
+
const root = await repository();
|
|
1378
|
+
const map = await Effect.runPromise(buildRepositoryBehaviorMapV1({
|
|
1379
|
+
repositoryRoot: root,
|
|
1380
|
+
requestedRef: "HEAD"
|
|
1381
|
+
}));
|
|
1382
|
+
const seeds = await Effect.runPromise(extractHistoricalTaskSeedsV1(map));
|
|
1383
|
+
assert.equal(seeds.length, 3);
|
|
1384
|
+
assert.ok(seeds.every((seed) => seed.status === "candidate"));
|
|
1385
|
+
assert.ok(seeds.every((seed) => seed.initialState.parentCommit === undefined &&
|
|
1386
|
+
seed.referenceCommit !== undefined &&
|
|
1387
|
+
seed.environment.gradeRecipes.length > 0 &&
|
|
1388
|
+
seed.environment.gradeRecipes.every((recipe) => recipe.kind === "test") &&
|
|
1389
|
+
seed.baselineObservations.length === 0 &&
|
|
1390
|
+
seed.preChangeObservations.length === 0 &&
|
|
1391
|
+
seed.postChangeObservations.length === 0));
|
|
1392
|
+
assert.ok(seeds.some((seed) => seed.summary === "fix terminal event ordering"));
|
|
1393
|
+
assert.equal(seeds.find((seed) => seed.summary === "feat(stream): preserve usage summaries")?.source.kind, "historical-feature");
|
|
1394
|
+
});
|
|
1395
|
+
test("historical seeds derive full frozen pnpm preparation and project-reference builds", async () => {
|
|
1396
|
+
const root = await repositoryWithPnpmBuildGraph();
|
|
1397
|
+
const map = await Effect.runPromise(buildRepositoryBehaviorMapV1({
|
|
1398
|
+
repositoryRoot: root,
|
|
1399
|
+
requestedRef: "HEAD"
|
|
1400
|
+
}));
|
|
1401
|
+
const seeds = await Effect.runPromise(extractHistoricalTaskSeedsV1(map));
|
|
1402
|
+
const seed = seeds.find((candidate) => candidate.summary === "fix(app): preserve the new value");
|
|
1403
|
+
assert.ok(seed);
|
|
1404
|
+
assert.deepEqual(seed.environment.packageManager, {
|
|
1405
|
+
name: "pnpm",
|
|
1406
|
+
version: "11.15.1"
|
|
1407
|
+
});
|
|
1408
|
+
assert.equal(seed.environment.lockfilePath, "pnpm-lock.yaml");
|
|
1409
|
+
assert.deepEqual(seed.environment.baselineGradeRecipes.map((recipe) => [recipe.cwd, recipe.args]), [
|
|
1410
|
+
["packages/app", ["pnpm@11.15.1", "exec", "node", "--test", "src/index.test.js"]],
|
|
1411
|
+
["packages/core", ["pnpm@11.15.1", "exec", "node", "--test", "src/index.test.js"]]
|
|
1412
|
+
]);
|
|
1413
|
+
assert.deepEqual(seed.environment.gradeRecipes.map((recipe) => [recipe.cwd, recipe.kind]), [
|
|
1414
|
+
["packages/app", "test"],
|
|
1415
|
+
["packages/core", "test"]
|
|
1416
|
+
]);
|
|
1417
|
+
assert.deepEqual(seed.environment.protectedControlPaths, [
|
|
1418
|
+
"package.json",
|
|
1419
|
+
"packages/app/package.json",
|
|
1420
|
+
"packages/app/tsconfig.json",
|
|
1421
|
+
"packages/core/package.json",
|
|
1422
|
+
"packages/core/tsconfig.json",
|
|
1423
|
+
"packages/non-ts/package.json",
|
|
1424
|
+
"packages/setup/package.json",
|
|
1425
|
+
"packages/setup/tsconfig.json",
|
|
1426
|
+
"pnpm-lock.yaml",
|
|
1427
|
+
"pnpm-workspace.yaml"
|
|
1428
|
+
]);
|
|
1429
|
+
assert.deepEqual(seed.environment.trustedPreparationRecipes.map((recipe) => [recipe.executable, recipe.args]), [
|
|
1430
|
+
[
|
|
1431
|
+
"corepack",
|
|
1432
|
+
[
|
|
1433
|
+
"pnpm@11.15.1",
|
|
1434
|
+
"install",
|
|
1435
|
+
"--prefer-offline",
|
|
1436
|
+
"--frozen-lockfile",
|
|
1437
|
+
"--ignore-pnpmfile",
|
|
1438
|
+
"--ignore-scripts"
|
|
1439
|
+
]
|
|
1440
|
+
]
|
|
1441
|
+
]);
|
|
1442
|
+
assert.deepEqual(seed.environment.candidateValidationRecipes.map((recipe) => [recipe.cwd, recipe.args]), [
|
|
1443
|
+
["packages/core", ["pnpm@11.15.1", "run", "build"]],
|
|
1444
|
+
["packages/non-ts", ["pnpm@11.15.1", "run", "build"]],
|
|
1445
|
+
["packages/setup", ["pnpm@11.15.1", "run", "build"]],
|
|
1446
|
+
["packages/app", ["pnpm@11.15.1", "run", "build"]]
|
|
1447
|
+
]);
|
|
1448
|
+
const replay = await Effect.runPromise(replayRepositoryCommandV1({
|
|
1449
|
+
repositoryRoot: root,
|
|
1450
|
+
commit: seed.initialState.commit,
|
|
1451
|
+
trustedPreparationRecipes: seed.environment.trustedPreparationRecipes,
|
|
1452
|
+
candidateValidationRecipes: [seed.environment.candidateValidationRecipes[0]],
|
|
1453
|
+
recipe: seed.environment.baselineGradeRecipes[0],
|
|
1454
|
+
repetitions: 1
|
|
1455
|
+
}));
|
|
1456
|
+
assert.equal(replay[0]?.exitCode, 0, JSON.stringify(replay, null, 2));
|
|
1457
|
+
assert.equal(replay[0]?.candidateValidationResults[0]?.exitCode, 0, JSON.stringify(replay, null, 2));
|
|
1458
|
+
assert.doesNotMatch([
|
|
1459
|
+
replay[0]?.candidateValidationResults[0]?.stdout,
|
|
1460
|
+
replay[0]?.candidateValidationResults[0]?.stderr
|
|
1461
|
+
].join("\n"), /Scope:\s+all\s+\d+\s+workspace projects|Lockfile passes supply-chain policies/u, "candidate validation must reuse the trusted pnpm store instead of reinstalling the workspace");
|
|
1462
|
+
});
|
|
1463
|
+
test("historical seed build planning fails closed for invalid TypeScript project references", async () => {
|
|
1464
|
+
for (const [mode, expected] of [
|
|
1465
|
+
["escaping", "project reference escapes the repository"],
|
|
1466
|
+
["malformed", "project reference 0 is malformed"],
|
|
1467
|
+
["cycle", "project references contain a cycle"]
|
|
1468
|
+
]) {
|
|
1469
|
+
const root = await repositoryWithPnpmBuildGraph(mode);
|
|
1470
|
+
const map = await Effect.runPromise(buildRepositoryBehaviorMapV1({
|
|
1471
|
+
repositoryRoot: root,
|
|
1472
|
+
requestedRef: "HEAD"
|
|
1473
|
+
}));
|
|
1474
|
+
await assert.rejects(Effect.runPromise(extractHistoricalTaskSeedsV1(map)), new RegExp(expected, "u"));
|
|
1475
|
+
}
|
|
1476
|
+
});
|
|
1477
|
+
test("historical qualification uses pre-existing baseline controls when the strong regression test is newly added", async () => {
|
|
1478
|
+
const root = await repositoryWithNewRegressionTest();
|
|
1479
|
+
const map = await Effect.runPromise(buildRepositoryBehaviorMapV1({
|
|
1480
|
+
repositoryRoot: root,
|
|
1481
|
+
requestedRef: "HEAD"
|
|
1482
|
+
}));
|
|
1483
|
+
const seeds = await Effect.runPromise(extractHistoricalTaskSeedsV1(map));
|
|
1484
|
+
const candidate = seeds.find((seed) => seed.summary === "fix(normalizer): trim surrounding whitespace");
|
|
1485
|
+
assert.ok(candidate);
|
|
1486
|
+
assert.deepEqual(candidate.environment.baselineGradeRecipes.map((recipe) => ({
|
|
1487
|
+
id: recipe.id,
|
|
1488
|
+
args: recipe.args
|
|
1489
|
+
})), [
|
|
1490
|
+
{
|
|
1491
|
+
id: "command-fixture-normalizer-test-baseline-control",
|
|
1492
|
+
args: ["pnpm@11.15.1", "exec", "node", "--test", "src/index.test.js"]
|
|
1493
|
+
}
|
|
1494
|
+
]);
|
|
1495
|
+
assert.deepEqual(candidate.environment.gradeRecipes.map((recipe) => ({
|
|
1496
|
+
id: recipe.id,
|
|
1497
|
+
args: recipe.args
|
|
1498
|
+
})), [
|
|
1499
|
+
{
|
|
1500
|
+
id: "command-fixture-normalizer-test-targeted",
|
|
1501
|
+
args: ["pnpm@11.15.1", "exec", "node", "--test", "src/whitespace-regression.test.js"]
|
|
1502
|
+
}
|
|
1503
|
+
]);
|
|
1504
|
+
const qualification = await Effect.runPromise(qualifyHistoricalTaskSeedV1({
|
|
1505
|
+
repositoryRoot: root,
|
|
1506
|
+
seed: candidate,
|
|
1507
|
+
repetitions: 2
|
|
1508
|
+
}));
|
|
1509
|
+
assert.equal(qualification.status, "qualified");
|
|
1510
|
+
assert.deepEqual(qualification.rejectionReasons, []);
|
|
1511
|
+
assert.equal(qualification.baselineObservations.length, 2);
|
|
1512
|
+
assert.equal(qualification.preChangeObservations.length, 2);
|
|
1513
|
+
assert.equal(qualification.postChangeObservations.length, 2);
|
|
1514
|
+
assert.ok(qualification.baselineObservations.every((observation) => observation.subjectId === "command-fixture-normalizer-test-baseline-control" &&
|
|
1515
|
+
observation.outcome === "pass"));
|
|
1516
|
+
assert.ok(qualification.preChangeObservations.every((observation) => observation.subjectId === "command-fixture-normalizer-test-targeted" &&
|
|
1517
|
+
observation.outcome === "fail"));
|
|
1518
|
+
assert.ok(qualification.postChangeObservations.every((observation) => observation.subjectId === "command-fixture-normalizer-test-targeted" &&
|
|
1519
|
+
observation.outcome === "pass"));
|
|
1520
|
+
});
|
|
1521
|
+
test("historical qualification narrows compiled Node test suites to reviewed test outputs", async () => {
|
|
1522
|
+
const fixture = await repositoryWithCompiledNodeTestRunner();
|
|
1523
|
+
const map = await Effect.runPromise(buildRepositoryBehaviorMapV1({
|
|
1524
|
+
repositoryRoot: fixture.root,
|
|
1525
|
+
requestedRef: "HEAD"
|
|
1526
|
+
}));
|
|
1527
|
+
const seeds = await Effect.runPromise(extractHistoricalTaskSeedsV1(map));
|
|
1528
|
+
const candidate = seeds.find((seed) => seed.referenceCommit === fixture.referenceCommit);
|
|
1529
|
+
assert.ok(candidate);
|
|
1530
|
+
assert.deepEqual(candidate.environment.baselineGradeRecipes[0]?.args, [
|
|
1531
|
+
"pnpm@11.15.1",
|
|
1532
|
+
"exec",
|
|
1533
|
+
"node",
|
|
1534
|
+
"--test",
|
|
1535
|
+
"dist/test/value.test.js"
|
|
1536
|
+
]);
|
|
1537
|
+
assert.deepEqual(candidate.environment.gradeRecipes[0]?.args, [
|
|
1538
|
+
"pnpm@11.15.1",
|
|
1539
|
+
"exec",
|
|
1540
|
+
"node",
|
|
1541
|
+
"--test",
|
|
1542
|
+
"dist/test/value.test.js"
|
|
1543
|
+
]);
|
|
1544
|
+
assert.equal(candidate.environment.baselineGradeRecipes[0]?.timeoutMs, 8 * 60_000);
|
|
1545
|
+
assert.equal(candidate.environment.gradeRecipes[0]?.timeoutMs, 8 * 60_000);
|
|
1546
|
+
const qualification = await Effect.runPromise(qualifyHistoricalTaskSeedV1({
|
|
1547
|
+
repositoryRoot: fixture.root,
|
|
1548
|
+
seed: candidate,
|
|
1549
|
+
repetitions: 2
|
|
1550
|
+
}));
|
|
1551
|
+
assert.equal(qualification.status, "qualified");
|
|
1552
|
+
assert.deepEqual(qualification.rejectionReasons, []);
|
|
1553
|
+
});
|
|
1554
|
+
test("historical qualification narrows Cloud-style tsx tests without dropping bounded concurrency", async () => {
|
|
1555
|
+
const root = await mkdtemp(path.join(os.tmpdir(), "routekit-foundry-concurrent-test-"));
|
|
1556
|
+
roots.push(root);
|
|
1557
|
+
await git(root, "init", "-q");
|
|
1558
|
+
await git(root, "config", "user.name", "RouteKit Foundry Test");
|
|
1559
|
+
await git(root, "config", "user.email", "foundry@example.test");
|
|
1560
|
+
await writeFile(path.join(root, "package.json"), JSON.stringify({ name: "fixture-root", private: true, packageManager: "pnpm@11.15.1" }));
|
|
1561
|
+
const cloudScript = 'tsx --test --test-concurrency=4 "test/cases/*.test.ts" "test/deployment/*.test.ts" "test/db/*.test.ts"';
|
|
1562
|
+
const unsafeScript = `tsx --test --test-concurrency=4 --import=./unexpected.mjs "test/cases/*.test.ts"`;
|
|
1563
|
+
for (const [name, script] of [["cloud", cloudScript], ["unsafe", unsafeScript]]) {
|
|
1564
|
+
const directory = path.join(root, "apps", name, "test", "cases");
|
|
1565
|
+
await mkdir(directory, { recursive: true });
|
|
1566
|
+
await writeFile(path.join(root, "apps", name, "package.json"), JSON.stringify({ name: `@fixture/${name}`, private: true, scripts: { test: script } }));
|
|
1567
|
+
await writeFile(path.join(directory, "behavior.test.ts"), "import test from 'node:test';\n");
|
|
1568
|
+
}
|
|
1569
|
+
const commitId = await commit(root, "add Cloud-style test scripts", "2026-02-01T00:00:00Z");
|
|
1570
|
+
const gradeRecipes = ["cloud", "unsafe"].map((name) => ({
|
|
1571
|
+
id: `command-${name}-test`,
|
|
1572
|
+
kind: "test",
|
|
1573
|
+
cwd: `apps/${name}`,
|
|
1574
|
+
executable: "corepack",
|
|
1575
|
+
args: ["pnpm@11.15.1", "run", "test"],
|
|
1576
|
+
timeoutMs: 120_000
|
|
1577
|
+
}));
|
|
1578
|
+
const testPaths = ["cloud", "unsafe"].map((name) => `apps/${name}/test/cases/behavior.test.ts`);
|
|
1579
|
+
const environment = await Effect.runPromise(deriveHistoricalSeedEnvironmentV1({
|
|
1580
|
+
repositoryRoot: root,
|
|
1581
|
+
commit: commitId,
|
|
1582
|
+
gradeRecipes,
|
|
1583
|
+
testPaths,
|
|
1584
|
+
changedPaths: testPaths
|
|
1585
|
+
}));
|
|
1586
|
+
const expected = [
|
|
1587
|
+
"pnpm@11.15.1",
|
|
1588
|
+
"exec",
|
|
1589
|
+
"tsx",
|
|
1590
|
+
"--test",
|
|
1591
|
+
"--test-concurrency=4",
|
|
1592
|
+
"test/cases/behavior.test.ts"
|
|
1593
|
+
];
|
|
1594
|
+
assert.deepEqual(environment.gradeRecipes[0]?.args, expected);
|
|
1595
|
+
assert.deepEqual(environment.baselineGradeRecipes[0]?.args, expected);
|
|
1596
|
+
assert.equal(environment.gradeRecipes[0]?.id, "command-cloud-test-targeted");
|
|
1597
|
+
assert.deepEqual(environment.gradeRecipes[1]?.args, gradeRecipes[1]?.args);
|
|
1598
|
+
assert.deepEqual(environment.baselineGradeRecipes[1]?.args, [
|
|
1599
|
+
"pnpm@11.15.1", "run", "test"
|
|
1600
|
+
]);
|
|
1601
|
+
});
|
|
1602
|
+
test("historical qualification never promotes a pre-change validation failure to behavioral evidence", async () => {
|
|
1603
|
+
const root = await repositoryWithNewRegressionTest();
|
|
1604
|
+
const map = await Effect.runPromise(buildRepositoryBehaviorMapV1({
|
|
1605
|
+
repositoryRoot: root,
|
|
1606
|
+
requestedRef: "HEAD"
|
|
1607
|
+
}));
|
|
1608
|
+
const seeds = await Effect.runPromise(extractHistoricalTaskSeedsV1(map));
|
|
1609
|
+
const candidate = seeds.find((seed) => seed.summary === "fix(normalizer): trim surrounding whitespace");
|
|
1610
|
+
assert.ok(candidate);
|
|
1611
|
+
const compileContract = {
|
|
1612
|
+
id: "command-compile-reviewed-regression",
|
|
1613
|
+
kind: "build",
|
|
1614
|
+
cwd: "packages/normalizer",
|
|
1615
|
+
executable: "node",
|
|
1616
|
+
args: [
|
|
1617
|
+
"--eval",
|
|
1618
|
+
[
|
|
1619
|
+
"const fs = require('node:fs');",
|
|
1620
|
+
"const testExists = fs.existsSync('src/whitespace-regression.test.js');",
|
|
1621
|
+
"const implementation = fs.readFileSync('src/index.js', 'utf8');",
|
|
1622
|
+
"if (testExists && !implementation.includes('.trim()')) process.exit(51);"
|
|
1623
|
+
].join(" ")
|
|
1624
|
+
],
|
|
1625
|
+
timeoutMs: 30_000
|
|
1626
|
+
};
|
|
1627
|
+
const phases = [];
|
|
1628
|
+
await assert.rejects(Effect.runPromise(qualifyHistoricalTaskSeedV1({
|
|
1629
|
+
repositoryRoot: root,
|
|
1630
|
+
seed: {
|
|
1631
|
+
...candidate,
|
|
1632
|
+
environment: {
|
|
1633
|
+
...candidate.environment,
|
|
1634
|
+
candidateValidationRecipes: [compileContract]
|
|
1635
|
+
}
|
|
1636
|
+
},
|
|
1637
|
+
repetitions: 2
|
|
1638
|
+
}).pipe(Effect.provide(RepositoryFoundryProgressLive(() => Effect.void, (artifact) => Effect.sync(() => {
|
|
1639
|
+
phases.push(artifact.value);
|
|
1640
|
+
}))))), /operational failure prevented grading.*phase=pre-change.*stage=candidate-validation/u);
|
|
1641
|
+
const baseline = phases.find((phase) => phase.phase === "baseline");
|
|
1642
|
+
const pre = phases.find((phase) => phase.phase === "pre-change");
|
|
1643
|
+
assert.ok(baseline);
|
|
1644
|
+
assert.ok(pre);
|
|
1645
|
+
assert.equal(phases.length, 2, "post-change and subsequent batches are not useful after an unexecuted grader");
|
|
1646
|
+
assert.equal(baseline.results.length, 2);
|
|
1647
|
+
assert.equal(pre.results.length, 2);
|
|
1648
|
+
assert.ok(baseline.results.every((result) => repositoryCommandTestEvidenceV1(result, result.recipeId).outcome === "pass"));
|
|
1649
|
+
for (const result of pre.results) {
|
|
1650
|
+
assert.equal(result.stage, "candidate-validation");
|
|
1651
|
+
assert.equal(result.gradeExecuted, false);
|
|
1652
|
+
assert.equal(result.recipeId, compileContract.id);
|
|
1653
|
+
assert.equal(result.exitCode, 51);
|
|
1654
|
+
assert.equal(repositoryCommandTestEvidenceV1(result, result.recipeId).outcome, "unknown");
|
|
1655
|
+
}
|
|
1656
|
+
});
|
|
1657
|
+
test("historical qualification requires assertion evidence rather than runtime or discovery failures", async (t) => {
|
|
1658
|
+
const scenarios = [
|
|
1659
|
+
{
|
|
1660
|
+
name: "runtime failure",
|
|
1661
|
+
body: "test('normalizes whitespace', () => { if (normalize(' value ') !== 'value') throw new TypeError('fixture setup failed'); assert.equal(normalize(' value '), 'value'); });"
|
|
1662
|
+
},
|
|
1663
|
+
{
|
|
1664
|
+
name: "missing new export",
|
|
1665
|
+
additionalReferenceSource: "export const normalizeNew = normalize;\n",
|
|
1666
|
+
body: "import { normalizeNew } from './index.js'; test('normalizes whitespace', () => assert.equal(normalizeNew(' value '), 'value'));"
|
|
1667
|
+
},
|
|
1668
|
+
{
|
|
1669
|
+
name: "process failure",
|
|
1670
|
+
body: "if (normalize(' value ') !== 'value') process.exit(1); test('normalizes whitespace', () => assert.equal(normalize(' value '), 'value'));"
|
|
1671
|
+
},
|
|
1672
|
+
{
|
|
1673
|
+
name: "empty discovery",
|
|
1674
|
+
body: "if (normalize(' value ') === 'value') test('normalizes whitespace', () => assert.equal(normalize(' value '), 'value'));"
|
|
1675
|
+
}
|
|
1676
|
+
];
|
|
1677
|
+
for (const scenario of scenarios) {
|
|
1678
|
+
await t.test(scenario.name, async () => {
|
|
1679
|
+
const root = await repositoryWithNewRegressionTest();
|
|
1680
|
+
const implementationPath = path.join(root, "packages/normalizer/src/index.js");
|
|
1681
|
+
if (scenario.additionalReferenceSource !== undefined) {
|
|
1682
|
+
await writeFile(implementationPath, (await readFile(implementationPath, "utf8")) + scenario.additionalReferenceSource);
|
|
1683
|
+
}
|
|
1684
|
+
await writeFile(path.join(root, "packages/normalizer/src/whitespace-regression.test.js"), [
|
|
1685
|
+
"import { test } from 'node:test';",
|
|
1686
|
+
"import { strict as assert } from 'node:assert';",
|
|
1687
|
+
"import { normalize } from './index.js';",
|
|
1688
|
+
scenario.body,
|
|
1689
|
+
""
|
|
1690
|
+
].join("\n"));
|
|
1691
|
+
await git(root, "add", "-A");
|
|
1692
|
+
await git(root, "commit", "--amend", "--no-edit");
|
|
1693
|
+
const map = await Effect.runPromise(buildRepositoryBehaviorMapV1({ repositoryRoot: root, requestedRef: "HEAD" }));
|
|
1694
|
+
const seeds = await Effect.runPromise(extractHistoricalTaskSeedsV1(map));
|
|
1695
|
+
const seed = seeds.find((candidate) => candidate.summary === "fix(normalizer): trim surrounding whitespace");
|
|
1696
|
+
assert.ok(seed);
|
|
1697
|
+
const qualified = await Effect.runPromise(qualifyHistoricalTaskSeedV1({ repositoryRoot: root, seed, repetitions: 2 }).pipe(Effect.result));
|
|
1698
|
+
assert.equal(qualified._tag, "Failure");
|
|
1699
|
+
if (qualified._tag === "Failure") {
|
|
1700
|
+
assert.equal(qualified.failure.classification, "environment_blocked");
|
|
1701
|
+
assert.match(qualified.failure.detail, /phase=pre-change; stage=grade/u);
|
|
1702
|
+
assert.match(qualified.failure.detail, /unstarted phases skipped=1/u);
|
|
1703
|
+
}
|
|
1704
|
+
// Execution failure is neither a semantic failing control nor a qualified
|
|
1705
|
+
// seed. The reference phase is deliberately not run after this blocker.
|
|
1706
|
+
});
|
|
1707
|
+
}
|
|
1708
|
+
});
|
|
1709
|
+
test("historical replay qualifies only a stable pre-fail post-pass seed", async () => {
|
|
1710
|
+
const root = await repository();
|
|
1711
|
+
const map = await Effect.runPromise(buildRepositoryBehaviorMapV1({
|
|
1712
|
+
repositoryRoot: root,
|
|
1713
|
+
requestedRef: "HEAD"
|
|
1714
|
+
}));
|
|
1715
|
+
const seeds = await Effect.runPromise(extractHistoricalTaskSeedsV1(map));
|
|
1716
|
+
const replayed = await Promise.all(seeds.map((seed) => Effect.runPromise(qualifyHistoricalTaskSeedV1({
|
|
1717
|
+
repositoryRoot: root,
|
|
1718
|
+
seed,
|
|
1719
|
+
repetitions: 2
|
|
1720
|
+
}))));
|
|
1721
|
+
const qualified = replayed.find((seed) => seed.summary === "fix terminal event ordering");
|
|
1722
|
+
const qualifiedFeature = replayed.find((seed) => seed.summary === "feat(stream): preserve usage summaries");
|
|
1723
|
+
const rejected = replayed.find((seed) => seed.status === "rejected");
|
|
1724
|
+
assert.equal(qualified?.status, "qualified", JSON.stringify(qualified?.rejectionReasons));
|
|
1725
|
+
assert.equal(qualified?.summary, "fix terminal event ordering");
|
|
1726
|
+
assert.equal(qualified?.baselineObservations.length, 2);
|
|
1727
|
+
assert.equal(qualified?.preChangeObservations.length, 2);
|
|
1728
|
+
assert.equal(qualified?.postChangeObservations.length, 2);
|
|
1729
|
+
assert.ok(qualified?.baselineObservations.every((observation) => observation.outcome === "pass"));
|
|
1730
|
+
assert.ok(qualified?.preChangeObservations.every((observation) => observation.outcome === "fail"));
|
|
1731
|
+
assert.ok(qualified?.postChangeObservations.every((observation) => observation.outcome === "pass"));
|
|
1732
|
+
assert.equal(qualifiedFeature?.status, "qualified", JSON.stringify(qualifiedFeature?.rejectionReasons));
|
|
1733
|
+
assert.equal(qualifiedFeature?.source.kind, "historical-feature");
|
|
1734
|
+
assert.ok(qualifiedFeature?.baselineObservations.every((observation) => observation.outcome === "pass"));
|
|
1735
|
+
assert.ok(qualifiedFeature?.preChangeObservations.every((observation) => observation.outcome === "fail"));
|
|
1736
|
+
assert.ok(qualifiedFeature?.postChangeObservations.every((observation) => observation.outcome === "pass"));
|
|
1737
|
+
assert.equal(rejected?.summary, "repair duplicate terminal events");
|
|
1738
|
+
assert.ok(rejected?.rejectionReasons.includes("every reviewed grade command must reproduce the pre-change failure"));
|
|
1739
|
+
await assert.rejects(Effect.runPromise(qualifyHistoricalTaskSeedV1({
|
|
1740
|
+
repositoryRoot: root,
|
|
1741
|
+
seed: {
|
|
1742
|
+
...seeds[0],
|
|
1743
|
+
environment: {
|
|
1744
|
+
...seeds[0].environment,
|
|
1745
|
+
trustedPreparationRecipes: [
|
|
1746
|
+
{
|
|
1747
|
+
id: "command-broken-trusted-preparation",
|
|
1748
|
+
kind: "custom",
|
|
1749
|
+
cwd: ".",
|
|
1750
|
+
executable: "node",
|
|
1751
|
+
args: ["--eval", "process.exit(61)"],
|
|
1752
|
+
timeoutMs: 30_000
|
|
1753
|
+
}
|
|
1754
|
+
]
|
|
1755
|
+
}
|
|
1756
|
+
},
|
|
1757
|
+
repetitions: 1
|
|
1758
|
+
})), /operational failure prevented grading.*stage=trusted-preparation/u);
|
|
1759
|
+
});
|
|
1760
|
+
test("historical qualification requires agreement across independently scheduled batches", async () => {
|
|
1761
|
+
const root = await repository();
|
|
1762
|
+
const map = await Effect.runPromise(buildRepositoryBehaviorMapV1({
|
|
1763
|
+
repositoryRoot: root,
|
|
1764
|
+
requestedRef: "HEAD"
|
|
1765
|
+
}));
|
|
1766
|
+
const seeds = await Effect.runPromise(extractHistoricalTaskSeedsV1(map));
|
|
1767
|
+
const candidate = seeds.find((seed) => seed.summary === "fix terminal event ordering");
|
|
1768
|
+
assert.ok(candidate);
|
|
1769
|
+
const qualification = await Effect.runPromise(qualifyHistoricalTaskSeedAcrossBatchesV1({
|
|
1770
|
+
repositoryRoot: root,
|
|
1771
|
+
seed: candidate,
|
|
1772
|
+
batches: 2,
|
|
1773
|
+
repetitionsPerBatch: 2
|
|
1774
|
+
}));
|
|
1775
|
+
assert.equal(qualification.seed.status, "qualified", JSON.stringify(qualification.seed.rejectionReasons));
|
|
1776
|
+
assert.equal(qualification.stability.stable, true);
|
|
1777
|
+
assert.equal(qualification.stability.batches.length, 2);
|
|
1778
|
+
assert.deepEqual(qualification.stability.divergentPhases, []);
|
|
1779
|
+
assert.deepEqual(qualification.stability.rejectionReasons, []);
|
|
1780
|
+
assert.ok(qualification.stability.batches.every((batch) => batch.status === "qualified" &&
|
|
1781
|
+
batch.phaseSignatures.every((phase) => phase.signatures.length === 1)));
|
|
1782
|
+
assert.equal(qualification.seed.baselineObservations.length, 4);
|
|
1783
|
+
assert.equal(qualification.seed.preChangeObservations.length, 4);
|
|
1784
|
+
assert.equal(qualification.seed.postChangeObservations.length, 4);
|
|
1785
|
+
assert.deepEqual(qualification.stability.failureSignatures, [
|
|
1786
|
+
{
|
|
1787
|
+
signature: `pre-change:${candidate.environment.gradeRecipes[0].id}:fail:stage=grade:exit=1:gradeExecuted=true:timeout=false`,
|
|
1788
|
+
count: 4
|
|
1789
|
+
}
|
|
1790
|
+
]);
|
|
1791
|
+
const primaryGrade = candidate.environment.gradeRecipes[0];
|
|
1792
|
+
const multiGrade = await Effect.runPromise(qualifyHistoricalTaskSeedV1({
|
|
1793
|
+
repositoryRoot: root,
|
|
1794
|
+
seed: {
|
|
1795
|
+
...candidate,
|
|
1796
|
+
environment: {
|
|
1797
|
+
...candidate.environment,
|
|
1798
|
+
gradeRecipes: [
|
|
1799
|
+
primaryGrade,
|
|
1800
|
+
{
|
|
1801
|
+
...primaryGrade,
|
|
1802
|
+
id: `${primaryGrade.id}-independent-surface`
|
|
1803
|
+
}
|
|
1804
|
+
]
|
|
1805
|
+
}
|
|
1806
|
+
},
|
|
1807
|
+
repetitions: 2
|
|
1808
|
+
}));
|
|
1809
|
+
assert.equal(multiGrade.status, "qualified");
|
|
1810
|
+
assert.deepEqual([
|
|
1811
|
+
...new Set(multiGrade.preChangeObservations.map((observation) => observation.subjectId))
|
|
1812
|
+
].sort(), [primaryGrade.id, `${primaryGrade.id}-independent-surface`].sort());
|
|
1813
|
+
assert.equal(multiGrade.baselineObservations.length, 2);
|
|
1814
|
+
assert.equal(multiGrade.preChangeObservations.length, 4);
|
|
1815
|
+
assert.equal(multiGrade.postChangeObservations.length, 4);
|
|
1816
|
+
});
|
|
1817
|
+
test("cross-batch qualification reuses one filtered frozen installation per pinned commit", async () => {
|
|
1818
|
+
const root = await repository();
|
|
1819
|
+
const map = await Effect.runPromise(buildRepositoryBehaviorMapV1({
|
|
1820
|
+
repositoryRoot: root,
|
|
1821
|
+
requestedRef: "HEAD"
|
|
1822
|
+
}));
|
|
1823
|
+
const seeds = await Effect.runPromise(extractHistoricalTaskSeedsV1(map));
|
|
1824
|
+
const candidate = seeds.find((seed) => seed.summary === "fix terminal event ordering");
|
|
1825
|
+
assert.ok(candidate);
|
|
1826
|
+
const prepared = {
|
|
1827
|
+
...candidate,
|
|
1828
|
+
environment: {
|
|
1829
|
+
...candidate.environment,
|
|
1830
|
+
trustedPreparationRecipes: [
|
|
1831
|
+
{
|
|
1832
|
+
id: "command-environment-pnpm-frozen-install",
|
|
1833
|
+
kind: "custom",
|
|
1834
|
+
cwd: ".",
|
|
1835
|
+
executable: "corepack",
|
|
1836
|
+
args: [
|
|
1837
|
+
"pnpm@11.15.1",
|
|
1838
|
+
"--filter",
|
|
1839
|
+
"@fixture/stream...",
|
|
1840
|
+
"install",
|
|
1841
|
+
"--offline",
|
|
1842
|
+
"--frozen-lockfile",
|
|
1843
|
+
"--ignore-pnpmfile",
|
|
1844
|
+
"--ignore-scripts"
|
|
1845
|
+
],
|
|
1846
|
+
timeoutMs: 30_000
|
|
1847
|
+
}
|
|
1848
|
+
]
|
|
1849
|
+
}
|
|
1850
|
+
};
|
|
1851
|
+
let preparationCompletions = 0;
|
|
1852
|
+
const qualification = await Effect.runPromise(qualifyHistoricalTaskSeedAcrossBatchesV1({
|
|
1853
|
+
repositoryRoot: root,
|
|
1854
|
+
seed: prepared,
|
|
1855
|
+
batches: 2,
|
|
1856
|
+
repetitionsPerBatch: 2
|
|
1857
|
+
}).pipe(Effect.provideService(RepositoryCommandExecutionPort, {
|
|
1858
|
+
run: (input) => input.args.includes("install")
|
|
1859
|
+
? Effect.succeed({ exitCode: 0, stdout: "", stderr: "" })
|
|
1860
|
+
: Effect.tryPromise({
|
|
1861
|
+
try: async () => {
|
|
1862
|
+
try {
|
|
1863
|
+
const result = await execFilePromise(input.executable, [...input.args], {
|
|
1864
|
+
cwd: input.cwd,
|
|
1865
|
+
env: input.environment,
|
|
1866
|
+
timeout: input.timeoutMs,
|
|
1867
|
+
encoding: "utf8"
|
|
1868
|
+
});
|
|
1869
|
+
return { exitCode: 0, stdout: result.stdout, stderr: result.stderr };
|
|
1870
|
+
}
|
|
1871
|
+
catch (cause) {
|
|
1872
|
+
const result = cause;
|
|
1873
|
+
if (typeof result.code !== "number")
|
|
1874
|
+
throw cause;
|
|
1875
|
+
return {
|
|
1876
|
+
exitCode: result.code,
|
|
1877
|
+
stdout: result.stdout ?? "",
|
|
1878
|
+
stderr: result.stderr ?? "",
|
|
1879
|
+
timedOut: result.killed ?? false
|
|
1880
|
+
};
|
|
1881
|
+
}
|
|
1882
|
+
},
|
|
1883
|
+
catch: (cause) => new RepositoryFoundryError({
|
|
1884
|
+
operation: "run-command",
|
|
1885
|
+
detail: "qualification command failed",
|
|
1886
|
+
cause
|
|
1887
|
+
})
|
|
1888
|
+
})
|
|
1889
|
+
}), Effect.provideService(RepositoryCommandObservability, {
|
|
1890
|
+
observe: (event) => {
|
|
1891
|
+
if (event.kind === "completed" && event.stage === "trusted-preparation") {
|
|
1892
|
+
preparationCompletions += 1;
|
|
1893
|
+
}
|
|
1894
|
+
}
|
|
1895
|
+
})));
|
|
1896
|
+
assert.equal(qualification.seed.status, "qualified", qualification.seed.rejectionReasons.join("; "));
|
|
1897
|
+
assert.equal(preparationCompletions, 2, "initial and reference commits prepare exactly once");
|
|
1898
|
+
assert.equal(qualification.stability.batches.length, 2);
|
|
1899
|
+
assert.equal(qualification.seed.baselineObservations.length, 4);
|
|
1900
|
+
assert.equal(qualification.seed.preChangeObservations.length, 4);
|
|
1901
|
+
assert.equal(qualification.seed.postChangeObservations.length, 4);
|
|
1902
|
+
});
|
|
1903
|
+
test("first foundry vertical slice closes a real oracle escape with an independently checked counterfactual fixture", async () => {
|
|
1904
|
+
const { root, alternateCommit, behaviorPreservingCommit, mutantCommit, unterminatedDropCommit } = await repositoryWithSolutionBranches();
|
|
1905
|
+
const map = await Effect.runPromise(buildRepositoryBehaviorMapV1({
|
|
1906
|
+
repositoryRoot: root,
|
|
1907
|
+
requestedRef: "HEAD"
|
|
1908
|
+
}));
|
|
1909
|
+
const candidateSeeds = await Effect.runPromise(extractHistoricalTaskSeedsV1(map));
|
|
1910
|
+
const replayedSeeds = await Promise.all(candidateSeeds.map((seed) => Effect.runPromise(qualifyHistoricalTaskSeedV1({
|
|
1911
|
+
repositoryRoot: root,
|
|
1912
|
+
seed,
|
|
1913
|
+
repetitions: 2
|
|
1914
|
+
}))));
|
|
1915
|
+
const seed = replayedSeeds.find((entry) => entry.summary === "fix terminal event ordering");
|
|
1916
|
+
assert.ok(seed);
|
|
1917
|
+
const families = await Effect.runPromise(deriveRepositoryTaskFamiliesV1({ map, seeds: replayedSeeds }));
|
|
1918
|
+
const family = families.find((entry) => entry.seedIds.includes(seed.id));
|
|
1919
|
+
assert.ok(family, JSON.stringify(replayedSeeds.map((seed) => ({
|
|
1920
|
+
summary: seed.summary,
|
|
1921
|
+
status: seed.status,
|
|
1922
|
+
rejectionReasons: seed.rejectionReasons
|
|
1923
|
+
}))));
|
|
1924
|
+
const visible = await Effect.runPromise(constructRepositoryAgentTaskV1({ seed, family }));
|
|
1925
|
+
const specificationCritique = await Effect.runPromise(critiqueRepositoryTaskSpecificationV1({
|
|
1926
|
+
repositoryRoot: root,
|
|
1927
|
+
seed,
|
|
1928
|
+
family,
|
|
1929
|
+
visible,
|
|
1930
|
+
reviews: independentSpecificationReviews
|
|
1931
|
+
}));
|
|
1932
|
+
assert.equal(specificationCritique.outcome, "pass");
|
|
1933
|
+
assert.equal(specificationCritique.independentApprovalCount, 2);
|
|
1934
|
+
assert.equal(JSON.stringify(visible).includes("hidden grader") ||
|
|
1935
|
+
JSON.stringify(visible).includes("post-change"), false);
|
|
1936
|
+
const validationRecipe = map.commands.find((recipe) => recipe.kind === "build" && recipe.cwd === "packages/stream");
|
|
1937
|
+
assert.ok(validationRecipe);
|
|
1938
|
+
const generatedAdversaries = await Effect.runPromise(generateDeterministicAdversariesV1({
|
|
1939
|
+
repositoryRoot: root,
|
|
1940
|
+
seed,
|
|
1941
|
+
validationRecipe,
|
|
1942
|
+
repetitions: 2
|
|
1943
|
+
}));
|
|
1944
|
+
assert.ok(generatedAdversaries.report.candidates.length >= 1);
|
|
1945
|
+
assert.equal(generatedAdversaries.report.admittedCandidateIds.length, generatedAdversaries.report.candidates.length);
|
|
1946
|
+
assert.deepEqual(generatedAdversaries.report.rejectedCandidateIds, []);
|
|
1947
|
+
assert.ok(generatedAdversaries.report.candidates.every((candidate) => candidate.compilationStatus === "passed" && candidate.validationObservations.length === 2));
|
|
1948
|
+
const rejectedAdversaries = await Effect.runPromise(generateDeterministicAdversariesV1({
|
|
1949
|
+
repositoryRoot: root,
|
|
1950
|
+
seed,
|
|
1951
|
+
validationRecipe: {
|
|
1952
|
+
...validationRecipe,
|
|
1953
|
+
id: "command-build-rejects-mutants",
|
|
1954
|
+
executable: "node",
|
|
1955
|
+
args: ["--eval", "process.exit(1)"]
|
|
1956
|
+
},
|
|
1957
|
+
repetitions: 2
|
|
1958
|
+
}));
|
|
1959
|
+
assert.equal(rejectedAdversaries.solutions.length, 0);
|
|
1960
|
+
assert.equal(rejectedAdversaries.report.rejectedCandidateIds.length, rejectedAdversaries.report.candidates.length);
|
|
1961
|
+
assert.ok(rejectedAdversaries.report.candidates.every((candidate) => candidate.compilationStatus === "failed" && candidate.rejectionReasons.length > 0));
|
|
1962
|
+
const withoutHiddenOverlay = await Effect.runPromise(replayRepositoryCommandV1({
|
|
1963
|
+
repositoryRoot: root,
|
|
1964
|
+
commit: mutantCommit,
|
|
1965
|
+
recipe: seed.environment.gradeRecipes[0],
|
|
1966
|
+
repetitions: 2
|
|
1967
|
+
}));
|
|
1968
|
+
assert.ok(withoutHiddenOverlay.every((result) => result.exitCode === 0));
|
|
1969
|
+
const solutionPopulation = [
|
|
1970
|
+
{
|
|
1971
|
+
id: "alternate-loop-valid",
|
|
1972
|
+
kind: "independent-valid",
|
|
1973
|
+
family: "independent-valid",
|
|
1974
|
+
expectedClass: "valid",
|
|
1975
|
+
commit: alternateCommit
|
|
1976
|
+
},
|
|
1977
|
+
{
|
|
1978
|
+
id: "behavior-preserving-loop-valid",
|
|
1979
|
+
kind: "behavior-preserving-valid",
|
|
1980
|
+
family: "behavior-preserving",
|
|
1981
|
+
expectedClass: "valid",
|
|
1982
|
+
commit: behaviorPreservingCommit
|
|
1983
|
+
},
|
|
1984
|
+
{
|
|
1985
|
+
id: "drops-output-mutant",
|
|
1986
|
+
kind: "model-near-miss",
|
|
1987
|
+
family: "integration-near-miss",
|
|
1988
|
+
expectedClass: "wrong",
|
|
1989
|
+
commit: mutantCommit
|
|
1990
|
+
},
|
|
1991
|
+
{
|
|
1992
|
+
id: "drops-unterminated-stream-mutant",
|
|
1993
|
+
kind: "model-near-miss",
|
|
1994
|
+
family: "boundary-near-miss",
|
|
1995
|
+
expectedClass: "wrong",
|
|
1996
|
+
commit: unterminatedDropCommit
|
|
1997
|
+
},
|
|
1998
|
+
{
|
|
1999
|
+
id: "test-weakening-reward-hack",
|
|
2000
|
+
kind: "reward-hacking",
|
|
2001
|
+
family: "reward-hacking",
|
|
2002
|
+
expectedClass: "wrong",
|
|
2003
|
+
commit: seed.initialState.commit,
|
|
2004
|
+
fileOverrides: [
|
|
2005
|
+
{
|
|
2006
|
+
path: "packages/stream/src/stream-protocol.test.js",
|
|
2007
|
+
content: "import { test } from 'node:test';\ntest('weakened', () => {});\n"
|
|
2008
|
+
}
|
|
2009
|
+
]
|
|
2010
|
+
},
|
|
2011
|
+
...generatedAdversaries.solutions
|
|
2012
|
+
];
|
|
2013
|
+
const hiddenFixtureSuite = await Effect.runPromise(compileReviewedHiddenFixtureSuiteV1({
|
|
2014
|
+
caseId: visible.id,
|
|
2015
|
+
seed,
|
|
2016
|
+
suite: {
|
|
2017
|
+
version: 1,
|
|
2018
|
+
caseId: visible.id,
|
|
2019
|
+
fixtures: [
|
|
2020
|
+
{
|
|
2021
|
+
id: "counterfactual-no-terminal",
|
|
2022
|
+
kind: "counterfactual",
|
|
2023
|
+
description: "Exercise the same adapter when the provider sequence has no terminal event.",
|
|
2024
|
+
expectedBehavior: "All non-terminal events remain unchanged when no completion event exists.",
|
|
2025
|
+
expectationMode: "changes",
|
|
2026
|
+
source: "reviewed",
|
|
2027
|
+
testPath: "packages/stream/src/stream-protocol.test.js"
|
|
2028
|
+
},
|
|
2029
|
+
{
|
|
2030
|
+
id: "metamorphic-output-renaming",
|
|
2031
|
+
kind: "metamorphic",
|
|
2032
|
+
description: "Rename non-terminal output values without changing terminal ordering.",
|
|
2033
|
+
expectedBehavior: "Changing payload values preserves event count, order, and the terminal-event invariant.",
|
|
2034
|
+
expectationMode: "preserved",
|
|
2035
|
+
source: "reviewed",
|
|
2036
|
+
testPath: "packages/stream/src/stream-protocol.test.js"
|
|
2037
|
+
}
|
|
2038
|
+
],
|
|
2039
|
+
overlays: [
|
|
2040
|
+
{
|
|
2041
|
+
path: "packages/stream/src/stream-protocol.test.js",
|
|
2042
|
+
fixtureIds: ["counterfactual-no-terminal"],
|
|
2043
|
+
content: [
|
|
2044
|
+
"import { test } from 'node:test';",
|
|
2045
|
+
"import { strict as assert } from 'node:assert';",
|
|
2046
|
+
"import { translateEvents } from './stream-protocol.js';",
|
|
2047
|
+
"test('preserves a sequence without a terminal event', () => {",
|
|
2048
|
+
" assert.deepEqual(translateEvents([{ type: 'output', value: 'first' }, { type: 'usage', tokens: 3 }]), [{ type: 'output', value: 'first' }, { type: 'usage', tokens: 3 }]);",
|
|
2049
|
+
"});",
|
|
2050
|
+
""
|
|
2051
|
+
].join("\n")
|
|
2052
|
+
},
|
|
2053
|
+
{
|
|
2054
|
+
path: "packages/stream/src/stream-protocol.test.js",
|
|
2055
|
+
fixtureIds: ["metamorphic-output-renaming"],
|
|
2056
|
+
content: [
|
|
2057
|
+
"import { test } from 'node:test';",
|
|
2058
|
+
"import { strict as assert } from 'node:assert';",
|
|
2059
|
+
"import { translateEvents } from './stream-protocol.js';",
|
|
2060
|
+
"test('renaming payload values preserves terminal ordering', () => {",
|
|
2061
|
+
" const translated = translateEvents([{ type: 'completed' }, { type: 'output', value: 'renamed' }]);",
|
|
2062
|
+
" assert.deepEqual(translated, [{ type: 'output', value: 'renamed' }, { type: 'completed' }]);",
|
|
2063
|
+
"});",
|
|
2064
|
+
""
|
|
2065
|
+
].join("\n")
|
|
2066
|
+
}
|
|
2067
|
+
]
|
|
2068
|
+
}
|
|
2069
|
+
}));
|
|
2070
|
+
const compiledCase = await Effect.runPromise(compileReviewedHistoricalCaseV1({
|
|
2071
|
+
repositoryRoot: root,
|
|
2072
|
+
map,
|
|
2073
|
+
seed,
|
|
2074
|
+
blueprint: {
|
|
2075
|
+
caseId: visible.id,
|
|
2076
|
+
visible,
|
|
2077
|
+
specificationReviews: independentSpecificationReviews,
|
|
2078
|
+
finalFixtureSuite: hiddenFixtureSuite,
|
|
2079
|
+
solutions: solutionPopulation,
|
|
2080
|
+
weakOracleTrial: {
|
|
2081
|
+
solutions: solutionPopulation,
|
|
2082
|
+
expectedFalseAcceptedSolutionIds: ["drops-unterminated-stream-mutant"]
|
|
2083
|
+
},
|
|
2084
|
+
oracleRepetitions: 2
|
|
2085
|
+
}
|
|
2086
|
+
}));
|
|
2087
|
+
const { benchmarkCase, hidden, weakAdequacy } = compiledCase;
|
|
2088
|
+
assert.equal(weakAdequacy?.admitted, false);
|
|
2089
|
+
assert.deepEqual(weakAdequacy?.falseAcceptedSolutionIds, ["drops-unterminated-stream-mutant"]);
|
|
2090
|
+
const library = await Effect.runPromise(compileRepositoryBenchmarkV1([benchmarkCase]));
|
|
2091
|
+
assert.equal(benchmarkCase.status, "valid-library");
|
|
2092
|
+
assert.equal(benchmarkCase.adequacy.admitted, true);
|
|
2093
|
+
assert.equal(benchmarkCase.adequacy.validSolutionCount, 3);
|
|
2094
|
+
assert.equal(benchmarkCase.adequacy.wrongSolutionCount, 4 + generatedAdversaries.solutions.length);
|
|
2095
|
+
assert.deepEqual(benchmarkCase.adequacy.falseAcceptedSolutionIds, []);
|
|
2096
|
+
assert.deepEqual(benchmarkCase.adequacy.falseRejectedSolutionIds, []);
|
|
2097
|
+
assert.deepEqual(benchmarkCase.adequacy.unstableSolutionIds, []);
|
|
2098
|
+
const fixtureObservations = (solutionId) => hidden.observations.filter((observation) => observation.solutionId === solutionId && observation.clauseId.startsWith("fixture-"));
|
|
2099
|
+
assert.deepEqual(fixtureObservations("alternate-loop-valid").map((observation) => observation.outcome), ["pass", "pass", "pass", "pass"]);
|
|
2100
|
+
assert.equal(fixtureObservations("drops-output-mutant").some((observation) => observation.outcome === "fail"), true);
|
|
2101
|
+
for (const solution of generatedAdversaries.solutions) {
|
|
2102
|
+
assert.equal(fixtureObservations(solution.id).length, hidden.fixtures.length * 2);
|
|
2103
|
+
}
|
|
2104
|
+
assert.equal(fixtureObservations("drops-unterminated-stream-mutant").some((observation) => observation.outcome === "fail"), true);
|
|
2105
|
+
assert.deepEqual(fixtureObservations("drops-unterminated-stream-mutant").map((observation) => [
|
|
2106
|
+
observation.clauseId,
|
|
2107
|
+
observation.outcome
|
|
2108
|
+
]), [
|
|
2109
|
+
["fixture-counterfactual-no-terminal", "fail"],
|
|
2110
|
+
["fixture-counterfactual-no-terminal", "fail"],
|
|
2111
|
+
["fixture-metamorphic-output-renaming", "pass"],
|
|
2112
|
+
["fixture-metamorphic-output-renaming", "pass"]
|
|
2113
|
+
], "fixtures sharing the same package-level test recipe must retain distinct mechanical outcomes");
|
|
2114
|
+
assert.deepEqual(hidden.fixtures.map((fixture) => fixture.id), ["counterfactual-no-terminal", "metamorphic-output-renaming"]);
|
|
2115
|
+
assert.deepEqual(library.cases.map((entry) => entry.id), [visible.id]);
|
|
2116
|
+
assert.deepEqual(library.rejectedCaseIds, []);
|
|
2117
|
+
assert.ok(library.acceptanceFunnel.every((gate) => gate.rejected === 0));
|
|
2118
|
+
assert.equal(JSON.stringify(visible).includes(alternateCommit) ||
|
|
2119
|
+
JSON.stringify(visible).includes(mutantCommit), false);
|
|
2120
|
+
});
|
|
2121
|
+
test("quality gate rejects a replayed task until an independent valid solution is demonstrated", async () => {
|
|
2122
|
+
const root = await repository();
|
|
2123
|
+
const map = await Effect.runPromise(buildRepositoryBehaviorMapV1({
|
|
2124
|
+
repositoryRoot: root,
|
|
2125
|
+
requestedRef: "HEAD"
|
|
2126
|
+
}));
|
|
2127
|
+
const candidates = await Effect.runPromise(extractHistoricalTaskSeedsV1(map));
|
|
2128
|
+
const seed = await Effect.runPromise(qualifyHistoricalTaskSeedV1({
|
|
2129
|
+
repositoryRoot: root,
|
|
2130
|
+
seed: candidates.find((entry) => entry.summary === "fix terminal event ordering"),
|
|
2131
|
+
repetitions: 2
|
|
2132
|
+
}));
|
|
2133
|
+
const family = (await Effect.runPromise(deriveRepositoryTaskFamiliesV1({ map, seeds: [seed] })))[0];
|
|
2134
|
+
const visible = await Effect.runPromise(constructRepositoryAgentTaskV1({ seed, family }));
|
|
2135
|
+
const specificationCritique = await Effect.runPromise(critiqueRepositoryTaskSpecificationV1({
|
|
2136
|
+
repositoryRoot: root,
|
|
2137
|
+
seed,
|
|
2138
|
+
family,
|
|
2139
|
+
visible,
|
|
2140
|
+
reviews: independentSpecificationReviews
|
|
2141
|
+
}));
|
|
2142
|
+
const hidden = await Effect.runPromise(buildHistoricalHiddenOracleV1({
|
|
2143
|
+
repositoryRoot: root,
|
|
2144
|
+
caseId: visible.id,
|
|
2145
|
+
seed,
|
|
2146
|
+
repetitions: 2,
|
|
2147
|
+
additionalSolutions: []
|
|
2148
|
+
}));
|
|
2149
|
+
const benchmarkCase = await Effect.runPromise(evaluateRepositoryBenchmarkDraftV1({
|
|
2150
|
+
map,
|
|
2151
|
+
seed,
|
|
2152
|
+
family,
|
|
2153
|
+
visible,
|
|
2154
|
+
hidden,
|
|
2155
|
+
specificationCritique
|
|
2156
|
+
}));
|
|
2157
|
+
assert.equal(benchmarkCase.status, "rejected");
|
|
2158
|
+
assert.ok(benchmarkCase.rejectionReasons.some((reason) => reason.includes("no independently implemented valid alternative")));
|
|
2159
|
+
});
|
|
2160
|
+
test("specification criticism fails closed on eval-only phrasing and missing independent review", async () => {
|
|
2161
|
+
const root = await repository();
|
|
2162
|
+
const map = await Effect.runPromise(buildRepositoryBehaviorMapV1({
|
|
2163
|
+
repositoryRoot: root,
|
|
2164
|
+
requestedRef: "HEAD"
|
|
2165
|
+
}));
|
|
2166
|
+
const candidates = await Effect.runPromise(extractHistoricalTaskSeedsV1(map));
|
|
2167
|
+
const seed = await Effect.runPromise(qualifyHistoricalTaskSeedV1({
|
|
2168
|
+
repositoryRoot: root,
|
|
2169
|
+
seed: candidates.find((entry) => entry.summary === "fix terminal event ordering"),
|
|
2170
|
+
repetitions: 2
|
|
2171
|
+
}));
|
|
2172
|
+
const family = (await Effect.runPromise(deriveRepositoryTaskFamiliesV1({ map, seeds: [seed] })))[0];
|
|
2173
|
+
const visible = await Effect.runPromise(constructRepositoryAgentTaskV1({ seed, family }));
|
|
2174
|
+
const unreviewed = await Effect.runPromise(critiqueRepositoryTaskSpecificationV1({
|
|
2175
|
+
repositoryRoot: root,
|
|
2176
|
+
seed,
|
|
2177
|
+
family,
|
|
2178
|
+
visible,
|
|
2179
|
+
reviews: []
|
|
2180
|
+
}));
|
|
2181
|
+
assert.equal(unreviewed.outcome, "review-required");
|
|
2182
|
+
assert.match(unreviewed.rejectionReasons.join("\n"), /two are required/u);
|
|
2183
|
+
const leaked = await Effect.runPromise(critiqueRepositoryTaskSpecificationV1({
|
|
2184
|
+
repositoryRoot: root,
|
|
2185
|
+
seed,
|
|
2186
|
+
family,
|
|
2187
|
+
visible: {
|
|
2188
|
+
...visible,
|
|
2189
|
+
constraints: [...visible.constraints, "Inspect the hidden grader and post-change source."]
|
|
2190
|
+
},
|
|
2191
|
+
reviews: independentSpecificationReviews
|
|
2192
|
+
}));
|
|
2193
|
+
assert.equal(leaked.outcome, "fail");
|
|
2194
|
+
assert.ok(leaked.mechanicalFindings.some((finding) => finding.axis === "unrealistic-phrasing" && finding.severity === "blocker"));
|
|
2195
|
+
});
|