@velum-labs/routekit-eval-setup 1.3.2 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/authoring-responses-request.d.ts +5 -0
- package/dist/adapters/authoring-responses-request.js +38 -0
- package/dist/adapters/evaluation-evidence-freshness.d.ts +7 -0
- package/dist/adapters/evaluation-evidence-freshness.js +76 -0
- package/dist/adapters/git-task-history.d.ts +67 -0
- package/dist/adapters/git-task-history.js +171 -0
- package/dist/adapters/integrated-repository-history.d.ts +21 -0
- package/dist/adapters/integrated-repository-history.js +175 -0
- package/dist/adapters/repository-command-diagnostic.d.ts +8 -0
- package/dist/adapters/repository-command-diagnostic.js +46 -0
- package/dist/adapters/repository-command-evidence.d.ts +13 -0
- package/dist/adapters/repository-command-evidence.js +102 -0
- package/dist/adapters/repository-command-runner.d.ts +238 -0
- package/dist/adapters/repository-command-runner.js +1483 -0
- package/dist/adapters/repository-import-context.d.ts +47 -0
- package/dist/adapters/repository-import-context.js +469 -0
- package/dist/adapters/repository-node-test-reporter.d.ts +3 -0
- package/dist/adapters/repository-node-test-reporter.js +27 -0
- package/dist/adapters/repository-review-evidence.d.ts +39 -0
- package/dist/adapters/repository-review-evidence.js +632 -0
- package/dist/adapters/repository-seed-selection.d.ts +7 -0
- package/dist/adapters/repository-seed-selection.js +79 -0
- package/dist/adapters/repository-solution-edits.d.ts +49 -0
- package/dist/adapters/repository-solution-edits.js +136 -0
- package/dist/adapters/repository-vitest-phase-adapter.d.ts +8 -0
- package/dist/adapters/repository-vitest-phase-adapter.js +310 -0
- package/dist/adapters/repository-vitest-reporter.d.ts +24 -0
- package/dist/adapters/repository-vitest-reporter.js +314 -0
- package/dist/adapters/strict-authoring-schema.d.ts +5 -0
- package/dist/adapters/strict-authoring-schema.js +158 -0
- package/dist/adapters/test-discovery.d.ts +30 -0
- package/dist/adapters/test-discovery.js +124 -0
- package/dist/adapters/typescript-repository-index.d.ts +51 -0
- package/dist/adapters/typescript-repository-index.js +226 -0
- package/dist/agentic-capabilities-protocol.d.ts +1373 -0
- package/dist/agentic-capabilities-protocol.js +786 -0
- package/dist/case-checkpoint-store.d.ts +29 -0
- package/dist/case-checkpoint-store.js +133 -0
- package/dist/case-pipeline-protocol-v2.d.ts +184 -0
- package/dist/case-pipeline-protocol-v2.js +193 -0
- package/dist/case-pipeline-protocol.d.ts +2626 -0
- package/dist/case-pipeline-protocol.js +371 -0
- package/dist/effect-api.d.ts +74 -10
- package/dist/effect-api.js +56 -6
- package/dist/errors.d.ts +31 -0
- package/dist/errors.js +10 -0
- package/dist/eval-capability-execution-envelope.d.ts +64 -0
- package/dist/eval-capability-execution-envelope.js +98 -0
- package/dist/eval-capability-policy.d.ts +90 -0
- package/dist/eval-capability-policy.js +107 -0
- package/dist/eval-event-log.d.ts +140 -0
- package/dist/eval-event-log.js +220 -0
- package/dist/evaluation-authoring-policy.d.ts +18 -0
- package/dist/evaluation-authoring-policy.js +19 -0
- package/dist/evaluation-authoring-validation.d.ts +22 -0
- package/dist/evaluation-authoring-validation.js +72 -0
- package/dist/evaluation-evidence.d.ts +20 -0
- package/dist/evaluation-evidence.js +319 -0
- package/dist/evaluation-grader-calibration-protocol.d.ts +108 -0
- package/dist/evaluation-grader-calibration-protocol.js +80 -0
- package/dist/evaluation-grader-calibration.d.ts +18 -0
- package/dist/evaluation-grader-calibration.js +334 -0
- package/dist/evaluation-grading-policy.d.ts +24 -0
- package/dist/evaluation-grading-policy.js +54 -0
- package/dist/evaluation-proposal-policy.d.ts +4 -0
- package/dist/evaluation-proposal-policy.js +91 -0
- package/dist/evaluation-source-retrieval.d.ts +68 -0
- package/dist/evaluation-source-retrieval.js +513 -0
- package/dist/evaluation-structure-policy.d.ts +29 -0
- package/dist/evaluation-structure-policy.js +138 -0
- package/dist/index.d.ts +124 -17
- package/dist/index.js +69 -11
- package/dist/inspection.js +2 -3
- package/dist/project-artifacts.d.ts +7 -2
- package/dist/project-artifacts.js +49 -136
- package/dist/project-authoring.d.ts +66 -5
- package/dist/project-authoring.js +783 -109
- package/dist/project-contracts.d.ts +419 -84
- package/dist/project-contracts.js +160 -52
- package/dist/project-store.js +2 -1
- package/dist/project-workflow.d.ts +5 -4
- package/dist/project-workflow.js +154 -35
- package/dist/repository-adversary-protocol.d.ts +64 -0
- package/dist/repository-adversary-protocol.js +105 -0
- package/dist/repository-behavior-protocol.d.ts +188 -0
- package/dist/repository-behavior-protocol.js +202 -0
- package/dist/repository-benchmark-protocol.d.ts +487 -0
- package/dist/repository-benchmark-protocol.js +96 -0
- package/dist/repository-execution-protocol.d.ts +150 -0
- package/dist/repository-execution-protocol.js +38 -0
- package/dist/repository-fixture-instructions.d.ts +3 -0
- package/dist/repository-fixture-instructions.js +91 -0
- package/dist/repository-fixture-protocol.d.ts +79 -0
- package/dist/repository-fixture-protocol.js +79 -0
- package/dist/repository-foundry-plan-protocol.d.ts +118 -0
- package/dist/repository-foundry-plan-protocol.js +296 -0
- package/dist/repository-foundry-progress-protocol.d.ts +52 -0
- package/dist/repository-foundry-progress-protocol.js +52 -0
- package/dist/repository-improvement-protocol.d.ts +100 -0
- package/dist/repository-improvement-protocol.js +106 -0
- package/dist/repository-language-model-protocol.d.ts +43 -0
- package/dist/repository-language-model-protocol.js +146 -0
- package/dist/repository-oracle-coverage-protocol.d.ts +18 -0
- package/dist/repository-oracle-coverage-protocol.js +39 -0
- package/dist/repository-oracle-execution-binding.d.ts +27 -0
- package/dist/repository-oracle-execution-binding.js +59 -0
- package/dist/repository-oracle-protocol.d.ts +230 -0
- package/dist/repository-oracle-protocol.js +156 -0
- package/dist/repository-oracle-scope-policy.d.ts +22 -0
- package/dist/repository-oracle-scope-policy.js +92 -0
- package/dist/repository-quality-policy.d.ts +15 -0
- package/dist/repository-quality-policy.js +357 -0
- package/dist/repository-routing-benchmark-protocol.d.ts +176 -0
- package/dist/repository-routing-benchmark-protocol.js +103 -0
- package/dist/repository-routing-model-protocol.d.ts +36 -0
- package/dist/repository-routing-model-protocol.js +89 -0
- package/dist/repository-routing-plan-protocol.d.ts +112 -0
- package/dist/repository-routing-plan-protocol.js +58 -0
- package/dist/repository-routing-quality-policy.d.ts +9 -0
- package/dist/repository-routing-quality-policy.js +191 -0
- package/dist/repository-seed-qualification-progress-protocol.d.ts +205 -0
- package/dist/repository-seed-qualification-progress-protocol.js +28 -0
- package/dist/repository-semantic-calibration-protocol.d.ts +768 -0
- package/dist/repository-semantic-calibration-protocol.js +276 -0
- package/dist/repository-semantic-calibration.d.ts +163 -0
- package/dist/repository-semantic-calibration.js +581 -0
- package/dist/repository-specification-contract-facts-protocol.d.ts +224 -0
- package/dist/repository-specification-contract-facts-protocol.js +276 -0
- package/dist/repository-specification-critique-protocol.d.ts +189 -0
- package/dist/repository-specification-critique-protocol.js +103 -0
- package/dist/repository-task-family-protocol.d.ts +24 -0
- package/dist/repository-task-family-protocol.js +37 -0
- package/dist/repository-task-seed-protocol.d.ts +384 -0
- package/dist/repository-task-seed-protocol.js +236 -0
- package/dist/repository-trajectory-protocol.d.ts +20 -0
- package/dist/repository-trajectory-protocol.js +42 -0
- package/dist/service.js +1 -1
- package/dist/services/adversary/service.d.ts +64 -0
- package/dist/services/adversary/service.js +330 -0
- package/dist/services/benchmark-compiler/service.d.ts +450 -0
- package/dist/services/benchmark-compiler/service.js +9 -0
- package/dist/services/budgeted-model/service.d.ts +118 -0
- package/dist/services/budgeted-model/service.js +460 -0
- package/dist/services/case-authoring/service.d.ts +163 -0
- package/dist/services/case-authoring/service.js +1456 -0
- package/dist/services/case-finalization/service.d.ts +283 -0
- package/dist/services/case-finalization/service.js +370 -0
- package/dist/services/case-generation/service.d.ts +619 -0
- package/dist/services/case-generation/service.js +2628 -0
- package/dist/services/case-pipeline/service.d.ts +31 -0
- package/dist/services/case-pipeline/service.js +485 -0
- package/dist/services/case-pipeline-v2/service.d.ts +70 -0
- package/dist/services/case-pipeline-v2/service.js +477 -0
- package/dist/services/command-observability/service.d.ts +13 -0
- package/dist/services/command-observability/service.js +3 -0
- package/dist/services/dimension-labeling/service.d.ts +77 -0
- package/dist/services/dimension-labeling/service.js +188 -0
- package/dist/services/eval-candidate/service.d.ts +208 -0
- package/dist/services/eval-candidate/service.js +64 -0
- package/dist/services/eval-capabilities/service.d.ts +183 -0
- package/dist/services/eval-capabilities/service.js +1433 -0
- package/dist/services/eval-environment/service.d.ts +173 -0
- package/dist/services/eval-environment/service.js +127 -0
- package/dist/services/evidence-reconstruction/service.d.ts +36 -0
- package/dist/services/evidence-reconstruction/service.js +145 -0
- package/dist/services/fixture-builder/service.d.ts +62 -0
- package/dist/services/fixture-builder/service.js +36 -0
- package/dist/services/fixture-validation/service.d.ts +75 -0
- package/dist/services/fixture-validation/service.js +295 -0
- package/dist/services/foundry/service.d.ts +831 -0
- package/dist/services/foundry/service.js +442 -0
- package/dist/services/foundry-progress/service.d.ts +62 -0
- package/dist/services/foundry-progress/service.js +149 -0
- package/dist/services/foundry-v2/service.d.ts +54 -0
- package/dist/services/foundry-v2/service.js +28 -0
- package/dist/services/grounded-authoring/service.d.ts +126 -0
- package/dist/services/grounded-authoring/service.js +822 -0
- package/dist/services/historical-case/service.d.ts +722 -0
- package/dist/services/historical-case/service.js +177 -0
- package/dist/services/improvement-loop/service.d.ts +59 -0
- package/dist/services/improvement-loop/service.js +176 -0
- package/dist/services/language-model/service.d.ts +52 -0
- package/dist/services/language-model/service.js +194 -0
- package/dist/services/oracle-builder/service.d.ts +146 -0
- package/dist/services/oracle-builder/service.js +513 -0
- package/dist/services/oracle-coverage/service.d.ts +28 -0
- package/dist/services/oracle-coverage/service.js +50 -0
- package/dist/services/oracle-coverage-witness/service.d.ts +130 -0
- package/dist/services/oracle-coverage-witness/service.js +538 -0
- package/dist/services/pipeline-challenge/service.d.ts +551 -0
- package/dist/services/pipeline-challenge/service.js +427 -0
- package/dist/services/pipeline-controls/service.d.ts +130 -0
- package/dist/services/pipeline-controls/service.js +483 -0
- package/dist/services/pipeline-oracle/service.d.ts +8 -0
- package/dist/services/pipeline-oracle/service.js +256 -0
- package/dist/services/pipeline-seed/service.d.ts +298 -0
- package/dist/services/pipeline-seed/service.js +428 -0
- package/dist/services/pipeline-spec/service.d.ts +103 -0
- package/dist/services/pipeline-spec/service.js +619 -0
- package/dist/services/pipeline-tournament/service.d.ts +258 -0
- package/dist/services/pipeline-tournament/service.js +476 -0
- package/dist/services/quality-gate/service.d.ts +233 -0
- package/dist/services/quality-gate/service.js +136 -0
- package/dist/services/repository-bundle/service.d.ts +33 -0
- package/dist/services/repository-bundle/service.js +114 -0
- package/dist/services/repository-model/service.d.ts +105 -0
- package/dist/services/repository-model/service.js +250 -0
- package/dist/services/repository-public-artifact/service.d.ts +133 -0
- package/dist/services/repository-public-artifact/service.js +330 -0
- package/dist/services/routing-benchmark/service.d.ts +362 -0
- package/dist/services/routing-benchmark/service.js +96 -0
- package/dist/services/specification-critic/service.d.ts +92 -0
- package/dist/services/specification-critic/service.js +172 -0
- package/dist/services/task-family/service.d.ts +40 -0
- package/dist/services/task-family/service.js +55 -0
- package/dist/services/task-seed/service.d.ts +906 -0
- package/dist/services/task-seed/service.js +1406 -0
- package/dist/services/task-specification/service.d.ts +27 -0
- package/dist/services/task-specification/service.js +40 -0
- package/dist/services/trajectory-policy/service.d.ts +110 -0
- package/dist/services/trajectory-policy/service.js +216 -0
- package/dist/test/agentic-capabilities-protocol.test.d.ts +1 -0
- package/dist/test/agentic-capabilities-protocol.test.js +570 -0
- package/dist/test/agentic-capabilities.test.d.ts +1 -0
- package/dist/test/agentic-capabilities.test.js +1461 -0
- package/dist/test/agentic-environment.test.d.ts +1 -0
- package/dist/test/agentic-environment.test.js +213 -0
- package/dist/test/case-pipeline-foundation.test.d.ts +1 -0
- package/dist/test/case-pipeline-foundation.test.js +535 -0
- package/dist/test/case-pipeline-protocol-v2.test.d.ts +1 -0
- package/dist/test/case-pipeline-protocol-v2.test.js +124 -0
- package/dist/test/case-pipeline-v2.test.d.ts +1 -0
- package/dist/test/case-pipeline-v2.test.js +286 -0
- package/dist/test/case-pipeline.test.d.ts +1 -0
- package/dist/test/case-pipeline.test.js +851 -0
- package/dist/test/eval-capability-policy.test.d.ts +1 -0
- package/dist/test/eval-capability-policy.test.js +50 -0
- package/dist/test/eval-event-log.test.d.ts +1 -0
- package/dist/test/eval-event-log.test.js +125 -0
- package/dist/test/evaluation-evidence-freshness.test.d.ts +1 -0
- package/dist/test/evaluation-evidence-freshness.test.js +44 -0
- package/dist/test/evaluation-evidence.test.d.ts +1 -0
- package/dist/test/evaluation-evidence.test.js +230 -0
- package/dist/test/evaluation-grader-calibration.test.d.ts +1 -0
- package/dist/test/evaluation-grader-calibration.test.js +373 -0
- package/dist/test/evaluation-proposal-digest.test.d.ts +1 -0
- package/dist/test/evaluation-proposal-digest.test.js +187 -0
- package/dist/test/evaluation-source-retrieval.test.d.ts +1 -0
- package/dist/test/evaluation-source-retrieval.test.js +237 -0
- package/dist/test/evaluation-structure-policy.test.d.ts +1 -0
- package/dist/test/evaluation-structure-policy.test.js +196 -0
- package/dist/test/fixtures/repository-resource-panel.d.ts +39 -0
- package/dist/test/fixtures/repository-resource-panel.js +111 -0
- package/dist/test/fixtures/vitest-boundary-panel.d.ts +84 -0
- package/dist/test/fixtures/vitest-boundary-panel.js +120 -0
- package/dist/test/fixtures/vitest-phase-panel.d.ts +135 -0
- package/dist/test/fixtures/vitest-phase-panel.js +213 -0
- package/dist/test/fixtures/vitest-reporter-results.d.ts +76 -0
- package/dist/test/fixtures/vitest-reporter-results.js +94 -0
- package/dist/test/grounded-authoring.test.d.ts +1 -0
- package/dist/test/grounded-authoring.test.js +565 -0
- package/dist/test/integrated-repository-history.test.d.ts +1 -0
- package/dist/test/integrated-repository-history.test.js +227 -0
- package/dist/test/project-authoring.test.js +593 -43
- package/dist/test/project-workflow.test.js +419 -40
- package/dist/test/repository-authoring-artifacts.test.d.ts +1 -0
- package/dist/test/repository-authoring-artifacts.test.js +185 -0
- package/dist/test/repository-bundle.test.d.ts +1 -0
- package/dist/test/repository-bundle.test.js +52 -0
- package/dist/test/repository-case-generation.test.d.ts +1 -0
- package/dist/test/repository-case-generation.test.js +3465 -0
- package/dist/test/repository-command-diagnostic.test.d.ts +1 -0
- package/dist/test/repository-command-diagnostic.test.js +55 -0
- package/dist/test/repository-command-signals.test.d.ts +1 -0
- package/dist/test/repository-command-signals.test.js +124 -0
- package/dist/test/repository-fixture-scope-coverage.test.d.ts +1 -0
- package/dist/test/repository-fixture-scope-coverage.test.js +127 -0
- package/dist/test/repository-fixture-validation.test.d.ts +1 -0
- package/dist/test/repository-fixture-validation.test.js +362 -0
- package/dist/test/repository-foundry-progress.test.d.ts +1 -0
- package/dist/test/repository-foundry-progress.test.js +110 -0
- package/dist/test/repository-foundry-quality.test.d.ts +1 -0
- package/dist/test/repository-foundry-quality.test.js +1138 -0
- package/dist/test/repository-import-context.test.d.ts +1 -0
- package/dist/test/repository-import-context.test.js +354 -0
- package/dist/test/repository-model-authoring.test.d.ts +1 -0
- package/dist/test/repository-model-authoring.test.js +544 -0
- package/dist/test/repository-model.test.d.ts +1 -0
- package/dist/test/repository-model.test.js +2195 -0
- package/dist/test/repository-node-test-reporter.test.d.ts +1 -0
- package/dist/test/repository-node-test-reporter.test.js +104 -0
- package/dist/test/repository-oracle-concurrency.test.d.ts +1 -0
- package/dist/test/repository-oracle-concurrency.test.js +542 -0
- package/dist/test/repository-oracle-coverage-witness.test.d.ts +1 -0
- package/dist/test/repository-oracle-coverage-witness.test.js +511 -0
- package/dist/test/repository-oracle-coverage.test.d.ts +1 -0
- package/dist/test/repository-oracle-coverage.test.js +168 -0
- package/dist/test/repository-oracle-evidence.test.d.ts +1 -0
- package/dist/test/repository-oracle-evidence.test.js +185 -0
- package/dist/test/repository-oracle-plan.test.d.ts +1 -0
- package/dist/test/repository-oracle-plan.test.js +176 -0
- package/dist/test/repository-overlay-isolation.test.d.ts +1 -0
- package/dist/test/repository-overlay-isolation.test.js +85 -0
- package/dist/test/repository-preparation-cache.test.d.ts +1 -0
- package/dist/test/repository-preparation-cache.test.js +414 -0
- package/dist/test/repository-public-artifact.test.d.ts +1 -0
- package/dist/test/repository-public-artifact.test.js +273 -0
- package/dist/test/repository-qualification-diagnostics.test.d.ts +1 -0
- package/dist/test/repository-qualification-diagnostics.test.js +524 -0
- package/dist/test/repository-reference-authoring.test.d.ts +1 -0
- package/dist/test/repository-reference-authoring.test.js +1633 -0
- package/dist/test/repository-review-evidence-v2.test.d.ts +1 -0
- package/dist/test/repository-review-evidence-v2.test.js +183 -0
- package/dist/test/repository-review-evidence.test.d.ts +1 -0
- package/dist/test/repository-review-evidence.test.js +124 -0
- package/dist/test/repository-seed-exclusions.test.d.ts +1 -0
- package/dist/test/repository-seed-exclusions.test.js +96 -0
- package/dist/test/repository-seed-selection.test.d.ts +1 -0
- package/dist/test/repository-seed-selection.test.js +504 -0
- package/dist/test/repository-semantic-calibration.test.d.ts +1 -0
- package/dist/test/repository-semantic-calibration.test.js +688 -0
- package/dist/test/repository-solution-edits.test.d.ts +1 -0
- package/dist/test/repository-solution-edits.test.js +377 -0
- package/dist/test/repository-specification-budget.test.d.ts +1 -0
- package/dist/test/repository-specification-budget.test.js +171 -0
- package/dist/test/repository-specification-contract-checkpoint.test.d.ts +1 -0
- package/dist/test/repository-specification-contract-checkpoint.test.js +228 -0
- package/dist/test/repository-specification-contract-facts.test.d.ts +1 -0
- package/dist/test/repository-specification-contract-facts.test.js +177 -0
- package/dist/test/repository-trajectory-authoring.test.d.ts +1 -0
- package/dist/test/repository-trajectory-authoring.test.js +176 -0
- package/dist/test/repository-valid-control-plan.test.d.ts +1 -0
- package/dist/test/repository-valid-control-plan.test.js +45 -0
- package/dist/test/repository-vitest-phase.test.d.ts +1 -0
- package/dist/test/repository-vitest-phase.test.js +848 -0
- package/dist/test/repository-vitest-reporter.test.d.ts +1 -0
- package/dist/test/repository-vitest-reporter.test.js +158 -0
- package/dist/test/repository-workspace-build.test.d.ts +1 -0
- package/dist/test/repository-workspace-build.test.js +160 -0
- package/dist/test/strict-authoring-schema.test.d.ts +1 -0
- package/dist/test/strict-authoring-schema.test.js +169 -0
- package/package.json +48 -6
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
import { Context, Effect, Layer } from "effect";
|
|
2
|
+
import { RepositoryFoundryError } from "../../errors.js";
|
|
3
|
+
import { evaluateRepositoryOracleAdequacyV1 } from "../../repository-quality-policy.js";
|
|
4
|
+
import { assertRepositoryTrajectoryPolicyV1 } from "../../repository-trajectory-protocol.js";
|
|
5
|
+
import { compileReviewedHiddenFixtureSuiteV1 } from "../fixture-builder/service.js";
|
|
6
|
+
import { buildHistoricalHiddenOracleV1 } from "../oracle-builder/service.js";
|
|
7
|
+
import { evaluateRepositoryBenchmarkDraftV1 } from "../quality-gate/service.js";
|
|
8
|
+
import { critiqueRepositoryTaskSpecificationV1 } from "../specification-critic/service.js";
|
|
9
|
+
import { deriveRepositoryTaskFamiliesV1 } from "../task-family/service.js";
|
|
10
|
+
const fail = (detail) => new RepositoryFoundryError({
|
|
11
|
+
operation: "compile-reviewed-case",
|
|
12
|
+
detail
|
|
13
|
+
});
|
|
14
|
+
const sortedUnique = (values) => [...new Set(values)].sort();
|
|
15
|
+
const sameStrings = (left, right) => JSON.stringify(sortedUnique(left)) === JSON.stringify(sortedUnique(right));
|
|
16
|
+
const assertBlueprintShape = (seed, blueprint) => {
|
|
17
|
+
if (seed.status !== "qualified" || seed.referenceCommit === undefined) {
|
|
18
|
+
throw fail("a reviewed historical case requires a replay-qualified seed with a reference commit");
|
|
19
|
+
}
|
|
20
|
+
if (blueprint.caseId.trim().length === 0 ||
|
|
21
|
+
blueprint.visible.id !== blueprint.caseId ||
|
|
22
|
+
blueprint.finalFixtureSuite.caseId !== blueprint.caseId) {
|
|
23
|
+
throw fail("the reviewed case, visible task, and final fixture suite must share one non-empty case id");
|
|
24
|
+
}
|
|
25
|
+
if (blueprint.visible.lane !== "repository-agent") {
|
|
26
|
+
throw fail("reviewed historical repository cases must use the repository-agent lane");
|
|
27
|
+
}
|
|
28
|
+
if (blueprint.trajectoryPolicy !== undefined) {
|
|
29
|
+
assertRepositoryTrajectoryPolicyV1(blueprint.trajectoryPolicy);
|
|
30
|
+
if (blueprint.trajectoryPolicy.caseId !== blueprint.caseId) {
|
|
31
|
+
throw fail("the reviewed trajectory policy must share the reviewed case id");
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
if (blueprint.visible.checkoutCommit !== seed.initialState.commit) {
|
|
35
|
+
throw fail("the reviewed visible task must expose the replay-qualified pre-task checkout");
|
|
36
|
+
}
|
|
37
|
+
const solutionIds = blueprint.solutions.map((solution) => solution.id);
|
|
38
|
+
if (solutionIds.some((id) => id.trim().length === 0) ||
|
|
39
|
+
new Set(solutionIds).size !== solutionIds.length) {
|
|
40
|
+
throw fail("the reviewed strong solution population requires unique ids");
|
|
41
|
+
}
|
|
42
|
+
const weak = blueprint.weakOracleTrial;
|
|
43
|
+
if (weak === undefined)
|
|
44
|
+
return;
|
|
45
|
+
if (weak.fixtureSuite !== undefined && weak.fixtureSuite.caseId !== blueprint.caseId) {
|
|
46
|
+
throw fail("the weak oracle fixture suite must share the reviewed case id");
|
|
47
|
+
}
|
|
48
|
+
const weakSolutionIds = weak.solutions.map((solution) => solution.id);
|
|
49
|
+
if (new Set(weakSolutionIds).size !== weakSolutionIds.length) {
|
|
50
|
+
throw fail("the reviewed weak solution population requires unique ids");
|
|
51
|
+
}
|
|
52
|
+
const expectedEscapeIds = sortedUnique(weak.expectedFalseAcceptedSolutionIds);
|
|
53
|
+
if (expectedEscapeIds.length === 0) {
|
|
54
|
+
throw fail("a weak oracle trial must name at least one reproduced false acceptance");
|
|
55
|
+
}
|
|
56
|
+
const strongSolutionIds = new Set(solutionIds);
|
|
57
|
+
for (const escapeId of expectedEscapeIds) {
|
|
58
|
+
const weakSolution = weak.solutions.find((solution) => solution.id === escapeId);
|
|
59
|
+
if (weakSolution === undefined || weakSolution.expectedClass !== "wrong") {
|
|
60
|
+
throw fail(`weak oracle escape ${JSON.stringify(escapeId)} must identify a plausible wrong solution`);
|
|
61
|
+
}
|
|
62
|
+
if (!strongSolutionIds.has(escapeId)) {
|
|
63
|
+
throw fail(`weak oracle escape ${JSON.stringify(escapeId)} must remain in the strengthened tournament`);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
};
|
|
67
|
+
export class HistoricalCase extends Context.Service()("@velum-labs/routekit-eval-setup/HistoricalCase") {
|
|
68
|
+
}
|
|
69
|
+
export const compileReviewedHistoricalCaseV1 = Effect.fn("HistoricalCase.compileReviewed")(function* (input) {
|
|
70
|
+
assertBlueprintShape(input.seed, input.blueprint);
|
|
71
|
+
const families = yield* deriveRepositoryTaskFamiliesV1({
|
|
72
|
+
map: input.map,
|
|
73
|
+
seeds: [input.seed]
|
|
74
|
+
});
|
|
75
|
+
const family = families.find((candidate) => candidate.seedIds.includes(input.seed.id));
|
|
76
|
+
if (family === undefined) {
|
|
77
|
+
return yield* fail("the replay-qualified seed has no derived task family");
|
|
78
|
+
}
|
|
79
|
+
if (family.lane !== input.blueprint.visible.lane || !family.seedIds.includes(input.seed.id)) {
|
|
80
|
+
return yield* fail("the visible task is not aligned with the seed's derived task family");
|
|
81
|
+
}
|
|
82
|
+
const finalFixtureSuite = yield* compileReviewedHiddenFixtureSuiteV1({
|
|
83
|
+
caseId: input.blueprint.caseId,
|
|
84
|
+
seed: input.seed,
|
|
85
|
+
suite: input.blueprint.finalFixtureSuite
|
|
86
|
+
});
|
|
87
|
+
const specificationCritique = yield* critiqueRepositoryTaskSpecificationV1({
|
|
88
|
+
repositoryRoot: input.repositoryRoot,
|
|
89
|
+
seed: input.seed,
|
|
90
|
+
family,
|
|
91
|
+
visible: input.blueprint.visible,
|
|
92
|
+
reviews: input.blueprint.specificationReviews,
|
|
93
|
+
...(input.blueprint.requestedDimension === undefined
|
|
94
|
+
? {}
|
|
95
|
+
: { requestedDimension: input.blueprint.requestedDimension })
|
|
96
|
+
});
|
|
97
|
+
if (specificationCritique.outcome !== "pass") {
|
|
98
|
+
return yield* fail(`the reviewed visible task is not sufficiently specified: ${specificationCritique.rejectionReasons.join("; ")}`);
|
|
99
|
+
}
|
|
100
|
+
const repetitions = Math.max(2, Math.min(input.blueprint.oracleRepetitions ?? 2, 5));
|
|
101
|
+
let weakOracle;
|
|
102
|
+
let weakAdequacy;
|
|
103
|
+
const weakTrial = input.blueprint.weakOracleTrial;
|
|
104
|
+
if (weakTrial !== undefined) {
|
|
105
|
+
const fixtureSuite = weakTrial.fixtureSuite === undefined
|
|
106
|
+
? undefined
|
|
107
|
+
: yield* compileReviewedHiddenFixtureSuiteV1({
|
|
108
|
+
caseId: input.blueprint.caseId,
|
|
109
|
+
seed: input.seed,
|
|
110
|
+
suite: weakTrial.fixtureSuite
|
|
111
|
+
});
|
|
112
|
+
weakOracle = yield* buildHistoricalHiddenOracleV1({
|
|
113
|
+
repositoryRoot: input.repositoryRoot,
|
|
114
|
+
caseId: input.blueprint.caseId,
|
|
115
|
+
seed: input.seed,
|
|
116
|
+
additionalSolutions: weakTrial.solutions,
|
|
117
|
+
...(input.blueprint.oracleConcurrency === undefined
|
|
118
|
+
? {}
|
|
119
|
+
: { oracleConcurrency: input.blueprint.oracleConcurrency }),
|
|
120
|
+
...(fixtureSuite === undefined ? {} : { hiddenFixtureSuite: fixtureSuite }),
|
|
121
|
+
...(input.blueprint.structuralClauses === undefined
|
|
122
|
+
? {}
|
|
123
|
+
: { structuralClauses: input.blueprint.structuralClauses }),
|
|
124
|
+
repetitions
|
|
125
|
+
});
|
|
126
|
+
weakAdequacy = evaluateRepositoryOracleAdequacyV1(weakOracle);
|
|
127
|
+
if (!sameStrings(weakAdequacy.falseAcceptedSolutionIds, weakTrial.expectedFalseAcceptedSolutionIds) ||
|
|
128
|
+
weakAdequacy.falseRejectedSolutionIds.length > 0 ||
|
|
129
|
+
weakAdequacy.unstableSolutionIds.length > 0) {
|
|
130
|
+
return yield* fail([
|
|
131
|
+
"the weak oracle trial did not reproduce the reviewed hill-climb evidence",
|
|
132
|
+
`expectedFalseAccepts=${sortedUnique(weakTrial.expectedFalseAcceptedSolutionIds).join(",")}`,
|
|
133
|
+
`actualFalseAccepts=${weakAdequacy.falseAcceptedSolutionIds.join(",")}`,
|
|
134
|
+
`falseRejects=${weakAdequacy.falseRejectedSolutionIds.join(",")}`,
|
|
135
|
+
`unstable=${weakAdequacy.unstableSolutionIds.join(",")}`
|
|
136
|
+
].join("; "));
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
const hidden = yield* buildHistoricalHiddenOracleV1({
|
|
140
|
+
repositoryRoot: input.repositoryRoot,
|
|
141
|
+
caseId: input.blueprint.caseId,
|
|
142
|
+
seed: input.seed,
|
|
143
|
+
additionalSolutions: input.blueprint.solutions,
|
|
144
|
+
...(input.blueprint.oracleConcurrency === undefined
|
|
145
|
+
? {}
|
|
146
|
+
: { oracleConcurrency: input.blueprint.oracleConcurrency }),
|
|
147
|
+
hiddenFixtureSuite: finalFixtureSuite,
|
|
148
|
+
...(input.blueprint.structuralClauses === undefined
|
|
149
|
+
? {}
|
|
150
|
+
: { structuralClauses: input.blueprint.structuralClauses }),
|
|
151
|
+
repetitions
|
|
152
|
+
});
|
|
153
|
+
const benchmarkCase = yield* evaluateRepositoryBenchmarkDraftV1({
|
|
154
|
+
map: input.map,
|
|
155
|
+
seed: input.seed,
|
|
156
|
+
family,
|
|
157
|
+
visible: input.blueprint.visible,
|
|
158
|
+
hidden,
|
|
159
|
+
specificationCritique,
|
|
160
|
+
...(input.blueprint.trajectoryPolicy === undefined
|
|
161
|
+
? {}
|
|
162
|
+
: { trajectoryPolicy: input.blueprint.trajectoryPolicy })
|
|
163
|
+
});
|
|
164
|
+
return {
|
|
165
|
+
seed: input.seed,
|
|
166
|
+
family,
|
|
167
|
+
specificationCritique,
|
|
168
|
+
...(weakOracle === undefined ? {} : { weakOracle }),
|
|
169
|
+
...(weakAdequacy === undefined ? {} : { weakAdequacy }),
|
|
170
|
+
hidden,
|
|
171
|
+
benchmarkCase
|
|
172
|
+
};
|
|
173
|
+
});
|
|
174
|
+
export const makeHistoricalCase = Effect.succeed(HistoricalCase.of({
|
|
175
|
+
compileReviewed: compileReviewedHistoricalCaseV1
|
|
176
|
+
}));
|
|
177
|
+
export const HistoricalCaseLive = Layer.effect(HistoricalCase, makeHistoricalCase);
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import { Context, Effect, Layer } from "effect";
|
|
2
|
+
import { RepositoryFoundryError } from "../../errors.js";
|
|
3
|
+
import type { RepositoryValidatedBenchmarkLibraryV1 } from "../../repository-benchmark-protocol.js";
|
|
4
|
+
import { type RepositoryFoundryImprovementCycleV1, type RepositoryFoundryImprovementDraftV1 } from "../../repository-improvement-protocol.js";
|
|
5
|
+
export type ImprovementLoopShape = {
|
|
6
|
+
readonly compileCycle: (input: {
|
|
7
|
+
readonly cycleId: string;
|
|
8
|
+
readonly library: RepositoryValidatedBenchmarkLibraryV1;
|
|
9
|
+
readonly auditedCoreCaseIds: readonly string[];
|
|
10
|
+
readonly drafts: readonly RepositoryFoundryImprovementDraftV1[];
|
|
11
|
+
}) => Effect.Effect<RepositoryFoundryImprovementCycleV1, RepositoryFoundryError>;
|
|
12
|
+
};
|
|
13
|
+
declare const ImprovementLoop_base: Context.ServiceClass<ImprovementLoop, "@velum-labs/routekit-eval-setup/ImprovementLoop", ImprovementLoopShape>;
|
|
14
|
+
export declare class ImprovementLoop extends ImprovementLoop_base {
|
|
15
|
+
}
|
|
16
|
+
export declare const compileRepositoryFoundryImprovementCycleV1: (input: {
|
|
17
|
+
readonly cycleId: string;
|
|
18
|
+
readonly library: RepositoryValidatedBenchmarkLibraryV1;
|
|
19
|
+
readonly auditedCoreCaseIds: readonly string[];
|
|
20
|
+
readonly drafts: readonly RepositoryFoundryImprovementDraftV1[];
|
|
21
|
+
}) => Effect.Effect<{
|
|
22
|
+
readonly version: 1;
|
|
23
|
+
readonly cycleId: string;
|
|
24
|
+
readonly inputCaseIds: readonly string[];
|
|
25
|
+
readonly records: readonly {
|
|
26
|
+
readonly version: 1;
|
|
27
|
+
readonly id: string;
|
|
28
|
+
readonly trigger: "superseded" | "new-authentic-work" | "oracle-escape" | "false-rejection" | "coverage-gap" | "instability" | "environment-drift" | "contract-drift" | "realism-drift" | "semantic-duplicate";
|
|
29
|
+
readonly action: "add-seed" | "add-fixture" | "strengthen-oracle" | "add-valid-alternative" | "regenerate-specification" | "quarantine-case" | "retire-case";
|
|
30
|
+
readonly summary: string;
|
|
31
|
+
readonly affectedCaseIds: readonly string[];
|
|
32
|
+
readonly evidence: readonly {
|
|
33
|
+
readonly kind: "symbol" | "package" | "public-surface" | "protocol" | "schema" | "test" | "command" | "history-episode" | "workload-signal" | "documentation";
|
|
34
|
+
readonly id: string;
|
|
35
|
+
readonly path?: string | undefined;
|
|
36
|
+
}[];
|
|
37
|
+
readonly reviews: readonly {
|
|
38
|
+
readonly reviewerId: string;
|
|
39
|
+
readonly independentlyProduced: boolean;
|
|
40
|
+
readonly verdict: "approve" | "reject";
|
|
41
|
+
readonly detail: string;
|
|
42
|
+
}[];
|
|
43
|
+
readonly decision: "rejected" | "applied";
|
|
44
|
+
readonly rejectionReasons: readonly string[];
|
|
45
|
+
}[];
|
|
46
|
+
readonly lifecycle: readonly {
|
|
47
|
+
readonly caseId: string;
|
|
48
|
+
readonly reason: string;
|
|
49
|
+
readonly state: "active" | "audited-core" | "quarantined" | "retired";
|
|
50
|
+
readonly improvementRecordId?: string | undefined;
|
|
51
|
+
}[];
|
|
52
|
+
readonly activeCaseIds: readonly string[];
|
|
53
|
+
readonly auditedCoreCaseIds: readonly string[];
|
|
54
|
+
readonly quarantinedCaseIds: readonly string[];
|
|
55
|
+
readonly retiredCaseIds: readonly string[];
|
|
56
|
+
}, RepositoryFoundryError, never>;
|
|
57
|
+
export declare const makeImprovementLoop: Effect.Effect<ImprovementLoopShape, never, never>;
|
|
58
|
+
export declare const ImprovementLoopLive: Layer.Layer<ImprovementLoop, never, never>;
|
|
59
|
+
export {};
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
import { Context, Effect, Layer } from "effect";
|
|
2
|
+
import { RepositoryFoundryError } from "../../errors.js";
|
|
3
|
+
import { assertRepositoryFoundryImprovementCycleV1 } from "../../repository-improvement-protocol.js";
|
|
4
|
+
const ALLOWED_ACTIONS = {
|
|
5
|
+
"new-authentic-work": new Set(["add-seed"]),
|
|
6
|
+
"oracle-escape": new Set([
|
|
7
|
+
"add-fixture",
|
|
8
|
+
"strengthen-oracle",
|
|
9
|
+
"quarantine-case"
|
|
10
|
+
]),
|
|
11
|
+
"false-rejection": new Set([
|
|
12
|
+
"add-valid-alternative",
|
|
13
|
+
"strengthen-oracle",
|
|
14
|
+
"quarantine-case"
|
|
15
|
+
]),
|
|
16
|
+
"coverage-gap": new Set([
|
|
17
|
+
"add-seed",
|
|
18
|
+
"add-fixture",
|
|
19
|
+
"regenerate-specification"
|
|
20
|
+
]),
|
|
21
|
+
instability: new Set(["quarantine-case"]),
|
|
22
|
+
"environment-drift": new Set(["quarantine-case", "retire-case"]),
|
|
23
|
+
"contract-drift": new Set([
|
|
24
|
+
"regenerate-specification",
|
|
25
|
+
"quarantine-case",
|
|
26
|
+
"retire-case"
|
|
27
|
+
]),
|
|
28
|
+
"realism-drift": new Set([
|
|
29
|
+
"regenerate-specification",
|
|
30
|
+
"quarantine-case",
|
|
31
|
+
"retire-case"
|
|
32
|
+
]),
|
|
33
|
+
"semantic-duplicate": new Set(["retire-case"]),
|
|
34
|
+
superseded: new Set(["retire-case"])
|
|
35
|
+
};
|
|
36
|
+
const failure = (detail, cause) => new RepositoryFoundryError({
|
|
37
|
+
operation: "compile-improvement-cycle",
|
|
38
|
+
detail,
|
|
39
|
+
...(cause === undefined ? {} : { cause })
|
|
40
|
+
});
|
|
41
|
+
const compileRecord = (draft, knownCaseIds, auditedCoreCaseIds, lifecycleChangedCaseIds) => {
|
|
42
|
+
const reviewerIds = new Set(draft.reviews
|
|
43
|
+
.filter((review) => review.independentlyProduced)
|
|
44
|
+
.map((review) => review.reviewerId));
|
|
45
|
+
const lifecycleAction = draft.action === "quarantine-case" || draft.action === "retire-case";
|
|
46
|
+
const rejectionReasons = [
|
|
47
|
+
...(draft.id.trim().length === 0 ? ["improvement id is empty"] : []),
|
|
48
|
+
...(draft.summary.trim().length === 0
|
|
49
|
+
? ["improvement summary is empty"]
|
|
50
|
+
: []),
|
|
51
|
+
...(ALLOWED_ACTIONS[draft.trigger].has(draft.action)
|
|
52
|
+
? []
|
|
53
|
+
: [
|
|
54
|
+
`action ${JSON.stringify(draft.action)} is invalid for trigger ${JSON.stringify(draft.trigger)}`
|
|
55
|
+
]),
|
|
56
|
+
...(draft.reviews.some((review) => review.verdict === "reject")
|
|
57
|
+
? ["an independent review rejected the proposed improvement"]
|
|
58
|
+
: []),
|
|
59
|
+
...(reviewerIds.size >= 2 &&
|
|
60
|
+
draft.reviews.filter((review) => review.independentlyProduced && review.verdict === "approve").length >= 2
|
|
61
|
+
? []
|
|
62
|
+
: ["two distinct independent approvals are required"]),
|
|
63
|
+
...(lifecycleAction && draft.affectedCaseIds.length === 0
|
|
64
|
+
? ["lifecycle improvements require an affected case"]
|
|
65
|
+
: []),
|
|
66
|
+
...draft.affectedCaseIds
|
|
67
|
+
.filter((caseId) => !knownCaseIds.has(caseId))
|
|
68
|
+
.map((caseId) => `affected case is unknown: ${caseId}`),
|
|
69
|
+
...(lifecycleAction
|
|
70
|
+
? draft.affectedCaseIds
|
|
71
|
+
.filter((caseId) => auditedCoreCaseIds.has(caseId))
|
|
72
|
+
.map((caseId) => `audited core case cannot be silently removed: ${caseId}`)
|
|
73
|
+
: []),
|
|
74
|
+
...(lifecycleAction
|
|
75
|
+
? draft.affectedCaseIds
|
|
76
|
+
.filter((caseId) => lifecycleChangedCaseIds.has(caseId))
|
|
77
|
+
.map((caseId) => `case already has a lifecycle decision in this cycle: ${caseId}`)
|
|
78
|
+
: [])
|
|
79
|
+
];
|
|
80
|
+
return {
|
|
81
|
+
version: 1,
|
|
82
|
+
...draft,
|
|
83
|
+
decision: rejectionReasons.length === 0 ? "applied" : "rejected",
|
|
84
|
+
rejectionReasons
|
|
85
|
+
};
|
|
86
|
+
};
|
|
87
|
+
export class ImprovementLoop extends Context.Service()("@velum-labs/routekit-eval-setup/ImprovementLoop") {
|
|
88
|
+
}
|
|
89
|
+
export const compileRepositoryFoundryImprovementCycleV1 = Effect.fn("ImprovementLoop.compileCycle")(function* (input) {
|
|
90
|
+
if (input.cycleId.trim().length === 0) {
|
|
91
|
+
return yield* failure("improvement cycle id is empty");
|
|
92
|
+
}
|
|
93
|
+
const inputCaseIds = input.library.cases.map((benchmarkCase) => benchmarkCase.id);
|
|
94
|
+
const knownCaseIds = new Set(inputCaseIds);
|
|
95
|
+
const auditedCoreCaseIds = new Set(input.auditedCoreCaseIds);
|
|
96
|
+
const unknownCoreCaseIds = [...auditedCoreCaseIds].filter((caseId) => !knownCaseIds.has(caseId));
|
|
97
|
+
if (unknownCoreCaseIds.length > 0) {
|
|
98
|
+
return yield* failure(`audited core contains unknown cases: ${unknownCoreCaseIds.join(", ")}`);
|
|
99
|
+
}
|
|
100
|
+
const draftIds = new Set(input.drafts.map((draft) => draft.id));
|
|
101
|
+
if (draftIds.size !== input.drafts.length) {
|
|
102
|
+
return yield* failure("improvement draft ids must be unique");
|
|
103
|
+
}
|
|
104
|
+
const lifecycleChangedCaseIds = new Set();
|
|
105
|
+
const records = [];
|
|
106
|
+
for (const draft of input.drafts) {
|
|
107
|
+
const record = compileRecord(draft, knownCaseIds, auditedCoreCaseIds, lifecycleChangedCaseIds);
|
|
108
|
+
records.push(record);
|
|
109
|
+
if (record.decision === "applied" &&
|
|
110
|
+
(record.action === "quarantine-case" ||
|
|
111
|
+
record.action === "retire-case")) {
|
|
112
|
+
for (const caseId of record.affectedCaseIds) {
|
|
113
|
+
lifecycleChangedCaseIds.add(caseId);
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
const lifecycle = inputCaseIds
|
|
118
|
+
.map((caseId) => {
|
|
119
|
+
const record = records.find((candidate) => candidate.decision === "applied" &&
|
|
120
|
+
(candidate.action === "quarantine-case" ||
|
|
121
|
+
candidate.action === "retire-case") &&
|
|
122
|
+
candidate.affectedCaseIds.includes(caseId));
|
|
123
|
+
if (record?.action === "quarantine-case") {
|
|
124
|
+
return {
|
|
125
|
+
caseId,
|
|
126
|
+
state: "quarantined",
|
|
127
|
+
improvementRecordId: record.id,
|
|
128
|
+
reason: record.summary
|
|
129
|
+
};
|
|
130
|
+
}
|
|
131
|
+
if (record?.action === "retire-case") {
|
|
132
|
+
return {
|
|
133
|
+
caseId,
|
|
134
|
+
state: "retired",
|
|
135
|
+
improvementRecordId: record.id,
|
|
136
|
+
reason: record.summary
|
|
137
|
+
};
|
|
138
|
+
}
|
|
139
|
+
if (auditedCoreCaseIds.has(caseId)) {
|
|
140
|
+
return {
|
|
141
|
+
caseId,
|
|
142
|
+
state: "audited-core",
|
|
143
|
+
reason: "Retained as a reviewed generator-regression case."
|
|
144
|
+
};
|
|
145
|
+
}
|
|
146
|
+
return {
|
|
147
|
+
caseId,
|
|
148
|
+
state: "active",
|
|
149
|
+
reason: "Accepted and active in the validated library."
|
|
150
|
+
};
|
|
151
|
+
})
|
|
152
|
+
.sort((left, right) => left.caseId.localeCompare(right.caseId));
|
|
153
|
+
const byState = (state) => lifecycle
|
|
154
|
+
.filter((entry) => entry.state === state)
|
|
155
|
+
.map((entry) => entry.caseId);
|
|
156
|
+
const cycle = {
|
|
157
|
+
version: 1,
|
|
158
|
+
cycleId: input.cycleId,
|
|
159
|
+
inputCaseIds: [...inputCaseIds].sort(),
|
|
160
|
+
records,
|
|
161
|
+
lifecycle,
|
|
162
|
+
activeCaseIds: byState("active"),
|
|
163
|
+
auditedCoreCaseIds: byState("audited-core"),
|
|
164
|
+
quarantinedCaseIds: byState("quarantined"),
|
|
165
|
+
retiredCaseIds: byState("retired")
|
|
166
|
+
};
|
|
167
|
+
yield* Effect.try({
|
|
168
|
+
try: () => assertRepositoryFoundryImprovementCycleV1(cycle),
|
|
169
|
+
catch: (cause) => failure("compiled improvement cycle failed validation", cause)
|
|
170
|
+
});
|
|
171
|
+
return cycle;
|
|
172
|
+
});
|
|
173
|
+
export const makeImprovementLoop = Effect.succeed(ImprovementLoop.of({
|
|
174
|
+
compileCycle: compileRepositoryFoundryImprovementCycleV1
|
|
175
|
+
}));
|
|
176
|
+
export const ImprovementLoopLive = Layer.effect(ImprovementLoop, makeImprovementLoop);
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import { Context, Effect, Layer, Schema } from "effect";
|
|
2
|
+
import { RepositoryFoundryError } from "../../errors.js";
|
|
3
|
+
import { EvalAuthoringTransport } from "../../project-authoring.js";
|
|
4
|
+
import { type RepositoryFoundryModelCallV1, type RepositoryFoundryModelPlanV1, type RepositoryFoundryModelRoleV1, type RepositoryFoundryReasoningEffortV1 } from "../../repository-language-model-protocol.js";
|
|
5
|
+
export type RepositoryFoundryStructuredGenerationV1<A> = {
|
|
6
|
+
readonly value: A;
|
|
7
|
+
readonly call: RepositoryFoundryModelCallV1;
|
|
8
|
+
};
|
|
9
|
+
export type RepositoryStructuredModelAssignmentV1<Role extends string> = {
|
|
10
|
+
readonly role: Role;
|
|
11
|
+
readonly model: string;
|
|
12
|
+
readonly reasoningEffort: RepositoryFoundryReasoningEffortV1;
|
|
13
|
+
};
|
|
14
|
+
export type RepositoryAssignedStructuredGenerationV1<A, Role extends string> = {
|
|
15
|
+
readonly value: A;
|
|
16
|
+
readonly call: {
|
|
17
|
+
readonly operationId: string;
|
|
18
|
+
readonly role: Role;
|
|
19
|
+
readonly model: string;
|
|
20
|
+
readonly reasoningEffort: RepositoryFoundryReasoningEffortV1;
|
|
21
|
+
readonly callId?: string;
|
|
22
|
+
};
|
|
23
|
+
};
|
|
24
|
+
export type RepositoryFoundryLanguageModelShape = {
|
|
25
|
+
readonly generateAssignedStructured: <A, Role extends string>(input: {
|
|
26
|
+
readonly assignment: RepositoryStructuredModelAssignmentV1<Role>;
|
|
27
|
+
readonly operationId: string;
|
|
28
|
+
readonly instructions: string;
|
|
29
|
+
readonly input: unknown;
|
|
30
|
+
readonly schemaName: string;
|
|
31
|
+
readonly outputSchema: Schema.Codec<A, unknown, never, unknown>;
|
|
32
|
+
readonly maximumOutputTokens: number;
|
|
33
|
+
readonly reviewInputVersion?: 2;
|
|
34
|
+
}) => Effect.Effect<RepositoryAssignedStructuredGenerationV1<A, Role>, RepositoryFoundryError>;
|
|
35
|
+
readonly generateStructured: <A>(input: {
|
|
36
|
+
readonly plan: RepositoryFoundryModelPlanV1;
|
|
37
|
+
readonly role: RepositoryFoundryModelRoleV1;
|
|
38
|
+
readonly operationId: string;
|
|
39
|
+
readonly instructions: string;
|
|
40
|
+
readonly input: unknown;
|
|
41
|
+
readonly schemaName: string;
|
|
42
|
+
readonly outputSchema: Schema.Codec<A, unknown, never, unknown>;
|
|
43
|
+
readonly maximumOutputTokens: number;
|
|
44
|
+
readonly reviewInputVersion?: 2;
|
|
45
|
+
}) => Effect.Effect<RepositoryFoundryStructuredGenerationV1<A>, RepositoryFoundryError>;
|
|
46
|
+
};
|
|
47
|
+
declare const RepositoryFoundryLanguageModel_base: Context.ServiceClass<RepositoryFoundryLanguageModel, "@velum-labs/routekit-eval-setup/RepositoryFoundryLanguageModel", RepositoryFoundryLanguageModelShape>;
|
|
48
|
+
export declare class RepositoryFoundryLanguageModel extends RepositoryFoundryLanguageModel_base {
|
|
49
|
+
}
|
|
50
|
+
export declare const makeRepositoryFoundryLanguageModel: Effect.Effect<RepositoryFoundryLanguageModelShape, never, EvalAuthoringTransport>;
|
|
51
|
+
export declare const RepositoryFoundryLanguageModelLive: Layer.Layer<RepositoryFoundryLanguageModel, never, EvalAuthoringTransport>;
|
|
52
|
+
export {};
|
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
import { Context, Effect, Layer, Schema } from "effect";
|
|
2
|
+
import { evalAuthoringRequestByteLimit, evalAuthoringResponsesRequestBody } from "../../adapters/authoring-responses-request.js";
|
|
3
|
+
import { encodeRepositoryReviewEvidenceV2, REPOSITORY_REVIEW_EVIDENCE_V2_INSTRUCTIONS } from "../../adapters/repository-review-evidence.js";
|
|
4
|
+
import { strictAuthoringSchema } from "../../adapters/strict-authoring-schema.js";
|
|
5
|
+
import { RepositoryFoundryError } from "../../errors.js";
|
|
6
|
+
import { EvalAuthoringTransport } from "../../project-authoring.js";
|
|
7
|
+
import { assertRepositoryFoundryModelPlanV1, repositoryFoundryModelAssignmentV1 } from "../../repository-language-model-protocol.js";
|
|
8
|
+
import { reportFoundryAuthoringArtifactV1, reportFoundryModelCallV1 } from "../foundry-progress/service.js";
|
|
9
|
+
export class RepositoryFoundryLanguageModel extends Context.Service()("@velum-labs/routekit-eval-setup/RepositoryFoundryLanguageModel") {
|
|
10
|
+
}
|
|
11
|
+
const failure = (detail, cause) => new RepositoryFoundryError({
|
|
12
|
+
operation: "invoke-language-model",
|
|
13
|
+
detail,
|
|
14
|
+
...(cause === undefined ? {} : { cause })
|
|
15
|
+
});
|
|
16
|
+
const malformedResponse = (modelResponseFailure, detail, cause) => new RepositoryFoundryError({
|
|
17
|
+
operation: "invoke-language-model",
|
|
18
|
+
modelResponseFailure,
|
|
19
|
+
detail,
|
|
20
|
+
cause
|
|
21
|
+
});
|
|
22
|
+
const detailOf = (cause) => cause instanceof Error && cause.message.trim().length > 0 ? cause.message : String(cause);
|
|
23
|
+
export const makeRepositoryFoundryLanguageModel = Effect.gen(function* () {
|
|
24
|
+
const transport = yield* EvalAuthoringTransport;
|
|
25
|
+
const generateAssignedStructured = (input) => Effect.gen(function* () {
|
|
26
|
+
yield* Effect.try({
|
|
27
|
+
try: () => {
|
|
28
|
+
if (input.assignment.role.trim().length === 0 ||
|
|
29
|
+
input.assignment.model.trim().length === 0 ||
|
|
30
|
+
input.assignment.reasoningEffort.trim().length === 0 ||
|
|
31
|
+
input.operationId.trim().length === 0 ||
|
|
32
|
+
input.instructions.trim().length === 0 ||
|
|
33
|
+
input.schemaName.trim().length === 0 ||
|
|
34
|
+
!Number.isInteger(input.maximumOutputTokens) ||
|
|
35
|
+
input.maximumOutputTokens < 1) {
|
|
36
|
+
throw new Error("repository foundry model request requires a bounded, named assignment");
|
|
37
|
+
}
|
|
38
|
+
},
|
|
39
|
+
catch: (cause) => failure("repository foundry model request failed validation", cause)
|
|
40
|
+
});
|
|
41
|
+
const strictSchema = yield* Effect.try({
|
|
42
|
+
try: () => {
|
|
43
|
+
const document = Schema.toJsonSchemaDocument(input.outputSchema);
|
|
44
|
+
return strictAuthoringSchema(Object.keys(document.definitions).length === 0
|
|
45
|
+
? document.schema
|
|
46
|
+
: {
|
|
47
|
+
...document.schema,
|
|
48
|
+
$defs: document.definitions
|
|
49
|
+
});
|
|
50
|
+
},
|
|
51
|
+
catch: (cause) => failure("repository foundry output schema is not strict-provider compatible", cause)
|
|
52
|
+
});
|
|
53
|
+
const jsonSchema = strictSchema.jsonSchema;
|
|
54
|
+
const request = yield* Effect.try({
|
|
55
|
+
try: () => {
|
|
56
|
+
const original = {
|
|
57
|
+
operationId: input.operationId,
|
|
58
|
+
model: input.assignment.model,
|
|
59
|
+
instructions: input.instructions,
|
|
60
|
+
input: JSON.stringify(input.input),
|
|
61
|
+
schemaName: input.schemaName,
|
|
62
|
+
jsonSchema,
|
|
63
|
+
maximumOutputTokens: input.maximumOutputTokens,
|
|
64
|
+
foundryRole: input.assignment.role,
|
|
65
|
+
reasoningEffort: input.assignment.reasoningEffort,
|
|
66
|
+
...(input.reviewInputVersion === undefined
|
|
67
|
+
? {}
|
|
68
|
+
: { reviewInputVersion: input.reviewInputVersion })
|
|
69
|
+
};
|
|
70
|
+
const requestByteLimit = evalAuthoringRequestByteLimit(original);
|
|
71
|
+
if (Buffer.byteLength(evalAuthoringResponsesRequestBody(original)) <= requestByteLimit)
|
|
72
|
+
return original;
|
|
73
|
+
const sourceBases = new Set();
|
|
74
|
+
const collect = (value) => {
|
|
75
|
+
if (Array.isArray(value)) {
|
|
76
|
+
for (const entry of value)
|
|
77
|
+
collect(entry);
|
|
78
|
+
}
|
|
79
|
+
else if (value !== null && typeof value === "object") {
|
|
80
|
+
const record = value;
|
|
81
|
+
if (typeof record.content === "string")
|
|
82
|
+
sourceBases.add(record.content);
|
|
83
|
+
for (const entry of Object.values(record))
|
|
84
|
+
collect(entry);
|
|
85
|
+
}
|
|
86
|
+
};
|
|
87
|
+
const originalInput = JSON.parse(original.input);
|
|
88
|
+
collect(originalInput);
|
|
89
|
+
const compact = {
|
|
90
|
+
...original,
|
|
91
|
+
instructions: `${original.instructions}\n\n${REPOSITORY_REVIEW_EVIDENCE_V2_INSTRUCTIONS}`,
|
|
92
|
+
input: JSON.stringify(encodeRepositoryReviewEvidenceV2(originalInput, [...sourceBases]))
|
|
93
|
+
};
|
|
94
|
+
if (Buffer.byteLength(evalAuthoringResponsesRequestBody(compact)) > requestByteLimit)
|
|
95
|
+
throw new Error("complete role evidence exceeds the bounded request after lossless compaction");
|
|
96
|
+
return compact;
|
|
97
|
+
},
|
|
98
|
+
catch: (cause) => failure(`language-model role ${input.assignment.role} input preparation failed: ${detailOf(cause)}`, cause)
|
|
99
|
+
});
|
|
100
|
+
yield* reportFoundryModelCallV1({
|
|
101
|
+
role: input.assignment.role,
|
|
102
|
+
operationId: input.operationId,
|
|
103
|
+
phase: "start"
|
|
104
|
+
});
|
|
105
|
+
const completion = yield* (transport.completeDetailed === undefined
|
|
106
|
+
? transport.complete(request).pipe(Effect.map((text) => ({ text })), Effect.mapError((cause) => failure(`language-model role ${input.assignment.role} failed: ${detailOf(cause)}`, cause)))
|
|
107
|
+
: transport
|
|
108
|
+
.completeDetailed(request)
|
|
109
|
+
.pipe(Effect.mapError((cause) => failure(`language-model role ${input.assignment.role} failed: ${detailOf(cause)}`, cause)))).pipe(Effect.tapError(() => reportFoundryModelCallV1({
|
|
110
|
+
role: input.assignment.role,
|
|
111
|
+
operationId: input.operationId,
|
|
112
|
+
phase: "failed"
|
|
113
|
+
})), Effect.onInterrupt(() => reportFoundryModelCallV1({
|
|
114
|
+
role: input.assignment.role,
|
|
115
|
+
operationId: input.operationId,
|
|
116
|
+
phase: "failed"
|
|
117
|
+
})));
|
|
118
|
+
const callId = !("callId" in completion) || completion.callId === undefined
|
|
119
|
+
? undefined
|
|
120
|
+
: completion.callId;
|
|
121
|
+
// A returned response consumed a provider call even when JSON decoding,
|
|
122
|
+
// schema validation, or durable artifact recording subsequently fails.
|
|
123
|
+
yield* reportFoundryModelCallV1({
|
|
124
|
+
role: input.assignment.role,
|
|
125
|
+
operationId: input.operationId,
|
|
126
|
+
phase: "complete",
|
|
127
|
+
...(callId === undefined ? {} : { callId })
|
|
128
|
+
});
|
|
129
|
+
const artifact = {
|
|
130
|
+
version: 1,
|
|
131
|
+
operationId: input.operationId,
|
|
132
|
+
role: input.assignment.role,
|
|
133
|
+
model: input.assignment.model,
|
|
134
|
+
reasoningEffort: input.assignment.reasoningEffort,
|
|
135
|
+
schemaName: input.schemaName,
|
|
136
|
+
validation: "received",
|
|
137
|
+
request: {
|
|
138
|
+
instructions: request.instructions,
|
|
139
|
+
input: JSON.parse(request.input),
|
|
140
|
+
jsonSchema,
|
|
141
|
+
maximumOutputTokens: input.maximumOutputTokens
|
|
142
|
+
},
|
|
143
|
+
response: {
|
|
144
|
+
text: completion.text,
|
|
145
|
+
...(callId === undefined ? {} : { callId })
|
|
146
|
+
}
|
|
147
|
+
};
|
|
148
|
+
yield* reportFoundryAuthoringArtifactV1(artifact);
|
|
149
|
+
const parsed = yield* Effect.try({
|
|
150
|
+
try: () => JSON.parse(completion.text),
|
|
151
|
+
catch: (cause) => malformedResponse("invalid-json", `language-model role ${input.assignment.role} returned invalid JSON`, cause)
|
|
152
|
+
});
|
|
153
|
+
const normalized = yield* Effect.try({
|
|
154
|
+
try: () => strictSchema.normalizeResponse(parsed),
|
|
155
|
+
catch: (cause) => malformedResponse("normalization", "repository foundry structured response could not restore optional fields", cause)
|
|
156
|
+
});
|
|
157
|
+
const value = yield* Schema.decodeUnknownEffect(input.outputSchema)(normalized).pipe(Effect.mapError((cause) => malformedResponse("schema", `language-model role ${input.assignment.role} violated ${input.schemaName}`, cause)));
|
|
158
|
+
yield* reportFoundryAuthoringArtifactV1({ ...artifact, validation: "validated", value });
|
|
159
|
+
return {
|
|
160
|
+
value,
|
|
161
|
+
call: {
|
|
162
|
+
operationId: input.operationId,
|
|
163
|
+
role: input.assignment.role,
|
|
164
|
+
model: input.assignment.model,
|
|
165
|
+
reasoningEffort: input.assignment.reasoningEffort,
|
|
166
|
+
...(callId === undefined ? {} : { callId })
|
|
167
|
+
}
|
|
168
|
+
};
|
|
169
|
+
});
|
|
170
|
+
const generateStructured = (input) => Effect.gen(function* () {
|
|
171
|
+
yield* Effect.try({
|
|
172
|
+
try: () => assertRepositoryFoundryModelPlanV1(input.plan),
|
|
173
|
+
catch: (cause) => failure("repository foundry model request failed validation", cause)
|
|
174
|
+
});
|
|
175
|
+
const assignment = repositoryFoundryModelAssignmentV1(input.plan, input.role);
|
|
176
|
+
return yield* generateAssignedStructured({
|
|
177
|
+
assignment,
|
|
178
|
+
operationId: input.operationId,
|
|
179
|
+
instructions: input.instructions,
|
|
180
|
+
input: input.input,
|
|
181
|
+
schemaName: input.schemaName,
|
|
182
|
+
outputSchema: input.outputSchema,
|
|
183
|
+
maximumOutputTokens: input.maximumOutputTokens,
|
|
184
|
+
...(input.reviewInputVersion === undefined
|
|
185
|
+
? {}
|
|
186
|
+
: { reviewInputVersion: input.reviewInputVersion })
|
|
187
|
+
});
|
|
188
|
+
});
|
|
189
|
+
return RepositoryFoundryLanguageModel.of({
|
|
190
|
+
generateAssignedStructured,
|
|
191
|
+
generateStructured
|
|
192
|
+
});
|
|
193
|
+
});
|
|
194
|
+
export const RepositoryFoundryLanguageModelLive = Layer.effect(RepositoryFoundryLanguageModel, makeRepositoryFoundryLanguageModel);
|