@velum-labs/routekit-eval-setup 1.3.2 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/authoring-responses-request.d.ts +5 -0
- package/dist/adapters/authoring-responses-request.js +38 -0
- package/dist/adapters/evaluation-evidence-freshness.d.ts +7 -0
- package/dist/adapters/evaluation-evidence-freshness.js +76 -0
- package/dist/adapters/git-task-history.d.ts +67 -0
- package/dist/adapters/git-task-history.js +171 -0
- package/dist/adapters/integrated-repository-history.d.ts +21 -0
- package/dist/adapters/integrated-repository-history.js +175 -0
- package/dist/adapters/repository-command-diagnostic.d.ts +8 -0
- package/dist/adapters/repository-command-diagnostic.js +46 -0
- package/dist/adapters/repository-command-evidence.d.ts +13 -0
- package/dist/adapters/repository-command-evidence.js +102 -0
- package/dist/adapters/repository-command-runner.d.ts +238 -0
- package/dist/adapters/repository-command-runner.js +1483 -0
- package/dist/adapters/repository-import-context.d.ts +47 -0
- package/dist/adapters/repository-import-context.js +469 -0
- package/dist/adapters/repository-node-test-reporter.d.ts +3 -0
- package/dist/adapters/repository-node-test-reporter.js +27 -0
- package/dist/adapters/repository-review-evidence.d.ts +39 -0
- package/dist/adapters/repository-review-evidence.js +632 -0
- package/dist/adapters/repository-seed-selection.d.ts +7 -0
- package/dist/adapters/repository-seed-selection.js +79 -0
- package/dist/adapters/repository-solution-edits.d.ts +49 -0
- package/dist/adapters/repository-solution-edits.js +136 -0
- package/dist/adapters/repository-vitest-phase-adapter.d.ts +8 -0
- package/dist/adapters/repository-vitest-phase-adapter.js +310 -0
- package/dist/adapters/repository-vitest-reporter.d.ts +24 -0
- package/dist/adapters/repository-vitest-reporter.js +314 -0
- package/dist/adapters/strict-authoring-schema.d.ts +5 -0
- package/dist/adapters/strict-authoring-schema.js +158 -0
- package/dist/adapters/test-discovery.d.ts +30 -0
- package/dist/adapters/test-discovery.js +124 -0
- package/dist/adapters/typescript-repository-index.d.ts +51 -0
- package/dist/adapters/typescript-repository-index.js +226 -0
- package/dist/agentic-capabilities-protocol.d.ts +1373 -0
- package/dist/agentic-capabilities-protocol.js +786 -0
- package/dist/case-checkpoint-store.d.ts +29 -0
- package/dist/case-checkpoint-store.js +133 -0
- package/dist/case-pipeline-protocol-v2.d.ts +184 -0
- package/dist/case-pipeline-protocol-v2.js +193 -0
- package/dist/case-pipeline-protocol.d.ts +2626 -0
- package/dist/case-pipeline-protocol.js +371 -0
- package/dist/effect-api.d.ts +74 -10
- package/dist/effect-api.js +56 -6
- package/dist/errors.d.ts +31 -0
- package/dist/errors.js +10 -0
- package/dist/eval-capability-execution-envelope.d.ts +64 -0
- package/dist/eval-capability-execution-envelope.js +98 -0
- package/dist/eval-capability-policy.d.ts +90 -0
- package/dist/eval-capability-policy.js +107 -0
- package/dist/eval-event-log.d.ts +140 -0
- package/dist/eval-event-log.js +220 -0
- package/dist/evaluation-authoring-policy.d.ts +18 -0
- package/dist/evaluation-authoring-policy.js +19 -0
- package/dist/evaluation-authoring-validation.d.ts +22 -0
- package/dist/evaluation-authoring-validation.js +72 -0
- package/dist/evaluation-evidence.d.ts +20 -0
- package/dist/evaluation-evidence.js +319 -0
- package/dist/evaluation-grader-calibration-protocol.d.ts +108 -0
- package/dist/evaluation-grader-calibration-protocol.js +80 -0
- package/dist/evaluation-grader-calibration.d.ts +18 -0
- package/dist/evaluation-grader-calibration.js +334 -0
- package/dist/evaluation-grading-policy.d.ts +24 -0
- package/dist/evaluation-grading-policy.js +54 -0
- package/dist/evaluation-proposal-policy.d.ts +4 -0
- package/dist/evaluation-proposal-policy.js +91 -0
- package/dist/evaluation-source-retrieval.d.ts +68 -0
- package/dist/evaluation-source-retrieval.js +513 -0
- package/dist/evaluation-structure-policy.d.ts +29 -0
- package/dist/evaluation-structure-policy.js +138 -0
- package/dist/index.d.ts +124 -17
- package/dist/index.js +69 -11
- package/dist/inspection.js +2 -3
- package/dist/project-artifacts.d.ts +7 -2
- package/dist/project-artifacts.js +49 -136
- package/dist/project-authoring.d.ts +66 -5
- package/dist/project-authoring.js +783 -109
- package/dist/project-contracts.d.ts +419 -84
- package/dist/project-contracts.js +160 -52
- package/dist/project-store.js +2 -1
- package/dist/project-workflow.d.ts +5 -4
- package/dist/project-workflow.js +154 -35
- package/dist/repository-adversary-protocol.d.ts +64 -0
- package/dist/repository-adversary-protocol.js +105 -0
- package/dist/repository-behavior-protocol.d.ts +188 -0
- package/dist/repository-behavior-protocol.js +202 -0
- package/dist/repository-benchmark-protocol.d.ts +487 -0
- package/dist/repository-benchmark-protocol.js +96 -0
- package/dist/repository-execution-protocol.d.ts +150 -0
- package/dist/repository-execution-protocol.js +38 -0
- package/dist/repository-fixture-instructions.d.ts +3 -0
- package/dist/repository-fixture-instructions.js +91 -0
- package/dist/repository-fixture-protocol.d.ts +79 -0
- package/dist/repository-fixture-protocol.js +79 -0
- package/dist/repository-foundry-plan-protocol.d.ts +118 -0
- package/dist/repository-foundry-plan-protocol.js +296 -0
- package/dist/repository-foundry-progress-protocol.d.ts +52 -0
- package/dist/repository-foundry-progress-protocol.js +52 -0
- package/dist/repository-improvement-protocol.d.ts +100 -0
- package/dist/repository-improvement-protocol.js +106 -0
- package/dist/repository-language-model-protocol.d.ts +43 -0
- package/dist/repository-language-model-protocol.js +146 -0
- package/dist/repository-oracle-coverage-protocol.d.ts +18 -0
- package/dist/repository-oracle-coverage-protocol.js +39 -0
- package/dist/repository-oracle-execution-binding.d.ts +27 -0
- package/dist/repository-oracle-execution-binding.js +59 -0
- package/dist/repository-oracle-protocol.d.ts +230 -0
- package/dist/repository-oracle-protocol.js +156 -0
- package/dist/repository-oracle-scope-policy.d.ts +22 -0
- package/dist/repository-oracle-scope-policy.js +92 -0
- package/dist/repository-quality-policy.d.ts +15 -0
- package/dist/repository-quality-policy.js +357 -0
- package/dist/repository-routing-benchmark-protocol.d.ts +176 -0
- package/dist/repository-routing-benchmark-protocol.js +103 -0
- package/dist/repository-routing-model-protocol.d.ts +36 -0
- package/dist/repository-routing-model-protocol.js +89 -0
- package/dist/repository-routing-plan-protocol.d.ts +112 -0
- package/dist/repository-routing-plan-protocol.js +58 -0
- package/dist/repository-routing-quality-policy.d.ts +9 -0
- package/dist/repository-routing-quality-policy.js +191 -0
- package/dist/repository-seed-qualification-progress-protocol.d.ts +205 -0
- package/dist/repository-seed-qualification-progress-protocol.js +28 -0
- package/dist/repository-semantic-calibration-protocol.d.ts +768 -0
- package/dist/repository-semantic-calibration-protocol.js +276 -0
- package/dist/repository-semantic-calibration.d.ts +163 -0
- package/dist/repository-semantic-calibration.js +581 -0
- package/dist/repository-specification-contract-facts-protocol.d.ts +224 -0
- package/dist/repository-specification-contract-facts-protocol.js +276 -0
- package/dist/repository-specification-critique-protocol.d.ts +189 -0
- package/dist/repository-specification-critique-protocol.js +103 -0
- package/dist/repository-task-family-protocol.d.ts +24 -0
- package/dist/repository-task-family-protocol.js +37 -0
- package/dist/repository-task-seed-protocol.d.ts +384 -0
- package/dist/repository-task-seed-protocol.js +236 -0
- package/dist/repository-trajectory-protocol.d.ts +20 -0
- package/dist/repository-trajectory-protocol.js +42 -0
- package/dist/service.js +1 -1
- package/dist/services/adversary/service.d.ts +64 -0
- package/dist/services/adversary/service.js +330 -0
- package/dist/services/benchmark-compiler/service.d.ts +450 -0
- package/dist/services/benchmark-compiler/service.js +9 -0
- package/dist/services/budgeted-model/service.d.ts +118 -0
- package/dist/services/budgeted-model/service.js +460 -0
- package/dist/services/case-authoring/service.d.ts +163 -0
- package/dist/services/case-authoring/service.js +1456 -0
- package/dist/services/case-finalization/service.d.ts +283 -0
- package/dist/services/case-finalization/service.js +370 -0
- package/dist/services/case-generation/service.d.ts +619 -0
- package/dist/services/case-generation/service.js +2628 -0
- package/dist/services/case-pipeline/service.d.ts +31 -0
- package/dist/services/case-pipeline/service.js +485 -0
- package/dist/services/case-pipeline-v2/service.d.ts +70 -0
- package/dist/services/case-pipeline-v2/service.js +477 -0
- package/dist/services/command-observability/service.d.ts +13 -0
- package/dist/services/command-observability/service.js +3 -0
- package/dist/services/dimension-labeling/service.d.ts +77 -0
- package/dist/services/dimension-labeling/service.js +188 -0
- package/dist/services/eval-candidate/service.d.ts +208 -0
- package/dist/services/eval-candidate/service.js +64 -0
- package/dist/services/eval-capabilities/service.d.ts +183 -0
- package/dist/services/eval-capabilities/service.js +1433 -0
- package/dist/services/eval-environment/service.d.ts +173 -0
- package/dist/services/eval-environment/service.js +127 -0
- package/dist/services/evidence-reconstruction/service.d.ts +36 -0
- package/dist/services/evidence-reconstruction/service.js +145 -0
- package/dist/services/fixture-builder/service.d.ts +62 -0
- package/dist/services/fixture-builder/service.js +36 -0
- package/dist/services/fixture-validation/service.d.ts +75 -0
- package/dist/services/fixture-validation/service.js +295 -0
- package/dist/services/foundry/service.d.ts +831 -0
- package/dist/services/foundry/service.js +442 -0
- package/dist/services/foundry-progress/service.d.ts +62 -0
- package/dist/services/foundry-progress/service.js +149 -0
- package/dist/services/foundry-v2/service.d.ts +54 -0
- package/dist/services/foundry-v2/service.js +28 -0
- package/dist/services/grounded-authoring/service.d.ts +126 -0
- package/dist/services/grounded-authoring/service.js +822 -0
- package/dist/services/historical-case/service.d.ts +722 -0
- package/dist/services/historical-case/service.js +177 -0
- package/dist/services/improvement-loop/service.d.ts +59 -0
- package/dist/services/improvement-loop/service.js +176 -0
- package/dist/services/language-model/service.d.ts +52 -0
- package/dist/services/language-model/service.js +194 -0
- package/dist/services/oracle-builder/service.d.ts +146 -0
- package/dist/services/oracle-builder/service.js +513 -0
- package/dist/services/oracle-coverage/service.d.ts +28 -0
- package/dist/services/oracle-coverage/service.js +50 -0
- package/dist/services/oracle-coverage-witness/service.d.ts +130 -0
- package/dist/services/oracle-coverage-witness/service.js +538 -0
- package/dist/services/pipeline-challenge/service.d.ts +551 -0
- package/dist/services/pipeline-challenge/service.js +427 -0
- package/dist/services/pipeline-controls/service.d.ts +130 -0
- package/dist/services/pipeline-controls/service.js +483 -0
- package/dist/services/pipeline-oracle/service.d.ts +8 -0
- package/dist/services/pipeline-oracle/service.js +256 -0
- package/dist/services/pipeline-seed/service.d.ts +298 -0
- package/dist/services/pipeline-seed/service.js +428 -0
- package/dist/services/pipeline-spec/service.d.ts +103 -0
- package/dist/services/pipeline-spec/service.js +619 -0
- package/dist/services/pipeline-tournament/service.d.ts +258 -0
- package/dist/services/pipeline-tournament/service.js +476 -0
- package/dist/services/quality-gate/service.d.ts +233 -0
- package/dist/services/quality-gate/service.js +136 -0
- package/dist/services/repository-bundle/service.d.ts +33 -0
- package/dist/services/repository-bundle/service.js +114 -0
- package/dist/services/repository-model/service.d.ts +105 -0
- package/dist/services/repository-model/service.js +250 -0
- package/dist/services/repository-public-artifact/service.d.ts +133 -0
- package/dist/services/repository-public-artifact/service.js +330 -0
- package/dist/services/routing-benchmark/service.d.ts +362 -0
- package/dist/services/routing-benchmark/service.js +96 -0
- package/dist/services/specification-critic/service.d.ts +92 -0
- package/dist/services/specification-critic/service.js +172 -0
- package/dist/services/task-family/service.d.ts +40 -0
- package/dist/services/task-family/service.js +55 -0
- package/dist/services/task-seed/service.d.ts +906 -0
- package/dist/services/task-seed/service.js +1406 -0
- package/dist/services/task-specification/service.d.ts +27 -0
- package/dist/services/task-specification/service.js +40 -0
- package/dist/services/trajectory-policy/service.d.ts +110 -0
- package/dist/services/trajectory-policy/service.js +216 -0
- package/dist/test/agentic-capabilities-protocol.test.d.ts +1 -0
- package/dist/test/agentic-capabilities-protocol.test.js +570 -0
- package/dist/test/agentic-capabilities.test.d.ts +1 -0
- package/dist/test/agentic-capabilities.test.js +1461 -0
- package/dist/test/agentic-environment.test.d.ts +1 -0
- package/dist/test/agentic-environment.test.js +213 -0
- package/dist/test/case-pipeline-foundation.test.d.ts +1 -0
- package/dist/test/case-pipeline-foundation.test.js +535 -0
- package/dist/test/case-pipeline-protocol-v2.test.d.ts +1 -0
- package/dist/test/case-pipeline-protocol-v2.test.js +124 -0
- package/dist/test/case-pipeline-v2.test.d.ts +1 -0
- package/dist/test/case-pipeline-v2.test.js +286 -0
- package/dist/test/case-pipeline.test.d.ts +1 -0
- package/dist/test/case-pipeline.test.js +851 -0
- package/dist/test/eval-capability-policy.test.d.ts +1 -0
- package/dist/test/eval-capability-policy.test.js +50 -0
- package/dist/test/eval-event-log.test.d.ts +1 -0
- package/dist/test/eval-event-log.test.js +125 -0
- package/dist/test/evaluation-evidence-freshness.test.d.ts +1 -0
- package/dist/test/evaluation-evidence-freshness.test.js +44 -0
- package/dist/test/evaluation-evidence.test.d.ts +1 -0
- package/dist/test/evaluation-evidence.test.js +230 -0
- package/dist/test/evaluation-grader-calibration.test.d.ts +1 -0
- package/dist/test/evaluation-grader-calibration.test.js +373 -0
- package/dist/test/evaluation-proposal-digest.test.d.ts +1 -0
- package/dist/test/evaluation-proposal-digest.test.js +187 -0
- package/dist/test/evaluation-source-retrieval.test.d.ts +1 -0
- package/dist/test/evaluation-source-retrieval.test.js +237 -0
- package/dist/test/evaluation-structure-policy.test.d.ts +1 -0
- package/dist/test/evaluation-structure-policy.test.js +196 -0
- package/dist/test/fixtures/repository-resource-panel.d.ts +39 -0
- package/dist/test/fixtures/repository-resource-panel.js +111 -0
- package/dist/test/fixtures/vitest-boundary-panel.d.ts +84 -0
- package/dist/test/fixtures/vitest-boundary-panel.js +120 -0
- package/dist/test/fixtures/vitest-phase-panel.d.ts +135 -0
- package/dist/test/fixtures/vitest-phase-panel.js +213 -0
- package/dist/test/fixtures/vitest-reporter-results.d.ts +76 -0
- package/dist/test/fixtures/vitest-reporter-results.js +94 -0
- package/dist/test/grounded-authoring.test.d.ts +1 -0
- package/dist/test/grounded-authoring.test.js +565 -0
- package/dist/test/integrated-repository-history.test.d.ts +1 -0
- package/dist/test/integrated-repository-history.test.js +227 -0
- package/dist/test/project-authoring.test.js +593 -43
- package/dist/test/project-workflow.test.js +419 -40
- package/dist/test/repository-authoring-artifacts.test.d.ts +1 -0
- package/dist/test/repository-authoring-artifacts.test.js +185 -0
- package/dist/test/repository-bundle.test.d.ts +1 -0
- package/dist/test/repository-bundle.test.js +52 -0
- package/dist/test/repository-case-generation.test.d.ts +1 -0
- package/dist/test/repository-case-generation.test.js +3465 -0
- package/dist/test/repository-command-diagnostic.test.d.ts +1 -0
- package/dist/test/repository-command-diagnostic.test.js +55 -0
- package/dist/test/repository-command-signals.test.d.ts +1 -0
- package/dist/test/repository-command-signals.test.js +124 -0
- package/dist/test/repository-fixture-scope-coverage.test.d.ts +1 -0
- package/dist/test/repository-fixture-scope-coverage.test.js +127 -0
- package/dist/test/repository-fixture-validation.test.d.ts +1 -0
- package/dist/test/repository-fixture-validation.test.js +362 -0
- package/dist/test/repository-foundry-progress.test.d.ts +1 -0
- package/dist/test/repository-foundry-progress.test.js +110 -0
- package/dist/test/repository-foundry-quality.test.d.ts +1 -0
- package/dist/test/repository-foundry-quality.test.js +1138 -0
- package/dist/test/repository-import-context.test.d.ts +1 -0
- package/dist/test/repository-import-context.test.js +354 -0
- package/dist/test/repository-model-authoring.test.d.ts +1 -0
- package/dist/test/repository-model-authoring.test.js +544 -0
- package/dist/test/repository-model.test.d.ts +1 -0
- package/dist/test/repository-model.test.js +2195 -0
- package/dist/test/repository-node-test-reporter.test.d.ts +1 -0
- package/dist/test/repository-node-test-reporter.test.js +104 -0
- package/dist/test/repository-oracle-concurrency.test.d.ts +1 -0
- package/dist/test/repository-oracle-concurrency.test.js +542 -0
- package/dist/test/repository-oracle-coverage-witness.test.d.ts +1 -0
- package/dist/test/repository-oracle-coverage-witness.test.js +511 -0
- package/dist/test/repository-oracle-coverage.test.d.ts +1 -0
- package/dist/test/repository-oracle-coverage.test.js +168 -0
- package/dist/test/repository-oracle-evidence.test.d.ts +1 -0
- package/dist/test/repository-oracle-evidence.test.js +185 -0
- package/dist/test/repository-oracle-plan.test.d.ts +1 -0
- package/dist/test/repository-oracle-plan.test.js +176 -0
- package/dist/test/repository-overlay-isolation.test.d.ts +1 -0
- package/dist/test/repository-overlay-isolation.test.js +85 -0
- package/dist/test/repository-preparation-cache.test.d.ts +1 -0
- package/dist/test/repository-preparation-cache.test.js +414 -0
- package/dist/test/repository-public-artifact.test.d.ts +1 -0
- package/dist/test/repository-public-artifact.test.js +273 -0
- package/dist/test/repository-qualification-diagnostics.test.d.ts +1 -0
- package/dist/test/repository-qualification-diagnostics.test.js +524 -0
- package/dist/test/repository-reference-authoring.test.d.ts +1 -0
- package/dist/test/repository-reference-authoring.test.js +1633 -0
- package/dist/test/repository-review-evidence-v2.test.d.ts +1 -0
- package/dist/test/repository-review-evidence-v2.test.js +183 -0
- package/dist/test/repository-review-evidence.test.d.ts +1 -0
- package/dist/test/repository-review-evidence.test.js +124 -0
- package/dist/test/repository-seed-exclusions.test.d.ts +1 -0
- package/dist/test/repository-seed-exclusions.test.js +96 -0
- package/dist/test/repository-seed-selection.test.d.ts +1 -0
- package/dist/test/repository-seed-selection.test.js +504 -0
- package/dist/test/repository-semantic-calibration.test.d.ts +1 -0
- package/dist/test/repository-semantic-calibration.test.js +688 -0
- package/dist/test/repository-solution-edits.test.d.ts +1 -0
- package/dist/test/repository-solution-edits.test.js +377 -0
- package/dist/test/repository-specification-budget.test.d.ts +1 -0
- package/dist/test/repository-specification-budget.test.js +171 -0
- package/dist/test/repository-specification-contract-checkpoint.test.d.ts +1 -0
- package/dist/test/repository-specification-contract-checkpoint.test.js +228 -0
- package/dist/test/repository-specification-contract-facts.test.d.ts +1 -0
- package/dist/test/repository-specification-contract-facts.test.js +177 -0
- package/dist/test/repository-trajectory-authoring.test.d.ts +1 -0
- package/dist/test/repository-trajectory-authoring.test.js +176 -0
- package/dist/test/repository-valid-control-plan.test.d.ts +1 -0
- package/dist/test/repository-valid-control-plan.test.js +45 -0
- package/dist/test/repository-vitest-phase.test.d.ts +1 -0
- package/dist/test/repository-vitest-phase.test.js +848 -0
- package/dist/test/repository-vitest-reporter.test.d.ts +1 -0
- package/dist/test/repository-vitest-reporter.test.js +158 -0
- package/dist/test/repository-workspace-build.test.d.ts +1 -0
- package/dist/test/repository-workspace-build.test.js +160 -0
- package/dist/test/strict-authoring-schema.test.d.ts +1 -0
- package/dist/test/strict-authoring-schema.test.js +169 -0
- package/package.json +48 -6
|
@@ -0,0 +1,450 @@
|
|
|
1
|
+
import { Context, Effect, Layer } from "effect";
|
|
2
|
+
import type { RepositoryBenchmarkCaseV1, RepositoryValidatedBenchmarkLibraryV1 } from "../../repository-benchmark-protocol.js";
|
|
3
|
+
export type BenchmarkCompilerShape = {
|
|
4
|
+
readonly compile: (cases: readonly RepositoryBenchmarkCaseV1[]) => Effect.Effect<RepositoryValidatedBenchmarkLibraryV1>;
|
|
5
|
+
};
|
|
6
|
+
declare const BenchmarkCompiler_base: Context.ServiceClass<BenchmarkCompiler, "@velum-labs/routekit-eval-setup/BenchmarkCompiler", BenchmarkCompilerShape>;
|
|
7
|
+
export declare class BenchmarkCompiler extends BenchmarkCompiler_base {
|
|
8
|
+
}
|
|
9
|
+
export declare const compileRepositoryBenchmarkV1: (cases: readonly {
|
|
10
|
+
readonly id: string;
|
|
11
|
+
readonly version: 1;
|
|
12
|
+
readonly rejectionReasons: readonly string[];
|
|
13
|
+
readonly status: "rejected" | "valid-library" | "routing-eligible" | "routing-active";
|
|
14
|
+
readonly seedId: string;
|
|
15
|
+
readonly lane: "request" | "repository-agent";
|
|
16
|
+
readonly qualityFindings: readonly {
|
|
17
|
+
readonly axis: "coverage" | "stability" | "authenticity" | "specification-sufficiency" | "oracle-adequacy" | "distinctness" | "realism" | "leakage" | "triviality";
|
|
18
|
+
readonly outcome: "pass" | "fail" | "review-required";
|
|
19
|
+
readonly detail: string;
|
|
20
|
+
readonly evidence: readonly {
|
|
21
|
+
readonly kind: "symbol" | "package" | "public-surface" | "protocol" | "schema" | "test" | "command" | "history-episode" | "workload-signal" | "documentation";
|
|
22
|
+
readonly id: string;
|
|
23
|
+
readonly path?: string | undefined;
|
|
24
|
+
}[];
|
|
25
|
+
readonly evidenceIds: readonly string[];
|
|
26
|
+
}[];
|
|
27
|
+
readonly capabilityId: string;
|
|
28
|
+
readonly familyId: string;
|
|
29
|
+
readonly visible: {
|
|
30
|
+
readonly lane: "request";
|
|
31
|
+
readonly id: string;
|
|
32
|
+
readonly request: {
|
|
33
|
+
readonly endpoint: "chat" | "responses" | "anthropic";
|
|
34
|
+
readonly method: "POST";
|
|
35
|
+
readonly path: string;
|
|
36
|
+
readonly headers: readonly {
|
|
37
|
+
readonly name: string;
|
|
38
|
+
readonly value: string;
|
|
39
|
+
}[];
|
|
40
|
+
readonly bodyJson: string;
|
|
41
|
+
};
|
|
42
|
+
} | {
|
|
43
|
+
readonly lane: "repository-agent";
|
|
44
|
+
readonly id: string;
|
|
45
|
+
readonly requestText: string;
|
|
46
|
+
readonly checkoutCommit: string;
|
|
47
|
+
readonly allowedTools: readonly ("test" | "read" | "search" | "edit" | "shell")[];
|
|
48
|
+
readonly constraints: readonly string[];
|
|
49
|
+
};
|
|
50
|
+
readonly hidden: {
|
|
51
|
+
readonly version: 1;
|
|
52
|
+
readonly fixtures: readonly ({
|
|
53
|
+
readonly kind: "historical-regression";
|
|
54
|
+
readonly expectationMode: "changes";
|
|
55
|
+
readonly id: string;
|
|
56
|
+
readonly description: string;
|
|
57
|
+
readonly expectedBehavior: string;
|
|
58
|
+
readonly source: "historical" | "generated" | "reviewed";
|
|
59
|
+
readonly testPath: string;
|
|
60
|
+
} | {
|
|
61
|
+
readonly kind: "boundary";
|
|
62
|
+
readonly expectationMode: "changes" | "preserved";
|
|
63
|
+
readonly id: string;
|
|
64
|
+
readonly description: string;
|
|
65
|
+
readonly expectedBehavior: string;
|
|
66
|
+
readonly source: "historical" | "generated" | "reviewed";
|
|
67
|
+
readonly testPath: string;
|
|
68
|
+
} | {
|
|
69
|
+
readonly kind: "counterfactual";
|
|
70
|
+
readonly expectationMode: "changes";
|
|
71
|
+
readonly id: string;
|
|
72
|
+
readonly description: string;
|
|
73
|
+
readonly expectedBehavior: string;
|
|
74
|
+
readonly source: "historical" | "generated" | "reviewed";
|
|
75
|
+
readonly testPath: string;
|
|
76
|
+
} | {
|
|
77
|
+
readonly kind: "metamorphic";
|
|
78
|
+
readonly expectationMode: "preserved";
|
|
79
|
+
readonly id: string;
|
|
80
|
+
readonly description: string;
|
|
81
|
+
readonly expectedBehavior: string;
|
|
82
|
+
readonly source: "historical" | "generated" | "reviewed";
|
|
83
|
+
readonly testPath: string;
|
|
84
|
+
})[];
|
|
85
|
+
readonly caseId: string;
|
|
86
|
+
readonly hardFailurePolicy: "any-critical-hard-clause-fails";
|
|
87
|
+
readonly clauses: readonly ({
|
|
88
|
+
readonly kind: "candidate-policy";
|
|
89
|
+
readonly policyId: string;
|
|
90
|
+
readonly forbiddenPaths: readonly string[];
|
|
91
|
+
readonly id: string;
|
|
92
|
+
readonly critical: boolean;
|
|
93
|
+
readonly description: string;
|
|
94
|
+
} | {
|
|
95
|
+
readonly kind: "test-command";
|
|
96
|
+
readonly recipeId: string;
|
|
97
|
+
readonly expectedExitCode: number;
|
|
98
|
+
readonly fixtureIds: readonly string[];
|
|
99
|
+
readonly id: string;
|
|
100
|
+
readonly critical: boolean;
|
|
101
|
+
readonly description: string;
|
|
102
|
+
} | {
|
|
103
|
+
readonly kind: "json-field";
|
|
104
|
+
readonly id: string;
|
|
105
|
+
readonly description: string;
|
|
106
|
+
readonly critical: boolean;
|
|
107
|
+
readonly operator: "equals" | "present" | "absent";
|
|
108
|
+
readonly pointer: string;
|
|
109
|
+
readonly expectedJson?: string | undefined;
|
|
110
|
+
} | {
|
|
111
|
+
readonly kind: "ordered-events";
|
|
112
|
+
readonly id: string;
|
|
113
|
+
readonly description: string;
|
|
114
|
+
readonly critical: boolean;
|
|
115
|
+
readonly eventTypes: readonly string[];
|
|
116
|
+
readonly terminalEvent?: string | undefined;
|
|
117
|
+
} | {
|
|
118
|
+
readonly kind: "property";
|
|
119
|
+
readonly propertyId: string;
|
|
120
|
+
readonly fixtureIds: readonly string[];
|
|
121
|
+
readonly id: string;
|
|
122
|
+
readonly critical: boolean;
|
|
123
|
+
readonly description: string;
|
|
124
|
+
} | {
|
|
125
|
+
readonly kind: "differential";
|
|
126
|
+
readonly implementationIds: readonly string[];
|
|
127
|
+
readonly fixtureIds: readonly string[];
|
|
128
|
+
readonly id: string;
|
|
129
|
+
readonly critical: boolean;
|
|
130
|
+
readonly description: string;
|
|
131
|
+
} | {
|
|
132
|
+
readonly kind: "architecture";
|
|
133
|
+
readonly ruleId: string;
|
|
134
|
+
readonly expectedPaths: readonly string[];
|
|
135
|
+
readonly id: string;
|
|
136
|
+
readonly critical: boolean;
|
|
137
|
+
readonly description: string;
|
|
138
|
+
} | {
|
|
139
|
+
readonly kind: "transformation-completeness";
|
|
140
|
+
readonly ruleId: string;
|
|
141
|
+
readonly forbiddenLegacyPaths: readonly string[];
|
|
142
|
+
readonly requiredNewPaths: readonly string[];
|
|
143
|
+
readonly id: string;
|
|
144
|
+
readonly critical: boolean;
|
|
145
|
+
readonly description: string;
|
|
146
|
+
} | {
|
|
147
|
+
readonly kind: "semantic-residual";
|
|
148
|
+
readonly criterion: string;
|
|
149
|
+
readonly id: string;
|
|
150
|
+
readonly critical: boolean;
|
|
151
|
+
readonly description: string;
|
|
152
|
+
})[];
|
|
153
|
+
readonly solutions: readonly {
|
|
154
|
+
readonly kind: "historical-reference" | "independent-valid" | "simplified-valid" | "behavior-preserving-valid" | "historical-defect" | "mutation" | "model-near-miss" | "reward-hacking";
|
|
155
|
+
readonly id: string;
|
|
156
|
+
readonly expectedClass: "valid" | "wrong";
|
|
157
|
+
readonly family: string;
|
|
158
|
+
readonly commit?: string | undefined;
|
|
159
|
+
}[];
|
|
160
|
+
readonly observations: readonly {
|
|
161
|
+
readonly outcome: "pass" | "fail" | "unknown";
|
|
162
|
+
readonly solutionId: string;
|
|
163
|
+
readonly clauseId: string;
|
|
164
|
+
readonly repetition: number;
|
|
165
|
+
readonly detail: string;
|
|
166
|
+
readonly inconclusiveReason?: "unattributed-test-failure" | undefined;
|
|
167
|
+
}[];
|
|
168
|
+
readonly knownLimitations: readonly string[];
|
|
169
|
+
readonly executionBindingDigest?: string | undefined;
|
|
170
|
+
readonly executionRepetitions?: number | undefined;
|
|
171
|
+
};
|
|
172
|
+
readonly adequacy: {
|
|
173
|
+
readonly version: 1;
|
|
174
|
+
readonly caseId: string;
|
|
175
|
+
readonly admitted: boolean;
|
|
176
|
+
readonly validSolutionCount: number;
|
|
177
|
+
readonly wrongSolutionCount: number;
|
|
178
|
+
readonly falseRejectedSolutionIds: readonly string[];
|
|
179
|
+
readonly falseAcceptedSolutionIds: readonly string[];
|
|
180
|
+
readonly unstableSolutionIds: readonly string[];
|
|
181
|
+
readonly unobservedClauseIds: readonly string[];
|
|
182
|
+
readonly insufficientObservationPairs: readonly string[];
|
|
183
|
+
readonly mutationFamilyKills: readonly {
|
|
184
|
+
readonly family: string;
|
|
185
|
+
readonly killed: number;
|
|
186
|
+
readonly total: number;
|
|
187
|
+
}[];
|
|
188
|
+
readonly rejectionReasons: readonly string[];
|
|
189
|
+
};
|
|
190
|
+
readonly behavioralFingerprint: string;
|
|
191
|
+
readonly trajectoryPolicy?: {
|
|
192
|
+
readonly version: 1;
|
|
193
|
+
readonly caseId: string;
|
|
194
|
+
readonly purpose: "routing-signal";
|
|
195
|
+
readonly validationRecipeIds: readonly string[];
|
|
196
|
+
readonly protectedPaths: readonly string[];
|
|
197
|
+
readonly thresholds: {
|
|
198
|
+
readonly maximumToolCalls: number;
|
|
199
|
+
readonly maximumFailedToolCalls: number;
|
|
200
|
+
readonly maximumRepeatedFailedActions: number;
|
|
201
|
+
};
|
|
202
|
+
readonly requirements: {
|
|
203
|
+
readonly passingValidationAfterLastMutation: boolean;
|
|
204
|
+
readonly noProtectedPathMutations: boolean;
|
|
205
|
+
};
|
|
206
|
+
readonly rationale: string;
|
|
207
|
+
} | undefined;
|
|
208
|
+
}[]) => Effect.Effect<{
|
|
209
|
+
readonly version: 1;
|
|
210
|
+
readonly cases: readonly {
|
|
211
|
+
readonly id: string;
|
|
212
|
+
readonly version: 1;
|
|
213
|
+
readonly rejectionReasons: readonly string[];
|
|
214
|
+
readonly status: "rejected" | "valid-library" | "routing-eligible" | "routing-active";
|
|
215
|
+
readonly seedId: string;
|
|
216
|
+
readonly lane: "request" | "repository-agent";
|
|
217
|
+
readonly qualityFindings: readonly {
|
|
218
|
+
readonly axis: "coverage" | "stability" | "authenticity" | "specification-sufficiency" | "oracle-adequacy" | "distinctness" | "realism" | "leakage" | "triviality";
|
|
219
|
+
readonly outcome: "pass" | "fail" | "review-required";
|
|
220
|
+
readonly detail: string;
|
|
221
|
+
readonly evidence: readonly {
|
|
222
|
+
readonly kind: "symbol" | "package" | "public-surface" | "protocol" | "schema" | "test" | "command" | "history-episode" | "workload-signal" | "documentation";
|
|
223
|
+
readonly id: string;
|
|
224
|
+
readonly path?: string | undefined;
|
|
225
|
+
}[];
|
|
226
|
+
readonly evidenceIds: readonly string[];
|
|
227
|
+
}[];
|
|
228
|
+
readonly capabilityId: string;
|
|
229
|
+
readonly familyId: string;
|
|
230
|
+
readonly visible: {
|
|
231
|
+
readonly lane: "request";
|
|
232
|
+
readonly id: string;
|
|
233
|
+
readonly request: {
|
|
234
|
+
readonly endpoint: "chat" | "responses" | "anthropic";
|
|
235
|
+
readonly method: "POST";
|
|
236
|
+
readonly path: string;
|
|
237
|
+
readonly headers: readonly {
|
|
238
|
+
readonly name: string;
|
|
239
|
+
readonly value: string;
|
|
240
|
+
}[];
|
|
241
|
+
readonly bodyJson: string;
|
|
242
|
+
};
|
|
243
|
+
} | {
|
|
244
|
+
readonly lane: "repository-agent";
|
|
245
|
+
readonly id: string;
|
|
246
|
+
readonly requestText: string;
|
|
247
|
+
readonly checkoutCommit: string;
|
|
248
|
+
readonly allowedTools: readonly ("test" | "read" | "search" | "edit" | "shell")[];
|
|
249
|
+
readonly constraints: readonly string[];
|
|
250
|
+
};
|
|
251
|
+
readonly hidden: {
|
|
252
|
+
readonly version: 1;
|
|
253
|
+
readonly fixtures: readonly ({
|
|
254
|
+
readonly kind: "historical-regression";
|
|
255
|
+
readonly expectationMode: "changes";
|
|
256
|
+
readonly id: string;
|
|
257
|
+
readonly description: string;
|
|
258
|
+
readonly expectedBehavior: string;
|
|
259
|
+
readonly source: "historical" | "generated" | "reviewed";
|
|
260
|
+
readonly testPath: string;
|
|
261
|
+
} | {
|
|
262
|
+
readonly kind: "boundary";
|
|
263
|
+
readonly expectationMode: "changes" | "preserved";
|
|
264
|
+
readonly id: string;
|
|
265
|
+
readonly description: string;
|
|
266
|
+
readonly expectedBehavior: string;
|
|
267
|
+
readonly source: "historical" | "generated" | "reviewed";
|
|
268
|
+
readonly testPath: string;
|
|
269
|
+
} | {
|
|
270
|
+
readonly kind: "counterfactual";
|
|
271
|
+
readonly expectationMode: "changes";
|
|
272
|
+
readonly id: string;
|
|
273
|
+
readonly description: string;
|
|
274
|
+
readonly expectedBehavior: string;
|
|
275
|
+
readonly source: "historical" | "generated" | "reviewed";
|
|
276
|
+
readonly testPath: string;
|
|
277
|
+
} | {
|
|
278
|
+
readonly kind: "metamorphic";
|
|
279
|
+
readonly expectationMode: "preserved";
|
|
280
|
+
readonly id: string;
|
|
281
|
+
readonly description: string;
|
|
282
|
+
readonly expectedBehavior: string;
|
|
283
|
+
readonly source: "historical" | "generated" | "reviewed";
|
|
284
|
+
readonly testPath: string;
|
|
285
|
+
})[];
|
|
286
|
+
readonly caseId: string;
|
|
287
|
+
readonly hardFailurePolicy: "any-critical-hard-clause-fails";
|
|
288
|
+
readonly clauses: readonly ({
|
|
289
|
+
readonly kind: "candidate-policy";
|
|
290
|
+
readonly policyId: string;
|
|
291
|
+
readonly forbiddenPaths: readonly string[];
|
|
292
|
+
readonly id: string;
|
|
293
|
+
readonly critical: boolean;
|
|
294
|
+
readonly description: string;
|
|
295
|
+
} | {
|
|
296
|
+
readonly kind: "test-command";
|
|
297
|
+
readonly recipeId: string;
|
|
298
|
+
readonly expectedExitCode: number;
|
|
299
|
+
readonly fixtureIds: readonly string[];
|
|
300
|
+
readonly id: string;
|
|
301
|
+
readonly critical: boolean;
|
|
302
|
+
readonly description: string;
|
|
303
|
+
} | {
|
|
304
|
+
readonly kind: "json-field";
|
|
305
|
+
readonly id: string;
|
|
306
|
+
readonly description: string;
|
|
307
|
+
readonly critical: boolean;
|
|
308
|
+
readonly operator: "equals" | "present" | "absent";
|
|
309
|
+
readonly pointer: string;
|
|
310
|
+
readonly expectedJson?: string | undefined;
|
|
311
|
+
} | {
|
|
312
|
+
readonly kind: "ordered-events";
|
|
313
|
+
readonly id: string;
|
|
314
|
+
readonly description: string;
|
|
315
|
+
readonly critical: boolean;
|
|
316
|
+
readonly eventTypes: readonly string[];
|
|
317
|
+
readonly terminalEvent?: string | undefined;
|
|
318
|
+
} | {
|
|
319
|
+
readonly kind: "property";
|
|
320
|
+
readonly propertyId: string;
|
|
321
|
+
readonly fixtureIds: readonly string[];
|
|
322
|
+
readonly id: string;
|
|
323
|
+
readonly critical: boolean;
|
|
324
|
+
readonly description: string;
|
|
325
|
+
} | {
|
|
326
|
+
readonly kind: "differential";
|
|
327
|
+
readonly implementationIds: readonly string[];
|
|
328
|
+
readonly fixtureIds: readonly string[];
|
|
329
|
+
readonly id: string;
|
|
330
|
+
readonly critical: boolean;
|
|
331
|
+
readonly description: string;
|
|
332
|
+
} | {
|
|
333
|
+
readonly kind: "architecture";
|
|
334
|
+
readonly ruleId: string;
|
|
335
|
+
readonly expectedPaths: readonly string[];
|
|
336
|
+
readonly id: string;
|
|
337
|
+
readonly critical: boolean;
|
|
338
|
+
readonly description: string;
|
|
339
|
+
} | {
|
|
340
|
+
readonly kind: "transformation-completeness";
|
|
341
|
+
readonly ruleId: string;
|
|
342
|
+
readonly forbiddenLegacyPaths: readonly string[];
|
|
343
|
+
readonly requiredNewPaths: readonly string[];
|
|
344
|
+
readonly id: string;
|
|
345
|
+
readonly critical: boolean;
|
|
346
|
+
readonly description: string;
|
|
347
|
+
} | {
|
|
348
|
+
readonly kind: "semantic-residual";
|
|
349
|
+
readonly criterion: string;
|
|
350
|
+
readonly id: string;
|
|
351
|
+
readonly critical: boolean;
|
|
352
|
+
readonly description: string;
|
|
353
|
+
})[];
|
|
354
|
+
readonly solutions: readonly {
|
|
355
|
+
readonly kind: "historical-reference" | "independent-valid" | "simplified-valid" | "behavior-preserving-valid" | "historical-defect" | "mutation" | "model-near-miss" | "reward-hacking";
|
|
356
|
+
readonly id: string;
|
|
357
|
+
readonly expectedClass: "valid" | "wrong";
|
|
358
|
+
readonly family: string;
|
|
359
|
+
readonly commit?: string | undefined;
|
|
360
|
+
}[];
|
|
361
|
+
readonly observations: readonly {
|
|
362
|
+
readonly outcome: "pass" | "fail" | "unknown";
|
|
363
|
+
readonly solutionId: string;
|
|
364
|
+
readonly clauseId: string;
|
|
365
|
+
readonly repetition: number;
|
|
366
|
+
readonly detail: string;
|
|
367
|
+
readonly inconclusiveReason?: "unattributed-test-failure" | undefined;
|
|
368
|
+
}[];
|
|
369
|
+
readonly knownLimitations: readonly string[];
|
|
370
|
+
readonly executionBindingDigest?: string | undefined;
|
|
371
|
+
readonly executionRepetitions?: number | undefined;
|
|
372
|
+
};
|
|
373
|
+
readonly adequacy: {
|
|
374
|
+
readonly version: 1;
|
|
375
|
+
readonly caseId: string;
|
|
376
|
+
readonly admitted: boolean;
|
|
377
|
+
readonly validSolutionCount: number;
|
|
378
|
+
readonly wrongSolutionCount: number;
|
|
379
|
+
readonly falseRejectedSolutionIds: readonly string[];
|
|
380
|
+
readonly falseAcceptedSolutionIds: readonly string[];
|
|
381
|
+
readonly unstableSolutionIds: readonly string[];
|
|
382
|
+
readonly unobservedClauseIds: readonly string[];
|
|
383
|
+
readonly insufficientObservationPairs: readonly string[];
|
|
384
|
+
readonly mutationFamilyKills: readonly {
|
|
385
|
+
readonly family: string;
|
|
386
|
+
readonly killed: number;
|
|
387
|
+
readonly total: number;
|
|
388
|
+
}[];
|
|
389
|
+
readonly rejectionReasons: readonly string[];
|
|
390
|
+
};
|
|
391
|
+
readonly behavioralFingerprint: string;
|
|
392
|
+
readonly trajectoryPolicy?: {
|
|
393
|
+
readonly version: 1;
|
|
394
|
+
readonly caseId: string;
|
|
395
|
+
readonly purpose: "routing-signal";
|
|
396
|
+
readonly validationRecipeIds: readonly string[];
|
|
397
|
+
readonly protectedPaths: readonly string[];
|
|
398
|
+
readonly thresholds: {
|
|
399
|
+
readonly maximumToolCalls: number;
|
|
400
|
+
readonly maximumFailedToolCalls: number;
|
|
401
|
+
readonly maximumRepeatedFailedActions: number;
|
|
402
|
+
};
|
|
403
|
+
readonly requirements: {
|
|
404
|
+
readonly passingValidationAfterLastMutation: boolean;
|
|
405
|
+
readonly noProtectedPathMutations: boolean;
|
|
406
|
+
};
|
|
407
|
+
readonly rationale: string;
|
|
408
|
+
} | undefined;
|
|
409
|
+
}[];
|
|
410
|
+
readonly rejectedCaseIds: readonly string[];
|
|
411
|
+
readonly acceptanceFunnel: readonly {
|
|
412
|
+
readonly gate: string;
|
|
413
|
+
readonly input: number;
|
|
414
|
+
readonly admitted: number;
|
|
415
|
+
readonly rejected: number;
|
|
416
|
+
}[];
|
|
417
|
+
readonly coverage: {
|
|
418
|
+
readonly capabilities: readonly {
|
|
419
|
+
readonly id: string;
|
|
420
|
+
readonly count: number;
|
|
421
|
+
}[];
|
|
422
|
+
readonly families: readonly {
|
|
423
|
+
readonly id: string;
|
|
424
|
+
readonly count: number;
|
|
425
|
+
}[];
|
|
426
|
+
readonly lanes: readonly {
|
|
427
|
+
readonly id: string;
|
|
428
|
+
readonly count: number;
|
|
429
|
+
}[];
|
|
430
|
+
readonly packages: readonly {
|
|
431
|
+
readonly id: string;
|
|
432
|
+
readonly count: number;
|
|
433
|
+
}[];
|
|
434
|
+
readonly oracleKinds: readonly {
|
|
435
|
+
readonly id: string;
|
|
436
|
+
readonly count: number;
|
|
437
|
+
}[];
|
|
438
|
+
readonly fixtureKinds: readonly {
|
|
439
|
+
readonly id: string;
|
|
440
|
+
readonly count: number;
|
|
441
|
+
}[];
|
|
442
|
+
readonly mutationFamilies: readonly {
|
|
443
|
+
readonly id: string;
|
|
444
|
+
readonly count: number;
|
|
445
|
+
}[];
|
|
446
|
+
};
|
|
447
|
+
}, never, never>;
|
|
448
|
+
export declare const makeBenchmarkCompiler: Effect.Effect<BenchmarkCompilerShape, never, never>;
|
|
449
|
+
export declare const BenchmarkCompilerLive: Layer.Layer<BenchmarkCompiler, never, never>;
|
|
450
|
+
export {};
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { Context, Effect, Layer } from "effect";
|
|
2
|
+
import { compileValidatedBenchmarkLibraryV1 } from "../../repository-quality-policy.js";
|
|
3
|
+
export class BenchmarkCompiler extends Context.Service()("@velum-labs/routekit-eval-setup/BenchmarkCompiler") {
|
|
4
|
+
}
|
|
5
|
+
export const compileRepositoryBenchmarkV1 = Effect.fn("BenchmarkCompiler.compile")(function* (cases) {
|
|
6
|
+
return compileValidatedBenchmarkLibraryV1(cases);
|
|
7
|
+
});
|
|
8
|
+
export const makeBenchmarkCompiler = Effect.succeed(BenchmarkCompiler.of({ compile: compileRepositoryBenchmarkV1 }));
|
|
9
|
+
export const BenchmarkCompilerLive = Layer.effect(BenchmarkCompiler, makeBenchmarkCompiler);
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
import { Context, Effect, Layer, Schema } from "effect";
|
|
2
|
+
import type { CaseCheckpointStoreV1 } from "../../case-checkpoint-store.js";
|
|
3
|
+
import { type HostFeedbackFindingV1, HostFeedbackV1, type PipelineWorkCheckpointStoreV1, PipelineBudgetExhaustedError, type PipelineBudgetV1, type PipelineStateV1 } from "../../case-pipeline-protocol.js";
|
|
4
|
+
import { RepositoryFoundryError } from "../../errors.js";
|
|
5
|
+
import { type RepositoryFoundryModelPlanV1, type RepositoryFoundryModelRoleV1, RepositoryFoundryReasoningEffortV1 } from "../../repository-language-model-protocol.js";
|
|
6
|
+
import { RepositoryFoundryLanguageModel, type RepositoryFoundryLanguageModelShape, type RepositoryStructuredModelAssignmentV1 } from "../language-model/service.js";
|
|
7
|
+
/**
|
|
8
|
+
* Wraps a RepositoryFoundryLanguageModel so every structured call in the
|
|
9
|
+
* pipeline, including calls made by reused substrate services, is budgeted
|
|
10
|
+
* by the single global budget, ledgered, persisted, and retried on transport
|
|
11
|
+
* failures. A call is counted when a response is received, not when it
|
|
12
|
+
* validates.
|
|
13
|
+
*/
|
|
14
|
+
export type PipelineStateAccessorV1 = {
|
|
15
|
+
readonly get: Effect.Effect<PipelineStateV1>;
|
|
16
|
+
readonly set: (state: PipelineStateV1) => Effect.Effect<void>;
|
|
17
|
+
};
|
|
18
|
+
/** Ref-backed accessor shared by the orchestrator and the budget wrapper. */
|
|
19
|
+
export declare const makePipelineStateAccessorV1: (initial: PipelineStateV1) => Effect.Effect<PipelineStateAccessorV1>;
|
|
20
|
+
export type PipelineBudgetSnapshotV1 = {
|
|
21
|
+
readonly modelCalls: number;
|
|
22
|
+
readonly elapsedMs: number;
|
|
23
|
+
readonly remainingCalls: number;
|
|
24
|
+
readonly remainingMs: number;
|
|
25
|
+
};
|
|
26
|
+
export type PipelineBudgetTrackerShape = {
|
|
27
|
+
readonly budget: PipelineBudgetV1;
|
|
28
|
+
readonly snapshot: Effect.Effect<PipelineBudgetSnapshotV1>;
|
|
29
|
+
/** Fails with PipelineBudgetExhaustedError when no call or wall time remains. */
|
|
30
|
+
readonly assertAvailable: Effect.Effect<void, PipelineBudgetExhaustedError>;
|
|
31
|
+
};
|
|
32
|
+
declare const PipelineBudgetTracker_base: Context.ServiceClass<PipelineBudgetTracker, "@velum-labs/routekit-eval-setup/PipelineBudgetTracker", PipelineBudgetTrackerShape>;
|
|
33
|
+
export declare class PipelineBudgetTracker extends PipelineBudgetTracker_base {
|
|
34
|
+
}
|
|
35
|
+
export declare const budgetSnapshotV1: Effect.Effect<PipelineBudgetSnapshotV1, never, PipelineBudgetTracker>;
|
|
36
|
+
export declare const TRANSPORT_RETRY_BACKOFF_MS: readonly [5000, 15000, 45000, 90000, 180000];
|
|
37
|
+
export declare const TRANSPORT_ERROR_MARKERS: readonly ["transport error", "before response headers", "econnrefused", "econnreset", "fetch failed", "timed out", "socket", "5xx", "502", "503", "504", "429", "stream"];
|
|
38
|
+
export declare const STRUCTURED_OUTPUT_INVALID_PREFIX = "structured-output-invalid:";
|
|
39
|
+
/** Transport failures are retried and are never a terminal outcome of a case. */
|
|
40
|
+
export declare const isTransportFailureV1: (error: RepositoryFoundryError) => boolean;
|
|
41
|
+
export declare const isStructuredOutputInvalidV1: (error: unknown) => boolean;
|
|
42
|
+
export declare const BUDGET_EXHAUSTED_PREFIX = "budget-exhausted:";
|
|
43
|
+
/**
|
|
44
|
+
* The language-model shape only carries RepositoryFoundryError, so the wrapper
|
|
45
|
+
* fails with a tagged RepositoryFoundryError whose cause is the typed budget
|
|
46
|
+
* error. Substrate services propagate it unchanged; stage and orchestrator
|
|
47
|
+
* code unwraps it with `normalizePipelineFailureV1`.
|
|
48
|
+
*/
|
|
49
|
+
export declare const budgetExhaustedErrorOfV1: (error: unknown) => PipelineBudgetExhaustedError | undefined;
|
|
50
|
+
export declare const normalizePipelineFailureV1: (error: RepositoryFoundryError | PipelineBudgetExhaustedError) => RepositoryFoundryError | PipelineBudgetExhaustedError;
|
|
51
|
+
/** The raw response text the wrapper captured for a received-but-invalid response. */
|
|
52
|
+
export declare const structuredOutputInvalidResponseTextV1: (error: RepositoryFoundryError) => string | undefined;
|
|
53
|
+
export type BudgetedLanguageModelInputV1 = {
|
|
54
|
+
readonly store: CaseCheckpointStoreV1;
|
|
55
|
+
readonly state: PipelineStateAccessorV1;
|
|
56
|
+
readonly budget: PipelineBudgetV1;
|
|
57
|
+
/** Wall clock at the start of this run; prior resumes are in state.elapsedMs. */
|
|
58
|
+
readonly startedAtMs: number;
|
|
59
|
+
readonly inner: RepositoryFoundryLanguageModelShape;
|
|
60
|
+
readonly sleep?: (ms: number) => Effect.Effect<void>;
|
|
61
|
+
readonly now?: Effect.Effect<number>;
|
|
62
|
+
};
|
|
63
|
+
export declare const makeBudgetedLanguageModelV1: (input: BudgetedLanguageModelInputV1) => Effect.Effect<{
|
|
64
|
+
languageModel: RepositoryFoundryLanguageModelShape;
|
|
65
|
+
tracker: PipelineBudgetTrackerShape;
|
|
66
|
+
}, never, never>;
|
|
67
|
+
/** Provides both the wrapped language model and the budget tracker. */
|
|
68
|
+
export declare const BudgetedLanguageModelLive: (input: BudgetedLanguageModelInputV1) => Layer.Layer<RepositoryFoundryLanguageModel | PipelineBudgetTracker>;
|
|
69
|
+
export declare const FEEDBACK_LOOP_UNRESOLVED_PREFIX = "feedback-loop-unresolved:";
|
|
70
|
+
export declare const DEFAULT_FEEDBACK_MAX_ATTEMPTS = 12;
|
|
71
|
+
export type GenerateWithFeedbackTargetV1 = {
|
|
72
|
+
readonly assignment: RepositoryStructuredModelAssignmentV1<string>;
|
|
73
|
+
readonly plan?: undefined;
|
|
74
|
+
readonly role?: undefined;
|
|
75
|
+
} | {
|
|
76
|
+
readonly plan: RepositoryFoundryModelPlanV1;
|
|
77
|
+
readonly role: RepositoryFoundryModelRoleV1;
|
|
78
|
+
readonly assignment?: undefined;
|
|
79
|
+
};
|
|
80
|
+
export type GenerateWithFeedbackInputV1<A, E, R> = GenerateWithFeedbackTargetV1 & {
|
|
81
|
+
/** Defaults to the RepositoryFoundryLanguageModel service in context. */
|
|
82
|
+
readonly languageModel?: RepositoryFoundryLanguageModelShape;
|
|
83
|
+
readonly operationId: string;
|
|
84
|
+
readonly instructions: string;
|
|
85
|
+
readonly input: unknown;
|
|
86
|
+
readonly schemaName: string;
|
|
87
|
+
readonly outputSchema: Schema.Codec<A, unknown, never, unknown>;
|
|
88
|
+
readonly maximumOutputTokens: number;
|
|
89
|
+
/** Durable authoring only. The binding includes every host-only validation input. */
|
|
90
|
+
readonly checkpoint?: {
|
|
91
|
+
readonly store: PipelineWorkCheckpointStoreV1;
|
|
92
|
+
readonly binding: unknown;
|
|
93
|
+
};
|
|
94
|
+
/** Empty findings accept the value; anything else re-calls the same role. */
|
|
95
|
+
readonly validate: (value: A) => Effect.Effect<readonly HostFeedbackFindingV1[], E, R>;
|
|
96
|
+
readonly maxAttempts?: number;
|
|
97
|
+
};
|
|
98
|
+
export type GenerateWithFeedbackResultV1<A> = {
|
|
99
|
+
readonly value: A;
|
|
100
|
+
readonly attempts: number;
|
|
101
|
+
readonly call: {
|
|
102
|
+
readonly operationId: string;
|
|
103
|
+
readonly role: string;
|
|
104
|
+
readonly model: string;
|
|
105
|
+
readonly reasoningEffort: RepositoryFoundryReasoningEffortV1;
|
|
106
|
+
readonly callId?: string;
|
|
107
|
+
};
|
|
108
|
+
/** Every feedback packet that was sent back, in order. */
|
|
109
|
+
readonly feedback: readonly HostFeedbackV1[];
|
|
110
|
+
};
|
|
111
|
+
/**
|
|
112
|
+
* Model proposes, host verifies, same role repairs. Shortfalls become
|
|
113
|
+
* hostFeedback instead of terminal failures; only the global budget bounds
|
|
114
|
+
* the loop. A received response and completed feedback survive interruption;
|
|
115
|
+
* resuming revalidates the response, never treats authoring as executed proof.
|
|
116
|
+
*/
|
|
117
|
+
export declare const generateWithFeedbackV1: <A, E = never, R = never>(input: GenerateWithFeedbackInputV1<A, E, R>) => Effect.Effect<GenerateWithFeedbackResultV1<A>, RepositoryFoundryError | PipelineBudgetExhaustedError | E, RepositoryFoundryLanguageModel | R>;
|
|
118
|
+
export {};
|