@velum-labs/routekit-eval-setup 1.3.2 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/authoring-responses-request.d.ts +5 -0
- package/dist/adapters/authoring-responses-request.js +38 -0
- package/dist/adapters/evaluation-evidence-freshness.d.ts +7 -0
- package/dist/adapters/evaluation-evidence-freshness.js +76 -0
- package/dist/adapters/git-task-history.d.ts +67 -0
- package/dist/adapters/git-task-history.js +171 -0
- package/dist/adapters/integrated-repository-history.d.ts +21 -0
- package/dist/adapters/integrated-repository-history.js +175 -0
- package/dist/adapters/repository-command-diagnostic.d.ts +8 -0
- package/dist/adapters/repository-command-diagnostic.js +46 -0
- package/dist/adapters/repository-command-evidence.d.ts +13 -0
- package/dist/adapters/repository-command-evidence.js +102 -0
- package/dist/adapters/repository-command-runner.d.ts +238 -0
- package/dist/adapters/repository-command-runner.js +1483 -0
- package/dist/adapters/repository-import-context.d.ts +47 -0
- package/dist/adapters/repository-import-context.js +469 -0
- package/dist/adapters/repository-node-test-reporter.d.ts +3 -0
- package/dist/adapters/repository-node-test-reporter.js +27 -0
- package/dist/adapters/repository-review-evidence.d.ts +39 -0
- package/dist/adapters/repository-review-evidence.js +632 -0
- package/dist/adapters/repository-seed-selection.d.ts +7 -0
- package/dist/adapters/repository-seed-selection.js +79 -0
- package/dist/adapters/repository-solution-edits.d.ts +49 -0
- package/dist/adapters/repository-solution-edits.js +136 -0
- package/dist/adapters/repository-vitest-phase-adapter.d.ts +8 -0
- package/dist/adapters/repository-vitest-phase-adapter.js +310 -0
- package/dist/adapters/repository-vitest-reporter.d.ts +24 -0
- package/dist/adapters/repository-vitest-reporter.js +314 -0
- package/dist/adapters/strict-authoring-schema.d.ts +5 -0
- package/dist/adapters/strict-authoring-schema.js +158 -0
- package/dist/adapters/test-discovery.d.ts +30 -0
- package/dist/adapters/test-discovery.js +124 -0
- package/dist/adapters/typescript-repository-index.d.ts +51 -0
- package/dist/adapters/typescript-repository-index.js +226 -0
- package/dist/agentic-capabilities-protocol.d.ts +1373 -0
- package/dist/agentic-capabilities-protocol.js +786 -0
- package/dist/case-checkpoint-store.d.ts +29 -0
- package/dist/case-checkpoint-store.js +133 -0
- package/dist/case-pipeline-protocol-v2.d.ts +184 -0
- package/dist/case-pipeline-protocol-v2.js +193 -0
- package/dist/case-pipeline-protocol.d.ts +2626 -0
- package/dist/case-pipeline-protocol.js +371 -0
- package/dist/effect-api.d.ts +74 -10
- package/dist/effect-api.js +56 -6
- package/dist/errors.d.ts +31 -0
- package/dist/errors.js +10 -0
- package/dist/eval-capability-execution-envelope.d.ts +64 -0
- package/dist/eval-capability-execution-envelope.js +98 -0
- package/dist/eval-capability-policy.d.ts +90 -0
- package/dist/eval-capability-policy.js +107 -0
- package/dist/eval-event-log.d.ts +140 -0
- package/dist/eval-event-log.js +220 -0
- package/dist/evaluation-authoring-policy.d.ts +18 -0
- package/dist/evaluation-authoring-policy.js +19 -0
- package/dist/evaluation-authoring-validation.d.ts +22 -0
- package/dist/evaluation-authoring-validation.js +72 -0
- package/dist/evaluation-evidence.d.ts +20 -0
- package/dist/evaluation-evidence.js +319 -0
- package/dist/evaluation-grader-calibration-protocol.d.ts +108 -0
- package/dist/evaluation-grader-calibration-protocol.js +80 -0
- package/dist/evaluation-grader-calibration.d.ts +18 -0
- package/dist/evaluation-grader-calibration.js +334 -0
- package/dist/evaluation-grading-policy.d.ts +24 -0
- package/dist/evaluation-grading-policy.js +54 -0
- package/dist/evaluation-proposal-policy.d.ts +4 -0
- package/dist/evaluation-proposal-policy.js +91 -0
- package/dist/evaluation-source-retrieval.d.ts +68 -0
- package/dist/evaluation-source-retrieval.js +513 -0
- package/dist/evaluation-structure-policy.d.ts +29 -0
- package/dist/evaluation-structure-policy.js +138 -0
- package/dist/index.d.ts +124 -17
- package/dist/index.js +69 -11
- package/dist/inspection.js +2 -3
- package/dist/project-artifacts.d.ts +7 -2
- package/dist/project-artifacts.js +49 -136
- package/dist/project-authoring.d.ts +66 -5
- package/dist/project-authoring.js +783 -109
- package/dist/project-contracts.d.ts +419 -84
- package/dist/project-contracts.js +160 -52
- package/dist/project-store.js +2 -1
- package/dist/project-workflow.d.ts +5 -4
- package/dist/project-workflow.js +154 -35
- package/dist/repository-adversary-protocol.d.ts +64 -0
- package/dist/repository-adversary-protocol.js +105 -0
- package/dist/repository-behavior-protocol.d.ts +188 -0
- package/dist/repository-behavior-protocol.js +202 -0
- package/dist/repository-benchmark-protocol.d.ts +487 -0
- package/dist/repository-benchmark-protocol.js +96 -0
- package/dist/repository-execution-protocol.d.ts +150 -0
- package/dist/repository-execution-protocol.js +38 -0
- package/dist/repository-fixture-instructions.d.ts +3 -0
- package/dist/repository-fixture-instructions.js +91 -0
- package/dist/repository-fixture-protocol.d.ts +79 -0
- package/dist/repository-fixture-protocol.js +79 -0
- package/dist/repository-foundry-plan-protocol.d.ts +118 -0
- package/dist/repository-foundry-plan-protocol.js +296 -0
- package/dist/repository-foundry-progress-protocol.d.ts +52 -0
- package/dist/repository-foundry-progress-protocol.js +52 -0
- package/dist/repository-improvement-protocol.d.ts +100 -0
- package/dist/repository-improvement-protocol.js +106 -0
- package/dist/repository-language-model-protocol.d.ts +43 -0
- package/dist/repository-language-model-protocol.js +146 -0
- package/dist/repository-oracle-coverage-protocol.d.ts +18 -0
- package/dist/repository-oracle-coverage-protocol.js +39 -0
- package/dist/repository-oracle-execution-binding.d.ts +27 -0
- package/dist/repository-oracle-execution-binding.js +59 -0
- package/dist/repository-oracle-protocol.d.ts +230 -0
- package/dist/repository-oracle-protocol.js +156 -0
- package/dist/repository-oracle-scope-policy.d.ts +22 -0
- package/dist/repository-oracle-scope-policy.js +92 -0
- package/dist/repository-quality-policy.d.ts +15 -0
- package/dist/repository-quality-policy.js +357 -0
- package/dist/repository-routing-benchmark-protocol.d.ts +176 -0
- package/dist/repository-routing-benchmark-protocol.js +103 -0
- package/dist/repository-routing-model-protocol.d.ts +36 -0
- package/dist/repository-routing-model-protocol.js +89 -0
- package/dist/repository-routing-plan-protocol.d.ts +112 -0
- package/dist/repository-routing-plan-protocol.js +58 -0
- package/dist/repository-routing-quality-policy.d.ts +9 -0
- package/dist/repository-routing-quality-policy.js +191 -0
- package/dist/repository-seed-qualification-progress-protocol.d.ts +205 -0
- package/dist/repository-seed-qualification-progress-protocol.js +28 -0
- package/dist/repository-semantic-calibration-protocol.d.ts +768 -0
- package/dist/repository-semantic-calibration-protocol.js +276 -0
- package/dist/repository-semantic-calibration.d.ts +163 -0
- package/dist/repository-semantic-calibration.js +581 -0
- package/dist/repository-specification-contract-facts-protocol.d.ts +224 -0
- package/dist/repository-specification-contract-facts-protocol.js +276 -0
- package/dist/repository-specification-critique-protocol.d.ts +189 -0
- package/dist/repository-specification-critique-protocol.js +103 -0
- package/dist/repository-task-family-protocol.d.ts +24 -0
- package/dist/repository-task-family-protocol.js +37 -0
- package/dist/repository-task-seed-protocol.d.ts +384 -0
- package/dist/repository-task-seed-protocol.js +236 -0
- package/dist/repository-trajectory-protocol.d.ts +20 -0
- package/dist/repository-trajectory-protocol.js +42 -0
- package/dist/service.js +1 -1
- package/dist/services/adversary/service.d.ts +64 -0
- package/dist/services/adversary/service.js +330 -0
- package/dist/services/benchmark-compiler/service.d.ts +450 -0
- package/dist/services/benchmark-compiler/service.js +9 -0
- package/dist/services/budgeted-model/service.d.ts +118 -0
- package/dist/services/budgeted-model/service.js +460 -0
- package/dist/services/case-authoring/service.d.ts +163 -0
- package/dist/services/case-authoring/service.js +1456 -0
- package/dist/services/case-finalization/service.d.ts +283 -0
- package/dist/services/case-finalization/service.js +370 -0
- package/dist/services/case-generation/service.d.ts +619 -0
- package/dist/services/case-generation/service.js +2628 -0
- package/dist/services/case-pipeline/service.d.ts +31 -0
- package/dist/services/case-pipeline/service.js +485 -0
- package/dist/services/case-pipeline-v2/service.d.ts +70 -0
- package/dist/services/case-pipeline-v2/service.js +477 -0
- package/dist/services/command-observability/service.d.ts +13 -0
- package/dist/services/command-observability/service.js +3 -0
- package/dist/services/dimension-labeling/service.d.ts +77 -0
- package/dist/services/dimension-labeling/service.js +188 -0
- package/dist/services/eval-candidate/service.d.ts +208 -0
- package/dist/services/eval-candidate/service.js +64 -0
- package/dist/services/eval-capabilities/service.d.ts +183 -0
- package/dist/services/eval-capabilities/service.js +1433 -0
- package/dist/services/eval-environment/service.d.ts +173 -0
- package/dist/services/eval-environment/service.js +127 -0
- package/dist/services/evidence-reconstruction/service.d.ts +36 -0
- package/dist/services/evidence-reconstruction/service.js +145 -0
- package/dist/services/fixture-builder/service.d.ts +62 -0
- package/dist/services/fixture-builder/service.js +36 -0
- package/dist/services/fixture-validation/service.d.ts +75 -0
- package/dist/services/fixture-validation/service.js +295 -0
- package/dist/services/foundry/service.d.ts +831 -0
- package/dist/services/foundry/service.js +442 -0
- package/dist/services/foundry-progress/service.d.ts +62 -0
- package/dist/services/foundry-progress/service.js +149 -0
- package/dist/services/foundry-v2/service.d.ts +54 -0
- package/dist/services/foundry-v2/service.js +28 -0
- package/dist/services/grounded-authoring/service.d.ts +126 -0
- package/dist/services/grounded-authoring/service.js +822 -0
- package/dist/services/historical-case/service.d.ts +722 -0
- package/dist/services/historical-case/service.js +177 -0
- package/dist/services/improvement-loop/service.d.ts +59 -0
- package/dist/services/improvement-loop/service.js +176 -0
- package/dist/services/language-model/service.d.ts +52 -0
- package/dist/services/language-model/service.js +194 -0
- package/dist/services/oracle-builder/service.d.ts +146 -0
- package/dist/services/oracle-builder/service.js +513 -0
- package/dist/services/oracle-coverage/service.d.ts +28 -0
- package/dist/services/oracle-coverage/service.js +50 -0
- package/dist/services/oracle-coverage-witness/service.d.ts +130 -0
- package/dist/services/oracle-coverage-witness/service.js +538 -0
- package/dist/services/pipeline-challenge/service.d.ts +551 -0
- package/dist/services/pipeline-challenge/service.js +427 -0
- package/dist/services/pipeline-controls/service.d.ts +130 -0
- package/dist/services/pipeline-controls/service.js +483 -0
- package/dist/services/pipeline-oracle/service.d.ts +8 -0
- package/dist/services/pipeline-oracle/service.js +256 -0
- package/dist/services/pipeline-seed/service.d.ts +298 -0
- package/dist/services/pipeline-seed/service.js +428 -0
- package/dist/services/pipeline-spec/service.d.ts +103 -0
- package/dist/services/pipeline-spec/service.js +619 -0
- package/dist/services/pipeline-tournament/service.d.ts +258 -0
- package/dist/services/pipeline-tournament/service.js +476 -0
- package/dist/services/quality-gate/service.d.ts +233 -0
- package/dist/services/quality-gate/service.js +136 -0
- package/dist/services/repository-bundle/service.d.ts +33 -0
- package/dist/services/repository-bundle/service.js +114 -0
- package/dist/services/repository-model/service.d.ts +105 -0
- package/dist/services/repository-model/service.js +250 -0
- package/dist/services/repository-public-artifact/service.d.ts +133 -0
- package/dist/services/repository-public-artifact/service.js +330 -0
- package/dist/services/routing-benchmark/service.d.ts +362 -0
- package/dist/services/routing-benchmark/service.js +96 -0
- package/dist/services/specification-critic/service.d.ts +92 -0
- package/dist/services/specification-critic/service.js +172 -0
- package/dist/services/task-family/service.d.ts +40 -0
- package/dist/services/task-family/service.js +55 -0
- package/dist/services/task-seed/service.d.ts +906 -0
- package/dist/services/task-seed/service.js +1406 -0
- package/dist/services/task-specification/service.d.ts +27 -0
- package/dist/services/task-specification/service.js +40 -0
- package/dist/services/trajectory-policy/service.d.ts +110 -0
- package/dist/services/trajectory-policy/service.js +216 -0
- package/dist/test/agentic-capabilities-protocol.test.d.ts +1 -0
- package/dist/test/agentic-capabilities-protocol.test.js +570 -0
- package/dist/test/agentic-capabilities.test.d.ts +1 -0
- package/dist/test/agentic-capabilities.test.js +1461 -0
- package/dist/test/agentic-environment.test.d.ts +1 -0
- package/dist/test/agentic-environment.test.js +213 -0
- package/dist/test/case-pipeline-foundation.test.d.ts +1 -0
- package/dist/test/case-pipeline-foundation.test.js +535 -0
- package/dist/test/case-pipeline-protocol-v2.test.d.ts +1 -0
- package/dist/test/case-pipeline-protocol-v2.test.js +124 -0
- package/dist/test/case-pipeline-v2.test.d.ts +1 -0
- package/dist/test/case-pipeline-v2.test.js +286 -0
- package/dist/test/case-pipeline.test.d.ts +1 -0
- package/dist/test/case-pipeline.test.js +851 -0
- package/dist/test/eval-capability-policy.test.d.ts +1 -0
- package/dist/test/eval-capability-policy.test.js +50 -0
- package/dist/test/eval-event-log.test.d.ts +1 -0
- package/dist/test/eval-event-log.test.js +125 -0
- package/dist/test/evaluation-evidence-freshness.test.d.ts +1 -0
- package/dist/test/evaluation-evidence-freshness.test.js +44 -0
- package/dist/test/evaluation-evidence.test.d.ts +1 -0
- package/dist/test/evaluation-evidence.test.js +230 -0
- package/dist/test/evaluation-grader-calibration.test.d.ts +1 -0
- package/dist/test/evaluation-grader-calibration.test.js +373 -0
- package/dist/test/evaluation-proposal-digest.test.d.ts +1 -0
- package/dist/test/evaluation-proposal-digest.test.js +187 -0
- package/dist/test/evaluation-source-retrieval.test.d.ts +1 -0
- package/dist/test/evaluation-source-retrieval.test.js +237 -0
- package/dist/test/evaluation-structure-policy.test.d.ts +1 -0
- package/dist/test/evaluation-structure-policy.test.js +196 -0
- package/dist/test/fixtures/repository-resource-panel.d.ts +39 -0
- package/dist/test/fixtures/repository-resource-panel.js +111 -0
- package/dist/test/fixtures/vitest-boundary-panel.d.ts +84 -0
- package/dist/test/fixtures/vitest-boundary-panel.js +120 -0
- package/dist/test/fixtures/vitest-phase-panel.d.ts +135 -0
- package/dist/test/fixtures/vitest-phase-panel.js +213 -0
- package/dist/test/fixtures/vitest-reporter-results.d.ts +76 -0
- package/dist/test/fixtures/vitest-reporter-results.js +94 -0
- package/dist/test/grounded-authoring.test.d.ts +1 -0
- package/dist/test/grounded-authoring.test.js +565 -0
- package/dist/test/integrated-repository-history.test.d.ts +1 -0
- package/dist/test/integrated-repository-history.test.js +227 -0
- package/dist/test/project-authoring.test.js +593 -43
- package/dist/test/project-workflow.test.js +419 -40
- package/dist/test/repository-authoring-artifacts.test.d.ts +1 -0
- package/dist/test/repository-authoring-artifacts.test.js +185 -0
- package/dist/test/repository-bundle.test.d.ts +1 -0
- package/dist/test/repository-bundle.test.js +52 -0
- package/dist/test/repository-case-generation.test.d.ts +1 -0
- package/dist/test/repository-case-generation.test.js +3465 -0
- package/dist/test/repository-command-diagnostic.test.d.ts +1 -0
- package/dist/test/repository-command-diagnostic.test.js +55 -0
- package/dist/test/repository-command-signals.test.d.ts +1 -0
- package/dist/test/repository-command-signals.test.js +124 -0
- package/dist/test/repository-fixture-scope-coverage.test.d.ts +1 -0
- package/dist/test/repository-fixture-scope-coverage.test.js +127 -0
- package/dist/test/repository-fixture-validation.test.d.ts +1 -0
- package/dist/test/repository-fixture-validation.test.js +362 -0
- package/dist/test/repository-foundry-progress.test.d.ts +1 -0
- package/dist/test/repository-foundry-progress.test.js +110 -0
- package/dist/test/repository-foundry-quality.test.d.ts +1 -0
- package/dist/test/repository-foundry-quality.test.js +1138 -0
- package/dist/test/repository-import-context.test.d.ts +1 -0
- package/dist/test/repository-import-context.test.js +354 -0
- package/dist/test/repository-model-authoring.test.d.ts +1 -0
- package/dist/test/repository-model-authoring.test.js +544 -0
- package/dist/test/repository-model.test.d.ts +1 -0
- package/dist/test/repository-model.test.js +2195 -0
- package/dist/test/repository-node-test-reporter.test.d.ts +1 -0
- package/dist/test/repository-node-test-reporter.test.js +104 -0
- package/dist/test/repository-oracle-concurrency.test.d.ts +1 -0
- package/dist/test/repository-oracle-concurrency.test.js +542 -0
- package/dist/test/repository-oracle-coverage-witness.test.d.ts +1 -0
- package/dist/test/repository-oracle-coverage-witness.test.js +511 -0
- package/dist/test/repository-oracle-coverage.test.d.ts +1 -0
- package/dist/test/repository-oracle-coverage.test.js +168 -0
- package/dist/test/repository-oracle-evidence.test.d.ts +1 -0
- package/dist/test/repository-oracle-evidence.test.js +185 -0
- package/dist/test/repository-oracle-plan.test.d.ts +1 -0
- package/dist/test/repository-oracle-plan.test.js +176 -0
- package/dist/test/repository-overlay-isolation.test.d.ts +1 -0
- package/dist/test/repository-overlay-isolation.test.js +85 -0
- package/dist/test/repository-preparation-cache.test.d.ts +1 -0
- package/dist/test/repository-preparation-cache.test.js +414 -0
- package/dist/test/repository-public-artifact.test.d.ts +1 -0
- package/dist/test/repository-public-artifact.test.js +273 -0
- package/dist/test/repository-qualification-diagnostics.test.d.ts +1 -0
- package/dist/test/repository-qualification-diagnostics.test.js +524 -0
- package/dist/test/repository-reference-authoring.test.d.ts +1 -0
- package/dist/test/repository-reference-authoring.test.js +1633 -0
- package/dist/test/repository-review-evidence-v2.test.d.ts +1 -0
- package/dist/test/repository-review-evidence-v2.test.js +183 -0
- package/dist/test/repository-review-evidence.test.d.ts +1 -0
- package/dist/test/repository-review-evidence.test.js +124 -0
- package/dist/test/repository-seed-exclusions.test.d.ts +1 -0
- package/dist/test/repository-seed-exclusions.test.js +96 -0
- package/dist/test/repository-seed-selection.test.d.ts +1 -0
- package/dist/test/repository-seed-selection.test.js +504 -0
- package/dist/test/repository-semantic-calibration.test.d.ts +1 -0
- package/dist/test/repository-semantic-calibration.test.js +688 -0
- package/dist/test/repository-solution-edits.test.d.ts +1 -0
- package/dist/test/repository-solution-edits.test.js +377 -0
- package/dist/test/repository-specification-budget.test.d.ts +1 -0
- package/dist/test/repository-specification-budget.test.js +171 -0
- package/dist/test/repository-specification-contract-checkpoint.test.d.ts +1 -0
- package/dist/test/repository-specification-contract-checkpoint.test.js +228 -0
- package/dist/test/repository-specification-contract-facts.test.d.ts +1 -0
- package/dist/test/repository-specification-contract-facts.test.js +177 -0
- package/dist/test/repository-trajectory-authoring.test.d.ts +1 -0
- package/dist/test/repository-trajectory-authoring.test.js +176 -0
- package/dist/test/repository-valid-control-plan.test.d.ts +1 -0
- package/dist/test/repository-valid-control-plan.test.js +45 -0
- package/dist/test/repository-vitest-phase.test.d.ts +1 -0
- package/dist/test/repository-vitest-phase.test.js +848 -0
- package/dist/test/repository-vitest-reporter.test.d.ts +1 -0
- package/dist/test/repository-vitest-reporter.test.js +158 -0
- package/dist/test/repository-workspace-build.test.d.ts +1 -0
- package/dist/test/repository-workspace-build.test.js +160 -0
- package/dist/test/strict-authoring-schema.test.d.ts +1 -0
- package/dist/test/strict-authoring-schema.test.js +169 -0
- package/package.json +48 -6
|
@@ -0,0 +1,477 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { Effect, Layer, Schema } from "effect";
|
|
4
|
+
import { makeCaseCheckpointStoreV1 } from "../../case-checkpoint-store.js";
|
|
5
|
+
import { PIPELINE_STAGES } from "../../case-pipeline-protocol.js";
|
|
6
|
+
import { assertPipelineCheckpointChainV2, assertPipelineCheckpointReplayV2, decodePipelineScientificIdentityV2, PIPELINE_STAGES_V2, pipelineCheckpointDescriptorDigestV2, pipelineDigestV2, pipelineScientificIdentityDigestV2 } from "../../case-pipeline-protocol-v2.js";
|
|
7
|
+
import { RepositoryCommandExecutionPort } from "../../adapters/repository-command-runner.js";
|
|
8
|
+
import { RepositoryFoundryError } from "../../errors.js";
|
|
9
|
+
import { assertRepositoryFoundryModelPlanV1, repositoryFoundryModelAssignmentV1 } from "../../repository-language-model-protocol.js";
|
|
10
|
+
import { runCasePipelineV1 } from "../case-pipeline/service.js";
|
|
11
|
+
import { RepositoryCommandObservability } from "../command-observability/service.js";
|
|
12
|
+
import { RepositoryFoundryLanguageModel } from "../language-model/service.js";
|
|
13
|
+
/** The platform port executes commands in the real isolated checkout, with the runner's exact environment. */
|
|
14
|
+
export const makeRepositoryCommandExecutionPortV2 = (ports) => ({
|
|
15
|
+
run: (command) => ports.execution
|
|
16
|
+
.run({
|
|
17
|
+
operationId: command.operationId,
|
|
18
|
+
runtimeRoot: command.cwd,
|
|
19
|
+
command: [
|
|
20
|
+
"/usr/bin/env",
|
|
21
|
+
"-i",
|
|
22
|
+
...Object.entries(command.environment).map(([key, value]) => `${key}=${value}`),
|
|
23
|
+
command.executable,
|
|
24
|
+
...command.args
|
|
25
|
+
],
|
|
26
|
+
timeoutMs: command.timeoutMs
|
|
27
|
+
})
|
|
28
|
+
.pipe(Effect.mapError((cause) => new RepositoryFoundryError({
|
|
29
|
+
operation: "run-command",
|
|
30
|
+
detail: `execution port failed: ${cause.detail}`,
|
|
31
|
+
cause
|
|
32
|
+
})))
|
|
33
|
+
});
|
|
34
|
+
const failure = (operation, detail, cause) => new RepositoryFoundryError({
|
|
35
|
+
operation,
|
|
36
|
+
detail,
|
|
37
|
+
...(cause === undefined ? {} : { cause })
|
|
38
|
+
});
|
|
39
|
+
const stageIndex = (stage) => PIPELINE_STAGES.indexOf(stage);
|
|
40
|
+
const REPOSITORY_URI = "routekit://repository";
|
|
41
|
+
const CASE_URI = "routekit://case";
|
|
42
|
+
const mapStrings = (value, transform) => {
|
|
43
|
+
if (typeof value === "string")
|
|
44
|
+
return transform(value);
|
|
45
|
+
if (Array.isArray(value))
|
|
46
|
+
return value.map((entry) => mapStrings(entry, transform));
|
|
47
|
+
if (value === null || typeof value !== "object")
|
|
48
|
+
return value;
|
|
49
|
+
return Object.fromEntries(Object.entries(value).map(([key, entry]) => [key, mapStrings(entry, transform)]));
|
|
50
|
+
};
|
|
51
|
+
const replacePath = (value, from, to) => {
|
|
52
|
+
if (value === from)
|
|
53
|
+
return to;
|
|
54
|
+
return value.startsWith(`${from}/`) ? `${to}${value.slice(from.length)}` : value;
|
|
55
|
+
};
|
|
56
|
+
const portableValue = (value, runtimeRoot, caseDirectory) => mapStrings(value, (text) => {
|
|
57
|
+
const withoutCase = caseDirectory === undefined ? text : replacePath(text, caseDirectory, CASE_URI);
|
|
58
|
+
return replacePath(withoutCase, runtimeRoot, REPOSITORY_URI);
|
|
59
|
+
});
|
|
60
|
+
const runtimeValue = (value, runtimeRoot, caseDirectory) => mapStrings(value, (text) => replacePath(replacePath(text, CASE_URI, caseDirectory), REPOSITORY_URI, runtimeRoot));
|
|
61
|
+
export const repositoryBehaviorMapDigestV2 = (map) => pipelineDigestV2(portableValue(map, map.repository.root));
|
|
62
|
+
const mappedRole = (role) => {
|
|
63
|
+
switch (role) {
|
|
64
|
+
case "repository-analyst":
|
|
65
|
+
return "seed-author";
|
|
66
|
+
case "specification-writer":
|
|
67
|
+
case "trajectory-designer":
|
|
68
|
+
return "specification-author";
|
|
69
|
+
case "specification-critic-a":
|
|
70
|
+
case "specification-critic-b":
|
|
71
|
+
case "trajectory-critic":
|
|
72
|
+
return "specification-reviewer";
|
|
73
|
+
case "oracle-designer":
|
|
74
|
+
return "oracle-author";
|
|
75
|
+
case "valid-solution-generator-a":
|
|
76
|
+
case "valid-solution-generator-b":
|
|
77
|
+
return "control-author";
|
|
78
|
+
case "adversary":
|
|
79
|
+
case "held-out-adversary":
|
|
80
|
+
return "adversary-author";
|
|
81
|
+
case "solution-critic-a":
|
|
82
|
+
case "solution-critic-b":
|
|
83
|
+
case "held-out-solution-critic-a":
|
|
84
|
+
case "held-out-solution-critic-b":
|
|
85
|
+
case "oracle-critic":
|
|
86
|
+
case "hill-climb-planner":
|
|
87
|
+
return "oracle-critic";
|
|
88
|
+
case "quality-reviewer":
|
|
89
|
+
return "admission-reviewer";
|
|
90
|
+
default:
|
|
91
|
+
throw new Error(`unsupported portable foundry role ${JSON.stringify(role)}`);
|
|
92
|
+
}
|
|
93
|
+
};
|
|
94
|
+
const stageForRole = (role) => {
|
|
95
|
+
switch (mappedRole(role)) {
|
|
96
|
+
case "seed-author":
|
|
97
|
+
return "seed";
|
|
98
|
+
case "specification-author":
|
|
99
|
+
case "specification-reviewer":
|
|
100
|
+
return "spec";
|
|
101
|
+
case "oracle-author":
|
|
102
|
+
return role === "hill-climb-planner" ? "tournament" : "oracle";
|
|
103
|
+
case "control-author":
|
|
104
|
+
return "controls";
|
|
105
|
+
case "adversary-author":
|
|
106
|
+
return role === "held-out-adversary" ? "challenge" : "tournament";
|
|
107
|
+
case "oracle-critic":
|
|
108
|
+
return role.startsWith("held-out-") ? "challenge" : "tournament";
|
|
109
|
+
case "admission-reviewer":
|
|
110
|
+
return "admit";
|
|
111
|
+
}
|
|
112
|
+
};
|
|
113
|
+
const modelPlanV1 = (identity) => {
|
|
114
|
+
const byRole = new Map(identity.modelPlan.map((binding) => [binding.role, binding]));
|
|
115
|
+
const assignments = [
|
|
116
|
+
"repository-analyst",
|
|
117
|
+
"specification-writer",
|
|
118
|
+
"specification-critic-a",
|
|
119
|
+
"specification-critic-b",
|
|
120
|
+
"trajectory-designer",
|
|
121
|
+
"trajectory-critic",
|
|
122
|
+
"oracle-designer",
|
|
123
|
+
"valid-solution-generator-a",
|
|
124
|
+
"valid-solution-generator-b",
|
|
125
|
+
"adversary",
|
|
126
|
+
"solution-critic-a",
|
|
127
|
+
"solution-critic-b",
|
|
128
|
+
"held-out-adversary",
|
|
129
|
+
"held-out-solution-critic-a",
|
|
130
|
+
"held-out-solution-critic-b",
|
|
131
|
+
"quality-reviewer",
|
|
132
|
+
"hill-climb-planner",
|
|
133
|
+
"oracle-critic"
|
|
134
|
+
].map((role) => {
|
|
135
|
+
const portableRole = mappedRole(role);
|
|
136
|
+
const binding = byRole.get(portableRole);
|
|
137
|
+
if (binding === undefined) {
|
|
138
|
+
throw new Error(`portable foundry model plan is missing role ${portableRole}`);
|
|
139
|
+
}
|
|
140
|
+
return {
|
|
141
|
+
role: role,
|
|
142
|
+
model: binding.model,
|
|
143
|
+
reasoningEffort: binding.reasoningEffort
|
|
144
|
+
};
|
|
145
|
+
});
|
|
146
|
+
const plan = { version: 1, assignments };
|
|
147
|
+
assertRepositoryFoundryModelPlanV1(plan);
|
|
148
|
+
return plan;
|
|
149
|
+
};
|
|
150
|
+
const encodePayload = (payload) => Buffer.from(JSON.stringify(payload), "utf8");
|
|
151
|
+
const decodePayload = (descriptor, bytes) => {
|
|
152
|
+
if (bytes.byteLength !== descriptor.payloadBytes) {
|
|
153
|
+
throw new Error("portable checkpoint payload byte length does not match its descriptor");
|
|
154
|
+
}
|
|
155
|
+
const digest = createHash("sha256").update(bytes).digest("hex");
|
|
156
|
+
if (digest !== descriptor.payloadDigest) {
|
|
157
|
+
throw new Error("portable checkpoint payload digest does not match its descriptor");
|
|
158
|
+
}
|
|
159
|
+
const value = JSON.parse(Buffer.from(bytes).toString("utf8"));
|
|
160
|
+
if (typeof value !== "object" ||
|
|
161
|
+
value === null ||
|
|
162
|
+
!("version" in value) ||
|
|
163
|
+
value.version !== 2 ||
|
|
164
|
+
!("action" in value) ||
|
|
165
|
+
!("stage" in value) ||
|
|
166
|
+
!PIPELINE_STAGES.includes(value.stage)) {
|
|
167
|
+
throw new Error("portable checkpoint payload is not a supported V2 operation");
|
|
168
|
+
}
|
|
169
|
+
if (value.action === "delete-from") {
|
|
170
|
+
return { version: 2, action: "delete-from", stage: value.stage };
|
|
171
|
+
}
|
|
172
|
+
if (value.action === "write" && "value" in value) {
|
|
173
|
+
return {
|
|
174
|
+
version: 2,
|
|
175
|
+
action: "write",
|
|
176
|
+
stage: value.stage,
|
|
177
|
+
value: value.value
|
|
178
|
+
};
|
|
179
|
+
}
|
|
180
|
+
throw new Error("portable checkpoint payload action is invalid");
|
|
181
|
+
};
|
|
182
|
+
const makeLanguageModelLayerV2 = (identity, ports) => {
|
|
183
|
+
const complete = (input) => Effect.gen(function* () {
|
|
184
|
+
const portableRole = yield* Effect.try({
|
|
185
|
+
try: () => mappedRole(input.assignment.role),
|
|
186
|
+
catch: (cause) => failure("invoke-language-model", "model role is not allowlisted", cause)
|
|
187
|
+
});
|
|
188
|
+
const binding = identity.modelPlan.find(({ role }) => role === portableRole);
|
|
189
|
+
if (binding === undefined ||
|
|
190
|
+
binding.model !== input.assignment.model ||
|
|
191
|
+
binding.reasoningEffort !== input.assignment.reasoningEffort ||
|
|
192
|
+
input.maximumOutputTokens > binding.maximumOutputTokens) {
|
|
193
|
+
return yield* failure("invoke-language-model", `model call does not match the immutable ${portableRole} binding`);
|
|
194
|
+
}
|
|
195
|
+
const response = yield* ports.model
|
|
196
|
+
.complete({
|
|
197
|
+
operationId: input.operationId,
|
|
198
|
+
caseId: identity.caseId,
|
|
199
|
+
stage: stageForRole(input.assignment.role),
|
|
200
|
+
role: portableRole,
|
|
201
|
+
model: binding.model,
|
|
202
|
+
reasoningEffort: binding.reasoningEffort,
|
|
203
|
+
input: JSON.stringify({
|
|
204
|
+
instructions: input.instructions,
|
|
205
|
+
input: input.input,
|
|
206
|
+
schemaName: input.schemaName,
|
|
207
|
+
reviewInputVersion: input.reviewInputVersion
|
|
208
|
+
}),
|
|
209
|
+
maximumOutputTokens: input.maximumOutputTokens
|
|
210
|
+
})
|
|
211
|
+
.pipe(Effect.mapError((cause) => failure("invoke-language-model", cause.detail, cause)));
|
|
212
|
+
const decoded = yield* Effect.try({
|
|
213
|
+
try: () => {
|
|
214
|
+
const parsed = JSON.parse(response.text);
|
|
215
|
+
return Schema.decodeUnknownSync(input.outputSchema, {
|
|
216
|
+
onExcessProperty: "error"
|
|
217
|
+
})(parsed);
|
|
218
|
+
},
|
|
219
|
+
catch: (cause) => failure("invoke-language-model", "model response did not satisfy the requested structured schema", cause)
|
|
220
|
+
});
|
|
221
|
+
return {
|
|
222
|
+
value: decoded,
|
|
223
|
+
call: {
|
|
224
|
+
operationId: input.operationId,
|
|
225
|
+
role: input.assignment.role,
|
|
226
|
+
model: binding.model,
|
|
227
|
+
reasoningEffort: binding.reasoningEffort,
|
|
228
|
+
callId: response.callId
|
|
229
|
+
}
|
|
230
|
+
};
|
|
231
|
+
});
|
|
232
|
+
const service = {
|
|
233
|
+
generateAssignedStructured: complete,
|
|
234
|
+
generateStructured: (input) => {
|
|
235
|
+
const assignment = repositoryFoundryModelAssignmentV1(input.plan, input.role);
|
|
236
|
+
return complete({ ...input, assignment });
|
|
237
|
+
}
|
|
238
|
+
};
|
|
239
|
+
return Layer.succeed(RepositoryFoundryLanguageModel, service);
|
|
240
|
+
};
|
|
241
|
+
export const makePortableCheckpointStoreV2 = Effect.fn("CasePipelineV2.makeCheckpointStore")(function* (input) {
|
|
242
|
+
const identityDigest = pipelineScientificIdentityDigestV2(input.identity);
|
|
243
|
+
const caseDirectory = join(input.runtimeRoot, ".routekit", "evals", "cloud", identityDigest.slice(0, 24));
|
|
244
|
+
const local = makeCaseCheckpointStoreV1({ caseDirectory });
|
|
245
|
+
const loaded = [
|
|
246
|
+
...(yield* input.ports.checkpoints
|
|
247
|
+
.list(identityDigest)
|
|
248
|
+
.pipe(Effect.mapError((cause) => failure("checkpoint-store", cause.detail, cause))))
|
|
249
|
+
].sort((left, right) => left.sequence - right.sequence);
|
|
250
|
+
yield* Effect.try({
|
|
251
|
+
try: () => {
|
|
252
|
+
assertPipelineCheckpointChainV2(input.identity, loaded);
|
|
253
|
+
const terminal = loaded.filter(({ kind }) => kind === "terminal");
|
|
254
|
+
if (terminal.length > 1 ||
|
|
255
|
+
(terminal.length === 1 &&
|
|
256
|
+
(loaded.at(-1) !== terminal[0] || terminal[0]?.stage !== "admit"))) {
|
|
257
|
+
throw new Error("the accepted terminal checkpoint must be the unique admit chain head");
|
|
258
|
+
}
|
|
259
|
+
},
|
|
260
|
+
catch: (cause) => failure("checkpoint-store", "checkpoint chain is invalid", cause)
|
|
261
|
+
});
|
|
262
|
+
for (const descriptor of loaded) {
|
|
263
|
+
const bytes = yield* input.ports.checkpoints
|
|
264
|
+
.read(descriptor)
|
|
265
|
+
.pipe(Effect.mapError((cause) => failure("checkpoint-store", cause.detail, cause)));
|
|
266
|
+
const payload = yield* Effect.try({
|
|
267
|
+
try: () => decodePayload(descriptor, bytes),
|
|
268
|
+
catch: (cause) => failure("checkpoint-store", "checkpoint payload is invalid", cause)
|
|
269
|
+
});
|
|
270
|
+
if (payload.action === "write") {
|
|
271
|
+
yield* local.writeStage(payload.stage, runtimeValue(payload.value, input.runtimeRoot, caseDirectory));
|
|
272
|
+
}
|
|
273
|
+
else {
|
|
274
|
+
yield* local.deleteFrom(payload.stage);
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
const descriptors = [...loaded];
|
|
278
|
+
const loopIterations = new Map();
|
|
279
|
+
let highestStageIndex = loaded.reduce((highest, descriptor) => Math.max(highest, PIPELINE_STAGES_V2.indexOf(descriptor.stage)), -1);
|
|
280
|
+
const append = Effect.fn("CasePipelineV2.appendCheckpoint")(function* (appendInput) {
|
|
281
|
+
const { payload } = appendInput;
|
|
282
|
+
if (descriptors.at(-1)?.kind === "terminal") {
|
|
283
|
+
return yield* failure("checkpoint-store", "the accepted terminal checkpoint is already the immutable chain head");
|
|
284
|
+
}
|
|
285
|
+
const bytes = encodePayload(payload);
|
|
286
|
+
const payloadDigest = createHash("sha256").update(bytes).digest("hex");
|
|
287
|
+
const sequence = (descriptors.at(-1)?.sequence ?? 0) + 1;
|
|
288
|
+
const actualStageIndex = stageIndex(payload.stage);
|
|
289
|
+
const movesBack = actualStageIndex < highestStageIndex;
|
|
290
|
+
const repeatsStage = descriptors.length > 0 && actualStageIndex <= highestStageIndex;
|
|
291
|
+
highestStageIndex = Math.max(highestStageIndex, actualStageIndex);
|
|
292
|
+
const descriptorStage = PIPELINE_STAGES_V2[highestStageIndex] ?? "seed";
|
|
293
|
+
const loopName = payload.action === "delete-from"
|
|
294
|
+
? `invalidate-${payload.stage}`
|
|
295
|
+
: `checkpoint-${payload.stage}`;
|
|
296
|
+
const loopIteration = loopIterations.get(loopName) ?? 0;
|
|
297
|
+
if (movesBack || repeatsStage || payload.action === "delete-from") {
|
|
298
|
+
loopIterations.set(loopName, loopIteration + 1);
|
|
299
|
+
}
|
|
300
|
+
const createdAt = (yield* input.ports.clock.now.pipe(Effect.mapError((cause) => failure("checkpoint-store", cause.detail, cause)))).toISOString();
|
|
301
|
+
const descriptor = {
|
|
302
|
+
version: 2,
|
|
303
|
+
checkpointId: `checkpoint-${String(sequence)}-${payloadDigest.slice(0, 16)}`,
|
|
304
|
+
identityDigest,
|
|
305
|
+
attemptGeneration: input.attemptGeneration,
|
|
306
|
+
sequence,
|
|
307
|
+
kind: appendInput.terminal
|
|
308
|
+
? "terminal"
|
|
309
|
+
: movesBack || repeatsStage || payload.action === "delete-from"
|
|
310
|
+
? "loop"
|
|
311
|
+
: "stage",
|
|
312
|
+
stage: descriptorStage,
|
|
313
|
+
...(!appendInput.terminal && (movesBack || repeatsStage || payload.action === "delete-from")
|
|
314
|
+
? { loopName, loopIteration }
|
|
315
|
+
: {}),
|
|
316
|
+
parentDigest: descriptors.length === 0
|
|
317
|
+
? identityDigest
|
|
318
|
+
: pipelineCheckpointDescriptorDigestV2(descriptors.at(-1)),
|
|
319
|
+
payloadDigest,
|
|
320
|
+
payloadBytes: bytes.byteLength,
|
|
321
|
+
createdAt
|
|
322
|
+
};
|
|
323
|
+
const accepted = yield* input.ports.checkpoints
|
|
324
|
+
.put(descriptor, bytes)
|
|
325
|
+
.pipe(Effect.mapError((cause) => failure("checkpoint-store", cause.detail, cause)));
|
|
326
|
+
yield* Effect.try({
|
|
327
|
+
try: () => {
|
|
328
|
+
assertPipelineCheckpointReplayV2(descriptor, accepted);
|
|
329
|
+
assertPipelineCheckpointChainV2(input.identity, [...descriptors, accepted]);
|
|
330
|
+
},
|
|
331
|
+
catch: (cause) => failure("checkpoint-store", "checkpoint authority acknowledged a conflicting descriptor", cause)
|
|
332
|
+
});
|
|
333
|
+
descriptors.push(accepted);
|
|
334
|
+
return accepted;
|
|
335
|
+
});
|
|
336
|
+
const store = {
|
|
337
|
+
caseDirectory: local.caseDirectory,
|
|
338
|
+
stagePath: local.stagePath,
|
|
339
|
+
readStage: local.readStage,
|
|
340
|
+
writeStage: (stage, value) => append({
|
|
341
|
+
payload: {
|
|
342
|
+
version: 2,
|
|
343
|
+
action: "write",
|
|
344
|
+
stage,
|
|
345
|
+
value: portableValue(value, input.runtimeRoot, caseDirectory)
|
|
346
|
+
}
|
|
347
|
+
}).pipe(Effect.andThen(local.writeStage(stage, value))),
|
|
348
|
+
deleteFrom: (stage) => append({ payload: { version: 2, action: "delete-from", stage } }).pipe(Effect.andThen(local.deleteFrom(stage))),
|
|
349
|
+
readState: local.readState,
|
|
350
|
+
writeState: local.writeState,
|
|
351
|
+
appendLedger: local.appendLedger,
|
|
352
|
+
readLedger: local.readLedger,
|
|
353
|
+
writeCall: local.writeCall,
|
|
354
|
+
readWorkCheckpoint: local.readWorkCheckpoint,
|
|
355
|
+
writeWorkCheckpoint: local.writeWorkCheckpoint,
|
|
356
|
+
deleteWorkCheckpoint: local.deleteWorkCheckpoint
|
|
357
|
+
};
|
|
358
|
+
return {
|
|
359
|
+
store,
|
|
360
|
+
descriptors: () => [...descriptors],
|
|
361
|
+
completeAdmit: (admit) => {
|
|
362
|
+
const payload = {
|
|
363
|
+
version: 2,
|
|
364
|
+
action: "write",
|
|
365
|
+
stage: "admit",
|
|
366
|
+
value: portableValue(admit, input.runtimeRoot, caseDirectory)
|
|
367
|
+
};
|
|
368
|
+
const terminal = descriptors.find(({ kind }) => kind === "terminal");
|
|
369
|
+
if (terminal !== undefined) {
|
|
370
|
+
return Effect.try({
|
|
371
|
+
try: () => {
|
|
372
|
+
const bytes = encodePayload(payload);
|
|
373
|
+
const payloadDigest = createHash("sha256").update(bytes).digest("hex");
|
|
374
|
+
if (terminal.payloadDigest !== payloadDigest ||
|
|
375
|
+
terminal.payloadBytes !== bytes.byteLength) {
|
|
376
|
+
throw new Error("terminal checkpoint replay conflicts with the durable admit result");
|
|
377
|
+
}
|
|
378
|
+
return terminal;
|
|
379
|
+
},
|
|
380
|
+
catch: (cause) => failure("checkpoint-store", "terminal checkpoint replay conflicts with the durable admit result", cause)
|
|
381
|
+
});
|
|
382
|
+
}
|
|
383
|
+
return append({
|
|
384
|
+
payload,
|
|
385
|
+
terminal: true
|
|
386
|
+
});
|
|
387
|
+
}
|
|
388
|
+
};
|
|
389
|
+
});
|
|
390
|
+
const assertRepositoryBinding = (identity, map) => {
|
|
391
|
+
if (map.repository.commit !== identity.repository.commitSha ||
|
|
392
|
+
map.repository.tree !== identity.repository.treeSha ||
|
|
393
|
+
repositoryBehaviorMapDigestV2(map) !== identity.policy.repositoryModelDigest) {
|
|
394
|
+
throw new Error("repository behavior map does not match the portable foundry identity");
|
|
395
|
+
}
|
|
396
|
+
};
|
|
397
|
+
export const runCasePipelineV2 = Effect.fn("CasePipelineV2.run")(function* (input) {
|
|
398
|
+
const identity = yield* Effect.try({
|
|
399
|
+
try: () => {
|
|
400
|
+
const decoded = decodePipelineScientificIdentityV2(input.identity);
|
|
401
|
+
assertRepositoryBinding(decoded, input.map);
|
|
402
|
+
if (!Number.isInteger(input.attemptGeneration) || input.attemptGeneration < 1) {
|
|
403
|
+
throw new Error("attempt generation must be a positive integer");
|
|
404
|
+
}
|
|
405
|
+
if (input.runtimeRoot.trim().length === 0) {
|
|
406
|
+
throw new Error("runtime root must be non-empty");
|
|
407
|
+
}
|
|
408
|
+
return decoded;
|
|
409
|
+
},
|
|
410
|
+
catch: (cause) => failure("execute-generation", "portable foundry input is invalid", cause)
|
|
411
|
+
});
|
|
412
|
+
yield* input.ports.repository
|
|
413
|
+
.materialize(identity, input.runtimeRoot)
|
|
414
|
+
.pipe(Effect.mapError((cause) => failure("resolve-repository", cause.detail, cause)));
|
|
415
|
+
yield* input.ports.repository
|
|
416
|
+
.verify(identity, input.runtimeRoot)
|
|
417
|
+
.pipe(Effect.mapError((cause) => failure("resolve-repository", cause.detail, cause)));
|
|
418
|
+
const portableStore = yield* makePortableCheckpointStoreV2({
|
|
419
|
+
identity,
|
|
420
|
+
runtimeRoot: input.runtimeRoot,
|
|
421
|
+
attemptGeneration: input.attemptGeneration,
|
|
422
|
+
ports: input.ports
|
|
423
|
+
});
|
|
424
|
+
const plan = yield* Effect.try({
|
|
425
|
+
try: () => modelPlanV1(identity),
|
|
426
|
+
catch: (cause) => failure("execute-generation", "model plan cannot execute the V1 foundry", cause)
|
|
427
|
+
});
|
|
428
|
+
const primary = plan.assignments[0];
|
|
429
|
+
const runtimeMap = {
|
|
430
|
+
...input.map,
|
|
431
|
+
repository: {
|
|
432
|
+
...input.map.repository,
|
|
433
|
+
root: input.runtimeRoot,
|
|
434
|
+
requestedRef: identity.repository.commitSha
|
|
435
|
+
},
|
|
436
|
+
commands: input.map.commands.map((command) => ({
|
|
437
|
+
...command,
|
|
438
|
+
cwd: replacePath(command.cwd, input.map.repository.root, input.runtimeRoot)
|
|
439
|
+
}))
|
|
440
|
+
};
|
|
441
|
+
const result = yield* runCasePipelineV1({
|
|
442
|
+
context: {
|
|
443
|
+
repositoryRoot: input.runtimeRoot,
|
|
444
|
+
caseId: identity.caseId,
|
|
445
|
+
caseDirectory: portableStore.store.caseDirectory,
|
|
446
|
+
dimension: identity.dimension,
|
|
447
|
+
primaryModel: primary.model,
|
|
448
|
+
reasoningEffort: primary.reasoningEffort,
|
|
449
|
+
modelPlan: plan,
|
|
450
|
+
budget: {
|
|
451
|
+
maxModelCalls: identity.budget.maximumModelCalls,
|
|
452
|
+
maxWallMs: identity.budget.maximumWallTimeMs
|
|
453
|
+
},
|
|
454
|
+
referenceCommit: identity.source.referenceCommitSha,
|
|
455
|
+
requestedRef: identity.repository.commitSha,
|
|
456
|
+
oracleRepetitions: identity.policy.oracleRepetitions,
|
|
457
|
+
oracleConcurrency: identity.policy.oracleConcurrency,
|
|
458
|
+
foldLimit: identity.policy.foldLimit
|
|
459
|
+
},
|
|
460
|
+
map: runtimeMap,
|
|
461
|
+
store: portableStore.store
|
|
462
|
+
}).pipe(Effect.provide(Layer.mergeAll(makeLanguageModelLayerV2(identity, input.ports), Layer.succeed(RepositoryCommandExecutionPort, makeRepositoryCommandExecutionPortV2(input.ports)), Layer.succeed(RepositoryCommandObservability, {
|
|
463
|
+
observe: (event) => input.ports.progress.observeCommand?.(event)
|
|
464
|
+
}))));
|
|
465
|
+
if (result.admit !== undefined &&
|
|
466
|
+
result.state.stage === "done" &&
|
|
467
|
+
(result.state.status === "admitted" || result.state.status === "rejected")) {
|
|
468
|
+
yield* portableStore.completeAdmit(result.admit);
|
|
469
|
+
}
|
|
470
|
+
return {
|
|
471
|
+
identityDigest: pipelineScientificIdentityDigestV2(identity),
|
|
472
|
+
runtimeRoot: input.runtimeRoot,
|
|
473
|
+
attemptGeneration: input.attemptGeneration,
|
|
474
|
+
checkpoints: portableStore.descriptors(),
|
|
475
|
+
result
|
|
476
|
+
};
|
|
477
|
+
});
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { Context } from "effect";
|
|
2
|
+
import type { RepositoryCommandObservationV1 } from "../../repository-execution-protocol.js";
|
|
3
|
+
export type RepositoryCommandObservabilityShape = {
|
|
4
|
+
/**
|
|
5
|
+
* Nonblocking operational telemetry. Implementations must enqueue and return;
|
|
6
|
+
* command execution and terminalization never wait for log delivery.
|
|
7
|
+
*/
|
|
8
|
+
readonly observe: (event: RepositoryCommandObservationV1) => void;
|
|
9
|
+
};
|
|
10
|
+
declare const RepositoryCommandObservability_base: Context.ServiceClass<RepositoryCommandObservability, "@velum-labs/routekit-eval-setup/RepositoryCommandObservability", RepositoryCommandObservabilityShape>;
|
|
11
|
+
export declare class RepositoryCommandObservability extends RepositoryCommandObservability_base {
|
|
12
|
+
}
|
|
13
|
+
export {};
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import { type RoutingBasisV3 } from "@velum-labs/routekit-eval-contracts";
|
|
2
|
+
import { Context, Effect, Layer } from "effect";
|
|
3
|
+
import { RepositoryFoundryError } from "../../errors.js";
|
|
4
|
+
import type { RepositoryBenchmarkCaseV1 } from "../../repository-benchmark-protocol.js";
|
|
5
|
+
import { type RepositoryReviewedDimensionLabelV1 } from "../../repository-routing-benchmark-protocol.js";
|
|
6
|
+
import { type RepositoryRoutingModelCallV1, type RepositoryRoutingModelPlanV1 } from "../../repository-routing-model-protocol.js";
|
|
7
|
+
import { RepositoryFoundryLanguageModel } from "../language-model/service.js";
|
|
8
|
+
export type RepositoryDimensionLabelingResultV1 = {
|
|
9
|
+
readonly version: 1;
|
|
10
|
+
readonly reviewed: RepositoryReviewedDimensionLabelV1;
|
|
11
|
+
readonly modelCalls: readonly RepositoryRoutingModelCallV1[];
|
|
12
|
+
};
|
|
13
|
+
export type DimensionLabelingShape = {
|
|
14
|
+
readonly authorReviewedLabel: (input: {
|
|
15
|
+
readonly operationId: string;
|
|
16
|
+
readonly basis: RoutingBasisV3;
|
|
17
|
+
readonly benchmarkCase: RepositoryBenchmarkCaseV1;
|
|
18
|
+
readonly modelPlan: RepositoryRoutingModelPlanV1;
|
|
19
|
+
}) => Effect.Effect<RepositoryDimensionLabelingResultV1, RepositoryFoundryError>;
|
|
20
|
+
};
|
|
21
|
+
declare const DimensionLabeling_base: Context.ServiceClass<DimensionLabeling, "@velum-labs/routekit-eval-setup/DimensionLabeling", DimensionLabelingShape>;
|
|
22
|
+
export declare class DimensionLabeling extends DimensionLabeling_base {
|
|
23
|
+
}
|
|
24
|
+
export declare const authorRepositoryReviewedDimensionLabelV1: (input: {
|
|
25
|
+
readonly operationId: string;
|
|
26
|
+
readonly basis: RoutingBasisV3;
|
|
27
|
+
readonly benchmarkCase: RepositoryBenchmarkCaseV1;
|
|
28
|
+
readonly modelPlan: RepositoryRoutingModelPlanV1;
|
|
29
|
+
}) => Effect.Effect<{
|
|
30
|
+
version: 1;
|
|
31
|
+
reviewed: {
|
|
32
|
+
readonly version: 1;
|
|
33
|
+
readonly caseId: string;
|
|
34
|
+
readonly label: {
|
|
35
|
+
readonly caseId: string;
|
|
36
|
+
readonly basisDigest: string;
|
|
37
|
+
readonly scores: readonly {
|
|
38
|
+
readonly dimensionId: string;
|
|
39
|
+
readonly score: number;
|
|
40
|
+
}[];
|
|
41
|
+
readonly unknownProbability: number;
|
|
42
|
+
};
|
|
43
|
+
readonly basisDigest: string;
|
|
44
|
+
readonly authorModel: string;
|
|
45
|
+
readonly reviews: readonly {
|
|
46
|
+
readonly outcome: "approve" | "reject";
|
|
47
|
+
readonly detail: string;
|
|
48
|
+
readonly model: string;
|
|
49
|
+
readonly evidenceIds: readonly string[];
|
|
50
|
+
readonly reviewerId: string;
|
|
51
|
+
readonly primaryDimensionId: string;
|
|
52
|
+
readonly independentLabel: {
|
|
53
|
+
readonly caseId: string;
|
|
54
|
+
readonly basisDigest: string;
|
|
55
|
+
readonly scores: readonly {
|
|
56
|
+
readonly dimensionId: string;
|
|
57
|
+
readonly score: number;
|
|
58
|
+
}[];
|
|
59
|
+
readonly unknownProbability: number;
|
|
60
|
+
};
|
|
61
|
+
readonly callId?: string | undefined;
|
|
62
|
+
}[];
|
|
63
|
+
readonly authorEvidenceIds: readonly string[];
|
|
64
|
+
readonly authorDetail: string;
|
|
65
|
+
readonly authorCallId?: string | undefined;
|
|
66
|
+
};
|
|
67
|
+
modelCalls: {
|
|
68
|
+
readonly operationId: string;
|
|
69
|
+
readonly role: "dimension-labeler" | "dimension-critic-terra" | "dimension-critic-luna";
|
|
70
|
+
readonly model: string;
|
|
71
|
+
readonly reasoningEffort: import("../../repository-language-model-protocol.js").RepositoryFoundryReasoningEffortV1;
|
|
72
|
+
readonly callId?: string;
|
|
73
|
+
}[];
|
|
74
|
+
}, RepositoryFoundryError, RepositoryFoundryLanguageModel>;
|
|
75
|
+
export declare const makeDimensionLabeling: Effect.Effect<DimensionLabelingShape, never, RepositoryFoundryLanguageModel>;
|
|
76
|
+
export declare const DimensionLabelingLive: Layer.Layer<DimensionLabeling, never, RepositoryFoundryLanguageModel>;
|
|
77
|
+
export {};
|