@velum-labs/routekit-eval-setup 1.0.0

package/dist/test/state-store.test.js

@@ -0,0 +1,30 @@
+import assert from "node:assert/strict";
+import { mkdtemp, rm, stat } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { after, test } from "node:test";
+import { layer as NodeServicesLayer } from "@effect/platform-node/NodeServices";
+import { Effect } from "effect";
+import { initialSetupState, makeFileEvalSetupStateStore } from "../state-store.js";
+const roots = [];
+after(async () => Promise.all(roots.map((root) => rm(root, { recursive: true, force: true }))));
+test("setup state and run checkpoints survive process interruption", async () => {
+    const root = await mkdtemp(path.join(os.tmpdir(), "routekit-eval-setup-state-"));
+    roots.push(root);
+    await Effect.runPromise(Effect.gen(function* () {
+        const store = yield* makeFileEvalSetupStateStore;
+        const state = {
+            ...initialSetupState({
+                profileId: "support",
+                repositoryRoot: root,
+                now: "2026-08-15T00:00:00.000Z"
+            }),
+            openQuestion: "Which workflow?"
+        };
+        yield* store.save(state);
+        assert.deepEqual(yield* store.load(root, "support"), state);
+    }).pipe(Effect.provide(NodeServicesLayer)));
+    const mode = (await stat(path.join(root, ".routekit", "eval-setup", "support", "state.json")))
+        .mode;
+    assert.equal(mode & 0o777, 0o600);
+});

package/dist/test/workflow.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/test/workflow.test.js

@@ -0,0 +1,167 @@
+import assert from "node:assert/strict";
+import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { after, test } from "node:test";
+import { layer as NodeServicesLayer } from "@effect/platform-node/NodeServices";
+import { Effect, Layer } from "effect";
+import { EvalSetup, EvalSetupLive, EvalSetupRunner, OriEvalAuthoring, oriAuthoringFromApi } from "../index.js";
+const roots = [];
+after(async () => Promise.all(roots.map((root) => rm(root, { recursive: true, force: true }))));
+const waiting = (tag, prompt) => ({
+    ok: true,
+    status: "waiting",
+    runDirectory: "/tmp/ori-run",
+    tag,
+    prompt,
+    question: prompt,
+    options: ["Support replies", "Documentation", "Routing"],
+    context: "Inspected the repository."
+});
+const fakeAuthoring = (turns) => {
+    let index = 0;
+    const next = () => turns[Math.min(index++, turns.length - 1)] ?? waiting("surface", "Which surface?");
+    return {
+        prepare: async () => ({ ok: true, status: "prepared", runDirectory: "/tmp/ori-run" }),
+        run: async () => next(),
+        answer: async (input) => {
+            if (/clarif|what do you mean/iu.test(input.answer)) {
+                return { ...waiting("surface", "Which surface?"), accepted: false };
+            }
+            return next();
+        },
+        status: async () => turns[Math.max(0, index - 1)] ?? {
+            ok: true,
+            status: "prepared",
+            runDirectory: "/tmp/ori-run"
+        }
+    };
+};
+const makeLayer = (api, publishes) => EvalSetupLive.pipe(Layer.provide(Layer.mergeAll(OriEvalAuthoring.layer(oriAuthoringFromApi(api)), EvalSetupRunner.layer({
+    validate: () => Effect.void,
+    estimate: () => Effect.succeed({ callCount: 12, maximumCostUsd: 0.42, pricingKnown: true }),
+    publish: () => Effect.sync(() => {
+        publishes.count += 1;
+        return {
+            comparison: {
+                version: 1,
+                comparisonId: "comparison-1",
+                profileId: "support",
+                suiteDigest: "suite",
+                judgeModel: "openai/judge",
+                startedAt: "2026-08-16T00:00:00.000Z",
+                finishedAt: "2026-08-16T00:01:00.000Z",
+                models: []
+            },
+            activation: {
+                version: 2,
+                generatedAt: "2026-08-16T00:01:00.000Z",
+                basisDigest: "basis",
+                evidenceDigest: "evidence",
+                classifierModel: "openai/classifier",
+                objective: { kind: "lowest-cost", minimumQuality: 0.8 },
+                maximumUnknownWeight: 0.2,
+                dimensions: [
+                    {
+                        id: "support",
+                        description: "Support requests",
+                        includes: ["support"],
+                        excludes: ["other"]
+                    }
+                ],
+                candidateModels: ["openai/cheap", "anthropic/strong"],
+                evidence: [
+                    {
+                        model: "openai/cheap",
+                        dimensionId: "support",
+                        suiteDigest: "suite",
+                        evidenceDigest: "cheap-evidence",
+                        quality: {
+                            passRate: 1,
+                            lowerConfidenceBound: 0.8,
+                            sampleCount: 1
+                        },
+                        failureRate: 0,
+                        unpricedCalls: 1
+                    },
+                    {
+                        model: "anthropic/strong",
+                        dimensionId: "support",
+                        suiteDigest: "suite",
+                        evidenceDigest: "strong-evidence",
+                        quality: {
+                            passRate: 1,
+                            lowerConfidenceBound: 0.8,
+                            sampleCount: 1
+                        },
+                        failureRate: 0,
+                        unpricedCalls: 1
+                    }
+                ]
+            }
+        };
+    })
+}))), Layer.provide(NodeServicesLayer));
+test("prepare, run, answer, and status relay Ori questions and resume durable host metadata", async () => {
+    const root = await mkdtemp(path.join(os.tmpdir(), "routekit-eval-setup-ori-"));
+    roots.push(root);
+    await writeFile(path.join(root, "source.txt"), "unchanged\n");
+    const publishes = { count: 0 };
+    const api = fakeAuthoring([
+        waiting("surface", "Which model call should we evaluate?"),
+        {
+            ok: true,
+            status: "completed",
+            runDirectory: "/tmp/ori-run",
+            scratchWorkspace: path.join(root, "scratch"),
+            evalRuns: []
+        }
+    ]);
+    const first = await Effect.runPromise(Effect.gen(function* () {
+        const setup = yield* EvalSetup;
+        const prepared = yield* setup.prepare(root, "support");
+        assert.equal(prepared.state.stage, "prepared");
+        const running = yield* setup.runApproved(root, "support");
+        assert.equal(running.state.stage, "surface");
+        assert.equal(running.question?.prompt, "Which model call should we evaluate?");
+        const clarification = yield* setup.answer(root, "support", "Can you clarify?");
+        assert.equal(clarification.state.stage, "surface");
+        return yield* setup.answer(root, "support", "1");
+    }).pipe(Effect.provide(makeLayer(api, publishes))));
+    assert.equal(first.state.stage, "completed");
+    assert.equal(first.state.answers.surface, "1");
+    const resumed = await Effect.runPromise(Effect.gen(function* () {
+        const setup = yield* EvalSetup;
+        return yield* setup.prepare(root, "support");
+    }).pipe(Effect.provide(makeLayer(api, publishes))));
+    assert.equal(resumed.state.answers.surface, "1");
+    assert.equal(await readFile(path.join(root, "source.txt"), "utf8"), "unchanged\n");
+});
+test("publication requires a completed Ori run and runs exactly once", async () => {
+    const root = await mkdtemp(path.join(os.tmpdir(), "routekit-eval-setup-publish-"));
+    roots.push(root);
+    const publishes = { count: 0 };
+    const completed = {
+        ok: true,
+        status: "completed",
+        runDirectory: "/tmp/ori-run",
+        scratchWorkspace: path.join(root, "scratch"),
+        evalRuns: [{ ok: true, model: "openai/cheap" }]
+    };
+    const api = fakeAuthoring([waiting("surface", "Which surface?"), completed]);
+    const outcome = await Effect.runPromise(Effect.gen(function* () {
+        const setup = yield* EvalSetup;
+        yield* setup.prepare(root, "support");
+        yield* setup.runApproved(root, "support");
+        const early = yield* Effect.exit(setup.publishApproved(root, "support"));
+        assert.equal(early._tag, "Failure");
+        yield* setup.answer(root, "support", "1");
+        const published = yield* setup.publishApproved(root, "support");
+        const duplicateRun = yield* Effect.exit(setup.runApproved(root, "support"));
+        assert.equal(duplicateRun._tag, "Failure");
+        return published;
+    }).pipe(Effect.provide(makeLayer(api, publishes))));
+    assert.equal(outcome.state.stage, "completed");
+    assert.deepEqual(outcome.activation?.candidateModels, ["openai/cheap", "anthropic/strong"]);
+    assert.equal(publishes.count, 1);
+});

package/dist/types.d.ts

@@ -0,0 +1,77 @@
+import type { EvalComparisonResult, EvalSetupEvent, EvalSetupRunMode, PublishedRoutingActivation, RoutingObjective } from "@velum-labs/routekit-eval-contracts";
+import type { OriEvalResult } from "./ori-result.js";
+export type SetupQuestion = {
+    readonly id: string;
+    readonly prompt: string;
+    readonly context?: string;
+    readonly options: readonly string[];
+};
+export type RepositorySurface = {
+    readonly name: string;
+    readonly path: string;
+    readonly model?: string;
+};
+export type RepositoryMaterial = {
+    readonly kind: "doc" | "prompt" | "dataset" | "fixture" | "test" | "schema";
+    readonly path: string;
+};
+export type RepositoryInspection = {
+    readonly repositoryRoot: string;
+    readonly surfaces: readonly RepositorySurface[];
+    readonly materials: readonly RepositoryMaterial[];
+    readonly summary: {
+        readonly entriesVisited: number;
+        readonly textFilesConsidered: number;
+        readonly filesRead: number;
+        readonly bytesRead: number;
+        readonly skippedOversizedFiles: number;
+        readonly truncated: boolean;
+    };
+};
+export type SetupEstimate = {
+    readonly callCount: number;
+    readonly maximumCostUsd?: number;
+    readonly pricingKnown: boolean;
+};
+export type SetupStateView = {
+    readonly profileId: string;
+    readonly repositoryRoot: string;
+    readonly stage: string;
+    readonly revision: number;
+    readonly updatedAt: string;
+    readonly answers: Record<string, string>;
+    readonly runDirectory?: string;
+    readonly scratchWorkspace?: string;
+    readonly publishApproved?: boolean;
+};
+export type SetupStatus = {
+    readonly state: SetupStateView;
+    readonly question?: SetupQuestion;
+    readonly result?: OriEvalResult;
+};
+export type SetupAnswerResult = SetupStatus & {
+    readonly events: readonly EvalSetupEvent[];
+};
+export type SetupRunResult = SetupAnswerResult & {
+    readonly comparison?: EvalComparisonResult;
+    readonly activation?: PublishedRoutingActivation;
+};
+export type EvalSetupRunnerShape = {
+    readonly validate: (result: OriEvalResult) => import("effect").Effect.Effect<void, import("./errors.js").EvalSetupRunnerError>;
+    readonly estimate: (result: OriEvalResult, mode: EvalSetupRunMode) => import("effect").Effect.Effect<SetupEstimate, import("./errors.js").EvalSetupRunnerError>;
+    readonly publish: (input: {
+        readonly profileId: string;
+        readonly description: string;
+        readonly repositoryRoot: string;
+        readonly objective: RoutingObjective;
+        readonly result: OriEvalResult;
+    }) => import("effect").Effect.Effect<{
+        readonly comparison: EvalComparisonResult;
+        readonly activation: PublishedRoutingActivation;
+    }, import("./errors.js").EvalSetupRunnerError>;
+};
+export type EvalSetupRunCheckpoint = {
+    readonly comparison: EvalComparisonResult;
+    readonly activation: PublishedRoutingActivation;
+};
+export type { EvalSetupRunMode };

package/dist/types.js

@@ -0,0 +1 @@
+export {};

package/package.json

@@ -0,0 +1,52 @@
+{
+  "name": "@velum-labs/routekit-eval-setup",
+  "private": false,
+  "version": "1.0.0",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/velum-labs/routekit.git",
+    "directory": "packages/eval-setup"
+  },
+  "description": "Durable Effect onboarding for compositional RouteKit eval routing.",
+  "license": "Apache-2.0",
+  "type": "module",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    },
+    "./effect": {
+      "types": "./dist/effect-api.d.ts",
+      "default": "./dist/effect-api.js"
+    }
+  },
+  "files": [
+    "dist",
+    "skills",
+    "LICENSE"
+  ],
+  "publishConfig": {
+    "registry": "https://registry.npmjs.org",
+    "access": "public",
+    "provenance": true
+  },
+  "dependencies": {
+    "effect": "4.0.0-rc.108",
+    "@velum-labs/routekit-eval-contracts": "1.0.0",
+    "@velum-labs/routekit-runtime": "1.0.0"
+  },
+  "keywords": [
+    "routekit",
+    "eval",
+    "onboarding",
+    "routing"
+  ],
+  "devDependencies": {
+    "@effect/platform-node": "4.0.0-rc.108"
+  },
+  "scripts": {
+    "build": "tsc -b",
+    "clean": "tsc -b --clean",
+    "test": "node --test \"dist/test/*.test.js\""
+  }
+}

package/skills/setup-eval-routing/SKILL.md

@@ -0,0 +1,149 @@
+---
+name: setup-eval-routing
+description: >-
+  Onboard or maintain a repository's compositional RouteKit eval routing through
+  the public routekit eval CLI. Use when the user wants to define a routing
+  basis, review workload dimensions, author or approve evaluations, validate or
+  estimate a billed plan, run model evidence, inspect results, activate
+  model:auto routing, or resume an interrupted eval project.
+---
+
+# Set Up Eval Routing
+
+Use the public `routekit eval` CLI as the product boundary. Do not substitute
+internal services, a standalone eval executable, or testkit qualification
+commands for missing CLI functionality.
+
+Inside the RouteKit source checkout, build first and use:
+
+```text
+node packages/cli/dist/index.js
+```
+
+For an installed release, use `routekit`. Refer to either form as `$ROUTEKIT`.
+
+## Discover the available interface
+
+Run `$ROUTEKIT eval --help` and the relevant subcommand help before acting. Use
+only commands and flags exposed by that CLI version. Prefer `--json` for state,
+plans, estimates, and results.
+
+Normal model-backed commands use RouteKit's standard target resolution:
+
+1. an explicit global `--remote <name>` or `--local` selection;
+2. the active remote, when configured; otherwise
+3. the local daemon.
+
+Do not ask for a gateway URL or credential when that configured target works.
+An explicitly supported external gateway mode may use `--gateway-url` and one
+private credential source, but it is for qualification only and must never
+publish a routing activation.
+
+## Resume or initialize
+
+1. Run `$ROUTEKIT --json eval status` from the repository root.
+2. If no eval project exists, run `$ROUTEKIT --json eval setup`.
+3. Follow `nextAction` and the returned artifact paths. Durable state lives
+   under `.routekit/evals`; resume it rather than starting over after an
+   interruption.
+4. When setup returns a question, relay exactly that question and its context.
+   Ask one question per turn and never answer it for the user.
+5. Submit the answer unchanged with `eval answer`. Prefer a private temporary
+   answer file for multiline text when the CLI supports one, then remove it.
+
+Candidate, classifier, author, and judge roles must use explicit
+`provider/model` IDs. Eval traffic must never use `model: auto`.
+
+## Build and review the routing basis
+
+Use the CLI workflow in this order, following the current `nextAction`:
+
+1. `eval propose dimensions`
+2. review the generated routing basis and every workload dimension;
+3. `eval approve dimensions`
+4. `eval propose evaluations`
+5. review every dimension suite, case identity, rubric, manifest, and source
+   boundary;
+6. `eval approve evaluations`
+7. `eval validate`
+
+Treat proposals as review material, not activation evidence. Approval is bound
+to the exact artifact digest. If an artifact changes, validate and approve the
+new digest rather than reusing an old approval.
+
+A useful routing basis normally contains 5–10 separable workload dimensions.
+Each definition should include positive scope, exclusions, and boundary
+examples. Request-envelope capabilities such as tools, vision, context, and
+maximum output are hard requirements, not semantic workload dimensions.
+
+Keep generated evaluations and sanitized structured results reviewable in the
+repository when the user approves committing them. Do not hand-edit immutable
+plans or measured run records.
+
+## Preserve the classifier boundary
+
+The decomposition classifier receives only:
+
+- the request; and
+- the reviewed workload-dimension definitions.
+
+It emits one weight per dimension plus an unknown weight, normalized to sum to
+one. It must not receive candidate models, evidence, prices, objectives,
+selected models, fallbacks, or previous routing decisions.
+
+Model selection is deterministic. It combines the request decomposition, hard
+requirements, the approved objective and constraints, and the published
+model-by-dimension evidence matrix.
+
+## Estimate and run
+
+Before every billed step:
+
+1. explain what will call models and show the resolved target and explicit model
+   roles;
+2. run `eval estimate` for the intended scope;
+3. report exact call and token limits and the CLI's pricing status—missing
+   pricing is unknown, never zero; and
+4. obtain explicit user approval for that plan.
+
+Run the immutable plan with `eval run` using the plan identifier returned by
+the CLI. Do not silently reduce case counts, change candidates, replace failed
+rows, or retry a completed paid plan. After an interruption or ambiguous
+result, use `eval status` and `eval results` before deciding whether work
+remains.
+
+Never expose credentials, prompts, responses, headers, or raw child output.
+Never recursively evaluate through `model: auto`.
+
+## Review results and activate
+
+Use `eval results` to review the decomposition benchmark, every dimension
+suite, the composition benchmark, the complete evidence matrix, accounting,
+and cleanup outcome.
+
+Run `eval publish` only when all of these are true:
+
+- dimensions and evaluations were approved at their current digests;
+- validation passed and the immutable plan is still fresh;
+- every configured candidate has exactly one judged result for every expected
+  case in every dimension;
+- the decomposition and composition benchmarks passed;
+- the run has zero active reservations and zero unknown measurements;
+- the user reviewed the results and explicitly approved activation; and
+- the run used a configured local or remote RouteKit target, not an external
+  gateway.
+
+Publication installs already-measured evidence atomically; it must not perform
+another billed run. After publication, check `eval status`, then verify an
+ordinary headerless `model: auto` request. Use `routekit calls inspect <call-id>`
+to inspect sanitized routing provenance when needed.
+
+## Safety
+
+- Never spend or publish silently.
+- Never send repository material before approval for the model-backed step.
+- Never log, echo, or commit credentials.
+- Never describe unknown cost as zero.
+- Never publish incomplete, stale, mismatched, duplicated, or cutoff evidence.
+- Never claim that a passing pilot or classifier-only run is production routing
+  qualification.