@velum-labs/routekit-eval-setup 1.0.0

package/dist/inspection.js

@@ -0,0 +1,261 @@
+import { Context, Effect, FileSystem, Layer, Option, Path } from "effect";
+import { EvalSetupInspectionError } from "./errors.js";
+export class EvalRepositoryInspector extends Context.Service()("@velum-labs/routekit-eval-setup/EvalRepositoryInspector") {
+}
+const IGNORED_SEGMENTS = new Set([
+    ".cache",
+    ".git",
+    ".next",
+    ".pnpm-store",
+    ".routekit",
+    ".turbo",
+    ".venv",
+    "build",
+    "coverage",
+    "dist",
+    "generated",
+    "node_modules",
+    "out",
+    "target",
+    "vendor"
+]);
+const MAX_INSPECTION_FILES = 2_000;
+const MAX_INSPECTION_ENTRIES = 10_000;
+const MAX_INSPECTION_DEPTH = 24;
+const MAX_INSPECTION_DIRECTORIES = 1_000;
+const MAX_FILE_BYTES = 256 * 1024;
+const MAX_TOTAL_BYTES = 8 * 1024 * 1024;
+const TEXT_EXTENSIONS = new Set([
+    ".cjs",
+    ".go",
+    ".js",
+    ".json",
+    ".jsx",
+    ".md",
+    ".mdx",
+    ".mjs",
+    ".py",
+    ".rs",
+    ".ts",
+    ".tsx",
+    ".yaml",
+    ".yml"
+]);
+const MODEL_CALL = /(?:chat\.completions|responses\.create|messages\.create|generateContent|model\s*[:=])/u;
+const MODEL_ID = /\b(?:openai|anthropic|google|meta-llama|mistralai|cohere|x-ai)\/[A-Za-z0-9._~:-]+\b/u;
+const PROMPT_NAME = /(?:^|[-_.])(prompt|system|instructions?)(?:[-_.]|$)/iu;
+const DATASET_NAME = /(?:dataset|cases?|examples?|samples?|traffic)/iu;
+const FIXTURE_NAME = /(?:fixture|gold|expected)/iu;
+const TEST_NAME = /(?:^|[-_.])(test|spec)(?:[-_.]|$)/iu;
+const SCHEMA_NAME = /(?:schema|output)/iu;
+const inspectionFailure = (path, cause) => new EvalSetupInspectionError({
+    path,
+    detail: cause instanceof Error ? cause.message : String(cause),
+    cause
+});
+const skippedPath = (relativePath) => relativePath.split(/[\\/]/u).some((segment) => IGNORED_SEGMENTS.has(segment));
+const pathIsWithin = (paths, root, candidate) => {
+    const relative = paths.relative(root, candidate);
+    return (relative === "" ||
+        (relative !== ".." && !relative.startsWith(`..${paths.sep}`) && !paths.isAbsolute(relative)));
+};
+const posixPath = (relativePath) => relativePath.split(/[\\/]/u).join("/");
+const isTopLevelReadme = (relativePath) => /^README\.mdx?$/iu.test(posixPath(relativePath));
+const isReadme = (relativePath) => /(^|\/)README\.mdx?$/iu.test(posixPath(relativePath));
+const isDocPath = (relativePath) => {
+    const posix = posixPath(relativePath);
+    if (isReadme(posix))
+        return true;
+    const segments = posix.split("/");
+    const docsIndex = segments.findIndex((segment) => segment.toLowerCase() === "docs");
+    if (docsIndex < 0 || docsIndex >= segments.length - 1)
+        return false;
+    const ext = segments[segments.length - 1]?.toLowerCase();
+    return ext !== undefined && (ext.endsWith(".md") || ext.endsWith(".mdx"));
+};
+const skipSurfacePath = (relativePath) => {
+    const posix = posixPath(relativePath);
+    const segments = posix.split("/");
+    if (segments.some((segment) => ["vendor", "generated", "test", "tests", "__tests__"].includes(segment.toLowerCase()))) {
+        return true;
+    }
+    const base = segments[segments.length - 1] ?? "";
+    return /\.(?:test|spec)\.[cm]?[jt]sx?$/iu.test(base);
+};
+const materialKind = (relativePath) => {
+    if (isDocPath(relativePath))
+        return "doc";
+    const lower = relativePath.toLowerCase();
+    if (TEST_NAME.test(lower))
+        return "test";
+    if (FIXTURE_NAME.test(lower))
+        return "fixture";
+    if (DATASET_NAME.test(lower))
+        return "dataset";
+    if (PROMPT_NAME.test(lower))
+        return "prompt";
+    if (SCHEMA_NAME.test(lower))
+        return "schema";
+    return undefined;
+};
+const materialRank = (material) => {
+    if (isTopLevelReadme(material.path))
+        return 0;
+    if (material.kind === "doc" && isReadme(material.path))
+        return 1;
+    if (material.kind === "doc")
+        return 2;
+    if (material.kind === "prompt")
+        return 3;
+    if (material.kind === "fixture")
+        return 4;
+    if (material.kind === "dataset")
+        return 5;
+    if (material.kind === "schema")
+        return 6;
+    return 7;
+};
+const surfaceRank = (surface) => {
+    const posix = posixPath(surface.path);
+    if (isTopLevelReadme(posix))
+        return 0;
+    if (posix.startsWith("docs/") || posix.includes("/docs/"))
+        return 1;
+    return 2;
+};
+const compareByPath = (left, right) => posixPath(left).localeCompare(posixPath(right));
+export const inspectRepository = Effect.fn("EvalSetup.inspectRepository")(function* (repositoryRoot) {
+    const fs = yield* FileSystem.FileSystem;
+    const paths = yield* Path.Path;
+    const root = yield* fs
+        .realPath(repositoryRoot)
+        .pipe(Effect.mapError((cause) => inspectionFailure(repositoryRoot, cause)));
+    const surfaces = [];
+    const materials = [];
+    const pendingDirectories = [{ absolutePath: root, relativePath: "", depth: 0 }];
+    const visitedDirectories = new Set([root]);
+    let entriesVisited = 0;
+    let textFilesConsidered = 0;
+    let filesRead = 0;
+    let bytesRead = 0;
+    let skippedOversizedFiles = 0;
+    let truncated = false;
+    while (pendingDirectories.length > 0 && !truncated) {
+        const directory = pendingDirectories.shift();
+        if (directory === undefined)
+            break;
+        const names = directory.depth === 0
+            ? yield* fs
+                .readDirectory(directory.absolutePath)
+                .pipe(Effect.mapError((cause) => inspectionFailure(directory.absolutePath, cause)))
+            : yield* fs.readDirectory(directory.absolutePath).pipe(Effect.orElseSucceed(() => []));
+        names.sort((left, right) => left.localeCompare(right));
+        for (const name of names) {
+            if (entriesVisited >= MAX_INSPECTION_ENTRIES) {
+                truncated = true;
+                break;
+            }
+            entriesVisited += 1;
+            const relativePath = directory.relativePath.length === 0 ? name : paths.join(directory.relativePath, name);
+            if (skippedPath(relativePath))
+                continue;
+            const unresolvedPath = paths.join(directory.absolutePath, name);
+            const canonicalOption = yield* fs.realPath(unresolvedPath).pipe(Effect.option);
+            if (Option.isNone(canonicalOption))
+                continue;
+            const canonicalPath = canonicalOption.value;
+            if (!pathIsWithin(paths, root, canonicalPath))
+                continue;
+            const infoOption = yield* fs.stat(canonicalPath).pipe(Effect.option);
+            if (Option.isNone(infoOption))
+                continue;
+            const info = infoOption.value;
+            if (info.type === "Directory") {
+                if (directory.depth >= MAX_INSPECTION_DEPTH) {
+                    truncated = true;
+                    continue;
+                }
+                if (visitedDirectories.has(canonicalPath))
+                    continue;
+                if (visitedDirectories.size >= MAX_INSPECTION_DIRECTORIES) {
+                    truncated = true;
+                    continue;
+                }
+                visitedDirectories.add(canonicalPath);
+                pendingDirectories.push({
+                    absolutePath: canonicalPath,
+                    relativePath,
+                    depth: directory.depth + 1
+                });
+                continue;
+            }
+            if (info.type !== "File")
+                continue;
+            if (!TEXT_EXTENSIONS.has(paths.extname(relativePath).toLowerCase()))
+                continue;
+            if (textFilesConsidered >= MAX_INSPECTION_FILES) {
+                truncated = true;
+                break;
+            }
+            textFilesConsidered += 1;
+            const kind = materialKind(relativePath);
+            if (kind !== undefined)
+                materials.push({ kind, path: relativePath });
+            const fileBytes = Number(info.size);
+            if (fileBytes > MAX_FILE_BYTES) {
+                skippedOversizedFiles += 1;
+                continue;
+            }
+            if (bytesRead + fileBytes > MAX_TOTAL_BYTES) {
+                skippedOversizedFiles += 1;
+                truncated = true;
+                break;
+            }
+            const contentOption = yield* fs.readFileString(canonicalPath).pipe(Effect.option);
+            if (Option.isNone(contentOption))
+                continue;
+            filesRead += 1;
+            bytesRead += fileBytes;
+            const content = contentOption.value;
+            if (skipSurfacePath(relativePath) || !MODEL_CALL.test(content))
+                continue;
+            const model = MODEL_ID.exec(content)?.[0];
+            surfaces.push({
+                name: relativePath.replace(/\.[^.]+$/u, ""),
+                path: relativePath,
+                ...(model === undefined ? {} : { model })
+            });
+        }
+    }
+    materials.sort((left, right) => {
+        const rank = materialRank(left) - materialRank(right);
+        return rank !== 0 ? rank : compareByPath(left.path, right.path);
+    });
+    surfaces.sort((left, right) => {
+        const rank = surfaceRank(left) - surfaceRank(right);
+        return rank !== 0 ? rank : compareByPath(left.path, right.path);
+    });
+    if (surfaces.length === 0 && materials.some((material) => isTopLevelReadme(material.path))) {
+        surfaces.push({ name: "repository-docs", path: "README.md" });
+    }
+    return {
+        repositoryRoot: root,
+        surfaces,
+        materials,
+        summary: {
+            entriesVisited,
+            textFilesConsidered,
+            filesRead,
+            bytesRead,
+            skippedOversizedFiles,
+            truncated
+        }
+    };
+});
+export const EvalRepositoryInspectorLive = Layer.effect(EvalRepositoryInspector, Effect.gen(function* () {
+    const fs = yield* FileSystem.FileSystem;
+    const paths = yield* Path.Path;
+    return EvalRepositoryInspector.of({
+        inspect: (repositoryRoot) => inspectRepository(repositoryRoot).pipe(Effect.provideService(FileSystem.FileSystem, fs), Effect.provideService(Path.Path, paths))
+    });
+}));

package/dist/model-selection.d.ts

@@ -0,0 +1,6 @@
+export type EvalModelSelection = {
+    readonly candidates: readonly [string, string];
+    readonly judgeModel: string;
+};
+export declare const assertEvalModelSelection: (candidates: readonly string[], judgeModel: string) => EvalModelSelection;
+export declare const modelSelectionFromAnswer: (answer: string) => EvalModelSelection;

package/dist/model-selection.js

@@ -0,0 +1,37 @@
+import { assertExplicitEvalModel } from "@velum-labs/routekit-eval-contracts";
+const EXPLICIT_MODEL_ID = /\b[A-Za-z0-9][A-Za-z0-9._-]*\/[A-Za-z0-9][A-Za-z0-9._~:-]*\b/gu;
+const FORBIDDEN_ALIAS_SEGMENTS = new Set(["auto", "default", "router"]);
+const assertNotAlias = (model) => {
+    const segments = model.toLowerCase().split("/");
+    if (segments.some((segment) => FORBIDDEN_ALIAS_SEGMENTS.has(segment))) {
+        throw new Error(`eval model must be a concrete provider/model id, not alias ${JSON.stringify(model)}`);
+    }
+};
+export const assertEvalModelSelection = (candidates, judgeModel) => {
+    if (candidates.length !== 2) {
+        throw new Error("provide exactly two candidate model ids");
+    }
+    const first = candidates[0];
+    const second = candidates[1];
+    if (first === undefined || second === undefined) {
+        throw new Error("provide exactly two candidate model ids");
+    }
+    assertExplicitEvalModel(first, "candidate");
+    assertExplicitEvalModel(second, "candidate");
+    assertExplicitEvalModel(judgeModel, "judge");
+    assertNotAlias(first);
+    assertNotAlias(second);
+    assertNotAlias(judgeModel);
+    const unique = new Set([first, second, judgeModel].map((model) => model.toLowerCase()));
+    if (unique.size !== 3) {
+        throw new Error("candidate and judge roles require three unique model ids; the judge must differ from both candidates");
+    }
+    return { candidates: [first, second], judgeModel };
+};
+export const modelSelectionFromAnswer = (answer) => {
+    const modelIds = answer.match(EXPLICIT_MODEL_ID) ?? [];
+    if (modelIds.length !== 3) {
+        throw new Error("provide exactly three explicit provider/model ids: two candidates followed by one distinct judge");
+    }
+    return assertEvalModelSelection(modelIds.slice(0, 2), modelIds[2] ?? "");
+};

package/dist/ori-authoring.d.ts

@@ -0,0 +1,16 @@
+import { Context, Effect, Layer } from "effect";
+import { EvalSetupRunnerError } from "./errors.js";
+import type { OriEvalAuthoringApi } from "./ori-result.js";
+export type OriAuthoringInput = {
+    readonly profileId: string;
+    readonly repositoryRoot: string;
+};
+export type OriEvalAuthoringShape = {
+    readonly withAuthoring: <A>(input: OriAuthoringInput, use: (api: OriEvalAuthoringApi) => Promise<A>) => Effect.Effect<A, EvalSetupRunnerError>;
+};
+declare const OriEvalAuthoring_base: Context.ServiceClass<OriEvalAuthoring, "@velum-labs/routekit-eval-setup/OriEvalAuthoring", OriEvalAuthoringShape>;
+export declare class OriEvalAuthoring extends OriEvalAuthoring_base {
+    static layer(service: OriEvalAuthoringShape): Layer.Layer<OriEvalAuthoring, never, never>;
+}
+export declare const oriAuthoringFromApi: (api: OriEvalAuthoringApi) => OriEvalAuthoringShape;
+export {};

package/dist/ori-authoring.js

@@ -0,0 +1,17 @@
+import { Context, Effect, Layer } from "effect";
+import { EvalSetupRunnerError } from "./errors.js";
+export class OriEvalAuthoring extends Context.Service()("@velum-labs/routekit-eval-setup/OriEvalAuthoring") {
+    static layer(service) {
+        return Layer.succeed(OriEvalAuthoring, OriEvalAuthoring.of(service));
+    }
+}
+export const oriAuthoringFromApi = (api) => ({
+    withAuthoring: (_input, use) => Effect.tryPromise({
+        try: () => use(api),
+        catch: (cause) => new EvalSetupRunnerError({
+            operation: "call Ori authoring",
+            detail: cause instanceof Error ? cause.message : String(cause),
+            cause
+        })
+    })
+});

package/dist/ori-result.d.ts

@@ -0,0 +1,45 @@
+/**
+ * Structural Ori library result. RouteKit setup persists and relays this JSON
+ * without importing the eval-engine authoring module.
+ */
+export type OriEvalResult = {
+    readonly ok: boolean;
+    readonly status?: string;
+    readonly runDirectory?: string;
+    readonly scratchWorkspace?: string;
+    readonly tag?: string;
+    readonly prompt?: string;
+    readonly question?: string;
+    readonly context?: string;
+    readonly options?: unknown;
+    readonly accepted?: boolean;
+    readonly state?: unknown;
+    readonly evalRuns?: unknown;
+    readonly evalRunTotals?: unknown;
+    readonly attemptTotals?: unknown;
+    readonly error?: string;
+    readonly [key: string]: unknown;
+};
+export type OriEvalAuthoringApi = {
+    readonly prepare: (input: {
+        readonly repository: string;
+        readonly request?: string;
+        readonly harness?: "pi" | "claude" | "codex";
+        readonly model?: string;
+        readonly judgeModel?: string;
+        readonly existing?: "resume" | "archive" | "stop";
+    }) => Promise<OriEvalResult>;
+    readonly run: (input: {
+        readonly repository?: string;
+        readonly runDirectory?: string;
+    }) => Promise<OriEvalResult>;
+    readonly answer: (input: {
+        readonly answer: string;
+        readonly repository?: string;
+        readonly runDirectory?: string;
+    }) => Promise<OriEvalResult>;
+    readonly status: (input: {
+        readonly repository?: string;
+        readonly runDirectory?: string;
+    }) => Promise<OriEvalResult>;
+};

package/dist/ori-result.js

@@ -0,0 +1 @@
+export {};

package/dist/project-artifacts.d.ts

@@ -0,0 +1,31 @@
+import { type RoutingBasis } from "@velum-labs/routekit-eval-contracts";
+import { Context, Effect, FileSystem, Layer, Path } from "effect";
+import { EvalProjectArtifactError } from "./errors.js";
+import { type EvalArtifactApproval, type EvalEvaluationProposal, type EvalExecutionPlan, type EvalRunReport } from "./project-contracts.js";
+export declare function routingBasisDigest(dimensions: RoutingBasis["dimensions"]): string;
+export declare function evaluationProposalDigest(proposal: Omit<EvalEvaluationProposal, "evaluationDigest">): string;
+export type EvalProjectArtifactsShape = {
+    readonly loadBasisProposal: (repositoryRoot: string) => Effect.Effect<RoutingBasis | undefined, EvalProjectArtifactError, never>;
+    readonly saveBasisProposal: (repositoryRoot: string, basis: RoutingBasis) => Effect.Effect<void, EvalProjectArtifactError, never>;
+    readonly loadBasisApproval: (repositoryRoot: string) => Effect.Effect<EvalArtifactApproval | undefined, EvalProjectArtifactError, never>;
+    readonly saveBasisApproval: (repositoryRoot: string, approval: EvalArtifactApproval) => Effect.Effect<void, EvalProjectArtifactError, never>;
+    readonly loadEvaluationProposal: (repositoryRoot: string) => Effect.Effect<EvalEvaluationProposal | undefined, EvalProjectArtifactError, never>;
+    readonly saveEvaluationProposal: (repositoryRoot: string, proposal: EvalEvaluationProposal) => Effect.Effect<void, EvalProjectArtifactError, never>;
+    readonly loadEvaluationsApproval: (repositoryRoot: string) => Effect.Effect<EvalArtifactApproval | undefined, EvalProjectArtifactError, never>;
+    readonly saveEvaluationsApproval: (repositoryRoot: string, approval: EvalArtifactApproval) => Effect.Effect<void, EvalProjectArtifactError, never>;
+    readonly savePlan: (repositoryRoot: string, plan: EvalExecutionPlan) => Effect.Effect<void, EvalProjectArtifactError, never>;
+    readonly materializePlanSuites: (repositoryRoot: string, plan: EvalExecutionPlan, proposal: EvalEvaluationProposal) => Effect.Effect<void, EvalProjectArtifactError, never>;
+    readonly planSuitePath: (repositoryRoot: string, planId: string, dimensionId: string) => Effect.Effect<string, EvalProjectArtifactError, never>;
+    readonly compositionSuitePath: (repositoryRoot: string, planId: string) => Effect.Effect<string, EvalProjectArtifactError, never>;
+    readonly loadPlan: (repositoryRoot: string, planId: string) => Effect.Effect<EvalExecutionPlan | undefined, EvalProjectArtifactError, never>;
+    readonly listPlans: (repositoryRoot: string) => Effect.Effect<readonly string[], EvalProjectArtifactError, never>;
+    readonly saveRunReport: (repositoryRoot: string, report: EvalRunReport) => Effect.Effect<string, EvalProjectArtifactError, never>;
+    readonly loadRunReport: (repositoryRoot: string, runId: string) => Effect.Effect<EvalRunReport | undefined, EvalProjectArtifactError, never>;
+    readonly listRunReports: (repositoryRoot: string) => Effect.Effect<readonly string[], EvalProjectArtifactError, never>;
+};
+declare const EvalProjectArtifacts_base: Context.ServiceClass<EvalProjectArtifacts, "@velum-labs/routekit-eval-setup/EvalProjectArtifacts", EvalProjectArtifactsShape>;
+export declare class EvalProjectArtifacts extends EvalProjectArtifacts_base {
+}
+export declare const makeFileEvalProjectArtifacts: Effect.Effect<EvalProjectArtifactsShape, never, Path.Path | FileSystem.FileSystem>;
+export declare const EvalProjectArtifactsLive: Layer.Layer<EvalProjectArtifacts, never, Path.Path | FileSystem.FileSystem>;
+export {};