@kimpton-ai/evalrouter 0.1.0

package/LICENSE

@@ -0,0 +1,29 @@
+Kimpton EvalRouter SDK License
+
+Copyright (c) 2026 Kimpton. All rights reserved.
+
+Permission is granted to download, install, execute, and make internal copies of
+this SDK and its documentation solely to access the EvalRouter service and to
+integrate that access into applications you own or control, subject to your
+applicable EvalRouter service agreement and authorization.
+
+This is proprietary software, not an open-source license. Except for the limited
+permission above or as required by applicable law, no permission is granted to
+modify, redistribute, sublicense, sell, or use this software to provide a
+competing service. Preserve this license and all copyright notices in permitted
+copies. All rights not expressly granted are reserved.
+
+This license covers only the distributed SDK, CLI, and accompanying
+documentation. It grants no rights to EvalRouter backend or service source,
+benchmarks, datasets, models, credentials, trademarks, or other excluded material.
+Third-party dependencies remain subject to their own licenses.
+
+Downloading this software does not create an account, grant service access,
+provide credits, or waive service charges. Access remains subject to separate
+account, workspace, authorization, and billing requirements.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE, AND NONINFRINGEMENT. TO THE MAXIMUM EXTENT PERMITTED BY APPLICABLE LAW,
+THE COPYRIGHT HOLDERS SHALL NOT BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER
+LIABILITY ARISING FROM THE SOFTWARE OR ITS USE.

package/README.md

@@ -0,0 +1,48 @@
+# EvalRouter TypeScript client
+
+Client for trusted Node.js 22+ applications, using ESM, distributed under the
+proprietary [Kimpton EvalRouter SDK License](LICENSE). Install with
+`npm install @kimpton-ai/evalrouter@0.1.0`. Discover evaluations and models,
+submit bounded evaluation runs, inspect results, and export reports.
+
+Use API keys only in a trusted server process, never a browser bundle. Set
+`EVALROUTER_BASE_URL`, `EVALROUTER_API_KEY` and `EVALROUTER_WORKSPACE_ID` after
+creating and funding an account in the web application. The public API origin is
+`https://api.evalrouter.ai`. Package download does not grant service access.
+While website access is restricted, obtain an invitation from your EvalRouter
+operator before setting up the account and workspace key.
+
+```typescript
+import { Client } from "@kimpton-ai/evalrouter";
+
+const client = new Client();
+const available = await client.evals.list();
+// Select an admitted exact reference and compatible model from discovery.
+const run = await client.run({
+  model: "YOUR_MANAGED_MODEL",
+  eval: "YOUR_EXACT_EVAL_REFERENCE",
+  budgetUsd: "1.00",
+  idempotencyKey: "my-durable-evaluation-request",
+});
+const result = await client.runs.results(run.id);
+await client.runs.exportToFile(run.id, "result.json", { format: "json" });
+```
+
+A budget is a spending cap, not a price guarantee or assurance that an eval fits.
+Managed model routes and checked `connection:<id>` endpoints retain their distinct
+billing bases. Alternatively use `quotes.create`, review the quote, then
+`runs.create(body, durableIdempotencyKey)`. Retry the same authorized request with
+the same idempotency key. Local timeouts do not establish that a submitted run
+failed to commit.
+
+The supported resources are `catalog`, `evals`, `connections`, `quotes`,
+`evaluations` and `runs`. `catalog.evals` and `catalog.eval` are discovery aliases.
+The package includes named request/response types, `Client` (`EvalRouter` alias),
+`APIError`, `ClientError`, `RequestTimeout`, `WaitCancelled`, and typed
+request/wait options. `runs.wait` supports local abort/deadlines; `runs.cancel`
+explicitly cancels server work. `runs.exportToFile` handles versioned JSON/CSV/HTML
+exports and refuses overwrites by default. Workspace and API configuration can
+also be passed in `ClientOptions`.
+
+The HTTP service enforces account access and workspace permissions. Configure
+the client with an existing workspace API key.

package/dist/boundary.d.ts

@@ -0,0 +1,2 @@
+/** Validate supported evaluation inputs before issuing a request. */
+export declare function validateSubmission(resource: "quotes" | "evaluations", input: unknown): void;

package/dist/boundary.js

@@ -0,0 +1,32 @@
+import { ClientError } from "./transport.js";
+/** Validate supported evaluation inputs before issuing a request. */
+export function validateSubmission(resource, input) {
+    const object = (value) => value !== null && typeof value === "object" && !Array.isArray(value);
+    const allowed = new Set(resource === "quotes"
+        ? ["selection", "model", "coverage", "max_charge_microusd"]
+        : ["model", "eval", "budget_usd", "max_charge_microusd", "provider", "coverage", "name", "metadata", "split"]);
+    const fail = () => { throw new ClientError("unsupported_feature", "Choose an admitted eval and a managed model or checked connection."); };
+    if (!object(input) || Object.keys(input).some((key) => !allowed.has(key)))
+        return fail();
+    const model = input.model;
+    if (typeof model === "string" && resource === "evaluations") {
+        if (!model || /^(checkpoint|agent|artifact):/.test(model))
+            return fail();
+    }
+    else if (object(model)) {
+        const keys = model.kind === "managed" ? ["kind", "route_id"] : model.kind === "connection" ? ["kind", "connection_id"] : [];
+        if (keys.length === 0 || Object.keys(model).length !== keys.length || keys.some((key) => !(key in model)))
+            return fail();
+    }
+    else
+        return fail();
+    if (resource === "quotes") {
+        const selection = input.selection;
+        if (!object(selection))
+            return fail();
+        const keys = Object.keys(selection);
+        if (!(keys.length === 1 && keys[0] === "profile_ids") &&
+            !("eval" in selection && keys.every((key) => ["eval", "provider", "split"].includes(key))))
+            return fail();
+    }
+}

package/dist/helpers.d.ts

@@ -0,0 +1,16 @@
+import type { RunRecord, RunsExportParams } from "./types.js";
+import { Resource, type RequestOptions } from "./transport.js";
+export type WaitOptions = {
+    timeoutMs?: number;
+    pollIntervalMs?: number;
+    signal?: AbortSignal;
+    onProgress?: (run: RunRecord) => void | Promise<void>;
+};
+export declare abstract class RunHelpers extends Resource {
+    abstract get(runId: string, options?: RequestOptions): Promise<RunRecord>;
+    abstract export(runId: string, params?: RunsExportParams, options?: RequestOptions): Promise<Uint8Array>;
+    wait(runId: string, options?: WaitOptions): Promise<RunRecord>;
+    exportToFile(runId: string, destination: string, params?: RunsExportParams, options?: RequestOptions & {
+        overwrite?: boolean;
+    }): Promise<string>;
+}

package/dist/helpers.js

@@ -0,0 +1,73 @@
+import { link, lstat, open, rename, unlink } from "node:fs/promises";
+import { randomUUID } from "node:crypto";
+import { dirname, join } from "node:path";
+import { ClientError, Resource, checkpoint, pause, positive, } from "./transport.js";
+const terminal = new Set(["completed", "partial", "failed", "cancelled"]);
+const states = new Set([
+    ...terminal,
+    "queued",
+    "running",
+    "cancelling",
+    "finalizing",
+]);
+export class RunHelpers extends Resource {
+    async wait(runId, options = {}) {
+        const deadline = performance.now() +
+            positive(options.timeoutMs ?? 3_600_000, "Wait timeout");
+        const interval = positive(options.pollIntervalMs ?? 2000, "Poll interval");
+        while (true) {
+            const run = await this.get(runId, {
+                timeoutMs: checkpoint(deadline, options.signal),
+                signal: options.signal,
+            });
+            checkpoint(deadline, options.signal);
+            if (!states.has(run.status))
+                throw new ClientError("invalid_response", "API returned an unknown evaluation state.");
+            if (options.onProgress)
+                await options.onProgress(run);
+            checkpoint(deadline, options.signal);
+            if (terminal.has(run.status))
+                return run;
+            await pause(interval, deadline, options.signal);
+        }
+    }
+    async exportToFile(runId, destination, params = {}, options = {}) {
+        if (!options.overwrite) {
+            try {
+                await lstat(destination);
+                throw new ClientError("destination_exists", "Export destination already exists.");
+            }
+            catch (e) {
+                if (e.code !== "ENOENT")
+                    throw e instanceof ClientError
+                        ? e
+                        : new ClientError("export_write_failed", "Could not access the export destination.");
+            }
+        }
+        const bytes = await this.export(runId, params, options);
+        const temporary = join(dirname(destination), `.kimpton-export-${randomUUID()}`);
+        try {
+            const file = await open(temporary, "wx", 0o600);
+            try {
+                await file.writeFile(bytes);
+                await file.sync();
+            }
+            finally {
+                await file.close();
+            }
+            if (options.overwrite)
+                await rename(temporary, destination);
+            else
+                await link(temporary, destination);
+            return destination;
+        }
+        catch (e) {
+            throw new ClientError(e.code === "EEXIST"
+                ? "destination_exists"
+                : "export_write_failed", "Could not write the export destination.");
+        }
+        finally {
+            await unlink(temporary).catch(() => undefined);
+        }
+    }
+}

package/dist/index.d.ts

@@ -0,0 +1,35 @@
+import { Catalog, Evals, Evaluations, Connections, Quotes, Runs } from "./resources.js";
+import type { RunRecord } from "./types.js";
+import { Transport } from "./transport.js";
+export { APIError, ClientError, RequestTimeout, WaitCancelled, } from "./transport.js";
+export type { ClientOptions, RequestOptions } from "./transport.js";
+export type { WaitOptions } from "./helpers.js";
+export type * from "./types.js";
+export declare class Client extends Transport {
+    readonly catalog: Catalog;
+    readonly connections: Connections;
+    readonly quotes: Quotes;
+    readonly runs: Runs;
+    readonly evals: Evals;
+    readonly evaluations: Evaluations;
+    /**
+     * The canonical call: route `eval` to whichever provider owns it and run it.
+     * `model` is a managed route id or `connection:<uuid>`;
+     * `eval` is `eval://provider/name/version` or `name@version`.
+     */
+    run(input: {
+        model: string;
+        eval: string;
+        budgetUsd?: number | string;
+        maxChargeMicrousd?: string;
+        provider?: string;
+        coverage?: Record<string, unknown>;
+        name?: string;
+        metadata?: Record<string, string>;
+        idempotencyKey?: string;
+        wait?: boolean;
+        timeoutMs?: number;
+        pollIntervalMs?: number;
+    }): Promise<RunRecord>;
+}
+export { Client as EvalRouter };

package/dist/index.js

@@ -0,0 +1,41 @@
+import { Catalog, Evals, Evaluations, Connections, Quotes, Runs } from "./resources.js";
+import { randomUUID } from "node:crypto";
+import { Transport } from "./transport.js";
+export { APIError, ClientError, RequestTimeout, WaitCancelled, } from "./transport.js";
+export class Client extends Transport {
+    catalog = new Catalog(this);
+    connections = new Connections(this);
+    quotes = new Quotes(this);
+    runs = new Runs(this);
+    evals = new Evals(this);
+    evaluations = new Evaluations(this);
+    /**
+     * The canonical call: route `eval` to whichever provider owns it and run it.
+     * `model` is a managed route id or `connection:<uuid>`;
+     * `eval` is `eval://provider/name/version` or `name@version`.
+     */
+    async run(input) {
+        const body = {
+            model: input.model,
+            eval: input.eval,
+            name: input.name ?? "Evaluation",
+            metadata: input.metadata ?? {},
+        };
+        if (input.provider)
+            body.provider = input.provider;
+        if (input.coverage)
+            body.coverage = input.coverage;
+        if (input.maxChargeMicrousd !== undefined)
+            body.max_charge_microusd = input.maxChargeMicrousd;
+        else
+            body.budget_usd = String(input.budgetUsd);
+        const run = await this.evaluations.create(body, input.idempotencyKey ?? `run-${randomUUID()}`);
+        if (input.wait === false)
+            return run;
+        return this.runs.wait(String(run.id), {
+            timeoutMs: input.timeoutMs,
+            pollIntervalMs: input.pollIntervalMs,
+        });
+    }
+}
+export { Client as EvalRouter };

package/dist/resources.d.ts

@@ -0,0 +1,43 @@
+import type * as types from "./types.js";
+import { Resource, type RequestOptions } from "./transport.js";
+import { RunHelpers } from "./helpers.js";
+export declare class Catalog extends Resource {
+    evals(params?: types.CatalogEvalsParams, options?: RequestOptions): Promise<types.Page_EvalRecord_>;
+    eval(ref: string, options?: RequestOptions): Promise<types.EvalRecord>;
+    benchmarks(params?: types.CatalogBenchmarksParams, options?: RequestOptions): Promise<types.Page_BenchmarkRecord_>;
+    benchmark(slug: string, options?: RequestOptions): Promise<types.BenchmarkDetail>;
+    models(params?: types.CatalogModelsParams, options?: RequestOptions): Promise<types.Page_ManagedRoute_>;
+    execution_options(options?: RequestOptions): Promise<types.ExecutionOptions>;
+    environments(params?: types.CatalogEnvironmentsParams, options?: RequestOptions): Promise<types.Page_EnvironmentRecord_>;
+    environment(family: string, options?: RequestOptions): Promise<types.EnvironmentFamily>;
+}
+export declare class Connections extends Resource {
+    list(params?: types.ConnectionsListParams, options?: RequestOptions): Promise<types.Page_ConnectionRecord_>;
+    create(body: types.NewConnection, options?: RequestOptions): Promise<types.ConnectionRecord>;
+    check(connection_id: string, options?: RequestOptions): Promise<types.ConnectionRecord>;
+    update(connection_id: string, body: types.UpdateConnection, options?: RequestOptions): Promise<types.ConnectionRecord>;
+    disable(connection_id: string, options?: RequestOptions): Promise<types.OKResponse>;
+}
+export declare class Quotes extends Resource {
+    create(body: types.NewQuote, options?: RequestOptions): Promise<types.QuoteRecord>;
+    get(quote_id: string, options?: RequestOptions): Promise<types.QuoteRecord>;
+}
+export declare class Runs extends RunHelpers {
+    create(body: types.NewRun, idempotency_key: string, options?: RequestOptions): Promise<types.RunRecord>;
+    list(params?: types.RunsListParams, options?: RequestOptions): Promise<types.Page_RunRecord_>;
+    get(run_id: string, options?: RequestOptions): Promise<types.RunRecord>;
+    receipt(run_id: string, options?: RequestOptions): Promise<types.RunReceipt>;
+    executions(run_id: string, params?: types.RunsExecutionsParams, options?: RequestOptions): Promise<types.SandboxExecutionPage>;
+    events(run_id: string, params?: types.RunsEventsParams, options?: RequestOptions): Promise<types.EventPage>;
+    cancel(run_id: string, options?: RequestOptions): Promise<types.RunRecord>;
+    results(run_id: string, params?: types.RunsResultsParams, options?: RequestOptions): Promise<types.ResultRecord>;
+    export(run_id: string, params?: types.RunsExportParams, options?: RequestOptions): Promise<Uint8Array>;
+}
+export declare class Evals extends Resource {
+    list(params?: types.EvalsListParams, options?: RequestOptions): Promise<types.Page_EvalRecord_>;
+    get(ref: string, options?: RequestOptions): Promise<types.EvalRecord>;
+}
+export declare class Evaluations extends Resource {
+    create(body: types.NewEvaluation, idempotency_key: string, options?: RequestOptions): Promise<types.RunRecord>;
+    get(evaluation_id: string, options?: RequestOptions): Promise<types.RunRecord>;
+}

package/dist/resources.js

@@ -0,0 +1,101 @@
+import { Resource, segment } from "./transport.js";
+import { RunHelpers } from "./helpers.js";
+import { validateSubmission } from "./boundary.js";
+export class Catalog extends Resource {
+    async evals(params = {}, options) {
+        return this.client.request("GET", `/evals`, { retry: true, options, query: params });
+    }
+    async eval(ref, options) {
+        return this.client.request("GET", `/evals/${segment(ref)}`, { retry: true, options });
+    }
+    async benchmarks(params = {}, options) {
+        return this.client.request("GET", `/catalog/benchmarks`, { retry: true, options, query: params });
+    }
+    async benchmark(slug, options) {
+        return this.client.request("GET", `/catalog/benchmarks/${segment(slug)}`, { retry: true, options });
+    }
+    async models(params = {}, options) {
+        return this.client.request("GET", `/catalog/models`, { retry: true, options, query: params });
+    }
+    async execution_options(options) {
+        return this.client.request("GET", `/catalog/execution-options`, { retry: true, options });
+    }
+    async environments(params = {}, options) {
+        return this.client.request("GET", `/catalog/environments`, { retry: true, options, query: params });
+    }
+    async environment(family, options) {
+        return this.client.request("GET", `/catalog/environments/${segment(family)}`, { retry: true, options });
+    }
+}
+export class Connections extends Resource {
+    async list(params = {}, options) {
+        return this.client.request("GET", `/connections`, { retry: true, options, query: params });
+    }
+    async create(body, options) {
+        return this.client.request("POST", `/connections`, { retry: false, options, body: body });
+    }
+    async check(connection_id, options) {
+        return this.client.request("POST", `/connections/${segment(connection_id)}/check`, { retry: false, options });
+    }
+    async update(connection_id, body, options) {
+        return this.client.request("PATCH", `/connections/${segment(connection_id)}`, { retry: false, options, body: body });
+    }
+    async disable(connection_id, options) {
+        return this.client.request("DELETE", `/connections/${segment(connection_id)}`, { retry: false, options });
+    }
+}
+export class Quotes extends Resource {
+    async create(body, options) {
+        validateSubmission("quotes", body);
+        return this.client.request("POST", `/quotes`, { retry: false, options, body: body });
+    }
+    async get(quote_id, options) {
+        return this.client.request("GET", `/quotes/${segment(quote_id)}`, { retry: true, options });
+    }
+}
+export class Runs extends RunHelpers {
+    async create(body, idempotency_key, options) {
+        return this.client.request("POST", `/runs`, { retry: true, options, body: body, idempotency_key: idempotency_key });
+    }
+    async list(params = {}, options) {
+        return this.client.request("GET", `/runs`, { retry: true, options, query: params });
+    }
+    async get(run_id, options) {
+        return this.client.request("GET", `/runs/${segment(run_id)}`, { retry: true, options });
+    }
+    async receipt(run_id, options) {
+        return this.client.request("GET", `/runs/${segment(run_id)}/receipt`, { retry: true, options });
+    }
+    async executions(run_id, params = {}, options) {
+        return this.client.request("GET", `/runs/${segment(run_id)}/executions`, { retry: true, options, query: params });
+    }
+    async events(run_id, params = {}, options) {
+        return this.client.request("GET", `/runs/${segment(run_id)}/events`, { retry: true, options, query: params });
+    }
+    async cancel(run_id, options) {
+        return this.client.request("POST", `/runs/${segment(run_id)}/cancel`, { retry: true, options });
+    }
+    async results(run_id, params = {}, options) {
+        return this.client.request("GET", `/runs/${segment(run_id)}/results`, { retry: true, options, query: params });
+    }
+    async export(run_id, params = {}, options) {
+        return this.client.request("GET", `/runs/${segment(run_id)}/export`, { retry: true, options, query: params, binary: true });
+    }
+}
+export class Evals extends Resource {
+    async list(params = {}, options) {
+        return this.client.request("GET", `/evals`, { retry: true, options, query: params });
+    }
+    async get(ref, options) {
+        return this.client.request("GET", `/evals/${segment(ref)}`, { retry: true, options });
+    }
+}
+export class Evaluations extends Resource {
+    async create(body, idempotency_key, options) {
+        validateSubmission("evaluations", body);
+        return this.client.request("POST", `/evaluations`, { retry: true, options, body: body, idempotency_key: idempotency_key });
+    }
+    async get(evaluation_id, options) {
+        return this.client.request("GET", `/evaluations/${segment(evaluation_id)}`, { retry: true, options });
+    }
+}

package/dist/transport.d.ts

@@ -0,0 +1,73 @@
+export declare class ClientError extends Error {
+    readonly code: string;
+    idempotency_key?: string;
+    constructor(code: string, message: string);
+    toJSON(): {
+        idempotency_key?: string | undefined;
+        code: string;
+        message: string;
+    };
+}
+export declare class APIError extends ClientError {
+    readonly status: number;
+    readonly requestId: string | null;
+    readonly details: unknown;
+    constructor(status: number, code: string, requestId?: string | null, message?: string, details?: unknown);
+    toJSON(): {
+        status: number;
+        request_id: string | null;
+        details: unknown;
+        idempotency_key?: string | undefined;
+        code: string;
+        message: string;
+    };
+}
+export declare class RequestTimeout extends ClientError {
+    constructor();
+}
+export declare class WaitCancelled extends ClientError {
+    constructor();
+}
+export type RequestOptions = {
+    timeoutMs?: number;
+    signal?: AbortSignal;
+};
+export type ClientOptions = {
+    baseUrl?: string;
+    apiKey?: string;
+    workspaceId?: string;
+    timeoutMs?: number;
+    requestTimeoutMs?: number;
+    maxRetries?: number;
+    maxResponseBytes?: number;
+    maxExportBytes?: number;
+    allowHttp?: boolean;
+    fetch?: typeof globalThis.fetch;
+};
+type Request = {
+    body?: unknown;
+    query?: object;
+    retry: boolean;
+    idempotency_key?: string;
+    binary?: boolean;
+    options?: RequestOptions;
+};
+export declare function positive(value: number, label: string): number;
+export declare function segment(value: string): string;
+export declare function checkpoint(deadline: number, signal?: AbortSignal): number;
+export declare function pause(delay: number, deadline: number, signal?: AbortSignal): Promise<void>;
+export declare class Transport {
+    #private;
+    readonly timeoutMs: number;
+    readonly requestTimeoutMs: number;
+    readonly maxRetries: number;
+    readonly maxResponseBytes: number;
+    readonly maxExportBytes: number;
+    constructor(options?: ClientOptions);
+    request<T>(method: string, path: string, request: Request): Promise<T>;
+}
+export declare class Resource {
+    protected readonly client: Transport;
+    constructor(client: Transport);
+}
+export {};