@mupt-ai/dari-router 0.2.3 → 0.2.5

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,4EAA4E;AAC5E,8EAA8E;AAC9E,OAAO,EACL,YAAY,GACb,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EACL,eAAe,GAChB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,gBAAgB,GACjB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EACL,eAAe,GAChB,MAAM,iBAAiB,CAAC;AAwDzB,OAAO,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,4EAA4E;AAC5E,8EAA8E;AAC9E,OAAO,EACL,YAAY,GACb,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EACL,eAAe,GAChB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,gBAAgB,GACjB,MAAM,kBAAkB,CAAC;AAuD1B,OAAO,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC"}

package/dist/jev_client.d.ts

@@ -0,0 +1,31 @@
+import { RouterCoreError } from "./errors.js";
+import type { JsonValue } from "./json.js";
+export declare const DEFAULT_JEV_ENDPOINT = "https://api.typesafe.ai/v1";
+export declare const DEFAULT_JEV_MODEL = "jev-latest";
+export type JevConfig = {
+    apiKey: string;
+    endpoint?: string;
+    model?: string;
+    timeoutMs?: number;
+};
+export type ResolvedJevConfig = {
+    apiKey: string;
+    endpoint: string;
+    model: string;
+    timeoutMs: number;
+};
+export type JevTokenUsage = {
+    input_tokens: number;
+    output_tokens: number;
+};
+export declare class JevOutputError extends RouterCoreError {
+    readonly usage: JevTokenUsage | null;
+    constructor(message: string, code: string, usage?: JevTokenUsage | null);
+}
+export declare function resolveJevConfig(input: JevConfig): ResolvedJevConfig;
+export declare function postSystemOne(request: {
+    model: string;
+    state: JsonValue;
+    questions: Record<string, unknown>;
+}, config: ResolvedJevConfig, signal: AbortSignal | undefined): Promise<unknown>;
+export declare function invalidJevResponse(message: string, usage?: JevTokenUsage | null): JevOutputError;

package/dist/jev_client.js

@@ -0,0 +1,115 @@
+// TypeSafe Jev System One transport. Jev (docs.typesafe.ai) is a System One
+// model: it answers typed questions about a state instead of generating
+// text, and the answers come back as calibrated probabilities. Callers own
+// the questions; this module owns the connection, retries, and errors.
+import { RouterCoreError } from "./errors.js";
+import { RouterFrameworkError } from "./framework_error.js";
+export const DEFAULT_JEV_ENDPOINT = "https://api.typesafe.ai/v1";
+export const DEFAULT_JEV_MODEL = "jev-latest";
+const DEFAULT_JEV_TIMEOUT_MS = 30_000;
+// Rate-limit and overload responses get one backoff retry; anything else is
+// the caller's failure to surface.
+const JEV_RETRY_STATUSES = new Set([429, 529]);
+const JEV_ATTEMPTS = 2;
+const JEV_RETRY_DELAY_MS = 500;
+const JEV_RETRY_MAX_DELAY_MS = 5_000;
+// Jev answered 200 with a body the caller cannot use. Token counts are on
+// the body even then, so hosts can still bill the call.
+export class JevOutputError extends RouterCoreError {
+    usage;
+    constructor(message, code, usage = null) {
+        super("selector_output", message, code);
+        this.name = "JevOutputError";
+        this.usage = usage;
+    }
+}
+export function resolveJevConfig(input) {
+    const apiKey = input.apiKey?.trim() ?? "";
+    if (!apiKey)
+        throw configurationError("Jev requires an apiKey.", "apiKey");
+    const endpoint = (input.endpoint ?? DEFAULT_JEV_ENDPOINT).trim().replace(/\/+$/, "");
+    let url;
+    try {
+        url = new URL(endpoint);
+    }
+    catch {
+        throw configurationError("Jev endpoint must be a valid URL.", "endpoint");
+    }
+    if ((url.protocol !== "https:" && url.protocol !== "http:") || url.username || url.password) {
+        throw configurationError("Jev endpoint must be an HTTP(S) URL without embedded credentials.", "endpoint");
+    }
+    const model = (input.model ?? DEFAULT_JEV_MODEL).trim();
+    if (!model)
+        throw configurationError("Jev model must not be empty.", "model");
+    const timeoutMs = input.timeoutMs ?? DEFAULT_JEV_TIMEOUT_MS;
+    if (!Number.isSafeInteger(timeoutMs) || timeoutMs <= 0) {
+        throw configurationError("Jev timeoutMs must be a positive integer.", "timeoutMs");
+    }
+    return { apiKey, endpoint, model, timeoutMs };
+}
+// One System One call: POSTs the request, retries once on rate limits and
+// overload, and returns the parsed body for the caller to interpret.
+export async function postSystemOne(request, config, signal) {
+    for (let attempt = 1;; attempt += 1) {
+        const response = await fetchSystemOne(request, config, signal);
+        if (response.ok) {
+            try {
+                return await response.json();
+            }
+            catch {
+                throw invalidJevResponse("Jev returned a non-JSON response body.");
+            }
+        }
+        if (attempt < JEV_ATTEMPTS && JEV_RETRY_STATUSES.has(response.status)) {
+            await delay(retryDelayMs(response.headers.get("retry-after")), signal);
+            continue;
+        }
+        const body = await response.text().catch(() => "");
+        throw new RouterFrameworkError("policy", `Jev returned HTTP ${response.status}: ${body}`, "jev_http_error");
+    }
+}
+export function invalidJevResponse(message, usage = null) {
+    return new JevOutputError(message, "selector_invalid_response", usage);
+}
+async function fetchSystemOne(request, config, signal) {
+    const timeout = AbortSignal.timeout(config.timeoutMs);
+    try {
+        return await fetch(`${config.endpoint}/systemone`, {
+            method: "POST",
+            headers: {
+                Authorization: `Bearer ${config.apiKey}`,
+                "Content-Type": "application/json",
+            },
+            body: JSON.stringify(request),
+            signal: signal === undefined ? timeout : AbortSignal.any([signal, timeout]),
+        });
+    }
+    catch (error) {
+        if (signal?.aborted)
+            throw error;
+        throw new RouterFrameworkError("policy", `Jev request failed: ${error instanceof Error ? error.message : String(error)}`, "jev_request_failed", undefined, { cause: error });
+    }
+}
+function retryDelayMs(retryAfter) {
+    const seconds = Number(retryAfter);
+    if (retryAfter === null || !Number.isFinite(seconds) || seconds <= 0)
+        return JEV_RETRY_DELAY_MS;
+    return Math.min(seconds * 1000, JEV_RETRY_MAX_DELAY_MS);
+}
+function delay(ms, signal) {
+    return new Promise((resolve, reject) => {
+        const timer = setTimeout(() => {
+            signal?.removeEventListener("abort", onAbort);
+            resolve();
+        }, ms);
+        const onAbort = () => {
+            clearTimeout(timer);
+            reject(signal?.reason);
+        };
+        signal?.addEventListener("abort", onAbort, { once: true });
+    });
+}
+function configurationError(message, param) {
+    return new RouterCoreError("configuration", message, "jev_configuration_invalid", param);
+}
+//# sourceMappingURL=jev_client.js.map

package/dist/jev_client.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"jev_client.js","sourceRoot":"","sources":["../src/jev_client.ts"],"names":[],"mappings":"AAAA,4EAA4E;AAC5E,wEAAwE;AACxE,2EAA2E;AAC3E,uEAAuE;AAEvE,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAG5D,MAAM,CAAC,MAAM,oBAAoB,GAAG,4BAA4B,CAAC;AACjE,MAAM,CAAC,MAAM,iBAAiB,GAAG,YAAY,CAAC;AAC9C,MAAM,sBAAsB,GAAG,MAAM,CAAC;AAEtC,4EAA4E;AAC5E,mCAAmC;AACnC,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAC/C,MAAM,YAAY,GAAG,CAAC,CAAC;AACvB,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAC/B,MAAM,sBAAsB,GAAG,KAAK,CAAC;AAkBrC,0EAA0E;AAC1E,wDAAwD;AACxD,MAAM,OAAO,cAAe,SAAQ,eAAe;IACxC,KAAK,CAAuB;IAErC,YAAY,OAAe,EAAE,IAAY,EAAE,QAA8B,IAAI;QAC3E,KAAK,CAAC,iBAAiB,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;QACxC,IAAI,CAAC,IAAI,GAAG,gBAAgB,CAAC;QAC7B,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACrB,CAAC;CACF;AAED,MAAM,UAAU,gBAAgB,CAAC,KAAgB;IAC/C,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IAC1C,IAAI,CAAC,MAAM;QAAE,MAAM,kBAAkB,CAAC,yBAAyB,EAAE,QAAQ,CAAC,CAAC;IAC3E,MAAM,QAAQ,GAAG,CAAC,KAAK,CAAC,QAAQ,IAAI,oBAAoB,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IACrF,IAAI,GAAQ,CAAC;IACb,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC1B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,kBAAkB,CAAC,mCAAmC,EAAE,UAAU,CAAC,CAAC;IAC5E,CAAC;IACD,IAAI,CAAC,GAAG,CAAC,QAAQ,KAAK,QAAQ,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO,CAAC,IAAI,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC;QAC5F,MAAM,kBAAkB,CACtB,mEAAmE,EACnE,UAAU,CACX,CAAC;IACJ,CAAC;IACD,MAAM,KAAK,GAAG,CAAC,KAAK,CAAC,KAAK,IAAI,iBAAiB,CAAC,CAAC,IAAI,EAAE,CAAC;IACxD,IAAI,CAAC,KAAK;QAAE,MAAM,kBAAkB,CAAC,8BAA8B,EAAE,OAAO,CAAC,CAAC;IAC9E,MAAM,SAAS,GAAG,KAAK,CAAC,SAAS,IAAI,sBAAsB,CAAC;IAC5D,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,SAAS,CAAC,IAAI,SAAS,IAAI,CAAC,EAAE,CAAC;QACvD,MAAM,kBAAkB,CAAC,2CAA2C,EAAE,WAAW,CAAC,CAAC;IACrF,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;AAChD,CAAC;AAED,0EAA0E;AAC1E,qEAAqE;AACrE,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,OAAgF,EAChF,MAAyB,EACzB,MAA+B;IAE/B,KAAK,IAAI,OAAO,GAAG,CAAC,GAAI,OAAO,IAAI,CAAC,EAAE,CAAC;QACrC,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;QAC/D,IAAI,QAAQ,CAAC,EAAE,EAAE,CAAC;YAChB,IAAI,CAAC;gBACH,OAAO,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YAC/B,CAAC;YAAC,MAAM,CAAC;gBACP,MAAM,kBAAkB,CAAC,wCAAwC,CAAC,CAAC;YACrE,CAAC;QACH,CAAC;QACD,IAAI,OAAO,GAAG,YAAY,IAAI,kBAAkB,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YACtE,MAAM,KAAK,CAAC,YAAY,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;YACvE,SAAS;QACX,CAAC;QACD,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;QACnD,MAAM,IAAI,oBAAoB,CAC5B,QAAQ,EACR,qBAAqB,QAAQ,CAAC,MAAM,KAAK,IAAI,EAAE,EAC/C,gBAAgB,CACjB,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,UAAU,kBAAkB,CAChC,OAAe,EACf,QAA8B,IAAI;IAElC,OAAO,IAAI,cAAc,CAAC,OAAO,EAAE,2BAA2B,EAAE,KAAK,CAAC,CAAC;AACzE,CAAC;AAED,KAAK,UAAU,cAAc,CAC3B,OAAgB,EAChB,MAAyB,EACzB,MAA+B;IAE/B,MAAM,OAAO,GAAG,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IACtD,IAAI,CAAC;QACH,OAAO,MAAM,KAAK,CAAC,GAAG,MAAM,CAAC,QAAQ,YAAY,EAAE;YACjD,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,aAAa,EAAE,UAAU,MAAM,CAAC,MAAM,EAAE;gBACxC,cAAc,EAAE,kBAAkB;aACnC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;YAC7B,MAAM,EAAE,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;SAC5E,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,MAAM,EAAE,OAAO;YAAE,MAAM,KAAK,CAAC;QACjC,MAAM,IAAI,oBAAoB,CAC5B,QAAQ,EACR,uBAAuB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,EAC/E,oBAAoB,EACpB,SAAS,EACT,EAAE,KAAK,EAAE,KAAK,EAAE,CACjB,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,YAAY,CAAC,UAAyB;IAC7C,MAAM,OAAO,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;IACnC,IAAI,UAAU,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,IAAI,CAAC;QAAE,OAAO,kBAAkB,CAAC;IAChG,OAAO,IAAI,CAAC,GAAG,CAAC,OAAO,GAAG,IAAI,EAAE,sBAAsB,CAAC,CAAC;AAC1D,CAAC;AAED,SAAS,KAAK,CAAC,EAAU,EAAE,MAA+B;IACxD,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC5B,MAAM,EAAE,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAC9C,OAAO,EAAE,CAAC;QACZ,CAAC,EAAE,EAAE,CAAC,CAAC;QACP,MAAM,OAAO,GAAG,GAAG,EAAE;YACnB,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACzB,CAAC,CAAC;QACF,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;IAC7D,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,kBAAkB,CAAC,OAAe,EAAE,KAAa;IACxD,OAAO,IAAI,eAAe,CAAC,eAAe,EAAE,OAAO,EAAE,2BAA2B,EAAE,KAAK,CAAC,CAAC;AAC3F,CAAC"}

package/dist/jev_eval_relevance.d.ts

@@ -0,0 +1,49 @@
+import type { JsonValue } from "./json.js";
+import { type ResolvedJevConfig } from "./jev_client.js";
+import { type ChatMessage, type RouterEval } from "./types.js";
+export declare const JEV_EVAL_RELEVANCE_THRESHOLD = 0.3;
+export declare const JEV_EVAL_RELEVANCE_BATCH_SIZE = 100;
+export declare const JEV_EVAL_RELEVANCE_HISTORY_CHAR_BUDGET = 16000;
+export type JevEvalRelevanceState = {
+    task: ChatMessage | null;
+    conversation: ChatMessage[];
+};
+export type JevNoulQuestion = {
+    type: "noul";
+    instructions: JsonValue;
+    criteria: {
+        true: string;
+        false: string;
+    };
+};
+export type JevEvalRelevanceRequest = {
+    model: string;
+    state: JsonValue;
+    questions: Record<string, JevNoulQuestion>;
+};
+export type JevEvalRelevanceCard = {
+    id: string;
+    name: string;
+    relevance: number;
+    kept: boolean;
+};
+export type JevEvalRelevanceSelection = {
+    evals: RouterEval[];
+    cards: JevEvalRelevanceCard[];
+    model: string;
+    usage: {
+        input_tokens: number;
+        output_tokens: number;
+    };
+    requests: JevEvalRelevanceRequest[];
+};
+export declare function buildJevEvalRelevanceRequest(cards: readonly RouterEval[], state: JevEvalRelevanceState, model: string): JevEvalRelevanceRequest;
+export declare function parseJevEvalRelevanceResponse(body: unknown, cards: readonly RouterEval[]): {
+    cards: JevEvalRelevanceCard[];
+    model: string;
+    usage: {
+        input_tokens: number;
+        output_tokens: number;
+    };
+};
+export declare function selectRelevantEvals(cards: readonly RouterEval[], state: JevEvalRelevanceState, config: ResolvedJevConfig, signal?: AbortSignal): Promise<JevEvalRelevanceSelection>;

package/dist/jev_eval_relevance.js

@@ -0,0 +1,176 @@
+// Jev-decided eval relevance. Instead of the router owner curating which
+// benchmark scorecards the selector sees, every candidate card is put to Jev
+// as one yes/no (Noul) question — does this benchmark measure the kind of
+// work the agent is doing? — all cards in a single System One call, and the
+// cards Jev answers yes to become the selector's imported_evals.
+//
+// Jev is literal, so the question names the exact condition and the criteria
+// carry the boundary: a benchmark of the same kind of work (coding, legal,
+// visual design, ...) is relevant even when its exact task differs; a
+// benchmark of a different kind of work is not, however capable a model
+// scoring well on it might be.
+import { JEV_EVAL_RELEVANCE_PROMPT } from "./prompts.js";
+import { invalidJevResponse, postSystemOne } from "./jev_client.js";
+import { compactRoutingHistory, contentText, elideMiddle, ROUTER_USER_MESSAGE_CHAR_CAP, } from "./routing_history.js";
+import { isRecord } from "./types.js";
+// A card is kept when Jev's probability that it is relevant exceeds this.
+// Below even odds on purpose: an irrelevant card is noise the selector can
+// discount, while a missed card is evidence it never sees.
+export const JEV_EVAL_RELEVANCE_THRESHOLD = 0.3;
+// Cards per System One call. Jev evaluates every question against the
+// state in one pass and its latency grows slowly with question count, but
+// each question also counts toward the request's token cap alongside the
+// state; probed against the API, 100 cards plus a 64k-char conversation fit
+// with room to spare while 400 cards alone did not.
+export const JEV_EVAL_RELEVANCE_BATCH_SIZE = 100;
+export const JEV_EVAL_RELEVANCE_HISTORY_CHAR_BUDGET = 16_000;
+export function buildJevEvalRelevanceRequest(cards, state, model) {
+    const questions = {};
+    for (const card of cards) {
+        questions[card.id] = {
+            type: "noul",
+            instructions: {
+                question: JEV_EVAL_RELEVANCE_PROMPT.question,
+                benchmark: {
+                    name: card.name,
+                    ...(card.description ? { description: card.description } : {}),
+                },
+                rules: JEV_EVAL_RELEVANCE_PROMPT.rules,
+            },
+            criteria: JEV_EVAL_RELEVANCE_PROMPT.criteria,
+        };
+    }
+    return {
+        model,
+        state: {
+            task: state.task,
+            conversation: state.conversation,
+        },
+        questions,
+    };
+}
+// Maps a System One body back onto the cards it was asked about. Every card
+// must be answered; a missing or non-Noul answer is a malformed response.
+export function parseJevEvalRelevanceResponse(body, cards) {
+    if (!isRecord(body) || typeof body.model !== "string" || !isRecord(body.answers)) {
+        throw invalidJevResponse("Jev returned an unrecognized eval relevance response body.");
+    }
+    const usage = body.usage;
+    if (!isRecord(usage)
+        || typeof usage.input_tokens !== "number"
+        || typeof usage.output_tokens !== "number") {
+        throw invalidJevResponse("Jev eval relevance response is missing token usage.");
+    }
+    const answers = body.answers;
+    const scored = cards.map((card) => {
+        const answer = answers[card.id];
+        if (!isRecord(answer)
+            || answer.type !== "noul"
+            || typeof answer.noul !== "number"
+            || !Number.isFinite(answer.noul)
+            || answer.noul < 0
+            || answer.noul > 1) {
+            throw invalidJevResponse(`Jev eval relevance response has an invalid noul answer for ${card.id}.`);
+        }
+        return {
+            id: card.id,
+            name: card.name,
+            relevance: answer.noul,
+            kept: answer.noul > JEV_EVAL_RELEVANCE_THRESHOLD,
+        };
+    });
+    return {
+        cards: scored,
+        model: body.model,
+        usage: { input_tokens: usage.input_tokens, output_tokens: usage.output_tokens },
+    };
+}
+// Asks Jev about every card (batched, batches in parallel) and returns the
+// kept cards in their original order plus the per-card probabilities.
+export async function selectRelevantEvals(cards, state, config, signal) {
+    const unique = dedupeById(cards);
+    if (unique.length === 0) {
+        return {
+            evals: [],
+            cards: [],
+            model: config.model,
+            usage: { input_tokens: 0, output_tokens: 0 },
+            requests: [],
+        };
+    }
+    const batches = [];
+    for (let start = 0; start < unique.length; start += JEV_EVAL_RELEVANCE_BATCH_SIZE) {
+        batches.push(unique.slice(start, start + JEV_EVAL_RELEVANCE_BATCH_SIZE));
+    }
+    // History compaction leaves a lone message and the leading instructions
+    // verbatim, so every message is also capped: one large paste must not push
+    // the request past Jev's token limit.
+    const boundedState = {
+        task: state.task === null ? null : boundedMessage(state.task, ROUTER_USER_MESSAGE_CHAR_CAP),
+        conversation: compactRoutingHistory(state.conversation, JEV_EVAL_RELEVANCE_HISTORY_CHAR_BUDGET).map((message) => boundedMessage(message, JEV_EVAL_RELEVANCE_HISTORY_CHAR_BUDGET)),
+    };
+    const requests = batches.map((batch) => buildJevEvalRelevanceRequest(batch, boundedState, config.model));
+    const parsed = await Promise.all(requests.map(async (request, index) => parseJevEvalRelevanceResponse(await postSystemOne(request, config, signal), batches[index])));
+    const scored = parsed.flatMap((result) => result.cards);
+    const kept = new Set(scored.filter((card) => card.kept).map((card) => card.id));
+    return {
+        evals: unique.filter((card) => kept.has(card.id)),
+        cards: scored,
+        model: parsed[0].model,
+        usage: parsed.reduce((total, result) => ({
+            input_tokens: total.input_tokens + result.usage.input_tokens,
+            output_tokens: total.output_tokens + result.usage.output_tokens,
+        }), { input_tokens: 0, output_tokens: 0 }),
+        requests,
+    };
+}
+// Jev only needs the words, so an oversized message becomes its text with the
+// middle of each long part elided, shrinking until the JSON (escapes
+// included) fits.
+function boundedMessage(message, maxChars) {
+    if (JSON.stringify(message).length <= maxChars)
+        return message;
+    const parts = typeof message.content === "string"
+        ? [message.content]
+        : Array.isArray(message.content)
+            ? message.content.map((part) => contentText([part])).filter((text) => text !== "")
+            : [];
+    for (let keepChars = maxChars;; keepChars = Math.floor(keepChars * 0.75)) {
+        const bounded = { role: message.role, content: elideParts(parts, keepChars) };
+        if (keepChars === 0 || JSON.stringify(bounded).length <= maxChars)
+            return bounded;
+    }
+}
+// Shares the budget across text parts so a short ask between long harness
+// parts (reminders, environment dumps) survives: parts within an even share
+// stay whole and the longer parts split what remains.
+function elideParts(parts, maxChars) {
+    const limits = new Array(parts.length).fill(0);
+    let remaining = Math.max(0, maxChars - (parts.length - 1));
+    const bySize = parts.map((_, index) => index).sort((left, right) => parts[left].length - parts[right].length);
+    for (let open = 0; open < bySize.length; open += 1) {
+        const share = Math.floor(remaining / (bySize.length - open));
+        const index = bySize[open];
+        if (parts[index].length > share) {
+            for (const rest of bySize.slice(open))
+                limits[rest] = share;
+            break;
+        }
+        limits[index] = parts[index].length;
+        remaining -= parts[index].length;
+    }
+    return parts.map((part, index) => elideMiddle(part, limits[index])).join("\n");
+}
+// The candidate pool is the router's own cards plus the public catalog; a
+// card the router imported from the catalog appears in both, and the first
+// occurrence (the router's copy) wins.
+function dedupeById(cards) {
+    const seen = new Set();
+    return cards.filter((card) => {
+        if (seen.has(card.id))
+            return false;
+        seen.add(card.id);
+        return true;
+    });
+}
+//# sourceMappingURL=jev_eval_relevance.js.map

package/dist/jev_eval_relevance.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"jev_eval_relevance.js","sourceRoot":"","sources":["../src/jev_eval_relevance.ts"],"names":[],"mappings":"AAAA,yEAAyE;AACzE,6EAA6E;AAC7E,0EAA0E;AAC1E,4EAA4E;AAC5E,iEAAiE;AACjE,EAAE;AACF,6EAA6E;AAC7E,2EAA2E;AAC3E,sEAAsE;AACtE,wEAAwE;AACxE,+BAA+B;AAG/B,OAAO,EAAE,yBAAyB,EAAE,MAAM,cAAc,CAAC;AACzD,OAAO,EAAE,kBAAkB,EAAE,aAAa,EAA0B,MAAM,iBAAiB,CAAC;AAC5F,OAAO,EACL,qBAAqB,EACrB,WAAW,EACX,WAAW,EACX,4BAA4B,GAC7B,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,QAAQ,EAAqC,MAAM,YAAY,CAAC;AAEzE,0EAA0E;AAC1E,2EAA2E;AAC3E,2DAA2D;AAC3D,MAAM,CAAC,MAAM,4BAA4B,GAAG,GAAG,CAAC;AAEhD,sEAAsE;AACtE,0EAA0E;AAC1E,yEAAyE;AACzE,4EAA4E;AAC5E,oDAAoD;AACpD,MAAM,CAAC,MAAM,6BAA6B,GAAG,GAAG,CAAC;AACjD,MAAM,CAAC,MAAM,sCAAsC,GAAG,MAAM,CAAC;AAmC7D,MAAM,UAAU,4BAA4B,CAC1C,KAA4B,EAC5B,KAA4B,EAC5B,KAAa;IAEb,MAAM,SAAS,GAAoC,EAAE,CAAC;IACtD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG;YACnB,IAAI,EAAE,MAAM;YACZ,YAAY,EAAE;gBACZ,QAAQ,EAAE,yBAAyB,CAAC,QAAQ;gBAC5C,SAAS,EAAE;oBACT,IAAI,EAAE,IAAI,CAAC,IAAI;oBACf,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBAC/D;gBACD,KAAK,EAAE,yBAAyB,CAAC,KAAK;aACvC;YACD,QAAQ,EAAE,yBAAyB,CAAC,QAAQ;SAC7C,CAAC;IACJ,CAAC;IACD,OAAO;QACL,KAAK;QACL,KAAK,EAAE;YACL,IAAI,EAAE,KAAK,CAAC,IAA4B;YACxC,YAAY,EAAE,KAAK,CAAC,YAAoC;SACzD;QACD,SAAS;KACV,CAAC;AACJ,CAAC;AAED,4EAA4E;AAC5E,0EAA0E;AAC1E,MAAM,UAAU,6BAA6B,CAC3C,IAAa,EACb,KAA4B;IAM5B,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QACjF,MAAM,kBAAkB,CAAC,4DAA4D,CAAC,CAAC;IACzF,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;IACzB,IACE,CAAC,QAAQ,CAAC,KAAK,CAAC;WACb,OAAO,KAAK,CAAC,YAAY,KAAK,QAAQ;WACtC,OAAO,KAAK,CAAC,aAAa,KAAK,QAAQ,EAC1C,CAAC;QACD,MAAM,kBAAkB,CAAC,qDAAqD,CAAC,CAAC;IAClF,CAAC;IACD,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;IAC7B,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAwB,EAAE;QACtD,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAChC,IACE,CAAC,QAAQ,CAAC,MAAM,CAAC;eACd,MAAM,CAAC,IAAI,KAAK,MAAM;eACtB,OAAO,MAAM,CAAC,IAAI,KAAK,QAAQ;eAC/B,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC;eAC7B,MAAM,CAAC,IAAI,GAAG,CAAC;eACf,MAAM,CAAC,IAAI,GAAG,CAAC,EAClB,CAAC;YACD,MAAM,kBAAkB,CAAC,8DAA8D,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC;QACrG,CAAC;QACD,OAAO;YACL,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,SAAS,EAAE,MAAM,CAAC,IAAI;YACtB,IAAI,EAAE,MAAM,CAAC,IAAI,GAAG,4BAA4B;SACjD,CAAC;IACJ,CAAC,CAAC,CAAC;IACH,OAAO;QACL,KAAK,EAAE,MAAM;QACb,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,KAAK,EAAE,EAAE,YAAY,EAAE,KAAK,CAAC,YAAY,EAAE,aAAa,EAAE,KAAK,CAAC,aAAa,EAAE;KAChF,CAAC;AACJ,CAAC;AAED,2EAA2E;AAC3E,sEAAsE;AACtE,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,KAA4B,EAC5B,KAA4B,EAC5B,MAAyB,EACzB,MAAoB;IAEpB,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;IACjC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO;YACL,KAAK,EAAE,EAAE;YACT,KAAK,EAAE,EAAE;YACT,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,KAAK,EAAE,EAAE,YAAY,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE;YAC5C,QAAQ,EAAE,EAAE;SACb,CAAC;IACJ,CAAC;IACD,MAAM,OAAO,GAAmB,EAAE,CAAC;IACnC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC,MAAM,EAAE,KAAK,IAAI,6BAA6B,EAAE,CAAC;QAClF,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,6BAA6B,CAAC,CAAC,CAAC;IAC3E,CAAC;IACD,wEAAwE;IACxE,2EAA2E;IAC3E,sCAAsC;IACtC,MAAM,YAAY,GAAG;QACnB,IAAI,EAAE,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,cAAc,CAAC,KAAK,CAAC,IAAI,EAAE,4BAA4B,CAAC;QAC3F,YAAY,EAAE,qBAAqB,CACjC,KAAK,CAAC,YAAY,EAClB,sCAAsC,CACvC,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,cAAc,CAAC,OAAO,EAAE,sCAAsC,CAAC,CAAC;KACpF,CAAC;IACF,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,4BAA4B,CAAC,KAAK,EAAE,YAAY,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;IACzG,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,CAC9B,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,CACpC,6BAA6B,CAAC,MAAM,aAAa,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,OAAO,CAAC,KAAK,CAAE,CAAC,CAC7F,CACF,CAAC;IACF,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACxD,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;IAChF,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACjD,KAAK,EAAE,MAAM;QACb,KAAK,EAAE,MAAM,CAAC,CAAC,CAAE,CAAC,KAAK;QACvB,KAAK,EAAE,MAAM,CAAC,MAAM,CAClB,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC;YAClB,YAAY,EAAE,KAAK,CAAC,YAAY,GAAG,MAAM,CAAC,KAAK,CAAC,YAAY;YAC5D,aAAa,EAAE,KAAK,CAAC,aAAa,GAAG,MAAM,CAAC,KAAK,CAAC,aAAa;SAChE,CAAC,EACF,EAAE,YAAY,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,CACtC;QACD,QAAQ;KACT,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,qEAAqE;AACrE,kBAAkB;AAClB,SAAS,cAAc,CAAC,OAAoB,EAAE,QAAgB;IAC5D,IAAI,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,MAAM,IAAI,QAAQ;QAAE,OAAO,OAAO,CAAC;IAC/D,MAAM,KAAK,GAAG,OAAO,OAAO,CAAC,OAAO,KAAK,QAAQ;QAC/C,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC;QACnB,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC;YAC9B,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC;YAClF,CAAC,CAAC,EAAE,CAAC;IACT,KAAK,IAAI,SAAS,GAAG,QAAQ,GAAI,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,EAAE,CAAC;QAC1E,MAAM,OAAO,GAAG,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,OAAO,EAAE,UAAU,CAAC,KAAK,EAAE,SAAS,CAAC,EAAE,CAAC;QAC9E,IAAI,SAAS,KAAK,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,MAAM,IAAI,QAAQ;YAAE,OAAO,OAAO,CAAC;IACpF,CAAC;AACH,CAAC;AAED,0EAA0E;AAC1E,4EAA4E;AAC5E,sDAAsD;AACtD,SAAS,UAAU,CAAC,KAAwB,EAAE,QAAgB;IAC5D,MAAM,MAAM,GAAG,IAAI,KAAK,CAAS,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACvD,IAAI,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,GAAG,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;IAC3D,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CACjE,KAAK,CAAC,IAAI,CAAE,CAAC,MAAM,GAAG,KAAK,CAAC,KAAK,CAAE,CAAC,MAAM,CAC3C,CAAC;IACF,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,MAAM,CAAC,MAAM,EAAE,IAAI,IAAI,CAAC,EAAE,CAAC;QACnD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC;QAC7D,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAE,CAAC;QAC5B,IAAI,KAAK,CAAC,KAAK,CAAE,CAAC,MAAM,GAAG,KAAK,EAAE,CAAC;YACjC,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;gBAAE,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC;YAC5D,MAAM;QACR,CAAC;QACD,MAAM,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,KAAK,CAAE,CAAC,MAAM,CAAC;QACrC,SAAS,IAAI,KAAK,CAAC,KAAK,CAAE,CAAC,MAAM,CAAC;IACpC,CAAC;IACD,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,WAAW,CAAC,IAAI,EAAE,MAAM,CAAC,KAAK,CAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAClF,CAAC;AAED,0EAA0E;AAC1E,2EAA2E;AAC3E,uCAAuC;AACvC,SAAS,UAAU,CAAC,KAA4B;IAC9C,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE;QAC3B,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAAE,OAAO,KAAK,CAAC;QACpC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAClB,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/jev_selector.d.ts

@@ -1,25 +1,11 @@
 import { type AnonymousActionSlot, type AnonymousSelectorInput } from "./anonymous_actions.js";
-import { RouterCoreError } from "./errors.js";
+import { type JevTokenUsage, type ResolvedJevConfig } from "./jev_client.js";
 import type { JsonValue } from "./json.js";
-import type { Selector } from "./route.js";
 import type { CustomSelectorInput, SelectorInput } from "./selector_input.js";
 import { type RoutingDecision } from "./types.js";
-export declare const DEFAULT_JEV_ENDPOINT = "https://api.typesafe.ai/v1";
-export declare const DEFAULT_JEV_MODEL = "jev-latest";
 export declare const JEV_SELECTOR_CONTEXT_WINDOW_CHARS = 64000;
-export type JevSelectorConfig = {
-    apiKey: string;
-    endpoint?: string;
-    model?: string;
-    leaseTurns?: readonly number[];
-    timeoutMs?: number;
-};
-export type ResolvedJevSelectorConfig = {
-    apiKey: string;
-    endpoint: string;
-    model: string;
+export type JevSelectorConfig = ResolvedJevConfig & {
     leaseTurns: readonly number[];
-    timeoutMs: number;
 };
 export type JevChoiceQuestion = {
     type: "choice";
@@ -40,10 +26,6 @@ export type JevChoiceAnswer = {
     confidence: number;
     probabilities: Record<string, number>;
 };
-export type JevTokenUsage = {
-    input_tokens: number;
-    output_tokens: number;
-};
 export type JevSystemOneResponse = {
     model: string;
     answers: {
@@ -52,16 +34,11 @@ export type JevSystemOneResponse = {
     };
     usage: JevTokenUsage;
 };
-export declare class JevSelectorOutputError extends RouterCoreError {
-    readonly usage: JevTokenUsage | null;
-    constructor(message: string, code: string, usage?: JevTokenUsage | null);
-}
 export type JevSelection = {
     decision: RoutingDecision;
     request: JevSystemOneRequest;
     response: JevSystemOneResponse;
 };
-export declare function resolveJevSelectorConfig(input: JevSelectorConfig): ResolvedJevSelectorConfig;
 export declare function buildJevSelectorRequest(anonymousInput: AnonymousSelectorInput, options: {
     model: string;
     leaseTurns: readonly number[];
@@ -73,5 +50,4 @@ export declare function parseJevSelectorResponse(body: unknown, args: {
     decision: RoutingDecision;
     response: JevSystemOneResponse;
 };
-export declare function selectWithJev(selectorInput: SelectorInput | CustomSelectorInput, slots: readonly AnonymousActionSlot[], config: ResolvedJevSelectorConfig, signal?: AbortSignal): Promise<JevSelection>;
-export declare function createJevSelector(input: JevSelectorConfig): Selector;
+export declare function selectWithJev(selectorInput: SelectorInput | CustomSelectorInput, slots: readonly AnonymousActionSlot[], config: JevSelectorConfig, signal?: AbortSignal): Promise<JevSelection>;