@mupt-ai/dari-router 0.2.2 → 0.2.4

package/dist/jev_eval_relevance.d.ts

@@ -0,0 +1,49 @@
+import type { JsonValue } from "./json.js";
+import { type ResolvedJevConfig } from "./jev_client.js";
+import { type ChatMessage, type RouterEval } from "./types.js";
+export declare const JEV_EVAL_RELEVANCE_THRESHOLD = 0.3;
+export declare const JEV_EVAL_RELEVANCE_BATCH_SIZE = 100;
+export declare const JEV_EVAL_RELEVANCE_HISTORY_CHAR_BUDGET = 16000;
+export type JevEvalRelevanceState = {
+    task: ChatMessage | null;
+    conversation: ChatMessage[];
+};
+export type JevNoulQuestion = {
+    type: "noul";
+    instructions: JsonValue;
+    criteria: {
+        true: string;
+        false: string;
+    };
+};
+export type JevEvalRelevanceRequest = {
+    model: string;
+    state: JsonValue;
+    questions: Record<string, JevNoulQuestion>;
+};
+export type JevEvalRelevanceCard = {
+    id: string;
+    name: string;
+    relevance: number;
+    kept: boolean;
+};
+export type JevEvalRelevanceSelection = {
+    evals: RouterEval[];
+    cards: JevEvalRelevanceCard[];
+    model: string;
+    usage: {
+        input_tokens: number;
+        output_tokens: number;
+    };
+    requests: JevEvalRelevanceRequest[];
+};
+export declare function buildJevEvalRelevanceRequest(cards: readonly RouterEval[], state: JevEvalRelevanceState, model: string): JevEvalRelevanceRequest;
+export declare function parseJevEvalRelevanceResponse(body: unknown, cards: readonly RouterEval[]): {
+    cards: JevEvalRelevanceCard[];
+    model: string;
+    usage: {
+        input_tokens: number;
+        output_tokens: number;
+    };
+};
+export declare function selectRelevantEvals(cards: readonly RouterEval[], state: JevEvalRelevanceState, config: ResolvedJevConfig, signal?: AbortSignal): Promise<JevEvalRelevanceSelection>;

package/dist/jev_eval_relevance.js

@@ -0,0 +1,176 @@
+// Jev-decided eval relevance. Instead of the router owner curating which
+// benchmark scorecards the selector sees, every candidate card is put to Jev
+// as one yes/no (Noul) question — does this benchmark measure the kind of
+// work the agent is doing? — all cards in a single System One call, and the
+// cards Jev answers yes to become the selector's imported_evals.
+//
+// Jev is literal, so the question names the exact condition and the criteria
+// carry the boundary: a benchmark of the same kind of work (coding, legal,
+// visual design, ...) is relevant even when its exact task differs; a
+// benchmark of a different kind of work is not, however capable a model
+// scoring well on it might be.
+import { JEV_EVAL_RELEVANCE_PROMPT } from "./prompts.js";
+import { invalidJevResponse, postSystemOne } from "./jev_client.js";
+import { compactRoutingHistory, contentText, elideMiddle, ROUTER_USER_MESSAGE_CHAR_CAP, } from "./routing_history.js";
+import { isRecord } from "./types.js";
+// A card is kept when Jev's probability that it is relevant exceeds this.
+// Below even odds on purpose: an irrelevant card is noise the selector can
+// discount, while a missed card is evidence it never sees.
+export const JEV_EVAL_RELEVANCE_THRESHOLD = 0.3;
+// Cards per System One call. Jev evaluates every question against the
+// state in one pass and its latency grows slowly with question count, but
+// each question also counts toward the request's token cap alongside the
+// state; probed against the API, 100 cards plus a 64k-char conversation fit
+// with room to spare while 400 cards alone did not.
+export const JEV_EVAL_RELEVANCE_BATCH_SIZE = 100;
+export const JEV_EVAL_RELEVANCE_HISTORY_CHAR_BUDGET = 16_000;
+export function buildJevEvalRelevanceRequest(cards, state, model) {
+    const questions = {};
+    for (const card of cards) {
+        questions[card.id] = {
+            type: "noul",
+            instructions: {
+                question: JEV_EVAL_RELEVANCE_PROMPT.question,
+                benchmark: {
+                    name: card.name,
+                    ...(card.description ? { description: card.description } : {}),
+                },
+                rules: JEV_EVAL_RELEVANCE_PROMPT.rules,
+            },
+            criteria: JEV_EVAL_RELEVANCE_PROMPT.criteria,
+        };
+    }
+    return {
+        model,
+        state: {
+            task: state.task,
+            conversation: state.conversation,
+        },
+        questions,
+    };
+}
+// Maps a System One body back onto the cards it was asked about. Every card
+// must be answered; a missing or non-Noul answer is a malformed response.
+export function parseJevEvalRelevanceResponse(body, cards) {
+    if (!isRecord(body) || typeof body.model !== "string" || !isRecord(body.answers)) {
+        throw invalidJevResponse("Jev returned an unrecognized eval relevance response body.");
+    }
+    const usage = body.usage;
+    if (!isRecord(usage)
+        || typeof usage.input_tokens !== "number"
+        || typeof usage.output_tokens !== "number") {
+        throw invalidJevResponse("Jev eval relevance response is missing token usage.");
+    }
+    const answers = body.answers;
+    const scored = cards.map((card) => {
+        const answer = answers[card.id];
+        if (!isRecord(answer)
+            || answer.type !== "noul"
+            || typeof answer.noul !== "number"
+            || !Number.isFinite(answer.noul)
+            || answer.noul < 0
+            || answer.noul > 1) {
+            throw invalidJevResponse(`Jev eval relevance response has an invalid noul answer for ${card.id}.`);
+        }
+        return {
+            id: card.id,
+            name: card.name,
+            relevance: answer.noul,
+            kept: answer.noul > JEV_EVAL_RELEVANCE_THRESHOLD,
+        };
+    });
+    return {
+        cards: scored,
+        model: body.model,
+        usage: { input_tokens: usage.input_tokens, output_tokens: usage.output_tokens },
+    };
+}
+// Asks Jev about every card (batched, batches in parallel) and returns the
+// kept cards in their original order plus the per-card probabilities.
+export async function selectRelevantEvals(cards, state, config, signal) {
+    const unique = dedupeById(cards);
+    if (unique.length === 0) {
+        return {
+            evals: [],
+            cards: [],
+            model: config.model,
+            usage: { input_tokens: 0, output_tokens: 0 },
+            requests: [],
+        };
+    }
+    const batches = [];
+    for (let start = 0; start < unique.length; start += JEV_EVAL_RELEVANCE_BATCH_SIZE) {
+        batches.push(unique.slice(start, start + JEV_EVAL_RELEVANCE_BATCH_SIZE));
+    }
+    // History compaction leaves a lone message and the leading instructions
+    // verbatim, so every message is also capped: one large paste must not push
+    // the request past Jev's token limit.
+    const boundedState = {
+        task: state.task === null ? null : boundedMessage(state.task, ROUTER_USER_MESSAGE_CHAR_CAP),
+        conversation: compactRoutingHistory(state.conversation, JEV_EVAL_RELEVANCE_HISTORY_CHAR_BUDGET).map((message) => boundedMessage(message, JEV_EVAL_RELEVANCE_HISTORY_CHAR_BUDGET)),
+    };
+    const requests = batches.map((batch) => buildJevEvalRelevanceRequest(batch, boundedState, config.model));
+    const parsed = await Promise.all(requests.map(async (request, index) => parseJevEvalRelevanceResponse(await postSystemOne(request, config, signal), batches[index])));
+    const scored = parsed.flatMap((result) => result.cards);
+    const kept = new Set(scored.filter((card) => card.kept).map((card) => card.id));
+    return {
+        evals: unique.filter((card) => kept.has(card.id)),
+        cards: scored,
+        model: parsed[0].model,
+        usage: parsed.reduce((total, result) => ({
+            input_tokens: total.input_tokens + result.usage.input_tokens,
+            output_tokens: total.output_tokens + result.usage.output_tokens,
+        }), { input_tokens: 0, output_tokens: 0 }),
+        requests,
+    };
+}
+// Jev only needs the words, so an oversized message becomes its text with the
+// middle of each long part elided, shrinking until the JSON (escapes
+// included) fits.
+function boundedMessage(message, maxChars) {
+    if (JSON.stringify(message).length <= maxChars)
+        return message;
+    const parts = typeof message.content === "string"
+        ? [message.content]
+        : Array.isArray(message.content)
+            ? message.content.map((part) => contentText([part])).filter((text) => text !== "")
+            : [];
+    for (let keepChars = maxChars;; keepChars = Math.floor(keepChars * 0.75)) {
+        const bounded = { role: message.role, content: elideParts(parts, keepChars) };
+        if (keepChars === 0 || JSON.stringify(bounded).length <= maxChars)
+            return bounded;
+    }
+}
+// Shares the budget across text parts so a short ask between long harness
+// parts (reminders, environment dumps) survives: parts within an even share
+// stay whole and the longer parts split what remains.
+function elideParts(parts, maxChars) {
+    const limits = new Array(parts.length).fill(0);
+    let remaining = Math.max(0, maxChars - (parts.length - 1));
+    const bySize = parts.map((_, index) => index).sort((left, right) => parts[left].length - parts[right].length);
+    for (let open = 0; open < bySize.length; open += 1) {
+        const share = Math.floor(remaining / (bySize.length - open));
+        const index = bySize[open];
+        if (parts[index].length > share) {
+            for (const rest of bySize.slice(open))
+                limits[rest] = share;
+            break;
+        }
+        limits[index] = parts[index].length;
+        remaining -= parts[index].length;
+    }
+    return parts.map((part, index) => elideMiddle(part, limits[index])).join("\n");
+}
+// The candidate pool is the router's own cards plus the public catalog; a
+// card the router imported from the catalog appears in both, and the first
+// occurrence (the router's copy) wins.
+function dedupeById(cards) {
+    const seen = new Set();
+    return cards.filter((card) => {
+        if (seen.has(card.id))
+            return false;
+        seen.add(card.id);
+        return true;
+    });
+}
+//# sourceMappingURL=jev_eval_relevance.js.map

package/dist/jev_eval_relevance.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"jev_eval_relevance.js","sourceRoot":"","sources":["../src/jev_eval_relevance.ts"],"names":[],"mappings":"AAAA,yEAAyE;AACzE,6EAA6E;AAC7E,0EAA0E;AAC1E,4EAA4E;AAC5E,iEAAiE;AACjE,EAAE;AACF,6EAA6E;AAC7E,2EAA2E;AAC3E,sEAAsE;AACtE,wEAAwE;AACxE,+BAA+B;AAG/B,OAAO,EAAE,yBAAyB,EAAE,MAAM,cAAc,CAAC;AACzD,OAAO,EAAE,kBAAkB,EAAE,aAAa,EAA0B,MAAM,iBAAiB,CAAC;AAC5F,OAAO,EACL,qBAAqB,EACrB,WAAW,EACX,WAAW,EACX,4BAA4B,GAC7B,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,QAAQ,EAAqC,MAAM,YAAY,CAAC;AAEzE,0EAA0E;AAC1E,2EAA2E;AAC3E,2DAA2D;AAC3D,MAAM,CAAC,MAAM,4BAA4B,GAAG,GAAG,CAAC;AAEhD,sEAAsE;AACtE,0EAA0E;AAC1E,yEAAyE;AACzE,4EAA4E;AAC5E,oDAAoD;AACpD,MAAM,CAAC,MAAM,6BAA6B,GAAG,GAAG,CAAC;AACjD,MAAM,CAAC,MAAM,sCAAsC,GAAG,MAAM,CAAC;AAmC7D,MAAM,UAAU,4BAA4B,CAC1C,KAA4B,EAC5B,KAA4B,EAC5B,KAAa;IAEb,MAAM,SAAS,GAAoC,EAAE,CAAC;IACtD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG;YACnB,IAAI,EAAE,MAAM;YACZ,YAAY,EAAE;gBACZ,QAAQ,EAAE,yBAAyB,CAAC,QAAQ;gBAC5C,SAAS,EAAE;oBACT,IAAI,EAAE,IAAI,CAAC,IAAI;oBACf,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBAC/D;gBACD,KAAK,EAAE,yBAAyB,CAAC,KAAK;aACvC;YACD,QAAQ,EAAE,yBAAyB,CAAC,QAAQ;SAC7C,CAAC;IACJ,CAAC;IACD,OAAO;QACL,KAAK;QACL,KAAK,EAAE;YACL,IAAI,EAAE,KAAK,CAAC,IAA4B;YACxC,YAAY,EAAE,KAAK,CAAC,YAAoC;SACzD;QACD,SAAS;KACV,CAAC;AACJ,CAAC;AAED,4EAA4E;AAC5E,0EAA0E;AAC1E,MAAM,UAAU,6BAA6B,CAC3C,IAAa,EACb,KAA4B;IAM5B,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QACjF,MAAM,kBAAkB,CAAC,4DAA4D,CAAC,CAAC;IACzF,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;IACzB,IACE,CAAC,QAAQ,CAAC,KAAK,CAAC;WACb,OAAO,KAAK,CAAC,YAAY,KAAK,QAAQ;WACtC,OAAO,KAAK,CAAC,aAAa,KAAK,QAAQ,EAC1C,CAAC;QACD,MAAM,kBAAkB,CAAC,qDAAqD,CAAC,CAAC;IAClF,CAAC;IACD,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;IAC7B,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAwB,EAAE;QACtD,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAChC,IACE,CAAC,QAAQ,CAAC,MAAM,CAAC;eACd,MAAM,CAAC,IAAI,KAAK,MAAM;eACtB,OAAO,MAAM,CAAC,IAAI,KAAK,QAAQ;eAC/B,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC;eAC7B,MAAM,CAAC,IAAI,GAAG,CAAC;eACf,MAAM,CAAC,IAAI,GAAG,CAAC,EAClB,CAAC;YACD,MAAM,kBAAkB,CAAC,8DAA8D,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC;QACrG,CAAC;QACD,OAAO;YACL,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,SAAS,EAAE,MAAM,CAAC,IAAI;YACtB,IAAI,EAAE,MAAM,CAAC,IAAI,GAAG,4BAA4B;SACjD,CAAC;IACJ,CAAC,CAAC,CAAC;IACH,OAAO;QACL,KAAK,EAAE,MAAM;QACb,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,KAAK,EAAE,EAAE,YAAY,EAAE,KAAK,CAAC,YAAY,EAAE,aAAa,EAAE,KAAK,CAAC,aAAa,EAAE;KAChF,CAAC;AACJ,CAAC;AAED,2EAA2E;AAC3E,sEAAsE;AACtE,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,KAA4B,EAC5B,KAA4B,EAC5B,MAAyB,EACzB,MAAoB;IAEpB,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;IACjC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO;YACL,KAAK,EAAE,EAAE;YACT,KAAK,EAAE,EAAE;YACT,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,KAAK,EAAE,EAAE,YAAY,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE;YAC5C,QAAQ,EAAE,EAAE;SACb,CAAC;IACJ,CAAC;IACD,MAAM,OAAO,GAAmB,EAAE,CAAC;IACnC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC,MAAM,EAAE,KAAK,IAAI,6BAA6B,EAAE,CAAC;QAClF,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,6BAA6B,CAAC,CAAC,CAAC;IAC3E,CAAC;IACD,wEAAwE;IACxE,2EAA2E;IAC3E,sCAAsC;IACtC,MAAM,YAAY,GAAG;QACnB,IAAI,EAAE,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,cAAc,CAAC,KAAK,CAAC,IAAI,EAAE,4BAA4B,CAAC;QAC3F,YAAY,EAAE,qBAAqB,CACjC,KAAK,CAAC,YAAY,EAClB,sCAAsC,CACvC,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,cAAc,CAAC,OAAO,EAAE,sCAAsC,CAAC,CAAC;KACpF,CAAC;IACF,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,4BAA4B,CAAC,KAAK,EAAE,YAAY,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;IACzG,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,CAC9B,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,CACpC,6BAA6B,CAAC,MAAM,aAAa,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,OAAO,CAAC,KAAK,CAAE,CAAC,CAC7F,CACF,CAAC;IACF,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACxD,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;IAChF,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACjD,KAAK,EAAE,MAAM;QACb,KAAK,EAAE,MAAM,CAAC,CAAC,CAAE,CAAC,KAAK;QACvB,KAAK,EAAE,MAAM,CAAC,MAAM,CAClB,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC;YAClB,YAAY,EAAE,KAAK,CAAC,YAAY,GAAG,MAAM,CAAC,KAAK,CAAC,YAAY;YAC5D,aAAa,EAAE,KAAK,CAAC,aAAa,GAAG,MAAM,CAAC,KAAK,CAAC,aAAa;SAChE,CAAC,EACF,EAAE,YAAY,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,CACtC;QACD,QAAQ;KACT,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,qEAAqE;AACrE,kBAAkB;AAClB,SAAS,cAAc,CAAC,OAAoB,EAAE,QAAgB;IAC5D,IAAI,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,MAAM,IAAI,QAAQ;QAAE,OAAO,OAAO,CAAC;IAC/D,MAAM,KAAK,GAAG,OAAO,OAAO,CAAC,OAAO,KAAK,QAAQ;QAC/C,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC;QACnB,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC;YAC9B,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC;YAClF,CAAC,CAAC,EAAE,CAAC;IACT,KAAK,IAAI,SAAS,GAAG,QAAQ,GAAI,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,EAAE,CAAC;QAC1E,MAAM,OAAO,GAAG,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,OAAO,EAAE,UAAU,CAAC,KAAK,EAAE,SAAS,CAAC,EAAE,CAAC;QAC9E,IAAI,SAAS,KAAK,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,MAAM,IAAI,QAAQ;YAAE,OAAO,OAAO,CAAC;IACpF,CAAC;AACH,CAAC;AAED,0EAA0E;AAC1E,4EAA4E;AAC5E,sDAAsD;AACtD,SAAS,UAAU,CAAC,KAAwB,EAAE,QAAgB;IAC5D,MAAM,MAAM,GAAG,IAAI,KAAK,CAAS,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACvD,IAAI,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,GAAG,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;IAC3D,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CACjE,KAAK,CAAC,IAAI,CAAE,CAAC,MAAM,GAAG,KAAK,CAAC,KAAK,CAAE,CAAC,MAAM,CAC3C,CAAC;IACF,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,MAAM,CAAC,MAAM,EAAE,IAAI,IAAI,CAAC,EAAE,CAAC;QACnD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC;QAC7D,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAE,CAAC;QAC5B,IAAI,KAAK,CAAC,KAAK,CAAE,CAAC,MAAM,GAAG,KAAK,EAAE,CAAC;YACjC,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;gBAAE,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC;YAC5D,MAAM;QACR,CAAC;QACD,MAAM,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,KAAK,CAAE,CAAC,MAAM,CAAC;QACrC,SAAS,IAAI,KAAK,CAAC,KAAK,CAAE,CAAC,MAAM,CAAC;IACpC,CAAC;IACD,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,WAAW,CAAC,IAAI,EAAE,MAAM,CAAC,KAAK,CAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAClF,CAAC;AAED,0EAA0E;AAC1E,2EAA2E;AAC3E,uCAAuC;AACvC,SAAS,UAAU,CAAC,KAA4B;IAC9C,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE;QAC3B,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAAE,OAAO,KAAK,CAAC;QACpC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAClB,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/jev_selector.d.ts

@@ -1,25 +1,11 @@
 import { type AnonymousActionSlot, type AnonymousSelectorInput } from "./anonymous_actions.js";
-import { RouterCoreError } from "./errors.js";
+import { type JevTokenUsage, type ResolvedJevConfig } from "./jev_client.js";
 import type { JsonValue } from "./json.js";
-import type { Selector } from "./route.js";
 import type { CustomSelectorInput, SelectorInput } from "./selector_input.js";
 import { type RoutingDecision } from "./types.js";
-export declare const DEFAULT_JEV_ENDPOINT = "https://api.typesafe.ai/v1";
-export declare const DEFAULT_JEV_MODEL = "jev-latest";
 export declare const JEV_SELECTOR_CONTEXT_WINDOW_CHARS = 64000;
-export type JevSelectorConfig = {
-    apiKey: string;
-    endpoint?: string;
-    model?: string;
-    leaseTurns?: readonly number[];
-    timeoutMs?: number;
-};
-export type ResolvedJevSelectorConfig = {
-    apiKey: string;
-    endpoint: string;
-    model: string;
+export type JevSelectorConfig = ResolvedJevConfig & {
     leaseTurns: readonly number[];
-    timeoutMs: number;
 };
 export type JevChoiceQuestion = {
     type: "choice";
@@ -40,10 +26,6 @@ export type JevChoiceAnswer = {
     confidence: number;
     probabilities: Record<string, number>;
 };
-export type JevTokenUsage = {
-    input_tokens: number;
-    output_tokens: number;
-};
 export type JevSystemOneResponse = {
     model: string;
     answers: {
@@ -52,16 +34,11 @@ export type JevSystemOneResponse = {
     };
     usage: JevTokenUsage;
 };
-export declare class JevSelectorOutputError extends RouterCoreError {
-    readonly usage: JevTokenUsage | null;
-    constructor(message: string, code: string, usage?: JevTokenUsage | null);
-}
 export type JevSelection = {
     decision: RoutingDecision;
     request: JevSystemOneRequest;
     response: JevSystemOneResponse;
 };
-export declare function resolveJevSelectorConfig(input: JevSelectorConfig): ResolvedJevSelectorConfig;
 export declare function buildJevSelectorRequest(anonymousInput: AnonymousSelectorInput, options: {
     model: string;
     leaseTurns: readonly number[];
@@ -73,5 +50,4 @@ export declare function parseJevSelectorResponse(body: unknown, args: {
     decision: RoutingDecision;
     response: JevSystemOneResponse;
 };
-export declare function selectWithJev(selectorInput: SelectorInput | CustomSelectorInput, slots: readonly AnonymousActionSlot[], config: ResolvedJevSelectorConfig, signal?: AbortSignal): Promise<JevSelection>;
-export declare function createJevSelector(input: JevSelectorConfig): Selector;
+export declare function selectWithJev(selectorInput: SelectorInput | CustomSelectorInput, slots: readonly AnonymousActionSlot[], config: JevSelectorConfig, signal?: AbortSignal): Promise<JevSelection>;

package/dist/jev_selector.js

@@ -1,74 +1,25 @@
-// TypeSafe Jev selector over the anonymous-action protocol. Jev
-// (docs.typesafe.ai) is a System One model: it answers typed questions about
-// a state instead of generating text, so the routing decision is two Choice
-// questions over the same state — which anonymous candidate action serves
-// the next turns, and how long a lease to commit to — and the answer comes
-// back as calibrated probabilities rather than parsed JSON.
+// TypeSafe Jev as a routing selector over the anonymous-action protocol.
+// Jev answers two Choice questions about the de-identified selector input —
+// which anonymous candidate action serves the next turns, and how long a
+// lease to commit to — and the answers come back as calibrated
+// probabilities rather than parsed JSON. The hosted router uses it as the
+// fallback when Dari's policy endpoint is unavailable.
 //
-// This is an alternative to Dari's trained routing policy. Candidates are
-// de-identified through the same anonymous-action protocol Dari uses
-// (anonymous_actions.ts), so Jev decides on benchmark and cost evidence and
-// never sees a model identity; comparisons Jev is weak at (cost ordering,
-// benchmark standing) are computed here and passed as text.
-import { anonymizeSelectorInput, assignAnonymousActions, sessionRng, } from "./anonymous_actions.js";
-import { RouterCoreError } from "./errors.js";
-import { RouterFrameworkError } from "./framework_error.js";
-import { FIXED_TURN_COST_COMPARISON_TURNS, SELECTOR_LEASE_TURNS } from "./fixed_turn_cost_config.js";
-import { isRecord, } from "./types.js";
-export const DEFAULT_JEV_ENDPOINT = "https://api.typesafe.ai/v1";
-export const DEFAULT_JEV_MODEL = "jev-latest";
-const DEFAULT_JEV_TIMEOUT_MS = 30_000;
+// Candidates are de-identified through the same anonymous-action protocol
+// Dari uses (anonymous_actions.ts), so Jev decides on benchmark and cost
+// evidence and never sees a model identity; comparisons Jev is weak at (cost
+// ordering, benchmark standing) are computed here and passed as text.
+import { anonymizeSelectorInput, } from "./anonymous_actions.js";
+import { FIXED_TURN_COST_COMPARISON_TURNS } from "./fixed_turn_cost_config.js";
+import { invalidJevResponse, JevOutputError, postSystemOne, } from "./jev_client.js";
+import { fillPrompt, JEV_SELECTOR_PROMPT } from "./prompts.js";
+import { isRecord } from "./types.js";
 // Jev evaluates up to 32k tokens of state plus the longest question, and
 // JSON-shaped conversation state tokenizes at roughly 2.7 characters per
 // token. The selector request is sized on its JSON-escaped messages, which
 // overstate the state Jev receives, so this budget leaves room for the
 // questions and the action descriptions.
 export const JEV_SELECTOR_CONTEXT_WINDOW_CHARS = 64_000;
-// Rate-limit and overload responses get one backoff retry; anything else is
-// the caller's failure to surface.
-const JEV_RETRY_STATUSES = new Set([429, 529]);
-const JEV_ATTEMPTS = 2;
-const JEV_RETRY_DELAY_MS = 500;
-const JEV_RETRY_MAX_DELAY_MS = 5_000;
-// selector_output after TypeSafe returned 200. Token counts are on the body
-// even when the action or lease answer is rejected, so hosts can still bill
-// the selector call.
-export class JevSelectorOutputError extends RouterCoreError {
-    usage;
-    constructor(message, code, usage = null) {
-        super("selector_output", message, code);
-        this.name = "JevSelectorOutputError";
-        this.usage = usage;
-    }
-}
-export function resolveJevSelectorConfig(input) {
-    const apiKey = input.apiKey?.trim() ?? "";
-    if (!apiKey)
-        throw configurationError("Jev selector requires an apiKey.", "apiKey");
-    const endpoint = (input.endpoint ?? DEFAULT_JEV_ENDPOINT).trim().replace(/\/+$/, "");
-    let url;
-    try {
-        url = new URL(endpoint);
-    }
-    catch {
-        throw configurationError("Jev selector endpoint must be a valid URL.", "endpoint");
-    }
-    if ((url.protocol !== "https:" && url.protocol !== "http:") || url.username || url.password) {
-        throw configurationError("Jev selector endpoint must be an HTTP(S) URL without embedded credentials.", "endpoint");
-    }
-    const model = (input.model ?? DEFAULT_JEV_MODEL).trim();
-    if (!model)
-        throw configurationError("Jev selector model must not be empty.", "model");
-    const leaseTurns = input.leaseTurns ?? SELECTOR_LEASE_TURNS;
-    if (leaseTurns.some((turns, index) => !Number.isSafeInteger(turns) || turns <= 0 || (index > 0 && turns <= leaseTurns[index - 1]))) {
-        throw configurationError("Jev selector leaseTurns must be ascending positive integers.", "leaseTurns");
-    }
-    const timeoutMs = input.timeoutMs ?? DEFAULT_JEV_TIMEOUT_MS;
-    if (!Number.isSafeInteger(timeoutMs) || timeoutMs <= 0) {
-        throw configurationError("Jev selector timeoutMs must be a positive integer.", "timeoutMs");
-    }
-    return { apiKey, endpoint, model, leaseTurns, timeoutMs };
-}
 // Renders the de-identified selector input as a Jev request. Every candidate
 // identity has already been replaced by its action letter; the only evidence
 // an action carries is its projected cost and its benchmark standing.
@@ -95,7 +46,6 @@ export function buildJevSelectorRequest(anonymousInput, options) {
             };
     }
     const state = {
-        task: anonymousInput.task,
         previous_action: anonymousInput.previous_action === null
             ? null
             : anonymousInput.previous_action.action,
@@ -113,15 +63,14 @@ export function buildJevSelectorRequest(anonymousInput, options) {
         action: {
             type: "choice",
             instructions: {
-                question: "Which action should serve the coding agent's next turns?",
-                rules: [
-                    "Capability first: the action must be able to handle the task; capability is never traded away for cost.",
-                    "Judge the task's difficulty and how much work remains from `task`, `lease_history`, and the latest turns of `conversation`.",
-                    "Use each action's benchmark scores and ranks as evidence of capability. A missing benchmark means the action was not measured on it, not that it scored poorly.",
-                    "When several actions are capable enough, choose the one with the best (lowest-numbered) cost rank.",
-                    "Prefer `previous_action` while the task and its current phase have not changed.",
-                ],
+                question: JEV_SELECTOR_PROMPT.action.question,
+                rules: anonymousInput.router_rules === undefined
+                    ? JEV_SELECTOR_PROMPT.action.rules
+                    : [...JEV_SELECTOR_PROMPT.action.rules, JEV_SELECTOR_PROMPT.action.router_rules_rule],
                 ...(Object.keys(glossary).length === 0 ? {} : { benchmarks: glossary }),
+                ...(anonymousInput.router_rules === undefined
+                    ? {}
+                    : { router_rules: anonymousInput.router_rules }),
             },
             criteria,
         },
@@ -130,23 +79,18 @@ export function buildJevSelectorRequest(anonymousInput, options) {
     return { model: options.model, state, questions };
 }
 function leaseQuestion(leaseTurns) {
+    const { question, guidance, criteria: templates } = JEV_SELECTOR_PROMPT.lease;
     const last = leaseTurns.length - 1;
     const criteria = {};
     leaseTurns.forEach((turns, index) => {
-        criteria[String(turns)] = index === 0
-            ? `${turns} turns: the task is nearly done, or the next step may change what the agent needs.`
+        const template = index === 0
+            ? templates.shortest
             : index === last
-                ? `${turns} turns: a long stretch of similar work remains, such as a large implementation or many edit-and-test cycles.`
-                : `${turns} turns: a bounded stretch of similar work remains.`;
+                ? templates.longest
+                : templates.middle;
+        criteria[String(turns)] = fillPrompt(template, { turns: String(turns) });
     });
-    return {
-        type: "choice",
-        instructions: {
-            question: "How many consecutive turns should the chosen action serve before the router reconsiders?",
-            guidance: "A longer lease amortizes the cold start paid on its first turn across warm cache hits on the turns that follow. The lease ends early only if the task finishes or the provider fails. Judge from how much similar work remains after the latest turn of `conversation`.",
-        },
-        criteria,
-    };
+    return { type: "choice", instructions: { question, guidance }, criteria };
 }
 export function parseJevSelectorResponse(body, args) {
     if (!isRecord(body) || typeof body.model !== "string" || !isRecord(body.answers)) {
@@ -165,7 +109,7 @@ export function parseJevSelectorResponse(body, args) {
     const action = choiceAnswer(body.answers.action, "action", tokenUsage);
     const slot = args.slots.find((entry) => entry.action === action.choice);
     if (slot === undefined) {
-        throw new JevSelectorOutputError("Jev selected an action outside this router's candidates.", "selector_invalid_candidate", tokenUsage);
+        throw new JevOutputError("Jev selected an action outside this router's candidates.", "selector_invalid_candidate", tokenUsage);
     }
     let lease;
     let turns;
@@ -203,9 +147,6 @@ function decisionReason(action, turns, lease) {
 // Anonymizes the named selector input against the slots, asks Jev, and maps
 // the answers back onto the slots' candidates.
 export async function selectWithJev(selectorInput, slots, config, signal) {
-    if ("custom_rules" in selectorInput || "default_target" in selectorInput) {
-        throw new RouterCoreError("configuration", "The Jev selector serves only the default routing policy and cannot apply custom routing rules.", "custom_rules_not_supported");
-    }
     const anonymousInput = anonymizeSelectorInput(selectorInput, slots);
     const request = buildJevSelectorRequest(anonymousInput, config);
     const body = await postSystemOne(request, config, signal);
@@ -221,126 +162,6 @@ export async function selectWithJev(selectorInput, slots, config, signal) {
 // Letters are assigned per call: the framework path carries no conversation
 // identity, and every candidate-bearing field is re-lettered together, so
 // correctness never depends on cross-turn stability.
-export function createJevSelector(input) {
-    const config = resolveJevSelectorConfig(input);
-    return {
-        async select(request, signal) {
-            const selectorInput = selectorInputFromRequest(request);
-            const single = singleCandidate(selectorInput);
-            if (single !== undefined) {
-                return JSON.stringify({
-                    selected_model: single.model,
-                    reasoning_effort: single.reasoningEffort,
-                    reason: "The router has only one compatible model/thinking-level pair.",
-                });
-            }
-            const candidates = selectorInput.candidate_pairs.map((pair) => ({
-                model: pair.model,
-                reasoningEffort: pair.thinking_level,
-            }));
-            const rng = typeof request.prompt_cache_key === "string" && request.prompt_cache_key !== ""
-                ? sessionRng(request.prompt_cache_key)
-                : Math.random;
-            let selection;
-            try {
-                selection = await selectWithJev(selectorInput, assignAnonymousActions(candidates, rng), config, signal);
-            }
-            catch (error) {
-                throw frameworkError(error);
-            }
-            const { decision, response } = selection;
-            return JSON.stringify({
-                selected_model: decision.selectedModel,
-                reasoning_effort: decision.reasoningEffort,
-                reason: decision.reason,
-                ...(decision.leaseTurnsRemaining === undefined
-                    ? {}
-                    : { lease_turns_remaining: decision.leaseTurnsRemaining }),
-                jev: response,
-            });
-        },
-    };
-}
-function singleCandidate(selectorInput) {
-    if (selectorInput.candidate_pairs.length !== 1)
-        return undefined;
-    const pair = selectorInput.candidate_pairs[0];
-    return { model: pair.model, reasoningEffort: pair.thinking_level };
-}
-async function postSystemOne(request, config, signal) {
-    for (let attempt = 1;; attempt += 1) {
-        const response = await fetchSystemOne(request, config, signal);
-        if (response.ok) {
-            try {
-                return await response.json();
-            }
-            catch {
-                throw invalidResponse("Jev returned a non-JSON response body.");
-            }
-        }
-        if (attempt < JEV_ATTEMPTS && JEV_RETRY_STATUSES.has(response.status)) {
-            await delay(retryDelayMs(response.headers.get("retry-after")), signal);
-            continue;
-        }
-        const body = await response.text().catch(() => "");
-        throw new RouterFrameworkError("policy", `Jev returned HTTP ${response.status}: ${body}`, "jev_http_error");
-    }
-}
-async function fetchSystemOne(request, config, signal) {
-    const timeout = AbortSignal.timeout(config.timeoutMs);
-    try {
-        return await fetch(`${config.endpoint}/systemone`, {
-            method: "POST",
-            headers: {
-                Authorization: `Bearer ${config.apiKey}`,
-                "Content-Type": "application/json",
-            },
-            body: JSON.stringify(request),
-            signal: signal === undefined ? timeout : AbortSignal.any([signal, timeout]),
-        });
-    }
-    catch (error) {
-        if (signal?.aborted)
-            throw error;
-        throw new RouterFrameworkError("policy", `Jev request failed: ${error instanceof Error ? error.message : String(error)}`, "jev_request_failed", undefined, { cause: error });
-    }
-}
-function retryDelayMs(retryAfter) {
-    const seconds = Number(retryAfter);
-    if (retryAfter === null || !Number.isFinite(seconds) || seconds <= 0)
-        return JEV_RETRY_DELAY_MS;
-    return Math.min(seconds * 1000, JEV_RETRY_MAX_DELAY_MS);
-}
-function delay(ms, signal) {
-    return new Promise((resolve, reject) => {
-        const timer = setTimeout(() => {
-            signal?.removeEventListener("abort", onAbort);
-            resolve();
-        }, ms);
-        const onAbort = () => {
-            clearTimeout(timer);
-            reject(signal?.reason);
-        };
-        signal?.addEventListener("abort", onAbort, { once: true });
-    });
-}
-function selectorInputFromRequest(request) {
-    const user = request.messages?.find((message) => message.role === "user");
-    if (typeof user?.content !== "string") {
-        throw new RouterCoreError("invalid_request", "Selector request is missing the JSON user message.", "invalid_request_error");
-    }
-    let parsed;
-    try {
-        parsed = JSON.parse(user.content);
-    }
-    catch {
-        throw new RouterCoreError("invalid_request", "Selector request user message is not valid JSON.", "invalid_request_error");
-    }
-    if (!isRecord(parsed) || !Array.isArray(parsed.candidate_pairs) || !Array.isArray(parsed.messages)) {
-        throw new RouterCoreError("invalid_request", "Selector input is missing candidate_pairs or messages.", "invalid_request_error");
-    }
-    return parsed;
-}
 // Cost per action as text Jev can compare literally: the loop projections at
 // each lease length, plus a rank across actions on the comparison horizon.
 function actionCosts(costs) {
@@ -403,16 +224,5 @@ function usd(value) {
 function percent(value) {
     return `${Math.round(value * 100)}%`;
 }
-function invalidResponse(message, usage = null) {
-    return new JevSelectorOutputError(message, "selector_invalid_response", usage);
-}
-function configurationError(message, param) {
-    return new RouterCoreError("configuration", message, "jev_configuration_invalid", param);
-}
-function frameworkError(error) {
-    if (!(error instanceof RouterCoreError))
-        return error;
-    const kind = error.kind === "selector_output" ? "policy" : error.kind;
-    return new RouterFrameworkError(kind, error.message, error.code, error.param, { cause: error });
-}
+const invalidResponse = invalidJevResponse;
 //# sourceMappingURL=jev_selector.js.map