nodebench-mcp 2.11.0 → 2.14.0

package/dist/__tests__/helpers/answerMatch.js

@@ -8,34 +8,12 @@ function normalizeBasic(value) {
         .toLowerCase();
 }
 const SIMPLE_NUMBER_WORDS = {
-    zero: 0,
-    one: 1,
-    two: 2,
-    three: 3,
-    four: 4,
-    five: 5,
-    six: 6,
-    seven: 7,
-    eight: 8,
-    nine: 9,
-    ten: 10,
-    eleven: 11,
-    twelve: 12,
-    thirteen: 13,
-    fourteen: 14,
-    fifteen: 15,
-    sixteen: 16,
-    seventeen: 17,
-    eighteen: 18,
-    nineteen: 19,
-    twenty: 20,
-    thirty: 30,
-    forty: 40,
-    fifty: 50,
-    sixty: 60,
-    seventy: 70,
-    eighty: 80,
-    ninety: 90,
+    zero: 0, one: 1, two: 2, three: 3, four: 4, five: 5,
+    six: 6, seven: 7, eight: 8, nine: 9, ten: 10,
+    eleven: 11, twelve: 12, thirteen: 13, fourteen: 14, fifteen: 15,
+    sixteen: 16, seventeen: 17, eighteen: 18, nineteen: 19,
+    twenty: 20, thirty: 30, forty: 40, fifty: 50,
+    sixty: 60, seventy: 70, eighty: 80, ninety: 90,
 };
 function tryParseLeadingInt(text) {
     const m = String(text ?? "").match(/-?\d+/);
@@ -48,15 +26,11 @@ function expectedIsStrictInt(expectedNorm) {
     return /^-?\d+$/.test(expectedNorm);
 }
 /**
- * GAIA capability scoring helper.
+ * GAIA capability scoring helper (deterministic, synchronous).
  *
- * We keep this intentionally conservative:
+ * Conservative rules:
  * - Case/whitespace normalization for all answers
- * - For integer-only expected answers, allow:
- *   - leading integer inside the model output (e.g. "8 clients" -> 8)
- *   - simple number words (e.g. "eight" -> 8)
- *
- * We do NOT loosen decimals/currency formatting by default to avoid inflating scores.
+ * - For integer-only expected answers: leading integer extraction + number words
  */
 export function answersMatch(expectedRaw, actualRaw) {
     const expected = normalizeBasic(expectedRaw);
@@ -75,4 +49,219 @@ export function answersMatch(expectedRaw, actualRaw) {
     }
     return actual === expected;
 }
+// ---------- LLM Judge ----------
+const JUDGE_SYSTEM_PROMPT = `You are a strict but fair answer-equivalence judge for the GAIA benchmark.
+
+Given:
+- EXPECTED: the ground-truth answer
+- ACTUAL: the candidate answer
+
+Decide whether ACTUAL is semantically equivalent to EXPECTED.
+
+Rules:
+1. MATCH if both convey the same factual content, even with:
+   - Minor typos or spelling variations (e.g. "Ploybius" vs "Polybius")
+   - Different punctuation, capitalization, or trailing periods
+   - Reworded but equivalent phrasing (e.g. "85 points" vs "85")
+   - Number format variations (e.g. "1,000" vs "1000")
+2. NO MATCH if:
+   - The factual content differs (different numbers, names, dates)
+   - One answer is a subset but missing key information
+   - The answers are about different things entirely
+
+Respond with ONLY a single JSON object:
+{"match": true, "reason": "brief explanation"}
+or
+{"match": false, "reason": "brief explanation"}`;
+// ---------- OpenRouter free model auto-discovery ----------
+/**
+ * Free models on OpenRouter (ordered by preference for judge tasks).
+ * These require an OPENROUTER_API_KEY but cost $0.
+ */
+const OPENROUTER_FREE_MODELS = [
+    "google/gemini-2.0-flash-exp:free",
+    "meta-llama/llama-3.3-70b-instruct:free",
+    "qwen/qwen-2.5-72b-instruct:free",
+    "mistralai/mistral-small-3.1-24b-instruct:free",
+];
+let _cachedFreeJudge = null;
+/**
+ * Try to create a free OpenRouter judge client.
+ * Uses OpenAI-compatible API. Returns null if no API key or all models fail.
+ */
+async function tryCreateOpenRouterFreeJudge() {
+    if (_cachedFreeJudge === "unavailable")
+        return null;
+    if (_cachedFreeJudge)
+        return _cachedFreeJudge;
+    const apiKey = process.env.OPENROUTER_API_KEY ||
+        process.env.OPENROUTER_KEY ||
+        "";
+    if (!apiKey) {
+        _cachedFreeJudge = "unavailable";
+        return null;
+    }
+    // Probe the first available model with a tiny request
+    for (const model of OPENROUTER_FREE_MODELS) {
+        try {
+            const response = await fetch("https://openrouter.ai/api/v1/chat/completions", {
+                method: "POST",
+                headers: {
+                    "Authorization": `Bearer ${apiKey}`,
+                    "Content-Type": "application/json",
+                    "HTTP-Referer": "https://github.com/nodebench/nodebench-ai",
+                    "X-Title": "nodebench-gaia-judge",
+                },
+                body: JSON.stringify({
+                    model,
+                    messages: [{ role: "user", content: "Reply with only: OK" }],
+                    temperature: 0,
+                    max_tokens: 5,
+                }),
+            });
+            if (!response.ok)
+                continue;
+            const data = (await response.json());
+            const text = data?.choices?.[0]?.message?.content ?? "";
+            if (!text)
+                continue;
+            // Model works — create a reusable client
+            const client = {
+                provider: "openrouter",
+                model,
+                generateText: async ({ prompt, temperature, maxOutputTokens }) => {
+                    const res = await fetch("https://openrouter.ai/api/v1/chat/completions", {
+                        method: "POST",
+                        headers: {
+                            "Authorization": `Bearer ${apiKey}`,
+                            "Content-Type": "application/json",
+                            "HTTP-Referer": "https://github.com/nodebench/nodebench-ai",
+                            "X-Title": "nodebench-gaia-judge",
+                        },
+                        body: JSON.stringify({
+                            model,
+                            messages: [{ role: "user", content: prompt }],
+                            temperature: typeof temperature === "number" ? temperature : 0,
+                            max_tokens: typeof maxOutputTokens === "number" ? maxOutputTokens : 200,
+                        }),
+                    });
+                    if (!res.ok) {
+                        const errText = await res.text().catch(() => "");
+                        throw new Error(`OpenRouter ${res.status}: ${errText.slice(0, 200)}`);
+                    }
+                    const json = (await res.json());
+                    return String(json?.choices?.[0]?.message?.content ?? "").trim();
+                },
+            };
+            _cachedFreeJudge = client;
+            return client;
+        }
+        catch {
+            continue;
+        }
+    }
+    _cachedFreeJudge = "unavailable";
+    return null;
+}
+/**
+ * Auto-discover the best available judge client.
+ *
+ * Priority:
+ *   1. OpenRouter free model ($0, just needs OPENROUTER_API_KEY)
+ *   2. Provided paid LLM client (Gemini/OpenAI/Anthropic)
+ *   3. null (deterministic-only fallback)
+ */
+export async function autoDiscoverJudge(paidFallback) {
+    // Try free first
+    const free = await tryCreateOpenRouterFreeJudge();
+    if (free)
+        return free;
+    // Fall back to paid client (if available and not a noop)
+    if (paidFallback && paidFallback.provider !== "none")
+        return paidFallback;
+    return null;
+}
+/**
+ * Tiered answer matching: deterministic first, LLM judge fallback.
+ *
+ * Tier 0 (instant, free): `answersMatch()` — normalization + integer fuzzy
+ * Tier 1 (async, free):   OpenRouter free model judge — semantic equivalence
+ * Tier 2 (async, paid):   Provided LLM client — semantic equivalence
+ * Tier 3 (instant, free): No API key → deterministic only
+ *
+ * If no `judge` client is provided, auto-discovers via `autoDiscoverJudge()`.
+ */
+export async function answersMatchWithJudge(expectedRaw, actualRaw, judge) {
+    // Tier 0: deterministic
+    const detMatch = answersMatch(expectedRaw, actualRaw);
+    if (detMatch) {
+        return { deterministicMatch: true, judgeInvoked: false, match: true };
+    }
+    // If no actual answer was produced, skip the judge
+    const actual = normalizeBasic(actualRaw);
+    if (!actual) {
+        return { deterministicMatch: false, judgeInvoked: false, match: false };
+    }
+    // Resolve judge: use provided client or auto-discover
+    const resolvedJudge = judge !== undefined ? judge : await autoDiscoverJudge();
+    if (!resolvedJudge || resolvedJudge.provider === "none") {
+        return { deterministicMatch: false, judgeInvoked: false, match: false };
+    }
+    try {
+        const prompt = [
+            JUDGE_SYSTEM_PROMPT,
+            "",
+            `EXPECTED: ${expectedRaw}`,
+            `ACTUAL: ${actualRaw}`,
+        ].join("\n");
+        const response = await resolvedJudge.generateText({
+            prompt,
+            temperature: 0,
+            maxOutputTokens: 200,
+        });
+        const parsed = parseJudgeResponse(response);
+        return {
+            deterministicMatch: false,
+            judgeInvoked: true,
+            judgeProvider: `${resolvedJudge.provider}:${resolvedJudge.model}`,
+            match: parsed.match,
+            reason: parsed.reason,
+        };
+    }
+    catch (err) {
+        return {
+            deterministicMatch: false,
+            judgeInvoked: true,
+            judgeProvider: `${resolvedJudge.provider}:${resolvedJudge.model}`,
+            match: false,
+            reason: `judge error: ${err?.message ?? String(err)}`,
+        };
+    }
+}
+function parseJudgeResponse(text) {
+    const trimmed = String(text ?? "").trim();
+    // Try JSON parse
+    const jsonMatch = trimmed.match(/\{[\s\S]*\}/);
+    if (jsonMatch) {
+        try {
+            const obj = JSON.parse(jsonMatch[0]);
+            if (typeof obj.match === "boolean") {
+                return { match: obj.match, reason: String(obj.reason ?? "") };
+            }
+        }
+        catch {
+            // Fall through to heuristic
+        }
+    }
+    // Heuristic fallback
+    const lower = trimmed.toLowerCase();
+    if (lower.includes('"match": true') || lower.includes('"match":true')) {
+        return { match: true, reason: trimmed };
+    }
+    if (lower.includes('"match": false') || lower.includes('"match":false')) {
+        return { match: false, reason: trimmed };
+    }
+    // Conservative default: no match
+    return { match: false, reason: `unparseable judge response: ${trimmed.slice(0, 200)}` };
+}
 //# sourceMappingURL=answerMatch.js.map

package/dist/__tests__/helpers/answerMatch.js.map

@@ -1 +1 @@
-{"version":3,"file":"answerMatch.js","sourceRoot":"","sources":["../../../src/__tests__/helpers/answerMatch.ts"],"names":[],"mappings":"AAAA,SAAS,cAAc,CAAC,KAAa;IACnC,OAAO,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC;SACvB,IAAI,EAAE;SACN,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC;SAClB,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;SACpB,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC;SAC3B,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC;SACrB,WAAW,EAAE,CAAC;AACnB,CAAC;AAED,MAAM,mBAAmB,GAA2B;IAClD,IAAI,EAAE,CAAC;IACP,GAAG,EAAE,CAAC;IACN,GAAG,EAAE,CAAC;IACN,KAAK,EAAE,CAAC;IACR,IAAI,EAAE,CAAC;IACP,IAAI,EAAE,CAAC;IACP,GAAG,EAAE,CAAC;IACN,KAAK,EAAE,CAAC;IACR,KAAK,EAAE,CAAC;IACR,IAAI,EAAE,CAAC;IACP,GAAG,EAAE,EAAE;IACP,MAAM,EAAE,EAAE;IACV,MAAM,EAAE,EAAE;IACV,QAAQ,EAAE,EAAE;IACZ,QAAQ,EAAE,EAAE;IACZ,OAAO,EAAE,EAAE;IACX,OAAO,EAAE,EAAE;IACX,SAAS,EAAE,EAAE;IACb,QAAQ,EAAE,EAAE;IACZ,QAAQ,EAAE,EAAE;IACZ,MAAM,EAAE,EAAE;IACV,MAAM,EAAE,EAAE;IACV,KAAK,EAAE,EAAE;IACT,KAAK,EAAE,EAAE;IACT,KAAK,EAAE,EAAE;IACT,OAAO,EAAE,EAAE;IACX,MAAM,EAAE,EAAE;IACV,MAAM,EAAE,EAAE;CACX,CAAC;AAEF,SAAS,kBAAkB,CAAC,IAAY;IACtC,MAAM,CAAC,GAAG,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC5C,IAAI,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IACpB,MAAM,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACpC,OAAO,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACvC,CAAC;AAED,SAAS,mBAAmB,CAAC,YAAoB;IAC/C,OAAO,SAAS,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;AACtC,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,YAAY,CAAC,WAAmB,EAAE,SAAiB;IACjE,MAAM,QAAQ,GAAG,cAAc,CAAC,WAAW,CAAC,CAAC;IAC7C,MAAM,MAAM,GAAG,cAAc,CAAC,SAAS,CAAC,CAAC;IACzC,IAAI,CAAC,QAAQ;QAAE,OAAO,KAAK,CAAC;IAE5B,IAAI,mBAAmB,CAAC,QAAQ,CAAC,EAAE,CAAC;QAClC,MAAM,WAAW,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QAClD,MAAM,SAAS,GAAG,kBAAkB,CAAC,MAAM,CAAC,CAAC;QAC7C,IAAI,SAAS,KAAK,IAAI;YAAE,OAAO,SAAS,KAAK,WAAW,CAAC;QAEzD,MAAM,IAAI,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC;QACzC,IAAI,OAAO,IAAI,KAAK,QAAQ;YAAE,OAAO,IAAI,KAAK,WAAW,CAAC;QAC1D,OAAO,KAAK,CAAC;IACf,CAAC;IAED,OAAO,MAAM,KAAK,QAAQ,CAAC;AAC7B,CAAC"}
+{"version":3,"file":"answerMatch.js","sourceRoot":"","sources":["../../../src/__tests__/helpers/answerMatch.ts"],"names":[],"mappings":"AAEA,SAAS,cAAc,CAAC,KAAa;IACnC,OAAO,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC;SACvB,IAAI,EAAE;SACN,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC;SAClB,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;SACpB,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC;SAC3B,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC;SACrB,WAAW,EAAE,CAAC;AACnB,CAAC;AAED,MAAM,mBAAmB,GAA2B;IAClD,IAAI,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC;IACnD,GAAG,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE;IAC5C,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE;IAC/D,OAAO,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE;IACtD,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE;IAC5C,KAAK,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE;CAC/C,CAAC;AAEF,SAAS,kBAAkB,CAAC,IAAY;IACtC,MAAM,CAAC,GAAG,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC5C,IAAI,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IACpB,MAAM,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACpC,OAAO,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACvC,CAAC;AAED,SAAS,mBAAmB,CAAC,YAAoB;IAC/C,OAAO,SAAS,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;AACtC,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,YAAY,CAAC,WAAmB,EAAE,SAAiB;IACjE,MAAM,QAAQ,GAAG,cAAc,CAAC,WAAW,CAAC,CAAC;IAC7C,MAAM,MAAM,GAAG,cAAc,CAAC,SAAS,CAAC,CAAC;IACzC,IAAI,CAAC,QAAQ;QAAE,OAAO,KAAK,CAAC;IAE5B,IAAI,mBAAmB,CAAC,QAAQ,CAAC,EAAE,CAAC;QAClC,MAAM,WAAW,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QAClD,MAAM,SAAS,GAAG,kBAAkB,CAAC,MAAM,CAAC,CAAC;QAC7C,IAAI,SAAS,KAAK,IAAI;YAAE,OAAO,SAAS,KAAK,WAAW,CAAC;QAEzD,MAAM,IAAI,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC;QACzC,IAAI,OAAO,IAAI,KAAK,QAAQ;YAAE,OAAO,IAAI,KAAK,WAAW,CAAC;QAC1D,OAAO,KAAK,CAAC;IACf,CAAC;IAED,OAAO,MAAM,KAAK,QAAQ,CAAC;AAC7B,CAAC;AAED,kCAAkC;AAElC,MAAM,mBAAmB,GAAG;;;;;;;;;;;;;;;;;;;;;;gDAsBoB,CAAC;AAejD,6DAA6D;AAE7D;;;GAGG;AACH,MAAM,sBAAsB,GAAG;IAC7B,kCAAkC;IAClC,wCAAwC;IACxC,iCAAiC;IACjC,+CAA+C;CAChD,CAAC;AAEF,IAAI,gBAAgB,GAAyC,IAAI,CAAC;AAElE;;;GAGG;AACH,KAAK,UAAU,4BAA4B;IACzC,IAAI,gBAAgB,KAAK,aAAa;QAAE,OAAO,IAAI,CAAC;IACpD,IAAI,gBAAgB;QAAE,OAAO,gBAAgB,CAAC;IAE9C,MAAM,MAAM,GACV,OAAO,CAAC,GAAG,CAAC,kBAAkB;QAC9B,OAAO,CAAC,GAAG,CAAC,cAAc;QAC1B,EAAE,CAAC;IACL,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,gBAAgB,GAAG,aAAa,CAAC;QACjC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,sDAAsD;IACtD,KAAK,MAAM,KAAK,IAAI,sBAAsB,EAAE,CAAC;QAC3C,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,+CAA+C,EAAE;gBAC5E,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,eAAe,EAAE,UAAU,MAAM,EAAE;oBACnC,cAAc,EAAE,kBAAkB;oBAClC,cAAc,EAAE,2CAA2C;oBAC3D,SAAS,EAAE,sBAAsB;iBAClC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,KAAK;oBACL,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,qBAAqB,EAAE,CAAC;oBAC5D,WAAW,EAAE,CAAC;oBACd,UAAU,EAAE,CAAC;iBACd,CAAC;aACH,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,EAAE;gBAAE,SAAS;YAE3B,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAQ,CAAC;YAC5C,MAAM,IAAI,GAAG,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,IAAI,EAAE,CAAC;YACxD,IAAI,CAAC,IAAI;gBAAE,SAAS;YAEpB,yCAAyC;YACzC,MAAM,MAAM,GAAkB;gBAC5B,QAAQ,EAAE,YAAmB;gBAC7B,KAAK;gBACL,YAAY,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,eAAe,EAAE,EAAE,EAAE;oBAC/D,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,+CAA+C,EAAE;wBACvE,MAAM,EAAE,MAAM;wBACd,OAAO,EAAE;4BACP,eAAe,EAAE,UAAU,MAAM,EAAE;4BACnC,cAAc,EAAE,kBAAkB;4BAClC,cAAc,EAAE,2CAA2C;4BAC3D,SAAS,EAAE,sBAAsB;yBAClC;wBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;4BACnB,KAAK;4BACL,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;4BAC7C,WAAW,EAAE,OAAO,WAAW,KAAK,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;4BAC9D,UAAU,EAAE,OAAO,eAAe,KAAK,QAAQ,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,GAAG;yBACxE,CAAC;qBACH,CAAC,CAAC;oBACH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;wBACZ,MAAM,OAAO,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;wBACjD,MAAM,IAAI,KAAK,CAAC,cAAc,GAAG,CAAC,MAAM,KAAK,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;oBACxE,CAAC;oBACD,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAQ,CAAC;oBACvC,OAAO,MAAM,CAAC,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;gBACnE,CAAC;aACF,CAAC;YAEF,gBAAgB,GAAG,MAAM,CAAC;YAC1B,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;IACH,CAAC;IAED,gBAAgB,GAAG,aAAa,CAAC;IACjC,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,YAAmC;IAEnC,iBAAiB;IACjB,MAAM,IAAI,GAAG,MAAM,4BAA4B,EAAE,CAAC;IAClD,IAAI,IAAI;QAAE,OAAO,IAAI,CAAC;IAEtB,yDAAyD;IACzD,IAAI,YAAY,IAAI,YAAY,CAAC,QAAQ,KAAK,MAAM;QAAE,OAAO,YAAY,CAAC;IAE1E,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,WAAmB,EACnB,SAAiB,EACjB,KAA4B;IAE5B,wBAAwB;IACxB,MAAM,QAAQ,GAAG,YAAY,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;IACtD,IAAI,QAAQ,EAAE,CAAC;QACb,OAAO,EAAE,kBAAkB,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;IACxE,CAAC;IAED,mDAAmD;IACnD,MAAM,MAAM,GAAG,cAAc,CAAC,SAAS,CAAC,CAAC;IACzC,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO,EAAE,kBAAkB,EAAE,KAAK,EAAE,YAAY,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;IAC1E,CAAC;IAED,sDAAsD;IACtD,MAAM,aAAa,GAAG,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,iBAAiB,EAAE,CAAC;IAC9E,IAAI,CAAC,aAAa,IAAK,aAAqB,CAAC,QAAQ,KAAK,MAAM,EAAE,CAAC;QACjE,OAAO,EAAE,kBAAkB,EAAE,KAAK,EAAE,YAAY,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;IAC1E,CAAC;IAED,IAAI,CAAC;QACH,MAAM,MAAM,GAAG;YACb,mBAAmB;YACnB,EAAE;YACF,aAAa,WAAW,EAAE;YAC1B,WAAW,SAAS,EAAE;SACvB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEb,MAAM,QAAQ,GAAG,MAAM,aAAa,CAAC,YAAY,CAAC;YAChD,MAAM;YACN,WAAW,EAAE,CAAC;YACd,eAAe,EAAE,GAAG;SACrB,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,kBAAkB,CAAC,QAAQ,CAAC,CAAC;QAC5C,OAAO;YACL,kBAAkB,EAAE,KAAK;YACzB,YAAY,EAAE,IAAI;YAClB,aAAa,EAAE,GAAG,aAAa,CAAC,QAAQ,IAAI,aAAa,CAAC,KAAK,EAAE;YACjE,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,MAAM,EAAE,MAAM,CAAC,MAAM;SACtB,CAAC;IACJ,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,OAAO;YACL,kBAAkB,EAAE,KAAK;YACzB,YAAY,EAAE,IAAI;YAClB,aAAa,EAAE,GAAG,aAAa,CAAC,QAAQ,IAAI,aAAa,CAAC,KAAK,EAAE;YACjE,KAAK,EAAE,KAAK;YACZ,MAAM,EAAE,gBAAgB,GAAG,EAAE,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC,EAAE;SACtD,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAY;IACtC,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAE1C,iBAAiB;IACjB,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;IAC/C,IAAI,SAAS,EAAE,CAAC;QACd,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;YACrC,IAAI,OAAO,GAAG,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;gBACnC,OAAO,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC,EAAE,CAAC;YAChE,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,4BAA4B;QAC9B,CAAC;IACH,CAAC;IAED,qBAAqB;IACrB,MAAM,KAAK,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IACpC,IAAI,KAAK,CAAC,QAAQ,CAAC,eAAe,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,cAAc,CAAC,EAAE,CAAC;QACtE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;IAC1C,CAAC;IACD,IAAI,KAAK,CAAC,QAAQ,CAAC,gBAAgB,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,eAAe,CAAC,EAAE,CAAC;QACxE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;IAC3C,CAAC;IAED,iCAAiC;IACjC,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,+BAA+B,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC;AAC1F,CAAC"}

package/dist/__tests__/helpers/textLlm.d.ts

@@ -1,4 +1,4 @@
-export type TextLlmProviderName = "gemini" | "openai" | "anthropic";
+export type TextLlmProviderName = "gemini" | "openai" | "anthropic" | "none";
 export type TextLlmClient = {
     provider: TextLlmProviderName;
     model: string;

package/dist/__tests__/presetRealWorldBench.test.d.ts

@@ -0,0 +1 @@
+export {};