@sbissoli/mcp-evals 0.1.0

package/dist/retry.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"retry.d.ts","sourceRoot":"","sources":["../src/retry.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,MAAM,MAAM,SAAS,GACjB,YAAY,GACZ,YAAY,GACZ,QAAQ,GACR,SAAS,GACT,MAAM,GACN,SAAS,GACT,OAAO,CAAC;AAEZ,qBAAa,YAAa,SAAQ,KAAK;aAMnB,MAAM,EAAE,MAAM;aACd,IAAI,EAAE,SAAS;aACf,SAAS,EAAE,OAAO;IAPpC,iEAAiE;IACjE,iBAAiB,EAAE,MAAM,GAAG,SAAS,CAAC;gBAGpC,OAAO,EAAE,MAAM,EACC,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,SAAS,EACf,SAAS,EAAE,OAAO;CAKrC;AAED,4FAA4F;AAC5F,wBAAgB,YAAY,CAAC,IAAI,EAAE,SAAS,GAAG,OAAO,CAErD;AAED;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG;IAAE,IAAI,EAAE,SAAS,CAAC;IAAC,SAAS,EAAE,OAAO,CAAA;CAAE,CAUtG;AAED,eAAO,MAAM,eAAe,OAAQ,CAAC;AACrC,eAAO,MAAM,cAAc,QAAS,CAAC;AACrC,eAAO,MAAM,WAAW,IAAI,CAAC;AAE7B;;;;;GAKG;AACH,wBAAgB,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,iBAAiB,CAAC,EAAE,MAAM,GAAG,MAAM,CAK7E;AAED,oFAAoF;AACpF,wBAAgB,eAAe,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,GAAG,MAAM,GAAG,SAAS,CAI9E"}

package/dist/retry.js

@@ -0,0 +1,72 @@
+/**
+ * Núcleo puro de retry/classificação de erro do runner. Sem rede, sem I/O — os testes
+ * offline o exercitam diretamente.
+ *
+ * O runner manda cada fixture à Anthropic Messages API. Duas classes de falha precisam
+ * ser distinguidas, porque nenhuma delas é sinal de acurácia (uma fixture que nunca
+ * chegou ao modelo não foi "respondida errado"):
+ *   - infra transitória (429 rate-limit, 529 overloaded, 5xx, rede) → vale retry;
+ *   - infra fatal (401 auth, 400 "credit balance too low") → retry nunca ajuda; a rodada
+ *     inteira deve falhar rápido com remédio claro em vez de martelar a API.
+ */
+export class EvalApiError extends Error {
+    status;
+    kind;
+    retryable;
+    /** Dica Retry-After (segundos) vinda da API, quando presente. */
+    retryAfterSeconds;
+    constructor(message, status, kind, retryable) {
+        super(message);
+        this.status = status;
+        this.kind = kind;
+        this.retryable = retryable;
+        this.name = "EvalApiError";
+    }
+}
+/** Kinds fatais curto-circuitam a rodada inteira — toda fixture bateria na mesma parede. */
+export function isFatalInfra(kind) {
+    return kind === "auth" || kind === "billing";
+}
+/**
+ * Classifica um erro HTTP da Anthropic API (status + corpo) em kind + retryable.
+ * Puro; o corpo só é inspecionado para distinguir um 400 de billing ("credit balance
+ * is too low") dos demais 400.
+ */
+export function classifyApiError(status, body) {
+    const lower = (body || "").toLowerCase();
+    if (status === 429)
+        return { kind: "rate_limit", retryable: true };
+    if (status === 529)
+        return { kind: "overloaded", retryable: true };
+    if (status === 401 || status === 403)
+        return { kind: "auth", retryable: false };
+    if (status === 400 && (lower.includes("credit balance") || lower.includes("plans & billing"))) {
+        return { kind: "billing", retryable: false };
+    }
+    if (status >= 500)
+        return { kind: "server", retryable: true };
+    return { kind: "other", retryable: false };
+}
+export const BASE_BACKOFF_MS = 2_000;
+export const MAX_BACKOFF_MS = 30_000;
+export const MAX_RETRIES = 5;
+/**
+ * Backoff base (ms) antes da tentativa de retry (0-based). Honra a dica Retry-After
+ * (segundos) da API quando presente — importante no 429, onde a janela de rate-limit
+ * precisa rolar antes da próxima tentativa. O chamador adiciona jitter; mantido puro
+ * para o teste unitário poder asserir valores exatos.
+ */
+export function backoffMs(attempt, retryAfterSeconds) {
+    if (retryAfterSeconds !== undefined && Number.isFinite(retryAfterSeconds) && retryAfterSeconds > 0) {
+        return Math.min(Math.ceil(retryAfterSeconds * 1000), MAX_BACKOFF_MS);
+    }
+    return Math.min(BASE_BACKOFF_MS * 2 ** attempt, MAX_BACKOFF_MS);
+}
+/** Interpreta um header `Retry-After` (só a forma inteiro-segundos) em segundos. */
+export function parseRetryAfter(headerValue) {
+    if (!headerValue)
+        return undefined;
+    const secs = parseInt(headerValue.trim(), 10);
+    return Number.isFinite(secs) && secs >= 0 ? secs : undefined;
+}
+//# sourceMappingURL=retry.js.map

package/dist/retry.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"retry.js","sourceRoot":"","sources":["../src/retry.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAWH,MAAM,OAAO,YAAa,SAAQ,KAAK;IAMnB;IACA;IACA;IAPlB,iEAAiE;IACjE,iBAAiB,CAAqB;IAEtC,YACE,OAAe,EACC,MAAc,EACd,IAAe,EACf,SAAkB;QAElC,KAAK,CAAC,OAAO,CAAC,CAAC;QAJC,WAAM,GAAN,MAAM,CAAQ;QACd,SAAI,GAAJ,IAAI,CAAW;QACf,cAAS,GAAT,SAAS,CAAS;QAGlC,IAAI,CAAC,IAAI,GAAG,cAAc,CAAC;IAC7B,CAAC;CACF;AAED,4FAA4F;AAC5F,MAAM,UAAU,YAAY,CAAC,IAAe;IAC1C,OAAO,IAAI,KAAK,MAAM,IAAI,IAAI,KAAK,SAAS,CAAC;AAC/C,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,gBAAgB,CAAC,MAAc,EAAE,IAAY;IAC3D,MAAM,KAAK,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;IACzC,IAAI,MAAM,KAAK,GAAG;QAAE,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;IACnE,IAAI,MAAM,KAAK,GAAG;QAAE,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;IACnE,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG;QAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;IAChF,IAAI,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,gBAAgB,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC,EAAE,CAAC;QAC9F,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;IAC/C,CAAC;IACD,IAAI,MAAM,IAAI,GAAG;QAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;IAC9D,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;AAC7C,CAAC;AAED,MAAM,CAAC,MAAM,eAAe,GAAG,KAAK,CAAC;AACrC,MAAM,CAAC,MAAM,cAAc,GAAG,MAAM,CAAC;AACrC,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,CAAC;AAE7B;;;;;GAKG;AACH,MAAM,UAAU,SAAS,CAAC,OAAe,EAAE,iBAA0B;IACnE,IAAI,iBAAiB,KAAK,SAAS,IAAI,MAAM,CAAC,QAAQ,CAAC,iBAAiB,CAAC,IAAI,iBAAiB,GAAG,CAAC,EAAE,CAAC;QACnG,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,EAAE,cAAc,CAAC,CAAC;IACvE,CAAC;IACD,OAAO,IAAI,CAAC,GAAG,CAAC,eAAe,GAAG,CAAC,IAAI,OAAO,EAAE,cAAc,CAAC,CAAC;AAClE,CAAC;AAED,oFAAoF;AACpF,MAAM,UAAU,eAAe,CAAC,WAA0B;IACxD,IAAI,CAAC,WAAW;QAAE,OAAO,SAAS,CAAC;IACnC,MAAM,IAAI,GAAG,QAAQ,CAAC,WAAW,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;IAC9C,OAAO,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;AAC/D,CAAC"}

package/dist/runner.d.ts

@@ -0,0 +1,63 @@
+/**
+ * Runner — dirige o eval real de seleção de tool contra a Anthropic Messages API.
+ *
+ * Para cada fixture, envia o catálogo inteiro como `tools` da requisição, força uma
+ * chamada de tool com `tool_choice: {type: "any"}` e registra a tool escolhida. O scorer
+ * puro (score.ts) transforma as escolhas em acurácia top-1/top-k/por-área e na decisão
+ * de gate.
+ *
+ * Princípios (herdados do harness do senado — reexecução barata):
+ *   - Sem dependência de SDK: `fetch` puro contra /v1/messages.
+ *   - Gateado em ANTHROPIC_API_KEY: ausente → imprime instruções e devolve exit 0
+ *     (nunca quebra CI, nunca exige rede).
+ *   - Catálogo + fixtures são a fonte de verdade; o modelo só escolhe entre tools reais.
+ *
+ * Uso típico no `evals/run.ts` de um projeto:
+ *
+ *   const { exitCode } = await runEval({ catalog, fixtures: FIXTURES, systemPrompt: "..." });
+ *   process.exit(exitCode);
+ *
+ * Variáveis de ambiente (todas opcionais): EVAL_MODEL, EVAL_CONCURRENCY, EVAL_LIMIT.
+ */
+import { type Catalog } from "./catalog.js";
+import type { EvalFixture } from "./fixtures.js";
+import { type GateOptions, type GateResult, type ScoreReport } from "./score.js";
+import { type FixtureError } from "./report.js";
+export interface EvalRunnerConfig {
+    catalog: Catalog;
+    fixtures: EvalFixture[];
+    /** System prompt de roteamento, na persona/idioma do servidor. */
+    systemPrompt: string;
+    /** Modelo padrão quando EVAL_MODEL não está definido. Padrão: claude-opus-4-8. */
+    defaultModel?: string;
+    /** Limiares/contexto do gate (toolCount é preenchido do catálogo se omitido). */
+    gate?: GateOptions;
+    /** Ambiente a consultar (padrão: process.env). */
+    env?: Record<string, string | undefined>;
+    /** Saída normal (padrão: console.log). */
+    log?: (line: string) => void;
+    /** Saída de erro/progresso de retry (padrão: console.error). */
+    logError?: (line: string) => void;
+    /** Implementação de fetch (padrão: fetch global) — injetável para teste offline. */
+    fetchImpl?: typeof fetch;
+    /** Implementação de sleep (padrão: setTimeout) — injetável para teste offline. */
+    sleepImpl?: (ms: number) => Promise<void>;
+    /** Máximo de retries por fixture em falha transitória. Padrão: MAX_RETRIES (5). */
+    maxRetries?: number;
+}
+export interface EvalRunResult {
+    /** 0 = completo (ou pulado sem API key); 2 = rodada incompleta (dropout de infra). */
+    exitCode: number;
+    /** True quando ANTHROPIC_API_KEY não estava definido e nada rodou. */
+    skipped: boolean;
+    report?: ScoreReport;
+    gate?: GateResult;
+    errors?: FixtureError[];
+}
+/**
+ * Roda o eval completo e imprime o relatório. Devolve o resultado com `exitCode`
+ * (0 completo/pulado, 2 rodada incompleta) — o `evals/run.ts` do projeto decide se
+ * chama `process.exit`.
+ */
+export declare function runEval(config: EvalRunnerConfig): Promise<EvalRunResult>;
+//# sourceMappingURL=runner.d.ts.map

package/dist/runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../src/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,EAA+C,KAAK,OAAO,EAAE,MAAM,cAAc,CAAC;AACzF,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AACjD,OAAO,EAAa,KAAK,WAAW,EAAE,KAAK,UAAU,EAAoC,KAAK,WAAW,EAAE,MAAM,YAAY,CAAC;AAC9H,OAAO,EAAgB,KAAK,YAAY,EAAE,MAAM,aAAa,CAAC;AAc9D,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,OAAO,CAAC;IACjB,QAAQ,EAAE,WAAW,EAAE,CAAC;IACxB,kEAAkE;IAClE,YAAY,EAAE,MAAM,CAAC;IACrB,kFAAkF;IAClF,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,iFAAiF;IACjF,IAAI,CAAC,EAAE,WAAW,CAAC;IACnB,kDAAkD;IAClD,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC;IACzC,0CAA0C;IAC1C,GAAG,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;IAC7B,gEAAgE;IAChE,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;IAClC,oFAAoF;IACpF,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;IACzB,kFAAkF;IAClF,SAAS,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1C,mFAAmF;IACnF,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,aAAa;IAC5B,sFAAsF;IACtF,QAAQ,EAAE,MAAM,CAAC;IACjB,sEAAsE;IACtE,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,IAAI,CAAC,EAAE,UAAU,CAAC;IAClB,MAAM,CAAC,EAAE,YAAY,EAAE,CAAC;CACzB;AA0ID;;;;GAIG;AACH,wBAAsB,OAAO,CAAC,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,aAAa,CAAC,CAmE9E"}

package/dist/runner.js

@@ -0,0 +1,201 @@
+/**
+ * Runner — dirige o eval real de seleção de tool contra a Anthropic Messages API.
+ *
+ * Para cada fixture, envia o catálogo inteiro como `tools` da requisição, força uma
+ * chamada de tool com `tool_choice: {type: "any"}` e registra a tool escolhida. O scorer
+ * puro (score.ts) transforma as escolhas em acurácia top-1/top-k/por-área e na decisão
+ * de gate.
+ *
+ * Princípios (herdados do harness do senado — reexecução barata):
+ *   - Sem dependência de SDK: `fetch` puro contra /v1/messages.
+ *   - Gateado em ANTHROPIC_API_KEY: ausente → imprime instruções e devolve exit 0
+ *     (nunca quebra CI, nunca exige rede).
+ *   - Catálogo + fixtures são a fonte de verdade; o modelo só escolhe entre tools reais.
+ *
+ * Uso típico no `evals/run.ts` de um projeto:
+ *
+ *   const { exitCode } = await runEval({ catalog, fixtures: FIXTURES, systemPrompt: "..." });
+ *   process.exit(exitCode);
+ *
+ * Variáveis de ambiente (todas opcionais): EVAL_MODEL, EVAL_CONCURRENCY, EVAL_LIMIT.
+ */
+import { catalogAsAnthropicTools } from "./catalog.js";
+import { scoreItem } from "./score.js";
+import { formatReport } from "./report.js";
+import { EvalApiError, classifyApiError, backoffMs, parseRetryAfter, isFatalInfra, MAX_RETRIES, } from "./retry.js";
+const API_URL = "https://api.anthropic.com/v1/messages";
+const API_VERSION = "2023-06-01";
+const DEFAULT_MODEL = "claude-opus-4-8";
+function defaultSleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+/** Uma tentativa HTTP. Lança EvalApiError tipado em qualquer não-2xx ou falha de rede. */
+async function callOnce(fixture, apiKey, model, tools, systemPrompt, fetchImpl) {
+    const body = {
+        model,
+        max_tokens: 1024,
+        system: systemPrompt,
+        tools,
+        tool_choice: { type: "any" },
+        messages: [{ role: "user", content: fixture.query }],
+    };
+    let res;
+    try {
+        res = await fetchImpl(API_URL, {
+            method: "POST",
+            headers: {
+                "content-type": "application/json",
+                "x-api-key": apiKey,
+                "anthropic-version": API_VERSION,
+            },
+            body: JSON.stringify(body),
+        });
+    }
+    catch (e) {
+        // DNS/TCP/TLS/abort — sempre vale retry.
+        throw new EvalApiError(`erro de rede: ${e.message || "desconhecido"}`, 0, "network", true);
+    }
+    if (!res.ok) {
+        const text = await res.text().catch(() => "");
+        const { kind, retryable } = classifyApiError(res.status, text);
+        const err = new EvalApiError(`Anthropic API ${res.status}: ${text.slice(0, 200)}`, res.status, kind, retryable);
+        err.retryAfterSeconds = parseRetryAfter(res.headers.get("retry-after"));
+        throw err;
+    }
+    const data = (await res.json());
+    const picks = [];
+    for (const block of data.content ?? []) {
+        if (block.type === "tool_use" && typeof block.name === "string") {
+            picks.push(block.name);
+        }
+    }
+    return { id: fixture.id, predictedTools: picks };
+}
+/**
+ * Chama a API para uma fixture, com retry de falhas transitórias (429/529/5xx/rede) em
+ * backoff exponencial limitado + jitter, honrando Retry-After. Falhas fatais de infra
+ * (auth/billing) lançam imediatamente — retry só desperdiça tempo. O EvalApiError
+ * lançado carrega `kind` para o chamador distinguir "dropout de infra" de "tool errada".
+ */
+async function predictOne(fixture, apiKey, model, tools, systemPrompt, fetchImpl, sleepImpl, maxRetries, logError) {
+    let lastError;
+    for (let attempt = 0; attempt <= maxRetries; attempt++) {
+        try {
+            return await callOnce(fixture, apiKey, model, tools, systemPrompt, fetchImpl);
+        }
+        catch (e) {
+            const err = e instanceof EvalApiError ? e : new EvalApiError(e.message, 0, "other", false);
+            lastError = err;
+            if (!err.retryable || attempt === maxRetries)
+                throw err;
+            const base = backoffMs(attempt, err.retryAfterSeconds);
+            const jitter = Math.floor(Math.random() * 500);
+            logError(`    ↻ ${fixture.id}: ${err.kind} (HTTP ${err.status}); retry ${attempt + 1}/${maxRetries} em ${base + jitter}ms`);
+            await sleepImpl(base + jitter);
+        }
+    }
+    throw lastError ?? new EvalApiError(`falha desconhecida em ${fixture.id}`, 0, "other", false);
+}
+/** Mapeia `items` com concorrência limitada, preservando a ordem. */
+async function mapWithConcurrency(items, limit, fn) {
+    const results = new Array(items.length);
+    let next = 0;
+    async function worker() {
+        while (true) {
+            const i = next++;
+            if (i >= items.length)
+                return;
+            results[i] = await fn(items[i], i);
+        }
+    }
+    const workers = Array.from({ length: Math.min(limit, items.length) }, () => worker());
+    await Promise.all(workers);
+    return results;
+}
+function printSetupInstructions(config, log) {
+    const lines = [
+        "",
+        "  ANTHROPIC_API_KEY não está definido — pulando a execução com modelo real.",
+        "  Este runner faz a seleção de tool real via Anthropic Messages API.",
+        "",
+        "  Para rodar de verdade:",
+        "    ANTHROPIC_API_KEY=sk-ant-... npx tsx evals/run.ts",
+        "",
+        "  Variáveis opcionais:",
+        `    EVAL_MODEL=...                 (padrão: ${config.defaultModel ?? DEFAULT_MODEL})`,
+        "    EVAL_CONCURRENCY=4             (requisições paralelas)",
+        "    EVAL_LIMIT=10                  (rodar só as N primeiras fixtures)",
+        "",
+        `  Catálogo: ${config.catalog.tools.length} tools · Fixtures: ${config.fixtures.length} consultas`,
+        "  O scorer e o catálogo são testados offline em `npm test`.",
+        "",
+    ];
+    log(lines.join("\n"));
+}
+/**
+ * Roda o eval completo e imprime o relatório. Devolve o resultado com `exitCode`
+ * (0 completo/pulado, 2 rodada incompleta) — o `evals/run.ts` do projeto decide se
+ * chama `process.exit`.
+ */
+export async function runEval(config) {
+    const env = config.env ?? process.env;
+    const log = config.log ?? console.log;
+    const logError = config.logError ?? console.error;
+    const fetchImpl = config.fetchImpl ?? fetch;
+    const sleepImpl = config.sleepImpl ?? defaultSleep;
+    const maxRetries = config.maxRetries ?? MAX_RETRIES;
+    const apiKey = env.ANTHROPIC_API_KEY;
+    if (!apiKey) {
+        printSetupInstructions(config, log);
+        return { exitCode: 0, skipped: true };
+    }
+    const model = env.EVAL_MODEL || config.defaultModel || DEFAULT_MODEL;
+    const concurrency = Math.max(1, parseInt(env.EVAL_CONCURRENCY || "4", 10) || 4);
+    const limit = parseInt(env.EVAL_LIMIT || "", 10);
+    const fixtures = Number.isFinite(limit) && limit > 0 ? config.fixtures.slice(0, limit) : config.fixtures;
+    const tools = catalogAsAnthropicTools(config.catalog);
+    log(`Rodando ${fixtures.length} fixtures contra ${tools.length} tools (modelo ${model}, concorrência ${concurrency})...`);
+    // Depois de uma falha fatal de infra (auth/billing), toda fixture restante bateria na
+    // mesma parede — curto-circuita em vez de martelar a API com requisições condenadas.
+    let fatalAbort = null;
+    const predictions = [];
+    const errors = [];
+    await mapWithConcurrency(fixtures, concurrency, async (f) => {
+        if (fatalAbort) {
+            errors.push({ id: f.id, kind: fatalAbort.kind, status: fatalAbort.status, message: "abortado (falha fatal anterior)" });
+            return;
+        }
+        try {
+            predictions.push(await predictOne(f, apiKey, model, tools, config.systemPrompt, fetchImpl, sleepImpl, maxRetries, logError));
+        }
+        catch (e) {
+            const err = e instanceof EvalApiError ? e : new EvalApiError(e.message, 0, "other", false);
+            if (isFatalInfra(err.kind))
+                fatalAbort = err;
+            logError(`  ! ${f.id}: [${err.kind}] ${err.message}`);
+            errors.push({ id: f.id, kind: err.kind, status: err.status, message: err.message });
+        }
+    });
+    const predById = new Map(predictions.map((p) => [p.id, p]));
+    const items = fixtures.map((f) => {
+        // A "área" da fixture é a área da primeira expectedTool (a resposta canônica).
+        const first = f.expectedTools[0];
+        const area = (first !== undefined ? config.catalog.areaByName.get(first) : undefined) ?? "desconhecida";
+        return scoreItem(f, predById.get(f.id), area);
+    });
+    const gateOptions = {
+        ...config.gate,
+        ...(config.gate?.toolCount === undefined ? { toolCount: config.catalog.tools.length } : {}),
+    };
+    const formatted = formatReport(items, errors, model, gateOptions);
+    for (const line of formatted.lines)
+        log(line);
+    return {
+        exitCode: formatted.exitCode,
+        skipped: false,
+        report: formatted.report,
+        gate: formatted.gate,
+        errors,
+    };
+}
+//# sourceMappingURL=runner.js.map

package/dist/runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.js","sourceRoot":"","sources":["../src/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,EAAE,uBAAuB,EAAoC,MAAM,cAAc,CAAC;AAEzF,OAAO,EAAE,SAAS,EAAyF,MAAM,YAAY,CAAC;AAC9H,OAAO,EAAE,YAAY,EAAqB,MAAM,aAAa,CAAC;AAC9D,OAAO,EACL,YAAY,EACZ,gBAAgB,EAChB,SAAS,EACT,eAAe,EACf,YAAY,EACZ,WAAW,GACZ,MAAM,YAAY,CAAC;AAEpB,MAAM,OAAO,GAAG,uCAAuC,CAAC;AACxD,MAAM,WAAW,GAAG,YAAY,CAAC;AACjC,MAAM,aAAa,GAAG,iBAAiB,CAAC;AAwCxC,SAAS,YAAY,CAAC,EAAU;IAC9B,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED,0FAA0F;AAC1F,KAAK,UAAU,QAAQ,CACrB,OAAoB,EACpB,MAAc,EACd,KAAa,EACb,KAAsB,EACtB,YAAoB,EACpB,SAAuB;IAEvB,MAAM,IAAI,GAAG;QACX,KAAK;QACL,UAAU,EAAE,IAAI;QAChB,MAAM,EAAE,YAAY;QACpB,KAAK;QACL,WAAW,EAAE,EAAE,IAAI,EAAE,KAAc,EAAE;QACrC,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,OAAO,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC;KAC9D,CAAC;IAEF,IAAI,GAAa,CAAC;IAClB,IAAI,CAAC;QACH,GAAG,GAAG,MAAM,SAAS,CAAC,OAAO,EAAE;YAC7B,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;gBAClC,WAAW,EAAE,MAAM;gBACnB,mBAAmB,EAAE,WAAW;aACjC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;SAC3B,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,yCAAyC;QACzC,MAAM,IAAI,YAAY,CAAC,iBAAkB,CAAW,CAAC,OAAO,IAAI,cAAc,EAAE,EAAE,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;IACxG,CAAC;IAED,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;QACZ,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;QAC9C,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,gBAAgB,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QAC/D,MAAM,GAAG,GAAG,IAAI,YAAY,CAAC,iBAAiB,GAAG,CAAC,MAAM,KAAK,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,SAAS,CAAC,CAAC;QAChH,GAAG,CAAC,iBAAiB,GAAG,eAAe,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC;QACxE,MAAM,GAAG,CAAC;IACZ,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAA0C,CAAC;IACzE,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC;QACvC,IAAI,KAAK,CAAC,IAAI,KAAK,UAAU,IAAI,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAChE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACzB,CAAC;IACH,CAAC;IACD,OAAO,EAAE,EAAE,EAAE,OAAO,CAAC,EAAE,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC;AACnD,CAAC;AAED;;;;;GAKG;AACH,KAAK,UAAU,UAAU,CACvB,OAAoB,EACpB,MAAc,EACd,KAAa,EACb,KAAsB,EACtB,YAAoB,EACpB,SAAuB,EACvB,SAAwC,EACxC,UAAkB,EAClB,QAAgC;IAEhC,IAAI,SAAmC,CAAC;IACxC,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC;QACvD,IAAI,CAAC;YACH,OAAO,MAAM,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,YAAY,EAAE,SAAS,CAAC,CAAC;QAChF,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,GAAG,GAAG,CAAC,YAAY,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,YAAY,CAAE,CAAW,CAAC,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;YACtG,SAAS,GAAG,GAAG,CAAC;YAChB,IAAI,CAAC,GAAG,CAAC,SAAS,IAAI,OAAO,KAAK,UAAU;gBAAE,MAAM,GAAG,CAAC;YACxD,MAAM,IAAI,GAAG,SAAS,CAAC,OAAO,EAAE,GAAG,CAAC,iBAAiB,CAAC,CAAC;YACvD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC;YAC/C,QAAQ,CAAC,SAAS,OAAO,CAAC,EAAE,KAAK,GAAG,CAAC,IAAI,UAAU,GAAG,CAAC,MAAM,YAAY,OAAO,GAAG,CAAC,IAAI,UAAU,OAAO,IAAI,GAAG,MAAM,IAAI,CAAC,CAAC;YAC5H,MAAM,SAAS,CAAC,IAAI,GAAG,MAAM,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;IACD,MAAM,SAAS,IAAI,IAAI,YAAY,CAAC,yBAAyB,OAAO,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;AAChG,CAAC;AAED,qEAAqE;AACrE,KAAK,UAAU,kBAAkB,CAC/B,KAAU,EACV,KAAa,EACb,EAA0C;IAE1C,MAAM,OAAO,GAAQ,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAC7C,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,UAAU,MAAM;QACnB,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC;YACjB,IAAI,CAAC,IAAI,KAAK,CAAC,MAAM;gBAAE,OAAO;YAC9B,OAAO,CAAC,CAAC,CAAC,GAAG,MAAM,EAAE,CAAC,KAAK,CAAC,CAAC,CAAM,EAAE,CAAC,CAAC,CAAC;QAC1C,CAAC;IACH,CAAC;IACD,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC;IACtF,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAC3B,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,sBAAsB,CAAC,MAAwB,EAAE,GAA2B;IACnF,MAAM,KAAK,GAAG;QACZ,EAAE;QACF,6EAA6E;QAC7E,sEAAsE;QACtE,EAAE;QACF,0BAA0B;QAC1B,uDAAuD;QACvD,EAAE;QACF,wBAAwB;QACxB,+CAA+C,MAAM,CAAC,YAAY,IAAI,aAAa,GAAG;QACtF,4DAA4D;QAC5D,uEAAuE;QACvE,EAAE;QACF,eAAe,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,sBAAsB,MAAM,CAAC,QAAQ,CAAC,MAAM,YAAY;QAClG,6DAA6D;QAC7D,EAAE;KACH,CAAC;IACF,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;AACxB,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,MAAwB;IACpD,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,CAAC;IACtC,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,CAAC;IACtC,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,IAAI,OAAO,CAAC,KAAK,CAAC;IAClD,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,IAAI,KAAK,CAAC;IAC5C,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,IAAI,YAAY,CAAC;IACnD,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,IAAI,WAAW,CAAC;IAEpD,MAAM,MAAM,GAAG,GAAG,CAAC,iBAAiB,CAAC;IACrC,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,sBAAsB,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QACpC,OAAO,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IACxC,CAAC;IAED,MAAM,KAAK,GAAG,GAAG,CAAC,UAAU,IAAI,MAAM,CAAC,YAAY,IAAI,aAAa,CAAC;IACrE,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,GAAG,CAAC,gBAAgB,IAAI,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC;IAChF,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,UAAU,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;IACjD,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC;IAEzG,MAAM,KAAK,GAAG,uBAAuB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACtD,GAAG,CAAC,WAAW,QAAQ,CAAC,MAAM,oBAAoB,KAAK,CAAC,MAAM,kBAAkB,KAAK,kBAAkB,WAAW,MAAM,CAAC,CAAC;IAE1H,sFAAsF;IACtF,qFAAqF;IACrF,IAAI,UAAU,GAAwB,IAAI,CAAC;IAC3C,MAAM,WAAW,GAAiB,EAAE,CAAC;IACrC,MAAM,MAAM,GAAmB,EAAE,CAAC;IAElC,MAAM,kBAAkB,CAAC,QAAQ,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;QAC1D,IAAI,UAAU,EAAE,CAAC;YACf,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,OAAO,EAAE,iCAAiC,EAAE,CAAC,CAAC;YACxH,OAAO;QACT,CAAC;QACD,IAAI,CAAC;YACH,WAAW,CAAC,IAAI,CACd,MAAM,UAAU,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,YAAY,EAAE,SAAS,EAAE,SAAS,EAAE,UAAU,EAAE,QAAQ,CAAC,CAC3G,CAAC;QACJ,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,GAAG,GAAG,CAAC,YAAY,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,YAAY,CAAE,CAAW,CAAC,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;YACtG,IAAI,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC;gBAAE,UAAU,GAAG,GAAG,CAAC;YAC7C,QAAQ,CAAC,OAAO,CAAC,CAAC,EAAE,MAAM,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;YACtD,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;QACtF,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5D,MAAM,KAAK,GAAiB,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QAC7C,+EAA+E;QAC/E,MAAM,KAAK,GAAG,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;QACjC,MAAM,IAAI,GAAG,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,cAAc,CAAC;QACxG,OAAO,SAAS,CAAC,CAAC,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC;IAChD,CAAC,CAAC,CAAC;IAEH,MAAM,WAAW,GAAgB;QAC/B,GAAG,MAAM,CAAC,IAAI;QACd,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC5F,CAAC;IACF,MAAM,SAAS,GAAG,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,WAAW,CAAC,CAAC;IAClE,KAAK,MAAM,IAAI,IAAI,SAAS,CAAC,KAAK;QAAE,GAAG,CAAC,IAAI,CAAC,CAAC;IAE9C,OAAO;QACL,QAAQ,EAAE,SAAS,CAAC,QAAQ;QAC5B,OAAO,EAAE,KAAK;QACd,MAAM,EAAE,SAAS,CAAC,MAAM;QACxB,IAAI,EAAE,SAAS,CAAC,IAAI;QACpB,MAAM;KACP,CAAC;AACJ,CAAC"}

package/dist/score.d.ts

@@ -0,0 +1,82 @@
+/**
+ * Núcleo de scoring puro do eval de seleção de tool. Sem rede, sem modelo, sem I/O.
+ *
+ * O runner produz, por fixture, a lista ordenada de tools que o modelo chamaria (mais
+ * provável primeiro). Este módulo transforma essas predições + os `expectedTools` das
+ * fixtures em números de acurácia: top-1, top-k e por área. São estas funções que os
+ * testes offline exercitam com dados sintéticos de acurácia conhecida.
+ */
+import type { EvalFixture } from "./fixtures.js";
+/** Predição do modelo para uma fixture: nomes de tool em ordem de prioridade. */
+export interface Prediction {
+    id: string;
+    /** Candidatas em ordem (mais provável primeiro). Vazio se o modelo não chamou tool. */
+    predictedTools: string[];
+}
+export interface ScoredItem {
+    id: string;
+    area: string;
+    expectedTools: string[];
+    predictedTools: string[];
+    /** True se a predição top-1 está em expectedTools. */
+    top1: boolean;
+    /** Rank (1-based) da primeira predição correta, ou null se nenhuma acertou. */
+    hitRank: number | null;
+}
+export interface AreaAccuracy {
+    area: string;
+    total: number;
+    top1Correct: number;
+    top1Accuracy: number;
+}
+export interface ScoreReport {
+    total: number;
+    /** Fixtures com ao menos uma predição (o modelo tentou chamar uma tool). */
+    attempted: number;
+    top1Correct: number;
+    top1Accuracy: number;
+    /** topKAccuracy[k] = fração cujo primeiro acerto tem rank <= k, para k em 1..maxK. */
+    topKAccuracy: Record<number, number>;
+    byArea: AreaAccuracy[];
+    items: ScoredItem[];
+}
+/** Pontua uma fixture contra sua predição. `area` vem do catálogo, via chamador. */
+export declare function scoreItem(fixture: Pick<EvalFixture, "id" | "expectedTools">, prediction: Prediction | undefined, area: string): ScoredItem;
+/**
+ * Agrega itens pontuados num relatório.
+ *
+ * @param maxK maior k para o cálculo de acurácia top-k (padrão 3).
+ */
+export declare function aggregate(items: ScoredItem[], maxK?: number): ScoreReport;
+/** Conveniência: pontua um lote inteiro dado fixtures, predições e lookup de área. */
+export declare function scoreAll(fixtures: Pick<EvalFixture, "id" | "expectedTools">[], predictions: Prediction[], areaByExpectedTool: (fixture: Pick<EvalFixture, "id" | "expectedTools">) => string, maxK?: number): ScoreReport;
+export type GateDecision = "remediar" | "despriorizar-refatoracao" | "zona-cinzenta";
+export interface GateResult {
+    decision: GateDecision;
+    accuracy: number;
+    /** Recomendação legível em pt-BR. */
+    message: string;
+}
+export interface GateOptions {
+    /** Abaixo deste valor → `remediar`. Padrão: 0.85. */
+    remediationThreshold?: number;
+    /** Neste valor ou acima → `despriorizar-refatoracao`. Padrão: 0.9. */
+    deprioritizeThreshold?: number;
+    /**
+     * Tamanho do catálogo, se informado citado na mensagem de despriorização
+     * ("mesmo com N tools") — dá contexto ao número de acurácia.
+     */
+    toolCount?: number;
+}
+export declare const GATE_REMEDIATION_THRESHOLD = 0.85;
+export declare const GATE_DEPRIORITIZE_THRESHOLD = 0.9;
+/**
+ * Aplica o gate de seleção de tool a uma acurácia top-1:
+ *   < remediationThreshold  → abrir sessão de remediação (deferred loading / Code Mode /
+ *                             agrupamento por sessão).
+ *   >= deprioritizeThreshold → despriorizar refatoração de catálogo (seguir só
+ *                             consolidando via enums).
+ *   entre os dois → zona cinzenta (manter sob observação).
+ */
+export declare function evaluateGate(top1Accuracy: number, options?: GateOptions): GateResult;
+//# sourceMappingURL=score.d.ts.map

package/dist/score.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"score.d.ts","sourceRoot":"","sources":["../src/score.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAEjD,iFAAiF;AACjF,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,uFAAuF;IACvF,cAAc,EAAE,MAAM,EAAE,CAAC;CAC1B;AAED,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,sDAAsD;IACtD,IAAI,EAAE,OAAO,CAAC;IACd,+EAA+E;IAC/E,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;CACxB;AAED,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,WAAW;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,4EAA4E;IAC5E,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,sFAAsF;IACtF,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACrC,MAAM,EAAE,YAAY,EAAE,CAAC;IACvB,KAAK,EAAE,UAAU,EAAE,CAAC;CACrB;AAED,oFAAoF;AACpF,wBAAgB,SAAS,CACvB,OAAO,EAAE,IAAI,CAAC,WAAW,EAAE,IAAI,GAAG,eAAe,CAAC,EAClD,UAAU,EAAE,UAAU,GAAG,SAAS,EAClC,IAAI,EAAE,MAAM,GACX,UAAU,CAmBZ;AAED;;;;GAIG;AACH,wBAAgB,SAAS,CAAC,KAAK,EAAE,UAAU,EAAE,EAAE,IAAI,SAAI,GAAG,WAAW,CAoCpE;AAED,sFAAsF;AACtF,wBAAgB,QAAQ,CACtB,QAAQ,EAAE,IAAI,CAAC,WAAW,EAAE,IAAI,GAAG,eAAe,CAAC,EAAE,EACrD,WAAW,EAAE,UAAU,EAAE,EACzB,kBAAkB,EAAE,CAAC,OAAO,EAAE,IAAI,CAAC,WAAW,EAAE,IAAI,GAAG,eAAe,CAAC,KAAK,MAAM,EAClF,IAAI,SAAI,GACP,WAAW,CAIb;AAID,MAAM,MAAM,YAAY,GAAG,UAAU,GAAG,0BAA0B,GAAG,eAAe,CAAC;AAErF,MAAM,WAAW,UAAU;IACzB,QAAQ,EAAE,YAAY,CAAC;IACvB,QAAQ,EAAE,MAAM,CAAC;IACjB,qCAAqC;IACrC,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,WAAW;IAC1B,qDAAqD;IACrD,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,sEAAsE;IACtE,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,eAAO,MAAM,0BAA0B,OAAO,CAAC;AAC/C,eAAO,MAAM,2BAA2B,MAAM,CAAC;AAO/C;;;;;;;GAOG;AACH,wBAAgB,YAAY,CAAC,YAAY,EAAE,MAAM,EAAE,OAAO,GAAE,WAAgB,GAAG,UAAU,CA8BxF"}

package/dist/score.js

@@ -0,0 +1,116 @@
+/**
+ * Núcleo de scoring puro do eval de seleção de tool. Sem rede, sem modelo, sem I/O.
+ *
+ * O runner produz, por fixture, a lista ordenada de tools que o modelo chamaria (mais
+ * provável primeiro). Este módulo transforma essas predições + os `expectedTools` das
+ * fixtures em números de acurácia: top-1, top-k e por área. São estas funções que os
+ * testes offline exercitam com dados sintéticos de acurácia conhecida.
+ */
+/** Pontua uma fixture contra sua predição. `area` vem do catálogo, via chamador. */
+export function scoreItem(fixture, prediction, area) {
+    const predicted = prediction?.predictedTools ?? [];
+    const expected = new Set(fixture.expectedTools);
+    let hitRank = null;
+    for (const [i, name] of predicted.entries()) {
+        if (expected.has(name)) {
+            hitRank = i + 1;
+            break;
+        }
+    }
+    const first = predicted[0];
+    return {
+        id: fixture.id,
+        area,
+        expectedTools: fixture.expectedTools,
+        predictedTools: predicted,
+        top1: first !== undefined && expected.has(first),
+        hitRank,
+    };
+}
+/**
+ * Agrega itens pontuados num relatório.
+ *
+ * @param maxK maior k para o cálculo de acurácia top-k (padrão 3).
+ */
+export function aggregate(items, maxK = 3) {
+    const total = items.length;
+    const attempted = items.filter((it) => it.predictedTools.length > 0).length;
+    const top1Correct = items.filter((it) => it.top1).length;
+    const topKAccuracy = {};
+    for (let k = 1; k <= maxK; k++) {
+        const hits = items.filter((it) => it.hitRank !== null && it.hitRank <= k).length;
+        topKAccuracy[k] = total === 0 ? 0 : hits / total;
+    }
+    const areas = new Map();
+    for (const it of items) {
+        const a = areas.get(it.area) ?? { total: 0, correct: 0 };
+        a.total += 1;
+        if (it.top1)
+            a.correct += 1;
+        areas.set(it.area, a);
+    }
+    const byArea = [...areas.entries()]
+        .map(([area, v]) => ({
+        area,
+        total: v.total,
+        top1Correct: v.correct,
+        top1Accuracy: v.total === 0 ? 0 : v.correct / v.total,
+    }))
+        .sort((x, y) => x.top1Accuracy - y.top1Accuracy || x.area.localeCompare(y.area));
+    return {
+        total,
+        attempted,
+        top1Correct,
+        top1Accuracy: total === 0 ? 0 : top1Correct / total,
+        topKAccuracy,
+        byArea,
+        items,
+    };
+}
+/** Conveniência: pontua um lote inteiro dado fixtures, predições e lookup de área. */
+export function scoreAll(fixtures, predictions, areaByExpectedTool, maxK = 3) {
+    const predById = new Map(predictions.map((p) => [p.id, p]));
+    const items = fixtures.map((f) => scoreItem(f, predById.get(f.id), areaByExpectedTool(f)));
+    return aggregate(items, maxK);
+}
+export const GATE_REMEDIATION_THRESHOLD = 0.85;
+export const GATE_DEPRIORITIZE_THRESHOLD = 0.9;
+/** Formata um limiar (fração) como percentual sem decimais espúrios: 0.85 → "85". */
+function thresholdPct(fraction) {
+    return String(Math.round(fraction * 100 * 1e6) / 1e6);
+}
+/**
+ * Aplica o gate de seleção de tool a uma acurácia top-1:
+ *   < remediationThreshold  → abrir sessão de remediação (deferred loading / Code Mode /
+ *                             agrupamento por sessão).
+ *   >= deprioritizeThreshold → despriorizar refatoração de catálogo (seguir só
+ *                             consolidando via enums).
+ *   entre os dois → zona cinzenta (manter sob observação).
+ */
+export function evaluateGate(top1Accuracy, options = {}) {
+    const { remediationThreshold = GATE_REMEDIATION_THRESHOLD, deprioritizeThreshold = GATE_DEPRIORITIZE_THRESHOLD, toolCount, } = options;
+    const pct = (top1Accuracy * 100).toFixed(1);
+    const remPct = thresholdPct(remediationThreshold);
+    const depPct = thresholdPct(deprioritizeThreshold);
+    if (top1Accuracy < remediationThreshold) {
+        return {
+            decision: "remediar",
+            accuracy: top1Accuracy,
+            message: `Acurácia top-1 = ${pct}% (< ${remPct}%). Recomendação: abrir SESSÃO DE REMEDIAÇÃO (deferred loading / Code Mode / agrupamento por sessão).`,
+        };
+    }
+    if (top1Accuracy >= deprioritizeThreshold) {
+        const context = toolCount !== undefined ? ` mesmo com ${toolCount} tools` : "";
+        return {
+            decision: "despriorizar-refatoracao",
+            accuracy: top1Accuracy,
+            message: `Acurácia top-1 = ${pct}% (>= ${depPct}%)${context}. Recomendação: DESPRIORIZAR refatoração de catálogo; seguir consolidando via enums.`,
+        };
+    }
+    return {
+        decision: "zona-cinzenta",
+        accuracy: top1Accuracy,
+        message: `Acurácia top-1 = ${pct}% (entre ${remPct}% e ${depPct}%). Zona cinzenta: manter sob observação; reavaliar após próxima mudança de tool/descrição.`,
+    };
+}
+//# sourceMappingURL=score.js.map

package/dist/score.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"score.js","sourceRoot":"","sources":["../src/score.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAyCH,oFAAoF;AACpF,MAAM,UAAU,SAAS,CACvB,OAAkD,EAClD,UAAkC,EAClC,IAAY;IAEZ,MAAM,SAAS,GAAG,UAAU,EAAE,cAAc,IAAI,EAAE,CAAC;IACnD,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;IAChD,IAAI,OAAO,GAAkB,IAAI,CAAC;IAClC,KAAK,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,SAAS,CAAC,OAAO,EAAE,EAAE,CAAC;QAC5C,IAAI,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YACvB,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC;YAChB,MAAM;QACR,CAAC;IACH,CAAC;IACD,MAAM,KAAK,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;IAC3B,OAAO;QACL,EAAE,EAAE,OAAO,CAAC,EAAE;QACd,IAAI;QACJ,aAAa,EAAE,OAAO,CAAC,aAAa;QACpC,cAAc,EAAE,SAAS;QACzB,IAAI,EAAE,KAAK,KAAK,SAAS,IAAI,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC;QAChD,OAAO;KACR,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,SAAS,CAAC,KAAmB,EAAE,IAAI,GAAG,CAAC;IACrD,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC;IAC3B,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;IAC5E,MAAM,WAAW,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;IAEzD,MAAM,YAAY,GAA2B,EAAE,CAAC;IAChD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;QAC/B,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,OAAO,KAAK,IAAI,IAAI,EAAE,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;QACjF,YAAY,CAAC,CAAC,CAAC,GAAG,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,KAAK,CAAC;IACnD,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,GAAG,EAA8C,CAAC;IACpE,KAAK,MAAM,EAAE,IAAI,KAAK,EAAE,CAAC;QACvB,MAAM,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;QACzD,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;QACb,IAAI,EAAE,CAAC,IAAI;YAAE,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC;QAC5B,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IACxB,CAAC;IACD,MAAM,MAAM,GAAmB,CAAC,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC;SAChD,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACnB,IAAI;QACJ,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,WAAW,EAAE,CAAC,CAAC,OAAO;QACtB,YAAY,EAAE,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,KAAK;KACtD,CAAC,CAAC;SACF,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,YAAY,IAAI,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAEnF,OAAO;QACL,KAAK;QACL,SAAS;QACT,WAAW;QACX,YAAY,EAAE,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,KAAK;QACnD,YAAY;QACZ,MAAM;QACN,KAAK;KACN,CAAC;AACJ,CAAC;AAED,sFAAsF;AACtF,MAAM,UAAU,QAAQ,CACtB,QAAqD,EACrD,WAAyB,EACzB,kBAAkF,EAClF,IAAI,GAAG,CAAC;IAER,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5D,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,CAAC,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3F,OAAO,SAAS,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;AAChC,CAAC;AAyBD,MAAM,CAAC,MAAM,0BAA0B,GAAG,IAAI,CAAC;AAC/C,MAAM,CAAC,MAAM,2BAA2B,GAAG,GAAG,CAAC;AAE/C,qFAAqF;AACrF,SAAS,YAAY,CAAC,QAAgB;IACpC,OAAO,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,GAAG,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC;AACxD,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,YAAY,CAAC,YAAoB,EAAE,UAAuB,EAAE;IAC1E,MAAM,EACJ,oBAAoB,GAAG,0BAA0B,EACjD,qBAAqB,GAAG,2BAA2B,EACnD,SAAS,GACV,GAAG,OAAO,CAAC;IACZ,MAAM,GAAG,GAAG,CAAC,YAAY,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAC5C,MAAM,MAAM,GAAG,YAAY,CAAC,oBAAoB,CAAC,CAAC;IAClD,MAAM,MAAM,GAAG,YAAY,CAAC,qBAAqB,CAAC,CAAC;IAEnD,IAAI,YAAY,GAAG,oBAAoB,EAAE,CAAC;QACxC,OAAO;YACL,QAAQ,EAAE,UAAU;YACpB,QAAQ,EAAE,YAAY;YACtB,OAAO,EAAE,oBAAoB,GAAG,QAAQ,MAAM,uGAAuG;SACtJ,CAAC;IACJ,CAAC;IACD,IAAI,YAAY,IAAI,qBAAqB,EAAE,CAAC;QAC1C,MAAM,OAAO,GAAG,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,cAAc,SAAS,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/E,OAAO;YACL,QAAQ,EAAE,0BAA0B;YACpC,QAAQ,EAAE,YAAY;YACtB,OAAO,EAAE,oBAAoB,GAAG,SAAS,MAAM,KAAK,OAAO,sFAAsF;SAClJ,CAAC;IACJ,CAAC;IACD,OAAO;QACL,QAAQ,EAAE,eAAe;QACzB,QAAQ,EAAE,YAAY;QACtB,OAAO,EAAE,oBAAoB,GAAG,YAAY,MAAM,OAAO,MAAM,6FAA6F;KAC7J,CAAC;AACJ,CAAC"}

package/package.json

@@ -0,0 +1,38 @@
+{
+  "name": "@sbissoli/mcp-evals",
+  "version": "0.1.0",
+  "description": "Harness de evals de seleção de tool para servidores MCP: extrator de catálogo (fake McpServer), validação de fixtures, scorer offline (top-1/top-k/por-área), gate de acurácia e runner via Anthropic Messages API.",
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "scripts": {
+    "build": "tsc -p tsconfig.build.json",
+    "typecheck": "tsc --noEmit",
+    "test": "vitest run"
+  },
+  "keywords": [
+    "mcp",
+    "evals",
+    "tool-selection",
+    "evaluation",
+    "open-data"
+  ],
+  "author": "Sidney Bissoli <sbissoli76@gmail.com>",
+  "license": "MIT",
+  "peerDependencies": {
+    "zod": "^4.0.0"
+  },
+  "devDependencies": {
+    "zod": "^4.0.0"
+  }
+}