@verdict-ai/mcp 0.0.0-stage → 0.2.0

package/LICENSE

@@ -0,0 +1,9 @@
+MIT License
+
+Copyright (c) 2026 Verdict contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package/README.md

@@ -1,3 +1,25 @@
-# Temporary Holding Version
+# @verdict-ai/mcp
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+Stdio MCP server exposing evaluation tools to agents.
+
+```bash
+npm install @verdict-ai/mcp
+```
+
+Add this to `claude_desktop_config.json`:
+
+```json
+{
+  "mcpServers": {
+    "verdict": {
+      "command": "npx",
+      "args": ["-y", "@verdict-ai/mcp"],
+      "env": {
+        "VERDICT_API_URL": "http://localhost:8787"
+      }
+    }
+  }
+}
+```
+
+Requires Node 20 or newer. [Documentation](https://verdict-docs.developerolim.workers.dev/reference/mcp). MIT licensed.

package/dist/agreement.d.ts

@@ -0,0 +1,67 @@
+/**
+ * Exposes check_against_humans: run a rubric file over a human-labelled gold file and report agreement.
+ * Agents edit the rubric between calls, so the file is re-read every time; the reply lists every disagreement with a
+ * preview so the caller can see what to change without opening the gold file.
+ */
+import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { type AgreementReport, type Engine } from "@verdict-ai/core";
+/** File access stays injectable so the tool can be exercised in memory. */
+export interface AgreementDependencies {
+    engine?: Engine;
+    read: (path: string) => Promise<string>;
+    resolve: (path: string) => string;
+    dirname: (path: string) => string;
+}
+/** Shapes a report for an agent: totals first, then only the subjects and criteria that disagree. */
+export declare function agreementSummary(report: AgreementReport): {
+    rubric: string;
+    items: number;
+    agreement: string;
+    decided: number;
+    agreed: number;
+    askedHuman: number;
+    failed: number;
+    criteria: {
+        [k: string]: {
+            agreed: number;
+            n: number;
+            agreement: string | null;
+            askedHuman: number;
+        };
+    };
+    splits: {
+        [k: string]: {
+            agreed: number;
+            decided: number;
+            items: number;
+            agreement: string | null;
+        };
+    };
+    labelSources: Record<string, number>;
+    disagreements: {
+        id: string;
+        name: string;
+        human: {
+            passed?: boolean | undefined;
+            criteria?: Record<string, string | number | boolean> | undefined;
+        };
+        preview: string | undefined;
+        verdict: {
+            passed: boolean | null;
+            criteria: {
+                [k: string]: {
+                    value: string | number | null;
+                    confidence: number | null;
+                };
+            };
+        };
+    }[];
+    askedHumanFor: string[];
+    invalidInput: {
+        id: string;
+        name: string;
+        errors: import("@verdict-ai/core").InputError[] | undefined;
+    }[];
+};
+/** Registers the tool; failures reach the caller as MCP error content. */
+export declare function registerAgreement(server: McpServer, deps: AgreementDependencies): void;

package/dist/agreement.js

@@ -0,0 +1,45 @@
+import { checkAgainstHumans, loadRubricText, parseGoldSet, parseRubric, ValidationError } from "@verdict-ai/core";
+import { z } from "zod";
+import { text, toolResult } from "./result.js";
+const percent = (x) => (x === null ? null : `${Math.round(x * 100)}%`);
+/** Shapes a report for an agent: totals first, then only the subjects and criteria that disagree. */
+export function agreementSummary(report) {
+    const disagreeing = report.results.filter((x) => x.agrees === false || Object.values(x.criteria).some((c) => c.agrees === false));
+    return {
+        rubric: report.rubric, items: report.items,
+        agreement: `${report.agreed}/${report.decided} decided by Verdict agree with the humans (${percent(report.agreement)})`,
+        decided: report.decided, agreed: report.agreed, askedHuman: report.askedHuman, failed: report.failed,
+        criteria: Object.fromEntries(Object.entries(report.criteria).map(([id, c]) => [id, { agreed: c.agreed, n: c.n, agreement: percent(c.agreement), askedHuman: c.askedHuman }])),
+        splits: Object.fromEntries(Object.entries(report.splits).map(([name, t]) => [name, { agreed: t.agreed, decided: t.decided, items: t.items, agreement: percent(t.agreement) }])),
+        labelSources: report.labelSources,
+        disagreements: disagreeing.map((x) => ({
+            id: x.id, name: x.name, human: x.human, preview: x.preview,
+            verdict: { passed: x.passed, criteria: Object.fromEntries(Object.entries(x.criteria).map(([id, c]) => [id, { value: c.verdict, confidence: c.confidence === null ? null : Math.round(c.confidence * 100) / 100 }])) },
+        })),
+        askedHumanFor: report.results.filter((x) => x.askedHuman).map((x) => x.name),
+        invalidInput: report.results.filter((x) => x.inputErrors).map((x) => ({ id: x.id, name: x.name, errors: x.inputErrors })),
+    };
+}
+/** Registers the tool; failures reach the caller as MCP error content. */
+export function registerAgreement(server, deps) {
+    server.registerTool("check_against_humans", {
+        title: "Check a rubric against human labels",
+        description: "Run a rubric over a gold set (subjects people already judged) and report how often Verdict reached the same verdict: overall pass/fail agreement, per-criterion agreement, how many subjects the judge handed to a human (low-confidence escalation), and every disagreement with a preview. `rubricPath` is a local YAML/JSON rubric file, re-read on every call; `gold` is a local JSON array or JSON-lines file of {id?, name, inputs, expected?: {criterion: label}, human?: {passed?, criteria?}, labelSource?, split?}; choice criteria are labelled with an option id, and unreadable labels are rejected before any judge runs. Local mode only.",
+        inputSchema: {
+            rubricPath: z.string().optional().describe("Local rubric file (YAML or JSON)"),
+            rubric: z.union([z.string(), z.record(z.string(), z.unknown())]).optional().describe("Stored rubric id or inline rubric, instead of rubricPath"),
+            gold: z.string().describe("Local gold-set file: JSON array or JSON lines"),
+        },
+    }, async ({ rubricPath, rubric, gold }) => toolResult(async () => {
+        // Gold files and rubric paths are local; a remote API has no access to them.
+        if (!deps.engine)
+            throw new ValidationError("check_against_humans needs the local engine (unset VERDICT_API_URL)");
+        const ref = rubricPath ? loadRubricText(await deps.read(deps.resolve(rubricPath))) : typeof rubric === "string" ? rubric : rubric ? parseRubric(rubric) : undefined;
+        if (!ref)
+            throw new ValidationError("pass rubricPath or rubric");
+        const goldPath = deps.resolve(gold);
+        const items = parseGoldSet(await deps.read(goldPath), deps.dirname(goldPath));
+        return text(agreementSummary(await checkAgainstHumans(deps.engine, ref, items)));
+    }));
+}
+//# sourceMappingURL=agreement.js.map

package/dist/agreement.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"agreement.js","sourceRoot":"","sources":["../src/agreement.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,kBAAkB,EAAE,cAAc,EAAE,YAAY,EAAE,WAAW,EAAE,eAAe,EAAqC,MAAM,kBAAkB,CAAC;AACrJ,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAK/C,MAAM,OAAO,GAAG,CAAC,CAAgB,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;AAEtF,qGAAqG;AACrG,MAAM,UAAU,gBAAgB,CAAC,MAAuB;IACtD,MAAM,WAAW,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,KAAK,IAAI,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,KAAK,CAAC,CAAC,CAAC;IAClI,OAAO;QACL,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK;QAC1C,SAAS,EAAE,GAAG,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,OAAO,8CAA8C,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG;QACvH,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM;QACpG,QAAQ,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;QAC7K,MAAM,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC;QAC/K,YAAY,EAAE,MAAM,CAAC,YAAY;QACjC,aAAa,EAAE,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACrC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO;YAC1D,OAAO,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,QAAQ,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,UAAU,EAAE,CAAC,CAAC,UAAU,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,GAAG,GAAG,CAAC,GAAG,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE;SACtN,CAAC,CAAC;QACH,aAAa,EAAE,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QAC5E,YAAY,EAAE,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;KAC1H,CAAC;AACJ,CAAC;AAED,0EAA0E;AAC1E,MAAM,UAAU,iBAAiB,CAAC,MAAiB,EAAE,IAA2B;IAC9E,MAAM,CAAC,YAAY,CAAC,sBAAsB,EAAE;QAC1C,KAAK,EAAE,qCAAqC;QAC5C,WAAW,EAAE,0nBAA0nB;QACvoB,WAAW,EAAE;YACX,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,kCAAkC,CAAC;YAC9E,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,0DAA0D,CAAC;YAChJ,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,+CAA+C,CAAC;SAC3E;KACF,EAAE,KAAK,EAAE,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,UAAU,CAAC,KAAK,IAAI,EAAE;QAC/D,6EAA6E;QAC7E,IAAI,CAAC,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,eAAe,CAAC,qEAAqE,CAAC,CAAC;QACnH,MAAM,GAAG,GAAG,UAAU,CAAC,CAAC,CAAC,cAAc,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACpK,IAAI,CAAC,GAAG;YAAE,MAAM,IAAI,eAAe,CAAC,2BAA2B,CAAC,CAAC;QACjE,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACpC,MAAM,KAAK,GAAG,YAAY,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;QAC9E,OAAO,IAAI,CAAC,gBAAgB,CAAC,MAAM,kBAAkB,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;IACnF,CAAC,CAAC,CAAC,CAAC;AACN,CAAC"}

package/dist/bin.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/dist/bin.js

@@ -0,0 +1,10 @@
+#!/usr/bin/env node
+/**
+ * Starts the executable transport at the process boundary.
+ * Library imports remain side-effect free; startup failures must be visible to the invoking process.
+ */
+import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
+import { createMcpServer } from "./index.js";
+const server = createMcpServer();
+await server.connect(new StdioServerTransport());
+//# sourceMappingURL=bin.js.map

package/dist/bin.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"bin.js","sourceRoot":"","sources":["../src/bin.ts"],"names":[],"mappings":";AACA;;;GAGG;AACH,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAC;AACjF,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAE7C,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;AACjC,MAAM,MAAM,CAAC,OAAO,CAAC,IAAI,oBAAoB,EAAE,CAAC,CAAC"}

package/dist/hub.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Hub tools for agents (docs/design/hub.md P3-4): search published cards, read one, and fork one into the rubrics this
+ * server keeps so `evaluate` can use it. Public reads only — no hub key is sent. A card is used only after its rubric
+ * freezes here to the version the hub names.
+ */
+import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { type Engine } from "@verdict-ai/core";
+import { Verdict } from "@verdict-ai/sdk";
+export interface HubToolOptions {
+    /** Default hub (VERDICT_HUB_URL); each call may name another. */
+    hubUrl?: string;
+    fetch?: typeof fetch;
+    engine?: Engine;
+    client?: Verdict;
+}
+export declare function registerHub(server: McpServer, opts: HubToolOptions): void;

package/dist/hub.js

@@ -0,0 +1,62 @@
+import { forkDraft, parseHubRef, questionAgreement, ValidationError, verifyPublishedVersion } from "@verdict-ai/core";
+import { Verdict } from "@verdict-ai/sdk";
+import { z } from "zod";
+import { text, toolResult } from "./result.js";
+const CASES_SHOWN = 20;
+export function registerHub(server, opts) {
+    const hubFor = (hub) => {
+        const url = (hub ?? opts.hubUrl)?.replace(/\/$/, "");
+        if (!url)
+            throw new ValidationError("no hub configured: pass `hub` or set VERDICT_HUB_URL");
+        // An empty key: never fall back to VERDICT_API_KEY, which belongs to another server.
+        return { url, client: new Verdict({ baseUrl: url, apiKey: "", fetch: opts.fetch }) };
+    };
+    const load = async (ref, hub) => {
+        const r = parseHubRef(ref), h = hubFor(hub);
+        const { publication, versions } = await h.client.hub.card(r.owner, r.slug, r.versionId ? { version: r.versionId } : {});
+        return { hub: h.url, publication, versions, version: await verifyPublishedVersion(publication, r.versionId, h.url) };
+    };
+    server.registerTool("hub_search", {
+        title: "Search the hub",
+        description: "Search published single-question judgment cards (rubrics with public test cases). Returns one result per card at its newest version, with a `hub:owner/slug@rv_…` ref for hub_get or fork_rubric.",
+        inputSchema: {
+            q: z.string().optional().describe("matches title, author, slug and domain"), domain: z.string().optional(), locale: z.string().optional(),
+            type: z.enum(["score", "noul", "choice"]).optional().describe("noul = yes/no"), owner: z.string().optional(), limit: z.number().int().min(1).max(50).optional(),
+            hub: z.string().url().optional(),
+        },
+    }, async ({ hub, ...q }) => toolResult(async () => {
+        const { data } = await hubFor(hub).client.hub.search(q);
+        return text(data.map((d) => ({ ref: `hub:${d.ref}@${d.latest.versionId}`, title: d.title, description: d.description, judgmentType: d.judgmentType, domain: d.domain, locale: d.locale, license: d.license, cases: d.cases, runs: d.runs })));
+    }));
+    server.registerTool("hub_get", {
+        title: "Read a hub card",
+        description: "Read a published card: its question and answers, threshold, licence, lineage, author-reported agreement numbers, versions and some public cases. `ref` is hub:owner/slug or hub:owner/slug@rv_….",
+        inputSchema: { ref: z.string(), hub: z.string().url().optional() },
+    }, async ({ ref, hub }) => toolResult(async () => {
+        const card = await load(ref, hub);
+        return text({ ref: `hub:${card.publication.ref}@${card.version.id}`, hub: card.hub, ...describe(card.publication), versions: card.versions, rubric: card.version.rubric });
+    }));
+    server.registerTool("fork_rubric", {
+        title: "Fork a hub card",
+        description: "Copy a published card into this server's rubrics (a new rubric id; the card is not changed) so `evaluate` can use it. The rubric's metadata records where it came from and its licence.",
+        inputSchema: { ref: z.string(), hub: z.string().url().optional(), name: z.string().optional().describe("name for the copy (default: the card's)") },
+    }, async ({ ref, hub, name }) => toolResult(async () => {
+        const card = await load(ref, hub);
+        const forkedFrom = { hub: card.hub, ref: `hub:${card.publication.ref}@${card.version.id}`, versionId: card.version.id, license: card.publication.license };
+        const draft = forkDraft(card.version);
+        const rubric = { ...draft, ...(name ? { name } : {}), metadata: { ...draft.metadata, forkedFrom } };
+        const stored = opts.client ? await opts.client.rubrics.create(rubric) : await opts.engine.createRubric(rubric);
+        return text({ id: stored.id, name: stored.name, forkedFrom, publicCases: card.publication.cases.length });
+    }));
+}
+function describe(p) {
+    const c = p.version.rubric.criteria[0];
+    return {
+        title: p.title, description: p.description, license: p.license, judgmentType: p.judgmentType, domain: p.domain, locale: p.locale,
+        question: { id: c.id, name: c.name, description: c.description, anchors: c.anchors, choices: c.choices, levels: c.levels, threshold: c.threshold },
+        parent: p.parent, changeKinds: p.version.changeKinds, forkNote: p.version.forkNote, sources: p.sources,
+        runs: p.runs.map((r) => ({ origin: r.origin, cases: r.counts.cases, agreement: questionAgreement(p, r), casesPublic: r.casesPublic, labelSources: r.labelSources, models: r.fingerprint.models })),
+        cases: { total: p.cases.length, shown: p.cases.slice(0, CASES_SHOWN) },
+    };
+}
+//# sourceMappingURL=hub.js.map

package/dist/hub.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"hub.js","sourceRoot":"","sources":["../src/hub.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,iBAAiB,EAAE,eAAe,EAAE,sBAAsB,EAAuC,MAAM,kBAAkB,CAAC;AAC3J,OAAO,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAC1C,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAU/C,MAAM,WAAW,GAAG,EAAE,CAAC;AAEvB,MAAM,UAAU,WAAW,CAAC,MAAiB,EAAE,IAAoB;IACjE,MAAM,MAAM,GAAG,CAAC,GAAY,EAAE,EAAE;QAC9B,MAAM,GAAG,GAAG,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACrD,IAAI,CAAC,GAAG;YAAE,MAAM,IAAI,eAAe,CAAC,sDAAsD,CAAC,CAAC;QAC5F,qFAAqF;QACrF,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,OAAO,CAAC,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC;IACvF,CAAC,CAAC;IACF,MAAM,IAAI,GAAG,KAAK,EAAE,GAAW,EAAE,GAAY,EAAE,EAAE;QAC/C,MAAM,CAAC,GAAG,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;QAC5C,MAAM,EAAE,WAAW,EAAE,QAAQ,EAAE,GAAG,MAAM,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACxH,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,WAAW,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC,WAAW,EAAE,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC;IACvH,CAAC,CAAC;IAEF,MAAM,CAAC,YAAY,CAAC,YAAY,EAAE;QAChC,KAAK,EAAE,gBAAgB;QACvB,WAAW,EAAE,mMAAmM;QAChN,WAAW,EAAE;YACX,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,wCAAwC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;YACzI,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,eAAe,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE;YAC/J,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;SACjC;KACF,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,EAAE,EAAE,CAAC,UAAU,CAAC,KAAK,IAAI,EAAE;QAChD,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QACxD,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,MAAM,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,WAAW,EAAE,CAAC,CAAC,WAAW,EAAE,YAAY,EAAE,CAAC,CAAC,YAAY,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAChP,CAAC,CAAC,CAAC,CAAC;IAEJ,MAAM,CAAC,YAAY,CAAC,SAAS,EAAE;QAC7B,KAAK,EAAE,iBAAiB;QACxB,WAAW,EAAE,kMAAkM;QAC/M,WAAW,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,EAAE;KACnE,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,UAAU,CAAC,KAAK,IAAI,EAAE;QAC/C,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QAClC,OAAO,IAAI,CAAC,EAAE,GAAG,EAAE,OAAO,IAAI,CAAC,WAAW,CAAC,GAAG,IAAI,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAC7K,CAAC,CAAC,CAAC,CAAC;IAEJ,MAAM,CAAC,YAAY,CAAC,aAAa,EAAE;QACjC,KAAK,EAAE,iBAAiB;QACxB,WAAW,EAAE,yLAAyL;QACtM,WAAW,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,yCAAyC,CAAC,EAAE;KACpJ,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,UAAU,CAAC,KAAK,IAAI,EAAE;QACrD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QAClC,MAAM,UAAU,GAAG,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,OAAO,IAAI,CAAC,WAAW,CAAC,GAAG,IAAI,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,EAAE,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;QAC3J,MAAM,KAAK,GAAG,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACtC,MAAM,MAAM,GAAG,EAAE,GAAG,KAAK,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,GAAG,KAAK,CAAC,QAAQ,EAAE,UAAU,EAAE,EAAE,CAAC;QACpG,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,MAAO,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QAChH,OAAO,IAAI,CAAC,EAAE,EAAE,EAAE,MAAM,CAAC,EAAE,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,UAAU,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;IAC5G,CAAC,CAAC,CAAC,CAAC;AACN,CAAC;AAED,SAAS,QAAQ,CAAC,CAAoB;IACpC,MAAM,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAE,CAAC;IACxC,OAAO;QACL,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,WAAW,EAAE,CAAC,CAAC,WAAW,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,YAAY,EAAE,CAAC,CAAC,YAAY,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM;QAChI,QAAQ,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC,WAAW,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE;QAClJ,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE,QAAQ,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO;QACtG,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,EAAE,SAAS,EAAE,iBAAiB,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,WAAW,EAAE,YAAY,EAAE,CAAC,CAAC,YAAY,EAAE,MAAM,EAAE,CAAC,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC;QAClM,KAAK,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,WAAW,CAAC,EAAE;KACvE,CAAC;AACJ,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,27 @@
+/**
+ * Exposes rubric, evaluation and human-review operations as MCP tools.
+ * Local engines and remote clients share result projections; transport schemas validate tool arguments.
+ */
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { Engine } from "@verdict-ai/core";
+import { Verdict } from "@verdict-ai/sdk";
+/**
+ * Verdict MCP server. Two modes:
+ *  - local:  runs the engine in-process (set VERDICT_DATA_DIR to persist; needs provider keys e.g. ANTHROPIC_API_KEY)
+ *  - remote: proxies to a Verdict API (set VERDICT_API_URL + VERDICT_API_KEY)
+ */
+export interface McpOptions {
+    engine?: Engine;
+    client?: Verdict;
+    mode?: "local" | "remote";
+    apiUrl?: string;
+    apiKey?: string;
+    dataDir?: string;
+    allowLocalFiles?: boolean;
+    /** Hub for hub_search / hub_get / fork_rubric (default: VERDICT_HUB_URL). */
+    hubUrl?: string;
+    /** Network for hub reads (tests). */
+    fetch?: typeof fetch;
+}
+/** Returns a configured MCP server using local or remote dependencies; transport startup remains the caller responsibility. */
+export declare function createMcpServer(opts?: McpOptions): McpServer;

package/dist/index.js

@@ -0,0 +1,149 @@
+import { summarize } from "./summary.js";
+import { evaluateSubject, compareSubjects } from "./operations.js";
+import { registerAgreement } from "./agreement.js";
+import { registerHub } from "./hub.js";
+import { text, toolResult } from "./result.js";
+import { readFile } from "node:fs/promises";
+import { dirname, resolve } from "node:path";
+/**
+ * Exposes rubric, evaluation and human-review operations as MCP tools.
+ * Local engines and remote clients share result projections; transport schemas validate tool arguments.
+ */
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { Aggregation, calibrate, coerceScore, criterionScale, scaleRange, NotFoundError, ValidationError, Engine, FileStore, Input, lazyPlaywrightScreenshot, loadRubricText, MemoryStore, parseRubric } from "@verdict-ai/core";
+import { Verdict } from "@verdict-ai/sdk";
+import { z } from "zod";
+/** Returns a configured MCP server using local or remote dependencies; transport startup remains the caller responsibility. */
+export function createMcpServer(opts = {}) {
+    const mode = opts.mode ?? (process.env.VERDICT_API_URL ? "remote" : "local");
+    const client = opts.client ?? (mode === "remote" ? new Verdict({ baseUrl: opts.apiUrl ?? process.env.VERDICT_API_URL, apiKey: opts.apiKey ?? process.env.VERDICT_API_KEY }) : undefined);
+    const engine = opts.engine ?? (mode === "local" ? new Engine({ store: (opts.dataDir ?? process.env.VERDICT_DATA_DIR) ? new FileStore(opts.dataDir ?? process.env.VERDICT_DATA_DIR) : new MemoryStore(), ingest: { allowLocalFiles: opts.allowLocalFiles ?? true, screenshot: lazyPlaywrightScreenshot() } }) : undefined);
+    const server = new McpServer({ name: "verdict", version: "0.1.0" }, { instructions: "Verdict scores documents, websites, YouTube videos, JSON and text against customizable rubrics. Judges: decision models (Jev / Clef / OpenAI Decisions — typed questions, probabilities, one call per rubric), humans, deterministic benchmarks, optional generative LLM for rationales. Create or inline a rubric, then call evaluate." });
+    registerRubrics(server, engine, client);
+    registerEvaluations(server, engine, client);
+    registerHuman(server, engine, client);
+    registerAgreement(server, { engine: client ? undefined : engine, read: (path) => readFile(path, "utf8"), resolve, dirname });
+    registerHub(server, { hubUrl: opts.hubUrl ?? process.env.VERDICT_HUB_URL, fetch: opts.fetch, engine, client });
+    return server;
+}
+function registerRubrics(server, engine, client) {
+    server.registerTool("validate_rubric", {
+        title: "Validate rubric",
+        description: "Validate a rubric (JSON object or YAML string) and return it with defaults filled in. Use before evaluate to catch mistakes.",
+        inputSchema: { rubric: z.union([z.string(), z.record(z.string(), z.unknown())]) },
+    }, async ({ rubric }) => toolResult(async () => {
+        try {
+            return text({ valid: true, rubric: typeof rubric === "string" ? loadRubricText(rubric) : parseRubric(rubric) });
+        }
+        catch (e) {
+            return text({ valid: false, error: e.message });
+        }
+    }));
+    server.registerTool("create_rubric", {
+        title: "Create rubric",
+        description: "Store a rubric and get back its id for reuse. Criteria need id/name; optional weight, scale, anchors (= rubric levels for decision models), judge ({type: decision|human|benchmark|llm|panel}; default decision/typesafe). Aggregation supports weighted_mean, weighted_sum, mean, median, min, max, geometric_mean, formula; gates; grades; passThreshold.",
+        inputSchema: { rubric: z.union([z.string(), z.record(z.string(), z.unknown())]) },
+    }, async ({ rubric }) => toolResult(async () => {
+        const parsed = typeof rubric === "string" ? loadRubricText(rubric) : parseRubric(rubric);
+        return text(client ? await client.rubrics.create(parsed) : await engine.createRubric(parsed));
+    }));
+    server.registerTool("list_rubrics", { title: "List rubrics", description: "List stored rubrics.", inputSchema: {} }, async () => toolResult(async () => (text(client ? (await client.rubrics.list()).data.map((r) => ({ id: r.id, name: r.name, criteria: r.criteria.length })) : (await engine.store.listRubrics()).map((r) => ({ id: r.id, name: r.name, criteria: r.criteria.length }))))));
+    server.registerTool("calibration_report", {
+        title: "Calibration report", description: "Compare decision confidence against human consensus on stored evaluations.",
+        inputSchema: { rubricId: z.string(), target: z.number().min(0).max(1).optional(), buckets: z.number().int().min(1).max(1000).optional() },
+    }, async ({ rubricId, ...options }) => toolResult(async () => {
+        if (client)
+            return text(await client.rubrics.calibration(rubricId, options));
+        await engine.resolveRubric(rubricId);
+        return text(calibrate(await engine.store.listEvaluations({ rubricId, limit: 1000 }), options));
+    }));
+}
+function registerEvaluations(server, engine, client) {
+    server.registerTool("evaluate", {
+        title: "Evaluate",
+        description: "Score a subject against a rubric. `rubric` is a stored rubric id or an inline rubric. Inputs can be text, json, file (path or base64), url (set `screenshot: true` to let vision judges such as Clef see the page), image (PNG/JPEG/WebP, path or base64), or youtube. Returns the total score, grade, pass/fail, per-criterion scores with rationales, and any pending human tasks.",
+        inputSchema: {
+            rubric: z.union([z.string(), z.record(z.string(), z.unknown())]),
+            inputs: z.array(Input).min(1),
+            name: z.string().optional().describe("Subject name for reports"),
+            idempotencyKey: z.string().min(1).max(255).optional(),
+            metadata: z.record(z.string(), z.unknown()).optional(),
+        },
+    }, async ({ rubric, inputs, name, metadata, idempotencyKey }) => toolResult(async () => {
+        await validateRubricReference(rubric, engine, client);
+        const result = await evaluateSubject({ engine, client }, { rubric, inputs, name, metadata, idempotencyKey });
+        return text({ ...summarize(result.evaluation), replayed: result.replayed });
+    }));
+    server.registerTool("compare", {
+        title: "Compare / rank",
+        description: "Evaluate several subjects against one rubric and return a ranking (leaderboard).",
+        inputSchema: {
+            rubric: z.union([z.string(), z.record(z.string(), z.unknown())]),
+            idempotencyKey: z.string().min(1).max(255).optional(),
+            subjects: z.array(z.object({ name: z.string(), inputs: z.array(Input).min(1) })).min(2),
+        },
+    }, async ({ rubric, subjects, idempotencyKey }) => toolResult(async () => {
+        await validateRubricReference(rubric, engine, client);
+        const result = await compareSubjects({ engine, client }, { rubric, subjects, idempotencyKey });
+        return text({ ranking: result.ranking, replayed: result.replayed });
+    }));
+}
+function registerHuman(server, engine, client) {
+    server.registerTool("get_evaluation", { title: "Get evaluation", description: "Fetch an evaluation by id (status, scores, rationales, pending human tasks).", inputSchema: { id: z.string() } }, async ({ id }) => toolResult(async () => {
+        if (client)
+            return text(summarize(await client.evaluations.get(id)));
+        const ev = await engine.store.getEvaluation(id);
+        return text(ev ? summarize(ev) : { error: `evaluation ${id} not found` });
+    }));
+    server.registerTool("submit_human_score", {
+        title: "Submit human score",
+        description: "Submit a human rater's score for a criterion that is waiting on a human. Score is a number on the criterion scale or an ordinal/binary label.",
+        inputSchema: { evaluationId: z.string(), criterionId: z.string(), raterId: z.string(), score: z.union([z.number(), z.string()]), rationale: z.string().optional() },
+    }, async ({ evaluationId, ...s }) => toolResult(async () => {
+        // Remote submissions use the API validation against its frozen rubric.
+        if (client)
+            return text(summarize(await client.evaluations.submitScore(evaluationId, s)));
+        const evaluation = await engine.store.getEvaluation(evaluationId);
+        if (!evaluation)
+            throw new NotFoundError(`evaluation ${evaluationId} not found`);
+        const criterion = evaluation.rubric.criteria.find(c => c.id === s.criterionId);
+        if (!criterion)
+            throw new ValidationError(`unknown criterion ${s.criterionId}`);
+        // Choices have optional score mappings; only scale-based submissions need range validation.
+        if (criterion.choices && !Object.hasOwn(criterion.choices, String(s.score)))
+            throw new ValidationError(`unknown choice ${s.score}`);
+        if (!criterion.choices) {
+            const scale = criterionScale(evaluation.rubric, criterion);
+            const [min, max] = scaleRange(scale);
+            const numeric = typeof s.score === "number" ? s.score : Number(s.score);
+            if (Number.isFinite(numeric) && (numeric < min || numeric > max))
+                throw new ValidationError(`score must be between ${min} and ${max}`);
+            coerceScore(s.score, scale);
+        }
+        return text(summarize(await engine.submitScore(evaluationId, s)));
+    }));
+    server.registerTool("reaggregate", {
+        title: "Re-aggregate",
+        description: "Recompute the total with a different aggregation (method, formula, weights via gates/grades/passThreshold) without re-running judges. Useful for what-if analysis.",
+        inputSchema: { evaluationId: z.string(), aggregation: z.record(z.string(), z.unknown()), persist: z.boolean().default(false) },
+    }, async ({ evaluationId, aggregation, persist }) => toolResult(async () => (text(summarize(client ? (await client.evaluations.reaggregate(evaluationId, Aggregation.partial().parse(aggregation), persist)) : await engine.reaggregate(evaluationId, Aggregation.partial().parse(aggregation), undefined, persist))))));
+    server.registerTool("list_human_tasks", { title: "List human tasks", description: "Criteria waiting on human raters.", inputSchema: { rater: z.string().optional() } }, async ({ rater }) => toolResult(async () => {
+        if (client)
+            return text((await client.tasks.list(rater)).data);
+        // why: filter only tenant-visible pending work; named assignments take precedence over allowed-rater pools.
+        const list = await engine.store.listEvaluations({ status: "pending_human", limit: TASK_INBOX_LIMIT });
+        return text(list.flatMap((e) => e.pendingTasks.filter((t) => !rater || (t.raterId ? t.raterId === rater : !t.allowedRaters || t.allowedRaters.includes(rater))).map((t) => ({ evaluationId: e.id, rubric: e.rubric.name, subject: e.subject.name, ...t }))));
+    }));
+}
+// why: match the REST inbox cap so local MCP calls do not scan unbounded evaluation history.
+const TASK_INBOX_LIMIT = 500;
+// Stored references need a lookup; inline rubrics are validated by the operation parser.
+async function validateRubricReference(rubric, engine, client) {
+    if (typeof rubric !== "string")
+        return;
+    // Remote APIs resolve ids themselves; avoid adding a lookup to every remote evaluation.
+    if (client)
+        return;
+    await engine.resolveRubric(rubric);
+}
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AACnD,OAAO,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACvC,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAC/C,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC7C;;;GAGG;AACH,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AACpE,OAAO,EAAE,WAAW,EAAE,SAAS,EAAE,WAAW,EAAE,cAAc,EAAE,UAAU,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,wBAAwB,EAAE,cAAc,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AACjO,OAAO,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAC1C,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAqBxB,+HAA+H;AAC/H,MAAM,UAAU,eAAe,CAAC,OAAmB,EAAE;IACnD,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;IAC7E,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IACzL,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,IAAI,SAAS,CAAC,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,GAAG,CAAC,gBAAiB,CAAC,CAAC,CAAC,CAAC,IAAI,WAAW,EAAE,EAAE,MAAM,EAAE,EAAE,eAAe,EAAE,IAAI,CAAC,eAAe,IAAI,IAAI,EAAE,UAAU,EAAE,wBAAwB,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IAE3T,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,YAAY,EAAE,yUAAyU,EAAE,CAAC,CAAC;IAEja,eAAe,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IACxC,mBAAmB,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IAC5C,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IACtC,iBAAiB,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;IAC7H,WAAW,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IAC/G,OAAO,MAAM,CAAC;AAChB,CAAC;AAGD,SAAS,eAAe,CAAC,MAAiB,EAAE,MAA0B,EAAE,MAA2B;IACjG,MAAM,CAAC,YAAY,CAAC,iBAAiB,EAAE;QACrC,KAAK,EAAE,iBAAiB;QACxB,WAAW,EAAE,8HAA8H;QAC3I,WAAW,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE;KAClF,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,UAAU,CAAC,KAAK,IAAI,EAAE;QAC7C,IAAI,CAAC;YAAC,OAAO,IAAI,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAAC,CAAC;QACxH,OAAO,CAAC,EAAE,CAAC;YAAC,OAAO,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAG,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;QAAC,CAAC;IAC3E,CAAC,CAAC,CAAC,CAAC;IAEJ,MAAM,CAAC,YAAY,CAAC,eAAe,EAAE;QACnC,KAAK,EAAE,eAAe;QACtB,WAAW,EAAE,6VAA6V;QAC1W,WAAW,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE;KAClF,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,UAAU,CAAC,KAAK,IAAI,EAAE;QAC7C,MAAM,MAAM,GAAG,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QACzF,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,MAAO,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC;IACjG,CAAC,CAAC,CAAC,CAAC;IAEJ,MAAM,CAAC,YAAY,CAAC,cAAc,EAAE,EAAE,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,sBAAsB,EAAE,WAAW,EAAE,EAAE,EAAE,EAAE,KAAK,IAAI,EAAE,CAC9H,UAAU,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,MAAO,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAEjQ,MAAM,CAAC,YAAY,CAAC,oBAAoB,EAAE;QACxC,KAAK,EAAE,oBAAoB,EAAE,WAAW,EAAE,4EAA4E;QACtH,WAAW,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE;KAC1I,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,GAAG,OAAO,EAAE,EAAE,EAAE,CAAC,UAAU,CAAC,KAAK,IAAI,EAAE;QAC3D,IAAI,MAAM;YAAE,OAAO,IAAI,CAAC,MAAM,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;QAC7E,MAAM,MAAO,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;QACtC,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,MAAO,CAAC,KAAK,CAAC,eAAe,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;IAClG,CAAC,CAAC,CAAC,CAAC;AAEN,CAAC;AAED,SAAS,mBAAmB,CAAC,MAAiB,EAAE,MAA0B,EAAE,MAA2B;IACrG,MAAM,CAAC,YAAY,CAAC,UAAU,EAAE;QAC9B,KAAK,EAAE,UAAU;QACjB,WAAW,EAAE,sXAAsX;QACnY,WAAW,EAAE;YACX,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YAChE,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YAC7B,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,0BAA0B,CAAC;YAChE,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE;YACrD,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;SACvD;KACF,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,cAAc,EAAE,EAAE,EAAE,CAAC,UAAU,CAAC,KAAK,IAAI,EAAE;QACrF,MAAM,uBAAuB,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;QACtD,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,cAAc,EAAE,CAAC,CAAC;QAC7G,OAAO,IAAI,CAAC,EAAE,GAAG,SAAS,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC9E,CAAC,CAAC,CAAC,CAAC;IAEJ,MAAM,CAAC,YAAY,CAAC,SAAS,EAAE;QAC7B,KAAK,EAAE,gBAAgB;QACvB,WAAW,EAAE,kFAAkF;QAC/F,WAAW,EAAE;YACX,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YAChE,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE;YACrD,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;SACxF;KACF,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,cAAc,EAAE,EAAE,EAAE,CAAC,UAAU,CAAC,KAAK,IAAI,EAAE;QACvE,MAAM,uBAAuB,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;QACtD,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,cAAc,EAAE,CAAC,CAAC;QAC/F,OAAO,IAAI,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;IACtE,CAAC,CAAC,CAAC,CAAC;AAEN,CAAC;AAED,SAAS,aAAa,CAAC,MAAiB,EAAE,MAA0B,EAAE,MAA2B;IAC/F,MAAM,CAAC,YAAY,CAAC,gBAAgB,EAAE,EAAE,KAAK,EAAE,gBAAgB,EAAE,WAAW,EAAE,8EAA8E,EAAE,WAAW,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,UAAU,CAAC,KAAK,IAAI,EAAE;QACvO,IAAI,MAAM;YAAE,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QACrE,MAAM,EAAE,GAAG,MAAM,MAAO,CAAC,KAAK,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;QACjD,OAAO,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE,YAAY,EAAE,CAAC,CAAC;IAC5E,CAAC,CAAC,CAAC,CAAC;IAEJ,MAAM,CAAC,YAAY,CAAC,oBAAoB,EAAE;QACxC,KAAK,EAAE,oBAAoB;QAC3B,WAAW,EAAE,+IAA+I;QAC5J,WAAW,EAAE,EAAE,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE;KACpK,EAAE,KAAK,EAAE,EAAE,YAAY,EAAE,GAAG,CAAC,EAAE,EAAE,EAAE,CAAC,UAAU,CAAC,KAAK,IAAI,EAAE;QACzD,uEAAuE;QACvE,IAAI,MAAM;YAAE,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,MAAM,CAAC,WAAW,CAAC,WAAW,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1F,MAAM,UAAU,GAAG,MAAM,MAAO,CAAC,KAAK,CAAC,aAAa,CAAC,YAAY,CAAC,CAAC;QACnE,IAAI,CAAC,UAAU;YAAE,MAAM,IAAI,aAAa,CAAC,cAAc,YAAY,YAAY,CAAC,CAAC;QACjF,MAAM,SAAS,GAAG,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,WAAW,CAAC,CAAC;QAC/E,IAAI,CAAC,SAAS;YAAE,MAAM,IAAI,eAAe,CAAC,qBAAqB,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;QAChF,4FAA4F;QAC5F,IAAI,SAAS,CAAC,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YAAE,MAAM,IAAI,eAAe,CAAC,kBAAkB,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;QACpI,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;YACvB,MAAM,KAAK,GAAG,cAAc,CAAC,UAAU,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;YAC3D,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;YACrC,MAAM,OAAO,GAAG,OAAO,CAAC,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YACxE,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,GAAG,GAAG,IAAI,OAAO,GAAG,GAAG,CAAC;gBAAE,MAAM,IAAI,eAAe,CAAC,yBAAyB,GAAG,QAAQ,GAAG,EAAE,CAAC,CAAC;YACvI,WAAW,CAAC,CAAC,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QAC9B,CAAC;QACD,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,MAAO,CAAC,WAAW,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IACrE,CAAC,CAAC,CAAC,CAAC;IAEJ,MAAM,CAAC,YAAY,CAAC,aAAa,EAAE;QACjC,KAAK,EAAE,cAAc;QACrB,WAAW,EAAE,oKAAoK;QACjL,WAAW,EAAE,EAAE,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE;KAC/H,EAAE,KAAK,EAAE,EAAE,YAAY,EAAE,WAAW,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,UAAU,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,WAAW,CAAC,WAAW,CAAC,YAAY,EAAE,WAAW,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,MAAO,CAAC,WAAW,CAAC,YAAY,EAAE,WAAW,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAE1T,MAAM,CAAC,YAAY,CAAC,kBAAkB,EAAE,EAAE,KAAK,EAAE,kBAAkB,EAAE,WAAW,EAAE,mCAAmC,EAAE,WAAW,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,UAAU,CAAC,KAAK,IAAI,EAAE;QACjN,IAAI,MAAM;YAAE,OAAO,IAAI,CAAC,CAAC,MAAM,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC/D,4GAA4G;QAC5G,MAAM,IAAI,GAAG,MAAM,MAAO,CAAC,KAAK,CAAC,eAAe,CAAC,EAAE,MAAM,EAAE,eAAe,EAAE,KAAK,EAAE,gBAAgB,EAAE,CAAC,CAAC;QACvG,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,IAAI,CAAC,CAAC,aAAa,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,YAAY,EAAE,CAAC,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/P,CAAC,CAAC,CAAC,CAAC;AAEN,CAAC;AAGD,6FAA6F;AAC7F,MAAM,gBAAgB,GAAG,GAAG,CAAC;AAG7B,yFAAyF;AACzF,KAAK,UAAU,uBAAuB,CAAC,MAAwC,EAAE,MAAe,EAAE,MAAgB;IAChH,IAAI,OAAO,MAAM,KAAK,QAAQ;QAAE,OAAO;IACvC,wFAAwF;IACxF,IAAI,MAAM;QAAE,OAAO;IACnB,MAAM,MAAO,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;AACtC,CAAC"}

package/dist/operations.d.ts

@@ -0,0 +1,52 @@
+import { type Engine, type Input, type Subject } from "@verdict-ai/core";
+import type { Verdict } from "@verdict-ai/sdk";
+type Dependencies = {
+    engine?: Engine;
+    client?: Verdict;
+};
+type RubricReference = string | Record<string, unknown>;
+/** Transport-independent operation; the idempotency key selects reservation before any judge work. */
+export declare function evaluateSubject(dependencies: Dependencies, params: {
+    rubric: RubricReference;
+    inputs: Input[];
+    name?: string;
+    metadata?: Record<string, unknown>;
+    idempotencyKey?: string;
+}): Promise<{
+    evaluation: import("@verdict-ai/sdk").EvaluationView;
+    replayed: boolean;
+} | {
+    evaluation: import("@verdict-ai/core").Evaluation;
+    replayed: boolean;
+}>;
+/** Shares local and remote ranking semantics without depending on the MCP server. */
+export declare function compareSubjects(dependencies: Dependencies, params: {
+    rubric: RubricReference;
+    subjects: Subject[];
+    idempotencyKey?: string;
+}): Promise<{
+    replayed: boolean;
+    ranking: Array<{
+        rank: number;
+        evaluationId: string;
+        subject: string;
+        score?: number;
+        grade?: string;
+        passed?: boolean;
+        status: string;
+    }>;
+    evaluations: import("@verdict-ai/sdk").EvaluationView[];
+} | {
+    ranking: {
+        rank: number;
+        evaluationId: string;
+        subject: string;
+        score: number | undefined;
+        grade: string | undefined;
+        passed: boolean | undefined;
+        status: import("@verdict-ai/core").EvaluationStatus;
+    }[];
+    evaluations: import("@verdict-ai/core").Evaluation[];
+    replayed: boolean;
+}>;
+export {};

package/dist/operations.js

@@ -0,0 +1,39 @@
+import { parseRubric, ValidationError } from "@verdict-ai/core";
+function localEngine(dependencies) {
+    if (!dependencies.engine)
+        throw new ValidationError("local MCP operation requires an engine");
+    return dependencies.engine;
+}
+/** Transport-independent operation; the idempotency key selects reservation before any judge work. */
+export async function evaluateSubject(dependencies, params) {
+    const { client } = dependencies;
+    const { inputs, name, metadata, idempotencyKey } = params;
+    const rubric = typeof params.rubric === "string" ? params.rubric : parseRubric(params.rubric);
+    if (client) {
+        const request = { rubric, inputs, name, metadata };
+        if (idempotencyKey)
+            return client.evaluations.createIdempotent(request, idempotencyKey);
+        return { evaluation: await client.evaluations.create(request), replayed: false };
+    }
+    const engine = localEngine(dependencies);
+    const request = { rubric, subject: { name, inputs }, metadata };
+    if (idempotencyKey)
+        return engine.evaluateIdempotent({ ...request, idempotencyKey });
+    return { evaluation: await engine.evaluate(request), replayed: false };
+}
+/** Shares local and remote ranking semantics without depending on the MCP server. */
+export async function compareSubjects(dependencies, params) {
+    const { client } = dependencies;
+    const { subjects, idempotencyKey } = params;
+    const rubric = typeof params.rubric === "string" ? params.rubric : parseRubric(params.rubric);
+    if (client) {
+        if (idempotencyKey)
+            return client.batches.createIdempotent({ rubric, subjects }, idempotencyKey);
+        return { ...await client.batches.create({ rubric, subjects }), replayed: false };
+    }
+    const engine = localEngine(dependencies);
+    if (idempotencyKey)
+        return engine.evaluateBatchIdempotent(rubric, subjects, { idempotencyKey });
+    return { ...await engine.evaluateBatch(rubric, subjects), replayed: false };
+}
+//# sourceMappingURL=operations.js.map

package/dist/operations.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"operations.js","sourceRoot":"","sources":["../src/operations.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,eAAe,EAAyC,MAAM,kBAAkB,CAAC;AAMvG,SAAS,WAAW,CAAC,YAA0B;IAC7C,IAAI,CAAC,YAAY,CAAC,MAAM;QAAE,MAAM,IAAI,eAAe,CAAC,wCAAwC,CAAC,CAAC;IAC9F,OAAO,YAAY,CAAC,MAAM,CAAC;AAC7B,CAAC;AAED,sGAAsG;AACtG,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,YAA0B,EAAE,MAEjE;IACC,MAAM,EAAE,MAAM,EAAE,GAAG,YAAY,CAAC;IAChC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,cAAc,EAAE,GAAG,MAAM,CAAC;IAC1D,MAAM,MAAM,GAAG,OAAO,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC9F,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,OAAO,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;QACnD,IAAI,cAAc;YAAE,OAAO,MAAM,CAAC,WAAW,CAAC,gBAAgB,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;QACxF,OAAO,EAAE,UAAU,EAAE,MAAM,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;IACnF,CAAC;IACD,MAAM,MAAM,GAAG,WAAW,CAAC,YAAY,CAAC,CAAC;IACzC,MAAM,OAAO,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,QAAQ,EAAE,CAAC;IAChE,IAAI,cAAc;QAAE,OAAO,MAAM,CAAC,kBAAkB,CAAC,EAAE,GAAG,OAAO,EAAE,cAAc,EAAE,CAAC,CAAC;IACrF,OAAO,EAAE,UAAU,EAAE,MAAM,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;AACzE,CAAC;AAED,qFAAqF;AACrF,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,YAA0B,EAAE,MAEjE;IACC,MAAM,EAAE,MAAM,EAAE,GAAG,YAAY,CAAC;IAChC,MAAM,EAAE,QAAQ,EAAE,cAAc,EAAE,GAAG,MAAM,CAAC;IAC5C,MAAM,MAAM,GAAG,OAAO,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC9F,IAAI,MAAM,EAAE,CAAC;QACX,IAAI,cAAc;YAAE,OAAO,MAAM,CAAC,OAAO,CAAC,gBAAgB,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,cAAc,CAAC,CAAC;QACjG,OAAO,EAAE,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;IACnF,CAAC;IACD,MAAM,MAAM,GAAG,WAAW,CAAC,YAAY,CAAC,CAAC;IACzC,IAAI,cAAc;QAAE,OAAO,MAAM,CAAC,uBAAuB,CAAC,MAAM,EAAE,QAAQ,EAAE,EAAE,cAAc,EAAE,CAAC,CAAC;IAChG,OAAO,EAAE,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,MAAM,EAAE,QAAQ,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;AAC9E,CAAC"}

package/dist/result.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Shapes MCP tool replies. Every tool answers with JSON text; failures become MCP error content instead of
+ * rejecting, so a client sees the message rather than a transport error.
+ */
+export declare const text: (v: unknown) => {
+    content: {
+        type: "text";
+        text: string;
+    }[];
+};
+/** Runs a tool handler; a rejection, including from a local adapter, becomes `isError` content. */
+export declare function toolResult<T>(work: () => Promise<T>): Promise<T | {
+    content: {
+        type: "text";
+        text: string;
+    }[];
+    isError: boolean;
+}>;

package/dist/result.js

@@ -0,0 +1,15 @@
+/**
+ * Shapes MCP tool replies. Every tool answers with JSON text; failures become MCP error content instead of
+ * rejecting, so a client sees the message rather than a transport error.
+ */
+export const text = (v) => ({ content: [{ type: "text", text: JSON.stringify(v, null, 2) }] });
+/** Runs a tool handler; a rejection, including from a local adapter, becomes `isError` content. */
+export async function toolResult(work) {
+    try {
+        return await work();
+    }
+    catch (error) {
+        return { isError: true, ...text({ error: error instanceof Error ? error.message : String(error) }) };
+    }
+}
+//# sourceMappingURL=result.js.map

package/dist/result.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"result.js","sourceRoot":"","sources":["../src/result.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,MAAM,CAAC,MAAM,IAAI,GAAG,CAAC,CAAU,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;AAEjH,mGAAmG;AACnG,MAAM,CAAC,KAAK,UAAU,UAAU,CAAI,IAAsB;IACxD,IAAI,CAAC;QAAC,OAAO,MAAM,IAAI,EAAE,CAAC;IAAC,CAAC;IAC5B,OAAO,KAAK,EAAE,CAAC;QAAC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,CAAC;IAAC,CAAC;AACzH,CAAC"}

package/dist/summary.d.ts

@@ -0,0 +1,45 @@
+import type { Evaluation } from "@verdict-ai/core";
+type SummaryEvaluation = Pick<Evaluation, "id" | "status" | "job" | "aggregate" | "criteria" | "pendingTasks" | "error" | "usage">;
+export declare function summarize(e: SummaryEvaluation): {
+    score: number | undefined;
+    grade: string | undefined;
+    passed: boolean | undefined;
+    partial: boolean;
+    criteria: {
+        id: string;
+        name: string;
+        status: "error" | "scored" | "pending" | "excluded";
+        choice: string | undefined;
+        score: number | undefined;
+        normalized: number | undefined;
+        pending: number;
+        confidence: number | undefined;
+        probabilities: Record<string, number> | undefined;
+        rationale: string | undefined;
+        quotes: string[];
+    }[];
+    pendingTasks: {
+        criterionId: string;
+        raterId?: string;
+        allowedRaters?: string[];
+        instructions?: string;
+    }[];
+    error: string | undefined;
+    usage: {
+        decisionCalls: number;
+        llmCalls: number;
+        inputTokens: number;
+        outputTokens: number;
+        durationMs: number;
+    };
+    job?: {
+        attempts: number;
+        maxAttempts: number;
+        lastError?: string;
+        nextAttemptAt?: string;
+        deadLettered?: boolean;
+    } | undefined;
+    id: string;
+    status: import("@verdict-ai/core").EvaluationStatus;
+};
+export {};

package/dist/summary.js

@@ -0,0 +1,16 @@
+export function summarize(e) {
+    return {
+        id: e.id,
+        status: e.status,
+        ...(e.job ? { job: e.job } : {}),
+        score: e.aggregate?.score,
+        grade: e.aggregate?.grade,
+        passed: e.aggregate?.passed,
+        partial: e.aggregate?.partial,
+        criteria: e.criteria?.map((c) => ({ id: c.criterionId, name: c.name, status: c.status, choice: c.choice, score: c.score, normalized: c.normalized, pending: c.pending, confidence: c.ratings?.find((r) => typeof r.confidence === "number")?.confidence, probabilities: c.ratings?.find((r) => r.probabilities)?.probabilities, rationale: c.ratings?.find((r) => r.rationale)?.rationale, quotes: c.ratings?.flatMap((r) => r.quotes ?? []).slice(0, 3) })),
+        pendingTasks: e.pendingTasks,
+        error: e.error,
+        usage: e.usage,
+    };
+}
+//# sourceMappingURL=summary.js.map

package/dist/summary.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"summary.js","sourceRoot":"","sources":["../src/summary.ts"],"names":[],"mappings":"AAIA,MAAM,UAAU,SAAS,CAAC,CAAoB;IAC5C,OAAO;QACL,EAAE,EAAE,CAAC,CAAC,EAAE;QACR,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAChC,KAAK,EAAE,CAAC,CAAC,SAAS,EAAE,KAAK;QACzB,KAAK,EAAE,CAAC,CAAC,SAAS,EAAE,KAAK;QACzB,MAAM,EAAE,CAAC,CAAC,SAAS,EAAE,MAAM;QAC3B,OAAO,EAAE,CAAC,CAAC,SAAS,EAAE,OAAO;QAC7B,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC,UAAU,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,UAAU,EAAE,CAAC,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,UAAU,KAAK,QAAQ,CAAC,EAAE,UAAU,EAAE,aAAa,EAAE,CAAC,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,EAAE,aAAa,EAAE,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;QAC5b,YAAY,EAAE,CAAC,CAAC,YAAY;QAC5B,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,KAAK,EAAE,CAAC,CAAC,KAAK;KACf,CAAC;AACJ,CAAC"}

package/package.json

@@ -1,6 +1,60 @@
 {
   "name": "@verdict-ai/mcp",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
+  "version": "0.2.0",
+  "description": "Verdict MCP server — expose the evaluation engine to any agent.",
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "bin": {
+    "verdict-mcp": "./dist/bin.js"
+  },
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "dependencies": {
+    "@modelcontextprotocol/sdk": "^1.32.2",
+    "zod": "^4.6.7",
+    "@verdict-ai/core": "0.2.0",
+    "@verdict-ai/sdk": "0.2.0"
+  },
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/Ustay-coder/verdict.git",
+    "directory": "packages/mcp"
+  },
+  "homepage": "https://github.com/Ustay-coder/verdict#readme",
+  "bugs": {
+    "url": "https://github.com/Ustay-coder/verdict/issues"
+  },
+  "keywords": [
+    "evaluation",
+    "rubric",
+    "decision-model",
+    "jev",
+    "clef",
+    "scoring",
+    "llm-judge",
+    "mcp"
+  ],
+  "engines": {
+    "node": ">=20"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "files": [
+    "dist",
+    "README.md",
+    "LICENSE"
+  ],
+  "sideEffects": false,
+  "scripts": {
+    "build": "tsc -p tsconfig.json && chmod +x dist/bin.js",
+    "typecheck": "tsc -p tsconfig.json --noEmit",
+    "test": "vitest run --passWithNoTests"
+  }
 }