@2kw/ai-mcp-server 4.0.0-dev.2

package/dist/tools/datasets.js

@@ -0,0 +1,200 @@
+import { z } from "zod";
+import { formatErrorForMcp } from "../errors.js";
+async function resolveLatestVersionId(client, datasetId) {
+    const { data } = await client.GET("/v1/datasets/{id}/versions/latest", { params: { path: { id: datasetId } } });
+    const versionId = data?.id;
+    if (!versionId) {
+        throw new Error(`Could not resolve latest version for dataset ${datasetId}. Create a version first.`);
+    }
+    return versionId;
+}
+export function register(server, client) {
+    // ── list_datasets ──────────────────────────────────────────────
+    server.tool("2kw_list_datasets", "List datasets in the organization with optional search, type filter, and pagination.", {
+        search: z.string().optional().describe("Filter datasets by name"),
+        type: z.string().optional().describe("Filter by dataset type"),
+        page: z.number().optional().default(0).describe("Page number (0-based)"),
+        size: z.number().optional().default(20).describe("Page size"),
+        sort: z.string().optional().describe("Sort field and direction"),
+    }, async ({ search, type, page, size, sort }) => {
+        try {
+            const { data } = await client.GET("/v1/datasets", {
+                params: {
+                    query: {
+                        search,
+                        type,
+                        pageable: {
+                            page,
+                            size,
+                            sort: sort ? [sort] : undefined,
+                        },
+                    },
+                },
+            });
+            const result = data;
+            const lines = (result.content ?? []).map((d) => `- ${d.name} (id: ${d.id}, type: ${d.type ?? "—"})${d.description ? ` — ${d.description}` : ""}`);
+            return {
+                content: [
+                    {
+                        type: "text",
+                        text: `Datasets (${result.totalElements} total, page ${(result.number ?? 0) + 1}/${result.totalPages}):\n${lines.join("\n") || "(none)"}`,
+                    },
+                ],
+            };
+        }
+        catch (error) {
+            return {
+                content: [{ type: "text", text: formatErrorForMcp(error) }],
+                isError: true,
+            };
+        }
+    });
+    // ── get_dataset ────────────────────────────────────────────────
+    server.tool("2kw_get_dataset", "Get a dataset by its ID.", {
+        datasetId: z.string().describe("The dataset ID"),
+    }, async ({ datasetId }) => {
+        try {
+            const { data } = await client.GET("/v1/datasets/{id}", {
+                params: { path: { id: datasetId } },
+            });
+            return {
+                content: [{ type: "text", text: JSON.stringify(data, null, 2) }],
+            };
+        }
+        catch (error) {
+            return {
+                content: [{ type: "text", text: formatErrorForMcp(error) }],
+                isError: true,
+            };
+        }
+    });
+    // ── create_dataset ─────────────────────────────────────────────
+    server.tool("2kw_create_dataset", "Create a new dataset in the organization.", {
+        name: z.string().min(1).describe("Dataset name"),
+        description: z.string().optional().describe("Dataset description"),
+        type: z.string().optional().describe("Dataset type"),
+        inputSchema: z.unknown().optional().describe("Input JSON Schema definition"),
+        expectedOutputSchema: z.unknown().optional().describe("Expected output JSON Schema definition"),
+        metadata: z.unknown().optional().describe("Arbitrary metadata (JSON object)"),
+    }, async ({ name, description, type, inputSchema, expectedOutputSchema, metadata }) => {
+        try {
+            const body = { name };
+            if (description !== undefined)
+                body.description = description;
+            if (type !== undefined)
+                body.type = type;
+            if (inputSchema !== undefined)
+                body.inputSchema = inputSchema;
+            if (expectedOutputSchema !== undefined)
+                body.expectedOutputSchema = expectedOutputSchema;
+            if (metadata !== undefined)
+                body.metadata = metadata;
+            const { data } = await client.POST("/v1/datasets", {
+                body: body,
+            });
+            return {
+                content: [{ type: "text", text: JSON.stringify(data, null, 2) }],
+            };
+        }
+        catch (error) {
+            return {
+                content: [{ type: "text", text: formatErrorForMcp(error) }],
+                isError: true,
+            };
+        }
+    });
+    // ── create_dataset_version ─────────────────────────────────────
+    server.tool("2kw_create_dataset_version", "Create a new version of a dataset.", {
+        datasetId: z.string().describe("The dataset ID"),
+        changeDescription: z.string().optional().describe("Description of what changed in this version"),
+    }, async ({ datasetId, changeDescription }) => {
+        try {
+            const { data } = await client.POST("/v1/datasets/{id}/versions", {
+                params: { path: { id: datasetId } },
+                body: { changeDescription },
+            });
+            return {
+                content: [{ type: "text", text: JSON.stringify(data, null, 2) }],
+            };
+        }
+        catch (error) {
+            return {
+                content: [{ type: "text", text: formatErrorForMcp(error) }],
+                isError: true,
+            };
+        }
+    });
+    // ── add_dataset_item ───────────────────────────────────────────
+    server.tool("2kw_add_dataset_item", "Add an item to a dataset. Uses the latest version unless versionId is specified.", {
+        datasetId: z.string().describe("The dataset ID"),
+        versionId: z.string().optional().describe("The dataset version ID (defaults to latest)"),
+        input: z.unknown().describe("Item input data (JSON object)"),
+        expectedOutput: z.unknown().optional().describe("Expected output (JSON object)"),
+        tags: z.array(z.string()).optional().describe("Tags for the item"),
+        metadata: z.unknown().optional().describe("Arbitrary metadata (JSON object)"),
+    }, async ({ datasetId, versionId, input, expectedOutput, tags, metadata }) => {
+        try {
+            const resolvedVersionId = versionId ?? (await resolveLatestVersionId(client, datasetId));
+            const body = { input };
+            if (expectedOutput !== undefined)
+                body.expectedOutput = expectedOutput;
+            if (tags !== undefined)
+                body.tags = tags;
+            if (metadata !== undefined)
+                body.metadata = metadata;
+            const { data } = await client.POST("/v1/datasets/{id}/versions/{versionId}/items", {
+                params: {
+                    // OpenAPI spec omits `id` in declared path params though `{id}` is in the URL template.
+                    path: { id: datasetId, versionId: resolvedVersionId },
+                },
+                body: body,
+            });
+            return {
+                content: [{ type: "text", text: JSON.stringify(data, null, 2) }],
+            };
+        }
+        catch (error) {
+            return {
+                content: [{ type: "text", text: formatErrorForMcp(error) }],
+                isError: true,
+            };
+        }
+    });
+    // ── get_dataset_items ──────────────────────────────────────────
+    server.tool("2kw_get_dataset_items", "List items in a dataset with pagination. Uses the latest version unless versionId is specified.", {
+        datasetId: z.string().describe("The dataset ID"),
+        versionId: z.string().optional().describe("The dataset version ID (defaults to latest)"),
+        page: z.number().optional().default(0).describe("Page number (0-based)"),
+        size: z.number().optional().default(20).describe("Page size"),
+    }, async ({ datasetId, versionId, page, size }) => {
+        try {
+            const resolvedVersionId = versionId ?? (await resolveLatestVersionId(client, datasetId));
+            const { data } = await client.GET("/v1/datasets/{id}/versions/{versionId}/items", {
+                params: {
+                    // OpenAPI spec omits `id` in declared path params though `{id}` is in the URL template.
+                    path: { id: datasetId, versionId: resolvedVersionId },
+                    query: { pageable: { page, size } },
+                },
+            });
+            const result = data;
+            const lines = (result.content ?? []).map(
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            (item) => `- Item ${item.id}: input=${JSON.stringify(item.input)}${item.expectedOutput ? `, expected=${JSON.stringify(item.expectedOutput)}` : ""}${item.tags?.length ? `, tags=[${item.tags.join(", ")}]` : ""}`);
+            return {
+                content: [
+                    {
+                        type: "text",
+                        text: `Items (${result.totalElements} total, page ${(result.number ?? 0) + 1}/${result.totalPages}):\n${lines.join("\n") || "(none)"}`,
+                    },
+                ],
+            };
+        }
+        catch (error) {
+            return {
+                content: [{ type: "text", text: formatErrorForMcp(error) }],
+                isError: true,
+            };
+        }
+    });
+}
+//# sourceMappingURL=datasets.js.map

package/dist/tools/docs.d.ts

@@ -0,0 +1,4 @@
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import type { ApiClient } from "../client.js";
+export declare function register(server: McpServer, _client: ApiClient, baseUrl: string, apiKey: string): void;
+//# sourceMappingURL=docs.d.ts.map

package/dist/tools/docs.js

@@ -0,0 +1,171 @@
+import { z } from "zod";
+import { formatErrorForMcp } from "../errors.js";
+/* eslint-enable @typescript-eslint/no-explicit-any */
+let specCache = null;
+async function fetchSpec(baseUrl, apiKey) {
+    if (specCache)
+        return specCache;
+    const specUrl = `${baseUrl.replace(/\/+$/, "")}/v3/api-docs`;
+    const res = await fetch(specUrl, {
+        headers: { Authorization: `Bearer ${apiKey}` },
+    });
+    if (!res.ok) {
+        if (res.status === 404 || res.status === 403) {
+            throw new Error("API documentation is not available. Enable it by setting SPRINGDOC_ENABLED=true on the backend.");
+        }
+        throw new Error(`Failed to fetch API docs: ${res.status} ${res.statusText}`);
+    }
+    specCache = await res.json();
+    return specCache;
+}
+/**
+ * Collect all $ref strings from an object tree.
+ */
+function collectRefs(obj, refs) {
+    if (obj === null || obj === undefined || typeof obj !== "object")
+        return;
+    if (Array.isArray(obj)) {
+        for (const item of obj)
+            collectRefs(item, refs);
+        return;
+    }
+    const record = obj;
+    if (typeof record["$ref"] === "string") {
+        refs.add(record["$ref"]);
+    }
+    for (const value of Object.values(record)) {
+        collectRefs(value, refs);
+    }
+}
+/**
+ * Recursively resolve schema refs — the initial set plus any schemas they reference.
+ */
+function resolveSchemas(refPaths, allSchemas) {
+    const resolved = {};
+    const queue = [...refPaths];
+    const visited = new Set();
+    while (queue.length > 0) {
+        const ref = queue.pop();
+        if (visited.has(ref))
+            continue;
+        visited.add(ref);
+        const prefix = "#/components/schemas/";
+        if (!ref.startsWith(prefix))
+            continue;
+        const name = ref.slice(prefix.length);
+        const schema = allSchemas[name];
+        if (!schema)
+            continue;
+        resolved[name] = schema;
+        // Find nested refs in this schema
+        const nested = new Set();
+        collectRefs(schema, nested);
+        for (const n of nested) {
+            if (!visited.has(n))
+                queue.push(n);
+        }
+    }
+    return resolved;
+}
+export function register(server, _client, baseUrl, apiKey) {
+    // ── list_api_doc_sections ───────────────────────────────────────────────
+    server.tool("2kw_list_api_doc_sections", "List available API documentation sections (tags) from the OpenAPI spec. Use this to discover which sections you can fetch detailed docs for.", {}, async () => {
+        try {
+            const spec = await fetchSpec(baseUrl, apiKey);
+            const tags = spec.tags ?? [];
+            const paths = spec.paths ?? {};
+            // Count endpoints per tag
+            const tagCounts = new Map();
+            for (const methods of Object.values(paths)) {
+                for (const operation of Object.values(methods)) {
+                    if (operation?.tags) {
+                        for (const tag of operation.tags) {
+                            tagCounts.set(tag, (tagCounts.get(tag) ?? 0) + 1);
+                        }
+                    }
+                }
+            }
+            const lines = tags.map((t) => {
+                const count = tagCounts.get(t.name) ?? 0;
+                const desc = t.description ? ` — ${t.description}` : "";
+                return `- ${t.name} (${count} endpoint${count !== 1 ? "s" : ""})${desc}`;
+            });
+            const info = spec.info ?? {};
+            const header = `${info.title ?? "API"} v${info.version ?? "?"}`;
+            return {
+                content: [
+                    {
+                        type: "text",
+                        text: `${header}\n\nAvailable sections:\n${lines.join("\n") || "(none)"}`,
+                    },
+                ],
+            };
+        }
+        catch (error) {
+            return {
+                content: [{ type: "text", text: formatErrorForMcp(error) }],
+                isError: true,
+            };
+        }
+    });
+    // ── get_api_docs ────────────────────────────────────────────────────────
+    server.tool("2kw_get_api_docs", "Fetch API documentation for a specific section (tag). Returns the filtered endpoints and their referenced schemas. Use 2kw_list_api_doc_sections first to see available tags.", {
+        tag: z.string().describe("The tag/section name to filter by (e.g. 'Projects', 'Extractions')"),
+    }, async ({ tag }) => {
+        try {
+            const spec = await fetchSpec(baseUrl, apiKey);
+            const allTags = spec.tags ?? [];
+            const tagNames = allTags.map((t) => t.name);
+            if (!tagNames.includes(tag)) {
+                return {
+                    content: [
+                        {
+                            type: "text",
+                            text: `Tag "${tag}" not found. Available tags: ${tagNames.join(", ")}`,
+                        },
+                    ],
+                    isError: true,
+                };
+            }
+            const allPaths = spec.paths ?? {};
+            const allSchemas = spec.components?.schemas ?? {};
+            // Filter paths to only operations matching the requested tag
+            const filteredPaths = {};
+            const refs = new Set();
+            for (const [path, methods] of Object.entries(allPaths)) {
+                const filteredMethods = {};
+                for (const [method, operation] of Object.entries(methods)) {
+                    const op = operation;
+                    if (op?.tags?.includes(tag)) {
+                        filteredMethods[method] = operation;
+                        collectRefs(operation, refs);
+                    }
+                }
+                if (Object.keys(filteredMethods).length > 0) {
+                    filteredPaths[path] = filteredMethods;
+                }
+            }
+            // Resolve referenced schemas (including transitive refs)
+            const referencedSchemas = resolveSchemas(refs, allSchemas);
+            const result = {
+                paths: filteredPaths,
+                schemas: referencedSchemas,
+            };
+            return {
+                content: [
+                    {
+                        type: "text",
+                        text: JSON.stringify(result, null, 2),
+                    },
+                ],
+            };
+        }
+        catch (error) {
+            return {
+                content: [{ type: "text", text: formatErrorForMcp(error) }],
+                isError: true,
+            };
+        }
+    });
+}
+//# sourceMappingURL=docs.js.map

package/dist/tools/evaluators.d.ts

@@ -0,0 +1,4 @@
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import type { ApiClient } from "../client.js";
+export declare function register(server: McpServer, client: ApiClient): void;
+//# sourceMappingURL=evaluators.d.ts.map

package/dist/tools/evaluators.js

@@ -0,0 +1,140 @@
+import { z } from "zod";
+import { formatErrorForMcp } from "../errors.js";
+export function register(server, client) {
+    // ── built-in types ──────────────────────────────────────────────────────
+    server.tool("2kw_list_evaluator_types", "List the built-in evaluator types that experiments can score against (e.g. exact_match, grounding, llm_judge). These are the primitive kinds — use evaluator templates to reuse a configured judge across experiments.", {}, async () => {
+        try {
+            const { data } = await client.GET("/v1/evaluators");
+            return {
+                content: [{ type: "text", text: JSON.stringify(data, null, 2) }],
+            };
+        }
+        catch (error) {
+            return {
+                content: [{ type: "text", text: formatErrorForMcp(error) }],
+                isError: true,
+            };
+        }
+    });
+    // ── templates ───────────────────────────────────────────────────────────
+    server.tool("2kw_list_evaluator_templates", "List org-scoped evaluator templates. Templates bundle a type + name + description + config and are referenced by id when attached to experiments.", {}, async () => {
+        try {
+            const { data } = await client.GET("/v1/evaluator-templates");
+            return {
+                content: [{ type: "text", text: JSON.stringify(data, null, 2) }],
+            };
+        }
+        catch (error) {
+            return {
+                content: [{ type: "text", text: formatErrorForMcp(error) }],
+                isError: true,
+            };
+        }
+    });
+    server.tool("2kw_get_evaluator_template", "Show a single evaluator template by id, including its config.", { id: z.string().describe("Evaluator template id") }, async (params) => {
+        try {
+            const { data } = await client.GET("/v1/evaluator-templates/{id}", {
+                params: { path: { id: params.id } },
+            });
+            return {
+                content: [{ type: "text", text: JSON.stringify(data, null, 2) }],
+            };
+        }
+        catch (error) {
+            return {
+                content: [{ type: "text", text: formatErrorForMcp(error) }],
+                isError: true,
+            };
+        }
+    });
+    server.tool("2kw_create_evaluator_template", "Create a reusable evaluator template. LLM-judge templates require a config object with rubricPrompt (string) and model (provider/model). CODE and HUMAN templates may omit config.", {
+        name: z.string().describe("Template name"),
+        type: z.enum(["CODE", "LLM_JUDGE", "HUMAN"]).describe("Evaluator type"),
+        description: z.string().optional().describe("Description"),
+        config: z
+            .record(z.string(), z.unknown())
+            .optional()
+            .describe("Type-specific config JSON (LLM_JUDGE requires rubricPrompt + model)"),
+    }, async (params) => {
+        try {
+            const { data } = await client.POST("/v1/evaluator-templates", {
+                body: {
+                    name: params.name,
+                    type: params.type,
+                    description: params.description,
+                    config: params.config,
+                },
+            });
+            return {
+                content: [
+                    {
+                        type: "text",
+                        text: `Evaluator template created.\n\n${JSON.stringify(data, null, 2)}`,
+                    },
+                ],
+            };
+        }
+        catch (error) {
+            return {
+                content: [{ type: "text", text: formatErrorForMcp(error) }],
+                isError: true,
+            };
+        }
+    });
+    server.tool("2kw_update_evaluator_template", "Replace an evaluator template. PUT semantics: omitted fields fall back to the current values (the tool reads current state first and merges).", {
+        id: z.string().describe("Evaluator template id"),
+        name: z.string().optional(),
+        type: z.enum(["CODE", "LLM_JUDGE", "HUMAN"]).optional(),
+        description: z.string().optional(),
+        config: z.record(z.string(), z.unknown()).optional(),
+    }, async (params) => {
+        try {
+            const { data: current } = await client.GET("/v1/evaluator-templates/{id}", {
+                params: { path: { id: params.id } },
+            });
+            const cur = current;
+            const { data } = await client.PUT("/v1/evaluator-templates/{id}", {
+                params: { path: { id: params.id } },
+                body: {
+                    name: params.name ?? cur?.name,
+                    type: params.type ?? cur?.type,
+                    description: params.description ?? cur?.description,
+                    config: params.config ?? cur?.config,
+                },
+            });
+            return {
+                content: [
+                    {
+                        type: "text",
+                        text: `Evaluator template updated.\n\n${JSON.stringify(data, null, 2)}`,
+                    },
+                ],
+            };
+        }
+        catch (error) {
+            return {
+                content: [{ type: "text", text: formatErrorForMcp(error) }],
+                isError: true,
+            };
+        }
+    });
+    server.tool("2kw_delete_evaluator_template", "Delete an evaluator template. Experiments that already scored against it keep their past scores.", { id: z.string().describe("Evaluator template id") }, async (params) => {
+        try {
+            await client.DELETE("/v1/evaluator-templates/{id}", {
+                params: { path: { id: params.id } },
+            });
+            return {
+                content: [
+                    { type: "text", text: `Deleted evaluator template ${params.id}` },
+                ],
+            };
+        }
+        catch (error) {
+            return {
+                content: [{ type: "text", text: formatErrorForMcp(error) }],
+                isError: true,
+            };
+        }
+    });
+}
+//# sourceMappingURL=evaluators.js.map

package/dist/tools/experiments.d.ts

@@ -0,0 +1,4 @@
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import type { ApiClient } from "../client.js";
+export declare function register(server: McpServer, client: ApiClient): void;
+//# sourceMappingURL=experiments.d.ts.map