@utdk/mcp-core 0.1.0-dev.37053ca

package/src/loader.ts

@@ -0,0 +1,203 @@
+import { OpenApiConverter } from "@utcp/http";
+import type { AuthProvider } from "@utdk/common";
+
+export interface ProviderTool {
+  /** MCP tool name (e.g. "github__repos_list") */
+  mcpName: string;
+  /** Original UTCP tool name (e.g. "github.repos/list") */
+  utcpName: string;
+  description: string;
+  /** JSON Schema for the tool's inputs */
+  inputSchema: Record<string, unknown>;
+  /** Provider name (e.g. "github") */
+  providerName: string;
+  /** OpenAPI operation tags (e.g. ["repos", "issues"]) */
+  tags: string[];
+  /** Transport method (e.g. "GET", "POST") */
+  method: string;
+  /** Route template with {param} placeholders */
+  routeTemplate: string;
+  /** Content-Type for requests */
+  contentType: string;
+  /** Parameter keys that appear in the URL path */
+  pathParamKeys: string[];
+  /** Parameter keys that should go in query string for GET/DELETE */
+  queryParamKeys: string[];
+  /** Auth provider for this provider */
+  auth: AuthProvider | undefined;
+}
+
+export interface UtdkAuthConfig {
+  auth_type: string;
+  api_key?: string;
+  var_name?: string;
+  location?: string;
+  token_url?: string;
+  client_id?: string;
+  client_secret?: string;
+  scope?: string;
+}
+
+interface UtdkPackageJson {
+  name: string;
+  utdk?: {
+    provider: string;
+    auth?: UtdkAuthConfig[];
+    openapi?: {
+      title?: string;
+    };
+  };
+}
+
+interface TransportCallTemplate {
+  call_template_type: string;
+  http_method?: string;
+  url?: string;
+  content_type?: string;
+}
+
+function extractPathParams(routeTemplate: string): string[] {
+  const matches = routeTemplate.match(/\{([^}]+)\}/g) ?? [];
+  return matches.map((m) => m.slice(1, -1));
+}
+
+function toMcpToolName(utcpName: string): string {
+  return utcpName.replace(/\./g, "__").replace(/[^a-zA-Z0-9_-]/g, "_");
+}
+
+/**
+ * Load a provider package's OpenAPI document and package metadata.
+ * This is the single choke point for the openapi.json dynamic-import contract.
+ * Returns null if the package cannot be found or loaded.
+ */
+export async function loadProviderPackage(providerName: string): Promise<{
+  openApiDoc: Record<string, unknown>;
+  packageJson: UtdkPackageJson;
+} | null> {
+  const packageName = `@utdk/${providerName}`;
+
+  try {
+    const [openApiMod, pkgMod] = await Promise.all([
+      import(`${packageName}/openapi.json`, { with: { type: "json" } }).catch(() => null),
+      import(`${packageName}/package.json`, { with: { type: "json" } }).catch(() => null),
+    ]);
+
+    if (!openApiMod || !pkgMod) {
+      process.stderr.write(
+        `[mcp-core] Warning: could not import openapi.json or package.json for ${packageName}\n`,
+      );
+      return null;
+    }
+
+    return {
+      openApiDoc: (openApiMod as { default: Record<string, unknown> }).default as Record<
+        string,
+        unknown
+      >,
+      packageJson: (pkgMod as { default: UtdkPackageJson }).default as UtdkPackageJson,
+    };
+  } catch (err) {
+    process.stderr.write(
+      `[mcp-core] Warning: failed to load provider ${providerName}: ${err}\n`,
+    );
+    return null;
+  }
+}
+
+function buildProviderTools(
+  providerName: string,
+  openApiDoc: Record<string, unknown>,
+  auth: AuthProvider | undefined,
+): ProviderTool[] {
+  const converter = new OpenApiConverter(openApiDoc, {
+    callTemplateName: providerName,
+  });
+
+  const manual = converter.convert();
+  const tools: ProviderTool[] = [];
+
+  for (const tool of manual.tools) {
+    const template = tool.tool_call_template as TransportCallTemplate | undefined;
+    if (!template) continue;
+
+    const routeTemplate = template.url ?? "/";
+    const method = (template.http_method ?? "GET").toUpperCase();
+    const contentType = template.content_type ?? "application/json";
+    const pathParamKeys = extractPathParams(routeTemplate);
+
+    const inputProperties = (tool.inputs?.properties ?? {}) as Record<string, unknown>;
+    const allInputKeys = Object.keys(inputProperties);
+    const isBodyMethod = ["POST", "PUT", "PATCH"].includes(method);
+    const queryParamKeys = isBodyMethod
+      ? []
+      : allInputKeys.filter((k) => !pathParamKeys.includes(k));
+
+    tools.push({
+      mcpName: toMcpToolName(tool.name),
+      utcpName: tool.name,
+      description: tool.description ?? `${method} ${routeTemplate}`,
+      inputSchema: tool.inputs as Record<string, unknown>,
+      providerName,
+      tags: (tool.tags as string[] | undefined) ?? [],
+      method,
+      routeTemplate,
+      contentType,
+      pathParamKeys,
+      queryParamKeys,
+      auth,
+    });
+  }
+
+  return tools;
+}
+
+/**
+ * Callback for authenticating and executing a tool call.
+ * The stdio server passes a fetch-based implementation; the gateway passes its own executor.
+ */
+export type AuthFactory = (
+  providerName: string,
+  authConfigs: UtdkAuthConfig[],
+) => AuthProvider | undefined;
+
+/**
+ * Load all tools for a list of provider names.
+ * Providers that fail to load are skipped with a warning.
+ * Pass `buildAuth` to attach credentials; omit it for unauthenticated tool loading.
+ */
+export async function loadProviders(
+  providerNames: string[],
+  buildAuth?: AuthFactory,
+): Promise<ProviderTool[]> {
+  const allTools: ProviderTool[] = [];
+
+  await Promise.all(
+    providerNames.map(async (providerName) => {
+      const pkg = await loadProviderPackage(providerName);
+      if (!pkg) return;
+
+      const authConfigs = pkg.packageJson.utdk?.auth ?? [];
+      const auth = buildAuth?.(providerName, authConfigs);
+
+      const tools = buildProviderTools(providerName, pkg.openApiDoc, auth);
+      process.stderr.write(
+        `[mcp-core] Loaded ${tools.length} tools from @utdk/${providerName}\n`,
+      );
+      allTools.push(...tools);
+    }),
+  );
+
+  return allTools;
+}
+
+/**
+ * Parse the UTDK_PROVIDERS environment variable.
+ * Returns an array of provider names (e.g. ["github", "slack", "stripe"]).
+ */
+export function parseProviderNames(envValue: string | undefined): string[] {
+  if (!envValue?.trim()) return [];
+  return envValue
+    .split(",")
+    .map((name) => name.trim().toLowerCase())
+    .filter(Boolean);
+}

package/src/meta-tools.ts

@@ -0,0 +1,285 @@
+import { searchTools, groupTools } from "./search.js";
+import type { ProviderTool } from "./loader.js";
+
+// ---------------------------------------------------------------------------
+// Result shape (structurally compatible with MCP SDK CallToolResult)
+// ---------------------------------------------------------------------------
+
+/**
+ * A text content block. Structurally compatible with the MCP SDK's text
+ * `ContentBlock` so handlers can be returned directly from a tools/call
+ * handler.
+ */
+export interface TextContent {
+  type: "text";
+  text: string;
+}
+
+/**
+ * Result of a tools/call handler. Structurally compatible with the MCP SDK's
+ * `CallToolResult` so values returned from these handlers can be passed
+ * straight back to the SDK without an adapter: the stdio server returns them
+ * directly from its `tools/call` request handler, and the hosted gateway
+ * (Phase 2) serializes the same shape over HTTP. mcp-core deliberately does
+ * not depend on `@modelcontextprotocol/sdk`; the handlers return fresh object
+ * literals whose inferred types are assignable to the SDK's `ServerResult`
+ * union, and this interface is the documented shape of those literals.
+ */
+export interface CallToolResult {
+  content: TextContent[];
+  isError?: boolean;
+}
+
+// ---------------------------------------------------------------------------
+// Schema normalization
+// ---------------------------------------------------------------------------
+
+/**
+ * Ensure the input schema is a valid MCP tool inputSchema (type: "object").
+ */
+export function normalizeInputSchema(schema: Record<string, unknown>): {
+  type: "object";
+  properties?: Record<string, object>;
+  required?: string[];
+} {
+  return {
+    type: "object",
+    ...(schema["properties"]
+      ? { properties: schema["properties"] as Record<string, object> }
+      : {}),
+    ...(schema["required"] ? { required: schema["required"] as string[] } : {}),
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Meta-tool definitions (served by tools/list)
+// ---------------------------------------------------------------------------
+
+export const META_TOOLS = [
+  {
+    name: "list_tools",
+    description:
+      "List available tool names without full schemas. Pass an optional provider to filter results, or group_by to organize by category.",
+    inputSchema: {
+      type: "object" as const,
+      properties: {
+        provider: {
+          type: "string",
+          description: "Filter results to tools from this provider name (e.g. 'github')",
+        },
+        group_by: {
+          type: "string",
+          enum: ["provider", "tag"],
+          description:
+            "Group results by 'provider' or by OpenAPI 'tag'. When omitted, returns a flat list of names.",
+        },
+      },
+    },
+  },
+  {
+    name: "search_tools",
+    description:
+      "Find tools by keyword. Searches tool names, OpenAPI tags, and descriptions using TF-IDF relevance ranking so the LLM can locate relevant tools without browsing the full catalog.",
+    inputSchema: {
+      type: "object" as const,
+      properties: {
+        query: {
+          type: "string",
+          description:
+            "Natural language or keyword search (all words must appear in name, tags, or description)",
+        },
+        provider: {
+          type: "string",
+          description: "Restrict search to tools from this provider name (e.g. 'github')",
+        },
+        limit: {
+          type: "number",
+          description: "Maximum number of results to return (default 10)",
+        },
+      },
+      required: ["query"],
+    },
+  },
+  {
+    name: "tool_info",
+    description:
+      "Get the full schema and metadata for a specific tool by name. Use this after list_tools or search_tools to retrieve the complete input schema before calling the tool.",
+    inputSchema: {
+      type: "object" as const,
+      properties: {
+        tool_name: {
+          type: "string",
+          description: "The MCP tool name (e.g. 'github__repos_list')",
+        },
+      },
+      required: ["tool_name"],
+    },
+  },
+  {
+    name: "call_tool",
+    description:
+      "Execute any registered tool by name with the provided arguments. Use tool_info first to get the correct argument schema.",
+    inputSchema: {
+      type: "object" as const,
+      properties: {
+        tool_name: {
+          type: "string",
+          description: "The MCP tool name to execute (e.g. 'github__repos_list')",
+        },
+        arguments: {
+          type: "object",
+          description: "Arguments to pass to the tool",
+        },
+      },
+      required: ["tool_name", "arguments"],
+    },
+  },
+];
+
+// ---------------------------------------------------------------------------
+// Meta-tool call handlers
+//
+// Each handler returns a fresh object literal (no explicit return annotation)
+// so the inferred type stays assignable to the MCP SDK's `ServerResult`
+// union, which includes an index-signature envelope that named interface types
+// cannot satisfy. `type: "text" as const` keeps the content block's `type`
+// literal so the result is still assignable to {@link CallToolResult}.
+// ---------------------------------------------------------------------------
+
+export function handleListTools(tools: ProviderTool[], args: Record<string, unknown>) {
+  const providerFilter =
+    typeof args["provider"] === "string" ? args["provider"] : undefined;
+  const groupBy =
+    args["group_by"] === "provider" || args["group_by"] === "tag"
+      ? (args["group_by"] as "provider" | "tag")
+      : undefined;
+  const filtered = providerFilter
+    ? tools.filter((t) => t.providerName === providerFilter)
+    : tools;
+  const result =
+    groupBy === "provider" || groupBy === "tag"
+      ? groupTools(filtered, groupBy)
+      : filtered.map((t) => t.mcpName);
+  return {
+    content: [{ type: "text" as const, text: JSON.stringify(result, null, 2) }],
+  };
+}
+
+export function handleSearchTools(tools: ProviderTool[], args: Record<string, unknown>) {
+  const query = typeof args["query"] === "string" ? args["query"] : "";
+  const providerFilter =
+    typeof args["provider"] === "string" ? args["provider"] : undefined;
+  const limit =
+    typeof args["limit"] === "number" && args["limit"] > 0 ? Math.floor(args["limit"]) : 10;
+  const results = searchTools(tools, query, providerFilter, limit);
+  return {
+    content: [{ type: "text" as const, text: JSON.stringify(results, null, 2) }],
+  };
+}
+
+export function handleToolInfo(
+  toolMap: Map<string, ProviderTool>,
+  args: Record<string, unknown>,
+) {
+  const requestedName = typeof args["tool_name"] === "string" ? args["tool_name"] : "";
+  const tool = toolMap.get(requestedName);
+  if (!tool) {
+    return {
+      isError: true,
+      content: [{ type: "text" as const, text: `Unknown tool: ${requestedName}` }],
+    };
+  }
+  return {
+    content: [
+      {
+        type: "text" as const,
+        text: JSON.stringify(
+          {
+            name: tool.mcpName,
+            description: tool.description,
+            provider: tool.providerName,
+            inputSchema: normalizeInputSchema(tool.inputSchema),
+          },
+          null,
+          2,
+        ),
+      },
+    ],
+  };
+}
+
+// ---------------------------------------------------------------------------
+// execute() contract for handleCallTool
+// ---------------------------------------------------------------------------
+
+/**
+ * Payload handed to an {@link Execute} callback: the resolved tool plus the
+ * arguments extracted from the `call_tool` meta-tool's `arguments` field.
+ */
+export interface ExecuteInput {
+  tool: ProviderTool;
+  args: Record<string, unknown>;
+}
+
+/**
+ * Transport-agnostic result of executing a tool.
+ *
+ * - `ok: true` carries the parsed tool output (`data`); mcp-core serializes it
+ *   as the `text` content of the {@link CallToolResult}.
+ * - `ok: false` carries an `error` message; mcp-core surfaces it as an
+ *   `isError` {@link CallToolResult}.
+ *
+ * The stdio server wraps its `fetch`-based executor (which throws on HTTP
+ * errors) into this discriminated union; the hosted gateway (Phase 2) maps its
+ * `IsolateResult` onto the same shape. Keeping errors as values (instead of
+ * throwing) lets both transports share a single handler.
+ */
+export type ExecuteResult = { ok: true; data: unknown } | { ok: false; error: string };
+
+/**
+ * Callback that performs the actual tool execution. Injected into
+ * {@link handleCallTool} so the handler stays transport-agnostic: the stdio
+ * server passes a `fetch`-based implementation, the gateway passes its own
+ * executor.
+ */
+export type Execute = (input: ExecuteInput) => Promise<ExecuteResult>;
+
+/**
+ * Handle a `call_tool` meta-tool invocation.
+ *
+ * `tool` is the *resolved* tool (the caller looks it up in its tool map, which
+ * for the gateway is the per-caller filtered catalog); `undefined` produces an
+ * "Unknown tool" result, matching the prior stdio behavior. The injected
+ * `execute` callback is invoked with `{ tool, args }` (the arguments extracted
+ * from `args["arguments"]`) and its {@link ExecuteResult} is wrapped in a
+ * {@link CallToolResult}.
+ */
+export async function handleCallTool(
+  tool: ProviderTool | undefined,
+  args: Record<string, unknown>,
+  execute: Execute,
+) {
+  const requestedName = typeof args["tool_name"] === "string" ? args["tool_name"] : "";
+  if (!tool) {
+    return {
+      isError: true,
+      content: [{ type: "text" as const, text: `Unknown tool: ${requestedName}` }],
+    };
+  }
+  const toolArgs =
+    typeof args["arguments"] === "object" && args["arguments"] !== null
+      ? (args["arguments"] as Record<string, unknown>)
+      : {};
+  const result = await execute({ tool, args: toolArgs });
+  if (!result.ok) {
+    return {
+      isError: true,
+      content: [
+        { type: "text" as const, text: `Error calling ${tool.mcpName}: ${result.error}` },
+      ],
+    };
+  }
+  const text =
+    typeof result.data === "string" ? result.data : JSON.stringify(result.data, null, 2);
+  return { content: [{ type: "text" as const, text }] };
+}

package/src/search.ts

@@ -0,0 +1,168 @@
+import type { ProviderTool } from "./loader.js";
+
+// ---------------------------------------------------------------------------
+// Tokenization
+// ---------------------------------------------------------------------------
+
+/**
+ * Tokenize text into lowercase words, splitting on non-alphanumeric characters.
+ * Also splits camelCase boundaries for better matching of MCP tool names.
+ *
+ * Examples:
+ *   "github__repos_list" → ["github", "repos", "list"]
+ *   "List public repositories" → ["list", "public", "repositories"]
+ *   "security-advisories" → ["security", "advisories"]
+ */
+export function tokenize(text: string): string[] {
+  return text
+    .replace(/([a-z])([A-Z])/g, "$1 $2") // split camelCase boundaries
+    .toLowerCase()
+    .split(/[^a-z0-9]+/)
+    .filter(Boolean);
+}
+
+// ---------------------------------------------------------------------------
+// TF-IDF scoring
+// ---------------------------------------------------------------------------
+
+/**
+ * Build a TF-IDF scoring function over a corpus of tools.
+ *
+ * Field weights applied during term-frequency calculation:
+ *   - name  × 3   (most signal)
+ *   - tags  × 2   (category signal)
+ *   - description × 1
+ *
+ * IDF formula: log((N + 1) / df)  — smoothed to avoid division-by-zero.
+ *
+ * Returns a function `score(tool, queryWords) → number` that can be called
+ * for each candidate. Build once per search call, reuse across candidates.
+ */
+export function buildTfIdf(
+  tools: ProviderTool[],
+): (tool: ProviderTool, words: string[]) => number {
+  const N = tools.length;
+  if (N === 0) return () => 0;
+
+  type TokenBag = Map<string, number>;
+
+  // Precompute weighted token bags for each tool
+  const bags: TokenBag[] = tools.map((tool) => {
+    const bag: TokenBag = new Map();
+    const add = (tokens: string[], weight: number) => {
+      for (const tok of tokens) {
+        bag.set(tok, (bag.get(tok) ?? 0) + weight);
+      }
+    };
+    add(tokenize(tool.mcpName), 3);
+    for (const tag of tool.tags) {
+      add(tokenize(tag), 2);
+    }
+    add(tokenize(tool.description), 1);
+    return bag;
+  });
+
+  // Document frequency: how many tools contain each token
+  const df = new Map<string, number>();
+  for (const bag of bags) {
+    for (const tok of bag.keys()) {
+      df.set(tok, (df.get(tok) ?? 0) + 1);
+    }
+  }
+
+  const bagIndex = new Map<ProviderTool, TokenBag>(tools.map((t, i) => [t, bags[i]!]));
+
+  return (tool: ProviderTool, words: string[]): number => {
+    const bag = bagIndex.get(tool);
+    if (!bag) return 0;
+    const totalWeight = [...bag.values()].reduce((s, v) => s + v, 0) || 1;
+    let score = 0;
+    for (const word of words) {
+      const tf = (bag.get(word) ?? 0) / totalWeight;
+      const docFreq = df.get(word) ?? 0;
+      if (docFreq === 0) continue;
+      const idf = Math.log((N + 1) / docFreq);
+      score += tf * idf;
+    }
+    return score;
+  };
+}
+
+// ---------------------------------------------------------------------------
+// searchTools
+// ---------------------------------------------------------------------------
+
+/**
+ * Search tools by keyword using TF-IDF relevance ranking.
+ *
+ * Matching: ALL query words must appear in at least one of name, tags, or
+ * description (case-insensitive substring match used for filtering; TF-IDF
+ * token scoring used for ranking).
+ *
+ * Returns up to `limit` results in descending relevance order.
+ */
+export function searchTools(
+  tools: ProviderTool[],
+  query: string,
+  provider: string | undefined,
+  limit: number,
+): Array<{ name: string; description: string; tags: string[] }> {
+  const words = tokenize(query);
+  if (words.length === 0) return [];
+
+  const source = provider ? tools.filter((t) => t.providerName === provider) : tools;
+  const scoreFn = buildTfIdf(source);
+
+  const candidates: Array<{ tool: ProviderTool; score: number }> = [];
+
+  for (const tool of source) {
+    const nameLower = tool.mcpName.toLowerCase();
+    const tagsText = tool.tags.join(" ").toLowerCase();
+    const descLower = tool.description.toLowerCase();
+
+    const allMatch = words.every(
+      (word) =>
+        nameLower.includes(word) || tagsText.includes(word) || descLower.includes(word),
+    );
+
+    if (allMatch) {
+      candidates.push({ tool, score: scoreFn(tool, words) });
+    }
+  }
+
+  candidates.sort((a, b) => b.score - a.score);
+
+  return candidates.slice(0, limit).map(({ tool }) => ({
+    name: tool.mcpName,
+    description: tool.description,
+    tags: tool.tags,
+  }));
+}
+
+// ---------------------------------------------------------------------------
+// groupTools
+// ---------------------------------------------------------------------------
+
+/**
+ * Group tools by provider name or by OpenAPI tag.
+ * Tools with no tags fall back to grouping under their provider name.
+ */
+export function groupTools(
+  tools: ProviderTool[],
+  by: "provider" | "tag",
+): Record<string, string[]> {
+  const grouped: Record<string, string[]> = {};
+
+  for (const tool of tools) {
+    if (by === "provider") {
+      (grouped[tool.providerName] ??= []).push(tool.mcpName);
+    } else {
+      const keys = tool.tags.length > 0 ? tool.tags : [tool.providerName];
+      for (const key of keys) {
+        (grouped[key] ??= []).push(tool.mcpName);
+      }
+    }
+  }
+
+  return grouped;
+}

package/tsconfig.json

@@ -0,0 +1,13 @@
+{
+  "extends": "../../tsconfig.json",
+  "compilerOptions": {
+    "moduleResolution": "Bundler",
+    "outDir": "dist",
+    "rootDir": "src",
+    "declaration": true,
+    "sourceMap": true,
+    "resolveJsonModule": true
+  },
+  "include": ["src/**/*.ts"],
+  "exclude": ["node_modules", "dist", "src/**/*.test.ts"]
+}

package/vitest.config.ts

@@ -0,0 +1,8 @@
+import { defineConfig } from "@aprovan/vitest-config";
+
+export default {
+  test: defineConfig("node", {
+    include: ["src/**/*.test.ts"],
+    testTimeout: 30000,
+  }),
+};