@semalt-ai/code 1.8.4 → 1.19.0

package/lib/mcp/boundary.js

@@ -0,0 +1,131 @@
+'use strict';
+
+// MCP boundary module (Task 3.2)
+// -----------------------------------------------------------------------------
+// The official MCP SDK (`@modelcontextprotocol/sdk`) is the project's first — and
+// so far only — runtime dependency. It is **ESM-only** (`"type": "module"`, no
+// CommonJS entry points), but this project is CommonJS: a CJS module cannot
+// `require()` an ESM-only package.
+//
+// This module is the SINGLE place that bridges the gap. It loads the SDK via
+// dynamic `import()` (the one mechanism CJS has for pulling in ESM) and re-exposes
+// a small, CommonJS-friendly **async** surface. Every other file in the codebase
+// stays plain CommonJS and talks to MCP through this boundary — no other module
+// imports the SDK directly. That keeps the ESM/CJS friction contained to one file
+// and means the rest of the project never has to change its module system.
+//
+// The dynamic import is memoized: the SDK's ESM graph is evaluated at most once
+// per process, on first use (lazy — importing this module costs nothing until a
+// boundary function is actually called). Task 3.3 builds the MCP client on top of
+// the helpers here.
+
+const { PACKAGE_JSON } = require('../constants');
+
+// The SDK subpaths we consume. The SDK exposes deep subpath exports
+// (`./client/index.js`, `./client/stdio.js`, …) rather than a single barrel, so
+// we import exactly what we use.
+const CLIENT_SUBPATH = '@modelcontextprotocol/sdk/client/index.js';
+const STDIO_SUBPATH = '@modelcontextprotocol/sdk/client/stdio.js';
+const HTTP_SUBPATH = '@modelcontextprotocol/sdk/client/streamableHttp.js';
+const SSE_SUBPATH = '@modelcontextprotocol/sdk/client/sse.js';
+
+let _sdkPromise = null;
+
+// Lazily load + memoize the SDK's client surface. `import()` is the ONLY bridge
+// from this CommonJS module to the ESM-only package — do not try to `require()`
+// the SDK anywhere. Returns just the named exports the rest of the code needs.
+async function loadSdk() {
+  if (!_sdkPromise) {
+    _sdkPromise = (async () => {
+      const [clientMod, stdioMod] = await Promise.all([
+        import(CLIENT_SUBPATH),
+        import(STDIO_SUBPATH),
+      ]);
+      return {
+        Client: clientMod.Client,
+        StdioClientTransport: stdioMod.StdioClientTransport,
+      };
+    })();
+    // If the import rejects (e.g. the dependency is not installed), clear the
+    // cache so a later call can retry rather than re-throwing a stale rejection.
+    _sdkPromise.catch(() => { _sdkPromise = null; });
+  }
+  return _sdkPromise;
+}
+
+// Whether the SDK is resolvable in this environment (installed in node_modules).
+// Synchronous and side-effect-free: used by the smoke test to skip gracefully
+// when the dependency could not be installed (e.g. an offline CI runner) instead
+// of failing the suite.
+function isSdkAvailable() {
+  try {
+    require.resolve(CLIENT_SUBPATH);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+// Default identity advertised to MCP servers during the connect handshake.
+const DEFAULT_CLIENT_INFO = Object.freeze({
+  name: PACKAGE_JSON.name,
+  version: PACKAGE_JSON.version,
+});
+
+// Instantiate an MCP `Client`. Does NOT connect — Task 3.3 owns transport wiring
+// and the `connect()` handshake. `clientInfo` defaults to this CLI's identity;
+// `options` defaults to declaring no client capabilities.
+async function createClient(clientInfo, options) {
+  const { Client } = await loadSdk();
+  return new Client(
+    clientInfo || { ...DEFAULT_CLIENT_INFO },
+    options || { capabilities: {} },
+  );
+}
+
+// Construct a stdio transport for launching a local MCP server subprocess.
+// `params` is the SDK's `StdioServerParameters` ({ command, args, env, … }).
+// When the caller supplies `env`, it is MERGED over the SDK's default safe
+// environment (getDefaultEnvironment) rather than replacing it — otherwise the
+// child would lose PATH/HOME and fail to launch most real servers.
+async function createStdioTransport(params) {
+  const stdioMod = await import(STDIO_SUBPATH);
+  const { StdioClientTransport, getDefaultEnvironment } = stdioMod;
+  const merged = { ...params };
+  if (params && params.env && typeof getDefaultEnvironment === 'function') {
+    merged.env = { ...getDefaultEnvironment(), ...params.env };
+  }
+  return new StdioClientTransport(merged);
+}
+
+// Construct a Streamable-HTTP transport for a remote MCP server. `url` is a URL
+// (or string); `opts` is the SDK's `StreamableHTTPClientTransportOptions`
+// ({ authProvider, requestInit, … }). OAuth is wired through `opts.authProvider`.
+async function createStreamableHttpTransport(url, opts) {
+  const mod = await import(HTTP_SUBPATH);
+  return new mod.StreamableHTTPClientTransport(url instanceof URL ? url : new URL(url), opts);
+}
+
+// Construct a legacy HTTP+SSE transport for a remote MCP server. Same shape as
+// the streamable-HTTP transport; used for servers that only speak the older
+// SSE protocol.
+async function createSseTransport(url, opts) {
+  const mod = await import(SSE_SUBPATH);
+  return new mod.SSEClientTransport(url instanceof URL ? url : new URL(url), opts);
+}
+
+// Test seam: drop the memoized import so a fresh load can be exercised.
+function _reset() {
+  _sdkPromise = null;
+}
+
+module.exports = {
+  loadSdk,
+  isSdkAvailable,
+  createClient,
+  createStdioTransport,
+  createStreamableHttpTransport,
+  createSseTransport,
+  DEFAULT_CLIENT_INFO,
+  _reset,
+};

package/lib/mcp/client.js

@@ -0,0 +1,270 @@
+'use strict';
+
+// MCP client manager (Task 3.3).
+// ----------------------------------------------------------------------------
+// Connects to the MCP servers configured under `config.mcp.servers`, discovers
+// each server's tools, and registers them into the runtime tool registry
+// (lib/tool_registry.js dynamic API) under the namespace `mcp__<server>__<tool>`
+// so they dispatch through the SAME agent loop as built-ins.
+//
+// Security posture (required by the task):
+//   * MCP tool RESULTS are untrusted external content — the execute() here
+//     returns `{ mcp:true, content, isError }`, and the agent loop wraps that
+//     payload in the UNTRUSTED_EXTERNAL_CONTENT delimiter (lib/agent.js),
+//     identical to http_get.
+//   * MCP tools are arbitrary/external, so they REQUIRE APPROVAL by default —
+//     never auto-allowed by the `--allow-*` tiers. Per-server/per-tool opt-in
+//     comes from config (`allow: [...]` or `allowAll: true`).
+//
+// Robustness: a server that fails to launch/connect degrades gracefully (the
+// failure is recorded in status, a warning is logged, and the CLI continues) —
+// it never crashes the process. The SDK itself is reached only through
+// lib/mcp/boundary.js (the single CJS↔ESM bridge); this module never imports it.
+
+const realBoundary = require('./boundary');
+const {
+  registerDynamicTool, unregisterDynamicTool,
+} = require('../tool_registry');
+const { logToolCall } = require('../audit');
+const { createKeychainOAuthProvider } = require('./oauth');
+
+const realRegistry = { registerDynamicTool, unregisterDynamicTool };
+
+const DEFAULT_CONNECT_TIMEOUT_MS = 15000;
+
+// `mcp__<server>__<tool>` with non-identifier chars folded to `_` so the result
+// is a valid native function name (the LLM echoes it back verbatim in tool_calls).
+function mcpToolName(server, tool) {
+  const s = String(server).replace(/[^a-zA-Z0-9_-]/g, '_');
+  const t = String(tool).replace(/[^a-zA-Z0-9_-]/g, '_');
+  return `mcp__${s}__${t}`;
+}
+
+// Flatten an MCP CallToolResult's content blocks into a single text string.
+// Text blocks pass through; non-text blocks (image/resource/…) are summarized
+// as a JSON line so nothing is silently dropped from the model's view.
+function mcpResultToText(result) {
+  if (!result) return '';
+  const blocks = Array.isArray(result.content) ? result.content : [];
+  const parts = [];
+  for (const b of blocks) {
+    if (b && b.type === 'text' && typeof b.text === 'string') parts.push(b.text);
+    else if (b && typeof b === 'object') parts.push(`[${b.type || 'content'}] ${JSON.stringify(b)}`);
+  }
+  if (!parts.length && result.structuredContent !== undefined) {
+    parts.push(JSON.stringify(result.structuredContent));
+  }
+  return parts.join('\n');
+}
+
+// Decide whether a discovered tool is pre-approved (opt-in) for this server.
+// Matches either the bare tool name or its full namespaced form in `allow`.
+function isToolAllowed(spec, toolName, namespacedName) {
+  if (!spec) return false;
+  if (spec.allowAll === true) return true;
+  const allow = Array.isArray(spec.allow) ? spec.allow : [];
+  return allow.includes(toolName) || allow.includes(namespacedName);
+}
+
+// Build the dynamic tool-registry entry for one discovered MCP tool. Shape is
+// identical to a static entry so the agent loop dispatches it the same way.
+function buildMcpToolEntry({ server, spec, tool, client }) {
+  const name = mcpToolName(server, tool.name);
+  const allowed = isToolAllowed(spec, tool.name, name);
+  const description = `[MCP:${server}] ${tool.description || tool.name}`;
+
+  return {
+    tool: name,
+    mcp: true,
+    server,
+    origName: tool.name,
+    spec: {
+      description,
+      parameters: tool.inputSchema && typeof tool.inputSchema === 'object'
+        ? tool.inputSchema
+        : { type: 'object', properties: {} },
+    },
+    // Native function-calling path: the model emits { name, arguments } → the
+    // whole arguments object becomes the single positional arg of the tuple.
+    fromParams: (p) => [name, p || {}],
+    // XML path (non-native models): <mcp__server__tool>{json args}</mcp__server__tool>.
+    parseXml: (text) => {
+      const out = [];
+      const re = new RegExp(`<${name}\\s*>([\\s\\S]*?)<\\/${name}>`, 'g');
+      for (const m of text.matchAll(re)) {
+        const body = (m[1] || '').trim();
+        let args = {};
+        if (body) { try { args = JSON.parse(body); } catch { args = {}; } }
+        out.push([name, args]);
+      }
+      // Self-closing form with no args: <mcp__server__tool/>
+      const selfRe = new RegExp(`<${name}\\s*/>`, 'g');
+      for (const _m of text.matchAll(selfRe)) out.push([name, {}]);
+      return out;
+    },
+    // Approval gate: MCP tools require approval by DEFAULT. Opt-in via config
+    // (allow/allowAll) returns null → no gate (treated like a read-only tool).
+    permission: () => {
+      if (allowed) return null;
+      return { actionType: 'mcp', description: `MCP ${server}/${tool.name}`, tag: name };
+    },
+    execute: async (_ctx, args, options) => {
+      const params = (args && args[0]) || {};
+      const signal = (options && options.signal) || undefined;
+      try {
+        const res = await client.callTool(
+          { name: tool.name, arguments: params },
+          undefined,
+          signal ? { signal } : undefined,
+        );
+        logToolCall(name, { server, tool: tool.name }, true, res && res.isError ? 'error' : 'ok');
+        return { mcp: true, content: mcpResultToText(res), isError: !!(res && res.isError) };
+      } catch (err) {
+        logToolCall(name, { server, tool: tool.name }, true, 'error');
+        return { error: err && err.message ? err.message : String(err) };
+      }
+    },
+  };
+}
+
+function withTimeout(promise, ms, onTimeoutMessage) {
+  let timer;
+  const timeout = new Promise((_, reject) => {
+    timer = setTimeout(() => reject(new Error(onTimeoutMessage)), ms);
+  });
+  return Promise.race([promise, timeout]).finally(() => clearTimeout(timer));
+}
+
+function createMcpManager({
+  getConfig,
+  boundary = realBoundary,
+  registry = realRegistry,
+  oauthFactory = createKeychainOAuthProvider,
+  logger = null,
+  connectTimeoutMs = DEFAULT_CONNECT_TIMEOUT_MS,
+} = {}) {
+  const _clients = new Map();   // server name → connected Client
+  const _toolNames = [];        // registered dynamic tool names
+  let _status = [];             // per-server status records
+
+  function warn(msg) {
+    if (typeof logger === 'function') logger(msg);
+  }
+
+  // Resolve the transport for a server spec. Throws on an invalid/missing spec.
+  async function buildTransport(name, spec) {
+    const transport = (spec.transport || (spec.url ? 'http' : 'stdio')).toLowerCase();
+    if (transport === 'stdio') {
+      if (!spec.command) throw new Error(`stdio server "${name}" requires a "command"`);
+      return boundary.createStdioTransport({
+        command: spec.command,
+        args: Array.isArray(spec.args) ? spec.args : [],
+        env: spec.env && typeof spec.env === 'object' ? spec.env : undefined,
+        cwd: spec.cwd || undefined,
+      });
+    }
+    if (transport === 'http' || transport === 'streamable-http' || transport === 'sse') {
+      if (!spec.url) throw new Error(`remote server "${name}" requires a "url"`);
+      const opts = {};
+      if (spec.headers && typeof spec.headers === 'object') {
+        opts.requestInit = { headers: spec.headers };
+      }
+      // OAuth: opt-in via spec.oauth (or spec.auth === 'oauth'). Tokens are
+      // persisted in the OS keychain by the provider, never in config.
+      if (spec.oauth === true || spec.auth === 'oauth') {
+        opts.authProvider = oauthFactory(name, { url: spec.url });
+      }
+      return transport === 'sse'
+        ? boundary.createSseTransport(spec.url, opts)
+        : boundary.createStreamableHttpTransport(spec.url, opts);
+    }
+    throw new Error(`server "${name}" has unknown transport "${transport}"`);
+  }
+
+  async function connectServer(name, spec) {
+    const transportKind = (spec.transport || (spec.url ? 'http' : 'stdio')).toLowerCase();
+    const record = { name, transport: transportKind, state: 'connecting', tools: [], error: null };
+    if (spec.disabled) {
+      record.state = 'disabled';
+      return record;
+    }
+    let client = null;
+    try {
+      const transport = await buildTransport(name, spec);
+      client = await boundary.createClient();
+      await withTimeout(
+        client.connect(transport),
+        connectTimeoutMs,
+        `connect timed out after ${connectTimeoutMs}ms`,
+      );
+      const listed = await client.listTools();
+      const tools = Array.isArray(listed && listed.tools) ? listed.tools : [];
+      for (const tool of tools) {
+        if (!tool || typeof tool.name !== 'string') continue;
+        const entry = buildMcpToolEntry({ server: name, spec, tool, client });
+        registry.registerDynamicTool(entry);
+        record.tools.push(entry.tool);
+        _toolNames.push(entry.tool);
+      }
+      record.state = 'connected';
+      _clients.set(name, client);
+    } catch (err) {
+      record.state = 'failed';
+      record.error = err && err.message ? err.message : String(err);
+      warn(`MCP server "${name}" failed: ${record.error}`);
+      // Best-effort cleanup of a half-open client so a failed server leaks nothing.
+      if (client) { try { await client.close(); } catch { /* ignore */ } }
+    }
+    return record;
+  }
+
+  // Connect to every configured server. Failures are isolated per-server (one
+  // bad server never blocks the others, and never throws out of here).
+  async function connectAll() {
+    const cfg = getConfig ? getConfig() : {};
+    const servers = (cfg && cfg.mcp && cfg.mcp.servers) || {};
+    const names = Object.keys(servers);
+    const records = [];
+    for (const name of names) {
+      records.push(await connectServer(name, servers[name] || {}));
+    }
+    _status = records;
+    return records;
+  }
+
+  function status() {
+    return _status.map((r) => ({ ...r, tools: r.tools.slice() }));
+  }
+
+  function registeredToolNames() {
+    return _toolNames.slice();
+  }
+
+  async function shutdown() {
+    for (const name of _toolNames) {
+      try { registry.unregisterDynamicTool(name); } catch { /* ignore */ }
+    }
+    _toolNames.length = 0;
+    for (const client of _clients.values()) {
+      try { await client.close(); } catch { /* ignore */ }
+    }
+    _clients.clear();
+    _status = [];
+  }
+
+  return {
+    connectAll,
+    connectServer,
+    status,
+    registeredToolNames,
+    shutdown,
+  };
+}
+
+module.exports = {
+  createMcpManager,
+  buildMcpToolEntry,
+  mcpToolName,
+  mcpResultToText,
+  isToolAllowed,
+};

package/lib/mcp/oauth.js

@@ -0,0 +1,134 @@
+'use strict';
+
+// MCP OAuth — keychain-backed token store (Task 3.3).
+// ----------------------------------------------------------------------------
+// Remote MCP servers (HTTP/SSE) may require OAuth. The SDK drives the OAuth 2.1
+// + PKCE flow through an `OAuthClientProvider` interface; this module implements
+// that interface and persists EVERYTHING sensitive — tokens, the dynamically
+// registered client credentials, and the PKCE code verifier — in the OS
+// keychain, never in plaintext config. That mirrors the Phase 0 secret path
+// (lib/secrets.js): secrets live in the keychain, config holds only references.
+//
+// The keychain access is injected (`store`) so it can be unit-tested with an
+// in-memory fake; in production it defaults to the generic keychain helpers in
+// lib/secrets.js under the service `semalt-code-mcp`, keyed per server.
+//
+// Records are JSON blobs stored under three accounts per server:
+//   <server>:tokens    — the OAuthTokens (access/refresh/expiry)
+//   <server>:client    — the registered OAuthClientInformation
+//   <server>:verifier  — the in-flight PKCE code verifier
+//
+// `redirectToAuthorization` opens the user's browser (best-effort) and prints
+// the URL so headless/remote sessions can complete the flow manually.
+
+const { spawn } = require('child_process');
+const {
+  keychainGetItem, keychainSetItem, keychainDeleteItem,
+} = require('../secrets');
+
+const MCP_KEYCHAIN_SERVICE = 'semalt-code-mcp';
+
+// Default production store: the OS keychain via lib/secrets.js generic helpers.
+function keychainStore(service = MCP_KEYCHAIN_SERVICE) {
+  return {
+    get(account) { return keychainGetItem(service, account); },
+    set(account, value) { return keychainSetItem(service, account, value, `MCP OAuth ${account}`); },
+    delete(account) { return keychainDeleteItem(service, account); },
+  };
+}
+
+// Best-effort browser opener — same idea the device-login flow uses. Never
+// throws; if no opener is available the URL is just printed for manual use.
+function _openBrowser(url) {
+  const platform = process.platform;
+  let cmd; let args;
+  if (platform === 'darwin') { cmd = 'open'; args = [url]; }
+  else if (platform === 'win32') { cmd = 'cmd'; args = ['/c', 'start', '', url]; }
+  else { cmd = 'xdg-open'; args = [url]; }
+  try {
+    const child = spawn(cmd, args, { stdio: 'ignore', detached: true });
+    child.on('error', () => {});
+    child.unref();
+  } catch { /* ignore — URL is printed below */ }
+}
+
+function _parse(raw) {
+  if (!raw) return undefined;
+  try { return JSON.parse(raw); } catch { return undefined; }
+}
+
+// Build a keychain-backed OAuthClientProvider for one server.
+//   server      — config key, used to namespace keychain accounts.
+//   url         — the server URL (origin used as redirect base).
+//   store       — injectable { get, set, delete } (defaults to OS keychain).
+//   onRedirect  — optional callback(url) instead of opening a browser (tests).
+function createKeychainOAuthProvider(server, {
+  url = '',
+  store = keychainStore(),
+  redirectUrl = 'http://127.0.0.1:8976/callback',
+  clientName = '@semalt-ai/code',
+  onRedirect = null,
+} = {}) {
+  const acct = (kind) => `${server}:${kind}`;
+
+  return {
+    get redirectUrl() { return redirectUrl; },
+
+    get clientMetadata() {
+      return {
+        client_name: clientName,
+        redirect_uris: [redirectUrl],
+        grant_types: ['authorization_code', 'refresh_token'],
+        response_types: ['code'],
+        token_endpoint_auth_method: 'none',
+      };
+    },
+
+    clientInformation() {
+      return _parse(store.get(acct('client')));
+    },
+    saveClientInformation(info) {
+      store.set(acct('client'), JSON.stringify(info));
+    },
+
+    tokens() {
+      return _parse(store.get(acct('tokens')));
+    },
+    saveTokens(tokens) {
+      store.set(acct('tokens'), JSON.stringify(tokens));
+    },
+
+    saveCodeVerifier(verifier) {
+      store.set(acct('verifier'), String(verifier));
+    },
+    codeVerifier() {
+      const v = store.get(acct('verifier'));
+      if (!v) throw new Error('No PKCE code verifier saved for this MCP server');
+      return v;
+    },
+
+    redirectToAuthorization(authorizationUrl) {
+      const href = authorizationUrl instanceof URL ? authorizationUrl.href : String(authorizationUrl);
+      if (typeof onRedirect === 'function') { onRedirect(href); return; }
+      // audit: allowed — pre/non-UI OAuth flow prompt; the user must visit this URL.
+      process.stderr.write(`\nOpen this URL to authorize the MCP server "${server}":\n${href}\n\n`);
+      _openBrowser(href);
+    },
+  };
+}
+
+// Forget all stored OAuth material for a server (used by `mcp remove`/re-auth).
+function clearOAuth(server, store = keychainStore()) {
+  let ok = true;
+  for (const kind of ['tokens', 'client', 'verifier']) {
+    if (!store.delete(`${server}:${kind}`)) ok = false;
+  }
+  return ok;
+}
+
+module.exports = {
+  MCP_KEYCHAIN_SERVICE,
+  keychainStore,
+  createKeychainOAuthProvider,
+  clearOAuth,
+};