@mhingston5/conduit 1.1.4 → 1.1.6

package/src/gateway/gateway.service.ts

@@ -1,4 +1,6 @@
 import { Logger } from 'pino';
+import { HostClient } from './host.client.js';
+import { StdioTransport } from '../transport/stdio.transport.js';
 import { UpstreamClient, UpstreamInfo } from './upstream.client.js';
 import { AuthService } from './auth.service.js';
 import { SchemaCache, ToolSchema } from './schema.cache.js';
@@ -13,7 +15,7 @@ import addFormats from 'ajv-formats';
 const BUILT_IN_TOOLS: ToolSchema[] = [
     {
         name: 'mcp_execute_typescript',
-        description: 'Executes TypeScript code in a secure sandbox with access to `tools.*` SDK.',
+        description: 'Executes TypeScript code in a secure sandbox. Access MCP tools via the global `tools` object (e.g. `filesystem__list_directory` -> `await tools.filesystem.list_directory(...)`).',
         inputSchema: {
             type: 'object',
             properties: {
@@ -24,7 +26,7 @@ const BUILT_IN_TOOLS: ToolSchema[] = [
                 allowedTools: {
                     type: 'array',
                     items: { type: 'string' },
-                    description: 'Optional list of tools the script is allowed to call (e.g. ["github.*"]).'
+                    description: 'List of tool names (e.g. "filesystem.list_directory" or "filesystem.*") that the script is allowed to call.'
                 }
             },
             required: ['code']
@@ -32,7 +34,7 @@ const BUILT_IN_TOOLS: ToolSchema[] = [
     },
     {
         name: 'mcp_execute_python',
-        description: 'Executes Python code in a secure sandbox with access to `tools.*` SDK.',
+        description: 'Executes Python code in a secure sandbox. Access MCP tools via the global `tools` object (e.g. `filesystem__list_directory` -> `await tools.filesystem.list_directory(...)`).',
         inputSchema: {
             type: 'object',
             properties: {
@@ -43,7 +45,7 @@ const BUILT_IN_TOOLS: ToolSchema[] = [
                 allowedTools: {
                     type: 'array',
                     items: { type: 'string' },
-                    description: 'Optional list of tools the script is allowed to call (e.g. ["github.*"]).'
+                    description: 'List of tool names (e.g. "filesystem.list_directory" or "filesystem.*") that the script is allowed to call.'
                 }
             },
             required: ['code']
@@ -51,7 +53,7 @@ const BUILT_IN_TOOLS: ToolSchema[] = [
     },
     {
         name: 'mcp_execute_isolate',
-        description: 'Executes JavaScript code in a high-speed V8 isolate (no Deno/Node APIs).',
+        description: 'Executes JavaScript code in a high-speed V8 isolate. Access MCP tools via the global `tools` object (e.g. `await tools.filesystem.list_directory(...)`). No Deno/Node APIs. Use `console.log` for output.',
         inputSchema: {
             type: 'object',
             properties: {
@@ -62,7 +64,7 @@ const BUILT_IN_TOOLS: ToolSchema[] = [
                 allowedTools: {
                     type: 'array',
                     items: { type: 'string' },
-                    description: 'Optional list of tools the script is allowed to call.'
+                    description: 'List of tool names (e.g. "filesystem.list_directory" or "filesystem.*") that the script is allowed to call.'
                 }
             },
             required: ['code']
@@ -72,41 +74,69 @@ const BUILT_IN_TOOLS: ToolSchema[] = [
 
 export class GatewayService {
     private logger: Logger;
-    private clients: Map<string, UpstreamClient> = new Map();
+    private clients: Map<string, any> = new Map();
     private authService: AuthService;
     private schemaCache: SchemaCache;
     private urlValidator: IUrlValidator;
-    private policyService: PolicyService;
+    public policyService: PolicyService;
     private ajv: Ajv;
     // Cache compiled validators to avoid recompilation on every call
     private validatorCache = new Map<string, any>();
 
     constructor(logger: Logger, urlValidator: IUrlValidator, policyService?: PolicyService) {
-        this.logger = logger;
+        this.logger = logger.child({ component: 'GatewayService' });
+        this.logger.debug('GatewayService instance created');
         this.urlValidator = urlValidator;
         this.authService = new AuthService(logger);
         this.schemaCache = new SchemaCache(logger);
         this.policyService = policyService ?? new PolicyService();
-        this.ajv = new Ajv({ strict: false }); // Strict mode off for now to be permissive with upstream schemas
-        // eslint-disable-next-line @typescript-eslint/no-var-requires
+        this.ajv = new Ajv({ strict: false });
         (addFormats as any).default(this.ajv);
     }
 
     registerUpstream(info: UpstreamInfo) {
         const client = new UpstreamClient(this.logger, info, this.authService, this.urlValidator);
         this.clients.set(info.id, client);
-        this.logger.info({ upstreamId: info.id }, 'Registered upstream MCP');
+        this.logger.info({ upstreamId: info.id, totalRegistered: this.clients.size }, 'Registered upstream MCP');
+    }
+
+    registerHost(transport: StdioTransport) {
+        // NOTE: The host (VS Code) cannot receive tools/call requests - it's the CLIENT.
+        // We only register it for potential future use (e.g., sampling requests).
+        // DO NOT use the host as a tool provider fallback.
+        this.logger.debug('Host transport available but not registered as tool upstream (protocol limitation)');
     }
 
     async listToolPackages(): Promise<ToolPackage[]> {
-        return Array.from(this.clients.entries()).map(([id, client]) => ({
+        const upstreams = Array.from(this.clients.entries()).map(([id, client]) => ({
             id,
-            description: `Upstream ${id}`, // NOTE: Upstream description fetching deferred to V2
+            description: `Upstream ${id}`,
             version: '1.0.0'
         }));
+
+        return [
+            { id: 'conduit', description: 'Conduit built-in execution tools', version: '1.0.0' },
+            ...upstreams
+        ];
+    }
+
+    getBuiltInTools(): ToolSchema[] {
+        return BUILT_IN_TOOLS;
     }
 
     async listToolStubs(packageId: string, context: ExecutionContext): Promise<ToolStub[]> {
+        if (packageId === 'conduit') {
+            const stubs = BUILT_IN_TOOLS.map(t => ({
+                id: `conduit__${t.name}`,
+                name: t.name,
+                description: t.description
+            }));
+            if (context.allowedTools) {
+                return stubs.filter(t => this.policyService.isToolAllowed(t.id, context.allowedTools!));
+            }
+            return stubs;
+        }
+
         const client = this.clients.get(packageId);
         if (!client) {
             throw new Error(`Upstream package not found: ${packageId}`);
@@ -117,40 +147,40 @@ export class GatewayService {
         // Try manifest first if tools not cached
         if (!tools) {
             try {
-                // Try to fetch manifest first
+                // Try to get manifest FIRST
                 const manifest = await client.getManifest(context);
-                if (manifest) {
-                    const stubs: ToolStub[] = manifest.tools.map((t: any) => ({
-                        id: `${packageId}__${t.name}`,
-                        name: t.name,
-                        description: t.description
-                    }));
-
-                    if (context.allowedTools) {
-                        return stubs.filter(t => this.policyService.isToolAllowed(t.id, context.allowedTools!));
+                if (manifest && manifest.tools) {
+                    tools = manifest.tools as ToolSchema[];
+                } else {
+                    // Fall back to RPC discovery
+                    if (typeof (client as any).listTools === 'function') {
+                        tools = await (client as any).listTools();
+                    } else {
+                        const response = await client.call({
+                            jsonrpc: '2.0',
+                            id: 'discovery',
+                            method: 'tools/list',
+                        }, context);
+
+                        if (response.result?.tools) {
+                            tools = response.result.tools as ToolSchema[];
+                        } else {
+                            this.logger.warn({ upstreamId: packageId, error: response.error }, 'Failed to discover tools via RPC');
+                        }
                     }
-                    return stubs;
                 }
-            } catch (e) {
-                // Manifest fetch failed, fall back
-                this.logger.debug({ packageId, err: e }, 'Manifest fetch failed, falling back to RPC');
-            }
 
-            const response = await client.call({
-                jsonrpc: '2.0',
-                id: 'discovery',
-                method: 'list_tools',
-            }, context);
-
-            if (response.result?.tools) {
-                tools = response.result.tools as ToolSchema[];
-                this.schemaCache.set(packageId, tools);
-            } else {
-                this.logger.warn({ upstreamId: packageId, error: response.error }, 'Failed to discover tools from upstream');
-                tools = [];
+                if (tools && tools.length > 0) {
+                    this.schemaCache.set(packageId, tools);
+                    this.logger.info({ upstreamId: packageId, toolCount: tools.length }, 'Discovered tools from upstream');
+                }
+            } catch (e: any) {
+                this.logger.error({ upstreamId: packageId, err: e.message }, 'Error during tool discovery');
             }
         }
 
+        if (!tools) tools = [];
+
         const stubs: ToolStub[] = tools.map(t => ({
             id: `${packageId}__${t.name}`,
             name: t.name,
@@ -170,17 +200,29 @@ export class GatewayService {
         }
 
         const parsed = this.policyService.parseToolName(toolId);
-        const toolName = parsed.name; // Use a new variable for the un-namespaced name
-
-        // Check for built-in tools
-        const builtIn = BUILT_IN_TOOLS.find(t => t.name === toolId); // Compare with the full toolId
-        if (builtIn) return builtIn;
+        const namespace = parsed.namespace;
+        const toolName = parsed.name;
+
+        // Check for built-in tools (now namespaced as conduit__*)
+        if (namespace === 'conduit' || namespace === '') {
+            const builtIn = BUILT_IN_TOOLS.find(t => t.name === toolName);
+            if (builtIn) {
+                return { ...builtIn, name: `conduit__${builtIn.name}` };
+            }
+        }
 
-        const upstreamId = parsed.namespace;
+        const upstreamId = namespace;
+        if (!upstreamId) {
+            // Un-namespaced tool lookup: try all upstreams
+            for (const id of this.clients.keys()) {
+                const schema = await this.getToolSchema(`${id}__${toolName}`, context);
+                if (schema) return schema;
+            }
+            return null;
+        }
 
         // Ensure we have schemas for this upstream
         if (!this.schemaCache.get(upstreamId)) {
-            // Force refresh if missing
             await this.listToolStubs(upstreamId, context);
         }
 
@@ -189,7 +231,6 @@ export class GatewayService {
 
         if (!tool) return null;
 
-        // Return schema with namespaced name
         return {
             ...tool,
             name: toolId
@@ -197,41 +238,48 @@ export class GatewayService {
     }
 
     async discoverTools(context: ExecutionContext): Promise<ToolSchema[]> {
-        const allTools: ToolSchema[] = [...BUILT_IN_TOOLS];
+        const allTools: ToolSchema[] = BUILT_IN_TOOLS.map(t => ({
+            ...t,
+            name: `conduit__${t.name}`
+        }));
+
+        this.logger.debug({ clientCount: this.clients.size, clientIds: Array.from(this.clients.keys()) }, 'Starting tool discovery');
 
         for (const [id, client] of this.clients.entries()) {
-            let tools = this.schemaCache.get(id);
+            // Skip host - it's not a tool provider
+            if (id === 'host') {
+                continue;
+            }
 
-            if (!tools) {
-                const response = await client.call({
-                    jsonrpc: '2.0',
-                    id: 'discovery',
-                    method: 'list_tools', // Standard MCP method
-                }, context);
+            this.logger.debug({ upstreamId: id }, 'Discovering tools from upstream');
 
-                if (response.result?.tools) {
-                    tools = response.result.tools as ToolSchema[];
-                    this.schemaCache.set(id, tools);
-                } else {
-                    this.logger.warn({ upstreamId: id, error: response.error }, 'Failed to discover tools from upstream');
-                    tools = [];
-                }
+            // reuse unified discovery logic
+            try {
+                await this.listToolStubs(id, context);
+            } catch (e: any) {
+                this.logger.error({ upstreamId: id, err: e.message }, 'Failed to list tool stubs');
             }
+            const tools = this.schemaCache.get(id) || [];
 
-            const prefixedTools = tools.map(t => ({ ...t, name: `${id}__${t.name}` }));
+            this.logger.debug({ upstreamId: id, toolCount: tools.length }, 'Discovery result');
 
-            if (context.allowedTools) {
-                // Support wildcard patterns: "mock.*" matches "mock__hello"
-                allTools.push(...prefixedTools.filter(t => this.policyService.isToolAllowed(t.name, context.allowedTools!)));
-            } else {
-                allTools.push(...prefixedTools);
+            if (tools && tools.length > 0) {
+                const prefixedTools = tools.map(t => ({ ...t, name: `${id}__${t.name}` }));
+                if (context.allowedTools) {
+                    allTools.push(...prefixedTools.filter(t => this.policyService.isToolAllowed(t.name, context.allowedTools!)));
+                } else {
+                    allTools.push(...prefixedTools);
+                }
             }
         }
 
+        this.logger.info({ totalTools: allTools.length }, 'Tool discovery complete');
         return allTools;
     }
 
     async callTool(name: string, params: any, context: ExecutionContext): Promise<JSONRPCResponse> {
+        this.logger.debug({ name, upstreamCount: this.clients.size }, 'GatewayService.callTool called');
+
         if (context.allowedTools && !this.policyService.isToolAllowed(name, context.allowedTools)) {
             this.logger.warn({ name, allowedTools: context.allowedTools }, 'Tool call blocked by allowlist');
             return {
@@ -248,14 +296,43 @@ export class GatewayService {
         const upstreamId = toolId.namespace;
         const toolName = toolId.name;
 
+        this.logger.debug({ name, upstreamId, toolName }, 'Parsed tool name');
+
+        // Fallback for namespaceless calls: try to find the tool in any registered upstream
+        if (!upstreamId) {
+            this.logger.debug({ toolName }, 'Namespaceless call, attempting discovery across upstreams');
+            const allStubs = await this.discoverTools(context);
+            const found = allStubs.find(t => {
+                const parts = t.name.split('__');
+                return parts[parts.length - 1] === toolName;
+            });
+
+            if (found) {
+                this.logger.debug({ original: name, resolved: found.name }, 'Resolved namespaceless tool');
+                return this.callTool(found.name, params, context);
+            }
+
+            // No fallback to host - it doesn't support server-to-client tool calls
+            const upstreamList = Array.from(this.clients.keys()).filter(k => k !== 'host');
+            return {
+                jsonrpc: '2.0',
+                id: 0,
+                error: {
+                    code: -32601,
+                    message: `Tool '${toolName}' not found. Discovered ${allStubs.length} tools from upstreams: [${upstreamList.join(', ') || 'none'}]. Available tools: ${allStubs.map(t => t.name).slice(0, 10).join(', ')}${allStubs.length > 10 ? '...' : ''}`,
+                },
+            };
+        }
+
         const client = this.clients.get(upstreamId);
         if (!client) {
+            this.logger.error({ upstreamId, availableUpstreams: Array.from(this.clients.keys()) }, 'Upstream not found');
             return {
                 jsonrpc: '2.0',
                 id: 0,
                 error: {
                     code: -32003,
-                    message: `Upstream not found: ${upstreamId}`,
+                    message: `Upstream not found: '${upstreamId}'. Available: ${Array.from(this.clients.keys()).join(', ') || 'none'}`,
                 },
             };
         }
@@ -319,7 +396,7 @@ export class GatewayService {
             response = await client.call({
                 jsonrpc: '2.0',
                 id: context.correlationId,
-                method: 'call_tool',
+                method: 'tools/call',
                 params: {
                     name: toolName,
                     arguments: params,
@@ -352,7 +429,7 @@ export class GatewayService {
                     const response = await client.call({
                         jsonrpc: '2.0',
                         id: 'health',
-                        method: 'list_tools',
+                        method: 'tools/list',
                     }, context);
                     upstreamStatus[id] = response.error ? 'degraded' : 'active';
                 } catch (err) {

package/src/gateway/host.client.ts

@@ -0,0 +1,65 @@
+import { Logger } from 'pino';
+import { JSONRPCRequest, JSONRPCResponse } from '../core/types.js';
+import { ExecutionContext } from '../core/execution.context.js';
+import { StdioTransport } from '../transport/stdio.transport.js';
+
+/**
+ * HostClient - Proxies tool calls back to the MCP client (e.g. VS Code)
+ * that is hosting this Conduit process.
+ */
+export class HostClient {
+    constructor(
+        private logger: Logger,
+        private transport: StdioTransport
+    ) { }
+
+    async call(request: JSONRPCRequest, context: ExecutionContext): Promise<JSONRPCResponse> {
+        try {
+            this.logger.debug({ method: request.method }, 'Forwarding request to host');
+
+            let method = request.method;
+            let params = request.params;
+
+            // Bridge mcp_* calls to standard MCP calls for the host
+            if (method === 'mcp_call_tool' || method === 'call_tool') {
+                method = 'tools/call';
+            } else if (method === 'mcp_discover_tools' || method === 'discover_tools') {
+                method = 'tools/list';
+                params = {};
+            }
+
+            const result = await this.transport.callHost(method, params);
+
+            return {
+                jsonrpc: '2.0',
+                id: request.id,
+                result
+            };
+        } catch (error: any) {
+            this.logger.error({ err: error.message }, 'Host call failed');
+            return {
+                jsonrpc: '2.0',
+                id: request.id,
+                error: {
+                    code: -32008,
+                    message: `Host error: ${error.message}`,
+                },
+            };
+        }
+    }
+
+    async listTools(): Promise<any[]> {
+        try {
+            this.logger.debug('Fetching tool list from host');
+            const result = await this.transport.callHost('tools/list', {});
+            return result.tools || [];
+        } catch (error: any) {
+            this.logger.warn({ err: error.message }, 'Failed to fetch tools from host');
+            return [];
+        }
+    }
+
+    async getManifest() {
+        return null;
+    }
+}

package/src/gateway/upstream.client.ts

@@ -24,6 +24,7 @@ export class UpstreamClient {
     private urlValidator: IUrlValidator;
     private mcpClient?: Client;
     private transport?: StdioClientTransport;
+    private connected: boolean = false;
 
     constructor(logger: Logger, info: UpstreamInfo, authService: AuthService, urlValidator: IUrlValidator) {
         this.logger = logger.child({ upstreamId: info.id });
@@ -55,18 +56,16 @@ export class UpstreamClient {
 
     private async ensureConnected() {
         if (!this.mcpClient || !this.transport) return;
-        // There isn't a public isConnected property easily accessible, 
-        // usually we just connect once.
-        // We can track connected state or just try/catch connect.
-        // For simplicity, we connect once and existing sdk handles reconnection or errors usually kill it.
-        // Actually SDK Client.connect() is for the transport.
+        if (this.connected) return;
+
         try {
-            // @ts-ignore - Check internal state or just attempt connect if we haven't
-            if (!this.transport.connection) {
-                await this.mcpClient.connect(this.transport);
-            }
-        } catch (e) {
-            // connection might already be active
+            this.logger.debug('Connecting to upstream transport...');
+            await this.mcpClient.connect(this.transport);
+            this.connected = true;
+            this.logger.info('Connected to upstream MCP');
+        } catch (e: any) {
+            this.logger.error({ err: e.message }, 'Failed to connect to upstream');
+            throw e;
         }
     }
 
@@ -90,23 +89,31 @@ export class UpstreamClient {
             await this.ensureConnected();
 
             // Map GatewayService method names to SDK typed methods
-            if (request.method === 'list_tools') {
+            if (request.method === 'list_tools' || request.method === 'tools/list') {
                 const result = await this.mcpClient.listTools();
                 return {
                     jsonrpc: '2.0',
                     id: request.id,
                     result: result
                 };
-            } else if (request.method === 'call_tool') {
+            } else if (request.method === 'call_tool' || request.method === 'tools/call') {
                 const params = request.params as { name: string; arguments?: Record<string, unknown> };
                 const result = await this.mcpClient.callTool({
                     name: params.name,
                     arguments: params.arguments,
                 });
+                const normalizedResult = (result && Array.isArray((result as any).content))
+                    ? result
+                    : {
+                        content: [{
+                            type: 'text',
+                            text: typeof result === 'string' ? result : JSON.stringify(result ?? null),
+                        }],
+                    };
                 return {
                     jsonrpc: '2.0',
                     id: request.id,
-                    result: result
+                    result: normalizedResult
                 };
             } else {
                 // Fallback to generic request for other methods

package/src/index.ts

@@ -29,9 +29,10 @@ program
     .command('serve', { isDefault: true })
     .description('Start the Conduit server')
     .option('--stdio', 'Use stdio transport')
+    .option('--config <path>', 'Path to config file')
     .action(async (options) => {
         try {
-            await startServer();
+            await startServer(options);
         } catch (err) {
             console.error('Failed to start Conduit:', err);
             process.exit(1);
@@ -43,10 +44,12 @@ program
     .description('Help set up OAuth for an upstream MCP server')
     .requiredOption('--client-id <id>', 'OAuth Client ID')
     .requiredOption('--client-secret <secret>', 'OAuth Client Secret')
-    .requiredOption('--auth-url <url>', 'OAuth Authorization URL')
-    .requiredOption('--token-url <url>', 'OAuth Token URL')
+    .option('--auth-url <url>', 'OAuth Authorization URL')
+    .option('--token-url <url>', 'OAuth Token URL')
+    .option('--mcp-url <url>', 'MCP base URL (auto-discover OAuth metadata)')
     .option('--scopes <scopes>', 'OAuth Scopes (comma separated)')
     .option('--port <port>', 'Port for the local callback server', '3333')
+    .option('--pkce', 'Use PKCE for the authorization code flow')
     .action(async (options) => {
         try {
             await handleAuth({
@@ -54,8 +57,10 @@ program
                 clientSecret: options.clientSecret,
                 authUrl: options.authUrl,
                 tokenUrl: options.tokenUrl,
+                mcpUrl: options.mcpUrl,
                 scopes: options.scopes,
                 port: parseInt(options.port, 10),
+                usePkce: options.pkce || Boolean(options.mcpUrl),
             });
             console.log('\nSuccess! Configuration generated.');
         } catch (err: any) {
@@ -64,8 +69,13 @@ program
         }
     });
 
-async function startServer() {
-    const configService = new ConfigService();
+async function startServer(options: any = {}) {
+    // Merge command line options into config overrides
+    const overrides: any = {};
+    if (options.stdio) overrides.transport = 'stdio';
+    if (options.config) process.env.CONFIG_FILE = options.config;
+
+    const configService = new ConfigService(overrides);
     const logger = createLogger(configService);
 
     const otelService = new OtelService(logger);
@@ -80,6 +90,7 @@ async function startServer() {
 
         const gatewayService = new GatewayService(logger, securityService);
         const upstreams = configService.get('upstreams') || [];
+        logger.info({ upstreamCount: upstreams.length, upstreamIds: upstreams.map((u: any) => u.id) }, 'Registering upstreams from config');
         for (const upstream of upstreams) {
             gatewayService.registerUpstream(upstream);
         }
@@ -118,15 +129,30 @@ async function startServer() {
         let address: string;
 
         if (configService.get('transport') === 'stdio') {
-            transport = new StdioTransport(logger, requestController, concurrencyService);
+            const stdioTransport = new StdioTransport(logger, requestController, concurrencyService);
+            transport = stdioTransport;
             await transport.start();
+            gatewayService.registerHost(stdioTransport);
             address = 'stdio';
+
+            // IMPORTANT: Even in stdio mode, we need a local socket for sandboxes to talk to
+            const internalTransport = new SocketTransport(logger, requestController, concurrencyService);
+            const internalPort = 0; // Random available port
+            const internalAddress = await internalTransport.listen({ port: internalPort });
+            executionService.ipcAddress = internalAddress;
+
+            // Register internal transport for shutdown
+            const originalShutdown = transport.close.bind(transport);
+            transport.close = async () => {
+                await originalShutdown();
+                await internalTransport.close();
+            };
         } else {
             transport = new SocketTransport(logger, requestController, concurrencyService);
             const port = configService.get('port');
             address = await transport.listen({ port });
+            executionService.ipcAddress = address;
         }
-        executionService.ipcAddress = address;
 
         // Pre-warm workers
         await requestController.warmup();