@mhingston5/conduit 1.0.0

package/src/executors/pyodide.executor.ts

@@ -0,0 +1,327 @@
+import { Worker } from 'node:worker_threads';
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { ExecutionContext } from '../core/execution.context.js';
+import { ResourceLimits as ConduitResourceLimits } from '../core/config.service.js';
+import { ConduitError } from '../core/types.js';
+import { resolveAssetPath } from '../core/asset.utils.js';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+import { Executor, ExecutorConfig, ExecutionResult } from '../core/interfaces/executor.interface.js';
+
+export { ExecutionResult };
+
+// Deprecated: use ExecutorConfig
+export interface IPCInfo {
+    ipcAddress: string;
+    ipcToken: string;
+    sdkCode?: string;
+}
+
+
+interface PooledWorker {
+    worker: Worker;
+    busy: boolean;
+    runs: number;
+    lastUsed: number;
+}
+
+export class PyodideExecutor implements Executor {
+    private shimContent: string = '';
+    private pool: PooledWorker[] = [];
+    private maxPoolSize: number;
+    private maxRunsPerWorker = 1;
+
+    constructor(maxPoolSize = 3) {
+        this.maxPoolSize = maxPoolSize;
+    }
+
+    private getShim(): string {
+        if (this.shimContent) return this.shimContent;
+        try {
+            const assetPath = resolveAssetPath('python-shim.py');
+            this.shimContent = fs.readFileSync(assetPath, 'utf-8');
+            return this.shimContent;
+        } catch (err: any) {
+            throw new Error(`Failed to load Python shim: ${err.message}`);
+        }
+    }
+
+    private waitQueue: Array<(worker: PooledWorker) => void> = [];
+
+    private async getWorker(logger: any, limits?: ConduitResourceLimits): Promise<PooledWorker> {
+        // Find available worker
+        let pooled = this.pool.find(w => !w.busy);
+        if (pooled) {
+            pooled.busy = true;
+            return pooled;
+        }
+
+        // Create new worker if pool not full
+        if (this.pool.length < this.maxPoolSize) {
+            logger.info('Creating new Pyodide worker for pool');
+            const worker = this.createWorker(limits);
+            pooled = { worker, busy: true, runs: 0, lastUsed: Date.now() };
+            this.pool.push(pooled);
+
+            // Wait for ready signal
+            await new Promise<void>((resolve, reject) => {
+                const onMessage = (msg: any) => {
+                    if (msg.type === 'ready') {
+                        worker.off('message', onMessage);
+                        resolve();
+                    }
+                };
+                worker.on('message', onMessage);
+                worker.on('error', reject);
+                setTimeout(() => {
+                    // Cleanup worker on timeout
+                    worker.terminate();
+                    this.pool = this.pool.filter(p => p !== pooled);
+                    reject(new Error('Worker init timeout'));
+                }, 10000);
+            });
+
+            return pooled;
+        }
+
+        // Wait for a worker to become available via queue
+        return new Promise((resolve) => {
+            this.waitQueue.push(resolve);
+        });
+    }
+
+    private createWorker(limits?: ConduitResourceLimits): Worker {
+        let workerPath = path.resolve(__dirname, './pyodide.worker.js');
+        if (!fs.existsSync(workerPath)) {
+            workerPath = path.resolve(__dirname, './pyodide.worker.ts');
+        }
+
+        return new Worker(workerPath, {
+            execArgv: process.execArgv.includes('--loader') ? process.execArgv : [],
+            resourceLimits: limits ? {
+                maxOldSpaceSizeMb: limits.memoryLimitMb,
+                // Stack size and young generation are usually fine with defaults
+            } as any : undefined
+        });
+    }
+
+    async warmup(limits: ConduitResourceLimits) {
+        // Pre-fill the pool up to maxPoolSize
+        const needed = this.maxPoolSize - this.pool.length;
+        if (needed <= 0) return;
+
+        console.info(`Pre-warming ${needed} Pyodide workers...`);
+        const promises = [];
+        for (let i = 0; i < needed; i++) {
+            promises.push(this.createAndPoolWorker(limits));
+        }
+        await Promise.all(promises);
+        console.info(`Pyodide pool pre-warmed with ${this.pool.length} workers.`);
+    }
+
+    private async createAndPoolWorker(limits: ConduitResourceLimits) {
+        // Small optimization: don't double-fill if racing
+        if (this.pool.length >= this.maxPoolSize) return;
+
+        const worker = this.createWorker(limits);
+        const pooled: PooledWorker = { worker, busy: true, runs: 0, lastUsed: Date.now() };
+        this.pool.push(pooled);
+
+        // Wait for ready signal
+        try {
+            await new Promise<void>((resolve, reject) => {
+                const onMessage = (msg: any) => {
+                    if (msg.type === 'ready') {
+                        worker.off('message', onMessage);
+                        resolve();
+                    }
+                };
+                worker.on('message', onMessage);
+                worker.on('error', reject);
+                setTimeout(() => reject(new Error('Worker init timeout')), 10000);
+            });
+            pooled.busy = false;
+
+            // Check if anyone is waiting for a worker immediately
+            if (this.waitQueue.length > 0) {
+                const nextResolve = this.waitQueue.shift();
+                if (nextResolve) {
+                    pooled.busy = true;
+                    nextResolve(pooled);
+                }
+            }
+        } catch (err) {
+            // If failed, remove from pool
+            this.pool = this.pool.filter(p => p !== pooled);
+            worker.terminate();
+        }
+    }
+
+    async execute(code: string, limits: ConduitResourceLimits, context: ExecutionContext, config?: ExecutorConfig): Promise<ExecutionResult> {
+        const { logger } = context;
+        const pooledWorker = await this.getWorker(logger, limits);
+        const worker = pooledWorker.worker;
+
+        return new Promise((resolve) => {
+            const timeout = setTimeout(() => {
+                logger.warn('Python execution timed out, terminating worker');
+                worker.terminate();
+                // Remove from pool
+                this.pool = this.pool.filter(w => w !== pooledWorker);
+                resolve({
+                    stdout: '',
+                    stderr: 'Execution timed out',
+                    exitCode: null,
+                    error: {
+                        code: ConduitError.RequestTimeout,
+                        message: 'Execution timed out',
+                    },
+                });
+            }, limits.timeoutMs);
+
+            const onMessage = (msg: any) => {
+                if (msg.type === 'ready' || msg.type === 'pong') return;
+
+                clearTimeout(timeout);
+                worker.off('message', onMessage);
+                worker.off('error', onError);
+
+                pooledWorker.busy = false;
+
+                // Notify next waiter if any
+                if (this.waitQueue.length > 0) {
+                    const nextResolve = this.waitQueue.shift();
+                    if (nextResolve) {
+                        pooledWorker.busy = true;
+                        nextResolve(pooledWorker);
+                    }
+                }
+
+                pooledWorker.runs++;
+                pooledWorker.lastUsed = Date.now();
+
+                // Recycle if too many runs
+                if (pooledWorker.runs >= this.maxRunsPerWorker) {
+                    logger.info('Recycling Pyodide worker after max runs');
+                    worker.terminate();
+                    this.pool = this.pool.filter(w => w !== pooledWorker);
+                }
+
+                if (msg.success) {
+                    resolve({
+                        stdout: msg.stdout,
+                        stderr: msg.stderr,
+                        exitCode: 0,
+                    });
+                } else {
+                    logger.warn({ error: msg.error }, 'Python execution failed or limit breached, terminating worker');
+                    worker.terminate();
+                    this.pool = this.pool.filter(w => w !== pooledWorker);
+
+                    logger.debug({ error: msg.error }, 'Python execution error from worker');
+                    const normalizedError = (msg.error || '').toLowerCase();
+                    const limitBreached = msg.limitBreached || '';
+
+                    const isLogLimit = limitBreached === 'log' || normalizedError.includes('[limit_log]');
+                    const isOutputLimit = limitBreached === 'output' || normalizedError.includes('[limit_output]');
+                    const isAmbiguousLimit = !isOutputLimit && !isLogLimit && (normalizedError.includes('i/o error') || normalizedError.includes('errno 29') || normalizedError.includes('limit exceeded'));
+
+                    resolve({
+                        stdout: msg.stdout,
+                        stderr: msg.stderr,
+                        exitCode: 1,
+                        error: {
+                            code: isLogLimit ? ConduitError.LogLimitExceeded : ((isOutputLimit || isAmbiguousLimit) ? ConduitError.OutputLimitExceeded : ConduitError.InternalError),
+                            message: isLogLimit ? 'Log entry limit exceeded' : ((isOutputLimit || isAmbiguousLimit) ? 'Output limit exceeded' : msg.error),
+                        },
+                    });
+                }
+            };
+
+            const onError = (err: any) => {
+                clearTimeout(timeout);
+                worker.off('message', onMessage);
+                worker.off('error', onError);
+
+                logger.error({ err }, 'Pyodide worker error');
+                worker.terminate();
+                this.pool = this.pool.filter(w => w !== pooledWorker);
+
+                resolve({
+                    stdout: '',
+                    stderr: err.message,
+                    exitCode: null,
+                    error: {
+                        code: ConduitError.InternalError,
+                        message: err.message,
+                    },
+                });
+            };
+
+            worker.on('message', onMessage);
+            worker.on('error', onError);
+
+            // Prepare shim with SDK injection
+            let shim = this.getShim();
+            if (config?.sdkCode) {
+                shim = shim.replace('# __CONDUIT_SDK_INJECTION__', config.sdkCode);
+            }
+
+            worker.postMessage({
+                type: 'execute',
+                data: { code, limits, ipcInfo: config, shim }
+            });
+        });
+    }
+
+    async shutdown() {
+        for (const pooled of this.pool) {
+            await pooled.worker.terminate();
+        }
+        this.pool = [];
+    }
+
+    async healthCheck(): Promise<{ status: string; workers: number; detail?: string }> {
+        try {
+            // Find an available worker or create a temporary one for health check
+            const pooled = await this.getWorker(console, {
+                timeoutMs: 5000,
+                memoryLimitMb: 128,
+                maxOutputBytes: 1024,
+                maxLogEntries: 10
+            });
+
+            return new Promise((resolve) => {
+                let timeout: NodeJS.Timeout;
+
+                const onMessage = (msg: any) => {
+                    if (msg.type === 'pong') {
+                        cleanup();
+                        pooled.busy = false;
+                        resolve({ status: 'ok', workers: this.pool.length });
+                    }
+                };
+
+                const cleanup = () => {
+                    clearTimeout(timeout);
+                    pooled.worker.off('message', onMessage);
+                };
+
+                timeout = setTimeout(() => {
+                    cleanup();
+                    pooled.busy = false;
+                    resolve({ status: 'error', workers: this.pool.length, detail: 'Health check timeout' });
+                }, 2000);
+
+                pooled.worker.on('message', onMessage);
+                pooled.worker.postMessage({ type: 'ping' });
+            });
+        } catch (err: any) {
+            return { status: 'error', workers: this.pool.length, detail: err.message };
+        }
+    }
+}
+

package/src/executors/pyodide.worker.ts

@@ -0,0 +1,195 @@
+import { parentPort, workerData } from 'node:worker_threads';
+import { loadPyodide, type PyodideInterface } from 'pyodide';
+import net from 'node:net';
+
+let pyodide: PyodideInterface | null = null;
+let currentStdout = '';
+let currentStderr = '';
+let totalOutputBytes = 0;
+let totalLogEntries = 0;
+let currentLimits: any = null;
+
+async function init() {
+    if (pyodide) return pyodide;
+
+    pyodide = await loadPyodide({
+        stdout: (text) => {
+            if (currentLimits && (totalOutputBytes > (currentLimits.maxOutputBytes || 1024 * 1024) || totalLogEntries > (currentLimits.maxLogEntries || 10000))) {
+                return; // Stop processing logs once limit breached
+            }
+            currentStdout += text + '\n';
+            totalOutputBytes += text.length + 1;
+            totalLogEntries++;
+        },
+        stderr: (text) => {
+            if (currentLimits && (totalOutputBytes > (currentLimits.maxOutputBytes || 1024 * 1024) || totalLogEntries > (currentLimits.maxLogEntries || 10000))) {
+                return; // Stop processing logs once limit breached
+            }
+            currentStderr += text + '\n';
+            totalOutputBytes += text.length + 1;
+            totalLogEntries++;
+        },
+    });
+
+    return pyodide;
+}
+
+async function handleTask(data: any) {
+    const { code, limits, ipcInfo, shim } = data;
+    currentStdout = '';
+    currentStderr = '';
+    totalOutputBytes = 0;
+    totalLogEntries = 0;
+    currentLimits = limits;
+
+    try {
+        const p = await init();
+
+        const sendIPCRequest = async (method: string, params: any) => {
+            if (!ipcInfo?.ipcAddress) throw new Error('Conduit IPC address not configured');
+
+            return new Promise((resolve, reject) => {
+                let client: net.Socket;
+
+                if (ipcInfo.ipcAddress.includes(':')) {
+                    const lastColon = ipcInfo.ipcAddress.lastIndexOf(':');
+                    const host = ipcInfo.ipcAddress.substring(0, lastColon);
+                    const port = ipcInfo.ipcAddress.substring(lastColon + 1);
+
+                    let targetHost = host.replace(/[\[\]]/g, '');
+                    if (targetHost === '0.0.0.0' || targetHost === '::' || targetHost === '::1' || targetHost === '') {
+                        targetHost = '127.0.0.1';
+                    }
+
+                    client = net.createConnection({
+                        host: targetHost,
+                        port: parseInt(port)
+                    });
+                } else {
+                    client = net.createConnection({ path: ipcInfo.ipcAddress });
+                }
+
+                const id = Math.random().toString(36).substring(7);
+                const request = {
+                    jsonrpc: '2.0',
+                    id,
+                    method,
+                    params: params || {},
+                    auth: { bearerToken: ipcInfo.ipcToken }
+                };
+
+                client.on('error', (err) => {
+                    reject(err);
+                    client.destroy();
+                });
+
+                client.write(JSON.stringify(request) + '\n');
+
+                let buffer = '';
+                client.on('data', (data) => {
+                    buffer += data.toString();
+                    // Robust framing: read until we find a complete JSON object on a line
+                    const lines = buffer.split('\n');
+                    buffer = lines.pop() || ''; // Keep the last partial line
+
+                    for (const line of lines) {
+                        if (!line.trim()) continue;
+                        try {
+                            const response = JSON.parse(line);
+                            if (response.id === id) {
+                                if (response.error) {
+                                    reject(new Error(response.error.message));
+                                } else {
+                                    resolve(response.result);
+                                }
+                                client.end();
+                                return;
+                            }
+                        } catch (e) {
+                            // If parse fails, it might be a partial line that we haven't seen the end of yet
+                            // but since we split by \n, this shouldn't happen unless the \n was inside the JSON.
+                            // However, Conduit ensures JSON-RPC is one line.
+                        }
+                    }
+                });
+
+                client.on('end', () => {
+                    if (buffer.trim()) {
+                        try {
+                            const response = JSON.parse(buffer);
+                            if (response.id === id) {
+                                if (response.error) {
+                                    reject(new Error(response.error.message));
+                                } else {
+                                    resolve(response.result);
+                                }
+                            }
+                        } catch (e) { }
+                    }
+                });
+            });
+        };
+
+        (p as any).globals.set('discover_mcp_tools_js', (options: any) => {
+            return sendIPCRequest('mcp.discoverTools', options);
+        });
+
+        (p as any).globals.set('call_mcp_tool_js', (name: string, args: any) => {
+            return sendIPCRequest('mcp.callTool', { name, arguments: args });
+        });
+
+        if (shim) {
+            await p.runPythonAsync(shim);
+        }
+
+        const result = await p.runPythonAsync(code);
+
+        if (totalOutputBytes > (limits.maxOutputBytes || 1024 * 1024)) {
+            throw new Error('[LIMIT_OUTPUT]');
+        }
+        if (totalLogEntries > (limits.maxLogEntries || 10000)) {
+            throw new Error('[LIMIT_LOG]');
+        }
+
+        parentPort?.postMessage({
+            stdout: currentStdout,
+            stderr: currentStderr,
+            result: String(result),
+            success: true,
+        });
+    } catch (err: any) {
+        let isOutput = err.message.includes('[LIMIT_OUTPUT]');
+        let isLog = err.message.includes('[LIMIT_LOG]');
+
+        // Fallback: check counters if message doesn't match (e.g. wrapped in OSError)
+        if (!isOutput && !isLog && currentLimits) {
+            if (totalOutputBytes > (currentLimits.maxOutputBytes || 1024 * 1024)) {
+                isOutput = true;
+            }
+            // Check specific log limit breach
+            if (totalLogEntries > (currentLimits.maxLogEntries || 10000)) {
+                isLog = true;
+            }
+        }
+
+        parentPort?.postMessage({
+            stdout: currentStdout,
+            stderr: currentStderr,
+            error: err.message,
+            limitBreached: isOutput ? 'output' : (isLog ? 'log' : undefined),
+            success: false,
+        });
+    }
+}
+
+parentPort?.on('message', async (msg) => {
+    if (msg.type === 'execute') {
+        await handleTask(msg.data);
+    } else if (msg.type === 'ping') {
+        parentPort?.postMessage({ type: 'pong' });
+    }
+});
+
+// Signal ready
+parentPort?.postMessage({ type: 'ready' });
+

package/src/gateway/auth.service.ts

@@ -0,0 +1,104 @@
+import { Logger } from 'pino';
+import axios from 'axios';
+
+export type AuthType = 'apiKey' | 'oauth2' | 'bearer';
+
+export interface UpstreamCredentials {
+    type: AuthType;
+    apiKey?: string;
+    bearerToken?: string;
+    oauth2?: {
+        clientId: string;
+        clientSecret: string;
+        tokenUrl: string;
+        refreshToken: string;
+    };
+}
+
+interface CachedToken {
+    accessToken: string;
+    expiresAt: number;
+}
+
+export class AuthService {
+    private logger: Logger;
+    // Cache tokens separately from credentials to avoid mutation
+    private tokenCache = new Map<string, CachedToken>();
+    // Prevent concurrent refresh requests for the same client
+    private refreshLocks = new Map<string, Promise<string>>();
+
+    constructor(logger: Logger) {
+        this.logger = logger;
+    }
+
+    async getAuthHeaders(creds: UpstreamCredentials): Promise<Record<string, string>> {
+        switch (creds.type) {
+            case 'apiKey':
+                return { 'X-API-Key': creds.apiKey || '' };
+            case 'bearer':
+                return { 'Authorization': `Bearer ${creds.bearerToken}` };
+            case 'oauth2':
+                return { 'Authorization': await this.getOAuth2Token(creds) };
+            default:
+                throw new Error(`Unsupported auth type: ${creds.type}`);
+        }
+    }
+
+    private async getOAuth2Token(creds: UpstreamCredentials): Promise<string> {
+        if (!creds.oauth2) throw new Error('OAuth2 credentials missing');
+
+        const { oauth2 } = creds;
+        const cacheKey = `${oauth2.clientId}:${oauth2.tokenUrl}`;
+
+        // Check cache first (with 30s buffer)
+        const cached = this.tokenCache.get(cacheKey);
+        if (cached && cached.expiresAt > Date.now() + 30000) {
+            return `Bearer ${cached.accessToken}`;
+        }
+
+        // Check if refresh is already in progress
+        const existingRefresh = this.refreshLocks.get(cacheKey);
+        if (existingRefresh) {
+            return existingRefresh;
+        }
+
+        // Start refresh with lock
+        const refreshPromise = this.doRefresh(creds, cacheKey);
+        this.refreshLocks.set(cacheKey, refreshPromise);
+
+        try {
+            return await refreshPromise;
+        } finally {
+            this.refreshLocks.delete(cacheKey);
+        }
+    }
+
+    private async doRefresh(creds: UpstreamCredentials, cacheKey: string): Promise<string> {
+        const { oauth2 } = creds;
+        if (!oauth2) throw new Error('OAuth2 credentials missing');
+
+        this.logger.info('Refreshing OAuth2 token');
+
+        try {
+            const response = await axios.post(oauth2.tokenUrl, {
+                grant_type: 'refresh_token',
+                refresh_token: oauth2.refreshToken,
+                client_id: oauth2.clientId,
+                client_secret: oauth2.clientSecret,
+            });
+
+            const { access_token, expires_in } = response.data;
+
+            // Cache the token (don't mutate the input credentials)
+            this.tokenCache.set(cacheKey, {
+                accessToken: access_token,
+                expiresAt: Date.now() + (expires_in * 1000),
+            });
+
+            return `Bearer ${access_token}`;
+        } catch (err: any) {
+            this.logger.error({ err: err.message }, 'Failed to refresh OAuth2 token');
+            throw new Error(`OAuth2 refresh failed: ${err.message}`);
+        }
+    }
+}