@mhingston5/conduit 1.0.0

package/src/core/asset.utils.ts

@@ -0,0 +1,42 @@
+import path from 'node:path';
+import fs from 'node:fs';
+import { fileURLToPath } from 'node:url';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+/**
+ * Resolves the absolute path to an asset file, handling both source (dev)
+ * and distribution (prod) directory structures.
+ * 
+ * Strategies:
+ * 1. ../assets/{filename} (Source: src/core -> src/assets)
+ * 2. ./assets/{filename} (Dist: dist/ -> dist/assets, if core is merged or similar)
+ * 3. ../../assets/{filename} (Dist: dist/core -> dist/assets)
+ * 4. assets/{filename} (Relative to cwd, unlikely but fallback)
+ * 
+ * @param filename The name of the asset file (e.g., 'deno-shim.ts')
+ * @returns The absolute path to the asset if found
+ * @throws Error if the asset cannot be found
+ */
+export function resolveAssetPath(filename: string): string {
+    const candidates = [
+        // Source structure: src/core/asset.utils.ts -> src/assets/
+        path.resolve(__dirname, '../assets', filename),
+        // Dist structure possibility 1: dist/ (flat) with assets/ subdir
+        path.resolve(__dirname, './assets', filename),
+        // Dist structure possibility 2: dist/core/ -> dist/assets/
+        path.resolve(__dirname, '../../assets', filename),
+        // Dist structure possibility 3: dist/ -> assets/ (if called from root)
+        path.resolve(process.cwd(), 'assets', filename),
+        // Dist structure possibility 4: dist/assets/ (from root)
+        path.resolve(process.cwd(), 'dist/assets', filename)
+    ];
+
+    for (const candidate of candidates) {
+        if (fs.existsSync(candidate)) {
+            return candidate;
+        }
+    }
+
+    throw new Error(`Asset not found: ${filename}. Checked paths: ${candidates.join(', ')}`);
+}

package/src/core/concurrency.service.ts

@@ -0,0 +1,70 @@
+import pLimit from 'p-limit';
+import { Logger } from 'pino';
+import { trace } from '@opentelemetry/api';
+import { metrics } from './metrics.service.js';
+
+export interface ConcurrencyOptions {
+    maxConcurrent: number;
+    maxQueueSize?: number;
+}
+
+export class QueueFullError extends Error {
+    constructor(message: string) {
+        super(message);
+        this.name = 'QueueFullError';
+    }
+}
+
+export class ConcurrencyService {
+    private limit: ReturnType<typeof pLimit>;
+    private logger: Logger;
+    private maxQueueSize: number;
+    private queueDepthHistogram: any; // Using explicit type locally if needed, or rely on metrics service later. Using direct OTEL for now involves refactor.
+    // Let's rely on internal state for rejection and let MetricsService handle reporting if possible, or add it here.
+    // simpler: usage of metrics service is better pattern.
+
+    constructor(logger: Logger, options: ConcurrencyOptions) {
+        this.logger = logger;
+        this.limit = pLimit(options.maxConcurrent);
+        this.maxQueueSize = options.maxQueueSize || 100; // Default to 100
+
+        metrics.registerQueueLengthProvider(() => this.limit.pendingCount);
+    }
+
+    async run<T>(fn: () => Promise<T>): Promise<T> {
+        if (this.limit.pendingCount >= this.maxQueueSize) {
+            this.logger.warn({ pending: this.limit.pendingCount, max: this.maxQueueSize }, 'Request queue full, rejecting request');
+            throw new QueueFullError('Server is too busy, please try again later');
+        }
+
+        const active = this.limit.activeCount;
+        const pending = this.limit.pendingCount;
+
+        this.logger.debug({ active, pending }, 'Concurrency status before task');
+
+        // Add attributes to current OTEL span if exists
+        const span = trace.getActiveSpan();
+        if (span) {
+            span.setAttributes({
+                'concurrency.active': active,
+                'concurrency.pending': pending,
+            });
+        }
+
+        try {
+            return await this.limit(fn);
+        } finally {
+            this.logger.debug({
+                active: this.limit.activeCount,
+                pending: this.limit.pendingCount
+            }, 'Concurrency status after task');
+        }
+    }
+
+    get stats() {
+        return {
+            activeCount: this.limit.activeCount,
+            pendingCount: this.limit.pendingCount,
+        };
+    }
+}

package/src/core/config.service.ts

@@ -0,0 +1,147 @@
+import { z } from 'zod';
+import dotenv from 'dotenv';
+import fs from 'node:fs';
+import path from 'node:path';
+import yaml from 'js-yaml';
+
+dotenv.config();
+
+import { AppConfig } from './interfaces/app.config.js';
+
+
+export const ResourceLimitsSchema = z.object({
+    timeoutMs: z.number().default(30000),
+    memoryLimitMb: z.number().default(256),
+    maxOutputBytes: z.number().default(1024 * 1024), // 1MB
+    maxLogEntries: z.number().default(10000),
+});
+
+export const UpstreamCredentialsSchema = z.object({
+    type: z.enum(['oauth2', 'apikey']), // Add other types as needed
+    clientId: z.string().optional(),
+    clientSecret: z.string().optional(),
+    tokenUrl: z.string().optional(),
+    scopes: z.array(z.string()).optional(),
+    apiKey: z.string().optional(),
+    headerName: z.string().optional(),
+});
+
+export const HttpUpstreamSchema = z.object({
+    id: z.string(),
+    type: z.literal('http').optional().default('http'),
+    url: z.string(),
+    credentials: UpstreamCredentialsSchema.optional(),
+});
+
+export const StdioUpstreamSchema = z.object({
+    id: z.string(),
+    type: z.literal('stdio'),
+    command: z.string(),
+    args: z.array(z.string()).optional(),
+    env: z.record(z.string(), z.string()).optional(),
+});
+
+export const UpstreamInfoSchema = z.union([HttpUpstreamSchema, StdioUpstreamSchema]);
+
+export type ResourceLimits = z.infer<typeof ResourceLimitsSchema>;
+
+export const ConfigSchema = z.object({
+    port: z.union([z.string(), z.number()]).default('3000').transform((v) => Number(v)),
+    nodeEnv: z.enum(['development', 'production', 'test']).default('development'),
+    logLevel: z.enum(['debug', 'info', 'warn', 'error']).default('info'),
+    resourceLimits: ResourceLimitsSchema.default({
+        timeoutMs: 30000,
+        memoryLimitMb: 256,
+        maxOutputBytes: 1024 * 1024,
+        maxLogEntries: 10000,
+    }),
+    secretRedactionPatterns: z.array(z.string()).default([
+        '[A-Za-z0-9-_]{20,}', // Default pattern from spec
+    ]),
+    ipcBearerToken: z.string().optional().default(() => Math.random().toString(36).substring(7)),
+    maxConcurrent: z.number().default(10),
+    denoMaxPoolSize: z.number().default(10),
+    pyodideMaxPoolSize: z.number().default(3),
+    metricsUrl: z.string().default('http://127.0.0.1:9464/metrics'),
+    opsPort: z.number().optional(),
+    upstreams: z.array(UpstreamInfoSchema).default([]),
+});
+
+export type Config = z.infer<typeof ConfigSchema>;
+
+export class ConfigService {
+    private config: AppConfig;
+
+    constructor(overrides: Partial<AppConfig> = {}) {
+        const fileConfig = this.loadConfigFile();
+
+        const envConfig = {
+            port: process.env.PORT,
+            nodeEnv: process.env.NODE_ENV,
+            logLevel: process.env.LOG_LEVEL,
+            metricsUrl: process.env.METRICS_URL,
+            // upstreams: process.env.UPSTREAMS ? JSON.parse(process.env.UPSTREAMS) : undefined, // Removed per user request
+        };
+
+        // Remove undefined keys from envConfig
+        Object.keys(envConfig).forEach(key => envConfig[key as keyof typeof envConfig] === undefined && delete envConfig[key as keyof typeof envConfig]);
+
+        const mergedConfig = {
+            ...fileConfig,
+            ...envConfig,
+            ...overrides,
+        };
+
+        const result = ConfigSchema.safeParse(mergedConfig);
+        if (!result.success) {
+            const error = result.error.format();
+            throw new Error(`Invalid configuration: ${JSON.stringify(error, null, 2)}`);
+        }
+
+        this.config = result.data as AppConfig;
+
+        // Default opsPort if not set
+        if (!this.config.opsPort) {
+            this.config.opsPort = this.config.port === 0 ? 0 : this.config.port + 1;
+        }
+    }
+
+    get<K extends keyof AppConfig>(key: K): AppConfig[K] {
+        return this.config[key];
+    }
+
+    get all(): AppConfig {
+        return { ...this.config };
+    }
+
+    private loadConfigFile(): Partial<AppConfig> {
+        const configPath = process.env.CONFIG_FILE ||
+            (fs.existsSync(path.resolve(process.cwd(), 'conduit.yaml')) ? 'conduit.yaml' :
+                (fs.existsSync(path.resolve(process.cwd(), 'conduit.json')) ? 'conduit.json' : null));
+
+        if (!configPath) return {};
+
+        try {
+            const fullPath = path.resolve(process.cwd(), configPath);
+            let fileContent = fs.readFileSync(fullPath, 'utf-8');
+
+            // Env var substitution: ${VAR} or ${VAR:-default}
+            fileContent = fileContent.replace(/\$\{([a-zA-Z0-9_]+)(?::-([^}]+))?\}/g, (match, varName, defaultValue) => {
+                const value = process.env[varName];
+                if (value !== undefined) {
+                    return value;
+                }
+                return defaultValue !== undefined ? defaultValue : '';
+            });
+
+            if (configPath.endsWith('.yaml') || configPath.endsWith('.yml')) {
+                return yaml.load(fileContent) as Partial<AppConfig>;
+            } else {
+                return JSON.parse(fileContent);
+            }
+        } catch (error) {
+            console.warn(`Failed to load config file ${configPath}:`, error);
+            return {};
+        }
+    }
+}

package/src/core/execution.context.ts

@@ -0,0 +1,37 @@
+import { v4 as uuidv4 } from 'uuid';
+import { Logger } from 'pino';
+
+export interface ExecutionContextOptions {
+    tenantId?: string;
+    logger: Logger;
+    allowedTools?: string[];
+    remoteAddress?: string;
+    strictValidation?: boolean;
+}
+
+export class ExecutionContext {
+    public readonly correlationId: string;
+    public readonly startTime: number;
+    public readonly tenantId?: string;
+    public logger: Logger;
+    public allowedTools?: string[];
+    public readonly remoteAddress?: string;
+    public readonly strictValidation: boolean;
+
+    constructor(options: ExecutionContextOptions) {
+        this.correlationId = uuidv4();
+        this.startTime = Date.now();
+        this.tenantId = options.tenantId;
+        this.allowedTools = options.allowedTools;
+        this.remoteAddress = options.remoteAddress;
+        this.strictValidation = options.strictValidation ?? false;
+        this.logger = options.logger.child({
+            correlationId: this.correlationId,
+            tenantId: this.tenantId,
+        });
+    }
+
+    getDuration(): number {
+        return Date.now() - this.startTime;
+    }
+}

package/src/core/execution.service.ts

@@ -0,0 +1,209 @@
+import { Logger } from 'pino';
+import { ExecutorRegistry } from './registries/executor.registry.js';
+import { ResourceLimits } from './config.service.js';
+import { GatewayService } from '../gateway/gateway.service.js';
+import { SecurityService } from './security.service.js';
+import { SDKGenerator, toToolBinding } from '../sdk/index.js';
+import { ExecutionContext } from './execution.context.js';
+import { ConduitError } from './types.js';
+import { ExecutionResult } from './interfaces/executor.interface.js';
+
+export { ExecutionResult };
+
+export class ExecutionService {
+    private logger: Logger;
+    private executorRegistry: ExecutorRegistry;
+    private sdkGenerator = new SDKGenerator();
+    private defaultLimits: ResourceLimits;
+    private gatewayService: GatewayService;
+    private securityService: SecurityService;
+    private _ipcAddress: string = '';
+
+    constructor(
+        logger: Logger,
+        defaultLimits: ResourceLimits,
+        gatewayService: GatewayService,
+        securityService: SecurityService,
+        executorRegistry: ExecutorRegistry
+    ) {
+        this.logger = logger;
+        this.defaultLimits = defaultLimits;
+        this.gatewayService = gatewayService;
+        this.securityService = securityService;
+        this.executorRegistry = executorRegistry;
+    }
+
+    set ipcAddress(addr: string) {
+        this._ipcAddress = addr;
+    }
+
+    async executeTypeScript(
+        code: string,
+        limits: ResourceLimits,
+        context: ExecutionContext,
+        allowedTools?: string[]
+    ): Promise<ExecutionResult> {
+        const effectiveLimits = { ...this.defaultLimits, ...limits };
+
+        // 1. Validation
+        const securityResult = this.securityService.validateCode(code);
+        if (!securityResult.valid) {
+            return this.createErrorResult(ConduitError.Forbidden, securityResult.message || 'Access denied');
+        }
+
+        // 2. Routing Logic (Isolate vs Deno)
+        // Strip simple comments to avoid false positives
+        const cleanCode = code.replace(/\/\*[\s\S]*?\*\/|([^:]|^)\/\/.*$/gm, '$1');
+        const hasImports = /^\s*import\s/m.test(cleanCode) ||
+            /^\s*export\s/m.test(cleanCode) ||
+            /\bDeno\./.test(cleanCode) ||
+            /\bDeno\b/.test(cleanCode);
+
+        // Try to use IsolateExecutor if available and code is simple
+        if (!hasImports && this.executorRegistry.has('isolate')) {
+            return await this.executeIsolate(code, effectiveLimits, context, allowedTools);
+        }
+
+        // Fix Sev2: Ensure IPC address is set before execution if Deno is needed
+        if (!this._ipcAddress) {
+            return this.createErrorResult(ConduitError.InternalError, 'IPC address not initialized');
+        }
+
+        // 3. Fallback to Deno
+        if (!this.executorRegistry.has('deno')) {
+            return this.createErrorResult(ConduitError.InternalError, 'Deno execution not available');
+        }
+
+        const executor = this.executorRegistry.get('deno')!;
+
+        // 4. SDK Generation
+        const bindings = await this.getToolBindings(context);
+        const sdkCode = this.sdkGenerator.generateTypeScript(bindings, allowedTools);
+
+        // 5. Session & Execution
+        const sessionToken = this.securityService.createSession(allowedTools);
+        try {
+            return await executor.execute(code, effectiveLimits, context, {
+                ipcAddress: this._ipcAddress,
+                ipcToken: sessionToken,
+                sdkCode
+            });
+        } finally {
+            this.securityService.invalidateSession(sessionToken);
+        }
+    }
+
+    async executePython(
+        code: string,
+        limits: ResourceLimits,
+        context: ExecutionContext,
+        allowedTools?: string[]
+    ): Promise<ExecutionResult> {
+        const effectiveLimits = { ...this.defaultLimits, ...limits };
+
+        if (!this.executorRegistry.has('python')) {
+            return this.createErrorResult(ConduitError.InternalError, 'Python execution not available');
+        }
+
+        // Fix Sev2: Ensure IPC address is set before execution for Python
+        if (!this._ipcAddress) {
+            return this.createErrorResult(ConduitError.InternalError, 'IPC address not initialized');
+        }
+
+        const executor = this.executorRegistry.get('python')!;
+
+        const securityResult = this.securityService.validateCode(code);
+        if (!securityResult.valid) {
+            return this.createErrorResult(ConduitError.Forbidden, securityResult.message || 'Access denied');
+        }
+
+        const bindings = await this.getToolBindings(context);
+        const sdkCode = this.sdkGenerator.generatePython(bindings, allowedTools);
+
+        const sessionToken = this.securityService.createSession(allowedTools);
+        try {
+            return await executor.execute(code, effectiveLimits, context, {
+                ipcAddress: this._ipcAddress,
+                ipcToken: sessionToken,
+                sdkCode
+            });
+        } finally {
+            this.securityService.invalidateSession(sessionToken);
+        }
+    }
+
+    private async getToolBindings(context: ExecutionContext) {
+        // Phase 1: Lazy loading - fetch stubs instead of full schemas
+        const packages = await this.gatewayService.listToolPackages();
+        const allBindings = [];
+
+        for (const pkg of packages) {
+            try {
+                // Determine if we need to fetch tools for this package
+                // Optimization: if allowedTools is strict, we could filter packages here
+
+                const stubs = await this.gatewayService.listToolStubs(pkg.id, context);
+                allBindings.push(...stubs.map(s => toToolBinding(s.id, undefined, s.description)));
+            } catch (err: any) {
+                this.logger.warn({ packageId: pkg.id, err: err.message }, 'Failed to list stubs for package');
+            }
+        }
+        return allBindings;
+    }
+
+    async executeIsolate(
+        code: string,
+        limits: ResourceLimits,
+        context: ExecutionContext,
+        allowedTools?: string[]
+    ): Promise<ExecutionResult> {
+        if (!this.executorRegistry.has('isolate')) {
+            return this.createErrorResult(ConduitError.InternalError, 'IsolateExecutor not available');
+        }
+        const executor = this.executorRegistry.get('isolate')!;
+
+        const effectiveLimits = { ...this.defaultLimits, ...limits };
+        const securityResult = this.securityService.validateCode(code);
+        if (!securityResult.valid) {
+            return this.createErrorResult(ConduitError.Forbidden, securityResult.message || 'Access denied');
+        }
+
+        const bindings = await this.getToolBindings(context);
+        const sdkCode = this.sdkGenerator.generateIsolateSDK(bindings, allowedTools);
+
+        try {
+            return await executor.execute(code, effectiveLimits, context, { sdkCode });
+        } catch (err: any) {
+            return this.createErrorResult(ConduitError.InternalError, err.message);
+        }
+    }
+
+    private createErrorResult(code: number, message: string): ExecutionResult {
+        return {
+            stdout: '',
+            stderr: '',
+            exitCode: null,
+            error: { code, message }
+        };
+    }
+
+    async shutdown(): Promise<void> {
+        await this.executorRegistry.shutdownAll();
+    }
+
+    async warmup(): Promise<void> {
+        const pythonExecutor = this.executorRegistry.get('python');
+        if (pythonExecutor && 'warmup' in pythonExecutor) {
+            // Cast to any because warmup is not in general Executor interface yet
+            await (pythonExecutor as any).warmup(this.defaultLimits);
+        }
+    }
+
+    async healthCheck(): Promise<any> {
+        const pythonExecutor = this.executorRegistry.get('python');
+        if (pythonExecutor && 'healthCheck' in pythonExecutor) {
+            return (pythonExecutor as any).healthCheck();
+        }
+        return { status: 'ok' };
+    }
+}

package/src/core/interfaces/app.config.ts

@@ -0,0 +1,17 @@
+import { ResourceLimits } from '../config.service.js';
+import { UpstreamInfo } from '../../gateway/upstream.client.js';
+
+export interface AppConfig {
+    port: number;
+    nodeEnv: 'development' | 'production' | 'test';
+    logLevel: 'debug' | 'info' | 'warn' | 'error';
+    resourceLimits: ResourceLimits;
+    secretRedactionPatterns: string[];
+    ipcBearerToken: string;
+    maxConcurrent: number;
+    denoMaxPoolSize: number;
+    pyodideMaxPoolSize: number;
+    metricsUrl: string;
+    opsPort?: number;
+    upstreams?: UpstreamInfo[];
+}

package/src/core/interfaces/executor.interface.ts

@@ -0,0 +1,31 @@
+import { ResourceLimits } from '../config.service.js';
+import { ExecutionContext } from '../execution.context.js';
+
+export interface ExecutionResult {
+    stdout: string;
+    stderr: string;
+    exitCode: number | null;
+    error?: {
+        code: number;
+        message: string;
+    };
+}
+
+export interface ExecutorConfig {
+    ipcAddress?: string;
+    ipcToken?: string;
+    sdkCode?: string;
+}
+
+export interface Executor {
+    execute(
+        code: string,
+        limits: ResourceLimits,
+        context: ExecutionContext,
+        config?: ExecutorConfig
+    ): Promise<ExecutionResult>;
+
+    shutdown?(): Promise<void>;
+    healthCheck?(): Promise<{ status: string; workers?: number; detail?: string }>;
+    warmup?(limits?: ResourceLimits): Promise<void>;
+}

package/src/core/interfaces/middleware.interface.ts

@@ -0,0 +1,12 @@
+import { ExecutionContext } from '../execution.context.js';
+import { JSONRPCRequest, JSONRPCResponse } from '../types.js';
+
+export type NextFunction = () => Promise<JSONRPCResponse>;
+
+export interface Middleware {
+    handle(
+        request: JSONRPCRequest,
+        context: ExecutionContext,
+        next: NextFunction
+    ): Promise<JSONRPCResponse>;
+}

package/src/core/interfaces/url.validator.interface.ts

@@ -0,0 +1,3 @@
+export interface IUrlValidator {
+    validateUrl(url: string): Promise<{ valid: boolean; message?: string; resolvedIp?: string }>;
+}

package/src/core/logger.ts

@@ -0,0 +1,64 @@
+import pino from 'pino';
+import { AsyncLocalStorage } from 'node:async_hooks';
+import { ConfigService } from './config.service.js';
+
+export const loggerStorage = new AsyncLocalStorage<{ correlationId: string }>();
+
+export function createLogger(configService: ConfigService) {
+    const logLevel = configService.get('logLevel');
+    const redactionPatterns = configService.get('secretRedactionPatterns');
+    const secretPatterns = redactionPatterns.map(p => new RegExp(p, 'g'));
+
+    const redactString = (str: string) => {
+        let result = str;
+        for (const pattern of secretPatterns) {
+            result = result.replace(pattern, '[REDACTED]');
+        }
+        return result;
+    };
+
+    return pino({
+        level: logLevel,
+        hooks: {
+            logMethod(inputArgs, method) {
+                const redactedArgs = inputArgs.map(arg => {
+                    try {
+                        if (typeof arg === 'string') {
+                            return redactString(arg);
+                        }
+                        if (typeof arg === 'object' && arg !== null) {
+                            // Shallow clone and redact keys if they are strings
+                            const clone = { ...arg } as any;
+                            for (const key in clone) {
+                                if (typeof clone[key] === 'string') {
+                                    clone[key] = redactString(clone[key]);
+                                }
+                            }
+                            return clone;
+                        }
+                    } catch (err) {
+                        return '[REDACTION_ERROR]';
+                    }
+                    return arg;
+                });
+                return method.apply(this, redactedArgs as any);
+            }
+        },
+        redact: {
+            paths: ['toolParams.*', 'headers.Authorization', 'headers.authorization', 'params.token'],
+            censor: '[REDACTED]',
+        },
+        mixin() {
+            const store = loggerStorage.getStore();
+            return {
+                correlationId: store?.correlationId,
+            };
+        },
+        transport: configService.get('nodeEnv') === 'development' ? {
+            target: 'pino-pretty',
+            options: {
+                colorize: true,
+            }
+        } : undefined,
+    });
+}