@cogitator-ai/self-modifying 0.1.0

package/src/tool-generation/tool-sandbox.ts

@@ -0,0 +1,332 @@
+import type {
+  ToolSandboxConfig,
+  ToolSandboxResult,
+  GeneratedTool,
+} from '@cogitator-ai/types';
+
+export const DEFAULT_SANDBOX_CONFIG: ToolSandboxConfig = {
+  enabled: true,
+  maxExecutionTime: 5000,
+  maxMemory: 50 * 1024 * 1024,
+  allowedModules: [],
+  isolationLevel: 'strict',
+};
+
+interface SandboxContext {
+  console: {
+    log: (...args: unknown[]) => void;
+    warn: (...args: unknown[]) => void;
+    error: (...args: unknown[]) => void;
+  };
+  Math: typeof Math;
+  JSON: typeof JSON;
+  Date: typeof Date;
+  Array: typeof Array;
+  Object: typeof Object;
+  String: typeof String;
+  Number: typeof Number;
+  Boolean: typeof Boolean;
+  RegExp: typeof RegExp;
+  Map: typeof Map;
+  Set: typeof Set;
+  Promise: typeof Promise;
+  Error: typeof Error;
+  TypeError: typeof TypeError;
+  RangeError: typeof RangeError;
+}
+
+export class ToolSandbox {
+  private readonly config: ToolSandboxConfig;
+  private readonly logs: string[] = [];
+
+  constructor(config: Partial<ToolSandboxConfig> = {}) {
+    this.config = { ...DEFAULT_SANDBOX_CONFIG, ...config };
+  }
+
+  async execute(
+    tool: GeneratedTool,
+    params: unknown
+  ): Promise<ToolSandboxResult> {
+    const startTime = Date.now();
+    this.logs.length = 0;
+
+    if (!this.config.enabled) {
+      return this.executeUnsandboxed(tool, params, startTime);
+    }
+
+    try {
+      this.validateImplementation(tool.implementation);
+
+      const context = this.createContext();
+      const wrappedCode = this.wrapImplementation(tool.implementation);
+
+      const result = await this.executeWithTimeout(
+        wrappedCode,
+        context,
+        params,
+        this.config.maxExecutionTime
+      );
+
+      return {
+        success: true,
+        result,
+        executionTime: Date.now() - startTime,
+        memoryUsed: this.estimateMemoryUsage(result),
+        logs: [...this.logs],
+      };
+    } catch (error) {
+      return {
+        success: false,
+        error: error instanceof Error ? error.message : String(error),
+        executionTime: Date.now() - startTime,
+        memoryUsed: 0,
+        logs: [...this.logs],
+      };
+    }
+  }
+
+  async testWithCases(
+    tool: GeneratedTool,
+    testCases: Array<{ input: unknown; expectedOutput?: unknown; shouldThrow?: boolean }>
+  ): Promise<{
+    passed: number;
+    failed: number;
+    results: Array<{
+      input: unknown;
+      output?: unknown;
+      error?: string;
+      passed: boolean;
+      executionTime: number;
+    }>;
+  }> {
+    const results: Array<{
+      input: unknown;
+      output?: unknown;
+      error?: string;
+      passed: boolean;
+      executionTime: number;
+    }> = [];
+
+    for (const testCase of testCases) {
+      const execResult = await this.execute(tool, testCase.input);
+
+      let passed = false;
+      if (testCase.shouldThrow) {
+        passed = !execResult.success;
+      } else if (testCase.expectedOutput !== undefined) {
+        passed =
+          execResult.success &&
+          this.deepEqual(execResult.result, testCase.expectedOutput);
+      } else {
+        passed = execResult.success;
+      }
+
+      results.push({
+        input: testCase.input,
+        output: execResult.result,
+        error: execResult.error,
+        passed,
+        executionTime: execResult.executionTime,
+      });
+    }
+
+    return {
+      passed: results.filter((r) => r.passed).length,
+      failed: results.filter((r) => !r.passed).length,
+      results,
+    };
+  }
+
+  private validateImplementation(code: string): void {
+    const forbidden = [
+      /\beval\s*\(/,
+      /\bFunction\s*\(/,
+      /\bnew\s+Function\s*\(/,
+      /\bimport\s*\(/,
+      /\brequire\s*\(/,
+      /\bprocess\./,
+      /\bglobal\./,
+      /\bglobalThis\./,
+      /\bwindow\./,
+      /\bdocument\./,
+      /\bchild_process/,
+      /\bfs\./,
+      /\bhttp\./,
+      /\bhttps\./,
+      /\bnet\./,
+      /\bdns\./,
+      /\bos\./,
+      /\bexec\s*\(/,
+      /\bspawn\s*\(/,
+      /__proto__/,
+      /\bconstructor\s*\[/,
+    ];
+
+    if (this.config.isolationLevel === 'strict') {
+      forbidden.push(
+        /\bfetch\s*\(/,
+        /\bXMLHttpRequest/,
+        /\bWebSocket/,
+        /\bsetTimeout\s*\(/,
+        /\bsetInterval\s*\(/
+      );
+    }
+
+    for (const pattern of forbidden) {
+      if (pattern.test(code)) {
+        throw new Error(
+          `Security violation: forbidden pattern detected - ${pattern.source}`
+        );
+      }
+    }
+
+    const lines = code.split('\n').length;
+    if (lines > 200) {
+      throw new Error(`Implementation too large: ${lines} lines (max 200)`);
+    }
+  }
+
+  private createContext(): SandboxContext {
+    const sandbox = this;
+
+    return {
+      console: {
+        log: (...args: unknown[]) => {
+          sandbox.logs.push(`[LOG] ${args.map(String).join(' ')}`);
+        },
+        warn: (...args: unknown[]) => {
+          sandbox.logs.push(`[WARN] ${args.map(String).join(' ')}`);
+        },
+        error: (...args: unknown[]) => {
+          sandbox.logs.push(`[ERROR] ${args.map(String).join(' ')}`);
+        },
+      },
+      Math,
+      JSON,
+      Date,
+      Array,
+      Object,
+      String,
+      Number,
+      Boolean,
+      RegExp,
+      Map,
+      Set,
+      Promise,
+      Error,
+      TypeError,
+      RangeError,
+    };
+  }
+
+  private wrapImplementation(code: string): string {
+    return `
+      "use strict";
+      return (async function sandboxedExecution(params, context) {
+        const { console, Math, JSON, Date, Array, Object, String, Number, Boolean, RegExp, Map, Set, Promise, Error, TypeError, RangeError } = context;
+
+        ${code}
+
+        if (typeof execute === 'function') {
+          return await execute(params);
+        }
+        throw new Error('Implementation must define an execute function');
+      });
+    `;
+  }
+
+  private async executeWithTimeout(
+    wrappedCode: string,
+    context: SandboxContext,
+    params: unknown,
+    timeout: number
+  ): Promise<unknown> {
+    return new Promise((resolve, reject) => {
+      const timer = setTimeout(() => {
+        reject(new Error(`Execution timeout: exceeded ${timeout}ms`));
+      }, timeout);
+
+      try {
+        const factory = new Function(wrappedCode);
+        const executor = factory();
+
+        Promise.resolve(executor(params, context))
+          .then((result) => {
+            clearTimeout(timer);
+            resolve(result);
+          })
+          .catch((error) => {
+            clearTimeout(timer);
+            reject(error);
+          });
+      } catch (error) {
+        clearTimeout(timer);
+        reject(error);
+      }
+    });
+  }
+
+  private async executeUnsandboxed(
+    tool: GeneratedTool,
+    params: unknown,
+    startTime: number
+  ): Promise<ToolSandboxResult> {
+    try {
+      const factory = new Function(`
+        "use strict";
+        ${tool.implementation}
+        return execute;
+      `);
+      const execute = factory();
+      const result = await execute(params);
+
+      return {
+        success: true,
+        result,
+        executionTime: Date.now() - startTime,
+        memoryUsed: this.estimateMemoryUsage(result),
+        logs: [],
+      };
+    } catch (error) {
+      return {
+        success: false,
+        error: error instanceof Error ? error.message : String(error),
+        executionTime: Date.now() - startTime,
+        memoryUsed: 0,
+        logs: [],
+      };
+    }
+  }
+
+  private estimateMemoryUsage(value: unknown): number {
+    try {
+      const str = JSON.stringify(value);
+      return str.length * 2;
+    } catch {
+      return 0;
+    }
+  }
+
+  private deepEqual(a: unknown, b: unknown): boolean {
+    if (a === b) return true;
+    if (typeof a !== typeof b) return false;
+    if (a === null || b === null) return a === b;
+
+    if (Array.isArray(a) && Array.isArray(b)) {
+      if (a.length !== b.length) return false;
+      return a.every((val, i) => this.deepEqual(val, b[i]));
+    }
+
+    if (typeof a === 'object' && typeof b === 'object') {
+      const aObj = a as Record<string, unknown>;
+      const bObj = b as Record<string, unknown>;
+      const keysA = Object.keys(aObj);
+      const keysB = Object.keys(bObj);
+
+      if (keysA.length !== keysB.length) return false;
+      return keysA.every((key) => this.deepEqual(aObj[key], bObj[key]));
+    }
+
+    return false;
+  }
+}

package/src/tool-generation/tool-validator.ts

@@ -0,0 +1,365 @@
+import type {
+  GeneratedTool,
+  ToolValidationResult,
+  LLMBackend,
+  ToolSelfGenerationConfig,
+} from '@cogitator-ai/types';
+import { ToolSandbox } from './tool-sandbox';
+import { buildToolValidationPrompt, parseValidationResponse } from './prompts';
+
+export interface ToolValidatorOptions {
+  llm?: LLMBackend;
+  config: ToolSelfGenerationConfig;
+}
+
+interface ValidationRule {
+  id: string;
+  name: string;
+  severity: 'error' | 'warning' | 'info';
+  check: (code: string, tool: GeneratedTool) => string | null;
+}
+
+const STATIC_VALIDATION_RULES: ValidationRule[] = [
+  {
+    id: 'no_eval',
+    name: 'No eval() or Function constructor',
+    severity: 'error',
+    check: (code) => {
+      if (/\beval\s*\(/.test(code)) return 'Uses eval()';
+      if (/\bnew\s+Function\s*\(/.test(code)) return 'Uses Function constructor';
+      return null;
+    },
+  },
+  {
+    id: 'no_dynamic_import',
+    name: 'No dynamic imports',
+    severity: 'error',
+    check: (code) => {
+      if (/\bimport\s*\(/.test(code)) return 'Uses dynamic import()';
+      if (/\brequire\s*\(/.test(code)) return 'Uses require()';
+      return null;
+    },
+  },
+  {
+    id: 'no_global_access',
+    name: 'No global object access',
+    severity: 'error',
+    check: (code) => {
+      if (/\bprocess\./.test(code)) return 'Accesses process object';
+      if (/\bglobal\./.test(code)) return 'Accesses global object';
+      if (/\bglobalThis\./.test(code)) return 'Accesses globalThis';
+      return null;
+    },
+  },
+  {
+    id: 'no_prototype_pollution',
+    name: 'No prototype pollution',
+    severity: 'error',
+    check: (code) => {
+      if (/__proto__/.test(code)) return 'Uses __proto__';
+      if (/\.prototype\s*=/.test(code)) return 'Modifies prototype';
+      if (/Object\.setPrototypeOf/.test(code)) return 'Uses setPrototypeOf';
+      return null;
+    },
+  },
+  {
+    id: 'no_infinite_loops',
+    name: 'No obvious infinite loops',
+    severity: 'warning',
+    check: (code) => {
+      if (/while\s*\(\s*true\s*\)/.test(code) && !/break/.test(code)) {
+        return 'Contains while(true) without break';
+      }
+      if (/for\s*\(\s*;\s*;\s*\)/.test(code) && !/break/.test(code)) {
+        return 'Contains for(;;) without break';
+      }
+      return null;
+    },
+  },
+  {
+    id: 'has_execute_function',
+    name: 'Has execute function',
+    severity: 'error',
+    check: (code) => {
+      const patterns = [
+        /(?:async\s+)?function\s+execute\s*\(/,
+        /const\s+execute\s*=\s*(?:async\s*)?\(/,
+        /const\s+execute\s*=\s*(?:async\s+)?function/,
+        /let\s+execute\s*=\s*(?:async\s*)?\(/,
+        /execute\s*=\s*(?:async\s*)?\(/,
+      ];
+      const hasExecute = patterns.some((p) => p.test(code));
+      return hasExecute ? null : 'Missing execute function';
+    },
+  },
+  {
+    id: 'reasonable_length',
+    name: 'Reasonable code length',
+    severity: 'warning',
+    check: (code) => {
+      const lines = code.split('\n').length;
+      if (lines > 150) return `Too long: ${lines} lines (recommended < 150)`;
+      return null;
+    },
+  },
+  {
+    id: 'no_shell_commands',
+    name: 'No shell command execution',
+    severity: 'error',
+    check: (code) => {
+      if (/child_process/.test(code)) return 'Uses child_process';
+      if (/\bexec\s*\(/.test(code)) return 'Uses exec()';
+      if (/\bspawn\s*\(/.test(code)) return 'Uses spawn()';
+      if (/\bexecSync\s*\(/.test(code)) return 'Uses execSync()';
+      return null;
+    },
+  },
+  {
+    id: 'no_file_system',
+    name: 'No file system access',
+    severity: 'error',
+    check: (code) => {
+      if (/\bfs\./.test(code)) return 'Uses fs module';
+      if (/readFileSync|writeFileSync/.test(code)) return 'Uses file system sync methods';
+      if (/readFile|writeFile/.test(code)) return 'Uses file system methods';
+      return null;
+    },
+  },
+  {
+    id: 'has_error_handling',
+    name: 'Has error handling',
+    severity: 'info',
+    check: (code) => {
+      if (!/try\s*\{/.test(code) && !/\.catch\s*\(/.test(code)) {
+        return 'No try-catch or .catch() error handling';
+      }
+      return null;
+    },
+  },
+];
+
+export class ToolValidator {
+  private readonly llm?: LLMBackend;
+  private readonly config: ToolSelfGenerationConfig;
+  private readonly sandbox: ToolSandbox;
+  private readonly customRules: ValidationRule[] = [];
+
+  constructor(options: ToolValidatorOptions) {
+    this.llm = options.llm;
+    this.config = options.config;
+    this.sandbox = new ToolSandbox(options.config.sandboxConfig);
+  }
+
+  async validate(
+    tool: GeneratedTool,
+    testCases?: Array<{ input: unknown; expectedOutput?: unknown; shouldThrow?: boolean }>
+  ): Promise<ToolValidationResult> {
+    const securityIssues: string[] = [];
+    const logicIssues: string[] = [];
+    const edgeCases: string[] = [];
+    const suggestions: string[] = [];
+
+    const staticResult = this.runStaticAnalysis(tool);
+    securityIssues.push(...staticResult.errors);
+    suggestions.push(...staticResult.warnings);
+
+    if (securityIssues.length > 0) {
+      return {
+        isValid: false,
+        securityIssues,
+        logicIssues,
+        edgeCases,
+        suggestions,
+        testResults: [],
+        overallScore: 0,
+      };
+    }
+
+    const effectiveTestCases = testCases || this.generateBasicTestCases(tool);
+    const sandboxResult = await this.sandbox.testWithCases(tool, effectiveTestCases);
+
+    const testResults = sandboxResult.results.map((r) => ({
+      input: r.input,
+      output: r.output,
+      passed: r.passed,
+      error: r.error,
+    }));
+
+    if (sandboxResult.failed > 0) {
+      const failedTests = sandboxResult.results.filter((r) => !r.passed);
+      logicIssues.push(
+        ...failedTests.map((t) => `Test failed for input ${JSON.stringify(t.input)}: ${t.error || 'unexpected output'}`)
+      );
+    }
+
+    if (this.llm && this.config.requireLLMValidation) {
+      const llmResult = await this.runLLMValidation(tool, effectiveTestCases);
+      if (llmResult) {
+        securityIssues.push(...llmResult.securityIssues);
+        logicIssues.push(...llmResult.logicIssues);
+        edgeCases.push(...llmResult.edgeCases);
+        suggestions.push(...llmResult.suggestions);
+      }
+    }
+
+    const score = this.calculateScore(
+      securityIssues.length,
+      logicIssues.length,
+      edgeCases.length,
+      sandboxResult.passed,
+      sandboxResult.failed
+    );
+
+    return {
+      isValid: securityIssues.length === 0 && logicIssues.length === 0 && score >= 0.7,
+      securityIssues: [...new Set(securityIssues)],
+      logicIssues: [...new Set(logicIssues)],
+      edgeCases: [...new Set(edgeCases)],
+      suggestions: [...new Set(suggestions)],
+      testResults,
+      overallScore: score,
+    };
+  }
+
+  addRule(rule: ValidationRule): void {
+    this.customRules.push(rule);
+  }
+
+  private runStaticAnalysis(tool: GeneratedTool): {
+    errors: string[];
+    warnings: string[];
+  } {
+    const errors: string[] = [];
+    const warnings: string[] = [];
+    const allRules = [...STATIC_VALIDATION_RULES, ...this.customRules];
+
+    for (const rule of allRules) {
+      const result = rule.check(tool.implementation, tool);
+      if (result) {
+        if (rule.severity === 'error') {
+          errors.push(`[${rule.id}] ${result}`);
+        } else if (rule.severity === 'warning') {
+          warnings.push(`[${rule.id}] ${result}`);
+        }
+      }
+    }
+
+    return { errors, warnings };
+  }
+
+  private async runLLMValidation(
+    tool: GeneratedTool,
+    testCases: Array<{ input: unknown; expectedOutput?: unknown; shouldThrow?: boolean }>
+  ): Promise<ToolValidationResult | null> {
+    if (!this.llm) return null;
+
+    try {
+      const prompt = buildToolValidationPrompt(
+        tool,
+        testCases.map((tc) => ({
+          input: tc.input,
+          expectedBehavior: tc.expectedOutput
+            ? `Should return ${JSON.stringify(tc.expectedOutput)}`
+            : tc.shouldThrow
+              ? 'Should throw an error'
+              : 'Should execute successfully',
+        }))
+      );
+
+      const response = await this.callLLM([
+        {
+          role: 'system',
+          content: `You are a security auditor and code reviewer.
+Analyze code for security vulnerabilities, logic errors, and edge cases.
+Be thorough but practical - focus on real issues.`,
+        },
+        { role: 'user', content: prompt },
+      ], 0.2);
+
+      return response ? parseValidationResponse(response.content) : null;
+    } catch {
+      return null;
+    }
+  }
+
+  private generateBasicTestCases(
+    tool: GeneratedTool
+  ): Array<{ input: unknown; expectedOutput?: unknown; shouldThrow?: boolean }> {
+    const testCases: Array<{ input: unknown; expectedOutput?: unknown; shouldThrow?: boolean }> = [];
+    const params = tool.parameters;
+
+    if (params.type === 'object' && params.properties) {
+      const validInput: Record<string, unknown> = {};
+      const properties = params.properties as Record<
+        string,
+        { type?: string; default?: unknown }
+      >;
+
+      for (const [key, schema] of Object.entries(properties)) {
+        validInput[key] = this.generateSampleValue(schema.type, schema.default);
+      }
+      testCases.push({ input: validInput });
+
+      const required = params.required as string[] | undefined;
+      testCases.push({ input: {}, shouldThrow: (required?.length ?? 0) > 0 });
+
+      testCases.push({ input: null, shouldThrow: true });
+    }
+
+    return testCases;
+  }
+
+  private generateSampleValue(type?: string, defaultValue?: unknown): unknown {
+    if (defaultValue !== undefined) return defaultValue;
+
+    switch (type) {
+      case 'string':
+        return 'test';
+      case 'number':
+      case 'integer':
+        return 42;
+      case 'boolean':
+        return true;
+      case 'array':
+        return [];
+      case 'object':
+        return {};
+      default:
+        return null;
+    }
+  }
+
+  private calculateScore(
+    securityCount: number,
+    logicCount: number,
+    edgeCaseCount: number,
+    testsPassed: number,
+    testsFailed: number
+  ): number {
+    if (securityCount > 0) return 0;
+
+    let score = 1.0;
+
+    score -= logicCount * 0.2;
+    score -= edgeCaseCount * 0.05;
+
+    const totalTests = testsPassed + testsFailed;
+    if (totalTests > 0) {
+      const testScore = testsPassed / totalTests;
+      score = score * 0.6 + testScore * 0.4;
+    }
+
+    return Math.max(0, Math.min(1, score));
+  }
+
+  private async callLLM(
+    messages: Array<{ role: 'system' | 'user' | 'assistant'; content: string }>,
+    temperature: number
+  ) {
+    if (!this.llm) return null;
+    if (this.llm.complete) {
+      return this.llm.complete({ messages, temperature });
+    }
+    return this.llm.chat({ model: 'default', messages, temperature });
+  }
+}

package/src/utils/index.ts

@@ -0,0 +1 @@
+export { llmChat } from './llm-helper';

package/src/utils/llm-helper.ts

@@ -0,0 +1,24 @@
+import type { LLMBackend, Message } from '@cogitator-ai/types';
+
+export async function llmChat(
+  llm: LLMBackend,
+  messages: Message[],
+  options?: { temperature?: number; maxTokens?: number }
+): Promise<string> {
+  if (llm.complete) {
+    const response = await llm.complete({
+      messages,
+      temperature: options?.temperature,
+      maxTokens: options?.maxTokens,
+    });
+    return response.content;
+  }
+
+  const response = await llm.chat({
+    model: 'default',
+    messages,
+    temperature: options?.temperature,
+    maxTokens: options?.maxTokens,
+  });
+  return response.content;
+}