@koi-language/koi 1.0.0

package/src/compiler/build-optimizer.js

@@ -0,0 +1,447 @@
+/**
+ * Build-Time Optimizer
+ *
+ * Pre-computes expensive operations during compilation to avoid runtime overhead:
+ * - Embeddings for agent affordances (with persistent caching)
+ * - Static agent metadata
+ * - Any other cacheable data
+ *
+ * Uses SHA-256 content hashing to detect changes and avoid redundant API calls.
+ */
+
+import { LLMProvider } from '../runtime/llm-provider.js';
+import { CacheManager } from './cache-manager.js';
+
+export class BuildTimeOptimizer {
+  constructor(config = {}) {
+    this.enableCache = config.cache !== false;
+    this.verbose = config.verbose || false;
+    this.llmProvider = null;  // For embeddings
+    this.chatProvider = null;  // For code introspection
+    this.cacheManager = new CacheManager({
+      verbose: config.verbose || false
+    });
+  }
+
+  /**
+   * Extract and pre-compute affordances from AST (with cache)
+   * @param ast - Abstract syntax tree
+   * @param sourceContent - Original source code content (for cache hashing)
+   * @param sourcePath - Path to source file (for cache tracking)
+   */
+  async optimizeAST(ast, sourceContent = '', sourcePath = 'unknown') {
+    if (!this.enableCache) {
+      if (this.verbose) console.log('[BuildOptimizer] Cache disabled, skipping');
+      return null;
+    }
+
+    console.log('🔄 [BuildOptimizer] Checking cache...');
+
+    // Check if we have cached data for this exact source content
+    const cached = this.cacheManager.get(sourceContent, sourcePath);
+    if (cached) {
+      const totalAffordances = (cached.metadata.totalAffordances || 0) + (cached.metadata.totalSkillAffordances || 0);
+      console.log(`✅ [BuildOptimizer] Using cached embeddings (${totalAffordances} affordances)`);
+      console.log(`   Last generated: ${new Date(cached.metadata.generatedAt).toLocaleString()}`);
+      console.log(`   💰 Saved API calls!`);
+      return cached;
+    }
+
+    // Cache miss - generate embeddings
+    console.log('🔄 [BuildOptimizer] Cache miss, pre-computing embeddings...');
+
+    const affordances = {};
+    const skillAffordances = {};
+    let totalEmbeddings = 0;
+    let totalSkillEmbeddings = 0;
+
+    // Find all declarations in AST
+    for (const decl of ast.declarations) {
+      if (decl.type === 'AgentDecl') {
+        const agentAffordances = await this.extractAgentAffordances(decl);
+
+        if (agentAffordances && Object.keys(agentAffordances).length > 0) {
+          affordances[decl.name.name] = agentAffordances;
+          totalEmbeddings += Object.keys(agentAffordances).length;
+        }
+      } else if (decl.type === 'SkillDecl') {
+        const skillData = await this.extractSkillAffordance(decl);
+
+        if (skillData) {
+          skillAffordances[decl.name.name] = skillData;
+          totalSkillEmbeddings++;
+        }
+      }
+    }
+
+    console.log(`✅ [BuildOptimizer] Pre-computed ${totalEmbeddings + totalSkillEmbeddings} embeddings (${totalEmbeddings} agents, ${totalSkillEmbeddings} skills)`);
+
+    const result = {
+      affordances,
+      skillAffordances,
+      metadata: {
+        generatedAt: Date.now(),
+        totalAgents: Object.keys(affordances).length,
+        totalAffordances: totalEmbeddings,
+        totalSkills: Object.keys(skillAffordances).length,
+        totalSkillAffordances: totalSkillEmbeddings
+      }
+    };
+
+    // Store in cache
+    this.cacheManager.set(sourceContent, sourcePath, result);
+
+    return result;
+  }
+
+  /**
+   * Extract and pre-compute affordances from AST (without cache)
+   * Generates embeddings but doesn't store them in persistent cache
+   * @param ast - Abstract syntax tree
+   */
+  async optimizeASTWithoutCache(ast) {
+    const affordances = {};
+    let totalEmbeddings = 0;
+
+    // Find all agent declarations in AST
+    for (const decl of ast.declarations) {
+      if (decl.type === 'AgentDecl') {
+        const agentAffordances = await this.extractAgentAffordances(decl);
+
+        if (agentAffordances && Object.keys(agentAffordances).length > 0) {
+          affordances[decl.name.name] = agentAffordances;
+          totalEmbeddings += Object.keys(agentAffordances).length;
+        }
+      }
+    }
+
+    console.log(`✅ [BuildOptimizer] Pre-computed ${totalEmbeddings} embeddings (no cache)`);
+
+    return {
+      affordances,
+      metadata: {
+        generatedAt: Date.now(),
+        totalAgents: Object.keys(affordances).length,
+        totalAffordances: totalEmbeddings
+      }
+    };
+  }
+
+  /**
+   * Extract affordances for a single agent
+   */
+  async extractAgentAffordances(agentNode) {
+    const affordances = {};
+
+    // Find event handlers
+    const eventHandlers = agentNode.body.filter(b => b.type === 'EventHandler');
+
+    if (eventHandlers.length === 0) {
+      return affordances;
+    }
+
+    if (this.verbose) {
+      console.log(`  [Agent:${agentNode.name.name}] Extracting ${eventHandlers.length} affordances...`);
+    }
+
+    for (const handler of eventHandlers) {
+      const eventName = handler.event.name;
+
+      // Try to find playbook
+      const playbook = this.findPlaybookForHandler(handler);
+
+      let description;
+      let confidence;
+      let hasPlaybook = false;
+
+      if (playbook) {
+        // Extract description from playbook
+        description = this.inferIntentFromPlaybook(playbook, eventName);
+        confidence = 0.9;
+        hasPlaybook = true;
+      } else {
+        // No playbook - use code introspection
+        description = await this.introspectHandlerCode(handler, eventName);
+        confidence = 0.8; // High confidence since we analyzed the actual code
+        hasPlaybook = false;
+      }
+
+      // Validate description
+      if (!description || description.trim() === '') {
+        console.warn(`⚠️  [BuildOptimizer] Empty description for ${agentNode.name.name}.${eventName}, skipping embedding`);
+        description = `Handler: ${eventName}`;
+      }
+
+      // Generate embedding
+      const embedding = await this.generateEmbedding(description);
+
+      affordances[eventName] = {
+        description: description,
+        embedding: embedding,
+        confidence: confidence,
+        hasPlaybook: hasPlaybook
+      };
+
+      if (this.verbose) {
+        console.log(`    ✓ ${eventName}: "${description.substring(0, 50)}..."`);
+      }
+    }
+
+    return affordances;
+  }
+
+  /**
+   * Find playbook for an event handler
+   */
+  findPlaybookForHandler(handler) {
+    // Check if handler has a playbook statement
+    for (const stmt of handler.body) {
+      if (stmt.type === 'PlaybookStatement') {
+        return stmt.content.value;
+      }
+    }
+
+    return null;
+  }
+
+  /**
+   * Infer intent from playbook text
+   */
+  inferIntentFromPlaybook(playbook, eventName) {
+    // Handle non-string playbooks
+    if (!playbook || typeof playbook !== 'string') {
+      return this.humanizeEventName(eventName);
+    }
+
+    // Remove template literals
+    const cleanText = playbook
+      .replace(/\$\{[^}]+\}/g, '')
+      .split('\n')
+      .map(line => line.trim())
+      .filter(line => line.length > 0 && !line.startsWith('//') && !line.startsWith('Return'))
+      .slice(0, 3) // Take first 3 lines
+      .join(' ');
+
+    if (cleanText.length > 10 && cleanText.length < 200) {
+      return cleanText;
+    }
+
+    // Fallback
+    return this.humanizeEventName(eventName);
+  }
+
+  /**
+   * Introspect handler code using LLM to understand what it does
+   * @param handler - Event handler AST node
+   * @param eventName - Name of the event
+   * @returns Description of what the handler does
+   */
+  async introspectHandlerCode(handler, eventName) {
+    // Serialize the handler body to source code
+    const codeLines = [];
+
+    for (const stmt of handler.body) {
+      const code = this.serializeStatement(stmt);
+      if (code) {
+        codeLines.push(code);
+      }
+    }
+
+    const sourceCode = codeLines.join('\n');
+
+    // If no code, fallback to event name
+    if (!sourceCode || sourceCode.trim().length === 0) {
+      return this.humanizeEventName(eventName);
+    }
+
+    // Use LLM to analyze the code
+    if (!this.chatProvider) {
+      this.initChatProvider();
+    }
+
+    const introspectionPrompt = `Analyze this event handler code and provide a concise description (1-2 sentences) of what it does.
+
+Event name: ${eventName}
+
+Code:
+\`\`\`javascript
+${sourceCode}
+\`\`\`
+
+Focus on:
+- What operations it performs
+- What data it processes or returns
+- Its main purpose or capability
+
+Return ONLY the description text, no markdown, no explanations, no prefix like "This handler...". Just a direct description.`;
+
+    try {
+      const response = await this.chatProvider.executeOpenAI(introspectionPrompt, false);
+
+      // Clean up the response
+      const description = response
+        .replace(/^(This handler|This event handler|This function|The handler|The function)\s*/i, '')
+        .replace(/^(handles?|processes?|performs?)\s*/i, '')
+        .trim();
+
+      if (description && description.length > 10) {
+        return description;
+      }
+    } catch (error) {
+      console.warn(`[BuildOptimizer] Code introspection failed for ${eventName}:`, error.message);
+    }
+
+    // Fallback to humanized event name
+    return this.humanizeEventName(eventName);
+  }
+
+  /**
+   * Serialize an AST statement to source code (simplified)
+   */
+  serializeStatement(stmt) {
+    if (!stmt) return '';
+
+    switch (stmt.type) {
+      case 'ConstDeclaration':
+        return `const ${stmt.name.name} = ...`;
+
+      case 'ReturnStatement':
+        if (stmt.value && stmt.value.type === 'ObjectLiteral') {
+          // Extract object keys
+          const keys = stmt.value.properties?.map(p => p.key?.name || p.key).join(', ') || '';
+          return `return { ${keys} }`;
+        }
+        return 'return ...';
+
+      case 'SendStatement':
+        const role = stmt.role?.name || 'Role';
+        const event = stmt.event?.name || 'event';
+        return `send to ${role}.${event}()`;
+
+      case 'ExpressionStatement':
+        if (stmt.expression?.type === 'CallExpression') {
+          const callee = stmt.expression.callee?.name || stmt.expression.callee?.property?.name || 'function';
+          return `${callee}(...)`;
+        }
+        return '...';
+
+      default:
+        return `// ${stmt.type}`;
+    }
+  }
+
+  /**
+   * Extract affordance for a skill
+   */
+  async extractSkillAffordance(skillNode) {
+    // Skills have an explicit affordance field
+    const affordanceText = skillNode.affordance?.value || skillNode.affordance || '';
+
+    if (!affordanceText || affordanceText.trim().length === 0) {
+      // No affordance defined - use skill name as fallback
+      const description = this.humanizeEventName(skillNode.name.name);
+      return {
+        description,
+        embedding: await this.generateEmbedding(description),
+        confidence: 0.5
+      };
+    }
+
+    // Clean up affordance text (remove extra whitespace/newlines)
+    let cleanDescription = affordanceText
+      .split('\n')
+      .map(line => line.trim())
+      .filter(line => line.length > 0)
+      .join(' ')
+      .trim();
+
+    // Validate description is not empty
+    if (!cleanDescription || cleanDescription.length === 0) {
+      cleanDescription = this.humanizeEventName(skillNode.name.name);
+    }
+
+    // Generate embedding for the affordance
+    const embedding = await this.generateEmbedding(cleanDescription);
+
+    if (this.verbose) {
+      console.log(`    ✓ Skill ${skillNode.name.name}: "${cleanDescription.substring(0, 50)}..."`);
+    }
+
+    return {
+      description: cleanDescription,
+      embedding: embedding,
+      confidence: 0.9
+    };
+  }
+
+  /**
+   * Humanize event name
+   */
+  humanizeEventName(eventName) {
+    return eventName
+      .replace(/([A-Z])/g, ' $1')
+      .replace(/_/g, ' ')
+      .toLowerCase()
+      .trim();
+  }
+
+  /**
+   * Initialize chat LLM provider for code introspection
+   */
+  initChatProvider() {
+    this.chatProvider = new LLMProvider({
+      provider: 'openai',
+      model: 'gpt-4o-mini',  // Fast and cheap model for code analysis
+      temperature: 0.1,
+      maxTokens: 150
+    });
+  }
+
+  /**
+   * Generate embedding for text
+   */
+  async generateEmbedding(text) {
+    if (!this.llmProvider) {
+      this.llmProvider = new LLMProvider({
+        provider: 'openai',
+        model: 'text-embedding-3-small'
+      });
+    }
+
+    try {
+      return await this.llmProvider.getEmbedding(text);
+    } catch (error) {
+      console.warn(`[BuildOptimizer] Failed to generate embedding: ${error.message}`);
+      return null;
+    }
+  }
+
+  /**
+   * Generate JavaScript code for cached data
+   */
+  generateCacheCode(cacheData) {
+    if (!cacheData) {
+      return '';
+    }
+
+    const totalAffordances = (cacheData.metadata.totalAffordances || 0) + (cacheData.metadata.totalSkillAffordances || 0);
+
+    const code = `
+// ============================================================
+// Pre-computed Affordances (Build-time Cache)
+// Generated at: ${new Date(cacheData.metadata.generatedAt).toISOString()}
+// Total agents: ${cacheData.metadata.totalAgents || 0}
+// Total agent affordances: ${cacheData.metadata.totalAffordances || 0}
+// Total skills: ${cacheData.metadata.totalSkills || 0}
+// Total skill affordances: ${cacheData.metadata.totalSkillAffordances || 0}
+// ============================================================
+
+const CACHED_AFFORDANCES = ${JSON.stringify(cacheData.affordances || {}, null, 2)};
+
+const CACHED_SKILL_AFFORDANCES = ${JSON.stringify(cacheData.skillAffordances || {}, null, 2)};
+
+`;
+
+    return code;
+  }
+}

package/src/compiler/cache-manager.js

@@ -0,0 +1,274 @@
+/**
+ * Persistent Cache Manager
+ *
+ * Manages a persistent cache directory for build-time optimizations.
+ * Uses SHA-256 hashing to detect source file changes and avoid
+ * regenerating expensive computations (embeddings, etc.)
+ */
+
+import fs from 'fs';
+import path from 'path';
+import crypto from 'crypto';
+import { fileURLToPath } from 'url';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+export class CacheManager {
+  constructor(config = {}) {
+    // Cache directory relative to project root
+    this.cacheDir = config.cacheDir || path.join(process.cwd(), '.koi-cache');
+    this.verbose = config.verbose || false;
+
+    // Ensure cache directory exists
+    this.ensureCacheDir();
+  }
+
+  /**
+   * Ensure cache directory exists
+   */
+  ensureCacheDir() {
+    if (!fs.existsSync(this.cacheDir)) {
+      fs.mkdirSync(this.cacheDir, { recursive: true });
+      if (this.verbose) {
+        console.log(`[Cache] Created cache directory: ${this.cacheDir}`);
+      }
+    }
+  }
+
+  /**
+   * Compute SHA-256 hash of file content
+   */
+  hashContent(content) {
+    return crypto.createHash('sha256').update(content).digest('hex');
+  }
+
+  /**
+   * Get cache file path for a source file
+   */
+  getCachePath(sourceHash) {
+    return path.join(this.cacheDir, `affordances-${sourceHash}.json`);
+  }
+
+  /**
+   * Get metadata file path
+   */
+  getMetadataPath() {
+    return path.join(this.cacheDir, 'cache-metadata.json');
+  }
+
+  /**
+   * Load cache metadata (tracks all cached files)
+   */
+  loadMetadata() {
+    const metaPath = this.getMetadataPath();
+
+    if (!fs.existsSync(metaPath)) {
+      return {
+        version: '1.0.0',
+        files: {}
+      };
+    }
+
+    try {
+      const content = fs.readFileSync(metaPath, 'utf-8');
+      return JSON.parse(content);
+    } catch (error) {
+      console.warn(`[Cache] Failed to load metadata: ${error.message}`);
+      return {
+        version: '1.0.0',
+        files: {}
+      };
+    }
+  }
+
+  /**
+   * Save cache metadata
+   */
+  saveMetadata(metadata) {
+    const metaPath = this.getMetadataPath();
+
+    try {
+      fs.writeFileSync(metaPath, JSON.stringify(metadata, null, 2));
+    } catch (error) {
+      console.warn(`[Cache] Failed to save metadata: ${error.message}`);
+    }
+  }
+
+  /**
+   * Check if cache exists for source content
+   */
+  has(sourceContent) {
+    const hash = this.hashContent(sourceContent);
+    const cachePath = this.getCachePath(hash);
+    return fs.existsSync(cachePath);
+  }
+
+  /**
+   * Get cached data for source content
+   */
+  get(sourceContent, sourcePath = null) {
+    const hash = this.hashContent(sourceContent);
+    const cachePath = this.getCachePath(hash);
+
+    if (!fs.existsSync(cachePath)) {
+      return null;
+    }
+
+    try {
+      const content = fs.readFileSync(cachePath, 'utf-8');
+      const cached = JSON.parse(content);
+
+      if (this.verbose) {
+        console.log(`[Cache] ✓ Cache hit for ${sourcePath || 'source'} (hash: ${hash.substring(0, 8)}...)`);
+        console.log(`[Cache]   Cached: ${cached.metadata.totalAffordances} affordances from ${new Date(cached.metadata.generatedAt).toLocaleString()}`);
+      }
+
+      return cached;
+    } catch (error) {
+      console.warn(`[Cache] Failed to read cache: ${error.message}`);
+      return null;
+    }
+  }
+
+  /**
+   * Store data in cache
+   */
+  set(sourceContent, sourcePath, data) {
+    const hash = this.hashContent(sourceContent);
+    const cachePath = this.getCachePath(hash);
+
+    // Add cache metadata
+    const cacheEntry = {
+      ...data,
+      cacheMetadata: {
+        sourceHash: hash,
+        sourcePath: sourcePath,
+        cachedAt: Date.now()
+      }
+    };
+
+    try {
+      fs.writeFileSync(cachePath, JSON.stringify(cacheEntry, null, 2));
+
+      if (this.verbose) {
+        console.log(`[Cache] ✓ Cached ${data.metadata.totalAffordances} affordances to ${path.basename(cachePath)}`);
+      }
+
+      // Update metadata index
+      const metadata = this.loadMetadata();
+      metadata.files[sourcePath] = {
+        hash: hash,
+        lastCached: Date.now(),
+        affordanceCount: data.metadata.totalAffordances
+      };
+      this.saveMetadata(metadata);
+
+      return true;
+    } catch (error) {
+      console.warn(`[Cache] Failed to write cache: ${error.message}`);
+      return false;
+    }
+  }
+
+  /**
+   * Clear cache for a specific file or all cache
+   */
+  clear(sourcePath = null) {
+    if (!sourcePath) {
+      // Clear all cache
+      if (fs.existsSync(this.cacheDir)) {
+        const files = fs.readdirSync(this.cacheDir);
+        files.forEach(file => {
+          fs.unlinkSync(path.join(this.cacheDir, file));
+        });
+        console.log(`[Cache] Cleared all cache (${files.length} files)`);
+      }
+      return;
+    }
+
+    // Clear cache for specific file
+    const metadata = this.loadMetadata();
+    const fileInfo = metadata.files[sourcePath];
+
+    if (fileInfo) {
+      const cachePath = this.getCachePath(fileInfo.hash);
+      if (fs.existsSync(cachePath)) {
+        fs.unlinkSync(cachePath);
+        delete metadata.files[sourcePath];
+        this.saveMetadata(metadata);
+        console.log(`[Cache] Cleared cache for ${sourcePath}`);
+      }
+    }
+  }
+
+  /**
+   * Get cache statistics
+   */
+  getStats() {
+    const metadata = this.loadMetadata();
+    const files = Object.keys(metadata.files).length;
+
+    let totalSize = 0;
+    let totalAffordances = 0;
+
+    if (fs.existsSync(this.cacheDir)) {
+      const cacheFiles = fs.readdirSync(this.cacheDir);
+      cacheFiles.forEach(file => {
+        const filePath = path.join(this.cacheDir, file);
+        const stats = fs.statSync(filePath);
+        totalSize += stats.size;
+      });
+    }
+
+    Object.values(metadata.files).forEach(info => {
+      totalAffordances += info.affordanceCount || 0;
+    });
+
+    return {
+      cacheDir: this.cacheDir,
+      files: files,
+      totalAffordances: totalAffordances,
+      totalSize: totalSize,
+      totalSizeFormatted: this.formatBytes(totalSize)
+    };
+  }
+
+  /**
+   * Format bytes to human readable
+   */
+  formatBytes(bytes) {
+    if (bytes === 0) return '0 Bytes';
+    const k = 1024;
+    const sizes = ['Bytes', 'KB', 'MB', 'GB'];
+    const i = Math.floor(Math.log(bytes) / Math.log(k));
+    return Math.round(bytes / Math.pow(k, i) * 100) / 100 + ' ' + sizes[i];
+  }
+
+  /**
+   * Print cache summary
+   */
+  printStats() {
+    const stats = this.getStats();
+
+    console.log(`\n📊 Cache Statistics:`);
+    console.log(`   Location: ${stats.cacheDir}`);
+    console.log(`   Cached files: ${stats.files}`);
+    console.log(`   Total affordances: ${stats.totalAffordances}`);
+    console.log(`   Cache size: ${stats.totalSizeFormatted}`);
+
+    if (stats.files > 0) {
+      const metadata = this.loadMetadata();
+      console.log(`\n   Recent files:`);
+      const sorted = Object.entries(metadata.files)
+        .sort((a, b) => b[1].lastCached - a[1].lastCached)
+        .slice(0, 5);
+
+      sorted.forEach(([filepath, info]) => {
+        const time = new Date(info.lastCached).toLocaleString();
+        console.log(`   • ${path.basename(filepath)} (${info.affordanceCount} affordances, ${time})`);
+      });
+    }
+    console.log('');
+  }
+}