@comfanion/workflow 4.3.0 → 4.5.1

package/src/opencode/tools/codeindex.ts

@@ -0,0 +1,255 @@
+/**
+ * Code Index Status & Management Tool
+ * 
+ * Check indexing status and trigger re-indexing.
+ * Supports multiple indexes: code, docs, config.
+ * 
+ * Usage by model:
+ *   codeindex({ action: "status" })
+ *   codeindex({ action: "status", index: "docs" })
+ *   codeindex({ action: "reindex", index: "code" })
+ *   codeindex({ action: "list" })
+ */
+
+import { tool } from "@opencode-ai/plugin"
+import path from "path"
+import fs from "fs/promises"
+import { glob } from "glob"
+import ignore from "ignore"
+
+// Index presets (duplicated from vectorizer for independence)
+const INDEX_PRESETS: Record<string, { pattern: string; description: string }> = {
+  code: {
+    pattern: '**/*.{js,ts,jsx,tsx,mjs,cjs,py,go,rs,java,kt,swift,c,cpp,h,hpp,cs,rb,php,scala,clj}',
+    description: 'Source code files'
+  },
+  docs: {
+    pattern: '**/*.{md,mdx,txt,rst,adoc}',
+    description: 'Documentation files'
+  },
+  config: {
+    pattern: '**/*.{yaml,yml,json,toml,ini,env,xml}',
+    description: 'Configuration files'
+  },
+  all: {
+    pattern: '**/*.{js,ts,jsx,tsx,mjs,cjs,py,go,rs,java,kt,swift,c,cpp,h,hpp,cs,rb,php,scala,clj,md,mdx,txt,rst,adoc,yaml,yml,json,toml}',
+    description: 'All supported files'
+  }
+}
+
+export default tool({
+  description: `Check codebase index status or trigger re-indexing for semantic search.
+
+Actions:
+- "status" → Show index statistics (specify index or see all)
+- "list" → List all available indexes with stats
+- "reindex" → Re-index files (specify which index)
+
+Available indexes:
+- "code" - Source code files
+- "docs" - Documentation files  
+- "config" - Configuration files
+
+Note: Initial indexing takes ~30s to load the embedding model.`,
+
+  args: {
+    action: tool.schema.enum(["status", "list", "reindex"]).describe("Action to perform"),
+    index: tool.schema.string().optional().default("code").describe("Index name for status/reindex: code, docs, config"),
+  },
+
+  async execute(args, context) {
+    const projectRoot = process.cwd()
+    const vectorizerDir = path.join(projectRoot, ".opencode", "vectorizer")
+    const vectorsDir = path.join(projectRoot, ".opencode", "vectors")
+
+    // Check if vectorizer is installed
+    const isInstalled = await fs.access(path.join(vectorizerDir, "node_modules"))
+      .then(() => true)
+      .catch(() => false)
+
+    if (!isInstalled) {
+      return `❌ Vectorizer not installed.
+
+To install:
+\`\`\`bash
+npx opencode-workflow vectorizer install
+\`\`\`
+
+This will download the embedding model (~100MB) and set up the vector database.`
+    }
+
+    try {
+      const vectorizerModule = path.join(vectorizerDir, "index.js")
+      const { CodebaseIndexer, INDEX_PRESETS: PRESETS } = await import(`file://${vectorizerModule}`)
+
+      // LIST: Show all indexes
+      if (args.action === "list") {
+        const tempIndexer = await new CodebaseIndexer(projectRoot, "code").init()
+        const allStats = await tempIndexer.getAllStats()
+
+        let output = `## Codebase Index Overview\n\n`
+        output += `✅ **Vectorizer installed**\n\n`
+
+        if (allStats.length === 0) {
+          output += `⚠️ **No indexes created yet**\n\n`
+          output += `Create indexes with:\n`
+          output += `\`\`\`bash\n`
+          output += `npx opencode-workflow index --index code   # Source code\n`
+          output += `npx opencode-workflow index --index docs   # Documentation\n`
+          output += `npx opencode-workflow index --index config # Config files\n`
+          output += `\`\`\`\n\n`
+        } else {
+          output += `### Active Indexes\n\n`
+          for (const stat of allStats) {
+            output += `**📁 ${stat.indexName}** - ${stat.description}\n`
+            output += `   Files: ${stat.fileCount}, Chunks: ${stat.chunkCount}\n\n`
+          }
+        }
+
+        output += `### Available Presets\n\n`
+        for (const [name, preset] of Object.entries(PRESETS || INDEX_PRESETS) as [string, any][]) {
+          const exists = allStats.find((s: any) => s.indexName === name)
+          const status = exists ? "✅" : "⬜"
+          output += `${status} **${name}**: ${preset.description}\n`
+        }
+
+        output += `\n### Usage\n`
+        output += `\`\`\`\n`
+        output += `codesearch({ query: "your query", index: "code" })\n`
+        output += `codesearch({ query: "deployment guide", index: "docs" })\n`
+        output += `codesearch({ query: "api keys", searchAll: true })\n`
+        output += `\`\`\``
+
+        return output
+      }
+
+      // STATUS: Show specific index status
+      if (args.action === "status") {
+        const indexName = args.index || "code"
+        const hashesFile = path.join(vectorsDir, indexName, "hashes.json")
+
+        try {
+          const indexer = await new CodebaseIndexer(projectRoot, indexName).init()
+          const stats = await indexer.getStats()
+
+          // Get sample files
+          const hashesContent = await fs.readFile(hashesFile, "utf8")
+          const hashes = JSON.parse(hashesContent)
+          const sampleFiles = Object.keys(hashes).slice(0, 5)
+
+          return `## Index Status: "${indexName}"
+
+✅ **Vectorizer installed**
+✅ **Index active**
+
+**Description:** ${stats.description}
+**Files indexed:** ${stats.fileCount}
+**Chunks:** ${stats.chunkCount}
+
+**Sample indexed files:**
+${sampleFiles.map(f => `- ${f}`).join("\n")}
+${stats.fileCount > 5 ? `- ... and ${stats.fileCount - 5} more` : ""}
+
+**Usage:**
+\`\`\`
+codesearch({ query: "your search query", index: "${indexName}" })
+\`\`\`
+
+To re-index:
+\`\`\`
+codeindex({ action: "reindex", index: "${indexName}" })
+\`\`\``
+
+        } catch {
+          return `## Index Status: "${indexName}"
+
+✅ **Vectorizer installed**
+⚠️ **Index "${indexName}" not created yet**
+
+To create this index:
+\`\`\`bash
+npx opencode-workflow index --index ${indexName}
+\`\`\`
+
+Or use:
+\`\`\`
+codeindex({ action: "reindex", index: "${indexName}" })
+\`\`\``
+        }
+      }
+
+      // REINDEX: Re-index specific index (do it directly, no shell)
+      if (args.action === "reindex") {
+        const indexName = args.index || "code"
+        
+        try {
+          const indexer = await new CodebaseIndexer(projectRoot, indexName).init()
+          
+          // Get pattern from preset
+          const preset = (PRESETS || INDEX_PRESETS)[indexName]
+          const pattern = preset?.pattern || '**/*.{js,ts,py,go,md,yaml,json}'
+          
+          // Load .gitignore
+          let ig = ignore()
+          try {
+            const gitignore = await fs.readFile(path.join(projectRoot, '.gitignore'), 'utf8')
+            ig = ig.add(gitignore)
+          } catch {}
+          ig.add(['node_modules', '.git', 'dist', 'build', '.opencode/vectors', '.opencode/vectorizer'])
+          
+          // Find files
+          const files = await glob(pattern, { cwd: projectRoot, nodir: true })
+          const filtered = files.filter((f: string) => !ig.ignores(f))
+          
+          let indexed = 0
+          let skipped = 0
+          
+          for (const file of filtered) {
+            const filePath = path.join(projectRoot, file)
+            try {
+              const wasIndexed = await indexer.indexFile(filePath)
+              if (wasIndexed) {
+                indexed++
+              } else {
+                skipped++
+              }
+            } catch {
+              // Skip files that can't be read
+            }
+          }
+          
+          // Unload model to free memory
+          await indexer.unloadModel()
+          
+          const stats = await indexer.getStats()
+
+          return `## Re-indexing Complete ✅
+
+**Index:** ${indexName}
+**Description:** ${stats.description}
+**Files found:** ${filtered.length}
+**Files indexed:** ${indexed}
+**Files unchanged:** ${skipped}
+**Total chunks:** ${stats.chunkCount}
+
+You can now use semantic search:
+\`\`\`
+codesearch({ query: "your search query", index: "${indexName}" })
+\`\`\``
+
+        } catch (error: any) {
+          return `❌ Re-indexing failed: ${error.message}
+
+Try:
+1. Check if vectorizer is installed: \`npx opencode-workflow vectorizer status\`
+2. Re-install vectorizer: \`npx opencode-workflow vectorizer install\``
+        }
+      }
+
+      return `Unknown action: ${args.action}. Use: status, list, or reindex`
+
+    } catch (error: any) {
+      return `❌ Error: ${error.message}`
+    }
+  },
+})

package/src/opencode/tools/codesearch.ts

@@ -0,0 +1,134 @@
+/**
+ * Semantic Code Search Tool
+ * 
+ * Allows the AI model to search the codebase using semantic similarity.
+ * Uses local embeddings (all-MiniLM-L6-v2) and LanceDB vector store.
+ * Supports multiple indexes: code, docs, config, or search all.
+ * 
+ * Usage by model:
+ *   codesearch({ query: "authentication middleware", limit: 5 })
+ *   codesearch({ query: "how to deploy", index: "docs" })
+ *   codesearch({ query: "database config", index: "config" })
+ *   codesearch({ query: "error handling", searchAll: true })
+ * 
+ * Prerequisites:
+ *   npx opencode-workflow vectorizer install
+ *   npx opencode-workflow index --index code
+ *   npx opencode-workflow index --index docs
+ */
+
+import { tool } from "@opencode-ai/plugin"
+import path from "path"
+import fs from "fs/promises"
+
+export default tool({
+  description: `Search the codebase semantically. Use this to find relevant code snippets, functions, or files based on meaning, not just text matching.
+
+Available indexes:
+- "code" (default) - Source code files (*.js, *.ts, *.py, *.go, etc.)
+- "docs" - Documentation files (*.md, *.txt, etc.)
+- "config" - Configuration files (*.yaml, *.json, etc.)
+- searchAll: true - Search across all indexes
+
+Examples:
+- "authentication logic" → finds auth-related code
+- "database connection handling" → finds DB setup code  
+- "how to deploy" with index: "docs" → finds deployment docs
+- "API keys" with index: "config" → finds config with API settings
+
+Prerequisites: Run 'npx opencode-workflow index --index <name>' first.`,
+
+  args: {
+    query: tool.schema.string().describe("Semantic search query describing what you're looking for"),
+    index: tool.schema.string().optional().default("code").describe("Index to search: code, docs, config, or custom name"),
+    limit: tool.schema.number().optional().default(5).describe("Number of results to return (default: 5)"),
+    searchAll: tool.schema.boolean().optional().default(false).describe("Search all indexes instead of just one"),
+  },
+
+  async execute(args, context) {
+    const projectRoot = process.cwd()
+    const vectorizerDir = path.join(projectRoot, ".opencode", "vectorizer")
+    const vectorizerModule = path.join(vectorizerDir, "index.js")
+
+    // Check if vectorizer is installed
+    try {
+      await fs.access(path.join(vectorizerDir, "node_modules"))
+    } catch {
+      return `❌ Vectorizer not installed. Run: npx opencode-workflow vectorizer install`
+    }
+
+    try {
+      // Dynamic import of the vectorizer
+      const { CodebaseIndexer } = await import(`file://${vectorizerModule}`)
+      
+      let allResults: any[] = []
+      const limit = args.limit || 5
+      const indexName = args.index || "code"
+
+      if (args.searchAll) {
+        // Search all indexes
+        const tempIndexer = await new CodebaseIndexer(projectRoot, "code").init()
+        const indexes = await tempIndexer.listIndexes()
+
+        if (indexes.length === 0) {
+          return `❌ No indexes found. Run: npx opencode-workflow index --index code`
+        }
+
+        for (const idx of indexes) {
+          const indexer = await new CodebaseIndexer(projectRoot, idx).init()
+          const results = await indexer.search(args.query, limit)
+          allResults.push(...results.map((r: any) => ({ ...r, _index: idx })))
+        }
+
+        // Sort by distance and take top N
+        allResults.sort((a, b) => (a._distance || 0) - (b._distance || 0))
+        allResults = allResults.slice(0, limit)
+
+      } else {
+        // Search specific index
+        const hashesFile = path.join(projectRoot, ".opencode", "vectors", indexName, "hashes.json")
+        try {
+          await fs.access(hashesFile)
+        } catch {
+          return `❌ Index "${indexName}" not found. Run: npx opencode-workflow index --index ${indexName}`
+        }
+
+        const indexer = await new CodebaseIndexer(projectRoot, indexName).init()
+        const results = await indexer.search(args.query, limit)
+        allResults = results.map((r: any) => ({ ...r, _index: indexName }))
+      }
+
+      if (allResults.length === 0) {
+        const scope = args.searchAll ? "any index" : `index "${indexName}"`
+        return `No results found in ${scope} for: "${args.query}"\n\nTry:\n- Different keywords\n- Re-index with: npx opencode-workflow index --index ${indexName} --force`
+      }
+
+      // Format results for the model
+      const scope = args.searchAll ? "all indexes" : `index "${indexName}"`
+      let output = `## Search Results for: "${args.query}" (${scope})\n\n`
+      
+      for (let i = 0; i < allResults.length; i++) {
+        const r = allResults[i]
+        const score = r._distance ? (1 - r._distance).toFixed(3) : "N/A"
+        const indexLabel = args.searchAll ? ` [${r._index}]` : ""
+        
+        output += `### ${i + 1}. ${r.file}${indexLabel}\n`
+        output += `**Relevance:** ${score}\n\n`
+        output += "```\n"
+        // Truncate long content
+        const content = r.content.length > 500 
+          ? r.content.substring(0, 500) + "\n... (truncated)"
+          : r.content
+        output += content
+        output += "\n```\n\n"
+      }
+
+      output += `---\n*Found ${allResults.length} results. Use Read tool to see full files.*`
+
+      return output
+
+    } catch (error: any) {
+      return `❌ Search failed: ${error.message}\n\nTry re-indexing: npx opencode-workflow index --index ${args.index || "code"} --force`
+    }
+  },
+})

package/src/vectorizer/index.js

@@ -0,0 +1,287 @@
+// OpenCode Vectorizer - Semantic Code Search with Multi-Index Support
+// Part of @comfanion/workflow
+
+import { pipeline } from '@xenova/transformers';
+import * as lancedb from 'vectordb';
+import fs from 'fs/promises';
+import path from 'path';
+import crypto from 'crypto';
+
+/**
+ * Index presets for different content types
+ */
+const INDEX_PRESETS = {
+  code: {
+    pattern: '**/*.{js,ts,jsx,tsx,mjs,cjs,py,go,rs,java,kt,swift,c,cpp,h,hpp,cs,rb,php,scala,clj}',
+    description: 'Source code files'
+  },
+  docs: {
+    pattern: '**/*.{md,mdx,txt,rst,adoc}',
+    description: 'Documentation files'
+  },
+  config: {
+    pattern: '**/*.{yaml,yml,json,toml,ini,env,xml}',
+    description: 'Configuration files'
+  },
+  all: {
+    pattern: '**/*.{js,ts,jsx,tsx,mjs,cjs,py,go,rs,java,kt,swift,c,cpp,h,hpp,cs,rb,php,scala,clj,md,mdx,txt,rst,adoc,yaml,yml,json,toml}',
+    description: 'All supported files'
+  }
+};
+
+class CodebaseIndexer {
+  /**
+   * @param {string} projectRoot - Project root directory
+   * @param {string} indexName - Name of the index (e.g., 'code', 'docs', 'config')
+   */
+  constructor(projectRoot, indexName = 'code') {
+    this.root = projectRoot;
+    this.indexName = indexName;
+    this.baseDir = path.join(projectRoot, '.opencode', 'vectors');
+    this.cacheDir = path.join(this.baseDir, indexName);
+    this.model = null;
+    this.db = null;
+    this.hashes = {};
+  }
+
+  async init() {
+    await fs.mkdir(this.cacheDir, { recursive: true });
+    this.db = await lancedb.connect(path.join(this.cacheDir, 'lancedb'));
+    await this.loadHashes();
+    return this;
+  }
+
+  async loadModel() {
+    if (!this.model) {
+      console.log('Loading embedding model (first time takes ~30s)...');
+      this.model = await pipeline('feature-extraction', 'Xenova/all-MiniLM-L6-v2');
+    }
+    return this.model;
+  }
+
+  async unloadModel() {
+    this.model = null;
+    if (global.gc) global.gc();
+  }
+
+  async loadHashes() {
+    try {
+      const hashFile = path.join(this.cacheDir, 'hashes.json');
+      const data = await fs.readFile(hashFile, 'utf8');
+      this.hashes = JSON.parse(data);
+    } catch {
+      this.hashes = {};
+    }
+  }
+
+  async saveHashes() {
+    const hashFile = path.join(this.cacheDir, 'hashes.json');
+    await fs.writeFile(hashFile, JSON.stringify(this.hashes, null, 2));
+  }
+
+  fileHash(content) {
+    return crypto.createHash('md5').update(content).digest('hex');
+  }
+
+  async embed(text) {
+    const model = await this.loadModel();
+    const result = await model(text, { pooling: 'mean', normalize: true });
+    return Array.from(result.data);
+  }
+
+  /**
+   * Chunk code into smaller pieces for embedding
+   * Tries to split on function/class boundaries when possible
+   */
+  chunkCode(content, maxChars = 1500) {
+    const chunks = [];
+    const lines = content.split('\n');
+    let current = [];
+    let currentLen = 0;
+
+    for (const line of lines) {
+      if (currentLen + line.length > maxChars && current.length > 0) {
+        chunks.push(current.join('\n'));
+        current = [];
+        currentLen = 0;
+      }
+      current.push(line);
+      currentLen += line.length + 1;
+    }
+    
+    if (current.length > 0) {
+      chunks.push(current.join('\n'));
+    }
+    
+    return chunks;
+  }
+
+  /**
+   * Check if file needs re-indexing based on content hash
+   */
+  needsIndex(filePath, content) {
+    const relPath = path.relative(this.root, filePath);
+    const currentHash = this.fileHash(content);
+    return this.hashes[relPath] !== currentHash;
+  }
+
+  /**
+   * Index a single file
+   * Returns true if file was indexed, false if skipped (unchanged)
+   */
+  async indexFile(filePath) {
+    const relPath = path.relative(this.root, filePath);
+    
+    let content;
+    try {
+      content = await fs.readFile(filePath, 'utf8');
+    } catch (e) {
+      console.warn(`Cannot read ${relPath}: ${e.message}`);
+      return false;
+    }
+
+    const hash = this.fileHash(content);
+
+    // Skip if unchanged
+    if (this.hashes[relPath] === hash) {
+      return false;
+    }
+
+    const chunks = this.chunkCode(content);
+    const data = [];
+
+    for (let i = 0; i < chunks.length; i++) {
+      const embedding = await this.embed(chunks[i]);
+      data.push({
+        file: relPath,
+        chunk_index: i,
+        content: chunks[i],
+        vector: embedding
+      });
+    }
+
+    // Add to database
+    const tableName = 'chunks';
+    const tables = await this.db.tableNames();
+    if (tables.includes(tableName)) {
+      const table = await this.db.openTable(tableName);
+      // Note: LanceDB doesn't support delete by filter in all versions
+      // So we just add new chunks (may have duplicates until reindex --force)
+      await table.add(data);
+    } else {
+      await this.db.createTable(tableName, data);
+    }
+
+    // Update hash cache
+    this.hashes[relPath] = hash;
+    await this.saveHashes();
+    
+    return true;
+  }
+
+  /**
+   * Semantic search across indexed codebase
+   */
+  async search(query, limit = 5) {
+    const tableName = 'chunks';
+    const tables = await this.db.tableNames();
+    if (!tables.includes(tableName)) {
+      return [];
+    }
+
+    const queryEmbedding = await this.embed(query);
+    const table = await this.db.openTable(tableName);
+    const results = await table.search(queryEmbedding).limit(limit).execute();
+    
+    return results;
+  }
+
+  /**
+   * Get indexing statistics for this index
+   */
+  async getStats() {
+    const fileCount = Object.keys(this.hashes).length;
+    let chunkCount = 0;
+    
+    try {
+      const tables = await this.db.tableNames();
+      if (tables.includes('chunks')) {
+        const table = await this.db.openTable('chunks');
+        chunkCount = await table.countRows();
+      }
+    } catch {}
+
+    const preset = INDEX_PRESETS[this.indexName];
+    
+    return { 
+      indexName: this.indexName,
+      description: preset?.description || 'Custom index',
+      fileCount, 
+      chunkCount 
+    };
+  }
+
+  /**
+   * Get statistics for all indexes
+   */
+  async getAllStats() {
+    const stats = [];
+    
+    try {
+      const entries = await fs.readdir(this.baseDir, { withFileTypes: true });
+      
+      for (const entry of entries) {
+        if (entry.isDirectory() && entry.name !== 'lancedb') {
+          try {
+            const indexer = await new CodebaseIndexer(this.root, entry.name).init();
+            const stat = await indexer.getStats();
+            if (stat.fileCount > 0 || stat.chunkCount > 0) {
+              stats.push(stat);
+            }
+          } catch {}
+        }
+      }
+    } catch {}
+    
+    return stats;
+  }
+
+  /**
+   * Clear this index's data
+   */
+  async clear() {
+    await fs.rm(this.cacheDir, { recursive: true, force: true });
+    this.hashes = {};
+    await this.init();
+  }
+
+  /**
+   * Clear all indexes
+   */
+  async clearAll() {
+    await fs.rm(this.baseDir, { recursive: true, force: true });
+    this.hashes = {};
+    await this.init();
+  }
+
+  /**
+   * List all available index names
+   */
+  async listIndexes() {
+    const indexes = [];
+    
+    try {
+      const entries = await fs.readdir(this.baseDir, { withFileTypes: true });
+      
+      for (const entry of entries) {
+        if (entry.isDirectory() && entry.name !== 'lancedb') {
+          indexes.push(entry.name);
+        }
+      }
+    } catch {}
+    
+    return indexes;
+  }
+}
+
+export { CodebaseIndexer, INDEX_PRESETS };

package/src/vectorizer/package.json

@@ -0,0 +1,15 @@
+{
+  "name": "opencode-vectorizer",
+  "version": "1.0.0",
+  "description": "Semantic code search for OpenCode Workflow",
+  "type": "module",
+  "private": true,
+  "main": "index.js",
+  "dependencies": {
+    "@xenova/transformers": "^2.17.0",
+    "vectordb": "^0.4.0"
+  },
+  "engines": {
+    "node": ">=18"
+  }
+}