claude-all-hands 1.0.2 → 1.0.4

package/.claude/envoy/src/lib/knowledge.ts

@@ -3,10 +3,15 @@
  * Uses @visheratin/web-ai-node for embeddings and usearch for HNSW indexing.
  */
 
-import { existsSync, mkdirSync, readFileSync, writeFileSync, readdirSync, statSync } from "fs";
-import { join, relative, extname } from "path";
-import { Index, MetricKind, ScalarKind, type Matches } from "usearch";
+import { createHash } from "crypto";
+import { existsSync, mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from "fs";
 import matter from "gray-matter";
+import { createRequire } from "module";
+import { basename, extname, join, relative } from "path";
+import { Index, MetricKind, ScalarKind } from "usearch";
+
+// Create require function for ESM compatibility (needed for fetch caching)
+const require = createRequire(import.meta.url);
 
 // Types
 interface DocumentMeta {
@@ -53,32 +58,13 @@ interface IndexConfig {
   hasFrontmatter: boolean;
 }
 
-// Configuration
-const INDEX_CONFIGS: Record<string, IndexConfig> = {
-  docs: {
-    name: "docs",
-    paths: ["docs/"],
-    extensions: [".md"],
-    description: "Project documentation for all agents",
-    hasFrontmatter: true,
-  },
-  curator: {
-    name: "curator",
-    paths: [
-      ".claude/agents/",
-      ".claude/hooks/",
-      ".claude/skills/",
-      ".claude/commands/",
-      ".claude/output-styles/",
-      ".claude/envoy/README.md",
-      ".claude/settings.json",
-      ".claude/envoy/src/",
-      ".claude/envoy/package.json",
-    ],
-    extensions: [".md", ".yaml", ".yml", ".ts", ".json"],
-    description: ".claude/ files for curator agent",
-    hasFrontmatter: false,
-  },
+// Docs index configuration (only supported index)
+const DOCS_CONFIG: IndexConfig = {
+  name: "docs",
+  paths: ["docs/"],
+  extensions: [".md"],
+  description: "Project documentation",
+  hasFrontmatter: true,
 };
 
 // File reference patterns for auto-populating relevant_files
@@ -90,14 +76,14 @@ const FILE_REF_PATTERNS = [
 
 // Environment config with defaults
 const SEARCH_SIMILARITY_THRESHOLD = parseFloat(
-  process.env.SEARCH_SIMILARITY_THRESHOLD ?? "0.64"
+  process.env.SEARCH_SIMILARITY_THRESHOLD ?? "0.65"
 );
 const SEARCH_CONTEXT_TOKEN_LIMIT = parseInt(
   process.env.SEARCH_CONTEXT_TOKEN_LIMIT ?? "5000",
   10
 );
 const SEARCH_FULL_CONTEXT_SIMILARITY_THRESHOLD = parseFloat(
-  process.env.SEARCH_FULL_CONTEXT_SIMILARITY_THRESHOLD ?? "0.72"
+  process.env.SEARCH_FULL_CONTEXT_SIMILARITY_THRESHOLD ?? "0.82"
 );
 
 export class KnowledgeService {
@@ -120,14 +106,101 @@ export class KnowledgeService {
   }
 
   /**
-   * Lazy-load embedding model (first use downloads ~100-400MB)
+   * Get model cache directory
+   */
+  private getModelCacheDir(): string {
+    return join(this.knowledgeDir, "models");
+  }
+
+  /**
+   * Install caching wrapper for node-fetch (must call before importing web-ai-node)
+   */
+  private installFetchCache(): void {
+    const cacheDir = this.getModelCacheDir();
+    if (!existsSync(cacheDir)) {
+      mkdirSync(cacheDir, { recursive: true });
+    }
+
+    // eslint-disable-next-line @typescript-eslint/no-require-imports
+    const nodeFetchModule = require("node-fetch");
+    const originalFetch = nodeFetchModule.default || nodeFetchModule;
+
+    // Skip if already patched
+    if ((originalFetch as { __cached?: boolean }).__cached) return;
+
+    const self = this;
+    const cachedFetch = async function (url: string, init?: RequestInit) {
+      // Only cache model files
+      if (!url.includes("web-ai-models.org") && !url.includes(".onnx")) {
+        return originalFetch(url, init);
+      }
+
+      const urlHash = createHash("md5").update(url).digest("hex").slice(0, 8);
+      const fileName = `${urlHash}-${basename(url)}`;
+      const cachePath = join(self.getModelCacheDir(), fileName);
+
+      if (existsSync(cachePath)) {
+        console.error(`[knowledge] Using cached model: ${fileName}`);
+        const data = readFileSync(cachePath);
+        // Return a mock response with arrayBuffer method
+        return {
+          ok: true,
+          status: 200,
+          arrayBuffer: async () => data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength),
+        };
+      }
+
+      console.error(`[knowledge] Downloading model: ${basename(url)}`);
+      const response = await originalFetch(url, init);
+      if (!response.ok) {
+        return response;
+      }
+
+      const arrayBuffer = await response.arrayBuffer();
+      const buffer = Buffer.from(arrayBuffer);
+      writeFileSync(cachePath, buffer);
+      console.error(`[knowledge] Cached model: ${fileName} (${(buffer.length / 1024 / 1024).toFixed(1)}MB)`);
+
+      // Return a mock response since we consumed the original
+      return {
+        ok: true,
+        status: 200,
+        arrayBuffer: async () => arrayBuffer,
+      };
+    };
+
+    (cachedFetch as { __cached?: boolean }).__cached = true;
+
+    // Patch the module's default export
+    if (nodeFetchModule.default) {
+      nodeFetchModule.default = cachedFetch;
+    }
+    // Also patch require.cache
+    const cacheKey = require.resolve("node-fetch");
+    if (require.cache[cacheKey]) {
+      require.cache[cacheKey]!.exports = cachedFetch;
+      require.cache[cacheKey]!.exports.default = cachedFetch;
+    }
+  }
+
+  /**
+   * Lazy-load embedding model with local caching
    */
   async getModel(): Promise<unknown> {
     if (this.model) return this.model;
 
+    this.ensureDir();
+    console.error("[knowledge] Loading embedding model...");
+    const startTime = Date.now();
+
+    // Install caching before importing the library
+    this.installFetchCache();
+
     const { TextModel } = await import("@visheratin/web-ai-node/text");
     const modelResult = await TextModel.create("gtr-t5-quant");
     this.model = modelResult.model;
+
+    console.error(`[knowledge] Model loaded in ${((Date.now() - startTime) / 1000).toFixed(1)}s`);
     return this.model;
   }
 
@@ -153,10 +226,10 @@ export class KnowledgeService {
   /**
    * Get index file paths
    */
-  private getIndexPaths(indexName: string): { index: string; meta: string } {
+  private getIndexPaths(): { index: string; meta: string } {
     return {
-      index: join(this.knowledgeDir, `${indexName}.usearch`),
-      meta: join(this.knowledgeDir, `${indexName}.meta.json`),
+      index: join(this.knowledgeDir, "docs.usearch"),
+      meta: join(this.knowledgeDir, "docs.meta.json"),
     };
   }
 
@@ -188,10 +261,8 @@ export class KnowledgeService {
   /**
    * Load index + metadata from disk
    */
-  async loadIndex(
-    indexName: string
-  ): Promise<{ index: Index; meta: IndexMetadata }> {
-    const paths = this.getIndexPaths(indexName);
+  async loadIndex(): Promise<{ index: Index; meta: IndexMetadata }> {
+    const paths = this.getIndexPaths();
 
     if (!existsSync(paths.index) || !existsSync(paths.meta)) {
       return {
@@ -210,13 +281,9 @@ export class KnowledgeService {
   /**
    * Save index + metadata to disk
    */
-  async saveIndex(
-    indexName: string,
-    index: Index,
-    meta: IndexMetadata
-  ): Promise<void> {
+  async saveIndex(index: Index, meta: IndexMetadata): Promise<void> {
     this.ensureDir();
-    const paths = this.getIndexPaths(indexName);
+    const paths = this.getIndexPaths();
 
     meta.lastUpdated = new Date().toISOString();
     index.save(paths.index);
@@ -356,10 +423,11 @@ export class KnowledgeService {
   }
 
   /**
-   * Search with similarity computation
+   * Search docs index with similarity computation
+   * @param metadataOnly - If true, only return file paths and descriptions (no full_resource_context)
    */
-  async search(indexName: string, query: string, k: number = 50): Promise<SearchResult[]> {
-    const { index, meta } = await this.loadIndex(indexName);
+  async search(query: string, k: number = 50, metadataOnly: boolean = false): Promise<SearchResult[]> {
+    const { index, meta } = await this.loadIndex();
 
     if (Object.keys(meta.documents).length === 0) {
       return [];
@@ -403,8 +471,8 @@ export class KnowledgeService {
         relevant_files: docMeta.relevant_files,
       };
 
-      // Include full context for high-similarity results
-      if (similarity >= SEARCH_FULL_CONTEXT_SIMILARITY_THRESHOLD) {
+      // Include full context for high-similarity results (unless metadata-only mode)
+      if (!metadataOnly && similarity >= SEARCH_FULL_CONTEXT_SIMILARITY_THRESHOLD) {
         const fullPath = join(this.projectRoot, path);
         if (existsSync(fullPath)) {
           result.full_resource_context = readFileSync(fullPath, "utf-8");
@@ -418,86 +486,76 @@ export class KnowledgeService {
   }
 
   /**
-   * Full reindex for an index
+   * Full reindex of docs
    */
-  async reindexAll(indexName?: string): Promise<Record<string, ReindexResult>> {
+  async reindexAll(): Promise<ReindexResult> {
     this.ensureDir();
-    const results: Record<string, ReindexResult> = {};
+    const startTime = Date.now();
+    console.error("[knowledge] Reindexing docs...");
 
-    const indexes = indexName ? [indexName] : Object.keys(INDEX_CONFIGS);
-
-    for (const name of indexes) {
-      const config = INDEX_CONFIGS[name];
-      if (!config) {
-        throw new Error(`Unknown index: ${name}`);
-      }
-
-      // Create fresh index
-      const index = this.createIndex();
-      const meta = this.createEmptyMetadata();
-
-      // Discover and index files
-      const files = this.discoverFiles(config);
-      let totalTokens = 0;
+    // Create fresh index
+    const index = this.createIndex();
+    const meta = this.createEmptyMetadata();
 
-      for (const filePath of files) {
-        const fullPath = join(this.projectRoot, filePath);
-        const content = readFileSync(fullPath, "utf-8");
+    // Discover and index files
+    const files = this.discoverFiles(DOCS_CONFIG);
+    console.error(`[knowledge] Found ${files.length} files`);
+    let totalTokens = 0;
 
-        // Parse front-matter only for indexes that support it
-        let frontMatter: Record<string, unknown> = {};
-        if (config.hasFrontmatter && filePath.endsWith(".md")) {
-          try {
-            const parsed = matter(content);
-            frontMatter = parsed.data;
-          } catch {
-            // Skip files with invalid front-matter
-          }
+    for (let i = 0; i < files.length; i++) {
+      const filePath = files[i];
+      const fullPath = join(this.projectRoot, filePath);
+      const content = readFileSync(fullPath, "utf-8");
+
+      // Parse front-matter
+      let frontMatter: Record<string, unknown> = {};
+      if (filePath.endsWith(".md")) {
+        try {
+          const parsed = matter(content);
+          frontMatter = parsed.data;
+        } catch {
+          // Skip files with invalid front-matter
         }
-
-        await this.indexDocument(index, meta, filePath, content, frontMatter);
-        totalTokens += meta.documents[filePath].token_count;
       }
 
-      // Save
-      await this.saveIndex(name, index, meta);
-
-      results[name] = {
-        files_indexed: files.length,
-        total_tokens: totalTokens,
-      };
+      console.error(`[knowledge] Embedding ${i + 1}/${files.length}: ${filePath}`);
+      await this.indexDocument(index, meta, filePath, content, frontMatter);
+      totalTokens += meta.documents[filePath].token_count;
     }
 
-    return results;
+    // Save
+    await this.saveIndex(index, meta);
+    const duration = ((Date.now() - startTime) / 1000).toFixed(1);
+    console.error(`[knowledge] Reindex complete: ${files.length} files, ${totalTokens} tokens in ${duration}s`);
+
+    return {
+      files_indexed: files.length,
+      total_tokens: totalTokens,
+    };
   }
 
   /**
    * Incremental reindex from changed files
    */
-  async reindexFromChanges(
-    indexName: string,
-    changes: FileChange[]
-  ): Promise<{
+  async reindexFromChanges(changes: FileChange[]): Promise<{
     success: boolean;
     message: string;
     missing_references?: { doc_path: string; missing_files: string[] }[];
     files: { path: string; action: string }[];
   }> {
-    const config = INDEX_CONFIGS[indexName];
-    if (!config) {
-      throw new Error(`Unknown index: ${indexName}`);
-    }
+    console.error(`[knowledge] Incremental reindex: ${changes.length} change(s)`);
+    const startTime = Date.now();
 
-    const { index, meta } = await this.loadIndex(indexName);
+    const { index, meta } = await this.loadIndex();
     const processedFiles: { path: string; action: string }[] = [];
     const missingReferences: { doc_path: string; missing_files: string[] }[] = [];
 
     for (const change of changes) {
       const { path, added, deleted, modified } = change;
 
-      // Check if file matches index config
-      const matchesConfig = config.paths.some((p) => path.startsWith(p)) &&
-        config.extensions.includes(extname(path));
+      // Check if file matches docs config
+      const matchesConfig = DOCS_CONFIG.paths.some((p: string) => path.startsWith(p)) &&
+        DOCS_CONFIG.extensions.includes(extname(path));
 
       if (!matchesConfig) continue;
 
@@ -511,6 +569,7 @@ export class KnowledgeService {
           delete meta.path_to_id[path];
           delete meta.documents[path];
           processedFiles.push({ path, action: "deleted" });
+          console.error(`[knowledge] Deleted: ${path}`);
         }
       } else if (added || modified) {
         const fullPath = join(this.projectRoot, path);
@@ -519,8 +578,8 @@ export class KnowledgeService {
         const content = readFileSync(fullPath, "utf-8");
         let frontMatter: Record<string, unknown> = {};
 
-        // Only process front-matter and file references for indexes that support it
-        if (config.hasFrontmatter && path.endsWith(".md")) {
+        // Process front-matter and file references
+        if (path.endsWith(".md")) {
           try {
             const parsed = matter(content);
             frontMatter = parsed.data;
@@ -546,13 +605,17 @@ export class KnowledgeService {
         }
 
         // Index document
+        const action = added ? "added" : "modified";
+        console.error(`[knowledge] Embedding (${action}): ${path}`);
         await this.indexDocument(index, meta, path, content, frontMatter);
-        processedFiles.push({ path, action: added ? "added" : "modified" });
+        processedFiles.push({ path, action });
       }
     }
 
     // Save updated index
-    await this.saveIndex(indexName, index, meta);
+    await this.saveIndex(index, meta);
+    const duration = ((Date.now() - startTime) / 1000).toFixed(1);
+    console.error(`[knowledge] Incremental reindex complete: ${processedFiles.length} file(s) in ${duration}s`);
 
     if (missingReferences.length > 0) {
       return {
@@ -571,24 +634,13 @@ export class KnowledgeService {
   }
 
   /**
-   * Check if indexes exist and are valid
+   * Check if docs index exists
    */
-  async checkIndexes(): Promise<{ valid: string[]; missing: string[] }> {
-    const valid: string[] = [];
-    const missing: string[] = [];
-
-    for (const name of Object.keys(INDEX_CONFIGS)) {
-      const paths = this.getIndexPaths(name);
-      if (existsSync(paths.index) && existsSync(paths.meta)) {
-        valid.push(name);
-      } else {
-        missing.push(name);
-      }
-    }
-
-    return { valid, missing };
+  async checkIndex(): Promise<{ exists: boolean }> {
+    const paths = this.getIndexPaths();
+    return { exists: existsSync(paths.index) && existsSync(paths.meta) };
   }
 }
 
-export { INDEX_CONFIGS };
-export type { SearchResult, ReindexResult, FileChange, IndexMetadata, DocumentMeta };
+export type { DocumentMeta, FileChange, IndexMetadata, ReindexResult, SearchResult };
+