cntx-ui 2.0.15 → 3.0.1

package/lib/agent-tools.js

@@ -3,8 +3,10 @@
  * Built on top of existing cntx-ui infrastructure
  */
 
-import { readFileSync, existsSync } from 'fs';
-import { join, relative } from 'path';
+import { readFileSync, existsSync, statSync } from 'fs';
+import { join, relative, extname } from 'path';
+import fs from 'fs';
+import path from 'path';
 import { exec } from 'child_process';
 import { promisify } from 'util';
 
@@ -106,8 +108,8 @@ export class AgentTools {
       // Apply semantic search if vector store is available
       if (this.cntxServer.vectorStoreInitialized) {
         try {
-          const searchResults = await this.cntxServer.vectorStore.search(query, maxResults * 2);
-          const chunkIds = searchResults.map(r => r.metadata?.chunkId).filter(Boolean);
+          const searchResults = await this.cntxServer.vectorStore.search(query, { limit: maxResults * 2 });
+          const chunkIds = searchResults.map(r => r.id || r.chunkId).filter(Boolean);
           chunks = chunks.filter(c => chunkIds.includes(c.id));
         } catch (error) {
           // Fall back to text-based search
@@ -318,7 +320,7 @@ export class AgentTools {
   getFileSize(filePath) {
     try {
       const fullPath = join(this.cntxServer.CWD, filePath);
-      const stats = require('fs').statSync(fullPath);
+      const stats = statSync(fullPath);
       return stats.size;
     } catch {
       return 0;
@@ -326,7 +328,7 @@ export class AgentTools {
   }
 
   getFileType(filePath) {
-    const ext = require('path').extname(filePath).toLowerCase();
+    const ext = extname(filePath).toLowerCase();
     const typeMap = {
       '.js': 'javascript',
       '.jsx': 'javascript',
@@ -345,7 +347,7 @@ export class AgentTools {
   }
 
   getMimeType(filePath) {
-    const ext = require('path').extname(filePath).toLowerCase();
+    const ext = path.extname(filePath).toLowerCase();
     const mimeTypes = {
       '.js': 'application/javascript',
       '.jsx': 'application/javascript',

package/lib/api-router.js

@@ -4,9 +4,12 @@
  */
 
 import { parse } from 'url';
+import fs from 'fs';
+import path from 'path';
 
 export default class APIRouter {
-  constructor(configManager, bundleManager, fileSystemManager, semanticAnalysisManager, vectorStore, activityManager) {
+  constructor(cntxServer, configManager, bundleManager, fileSystemManager, semanticAnalysisManager, vectorStore, activityManager) {
+    this.cntxServer = cntxServer;
     this.configManager = configManager;
     this.bundleManager = bundleManager;
     this.fileSystemManager = fileSystemManager;
@@ -19,6 +22,12 @@ export default class APIRouter {
     const method = req.method;
     const pathname = url.pathname;
 
+    // DEBUG: Log all incoming API requests
+    console.log('[API REQUEST]', method, pathname);
+    if (pathname.includes('database')) {
+      console.log('[DATABASE] Route requested:', pathname, method);
+    }
+
     try {
       // Route to appropriate handler
       if (pathname === '/api/bundles' && method === 'GET') {
@@ -164,6 +173,15 @@ export default class APIRouter {
         return await this.handlePostVectorDbSearch(req, res);
       }
 
+      if (pathname === '/api/vector-db/network' && method === 'GET') {
+        return await this.handleGetVectorDbNetwork(req, res);
+      }
+
+      if (pathname === '/api/semantic-search' && method === 'POST') {
+        return await this.handlePostSemanticSearch(req, res);
+      }
+
+
       if (pathname === '/api/vector-db/search-by-type' && method === 'POST') {
         return await this.handlePostVectorDbSearchByType(req, res);
       }
@@ -176,6 +194,11 @@ export default class APIRouter {
         return await this.handleGetActivities(req, res);
       }
 
+      if (pathname.startsWith('/api/activities/') && pathname.endsWith('/reasoning') && method === 'GET') {
+        const activityId = pathname.split('/')[3];
+        return await this.handleGetActivityReasoning(req, res, activityId);
+      }
+
       if (pathname.startsWith('/api/activities/') && pathname.endsWith('/execute') && method === 'POST') {
         const activityId = pathname.split('/')[3];
         return await this.handlePostActivityExecute(req, res, activityId);
@@ -190,6 +213,22 @@ export default class APIRouter {
         return await this.handleOpenFile(req, res);
       }
 
+      if (pathname === '/api/bundle-sync-status' && method === 'GET') {
+        return await this.handleGetBundleSyncStatus(req, res);
+      }
+
+      if (pathname === '/api/bundle-sync-status' && method === 'POST') {
+        return await this.handlePostBundleSyncStatus(req, res);
+      }
+
+      if (pathname === '/api/database/info' && method === 'GET') {
+        return await this.handleGetDatabaseInfo(req, res);
+      }
+
+      if (pathname === '/api/database/query' && method === 'POST') {
+        return await this.handlePostDatabaseQuery(req, res);
+      }
+
       // If no route matches, return 404
       res.writeHead(404, { 'Content-Type': 'application/json' });
       res.end(JSON.stringify({ error: 'API endpoint not found' }));
@@ -204,20 +243,13 @@ export default class APIRouter {
   // === Bundle Operations ===
 
   async handleGetBundles(req, res, url) {
-    const bundles = this.configManager.getBundles();
-    const bundleData = Array.from(bundles.entries()).map(([name, bundle]) => ({
-      name,
-      changed: bundle.changed,
-      fileCount: bundle.files.length,
-      contentPreview: bundle.content.substring(0, 200) + (bundle.content.length > 200 ? '...' : ''),
-      files: bundle.files,
-      patterns: bundle.patterns,
-      size: bundle.size,
-      generated: bundle.generated
-    }));
-
-    res.writeHead(200, { 'Content-Type': 'application/json' });
-    res.end(JSON.stringify(bundleData));
+    try {
+      const bundleInfo = this.bundleManager.getAllBundleInfo();
+      res.writeHead(200, { 'Content-Type': 'application/json' });
+      res.end(JSON.stringify(bundleInfo));
+    } catch (error) {
+      this.sendError(res, 500, error.message);
+    }
   }
 
   async handlePostBundles(req, res) {
@@ -243,7 +275,7 @@ export default class APIRouter {
           }
           // Ensure we're working with relative paths
           const relativeAddFileName = fileName.startsWith('/') ?
-            require('path').relative(this.configManager.CWD, fileName) : fileName;
+            path.relative(this.configManager.CWD, fileName) : fileName;
           if (!bundle.files.includes(relativeAddFileName)) {
             bundle.files.push(relativeAddFileName);
             bundle.changed = true;
@@ -257,7 +289,7 @@ export default class APIRouter {
           }
           // Ensure we're working with relative paths for both search and removal
           const relativeRemoveFileName = fileName.startsWith('/') ?
-            require('path').relative(this.configManager.CWD, fileName) : fileName;
+            path.relative(this.configManager.CWD, fileName) : fileName;
           const removeIndex = bundle.files.indexOf(relativeRemoveFileName);
           if (removeIndex > -1) {
             bundle.files.splice(removeIndex, 1);
@@ -273,7 +305,7 @@ export default class APIRouter {
           fileNames.forEach(file => {
             // Ensure we're working with relative paths
             const relativeFile = file.startsWith('/') ?
-              require('path').relative(this.configManager.CWD, file) : file;
+              path.relative(this.configManager.CWD, file) : file;
             if (!bundle.files.includes(relativeFile)) {
               bundle.files.push(relativeFile);
             }
@@ -289,7 +321,7 @@ export default class APIRouter {
           fileNames.forEach(file => {
             // Ensure we're working with relative paths
             const relativeFile = file.startsWith('/') ?
-              require('path').relative(this.configManager.CWD, file) : file;
+              path.relative(this.configManager.CWD, file) : file;
             const index = bundle.files.indexOf(relativeFile);
             if (index > -1) {
               bundle.files.splice(index, 1);
@@ -378,9 +410,8 @@ export default class APIRouter {
       editor: this.configManager.getEditor()
     };
 
-    bundles.forEach((bundle, name) => {
-      config.bundles[name] = bundle.patterns;
-    });
+    // Note: bundles are now managed separately in bundle-states.json
+    // config.json only contains non-bundle settings
 
     res.writeHead(200, { 'Content-Type': 'application/json' });
     res.end(JSON.stringify(config));
@@ -570,12 +601,14 @@ export default class APIRouter {
     const { content } = JSON.parse(body);
 
     // Save content and reload patterns
-    this.configManager.saveIgnoreFile ? this.configManager.saveIgnoreFile(content) : null;
-    this.configManager.loadIgnorePatterns();
-    this.fileSystemManager.setIgnorePatterns(this.configManager.getIgnorePatterns());
+    const success = this.configManager.saveCntxignore(content);
+    if (!success) {
+      return this.sendError(res, 500, 'Failed to save .cntxignore');
+    }
 
-    res.writeHead(200, { 'Content-Type': 'application/json' });
-    res.end(JSON.stringify({ success: true }));
+    this.fileSystemManager.setIgnorePatterns(this.configManager.ignorePatterns);
+
+    this.sendResponse(res, 200, { success: true });
   }
 
   async handleGetGitignore(req, res) {
@@ -624,37 +657,20 @@ export default class APIRouter {
         name: chunk.name,
         code: chunk.code,
         semanticType: chunk.subtype || chunk.type || 'unknown',
-        businessDomain: chunk.tags || [],
-        technicalPatterns: chunk.tags || [],
+        businessDomain: chunk.businessDomain || [],
+        technicalPatterns: chunk.technicalPatterns || [],
         purpose: chunk.purpose || '',
         filePath: chunk.filePath,
         files: chunk.filePath ? [chunk.filePath] : [],
         size: chunk.size || 0,
         complexity: chunk.complexity || 0,
+        tags: chunk.tags || [],
         startLine: chunk.startLine,
         isExported: chunk.isExported,
         isAsync: chunk.isAsync,
-        bundles: chunk.bundles || [],
-        embedding: chunk.embedding,
-        // Also include nested metadata format that VectorVisualization expects
-        metadata: {
-          content: chunk.code || '',
-          semanticType: chunk.subtype || chunk.type || 'unknown',
-          businessDomain: chunk.tags || [],
-          technicalPatterns: chunk.tags || [],
-          purpose: chunk.purpose || '',
-          files: chunk.filePath ? [chunk.filePath] : [],
-          size: chunk.size || 0,
-          complexity: chunk.complexity || 0
-        }
+        bundles: chunk.bundles || []
       }));
 
-      // console.log('📊 Transformed chunks sample:', chunks[0] ? {
-      //   id: chunks[0].id,
-      //   semanticType: chunks[0].semanticType,
-      //   hasMetadata: !!chunks[0].metadata
-      // } : 'No chunks');
-
       this.sendResponse(res, 200, {
         summary: {
           totalFiles: analysis?.summary?.totalFiles || analysis?.fileCount || 0,
@@ -703,17 +719,78 @@ export default class APIRouter {
     }));
   }
 
+  async handlePostSemanticSearch(req, res) {
+    try {
+      const body = await this.getRequestBody(req);
+      const { query, limit = 20 } = JSON.parse(body);
+      
+      if (!query) {
+        return this.sendError(res, 400, 'Search query is required');
+      }
+
+      const results = await this.vectorStore.search(query, { limit });
+      this.sendResponse(res, 200, { results });
+    } catch (error) {
+      this.sendError(res, 500, error.message);
+    }
+  }
+
   async handleGetMcpStatus(req, res) {
+    const isRunning = this.cntxServer.mcpServerStarted || false;
     const status = {
-      enabled: this.mcpServerStarted || false,
+      enabled: isRunning,
+      running: isRunning,
       available: true,
-      message: 'MCP server integration available'
+      message: isRunning ? 'MCP server is running' : 'MCP server integration available'
     };
 
     res.writeHead(200, { 'Content-Type': 'application/json' });
     res.end(JSON.stringify(status));
   }
 
+  async handleGetVectorDbNetwork(req, res) {
+    try {
+      // 1. Get all chunks and embeddings
+      const chunks = this.configManager.dbManager.db.prepare('SELECT * FROM semantic_chunks').all();
+      const embeddings = this.configManager.dbManager.db.prepare('SELECT * FROM vector_embeddings').all();
+      
+      const nodes = chunks.map(c => this.configManager.dbManager.mapChunkRow(c));
+      const edges = [];
+      const threshold = 0.7; // High threshold for clean network
+
+      // 2. Perform pairwise similarity on backend (efficient for small/medium repos)
+      // Limit to top 100 most complex chunks to keep graph readable
+      const topNodes = nodes.sort((a, b) => (b.complexity?.score || 0) - (a.complexity?.score || 0)).slice(0, 100);
+      
+      const embeddingMap = new Map();
+      embeddings.forEach(e => {
+        embeddingMap.set(e.chunk_id, new Float32Array(e.embedding.buffer, e.embedding.byteOffset, e.embedding.byteLength / 4));
+      });
+
+      for (let i = 0; i < topNodes.length; i++) {
+        for (let j = i + 1; j < topNodes.length; j++) {
+          const vecA = embeddingMap.get(topNodes[i].id);
+          const vecB = embeddingMap.get(topNodes[j].id);
+          
+          if (vecA && vecB) {
+            const similarity = this.vectorStore.cosineSimilarity(vecA, vecB);
+            if (similarity >= threshold) {
+              edges.push({
+                source: topNodes[i].id,
+                target: topNodes[j].id,
+                similarity
+              });
+            }
+          }
+        }
+      }
+
+      this.sendResponse(res, 200, { nodes: topNodes, edges });
+    } catch (error) {
+      this.sendError(res, 500, error.message);
+    }
+  }
+
   async handleGetStatus(req, res) {
     const bundles = this.configManager.getBundles();
     const bundleStats = Array.from(bundles.entries()).map(([name, bundle]) => ({
@@ -742,50 +819,55 @@ export default class APIRouter {
 
   // === Vector Database Operations ===
 
-  async handleGetVectorDbStatus(req, res) {
+  async handlePostVectorDbSearch(req, res) {
     try {
+      const body = await this.getRequestBody(req);
+      const { query, limit = 10 } = JSON.parse(body);
+
       // Initialize vector store if needed
-      if (!this.vectorStore.embedder) {
+      if (!this.vectorStore.initialized) {
         await this.vectorStore.init();
       }
 
-      const stats = await this.vectorStore.getStats();
-      this.sendResponse(res, 200, { stats });
+      const results = await this.vectorStore.search(query, { limit });
+      this.sendResponse(res, 200, results);
     } catch (error) {
       this.sendError(res, 500, error.message);
     }
   }
 
-  async handlePostVectorDbRebuild(req, res) {
+  async handleGetVectorDbStatus(req, res) {
     try {
-      await this.vectorStore.clear();
-      const analysis = await this.semanticAnalysisManager.getSemanticAnalysis();
-
-      if (analysis && analysis.chunks) {
-        await this.vectorStore.storePrecomputedChunks(analysis.chunks);
-      }
-
-      const stats = await this.vectorStore.getStats();
-      res.writeHead(200, { 'Content-Type': 'application/json' });
-      res.end(JSON.stringify({ success: true, stats }));
+      const info = this.configManager.dbManager.getInfo();
+      this.sendResponse(res, 200, {
+        stats: {
+          totalChunks: info.chunkCount,
+          embeddingCount: info.embeddingCount,
+          modelName: this.vectorStore.modelName
+        }
+      });
     } catch (error) {
-      res.writeHead(500, { 'Content-Type': 'application/json' });
-      res.end(JSON.stringify({ error: error.message }));
+      this.sendError(res, 500, error.message);
     }
   }
 
-  async handlePostVectorDbSearch(req, res) {
+  async handlePostVectorDbRebuild(req, res) {
     try {
-      const body = await this.getRequestBody(req);
-      const { query, limit = 10 } = JSON.parse(body);
-
-      // Initialize vector store if needed
-      if (!this.vectorStore.embedder) {
-        await this.vectorStore.init();
+      console.log('🔄 Rebuilding vector database...');
+      // 1. Get all chunks from SQLite
+      const chunks = this.configManager.dbManager.db.prepare('SELECT * FROM semantic_chunks').all()
+        .map(row => this.configManager.dbManager.mapChunkRow(row));
+
+      // 2. Generate/Persist embeddings for every chunk
+      for (const chunk of chunks) {
+        await this.vectorStore.upsertChunk(chunk);
       }
 
-      const results = await this.vectorStore.findSimilar(query, { limit });
-      this.sendResponse(res, 200, results);
+      const info = this.configManager.dbManager.getInfo();
+      this.sendResponse(res, 200, { 
+        success: true, 
+        embeddingCount: info.embeddingCount 
+      });
     } catch (error) {
       this.sendError(res, 500, error.message);
     }
@@ -829,13 +911,21 @@ export default class APIRouter {
 
   async handleGetActivities(req, res) {
     try {
-      console.log('API: /api/activities called');
       const activities = await this.activityManager.loadActivities();
-      console.log('API: Loaded activities:', activities.length);
       res.writeHead(200, { 'Content-Type': 'application/json' });
       res.end(JSON.stringify(activities));
     } catch (error) {
-      console.log('API: Error loading activities:', error.message);
+      res.writeHead(500, { 'Content-Type': 'application/json' });
+      res.end(JSON.stringify({ error: error.message }));
+    }
+  }
+
+  async handleGetActivityReasoning(req, res, activityId) {
+    try {
+      const history = this.configManager.dbManager.getSessionHistory(activityId);
+      res.writeHead(200, { 'Content-Type': 'application/json' });
+      res.end(JSON.stringify({ history }));
+    } catch (error) {
       res.writeHead(500, { 'Content-Type': 'application/json' });
       res.end(JSON.stringify({ error: error.message }));
     }
@@ -975,4 +1065,97 @@ export default class APIRouter {
       req.on('error', reject);
     });
   }
+
+  async handleGetBundleSyncStatus(req, res) {
+    const bundleStatesPath = path.join(this.configManager.CNTX_DIR, 'bundle-states.json');
+
+    // Since bundles are now only stored in bundle-states.json, we just check if the file exists and is valid
+    let bundleStates = [];
+    let bundleStatesExists = fs.existsSync(bundleStatesPath);
+    
+    if (bundleStatesExists) {
+      try {
+        bundleStates = JSON.parse(fs.readFileSync(bundleStatesPath, 'utf8'));
+      } catch (error) {
+        bundleStatesExists = false;
+      }
+    }
+
+    const details = {
+      inSync: bundleStatesExists && bundleStates.length > 0,
+      bundleCount: bundleStates.length,
+      bundleNames: bundleStates.map(b => b.name),
+      hasValidBundleFile: bundleStatesExists,
+      message: bundleStatesExists 
+        ? `Found ${bundleStates.length} bundles in bundle-states.json`
+        : 'bundle-states.json file not found or invalid'
+    };
+
+    res.writeHead(200, { 'Content-Type': 'application/json' });
+    res.end(JSON.stringify(details));
+  }
+
+  async handlePostBundleSyncStatus(req, res) {
+    const bundleStatesPath = path.join(this.configManager.CNTX_DIR, 'bundle-states.json');
+
+    if (!fs.existsSync(bundleStatesPath)) {
+      res.writeHead(404, { 'Content-Type': 'application/json' });
+      res.end(JSON.stringify({ message: 'bundle-states.json not found' }));
+      return;
+    }
+
+    try {
+      // Validate and reload bundle states to ensure they're in sync with the file
+      const bundleStates = JSON.parse(fs.readFileSync(bundleStatesPath, 'utf8'));
+      
+      // Reload bundle states in configuration manager
+      this.configManager.loadBundleStates();
+      
+      res.writeHead(200, { 'Content-Type': 'application/json' });
+      res.end(JSON.stringify({ 
+        message: 'Bundle states reloaded successfully',
+        bundleCount: bundleStates.length,
+        bundles: bundleStates.map(b => b.name)
+      }));
+    } catch (error) {
+      res.writeHead(500, { 'Content-Type': 'application/json' });
+      res.end(JSON.stringify({ 
+        message: 'Failed to reload bundle states',
+        error: error.message 
+      }));
+    }
+  }
+
+  // === Database API Handlers ===
+
+  async handleGetDatabaseInfo(req, res) {
+    try {
+      const info = this.configManager.dbManager.getInfo();
+      res.writeHead(200, { 'Content-Type': 'application/json' });
+      res.end(JSON.stringify(info));
+    } catch (error) {
+      res.writeHead(500, { 'Content-Type': 'application/json' });
+      res.end(JSON.stringify({ error: error.message }));
+    }
+  }
+
+  async handlePostDatabaseQuery(req, res) {
+    try {
+      const body = await this.getRequestBody(req);
+      const { query } = JSON.parse(body);
+      
+      if (!query || typeof query !== 'string') {
+        res.writeHead(400, { 'Content-Type': 'application/json' });
+        res.end(JSON.stringify({ error: 'Query is required' }));
+        return;
+      }
+      
+      const results = this.configManager.dbManager.query(query);
+      res.writeHead(200, { 'Content-Type': 'application/json' });
+      res.end(JSON.stringify({ results }));
+    } catch (error) {
+      res.writeHead(400, { 'Content-Type': 'application/json' });
+      res.end(JSON.stringify({ error: error.message }));
+    }
+  }
 }