cntx-ui 2.0.15 → 3.0.1

package/server.js

@@ -12,10 +12,12 @@ import { homedir } from 'os';
 
 // Import our modular components
 import ConfigurationManager from './lib/configuration-manager.js';
+import DatabaseManager from './lib/database-manager.js';
 import FileSystemManager from './lib/file-system-manager.js';
 import BundleManager from './lib/bundle-manager.js';
 import APIRouter from './lib/api-router.js';
 import WebSocketManager from './lib/websocket-manager.js';
+import { AgentRuntime } from './lib/agent-runtime.js';
 
 // Import existing lib modules
 import { startMCPTransport } from './lib/mcp-transport.js';
@@ -51,8 +53,12 @@ export class CntxServer {
     this.mcpServer = null;
     this.initMessages = []; // Track initialization messages
 
+    // Ensure directory exists early
+    if (!existsSync(this.CNTX_DIR)) mkdirSync(this.CNTX_DIR, { recursive: true });
+
     // Initialize modular components
     this.configManager = new ConfigurationManager(cwd, { verbose: this.verbose });
+    this.databaseManager = new DatabaseManager(this.CNTX_DIR, { verbose: this.verbose });
     this.fileSystemManager = new FileSystemManager(cwd, { verbose: this.verbose });
     this.bundleManager = new BundleManager(this.configManager, this.fileSystemManager, this.verbose);
     this.webSocketManager = new WebSocketManager(this.bundleManager, this.configManager, { verbose: this.verbose });
@@ -61,19 +67,20 @@ export class CntxServer {
     this.semanticSplitter = new SemanticSplitter({
       maxChunkSize: 2000,
       includeContext: true,
-      groupRelated: true,
       minFunctionSize: 50
     });
 
-    this.vectorStore = new SimpleVectorStore({
-      modelName: 'Xenova/all-MiniLM-L6-v2',
-      collectionName: 'code-chunks'
+    this.vectorStore = new SimpleVectorStore(this.databaseManager, {
+      modelName: 'Xenova/all-MiniLM-L6-v2'
     });
 
     this.semanticCache = null;
     this.lastSemanticAnalysis = null;
     this.vectorStoreInitialized = false;
 
+    // Initialize Agent Runtime
+    this.agentRuntime = new AgentRuntime(this);
+
     // Create semantic analysis manager object for API router
     this.semanticAnalysisManager = {
       getSemanticAnalysis: () => this.getSemanticAnalysis(),
@@ -91,6 +98,7 @@ export class CntxServer {
 
     // Initialize API router with all managers
     this.apiRouter = new APIRouter(
+      this,
       this.configManager,
       this.bundleManager,
       this.fileSystemManager,
@@ -102,7 +110,6 @@ export class CntxServer {
     // Add references for cross-module communication
     this.bundleManager.fileSystemManager = this.fileSystemManager;
     this.bundleManager.webSocketManager = this.webSocketManager;
-    this.apiRouter.mcpServerStarted = this.mcpServerStarted;
   }
 
   // Progress bar utility
@@ -227,6 +234,11 @@ export class CntxServer {
       this.startWatching();
       await progress.next(steps[3], 600);
 
+      // Trigger initial semantic analysis in background if no cache
+      if (!this.semanticCache) {
+        this.getSemanticAnalysis().catch(err => console.error('Initial semantic analysis failed:', err.message));
+      }
+
       // Step 5: Generating bundles
       if (!skipBundleGeneration) {
         this.bundleManager.generateAllBundles();
@@ -236,6 +248,9 @@ export class CntxServer {
 
     // Complete progress bar
     progress.complete();
+
+    // Generate Agent Manifest
+    await this.agentRuntime.generateAgentManifest();
   }
 
   // Display initialization summary
@@ -402,73 +417,78 @@ export class CntxServer {
   // === Semantic Analysis (Legacy methods for compatibility) ===
 
   async getSemanticAnalysis() {
-    // First, try to load from cache
-    if (!this.semanticCache) {
-      const cacheData = this.configManager.loadSemanticCache();
-      if (cacheData) {
-        this.semanticCache = cacheData.analysis;
-        this.lastSemanticAnalysis = cacheData.timestamp;
-        return cacheData.analysis;
+    // 1. Try to load from SQLite first
+    try {
+      const dbChunks = this.databaseManager.db.prepare('SELECT * FROM semantic_chunks').all();
+      if (dbChunks.length > 0) {
+        if (!this.semanticCache) {
+          this.semanticCache = {
+            chunks: dbChunks.map(row => this.databaseManager.mapChunkRow(row)),
+            summary: { totalChunks: dbChunks.length }
+          };
+        }
+        return this.semanticCache;
       }
+    } catch (e) {
+      console.warn('Failed to load chunks from SQLite, performing fresh analysis...');
     }
 
-    // Check if we need to refresh the semantic analysis
-    const shouldRefresh = !this.semanticCache || !this.lastSemanticAnalysis;
-
-    if (shouldRefresh) {
-      try {
-        // Auto-discover JavaScript/TypeScript files in the entire project
-        const patterns = ['**/*.{js,jsx,ts,tsx,mjs}'];
-
-        // Load bundle configuration for chunk grouping
-        let bundleConfig = null;
-        if (existsSync(this.configManager.CONFIG_FILE)) {
-          bundleConfig = JSON.parse(readFileSync(this.configManager.CONFIG_FILE, 'utf8'));
-        }
+    // 2. Perform fresh analysis if DB is empty
+    try {
+      const patterns = ['**/*.{js,jsx,ts,tsx,mjs}'];
+      let bundleConfig = null;
+      if (existsSync(this.configManager.CONFIG_FILE)) {
+        bundleConfig = JSON.parse(readFileSync(this.configManager.CONFIG_FILE, 'utf8'));
+      }
 
-        this.semanticCache = await this.semanticSplitter.extractSemanticChunks(this.CWD, patterns, bundleConfig);
-        this.lastSemanticAnalysis = Date.now();
+      this.semanticCache = await this.semanticSplitter.extractSemanticChunks(this.CWD, patterns, bundleConfig);
+      this.lastSemanticAnalysis = Date.now();
 
-        // Only enhance chunks with embeddings if they don't already have them
-        await this.enhanceSemanticChunksIfNeeded(this.semanticCache);
+      // 3. Persist chunks to SQLite immediately
+      if (this.semanticCache.chunks.length > 0) {
+        this.databaseManager.saveChunks(this.semanticCache.chunks);
+      }
 
-        // Save to disk cache
-        this.configManager.saveSemanticCache(this.semanticCache);
+      // 4. Trigger background embedding enhancement
+      this.enhanceSemanticChunksIfNeeded(this.semanticCache);
 
-        console.log('🔍 Semantic analysis complete');
-      } catch (error) {
-        console.error('Semantic analysis failed:', error.message);
-        throw new Error(`Semantic analysis failed: ${error.message}`);
-      }
+      return this.semanticCache;
+    } catch (error) {
+      console.error('Semantic analysis failed:', error.message);
+      throw new Error(`Semantic analysis failed: ${error.message}`);
     }
-
-    return this.semanticCache;
   }
 
   async refreshSemanticAnalysis() {
-    console.log('🔄 Forcing semantic analysis refresh...');
-
-    // Clear memory cache
+    console.log('🔄 Refreshing semantic analysis and database...');
+    
+    // Clear the database table but keep other data
+    this.databaseManager.db.prepare('DELETE FROM semantic_chunks').run();
+    this.databaseManager.db.prepare('DELETE FROM vector_embeddings').run();
+    
     this.semanticCache = null;
     this.lastSemanticAnalysis = null;
 
-    // Remove disk cache file
-    this.configManager.invalidateSemanticCache();
-
     return this.getSemanticAnalysis();
   }
 
   async enhanceSemanticChunksIfNeeded(analysis) {
     if (!analysis || !analysis.chunks) return;
 
-    const chunksNeedingEmbeddings = analysis.chunks.filter(chunk => !chunk.embedding);
+    // Check DB for existing embeddings to find only what's missing
+    const chunksNeedingEmbeddings = [];
+    for (const chunk of analysis.chunks) {
+      if (!this.databaseManager.getEmbedding(chunk.id)) {
+        chunksNeedingEmbeddings.push(chunk);
+      }
+    }
 
     if (chunksNeedingEmbeddings.length === 0) {
-      console.log('✅ All chunks already have embeddings');
+      console.log('✅ All chunks already have persistent embeddings');
       return;
     }
 
-    console.log(`🔧 Enhancing ${chunksNeedingEmbeddings.length} chunks with embeddings...`);
+    console.log(`🔧 Enhancing ${chunksNeedingEmbeddings.length} chunks with persistent embeddings...`);
 
     // Initialize vector store if needed
     if (!this.vectorStoreInitialized) {
@@ -476,29 +496,20 @@ export class CntxServer {
       this.vectorStoreInitialized = true;
     }
 
-    // Add embeddings to chunks that need them
+    // Add embeddings to chunks that need them and persist
     for (const chunk of chunksNeedingEmbeddings) {
       try {
-        const content = this.getChunkContentForEmbedding(chunk);
-        chunk.embedding = await this.vectorStore.generateEmbedding(content);
+        await this.vectorStore.upsertChunk(chunk);
       } catch (error) {
-        console.error(`Failed to generate embedding for chunk ${chunk.id}:`, error.message);
+        console.error(`Failed to generate/persist embedding for chunk ${chunk.id}:`, error.message);
       }
     }
+    console.log('✅ Background embedding enhancement complete');
   }
 
-  getChunkContentForEmbedding(chunk) {
-    let content = chunk.content || '';
-
-    if (chunk.businessDomains?.length > 0) {
-      content += ' ' + chunk.businessDomains.join(' ');
-    }
-
-    if (chunk.technicalPatterns?.length > 0) {
-      content += ' ' + chunk.technicalPatterns.join(' ');
-    }
-
-    return content.trim();
+  invalidateSemanticCache() {
+    this.semanticCache = null;
+    this.lastSemanticAnalysis = null;
   }
 
   async exportSemanticChunk(chunkName) {
@@ -633,7 +644,6 @@ export class CntxServer {
     if (!this.mcpServer) {
       this.mcpServer = new MCPServer(this);
       this.mcpServerStarted = true;
-      this.apiRouter.mcpServerStarted = true;
 
       if (this.verbose) {
         console.log('🔗 MCP server started');
@@ -726,26 +736,94 @@ export async function generateBundle(bundleName = 'master') {
   return bundleInfo;
 }
 
-export async function initConfig() {
-  const server = new CntxServer(process.cwd(), { verbose: false });
-  const templateDir = join(__dirname, 'templates');
-
-  // Initialize directory structure
+// Initialize project configuration
+export async function initConfig(cwd = process.cwd()) {
+  const server = new CntxServer(cwd);
+  
+  // 1. Initialize directory structure
   if (!existsSync(server.CNTX_DIR)) {
     mkdirSync(server.CNTX_DIR, { recursive: true });
     console.log('📁 Created .cntx directory');
   }
 
-  // Initialize basic configuration
+  // 2. Create .mcp.json for Claude Code discovery
+  const mcpConfigPath = join(cwd, '.mcp.json');
+  const mcpConfig = {
+    mcpServers: {
+      "cntx-ui": {
+        command: "cntx-ui",
+        args: ["mcp"],
+        cwd: "."
+      }
+    }
+  };
+  writeFileSync(mcpConfigPath, JSON.stringify(mcpConfig, null, 2), 'utf8');
+  console.log('📄 Created .mcp.json for agent auto-discovery');
+
+  // 3. Initialize basic configuration with better defaults and auto-suggestions
   server.configManager.loadConfig();
-  server.configManager.saveConfig({
-    bundles: {
-      master: ['**/*']
+  
+  const suggestedBundles = {
+    master: ['**/*']
+  };
+
+  // Directory-based auto-suggestions
+  const commonDirs = [
+    { dir: 'src/components', name: 'ui-components' },
+    { dir: 'src/services', name: 'services' },
+    { dir: 'src/lib', name: 'libraries' },
+    { dir: 'src/hooks', name: 'react-hooks' },
+    { dir: 'server', name: 'backend-api' },
+    { dir: 'tests', name: 'test-suite' }
+  ];
+
+  commonDirs.forEach(d => {
+    if (existsSync(join(cwd, d.dir))) {
+      suggestedBundles[d.name] = [`${d.dir}/**`];
+      console.log(`💡 Suggested bundle: ${d.name} (${d.dir}/**)`);
     }
   });
 
+  server.configManager.saveConfig({
+    bundles: suggestedBundles
+  });
+
+  // 4. Create robust default .cntxignore
+  const ignorePath = join(cwd, '.cntxignore');
+  if (!existsSync(ignorePath)) {
+    const defaultIgnore = `# Binary files
+*.db
+*.db-journal
+*.png
+*.jpg
+*.jpeg
+*.ico
+*.icns
+*.gif
+*.zip
+*.tar.gz
+
+# Generated files
+**/gen/**
+**/dist/**
+**/build/**
+**/node_modules/**
+**/.next/**
+**/.cache/**
+
+# cntx-ui internals
+.cntx/**
+.mcp.json
+`;
+    writeFileSync(ignorePath, defaultIgnore, 'utf8');
+    console.log('📄 Created .cntxignore with smart defaults');
+  }
+
   console.log('⚙️ Basic configuration initialized');
 
+  // ... (rest of the file remains same)
+  const templateDir = join(__dirname, 'templates');
+
   // Copy agent configuration files
   const agentFiles = [
     'agent-config.yaml',

package/templates/TOOLS.md

@@ -0,0 +1,41 @@
+# cntx-ui Tool Reference
+
+This document maps the **Model Context Protocol (MCP)** tools to their **HTTP API** equivalents.
+
+## Core Agent Tools
+
+| MCP Tool | HTTP Endpoint | Parameters | Description |
+| :--- | :--- | :--- | :--- |
+| `agent/discover` | `GET /api/status` | `scope?: string` | Architectural overview and health check. |
+| `agent/query` | `POST /api/semantic-search` | `question: string` | Semantic search across the entire codebase. |
+| `agent/investigate` | `POST /api/vector-db/search` | `feature: string` | Suggest integration points for new features. |
+| `agent/organize` | `GET /api/bundles` | `activity: string` | Audit and optimize project bundle health. |
+
+## Bundle Management
+
+| MCP Tool | HTTP Endpoint | Parameters | Description |
+| :--- | :--- | :--- | :--- |
+| `list_bundles` | `GET /api/bundles` | *None* | List all manual and smart bundles. |
+| `get_bundle` | `GET /api/bundles/:name` | `name: string` | Get full XML content of a specific bundle. |
+
+## Semantic Analysis
+
+| MCP Tool | HTTP Endpoint | Parameters | Description |
+| :--- | :--- | :--- | :--- |
+| `get_semantic_chunks`| `GET /api/semantic-chunks` | `refresh?: bool` | Get all surgically extracted code chunks. |
+| `search_by_type` | `POST /api/vector-db/search-by-type` | `type: string` | Find chunks by AST type (e.g. arrow_function). |
+| `search_by_domain` | `POST /api/vector-db/search-by-domain`| `domain: string` | Find chunks by business domain (e.g. auth). |
+
+## File Operations
+
+| MCP Tool | HTTP Endpoint | Parameters | Description |
+| :--- | :--- | :--- | :--- |
+| `read_file` | `GET /api/files/:path` | `path: string` | Read file with injected semantic metadata. |
+| `write_file` | `POST /api/files` | `path, content` | Write file with automatic backup. |
+
+## Activities
+
+| MCP Tool | HTTP Endpoint | Parameters | Description |
+| :--- | :--- | :--- | :--- |
+| `list_activities` | `GET /api/activities` | *None* | List all ongoing agent missions. |
+| `get_reasoning` | `GET /api/activities/:id/reasoning` | `id: string` | Recall agent interaction history for a task. |

package/templates/activities/activities/create-project-bundles/README.md

@@ -24,7 +24,7 @@ Set up organized bundles for a new codebase using agent assistance to analyze pr
 1. The system must analyze the existing project structure and file types
 2. The system must suggest logical bundle groupings based on project architecture
 3. The system must create bundle configuration files with clear naming and descriptions
-4. **CRITICAL**: The system must update both .cntx/bundles.json (metadata) AND .cntx/config.json (tracking patterns)
+4. **CRITICAL**: The system must update .cntx/bundle-states.json as the single source of truth for bundle definitions
 5. The system must establish rules for automatic file categorization
 6. The system must generate documentation explaining each bundle's purpose
 7. The system must allow iterative refinement of bundle structure based on user feedback
@@ -77,7 +77,8 @@ Set up organized bundles for a new codebase using agent assistance to analyze pr
 
 ## Related Files
 
-- `.cntx/config.json` - Main cntx-ui configuration
-- `.cntx/bundles.json` - Bundle definitions and rules
+- `.cntx/config.json` - Non-bundle configuration settings (editor, etc.)
+- `.cntx/bundle-states.json` - Single source of truth for all bundle definitions
 - `lib/bundle-manager.js` - Bundle management logic
+- `lib/configuration-manager.js` - Configuration and bundle state management
 - `web/src/components/BundleList.tsx` - Bundle UI components

package/templates/activities/activities/create-project-bundles/notes.md

@@ -47,33 +47,29 @@ _Consider how bundle structure should accommodate project expansion_
 
 ### Sample Bundle Configurations
 
-**bundles.json** (detailed metadata):
+**bundle-states.json** (single source of truth):
 ```json
-{
-  "name": "ui-components",
-  "description": "Reusable UI components and design system elements",
-  "patterns": ["src/components/**/*.tsx", "src/ui/**/*.tsx"],
-  "exclude": ["**/*.test.tsx", "**/*.stories.tsx"],
-  "tags": ["ui", "components"],
-  "priority": 1
-}
+[
+  {
+    "name": "ui-components",
+    "patterns": ["src/components/**/*.tsx", "src/ui/**/*.tsx"],
+    "files": ["src/components/Button.tsx", "src/ui/Card.tsx"],
+    "content": "",
+    "size": 45231,
+    "changed": false,
+    "generated": "2025-06-30T08:00:00.000Z"
+  }
+]
 ```
 
-**config.json** (tracking patterns):
+**config.json** (non-bundle settings only):
 ```json
 {
-  "bundles": {
-    "ui-components": [
-      "src/components/**/*.tsx",
-      "src/ui/**/*.tsx",
-      "!**/*.test.tsx",
-      "!**/*.stories.tsx"
-    ]
-  }
+  "editor": "cursor"
 }
 ```
 
-⚠️ **CRITICAL**: Both files must be updated for bundles to appear in the UI!
+⚠️ **CRITICAL**: Use .cntx/bundle-states.json as the single source of truth for all bundle data!
 
 ### File Organization Patterns
 - Component files: Component.tsx, Component.test.tsx, Component.stories.tsx

package/templates/activities/activities/create-project-bundles/tasks.md

@@ -1,7 +1,7 @@
 ## Relevant Files
 
-- `.cntx/config.json` - Main configuration file that will be updated with bundle settings
-- `.cntx/bundles.json` - Bundle definitions file that will contain all bundle configurations
+- `.cntx/config.json` - Non-bundle configuration settings (editor preferences, etc.)
+- `.cntx/bundle-states.json` - Single source of truth for all bundle definitions and metadata
 - `lib/bundle-manager.js` - Core bundle management logic for processing and applying bundle rules
 - `web/src/components/BundleList.tsx` - UI component for displaying and managing bundles
 - `web/src/components/BundleDetails.tsx` - UI component for viewing individual bundle contents
@@ -27,8 +27,8 @@
   - [ ] 2.3 Create bundle descriptions that explain their purpose and contents
   - [ ] 2.4 Validate bundle strategy with user and refine based on feedback
 - [ ] 3.0 Bundle Configuration Implementation
-  - [ ] 3.1 Create bundle definitions in .cntx/bundles.json with detailed metadata
-  - [ ] 3.2 **CRITICAL**: Update .cntx/config.json with bundle patterns for tracking
+  - [ ] 3.1 Create bundle definitions in .cntx/bundle-states.json with patterns and metadata
+  - [ ] 3.2 **CRITICAL**: Use .cntx/bundle-states.json as single source of truth (no config.json bundle data)
   - [ ] 3.3 Set up automatic file categorization rules
   - [ ] 3.4 Configure bundle metadata (descriptions, tags, priorities)
   - [ ] 3.5 Test bundle rules with sample files to ensure proper categorization

package/templates/activities/activities.json

@@ -23,7 +23,7 @@
       },
       {
         "title": "Bundle Configuration Implementation",
-        "description": "Create bundle definitions in .cntx/bundles.json and set up automatic file categorization rules.",
+        "description": "Create bundle definitions in .cntx/bundle-states.json and set up automatic file categorization rules.",
         "status": "todo"
       },
       {

package/templates/agent-config.yaml

@@ -31,8 +31,6 @@ capabilities:
 # Project-specific context
 project_context:
   - "project-specific/architecture.md"
-  - "project-specific/patterns.md"
-  - "project-specific/conventions.md"
 
 # Tool-specific optimizations
 tools:
@@ -40,22 +38,11 @@ tools:
     include: ["core_rules", "capabilities", "project_context"]
     output_file: ".cursorrules"
     format: "markdown"
-    additional:
-      - "tools/cursor-specific.md"
       
   claude_code:
     include: ["core_rules", "capabilities", "project_context"]
     output_file: ".cntx/agent-instructions.md"
     format: "markdown"
-    additional:
-      - "tools/claude-specific.md"
-      
-  github_copilot:
-    include: ["core_rules", "capabilities"] 
-    output_file: ".github/copilot-instructions.md"
-    format: "markdown"
-    additional:
-      - "tools/copilot-specific.md"
 
 # Performance metrics and validation
 validation:

package/templates/agent-instructions.md

@@ -25,15 +25,31 @@ These files contain the complete workflow for creating and managing activities w
 
 ## Your Role
 
-You are an AI agent with access to semantic code analysis, bundle organization, and vector search capabilities. Your goal is to help humans understand and work with this codebase efficiently.
+You are an AI agent with access to a specialized "Repository Intelligence" engine. Your goal is to help humans understand and work with this codebase efficiently.
 
 ## Available Capabilities
 
-- **Vector Database** (PRIMARY): Real-time semantic search across 315+ code chunks with ~20ms response time
-- **Semantic Analysis**: Pre-analyzed code chunks with purpose, complexity, and relationships  
-- **Bundle System**: Logical file groupings (frontend, backend, ui-components, etc.)
-- **Activities System**: Agent task definitions and progress tracking
-- **AST Parsing**: Precise symbol and dependency information (fallback only)
+### 1. Model Context Protocol (MCP) - PRIMARY
+You have direct access to surgical intelligence tools. Refer to the **Intelligence Interface** section in `.cntx/AGENT.md` for full parameter schemas.
+
+### 2. HTTP API - FALLBACK
+If MCP is unavailable, use the HTTP endpoints documented in `.cntx/AGENT.md`.
+
+## Performance Hierarchy (Use in this order):
+
+1. **Semantic Search** (20ms, 90% token savings) - `agent/query` (MCP) or `POST /api/semantic-search` (HTTP)
+   - Use for: code discovery, pattern matching, "find functions that..."
+
+2. **Bundle System** (50ms) - `list_bundles` (MCP) or `GET /api/bundles` (HTTP)
+   - Use for: project structure, file organization, high-level overview
+
+3. **Discovery Mode** - `agent/discover` (MCP) or `GET /api/status` (HTTP)
+   - Use for: architectural overview and health check.
+
+4. **Traditional Search** (100ms+, high token cost) - `grep/rg/Read`
+   - Use ONLY when: exact string matching needed, semantic search fails.
+
+---
 
 ## Operating Modes
 

package/templates/agent-rules/capabilities/bundle-system.md

@@ -5,7 +5,7 @@ The bundle system provides logical organization of project files into meaningful
 
 ## When Available
 - Check for bundle endpoint: `GET /api/bundles`
-- Look for bundle configuration files: `.cntx/bundles.json`
+- Look for bundle configuration files: `.cntx/bundle-states.json`
 - Verify bundle status and organization
 
 ## Core Capabilities

package/templates/agent-rules/project-specific/architecture.md

@@ -61,7 +61,7 @@ cntx-ui/
 ├── .cntx/                        # Configuration and cache
 │   ├── activities/               # Agent task definitions
 │   ├── agent-rules/              # Modular agent instructions
-│   ├── bundles.json             # Bundle configuration
+│   ├── bundle-states.json       # Bundle configuration and metadata
 │   └── semantic-cache.json      # Analysis cache
 └── server.js                    # Main HTTP server and API
 ```