cntx-ui 3.0.7 → 3.0.8

package/lib/semantic-splitter.js

@@ -9,6 +9,7 @@ import { join, extname } from 'path'
 import Parser from 'tree-sitter'
 import JavaScript from 'tree-sitter-javascript'
 import TypeScript from 'tree-sitter-typescript'
+import Rust from 'tree-sitter-rust'
 import HeuristicsManager from './heuristics-manager.js'
 
 export default class SemanticSplitter {
@@ -24,11 +25,13 @@ export default class SemanticSplitter {
     this.parsers = {
       javascript: new Parser(),
       typescript: new Parser(),
-      tsx: new Parser()
+      tsx: new Parser(),
+      rust: new Parser()
     }
     this.parsers.javascript.setLanguage(JavaScript)
     this.parsers.typescript.setLanguage(TypeScript.typescript)
     this.parsers.tsx.setLanguage(TypeScript.tsx)
+    this.parsers.rust.setLanguage(Rust)
 
     this.heuristicsManager = new HeuristicsManager()
   }
@@ -38,6 +41,7 @@ export default class SemanticSplitter {
     switch (ext) {
       case '.ts': return this.parsers.typescript
       case '.tsx': return this.parsers.tsx
+      case '.rs': return this.parsers.rust
       default: return this.parsers.javascript
     }
   }
@@ -98,7 +102,7 @@ export default class SemanticSplitter {
   }
 
   traverse(node, content, filePath, elements) {
-    // Detect Function Declarations
+    // Detect Function Declarations (JS/TS)
     if (node.type === 'function_declaration' || node.type === 'method_definition' || node.type === 'arrow_function') {
       const func = this.mapFunctionNode(node, content, filePath)
       if (func && func.code.length > this.options.minFunctionSize) {
@@ -106,14 +110,39 @@ export default class SemanticSplitter {
       }
     }
 
+    // Detect Rust function items
+    if (node.type === 'function_item') {
+      const func = this.mapFunctionNode(node, content, filePath)
+      if (func && func.code.length > this.options.minFunctionSize) {
+        elements.functions.push(func)
+      }
+    }
+
     // Detect Type Definitions (TS)
     if (node.type === 'interface_declaration' || node.type === 'type_alias_declaration') {
       const typeDef = this.mapTypeNode(node, content, filePath)
       if (typeDef) elements.types.push(typeDef)
     }
 
+    // Detect Rust type definitions
+    if (node.type === 'struct_item' || node.type === 'enum_item' || node.type === 'trait_item') {
+      const typeDef = this.mapTypeNode(node, content, filePath)
+      if (typeDef) elements.types.push(typeDef)
+    }
+
+    // Detect Rust impl blocks — traverse into body for methods
+    if (node.type === 'impl_item') {
+      const body = node.childForFieldName('body')
+      if (body) {
+        for (let i = 0; i < body.namedChildCount; i++) {
+          this.traverse(body.namedChild(i), content, filePath, elements)
+        }
+      }
+      return // Don't recurse again below
+    }
+
     // Recurse unless we've already captured the block (like a function body)
-    if (node.type !== 'function_declaration' && node.type !== 'method_definition') {
+    if (node.type !== 'function_declaration' && node.type !== 'method_definition' && node.type !== 'function_item') {
       for (let i = 0; i < node.namedChildCount; i++) {
         this.traverse(node.namedChild(i), content, filePath, elements)
       }
@@ -124,7 +153,7 @@ export default class SemanticSplitter {
     let name = 'anonymous';
     
     // Find name identifier based on node type
-    if (node.type === 'function_declaration' || node.type === 'method_definition') {
+    if (node.type === 'function_declaration' || node.type === 'method_definition' || node.type === 'function_item') {
       const nameNode = node.childForFieldName('name');
       if (nameNode) name = content.slice(nameNode.startIndex, nameNode.endIndex);
     } else if (node.type === 'arrow_function') {
@@ -175,10 +204,10 @@ export default class SemanticSplitter {
 
   extractImports(root, content, filePath) {
     const imports = []
-    // Simple traversal for import statements
+    // Simple traversal for import/use statements
     for (let i = 0; i < root.namedChildCount; i++) {
       const node = root.namedChild(i)
-      if (node.type === 'import_statement') {
+      if (node.type === 'import_statement' || node.type === 'use_declaration') {
         imports.push({
           statement: content.slice(node.startIndex, node.endIndex),
           filePath
@@ -189,6 +218,11 @@ export default class SemanticSplitter {
   }
 
   isExported(node) {
+    // Rust: check for visibility_modifier (pub) as direct child
+    for (let i = 0; i < node.namedChildCount; i++) {
+      if (node.namedChild(i).type === 'visibility_modifier') return true
+    }
+    // JS/TS: check for export_statement ancestor
     let current = node
     while (current) {
       if (current.type === 'export_statement') return true
@@ -250,7 +284,16 @@ export default class SemanticSplitter {
   }
 
   isImportRelevant(importStatement, functionCode) {
-    // Heuristic: does the function use any name from the import?
+    // Rust use statements: use std::collections::HashMap;
+    const useMatch = importStatement.match(/^use\s+(.+);?\s*$/)
+    if (useMatch) {
+      const path = useMatch[1]
+      // Extract the last segment (the actual imported name)
+      const segments = path.replace(/[{}]/g, '').split('::')
+      const lastSegment = segments[segments.length - 1].trim()
+      return functionCode.includes(lastSegment)
+    }
+    // JS/TS import statements
     const match = importStatement.match(/import\s+(?:\{([^}]+)\}|(\w+))/i)
     if (!match) return false
     const importedNames = match[1] ? match[1].split(',').map(n => n.trim()) : [match[2]]
@@ -258,7 +301,7 @@ export default class SemanticSplitter {
   }
 
   calculateComplexity(code) {
-    const indicators = ['if', 'else', 'for', 'while', 'switch', 'case', 'catch', '?', '&&', '||'];
+    const indicators = ['if', 'else', 'for', 'while', 'switch', 'case', 'catch', '?', '&&', '||', 'match', 'loop', 'unsafe', 'unwrap', 'expect'];
     let score = 1;
     indicators.forEach(ind => {
       // Escape special regex characters

package/lib/treesitter-semantic-chunker.js

@@ -1,19 +1,18 @@
 /**
- * Treesitter-based Semantic Chunker for JavaScript/TypeScript Files
+ * Treesitter-based Semantic Chunker for JavaScript/TypeScript and Rust Files
  * Uses tree-sitter for true AST-based code analysis and semantic chunking
- * Supports JS/TS/JSX/TSX with equal treatment
+ * Supports JS/TS/JSX/TSX and Rust with equal treatment
  * Node ecosystem focus: React components, Express APIs, CLI tools, utilities
+ * Rust ecosystem focus: standalone functions, structs, enums, traits
  */
 
 import { readFileSync, existsSync } from 'fs'
 import { extname, basename, dirname, relative, join } from 'path'
-import glob from 'glob'
-import { promisify } from 'util'
+import { glob } from 'glob'
 import Parser from 'tree-sitter'
 import JavaScript from 'tree-sitter-javascript'
 import TypeScript from 'tree-sitter-typescript'
-
-const globAsync = promisify(glob)
+import Rust from 'tree-sitter-rust'
 
 class TreesitterSemanticChunker {
   constructor(options = {}) {
@@ -32,7 +31,7 @@ class TreesitterSemanticChunker {
     this.parsers = {}
     this.initializeParsers()
     
-    // Semantic patterns for Node ecosystem
+    // Semantic patterns for Node/Rust ecosystem
     this.semanticPatterns = {
       reactComponent: this.isReactComponent.bind(this),
       reactHook: this.isReactHook.bind(this),
@@ -61,6 +60,10 @@ class TreesitterSemanticChunker {
     // TSX parser
     this.parsers.tsx = new Parser()
     this.parsers.tsx.setLanguage(TypeScript.tsx)
+
+    // Rust parser
+    this.parsers.rust = new Parser()
+    this.parsers.rust.setLanguage(Rust)
   }
 
   /**
@@ -71,6 +74,7 @@ class TreesitterSemanticChunker {
     switch (ext) {
       case '.ts': return this.parsers.typescript
       case '.tsx': return this.parsers.tsx
+      case '.rs': return this.parsers.rust
       case '.js':
       case '.jsx':
       default: return this.parsers.javascript
@@ -80,7 +84,7 @@ class TreesitterSemanticChunker {
   /**
    * Main entry point - analyze files and create semantic chunks
    */
-  async analyzeProject(projectPath, patterns = ['**/*.{js,jsx,ts,tsx}']) {
+  async analyzeProject(projectPath, patterns = ['**/*.{js,jsx,ts,tsx,rs}']) {
     console.log('🔍 Starting treesitter-based semantic analysis...')
     
     const files = await this.findFiles(projectPath, patterns)
@@ -115,7 +119,7 @@ class TreesitterSemanticChunker {
     const files = []
     
     for (const pattern of patterns) {
-      const matches = await globAsync(pattern, {
+      const matches = await glob(pattern, {
         cwd: projectPath,
         ignore: [
           'node_modules/**', 'dist/**', 'build/**', '.git/**',
@@ -186,11 +190,6 @@ class TreesitterSemanticChunker {
       // Use simple string parsing (confirmed working in tests)
       tree = parser.parse(content)
       rootNode = tree.rootNode
-      
-      // Check for parse errors
-      if (rootNode.hasError()) {
-        throw new Error('Parse error in file')
-      }
     } catch (error) {
       throw new Error(`Tree-sitter parse failed: ${error.message}`)
     }
@@ -239,7 +238,7 @@ class TreesitterSemanticChunker {
   extractFunctions(rootNode, content) {
     const functions = []
     
-    // Function declarations
+    // JS/TS Function declarations
     const functionDeclarations = this.queryNode(rootNode, '(function_declaration name: (identifier) @name)')
     functions.push(...functionDeclarations.map(capture => ({
       name: this.getNodeText(capture.node, content),
@@ -249,7 +248,7 @@ class TreesitterSemanticChunker {
       isExported: this.isNodeExported(capture.node)
     })))
     
-    // Arrow functions
+    // JS/TS Arrow functions
     const arrowFunctions = this.queryNode(rootNode, '(variable_declarator name: (identifier) @name value: (arrow_function))')
     functions.push(...arrowFunctions.map(capture => ({
       name: this.getNodeText(capture.node, content),
@@ -259,7 +258,7 @@ class TreesitterSemanticChunker {
       isExported: this.isNodeExported(capture.node.parent.parent)
     })))
     
-    // Method definitions
+    // JS/TS Method definitions
     const methods = this.queryNode(rootNode, '(method_definition name: (property_name) @name)')
     functions.push(...methods.map(capture => ({
       name: this.getNodeText(capture.node, content),
@@ -268,6 +267,16 @@ class TreesitterSemanticChunker {
       endPosition: capture.node.endPosition,
       isExported: false // methods are part of classes
     })))
+
+    // Rust function items
+    const rustFunctions = this.queryNode(rootNode, '(function_item name: (identifier) @name)')
+    functions.push(...rustFunctions.map(capture => ({
+      name: this.getNodeText(capture.node, content),
+      type: 'function_item',
+      startPosition: capture.node.startPosition,
+      endPosition: capture.node.endPosition,
+      isExported: this.isNodeExported(capture.node)
+    })))
     
     return functions
   }
@@ -318,6 +327,7 @@ class TreesitterSemanticChunker {
   extractImports(rootNode, content) {
     const imports = []
     
+    // JS/TS imports
     const importStatements = this.queryNode(rootNode, '(import_statement source: (string) @source)')
     imports.push(...importStatements.map(capture => {
       const source = this.getNodeText(capture.node, content).replace(/['"]/g, '')
@@ -329,6 +339,19 @@ class TreesitterSemanticChunker {
         importedNames: this.extractImportedNames(capture.node.parent, content)
       }
     }))
+
+    // Rust use declarations
+    const rustUseStatements = this.queryNode(rootNode, '(use_declaration)')
+    imports.push(...rustUseStatements.map(capture => {
+      const statement = this.getNodeText(capture.node, content)
+      return {
+        source: 'rust',
+        statement,
+        isRelative: statement.includes('self::') || statement.includes('super::'),
+        isExternal: !statement.includes('crate::') && !statement.includes('self::') && !statement.includes('super::'),
+        importedNames: [] // Complexity of parsing Rust use paths is high for this simple chunker
+      }
+    }))
     
     return imports
   }
@@ -339,7 +362,7 @@ class TreesitterSemanticChunker {
   extractExports(rootNode, content) {
     const exports = []
     
-    // Export declarations
+    // Export declarations (JS/TS)
     const exportDeclarations = this.queryNode(rootNode, '(export_statement)')
     exports.push(...exportDeclarations.map(capture => {
       const exportNode = capture.node
@@ -405,7 +428,7 @@ class TreesitterSemanticChunker {
     const types = []
     
     try {
-      // Interface declarations
+      // JS/TS Interface declarations
       const interfaces = this.queryNode(rootNode, '(interface_declaration name: (type_identifier) @name)')
       types.push(...interfaces.map(capture => ({
         name: this.getNodeText(capture.node, content),
@@ -415,7 +438,7 @@ class TreesitterSemanticChunker {
         isExported: this.isNodeExported(capture.node.parent)
       })))
       
-      // Type alias declarations  
+      // JS/TS Type alias declarations  
       const typeAliases = this.queryNode(rootNode, '(type_alias_declaration name: (type_identifier) @name)')
       types.push(...typeAliases.map(capture => ({
         name: this.getNodeText(capture.node, content),
@@ -424,8 +447,38 @@ class TreesitterSemanticChunker {
         endPosition: capture.node.endPosition,
         isExported: this.isNodeExported(capture.node.parent)
       })))
+
+      // Rust struct definitions
+      const structs = this.queryNode(rootNode, '(struct_item name: (type_identifier) @name)')
+      types.push(...structs.map(capture => ({
+        name: this.getNodeText(capture.node, content),
+        type: 'struct',
+        startPosition: capture.node.startPosition,
+        endPosition: capture.node.endPosition,
+        isExported: this.isNodeExported(capture.node)
+      })))
+
+      // Rust enum definitions
+      const enums = this.queryNode(rootNode, '(enum_item name: (type_identifier) @name)')
+      types.push(...enums.map(capture => ({
+        name: this.getNodeText(capture.node, content),
+        type: 'enum',
+        startPosition: capture.node.startPosition,
+        endPosition: capture.node.endPosition,
+        isExported: this.isNodeExported(capture.node)
+      })))
+
+      // Rust trait definitions
+      const traits = this.queryNode(rootNode, '(trait_item name: (type_identifier) @name)')
+      types.push(...traits.map(capture => ({
+        name: this.getNodeText(capture.node, content),
+        type: 'trait',
+        startPosition: capture.node.startPosition,
+        endPosition: capture.node.endPosition,
+        isExported: this.isNodeExported(capture.node)
+      })))
     } catch (error) {
-      // TypeScript types might not be available in JavaScript parser
+      // TypeScript/Rust types might not be available
     }
     
     return types
@@ -576,6 +629,18 @@ class TreesitterSemanticChunker {
     return hasTypeFileName || hasOnlyTypes
   }
 
+  /**
+   * Semantic pattern: UI Component (generic for any language)
+   */
+  isUiComponent(rootNode, content, filePath) {
+    // JS/TS logic
+    if (extname(filePath).match(/\.(jsx|tsx|js|ts)$/)) {
+      return this.isReactComponent(rootNode, content, filePath)
+    }
+    // TODO: Add generic patterns for other languages (Rust templates, etc.)
+    return false
+  }
+
   /**
    * Semantic pattern: Config Module
    */
@@ -583,7 +648,7 @@ class TreesitterSemanticChunker {
     const fileName = basename(filePath).toLowerCase()
     const hasConfigName = fileName.includes('config') || fileName.includes('setting')
     
-    const hasConfigPatterns = content.includes('module.exports') || content.includes('export default')
+    const hasConfigPatterns = content.includes('module.exports') || content.includes('export default') || content.includes('Cargo.toml')
     const hasConfigObject = /\{[\s\S]*\}/.test(content) && !/function|class/.test(content)
     
     return hasConfigName && (hasConfigPatterns || hasConfigObject)
@@ -630,11 +695,13 @@ class TreesitterSemanticChunker {
     if (content.includes('react')) patterns.push('react')
     if (content.includes('express')) patterns.push('express')
     if (content.includes('typescript')) patterns.push('typescript')
+    if (content.includes('cargo') || extname(content) === '.rs') patterns.push('rust')
     
     // Architecture patterns
-    if (content.includes('async') && content.includes('await')) patterns.push('async-await')
+    if (content.includes('async') && (content.includes('await') || content.includes('.await'))) patterns.push('async-await')
     if (content.includes('Promise')) patterns.push('promises')
     if (content.includes('class') && content.includes('extends')) patterns.push('inheritance')
+    if (content.includes('unsafe')) patterns.push('unsafe-code')
     
     // Design patterns
     const functions = this.extractFunctions(rootNode, content)
@@ -996,10 +1063,14 @@ class TreesitterSemanticChunker {
   }
 
   isNodeExported(node) {
-    // Check if node is part of an export statement
+    // Rust: check for visibility_modifier (pub) as direct child
+    for (let i = 0; i < node.namedChildCount; i++) {
+      if (node.namedChild(i).type === 'visibility_modifier') return true
+    }
+    // JS/TS: check for export_statement ancestor or parent
     let parent = node.parent
     while (parent) {
-      if (parent.type === 'export_statement') {
+      if (parent.type === 'export_statement' || parent.type === 'export_declaration') {
         return true
       }
       parent = parent.parent
@@ -1048,7 +1119,8 @@ class TreesitterSemanticChunker {
     let complexity = 1
     
     const complexityNodes = ['if_statement', 'while_statement', 'for_statement',
-                            'switch_statement', 'try_statement', 'catch_clause']
+                            'switch_statement', 'try_statement', 'catch_clause',
+                            'match_arm', 'loop_expression']
     
     const traverse = (node) => {
       if (complexityNodes.includes(node.type)) {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "cntx-ui",
   "type": "module",
-  "version": "3.0.7",
+  "version": "3.0.8",
   "description": "Autonomous Repository Intelligence engine with web UI and MCP server. Unified semantic code understanding, local RAG, and agent working memory.",
   "keywords": [
     "repository-intelligence",
@@ -49,6 +49,7 @@
     "glob": "^9.0.0",
     "tree-sitter": "^0.21.1",
     "tree-sitter-javascript": "^0.23.1",
+    "tree-sitter-rust": "^0.21.0",
     "tree-sitter-typescript": "^0.23.2",
     "ws": "^8.13.0"
   }

package/server.js

@@ -361,7 +361,7 @@ export class CntxServer {
 
     // 2. Perform fresh analysis if DB is empty
     try {
-      const supportedExtensions = ['.js', '.jsx', '.ts', '.tsx', '.mjs'];
+      const supportedExtensions = ['.js', '.jsx', '.ts', '.tsx', '.mjs', '.rs'];
       const files = this.fileSystemManager.getAllFiles()
         .filter(f => supportedExtensions.includes(extname(f).toLowerCase()))
         .map(f => relative(this.CWD, f));