gitnexus 1.0.0

package/dist/core/embeddings/embedding-pipeline.d.ts

@@ -0,0 +1,50 @@
+/**
+ * Embedding Pipeline Module
+ *
+ * Orchestrates the background embedding process:
+ * 1. Query embeddable nodes from KuzuDB
+ * 2. Generate text representations
+ * 3. Batch embed using transformers.js
+ * 4. Update KuzuDB with embeddings
+ * 5. Create vector index for semantic search
+ */
+import { type EmbeddingProgress, type EmbeddingConfig, type SemanticSearchResult } from './types.js';
+/**
+ * Progress callback type
+ */
+export type EmbeddingProgressCallback = (progress: EmbeddingProgress) => void;
+/**
+ * Run the embedding pipeline
+ *
+ * @param executeQuery - Function to execute Cypher queries against KuzuDB
+ * @param executeWithReusedStatement - Function to execute with reused prepared statement
+ * @param onProgress - Callback for progress updates
+ * @param config - Optional configuration override
+ */
+export declare const runEmbeddingPipeline: (executeQuery: (cypher: string) => Promise<any[]>, executeWithReusedStatement: (cypher: string, paramsList: Array<Record<string, any>>) => Promise<void>, onProgress: EmbeddingProgressCallback, config?: Partial<EmbeddingConfig>) => Promise<void>;
+/**
+ * Perform semantic search using the vector index
+ *
+ * Uses CodeEmbedding table and queries each node table to get metadata
+ *
+ * @param executeQuery - Function to execute Cypher queries
+ * @param query - Search query text
+ * @param k - Number of results to return (default: 10)
+ * @param maxDistance - Maximum distance threshold (default: 0.5)
+ * @returns Array of search results ordered by relevance
+ */
+export declare const semanticSearch: (executeQuery: (cypher: string) => Promise<any[]>, query: string, k?: number, maxDistance?: number) => Promise<SemanticSearchResult[]>;
+/**
+ * Semantic search with graph expansion (flattened results)
+ *
+ * Note: With multi-table schema, graph traversal is simplified.
+ * Returns semantic matches with their metadata.
+ * For full graph traversal, use execute_vector_cypher tool directly.
+ *
+ * @param executeQuery - Function to execute Cypher queries
+ * @param query - Search query text
+ * @param k - Number of initial semantic matches (default: 5)
+ * @param _hops - Unused (kept for API compatibility).
+ * @returns Semantic matches with metadata
+ */
+export declare const semanticSearchWithContext: (executeQuery: (cypher: string) => Promise<any[]>, query: string, k?: number, _hops?: number) => Promise<any[]>;

package/dist/core/embeddings/embedding-pipeline.js

@@ -0,0 +1,321 @@
+/**
+ * Embedding Pipeline Module
+ *
+ * Orchestrates the background embedding process:
+ * 1. Query embeddable nodes from KuzuDB
+ * 2. Generate text representations
+ * 3. Batch embed using transformers.js
+ * 4. Update KuzuDB with embeddings
+ * 5. Create vector index for semantic search
+ */
+import { initEmbedder, embedBatch, embedText, embeddingToArray, isEmbedderReady } from './embedder.js';
+import { generateBatchEmbeddingTexts } from './text-generator.js';
+import { DEFAULT_EMBEDDING_CONFIG, EMBEDDABLE_LABELS, } from './types.js';
+const isDev = process.env.NODE_ENV !== 'production';
+/**
+ * Query all embeddable nodes from KuzuDB
+ * Uses table-specific queries (File has different schema than code elements)
+ */
+const queryEmbeddableNodes = async (executeQuery) => {
+    const allNodes = [];
+    // Query each embeddable table with table-specific columns
+    for (const label of EMBEDDABLE_LABELS) {
+        try {
+            let query;
+            if (label === 'File') {
+                // File nodes don't have startLine/endLine
+                query = `
+          MATCH (n:File)
+          RETURN n.id AS id, n.name AS name, 'File' AS label, 
+                 n.filePath AS filePath, n.content AS content
+        `;
+            }
+            else {
+                // Code elements have startLine/endLine
+                query = `
+          MATCH (n:${label})
+          RETURN n.id AS id, n.name AS name, '${label}' AS label, 
+                 n.filePath AS filePath, n.content AS content,
+                 n.startLine AS startLine, n.endLine AS endLine
+        `;
+            }
+            const rows = await executeQuery(query);
+            for (const row of rows) {
+                allNodes.push({
+                    id: row.id ?? row[0],
+                    name: row.name ?? row[1],
+                    label: row.label ?? row[2],
+                    filePath: row.filePath ?? row[3],
+                    content: row.content ?? row[4] ?? '',
+                    startLine: row.startLine ?? row[5],
+                    endLine: row.endLine ?? row[6],
+                });
+            }
+        }
+        catch (error) {
+            // Table might not exist or be empty, continue
+            if (isDev) {
+                console.warn(`Query for ${label} nodes failed:`, error);
+            }
+        }
+    }
+    return allNodes;
+};
+/**
+ * Batch INSERT embeddings into separate CodeEmbedding table
+ * Using a separate lightweight table avoids copy-on-write overhead
+ * that occurs when UPDATEing nodes with large content fields
+ */
+const batchInsertEmbeddings = async (executeWithReusedStatement, updates) => {
+    // INSERT into separate embedding table - much more memory efficient!
+    const cypher = `CREATE (e:CodeEmbedding {nodeId: $nodeId, embedding: $embedding})`;
+    const paramsList = updates.map(u => ({ nodeId: u.id, embedding: u.embedding }));
+    await executeWithReusedStatement(cypher, paramsList);
+};
+/**
+ * Create the vector index for semantic search
+ * Now indexes the separate CodeEmbedding table
+ */
+const createVectorIndex = async (executeQuery) => {
+    const cypher = `
+    CALL CREATE_VECTOR_INDEX('CodeEmbedding', 'code_embedding_idx', 'embedding', metric := 'cosine')
+  `;
+    try {
+        await executeQuery(cypher);
+    }
+    catch (error) {
+        // Index might already exist
+        if (isDev) {
+            console.warn('Vector index creation warning:', error);
+        }
+    }
+};
+/**
+ * Run the embedding pipeline
+ *
+ * @param executeQuery - Function to execute Cypher queries against KuzuDB
+ * @param executeWithReusedStatement - Function to execute with reused prepared statement
+ * @param onProgress - Callback for progress updates
+ * @param config - Optional configuration override
+ */
+export const runEmbeddingPipeline = async (executeQuery, executeWithReusedStatement, onProgress, config = {}) => {
+    const finalConfig = { ...DEFAULT_EMBEDDING_CONFIG, ...config };
+    try {
+        // Phase 1: Load embedding model
+        onProgress({
+            phase: 'loading-model',
+            percent: 0,
+            modelDownloadPercent: 0,
+        });
+        await initEmbedder((modelProgress) => {
+            // Report model download progress
+            const downloadPercent = modelProgress.progress ?? 0;
+            onProgress({
+                phase: 'loading-model',
+                percent: Math.round(downloadPercent * 0.2), // 0-20% for model loading
+                modelDownloadPercent: downloadPercent,
+            });
+        }, finalConfig);
+        onProgress({
+            phase: 'loading-model',
+            percent: 20,
+            modelDownloadPercent: 100,
+        });
+        if (isDev) {
+            console.log('🔍 Querying embeddable nodes...');
+        }
+        // Phase 2: Query embeddable nodes
+        const nodes = await queryEmbeddableNodes(executeQuery);
+        const totalNodes = nodes.length;
+        if (isDev) {
+            console.log(`📊 Found ${totalNodes} embeddable nodes`);
+        }
+        if (totalNodes === 0) {
+            onProgress({
+                phase: 'ready',
+                percent: 100,
+                nodesProcessed: 0,
+                totalNodes: 0,
+            });
+            return;
+        }
+        // Phase 3: Batch embed nodes
+        const batchSize = finalConfig.batchSize;
+        const totalBatches = Math.ceil(totalNodes / batchSize);
+        let processedNodes = 0;
+        onProgress({
+            phase: 'embedding',
+            percent: 20,
+            nodesProcessed: 0,
+            totalNodes,
+            currentBatch: 0,
+            totalBatches,
+        });
+        for (let batchIndex = 0; batchIndex < totalBatches; batchIndex++) {
+            const start = batchIndex * batchSize;
+            const end = Math.min(start + batchSize, totalNodes);
+            const batch = nodes.slice(start, end);
+            // Generate texts for this batch
+            const texts = generateBatchEmbeddingTexts(batch, finalConfig);
+            // Embed the batch
+            const embeddings = await embedBatch(texts);
+            // Update KuzuDB with embeddings
+            const updates = batch.map((node, i) => ({
+                id: node.id,
+                embedding: embeddingToArray(embeddings[i]),
+            }));
+            await batchInsertEmbeddings(executeWithReusedStatement, updates);
+            processedNodes += batch.length;
+            // Report progress (20-90% for embedding phase)
+            const embeddingProgress = 20 + ((processedNodes / totalNodes) * 70);
+            onProgress({
+                phase: 'embedding',
+                percent: Math.round(embeddingProgress),
+                nodesProcessed: processedNodes,
+                totalNodes,
+                currentBatch: batchIndex + 1,
+                totalBatches,
+            });
+        }
+        // Phase 4: Create vector index
+        onProgress({
+            phase: 'indexing',
+            percent: 90,
+            nodesProcessed: totalNodes,
+            totalNodes,
+        });
+        if (isDev) {
+            console.log('📇 Creating vector index...');
+        }
+        await createVectorIndex(executeQuery);
+        // Complete
+        onProgress({
+            phase: 'ready',
+            percent: 100,
+            nodesProcessed: totalNodes,
+            totalNodes,
+        });
+        if (isDev) {
+            console.log('✅ Embedding pipeline complete!');
+        }
+    }
+    catch (error) {
+        const errorMessage = error instanceof Error ? error.message : 'Unknown error';
+        if (isDev) {
+            console.error('❌ Embedding pipeline error:', error);
+        }
+        onProgress({
+            phase: 'error',
+            percent: 0,
+            error: errorMessage,
+        });
+        throw error;
+    }
+};
+/**
+ * Perform semantic search using the vector index
+ *
+ * Uses CodeEmbedding table and queries each node table to get metadata
+ *
+ * @param executeQuery - Function to execute Cypher queries
+ * @param query - Search query text
+ * @param k - Number of results to return (default: 10)
+ * @param maxDistance - Maximum distance threshold (default: 0.5)
+ * @returns Array of search results ordered by relevance
+ */
+export const semanticSearch = async (executeQuery, query, k = 10, maxDistance = 0.5) => {
+    if (!isEmbedderReady()) {
+        throw new Error('Embedding model not initialized. Run embedding pipeline first.');
+    }
+    // Embed the query
+    const queryEmbedding = await embedText(query);
+    const queryVec = embeddingToArray(queryEmbedding);
+    const queryVecStr = `[${queryVec.join(',')}]`;
+    // Query the vector index on CodeEmbedding to get nodeIds and distances
+    const vectorQuery = `
+    CALL QUERY_VECTOR_INDEX('CodeEmbedding', 'code_embedding_idx', 
+      CAST(${queryVecStr} AS FLOAT[384]), ${k})
+    YIELD node AS emb, distance
+    WITH emb, distance
+    WHERE distance < ${maxDistance}
+    RETURN emb.nodeId AS nodeId, distance
+    ORDER BY distance
+  `;
+    const embResults = await executeQuery(vectorQuery);
+    if (embResults.length === 0) {
+        return [];
+    }
+    // Get metadata for each result by querying each node table
+    const results = [];
+    for (const embRow of embResults) {
+        const nodeId = embRow.nodeId ?? embRow[0];
+        const distance = embRow.distance ?? embRow[1];
+        // Extract label from node ID (format: Label:path:name)
+        const labelEndIdx = nodeId.indexOf(':');
+        const label = labelEndIdx > 0 ? nodeId.substring(0, labelEndIdx) : 'Unknown';
+        // Query the specific table for this node
+        // File nodes don't have startLine/endLine
+        try {
+            let nodeQuery;
+            if (label === 'File') {
+                nodeQuery = `
+          MATCH (n:File {id: '${nodeId.replace(/'/g, "''")}'}) 
+          RETURN n.name AS name, n.filePath AS filePath
+        `;
+            }
+            else {
+                nodeQuery = `
+          MATCH (n:${label} {id: '${nodeId.replace(/'/g, "''")}'}) 
+          RETURN n.name AS name, n.filePath AS filePath, 
+                 n.startLine AS startLine, n.endLine AS endLine
+        `;
+            }
+            const nodeRows = await executeQuery(nodeQuery);
+            if (nodeRows.length > 0) {
+                const nodeRow = nodeRows[0];
+                results.push({
+                    nodeId,
+                    name: nodeRow.name ?? nodeRow[0] ?? '',
+                    label,
+                    filePath: nodeRow.filePath ?? nodeRow[1] ?? '',
+                    distance,
+                    startLine: label !== 'File' ? (nodeRow.startLine ?? nodeRow[2]) : undefined,
+                    endLine: label !== 'File' ? (nodeRow.endLine ?? nodeRow[3]) : undefined,
+                });
+            }
+        }
+        catch {
+            // Table might not exist, skip
+        }
+    }
+    return results;
+};
+/**
+ * Semantic search with graph expansion (flattened results)
+ *
+ * Note: With multi-table schema, graph traversal is simplified.
+ * Returns semantic matches with their metadata.
+ * For full graph traversal, use execute_vector_cypher tool directly.
+ *
+ * @param executeQuery - Function to execute Cypher queries
+ * @param query - Search query text
+ * @param k - Number of initial semantic matches (default: 5)
+ * @param _hops - Unused (kept for API compatibility).
+ * @returns Semantic matches with metadata
+ */
+export const semanticSearchWithContext = async (executeQuery, query, k = 5, _hops = 1) => {
+    // For multi-table schema, just return semantic search results
+    // Graph traversal is complex with separate tables - use execute_vector_cypher instead
+    const results = await semanticSearch(executeQuery, query, k, 0.5);
+    return results.map(r => ({
+        matchId: r.nodeId,
+        matchName: r.name,
+        matchLabel: r.label,
+        matchPath: r.filePath,
+        distance: r.distance,
+        connectedId: null,
+        connectedName: null,
+        connectedLabel: null,
+        relationType: null,
+    }));
+};

package/dist/core/embeddings/index.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Embeddings Module
+ *
+ * Re-exports for the embedding pipeline system.
+ */
+export * from './types.js';
+export * from './embedder.js';
+export * from './text-generator.js';
+export * from './embedding-pipeline.js';

package/dist/core/embeddings/index.js

@@ -0,0 +1,9 @@
+/**
+ * Embeddings Module
+ *
+ * Re-exports for the embedding pipeline system.
+ */
+export * from './types.js';
+export * from './embedder.js';
+export * from './text-generator.js';
+export * from './embedding-pipeline.js';

package/dist/core/embeddings/text-generator.d.ts

@@ -0,0 +1,24 @@
+/**
+ * Text Generator Module
+ *
+ * Pure functions to generate embedding text from code nodes.
+ * Combines node metadata with code snippets for semantic matching.
+ */
+import type { EmbeddableNode, EmbeddingConfig } from './types.js';
+/**
+ * Generate embedding text for any embeddable node
+ * Dispatches to the appropriate generator based on node label
+ *
+ * @param node - The node to generate text for
+ * @param config - Optional configuration for max snippet length
+ * @returns Text suitable for embedding
+ */
+export declare const generateEmbeddingText: (node: EmbeddableNode, config?: Partial<EmbeddingConfig>) => string;
+/**
+ * Generate embedding texts for a batch of nodes
+ *
+ * @param nodes - Array of nodes to generate text for
+ * @param config - Optional configuration
+ * @returns Array of texts in the same order as input nodes
+ */
+export declare const generateBatchEmbeddingTexts: (nodes: EmbeddableNode[], config?: Partial<EmbeddingConfig>) => string[];

package/dist/core/embeddings/text-generator.js

@@ -0,0 +1,182 @@
+/**
+ * Text Generator Module
+ *
+ * Pure functions to generate embedding text from code nodes.
+ * Combines node metadata with code snippets for semantic matching.
+ */
+import { DEFAULT_EMBEDDING_CONFIG } from './types.js';
+/**
+ * Extract the filename from a file path
+ */
+const getFileName = (filePath) => {
+    const parts = filePath.split('/');
+    return parts[parts.length - 1] || filePath;
+};
+/**
+ * Extract the directory path from a file path
+ */
+const getDirectory = (filePath) => {
+    const parts = filePath.split('/');
+    parts.pop();
+    return parts.join('/') || '';
+};
+/**
+ * Truncate content to max length, preserving word boundaries
+ */
+const truncateContent = (content, maxLength) => {
+    if (content.length <= maxLength) {
+        return content;
+    }
+    // Find last space before maxLength to avoid cutting words
+    const truncated = content.slice(0, maxLength);
+    const lastSpace = truncated.lastIndexOf(' ');
+    if (lastSpace > maxLength * 0.8) {
+        return truncated.slice(0, lastSpace) + '...';
+    }
+    return truncated + '...';
+};
+/**
+ * Clean code content for embedding
+ * Removes excessive whitespace while preserving structure
+ */
+const cleanContent = (content) => {
+    return content
+        // Normalize line endings
+        .replace(/\r\n/g, '\n')
+        // Remove excessive blank lines (more than 2)
+        .replace(/\n{3,}/g, '\n\n')
+        // Trim each line
+        .split('\n')
+        .map(line => line.trimEnd())
+        .join('\n')
+        .trim();
+};
+/**
+ * Generate embedding text for a Function node
+ */
+const generateFunctionText = (node, maxSnippetLength) => {
+    const parts = [
+        `Function: ${node.name}`,
+        `File: ${getFileName(node.filePath)}`,
+    ];
+    const dir = getDirectory(node.filePath);
+    if (dir) {
+        parts.push(`Directory: ${dir}`);
+    }
+    if (node.content) {
+        const cleanedContent = cleanContent(node.content);
+        const snippet = truncateContent(cleanedContent, maxSnippetLength);
+        parts.push('', snippet);
+    }
+    return parts.join('\n');
+};
+/**
+ * Generate embedding text for a Class node
+ */
+const generateClassText = (node, maxSnippetLength) => {
+    const parts = [
+        `Class: ${node.name}`,
+        `File: ${getFileName(node.filePath)}`,
+    ];
+    const dir = getDirectory(node.filePath);
+    if (dir) {
+        parts.push(`Directory: ${dir}`);
+    }
+    if (node.content) {
+        const cleanedContent = cleanContent(node.content);
+        const snippet = truncateContent(cleanedContent, maxSnippetLength);
+        parts.push('', snippet);
+    }
+    return parts.join('\n');
+};
+/**
+ * Generate embedding text for a Method node
+ */
+const generateMethodText = (node, maxSnippetLength) => {
+    const parts = [
+        `Method: ${node.name}`,
+        `File: ${getFileName(node.filePath)}`,
+    ];
+    const dir = getDirectory(node.filePath);
+    if (dir) {
+        parts.push(`Directory: ${dir}`);
+    }
+    if (node.content) {
+        const cleanedContent = cleanContent(node.content);
+        const snippet = truncateContent(cleanedContent, maxSnippetLength);
+        parts.push('', snippet);
+    }
+    return parts.join('\n');
+};
+/**
+ * Generate embedding text for an Interface node
+ */
+const generateInterfaceText = (node, maxSnippetLength) => {
+    const parts = [
+        `Interface: ${node.name}`,
+        `File: ${getFileName(node.filePath)}`,
+    ];
+    const dir = getDirectory(node.filePath);
+    if (dir) {
+        parts.push(`Directory: ${dir}`);
+    }
+    if (node.content) {
+        const cleanedContent = cleanContent(node.content);
+        const snippet = truncateContent(cleanedContent, maxSnippetLength);
+        parts.push('', snippet);
+    }
+    return parts.join('\n');
+};
+/**
+ * Generate embedding text for a File node
+ * Uses file name and first N characters of content
+ */
+const generateFileText = (node, maxSnippetLength) => {
+    const parts = [
+        `File: ${node.name}`,
+        `Path: ${node.filePath}`,
+    ];
+    if (node.content) {
+        const cleanedContent = cleanContent(node.content);
+        // For files, use a shorter snippet since they can be very long
+        const snippet = truncateContent(cleanedContent, Math.min(maxSnippetLength, 300));
+        parts.push('', snippet);
+    }
+    return parts.join('\n');
+};
+/**
+ * Generate embedding text for any embeddable node
+ * Dispatches to the appropriate generator based on node label
+ *
+ * @param node - The node to generate text for
+ * @param config - Optional configuration for max snippet length
+ * @returns Text suitable for embedding
+ */
+export const generateEmbeddingText = (node, config = {}) => {
+    const maxSnippetLength = config.maxSnippetLength ?? DEFAULT_EMBEDDING_CONFIG.maxSnippetLength;
+    switch (node.label) {
+        case 'Function':
+            return generateFunctionText(node, maxSnippetLength);
+        case 'Class':
+            return generateClassText(node, maxSnippetLength);
+        case 'Method':
+            return generateMethodText(node, maxSnippetLength);
+        case 'Interface':
+            return generateInterfaceText(node, maxSnippetLength);
+        case 'File':
+            return generateFileText(node, maxSnippetLength);
+        default:
+            // Fallback for any other embeddable type
+            return `${node.label}: ${node.name}\nPath: ${node.filePath}`;
+    }
+};
+/**
+ * Generate embedding texts for a batch of nodes
+ *
+ * @param nodes - Array of nodes to generate text for
+ * @param config - Optional configuration
+ * @returns Array of texts in the same order as input nodes
+ */
+export const generateBatchEmbeddingTexts = (nodes, config = {}) => {
+    return nodes.map(node => generateEmbeddingText(node, config));
+};

package/dist/core/embeddings/types.d.ts

@@ -0,0 +1,87 @@
+/**
+ * Embedding Pipeline Types
+ *
+ * Type definitions for the embedding generation and semantic search system.
+ */
+/**
+ * Node labels that should be embedded for semantic search
+ * These are code elements that benefit from semantic matching
+ */
+export declare const EMBEDDABLE_LABELS: readonly ["Function", "Class", "Method", "Interface", "File"];
+export type EmbeddableLabel = typeof EMBEDDABLE_LABELS[number];
+/**
+ * Check if a label should be embedded
+ */
+export declare const isEmbeddableLabel: (label: string) => label is EmbeddableLabel;
+/**
+ * Embedding pipeline phases
+ */
+export type EmbeddingPhase = 'idle' | 'loading-model' | 'embedding' | 'indexing' | 'ready' | 'error';
+/**
+ * Progress information for the embedding pipeline
+ */
+export interface EmbeddingProgress {
+    phase: EmbeddingPhase;
+    percent: number;
+    modelDownloadPercent?: number;
+    nodesProcessed?: number;
+    totalNodes?: number;
+    currentBatch?: number;
+    totalBatches?: number;
+    error?: string;
+}
+/**
+ * Configuration for the embedding pipeline
+ */
+export interface EmbeddingConfig {
+    /** Model identifier for transformers.js */
+    modelId: string;
+    /** Number of nodes to embed in each batch */
+    batchSize: number;
+    /** Embedding vector dimensions */
+    dimensions: number;
+    /** Device to use for inference: 'auto' tries GPU first, falls back to CPU */
+    device: 'auto' | 'webgpu' | 'cuda' | 'cpu' | 'wasm';
+    /** Maximum characters of code snippet to include */
+    maxSnippetLength: number;
+}
+/**
+ * Default embedding configuration
+ * Uses snowflake-arctic-embed-xs for browser efficiency
+ * Tries WebGPU first (fast), user can choose WASM fallback if unavailable
+ */
+export declare const DEFAULT_EMBEDDING_CONFIG: EmbeddingConfig;
+/**
+ * Result from semantic search
+ */
+export interface SemanticSearchResult {
+    nodeId: string;
+    name: string;
+    label: string;
+    filePath: string;
+    distance: number;
+    startLine?: number;
+    endLine?: number;
+}
+/**
+ * Node data for embedding (minimal structure from KuzuDB query)
+ */
+export interface EmbeddableNode {
+    id: string;
+    name: string;
+    label: string;
+    filePath: string;
+    content: string;
+    startLine?: number;
+    endLine?: number;
+}
+/**
+ * Model download progress from transformers.js
+ */
+export interface ModelProgress {
+    status: 'initiate' | 'download' | 'progress' | 'done' | 'ready';
+    file?: string;
+    progress?: number;
+    loaded?: number;
+    total?: number;
+}