code-graph-context 1.1.0 → 2.0.0

package/dist/mcp/handlers/graph-generator.handler.js

@@ -3,6 +3,7 @@
  * Handles importing parsed graph data into Neo4j with embeddings
  */
 import fs from 'fs/promises';
+import { EMBEDDING_BATCH_CONFIG } from '../../core/embeddings/embeddings.service.js';
 import { QUERIES } from '../../storage/neo4j/neo4j.service.js';
 import { DEFAULTS } from '../constants.js';
 import { debugLog } from '../utils.js';
@@ -10,13 +11,20 @@ export class GraphGeneratorHandler {
     neo4jService;
     embeddingsService;
     static EMBEDDED_LABEL = 'Embedded';
+    projectId = null;
     constructor(neo4jService, embeddingsService) {
         this.neo4jService = neo4jService;
         this.embeddingsService = embeddingsService;
     }
+    /**
+     * Set the projectId for project-scoped operations
+     */
+    setProjectId(projectId) {
+        this.projectId = projectId;
+    }
     async generateGraph(graphJsonPath, batchSize = DEFAULTS.batchSize, clearExisting = true) {
         console.log(`Generating graph from JSON file: ${graphJsonPath}`);
-        await debugLog('Starting graph generation', { graphJsonPath, batchSize, clearExisting });
+        await debugLog('Starting graph generation', { graphJsonPath, batchSize, clearExisting, projectId: this.projectId });
         try {
             const graphData = await this.loadGraphData(graphJsonPath);
             const { nodes, edges, metadata } = graphData;
@@ -25,6 +33,7 @@ export class GraphGeneratorHandler {
             if (clearExisting) {
                 await this.clearExistingData();
             }
+            await this.createProjectIndexes();
             await this.importNodes(nodes, batchSize);
             await this.importEdges(edges, batchSize);
             await this.createVectorIndexes();
@@ -47,9 +56,24 @@ export class GraphGeneratorHandler {
         return JSON.parse(fileContent);
     }
     async clearExistingData() {
-        console.log('Clearing existing graph data...');
-        await this.neo4jService.run(QUERIES.CLEAR_DATABASE);
-        await debugLog('Existing graph data cleared');
+        if (this.projectId) {
+            console.log(`Clearing existing graph data for project: ${this.projectId}...`);
+            await this.neo4jService.run(QUERIES.CLEAR_PROJECT, { projectId: this.projectId });
+            await debugLog('Existing project graph data cleared', { projectId: this.projectId });
+        }
+        else {
+            console.log('Clearing ALL existing graph data (no projectId set)...');
+            await this.neo4jService.run(QUERIES.CLEAR_DATABASE);
+            await debugLog('Existing graph data cleared');
+        }
+    }
+    async createProjectIndexes() {
+        console.log('Creating project indexes...');
+        await this.neo4jService.run(QUERIES.CREATE_PROJECT_INDEX_EMBEDDED);
+        await this.neo4jService.run(QUERIES.CREATE_PROJECT_INDEX_SOURCEFILE);
+        await this.neo4jService.run(QUERIES.CREATE_PROJECT_ID_INDEX_EMBEDDED);
+        await this.neo4jService.run(QUERIES.CREATE_PROJECT_ID_INDEX_SOURCEFILE);
+        await debugLog('Project indexes created');
     }
     async importNodes(nodes, batchSize) {
         console.log(`Importing ${nodes.length} nodes with embeddings...`);
@@ -65,18 +89,72 @@ export class GraphGeneratorHandler {
             });
         }
     }
+    /**
+     * Process a batch of nodes with batched embedding calls.
+     * Collects all texts needing embedding, makes a single batched API call,
+     * then maps embeddings back to their respective nodes.
+     */
     async processNodeBatch(nodes) {
-        return Promise.all(nodes.map(async (node) => {
-            const embedding = await this.embedNodeSourceCode(node);
-            return {
-                ...node,
-                labels: embedding ? [...node.labels, GraphGeneratorHandler.EMBEDDED_LABEL] : node.labels,
-                properties: {
-                    ...this.flattenProperties(node.properties),
-                    embedding,
-                },
-            };
-        }));
+        // Separate nodes that need embedding from those that don't
+        const nodesNeedingEmbedding = [];
+        const nodeResults = new Array(nodes.length);
+        // First pass: identify nodes needing embedding and prepare texts
+        nodes.forEach((node, index) => {
+            if (node.properties?.sourceCode && !node.skipEmbedding) {
+                // Truncate to stay under embedding model's 8192 token limit (~4 chars/token)
+                const truncatedCode = node.properties.sourceCode.slice(0, DEFAULTS.maxEmbeddingChars);
+                // Include node name and type in embedding for better search matching
+                // e.g., "ProfileService ClassDeclaration" helps "profile service" queries match
+                const metadata = `${node.properties.name ?? ''} ${node.labels?.join(' ') ?? ''}`.trim();
+                const embeddingText = metadata ? `${metadata}\n${truncatedCode}` : truncatedCode;
+                nodesNeedingEmbedding.push({
+                    node,
+                    index,
+                    text: embeddingText,
+                });
+            }
+            else {
+                // Node doesn't need embedding - prepare it immediately
+                nodeResults[index] = {
+                    ...node,
+                    labels: node.labels,
+                    properties: {
+                        ...this.flattenProperties(node.properties),
+                        embedding: null,
+                    },
+                };
+            }
+        });
+        // Batch embed all texts that need it
+        if (nodesNeedingEmbedding.length > 0) {
+            const texts = nodesNeedingEmbedding.map((n) => n.text);
+            try {
+                const embeddings = await this.embeddingsService.embedTextsInBatches(texts, EMBEDDING_BATCH_CONFIG.maxBatchSize);
+                // Map embeddings back to their nodes
+                nodesNeedingEmbedding.forEach((item, i) => {
+                    const embedding = embeddings[i];
+                    nodeResults[item.index] = {
+                        ...item.node,
+                        labels: embedding ? [...item.node.labels, GraphGeneratorHandler.EMBEDDED_LABEL] : item.node.labels,
+                        properties: {
+                            ...this.flattenProperties(item.node.properties),
+                            embedding,
+                        },
+                    };
+                });
+                await debugLog('Batch embedding completed', {
+                    totalNodes: nodes.length,
+                    nodesEmbedded: nodesNeedingEmbedding.length,
+                    batchesUsed: Math.ceil(texts.length / EMBEDDING_BATCH_CONFIG.maxBatchSize),
+                });
+            }
+            catch (error) {
+                // DON'T silently continue - propagate the error so user knows what's wrong
+                await debugLog('Embedding failed', { error: error instanceof Error ? error.message : String(error) });
+                throw error;
+            }
+        }
+        return nodeResults;
     }
     async importEdges(edges, batchSize) {
         console.log(`Importing ${edges.length} edges using APOC...`);
@@ -85,7 +163,10 @@ export class GraphGeneratorHandler {
                 ...edge,
                 properties: this.flattenProperties(edge.properties),
             }));
-            const result = await this.neo4jService.run(QUERIES.CREATE_RELATIONSHIP, { edges: batch });
+            const result = await this.neo4jService.run(QUERIES.CREATE_RELATIONSHIP, {
+                edges: batch,
+                projectId: this.projectId,
+            });
             const batchEnd = Math.min(i + batchSize, edges.length);
             console.log(`Created ${result[0].created} edges in batch ${i + 1}-${batchEnd}`);
             await debugLog('Edge batch imported', {
@@ -100,22 +181,6 @@ export class GraphGeneratorHandler {
         await this.neo4jService.run(QUERIES.CREATE_EMBEDDED_VECTOR_INDEX);
         await debugLog('Vector indexes created');
     }
-    async embedNodeSourceCode(node) {
-        if (!node.properties?.sourceCode || node.skipEmbedding) {
-            return null;
-        }
-        try {
-            const sourceCode = node.properties.sourceCode;
-            const embedding = await this.embeddingsService.embedText(sourceCode);
-            await debugLog('Node embedded', { nodeId: node.id, codeLength: sourceCode.length });
-            return embedding;
-        }
-        catch (error) {
-            console.warn(`Failed to embed node ${node.id}:`, error);
-            await debugLog('Embedding failed', { nodeId: node.id, error });
-            return null;
-        }
-    }
     flattenProperties(properties) {
         const flattened = {};
         for (const [key, value] of Object.entries(properties)) {

package/dist/mcp/handlers/incremental-parse.handler.js

@@ -0,0 +1,146 @@
+/**
+ * Incremental Parse Handler
+ * Handles incremental graph updates triggered by file watchers
+ */
+import { writeFileSync, unlinkSync } from 'fs';
+import { join } from 'path';
+import { CORE_TYPESCRIPT_SCHEMA } from '../../core/config/schema.js';
+import { EmbeddingsService } from '../../core/embeddings/embeddings.service.js';
+import { ParserFactory } from '../../core/parsers/parser-factory.js';
+import { detectChangedFiles } from '../../core/utils/file-change-detection.js';
+import { resolveProjectId, getProjectName, UPSERT_PROJECT_QUERY } from '../../core/utils/project-id.js';
+import { Neo4jService, QUERIES } from '../../storage/neo4j/neo4j.service.js';
+import { DEFAULTS, FILE_PATHS, LOG_CONFIG } from '../constants.js';
+import { debugLog } from '../utils.js';
+import { deleteSourceFileSubgraphs, loadExistingNodesForEdgeDetection, getCrossFileEdges, } from './cross-file-edge.helpers.js';
+import { GraphGeneratorHandler } from './graph-generator.handler.js';
+/**
+ * Performs incremental parsing for a project
+ * This is used by the WatchManager when files change
+ */
+export const performIncrementalParse = async (projectPath, projectId, tsconfigPath) => {
+    const neo4jService = new Neo4jService();
+    const embeddingsService = new EmbeddingsService();
+    const graphHandler = new GraphGeneratorHandler(neo4jService, embeddingsService);
+    try {
+        await debugLog('Incremental parse started (watch)', { projectPath, projectId });
+        // Resolve project ID
+        const resolvedId = resolveProjectId(projectPath, projectId);
+        const projectName = await getProjectName(projectPath);
+        // Create parser with auto-detection and lazy loading enabled for memory efficiency
+        const parser = await ParserFactory.createParserWithAutoDetection(projectPath, tsconfigPath, resolvedId, true);
+        // Detect changed files
+        const { filesToReparse, filesToDelete } = await detectChangedFiles(projectPath, neo4jService, resolvedId);
+        await debugLog('Watch incremental change detection', {
+            filesToReparse: filesToReparse.length,
+            filesToDelete: filesToDelete.length,
+        });
+        // If no changes, return early
+        if (filesToReparse.length === 0 && filesToDelete.length === 0) {
+            await debugLog('Watch incremental: no changes detected');
+            return {
+                nodesUpdated: 0,
+                edgesUpdated: 0,
+                filesReparsed: 0,
+                filesDeleted: filesToDelete.length,
+            };
+        }
+        let savedCrossFileEdges = [];
+        const filesToRemoveFromGraph = [...filesToDelete, ...filesToReparse];
+        if (filesToRemoveFromGraph.length > 0) {
+            // Save cross-file edges before deletion
+            savedCrossFileEdges = await getCrossFileEdges(neo4jService, filesToRemoveFromGraph, resolvedId);
+            await debugLog('Watch: saved cross-file edges', { count: savedCrossFileEdges.length });
+            // Delete old subgraphs
+            await deleteSourceFileSubgraphs(neo4jService, filesToRemoveFromGraph, resolvedId);
+        }
+        let nodesImported = 0;
+        let edgesImported = 0;
+        if (filesToReparse.length > 0) {
+            // Load existing nodes for edge detection
+            const existingNodes = await loadExistingNodesForEdgeDetection(neo4jService, filesToRemoveFromGraph, resolvedId);
+            parser.setExistingNodes(existingNodes);
+            // Parse only changed files
+            await parser.parseWorkspace(filesToReparse);
+            // Export graph data
+            const { nodes, edges } = parser.exportToJson();
+            // Get framework schemas if available (use unknown as intermediate to access private property)
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            const parserAny = parser;
+            const frameworkSchemas = Array.isArray(parserAny.frameworkSchemas)
+                ? parserAny.frameworkSchemas.map((s) => s.name)
+                : ['Auto-detected'];
+            const graphData = {
+                nodes,
+                edges,
+                metadata: {
+                    coreSchema: CORE_TYPESCRIPT_SCHEMA.name,
+                    frameworkSchemas,
+                    projectType: 'auto',
+                    projectId: resolvedId,
+                    generated: new Date().toISOString(),
+                },
+            };
+            // Write to JSON file (required by GraphGeneratorHandler)
+            const outputPath = join(projectPath, FILE_PATHS.graphOutput);
+            writeFileSync(outputPath, JSON.stringify(graphData, null, LOG_CONFIG.jsonIndentation));
+            // Update Project node
+            await neo4jService.run(UPSERT_PROJECT_QUERY, {
+                projectId: resolvedId,
+                path: projectPath,
+                name: projectName,
+                status: 'complete',
+            });
+            // Import nodes and edges (clearExisting = false for incremental)
+            graphHandler.setProjectId(resolvedId);
+            try {
+                const result = await graphHandler.generateGraph(outputPath, DEFAULTS.batchSize, false);
+                nodesImported = result.nodesImported;
+                edgesImported = result.edgesImported;
+            }
+            finally {
+                // Clean up temporary graph.json file
+                try {
+                    unlinkSync(outputPath);
+                }
+                catch {
+                    // Ignore cleanup errors - file may not exist or be inaccessible
+                }
+            }
+            // Recreate cross-file edges
+            if (savedCrossFileEdges.length > 0) {
+                const recreateResult = await neo4jService.run(QUERIES.RECREATE_CROSS_FILE_EDGES, {
+                    projectId: resolvedId,
+                    edges: savedCrossFileEdges.map((e) => ({
+                        startNodeId: e.startNodeId,
+                        endNodeId: e.endNodeId,
+                        edgeType: e.edgeType,
+                        edgeProperties: e.edgeProperties,
+                    })),
+                });
+                // Safely extract recreatedCount with runtime validation
+                const firstResult = recreateResult[0];
+                const recreatedCount = firstResult && typeof firstResult === 'object' && 'recreatedCount' in firstResult
+                    ? Number(firstResult.recreatedCount) || 0
+                    : 0;
+                edgesImported += recreatedCount;
+                await debugLog('Watch: cross-file edges recreated', { recreatedCount });
+            }
+        }
+        await debugLog('Watch incremental parse completed', {
+            nodesImported,
+            edgesImported,
+            filesReparsed: filesToReparse.length,
+            filesDeleted: filesToDelete.length,
+        });
+        return {
+            nodesUpdated: nodesImported,
+            edgesUpdated: edgesImported,
+            filesReparsed: filesToReparse.length,
+            filesDeleted: filesToDelete.length,
+        };
+    }
+    finally {
+        await neo4jService.close();
+    }
+};

package/dist/mcp/handlers/streaming-import.handler.js

@@ -0,0 +1,210 @@
+/**
+ * Streaming Import Handler
+ * Orchestrates chunked parsing and import for large codebases
+ */
+import { randomBytes } from 'crypto';
+import { tmpdir } from 'os';
+import { join } from 'path';
+import { ProgressReporter } from '../../core/utils/progress-reporter.js';
+import { DEFAULTS } from '../constants.js';
+import { debugLog } from '../utils.js';
+/**
+ * Generate a secure temporary file path using crypto random bytes
+ * to avoid race conditions and predictable filenames
+ */
+const generateTempPath = (prefix) => {
+    const randomSuffix = randomBytes(16).toString('hex');
+    return join(tmpdir(), `${prefix}-${Date.now()}-${randomSuffix}.json`);
+};
+export class StreamingImportHandler {
+    graphGeneratorHandler;
+    progressReporter;
+    constructor(graphGeneratorHandler) {
+        this.graphGeneratorHandler = graphGeneratorHandler;
+        this.progressReporter = new ProgressReporter();
+    }
+    /**
+     * Import a project using chunked parsing to reduce memory usage.
+     * Files are parsed and imported in chunks, with progress reporting.
+     * Supports both TypeScriptParser (single project) and WorkspaceParser (monorepo).
+     */
+    async importProjectStreaming(parser, config) {
+        const startTime = Date.now();
+        if (config.onProgress) {
+            this.progressReporter.setCallback(config.onProgress);
+        }
+        // Set project ID on graph generator
+        this.graphGeneratorHandler.setProjectId(config.projectId);
+        // Phase 1: Get discovered files (already discovered by worker, this returns cached result)
+        const allFilePaths = await parser.discoverSourceFiles();
+        console.log(`📁 Found ${allFilePaths.length} files to parse`);
+        await debugLog('Streaming import started', {
+            totalFiles: allFilePaths.length,
+            chunkSize: config.chunkSize,
+        });
+        // Create chunks
+        const chunks = [];
+        for (let i = 0; i < allFilePaths.length; i += config.chunkSize) {
+            chunks.push(allFilePaths.slice(i, i + config.chunkSize));
+        }
+        console.log(`📦 Split into ${chunks.length} chunks of ~${config.chunkSize} files each`);
+        let totalNodesImported = 0;
+        let totalEdgesImported = 0;
+        // Phase 2: Parse and import chunks
+        for (let chunkIndex = 0; chunkIndex < chunks.length; chunkIndex++) {
+            const chunk = chunks[chunkIndex];
+            const filesProcessed = chunkIndex * config.chunkSize + chunk.length;
+            console.log(`\n🔄 Processing chunk ${chunkIndex + 1}/${chunks.length} (${chunk.length} files)`);
+            try {
+                // Parse the chunk (skip edge resolution for streaming)
+                const { nodes, edges } = await parser.parseChunk(chunk, true);
+                // Add parsed nodes to existing nodes for cross-chunk edge resolution
+                parser.addExistingNodesFromChunk(nodes);
+                // Import to Neo4j if we have data
+                if (nodes.length > 0 || edges.length > 0) {
+                    await debugLog('Importing chunk - generating embeddings', {
+                        chunkIndex: chunkIndex + 1,
+                        totalChunks: chunks.length,
+                        nodeCount: nodes.length,
+                    });
+                    await this.importChunkToNeo4j(nodes, edges);
+                    totalNodesImported += nodes.length;
+                    totalEdgesImported += edges.length;
+                }
+                else {
+                    console.warn(`⚠️ Chunk ${chunkIndex + 1} produced 0 nodes/edges from ${chunk.length} files`);
+                    await debugLog('Empty chunk result', {
+                        chunkIndex: chunkIndex + 1,
+                        fileCount: chunk.length,
+                        sampleFiles: chunk.slice(0, 3),
+                    });
+                }
+                // Report progress with all relevant data
+                await this.progressReporter.report({
+                    phase: 'importing',
+                    current: filesProcessed,
+                    total: allFilePaths.length,
+                    message: `Processed chunk ${chunkIndex + 1}/${chunks.length}: ${totalNodesImported} nodes, ${totalEdgesImported} edges`,
+                    details: {
+                        filesProcessed,
+                        nodesCreated: totalNodesImported,
+                        edgesCreated: totalEdgesImported,
+                        currentFile: chunk[0],
+                        chunkIndex: chunkIndex + 1,
+                        totalChunks: chunks.length,
+                    },
+                });
+                console.log(`✅ Chunk ${chunkIndex + 1}: ${nodes.length} nodes, ${edges.length} edges imported`);
+            }
+            catch (chunkError) {
+                console.error(`❌ Error processing chunk ${chunkIndex + 1}:`, chunkError);
+                await debugLog('Chunk processing error', {
+                    chunkIndex: chunkIndex + 1,
+                    fileCount: chunk.length,
+                    sampleFiles: chunk.slice(0, 3),
+                    error: chunkError instanceof Error ? chunkError.message : String(chunkError),
+                    stack: chunkError instanceof Error ? chunkError.stack : undefined,
+                });
+                // Re-throw to fail the entire import - don't silently continue
+                throw chunkError;
+            }
+            // Note: Don't clear parsed data during streaming - we need accumulated nodes for cross-chunk edge resolution
+            // Memory usage is bounded because we only keep Neo4jNode references (not full AST)
+        }
+        // Phase 3: Resolve cross-chunk deferred edges
+        await this.progressReporter.reportResolving(0, totalEdgesImported);
+        console.log('\n🔗 Resolving cross-chunk edges...');
+        const resolvedEdges = await parser.resolveDeferredEdgesManually();
+        if (resolvedEdges.length > 0) {
+            await this.importEdgesToNeo4j(resolvedEdges);
+            totalEdgesImported += resolvedEdges.length;
+            console.log(`✅ Resolved ${resolvedEdges.length} cross-chunk edges`);
+        }
+        else {
+            console.log('ℹ️ No cross-chunk edges to resolve');
+        }
+        // Phase 3b: Apply edge enhancements on all accumulated nodes
+        // This catches context-dependent edges (like INTERNAL_API_CALL) that span chunks
+        console.log('\n🔗 Applying edge enhancements on all nodes...');
+        const enhancedEdges = await parser.applyEdgeEnhancementsManually();
+        if (enhancedEdges.length > 0) {
+            await this.importEdgesToNeo4j(enhancedEdges);
+            totalEdgesImported += enhancedEdges.length;
+            console.log(`✅ Created ${enhancedEdges.length} edges from edge enhancements`);
+        }
+        else {
+            console.log('ℹ️ No edges from edge enhancements');
+        }
+        // Clear accumulated data now that edge resolution is complete
+        parser.clearParsedData();
+        await this.progressReporter.reportResolving(resolvedEdges.length, resolvedEdges.length);
+        // Phase 4: Complete
+        const elapsedMs = Date.now() - startTime;
+        await this.progressReporter.reportComplete(totalNodesImported, totalEdgesImported);
+        const result = {
+            nodesImported: totalNodesImported,
+            edgesImported: totalEdgesImported,
+            filesProcessed: allFilePaths.length,
+            chunksProcessed: chunks.length,
+            elapsedMs,
+        };
+        console.log(`\n🎉 Streaming import complete!`);
+        console.log(`   Files: ${allFilePaths.length}`);
+        console.log(`   Nodes: ${totalNodesImported}`);
+        console.log(`   Edges: ${totalEdgesImported}`);
+        console.log(`   Time: ${(elapsedMs / 1000).toFixed(2)}s`);
+        await debugLog('Streaming import completed', result);
+        return result;
+    }
+    /**
+     * Import a chunk of nodes and edges to Neo4j using the graph generator handler
+     */
+    async importChunkToNeo4j(nodes, edges) {
+        // Write to temporary JSON and use existing import mechanism
+        // This reuses the batched embedding and import logic
+        const tempPath = generateTempPath('chunk');
+        const fs = await import('fs/promises');
+        try {
+            await fs.writeFile(tempPath, JSON.stringify({
+                nodes,
+                edges,
+                metadata: { chunked: true },
+            }));
+            await this.graphGeneratorHandler.generateGraph(tempPath, DEFAULTS.batchSize, false);
+        }
+        finally {
+            // Clean up temp file
+            try {
+                await fs.unlink(tempPath);
+            }
+            catch {
+                // Ignore cleanup errors
+            }
+        }
+    }
+    /**
+     * Import resolved edges to Neo4j
+     */
+    async importEdgesToNeo4j(edges) {
+        if (edges.length === 0)
+            return;
+        const tempPath = generateTempPath('edges');
+        const fs = await import('fs/promises');
+        try {
+            await fs.writeFile(tempPath, JSON.stringify({
+                nodes: [],
+                edges,
+                metadata: { edgesOnly: true },
+            }));
+            await this.graphGeneratorHandler.generateGraph(tempPath, DEFAULTS.batchSize, false);
+        }
+        finally {
+            try {
+                await fs.unlink(tempPath);
+            }
+            catch {
+                // Ignore cleanup errors
+            }
+        }
+    }
+}