homegraph 1.5.1 → 1.5.3

package/dist/extraction/index.js

@@ -65,6 +65,7 @@ const errors_1 = require("../errors");
 const utils_1 = require("../utils");
 const ignore_1 = __importDefault(require("ignore"));
 const default_ignore_1 = require("./default-ignore");
+const generated_detection_1 = require("./generated-detection");
 var default_ignore_2 = require("./default-ignore");
 Object.defineProperty(exports, "buildDefaultIgnore", { enumerable: true, get: function () { return default_ignore_2.buildDefaultIgnore; } });
 const frameworks_1 = require("../resolution/frameworks");
@@ -1314,57 +1315,25 @@ class ExtractionOrchestrator {
                     durationMs: Date.now() - startTime,
                 };
             }
-            // Phase 2: Parse files in a worker thread (keeps main thread unblocked for UI)
+            // Do NOT emit `parsing` before the ArkTS Scene batch — that makes the
+            // shimmer print "Parsing code - done" when Scene starts, which looks like
+            // a restart. Parsing progress starts after primeArkTSBatch below.
             const total = files.length;
             let processed = 0;
-            // Emit parsing phase immediately so the progress bar appears during worker setup.
-            // The yield lets the shimmer worker flush the phase transition to stdout before
-            // the main thread starts synchronous grammar detection work.
-            onProgress?.({
-                phase: 'parsing',
-                current: 0,
-                total,
-            });
-            await new Promise(resolve => setImmediate(resolve));
-            // Detect needed languages and load grammars in the parse worker
+            // Detect needed languages. Parse workers start AFTER the ArkTS Scene batch
+            // so peak RSS is not Scene IR + N×WASM grammars at once.
             const neededLanguages = [...new Set(files.map((f) => (0, grammars_1.detectLanguage)(f, undefined, overrides)))];
             // .h files default to 'c' but may be C++ — ensure cpp grammar is loaded when c is needed
             if (neededLanguages.includes('c') && !neededLanguages.includes('cpp')) {
                 neededLanguages.push('cpp');
             }
-            // Parse files on a pool of worker threads (keeps the main thread free for UI
-            // and uses every core). Falls back to in-process parsing when the compiled
-            // worker is unavailable (e.g. running from source in tests).
             const parseWorkerPath = path.join(__dirname, 'parse-worker.js');
             const useWorker = fs.existsSync(parseWorkerPath);
+            // Assigned after ArkTS batch (see below).
             let pool = null;
-            if (useWorker) {
-                // CODEGRAPH_PARSE_WORKERS: explicit worker count; 1 = the old single-worker
-                // behaviour (the conservative rollback). Unset → clamp(cores-1, 1, 8).
-                const poolSize = (0, parse_pool_1.resolveParsePoolSize)(process.env.CODEGRAPH_PARSE_WORKERS, os.cpus().length);
-                pool = new parse_pool_1.ParseWorkerPool({
-                    languages: neededLanguages,
-                    size: poolSize,
-                    workerScriptPath: parseWorkerPath,
-                    recycleInterval: WORKER_RECYCLE_INTERVAL,
-                    parseTimeoutMs: PARSE_TIMEOUT_MS,
-                    log,
-                });
-                log(`Parse worker pool: ${poolSize} worker(s)`);
-            }
-            else {
-                // In-process fallback: load grammars locally and parse on the main thread.
-                await (0, grammars_1.loadGrammarsForLanguages)(neededLanguages);
-            }
-            const storeWorkerPath = path.join(__dirname, 'store-worker.js');
             let storeWriter = null;
-            if (storeWriterOpts &&
-                process.env.HOMEGRAPH_NO_STORE_WORKER !== '1' &&
-                fs.existsSync(storeWorkerPath)) {
-                storeWriter = new store_writer_1.StoreWriter(storeWorkerPath, storeWriterOpts.dbPath, storeWriterOpts.fastInit);
-                log('Store writer thread active');
-            }
             const STORE_WRITER_WINDOW = 64;
+            let windowSize = 1;
             /**
              * Parse one file: on the pool when available (the promise REJECTS on a worker
              * crash/timeout — the caller records it and the retry pass re-attempts), or
@@ -1387,8 +1356,7 @@ class ExtractionOrchestrator {
             // stable commit order keeps the resulting graph deterministic — byte-identical
             // to the single-worker path — instead of drifting with parse timing. The
             // `completed` buffer holds at most ~windowSize out-of-order results, so memory
-            // stays bounded.
-            const windowSize = pool ? Math.max(4, pool.size * 2) : 1;
+            // stays bounded. windowSize is set when the parse pool starts (after ArkTS).
             const inFlight = new Set();
             const completed = new Map();
             let nextSeq = 0; // file-order sequence assigned at dispatch
@@ -1507,6 +1475,46 @@ class ExtractionOrchestrator {
                 await (0, arkts_1.primeArkTSBatch)(this.rootDir, this.queries, firstEtsFile);
                 errors.push(...(0, arkts_1.drainArkTSIndexNotices)());
             }
+            // Start parse pool only after Scene teardown — most .ets are already in DB.
+            const remainingToParse = files.filter((f) => !((0, arkts_1.isArkTSBatchPersisted)(f) || ((0, arkts_1.isArkTSBatchCommitted)() && (0, arkts_1.isArkAnalyzerSourcePath)(f)))).length;
+            if (useWorker) {
+                // CODEGRAPH_PARSE_WORKERS: explicit worker count; 1 = the old single-worker
+                // behaviour (the conservative rollback). Unset → clamp(cores-1, 1, 8).
+                let poolSize = (0, parse_pool_1.resolveParsePoolSize)(process.env.CODEGRAPH_PARSE_WORKERS, os.cpus().length);
+                if (!process.env.CODEGRAPH_PARSE_WORKERS?.trim() && remainingToParse < 500) {
+                    poolSize = Math.min(poolSize, 2);
+                }
+                else if (!process.env.CODEGRAPH_PARSE_WORKERS?.trim() && remainingToParse < 2000) {
+                    poolSize = Math.min(poolSize, 4);
+                }
+                pool = new parse_pool_1.ParseWorkerPool({
+                    languages: neededLanguages,
+                    size: poolSize,
+                    workerScriptPath: parseWorkerPath,
+                    recycleInterval: WORKER_RECYCLE_INTERVAL,
+                    parseTimeoutMs: PARSE_TIMEOUT_MS,
+                    log,
+                });
+                windowSize = Math.max(4, pool.size * 2);
+                log(`Parse worker pool: ${poolSize} worker(s) (remaining files≈${remainingToParse})`);
+            }
+            else {
+                await (0, grammars_1.loadGrammarsForLanguages)(neededLanguages);
+            }
+            const storeWorkerPath = path.join(__dirname, 'store-worker.js');
+            if (storeWriterOpts &&
+                process.env.HOMEGRAPH_NO_STORE_WORKER !== '1' &&
+                fs.existsSync(storeWorkerPath)) {
+                storeWriter = new store_writer_1.StoreWriter(storeWorkerPath, storeWriterOpts.dbPath, storeWriterOpts.fastInit);
+                log('Store writer thread active');
+            }
+            // Parsing progress only after ArkTS batch (avoids false "Parsing - done").
+            onProgress?.({
+                phase: 'parsing',
+                current: 0,
+                total,
+            });
+            await new Promise((resolve) => setImmediate(resolve));
             for (let i = 0; i < files.length; i += FILE_IO_BATCH_SIZE) {
                 if (signal?.aborted) {
                     aborted = true;
@@ -1515,7 +1523,8 @@ class ExtractionOrchestrator {
                 const batch = files.slice(i, i + FILE_IO_BATCH_SIZE);
                 // Read files in parallel (with path validation before any I/O)
                 const fileContents = await Promise.all(batch.map(async (fp) => {
-                    if ((0, arkts_1.isArkTSBatchPersisted)(fp)) {
+                    if ((0, arkts_1.isArkTSBatchPersisted)(fp) ||
+                        ((0, arkts_1.isArkTSBatchCommitted)() && (0, arkts_1.isArkAnalyzerSourcePath)(fp))) {
                         return {
                             filePath: fp,
                             content: null,
@@ -1550,7 +1559,9 @@ class ExtractionOrchestrator {
                         aborted = true;
                         break;
                     }
-                    if ((0, arkts_1.isArkTSBatchPersisted)(filePath) || arktsBatchSkipped) {
+                    if ((0, arkts_1.isArkTSBatchPersisted)(filePath) ||
+                        arktsBatchSkipped ||
+                        ((0, arkts_1.isArkTSBatchCommitted)() && (0, arkts_1.isArkAnalyzerSourcePath)(filePath))) {
                         processed++;
                         filesIndexed++;
                         onProgress?.({ phase: 'parsing', current: processed, total, currentFile: filePath });
@@ -1748,6 +1759,7 @@ class ExtractionOrchestrator {
             // Shut down the parse worker pool.
             if (pool)
                 await pool.destroy();
+            (0, arkts_1.shrinkArkTSBatchPostParse)();
             return {
                 success: filesIndexed > 0 || errors.filter((e) => e.severity === 'error').length === 0,
                 filesIndexed,
@@ -1957,6 +1969,7 @@ class ExtractionOrchestrator {
                     indexedAt: Date.now(),
                     nodeCount: result.nodes.length,
                     errors: result.errors.length > 0 ? result.errors : undefined,
+                    generated: (0, generated_detection_1.detectGeneratedFile)(filePath, content),
                 },
             });
             if (crossFileIncomingEdges.length > 0) {
@@ -1990,6 +2003,7 @@ class ExtractionOrchestrator {
             indexedAt: Date.now(),
             nodeCount: result.nodes.length,
             errors: result.errors.length > 0 ? result.errors : undefined,
+            generated: (0, generated_detection_1.detectGeneratedFile)(filePath, content),
         };
         this.queries.upsertFile(fileRecord);
     }
@@ -2003,6 +2017,7 @@ class ExtractionOrchestrator {
             indexedAt: Date.now(),
             nodeCount,
             errors: resultErrors.length > 0 ? resultErrors : undefined,
+            generated: (0, generated_detection_1.detectGeneratedFile)(filePath, content),
         };
     }
     buildFreshStoreBundle(filePath, content, language, stats, result) {
@@ -2033,6 +2048,60 @@ class ExtractionOrchestrator {
             this.queries.insertUnresolvedRefsBatch(resurrected);
         }
     }
+    /**
+     * Re-open resolution edges whose target name's definition set this sync
+     * changed, when those edges live in files the sync did NOT re-extract.
+     *
+     * Resolution binds a reference to one of the same-named definitions
+     * PROJECT-WIDE. Adding or removing a definition therefore changes the answer
+     * for references in untouched files — but those references resolved once and
+     * nothing existed to revisit them with — the index kept an answer that was
+     * correct against an older graph.
+     *
+     * This deletes each affected edge and re-inserts it as the reference that
+     * created it (the refName/refKind stamp), status='pending', for the sync's
+     * resolution sweep to bind against the post-sync graph — the same input a
+     * full rebuild resolves from, which is what makes the two converge.
+     *
+     * Deliberately conservative in three ways, because a wrong deletion is a
+     * permanent edge loss while a missed rebind is only residual drift:
+     * - an edge with no refName stamp (synthesized, or built by an engine older
+     *   than the stamp) is left ALONE rather than reconstructed from the target's
+     *   plain name, same rule as `resurrectRefFromDroppedEdge`;
+     * - edges whose source is in a file this sync already re-extracted are
+     *   skipped — their references were re-resolved from scratch moments ago;
+     * - very common names are skipped by the per-name ceiling in
+     *   `getResolutionEdgesByTargetName`.
+     *
+     * Returns the number of references resurrected.
+     */
+    resurrectStaleResolutionEdges(definitionDelta, changedFilePaths) {
+        if (definitionDelta.length === 0)
+            return 0;
+        const alreadyFresh = new Set(changedFilePaths);
+        const candidates = this.queries.getResolutionEdgesByTargetName(definitionDelta);
+        const edgeIds = [];
+        const refs = [];
+        for (const e of candidates) {
+            if (alreadyFresh.has(e.sourceFilePath))
+                continue;
+            const ref = resurrectRefFromDroppedEdge(e);
+            if (!ref)
+                continue; // no stamp — never delete what we cannot restore
+            edgeIds.push(e.edgeId);
+            refs.push(ref);
+        }
+        if (refs.length === 0)
+            return 0;
+        // Delete first. The sweep re-inserts whichever edge resolution now picks,
+        // and `insertEdges` is INSERT OR IGNORE against idx_edges_identity — so a
+        // rebind to the same target is a clean no-op, but leaving the old row in
+        // place for a rebind ELSEWHERE would keep both, turning drift into
+        // duplication.
+        this.queries.deleteEdgesByIds(edgeIds);
+        this.queries.insertUnresolvedRefsBatch(refs);
+        return refs.length;
+    }
     /**
      * Sync the index with the current file state.
      *
@@ -2050,6 +2119,10 @@ class ExtractionOrchestrator {
         let filesRemoved = 0;
         let nodesUpdated = 0;
         const changedFilePaths = [];
+        // `file\0name` definition pairs for the files this sync touches, sampled
+        // BEFORE their nodes are replaced/deleted. Compared against the post-store
+        // pairs below to derive `definitionDelta`.
+        const pairsBefore = new Set();
         onProgress?.({
             phase: 'scanning',
             current: 0,
@@ -2086,6 +2159,10 @@ class ExtractionOrchestrator {
                 if (tracked.path.startsWith('@')) {
                     continue;
                 }
+                // Every name this file defined is about to stop existing here, which
+                // narrows the candidate set for that name repo-wide.
+                for (const pair of this.queries.getNodeNamePairsByFiles([tracked.path]))
+                    pairsBefore.add(pair);
                 const incoming = this.queries.getCrossFileIncomingEdgesWithTarget(tracked.path);
                 if (incoming.length > 0) {
                     const resurrected = incoming
@@ -2155,6 +2232,14 @@ class ExtractionOrchestrator {
                 filesModified++;
             }
         }
+        // Sampled here — after the add/modify classification, before any file is
+        // re-extracted — because `storeExtractionResult` deletes a file's nodes
+        // before inserting the new ones, so this is the last point the pre-edit
+        // definition set is readable.
+        if (filesToIndex.length > 0) {
+            for (const pair of this.queries.getNodeNamePairsByFiles(filesToIndex))
+                pairsBefore.add(pair);
+        }
         // Load only grammars needed for changed files
         if (filesToIndex.length > 0) {
             const overrides = (0, project_config_1.loadExtensionOverrides)(this.rootDir);
@@ -2225,6 +2310,27 @@ class ExtractionOrchestrator {
                 (0, context_1.setArkTSBatchProgressCallback)(null);
             }
         }
+        // Names whose definition set this sync changed: a `file\0name` pair present
+        // before but not after (removed/renamed away) or after but not before
+        // (added). A pair on both sides is untouched as far as resolution's
+        // candidate set is concerned — only its node id moved, which
+        // reattachCrossFileEdges already follows — so an edit that only changes
+        // bodies yields an empty delta and no downstream rebind work.
+        //
+        // Compared per FILE, not as one name set over the whole batch: a commit
+        // that adds `collect` to a new file while an unrelated changed file already
+        // defined `collect` must still flag the name, and a bare name set cancels
+        // exactly that case out.
+        const pairsAfter = this.queries.getNodeNamePairsByFiles(filesToIndex);
+        const deltaNames = new Set();
+        const nameOf = (pair) => pair.slice(pair.indexOf('\0') + 1);
+        for (const pair of pairsBefore)
+            if (!pairsAfter.has(pair))
+                deltaNames.add(nameOf(pair));
+        for (const pair of pairsAfter)
+            if (!pairsBefore.has(pair))
+                deltaNames.add(nameOf(pair));
+        const definitionDelta = [...deltaNames];
         return {
             filesChecked,
             filesAdded,
@@ -2233,6 +2339,7 @@ class ExtractionOrchestrator {
             nodesUpdated,
             durationMs: Date.now() - startTime,
             changedFilePaths: changedFilePaths.length > 0 ? changedFilePaths : undefined,
+            definitionDelta: definitionDelta.length > 0 ? definitionDelta : undefined,
         };
     }
     /**