gitnexus 1.6.11-rc.2 → 1.6.11-rc.21

package/dist/core/ingestion/pipeline-phases/parse-impl.js

@@ -2,7 +2,7 @@
  * Parse implementation — chunked parse + resolve loop.
  *
  * This is the core parsing engine of the ingestion pipeline. It reads
- * source files in byte-budget chunks (~20MB each), parses via the worker
+ * source files in stable hash-bucket packs (~2MB each by default), parses via the worker
  * pool (the sole parse path — there is no sequential fallback), and emits
  * route CALLS edges. Import,
  * call, and inheritance resolution are owned by the scope-resolution
@@ -16,8 +16,8 @@
  */
 import { BindingAccumulator, enrichExportedTypeMap, } from '../binding-accumulator.js';
 import { mergeChunkResults, dispatchChunkParse } from '../parsing-processor.js';
-import { fileContentHash, computeChunkHash, loadParseCacheChunk, persistParseCacheChunk, PARSE_CACHE_VERSION, } from '../../../storage/parse-cache.js';
-import { clearParsedFileStore, persistParsedFileChunk, loadParsedFilesForPaths, getDurableParsedFileDir, loadDurableParsedFileIndex, prepareDurableParsedFileChunk, restoreDurableParsedFileShard, } from '../../../storage/parsedfile-store.js';
+import { fileContentHash, computeChunkHash, loadParseCacheChunk, persistParseCacheChunk, PARSE_CACHE_VERSION, packParseCacheChunks, } from '../../../storage/parse-cache.js';
+import { clearParsedFileStore, persistParsedFileChunk, loadParsedFilesForPaths, getDurableParsedFileDir, loadDurableParsedFileIndex, prepareDurableParsedFileChunk, durableChunkHasShards, } from '../../../storage/parsedfile-store.js';
 import { DEFAULT_PDG_MAX_FUNCTION_LINES } from '../cfg/collect.js';
 import { processRoutesFromExtracted, resolveRouteHandlerSymbols, buildExportedTypeMapFromGraph, } from '../call-processor.js';
 import { createSemanticModel } from '../model/index.js';
@@ -111,38 +111,18 @@ export function heapPressureRemedy(heapLimitBytes) {
     return (`This machine is at its memory ceiling: exclude generated or vendored directories ` +
         `via .gitnexusignore, or analyze on a machine with more memory.`);
 }
-/** Max bytes of source content to load per parse chunk.
+/** Max bytes of source content to load per parse cache pack.
  *
- * Memory bound for the worker pool dispatch + a granularity knob for
- * the parse cache. A single file change invalidates only its enclosing
- * chunk, so smaller budgets → finer-grained invalidation.
- *
- * Override via GITNEXUS_CHUNK_BYTE_BUDGET (bytes) — the default of 2MB
- * gives a useful invalidation floor (~1/N chunks on a multi-MB repo)
- * while keeping worker dispatch overhead under 5% on cold runs.
- */
-/**
- * Built-in chunk byte budget when neither `PipelineOptions.chunkByteBudget`
- * nor `GITNEXUS_CHUNK_BYTE_BUDGET` is set. Tuned to give a useful
- * cache-invalidation floor (~1/N chunks on a multi-MB repo) while keeping
- * worker dispatch overhead under 5% on cold runs. Resolution happens at
- * call time inside `runChunkedParseAndResolve` (U14 from PR #1693 review)
- * — previously this was a module-load IIFE, which froze the env value at
- * import time and meant per-call option threading silently no-op'd.
+ * Granularity knob for the parse cache: a single file change invalidates only
+ * its enclosing pack. Override via GITNEXUS_CHUNK_BYTE_BUDGET. Resolution
+ * happens at call time (U14 from PR #1693) — not at module load.
  */
 const DEFAULT_CHUNK_BYTE_BUDGET = 2 * 1024 * 1024;
 /**
- * Per-worker share of a chunk's byte budget when auto-scaling (#worker-idle).
- *
- * A chunk is a single `WorkerPool.dispatch` unit; the pool fans a chunk's files
- * into sub-batch jobs and assigns them to idle workers (`wakeIdleSlots`). When
- * the chunk budget (2 MB) was far below the 8 MB sub-batch cap, every chunk
- * produced exactly ONE job → ONE busy worker while the other N-1 sat idle. To
- * keep all workers fed, the auto chunk budget now scales as
- * `poolSize × CHUNK_BYTES_PER_WORKER`, so each dispatch carries enough work to
- * fan across the whole pool. Sequential / explicit-budget runs are unaffected.
+ * Byte unit for auto pool sizing (one worker per this much source). Same
+ * magnitude as the default cache pack, but not a membership input (#3088).
  */
-const CHUNK_BYTES_PER_WORKER = 2 * 1024 * 1024;
+const CHUNK_BYTES_PER_WORKER = DEFAULT_CHUNK_BYTE_BUDGET;
 /**
  * Target jobs-per-worker per dispatch. More jobs than workers gives the pool's
  * idle-slot assignment room to load-balance (a slow job doesn't strand a worker
@@ -151,16 +131,14 @@ const CHUNK_BYTES_PER_WORKER = 2 * 1024 * 1024;
 const TARGET_JOBS_PER_WORKER = 3;
 /** Floor for a derived sub-batch so jobs don't shrink to per-file IPC churn. */
 const MIN_SUB_BATCH_BYTES = 256 * 1024;
-function resolveChunkByteBudget(options, effectivePoolSize = 1) {
+function resolveChunkByteBudget(options) {
     const opt = options?.chunkByteBudget;
     if (typeof opt === 'number' && Number.isFinite(opt) && opt > 0)
         return opt;
     const env = Number(process.env.GITNEXUS_CHUNK_BYTE_BUDGET);
     if (Number.isFinite(env) && env > 0)
         return env;
-    // Auto: size each chunk so a dispatch can fan across the whole pool. A
-    // single-worker (tiny-repo) run keeps the original 2 MB invalidation floor.
-    return Math.max(DEFAULT_CHUNK_BYTE_BUDGET, effectivePoolSize * CHUNK_BYTES_PER_WORKER);
+    return DEFAULT_CHUNK_BYTE_BUDGET;
 }
 /**
  * Whole-repo, cross-file route extraction (main thread).
@@ -367,14 +345,6 @@ export async function runChunkedParseAndResolve(graph, scannedFiles, allPaths, t
         }
     }
     const unavailableScopeLanguageFiles = [...skippedByLang.values()].reduce((total, count) => total + count, 0);
-    // Sort parseableScanned alphabetically for stable chunk membership
-    // across runs (Finding 4). Without this, filesystem-scan order can
-    // shift between runs (notably on macOS APFS where directory entry
-    // order can change after modifications) — different files in the
-    // same chunk → different chunk hash → cache miss even when no file
-    // content changed. The cache also becomes platform-specific: a
-    // Linux-built cache misses on macOS for the same repo.
-    parseableScanned.sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0));
     const totalParseable = parseableScanned.length;
     const totalBytes = parseableScanned.reduce((sum, f) => sum + f.size, 0);
     if (totalParseable === 0) {
@@ -416,24 +386,24 @@ export async function runChunkedParseAndResolve(graph, scannedFiles, allPaths, t
     // runs. Resolving in the function body restores per-call configurability
     // and matches the pattern used by resolveAutoPoolSize and the U1
     // parseChunkConcurrency resolver.
-    // Effective worker count, computed up-front so the chunk budget can scale to
-    // keep the whole pool busy (#worker-idle). The pool is ALWAYS used (sequential
-    // parsing was removed; the disabled channels threw above). Size it to the
-    // work: an explicit `--workers <N>` pins the size; otherwise the cores-based
-    // auto size is capped by the repo's worth of work (~one worker per
-    // CHUNK_BYTES_PER_WORKER of source) so a tiny repo spawns ~1 worker instead of
-    // a full pool, replacing the job the deleted small-repo threshold used to do.
-    // KTD-3 of the remove-sequential plan; the cap formula is intentionally coarse
-    // (tuning deferred).
+    // Effective worker count: explicit `--workers <N>` pins it; otherwise
+    // cores-based auto size is capped by source bytes / CHUNK_BYTES_PER_WORKER
+    // so a tiny repo does not spawn a full idle pool. Cache pack membership
+    // is independent of this number (#3088).
     const explicitPoolSize = options?.workerPoolSize;
     const workProportionalCap = Math.max(1, Math.ceil(totalBytes / CHUNK_BYTES_PER_WORKER));
     const effectivePoolSize = explicitPoolSize && explicitPoolSize > 0
         ? explicitPoolSize
         : Math.min(resolveAutoPoolSize(), workProportionalCap);
-    const chunkByteBudget = resolveChunkByteBudget(options, effectivePoolSize);
-    // Sub-batch size so each chunk fans into ~`TARGET_JOBS_PER_WORKER` jobs per
-    // worker, giving the pool's idle-slot assignment room to load-balance. An
-    // explicit `GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES` operator override wins.
+    // Cache packs: stable (language, hash(path) mod 128) buckets, then the
+    // per-call byte budget inside each bucket (#3088). Pool size is used only
+    // for worker count and sub-batch fan-out, not membership.
+    const chunkByteBudget = resolveChunkByteBudget(options);
+    // Sub-batch size so a 2 MiB pack fans into ~TARGET_JOBS_PER_WORKER jobs
+    // per worker, floored at MIN_SUB_BATCH_BYTES (256 KiB) so an 8-worker
+    // pool still gets ~8 jobs from one pack instead of one idle-heavy job
+    // (#worker-idle). Do not derive this from pool×2 MiB while dispatching a
+    // 2 MiB pack. An explicit GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES wins.
     const subBatchEnv = Number(process.env.GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES);
     const dispatchSubBatchMaxBytes = Number.isFinite(subBatchEnv) && subBatchEnv > 0
         ? subBatchEnv
@@ -451,20 +421,11 @@ export async function runChunkedParseAndResolve(graph, scannedFiles, allPaths, t
         logger.warn(`Large repository: analyzing ${parseableScanned.length} files needs roughly ${Math.round(projectedHeapNeedBytes / 1024 / 1024 / 1024)}GB of memory, ` +
             `but Node is limited to ${Math.round(heapLimitBytes / 1024 / 1024 / 1024)}GB — analyze may stop early. ${heapPressureRemedy(heapLimitBytes)}`);
     }
-    const chunks = [];
-    let currentChunk = [];
-    let currentBytes = 0;
-    for (const file of parseableScanned) {
-        if (currentChunk.length > 0 && currentBytes + file.size > chunkByteBudget) {
-            chunks.push(currentChunk);
-            currentChunk = [];
-            currentBytes = 0;
-        }
-        currentChunk.push(file.path);
-        currentBytes += file.size;
-    }
-    if (currentChunk.length > 0)
-        chunks.push(currentChunk);
+    const chunks = packParseCacheChunks(parseableScanned.map((file) => ({
+        path: file.path,
+        size: file.size,
+        language: getLanguageFromFilename(file.path) ?? 'unknown',
+    })), chunkByteBudget);
     const numChunks = chunks.length;
     if (isDev) {
         const totalMB = parseableScanned.reduce((s, f) => s + f.size, 0) / (1024 * 1024);
@@ -586,15 +547,16 @@ export async function runChunkedParseAndResolve(graph, scannedFiles, allPaths, t
     // a sibling of the run-scoped store, NOT cleared per run. Workers write a
     // shard per chunk hash; on a warm parse-cache hit we restore the chunk's
     // shards into the run-scoped store so scope-resolution streams them without
-    // re-parsing. `durableHitKeys` is the prior run's index, version-gated by
-    // PARSE_CACHE_VERSION (a mismatch ⇒ empty ⇒ every chunk re-dispatches, which
-    // repopulates the durable store — never the main-thread extract fallback).
+    // re-parsing. `durableHitEntries` is the prior run's path-coverage index,
+    // version-gated by PARSE_CACHE_VERSION (a mismatch ⇒ empty ⇒ every chunk
+    // re-dispatches, which repopulates the durable store).
     const durableParsedFileDir = parsedFileStorePath !== undefined ? getDurableParsedFileDir(parsedFileStorePath) : undefined;
-    const durableHitKeys = durableParsedFileDir !== undefined
+    const durableHitEntries = durableParsedFileDir !== undefined
         ? await loadDurableParsedFileIndex(durableParsedFileDir, PARSE_CACHE_VERSION)
-        : new Set();
+        : new Map();
     let chunkCacheHits = 0;
     let chunkCacheMisses = 0;
+    let reparsedFileCount = 0;
     try {
         // U1 — bounded chunk concurrency (B1 from PR #1693 review): pre-fetch
         // chunk file contents up to `parseChunkConcurrency` chunks ahead of the
@@ -645,7 +607,11 @@ export async function runChunkedParseAndResolve(graph, scannedFiles, allPaths, t
                 }
                 if (chunkWorkerData.parsedFiles?.length) {
                     if (parsedFileStorePath) {
-                        await persistParsedFileChunk(parsedFileStorePath, `chunk-${chunkIdx}`, chunkWorkerData.parsedFiles);
+                        const wrote = await persistParsedFileChunk(parsedFileStorePath, `chunk-${chunkIdx}`, chunkWorkerData.parsedFiles);
+                        if (!wrote) {
+                            for (const item of chunkWorkerData.parsedFiles)
+                                allParsedFiles.push(item);
+                        }
                     }
                     else {
                         for (const item of chunkWorkerData.parsedFiles)
@@ -832,7 +798,14 @@ export async function runChunkedParseAndResolve(graph, scannedFiles, allPaths, t
             // store was introduced, or a pruned/version-stale shard — fall through to
             // a worker re-dispatch to repopulate them. NEVER let scope-resolution
             // re-extract on the main thread (the #1983 OOM the durable store closes).
-            const durableHit = chunkHash !== null && durableParsedFileDir !== undefined && durableHitKeys.has(chunkHash);
+            const durableExpectedPaths = chunkHash === null ? undefined : durableHitEntries.get(chunkHash);
+            const durableHit = cachedRaw !== undefined &&
+                cachedRaw.length > 0 &&
+                chunkHash !== null &&
+                durableParsedFileDir !== undefined &&
+                parsedFileStorePath !== undefined &&
+                durableExpectedPaths !== undefined &&
+                (await durableChunkHasShards(parsedFileStorePath, chunkHash, durableExpectedPaths));
             if (cachedRaw && cachedRaw.length > 0 && (durableHit || parsedFileStorePath === undefined)) {
                 // Cache hit: replay cached worker output. Finalize any parked worker
                 // chunk FIRST so deferred aggregation stays in chunk order, then merge
@@ -862,22 +835,15 @@ export async function runChunkedParseAndResolve(graph, scannedFiles, allPaths, t
                         nodesCreated: graph.nodeCount,
                     },
                 });
-                // Restore the chunk's durable ParsedFile shards into the run-scoped
-                // store so scope-resolution finds full coverage with ZERO main-thread
-                // re-parse. A verbatim byte copy — byte-identical to a cold run.
-                if (durableHit && durableParsedFileDir && parsedFileStorePath && chunkHash) {
-                    const restored = await restoreDurableParsedFileShard(durableParsedFileDir, parsedFileStorePath, chunkHash);
-                    if (restored === 0) {
-                        logger.warn(`parsedfile-cache: durable shards missing for cached chunk ` +
-                            `${chunkHash.slice(0, 8)} — scope-resolution will re-extract these files`);
-                    }
-                }
+                // The durable gate already snapshotted warm `.v8` shards into the
+                // run-scoped store for scope resolution.
                 await applyChunkResults(chunkWorkerData, chunkIdx, chunkFiles, chunkStartMs);
             }
             else {
                 // Cache miss: dispatch to workers, capture the raw results, store
                 // them under the chunk hash for the next run.
                 chunkCacheMisses++;
+                reparsedFileCount += chunkFiles.length;
                 if (durableParsedFileDir !== undefined && chunkHash !== null) {
                     try {
                         await prepareDurableParsedFileChunk(durableParsedFileDir, chunkHash);
@@ -1336,6 +1302,11 @@ export async function runChunkedParseAndResolve(graph, scannedFiles, allPaths, t
         // no pool was needed: a warm all-cache-hit run replays cached worker output
         // without spawning workers, or there were no parseable files.
         usedWorkerPool: workerPool !== undefined,
+        // Exact number of files sent through workers on parse-cache misses. A
+        // changed file can invalidate its whole content-addressed chunk, so this
+        // is intentionally measured at dispatch time rather than inferred from
+        // the git/hash diff.
+        reparsedFileCount,
         // Per-file ParsedFile artifacts produced by workers' calls to
         // `extractParsedFile`. Consumed by scope-resolution as a re-extraction
         // cache: when the file's ParsedFile is here, scope-resolution skips its own

package/dist/core/ingestion/pipeline-phases/parse.d.ts

@@ -59,6 +59,8 @@ export interface ParseOutput {
      * is no sequential parser; the pool is the sole parse path on a cache miss.
      */
     readonly usedWorkerPool: boolean;
+    /** Files actually dispatched to parser workers after parse-cache lookup. */
+    readonly reparsedFileCount: number;
     /**
      * Per-file `ParsedFile` artifacts produced by workers' calls to
      * `extractParsedFile`. Threaded through to `scopeResolutionPhase`

package/dist/core/ingestion/pipeline-phases/runner.d.ts

@@ -17,6 +17,9 @@ import type { PipelinePhase, PipelineContext, PhaseResult } from './types.js';
  *
  * @param phases  All phases to execute (order doesn't matter — sorted internally)
  * @param ctx     Shared pipeline context
+ * @param seed    Results of phases that already ran against this same context,
+ *                available to `phases` as dependencies (#3016 deferred derived
+ *                phases). Included in the returned map.
  * @returns       Map of phase name → PhaseResult (all completed phases)
  */
-export declare function runPipeline(phases: readonly PipelinePhase[], ctx: PipelineContext): Promise<ReadonlyMap<string, PhaseResult<unknown>>>;
+export declare function runPipeline(phases: readonly PipelinePhase[], ctx: PipelineContext, seed?: ReadonlyMap<string, PhaseResult<unknown>>): Promise<ReadonlyMap<string, PhaseResult<unknown>>>;

package/dist/core/ingestion/pipeline-phases/runner.js

@@ -13,22 +13,30 @@
  */
 import { isDev } from '../utils/env.js';
 import { logger } from '../../logger.js';
-/**
- * Validate that the phases form a valid dependency graph (no cycles, all deps present).
- * Returns phases in topological execution order.
- */
-function topologicalSort(phases) {
-    const phaseMap = new Map();
+function assertUniquePhaseNames(phases) {
+    const seen = new Set();
     for (const phase of phases) {
-        if (phaseMap.has(phase.name)) {
+        if (seen.has(phase.name)) {
             throw new Error(`Duplicate phase name: '${phase.name}'`);
         }
-        phaseMap.set(phase.name, phase);
+        seen.add(phase.name);
     }
+}
+/**
+ * Validate that the phases form a valid dependency graph (no cycles, all deps present).
+ * Returns phases in topological execution order.
+ *
+ * `satisfied` names phases whose results are already available (a deferred
+ * follow-up run over the same context, #3016). Their edges are dropped rather
+ * than validated, because they are resolved by definition.
+ */
+function topologicalSort(phases, satisfied = new Set()) {
+    assertUniquePhaseNames(phases);
+    const phaseMap = new Map(phases.map((p) => [p.name, p]));
     // Validate all deps exist
     for (const phase of phases) {
         for (const dep of phase.deps) {
-            if (!phaseMap.has(dep)) {
+            if (!phaseMap.has(dep) && !satisfied.has(dep)) {
                 throw new Error(`Phase '${phase.name}' depends on '${dep}', which is not registered`);
             }
         }
@@ -37,8 +45,9 @@ function topologicalSort(phases) {
     const inDegree = new Map();
     const reverseDeps = new Map();
     for (const phase of phases) {
-        inDegree.set(phase.name, phase.deps.length);
-        for (const dep of phase.deps) {
+        const pendingDeps = phase.deps.filter((dep) => !satisfied.has(dep));
+        inDegree.set(phase.name, pendingDeps.length);
+        for (const dep of pendingDeps) {
             let rev = reverseDeps.get(dep);
             if (!rev) {
                 rev = [];
@@ -125,12 +134,23 @@ function findCyclePath(remaining, phaseMap) {
  *
  * @param phases  All phases to execute (order doesn't matter — sorted internally)
  * @param ctx     Shared pipeline context
+ * @param seed    Results of phases that already ran against this same context,
+ *                available to `phases` as dependencies (#3016 deferred derived
+ *                phases). Included in the returned map.
  * @returns       Map of phase name → PhaseResult (all completed phases)
  */
-export async function runPipeline(phases, ctx) {
+export async function runPipeline(phases, ctx, seed) {
+    // A seeded phase has already run against this context; re-running it would
+    // apply its graph writes a second time. "Already ran" is the whole meaning of
+    // the seed, so honour it here rather than making every caller pre-filter.
+    const satisfied = new Set(seed?.keys() ?? []);
     let sorted;
     try {
-        sorted = topologicalSort(phases);
+        // Duplicate names must be rejected on the caller-supplied list *before*
+        // seed-filtering. Filtering first would drop a seeded duplicate and let
+        // `topologicalSort` see a unique name (#3102).
+        assertUniquePhaseNames(phases);
+        sorted = topologicalSort(phases.filter((p) => !satisfied.has(p.name)), satisfied);
     }
     catch (err) {
         // Emit a terminal 'error' progress event for graph-validation failures
@@ -152,7 +172,7 @@ export async function runPipeline(phases, ctx) {
         }
         throw err;
     }
-    const results = new Map();
+    const results = new Map(seed);
     for (const phase of sorted) {
         const start = Date.now();
         if (isDev) {

package/dist/core/ingestion/pipeline-phases/scan.js

@@ -19,20 +19,32 @@ export const scanPhase = {
             percent: 0,
             message: 'Scanning repository...',
         });
-        const scannedFiles = await walkRepositoryPaths(ctx.repoPath, (current, total, filePath) => {
-            const scanProgress = Math.round((current / total) * 15);
-            ctx.onProgress({
-                phase: 'extracting',
-                percent: scanProgress,
-                message: 'Scanning repository...',
-                detail: filePath,
-                stats: {
-                    filesProcessed: current,
-                    totalFiles: total,
-                    nodesCreated: ctx.graph.nodeCount,
-                },
+        let scannedFiles;
+        try {
+            scannedFiles = await walkRepositoryPaths(ctx.repoPath, (current, total, filePath) => {
+                const scanProgress = Math.round((current / total) * 15);
+                ctx.onProgress({
+                    phase: 'extracting',
+                    percent: scanProgress,
+                    message: 'Scanning repository...',
+                    detail: filePath,
+                    stats: {
+                        filesProcessed: current,
+                        totalFiles: total,
+                        nodesCreated: ctx.graph.nodeCount,
+                    },
+                });
             });
-        });
+        }
+        catch (err) {
+            // Missing roots throw so status cannot treat an empty glob as "every
+            // covered file was deleted". The pipeline still reports an empty scan
+            // for a path that is not a directory, matching analyze of a bad cwd.
+            if (err instanceof Error && err.message.startsWith('walkRepositoryPaths:')) {
+                return { scannedFiles: [], allPaths: [], totalFiles: 0 };
+            }
+            throw err;
+        }
         const totalFiles = scannedFiles.length;
         const allPaths = scannedFiles.map((f) => f.path);
         ctx.onProgress({

package/dist/core/ingestion/pipeline.d.ts

@@ -25,6 +25,12 @@ export interface PipelineOptions {
      * to retain those nodes under `skipGraphPhases`.
      */
     skipGraphPhases?: boolean;
+    /**
+     * Skip only Leiden community detection and process/flow extraction (#3016).
+     * MRO/DI still run. Used on warm incremental analyze so persisted
+     * Community/Process rows can be kept instead of wipe+rewrite.
+     */
+    skipDerivedGraphPhases?: boolean;
     /** Per-advice Spring AOP candidate inspection cap. `0` disables this cap. */
     springAopMaxCandidateInspectionsPerAdvice?: number;
     /** Aggregate Spring AOP candidate inspection cap for one analysis. `0` disables this cap. */

package/dist/core/ingestion/pipeline.js

@@ -61,8 +61,12 @@ export function buildPhaseList(options) {
         .register(mroPhase, { enabledWhen: (o) => !o.skipGraphPhases })
         .register(springAopInheritancePhase, { enabledWhen: (o) => !o.skipGraphPhases })
         .register(diPhase, { enabledWhen: (o) => !o.skipGraphPhases })
-        .register(communitiesPhase, { enabledWhen: (o) => !o.skipGraphPhases })
-        .register(processesPhase, { enabledWhen: (o) => !o.skipGraphPhases })
+        .register(communitiesPhase, {
+        enabledWhen: (o) => !o.skipGraphPhases && o.skipDerivedGraphPhases !== true,
+    })
+        .register(processesPhase, {
+        enabledWhen: (o) => !o.skipGraphPhases && o.skipDerivedGraphPhases !== true,
+    })
         // Normalize a missing options object once here so phase predicates above
         // take a required PipelineOptions and need no `?.` guard (#2080 review S1).
         .build(options ?? {}));
@@ -91,17 +95,18 @@ export const runPipelineFromRepo = async (repoPath, onProgress, options) => {
         graphEmitSink = new GraphEmitSink(graph, options.graphEmitCsvDir);
     }
     const phases = buildPhaseList(options);
+    const ctx = {
+        repoPath,
+        graph: graphEmitSink ?? graph,
+        onProgress,
+        options,
+        pipelineStart,
+        graphEmit: graphEmitSink,
+    };
     let graphEmitManifest;
     let results;
     try {
-        results = await runPipeline(phases, {
-            repoPath,
-            graph: graphEmitSink ?? graph,
-            onProgress,
-            options,
-            pipelineStart,
-            graphEmit: graphEmitSink,
-        });
+        results = await runPipeline(phases, ctx);
         graphEmitManifest = graphEmitSink?.finalize();
     }
     finally {
@@ -109,7 +114,7 @@ export const runPipelineFromRepo = async (repoPath, onProgress, options) => {
         graphEmitSink?.close();
     }
     // Extract final results for the PipelineResult contract
-    const { totalFiles, usedWorkerPool, unavailableScopeLanguageFiles } = getPhaseOutput(results, 'parse');
+    const { totalFiles, usedWorkerPool, reparsedFileCount, unavailableScopeLanguageFiles } = getPhaseOutput(results, 'parse');
     let communityResult;
     let processResult;
     const scopeResolutionOutput = getPhaseOutput(results, 'scopeResolution');
@@ -140,7 +145,7 @@ export const runPipelineFromRepo = async (repoPath, onProgress, options) => {
             nodesCreated: graph.nodeCount,
         },
     });
-    return {
+    const result = {
         // The RAW graph, deliberately — NOT `graphEmitSink`. Phases above received
         // the sink so their reads are complete, but `loadGraphToLbug` feeds this to
         // `streamAllCSVsToDisk`, and the sink's complete iterator would then emit
@@ -156,9 +161,36 @@ export const runPipelineFromRepo = async (repoPath, onProgress, options) => {
         resolutionOutcomes,
         undecidedSatisfaction,
         usedWorkerPool,
+        reparsedFileCount,
         scopeExtractionFailures,
         unavailableScopeLanguageFiles,
         pdgEmitManifest,
         propertyInference,
     };
+    // #3016: hand back a way to run the derived phases `skipDerivedGraphPhases`
+    // held back. Which phases those are is answered by re-asking the registry
+    // with only that flag cleared — the one form of the question that stays
+    // correct when a different predicate (`skipGraphPhases`) also disables them,
+    // since then they are absent for a reason a deferred run cannot fix and the
+    // filter yields nothing. The sink guard mirrors the `graph` note above: a
+    // streaming run is a full rebuild, which never sets the skip flag, so an
+    // active sink here means the two got combined by mistake — and deferred
+    // phases writing into a finalized sink would emit past its manifest.
+    const deferredDerivedPhases = options?.skipDerivedGraphPhases === true && graphEmitSink === undefined
+        ? buildPhaseList({ ...options, skipDerivedGraphPhases: false }).filter((p) => (p.name === 'communities' || p.name === 'processes') && !results.has(p.name))
+        : [];
+    if (deferredDerivedPhases.length > 0) {
+        result.runDeferredDerivedPhases = async () => {
+            const derived = await runPipeline(deferredDerivedPhases, ctx, results);
+            // Presence-checked for the same reason as the block above: a phase the
+            // registry filtered out is absent, and `getPhaseOutput` throws on absent.
+            if (derived.has('communities')) {
+                result.communityResult = getPhaseOutput(derived, 'communities').communityResult;
+            }
+            if (derived.has('processes')) {
+                result.processResult = getPhaseOutput(derived, 'processes').processResult;
+            }
+        };
+    }
+    return result;
 };

package/dist/core/ingestion/scope-extractor.js

@@ -1450,6 +1450,7 @@ const KNOWN_SUB_TAGS = new Set([
     '@scope.lexical-names',
     '@declaration.name',
     '@declaration.qualified_name',
+    '@declaration.is-synthetic',
     '@import.name',
     '@import.source',
     '@import.alias',

package/dist/core/ingestion/utils/symbol-labels.d.ts

@@ -12,8 +12,8 @@ import type { NodeLabel } from '../../../_shared/index.js';
  * Single source of truth so the set can't silently drift the way the inline copy
  * did in #2379.
  *
- * NOTE: `group/extractors/manifest-extractor.ts`'s `CUSTOM_CONTRACT_RESOLVE_QUERY`
- * carries a near-identical hand-list that is intentionally a SUBSET — it excludes
+ * NOTE: group extractor queries in `manifest-extractor.ts` and `graphql-extractor.ts`
+ * carry near-identical hand-lists that are intentionally SUBSETS — they exclude
  * `Namespace`, `Variable`, `Module`. Unifying the two needs a contract-resolution
  * behavior check (would widen which nodes resolve as contract symbols), so it is
  * deliberately left separate for now.

package/dist/core/ingestion/utils/symbol-labels.js

@@ -11,8 +11,8 @@
  * Single source of truth so the set can't silently drift the way the inline copy
  * did in #2379.
  *
- * NOTE: `group/extractors/manifest-extractor.ts`'s `CUSTOM_CONTRACT_RESOLVE_QUERY`
- * carries a near-identical hand-list that is intentionally a SUBSET — it excludes
+ * NOTE: group extractor queries in `manifest-extractor.ts` and `graphql-extractor.ts`
+ * carry near-identical hand-lists that are intentionally SUBSETS — they exclude
  * `Namespace`, `Variable`, `Module`. Unifying the two needs a contract-resolution
  * behavior check (would widen which nodes resolve as contract symbols), so it is
  * deliberately left separate for now.

package/dist/core/ingestion/workers/parse-worker.js

@@ -1025,6 +1025,10 @@ const processFileGroup = (files, language, queryString, result, onFileProcessed)
             continue;
         }
         const provider = getProvider(language);
+        // Owner map for provider.synthesizeStructureMembers: type-declaration AST
+        // node id → graph node id for classes THIS file's capture loop materialized.
+        // Keyed by in-memory AST identity (never persisted); filled below.
+        const classOwnersByNodeId = new Map();
         // #2687: ONE pass over `matches` yields both suppression sets — the
         // definition-name claims by rank (callable > Property > value), so the dedup
         // below cannot depend on tree-sitter's match order, and the concrete-typedef
@@ -2248,6 +2252,14 @@ const processFileGroup = (files, language, queryString, result, onFileProcessed)
                     ? { annotations: methodProps.annotations }
                     : {}),
             });
+            // Class-like definitions register their AST node id → graph node id for
+            // provider.synthesizeStructureMembers. The definition node is the same
+            // type-declaration AST node that the provider-specific planner receives.
+            if (isClassLikeLabel &&
+                definitionNode &&
+                provider.classExtractor?.isTypeDeclaration(definitionNode)) {
+                classOwnersByNodeId.set(definitionNode.id, nodeId);
+            }
             // Object-literal callables remain file definitions as well as members of
             // their exported binding. Class members still use HAS_METHOD alone.
             const isTopLevelObjectCallable = objectLiteralBindingInfo?.ownerName !== undefined &&
@@ -2337,6 +2349,18 @@ const processFileGroup = (files, language, queryString, result, onFileProcessed)
             if (springTypes.length > 0)
                 (result.springTypes ??= []).push(...springTypes);
         }
+        if (provider.synthesizeStructureMembers) {
+            const synthetic = provider.synthesizeStructureMembers(tree, file.path, classOwnersByNodeId);
+            for (const node of synthetic.nodes) {
+                result.nodes.push(node);
+            }
+            for (const sym of synthetic.symbols) {
+                result.symbols.push(sym);
+            }
+            for (const rel of synthetic.relationships) {
+                result.relationships.push(rel);
+            }
+        }
         // Vue: emit CALLS edges for components used in <template>
         if (language === SupportedLanguages.Vue) {
             const templateComponents = extractTemplateComponents(file.content);
@@ -2469,8 +2493,10 @@ parentPort.on('message', (msg) => {
                     persistDurableParsedFileShardSync(DURABLE_PARSED_FILE_STORAGE_PATH, msg.chunkHash, threadId, seq, accumulated.parsedFiles);
                 }
                 if (PARSED_FILE_STORE_STORAGE_PATH) {
-                    persistParsedFileShardSync(PARSED_FILE_STORE_STORAGE_PATH, `w${threadId}-${seq}`, accumulated.parsedFiles);
-                    accumulated.parsedFiles = [];
+                    const wrote = persistParsedFileShardSync(PARSED_FILE_STORE_STORAGE_PATH, `w${threadId}-${seq}`, accumulated.parsedFiles);
+                    if (wrote) {
+                        accumulated.parsedFiles = [];
+                    }
                 }
             }
             postResultCloneSafe(accumulated);