cgraphx 2.0.2 → 2.0.3

package/dist/core/code/engine/core/ingestion/parsing-processor.js

@@ -0,0 +1,312 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.processParsing = exports.dispatchChunkParse = exports.mergeChunkResults = void 0;
+const index_js_1 = require("../../../shared/index.js");
+const call_processor_js_1 = require("./call-processor.js");
+const logger_js_1 = require("../logger.js");
+function compareText(left, right) {
+    return left < right ? -1 : left > right ? 1 : 0;
+}
+function sourceLine(node) {
+    const value = node.properties.startLine;
+    return typeof value === 'number' && Number.isFinite(value) ? value : Number.MAX_SAFE_INTEGER;
+}
+function compareParsedNodeSourceOrder(left, right) {
+    const leftPath = typeof left.properties.filePath === 'string' ? left.properties.filePath : '';
+    const rightPath = typeof right.properties.filePath === 'string' ? right.properties.filePath : '';
+    const fileOrder = compareText(leftPath, rightPath);
+    if (fileOrder !== 0)
+        return fileOrder;
+    const leftLine = sourceLine(left);
+    const rightLine = sourceLine(right);
+    if (leftLine !== rightLine)
+        return leftLine < rightLine ? -1 : 1;
+    return compareText(left.id, right.id);
+}
+function nodesInSourceOrder(nodes) {
+    for (let index = 1; index < nodes.length; index++) {
+        if (compareParsedNodeSourceOrder(nodes[index - 1], nodes[index]) > 0) {
+            return [...nodes].sort(compareParsedNodeSourceOrder);
+        }
+    }
+    return nodes;
+}
+// ============================================================================
+// Worker-based parallel parsing
+// ============================================================================
+/**
+ * Merge a list of `ParseWorkerResult`s into the running graph + symbol
+ * table state and produce the chunk-aggregated `WorkerExtractedData`.
+ *
+ * Split out from the worker-parse path so the same merge logic can
+ * be applied to both freshly-parsed worker output AND cached worker
+ * output replayed during incremental analyze. Idempotent on the
+ * accumulator fields (push-only); idempotent on graph if the caller
+ * starts from a clean graph (otherwise duplicate `addNode` calls are
+ * silently no-op'd by `KnowledgeGraph`).
+ */
+const mergeChunkResults = (graph, symbolTable, chunkResults, exportedTypeMap) => {
+    const allRoutes = [];
+    const allFetchCalls = [];
+    const allFetchWrapperDefs = [];
+    const allDecoratorRoutes = [];
+    const allRouterIncludes = [];
+    const allRouterImports = [];
+    const allRouterConstructorPrefixes = [];
+    const allRouterModuleAliases = [];
+    const allModuleConstants = [];
+    const allSpringTypes = [];
+    const allToolDefs = [];
+    const allORMQueries = [];
+    const fileScopeBindingsByFile = [];
+    const allParsedFiles = [];
+    for (const result of chunkResults) {
+        // Worker jobs and input files are already merged in stable start-index/path
+        // order. Canonicalize the final per-result node boundary once so graph
+        // insertion, cache replay, and first-wins graph indexes share source order.
+        // The common already-ordered path stays allocation-free and linear.
+        for (const node of nodesInSourceOrder(result.nodes)) {
+            graph.addNode({
+                id: node.id,
+                label: node.label,
+                properties: node.properties,
+            });
+        }
+        for (const rel of result.relationships) {
+            graph.addRelationship(rel);
+        }
+        for (const sym of result.symbols) {
+            symbolTable.add(sym.filePath, sym.name, sym.nodeId, sym.type, {
+                parameterCount: sym.parameterCount,
+                requiredParameterCount: sym.requiredParameterCount,
+                parameterTypes: sym.parameterTypes,
+                parameterTypeClasses: sym.parameterTypeClasses,
+                returnType: sym.returnType,
+                declaredType: sym.declaredType,
+                templateArguments: sym.templateArguments,
+                ownerId: sym.ownerId,
+                qualifiedName: sym.qualifiedName,
+                isDeleted: sym.isDeleted,
+            });
+        }
+        if (exportedTypeMap) {
+            for (const node of result.nodes) {
+                (0, call_processor_js_1.accumulateExportedTypesFromParsedNode)(exportedTypeMap, node, symbolTable);
+            }
+        }
+        for (const item of result.routes)
+            allRoutes.push(item);
+        for (const item of result.fetchCalls)
+            allFetchCalls.push(item);
+        for (const item of result.fetchWrapperDefs ?? [])
+            allFetchWrapperDefs.push(item);
+        for (const item of result.decoratorRoutes)
+            allDecoratorRoutes.push(item);
+        for (const item of result.routerIncludes ?? [])
+            allRouterIncludes.push(item);
+        for (const item of result.routerImports ?? [])
+            allRouterImports.push(item);
+        for (const item of result.routerConstructorPrefixes ?? []) {
+            allRouterConstructorPrefixes.push(item);
+        }
+        for (const item of result.routerModuleAliases ?? [])
+            allRouterModuleAliases.push(item);
+        for (const item of result.moduleConstants ?? [])
+            allModuleConstants.push(item);
+        for (const item of result.springTypes ?? [])
+            allSpringTypes.push(item);
+        for (const item of result.toolDefs)
+            allToolDefs.push(item);
+        if (result.ormQueries)
+            for (const item of result.ormQueries)
+                allORMQueries.push(item);
+        if (result.fileScopeBindings)
+            for (const item of result.fileScopeBindings)
+                fileScopeBindingsByFile.push(item);
+        if (result.parsedFiles)
+            for (const item of result.parsedFiles)
+                allParsedFiles.push(item);
+    }
+    return {
+        routes: allRoutes,
+        fetchCalls: allFetchCalls,
+        fetchWrapperDefs: allFetchWrapperDefs,
+        decoratorRoutes: allDecoratorRoutes,
+        routerIncludes: allRouterIncludes,
+        routerImports: allRouterImports,
+        routerConstructorPrefixes: allRouterConstructorPrefixes,
+        routerModuleAliases: allRouterModuleAliases,
+        moduleConstants: allModuleConstants,
+        toolDefs: allToolDefs,
+        ormQueries: allORMQueries,
+        springTypes: allSpringTypes,
+        fileScopeBindings: fileScopeBindingsByFile,
+        parsedFiles: allParsedFiles,
+    };
+};
+exports.mergeChunkResults = mergeChunkResults;
+/**
+ * Dispatch a chunk's files to the worker pool and return the RAW per-worker
+ * results, WITHOUT merging them into the graph. Split out from
+ * {@link processParsing} so the parse loop can overlap one chunk's
+ * merge (main-thread, via {@link mergeChunkResults}) with the NEXT chunk's
+ * worker parse — the merge is the only remaining serial main-thread step once
+ * ParsedFile serialization moved into the workers (#worker-idle pipelining).
+ * Returns `[]` for an all-unparseable chunk (the caller merges `[]` → empty).
+ */
+const dispatchChunkParse = async (files, workerPool, onFileProgress, 
+/** Populated in-place with the raw results (parse-cache capture). */
+outRawResults, 
+/**
+ * Content hash of this parse chunk. When set, the workers tag their durable
+ * ParsedFile shards with it so a future warm cache hit can restore them
+ * (#2038). `undefined` ⇒ no durable write (tests / no-cache path).
+ */
+chunkHash) => {
+    const parseableFiles = [];
+    for (const file of files) {
+        const lang = (0, index_js_1.getLanguageFromFilename)(file.path);
+        if (lang)
+            parseableFiles.push({ path: file.path, content: file.content });
+    }
+    if (parseableFiles.length === 0)
+        return [];
+    const total = files.length;
+    const chunkResults = await workerPool.dispatch(parseableFiles, (filesProcessed) => {
+        onFileProgress?.(Math.min(filesProcessed, total), total, 'Parsing...');
+    }, chunkHash);
+    // Capture raw results for the incremental parse cache before merging.
+    if (outRawResults) {
+        for (const r of chunkResults)
+            outRawResults.push(r);
+    }
+    // Skipped-language telemetry (worker output, independent of the merge).
+    const skippedLanguages = new Map();
+    for (const result of chunkResults) {
+        for (const [lang, count] of Object.entries(result.skippedLanguages)) {
+            skippedLanguages.set(lang, (skippedLanguages.get(lang) || 0) + count);
+        }
+    }
+    if (skippedLanguages.size > 0) {
+        const summary = Array.from(skippedLanguages.entries())
+            .map(([lang, count]) => `${lang}: ${count}`)
+            .join(', ');
+        logger_js_1.logger.warn(`  Skipped unsupported languages: ${summary}`);
+    }
+    // Per-language CFG skip telemetry (#2195): functions skipped during the worker
+    // CFG walk, bucketed by reason. Only surfaced for a `--pdg` run (otherwise
+    // `cfgSkipped` is empty). Warn ONLY when a robustness-relevant bucket
+    // (too-deeply-nested / build-error) is non-zero — a too-many-lines skip is the
+    // expected, benign minified/generated-code case and would otherwise be spam.
+    const cfgSkipped = new Map();
+    for (const result of chunkResults) {
+        for (const [lang, counts] of Object.entries(result.cfgSkipped ?? {})) {
+            const prev = cfgSkipped.get(lang) ?? { tooManyLines: 0, tooDeeplyNested: 0, buildError: 0 };
+            cfgSkipped.set(lang, {
+                tooManyLines: prev.tooManyLines + counts.tooManyLines,
+                tooDeeplyNested: prev.tooDeeplyNested + counts.tooDeeplyNested,
+                buildError: prev.buildError + counts.buildError,
+            });
+        }
+    }
+    for (const [lang, c] of cfgSkipped) {
+        if (c.tooDeeplyNested > 0 || c.buildError > 0) {
+            logger_js_1.logger.warn(`  CFG functions skipped (${lang}): ${c.tooDeeplyNested} too-deeply-nested, ` +
+                `${c.buildError} build-error(s), ${c.tooManyLines} over line cap`);
+        }
+    }
+    // Clone-safety telemetry (#2112): files whose parse output carried a value
+    // the structured-clone algorithm couldn't serialize across the worker
+    // boundary. The worker sanitized/dropped the offending value so the run
+    // could complete; surface the (rare) data loss so it's visible and the
+    // offending extractor can be fixed at source.
+    const skippedPaths = [];
+    for (const result of chunkResults) {
+        for (const entry of result.skippedPaths ?? [])
+            skippedPaths.push(entry);
+    }
+    if (skippedPaths.length > 0) {
+        // Keep the per-file reason ("stripped N value(s) from nodes" /
+        // "dropped non-serializable parsedFiles entry") — it distinguishes a
+        // recoverable strip from a whole-record drop, which a path-only line loses.
+        const shown = skippedPaths
+            .slice(0, 10)
+            .map((e) => `${e.path} (${e.reason})`)
+            .join(', ');
+        const more = skippedPaths.length > 10 ? ` …and ${skippedPaths.length - 10} more` : '';
+        logger_js_1.logger.warn(`  Sanitized ${skippedPaths.length} file(s) with non-serializable parse output: ${shown}${more}`);
+    }
+    onFileProgress?.(total, total, 'done');
+    return chunkResults;
+};
+exports.dispatchChunkParse = dispatchChunkParse;
+// ============================================================================
+// Public API
+// ============================================================================
+/**
+ * Per-`WorkerPool` log-dedup state for quarantine reporting. Keyed on the
+ * pool instance so multiple concurrent pools (test fixtures, future
+ * multi-pool callers) each get their own seen-set. WeakMap entries vanish
+ * when the pool is garbage-collected.
+ */
+const loggedQuarantineByPool = new WeakMap();
+const processParsing = async (graph, files, symbolTable, workerPool, onFileProgress, 
+/**
+ * Optional out-parameter for the incremental parse cache. When provided,
+ * populated with the raw `ParseWorkerResult[]` from the workers (pre-merge).
+ * See `cgraph/src/storage/parse-cache.ts`.
+ */
+outRawResults, exportedTypeMap) => {
+    let lastProgress = 0;
+    const reportProgress = onFileProgress
+        ? (current, total, detail) => {
+            lastProgress = Math.max(lastProgress, current);
+            onFileProgress(lastProgress, total, detail);
+        }
+        : undefined;
+    // U20 design pivot: the worker pool's resilience layers (respawn budget,
+    // circuit breaker, quarantine, slot-attribution, cumulative timeout) are the
+    // SOLE contract for handling worker failures. There is no sequential parser:
+    // a partial quarantine drops the file from this run's graph (surfaced by the
+    // per-chunk warn below; the chunk-cache write-guard in parse-impl.ts keeps the
+    // chunk uncached so the next analyze retries with a fresh pool), and a full
+    // pool failure propagates `WorkerPoolDispatchError` so the run errors out.
+    const chunkResults = await (0, exports.dispatchChunkParse)(files, workerPool, reportProgress, outRawResults);
+    const data = (0, exports.mergeChunkResults)(graph, symbolTable, chunkResults, exportedTypeMap);
+    // Session-scoped quarantine (worker-pool resilience Layer 3): surface any
+    // files this pool has decided are unsafe for workers so the operator can see
+    // what was skipped. The pool already filtered them out of dispatch; we only
+    // need to log + progress-report. Quarantine is session-scoped per pool
+    // instance — a fresh `createWorkerPool` call clears it.
+    //
+    // Dedup: log the full path list only for entries newly quarantined since the
+    // previous dispatch on the same pool. The per-chunk progress message still
+    // surfaces the count for UX continuity, but the structured `quarantinedFiles`
+    // payload is only emitted when there is new signal — prevents
+    // O(quarantine × chunks) log spam.
+    const quarantineSnapshot = workerPool.getQuarantinedPaths?.() ?? [];
+    const quarantineSet = new Set(quarantineSnapshot);
+    if (quarantineSet.size > 0) {
+        const quarantinedInChunk = files.filter((file) => quarantineSet.has(file.path));
+        if (quarantinedInChunk.length > 0) {
+            const seenForPool = loggedQuarantineByPool.get(workerPool) ?? new Set();
+            const newlyQuarantined = quarantinedInChunk
+                .map((file) => file.path)
+                .filter((p) => !seenForPool.has(p));
+            for (const p of newlyQuarantined)
+                seenForPool.add(p);
+            loggedQuarantineByPool.set(workerPool, seenForPool);
+            if (newlyQuarantined.length > 0) {
+                logger_js_1.logger.warn({
+                    newlyQuarantined,
+                    cumulativeQuarantine: quarantineSet.size,
+                    chunkSkipped: quarantinedInChunk.length,
+                }, `Worker quarantine: ${newlyQuarantined.length} new file(s) skipped this chunk ` +
+                    `(${quarantinedInChunk.length} skipped total, ${quarantineSet.size} cumulative).`);
+            }
+            reportProgress?.(lastProgress, files.length, `${quarantinedInChunk.length} worker-quarantined file(s) skipped`);
+        }
+    }
+    return data;
+};
+exports.processParsing = processParsing;

package/dist/core/code/engine/core/ingestion/pipeline-phases/communities.js

@@ -0,0 +1,69 @@
+"use strict";
+/**
+ * Phase: communities
+ *
+ * Detects code communities via Leiden algorithm and creates
+ * Community nodes + MEMBER_OF edges.
+ *
+ * @deps    mro, pruneLocalSymbols
+ * @reads   graph (all nodes and relationships)
+ * @writes  graph (Community nodes, MEMBER_OF edges)
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.communitiesPhase = void 0;
+const types_js_1 = require("./types.js");
+const community_processor_js_1 = require("../community-processor.js");
+const env_js_1 = require("../utils/env.js");
+const logger_js_1 = require("../../logger.js");
+exports.communitiesPhase = {
+    name: 'communities',
+    // `pruneLocalSymbols` is declared explicitly (not just transitively via `mro`)
+    // so community detection always reads the trimmed graph even if a future option
+    // drops `mro` from the phase list.
+    deps: ['mro', 'pruneLocalSymbols', 'structure'],
+    async execute(ctx, deps) {
+        const { totalFiles } = (0, types_js_1.getPhaseOutput)(deps, 'structure');
+        ctx.onProgress({
+            phase: 'communities',
+            percent: 98,
+            message: 'Detecting code communities...',
+            stats: { filesProcessed: totalFiles, totalFiles, nodesCreated: ctx.graph.nodeCount },
+        });
+        const communityResult = await (0, community_processor_js_1.processCommunities)(ctx.graph, (message, progress) => {
+            const communityProgress = 98 + progress * 0.01;
+            ctx.onProgress({
+                phase: 'communities',
+                percent: Math.round(communityProgress),
+                message,
+                stats: { filesProcessed: totalFiles, totalFiles, nodesCreated: ctx.graph.nodeCount },
+            });
+        });
+        if (env_js_1.isDev) {
+            logger_js_1.logger.info(`🏘️ Community detection: ${communityResult.stats.totalCommunities} communities found (modularity: ${communityResult.stats.modularity.toFixed(3)})`);
+        }
+        communityResult.communities.forEach((comm) => {
+            ctx.graph.addNode({
+                id: comm.id,
+                label: 'Community',
+                properties: {
+                    name: comm.label,
+                    filePath: '',
+                    heuristicLabel: comm.heuristicLabel,
+                    cohesion: comm.cohesion,
+                    symbolCount: comm.symbolCount,
+                },
+            });
+        });
+        communityResult.memberships.forEach((membership) => {
+            ctx.graph.addRelationship({
+                id: `${membership.nodeId}_member_of_${membership.communityId}`,
+                type: 'MEMBER_OF',
+                sourceId: membership.nodeId,
+                targetId: membership.communityId,
+                confidence: 1.0,
+                reason: 'leiden-algorithm',
+            });
+        });
+        return { communityResult };
+    },
+};

package/dist/core/code/engine/core/ingestion/pipeline-phases/cross-file.js

@@ -0,0 +1,71 @@
+"use strict";
+/**
+ * Phase: crossFile
+ *
+ * Accumulator disposal anchor. The legacy cross-file call re-resolution that
+ * this phase used to run (`runCrossFileBindingPropagation`) was owned by the
+ * call-resolution DAG and skipped every registry-primary language; RING4-1
+ * (#942) deleted the DAG, so the propagation is gone and this phase now only
+ * disposes the `BindingAccumulator`. It is kept as a phase (rather than folded
+ * into `parse`) so disposal stays sequenced after every accumulator consumer.
+ *
+ * @deps    parse, routes, tools, orm (waits for all post-parse phases)
+ * @reads   totalFiles, bindingAccumulator
+ * @writes  nothing (disposal only)
+ *
+ * **Accumulator ownership / residual risk.** This phase is the sole
+ * disposer of the `BindingAccumulator` produced by `parse`. The dispose
+ * call lives inside a `finally` block in `execute()` so that a throw
+ * anywhere in the body still releases the accumulator's heap. The dependency declaration
+ * (`deps: ['parse', 'routes', 'tools', 'orm']`) plus the runner's
+ * topological scheduling guarantee that every other consumer of the
+ * accumulator has finished before this phase starts, so disposing here
+ * is correct.
+ *
+ * The residual risk is intentional and accepted: if a future phase is
+ * inserted between `parse` and `crossFile` that reads the accumulator
+ * and throws, `crossFile.execute()` never runs and the accumulator
+ * leaks. Any author inserting a new phase between `parse` and
+ * `crossFile` MUST either route the new phase's output through
+ * `crossFile` (so disposal still happens here) or take ownership of
+ * the accumulator's lifetime explicitly (its own try/finally that
+ * disposes on the failure path). Do not silently rely on the GC.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.crossFilePhase = void 0;
+const types_js_1 = require("./types.js");
+const env_js_1 = require("../utils/env.js");
+const logger_js_1 = require("../../logger.js");
+exports.crossFilePhase = {
+    name: 'crossFile',
+    deps: ['parse', 'routes', 'tools', 'orm'],
+    async execute(ctx, deps) {
+        const { totalFiles, bindingAccumulator } = (0, types_js_1.getPhaseOutput)(deps, 'parse');
+        try {
+            // Telemetry must run BEFORE dispose: totalBindings, fileCount, and
+            // estimateMemoryBytes() all return 0 once dispose() clears the
+            // internal maps.
+            if (env_js_1.isDev) {
+                if (bindingAccumulator.totalBindings > 0) {
+                    const memKB = Math.round(bindingAccumulator.estimateMemoryBytes() / 1024);
+                    logger_js_1.logger.info(`📦 BindingAccumulator: ${bindingAccumulator.totalBindings} bindings across ${bindingAccumulator.fileCount} files (~${memKB} KB)`);
+                }
+                else if (totalFiles > 0) {
+                    logger_js_1.logger.info(`📦 BindingAccumulator: EMPTY — 0 bindings across 0 files despite ${totalFiles} parsed files. If the codebase has typed bindings, this indicates an upstream regression.`);
+                }
+            }
+            // Legacy cross-file call re-resolution was owned by the call-resolution
+            // DAG (registry-primary languages skipped it entirely). With the DAG
+            // removed (RING4-1 #942), scope-resolution owns all CALLS edges and no
+            // cross-file re-resolution pass runs here. This phase survives solely to
+            // dispose the BindingAccumulator on the runner's behalf (see finally).
+            return { filesReprocessed: 0 };
+        }
+        finally {
+            // Single dispose call site for the accumulator — runs on both the
+            // happy path and the throw path so the heap is always released
+            // before the runner moves on (or surfaces the error).
+            bindingAccumulator.dispose();
+        }
+    },
+};