gitnexus 1.6.8-rc.9 → 1.6.8

package/dist/core/ingestion/workers/parse-worker.js

@@ -11,7 +11,7 @@ import Go from 'tree-sitter-go';
 import Rust from 'tree-sitter-rust';
 import PHP from 'tree-sitter-php';
 import Ruby from 'tree-sitter-ruby';
-import { createRequire } from 'node:module';
+import { requireVendoredGrammar } from '../../tree-sitter/vendored-grammars.js';
 import { SupportedLanguages } from '../../../_shared/index.js';
 import { getProvider } from '../languages/index.js';
 import { getTreeSitterBufferSize, getTreeSitterContentByteLength, TREE_SITTER_MAX_BUFFER, } from '../constants.js';
@@ -20,8 +20,8 @@ import { parseSourceSafe } from '../../tree-sitter/safe-parse.js';
 import { postResultCloneSafe } from './post-result.js';
 import { mergeResult } from './result-merge.js';
 // ── Worker grammar loading — enforcement boundary (#2091/#2093, #2101) ───────
-// The worker maintains its own grammar table (the guarded `_require`s below +
-// `languageMap`) and intentionally does NOT consult the runtime
+// The worker maintains its own grammar table (the guarded vendored-grammar
+// loads below + `languageMap`) and intentionally does NOT consult the runtime
 // `GITNEXUS_SKIP_OPTIONAL_GRAMMARS` opt-out. It does not need to: the MAIN
 // THREAD's `parseableScanned` filter (pipeline-phases/parse-impl.ts, gated on
 // `parser-loader.isLanguageAvailable`, which honors the runtime opt-out and a
@@ -32,33 +32,29 @@ import { mergeResult } from './result-merge.js';
 // `isLanguageAvailable` must re-introduce the gate here. (The cleaner end-state
 // — routing this table through `parser-loader.getLanguageGrammar` so there is
 // one loader — is the deferred Tier-1 consolidation.)
-// tree-sitter-swift is an optionalDependency — may not be installed
-const _require = createRequire(import.meta.url);
+// Swift/Dart/Kotlin/C are vendored grammars loaded from `vendor/` by absolute
+// path (NEVER copied into node_modules — see vendored-grammars.ts / #2111). Each
+// may be absent on a platform without a prebuild or a toolchain-less /
+// `--ignore-scripts` install, so every load is guarded so a missing binding
+// cannot crash the worker at module-load (#2091/#2093, #2116).
 let Swift = null;
 try {
-    Swift = _require('tree-sitter-swift');
+    Swift = requireVendoredGrammar('tree-sitter-swift');
 }
 catch { }
-// tree-sitter-dart is an optionalDependency — may not be installed
 let Dart = null;
 try {
-    Dart = _require('tree-sitter-dart');
+    Dart = requireVendoredGrammar('tree-sitter-dart');
 }
 catch { }
-// tree-sitter-kotlin is an optionalDependency — may not be installed
 let Kotlin = null;
 try {
-    Kotlin = _require('tree-sitter-kotlin');
+    Kotlin = requireVendoredGrammar('tree-sitter-kotlin');
 }
 catch { }
-// tree-sitter-c is now vendored prebuild-only (#2116) and may be absent on a
-// toolchain-less / `--ignore-scripts` install. Guard it like Swift/Dart/Kotlin so
-// a missing binding cannot crash the worker at module-load (#2091/#2093); the
-// main-thread `isLanguageAvailable` filter keeps C files from being dispatched
-// here when the entry is absent.
 let C = null;
 try {
-    C = _require('tree-sitter-c');
+    C = requireVendoredGrammar('tree-sitter-c');
 }
 catch { }
 import { getLanguageFromFilename } from '../../../_shared/index.js';
@@ -73,6 +69,7 @@ import { extractTemplateArguments, templateArgumentsIdTag, templateConstraintsId
 import { extractParsedFile } from '../scope-extractor-bridge.js';
 import { persistParsedFileShardSync, persistDurableParsedFileShardSync, } from '../../../storage/parsedfile-store.js';
 import { extractLaravelRoutes } from '../route-extractors/laravel.js';
+import { collectFunctionCfgs, DEFAULT_PDG_MAX_FUNCTION_LINES, } from '../cfg/collect.js';
 import { logger } from '../../logger.js';
 // ── ParsedFile store (#1983 parallel serialization) ─────────────────────────
 // Read ONCE at worker init from `workerData` (immutable for the run, inherited
@@ -90,6 +87,17 @@ const PARSED_FILE_STORE_STORAGE_PATH = workerData?.parsedFileStoreStoragePath;
 // restores them without re-parsing. `undefined` ⇒ no durable write.
 const DURABLE_PARSED_FILE_STORAGE_PATH = workerData?.durableParsedFileStoragePath;
 let shardSeq = 0;
+// ── PDG/CFG opt-in (#2081 M1) ───────────────────────────────────────────────
+// Read ONCE at worker init from `workerData` (the worker never sees
+// PipelineOptions — config arrives via the pool factory's `workerData`, see
+// KTD7 / U5). When `pdg` is set, the worker builds a per-function control-flow
+// graph from the tree-sitter AST (where it lives) and serializes it onto
+// `ParsedFile.cfgSideChannel`. Off ⇒ no CFG work and no field — the default for
+// every run today. `pdgMaxFunctionLines` bounds per-function CFG cost
+// (0/undefined ⇒ no cap; see collectFunctionCfgs).
+const PDG_ENABLED = workerData?.pdg === true;
+const PDG_MAX_FUNCTION_LINES = workerData?.pdgMaxFunctionLines ??
+    DEFAULT_PDG_MAX_FUNCTION_LINES;
 // ── Bootstrap-stage diagnostics (#1741) ────────────────────────────────────
 // When GITNEXUS_WORKER_BOOTSTRAP=1 (or --verbose sets GITNEXUS_VERBOSE), each
 // worker reports its startup stage timings to stderr — which the pool tees
@@ -503,6 +511,7 @@ const processBatch = (files, onProgress) => {
         fileScopeBindings: [],
         parsedFiles: [],
         skippedLanguages: {},
+        cfgSkipped: {},
         fileCount: 0,
     };
     // Group by language to minimize setLanguage calls
@@ -837,7 +846,52 @@ const processFileGroup = (files, language, queryString, result, onFileProcessed)
             // copy — scopes/defs are carried by reference) to attach the field rather
             // than mutate the frozen object.
             const sideChannel = provider.collectCaptureSideChannel?.(file.path);
-            result.parsedFiles.push(sideChannel !== undefined ? { ...parsedFile, captureSideChannel: sideChannel } : parsedFile);
+            let withChannels = sideChannel !== undefined ? { ...parsedFile, captureSideChannel: sideChannel } : parsedFile;
+            // CFG side-channel (#2081 M1): build the per-function control-flow graph
+            // here, where the tree-sitter AST is still in hand, and attach it as plain
+            // serializable data. Only on a --pdg run and only for languages with a
+            // cfgVisitor (TS/JS in M1). The same disk-store/warm-cache machinery that
+            // carries captureSideChannel carries this — its coherence rests on the
+            // SCHEMA_BUMP + the pdg-folded chunk-hash key (see parse-cache.ts).
+            if (PDG_ENABLED && provider.cfgVisitor) {
+                // Isolate the CFG build per file: a throw here (an unexpected tree-sitter
+                // node shape) must NOT propagate — it would escape processFileGroup to the
+                // language-group catch, which treats any throw as "parser unavailable" and
+                // silently drops EVERY remaining file in the group. Skip CFG for this one
+                // file; parsing + scope resolution proceed unaffected (CFG is a
+                // strictly-additive opt-in). collectFunctionCfgs ALSO isolates per
+                // FUNCTION now (#2195) — a deep-nesting bail or a single malformed function
+                // is counted in `skipped` and skipped, not allowed to lose the whole file.
+                try {
+                    const { cfgs, skipped } = collectFunctionCfgs(tree.rootNode, provider.cfgVisitor, file.path, PDG_MAX_FUNCTION_LINES, 
+                    // Embedded scripts (Vue SFC <script>) parse at row 0 but live at
+                    // `lineOffset` in the file — shift the CFG into file coordinates so
+                    // it joins its graph node and BasicBlock lines map to source.
+                    lineOffset);
+                    if (cfgs.length)
+                        withChannels = { ...withChannels, cfgSideChannel: cfgs };
+                    // Surface per-function CFG skips per-language (#2195): merged + logged
+                    // in mergeChunkResults. Only accumulate when something was skipped so
+                    // the common (nothing-skipped) case stays a no-op.
+                    if (skipped.tooManyLines || skipped.tooDeeplyNested || skipped.buildError) {
+                        const agg = (result.cfgSkipped ??= {});
+                        const prev = agg[language] ?? { tooManyLines: 0, tooDeeplyNested: 0, buildError: 0 };
+                        agg[language] = {
+                            tooManyLines: prev.tooManyLines + skipped.tooManyLines,
+                            tooDeeplyNested: prev.tooDeeplyNested + skipped.tooDeeplyNested,
+                            buildError: prev.buildError + skipped.buildError,
+                        };
+                    }
+                }
+                catch (err) {
+                    const message = `CFG build failed for ${file.path}: ${err instanceof Error ? err.message : String(err)}`;
+                    if (parentPort)
+                        parentPort.postMessage({ type: 'warning', message });
+                    else
+                        logger.warn(message);
+                }
+            }
+            result.parsedFiles.push(withChannels);
         }
         // Build per-file type environment + constructor bindings in a single AST walk.
         // The legacy heritage pre-pass that seeded a file-local parentMap for
@@ -1677,6 +1731,9 @@ const processFileGroup = (files, language, queryString, result, onFileProcessed)
                 isReadonly: methodProps.isReadonly,
                 isAbstract: methodProps.isAbstract,
                 isFinal: methodProps.isFinal,
+                ...(methodProps.isDeleted !== undefined
+                    ? { isDeleted: methodProps.isDeleted }
+                    : {}),
                 ...(methodProps.isVirtual !== undefined
                     ? { isVirtual: methodProps.isVirtual }
                     : {}),
@@ -1792,6 +1849,7 @@ let accumulated = {
     fileScopeBindings: [],
     parsedFiles: [],
     skippedLanguages: {},
+    cfgSkipped: {},
     fileCount: 0,
 };
 let cumulativeProcessed = 0;
@@ -1910,6 +1968,7 @@ parentPort.on('message', (msg) => {
                 fileScopeBindings: [],
                 parsedFiles: [],
                 skippedLanguages: {},
+                cfgSkipped: {},
                 fileCount: 0,
             };
             cumulativeProcessed = 0;

package/dist/core/ingestion/workers/worker-pool.d.ts

@@ -1,30 +1,4 @@
 import { Worker } from 'node:worker_threads';
-/**
- * Build the sub-batch dispatch payload + transferList.
- *
- * For the parse-worker shape `{path, content: string}[]`, encodes each
- * file's content as a `Uint8Array` via `TextEncoder` so the underlying
- * `ArrayBuffer` can be transferred zero-copy. For any other input
- * shape, the items array is passed through verbatim (no transfer).
- *
- * @internal Exported for the unit test suite
- * (`test/unit/worker-pool-transferlist.test.ts`) so the
- * Uint8Array-per-content allocation contract can be pinned without
- * spinning up a real worker_threads.
- */
-export declare function buildDispatchMessage<T>(items: readonly T[]): {
-    message: {
-        type: 'sub-batch';
-        files: Array<{
-            path: string;
-            content: Uint8Array;
-        }>;
-    } | {
-        type: 'sub-batch';
-        files: readonly T[];
-    };
-    transferList?: ArrayBuffer[];
-};
 export interface WorkerPool {
     /**
      * Dispatch items across workers. Items are split into bounded jobs, each job
@@ -152,6 +126,15 @@ export interface WorkerPoolOptions {
      * `undefined` ⇒ no durable write.
      */
     durableParsedFileStoragePath?: string;
+    /**
+     * CFG/PDG opt-in (#2081 M1). Baked into every spawned worker's `workerData`
+     * (like the store paths above); when `true`, workers build a per-function
+     * control-flow graph from the tree-sitter AST and attach it to
+     * `ParsedFile.cfgSideChannel`. `undefined`/`false` ⇒ no CFG work.
+     */
+    pdg?: boolean;
+    /** Per-function source-line cap for worker-side CFG construction (0 ⇒ no cap). */
+    pdgMaxFunctionLines?: number;
 }
 export declare class WorkerPoolDispatchError extends Error {
     /**
@@ -201,20 +184,6 @@ export declare class WorkerPoolInitializationError extends WorkerPoolDispatchErr
 export declare class WorkerPoolDisabledError extends Error {
     constructor(message: string);
 }
-/**
- * Normalize a worker crash message into a stable signature so two instances of
- * the SAME deterministic crash compare equal while unrelated crashes don't.
- * Strips hex addresses, digit runs (pids / line numbers / timestamps) and
- * absolute paths. Best-effort by design: the deterministic classification's
- * correctness rests on the STRUCTURAL signal (zero workers ever ready + startup
- * budget exhausted), so an imperfect signature only changes how fast the
- * short-circuit fires, never whether the pool ultimately fails fast. Even a
- * stderr-less crash normalizes its "exited with code N" message to a stable
- * key, so the empty-stderr timing case still groups.
- *
- * @internal Exported for unit tests; production callers are in this module.
- */
-export declare function crashSignature(message: string): string;
 interface ResolvedWorkerPoolOptions {
     subBatchSize: number;
     subBatchMaxBytes: number;

package/dist/core/ingestion/workers/worker-pool.js

@@ -550,13 +550,26 @@ export const createWorkerPool = (workerUrl, poolSize, options) => {
     // signature is unchanged so the zero-arg test factories keep working.
     const parsedFileStoreStoragePath = options?.parsedFileStoreStoragePath;
     const durableParsedFileStoragePath = options?.durableParsedFileStoragePath;
-    const workerStoreData = parsedFileStoreStoragePath || durableParsedFileStoragePath
-        ? { parsedFileStoreStoragePath, durableParsedFileStoragePath }
+    // CFG/PDG opt-in (#2081 M1) — carried in workerData alongside the store paths.
+    const pdg = options?.pdg === true;
+    const pdgMaxFunctionLines = options?.pdgMaxFunctionLines;
+    const workerStoreData = parsedFileStoreStoragePath || durableParsedFileStoragePath || pdg
+        ? { parsedFileStoreStoragePath, durableParsedFileStoragePath, pdg, pdgMaxFunctionLines }
         : undefined;
     const spawnWorker = options?.workerFactory ??
         ((url) => new Worker(url, {
             stderr: true,
             workerData: workerStoreData,
+            // The CFG visitors build per-function control-flow graphs by RECURSIVE
+            // descent over the tree-sitter AST, so deeply-nested source overflows
+            // the worker thread's call stack. A worker's stack is governed by
+            // `resourceLimits.stackSizeMb` (Node default 4 MB) — the main process's
+            // `--stack-size` flag does NOT propagate to worker threads — so raise it
+            // here. This pushes the overflow threshold from ~1.5k to several-k
+            // nesting levels (far beyond any hand-written code); a deeper machine-
+            // generated nest is still caught per-function (buildFunctionCfg's R4
+            // try/catch) and only that function's PDG is skipped, never a crash.
+            resourceLimits: { stackSizeMb: 16 },
         }));
     /** Spawn + wire stderr capture in one step (used by all spawn sites). */
     const spawnAndCapture = (url) => {

package/dist/core/lbug/csv-generator.d.ts

@@ -11,23 +11,71 @@
  * - Double quotes within fields are escaped by doubling them ("")
  * - All fields are consistently quoted for safety with code content
  */
+import type { GraphNode, GraphRelationship } from '../../_shared/index.js';
 import { KnowledgeGraph } from '../graph/types.js';
 import { NodeTableName } from './schema.js';
 export declare const sanitizeUTF8: (str: string) => string;
 export declare const escapeCSVField: (value: string | number | undefined | null) => string;
 export declare const escapeCSVNumber: (value: number | undefined | null, defaultValue?: number) => string;
 export declare const isBinaryContent: (content: string) => boolean;
+/** Canonical relationship CSV header — shared by the emit pass and the
+ * `splitRelCsvByLabelPair` differential oracle. */
+export declare const REL_CSV_HEADER = "from,to,type,confidence,reason,step";
+/** Build the escaped CSV row (no trailing newline) for one relationship.
+ * Single source of the relationship row bytes — used by the emit pass and by
+ * the byte-identity differential test that feeds the legacy split oracle. */
+export declare const buildRelRow: (rel: GraphRelationship) => string;
+/** Canonical BasicBlock node CSV header — taint/PDG substrate (issue #2080).
+ * No `name` column; blocks are identified by id + source span. Shared by the
+ * whole-graph emit pass and the streaming PDG emit sink (issue #2202) so the
+ * two paths produce byte-identical BasicBlock rows by construction. */
+export declare const BASICBLOCK_CSV_HEADER = "id,filePath,startLine,endLine,text,callees,calleeIds";
+/** Build the escaped CSV row (no trailing newline) for one BasicBlock node.
+ * Single source of the BasicBlock row bytes — used by `streamAllCSVsToDisk`
+ * and by the streaming `PdgEmitSink` (issue #2202). `callees` is a comma-free
+ * (space-joined) list of the leaf callee names invoked in the block — the
+ * statement-precise inter-procedural reach substrate (the field is itself a CSV
+ * cell, so the inner separator must NOT be a comma). `calleeIds` is the SOUND
+ * parallel to `callees`: the space-joined RESOLVED callee symbol ids for the
+ * block (#2227 follow-up), likewise a comma-free cell. */
+export declare const buildBasicBlockRow: (node: GraphNode) => string;
 export interface StreamedCSVResult {
     nodeFiles: Map<NodeTableName, {
         csvPath: string;
         rows: number;
     }>;
-    relCsvPath: string;
-    relRows: number;
+    /** pairKey (`From|To`) → per-FROM→TO-label-pair CSV file. */
+    relsByPair: Map<string, {
+        csvPath: string;
+        rows: number;
+    }>;
+    /** Header line shared by every per-pair file. */
+    relHeader: string;
+    /** Edges skipped because an endpoint label is not a valid node table. */
+    skippedRels: number;
+    /** Edges routed to a per-pair file. */
+    totalValidRels: number;
 }
 /**
  * Stream all CSV data directly to disk files.
  * Iterates graph nodes exactly ONCE — routes each node to the right writer.
  * File contents are lazy-read from disk with a generous LRU cache.
+ *
+ * `onNodePhaseComplete` (optional, #2203 parallelism leg): fired exactly once,
+ * right after every node CSV is fully flushed to disk and BEFORE the
+ * relationship pass starts writing any `rel_*.csv`. It receives the finished
+ * node-file manifest so the caller can begin `COPY`-ing nodes while this
+ * function keeps generating relationship CSVs (the only single-writer-safe
+ * overlap — node `COPY` ‖ relationship emit). It is intentionally NOT awaited:
+ * the relationship pass proceeds concurrently with whatever the caller
+ * schedules. A synchronous throw from the callback is allowed and propagates out
+ * of this function (rejecting the returned promise) — it is raised before the
+ * relationship pass begins, so no `rel_*.csv` is written; `loadGraphToLbug` uses
+ * this to surface its PDG-manifest collision guard. The callback must NOT, however,
+ * schedule un-awaited async work that can reject unobserved. Absent ⇒ today's
+ * behavior, byte-for-byte.
  */
-export declare const streamAllCSVsToDisk: (graph: KnowledgeGraph, repoPath: string, csvDir: string) => Promise<StreamedCSVResult>;
+export declare const streamAllCSVsToDisk: (graph: KnowledgeGraph, repoPath: string, csvDir: string, onNodePhaseComplete?: (nodeFiles: Map<NodeTableName, {
+    csvPath: string;
+    rows: number;
+}>) => void) => Promise<StreamedCSVResult>;