gitnexus 1.6.0 → 1.6.1

package/dist/core/ingestion/pipeline-phases/scan.js

@@ -0,0 +1,46 @@
+/**
+ * Phase: scan
+ *
+ * Walks the repository filesystem and collects file paths + sizes.
+ * Does NOT read file contents — that happens in downstream phases.
+ *
+ * @deps    (none — this is the pipeline root)
+ * @reads   repoPath (filesystem)
+ * @writes  graph (nothing yet — just returns scanned paths)
+ * @output  ScannedFile[], allPaths[], totalFiles
+ */
+import { walkRepositoryPaths } from '../filesystem-walker.js';
+export const scanPhase = {
+    name: 'scan',
+    deps: [],
+    async execute(ctx) {
+        ctx.onProgress({
+            phase: 'extracting',
+            percent: 0,
+            message: 'Scanning repository...',
+        });
+        const scannedFiles = await walkRepositoryPaths(ctx.repoPath, (current, total, filePath) => {
+            const scanProgress = Math.round((current / total) * 15);
+            ctx.onProgress({
+                phase: 'extracting',
+                percent: scanProgress,
+                message: 'Scanning repository...',
+                detail: filePath,
+                stats: {
+                    filesProcessed: current,
+                    totalFiles: total,
+                    nodesCreated: ctx.graph.nodeCount,
+                },
+            });
+        });
+        const totalFiles = scannedFiles.length;
+        const allPaths = scannedFiles.map((f) => f.path);
+        ctx.onProgress({
+            phase: 'extracting',
+            percent: 15,
+            message: 'Repository scanned successfully',
+            stats: { filesProcessed: totalFiles, totalFiles, nodesCreated: ctx.graph.nodeCount },
+        });
+        return { scannedFiles, allPaths, totalFiles };
+    },
+};

package/dist/core/ingestion/pipeline-phases/structure.d.ts

@@ -0,0 +1,27 @@
+/**
+ * Phase: structure
+ *
+ * Builds File and Folder nodes in the graph from scanned paths.
+ *
+ * @deps    scan
+ * @reads   allPaths (from scan phase)
+ * @writes  graph (File, Folder nodes + CONTAINS edges)
+ */
+import type { PipelinePhase } from './types.js';
+/** Structure phase produces no additional data — it writes directly to the graph. */
+export interface StructureOutput {
+    /** Pass-through from scan for downstream phases. */
+    scannedFiles: {
+        path: string;
+        size: number;
+    }[];
+    allPaths: string[];
+    /**
+     * Materialized once here and shared across all downstream consumers
+     * (cobol, markdown, cross-file propagation). Avoids the previous
+     * per-phase `new Set(allPaths)` allocations on multi-thousand-file repos.
+     */
+    allPathSet: ReadonlySet<string>;
+    totalFiles: number;
+}
+export declare const structurePhase: PipelinePhase<StructureOutput>;

package/dist/core/ingestion/pipeline-phases/structure.js

@@ -0,0 +1,35 @@
+/**
+ * Phase: structure
+ *
+ * Builds File and Folder nodes in the graph from scanned paths.
+ *
+ * @deps    scan
+ * @reads   allPaths (from scan phase)
+ * @writes  graph (File, Folder nodes + CONTAINS edges)
+ */
+import { getPhaseOutput } from './types.js';
+import { processStructure } from '../structure-processor.js';
+export const structurePhase = {
+    name: 'structure',
+    deps: ['scan'],
+    async execute(ctx, deps) {
+        const { scannedFiles, allPaths, totalFiles } = getPhaseOutput(deps, 'scan');
+        ctx.onProgress({
+            phase: 'structure',
+            percent: 15,
+            message: 'Analyzing project structure...',
+            stats: { filesProcessed: 0, totalFiles, nodesCreated: ctx.graph.nodeCount },
+        });
+        processStructure(ctx.graph, allPaths);
+        ctx.onProgress({
+            phase: 'structure',
+            percent: 20,
+            message: 'Project structure analyzed',
+            stats: { filesProcessed: totalFiles, totalFiles, nodesCreated: ctx.graph.nodeCount },
+        });
+        // Build the set once here so cobol, markdown, and cross-file propagation
+        // can all reuse it instead of re-materializing `new Set(allPaths)` each.
+        const allPathSet = new Set(allPaths);
+        return { scannedFiles, allPaths, allPathSet, totalFiles };
+    },
+};

package/dist/core/ingestion/pipeline-phases/tools.d.ts

@@ -0,0 +1,20 @@
+/**
+ * Phase: tools
+ *
+ * Detects MCP/RPC tool definitions and creates Tool graph nodes.
+ *
+ * @deps    parse
+ * @reads   allToolDefs (from parse), allPaths
+ * @writes  graph (Tool nodes, HANDLES_TOOL edges)
+ * @output  toolDefs array
+ */
+import type { PipelinePhase } from './types.js';
+export interface ToolDef {
+    name: string;
+    filePath: string;
+    description: string;
+}
+export interface ToolsOutput {
+    toolDefs: ToolDef[];
+}
+export declare const toolsPhase: PipelinePhase<ToolsOutput>;

package/dist/core/ingestion/pipeline-phases/tools.js

@@ -0,0 +1,79 @@
+/**
+ * Phase: tools
+ *
+ * Detects MCP/RPC tool definitions and creates Tool graph nodes.
+ *
+ * @deps    parse
+ * @reads   allToolDefs (from parse), allPaths
+ * @writes  graph (Tool nodes, HANDLES_TOOL edges)
+ * @output  toolDefs array
+ */
+import { getPhaseOutput } from './types.js';
+import { generateId } from '../../../lib/utils.js';
+import { readFileContents } from '../filesystem-walker.js';
+import { isDev } from '../utils/env.js';
+export const toolsPhase = {
+    name: 'tools',
+    deps: ['parse'],
+    async execute(ctx, deps) {
+        const { allToolDefs, allPaths } = getPhaseOutput(deps, 'parse');
+        const toolDefs = [];
+        const seenToolNames = new Set();
+        for (const td of allToolDefs) {
+            if (seenToolNames.has(td.toolName))
+                continue;
+            seenToolNames.add(td.toolName);
+            toolDefs.push({ name: td.toolName, filePath: td.filePath, description: td.description });
+        }
+        // TS tool definition arrays — require inputSchema nearby
+        const toolCandidatePaths = allPaths.filter((p) => (p.endsWith('.ts') || p.endsWith('.js')) &&
+            p.toLowerCase().includes('tool') &&
+            !p.includes('node_modules') &&
+            !p.includes('test') &&
+            !p.includes('__'));
+        if (toolCandidatePaths.length > 0) {
+            const toolContents = await readFileContents(ctx.repoPath, toolCandidatePaths);
+            for (const [filePath, content] of toolContents) {
+                if (!content.includes('inputSchema'))
+                    continue;
+                const toolPattern = /name:\s*['"](\w+)['"]\s*,\s*\n?\s*description:\s*[`'"]([\s\S]*?)[`'"]/g;
+                let match;
+                while ((match = toolPattern.exec(content)) !== null) {
+                    const name = match[1];
+                    if (seenToolNames.has(name))
+                        continue;
+                    seenToolNames.add(name);
+                    toolDefs.push({
+                        name,
+                        filePath,
+                        description: match[2].slice(0, 200).replace(/\n/g, ' ').trim(),
+                    });
+                }
+            }
+        }
+        // Create Tool nodes and HANDLES_TOOL edges
+        if (toolDefs.length > 0) {
+            for (const td of toolDefs) {
+                const toolNodeId = generateId('Tool', td.name);
+                ctx.graph.addNode({
+                    id: toolNodeId,
+                    label: 'Tool',
+                    properties: { name: td.name, filePath: td.filePath, description: td.description },
+                });
+                const handlerFileId = generateId('File', td.filePath);
+                ctx.graph.addRelationship({
+                    id: generateId('HANDLES_TOOL', `${handlerFileId}->${toolNodeId}`),
+                    sourceId: handlerFileId,
+                    targetId: toolNodeId,
+                    type: 'HANDLES_TOOL',
+                    confidence: 1.0,
+                    reason: 'tool-definition',
+                });
+            }
+            if (isDev) {
+                console.log(`🔧 Tool registry: ${toolDefs.length} tools detected`);
+            }
+        }
+        return { toolDefs };
+    },
+};

package/dist/core/ingestion/pipeline-phases/types.d.ts

@@ -0,0 +1,79 @@
+/**
+ * Pipeline Phase — Type definitions.
+ *
+ * Each phase is a named node in the dependency graph with typed inputs and outputs.
+ * The runner resolves dependencies via topological sort and passes
+ * typed results from upstream phases as inputs to downstream phases.
+ *
+ * Design goals:
+ *  - Explicit data flow between phases via typed outputs
+ *  - The knowledge graph is a shared mutable accumulator — phases add nodes/edges
+ *    and may read prior phases' contributions. This is intentional: the graph is
+ *    the pipeline's primary output, not an inter-phase communication channel.
+ *  - Compile-time exhaustiveness (adding a phase = type error until wired)
+ *  - Each phase is independently testable with mocked inputs
+ */
+import type { KnowledgeGraph } from '../../graph/types.js';
+import type { PipelineProgress } from '../../../_shared/index.js';
+import type { PipelineOptions } from '../pipeline.js';
+/** Immutable context available to every phase. */
+export interface PipelineContext {
+    /** Absolute path to the repository root. */
+    readonly repoPath: string;
+    /** Mutable knowledge graph — the single shared accumulator. */
+    readonly graph: KnowledgeGraph;
+    /** Progress callback for UI updates. */
+    readonly onProgress: (progress: PipelineProgress) => void;
+    /** Pipeline options (skipGraphPhases, skipWorkers, etc.). */
+    readonly options?: PipelineOptions;
+    /** Pipeline start timestamp (for elapsed-time logging). */
+    readonly pipelineStart: number;
+}
+/** Wraps a phase's output with timing metadata. */
+export interface PhaseResult<T> {
+    /** Phase name (matches the phase's `name` field). */
+    readonly phaseName: string;
+    /** The typed output of the phase. */
+    readonly output: T;
+    /** Wall-clock duration in milliseconds. */
+    readonly durationMs: number;
+}
+/**
+ * A single phase in the ingestion pipeline.
+ *
+ * @typeParam TDeps - Tuple of dependency phase output types
+ * @typeParam TOutput - This phase's output type
+ */
+export interface PipelinePhase<TOutput = unknown> {
+    /** Unique name for logging and result lookup. */
+    readonly name: string;
+    /**
+     * Names of phases this phase depends on.
+     * The runner guarantees these have completed before execute() is called.
+     */
+    readonly deps: readonly string[];
+    /**
+     * Execute the phase.
+     *
+     * @param ctx    Shared pipeline context (graph, repoPath, progress, options)
+     * @param deps   Map of dependency name → PhaseResult (typed outputs from upstream phases)
+     * @returns      The phase's typed output
+     */
+    execute(ctx: PipelineContext, deps: ReadonlyMap<string, PhaseResult<unknown>>): Promise<TOutput>;
+}
+/**
+ * Helper to extract the typed output of a dependency phase.
+ *
+ * Type safety note: This uses an `as T` cast because the runner stores
+ * heterogeneous phase outputs in a single `Map<string, PhaseResult<unknown>>`.
+ * The cast is safe as long as callers use the correct output type for the
+ * named phase. Mismatches will surface as runtime type errors, not compile-time
+ * errors — this is an intentional trade-off for a static phase graph without
+ * a dynamic type registry.
+ *
+ * @param deps       The resolved dependency map from the runner
+ * @param phaseName  The name of the upstream phase whose output you need
+ * @returns          The typed output of the phase
+ * @throws           If the phase is not found in the dependency map
+ */
+export declare function getPhaseOutput<T>(deps: ReadonlyMap<string, PhaseResult<unknown>>, phaseName: string): T;

package/dist/core/ingestion/pipeline-phases/types.js

@@ -0,0 +1,37 @@
+/**
+ * Pipeline Phase — Type definitions.
+ *
+ * Each phase is a named node in the dependency graph with typed inputs and outputs.
+ * The runner resolves dependencies via topological sort and passes
+ * typed results from upstream phases as inputs to downstream phases.
+ *
+ * Design goals:
+ *  - Explicit data flow between phases via typed outputs
+ *  - The knowledge graph is a shared mutable accumulator — phases add nodes/edges
+ *    and may read prior phases' contributions. This is intentional: the graph is
+ *    the pipeline's primary output, not an inter-phase communication channel.
+ *  - Compile-time exhaustiveness (adding a phase = type error until wired)
+ *  - Each phase is independently testable with mocked inputs
+ */
+/**
+ * Helper to extract the typed output of a dependency phase.
+ *
+ * Type safety note: This uses an `as T` cast because the runner stores
+ * heterogeneous phase outputs in a single `Map<string, PhaseResult<unknown>>`.
+ * The cast is safe as long as callers use the correct output type for the
+ * named phase. Mismatches will surface as runtime type errors, not compile-time
+ * errors — this is an intentional trade-off for a static phase graph without
+ * a dynamic type registry.
+ *
+ * @param deps       The resolved dependency map from the runner
+ * @param phaseName  The name of the upstream phase whose output you need
+ * @returns          The typed output of the phase
+ * @throws           If the phase is not found in the dependency map
+ */
+export function getPhaseOutput(deps, phaseName) {
+    const result = deps.get(phaseName);
+    if (!result) {
+        throw new Error(`Phase '${phaseName}' not found in resolved dependencies`);
+    }
+    return result.output;
+}

package/dist/core/ingestion/pipeline-phases/wildcard-synthesis.d.ts

@@ -0,0 +1,35 @@
+/**
+ * Wildcard import binding synthesis.
+ *
+ * Languages with whole-module import semantics (Go, Ruby, C/C++, Swift)
+ * import all exported symbols from a file, not specific named symbols.
+ * After parsing, we know which symbols each file exports (via graph
+ * `isExported`), so we can expand IMPORTS edges into per-symbol bindings
+ * that the cross-file propagation phase can use for type resolution.
+ *
+ * Also builds Python module-alias maps for namespace-import languages
+ * (`import models` → `models.User()` resolves to `models.py:User`).
+ *
+ * @module
+ */
+import type { KnowledgeGraph } from '../../graph/types.js';
+import type { createResolutionContext } from '../model/resolution-context.js';
+import { SupportedLanguages } from '../../../_shared/index.js';
+/** Check if a language uses wildcard (whole-module) import semantics. */
+export declare function isWildcardImportLanguage(lang: SupportedLanguages): boolean;
+/** Check if a language needs synthesis before call resolution.
+ *  True for wildcard-import languages AND namespace-import languages (Python). */
+export declare function needsSynthesis(lang: SupportedLanguages): boolean;
+/**
+ * Synthesize namedImportMap entries for languages with whole-module imports.
+ *
+ * For each file that imports another file via wildcard semantics:
+ * 1. Look up all exported symbols from the imported file (via graph nodes)
+ * 2. Create synthetic named bindings: `{ name → { sourcePath, exportedName } }`
+ * 3. Build Python module-alias maps for namespace-import languages
+ *
+ * @param graph  The knowledge graph with parsed symbol nodes
+ * @param ctx    Resolution context with importMap and namedImportMap
+ * @returns      Number of synthetic bindings created
+ */
+export declare function synthesizeWildcardImportBindings(graph: KnowledgeGraph, ctx: ReturnType<typeof createResolutionContext>): number;

package/dist/core/ingestion/pipeline-phases/wildcard-synthesis.js

@@ -0,0 +1,174 @@
+/**
+ * Wildcard import binding synthesis.
+ *
+ * Languages with whole-module import semantics (Go, Ruby, C/C++, Swift)
+ * import all exported symbols from a file, not specific named symbols.
+ * After parsing, we know which symbols each file exports (via graph
+ * `isExported`), so we can expand IMPORTS edges into per-symbol bindings
+ * that the cross-file propagation phase can use for type resolution.
+ *
+ * Also builds Python module-alias maps for namespace-import languages
+ * (`import models` → `models.User()` resolves to `models.py:User`).
+ *
+ * @module
+ */
+import { getLanguageFromFilename } from '../../../_shared/index.js';
+import { providers, getProviderForFile } from '../languages/index.js';
+// ── Constants ──────────────────────────────────────────────────────────────
+/** Node labels that represent top-level importable symbols. */
+const IMPORTABLE_SYMBOL_LABELS = new Set([
+    'Function',
+    'Class',
+    'Interface',
+    'Struct',
+    'Enum',
+    'Trait',
+    'TypeAlias',
+    'Const',
+    'Static',
+    'Record',
+    'Union',
+    'Typedef',
+    'Macro',
+]);
+/** Max synthetic bindings per importing file — prevents memory bloat
+ *  for C/C++ files that include many large headers. */
+const MAX_SYNTHETIC_BINDINGS_PER_FILE = 1000;
+/** Languages with whole-module import semantics (derived from providers at module load). */
+const WILDCARD_LANGUAGES = new Set(Object.values(providers)
+    .filter((p) => p.importSemantics === 'wildcard')
+    .map((p) => p.id));
+/** Languages that need binding synthesis before call resolution. */
+const SYNTHESIS_LANGUAGES = new Set(Object.values(providers)
+    .filter((p) => p.importSemantics !== 'named')
+    .map((p) => p.id));
+/** Check if a language uses wildcard (whole-module) import semantics. */
+export function isWildcardImportLanguage(lang) {
+    return WILDCARD_LANGUAGES.has(lang);
+}
+/** Check if a language needs synthesis before call resolution.
+ *  True for wildcard-import languages AND namespace-import languages (Python). */
+export function needsSynthesis(lang) {
+    return SYNTHESIS_LANGUAGES.has(lang);
+}
+// ── Main synthesis function ────────────────────────────────────────────────
+/**
+ * Synthesize namedImportMap entries for languages with whole-module imports.
+ *
+ * For each file that imports another file via wildcard semantics:
+ * 1. Look up all exported symbols from the imported file (via graph nodes)
+ * 2. Create synthetic named bindings: `{ name → { sourcePath, exportedName } }`
+ * 3. Build Python module-alias maps for namespace-import languages
+ *
+ * @param graph  The knowledge graph with parsed symbol nodes
+ * @param ctx    Resolution context with importMap and namedImportMap
+ * @returns      Number of synthetic bindings created
+ */
+export function synthesizeWildcardImportBindings(graph, ctx) {
+    // Build exported symbols index from graph nodes (single pass)
+    const exportedSymbolsByFile = new Map();
+    graph.forEachNode((node) => {
+        if (!node.properties?.isExported)
+            return;
+        if (!IMPORTABLE_SYMBOL_LABELS.has(node.label))
+            return;
+        const fp = node.properties.filePath;
+        const name = node.properties.name;
+        if (!fp || !name)
+            return;
+        let symbols = exportedSymbolsByFile.get(fp);
+        if (!symbols) {
+            symbols = [];
+            exportedSymbolsByFile.set(fp, symbols);
+        }
+        symbols.push({ name, filePath: fp });
+    });
+    if (exportedSymbolsByFile.size === 0)
+        return 0;
+    // Collect graph-level IMPORTS edges for wildcard languages missing from ctx.importMap
+    const FILE_PREFIX = 'File:';
+    const graphImports = new Map();
+    graph.forEachRelationship((rel) => {
+        if (rel.type !== 'IMPORTS')
+            return;
+        if (!rel.sourceId.startsWith(FILE_PREFIX) || !rel.targetId.startsWith(FILE_PREFIX))
+            return;
+        const srcFile = rel.sourceId.slice(FILE_PREFIX.length);
+        const tgtFile = rel.targetId.slice(FILE_PREFIX.length);
+        const lang = getLanguageFromFilename(srcFile);
+        if (!lang || !isWildcardImportLanguage(lang))
+            return;
+        if (ctx.importMap.get(srcFile)?.has(tgtFile))
+            return;
+        let set = graphImports.get(srcFile);
+        if (!set) {
+            set = new Set();
+            graphImports.set(srcFile, set);
+        }
+        set.add(tgtFile);
+    });
+    let totalSynthesized = 0;
+    const synthesizeForFile = (filePath, importedFiles) => {
+        let fileBindings = ctx.namedImportMap.get(filePath);
+        let fileCount = fileBindings?.size ?? 0;
+        for (const importedFile of importedFiles) {
+            const exportedSymbols = exportedSymbolsByFile.get(importedFile);
+            if (!exportedSymbols)
+                continue;
+            for (const sym of exportedSymbols) {
+                if (fileCount >= MAX_SYNTHETIC_BINDINGS_PER_FILE)
+                    return;
+                if (fileBindings?.has(sym.name))
+                    continue;
+                if (!fileBindings) {
+                    fileBindings = new Map();
+                    ctx.namedImportMap.set(filePath, fileBindings);
+                }
+                fileBindings.set(sym.name, {
+                    sourcePath: importedFile,
+                    exportedName: sym.name,
+                });
+                fileCount++;
+                totalSynthesized++;
+            }
+        }
+    };
+    // Synthesize from ctx.importMap (Ruby, C/C++, Swift file-based imports)
+    for (const [filePath, importedFiles] of ctx.importMap) {
+        const lang = getLanguageFromFilename(filePath);
+        if (!lang || !isWildcardImportLanguage(lang))
+            continue;
+        synthesizeForFile(filePath, importedFiles);
+    }
+    // Synthesize from graph IMPORTS edges (Go and other wildcard-import languages)
+    for (const [filePath, importedFiles] of graphImports) {
+        synthesizeForFile(filePath, importedFiles);
+    }
+    // Build Python module-alias maps for namespace-import languages.
+    // `import models` in app.py → moduleAliasMap['app.py']['models'] = 'models.py'
+    // Enables `models.User()` to resolve without ambiguous symbol expansion.
+    for (const [filePath, importedFiles] of ctx.importMap) {
+        const provider = getProviderForFile(filePath);
+        if (!provider || provider.importSemantics !== 'namespace')
+            continue;
+        buildPythonModuleAliasForFile(ctx, filePath, importedFiles);
+    }
+    return totalSynthesized;
+}
+/** Build module alias entries for namespace-import files (e.g. Python). */
+function buildPythonModuleAliasForFile(ctx, callerFile, importedFiles) {
+    let aliasMap = ctx.moduleAliasMap.get(callerFile);
+    for (const importedFile of importedFiles) {
+        const lastSlash = importedFile.lastIndexOf('/');
+        const base = lastSlash >= 0 ? importedFile.slice(lastSlash + 1) : importedFile;
+        const dot = base.lastIndexOf('.');
+        const stem = dot >= 0 ? base.slice(0, dot) : base;
+        if (!stem)
+            continue;
+        if (!aliasMap) {
+            aliasMap = new Map();
+            ctx.moduleAliasMap.set(callerFile, aliasMap);
+        }
+        aliasMap.set(stem, importedFile);
+    }
+}

package/dist/core/ingestion/pipeline.d.ts

@@ -1,14 +1,21 @@
+/**
+ * Pipeline orchestrator — dependency-ordered ingestion pipeline.
+ *
+ * The pipeline is composed of named phases with explicit dependencies.
+ * Each phase is defined in its own file under `pipeline-phases/`.
+ * The runner in `pipeline-phases/runner.ts` executes phases in
+ * topological order, passing typed outputs from upstream phases as
+ * inputs to downstream phases.
+ *
+ * To add a new phase:
+ * 1. Create a new file in `pipeline-phases/` following the pattern
+ * 2. Export it from `pipeline-phases/index.ts`
+ * 3. Add it to the `ALL_PHASES` array below
+ *
+ * See ARCHITECTURE.md for the full phase dependency diagram.
+ */
 import { type PipelineProgress } from '../../_shared/index.js';
 import { PipelineResult } from '../../types/pipeline.js';
-/** A group of files with no mutual dependencies, safe to process in parallel. */
-type IndependentFileGroup = readonly string[];
-/** Kahn's algorithm: returns files grouped by topological level.
- *  Files in the same level have no mutual dependencies — safe to process in parallel.
- *  Files in cycles are returned as a final group (no cross-cycle propagation). */
-export declare function topologicalLevelSort(importMap: ReadonlyMap<string, ReadonlySet<string>>): {
-    levels: readonly IndependentFileGroup[];
-    cycleCount: number;
-};
 export interface PipelineOptions {
     /** Skip MRO, community detection, and process extraction for faster test runs. */
     skipGraphPhases?: boolean;
@@ -16,4 +23,3 @@ export interface PipelineOptions {
     skipWorkers?: boolean;
 }
 export declare const runPipelineFromRepo: (repoPath: string, onProgress: (progress: PipelineProgress) => void, options?: PipelineOptions) => Promise<PipelineResult>;
-export {};