arceus-s 1.6.4

package/dist/core/ingestion/pipeline-phases/wildcard-synthesis.js

@@ -0,0 +1,312 @@
+/**
+ * Wildcard import binding synthesis.
+ *
+ * Languages with whole-module import semantics (Go, Ruby, C/C++, Swift)
+ * import all exported symbols from a file, not specific named symbols.
+ * After parsing, we know which symbols each file exports (via graph
+ * `isExported`), so we can expand IMPORTS edges into per-symbol bindings
+ * that the cross-file propagation phase can use for type resolution.
+ *
+ * Also builds Python module-alias maps for namespace-import languages
+ * (`import models` → `models.User()` resolves to `models.py:User`).
+ *
+ * @module
+ */
+import { getLanguageFromFilename } from '../../../_shared/index.js';
+import { providers, getProviderForFile } from '../languages/index.js';
+// ── Constants ──────────────────────────────────────────────────────────────
+/** Node labels that represent top-level importable symbols. */
+const IMPORTABLE_SYMBOL_LABELS = new Set([
+    'Function',
+    'Class',
+    'Interface',
+    'Struct',
+    'Enum',
+    'Trait',
+    'TypeAlias',
+    'Const',
+    'Static',
+    'Record',
+    'Union',
+    'Typedef',
+    'Macro',
+]);
+/** Max synthetic bindings per importing file — prevents memory bloat
+ *  for C/C++ files that include many large headers. */
+const MAX_SYNTHETIC_BINDINGS_PER_FILE = 1000;
+/** Max files allowed in a single transitive include closure. Guards against
+ *  OOM on pathological C/C++ codebases (boost, Linux kernel-style monoheaders)
+ *  where a single translation unit can transitively reach many thousands of
+ *  headers. When the cap is hit, BFS expansion stops early — the file still
+ *  synthesizes bindings from the partial closure rather than failing. */
+const MAX_TRANSITIVE_CLOSURE_SIZE = 5000;
+/** Import semantics tags whose languages need synthesis of whole-module imports.
+ *  `wildcard-transitive` (C/C++) and `wildcard-leaf` (Go, Ruby, Swift, Dart) are
+ *  the file-based wildcard strategies. `explicit-reexport` is a scaffold tag —
+ *  no provider uses it yet, but it goes through the same leaf-style synthesis
+ *  path today because a re-exporter is still an importer; only the extra DAG
+ *  walk to surface re-exported symbols is missing (future work). */
+const WILDCARD_SEMANTICS = new Set([
+    'wildcard-transitive',
+    'wildcard-leaf',
+    'explicit-reexport',
+]);
+/** Languages with whole-module import semantics (derived from providers at module load). */
+const WILDCARD_LANGUAGES = new Set(Object.values(providers)
+    .filter((p) => WILDCARD_SEMANTICS.has(p.importSemantics))
+    .map((p) => p.id));
+/** Languages that need binding synthesis before call resolution. */
+const SYNTHESIS_LANGUAGES = new Set(Object.values(providers)
+    .filter((p) => p.importSemantics !== 'named')
+    .map((p) => p.id));
+/** Check if a language uses wildcard (whole-module) import semantics. */
+export function isWildcardImportLanguage(lang) {
+    return WILDCARD_LANGUAGES.has(lang);
+}
+/** Check if a language needs synthesis before call resolution.
+ *  True for wildcard-import languages AND namespace-import languages (Python). */
+export function needsSynthesis(lang) {
+    return SYNTHESIS_LANGUAGES.has(lang);
+}
+// ── Strategy implementations ───────────────────────────────────────────────
+/**
+ * Strategy implementation for `importSemantics: 'wildcard-transitive'` (C, C++).
+ *
+ * Textual-include languages chain symbols through files: if `dict.c` includes
+ * `server.h` and `server.h` includes `dict.h`, then `dict.c` sees symbols from
+ * all three files. This helper walks the include graph (combining both the
+ * ingestion-context `importMap` and the graph-level IMPORTS edges) until the
+ * closure is stable.
+ *
+ * **Order matters.** The returned `Set` preserves iteration order (insertion
+ * order). `synthesizeWildcardImportBindings` dedupes bindings by symbol name
+ * on a first-seen-wins basis, so this closure's ordering determines which
+ * declaration wins when multiple headers export the same name (e.g. overloaded
+ * free functions like `write_audit()` vs `write_audit(const char*)` in
+ * different headers). We therefore:
+ *   1. Seed the closure with direct imports in declaration order (matches the
+ *      order of `#include` directives in the source file).
+ *   2. Use FIFO / true BFS (`queue.shift()`) for transitive expansion, so
+ *      closer headers are seen before deeper ones.
+ *
+ * Cycle-safe: the `closure.has(file)` guard prevents infinite loops on circular
+ * header includes, which are valid C/C++ when paired with `#pragma once` or
+ * include guards.
+ *
+ * Size-bounded: the closure is capped at `MAX_TRANSITIVE_CLOSURE_SIZE` files to
+ * prevent OOM on pathological codebases (e.g. boost, monoheader kernel code)
+ * where one translation unit can transitively reach tens of thousands of
+ * headers. Partial closures still yield useful bindings for the cluster of
+ * headers closest to the importer, which is what overload resolution and
+ * cross-file call resolution care about.
+ *
+ * Queue implementation: uses a head-index over a growing array (O(1) dequeue)
+ * instead of `Array.prototype.shift()` (O(n)) so deep chains stay linear.
+ */
+export function expandTransitiveIncludeClosure(directImports, importMap, graphImports) {
+    const closure = new Set();
+    const queue = [];
+    let head = 0; // O(1) dequeue: advance the head index instead of shift()-ing.
+    const tryEnqueue = (file) => {
+        if (closure.has(file))
+            return true;
+        if (closure.size >= MAX_TRANSITIVE_CLOSURE_SIZE)
+            return false;
+        closure.add(file);
+        queue.push(file);
+        return true;
+    };
+    // Seed direct imports in declaration order (see JSDoc on order-sensitivity).
+    for (const f of directImports) {
+        if (!tryEnqueue(f))
+            break;
+    }
+    // True BFS for transitive reach: head-index FIFO preserves the "closer
+    // headers first" ordering that overload resolution depends on.
+    while (head < queue.length) {
+        if (closure.size >= MAX_TRANSITIVE_CLOSURE_SIZE)
+            break;
+        const file = queue[head++];
+        const nested = importMap.get(file);
+        if (nested) {
+            for (const n of nested) {
+                if (!tryEnqueue(n))
+                    break;
+            }
+        }
+        const nestedGraph = graphImports.get(file);
+        if (nestedGraph) {
+            for (const n of nestedGraph) {
+                if (!tryEnqueue(n))
+                    break;
+            }
+        }
+    }
+    return closure;
+}
+// ── Main synthesis function ────────────────────────────────────────────────
+/**
+ * Synthesize namedImportMap entries for languages with whole-module imports.
+ *
+ * For each file that imports another file via wildcard semantics:
+ * 1. Look up all exported symbols from the imported file (via graph nodes)
+ * 2. Create synthetic named bindings: `{ name → { sourcePath, exportedName } }`
+ * 3. Build Python module-alias maps for namespace-import languages
+ *
+ * @param graph  The knowledge graph with parsed symbol nodes
+ * @param ctx    Resolution context with importMap and namedImportMap
+ * @returns      Number of synthetic bindings created
+ */
+export function synthesizeWildcardImportBindings(graph, ctx) {
+    // Build exported symbols index from graph nodes (single pass)
+    const exportedSymbolsByFile = new Map();
+    graph.forEachNode((node) => {
+        if (!node.properties?.isExported)
+            return;
+        if (!IMPORTABLE_SYMBOL_LABELS.has(node.label))
+            return;
+        const fp = node.properties.filePath;
+        const name = node.properties.name;
+        if (!fp || !name)
+            return;
+        let symbols = exportedSymbolsByFile.get(fp);
+        if (!symbols) {
+            symbols = [];
+            exportedSymbolsByFile.set(fp, symbols);
+        }
+        symbols.push({ name, filePath: fp });
+    });
+    if (exportedSymbolsByFile.size === 0)
+        return 0;
+    // Collect graph-level IMPORTS edges for wildcard languages missing from ctx.importMap
+    const FILE_PREFIX = 'File:';
+    const graphImports = new Map();
+    graph.forEachRelationship((rel) => {
+        if (rel.type !== 'IMPORTS')
+            return;
+        if (!rel.sourceId.startsWith(FILE_PREFIX) || !rel.targetId.startsWith(FILE_PREFIX))
+            return;
+        const srcFile = rel.sourceId.slice(FILE_PREFIX.length);
+        const tgtFile = rel.targetId.slice(FILE_PREFIX.length);
+        const lang = getLanguageFromFilename(srcFile);
+        if (!lang || !isWildcardImportLanguage(lang))
+            return;
+        if (ctx.importMap.get(srcFile)?.has(tgtFile))
+            return;
+        let set = graphImports.get(srcFile);
+        if (!set) {
+            set = new Set();
+            graphImports.set(srcFile, set);
+        }
+        set.add(tgtFile);
+    });
+    let totalSynthesized = 0;
+    const synthesizeForFile = (filePath, importedFiles) => {
+        let fileBindings = ctx.namedImportMap.get(filePath);
+        let fileCount = fileBindings?.size ?? 0;
+        for (const importedFile of importedFiles) {
+            const exportedSymbols = exportedSymbolsByFile.get(importedFile);
+            if (!exportedSymbols)
+                continue;
+            for (const sym of exportedSymbols) {
+                if (fileCount >= MAX_SYNTHETIC_BINDINGS_PER_FILE)
+                    return;
+                if (fileBindings?.has(sym.name))
+                    continue;
+                if (!fileBindings) {
+                    fileBindings = new Map();
+                    ctx.namedImportMap.set(filePath, fileBindings);
+                }
+                fileBindings.set(sym.name, {
+                    sourcePath: importedFile,
+                    exportedName: sym.name,
+                });
+                fileCount++;
+                totalSynthesized++;
+            }
+        }
+    };
+    /**
+     * Dispatch wildcard synthesis by the file's language provider strategy.
+     *
+     * Strategy tags (see `ImportSemantics`):
+     *   - `wildcard-transitive`: expand the include closure first (C/C++ #include
+     *     chains — e.g. `dict.c` → `server.h` → `dict.h` so `dictFind` resolves
+     *     across header chains)
+     *   - `wildcard-leaf`: synthesize from direct imports only (Go, Ruby, Swift, Dart)
+     *   - `explicit-reexport`: scaffold tag; falls through to leaf behavior.
+     *     TODO(#821): implement re-export DAG walk for TS `export *` / Rust
+     *     `pub use`. The leaf fallthrough preserves today's TS/Rust behavior
+     *     (their direct imports still synthesize correctly); only the extra
+     *     re-export DAG walk for barrel-file correctness is missing.
+     *   - `namespace` / `named`: no-op here (namespace handled in Loop 3 below,
+     *     named needs no synthesis).
+     *
+     * Used by both Loop 1 (ctx.importMap) and Loop 2 (graphImports) so a future
+     * transitive-import language whose edges arrive via graphImports gets closure
+     * expansion consistently regardless of edge source.
+     */
+    const dispatchSynthesis = (filePath, importedFiles, provider) => {
+        switch (provider.importSemantics) {
+            case 'wildcard-transitive':
+                synthesizeForFile(filePath, expandTransitiveIncludeClosure(importedFiles, ctx.importMap, graphImports));
+                return;
+            case 'wildcard-leaf':
+            case 'explicit-reexport':
+                synthesizeForFile(filePath, importedFiles);
+                return;
+            case 'namespace':
+            case 'named':
+                return;
+            default: {
+                const _exhaustive = provider.importSemantics;
+                void _exhaustive;
+            }
+        }
+    };
+    // Loop 1: synthesize from ctx.importMap (Ruby, C/C++, Swift, Dart file-based imports).
+    for (const [filePath, importedFiles] of ctx.importMap) {
+        const lang = getLanguageFromFilename(filePath);
+        if (!lang || !isWildcardImportLanguage(lang))
+            continue;
+        const provider = getProviderForFile(filePath);
+        if (!provider)
+            continue;
+        dispatchSynthesis(filePath, importedFiles, provider);
+    }
+    // Loop 2: synthesize from graph IMPORTS edges (Go and other wildcard-import
+    // languages whose edges live in the graph rather than ctx.importMap).
+    for (const [filePath, importedFiles] of graphImports) {
+        const provider = getProviderForFile(filePath);
+        if (!provider)
+            continue;
+        dispatchSynthesis(filePath, importedFiles, provider);
+    }
+    // Build Python module-alias maps for namespace-import languages.
+    // `import models` in app.py → moduleAliasMap['app.py']['models'] = 'models.py'
+    // Enables `models.User()` to resolve without ambiguous symbol expansion.
+    for (const [filePath, importedFiles] of ctx.importMap) {
+        const provider = getProviderForFile(filePath);
+        if (!provider || provider.importSemantics !== 'namespace')
+            continue;
+        buildPythonModuleAliasForFile(ctx, filePath, importedFiles);
+    }
+    return totalSynthesized;
+}
+/** Build module alias entries for namespace-import files (e.g. Python). */
+function buildPythonModuleAliasForFile(ctx, callerFile, importedFiles) {
+    let aliasMap = ctx.moduleAliasMap.get(callerFile);
+    for (const importedFile of importedFiles) {
+        const lastSlash = importedFile.lastIndexOf('/');
+        const base = lastSlash >= 0 ? importedFile.slice(lastSlash + 1) : importedFile;
+        const dot = base.lastIndexOf('.');
+        const stem = dot >= 0 ? base.slice(0, dot) : base;
+        if (!stem)
+            continue;
+        if (!aliasMap) {
+            aliasMap = new Map();
+            ctx.moduleAliasMap.set(callerFile, aliasMap);
+        }
+        aliasMap.set(stem, importedFile);
+    }
+}

package/dist/core/ingestion/pipeline.d.ts

@@ -0,0 +1,49 @@
+/**
+ * Pipeline orchestrator — dependency-ordered ingestion pipeline.
+ *
+ * The pipeline is composed of named phases with explicit dependencies.
+ * Each phase is defined in its own file under `pipeline-phases/`.
+ * The runner in `pipeline-phases/runner.ts` executes phases in
+ * topological order, passing typed outputs from upstream phases as
+ * inputs to downstream phases.
+ *
+ * To add a new phase:
+ * 1. Create a new file in `pipeline-phases/` following the pattern
+ * 2. Export it from `pipeline-phases/index.ts`
+ * 3. Add it to the `ALL_PHASES` array below
+ *
+ * See ARCHITECTURE.md for the full phase dependency diagram.
+ */
+import { type PipelineProgress } from '../../_shared/index.js';
+import { PipelineResult } from '../../types/pipeline.js';
+export interface PipelineOptions {
+    /** Skip MRO, community detection, and process extraction for faster test runs. */
+    skipGraphPhases?: boolean;
+    /** Force sequential parsing (no worker pool). Useful for testing the sequential path. */
+    skipWorkers?: boolean;
+    /**
+     * @internal Test-only override for worker-pool gating thresholds.
+     * When unset, production defaults apply (15 files OR 512 KB total bytes).
+     * Setting either field lowers the corresponding threshold so small test
+     * fixtures can still exercise the worker-pool path. Do not use from
+     * production call sites.
+     */
+    workerThresholdsForTest?: {
+        minFiles?: number;
+        minBytes?: number;
+    };
+    /**
+     * Incremental-indexing parse cache. When provided:
+     *   - The parse phase looks up each chunk's content hash in
+     *     `parseCache.entries`. On hit, it replays the cached
+     *     `ParseWorkerResult[]` instead of dispatching to workers.
+     *   - On miss, it runs the workers as today and stores the new
+     *     results in `parseCache.entries` keyed by chunk hash.
+     * The caller (`run-analyze.ts`) is responsible for loading the cache
+     * before the pipeline runs and persisting it after. Cache survives
+     * `--force` because keys are content-addressed.
+     * See `arc/src/storage/parse-cache.ts`.
+     */
+    parseCache?: import('../../storage/parse-cache.js').ParseCache;
+}
+export declare const runPipelineFromRepo: (repoPath: string, onProgress: (progress: PipelineProgress) => void, options?: PipelineOptions) => Promise<PipelineResult>;

package/dist/core/ingestion/pipeline.js

@@ -0,0 +1,89 @@
+/**
+ * Pipeline orchestrator — dependency-ordered ingestion pipeline.
+ *
+ * The pipeline is composed of named phases with explicit dependencies.
+ * Each phase is defined in its own file under `pipeline-phases/`.
+ * The runner in `pipeline-phases/runner.ts` executes phases in
+ * topological order, passing typed outputs from upstream phases as
+ * inputs to downstream phases.
+ *
+ * To add a new phase:
+ * 1. Create a new file in `pipeline-phases/` following the pattern
+ * 2. Export it from `pipeline-phases/index.ts`
+ * 3. Add it to the `ALL_PHASES` array below
+ *
+ * See ARCHITECTURE.md for the full phase dependency diagram.
+ */
+import { createKnowledgeGraph } from '../graph/graph.js';
+import { runPipeline, getPhaseOutput, scanPhase, structurePhase, markdownPhase, cobolPhase, parsePhase, routesPhase, toolsPhase, ormPhase, crossFilePhase, scopeResolutionPhase, mroPhase, communitiesPhase, processesPhase, } from './pipeline-phases/index.js';
+// ── Phase registry ─────────────────────────────────────────────────────────
+/**
+ * All pipeline phases with their dependency relationships.
+ *
+ * Phase dependency graph:
+ *
+ *   scan → structure → [markdown, cobol] → parse → [routes, tools, orm]
+ *     → crossFile → mro → communities → processes
+ *
+ * To add a new phase: create a file in pipeline-phases/, export the phase
+ * object, and add it to the appropriate position in this array.
+ */
+function buildPhaseList(options) {
+    const phases = [
+        scanPhase,
+        structurePhase,
+        markdownPhase,
+        cobolPhase,
+        parsePhase,
+        routesPhase,
+        toolsPhase,
+        ormPhase,
+        crossFilePhase,
+        scopeResolutionPhase,
+    ];
+    if (!options?.skipGraphPhases) {
+        phases.push(mroPhase, communitiesPhase, processesPhase);
+    }
+    return phases;
+}
+// ── Pipeline orchestrator ─────────────────────────────────────────────────
+export const runPipelineFromRepo = async (repoPath, onProgress, options) => {
+    const graph = createKnowledgeGraph();
+    const pipelineStart = Date.now();
+    const phases = buildPhaseList(options);
+    const results = await runPipeline(phases, {
+        repoPath,
+        graph,
+        onProgress,
+        options,
+        pipelineStart,
+    });
+    // Extract final results for the PipelineResult contract
+    const { totalFiles, usedWorkerPool } = getPhaseOutput(results, 'parse');
+    let communityResult;
+    let processResult;
+    if (!options?.skipGraphPhases) {
+        communityResult = getPhaseOutput(results, 'communities').communityResult;
+        processResult = getPhaseOutput(results, 'processes').processResult;
+    }
+    onProgress({
+        phase: 'complete',
+        percent: 100,
+        message: communityResult && processResult
+            ? `Graph complete! ${communityResult.stats.totalCommunities} communities, ${processResult.stats.totalProcesses} processes detected.`
+            : 'Graph complete! (graph phases skipped)',
+        stats: {
+            filesProcessed: totalFiles,
+            totalFiles,
+            nodesCreated: graph.nodeCount,
+        },
+    });
+    return {
+        graph,
+        repoPath,
+        totalFileCount: totalFiles,
+        communityResult,
+        processResult,
+        usedWorkerPool,
+    };
+};

package/dist/core/ingestion/process-processor.d.ts

@@ -0,0 +1,51 @@
+/**
+ * Process Detection Processor
+ *
+ * Detects execution flows (Processes) in the code graph by:
+ * 1. Finding entry points (functions with no internal callers)
+ * 2. Tracing forward via CALLS edges (BFS)
+ * 3. Grouping and deduplicating similar paths
+ * 4. Labeling with heuristic names
+ *
+ * Processes help agents understand how features work through the codebase.
+ */
+import { KnowledgeGraph } from '../graph/types.js';
+import { CommunityMembership } from './community-processor.js';
+export interface ProcessDetectionConfig {
+    maxTraceDepth: number;
+    maxBranching: number;
+    maxProcesses: number;
+    minSteps: number;
+}
+export interface ProcessNode {
+    id: string;
+    label: string;
+    heuristicLabel: string;
+    processType: 'intra_community' | 'cross_community';
+    stepCount: number;
+    communities: string[];
+    entryPointId: string;
+    terminalId: string;
+    trace: string[];
+}
+export interface ProcessStep {
+    nodeId: string;
+    processId: string;
+    step: number;
+}
+export interface ProcessDetectionResult {
+    processes: ProcessNode[];
+    steps: ProcessStep[];
+    stats: {
+        totalProcesses: number;
+        crossCommunityCount: number;
+        avgStepCount: number;
+        entryPointsFound: number;
+    };
+}
+/**
+ * Detect processes (execution flows) in the knowledge graph
+ *
+ * This runs AFTER community detection, using CALLS edges to trace flows.
+ */
+export declare const processProcesses: (knowledgeGraph: KnowledgeGraph, memberships: CommunityMembership[], onProgress?: (message: string, progress: number) => void, config?: Partial<ProcessDetectionConfig>) => Promise<ProcessDetectionResult>;