arceus-s 1.6.4

package/dist/core/incremental/subgraph-extract.js

@@ -0,0 +1,111 @@
+/**
+ * Subgraph extraction for incremental DB writeback.
+ *
+ * Given the FULL ctx.graph produced by the pipeline (all files parsed,
+ * all phases run) and the set of file paths whose DB rows must be
+ * replaced, produce a smaller KnowledgeGraph that contains:
+ *
+ *   - Every node whose `properties.filePath` is in `toWriteSet`.
+ *   - Every graph-wide node (Community, Process) — these are regenerated
+ *     each run by the communities/processes phases and must be fully
+ *     rewritten.
+ *   - Every relationship where AT LEAST ONE endpoint is in the writable
+ *     set above. Relationships entirely between unchanged-file nodes
+ *     are skipped — their rows are still in the DB and re-inserting
+ *     them would PK-conflict at COPY time.
+ *
+ * The resulting subgraph is what gets passed to `loadGraphToLbug` after
+ * the orchestrator has deleted the corresponding DB rows. Hydrated
+ * unchanged-file rows are never touched in the DB.
+ *
+ * # Cross-file edge consistency (Finding 1)
+ *
+ * `extractChangedSubgraph` intentionally does NOT expand the set it is
+ * given — expansion is the orchestrator's job, so the SAME expanded set
+ * can be fed to both `deleteNodesForFile` and this function (asymmetry
+ * between the delete set and the write set silently corrupts the DB).
+ * `computeEffectiveWriteSet` below performs the boundary-crossing 1-hop
+ * walk; the orchestrator composes it with its importer-BFS expansion and
+ * passes the result here.
+ *
+ * Why the 1-hop walk is needed: consider a barrel re-export change —
+ * file C (a barrel) shifts `export { foo } from './b'` to
+ * `export { foo } from './d'`. After scope resolution, file A's CALLS
+ * edge to `foo` resolves to D instead of B, even though A's content is
+ * byte-for-byte identical:
+ *
+ *   - Old A→B edge survives in DB (neither A nor B is changed → not deleted)
+ *   - New A→D edge is missing (neither A nor D in writable set → skipped)
+ *
+ * Pulling the unchanged-side file of every writable-boundary-crossing
+ * edge into the write set fixes both halves: the orchestrator's
+ * `DETACH DELETE` cleans up the stale unchanged-side rows, and the new
+ * cross-file edges land because at least one endpoint is now writable.
+ *
+ * Limitation (documented): if a file X *stopped* importing from a
+ * changed file C, X has no edge to C in the new graph, so this 1-hop
+ * walk doesn't catch it. The orchestrator's importer-BFS (which reads
+ * IMPORTS from the pre-pipeline DB) covers that case instead.
+ */
+import { createKnowledgeGraph } from '../graph/graph.js';
+const isGraphWide = (label) => label === 'Community' || label === 'Process';
+/**
+ * Build a Map<nodeId, filePath> for every File-bound node in the graph.
+ * Graph-wide nodes (Community/Process) have no filePath and are filtered.
+ */
+const indexNodeFilePaths = (fullGraph) => {
+    const idx = new Map();
+    fullGraph.forEachNode((n) => {
+        const fp = n.properties?.filePath;
+        if (fp)
+            idx.set(n.id, fp);
+    });
+    return idx;
+};
+export const extractChangedSubgraph = (fullGraph, toWriteSet) => {
+    const sub = createKnowledgeGraph();
+    const writableNodeIds = new Set();
+    fullGraph.forEachNode((n) => {
+        const filePath = n.properties?.filePath;
+        const include = (filePath && toWriteSet.has(filePath)) || isGraphWide(n.label);
+        if (include) {
+            sub.addNode(n);
+            writableNodeIds.add(n.id);
+        }
+    });
+    fullGraph.forEachRelationship((r) => {
+        if (writableNodeIds.has(r.sourceId) || writableNodeIds.has(r.targetId)) {
+            sub.addRelationship(r);
+        }
+    });
+    return sub;
+};
+/**
+ * Public — derive the EFFECTIVE write-set: `toWriteSet` expanded by one
+ * hop along every edge in the new graph that crosses the writable
+ * boundary (one endpoint in a writable file, the other in an unchanged
+ * file). The unchanged-side file is pulled in so its stale rows are
+ * deleted + rewritten in lockstep with the changed side.
+ *
+ * Single pass over the edge list. Does NOT mutate `toWriteSet`. The
+ * orchestrator MUST feed the returned set to both `deleteNodesForFile`
+ * and `extractChangedSubgraph` — feeding the unexpanded set to either
+ * one leaves stale rows or PK-conflicts at COPY time.
+ */
+export const computeEffectiveWriteSet = (fullGraph, toWriteSet) => {
+    const nodeFilePaths = indexNodeFilePaths(fullGraph);
+    const expanded = new Set(toWriteSet);
+    fullGraph.forEachRelationship((r) => {
+        const sourcePath = nodeFilePaths.get(r.sourceId);
+        const targetPath = nodeFilePaths.get(r.targetId);
+        if (!sourcePath || !targetPath)
+            return; // skip edges to graph-wide nodes
+        const sourceWritable = toWriteSet.has(sourcePath);
+        const targetWritable = toWriteSet.has(targetPath);
+        if (sourceWritable && !targetWritable)
+            expanded.add(targetPath);
+        else if (targetWritable && !sourceWritable)
+            expanded.add(sourcePath);
+    });
+    return expanded;
+};

package/dist/core/ingestion/ast-cache.d.ts

@@ -0,0 +1,26 @@
+import Parser from 'tree-sitter';
+/**
+ * Minimal structural shape consumers need when reading Trees back
+ * through a phase-dependency boundary. Declared here so phases that
+ * receive ASTCache via `getPhaseOutput<...>` don't hand-roll their
+ * own inline structural types that silently drift when ASTCache's
+ * contract changes.
+ *
+ * Typed as `unknown` at the Tree boundary because consumers on the
+ * other side of the phase-output map don't share tree-sitter's type
+ * graph (e.g. COBOL's standalone processor).
+ */
+export interface ASTCacheReader {
+    get(filePath: string): unknown;
+    clear(): void;
+}
+export interface ASTCache extends ASTCacheReader {
+    get: (filePath: string) => Parser.Tree | undefined;
+    set: (filePath: string, tree: Parser.Tree) => void;
+    clear: () => void;
+    stats: () => {
+        size: number;
+        maxSize: number;
+    };
+}
+export declare const createASTCache: (maxSize?: number) => ASTCache;

package/dist/core/ingestion/ast-cache.js

@@ -0,0 +1,48 @@
+import { LRUCache } from 'lru-cache';
+import { logger } from '../logger.js';
+export const createASTCache = (maxSize = 50) => {
+    const effectiveMax = Math.max(maxSize, 1);
+    // Initialize the cache with a 'dispose' handler
+    // This is the magic: When an item is evicted (dropped), this runs automatically.
+    const cache = new LRUCache({
+        max: effectiveMax,
+        dispose: (tree) => {
+            try {
+                // NOTE: web-tree-sitter has tree.delete(); native tree-sitter
+                // trees are GC-managed and .delete is absent (no-op here).
+                //
+                // Single-owner invariant (load-bearing under WASM): a given
+                // Parser.Tree reference must live in AT MOST ONE ASTCache
+                // that disposes. The parse-phase chunk-local cache clears
+                // between chunks; the cross-phase `scopeTreeCache` (also an
+                // ASTCache today) holds the same Tree by reference. Under
+                // native tree-sitter this is benign (dispose is a no-op).
+                // If/when Arceus adopts web-tree-sitter for sequential
+                // parsing, the cross-phase cache must either (a) skip
+                // writing Trees that are already owned by a disposing cache,
+                // or (b) use tree.copy() per entry. Failing to pick one
+                // will hand freed memory to scope-resolution.
+                tree.delete?.();
+            }
+            catch (e) {
+                logger.warn({ e }, 'Failed to delete tree from WASM memory');
+            }
+        },
+    });
+    return {
+        get: (filePath) => {
+            const tree = cache.get(filePath);
+            return tree; // Returns undefined if not found
+        },
+        set: (filePath, tree) => {
+            cache.set(filePath, tree);
+        },
+        clear: () => {
+            cache.clear();
+        },
+        stats: () => ({
+            size: cache.size,
+            maxSize: effectiveMax,
+        }),
+    };
+};

package/dist/core/ingestion/binding-accumulator.d.ts

@@ -0,0 +1,212 @@
+/**
+ * BindingAccumulator — read-append-only accumulator that collects TypeEnv
+ * bindings across files in the Arceus analyzer pipeline.
+ *
+ * **Current behavior (both execution paths):** The accumulator carries only
+ * file-scope (`scope = ''`) entries. Function-scope bindings are stripped
+ * at both write sites:
+ *
+ * - **Worker path**: `parse-worker.ts` serializes only
+ *   `typeEnv.fileScope()` entries across the IPC boundary.
+ * - **Sequential path**: `type-env.ts::flush()` iterates only the FILE_SCOPE
+ *   entry of the env map and writes `BindingEntry` records with
+ *   `scope: ''` hardcoded.
+ *
+ * The narrowing exists because function-scope bindings have zero downstream
+ * consumers today and were previously costing ~4.9 MB of heap + IPC on
+ * every pipeline run. See `type-env.ts::flush()` and the `FileScopeBindings`
+ * JSDoc in `parse-worker.ts` for the paired Phase 9 reversion checklist.
+ *
+ * **Historical quality asymmetry (Phase 9 consideration):** Even though
+ * both paths now carry only file-scope data, the two paths were built
+ * under different resolution capabilities, and a future Phase 9 reverter
+ * that widens them back to all scopes will inherit that asymmetry:
+ *
+ * - **Sequential path** had (and would regain) access to the full
+ *   `SymbolTable` and `importedBindings`, so its bindings benefit from
+ *   Tier 2 cross-file propagation.
+ * - **Worker path** runs without `SymbolTable` / `importedBindings` and
+ *   can only produce Tier 0 (annotation-declared) and local Tier 1
+ *   (same-file constructor inference) bindings.
+ *
+ * Phase 9 consumers that trust every entry equally will silently produce
+ * worse results for large repos (worker-dominant) than small ones
+ * (sequential-dominant). If Phase 9 needs homogeneous quality, either
+ * (a) tag entries with their tier at insert time so consumers can filter,
+ * or (b) post-process worker-path entries through a follow-up resolution
+ * pass after the main-thread `SymbolTable` is complete.
+ *
+ * **Lifecycle contract**: single-use — `append* → finalize → consume → dispose`.
+ * After `dispose()` the accumulator is permanently dead: any mutating call
+ * (`appendFile`) throws, and read methods return empty/undefined as if the
+ * accumulator had never been appended to. The instance is not recyclable;
+ * construct a new one for a new pipeline run. Finalization and disposal are
+ * orthogonal state dimensions and may be invoked in either order.
+ */
+export interface BindingEntry {
+    readonly scope: string;
+    readonly varName: string;
+    readonly typeName: string;
+}
+/**
+ * Minimal graph-node shape required by `enrichExportedTypeMap()`. Intentionally
+ * narrower than the full `GraphNode` type in `graph/types.ts` so tests can
+ * construct a minimal mock without depending on the full graph module, and
+ * so the enrichment logic is a pure function over this contract.
+ *
+ * Matches the shape of the real `KnowledgeGraph` node's `properties.isExported`
+ * access path — tests that use a different shape silently pass while
+ * production fails.
+ */
+export interface EnrichmentGraphNode {
+    readonly id: string;
+    readonly properties?: {
+        readonly isExported?: boolean;
+    } | undefined;
+}
+/**
+ * Minimal graph lookup interface used by `enrichExportedTypeMap()`.
+ * Consumes only the method the enrichment loop actually calls.
+ */
+export interface EnrichmentGraphLookup {
+    getNode(id: string): EnrichmentGraphNode | undefined;
+}
+/**
+ * Merge file-scope bindings from a (finalized) `BindingAccumulator` into an
+ * `exportedTypeMap` for symbols whose graph nodes are marked as exported.
+ *
+ * This is the single source of truth for the worker-path ExportedTypeMap
+ * enrichment loop. Previously the logic lived inline in `pipeline.ts` and
+ * the test suite reimplemented it as a `runEnrichmentLoop` helper — a
+ * drift-prone pattern that meant tests could pass while production regressed.
+ * Extracting it here makes the production code call the same function the
+ * tests call.
+ *
+ * **Node ID candidate order**: `Function:{filePath}:{name}` →
+ * `Variable:{filePath}:{name}` → `Const:{filePath}:{name}`. First match wins.
+ *
+ * **Tier 0 priority**: if `exportedTypeMap` already has an entry for a
+ * `(filePath, name)` pair, the accumulator entry does NOT overwrite it —
+ * the SymbolTable tier-0 pass is authoritative. Without this guard, a
+ * worker-path binding could clobber a higher-quality type from SymbolTable.
+ *
+ * **Finalize precondition**: the accumulator should be finalized before
+ * calling this function. The lifecycle contract is
+ * `append → finalize → enrich → dispose`. Finalization is not asserted
+ * here (the test suite and pipeline both honor it separately), but any
+ * append happening concurrently with this enrichment would be a lifecycle
+ * bug at the caller level.
+ *
+ * @returns The number of new entries written into `exportedTypeMap`
+ *          (0 on empty accumulator or when every candidate was filtered
+ *          out by the export check or the Tier 0 guard).
+ */
+export declare function enrichExportedTypeMap(bindingAccumulator: BindingAccumulator, graph: EnrichmentGraphLookup, exportedTypeMap: Map<string, Map<string, string>>): number;
+export declare class BindingAccumulator {
+    private readonly _allByFile;
+    private readonly _fileScopeByFile;
+    private _totalBindings;
+    private _finalized;
+    private _disposed;
+    /**
+     * Append bindings for a file. Safe to call multiple times for the same file.
+     * Throws if the accumulator has been finalized. Skips if entries is empty.
+     *
+     * The `entries` parameter is `readonly` — this method never mutates the
+     * caller's array. Internally, the first `appendFile` call per filePath
+     * makes a defensive copy (`slice()`), and subsequent calls push into the
+     * accumulator's own storage.
+     */
+    appendFile(filePath: string, entries: readonly BindingEntry[]): void;
+    /** Lock the accumulator — no further appends. Idempotent. */
+    finalize(): void;
+    /**
+     * Release the accumulator's heap footprint. Clears both internal storage
+     * maps and resets `_totalBindings` to zero. Idempotent — calling twice
+     * is a no-op. Orthogonal to `finalize()` — calling `dispose()` does not
+     * change the finalized state.
+     *
+     * **Single-use lifecycle.** This is a one-way terminal transition: the
+     * accumulator is not recyclable. Any subsequent `appendFile` call throws
+     * (`'BindingAccumulator: use after dispose'`), regardless of whether
+     * `finalize()` was called first. Post-dispose reads do not throw —
+     * they return empty/undefined state matching a never-appended-to
+     * accumulator:
+     *   - `fileCount === 0`
+     *   - `totalBindings === 0`
+     *   - `files()` yields an empty iterator
+     *   - `getFile(x)` returns `undefined` for all `x`
+     *   - `fileScopeEntries(x)` returns `[]` for all `x`
+     *   - `fileScopeGet(x, y)` returns `undefined` for all `x, y`
+     *   - `estimateMemoryBytes()` returns `0`
+     *
+     * Lifecycle note: the pipeline disposes the accumulator inside the
+     * `finally` of the `crossFile` phase, which is scheduled after every
+     * other accumulator consumer (Phase 9 call/assignment processing and
+     * the ExportedTypeMap enrichment loop). The dispose call therefore
+     * runs once, on both the happy path and the throw path of the
+     * crossFile phase.
+     */
+    dispose(): void;
+    /** Get all bindings for a file, or undefined if the file is unknown. */
+    getFile(filePath: string): readonly BindingEntry[] | undefined;
+    /**
+     * Get only scope='' (file-level) entries as [varName, typeName] tuples.
+     * For iteration-based consumers (e.g., `enrichExportedTypeMap`).
+     * Returns an empty array for an unknown file.
+     *
+     * O(1) map lookup + O(n_file_scope) tuple reconstruction from the inner
+     * Map. Does NOT walk function-scope entries.
+     *
+     * For point-lookup consumers (e.g., Phase 9 fallback), prefer
+     * `fileScopeGet(filePath, name)` — O(1) with no allocation.
+     */
+    fileScopeEntries(filePath: string): readonly (readonly [string, string])[];
+    /**
+     * O(1) point-lookup for a single file-scope binding by (filePath, name).
+     * Returns the typeName if found, `undefined` otherwise.
+     *
+     * This is the preferred lookup path for Phase 9 consumers that resolve
+     * a single callee's return type — avoids the O(n_file_scope) iteration
+     * and defensive-copy allocation of `fileScopeEntries()`.
+     */
+    fileScopeGet(filePath: string, name: string): string | undefined;
+    /** Iterate over all file paths in insertion order. */
+    files(): IterableIterator<string>;
+    /** Number of distinct files with at least one binding. */
+    get fileCount(): number;
+    /** Total number of binding entries across all files. */
+    get totalBindings(): number;
+    /** Whether the accumulator has been finalized. */
+    get finalized(): boolean;
+    /**
+     * Whether the accumulator has been disposed. Exposed for symmetry with
+     * `finalized` so debug tooling and future Phase 9 consumers can detect a
+     * disposed accumulator without inspecting empty state heuristically.
+     *
+     * Disposal and finalization are orthogonal: a disposed accumulator may or
+     * may not be finalized, and vice versa. See `dispose()` for the full
+     * lifecycle contract.
+     */
+    get disposed(): boolean;
+    /**
+     * Rough memory estimate in bytes (intentionally pessimistic).
+     * Formula: sum of (ENTRY_OVERHEAD + char bytes of scope+varName+typeName) per entry
+     *          + MAP_ENTRY_OVERHEAD + char bytes of filePath per file.
+     *
+     * Note: V8 stores all-ASCII strings as Latin-1 (1 byte/char) and only upgrades
+     * to UCS-2 (2 bytes/char) for non-Latin-1 code points. Source paths and type names
+     * are typically all-ASCII, so actual heap cost is roughly half what this returns.
+     * The pessimistic factor is intentional — better to over-budget than under-budget.
+     *
+     * **⚠ Cost profile**: O(totalBindings) — iterates every entry in
+     * `_allByFile` and reads three string `.length` properties per entry.
+     * At a typical repo scale (10k files × ~20 file-scope bindings) this is
+     * ~200k property reads per call. Call at most once per pipeline run,
+     * NOT per file, per chunk, or per progress tick. The current single
+     * call site is the dev-mode telemetry log at the pipeline finalize
+     * seam. Adding a per-file-progress caller would silently make it
+     * quadratic in repo size.
+     */
+    estimateMemoryBytes(): number;
+}