@massa-ai/core 1.2.1

package/dist/services/etl/stage-context.d.ts

@@ -0,0 +1,153 @@
+/**
+ * ETL Stage Context & Shared Types
+ *
+ * Contracts shared across all 4 ETL stages:
+ *   discover → parse → resolve → load
+ *
+ * Each stage receives an EtlStageContext for progress reporting
+ * and passes its output as the next stage's input.
+ */
+import type { Chunk } from "../search/smart-chunker.js";
+import type { NormalizedStructure, ParseDiagnostic, SourceSpan, StructuralEdgeKind, StructuralSymbolKind } from "../structural/types.js";
+import type { GraphGenerationLease } from "../../data/graph-generation/graph-generation-contract.js";
+import type { HeaderLanguageEvidence } from "../structural/language-manifest.js";
+import type { FileCursor, ManagedRunLease } from "../../data/managed-runs/managed-run-contract.js";
+import type { ActiveGraphDiagnostics } from "@massa-ai/shared";
+export type EtlStage = "discover" | "parse" | "resolve" | "load";
+export interface EtlEvent {
+    type: "stage_start" | "stage_end" | "file_processed" | "file_error" | "progress";
+    stage: EtlStage;
+    payload: Record<string, unknown>;
+    timestamp: number;
+}
+export interface EtlStageContext {
+    projectId: string;
+    projectPath: string;
+    jobId: string;
+    graphGenerationLease?: GraphGenerationLease;
+    /**
+     * Wave 5 FR-09 / FR-10: managed_runs lease for this run. Present when the
+     * caller (IndexProjectTool or auto-reindex) acquired the lease; the Load
+     * stage uses it to persist a FileCursor after each file-batch commits so
+     * a kill/restart resumes from the next file.
+     */
+    managedRunLease?: ManagedRunLease;
+    abortSignal?: AbortSignal;
+    structuralHeaderEvidenceByFile?: Readonly<Record<string, HeaderLanguageEvidence>>;
+    /**
+     * Wave 5 FR-10: persisted FileCursor from the previous run. When present,
+     * the Discover stage skips files at-or-before the cursor (already-applied)
+     * so a kill/restart resumes from the next file. Set by the pipeline from
+     * {@link ManagedRunRepository.getActive} → file_cursor before stage 1.
+     */
+    resumeCursor?: FileCursor;
+    /** Hook for emitting progress events to the EventBus. */
+    emit: (event: EtlEvent) => void;
+}
+/** Output of Discover stage / Input of Parse stage */
+export interface DiscoveredFile {
+    absolutePath: string;
+    relativePath: string;
+    mtime: number;
+    size: number;
+    contentHash: string;
+    /** Immutable UTF-8 content captured during discovery for this generation. */
+    snapshotContent?: string;
+    /** True when content hash matches stored hash → skip parse/load. */
+    needsReparse: boolean;
+}
+/** A raw symbol extracted by the Parse stage before FQN resolution. */
+export interface RawSymbol {
+    kind: StructuralSymbolKind;
+    name: string;
+    /** Filled by Resolve stage: '{relativePath}#{name}' */
+    fqn?: string;
+    lineStart: number;
+    lineEnd: number;
+    exported: boolean;
+    docComment?: string;
+    span?: SourceSpan;
+}
+/** A raw import statement before path resolution. */
+export interface RawImport {
+    specifier: string;
+    names: string[];
+    isTypeOnly: boolean;
+    form?: "esm_import" | "esm_re_export" | "commonjs_require" | "dynamic_import" | "python_import" | "ruby_require" | "php_use" | "lua_require" | "c_include" | "cpp_include" | "go_import" | "rust_use" | "zig_import" | "java_import" | "java_static_import" | "kotlin_import" | "scala_import" | "csharp_using" | "swift_import" | "dart_import" | "elixir_alias" | "elixir_import" | "elixir_require" | "elixir_use" | "erlang_import" | "clojure_require" | "clojure_import" | "ocaml_open" | "ocaml_include" | "ocaml_module_alias" | "haskell_import";
+    span?: SourceSpan;
+    bindings?: readonly Readonly<{
+        imported: string;
+        local: string;
+        typeOnly: boolean;
+        arity?: number;
+    }>[];
+}
+/**
+ * A raw typed structural edge extracted by the Parse stage (D1).
+ *
+ * These are TS/JS best-effort call/control-flow edges resolved to FQNs in the
+ * Resolve stage and persisted as `symbol_references` rows with the new typed
+ * `ref_kind` values + `meta` JSON metadata.
+ *
+ * Edge types:
+ *   - call       : A() calls B() — caller → callee
+ *   - data_flow  : a value flows source → call-arg → callee-param (param binding)
+ *   - http_call  : fetch/axios/http/GraphQL/tRPC call site → route/URL/service
+ *   - emit       : emitter.emit('event', ...) — event producer
+ *   - listen     : emitter.on('event', ...) — event consumer
+ */
+export type RawEdgeKind = StructuralEdgeKind;
+export interface RawEdge {
+    kind: RawEdgeKind;
+    /** 1-based line where the edge originates. */
+    line: number;
+    /** The callee/target symbol name (e.g. 'fetch', 'bar', 'on'). */
+    symbolName: string;
+    /** Best-effort caller symbol (enclosing function/method name). */
+    callerSymbol?: string;
+    /** Typed-edge metadata. Resolved target_fqn is filled by Resolve stage. */
+    meta?: Record<string, unknown>;
+    span?: SourceSpan;
+    sourceFqn?: string;
+}
+/** Output of Parse stage / Input of Resolve stage */
+export interface ParsedFile {
+    file: DiscoveredFile;
+    chunks: Chunk[];
+    symbols: RawSymbol[];
+    rawImports: RawImport[];
+    rawEdges: RawEdge[];
+    /** Immutable native-query result retained after Tree-sitter objects are deleted. */
+    structure?: NormalizedStructure;
+    structuralDiagnostics?: readonly ParseDiagnostic[];
+    structuralDiagnosticCount?: number;
+    structuralRecovered?: boolean;
+}
+/** A resolved import with the concrete file path (or null if external). */
+export interface ResolvedImport {
+    raw: RawImport;
+    resolvedPath: string | null;
+    external: boolean;
+}
+/** A typed structural edge with its target FQN resolved where possible. */
+export interface ResolvedEdge extends RawEdge {
+    /** '{relativePath}#{symbolName}' if the callee/target was resolved, else undefined. */
+    targetFqn?: string;
+}
+/** Output of Resolve stage / Input of Load stage */
+export interface ResolvedFile extends ParsedFile {
+    resolvedImports: ResolvedImport[];
+    resolvedEdges: ResolvedEdge[];
+}
+export interface EtlResult extends ActiveGraphDiagnostics<string> {
+    filesDiscovered: number;
+    filesIndexed: number;
+    filesSkipped: number;
+    chunksIndexed: number;
+    symbolsIndexed: number;
+    errors: number;
+    durationMs: number;
+    stageTimings: Record<EtlStage, number>;
+}
+export type { ParserDiagnosticsSummary } from "@massa-ai/shared";
+//# sourceMappingURL=stage-context.d.ts.map

package/dist/services/etl/stage-context.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"stage-context.d.ts","sourceRoot":"","sources":["../../../src/services/etl/stage-context.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,4BAA4B,CAAC;AACxD,OAAO,KAAK,EACV,mBAAmB,EACnB,eAAe,EACf,UAAU,EACV,kBAAkB,EAClB,oBAAoB,EACrB,MAAM,wBAAwB,CAAC;AAChC,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,0DAA0D,CAAC;AACrG,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,oCAAoC,CAAC;AACjF,OAAO,KAAK,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,iDAAiD,CAAC;AACnG,OAAO,KAAK,EACV,sBAAsB,EAEvB,MAAM,kBAAkB,CAAC;AAI1B,MAAM,MAAM,QAAQ,GAAG,UAAU,GAAG,OAAO,GAAG,SAAS,GAAG,MAAM,CAAC;AAEjE,MAAM,WAAW,QAAQ;IACvB,IAAI,EACA,aAAa,GACb,WAAW,GACX,gBAAgB,GAChB,YAAY,GACZ,UAAU,CAAC;IACf,KAAK,EAAE,QAAQ,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjC,SAAS,EAAE,MAAM,CAAC;CACnB;AAID,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C;;;;;OAKG;IACH,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,8BAA8B,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,sBAAsB,CAAC,CAAC,CAAC;IAClF;;;;;OAKG;IACH,YAAY,CAAC,EAAE,UAAU,CAAC;IAC1B,yDAAyD;IACzD,IAAI,EAAE,CAAC,KAAK,EAAE,QAAQ,KAAK,IAAI,CAAC;CACjC;AAID,sDAAsD;AACtD,MAAM,WAAW,cAAc;IAC7B,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,6EAA6E;IAC7E,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,oEAAoE;IACpE,YAAY,EAAE,OAAO,CAAC;CACvB;AAED,uEAAuE;AACvE,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,oBAAoB,CAAC;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,uDAAuD;IACvD,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,OAAO,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,IAAI,CAAC,EAAE,UAAU,CAAC;CACnB;AAED,qDAAqD;AACrD,MAAM,WAAW,SAAS;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,UAAU,EAAE,OAAO,CAAC;IACpB,IAAI,CAAC,EAAE,YAAY,GAAG,eAAe,GAAG,kBAAkB,GAAG,gBAAgB,GAC3E,eAAe,GAAG,cAAc,GAAG,SAAS,GAAG,aAAa,GAC5D,WAAW,GAAG,aAAa,GAAG,WAAW,GAAG,UAAU,GAAG,YAAY,GACrE,aAAa,GAAG,oBAAoB,GAAG,eAAe,GAAG,cAAc,GAAG,cAAc,GAAG,cAAc,GAAG,aAAa,GACzH,cAAc,GAAG,eAAe,GAAG,gBAAgB,GAAG,YAAY,GAAG,eAAe,GACpF,iBAAiB,GAAG,gBAAgB,GAAG,YAAY,GAAG,eAAe,GAAG,oBAAoB,GAAG,gBAAgB,CAAC;IAClH,IAAI,CAAC,EAAE,UAAU,CAAC;IAClB,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC;QAC3B,QAAQ,EAAE,MAAM,CAAC;QACjB,KAAK,EAAE,MAAM,CAAC;QACd,QAAQ,EAAE,OAAO,CAAC;QAClB,KAAK,CAAC,EAAE,MAAM,CAAC;KAChB,CAAC,EAAE,CAAC;CACN;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,MAAM,WAAW,GAAG,kBAAkB,CAAC;AAE7C,MAAM,WAAW,OAAO;IACtB,IAAI,EAAE,WAAW,CAAC;IAClB,8CAA8C;IAC9C,IAAI,EAAE,MAAM,CAAC;IACb,iEAAiE;IACjE,UAAU,EAAE,MAAM,CAAC;IACnB,kEAAkE;IAClE,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,2EAA2E;IAC3E,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC/B,IAAI,CAAC,EAAE,UAAU,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,qDAAqD;AACrD,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,cAAc,CAAC;IACrB,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,OAAO,EAAE,SAAS,EAAE,CAAC;IACrB,UAAU,EAAE,SAAS,EAAE,CAAC;IACxB,QAAQ,EAAE,OAAO,EAAE,CAAC;IACpB,oFAAoF;IACpF,SAAS,CAAC,EAAE,mBAAmB,CAAC;IAChC,qBAAqB,CAAC,EAAE,SAAS,eAAe,EAAE,CAAC;IACnD,yBAAyB,CAAC,EAAE,MAAM,CAAC;IACnC,mBAAmB,CAAC,EAAE,OAAO,CAAC;CAC/B;AAED,2EAA2E;AAC3E,MAAM,WAAW,cAAc;IAC7B,GAAG,EAAE,SAAS,CAAC;IACf,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,QAAQ,EAAE,OAAO,CAAC;CACnB;AAED,2EAA2E;AAC3E,MAAM,WAAW,YAAa,SAAQ,OAAO;IAC3C,uFAAuF;IACvF,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,oDAAoD;AACpD,MAAM,WAAW,YAAa,SAAQ,UAAU;IAC9C,eAAe,EAAE,cAAc,EAAE,CAAC;IAClC,aAAa,EAAE,YAAY,EAAE,CAAC;CAC/B;AAID,MAAM,WAAW,SAAU,SAAQ,sBAAsB,CAAC,MAAM,CAAC;IAC/D,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;CACxC;AAED,YAAY,EAAE,wBAAwB,EAAE,MAAM,kBAAkB,CAAC"}

package/dist/services/etl/stage-context.js

@@ -0,0 +1,11 @@
+/**
+ * ETL Stage Context & Shared Types
+ *
+ * Contracts shared across all 4 ETL stages:
+ *   discover → parse → resolve → load
+ *
+ * Each stage receives an EtlStageContext for progress reporting
+ * and passes its output as the next stage's input.
+ */
+export {};
+//# sourceMappingURL=stage-context.js.map

package/dist/services/etl/stage-context.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"stage-context.js","sourceRoot":"","sources":["../../../src/services/etl/stage-context.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG"}

package/dist/services/etl/stages/discover.d.ts

@@ -0,0 +1,40 @@
+/**
+ * ETL Stage 1 — Discover
+ *
+ * Scans the project directory for indexable files.
+ * Computes SHA-256 content hashes and compares against stored hashes
+ * to skip files that haven't changed (fingerprint cache).
+ *
+ * Output order: high-centrality + recently-modified files first,
+ * so the Load stage populates the most important context earliest.
+ */
+import type { EtlStageContext, DiscoveredFile } from "../stage-context.js";
+export declare class DiscoverStage {
+    /**
+     * Run the discovery stage.
+     *
+     * @param ctx - Stage execution context (projectId, projectPath, jobId, emit)
+     * @param opts.forceReindex - When true, marks all files as needsReparse=true
+     * @param opts.filesToProcess - Explicit list of relative paths for incremental reindex
+     * @param opts.includeTests - When true, do NOT exclude test/benchmark files.
+     *   Builds a discover-local Ignore that omits the test globs from
+     *   DEFAULT_IGNORES; loadProjectIgnore itself is unchanged so query-time
+     *   callers (index-manager, contextual-search-rlm) still ignore tests.
+     */
+    run(ctx: EtlStageContext, opts?: {
+        forceReindex?: boolean;
+        filesToProcess?: string[];
+        includeTests?: boolean;
+    }): Promise<DiscoveredFile[]>;
+    private processFile;
+    /**
+     * Build the project Ignore. When includeTests is false (default), delegates
+     * to {@link loadProjectIgnore} unchanged (keeps query-time callers ignoring
+     * tests). When includeTests is true, constructs a discover-local Ignore that
+     * omits the test/benchmark globs from DEFAULT_IGNORES so test files are
+     * indexed but everything else (build artifacts, locks, generated, etc.)
+     * stays ignored. This does NOT mutate loadProjectIgnore.
+     */
+    private loadIgnore;
+}
+//# sourceMappingURL=discover.d.ts.map

package/dist/services/etl/stages/discover.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"discover.d.ts","sourceRoot":"","sources":["../../../../src/services/etl/stages/discover.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAUH,OAAO,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAyB3E,qBAAa,aAAa;IACxB;;;;;;;;;;OAUG;IACG,GAAG,CACP,GAAG,EAAE,eAAe,EACpB,IAAI,GAAE;QACJ,YAAY,CAAC,EAAE,OAAO,CAAC;QACvB,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;QAC1B,YAAY,CAAC,EAAE,OAAO,CAAC;KACnB,GACL,OAAO,CAAC,cAAc,EAAE,CAAC;YAiHd,WAAW;IAqCzB;;;;;;;OAOG;YACW,UAAU;CAsBzB"}

package/dist/services/etl/stages/discover.js

@@ -0,0 +1,213 @@
+/**
+ * ETL Stage 1 — Discover
+ *
+ * Scans the project directory for indexable files.
+ * Computes SHA-256 content hashes and compares against stored hashes
+ * to skip files that haven't changed (fingerprint cache).
+ *
+ * Output order: high-centrality + recently-modified files first,
+ * so the Load stage populates the most important context earliest.
+ */
+import { glob } from "glob";
+import fs from "fs/promises";
+import path from "path";
+import { createHash } from "crypto";
+import ignoreModule from "ignore";
+import { config, logger } from "@massa-ai/shared";
+import { getSymbolRepository } from "../../../data/symbol/symbol-repository-factory.js";
+import { DEFAULT_EXTENSIONS, DEFAULT_IGNORES, loadProjectIgnore } from "../../search/ignore-patterns.js";
+const ignore = ignoreModule.default ?? ignoreModule;
+// Test/benchmark glob patterns inside DEFAULT_IGNORES (ignore-patterns.ts:33-45).
+// When includeTests is true, the discover-local Ignore omits ONLY these so test
+// files are indexed; everything else in DEFAULT_IGNORES (build artifacts, locks,
+// generated, etc.) still applies. loadProjectIgnore itself is untouched so
+// query-time callers keep excluding tests.
+const TEST_IGNORE_PATTERNS = new Set([
+    "**/__tests__/**",
+    "**/tests/**",
+    "**/*.test.ts",
+    "**/*.test.tsx",
+    "**/*.test.js",
+    "**/*.test.jsx",
+    "**/*.spec.ts",
+    "**/*.spec.tsx",
+    "**/*.spec.js",
+    "**/*.spec.jsx",
+    "**/benchmarks/**",
+    "**/fixtures/**",
+]);
+export class DiscoverStage {
+    /**
+     * Run the discovery stage.
+     *
+     * @param ctx - Stage execution context (projectId, projectPath, jobId, emit)
+     * @param opts.forceReindex - When true, marks all files as needsReparse=true
+     * @param opts.filesToProcess - Explicit list of relative paths for incremental reindex
+     * @param opts.includeTests - When true, do NOT exclude test/benchmark files.
+     *   Builds a discover-local Ignore that omits the test globs from
+     *   DEFAULT_IGNORES; loadProjectIgnore itself is unchanged so query-time
+     *   callers (index-manager, contextual-search-rlm) still ignore tests.
+     */
+    async run(ctx, opts = {}) {
+        const t0 = performance.now();
+        ctx.emit({
+            type: "stage_start",
+            stage: "discover",
+            payload: { projectId: ctx.projectId, projectPath: ctx.projectPath },
+            timestamp: Date.now(),
+        });
+        const ig = await this.loadIgnore(ctx.projectPath, opts.includeTests ?? false);
+        const allowedExts = config.get("security").allowedExtensions ??
+            DEFAULT_EXTENSIONS;
+        let relPaths;
+        if (opts.filesToProcess && opts.filesToProcess.length > 0) {
+            relPaths = opts.filesToProcess;
+        }
+        else {
+            const pattern = `**/*{${allowedExts.join(",")}}`;
+            const found = await glob(pattern, {
+                cwd: ctx.projectPath,
+                nodir: true,
+                dot: false,
+                absolute: false,
+            });
+            // Ensure paths are relative (ignore library requires relative paths)
+            relPaths = found
+                .map((p) => (path.isAbsolute(p) ? path.relative(ctx.projectPath, p) : p))
+                .filter((p) => !ig.ignores(p));
+        }
+        // ── Wave 5 FR-10: FileCursor resume. If a cursor from a previous run
+        // is present, skip files at-or-before the cursor path (already-applied).
+        // The cursor's `path` is the LAST file successfully applied before the
+        // cursor was written; everything at-or-before it has been applied.
+        // `offset` is reserved for byte-level resume within a file (future
+        // work); at file-level granularity the cursor file is treated as fully
+        // applied (vectors upsert idempotently via deterministic doc ids, so a
+        // re-apply is safe, but we skip it to avoid redundant work). The filter
+        // applies to both glob-discovered and explicitly-passed `filesToProcess`
+        // so callers that pass an incremental hint still resume correctly.
+        // AC-24: kill mid-load leaves the cursor at the PREVIOUS file (the
+        // killed file's cursor write never commits), so restart re-processes
+        // the killed file (vectors upsert idempotently — partial re-apply safe).
+        if (ctx.resumeCursor?.path) {
+            const cursorPath = ctx.resumeCursor.path;
+            relPaths = relPaths.filter((rel) => rel.localeCompare(cursorPath) > 0);
+        }
+        // Load stored centrality scores for priority ordering
+        const centralityMap = await getSymbolRepository().getCentrality(ctx.projectId);
+        // Process files and compute fingerprints in parallel (batches of 30)
+        const discovered = [];
+        const BATCH = 30;
+        for (let i = 0; i < relPaths.length; i += BATCH) {
+            if (ctx.abortSignal?.aborted)
+                throw ctx.abortSignal.reason;
+            const batch = relPaths.slice(i, i + BATCH);
+            const results = await Promise.all(batch.map((rel) => this.processFile(ctx, rel, opts.forceReindex ?? false)));
+            for (const f of results) {
+                if (f)
+                    discovered.push(f);
+            }
+            ctx.emit({
+                type: "progress",
+                stage: "discover",
+                payload: {
+                    current: Math.min(i + BATCH, relPaths.length),
+                    total: relPaths.length,
+                    percentage: Math.round((Math.min(i + BATCH, relPaths.length) / relPaths.length) * 100),
+                },
+                timestamp: Date.now(),
+            });
+        }
+        // Sort: centrality desc → mtime desc → path asc
+        discovered.sort((a, b) => {
+            const ca = centralityMap.get(a.relativePath) ?? 0;
+            const cb = centralityMap.get(b.relativePath) ?? 0;
+            if (cb !== ca)
+                return cb - ca;
+            if (b.mtime !== a.mtime)
+                return b.mtime - a.mtime;
+            return a.relativePath.localeCompare(b.relativePath);
+        });
+        const durationMs = Math.round(performance.now() - t0);
+        ctx.emit({
+            type: "stage_end",
+            stage: "discover",
+            payload: {
+                total: discovered.length,
+                needsReparse: discovered.filter((f) => f.needsReparse).length,
+                skipped: discovered.filter((f) => !f.needsReparse).length,
+                durationMs,
+            },
+            timestamp: Date.now(),
+        });
+        logger.info("ETL Discover complete", {
+            projectId: ctx.projectId,
+            total: discovered.length,
+            needsReparse: discovered.filter((f) => f.needsReparse).length,
+            durationMs,
+        });
+        return discovered;
+    }
+    async processFile(ctx, relativePath, forceReindex) {
+        const absolutePath = path.join(ctx.projectPath, relativePath);
+        try {
+            const stat = await fs.stat(absolutePath);
+            const content = await fs.readFile(absolutePath, "utf-8");
+            const contentHash = createHash("sha256").update(content).digest("hex");
+            let needsReparse = forceReindex;
+            if (!forceReindex) {
+                const stored = await getSymbolRepository().getFile(ctx.projectId, relativePath);
+                needsReparse = !stored || stored.content_hash !== contentHash;
+            }
+            return {
+                absolutePath,
+                relativePath,
+                mtime: stat.mtimeMs,
+                size: stat.size,
+                contentHash,
+                snapshotContent: content,
+                needsReparse,
+            };
+        }
+        catch (err) {
+            logger.warn("DiscoverStage: failed to stat/read file", {
+                relativePath,
+                error: err.message,
+            });
+            throw new Error(`required_file_unreadable:${relativePath}:${err.message}`);
+        }
+    }
+    /**
+     * Build the project Ignore. When includeTests is false (default), delegates
+     * to {@link loadProjectIgnore} unchanged (keeps query-time callers ignoring
+     * tests). When includeTests is true, constructs a discover-local Ignore that
+     * omits the test/benchmark globs from DEFAULT_IGNORES so test files are
+     * indexed but everything else (build artifacts, locks, generated, etc.)
+     * stays ignored. This does NOT mutate loadProjectIgnore.
+     */
+    async loadIgnore(projectPath, includeTests) {
+        if (!includeTests)
+            return loadProjectIgnore(projectPath);
+        // Build a discover-local Ignore with test globs stripped.
+        const ig = ignore();
+        for (const pattern of DEFAULT_IGNORES) {
+            if (TEST_IGNORE_PATTERNS.has(pattern))
+                continue;
+            ig.add(pattern);
+        }
+        try {
+            const gitignorePath = path.join(projectPath, ".gitignore");
+            const gitignoreContent = await fs.readFile(gitignorePath, "utf8");
+            const rules = gitignoreContent
+                .split("\n")
+                .map((line) => line.trim())
+                .filter((line) => line && !line.startsWith("#"));
+            ig.add(rules);
+        }
+        catch {
+            // No .gitignore — defaults only.
+        }
+        return ig;
+    }
+}
+//# sourceMappingURL=discover.js.map

package/dist/services/etl/stages/discover.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"discover.js","sourceRoot":"","sources":["../../../../src/services/etl/stages/discover.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAE5B,OAAO,EAAE,MAAM,aAAa,CAAC;AAC7B,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,YAAY,MAAM,QAAQ,CAAC;AAClC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EAAE,mBAAmB,EAAE,MAAM,mDAAmD,CAAC;AAExF,OAAO,EAAE,kBAAkB,EAAE,eAAe,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAC;AAEzG,MAAM,MAAM,GAAI,YAA4D,CAAC,OAAO,IAAI,YAAY,CAAC;AAErG,kFAAkF;AAClF,gFAAgF;AAChF,iFAAiF;AACjF,2EAA2E;AAC3E,2CAA2C;AAC3C,MAAM,oBAAoB,GAAG,IAAI,GAAG,CAAC;IACnC,iBAAiB;IACjB,aAAa;IACb,cAAc;IACd,eAAe;IACf,cAAc;IACd,eAAe;IACf,cAAc;IACd,eAAe;IACf,cAAc;IACd,eAAe;IACf,kBAAkB;IAClB,gBAAgB;CACjB,CAAC,CAAC;AAEH,MAAM,OAAO,aAAa;IACxB;;;;;;;;;;OAUG;IACH,KAAK,CAAC,GAAG,CACP,GAAoB,EACpB,OAII,EAAE;QAEN,MAAM,EAAE,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAE7B,GAAG,CAAC,IAAI,CAAC;YACP,IAAI,EAAE,aAAa;YACnB,KAAK,EAAE,UAAU;YACjB,OAAO,EAAE,EAAE,SAAS,EAAE,GAAG,CAAC,SAAS,EAAE,WAAW,EAAE,GAAG,CAAC,WAAW,EAAE;YACnE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;SACtB,CAAC,CAAC;QAEH,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,WAAW,EAAE,IAAI,CAAC,YAAY,IAAI,KAAK,CAAC,CAAC;QAC9E,MAAM,WAAW,GACd,MAAM,CAAC,GAAG,CAAC,UAAU,CAA6B,CAAC,iBAA6B;YACjF,kBAAkB,CAAC;QAErB,IAAI,QAAkB,CAAC;QAEvB,IAAI,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1D,QAAQ,GAAG,IAAI,CAAC,cAAc,CAAC;QACjC,CAAC;aAAM,CAAC;YACN,MAAM,OAAO,GAAG,QAAQ,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YACjD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE;gBAChC,GAAG,EAAE,GAAG,CAAC,WAAW;gBACpB,KAAK,EAAE,IAAI;gBACX,GAAG,EAAE,KAAK;gBACV,QAAQ,EAAE,KAAK;aAChB,CAAC,CAAC;YACH,qEAAqE;YACrE,QAAQ,GAAG,KAAK;iBACb,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;iBACxE,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;QACnC,CAAC;QAED,sEAAsE;QACtE,yEAAyE;QACzE,uEAAuE;QACvE,mEAAmE;QACnE,mEAAmE;QACnE,uEAAuE;QACvE,uEAAuE;QACvE,wEAAwE;QACxE,yEAAyE;QACzE,mEAAmE;QACnE,mEAAmE;QACnE,qEAAqE;QACrE,yEAAyE;QACzE,IAAI,GAAG,CAAC,YAAY,EAAE,IAAI,EAAE,CAAC;YAC3B,MAAM,UAAU,GAAG,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC;YACzC,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,aAAa,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;QACzE,CAAC;QAED,sDAAsD;QACtD,MAAM,aAAa,GAAG,MAAM,mBAAmB,EAAE,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAE/E,qEAAqE;QACrE,MAAM,UAAU,GAAqB,EAAE,CAAC;QACxC,MAAM,KAAK,GAAG,EAAE,CAAC;QAEjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC;YAChD,IAAI,GAAG,CAAC,WAAW,EAAE,OAAO;gBAAE,MAAM,GAAG,CAAC,WAAW,CAAC,MAAM,CAAC;YAC3D,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,CAAC;YAC3C,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAC/B,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,CAAC,YAAY,IAAI,KAAK,CAAC,CAAC,CAC3E,CAAC;YACF,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;gBACxB,IAAI,CAAC;oBAAE,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC5B,CAAC;YAED,GAAG,CAAC,IAAI,CAAC;gBACP,IAAI,EAAE,UAAU;gBAChB,KAAK,EAAE,UAAU;gBACjB,OAAO,EAAE;oBACP,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC;oBAC7C,KAAK,EAAE,QAAQ,CAAC,MAAM;oBACtB,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC;iBACvF;gBACD,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;aACtB,CAAC,CAAC;QACL,CAAC;QAED,gDAAgD;QAChD,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YACvB,MAAM,EAAE,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YAClD,MAAM,EAAE,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YAClD,IAAI,EAAE,KAAK,EAAE;gBAAE,OAAO,EAAE,GAAG,EAAE,CAAC;YAC9B,IAAI,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,KAAK;gBAAE,OAAO,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;YAClD,OAAO,CAAC,CAAC,YAAY,CAAC,aAAa,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;QACtD,CAAC,CAAC,CAAC;QAEH,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;QAEtD,GAAG,CAAC,IAAI,CAAC;YACP,IAAI,EAAE,WAAW;YACjB,KAAK,EAAE,UAAU;YACjB,OAAO,EAAE;gBACP,KAAK,EAAE,UAAU,CAAC,MAAM;gBACxB,YAAY,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,MAAM;gBAC7D,OAAO,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,MAAM;gBACzD,UAAU;aACX;YACD,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;SACtB,CAAC,CAAC;QAEH,MAAM,CAAC,IAAI,CAAC,uBAAuB,EAAE;YACnC,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,KAAK,EAAE,UAAU,CAAC,MAAM;YACxB,YAAY,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,MAAM;YAC7D,UAAU;SACX,CAAC,CAAC;QAEH,OAAO,UAAU,CAAC;IACpB,CAAC;IAEO,KAAK,CAAC,WAAW,CACvB,GAAoB,EACpB,YAAoB,EACpB,YAAqB;QAErB,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,YAAY,CAAC,CAAC;QAE9D,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YACzC,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;YACzD,MAAM,WAAW,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAEvE,IAAI,YAAY,GAAG,YAAY,CAAC;YAEhC,IAAI,CAAC,YAAY,EAAE,CAAC;gBAClB,MAAM,MAAM,GAAG,MAAM,mBAAmB,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;gBAChF,YAAY,GAAG,CAAC,MAAM,IAAI,MAAM,CAAC,YAAY,KAAK,WAAW,CAAC;YAChE,CAAC;YAED,OAAO;gBACL,YAAY;gBACZ,YAAY;gBACZ,KAAK,EAAE,IAAI,CAAC,OAAO;gBACnB,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,WAAW;gBACX,eAAe,EAAE,OAAO;gBACxB,YAAY;aACb,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,IAAI,CAAC,yCAAyC,EAAE;gBACrD,YAAY;gBACZ,KAAK,EAAG,GAAa,CAAC,OAAO;aAC9B,CAAC,CAAC;YACH,MAAM,IAAI,KAAK,CAAC,4BAA4B,YAAY,IAAK,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC;QACxF,CAAC;IACH,CAAC;IAED;;;;;;;OAOG;IACK,KAAK,CAAC,UAAU,CAAC,WAAmB,EAAE,YAAqB;QACjE,IAAI,CAAC,YAAY;YAAE,OAAO,iBAAiB,CAAC,WAAW,CAAC,CAAC;QAEzD,0DAA0D;QAC1D,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC;QACpB,KAAK,MAAM,OAAO,IAAI,eAAe,EAAE,CAAC;YACtC,IAAI,oBAAoB,CAAC,GAAG,CAAC,OAAO,CAAC;gBAAE,SAAS;YAChD,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAClB,CAAC;QACD,IAAI,CAAC;YACH,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,YAAY,CAAC,CAAC;YAC3D,MAAM,gBAAgB,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;YAClE,MAAM,KAAK,GAAG,gBAAgB;iBAC3B,KAAK,CAAC,IAAI,CAAC;iBACX,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;iBAC1B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;YACnD,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAChB,CAAC;QAAC,MAAM,CAAC;YACP,iCAAiC;QACnC,CAAC;QACD,OAAO,EAAE,CAAC;IACZ,CAAC;CACF"}

package/dist/services/etl/stages/load.d.ts

@@ -0,0 +1,39 @@
+/**
+ * ETL Stage 4 — Load
+ *
+ * Persists ResolvedFile data in parallel to:
+ *   1. Vector store   — embedding chunks for semantic search
+ *   2. Keyword store  — lexical/trigram/fuzzy search over the same chunks
+ *   3. Symbol DB      — definitions, references, imports for graph navigation
+ *
+ * Each file is written atomically (PostgreSQL transaction per file).
+ * Updates the symbol_files fingerprint table on success.
+ */
+import type { SymbolDefinition, SymbolReference, SymbolImport } from "../../../data/symbol/symbol-repository-pg.js";
+import type { EtlStageContext, ResolvedFile } from "../stage-context.js";
+export interface LoadResult {
+    filesLoaded: number;
+    chunksLoaded: number;
+    symbolsLoaded: number;
+    errors: number;
+}
+export type LoadMode = "all" | "structural" | "semantic";
+export declare function buildSymbolPersistenceBatch(projectId: string, file: ResolvedFile, now?: number): {
+    definitions: SymbolDefinition[];
+    references: SymbolReference[];
+    imports: SymbolImport[];
+};
+export declare class LoadStage {
+    private vectorStore;
+    private keywordSearch;
+    private managedRunRepo;
+    private ensureVectorStore;
+    private ensureKeywordSearch;
+    private ensureManagedRunRepo;
+    run(ctx: EtlStageContext, files: ResolvedFile[], mode?: LoadMode, emitLifecycle?: boolean): Promise<LoadResult>;
+    /** Insert identical chunks into both semantic and lexical stores. */
+    private loadToSearchStores;
+    /** Insert symbols, references, and imports into the symbol DB. Returns symbol count. */
+    private loadToSymbolDb;
+}
+//# sourceMappingURL=load.d.ts.map

package/dist/services/etl/stages/load.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"load.d.ts","sourceRoot":"","sources":["../../../../src/services/etl/stages/load.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAOH,OAAO,KAAK,EACV,gBAAgB,EAChB,eAAe,EACf,YAAY,EACb,MAAM,8CAA8C,CAAC;AACtD,OAAO,KAAK,EACV,eAAe,EACf,YAAY,EAGb,MAAM,qBAAqB,CAAC;AAI7B,MAAM,WAAW,UAAU;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,MAAM,QAAQ,GAAG,KAAK,GAAG,YAAY,GAAG,UAAU,CAAC;AAczD,wBAAgB,2BAA2B,CACzC,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,YAAY,EAClB,GAAG,SAAa,GACf;IAAE,WAAW,EAAE,gBAAgB,EAAE,CAAC;IAAC,UAAU,EAAE,eAAe,EAAE,CAAC;IAAC,OAAO,EAAE,YAAY,EAAE,CAAA;CAAE,CAoD7F;AAED,qBAAa,SAAS;IACpB,OAAO,CAAC,WAAW,CAA2D;IAC9E,OAAO,CAAC,aAAa,CAAoD;IAGzE,OAAO,CAAC,cAAc,CAAuC;YAE/C,iBAAiB;IAO/B,OAAO,CAAC,mBAAmB;IAO3B,OAAO,CAAC,oBAAoB;IAKtB,GAAG,CAAC,GAAG,EAAE,eAAe,EAAE,KAAK,EAAE,YAAY,EAAE,EAAE,IAAI,GAAE,QAAgB,EAAE,aAAa,UAAO,GAAG,OAAO,CAAC,UAAU,CAAC;IAkMzH,qEAAqE;YACvD,kBAAkB;IAmChC,wFAAwF;YAC1E,cAAc;CAkD7B"}