@spikedpunch/mast 0.1.0

package/dist/indexer/index.js

@@ -0,0 +1,560 @@
+import { existsSync, readFileSync, statSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { sql } from 'kysely';
+import { CURRENT_SCHEMA_VERSION } from '../store/config.js';
+import { initLockMarkers, withLock } from '../store/lock.js';
+import { SqliteChunkStore } from '../store/sqliteChunkStore.js';
+import { openDatabase, readPragmaValue } from '../graph/db.js';
+import { populateFile, insertEdges, insertReExportFiles, removeDeletedFiles } from '../graph/populate.js';
+import { extractFile } from '../ast/extract.js';
+import { walkProject, buildManifest, diffManifest } from './walker.js';
+import { getImportResolver } from './import-resolver.js';
+/**
+ * Fresh best-effort mtime read (seconds), used wherever a value written to
+ * `files.mtime` or `file_manifest.json` must reflect disk state as observed
+ * at READ time rather than at walk time (F1/F12 — see the WHY-comments in
+ * {@link runIndex} for why the two can diverge once `structure.lock` no
+ * longer spans the whole run). Two call sites, two different reasons: the
+ * parse loop calls this IMMEDIATELY BEFORE `doExtract` so the stamp can never
+ * be newer than the content it accompanies (F12 invariant 1); the final
+ * manifest phase calls this with no accompanying content read at all, purely
+ * for `file_manifest.json` bookkeeping (see invariant 1's note on why that
+ * one doesn't need the same fix). Falls back to `fallbackSeconds` if the file
+ * has vanished since it was walked — mirrors `walkProject`'s own
+ * best-effort disappearance handling (walker.ts:63-65).
+ */
+function statMtimeSecondsOrFallback(absPath, fallbackSeconds) {
+    try {
+        return statSync(absPath).mtimeMs / 1_000;
+    }
+    catch {
+        return fallbackSeconds;
+    }
+}
+/**
+ * Walk the project, parse changed files into chunks, populate the graph
+ * database and FTS indexes.
+ *
+ * F1 (GITNEXUS_COMPARISON.md §13.2, MAST_SPEC.md:822-826): `structure.lock`
+ * is acquired and released separately for each write phase — deleted-file
+ * cleanup, each `LANCE_BATCH`-sized write batch, each edge-insertion batch,
+ * and the final manifest/index.json write — instead of once for the whole
+ * run. The read-only walk and the CPU-bound parse phase never hold the lock
+ * at all. Before this change the lock was held for the run's full duration
+ * (measured max: 280,782 ms on a 1,335-file corpus — eval/baseline-locks.json),
+ * which starved the JIT re-parse path (mcp/staleness.ts:56, 3 retries x
+ * 100 ms) for minutes at a time — directly contradicting MAST_SPEC.md:822-826
+ * ("lock holding is per-file-parse (10-50ms), not per-tool-call").
+ *
+ * Narrowing the lock's scope means a JIT re-parse can now interleave BETWEEN
+ * phases of an in-flight run. Two invariants keep that safe:
+ *
+ * 1. **[F12, `GITNEXUS_COMPARISON.md` Stage 1]** The row's stamped mtime must
+ *    never be NEWER than the content it is stamped against — i.e.
+ *    `stamp <= mtime-at-content-read`. This is an ORDERING property, not a
+ *    mutual-exclusion one: no lock can enforce it, because a file's editor
+ *    never takes `structure.lock`. An earlier version of this invariant
+ *    re-stat'd each file AFTER `doExtract` (inside the locked write phase) to
+ *    stop `files.mtime` *regressing* below `entry.mtime` — but that opened
+ *    the real hazard: an edit landing in the (unlocked, seconds-wide under
+ *    §15.2's FTS growth) parse-to-write window would be stamped with a mtime
+ *    NEWER than the content that was actually parsed, permanently disarming
+ *    staleness detection for that file (`diskMtime <= storedMtime` reads
+ *    "fresh" forever — the exact P0 class this stage exists to eliminate).
+ *    The fix stats each file IMMEDIATELY BEFORE `doExtract`, in the parse
+ *    loop below (unlocked, same as `mcp/staleness.ts`'s JIT path, which has
+ *    always done this correctly), and carries that pre-parse stamp through to
+ *    the write untouched — no re-stat at write time. A post-stat edit then
+ *    leaves stamp < content, so staleness re-fires on the next check and the
+ *    system self-heals, the same property the JIT path already had.
+ * 2. **Monotonic write-guard, `graph/populate.ts`'s `populateFile`**: even
+ *    with (1), two writers (e.g. this run's batch write and a concurrent JIT
+ *    refresh of the same file) can still race to write the SAME row with
+ *    DIFFERENT stamps, and `structure.lock` only serializes the two writes —
+ *    it does not guarantee the one with the fresher stamp wins. `populateFile`
+ *    refuses to replace a row whose stored mtime already exceeds the incoming
+ *    stamp, so whichever write carries the NEWER stamp wins regardless of
+ *    which one reaches the lock first — fixing the reindex-vs-JIT race
+ *    *better* than the lock alone does (the lock only serializes; the guard
+ *    orders).
+ *
+ * Known limitation (not solved here): mtime-granularity blindness. An edit
+ * landing in the SAME mtime tick as the stamp compares equal, not greater,
+ * and reads as fresh (the git racy-lstat problem). Near-moot on APFS's
+ * nanosecond-resolution mtimes; real on coarse-mtime volumes (e.g. some
+ * shared/network filesystems), which is exactly the container-deployment
+ * case MAST_SPEC.md §7.6 targets. Solving it properly means content hashing,
+ * not mtime comparison, and is out of scope for F12.
+ *
+ * `file_manifest.json` (which drives the NEXT run's incremental diff, see
+ * `diffManifest`) is rebuilt from a fresh re-stat of every file under the
+ * FINAL lock acquisition (below), not from the run-start walk snapshot or
+ * from the parse-phase stamps above. This one does NOT need invariant 1's
+ * fix: it is a stat with no accompanying content read in this final phase, so
+ * there is no stamp/content pair to invert. If it ever disagrees with
+ * `files.mtime` for a file JIT touched concurrently, that disagreement
+ * self-heals on the next JIT check (`checkAndRefreshIfStale` compares disk
+ * mtime, not the manifest, against `files.mtime`) — the manifest only feeds
+ * the NEXT run's `diffManifest`, and a stale manifest entry there causes an
+ * extra, harmless re-parse, never a missed one.
+ *
+ * The two-pass edge-insertion structure (`insertEdges` needs every file's
+ * symbols already present) is preserved: ALL pass-1 batches — across the
+ * whole run, not just the current batch — complete before pass 2 starts, so
+ * cross-file edge targets are always resolvable regardless of how finely
+ * pass 1's own lock acquisitions are chunked.
+ */
+export async function runIndex(config, options) {
+    const startMs = Date.now();
+    // initLockMarkers is idempotent — safe to call on every run so that
+    // `mast index` works even without a prior `mast init`.
+    initLockMarkers(config.resolved_state_dir);
+    const lockOptions = {
+        maxRetries: 5,
+        retryIntervalMs: 1_000,
+        caller: 'index-run',
+        sink: options.lockMetricsSink,
+    };
+    const doExtract = options.extractFileFn ?? extractFile;
+    // Opening the db handle is connection setup, not a graph/chunk-store
+    // mutation — no lock needed.
+    const db = openDatabase(config.resolved_state_dir, options.dbOptions ?? {});
+    // M1 (eval/GITNEXUS_COMPARISON.md §15.1): chunks live in graph.db's
+    // `chunks` table by default — `SqliteChunkStore` wraps the same `db`
+    // connection every write below already uses. `chunkStoreOverride` (test
+    // DI, §4.4) substitutes a fake for write-failure tests; see its docstring
+    // on `IndexOptions` and `graph/populate.ts`'s `ChunkWriter`.
+    const chunkStore = options.chunkStoreOverride ?? new SqliteChunkStore(db);
+    const chunkWriter = options.chunkStoreOverride
+        ? options.chunkStoreOverride.replaceChunksForFile.bind(options.chunkStoreOverride)
+        : undefined;
+    // Built with conditional spreads for the same reason `index-cmd.ts` builds
+    // `dbOptions` that way: `exactOptionalPropertyTypes` distinguishes an absent
+    // property from one explicitly set to undefined, and the production path
+    // depends on `spans` being genuinely absent so `populateFile` never takes a
+    // clock reading at all.
+    const populateOptions = {
+        ...(chunkWriter !== undefined ? { chunkWriter } : {}),
+        ...(options.writeSpans !== undefined ? { spans: options.writeSpans } : {}),
+        ...(options.unsafeSkipFtsDeletes === true ? { skipFtsDeletes: true } : {}),
+    };
+    // Load previous manifest for incremental comparison.
+    const manifestPath = join(config.resolved_state_dir, 'file_manifest.json');
+    const prevManifest = existsSync(manifestPath)
+        ? JSON.parse(readFileSync(manifestPath, 'utf-8'))
+        : {};
+    const currentFiles = await walkProject(config);
+    const { stale, added, deleted } = diffManifest(currentFiles, prevManifest);
+    let chunksRemoved = 0;
+    // Deleted-file cleanup — cascade removes symbols/edges/imports/chunks from
+    // the graph (one transaction, `removeDeletedFiles`). Its own short lock
+    // acquisition, separate from pass 1's per-batch locks.
+    await withLock(config.resolved_state_dir, 'structure', lockOptions, async () => {
+        chunksRemoved += await removeDeletedFiles(db, deleted);
+        // Full reindex: also purge DB entries for files no longer in the current
+        // walk (e.g. dist/ files left over from a previous indexing configuration
+        // that predates the current exclude_patterns). These are invisible to the
+        // manifest-diff above because they were never written to file_manifest.json
+        // under the current config.
+        if (!options.incremental) {
+            const currentPaths = new Set(currentFiles.map((e) => e.relativePath));
+            const dbRows = await db.selectFrom('files').select('path').execute();
+            const orphans = dbRows.map((r) => r.path).filter((p) => !currentPaths.has(p));
+            if (orphans.length > 0) {
+                chunksRemoved += await removeDeletedFiles(db, orphans);
+            }
+        }
+    });
+    // An incremental run's work set is the manifest diff PLUS any walked file
+    // with no `files` row. The manifest alone is not sufficient: a run that
+    // failed on a file before 2026-08-20 recorded it as up to date anyway
+    // (D034), so the diff reports it unchanged forever and the file never comes
+    // back. Without this, `mast_status` would correctly report such an index
+    // stale (`measureFreshness` reads the `files` stamps too) and every
+    // incremental run would do nothing about it — a state that never converges.
+    // On a full run every file is reindexed regardless, so this is scoped to the
+    // incremental path and costs one indexed `path` scan.
+    let toIndex;
+    if (options.incremental) {
+        const queued = new Set([...stale, ...added].map((e) => e.relativePath));
+        const indexedPaths = new Set((await db.selectFrom('files').select('path').execute()).map((r) => r.path));
+        const neverWritten = currentFiles.filter((e) => !queued.has(e.relativePath) && !indexedPaths.has(e.relativePath));
+        toIndex = [...stale, ...added, ...neverWritten];
+    }
+    else {
+        toIndex = currentFiles;
+    }
+    let filesIndexed = 0;
+    /**
+     * Files parsed and queued for a write. NOT `filesIndexed` — a queued file
+     * whose write then fails never lands, and conflating the two is what made
+     * `files: N indexed` and `files_indexed: N` overstate the index (D038).
+     * Kept solely so `onProgress` still advances during the parse phase, which
+     * is where the wall-clock is spent on a large batch.
+     */
+    let filesQueued = 0;
+    let parseErrors = 0;
+    let writeErrors = 0;
+    let staleWriteRejections = 0;
+    let chunksAdded = 0;
+    let filesStable = 0;
+    /**
+     * Relative paths this run attempted and did not get into the index — a parse
+     * that threw, or a write that failed. They are withheld from the manifest at
+     * finalise, so the next run's `diffManifest` sees them as new and retries
+     * them. Recording them as up-to-date is what made one transient failure a
+     * permanent hole (`docs/defects/LEDGER.md` D034).
+     */
+    const failedPaths = new Set();
+    // Pass 1: parse, then write (chunks + graph + FTS together via
+    // populateFile), processed in batches so a batch's worth of parse results
+    // doesn't sit in memory indefinitely. SQLite writes are sequential —
+    // better-sqlite3 serialises internally, and M1 (§15.1) removed the
+    // separate concurrent chunk-store write phase that existed only because
+    // the spike's chunk store was a different file/connection from graph.db.
+    // F1: `structure.lock` now wraps only the write half of each batch (below),
+    // not the parse phase and not the whole loop — see the runIndex WHY-comment.
+    const LANCE_BATCH = 16;
+    const edgeDataByFile = new Map();
+    // Phase accumulators. Summed across batches rather than sampled, because the parse/write
+    // interleave runs once per 16-file batch and a single stamp would capture only one of them.
+    const phase = { walk: 0, parse: 0, write: 0, edges: 0, finalise: 0 };
+    // Everything from `startMs` to here is the walk and manifest diff.
+    phase.walk = Date.now() - startMs;
+    for (let i = 0; i < toIndex.length; i += LANCE_BATCH) {
+        const batch = toIndex.slice(i, i + LANCE_BATCH);
+        const batchParseStart = Date.now();
+        // Parse phase — synchronous tree-sitter, cannot be parallelised without
+        // workers, and does not touch the graph db / chunk store, so it runs
+        // UNLOCKED. This is exactly the window in which a concurrent JIT
+        // re-parse of a not-yet-reached file can acquire structure.lock.
+        const parsed = [];
+        for (const entry of batch) {
+            try {
+                // F12: stat IMMEDIATELY BEFORE doExtract, not after — mirrors
+                // mcp/staleness.ts's JIT path (which has always stat'd before
+                // parsing). This is invariant 1 from the runIndex WHY-comment:
+                // `stamp <= mtime-at-content-read`. Stat'ing AFTER doExtract (the
+                // prior ordering) let a concurrent edit land in between, stamping
+                // this row with a mtime NEWER than the content that was actually
+                // parsed — permanently disarming staleness detection for the file
+                // (F12), and no lock can fix that ordering because editors never
+                // take structure.lock.
+                const preParseMtime = statMtimeSecondsOrFallback(entry.path, entry.mtime);
+                const result = doExtract(entry.path, config.resolved_project_root, config.context_lines, config.chunk_split_threshold, config.markdown_heading_depth);
+                // §7.1 stability skip: a file whose mtime changed but whose chunked
+                // content is identical (e.g. `git checkout` rewriting the same bytes)
+                // is re-parsed but need not be re-written. Incremental only; the
+                // check is conservative and bails to a full rewrite on any doubt.
+                if (options.incremental && await isFileUnchanged(db, chunkStore, entry.relativePath, result)) {
+                    filesStable++;
+                }
+                else {
+                    parsed.push({ entry, result, mtime: preParseMtime });
+                    // `filesIndexed` / `chunksAdded` are NOT incremented here. Both are
+                    // counted in the write loop below, once `populateFile` reports the
+                    // write actually landed — see D038.
+                    filesQueued++;
+                }
+            }
+            catch (err) {
+                process.stderr.write(`[mast] WARN: parse error in ${entry.path}: ${String(err)}\n`);
+                parseErrors++;
+                failedPaths.add(entry.relativePath);
+            }
+            options.onProgress?.(filesQueued + filesStable + parseErrors, toIndex.length);
+        }
+        phase.parse += Date.now() - batchParseStart;
+        const batchWriteStart = Date.now();
+        // E1-FTS `lock` span — see WriteSpansMs.lock. Acquisition is timed from
+        // here to the first statement of the callback, release from the callback's
+        // last statement to `withLock`'s return; both directly, neither derived.
+        const lockAcquireStart = options.writeSpans === undefined ? 0 : performance.now();
+        let lockBodyEnd = 0;
+        // Write phase — LOCKED, scoped to this batch only (F1). Bounds how long
+        // any caller (including a concurrent JIT re-parse) can be starved to
+        // roughly one batch's writes, not the whole run.
+        await withLock(config.resolved_state_dir, 'structure', lockOptions, async () => {
+            if (options.writeSpans !== undefined) {
+                options.writeSpans.lock += performance.now() - lockAcquireStart;
+            }
+            // populateFile now writes chunks + symbols + edges + imports + FTS in
+            // ONE per-file transaction (M1, §15.1) — a failure anywhere in it rolls
+            // back everything for that file, so there is no separate "chunk store
+            // write" step to run before this loop any more.
+            //
+            // A write failure is counted as `writeErrors`, NEVER `parseErrors` — the
+            // file parsed correctly; only its storage failed. Conflating the two
+            // (the original defect, GITNEXUS_COMPARISON.md §15.3 item 3) hid a
+            // storage failure behind a "parse error" label, which is the wrong
+            // diagnosis for anyone debugging it. Logged at ERROR (not WARN): the
+            // file's chunks/symbols/edges/imports/FTS rows are skipped entirely —
+            // this is not a soft warning.
+            //
+            // Deliberately NOT aborted and NOT quarantined per-chunk: one
+            // pathological file must not turn a monorepo index into a total outage,
+            // and partial-chunk recovery is out of scope for this fix. The file
+            // still fails to index — it just fails loudly and is correctly
+            // classified, instead of silently amputating the file under a WARN and
+            // a miscounted `parseErrors`.
+            for (const { entry, result, mtime } of parsed) {
+                // `mtime` is the PRE-parse stamp captured in the parse loop above
+                // (F12 invariant 1) — carried through untouched, NOT re-stat'd here.
+                // Re-statting at this point (the prior ordering) is exactly the bug:
+                // it reads disk state AFTER doExtract already read the content, so a
+                // concurrent edit landing in between gets stamped as newer-than-content.
+                try {
+                    const { chunksRemoved: removed, written } = await populateFile(db, {
+                        filePath: entry.relativePath,
+                        language: result.language,
+                        mtime,
+                        chunks: result.chunks,
+                        imports: result.imports,
+                        symbols: result.symbols,
+                        identifierRows: result.identifierRows,
+                    }, populateOptions);
+                    if (!written) {
+                        // Monotonic write-guard (F12 invariant 2, graph/populate.ts)
+                        // rejected this write: some other writer — a concurrent JIT
+                        // refresh, or a later batch in this same run reprocessing the
+                        // file — already stamped a NEWER mtime. The DB row is already at
+                        // least as fresh as what this parse would have written, so this
+                        // is correct behaviour, not a failure, but it must stay visible
+                        // rather than look like an ordinary successful write (the
+                        // `writeErrors` precedent).
+                        staleWriteRejections++;
+                        continue;
+                    }
+                    chunksRemoved += removed;
+                    // Counted HERE, not in the parse loop: this is the first point at
+                    // which the file is known to be in the index (D038). A stale-write
+                    // rejection `continue`s above and a write failure `continue`s below,
+                    // so neither reaches this line.
+                    filesIndexed++;
+                    chunksAdded += result.chunks.length;
+                }
+                catch (err) {
+                    process.stderr.write(`[mast] ERROR: chunk store write failed for ${entry.path} — file will be absent from the index: ${String(err)}\n`);
+                    writeErrors++;
+                    failedPaths.add(entry.relativePath);
+                    continue;
+                }
+                edgeDataByFile.set(entry.relativePath, result);
+            }
+            lockBodyEnd = performance.now();
+        });
+        if (options.writeSpans !== undefined) {
+            options.writeSpans.lock += performance.now() - lockBodyEnd;
+        }
+        phase.write += Date.now() - batchWriteStart;
+    }
+    // Pass 2: insert edges and star re-export rows now that all files' symbols
+    // and file rows exist (both may reference files indexed later in pass 1).
+    // Batched the same way as pass 1 and for the same reason (F1) — each batch
+    // gets its own short lock instead of one lock spanning every file's edges.
+    // Safe because ALL of pass 1 (every batch above) has already completed by
+    // the time this loop starts, so every cross-file edge target this run
+    // could produce already exists, independent of how pass 2's own lock
+    // acquisitions are chunked.
+    const edgeEntries = [...edgeDataByFile];
+    const edgesStart = Date.now();
+    for (let i = 0; i < edgeEntries.length; i += LANCE_BATCH) {
+        const batch = edgeEntries.slice(i, i + LANCE_BATCH);
+        await withLock(config.resolved_state_dir, 'structure', lockOptions, async () => {
+            for (const [filePath, data] of batch) {
+                await insertEdges(db, filePath, data.edges);
+                await insertReExportFiles(db, filePath, data.starReExports);
+            }
+        });
+    }
+    phase.edges = Date.now() - edgesStart;
+    const finaliseStart = Date.now();
+    // Final phase — manifest + index.json, under one more short lock. See
+    // invariant 2 in the runIndex WHY-comment for why this re-stats every file
+    // instead of reusing `currentFiles`' run-start walk snapshot, and why it
+    // needs the lock at all (to not race a JIT write that is already in flight).
+    await withLock(config.resolved_state_dir, 'structure', lockOptions, async () => {
+        // Files this run failed on are withheld: a manifest entry asserts "the
+        // index reflects this file at this mtime", which is false for them. Leaving
+        // the entry out makes the next run see the file as new and retry it, and —
+        // because a file with no manifest entry is also what `measureFreshness`
+        // counts as unindexed — keeps both status surfaces honest in the meantime.
+        // A file that was never attempted (unchanged on an incremental run) is not
+        // in `failedPaths` and keeps its entry, as it should.
+        const freshEntries = currentFiles
+            .filter((entry) => !failedPaths.has(entry.relativePath))
+            .map((entry) => ({
+            ...entry,
+            mtime: statMtimeSecondsOrFallback(entry.path, entry.mtime),
+        }));
+        const newManifest = buildManifest(freshEntries);
+        writeFileSync(manifestPath, JSON.stringify(newManifest, null, 2));
+        const chunkCount = await chunkStore.chunkCount();
+        const meta = {
+            schema_version: CURRENT_SCHEMA_VERSION,
+            last_indexed: new Date().toISOString(),
+            file_count: currentFiles.length,
+            chunk_count: chunkCount,
+            parse_errors: parseErrors > 0 ? parseErrors : undefined,
+            write_errors: writeErrors > 0 ? writeErrors : undefined,
+        };
+        writeFileSync(join(config.resolved_state_dir, 'index.json'), JSON.stringify(meta, null, 2));
+    });
+    phase.finalise = Date.now() - finaliseStart;
+    // Read the tuning pragmas back BEFORE the connection closes — they are
+    // connection-scoped state, not file state, and are unrecoverable afterwards.
+    const appliedPragmas = {
+        cache_size: readPragmaValue('cache_size', (await sql `PRAGMA cache_size`.execute(db)).rows),
+        mmap_size: readPragmaValue('mmap_size', (await sql `PRAGMA mmap_size`.execute(db)).rows),
+    };
+    await db.destroy();
+    // SqliteChunkStore (default path) wraps this same `db` connection (§15.1) —
+    // no separate handle to close. `chunkStoreOverride`, when supplied, is
+    // owned by the caller regardless.
+    return {
+        // Files skipped = never-parsed (mtime unchanged) + parsed-but-stable.
+        filesIndexed,
+        filesSkipped: currentFiles.length - toIndex.length + filesStable,
+        chunksAdded,
+        chunksRemoved,
+        parseErrors,
+        writeErrors,
+        staleWriteRejections,
+        // Drained, not read: the resolver is cached for the process lifetime and
+        // serves every run, so each run must clear what it reported.
+        miscasedImports: getImportResolver(config.resolved_project_root).drainMiscased(),
+        durationMs: Date.now() - startMs,
+        phaseMs: phase,
+        appliedPragmas,
+    };
+}
+/**
+ * True when `result` is equivalent to what is already stored for `filePath`, so
+ * re-writing it would be redundant (§7.1). Conservative:
+ *
+ *  - bails (returns false) if the file has any `block` chunk, whose content is
+ *    not captured by a symbol hash and so cannot be verified here;
+ *  - requires the exact same set of chunk ids (catches additions, removals, and
+ *    declarations that moved to a different line);
+ *  - requires every stored chunk's CONTENT to match byte-for-byte;
+ *  - requires every symbol's declaration_hash AND body_hash to match the stored
+ *    symbol of the same name+line;
+ *  - requires the stored `imports` rows to match module-for-module.
+ *
+ * Any mismatch — or any uncertainty — falls through to a full rewrite.
+ *
+ * D030 — WHY THE LAST TWO CHECKS EXIST. This function decides that a write can
+ * be skipped, so its equivalence check must cover everything the write covers.
+ * It originally compared chunk IDs and symbol hashes only, and neither moves
+ * when an edit lands OUTSIDE every symbol body. An `import` statement always
+ * does. `populateFile` writes the `imports` table, and a skipped file is also
+ * never added to `edgeDataByFile`, so pass 2 never re-resolves its edges —
+ * which made an import-only edit invisible in both places at once. It never
+ * self-healed: the finalise phase stamps the manifest from a fresh re-stat of
+ * EVERY walked file, written or not, so the next run did not see the file as
+ * stale either. Measured: after moving a declaration and repointing its
+ * importer, `mast index --incremental` reported `0 indexed, 2 skipped`, the
+ * `imports` row still read `./alpha.js` after that path had stopped existing,
+ * and `mast_callers` returned `verified_callers: []` for a function with a
+ * live caller.
+ *
+ * Chunk IDs deliberately do NOT change on a content edit (`dedupeChunkIds`,
+ * ast/extract.ts — `vectorKey` depends on that), so ID equality can never
+ * stand in for content equality here.
+ */
+async function isFileUnchanged(db, chunkStore, filePath, result) {
+    // `block` chunks carry no symbol hash, and `doc` chunks carry no symbols at
+    // all — neither can be content-verified here, so both bail to a full rewrite.
+    if (result.chunks.some((c) => c.chunk_type === 'block' || c.chunk_type === 'doc'))
+        return false;
+    // Structure: identical set of chunk ids, each carrying identical content.
+    // The content comparison is what catches an edit outside every symbol body
+    // — the import line, a top-level comment, a re-export (D030).
+    const storedChunks = await chunkStore.getChunksByFilePath(filePath);
+    if (storedChunks.length !== result.chunks.length)
+        return false;
+    const storedContentById = new Map(storedChunks.map((c) => [c.chunk_id, c.content]));
+    if (!result.chunks.every((c) => storedContentById.get(c.chunk_id) === c.content))
+        return false;
+    // Content: identical symbol hash signature.
+    const storedSymbols = await db
+        .selectFrom('symbols as s')
+        .innerJoin('files as f', 'f.id', 's.file_id')
+        .select(['s.name', 's.line', 's.declaration_hash', 's.body_hash'])
+        .where('f.path', '=', filePath)
+        .execute();
+    if (symbolSignature(result.symbols) !== symbolSignature(storedSymbols))
+        return false;
+    // Imports: an import statement can sit outside every chunk entirely (a file
+    // whose only chunks are its declarations, with `context_lines: 0`), so the
+    // chunk-content check above does not subsume this one.
+    const storedImports = await db
+        .selectFrom('imports as i')
+        .innerJoin('files as f', 'f.id', 'i.file_id')
+        .select(['i.module', 'i.symbols', 'i.is_external', 'i.resolved_path'])
+        .where('f.path', '=', filePath)
+        .execute();
+    return importSignature(result.imports) === importSignature(storedImports);
+}
+/**
+ * Order-independent signature of a file's import set, over exactly the columns
+ * `populateFile` writes to `imports`. The extracted and stored shapes differ
+ * (camelCase vs snake_case, boolean vs 0/1, array vs JSON text), so both are
+ * normalised to one canonical string rather than compared field by field.
+ */
+function importSignature(imports) {
+    return imports
+        .map((i) => {
+        const symbols = typeof i.symbols === 'string' ? i.symbols : JSON.stringify(i.symbols ?? []);
+        const external = i.isExternal ?? i.is_external === 1;
+        const resolved = i.resolvedPath ?? i.resolved_path ?? '';
+        return `${i.module}|${symbols}|${external ? 1 : 0}|${resolved}`;
+    })
+        .sort()
+        .join('\n');
+}
+/** Order-independent signature of a symbol set's identity + stability hashes. */
+function symbolSignature(symbols) {
+    return symbols
+        .map((s) => {
+        const decl = s.declarationHash ?? s.declaration_hash ?? '';
+        const body = s.bodyHash ?? s.body_hash ?? '';
+        return `${s.name}|${s.line}|${decl}|${body}`;
+    })
+        .sort()
+        .join('\n');
+}
+// ---------------------------------------------------------------------------
+// Freshness diagnostics (mast_status, `mast status`)
+// ---------------------------------------------------------------------------
+/**
+ * Derive the `freshness_cause` discriminator (§9 mast_status) from the stale
+ * file count. Stage 7.2 (IMPLEMENTATION_PLAN.md "Stage 7: Vector-store
+ * deletion") dropped the second parameter this used to take
+ * (`pendingEmbeddings`) along with the `'embedding_backlog'`/`'both'` cases —
+ * the Phase 2 embedder that could produce a backlog distinct from Phase 1
+ * staleness was excised in Stage 7.1, so a two-cause signature asserted a
+ * distinction the code could no longer draw.
+ */
+export function freshnessCause(staleFiles) {
+    return staleFiles > 0 ? 'phase1_stale' : null;
+}
+/**
+ * Load `index.json` from the state directory, or return null if absent.
+ *
+ * Unvalidated cast, not a zod parse — an `index.json` written before Stage
+ * 7.2 (IMPLEMENTATION_PLAN.md "Stage 7: Vector-store deletion") may still
+ * carry the removed `model` field; it rides along as an untyped extra key
+ * and is simply never read, so old files load without error.
+ */
+export function loadIndexMeta(stateDir) {
+    const metaPath = join(stateDir, 'index.json');
+    if (!existsSync(metaPath))
+        return null;
+    return JSON.parse(readFileSync(metaPath, 'utf-8'));
+}
+/** Write `index.json` to the state directory. */
+export function writeIndexMeta(stateDir, meta) {
+    writeFileSync(join(stateDir, 'index.json'), JSON.stringify(meta, null, 2));
+}
+//# sourceMappingURL=index.js.map

package/dist/indexer/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/indexer/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAC5E,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,GAAG,EAAE,MAAM,QAAQ,CAAC;AAE7B,OAAO,EAAE,sBAAsB,EAAE,MAAM,oBAAoB,CAAC;AAC5D,OAAO,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAE7D,OAAO,EAAE,gBAAgB,EAAmB,MAAM,8BAA8B,CAAC;AACjF,OAAO,EAAE,YAAY,EAAE,eAAe,EAA4B,MAAM,gBAAgB,CAAC;AACzF,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAE1G,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,WAAW,EAAE,aAAa,EAAE,YAAY,EAAkB,MAAM,aAAa,CAAC;AACvF,OAAO,EAAE,iBAAiB,EAA6B,MAAM,sBAAsB,CAAC;AAsJpF;;;;;;;;;;;;;GAaG;AACH,SAAS,0BAA0B,CAAC,OAAe,EAAE,eAAuB;IAC1E,IAAI,CAAC;QACH,OAAO,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,GAAG,KAAK,CAAC;IAC3C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,eAAe,CAAC;IACzB,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAwEG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,MAAsB,EACtB,OAAqB;IAErB,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC3B,oEAAoE;IACpE,uDAAuD;IACvD,eAAe,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC;IAE3C,MAAM,WAAW,GAAG;QAClB,UAAU,EAAE,CAAC;QACb,eAAe,EAAE,KAAK;QACtB,MAAM,EAAE,WAAW;QACnB,IAAI,EAAE,OAAO,CAAC,eAAe;KAC9B,CAAC;IACF,MAAM,SAAS,GAAG,OAAO,CAAC,aAAa,IAAI,WAAW,CAAC;IAEvD,qEAAqE;IACrE,6BAA6B;IAC7B,MAAM,EAAE,GAAG,YAAY,CAAC,MAAM,CAAC,kBAAkB,EAAE,OAAO,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC;IAE5E,oEAAoE;IACpE,qEAAqE;IACrE,wEAAwE;IACxE,0EAA0E;IAC1E,6DAA6D;IAC7D,MAAM,UAAU,GAAe,OAAO,CAAC,kBAAkB,IAAI,IAAI,gBAAgB,CAAC,EAAE,CAAC,CAAC;IACtF,MAAM,WAAW,GAAG,OAAO,CAAC,kBAAkB;QAC5C,CAAC,CAAC,OAAO,CAAC,kBAAkB,CAAC,oBAAoB,CAAC,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC;QAClF,CAAC,CAAC,SAAS,CAAC;IAEd,2EAA2E;IAC3E,6EAA6E;IAC7E,yEAAyE;IACzE,4EAA4E;IAC5E,wBAAwB;IACxB,MAAM,eAAe,GAAwB;QAC3C,GAAG,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACrD,GAAG,CAAC,OAAO,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1E,GAAG,CAAC,OAAO,CAAC,oBAAoB,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC3E,CAAC;IAEF,qDAAqD;IACrD,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,kBAAkB,EAAE,oBAAoB,CAAC,CAAC;IAC3E,MAAM,YAAY,GAA2B,UAAU,CAAC,YAAY,CAAC;QACnE,CAAC,CAAE,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,YAAY,EAAE,OAAO,CAAC,CAA4B;QAC7E,CAAC,CAAC,EAAE,CAAC;IAEP,MAAM,YAAY,GAAG,MAAM,WAAW,CAAC,MAAM,CAAC,CAAC;IAC/C,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,YAAY,CAAC,YAAY,EAAE,YAAY,CAAC,CAAC;IAE3E,IAAI,aAAa,GAAG,CAAC,CAAC;IAEtB,2EAA2E;IAC3E,wEAAwE;IACxE,uDAAuD;IACvD,MAAM,QAAQ,CAAC,MAAM,CAAC,kBAAkB,EAAE,WAAW,EAAE,WAAW,EAAE,KAAK,IAAI,EAAE;QAC7E,aAAa,IAAI,MAAM,kBAAkB,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;QAEvD,yEAAyE;QACzE,0EAA0E;QAC1E,0EAA0E;QAC1E,4EAA4E;QAC5E,4BAA4B;QAC5B,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;YACzB,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC;YACtE,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,CAAC;YACrE,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YAC9E,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvB,aAAa,IAAI,MAAM,kBAAkB,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;YACzD,CAAC;QACH,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,0EAA0E;IAC1E,wEAAwE;IACxE,sEAAsE;IACtE,4EAA4E;IAC5E,yEAAyE;IACzE,oEAAoE;IACpE,4EAA4E;IAC5E,6EAA6E;IAC7E,sDAAsD;IACtD,IAAI,OAAoB,CAAC;IACzB,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;QACxB,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,KAAK,EAAE,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC;QACxE,MAAM,YAAY,GAAG,IAAI,GAAG,CAC1B,CAAC,MAAM,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAC3E,CAAC;QACF,MAAM,YAAY,GAAG,YAAY,CAAC,MAAM,CACtC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,CACxE,CAAC;QACF,OAAO,GAAG,CAAC,GAAG,KAAK,EAAE,GAAG,KAAK,EAAE,GAAG,YAAY,CAAC,CAAC;IAClD,CAAC;SAAM,CAAC;QACN,OAAO,GAAG,YAAY,CAAC;IACzB,CAAC;IAED,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB;;;;;;OAMG;IACH,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,oBAAoB,GAAG,CAAC,CAAC;IAC7B,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB;;;;;;OAMG;IACH,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;IAEtC,+DAA+D;IAC/D,0EAA0E;IAC1E,qEAAqE;IACrE,mEAAmE;IACnE,wEAAwE;IACxE,yEAAyE;IACzE,4EAA4E;IAC5E,6EAA6E;IAC7E,MAAM,WAAW,GAAG,EAAE,CAAC;IAEvB,MAAM,cAAc,GAAG,IAAI,GAAG,EAA0C,CAAC;IAEzE,yFAAyF;IACzF,4FAA4F;IAC5F,MAAM,KAAK,GAAG,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;IACrE,mEAAmE;IACnE,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;IAElC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,WAAW,EAAE,CAAC;QACrD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,CAAC;QAChD,MAAM,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEnC,wEAAwE;QACxE,qEAAqE;QACrE,iEAAiE;QACjE,iEAAiE;QACjE,MAAM,MAAM,GAAiB,EAAE,CAAC;QAChC,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;YAC1B,IAAI,CAAC;gBACH,8DAA8D;gBAC9D,8DAA8D;gBAC9D,+DAA+D;gBAC/D,kEAAkE;gBAClE,kEAAkE;gBAClE,iEAAiE;gBACjE,kEAAkE;gBAClE,iEAAiE;gBACjE,uBAAuB;gBACvB,MAAM,aAAa,GAAG,0BAA0B,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;gBAC1E,MAAM,MAAM,GAAG,SAAS,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,CAAC,qBAAqB,EAAE,MAAM,CAAC,aAAa,EAAE,MAAM,CAAC,qBAAqB,EAAE,MAAM,CAAC,sBAAsB,CAAC,CAAC;gBACtJ,oEAAoE;gBACpE,sEAAsE;gBACtE,iEAAiE;gBACjE,kEAAkE;gBAClE,IAAI,OAAO,CAAC,WAAW,IAAI,MAAM,eAAe,CAAC,EAAE,EAAE,UAAU,EAAE,KAAK,CAAC,YAAY,EAAE,MAAM,CAAC,EAAE,CAAC;oBAC7F,WAAW,EAAE,CAAC;gBAChB,CAAC;qBAAM,CAAC;oBACN,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,aAAa,EAAE,CAAC,CAAC;oBACrD,oEAAoE;oBACpE,mEAAmE;oBACnE,oCAAoC;oBACpC,WAAW,EAAE,CAAC;gBAChB,CAAC;YACH,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,+BAA+B,KAAK,CAAC,IAAI,KAAK,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBACpF,WAAW,EAAE,CAAC;gBACd,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;YACtC,CAAC;YACD,OAAO,CAAC,UAAU,EAAE,CAAC,WAAW,GAAG,WAAW,GAAG,WAAW,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QAChF,CAAC;QAED,KAAK,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,eAAe,CAAC;QAC5C,MAAM,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACnC,wEAAwE;QACxE,2EAA2E;QAC3E,yEAAyE;QACzE,MAAM,gBAAgB,GAAG,OAAO,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;QAClF,IAAI,WAAW,GAAG,CAAC,CAAC;QAEpB,wEAAwE;QACxE,qEAAqE;QACrE,iDAAiD;QACjD,MAAM,QAAQ,CAAC,MAAM,CAAC,kBAAkB,EAAE,WAAW,EAAE,WAAW,EAAE,KAAK,IAAI,EAAE;YAC7E,IAAI,OAAO,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;gBACrC,OAAO,CAAC,UAAU,CAAC,IAAI,IAAI,WAAW,CAAC,GAAG,EAAE,GAAG,gBAAgB,CAAC;YAClE,CAAC;YACD,sEAAsE;YACtE,wEAAwE;YACxE,sEAAsE;YACtE,gDAAgD;YAChD,EAAE;YACF,yEAAyE;YACzE,qEAAqE;YACrE,mEAAmE;YACnE,mEAAmE;YACnE,qEAAqE;YACrE,sEAAsE;YACtE,8BAA8B;YAC9B,EAAE;YACF,8DAA8D;YAC9D,wEAAwE;YACxE,oEAAoE;YACpE,+DAA+D;YAC/D,uEAAuE;YACvE,8BAA8B;YAC9B,KAAK,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,MAAM,EAAE,CAAC;gBAC9C,kEAAkE;gBAClE,qEAAqE;gBACrE,qEAAqE;gBACrE,qEAAqE;gBACrE,yEAAyE;gBACzE,IAAI,CAAC;oBACH,MAAM,EAAE,aAAa,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,MAAM,YAAY,CAAC,EAAE,EAAE;wBACjE,QAAQ,EAAE,KAAK,CAAC,YAAY;wBAC5B,QAAQ,EAAE,MAAM,CAAC,QAAQ;wBACzB,KAAK;wBACL,MAAM,EAAE,MAAM,CAAC,MAAM;wBACrB,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,cAAc,EAAE,MAAM,CAAC,cAAc;qBACtC,EAAE,eAAe,CAAC,CAAC;oBACpB,IAAI,CAAC,OAAO,EAAE,CAAC;wBACb,6DAA6D;wBAC7D,4DAA4D;wBAC5D,8DAA8D;wBAC9D,iEAAiE;wBACjE,gEAAgE;wBAChE,gEAAgE;wBAChE,0DAA0D;wBAC1D,4BAA4B;wBAC5B,oBAAoB,EAAE,CAAC;wBACvB,SAAS;oBACX,CAAC;oBACD,aAAa,IAAI,OAAO,CAAC;oBACzB,kEAAkE;oBAClE,mEAAmE;oBACnE,qEAAqE;oBACrE,gCAAgC;oBAChC,YAAY,EAAE,CAAC;oBACf,WAAW,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC;gBACtC,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,8CAA8C,KAAK,CAAC,IAAI,0CAA0C,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;oBACxI,WAAW,EAAE,CAAC;oBACd,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;oBACpC,SAAS;gBACX,CAAC;gBACD,cAAc,CAAC,GAAG,CAAC,KAAK,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;YACjD,CAAC;YACD,WAAW,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAClC,CAAC,CAAC,CAAC;QACH,IAAI,OAAO,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;YACrC,OAAO,CAAC,UAAU,CAAC,IAAI,IAAI,WAAW,CAAC,GAAG,EAAE,GAAG,WAAW,CAAC;QAC7D,CAAC;QACD,KAAK,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,eAAe,CAAC;IAC9C,CAAC;IAED,2EAA2E;IAC3E,0EAA0E;IAC1E,2EAA2E;IAC3E,2EAA2E;IAC3E,0EAA0E;IAC1E,sEAAsE;IACtE,qEAAqE;IACrE,4BAA4B;IAC5B,MAAM,WAAW,GAAG,CAAC,GAAG,cAAc,CAAC,CAAC;IACxC,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC,IAAI,WAAW,EAAE,CAAC;QACzD,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,CAAC;QACpD,MAAM,QAAQ,CAAC,MAAM,CAAC,kBAAkB,EAAE,WAAW,EAAE,WAAW,EAAE,KAAK,IAAI,EAAE;YAC7E,KAAK,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,IAAI,KAAK,EAAE,CAAC;gBACrC,MAAM,WAAW,CAAC,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;gBAC5C,MAAM,mBAAmB,CAAC,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC;YAC9D,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,CAAC;IACtC,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAEjC,sEAAsE;IACtE,2EAA2E;IAC3E,yEAAyE;IACzE,6EAA6E;IAC7E,MAAM,QAAQ,CAAC,MAAM,CAAC,kBAAkB,EAAE,WAAW,EAAE,WAAW,EAAE,KAAK,IAAI,EAAE;QAC7E,uEAAuE;QACvE,4EAA4E;QAC5E,2EAA2E;QAC3E,wEAAwE;QACxE,2EAA2E;QAC3E,2EAA2E;QAC3E,sDAAsD;QACtD,MAAM,YAAY,GAAgB,YAAY;aAC3C,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;aACvD,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YACf,GAAG,KAAK;YACR,KAAK,EAAE,0BAA0B,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC;SAC3D,CAAC,CAAC,CAAC;QACN,MAAM,WAAW,GAAG,aAAa,CAAC,YAAY,CAAC,CAAC;QAChD,aAAa,CAAC,YAAY,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAElE,MAAM,UAAU,GAAG,MAAM,UAAU,CAAC,UAAU,EAAE,CAAC;QACjD,MAAM,IAAI,GAAc;YACtB,cAAc,EAAE,sBAAsB;YACtC,YAAY,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACtC,UAAU,EAAE,YAAY,CAAC,MAAM;YAC/B,WAAW,EAAE,UAAU;YACvB,YAAY,EAAE,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS;YACvD,YAAY,EAAE,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS;SACxD,CAAC;QACF,aAAa,CACX,IAAI,CAAC,MAAM,CAAC,kBAAkB,EAAE,YAAY,CAAC,EAC7C,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAC9B,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,aAAa,CAAC;IAE5C,uEAAuE;IACvE,6EAA6E;IAC7E,MAAM,cAAc,GAAG;QACrB,UAAU,EAAE,eAAe,CACzB,YAAY,EACZ,CAAC,MAAM,GAAG,CAAyB,mBAAmB,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CACzE;QACD,SAAS,EAAE,eAAe,CACxB,WAAW,EACX,CAAC,MAAM,GAAG,CAAyB,kBAAkB,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CACxE;KACF,CAAC;IAEF,MAAM,EAAE,CAAC,OAAO,EAAE,CAAC;IACnB,4EAA4E;IAC5E,uEAAuE;IACvE,kCAAkC;IAElC,OAAO;QACL,sEAAsE;QACtE,YAAY;QACZ,YAAY,EAAE,YAAY,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,WAAW;QAChE,WAAW;QACX,aAAa;QACb,WAAW;QACX,WAAW;QACX,oBAAoB;QACpB,yEAAyE;QACzE,6DAA6D;QAC7D,eAAe,EAAE,iBAAiB,CAAC,MAAM,CAAC,qBAAqB,CAAC,CAAC,aAAa,EAAE;QAChF,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;QAChC,OAAO,EAAE,KAAK;QACd,cAAc;KACf,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,KAAK,UAAU,eAAe,CAC5B,EAAmC,EACnC,UAAsB,EACtB,QAAgB,EAChB,MAAsC;IAEtC,4EAA4E;IAC5E,8EAA8E;IAC9E,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,KAAK,OAAO,IAAI,CAAC,CAAC,UAAU,KAAK,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAEhG,0EAA0E;IAC1E,2EAA2E;IAC3E,8DAA8D;IAC9D,MAAM,YAAY,GAAG,MAAM,UAAU,CAAC,mBAAmB,CAAC,QAAQ,CAAC,CAAC;IACpE,IAAI,YAAY,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,CAAC,MAAM;QAAE,OAAO,KAAK,CAAC;IAC/D,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IACpF,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC;QAAE,OAAO,KAAK,CAAC;IAE/F,4CAA4C;IAC5C,MAAM,aAAa,GAAG,MAAM,EAAE;SAC3B,UAAU,CAAC,cAAc,CAAC;SAC1B,SAAS,CAAC,YAAY,EAAE,MAAM,EAAE,WAAW,CAAC;SAC5C,MAAM,CAAC,CAAC,QAAQ,EAAE,QAAQ,EAAE,oBAAoB,EAAE,aAAa,CAAC,CAAC;SACjE,KAAK,CAAC,QAAQ,EAAE,GAAG,EAAE,QAAQ,CAAC;SAC9B,OAAO,EAAE,CAAC;IAEb,IAAI,eAAe,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,eAAe,CAAC,aAAa,CAAC;QAAE,OAAO,KAAK,CAAC;IAErF,4EAA4E;IAC5E,2EAA2E;IAC3E,uDAAuD;IACvD,MAAM,aAAa,GAAG,MAAM,EAAE;SAC3B,UAAU,CAAC,cAAc,CAAC;SAC1B,SAAS,CAAC,YAAY,EAAE,MAAM,EAAE,WAAW,CAAC;SAC5C,MAAM,CAAC,CAAC,UAAU,EAAE,WAAW,EAAE,eAAe,EAAE,iBAAiB,CAAC,CAAC;SACrE,KAAK,CAAC,QAAQ,EAAE,GAAG,EAAE,QAAQ,CAAC;SAC9B,OAAO,EAAE,CAAC;IAEb,OAAO,eAAe,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,eAAe,CAAC,aAAa,CAAC,CAAC;AAC5E,CAAC;AAED;;;;;GAKG;AACH,SAAS,eAAe,CACtB,OAOG;IAEH,OAAO,OAAO;SACX,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACT,MAAM,OAAO,GAAG,OAAO,CAAC,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC;QAC5F,MAAM,QAAQ,GAAG,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,WAAW,KAAK,CAAC,CAAC;QACrD,MAAM,QAAQ,GAAG,CAAC,CAAC,YAAY,IAAI,CAAC,CAAC,aAAa,IAAI,EAAE,CAAC;QACzD,OAAO,GAAG,CAAC,CAAC,MAAM,IAAI,OAAO,IAAI,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,QAAQ,EAAE,CAAC;IAClE,CAAC,CAAC;SACD,IAAI,EAAE;SACN,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED,iFAAiF;AACjF,SAAS,eAAe,CACtB,OAA0K;IAE1K,OAAO,OAAO;SACX,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACT,MAAM,IAAI,GAAG,CAAC,CAAC,eAAe,IAAI,CAAC,CAAC,gBAAgB,IAAI,EAAE,CAAC;QAC3D,MAAM,IAAI,GAAG,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,SAAS,IAAI,EAAE,CAAC;QAC7C,OAAO,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC;IAC/C,CAAC,CAAC;SACD,IAAI,EAAE;SACN,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED,8EAA8E;AAC9E,qDAAqD;AACrD,8EAA8E;AAE9E;;;;;;;;GAQG;AACH,MAAM,UAAU,cAAc,CAAC,UAAkB;IAC/C,OAAO,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC;AAChD,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,aAAa,CAAC,QAAgB;IAC5C,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IAC9C,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC;QAAE,OAAO,IAAI,CAAC;IACvC,OAAO,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAc,CAAC;AAClE,CAAC;AAED,iDAAiD;AACjD,MAAM,UAAU,cAAc,CAAC,QAAgB,EAAE,IAAe;IAC9D,aAAa,CAAC,IAAI,CAAC,QAAQ,EAAE,YAAY,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AAC7E,CAAC"}

package/dist/indexer/walker.d.ts

@@ -0,0 +1,60 @@
+import type { MastConfig } from '../ast/types.js';
+export interface FileEntry {
+    /** Absolute path to the file. */
+    readonly path: string;
+    /** Relative path from project root (used as the stored `file_path`). */
+    readonly relativePath: string;
+    /** File mtime in unix seconds. */
+    readonly mtime: number;
+}
+/**
+ * Convert a glob pattern to a RegExp.
+ * `**\/` — matches zero or more leading directories
+ * `**`   — matches any sequence including `/`
+ * `*`    — matches any sequence not containing `/`
+ * `?`    — matches a single non-`/` character
+ *
+ * Lives in the walker (file-discovery domain) because both watch mode and the
+ * MCP tools' `file_pattern` filters need the same glob semantics — and because
+ * `walkProject` below hands the same patterns to fast-glob, so this function's
+ * output has to agree with fast-glob's for the same pattern. It is checked
+ * against fast-glob directly by `__tests__/glob-to-regex.test.ts`.
+ *
+ * Written as a single left-to-right scan rather than a chain of `.replace`
+ * calls. The chain was re-entrant: each rule rewrote the output of the ones
+ * before it, so `**` → `.*` left a `*` for the `*` rule to turn into
+ * `.[^/]*`, and `**\/` → `(.+/)?` left a `?` for the `?` rule to turn into
+ * `(.+/)[^/]`. Every shipped default in `exclude_patterns` (`store/config.ts`)
+ * contains one of those two sequences, so every one of them compiled to a
+ * regex that matched neither the directory it named nor its contents.
+ */
+export declare function globToRegex(pattern: string): RegExp;
+/**
+ * Walk the project and return all indexable files.
+ *
+ * Applies `file_extensions` allowlist and `exclude_patterns` denylist from
+ * config. Results are sorted lexicographically by `relativePath` (D1,
+ * IMPLEMENTATION_PLAN.md Stage 4): fast-glob returns filesystem order, which
+ * varies between identical runs — and because edge insertion order feeds the
+ * bare-name fallback in `insertEdges`' name resolution, two identical index
+ * runs produced edge sets differing by ±4/3,940 (§15.5). Sorting here makes
+ * every downstream consumer (index order, manifest, edge insertion)
+ * deterministic at the source. Callers still must not attach SEMANTIC meaning
+ * to the order — the guarantee is reproducibility, not priority.
+ */
+export declare function walkProject(config: MastConfig): Promise<FileEntry[]>;
+/**
+ * Build a `{ relativePath → mtime }` snapshot map from `walkProject` results.
+ * Stored as `file_manifest.json` at the end of each index run.
+ */
+export declare function buildManifest(entries: readonly FileEntry[]): Record<string, number>;
+/**
+ * Compare the current filesystem scan against the previous manifest.
+ * Returns sets of paths for changed, new, and deleted files.
+ */
+export declare function diffManifest(current: readonly FileEntry[], previousManifest: Record<string, number>): {
+    stale: FileEntry[];
+    added: FileEntry[];
+    deleted: string[];
+};
+//# sourceMappingURL=walker.d.ts.map

package/dist/indexer/walker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"walker.d.ts","sourceRoot":"","sources":["../../src/indexer/walker.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAElD,MAAM,WAAW,SAAS;IACxB,iCAAiC;IACjC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,wEAAwE;IACxE,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,kCAAkC;IAClC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAgB,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAwBnD;AAED;;;;;;;;;;;;GAYG;AACH,wBAAsB,WAAW,CAAC,MAAM,EAAE,UAAU,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CAgC1E;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAAC,OAAO,EAAE,SAAS,SAAS,EAAE,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAMnF;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAC1B,OAAO,EAAE,SAAS,SAAS,EAAE,EAC7B,gBAAgB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GACvC;IACD,KAAK,EAAE,SAAS,EAAE,CAAC;IACnB,KAAK,EAAE,SAAS,EAAE,CAAC;IACnB,OAAO,EAAE,MAAM,EAAE,CAAC;CACnB,CAoBA"}