dsh-plugin-codegraph-tree-sitter 0.1.7 → 0.2.0

package/lib/walk.js

@@ -11,6 +11,7 @@
 import { createHash } from 'node:crypto';
 import { readdir, readFile, stat } from 'node:fs/promises';
 import { posix } from 'node:path';
+import { readParseCache } from "./cache.js";
 import { createParser } from "./grammar.js";
 import { extractFile } from "./extract.js";
 import { loadGitignore, matchesGitignore } from "./gitignore.js";
@@ -71,28 +72,73 @@ async function discover(projectRoot, config, signal) {
     return { candidates, overflow };
 }
 /**
- * Parse one candidate.
+ * The walk's view of a file whose extraction came from the cache: the same record a fresh parse
+ * would have produced, with the extraction the cache matched.
+ *
+ * The modification time and size come from the stat taken just now, not from the cache entry, because
+ * the graph records them and `status` measures staleness as `file mtime > recorded mtime`. Under
+ * stat verification the two are equal by construction; under hash verification a file whose content
+ * matched but whose mtime was touched would otherwise be recorded with the old mtime and reported
+ * stale on every later status call, even though the graph describes its exact content.
+ * @param path - the candidate's project-relative path, which the cache entry does not store itself.
+ * @param cached - the entry this file's content matched.
+ * @param stats - the file's stat from this run.
+ * @returns the file as the resolver and graph writer expect it.
+ */
+function reusedFile(path, cached, stats) {
+    return {
+        path,
+        language: cached.language,
+        size: stats.size,
+        modifiedAt: stats.mtimeMs,
+        contentHash: cached.contentHash,
+        lineCount: cached.lineCount,
+        extraction: cached.extraction,
+    };
+}
+/**
+ * Parse one candidate, or reuse the extraction a previous run cached for it.
  * @param candidate - the file to parse.
  * @param config - the walk's bounds.
+ * @param cache - the previous run's extractions, keyed by project-relative path.
  * @param signal - aborts the parse.
- * @returns the extracted file, `'minified'` when it was skipped as generated output, or `undefined`
- *   when it is over {@link WalkConfig.maxFileBytes}.
+ * @returns the extracted file (and whether it came from the cache), `'minified'` when it was skipped
+ *   as generated output, or `'skipped'` when it is over {@link WalkConfig.maxFileBytes}.
  */
-async function parseCandidate(candidate, config, signal) {
+async function parseCandidate(candidate, config, cache, signal) {
     // Checked before the stat and read: a `.min.` name settles it with no I/O at all.
     if (config.skipMinified && isMinifiedName(candidate.path))
-        return 'minified';
+        return { outcome: 'minified' };
     const stats = await stat(candidate.absolute);
     if (stats.size > config.maxFileBytes)
-        return undefined;
+        return { outcome: 'skipped' };
     const spec = languageFor(posix.extname(candidate.path));
     /* v8 ignore next 2 -- discover() only ever produces a candidate whose extension resolved to a spec. */
     if (spec === undefined)
-        return undefined;
-    const text = await readFile(candidate.absolute, 'utf8');
+        return { outcome: 'skipped' };
+    // Verified before the parser is even created: this is the whole point of the cache — an unchanged
+    // file costs one stat when stat verification is configured, and nothing beyond that.
+    const cached = cache.get(candidate.path);
+    let text;
+    if (cached !== undefined) {
+        if (config.cacheVerify === 'stat') {
+            if (cached.modifiedAt === stats.mtimeMs && cached.size === stats.size) {
+                return { outcome: 'extracted', file: reusedFile(candidate.path, cached, stats), reused: true };
+            }
+        }
+        else {
+            // Content is the only thing this mode trusts, so the read it needs to verify is also the read
+            // the parse below would need — a miss here costs nothing extra.
+            text = await readFile(candidate.absolute, 'utf8');
+            if (createHash('sha256').update(text).digest('hex') === cached.contentHash) {
+                return { outcome: 'extracted', file: reusedFile(candidate.path, cached, stats), reused: true };
+            }
+        }
+    }
+    text ??= await readFile(candidate.absolute, 'utf8');
     const shape = measureText(text);
     if (config.skipMinified && looksMinified(candidate.path, shape))
-        return 'minified';
+        return { outcome: 'minified' };
     const parser = await createParser(spec);
     try {
         signal?.throwIfAborted();
@@ -100,16 +146,20 @@ async function parseCandidate(candidate, config, signal) {
         /* v8 ignore next 2 -- createParser() always assigns a language before returning; parse() returns
          * null only when the parser has none. */
         if (tree === null)
-            return undefined;
+            return { outcome: 'skipped' };
         const extraction = extractFile(tree, spec);
         return {
-            path: candidate.path,
-            language: spec.language,
-            size: stats.size,
-            modifiedAt: stats.mtimeMs,
-            contentHash: createHash('sha256').update(text).digest('hex'),
-            lineCount: shape.lineCount,
-            extraction,
+            outcome: 'extracted',
+            reused: false,
+            file: {
+                path: candidate.path,
+                language: spec.language,
+                size: stats.size,
+                modifiedAt: stats.mtimeMs,
+                contentHash: createHash('sha256').update(text).digest('hex'),
+                lineCount: shape.lineCount,
+                extraction,
+            },
         };
     }
     finally {
@@ -138,26 +188,33 @@ async function mapWithConcurrency(items, concurrency, task) {
     return results;
 }
 /**
- * Walk `projectRoot` and parse every candidate file it finds.
+ * Walk `projectRoot` and parse every candidate file it finds — or reuse the extraction a previous
+ * run cached for it.
  * @param projectRoot - absolute path of the workspace to index.
  * @param config - the walk's bounds.
  * @param signal - aborts the walk.
- * @returns the parsed files and how many candidates were skipped (over size, or past `maxFiles`).
+ * @returns the parsed files, how many candidates were skipped (over size, or past `maxFiles`), and
+ *   how many files reused a cached extraction.
  */
 export async function walkAndExtract(projectRoot, config, signal) {
     const { candidates, overflow } = await discover(projectRoot, config, signal);
-    const parsed = await mapWithConcurrency(candidates, config.concurrency, candidate => parseCandidate(candidate, config, signal));
+    const cache = config.reuseParsedCache ? await readParseCache(projectRoot) : new Map();
+    const parsed = await mapWithConcurrency(candidates, config.concurrency, candidate => parseCandidate(candidate, config, cache, signal));
     const files = [];
     let filesSkipped = overflow;
     let filesMinified = 0;
+    let filesReused = 0;
     for (const result of parsed) {
-        if (result === undefined)
+        if (result.outcome === 'skipped')
             filesSkipped++;
-        else if (result === 'minified')
+        else if (result.outcome === 'minified')
             filesMinified++;
-        else
-            files.push(result);
+        else {
+            files.push(result.file);
+            if (result.reused)
+                filesReused++;
+        }
     }
-    return { files, filesSkipped, filesMinified };
+    return { files, filesSkipped, filesMinified, filesReused };
 }
 //# sourceMappingURL=walk.js.map

package/lib/watcher.d.ts

@@ -1,52 +1,112 @@
 /**
- * File-watch-driven reindexing: debounce a workspace's file-change events into a bounded number of
- * `sync` calls, using a per-platform strategy chosen to keep the open-descriptor / kernel-watch cost
- * BOUNDED rather than growing with the number of files:
+ * File-watch-driven reindexing over the harness filesystem seam (`ctx.fs`): turn a workspace's
+ * file-change notifications into a bounded number of `sync` calls, using ONE WATCH PER DIRECTORY.
  *
- *  - macOS / Windows: a SINGLE recursive `fs.watch(root, {recursive:true})`. This is the fix for the
- *    macOS file-table exhaustion a reference implementation hit watching one file descriptor per
- *    watched file (tens of thousands of REG fds), which exhausted `kern.maxfiles` and crashed
- *    unrelated processes system-wide (see NOTES.local.md). One recursive stream costs O(1)
- *    descriptors no matter how large the tree.
- *  - Linux: recursive `fs.watch` is unsupported, so one inotify watch is installed per (non-excluded)
- *    directory — O(directories), NOT O(files). New directories are picked up dynamically as events
- *    reveal them, and {@link WatchConfig.maxWatchedDirectories} bounds inotify usage on pathological
- *    monorepos.
+ * Two facts about the seam decide this module's shape. `watch()` reports a directory's DIRECT
+ * entries only (`depth: 0`, so a nested change never arrives on an ancestor's watch), and it never
+ * names the path that changed:
  *
- * `sync` is always a full rebuild in this version: the `paths` a debounce window collected are passed
- * through but never consulted by the caller supplied here (`runIndex` reruns `walkAndExtract` over the
- * whole tree regardless). That keeps this file's only new correctness surface confined to "when do we
- * call sync", not "what does sync do with a subset of files" — the latter is deliberately deferred
- * until a real incremental reindex is built (see NOTES.local.md). Because sync is always full, a
- * directory delete or a `mkdir` immediately followed by writes inside it never needs special-casing:
- * the parent directory's own create event is enough to schedule a rebuild that discovers (or removes)
- * everything under the path from scratch, watched or not.
+ *  - Coverage, therefore, comes from watching every in-scope directory: O(directories) watches, never
+ *    O(files). A directory created after startup is picked up by the same re-listing that a parent's
+ *    event triggers, so the watch tree grows with the workspace instead of freezing at `start()`.
+ *  - A notification is a directory-level invalidation with no filename, so attributing it to paths
+ *    means re-listing that directory and diffing it against the previous listing, using the
+ *    `version`/`size` the listing itself carries. That diff is what keeps a rebuild from being
+ *    scheduled by a change this indexer excludes — notably a change to `.codegraph`, which is what
+ *    stops the watcher from rebuilding forever off its own graph write.
+ *
+ * `sync` is still always a full rebuild: the paths a debounce window collected are passed through but
+ * never consulted by the caller supplied here (see `runSync` in `./index.ts`). That keeps this file's
+ * correctness surface confined to "when do we call sync", not "what does sync do with a subset of
+ * files".
+ *
+ * A change landing between the baseline index that starts the watcher and the listing that arms its
+ * root is in no snapshot and so produces no notification. That window is one directory listing wide,
+ * and the next attributable change — or the next explicit index — covers whatever it missed.
+ *
+ * Resource discipline is explicit. Every watch's disposer is retained and invoked on `stop()` or
+ * permanent degradation, and a watch whose initialization finishes after its watcher was stopped is
+ * closed immediately instead of being adopted — a late init can never leak a live watch. A
+ * generation counter invalidates every in-flight async step of a previous start(), so a restart
+ * cannot inherit state from the session it replaced. {@link WatchConfig.maxWatchedDirectories}
+ * bounds the number of open watches, and reaching it degrades the whole watcher rather than silently
+ * covering only part of the tree.
  * @module dsh-plugin-codegraph-tree-sitter/watcher
  */
-/** One raw watch event, in the shape `node:fs`'s `fs.watch` listener callback receives. */
-export type WatchEventListener = (eventType: 'rename' | 'change', filename: string | null) => void;
-/** A watch error, in the shape `fs.watch`'s `'error'` event delivers. */
-export type WatchErrorListener = (error: unknown) => void;
-/** A live OS-level watch, closeable independent of anything else this module tracks. */
-export interface WatchHandle {
-    close(): void;
+/**
+ * One resolved filesystem target — the opaque identity `ctx.fs` hands back and every other fs call
+ * takes. Only `targetKey` is read here: the module never parses it, and a remote backend's key is
+ * as valid as a local path.
+ */
+export interface WatchTarget {
+    /** Opaque backend identity of the target. */
+    readonly targetKey: string;
+}
+/** One direct child of a listed directory, as `ctx.fs.listDir` reports it. */
+export interface WatchDirEntry {
+    /** Basename of the child inside the listed directory. */
+    readonly name: string;
+    /** Whether the child is a regular file, a directory, or something else. */
+    readonly type: 'file' | 'directory' | 'other';
+    /** Resolved child target, usable for a follow-up `watch`/`listDir`. */
+    readonly target: WatchTarget;
+    /**
+     * Opaque freshness token, when the backend can report one cheaply. Opting into it is what makes a
+     * same-size content change visible to the change diff below, so a backend that omits it can still
+     * attribute creations, removals, and size-changing rewrites — but not a rewrite that preserved the
+     * size, which is then covered by the next attributable change or by the next explicit index.
+     */
+    readonly version?: string;
+    /** Byte size of a regular file, when the backend can report one. */
+    readonly size?: number;
 }
 /**
- * Indirection over `fs.watch` so tests can inject a fake that emits events/errors deterministically —
- * `fs.watch` is a non-configurable module property that can't be spied, and provoking real EMFILE/
- * ENOSPC exhaustion in a test run isn't reliable. Production always uses {@link nodeWatch}.
+ * The narrow slice of `ctx.fs` this module consumes.
+ *
+ * Deliberately structural rather than an import of the host's `FileSystem`: the tree-sitter package
+ * declares no dependency on `@deepseek-ai/dsh-fs`, and `watch` only exists from host 0.2 onward,
+ * while the package's supported peer range starts earlier. Naming the three calls it uses — resolve a
+ * directory, list it, observe it — keeps this module compilable and loadable across that range, and
+ * keeps the port small enough that a test double is a few lines.
  */
-export type WatchPrimitive = (path: string, options: {
-    readonly recursive: boolean;
-}, onEvent: WatchEventListener, onError: WatchErrorListener) => WatchHandle;
-/** The real `fs.watch`-backed primitive. */
-export declare const nodeWatch: WatchPrimitive;
+export interface WatchFileSystem {
+    /**
+     * Resolve a path into a stable backend target.
+     * @param path - the path to resolve; absolute paths are what this module passes.
+     * @param opts - optional resolution base and cancellation signal.
+     * @returns the resolved target.
+     */
+    resolve(path: string, opts?: {
+        readonly cwd?: string;
+        readonly signal?: AbortSignal;
+    }): Promise<WatchTarget>;
+    /**
+     * List one directory's direct children.
+     * @param target - the resolved directory target.
+     * @param signal - aborts the listing.
+     * @returns one entry per direct child.
+     */
+    listDir(target: WatchTarget, signal?: AbortSignal): Promise<readonly WatchDirEntry[]>;
+    /**
+     * Observe one directory's direct entries.
+     * @param target - the resolved directory target.
+     * @param changed - invalidation callback, called with no argument for a change and with an error
+     *   when the backend loses the watch; initialization failures are reported by rejecting instead.
+     * @param signal - cancels initialization only; an initialized watch is released through the
+     *   returned disposer.
+     * @returns the disposer that releases the watch.
+     */
+    watch(target: WatchTarget, changed: (error?: Error) => void, signal: AbortSignal): Promise<() => Promise<void>>;
+}
 /** Why live watching was permanently disabled for this watcher instance. */
 export type DegradeReason = {
     readonly code: 'RESOURCE_EXHAUSTED';
     readonly cause: NodeJS.ErrnoException;
 } | {
     readonly code: 'WATCH_LIMIT_EXCEEDED';
+} | {
+    readonly code: 'WATCH_UNAVAILABLE';
+    readonly cause: unknown;
 } | {
     readonly code: 'SYNC_FAILURE_LIMIT';
     readonly cause: unknown;
@@ -61,11 +121,11 @@ export interface WatchConfig {
     readonly respectGitignore: boolean;
     /** Milliseconds of quiet after the last change before `sync` runs. */
     readonly debounceMs: number;
-    /** Hard cap on directories watched individually (Linux only); exceeding it degrades the watcher. */
+    /** Hard cap on directories watched individually; exceeding it degrades the watcher. */
     readonly maxWatchedDirectories: number;
     /**
-     * Run one reindex. `paths` are the project-relative paths a debounce window collected, offered for
-     * a future incremental sync to consult; this version's caller always ignores them and rebuilds the
+     * Run one reindex. `paths` are the project-relative paths of the direct entries whose change the
+     * per-directory watch could attribute; this version's caller always ignores them and rebuilds the
      * whole graph.
      */
     readonly sync: (paths?: readonly string[]) => Promise<{
@@ -83,13 +143,18 @@ export interface Watcher {
     stop(): void;
     /** Whether live watching has permanently degraded (until the next {@link start}). */
     isDegraded(): boolean;
-    /** Project-relative paths seen since the last debounce window fired a sync. */
+    /**
+     * Project-relative paths of the entries whose change is queued for the next sync, collected since
+     * the last debounce window fired one. A path names a direct entry of a watched directory — file or
+     * directory — because that is the finest grain the seam can attribute a change to.
+     */
     getPendingFiles(): readonly string[];
 }
 /**
  * Build a watcher over one workspace root. Nothing runs until {@link Watcher.start} is called.
  * @param config - scope, timing, and the sync callback to drive.
- * @param primitive - the watch primitive to use; defaults to the real `fs.watch`.
+ * @param fs - the filesystem seam to resolve, enumerate, and watch through.
+ * @returns the watcher; every resource it acquires is released by {@link Watcher.stop}.
  */
-export declare function createWatcher(config: WatchConfig, primitive?: WatchPrimitive): Watcher;
+export declare function createWatcher(config: WatchConfig, fs: WatchFileSystem): Watcher;
 //# sourceMappingURL=watcher.d.ts.map