dsh-plugin-codegraph-tree-sitter 0.5.0 → 0.5.2

package/lib/index.d.ts

@@ -73,12 +73,25 @@ export declare const DEFAULT_EXCLUDE: string[];
 export declare const DEFAULT_MAX_FILE_BYTES = 2000000;
 /** Default ceiling on how many files one run indexes. */
 export declare const DEFAULT_MAX_FILES = 50000;
+/**
+ * How many candidate source files a workspace may contain before a **session-start** index declines
+ * to run (default 8,000).
+ *
+ * `maxFiles` above is the bound for a run someone asked for; this one guards the run nobody asked
+ * for. A session opened with `cwd` pointing at a home directory or a tool's own data directory can
+ * cover tens of thousands of files — a real case reached 21,752 under `~/.dsh`, under the 50,000
+ * ceiling, and the Tree-sitter WASM heap died mid-parse while the desktop froze. Declining early
+ * costs one `readdir` pass; declining late costs the whole machine.
+ */
+export declare const DEFAULT_AUTO_INDEX_MAX_FILES = 8000;
 /** Default number of files parsed concurrently. */
 export declare const DEFAULT_CONCURRENCY = 4;
 /** Default quiet period after the last change before a watch-triggered refresh runs. */
 export declare const DEFAULT_WATCH_DEBOUNCE_MS = 2000;
 /** Default cap on directories watched individually, one watch each, by the fs seam. */
 export declare const DEFAULT_MAX_WATCHED_DIRECTORIES = 4000;
+/** Default age past which a session-open refreshes an existing index (ten minutes). */
+export declare const DEFAULT_AUTO_REFRESH_AFTER_MS: number;
 /** Plugin configuration: the indexer's identity and every deployment-varying bound on a run. */
 export interface Config {
     /**
@@ -173,7 +186,16 @@ export interface Config {
      * Requires the host's `jobs` service: the build is never run synchronously inside session creation,
      * so a composition without background jobs simply schedules nothing.
      */
+    /** Cap on candidate source files a session-start scan may find before it declines to index. */
+    autoIndexMaxFiles?: number;
     autoIndexOnSession?: boolean;
+    /**
+     * How stale an existing index may be before a new session refreshes it in the background (default
+     * 600000, ten minutes). An index older than this is rebuilt on session start using the parse cache,
+     * which is milliseconds — and rebuilding is also what restarts the file watcher, since watching
+     * only ever begins after a successful index.
+     */
+    autoRefreshAfterMs?: number;
 }
 export declare const Config: z<Config>;
 /**

package/lib/index.js

@@ -27,13 +27,14 @@
  * this package ships with.
  * @module dsh-plugin-codegraph-tree-sitter
  */
-import { existsSync } from 'node:fs';
+import { readFileSync } from 'node:fs';
+import { readdir } from 'node:fs/promises';
 import { stat } from 'node:fs/promises';
-import { join } from 'node:path';
+import { join, posix } from 'node:path';
 import z from '@deepseek-ai/schemastery';
 import { CodegraphIndexerId, INDEX_STATUS_RELATIVE_PATH } from 'dsh-plugin-codegraph-service';
-import { LANGUAGE_TABLE } from "./languages.js";
-import { walkAndExtract } from "./walk.js";
+import { LANGUAGE_TABLE, languageFor } from "./languages.js";
+import { isExcluded, walkAndExtract } from "./walk.js";
 import { writeParseCache } from "./cache.js";
 import { resolveWorkspace } from "./resolve.js";
 import { writeGraph } from "./schema.js";
@@ -77,12 +78,25 @@ export const DEFAULT_EXCLUDE = ['node_modules', 'dist', 'build', 'coverage', '.g
 export const DEFAULT_MAX_FILE_BYTES = 2_000_000;
 /** Default ceiling on how many files one run indexes. */
 export const DEFAULT_MAX_FILES = 50_000;
+/**
+ * How many candidate source files a workspace may contain before a **session-start** index declines
+ * to run (default 8,000).
+ *
+ * `maxFiles` above is the bound for a run someone asked for; this one guards the run nobody asked
+ * for. A session opened with `cwd` pointing at a home directory or a tool's own data directory can
+ * cover tens of thousands of files — a real case reached 21,752 under `~/.dsh`, under the 50,000
+ * ceiling, and the Tree-sitter WASM heap died mid-parse while the desktop froze. Declining early
+ * costs one `readdir` pass; declining late costs the whole machine.
+ */
+export const DEFAULT_AUTO_INDEX_MAX_FILES = 8_000;
 /** Default number of files parsed concurrently. */
 export const DEFAULT_CONCURRENCY = 4;
 /** Default quiet period after the last change before a watch-triggered refresh runs. */
 export const DEFAULT_WATCH_DEBOUNCE_MS = 2_000;
 /** Default cap on directories watched individually, one watch each, by the fs seam. */
 export const DEFAULT_MAX_WATCHED_DIRECTORIES = 4_000;
+/** Default age past which a session-open refreshes an existing index (ten minutes). */
+export const DEFAULT_AUTO_REFRESH_AFTER_MS = 10 * 60 * 1000;
 export const Config = z.object({
     indexerId: z.string().default(DEFAULT_INDEXER_ID),
     languages: z.array(z.string()).default([...new Set(LANGUAGE_TABLE.map(spec => spec.language))]),
@@ -98,7 +112,10 @@ export const Config = z.object({
     watch: z.boolean().default(true),
     watchDebounceMs: z.number().default(DEFAULT_WATCH_DEBOUNCE_MS),
     maxWatchedDirectories: z.number().default(DEFAULT_MAX_WATCHED_DIRECTORIES),
+    /** Cap on candidate source files a session-start scan may find before it declines to index. */
+    autoIndexMaxFiles: z.number().default(DEFAULT_AUTO_INDEX_MAX_FILES),
     autoIndexOnSession: z.boolean().default(true),
+    autoRefreshAfterMs: z.number().default(DEFAULT_AUTO_REFRESH_AFTER_MS),
 });
 /**
  * Register the tree-sitter indexer.
@@ -124,10 +141,11 @@ export function apply(ctx, config) {
     // cross-plugin relation here for an invariant to describe — just process-local bookkeeping this
     // effect's cleanup below tears down on unload.
     const watchers = new Map();
-    // Roots this plugin instance has already started an automatic index for. The set is only a guard
-    // against a burst of agents opened on one workspace each scheduling the same rebuild; what a
-    // previous *process* indexed is recorded by the status sidecar `autoIndexOnSession` reads instead.
-    const autoIndexed = new Set();
+    // When this instance last started an automatic index, keyed by root. This is not a "already done"
+    // set: an index goes stale while the process runs, so a later session in the same workspace may
+    // legitimately trigger another refresh. The map only keeps a burst of sessions opened at once from
+    // queueing the same rebuild repeatedly; the freshness decision itself reads the status sidecar.
+    const lastAutoRun = new Map();
     // `agent/created` is typed by `@deepseek-ai/dsh-agent`'s module augmentation. Reading the event
     // through a local structural face keeps this package's host dependencies unchanged: it needs one
     // event name and one payload shape, not the whole agent package.
@@ -192,7 +210,7 @@ export function apply(ctx, config) {
         if (resolved.autoIndexOnSession) {
             // Registered inside this effect so the listener follows the plugin's lifetime: unloading the
             // plugin must stop new sessions from scheduling work this instance can no longer run.
-            yield agentEvents.on('agent/created', payload => autoIndexOnSession(ctx, autoIndexed, payload));
+            yield agentEvents.on('agent/created', payload => autoIndexOnSession(ctx, resolved, lastAutoRun, payload));
         }
         yield ctx.codegraph.registerIndexer(indexer);
     }, 'codegraph-tree-sitter');
@@ -200,30 +218,39 @@ export function apply(ctx, config) {
 /** Stable job kind of an automatic session-start index, used for retention and diagnostics. */
 const AUTO_INDEX_JOB_KIND = 'codegraph-index';
 /**
- * Index a new session's workspace in the background, when nothing has indexed it yet.
+ * Index a new session's workspace in the background — when nothing has indexed it yet, or when what
+ * is there has gone stale.
  *
  * Runs synchronously inside `agent/created` dispatch, so it must never block or throw: it only decides
  * whether a build is warranted and hands one to the host's job registry. A synchronous build here
  * would stall agent creation for the length of a parse, which is why a composition without background
  * jobs does nothing instead.
+ *
+ * Refreshing a stale index is what restarts the file watcher after a host restart: watching only
+ * begins once a successful index has established a baseline, so a workspace whose index is minutes
+ * old would otherwise stay frozen at whatever the last process left behind.
  * @param ctx - the plugin context, consulted for the optional job registry.
- * @param autoIndexed - roots this process has already started an automatic index for.
+ * @param config - the resolved plugin configuration, for the staleness threshold.
+ * @param lastAutoRun - when this instance last started an automatic index for each root.
  * @param payload - the `agent/created` dispatch payload.
  */
-function autoIndexOnSession(ctx, autoIndexed, payload) {
+function autoIndexOnSession(ctx, config, lastAutoRun, payload) {
     try {
         const agent = payload.agent;
         const root = agent?.session?.header?.cwd;
         // No workspace means no root to claim: an agent created without a `cwd` has nothing to index.
         if (agent === undefined || !root)
             return;
-        // Once per root per process. A second agent opened on the same workspace must not queue a second
-        // full rebuild of a graph the first one is already building.
-        if (autoIndexed.has(root))
+        const now = Date.now();
+        // A burst of sessions opened on one workspace must not queue the same rebuild repeatedly.
+        const previous = lastAutoRun.get(root);
+        if (previous !== undefined && now - previous < config.autoRefreshAfterMs)
             return;
-        // An existing status sidecar means a previous run already indexed this workspace. Checked before
-        // the job lookup so a composition without background jobs never pays for a filesystem probe.
-        if (existsSync(join(root, INDEX_STATUS_RELATIVE_PATH)))
+        // A fresh sidecar means a recent run is already describing this workspace; an absent, unreadable
+        // or old one means the graph is missing or behind. Checked before the job lookup so a composition
+        // without background jobs never pays for the probe.
+        const indexedAt = readStatusIndexedAt(root);
+        if (indexedAt !== undefined && now - indexedAt < config.autoRefreshAfterMs)
             return;
         // `ctx.get` rather than an `inject` entry: `jobs` is a host composition choice, and requiring it in
         // `inject` would hide this whole indexer from a profile that runs no background jobs.
@@ -235,11 +262,11 @@ function autoIndexOnSession(ctx, autoIndexed, payload) {
             owner: agent.id,
             kind: AUTO_INDEX_JOB_KIND,
             label: `codegraph auto index ${root}`,
-            run: handle => runAutoIndexJob(ctx, root, handle),
+            run: handle => runAutoIndexJob(ctx, root, config, handle),
         });
         // Marked only after the registry accepted the job, so a transient refusal leaves the workspace
         // eligible for the next session that opens in it.
-        autoIndexed.add(root);
+        lastAutoRun.set(root, now);
     }
     catch (error) {
         // The host treats a throwing listener as vetoing publication. Losing an index is a degradation,
@@ -247,6 +274,71 @@ function autoIndexOnSession(ctx, autoIndexed, payload) {
         console.error(`codegraph-tree-sitter: automatic indexing for a new session failed: ${String(error)}`);
     }
 }
+/**
+ * How many candidate source files `root` holds, when that is past the session-start limit.
+ *
+ * Deliberately a **bounded directory walk that never reads a file**: it counts what `discover` would
+ * hand the parser and stops one past the limit. The alternative — starting the real run and bailing
+ * when memory runs out — is what froze the machine this guard exists to prevent.
+ * @param root - absolute workspace path.
+ * @param config - the resolved plugin configuration (for the limit and the exclude list).
+ * @param signal - aborts the walk.
+ * @returns the count once it exceeds the limit, or `undefined` when the workspace is small enough.
+ */
+async function exceedsAutoIndexLimit(root, config, signal) {
+    const excluded = new Set(config.exclude);
+    const limit = config.autoIndexMaxFiles;
+    let count = 0;
+    async function scan(relativeDir) {
+        signal.throwIfAborted();
+        let entries;
+        try {
+            entries = await readdir(posix.join(root, relativeDir), { withFileTypes: true });
+        }
+        catch {
+            // An unreadable directory contributes nothing to a count that exists to protect against size.
+            return false;
+        }
+        for (const entry of entries) {
+            const relativePath = relativeDir === '' ? entry.name : `${relativeDir}/${entry.name}`;
+            if (isExcluded(entry.name, relativePath, entry.isDirectory(), excluded, []))
+                continue;
+            if (entry.isDirectory()) {
+                if (await scan(relativePath))
+                    return true;
+                continue;
+            }
+            if (!entry.isFile())
+                continue;
+            const spec = languageFor(posix.extname(entry.name));
+            if (spec === undefined)
+                continue;
+            count += 1;
+            if (count > limit)
+                return true;
+        }
+        return false;
+    }
+    return (await scan('')) ? count : undefined;
+}
+/**
+ * When the workspace's sidecar says its index was built.
+ * @param root - the workspace root.
+ * @returns the epoch-millisecond timestamp, or `undefined` when the sidecar is absent or unreadable —
+ *   either way the graph cannot be trusted to be current.
+ */
+function readStatusIndexedAt(root) {
+    try {
+        const parsed = JSON.parse(readFileSync(join(root, INDEX_STATUS_RELATIVE_PATH), 'utf8'));
+        if (typeof parsed !== 'object' || parsed === null)
+            return undefined;
+        const indexedAt = parsed.indexedAt;
+        return typeof indexedAt === 'number' && Number.isFinite(indexedAt) ? indexedAt : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
 /**
  * The producer body of one automatic indexing job.
  *
@@ -258,10 +350,19 @@ function autoIndexOnSession(ctx, autoIndexed, payload) {
  * @param handle - the job's producer face, for the job's live log.
  * @returns the job's cancellation hook and settlement promise.
  */
-function runAutoIndexJob(ctx, root, handle) {
+function runAutoIndexJob(ctx, root, config, handle) {
     const controller = new AbortController();
     const done = (async () => {
         try {
+            const oversized = await exceedsAutoIndexLimit(root, config, controller.signal);
+            if (oversized !== undefined) {
+                // Declining here is the whole point of the check: parsing this workspace would exhaust the
+                // WASM heap and take the desktop down with it, and an automatic index is a convenience.
+                handle.append(`Skipped automatic indexing of ${root}: it holds at least ${String(oversized)} candidate ` +
+                    `source files, past the ${String(config.autoIndexMaxFiles)}-file session-start limit. ` +
+                    `Run codegraph_index yourself if you want a graph for it.\n`);
+                return { status: 'failed', detail: 'workspace too large for automatic indexing' };
+            }
             const report = await ctx.codegraph.index(root, controller.signal);
             handle.append(`Indexed ${root}: ${report.filesIndexed} files, ${report.nodeCount} symbols, ` +
                 `${report.edgeCount} relationships.\n`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dsh-plugin-codegraph-tree-sitter",
-  "version": "0.5.0",
+  "version": "0.5.2",
   "description": "Self-built code-graph indexer for DeepSeek Harness — parses a workspace with web-tree-sitter and writes schema version 4 to .codegraph/codegraph.db, the same on-disk format the codegraph CLI writes, so no external indexer is required",
   "keywords": [
     "dsh",
@@ -51,7 +51,7 @@
     "@deepseek-ai/schemastery": "^3.18.1",
     "tree-sitter-wasms": "^0.1.13",
     "web-tree-sitter": "^0.25.10",
-    "dsh-plugin-codegraph-service": "^0.5.0"
+    "dsh-plugin-codegraph-service": "^0.5.2"
   },
   "peerDependencies": {
     "@deepseek-ai/cordis": ">=4.0.1",