dsh-plugin-codegraph-tree-sitter 0.4.2 → 0.5.1

package/lib/index.d.ts

@@ -29,6 +29,7 @@
  */
 import type { Context } from '@deepseek-ai/cordis';
 import z from '@deepseek-ai/schemastery';
+import type { EngineMode } from './walk.ts';
 export { LANGUAGE_TABLE, languageFor } from './languages.ts';
 export type { DefinitionRule, ImportRule, LanguageSpec } from './languages.ts';
 export { extractFile } from './extract.ts';
@@ -43,7 +44,9 @@ export { SCHEMA_VERSION, writeGraph } from './schema.ts';
 export { EXTRACTION_CACHE_RELATIVE_PATH, EXTRACTION_CACHE_VERSION, readParseCache, writeParseCache } from './cache.ts';
 export type { CachedFile, CacheTarget, CacheTargetInfo, ParseCacheFileSystem } from './cache.ts';
 export { isExcluded, walkAndExtract } from './walk.ts';
-export type { WalkConfig, WalkResult } from './walk.ts';
+export type { EngineMode, WalkConfig, WalkResult } from './walk.ts';
+export { extractWithRust, loadRustEngine, RUST_BATCH_SIZE, RUST_LANGUAGES } from './rust-engine.ts';
+export type { LoadRustEngineOptions, RustEngine, RustExtractInput } from './rust-engine.ts';
 export { createWatcher } from './watcher.ts';
 export type { DegradeReason, WatchConfig, WatchDirEntry, WatchFileSystem, WatchTarget, Watcher, } from './watcher.ts';
 export { decideWatch, readProcVersion } from './watch-policy.ts';
@@ -76,6 +79,8 @@ export declare const DEFAULT_CONCURRENCY = 4;
 export declare const DEFAULT_WATCH_DEBOUNCE_MS = 2000;
 /** Default cap on directories watched individually, one watch each, by the fs seam. */
 export declare const DEFAULT_MAX_WATCHED_DIRECTORIES = 4000;
+/** Default age past which a session-open refreshes an existing index (ten minutes). */
+export declare const DEFAULT_AUTO_REFRESH_AFTER_MS: number;
 /** Plugin configuration: the indexer's identity and every deployment-varying bound on a run. */
 export interface Config {
     /**
@@ -130,6 +135,17 @@ export interface Config {
      * parse and the tree walk, which dominate the per-file cost.
      */
     cacheVerify?: 'stat' | 'hash';
+    /**
+     * Which extractor parses a file (default `'auto'`): this package's own `extractFile` (`'js'`), the
+     * native Rust addon from `crates/indexer-rust` (`'rust'`), or whichever is available (`'auto'`).
+     *
+     * `'rust'` and `'auto'` hand every TypeScript/JavaScript/JSX/TSX file to the addon in batches and
+     * fall back to `extractFile` for anything it does not answer for — a language it does not handle, a
+     * file it declines, or a deployment with no built artifact at all. Both paths produce the same
+     * extraction, pinned by `rust-parity.spec.ts`, so this setting only trades speed against how the
+     * work is scheduled. `'js'` is exactly the behaviour this package had before the addon existed.
+     */
+    engine?: EngineMode;
     /**
      * Watch the workspace for file changes and refresh the index automatically after a successful
      * `index()` call establishes a baseline (default true). Set `watch: false` to keep indexing purely
@@ -160,6 +176,13 @@ export interface Config {
      * so a composition without background jobs simply schedules nothing.
      */
     autoIndexOnSession?: boolean;
+    /**
+     * How stale an existing index may be before a new session refreshes it in the background (default
+     * 600000, ten minutes). An index older than this is rebuilt on session start using the parse cache,
+     * which is milliseconds — and rebuilding is also what restarts the file watcher, since watching
+     * only ever begins after a successful index.
+     */
+    autoRefreshAfterMs?: number;
 }
 export declare const Config: z<Config>;
 /**

package/lib/index.js

@@ -27,7 +27,7 @@
  * this package ships with.
  * @module dsh-plugin-codegraph-tree-sitter
  */
-import { existsSync } from 'node:fs';
+import { readFileSync } from 'node:fs';
 import { stat } from 'node:fs/promises';
 import { join } from 'node:path';
 import z from '@deepseek-ai/schemastery';
@@ -47,6 +47,10 @@ export { isMinifiedName, looksMinified, measureText, MINIFIED_AVERAGE_LINE_LENGT
 export { SCHEMA_VERSION, writeGraph } from "./schema.js";
 export { EXTRACTION_CACHE_RELATIVE_PATH, EXTRACTION_CACHE_VERSION, readParseCache, writeParseCache } from "./cache.js";
 export { isExcluded, walkAndExtract } from "./walk.js";
+// The native engine is part of this package's story even though it is optional at runtime: a
+// deployment that wants to know *why* the faster path is not running can call `loadRustEngine()`
+// itself and read the answer, instead of inferring it from timings.
+export { extractWithRust, loadRustEngine, RUST_BATCH_SIZE, RUST_LANGUAGES } from "./rust-engine.js";
 export { createWatcher } from "./watcher.js";
 export { decideWatch, readProcVersion } from "./watch-policy.js";
 // Git diagnostics stay library functions rather than plugin wiring: nothing in `apply` runs git, so
@@ -79,6 +83,8 @@ export const DEFAULT_CONCURRENCY = 4;
 export const DEFAULT_WATCH_DEBOUNCE_MS = 2_000;
 /** Default cap on directories watched individually, one watch each, by the fs seam. */
 export const DEFAULT_MAX_WATCHED_DIRECTORIES = 4_000;
+/** Default age past which a session-open refreshes an existing index (ten minutes). */
+export const DEFAULT_AUTO_REFRESH_AFTER_MS = 10 * 60 * 1000;
 export const Config = z.object({
     indexerId: z.string().default(DEFAULT_INDEXER_ID),
     languages: z.array(z.string()).default([...new Set(LANGUAGE_TABLE.map(spec => spec.language))]),
@@ -90,10 +96,12 @@ export const Config = z.object({
     concurrency: z.number().default(DEFAULT_CONCURRENCY),
     reuseParsedCache: z.boolean().default(true),
     cacheVerify: z.union([z.const('stat'), z.const('hash')]).default('stat'),
+    engine: z.union([z.const('auto'), z.const('js'), z.const('rust')]).default('auto'),
     watch: z.boolean().default(true),
     watchDebounceMs: z.number().default(DEFAULT_WATCH_DEBOUNCE_MS),
     maxWatchedDirectories: z.number().default(DEFAULT_MAX_WATCHED_DIRECTORIES),
     autoIndexOnSession: z.boolean().default(true),
+    autoRefreshAfterMs: z.number().default(DEFAULT_AUTO_REFRESH_AFTER_MS),
 });
 /**
  * Register the tree-sitter indexer.
@@ -119,10 +127,11 @@ export function apply(ctx, config) {
     // cross-plugin relation here for an invariant to describe — just process-local bookkeeping this
     // effect's cleanup below tears down on unload.
     const watchers = new Map();
-    // Roots this plugin instance has already started an automatic index for. The set is only a guard
-    // against a burst of agents opened on one workspace each scheduling the same rebuild; what a
-    // previous *process* indexed is recorded by the status sidecar `autoIndexOnSession` reads instead.
-    const autoIndexed = new Set();
+    // When this instance last started an automatic index, keyed by root. This is not a "already done"
+    // set: an index goes stale while the process runs, so a later session in the same workspace may
+    // legitimately trigger another refresh. The map only keeps a burst of sessions opened at once from
+    // queueing the same rebuild repeatedly; the freshness decision itself reads the status sidecar.
+    const lastAutoRun = new Map();
     // `agent/created` is typed by `@deepseek-ai/dsh-agent`'s module augmentation. Reading the event
     // through a local structural face keeps this package's host dependencies unchanged: it needs one
     // event name and one payload shape, not the whole agent package.
@@ -187,7 +196,7 @@ export function apply(ctx, config) {
         if (resolved.autoIndexOnSession) {
             // Registered inside this effect so the listener follows the plugin's lifetime: unloading the
             // plugin must stop new sessions from scheduling work this instance can no longer run.
-            yield agentEvents.on('agent/created', payload => autoIndexOnSession(ctx, autoIndexed, payload));
+            yield agentEvents.on('agent/created', payload => autoIndexOnSession(ctx, resolved, lastAutoRun, payload));
         }
         yield ctx.codegraph.registerIndexer(indexer);
     }, 'codegraph-tree-sitter');
@@ -195,30 +204,39 @@ export function apply(ctx, config) {
 /** Stable job kind of an automatic session-start index, used for retention and diagnostics. */
 const AUTO_INDEX_JOB_KIND = 'codegraph-index';
 /**
- * Index a new session's workspace in the background, when nothing has indexed it yet.
+ * Index a new session's workspace in the background — when nothing has indexed it yet, or when what
+ * is there has gone stale.
  *
  * Runs synchronously inside `agent/created` dispatch, so it must never block or throw: it only decides
  * whether a build is warranted and hands one to the host's job registry. A synchronous build here
  * would stall agent creation for the length of a parse, which is why a composition without background
  * jobs does nothing instead.
+ *
+ * Refreshing a stale index is what restarts the file watcher after a host restart: watching only
+ * begins once a successful index has established a baseline, so a workspace whose index is minutes
+ * old would otherwise stay frozen at whatever the last process left behind.
  * @param ctx - the plugin context, consulted for the optional job registry.
- * @param autoIndexed - roots this process has already started an automatic index for.
+ * @param config - the resolved plugin configuration, for the staleness threshold.
+ * @param lastAutoRun - when this instance last started an automatic index for each root.
  * @param payload - the `agent/created` dispatch payload.
  */
-function autoIndexOnSession(ctx, autoIndexed, payload) {
+function autoIndexOnSession(ctx, config, lastAutoRun, payload) {
     try {
         const agent = payload.agent;
         const root = agent?.session?.header?.cwd;
         // No workspace means no root to claim: an agent created without a `cwd` has nothing to index.
         if (agent === undefined || !root)
             return;
-        // Once per root per process. A second agent opened on the same workspace must not queue a second
-        // full rebuild of a graph the first one is already building.
-        if (autoIndexed.has(root))
+        const now = Date.now();
+        // A burst of sessions opened on one workspace must not queue the same rebuild repeatedly.
+        const previous = lastAutoRun.get(root);
+        if (previous !== undefined && now - previous < config.autoRefreshAfterMs)
             return;
-        // An existing status sidecar means a previous run already indexed this workspace. Checked before
-        // the job lookup so a composition without background jobs never pays for a filesystem probe.
-        if (existsSync(join(root, INDEX_STATUS_RELATIVE_PATH)))
+        // A fresh sidecar means a recent run is already describing this workspace; an absent, unreadable
+        // or old one means the graph is missing or behind. Checked before the job lookup so a composition
+        // without background jobs never pays for the probe.
+        const indexedAt = readStatusIndexedAt(root);
+        if (indexedAt !== undefined && now - indexedAt < config.autoRefreshAfterMs)
             return;
         // `ctx.get` rather than an `inject` entry: `jobs` is a host composition choice, and requiring it in
         // `inject` would hide this whole indexer from a profile that runs no background jobs.
@@ -234,7 +252,7 @@ function autoIndexOnSession(ctx, autoIndexed, payload) {
         });
         // Marked only after the registry accepted the job, so a transient refusal leaves the workspace
         // eligible for the next session that opens in it.
-        autoIndexed.add(root);
+        lastAutoRun.set(root, now);
     }
     catch (error) {
         // The host treats a throwing listener as vetoing publication. Losing an index is a degradation,
@@ -242,6 +260,24 @@ function autoIndexOnSession(ctx, autoIndexed, payload) {
         console.error(`codegraph-tree-sitter: automatic indexing for a new session failed: ${String(error)}`);
     }
 }
+/**
+ * When the workspace's sidecar says its index was built.
+ * @param root - the workspace root.
+ * @returns the epoch-millisecond timestamp, or `undefined` when the sidecar is absent or unreadable —
+ *   either way the graph cannot be trusted to be current.
+ */
+function readStatusIndexedAt(root) {
+    try {
+        const parsed = JSON.parse(readFileSync(join(root, INDEX_STATUS_RELATIVE_PATH), 'utf8'));
+        if (typeof parsed !== 'object' || parsed === null)
+            return undefined;
+        const indexedAt = parsed.indexedAt;
+        return typeof indexedAt === 'number' && Number.isFinite(indexedAt) ? indexedAt : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
 /**
  * The producer body of one automatic indexing job.
  *
@@ -302,6 +338,7 @@ async function runIndex(projectRoot, config, fs, signal) {
         languages: config.languages,
         reuseParsedCache: config.reuseParsedCache,
         cacheVerify: config.cacheVerify,
+        engine: config.engine,
     }, fs, signal);
     signal?.throwIfAborted();
     const indexedAt = Date.now();

package/lib/rust-engine.d.ts

@@ -0,0 +1,104 @@
+/**
+ * The optional Rust extraction engine: a native addon built from `crates/indexer-rust`, used in place
+ * of this package's own `extractFile` for the four ECMAScript-family grammars.
+ *
+ * Everything here is optional by construction. The addon is a build artifact — this package's `files`
+ * list does not ship it, and a deployment without a Rust toolchain has no reason to produce one — so a
+ * missing artifact, a stale artifact, a thrown call, or an unparsable answer all degrade to "no Rust
+ * extraction". The walk then parses those files with `extractFile`, exactly as it would without this
+ * module; a slower index is acceptable, a failed one is not.
+ *
+ * The addon never touches the filesystem: it takes every file's text as an input and answers with
+ * JSON. That keeps the seam honest — filesystem policy (mtime, minified skips, cache reuse) stays in
+ * `walk.ts`, and this module only turns text into {@link FileExtraction}s.
+ * @module dsh-plugin-codegraph-tree-sitter/rust-engine
+ */
+import type { FileExtraction } from './extract.ts';
+/** Seam language labels the addon extracts. Every other language is left to `extractFile`. */
+export declare const RUST_LANGUAGES: readonly string[];
+/**
+ * Files handed to one addon call. The addon parses a batch across every core it can, so a large batch
+ * is what makes it fast — but the call itself blocks the event loop for its whole duration, so the
+ * batch is bounded to keep one call short enough that a watcher rebuild or a host request is not
+ * stalled behind it.
+ */
+export declare const RUST_BATCH_SIZE = 200;
+/** One file as the addon takes it. */
+export interface RustExtractInput {
+    /** Workspace-relative path, echoed back verbatim in the answer. */
+    readonly path: string;
+    /** Seam language label — one of {@link RUST_LANGUAGES} to be extracted. */
+    readonly language: string;
+    /** The file's full text. */
+    readonly text: string;
+}
+/**
+ * The addon's one exported entry point, as this package uses it. The real artifact also exports
+ * `__napiBindingTarget`, which nothing here reads.
+ */
+export interface RustEngine {
+    /**
+     * Extract definitions, calls, imports, and heritage for a batch of files.
+     * @param inputs - the files to extract; inputs in an unsupported language are silently skipped.
+     * @param rulesJson - the `LanguageSpec` table, as JSON.
+     * @returns JSON of `{ results: [{ path, extraction }], supported: string[] }`.
+     */
+    extractWorkspace(inputs: RustExtractInput[], rulesJson: string): string;
+}
+/** Options for {@link loadRustEngine}. */
+export interface LoadRustEngineOptions {
+    /**
+     * Load the addon from this path instead of the default artifact under `crates/indexer-rust`. Tests
+     * point it at a stand-in module or at a path that does not exist; production never sets it.
+     */
+    readonly modulePath?: string;
+}
+/**
+ * Every place the addon may live, most specific first.
+ *
+ * Two layouts matter: a package that ships the binary puts it in `artifacts/` beside `lib/` (the
+ * published shape, and what a profile install gets when someone copies it in), while a repository
+ * checkout builds it under `crates/indexer-rust/artifacts/`, three directories up from both `src/`
+ * and `lib/` since those sit at the same depth. Trying both means one module works whether it was
+ * installed or checked out.
+ * @param base - the importing module's URL; parameterized so a test can point at a scratch tree.
+ * @returns absolute paths to try, in order.
+ */
+export declare function candidateModulePaths(base?: string): string[];
+/**
+ * Load the native addon, or report that it is unavailable.
+ *
+ * Never throws and never repeats a failure: the first failure is logged once (see
+ * {@link reportUnavailable}), later calls answer `undefined` without touching the filesystem again.
+ * @param options - see {@link LoadRustEngineOptions}; production passes none.
+ * @returns the loaded addon, or `undefined` when this deployment has no usable one.
+ */
+export declare function loadRustEngine(options?: LoadRustEngineOptions): RustEngine | undefined;
+/**
+ * Extract a batch of files with the addon, in at most {@link RUST_BATCH_SIZE} files per call.
+ *
+ * A file the addon did not answer for is simply absent from the returned map: an unsupported
+ * language, a file it declines (a root parse error, whose extraction it will not vouch for), or one
+ * lost to a batch whose call failed. That is the contract the caller relies on — **per-path** absence
+ * means "extract this one with `extractFile`", not "discard the batch".
+ * `engine` is required rather than defaulted to {@link loadRustEngine}: `undefined` has to mean "no
+ * addon" here, and a default parameter would silently load one instead of accepting that answer — the
+ * caller that already resolved the engine (`walk.ts`) is the one that should decide.
+ * @param files - the files to extract, in any language; unsupported ones are dropped here.
+ * @param engine - the loaded addon, or `undefined` when this deployment has none.
+ * @returns one extraction per path the addon answered for, or `undefined` when no answer can be
+ *   trusted at all — no addon, a thrown call, or JSON that is not the documented shape.
+ */
+export declare function extractWithRust(files: readonly RustExtractInput[], engine: RustEngine | undefined): Map<string, FileExtraction> | undefined;
+/**
+ * Install a stand-in for the addon the walk resolves, or restore the real one.
+ *
+ * This exists for tests, and only for tests: the per-path fallback contract above is a promise about
+ * what happens when the addon *declines a file*, which the real artifact only does for its own
+ * reasons — a workspace with a syntactically broken file, at the time of writing. A test that needs
+ * that answer deterministically installs one here instead of depending on the artifact's current
+ * behaviour. Not exported from the package entry point; production code has no reason to call it.
+ * @param engine - the double to install, or `undefined` to let the next call load the real artifact.
+ */
+export declare function installRustEngineForTests(engine: RustEngine | undefined): void;
+//# sourceMappingURL=rust-engine.d.ts.map

package/lib/rust-engine.js

@@ -0,0 +1,201 @@
+/**
+ * The optional Rust extraction engine: a native addon built from `crates/indexer-rust`, used in place
+ * of this package's own `extractFile` for the four ECMAScript-family grammars.
+ *
+ * Everything here is optional by construction. The addon is a build artifact — this package's `files`
+ * list does not ship it, and a deployment without a Rust toolchain has no reason to produce one — so a
+ * missing artifact, a stale artifact, a thrown call, or an unparsable answer all degrade to "no Rust
+ * extraction". The walk then parses those files with `extractFile`, exactly as it would without this
+ * module; a slower index is acceptable, a failed one is not.
+ *
+ * The addon never touches the filesystem: it takes every file's text as an input and answers with
+ * JSON. That keeps the seam honest — filesystem policy (mtime, minified skips, cache reuse) stays in
+ * `walk.ts`, and this module only turns text into {@link FileExtraction}s.
+ * @module dsh-plugin-codegraph-tree-sitter/rust-engine
+ */
+import { createRequire } from 'node:module';
+import { fileURLToPath } from 'node:url';
+import { LANGUAGE_TABLE } from "./languages.js";
+/** Seam language labels the addon extracts. Every other language is left to `extractFile`. */
+export const RUST_LANGUAGES = ['typescript', 'tsx', 'javascript', 'jsx'];
+/**
+ * Files handed to one addon call. The addon parses a batch across every core it can, so a large batch
+ * is what makes it fast — but the call itself blocks the event loop for its whole duration, so the
+ * batch is bounded to keep one call short enough that a watcher rebuild or a host request is not
+ * stalled behind it.
+ */
+export const RUST_BATCH_SIZE = 200;
+const require = createRequire(import.meta.url);
+/**
+ * Every place the addon may live, most specific first.
+ *
+ * Two layouts matter: a package that ships the binary puts it in `artifacts/` beside `lib/` (the
+ * published shape, and what a profile install gets when someone copies it in), while a repository
+ * checkout builds it under `crates/indexer-rust/artifacts/`, three directories up from both `src/`
+ * and `lib/` since those sit at the same depth. Trying both means one module works whether it was
+ * installed or checked out.
+ * @param base - the importing module's URL; parameterized so a test can point at a scratch tree.
+ * @returns absolute paths to try, in order.
+ */
+export function candidateModulePaths(base = import.meta.url) {
+    const artifact = `codegraph-indexer.${process.platform}-${process.arch}.node`;
+    return [
+        fileURLToPath(new URL(`../artifacts/${artifact}`, base)),
+        fileURLToPath(new URL(`../../../crates/indexer-rust/artifacts/${artifact}`, base)),
+    ];
+}
+/** Addons already loaded, keyed by path, so a test's stand-in never shadows the real artifact. */
+const loaded = new Map();
+/**
+ * Paths whose load already failed. Kept so a workspace without the artifact pays one failed `require`
+ * for the process rather than one per walk.
+ */
+const failed = new Set();
+/** Whether this process has already said why the addon is unusable — see {@link reportUnavailable}. */
+let reportedFailure = false;
+/**
+ * Say once, per process, why the addon could not be used. Never once per file and never once per
+ * walk: a missing optional artifact must not turn into a console flood, but it must not be silent
+ * either — otherwise a deployment that expected the faster engine has no way to notice it is not
+ * running.
+ * @param reason - what went wrong, in one phrase the reader can act on.
+ */
+function reportUnavailable(reason) {
+    if (reportedFailure)
+        return;
+    reportedFailure = true;
+    console.error(`codegraph-tree-sitter: Rust extraction unavailable (${reason}); falling back to the JavaScript extractor`);
+}
+/** Whether `value` is a plain object, which is all the JSON and module shapes checked here ever are. */
+function isRecord(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+/**
+ * Whether a required module is the addon this package expects. A stale or mismatched artifact is a
+ * load failure like any other: it must fall back rather than throw on the first batch call.
+ * @param value - whatever `require` returned for the artifact path.
+ */
+function isRustEngine(value) {
+    return isRecord(value) && typeof value.extractWorkspace === 'function';
+}
+/**
+ * Load the native addon, or report that it is unavailable.
+ *
+ * Never throws and never repeats a failure: the first failure is logged once (see
+ * {@link reportUnavailable}), later calls answer `undefined` without touching the filesystem again.
+ * @param options - see {@link LoadRustEngineOptions}; production passes none.
+ * @returns the loaded addon, or `undefined` when this deployment has no usable one.
+ */
+export function loadRustEngine(options) {
+    const candidates = options?.modulePath === undefined ? candidateModulePaths() : [options.modulePath];
+    // Local, not module-level: a second call whose candidates are all already cached as failed must not
+    // resurrect the previous call's reason and try to report it again.
+    let lastFailure;
+    // Both caches are consulted before any `require`: a successful load is the process's engine, and a
+    // failed path is not worth retrying on the next walk.
+    for (const modulePath of candidates) {
+        const cached = loaded.get(modulePath);
+        if (cached !== undefined)
+            return cached;
+        if (failed.has(modulePath))
+            continue;
+        let addon;
+        try {
+            addon = require(modulePath);
+        }
+        catch (error) {
+            failed.add(modulePath);
+            lastFailure = `${modulePath}: ${String(error)}`;
+            continue;
+        }
+        if (!isRustEngine(addon)) {
+            failed.add(modulePath);
+            lastFailure = `${modulePath}: no extractWorkspace export`;
+            continue;
+        }
+        loaded.set(modulePath, addon);
+        return addon;
+    }
+    // Reported only once every candidate has been tried, so an installed package does not complain
+    // about the checkout path it was never going to have.
+    if (lastFailure !== undefined)
+        reportUnavailable(lastFailure);
+    return undefined;
+}
+/** The rules the addon extracts by, as JSON — built once from {@link LANGUAGE_TABLE}, then reused. */
+let rulesJson;
+/**
+ * The addon's rule table: this package's own `LANGUAGE_TABLE` entries for the languages the addon
+ * handles, so the two extractors can never disagree about what a rule matches. The table stays the
+ * single source of truth — the crate keeps no copy of it.
+ * @returns the `LanguageSpec[]` the addon parses, JSON-encoded.
+ */
+function rustRulesJson() {
+    rulesJson ??= JSON.stringify(LANGUAGE_TABLE.filter(spec => RUST_LANGUAGES.includes(spec.language)));
+    return rulesJson;
+}
+/**
+ * Extract a batch of files with the addon, in at most {@link RUST_BATCH_SIZE} files per call.
+ *
+ * A file the addon did not answer for is simply absent from the returned map: an unsupported
+ * language, a file it declines (a root parse error, whose extraction it will not vouch for), or one
+ * lost to a batch whose call failed. That is the contract the caller relies on — **per-path** absence
+ * means "extract this one with `extractFile`", not "discard the batch".
+ * `engine` is required rather than defaulted to {@link loadRustEngine}: `undefined` has to mean "no
+ * addon" here, and a default parameter would silently load one instead of accepting that answer — the
+ * caller that already resolved the engine (`walk.ts`) is the one that should decide.
+ * @param files - the files to extract, in any language; unsupported ones are dropped here.
+ * @param engine - the loaded addon, or `undefined` when this deployment has none.
+ * @returns one extraction per path the addon answered for, or `undefined` when no answer can be
+ *   trusted at all — no addon, a thrown call, or JSON that is not the documented shape.
+ */
+export function extractWithRust(files, engine) {
+    const supported = files.filter(file => RUST_LANGUAGES.includes(file.language));
+    if (supported.length === 0)
+        return new Map();
+    if (engine === undefined)
+        return undefined;
+    const extractions = new Map();
+    try {
+        for (let start = 0; start < supported.length; start += RUST_BATCH_SIZE) {
+            const batch = supported.slice(start, start + RUST_BATCH_SIZE);
+            const answer = JSON.parse(engine.extractWorkspace(batch, rustRulesJson()));
+            if (!isRecord(answer) || !Array.isArray(answer.results))
+                return undefined;
+            for (const entry of answer.results) {
+                // A malformed entry is treated as a path the addon did not answer for: the caller falls back
+                // for it, and a bad answer for one file never costs the rest of the batch its extraction.
+                if (!isRecord(entry) || typeof entry.path !== 'string' || !isRecord(entry.extraction))
+                    continue;
+                // The wire shape is `FileExtraction` field for field — camelCase `Raw*` fields, arrays of the
+                // same objects — which `rust-parity.spec.ts` pins with `toStrictEqual` against `extractFile`.
+                extractions.set(entry.path, entry.extraction);
+            }
+        }
+    }
+    catch (error) {
+        reportUnavailable(`extractWorkspace failed: ${String(error)}`);
+        return undefined;
+    }
+    return extractions;
+}
+/**
+ * Install a stand-in for the addon the walk resolves, or restore the real one.
+ *
+ * This exists for tests, and only for tests: the per-path fallback contract above is a promise about
+ * what happens when the addon *declines a file*, which the real artifact only does for its own
+ * reasons — a workspace with a syntactically broken file, at the time of writing. A test that needs
+ * that answer deterministically installs one here instead of depending on the artifact's current
+ * behaviour. Not exported from the package entry point; production code has no reason to call it.
+ * @param engine - the double to install, or `undefined` to let the next call load the real artifact.
+ */
+export function installRustEngineForTests(engine) {
+    // Every candidate, so a stand-in covers whichever layout this checkout resolves to.
+    for (const modulePath of candidateModulePaths()) {
+        if (engine === undefined)
+            loaded.delete(modulePath);
+        else
+            loaded.set(modulePath, engine);
+    }
+}
+//# sourceMappingURL=rust-engine.js.map

package/lib/walk.d.ts

@@ -22,6 +22,16 @@ import type { ExtractedFile } from './resolve.ts';
  * @param gitignoreRules - the project's parsed `.gitignore`, or empty when not respected.
  */
 export declare function isExcluded(name: string, relativePath: string, isDirectory: boolean, excluded: ReadonlySet<string>, gitignoreRules: readonly GitignoreRule[]): boolean;
+/**
+ * Which extractor parses a candidate file.
+ *
+ * `'js'` is this package's own `extractFile` for everything — the pre-engine behaviour, kept as an
+ * exact escape hatch. `'rust'` prefers the native addon. `'auto'` is `'rust'` when the artifact loads
+ * and `'js'` otherwise, which is what a deployment wants: the faster path when it exists, never a
+ * failure when it does not. In both non-`'js'` modes a language the addon does not handle — and any
+ * single file it declines — is still parsed by `extractFile`.
+ */
+export type EngineMode = 'auto' | 'js' | 'rust';
 /** The walk's tunable bounds, mirroring this package's `Config`. */
 export interface WalkConfig {
     /** Directory segment names never descended into. */
@@ -34,6 +44,13 @@ export interface WalkConfig {
     readonly concurrency: number;
     /** Restrict extraction to these seam language labels when set; every loaded grammar otherwise. */
     readonly languages?: readonly string[];
+    /**
+     * Which extractor parses a candidate — see {@link EngineMode}. Absent means `'auto'`. The choice
+     * never changes the files that are indexed or the answer's shape: both extractors produce the same
+     * {@link FileExtraction}, which `rust-parity.spec.ts` pins, so this only trades speed against
+     * leaving the process's own parser out of the run.
+     */
+    readonly engine?: EngineMode;
     /**
      * Leave generated (bundled or minified) artifacts out of the index instead of parsing them. See
      * `./minified.ts` for the two signals; a workspace that commits its build output is the case this
@@ -88,6 +105,11 @@ export interface WalkResult {
  * is the identity every cache entry is recorded under — and the only seam-mediated work here is the
  * parse cache's read. A capability is not a bound, so it does not belong in the config that mirrors
  * this package's `Config`.
+ *
+ * Every candidate is read and decided first, then the files needing extraction go to the native
+ * engine in batches with `extractFile` as the fallback — see {@link WalkConfig.engine}. The two phases
+ * exist so one addon call can cover a whole workspace; neither changes what a per-file decision
+ * (size, minified, cache) does.
  * @param projectRoot - absolute path of the workspace to index.
  * @param config - the walk's bounds.
  * @param fs - the filesystem seam the parse cache is read through.

package/lib/walk.js

@@ -17,6 +17,7 @@ import { extractFile } from "./extract.js";
 import { loadGitignore, matchesGitignore } from "./gitignore.js";
 import { languageFor } from "./languages.js";
 import { isMinifiedName, looksMinified, measureText } from "./minified.js";
+import { extractWithRust, loadRustEngine } from "./rust-engine.js";
 /**
  * Whether one directory entry is excluded from both the index and (for `watcher.ts`) the watch tree —
  * the single definition both consult, so "what the index covers" and "what gets watched" can never
@@ -97,15 +98,22 @@ function reusedFile(path, cached, stats) {
     };
 }
 /**
- * Parse one candidate, or reuse the extraction a previous run cached for it.
- * @param candidate - the file to parse.
+ * Read one candidate's text and apply every decision that does not need an extractor, or reuse the
+ * extraction a previous run cached for it.
+ *
+ * No parsing happens here on purpose: a fresh file is reported as {@link PendingFile} so that
+ * `walkAndExtract` can batch all of them into the native engine at once. That split is why the cache
+ * checks, the size bound, and the minified decision must stay in this function — they are the per-file
+ * policy the batch above them knows nothing about.
+ * @param candidate - the file to read.
  * @param config - the walk's bounds.
  * @param cache - the previous run's extractions, keyed by project-relative path.
- * @param signal - aborts the parse.
- * @returns the extracted file (and whether it came from the cache), `'minified'` when it was skipped
- *   as generated output, or `'skipped'` when it is over {@link WalkConfig.maxFileBytes}.
+ * @param signal - aborts the read.
+ * @returns the record a previous run cached for this file, `'minified'` when it was skipped as
+ *   generated output, `'skipped'` when it is over {@link WalkConfig.maxFileBytes}, or the pending file
+ *   an extractor still has to handle.
  */
-async function parseCandidate(candidate, config, cache, signal) {
+async function prepareCandidate(candidate, config, cache, signal) {
     // Checked before the stat and read: a `.min.` name settles it with no I/O at all.
     if (config.skipMinified && isMinifiedName(candidate.path))
         return { outcome: 'minified' };
@@ -123,7 +131,7 @@ async function parseCandidate(candidate, config, cache, signal) {
     if (cached !== undefined) {
         if (config.cacheVerify === 'stat') {
             if (cached.modifiedAt === stats.mtimeMs && cached.size === stats.size) {
-                return { outcome: 'extracted', file: reusedFile(candidate.path, cached, stats), reused: true };
+                return { outcome: 'cached', file: reusedFile(candidate.path, cached, stats) };
             }
         }
         else {
@@ -131,41 +139,119 @@ async function parseCandidate(candidate, config, cache, signal) {
             // the parse below would need — a miss here costs nothing extra.
             text = await readFile(candidate.absolute, 'utf8');
             if (createHash('sha256').update(text).digest('hex') === cached.contentHash) {
-                return { outcome: 'extracted', file: reusedFile(candidate.path, cached, stats), reused: true };
+                return { outcome: 'cached', file: reusedFile(candidate.path, cached, stats) };
             }
         }
     }
     text ??= await readFile(candidate.absolute, 'utf8');
+    signal?.throwIfAborted();
     const shape = measureText(text);
     if (config.skipMinified && looksMinified(candidate.path, shape))
         return { outcome: 'minified' };
-    const parser = await createParser(spec);
+    // Everything above is filesystem policy; everything below is extraction, and extraction is the one
+    // step that can run on the native engine — so this is exactly where the candidate waits for one.
+    return {
+        outcome: 'pending',
+        file: {
+            path: candidate.path,
+            language: spec.language,
+            spec,
+            text,
+            size: stats.size,
+            modifiedAt: stats.mtimeMs,
+            lineCount: shape.lineCount,
+        },
+    };
+}
+/**
+ * Parse one pending file with this package's own extractor.
+ * @param file - the file, whose text is already in memory.
+ * @param signal - aborts the parse.
+ * @returns what the parser found.
+ */
+async function extractWithJs(file, signal) {
+    const parser = await createParser(file.spec);
     try {
         signal?.throwIfAborted();
-        const tree = parser.parse(text);
+        const tree = parser.parse(file.text);
         /* v8 ignore next 2 -- createParser() always assigns a language before returning; parse() returns
          * null only when the parser has none. */
         if (tree === null)
-            return { outcome: 'skipped' };
-        const extraction = extractFile(tree, spec);
-        return {
-            outcome: 'extracted',
-            reused: false,
-            file: {
-                path: candidate.path,
-                language: spec.language,
-                size: stats.size,
-                modifiedAt: stats.mtimeMs,
-                contentHash: createHash('sha256').update(text).digest('hex'),
-                lineCount: shape.lineCount,
-                extraction,
-            },
-        };
+            return { definitions: [], calls: [], imports: [], heritage: [] };
+        return extractFile(tree, file.spec);
     }
     finally {
         parser.delete();
     }
 }
+/**
+ * The file record an extraction belongs to: the metadata `prepareCandidate` measured, plus a content
+ * hash of the text it read.
+ * @param file - the pending file this extraction came from.
+ * @param extraction - what an extractor found in it.
+ * @returns the file as the resolver, the parse cache, and the graph writer all expect it.
+ */
+function extractedFile(file, extraction) {
+    return {
+        path: file.path,
+        language: file.language,
+        size: file.size,
+        modifiedAt: file.modifiedAt,
+        contentHash: createHash('sha256').update(file.text).digest('hex'),
+        lineCount: file.lineCount,
+        extraction,
+    };
+}
+/**
+ * Extract every pending file, native engine first and `extractFile` for whatever it did not answer
+ * for, returning the results in the order they were given.
+ *
+ * The native call is one batch per {@link RUST_BATCH_SIZE} files (see `rust-engine.ts`); a file it
+ * did not answer for is parsed here instead. That fallback is per path, not per batch: one broken file
+ * costs one `extractFile` call, not the whole workspace's fast path.
+ * @param pending - the files waiting for an extractor, in discovery order.
+ * @param config - the walk's bounds, for the engine choice and the concurrency of the JS fallback.
+ * @param signal - aborts the extraction.
+ * @returns one record per pending file, in the same order.
+ */
+async function extractPending(pending, config, signal) {
+    if (pending.length === 0)
+        return [];
+    // `'js'` never even loads the addon: the mode exists to run exactly the pre-engine code path.
+    const engine = config.engine === 'js' ? undefined : loadRustEngine();
+    // Every pending file is offered to the engine, including the languages it does not handle: it drops
+    // those itself (`RUST_LANGUAGES`), so the walk never keeps a second copy of that list in sync.
+    const rusted = engine === undefined
+        ? new Map()
+        : extractWithRust(pending.map(toRustInput), engine) ?? new Map();
+    const extracted = new Array(pending.length);
+    const fallback = [];
+    for (const [index, file] of pending.entries()) {
+        const extraction = rusted.get(file.path);
+        // A hit is the addon's extraction; a miss is this one path falling back, for any of the reasons
+        // `rust-engine.ts` documents — never a reason to distrust the batch.
+        if (extraction === undefined)
+            fallback.push({ index, file });
+        else
+            extracted[index] = extractedFile(file, extraction);
+    }
+    const parsed = await mapWithConcurrency(fallback, config.concurrency, async (entry) => ({
+        index: entry.index,
+        file: extractedFile(entry.file, await extractWithJs(entry.file, signal)),
+    }));
+    for (const result of parsed)
+        extracted[result.index] = result.file;
+    return extracted;
+}
+/**
+ * One pending file as the native engine takes it: only the three fields it reads, so the walk's own
+ * bookkeeping (stat data, spec, line count) never crosses the boundary.
+ * @param file - the pending file.
+ * @returns the addon's input shape.
+ */
+function toRustInput(file) {
+    return { path: file.path, language: file.language, text: file.text };
+}
 /** Run `tasks` with at most `concurrency` in flight at once, preserving input order in the result. */
 async function mapWithConcurrency(items, concurrency, task) {
     const results = new Array(items.length);
@@ -196,6 +282,11 @@ async function mapWithConcurrency(items, concurrency, task) {
  * is the identity every cache entry is recorded under — and the only seam-mediated work here is the
  * parse cache's read. A capability is not a bound, so it does not belong in the config that mirrors
  * this package's `Config`.
+ *
+ * Every candidate is read and decided first, then the files needing extraction go to the native
+ * engine in batches with `extractFile` as the fallback — see {@link WalkConfig.engine}. The two phases
+ * exist so one addon call can cover a whole workspace; neither changes what a per-file decision
+ * (size, minified, cache) does.
  * @param projectRoot - absolute path of the workspace to index.
  * @param config - the walk's bounds.
  * @param fs - the filesystem seam the parse cache is read through.
@@ -206,21 +297,38 @@ async function mapWithConcurrency(items, concurrency, task) {
 export async function walkAndExtract(projectRoot, config, fs, signal) {
     const { candidates, overflow } = await discover(projectRoot, config, signal);
     const cache = config.reuseParsedCache ? await readParseCache(fs, projectRoot) : new Map();
-    const parsed = await mapWithConcurrency(candidates, config.concurrency, candidate => parseCandidate(candidate, config, cache, signal));
-    const files = [];
+    const prepared = await mapWithConcurrency(candidates, config.concurrency, candidate => prepareCandidate(candidate, config, cache, signal));
+    const pending = [];
     let filesSkipped = overflow;
     let filesMinified = 0;
-    let filesReused = 0;
-    for (const result of parsed) {
+    for (const result of prepared) {
         if (result.outcome === 'skipped')
             filesSkipped++;
         else if (result.outcome === 'minified')
             filesMinified++;
-        else {
-            files.push(result.file);
-            if (result.reused)
-                filesReused++;
+        else if (result.outcome === 'pending')
+            pending.push(result.file);
+    }
+    const extracted = await extractPending(pending, config, signal);
+    // One pass in discovery order: a cache hit is already a record, a pending file takes the next record
+    // `extractPending` produced. Which engine extracted a file therefore never shows in the ordering.
+    const files = [];
+    let filesReused = 0;
+    let next = 0;
+    for (const result of prepared) {
+        if (result.outcome === 'skipped' || result.outcome === 'minified')
+            continue;
+        if (result.outcome === 'pending') {
+            const file = extracted[next];
+            next++;
+            /* v8 ignore next 2 -- extractPending() returns exactly one record per pending file, in order. */
+            if (file === undefined)
+                continue;
+            files.push(file);
+            continue;
         }
+        files.push(result.file);
+        filesReused++;
     }
     return { files, filesSkipped, filesMinified, filesReused };
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dsh-plugin-codegraph-tree-sitter",
-  "version": "0.4.2",
+  "version": "0.5.1",
   "description": "Self-built code-graph indexer for DeepSeek Harness — parses a workspace with web-tree-sitter and writes schema version 4 to .codegraph/codegraph.db, the same on-disk format the codegraph CLI writes, so no external indexer is required",
   "keywords": [
     "dsh",
@@ -51,7 +51,7 @@
     "@deepseek-ai/schemastery": "^3.18.1",
     "tree-sitter-wasms": "^0.1.13",
     "web-tree-sitter": "^0.25.10",
-    "dsh-plugin-codegraph-service": "^0.4.2"
+    "dsh-plugin-codegraph-service": "^0.5.1"
   },
   "peerDependencies": {
     "@deepseek-ai/cordis": ">=4.0.1",