nexus-prime 7.2.0 → 7.3.1

package/dist/engines/ngram-index.js

@@ -40,6 +40,26 @@ function safeStatSize(filePath) {
     }
 }
 // ─────────────────────────────────────────────────────────────────────────────
+// SQLite WAL footprint helpers — used by quota guard, rotation, and cleanup.
+// SQLite databases are db + db-wal + db-shm. Any size accounting that ignores
+// wal/shm misses the bug that turned ngram-index.db-wal into 84GB on disk.
+// ─────────────────────────────────────────────────────────────────────────────
+const NGRAM_DEFAULT_WAL_LIMIT_BYTES = 64 * 1024 * 1024; // 64 MB
+const NGRAM_DEFAULT_FOOTPRINT_BYTES = 512 * 1024 * 1024; // 512 MB
+const NGRAM_DEFAULT_CHECKPOINT_INTERVAL_MS = 30_000;
+const NGRAM_DEFAULT_CHECKPOINT_DOC_COUNT = 200;
+export function getNgramWalPath(dbPath) {
+    return `${dbPath}-wal`;
+}
+export function getNgramShmPath(dbPath) {
+    return `${dbPath}-shm`;
+}
+export function getNgramFootprintBytes(dbPath) {
+    return safeStatSize(dbPath)
+        + safeStatSize(getNgramWalPath(dbPath))
+        + safeStatSize(getNgramShmPath(dbPath));
+}
+// ─────────────────────────────────────────────────────────────────────────────
 // Character pair frequency weights (precomputed from common code patterns)
 // Rarer pairs get higher weights for sparse n-gram extraction.
 // ─────────────────────────────────────────────────────────────────────────────
@@ -157,6 +177,10 @@ export class NgramIndex {
     deleteStmt;
     lookupStmt;
     docExistsStmt;
+    // Write-side checkpoint accounting (Phase 2): bound WAL growth on hot indexers.
+    writesSinceCheckpoint = 0;
+    lastCheckpointAt = 0;
+    quotaSkipNoticeShown = false;
     // Search analytics
     _searchStats = {
         totalQueries: 0,
@@ -190,36 +214,69 @@ export class NgramIndex {
         }
     }
     rotateOversizeDbIfNeeded() {
+        // Count the full SQLite footprint (db + wal + shm). The 84GB regression
+        // happened because a 32MB db file had a 84GB -wal sibling that this
+        // routine never inspected.
         const rotateBytes = readEnvBytes('NEXUS_NGRAM_ROTATE_BYTES', 1024 * 1024 * 1024); // 1GB default
-        const sizeBytes = safeStatSize(this.dbPath);
-        if (sizeBytes <= 0 || sizeBytes < rotateBytes)
+        const dbBytes = safeStatSize(this.dbPath);
+        const footprint = getNgramFootprintBytes(this.dbPath);
+        if (footprint <= 0 || footprint < rotateBytes)
             return;
-        // Keep only the most recent oversize archive to bound disk usage.
+        // Keep only the most recent oversize archive to bound disk usage. When the
+        // operator opts out of archiving (ARCHIVE_OVERSIZE=0), drop the leftovers
+        // outright instead of growing a backlog of multi-GB carcasses.
         const dir = path.dirname(this.dbPath);
         const base = path.basename(this.dbPath);
+        const archiveEnabled = process.env.NEXUS_NGRAM_ARCHIVE_OVERSIZE !== '0';
         const existing = fs.existsSync(dir)
             ? fs.readdirSync(dir).filter((entry) => entry.startsWith(`${base}.oversize.`)).sort().reverse()
             : [];
-        for (const entry of existing.slice(1)) {
+        const keepCount = archiveEnabled ? 1 : 0;
+        for (const entry of existing.slice(keepCount)) {
             try {
                 fs.rmSync(path.join(dir, entry), { force: true });
             }
             catch { /* best effort */ }
         }
-        const rotatedPath = `${this.dbPath}.oversize.${Date.now()}`;
+        const stamp = Date.now();
+        const rotatedPath = `${this.dbPath}.oversize.${stamp}`;
+        const removeSibling = (suffix) => {
+            try {
+                fs.rmSync(`${this.dbPath}${suffix}`, { force: true });
+            }
+            catch { /* best effort */ }
+        };
         try {
-            fs.renameSync(this.dbPath, rotatedPath);
-            logNgramNoticeOnce(`ngram:rotated:${this.dbPath}`, `[NgramIndex] rotated oversized DB (${Math.round(sizeBytes / 1024 / 1024)}MB >= ${Math.round(rotateBytes / 1024 / 1024)}MB) db=${this.dbPath} rotated=${rotatedPath}`);
+            if (archiveEnabled) {
+                fs.renameSync(this.dbPath, rotatedPath);
+            }
+            else {
+                fs.rmSync(this.dbPath, { force: true });
+            }
+            // db is gone; the WAL/SHM siblings are stale and unsafe to keep — drop
+            // them so SQLite never re-attaches them on the next open.
+            removeSibling('-wal');
+            removeSibling('-shm');
+            logNgramNoticeOnce(`ngram:rotated:${this.dbPath}:${stamp}`, `[NgramIndex] rotated oversized DB (footprint=${Math.round(footprint / 1024 / 1024)}MB db=${Math.round(dbBytes / 1024 / 1024)}MB threshold=${Math.round(rotateBytes / 1024 / 1024)}MB archive=${archiveEnabled ? 'on' : 'off'}) db=${this.dbPath}${archiveEnabled ? ` rotated=${rotatedPath}` : ''}`);
         }
         catch (err) {
             logNgramNoticeOnce(`ngram:rotate-failed:${this.dbPath}`, `[NgramIndex] oversize rotation failed; continuing with existing DB db=${this.dbPath} err=${String(err?.message ?? err)}`);
         }
     }
     initSchema() {
+        const walLimitBytes = readEnvBytes('NEXUS_NGRAM_WAL_LIMIT_BYTES', NGRAM_DEFAULT_WAL_LIMIT_BYTES);
         this.db.pragma('journal_mode = WAL');
         this.db.pragma('synchronous = NORMAL');
         this.db.pragma('cache_size = -16000');
         this.db.pragma('busy_timeout = 5000');
+        // Bound WAL growth: autocheckpoint every N pages and hard-cap journal size
+        // so it never balloons (the 84GB regression). temp_store=MEMORY avoids
+        // spilling sort/temp tables into another on-disk file. mmap_size=0
+        // disables the OS-mapped read cache that can keep pages alive on small VMs.
+        this.db.pragma('wal_autocheckpoint = 1000');
+        this.db.pragma(`journal_size_limit = ${walLimitBytes}`);
+        this.db.pragma('temp_store = MEMORY');
+        this.db.pragma('mmap_size = 0');
         this.db.exec(`
       CREATE TABLE IF NOT EXISTS ngram_postings (
         ngram_hash INTEGER NOT NULL,
@@ -288,8 +345,62 @@ export class NgramIndex {
         }
     }
     // ── Document Management ─────────────────────────────────────────────────
+    /** Run an opportunistic WAL checkpoint. PASSIVE first, escalate to TRUNCATE
+     *  if the WAL is still over its configured limit. Returns the WAL byte size
+     *  observed after the attempt so callers can report progress. */
+    checkpointIfNeeded(reason) {
+        const walLimit = readEnvBytes('NEXUS_NGRAM_WAL_LIMIT_BYTES', NGRAM_DEFAULT_WAL_LIMIT_BYTES);
+        try {
+            this.db.exec('PRAGMA wal_checkpoint(PASSIVE)');
+        }
+        catch {
+            // best effort
+        }
+        let walBytes = this.getWalBytes();
+        if (walBytes > walLimit || reason !== 'periodic') {
+            try {
+                this.db.exec('PRAGMA wal_checkpoint(TRUNCATE)');
+            }
+            catch {
+                // best effort
+            }
+            walBytes = this.getWalBytes();
+        }
+        this.writesSinceCheckpoint = 0;
+        this.lastCheckpointAt = Date.now();
+        return walBytes;
+    }
+    /** Footprint quota check before a write. Returns true if the write should
+     *  proceed; false if it should be skipped. Throws when NEXUS_NGRAM_STRICT_QUOTA=1
+     *  so callers can surface hard quota failures during tests/CI. */
+    allowWrite() {
+        const walLimit = readEnvBytes('NEXUS_NGRAM_WAL_LIMIT_BYTES', NGRAM_DEFAULT_WAL_LIMIT_BYTES);
+        const maxFootprint = readEnvBytes('NEXUS_NGRAM_MAX_FOOTPRINT_BYTES', NGRAM_DEFAULT_FOOTPRINT_BYTES);
+        const strict = process.env.NEXUS_NGRAM_STRICT_QUOTA === '1';
+        let walBytes = this.getWalBytes();
+        if (walBytes > walLimit) {
+            walBytes = this.checkpointIfNeeded('quota');
+        }
+        const footprint = this.getSqliteFootprintBytes();
+        if (footprint > maxFootprint) {
+            const msg = `[NgramIndex] footprint quota exceeded — db=${this.dbPath} footprint=${footprint} cap=${maxFootprint} wal=${walBytes}`;
+            if (strict) {
+                throw new Error(msg);
+            }
+            if (!this.quotaSkipNoticeShown) {
+                this.quotaSkipNoticeShown = true;
+                logNgramNoticeOnce(`ngram:quota-skip:${this.dbPath}`, msg);
+            }
+            return false;
+        }
+        // Recovered — re-arm the warning so a later breach prints again.
+        this.quotaSkipNoticeShown = false;
+        return true;
+    }
     /** Index a document's text content */
     addDocument(docId, text) {
+        if (!this.allowWrite())
+            return;
         // Remove existing postings for this doc (idempotent)
         this.deleteStmt.run(docId);
         const trigrams = extractTrigrams(text);
@@ -319,6 +430,19 @@ export class NgramIndex {
       `).run(docId, text.length, Date.now());
         });
         insertMany();
+        this.maybePeriodicCheckpoint();
+    }
+    /** Trigger a passive checkpoint when either the doc or time interval threshold
+     *  is hit. Cheap on a healthy DB, prevents WAL growth on bulk indexing. */
+    maybePeriodicCheckpoint() {
+        this.writesSinceCheckpoint += 1;
+        const intervalMs = readEnvBytes('NEXUS_NGRAM_CHECKPOINT_INTERVAL_MS', NGRAM_DEFAULT_CHECKPOINT_INTERVAL_MS);
+        const docThreshold = readEnvBytes('NEXUS_NGRAM_CHECKPOINT_DOCS', NGRAM_DEFAULT_CHECKPOINT_DOC_COUNT);
+        const dueByDocs = this.writesSinceCheckpoint >= docThreshold;
+        const dueByTime = Date.now() - this.lastCheckpointAt > intervalMs;
+        if (!dueByDocs && !dueByTime)
+            return;
+        this.checkpointIfNeeded('periodic');
     }
     /** Remove a document from the index */
     removeDocument(docId) {
@@ -337,6 +461,27 @@ export class NgramIndex {
         const row = this.db.prepare('SELECT COUNT(*) as cnt FROM ngram_docs').get();
         return row.cnt;
     }
+    // ── Footprint accessors ─────────────────────────────────────────────────
+    /** Path to the underlying SQLite database file. */
+    getDbPath() {
+        return this.dbPath;
+    }
+    /** Bytes occupied by the .db file alone. */
+    getDbBytes() {
+        return safeStatSize(this.dbPath);
+    }
+    /** Bytes occupied by the .db-wal file. */
+    getWalBytes() {
+        return safeStatSize(getNgramWalPath(this.dbPath));
+    }
+    /** Bytes occupied by the .db-shm file. */
+    getShmBytes() {
+        return safeStatSize(getNgramShmPath(this.dbPath));
+    }
+    /** Total SQLite footprint = db + wal + shm. Use this for quota checks. */
+    getSqliteFootprintBytes() {
+        return getNgramFootprintBytes(this.dbPath);
+    }
     // ── Search ──────────────────────────────────────────────────────────────
     /**
      * Search for documents matching a text query.
@@ -522,15 +667,24 @@ export class NgramIndex {
     }
     /**
      * Operator-focused maintenance for the on-disk ngram DB.
-     * - Bounds runaway DB growth via rotation (default >= 1GB)
-     * - Vacuums only when safe (<= vacuumMaxBytes) and either forced or dirty
+     * - Bounds runaway DB growth via rotation (default >= 1GB), counting the
+     *   full SQLite footprint (db + wal + shm) so a runaway WAL triggers it.
+     * - Vacuums only when safe (<= vacuumMaxBytes) and either forced or dirty.
      */
     maintainBounded(options = {}) {
-        const sizeBytes = safeStatSize(this.dbPath);
+        const dbBytes = safeStatSize(this.dbPath);
+        const footprint = this.getSqliteFootprintBytes();
         const rotateBytes = readEnvBytes('NEXUS_NGRAM_ROTATE_BYTES', 1024 * 1024 * 1024);
         const vacuumMaxBytes = readEnvBytes('NEXUS_NGRAM_VACUUM_MAX_BYTES', 256 * 1024 * 1024);
-        if (sizeBytes >= rotateBytes && sizeBytes > 0) {
+        if (footprint >= rotateBytes && footprint > 0) {
+            const archiveEnabled = process.env.NEXUS_NGRAM_ARCHIVE_OVERSIZE !== '0';
             const rotatedPath = `${this.dbPath}.oversize.${Date.now()}`;
+            const removeSibling = (suffix) => {
+                try {
+                    fs.rmSync(`${this.dbPath}${suffix}`, { force: true });
+                }
+                catch { /* best effort */ }
+            };
             try {
                 try {
                     this.db.exec('PRAGMA wal_checkpoint(TRUNCATE)');
@@ -540,7 +694,14 @@ export class NgramIndex {
                     this.db.close();
                 }
                 catch { /* ignore */ }
-                fs.renameSync(this.dbPath, rotatedPath);
+                if (archiveEnabled) {
+                    fs.renameSync(this.dbPath, rotatedPath);
+                }
+                else {
+                    fs.rmSync(this.dbPath, { force: true });
+                }
+                removeSibling('-wal');
+                removeSibling('-shm');
             }
             catch (err) {
                 // Re-open if close succeeded but rename failed
@@ -553,13 +714,16 @@ export class NgramIndex {
                 return {
                     action: 'none',
                     dbPath: this.dbPath,
-                    sizeBytes,
+                    sizeBytes: footprint,
                     reason: `rotate failed: ${String(err?.message ?? err)}`,
                 };
             }
             // Recreate a fresh DB
             this.knownHashes.clear();
             this.storageDirty = false;
+            this.writesSinceCheckpoint = 0;
+            this.lastCheckpointAt = Date.now();
+            this.quotaSkipNoticeShown = false;
             this.db = new Database(this.dbPath);
             this.initSchema();
             this.prepareStatements();
@@ -567,26 +731,26 @@ export class NgramIndex {
             return {
                 action: 'rotated',
                 dbPath: this.dbPath,
-                sizeBytes,
-                previousPath: rotatedPath,
-                reason: `db exceeded rotate threshold (${rotateBytes} bytes)`,
+                sizeBytes: footprint,
+                previousPath: archiveEnabled ? rotatedPath : undefined,
+                reason: `footprint exceeded rotate threshold (db=${dbBytes}, footprint=${footprint}, threshold=${rotateBytes})`,
             };
         }
-        if (sizeBytes > 0 && sizeBytes <= vacuumMaxBytes && (options.force || this.storageDirty)) {
+        if (dbBytes > 0 && dbBytes <= vacuumMaxBytes && (options.force || this.storageDirty)) {
             this.optimizeStorage(true);
             return {
                 action: 'vacuum',
                 dbPath: this.dbPath,
-                sizeBytes,
+                sizeBytes: dbBytes,
                 reason: options.force ? 'forced vacuum' : 'storageDirty vacuum',
             };
         }
         return {
             action: 'none',
             dbPath: this.dbPath,
-            sizeBytes,
-            reason: sizeBytes > vacuumMaxBytes
-                ? `skip vacuum: db too large (${sizeBytes} > ${vacuumMaxBytes})`
+            sizeBytes: footprint,
+            reason: dbBytes > vacuumMaxBytes
+                ? `skip vacuum: db too large (${dbBytes} > ${vacuumMaxBytes})`
                 : 'no maintenance needed',
         };
     }
@@ -607,9 +771,31 @@ export class NgramIndex {
             dbSizeBytes,
         };
     }
-    /** Close the database connection */
+    /** Close the database connection.
+     *
+     *  Always truncates the WAL so the .db-wal sibling can never outgrow the
+     *  configured cap on shutdown — the failure mode that produced the 84GB
+     *  WAL on a user machine. VACUUM is only run when the operator has both a
+     *  small DB and explicitly opts in via NEXUS_NGRAM_VACUUM_ON_CLOSE=1, since
+     *  blind VACUUM on multi-GB DBs blocks shutdown indefinitely. */
     close() {
-        this.optimizeStorage(this.storageDirty);
+        try {
+            this.db.exec('PRAGMA wal_checkpoint(TRUNCATE)');
+        }
+        catch {
+            // best effort — never block shutdown on checkpoint failure
+        }
+        if (process.env.NEXUS_NGRAM_VACUUM_ON_CLOSE === '1') {
+            const vacuumMaxBytes = readEnvBytes('NEXUS_NGRAM_VACUUM_MAX_BYTES', 256 * 1024 * 1024);
+            const dbBytes = safeStatSize(this.dbPath);
+            if (this.storageDirty && dbBytes > 0 && dbBytes <= vacuumMaxBytes) {
+                try {
+                    this.db.exec('VACUUM');
+                    this.storageDirty = false;
+                }
+                catch { /* best effort */ }
+            }
+        }
         this.db.close();
     }
 }

package/dist/engines/orchestrator/types.d.ts

@@ -267,4 +267,9 @@ export declare const MAX_DISCOVERED_FILES = 32;
 export declare const ABSOLUTE_SECONDARY_MIN = 0.3;
 export declare const DISCOVERY_EXTENSIONS: Set<string>;
 export declare const DISCOVERY_IGNORES: Set<string>;
+export declare const DISCOVERY_FILENAME_SKIPS: Set<string>;
+export declare const DISCOVERY_BINARY_EXTENSIONS: Set<string>;
 export declare const REPO_SEARCH_HEAD_BYTES = 2000;
+/** Default per-file byte cap for the ngram indexer. Override with
+ *  NEXUS_NGRAM_MAX_FILE_BYTES. Files above this are skipped (not truncated). */
+export declare const DISCOVERY_DEFAULT_MAX_FILE_BYTES = 256000;

package/dist/engines/orchestrator/types.js

@@ -7,6 +7,68 @@
 export const MAX_AUTONOMY_HISTORY = 24;
 export const MAX_DISCOVERED_FILES = 32;
 export const ABSOLUTE_SECONDARY_MIN = 0.3;
-export const DISCOVERY_EXTENSIONS = new Set(['.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs', '.json', '.md', '.yml', '.yaml']);
-export const DISCOVERY_IGNORES = new Set(['.git', 'node_modules', 'dist', 'coverage', '.next', '.playwright-cli', 'tmp', '.agents', '.agent']);
+// Source-like file extensions worth indexing. Covers JS/TS, Python, Go, Rust,
+// JVM langs, native, web, configs and prose. NEXUS_NGRAM_INDEX_ALL=1 disables
+// the allowlist (useful for diagnostics on non-mainstream repos).
+export const DISCOVERY_EXTENSIONS = new Set([
+    '.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs',
+    '.py', '.go', '.rs', '.java', '.kt', '.swift',
+    '.cpp', '.cc', '.c', '.h', '.hpp', '.cs',
+    '.rb', '.php',
+    '.md', '.mdx',
+    '.json', '.yaml', '.yml', '.toml',
+    '.sql', '.sh',
+    '.css', '.scss', '.html',
+]);
+// Directories the indexer must skip — generated artefacts, dependency caches,
+// virtualenvs, build outputs, and Nexus's own working dirs.
+export const DISCOVERY_IGNORES = new Set([
+    '.git',
+    'node_modules',
+    'dist',
+    'build',
+    'out',
+    'coverage',
+    '.next',
+    '.turbo',
+    '.playwright-cli',
+    '.cache',
+    'vendor',
+    'target',
+    '.venv',
+    '__pycache__',
+    'tmp',
+    '.agents',
+    '.agent',
+]);
+// Specific filenames to skip even when they match an allowed extension.
+// Lockfiles and large generated manifests blow up trigram indexes without
+// any retrieval value.
+export const DISCOVERY_FILENAME_SKIPS = new Set([
+    'package-lock.json',
+    'pnpm-lock.yaml',
+    'yarn.lock',
+    'bun.lockb',
+    'composer.lock',
+    'poetry.lock',
+    'Pipfile.lock',
+    'Cargo.lock',
+    'Gemfile.lock',
+    'go.sum',
+]);
+// Binary / media / archive extensions never to read into the indexer even
+// when the user passes NEXUS_NGRAM_INDEX_ALL=1 — these contain no useful
+// trigrams and inflate the WAL when accidentally included.
+export const DISCOVERY_BINARY_EXTENSIONS = new Set([
+    '.png', '.jpg', '.jpeg', '.gif', '.webp', '.bmp', '.ico', '.tiff',
+    '.mp3', '.mp4', '.mov', '.avi', '.mkv', '.webm', '.wav', '.flac', '.ogg',
+    '.zip', '.tar', '.gz', '.tgz', '.bz2', '.xz', '.7z', '.rar',
+    '.pdf', '.psd', '.ai',
+    '.so', '.dll', '.dylib', '.a', '.o', '.node', '.exe', '.bin',
+    '.wasm', '.class', '.jar', '.pyc', '.pyo',
+    '.lockb',
+]);
 export const REPO_SEARCH_HEAD_BYTES = 2_000;
+/** Default per-file byte cap for the ngram indexer. Override with
+ *  NEXUS_NGRAM_MAX_FILE_BYTES. Files above this are skipped (not truncated). */
+export const DISCOVERY_DEFAULT_MAX_FILE_BYTES = 256_000;

package/dist/engines/orchestrator.js

@@ -38,7 +38,7 @@ import { computeFilePriors } from './priors/file-priors.js';
 import { GhostPass, createSubAgentRuntime, } from '../phantom/index.js';
 // ─── Types and constants (extracted to orchestrator/types.ts) ─────────────────
 export * from './orchestrator/types.js';
-import { MAX_AUTONOMY_HISTORY, MAX_DISCOVERED_FILES, DISCOVERY_EXTENSIONS, DISCOVERY_IGNORES, } from './orchestrator/types.js';
+import { MAX_AUTONOMY_HISTORY, MAX_DISCOVERED_FILES, DISCOVERY_EXTENSIONS, DISCOVERY_IGNORES, DISCOVERY_FILENAME_SKIPS, DISCOVERY_BINARY_EXTENSIONS, DISCOVERY_DEFAULT_MAX_FILE_BYTES, } from './orchestrator/types.js';
 // ─── Scoring / planning utilities (extracted to orchestrator/scoring.ts) ───────
 import { dedupeStrings, shortLabel, extractKeywords, scoreText, scorePath, decomposeTask as decomposeTaskFn, classifyIntent, buildTaskGraph, buildWorkerPlan, decideWorkers, determineMode, toSourceAwareTokenBudget, toRagCandidateStatus, toRagUsageSummary, buildArtifactOutcome, mergeArtifactOutcomeHistory, getArtifactHistoryRecord, getArtifactHistoryScore, deriveArtifactMemoryClass, deriveArtifactFreshness, buildRepoSearchDocument, readRepoSearchHead, extractRepoSearchTerms, } from './orchestrator/scoring.js';
 import { categoryFilter, emitFunnelStage } from './orchestrator/funnel.js';
@@ -2103,14 +2103,33 @@ export class OrchestratorEngine {
         // Cheap extension check first (no I/O), then bounded-parallel stat for
         // the size + isFile gate. Replaces a per-file synchronous statSync that
         // ran inside .filter() loops on the hot path. CLAUDE.md §2 hard rule #1.
-        const candidates = filePaths.filter((f) => DISCOVERY_EXTENSIONS.has(path.extname(f)));
+        //
+        // Indexing discipline: skip lockfiles and binary/media extensions even
+        // when NEXUS_NGRAM_INDEX_ALL=1 — these never produce useful trigrams and
+        // bloat the WAL on big monorepos.
+        const indexAll = process.env.NEXUS_NGRAM_INDEX_ALL === '1';
+        const rawCap = Number(process.env.NEXUS_NGRAM_MAX_FILE_BYTES);
+        const maxFileBytes = Number.isFinite(rawCap) && rawCap > 0
+            ? Math.floor(rawCap)
+            : DISCOVERY_DEFAULT_MAX_FILE_BYTES;
+        const candidates = filePaths.filter((f) => {
+            const base = path.basename(f);
+            if (DISCOVERY_FILENAME_SKIPS.has(base))
+                return false;
+            const ext = path.extname(f).toLowerCase();
+            if (DISCOVERY_BINARY_EXTENSIONS.has(ext))
+                return false;
+            if (indexAll)
+                return true;
+            return DISCOVERY_EXTENSIONS.has(ext);
+        });
         if (candidates.length === 0)
             return [];
         const limit = pLimit(8);
         const results = await Promise.all(candidates.map((f) => limit(async () => {
             try {
                 const stat = await fs.promises.stat(f);
-                return stat.isFile() && stat.size <= 200_000 ? f : null;
+                return stat.isFile() && stat.size <= maxFileBytes ? f : null;
             }
             catch {
                 return null;
@@ -2202,6 +2221,7 @@ export class OrchestratorEngine {
         // we don't need a separate stat per entry, with bounded parallel reads
         // via pLimit(8). Size filtering is done in a single second pass through
         // filterDiscoverableFiles. CLAUDE.md §2 hard rule #1.
+        const walkIndexAll = process.env.NEXUS_NGRAM_INDEX_ALL === '1';
         const limit = pLimit(8);
         const candidates = [];
         let queue = [root];
@@ -2228,7 +2248,12 @@ export class OrchestratorEngine {
                     }
                     if (!entry.isFile())
                         continue;
-                    if (!DISCOVERY_EXTENSIONS.has(path.extname(entry.name)))
+                    if (DISCOVERY_FILENAME_SKIPS.has(entry.name))
+                        continue;
+                    const ext = path.extname(entry.name).toLowerCase();
+                    if (DISCOVERY_BINARY_EXTENSIONS.has(ext))
+                        continue;
+                    if (!walkIndexAll && !DISCOVERY_EXTENSIONS.has(ext))
                         continue;
                     candidates.push(fullPath);
                 }

package/dist/engines/runtime-hygiene.d.ts

@@ -12,6 +12,10 @@ export interface HygieneReport {
     orphanWorktreesRemoved: number;
     /** Runs dirs swept due to budgets. */
     boundedRunsSweeps: number;
+    /** Orphan ngram-index sidecar files removed (-wal/-shm without .db). */
+    ngramSidecarOrphansRemoved: number;
+    /** ngram-index oversize archives pruned beyond keep=1. */
+    ngramArchivesPruned: number;
 }
 export declare function runStartupHygiene(input: {
     repoRoot: string;

package/dist/engines/runtime-hygiene.js

@@ -6,7 +6,7 @@ import { SessionDNAManager } from './session-dna.js';
 import { clearBootstrapReceipt, readBootstrapReceipt } from './bootstrap/bootstrap-registry.js';
 import { doctorGitWorktrees } from './worktree-health.js';
 import { sweepDirectory, sweepOrphanWorktrees } from '../install/fs-purge.js';
-import { getRuntimeTmpRoots, getWorktreeRoots } from '../install/state-locator.js';
+import { enumerateNgramArchives, getRuntimeTmpRoots, getWorktreeRoots } from '../install/state-locator.js';
 import { resolveWorktreeBudget, resolveRunsBudget } from '../cli/cleanup.js';
 function isOlderThan(target, maxAgeMs) {
     try {
@@ -140,6 +140,7 @@ export async function runStartupHygiene(input) {
             // best-effort
         }
     }
+    const { ngramSidecarOrphansRemoved, ngramArchivesPruned } = pruneNgramArtifacts(stateDir);
     return {
         mode,
         cleanedRuntimeRegistryEntries: Math.max(0, registryBefore - registryAfter),
@@ -150,5 +151,48 @@ export async function runStartupHygiene(input) {
         boundedWorktreeSweeps,
         orphanWorktreesRemoved,
         boundedRunsSweeps,
+        ngramSidecarOrphansRemoved,
+        ngramArchivesPruned,
     };
 }
+/**
+ * Best-effort ngram cleanup at startup:
+ *   - delete orphan `*.db-wal` / `*.db-shm` when the matching `.db` is missing
+ *   - keep at most one `ngram-index.db.oversize.*` archive
+ * The live DB is left alone — NgramIndex itself enforces WAL bounds via
+ * journal_size_limit + autocheckpoint when it opens.
+ */
+function pruneNgramArtifacts(stateDir) {
+    let ngramSidecarOrphansRemoved = 0;
+    let ngramArchivesPruned = 0;
+    if (!fs.existsSync(stateDir)) {
+        return { ngramSidecarOrphansRemoved, ngramArchivesPruned };
+    }
+    // Orphan sidecars: foo.db-wal / foo.db-shm with no foo.db.
+    try {
+        for (const entry of fs.readdirSync(stateDir)) {
+            if (!entry.endsWith('.db-wal') && !entry.endsWith('.db-shm'))
+                continue;
+            const dbBase = entry.replace(/-(?:wal|shm)$/, '');
+            if (fs.existsSync(path.join(stateDir, dbBase)))
+                continue;
+            if (safeUnlink(path.join(stateDir, entry)))
+                ngramSidecarOrphansRemoved += 1;
+        }
+    }
+    catch {
+        // best-effort
+    }
+    // Archive cap: keep newest, drop the rest.
+    try {
+        const archives = enumerateNgramArchives(stateDir).sort((a, b) => b.modifiedAt - a.modifiedAt);
+        for (const archive of archives.slice(1)) {
+            if (safeUnlink(archive.path))
+                ngramArchivesPruned += 1;
+        }
+    }
+    catch {
+        // best-effort
+    }
+    return { ngramSidecarOrphansRemoved, ngramArchivesPruned };
+}

package/dist/install/claude-code-hooks.d.ts

@@ -0,0 +1,33 @@
+/** Marker that identifies a hook entry as Nexus-owned. */
+export declare const NEXUS_HOOK_COMMAND_MARKER = "nexus-prime hook";
+interface HookCommand {
+    type: 'command';
+    command: string;
+    timeout?: number;
+}
+interface HookEntry {
+    matcher?: string;
+    hooks: HookCommand[];
+}
+/**
+ * Canonical Nexus hook spec. Edit here, not in callers.
+ * Timeouts protect Claude Code from a wedged Nexus process — leave them set.
+ */
+export declare function getNexusHookSpec(): Record<string, HookEntry[]>;
+export interface NexusHookWriteResult {
+    /** True when the file would be written (or was written). */
+    changed: boolean;
+    /** Hook event names that ended up with Nexus entries. */
+    events: string[];
+    /** Number of stale Nexus entries removed before re-adding. */
+    staleRemoved: number;
+}
+/**
+ * Idempotent writer for the Nexus hook block. Returns metadata so callers can
+ * report status (`installed | unchanged | dry-run`) without re-implementing
+ * the merge logic.
+ */
+export declare function writeNexusClaudeCodeHooks(settingsPath: string, options?: {
+    dryRun?: boolean;
+}): NexusHookWriteResult;
+export {};