gitnexus 1.6.11-rc.11 → 1.6.11-rc.13

package/dist/cli/i18n/en.d.ts

@@ -187,7 +187,7 @@ export declare const en: {
     readonly 'help.option.serve.host': "Bind address (default: 127.0.0.1, use 0.0.0.0 for remote access)";
     readonly 'help.option.mcp.http': "Serve MCP over HTTP instead of stdio (for remote clients)";
     readonly 'help.option.mcp.host': "HTTP bind address (only with --http). Default: 127.0.0.1 (loopback). Use 0.0.0.0 to expose to all interfaces.";
-    readonly 'help.option.mcp.authToken': "Require this bearer token in the Authorization header (only with --http); may also be set via the GITNEXUS_MCP_AUTH_TOKEN env var. Required for a non-loopback bind (--host 0.0.0.0/::), which otherwise refuses to start.";
+    readonly 'help.option.mcp.authToken': "Require this bearer token in the Authorization header (only with --http); may also be set via the GITNEXUS_MCP_AUTH_TOKEN env var, which also enables MCP Bearer auth on gitnexus serve's /api/mcp route. Required for a non-loopback bind (--host 0.0.0.0/::), which otherwise refuses to start.";
     readonly 'help.option.force.confirmation': "Skip confirmation prompt";
     readonly 'help.option.uninstall.force': "Apply the changes (default is a dry-run preview)";
     readonly 'help.option.clean.all': "Clean all indexed repos";

package/dist/cli/i18n/en.js

@@ -189,7 +189,7 @@ export const en = {
     'help.option.serve.host': 'Bind address (default: 127.0.0.1, use 0.0.0.0 for remote access)',
     'help.option.mcp.http': 'Serve MCP over HTTP instead of stdio (for remote clients)',
     'help.option.mcp.host': 'HTTP bind address (only with --http). Default: 127.0.0.1 (loopback). Use 0.0.0.0 to expose to all interfaces.',
-    'help.option.mcp.authToken': 'Require this bearer token in the Authorization header (only with --http); may also be set via the GITNEXUS_MCP_AUTH_TOKEN env var. Required for a non-loopback bind (--host 0.0.0.0/::), which otherwise refuses to start.',
+    'help.option.mcp.authToken': "Require this bearer token in the Authorization header (only with --http); may also be set via the GITNEXUS_MCP_AUTH_TOKEN env var, which also enables MCP Bearer auth on gitnexus serve's /api/mcp route. Required for a non-loopback bind (--host 0.0.0.0/::), which otherwise refuses to start.",
     'help.option.force.confirmation': 'Skip confirmation prompt',
     'help.option.uninstall.force': 'Apply the changes (default is a dry-run preview)',
     'help.option.clean.all': 'Clean all indexed repos',

package/dist/cli/i18n/resources.d.ts

@@ -188,7 +188,7 @@ export declare const cliResources: {
         readonly 'help.option.serve.host': "Bind address (default: 127.0.0.1, use 0.0.0.0 for remote access)";
         readonly 'help.option.mcp.http': "Serve MCP over HTTP instead of stdio (for remote clients)";
         readonly 'help.option.mcp.host': "HTTP bind address (only with --http). Default: 127.0.0.1 (loopback). Use 0.0.0.0 to expose to all interfaces.";
-        readonly 'help.option.mcp.authToken': "Require this bearer token in the Authorization header (only with --http); may also be set via the GITNEXUS_MCP_AUTH_TOKEN env var. Required for a non-loopback bind (--host 0.0.0.0/::), which otherwise refuses to start.";
+        readonly 'help.option.mcp.authToken': "Require this bearer token in the Authorization header (only with --http); may also be set via the GITNEXUS_MCP_AUTH_TOKEN env var, which also enables MCP Bearer auth on gitnexus serve's /api/mcp route. Required for a non-loopback bind (--host 0.0.0.0/::), which otherwise refuses to start.";
         readonly 'help.option.force.confirmation': "Skip confirmation prompt";
         readonly 'help.option.uninstall.force': "Apply the changes (default is a dry-run preview)";
         readonly 'help.option.clean.all': "Clean all indexed repos";

package/dist/cli/i18n/zh-CN.js

@@ -187,7 +187,7 @@ export const zhCN = {
     'help.option.serve.host': '绑定地址（默认：127.0.0.1；远程访问可用 0.0.0.0）',
     'help.option.mcp.http': '使用 HTTP 代替 stdio 提供 MCP 服务（适合远程客户端）',
     'help.option.mcp.host': 'HTTP 绑定地址（仅与 --http 搭配使用）。默认：127.0.0.1（回环）。使用 0.0.0.0 向所有接口开放。',
-    'help.option.mcp.authToken': '要求 Authorization 头携带此 Bearer Token（仅与 --http 搭配使用）；也可通过 GITNEXUS_MCP_AUTH_TOKEN 环境变量设置。非回环绑定（--host 0.0.0.0/::）时必填，否则拒绝启动。',
+    'help.option.mcp.authToken': '要求 Authorization 头携带此 Bearer Token（仅与 --http 搭配使用）；也可通过 GITNEXUS_MCP_AUTH_TOKEN 环境变量设置，该变量同时为 gitnexus serve 的 /api/mcp 路由启用 MCP Bearer 认证。非回环绑定（--host 0.0.0.0/::）时必填，否则拒绝启动。',
     'help.option.force.confirmation': '跳过确认提示',
     'help.option.uninstall.force': '应用更改（默认仅为预演预览）',
     'help.option.clean.all': '清理所有已索引仓库',

package/dist/cli/index.js

@@ -149,7 +149,7 @@ program
     .option('--http', 'Serve MCP over HTTP instead of stdio (for remote clients)')
     .option('-p, --port <port>', 'HTTP port (only with --http). Default: 3000', '3000')
     .option('--host <host>', 'HTTP bind address (only with --http). Default: 127.0.0.1 (loopback). Use 0.0.0.0 to expose to all interfaces.', '127.0.0.1')
-    .option('--auth-token <token>', 'Require this bearer token in the Authorization header (only with --http); may also be set via the GITNEXUS_MCP_AUTH_TOKEN env var. Required for a non-loopback bind (--host 0.0.0.0/::), which otherwise refuses to start.')
+    .option('--auth-token <token>', "Require this bearer token in the Authorization header (only with --http); may also be set via the GITNEXUS_MCP_AUTH_TOKEN env var, which also enables MCP Bearer auth on gitnexus serve's /api/mcp route. Required for a non-loopback bind (--host 0.0.0.0/::), which otherwise refuses to start.")
     .action(createLbugLazyAction(() => import('./mcp.js'), 'mcpCommand'));
 program
     .command('list')

package/dist/core/ingestion/pipeline-phases/parse-impl.d.ts

@@ -2,7 +2,7 @@
  * Parse implementation — chunked parse + resolve loop.
  *
  * This is the core parsing engine of the ingestion pipeline. It reads
- * source files in byte-budget chunks (~20MB each), parses via the worker
+ * source files in stable hash-bucket packs (~2MB each by default), parses via the worker
  * pool (the sole parse path — there is no sequential fallback), and emits
  * route CALLS edges. Import,
  * call, and inheritance resolution are owned by the scope-resolution

package/dist/core/ingestion/pipeline-phases/parse-impl.js

@@ -2,7 +2,7 @@
  * Parse implementation — chunked parse + resolve loop.
  *
  * This is the core parsing engine of the ingestion pipeline. It reads
- * source files in byte-budget chunks (~20MB each), parses via the worker
+ * source files in stable hash-bucket packs (~2MB each by default), parses via the worker
  * pool (the sole parse path — there is no sequential fallback), and emits
  * route CALLS edges. Import,
  * call, and inheritance resolution are owned by the scope-resolution
@@ -16,7 +16,7 @@
  */
 import { BindingAccumulator, enrichExportedTypeMap, } from '../binding-accumulator.js';
 import { mergeChunkResults, dispatchChunkParse } from '../parsing-processor.js';
-import { fileContentHash, computeChunkHash, loadParseCacheChunk, persistParseCacheChunk, PARSE_CACHE_VERSION, } from '../../../storage/parse-cache.js';
+import { fileContentHash, computeChunkHash, loadParseCacheChunk, persistParseCacheChunk, PARSE_CACHE_VERSION, packParseCacheChunks, } from '../../../storage/parse-cache.js';
 import { clearParsedFileStore, persistParsedFileChunk, loadParsedFilesForPaths, getDurableParsedFileDir, loadDurableParsedFileIndex, prepareDurableParsedFileChunk, restoreDurableParsedFileShard, } from '../../../storage/parsedfile-store.js';
 import { DEFAULT_PDG_MAX_FUNCTION_LINES } from '../cfg/collect.js';
 import { processRoutesFromExtracted, resolveRouteHandlerSymbols, buildExportedTypeMapFromGraph, } from '../call-processor.js';
@@ -111,38 +111,18 @@ export function heapPressureRemedy(heapLimitBytes) {
     return (`This machine is at its memory ceiling: exclude generated or vendored directories ` +
         `via .gitnexusignore, or analyze on a machine with more memory.`);
 }
-/** Max bytes of source content to load per parse chunk.
+/** Max bytes of source content to load per parse cache pack.
  *
- * Memory bound for the worker pool dispatch + a granularity knob for
- * the parse cache. A single file change invalidates only its enclosing
- * chunk, so smaller budgets → finer-grained invalidation.
- *
- * Override via GITNEXUS_CHUNK_BYTE_BUDGET (bytes) — the default of 2MB
- * gives a useful invalidation floor (~1/N chunks on a multi-MB repo)
- * while keeping worker dispatch overhead under 5% on cold runs.
- */
-/**
- * Built-in chunk byte budget when neither `PipelineOptions.chunkByteBudget`
- * nor `GITNEXUS_CHUNK_BYTE_BUDGET` is set. Tuned to give a useful
- * cache-invalidation floor (~1/N chunks on a multi-MB repo) while keeping
- * worker dispatch overhead under 5% on cold runs. Resolution happens at
- * call time inside `runChunkedParseAndResolve` (U14 from PR #1693 review)
- * — previously this was a module-load IIFE, which froze the env value at
- * import time and meant per-call option threading silently no-op'd.
+ * Granularity knob for the parse cache: a single file change invalidates only
+ * its enclosing pack. Override via GITNEXUS_CHUNK_BYTE_BUDGET. Resolution
+ * happens at call time (U14 from PR #1693) — not at module load.
  */
 const DEFAULT_CHUNK_BYTE_BUDGET = 2 * 1024 * 1024;
 /**
- * Per-worker share of a chunk's byte budget when auto-scaling (#worker-idle).
- *
- * A chunk is a single `WorkerPool.dispatch` unit; the pool fans a chunk's files
- * into sub-batch jobs and assigns them to idle workers (`wakeIdleSlots`). When
- * the chunk budget (2 MB) was far below the 8 MB sub-batch cap, every chunk
- * produced exactly ONE job → ONE busy worker while the other N-1 sat idle. To
- * keep all workers fed, the auto chunk budget now scales as
- * `poolSize × CHUNK_BYTES_PER_WORKER`, so each dispatch carries enough work to
- * fan across the whole pool. Sequential / explicit-budget runs are unaffected.
+ * Byte unit for auto pool sizing (one worker per this much source). Same
+ * magnitude as the default cache pack, but not a membership input (#3088).
  */
-const CHUNK_BYTES_PER_WORKER = 2 * 1024 * 1024;
+const CHUNK_BYTES_PER_WORKER = DEFAULT_CHUNK_BYTE_BUDGET;
 /**
  * Target jobs-per-worker per dispatch. More jobs than workers gives the pool's
  * idle-slot assignment room to load-balance (a slow job doesn't strand a worker
@@ -151,16 +131,14 @@ const CHUNK_BYTES_PER_WORKER = 2 * 1024 * 1024;
 const TARGET_JOBS_PER_WORKER = 3;
 /** Floor for a derived sub-batch so jobs don't shrink to per-file IPC churn. */
 const MIN_SUB_BATCH_BYTES = 256 * 1024;
-function resolveChunkByteBudget(options, effectivePoolSize = 1) {
+function resolveChunkByteBudget(options) {
     const opt = options?.chunkByteBudget;
     if (typeof opt === 'number' && Number.isFinite(opt) && opt > 0)
         return opt;
     const env = Number(process.env.GITNEXUS_CHUNK_BYTE_BUDGET);
     if (Number.isFinite(env) && env > 0)
         return env;
-    // Auto: size each chunk so a dispatch can fan across the whole pool. A
-    // single-worker (tiny-repo) run keeps the original 2 MB invalidation floor.
-    return Math.max(DEFAULT_CHUNK_BYTE_BUDGET, effectivePoolSize * CHUNK_BYTES_PER_WORKER);
+    return DEFAULT_CHUNK_BYTE_BUDGET;
 }
 /**
  * Whole-repo, cross-file route extraction (main thread).
@@ -367,14 +345,6 @@ export async function runChunkedParseAndResolve(graph, scannedFiles, allPaths, t
         }
     }
     const unavailableScopeLanguageFiles = [...skippedByLang.values()].reduce((total, count) => total + count, 0);
-    // Sort parseableScanned alphabetically for stable chunk membership
-    // across runs (Finding 4). Without this, filesystem-scan order can
-    // shift between runs (notably on macOS APFS where directory entry
-    // order can change after modifications) — different files in the
-    // same chunk → different chunk hash → cache miss even when no file
-    // content changed. The cache also becomes platform-specific: a
-    // Linux-built cache misses on macOS for the same repo.
-    parseableScanned.sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0));
     const totalParseable = parseableScanned.length;
     const totalBytes = parseableScanned.reduce((sum, f) => sum + f.size, 0);
     if (totalParseable === 0) {
@@ -416,24 +386,24 @@ export async function runChunkedParseAndResolve(graph, scannedFiles, allPaths, t
     // runs. Resolving in the function body restores per-call configurability
     // and matches the pattern used by resolveAutoPoolSize and the U1
     // parseChunkConcurrency resolver.
-    // Effective worker count, computed up-front so the chunk budget can scale to
-    // keep the whole pool busy (#worker-idle). The pool is ALWAYS used (sequential
-    // parsing was removed; the disabled channels threw above). Size it to the
-    // work: an explicit `--workers <N>` pins the size; otherwise the cores-based
-    // auto size is capped by the repo's worth of work (~one worker per
-    // CHUNK_BYTES_PER_WORKER of source) so a tiny repo spawns ~1 worker instead of
-    // a full pool, replacing the job the deleted small-repo threshold used to do.
-    // KTD-3 of the remove-sequential plan; the cap formula is intentionally coarse
-    // (tuning deferred).
+    // Effective worker count: explicit `--workers <N>` pins it; otherwise
+    // cores-based auto size is capped by source bytes / CHUNK_BYTES_PER_WORKER
+    // so a tiny repo does not spawn a full idle pool. Cache pack membership
+    // is independent of this number (#3088).
     const explicitPoolSize = options?.workerPoolSize;
     const workProportionalCap = Math.max(1, Math.ceil(totalBytes / CHUNK_BYTES_PER_WORKER));
     const effectivePoolSize = explicitPoolSize && explicitPoolSize > 0
         ? explicitPoolSize
         : Math.min(resolveAutoPoolSize(), workProportionalCap);
-    const chunkByteBudget = resolveChunkByteBudget(options, effectivePoolSize);
-    // Sub-batch size so each chunk fans into ~`TARGET_JOBS_PER_WORKER` jobs per
-    // worker, giving the pool's idle-slot assignment room to load-balance. An
-    // explicit `GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES` operator override wins.
+    // Cache packs: stable (language, hash(path) mod 128) buckets, then the
+    // per-call byte budget inside each bucket (#3088). Pool size is used only
+    // for worker count and sub-batch fan-out, not membership.
+    const chunkByteBudget = resolveChunkByteBudget(options);
+    // Sub-batch size so a 2 MiB pack fans into ~TARGET_JOBS_PER_WORKER jobs
+    // per worker, floored at MIN_SUB_BATCH_BYTES (256 KiB) so an 8-worker
+    // pool still gets ~8 jobs from one pack instead of one idle-heavy job
+    // (#worker-idle). Do not derive this from pool×2 MiB while dispatching a
+    // 2 MiB pack. An explicit GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES wins.
     const subBatchEnv = Number(process.env.GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES);
     const dispatchSubBatchMaxBytes = Number.isFinite(subBatchEnv) && subBatchEnv > 0
         ? subBatchEnv
@@ -451,20 +421,11 @@ export async function runChunkedParseAndResolve(graph, scannedFiles, allPaths, t
         logger.warn(`Large repository: analyzing ${parseableScanned.length} files needs roughly ${Math.round(projectedHeapNeedBytes / 1024 / 1024 / 1024)}GB of memory, ` +
             `but Node is limited to ${Math.round(heapLimitBytes / 1024 / 1024 / 1024)}GB — analyze may stop early. ${heapPressureRemedy(heapLimitBytes)}`);
     }
-    const chunks = [];
-    let currentChunk = [];
-    let currentBytes = 0;
-    for (const file of parseableScanned) {
-        if (currentChunk.length > 0 && currentBytes + file.size > chunkByteBudget) {
-            chunks.push(currentChunk);
-            currentChunk = [];
-            currentBytes = 0;
-        }
-        currentChunk.push(file.path);
-        currentBytes += file.size;
-    }
-    if (currentChunk.length > 0)
-        chunks.push(currentChunk);
+    const chunks = packParseCacheChunks(parseableScanned.map((file) => ({
+        path: file.path,
+        size: file.size,
+        language: getLanguageFromFilename(file.path) ?? 'unknown',
+    })), chunkByteBudget);
     const numChunks = chunks.length;
     if (isDev) {
         const totalMB = parseableScanned.reduce((s, f) => s + f.size, 0) / (1024 * 1024);

package/dist/server/api.js

@@ -20,7 +20,7 @@ import { searchFTSFromLbug } from '../core/search/bm25-index.js';
 import { hybridSearch } from '../core/search/hybrid-search.js';
 import { ftsDegradedWarning } from '../core/search/fts-indexes.js';
 import { LocalBackend } from '../mcp/local/local-backend.js';
-import { mountMCPEndpoints } from './mcp-http.js';
+import { installServeMcpAuth, mountMCPEndpoints } from './mcp-http.js';
 import { fileURLToPath } from 'url';
 import { isTerminalJobStatus, JobManager } from './analyze-job.js';
 import { mountSSEProgress } from './sse-progress.js';
@@ -616,6 +616,9 @@ export const createServer = async (port, host = '127.0.0.1') => {
             callback(null, isAllowedOrigin(origin));
         },
     }));
+    // Optional protocol-layer auth for the MCP route. Keep this before the
+    // global body parser so rejected requests do not consume the JSON budget.
+    installServeMcpAuth(app);
     app.use(express.json({ limit: '10mb' }));
     // Origin guard for write routes: loopback, the server's own bound host, and
     // any configured public origin — prevents CSRF from other devices.

package/dist/server/mcp-http.d.ts

@@ -9,4 +9,12 @@
  */
 import type { Express } from 'express';
 import type { LocalBackend } from '../mcp/local/local-backend.js';
+/**
+ * Protect serve's /api/mcp route when the shared MCP bearer token is configured.
+ *
+ * This middleware must be installed before Express's global JSON parser so an
+ * unauthenticated request body is rejected before it is parsed. The standalone
+ * `gitnexus mcp --http` server resolves the same environment variable.
+ */
+export declare function installServeMcpAuth(app: Express, env?: NodeJS.ProcessEnv): boolean;
 export declare function mountMCPEndpoints(app: Express, backend: LocalBackend): Promise<() => Promise<void>>;

package/dist/server/mcp-http.js

@@ -7,9 +7,24 @@
  *
  * Used by server/api.ts to wire up the full web server.
  */
-import { createStreamableHttpHandler } from '../mcp/http-transport.js';
+import { createAuthMiddleware, createStreamableHttpHandler, resolveAuthToken, } from '../mcp/http-transport.js';
 import { createMcpRepositoryPolicy } from '../mcp/repository-policy.js';
 import { logger } from '../core/logger.js';
+/**
+ * Protect serve's /api/mcp route when the shared MCP bearer token is configured.
+ *
+ * This middleware must be installed before Express's global JSON parser so an
+ * unauthenticated request body is rejected before it is parsed. The standalone
+ * `gitnexus mcp --http` server resolves the same environment variable.
+ */
+export function installServeMcpAuth(app, env = process.env) {
+    const authToken = resolveAuthToken(undefined, env);
+    if (!authToken)
+        return false;
+    app.use('/api/mcp', createAuthMiddleware(authToken));
+    logger.info('Bearer authentication enabled for serve /api/mcp');
+    return true;
+}
 export async function mountMCPEndpoints(app, backend) {
     const repositoryPolicy = await createMcpRepositoryPolicy(backend);
     const { handler, cleanup } = createStreamableHttpHandler(backend, { repositoryPolicy });

package/dist/storage/parse-cache.d.ts

@@ -6,10 +6,12 @@
  * does is skip the tree-sitter worker dispatch when a chunk's contents
  * haven't changed since the last run.
  *
- * Granularity: chunk-level. The parse phase chunks files into ~20MB byte
- * budgets. The cache key is `sha256(joined(filePath:contentHash for each
- * file in the chunk, sorted))`. A change to a single file invalidates only
- * that file's chunk — typically 1 of ~50 chunks on a 1000-file repo.
+ * Granularity: chunk-level. Files are assigned to a stable
+ * `(language, hash(path) mod 128)` bucket, then packed to a 2 MiB (or
+ * operator) byte budget *inside* that bucket. Membership does not depend
+ * on worker count. The cache key is `sha256(joined(filePath:contentHash
+ * for each file in the chunk, sorted))`. A content edit invalidates only
+ * that file's pack; add/delete/rename only the affected bucket.
  *
  * Why not per-file:
  * - Workers process sub-batches and emit aggregated `ParseWorkerResult`s.
@@ -23,6 +25,22 @@
  */
 import type { ParseWorkerResult } from '../core/ingestion/workers/parse-worker.js';
 export declare const PARSE_CACHE_VERSION: string;
+/** Stable parse-cache bucket count (#3088). Changing this requires SCHEMA_BUMP. */
+export declare const PARSE_CACHE_BUCKET_COUNT = 128;
+/** Bucket id for cache membership: `sha256(path) mod N` without IEEE-754 truncation. */
+export declare const parseCacheBucketId: (filePath: string) => number;
+export type ParseCachePackFile = {
+    path: string;
+    size: number;
+    language: string;
+};
+/**
+ * Pack files into parse-cache chunks: group by (language, bucket id), sort
+ * paths inside the group, then cut at `byteBudget`. Bucket visit order is
+ * the lexicographic order of `${language}\\0${bucketId}` keys (deterministic,
+ * independent of scan order and worker count).
+ */
+export declare const packParseCacheChunks: (files: readonly ParseCachePackFile[], byteBudget: number) => string[][];
 /** Runtime view: keyed Map for fast lookup; mutated in place during a run. */
 export interface ParseCache {
     version: string;

package/dist/storage/parse-cache.js

@@ -6,10 +6,12 @@
  * does is skip the tree-sitter worker dispatch when a chunk's contents
  * haven't changed since the last run.
  *
- * Granularity: chunk-level. The parse phase chunks files into ~20MB byte
- * budgets. The cache key is `sha256(joined(filePath:contentHash for each
- * file in the chunk, sorted))`. A change to a single file invalidates only
- * that file's chunk — typically 1 of ~50 chunks on a 1000-file repo.
+ * Granularity: chunk-level. Files are assigned to a stable
+ * `(language, hash(path) mod 128)` bucket, then packed to a 2 MiB (or
+ * operator) byte budget *inside* that bucket. Membership does not depend
+ * on worker count. The cache key is `sha256(joined(filePath:contentHash
+ * for each file in the chunk, sorted))`. A content edit invalidates only
+ * that file's pack; add/delete/rename only the affected bucket.
  *
  * Why not per-file:
  * - Workers process sub-batches and emit aggregated `ParseWorkerResult`s.
@@ -26,6 +28,7 @@ import { createRequire } from 'module';
 import fs from 'fs/promises';
 import path from 'path';
 import { fileURLToPath } from 'url';
+import { compareCodeUnits } from '../lib/utils.js';
 /**
  * Cache version composed of:
  *   - A schema bump knob (`SCHEMA_BUMP`) for hand-controlled invalidation
@@ -627,6 +630,16 @@ import { fileURLToPath } from 'url';
 // therefore takes 79, the next free value above origin/main and every open PR
 // found by the contents-API scan at their exact head SHAs.
 //
+// 79 -> 80 for #3088: parse-cache membership is `(language, sha256(path) mod
+// 128)` then the byte budget *inside* that bucket. Worker count is no longer a
+// membership input, so a warm v79 cache keyed sequential scan-order packs (and
+// on multi-worker hosts, pool×2 MiB mega-chunks) must miss. Sidecar-era
+// ParsedFile stores (#3086/#3087) share PARSE_CACHE_VERSION, so both stores
+// invalidate in lockstep. origin/main at allocation is 79; open PRs that still
+// touch gitnexus/src/storage/parse-cache.ts claim 78 (#3060), 71 (#2840), and
+// 2 (#1616) — none claim 80. RE-CHECK AGAINST origin/main AND OPEN PRs
+// IMMEDIATELY BEFORE MERGING.
+//
 // WHY THIS IS STILL A HAND-PICKED NUMBER, when `SCHEMA_FINGERPRINT` next door
 // is a derived sha256 that cannot collide. The derivation exists and already
 // runs: `resolveAnalyzerRunnerIdentity` computes `build.digest` over the
@@ -645,7 +658,7 @@ import { fileURLToPath } from 'url';
 // `route-extractors/` and `workers/` module content — would close the missing-
 // bump axis without invalidating on unrelated churn, and is the real follow-up.
 // RE-CHECK AGAINST origin/main AND OPEN PRs IMMEDIATELY BEFORE MERGING.
-const SCHEMA_BUMP = 79;
+const SCHEMA_BUMP = 80;
 const GITNEXUS_PKG_VERSION = (() => {
     try {
         // package.json sits at gitnexus/package.json — two levels up from
@@ -673,16 +686,56 @@ const GITNEXUS_PKG_VERSION = (() => {
     return '0.0.0-unknown';
 })();
 export const PARSE_CACHE_VERSION = `${SCHEMA_BUMP}+${GITNEXUS_PKG_VERSION}`;
+/** SHA-256 hex of a string or buffer (paths for bucket ids, contents for cache keys). */
+const sha256Hex = (input) => createHash('sha256')
+    .update(typeof input === 'string' ? Buffer.from(input) : input)
+    .digest('hex');
+/** Stable parse-cache bucket count (#3088). Changing this requires SCHEMA_BUMP. */
+export const PARSE_CACHE_BUCKET_COUNT = 128;
+/** Bucket id for cache membership: `sha256(path) mod N` without IEEE-754 truncation. */
+export const parseCacheBucketId = (filePath) => Number(BigInt(`0x${sha256Hex(filePath)}`) % BigInt(PARSE_CACHE_BUCKET_COUNT));
+/**
+ * Pack files into parse-cache chunks: group by (language, bucket id), sort
+ * paths inside the group, then cut at `byteBudget`. Bucket visit order is
+ * the lexicographic order of `${language}\\0${bucketId}` keys (deterministic,
+ * independent of scan order and worker count).
+ */
+export const packParseCacheChunks = (files, byteBudget) => {
+    const buckets = new Map();
+    for (const file of files) {
+        const key = `${file.language}\0${parseCacheBucketId(file.path)}`;
+        const list = buckets.get(key);
+        if (list)
+            list.push(file);
+        else
+            buckets.set(key, [file]);
+    }
+    const chunks = [];
+    for (const key of [...buckets.keys()].sort()) {
+        const group = buckets.get(key);
+        group.sort((a, b) => compareCodeUnits(a.path, b.path));
+        let current = [];
+        let bytes = 0;
+        for (const file of group) {
+            if (current.length > 0 && bytes + file.size > byteBudget) {
+                chunks.push(current);
+                current = [];
+                bytes = 0;
+            }
+            current.push(file.path);
+            bytes += file.size;
+        }
+        if (current.length > 0)
+            chunks.push(current);
+    }
+    return chunks;
+};
 const LEGACY_CACHE_FILENAME = 'parse-cache.json';
 const CACHE_DIRNAME = 'parse-cache';
 const CACHE_INDEX_FILENAME = 'index.json';
 /** Keys on disk always come from `computeChunkHash` — 64-char lowercase hex. */
 const CHUNK_CACHE_KEY_HEX_RE = /^[a-f0-9]{64}$/;
 const isValidChunkCacheKey = (chunkHash) => CHUNK_CACHE_KEY_HEX_RE.test(chunkHash);
-/** SHA-256 hex of a single string or buffer. */
-const sha256Hex = (input) => createHash('sha256')
-    .update(typeof input === 'string' ? Buffer.from(input) : input)
-    .digest('hex');
 /** Stable hash of a single file's contents — used by callers to compose a chunk hash. */
 export const fileContentHash = (content) => sha256Hex(content);
 export const computeChunkHash = (entries, pdg = false) => {

package/dist/storage/parsedfile-store.d.ts

@@ -72,6 +72,15 @@ export declare const forceGc: () => void;
 export declare const getParsedFileStoreDir: (storagePath: string) => string;
 /** Remove any prior run's shards so a fresh parse starts clean. Idempotent. */
 export declare const clearParsedFileStore: (storagePath: string) => Promise<void>;
+/**
+ * Test seam for #3086. Production always calls {@link forceGc}; unit tests
+ * replace `run` to count cadence without requiring `--expose-gc`.
+ */
+export declare const parsedFileLoadGc: {
+    run: () => void;
+    /** Raw UTF-8 JSON shard bytes between GCs (#3086). Tests may lower this. */
+    byteBudget: number;
+};
 /**
  * Write one parse chunk's `ParsedFile[]` to the store as a single shard (async).
  * No-op for an empty chunk. `shardId` must be unique within a run. Used by the

package/dist/storage/parsedfile-store.js

@@ -47,7 +47,7 @@
  * on a cold run. Content-addressing makes stale reuse impossible: a changed
  * file changes its chunk hash, which misses BOTH stores and re-dispatches.
  */
-import { promises as fs, mkdirSync, writeFileSync } from 'node:fs';
+import { promises as fs, mkdirSync, writeFileSync, unlinkSync } from 'node:fs';
 import path from 'node:path';
 import v8 from 'node:v8';
 import vm from 'node:vm';
@@ -162,6 +162,119 @@ const serializeParsedFileShard = (parsedFiles) => {
     return JSON.stringify(parsedFiles, mapReplacer);
 };
 const shardPath = (storagePath, shardId) => path.join(getParsedFileStoreDir(storagePath), `${shardId}.json`);
+/** Sidecar listing `filePath`s in a shard; not matched by `endsWith('.json')`. */
+const shardPathsSidecarPath = (jsonPath) => `${jsonPath}.paths`;
+const LOAD_YIELD_EVERY_SHARDS = 128;
+/**
+ * Test seam for #3086. Production always calls {@link forceGc}; unit tests
+ * replace `run` to count cadence without requiring `--expose-gc`.
+ */
+export const parsedFileLoadGc = {
+    run: forceGc,
+    /** Raw UTF-8 JSON shard bytes between GCs (#3086). Tests may lower this. */
+    byteBudget: 128 * 1024 * 1024,
+};
+const encodeShardPathsSidecar = (parsedFiles) => {
+    const paths = parsedFiles.map((pf) => pf.filePath);
+    return `${paths.length}\n${paths.length === 0 ? '' : `${paths.join('\n')}\n`}`;
+};
+/**
+ * Parse a counted NDJSON path listing. Returns `null` when the sidecar must
+ * not be trusted to skip the JSON shard: missing trailing newline, CR/NUL,
+ * a truncated listing that still ends on a complete line, or a count that
+ * does not match the remaining lines.
+ */
+const parseShardPathsSidecar = (sidecarRaw) => {
+    if (sidecarRaw.includes('\0') || sidecarRaw.includes('\r') || !sidecarRaw.endsWith('\n')) {
+        return null;
+    }
+    const nl = sidecarRaw.indexOf('\n');
+    if (nl < 0)
+        return null;
+    const countToken = sidecarRaw.slice(0, nl);
+    if (!/^[0-9]+$/.test(countToken))
+        return null;
+    const count = Number(countToken);
+    const body = sidecarRaw.slice(nl + 1);
+    const listed = body === '' ? [] : body.slice(0, -1).split('\n');
+    if (listed.length !== count)
+        return null;
+    return listed;
+};
+/** NDJSON sidecars cannot encode paths that themselves contain CR/LF/NUL. */
+const shardPathsSidecarSafe = (parsedFiles) => parsedFiles.every((pf) => !/[\r\n\0]/.test(pf.filePath));
+const isEnoent = (err) => err.code === 'ENOENT';
+const warnSidecarIo = (err, jsonPath, msg) => {
+    logger.warn({ err, jsonPath }, msg);
+};
+const ignoreMissingSidecarUnlink = (err, jsonPath) => {
+    if (isEnoent(err))
+        return;
+    warnSidecarIo(err, jsonPath, 'parsedfile-store: failed to drop path sidecar; JSON remains authoritative');
+};
+/** Drop a leftover listing before publishing JSON so load cannot skip new paths. */
+const dropPathSidecar = async (jsonPath) => {
+    try {
+        await fs.unlink(shardPathsSidecarPath(jsonPath));
+    }
+    catch (err) {
+        ignoreMissingSidecarUnlink(err, jsonPath);
+    }
+};
+const dropPathSidecarSync = (jsonPath) => {
+    try {
+        unlinkSync(shardPathsSidecarPath(jsonPath));
+    }
+    catch (err) {
+        ignoreMissingSidecarUnlink(err, jsonPath);
+    }
+};
+const writeShardPathsSidecar = async (jsonPath, parsedFiles) => {
+    if (!shardPathsSidecarSafe(parsedFiles)) {
+        try {
+            await fs.unlink(shardPathsSidecarPath(jsonPath));
+        }
+        catch (err) {
+            ignoreMissingSidecarUnlink(err, jsonPath);
+        }
+        return;
+    }
+    try {
+        await fs.writeFile(shardPathsSidecarPath(jsonPath), encodeShardPathsSidecar(parsedFiles), 'utf-8');
+    }
+    catch (err) {
+        warnSidecarIo(err, jsonPath, 'parsedfile-store: path sidecar write failed; JSON shard remains authoritative');
+        try {
+            await fs.unlink(shardPathsSidecarPath(jsonPath));
+        }
+        catch (unlinkErr) {
+            ignoreMissingSidecarUnlink(unlinkErr, jsonPath);
+        }
+    }
+};
+const writeShardPathsSidecarSync = (jsonPath, parsedFiles) => {
+    if (!shardPathsSidecarSafe(parsedFiles)) {
+        try {
+            unlinkSync(shardPathsSidecarPath(jsonPath));
+        }
+        catch (err) {
+            ignoreMissingSidecarUnlink(err, jsonPath);
+        }
+        return;
+    }
+    try {
+        writeFileSync(shardPathsSidecarPath(jsonPath), encodeShardPathsSidecar(parsedFiles), 'utf-8');
+    }
+    catch (err) {
+        warnSidecarIo(err, jsonPath, 'parsedfile-store: path sidecar write failed; JSON shard remains authoritative');
+        try {
+            unlinkSync(shardPathsSidecarPath(jsonPath));
+        }
+        catch (unlinkErr) {
+            ignoreMissingSidecarUnlink(unlinkErr, jsonPath);
+        }
+    }
+};
 /**
  * Write one parse chunk's `ParsedFile[]` to the store as a single shard (async).
  * No-op for an empty chunk. `shardId` must be unique within a run. Used by the
@@ -173,7 +286,10 @@ export const persistParsedFileChunk = async (storagePath, shardId, parsedFiles)
     if (payload === null)
         return;
     await fs.mkdir(getParsedFileStoreDir(storagePath), { recursive: true });
-    await fs.writeFile(shardPath(storagePath, shardId), payload, 'utf-8');
+    const dest = shardPath(storagePath, shardId);
+    await dropPathSidecar(dest);
+    await fs.writeFile(dest, payload, 'utf-8');
+    await writeShardPathsSidecar(dest, parsedFiles);
 };
 // Per-process set of store dirs we've already `mkdir`ed, so the sync worker
 // writer (called once per job, many times into the same dir) doesn't issue a
@@ -197,7 +313,10 @@ export const persistParsedFileShardSync = (storagePath, shardId, parsedFiles) =>
         mkdirSync(dir, { recursive: true });
         createdStoreDirs.add(dir);
     }
-    writeFileSync(shardPath(storagePath, shardId), payload, 'utf-8');
+    const dest = shardPath(storagePath, shardId);
+    dropPathSidecarSync(dest);
+    writeFileSync(dest, payload, 'utf-8');
+    writeShardPathsSidecarSync(dest, parsedFiles);
 };
 /**
  * Stream the store and return the `ParsedFile`s whose `filePath` is in
@@ -227,58 +346,95 @@ export const loadParsedFilesForPaths = async (storagePath, wantPaths) => {
     let filesWithDroppedSites = 0;
     let droppedChains = 0;
     let rejectedFiles = 0;
+    let bytesSinceGc = 0;
+    let shardsSinceYield = 0;
+    const maybeYieldAndGc = async (forceByteGc) => {
+        if (forceByteGc) {
+            parsedFileLoadGc.run();
+            bytesSinceGc = 0;
+            shardsSinceYield = 0;
+            await new Promise((resolve) => setImmediate(resolve));
+            return;
+        }
+        shardsSinceYield++;
+        if (shardsSinceYield >= LOAD_YIELD_EVERY_SHARDS) {
+            shardsSinceYield = 0;
+            await new Promise((resolve) => setImmediate(resolve));
+        }
+    };
     for (let i = 0; i < shards.length; i++) {
+        const jsonName = shards[i];
+        const jsonFull = path.join(dir, jsonName);
+        try {
+            const sidecarRaw = await fs.readFile(shardPathsSidecarPath(jsonFull), 'utf-8');
+            // Fail closed: complete writers emit `<count>\n` plus one path per line
+            // and a trailing newline, never CR. Stripping CR (or accepting a
+            // newline-terminated prefix) would let a truncated listing skip JSON.
+            const listed = parseShardPathsSidecar(sidecarRaw);
+            if (listed === null) {
+                throw new Error('corrupt sidecar');
+            }
+            if (listed.length > 0 && !listed.some((p) => wantPaths.has(p))) {
+                await maybeYieldAndGc(false);
+                continue;
+            }
+        }
+        catch {
+            // Missing or unreadable sidecar → read the shard (pre-sidecar stores).
+        }
         // Per-shard def pool: a SymbolDefinition's three serialized copies live within
         // a single shard (one ParsedFile), so the dedup is shard-local. A cross-shard
         // pool would retain defs of files NOT in `wantPaths` (loaded-but-discarded
         // shards), reintroducing the leak; per-shard drops them with the shard.
         const defPool = new Map();
         const reviver = makeInterningReviver(pool, defPool);
+        let raw;
+        try {
+            raw = await fs.readFile(jsonFull, 'utf-8');
+        }
+        catch {
+            continue; // skip a missing shard; missing files fall back to fresh extract
+        }
+        bytesSinceGc += Buffer.byteLength(raw, 'utf8');
+        const crossedBudget = bytesSinceGc >= parsedFileLoadGc.byteBudget;
         let parsed;
         try {
-            const raw = await fs.readFile(path.join(dir, shards[i]), 'utf-8');
             parsed = JSON.parse(raw, reviver);
         }
         catch {
-            continue; // skip a corrupt shard; missing files fall back to fresh extract
+            parsed = undefined;
         }
-        if (!Array.isArray(parsed))
-            continue;
-        for (const pf of parsed) {
-            if (!pf || typeof pf.filePath !== 'string' || !wantPaths.has(pf.filePath))
-                continue;
-            const flow = sanitizeCallableFlowSites(pf.callableFlowSites);
-            if (flow === undefined) {
-                // non-array garbage → distrust the file, re-extract
-                rejectedFiles++;
-                continue;
-            }
-            const chains = sanitizeReceiverChains(pf.referenceSites);
-            if (chains === undefined) {
-                rejectedFiles++;
-                continue;
-            }
-            if (flow.dropped === 0 && chains.dropped === 0) {
-                out.set(pf.filePath, pf);
-            }
-            else {
-                droppedSites += flow.dropped;
-                droppedChains += chains.dropped;
-                filesWithDroppedSites++;
-                out.set(pf.filePath, {
-                    ...pf,
-                    ...(flow.dropped === 0 ? {} : { callableFlowSites: flow.sites }),
-                    ...(chains.dropped === 0 ? {} : { referenceSites: chains.sites }),
-                });
+        if (Array.isArray(parsed)) {
+            for (const pf of parsed) {
+                if (!pf || typeof pf.filePath !== 'string' || !wantPaths.has(pf.filePath))
+                    continue;
+                const flow = sanitizeCallableFlowSites(pf.callableFlowSites);
+                if (flow === undefined) {
+                    // non-array garbage → distrust the file, re-extract
+                    rejectedFiles++;
+                    continue;
+                }
+                const chains = sanitizeReceiverChains(pf.referenceSites);
+                if (chains === undefined) {
+                    rejectedFiles++;
+                    continue;
+                }
+                if (flow.dropped === 0 && chains.dropped === 0) {
+                    out.set(pf.filePath, pf);
+                }
+                else {
+                    droppedSites += flow.dropped;
+                    droppedChains += chains.dropped;
+                    filesWithDroppedSites++;
+                    out.set(pf.filePath, {
+                        ...pf,
+                        ...(flow.dropped === 0 ? {} : { callableFlowSites: flow.sites }),
+                        ...(chains.dropped === 0 ? {} : { referenceSites: chains.sites }),
+                    });
+                }
             }
         }
-        // Every few shards, reclaim the transient pre-intern parse churn before it
-        // piles up against the heap limit (~5 GB avoidable on the kernel), and
-        // yield so the GC + any pending I/O can run.
-        if ((i & 7) === 7) {
-            forceGc();
-            await new Promise((resolve) => setImmediate(resolve));
-        }
+        await maybeYieldAndGc(crossedBudget);
     }
     if (droppedSites > 0 || droppedChains > 0) {
         // Facts for the dropped sites are omitted this run (the file itself is
@@ -507,7 +663,10 @@ export const persistDurableParsedFileShardSync = (durableDir, chunkHash, threadI
         mkdirSync(dir, { recursive: true });
         createdDurableDirs.add(dir);
     }
-    writeFileSync(path.join(dir, `${chunkHash}-w${threadId}-${shardSeq}.json`), payload, 'utf-8');
+    const dest = path.join(dir, `${chunkHash}-w${threadId}-${shardSeq}.json`);
+    dropPathSidecarSync(dest);
+    writeFileSync(dest, payload, 'utf-8');
+    writeShardPathsSidecarSync(dest, parsedFiles);
 };
 /**
  * Restore a cached chunk's durable shards into the run-scoped store on a warm
@@ -533,7 +692,25 @@ export const restoreDurableParsedFileShard = async (durableDir, runStoragePath,
     const dst = getParsedFileStoreDir(runStoragePath);
     await fs.mkdir(dst, { recursive: true });
     for (const name of shards) {
-        await fs.copyFile(path.join(src, name), path.join(dst, name));
+        const srcJson = path.join(src, name);
+        const dstJson = path.join(dst, name);
+        await dropPathSidecar(dstJson);
+        await fs.copyFile(srcJson, dstJson);
+        try {
+            await fs.copyFile(shardPathsSidecarPath(srcJson), shardPathsSidecarPath(dstJson));
+        }
+        catch (copyErr) {
+            if (!isEnoent(copyErr)) {
+                warnSidecarIo(copyErr, srcJson, 'parsedfile-store: durable path sidecar copy failed; JSON remains authoritative');
+                continue;
+            }
+            try {
+                await fs.unlink(shardPathsSidecarPath(dstJson));
+            }
+            catch (err) {
+                ignoreMissingSidecarUnlink(err, dstJson);
+            }
+        }
     }
     return shards.length;
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gitnexus",
-  "version": "1.6.11-rc.11",
+  "version": "1.6.11-rc.13",
   "description": "Graph-powered code intelligence for AI agents. Index any codebase, query via MCP or CLI.",
   "author": "Abhigyan Patwari",
   "license": "PolyForm-Noncommercial-1.0.0",