gitnexus 1.6.5-rc.45 → 1.6.5-rc.46

package/dist/storage/parse-cache.d.ts

@@ -55,8 +55,14 @@ export declare const computeChunkHash: (entries: Array<{
  */
 export declare const loadParseCache: (storagePath: string) => Promise<ParseCache>;
 /**
- * Persist the cache to disk atomically (write-and-rename) so a crash
- * mid-write doesn't leave a corrupt file.
+ * Persist the cache to disk using a temp directory + rename.
+ *
+ * Writes shards under `${cacheDir}.tmp`, then removes the old `cacheDir` and
+ * renames the temp directory into place. There is a crash window after
+ * `rm(cacheDir)` and before `rename(tmpDir, cacheDir)` where no cache exists;
+ * that is acceptable — `loadParseCache` yields empty and the next run
+ * reparses. This is not a single atomic swap of the whole tree, but avoids
+ * leaving a half-written shard set visible to readers.
  */
 export declare const saveParseCache: (storagePath: string, cache: ParseCache) => Promise<void>;
 /**

package/dist/storage/parse-cache.js

@@ -69,7 +69,12 @@ const GITNEXUS_PKG_VERSION = (() => {
     return '0.0.0-unknown';
 })();
 export const PARSE_CACHE_VERSION = `${SCHEMA_BUMP}+${GITNEXUS_PKG_VERSION}`;
-const CACHE_FILENAME = 'parse-cache.json';
+const LEGACY_CACHE_FILENAME = 'parse-cache.json';
+const CACHE_DIRNAME = 'parse-cache';
+const CACHE_INDEX_FILENAME = 'index.json';
+/** Keys on disk always come from `computeChunkHash` — 64-char lowercase hex. */
+const CHUNK_CACHE_KEY_HEX_RE = /^[a-f0-9]{64}$/;
+const isValidChunkCacheKey = (chunkHash) => CHUNK_CACHE_KEY_HEX_RE.test(chunkHash);
 /** SHA-256 hex of a single string or buffer. */
 const sha256Hex = (input) => createHash('sha256')
     .update(typeof input === 'string' ? Buffer.from(input) : input)
@@ -116,12 +121,12 @@ const mapReviver = (_key, value) => {
     }
     return value;
 };
-/**
- * Load the parse cache. Returns an empty cache on any failure (missing
- * file, corrupt JSON, version mismatch). Never throws on a normal load.
- */
-export const loadParseCache = async (storagePath) => {
-    const cachePath = path.join(storagePath, CACHE_FILENAME);
+const getLegacyCachePath = (storagePath) => path.join(storagePath, LEGACY_CACHE_FILENAME);
+const getCacheDirPath = (storagePath) => path.join(storagePath, CACHE_DIRNAME);
+const getCacheIndexPath = (storagePath) => path.join(getCacheDirPath(storagePath), CACHE_INDEX_FILENAME);
+const getCacheChunkPath = (storagePath, chunkHash) => path.join(getCacheDirPath(storagePath), `${chunkHash}.json`);
+const loadLegacyParseCache = async (storagePath) => {
+    const cachePath = getLegacyCachePath(storagePath);
     try {
         const raw = await fs.readFile(cachePath, 'utf-8');
         const data = JSON.parse(raw, mapReviver);
@@ -143,22 +148,88 @@ export const loadParseCache = async (storagePath) => {
         return emptyCache();
     }
 };
+const loadShardedParseCache = async (storagePath) => {
+    const indexPath = getCacheIndexPath(storagePath);
+    try {
+        const raw = await fs.readFile(indexPath, 'utf-8');
+        const data = JSON.parse(raw);
+        if (typeof data !== 'object' ||
+            data === null ||
+            data.version !== PARSE_CACHE_VERSION ||
+            !Array.isArray(data.keys)) {
+            return emptyCache();
+        }
+        const entries = new Map();
+        for (const chunkHash of data.keys) {
+            if (typeof chunkHash !== 'string' || !isValidChunkCacheKey(chunkHash))
+                continue;
+            try {
+                const chunkRaw = await fs.readFile(getCacheChunkPath(storagePath, chunkHash), 'utf-8');
+                const chunkData = JSON.parse(chunkRaw, mapReviver);
+                if (Array.isArray(chunkData))
+                    entries.set(chunkHash, chunkData);
+            }
+            catch {
+                /* skip corrupt or missing shard */
+            }
+        }
+        return { version: PARSE_CACHE_VERSION, entries, usedKeys: new Set() };
+    }
+    catch {
+        return null;
+    }
+};
+/**
+ * Load the parse cache. Returns an empty cache on any failure (missing
+ * file, corrupt JSON, version mismatch). Never throws on a normal load.
+ */
+export const loadParseCache = async (storagePath) => {
+    const sharded = await loadShardedParseCache(storagePath);
+    if (sharded)
+        return sharded;
+    return loadLegacyParseCache(storagePath);
+};
 /**
- * Persist the cache to disk atomically (write-and-rename) so a crash
- * mid-write doesn't leave a corrupt file.
+ * Persist the cache to disk using a temp directory + rename.
+ *
+ * Writes shards under `${cacheDir}.tmp`, then removes the old `cacheDir` and
+ * renames the temp directory into place. There is a crash window after
+ * `rm(cacheDir)` and before `rename(tmpDir, cacheDir)` where no cache exists;
+ * that is acceptable — `loadParseCache` yields empty and the next run
+ * reparses. This is not a single atomic swap of the whole tree, but avoids
+ * leaving a half-written shard set visible to readers.
  */
 export const saveParseCache = async (storagePath, cache) => {
     await fs.mkdir(storagePath, { recursive: true });
-    const cachePath = path.join(storagePath, CACHE_FILENAME);
-    const tmpPath = `${cachePath}.tmp`;
-    const out = {
+    const cacheDir = getCacheDirPath(storagePath);
+    const tmpDir = `${cacheDir}.tmp`;
+    await fs.rm(tmpDir, { recursive: true, force: true });
+    await fs.mkdir(tmpDir, { recursive: true });
+    const keys = [];
+    for (const [chunkHash, chunkResults] of cache.entries) {
+        if (!isValidChunkCacheKey(chunkHash))
+            continue;
+        let payload;
+        try {
+            payload = JSON.stringify(chunkResults, mapReplacer);
+        }
+        catch {
+            // Extremely dense chunks could theoretically exceed string limits; skip
+            // rather than failing the entire save (orchestrator catches save errors).
+            continue;
+        }
+        keys.push(chunkHash);
+        const chunkPath = path.join(tmpDir, `${chunkHash}.json`);
+        await fs.writeFile(chunkPath, payload, 'utf-8');
+    }
+    const index = {
         version: cache.version,
-        entries: Object.fromEntries(cache.entries),
+        keys,
     };
-    // Compact JSON; this file can be tens of MB on a large repo and pretty-
-    // printing roughly doubles size for no value.
-    await fs.writeFile(tmpPath, JSON.stringify(out, mapReplacer), 'utf-8');
-    await fs.rename(tmpPath, cachePath);
+    await fs.writeFile(path.join(tmpDir, CACHE_INDEX_FILENAME), JSON.stringify(index), 'utf-8');
+    await fs.rm(cacheDir, { recursive: true, force: true });
+    await fs.rename(tmpDir, cacheDir);
+    await fs.rm(getLegacyCachePath(storagePath), { force: true });
 };
 /**
  * Drop entries whose hashes are not in `usedHashes`. Called at the end

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gitnexus",
-  "version": "1.6.5-rc.45",
+  "version": "1.6.5-rc.46",
   "description": "Graph-powered code intelligence for AI agents. Index any codebase, query via MCP or CLI.",
   "author": "Abhigyan Patwari",
   "license": "PolyForm-Noncommercial-1.0.0",