gitnexus 1.6.11-rc.2 → 1.6.11-rc.21

package/dist/storage/parse-cache.js

@@ -6,10 +6,12 @@
  * does is skip the tree-sitter worker dispatch when a chunk's contents
  * haven't changed since the last run.
  *
- * Granularity: chunk-level. The parse phase chunks files into ~20MB byte
- * budgets. The cache key is `sha256(joined(filePath:contentHash for each
- * file in the chunk, sorted))`. A change to a single file invalidates only
- * that file's chunk — typically 1 of ~50 chunks on a 1000-file repo.
+ * Granularity: chunk-level. Files are assigned to a stable
+ * `(language, hash(path) mod 128)` bucket, then packed to a 2 MiB (or
+ * operator) byte budget *inside* that bucket. Membership does not depend
+ * on worker count. The cache key is `sha256(joined(filePath:contentHash
+ * for each file in the chunk, sorted))`. A content edit invalidates only
+ * that file's pack; add/delete/rename only the affected bucket.
  *
  * Why not per-file:
  * - Workers process sub-batches and emit aggregated `ParseWorkerResult`s.
@@ -26,6 +28,8 @@ import { createRequire } from 'module';
 import fs from 'fs/promises';
 import path from 'path';
 import { fileURLToPath } from 'url';
+import { compareCodeUnits } from '../lib/utils.js';
+import { copyV8CacheIfPresent, tryLoadV8Cache, writeV8CacheFile } from './v8-sidecar.js';
 /**
  * Cache version composed of:
  *   - A schema bump knob (`SCHEMA_BUMP`) for hand-controlled invalidation
@@ -627,6 +631,16 @@ import { fileURLToPath } from 'url';
 // therefore takes 79, the next free value above origin/main and every open PR
 // found by the contents-API scan at their exact head SHAs.
 //
+// 79 -> 80 for #3088: parse-cache membership is `(language, sha256(path) mod
+// 128)` then the byte budget *inside* that bucket. Worker count is no longer a
+// membership input, so a warm v79 cache keyed sequential scan-order packs (and
+// on multi-worker hosts, pool×2 MiB mega-chunks) must miss. Sidecar-era
+// ParsedFile stores (#3086/#3087) share PARSE_CACHE_VERSION, so both stores
+// invalidate in lockstep. origin/main at allocation is 79; open PRs that still
+// touch gitnexus/src/storage/parse-cache.ts claim 78 (#3060), 71 (#2840), and
+// 2 (#1616) — none claim 80. RE-CHECK AGAINST origin/main AND OPEN PRs
+// IMMEDIATELY BEFORE MERGING.
+//
 // WHY THIS IS STILL A HAND-PICKED NUMBER, when `SCHEMA_FINGERPRINT` next door
 // is a derived sha256 that cannot collide. The derivation exists and already
 // runs: `resolveAnalyzerRunnerIdentity` computes `build.digest` over the
@@ -645,7 +659,28 @@ import { fileURLToPath } from 'url';
 // `route-extractors/` and `workers/` module content — would close the missing-
 // bump axis without invalidating on unrelated churn, and is the real follow-up.
 // RE-CHECK AGAINST origin/main AND OPEN PRs IMMEDIATELY BEFORE MERGING.
-const SCHEMA_BUMP = 79;
+// 80 -> 81: ParsedFile and parse-cache shards are one immutable `.v8` envelope
+// each (no JSON/path/generation siblings). A v80 index still names `.json`
+// keys and would skip workers while scope-resolution found nothing — the
+// #1983 main-thread reparse. origin/main at allocation is 80.
+// 81 -> 82: Java and Kotlin ParsedFile capture side channels now carry
+// programmatic Spring lookup facts. A warm v81 cache has no such facts, so it
+// would skip workers and silently omit the new INJECTS edges. origin/main at
+// allocation is 81.
+// 82 -> 83: Java Lombok @Data/@Getter/@Setter accessor synthesis emits
+// synthetic Method nodes, HAS_METHOD edges, and matching scope captures into
+// ParseWorkerResult / ParsedFile. A warm v82 cache replays pre-Lombok worker
+// output and silently omits those callables. origin/main at allocation is 82.
+// 83 -> 84: Kotlin val/var properties synthesize JVM get/set Method nodes
+// (same provider hook as Java Lombok). A warm v83 cache omits those callables.
+// origin/main at allocation is 83 (Java Lombok on this branch).
+// 84 -> 85: JVM synthetic accessor captures now use the declaration
+// qualified_name key consumed by scope extraction, and Kotlin accessor planning
+// follows JvmAbi naming plus conservative @JvmName suppression. A warm v84
+// cache can replay stale names and declaration metadata.
+// 85 -> 86: Kotlin interface property accessors now record isAbstract on the
+// synthetic Method. A warm v85 cache replays them as concrete.
+const SCHEMA_BUMP = 86;
 const GITNEXUS_PKG_VERSION = (() => {
     try {
         // package.json sits at gitnexus/package.json — two levels up from
@@ -673,16 +708,56 @@ const GITNEXUS_PKG_VERSION = (() => {
     return '0.0.0-unknown';
 })();
 export const PARSE_CACHE_VERSION = `${SCHEMA_BUMP}+${GITNEXUS_PKG_VERSION}`;
+/** SHA-256 hex of a string or buffer (paths for bucket ids, contents for cache keys). */
+const sha256Hex = (input) => createHash('sha256')
+    .update(typeof input === 'string' ? Buffer.from(input) : input)
+    .digest('hex');
+/** Stable parse-cache bucket count (#3088). Changing this requires SCHEMA_BUMP. */
+export const PARSE_CACHE_BUCKET_COUNT = 128;
+/** Bucket id for cache membership: `sha256(path) mod N` without IEEE-754 truncation. */
+export const parseCacheBucketId = (filePath) => Number(BigInt(`0x${sha256Hex(filePath)}`) % BigInt(PARSE_CACHE_BUCKET_COUNT));
+/**
+ * Pack files into parse-cache chunks: group by (language, bucket id), sort
+ * paths inside the group, then cut at `byteBudget`. Bucket visit order is
+ * the lexicographic order of `${language}\\0${bucketId}` keys (deterministic,
+ * independent of scan order and worker count).
+ */
+export const packParseCacheChunks = (files, byteBudget) => {
+    const buckets = new Map();
+    for (const file of files) {
+        const key = `${file.language}\0${parseCacheBucketId(file.path)}`;
+        const list = buckets.get(key);
+        if (list)
+            list.push(file);
+        else
+            buckets.set(key, [file]);
+    }
+    const chunks = [];
+    for (const key of [...buckets.keys()].sort()) {
+        const group = buckets.get(key);
+        group.sort((a, b) => compareCodeUnits(a.path, b.path));
+        let current = [];
+        let bytes = 0;
+        for (const file of group) {
+            if (current.length > 0 && bytes + file.size > byteBudget) {
+                chunks.push(current);
+                current = [];
+                bytes = 0;
+            }
+            current.push(file.path);
+            bytes += file.size;
+        }
+        if (current.length > 0)
+            chunks.push(current);
+    }
+    return chunks;
+};
 const LEGACY_CACHE_FILENAME = 'parse-cache.json';
 const CACHE_DIRNAME = 'parse-cache';
 const CACHE_INDEX_FILENAME = 'index.json';
 /** Keys on disk always come from `computeChunkHash` — 64-char lowercase hex. */
 const CHUNK_CACHE_KEY_HEX_RE = /^[a-f0-9]{64}$/;
 const isValidChunkCacheKey = (chunkHash) => CHUNK_CACHE_KEY_HEX_RE.test(chunkHash);
-/** SHA-256 hex of a single string or buffer. */
-const sha256Hex = (input) => createHash('sha256')
-    .update(typeof input === 'string' ? Buffer.from(input) : input)
-    .digest('hex');
 /** Stable hash of a single file's contents — used by callers to compose a chunk hash. */
 export const fileContentHash = (content) => sha256Hex(content);
 export const computeChunkHash = (entries, pdg = false) => {
@@ -759,7 +834,7 @@ export const mapReviver = (_key, value) => {
 const getLegacyCachePath = (storagePath) => path.join(storagePath, LEGACY_CACHE_FILENAME);
 const getCacheDirPath = (storagePath) => path.join(storagePath, CACHE_DIRNAME);
 const getCacheIndexPath = (storagePath) => path.join(getCacheDirPath(storagePath), CACHE_INDEX_FILENAME);
-const getCacheChunkPath = (storagePath, chunkHash) => path.join(getCacheDirPath(storagePath), `${chunkHash}.json`);
+const getCacheChunkPath = (storagePath, chunkHash) => path.join(getCacheDirPath(storagePath), `${chunkHash}.v8`);
 /**
  * Drop fields that are not replayed by `mergeChunkResults` / parse-impl after
  * RING4-1 (#942). Shrinks on-disk shards and peak RSS during cold runs.
@@ -784,9 +859,12 @@ const readParseCacheChunkFromDisk = async (storagePath, chunkHash) => {
     if (!isValidChunkCacheKey(chunkHash))
         return undefined;
     try {
-        const chunkRaw = await fs.readFile(getCacheChunkPath(storagePath, chunkHash), 'utf-8');
-        const chunkData = JSON.parse(chunkRaw, mapReviver);
-        return Array.isArray(chunkData) ? chunkData : undefined;
+        const chunkPath = getCacheChunkPath(storagePath, chunkHash);
+        const v8Hit = await tryLoadV8Cache(chunkPath);
+        if (v8Hit?.kind === 'hit' && Array.isArray(v8Hit.value)) {
+            return v8Hit.value;
+        }
+        return undefined;
     }
     catch {
         return undefined;
@@ -821,8 +899,17 @@ export const persistParseCacheChunk = async (cache, chunkHash, chunkResults) =>
             await fs.mkdir(cacheDir, { recursive: true });
             createdCacheDirs.add(cacheDir);
         }
-        const payload = JSON.stringify(slim, mapReplacer);
-        await fs.writeFile(getCacheChunkPath(cache.storagePath, chunkHash), payload, 'utf-8');
+        const chunkPath = getCacheChunkPath(cache.storagePath, chunkHash);
+        let ok = await writeV8CacheFile(chunkPath, slim);
+        if (!ok) {
+            await fs.mkdir(cacheDir, { recursive: true });
+            createdCacheDirs.add(cacheDir);
+            ok = await writeV8CacheFile(chunkPath, slim);
+        }
+        if (!ok) {
+            cache.entries.set(chunkHash, slim);
+            return;
+        }
         cache.onDiskKeys ??= new Set();
         cache.onDiskKeys.add(chunkHash);
         cache.entries.delete(chunkHash);
@@ -918,28 +1005,18 @@ export const saveParseCache = async (storagePath, cache) => {
     // index from what we persisted, not from the raw usedKeys snapshot.
     const writtenKeys = [];
     for (const chunkHash of keys) {
-        const chunkPath = path.join(tmpDir, `${chunkHash}.json`);
+        const chunkPath = path.join(tmpDir, `${chunkHash}.v8`);
         const inMemory = cache.entries.get(chunkHash);
         if (inMemory !== undefined) {
-            let payload;
-            try {
-                payload = JSON.stringify(inMemory, mapReplacer);
-            }
-            catch {
-                continue;
+            if (await writeV8CacheFile(chunkPath, inMemory)) {
+                writtenKeys.push(chunkHash);
             }
-            await fs.writeFile(chunkPath, payload, 'utf-8');
-            writtenKeys.push(chunkHash);
             continue;
         }
         const existingPath = getCacheChunkPath(storagePath, chunkHash);
-        try {
-            await fs.copyFile(existingPath, chunkPath);
+        if (await copyV8CacheIfPresent(existingPath, chunkPath)) {
             writtenKeys.push(chunkHash);
         }
-        catch {
-            /* shard missing — skip; next run treats as cache miss */
-        }
     }
     const index = {
         version: cache.version,

package/dist/storage/parsedfile-store.d.ts

@@ -24,11 +24,11 @@
  *
  * ## Shape
  *
- * `<storagePath>/parsedfile-store/<shardId>.json` — one shard per parse chunk,
- * a JSON array of `ParsedFile` serialized with the same `mapReplacer` the parse
- * cache uses (Scope.bindings / Scope.typeBindings are `Map`s). The store is
- * cleared at the start of each parse and after scope-resolution consumes it, so
- * it never lingers and never goes stale across runs.
+ * `<storagePath>/parsedfile-store/<shardId>.v8` — one shard per parse chunk,
+ * a V8 envelope of `ParsedFile[]` (Scope.bindings / Scope.typeBindings stay
+ * `Map`s). The store is cleared at the start of each parse and after
+ * scope-resolution consumes it, so it never lingers and never goes stale
+ * across runs.
  *
  * ## Durable sibling store (`parsedfile-cache/`, warm-cache coverage)
  *
@@ -41,11 +41,11 @@
  * that gap we ALSO write the worker's ParsedFiles to a second, CONTENT-ADDRESSED
  * store keyed by the parse chunk hash (`getDurableParsedFileDir`), which mirrors
  * the parse cache's lifecycle (persists across runs, pruned by `usedKeys`,
- * version-tied via `PARSE_CACHE_VERSION`). On a warm hit the chunk's durable
- * shards are byte-COPIED into the run-scoped store (no re-parse, no
- * re-serialize → byte-identical), so scope-resolution streams them exactly as
- * on a cold run. Content-addressing makes stale reuse impossible: a changed
- * file changes its chunk hash, which misses BOTH stores and re-dispatches.
+ * version-tied via `PARSE_CACHE_VERSION`). On a warm hit the chunk's immutable
+ * durable shards are hardlinked (or atomically copied) into the run store after
+ * their envelope metadata proves complete coverage. That pins a stable snapshot
+ * before workers are skipped, even when another branch refreshes the shared
+ * durable directory concurrently.
  */
 import type { ParsedFile, SymbolDefinition } from '../_shared/index.js';
 /**
@@ -73,30 +73,25 @@ export declare const getParsedFileStoreDir: (storagePath: string) => string;
 /** Remove any prior run's shards so a fresh parse starts clean. Idempotent. */
 export declare const clearParsedFileStore: (storagePath: string) => Promise<void>;
 /**
- * Write one parse chunk's `ParsedFile[]` to the store as a single shard (async).
- * No-op for an empty chunk. `shardId` must be unique within a run. Used by the
- * main-thread no-store-disabled fallback and any non-worker writer; the worker
- * store path uses {@link persistParsedFileShardSync}.
+ * Test seam for #3086. Production always calls {@link forceGc}; unit tests
+ * replace `run` to count cadence without requiring `--expose-gc`.
  */
-export declare const persistParsedFileChunk: (storagePath: string, shardId: string, parsedFiles: readonly ParsedFile[]) => Promise<void>;
+export declare const parsedFileLoadGc: {
+    run: () => void;
+    /** V8 envelope bytes visited between GCs (#3086). Tests may lower this. */
+    byteBudget: number;
+};
 /**
- * Synchronous shard writer for use INSIDE a parse worker (#1983 parallel
- * serialization). The worker is a dedicated thread, so a blocking write there
- * protects the main thread, and a sync write avoids threading `async`/`await`
- * through the synchronous per-file extract loop. Produces byte-identical shards
- * to {@link persistParsedFileChunk} via the shared {@link serializeParsedFileShard}.
- * No-op for an empty chunk. `shardId` must be globally unique for the run (the
- * worker uses `w<threadId>-<seq>`); a duplicate would silently overwrite.
+ * Write one parse chunk's `ParsedFile[]` to the store as a single `.v8` shard.
+ * No-op for an empty chunk. `shardId` must be unique within a run.
  */
-export declare const persistParsedFileShardSync: (storagePath: string, shardId: string, parsedFiles: readonly ParsedFile[]) => void;
+export declare const persistParsedFileChunk: (storagePath: string, shardId: string, parsedFiles: readonly ParsedFile[]) => Promise<boolean>;
 /**
- * Stream the store and return the `ParsedFile`s whose `filePath` is in
- * `wantPaths`, keyed by path. Loads one shard at a time and retains only the
- * matching entries, so peak heap is bounded by (matched set) + (one shard)
- * rather than the whole store. Returns an empty map when the store is absent
- * (e.g. tests, or a run with no worker pool) — callers fall back to a fresh
- * extract for the missing files.
+ * Synchronous shard writer for use INSIDE a parse worker (#1983 parallel
+ * serialization). Returns false on write failure so the worker can keep
+ * ParsedFiles in the result instead of dropping them.
  */
+export declare const persistParsedFileShardSync: (storagePath: string, shardId: string, parsedFiles: readonly ParsedFile[]) => boolean;
 export declare const loadParsedFilesForPaths: (storagePath: string, wantPaths: ReadonlySet<string>) => Promise<Map<string, ParsedFile>>;
 /** Durable store dir — a sibling of `parsedfile-store/`, NEVER cleared per run. */
 export declare const getDurableParsedFileDir: (storagePath: string) => string;
@@ -108,43 +103,22 @@ export declare const getDurableParsedFileDir: (storagePath: string) => string;
  * from accumulating across analyses.
  */
 export declare const prepareDurableParsedFileChunk: (durableDir: string, chunkHash: string) => Promise<void>;
+export declare const persistDurableParsedFileShardSync: (durableDir: string, chunkHash: string, threadId: number, shardSeq: number, parsedFiles: readonly ParsedFile[]) => boolean;
 /**
- * Synchronous durable-shard writer for use INSIDE a parse worker, alongside
- * {@link persistParsedFileShardSync}. Writes the SAME bytes to a content-addressed
- * durable location keyed by the parse chunk hash so a future warm hit can reuse
- * them. `chunkHash`+`threadId`+`shardSeq` is collision-free across the
- * N-shards-per-chunk fan-out and across worker-death retries — the same
- * uniqueness that makes the run-scoped `w<tid>-<seq>` name safe, prefixed by
- * content. No-op for an empty chunk.
- */
-export declare const persistDurableParsedFileShardSync: (durableDir: string, chunkHash: string, threadId: number, shardSeq: number, parsedFiles: readonly ParsedFile[]) => void;
-/**
- * Restore a cached chunk's durable shards into the run-scoped store on a warm
- * hit. A verbatim byte copy (no parse, no re-serialize), so the restored
- * ParsedFiles are byte-identical to a cold run and `loadParsedFilesForPaths`
- * (which keys on `filePath`, not shard name) gives scope-resolution full
- * coverage. The durable shard names already carry the chunk hash, so they never
- * collide with the worker's run-scoped `w<tid>-<seq>` shards. Returns the number
- * of shards restored (0 ⇒ no durable coverage for this chunk; caller treats it
- * as a miss).
- */
-export declare const restoreDurableParsedFileShard: (durableDir: string, runStoragePath: string, chunkHash: string) => Promise<number>;
-/**
- * Read the durable index and return the set of chunk hashes it vouches for,
- * gated on `expectedVersion` (`PARSE_CACHE_VERSION`). A version mismatch or a
- * missing/corrupt index returns the empty set — the caller then treats every
- * chunk as a durable miss and re-dispatches workers (NEVER the main-thread
- * `extractParsedFile` fallback), which rewrites the durable store under the new
- * version. Mirrors `loadParseCache`'s version-invalidation contract.
+ * Validate and snapshot one durable chunk into the run store. Every envelope
+ * must be runtime-compatible and integrity-valid, and together they must match
+ * the path coverage recorded when the durable index was published. Linking
+ * before returning pins the inodes against concurrent branch-cache rotation.
  */
-export declare const loadDurableParsedFileIndex: (durableDir: string, expectedVersion: string) => Promise<Set<string>>;
+export declare const durableChunkHasShards: (runStoragePath: string, chunkHash: string, expectedPaths: ReadonlySet<string>) => Promise<boolean>;
+export declare const loadDurableParsedFileIndex: (durableDir: string, expectedVersion: string) => Promise<Map<string, ReadonlySet<string>>>;
 /**
  * Prune the durable store to `keepKeys` and rewrite its index. `keepKeys` must
  * be the parse cache's surviving on-disk keys (so the two stores stay coherent:
  * a chunk is "cached" iff BOTH its parse-cache shard and its durable shards
  * exist; a quarantined chunk — no parse-cache shard — drops its durable subdir
- * here and re-dispatches next run). Only subdirs with ≥1 shard are indexed
- * (mirrors `saveParseCache`'s written-keys discipline — never vouch for a chunk
- * hash with no backing shard). The index write is tmp+rename atomic.
+ * here and re-dispatches next run). Only chunks whose envelopes all validate
+ * are indexed, together with their exact persisted path coverage (never vouch
+ * for a missing/corrupt shard). The index write is tmp+rename atomic.
  */
 export declare const pruneAndSaveDurableParsedFileStore: (durableDir: string, version: string, keepKeys: ReadonlySet<string>) => Promise<void>;