gitnexus 1.6.10-rc.61 → 1.6.10-rc.63

package/dist/core/lbug/lbug-config.d.ts

@@ -61,6 +61,16 @@ export declare function resolveNativeSafeStorageDir(storagePath: string, subdir:
  */
 export declare const LBUG_MAX_DB_SIZE: number;
 export declare const parseWalCheckpointThreshold: (raw: string | undefined) => number | undefined;
+/**
+ * Size the buffer pool to an estimated graph size (node + relationship count),
+ * clamped to [ADAPTIVE_POOL_FLOOR, defaultBufferPoolSize()]. The estimate can
+ * only *shrink* the pool from the default — never above the 2 GiB / 80%-RAM cap,
+ * never below the COPY-safety floor — so no repo is under-sized or gets more
+ * than the default it would have today.
+ */
+export declare const estimateBufferPool: (graphElementCount: number) => number;
+/** Set (bytes) or clear (`undefined`) the per-run buffer-pool size hint. */
+export declare const setBufferPoolSizeHint: (bytes: number | undefined) => void;
 export declare const WAL_RECOVERY_SUGGESTION = "WAL corruption detected. Run `gitnexus analyze --force` to rebuild the index.";
 export declare function isWalCorruptionError(err: unknown): boolean;
 /**

package/dist/core/lbug/lbug-config.js

@@ -307,6 +307,15 @@ const resolveCheckpointThreshold = () => {
  */
 const DEFAULT_BUFFER_POOL_CAP = 2 * 1024 * 1024 * 1024;
 const BUFFER_POOL_FLOOR = 64 * 1024 * 1024;
+// COPY-safety floor for the adaptive hint (below). LadybugDB's bulk COPY needs
+// working buffer-pool memory that scales with the repo: a 64 MiB pool fails
+// ("buffer pool is full and no memory could be freed") on any non-trivial repo,
+// and even the ~1800-file GitNexus checkout needs ≥256 MiB. So the adaptive
+// size never drops a repo below this — a distinct, higher floor than
+// BUFFER_POOL_FLOOR, which only guards defaultBufferPoolSize on tiny-RAM
+// machines. It is still clamped up to defaultBufferPoolSize, so a machine whose
+// default is below this floor keeps its default rather than over-committing.
+const ADAPTIVE_POOL_FLOOR = 256 * 1024 * 1024;
 const parseBufferPoolSize = (raw) => {
     if (raw === undefined)
         return undefined;
@@ -319,17 +328,66 @@ const parseBufferPoolSize = (raw) => {
     return Math.floor(parsed);
 };
 const defaultBufferPoolSize = () => Math.min(DEFAULT_BUFFER_POOL_CAP, Math.max(BUFFER_POOL_FLOOR, Math.floor(os.totalmem() * 0.8)));
+/**
+ * Clamp an adaptive pool request to [ADAPTIVE_POOL_FLOOR, default]. The lower
+ * bound keeps LadybugDB's COPY viable; the upper bound (defaultBufferPoolSize)
+ * means the hint can only shrink the pool from today's default and can never
+ * exceed the 2 GiB / 80%-RAM cap — and on a machine whose default is below the
+ * COPY floor, the default wins, so the pool is never over-committed.
+ */
+const clampBufferPool = (bytes) => Math.min(defaultBufferPoolSize(), Math.max(ADAPTIVE_POOL_FLOOR, Math.floor(bytes)));
+/**
+ * Buffer-pool bytes to provision per graph element (node + relationship).
+ *
+ * The fixed 2 GiB default is far larger than most repos' working set, and
+ * LadybugDB eagerly commits the pool at DB open — measured: a full
+ * `analyze --force` of the GitNexus checkout takes ~51 s with the 2 GiB pool
+ * vs ~35 s with the ~414 MiB this factor yields (31% faster; the oversized
+ * pool's commit dominates). The pool is a page cache over the on-disk index,
+ * which scales with node/edge count, so a per-element budget sizes it to the
+ * repo. Kept generous so the whole index stays resident (no COPY thrash) and
+ * always clamped to at least ADAPTIVE_POOL_FLOOR; tuned by timing a real
+ * large-repo `analyze --force` at this factor vs a forced 2 GiB pool (the pool
+ * is a native eager allocation, measured with a real analyze, not a build-free
+ * bench — see the emit-path COPY timing note in bench/emit-persistence).
+ */
+const POOL_BYTES_PER_ELEMENT = 4 * 1024;
+/**
+ * Size the buffer pool to an estimated graph size (node + relationship count),
+ * clamped to [ADAPTIVE_POOL_FLOOR, defaultBufferPoolSize()]. The estimate can
+ * only *shrink* the pool from the default — never above the 2 GiB / 80%-RAM cap,
+ * never below the COPY-safety floor — so no repo is under-sized or gets more
+ * than the default it would have today.
+ */
+export const estimateBufferPool = (graphElementCount) => clampBufferPool(graphElementCount * POOL_BYTES_PER_ELEMENT);
+/**
+ * Optional per-run buffer-pool size hint (bytes). The analyze orchestrator sets
+ * it once the graph size is known (after the pipeline, before the DB open) so
+ * the pool is sized to the repo instead of the fixed 2 GiB default, and clears
+ * it at run end. Non-analyze opens (MCP serve, `native-check` `:memory:`) never
+ * set it and keep the default.
+ */
+let bufferPoolSizeHint;
+/** Set (bytes) or clear (`undefined`) the per-run buffer-pool size hint. */
+export const setBufferPoolSizeHint = (bytes) => {
+    bufferPoolSizeHint = bytes;
+};
 /**
  * Resolve the `bufferManagerSize` passed to every `new lbug.Database(...)`.
- * `GITNEXUS_LBUG_BUFFER_POOL_SIZE` (bytes) overrides the default; `0` is a
+ * `GITNEXUS_LBUG_BUFFER_POOL_SIZE` (bytes) overrides everything; `0` is a
  * deliberate escape hatch that restores LadybugDB's native unbounded
- * 80%-of-RAM default. Resolved at call time (not module load) so tests can
- * stub the env var and `os.totalmem`.
+ * 80%-of-RAM default. With no env override, a per-run `setBufferPoolSizeHint`
+ * (clamped to [floor, default]) sizes the pool to the repo; otherwise the
+ * default. Resolved at call time (not module load) so tests can stub the env
+ * var, the hint, and `os.totalmem`.
  */
 const resolveBufferManagerSize = () => {
     const raw = process.env.GITNEXUS_LBUG_BUFFER_POOL_SIZE;
-    if (raw === undefined)
-        return defaultBufferPoolSize();
+    if (raw === undefined) {
+        return bufferPoolSizeHint !== undefined
+            ? clampBufferPool(bufferPoolSizeHint)
+            : defaultBufferPoolSize();
+    }
     const parsed = parseBufferPoolSize(raw);
     if (parsed !== undefined)
         return parsed;

package/dist/core/run-analyze.js

@@ -13,6 +13,7 @@ import fs from 'fs/promises';
 import { runPipelineFromRepo } from './ingestion/pipeline.js';
 import { resetDegradedParseCounter } from './tree-sitter/safe-parse.js';
 import { initLbug, loadGraphToLbug, getLbugStats, executeQuery, executeWithReusedStatement, closeLbug, closeLbugBeforeExit, loadCachedEmbeddings, deleteNodesForFiles, deleteAllCommunitiesAndProcesses, deleteAllInterprocTaintPaths, deleteAllCallSummaries, deleteAllInjects, queryImportersBatch, loadFTSExtension, wipeLbugDbFiles, LbugWipeError, DELETE_FILES_CHUNK_SIZE, } from './lbug/lbug-adapter.js';
+import { estimateBufferPool, setBufferPoolSizeHint } from './lbug/lbug-config.js';
 import { escapeCypherString } from './lbug/cypher-escape.js';
 import { buildSearchIndexesOrDegrade, createSearchFTSIndexes, initialiseSearchFTSStemmer, verifySearchFTSIndexes, } from './search/fts-indexes.js';
 import { cjkSegmentationModeMismatch, getSearchFTSCjkSegmentation, initialiseSearchFTSCjkSegmentation, } from './search/cjk-segmentation.js';
@@ -312,6 +313,10 @@ export async function runFullAnalysis(repoPath, options, callbacks, runnerIdenti
     // (parse-cache, parsedfile-store) and the kuzu-migration cleanup live there
     // and are shared across branches (#2106 KTD7).
     const { storagePath } = getStoragePaths(repoPath);
+    // Start each analyze with a clean buffer-pool hint: any pre-pipeline DB open
+    // (e.g. the embeddings-cache open) falls back to the default until the hint is
+    // set from the built graph below, so a prior run's size can't leak in.
+    setBufferPoolSizeHint(undefined);
     // Clean up stale KuzuDB files from before the LadybugDB migration.
     const kuzuResult = await cleanupOldKuzuFiles(storagePath);
     if (kuzuResult.found && kuzuResult.needsReindex) {
@@ -944,6 +949,13 @@ export async function runFullAnalysis(repoPath, options, callbacks, runnerIdenti
         // a still-populated DB this run believes it wiped.
         await wipeLbugDbFiles(lbugPath);
     }
+    // Size the buffer pool to the graph just built by the pipeline (a page cache
+    // over the on-disk index, which scales with node/edge count) instead of the
+    // fixed 2 GiB default, whose eager commit dominates large-repo analyze. The
+    // size is clamped to [COPY-safety floor, default], so it only ever shrinks
+    // the pool; env override / no-hint paths are unchanged. See
+    // resolveBufferManagerSize / estimateBufferPool.
+    setBufferPoolSizeHint(estimateBufferPool(pipelineResult.graph.nodeCount + pipelineResult.graph.relationshipCount));
     await initLbug(lbugPath);
     // Manual WAL checkpoint driver (#1741): periodically drain the WAL
     // from JS so the un-retriable native auto-checkpoint almost never

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gitnexus",
-  "version": "1.6.10-rc.61",
+  "version": "1.6.10-rc.63",
   "description": "Graph-powered code intelligence for AI agents. Index any codebase, query via MCP or CLI.",
   "author": "Abhigyan Patwari",
   "license": "PolyForm-Noncommercial-1.0.0",