gitnexus 1.6.13-rc.26 → 1.6.13-rc.28

package/dist/core/lbug/lbug-adapter.js

@@ -31,7 +31,7 @@ import { extensionManager, getFtsCapability, resolveAnalyzeInstallPolicy, } from
 import { diagnoseExtensionLoad, extractExtensionPath } from './extension-load-error.js';
 import { resolveFtsVersionPair } from './vendored-extension-path.js';
 import { classifyDeleteAllError, closeLbugConnection, HANDLE_RELEASE_PROBE_ATTEMPTS, HANDLE_RELEASE_PROBE_DELAY_MS, isDbBusyError, isOpenRetryExhausted, isStorageVersionMismatchError, isWalCorruptionError, throwIfStorageVersionMismatch, bufferPoolExhaustionRemedy, openLbugConnection, sleep, toNativeSafePath, resolveNativeSafeStorageDir, WAL_RECOVERY_SUGGESTION, waitForWindowsHandleRelease, } from './lbug-config.js';
-import { assertReadOnlyFtsCrashSafe, cleanQuarantinedMissingShadowWals, finalizeLbugSidecarsAfterClose, guardWalQuarantine, isMissingShadowSidecarError, isReadOnlyShadowReplayError, lbugLockRemediation, preflightLbugSidecars, quarantineWalForMissingShadow, renameFailureMessage, shadowSidecarRecoveryMessage, sidecarPreflightDisabled, } from './sidecar-recovery.js';
+import { assertReadOnlyFtsCrashSafe, cleanQuarantinedMissingShadowWals, finalizeLbugSidecarsAfterClose, guardWalQuarantine, isMissingShadowSidecarError, isReadOnlyCheckpointInProgressError, isReadOnlyRecoveryFailure, isReadOnlyShadowReplayError, lbugLockRemediation, preflightLbugSidecars, quarantineWalForMissingShadow, readOnlyRecoveryFailureMessage, renameFailureMessage, shadowSidecarRecoveryMessage, sidecarPreflightDisabled, } from './sidecar-recovery.js';
 import { isProcessAlive } from '../../utils/process-identity.js';
 import { logger } from '../logger.js';
 import { SPRING_AUTO_CONFIGURATION_REASONS, SPRING_AUTO_CONFIGURATION_SYNTHETIC_ID_PREFIX, } from '../ingestion/frameworks/spring/auto-configuration.js';
@@ -422,6 +422,10 @@ const queryAndDrain = async (targetConn, cypher) => {
 // `queryAndDrain`, which drains and discards the rows; the ONLY observable is
 // whether the read-only shadow replay throws, so no row identity is read.
 const READ_ONLY_SHADOW_REPLAY_PROBE = 'MATCH (n) RETURN n LIMIT 1';
+// The durability half of writable recovery: replayed pages only persist when
+// an explicit CHECKPOINT applies them to the main file (see
+// recoverReadOnlyViaWritableOpen).
+const RECOVERY_CHECKPOINT_QUERY = 'CHECKPOINT';
 /**
  * Serve-side entry to the shared WAL-quarantine safety gate. Refuses (throws)
  * when the `.shadow` is present on disk or the orphan WAL is too large to
@@ -501,13 +505,30 @@ const ensureReadOnlyConnectionUsable = async (dbPath, handle) => {
             await closeLbugConnection(handle);
             return await reopenReadOnlyAfterMissingShadow(dbPath, err);
         }
-        if (!isReadOnlyShadowReplayError(err)) {
+        if (!isReadOnlyShadowReplayError(err) && !isReadOnlyCheckpointInProgressError(err)) {
             await closeLbugConnection(handle);
             throw err;
         }
         shadowReplayErr = err;
     }
     await closeLbugConnection(handle);
+    return await recoverReadOnlyViaWritableOpen(dbPath, shadowReplayErr);
+};
+/**
+ * Clear an interrupted-checkpoint / pending-shadow-replay state by opening the
+ * database WRITABLE once — probe (forces the WAL replay) then an explicit
+ * CHECKPOINT (persists it; see the comment at the call site) — and reopening
+ * read-only. Recovery for every read-only refusal an interrupted checkpoint
+ * produces — `isReadOnlyShadowReplayError` and
+ * `isReadOnlyCheckpointInProgressError` — shared by the probe path
+ * (`ensureReadOnlyConnectionUsable`) and the open path (`doInitLbug`'s
+ * read-only branch, where the native open itself refuses before any probe can
+ * run). Homelab repro 2026-09-19: a wiki pod killed mid-CHECKPOINT left the
+ * checkpoint sidecars behind, and every read-only open thereafter failed until
+ * a writable open (any `gitnexus analyze`) recovered it — this makes the read
+ * path self-heal instead.
+ */
+const recoverReadOnlyViaWritableOpen = async (dbPath, triggeringErr) => {
     let writable;
     try {
         writable = await openLbugConnection(lbug, dbPath);
@@ -515,18 +536,28 @@ const ensureReadOnlyConnectionUsable = async (dbPath, handle) => {
     catch (openErr) {
         const code = extractErrnoCode(openErr);
         if (code === 'EROFS' || code === 'EACCES' || code === 'EPERM') {
-            throw new Error(shadowSidecarRecoveryMessage(dbPath, shadowReplayErr) +
-                '\n  The workspace appears to be read-only — mount it read-write to perform shadow replay recovery,' +
+            throw new Error(readOnlyRecoveryFailureMessage(dbPath, triggeringErr) +
+                '\n  The workspace appears to be read-only — mount it read-write to perform WAL recovery,' +
                 ' or re-run `gitnexus analyze` on a writable filesystem to rebuild the index.');
         }
         throw openErr;
     }
     let missingShadowError;
+    let probeSucceeded = false;
     try {
         await queryAndDrain(writable.conn, READ_ONLY_SHADOW_REPLAY_PROBE);
+        probeSucceeded = true;
+        // Load-bearing durability step (engine 0.19.1 matrix, homelab repro
+        // 2026-09-19): the probe replays the WAL in MEMORY only. Without an
+        // explicit CHECKPOINT the engine drops those pages at close and the
+        // follow-up read-only open silently serves the pre-checkpoint state.
+        // CHECKPOINT applies the replay to the main file, consuming the
+        // `lbug.wal.checkpoint` / `lbug.shadow` sidecars and clearing the
+        // checkpoint locks the interrupted checkpoint left behind.
+        await queryAndDrain(writable.conn, RECOVERY_CHECKPOINT_QUERY);
     }
     catch (err) {
-        if (isMissingShadowSidecarError(err)) {
+        if (!probeSucceeded && isMissingShadowSidecarError(err)) {
             missingShadowError = err;
         }
         else {
@@ -546,8 +577,10 @@ const ensureReadOnlyConnectionUsable = async (dbPath, handle) => {
     }
     catch (err) {
         await closeLbugConnection(reopened);
-        if (isMissingShadowSidecarError(err)) {
-            throw new Error(shadowSidecarRecoveryMessage(dbPath, err));
+        if (isMissingShadowSidecarError(err) ||
+            isReadOnlyShadowReplayError(err) ||
+            isReadOnlyCheckpointInProgressError(err)) {
+            throw new Error(readOnlyRecoveryFailureMessage(dbPath, err));
         }
         throw err;
     }
@@ -700,17 +733,27 @@ const doInitLbug = async (dbPath, options = {}) => {
             usable = await ensureReadOnlyConnectionUsable(dbPath, opened);
         }
         catch (err) {
-            // Not retryable: the on-disk file's storage version doesn't change on
-            // its own, so withLbugDb's retry loop (which only handles
-            // isDbBusyError) would just repeat the same native exception. Fail
-            // immediately with an actionable message instead (review finding on
-            // PR #3189 — this became reachable once the pinned engine version can
-            // trail behind whatever version last wrote an index, e.g. after
-            // downgrading the dependency). Mirrors the pool-adapter.ts check for
-            // the same error, on the separate open path /api/graph and /api/query
-            // actually use (withLbugDb, not the pool).
-            throwIfStorageVersionMismatch(err);
-            throw err;
+            // An interrupted checkpoint can make the OPEN itself refuse read-only
+            // ("Cannot open database in read-only mode while checkpoint is in
+            // progress") before any probe runs. Clear it with one writable open,
+            // then reopen read-only — the same self-heal the probe path applies.
+            // Skip already-wrapped failures so we do not re-enter writable recovery.
+            if (isReadOnlyCheckpointInProgressError(err) && !isReadOnlyRecoveryFailure(err)) {
+                usable = await recoverReadOnlyViaWritableOpen(dbPath, err);
+            }
+            else {
+                // Not retryable: the on-disk file's storage version doesn't change on
+                // its own, so withLbugDb's retry loop (which only handles
+                // isDbBusyError) would just repeat the same native exception. Fail
+                // immediately with an actionable message instead (review finding on
+                // PR #3189 — this became reachable once the pinned engine version can
+                // trail behind whatever version last wrote an index, e.g. after
+                // downgrading the dependency). Mirrors the pool-adapter.ts check for
+                // the same error, on the separate open path /api/graph and /api/query
+                // actually use (withLbugDb, not the pool).
+                throwIfStorageVersionMismatch(err);
+                throw err;
+            }
         }
         db = usable.db;
         conn = usable.conn;

package/dist/core/lbug/pool-adapter.js

@@ -20,7 +20,7 @@ import { isReadOnlyDbError, loadFTSExtension, loadVectorExtension } from './lbug
 import { closeQueryResults } from './query-result-utils.js';
 import { warnIfQueryTextUnbounded } from './query-batch.js';
 import { createLbugDatabase, isWalCorruptionError, sleep, throwIfStorageVersionMismatch, toNativeSafePath, WAL_RECOVERY_SUGGESTION, } from './lbug-config.js';
-import { assertReadOnlyFtsCrashSafe, FtsReaderUnrepairableError, guardWalQuarantine, isMissingFsError, isMissingShadowSidecarError, isReadOnlyShadowReplayError, preflightLbugSidecars, quarantineWalForMissingShadow, renameFailureMessage, statIfExists, } from './sidecar-recovery.js';
+import { assertReadOnlyFtsCrashSafe, FtsReaderUnrepairableError, guardWalQuarantine, isMissingFsError, isMissingShadowSidecarError, isReadOnlyCheckpointInProgressError, isReadOnlyShadowReplayError, preflightLbugSidecars, quarantineWalForMissingShadow, renameFailureMessage, statIfExists, } from './sidecar-recovery.js';
 export async function statDbIdentity(dbPath) {
     try {
         const s = await fs.stat(dbPath);
@@ -481,13 +481,35 @@ async function probeDatabaseForShadowReplay(db) {
 }
 async function replayShadowPagesWithWritableOpen(dbPath) {
     let db;
+    // Mirrors the direct adapter's `probeSucceeded` guard: once the probe has
+    // replayed, a MISSING-SHADOW error can only come from the CHECKPOINT itself
+    // — quarantining the WAL then would park a live sidecar on a db whose main
+    // file just changed underneath it. Fail closed instead (review finding:
+    // policy drift vs the serve path).
+    let replaySucceeded = false;
     try {
         db = createLbugDatabase(lbug, toNativeSafePath(dbPath), { throwOnWalReplayFailure: false });
         await db.init();
         await probeDatabaseForShadowReplay(db);
+        replaySucceeded = true;
+        // Load-bearing durability step (engine 0.19.1 matrix, homelab repro
+        // 2026-09-19): the probe replays the WAL in MEMORY only. Without an
+        // explicit CHECKPOINT the engine drops those pages at close and the
+        // follow-up read-only open silently serves the pre-checkpoint state.
+        const conn = createConnection(db);
+        try {
+            const checkpointResult = await conn.query('CHECKPOINT');
+            const result = Array.isArray(checkpointResult) ? checkpointResult[0] : checkpointResult;
+            await result.getAll();
+            // Shared best-effort closer (awaits + swallows) — never roll this loop.
+            await closeQueryResults(result);
+        }
+        finally {
+            await conn.close().catch(() => { });
+        }
     }
     catch (err) {
-        if (isMissingShadowSidecarError(err)) {
+        if (isMissingShadowSidecarError(err) && !replaySucceeded) {
             await tryQuarantineForMissingShadow(dbPath, {
                 reason: 'pool writable replay recovery',
                 err,
@@ -515,8 +537,14 @@ async function openReadOnlyDatabase(dbPath) {
             readOnly: true,
             throwOnWalReplayFailure: false,
         });
-        await db.init();
+        // init() is inside the try: an interrupted checkpoint (pod killed
+        // mid-CHECKPOINT leaves `lbug.wal` + `lbug.shadow`) can make the read-only
+        // OPEN itself refuse — "Cannot open database in read-only mode while
+        // checkpoint is in progress" — before any probe runs (homelab repro
+        // 2026-09-19). Both refusal classes recover identically below: one
+        // writable open replays the WAL/completes the checkpoint.
         try {
+            await db.init();
             await probeDatabaseForShadowReplay(db);
         }
         catch (err) {
@@ -540,7 +568,7 @@ async function openReadOnlyDatabase(dbPath) {
                 await probeDatabaseForShadowReplay(db);
                 return db;
             }
-            if (!isReadOnlyShadowReplayError(err)) {
+            if (!isReadOnlyShadowReplayError(err) && !isReadOnlyCheckpointInProgressError(err)) {
                 throw err;
             }
             await db.close().catch(() => { });

package/dist/core/lbug/sidecar-recovery.d.ts

@@ -57,6 +57,20 @@ export declare const statIfExists: (filePath: string) => Promise<{
 } | null>;
 export declare const isMissingShadowSidecarError: (err: unknown) => boolean;
 export declare const isReadOnlyShadowReplayError: (err: unknown) => boolean;
+export declare const isReadOnlyCheckpointInProgressError: (err: unknown) => boolean;
+/** Prefix of the interrupted-checkpoint wrap — must not rematch the native classifier. */
+export declare const INTERRUPTED_CHECKPOINT_RECOVERY_PREFIX = "LadybugDB could not finish an interrupted checkpoint";
+/** Prefix of the pending-shadow-replay wrap — must not rematch the native classifier. */
+export declare const PENDING_SHADOW_REPLAY_RECOVERY_PREFIX = "LadybugDB could not finish a pending shadow replay";
+/** True when `err` is already a classifier-aware recovery wrap (not a native refusal). */
+export declare const isReadOnlyRecoveryFailure: (err: unknown) => boolean;
+/**
+ * Operator-facing wrap for a failed read-only self-heal. Checkpoint-in-progress
+ * and pending-shadow-replay must not reuse `shadowSidecarRecoveryMessage` —
+ * that copy claims the sidecar is missing and appends the native text, which
+ * rematches the classifiers and can re-enter writable CHECKPOINT.
+ */
+export declare const readOnlyRecoveryFailureMessage: (dbPath: string, err: unknown) => string;
 export declare const shadowSidecarRecoveryMessage: (dbPath: string, err: unknown) => string;
 /**
  * Actionable message for the case where LadybugDB reports a "missing shadow"

package/dist/core/lbug/sidecar-recovery.js

@@ -189,6 +189,48 @@ export const isReadOnlyShadowReplayError = (err) => {
     const msg = err instanceof Error ? err.message : String(err);
     return /replay shadow pages under read-only mode/i.test(msg);
 };
+// LADYBUGDB-CONTRACT: native error text. When bumping LadybugDB, re-validate
+// this regex against the new error format — `git grep "LADYBUGDB-CONTRACT"`
+// enumerates every version-coupled spot.
+// Version honesty (review finding on the interrupted-checkpoint PR): this
+// string is FIRST OBSERVED on @ladybugdb/core 0.19.1 — live-reproduced by
+// SIGKILLing a writer mid-CHECKPOINT (homelab 2026-09-19, GitNexus image
+// 1.6.10-20260917, built on 0.19.x). The 0.18.3 binary does NOT contain it
+// (checked via `strings`), and 0.18.3 tolerates the same on-disk state, so on
+// 0.18.x this classifier simply never fires. If the engine pin ever moves
+// back to ^0.19, the read-path self-heal is already in place.
+export const isReadOnlyCheckpointInProgressError = (err) => {
+    const msg = err instanceof Error ? err.message : String(err);
+    return /cannot open database in read-only mode while checkpoint is in progress/i.test(msg);
+};
+/** Prefix of the interrupted-checkpoint wrap — must not rematch the native classifier. */
+export const INTERRUPTED_CHECKPOINT_RECOVERY_PREFIX = 'LadybugDB could not finish an interrupted checkpoint';
+/** Prefix of the pending-shadow-replay wrap — must not rematch the native classifier. */
+export const PENDING_SHADOW_REPLAY_RECOVERY_PREFIX = 'LadybugDB could not finish a pending shadow replay';
+const READ_ONLY_RECOVERY_REMOUNT = 'Mount the workspace read-write or re-run `gitnexus analyze` to complete recovery.';
+/** True when `err` is already a classifier-aware recovery wrap (not a native refusal). */
+export const isReadOnlyRecoveryFailure = (err) => {
+    const msg = err instanceof Error ? err.message : String(err);
+    return (msg.startsWith(INTERRUPTED_CHECKPOINT_RECOVERY_PREFIX) ||
+        msg.startsWith(PENDING_SHADOW_REPLAY_RECOVERY_PREFIX));
+};
+/**
+ * Operator-facing wrap for a failed read-only self-heal. Checkpoint-in-progress
+ * and pending-shadow-replay must not reuse `shadowSidecarRecoveryMessage` —
+ * that copy claims the sidecar is missing and appends the native text, which
+ * rematches the classifiers and can re-enter writable CHECKPOINT.
+ */
+export const readOnlyRecoveryFailureMessage = (dbPath, err) => {
+    if (isReadOnlyCheckpointInProgressError(err)) {
+        return (`${INTERRUPTED_CHECKPOINT_RECOVERY_PREFIX} for ${dbPath}. ` +
+            `${READ_ONLY_RECOVERY_REMOUNT} ` +
+            'Retry later if another writer is still checkpointing.');
+    }
+    if (isReadOnlyShadowReplayError(err)) {
+        return `${PENDING_SHADOW_REPLAY_RECOVERY_PREFIX} for ${dbPath}. ${READ_ONLY_RECOVERY_REMOUNT}`;
+    }
+    return shadowSidecarRecoveryMessage(dbPath, err);
+};
 export const shadowSidecarRecoveryMessage = (dbPath, err) => {
     const msg = err instanceof Error ? err.message : String(err);
     return (`LadybugDB checkpoint sidecar is missing for ${dbPath}. ` +

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gitnexus",
-  "version": "1.6.13-rc.26",
+  "version": "1.6.13-rc.28",
   "description": "Graph-powered code intelligence for AI agents. Index any codebase, query via MCP or CLI.",
   "author": "Abhigyan Patwari",
   "license": "PolyForm-Noncommercial-1.0.0",

package/scripts/cross-platform-tests.ts

@@ -165,6 +165,7 @@ const LBUG_NATIVE = [
   'test/integration/lbug-open-retry.test.ts',
   'test/integration/lbug-close-handle-release.test.ts',
   'test/integration/lbug-orphan-sidecar-recovery.test.ts',
+  'test/integration/lbug-interrupted-checkpoint-recovery.test.ts',
   'test/integration/lbug-readonly-init.test.ts',
   'test/integration/lbug-non-ascii-path.test.ts',
   // Cross-repo trace e2e: builds two real lbug indexes + a real bridge and