gitnexus 1.6.9 → 1.6.10-aptos.1

package/dist/core/lbug/extension-loader.d.ts

@@ -1,3 +1,4 @@
+import { type ExtensionLoadDiagnosis } from './extension-load-error.js';
 /**
  * Lifecycle policy for an optional DuckDB extension.
  *
@@ -20,6 +21,12 @@ export interface ExtensionCapability {
     loaded: boolean;
     /** Human-readable reason when `loaded` is false. */
     reason?: string;
+    /**
+     * Classified diagnosis of `reason`, computed ONCE at mark-unavailable time so
+     * per-request surfaces (ftsDegradedWarning on /api/search + MCP query) read the
+     * cached remedy instead of re-inspecting the extension file on every call (#2383 F3).
+     */
+    diagnosis?: ExtensionLoadDiagnosis;
 }
 /** Per-call overrides applied on top of `ExtensionManager` defaults. */
 export interface ExtensionEnsureOptions {
@@ -29,7 +36,7 @@ export interface ExtensionEnsureOptions {
 export interface ExtensionManagerOptions {
     policy?: ExtensionInstallPolicy;
     installTimeoutMs?: number;
-    installExtension?: (extensionName: string, timeoutMs: number) => Promise<ExtensionInstallResult>;
+    installExtension?: (extensionName: string, timeoutMs: number, loadError?: string) => Promise<ExtensionInstallResult>;
     warn?: (message: string) => void;
 }
 export declare const getExtensionInstallPolicy: () => ExtensionInstallPolicy;
@@ -58,7 +65,7 @@ export declare const getExtensionInstallChildProcessArgs: (extensionName: string
  * If the child exceeds `timeoutMs` the parent kills it with SIGKILL and
  * resolves with `timedOut: true`.
  */
-export declare const installDuckDbExtensionOutOfProcess: (extensionName: string, timeoutMs?: number) => Promise<ExtensionInstallResult>;
+export declare const installDuckDbExtensionOutOfProcess: (extensionName: string, timeoutMs?: number, loadError?: string) => Promise<ExtensionInstallResult>;
 /**
  * Centralized lifecycle manager for optional LadybugDB extensions.
  *
@@ -90,6 +97,14 @@ export declare class ExtensionManager {
      * paths are expected to degrade gracefully.
      */
     ensure(query: (sql: string) => Promise<unknown>, name: string, label: string, opts?: ExtensionEnsureOptions): Promise<boolean>;
+    /**
+     * Attempt `LOAD EXTENSION <name>`; returns `null` on success and the
+     * collapsed error message on failure. The message is the load-side ground
+     * truth — LadybugDB distinguishes a missing extension file from a present
+     * but unloadable one (wrong platform, truncated download, version mismatch),
+     * and discarding it left users staring at "not pre-installed" when the file
+     * existed all along (#2374).
+     */
     private tryLoad;
     private markLoaded;
     private markUnavailable;

package/dist/core/lbug/extension-loader.js

@@ -1,12 +1,15 @@
 import { spawn } from 'child_process';
 import { fileURLToPath } from 'node:url';
 import { LBUG_MAX_DB_SIZE } from './lbug-config.js';
+import { diagnoseExtensionLoad } from './extension-load-error.js';
 import { logger } from '../logger.js';
 const DEFAULT_EXTENSION_INSTALL_TIMEOUT_MS = 15_000;
 const EXTENSION_NAME_PATTERN = /^[A-Za-z][A-Za-z0-9_]*$/;
 const alreadyAvailable = (message) => message.includes('already loaded') ||
     message.includes('already installed') ||
     message.includes('already exists');
+/** LadybugDB errors are multi-line; collapse for single-line warn/reason strings. */
+const oneLine = (value) => value.replace(/\s+/g, ' ').trim();
 const resolvePolicyFromEnv = () => {
     const raw = process.env.GITNEXUS_LBUG_EXTENSION_INSTALL;
     if (raw === 'load-only' || raw === 'never' || raw === 'auto')
@@ -51,7 +54,7 @@ export const getExtensionInstallChildProcessArgs = (extensionName, maxDbSize = L
  * If the child exceeds `timeoutMs` the parent kills it with SIGKILL and
  * resolves with `timedOut: true`.
  */
-export const installDuckDbExtensionOutOfProcess = async (extensionName, timeoutMs = getExtensionInstallTimeoutMs()) => {
+export const installDuckDbExtensionOutOfProcess = async (extensionName, timeoutMs = getExtensionInstallTimeoutMs(), loadError) => {
     if (!EXTENSION_NAME_PATTERN.test(extensionName)) {
         throw new Error(`Invalid DuckDB extension name: ${extensionName}`);
     }
@@ -60,6 +63,9 @@ export const installDuckDbExtensionOutOfProcess = async (extensionName, timeoutM
             env: {
                 ...process.env,
                 GITNEXUS_LBUG_EXTENSION_NAME: extensionName,
+                // The child picks INSTALL vs FORCE INSTALL from this LOAD error so it
+                // only re-downloads when the on-disk extension file is actually broken.
+                ...(loadError ? { GITNEXUS_LBUG_EXTENSION_LOAD_ERROR: loadError } : {}),
             },
             stdio: ['ignore', 'ignore', 'pipe'],
             windowsHide: true,
@@ -78,7 +84,7 @@ export const installDuckDbExtensionOutOfProcess = async (extensionName, timeoutM
             resolve({
                 success: false,
                 timedOut: true,
-                message: `INSTALL ${extensionName} timed out after ${timeoutMs}ms`,
+                message: `extension install for ${extensionName} timed out after ${timeoutMs}ms`,
             });
         }, timeoutMs);
         child.on('error', (err) => {
@@ -97,8 +103,8 @@ export const installDuckDbExtensionOutOfProcess = async (extensionName, timeoutM
                 success: code === 0,
                 timedOut: false,
                 message: code === 0
-                    ? `INSTALL ${extensionName} completed`
-                    : `INSTALL ${extensionName} failed with ${signal ?? `exit code ${code}`}${stderr ? `: ${stderr.trim()}` : ''}`,
+                    ? `extension install for ${extensionName} completed`
+                    : `extension install for ${extensionName} failed with ${signal ?? `exit code ${code}`}${stderr ? `: ${stderr.trim()}` : ''}`,
             });
         });
     });
@@ -152,46 +158,65 @@ export class ExtensionManager {
             this.markUnavailable(name, label, 'extension install policy is "never"', warn);
             return false;
         }
-        if (await this.tryLoad(query, name)) {
+        const loadError = await this.tryLoad(query, name);
+        if (loadError === null) {
             this.markLoaded(name);
             return true;
         }
         if (policy === 'load-only') {
-            this.markUnavailable(name, label, 'load-only policy: extension not pre-installed', warn);
+            this.markUnavailable(name, label, `load-only policy (no install attempted); LOAD ${name} failed: ${loadError}`, warn);
             return false;
         }
         let install = this.installAttempted.get(name);
         if (!install) {
             const installFn = this.options.installExtension ?? installDuckDbExtensionOutOfProcess;
-            install = await installFn(name, timeoutMs);
+            // Hand the child the LOAD error so it re-downloads (FORCE) only when the
+            // present extension file is provably broken, not on every LOAD failure.
+            install = await installFn(name, timeoutMs, loadError);
             this.installAttempted.set(name, install);
         }
         if (!install.success) {
-            this.markUnavailable(name, label, install.message, warn);
+            this.markUnavailable(name, label, `${install.message}; LOAD ${name} had failed: ${loadError}`, warn);
             return false;
         }
-        if (await this.tryLoad(query, name)) {
+        const retryError = await this.tryLoad(query, name);
+        if (retryError === null) {
             this.markLoaded(name);
             return true;
         }
-        this.markUnavailable(name, label, `LOAD ${name} failed after successful INSTALL`, warn);
+        this.markUnavailable(name, label, `LOAD ${name} failed after successful INSTALL: ${retryError}`, warn);
         return false;
     }
+    /**
+     * Attempt `LOAD EXTENSION <name>`; returns `null` on success and the
+     * collapsed error message on failure. The message is the load-side ground
+     * truth — LadybugDB distinguishes a missing extension file from a present
+     * but unloadable one (wrong platform, truncated download, version mismatch),
+     * and discarding it left users staring at "not pre-installed" when the file
+     * existed all along (#2374).
+     */
     async tryLoad(query, name) {
         try {
             await query(`LOAD EXTENSION ${name}`);
-            return true;
+            return null;
         }
         catch (err) {
             const msg = err instanceof Error ? err.message : String(err);
-            return alreadyAvailable(msg);
+            return alreadyAvailable(msg) ? null : oneLine(msg);
         }
     }
     markLoaded(name) {
         this.capabilities.set(name, { name, loaded: true });
     }
     markUnavailable(name, label, reason, warn) {
-        this.capabilities.set(name, { name, loaded: false, reason });
+        // Classify once here (the single load-failure sink, run per Database not per
+        // request) so the hot per-request warning path does no file I/O (#2383 F3).
+        this.capabilities.set(name, {
+            name,
+            loaded: false,
+            reason,
+            diagnosis: diagnoseExtensionLoad(reason, label),
+        });
         const key = `${name}:${reason}`;
         if (this.warnedKeys.has(key))
             return;

package/dist/core/lbug/lbug-adapter.d.ts

@@ -2,7 +2,7 @@ import lbug from '@ladybugdb/core';
 import { KnowledgeGraph } from '../graph/types.js';
 import { NodeTableName } from './schema.js';
 import type { PdgEmitManifest } from './pdg-emit-sink.js';
-import type { CachedEmbedding } from '../embeddings/types.js';
+import { type CachedEmbedding } from '../embeddings/types.js';
 import { type ExtensionEnsureOptions } from './extension-loader.js';
 /** Result of splitting the relationship CSV into per-label-pair files. */
 export interface RelCsvSplitResult {
@@ -83,6 +83,21 @@ pdgEmitManifest?: PdgEmitManifest) => Promise<{
     warnings: string[];
 }>;
 export declare const COPY_CSV_OPTS = "(HEADER=true, ESCAPE='\"', DELIM=',', QUOTE='\"', PARALLEL=false, auto_detect=false)";
+/**
+ * Fallback: insert relationships one-by-one if COPY fails.
+ *
+ * Exported for the quoted-id round-trip tests in
+ * `test/integration/lbug-core-adapter.test.ts` (the `DELETE_FILES_CHUNK_SIZE`
+ * exported-for-tests precedent); production callers stay in this module.
+ * Bails silently when the adapter singleton is closed.
+ *
+ * KNOWN PRE-EXISTING NARROWING (distinct from the `''` escaping bug, NOT
+ * fixed here): the row regex below matches CSV fields with `[^"]*`, so an id
+ * containing a double quote (CSV-escaped as `""`) never matches and the edge
+ * is skipped. Tracked as part of the quote-in-id divergence documented in
+ * `rel-pair-routing.ts`.
+ */
+export declare const fallbackRelationshipInserts: (validRelLines: string[], validTables: Set<string>, getNodeLabel: (id: string) => string) => Promise<void>;
 export declare const getCopyQuery: (table: NodeTableName, filePath: string) => string;
 /**
  * Insert a single node to LadybugDB
@@ -198,6 +213,76 @@ export declare const safeClose: () => Promise<void>;
  */
 export declare const closeLbugBeforeExit: () => Promise<void>;
 export declare const closeLbug: () => Promise<void>;
+/**
+ * Thrown by {@link wipeLbugDbFiles} when a data-bearing member of the
+ * LadybugDB file family is still present after the bounded
+ * remove-and-verify retries (#2409, tri-review 4669518496 P2-4), and by
+ * run-analyze's dirty-recovery block when the crashed run's sidecars can
+ * neither be parked nor removed (this shipping review, FIX 1 — same lock
+ * class, same remediation, and the CLI already renders this type).
+ *
+ * Classify by TYPE (`err instanceof LbugWipeError`) — the repo norm from
+ * #2385 — never by message text. The MESSAGE is nonetheless fully
+ * self-contained (headline + blocked paths + remediation) because
+ * `gitnexus serve` forwards only `err.message` over worker IPC
+ * (analyze-worker-core.ts), so the serve surface has nothing but this
+ * string to show the user. The holder framing deliberately covers the
+ * own-process case (FIX 2, finder A): the blocking handle is often a
+ * lingering one from THIS process's just-closed DB or a transient AV scan
+ * — not necessarily another process — so an immediate re-run often
+ * succeeds.
+ */
+export declare class LbugWipeError extends Error {
+    /** Paths still present (or unverifiable) after all retries. */
+    readonly survivors: readonly string[];
+    constructor(survivors: readonly string[], options?: {
+        headline?: string;
+    });
+}
+/**
+ * Remove the LadybugDB file family and VERIFY each member is really gone.
+ *
+ * Owns the canonical 4-file family list — `<lbugPath>`, `.wal`, `.shadow`,
+ * `.lock` — so run-analyze's two wipe sites (full rebuild + the #2409
+ * escalation valve) can never drift apart. `.shadow` is included because a
+ * checkpoint-in-flight crash leaves a shadow sidecar, and a stale shadow next
+ * to a freshly created DB file is replay poison on the next open (#2409).
+ *
+ * Verification contract (tri-review 4669518496 P2-4 — the old inline loops
+ * swallowed rm failures and let `initLbug` reopen a still-populated DB the
+ * run believed it wiped): after `fs.rm({ recursive, force })`, each path is
+ * probed and counts as GONE only when the probe rejects with **ENOENT**. A
+ * resolving probe, or a rejection in the EPERM/EBUSY/EACCES class (Windows
+ * delete-pending / handle-release lag — see HANDLE_RELEASE_LOCK_CODES in
+ * lbug-config.ts), or any other code means the path is not verifiably gone:
+ * it is retried on the shared handle-release budget
+ * (HANDLE_RELEASE_PROBE_ATTEMPTS × linear HANDLE_RELEASE_PROBE_DELAY_MS,
+ * lbug-config.ts — the previous private mirror constants were
+ * documentation-coupled copies) and then handled by CLASS (this shipping
+ * review, FIX 2):
+ *
+ *   - DATA-BEARING members (`<lbugPath>`, `.wal`, `.shadow`) — a survivor
+ *     means the reopen would resurrect rows this run believes wiped: throw
+ *     a typed {@link LbugWipeError}.
+ *   - `.lock` — contentless: `initLbug` recreates it, and a genuinely held
+ *     lock surfaces as initLbug's own lock-busy classification (a better
+ *     error than this one). A `.lock`-only survivor (an AV-held
+ *     delete-pending handle outlasting the budget previously failed a
+ *     perfectly sound rebuild) logs a warning and CONTINUES.
+ *
+ * Linux unlinked-but-open (name gone, holder keeps the old inode) probes
+ * ENOENT and is accepted by design — both production wipe sites run after a
+ * real `closeLbug()`.
+ *
+ * Deliberately OUT of this contract: `cleanupOldKuzuFiles`
+ * (repo-manager.ts) sweeps the LEGACY kuzu-era file family during storage
+ * migration — different family, best-effort by design; and
+ * `sweepStaleSidecars` (lbug-config.ts) is a test-fixture-gated open-retry
+ * fallback that must never delete production files. Neither wipes the live
+ * DB the run is about to recreate, so neither needs (or may share) the
+ * loud-failure contract here.
+ */
+export declare const wipeLbugDbFiles: (lbugPath: string) => Promise<void>;
 export declare const isLbugReady: () => boolean;
 /**
  * Delete all nodes (and their relationships) for a specific file from LadybugDB
@@ -208,6 +293,43 @@ export declare const isLbugReady: () => boolean;
 export declare const deleteNodesForFile: (filePath: string, dbPath?: string) => Promise<{
     deletedNodes: number;
 }>;
+/**
+ * Chunk size for {@link deleteNodesForFiles}. 200 paths keeps each
+ * statement ~13KB (well inside parser limits) while a ~700-file write set
+ * still collapses from ~13,000 statements to 124: 31 statements per chunk
+ * (1 CodeEmbedding join-delete + 30 filePath-bearing node tables — the
+ * 32-table NODE_TABLES roster minus Community/Process) × 4 chunks. The
+ * original "~40" claim under-counted the per-chunk statement fan-out
+ * (tri-review 4669518496 accuracy sweep).
+ */
+export declare const DELETE_FILES_CHUNK_SIZE = 200;
+/**
+ * Batched variant of {@link deleteNodesForFile} for the incremental
+ * writeback (#2409). One `DETACH DELETE … WHERE n.filePath IN […]` per
+ * node table per chunk of paths, instead of a count + delete per table
+ * per FILE. The per-file loop issued ~13,000 single-row write
+ * transactions on a ~700-file write set — a WAL-append storm that made
+ * the incremental path slower than a full rebuild and is the write
+ * pattern behind the native mid-writeback deaths reported in #2409.
+ *
+ * NO general error swallowing: a zero-match chunk is a no-op success by
+ * construction (every node table except Community/Process has a filePath
+ * column), so anything thrown here is a real engine failure the caller
+ * must see — silently skipping was exactly how #2409 hid its root cause.
+ * The single tolerated exception (FIX 4) is the missing-embedding-table
+ * binder error on the embedding join-delete: a DB created without
+ * EMBEDDING_SCHEMA cannot own embedding rows, so skipping that one
+ * statement is sound, while failing would brick every incremental run on
+ * such a DB until `--force`. Statement count per chunk is unchanged by the
+ * multi-label join: 1 embedding join-delete + 30 node-table deletes = 31
+ * (the rejected per-label fallback shape would have been 19 + 30 = 49).
+ * Singleton-connection only: the analyze writeback owns the write lock,
+ * and `queryAndDrain` routes through `withConnLock` for it (the WAL
+ * checkpoint driver is live during this).
+ */
+export declare const deleteNodesForFiles: (filePaths: readonly string[], options?: {
+    onChunk?: (filesDone: number, filesTotal: number) => void;
+}) => Promise<void>;
 export declare const getEmbeddingTableName: () => string;
 /**
  * Return the distinct repo-relative paths of files that import
@@ -225,6 +347,30 @@ export declare const getEmbeddingTableName: () => string;
  * exports.
  */
 export declare const queryImporters: (targetFilePath: string) => Promise<string[]>;
+/**
+ * Batched variant of {@link queryImporters} for the incremental importer
+ * BFS (#2409): distinct importers of ANY of the target paths, one query per
+ * chunk per BFS depth instead of one query per frontier FILE (a ~700-file
+ * frontier was ~700 sequential round-trips, each taking the connection lock
+ * against the live WAL checkpoint driver — ~5.6s of the writeback measured).
+ *
+ * Same contract as the singular form: reads the pre-pipeline DB state and
+ * swallows per-chunk query failures into a smaller result (correctness
+ * degrades on that branch — under-expansion means possibly-stale edges —
+ * but the DB stays writable and the writeback proceeds). Unlike the singular
+ * form the degradation is not silent (tri-review 4669518496 P2-5): every
+ * dropped chunk is logged and reported through `options.onChunkFailure`, so
+ * the orchestrator can count it into the #2410 crash diagnostics
+ * (`incrementalInProgress.droppedImporterChunks`).
+ */
+export declare const queryImportersBatch: (targetFilePaths: readonly string[], options?: {
+    /**
+     * Invoked once per chunk whose IMPORTS query failed and was dropped from
+     * the expansion. Observability only — the degrade-don't-fail contract is
+     * unchanged (the result just shrinks by the failed chunk's importers).
+     */
+    onChunkFailure?: (chunkIndex: number, chunkSize: number, err: unknown) => void;
+}) => Promise<string[]>;
 /**
  * Drop every Community and Process node (and their MEMBER_OF /
  * STEP_IN_PROCESS edges via DETACH DELETE). Used at the start of an
@@ -338,6 +484,40 @@ export declare const createFTSIndex: (tableName: string, indexName: string, prop
  * failure propagates so the caller can log it.
  */
 export declare const createVectorIndex: () => Promise<boolean>;
+/**
+ * Make DML against {@link EMBEDDING_TABLE_NAME} legal on the writable
+ * connection when it can be, and report whether it is.
+ *
+ * LadybugDB refuses EVERY mutation of a table carrying an HNSW index while
+ * the VECTOR extension is not loaded on that connection: `DELETE` fails with
+ * "Trying to delete from an index on table CodeEmbedding but its extension is
+ * not loaded", `CREATE` with the matching "insert into an index" variant,
+ * `DROP TABLE` is refused while the index references it, and `SET` — even on
+ * a NON-indexed property — segfaults the process outright. Probed against
+ * @ladybugdb/core 0.18.2 (the lockfile-pinned version) and 0.18.0 — every
+ * result identical on both (#2623).
+ *
+ * Dropping the index is NOT an available recovery: `CALL DROP_VECTOR_INDEX`
+ * is itself a VECTOR-extension function and resolves to "Catalog exception:
+ * function DROP_VECTOR_INDEX is not defined" in exactly the state it would
+ * need to rescue. Loading the extension is the only in-place repair, which is
+ * why this returns a verdict instead of attempting a fixup.
+ *
+ * `true` = embedding-row DML is safe: either VECTOR is now loaded, or the
+ * table carries no index to trip over. `false` = genuinely blocked (index
+ * present, extension unloadable); the analyze orchestrator answers that by
+ * escalating to the wipe-and-rebuild write plan instead of failing
+ * mid-writeback.
+ *
+ * Cheap by construction: one local `SHOW_INDEXES` read settles the common
+ * "this repo never built an embedding index" case without touching the
+ * extension machinery at all, so a VECTOR-less machine is not charged a
+ * bounded INSTALL attempt on every incremental analyze. `SHOW_INDEXES` is
+ * readable WITHOUT the extension and reports `extension_loaded` per index, so
+ * no error-string sniffing is needed; it runs through the unprepared
+ * `conn.query()` path like every other `CALL` procedure here (#2114).
+ */
+export declare const ensureEmbeddingRowDmlSafe: () => Promise<boolean>;
 /**
  * Lazy-create an FTS index, caching the fact in-process.
  *
@@ -373,6 +553,27 @@ export declare const queryFTS: (tableName: string, indexName: string, query: str
     [key: string]: any;
 }>>;
 /**
- * Drop an FTS index
+ * True for the two benign "nothing to drop" `DROP_FTS_INDEX` failures —
+ * both catalog/binder exceptions, LadybugDB's classes for "this name isn't
+ * bound to anything right now" (probe-verified end-to-end through
+ * `dropFTSIndex`'s real `conn.query()` path against @ladybugdb/core
+ * 0.18.x): the named index was never created (`Binder exception: Table <T>
+ * doesn't have an index with name <name>.`), or the FTS extension/function
+ * isn't registered at all (`Catalog exception: function DROP_FTS_INDEX is
+ * not defined...`). A real engine failure — e.g. the `Runtime exception:
+ * FTS index '<name>' is inconsistent: ...` class from #2589 — is a
+ * DIFFERENT exception class (an execution-time failure, not a catalog/bind
+ * lookup miss), so this returns false for it. Anchored to the START of the
+ * message (not a bare substring search): every probed LadybugDB error leads
+ * with its exception class, and anchoring means a future message that merely
+ * mentions "Binder exception" or "Catalog exception" further in in the body
+ * of an otherwise-genuine failure can't be misclassified as benign. Pure
+ * string logic so it is unit-testable without a native LadybugDB connection.
+ */
+export declare const isBenignDropFtsIndexError: (message: string) => boolean;
+/**
+ * Drop an FTS index. Tolerates only {@link isBenignDropFtsIndexError} —
+ * anything else rethrows instead of being silently masked, which previously
+ * let a corrupted index persist across analyze runs undetected.
  */
 export declare const dropFTSIndex: (tableName: string, indexName: string) => Promise<void>;