gitnexus 1.6.10-rc.185 → 1.6.10-rc.186

package/dist/core/lbug/csv-generator.d.ts

@@ -53,6 +53,29 @@ export declare const shouldFlushCSVBuffer: (byteCount: number) => boolean;
 export declare const sanitizeUTF8: (str: string) => string;
 export declare const escapeCSVField: (value: string | number | undefined | null) => string;
 export declare const escapeCSVNumber: (value: number | undefined | null, defaultValue?: number) => string;
+/**
+ * Did this text come from a binary payload? Density of non-printables over the
+ * first {@link BINARY_SAMPLE_CHARS} characters, above 10%.
+ *
+ * U+FFFD counts, and it is the character that matters most here (#2889). Every
+ * source file enters the pipeline through a `utf-8` decode — the content cache
+ * below reads with `fs.readFile(path, 'utf-8')`, and the parse worker decodes
+ * the same way. That decode is lossy and total: an invalid byte sequence never
+ * survives as invalid bytes, it is REPLACED with U+FFFD. So the embedded-binary
+ * payloads #2889 describes (webpack bundles, class-file constant pools,
+ * serialized objects inside .js/.vue sources) arrive here as long runs of
+ * U+FFFD, not as the control bytes this scan was originally written to count —
+ * charCode 0xFFFD is neither `< 9`, nor between 13 and 32, nor 127, so the
+ * detector scored a wholly corrupt payload as clean text and every caller waved
+ * it through.
+ *
+ * The threshold stays at 10%: a legitimate source file carries no replacement
+ * characters at all unless it was mis-decoded, and a handful (a stray latin-1
+ * comment, one bad byte in a license header) still scores far under the bar.
+ * Density rather than "contains any binary run" is deliberate — a description
+ * that is mostly real prose with one stray replacement character is worth more
+ * indexed than dropped.
+ */
 export declare const isBinaryContent: (content: string) => boolean;
 /**
  * Flatten newlines and tabs to single spaces for FTS-indexed text columns

package/dist/core/lbug/csv-generator.js

@@ -113,17 +113,44 @@ const formatCSVStringArray = (value) => {
 // ============================================================================
 // CONTENT EXTRACTION (lazy — reads from disk on demand)
 // ============================================================================
+const BINARY_SAMPLE_CHARS = 1000;
+const UNICODE_REPLACEMENT_CHAR = 0xfffd;
+/**
+ * Did this text come from a binary payload? Density of non-printables over the
+ * first {@link BINARY_SAMPLE_CHARS} characters, above 10%.
+ *
+ * U+FFFD counts, and it is the character that matters most here (#2889). Every
+ * source file enters the pipeline through a `utf-8` decode — the content cache
+ * below reads with `fs.readFile(path, 'utf-8')`, and the parse worker decodes
+ * the same way. That decode is lossy and total: an invalid byte sequence never
+ * survives as invalid bytes, it is REPLACED with U+FFFD. So the embedded-binary
+ * payloads #2889 describes (webpack bundles, class-file constant pools,
+ * serialized objects inside .js/.vue sources) arrive here as long runs of
+ * U+FFFD, not as the control bytes this scan was originally written to count —
+ * charCode 0xFFFD is neither `< 9`, nor between 13 and 32, nor 127, so the
+ * detector scored a wholly corrupt payload as clean text and every caller waved
+ * it through.
+ *
+ * The threshold stays at 10%: a legitimate source file carries no replacement
+ * characters at all unless it was mis-decoded, and a handful (a stray latin-1
+ * comment, one bad byte in a license header) still scores far under the bar.
+ * Density rather than "contains any binary run" is deliberate — a description
+ * that is mostly real prose with one stray replacement character is worth more
+ * indexed than dropped.
+ */
 export const isBinaryContent = (content) => {
-    if (!content || content.length === 0)
+    // `content &&` keeps the original tolerance for a null/undefined caller —
+    // `strict` is off in this package, so the type alone does not rule it out.
+    const end = content ? Math.min(content.length, BINARY_SAMPLE_CHARS) : 0;
+    if (end === 0)
         return false;
-    const sample = content.slice(0, 1000);
     let nonPrintable = 0;
-    for (let i = 0; i < sample.length; i++) {
-        const code = sample.charCodeAt(i);
-        if (code < 9 || (code > 13 && code < 32) || code === 127)
+    for (let i = 0; i < end; i++) {
+        const code = content.charCodeAt(i);
+        if (code < 9 || (code > 13 && code < 32) || code === 127 || code === UNICODE_REPLACEMENT_CHAR)
             nonPrintable++;
     }
-    return nonPrintable / sample.length > 0.1;
+    return nonPrintable / end > 0.1;
 };
 /**
  * LRU content cache — avoids re-reading the same source file for every
@@ -200,8 +227,20 @@ class FileContentCache {
  * otherwise fail to match whitespace-normalized indexed content.
  */
 export const normalizeFtsText = (text) => text.replace(/[\r\n\t]+/g, ' ');
-/** Composes both FTS-text transforms for the `description` column — one place for the six emission sites below to call, instead of repeating the composition. */
-const formatFtsDescription = (description) => normalizeFtsText(applyCjkSegmentationIfEnabled(description));
+/**
+ * Composes both FTS-text transforms for the `description` column — one place for
+ * the six emission sites below to call, instead of repeating the composition.
+ *
+ * Binary-looking descriptions are dropped rather than transformed (#2889). The
+ * `content` column has always been gated on {@link isBinaryContent} inside
+ * {@link extractContent}; `description` never was, so a symbol whose "doc
+ * comment" is really a slice of an embedded binary payload had that payload
+ * copied verbatim into an FTS-indexed column. Dropping it here rather than at
+ * each of the six call sites keeps the gate in the same place as the transforms
+ * it guards. Empty string, not a sentinel: unlike `content`, a description has
+ * no reader that needs to be told why it is missing.
+ */
+const formatFtsDescription = (description) => isBinaryContent(description) ? '' : normalizeFtsText(applyCjkSegmentationIfEnabled(description));
 // Labels that get exact source-span content (no ±2 window). Single source of
 // truth in `symbol-labels.ts` — see there for why the exactness depends on the
 // 0-based line invariant. Kept as a named alias to read intent at the use site.

package/dist/core/run-analyze.js

@@ -21,7 +21,7 @@ import { resetDegradedParseCounter } from './tree-sitter/safe-parse.js';
 import { initLbug, loadGraphToLbug, getLbugStats, executeQuery, executeWithReusedStatement, closeLbug, closeLbugBeforeExit, loadCachedEmbeddings, deleteNodesForFiles, ensureEmbeddingRowDmlSafe, ensureFtsRowDmlSafe, readIndexCatalogSnapshot, INDEX_CATALOG_UNREADABLE, deleteAllCommunitiesAndProcesses, deleteAllInterprocTaintPaths, deleteAllCallSummaries, deleteAllInjects, deleteAllAdvisedBy, deleteSpringAopEvidenceNodes, deleteSpringAutoConfigurationDeclarations, deleteSpringAutoConfigurationSyntheticClasses, queryImportersBatch, loadFTSExtension, wipeLbugDbFiles, LbugWipeError, DELETE_FILES_CHUNK_SIZE, } from './lbug/lbug-adapter.js';
 import { estimateBufferPool, setBufferPoolSizeHint, resolveNativeSafeStorageDir, } from './lbug/lbug-config.js';
 import { escapeCypherString } from './lbug/cypher-escape.js';
-import { buildSearchIndexesOrDegrade, ftsFailureIsFatal, createSearchFTSIndexes, dropSearchFTSIndexes, initialiseSearchFTSStemmer, verifySearchFTSIndexes, } from './search/fts-indexes.js';
+import { buildSearchIndexesOrDegrade, ftsFailureIsFatal, createSearchFTSIndexes, summarizeFtsIndexBuildFailures, dropSearchFTSIndexes, initialiseSearchFTSStemmer, verifySearchFTSIndexes, } from './search/fts-indexes.js';
 import { cjkSegmentationModeMismatch, getSearchFTSCjkSegmentation, initialiseSearchFTSCjkSegmentation, } from './search/cjk-segmentation.js';
 import { getExtensionCapability, getExtensionCapabilities, getFtsCapability, resolveAnalyzeInstallPolicy, } from './lbug/extension-loader.js';
 import { diagnoseExtensionLoad } from './lbug/extension-load-error.js';
@@ -716,7 +716,7 @@ async function runFullAnalysisInner(repoPath, options, callbacks, writeTarget, r
                     remedyTail);
             }
             progress('fts', 85, 'Repairing search indexes...');
-            await createSearchFTSIndexes({
+            const repairFailures = await createSearchFTSIndexes({
                 onIndexStart: options.verbose
                     ? (table, indexName) => log(`FTS: creating ${table}.${indexName}`)
                     : undefined,
@@ -726,7 +726,14 @@ async function runFullAnalysisInner(repoPath, options, callbacks, writeTarget, r
             });
             const missing = await verifySearchFTSIndexes(executeQuery);
             if (missing.length > 0) {
-                throw new Error(`FTS repair failed - missing indexes after rebuild: ${missing.join(', ')}. ` +
+                // #2889: name WHY each index is missing when the build itself said so.
+                // Repair now rebuilds every table it can before reporting, so the tables
+                // absent from this list were genuinely repaired even on a failed run —
+                // previously the first failure aborted the sweep and the message could
+                // only ever list "missing", never a reason. Same sentence the analyze
+                // degrade path prints, so one failure does not read two ways.
+                const reasons = repairFailures.length > 0 ? ` ${summarizeFtsIndexBuildFailures(repairFailures)}.` : '';
+                throw new Error(`FTS repair failed - missing indexes after rebuild: ${missing.join(', ')}.${reasons} ` +
                     'Run `gitnexus analyze --force` to perform a full graph+FTS rebuild; ' +
                     'if that also fails, verify FTS extension availability via `gitnexus doctor`.');
             }

package/dist/core/search/fts-indexes.d.ts

@@ -99,7 +99,46 @@ export declare function getSearchFTSStemmer(): string;
  * `lbug-adapter.ts`. Omit it to have the sweep read the catalog itself.
  */
 export declare function dropSearchFTSIndexes(indexRows?: IndexCatalogSnapshot): Promise<void>;
-export declare function createSearchFTSIndexes(options?: CreateSearchFTSIndexesOptions): Promise<void>;
+/** One configured index that could not be (re)built, and why. */
+export interface FtsIndexBuildFailure {
+    table: string;
+    indexName: string;
+    /** The raw LadybugDB message, unmodified — it is the only row-level evidence there is. */
+    error: string;
+}
+/**
+ * Build every configured FTS index, and keep going when one of them fails
+ * (#2889).
+ *
+ * The loop used to let the first failure propagate, which made a single
+ * untokenizable row far more expensive than it looks: `dropFTSIndex` has
+ * already run for the failing table, so that table ends with NO index, and
+ * every table after it in {@link FTS_INDEXES} order is never reached — on a
+ * fresh build, or on the incremental path where `dropSearchFTSIndexes` cleared
+ * them all up front, those tables end with no index either. One bad `Method`
+ * row therefore cost keyword search on Namespace, Property, Record, Union,
+ * Static and Variable as well, and `verifySearchFTSIndexes` never ran to say
+ * so. The blast radius was an artifact of loop control flow, not of the data.
+ *
+ * Isolating per index bounds the damage to the table that actually holds the
+ * bad row, and makes `--repair-fts` able to recover everything else. Failures
+ * are returned rather than thrown so the caller can decide — degrade or abort —
+ * with every failure in hand instead of only the first.
+ */
+export declare function createSearchFTSIndexes(options?: CreateSearchFTSIndexesOptions): Promise<FtsIndexBuildFailure[]>;
+/**
+ * One sentence naming every table that failed and why, e.g. `FTS index build
+ * failed for 2 of 21 tables: Method.method_fts (Runtime exception: …), …`.
+ *
+ * Lives here rather than at the call sites because only this module knows the
+ * denominator. Both the analyze degrade path and `--repair-fts` render it, so
+ * one failure reads the same way whichever command produced it.
+ *
+ * Embeds the raw LadybugDB message UNREDACTED — CLI and log surfaces only.
+ * Anything heading for a network response has to pass it through
+ * {@link redactPaths} first, the same rule the query-side warnings follow.
+ */
+export declare const summarizeFtsIndexBuildFailures: (failures: readonly FtsIndexBuildFailure[]) => string;
 export declare function verifySearchFTSIndexes(executeQuery: (cypher: string) => Promise<unknown[]>): Promise<string[]>;
 /**
  * Why an FTS build failed, so the caller can react correctly (#2658):

package/dist/core/search/fts-indexes.js

@@ -209,8 +209,28 @@ export async function dropSearchFTSIndexes(indexRows) {
         await dropFTSIndex(table, indexName);
     }
 }
+/**
+ * Build every configured FTS index, and keep going when one of them fails
+ * (#2889).
+ *
+ * The loop used to let the first failure propagate, which made a single
+ * untokenizable row far more expensive than it looks: `dropFTSIndex` has
+ * already run for the failing table, so that table ends with NO index, and
+ * every table after it in {@link FTS_INDEXES} order is never reached — on a
+ * fresh build, or on the incremental path where `dropSearchFTSIndexes` cleared
+ * them all up front, those tables end with no index either. One bad `Method`
+ * row therefore cost keyword search on Namespace, Property, Record, Union,
+ * Static and Variable as well, and `verifySearchFTSIndexes` never ran to say
+ * so. The blast radius was an artifact of loop control flow, not of the data.
+ *
+ * Isolating per index bounds the damage to the table that actually holds the
+ * bad row, and makes `--repair-fts` able to recover everything else. Failures
+ * are returned rather than thrown so the caller can decide — degrade or abort —
+ * with every failure in hand instead of only the first.
+ */
 export async function createSearchFTSIndexes(options) {
     const stemmer = getSearchFTSStemmer();
+    const failures = [];
     for (const { table, indexName, properties } of FTS_INDEXES) {
         options?.onIndexStart?.(table, indexName);
         // Drop first so the live `properties` always win. `createFTSIndex` is
@@ -224,11 +244,34 @@ export async function createSearchFTSIndexes(options) {
         // skipping when present; FTS build is proportional to symbol-table size and
         // runs inside the existing FTS phase. Gate on a stored schema fingerprint if
         // this rebuild cost ever shows up in analyze profiles.
-        await dropFTSIndex(table, indexName);
-        await createFTSIndex(table, indexName, [...properties], stemmer);
+        try {
+            await dropFTSIndex(table, indexName);
+            await createFTSIndex(table, indexName, [...properties], stemmer);
+        }
+        catch (e) {
+            // The message, never the Error: holding the Error would pin its stack and
+            // whatever the native binding attached to it, for up to one per table.
+            failures.push({ table, indexName, error: e instanceof Error ? e.message : String(e) });
+            continue;
+        }
         options?.onIndexReady?.(table, indexName);
     }
+    return failures;
 }
+/**
+ * One sentence naming every table that failed and why, e.g. `FTS index build
+ * failed for 2 of 21 tables: Method.method_fts (Runtime exception: …), …`.
+ *
+ * Lives here rather than at the call sites because only this module knows the
+ * denominator. Both the analyze degrade path and `--repair-fts` render it, so
+ * one failure reads the same way whichever command produced it.
+ *
+ * Embeds the raw LadybugDB message UNREDACTED — CLI and log surfaces only.
+ * Anything heading for a network response has to pass it through
+ * {@link redactPaths} first, the same rule the query-side warnings follow.
+ */
+export const summarizeFtsIndexBuildFailures = (failures) => `FTS index build failed for ${failures.length} of ${FTS_INDEXES.length} tables: ` +
+    failures.map((f) => `${f.table}.${f.indexName} (${f.error})`).join(', ');
 export async function verifySearchFTSIndexes(executeQuery) {
     // Read the catalog once and check each configured index both EXISTS and
     // covers its expected columns. A queryability-only probe (CALL QUERY_FTS_INDEX
@@ -328,16 +371,35 @@ export const ftsFailureIsFatal = (failureClass, useAtomicSwap) => failureClass =
  */
 export async function buildSearchIndexesOrDegrade(executeQuery, options) {
     try {
-        await createSearchFTSIndexes(options);
+        // Verify ALWAYS runs, failures or not (#2889). Reporting must not jump the
+        // queue ahead of verification: a partial build is exactly when "the other
+        // tables are fine" needs proving rather than asserting, and a stale
+        // name+content-only index is invisible to the build (it succeeds) yet still
+        // means description search is broken (#2299).
+        const failures = await createSearchFTSIndexes(options);
         const missing = await verifySearchFTSIndexes(executeQuery);
-        if (missing.length > 0) {
-            // Structural incompleteness with no thrown error — treat as capability
-            // (degrade), matching prior behavior; a broken *write* surfaces as a
-            // thrown IO/checkpoint error below and is classified integrity there.
-            const error = `missing indexes after build: ${missing.join(', ')}`;
-            return { ok: false, error, failureClass: classifyFtsBuildError(error) };
-        }
-        return { ok: true };
+        if (failures.length === 0 && missing.length === 0)
+            return { ok: true };
+        // A table that failed to build is necessarily missing too — report it once,
+        // with its reason, and keep `missing` for indexes nothing explains.
+        const named = new Set(failures.map((f) => `${f.table}.${f.indexName}`));
+        const unexplained = missing.filter((name) => !named.has(name));
+        const error = [
+            failures.length > 0 ? summarizeFtsIndexBuildFailures(failures) : '',
+            // Structural incompleteness with no thrown error — classified capability
+            // (degrade) below, matching prior behavior; a broken *write* surfaces as
+            // a thrown IO/checkpoint error and is classified integrity there.
+            unexplained.length > 0 ? `missing indexes after build: ${unexplained.join(', ')}` : '',
+        ]
+            .filter((part) => part.length > 0)
+            .join('; ');
+        // Classify per failure, not over the joined text: capability signatures are
+        // checked first, so folding the messages together would let an untokenizable
+        // row mask a genuinely broken write and downgrade an abort into a degrade.
+        const failureClass = failures.some((f) => classifyFtsBuildError(f.error) === 'integrity')
+            ? 'integrity'
+            : classifyFtsBuildError(error);
+        return { ok: false, error, failureClass };
     }
     catch (e) {
         const error = e instanceof Error ? e.message : String(e);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gitnexus",
-  "version": "1.6.10-rc.185",
+  "version": "1.6.10-rc.186",
   "description": "Graph-powered code intelligence for AI agents. Index any codebase, query via MCP or CLI.",
   "author": "Abhigyan Patwari",
   "license": "PolyForm-Noncommercial-1.0.0",