gitnexus 1.6.12-rc.33 → 1.6.12-rc.35
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/analyze-options.d.ts +1 -0
- package/dist/cli/analyze-watch.js +1 -0
- package/dist/cli/analyze.js +8 -1
- package/dist/cli/index.js +1 -0
- package/dist/cli/status.js +4 -0
- package/dist/core/lbug/lbug-adapter.d.ts +7 -2
- package/dist/core/lbug/lbug-adapter.js +24 -9
- package/dist/core/run-analyze.d.ts +5 -1
- package/dist/core/run-analyze.js +70 -30
- package/dist/core/search/bm25-index.d.ts +13 -3
- package/dist/core/search/bm25-index.js +14 -3
- package/dist/core/search/fts-indexes.d.ts +2 -1
- package/dist/core/search/fts-indexes.js +4 -1
- package/dist/core/search/fts-policy.d.ts +18 -0
- package/dist/core/search/fts-policy.js +56 -0
- package/dist/core/search/hybrid-search.d.ts +7 -4
- package/dist/core/search/hybrid-search.js +9 -6
- package/dist/mcp/local/local-backend.js +8 -4
- package/dist/server/api.js +34 -15
- package/dist/storage/repo-meta.d.ts +5 -4
- package/package.json +1 -1
- package/scripts/cross-platform-tests.ts +1 -0
|
@@ -23,6 +23,7 @@ export interface AnalyzeOptions {
|
|
|
23
23
|
/** Commander negated flag: false only when --no-parse-cache is passed. */
|
|
24
24
|
parseCache?: boolean;
|
|
25
25
|
repairFts?: boolean;
|
|
26
|
+
skipFts?: boolean;
|
|
26
27
|
/**
|
|
27
28
|
* Embedding generation toggle. Commander parses `--embeddings [limit]` as:
|
|
28
29
|
* - `undefined` when the flag is omitted
|
|
@@ -127,6 +127,7 @@ export async function resolveWatchOptions(repoPath, cli, baseline, reportIgnored
|
|
|
127
127
|
setEnvironment('GITNEXUS_VERBOSE', merged.verbose ? '1' : baseline.verbose);
|
|
128
128
|
return {
|
|
129
129
|
pdg: merged.pdg,
|
|
130
|
+
skipFts: merged.skipFts,
|
|
130
131
|
branch,
|
|
131
132
|
registryName: merged.name,
|
|
132
133
|
allowDuplicateName: merged.allowDuplicateName,
|
package/dist/cli/analyze.js
CHANGED
|
@@ -12,6 +12,7 @@ import os from 'os';
|
|
|
12
12
|
import { spawn } from 'child_process';
|
|
13
13
|
import v8 from 'v8';
|
|
14
14
|
import cliProgress from 'cli-progress';
|
|
15
|
+
import { FTS_DISABLED_MESSAGE, isExplicitFtsDisablement } from '../core/search/fts-policy.js';
|
|
15
16
|
import { isLbugReady, LbugWipeError } from '../core/lbug/lbug-adapter.js';
|
|
16
17
|
import { boundedCheckpointBeforeExit } from '../core/lbug/shutdown-helpers.js';
|
|
17
18
|
import { findUndeclaredRelationPairError } from '../core/lbug/rel-pair-routing.js';
|
|
@@ -1089,6 +1090,7 @@ const analyzeCommandImpl = async (inputPath, cliOptions, runnerIdentityAtBootstr
|
|
|
1089
1090
|
force: options.force || options.skills || options.parseCache === false,
|
|
1090
1091
|
useParseCache: options.parseCache !== false,
|
|
1091
1092
|
repairFts: options.repairFts,
|
|
1093
|
+
skipFts: options.skipFts,
|
|
1092
1094
|
embeddings: embeddingsEnabled,
|
|
1093
1095
|
embeddingsNodeLimit,
|
|
1094
1096
|
dropEmbeddings: options.dropEmbeddings,
|
|
@@ -1187,6 +1189,8 @@ const analyzeCommandImpl = async (inputPath, cliOptions, runnerIdentityAtBootstr
|
|
|
1187
1189
|
console.error = origError;
|
|
1188
1190
|
bar.stop();
|
|
1189
1191
|
console.log(' Already up to date\n');
|
|
1192
|
+
if (isExplicitFtsDisablement(result.ftsSkipReason))
|
|
1193
|
+
console.log(` ${FTS_DISABLED_MESSAGE}\n`);
|
|
1190
1194
|
if (runOptions.registryName) {
|
|
1191
1195
|
console.log(` Registry name: ${result.repoName}\n`);
|
|
1192
1196
|
}
|
|
@@ -1319,7 +1323,10 @@ const analyzeCommandImpl = async (inputPath, cliOptions, runnerIdentityAtBootstr
|
|
|
1319
1323
|
if (result.ftsSkipped) {
|
|
1320
1324
|
// #2658 review L2: a build/verify failure is NOT an extension-unavailable
|
|
1321
1325
|
// problem — sending the user to install the extension is the wrong remedy.
|
|
1322
|
-
if (result.ftsSkipReason
|
|
1326
|
+
if (isExplicitFtsDisablement(result.ftsSkipReason)) {
|
|
1327
|
+
console.log(`\n ${FTS_DISABLED_MESSAGE}`);
|
|
1328
|
+
}
|
|
1329
|
+
else if (result.ftsSkipReason === 'build-failed') {
|
|
1323
1330
|
console.log(`\n Warning: full-text/BM25 search is disabled — the search index build failed this run.\n` +
|
|
1324
1331
|
` The FTS extension is available; rerun \`gitnexus analyze --repair-fts\`. If it persists,\n` +
|
|
1325
1332
|
` check the disk for space or corruption. Run \`gitnexus doctor\` for details.`);
|
package/dist/cli/index.js
CHANGED
|
@@ -51,6 +51,7 @@ program
|
|
|
51
51
|
.option('-f, --force', 'Force graph and FTS rebuild; unchanged parser output may be reused')
|
|
52
52
|
.option('--no-parse-cache', 'Re-parse every source file instead of replaying cached parser output')
|
|
53
53
|
.option('--repair-fts', 'Repair/rebuild search FTS indexes without full re-analysis')
|
|
54
|
+
.option('--skip-fts', 'Skip FTS extension loading and keyword search indexes')
|
|
54
55
|
.option('--embeddings [limit]', 'Enable embedding generation for semantic search (off by default). ' +
|
|
55
56
|
'Optional [limit] overrides the 50,000-node safety cap; pass 0 to disable the cap entirely.')
|
|
56
57
|
.option('--drop-embeddings', 'Drop existing embeddings on rebuild. By default, an `analyze` without `--embeddings` ' +
|
package/dist/cli/status.js
CHANGED
|
@@ -8,6 +8,7 @@ import { findRepo, getStoragePaths, loadMeta, hasKuzuIndex } from '../storage/re
|
|
|
8
8
|
import { getCurrentCommit, getCurrentBranch, isGitRepo, getGitRoot, isWorkingTreeDirty, } from '../storage/git.js';
|
|
9
9
|
import { analyzerRunnerIdentitiesEqual, resolveAnalyzerRunnerIdentity, } from '../core/analyzer-identity.js';
|
|
10
10
|
import { getIndexIncompleteReasons } from '../core/index-freshness.js';
|
|
11
|
+
import { getFtsDisabledReason, FTS_DISABLED_MESSAGE } from '../core/search/fts-policy.js';
|
|
11
12
|
import { detectIndexContentDrift } from '../core/index-content-drift.js';
|
|
12
13
|
import { t } from './i18n/index.js';
|
|
13
14
|
/** How many drifted paths the report names before summarizing the rest. */
|
|
@@ -149,6 +150,7 @@ export const statusCommand = async (options = {}) => {
|
|
|
149
150
|
index: {
|
|
150
151
|
indexedAt: activeMeta.indexedAt,
|
|
151
152
|
commit: activeMeta.lastCommit,
|
|
153
|
+
...(activeMeta.capabilities ? { capabilities: activeMeta.capabilities } : {}),
|
|
152
154
|
runnerIdentity: activeMeta.runnerIdentity ?? null,
|
|
153
155
|
runnerIdentityStatus: runnerIdentityIsCurrent ? 'current' : 'stale-or-unknown',
|
|
154
156
|
incompleteReasons,
|
|
@@ -170,6 +172,8 @@ export const statusCommand = async (options = {}) => {
|
|
|
170
172
|
console.log(`${t('status.indexed')}: ${new Date(activeMeta.indexedAt).toLocaleString()}`);
|
|
171
173
|
console.log(`${t('status.indexedCommit')}: ${activeMeta.lastCommit?.slice(0, 7)}`);
|
|
172
174
|
console.log(`${t('status.currentCommit')}: ${currentCommit?.slice(0, 7)}`);
|
|
175
|
+
if (getFtsDisabledReason(activeMeta.capabilities?.fts))
|
|
176
|
+
console.log(FTS_DISABLED_MESSAGE);
|
|
173
177
|
// Emit the complete, versioned receipt as JSON so humans can inspect it and
|
|
174
178
|
// automation can compare it without reverse-engineering a display string.
|
|
175
179
|
// `null` is the backward-compatible signal for pre-receipt metadata.
|
|
@@ -40,7 +40,9 @@ export declare const isReadOnlyDbError: (err: unknown) => boolean;
|
|
|
40
40
|
export declare const acquireInitLock: (dbPath: string) => Promise<() => Promise<void>>;
|
|
41
41
|
/** Exported for testing — returns the lock file path for a given dbPath. */
|
|
42
42
|
export declare const _initLockPathForTest: (dbPath: string) => string;
|
|
43
|
-
export declare const initLbug: (dbPath: string
|
|
43
|
+
export declare const initLbug: (dbPath: string, options?: {
|
|
44
|
+
skipFts?: boolean;
|
|
45
|
+
}) => Promise<{
|
|
44
46
|
db: lbug.Database;
|
|
45
47
|
conn: lbug.Connection;
|
|
46
48
|
}>;
|
|
@@ -54,6 +56,7 @@ export declare const initLbug: (dbPath: string) => Promise<{
|
|
|
54
56
|
*/
|
|
55
57
|
export declare const withLbugDb: <T>(dbPath: string, operation: () => Promise<T>, options?: {
|
|
56
58
|
readOnly?: boolean;
|
|
59
|
+
skipFts?: boolean;
|
|
57
60
|
}) => Promise<T>;
|
|
58
61
|
export type LbugProgressCallback = (message: string) => void;
|
|
59
62
|
/**
|
|
@@ -839,7 +842,9 @@ export declare const ensureEmbeddingRowDmlSafe: (indexRows?: IndexCatalogSnapsho
|
|
|
839
842
|
* read once per run. {@link INDEX_CATALOG_UNREADABLE} fails closed here without
|
|
840
843
|
* a second read; omitting the argument makes the gate read for itself.
|
|
841
844
|
*/
|
|
842
|
-
export declare const ensureFtsRowDmlSafe: (indexRows?: IndexCatalogSnapshot
|
|
845
|
+
export declare const ensureFtsRowDmlSafe: (indexRows?: IndexCatalogSnapshot, options?: {
|
|
846
|
+
skipFts?: boolean;
|
|
847
|
+
}) => Promise<boolean>;
|
|
843
848
|
/**
|
|
844
849
|
* Lazy-create an FTS index, caching the fact in-process.
|
|
845
850
|
*
|
|
@@ -576,8 +576,8 @@ const runSchemaCreationQueries = async (dbPath) => {
|
|
|
576
576
|
}
|
|
577
577
|
return null;
|
|
578
578
|
};
|
|
579
|
-
export const initLbug = async (dbPath) => {
|
|
580
|
-
return runWithSessionLock(() => ensureLbugInitialized(dbPath));
|
|
579
|
+
export const initLbug = async (dbPath, options = {}) => {
|
|
580
|
+
return runWithSessionLock(() => ensureLbugInitialized(dbPath, options));
|
|
581
581
|
};
|
|
582
582
|
/**
|
|
583
583
|
* Execute multiple queries against one repo DB atomically.
|
|
@@ -593,7 +593,7 @@ export const withLbugDb = async (dbPath, operation, options = {}) => {
|
|
|
593
593
|
for (let attempt = 1; attempt <= DB_LOCK_RETRY_ATTEMPTS; attempt++) {
|
|
594
594
|
try {
|
|
595
595
|
return await runWithSessionLock(async () => {
|
|
596
|
-
await ensureLbugInitialized(dbPath, readOnly);
|
|
596
|
+
await ensureLbugInitialized(dbPath, { readOnly, skipFts: options.skipFts });
|
|
597
597
|
return operation();
|
|
598
598
|
});
|
|
599
599
|
}
|
|
@@ -624,14 +624,22 @@ export const withLbugDb = async (dbPath, operation, options = {}) => {
|
|
|
624
624
|
// but TypeScript needs an explicit throw to satisfy the return type.
|
|
625
625
|
throw lastError;
|
|
626
626
|
};
|
|
627
|
-
|
|
628
|
-
|
|
627
|
+
let currentDbSkipFts = false;
|
|
628
|
+
const ensureLbugInitialized = async (dbPath, options = {}) => {
|
|
629
|
+
const readOnly = options.readOnly === true;
|
|
630
|
+
const skipFts = options.skipFts === true;
|
|
631
|
+
if (conn &&
|
|
632
|
+
currentDbPath === dbPath &&
|
|
633
|
+
currentDbReadOnly === readOnly &&
|
|
634
|
+
currentDbSkipFts === skipFts) {
|
|
629
635
|
return { db, conn };
|
|
630
636
|
}
|
|
631
|
-
await doInitLbug(dbPath, readOnly);
|
|
637
|
+
await doInitLbug(dbPath, { readOnly, skipFts });
|
|
632
638
|
return { db, conn };
|
|
633
639
|
};
|
|
634
|
-
const doInitLbug = async (dbPath,
|
|
640
|
+
const doInitLbug = async (dbPath, options = {}) => {
|
|
641
|
+
const readOnly = options.readOnly === true;
|
|
642
|
+
const skipFts = options.skipFts === true;
|
|
635
643
|
// Different database requested — close the old one first
|
|
636
644
|
if (conn || db) {
|
|
637
645
|
await safeClose();
|
|
@@ -790,7 +798,10 @@ const doInitLbug = async (dbPath, readOnly = false) => {
|
|
|
790
798
|
// Phase 3 installs it moments later in the same run. Warning here reported a
|
|
791
799
|
// degradation that never happened — the run went on to build every FTS index.
|
|
792
800
|
// Phase 3 (and the read-only branch) still warn for real failures.
|
|
793
|
-
|
|
801
|
+
if (!skipFts) {
|
|
802
|
+
await loadFTSExtension(undefined, readOnly ? { policy: 'load-only' } : { quiet: true });
|
|
803
|
+
}
|
|
804
|
+
currentDbSkipFts = skipFts;
|
|
794
805
|
currentDbPath = dbPath;
|
|
795
806
|
return { db, conn };
|
|
796
807
|
};
|
|
@@ -3204,7 +3215,7 @@ export const ensureEmbeddingRowDmlSafe = async (indexRows) => {
|
|
|
3204
3215
|
* read once per run. {@link INDEX_CATALOG_UNREADABLE} fails closed here without
|
|
3205
3216
|
* a second read; omitting the argument makes the gate read for itself.
|
|
3206
3217
|
*/
|
|
3207
|
-
export const ensureFtsRowDmlSafe = async (indexRows) => {
|
|
3218
|
+
export const ensureFtsRowDmlSafe = async (indexRows, options = {}) => {
|
|
3208
3219
|
// Unconditional precondition, same regression as the VECTOR twin's (#2841
|
|
3209
3220
|
// review §5.B): a caller-supplied snapshot must not let a closed DB be
|
|
3210
3221
|
// answered `true`.
|
|
@@ -3231,6 +3242,10 @@ export const ensureFtsRowDmlSafe = async (indexRows) => {
|
|
|
3231
3242
|
});
|
|
3232
3243
|
if (!indexGatesDml)
|
|
3233
3244
|
return true;
|
|
3245
|
+
// Existing/unknown native indexes still gate writes. Rebuild into a fresh
|
|
3246
|
+
// database rather than loading FTS or issuing unsafe DML when opted out.
|
|
3247
|
+
if (options.skipFts)
|
|
3248
|
+
return false;
|
|
3234
3249
|
return await loadFTSExtension(undefined, { policy: resolveAnalyzeInstallPolicy() });
|
|
3235
3250
|
};
|
|
3236
3251
|
/**
|
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
* wrapper or server worker) is responsible for process lifecycle.
|
|
10
10
|
*/
|
|
11
11
|
import { type GraphWriteCollapseVerdict } from './index-freshness.js';
|
|
12
|
+
import { type FtsSkipReason } from './search/fts-policy.js';
|
|
12
13
|
import type { KnowledgeGraph } from './graph/types.js';
|
|
13
14
|
import { type AnalyzerRunnerIdentity, type RepoMeta } from '../storage/repo-manager.js';
|
|
14
15
|
export interface AnalyzeCallbacks {
|
|
@@ -32,6 +33,7 @@ export interface AnalyzeOptions {
|
|
|
32
33
|
useParseCache?: boolean;
|
|
33
34
|
/** Repair only search indexes without re-running full parsing/indexing. */
|
|
34
35
|
repairFts?: boolean;
|
|
36
|
+
skipFts?: boolean;
|
|
35
37
|
/** Emit per-index FTS create logs. */
|
|
36
38
|
verbose?: boolean;
|
|
37
39
|
embeddings?: boolean;
|
|
@@ -226,8 +228,10 @@ export interface AnalyzeResult {
|
|
|
226
228
|
* extension loaded but the index build/verify failed non-fatally — remedied by
|
|
227
229
|
* `--repair-fts`, not by installing the extension). Lets the CLI show the
|
|
228
230
|
* correct recovery hint instead of always blaming a missing extension.
|
|
231
|
+
* `disabled-by-flag` and `disabled-by-env` record intentional opt-out;
|
|
232
|
+
* neither calls for extension installation or repair.
|
|
229
233
|
*/
|
|
230
|
-
ftsSkipReason?:
|
|
234
|
+
ftsSkipReason?: FtsSkipReason;
|
|
231
235
|
/**
|
|
232
236
|
* True when the index this run produced/validated is the flat workspace
|
|
233
237
|
* slot (#2106 R2, inverted by #2354 to follow the checked-out branch).
|
package/dist/core/run-analyze.js
CHANGED
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
* wrapper or server worker) is responsible for process lifecycle.
|
|
10
10
|
*/
|
|
11
11
|
import { detectGraphWriteCollapse } from './index-freshness.js';
|
|
12
|
+
import { resolveFtsDisableReason, getFtsDisabledReason, withExplicitFtsDisablement, FTS_DISABLED_MESSAGE, DEFAULT_GRAPH_CAPABILITY, DEFAULT_VECTOR_SEARCH_CAPABILITY, } from './search/fts-policy.js';
|
|
12
13
|
import { PDG_EDGE_TYPES } from './lbug/pdg-emit-sink.js';
|
|
13
14
|
import path from 'path';
|
|
14
15
|
import fs from 'fs/promises';
|
|
@@ -156,6 +157,11 @@ function recordLiveIndexMutationRisk(error) {
|
|
|
156
157
|
liveIndexMutationRisks.add(error);
|
|
157
158
|
}
|
|
158
159
|
}
|
|
160
|
+
function formatMetaWriteFailureReason(err) {
|
|
161
|
+
return isReadOnlyFilesystemError(err)
|
|
162
|
+
? `${err.message} — storage may be read-only (#1549)`
|
|
163
|
+
: err.message;
|
|
164
|
+
}
|
|
159
165
|
/** Whether a failed analyze may already have changed the live DB. */
|
|
160
166
|
export function analyzeFailureMayHaveMutatedLiveIndex(error) {
|
|
161
167
|
return (((typeof error === 'object' && error !== null) || typeof error === 'function') &&
|
|
@@ -584,6 +590,9 @@ export async function runFullAnalysis(repoPath, options, callbacks, runnerIdenti
|
|
|
584
590
|
// Validate operator-provided FTS config before anything else — a typo fails
|
|
585
591
|
// here in ms, without taking the lock. (createSearchFTSIndexes reuses the
|
|
586
592
|
// cached value via getSearchFTSStemmer.)
|
|
593
|
+
if (options.repairFts && resolveFtsDisableReason(options.skipFts)) {
|
|
594
|
+
throw new Error('--repair-fts cannot be used with --skip-fts or GITNEXUS_SKIP_FTS=1.');
|
|
595
|
+
}
|
|
587
596
|
initialiseSearchFTSStemmer();
|
|
588
597
|
initialiseSearchFTSCjkSegmentation();
|
|
589
598
|
// Scope the degraded-parse log throttle to this run (module-level counter
|
|
@@ -635,6 +644,8 @@ export async function runFullAnalysis(repoPath, options, callbacks, runnerIdenti
|
|
|
635
644
|
}
|
|
636
645
|
}
|
|
637
646
|
async function runFullAnalysisInner(repoPath, options, callbacks, writeTarget, runnerIdentityAtBootstrap) {
|
|
647
|
+
const ftsDisabledReason = resolveFtsDisableReason(options.skipFts);
|
|
648
|
+
const initAnalysisLbug = (dbPath) => ftsDisabledReason ? initLbug(dbPath, { skipFts: true }) : initLbug(dbPath);
|
|
638
649
|
const log = (msg) => callbacks.onLog?.(stripControlCharacters(msg));
|
|
639
650
|
const progress = (phase, percent, message) => callbacks.onProgress(phase, percent, message);
|
|
640
651
|
// FTS-config validation and the degraded-parse counter reset happen in the
|
|
@@ -668,7 +679,19 @@ async function runFullAnalysisInner(repoPath, options, callbacks, writeTarget, r
|
|
|
668
679
|
const code = err?.code;
|
|
669
680
|
log(`Metadata reconciliation failed (non-critical${code ? `, ${code}` : ''}); continuing.`);
|
|
670
681
|
}
|
|
671
|
-
const
|
|
682
|
+
const loadedMeta = await loadMeta(metaDir);
|
|
683
|
+
const previousFtsDisabledReason = getFtsDisabledReason(loadedMeta?.capabilities?.fts);
|
|
684
|
+
// Flag and env are equivalent disablements. Only a true enable↔disable flip
|
|
685
|
+
// needs a write plan; a discriminator-only change restamps on the
|
|
686
|
+
// already-up-to-date path.
|
|
687
|
+
const ftsModeChanged = Boolean(ftsDisabledReason) !== Boolean(previousFtsDisabledReason);
|
|
688
|
+
// Fold explicit disablement into the in-memory prior meta so every later
|
|
689
|
+
// saveMeta that spreads it (dirty flag, incremental phase stamps) advertises
|
|
690
|
+
// "FTS disabled" instead of leftover available/build-failed while a wipe is
|
|
691
|
+
// in flight. Re-enable leaves the prior stamp untouched.
|
|
692
|
+
const existingMeta = loadedMeta
|
|
693
|
+
? withExplicitFtsDisablement(loadedMeta, ftsDisabledReason)
|
|
694
|
+
: undefined;
|
|
672
695
|
// ── FTS-only repair path ────────────────────────────────────────────
|
|
673
696
|
if (options.repairFts) {
|
|
674
697
|
if (!existingMeta) {
|
|
@@ -713,7 +736,7 @@ async function runFullAnalysisInner(repoPath, options, callbacks, writeTarget, r
|
|
|
713
736
|
'Run `gitnexus analyze` (full) to rebuild from scratch.');
|
|
714
737
|
}
|
|
715
738
|
try {
|
|
716
|
-
await
|
|
739
|
+
await initAnalysisLbug(lbugPath);
|
|
717
740
|
// Gate on FTS availability BEFORE touching any index. createSearchFTSIndexes
|
|
718
741
|
// now DROPs each index before recreating it (so schema changes reach existing
|
|
719
742
|
// DBs); if the extension were unavailable, the drops would run and leave the
|
|
@@ -793,16 +816,9 @@ async function runFullAnalysisInner(repoPath, options, callbacks, writeTarget, r
|
|
|
793
816
|
await saveMeta(metaDir, {
|
|
794
817
|
...latestMeta,
|
|
795
818
|
capabilities: {
|
|
796
|
-
graph: latestMeta.capabilities?.graph ??
|
|
797
|
-
provider: 'ladybugdb',
|
|
798
|
-
status: 'available',
|
|
799
|
-
},
|
|
819
|
+
graph: latestMeta.capabilities?.graph ?? DEFAULT_GRAPH_CAPABILITY,
|
|
800
820
|
fts: { provider: 'ladybugdb-fts', status: 'available' },
|
|
801
|
-
vectorSearch: latestMeta.capabilities?.vectorSearch ??
|
|
802
|
-
provider: 'exact-scan',
|
|
803
|
-
status: 'unavailable',
|
|
804
|
-
exactScanLimit: 0,
|
|
805
|
-
},
|
|
821
|
+
vectorSearch: latestMeta.capabilities?.vectorSearch ?? DEFAULT_VECTOR_SEARCH_CAPABILITY,
|
|
806
822
|
},
|
|
807
823
|
});
|
|
808
824
|
}
|
|
@@ -1192,7 +1208,8 @@ async function runFullAnalysisInner(repoPath, options, callbacks, writeTarget, r
|
|
|
1192
1208
|
if (existingMeta &&
|
|
1193
1209
|
!existingMeta.embeddingCheckpoint &&
|
|
1194
1210
|
!options.force &&
|
|
1195
|
-
existingMeta.lastCommit === currentCommit
|
|
1211
|
+
existingMeta.lastCommit === currentCommit &&
|
|
1212
|
+
!ftsModeChanged) {
|
|
1196
1213
|
// Non-git folders have currentCommit = '' — always rebuild since we can't detect changes
|
|
1197
1214
|
if (currentCommit !== '') {
|
|
1198
1215
|
// For git repos, even if HEAD matches lastCommit, the working tree
|
|
@@ -1287,10 +1304,17 @@ async function runFullAnalysisInner(repoPath, options, callbacks, writeTarget, r
|
|
|
1287
1304
|
// EACCES/EPERM also arise from ownership problems and transient
|
|
1288
1305
|
// Windows locks, so keep the real error visible alongside the
|
|
1289
1306
|
// #1549 read-only hint instead of replacing it.
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1307
|
+
log(`Warning: could not restamp the workspace branch label (${formatMetaWriteFailureReason(err)}); will retry on the next run.`);
|
|
1308
|
+
}
|
|
1309
|
+
}
|
|
1310
|
+
else if (ftsDisabledReason && ftsDisabledReason !== previousFtsDisabledReason) {
|
|
1311
|
+
// Discriminator-only restamp (flag↔env). `existingMeta` already
|
|
1312
|
+
// carries the folded skipReason; persist it without a write plan.
|
|
1313
|
+
try {
|
|
1314
|
+
await saveMeta(metaDir, existingMeta);
|
|
1315
|
+
}
|
|
1316
|
+
catch (err) {
|
|
1317
|
+
log(`Warning: could not restamp the FTS skip reason (${formatMetaWriteFailureReason(err)}); will retry on the next run.`);
|
|
1294
1318
|
}
|
|
1295
1319
|
}
|
|
1296
1320
|
await ensureGitNexusIgnored(repoPath);
|
|
@@ -1304,6 +1328,7 @@ async function runFullAnalysisInner(repoPath, options, callbacks, writeTarget, r
|
|
|
1304
1328
|
repoPath,
|
|
1305
1329
|
stats: existingMeta.stats ?? {},
|
|
1306
1330
|
alreadyUpToDate: true,
|
|
1331
|
+
...(ftsDisabledReason ? { ftsSkipped: true, ftsSkipReason: ftsDisabledReason } : {}),
|
|
1307
1332
|
isPrimaryBranch: !placement.branch,
|
|
1308
1333
|
};
|
|
1309
1334
|
}
|
|
@@ -1361,7 +1386,7 @@ async function runFullAnalysisInner(repoPath, options, callbacks, writeTarget, r
|
|
|
1361
1386
|
if (shouldLoadCache && existingMeta) {
|
|
1362
1387
|
try {
|
|
1363
1388
|
progress('embeddings', 0, 'Caching embeddings...');
|
|
1364
|
-
await
|
|
1389
|
+
await initAnalysisLbug(lbugPath);
|
|
1365
1390
|
const cached = await loadCachedEmbeddings();
|
|
1366
1391
|
cachedEmbeddingNodeIds = cached.embeddingNodeIds;
|
|
1367
1392
|
cachedEmbeddings = cached.embeddings;
|
|
@@ -1707,7 +1732,7 @@ async function runFullAnalysisInner(repoPath, options, callbacks, writeTarget, r
|
|
|
1707
1732
|
// Full rebuild (POSIX) builds into the temp `buildPath`; incremental and
|
|
1708
1733
|
// Windows use `buildPath === lbugPath` in place.
|
|
1709
1734
|
try {
|
|
1710
|
-
await
|
|
1735
|
+
await initAnalysisLbug(buildPath);
|
|
1711
1736
|
}
|
|
1712
1737
|
catch (error) {
|
|
1713
1738
|
if (liveIndexMutationStarted)
|
|
@@ -1990,7 +2015,9 @@ async function runFullAnalysisInner(repoPath, options, callbacks, writeTarget, r
|
|
|
1990
2015
|
// creates or drops an index.
|
|
1991
2016
|
const indexCatalogRows = await readIndexCatalogSnapshot();
|
|
1992
2017
|
const embeddingRowDmlSafe = await ensureEmbeddingRowDmlSafe(indexCatalogRows);
|
|
1993
|
-
const ftsRowDmlSafe = await ensureFtsRowDmlSafe(indexCatalogRows
|
|
2018
|
+
const ftsRowDmlSafe = await ensureFtsRowDmlSafe(indexCatalogRows, {
|
|
2019
|
+
skipFts: Boolean(ftsDisabledReason),
|
|
2020
|
+
});
|
|
1994
2021
|
const extensionForcedRebuild = !embeddingRowDmlSafe || !ftsRowDmlSafe;
|
|
1995
2022
|
// `!options.dropEmbeddings` (H1): this rescue reads the rows back OUT of
|
|
1996
2023
|
// the DB, so it must never fire on the one path whose entire purpose is to
|
|
@@ -2074,7 +2101,7 @@ async function runFullAnalysisInner(repoPath, options, callbacks, writeTarget, r
|
|
|
2074
2101
|
// catalog read happened to fail still had both gates answer "safe"
|
|
2075
2102
|
// (both extensions loaded), and claiming otherwise would trade one
|
|
2076
2103
|
// invented cause for another.
|
|
2077
|
-
if (extensionForcedRebuild && indexCatalogUnreadable) {
|
|
2104
|
+
if (extensionForcedRebuild && indexCatalogUnreadable && !ftsDisabledReason) {
|
|
2078
2105
|
const blockedExtensions = [
|
|
2079
2106
|
!embeddingRowDmlSafe ? 'VECTOR' : undefined,
|
|
2080
2107
|
!ftsRowDmlSafe ? 'FTS' : undefined,
|
|
@@ -2092,7 +2119,11 @@ async function runFullAnalysisInner(repoPath, options, callbacks, writeTarget, r
|
|
|
2092
2119
|
}
|
|
2093
2120
|
degradedEffects.push('Semantic search falls back to exact scan until VECTOR is available.');
|
|
2094
2121
|
}
|
|
2095
|
-
if (!ftsRowDmlSafe) {
|
|
2122
|
+
if (!ftsRowDmlSafe && ftsDisabledReason) {
|
|
2123
|
+
escalationCauses.push('FTS is explicitly disabled and existing search indexes could not be ruled out; ' +
|
|
2124
|
+
'a fresh graph store is required before writing rows without the extension');
|
|
2125
|
+
}
|
|
2126
|
+
else if (!ftsRowDmlSafe) {
|
|
2096
2127
|
if (!indexCatalogUnreadable) {
|
|
2097
2128
|
// Self-contained subject (H5): `join('; and ')` used to render "…the
|
|
2098
2129
|
// CodeEmbedding vector index exists … and THIS INDEX carries FTS
|
|
@@ -2139,7 +2170,9 @@ async function runFullAnalysisInner(repoPath, options, callbacks, writeTarget, r
|
|
|
2139
2170
|
!embeddingRowDmlSafe
|
|
2140
2171
|
? { reason: getExtensionCapability('VECTOR')?.reason, label: 'VECTOR' }
|
|
2141
2172
|
: undefined,
|
|
2142
|
-
!ftsRowDmlSafe
|
|
2173
|
+
!ftsRowDmlSafe && !ftsDisabledReason
|
|
2174
|
+
? { reason: getFtsCapability()?.reason, label: 'FTS' }
|
|
2175
|
+
: undefined,
|
|
2143
2176
|
]
|
|
2144
2177
|
.filter((e) => e !== undefined)
|
|
2145
2178
|
.map(({ reason, label }) => diagnoseExtensionLoad(reason, label).remedy);
|
|
@@ -2219,7 +2252,7 @@ async function runFullAnalysisInner(repoPath, options, callbacks, writeTarget, r
|
|
|
2219
2252
|
if (buildPath === lbugPath)
|
|
2220
2253
|
liveIndexMutationStarted = true;
|
|
2221
2254
|
await wipeLbugDbFiles(buildPath);
|
|
2222
|
-
await
|
|
2255
|
+
await initAnalysisLbug(buildPath);
|
|
2223
2256
|
walCheckpointDriver = startWalCheckpointDriver();
|
|
2224
2257
|
await loadGraphToLbug(pipelineResult.graph, pipelineResult.repoPath, storagePath, (msg) => {
|
|
2225
2258
|
lbugMsgCount++;
|
|
@@ -2429,20 +2462,23 @@ async function runFullAnalysisInner(repoPath, options, callbacks, writeTarget, r
|
|
|
2429
2462
|
// analyze still produces a fully queryable graph; only full-text/BM25
|
|
2430
2463
|
// search falls back. `--repair-fts` (whose sole job is FTS) still fails
|
|
2431
2464
|
// loudly on its own path above.
|
|
2432
|
-
progress('fts', 85, 'Creating search indexes...');
|
|
2433
|
-
const ftsAvailable =
|
|
2434
|
-
|
|
2435
|
-
|
|
2465
|
+
progress('fts', 85, ftsDisabledReason ? 'Skipping search indexes...' : 'Creating search indexes...');
|
|
2466
|
+
const ftsAvailable = !ftsDisabledReason &&
|
|
2467
|
+
(await loadFTSExtension(undefined, {
|
|
2468
|
+
policy: resolveAnalyzeInstallPolicy(),
|
|
2469
|
+
}));
|
|
2436
2470
|
// Tracks whether search indexes actually ended up usable this run — starts
|
|
2437
2471
|
// as ftsAvailable (extension loaded) but flips to false below when the
|
|
2438
2472
|
// build/verify step itself fails, so capabilities.fts.status / ftsSkipped
|
|
2439
2473
|
// stay honest even though that failure no longer aborts the whole analyze.
|
|
2440
2474
|
let ftsReady = ftsAvailable;
|
|
2441
|
-
// Why FTS ended up skipped (#2658 review L2):
|
|
2442
|
-
//
|
|
2475
|
+
// Why FTS ended up skipped (#2658 review L2): an explicit opt-out
|
|
2476
|
+
// (`disabled-by-flag` / `disabled-by-env`, #3091) when one was recorded,
|
|
2477
|
+
// else extension-unavailable up front, or build-failed in the degrade
|
|
2478
|
+
// branch below.
|
|
2443
2479
|
let ftsSkipReason = ftsAvailable
|
|
2444
2480
|
? undefined
|
|
2445
|
-
: 'extension-unavailable';
|
|
2481
|
+
: (ftsDisabledReason ?? 'extension-unavailable');
|
|
2446
2482
|
if (ftsAvailable) {
|
|
2447
2483
|
// Degrade rather than throw: createSearchFTSIndexes re-tokenizes every
|
|
2448
2484
|
// stored row on every run, so a native tokenizer error on a single
|
|
@@ -2482,6 +2518,10 @@ async function runFullAnalysisInner(repoPath, options, callbacks, writeTarget, r
|
|
|
2482
2518
|
progress('fts', 90, 'Search indexes skipped (build failed)');
|
|
2483
2519
|
}
|
|
2484
2520
|
}
|
|
2521
|
+
else if (ftsDisabledReason) {
|
|
2522
|
+
log(FTS_DISABLED_MESSAGE);
|
|
2523
|
+
progress('fts', 90, 'Search indexes skipped (explicitly disabled)');
|
|
2524
|
+
}
|
|
2485
2525
|
else {
|
|
2486
2526
|
// For a missing runtime dependency (#2374) the file is present, so the
|
|
2487
2527
|
// generic "install it with network access" tail in FTS_UNAVAILABLE_MESSAGE
|
|
@@ -2,8 +2,12 @@
|
|
|
2
2
|
* Full-Text Search via LadybugDB FTS
|
|
3
3
|
*
|
|
4
4
|
* Uses LadybugDB's built-in full-text search indexes for keyword-based search.
|
|
5
|
-
*
|
|
5
|
+
* Reads from the database on every query it runs (no cached state to drift).
|
|
6
|
+
* The one exception is an explicit opt-out (#3091): when a caller passes an
|
|
7
|
+
* `FtsDisabledReason`, the search short-circuits to an empty, unavailable
|
|
8
|
+
* response without opening or querying the database at all.
|
|
6
9
|
*/
|
|
10
|
+
import type { FtsDisabledReason } from './fts-policy.js';
|
|
7
11
|
export interface BM25SearchResult {
|
|
8
12
|
filePath: string;
|
|
9
13
|
score: number;
|
|
@@ -27,14 +31,20 @@ export interface FTSSearchResponse {
|
|
|
27
31
|
nonBenignErrors?: string[];
|
|
28
32
|
}
|
|
29
33
|
/**
|
|
30
|
-
* Search using LadybugDB's built-in FTS (
|
|
34
|
+
* Search using LadybugDB's built-in FTS (fresh, reads from disk)
|
|
31
35
|
*
|
|
32
36
|
* Queries multiple node tables (File, Function, Class, Method) in parallel
|
|
33
37
|
* and merges results by filePath, summing scores for the same file.
|
|
34
38
|
*
|
|
39
|
+
* When `disabledReason` is set the index intentionally has no FTS (#3091), so
|
|
40
|
+
* this returns an empty `ftsAvailable: false` response immediately and never
|
|
41
|
+
* touches the database — callers render that as deliberate disablement rather
|
|
42
|
+
* than as a missing index or a failed extension load.
|
|
43
|
+
*
|
|
35
44
|
* @param query - Search query string
|
|
36
45
|
* @param limit - Maximum results
|
|
37
46
|
* @param repoId - If provided, queries will be routed via the MCP connection pool
|
|
47
|
+
* @param disabledReason - Explicit FTS opt-out recorded for this index; short-circuits the search
|
|
38
48
|
* @returns Ranked search results from FTS indexes
|
|
39
49
|
*/
|
|
40
|
-
export declare const searchFTSFromLbug: (query: string, limit?: number, repoId?: string) => Promise<FTSSearchResponse>;
|
|
50
|
+
export declare const searchFTSFromLbug: (query: string, limit?: number, repoId?: string, disabledReason?: FtsDisabledReason) => Promise<FTSSearchResponse>;
|
|
@@ -2,7 +2,10 @@
|
|
|
2
2
|
* Full-Text Search via LadybugDB FTS
|
|
3
3
|
*
|
|
4
4
|
* Uses LadybugDB's built-in full-text search indexes for keyword-based search.
|
|
5
|
-
*
|
|
5
|
+
* Reads from the database on every query it runs (no cached state to drift).
|
|
6
|
+
* The one exception is an explicit opt-out (#3091): when a caller passes an
|
|
7
|
+
* `FtsDisabledReason`, the search short-circuits to an empty, unavailable
|
|
8
|
+
* response without opening or querying the database at all.
|
|
6
9
|
*/
|
|
7
10
|
// tri-review Residual-1: `classifyFtsQueryError` now lives in lbug-adapter.ts
|
|
8
11
|
// (see its doc comment) so `queryFTS`'s own catch can share the SAME
|
|
@@ -46,17 +49,25 @@ async function queryFTSViaExecutor(executor, tableName, indexName, query, limit)
|
|
|
46
49
|
}
|
|
47
50
|
}
|
|
48
51
|
/**
|
|
49
|
-
* Search using LadybugDB's built-in FTS (
|
|
52
|
+
* Search using LadybugDB's built-in FTS (fresh, reads from disk)
|
|
50
53
|
*
|
|
51
54
|
* Queries multiple node tables (File, Function, Class, Method) in parallel
|
|
52
55
|
* and merges results by filePath, summing scores for the same file.
|
|
53
56
|
*
|
|
57
|
+
* When `disabledReason` is set the index intentionally has no FTS (#3091), so
|
|
58
|
+
* this returns an empty `ftsAvailable: false` response immediately and never
|
|
59
|
+
* touches the database — callers render that as deliberate disablement rather
|
|
60
|
+
* than as a missing index or a failed extension load.
|
|
61
|
+
*
|
|
54
62
|
* @param query - Search query string
|
|
55
63
|
* @param limit - Maximum results
|
|
56
64
|
* @param repoId - If provided, queries will be routed via the MCP connection pool
|
|
65
|
+
* @param disabledReason - Explicit FTS opt-out recorded for this index; short-circuits the search
|
|
57
66
|
* @returns Ranked search results from FTS indexes
|
|
58
67
|
*/
|
|
59
|
-
export const searchFTSFromLbug = async (query, limit = 20, repoId) => {
|
|
68
|
+
export const searchFTSFromLbug = async (query, limit = 20, repoId, disabledReason) => {
|
|
69
|
+
if (disabledReason)
|
|
70
|
+
return { results: [], ftsAvailable: false };
|
|
60
71
|
// Applied once, up front, so every downstream branch searches with the
|
|
61
72
|
// same text the index was built from (#2331/#2339) — index-time and
|
|
62
73
|
// query-time text transforms must never diverge, since QUERY_FTS_INDEX
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { type IndexCatalogSnapshot } from '../lbug/lbug-adapter.js';
|
|
2
|
+
import { type FtsDisabledReason } from './fts-policy.js';
|
|
2
3
|
/**
|
|
3
4
|
* Strip filesystem paths from a LadybugDB error before it reaches the HTTP
|
|
4
5
|
* `/api/search` and MCP query surfaces (#2374, PR #2375): the raw LOAD error
|
|
@@ -44,7 +45,7 @@ export interface FtsWarningContext {
|
|
|
44
45
|
* text itself (#2767). Optional and additive: omitting it reproduces today's
|
|
45
46
|
* exact message.
|
|
46
47
|
*/
|
|
47
|
-
export declare const ftsDegradedWarning: (context?: FtsWarningContext) => string;
|
|
48
|
+
export declare const ftsDegradedWarning: (context?: FtsWarningContext, disabledReason?: FtsDisabledReason) => string;
|
|
48
49
|
/**
|
|
49
50
|
* Warning for when the FTS extension is loaded and indexes exist, but every
|
|
50
51
|
* configured table's query failed for a real, non-benign reason (timeout,
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { createFTSIndex, dropFTSIndex, indexRowName, indexRowTable, resolveGateRows, DEFAULT_FTS_STEMMER, } from '../lbug/lbug-adapter.js';
|
|
2
2
|
import { getFtsCapability } from '../lbug/extension-loader.js';
|
|
3
|
+
import { FTS_DISABLED_MESSAGE } from './fts-policy.js';
|
|
3
4
|
import { classifyExtensionLoadError } from '../lbug/extension-load-error.js';
|
|
4
5
|
import { FTS_INDEXES } from './fts-schema.js';
|
|
5
6
|
/**
|
|
@@ -45,8 +46,10 @@ const formatWarningContext = (context) => {
|
|
|
45
46
|
* text itself (#2767). Optional and additive: omitting it reproduces today's
|
|
46
47
|
* exact message.
|
|
47
48
|
*/
|
|
48
|
-
export const ftsDegradedWarning = (context) => {
|
|
49
|
+
export const ftsDegradedWarning = (context, disabledReason) => {
|
|
49
50
|
const suffix = context ? formatWarningContext(context) : '';
|
|
51
|
+
if (disabledReason)
|
|
52
|
+
return FTS_DISABLED_MESSAGE + suffix;
|
|
50
53
|
const fts = getFtsCapability();
|
|
51
54
|
if (fts && !fts.loaded) {
|
|
52
55
|
const reason = fts.reason ? redactPaths(fts.reason).replace(/\.$/, '') : undefined;
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { RepoMeta } from '../../storage/repo-meta.js';
|
|
2
|
+
export type FtsDisabledReason = 'disabled-by-flag' | 'disabled-by-env';
|
|
3
|
+
export type FtsSkipReason = FtsDisabledReason | 'extension-unavailable' | 'build-failed';
|
|
4
|
+
type RepoCapabilities = NonNullable<RepoMeta['capabilities']>;
|
|
5
|
+
export declare const DEFAULT_GRAPH_CAPABILITY: RepoCapabilities['graph'];
|
|
6
|
+
export declare const DEFAULT_VECTOR_SEARCH_CAPABILITY: RepoCapabilities['vectorSearch'];
|
|
7
|
+
export declare function resolveFtsDisableReason(skipFts?: boolean, envValue?: string): FtsDisabledReason | undefined;
|
|
8
|
+
export declare function isExplicitFtsDisablement(reason: string | undefined): reason is FtsDisabledReason;
|
|
9
|
+
export declare function getFtsDisabledReason(capability: RepoCapabilities['fts'] | undefined): FtsDisabledReason | undefined;
|
|
10
|
+
/**
|
|
11
|
+
* Overlay an explicit FTS opt-out onto an existing meta snapshot without
|
|
12
|
+
* touching freshness (`indexedAt` / `lastCommit`) or sibling capabilities.
|
|
13
|
+
* Flag and env are equivalent disablements; only the discriminator changes.
|
|
14
|
+
* Returns `meta` unchanged when `reason` is absent (re-enable) or already stamped.
|
|
15
|
+
*/
|
|
16
|
+
export declare function withExplicitFtsDisablement(meta: RepoMeta, reason: FtsDisabledReason | undefined): RepoMeta;
|
|
17
|
+
export declare const FTS_DISABLED_MESSAGE: string;
|
|
18
|
+
export {};
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
export const DEFAULT_GRAPH_CAPABILITY = {
|
|
2
|
+
provider: 'ladybugdb',
|
|
3
|
+
status: 'available',
|
|
4
|
+
};
|
|
5
|
+
export const DEFAULT_VECTOR_SEARCH_CAPABILITY = {
|
|
6
|
+
provider: 'exact-scan',
|
|
7
|
+
status: 'unavailable',
|
|
8
|
+
exactScanLimit: 0,
|
|
9
|
+
};
|
|
10
|
+
export function resolveFtsDisableReason(skipFts, envValue = process.env.GITNEXUS_SKIP_FTS) {
|
|
11
|
+
if (skipFts === true)
|
|
12
|
+
return 'disabled-by-flag';
|
|
13
|
+
if (envValue === '1')
|
|
14
|
+
return 'disabled-by-env';
|
|
15
|
+
return undefined;
|
|
16
|
+
}
|
|
17
|
+
export function isExplicitFtsDisablement(reason) {
|
|
18
|
+
return reason === 'disabled-by-flag' || reason === 'disabled-by-env';
|
|
19
|
+
}
|
|
20
|
+
export function getFtsDisabledReason(capability) {
|
|
21
|
+
if (capability?.status !== 'unavailable')
|
|
22
|
+
return undefined;
|
|
23
|
+
return isExplicitFtsDisablement(capability.skipReason) ? capability.skipReason : undefined;
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Overlay an explicit FTS opt-out onto an existing meta snapshot without
|
|
27
|
+
* touching freshness (`indexedAt` / `lastCommit`) or sibling capabilities.
|
|
28
|
+
* Flag and env are equivalent disablements; only the discriminator changes.
|
|
29
|
+
* Returns `meta` unchanged when `reason` is absent (re-enable) or already stamped.
|
|
30
|
+
*/
|
|
31
|
+
export function withExplicitFtsDisablement(meta, reason) {
|
|
32
|
+
if (!reason)
|
|
33
|
+
return meta;
|
|
34
|
+
const existing = meta.capabilities;
|
|
35
|
+
const existingFts = existing?.fts;
|
|
36
|
+
if (existingFts?.status === 'unavailable' &&
|
|
37
|
+
existingFts.skipReason === reason &&
|
|
38
|
+
existing?.graph &&
|
|
39
|
+
existing.vectorSearch) {
|
|
40
|
+
return meta;
|
|
41
|
+
}
|
|
42
|
+
return {
|
|
43
|
+
...meta,
|
|
44
|
+
capabilities: {
|
|
45
|
+
graph: existing?.graph ?? DEFAULT_GRAPH_CAPABILITY,
|
|
46
|
+
fts: {
|
|
47
|
+
provider: existingFts?.provider ?? 'ladybugdb-fts',
|
|
48
|
+
status: 'unavailable',
|
|
49
|
+
skipReason: reason,
|
|
50
|
+
},
|
|
51
|
+
vectorSearch: existing?.vectorSearch ?? DEFAULT_VECTOR_SEARCH_CAPABILITY,
|
|
52
|
+
},
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
export const FTS_DISABLED_MESSAGE = 'FTS disabled for this index. To enable keyword search, run gitnexus analyze ' +
|
|
56
|
+
'without --skip-fts and with GITNEXUS_SKIP_FTS unset.';
|
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
*/
|
|
10
10
|
import { type BM25SearchResult } from './bm25-index.js';
|
|
11
11
|
import type { SemanticSearchResult } from '../embeddings/types.js';
|
|
12
|
+
import type { FtsDisabledReason } from './fts-policy.js';
|
|
12
13
|
export interface HybridSearchResult {
|
|
13
14
|
filePath: string;
|
|
14
15
|
score: number;
|
|
@@ -44,10 +45,12 @@ export declare const isHybridSearchReady: () => boolean;
|
|
|
44
45
|
export declare const formatHybridResults: (results: HybridSearchResult[]) => string;
|
|
45
46
|
/**
|
|
46
47
|
* Execute BM25 + semantic search and merge with RRF.
|
|
47
|
-
* Uses LadybugDB FTS for
|
|
48
|
+
* Uses LadybugDB FTS for fresh BM25 results (no cached data).
|
|
48
49
|
* The semanticSearch function is injected to keep this module environment-agnostic.
|
|
49
50
|
*
|
|
50
|
-
* When FTS is unavailable (e.g. read-only MCP connection, missing indexes)
|
|
51
|
-
*
|
|
51
|
+
* When FTS is unavailable (e.g. read-only MCP connection, missing indexes) or
|
|
52
|
+
* explicitly disabled for this index (`disabledReason`, #3091), falls back to
|
|
53
|
+
* semantic-only results instead of crashing (#1489). In the disabled case no
|
|
54
|
+
* BM25 query is issued at all.
|
|
52
55
|
*/
|
|
53
|
-
export declare const hybridSearch: (query: string, limit: number, executeQuery: (cypher: string) => Promise<any[]>, semanticSearch: (executeQuery: (cypher: string) => Promise<any[]>, query: string, k?: number) => Promise<SemanticSearchResult[]
|
|
56
|
+
export declare const hybridSearch: (query: string, limit: number, executeQuery: (cypher: string) => Promise<any[]>, semanticSearch: (executeQuery: (cypher: string) => Promise<any[]>, query: string, k?: number) => Promise<SemanticSearchResult[]>, disabledReason?: FtsDisabledReason) => Promise<HybridSearchResult[]>;
|
|
@@ -113,19 +113,22 @@ export const formatHybridResults = (results) => {
|
|
|
113
113
|
};
|
|
114
114
|
/**
|
|
115
115
|
* Execute BM25 + semantic search and merge with RRF.
|
|
116
|
-
* Uses LadybugDB FTS for
|
|
116
|
+
* Uses LadybugDB FTS for fresh BM25 results (no cached data).
|
|
117
117
|
* The semanticSearch function is injected to keep this module environment-agnostic.
|
|
118
118
|
*
|
|
119
|
-
* When FTS is unavailable (e.g. read-only MCP connection, missing indexes)
|
|
120
|
-
*
|
|
119
|
+
* When FTS is unavailable (e.g. read-only MCP connection, missing indexes) or
|
|
120
|
+
* explicitly disabled for this index (`disabledReason`, #3091), falls back to
|
|
121
|
+
* semantic-only results instead of crashing (#1489). In the disabled case no
|
|
122
|
+
* BM25 query is issued at all.
|
|
121
123
|
*/
|
|
122
|
-
export const hybridSearch = async (query, limit, executeQuery, semanticSearch) => {
|
|
123
|
-
// Use LadybugDB FTS for
|
|
124
|
+
export const hybridSearch = async (query, limit, executeQuery, semanticSearch, disabledReason) => {
|
|
125
|
+
// Use LadybugDB FTS for fresh BM25 results — skipped entirely when this
|
|
126
|
+
// index recorded an explicit FTS opt-out (`disabledReason`, #3091).
|
|
124
127
|
// If FTS fails (e.g. extension not loaded in MCP process), fall back to
|
|
125
128
|
// semantic-only search instead of crashing with "bm25Results is not iterable".
|
|
126
129
|
let bm25Results = [];
|
|
127
130
|
try {
|
|
128
|
-
const ftsResponse = await searchFTSFromLbug(query, limit);
|
|
131
|
+
const ftsResponse = await searchFTSFromLbug(query, limit, undefined, disabledReason);
|
|
129
132
|
bm25Results = ftsResponse?.results ?? [];
|
|
130
133
|
}
|
|
131
134
|
catch {
|
|
@@ -48,6 +48,7 @@ import { EMBEDDING_TABLE_NAME, EMBEDDING_INDEX_NAME, embeddingDimsMismatch, } fr
|
|
|
48
48
|
import { getExactScanLimit } from '../../core/platform/capabilities.js';
|
|
49
49
|
import { PhaseTimer } from '../../core/search/phase-timer.js';
|
|
50
50
|
import { ftsDegradedWarning, ftsQueryFailedWarning } from '../../core/search/fts-indexes.js';
|
|
51
|
+
import { getFtsDisabledReason } from '../../core/search/fts-policy.js';
|
|
51
52
|
import { cjkSegmentationModeMismatch, containsSegmentableCjkRun, getSearchFTSCjkSegmentation, isSupportedCjkSegmentationMode, MAX_CJK_SEGMENTATION_QUERY_LENGTH, } from '../../core/search/cjk-segmentation.js';
|
|
52
53
|
import { checkStalenessAsync, checkCwdMatch } from '../../core/git-staleness.js';
|
|
53
54
|
import { stalenessPayload, } from '../../core/staleness-status.js';
|
|
@@ -2215,8 +2216,10 @@ export class LocalBackend {
|
|
|
2215
2216
|
// each so both get independent wall-time records without fighting
|
|
2216
2217
|
// over a single `current` phase slot.
|
|
2217
2218
|
const searchLimit = processLimit * maxSymbolsPerProcess; // fetch enough raw results
|
|
2219
|
+
const meta = await loadMeta(path.dirname(repo.lbugPath));
|
|
2220
|
+
const ftsDisabledReason = getFtsDisabledReason(meta?.capabilities?.fts);
|
|
2218
2221
|
const [bm25SearchResult, semanticResults] = await Promise.all([
|
|
2219
|
-
timer.time('bm25', this.bm25Search(repo, searchQuery, searchLimit)),
|
|
2222
|
+
timer.time('bm25', this.bm25Search(repo, searchQuery, searchLimit, ftsDisabledReason)),
|
|
2220
2223
|
timer.time('vector', this.semanticSearch(repo, searchQuery, searchLimit)),
|
|
2221
2224
|
]);
|
|
2222
2225
|
// Guard against undefined results (#1489) — when FTS is entirely
|
|
@@ -2508,7 +2511,7 @@ export class LocalBackend {
|
|
|
2508
2511
|
// instead of burying the real cause as a trailing suffix on bad advice.
|
|
2509
2512
|
warnings.push(ftsQueryErrors
|
|
2510
2513
|
? ftsQueryFailedWarning({ ...warningContext, lastErrorRedacted: ftsQueryErrors[0] })
|
|
2511
|
-
: ftsDegradedWarning(warningContext));
|
|
2514
|
+
: ftsDegradedWarning(warningContext, ftsDisabledReason));
|
|
2512
2515
|
}
|
|
2513
2516
|
else if (ftsQueryErrors) {
|
|
2514
2517
|
// #2767: at least one FTS table succeeded (ftsUsed=true) but another
|
|
@@ -2556,7 +2559,6 @@ export class LocalBackend {
|
|
|
2556
2559
|
// GITNEXUS_FTS_CJK_SEGMENTATION (the only thing that actually throws in
|
|
2557
2560
|
// there) cannot take an unrelated diagnostic down with it. Needs no guard
|
|
2558
2561
|
// of its own: loadMeta() returns null on any read/parse failure.
|
|
2559
|
-
const meta = await loadMeta(path.dirname(repo.lbugPath));
|
|
2560
2562
|
try {
|
|
2561
2563
|
// meta.json is on-disk state inside the analyzed repo, read via a
|
|
2562
2564
|
// schema-less JSON.parse — not trusted input. Validate before
|
|
@@ -2645,7 +2647,9 @@ export class LocalBackend {
|
|
|
2645
2647
|
/**
|
|
2646
2648
|
* BM25 keyword search helper - uses LadybugDB FTS for always-fresh results
|
|
2647
2649
|
*/
|
|
2648
|
-
async bm25Search(repo, query, limit) {
|
|
2650
|
+
async bm25Search(repo, query, limit, disabledReason) {
|
|
2651
|
+
if (disabledReason)
|
|
2652
|
+
return { results: [], ftsUsed: false };
|
|
2649
2653
|
let searchFTSFromLbug;
|
|
2650
2654
|
try {
|
|
2651
2655
|
({ searchFTSFromLbug } = await import('../../core/search/bm25-index.js'));
|
package/dist/server/api.js
CHANGED
|
@@ -18,6 +18,7 @@ import { NODE_TABLES } from '../_shared/index.js';
|
|
|
18
18
|
import { searchFTSFromLbug } from '../core/search/bm25-index.js';
|
|
19
19
|
import { hybridSearch } from '../core/search/hybrid-search.js';
|
|
20
20
|
import { ftsDegradedWarning } from '../core/search/fts-indexes.js';
|
|
21
|
+
import { getFtsDisabledReason } from '../core/search/fts-policy.js';
|
|
21
22
|
import { LocalBackend } from '../mcp/local/local-backend.js';
|
|
22
23
|
import { installServeMcpAuth, mountMCPEndpoints } from './mcp-http.js';
|
|
23
24
|
import { fileURLToPath } from 'url';
|
|
@@ -574,6 +575,21 @@ export const handleFileRequest = async (req, res, repoPath) => {
|
|
|
574
575
|
}
|
|
575
576
|
}
|
|
576
577
|
};
|
|
578
|
+
async function loadFtsSession(storagePath) {
|
|
579
|
+
const meta = await loadMeta(storagePath);
|
|
580
|
+
const ftsDisabledReason = getFtsDisabledReason(meta?.capabilities?.fts);
|
|
581
|
+
return {
|
|
582
|
+
meta,
|
|
583
|
+
ftsDisabledReason,
|
|
584
|
+
...skipFtsOption(Boolean(ftsDisabledReason)),
|
|
585
|
+
};
|
|
586
|
+
}
|
|
587
|
+
function skipFtsOption(skipFts) {
|
|
588
|
+
return skipFts ? { skipFts: true } : {};
|
|
589
|
+
}
|
|
590
|
+
function readOnlyFtsOptions(skipFts) {
|
|
591
|
+
return skipFts ? { readOnly: true, skipFts: true } : { readOnly: true };
|
|
592
|
+
}
|
|
577
593
|
export const handleQueryRequest = async (req, res, resolveRepo) => {
|
|
578
594
|
try {
|
|
579
595
|
const cypher = req.body.cypher;
|
|
@@ -594,9 +610,8 @@ export const handleQueryRequest = async (req, res, resolveRepo) => {
|
|
|
594
610
|
return;
|
|
595
611
|
}
|
|
596
612
|
const lbugPath = path.join(entry.storagePath, 'lbug');
|
|
597
|
-
const
|
|
598
|
-
|
|
599
|
-
});
|
|
613
|
+
const { skipFts } = await loadFtsSession(entry.storagePath);
|
|
614
|
+
const result = await withLbugDb(lbugPath, () => executePrepared(cypher, queryParams ?? {}), readOnlyFtsOptions(skipFts));
|
|
600
615
|
res.json({ result });
|
|
601
616
|
}
|
|
602
617
|
catch (err) {
|
|
@@ -950,6 +965,7 @@ export const createServer = async (port, host = '127.0.0.1') => {
|
|
|
950
965
|
const lbugPath = path.join(entry.storagePath, 'lbug');
|
|
951
966
|
const includeContent = req.query.includeContent === 'true';
|
|
952
967
|
const stream = req.query.stream === 'true';
|
|
968
|
+
const { skipFts } = await loadFtsSession(entry.storagePath);
|
|
953
969
|
if (stream) {
|
|
954
970
|
const abortController = new AbortController();
|
|
955
971
|
let responseFinished = false;
|
|
@@ -974,7 +990,7 @@ export const createServer = async (port, host = '127.0.0.1') => {
|
|
|
974
990
|
// "Cannot open file ... lbug.shadow - Error 2". See pool-adapter.ts
|
|
975
991
|
// which already opens read-only for the same reason, and the
|
|
976
992
|
// /api/query precedent in PR #1655.
|
|
977
|
-
await withLbugDb(lbugPath, async () => streamGraphNdjson(res, includeContent, abortController.signal),
|
|
993
|
+
await withLbugDb(lbugPath, async () => streamGraphNdjson(res, includeContent, abortController.signal), readOnlyFtsOptions(skipFts));
|
|
978
994
|
if (!abortController.signal.aborted && !res.writableEnded) {
|
|
979
995
|
res.end();
|
|
980
996
|
}
|
|
@@ -986,9 +1002,7 @@ export const createServer = async (port, host = '127.0.0.1') => {
|
|
|
986
1002
|
}
|
|
987
1003
|
return;
|
|
988
1004
|
}
|
|
989
|
-
const graph = await withLbugDb(lbugPath, async () => buildGraph(includeContent),
|
|
990
|
-
readOnly: true,
|
|
991
|
-
});
|
|
1005
|
+
const graph = await withLbugDb(lbugPath, async () => buildGraph(includeContent), readOnlyFtsOptions(skipFts));
|
|
992
1006
|
res.json(graph);
|
|
993
1007
|
}
|
|
994
1008
|
catch (err) {
|
|
@@ -1028,6 +1042,7 @@ export const createServer = async (port, host = '127.0.0.1') => {
|
|
|
1028
1042
|
}
|
|
1029
1043
|
const lbugPath = path.join(entry.storagePath, 'lbug');
|
|
1030
1044
|
const parsedLimit = Number(req.body.limit ?? 10);
|
|
1045
|
+
const { ftsDisabledReason, skipFts } = await loadFtsSession(entry.storagePath);
|
|
1031
1046
|
const limit = Number.isFinite(parsedLimit)
|
|
1032
1047
|
? Math.max(1, Math.min(100, Math.trunc(parsedLimit)))
|
|
1033
1048
|
: 10;
|
|
@@ -1052,7 +1067,7 @@ export const createServer = async (port, host = '127.0.0.1') => {
|
|
|
1052
1067
|
}));
|
|
1053
1068
|
}
|
|
1054
1069
|
else if (mode === 'bm25') {
|
|
1055
|
-
const ftsResponse = await searchFTSFromLbug(query, limit);
|
|
1070
|
+
const ftsResponse = await searchFTSFromLbug(query, limit, undefined, ftsDisabledReason);
|
|
1056
1071
|
ftsAvailable = ftsResponse.ftsAvailable;
|
|
1057
1072
|
searchResults = ftsResponse.results.map((r, i) => ({
|
|
1058
1073
|
...r,
|
|
@@ -1065,10 +1080,12 @@ export const createServer = async (port, host = '127.0.0.1') => {
|
|
|
1065
1080
|
const { isEmbedderReady } = await import('../core/embeddings/embedder.js');
|
|
1066
1081
|
if (isEmbedderReady()) {
|
|
1067
1082
|
const { semanticSearch: semSearch } = await import('../core/embeddings/embedding-pipeline.js');
|
|
1068
|
-
searchResults = await hybridSearch(query, limit, executeQuery, semSearch);
|
|
1083
|
+
searchResults = await hybridSearch(query, limit, executeQuery, semSearch, ftsDisabledReason);
|
|
1084
|
+
if (ftsDisabledReason)
|
|
1085
|
+
ftsAvailable = false;
|
|
1069
1086
|
}
|
|
1070
1087
|
else {
|
|
1071
|
-
const ftsResponse = await searchFTSFromLbug(query, limit);
|
|
1088
|
+
const ftsResponse = await searchFTSFromLbug(query, limit, undefined, ftsDisabledReason);
|
|
1072
1089
|
ftsAvailable = ftsResponse.ftsAvailable;
|
|
1073
1090
|
searchResults = ftsResponse.results;
|
|
1074
1091
|
}
|
|
@@ -1141,10 +1158,10 @@ export const createServer = async (port, host = '127.0.0.1') => {
|
|
|
1141
1158
|
return { ...r, ...enrichment };
|
|
1142
1159
|
}));
|
|
1143
1160
|
return { searchResults: enriched, ftsAvailable };
|
|
1144
|
-
},
|
|
1161
|
+
}, readOnlyFtsOptions(skipFts));
|
|
1145
1162
|
const response = { results: results.searchResults ?? results };
|
|
1146
1163
|
if (results.ftsAvailable === false) {
|
|
1147
|
-
response.warning = ftsDegradedWarning();
|
|
1164
|
+
response.warning = ftsDegradedWarning(undefined, ftsDisabledReason);
|
|
1148
1165
|
}
|
|
1149
1166
|
res.json(response);
|
|
1150
1167
|
}
|
|
@@ -1179,8 +1196,9 @@ export const createServer = async (port, host = '127.0.0.1') => {
|
|
|
1179
1196
|
// cut a stuck regex.test() when the wall-clock budget expires.
|
|
1180
1197
|
const { regex, fileFilter, limit } = parseGrepQuery(req.query);
|
|
1181
1198
|
const repoRoot = path.resolve(entry.path);
|
|
1199
|
+
const { skipFts } = await loadFtsSession(entry.storagePath);
|
|
1182
1200
|
const lbugPath = path.join(entry.storagePath, 'lbug');
|
|
1183
|
-
const fileRows = await withLbugDb(lbugPath, () => executeQuery(`MATCH (n:File) WHERE n.content IS NOT NULL RETURN n.filePath AS filePath`),
|
|
1201
|
+
const fileRows = await withLbugDb(lbugPath, () => executeQuery(`MATCH (n:File) WHERE n.content IS NOT NULL RETURN n.filePath AS filePath`), readOnlyFtsOptions(skipFts));
|
|
1184
1202
|
const filePaths = [];
|
|
1185
1203
|
for (const row of fileRows) {
|
|
1186
1204
|
const filePath = row.filePath || '';
|
|
@@ -1512,11 +1530,12 @@ export const createServer = async (port, host = '127.0.0.1') => {
|
|
|
1512
1530
|
let partialRunDetail;
|
|
1513
1531
|
try {
|
|
1514
1532
|
const lbugPath = path.join(entry.storagePath, 'lbug');
|
|
1533
|
+
const ftsSession = await loadFtsSession(entry.storagePath);
|
|
1534
|
+
let embeddingMeta = ftsSession.meta;
|
|
1515
1535
|
await withLbugDb(lbugPath, async () => {
|
|
1516
1536
|
const { runEmbeddingPipeline } = await import('../core/embeddings/embedding-pipeline.js');
|
|
1517
1537
|
const { resolveEmbeddingIdentity } = await import('../core/embeddings/embedding-identity.js');
|
|
1518
1538
|
const embeddingIdentity = resolveEmbeddingIdentity();
|
|
1519
|
-
let embeddingMeta = await loadMeta(entry.storagePath);
|
|
1520
1539
|
if (!embeddingMeta) {
|
|
1521
1540
|
throw new Error('Repository metadata is missing; run gitnexus analyze first');
|
|
1522
1541
|
}
|
|
@@ -1654,7 +1673,7 @@ export const createServer = async (port, host = '127.0.0.1') => {
|
|
|
1654
1673
|
partialRunDetail = outcome.partial;
|
|
1655
1674
|
embeddingMeta = withMeasuredEmbeddingCount({ ...finalMeta, embeddingCheckpoint: outcome.checkpoint }, measuredEmbeddings);
|
|
1656
1675
|
await saveMeta(entry.storagePath, embeddingMeta);
|
|
1657
|
-
});
|
|
1676
|
+
}, skipFtsOption(ftsSession.skipFts));
|
|
1658
1677
|
// Don't overwrite 'failed' if the job was cancelled while the pipeline was running
|
|
1659
1678
|
const current = embedJobManager.getJob(job.id);
|
|
1660
1679
|
if (!current || current.status !== 'failed') {
|
|
@@ -159,23 +159,24 @@ export interface RepoMeta {
|
|
|
159
159
|
* `'unavailable'` (#2841). Mirrors `AnalysisResult.ftsSkipReason` in
|
|
160
160
|
* core/run-analyze.ts — the same discriminator that surface already
|
|
161
161
|
* reports to the CLI, persisted rather than re-derived because the two
|
|
162
|
-
* causes need
|
|
162
|
+
* causes need distinct diagnostics and recovery handling:
|
|
163
163
|
*
|
|
164
164
|
* - `extension-unavailable` — the FTS extension could not load. Healable
|
|
165
|
-
* from outside the repo (install it),
|
|
166
|
-
* probes whether it loads now and re-analyzes when it does.
|
|
165
|
+
* from outside the repo (install it), then rebuild with --repair-fts.
|
|
167
166
|
* - `build-failed` — the extension loaded fine and the index BUILD
|
|
168
167
|
* failed (e.g. one un-tokenizable pre-existing row, #2544/#2546).
|
|
169
168
|
* Deterministic: the same probe would "heal" it into a full
|
|
170
169
|
* re-analysis that degrades identically and restamps, forever. Only
|
|
171
170
|
* `--repair-fts` or a content change addresses it.
|
|
171
|
+
* - `disabled-by-flag` / `disabled-by-env` — deliberate opt-out.
|
|
172
|
+
* A later analyze without the opt-out rebuilds FTS at the same commit.
|
|
172
173
|
*
|
|
173
174
|
* Collapsing both into `status: 'unavailable'` is exactly what made that
|
|
174
175
|
* loop reachable. ABSENT on indexes written before #2841 and on the
|
|
175
176
|
* `--repair-fts` stamp (which writes `status: 'available'`); `undefined`
|
|
176
177
|
* therefore reads as "cause unknown" and keeps the pre-#2841 behaviour.
|
|
177
178
|
*/
|
|
178
|
-
skipReason?: 'extension-unavailable' | 'build-failed';
|
|
179
|
+
skipReason?: 'extension-unavailable' | 'build-failed' | 'disabled-by-flag' | 'disabled-by-env';
|
|
179
180
|
};
|
|
180
181
|
vectorSearch: {
|
|
181
182
|
provider: string;
|
package/package.json
CHANGED
|
@@ -142,6 +142,7 @@ const PLATFORM_LOGIC = [
|
|
|
142
142
|
// N-API addon which has known platform-specific behavior (Windows
|
|
143
143
|
// file-lock lag after close, macOS N-API destructor segfaults)
|
|
144
144
|
const LBUG_NATIVE = [
|
|
145
|
+
'test/integration/skip-fts.test.ts',
|
|
145
146
|
'test/integration/lbug-core-adapter.test.ts',
|
|
146
147
|
'test/integration/lbug-vector-extension.test.ts',
|
|
147
148
|
'test/integration/lbug-pool.test.ts',
|