gitnexus 1.6.10-rc.85 → 1.6.10-rc.87
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/doctor.js +21 -3
- package/dist/core/lbug/extension-load-error.d.ts +2 -2
- package/dist/core/lbug/extension-load-error.js +31 -25
- package/dist/core/lbug/extension-loader.js +1 -1
- package/dist/core/lbug/lbug-adapter.d.ts +34 -0
- package/dist/core/lbug/lbug-adapter.js +79 -12
- package/dist/core/lbug/native-check.d.ts +18 -0
- package/dist/core/lbug/native-check.js +25 -1
- package/dist/core/lbug/pool-adapter.js +14 -1
- package/dist/core/platform/capabilities.d.ts +0 -1
- package/dist/core/platform/capabilities.js +12 -7
- package/dist/core/run-analyze.js +53 -11
- package/dist/mcp/local/local-backend.js +29 -25
- package/package.json +1 -1
- package/scripts/cross-platform-tests.ts +6 -0
- package/scripts/ensure-fts.ts +14 -3
package/dist/cli/doctor.js
CHANGED
|
@@ -4,7 +4,7 @@ import { isHttpMode } from '../core/embeddings/http-client.js';
|
|
|
4
4
|
import { getLocalEmbeddingRuntimeBlocker, localEmbeddingPrefixUnloadableMessage, localEmbeddingStackMissingMessage, } from '../core/embeddings/runtime-support.js';
|
|
5
5
|
import { isPrefixRuntimeLoadable, resolveEmbeddingRuntime, } from '../core/embeddings/runtime-install.js';
|
|
6
6
|
import { cudaRedirectDoctorStatus } from '../core/embeddings/onnxruntime-node-resolver.js';
|
|
7
|
-
import { checkLbugNative, probeFtsExtensionLoad } from '../core/lbug/native-check.js';
|
|
7
|
+
import { checkLbugNative, probeFtsExtensionLoad, probeVectorExtensionLoad, } from '../core/lbug/native-check.js';
|
|
8
8
|
import { getOsPageSize, isPageSizeAwareLadybug } from '../core/lbug/lbug-config.js';
|
|
9
9
|
import { diagnoseExtensionLoad } from '../core/lbug/extension-load-error.js';
|
|
10
10
|
import { getExtensionInstallPolicy } from '../core/lbug/extension-loader.js';
|
|
@@ -162,8 +162,26 @@ export const doctorCommand = async () => {
|
|
|
162
162
|
console.log(` ${padDisplayEnd('', 18)}${remedy}`);
|
|
163
163
|
}
|
|
164
164
|
}
|
|
165
|
-
|
|
166
|
-
|
|
165
|
+
// Live LOAD probe for VECTOR too (#2623). The static capability is just
|
|
166
|
+
// `platform !== 'win32'`, so it printed "available" on the very machines
|
|
167
|
+
// where analyze was failing to load the extension — the same contradiction
|
|
168
|
+
// #2374 fixed for FTS above, and exactly what #2623's reporter saw while
|
|
169
|
+
// every incremental analyze died on an unloaded VECTOR extension.
|
|
170
|
+
const vectorProbe = nativeCheck.ok
|
|
171
|
+
? await probeVectorExtensionLoad()
|
|
172
|
+
: { loaded: false, reason: 'LadybugDB native module (lbugjs.node) failed to load' };
|
|
173
|
+
console.log(` ${label('doctor.labels.vectorIndex', 18)}${vectorProbe.loaded ? 'available' : 'unavailable'}`);
|
|
174
|
+
if (!vectorProbe.loaded && vectorProbe.reason) {
|
|
175
|
+
console.log(` ${padDisplayEnd('', 18)}${vectorProbe.reason}`);
|
|
176
|
+
const { kind, remedy } = diagnoseExtensionLoad(vectorProbe.reason, 'VECTOR');
|
|
177
|
+
if (kind !== 'unknown') {
|
|
178
|
+
console.log(` ${padDisplayEnd('', 18)}${remedy}`);
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
// Semantic mode follows the probe, not the platform: without a loadable
|
|
182
|
+
// VECTOR extension the index can be neither built nor queried, so search is
|
|
183
|
+
// really on exact scan no matter what the platform would allow.
|
|
184
|
+
console.log(` ${label('doctor.labels.semanticMode', 18)}${vectorProbe.loaded ? capabilities.semanticMode : 'exact-scan'}`);
|
|
167
185
|
// Surface the optional-extension install policy so offline users can see
|
|
168
186
|
// whether analyze/query will reach the network (extension.ladybugdb.com).
|
|
169
187
|
// Literal label (like the 'native' line) to avoid adding i18n keys.
|
|
@@ -20,7 +20,7 @@ export declare const FILE_CORRUPTION_SIGNATURES: readonly RegExp[];
|
|
|
20
20
|
* Windows download (error 193, matched by `/not a valid/i`) routes to
|
|
21
21
|
* FORCE-reinstall rather than to the runtime-install remedy.
|
|
22
22
|
*/
|
|
23
|
-
export declare function classifyExtensionLoadError(reason: string | undefined | null): ExtensionLoadDiagnosis;
|
|
23
|
+
export declare function classifyExtensionLoadError(reason: string | undefined | null, label?: string): ExtensionLoadDiagnosis;
|
|
24
24
|
/** Well-formedness of the extension binary for the host platform + arch. */
|
|
25
25
|
export type ExtensionBinaryState = 'absent' | 'corrupt' | 'valid' | 'indeterminate';
|
|
26
26
|
/**
|
|
@@ -63,5 +63,5 @@ export declare function inspectExtensionBinary(extensionPath: string | null | un
|
|
|
63
63
|
* classifier (which still carries the language-independent hedged fallback). This
|
|
64
64
|
* is the entry point every surface should call.
|
|
65
65
|
*/
|
|
66
|
-
export declare function diagnoseExtensionLoad(reason: string | undefined | null): ExtensionLoadDiagnosis;
|
|
66
|
+
export declare function diagnoseExtensionLoad(reason: string | undefined | null, label?: string): ExtensionLoadDiagnosis;
|
|
67
67
|
export {};
|
|
@@ -85,26 +85,30 @@ const POSIX_MISSING_DEPENDENCY_SIGNATURES = [
|
|
|
85
85
|
* display language — the only localized part is the OS-error tail after it. So
|
|
86
86
|
* it is the language-independent fallback signal once the specific tails miss: a
|
|
87
87
|
* French/German/Japanese Windows 126 has a localized tail we cannot enumerate,
|
|
88
|
-
* but it still carries this wrapper. See
|
|
88
|
+
* but it still carries this wrapper. See hedgedLoadFailureRemedy.
|
|
89
89
|
*/
|
|
90
90
|
const LOAD_FAILURE_WRAPPER = /failed to load library/i;
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
91
|
+
// Remedies are label-parameterized (#2623 follow-up): doctor now live-probes
|
|
92
|
+
// VECTOR through the same classifier, and FTS-specific advice (`--repair-fts`
|
|
93
|
+
// repairs FTS indexes only) must not be dispensed for other extensions.
|
|
94
|
+
const repairFtsHint = (label, lead) => label === 'FTS' ? ` (${lead}\`gitnexus analyze --repair-fts\`)` : '';
|
|
95
|
+
const missingFileRemedy = (label) => `The ${label} extension is not installed. Re-run with network access and ` +
|
|
96
|
+
`GITNEXUS_LBUG_EXTENSION_INSTALL=auto${repairFtsHint(label, 'or ')} to download it.`;
|
|
97
|
+
const corruptFileRemedy = (label) => `The ${label} extension file is present but unreadable (corrupt, truncated, or built for another ` +
|
|
98
|
+
`platform). Re-download it with network access and ` +
|
|
99
|
+
`GITNEXUS_LBUG_EXTENSION_INSTALL=auto${repairFtsHint(label, '')}.`;
|
|
96
100
|
// Single source of truth for the VC++ runtime-install pointer, shared by the
|
|
97
101
|
// Windows-126 and structural missing-dependency remedies so the name/URL cannot
|
|
98
102
|
// drift between them (#2383 F5).
|
|
99
103
|
const VC_REDIST_INSTALL_HINT = 'the Microsoft Visual C++ 2015-2022 Redistributable (x64) from ' +
|
|
100
104
|
'https://aka.ms/vs/17/release/vc_redist.x64.exe';
|
|
101
105
|
// MSVC-first per DuckDB's canonical answer for this exact error; OpenSSL second.
|
|
102
|
-
const
|
|
106
|
+
const windowsMissingDependencyRemedy = (label) => `The ${label} extension is present but a required runtime library is missing (Windows error 126). ` +
|
|
103
107
|
'Reinstalling the extension will NOT help. Install ' +
|
|
104
108
|
VC_REDIST_INSTALL_HINT +
|
|
105
109
|
'; if the error persists, the extension also needs OpenSSL 3 ' +
|
|
106
110
|
'(libcrypto-3-x64.dll / libssl-3-x64.dll) on the DLL search path.';
|
|
107
|
-
const
|
|
111
|
+
const posixMissingDependencyRemedy = (label) => `The ${label} extension is present but a shared library it depends on could not be loaded (named in ` +
|
|
108
112
|
'the error above). Reinstalling the extension will NOT help — install that library or add it to ' +
|
|
109
113
|
'your loader search path.';
|
|
110
114
|
// Language-independent fallback: we know the extension failed to load, but the
|
|
@@ -113,13 +117,15 @@ const POSIX_MISSING_DEPENDENCY_REMEDY = 'The FTS extension is present but a shar
|
|
|
113
117
|
// branches — rather than confidently prescribing the wrong single fix. The clean
|
|
114
118
|
// long-term fix is upstream: have LadybugDB include the numeric GetLastError/errno
|
|
115
119
|
// in the message (as it already does elsewhere), so this becomes a code match.
|
|
116
|
-
const
|
|
120
|
+
const hedgedLoadFailureRemedy = (label) => `The ${label} extension file was found but could not be loaded — see the "Error:" text above (shown ` +
|
|
117
121
|
"in your system's language). Reinstalling usually will not help. If it names a missing module or " +
|
|
118
122
|
'library, install the required runtime (on Windows: the Microsoft Visual C++ 2015-2022 ' +
|
|
119
|
-
'Redistributable x64 and OpenSSL 3); if it names a corrupt or invalid file,
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
+
'Redistributable x64 and OpenSSL 3); if it names a corrupt or invalid file, ' +
|
|
124
|
+
(label === 'FTS'
|
|
125
|
+
? 'run `gitnexus analyze --repair-fts` to re-download.'
|
|
126
|
+
: 're-run analyze with network access and GITNEXUS_LBUG_EXTENSION_INSTALL=auto to re-download.');
|
|
127
|
+
const unknownRemedy = (label) => `The ${label} extension failed to load for an unrecognized reason. Run \`gitnexus doctor\` for live ` +
|
|
128
|
+
`${label} status and verify the extension file and platform.`;
|
|
123
129
|
const matchesAny = (reason, signatures) => signatures.some((re) => re.test(reason));
|
|
124
130
|
/**
|
|
125
131
|
* Classify a collapsed LadybugDB LOAD error. Order is most-specific-first and is
|
|
@@ -127,19 +133,19 @@ const matchesAny = (reason, signatures) => signatures.some((re) => re.test(reaso
|
|
|
127
133
|
* Windows download (error 193, matched by `/not a valid/i`) routes to
|
|
128
134
|
* FORCE-reinstall rather than to the runtime-install remedy.
|
|
129
135
|
*/
|
|
130
|
-
export function classifyExtensionLoadError(reason) {
|
|
136
|
+
export function classifyExtensionLoadError(reason, label = 'FTS') {
|
|
131
137
|
const text = reason ?? '';
|
|
132
138
|
if (matchesAny(text, MISSING_FILE_SIGNATURES)) {
|
|
133
|
-
return { kind: 'missing_file', remedy:
|
|
139
|
+
return { kind: 'missing_file', remedy: missingFileRemedy(label) };
|
|
134
140
|
}
|
|
135
141
|
if (matchesAny(text, FILE_CORRUPTION_SIGNATURES)) {
|
|
136
|
-
return { kind: 'corrupt_file', remedy:
|
|
142
|
+
return { kind: 'corrupt_file', remedy: corruptFileRemedy(label) };
|
|
137
143
|
}
|
|
138
144
|
if (matchesAny(text, WINDOWS_MISSING_DEPENDENCY_SIGNATURES)) {
|
|
139
|
-
return { kind: 'missing_dependency', remedy:
|
|
145
|
+
return { kind: 'missing_dependency', remedy: windowsMissingDependencyRemedy(label) };
|
|
140
146
|
}
|
|
141
147
|
if (matchesAny(text, POSIX_MISSING_DEPENDENCY_SIGNATURES)) {
|
|
142
|
-
return { kind: 'missing_dependency', remedy:
|
|
148
|
+
return { kind: 'missing_dependency', remedy: posixMissingDependencyRemedy(label) };
|
|
143
149
|
}
|
|
144
150
|
// Language-independent fallback: the extension demonstrably failed to load
|
|
145
151
|
// (lbug's English wrapper is present) but the localized OS tail matched no
|
|
@@ -147,11 +153,11 @@ export function classifyExtensionLoadError(reason) {
|
|
|
147
153
|
// remedy — strictly better than the generic `unknown` for non-English hosts,
|
|
148
154
|
// and it never prescribes the wrong fix.
|
|
149
155
|
if (LOAD_FAILURE_WRAPPER.test(text)) {
|
|
150
|
-
return { kind: 'missing_dependency', remedy:
|
|
156
|
+
return { kind: 'missing_dependency', remedy: hedgedLoadFailureRemedy(label) };
|
|
151
157
|
}
|
|
152
|
-
return { kind: 'unknown', remedy:
|
|
158
|
+
return { kind: 'unknown', remedy: unknownRemedy(label) };
|
|
153
159
|
}
|
|
154
|
-
const
|
|
160
|
+
const structuralMissingDependencyRemedy = (label) => `The ${label} extension file is valid, so the failure is a missing or incompatible runtime dependency, ` +
|
|
155
161
|
'not the extension itself — reinstalling will NOT help. On Windows, install ' +
|
|
156
162
|
VC_REDIST_INSTALL_HINT +
|
|
157
163
|
' and ensure OpenSSL 3 is available; on Linux/macOS install the shared library named in the error above.';
|
|
@@ -284,12 +290,12 @@ export function inspectExtensionBinary(extensionPath) {
|
|
|
284
290
|
* classifier (which still carries the language-independent hedged fallback). This
|
|
285
291
|
* is the entry point every surface should call.
|
|
286
292
|
*/
|
|
287
|
-
export function diagnoseExtensionLoad(reason) {
|
|
293
|
+
export function diagnoseExtensionLoad(reason, label = 'FTS') {
|
|
288
294
|
const text = reason ?? '';
|
|
289
|
-
const stringResult = classifyExtensionLoadError(text);
|
|
295
|
+
const stringResult = classifyExtensionLoadError(text, label);
|
|
290
296
|
const fileState = inspectExtensionBinary(extractExtensionPath(text));
|
|
291
297
|
if (fileState === 'corrupt') {
|
|
292
|
-
return { kind: 'corrupt_file', remedy:
|
|
298
|
+
return { kind: 'corrupt_file', remedy: corruptFileRemedy(label) };
|
|
293
299
|
}
|
|
294
300
|
if (fileState === 'valid') {
|
|
295
301
|
// The structural probe only inspects the first BINARY_HEADER_BYTES, so a file
|
|
@@ -307,7 +313,7 @@ export function diagnoseExtensionLoad(reason) {
|
|
|
307
313
|
// sharper remedy when it recognized the specific case (e.g. English 126).
|
|
308
314
|
const remedy = stringResult.kind === 'missing_dependency'
|
|
309
315
|
? stringResult.remedy
|
|
310
|
-
:
|
|
316
|
+
: structuralMissingDependencyRemedy(label);
|
|
311
317
|
return { kind: 'missing_dependency', remedy };
|
|
312
318
|
}
|
|
313
319
|
// 'absent' or 'indeterminate' → no positive structural evidence, so defer to the
|
|
@@ -484,6 +484,40 @@ export declare const createFTSIndex: (tableName: string, indexName: string, prop
|
|
|
484
484
|
* failure propagates so the caller can log it.
|
|
485
485
|
*/
|
|
486
486
|
export declare const createVectorIndex: () => Promise<boolean>;
|
|
487
|
+
/**
|
|
488
|
+
* Make DML against {@link EMBEDDING_TABLE_NAME} legal on the writable
|
|
489
|
+
* connection when it can be, and report whether it is.
|
|
490
|
+
*
|
|
491
|
+
* LadybugDB refuses EVERY mutation of a table carrying an HNSW index while
|
|
492
|
+
* the VECTOR extension is not loaded on that connection: `DELETE` fails with
|
|
493
|
+
* "Trying to delete from an index on table CodeEmbedding but its extension is
|
|
494
|
+
* not loaded", `CREATE` with the matching "insert into an index" variant,
|
|
495
|
+
* `DROP TABLE` is refused while the index references it, and `SET` — even on
|
|
496
|
+
* a NON-indexed property — segfaults the process outright. Probed against
|
|
497
|
+
* @ladybugdb/core 0.18.2 (the lockfile-pinned version) and 0.18.0 — every
|
|
498
|
+
* result identical on both (#2623).
|
|
499
|
+
*
|
|
500
|
+
* Dropping the index is NOT an available recovery: `CALL DROP_VECTOR_INDEX`
|
|
501
|
+
* is itself a VECTOR-extension function and resolves to "Catalog exception:
|
|
502
|
+
* function DROP_VECTOR_INDEX is not defined" in exactly the state it would
|
|
503
|
+
* need to rescue. Loading the extension is the only in-place repair, which is
|
|
504
|
+
* why this returns a verdict instead of attempting a fixup.
|
|
505
|
+
*
|
|
506
|
+
* `true` = embedding-row DML is safe: either VECTOR is now loaded, or the
|
|
507
|
+
* table carries no index to trip over. `false` = genuinely blocked (index
|
|
508
|
+
* present, extension unloadable); the analyze orchestrator answers that by
|
|
509
|
+
* escalating to the wipe-and-rebuild write plan instead of failing
|
|
510
|
+
* mid-writeback.
|
|
511
|
+
*
|
|
512
|
+
* Cheap by construction: one local `SHOW_INDEXES` read settles the common
|
|
513
|
+
* "this repo never built an embedding index" case without touching the
|
|
514
|
+
* extension machinery at all, so a VECTOR-less machine is not charged a
|
|
515
|
+
* bounded INSTALL attempt on every incremental analyze. `SHOW_INDEXES` is
|
|
516
|
+
* readable WITHOUT the extension and reports `extension_loaded` per index, so
|
|
517
|
+
* no error-string sniffing is needed; it runs through the unprepared
|
|
518
|
+
* `conn.query()` path like every other `CALL` procedure here (#2114).
|
|
519
|
+
*/
|
|
520
|
+
export declare const ensureEmbeddingRowDmlSafe: () => Promise<boolean>;
|
|
487
521
|
/**
|
|
488
522
|
* Lazy-create an FTS index, caching the fact in-process.
|
|
489
523
|
*
|
|
@@ -13,10 +13,9 @@ import { NODE_TABLES, REL_TABLE_NAME, SCHEMA_QUERIES, EMBEDDING_TABLE_NAME, CREA
|
|
|
13
13
|
import { streamAllCSVsToDisk } from './csv-generator.js';
|
|
14
14
|
import { getNodeLabel as deriveNodeLabel } from './rel-pair-routing.js';
|
|
15
15
|
import { EMBEDDABLE_LABELS } from '../embeddings/types.js';
|
|
16
|
-
import { extensionManager } from './extension-loader.js';
|
|
16
|
+
import { extensionManager, resolveAnalyzeInstallPolicy, } from './extension-loader.js';
|
|
17
17
|
import { classifyDeleteAllError, closeLbugConnection, HANDLE_RELEASE_PROBE_ATTEMPTS, HANDLE_RELEASE_PROBE_DELAY_MS, isDbBusyError, isOpenRetryExhausted, isWalCorruptionError, openLbugConnection, sleep, toNativeSafePath, resolveNativeSafeStorageDir, WAL_RECOVERY_SUGGESTION, waitForWindowsHandleRelease, } from './lbug-config.js';
|
|
18
18
|
import { finalizeLbugSidecarsAfterClose, guardWalQuarantine, isMissingShadowSidecarError, isReadOnlyShadowReplayError, lbugLockRemediation, preflightLbugSidecars, quarantineWalForMissingShadow, renameFailureMessage, shadowSidecarRecoveryMessage, } from './sidecar-recovery.js';
|
|
19
|
-
import { isVectorExtensionSupportedByPlatform } from '../platform/capabilities.js';
|
|
20
19
|
import { logger } from '../logger.js';
|
|
21
20
|
/**
|
|
22
21
|
* Split a relationship CSV into per-label-pair files on disk.
|
|
@@ -2396,16 +2395,16 @@ export const loadVectorExtension = async (targetConn, opts = {}) => {
|
|
|
2396
2395
|
const useModuleState = targetConn === undefined;
|
|
2397
2396
|
if (useModuleState && vectorExtensionLoaded)
|
|
2398
2397
|
return true;
|
|
2399
|
-
//
|
|
2400
|
-
//
|
|
2401
|
-
//
|
|
2402
|
-
//
|
|
2403
|
-
//
|
|
2404
|
-
//
|
|
2405
|
-
|
|
2406
|
-
|
|
2407
|
-
|
|
2408
|
-
|
|
2398
|
+
// No platform gate. Windows was hard-refused here for years on the strength
|
|
2399
|
+
// of an early-era report that in-process INSTALL VECTOR could SIGSEGV
|
|
2400
|
+
// (#1365) — but the extension server ships win_amd64 VECTOR artifacts for
|
|
2401
|
+
// every 0.18.x extension version (probed live: v0.18.0 and v0.18.1 both
|
|
2402
|
+
// serve a real PE32+ DLL; the pinned 0.18.2 core resolves its extension
|
|
2403
|
+
// directory to 0.18.1, strace-verified), and INSTALL now runs in a spawned
|
|
2404
|
+
// child process (installDuckDbExtensionOutOfProcess), so even a crashing
|
|
2405
|
+
// installer kills only the child and degrades to `false` here. LOAD of a
|
|
2406
|
+
// present extension file is an ordinary in-process load whose failures
|
|
2407
|
+
// surface as catchable errors, exactly like FTS.
|
|
2409
2408
|
const c = targetConn ?? conn;
|
|
2410
2409
|
if (!c) {
|
|
2411
2410
|
throw new Error('LadybugDB not initialized. Call initLbug first.');
|
|
@@ -2499,6 +2498,74 @@ export const createVectorIndex = async () => {
|
|
|
2499
2498
|
throw e;
|
|
2500
2499
|
}
|
|
2501
2500
|
};
|
|
2501
|
+
/**
|
|
2502
|
+
* Make DML against {@link EMBEDDING_TABLE_NAME} legal on the writable
|
|
2503
|
+
* connection when it can be, and report whether it is.
|
|
2504
|
+
*
|
|
2505
|
+
* LadybugDB refuses EVERY mutation of a table carrying an HNSW index while
|
|
2506
|
+
* the VECTOR extension is not loaded on that connection: `DELETE` fails with
|
|
2507
|
+
* "Trying to delete from an index on table CodeEmbedding but its extension is
|
|
2508
|
+
* not loaded", `CREATE` with the matching "insert into an index" variant,
|
|
2509
|
+
* `DROP TABLE` is refused while the index references it, and `SET` — even on
|
|
2510
|
+
* a NON-indexed property — segfaults the process outright. Probed against
|
|
2511
|
+
* @ladybugdb/core 0.18.2 (the lockfile-pinned version) and 0.18.0 — every
|
|
2512
|
+
* result identical on both (#2623).
|
|
2513
|
+
*
|
|
2514
|
+
* Dropping the index is NOT an available recovery: `CALL DROP_VECTOR_INDEX`
|
|
2515
|
+
* is itself a VECTOR-extension function and resolves to "Catalog exception:
|
|
2516
|
+
* function DROP_VECTOR_INDEX is not defined" in exactly the state it would
|
|
2517
|
+
* need to rescue. Loading the extension is the only in-place repair, which is
|
|
2518
|
+
* why this returns a verdict instead of attempting a fixup.
|
|
2519
|
+
*
|
|
2520
|
+
* `true` = embedding-row DML is safe: either VECTOR is now loaded, or the
|
|
2521
|
+
* table carries no index to trip over. `false` = genuinely blocked (index
|
|
2522
|
+
* present, extension unloadable); the analyze orchestrator answers that by
|
|
2523
|
+
* escalating to the wipe-and-rebuild write plan instead of failing
|
|
2524
|
+
* mid-writeback.
|
|
2525
|
+
*
|
|
2526
|
+
* Cheap by construction: one local `SHOW_INDEXES` read settles the common
|
|
2527
|
+
* "this repo never built an embedding index" case without touching the
|
|
2528
|
+
* extension machinery at all, so a VECTOR-less machine is not charged a
|
|
2529
|
+
* bounded INSTALL attempt on every incremental analyze. `SHOW_INDEXES` is
|
|
2530
|
+
* readable WITHOUT the extension and reports `extension_loaded` per index, so
|
|
2531
|
+
* no error-string sniffing is needed; it runs through the unprepared
|
|
2532
|
+
* `conn.query()` path like every other `CALL` procedure here (#2114).
|
|
2533
|
+
*/
|
|
2534
|
+
export const ensureEmbeddingRowDmlSafe = async () => {
|
|
2535
|
+
const targetConn = conn;
|
|
2536
|
+
if (!targetConn) {
|
|
2537
|
+
throw new Error('LadybugDB not initialized. Call initLbug first.');
|
|
2538
|
+
}
|
|
2539
|
+
// Catalog FIRST. The overwhelmingly common case on a repo that never enabled
|
|
2540
|
+
// embeddings is "no index at all", and that is provable with one local read
|
|
2541
|
+
// — no extension needed. Loading first would make every incremental analyze
|
|
2542
|
+
// on a VECTOR-less machine pay a bounded out-of-process INSTALL attempt (the
|
|
2543
|
+
// `auto` policy) plus an "extension unavailable" warning, for a repo that
|
|
2544
|
+
// can never hit this hazard.
|
|
2545
|
+
let indexRows;
|
|
2546
|
+
try {
|
|
2547
|
+
indexRows = await withConnLock(async () => readQueryRows(await targetConn.query('CALL SHOW_INDEXES() RETURN *')));
|
|
2548
|
+
}
|
|
2549
|
+
catch (err) {
|
|
2550
|
+
// Fall through to the load attempt: unable to prove the index is absent,
|
|
2551
|
+
// so the extension is the only thing that can make DML safe.
|
|
2552
|
+
logger.warn({ err }, `Could not read the index catalog to check for a ${EMBEDDING_TABLE_NAME} vector index; ` +
|
|
2553
|
+
'falling back to loading the VECTOR extension.');
|
|
2554
|
+
}
|
|
2555
|
+
// Any non-HASH index on the embedding table gates DML. Keyed on index TYPE,
|
|
2556
|
+
// not name, so an index built under a different name still counts; the
|
|
2557
|
+
// implicit primary-key HASH index is engine-internal and never gates.
|
|
2558
|
+
const indexGatesDml = indexRows === undefined ||
|
|
2559
|
+
indexRows.some((row) => {
|
|
2560
|
+
const table = row?.table_name ?? row?.[0];
|
|
2561
|
+
if (table !== EMBEDDING_TABLE_NAME)
|
|
2562
|
+
return false;
|
|
2563
|
+
return (row?.index_type ?? row?.[2]) !== 'HASH';
|
|
2564
|
+
});
|
|
2565
|
+
if (!indexGatesDml)
|
|
2566
|
+
return true;
|
|
2567
|
+
return await loadVectorExtension(undefined, { policy: resolveAnalyzeInstallPolicy() });
|
|
2568
|
+
};
|
|
2502
2569
|
/**
|
|
2503
2570
|
* Lazy-create an FTS index, caching the fact in-process.
|
|
2504
2571
|
*
|
|
@@ -9,6 +9,8 @@ export interface FtsProbeResult {
|
|
|
9
9
|
/** Collapsed LadybugDB error when `loaded` is false. */
|
|
10
10
|
reason?: string;
|
|
11
11
|
}
|
|
12
|
+
/** Same shape for every optional extension; `FtsProbeResult` is the legacy name. */
|
|
13
|
+
export type ExtensionProbeResult = FtsProbeResult;
|
|
12
14
|
/**
|
|
13
15
|
* Live-probe `LOAD EXTENSION fts` on a throwaway in-memory database.
|
|
14
16
|
*
|
|
@@ -26,3 +28,19 @@ export interface FtsProbeResult {
|
|
|
26
28
|
* would need an out-of-process probe.
|
|
27
29
|
*/
|
|
28
30
|
export declare function probeFtsExtensionLoad(timeoutMs?: number): Promise<FtsProbeResult>;
|
|
31
|
+
/**
|
|
32
|
+
* Live-probe `LOAD EXTENSION vector`, the VECTOR counterpart of the FTS probe.
|
|
33
|
+
*
|
|
34
|
+
* Needed for the same reason #2374 needed the FTS one, and reported the same
|
|
35
|
+
* way: #2623's reporter saw `doctor` print `VECTOR index: available` while
|
|
36
|
+
* every incremental `analyze` was dying because the extension had not loaded.
|
|
37
|
+
* `doctor` derived that line from a static platform capability, so it read
|
|
38
|
+
* "available" no matter what the extension file was doing.
|
|
39
|
+
*
|
|
40
|
+
* Probes for real on every platform, Windows included: the extension server
|
|
41
|
+
* ships win_amd64 VECTOR artifacts for every 0.18.x extension version (the
|
|
42
|
+
* old blanket Windows refusal was stale, #1365-era). LOAD never touches the
|
|
43
|
+
* network and never invokes the installer, so this probe is exactly as safe
|
|
44
|
+
* as the FTS one above.
|
|
45
|
+
*/
|
|
46
|
+
export declare function probeVectorExtensionLoad(timeoutMs?: number): Promise<ExtensionProbeResult>;
|
|
@@ -113,6 +113,30 @@ const closeProbeResults = (result) => {
|
|
|
113
113
|
* would need an out-of-process probe.
|
|
114
114
|
*/
|
|
115
115
|
export async function probeFtsExtensionLoad(timeoutMs = DEFAULT_FTS_PROBE_TIMEOUT_MS) {
|
|
116
|
+
return await probeExtensionLoad('fts', timeoutMs);
|
|
117
|
+
}
|
|
118
|
+
/**
|
|
119
|
+
* Live-probe `LOAD EXTENSION vector`, the VECTOR counterpart of the FTS probe.
|
|
120
|
+
*
|
|
121
|
+
* Needed for the same reason #2374 needed the FTS one, and reported the same
|
|
122
|
+
* way: #2623's reporter saw `doctor` print `VECTOR index: available` while
|
|
123
|
+
* every incremental `analyze` was dying because the extension had not loaded.
|
|
124
|
+
* `doctor` derived that line from a static platform capability, so it read
|
|
125
|
+
* "available" no matter what the extension file was doing.
|
|
126
|
+
*
|
|
127
|
+
* Probes for real on every platform, Windows included: the extension server
|
|
128
|
+
* ships win_amd64 VECTOR artifacts for every 0.18.x extension version (the
|
|
129
|
+
* old blanket Windows refusal was stale, #1365-era). LOAD never touches the
|
|
130
|
+
* network and never invokes the installer, so this probe is exactly as safe
|
|
131
|
+
* as the FTS one above.
|
|
132
|
+
*/
|
|
133
|
+
export async function probeVectorExtensionLoad(timeoutMs = DEFAULT_FTS_PROBE_TIMEOUT_MS) {
|
|
134
|
+
return await probeExtensionLoad('vector', timeoutMs);
|
|
135
|
+
}
|
|
136
|
+
/**
|
|
137
|
+
* Shared LOAD probe. `extension` is a fixed internal literal, never user input.
|
|
138
|
+
*/
|
|
139
|
+
async function probeExtensionLoad(extension, timeoutMs) {
|
|
116
140
|
let timer;
|
|
117
141
|
const timeout = new Promise((resolve) => {
|
|
118
142
|
timer = setTimeout(() => resolve({
|
|
@@ -128,7 +152,7 @@ export async function probeFtsExtensionLoad(timeoutMs = DEFAULT_FTS_PROBE_TIMEOU
|
|
|
128
152
|
try {
|
|
129
153
|
const conn = new lbug.Connection(db);
|
|
130
154
|
try {
|
|
131
|
-
const result = await conn.query(
|
|
155
|
+
const result = await conn.query(`LOAD EXTENSION ${extension}`);
|
|
132
156
|
closeProbeResults(result);
|
|
133
157
|
return { loaded: true };
|
|
134
158
|
}
|
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
*/
|
|
17
17
|
import fs from 'fs/promises';
|
|
18
18
|
import lbug from '@ladybugdb/core';
|
|
19
|
-
import { isReadOnlyDbError, loadFTSExtension } from './lbug-adapter.js';
|
|
19
|
+
import { isReadOnlyDbError, loadFTSExtension, loadVectorExtension } from './lbug-adapter.js';
|
|
20
20
|
import { closeQueryResults } from './query-result-utils.js';
|
|
21
21
|
import { createLbugDatabase, isWalCorruptionError, toNativeSafePath, WAL_RECOVERY_SUGGESTION, } from './lbug-config.js';
|
|
22
22
|
import { guardWalQuarantine, isMissingFsError, isMissingShadowSidecarError, isReadOnlyShadowReplayError, preflightLbugSidecars, quarantineWalForMissingShadow, renameFailureMessage, statIfExists, } from './sidecar-recovery.js';
|
|
@@ -263,6 +263,7 @@ function closeOne(repoId) {
|
|
|
263
263
|
// for the same dbPath reuse it instead of hitting a file lock.
|
|
264
264
|
shared.refCount = 0;
|
|
265
265
|
shared.ftsLoaded = false;
|
|
266
|
+
shared.vectorLoaded = false;
|
|
266
267
|
}
|
|
267
268
|
else {
|
|
268
269
|
shared.db.close().catch(() => { });
|
|
@@ -687,6 +688,13 @@ async function doInitLbug(repoId, dbPath) {
|
|
|
687
688
|
if (!shared.ftsLoaded) {
|
|
688
689
|
shared.ftsLoaded = await loadFTSExtension(available[0], { policy: 'load-only' });
|
|
689
690
|
}
|
|
691
|
+
// VECTOR too — extension load scope is per-Database, so this one load
|
|
692
|
+
// makes QUERY_VECTOR_INDEX legal on every pooled connection. Same
|
|
693
|
+
// load-only contract as FTS above; on failure the semantic-query lane
|
|
694
|
+
// falls back to the exact scan with its own diagnostic (#2623 follow-up).
|
|
695
|
+
if (!shared.vectorLoaded) {
|
|
696
|
+
shared.vectorLoaded = await loadVectorExtension(available[0], { policy: 'load-only' });
|
|
697
|
+
}
|
|
690
698
|
// Register pool entry only after all connections are pre-warmed and FTS is
|
|
691
699
|
// loaded. Concurrent executeQuery calls see either "not initialized"
|
|
692
700
|
// (and throw cleanly) or a fully ready pool — never a half-built one.
|
|
@@ -749,6 +757,11 @@ export async function initLbugWithDb(repoId, existingDb, dbPath) {
|
|
|
749
757
|
if (!shared.ftsLoaded) {
|
|
750
758
|
shared.ftsLoaded = await loadFTSExtension(available[0], { policy: 'load-only' });
|
|
751
759
|
}
|
|
760
|
+
// VECTOR too — same per-Database scope and load-only contract as the
|
|
761
|
+
// doInitLbug site above (#2623 follow-up).
|
|
762
|
+
if (!shared.vectorLoaded) {
|
|
763
|
+
shared.vectorLoaded = await loadVectorExtension(available[0], { policy: 'load-only' });
|
|
764
|
+
}
|
|
752
765
|
pool.set(repoId, {
|
|
753
766
|
db: existingDb,
|
|
754
767
|
available,
|
|
@@ -19,6 +19,5 @@ export interface RuntimeCapabilities {
|
|
|
19
19
|
export declare const DEFAULT_EXACT_SCAN_LIMIT = 10000;
|
|
20
20
|
export declare const getExactScanLimit: () => number;
|
|
21
21
|
export declare const getRuntimeFingerprint: () => RuntimeFingerprint;
|
|
22
|
-
export declare const isVectorExtensionSupportedByPlatform: (platform?: NodeJS.Platform) => boolean;
|
|
23
22
|
export declare const getRuntimeCapabilities: () => RuntimeCapabilities;
|
|
24
23
|
export declare const defaultEmbeddingThreads: () => number;
|
|
@@ -59,19 +59,24 @@ export const getRuntimeFingerprint = () => ({
|
|
|
59
59
|
ladybugdb: packageVersion('@ladybugdb/core'),
|
|
60
60
|
onnxruntime: packageVersion('onnxruntime-node'),
|
|
61
61
|
});
|
|
62
|
-
export const isVectorExtensionSupportedByPlatform = (platform = process.platform) => platform !== 'win32';
|
|
63
62
|
export const getRuntimeCapabilities = () => {
|
|
64
|
-
const vector = isVectorExtensionSupportedByPlatform() ? 'available' : 'unavailable';
|
|
65
63
|
const exactScanLimit = getExactScanLimit();
|
|
64
|
+
// Static PLATFORM capability only. LadybugDB ships the VECTOR extension for
|
|
65
|
+
// every platform gitnexus supports — the extension server hosts win_amd64
|
|
66
|
+
// artifacts for every 0.18.x extension version (probed: v0.18.0 and v0.18.1
|
|
67
|
+
// both return a real 14 MB PE32+ DLL; the pinned 0.18.2 core resolves its
|
|
68
|
+
// extension directory to 0.18.1, strace-verified), so the old
|
|
69
|
+
// `platform !== 'win32'` gate was stale (#1365-era). Whether the extension
|
|
70
|
+
// actually LOADS on a given machine is a runtime question — doctor answers
|
|
71
|
+
// it with probeVectorExtensionLoad, and analyze/query degrade to exact scan
|
|
72
|
+
// when the load fails.
|
|
66
73
|
return {
|
|
67
74
|
graph: 'available',
|
|
68
75
|
fts: 'available',
|
|
69
|
-
vector,
|
|
70
|
-
semanticMode:
|
|
76
|
+
vector: 'available',
|
|
77
|
+
semanticMode: 'vector-index',
|
|
71
78
|
exactScanLimit,
|
|
72
|
-
reason:
|
|
73
|
-
? 'LadybugDB VECTOR is disabled on this platform; semantic search uses exact scan when embeddings exist.'
|
|
74
|
-
: undefined,
|
|
79
|
+
reason: undefined,
|
|
75
80
|
};
|
|
76
81
|
};
|
|
77
82
|
export const defaultEmbeddingThreads = () => {
|
package/dist/core/run-analyze.js
CHANGED
|
@@ -13,7 +13,7 @@ import fs from 'fs/promises';
|
|
|
13
13
|
import { retryRename } from '../storage/fs-atomic.js';
|
|
14
14
|
import { runPipelineFromRepo } from './ingestion/pipeline.js';
|
|
15
15
|
import { resetDegradedParseCounter } from './tree-sitter/safe-parse.js';
|
|
16
|
-
import { initLbug, loadGraphToLbug, getLbugStats, executeQuery, executeWithReusedStatement, closeLbug, closeLbugBeforeExit, loadCachedEmbeddings, deleteNodesForFiles, deleteAllCommunitiesAndProcesses, deleteAllInterprocTaintPaths, deleteAllCallSummaries, deleteAllInjects, queryImportersBatch, loadFTSExtension, wipeLbugDbFiles, LbugWipeError, DELETE_FILES_CHUNK_SIZE, } from './lbug/lbug-adapter.js';
|
|
16
|
+
import { initLbug, loadGraphToLbug, getLbugStats, executeQuery, executeWithReusedStatement, closeLbug, closeLbugBeforeExit, loadCachedEmbeddings, deleteNodesForFiles, ensureEmbeddingRowDmlSafe, deleteAllCommunitiesAndProcesses, deleteAllInterprocTaintPaths, deleteAllCallSummaries, deleteAllInjects, queryImportersBatch, loadFTSExtension, wipeLbugDbFiles, LbugWipeError, DELETE_FILES_CHUNK_SIZE, } from './lbug/lbug-adapter.js';
|
|
17
17
|
import { estimateBufferPool, setBufferPoolSizeHint } from './lbug/lbug-config.js';
|
|
18
18
|
import { escapeCypherString } from './lbug/cypher-escape.js';
|
|
19
19
|
import { buildSearchIndexesOrDegrade, createSearchFTSIndexes, dropSearchFTSIndexes, initialiseSearchFTSStemmer, verifySearchFTSIndexes, } from './search/fts-indexes.js';
|
|
@@ -1223,15 +1223,57 @@ export async function runFullAnalysis(repoPath, options, callbacks, runnerIdenti
|
|
|
1223
1223
|
// DB write plan changes here; fileHashes/meta bookkeeping is identical.
|
|
1224
1224
|
// Thresholds + the AND-gate live in incremental/escalation-gate.ts.
|
|
1225
1225
|
const writeFraction = effectiveWriteSet.size / Math.max(1, allFilePaths.length);
|
|
1226
|
-
|
|
1226
|
+
// VECTOR gate (#2623) — load the extension BEFORE a single embedding row
|
|
1227
|
+
// is touched. `deleteNodesForFiles` below opens with the CodeEmbedding
|
|
1228
|
+
// join-delete, and LadybugDB refuses all DML on a table carrying its HNSW
|
|
1229
|
+
// index unless VECTOR is loaded on this connection; nothing else on this
|
|
1230
|
+
// path loads it until Phase 4, so every incremental run over a DB that
|
|
1231
|
+
// already built `code_embedding_idx` died here. Same seam the FTS drop
|
|
1232
|
+
// occupies at the head of this branch (#2589): index lifecycle first,
|
|
1233
|
+
// then rows. UNCONDITIONAL — not gated on `shouldGenerateEmbeddings` —
|
|
1234
|
+
// because a DB carrying the index from an earlier `--embeddings` run hits
|
|
1235
|
+
// the identical wall on a plain incremental run.
|
|
1236
|
+
//
|
|
1237
|
+
// When VECTOR genuinely cannot load, the table is immutable (the index
|
|
1238
|
+
// cannot be dropped without the extension either), so surgery is
|
|
1239
|
+
// impossible: fall through to the escalation valve's wipe-and-COPY plan,
|
|
1240
|
+
// which rebuilds the DB files outright and needs no embedding-row DML.
|
|
1241
|
+
const embeddingRowDmlSafe = await ensureEmbeddingRowDmlSafe();
|
|
1242
|
+
if (!embeddingRowDmlSafe && cachedEmbeddings.length === 0) {
|
|
1243
|
+
// The escalation below WIPES the DB files, and Phase 3.5 restores
|
|
1244
|
+
// embedding rows from `cachedEmbeddings` — which is only populated when
|
|
1245
|
+
// `deriveEmbeddingMode` saw `meta.stats.embeddings > 0`. A DB whose meta
|
|
1246
|
+
// under-reports its embeddings (meta restored from an older run, or a
|
|
1247
|
+
// count that never got stamped) would therefore have every vector
|
|
1248
|
+
// silently destroyed by a rebuild it did not ask for. Read them now,
|
|
1249
|
+
// while the DB is still intact — a plain MATCH, which needs no VECTOR
|
|
1250
|
+
// extension. Rows whose owning node is gone are dropped by Phase 3.5's
|
|
1251
|
+
// live-graph filter, exactly as on any other wiped path.
|
|
1252
|
+
const rescued = await loadCachedEmbeddings();
|
|
1253
|
+
if (rescued.embeddings.length > 0) {
|
|
1254
|
+
cachedEmbeddings = rescued.embeddings;
|
|
1255
|
+
cachedEmbeddingNodeIds = rescued.embeddingNodeIds;
|
|
1256
|
+
log(`Preserving ${rescued.embeddings.length} embedding row(s) across the forced rebuild ` +
|
|
1257
|
+
`(the index metadata did not account for them).`);
|
|
1258
|
+
}
|
|
1259
|
+
}
|
|
1260
|
+
if (!embeddingRowDmlSafe ||
|
|
1261
|
+
shouldEscalateIncrementalWrite(filesToDelete.length, effectiveWriteSet.size, allFilePaths.length)) {
|
|
1227
1262
|
escalatedFullWrite = true;
|
|
1228
|
-
log(
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1263
|
+
log(!embeddingRowDmlSafe
|
|
1264
|
+
? `Incremental: the ${EMBEDDING_TABLE_NAME} vector index exists but the VECTOR ` +
|
|
1265
|
+
`extension could not be loaded, so embedding rows cannot be rewritten in place — ` +
|
|
1266
|
+
`switching to a full DB write (wipe + bulk COPY) for this run. Semantic search ` +
|
|
1267
|
+
`falls back to exact scan until VECTOR is available; run \`gitnexus doctor\` for ` +
|
|
1268
|
+
`live extension status, or set GITNEXUS_LBUG_EXTENSION_INSTALL=auto to allow one ` +
|
|
1269
|
+
`bounded install attempt.`
|
|
1270
|
+
: `Incremental: effective write set covers ${effectiveWriteSet.size}/${allFilePaths.length} ` +
|
|
1271
|
+
// Display clamp only (predicate unchanged): BFS-found deleted
|
|
1272
|
+
// importers can push the numerator past the CURRENT file list, so
|
|
1273
|
+
// the raw fraction can exceed 1 — see the population-mismatch note
|
|
1274
|
+
// on shouldEscalateIncrementalWrite (tri-review 4669518496).
|
|
1275
|
+
`files (${Math.min(100, Math.round(writeFraction * 100))}%) — switching to a full DB write ` +
|
|
1276
|
+
`(wipe + bulk COPY) for this run; file-level incremental bookkeeping is unaffected.`);
|
|
1235
1277
|
// toWriteCount: 0 is the established full-path dirty-flag sentinel;
|
|
1236
1278
|
// the real counters ride along for crash diagnostics.
|
|
1237
1279
|
await saveIncrementalDirtyState('escalated-full-write', {
|
|
@@ -1565,8 +1607,8 @@ export async function runFullAnalysis(repoPath, options, callbacks, runnerIdenti
|
|
|
1565
1607
|
// the case a naive gate would leave index-less again.
|
|
1566
1608
|
// buildVectorIndex carries its own extension-policy gate and
|
|
1567
1609
|
// warn-on-failure; the boolean feeds semanticMode so the finalize stamp
|
|
1568
|
-
// reflects the DB's ACTUAL state even when recreation fails (
|
|
1569
|
-
//
|
|
1610
|
+
// reflects the DB's ACTUAL state even when recreation fails (extension
|
|
1611
|
+
// unavailable → 'exact-scan').
|
|
1570
1612
|
const dbWasWiped = !isIncremental || escalatedFullWrite;
|
|
1571
1613
|
if (restoredEmbeddingCount > 0 && dbWasWiped && embeddingSkipped) {
|
|
1572
1614
|
// Re-import at the seam rather than thread a mutable capture from
|
|
@@ -26,7 +26,7 @@ import { collectBestChunks } from '../../core/embeddings/types.js';
|
|
|
26
26
|
import { DEFAULT_MCP_VECTOR_MAX_DISTANCE, getVectorMaxDistance, } from '../../core/embeddings/config.js';
|
|
27
27
|
import { rankExactEmbeddingRows, } from '../../core/embeddings/exact-search.js';
|
|
28
28
|
import { EMBEDDING_TABLE_NAME, EMBEDDING_INDEX_NAME } from '../../core/lbug/schema.js';
|
|
29
|
-
import { getExactScanLimit
|
|
29
|
+
import { getExactScanLimit } from '../../core/platform/capabilities.js';
|
|
30
30
|
import { PhaseTimer } from '../../core/search/phase-timer.js';
|
|
31
31
|
import { ftsDegradedWarning } from '../../core/search/fts-indexes.js';
|
|
32
32
|
import { cjkSegmentationModeMismatch, containsSegmentableCjkRun, getSearchFTSCjkSegmentation, isSupportedCjkSegmentationMode, MAX_CJK_SEGMENTATION_QUERY_LENGTH, } from '../../core/search/cjk-segmentation.js';
|
|
@@ -1938,10 +1938,16 @@ export class LocalBackend {
|
|
|
1938
1938
|
const queryVecStr = `[${queryVec.join(',')}]`;
|
|
1939
1939
|
const maxDistance = getVectorMaxDistance(DEFAULT_MCP_VECTOR_MAX_DISTANCE);
|
|
1940
1940
|
let bestChunks = new Map();
|
|
1941
|
-
|
|
1942
|
-
|
|
1943
|
-
|
|
1944
|
-
|
|
1941
|
+
// Always TRY the vector lane — no platform gate. LadybugDB ships the
|
|
1942
|
+
// VECTOR extension for every supported platform, Windows included
|
|
1943
|
+
// (#2623 follow-up; the old `platform !== 'win32'` gate was stale), so
|
|
1944
|
+
// whether the index is queryable is a per-machine runtime fact. The
|
|
1945
|
+
// catch below is the fallback: any failure (extension unloadable, index
|
|
1946
|
+
// absent, older DB) degrades to the exact scan with a once-per-backend
|
|
1947
|
+
// diagnostic instead of being silently swallowed.
|
|
1948
|
+
try {
|
|
1949
|
+
bestChunks = await collectBestChunks(limit, async (fetchLimit) => {
|
|
1950
|
+
const vectorQuery = `
|
|
1945
1951
|
CALL QUERY_VECTOR_INDEX('${EMBEDDING_TABLE_NAME}', '${EMBEDDING_INDEX_NAME}',
|
|
1946
1952
|
CAST(${queryVecStr} AS FLOAT[${dims}]), ${fetchLimit})
|
|
1947
1953
|
YIELD node AS emb, distance
|
|
@@ -1951,27 +1957,25 @@ export class LocalBackend {
|
|
|
1951
1957
|
emb.startLine AS startLine, emb.endLine AS endLine, distance
|
|
1952
1958
|
ORDER BY distance
|
|
1953
1959
|
`;
|
|
1954
|
-
|
|
1955
|
-
|
|
1956
|
-
|
|
1957
|
-
|
|
1958
|
-
|
|
1959
|
-
|
|
1960
|
-
|
|
1961
|
-
|
|
1962
|
-
|
|
1963
|
-
}
|
|
1964
|
-
catch {
|
|
1965
|
-
bestChunks = new Map();
|
|
1966
|
-
}
|
|
1960
|
+
const embResults = await executeQuery(repo.lbugPath, vectorQuery);
|
|
1961
|
+
return embResults.map((row) => ({
|
|
1962
|
+
nodeId: row.nodeId ?? row[0],
|
|
1963
|
+
chunkIndex: row.chunkIndex ?? row[1] ?? 0,
|
|
1964
|
+
startLine: row.startLine ?? row[2] ?? 0,
|
|
1965
|
+
endLine: row.endLine ?? row[3] ?? 0,
|
|
1966
|
+
distance: row.distance ?? row[4],
|
|
1967
|
+
}));
|
|
1968
|
+
});
|
|
1967
1969
|
}
|
|
1968
|
-
|
|
1969
|
-
|
|
1970
|
-
|
|
1971
|
-
|
|
1972
|
-
|
|
1973
|
-
|
|
1974
|
-
|
|
1970
|
+
catch (err) {
|
|
1971
|
+
bestChunks = new Map();
|
|
1972
|
+
if (!this.warnedVectorUnsupported) {
|
|
1973
|
+
// Rare diagnostic: surface why semantic search fell back to the
|
|
1974
|
+
// exact scan. Emitted once per `LocalBackend` instance lifetime to
|
|
1975
|
+
// avoid noisy stderr on hot semantic-search paths (DoD §2.8).
|
|
1976
|
+
this.warnedVectorUnsupported = true;
|
|
1977
|
+
logger.warn({ err }, 'GitNexus [query:vector]: vector index query failed; using exact scan fallback');
|
|
1978
|
+
}
|
|
1975
1979
|
}
|
|
1976
1980
|
if (bestChunks.size === 0) {
|
|
1977
1981
|
const embeddingCount = Number(tableCheck[0].cnt ?? tableCheck[0][0] ?? 0);
|
package/package.json
CHANGED
|
@@ -117,6 +117,12 @@ const LBUG_NATIVE = [
|
|
|
117
117
|
// to a live native DB, rm-then-rename over an existing parked copy) before
|
|
118
118
|
// any open — rename semantics are exactly what differs on Windows.
|
|
119
119
|
'test/unit/incremental-dirty-recovery.test.ts',
|
|
120
|
+
// #2623: the incremental writeback must load VECTOR before the CodeEmbedding
|
|
121
|
+
// join-delete, and the blocked path must escalate instead of crashing. The
|
|
122
|
+
// win32 VECTOR gate was removed in the same PR, so this ordering must be
|
|
123
|
+
// proven on the windows-latest native addon, not just Ubuntu. Budget: ~25s
|
|
124
|
+
// on Linux → expect ~2min on the slowest Windows shard.
|
|
125
|
+
'test/unit/incremental-vector-extension-ordering.test.ts',
|
|
120
126
|
];
|
|
121
127
|
|
|
122
128
|
// Process spawning and CLI tests — exercise child_process with real
|
package/scripts/ensure-fts.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Install the LadybugDB FTS
|
|
3
|
-
* every test in a sharded CI run finds
|
|
2
|
+
* Install the LadybugDB FTS and VECTOR extensions into the shared home (~/.lbdb)
|
|
3
|
+
* up front, so every test in a sharded CI run finds them regardless of shard.
|
|
4
4
|
*
|
|
5
5
|
* FTS-dependent tests split two ways: the LOAD-path gate (skipUnlessFtsAvailable)
|
|
6
6
|
* self-installs on miss, but the FILE-path gate (requireFtsResourceOrSkip, e.g.
|
|
@@ -17,13 +17,24 @@
|
|
|
17
17
|
import { mkdtempSync, rmSync } from 'node:fs';
|
|
18
18
|
import { tmpdir } from 'node:os';
|
|
19
19
|
import { join } from 'node:path';
|
|
20
|
-
import {
|
|
20
|
+
import {
|
|
21
|
+
initLbug,
|
|
22
|
+
loadFTSExtension,
|
|
23
|
+
loadVectorExtension,
|
|
24
|
+
closeLbug,
|
|
25
|
+
} from '../src/core/lbug/lbug-adapter.js';
|
|
21
26
|
|
|
22
27
|
const dir = mkdtempSync(join(tmpdir(), 'gn-ensure-fts-'));
|
|
23
28
|
try {
|
|
24
29
|
await initLbug(join(dir, 'ensure-fts.lbug'));
|
|
25
30
|
const ok = await loadFTSExtension(undefined, { policy: 'auto' });
|
|
26
31
|
console.log(ok ? 'FTS extension ready.' : 'FTS extension unavailable (continuing).');
|
|
32
|
+
// VECTOR rides the same pre-install (#2623): the win32 gate is gone, so the
|
|
33
|
+
// vector suites genuinely run on Windows/macOS — installing once here means
|
|
34
|
+
// every sharded test process LOADs from ~/.lbdb instead of racing its own
|
|
35
|
+
// out-of-process INSTALL (bounded 15s each when the server is unreachable).
|
|
36
|
+
const vec = await loadVectorExtension(undefined, { policy: 'auto' });
|
|
37
|
+
console.log(vec ? 'VECTOR extension ready.' : 'VECTOR extension unavailable (continuing).');
|
|
27
38
|
} catch (err) {
|
|
28
39
|
console.warn(`ensure-fts: skipped (${err instanceof Error ? err.message : String(err)})`);
|
|
29
40
|
} finally {
|