gitnexus 1.6.10-rc.3 → 1.6.10-rc.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/dist/cli/doctor.js +10 -2
- package/dist/core/lbug/extension-loader.d.ts +10 -2
- package/dist/core/lbug/extension-loader.js +29 -12
- package/dist/core/lbug/native-check.d.ts +22 -0
- package/dist/core/lbug/native-check.js +66 -0
- package/dist/core/platform/capabilities.js +27 -1
- package/dist/core/run-analyze.js +11 -4
- package/dist/core/search/fts-indexes.d.ts +7 -0
- package/dist/core/search/fts-indexes.js +27 -0
- package/dist/mcp/local/local-backend.js +2 -1
- package/dist/server/api.js +2 -2
- package/package.json +1 -1
- package/scripts/cross-platform-tests.ts +4 -0
- package/scripts/install-duckdb-extension.mjs +83 -24
package/README.md
CHANGED
|
@@ -444,14 +444,14 @@ The prefix defaults to `~/.gitnexus/embedding-runtime`; set `GITNEXUS_EMBEDDING_
|
|
|
444
444
|
|
|
445
445
|
### Analyze warns about unavailable FTS or VECTOR extensions
|
|
446
446
|
|
|
447
|
-
GitNexus uses optional DuckDB extensions for BM25 and vector search. The `gitnexus serve` and MCP read paths only ever try to `LOAD` the extensions — they never block on a network install. The `analyze` command, by default, attempts one bounded out-of-process `INSTALL`
|
|
447
|
+
GitNexus uses optional DuckDB extensions for BM25 and vector search. The `gitnexus serve` and MCP read paths only ever try to `LOAD` the extensions — they never block on a network install. The `analyze` command, by default, attempts one bounded out-of-process install if `LOAD` fails (a plain `INSTALL` to download a missing extension, escalating to `FORCE INSTALL` only when the `LOAD` error shows the existing file is broken or truncated, so a permanent non-file failure does not re-download on every run) and proceeds even when that install times out, so the index is always written to disk; BM25/vector search degrade gracefully until the extensions become available.
|
|
448
448
|
|
|
449
449
|
Configure the behavior with these environment variables:
|
|
450
450
|
|
|
451
451
|
| Variable | Values | Default | Effect |
|
|
452
452
|
| -------------------------------------------- | ---------------------------- | ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
453
|
-
| `GITNEXUS_LBUG_EXTENSION_INSTALL` | `auto`, `load-only`, `never` | `auto` | `auto` runs one bounded
|
|
454
|
-
| `GITNEXUS_LBUG_EXTENSION_INSTALL_TIMEOUT_MS` | positive integer | `15000` | Wall-clock budget for the out-of-process
|
|
453
|
+
| `GITNEXUS_LBUG_EXTENSION_INSTALL` | `auto`, `load-only`, `never` | `auto` | `auto` runs one bounded install if LOAD fails — a plain `INSTALL`, escalating to `FORCE INSTALL` only when the LOAD error shows the present extension file is broken. `load-only` only uses already-installed extensions (recommended for offline / firewalled environments). `never` skips optional extensions entirely. |
|
|
454
|
+
| `GITNEXUS_LBUG_EXTENSION_INSTALL_TIMEOUT_MS` | positive integer | `15000` | Wall-clock budget for the out-of-process extension-install child before it is killed. |
|
|
455
455
|
| `GITNEXUS_FTS_STEMMER` | supported LadybugDB stemmer | `porter` | Stemmer used when rebuilding BM25/FTS indexes. Use `none` for CJK-heavy repositories, or a language stemmer such as `german`, `french`, or `spanish` when that better matches repository comments and identifiers. Re-run `gitnexus analyze --repair-fts` after changing it. |
|
|
456
456
|
| `GITNEXUS_FTS_CJK_SEGMENTATION` | `none`, `bigram` | `none` | `bigram` inserts overlapping character-bigram boundaries into Chinese/Japanese Han-ideograph spans in `content`/`description` before FTS indexing, so LadybugDB's space-only tokenizer can see sub-phrase word boundaries. Scoped to CJK Unified Ideographs only — Japanese Hiragana/Katakana and Korean Hangul are not currently segmented. Unlike `GITNEXUS_FTS_STEMMER`, this rewrites stored text — enabling it on an already-indexed repo requires a full `gitnexus analyze --force`; neither `--repair-fts` nor a plain incremental `analyze` applies it to previously-indexed files. Set the same value wherever `analyze` and search-serving processes (CLI query, MCP server, web server) run. |
|
|
457
457
|
| `GITNEXUS_WAL_CHECKPOINT_THRESHOLD` | integer `>= -1` | `67108864` (64 MiB) | LadybugDB WAL auto-checkpoint threshold during analyze (bytes). Auto-checkpoint remains enabled; `-1` keeps Ladybug's stock ~16 MiB. Larger thresholds reduce checkpoint frequency but increase the WAL size at rotation time — choose a smaller value on disk-constrained environments. |
|
package/dist/cli/doctor.js
CHANGED
|
@@ -4,7 +4,7 @@ import { isHttpMode } from '../core/embeddings/http-client.js';
|
|
|
4
4
|
import { getLocalEmbeddingRuntimeBlocker, localEmbeddingPrefixUnloadableMessage, localEmbeddingStackMissingMessage, } from '../core/embeddings/runtime-support.js';
|
|
5
5
|
import { isPrefixRuntimeLoadable, resolveEmbeddingRuntime, } from '../core/embeddings/runtime-install.js';
|
|
6
6
|
import { cudaRedirectDoctorStatus } from '../core/embeddings/onnxruntime-node-resolver.js';
|
|
7
|
-
import { checkLbugNative } from '../core/lbug/native-check.js';
|
|
7
|
+
import { checkLbugNative, probeFtsExtensionLoad } from '../core/lbug/native-check.js';
|
|
8
8
|
import { getExtensionInstallPolicy } from '../core/lbug/extension-loader.js';
|
|
9
9
|
import { t } from './i18n/index.js';
|
|
10
10
|
function isCombiningMark(codePoint) {
|
|
@@ -108,7 +108,15 @@ export const doctorCommand = async () => {
|
|
|
108
108
|
console.log('');
|
|
109
109
|
console.log(t('doctor.capabilities'));
|
|
110
110
|
console.log(` ${label('doctor.labels.graphStore', 18)}${capabilities.graph}`);
|
|
111
|
-
|
|
111
|
+
// Live LOAD probe, not the static platform capability — the static value
|
|
112
|
+
// said "available" while analyze failed to load the extension (#2374).
|
|
113
|
+
const ftsProbe = nativeCheck.ok
|
|
114
|
+
? await probeFtsExtensionLoad()
|
|
115
|
+
: { loaded: false, reason: 'LadybugDB native module (lbugjs.node) failed to load' };
|
|
116
|
+
console.log(` ${label('doctor.labels.fullTextSearch', 18)}${ftsProbe.loaded ? 'available' : 'unavailable'}`);
|
|
117
|
+
if (!ftsProbe.loaded && ftsProbe.reason) {
|
|
118
|
+
console.log(` ${padDisplayEnd('', 18)}${ftsProbe.reason}`);
|
|
119
|
+
}
|
|
112
120
|
console.log(` ${label('doctor.labels.vectorIndex', 18)}${capabilities.vector}`);
|
|
113
121
|
console.log(` ${label('doctor.labels.semanticMode', 18)}${capabilities.semanticMode}`);
|
|
114
122
|
// Surface the optional-extension install policy so offline users can see
|
|
@@ -29,7 +29,7 @@ export interface ExtensionEnsureOptions {
|
|
|
29
29
|
export interface ExtensionManagerOptions {
|
|
30
30
|
policy?: ExtensionInstallPolicy;
|
|
31
31
|
installTimeoutMs?: number;
|
|
32
|
-
installExtension?: (extensionName: string, timeoutMs: number) => Promise<ExtensionInstallResult>;
|
|
32
|
+
installExtension?: (extensionName: string, timeoutMs: number, loadError?: string) => Promise<ExtensionInstallResult>;
|
|
33
33
|
warn?: (message: string) => void;
|
|
34
34
|
}
|
|
35
35
|
export declare const getExtensionInstallPolicy: () => ExtensionInstallPolicy;
|
|
@@ -58,7 +58,7 @@ export declare const getExtensionInstallChildProcessArgs: (extensionName: string
|
|
|
58
58
|
* If the child exceeds `timeoutMs` the parent kills it with SIGKILL and
|
|
59
59
|
* resolves with `timedOut: true`.
|
|
60
60
|
*/
|
|
61
|
-
export declare const installDuckDbExtensionOutOfProcess: (extensionName: string, timeoutMs?: number) => Promise<ExtensionInstallResult>;
|
|
61
|
+
export declare const installDuckDbExtensionOutOfProcess: (extensionName: string, timeoutMs?: number, loadError?: string) => Promise<ExtensionInstallResult>;
|
|
62
62
|
/**
|
|
63
63
|
* Centralized lifecycle manager for optional LadybugDB extensions.
|
|
64
64
|
*
|
|
@@ -90,6 +90,14 @@ export declare class ExtensionManager {
|
|
|
90
90
|
* paths are expected to degrade gracefully.
|
|
91
91
|
*/
|
|
92
92
|
ensure(query: (sql: string) => Promise<unknown>, name: string, label: string, opts?: ExtensionEnsureOptions): Promise<boolean>;
|
|
93
|
+
/**
|
|
94
|
+
* Attempt `LOAD EXTENSION <name>`; returns `null` on success and the
|
|
95
|
+
* collapsed error message on failure. The message is the load-side ground
|
|
96
|
+
* truth — LadybugDB distinguishes a missing extension file from a present
|
|
97
|
+
* but unloadable one (wrong platform, truncated download, version mismatch),
|
|
98
|
+
* and discarding it left users staring at "not pre-installed" when the file
|
|
99
|
+
* existed all along (#2374).
|
|
100
|
+
*/
|
|
93
101
|
private tryLoad;
|
|
94
102
|
private markLoaded;
|
|
95
103
|
private markUnavailable;
|
|
@@ -7,6 +7,8 @@ const EXTENSION_NAME_PATTERN = /^[A-Za-z][A-Za-z0-9_]*$/;
|
|
|
7
7
|
const alreadyAvailable = (message) => message.includes('already loaded') ||
|
|
8
8
|
message.includes('already installed') ||
|
|
9
9
|
message.includes('already exists');
|
|
10
|
+
/** LadybugDB errors are multi-line; collapse for single-line warn/reason strings. */
|
|
11
|
+
const oneLine = (value) => value.replace(/\s+/g, ' ').trim();
|
|
10
12
|
const resolvePolicyFromEnv = () => {
|
|
11
13
|
const raw = process.env.GITNEXUS_LBUG_EXTENSION_INSTALL;
|
|
12
14
|
if (raw === 'load-only' || raw === 'never' || raw === 'auto')
|
|
@@ -51,7 +53,7 @@ export const getExtensionInstallChildProcessArgs = (extensionName, maxDbSize = L
|
|
|
51
53
|
* If the child exceeds `timeoutMs` the parent kills it with SIGKILL and
|
|
52
54
|
* resolves with `timedOut: true`.
|
|
53
55
|
*/
|
|
54
|
-
export const installDuckDbExtensionOutOfProcess = async (extensionName, timeoutMs = getExtensionInstallTimeoutMs()) => {
|
|
56
|
+
export const installDuckDbExtensionOutOfProcess = async (extensionName, timeoutMs = getExtensionInstallTimeoutMs(), loadError) => {
|
|
55
57
|
if (!EXTENSION_NAME_PATTERN.test(extensionName)) {
|
|
56
58
|
throw new Error(`Invalid DuckDB extension name: ${extensionName}`);
|
|
57
59
|
}
|
|
@@ -60,6 +62,9 @@ export const installDuckDbExtensionOutOfProcess = async (extensionName, timeoutM
|
|
|
60
62
|
env: {
|
|
61
63
|
...process.env,
|
|
62
64
|
GITNEXUS_LBUG_EXTENSION_NAME: extensionName,
|
|
65
|
+
// The child picks INSTALL vs FORCE INSTALL from this LOAD error so it
|
|
66
|
+
// only re-downloads when the on-disk extension file is actually broken.
|
|
67
|
+
...(loadError ? { GITNEXUS_LBUG_EXTENSION_LOAD_ERROR: loadError } : {}),
|
|
63
68
|
},
|
|
64
69
|
stdio: ['ignore', 'ignore', 'pipe'],
|
|
65
70
|
windowsHide: true,
|
|
@@ -78,7 +83,7 @@ export const installDuckDbExtensionOutOfProcess = async (extensionName, timeoutM
|
|
|
78
83
|
resolve({
|
|
79
84
|
success: false,
|
|
80
85
|
timedOut: true,
|
|
81
|
-
message: `
|
|
86
|
+
message: `extension install for ${extensionName} timed out after ${timeoutMs}ms`,
|
|
82
87
|
});
|
|
83
88
|
}, timeoutMs);
|
|
84
89
|
child.on('error', (err) => {
|
|
@@ -97,8 +102,8 @@ export const installDuckDbExtensionOutOfProcess = async (extensionName, timeoutM
|
|
|
97
102
|
success: code === 0,
|
|
98
103
|
timedOut: false,
|
|
99
104
|
message: code === 0
|
|
100
|
-
? `
|
|
101
|
-
: `
|
|
105
|
+
? `extension install for ${extensionName} completed`
|
|
106
|
+
: `extension install for ${extensionName} failed with ${signal ?? `exit code ${code}`}${stderr ? `: ${stderr.trim()}` : ''}`,
|
|
102
107
|
});
|
|
103
108
|
});
|
|
104
109
|
});
|
|
@@ -152,39 +157,51 @@ export class ExtensionManager {
|
|
|
152
157
|
this.markUnavailable(name, label, 'extension install policy is "never"', warn);
|
|
153
158
|
return false;
|
|
154
159
|
}
|
|
155
|
-
|
|
160
|
+
const loadError = await this.tryLoad(query, name);
|
|
161
|
+
if (loadError === null) {
|
|
156
162
|
this.markLoaded(name);
|
|
157
163
|
return true;
|
|
158
164
|
}
|
|
159
165
|
if (policy === 'load-only') {
|
|
160
|
-
this.markUnavailable(name, label,
|
|
166
|
+
this.markUnavailable(name, label, `load-only policy (no install attempted); LOAD ${name} failed: ${loadError}`, warn);
|
|
161
167
|
return false;
|
|
162
168
|
}
|
|
163
169
|
let install = this.installAttempted.get(name);
|
|
164
170
|
if (!install) {
|
|
165
171
|
const installFn = this.options.installExtension ?? installDuckDbExtensionOutOfProcess;
|
|
166
|
-
|
|
172
|
+
// Hand the child the LOAD error so it re-downloads (FORCE) only when the
|
|
173
|
+
// present extension file is provably broken, not on every LOAD failure.
|
|
174
|
+
install = await installFn(name, timeoutMs, loadError);
|
|
167
175
|
this.installAttempted.set(name, install);
|
|
168
176
|
}
|
|
169
177
|
if (!install.success) {
|
|
170
|
-
this.markUnavailable(name, label, install.message
|
|
178
|
+
this.markUnavailable(name, label, `${install.message}; LOAD ${name} had failed: ${loadError}`, warn);
|
|
171
179
|
return false;
|
|
172
180
|
}
|
|
173
|
-
|
|
181
|
+
const retryError = await this.tryLoad(query, name);
|
|
182
|
+
if (retryError === null) {
|
|
174
183
|
this.markLoaded(name);
|
|
175
184
|
return true;
|
|
176
185
|
}
|
|
177
|
-
this.markUnavailable(name, label, `LOAD ${name} failed after successful INSTALL`, warn);
|
|
186
|
+
this.markUnavailable(name, label, `LOAD ${name} failed after successful INSTALL: ${retryError}`, warn);
|
|
178
187
|
return false;
|
|
179
188
|
}
|
|
189
|
+
/**
|
|
190
|
+
* Attempt `LOAD EXTENSION <name>`; returns `null` on success and the
|
|
191
|
+
* collapsed error message on failure. The message is the load-side ground
|
|
192
|
+
* truth — LadybugDB distinguishes a missing extension file from a present
|
|
193
|
+
* but unloadable one (wrong platform, truncated download, version mismatch),
|
|
194
|
+
* and discarding it left users staring at "not pre-installed" when the file
|
|
195
|
+
* existed all along (#2374).
|
|
196
|
+
*/
|
|
180
197
|
async tryLoad(query, name) {
|
|
181
198
|
try {
|
|
182
199
|
await query(`LOAD EXTENSION ${name}`);
|
|
183
|
-
return
|
|
200
|
+
return null;
|
|
184
201
|
}
|
|
185
202
|
catch (err) {
|
|
186
203
|
const msg = err instanceof Error ? err.message : String(err);
|
|
187
|
-
return alreadyAvailable(msg);
|
|
204
|
+
return alreadyAvailable(msg) ? null : oneLine(msg);
|
|
188
205
|
}
|
|
189
206
|
}
|
|
190
207
|
markLoaded(name) {
|
|
@@ -4,3 +4,25 @@ export interface NativeCheckResult {
|
|
|
4
4
|
message?: string;
|
|
5
5
|
}
|
|
6
6
|
export declare function checkLbugNative(overridePkgDir?: string): NativeCheckResult;
|
|
7
|
+
export interface FtsProbeResult {
|
|
8
|
+
loaded: boolean;
|
|
9
|
+
/** Collapsed LadybugDB error when `loaded` is false. */
|
|
10
|
+
reason?: string;
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* Live-probe `LOAD EXTENSION fts` on a throwaway in-memory database.
|
|
14
|
+
*
|
|
15
|
+
* `doctor` used to print the static platform capability, which contradicted
|
|
16
|
+
* analyze whenever the extension file was missing or unloadable (#2374).
|
|
17
|
+
* LOAD never touches the network, so the probe is safe offline, and it
|
|
18
|
+
* surfaces LadybugDB's real error — which distinguishes a missing extension
|
|
19
|
+
* file from a present-but-broken one (wrong platform, truncated download).
|
|
20
|
+
* Dynamic import so doctor still runs when the native module itself is broken.
|
|
21
|
+
*
|
|
22
|
+
* Bounded by `timeoutMs`: an unresponsive extension file (e.g. on a hung
|
|
23
|
+
* network home dir) must never freeze `doctor` — the tool the degradation
|
|
24
|
+
* warnings send users to. `Promise.race` lets doctor report and move on; it
|
|
25
|
+
* cannot cancel an in-flight native call, so a future thread-blocking case
|
|
26
|
+
* would need an out-of-process probe.
|
|
27
|
+
*/
|
|
28
|
+
export declare function probeFtsExtensionLoad(timeoutMs?: number): Promise<FtsProbeResult>;
|
|
@@ -81,3 +81,69 @@ export function checkLbugNative(overridePkgDir) {
|
|
|
81
81
|
}
|
|
82
82
|
return { ok: true, binaryPath };
|
|
83
83
|
}
|
|
84
|
+
const DEFAULT_FTS_PROBE_TIMEOUT_MS = 10_000;
|
|
85
|
+
/** Close each result, swallowing close-time errors so a successful LOAD is not
|
|
86
|
+
* misreported as a failure (native-check keeps no static lbug dependency, so it
|
|
87
|
+
* cannot reuse the adapter's closeQueryResults — that would eagerly load the
|
|
88
|
+
* module and defeat the dynamic import below). */
|
|
89
|
+
const closeProbeResults = (result) => {
|
|
90
|
+
for (const r of Array.isArray(result) ? result : [result]) {
|
|
91
|
+
try {
|
|
92
|
+
r?.close?.();
|
|
93
|
+
}
|
|
94
|
+
catch {
|
|
95
|
+
// ignore — a close failure must not flip a successful LOAD to failed
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
};
|
|
99
|
+
/**
|
|
100
|
+
* Live-probe `LOAD EXTENSION fts` on a throwaway in-memory database.
|
|
101
|
+
*
|
|
102
|
+
* `doctor` used to print the static platform capability, which contradicted
|
|
103
|
+
* analyze whenever the extension file was missing or unloadable (#2374).
|
|
104
|
+
* LOAD never touches the network, so the probe is safe offline, and it
|
|
105
|
+
* surfaces LadybugDB's real error — which distinguishes a missing extension
|
|
106
|
+
* file from a present-but-broken one (wrong platform, truncated download).
|
|
107
|
+
* Dynamic import so doctor still runs when the native module itself is broken.
|
|
108
|
+
*
|
|
109
|
+
* Bounded by `timeoutMs`: an unresponsive extension file (e.g. on a hung
|
|
110
|
+
* network home dir) must never freeze `doctor` — the tool the degradation
|
|
111
|
+
* warnings send users to. `Promise.race` lets doctor report and move on; it
|
|
112
|
+
* cannot cancel an in-flight native call, so a future thread-blocking case
|
|
113
|
+
* would need an out-of-process probe.
|
|
114
|
+
*/
|
|
115
|
+
export async function probeFtsExtensionLoad(timeoutMs = DEFAULT_FTS_PROBE_TIMEOUT_MS) {
|
|
116
|
+
let timer;
|
|
117
|
+
const timeout = new Promise((resolve) => {
|
|
118
|
+
timer = setTimeout(() => resolve({
|
|
119
|
+
loaded: false,
|
|
120
|
+
reason: 'probe timed out — extension file or filesystem unresponsive',
|
|
121
|
+
}), timeoutMs);
|
|
122
|
+
});
|
|
123
|
+
const probe = (async () => {
|
|
124
|
+
try {
|
|
125
|
+
const { default: lbug } = await import('@ladybugdb/core');
|
|
126
|
+
const db = new lbug.Database(':memory:');
|
|
127
|
+
// Nested finallys so `db` is closed even if the Connection ctor throws.
|
|
128
|
+
try {
|
|
129
|
+
const conn = new lbug.Connection(db);
|
|
130
|
+
try {
|
|
131
|
+
const result = await conn.query('LOAD EXTENSION fts');
|
|
132
|
+
closeProbeResults(result);
|
|
133
|
+
return { loaded: true };
|
|
134
|
+
}
|
|
135
|
+
finally {
|
|
136
|
+
await conn.close().catch(() => { });
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
finally {
|
|
140
|
+
await db.close().catch(() => { });
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
catch (err) {
|
|
144
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
145
|
+
return { loaded: false, reason: message.replace(/\s+/g, ' ').trim() };
|
|
146
|
+
}
|
|
147
|
+
})();
|
|
148
|
+
return await Promise.race([probe, timeout]).finally(() => clearTimeout(timer));
|
|
149
|
+
}
|
|
@@ -1,4 +1,6 @@
|
|
|
1
|
+
import fs from 'fs';
|
|
1
2
|
import os from 'os';
|
|
3
|
+
import path from 'path';
|
|
2
4
|
import { createRequire } from 'module';
|
|
3
5
|
const require = createRequire(import.meta.url);
|
|
4
6
|
const packageVersion = (name) => {
|
|
@@ -6,7 +8,31 @@ const packageVersion = (name) => {
|
|
|
6
8
|
return require(`${name}/package.json`).version;
|
|
7
9
|
}
|
|
8
10
|
catch {
|
|
9
|
-
|
|
11
|
+
// Packages whose `exports` map omits ./package.json (e.g. @ladybugdb/core)
|
|
12
|
+
// reject the direct require with ERR_PACKAGE_PATH_NOT_EXPORTED, which made
|
|
13
|
+
// doctor print "LadybugDB: unknown" on every platform (#2374). Resolve the
|
|
14
|
+
// entry point instead and walk up to the package's own package.json.
|
|
15
|
+
try {
|
|
16
|
+
let dir = path.dirname(require.resolve(name));
|
|
17
|
+
// Entry points sit at the package root or a shallow dist/ dir; a few
|
|
18
|
+
// hops always reach the package's own package.json.
|
|
19
|
+
for (let hops = 0; hops < 5; hops++) {
|
|
20
|
+
const candidate = path.join(dir, 'package.json');
|
|
21
|
+
if (fs.existsSync(candidate)) {
|
|
22
|
+
const pkg = JSON.parse(fs.readFileSync(candidate, 'utf8'));
|
|
23
|
+
if (pkg.name === name)
|
|
24
|
+
return pkg.version;
|
|
25
|
+
}
|
|
26
|
+
const parent = path.dirname(dir);
|
|
27
|
+
if (parent === dir)
|
|
28
|
+
break;
|
|
29
|
+
dir = parent;
|
|
30
|
+
}
|
|
31
|
+
return undefined;
|
|
32
|
+
}
|
|
33
|
+
catch {
|
|
34
|
+
return undefined;
|
|
35
|
+
}
|
|
10
36
|
}
|
|
11
37
|
};
|
|
12
38
|
const gitnexusVersion = () => {
|
package/dist/core/run-analyze.js
CHANGED
|
@@ -16,7 +16,7 @@ import { resetDegradedParseCounter } from './tree-sitter/safe-parse.js';
|
|
|
16
16
|
import { initLbug, loadGraphToLbug, getLbugStats, executeQuery, executeWithReusedStatement, closeLbug, closeLbugBeforeExit, loadCachedEmbeddings, deleteNodesForFile, deleteAllCommunitiesAndProcesses, deleteAllInterprocTaintPaths, deleteAllCallSummaries, deleteAllInjects, queryImporters, loadFTSExtension, } from './lbug/lbug-adapter.js';
|
|
17
17
|
import { createSearchFTSIndexes, initialiseSearchFTSStemmer, verifySearchFTSIndexes, } from './search/fts-indexes.js';
|
|
18
18
|
import { cjkSegmentationModeMismatch, getSearchFTSCjkSegmentation, initialiseSearchFTSCjkSegmentation, } from './search/cjk-segmentation.js';
|
|
19
|
-
import { resolveAnalyzeInstallPolicy } from './lbug/extension-loader.js';
|
|
19
|
+
import { getExtensionCapabilities, resolveAnalyzeInstallPolicy } from './lbug/extension-loader.js';
|
|
20
20
|
import { startWalCheckpointDriver, } from './lbug/wal-checkpoint-driver.js';
|
|
21
21
|
import { getStoragePaths, resolveBranchPlacement, saveMeta, loadMeta, ensureGitNexusIgnored, registerRepo, adoptFlatBranchLabel, isReadOnlyFilesystemError, isRepoRegistered, cleanupOldKuzuFiles, reconcileMetadataFiles, isMissingFilesystemError, INDEX_METADATA_FILE, INCREMENTAL_SCHEMA_VERSION, } from '../storage/repo-manager.js';
|
|
22
22
|
import { DEFAULT_PDG_MAX_FUNCTION_LINES } from './ingestion/cfg/collect.js';
|
|
@@ -365,9 +365,16 @@ export async function runFullAnalysis(repoPath, options, callbacks) {
|
|
|
365
365
|
policy: resolveAnalyzeInstallPolicy(),
|
|
366
366
|
});
|
|
367
367
|
if (!repairFtsAvailable) {
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
368
|
+
// Surface the load-side reason (#2374): "not pre-installed" was wrong
|
|
369
|
+
// and doctor never installed anything, so the old message trapped
|
|
370
|
+
// users in a query → repair-fts → doctor loop with no way out.
|
|
371
|
+
const ftsReason = getExtensionCapabilities()
|
|
372
|
+
.find((c) => c.name === 'fts')
|
|
373
|
+
?.reason?.replace(/\.$/, '');
|
|
374
|
+
throw new Error('Cannot repair FTS indexes: the LadybugDB FTS extension failed to load' +
|
|
375
|
+
(ftsReason ? ` — ${ftsReason}` : '') +
|
|
376
|
+
'. Retry with network access and GITNEXUS_LBUG_EXTENSION_INSTALL=auto to install it, ' +
|
|
377
|
+
'or pre-install the extension file; run `gitnexus doctor` for live FTS status.');
|
|
371
378
|
}
|
|
372
379
|
progress('fts', 85, 'Repairing search indexes...');
|
|
373
380
|
await createSearchFTSIndexes({
|
|
@@ -1,3 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Warning attached to search responses when BM25/FTS is degraded. Prefers the
|
|
3
|
+
* live extension-load failure (with LadybugDB's real reason, #2374) over the
|
|
4
|
+
* generic indexes-missing message, so "indexes exist but the extension broke"
|
|
5
|
+
* is not misreported as missing indexes.
|
|
6
|
+
*/
|
|
7
|
+
export declare const ftsDegradedWarning: () => string;
|
|
1
8
|
export declare const SUPPORTED_FTS_STEMMERS: ReadonlySet<string>;
|
|
2
9
|
export interface CreateSearchFTSIndexesOptions {
|
|
3
10
|
onIndexStart?: (table: string, indexName: string) => void;
|
|
@@ -1,5 +1,32 @@
|
|
|
1
1
|
import { createFTSIndex, dropFTSIndex, DEFAULT_FTS_STEMMER } from '../lbug/lbug-adapter.js';
|
|
2
|
+
import { getExtensionCapabilities } from '../lbug/extension-loader.js';
|
|
2
3
|
import { FTS_INDEXES } from './fts-schema.js';
|
|
4
|
+
/**
|
|
5
|
+
* Strip filesystem paths from a LadybugDB error before it reaches the HTTP
|
|
6
|
+
* `/api/search` and MCP query surfaces (#2374, PR #2375): the raw LOAD error
|
|
7
|
+
* embeds the absolute extension path (username, home dir) which must not leak to
|
|
8
|
+
* a network client. The error class words ("Failed to load library", "invalid
|
|
9
|
+
* ELF header", "has not been installed") have no leading path separator and
|
|
10
|
+
* survive. CLI/doctor/log surfaces keep the full path (they read the reason
|
|
11
|
+
* directly, not through this function).
|
|
12
|
+
*/
|
|
13
|
+
const redactPaths = (reason) => reason.replace(/(?:[A-Za-z]:\\|\/)[^\s'"]+/g, '<path>');
|
|
14
|
+
/**
|
|
15
|
+
* Warning attached to search responses when BM25/FTS is degraded. Prefers the
|
|
16
|
+
* live extension-load failure (with LadybugDB's real reason, #2374) over the
|
|
17
|
+
* generic indexes-missing message, so "indexes exist but the extension broke"
|
|
18
|
+
* is not misreported as missing indexes.
|
|
19
|
+
*/
|
|
20
|
+
export const ftsDegradedWarning = () => {
|
|
21
|
+
const fts = getExtensionCapabilities().find((c) => c.name === 'fts');
|
|
22
|
+
if (fts && !fts.loaded) {
|
|
23
|
+
const reason = fts.reason ? redactPaths(fts.reason).replace(/\.$/, '') : undefined;
|
|
24
|
+
return ('FTS extension failed to load — keyword search degraded' +
|
|
25
|
+
(reason ? ` (${reason})` : '') +
|
|
26
|
+
'. Run `gitnexus doctor` for details, then `gitnexus analyze --repair-fts` with network access to reinstall.');
|
|
27
|
+
}
|
|
28
|
+
return 'FTS indexes missing — keyword search degraded. Run: gitnexus analyze --repair-fts (or gitnexus analyze --force) to rebuild indexes.';
|
|
29
|
+
};
|
|
3
30
|
// Stemmers shipped by the LadybugDB FTS extension. Mirrors the lowercase token
|
|
4
31
|
// set in the extension bundled with @ladybugdb/core 0.18.x (see package.json).
|
|
5
32
|
// Keep in sync on a LadybugDB minor bump — a value here that the installed
|
|
@@ -26,6 +26,7 @@ import { rankExactEmbeddingRows, } from '../../core/embeddings/exact-search.js';
|
|
|
26
26
|
import { EMBEDDING_TABLE_NAME, EMBEDDING_INDEX_NAME } from '../../core/lbug/schema.js';
|
|
27
27
|
import { getExactScanLimit, isVectorExtensionSupportedByPlatform, } from '../../core/platform/capabilities.js';
|
|
28
28
|
import { PhaseTimer } from '../../core/search/phase-timer.js';
|
|
29
|
+
import { ftsDegradedWarning } from '../../core/search/fts-indexes.js';
|
|
29
30
|
import { cjkSegmentationModeMismatch, containsSegmentableCjkRun, getSearchFTSCjkSegmentation, isSupportedCjkSegmentationMode, MAX_CJK_SEGMENTATION_QUERY_LENGTH, } from '../../core/search/cjk-segmentation.js';
|
|
30
31
|
import { checkStalenessAsync, checkCwdMatch } from '../../core/git-staleness.js';
|
|
31
32
|
import { logger } from '../../core/logger.js';
|
|
@@ -1659,7 +1660,7 @@ export class LocalBackend {
|
|
|
1659
1660
|
// path, leaving the success-path response shape byte-identical.
|
|
1660
1661
|
const warnings = [];
|
|
1661
1662
|
if (!ftsUsed) {
|
|
1662
|
-
warnings.push(
|
|
1663
|
+
warnings.push(ftsDegradedWarning());
|
|
1663
1664
|
}
|
|
1664
1665
|
// #2331: a CJK query against a server process resolving
|
|
1665
1666
|
// GITNEXUS_FTS_CJK_SEGMENTATION to 'none' silently misses sub-phrase
|
package/dist/server/api.js
CHANGED
|
@@ -18,6 +18,7 @@ import { isValidQueryParams } from '../core/lbug/query-params.js';
|
|
|
18
18
|
import { NODE_TABLES } from '../_shared/index.js';
|
|
19
19
|
import { searchFTSFromLbug } from '../core/search/bm25-index.js';
|
|
20
20
|
import { hybridSearch } from '../core/search/hybrid-search.js';
|
|
21
|
+
import { ftsDegradedWarning } from '../core/search/fts-indexes.js';
|
|
21
22
|
import { LocalBackend } from '../mcp/local/local-backend.js';
|
|
22
23
|
import { mountMCPEndpoints } from './mcp-http.js';
|
|
23
24
|
import { fileURLToPath } from 'url';
|
|
@@ -1140,8 +1141,7 @@ export const createServer = async (port, host = '127.0.0.1') => {
|
|
|
1140
1141
|
}, { readOnly: true });
|
|
1141
1142
|
const response = { results: results.searchResults ?? results };
|
|
1142
1143
|
if (results.ftsAvailable === false) {
|
|
1143
|
-
response.warning =
|
|
1144
|
-
'FTS indexes missing — keyword search degraded. Run: gitnexus analyze --repair-fts (or gitnexus analyze --force) to rebuild indexes.';
|
|
1144
|
+
response.warning = ftsDegradedWarning();
|
|
1145
1145
|
}
|
|
1146
1146
|
res.json(response);
|
|
1147
1147
|
}
|
package/package.json
CHANGED
|
@@ -100,6 +100,10 @@ const SPAWN_CLI = [
|
|
|
100
100
|
'test/integration/cli-limit-e2e.test.ts',
|
|
101
101
|
'test/integration/hooks-e2e.test.ts',
|
|
102
102
|
'test/integration/skills-e2e.test.ts',
|
|
103
|
+
// Spawns the real CLI across hermetic HOME/USERPROFILE homes to exercise the
|
|
104
|
+
// FTS extension lifecycle — the #2374 bug was Windows-reported, so this must
|
|
105
|
+
// run on the Windows/macOS matrix, not just the Ubuntu full suite.
|
|
106
|
+
'test/integration/fts-extension-e2e.test.ts',
|
|
103
107
|
'test/integration/server-http-startup.test.ts',
|
|
104
108
|
'test/integration/mcp/server-startup.test.ts',
|
|
105
109
|
'test/integration/analyze-heap-oom-e2e.test.ts',
|
|
@@ -3,9 +3,39 @@ import fs from 'node:fs/promises';
|
|
|
3
3
|
import os from 'node:os';
|
|
4
4
|
import path from 'node:path';
|
|
5
5
|
import { createRequire } from 'node:module';
|
|
6
|
+
import { pathToFileURL } from 'node:url';
|
|
6
7
|
|
|
7
8
|
const EXTENSION_NAME_PATTERN = /^[A-Za-z][A-Za-z0-9_]*$/;
|
|
8
9
|
|
|
10
|
+
// Positive on-disk-corruption signatures. `FORCE INSTALL` re-downloads even when
|
|
11
|
+
// a file is already present; we only want that when the LOAD error proves the
|
|
12
|
+
// existing file is bad (truncated/wrong-platform, #2374). For everything else —
|
|
13
|
+
// a missing file (plain INSTALL downloads it), or a permanent non-file failure a
|
|
14
|
+
// re-download can never fix (missing runtime dep: "cannot open shared object") —
|
|
15
|
+
// plain INSTALL avoids re-downloading ~2 MB on every analyze run forever.
|
|
16
|
+
const FILE_CORRUPTION_SIGNATURES = [
|
|
17
|
+
/invalid elf/i,
|
|
18
|
+
/file too short/i,
|
|
19
|
+
/not a valid/i,
|
|
20
|
+
/bad magic/i,
|
|
21
|
+
/wrong architecture/i,
|
|
22
|
+
/mach-o/i,
|
|
23
|
+
/truncat/i,
|
|
24
|
+
];
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Decide the install verb from the LOAD error that triggered this install.
|
|
28
|
+
* `FORCE INSTALL` only when the error positively indicates file-level breakage;
|
|
29
|
+
* otherwise plain `INSTALL` (missing file, missing-dependency dlopen failure,
|
|
30
|
+
* or unknown/absent error).
|
|
31
|
+
*/
|
|
32
|
+
export function chooseInstallVerb(loadError) {
|
|
33
|
+
if (loadError && FILE_CORRUPTION_SIGNATURES.some((re) => re.test(loadError))) {
|
|
34
|
+
return 'FORCE INSTALL';
|
|
35
|
+
}
|
|
36
|
+
return 'INSTALL';
|
|
37
|
+
}
|
|
38
|
+
|
|
9
39
|
function parseLbugMaxDbSize(raw) {
|
|
10
40
|
const parsed = raw ? Number(raw) : NaN;
|
|
11
41
|
if (!Number.isFinite(parsed) || parsed <= 0) {
|
|
@@ -14,28 +44,54 @@ function parseLbugMaxDbSize(raw) {
|
|
|
14
44
|
return Math.floor(parsed);
|
|
15
45
|
}
|
|
16
46
|
|
|
17
|
-
|
|
18
|
-
if (!extensionName || !EXTENSION_NAME_PATTERN.test(extensionName)) {
|
|
19
|
-
throw new Error(`Invalid DuckDB extension name: ${extensionName ?? '<missing>'}`);
|
|
20
|
-
}
|
|
21
|
-
|
|
22
|
-
const require = createRequire(import.meta.url);
|
|
23
|
-
const lbugModule = require('@ladybugdb/core');
|
|
24
|
-
const lbug = lbugModule.default ?? lbugModule;
|
|
47
|
+
function resolveMaxDbSize() {
|
|
25
48
|
// argv[3] is the optional positional size; ignore it when it is actually a
|
|
26
49
|
// flag token (e.g. `--verify-only`) and fall back to the env default.
|
|
27
50
|
const sizeArg =
|
|
28
51
|
process.argv[3] && !process.argv[3].startsWith('--') ? process.argv[3] : undefined;
|
|
29
|
-
|
|
52
|
+
return parseLbugMaxDbSize(sizeArg ?? process.env.GITNEXUS_LBUG_MAX_DB_SIZE);
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/** Open a scratch LadybugDB and return its connection plus a disposer. */
|
|
56
|
+
async function defaultConnect(lbugMaxDbSize) {
|
|
57
|
+
const require = createRequire(import.meta.url);
|
|
58
|
+
const lbugModule = require('@ladybugdb/core');
|
|
59
|
+
const lbug = lbugModule.default ?? lbugModule;
|
|
30
60
|
|
|
31
61
|
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-ext-install-'));
|
|
32
62
|
const dbPath = path.join(tmpDir, 'install.lbug');
|
|
33
|
-
|
|
34
|
-
|
|
63
|
+
const db = new lbug.Database(dbPath, 0, false, false, lbugMaxDbSize);
|
|
64
|
+
const conn = new lbug.Connection(db);
|
|
65
|
+
return {
|
|
66
|
+
conn,
|
|
67
|
+
dispose: async () => {
|
|
68
|
+
await conn.close().catch(() => {});
|
|
69
|
+
await db.close().catch(() => {});
|
|
70
|
+
await fs.rm(tmpDir, { recursive: true, force: true }).catch(() => {});
|
|
71
|
+
},
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Install (or verify) an optional LadybugDB extension in this short-lived process.
|
|
77
|
+
*
|
|
78
|
+
* @param {string} extensionName
|
|
79
|
+
* @param {object} [options]
|
|
80
|
+
* @param {boolean} [options.verifyOnly] LOAD-only Docker build gate — no install.
|
|
81
|
+
* @param {string} [options.loadError] The parent's LOAD failure; selects the verb.
|
|
82
|
+
* @param {(size: number) => Promise<{conn: {query: (sql: string) => Promise<unknown>}, dispose: () => Promise<void>}>} [options.connect]
|
|
83
|
+
* Connection factory; injectable for offline unit tests.
|
|
84
|
+
*/
|
|
85
|
+
export async function installDuckDbExtension(extensionName, options = {}) {
|
|
86
|
+
const { verifyOnly = false, loadError, connect } = options;
|
|
87
|
+
if (!extensionName || !EXTENSION_NAME_PATTERN.test(extensionName)) {
|
|
88
|
+
throw new Error(`Invalid DuckDB extension name: ${extensionName ?? '<missing>'}`);
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
const makeConnection = connect ?? (() => defaultConnect(resolveMaxDbSize()));
|
|
92
|
+
const { conn, dispose } = await makeConnection();
|
|
35
93
|
|
|
36
94
|
try {
|
|
37
|
-
db = new lbug.Database(dbPath, 0, false, false, lbugMaxDbSize);
|
|
38
|
-
conn = new lbug.Connection(db);
|
|
39
95
|
if (verifyOnly) {
|
|
40
96
|
// Prove a previously-baked extension is resolvable by a FRESH process
|
|
41
97
|
// under the current HOME (the runtime `LOAD EXTENSION` path) — no INSTALL,
|
|
@@ -46,19 +102,22 @@ async function installDuckDbExtension(extensionName, verifyOnly = false) {
|
|
|
46
102
|
`[install-ext] LOAD-only verify OK for '${extensionName}' (HOME=${process.env.HOME})`,
|
|
47
103
|
);
|
|
48
104
|
} else {
|
|
49
|
-
|
|
105
|
+
// Plain INSTALL is a no-op when the file already exists; escalate to FORCE
|
|
106
|
+
// only when the LOAD error proves the on-disk file is broken (#2374).
|
|
107
|
+
await conn.query(`${chooseInstallVerb(loadError)} ${extensionName}`);
|
|
50
108
|
}
|
|
51
109
|
} finally {
|
|
52
|
-
|
|
53
|
-
if (db) await db.close().catch(() => {});
|
|
54
|
-
await fs.rm(tmpDir, { recursive: true, force: true }).catch(() => {});
|
|
110
|
+
await dispose();
|
|
55
111
|
}
|
|
56
112
|
}
|
|
57
113
|
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
process.argv.
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
114
|
+
// Only run when executed directly — imported (e.g. by unit tests) it stays inert.
|
|
115
|
+
if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) {
|
|
116
|
+
installDuckDbExtension(process.argv[2] ?? process.env.GITNEXUS_LBUG_EXTENSION_NAME, {
|
|
117
|
+
verifyOnly: process.argv.includes('--verify-only'),
|
|
118
|
+
loadError: process.env.GITNEXUS_LBUG_EXTENSION_LOAD_ERROR,
|
|
119
|
+
}).catch((err) => {
|
|
120
|
+
console.error(err instanceof Error ? (err.stack ?? err.message) : String(err));
|
|
121
|
+
process.exitCode = 1;
|
|
122
|
+
});
|
|
123
|
+
}
|