gitnexus 1.6.10-rc.5 → 1.6.10-rc.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/analyze.js +46 -2
- package/dist/cli/cli-message.d.ts +1 -1
- package/dist/cli/doctor.js +10 -0
- package/dist/core/embeddings/http-client.d.ts +28 -1
- package/dist/core/embeddings/http-client.js +107 -15
- package/dist/core/lbug/extension-load-error.d.ts +67 -0
- package/dist/core/lbug/extension-load-error.js +320 -0
- package/dist/core/lbug/extension-loader.d.ts +7 -0
- package/dist/core/lbug/extension-loader.js +9 -1
- package/dist/core/lbug/lbug-adapter.js +7 -16
- package/dist/core/lbug/pool-adapter.js +10 -1
- package/dist/core/lbug/sidecar-recovery.d.ts +32 -0
- package/dist/core/lbug/sidecar-recovery.js +91 -7
- package/dist/core/run-analyze.js +27 -7
- package/dist/core/search/fts-indexes.js +11 -1
- package/package.json +1 -1
- package/scripts/cross-platform-tests.ts +6 -0
- package/scripts/install-duckdb-extension.mjs +3 -1
package/dist/cli/analyze.js
CHANGED
|
@@ -27,7 +27,7 @@ import { cliError } from './cli-message.js';
|
|
|
27
27
|
import { EMBEDDING_DIMS_ERROR, normalizeEmbeddingDims } from './embedding-dims.js';
|
|
28
28
|
import { formatElapsed } from './format-elapsed.js';
|
|
29
29
|
import { isHfDownloadFailure } from '../core/embeddings/hf-env.js';
|
|
30
|
-
import { isHttpMode, safeUrl } from '../core/embeddings/http-client.js';
|
|
30
|
+
import { isHttpEmbeddingDimsError, isHttpEmbeddingError, isHttpMode, safeUrl, } from '../core/embeddings/http-client.js';
|
|
31
31
|
import { isLocalEmbeddingRuntimeBlockerMessage, isMissingLocalEmbeddingStackMessage, localEmbeddingPrefixUnloadableMessage, localEmbeddingStackMissingMessage, } from '../core/embeddings/runtime-support.js';
|
|
32
32
|
import { ANALYZE_EMBEDDING_INSTALL_TIMEOUT_MS, getEmbeddingInstallTimeoutMs, getEmbeddingRuntimeDir, installEmbeddingRuntime, isPrefixRuntimeLoadable, resolveEmbeddingRuntime, } from '../core/embeddings/runtime-install.js';
|
|
33
33
|
import { warnIfNpm11NpxRisk } from './resolve-invocation.js';
|
|
@@ -1262,10 +1262,54 @@ const analyzeCommandImpl = async (inputPath, cliOptions) => {
|
|
|
1262
1262
|
process.exitCode = 1;
|
|
1263
1263
|
return;
|
|
1264
1264
|
}
|
|
1265
|
+
// Malformed GITNEXUS_EMBEDDING_DIMS env var (#2385). readConfig() throws a
|
|
1266
|
+
// plain Error (a config mistake, not an endpoint failure), surfacing here from
|
|
1267
|
+
// httpEmbed()->readConfig() inside the analysis run. Show a clean config
|
|
1268
|
+
// message rather than a raw stack dump. The --embedding-dims CLI flag is
|
|
1269
|
+
// validated up front (EMBEDDING_DIMS_ERROR); this covers the env-var path.
|
|
1270
|
+
// Checked before the endpoint/HF branches: it is a plain Error, so
|
|
1271
|
+
// isHttpEmbeddingError() is false and the HF network heuristic must not claim it.
|
|
1272
|
+
if (isHttpEmbeddingDimsError(msg)) {
|
|
1273
|
+
cliError(` ${msg.replace(/\n/g, '\n ')}\n`, {
|
|
1274
|
+
recoveryHint: 'embedding-dims-invalid',
|
|
1275
|
+
});
|
|
1276
|
+
process.exitCode = 1;
|
|
1277
|
+
return;
|
|
1278
|
+
}
|
|
1279
|
+
// Custom HTTP embedding endpoint failure (#2385). When a `--embedding-base-url`
|
|
1280
|
+
// is configured, HTTP mode never downloads a model — so a failure talking to
|
|
1281
|
+
// that endpoint must NOT show the huggingface-download guidance. Keyed on the
|
|
1282
|
+
// error *type* (HttpEmbeddingError), not its message text, so it stays correct
|
|
1283
|
+
// regardless of locale or wording. Checked before the HF branch, whose network
|
|
1284
|
+
// heuristic (`fetch failed` / `ECONNREFUSED`) would otherwise also match a
|
|
1285
|
+
// wrapped endpoint-connection error. The header is deliberately neutral: this
|
|
1286
|
+
// type covers both never-reached failures (connection/timeout/DNS) and
|
|
1287
|
+
// reached-but-failed ones (4xx/5xx, dimension/shape mismatch), so it must not
|
|
1288
|
+
// assert "unreachable". The thrown `msg` carries the specific reason (and the
|
|
1289
|
+
// masked URL where one applies), so it is surfaced verbatim.
|
|
1290
|
+
if (isHttpEmbeddingError(err)) {
|
|
1291
|
+
cliError(` The custom embedding endpoint request failed.\n` +
|
|
1292
|
+
` ${msg.replace(/\n/g, '\n ')}\n` +
|
|
1293
|
+
` Suggestions:\n` +
|
|
1294
|
+
` 1. Verify the endpoint URL is reachable and running ` +
|
|
1295
|
+
`(--embedding-base-url / GITNEXUS_EMBEDDING_URL: host, port, /v1 path).\n` +
|
|
1296
|
+
` 2. Confirm the model name and embedding dimensions match what the endpoint serves.\n` +
|
|
1297
|
+
` 3. Re-run without --embeddings to index without vectors.\n`, { recoveryHint: 'http-embedding-endpoint-error' });
|
|
1298
|
+
process.exitCode = 1;
|
|
1299
|
+
return;
|
|
1300
|
+
}
|
|
1301
|
+
// isHttpMode() is a pure presence probe (URL+MODEL) that never throws — a
|
|
1302
|
+
// malformed GITNEXUS_EMBEDDING_DIMS is handled by the dims branch above — so
|
|
1303
|
+
// no defensive try/catch is needed here (#2385).
|
|
1304
|
+
const inHttpMode = isHttpMode();
|
|
1265
1305
|
// HF download failure — show clean guidance without the raw stack trace.
|
|
1266
1306
|
// Checked before writeFatalToStderr so the user sees one focused message
|
|
1267
1307
|
// rather than a stack-trace dump followed by a second remediation block.
|
|
1268
|
-
|
|
1308
|
+
// Gated on !inHttpMode: with a custom endpoint configured no model download
|
|
1309
|
+
// is ever attempted, so a network error there is the endpoint's, handled by
|
|
1310
|
+
// the HttpEmbeddingError branch above — never HF's (#2385).
|
|
1311
|
+
if ((isHfDownloadFailure(msg) || msg.includes('Failed to download embedding model')) &&
|
|
1312
|
+
!inHttpMode) {
|
|
1269
1313
|
cliError(` The embedding model could not be downloaded.\n` +
|
|
1270
1314
|
` huggingface.co may be unreachable from your network\n` +
|
|
1271
1315
|
` (e.g. behind a corporate proxy or a regional firewall).\n` +
|
|
@@ -12,7 +12,7 @@ import { type CliMessageKey, type CliMessageVars } from './i18n/index.js';
|
|
|
12
12
|
* Consumers can import this type to narrow log-record `recoveryHint`
|
|
13
13
|
* fields without restating the literal list.
|
|
14
14
|
*/
|
|
15
|
-
export type RecoveryHint = 'wal-corruption' | 'wal-checkpoint-threshold' | 'heap-oom-respawn' | 'native-worker-abort' | 'hf-endpoint-unreachable' | 'local-embedding-unsupported' | 'local-embedding-stack-missing' | 'large-repo' | 'npm-resolution' | 'module-not-found' | 'gitnexusrc-invalid' | 'default-branch-invalid';
|
|
15
|
+
export type RecoveryHint = 'wal-corruption' | 'wal-checkpoint-threshold' | 'heap-oom-respawn' | 'native-worker-abort' | 'hf-endpoint-unreachable' | 'http-embedding-endpoint-error' | 'embedding-dims-invalid' | 'local-embedding-unsupported' | 'local-embedding-stack-missing' | 'large-repo' | 'npm-resolution' | 'module-not-found' | 'gitnexusrc-invalid' | 'default-branch-invalid';
|
|
16
16
|
/**
|
|
17
17
|
* Common shape for the optional structured-field bag passed to
|
|
18
18
|
* `cliError`/`cliWarn`/`cliInfo`. Typed so the `recoveryHint` slot is
|
package/dist/cli/doctor.js
CHANGED
|
@@ -5,6 +5,7 @@ import { getLocalEmbeddingRuntimeBlocker, localEmbeddingPrefixUnloadableMessage,
|
|
|
5
5
|
import { isPrefixRuntimeLoadable, resolveEmbeddingRuntime, } from '../core/embeddings/runtime-install.js';
|
|
6
6
|
import { cudaRedirectDoctorStatus } from '../core/embeddings/onnxruntime-node-resolver.js';
|
|
7
7
|
import { checkLbugNative, probeFtsExtensionLoad } from '../core/lbug/native-check.js';
|
|
8
|
+
import { diagnoseExtensionLoad } from '../core/lbug/extension-load-error.js';
|
|
8
9
|
import { getExtensionInstallPolicy } from '../core/lbug/extension-loader.js';
|
|
9
10
|
import { t } from './i18n/index.js';
|
|
10
11
|
function isCombiningMark(codePoint) {
|
|
@@ -116,6 +117,15 @@ export const doctorCommand = async () => {
|
|
|
116
117
|
console.log(` ${label('doctor.labels.fullTextSearch', 18)}${ftsProbe.loaded ? 'available' : 'unavailable'}`);
|
|
117
118
|
if (!ftsProbe.loaded && ftsProbe.reason) {
|
|
118
119
|
console.log(` ${padDisplayEnd('', 18)}${ftsProbe.reason}`);
|
|
120
|
+
// Add an actionable remedy for recognized failure classes (#2374). The
|
|
121
|
+
// Windows missing-dependency case is the point of this: the raw error 126
|
|
122
|
+
// ("specified module could not be found") is opaque, so name the fix (VC++
|
|
123
|
+
// redist, then OpenSSL) instead of leaving the user to reinstall in vain.
|
|
124
|
+
// `unknown`'s remedy is "run doctor", which would be circular here.
|
|
125
|
+
const { kind, remedy } = diagnoseExtensionLoad(ftsProbe.reason);
|
|
126
|
+
if (kind !== 'unknown') {
|
|
127
|
+
console.log(` ${padDisplayEnd('', 18)}${remedy}`);
|
|
128
|
+
}
|
|
119
129
|
}
|
|
120
130
|
console.log(` ${label('doctor.labels.vectorIndex', 18)}${capabilities.vector}`);
|
|
121
131
|
console.log(` ${label('doctor.labels.semanticMode', 18)}${capabilities.semanticMode}`);
|
|
@@ -11,7 +11,13 @@
|
|
|
11
11
|
* via `AbortSignal.timeout` on the underlying fetch.
|
|
12
12
|
*/
|
|
13
13
|
/**
|
|
14
|
-
*
|
|
14
|
+
* Whether HTTP embedding mode is active — i.e. both `GITNEXUS_EMBEDDING_URL` and
|
|
15
|
+
* `GITNEXUS_EMBEDDING_MODEL` are set. A pure presence probe: it deliberately does
|
|
16
|
+
* NOT call {@link readConfig}, so it never throws on a malformed
|
|
17
|
+
* `GITNEXUS_EMBEDDING_DIMS`. This lets its ~13 call sites (analyze, doctor,
|
|
18
|
+
* run-analyze, embedder, mcp) probe the mode without a defensive try/catch; the
|
|
19
|
+
* DIMS value is validated where it is actually used (`readConfig` in
|
|
20
|
+
* `httpEmbed`/`httpEmbedQuery`), surfacing a recognizable config error. See #2385.
|
|
15
21
|
*/
|
|
16
22
|
export declare const isHttpMode: () => boolean;
|
|
17
23
|
/**
|
|
@@ -26,6 +32,27 @@ export declare const getHttpDimensions: () => number | undefined;
|
|
|
26
32
|
* custom-endpoint confirmation can mask the same way.
|
|
27
33
|
*/
|
|
28
34
|
export declare const safeUrl: (url: string) => string;
|
|
35
|
+
/**
|
|
36
|
+
* Error thrown by this module's HTTP embedding path (`httpEmbedBatch` /
|
|
37
|
+
* `httpEmbed` / `httpEmbedQuery`) for any endpoint failure — a
|
|
38
|
+
* connection/timeout/DNS error, an open circuit, a non-OK status, an
|
|
39
|
+
* unparseable or wrong-shape response body, an empty response, or a dimension
|
|
40
|
+
* mismatch.
|
|
41
|
+
*
|
|
42
|
+
* Carrying a distinct type (rather than a plain `Error`) lets the CLI tell a
|
|
43
|
+
* *custom endpoint* failure apart from a HuggingFace *model download* failure
|
|
44
|
+
* without matching message text: the two share the same underlying network
|
|
45
|
+
* substrings (`fetch failed`, `ECONNREFUSED`, …), which is exactly why
|
|
46
|
+
* `isNetworkFetchError` in `hf-env.ts` cannot tell them apart. Keying on the
|
|
47
|
+
* type instead of the message is also locale-proof and survives message
|
|
48
|
+
* rewording. The human-readable `.message` (built with `safeUrl` and the
|
|
49
|
+
* underlying reason) is what the CLI surfaces to the user. See #2385.
|
|
50
|
+
*/
|
|
51
|
+
export declare class HttpEmbeddingError extends Error {
|
|
52
|
+
constructor(message: string, options?: {
|
|
53
|
+
cause?: unknown;
|
|
54
|
+
});
|
|
55
|
+
}
|
|
29
56
|
/**
|
|
30
57
|
* Embed texts via the HTTP backend, splitting into batches.
|
|
31
58
|
* Reads config from env vars on every call.
|
|
@@ -17,10 +17,26 @@ const HTTP_RETRY_BACKOFF_MS = 1_000;
|
|
|
17
17
|
const HTTP_BATCH_SIZE = 64;
|
|
18
18
|
const DEFAULT_DIMS = 384;
|
|
19
19
|
const HTTP_BREAKER_KEY = 'embeddings-http';
|
|
20
|
+
/**
|
|
21
|
+
* Stable lead of the {@link readConfig} malformed-`GITNEXUS_EMBEDDING_DIMS`
|
|
22
|
+
* error. `readConfig` throws a plain `Error` (not an {@link HttpEmbeddingError})
|
|
23
|
+
* because this is a *config* mistake, not an endpoint failure — so the CLI
|
|
24
|
+
* recognizes it by this lead ({@link isHttpEmbeddingDimsError}) and prints a
|
|
25
|
+
* clean config message instead of a raw stack dump. See #2385.
|
|
26
|
+
*/
|
|
27
|
+
const EMBEDDING_DIMS_ENV_ERROR_LEAD = 'GITNEXUS_EMBEDDING_DIMS must be a positive integer';
|
|
28
|
+
/**
|
|
29
|
+
* @internal Exported for the CLI analyze error handler. True when `message` is
|
|
30
|
+
* the {@link readConfig} malformed-DIMS config error (a plain `Error`).
|
|
31
|
+
*/
|
|
32
|
+
export const isHttpEmbeddingDimsError = (message) => message.includes(EMBEDDING_DIMS_ENV_ERROR_LEAD);
|
|
20
33
|
/**
|
|
21
34
|
* Build config from the current process.env snapshot.
|
|
22
35
|
* Returns null when GITNEXUS_EMBEDDING_URL + GITNEXUS_EMBEDDING_MODEL are unset.
|
|
23
36
|
* Not cached — env vars are read fresh so late configuration takes effect.
|
|
37
|
+
* Validates GITNEXUS_EMBEDDING_DIMS and throws on a malformed value; callers
|
|
38
|
+
* that only need to know whether HTTP mode is *configured* must use
|
|
39
|
+
* {@link isHttpMode} (a presence probe that never throws), not this.
|
|
24
40
|
*/
|
|
25
41
|
const readConfig = () => {
|
|
26
42
|
const baseUrl = process.env.GITNEXUS_EMBEDDING_URL;
|
|
@@ -31,11 +47,11 @@ const readConfig = () => {
|
|
|
31
47
|
let dimensions;
|
|
32
48
|
if (rawDims !== undefined) {
|
|
33
49
|
if (!/^\d+$/.test(rawDims)) {
|
|
34
|
-
throw new Error(
|
|
50
|
+
throw new Error(`${EMBEDDING_DIMS_ENV_ERROR_LEAD}, got "${rawDims}"`);
|
|
35
51
|
}
|
|
36
52
|
const parsed = parseInt(rawDims, 10);
|
|
37
53
|
if (parsed <= 0) {
|
|
38
|
-
throw new Error(
|
|
54
|
+
throw new Error(`${EMBEDDING_DIMS_ENV_ERROR_LEAD}, got "${rawDims}"`);
|
|
39
55
|
}
|
|
40
56
|
dimensions = parsed;
|
|
41
57
|
}
|
|
@@ -47,9 +63,15 @@ const readConfig = () => {
|
|
|
47
63
|
};
|
|
48
64
|
};
|
|
49
65
|
/**
|
|
50
|
-
*
|
|
66
|
+
* Whether HTTP embedding mode is active — i.e. both `GITNEXUS_EMBEDDING_URL` and
|
|
67
|
+
* `GITNEXUS_EMBEDDING_MODEL` are set. A pure presence probe: it deliberately does
|
|
68
|
+
* NOT call {@link readConfig}, so it never throws on a malformed
|
|
69
|
+
* `GITNEXUS_EMBEDDING_DIMS`. This lets its ~13 call sites (analyze, doctor,
|
|
70
|
+
* run-analyze, embedder, mcp) probe the mode without a defensive try/catch; the
|
|
71
|
+
* DIMS value is validated where it is actually used (`readConfig` in
|
|
72
|
+
* `httpEmbed`/`httpEmbedQuery`), surfacing a recognizable config error. See #2385.
|
|
51
73
|
*/
|
|
52
|
-
export const isHttpMode = () =>
|
|
74
|
+
export const isHttpMode = () => Boolean(process.env.GITNEXUS_EMBEDDING_URL && process.env.GITNEXUS_EMBEDDING_MODEL);
|
|
53
75
|
/**
|
|
54
76
|
* Return the configured embedding dimensions for HTTP mode, or undefined
|
|
55
77
|
* if HTTP mode is not active or no explicit dimensions are set.
|
|
@@ -70,6 +92,63 @@ export const safeUrl = (url) => {
|
|
|
70
92
|
return '<invalid-url>';
|
|
71
93
|
}
|
|
72
94
|
};
|
|
95
|
+
/**
|
|
96
|
+
* Strip credentials from an underlying transport error message before it is
|
|
97
|
+
* surfaced. A credential-bearing endpoint URL (`https://user:secret@host/v1`)
|
|
98
|
+
* makes undici throw `TypeError: Request cannot be constructed from a URL that
|
|
99
|
+
* includes credentials: <that full URL>`; interpolating `err.message` verbatim
|
|
100
|
+
* would re-leak the secret to stderr + logs even though the URL argument is
|
|
101
|
+
* already masked with {@link safeUrl}. First swap the exact configured `url` for
|
|
102
|
+
* its masked form, then strip any residual `scheme://userinfo@` the transport may
|
|
103
|
+
* have echoed in a normalized (non-exact) form. See #2385.
|
|
104
|
+
*/
|
|
105
|
+
const sanitizeReason = (reason, url) => reason
|
|
106
|
+
.split(url)
|
|
107
|
+
.join(safeUrl(url))
|
|
108
|
+
.replace(/([a-z][a-z0-9+.-]*:\/\/)[^/@\s]*@/gi, '$1');
|
|
109
|
+
/**
|
|
110
|
+
* Error thrown by this module's HTTP embedding path (`httpEmbedBatch` /
|
|
111
|
+
* `httpEmbed` / `httpEmbedQuery`) for any endpoint failure — a
|
|
112
|
+
* connection/timeout/DNS error, an open circuit, a non-OK status, an
|
|
113
|
+
* unparseable or wrong-shape response body, an empty response, or a dimension
|
|
114
|
+
* mismatch.
|
|
115
|
+
*
|
|
116
|
+
* Carrying a distinct type (rather than a plain `Error`) lets the CLI tell a
|
|
117
|
+
* *custom endpoint* failure apart from a HuggingFace *model download* failure
|
|
118
|
+
* without matching message text: the two share the same underlying network
|
|
119
|
+
* substrings (`fetch failed`, `ECONNREFUSED`, …), which is exactly why
|
|
120
|
+
* `isNetworkFetchError` in `hf-env.ts` cannot tell them apart. Keying on the
|
|
121
|
+
* type instead of the message is also locale-proof and survives message
|
|
122
|
+
* rewording. The human-readable `.message` (built with `safeUrl` and the
|
|
123
|
+
* underlying reason) is what the CLI surfaces to the user. See #2385.
|
|
124
|
+
*/
|
|
125
|
+
export class HttpEmbeddingError extends Error {
|
|
126
|
+
constructor(message, options) {
|
|
127
|
+
super(message, options?.cause !== undefined ? { cause: options.cause } : undefined);
|
|
128
|
+
this.name = 'HttpEmbeddingError';
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
/**
|
|
132
|
+
* @internal Exported for the CLI analyze error handler and unit tests.
|
|
133
|
+
*
|
|
134
|
+
* Type-guard for {@link HttpEmbeddingError}. The `name` fallback keeps the
|
|
135
|
+
* check working across module-realm boundaries where `instanceof` can fail
|
|
136
|
+
* (two loaded copies of the class) — mirroring the codebase's existing
|
|
137
|
+
* `err.name === 'TimeoutError'` idiom. Matches on the stable class
|
|
138
|
+
* discriminator, never on the human-readable (potentially localized) message.
|
|
139
|
+
*/
|
|
140
|
+
export const isHttpEmbeddingError = (err) => err instanceof HttpEmbeddingError || (err instanceof Error && err.name === 'HttpEmbeddingError');
|
|
141
|
+
/**
|
|
142
|
+
* Runtime guard for a single response item. The `Array.isArray(data.data)` shape
|
|
143
|
+
* check only validates the outer array — a 200 body like `{"data":[null]}` passes
|
|
144
|
+
* it, then crashes at `new Float32Array(item.embedding)` (`httpEmbed`) or
|
|
145
|
+
* `items[0].embedding` (`httpEmbedQuery`) with a raw `TypeError` that escapes the
|
|
146
|
+
* typed boundary, landing on the CLI's generic stack-dump path — the exact class
|
|
147
|
+
* #2385 closes. Validate each item so every wrong-shape body stays classifiable.
|
|
148
|
+
*/
|
|
149
|
+
const isEmbeddingItem = (item) => typeof item === 'object' &&
|
|
150
|
+
item !== null &&
|
|
151
|
+
Array.isArray(item.embedding);
|
|
73
152
|
/**
|
|
74
153
|
* Send a single batch of texts to the embedding endpoint with retry.
|
|
75
154
|
*
|
|
@@ -111,23 +190,36 @@ const httpEmbedBatch = async (url, batch, model, apiKey, batchIndex = 0, dimensi
|
|
|
111
190
|
}
|
|
112
191
|
catch (err) {
|
|
113
192
|
if (err instanceof CircuitOpenError) {
|
|
114
|
-
throw new
|
|
193
|
+
throw new HttpEmbeddingError(`Embedding endpoint circuit open (${safeUrl(url)}, batch ${batchIndex}): retry in ${Math.ceil(err.retryAfterMs / 1000)}s`, { cause: err });
|
|
115
194
|
}
|
|
116
195
|
if (err instanceof DOMException && err.name === 'TimeoutError') {
|
|
117
|
-
throw new
|
|
196
|
+
throw new HttpEmbeddingError(`Embedding request timed out after ${HTTP_TIMEOUT_MS}ms (${safeUrl(url)}, batch ${batchIndex})`, { cause: err });
|
|
118
197
|
}
|
|
119
198
|
if (err instanceof ResilientFetchExhaustedError) {
|
|
120
|
-
throw new
|
|
199
|
+
throw new HttpEmbeddingError(`Embedding endpoint returned ${err.response.status} (${safeUrl(url)}, batch ${batchIndex})`, { cause: err });
|
|
121
200
|
}
|
|
122
|
-
const reason = err instanceof Error ? err.message : String(err);
|
|
123
|
-
throw new
|
|
201
|
+
const reason = sanitizeReason(err instanceof Error ? err.message : String(err), url);
|
|
202
|
+
throw new HttpEmbeddingError(`Embedding request failed (${safeUrl(url)}, batch ${batchIndex}): ${reason}`, { cause: err });
|
|
124
203
|
}
|
|
125
204
|
if (!resp.ok) {
|
|
126
205
|
// resilientFetch already retried 5xx/429; any non-OK response here is
|
|
127
206
|
// a terminal client error (4xx other than 429).
|
|
128
|
-
throw new
|
|
207
|
+
throw new HttpEmbeddingError(`Embedding endpoint returned ${resp.status} (${safeUrl(url)}, batch ${batchIndex})`);
|
|
208
|
+
}
|
|
209
|
+
// A reachable-but-wrong endpoint (e.g. a captive portal or a non-embeddings
|
|
210
|
+
// service) can answer 200 with an HTML/truncated body. Parse inside the
|
|
211
|
+
// typed-error boundary so that lands as an endpoint failure the CLI can
|
|
212
|
+
// classify, not a raw SyntaxError/TypeError on the generic stack-dump path.
|
|
213
|
+
let data;
|
|
214
|
+
try {
|
|
215
|
+
data = (await resp.json());
|
|
216
|
+
}
|
|
217
|
+
catch (err) {
|
|
218
|
+
throw new HttpEmbeddingError(`Embedding endpoint returned an unparseable response (${safeUrl(url)}, batch ${batchIndex})`, { cause: err });
|
|
219
|
+
}
|
|
220
|
+
if (!Array.isArray(data?.data) || !data.data.every(isEmbeddingItem)) {
|
|
221
|
+
throw new HttpEmbeddingError(`Embedding endpoint returned an unexpected response shape (${safeUrl(url)}, batch ${batchIndex})`);
|
|
129
222
|
}
|
|
130
|
-
const data = (await resp.json());
|
|
131
223
|
return data.data;
|
|
132
224
|
};
|
|
133
225
|
/**
|
|
@@ -150,7 +242,7 @@ export const httpEmbed = async (texts) => {
|
|
|
150
242
|
const batchIndex = Math.floor(i / HTTP_BATCH_SIZE);
|
|
151
243
|
const items = await httpEmbedBatch(url, batch, config.model, config.apiKey, batchIndex, config.dimensions);
|
|
152
244
|
if (items.length !== batch.length) {
|
|
153
|
-
throw new
|
|
245
|
+
throw new HttpEmbeddingError(`Embedding endpoint returned ${items.length} vectors for ${batch.length} texts ` +
|
|
154
246
|
`(${safeUrl(url)}, batch ${batchIndex})`);
|
|
155
247
|
}
|
|
156
248
|
for (const item of items) {
|
|
@@ -162,7 +254,7 @@ export const httpEmbed = async (texts) => {
|
|
|
162
254
|
const hint = config.dimensions
|
|
163
255
|
? 'Update GITNEXUS_EMBEDDING_DIMS to match your model output.'
|
|
164
256
|
: `Set GITNEXUS_EMBEDDING_DIMS=${vec.length} to match your model output.`;
|
|
165
|
-
throw new
|
|
257
|
+
throw new HttpEmbeddingError(`Embedding dimension mismatch: endpoint returned ${vec.length}d vector, ` +
|
|
166
258
|
`but expected ${expected}d. ${hint}`);
|
|
167
259
|
}
|
|
168
260
|
allVectors.push(vec);
|
|
@@ -184,7 +276,7 @@ export const httpEmbedQuery = async (text) => {
|
|
|
184
276
|
const url = `${config.baseUrl}/embeddings`;
|
|
185
277
|
const items = await httpEmbedBatch(url, [text], config.model, config.apiKey, 0, config.dimensions);
|
|
186
278
|
if (!items.length) {
|
|
187
|
-
throw new
|
|
279
|
+
throw new HttpEmbeddingError(`Embedding endpoint returned empty response (${safeUrl(url)})`);
|
|
188
280
|
}
|
|
189
281
|
const embedding = items[0].embedding;
|
|
190
282
|
// Same dimension checks as httpEmbed — catch mismatches before they
|
|
@@ -194,7 +286,7 @@ export const httpEmbedQuery = async (text) => {
|
|
|
194
286
|
const hint = config.dimensions
|
|
195
287
|
? 'Update GITNEXUS_EMBEDDING_DIMS to match your model output.'
|
|
196
288
|
: `Set GITNEXUS_EMBEDDING_DIMS=${embedding.length} to match your model output.`;
|
|
197
|
-
throw new
|
|
289
|
+
throw new HttpEmbeddingError(`Embedding dimension mismatch: endpoint returned ${embedding.length}d vector, ` +
|
|
198
290
|
`but expected ${expected}d. ${hint}`);
|
|
199
291
|
}
|
|
200
292
|
return embedding;
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
export type ExtensionLoadErrorKind = 'missing_file' | 'corrupt_file' | 'missing_dependency' | 'unknown';
|
|
2
|
+
export interface ExtensionLoadDiagnosis {
|
|
3
|
+
readonly kind: ExtensionLoadErrorKind;
|
|
4
|
+
/** Actionable, literal-English remedy suited to the class. */
|
|
5
|
+
readonly remedy: string;
|
|
6
|
+
}
|
|
7
|
+
/**
|
|
8
|
+
* On-disk file corruption / wrong-platform. FORCE INSTALL re-downloads.
|
|
9
|
+
* Kept byte-identical to `FILE_CORRUPTION_SIGNATURES` in
|
|
10
|
+
* scripts/install-duckdb-extension.mjs (that `.mjs` cannot import this `.ts`;
|
|
11
|
+
* the duplication is deliberate — the two serve different call sites). Note
|
|
12
|
+
* `/not a valid/i` already covers Windows error 193 ("is not a valid Win32
|
|
13
|
+
* application"), so a truncated Windows download is caught here, before the
|
|
14
|
+
* missing-dependency branch.
|
|
15
|
+
*/
|
|
16
|
+
export declare const FILE_CORRUPTION_SIGNATURES: readonly RegExp[];
|
|
17
|
+
/**
|
|
18
|
+
* Classify a collapsed LadybugDB LOAD error. Order is most-specific-first and is
|
|
19
|
+
* load-bearing: corrupt-file is tested before missing-dependency so a truncated
|
|
20
|
+
* Windows download (error 193, matched by `/not a valid/i`) routes to
|
|
21
|
+
* FORCE-reinstall rather than to the runtime-install remedy.
|
|
22
|
+
*/
|
|
23
|
+
export declare function classifyExtensionLoadError(reason: string | undefined | null): ExtensionLoadDiagnosis;
|
|
24
|
+
/** Well-formedness of the extension binary for the host platform + arch. */
|
|
25
|
+
export type ExtensionBinaryState = 'absent' | 'corrupt' | 'valid' | 'indeterminate';
|
|
26
|
+
/**
|
|
27
|
+
* Pull the extension file path out of lbug's load error. lbug's wrapper is
|
|
28
|
+
* English regardless of OS language — `Failed to load library: {path} which is
|
|
29
|
+
* needed by extension: {name}` (real lbug), or the quoted `Failed to load
|
|
30
|
+
* library '{path}': {reason}` variant — so the path is recoverable in any locale.
|
|
31
|
+
* Only paths ending in `.lbug_extension` are accepted, so a regex misfire can
|
|
32
|
+
* never point the inspector at an arbitrary file.
|
|
33
|
+
*/
|
|
34
|
+
export declare function extractExtensionPath(reason: string | undefined | null): string | null;
|
|
35
|
+
/**
|
|
36
|
+
* A structural verdict on a binary header. `indeterminate` means the probe could
|
|
37
|
+
* not prove validity OR corruption from what it read (e.g. the PE header sits past
|
|
38
|
+
* the BINARY_HEADER_BYTES window) — the caller defers to the string classifier
|
|
39
|
+
* rather than assert a false verdict.
|
|
40
|
+
*/
|
|
41
|
+
type HeaderVerdict = 'valid' | 'corrupt' | 'indeterminate';
|
|
42
|
+
/**
|
|
43
|
+
* Decide whether a binary header is a well-formed shared library for the given
|
|
44
|
+
* platform + architecture — using only the file's structure, no localized text.
|
|
45
|
+
* Pure and injectable (platform/arch as params) so every format+arch combination
|
|
46
|
+
* is unit-testable regardless of the host it runs on.
|
|
47
|
+
*/
|
|
48
|
+
export declare function classifyBinaryHeader(buf: Buffer, bytesRead: number, platform: NodeJS.Platform, arch: string): HeaderVerdict;
|
|
49
|
+
/**
|
|
50
|
+
* Best-effort language-independent inspection of the extension file. Reads the
|
|
51
|
+
* header and classifies it; never throws — a missing file is `absent`, an
|
|
52
|
+
* unreadable one is `indeterminate`.
|
|
53
|
+
*/
|
|
54
|
+
export declare function inspectExtensionBinary(extensionPath: string | null | undefined): ExtensionBinaryState;
|
|
55
|
+
/**
|
|
56
|
+
* Diagnose a LadybugDB load failure, preferring a LANGUAGE-INDEPENDENT structural
|
|
57
|
+
* check of the extension binary over the localized error text:
|
|
58
|
+
* - file absent → missing_file
|
|
59
|
+
* - present but malformed → corrupt_file (bad magic / wrong architecture)
|
|
60
|
+
* - present and well-formed → missing_dependency (a valid binary the loader rejected)
|
|
61
|
+
* The path comes from lbug's own English wrapper, so this holds in any OS display
|
|
62
|
+
* language. When the file cannot be located or read, it falls back to the string
|
|
63
|
+
* classifier (which still carries the language-independent hedged fallback). This
|
|
64
|
+
* is the entry point every surface should call.
|
|
65
|
+
*/
|
|
66
|
+
export declare function diagnoseExtensionLoad(reason: string | undefined | null): ExtensionLoadDiagnosis;
|
|
67
|
+
export {};
|
|
@@ -0,0 +1,320 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Classify a LadybugDB `LOAD EXTENSION` failure into one of four actionable
|
|
3
|
+
* classes and produce an accurate, literal-English remedy.
|
|
4
|
+
*
|
|
5
|
+
* Background (#2374): PR #2375 made the real LadybugDB LOAD error visible
|
|
6
|
+
* (instead of a false "not pre-installed" message). The rc.4 reproduction then
|
|
7
|
+
* showed the remaining defect — on Windows the extension file downloads and
|
|
8
|
+
* INSTALLs fine, but `LoadLibrary` fails with error 126 ("the specified module
|
|
9
|
+
* could not be found" / `找不到指定的模块`) because the extension dynamically
|
|
10
|
+
* imports OpenSSL 3 / MSVC 14 DLLs that ship nowhere. For that class, telling
|
|
11
|
+
* the user to reinstall/redownload is wrong — the file is fine; a *runtime
|
|
12
|
+
* dependency* is missing. This module decides which class an error is so each
|
|
13
|
+
* surface (doctor, --repair-fts, the analyze degrade warning, and
|
|
14
|
+
* ftsDegradedWarning) can emit the right remedy instead of a one-size-fits-all
|
|
15
|
+
* "reinstall over the network".
|
|
16
|
+
*
|
|
17
|
+
* `classifyExtensionLoadError` is pure string logic — no `@ladybugdb/core`
|
|
18
|
+
* import, no filesystem — which keeps `native-check.ts` free of a static lbug
|
|
19
|
+
* dependency. `diagnoseExtensionLoad` layers a LANGUAGE-INDEPENDENT structural
|
|
20
|
+
* check on top: it pulls the extension's file path out of lbug's own (English)
|
|
21
|
+
* wrapper and inspects the binary header directly (PE/ELF/Mach-O magic +
|
|
22
|
+
* architecture), so corrupt-vs-valid is decided by the file itself, not by the
|
|
23
|
+
* localized OS error tail. It reads the file (node:fs core module only, still no
|
|
24
|
+
* lbug) and never throws — any read failure degrades to the string classifier.
|
|
25
|
+
*/
|
|
26
|
+
import { closeSync, openSync, readSync } from 'node:fs';
|
|
27
|
+
/** LadybugDB says the extension file was never installed. INSTALL can heal it. */
|
|
28
|
+
const MISSING_FILE_SIGNATURES = [
|
|
29
|
+
/has not been installed/i,
|
|
30
|
+
/not been installed/i,
|
|
31
|
+
];
|
|
32
|
+
/**
|
|
33
|
+
* On-disk file corruption / wrong-platform. FORCE INSTALL re-downloads.
|
|
34
|
+
* Kept byte-identical to `FILE_CORRUPTION_SIGNATURES` in
|
|
35
|
+
* scripts/install-duckdb-extension.mjs (that `.mjs` cannot import this `.ts`;
|
|
36
|
+
* the duplication is deliberate — the two serve different call sites). Note
|
|
37
|
+
* `/not a valid/i` already covers Windows error 193 ("is not a valid Win32
|
|
38
|
+
* application"), so a truncated Windows download is caught here, before the
|
|
39
|
+
* missing-dependency branch.
|
|
40
|
+
*/
|
|
41
|
+
// Exported so a parity test can assert this stays byte-identical to the copy in
|
|
42
|
+
// scripts/install-duckdb-extension.mjs (that `.mjs` cannot import this `.ts`), #2383 F5b.
|
|
43
|
+
export const FILE_CORRUPTION_SIGNATURES = [
|
|
44
|
+
/invalid elf/i,
|
|
45
|
+
/file too short/i,
|
|
46
|
+
/not a valid/i,
|
|
47
|
+
/bad magic/i,
|
|
48
|
+
/wrong architecture/i,
|
|
49
|
+
/mach-o/i,
|
|
50
|
+
/truncat/i,
|
|
51
|
+
];
|
|
52
|
+
/**
|
|
53
|
+
* A *transitive dependency* of the extension is missing — the file loaded far
|
|
54
|
+
* enough to be found, but a library it needs is absent. Reinstalling the
|
|
55
|
+
* extension is a no-op for this class.
|
|
56
|
+
*
|
|
57
|
+
* WINDOWS CATCH-ALL GUARD (adversarial review): LadybugDB wraps *every* Windows
|
|
58
|
+
* load failure in `Failed to load library … which is needed by extension`, so
|
|
59
|
+
* that generic wrapper must NOT be sufficient — otherwise error 127 (wrong
|
|
60
|
+
* OpenSSL minor / unresolved procedure), 5 (AV/permission lock), and 1114
|
|
61
|
+
* (dependency DllMain failure) would all be mislabeled `missing_dependency` and
|
|
62
|
+
* told to install a runtime, the opposite of their real fix. We key strictly on
|
|
63
|
+
* the specific error-126 tail. Linux/macOS loaders name the missing library
|
|
64
|
+
* directly, so their signals are unambiguous.
|
|
65
|
+
*
|
|
66
|
+
* Localized Windows tails we do not enumerate (French, German, Japanese, …) and
|
|
67
|
+
* mojibake renderings of the Chinese text won't match here — but they still
|
|
68
|
+
* carry lbug's language-independent `Failed to load library` wrapper, so they
|
|
69
|
+
* are caught by the hedged fallback (LOAD_FAILURE_WRAPPER) with a non-committal
|
|
70
|
+
* remedy, never a wrong confident "reinstall" instruction.
|
|
71
|
+
*/
|
|
72
|
+
const WINDOWS_MISSING_DEPENDENCY_SIGNATURES = [
|
|
73
|
+
/找不到指定的模块/,
|
|
74
|
+
/specified module could not be found/i,
|
|
75
|
+
];
|
|
76
|
+
const POSIX_MISSING_DEPENDENCY_SIGNATURES = [
|
|
77
|
+
/cannot open shared object file/i, // Linux ld.so
|
|
78
|
+
/image not found/i, // macOS dyld
|
|
79
|
+
/library not loaded/i, // macOS dyld
|
|
80
|
+
];
|
|
81
|
+
/**
|
|
82
|
+
* LadybugDB's own English wrapper for a dlopen/LoadLibrary failure
|
|
83
|
+
* (extension.cpp: `Failed to load library: {path} which is needed by extension:
|
|
84
|
+
* {name}`). It is emitted for EVERY extension load failure regardless of the OS
|
|
85
|
+
* display language — the only localized part is the OS-error tail after it. So
|
|
86
|
+
* it is the language-independent fallback signal once the specific tails miss: a
|
|
87
|
+
* French/German/Japanese Windows 126 has a localized tail we cannot enumerate,
|
|
88
|
+
* but it still carries this wrapper. See HEDGED_LOAD_FAILURE_REMEDY.
|
|
89
|
+
*/
|
|
90
|
+
const LOAD_FAILURE_WRAPPER = /failed to load library/i;
|
|
91
|
+
const MISSING_FILE_REMEDY = 'The FTS extension is not installed. Re-run with network access and ' +
|
|
92
|
+
'GITNEXUS_LBUG_EXTENSION_INSTALL=auto (or `gitnexus analyze --repair-fts`) to download it.';
|
|
93
|
+
const CORRUPT_FILE_REMEDY = 'The FTS extension file is present but unreadable (corrupt, truncated, or built for another ' +
|
|
94
|
+
'platform). Re-download it with network access and GITNEXUS_LBUG_EXTENSION_INSTALL=auto ' +
|
|
95
|
+
'(`gitnexus analyze --repair-fts`).';
|
|
96
|
+
// Single source of truth for the VC++ runtime-install pointer, shared by the
|
|
97
|
+
// Windows-126 and structural missing-dependency remedies so the name/URL cannot
|
|
98
|
+
// drift between them (#2383 F5).
|
|
99
|
+
const VC_REDIST_INSTALL_HINT = 'the Microsoft Visual C++ 2015-2022 Redistributable (x64) from ' +
|
|
100
|
+
'https://aka.ms/vs/17/release/vc_redist.x64.exe';
|
|
101
|
+
// MSVC-first per DuckDB's canonical answer for this exact error; OpenSSL second.
|
|
102
|
+
const WINDOWS_MISSING_DEPENDENCY_REMEDY = 'The FTS extension is present but a required runtime library is missing (Windows error 126). ' +
|
|
103
|
+
'Reinstalling the extension will NOT help. Install ' +
|
|
104
|
+
VC_REDIST_INSTALL_HINT +
|
|
105
|
+
'; if the error persists, the extension also needs OpenSSL 3 ' +
|
|
106
|
+
'(libcrypto-3-x64.dll / libssl-3-x64.dll) on the DLL search path.';
|
|
107
|
+
const POSIX_MISSING_DEPENDENCY_REMEDY = 'The FTS extension is present but a shared library it depends on could not be loaded (named in ' +
|
|
108
|
+
'the error above). Reinstalling the extension will NOT help — install that library or add it to ' +
|
|
109
|
+
'your loader search path.';
|
|
110
|
+
// Language-independent fallback: we know the extension failed to load, but the
|
|
111
|
+
// OS-error tail is in a locale we did not enumerate, so we cannot say which class
|
|
112
|
+
// it is. Hedge honestly — point at the user's own localized error and give both
|
|
113
|
+
// branches — rather than confidently prescribing the wrong single fix. The clean
|
|
114
|
+
// long-term fix is upstream: have LadybugDB include the numeric GetLastError/errno
|
|
115
|
+
// in the message (as it already does elsewhere), so this becomes a code match.
|
|
116
|
+
const HEDGED_LOAD_FAILURE_REMEDY = 'The FTS extension file was found but could not be loaded — see the "Error:" text above (shown ' +
|
|
117
|
+
"in your system's language). Reinstalling usually will not help. If it names a missing module or " +
|
|
118
|
+
'library, install the required runtime (on Windows: the Microsoft Visual C++ 2015-2022 ' +
|
|
119
|
+
'Redistributable x64 and OpenSSL 3); if it names a corrupt or invalid file, run ' +
|
|
120
|
+
'`gitnexus analyze --repair-fts` to re-download.';
|
|
121
|
+
const UNKNOWN_REMEDY = 'The FTS extension failed to load for an unrecognized reason. Run `gitnexus doctor` for live ' +
|
|
122
|
+
'FTS status and verify the extension file and platform.';
|
|
123
|
+
const matchesAny = (reason, signatures) => signatures.some((re) => re.test(reason));
|
|
124
|
+
/**
|
|
125
|
+
* Classify a collapsed LadybugDB LOAD error. Order is most-specific-first and is
|
|
126
|
+
* load-bearing: corrupt-file is tested before missing-dependency so a truncated
|
|
127
|
+
* Windows download (error 193, matched by `/not a valid/i`) routes to
|
|
128
|
+
* FORCE-reinstall rather than to the runtime-install remedy.
|
|
129
|
+
*/
|
|
130
|
+
export function classifyExtensionLoadError(reason) {
|
|
131
|
+
const text = reason ?? '';
|
|
132
|
+
if (matchesAny(text, MISSING_FILE_SIGNATURES)) {
|
|
133
|
+
return { kind: 'missing_file', remedy: MISSING_FILE_REMEDY };
|
|
134
|
+
}
|
|
135
|
+
if (matchesAny(text, FILE_CORRUPTION_SIGNATURES)) {
|
|
136
|
+
return { kind: 'corrupt_file', remedy: CORRUPT_FILE_REMEDY };
|
|
137
|
+
}
|
|
138
|
+
if (matchesAny(text, WINDOWS_MISSING_DEPENDENCY_SIGNATURES)) {
|
|
139
|
+
return { kind: 'missing_dependency', remedy: WINDOWS_MISSING_DEPENDENCY_REMEDY };
|
|
140
|
+
}
|
|
141
|
+
if (matchesAny(text, POSIX_MISSING_DEPENDENCY_SIGNATURES)) {
|
|
142
|
+
return { kind: 'missing_dependency', remedy: POSIX_MISSING_DEPENDENCY_REMEDY };
|
|
143
|
+
}
|
|
144
|
+
// Language-independent fallback: the extension demonstrably failed to load
|
|
145
|
+
// (lbug's English wrapper is present) but the localized OS tail matched no
|
|
146
|
+
// specific class. Treat as a dependency/runtime load failure with a hedged
|
|
147
|
+
// remedy — strictly better than the generic `unknown` for non-English hosts,
|
|
148
|
+
// and it never prescribes the wrong fix.
|
|
149
|
+
if (LOAD_FAILURE_WRAPPER.test(text)) {
|
|
150
|
+
return { kind: 'missing_dependency', remedy: HEDGED_LOAD_FAILURE_REMEDY };
|
|
151
|
+
}
|
|
152
|
+
return { kind: 'unknown', remedy: UNKNOWN_REMEDY };
|
|
153
|
+
}
|
|
154
|
+
const STRUCTURAL_MISSING_DEPENDENCY_REMEDY = 'The FTS extension file is valid, so the failure is a missing or incompatible runtime dependency, ' +
|
|
155
|
+
'not the extension itself — reinstalling will NOT help. On Windows, install ' +
|
|
156
|
+
VC_REDIST_INSTALL_HINT +
|
|
157
|
+
' and ensure OpenSSL 3 is available; on Linux/macOS install the shared library named in the error above.';
|
|
158
|
+
/**
|
|
159
|
+
* Pull the extension file path out of lbug's load error. lbug's wrapper is
|
|
160
|
+
* English regardless of OS language — `Failed to load library: {path} which is
|
|
161
|
+
* needed by extension: {name}` (real lbug), or the quoted `Failed to load
|
|
162
|
+
* library '{path}': {reason}` variant — so the path is recoverable in any locale.
|
|
163
|
+
* Only paths ending in `.lbug_extension` are accepted, so a regex misfire can
|
|
164
|
+
* never point the inspector at an arbitrary file.
|
|
165
|
+
*/
|
|
166
|
+
export function extractExtensionPath(reason) {
|
|
167
|
+
const text = reason ?? '';
|
|
168
|
+
const m = /failed to load library:?\s*['"]?(.+?\.lbug_extension)/i.exec(text);
|
|
169
|
+
const path = m?.[1]?.trim();
|
|
170
|
+
return path && path.length > 0 ? path : null;
|
|
171
|
+
}
|
|
172
|
+
/** Node `process.arch` → PE `Machine`. Undefined for arches we don't map. */
|
|
173
|
+
const PE_MACHINE = { x64: 0x8664, arm64: 0xaa64 };
|
|
174
|
+
/** Node `process.arch` → ELF `e_machine`. */
|
|
175
|
+
const ELF_MACHINE = { x64: 0x3e, arm64: 0xb7 };
|
|
176
|
+
/** Node `process.arch` → Mach-O `cputype`. */
|
|
177
|
+
const MACHO_CPUTYPE = { x64: 0x01000007, arm64: 0x0100000c };
|
|
178
|
+
function classifyPE(buf, bytesRead, arch) {
|
|
179
|
+
if (bytesRead < 0x40 || buf[0] !== 0x4d || buf[1] !== 0x5a)
|
|
180
|
+
return 'corrupt'; // 'MZ'
|
|
181
|
+
const peOffset = buf.readUInt32LE(0x3c);
|
|
182
|
+
// The PE header (e_lfanew) points beyond what we read. A large-DOS-stub VALID PE
|
|
183
|
+
// and a garbage e_lfanew are indistinguishable from here, so don't claim 'corrupt'
|
|
184
|
+
// — defer to the loader's own report (#2383 F1-secondary).
|
|
185
|
+
if (peOffset + 6 > bytesRead)
|
|
186
|
+
return 'indeterminate';
|
|
187
|
+
const isPE = buf[peOffset] === 0x50 &&
|
|
188
|
+
buf[peOffset + 1] === 0x45 &&
|
|
189
|
+
buf[peOffset + 2] === 0 &&
|
|
190
|
+
buf[peOffset + 3] === 0;
|
|
191
|
+
if (!isPE)
|
|
192
|
+
return 'corrupt';
|
|
193
|
+
const expected = PE_MACHINE[arch];
|
|
194
|
+
if (expected === undefined)
|
|
195
|
+
return 'valid'; // arch we don't map: don't claim corrupt
|
|
196
|
+
return buf.readUInt16LE(peOffset + 4) === expected ? 'valid' : 'corrupt';
|
|
197
|
+
}
|
|
198
|
+
function classifyELF(buf, bytesRead, arch) {
|
|
199
|
+
if (bytesRead < 20)
|
|
200
|
+
return 'corrupt';
|
|
201
|
+
if (buf[0] !== 0x7f || buf[1] !== 0x45 || buf[2] !== 0x4c || buf[3] !== 0x46)
|
|
202
|
+
return 'corrupt'; // 0x7F ELF
|
|
203
|
+
const littleEndian = buf[5] === 1; // EI_DATA
|
|
204
|
+
const eMachine = littleEndian ? buf.readUInt16LE(18) : buf.readUInt16BE(18);
|
|
205
|
+
const expected = ELF_MACHINE[arch];
|
|
206
|
+
if (expected === undefined)
|
|
207
|
+
return 'valid';
|
|
208
|
+
return eMachine === expected ? 'valid' : 'corrupt';
|
|
209
|
+
}
|
|
210
|
+
function classifyMachO(buf, bytesRead, arch) {
|
|
211
|
+
if (bytesRead < 8)
|
|
212
|
+
return 'corrupt';
|
|
213
|
+
const magicLE = buf.readUInt32LE(0);
|
|
214
|
+
const magicBE = buf.readUInt32BE(0);
|
|
215
|
+
// Universal ("fat") binary — assume it carries the host slice.
|
|
216
|
+
if (magicBE === 0xcafebabe || magicLE === 0xcafebabe)
|
|
217
|
+
return 'valid';
|
|
218
|
+
const thin = magicLE === 0xfeedfacf || magicLE === 0xfeedface;
|
|
219
|
+
const thinSwapped = magicBE === 0xfeedfacf || magicBE === 0xfeedface;
|
|
220
|
+
if (!thin && !thinSwapped)
|
|
221
|
+
return 'corrupt';
|
|
222
|
+
const cpuType = thin ? buf.readUInt32LE(4) : buf.readUInt32BE(4);
|
|
223
|
+
const expected = MACHO_CPUTYPE[arch];
|
|
224
|
+
if (expected === undefined)
|
|
225
|
+
return 'valid';
|
|
226
|
+
return cpuType === expected ? 'valid' : 'corrupt';
|
|
227
|
+
}
|
|
228
|
+
/**
|
|
229
|
+
* Decide whether a binary header is a well-formed shared library for the given
|
|
230
|
+
* platform + architecture — using only the file's structure, no localized text.
|
|
231
|
+
* Pure and injectable (platform/arch as params) so every format+arch combination
|
|
232
|
+
* is unit-testable regardless of the host it runs on.
|
|
233
|
+
*/
|
|
234
|
+
export function classifyBinaryHeader(buf, bytesRead, platform, arch) {
|
|
235
|
+
if (platform === 'win32')
|
|
236
|
+
return classifyPE(buf, bytesRead, arch);
|
|
237
|
+
if (platform === 'linux')
|
|
238
|
+
return classifyELF(buf, bytesRead, arch);
|
|
239
|
+
if (platform === 'darwin')
|
|
240
|
+
return classifyMachO(buf, bytesRead, arch);
|
|
241
|
+
return 'valid'; // unknown host: never claim corrupt
|
|
242
|
+
}
|
|
243
|
+
const BINARY_HEADER_BYTES = 4096;
|
|
244
|
+
/**
|
|
245
|
+
* Best-effort language-independent inspection of the extension file. Reads the
|
|
246
|
+
* header and classifies it; never throws — a missing file is `absent`, an
|
|
247
|
+
* unreadable one is `indeterminate`.
|
|
248
|
+
*/
|
|
249
|
+
export function inspectExtensionBinary(extensionPath) {
|
|
250
|
+
if (!extensionPath)
|
|
251
|
+
return 'indeterminate';
|
|
252
|
+
let fd;
|
|
253
|
+
try {
|
|
254
|
+
fd = openSync(extensionPath, 'r');
|
|
255
|
+
}
|
|
256
|
+
catch (err) {
|
|
257
|
+
return err?.code === 'ENOENT' ? 'absent' : 'indeterminate';
|
|
258
|
+
}
|
|
259
|
+
try {
|
|
260
|
+
const buf = Buffer.alloc(BINARY_HEADER_BYTES);
|
|
261
|
+
const bytesRead = readSync(fd, buf, 0, BINARY_HEADER_BYTES, 0);
|
|
262
|
+
return classifyBinaryHeader(buf, bytesRead, process.platform, process.arch);
|
|
263
|
+
}
|
|
264
|
+
catch {
|
|
265
|
+
return 'indeterminate';
|
|
266
|
+
}
|
|
267
|
+
finally {
|
|
268
|
+
try {
|
|
269
|
+
closeSync(fd);
|
|
270
|
+
}
|
|
271
|
+
catch {
|
|
272
|
+
/* closing the probe fd must never surface */
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
/**
|
|
277
|
+
* Diagnose a LadybugDB load failure, preferring a LANGUAGE-INDEPENDENT structural
|
|
278
|
+
* check of the extension binary over the localized error text:
|
|
279
|
+
* - file absent → missing_file
|
|
280
|
+
* - present but malformed → corrupt_file (bad magic / wrong architecture)
|
|
281
|
+
* - present and well-formed → missing_dependency (a valid binary the loader rejected)
|
|
282
|
+
* The path comes from lbug's own English wrapper, so this holds in any OS display
|
|
283
|
+
* language. When the file cannot be located or read, it falls back to the string
|
|
284
|
+
* classifier (which still carries the language-independent hedged fallback). This
|
|
285
|
+
* is the entry point every surface should call.
|
|
286
|
+
*/
|
|
287
|
+
export function diagnoseExtensionLoad(reason) {
|
|
288
|
+
const text = reason ?? '';
|
|
289
|
+
const stringResult = classifyExtensionLoadError(text);
|
|
290
|
+
const fileState = inspectExtensionBinary(extractExtensionPath(text));
|
|
291
|
+
if (fileState === 'corrupt') {
|
|
292
|
+
return { kind: 'corrupt_file', remedy: CORRUPT_FILE_REMEDY };
|
|
293
|
+
}
|
|
294
|
+
if (fileState === 'valid') {
|
|
295
|
+
// The structural probe only inspects the first BINARY_HEADER_BYTES, so a file
|
|
296
|
+
// truncated AFTER its header still reads 'valid'. When the loader itself reported
|
|
297
|
+
// corruption (e.g. "file too short" / Windows error 193 "not a valid Win32
|
|
298
|
+
// application"), that whole-file verdict is stronger evidence than an intact-looking
|
|
299
|
+
// header — honor it and route to re-download, not a runtime-dependency install (#2383
|
|
300
|
+
// F1). Localized corrupt tails classify as hedged missing_dependency (not
|
|
301
|
+
// corrupt_file), so they still fall through to the dependency remedy below.
|
|
302
|
+
if (stringResult.kind === 'corrupt_file') {
|
|
303
|
+
return stringResult;
|
|
304
|
+
}
|
|
305
|
+
// A structurally sound binary that still failed to load ⇒ a dependency/runtime
|
|
306
|
+
// problem, decided WITHOUT the localized tail. Keep the string classifier's
|
|
307
|
+
// sharper remedy when it recognized the specific case (e.g. English 126).
|
|
308
|
+
const remedy = stringResult.kind === 'missing_dependency'
|
|
309
|
+
? stringResult.remedy
|
|
310
|
+
: STRUCTURAL_MISSING_DEPENDENCY_REMEDY;
|
|
311
|
+
return { kind: 'missing_dependency', remedy };
|
|
312
|
+
}
|
|
313
|
+
// 'absent' or 'indeterminate' → no positive structural evidence, so defer to the
|
|
314
|
+
// string classifier. Note a real never-installed extension has NO path in its
|
|
315
|
+
// reason (lbug says "has not been installed"), so it lands here via
|
|
316
|
+
// 'indeterminate' and the string classifier reports missing_file correctly; a
|
|
317
|
+
// path that lbug named but that is now gone (stale/racy) is better judged by
|
|
318
|
+
// what lbug actually reported than by re-deriving from disk.
|
|
319
|
+
return stringResult;
|
|
320
|
+
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { type ExtensionLoadDiagnosis } from './extension-load-error.js';
|
|
1
2
|
/**
|
|
2
3
|
* Lifecycle policy for an optional DuckDB extension.
|
|
3
4
|
*
|
|
@@ -20,6 +21,12 @@ export interface ExtensionCapability {
|
|
|
20
21
|
loaded: boolean;
|
|
21
22
|
/** Human-readable reason when `loaded` is false. */
|
|
22
23
|
reason?: string;
|
|
24
|
+
/**
|
|
25
|
+
* Classified diagnosis of `reason`, computed ONCE at mark-unavailable time so
|
|
26
|
+
* per-request surfaces (ftsDegradedWarning on /api/search + MCP query) read the
|
|
27
|
+
* cached remedy instead of re-inspecting the extension file on every call (#2383 F3).
|
|
28
|
+
*/
|
|
29
|
+
diagnosis?: ExtensionLoadDiagnosis;
|
|
23
30
|
}
|
|
24
31
|
/** Per-call overrides applied on top of `ExtensionManager` defaults. */
|
|
25
32
|
export interface ExtensionEnsureOptions {
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { spawn } from 'child_process';
|
|
2
2
|
import { fileURLToPath } from 'node:url';
|
|
3
3
|
import { LBUG_MAX_DB_SIZE } from './lbug-config.js';
|
|
4
|
+
import { diagnoseExtensionLoad } from './extension-load-error.js';
|
|
4
5
|
import { logger } from '../logger.js';
|
|
5
6
|
const DEFAULT_EXTENSION_INSTALL_TIMEOUT_MS = 15_000;
|
|
6
7
|
const EXTENSION_NAME_PATTERN = /^[A-Za-z][A-Za-z0-9_]*$/;
|
|
@@ -208,7 +209,14 @@ export class ExtensionManager {
|
|
|
208
209
|
this.capabilities.set(name, { name, loaded: true });
|
|
209
210
|
}
|
|
210
211
|
markUnavailable(name, label, reason, warn) {
|
|
211
|
-
|
|
212
|
+
// Classify once here (the single load-failure sink, run per Database not per
|
|
213
|
+
// request) so the hot per-request warning path does no file I/O (#2383 F3).
|
|
214
|
+
this.capabilities.set(name, {
|
|
215
|
+
name,
|
|
216
|
+
loaded: false,
|
|
217
|
+
reason,
|
|
218
|
+
diagnosis: diagnoseExtensionLoad(reason),
|
|
219
|
+
});
|
|
212
220
|
const key = `${name}:${reason}`;
|
|
213
221
|
if (this.warnedKeys.has(key))
|
|
214
222
|
return;
|
|
@@ -13,7 +13,7 @@ import { streamAllCSVsToDisk } from './csv-generator.js';
|
|
|
13
13
|
import { getNodeLabel as deriveNodeLabel } from './rel-pair-routing.js';
|
|
14
14
|
import { extensionManager } from './extension-loader.js';
|
|
15
15
|
import { classifyDeleteAllError, closeLbugConnection, isDbBusyError, isOpenRetryExhausted, isWalCorruptionError, openLbugConnection, toNativeSafePath, resolveNativeSafeStorageDir, WAL_RECOVERY_SUGGESTION, waitForWindowsHandleRelease, } from './lbug-config.js';
|
|
16
|
-
import { finalizeLbugSidecarsAfterClose,
|
|
16
|
+
import { finalizeLbugSidecarsAfterClose, guardWalQuarantine, isMissingShadowSidecarError, isReadOnlyShadowReplayError, preflightLbugSidecars, quarantineWalForMissingShadow, renameFailureMessage, shadowSidecarRecoveryMessage, } from './sidecar-recovery.js';
|
|
17
17
|
import { isVectorExtensionSupportedByPlatform } from '../platform/capabilities.js';
|
|
18
18
|
import { logger } from '../logger.js';
|
|
19
19
|
/**
|
|
@@ -409,23 +409,14 @@ const queryAndDrain = async (targetConn, cypher) => {
|
|
|
409
409
|
};
|
|
410
410
|
const READ_ONLY_SHADOW_REPLAY_PROBE = 'MATCH (n) RETURN n LIMIT 1';
|
|
411
411
|
/**
|
|
412
|
-
*
|
|
413
|
-
*
|
|
414
|
-
*
|
|
415
|
-
*
|
|
416
|
-
*
|
|
417
|
-
* Throws shadowSidecarRecoveryMessage immediately when the WAL is large,
|
|
418
|
-
* preserving the uncheckpointed pages for explicit operator recovery.
|
|
419
|
-
* Returns silently when the WAL is absent, tiny, or in any other state
|
|
420
|
-
* where the existing recovery path is safe to proceed.
|
|
412
|
+
* Serve-side entry to the shared WAL-quarantine safety gate. Refuses (throws)
|
|
413
|
+
* when the `.shadow` is present on disk or the orphan WAL is too large to
|
|
414
|
+
* safely discard; returns silently otherwise. The policy itself lives in
|
|
415
|
+
* `guardWalQuarantine` (sidecar-recovery.ts) so serve and the MCP pool share
|
|
416
|
+
* one source of truth (PR #1747 review D2; issue #2382 review, Finding B).
|
|
421
417
|
*/
|
|
422
418
|
const refuseLargeWalQuarantine = async (dbPath, mode, triggeringErr) => {
|
|
423
|
-
|
|
424
|
-
if (state.kind === 'orphan-wal') {
|
|
425
|
-
logger.warn(`GitNexus: refusing to quarantine large WAL (${state.walBytes} bytes) at ${dbPath}.wal during ${mode} recovery; ` +
|
|
426
|
-
'manual recovery required — run `gitnexus analyze --force <repo-path> --index-only`.');
|
|
427
|
-
throw new Error(shadowSidecarRecoveryMessage(dbPath, triggeringErr));
|
|
428
|
-
}
|
|
419
|
+
await guardWalQuarantine(dbPath, mode, triggeringErr, logger);
|
|
429
420
|
};
|
|
430
421
|
const reopenReadOnlyAfterMissingShadow = async (dbPath, err) => {
|
|
431
422
|
await refuseLargeWalQuarantine(dbPath, 'read-only', err);
|
|
@@ -19,7 +19,7 @@ import lbug from '@ladybugdb/core';
|
|
|
19
19
|
import { isReadOnlyDbError, loadFTSExtension } from './lbug-adapter.js';
|
|
20
20
|
import { closeQueryResults } from './query-result-utils.js';
|
|
21
21
|
import { createLbugDatabase, isWalCorruptionError, toNativeSafePath, WAL_RECOVERY_SUGGESTION, } from './lbug-config.js';
|
|
22
|
-
import { isMissingFsError, isMissingShadowSidecarError, isReadOnlyShadowReplayError, preflightLbugSidecars, quarantineWalForMissingShadow, renameFailureMessage, statIfExists, } from './sidecar-recovery.js';
|
|
22
|
+
import { guardWalQuarantine, isMissingFsError, isMissingShadowSidecarError, isReadOnlyShadowReplayError, preflightLbugSidecars, quarantineWalForMissingShadow, renameFailureMessage, statIfExists, } from './sidecar-recovery.js';
|
|
23
23
|
const pool = new Map();
|
|
24
24
|
const poolCloseListeners = new Set();
|
|
25
25
|
/**
|
|
@@ -349,6 +349,12 @@ const poolSidecarLogger = {
|
|
|
349
349
|
* See plan: docs/plans/2026-05-21-001-fix-pr-1747-quarantine-enoent-and-large-wal-plan.md (U2)
|
|
350
350
|
*/
|
|
351
351
|
async function tryQuarantineForMissingShadow(dbPath, opts) {
|
|
352
|
+
// Refuse (throw) before renaming a live WAL when the shadow is present on
|
|
353
|
+
// disk or the orphan WAL is too large — parity with the serve path's
|
|
354
|
+
// refuseLargeWalQuarantine (issue #2382 review, Finding B). Kept OUTSIDE the
|
|
355
|
+
// try so the actionable recovery message propagates to the MCP caller rather
|
|
356
|
+
// than being re-wrapped as a rename failure.
|
|
357
|
+
await guardWalQuarantine(dbPath, opts.reason, opts.err, poolSidecarLogger);
|
|
352
358
|
try {
|
|
353
359
|
const quarantinePath = await quarantineWalForMissingShadow(dbPath, {
|
|
354
360
|
logger: poolSidecarLogger,
|
|
@@ -396,6 +402,7 @@ async function replayShadowPagesWithWritableOpen(dbPath) {
|
|
|
396
402
|
if (isMissingShadowSidecarError(err)) {
|
|
397
403
|
await tryQuarantineForMissingShadow(dbPath, {
|
|
398
404
|
reason: 'pool writable replay recovery',
|
|
405
|
+
err,
|
|
399
406
|
});
|
|
400
407
|
return;
|
|
401
408
|
}
|
|
@@ -429,6 +436,7 @@ async function openReadOnlyDatabase(dbPath) {
|
|
|
429
436
|
db = undefined;
|
|
430
437
|
await tryQuarantineForMissingShadow(dbPath, {
|
|
431
438
|
reason: 'pool read-only recovery',
|
|
439
|
+
err,
|
|
432
440
|
});
|
|
433
441
|
await preflightLbugSidecars(dbPath, {
|
|
434
442
|
mode: 'read-only',
|
|
@@ -559,6 +567,7 @@ async function doInitLbug(repoId, dbPath) {
|
|
|
559
567
|
}
|
|
560
568
|
}
|
|
561
569
|
if (lastError.message.startsWith('LadybugDB checkpoint sidecar is missing') ||
|
|
570
|
+
lastError.message.startsWith('LadybugDB checkpoint sidecar is present but unreachable') ||
|
|
562
571
|
lastError.message.startsWith('GitNexus could not move the LadybugDB WAL sidecar') ||
|
|
563
572
|
isMissingShadowSidecarError(lastError)) {
|
|
564
573
|
throw lastError;
|
|
@@ -32,6 +32,16 @@ export declare const statIfExists: (filePath: string) => Promise<{
|
|
|
32
32
|
export declare const isMissingShadowSidecarError: (err: unknown) => boolean;
|
|
33
33
|
export declare const isReadOnlyShadowReplayError: (err: unknown) => boolean;
|
|
34
34
|
export declare const shadowSidecarRecoveryMessage: (dbPath: string, err: unknown) => string;
|
|
35
|
+
/**
|
|
36
|
+
* Actionable message for the case where LadybugDB reports a "missing shadow"
|
|
37
|
+
* but `inspectLbugSidecars` finds the `.shadow` PRESENT on disk — the open
|
|
38
|
+
* failed on path reachability or a lock, not a genuinely-missing sidecar (issue
|
|
39
|
+
* #2382 review, S2). Unlike `shadowSidecarRecoveryMessage` it does NOT tell the
|
|
40
|
+
* operator to rebuild the index (the remedy is fixing the lock/path). Keeps the
|
|
41
|
+
* `Original error:` tail so downstream `isMissingShadowSidecarError` recognition
|
|
42
|
+
* still matches the wrapped error.
|
|
43
|
+
*/
|
|
44
|
+
export declare const presentShadowUnreachableMessage: (dbPath: string, err: unknown) => string;
|
|
35
45
|
export declare const isPermissionRenameError: (err: unknown) => boolean;
|
|
36
46
|
/**
|
|
37
47
|
* Classify a failure surfaced by quarantine rename into an actionable user-facing
|
|
@@ -50,6 +60,28 @@ export declare const isPermissionRenameError: (err: unknown) => boolean;
|
|
|
50
60
|
*/
|
|
51
61
|
export declare const renameFailureMessage: (dbPath: string, err: unknown) => string;
|
|
52
62
|
export declare function inspectLbugSidecars(dbPath: string): Promise<LbugSidecarState>;
|
|
63
|
+
/**
|
|
64
|
+
* Reject the WAL-quarantine path when discarding the WAL would be unsafe or
|
|
65
|
+
* wrong. Shared by every reactive missing-shadow recovery consumer — serve (via
|
|
66
|
+
* lbug-adapter's `refuseLargeWalQuarantine`) and the MCP/wiki/augmentation pool
|
|
67
|
+
* (via pool-adapter's `tryQuarantineForMissingShadow`) — so the quarantine
|
|
68
|
+
* safety policy has a single source of truth (issue #2382 review, Finding B).
|
|
69
|
+
*
|
|
70
|
+
* 1. `wal-with-shadow` — the `.shadow` sidecar is PRESENT on disk. A
|
|
71
|
+
* "missing shadow" error alongside a present shadow means the open failed
|
|
72
|
+
* on path reachability or a lock (the #1811 non-ASCII path-garble on
|
|
73
|
+
* Windows), not a genuinely-missing shadow; quarantining would move a live
|
|
74
|
+
* WAL sitting next to its shadow — data loss.
|
|
75
|
+
* 2. `orphan-wal` — the orphan WAL is too large to safely discard
|
|
76
|
+
* (>TINY_ORPHAN_WAL_BYTES); preserve the uncheckpointed pages for explicit
|
|
77
|
+
* operator recovery.
|
|
78
|
+
*
|
|
79
|
+
* Throws `shadowSidecarRecoveryMessage` in either case. Returns silently only
|
|
80
|
+
* when the shadow is absent AND the WAL is absent or tiny — the states where
|
|
81
|
+
* the existing recovery path is safe to proceed. `mode` is a label used only in
|
|
82
|
+
* the warning text (e.g. 'read-only', 'writable', 'pool read-only recovery').
|
|
83
|
+
*/
|
|
84
|
+
export declare const guardWalQuarantine: (dbPath: string, mode: string, triggeringErr: unknown, logger: SidecarRecoveryLogger) => Promise<void>;
|
|
53
85
|
export declare function quarantineWalForMissingShadow(dbPath: string, options: {
|
|
54
86
|
logger: SidecarRecoveryLogger;
|
|
55
87
|
level?: 'debug' | 'info' | 'warn';
|
|
@@ -87,16 +87,47 @@ const warnOnce = (logger, key, message) => {
|
|
|
87
87
|
logger.warn(`${message} (${ordinal(next)} occurrence of this condition)`);
|
|
88
88
|
};
|
|
89
89
|
// LADYBUGDB-CONTRACT: matches @ladybugdb/core ^0.18.0 native error text.
|
|
90
|
-
// When bumping LadybugDB, re-validate this
|
|
90
|
+
// When bumping LadybugDB, re-validate this against the new error format
|
|
91
91
|
// — `git grep "LADYBUGDB-CONTRACT"` enumerates every version-coupled spot.
|
|
92
|
-
//
|
|
93
|
-
//
|
|
94
|
-
//
|
|
95
|
-
//
|
|
96
|
-
//
|
|
92
|
+
//
|
|
93
|
+
// Two native formats reach here for a genuinely-missing shadow sidecar:
|
|
94
|
+
// POSIX: `Cannot open file <path>.shadow: No such file or directory`
|
|
95
|
+
// Windows: `Cannot open file. path: <path>.shadow - Error 2: <system text>`
|
|
96
|
+
// Windows OS text is localized on non-English installs (issue #2382 was filed
|
|
97
|
+
// from a non-English Windows), so we key on the locale-invariant Win32 code
|
|
98
|
+
// (2 = ERROR_FILE_NOT_FOUND), NOT the English phrase. The code is matched only
|
|
99
|
+
// in the reason AFTER the LAST `.shadow` token (the real failing sidecar; the
|
|
100
|
+
// reason text never contains `.shadow`), so a repo *path* containing e.g.
|
|
101
|
+
// `\error 2\` — even under a `.shadow`-suffixed parent directory — cannot trip
|
|
102
|
+
// it. Deliberate exclusions:
|
|
103
|
+
// - `Error 3` (ERROR_PATH_NOT_FOUND): the #1811 non-ASCII path-garble
|
|
104
|
+
// artifact (see lbug-config.ts) where the shadow is PRESENT on disk;
|
|
105
|
+
// treating it as missing would quarantine a live WAL — data loss.
|
|
106
|
+
// - `Error 5` / `Error 32` / POSIX `Permission denied`: present-but-locked;
|
|
107
|
+
// handled as permission/lock classes, must not quarantine.
|
|
108
|
+
// The quarantine path adds a present-shadow disk check as a belt (see
|
|
109
|
+
// refuseLargeWalQuarantine in lbug-adapter.ts).
|
|
110
|
+
//
|
|
111
|
+
// The Windows branch is derived from the issue #2382 reported string, not a
|
|
112
|
+
// self-produced live crash; unit/consumer tests inject that same string, so
|
|
113
|
+
// GREEN TESTS DO NOT PROVE the byte-exact 0.18.0 Windows format — confirm
|
|
114
|
+
// against a real Windows run before closing #2382.
|
|
97
115
|
export const isMissingShadowSidecarError = (err) => {
|
|
98
116
|
const msg = err instanceof Error ? err.message : String(err);
|
|
99
|
-
|
|
117
|
+
if (!/cannot open file/i.test(msg))
|
|
118
|
+
return false;
|
|
119
|
+
// Anchor on the LAST `.shadow`, not the first: LadybugDB names the failing
|
|
120
|
+
// sidecar as the final `.shadow` token and its reason text (POSIX
|
|
121
|
+
// `: No such file or directory` / Windows ` - Error N: ...`) never contains
|
|
122
|
+
// `.shadow`. Slicing from the last match isolates the true reason, so an
|
|
123
|
+
// earlier `.shadow`-suffixed path segment (e.g. a `branch=subdir` directory
|
|
124
|
+
// like `snap.shadow\`) can't shift the anchor and let a path-embedded
|
|
125
|
+
// `error 2` be read as the Win32 code (issue #2382 review, Finding A).
|
|
126
|
+
const lastShadow = [...msg.matchAll(/\.shadow\b/gi)].at(-1);
|
|
127
|
+
if (lastShadow?.index === undefined)
|
|
128
|
+
return false;
|
|
129
|
+
const reason = msg.slice(lastShadow.index);
|
|
130
|
+
return /no such file or directory/i.test(reason) || /\berror\s+2\b/i.test(reason);
|
|
100
131
|
};
|
|
101
132
|
// LADYBUGDB-CONTRACT: matches @ladybugdb/core ^0.18.0 native error text.
|
|
102
133
|
// When bumping LadybugDB, re-validate this regex against the new error format
|
|
@@ -114,6 +145,24 @@ export const shadowSidecarRecoveryMessage = (dbPath, err) => {
|
|
|
114
145
|
'Rebuild the index with `gitnexus analyze --force <repo-path> --index-only` and restart `gitnexus serve`.' +
|
|
115
146
|
`\n Original error: ${msg.slice(0, 200)}`);
|
|
116
147
|
};
|
|
148
|
+
/**
|
|
149
|
+
* Actionable message for the case where LadybugDB reports a "missing shadow"
|
|
150
|
+
* but `inspectLbugSidecars` finds the `.shadow` PRESENT on disk — the open
|
|
151
|
+
* failed on path reachability or a lock, not a genuinely-missing sidecar (issue
|
|
152
|
+
* #2382 review, S2). Unlike `shadowSidecarRecoveryMessage` it does NOT tell the
|
|
153
|
+
* operator to rebuild the index (the remedy is fixing the lock/path). Keeps the
|
|
154
|
+
* `Original error:` tail so downstream `isMissingShadowSidecarError` recognition
|
|
155
|
+
* still matches the wrapped error.
|
|
156
|
+
*/
|
|
157
|
+
export const presentShadowUnreachableMessage = (dbPath, err) => {
|
|
158
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
159
|
+
return (`LadybugDB checkpoint sidecar is present but unreachable for ${dbPath}. ` +
|
|
160
|
+
'The .shadow file is on disk, so the open likely failed on path reachability or a file lock ' +
|
|
161
|
+
'(antivirus, another process holding a handle, or a non-ASCII path) rather than a missing sidecar. ' +
|
|
162
|
+
'Check filesystem access and locks; only run `gitnexus analyze --force <repo-path> --index-only` ' +
|
|
163
|
+
'if the index is genuinely broken.' +
|
|
164
|
+
`\n Original error: ${msg.slice(0, 200)}`);
|
|
165
|
+
};
|
|
117
166
|
const PERMISSION_RENAME_CODES = new Set(['EACCES', 'EPERM', 'EBUSY']);
|
|
118
167
|
export const isPermissionRenameError = (err) => {
|
|
119
168
|
const code = err?.code;
|
|
@@ -165,6 +214,41 @@ export async function inspectLbugSidecars(dbPath) {
|
|
|
165
214
|
}
|
|
166
215
|
return { kind: 'clean', dbPath };
|
|
167
216
|
}
|
|
217
|
+
/**
|
|
218
|
+
* Reject the WAL-quarantine path when discarding the WAL would be unsafe or
|
|
219
|
+
* wrong. Shared by every reactive missing-shadow recovery consumer — serve (via
|
|
220
|
+
* lbug-adapter's `refuseLargeWalQuarantine`) and the MCP/wiki/augmentation pool
|
|
221
|
+
* (via pool-adapter's `tryQuarantineForMissingShadow`) — so the quarantine
|
|
222
|
+
* safety policy has a single source of truth (issue #2382 review, Finding B).
|
|
223
|
+
*
|
|
224
|
+
* 1. `wal-with-shadow` — the `.shadow` sidecar is PRESENT on disk. A
|
|
225
|
+
* "missing shadow" error alongside a present shadow means the open failed
|
|
226
|
+
* on path reachability or a lock (the #1811 non-ASCII path-garble on
|
|
227
|
+
* Windows), not a genuinely-missing shadow; quarantining would move a live
|
|
228
|
+
* WAL sitting next to its shadow — data loss.
|
|
229
|
+
* 2. `orphan-wal` — the orphan WAL is too large to safely discard
|
|
230
|
+
* (>TINY_ORPHAN_WAL_BYTES); preserve the uncheckpointed pages for explicit
|
|
231
|
+
* operator recovery.
|
|
232
|
+
*
|
|
233
|
+
* Throws `shadowSidecarRecoveryMessage` in either case. Returns silently only
|
|
234
|
+
* when the shadow is absent AND the WAL is absent or tiny — the states where
|
|
235
|
+
* the existing recovery path is safe to proceed. `mode` is a label used only in
|
|
236
|
+
* the warning text (e.g. 'read-only', 'writable', 'pool read-only recovery').
|
|
237
|
+
*/
|
|
238
|
+
export const guardWalQuarantine = async (dbPath, mode, triggeringErr, logger) => {
|
|
239
|
+
const state = await inspectLbugSidecars(dbPath);
|
|
240
|
+
if (state.kind === 'wal-with-shadow') {
|
|
241
|
+
warnOnce(logger, `${dbPath}:present-shadow-refuse:${mode}`, `GitNexus: refusing to quarantine WAL at ${dbPath}.wal during ${mode} recovery — ` +
|
|
242
|
+
'the .shadow sidecar is present on disk, so the open likely failed on path reachability or a lock ' +
|
|
243
|
+
'rather than a missing shadow. Run `gitnexus analyze --force <repo-path> --index-only` if the index is genuinely broken.');
|
|
244
|
+
throw new Error(presentShadowUnreachableMessage(dbPath, triggeringErr));
|
|
245
|
+
}
|
|
246
|
+
if (state.kind === 'orphan-wal') {
|
|
247
|
+
warnOnce(logger, `${dbPath}:large-wal-refuse:${mode}`, `GitNexus: refusing to quarantine large WAL (${state.walBytes} bytes) at ${dbPath}.wal during ${mode} recovery; ` +
|
|
248
|
+
'manual recovery required — run `gitnexus analyze --force <repo-path> --index-only`.');
|
|
249
|
+
throw new Error(shadowSidecarRecoveryMessage(dbPath, triggeringErr));
|
|
250
|
+
}
|
|
251
|
+
};
|
|
168
252
|
export async function quarantineWalForMissingShadow(dbPath, options) {
|
|
169
253
|
const walPath = `${dbPath}.wal`;
|
|
170
254
|
const quarantinePath = `${walPath}.missing-shadow.${Date.now()}-${Math.random()
|
package/dist/core/run-analyze.js
CHANGED
|
@@ -17,6 +17,7 @@ import { initLbug, loadGraphToLbug, getLbugStats, executeQuery, executeWithReuse
|
|
|
17
17
|
import { createSearchFTSIndexes, initialiseSearchFTSStemmer, verifySearchFTSIndexes, } from './search/fts-indexes.js';
|
|
18
18
|
import { cjkSegmentationModeMismatch, getSearchFTSCjkSegmentation, initialiseSearchFTSCjkSegmentation, } from './search/cjk-segmentation.js';
|
|
19
19
|
import { getExtensionCapabilities, resolveAnalyzeInstallPolicy } from './lbug/extension-loader.js';
|
|
20
|
+
import { diagnoseExtensionLoad } from './lbug/extension-load-error.js';
|
|
20
21
|
import { startWalCheckpointDriver, } from './lbug/wal-checkpoint-driver.js';
|
|
21
22
|
import { getStoragePaths, resolveBranchPlacement, saveMeta, loadMeta, ensureGitNexusIgnored, registerRepo, adoptFlatBranchLabel, isReadOnlyFilesystemError, isRepoRegistered, cleanupOldKuzuFiles, reconcileMetadataFiles, isMissingFilesystemError, INDEX_METADATA_FILE, INCREMENTAL_SCHEMA_VERSION, } from '../storage/repo-manager.js';
|
|
22
23
|
import { DEFAULT_PDG_MAX_FUNCTION_LINES } from './ingestion/cfg/collect.js';
|
|
@@ -41,7 +42,11 @@ import { STALE_HASH_SENTINEL } from './lbug/schema.js';
|
|
|
41
42
|
* a full analyze. Kept as a named constant so the env-var/command guidance
|
|
42
43
|
* stays in one place (mirrors the VECTOR message in embedding-pipeline.ts).
|
|
43
44
|
*/
|
|
44
|
-
|
|
45
|
+
// Class-neutral lead, reused for the missing-dependency degrade path (#2383 F2):
|
|
46
|
+
// its remedy already explains that reinstalling will NOT help, so appending the
|
|
47
|
+
// generic "install with network access" tail below would contradict it.
|
|
48
|
+
const FTS_UNAVAILABLE_LEAD = 'FTS extension unavailable; skipping search-index creation.';
|
|
49
|
+
const FTS_UNAVAILABLE_MESSAGE = `${FTS_UNAVAILABLE_LEAD} ` +
|
|
45
50
|
'Full-text/BM25 search will be disabled until the LadybugDB FTS extension is ' +
|
|
46
51
|
'installed once with network access (GITNEXUS_LBUG_EXTENSION_INSTALL=auto) or ' +
|
|
47
52
|
'pre-installed for offline use. Run `gitnexus doctor` for details.';
|
|
@@ -368,13 +373,20 @@ export async function runFullAnalysis(repoPath, options, callbacks) {
|
|
|
368
373
|
// Surface the load-side reason (#2374): "not pre-installed" was wrong
|
|
369
374
|
// and doctor never installed anything, so the old message trapped
|
|
370
375
|
// users in a query → repair-fts → doctor loop with no way out.
|
|
371
|
-
const
|
|
372
|
-
|
|
373
|
-
|
|
376
|
+
const rawFtsReason = getExtensionCapabilities().find((c) => c.name === 'fts')?.reason;
|
|
377
|
+
const ftsReason = rawFtsReason?.replace(/\.$/, '');
|
|
378
|
+
// A missing runtime dependency (Windows error 126, #2374) is not healed
|
|
379
|
+
// by re-installing — the file is already present. Route that class to the
|
|
380
|
+
// classified remedy (install VC++ redist / OpenSSL) instead of the old
|
|
381
|
+
// "retry the network install" text that trapped the user in a loop.
|
|
382
|
+
const { kind, remedy } = diagnoseExtensionLoad(rawFtsReason);
|
|
383
|
+
const remedyTail = kind === 'missing_dependency'
|
|
384
|
+
? ` ${remedy}`
|
|
385
|
+
: '. Retry with network access and GITNEXUS_LBUG_EXTENSION_INSTALL=auto to install it, ' +
|
|
386
|
+
'or pre-install the extension file; run `gitnexus doctor` for live FTS status.';
|
|
374
387
|
throw new Error('Cannot repair FTS indexes: the LadybugDB FTS extension failed to load' +
|
|
375
388
|
(ftsReason ? ` — ${ftsReason}` : '') +
|
|
376
|
-
|
|
377
|
-
'or pre-install the extension file; run `gitnexus doctor` for live FTS status.');
|
|
389
|
+
remedyTail);
|
|
378
390
|
}
|
|
379
391
|
progress('fts', 85, 'Repairing search indexes...');
|
|
380
392
|
await createSearchFTSIndexes({
|
|
@@ -963,7 +975,15 @@ export async function runFullAnalysis(repoPath, options, callbacks) {
|
|
|
963
975
|
progress('fts', 90, 'Search indexes ready');
|
|
964
976
|
}
|
|
965
977
|
else {
|
|
966
|
-
|
|
978
|
+
// For a missing runtime dependency (#2374) the file is present, so the
|
|
979
|
+
// generic "install it with network access" tail in FTS_UNAVAILABLE_MESSAGE
|
|
980
|
+
// contradicts the remedy's own "reinstalling will NOT help" (#2383 F2). Lead
|
|
981
|
+
// with the class-neutral sentence and append only the classified remedy.
|
|
982
|
+
const ftsReason = getExtensionCapabilities().find((c) => c.name === 'fts')?.reason;
|
|
983
|
+
const { kind, remedy } = diagnoseExtensionLoad(ftsReason);
|
|
984
|
+
log(kind === 'missing_dependency'
|
|
985
|
+
? `${FTS_UNAVAILABLE_LEAD} ${remedy}`
|
|
986
|
+
: FTS_UNAVAILABLE_MESSAGE);
|
|
967
987
|
progress('fts', 90, 'Search indexes skipped (FTS unavailable)');
|
|
968
988
|
}
|
|
969
989
|
// ── Phase 3.5: Re-insert cached embeddings ────────────────────────
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { createFTSIndex, dropFTSIndex, DEFAULT_FTS_STEMMER } from '../lbug/lbug-adapter.js';
|
|
2
2
|
import { getExtensionCapabilities } from '../lbug/extension-loader.js';
|
|
3
|
+
import { classifyExtensionLoadError } from '../lbug/extension-load-error.js';
|
|
3
4
|
import { FTS_INDEXES } from './fts-schema.js';
|
|
4
5
|
/**
|
|
5
6
|
* Strip filesystem paths from a LadybugDB error before it reaches the HTTP
|
|
@@ -21,9 +22,18 @@ export const ftsDegradedWarning = () => {
|
|
|
21
22
|
const fts = getExtensionCapabilities().find((c) => c.name === 'fts');
|
|
22
23
|
if (fts && !fts.loaded) {
|
|
23
24
|
const reason = fts.reason ? redactPaths(fts.reason).replace(/\.$/, '') : undefined;
|
|
25
|
+
// A missing *runtime dependency* (Windows error 126, etc.) is not healed by
|
|
26
|
+
// reinstalling (#2374) — surface the classified remedy instead of the generic
|
|
27
|
+
// reinstall tail. Read the diagnosis cached at mark-unavailable time so this
|
|
28
|
+
// per-request path (HTTP /api/search + MCP query) does NO file I/O (#2383 F3);
|
|
29
|
+
// fall back to the pure, no-I/O string classifier if it is somehow absent.
|
|
30
|
+
const { kind, remedy } = fts.diagnosis ?? classifyExtensionLoadError(fts.reason);
|
|
31
|
+
const tail = kind === 'missing_dependency'
|
|
32
|
+
? ` ${remedy}`
|
|
33
|
+
: '. Run `gitnexus doctor` for details, then `gitnexus analyze --repair-fts` with network access to reinstall.';
|
|
24
34
|
return ('FTS extension failed to load — keyword search degraded' +
|
|
25
35
|
(reason ? ` (${reason})` : '') +
|
|
26
|
-
|
|
36
|
+
tail);
|
|
27
37
|
}
|
|
28
38
|
return 'FTS indexes missing — keyword search degraded. Run: gitnexus analyze --repair-fts (or gitnexus analyze --force) to rebuild indexes.';
|
|
29
39
|
};
|
package/package.json
CHANGED
|
@@ -43,6 +43,7 @@ const PLATFORM_LOGIC = [
|
|
|
43
43
|
'test/unit/cursor-hook.test.ts',
|
|
44
44
|
'test/unit/sidecar-recovery.test.ts',
|
|
45
45
|
'test/unit/pool-wal-recovery.test.ts',
|
|
46
|
+
'test/unit/lbug-adapter-wal-schema.test.ts',
|
|
46
47
|
'test/unit/detect-changes-worktree.test.ts',
|
|
47
48
|
'test/unit/eval-server-bind-restriction.test.ts',
|
|
48
49
|
'test/unit/ignore-service.test.ts',
|
|
@@ -61,6 +62,11 @@ const PLATFORM_LOGIC = [
|
|
|
61
62
|
// array form. Runs on every platform (the ubuntu suite covers Linux; this
|
|
62
63
|
// registration adds windows + macos).
|
|
63
64
|
'test/unit/embedding-install-arg-delivery.test.ts',
|
|
65
|
+
// Structural FTS-extension classifier against REAL binaries (#2374): on this
|
|
66
|
+
// matrix `process.execPath` / `lbugjs.node` are a real PE (windows) and Mach-O
|
|
67
|
+
// (macos), so the header parsing is proven on genuine binaries, not synthetic
|
|
68
|
+
// buffers (the ubuntu suite covers the ELF path).
|
|
69
|
+
'test/integration/extension-binary-real.test.ts',
|
|
64
70
|
];
|
|
65
71
|
|
|
66
72
|
// Native LadybugDB integration tests — exercise the @ladybugdb/core
|
|
@@ -13,7 +13,9 @@ const EXTENSION_NAME_PATTERN = /^[A-Za-z][A-Za-z0-9_]*$/;
|
|
|
13
13
|
// a missing file (plain INSTALL downloads it), or a permanent non-file failure a
|
|
14
14
|
// re-download can never fix (missing runtime dep: "cannot open shared object") —
|
|
15
15
|
// plain INSTALL avoids re-downloading ~2 MB on every analyze run forever.
|
|
16
|
-
|
|
16
|
+
// Exported so a parity test keeps this byte-identical to the copy in
|
|
17
|
+
// src/core/lbug/extension-load-error.ts (this `.mjs` cannot import that `.ts`), #2383 F5b.
|
|
18
|
+
export const FILE_CORRUPTION_SIGNATURES = [
|
|
17
19
|
/invalid elf/i,
|
|
18
20
|
/file too short/i,
|
|
19
21
|
/not a valid/i,
|