gitnexus 1.6.12 → 1.6.13-rc.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -31
- package/dist/cli/analyze.js +7 -24
- package/dist/cli/doctor.js +14 -9
- package/dist/core/lbug/extension-load-error.d.ts +8 -2
- package/dist/core/lbug/extension-load-error.js +23 -16
- package/dist/core/lbug/extension-loader.d.ts +21 -6
- package/dist/core/lbug/extension-loader.js +72 -17
- package/dist/core/lbug/lbug-adapter.d.ts +6 -25
- package/dist/core/lbug/lbug-adapter.js +36 -47
- package/dist/core/lbug/native-check.d.ts +15 -1
- package/dist/core/lbug/native-check.js +45 -22
- package/dist/core/lbug/pool-adapter.js +8 -1
- package/dist/core/lbug/sidecar-recovery.d.ts +32 -1
- package/dist/core/lbug/sidecar-recovery.js +63 -1
- package/dist/core/lbug/vendored-extension-path.d.ts +26 -0
- package/dist/core/lbug/vendored-extension-path.js +98 -0
- package/dist/core/lbug/wal-checkpoint-driver.d.ts +5 -2
- package/dist/core/lbug/wal-checkpoint-driver.js +7 -3
- package/dist/core/run-analyze.d.ts +4 -2
- package/dist/core/run-analyze.js +205 -68
- package/dist/core/search/fts-crash-marker.d.ts +59 -0
- package/dist/core/search/fts-crash-marker.js +55 -0
- package/dist/core/search/fts-indexes.js +2 -2
- package/dist/core/search/fts-policy.d.ts +11 -1
- package/dist/core/search/fts-policy.js +47 -0
- package/dist/core/tree-sitter/vendored-grammars.d.ts +2 -10
- package/dist/core/tree-sitter/vendored-grammars.js +2 -11
- package/dist/core/vendor-root.d.ts +8 -0
- package/dist/core/vendor-root.js +10 -0
- package/dist/mcp/server.js +10 -3
- package/dist/server/api.js +21 -14
- package/dist/storage/repo-meta.d.ts +31 -2
- package/package.json +2 -2
- package/scripts/assert-publish-fts-coverage.cjs +246 -0
- package/scripts/cross-platform-shard.ts +7 -2
- package/scripts/cross-platform-tests.ts +15 -0
- package/scripts/ensure-fts.ts +13 -11
- package/vendor/lbug-fts/LICENSE +27 -0
- package/vendor/lbug-fts/manifest.json +21 -0
- package/vendor/lbug-fts/prebuilds/SHA256SUMS +5 -0
- package/vendor/lbug-fts/prebuilds/darwin-arm64/libfts.lbug_extension +0 -0
- package/vendor/lbug-fts/prebuilds/darwin-x64/libfts.lbug_extension +0 -0
- package/vendor/lbug-fts/prebuilds/linux-arm64/libfts.lbug_extension +0 -0
- package/vendor/lbug-fts/prebuilds/linux-x64/libfts.lbug_extension +0 -0
- package/vendor/lbug-fts/prebuilds/win32-x64/libfts.lbug_extension +0 -0
- package/web/assets/{agent-DeT_Hy8W.js → agent-DiLVv9Xg.js} +8 -8
- package/web/assets/{index-K4KOdfVv.js → index-coee0RPm.js} +3 -3
- package/web/index.html +1 -1
|
@@ -1,13 +1,5 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
*
|
|
4
|
-
* This module compiles to `<pkg>/dist/core/tree-sitter/vendored-grammars.js`
|
|
5
|
-
* and runs from `<pkg>/src/core/tree-sitter/...` under tsx in dev — both sit
|
|
6
|
-
* three directories below the package root, and the build (`tsc`) never bundles,
|
|
7
|
-
* so `import.meta.url` resolves the same way in both. `vendor/` ships in the
|
|
8
|
-
* published package via package.json `files`.
|
|
9
|
-
*/
|
|
10
|
-
export declare const VENDOR_ROOT: string;
|
|
1
|
+
import { VENDOR_ROOT } from '../vendor-root.js';
|
|
2
|
+
export { VENDOR_ROOT };
|
|
11
3
|
/**
|
|
12
4
|
* The tree-sitter grammars GitNexus vendors inside its own package (NOT npm
|
|
13
5
|
* dependencies). Kept in one place so consumers (runtime loaders, the CLI
|
|
@@ -1,17 +1,8 @@
|
|
|
1
1
|
import { createRequire } from 'node:module';
|
|
2
|
-
import { fileURLToPath } from 'node:url';
|
|
3
2
|
import path from 'node:path';
|
|
3
|
+
import { VENDOR_ROOT } from '../vendor-root.js';
|
|
4
4
|
const _require = createRequire(import.meta.url);
|
|
5
|
-
|
|
6
|
-
* Absolute path to the vendored grammar tree (`<pkg>/vendor`).
|
|
7
|
-
*
|
|
8
|
-
* This module compiles to `<pkg>/dist/core/tree-sitter/vendored-grammars.js`
|
|
9
|
-
* and runs from `<pkg>/src/core/tree-sitter/...` under tsx in dev — both sit
|
|
10
|
-
* three directories below the package root, and the build (`tsc`) never bundles,
|
|
11
|
-
* so `import.meta.url` resolves the same way in both. `vendor/` ships in the
|
|
12
|
-
* published package via package.json `files`.
|
|
13
|
-
*/
|
|
14
|
-
export const VENDOR_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..', '..', 'vendor');
|
|
5
|
+
export { VENDOR_ROOT };
|
|
15
6
|
/**
|
|
16
7
|
* The tree-sitter grammars GitNexus vendors inside its own package (NOT npm
|
|
17
8
|
* dependencies). Kept in one place so consumers (runtime loaders, the CLI
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Absolute path to the published `vendor/` tree (`<pkg>/vendor`).
|
|
3
|
+
*
|
|
4
|
+
* This module compiles to `<pkg>/dist/core/vendor-root.js` and runs from
|
|
5
|
+
* `<pkg>/src/core/` under tsx — both sit two directories below the package
|
|
6
|
+
* root. Shared so grammar loaders and the FTS artifact resolver cannot drift.
|
|
7
|
+
*/
|
|
8
|
+
export declare const VENDOR_ROOT: string;
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { fileURLToPath } from 'node:url';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
/**
|
|
4
|
+
* Absolute path to the published `vendor/` tree (`<pkg>/vendor`).
|
|
5
|
+
*
|
|
6
|
+
* This module compiles to `<pkg>/dist/core/vendor-root.js` and runs from
|
|
7
|
+
* `<pkg>/src/core/` under tsx — both sit two directories below the package
|
|
8
|
+
* root. Shared so grammar loaders and the FTS artifact resolver cannot drift.
|
|
9
|
+
*/
|
|
10
|
+
export const VENDOR_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..', 'vendor');
|
package/dist/mcp/server.js
CHANGED
|
@@ -60,6 +60,14 @@ function getNextStepHint(toolName, args) {
|
|
|
60
60
|
return '';
|
|
61
61
|
}
|
|
62
62
|
}
|
|
63
|
+
/** Include a string `Error.code` in MCP error text when present. */
|
|
64
|
+
function formatMcpToolError(error) {
|
|
65
|
+
const message = error instanceof Error ? error.message : 'Unknown error';
|
|
66
|
+
const code = typeof error === 'object' && error !== null && 'code' in error
|
|
67
|
+
? error.code
|
|
68
|
+
: undefined;
|
|
69
|
+
return typeof code === 'string' ? `Error [${code}]: ${message}` : `Error: ${message}`;
|
|
70
|
+
}
|
|
63
71
|
/**
|
|
64
72
|
* Create a configured MCP Server with all handlers registered.
|
|
65
73
|
* Transport-agnostic — caller connects the desired transport.
|
|
@@ -129,7 +137,7 @@ export function createMCPServer(backend, options = {}) {
|
|
|
129
137
|
{
|
|
130
138
|
uri,
|
|
131
139
|
mimeType: 'text/plain',
|
|
132
|
-
text:
|
|
140
|
+
text: formatMcpToolError(err),
|
|
133
141
|
},
|
|
134
142
|
],
|
|
135
143
|
};
|
|
@@ -185,12 +193,11 @@ export function createMCPServer(backend, options = {}) {
|
|
|
185
193
|
};
|
|
186
194
|
}
|
|
187
195
|
catch (error) {
|
|
188
|
-
const message = error instanceof Error ? error.message : 'Unknown error';
|
|
189
196
|
return {
|
|
190
197
|
content: [
|
|
191
198
|
{
|
|
192
199
|
type: 'text',
|
|
193
|
-
text: applyMcpMaxTokens(
|
|
200
|
+
text: applyMcpMaxTokens(formatMcpToolError(error), maxTokens),
|
|
194
201
|
},
|
|
195
202
|
],
|
|
196
203
|
isError: true,
|
package/dist/server/api.js
CHANGED
|
@@ -14,6 +14,7 @@ import fs from 'fs/promises';
|
|
|
14
14
|
import { canonicalizePath, cloneDirBelongsToEntry, loadMeta, saveMeta, listRegisteredRepos, registryPathEquals, } from '../storage/repo-manager.js';
|
|
15
15
|
import { requireDeletableStoragePath, requireRegisteredStoragePath, STATUS_STORAGE_REQUIREMENTS, StorageDeletionError, StorageRequirementError, } from '../storage/storage-resolver.js';
|
|
16
16
|
import { executeQuery, executePrepared, executeWithReusedStatement, streamQuery, flushWAL, closeLbug, withLbugDb, isReadOnlyDbError, } from '../core/lbug/lbug-adapter.js';
|
|
17
|
+
import { assertReadOnlyFtsCrashSafe } from '../core/lbug/sidecar-recovery.js';
|
|
17
18
|
import { isValidQueryParams } from '../core/lbug/query-params.js';
|
|
18
19
|
import { NODE_TABLES } from '../_shared/index.js';
|
|
19
20
|
import { searchFTSFromLbug } from '../core/search/bm25-index.js';
|
|
@@ -449,6 +450,12 @@ export const streamGraphNdjson = async (res, includeContent = false, signal) =>
|
|
|
449
450
|
}, signal);
|
|
450
451
|
});
|
|
451
452
|
};
|
|
453
|
+
const httpErrorBody = (err, fallback) => {
|
|
454
|
+
const body = { error: err.message || fallback };
|
|
455
|
+
if (typeof err?.code === 'string')
|
|
456
|
+
body.code = err.code;
|
|
457
|
+
return body;
|
|
458
|
+
};
|
|
452
459
|
const statusFromError = (err) => {
|
|
453
460
|
// Validation helpers throw BadRequestError / ForbiddenError with a typed
|
|
454
461
|
// .status field — honor it before falling back to message-string matching.
|
|
@@ -666,7 +673,7 @@ export const handleQueryRequest = async (req, res, resolveRepo) => {
|
|
|
666
673
|
res.status(403).json({ error: 'Write queries are not allowed via the HTTP API' });
|
|
667
674
|
return;
|
|
668
675
|
}
|
|
669
|
-
res.status(500).json(
|
|
676
|
+
res.status(500).json(httpErrorBody(err, 'Query failed'));
|
|
670
677
|
}
|
|
671
678
|
};
|
|
672
679
|
/**
|
|
@@ -1099,10 +1106,10 @@ export const createServer = async (port, host = '127.0.0.1') => {
|
|
|
1099
1106
|
if (err instanceof ClientDisconnectedError) {
|
|
1100
1107
|
return;
|
|
1101
1108
|
}
|
|
1102
|
-
const
|
|
1109
|
+
const body = httpErrorBody(err, 'Failed to build graph');
|
|
1103
1110
|
if (res.headersSent) {
|
|
1104
1111
|
try {
|
|
1105
|
-
res.write(JSON.stringify({ type: 'error',
|
|
1112
|
+
res.write(JSON.stringify({ type: 'error', ...body }) + '\n');
|
|
1106
1113
|
}
|
|
1107
1114
|
catch {
|
|
1108
1115
|
// Best-effort only after streaming has started.
|
|
@@ -1110,7 +1117,7 @@ export const createServer = async (port, host = '127.0.0.1') => {
|
|
|
1110
1117
|
res.end();
|
|
1111
1118
|
return;
|
|
1112
1119
|
}
|
|
1113
|
-
res.status(500).json(
|
|
1120
|
+
res.status(500).json(body);
|
|
1114
1121
|
}
|
|
1115
1122
|
});
|
|
1116
1123
|
// Execute Cypher query
|
|
@@ -1258,7 +1265,7 @@ export const createServer = async (port, host = '127.0.0.1') => {
|
|
|
1258
1265
|
catch (err) {
|
|
1259
1266
|
if (sendStorageRequirementHttp(err, res))
|
|
1260
1267
|
return;
|
|
1261
|
-
res.status(500).json(
|
|
1268
|
+
res.status(500).json(httpErrorBody(err, 'Search failed'));
|
|
1262
1269
|
}
|
|
1263
1270
|
});
|
|
1264
1271
|
// Read file — with path traversal guard
|
|
@@ -1323,7 +1330,7 @@ export const createServer = async (port, host = '127.0.0.1') => {
|
|
|
1323
1330
|
catch (err) {
|
|
1324
1331
|
if (sendStorageRequirementHttp(err, res))
|
|
1325
1332
|
return;
|
|
1326
|
-
res.status(statusFromError(err)).json(
|
|
1333
|
+
res.status(statusFromError(err)).json(httpErrorBody(err, 'Grep failed'));
|
|
1327
1334
|
}
|
|
1328
1335
|
});
|
|
1329
1336
|
// List all processes
|
|
@@ -1333,7 +1340,7 @@ export const createServer = async (port, host = '127.0.0.1') => {
|
|
|
1333
1340
|
res.json(result);
|
|
1334
1341
|
}
|
|
1335
1342
|
catch (err) {
|
|
1336
|
-
res.status(statusFromError(err)).json(
|
|
1343
|
+
res.status(statusFromError(err)).json(httpErrorBody(err, 'Failed to query processes'));
|
|
1337
1344
|
}
|
|
1338
1345
|
});
|
|
1339
1346
|
// Process detail
|
|
@@ -1352,9 +1359,7 @@ export const createServer = async (port, host = '127.0.0.1') => {
|
|
|
1352
1359
|
res.json(result);
|
|
1353
1360
|
}
|
|
1354
1361
|
catch (err) {
|
|
1355
|
-
res
|
|
1356
|
-
.status(statusFromError(err))
|
|
1357
|
-
.json({ error: err.message || 'Failed to query process detail' });
|
|
1362
|
+
res.status(statusFromError(err)).json(httpErrorBody(err, 'Failed to query process detail'));
|
|
1358
1363
|
}
|
|
1359
1364
|
});
|
|
1360
1365
|
// List all clusters
|
|
@@ -1364,7 +1369,7 @@ export const createServer = async (port, host = '127.0.0.1') => {
|
|
|
1364
1369
|
res.json(result);
|
|
1365
1370
|
}
|
|
1366
1371
|
catch (err) {
|
|
1367
|
-
res.status(statusFromError(err)).json(
|
|
1372
|
+
res.status(statusFromError(err)).json(httpErrorBody(err, 'Failed to query clusters'));
|
|
1368
1373
|
}
|
|
1369
1374
|
});
|
|
1370
1375
|
// Cluster detail
|
|
@@ -1383,9 +1388,7 @@ export const createServer = async (port, host = '127.0.0.1') => {
|
|
|
1383
1388
|
res.json(result);
|
|
1384
1389
|
}
|
|
1385
1390
|
catch (err) {
|
|
1386
|
-
res
|
|
1387
|
-
.status(statusFromError(err))
|
|
1388
|
-
.json({ error: err.message || 'Failed to query cluster detail' });
|
|
1391
|
+
res.status(statusFromError(err)).json(httpErrorBody(err, 'Failed to query cluster detail'));
|
|
1389
1392
|
}
|
|
1390
1393
|
});
|
|
1391
1394
|
// ── Analyze API ──────────────────────────────────────────────────────
|
|
@@ -1640,6 +1643,10 @@ export const createServer = async (port, host = '127.0.0.1') => {
|
|
|
1640
1643
|
const lbugPath = path.join(storagePath, LBUG_DIRECTORY);
|
|
1641
1644
|
const ftsSession = await loadFtsSession(storagePath);
|
|
1642
1645
|
let embeddingMeta = ftsSession.meta;
|
|
1646
|
+
// Writable embed still replays a leftover FTS-abort WAL. Refuse
|
|
1647
|
+
// here — doInitLbug only gates the readOnly path, and analyze
|
|
1648
|
+
// writers must still be able to park/rebuild.
|
|
1649
|
+
await assertReadOnlyFtsCrashSafe(lbugPath);
|
|
1643
1650
|
await withLbugDb(lbugPath, async () => {
|
|
1644
1651
|
const { runEmbeddingPipeline } = await import('../core/embeddings/embedding-pipeline.js');
|
|
1645
1652
|
const { resolveEmbeddingIdentity } = await import('../core/embeddings/embedding-identity.js');
|
|
@@ -77,6 +77,11 @@ export interface AnalyzerRunnerIdentity {
|
|
|
77
77
|
digest: string;
|
|
78
78
|
};
|
|
79
79
|
}
|
|
80
|
+
/**
|
|
81
|
+
* Hand-mirrored `FtsSkipReason` from core/search/fts-policy.ts so storage
|
|
82
|
+
* takes no core import. Change both declarations together.
|
|
83
|
+
*/
|
|
84
|
+
export type PersistedFtsSkipReason = 'extension-unavailable' | 'build-failed' | 'disabled-by-flag' | 'disabled-by-env' | 'native-abort' | 'tuple-missing';
|
|
80
85
|
export interface RepoMeta {
|
|
81
86
|
repoPath: string;
|
|
82
87
|
/** Complete index directory selected for this successful analysis. */
|
|
@@ -178,13 +183,28 @@ export interface RepoMeta {
|
|
|
178
183
|
* `--repair-fts` or a content change addresses it.
|
|
179
184
|
* - `disabled-by-flag` / `disabled-by-env` — deliberate opt-out.
|
|
180
185
|
* A later analyze without the opt-out rebuilds FTS at the same commit.
|
|
186
|
+
* - `native-abort` — inferred on the next run from an FTS-phase dirty
|
|
187
|
+
* flag after the previous process died in the native FTS build.
|
|
188
|
+
* - `tuple-missing` — no packaged artifact for this platform tuple.
|
|
189
|
+
* The tuple itself is not persisted (closed enum; live messages name it).
|
|
181
190
|
*
|
|
182
191
|
* Collapsing both into `status: 'unavailable'` is exactly what made that
|
|
183
192
|
* loop reachable. ABSENT on indexes written before #2841 and on the
|
|
184
193
|
* `--repair-fts` stamp (which writes `status: 'available'`); `undefined`
|
|
185
194
|
* therefore reads as "cause unknown" and keeps the pre-#2841 behaviour.
|
|
195
|
+
* No schema version: meta reads are unchecked casts. An older binary
|
|
196
|
+
* still sees the raw JSON string for an unknown member; it does not
|
|
197
|
+
* drop the field to `undefined` at parse time.
|
|
198
|
+
*/
|
|
199
|
+
skipReason?: PersistedFtsSkipReason;
|
|
200
|
+
/**
|
|
201
|
+
* Write plan of the analyze that aborted, persisted with
|
|
202
|
+
* `skipReason: 'native-abort'` so readers can refuse a leftover live
|
|
203
|
+
* WAL after recovery clears `incrementalInProgress`. Staging persist
|
|
204
|
+
* also writes `native-abort` and must keep its live WAL. Absent on
|
|
205
|
+
* older indexes and on non-abort skips.
|
|
186
206
|
*/
|
|
187
|
-
|
|
207
|
+
writePlan?: 'in-place' | 'staging';
|
|
188
208
|
};
|
|
189
209
|
vectorSearch: {
|
|
190
210
|
provider: string;
|
|
@@ -396,8 +416,17 @@ export interface RepoMeta {
|
|
|
396
416
|
/** Number of files in the writable set, for diagnostic logs.
|
|
397
417
|
* `0` on the full-rebuild path (no incremental write set exists). */
|
|
398
418
|
toWriteCount: number;
|
|
399
|
-
/**
|
|
419
|
+
/**
|
|
420
|
+
* Last completed writeback phase before the process stopped.
|
|
421
|
+
* `'fts'` is the graph-boundary / FTS-build marker: recovery may act
|
|
422
|
+
* more narrowly than a bare dirty flag, and `--repair-fts` must not
|
|
423
|
+
* treat it as a half-written graph.
|
|
424
|
+
*/
|
|
400
425
|
phase?: string;
|
|
426
|
+
/** Settled write plan at the FTS boundary (in-place vs unpublished staging). */
|
|
427
|
+
writePlan?: 'in-place' | 'staging';
|
|
428
|
+
/** Whether the converged graph-boundary CHECKPOINT succeeded. */
|
|
429
|
+
checkpointSucceeded?: boolean;
|
|
401
430
|
/** Directly changed/added files before importer expansion. */
|
|
402
431
|
directWriteCount?: number;
|
|
403
432
|
/** Extra files pulled into the writable set by importer BFS. */
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "gitnexus",
|
|
3
|
-
"version": "1.6.
|
|
3
|
+
"version": "1.6.13-rc.2",
|
|
4
4
|
"description": "Graph-powered code intelligence for AI agents. Index any codebase, query via MCP or CLI.",
|
|
5
5
|
"author": "Abhigyan Patwari",
|
|
6
6
|
"license": "PolyForm-Noncommercial-1.0.0",
|
|
@@ -53,7 +53,7 @@
|
|
|
53
53
|
"postinstall": "node scripts/build-tree-sitter-grammars.cjs",
|
|
54
54
|
"assert-publish-coverage": "node scripts/assert-publish-grammar-coverage.cjs",
|
|
55
55
|
"prepare": "node scripts/build.js",
|
|
56
|
-
"prepack": "node scripts/assert-publish-grammar-coverage.cjs && node scripts/build.js --web && node scripts/assert-web-assets.mjs web",
|
|
56
|
+
"prepack": "node scripts/assert-publish-grammar-coverage.cjs && node scripts/assert-publish-fts-coverage.cjs && node scripts/build.js --web && node scripts/assert-web-assets.mjs web",
|
|
57
57
|
"version": "node scripts/sync-plugin-manifests.mjs"
|
|
58
58
|
},
|
|
59
59
|
"dependencies": {
|
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* Publish guard: core↔extension pairing plus vendored FTS artifact integrity.
|
|
4
|
+
*
|
|
5
|
+
* Does not shell out to `npm pack` (prepack re-entrancy; see the grammar gate).
|
|
6
|
+
* Checksums and the `files` allow-list are asserted as pure predicates so a
|
|
7
|
+
* future lean-publish narrowing cannot drop the artifacts silently.
|
|
8
|
+
*/
|
|
9
|
+
const fs = require('fs');
|
|
10
|
+
const path = require('path');
|
|
11
|
+
const crypto = require('crypto');
|
|
12
|
+
|
|
13
|
+
const WIN32_ARM64 = 'win32-arm64';
|
|
14
|
+
const SAFE_FILENAME = /^[\w.-]+\.lbug_extension$/;
|
|
15
|
+
const REQUIRED_SUPPORTED_TUPLES = [
|
|
16
|
+
'linux-x64',
|
|
17
|
+
'linux-arm64',
|
|
18
|
+
'darwin-x64',
|
|
19
|
+
'darwin-arm64',
|
|
20
|
+
'win32-x64',
|
|
21
|
+
];
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Pure pairing core (exported for tests). Returns human-readable problem
|
|
25
|
+
* strings; an empty array means the core↔extension pin is consistent.
|
|
26
|
+
*/
|
|
27
|
+
const EXACT_CORE_PIN = /^\d+\.\d+\.\d+$/;
|
|
28
|
+
|
|
29
|
+
function findPairingProblems({
|
|
30
|
+
installedCoreVersion,
|
|
31
|
+
manifestCoreVersion,
|
|
32
|
+
manifestExtensionVersion,
|
|
33
|
+
}) {
|
|
34
|
+
const problems = [];
|
|
35
|
+
const installed = String(installedCoreVersion ?? '');
|
|
36
|
+
const pinned = String(manifestCoreVersion ?? '');
|
|
37
|
+
if (!EXACT_CORE_PIN.test(installed) || !EXACT_CORE_PIN.test(pinned)) {
|
|
38
|
+
problems.push(
|
|
39
|
+
`core pin must be exact x.y.z: installed '${installedCoreVersion ?? ''}' vs manifest '${manifestCoreVersion ?? ''}'`,
|
|
40
|
+
);
|
|
41
|
+
return problems;
|
|
42
|
+
}
|
|
43
|
+
if (installed !== pinned) {
|
|
44
|
+
problems.push(
|
|
45
|
+
`core pin mismatch: installed ${installed} vs manifest ${pinned}` +
|
|
46
|
+
(manifestExtensionVersion ? ` (extension ${manifestExtensionVersion})` : ''),
|
|
47
|
+
);
|
|
48
|
+
}
|
|
49
|
+
return problems;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function readInstalledCoreVersion(pkg) {
|
|
53
|
+
const raw = pkg?.dependencies?.['@ladybugdb/core'];
|
|
54
|
+
return raw == null ? '' : String(raw);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
function normalizeFilesEntry(value) {
|
|
58
|
+
return String(value ?? '')
|
|
59
|
+
.replace(/\\/g, '/')
|
|
60
|
+
.replace(/\/+$/, '')
|
|
61
|
+
.replace(/\/\*\*?$/, '');
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** True when package.json `files` still ships the FTS prebuild tree. */
|
|
65
|
+
function filesCoverFtsArtifacts(filesField) {
|
|
66
|
+
return (filesField || []).some((entry) => {
|
|
67
|
+
const n = normalizeFilesEntry(entry);
|
|
68
|
+
return (
|
|
69
|
+
n === 'vendor' ||
|
|
70
|
+
n === 'vendor/lbug-fts' ||
|
|
71
|
+
n === 'vendor/lbug-fts/prebuilds' ||
|
|
72
|
+
n === 'vendor/**/prebuilds'
|
|
73
|
+
);
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
function supportedTuplesFromManifest(manifest) {
|
|
78
|
+
return (manifest?.tuples ?? []).map((entry) => entry.tuple);
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function unsupportedTuplesFromManifest(manifest) {
|
|
82
|
+
return (manifest?.unsupportedTuples ?? []).map((entry) => entry.tuple);
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
function parseSha256Sums(text) {
|
|
86
|
+
const out = {};
|
|
87
|
+
for (const line of String(text ?? '').split(/\r?\n/)) {
|
|
88
|
+
const m = /^([a-fA-F0-9]{64})\s+\.\/(\S+)$/.exec(line.trim());
|
|
89
|
+
if (!m) continue;
|
|
90
|
+
out[m[2]] = m[1].toLowerCase();
|
|
91
|
+
}
|
|
92
|
+
return out;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
function sha256File(filePath) {
|
|
96
|
+
return crypto.createHash('sha256').update(fs.readFileSync(filePath)).digest('hex');
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* Integrity + coverage predicates. `artifactByTuple` is injected so tests
|
|
101
|
+
* never invoke pack and never need the real binaries.
|
|
102
|
+
*/
|
|
103
|
+
function findArtifactProblems({
|
|
104
|
+
tuples,
|
|
105
|
+
unsupportedTuples,
|
|
106
|
+
filesField,
|
|
107
|
+
checksumByRelPath,
|
|
108
|
+
artifactByTuple,
|
|
109
|
+
filename,
|
|
110
|
+
}) {
|
|
111
|
+
const problems = [];
|
|
112
|
+
const filenameSafe = filename || 'libfts.lbug_extension';
|
|
113
|
+
if (!SAFE_FILENAME.test(filenameSafe)) {
|
|
114
|
+
problems.push(`invalid FTS artifact filename: ${filenameSafe}`);
|
|
115
|
+
}
|
|
116
|
+
const listed = new Set(tuples || []);
|
|
117
|
+
for (const required of REQUIRED_SUPPORTED_TUPLES) {
|
|
118
|
+
if (!listed.has(required)) {
|
|
119
|
+
problems.push(`manifest.tuples is missing required ${required}`);
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
if (!tuples || tuples.length === 0) {
|
|
123
|
+
problems.push('manifest.tuples is empty — refusing to publish with 0 artifacts');
|
|
124
|
+
}
|
|
125
|
+
if (!filesCoverFtsArtifacts(filesField)) {
|
|
126
|
+
problems.push('package.json files no longer covers vendor/lbug-fts/prebuilds');
|
|
127
|
+
}
|
|
128
|
+
if (!(unsupportedTuples || []).includes(WIN32_ARM64)) {
|
|
129
|
+
problems.push('win32-arm64 must be declared unsupported (no upstream artifact)');
|
|
130
|
+
}
|
|
131
|
+
if ((tuples || []).includes(WIN32_ARM64)) {
|
|
132
|
+
problems.push('win32-arm64 is listed as supported but has no upstream artifact');
|
|
133
|
+
}
|
|
134
|
+
for (const tuple of tuples || []) {
|
|
135
|
+
const rel = `${tuple}/${filenameSafe}`;
|
|
136
|
+
const artifact = artifactByTuple?.[tuple];
|
|
137
|
+
if (!artifact?.exists) {
|
|
138
|
+
problems.push(`missing artifact for ${tuple} (${rel})`);
|
|
139
|
+
continue;
|
|
140
|
+
}
|
|
141
|
+
const expected = checksumByRelPath?.[rel];
|
|
142
|
+
if (!expected) {
|
|
143
|
+
problems.push(`missing SHA-256 for ${rel}`);
|
|
144
|
+
continue;
|
|
145
|
+
}
|
|
146
|
+
if (artifact.hash !== expected) {
|
|
147
|
+
problems.push(
|
|
148
|
+
`checksum mismatch for ${tuple}: expected ${expected} got ${artifact.hash}` +
|
|
149
|
+
(artifact.sizeBytes != null ? ` (${artifact.sizeBytes} bytes)` : ''),
|
|
150
|
+
);
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
return problems;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
function readDiskArtifacts(prebuildsDir, tuples, filename) {
|
|
157
|
+
const artifactByTuple = {};
|
|
158
|
+
for (const tuple of tuples) {
|
|
159
|
+
const filePath = path.join(prebuildsDir, tuple, filename);
|
|
160
|
+
if (!fs.existsSync(filePath)) {
|
|
161
|
+
artifactByTuple[tuple] = { exists: false };
|
|
162
|
+
continue;
|
|
163
|
+
}
|
|
164
|
+
const buf = fs.readFileSync(filePath);
|
|
165
|
+
artifactByTuple[tuple] = {
|
|
166
|
+
exists: true,
|
|
167
|
+
hash: crypto.createHash('sha256').update(buf).digest('hex'),
|
|
168
|
+
sizeBytes: buf.byteLength,
|
|
169
|
+
};
|
|
170
|
+
}
|
|
171
|
+
return artifactByTuple;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
function main() {
|
|
175
|
+
const gitnexusRoot = path.join(__dirname, '..');
|
|
176
|
+
const pkg = JSON.parse(fs.readFileSync(path.join(gitnexusRoot, 'package.json'), 'utf8'));
|
|
177
|
+
const vendorDir = path.join(gitnexusRoot, 'vendor', 'lbug-fts');
|
|
178
|
+
const manifestPath = path.join(vendorDir, 'manifest.json');
|
|
179
|
+
let manifest;
|
|
180
|
+
try {
|
|
181
|
+
manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8'));
|
|
182
|
+
} catch (err) {
|
|
183
|
+
console.error(
|
|
184
|
+
`[fts-pairing] Refusing to publish — cannot read ${manifestPath}: ${err.message}`,
|
|
185
|
+
);
|
|
186
|
+
process.exit(1);
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
const installedCoreVersion = readInstalledCoreVersion(pkg);
|
|
190
|
+
const pairing = findPairingProblems({
|
|
191
|
+
installedCoreVersion,
|
|
192
|
+
manifestCoreVersion: manifest.coreVersion,
|
|
193
|
+
manifestExtensionVersion: manifest.extensionVersion,
|
|
194
|
+
});
|
|
195
|
+
|
|
196
|
+
const tuples = supportedTuplesFromManifest(manifest);
|
|
197
|
+
const unsupportedTuples = unsupportedTuplesFromManifest(manifest);
|
|
198
|
+
const filename = manifest.filename || 'libfts.lbug_extension';
|
|
199
|
+
const prebuildsDir = path.join(vendorDir, 'prebuilds');
|
|
200
|
+
let checksumByRelPath = {};
|
|
201
|
+
try {
|
|
202
|
+
checksumByRelPath = parseSha256Sums(
|
|
203
|
+
fs.readFileSync(path.join(prebuildsDir, 'SHA256SUMS'), 'utf8'),
|
|
204
|
+
);
|
|
205
|
+
} catch (err) {
|
|
206
|
+
pairing.push(`cannot read SHA256SUMS: ${err.message}`);
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
const artifacts = findArtifactProblems({
|
|
210
|
+
tuples,
|
|
211
|
+
unsupportedTuples,
|
|
212
|
+
filesField: pkg.files,
|
|
213
|
+
checksumByRelPath,
|
|
214
|
+
artifactByTuple: readDiskArtifacts(prebuildsDir, tuples, filename),
|
|
215
|
+
filename,
|
|
216
|
+
});
|
|
217
|
+
|
|
218
|
+
const problems = [...pairing, ...artifacts];
|
|
219
|
+
if (problems.length > 0) {
|
|
220
|
+
console.error('[fts-pairing] Refusing to publish — FTS artifact coverage failed:');
|
|
221
|
+
for (const p of problems) console.error(` - ${p}`);
|
|
222
|
+
console.error(
|
|
223
|
+
'\nFix: refresh vendor/lbug-fts via .github/scripts/fetch-lbug-fts-artifacts.mjs, ' +
|
|
224
|
+
'or restore the core pin / files allow-list.',
|
|
225
|
+
);
|
|
226
|
+
process.exit(1);
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
console.log(
|
|
230
|
+
`[fts-pairing] OK — core ${installedCoreVersion} ↔ extension ${manifest.extensionVersion}; ` +
|
|
231
|
+
`${tuples.length} artifacts.`,
|
|
232
|
+
);
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
if (require.main === module) main();
|
|
236
|
+
|
|
237
|
+
module.exports = {
|
|
238
|
+
findPairingProblems,
|
|
239
|
+
findArtifactProblems,
|
|
240
|
+
filesCoverFtsArtifacts,
|
|
241
|
+
parseSha256Sums,
|
|
242
|
+
readInstalledCoreVersion,
|
|
243
|
+
supportedTuplesFromManifest,
|
|
244
|
+
unsupportedTuplesFromManifest,
|
|
245
|
+
sha256File,
|
|
246
|
+
};
|
|
@@ -55,8 +55,13 @@ export const WINDOWS_WEIGHTS_SEC: Readonly<Record<string, number>> = {
|
|
|
55
55
|
// Upstream speedups may reduce these figures; retaining conservative weights
|
|
56
56
|
// keeps the expensive suites distributed without changing the watchdog.
|
|
57
57
|
'test/unit/incremental-index-extension-dml-gate.test.ts': 414,
|
|
58
|
-
|
|
59
|
-
|
|
58
|
+
// Re-measured on windows-latest run 34815870795 after vendored-first FTS
|
|
59
|
+
// rewrote the HOME-layout e2e (373s) and skills-e2e grew to 542s. The old
|
|
60
|
+
// 146s/444s entries packed both onto shard 2/3 and blew the 20-minute
|
|
61
|
+
// watchdog with one file still queued.
|
|
62
|
+
'test/integration/skills-e2e.test.ts': 550,
|
|
63
|
+
'test/integration/fts-extension-e2e.test.ts': 380,
|
|
64
|
+
'test/integration/skip-fts.test.ts': 110,
|
|
60
65
|
'test/integration/analyze-wal-checkpoint-failure.test.ts': 86,
|
|
61
66
|
'test/integration/cli-limit-e2e.test.ts': 75,
|
|
62
67
|
'test/unit/hooks.test.ts': 26,
|
|
@@ -85,6 +85,20 @@ const PLATFORM_LOGIC = [
|
|
|
85
85
|
'test/unit/lbug-config-pagesize.test.ts',
|
|
86
86
|
'test/unit/worker-pool-windows-quarantine.test.ts',
|
|
87
87
|
'test/unit/lbug-pool-fts-load.test.ts',
|
|
88
|
+
// U7 arm B: Windows FTS names a vendor-neutral OpenSSL/VC++ prerequisite
|
|
89
|
+
// and must never recommend borrowing Git for Windows DLLs. The file's
|
|
90
|
+
// assertions are unconditional so a skip-only suite cannot stay green.
|
|
91
|
+
'test/integration/fts-windows-dependency.test.ts',
|
|
92
|
+
// Remedy-text suites: discoverability only. They pass explicit platform
|
|
93
|
+
// strings into pure classifiers and drive mocked rejections with hardcoded
|
|
94
|
+
// literals, so they assert identically on every runner. Registering them
|
|
95
|
+
// here does not claim Windows-specific behavioral coverage.
|
|
96
|
+
'test/unit/extension-load-error.test.ts',
|
|
97
|
+
'test/unit/fts-degraded-warning.test.ts',
|
|
98
|
+
// Vendored-root symlink containment uses realpathSync + path.relative; a
|
|
99
|
+
// prefix-only leak follows a Windows junction / POSIX symlink out of
|
|
100
|
+
// vendor/. Ubuntu-only would leave that guard unverified on the OS matrix.
|
|
101
|
+
'test/unit/lbug-extension-loader.test.ts',
|
|
88
102
|
// Global registry writes use the platform-specific index-lock backend
|
|
89
103
|
// (Windows named pipe, Linux socket, or macOS file lock). This includes the
|
|
90
104
|
// overlapping-registration regression from #2716 on every OS matrix.
|
|
@@ -208,6 +222,7 @@ const SPAWN_CLI = [
|
|
|
208
222
|
// FTS extension lifecycle — the #2374 bug was Windows-reported, so this must
|
|
209
223
|
// run on the Windows/macOS matrix, not just the Ubuntu full suite.
|
|
210
224
|
'test/integration/fts-extension-e2e.test.ts',
|
|
225
|
+
'test/integration/fts-vendored-root-seam.test.ts',
|
|
211
226
|
'test/integration/server-http-startup.test.ts',
|
|
212
227
|
'test/integration/mcp/server-startup.test.ts',
|
|
213
228
|
'test/integration/analyze-heap-oom-e2e.test.ts',
|
package/scripts/ensure-fts.ts
CHANGED
|
@@ -1,14 +1,11 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
3
|
-
* up front, so every test in a sharded CI run finds them regardless of shard.
|
|
2
|
+
* Make FTS and VECTOR resolvable for every shard before vitest starts.
|
|
4
3
|
*
|
|
5
|
-
* FTS
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
* dependency by installing FTS once before vitest starts. `auto` is LOAD-first,
|
|
11
|
-
* so a cache-warmed extension costs no network.
|
|
4
|
+
* After vendoring, FTS is ready when the packaged artifact exists — LOAD
|
|
5
|
+
* no longer writes `~/.lbdb`, and the FILE-path gates resolve that artifact
|
|
6
|
+
* (or a leftover home install). When no artifact is present yet, fall back
|
|
7
|
+
* to one bounded `auto` INSTALL so shards without an installer sibling still
|
|
8
|
+
* find a file.
|
|
12
9
|
*
|
|
13
10
|
* Best-effort: exits 0 on failure (offline etc.) — the per-test gates still
|
|
14
11
|
* hard-fail under GITNEXUS_REQUIRE_FTS=1 if FTS is genuinely unavailable, which
|
|
@@ -23,12 +20,17 @@ import {
|
|
|
23
20
|
loadVectorExtension,
|
|
24
21
|
closeLbug,
|
|
25
22
|
} from '../src/core/lbug/lbug-adapter.js';
|
|
23
|
+
import { resolveVendoredFtsPath } from '../src/core/lbug/vendored-extension-path.js';
|
|
26
24
|
|
|
27
25
|
const dir = mkdtempSync(join(tmpdir(), 'gn-ensure-fts-'));
|
|
28
26
|
try {
|
|
29
27
|
await initLbug(join(dir, 'ensure-fts.lbug'));
|
|
30
|
-
|
|
31
|
-
|
|
28
|
+
if (resolveVendoredFtsPath()) {
|
|
29
|
+
console.log('FTS extension ready (vendored artifact).');
|
|
30
|
+
} else {
|
|
31
|
+
const ok = await loadFTSExtension(undefined, { policy: 'auto' });
|
|
32
|
+
console.log(ok ? 'FTS extension ready.' : 'FTS extension unavailable (continuing).');
|
|
33
|
+
}
|
|
32
34
|
// VECTOR rides the same pre-install (#2623): the win32 gate is gone, so the
|
|
33
35
|
// vector suites genuinely run on Windows/macOS — installing once here means
|
|
34
36
|
// every sharded test process LOADs from ~/.lbdb instead of racing its own
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
The files under `prebuilds/` are the LadybugDB FTS extension, redistributed
|
|
2
|
+
unchanged from https://extension.ladybugdb.com/. LadybugDB is MIT-licensed.
|
|
3
|
+
There is no upstream-published digest for these artifacts; GitNexus records
|
|
4
|
+
SHA-256 locally in `prebuilds/SHA256SUMS`.
|
|
5
|
+
|
|
6
|
+
MIT License
|
|
7
|
+
|
|
8
|
+
Copyright (c) 2022-2025 Kùzu Inc.
|
|
9
|
+
Copyright (c) 2025-2026 Ladybug Memory Inc.
|
|
10
|
+
|
|
11
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
12
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
13
|
+
in the Software without restriction, including without limitation the rights
|
|
14
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
15
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
16
|
+
furnished to do so, subject to the following conditions:
|
|
17
|
+
|
|
18
|
+
The above copyright notice and this permission notice shall be included in all
|
|
19
|
+
copies or substantial portions of the Software.
|
|
20
|
+
|
|
21
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
22
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
23
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
24
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
25
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
26
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
27
|
+
SOFTWARE.
|