d365fo-mcp 1.16.0 → 1.16.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/metadata/symbolIndex.d.ts +53 -3
- package/dist/metadata/symbolIndex.js +63 -6
- package/dist/scripts/build-database.js +64 -7
- package/dist/scripts/build-fts.js +63 -7
- package/dist/tools/sdlc/sysTestRunner.js +1 -1
- package/dist/tools/sdlc/sysTestXml.d.ts +32 -0
- package/dist/tools/sdlc/sysTestXml.js +54 -0
- package/package.json +1 -1
|
@@ -36,6 +36,35 @@ export interface ExtensionMetadataRecord {
|
|
|
36
36
|
eventSubscriptions?: string[];
|
|
37
37
|
model: string;
|
|
38
38
|
}
|
|
39
|
+
/**
|
|
40
|
+
* Construction-time switches for the deferred file_path index builds.
|
|
41
|
+
*
|
|
42
|
+
* The defaults describe the long-running server: build in the background, treat
|
|
43
|
+
* anything over 200 MB as big enough to be worth a thread. A one-shot CLI wants
|
|
44
|
+
* the opposite of both — see scripts/build-database.ts.
|
|
45
|
+
*/
|
|
46
|
+
export interface XppSymbolIndexOptions {
|
|
47
|
+
/**
|
|
48
|
+
* Dispatch large index builds to a worker thread (default true).
|
|
49
|
+
*
|
|
50
|
+
* Set false in one-shot scripts. A worker writes through a SECOND connection to
|
|
51
|
+
* the same file, which cannot coexist with `locking_mode = EXCLUSIVE` on the
|
|
52
|
+
* writer — the build script takes that lock and the worker (or the writer,
|
|
53
|
+
* depending on who gets there first) fails with SQLITE_BUSY. Blocking the event
|
|
54
|
+
* loop, the reason the worker exists at all, costs a CLI nothing.
|
|
55
|
+
*/
|
|
56
|
+
backgroundIndexBuilds?: boolean;
|
|
57
|
+
/**
|
|
58
|
+
* Skip the file_path index builds during construction; the caller runs
|
|
59
|
+
* ensureFilePathIndexes() itself once its bulk load is done (default false).
|
|
60
|
+
*
|
|
61
|
+
* Building them up front makes a bulk load maintain two extra B-trees per row,
|
|
62
|
+
* and on a full rebuild the work is thrown away by clear() anyway.
|
|
63
|
+
*/
|
|
64
|
+
deferFilePathIndexes?: boolean;
|
|
65
|
+
/** Byte size at which a DB is "large" enough to hand its index build to a worker. */
|
|
66
|
+
largeDbThresholdBytes?: number;
|
|
67
|
+
}
|
|
39
68
|
export declare class XppSymbolIndex {
|
|
40
69
|
db: Database;
|
|
41
70
|
labelsDb: Database;
|
|
@@ -54,12 +83,16 @@ export declare class XppSymbolIndex {
|
|
|
54
83
|
private suggestionNamesCache;
|
|
55
84
|
private symbolsByTermCache;
|
|
56
85
|
private perConnStmtCache;
|
|
86
|
+
private backgroundIndexBuilds;
|
|
87
|
+
private deferFilePathIndexes;
|
|
88
|
+
private largeDbThresholdBytes;
|
|
89
|
+
private pendingIndexWorkers;
|
|
57
90
|
/**
|
|
58
91
|
* Directory holding the metadata databases. Sibling marker files (the blob-download
|
|
59
92
|
* note, the last-build record) live here so they travel with the index they describe.
|
|
60
93
|
*/
|
|
61
94
|
get dataDir(): string;
|
|
62
|
-
constructor(dbPath: string, labelsDbPath?: string);
|
|
95
|
+
constructor(dbPath: string, labelsDbPath?: string, options?: XppSymbolIndexOptions);
|
|
63
96
|
/**
|
|
64
97
|
* Returns the next read-only connection from the pool (round-robin).
|
|
65
98
|
* Falls back to the main writer connection when the pool is empty
|
|
@@ -69,11 +102,20 @@ export declare class XppSymbolIndex {
|
|
|
69
102
|
* to benefit from read-pool parallelism and per-connection stmt caching.
|
|
70
103
|
*/
|
|
71
104
|
getReadDb(): Database;
|
|
105
|
+
/** True while a background file_path index build is still running. */
|
|
106
|
+
hasPendingIndexBuilds(): boolean;
|
|
72
107
|
/**
|
|
73
108
|
* Close and drain all read-pool connections.
|
|
74
109
|
* Must be called before setting locking_mode = EXCLUSIVE on the writer
|
|
75
110
|
* connection (e.g. in build scripts) — SQLite cannot grant EXCLUSIVE while
|
|
76
111
|
* any other connection (even read-only, even in-process) holds a shared lock.
|
|
112
|
+
*
|
|
113
|
+
* The read pool is NOT the only other connection this class opens: an index
|
|
114
|
+
* build dispatched to a worker holds a writer connection to the same file, and
|
|
115
|
+
* this method cannot drain it (worker shutdown is asynchronous; this is not).
|
|
116
|
+
* Callers that take an EXCLUSIVE lock must construct with
|
|
117
|
+
* `backgroundIndexBuilds: false` so no such worker ever exists — the warning
|
|
118
|
+
* below is the tripwire for the ones that forget.
|
|
77
119
|
*/
|
|
78
120
|
closeReadPool(): void;
|
|
79
121
|
/**
|
|
@@ -150,11 +192,19 @@ export declare class XppSymbolIndex {
|
|
|
150
192
|
* build is instant and runs here; on a large existing DB it is handed to a
|
|
151
193
|
* worker thread, and until it finishes those deletes simply stay as slow as
|
|
152
194
|
* they are today.
|
|
195
|
+
*
|
|
196
|
+
* Public because build scripts defer it (see XppSymbolIndexOptions) and run it
|
|
197
|
+
* themselves after their bulk load, with the worker dispatch turned off.
|
|
153
198
|
*/
|
|
154
|
-
|
|
199
|
+
ensureFilePathIndexes(): void;
|
|
155
200
|
/**
|
|
156
201
|
* Build one index on a separate thread so the main event loop keeps serving.
|
|
157
|
-
*
|
|
202
|
+
*
|
|
203
|
+
* The worker opens its OWN write connection to the same file, so this is only
|
|
204
|
+
* legal while the database stays in WAL mode and no one takes an EXCLUSIVE lock
|
|
205
|
+
* on it for the duration — both guaranteed by the caller (ensureFilePathIndexes
|
|
206
|
+
* checks the journal mode, and build scripts opt out of workers entirely).
|
|
207
|
+
*
|
|
158
208
|
* Best-effort: a failure leaves the index absent, which is exactly the state
|
|
159
209
|
* the server ran in before, so it is logged and never thrown.
|
|
160
210
|
*/
|
|
@@ -77,6 +77,15 @@ export class XppSymbolIndex {
|
|
|
77
77
|
// Per-connection prepared-statement cache. Prepared statements are bound to
|
|
78
78
|
// their originating connection and cannot be shared across connections.
|
|
79
79
|
perConnStmtCache = new WeakMap();
|
|
80
|
+
// See XppSymbolIndexOptions. Held as fields so ensureFilePathIndexes() reads the
|
|
81
|
+
// same answer whether it runs from the constructor or from a build script later.
|
|
82
|
+
backgroundIndexBuilds;
|
|
83
|
+
deferFilePathIndexes;
|
|
84
|
+
largeDbThresholdBytes;
|
|
85
|
+
// Index-build workers still running. closeReadPool() cannot drain these (it only
|
|
86
|
+
// owns the read pool), so track them to warn about the EXCLUSIVE-lock race and to
|
|
87
|
+
// tear them down in close().
|
|
88
|
+
pendingIndexWorkers = new Set();
|
|
80
89
|
/**
|
|
81
90
|
* Directory holding the metadata databases. Sibling marker files (the blob-download
|
|
82
91
|
* note, the last-build record) live here so they travel with the index they describe.
|
|
@@ -84,8 +93,11 @@ export class XppSymbolIndex {
|
|
|
84
93
|
get dataDir() {
|
|
85
94
|
return path.dirname(this.dbPath);
|
|
86
95
|
}
|
|
87
|
-
constructor(dbPath, labelsDbPath) {
|
|
96
|
+
constructor(dbPath, labelsDbPath, options = {}) {
|
|
88
97
|
this.dbPath = dbPath;
|
|
98
|
+
this.backgroundIndexBuilds = options.backgroundIndexBuilds !== false;
|
|
99
|
+
this.deferFilePathIndexes = options.deferFilePathIndexes === true;
|
|
100
|
+
this.largeDbThresholdBytes = options.largeDbThresholdBytes ?? 200 * 1024 * 1024;
|
|
89
101
|
// Ensure database directory exists
|
|
90
102
|
const dbDir = path.dirname(dbPath);
|
|
91
103
|
if (!fs.existsSync(dbDir)) {
|
|
@@ -167,13 +179,29 @@ export class XppSymbolIndex {
|
|
|
167
179
|
return this.db;
|
|
168
180
|
return this.readPool[this.readPoolRR++ % this.readPool.length];
|
|
169
181
|
}
|
|
182
|
+
/** True while a background file_path index build is still running. */
|
|
183
|
+
hasPendingIndexBuilds() {
|
|
184
|
+
return this.pendingIndexWorkers.size > 0;
|
|
185
|
+
}
|
|
170
186
|
/**
|
|
171
187
|
* Close and drain all read-pool connections.
|
|
172
188
|
* Must be called before setting locking_mode = EXCLUSIVE on the writer
|
|
173
189
|
* connection (e.g. in build scripts) — SQLite cannot grant EXCLUSIVE while
|
|
174
190
|
* any other connection (even read-only, even in-process) holds a shared lock.
|
|
191
|
+
*
|
|
192
|
+
* The read pool is NOT the only other connection this class opens: an index
|
|
193
|
+
* build dispatched to a worker holds a writer connection to the same file, and
|
|
194
|
+
* this method cannot drain it (worker shutdown is asynchronous; this is not).
|
|
195
|
+
* Callers that take an EXCLUSIVE lock must construct with
|
|
196
|
+
* `backgroundIndexBuilds: false` so no such worker ever exists — the warning
|
|
197
|
+
* below is the tripwire for the ones that forget.
|
|
175
198
|
*/
|
|
176
199
|
closeReadPool() {
|
|
200
|
+
if (this.pendingIndexWorkers.size > 0) {
|
|
201
|
+
console.error(`[SymbolIndex] closeReadPool() with ${this.pendingIndexWorkers.size} background index ` +
|
|
202
|
+
`build(s) still running — these hold their own write connections and will contend ` +
|
|
203
|
+
`with locking_mode = EXCLUSIVE. Construct with { backgroundIndexBuilds: false }.`);
|
|
204
|
+
}
|
|
177
205
|
for (const conn of this.readPool) {
|
|
178
206
|
try {
|
|
179
207
|
conn.close();
|
|
@@ -709,7 +737,9 @@ export class XppSymbolIndex {
|
|
|
709
737
|
CREATE INDEX IF NOT EXISTS idx_md_define ON macro_defines(define_name);
|
|
710
738
|
CREATE INDEX IF NOT EXISTS idx_md_model ON macro_defines(model);
|
|
711
739
|
`);
|
|
712
|
-
this.
|
|
740
|
+
if (!this.deferFilePathIndexes) {
|
|
741
|
+
this.ensureFilePathIndexes();
|
|
742
|
+
}
|
|
713
743
|
}
|
|
714
744
|
/**
|
|
715
745
|
* The `labels` indexes that exist purely to accelerate reads, keyed by name so
|
|
@@ -810,6 +840,9 @@ export class XppSymbolIndex {
|
|
|
810
840
|
* build is instant and runs here; on a large existing DB it is handed to a
|
|
811
841
|
* worker thread, and until it finishes those deletes simply stay as slow as
|
|
812
842
|
* they are today.
|
|
843
|
+
*
|
|
844
|
+
* Public because build scripts defer it (see XppSymbolIndexOptions) and run it
|
|
845
|
+
* themselves after their bulk load, with the worker dispatch turned off.
|
|
813
846
|
*/
|
|
814
847
|
ensureFilePathIndexes() {
|
|
815
848
|
const missing = (db, indexName) => !db.prepare(`SELECT 1 FROM sqlite_master WHERE type = 'index' AND name = ?`).get(indexName);
|
|
@@ -819,7 +852,7 @@ export class XppSymbolIndex {
|
|
|
819
852
|
if (dbFile === ':memory:')
|
|
820
853
|
return false;
|
|
821
854
|
try {
|
|
822
|
-
return fs.statSync(dbFile).size >
|
|
855
|
+
return fs.statSync(dbFile).size > this.largeDbThresholdBytes;
|
|
823
856
|
}
|
|
824
857
|
catch {
|
|
825
858
|
return false;
|
|
@@ -861,7 +894,12 @@ export class XppSymbolIndex {
|
|
|
861
894
|
// .label.txt (813 on a default build), so building them is instant and needs
|
|
862
895
|
// neither the deferral nor the worker dispatch the per-label table needed.
|
|
863
896
|
for (const item of work) {
|
|
864
|
-
|
|
897
|
+
// A worker writes through its own connection, which only works while the
|
|
898
|
+
// database is in WAL mode and nothing holds an EXCLUSIVE lock. Build scripts
|
|
899
|
+
// switch to journal_mode = MEMORY and take that lock, so the journal mode is
|
|
900
|
+
// checked here rather than assumed — off WAL the only safe build is inline.
|
|
901
|
+
const inWal = item.db.pragma('journal_mode', { simple: true }) === 'wal';
|
|
902
|
+
if (isLarge(item.dbFile) && this.backgroundIndexBuilds && inWal) {
|
|
865
903
|
this.buildIndexInWorker(item.dbFile, item.sql, item.name);
|
|
866
904
|
}
|
|
867
905
|
else {
|
|
@@ -871,7 +909,12 @@ export class XppSymbolIndex {
|
|
|
871
909
|
}
|
|
872
910
|
/**
|
|
873
911
|
* Build one index on a separate thread so the main event loop keeps serving.
|
|
874
|
-
*
|
|
912
|
+
*
|
|
913
|
+
* The worker opens its OWN write connection to the same file, so this is only
|
|
914
|
+
* legal while the database stays in WAL mode and no one takes an EXCLUSIVE lock
|
|
915
|
+
* on it for the duration — both guaranteed by the caller (ensureFilePathIndexes
|
|
916
|
+
* checks the journal mode, and build scripts opt out of workers entirely).
|
|
917
|
+
*
|
|
875
918
|
* Best-effort: a failure leaves the index absent, which is exactly the state
|
|
876
919
|
* the server ran in before, so it is logged and never thrown.
|
|
877
920
|
*/
|
|
@@ -880,6 +923,7 @@ export class XppSymbolIndex {
|
|
|
880
923
|
const worker = new Worker(new URL('./buildIndexWorker.js', import.meta.url), {
|
|
881
924
|
workerData: { dbPath, sql, indexName },
|
|
882
925
|
});
|
|
926
|
+
this.pendingIndexWorkers.add(worker);
|
|
883
927
|
// unref() so a pending index build never keeps the process alive on exit.
|
|
884
928
|
worker.unref();
|
|
885
929
|
worker.once('message', (msg) => {
|
|
@@ -889,9 +933,14 @@ export class XppSymbolIndex {
|
|
|
889
933
|
else {
|
|
890
934
|
console.error(`[SymbolIndex] Background build of ${indexName} failed: ${msg.error}`);
|
|
891
935
|
}
|
|
936
|
+
this.pendingIndexWorkers.delete(worker);
|
|
892
937
|
void worker.terminate();
|
|
893
938
|
});
|
|
894
|
-
worker.once('error', e =>
|
|
939
|
+
worker.once('error', e => {
|
|
940
|
+
this.pendingIndexWorkers.delete(worker);
|
|
941
|
+
console.error(`[SymbolIndex] ${indexName} worker error: ${e}`);
|
|
942
|
+
});
|
|
943
|
+
worker.once('exit', () => this.pendingIndexWorkers.delete(worker));
|
|
895
944
|
}
|
|
896
945
|
catch (e) {
|
|
897
946
|
console.error(`[SymbolIndex] Could not start ${indexName} worker: ${e}`);
|
|
@@ -3836,6 +3885,14 @@ export class XppSymbolIndex {
|
|
|
3836
3885
|
console.error(`[SymbolIndex] Final labels FTS flush failed: ${e}`);
|
|
3837
3886
|
this._labelsFtsTimer = null;
|
|
3838
3887
|
}
|
|
3888
|
+
// Background index builds hold their own write connection to the same file;
|
|
3889
|
+
// leaving one running past close() writes into a database the owner considers
|
|
3890
|
+
// shut. terminate() is async and best-effort — we do not await it, we only stop
|
|
3891
|
+
// the build from outliving us.
|
|
3892
|
+
for (const worker of this.pendingIndexWorkers) {
|
|
3893
|
+
void worker.terminate();
|
|
3894
|
+
}
|
|
3895
|
+
this.pendingIndexWorkers.clear();
|
|
3839
3896
|
// Drain read pools first — writer close will fail on WAL if readers hold a lock.
|
|
3840
3897
|
this.closeReadPool();
|
|
3841
3898
|
this.stmtCache.clear();
|
|
@@ -1208,6 +1208,15 @@ var XppSymbolIndex = class _XppSymbolIndex {
|
|
|
1208
1208
|
// Per-connection prepared-statement cache. Prepared statements are bound to
|
|
1209
1209
|
// their originating connection and cannot be shared across connections.
|
|
1210
1210
|
perConnStmtCache = /* @__PURE__ */ new WeakMap();
|
|
1211
|
+
// See XppSymbolIndexOptions. Held as fields so ensureFilePathIndexes() reads the
|
|
1212
|
+
// same answer whether it runs from the constructor or from a build script later.
|
|
1213
|
+
backgroundIndexBuilds;
|
|
1214
|
+
deferFilePathIndexes;
|
|
1215
|
+
largeDbThresholdBytes;
|
|
1216
|
+
// Index-build workers still running. closeReadPool() cannot drain these (it only
|
|
1217
|
+
// owns the read pool), so track them to warn about the EXCLUSIVE-lock race and to
|
|
1218
|
+
// tear them down in close().
|
|
1219
|
+
pendingIndexWorkers = /* @__PURE__ */ new Set();
|
|
1211
1220
|
/**
|
|
1212
1221
|
* Directory holding the metadata databases. Sibling marker files (the blob-download
|
|
1213
1222
|
* note, the last-build record) live here so they travel with the index they describe.
|
|
@@ -1215,8 +1224,11 @@ var XppSymbolIndex = class _XppSymbolIndex {
|
|
|
1215
1224
|
get dataDir() {
|
|
1216
1225
|
return path.dirname(this.dbPath);
|
|
1217
1226
|
}
|
|
1218
|
-
constructor(dbPath, labelsDbPath) {
|
|
1227
|
+
constructor(dbPath, labelsDbPath, options = {}) {
|
|
1219
1228
|
this.dbPath = dbPath;
|
|
1229
|
+
this.backgroundIndexBuilds = options.backgroundIndexBuilds !== false;
|
|
1230
|
+
this.deferFilePathIndexes = options.deferFilePathIndexes === true;
|
|
1231
|
+
this.largeDbThresholdBytes = options.largeDbThresholdBytes ?? 200 * 1024 * 1024;
|
|
1220
1232
|
const dbDir = path.dirname(dbPath);
|
|
1221
1233
|
if (!fs2.existsSync(dbDir)) {
|
|
1222
1234
|
fs2.mkdirSync(dbDir, { recursive: true });
|
|
@@ -1281,13 +1293,29 @@ var XppSymbolIndex = class _XppSymbolIndex {
|
|
|
1281
1293
|
if (this.readPool.length === 0) return this.db;
|
|
1282
1294
|
return this.readPool[this.readPoolRR++ % this.readPool.length];
|
|
1283
1295
|
}
|
|
1296
|
+
/** True while a background file_path index build is still running. */
|
|
1297
|
+
hasPendingIndexBuilds() {
|
|
1298
|
+
return this.pendingIndexWorkers.size > 0;
|
|
1299
|
+
}
|
|
1284
1300
|
/**
|
|
1285
1301
|
* Close and drain all read-pool connections.
|
|
1286
1302
|
* Must be called before setting locking_mode = EXCLUSIVE on the writer
|
|
1287
1303
|
* connection (e.g. in build scripts) — SQLite cannot grant EXCLUSIVE while
|
|
1288
1304
|
* any other connection (even read-only, even in-process) holds a shared lock.
|
|
1305
|
+
*
|
|
1306
|
+
* The read pool is NOT the only other connection this class opens: an index
|
|
1307
|
+
* build dispatched to a worker holds a writer connection to the same file, and
|
|
1308
|
+
* this method cannot drain it (worker shutdown is asynchronous; this is not).
|
|
1309
|
+
* Callers that take an EXCLUSIVE lock must construct with
|
|
1310
|
+
* `backgroundIndexBuilds: false` so no such worker ever exists — the warning
|
|
1311
|
+
* below is the tripwire for the ones that forget.
|
|
1289
1312
|
*/
|
|
1290
1313
|
closeReadPool() {
|
|
1314
|
+
if (this.pendingIndexWorkers.size > 0) {
|
|
1315
|
+
console.error(
|
|
1316
|
+
`[SymbolIndex] closeReadPool() with ${this.pendingIndexWorkers.size} background index build(s) still running \u2014 these hold their own write connections and will contend with locking_mode = EXCLUSIVE. Construct with { backgroundIndexBuilds: false }.`
|
|
1317
|
+
);
|
|
1318
|
+
}
|
|
1291
1319
|
for (const conn of this.readPool) {
|
|
1292
1320
|
try {
|
|
1293
1321
|
conn.close();
|
|
@@ -1766,7 +1794,9 @@ var XppSymbolIndex = class _XppSymbolIndex {
|
|
|
1766
1794
|
CREATE INDEX IF NOT EXISTS idx_md_define ON macro_defines(define_name);
|
|
1767
1795
|
CREATE INDEX IF NOT EXISTS idx_md_model ON macro_defines(model);
|
|
1768
1796
|
`);
|
|
1769
|
-
this.
|
|
1797
|
+
if (!this.deferFilePathIndexes) {
|
|
1798
|
+
this.ensureFilePathIndexes();
|
|
1799
|
+
}
|
|
1770
1800
|
}
|
|
1771
1801
|
/**
|
|
1772
1802
|
* The `labels` indexes that exist purely to accelerate reads, keyed by name so
|
|
@@ -1860,13 +1890,16 @@ var XppSymbolIndex = class _XppSymbolIndex {
|
|
|
1860
1890
|
* build is instant and runs here; on a large existing DB it is handed to a
|
|
1861
1891
|
* worker thread, and until it finishes those deletes simply stay as slow as
|
|
1862
1892
|
* they are today.
|
|
1893
|
+
*
|
|
1894
|
+
* Public because build scripts defer it (see XppSymbolIndexOptions) and run it
|
|
1895
|
+
* themselves after their bulk load, with the worker dispatch turned off.
|
|
1863
1896
|
*/
|
|
1864
1897
|
ensureFilePathIndexes() {
|
|
1865
1898
|
const missing = (db, indexName) => !db.prepare(`SELECT 1 FROM sqlite_master WHERE type = 'index' AND name = ?`).get(indexName);
|
|
1866
1899
|
const isLarge = (dbFile) => {
|
|
1867
1900
|
if (dbFile === ":memory:") return false;
|
|
1868
1901
|
try {
|
|
1869
|
-
return fs2.statSync(dbFile).size >
|
|
1902
|
+
return fs2.statSync(dbFile).size > this.largeDbThresholdBytes;
|
|
1870
1903
|
} catch {
|
|
1871
1904
|
return false;
|
|
1872
1905
|
}
|
|
@@ -1898,7 +1931,8 @@ var XppSymbolIndex = class _XppSymbolIndex {
|
|
|
1898
1931
|
});
|
|
1899
1932
|
}
|
|
1900
1933
|
for (const item of work) {
|
|
1901
|
-
|
|
1934
|
+
const inWal = item.db.pragma("journal_mode", { simple: true }) === "wal";
|
|
1935
|
+
if (isLarge(item.dbFile) && this.backgroundIndexBuilds && inWal) {
|
|
1902
1936
|
this.buildIndexInWorker(item.dbFile, item.sql, item.name);
|
|
1903
1937
|
} else {
|
|
1904
1938
|
item.db.exec(item.sql);
|
|
@@ -1907,7 +1941,12 @@ var XppSymbolIndex = class _XppSymbolIndex {
|
|
|
1907
1941
|
}
|
|
1908
1942
|
/**
|
|
1909
1943
|
* Build one index on a separate thread so the main event loop keeps serving.
|
|
1910
|
-
*
|
|
1944
|
+
*
|
|
1945
|
+
* The worker opens its OWN write connection to the same file, so this is only
|
|
1946
|
+
* legal while the database stays in WAL mode and no one takes an EXCLUSIVE lock
|
|
1947
|
+
* on it for the duration — both guaranteed by the caller (ensureFilePathIndexes
|
|
1948
|
+
* checks the journal mode, and build scripts opt out of workers entirely).
|
|
1949
|
+
*
|
|
1911
1950
|
* Best-effort: a failure leaves the index absent, which is exactly the state
|
|
1912
1951
|
* the server ran in before, so it is logged and never thrown.
|
|
1913
1952
|
*/
|
|
@@ -1916,6 +1955,7 @@ var XppSymbolIndex = class _XppSymbolIndex {
|
|
|
1916
1955
|
const worker = new Worker(new URL("./buildIndexWorker.js", import.meta.url), {
|
|
1917
1956
|
workerData: { dbPath, sql, indexName }
|
|
1918
1957
|
});
|
|
1958
|
+
this.pendingIndexWorkers.add(worker);
|
|
1919
1959
|
worker.unref();
|
|
1920
1960
|
worker.once("message", (msg) => {
|
|
1921
1961
|
if (msg.ok) {
|
|
@@ -1923,9 +1963,14 @@ var XppSymbolIndex = class _XppSymbolIndex {
|
|
|
1923
1963
|
} else {
|
|
1924
1964
|
console.error(`[SymbolIndex] Background build of ${indexName} failed: ${msg.error}`);
|
|
1925
1965
|
}
|
|
1966
|
+
this.pendingIndexWorkers.delete(worker);
|
|
1926
1967
|
void worker.terminate();
|
|
1927
1968
|
});
|
|
1928
|
-
worker.once("error", (e) =>
|
|
1969
|
+
worker.once("error", (e) => {
|
|
1970
|
+
this.pendingIndexWorkers.delete(worker);
|
|
1971
|
+
console.error(`[SymbolIndex] ${indexName} worker error: ${e}`);
|
|
1972
|
+
});
|
|
1973
|
+
worker.once("exit", () => this.pendingIndexWorkers.delete(worker));
|
|
1929
1974
|
} catch (e) {
|
|
1930
1975
|
console.error(`[SymbolIndex] Could not start ${indexName} worker: ${e}`);
|
|
1931
1976
|
}
|
|
@@ -4601,6 +4646,10 @@ Point the installation at a drive with room (a full index needs several GB): re-
|
|
|
4601
4646
|
console.error(`[SymbolIndex] Final labels FTS flush failed: ${e}`);
|
|
4602
4647
|
this._labelsFtsTimer = null;
|
|
4603
4648
|
}
|
|
4649
|
+
for (const worker of this.pendingIndexWorkers) {
|
|
4650
|
+
void worker.terminate();
|
|
4651
|
+
}
|
|
4652
|
+
this.pendingIndexWorkers.clear();
|
|
4604
4653
|
this.closeReadPool();
|
|
4605
4654
|
this.stmtCache.clear();
|
|
4606
4655
|
try {
|
|
@@ -7140,7 +7189,10 @@ async function buildDatabase() {
|
|
|
7140
7189
|
console.log(kv("Labels DB", shortPath(OUTPUT_LABELS_DB)));
|
|
7141
7190
|
console.log(kv("VACUUM", EXTRACT_MODE === "all" || FORCE_VACUUM ? c.green("enabled") : c.dim("disabled (incremental build)")));
|
|
7142
7191
|
console.log("");
|
|
7143
|
-
const symbolIndex = new XppSymbolIndex(OUTPUT_DB, OUTPUT_LABELS_DB
|
|
7192
|
+
const symbolIndex = new XppSymbolIndex(OUTPUT_DB, OUTPUT_LABELS_DB, {
|
|
7193
|
+
backgroundIndexBuilds: false,
|
|
7194
|
+
deferFilePathIndexes: true
|
|
7195
|
+
});
|
|
7144
7196
|
const extractManifestCustomModels = readExtractedCustomModels(INPUT_PATH);
|
|
7145
7197
|
if (extractManifestCustomModels !== void 0) {
|
|
7146
7198
|
symbolIndex.setNonMicrosoftModels(extractManifestCustomModels);
|
|
@@ -7345,6 +7397,11 @@ async function buildDatabase() {
|
|
|
7345
7397
|
console.log("");
|
|
7346
7398
|
log.info("Skipping label indexing (INCLUDE_LABELS=false)");
|
|
7347
7399
|
}
|
|
7400
|
+
console.log("");
|
|
7401
|
+
log.step("Building file_path indexes...");
|
|
7402
|
+
const filePathIdxStart = Date.now();
|
|
7403
|
+
symbolIndex.ensureFilePathIndexes();
|
|
7404
|
+
log.ok(`file_path indexes built in ${((Date.now() - filePathIdxStart) / 1e3).toFixed(2)}s`);
|
|
7348
7405
|
if (SKIP_FTS) {
|
|
7349
7406
|
console.log("");
|
|
7350
7407
|
log.info("Skipping WAL conversion (database will be finalized by build-fts step)");
|
|
@@ -1147,6 +1147,15 @@ var XppSymbolIndex = class _XppSymbolIndex {
|
|
|
1147
1147
|
// Per-connection prepared-statement cache. Prepared statements are bound to
|
|
1148
1148
|
// their originating connection and cannot be shared across connections.
|
|
1149
1149
|
perConnStmtCache = /* @__PURE__ */ new WeakMap();
|
|
1150
|
+
// See XppSymbolIndexOptions. Held as fields so ensureFilePathIndexes() reads the
|
|
1151
|
+
// same answer whether it runs from the constructor or from a build script later.
|
|
1152
|
+
backgroundIndexBuilds;
|
|
1153
|
+
deferFilePathIndexes;
|
|
1154
|
+
largeDbThresholdBytes;
|
|
1155
|
+
// Index-build workers still running. closeReadPool() cannot drain these (it only
|
|
1156
|
+
// owns the read pool), so track them to warn about the EXCLUSIVE-lock race and to
|
|
1157
|
+
// tear them down in close().
|
|
1158
|
+
pendingIndexWorkers = /* @__PURE__ */ new Set();
|
|
1150
1159
|
/**
|
|
1151
1160
|
* Directory holding the metadata databases. Sibling marker files (the blob-download
|
|
1152
1161
|
* note, the last-build record) live here so they travel with the index they describe.
|
|
@@ -1154,8 +1163,11 @@ var XppSymbolIndex = class _XppSymbolIndex {
|
|
|
1154
1163
|
get dataDir() {
|
|
1155
1164
|
return path.dirname(this.dbPath);
|
|
1156
1165
|
}
|
|
1157
|
-
constructor(dbPath, labelsDbPath) {
|
|
1166
|
+
constructor(dbPath, labelsDbPath, options = {}) {
|
|
1158
1167
|
this.dbPath = dbPath;
|
|
1168
|
+
this.backgroundIndexBuilds = options.backgroundIndexBuilds !== false;
|
|
1169
|
+
this.deferFilePathIndexes = options.deferFilePathIndexes === true;
|
|
1170
|
+
this.largeDbThresholdBytes = options.largeDbThresholdBytes ?? 200 * 1024 * 1024;
|
|
1159
1171
|
const dbDir = path.dirname(dbPath);
|
|
1160
1172
|
if (!fs2.existsSync(dbDir)) {
|
|
1161
1173
|
fs2.mkdirSync(dbDir, { recursive: true });
|
|
@@ -1220,13 +1232,29 @@ var XppSymbolIndex = class _XppSymbolIndex {
|
|
|
1220
1232
|
if (this.readPool.length === 0) return this.db;
|
|
1221
1233
|
return this.readPool[this.readPoolRR++ % this.readPool.length];
|
|
1222
1234
|
}
|
|
1235
|
+
/** True while a background file_path index build is still running. */
|
|
1236
|
+
hasPendingIndexBuilds() {
|
|
1237
|
+
return this.pendingIndexWorkers.size > 0;
|
|
1238
|
+
}
|
|
1223
1239
|
/**
|
|
1224
1240
|
* Close and drain all read-pool connections.
|
|
1225
1241
|
* Must be called before setting locking_mode = EXCLUSIVE on the writer
|
|
1226
1242
|
* connection (e.g. in build scripts) — SQLite cannot grant EXCLUSIVE while
|
|
1227
1243
|
* any other connection (even read-only, even in-process) holds a shared lock.
|
|
1244
|
+
*
|
|
1245
|
+
* The read pool is NOT the only other connection this class opens: an index
|
|
1246
|
+
* build dispatched to a worker holds a writer connection to the same file, and
|
|
1247
|
+
* this method cannot drain it (worker shutdown is asynchronous; this is not).
|
|
1248
|
+
* Callers that take an EXCLUSIVE lock must construct with
|
|
1249
|
+
* `backgroundIndexBuilds: false` so no such worker ever exists — the warning
|
|
1250
|
+
* below is the tripwire for the ones that forget.
|
|
1228
1251
|
*/
|
|
1229
1252
|
closeReadPool() {
|
|
1253
|
+
if (this.pendingIndexWorkers.size > 0) {
|
|
1254
|
+
console.error(
|
|
1255
|
+
`[SymbolIndex] closeReadPool() with ${this.pendingIndexWorkers.size} background index build(s) still running \u2014 these hold their own write connections and will contend with locking_mode = EXCLUSIVE. Construct with { backgroundIndexBuilds: false }.`
|
|
1256
|
+
);
|
|
1257
|
+
}
|
|
1230
1258
|
for (const conn of this.readPool) {
|
|
1231
1259
|
try {
|
|
1232
1260
|
conn.close();
|
|
@@ -1705,7 +1733,9 @@ var XppSymbolIndex = class _XppSymbolIndex {
|
|
|
1705
1733
|
CREATE INDEX IF NOT EXISTS idx_md_define ON macro_defines(define_name);
|
|
1706
1734
|
CREATE INDEX IF NOT EXISTS idx_md_model ON macro_defines(model);
|
|
1707
1735
|
`);
|
|
1708
|
-
this.
|
|
1736
|
+
if (!this.deferFilePathIndexes) {
|
|
1737
|
+
this.ensureFilePathIndexes();
|
|
1738
|
+
}
|
|
1709
1739
|
}
|
|
1710
1740
|
/**
|
|
1711
1741
|
* The `labels` indexes that exist purely to accelerate reads, keyed by name so
|
|
@@ -1799,13 +1829,16 @@ var XppSymbolIndex = class _XppSymbolIndex {
|
|
|
1799
1829
|
* build is instant and runs here; on a large existing DB it is handed to a
|
|
1800
1830
|
* worker thread, and until it finishes those deletes simply stay as slow as
|
|
1801
1831
|
* they are today.
|
|
1832
|
+
*
|
|
1833
|
+
* Public because build scripts defer it (see XppSymbolIndexOptions) and run it
|
|
1834
|
+
* themselves after their bulk load, with the worker dispatch turned off.
|
|
1802
1835
|
*/
|
|
1803
1836
|
ensureFilePathIndexes() {
|
|
1804
1837
|
const missing = (db, indexName) => !db.prepare(`SELECT 1 FROM sqlite_master WHERE type = 'index' AND name = ?`).get(indexName);
|
|
1805
1838
|
const isLarge = (dbFile) => {
|
|
1806
1839
|
if (dbFile === ":memory:") return false;
|
|
1807
1840
|
try {
|
|
1808
|
-
return fs2.statSync(dbFile).size >
|
|
1841
|
+
return fs2.statSync(dbFile).size > this.largeDbThresholdBytes;
|
|
1809
1842
|
} catch {
|
|
1810
1843
|
return false;
|
|
1811
1844
|
}
|
|
@@ -1837,7 +1870,8 @@ var XppSymbolIndex = class _XppSymbolIndex {
|
|
|
1837
1870
|
});
|
|
1838
1871
|
}
|
|
1839
1872
|
for (const item of work) {
|
|
1840
|
-
|
|
1873
|
+
const inWal = item.db.pragma("journal_mode", { simple: true }) === "wal";
|
|
1874
|
+
if (isLarge(item.dbFile) && this.backgroundIndexBuilds && inWal) {
|
|
1841
1875
|
this.buildIndexInWorker(item.dbFile, item.sql, item.name);
|
|
1842
1876
|
} else {
|
|
1843
1877
|
item.db.exec(item.sql);
|
|
@@ -1846,7 +1880,12 @@ var XppSymbolIndex = class _XppSymbolIndex {
|
|
|
1846
1880
|
}
|
|
1847
1881
|
/**
|
|
1848
1882
|
* Build one index on a separate thread so the main event loop keeps serving.
|
|
1849
|
-
*
|
|
1883
|
+
*
|
|
1884
|
+
* The worker opens its OWN write connection to the same file, so this is only
|
|
1885
|
+
* legal while the database stays in WAL mode and no one takes an EXCLUSIVE lock
|
|
1886
|
+
* on it for the duration — both guaranteed by the caller (ensureFilePathIndexes
|
|
1887
|
+
* checks the journal mode, and build scripts opt out of workers entirely).
|
|
1888
|
+
*
|
|
1850
1889
|
* Best-effort: a failure leaves the index absent, which is exactly the state
|
|
1851
1890
|
* the server ran in before, so it is logged and never thrown.
|
|
1852
1891
|
*/
|
|
@@ -1855,6 +1894,7 @@ var XppSymbolIndex = class _XppSymbolIndex {
|
|
|
1855
1894
|
const worker = new Worker(new URL("./buildIndexWorker.js", import.meta.url), {
|
|
1856
1895
|
workerData: { dbPath, sql, indexName }
|
|
1857
1896
|
});
|
|
1897
|
+
this.pendingIndexWorkers.add(worker);
|
|
1858
1898
|
worker.unref();
|
|
1859
1899
|
worker.once("message", (msg) => {
|
|
1860
1900
|
if (msg.ok) {
|
|
@@ -1862,9 +1902,14 @@ var XppSymbolIndex = class _XppSymbolIndex {
|
|
|
1862
1902
|
} else {
|
|
1863
1903
|
console.error(`[SymbolIndex] Background build of ${indexName} failed: ${msg.error}`);
|
|
1864
1904
|
}
|
|
1905
|
+
this.pendingIndexWorkers.delete(worker);
|
|
1865
1906
|
void worker.terminate();
|
|
1866
1907
|
});
|
|
1867
|
-
worker.once("error", (e) =>
|
|
1908
|
+
worker.once("error", (e) => {
|
|
1909
|
+
this.pendingIndexWorkers.delete(worker);
|
|
1910
|
+
console.error(`[SymbolIndex] ${indexName} worker error: ${e}`);
|
|
1911
|
+
});
|
|
1912
|
+
worker.once("exit", () => this.pendingIndexWorkers.delete(worker));
|
|
1868
1913
|
} catch (e) {
|
|
1869
1914
|
console.error(`[SymbolIndex] Could not start ${indexName} worker: ${e}`);
|
|
1870
1915
|
}
|
|
@@ -4540,6 +4585,10 @@ Point the installation at a drive with room (a full index needs several GB): re-
|
|
|
4540
4585
|
console.error(`[SymbolIndex] Final labels FTS flush failed: ${e}`);
|
|
4541
4586
|
this._labelsFtsTimer = null;
|
|
4542
4587
|
}
|
|
4588
|
+
for (const worker of this.pendingIndexWorkers) {
|
|
4589
|
+
void worker.terminate();
|
|
4590
|
+
}
|
|
4591
|
+
this.pendingIndexWorkers.clear();
|
|
4543
4592
|
this.closeReadPool();
|
|
4544
4593
|
this.stmtCache.clear();
|
|
4545
4594
|
try {
|
|
@@ -5228,7 +5277,10 @@ async function buildFts() {
|
|
|
5228
5277
|
console.error(' Run "npm run build-database" (with SKIP_FTS=true) first.');
|
|
5229
5278
|
process.exit(1);
|
|
5230
5279
|
}
|
|
5231
|
-
const symbolIndex = new XppSymbolIndex(OUTPUT_DB, OUTPUT_LABELS_DB
|
|
5280
|
+
const symbolIndex = new XppSymbolIndex(OUTPUT_DB, OUTPUT_LABELS_DB, {
|
|
5281
|
+
backgroundIndexBuilds: false,
|
|
5282
|
+
deferFilePathIndexes: true
|
|
5283
|
+
});
|
|
5232
5284
|
symbolIndex.closeReadPool();
|
|
5233
5285
|
symbolIndex.db.pragma("journal_mode = MEMORY");
|
|
5234
5286
|
symbolIndex.db.pragma("synchronous = OFF");
|
|
@@ -5289,6 +5341,10 @@ async function buildFts() {
|
|
|
5289
5341
|
} else {
|
|
5290
5342
|
console.log("\n\u23ED\uFE0F Skipping label indexing (INCLUDE_LABELS=false)");
|
|
5291
5343
|
}
|
|
5344
|
+
console.log("\n\u{1F511} Building file_path indexes...");
|
|
5345
|
+
const filePathIdxStart = Date.now();
|
|
5346
|
+
symbolIndex.ensureFilePathIndexes();
|
|
5347
|
+
console.log(` \u2705 Done in ${((Date.now() - filePathIdxStart) / 1e3).toFixed(2)}s`);
|
|
5292
5348
|
console.log("\n\u{1F504} Converting databases to WAL mode for production...");
|
|
5293
5349
|
symbolIndex.db.pragma("locking_mode = NORMAL");
|
|
5294
5350
|
symbolIndex.db.pragma("journal_mode = WAL");
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Reader for the `/xml:` result document SysTestConsole writes.
|
|
3
|
+
*
|
|
4
|
+
* This lives beside the runner rather than in `src/eval/` because it is loaded
|
|
5
|
+
* at runtime by `sysTestRunner.ts`, and `src/eval/**` is a dev-only tree that
|
|
6
|
+
* package.json's `files` list keeps out of the published tarball
|
|
7
|
+
* (`"!dist/eval/**"`). A shipped module importing across that boundary makes
|
|
8
|
+
* the installed server fail to start with ERR_MODULE_NOT_FOUND. The eval
|
|
9
|
+
* oracle re-exports this from `src/eval/oracle/systest.ts`; the dependency
|
|
10
|
+
* runs eval → tools, never the other way.
|
|
11
|
+
*/
|
|
12
|
+
/** One test method's outcome, read from the runner's XML result document. */
|
|
13
|
+
export interface SysTestCaseOutcome {
|
|
14
|
+
name: string;
|
|
15
|
+
passed: boolean;
|
|
16
|
+
message?: string;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Per-method outcomes from the `/xml:` document SysTestConsole writes.
|
|
20
|
+
*
|
|
21
|
+
* The shape is the platform's own: SysTestListenerXML builds
|
|
22
|
+
* `<test-results><results><test-case name="…" success="true|false">` and adds a
|
|
23
|
+
* `<failure><message>…</message></failure>` child for a failing one (the element
|
|
24
|
+
* names are #define'd at the top of that class). Reading it beats the regex over
|
|
25
|
+
* combined stdout that the runner used to classify with: a class named
|
|
26
|
+
* …ErrorHandlingTest made "error" appear in the output of a passing run, and a
|
|
27
|
+
* green run was reported as failed.
|
|
28
|
+
*
|
|
29
|
+
* Returns [] when the text is not such a document, so callers can fall back.
|
|
30
|
+
*/
|
|
31
|
+
export declare function parseSysTestXml(xml: string | null | undefined): SysTestCaseOutcome[];
|
|
32
|
+
//# sourceMappingURL=sysTestXml.d.ts.map
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Reader for the `/xml:` result document SysTestConsole writes.
|
|
3
|
+
*
|
|
4
|
+
* This lives beside the runner rather than in `src/eval/` because it is loaded
|
|
5
|
+
* at runtime by `sysTestRunner.ts`, and `src/eval/**` is a dev-only tree that
|
|
6
|
+
* package.json's `files` list keeps out of the published tarball
|
|
7
|
+
* (`"!dist/eval/**"`). A shipped module importing across that boundary makes
|
|
8
|
+
* the installed server fail to start with ERR_MODULE_NOT_FOUND. The eval
|
|
9
|
+
* oracle re-exports this from `src/eval/oracle/systest.ts`; the dependency
|
|
10
|
+
* runs eval → tools, never the other way.
|
|
11
|
+
*/
|
|
12
|
+
/**
|
|
13
|
+
* Per-method outcomes from the `/xml:` document SysTestConsole writes.
|
|
14
|
+
*
|
|
15
|
+
* The shape is the platform's own: SysTestListenerXML builds
|
|
16
|
+
* `<test-results><results><test-case name="…" success="true|false">` and adds a
|
|
17
|
+
* `<failure><message>…</message></failure>` child for a failing one (the element
|
|
18
|
+
* names are #define'd at the top of that class). Reading it beats the regex over
|
|
19
|
+
* combined stdout that the runner used to classify with: a class named
|
|
20
|
+
* …ErrorHandlingTest made "error" appear in the output of a passing run, and a
|
|
21
|
+
* green run was reported as failed.
|
|
22
|
+
*
|
|
23
|
+
* Returns [] when the text is not such a document, so callers can fall back.
|
|
24
|
+
*/
|
|
25
|
+
export function parseSysTestXml(xml) {
|
|
26
|
+
if (!xml || !/<test-case\b/i.test(xml))
|
|
27
|
+
return [];
|
|
28
|
+
const outcomes = [];
|
|
29
|
+
// The SELF-CLOSING form has to be tried first. With the paired form leading, its
|
|
30
|
+
// `[^>]*` happily consumed the `/` of `<test-case … />` and the lazy body then ran
|
|
31
|
+
// on to the NEXT `</test-case>`, swallowing two results into one.
|
|
32
|
+
const caseRe = /<test-case\b([^>]*?)\/>|<test-case\b([^>]*)>([\s\S]*?)<\/test-case>/gi;
|
|
33
|
+
let m;
|
|
34
|
+
while ((m = caseRe.exec(xml)) !== null) {
|
|
35
|
+
const attrs = m[1] ?? m[2] ?? '';
|
|
36
|
+
const body = m[3] ?? '';
|
|
37
|
+
const name = /\bname\s*=\s*"([^"]*)"/i.exec(attrs)?.[1] ?? '';
|
|
38
|
+
const successAttr = /\bsuccess\s*=\s*"([^"]*)"/i.exec(attrs)?.[1];
|
|
39
|
+
const failure = /<failure\b[^>]*>([\s\S]*?)<\/failure>/i.exec(body)?.[1];
|
|
40
|
+
const message = failure
|
|
41
|
+
? (/<message\b[^>]*>([\s\S]*?)<\/message>/i.exec(failure)?.[1] ?? failure)
|
|
42
|
+
.replace(/<[^>]+>/g, ' ')
|
|
43
|
+
.replace(/\s+/g, ' ')
|
|
44
|
+
.trim()
|
|
45
|
+
: undefined;
|
|
46
|
+
// `success` is authoritative when present; a <failure> child decides otherwise.
|
|
47
|
+
const passed = successAttr !== undefined
|
|
48
|
+
? /^(true|1)$/i.test(successAttr)
|
|
49
|
+
: failure === undefined;
|
|
50
|
+
outcomes.push(message ? { name, passed, message } : { name, passed });
|
|
51
|
+
}
|
|
52
|
+
return outcomes;
|
|
53
|
+
}
|
|
54
|
+
//# sourceMappingURL=sysTestXml.js.map
|