d365fo-mcp 1.16.0 → 1.16.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -36,6 +36,35 @@ export interface ExtensionMetadataRecord {
36
36
  eventSubscriptions?: string[];
37
37
  model: string;
38
38
  }
39
+ /**
40
+ * Construction-time switches for the deferred file_path index builds.
41
+ *
42
+ * The defaults describe the long-running server: build in the background, treat
43
+ * anything over 200 MB as big enough to be worth a thread. A one-shot CLI wants
44
+ * the opposite of both — see scripts/build-database.ts.
45
+ */
46
+ export interface XppSymbolIndexOptions {
47
+ /**
48
+ * Dispatch large index builds to a worker thread (default true).
49
+ *
50
+ * Set false in one-shot scripts. A worker writes through a SECOND connection to
51
+ * the same file, which cannot coexist with `locking_mode = EXCLUSIVE` on the
52
+ * writer — the build script takes that lock and the worker (or the writer,
53
+ * depending on who gets there first) fails with SQLITE_BUSY. Blocking the event
54
+ * loop, the reason the worker exists at all, costs a CLI nothing.
55
+ */
56
+ backgroundIndexBuilds?: boolean;
57
+ /**
58
+ * Skip the file_path index builds during construction; the caller runs
59
+ * ensureFilePathIndexes() itself once its bulk load is done (default false).
60
+ *
61
+ * Building them up front makes a bulk load maintain two extra B-trees per row,
62
+ * and on a full rebuild the work is thrown away by clear() anyway.
63
+ */
64
+ deferFilePathIndexes?: boolean;
65
+ /** Byte size at which a DB is "large" enough to hand its index build to a worker. */
66
+ largeDbThresholdBytes?: number;
67
+ }
39
68
  export declare class XppSymbolIndex {
40
69
  db: Database;
41
70
  labelsDb: Database;
@@ -54,12 +83,16 @@ export declare class XppSymbolIndex {
54
83
  private suggestionNamesCache;
55
84
  private symbolsByTermCache;
56
85
  private perConnStmtCache;
86
+ private backgroundIndexBuilds;
87
+ private deferFilePathIndexes;
88
+ private largeDbThresholdBytes;
89
+ private pendingIndexWorkers;
57
90
  /**
58
91
  * Directory holding the metadata databases. Sibling marker files (the blob-download
59
92
  * note, the last-build record) live here so they travel with the index they describe.
60
93
  */
61
94
  get dataDir(): string;
62
- constructor(dbPath: string, labelsDbPath?: string);
95
+ constructor(dbPath: string, labelsDbPath?: string, options?: XppSymbolIndexOptions);
63
96
  /**
64
97
  * Returns the next read-only connection from the pool (round-robin).
65
98
  * Falls back to the main writer connection when the pool is empty
@@ -69,11 +102,20 @@ export declare class XppSymbolIndex {
69
102
  * to benefit from read-pool parallelism and per-connection stmt caching.
70
103
  */
71
104
  getReadDb(): Database;
105
+ /** True while a background file_path index build is still running. */
106
+ hasPendingIndexBuilds(): boolean;
72
107
  /**
73
108
  * Close and drain all read-pool connections.
74
109
  * Must be called before setting locking_mode = EXCLUSIVE on the writer
75
110
  * connection (e.g. in build scripts) — SQLite cannot grant EXCLUSIVE while
76
111
  * any other connection (even read-only, even in-process) holds a shared lock.
112
+ *
113
+ * The read pool is NOT the only other connection this class opens: an index
114
+ * build dispatched to a worker holds a writer connection to the same file, and
115
+ * this method cannot drain it (worker shutdown is asynchronous; this is not).
116
+ * Callers that take an EXCLUSIVE lock must construct with
117
+ * `backgroundIndexBuilds: false` so no such worker ever exists — the warning
118
+ * below is the tripwire for the ones that forget.
77
119
  */
78
120
  closeReadPool(): void;
79
121
  /**
@@ -150,11 +192,19 @@ export declare class XppSymbolIndex {
150
192
  * build is instant and runs here; on a large existing DB it is handed to a
151
193
  * worker thread, and until it finishes those deletes simply stay as slow as
152
194
  * they are today.
195
+ *
196
+ * Public because build scripts defer it (see XppSymbolIndexOptions) and run it
197
+ * themselves after their bulk load, with the worker dispatch turned off.
153
198
  */
154
- private ensureFilePathIndexes;
199
+ ensureFilePathIndexes(): void;
155
200
  /**
156
201
  * Build one index on a separate thread so the main event loop keeps serving.
157
- * WAL mode allows the worker's write to proceed alongside main-thread readers.
202
+ *
203
+ * The worker opens its OWN write connection to the same file, so this is only
204
+ * legal while the database stays in WAL mode and no one takes an EXCLUSIVE lock
205
+ * on it for the duration — both guaranteed by the caller (ensureFilePathIndexes
206
+ * checks the journal mode, and build scripts opt out of workers entirely).
207
+ *
158
208
  * Best-effort: a failure leaves the index absent, which is exactly the state
159
209
  * the server ran in before, so it is logged and never thrown.
160
210
  */
@@ -77,6 +77,15 @@ export class XppSymbolIndex {
77
77
  // Per-connection prepared-statement cache. Prepared statements are bound to
78
78
  // their originating connection and cannot be shared across connections.
79
79
  perConnStmtCache = new WeakMap();
80
+ // See XppSymbolIndexOptions. Held as fields so ensureFilePathIndexes() reads the
81
+ // same answer whether it runs from the constructor or from a build script later.
82
+ backgroundIndexBuilds;
83
+ deferFilePathIndexes;
84
+ largeDbThresholdBytes;
85
+ // Index-build workers still running. closeReadPool() cannot drain these (it only
86
+ // owns the read pool), so track them to warn about the EXCLUSIVE-lock race and to
87
+ // tear them down in close().
88
+ pendingIndexWorkers = new Set();
80
89
  /**
81
90
  * Directory holding the metadata databases. Sibling marker files (the blob-download
82
91
  * note, the last-build record) live here so they travel with the index they describe.
@@ -84,8 +93,11 @@ export class XppSymbolIndex {
84
93
  get dataDir() {
85
94
  return path.dirname(this.dbPath);
86
95
  }
87
- constructor(dbPath, labelsDbPath) {
96
+ constructor(dbPath, labelsDbPath, options = {}) {
88
97
  this.dbPath = dbPath;
98
+ this.backgroundIndexBuilds = options.backgroundIndexBuilds !== false;
99
+ this.deferFilePathIndexes = options.deferFilePathIndexes === true;
100
+ this.largeDbThresholdBytes = options.largeDbThresholdBytes ?? 200 * 1024 * 1024;
89
101
  // Ensure database directory exists
90
102
  const dbDir = path.dirname(dbPath);
91
103
  if (!fs.existsSync(dbDir)) {
@@ -167,13 +179,29 @@ export class XppSymbolIndex {
167
179
  return this.db;
168
180
  return this.readPool[this.readPoolRR++ % this.readPool.length];
169
181
  }
182
+ /** True while a background file_path index build is still running. */
183
+ hasPendingIndexBuilds() {
184
+ return this.pendingIndexWorkers.size > 0;
185
+ }
170
186
  /**
171
187
  * Close and drain all read-pool connections.
172
188
  * Must be called before setting locking_mode = EXCLUSIVE on the writer
173
189
  * connection (e.g. in build scripts) — SQLite cannot grant EXCLUSIVE while
174
190
  * any other connection (even read-only, even in-process) holds a shared lock.
191
+ *
192
+ * The read pool is NOT the only other connection this class opens: an index
193
+ * build dispatched to a worker holds a writer connection to the same file, and
194
+ * this method cannot drain it (worker shutdown is asynchronous; this is not).
195
+ * Callers that take an EXCLUSIVE lock must construct with
196
+ * `backgroundIndexBuilds: false` so no such worker ever exists — the warning
197
+ * below is the tripwire for the ones that forget.
175
198
  */
176
199
  closeReadPool() {
200
+ if (this.pendingIndexWorkers.size > 0) {
201
+ console.error(`[SymbolIndex] closeReadPool() with ${this.pendingIndexWorkers.size} background index ` +
202
+ `build(s) still running — these hold their own write connections and will contend ` +
203
+ `with locking_mode = EXCLUSIVE. Construct with { backgroundIndexBuilds: false }.`);
204
+ }
177
205
  for (const conn of this.readPool) {
178
206
  try {
179
207
  conn.close();
@@ -709,7 +737,9 @@ export class XppSymbolIndex {
709
737
  CREATE INDEX IF NOT EXISTS idx_md_define ON macro_defines(define_name);
710
738
  CREATE INDEX IF NOT EXISTS idx_md_model ON macro_defines(model);
711
739
  `);
712
- this.ensureFilePathIndexes();
740
+ if (!this.deferFilePathIndexes) {
741
+ this.ensureFilePathIndexes();
742
+ }
713
743
  }
714
744
  /**
715
745
  * The `labels` indexes that exist purely to accelerate reads, keyed by name so
@@ -810,6 +840,9 @@ export class XppSymbolIndex {
810
840
  * build is instant and runs here; on a large existing DB it is handed to a
811
841
  * worker thread, and until it finishes those deletes simply stay as slow as
812
842
  * they are today.
843
+ *
844
+ * Public because build scripts defer it (see XppSymbolIndexOptions) and run it
845
+ * themselves after their bulk load, with the worker dispatch turned off.
813
846
  */
814
847
  ensureFilePathIndexes() {
815
848
  const missing = (db, indexName) => !db.prepare(`SELECT 1 FROM sqlite_master WHERE type = 'index' AND name = ?`).get(indexName);
@@ -819,7 +852,7 @@ export class XppSymbolIndex {
819
852
  if (dbFile === ':memory:')
820
853
  return false;
821
854
  try {
822
- return fs.statSync(dbFile).size > 200 * 1024 * 1024;
855
+ return fs.statSync(dbFile).size > this.largeDbThresholdBytes;
823
856
  }
824
857
  catch {
825
858
  return false;
@@ -861,7 +894,12 @@ export class XppSymbolIndex {
861
894
  // .label.txt (813 on a default build), so building them is instant and needs
862
895
  // neither the deferral nor the worker dispatch the per-label table needed.
863
896
  for (const item of work) {
864
- if (isLarge(item.dbFile)) {
897
+ // A worker writes through its own connection, which only works while the
898
+ // database is in WAL mode and nothing holds an EXCLUSIVE lock. Build scripts
899
+ // switch to journal_mode = MEMORY and take that lock, so the journal mode is
900
+ // checked here rather than assumed — off WAL the only safe build is inline.
901
+ const inWal = item.db.pragma('journal_mode', { simple: true }) === 'wal';
902
+ if (isLarge(item.dbFile) && this.backgroundIndexBuilds && inWal) {
865
903
  this.buildIndexInWorker(item.dbFile, item.sql, item.name);
866
904
  }
867
905
  else {
@@ -871,7 +909,12 @@ export class XppSymbolIndex {
871
909
  }
872
910
  /**
873
911
  * Build one index on a separate thread so the main event loop keeps serving.
874
- * WAL mode allows the worker's write to proceed alongside main-thread readers.
912
+ *
913
+ * The worker opens its OWN write connection to the same file, so this is only
914
+ * legal while the database stays in WAL mode and no one takes an EXCLUSIVE lock
915
+ * on it for the duration — both guaranteed by the caller (ensureFilePathIndexes
916
+ * checks the journal mode, and build scripts opt out of workers entirely).
917
+ *
875
918
  * Best-effort: a failure leaves the index absent, which is exactly the state
876
919
  * the server ran in before, so it is logged and never thrown.
877
920
  */
@@ -880,6 +923,7 @@ export class XppSymbolIndex {
880
923
  const worker = new Worker(new URL('./buildIndexWorker.js', import.meta.url), {
881
924
  workerData: { dbPath, sql, indexName },
882
925
  });
926
+ this.pendingIndexWorkers.add(worker);
883
927
  // unref() so a pending index build never keeps the process alive on exit.
884
928
  worker.unref();
885
929
  worker.once('message', (msg) => {
@@ -889,9 +933,14 @@ export class XppSymbolIndex {
889
933
  else {
890
934
  console.error(`[SymbolIndex] Background build of ${indexName} failed: ${msg.error}`);
891
935
  }
936
+ this.pendingIndexWorkers.delete(worker);
892
937
  void worker.terminate();
893
938
  });
894
- worker.once('error', e => console.error(`[SymbolIndex] ${indexName} worker error: ${e}`));
939
+ worker.once('error', e => {
940
+ this.pendingIndexWorkers.delete(worker);
941
+ console.error(`[SymbolIndex] ${indexName} worker error: ${e}`);
942
+ });
943
+ worker.once('exit', () => this.pendingIndexWorkers.delete(worker));
895
944
  }
896
945
  catch (e) {
897
946
  console.error(`[SymbolIndex] Could not start ${indexName} worker: ${e}`);
@@ -3836,6 +3885,14 @@ export class XppSymbolIndex {
3836
3885
  console.error(`[SymbolIndex] Final labels FTS flush failed: ${e}`);
3837
3886
  this._labelsFtsTimer = null;
3838
3887
  }
3888
+ // Background index builds hold their own write connection to the same file;
3889
+ // leaving one running past close() writes into a database the owner considers
3890
+ // shut. terminate() is async and best-effort — we do not await it, we only stop
3891
+ // the build from outliving us.
3892
+ for (const worker of this.pendingIndexWorkers) {
3893
+ void worker.terminate();
3894
+ }
3895
+ this.pendingIndexWorkers.clear();
3839
3896
  // Drain read pools first — writer close will fail on WAL if readers hold a lock.
3840
3897
  this.closeReadPool();
3841
3898
  this.stmtCache.clear();
@@ -1208,6 +1208,15 @@ var XppSymbolIndex = class _XppSymbolIndex {
1208
1208
  // Per-connection prepared-statement cache. Prepared statements are bound to
1209
1209
  // their originating connection and cannot be shared across connections.
1210
1210
  perConnStmtCache = /* @__PURE__ */ new WeakMap();
1211
+ // See XppSymbolIndexOptions. Held as fields so ensureFilePathIndexes() reads the
1212
+ // same answer whether it runs from the constructor or from a build script later.
1213
+ backgroundIndexBuilds;
1214
+ deferFilePathIndexes;
1215
+ largeDbThresholdBytes;
1216
+ // Index-build workers still running. closeReadPool() cannot drain these (it only
1217
+ // owns the read pool), so track them to warn about the EXCLUSIVE-lock race and to
1218
+ // tear them down in close().
1219
+ pendingIndexWorkers = /* @__PURE__ */ new Set();
1211
1220
  /**
1212
1221
  * Directory holding the metadata databases. Sibling marker files (the blob-download
1213
1222
  * note, the last-build record) live here so they travel with the index they describe.
@@ -1215,8 +1224,11 @@ var XppSymbolIndex = class _XppSymbolIndex {
1215
1224
  get dataDir() {
1216
1225
  return path.dirname(this.dbPath);
1217
1226
  }
1218
- constructor(dbPath, labelsDbPath) {
1227
+ constructor(dbPath, labelsDbPath, options = {}) {
1219
1228
  this.dbPath = dbPath;
1229
+ this.backgroundIndexBuilds = options.backgroundIndexBuilds !== false;
1230
+ this.deferFilePathIndexes = options.deferFilePathIndexes === true;
1231
+ this.largeDbThresholdBytes = options.largeDbThresholdBytes ?? 200 * 1024 * 1024;
1220
1232
  const dbDir = path.dirname(dbPath);
1221
1233
  if (!fs2.existsSync(dbDir)) {
1222
1234
  fs2.mkdirSync(dbDir, { recursive: true });
@@ -1281,13 +1293,29 @@ var XppSymbolIndex = class _XppSymbolIndex {
1281
1293
  if (this.readPool.length === 0) return this.db;
1282
1294
  return this.readPool[this.readPoolRR++ % this.readPool.length];
1283
1295
  }
1296
+ /** True while a background file_path index build is still running. */
1297
+ hasPendingIndexBuilds() {
1298
+ return this.pendingIndexWorkers.size > 0;
1299
+ }
1284
1300
  /**
1285
1301
  * Close and drain all read-pool connections.
1286
1302
  * Must be called before setting locking_mode = EXCLUSIVE on the writer
1287
1303
  * connection (e.g. in build scripts) — SQLite cannot grant EXCLUSIVE while
1288
1304
  * any other connection (even read-only, even in-process) holds a shared lock.
1305
+ *
1306
+ * The read pool is NOT the only other connection this class opens: an index
1307
+ * build dispatched to a worker holds a writer connection to the same file, and
1308
+ * this method cannot drain it (worker shutdown is asynchronous; this is not).
1309
+ * Callers that take an EXCLUSIVE lock must construct with
1310
+ * `backgroundIndexBuilds: false` so no such worker ever exists — the warning
1311
+ * below is the tripwire for the ones that forget.
1289
1312
  */
1290
1313
  closeReadPool() {
1314
+ if (this.pendingIndexWorkers.size > 0) {
1315
+ console.error(
1316
+ `[SymbolIndex] closeReadPool() with ${this.pendingIndexWorkers.size} background index build(s) still running \u2014 these hold their own write connections and will contend with locking_mode = EXCLUSIVE. Construct with { backgroundIndexBuilds: false }.`
1317
+ );
1318
+ }
1291
1319
  for (const conn of this.readPool) {
1292
1320
  try {
1293
1321
  conn.close();
@@ -1766,7 +1794,9 @@ var XppSymbolIndex = class _XppSymbolIndex {
1766
1794
  CREATE INDEX IF NOT EXISTS idx_md_define ON macro_defines(define_name);
1767
1795
  CREATE INDEX IF NOT EXISTS idx_md_model ON macro_defines(model);
1768
1796
  `);
1769
- this.ensureFilePathIndexes();
1797
+ if (!this.deferFilePathIndexes) {
1798
+ this.ensureFilePathIndexes();
1799
+ }
1770
1800
  }
1771
1801
  /**
1772
1802
  * The `labels` indexes that exist purely to accelerate reads, keyed by name so
@@ -1860,13 +1890,16 @@ var XppSymbolIndex = class _XppSymbolIndex {
1860
1890
  * build is instant and runs here; on a large existing DB it is handed to a
1861
1891
  * worker thread, and until it finishes those deletes simply stay as slow as
1862
1892
  * they are today.
1893
+ *
1894
+ * Public because build scripts defer it (see XppSymbolIndexOptions) and run it
1895
+ * themselves after their bulk load, with the worker dispatch turned off.
1863
1896
  */
1864
1897
  ensureFilePathIndexes() {
1865
1898
  const missing = (db, indexName) => !db.prepare(`SELECT 1 FROM sqlite_master WHERE type = 'index' AND name = ?`).get(indexName);
1866
1899
  const isLarge = (dbFile) => {
1867
1900
  if (dbFile === ":memory:") return false;
1868
1901
  try {
1869
- return fs2.statSync(dbFile).size > 200 * 1024 * 1024;
1902
+ return fs2.statSync(dbFile).size > this.largeDbThresholdBytes;
1870
1903
  } catch {
1871
1904
  return false;
1872
1905
  }
@@ -1898,7 +1931,8 @@ var XppSymbolIndex = class _XppSymbolIndex {
1898
1931
  });
1899
1932
  }
1900
1933
  for (const item of work) {
1901
- if (isLarge(item.dbFile)) {
1934
+ const inWal = item.db.pragma("journal_mode", { simple: true }) === "wal";
1935
+ if (isLarge(item.dbFile) && this.backgroundIndexBuilds && inWal) {
1902
1936
  this.buildIndexInWorker(item.dbFile, item.sql, item.name);
1903
1937
  } else {
1904
1938
  item.db.exec(item.sql);
@@ -1907,7 +1941,12 @@ var XppSymbolIndex = class _XppSymbolIndex {
1907
1941
  }
1908
1942
  /**
1909
1943
  * Build one index on a separate thread so the main event loop keeps serving.
1910
- * WAL mode allows the worker's write to proceed alongside main-thread readers.
1944
+ *
1945
+ * The worker opens its OWN write connection to the same file, so this is only
1946
+ * legal while the database stays in WAL mode and no one takes an EXCLUSIVE lock
1947
+ * on it for the duration — both guaranteed by the caller (ensureFilePathIndexes
1948
+ * checks the journal mode, and build scripts opt out of workers entirely).
1949
+ *
1911
1950
  * Best-effort: a failure leaves the index absent, which is exactly the state
1912
1951
  * the server ran in before, so it is logged and never thrown.
1913
1952
  */
@@ -1916,6 +1955,7 @@ var XppSymbolIndex = class _XppSymbolIndex {
1916
1955
  const worker = new Worker(new URL("./buildIndexWorker.js", import.meta.url), {
1917
1956
  workerData: { dbPath, sql, indexName }
1918
1957
  });
1958
+ this.pendingIndexWorkers.add(worker);
1919
1959
  worker.unref();
1920
1960
  worker.once("message", (msg) => {
1921
1961
  if (msg.ok) {
@@ -1923,9 +1963,14 @@ var XppSymbolIndex = class _XppSymbolIndex {
1923
1963
  } else {
1924
1964
  console.error(`[SymbolIndex] Background build of ${indexName} failed: ${msg.error}`);
1925
1965
  }
1966
+ this.pendingIndexWorkers.delete(worker);
1926
1967
  void worker.terminate();
1927
1968
  });
1928
- worker.once("error", (e) => console.error(`[SymbolIndex] ${indexName} worker error: ${e}`));
1969
+ worker.once("error", (e) => {
1970
+ this.pendingIndexWorkers.delete(worker);
1971
+ console.error(`[SymbolIndex] ${indexName} worker error: ${e}`);
1972
+ });
1973
+ worker.once("exit", () => this.pendingIndexWorkers.delete(worker));
1929
1974
  } catch (e) {
1930
1975
  console.error(`[SymbolIndex] Could not start ${indexName} worker: ${e}`);
1931
1976
  }
@@ -4601,6 +4646,10 @@ Point the installation at a drive with room (a full index needs several GB): re-
4601
4646
  console.error(`[SymbolIndex] Final labels FTS flush failed: ${e}`);
4602
4647
  this._labelsFtsTimer = null;
4603
4648
  }
4649
+ for (const worker of this.pendingIndexWorkers) {
4650
+ void worker.terminate();
4651
+ }
4652
+ this.pendingIndexWorkers.clear();
4604
4653
  this.closeReadPool();
4605
4654
  this.stmtCache.clear();
4606
4655
  try {
@@ -7140,7 +7189,10 @@ async function buildDatabase() {
7140
7189
  console.log(kv("Labels DB", shortPath(OUTPUT_LABELS_DB)));
7141
7190
  console.log(kv("VACUUM", EXTRACT_MODE === "all" || FORCE_VACUUM ? c.green("enabled") : c.dim("disabled (incremental build)")));
7142
7191
  console.log("");
7143
- const symbolIndex = new XppSymbolIndex(OUTPUT_DB, OUTPUT_LABELS_DB);
7192
+ const symbolIndex = new XppSymbolIndex(OUTPUT_DB, OUTPUT_LABELS_DB, {
7193
+ backgroundIndexBuilds: false,
7194
+ deferFilePathIndexes: true
7195
+ });
7144
7196
  const extractManifestCustomModels = readExtractedCustomModels(INPUT_PATH);
7145
7197
  if (extractManifestCustomModels !== void 0) {
7146
7198
  symbolIndex.setNonMicrosoftModels(extractManifestCustomModels);
@@ -7345,6 +7397,11 @@ async function buildDatabase() {
7345
7397
  console.log("");
7346
7398
  log.info("Skipping label indexing (INCLUDE_LABELS=false)");
7347
7399
  }
7400
+ console.log("");
7401
+ log.step("Building file_path indexes...");
7402
+ const filePathIdxStart = Date.now();
7403
+ symbolIndex.ensureFilePathIndexes();
7404
+ log.ok(`file_path indexes built in ${((Date.now() - filePathIdxStart) / 1e3).toFixed(2)}s`);
7348
7405
  if (SKIP_FTS) {
7349
7406
  console.log("");
7350
7407
  log.info("Skipping WAL conversion (database will be finalized by build-fts step)");
@@ -1147,6 +1147,15 @@ var XppSymbolIndex = class _XppSymbolIndex {
1147
1147
  // Per-connection prepared-statement cache. Prepared statements are bound to
1148
1148
  // their originating connection and cannot be shared across connections.
1149
1149
  perConnStmtCache = /* @__PURE__ */ new WeakMap();
1150
+ // See XppSymbolIndexOptions. Held as fields so ensureFilePathIndexes() reads the
1151
+ // same answer whether it runs from the constructor or from a build script later.
1152
+ backgroundIndexBuilds;
1153
+ deferFilePathIndexes;
1154
+ largeDbThresholdBytes;
1155
+ // Index-build workers still running. closeReadPool() cannot drain these (it only
1156
+ // owns the read pool), so track them to warn about the EXCLUSIVE-lock race and to
1157
+ // tear them down in close().
1158
+ pendingIndexWorkers = /* @__PURE__ */ new Set();
1150
1159
  /**
1151
1160
  * Directory holding the metadata databases. Sibling marker files (the blob-download
1152
1161
  * note, the last-build record) live here so they travel with the index they describe.
@@ -1154,8 +1163,11 @@ var XppSymbolIndex = class _XppSymbolIndex {
1154
1163
  get dataDir() {
1155
1164
  return path.dirname(this.dbPath);
1156
1165
  }
1157
- constructor(dbPath, labelsDbPath) {
1166
+ constructor(dbPath, labelsDbPath, options = {}) {
1158
1167
  this.dbPath = dbPath;
1168
+ this.backgroundIndexBuilds = options.backgroundIndexBuilds !== false;
1169
+ this.deferFilePathIndexes = options.deferFilePathIndexes === true;
1170
+ this.largeDbThresholdBytes = options.largeDbThresholdBytes ?? 200 * 1024 * 1024;
1159
1171
  const dbDir = path.dirname(dbPath);
1160
1172
  if (!fs2.existsSync(dbDir)) {
1161
1173
  fs2.mkdirSync(dbDir, { recursive: true });
@@ -1220,13 +1232,29 @@ var XppSymbolIndex = class _XppSymbolIndex {
1220
1232
  if (this.readPool.length === 0) return this.db;
1221
1233
  return this.readPool[this.readPoolRR++ % this.readPool.length];
1222
1234
  }
1235
+ /** True while a background file_path index build is still running. */
1236
+ hasPendingIndexBuilds() {
1237
+ return this.pendingIndexWorkers.size > 0;
1238
+ }
1223
1239
  /**
1224
1240
  * Close and drain all read-pool connections.
1225
1241
  * Must be called before setting locking_mode = EXCLUSIVE on the writer
1226
1242
  * connection (e.g. in build scripts) — SQLite cannot grant EXCLUSIVE while
1227
1243
  * any other connection (even read-only, even in-process) holds a shared lock.
1244
+ *
1245
+ * The read pool is NOT the only other connection this class opens: an index
1246
+ * build dispatched to a worker holds a writer connection to the same file, and
1247
+ * this method cannot drain it (worker shutdown is asynchronous; this is not).
1248
+ * Callers that take an EXCLUSIVE lock must construct with
1249
+ * `backgroundIndexBuilds: false` so no such worker ever exists — the warning
1250
+ * below is the tripwire for the ones that forget.
1228
1251
  */
1229
1252
  closeReadPool() {
1253
+ if (this.pendingIndexWorkers.size > 0) {
1254
+ console.error(
1255
+ `[SymbolIndex] closeReadPool() with ${this.pendingIndexWorkers.size} background index build(s) still running \u2014 these hold their own write connections and will contend with locking_mode = EXCLUSIVE. Construct with { backgroundIndexBuilds: false }.`
1256
+ );
1257
+ }
1230
1258
  for (const conn of this.readPool) {
1231
1259
  try {
1232
1260
  conn.close();
@@ -1705,7 +1733,9 @@ var XppSymbolIndex = class _XppSymbolIndex {
1705
1733
  CREATE INDEX IF NOT EXISTS idx_md_define ON macro_defines(define_name);
1706
1734
  CREATE INDEX IF NOT EXISTS idx_md_model ON macro_defines(model);
1707
1735
  `);
1708
- this.ensureFilePathIndexes();
1736
+ if (!this.deferFilePathIndexes) {
1737
+ this.ensureFilePathIndexes();
1738
+ }
1709
1739
  }
1710
1740
  /**
1711
1741
  * The `labels` indexes that exist purely to accelerate reads, keyed by name so
@@ -1799,13 +1829,16 @@ var XppSymbolIndex = class _XppSymbolIndex {
1799
1829
  * build is instant and runs here; on a large existing DB it is handed to a
1800
1830
  * worker thread, and until it finishes those deletes simply stay as slow as
1801
1831
  * they are today.
1832
+ *
1833
+ * Public because build scripts defer it (see XppSymbolIndexOptions) and run it
1834
+ * themselves after their bulk load, with the worker dispatch turned off.
1802
1835
  */
1803
1836
  ensureFilePathIndexes() {
1804
1837
  const missing = (db, indexName) => !db.prepare(`SELECT 1 FROM sqlite_master WHERE type = 'index' AND name = ?`).get(indexName);
1805
1838
  const isLarge = (dbFile) => {
1806
1839
  if (dbFile === ":memory:") return false;
1807
1840
  try {
1808
- return fs2.statSync(dbFile).size > 200 * 1024 * 1024;
1841
+ return fs2.statSync(dbFile).size > this.largeDbThresholdBytes;
1809
1842
  } catch {
1810
1843
  return false;
1811
1844
  }
@@ -1837,7 +1870,8 @@ var XppSymbolIndex = class _XppSymbolIndex {
1837
1870
  });
1838
1871
  }
1839
1872
  for (const item of work) {
1840
- if (isLarge(item.dbFile)) {
1873
+ const inWal = item.db.pragma("journal_mode", { simple: true }) === "wal";
1874
+ if (isLarge(item.dbFile) && this.backgroundIndexBuilds && inWal) {
1841
1875
  this.buildIndexInWorker(item.dbFile, item.sql, item.name);
1842
1876
  } else {
1843
1877
  item.db.exec(item.sql);
@@ -1846,7 +1880,12 @@ var XppSymbolIndex = class _XppSymbolIndex {
1846
1880
  }
1847
1881
  /**
1848
1882
  * Build one index on a separate thread so the main event loop keeps serving.
1849
- * WAL mode allows the worker's write to proceed alongside main-thread readers.
1883
+ *
1884
+ * The worker opens its OWN write connection to the same file, so this is only
1885
+ * legal while the database stays in WAL mode and no one takes an EXCLUSIVE lock
1886
+ * on it for the duration — both guaranteed by the caller (ensureFilePathIndexes
1887
+ * checks the journal mode, and build scripts opt out of workers entirely).
1888
+ *
1850
1889
  * Best-effort: a failure leaves the index absent, which is exactly the state
1851
1890
  * the server ran in before, so it is logged and never thrown.
1852
1891
  */
@@ -1855,6 +1894,7 @@ var XppSymbolIndex = class _XppSymbolIndex {
1855
1894
  const worker = new Worker(new URL("./buildIndexWorker.js", import.meta.url), {
1856
1895
  workerData: { dbPath, sql, indexName }
1857
1896
  });
1897
+ this.pendingIndexWorkers.add(worker);
1858
1898
  worker.unref();
1859
1899
  worker.once("message", (msg) => {
1860
1900
  if (msg.ok) {
@@ -1862,9 +1902,14 @@ var XppSymbolIndex = class _XppSymbolIndex {
1862
1902
  } else {
1863
1903
  console.error(`[SymbolIndex] Background build of ${indexName} failed: ${msg.error}`);
1864
1904
  }
1905
+ this.pendingIndexWorkers.delete(worker);
1865
1906
  void worker.terminate();
1866
1907
  });
1867
- worker.once("error", (e) => console.error(`[SymbolIndex] ${indexName} worker error: ${e}`));
1908
+ worker.once("error", (e) => {
1909
+ this.pendingIndexWorkers.delete(worker);
1910
+ console.error(`[SymbolIndex] ${indexName} worker error: ${e}`);
1911
+ });
1912
+ worker.once("exit", () => this.pendingIndexWorkers.delete(worker));
1868
1913
  } catch (e) {
1869
1914
  console.error(`[SymbolIndex] Could not start ${indexName} worker: ${e}`);
1870
1915
  }
@@ -4540,6 +4585,10 @@ Point the installation at a drive with room (a full index needs several GB): re-
4540
4585
  console.error(`[SymbolIndex] Final labels FTS flush failed: ${e}`);
4541
4586
  this._labelsFtsTimer = null;
4542
4587
  }
4588
+ for (const worker of this.pendingIndexWorkers) {
4589
+ void worker.terminate();
4590
+ }
4591
+ this.pendingIndexWorkers.clear();
4543
4592
  this.closeReadPool();
4544
4593
  this.stmtCache.clear();
4545
4594
  try {
@@ -5228,7 +5277,10 @@ async function buildFts() {
5228
5277
  console.error(' Run "npm run build-database" (with SKIP_FTS=true) first.');
5229
5278
  process.exit(1);
5230
5279
  }
5231
- const symbolIndex = new XppSymbolIndex(OUTPUT_DB, OUTPUT_LABELS_DB);
5280
+ const symbolIndex = new XppSymbolIndex(OUTPUT_DB, OUTPUT_LABELS_DB, {
5281
+ backgroundIndexBuilds: false,
5282
+ deferFilePathIndexes: true
5283
+ });
5232
5284
  symbolIndex.closeReadPool();
5233
5285
  symbolIndex.db.pragma("journal_mode = MEMORY");
5234
5286
  symbolIndex.db.pragma("synchronous = OFF");
@@ -5289,6 +5341,10 @@ async function buildFts() {
5289
5341
  } else {
5290
5342
  console.log("\n\u23ED\uFE0F Skipping label indexing (INCLUDE_LABELS=false)");
5291
5343
  }
5344
+ console.log("\n\u{1F511} Building file_path indexes...");
5345
+ const filePathIdxStart = Date.now();
5346
+ symbolIndex.ensureFilePathIndexes();
5347
+ console.log(` \u2705 Done in ${((Date.now() - filePathIdxStart) / 1e3).toFixed(2)}s`);
5292
5348
  console.log("\n\u{1F504} Converting databases to WAL mode for production...");
5293
5349
  symbolIndex.db.pragma("locking_mode = NORMAL");
5294
5350
  symbolIndex.db.pragma("journal_mode = WAL");
@@ -1,5 +1,5 @@
1
1
  import { execFile } from 'child_process';
2
- import { parseSysTestXml } from '../../eval/oracle/systest.js';
2
+ import { parseSysTestXml } from './sysTestXml.js';
3
3
  import util from 'util';
4
4
  import path from 'path';
5
5
  import os from 'os';
@@ -0,0 +1,32 @@
1
+ /**
2
+ * Reader for the `/xml:` result document SysTestConsole writes.
3
+ *
4
+ * This lives beside the runner rather than in `src/eval/` because it is loaded
5
+ * at runtime by `sysTestRunner.ts`, and `src/eval/**` is a dev-only tree that
6
+ * package.json's `files` list keeps out of the published tarball
7
+ * (`"!dist/eval/**"`). A shipped module importing across that boundary makes
8
+ * the installed server fail to start with ERR_MODULE_NOT_FOUND. The eval
9
+ * oracle re-exports this from `src/eval/oracle/systest.ts`; the dependency
10
+ * runs eval → tools, never the other way.
11
+ */
12
+ /** One test method's outcome, read from the runner's XML result document. */
13
+ export interface SysTestCaseOutcome {
14
+ name: string;
15
+ passed: boolean;
16
+ message?: string;
17
+ }
18
+ /**
19
+ * Per-method outcomes from the `/xml:` document SysTestConsole writes.
20
+ *
21
+ * The shape is the platform's own: SysTestListenerXML builds
22
+ * `<test-results><results><test-case name="…" success="true|false">` and adds a
23
+ * `<failure><message>…</message></failure>` child for a failing one (the element
24
+ * names are #define'd at the top of that class). Reading it beats the regex over
25
+ * combined stdout that the runner used to classify with: a class named
26
+ * …ErrorHandlingTest made "error" appear in the output of a passing run, and a
27
+ * green run was reported as failed.
28
+ *
29
+ * Returns [] when the text is not such a document, so callers can fall back.
30
+ */
31
+ export declare function parseSysTestXml(xml: string | null | undefined): SysTestCaseOutcome[];
32
+ //# sourceMappingURL=sysTestXml.d.ts.map
@@ -0,0 +1,54 @@
1
+ /**
2
+ * Reader for the `/xml:` result document SysTestConsole writes.
3
+ *
4
+ * This lives beside the runner rather than in `src/eval/` because it is loaded
5
+ * at runtime by `sysTestRunner.ts`, and `src/eval/**` is a dev-only tree that
6
+ * package.json's `files` list keeps out of the published tarball
7
+ * (`"!dist/eval/**"`). A shipped module importing across that boundary makes
8
+ * the installed server fail to start with ERR_MODULE_NOT_FOUND. The eval
9
+ * oracle re-exports this from `src/eval/oracle/systest.ts`; the dependency
10
+ * runs eval → tools, never the other way.
11
+ */
12
+ /**
13
+ * Per-method outcomes from the `/xml:` document SysTestConsole writes.
14
+ *
15
+ * The shape is the platform's own: SysTestListenerXML builds
16
+ * `<test-results><results><test-case name="…" success="true|false">` and adds a
17
+ * `<failure><message>…</message></failure>` child for a failing one (the element
18
+ * names are #define'd at the top of that class). Reading it beats the regex over
19
+ * combined stdout that the runner used to classify with: a class named
20
+ * …ErrorHandlingTest made "error" appear in the output of a passing run, and a
21
+ * green run was reported as failed.
22
+ *
23
+ * Returns [] when the text is not such a document, so callers can fall back.
24
+ */
25
+ export function parseSysTestXml(xml) {
26
+ if (!xml || !/<test-case\b/i.test(xml))
27
+ return [];
28
+ const outcomes = [];
29
+ // The SELF-CLOSING form has to be tried first. With the paired form leading, its
30
+ // `[^>]*` happily consumed the `/` of `<test-case … />` and the lazy body then ran
31
+ // on to the NEXT `</test-case>`, swallowing two results into one.
32
+ const caseRe = /<test-case\b([^>]*?)\/>|<test-case\b([^>]*)>([\s\S]*?)<\/test-case>/gi;
33
+ let m;
34
+ while ((m = caseRe.exec(xml)) !== null) {
35
+ const attrs = m[1] ?? m[2] ?? '';
36
+ const body = m[3] ?? '';
37
+ const name = /\bname\s*=\s*"([^"]*)"/i.exec(attrs)?.[1] ?? '';
38
+ const successAttr = /\bsuccess\s*=\s*"([^"]*)"/i.exec(attrs)?.[1];
39
+ const failure = /<failure\b[^>]*>([\s\S]*?)<\/failure>/i.exec(body)?.[1];
40
+ const message = failure
41
+ ? (/<message\b[^>]*>([\s\S]*?)<\/message>/i.exec(failure)?.[1] ?? failure)
42
+ .replace(/<[^>]+>/g, ' ')
43
+ .replace(/\s+/g, ' ')
44
+ .trim()
45
+ : undefined;
46
+ // `success` is authoritative when present; a <failure> child decides otherwise.
47
+ const passed = successAttr !== undefined
48
+ ? /^(true|1)$/i.test(successAttr)
49
+ : failure === undefined;
50
+ outcomes.push(message ? { name, passed, message } : { name, passed });
51
+ }
52
+ return outcomes;
53
+ }
54
+ //# sourceMappingURL=sysTestXml.js.map
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "d365fo-mcp",
3
- "version": "1.16.0",
3
+ "version": "1.16.2",
4
4
  "description": "MCP Server for X++ Code Completion in D365 Finance & Operations",
5
5
  "type": "module",
6
6
  "main": "dist/index.js",