d365fo-mcp 1.15.0 → 1.16.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. package/dist/bridge/bridgeAdapter.d.ts +33 -1
  2. package/dist/bridge/bridgeAdapter.js +50 -3
  3. package/dist/cli/commands/bpCatalog.js +5 -2
  4. package/dist/knowledge/kernelEnums.js +27 -0
  5. package/dist/metadata/symbolIndex.d.ts +53 -3
  6. package/dist/metadata/symbolIndex.js +63 -6
  7. package/dist/scripts/build-database.js +64 -7
  8. package/dist/scripts/build-fts.js +63 -7
  9. package/dist/scripts/buildIndexWorker.js +149 -0
  10. package/dist/scripts/indexWarmupWorker.js +201 -0
  11. package/dist/server/toolSchemas/prepare.js +9 -3
  12. package/dist/server/toolSchemas/validateCode.js +2 -2
  13. package/dist/tools/analysis/validateXpp.d.ts +2 -1
  14. package/dist/tools/analysis/validateXpp.js +352 -6
  15. package/dist/tools/knowledge/bpMonikerHelp.js +22 -5
  16. package/dist/tools/knowledge/xppKnowledge.js +231 -6
  17. package/dist/tools/prepare/prepareCreate.d.ts +5 -0
  18. package/dist/tools/prepare/prepareCreate.js +196 -10
  19. package/dist/tools/prepare/prepareTest.js +58 -10
  20. package/dist/tools/readers/formInfo.js +130 -7
  21. package/dist/tools/smart/codeGen.js +166 -7
  22. package/dist/tools/smart/generateSmartForm.js +85 -15
  23. package/dist/tools/smart/generateSmartReport.js +18 -2
  24. package/dist/tools/specs/d365foFileOpSpecs.js +20 -6
  25. package/dist/tools/specs/generateObjectOpSpecs.js +23 -4
  26. package/dist/tools/specs/getFormPatternSpec.js +1 -0
  27. package/dist/tools/write/createD365File.js +76 -21
  28. package/dist/tools/write/createLabel.js +4 -1
  29. package/dist/tools/write/directXmlWriters.d.ts +28 -5
  30. package/dist/tools/write/directXmlWriters.js +56 -10
  31. package/dist/tools/write/inlineWriteVerification.d.ts +19 -0
  32. package/dist/tools/write/inlineWriteVerification.js +37 -0
  33. package/dist/tools/write/modifyD365File.d.ts +1 -0
  34. package/dist/tools/write/modifyD365File.js +44 -3
  35. package/dist/tools/write/objectIdentityGate.d.ts +71 -0
  36. package/dist/tools/write/objectIdentityGate.js +126 -0
  37. package/dist/tools/write/preserveMetadataElements.d.ts +81 -0
  38. package/dist/tools/write/preserveMetadataElements.js +217 -0
  39. package/dist/tools/write/resolveReferences.js +28 -0
  40. package/dist/tools/xml/createTablePropertyHonesty.d.ts +2 -0
  41. package/dist/tools/xml/createTablePropertyHonesty.js +24 -1
  42. package/dist/tools/xml/xmlTemplateGenerator.js +33 -4
  43. package/dist/utils/axEnumProperties.d.ts +31 -21
  44. package/dist/utils/axEnumProperties.js +48 -46
  45. package/dist/utils/formPatternTemplates.d.ts +27 -0
  46. package/dist/utils/formPatternTemplates.js +40 -14
  47. package/dist/utils/modelClassifier.js +19 -0
  48. package/dist/utils/objectNamingRules.d.ts +12 -0
  49. package/dist/utils/objectNamingRules.js +74 -17
  50. package/dist/utils/symbolLookup.d.ts +18 -0
  51. package/dist/utils/symbolLookup.js +41 -0
  52. package/dist/utils/xmlScan.d.ts +50 -0
  53. package/dist/utils/xmlScan.js +88 -0
  54. package/dist/utils/xppLexer.d.ts +3 -1
  55. package/dist/utils/xppLexer.js +3 -1
  56. package/dist/validation/formControlElementOrder.d.ts +86 -0
  57. package/dist/validation/formControlElementOrder.generated.d.ts +13 -0
  58. package/dist/validation/formControlElementOrder.generated.js +1705 -0
  59. package/dist/validation/formControlElementOrder.js +227 -0
  60. package/package.json +8 -3
  61. package/scripts/extract-bp-catalog.ps1 +52 -13
@@ -22,7 +22,7 @@
22
22
  */
23
23
  import type { BridgeClient } from './bridgeClient.js';
24
24
  import type { BridgeAttempt } from './bridgeFailure.js';
25
- import type { BridgeSmartTableResult } from './bridgeTypes.js';
25
+ import type { BridgeCompletionResult, BridgeSmartTableResult } from './bridgeTypes.js';
26
26
  /** Standard MCP tool response shape */
27
27
  export interface ToolResult {
28
28
  content: Array<{
@@ -538,7 +538,39 @@ export declare function tryBridgeSecurityArtifact(bridge: BridgeClient | undefin
538
538
  export declare function tryBridgeMenuItem(bridge: BridgeClient | undefined, name: string, itemType?: string): Promise<ToolResult | null>;
539
539
  export declare function tryBridgeTableExtensions(bridge: BridgeClient | undefined, baseTableName: string): Promise<ToolResult | null>;
540
540
  export declare function tryBridgeCompletion(bridge: BridgeClient | undefined, symbolName: string, prefix?: string, ancestors?: string[]): Promise<ToolResult | null>;
541
+ /**
542
+ * One line of a member list — NAME first, always.
543
+ *
544
+ * This used to print the signature INSTEAD of the name whenever a signature
545
+ * existed, which made a member unidentifiable the moment the signature was
546
+ * wrong. It is sometimes wrong: the bridge's extractor hands back the line
547
+ * preceding the body when a macro sits between the doc block and the signature,
548
+ * so `SysQuery.range` was listed as `#ISOCountryRegionCodes`. Filtering runs on
549
+ * the NAME, so asking for prefix "range" returned "1 member" that appeared to be
550
+ * something else entirely — and a member list that renders a public API as
551
+ * another string argues the API does not exist. An eval run concluded exactly
552
+ * that and worked around a method that was there all along.
553
+ *
554
+ * The signature is still shown, as detail, and only when it plausibly belongs to
555
+ * this member. Fixing the extractor is a separate, C#-side change; this makes the
556
+ * output honest either way.
557
+ */
558
+ declare function memberLine(m: {
559
+ name: string;
560
+ signature?: string;
561
+ }): string;
562
+ declare function formatCompletion(r: BridgeCompletionResult, prefix?: string): string;
541
563
  export declare function tryBridgeCocExtensions(bridge: BridgeClient | undefined, baseClassName: string, methodName?: string): Promise<ToolResult | null>;
542
564
  export declare function tryBridgeEventHandlers(bridge: BridgeClient | undefined, targetName: string, eventName?: string, handlerType?: string): Promise<ToolResult | null>;
543
565
  export declare function tryBridgeApiUsageCallers(bridge: BridgeClient | undefined, apiName: string, limit?: number): Promise<ToolResult | null>;
566
+ /**
567
+ * Internals exposed for tests only. `formatCompletion` renders a member list,
568
+ * and a member list that cannot be trusted to name its members is worse than no
569
+ * list at all — see tests/bridge/completionMemberLine.test.ts.
570
+ */
571
+ export declare const __testing: {
572
+ formatCompletion: typeof formatCompletion;
573
+ memberLine: typeof memberLine;
574
+ };
575
+ export {};
544
576
  //# sourceMappingURL=bridgeAdapter.d.ts.map
@@ -758,7 +758,23 @@ export async function tryBridgeSearch(bridge, query, objectType, maxResults = 50
758
758
  if (!sr)
759
759
  return null;
760
760
  // Splice in exact matches the bridge's truncated window missed (#15).
761
- const bridgeHits = sr.results ?? [];
761
+ //
762
+ // …but first, honour the type filter the caller asked for, because the bridge
763
+ // does not always honour it. Its C#-side type map has no entry for every AOT
764
+ // kind — `report` is one — and an unmapped type runs the query UNFILTERED, so
765
+ // `search(type="report")` answered with tables and queries and looked
766
+ // authoritative doing it ("Cust" returned six tables). A report reached the
767
+ // caller only when the SQLite exact-name splice happened to carry one, which
768
+ // is why the failure looked intermittent rather than total.
769
+ //
770
+ // Only the BRIDGE's hits are filtered. The spliced exact and custom matches
771
+ // are deliberately allowed to differ in type — a `table-extension` is a
772
+ // wanted answer to a `type="table"` search for the table it extends — and
773
+ // filtering those too would delete that behaviour.
774
+ const rawBridgeHits = sr.results ?? [];
775
+ const bridgeHits = objectType && objectType !== 'all'
776
+ ? rawBridgeHits.filter(r => r.type === objectType)
777
+ : rawBridgeHits;
762
778
  const known = new Set(bridgeHits.map(r => `${r.name.toLowerCase()}\0${r.type}`));
763
779
  const spliced = [];
764
780
  for (const cand of opts?.exactMatches ?? []) {
@@ -2437,6 +2453,30 @@ export async function tryBridgeCompletion(bridge, symbolName, prefix, ancestors)
2437
2453
  return null;
2438
2454
  }
2439
2455
  }
2456
+ /**
2457
+ * One line of a member list — NAME first, always.
2458
+ *
2459
+ * This used to print the signature INSTEAD of the name whenever a signature
2460
+ * existed, which made a member unidentifiable the moment the signature was
2461
+ * wrong. It is sometimes wrong: the bridge's extractor hands back the line
2462
+ * preceding the body when a macro sits between the doc block and the signature,
2463
+ * so `SysQuery.range` was listed as `#ISOCountryRegionCodes`. Filtering runs on
2464
+ * the NAME, so asking for prefix "range" returned "1 member" that appeared to be
2465
+ * something else entirely — and a member list that renders a public API as
2466
+ * another string argues the API does not exist. An eval run concluded exactly
2467
+ * that and worked around a method that was there all along.
2468
+ *
2469
+ * The signature is still shown, as detail, and only when it plausibly belongs to
2470
+ * this member. Fixing the extractor is a separate, C#-side change; this makes the
2471
+ * output honest either way.
2472
+ */
2473
+ function memberLine(m) {
2474
+ const sig = m.signature?.trim();
2475
+ const belongs = sig && new RegExp(`\\b${m.name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\s*\\(`, 'i').test(sig);
2476
+ if (belongs)
2477
+ return `\`${sig}\``;
2478
+ return sig ? `**${m.name}** _(signature unavailable)_` : `**${m.name}**`;
2479
+ }
2440
2480
  function formatCompletion(r, prefix) {
2441
2481
  let members = r.members;
2442
2482
  if (prefix) {
@@ -2458,8 +2498,9 @@ function formatCompletion(r, prefix) {
2458
2498
  const inheritedCount = methodMembers.filter(m => m.inheritedFrom).length;
2459
2499
  out += `## Methods (${methodMembers.length})\n`;
2460
2500
  for (const m of methodMembers) {
2461
- const body = m.signature ? `\`${m.signature}\`` : m.name;
2462
- out += m.inheritedFrom ? `- ${body} _(inherited from ${m.inheritedFrom})_\n` : `- ${body}\n`;
2501
+ out += m.inheritedFrom
2502
+ ? `- ${memberLine(m)} _(inherited from ${m.inheritedFrom})_\n`
2503
+ : `- ${memberLine(m)}\n`;
2463
2504
  }
2464
2505
  if (inheritedCount > 0) {
2465
2506
  out += `\n> ${inheritedCount} of these are inherited — callable on ${r.symbolName}, but ` +
@@ -2602,4 +2643,10 @@ function formatApiUsageCallers(r) {
2602
2643
  }
2603
2644
  return out;
2604
2645
  }
2646
+ /**
2647
+ * Internals exposed for tests only. `formatCompletion` renders a member list,
2648
+ * and a member list that cannot be trusted to name its members is worse than no
2649
+ * list at all — see tests/bridge/completionMemberLine.test.ts.
2650
+ */
2651
+ export const __testing = { formatCompletion, memberLine };
2605
2652
  //# sourceMappingURL=bridgeAdapter.js.map
@@ -18,7 +18,7 @@
18
18
  */
19
19
  import { createHash } from 'node:crypto';
20
20
  import { existsSync, readFileSync, readdirSync, renameSync, rmSync, statSync } from 'node:fs';
21
- import { basename, join, resolve } from 'node:path';
21
+ import { join, resolve } from 'node:path';
22
22
  import { settingByPath } from '../../config/settings.js';
23
23
  import { findPackagesRoot } from '../../utils/packagesRoot.js';
24
24
  import { commandExists, runExe } from '../exec.js';
@@ -243,7 +243,10 @@ async function refreshCatalog(target, deps) {
243
243
  return;
244
244
  }
245
245
  }
246
- p.log.step(`Refreshing BP moniker catalog (${target.label}, ${basename(source.packagesPath)})…`);
246
+ // No packages path in this line: the script's own report opens with the
247
+ // full path, and the tail of it ('PackagesLocalDirectory') identified
248
+ // nothing on a box with several installs.
249
+ p.log.step(`Refreshing BP moniker catalog (${target.label})…`);
247
250
  // Extract beside the real catalog, not over it. The script writes its output
248
251
  // in one go with no staging of its own, so pointing it straight at the live
249
252
  // file means a partial run destroys a good catalog before anything can judge
@@ -50,6 +50,33 @@ const ENTRIES = [
50
50
  'Integer', 'List', 'RString', 'Real', 'Record', 'String', 'Time',
51
51
  'UserType', 'UtcDateTime', 'VarArg', 'VarString', 'Void',
52
52
  ] },
53
+ // What Box::yesNo/confirm/okCancel answer with. Found by the knowledge audit,
54
+ // which reported "DialogButton does not exist in the symbol index" for a topic
55
+ // whose call shape had already compiled in an xppc probe — the same
56
+ // absent-from-the-AOT-is-not-absent trap this module exists for. Values are the
57
+ // ones shipped code actually uses, counted over 30,000 AxClass/AxForm files:
58
+ // Yes 602, No 561, Cancel 154, Ok 129, YesToAll 7, NoToAll 6.
59
+ { name: 'DialogButton', values: ['Cancel', 'No', 'NoToAll', 'Ok', 'Yes', 'YesToAll'] },
60
+ // Added 2026-08-31 after an eval run tripped over AccessType and MenuItemType:
61
+ // validate_code(mode="references") called both hard ERRORS on code that
62
+ // compiles, and with GROUNDING_ENFORCE on it would have REFUSED the write —
63
+ // the exact failure this module was created to prevent. That both were absent
64
+ // was the signal that the list had never been derived from anything, so it is
65
+ // now checked by `npm run oracle:kernel-enums`, which finds every `Name::Value`
66
+ // in shipped X++ with no AOT element behind it. Values below are what the
67
+ // product itself writes, counted by that audit, not recalled.
68
+ { name: 'MenuItemType', values: ['Action', 'Display', 'Output'] },
69
+ { name: 'AccessType', values: ['Add', 'Correction', 'Delete', 'Edit', 'NoAccess', 'View'] },
70
+ { name: 'AccessRight', values: ['Add', 'Correction', 'Delete', 'Edit', 'NoAccess', 'View'] },
71
+ { name: 'SortOrder', values: ['Ascending', 'Descending'] },
72
+ // QueryBuildDataSource.joinMode(). Reported by a second eval run, through the
73
+ // static-member rule rather than the declared-type one — the two rules did not
74
+ // share the kernel-name exemption, so the same fact was a warning in one place
75
+ // and an error in the other.
76
+ { name: 'JoinMode', values: ['ExistsJoin', 'InnerJoin', 'NoExistsJoin', 'OuterJoin'] },
77
+ // The argument to an `unchecked(...)` block. Only these two members exist —
78
+ // DBSchemaDrift and Deadlock read plausibly and are not real (xppc-verified).
79
+ { name: 'Uncheck', values: ['TableSecurityPermission', 'XDS'] },
53
80
  { name: 'TableScope', values: ['CurrentTableOnly', 'IncludeBaseTables', 'IncludeDerivedTables'] },
54
81
  { name: 'ConcurrencyModel', values: ['Auto', 'Optimistic', 'Pessimistic'] },
55
82
  { name: 'StatementType', values: ['Delete', 'Insert', 'Select', 'Update'] },
@@ -36,6 +36,35 @@ export interface ExtensionMetadataRecord {
36
36
  eventSubscriptions?: string[];
37
37
  model: string;
38
38
  }
39
+ /**
40
+ * Construction-time switches for the deferred file_path index builds.
41
+ *
42
+ * The defaults describe the long-running server: build in the background, treat
43
+ * anything over 200 MB as big enough to be worth a thread. A one-shot CLI wants
44
+ * the opposite of both — see scripts/build-database.ts.
45
+ */
46
+ export interface XppSymbolIndexOptions {
47
+ /**
48
+ * Dispatch large index builds to a worker thread (default true).
49
+ *
50
+ * Set false in one-shot scripts. A worker writes through a SECOND connection to
51
+ * the same file, which cannot coexist with `locking_mode = EXCLUSIVE` on the
52
+ * writer — the build script takes that lock and the worker (or the writer,
53
+ * depending on who gets there first) fails with SQLITE_BUSY. Blocking the event
54
+ * loop, the reason the worker exists at all, costs a CLI nothing.
55
+ */
56
+ backgroundIndexBuilds?: boolean;
57
+ /**
58
+ * Skip the file_path index builds during construction; the caller runs
59
+ * ensureFilePathIndexes() itself once its bulk load is done (default false).
60
+ *
61
+ * Building them up front makes a bulk load maintain two extra B-trees per row,
62
+ * and on a full rebuild the work is thrown away by clear() anyway.
63
+ */
64
+ deferFilePathIndexes?: boolean;
65
+ /** Byte size at which a DB is "large" enough to hand its index build to a worker. */
66
+ largeDbThresholdBytes?: number;
67
+ }
39
68
  export declare class XppSymbolIndex {
40
69
  db: Database;
41
70
  labelsDb: Database;
@@ -54,12 +83,16 @@ export declare class XppSymbolIndex {
54
83
  private suggestionNamesCache;
55
84
  private symbolsByTermCache;
56
85
  private perConnStmtCache;
86
+ private backgroundIndexBuilds;
87
+ private deferFilePathIndexes;
88
+ private largeDbThresholdBytes;
89
+ private pendingIndexWorkers;
57
90
  /**
58
91
  * Directory holding the metadata databases. Sibling marker files (the blob-download
59
92
  * note, the last-build record) live here so they travel with the index they describe.
60
93
  */
61
94
  get dataDir(): string;
62
- constructor(dbPath: string, labelsDbPath?: string);
95
+ constructor(dbPath: string, labelsDbPath?: string, options?: XppSymbolIndexOptions);
63
96
  /**
64
97
  * Returns the next read-only connection from the pool (round-robin).
65
98
  * Falls back to the main writer connection when the pool is empty
@@ -69,11 +102,20 @@ export declare class XppSymbolIndex {
69
102
  * to benefit from read-pool parallelism and per-connection stmt caching.
70
103
  */
71
104
  getReadDb(): Database;
105
+ /** True while a background file_path index build is still running. */
106
+ hasPendingIndexBuilds(): boolean;
72
107
  /**
73
108
  * Close and drain all read-pool connections.
74
109
  * Must be called before setting locking_mode = EXCLUSIVE on the writer
75
110
  * connection (e.g. in build scripts) — SQLite cannot grant EXCLUSIVE while
76
111
  * any other connection (even read-only, even in-process) holds a shared lock.
112
+ *
113
+ * The read pool is NOT the only other connection this class opens: an index
114
+ * build dispatched to a worker holds a writer connection to the same file, and
115
+ * this method cannot drain it (worker shutdown is asynchronous; this is not).
116
+ * Callers that take an EXCLUSIVE lock must construct with
117
+ * `backgroundIndexBuilds: false` so no such worker ever exists — the warning
118
+ * below is the tripwire for the ones that forget.
77
119
  */
78
120
  closeReadPool(): void;
79
121
  /**
@@ -150,11 +192,19 @@ export declare class XppSymbolIndex {
150
192
  * build is instant and runs here; on a large existing DB it is handed to a
151
193
  * worker thread, and until it finishes those deletes simply stay as slow as
152
194
  * they are today.
195
+ *
196
+ * Public because build scripts defer it (see XppSymbolIndexOptions) and run it
197
+ * themselves after their bulk load, with the worker dispatch turned off.
153
198
  */
154
- private ensureFilePathIndexes;
199
+ ensureFilePathIndexes(): void;
155
200
  /**
156
201
  * Build one index on a separate thread so the main event loop keeps serving.
157
- * WAL mode allows the worker's write to proceed alongside main-thread readers.
202
+ *
203
+ * The worker opens its OWN write connection to the same file, so this is only
204
+ * legal while the database stays in WAL mode and no one takes an EXCLUSIVE lock
205
+ * on it for the duration — both guaranteed by the caller (ensureFilePathIndexes
206
+ * checks the journal mode, and build scripts opt out of workers entirely).
207
+ *
158
208
  * Best-effort: a failure leaves the index absent, which is exactly the state
159
209
  * the server ran in before, so it is logged and never thrown.
160
210
  */
@@ -77,6 +77,15 @@ export class XppSymbolIndex {
77
77
  // Per-connection prepared-statement cache. Prepared statements are bound to
78
78
  // their originating connection and cannot be shared across connections.
79
79
  perConnStmtCache = new WeakMap();
80
+ // See XppSymbolIndexOptions. Held as fields so ensureFilePathIndexes() reads the
81
+ // same answer whether it runs from the constructor or from a build script later.
82
+ backgroundIndexBuilds;
83
+ deferFilePathIndexes;
84
+ largeDbThresholdBytes;
85
+ // Index-build workers still running. closeReadPool() cannot drain these (it only
86
+ // owns the read pool), so track them to warn about the EXCLUSIVE-lock race and to
87
+ // tear them down in close().
88
+ pendingIndexWorkers = new Set();
80
89
  /**
81
90
  * Directory holding the metadata databases. Sibling marker files (the blob-download
82
91
  * note, the last-build record) live here so they travel with the index they describe.
@@ -84,8 +93,11 @@ export class XppSymbolIndex {
84
93
  get dataDir() {
85
94
  return path.dirname(this.dbPath);
86
95
  }
87
- constructor(dbPath, labelsDbPath) {
96
+ constructor(dbPath, labelsDbPath, options = {}) {
88
97
  this.dbPath = dbPath;
98
+ this.backgroundIndexBuilds = options.backgroundIndexBuilds !== false;
99
+ this.deferFilePathIndexes = options.deferFilePathIndexes === true;
100
+ this.largeDbThresholdBytes = options.largeDbThresholdBytes ?? 200 * 1024 * 1024;
89
101
  // Ensure database directory exists
90
102
  const dbDir = path.dirname(dbPath);
91
103
  if (!fs.existsSync(dbDir)) {
@@ -167,13 +179,29 @@ export class XppSymbolIndex {
167
179
  return this.db;
168
180
  return this.readPool[this.readPoolRR++ % this.readPool.length];
169
181
  }
182
+ /** True while a background file_path index build is still running. */
183
+ hasPendingIndexBuilds() {
184
+ return this.pendingIndexWorkers.size > 0;
185
+ }
170
186
  /**
171
187
  * Close and drain all read-pool connections.
172
188
  * Must be called before setting locking_mode = EXCLUSIVE on the writer
173
189
  * connection (e.g. in build scripts) — SQLite cannot grant EXCLUSIVE while
174
190
  * any other connection (even read-only, even in-process) holds a shared lock.
191
+ *
192
+ * The read pool is NOT the only other connection this class opens: an index
193
+ * build dispatched to a worker holds a writer connection to the same file, and
194
+ * this method cannot drain it (worker shutdown is asynchronous; this is not).
195
+ * Callers that take an EXCLUSIVE lock must construct with
196
+ * `backgroundIndexBuilds: false` so no such worker ever exists — the warning
197
+ * below is the tripwire for the ones that forget.
175
198
  */
176
199
  closeReadPool() {
200
+ if (this.pendingIndexWorkers.size > 0) {
201
+ console.error(`[SymbolIndex] closeReadPool() with ${this.pendingIndexWorkers.size} background index ` +
202
+ `build(s) still running — these hold their own write connections and will contend ` +
203
+ `with locking_mode = EXCLUSIVE. Construct with { backgroundIndexBuilds: false }.`);
204
+ }
177
205
  for (const conn of this.readPool) {
178
206
  try {
179
207
  conn.close();
@@ -709,7 +737,9 @@ export class XppSymbolIndex {
709
737
  CREATE INDEX IF NOT EXISTS idx_md_define ON macro_defines(define_name);
710
738
  CREATE INDEX IF NOT EXISTS idx_md_model ON macro_defines(model);
711
739
  `);
712
- this.ensureFilePathIndexes();
740
+ if (!this.deferFilePathIndexes) {
741
+ this.ensureFilePathIndexes();
742
+ }
713
743
  }
714
744
  /**
715
745
  * The `labels` indexes that exist purely to accelerate reads, keyed by name so
@@ -810,6 +840,9 @@ export class XppSymbolIndex {
810
840
  * build is instant and runs here; on a large existing DB it is handed to a
811
841
  * worker thread, and until it finishes those deletes simply stay as slow as
812
842
  * they are today.
843
+ *
844
+ * Public because build scripts defer it (see XppSymbolIndexOptions) and run it
845
+ * themselves after their bulk load, with the worker dispatch turned off.
813
846
  */
814
847
  ensureFilePathIndexes() {
815
848
  const missing = (db, indexName) => !db.prepare(`SELECT 1 FROM sqlite_master WHERE type = 'index' AND name = ?`).get(indexName);
@@ -819,7 +852,7 @@ export class XppSymbolIndex {
819
852
  if (dbFile === ':memory:')
820
853
  return false;
821
854
  try {
822
- return fs.statSync(dbFile).size > 200 * 1024 * 1024;
855
+ return fs.statSync(dbFile).size > this.largeDbThresholdBytes;
823
856
  }
824
857
  catch {
825
858
  return false;
@@ -861,7 +894,12 @@ export class XppSymbolIndex {
861
894
  // .label.txt (813 on a default build), so building them is instant and needs
862
895
  // neither the deferral nor the worker dispatch the per-label table needed.
863
896
  for (const item of work) {
864
- if (isLarge(item.dbFile)) {
897
+ // A worker writes through its own connection, which only works while the
898
+ // database is in WAL mode and nothing holds an EXCLUSIVE lock. Build scripts
899
+ // switch to journal_mode = MEMORY and take that lock, so the journal mode is
900
+ // checked here rather than assumed — off WAL the only safe build is inline.
901
+ const inWal = item.db.pragma('journal_mode', { simple: true }) === 'wal';
902
+ if (isLarge(item.dbFile) && this.backgroundIndexBuilds && inWal) {
865
903
  this.buildIndexInWorker(item.dbFile, item.sql, item.name);
866
904
  }
867
905
  else {
@@ -871,7 +909,12 @@ export class XppSymbolIndex {
871
909
  }
872
910
  /**
873
911
  * Build one index on a separate thread so the main event loop keeps serving.
874
- * WAL mode allows the worker's write to proceed alongside main-thread readers.
912
+ *
913
+ * The worker opens its OWN write connection to the same file, so this is only
914
+ * legal while the database stays in WAL mode and no one takes an EXCLUSIVE lock
915
+ * on it for the duration — both guaranteed by the caller (ensureFilePathIndexes
916
+ * checks the journal mode, and build scripts opt out of workers entirely).
917
+ *
875
918
  * Best-effort: a failure leaves the index absent, which is exactly the state
876
919
  * the server ran in before, so it is logged and never thrown.
877
920
  */
@@ -880,6 +923,7 @@ export class XppSymbolIndex {
880
923
  const worker = new Worker(new URL('./buildIndexWorker.js', import.meta.url), {
881
924
  workerData: { dbPath, sql, indexName },
882
925
  });
926
+ this.pendingIndexWorkers.add(worker);
883
927
  // unref() so a pending index build never keeps the process alive on exit.
884
928
  worker.unref();
885
929
  worker.once('message', (msg) => {
@@ -889,9 +933,14 @@ export class XppSymbolIndex {
889
933
  else {
890
934
  console.error(`[SymbolIndex] Background build of ${indexName} failed: ${msg.error}`);
891
935
  }
936
+ this.pendingIndexWorkers.delete(worker);
892
937
  void worker.terminate();
893
938
  });
894
- worker.once('error', e => console.error(`[SymbolIndex] ${indexName} worker error: ${e}`));
939
+ worker.once('error', e => {
940
+ this.pendingIndexWorkers.delete(worker);
941
+ console.error(`[SymbolIndex] ${indexName} worker error: ${e}`);
942
+ });
943
+ worker.once('exit', () => this.pendingIndexWorkers.delete(worker));
895
944
  }
896
945
  catch (e) {
897
946
  console.error(`[SymbolIndex] Could not start ${indexName} worker: ${e}`);
@@ -3836,6 +3885,14 @@ export class XppSymbolIndex {
3836
3885
  console.error(`[SymbolIndex] Final labels FTS flush failed: ${e}`);
3837
3886
  this._labelsFtsTimer = null;
3838
3887
  }
3888
+ // Background index builds hold their own write connection to the same file;
3889
+ // leaving one running past close() writes into a database the owner considers
3890
+ // shut. terminate() is async and best-effort — we do not await it, we only stop
3891
+ // the build from outliving us.
3892
+ for (const worker of this.pendingIndexWorkers) {
3893
+ void worker.terminate();
3894
+ }
3895
+ this.pendingIndexWorkers.clear();
3839
3896
  // Drain read pools first — writer close will fail on WAL if readers hold a lock.
3840
3897
  this.closeReadPool();
3841
3898
  this.stmtCache.clear();
@@ -1208,6 +1208,15 @@ var XppSymbolIndex = class _XppSymbolIndex {
1208
1208
  // Per-connection prepared-statement cache. Prepared statements are bound to
1209
1209
  // their originating connection and cannot be shared across connections.
1210
1210
  perConnStmtCache = /* @__PURE__ */ new WeakMap();
1211
+ // See XppSymbolIndexOptions. Held as fields so ensureFilePathIndexes() reads the
1212
+ // same answer whether it runs from the constructor or from a build script later.
1213
+ backgroundIndexBuilds;
1214
+ deferFilePathIndexes;
1215
+ largeDbThresholdBytes;
1216
+ // Index-build workers still running. closeReadPool() cannot drain these (it only
1217
+ // owns the read pool), so track them to warn about the EXCLUSIVE-lock race and to
1218
+ // tear them down in close().
1219
+ pendingIndexWorkers = /* @__PURE__ */ new Set();
1211
1220
  /**
1212
1221
  * Directory holding the metadata databases. Sibling marker files (the blob-download
1213
1222
  * note, the last-build record) live here so they travel with the index they describe.
@@ -1215,8 +1224,11 @@ var XppSymbolIndex = class _XppSymbolIndex {
1215
1224
  get dataDir() {
1216
1225
  return path.dirname(this.dbPath);
1217
1226
  }
1218
- constructor(dbPath, labelsDbPath) {
1227
+ constructor(dbPath, labelsDbPath, options = {}) {
1219
1228
  this.dbPath = dbPath;
1229
+ this.backgroundIndexBuilds = options.backgroundIndexBuilds !== false;
1230
+ this.deferFilePathIndexes = options.deferFilePathIndexes === true;
1231
+ this.largeDbThresholdBytes = options.largeDbThresholdBytes ?? 200 * 1024 * 1024;
1220
1232
  const dbDir = path.dirname(dbPath);
1221
1233
  if (!fs2.existsSync(dbDir)) {
1222
1234
  fs2.mkdirSync(dbDir, { recursive: true });
@@ -1281,13 +1293,29 @@ var XppSymbolIndex = class _XppSymbolIndex {
1281
1293
  if (this.readPool.length === 0) return this.db;
1282
1294
  return this.readPool[this.readPoolRR++ % this.readPool.length];
1283
1295
  }
1296
+ /** True while a background file_path index build is still running. */
1297
+ hasPendingIndexBuilds() {
1298
+ return this.pendingIndexWorkers.size > 0;
1299
+ }
1284
1300
  /**
1285
1301
  * Close and drain all read-pool connections.
1286
1302
  * Must be called before setting locking_mode = EXCLUSIVE on the writer
1287
1303
  * connection (e.g. in build scripts) — SQLite cannot grant EXCLUSIVE while
1288
1304
  * any other connection (even read-only, even in-process) holds a shared lock.
1305
+ *
1306
+ * The read pool is NOT the only other connection this class opens: an index
1307
+ * build dispatched to a worker holds a writer connection to the same file, and
1308
+ * this method cannot drain it (worker shutdown is asynchronous; this is not).
1309
+ * Callers that take an EXCLUSIVE lock must construct with
1310
+ * `backgroundIndexBuilds: false` so no such worker ever exists — the warning
1311
+ * below is the tripwire for the ones that forget.
1289
1312
  */
1290
1313
  closeReadPool() {
1314
+ if (this.pendingIndexWorkers.size > 0) {
1315
+ console.error(
1316
+ `[SymbolIndex] closeReadPool() with ${this.pendingIndexWorkers.size} background index build(s) still running \u2014 these hold their own write connections and will contend with locking_mode = EXCLUSIVE. Construct with { backgroundIndexBuilds: false }.`
1317
+ );
1318
+ }
1291
1319
  for (const conn of this.readPool) {
1292
1320
  try {
1293
1321
  conn.close();
@@ -1766,7 +1794,9 @@ var XppSymbolIndex = class _XppSymbolIndex {
1766
1794
  CREATE INDEX IF NOT EXISTS idx_md_define ON macro_defines(define_name);
1767
1795
  CREATE INDEX IF NOT EXISTS idx_md_model ON macro_defines(model);
1768
1796
  `);
1769
- this.ensureFilePathIndexes();
1797
+ if (!this.deferFilePathIndexes) {
1798
+ this.ensureFilePathIndexes();
1799
+ }
1770
1800
  }
1771
1801
  /**
1772
1802
  * The `labels` indexes that exist purely to accelerate reads, keyed by name so
@@ -1860,13 +1890,16 @@ var XppSymbolIndex = class _XppSymbolIndex {
1860
1890
  * build is instant and runs here; on a large existing DB it is handed to a
1861
1891
  * worker thread, and until it finishes those deletes simply stay as slow as
1862
1892
  * they are today.
1893
+ *
1894
+ * Public because build scripts defer it (see XppSymbolIndexOptions) and run it
1895
+ * themselves after their bulk load, with the worker dispatch turned off.
1863
1896
  */
1864
1897
  ensureFilePathIndexes() {
1865
1898
  const missing = (db, indexName) => !db.prepare(`SELECT 1 FROM sqlite_master WHERE type = 'index' AND name = ?`).get(indexName);
1866
1899
  const isLarge = (dbFile) => {
1867
1900
  if (dbFile === ":memory:") return false;
1868
1901
  try {
1869
- return fs2.statSync(dbFile).size > 200 * 1024 * 1024;
1902
+ return fs2.statSync(dbFile).size > this.largeDbThresholdBytes;
1870
1903
  } catch {
1871
1904
  return false;
1872
1905
  }
@@ -1898,7 +1931,8 @@ var XppSymbolIndex = class _XppSymbolIndex {
1898
1931
  });
1899
1932
  }
1900
1933
  for (const item of work) {
1901
- if (isLarge(item.dbFile)) {
1934
+ const inWal = item.db.pragma("journal_mode", { simple: true }) === "wal";
1935
+ if (isLarge(item.dbFile) && this.backgroundIndexBuilds && inWal) {
1902
1936
  this.buildIndexInWorker(item.dbFile, item.sql, item.name);
1903
1937
  } else {
1904
1938
  item.db.exec(item.sql);
@@ -1907,7 +1941,12 @@ var XppSymbolIndex = class _XppSymbolIndex {
1907
1941
  }
1908
1942
  /**
1909
1943
  * Build one index on a separate thread so the main event loop keeps serving.
1910
- * WAL mode allows the worker's write to proceed alongside main-thread readers.
1944
+ *
1945
+ * The worker opens its OWN write connection to the same file, so this is only
1946
+ * legal while the database stays in WAL mode and no one takes an EXCLUSIVE lock
1947
+ * on it for the duration — both guaranteed by the caller (ensureFilePathIndexes
1948
+ * checks the journal mode, and build scripts opt out of workers entirely).
1949
+ *
1911
1950
  * Best-effort: a failure leaves the index absent, which is exactly the state
1912
1951
  * the server ran in before, so it is logged and never thrown.
1913
1952
  */
@@ -1916,6 +1955,7 @@ var XppSymbolIndex = class _XppSymbolIndex {
1916
1955
  const worker = new Worker(new URL("./buildIndexWorker.js", import.meta.url), {
1917
1956
  workerData: { dbPath, sql, indexName }
1918
1957
  });
1958
+ this.pendingIndexWorkers.add(worker);
1919
1959
  worker.unref();
1920
1960
  worker.once("message", (msg) => {
1921
1961
  if (msg.ok) {
@@ -1923,9 +1963,14 @@ var XppSymbolIndex = class _XppSymbolIndex {
1923
1963
  } else {
1924
1964
  console.error(`[SymbolIndex] Background build of ${indexName} failed: ${msg.error}`);
1925
1965
  }
1966
+ this.pendingIndexWorkers.delete(worker);
1926
1967
  void worker.terminate();
1927
1968
  });
1928
- worker.once("error", (e) => console.error(`[SymbolIndex] ${indexName} worker error: ${e}`));
1969
+ worker.once("error", (e) => {
1970
+ this.pendingIndexWorkers.delete(worker);
1971
+ console.error(`[SymbolIndex] ${indexName} worker error: ${e}`);
1972
+ });
1973
+ worker.once("exit", () => this.pendingIndexWorkers.delete(worker));
1929
1974
  } catch (e) {
1930
1975
  console.error(`[SymbolIndex] Could not start ${indexName} worker: ${e}`);
1931
1976
  }
@@ -4601,6 +4646,10 @@ Point the installation at a drive with room (a full index needs several GB): re-
4601
4646
  console.error(`[SymbolIndex] Final labels FTS flush failed: ${e}`);
4602
4647
  this._labelsFtsTimer = null;
4603
4648
  }
4649
+ for (const worker of this.pendingIndexWorkers) {
4650
+ void worker.terminate();
4651
+ }
4652
+ this.pendingIndexWorkers.clear();
4604
4653
  this.closeReadPool();
4605
4654
  this.stmtCache.clear();
4606
4655
  try {
@@ -7140,7 +7189,10 @@ async function buildDatabase() {
7140
7189
  console.log(kv("Labels DB", shortPath(OUTPUT_LABELS_DB)));
7141
7190
  console.log(kv("VACUUM", EXTRACT_MODE === "all" || FORCE_VACUUM ? c.green("enabled") : c.dim("disabled (incremental build)")));
7142
7191
  console.log("");
7143
- const symbolIndex = new XppSymbolIndex(OUTPUT_DB, OUTPUT_LABELS_DB);
7192
+ const symbolIndex = new XppSymbolIndex(OUTPUT_DB, OUTPUT_LABELS_DB, {
7193
+ backgroundIndexBuilds: false,
7194
+ deferFilePathIndexes: true
7195
+ });
7144
7196
  const extractManifestCustomModels = readExtractedCustomModels(INPUT_PATH);
7145
7197
  if (extractManifestCustomModels !== void 0) {
7146
7198
  symbolIndex.setNonMicrosoftModels(extractManifestCustomModels);
@@ -7345,6 +7397,11 @@ async function buildDatabase() {
7345
7397
  console.log("");
7346
7398
  log.info("Skipping label indexing (INCLUDE_LABELS=false)");
7347
7399
  }
7400
+ console.log("");
7401
+ log.step("Building file_path indexes...");
7402
+ const filePathIdxStart = Date.now();
7403
+ symbolIndex.ensureFilePathIndexes();
7404
+ log.ok(`file_path indexes built in ${((Date.now() - filePathIdxStart) / 1e3).toFixed(2)}s`);
7348
7405
  if (SKIP_FTS) {
7349
7406
  console.log("");
7350
7407
  log.info("Skipping WAL conversion (database will be finalized by build-fts step)");