d365fo-mcp 1.10.0 → 1.10.1

package/dist/metadata/labelParser.d.ts

@@ -55,8 +55,10 @@ export declare function discoverLabelFiles(modelDir: string): Promise<Array<{
  */
 export declare function indexAllLabels(symbolIndex: XppSymbolIndex, packagesPath: string, modelFilter?: (modelName: string) => boolean, opts?: {
     ftsStrategy?: 'rebuild' | 'incremental';
+    skipFtsRebuild?: boolean;
 }): Promise<{
     totalLabels: number;
     modelsIndexed: number;
+    ftsRebuildPending: boolean;
 }>;
 //# sourceMappingURL=labelParser.d.ts.map

package/dist/metadata/labelParser.js

@@ -15,6 +15,7 @@ import * as fs from 'fs/promises';
 import * as fsSync from 'fs';
 import * as path from 'path';
 import { settingByEnv } from '../config/settings.js';
+import { defaultFileConcurrency, mapWithConcurrency } from '../utils/concurrency.js';
 /**
  * Fallback when LABEL_LANGUAGES is unset, taken from the registry rather than
  * spelled out here. This used to be the literal 'en-US,cs,sk,de' while
@@ -158,15 +159,23 @@ async function indexModelLabels(symbolIndex, modelDir, model, opts) {
     if (labelFiles.length === 0)
         return 0;
     const allEntries = [];
-    for (const { labelFileId, language, filePath } of labelFiles) {
+    // Read the model's label files with a bounded number in flight rather than one at
+    // a time. Barely matters on a default en-US build (813 files / ~38 MB across the
+    // whole PackagesLocalDirectory), but LABEL_LANGUAGES=all multiplies that by the ~74
+    // shipped locales — ~60 K files and a few GB — and a serial await per file makes
+    // that phase cost the sum of every read's latency. mapWithConcurrency preserves
+    // input order, so the rows still go in deterministically.
+    const perFile = await mapWithConcurrency(labelFiles, defaultFileConcurrency(), async ({ labelFileId, language, filePath }) => {
         let content;
         try {
             content = await fs.readFile(filePath, 'utf-8');
         }
         catch {
-            continue;
+            return [];
         }
-        const labels = parseLabelFile(content, labelFileId, model, language, filePath);
+        return parseLabelFile(content, labelFileId, model, language, filePath);
+    });
+    for (const labels of perFile) {
         for (const lbl of labels) {
             allEntries.push({
                 labelId: lbl.labelId,
@@ -203,7 +212,7 @@ export async function indexAllLabels(symbolIndex, packagesPath, modelFilter, opt
     }
     catch {
         console.error(`[LabelParser] Cannot read packages path: ${packagesPath}`);
-        return { totalLabels, modelsIndexed };
+        return { totalLabels, modelsIndexed, ftsRebuildPending: false };
     }
     let skippedByFilter = 0;
     let skippedMissingDir = 0;
@@ -263,9 +272,13 @@ export async function indexAllLabels(symbolIndex, packagesPath, modelFilter, opt
             }
         }
     }
-    // Rebuild FTS once for all models rather than per-model (avoids O(n^2) rebuild cost).
-    // Skipped under 'incremental': the triggers already kept labels_fts in sync per row.
-    if (totalLabels > 0 && !incrementalFts) {
+    // The rebuild is O(every label in the database) no matter how narrow this scan was,
+    // so a caller looping over several package roots must not pay it once per root —
+    // `skipFtsRebuild` lets it hoist the single rebuild out of its own loop and run it
+    // when `ftsRebuildPending` comes back true. Skipped under 'incremental' regardless:
+    // the triggers already kept labels_fts in sync per row.
+    const ftsRebuildPending = totalLabels > 0 && !incrementalFts;
+    if (ftsRebuildPending && !opts?.skipFtsRebuild) {
         symbolIndex.rebuildLabelsFts();
     }
     if (modelsIndexed === 0) {
@@ -275,6 +288,10 @@ export async function indexAllLabels(symbolIndex, packagesPath, modelFilter, opt
         console.log(`      - Models with no labels: ${skippedNoLabels}`);
         console.log(`      - Total models found: ${models.length}`);
     }
-    return { totalLabels, modelsIndexed };
+    return {
+        totalLabels,
+        modelsIndexed,
+        ftsRebuildPending: ftsRebuildPending && !!opts?.skipFtsRebuild,
+    };
 }
 //# sourceMappingURL=labelParser.js.map

package/dist/metadata/symbolIndex.d.ts

@@ -69,6 +69,42 @@ export declare class XppSymbolIndex {
      */
     private rowToSymbol;
     private initializeDatabase;
+    /**
+     * The `labels` indexes that exist purely to accelerate reads, keyed by name so
+     * they can be dropped for a bulk load and rebuilt afterwards.
+     *
+     * `idx_labels_unique` is NOT in here — it enforces the dedupe that
+     * INSERT OR REPLACE relies on and has to stay live through the load.
+     *
+     * The two file_path entries are also created on demand by ensureFilePathIndexes()
+     * (which adds the large-DB worker dispatch that startup needs); this list is the
+     * single definition of their SQL so the two paths cannot drift apart.
+     */
+    private static readonly LABEL_SECONDARY_INDEXES;
+    /** The subset created at schema-init time; the file_path_id index is left to ensureFilePathIndexes(). */
+    private static readonly LABEL_SECONDARY_INDEX_SQL;
+    /**
+     * Drop the read-only `labels` indexes ahead of a bulk load, and report which ones
+     * were actually there so the caller can put back exactly that set.
+     *
+     * Every row of a bulk load otherwise maintains eight B-trees, two of them keyed on
+     * a ~130-character absolute path. Measured on a 400 K-row / 150-model reproduction
+     * of this exact write path: 17.2 s with the indexes live versus 10.6 s dropping
+     * these seven and rebuilding them at the end (the insert itself, 14.9 s → 4.4 s).
+     * This is the same trade the symbols side already makes in ensureFilePathIndexes().
+     *
+     * Build-time only. Do NOT call this on a server that is answering queries — label
+     * search degrades to a full scan until createLabelSecondaryIndexes() finishes.
+     */
+    dropLabelSecondaryIndexes(): string[];
+    /**
+     * Rebuild the indexes dropped by dropLabelSecondaryIndexes().
+     *
+     * Pass the array that call returned to restore exactly the set that was there;
+     * omit it to create all of them. Returns the elapsed milliseconds so build scripts
+     * can report the cost they moved out of the insert loop.
+     */
+    createLabelSecondaryIndexes(only?: string[]): number;
     /**
      * Index `symbols.file_path` and `labels.file_path`.
      *
@@ -138,6 +174,27 @@ export declare class XppSymbolIndex {
      * on every database that predates this, so the one-time rebuild is self-triggering
      * and costs nothing on an already-migrated file.
      */
+    /**
+     * Move an existing database from the inline `labels.file_path` column to the
+     * `label_files` lookup table.
+     *
+     * Runs once, gated on `PRAGMA user_version` like the FTS coverage migration below.
+     * A database built before this carries the path spelled out on every row; the
+     * column cannot simply be dropped, because the rows have to be rewritten to point
+     * at the extracted paths instead.
+     *
+     * This is a full table rewrite, so it is minutes on a multi-GB labels database
+     * rather than the ~23 s the FTS migration costs — announced up front for the same
+     * reason: a silent stall of that length reads as a hung server. It is worth paying
+     * once. The rewritten table is less than half the size, and every full-table
+     * operation on it afterwards (FTS rebuild, ANALYZE, VACUUM, any cold-cache scan)
+     * moves proportionally less disk.
+     *
+     * Row ids are carried over deliberately: `labels_fts` is an external-content index
+     * keyed on `labels.id`, so preserving them keeps it valid. It is rebuilt at the end
+     * anyway, because the content table it points at is a different table object by then.
+     */
+    private migrateLabelPathsToLabelFiles;
     private migrateLabelsFtsLanguageCoverage;
     /**
      * Add a symbol to the index with enhanced metadata
@@ -600,6 +657,19 @@ export declare class XppSymbolIndex {
         comment?: string;
         filePath: string;
     }): void;
+    /**
+     * Row id of `filePath` in `label_files`, inserting it if it is new.
+     *
+     * Memoised for the process: a bulk load calls this once per label but there is
+     * one distinct path per .label.txt (813 across a default en-US build of 374 K
+     * rows), so without the cache it would be ~374 K index probes to learn 813 answers.
+     * The cache is only ever added to — rows in label_files are never deleted, since a
+     * path that had labels once may have them again after the next scan and the table
+     * is trivially small either way.
+     */
+    private labelFilePathId;
+    /** filePath -> label_files.id, populated lazily by labelFilePathId(). */
+    private readonly labelFilePathIds;
     /**
      * Bulk-insert labels (drops FTS triggers for speed).
      * Pass `{ skipFtsRebuild: true }` when indexing many models sequentially;
@@ -672,7 +742,22 @@ export declare class XppSymbolIndex {
      */
     private searchLabelsLike;
     /**
-     * Get a single label by exact ID (returns all languages)
+     * Get a single label by ID (returns all languages).
+     *
+     * The ID may be spelled any way the rest of the server emits it: a reference
+     * (`@ContosoExt:EquipmentName`, `@GLS4170035`, even the doubled
+     * `@SYS:@SYS67433`) or the bare key. #888: matching the caller's string
+     * against the stored one verbatim made the natural spelling fail for the 27
+     * legacy label files, whose keys are stored WITH the sigil — 61% of the
+     * indexed rows — and `search` output, which is always a reference, was never
+     * valid input here. Both branches of the IN-list still use `idx_labels_id`
+     * (and `idx_labels_unique` once the file/model filters are added), so the
+     * widening costs nothing; see labelIdSpellings for why the sigil is not
+     * normalised away at storage time instead.
+     *
+     * Rows come back with the id EXACTLY as stored, which is the spelling
+     * callers must keep using for anything that reads the .label.txt (see
+     * labelMissingOnDisk) or writes a reference.
      */
     getLabelById(labelId: string, labelFileId?: string, model?: string): Array<{
         labelId: string;
@@ -720,8 +805,15 @@ export declare class XppSymbolIndex {
     getLabelCount(): number;
     /**
      * Rename a label ID in the index (used by rename_label tool).
-     * Updates all rows for the given labelId + labelFileId + model combination
-     * and rebuilds the FTS index.
+     * Updates all rows for the given labelId + labelFileId + model combination.
+     *
+     * No FTS rebuild: the `labels_au` trigger deletes the old term and inserts the new
+     * one for each updated row, which is the whole of the work a rebuild would redo.
+     * This used to call rebuildLabelsFts() afterwards — re-tokenising every label in
+     * the database (~105 s on the production DB) to reflect a handful of renamed rows,
+     * and because node:sqlite is synchronous the server answered nothing for its whole
+     * duration. create_label and update_symbol_index were moved off that pattern
+     * earlier; this was the last caller still on it.
      */
     renameLabelInIndex(oldLabelId: string, newLabelId: string, labelFileId: string, model: string): void;
 }