d365fo-mcp 1.8.0 → 1.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bootstrapEnv.d.ts +33 -0
- package/dist/bootstrapEnv.js +34 -0
- package/dist/bridge/bridgeAdapter.js +3 -0
- package/dist/bridge/debouncedRefresh.d.ts +6 -0
- package/dist/bridge/debouncedRefresh.js +25 -0
- package/dist/index.d.ts +1 -1
- package/dist/index.js +38 -4
- package/dist/metadata/buildIndexWorker.d.ts +18 -0
- package/dist/metadata/buildIndexWorker.js +38 -0
- package/dist/metadata/symbolIndex.d.ts +38 -0
- package/dist/metadata/symbolIndex.js +142 -3
- package/dist/middleware/apiKeyAuth.js +14 -3
- package/dist/middleware/rateLimiter.d.ts +5 -4
- package/dist/middleware/rateLimiter.js +51 -29
- package/dist/scripts/build-database.js +114 -3
- package/dist/scripts/build-fts.js +114 -3
- package/dist/scripts/extract-metadata.js +3 -1
- package/dist/server/toolSchemas/index.d.ts +17 -1
- package/dist/server/toolSchemas/updateSymbolIndex.d.ts +4 -1
- package/dist/server/toolSchemas/updateSymbolIndex.js +1 -1
- package/dist/server/transport.js +16 -4
- package/dist/tools/createD365File.js +50 -23
- package/dist/tools/edtInfo.js +65 -3
- package/dist/tools/findEventHandlers.js +46 -10
- package/dist/tools/modifyD365File.d.ts +11 -0
- package/dist/tools/modifyD365File.js +71 -1
- package/dist/tools/toolHandler.js +1 -1
- package/dist/tools/updateSymbolIndex.d.ts +0 -7
- package/dist/tools/updateSymbolIndex.js +139 -88
- package/dist/utils/configManager.d.ts +11 -0
- package/dist/utils/configManager.js +39 -2
- package/dist/utils/gracefulShutdown.d.ts +52 -0
- package/dist/utils/gracefulShutdown.js +97 -0
- package/dist/utils/modelClassifier.d.ts +13 -6
- package/dist/utils/modelClassifier.js +62 -30
- package/dist/utils/modelPrefixInference.d.ts +62 -0
- package/dist/utils/modelPrefixInference.js +200 -0
- package/dist/utils/workspaceDetector.d.ts +15 -0
- package/dist/utils/workspaceDetector.js +106 -11
- package/package.json +1 -1
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Configuration bootstrap — import this FIRST from every entry point.
|
|
3
|
+
*
|
|
4
|
+
* Importing this module loads config/d365fo-mcp.json, config/secrets.json and
|
|
5
|
+
* .env onto process.env as a side effect (see loadEnv for the precedence rules).
|
|
6
|
+
*
|
|
7
|
+
* Why a module instead of a plain call
|
|
8
|
+
* ------------------------------------
|
|
9
|
+
* ESM evaluates every `import` declaration before the first statement of the
|
|
10
|
+
* importing module's body. Entry points used to be written as
|
|
11
|
+
*
|
|
12
|
+
* import { loadEnv } from './utils/loadEnv.js';
|
|
13
|
+
* loadEnv(import.meta.url); // <- reads first, runs last
|
|
14
|
+
* import { apiKeyAuth } from './middleware/apiKeyAuth.js';
|
|
15
|
+
*
|
|
16
|
+
* which looks like configuration is loaded up front but is not: every module in
|
|
17
|
+
* the import graph — including the ones that snapshot process.env into a
|
|
18
|
+
* module-level const — had already been evaluated by the time loadEnv() ran, so
|
|
19
|
+
* they all saw the unconfigured environment. That silently disabled API-key
|
|
20
|
+
* authentication for keys supplied through .env or config/secrets.json, and made
|
|
21
|
+
* METADATA_PATH, the MCP tool timeouts and the rate-limit settings unreadable
|
|
22
|
+
* from those files.
|
|
23
|
+
*
|
|
24
|
+
* A side-effect import is subject to the same evaluation order it is trying to
|
|
25
|
+
* control, so being *first* is what makes it correct — an import placed above it
|
|
26
|
+
* still wins. tests/server/envConfigTiming.test.ts enforces that ordering.
|
|
27
|
+
*
|
|
28
|
+
* This file deliberately lives next to index.ts rather than in utils/: loadEnv
|
|
29
|
+
* resolves the installation directory from the caller's own location, so the
|
|
30
|
+
* bootstrap must sit at the same depth the entry points do.
|
|
31
|
+
*/
|
|
32
|
+
export {};
|
|
33
|
+
//# sourceMappingURL=bootstrapEnv.d.ts.map
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Configuration bootstrap — import this FIRST from every entry point.
|
|
3
|
+
*
|
|
4
|
+
* Importing this module loads config/d365fo-mcp.json, config/secrets.json and
|
|
5
|
+
* .env onto process.env as a side effect (see loadEnv for the precedence rules).
|
|
6
|
+
*
|
|
7
|
+
* Why a module instead of a plain call
|
|
8
|
+
* ------------------------------------
|
|
9
|
+
* ESM evaluates every `import` declaration before the first statement of the
|
|
10
|
+
* importing module's body. Entry points used to be written as
|
|
11
|
+
*
|
|
12
|
+
* import { loadEnv } from './utils/loadEnv.js';
|
|
13
|
+
* loadEnv(import.meta.url); // <- reads first, runs last
|
|
14
|
+
* import { apiKeyAuth } from './middleware/apiKeyAuth.js';
|
|
15
|
+
*
|
|
16
|
+
* which looks like configuration is loaded up front but is not: every module in
|
|
17
|
+
* the import graph — including the ones that snapshot process.env into a
|
|
18
|
+
* module-level const — had already been evaluated by the time loadEnv() ran, so
|
|
19
|
+
* they all saw the unconfigured environment. That silently disabled API-key
|
|
20
|
+
* authentication for keys supplied through .env or config/secrets.json, and made
|
|
21
|
+
* METADATA_PATH, the MCP tool timeouts and the rate-limit settings unreadable
|
|
22
|
+
* from those files.
|
|
23
|
+
*
|
|
24
|
+
* A side-effect import is subject to the same evaluation order it is trying to
|
|
25
|
+
* control, so being *first* is what makes it correct — an import placed above it
|
|
26
|
+
* still wins. tests/server/envConfigTiming.test.ts enforces that ordering.
|
|
27
|
+
*
|
|
28
|
+
* This file deliberately lives next to index.ts rather than in utils/: loadEnv
|
|
29
|
+
* resolves the installation directory from the caller's own location, so the
|
|
30
|
+
* bootstrap must sit at the same depth the entry points do.
|
|
31
|
+
*/
|
|
32
|
+
import { loadEnv } from './utils/loadEnv.js';
|
|
33
|
+
loadEnv(import.meta.url);
|
|
34
|
+
//# sourceMappingURL=bootstrapEnv.js.map
|
|
@@ -965,6 +965,9 @@ export async function bridgeRefreshProvider(bridge) {
|
|
|
965
965
|
if (!bridge?.isReady || !bridge.metadataAvailable)
|
|
966
966
|
return null;
|
|
967
967
|
try {
|
|
968
|
+
// Recorded so callers that only need the provider to be no older than a
|
|
969
|
+
// given write can skip a redundant rebuild — see debouncedRefresh.
|
|
970
|
+
debouncedRefresh.markRefreshStarted();
|
|
968
971
|
return await bridge.refreshProvider();
|
|
969
972
|
}
|
|
970
973
|
catch (e) {
|
|
@@ -4,6 +4,12 @@
|
|
|
4
4
|
*/
|
|
5
5
|
import type { BridgeClient } from './bridgeClient.js';
|
|
6
6
|
import type { BridgeRefreshResult } from './bridgeClient.js';
|
|
7
|
+
/** Epoch ms at which the last provider refresh started; 0 if none yet. */
|
|
8
|
+
export declare function getLastRefreshStartedAt(): number;
|
|
9
|
+
/** Record that a refresh is starting now. Called by every refresh path. */
|
|
10
|
+
export declare function markRefreshStarted(at?: number): void;
|
|
11
|
+
/** Forget the recorded refresh time (test isolation). */
|
|
12
|
+
export declare function resetRefreshTracking(): void;
|
|
7
13
|
/**
|
|
8
14
|
* Request a bridge refresh. If one is already pending, the settle timer
|
|
9
15
|
* resets (up to MAX_WAIT_MS). All callers receive the same result.
|
|
@@ -5,6 +5,30 @@
|
|
|
5
5
|
const SETTLE_MS = 400;
|
|
6
6
|
const MAX_WAIT_MS = 2_000;
|
|
7
7
|
let pending = null;
|
|
8
|
+
/**
|
|
9
|
+
* When the most recent provider refresh STARTED (epoch ms), across both this
|
|
10
|
+
* module and the direct bridgeRefreshProvider() path.
|
|
11
|
+
*
|
|
12
|
+
* Start rather than completion, because only a refresh that began after a file
|
|
13
|
+
* was written is guaranteed to have read it. Callers use this to skip a refresh
|
|
14
|
+
* that would rediscover nothing: a create/modify already refreshes the provider
|
|
15
|
+
* on its way out, so the update_symbol_index call that follows it was paying for
|
|
16
|
+
* a second full DiskProvider rebuild that could not see anything new.
|
|
17
|
+
*/
|
|
18
|
+
let lastRefreshStartedAt = 0;
|
|
19
|
+
/** Epoch ms at which the last provider refresh started; 0 if none yet. */
|
|
20
|
+
export function getLastRefreshStartedAt() {
|
|
21
|
+
return lastRefreshStartedAt;
|
|
22
|
+
}
|
|
23
|
+
/** Record that a refresh is starting now. Called by every refresh path. */
|
|
24
|
+
export function markRefreshStarted(at = Date.now()) {
|
|
25
|
+
if (at > lastRefreshStartedAt)
|
|
26
|
+
lastRefreshStartedAt = at;
|
|
27
|
+
}
|
|
28
|
+
/** Forget the recorded refresh time (test isolation). */
|
|
29
|
+
export function resetRefreshTracking() {
|
|
30
|
+
lastRefreshStartedAt = 0;
|
|
31
|
+
}
|
|
8
32
|
/**
|
|
9
33
|
* Request a bridge refresh. If one is already pending, the settle timer
|
|
10
34
|
* resets (up to MAX_WAIT_MS). All callers receive the same result.
|
|
@@ -50,6 +74,7 @@ function executeRefresh() {
|
|
|
50
74
|
return;
|
|
51
75
|
const { resolve, bridge } = pending;
|
|
52
76
|
pending = null;
|
|
77
|
+
markRefreshStarted();
|
|
53
78
|
bridge.refreshProvider()
|
|
54
79
|
.then(result => resolve(result))
|
|
55
80
|
.catch(err => {
|
package/dist/index.d.ts
CHANGED
package/dist/index.js
CHANGED
|
@@ -2,9 +2,10 @@
|
|
|
2
2
|
* X++ MCP Code Completion Server
|
|
3
3
|
* Main entry point
|
|
4
4
|
*/
|
|
5
|
-
// Load .env —
|
|
6
|
-
|
|
7
|
-
|
|
5
|
+
// Load configuration onto process.env — MUST stay the first import: ESM
|
|
6
|
+
// evaluates imports before any module body, so anything above this line is
|
|
7
|
+
// evaluated before the configuration exists (see src/bootstrapEnv.ts).
|
|
8
|
+
import './bootstrapEnv.js';
|
|
8
9
|
import { fileURLToPath } from 'url';
|
|
9
10
|
import { dirname, resolve } from 'path';
|
|
10
11
|
import express from 'express';
|
|
@@ -23,6 +24,8 @@ import { TOOL_ANNOTATIONS } from './server/toolAnnotations.js';
|
|
|
23
24
|
import { apiKeyAuth } from './middleware/apiKeyAuth.js';
|
|
24
25
|
import { VERSION } from './version.js';
|
|
25
26
|
import { setInitializeParams } from './utils/stdioSessionInfo.js';
|
|
27
|
+
import { setModelObjectNameSource } from './utils/modelPrefixInference.js';
|
|
28
|
+
import { createShutdownCoordinator } from './utils/gracefulShutdown.js';
|
|
26
29
|
import { box, kv, sectionTitle, statusLine, spread, c, glyph, sanitize, supportsUnicode, log, shortPath, startupWarnings } from './utils/terminalUi.js';
|
|
27
30
|
import * as fs from 'fs/promises';
|
|
28
31
|
import * as fsSync from 'node:fs';
|
|
@@ -120,6 +123,11 @@ const serverState = {
|
|
|
120
123
|
isHealthy: false,
|
|
121
124
|
statusMessage: 'Starting...',
|
|
122
125
|
};
|
|
126
|
+
// Graceful shutdown — see src/utils/gracefulShutdown.ts for why and how.
|
|
127
|
+
const shutdownCoordinator = createShutdownCoordinator({
|
|
128
|
+
deadlineMs: Math.max(1_000, parseInt(process.env.SHUTDOWN_TIMEOUT_MS || '5000', 10) || 5_000),
|
|
129
|
+
});
|
|
130
|
+
const onShutdown = shutdownCoordinator.onShutdown;
|
|
123
131
|
async function initializeServices() {
|
|
124
132
|
// -----------------------------------------------------------------------
|
|
125
133
|
// write-only mode: skip all database/symbol work — LOCAL_TOOLS
|
|
@@ -239,6 +247,10 @@ async function initializeServices() {
|
|
|
239
247
|
throw error;
|
|
240
248
|
}
|
|
241
249
|
}
|
|
250
|
+
// Let object naming learn each model's prefix from the objects that model
|
|
251
|
+
// already contains, instead of applying one configured EXTENSION_PREFIX to
|
|
252
|
+
// every model a developer works in (see utils/modelPrefixInference.ts).
|
|
253
|
+
setModelObjectNameSource(model => symbolIndex.getModelObjectNames(model));
|
|
242
254
|
const parser = new XppMetadataParser();
|
|
243
255
|
// Check if database needs indexing
|
|
244
256
|
if (!dbHasSymbols) {
|
|
@@ -370,6 +382,10 @@ async function initializeBridge(targetContext) {
|
|
|
370
382
|
});
|
|
371
383
|
if (bridge) {
|
|
372
384
|
targetContext.bridge = bridge;
|
|
385
|
+
// dispose() ends the child's stdin and escalates to SIGTERM/SIGKILL, so the
|
|
386
|
+
// bridge gets the chance to finish an in-flight AOT write and close its own
|
|
387
|
+
// metadata handles rather than being cut off mid-file.
|
|
388
|
+
onShutdown('C# bridge', () => bridge.dispose());
|
|
373
389
|
const cap = `metadata ${bridge.metadataAvailable ? 'yes' : 'no'} ${glyph.dot} xref ${bridge.xrefAvailable ? 'yes' : 'no'}`;
|
|
374
390
|
return { ok: true, summary: `C# bridge connected (${devEnvType}) ${glyph.dot} ${cap}` };
|
|
375
391
|
}
|
|
@@ -384,6 +400,18 @@ async function initializeBridge(targetContext) {
|
|
|
384
400
|
}
|
|
385
401
|
}
|
|
386
402
|
async function main() {
|
|
403
|
+
// Registered first so it runs LAST (cleanups run in reverse): the log file is
|
|
404
|
+
// where the other steps report, so it has to outlive them.
|
|
405
|
+
onShutdown('log file', () => {
|
|
406
|
+
if (_logStream) {
|
|
407
|
+
_logStream.end();
|
|
408
|
+
_logStream = undefined;
|
|
409
|
+
}
|
|
410
|
+
});
|
|
411
|
+
// Covers whichever index ended up in serverState — the real one, the in-memory
|
|
412
|
+
// stub, or the replacement built after a corrupt-DB recovery.
|
|
413
|
+
onShutdown('symbol index', () => serverState.symbolIndex?.close?.());
|
|
414
|
+
shutdownCoordinator.registerSignalHandlers({ stdio: isStdioMode });
|
|
387
415
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
388
416
|
// Stdin sniffer: capture the `initialize` request params for get_workspace_info.
|
|
389
417
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
@@ -635,7 +663,13 @@ async function main() {
|
|
|
635
663
|
});
|
|
636
664
|
});
|
|
637
665
|
// Bind port immediately — Azure requires the port to be open within ~230 s
|
|
638
|
-
await new Promise(resolve =>
|
|
666
|
+
const httpServer = await new Promise(resolve => {
|
|
667
|
+
const s = app.listen(PORT, host, () => resolve(s));
|
|
668
|
+
});
|
|
669
|
+
// Stop accepting new connections and let in-flight requests finish. Bounded
|
|
670
|
+
// by the shutdown deadline, so a held-open keep-alive socket cannot stall the
|
|
671
|
+
// exit.
|
|
672
|
+
onShutdown('HTTP server', () => new Promise(resolve => httpServer.close(() => resolve())));
|
|
639
673
|
// Initialise services in the background; register MCP routes once ready
|
|
640
674
|
initializeServices().then(async ({ mcpServer, symbolIndex, parser, workspaceScanner, hybridSearch, context }) => {
|
|
641
675
|
// Register MCP transport (Express supports dynamic route registration)
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Index-build worker thread.
|
|
3
|
+
*
|
|
4
|
+
* CREATE INDEX over an already-populated table is a full table read plus a
|
|
5
|
+
* B-tree write — ~8 s on the 2 GB production symbol DB. node:sqlite is
|
|
6
|
+
* synchronous, so running that on the main thread blocks the event loop for its
|
|
7
|
+
* whole duration and the MCP client (VS Code Copilot) can time the server out
|
|
8
|
+
* before it answers its first request. Running it here, on its own connection,
|
|
9
|
+
* keeps the main thread serving; WAL mode lets this write proceed alongside the
|
|
10
|
+
* main thread's readers.
|
|
11
|
+
*
|
|
12
|
+
* Spawned by XppSymbolIndex.ensureFilePathIndexes() and posts a single message:
|
|
13
|
+
* { ok: true, elapsedMs } | { ok: false, error }
|
|
14
|
+
*
|
|
15
|
+
* The connection is NOT read-only (unlike symbolCountsWorker) — this one writes.
|
|
16
|
+
*/
|
|
17
|
+
export {};
|
|
18
|
+
//# sourceMappingURL=buildIndexWorker.d.ts.map
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Index-build worker thread.
|
|
3
|
+
*
|
|
4
|
+
* CREATE INDEX over an already-populated table is a full table read plus a
|
|
5
|
+
* B-tree write — ~8 s on the 2 GB production symbol DB. node:sqlite is
|
|
6
|
+
* synchronous, so running that on the main thread blocks the event loop for its
|
|
7
|
+
* whole duration and the MCP client (VS Code Copilot) can time the server out
|
|
8
|
+
* before it answers its first request. Running it here, on its own connection,
|
|
9
|
+
* keeps the main thread serving; WAL mode lets this write proceed alongside the
|
|
10
|
+
* main thread's readers.
|
|
11
|
+
*
|
|
12
|
+
* Spawned by XppSymbolIndex.ensureFilePathIndexes() and posts a single message:
|
|
13
|
+
* { ok: true, elapsedMs } | { ok: false, error }
|
|
14
|
+
*
|
|
15
|
+
* The connection is NOT read-only (unlike symbolCountsWorker) — this one writes.
|
|
16
|
+
*/
|
|
17
|
+
import { parentPort, workerData } from 'node:worker_threads';
|
|
18
|
+
import Database from '../database/sqlite.js';
|
|
19
|
+
const { dbPath, sql } = workerData;
|
|
20
|
+
try {
|
|
21
|
+
const db = new Database(dbPath);
|
|
22
|
+
try {
|
|
23
|
+
// A WAL checkpoint racing with a main-thread reader returns SQLITE_BUSY
|
|
24
|
+
// immediately without this; the build is background work, so it can afford
|
|
25
|
+
// to wait far longer than the 5 s used elsewhere.
|
|
26
|
+
db.pragma('busy_timeout = 120000');
|
|
27
|
+
const started = Date.now();
|
|
28
|
+
db.exec(sql);
|
|
29
|
+
parentPort.postMessage({ ok: true, elapsedMs: Date.now() - started });
|
|
30
|
+
}
|
|
31
|
+
finally {
|
|
32
|
+
db.close();
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
catch (e) {
|
|
36
|
+
parentPort.postMessage({ ok: false, error: String(e) });
|
|
37
|
+
}
|
|
38
|
+
//# sourceMappingURL=buildIndexWorker.js.map
|
|
@@ -21,6 +21,7 @@ export declare class XppSymbolIndex {
|
|
|
21
21
|
private labelsReadPool;
|
|
22
22
|
private readPoolRR;
|
|
23
23
|
private dbPath;
|
|
24
|
+
private labelsDbPath;
|
|
24
25
|
private symbolCountsCache;
|
|
25
26
|
private symbolCountsPromise;
|
|
26
27
|
private perConnStmtCache;
|
|
@@ -61,6 +62,33 @@ export declare class XppSymbolIndex {
|
|
|
61
62
|
*/
|
|
62
63
|
private rowToSymbol;
|
|
63
64
|
private initializeDatabase;
|
|
65
|
+
/**
|
|
66
|
+
* Index `symbols.file_path` and `labels.file_path`.
|
|
67
|
+
*
|
|
68
|
+
* Both are the lookup key of removeSymbolsByFile()/removeLabelsByFile(), which
|
|
69
|
+
* every update_symbol_index, undo_last_modification and resync runs first.
|
|
70
|
+
* Unindexed, each of those calls scans the entire table — measured on the 2 GB
|
|
71
|
+
* production DB at 319 s (the SELECT of object names) + 173 s (the DELETE) for
|
|
72
|
+
* indexing a SINGLE new object, versus 0 ms once the index exists. That is why
|
|
73
|
+
* indexing one freshly created object cost as much as a rebuild.
|
|
74
|
+
*
|
|
75
|
+
* Deliberately not part of the CREATE INDEX block above. node:sqlite is
|
|
76
|
+
* synchronous, and building this index over an already-populated production
|
|
77
|
+
* table takes ~8 s, so doing it inline would block the event loop for the whole
|
|
78
|
+
* of startup — the failure mode that makes MCP clients time out and kill the
|
|
79
|
+
* server. On an empty or small DB (a fresh build, the test suite, :memory:) the
|
|
80
|
+
* build is instant and runs here; on a large existing DB it is handed to a
|
|
81
|
+
* worker thread, and until it finishes those deletes simply stay as slow as
|
|
82
|
+
* they are today.
|
|
83
|
+
*/
|
|
84
|
+
private ensureFilePathIndexes;
|
|
85
|
+
/**
|
|
86
|
+
* Build one index on a separate thread so the main event loop keeps serving.
|
|
87
|
+
* WAL mode allows the worker's write to proceed alongside main-thread readers.
|
|
88
|
+
* Best-effort: a failure leaves the index absent, which is exactly the state
|
|
89
|
+
* the server ran in before, so it is logged and never thrown.
|
|
90
|
+
*/
|
|
91
|
+
private buildIndexInWorker;
|
|
64
92
|
/**
|
|
65
93
|
* Create FTS triggers for keeping symbols_fts in sync
|
|
66
94
|
* Extracted to allow disabling during bulk inserts and re-enabling after
|
|
@@ -94,6 +122,16 @@ export declare class XppSymbolIndex {
|
|
|
94
122
|
* even when the DB stores a different path form than the caller passed.
|
|
95
123
|
* Returns the names of top-level objects that were removed (for cache invalidation).
|
|
96
124
|
*/
|
|
125
|
+
/**
|
|
126
|
+
* Top-level object names belonging to one model — the evidence from which a
|
|
127
|
+
* model's naming prefix is inferred (see utils/modelPrefixInference.ts).
|
|
128
|
+
*
|
|
129
|
+
* Deliberately narrow and bounded: only `name`, only root objects, capped at
|
|
130
|
+
* `limit`. Reading whole rows here would pull source snippets across the wire
|
|
131
|
+
* and turn a 450 ms lookup into a slow one. Extension objects are included on
|
|
132
|
+
* purpose — a dot-notation extension states the model's infix outright.
|
|
133
|
+
*/
|
|
134
|
+
getModelObjectNames(model: string, limit?: number): string[];
|
|
97
135
|
removeSymbolsByFile(filePath: string): {
|
|
98
136
|
deletedCount: number;
|
|
99
137
|
objectNames: string[];
|
|
@@ -39,6 +39,9 @@ export class XppSymbolIndex {
|
|
|
39
39
|
// Symbol-count scans are expensive (full index scan of 1M+ rows, 30-60 s
|
|
40
40
|
// cold) — memoize the result and compute it off-thread (see getSymbolCounts).
|
|
41
41
|
dbPath;
|
|
42
|
+
// Needed alongside dbPath so ensureFilePathIndexes() can size the labels DB
|
|
43
|
+
// and hand its path to the background index builder.
|
|
44
|
+
labelsDbPath = ':memory:';
|
|
42
45
|
symbolCountsCache = null;
|
|
43
46
|
symbolCountsPromise = null;
|
|
44
47
|
// Per-connection prepared-statement cache. Prepared statements are bound to
|
|
@@ -55,6 +58,7 @@ export class XppSymbolIndex {
|
|
|
55
58
|
// Labels live in a separate DB so the main symbol DB stays small and fast;
|
|
56
59
|
// labels can be huge (20M+ rows) without affecting search performance.
|
|
57
60
|
const labelPath = labelsDbPath || dbPath.replace('.db', '-labels.db');
|
|
61
|
+
this.labelsDbPath = labelPath;
|
|
58
62
|
this.labelsDb = new Database(labelPath);
|
|
59
63
|
// journal_mode should be set by caller (MEMORY for build, WAL for production).
|
|
60
64
|
// pragma() returns a string like "wal"/"delete" — always truthy, so compare the value, not !pragma(...).
|
|
@@ -649,6 +653,95 @@ export class XppSymbolIndex {
|
|
|
649
653
|
CREATE INDEX IF NOT EXISTS idx_md_define ON macro_defines(define_name);
|
|
650
654
|
CREATE INDEX IF NOT EXISTS idx_md_model ON macro_defines(model);
|
|
651
655
|
`);
|
|
656
|
+
this.ensureFilePathIndexes();
|
|
657
|
+
}
|
|
658
|
+
/**
|
|
659
|
+
* Index `symbols.file_path` and `labels.file_path`.
|
|
660
|
+
*
|
|
661
|
+
* Both are the lookup key of removeSymbolsByFile()/removeLabelsByFile(), which
|
|
662
|
+
* every update_symbol_index, undo_last_modification and resync runs first.
|
|
663
|
+
* Unindexed, each of those calls scans the entire table — measured on the 2 GB
|
|
664
|
+
* production DB at 319 s (the SELECT of object names) + 173 s (the DELETE) for
|
|
665
|
+
* indexing a SINGLE new object, versus 0 ms once the index exists. That is why
|
|
666
|
+
* indexing one freshly created object cost as much as a rebuild.
|
|
667
|
+
*
|
|
668
|
+
* Deliberately not part of the CREATE INDEX block above. node:sqlite is
|
|
669
|
+
* synchronous, and building this index over an already-populated production
|
|
670
|
+
* table takes ~8 s, so doing it inline would block the event loop for the whole
|
|
671
|
+
* of startup — the failure mode that makes MCP clients time out and kill the
|
|
672
|
+
* server. On an empty or small DB (a fresh build, the test suite, :memory:) the
|
|
673
|
+
* build is instant and runs here; on a large existing DB it is handed to a
|
|
674
|
+
* worker thread, and until it finishes those deletes simply stay as slow as
|
|
675
|
+
* they are today.
|
|
676
|
+
*/
|
|
677
|
+
ensureFilePathIndexes() {
|
|
678
|
+
const missing = (db, indexName) => !db.prepare(`SELECT 1 FROM sqlite_master WHERE type = 'index' AND name = ?`).get(indexName);
|
|
679
|
+
// Size is read off the file rather than counted, because COUNT(*) on the
|
|
680
|
+
// table we are trying to speed up is itself one of the slow scans.
|
|
681
|
+
const isLarge = (dbFile) => {
|
|
682
|
+
if (dbFile === ':memory:')
|
|
683
|
+
return false;
|
|
684
|
+
try {
|
|
685
|
+
return fs.statSync(dbFile).size > 200 * 1024 * 1024;
|
|
686
|
+
}
|
|
687
|
+
catch {
|
|
688
|
+
return false;
|
|
689
|
+
}
|
|
690
|
+
};
|
|
691
|
+
const labelsPath = this.labelsDbPath;
|
|
692
|
+
const work = [];
|
|
693
|
+
if (missing(this.db, 'idx_symbols_file_path')) {
|
|
694
|
+
work.push({
|
|
695
|
+
db: this.db,
|
|
696
|
+
dbFile: this.dbPath,
|
|
697
|
+
name: 'idx_symbols_file_path',
|
|
698
|
+
sql: 'CREATE INDEX IF NOT EXISTS idx_symbols_file_path ON symbols(file_path);',
|
|
699
|
+
});
|
|
700
|
+
}
|
|
701
|
+
if (missing(this.labelsDb, 'idx_labels_file_path')) {
|
|
702
|
+
work.push({
|
|
703
|
+
db: this.labelsDb,
|
|
704
|
+
dbFile: labelsPath,
|
|
705
|
+
name: 'idx_labels_file_path',
|
|
706
|
+
sql: 'CREATE INDEX IF NOT EXISTS idx_labels_file_path ON labels(file_path);',
|
|
707
|
+
});
|
|
708
|
+
}
|
|
709
|
+
for (const item of work) {
|
|
710
|
+
if (isLarge(item.dbFile)) {
|
|
711
|
+
this.buildIndexInWorker(item.dbFile, item.sql, item.name);
|
|
712
|
+
}
|
|
713
|
+
else {
|
|
714
|
+
item.db.exec(item.sql);
|
|
715
|
+
}
|
|
716
|
+
}
|
|
717
|
+
}
|
|
718
|
+
/**
|
|
719
|
+
* Build one index on a separate thread so the main event loop keeps serving.
|
|
720
|
+
* WAL mode allows the worker's write to proceed alongside main-thread readers.
|
|
721
|
+
* Best-effort: a failure leaves the index absent, which is exactly the state
|
|
722
|
+
* the server ran in before, so it is logged and never thrown.
|
|
723
|
+
*/
|
|
724
|
+
buildIndexInWorker(dbPath, sql, indexName) {
|
|
725
|
+
try {
|
|
726
|
+
const worker = new Worker(new URL('./buildIndexWorker.js', import.meta.url), {
|
|
727
|
+
workerData: { dbPath, sql, indexName },
|
|
728
|
+
});
|
|
729
|
+
// unref() so a pending index build never keeps the process alive on exit.
|
|
730
|
+
worker.unref();
|
|
731
|
+
worker.once('message', (msg) => {
|
|
732
|
+
if (msg.ok) {
|
|
733
|
+
console.error(`[SymbolIndex] Built ${indexName} in background (${msg.elapsedMs}ms)`);
|
|
734
|
+
}
|
|
735
|
+
else {
|
|
736
|
+
console.error(`[SymbolIndex] Background build of ${indexName} failed: ${msg.error}`);
|
|
737
|
+
}
|
|
738
|
+
void worker.terminate();
|
|
739
|
+
});
|
|
740
|
+
worker.once('error', e => console.error(`[SymbolIndex] ${indexName} worker error: ${e}`));
|
|
741
|
+
}
|
|
742
|
+
catch (e) {
|
|
743
|
+
console.error(`[SymbolIndex] Could not start ${indexName} worker: ${e}`);
|
|
744
|
+
}
|
|
652
745
|
}
|
|
653
746
|
/**
|
|
654
747
|
* Create FTS triggers for keeping symbols_fts in sync
|
|
@@ -736,6 +829,27 @@ export class XppSymbolIndex {
|
|
|
736
829
|
* even when the DB stores a different path form than the caller passed.
|
|
737
830
|
* Returns the names of top-level objects that were removed (for cache invalidation).
|
|
738
831
|
*/
|
|
832
|
+
/**
|
|
833
|
+
* Top-level object names belonging to one model — the evidence from which a
|
|
834
|
+
* model's naming prefix is inferred (see utils/modelPrefixInference.ts).
|
|
835
|
+
*
|
|
836
|
+
* Deliberately narrow and bounded: only `name`, only root objects, capped at
|
|
837
|
+
* `limit`. Reading whole rows here would pull source snippets across the wire
|
|
838
|
+
* and turn a 450 ms lookup into a slow one. Extension objects are included on
|
|
839
|
+
* purpose — a dot-notation extension states the model's infix outright.
|
|
840
|
+
*/
|
|
841
|
+
getModelObjectNames(model, limit = 400) {
|
|
842
|
+
if (!model)
|
|
843
|
+
return [];
|
|
844
|
+
const rows = this.getReadDb()
|
|
845
|
+
.prepare(`SELECT name FROM symbols
|
|
846
|
+
WHERE model = ?
|
|
847
|
+
AND parent_name IS NULL
|
|
848
|
+
AND type NOT IN ('method', 'field')
|
|
849
|
+
LIMIT ?`)
|
|
850
|
+
.all(model, limit);
|
|
851
|
+
return rows.map(r => r.name);
|
|
852
|
+
}
|
|
739
853
|
removeSymbolsByFile(filePath) {
|
|
740
854
|
const forms = this.filePathForms(filePath);
|
|
741
855
|
const placeholders = forms.map(() => '?').join(', ');
|
|
@@ -2782,17 +2896,42 @@ export class XppSymbolIndex {
|
|
|
2782
2896
|
}));
|
|
2783
2897
|
}
|
|
2784
2898
|
getApiUsagePatterns(className) {
|
|
2785
|
-
// Find
|
|
2899
|
+
// Find methods that reference this class in their used_types.
|
|
2786
2900
|
// Cap at 20 rows — fetching source_snippet for 50+ rows on a 584K-row table causes timeout.
|
|
2901
|
+
//
|
|
2902
|
+
// `used_types LIKE '%name%'` was wrong on both counts. used_types is a
|
|
2903
|
+
// comma-separated list of exact type names, so a substring test answered
|
|
2904
|
+
// "SalesTable" with methods that only use AxSalesTable or
|
|
2905
|
+
// MCRSalesTableRefRecId. And it could not use an index: LIMIT 20 only stops
|
|
2906
|
+
// the scan once 20 matches exist, so a name with no matches read used_types
|
|
2907
|
+
// out of all 627 K method rows — each carrying source_snippet and source in
|
|
2908
|
+
// overflow pages. Measured on the 2 GB index: over 7 minutes, synchronously,
|
|
2909
|
+
// which blocks the whole server until the client gives up and kills it.
|
|
2910
|
+
//
|
|
2911
|
+
// The FTS pre-filter makes the candidate set small (an indexed token match on
|
|
2912
|
+
// source_snippet), and exact membership in the list then decides. Same
|
|
2913
|
+
// measurement: 0–230 ms, with the false positives gone.
|
|
2914
|
+
//
|
|
2915
|
+
// The pre-filter also narrows what can be returned: a method whose
|
|
2916
|
+
// used_types names the class but whose stored snippet does not mention it is
|
|
2917
|
+
// no longer reported. That is acceptable here specifically — this tool
|
|
2918
|
+
// exists to show usage examples, and the caller reads initialization
|
|
2919
|
+
// patterns straight out of source_snippet, so a row whose snippet lacks the
|
|
2920
|
+
// name has no example to contribute.
|
|
2921
|
+
const safe = className.replace(/["\(\)\\]/g, '').trim();
|
|
2922
|
+
if (!safe)
|
|
2923
|
+
return [];
|
|
2787
2924
|
let stmt = this.stmtCache.get('getApiUsagePatterns');
|
|
2788
2925
|
if (!stmt) {
|
|
2789
2926
|
stmt = this.db.prepare(`SELECT name, parent_name, method_calls, source_snippet
|
|
2790
2927
|
FROM symbols
|
|
2791
|
-
WHERE type = 'method'
|
|
2928
|
+
WHERE type = 'method'
|
|
2929
|
+
AND id IN (SELECT rowid FROM symbols_fts WHERE symbols_fts MATCH ?)
|
|
2930
|
+
AND ', ' || used_types || ', ' LIKE ? COLLATE NOCASE
|
|
2792
2931
|
LIMIT 20`);
|
|
2793
2932
|
this.stmtCache.set('getApiUsagePatterns', stmt);
|
|
2794
2933
|
}
|
|
2795
|
-
const methods = stmt.all(
|
|
2934
|
+
const methods = stmt.all(`{source_snippet} : "${safe}"`, `%, ${className}, %`);
|
|
2796
2935
|
if (methods.length === 0) {
|
|
2797
2936
|
return [];
|
|
2798
2937
|
}
|
|
@@ -13,8 +13,18 @@ import { timingSafeEqual } from 'node:crypto';
|
|
|
13
13
|
* existing deployments keep working without changes.
|
|
14
14
|
*
|
|
15
15
|
* Timing-safe comparison is used to prevent timing side-channel attacks.
|
|
16
|
+
*
|
|
17
|
+
* The key is read per request rather than snapshotted at module load. ESM
|
|
18
|
+
* evaluates this module as part of the entry point's import graph, which under
|
|
19
|
+
* the old ordering happened before the configuration was loaded onto
|
|
20
|
+
* process.env — so a key set in .env or config/secrets.json read as "no key
|
|
21
|
+
* configured" and authentication silently disabled itself. src/bootstrapEnv.ts
|
|
22
|
+
* fixes that ordering; reading late means this file no longer depends on it.
|
|
16
23
|
*/
|
|
17
|
-
|
|
24
|
+
function configuredApiKey() {
|
|
25
|
+
const key = process.env.API_KEY?.trim();
|
|
26
|
+
return key ? key : undefined;
|
|
27
|
+
}
|
|
18
28
|
/** Paths that never require authentication */
|
|
19
29
|
const PUBLIC_PATHS = new Set(['/', '/health']);
|
|
20
30
|
/**
|
|
@@ -47,8 +57,9 @@ function extractApiKey(req) {
|
|
|
47
57
|
* Mount BEFORE any route handlers.
|
|
48
58
|
*/
|
|
49
59
|
export function apiKeyAuth(req, res, next) {
|
|
60
|
+
const apiKey = configuredApiKey();
|
|
50
61
|
// No API_KEY configured → auth disabled, pass through
|
|
51
|
-
if (!
|
|
62
|
+
if (!apiKey) {
|
|
52
63
|
next();
|
|
53
64
|
return;
|
|
54
65
|
}
|
|
@@ -58,7 +69,7 @@ export function apiKeyAuth(req, res, next) {
|
|
|
58
69
|
return;
|
|
59
70
|
}
|
|
60
71
|
const provided = extractApiKey(req);
|
|
61
|
-
if (!provided || !safeCompare(provided,
|
|
72
|
+
if (!provided || !safeCompare(provided, apiKey)) {
|
|
62
73
|
res.status(401).json({
|
|
63
74
|
error: 'Unauthorized',
|
|
64
75
|
message: 'Missing or invalid API key. Provide it via X-Api-Key header or Authorization: Bearer <key>.',
|
|
@@ -1,7 +1,8 @@
|
|
|
1
|
+
import type { Request, RequestHandler } from 'express';
|
|
2
|
+
export declare function generateRateLimitKey(req: Request): string;
|
|
1
3
|
/**
|
|
2
|
-
*
|
|
3
|
-
*
|
|
4
|
-
* requests, so this stays generous. Override via RATE_LIMIT_MAX_REQUESTS.
|
|
4
|
+
* Express middleware entry point. Delegates to the limiter, constructing it on
|
|
5
|
+
* the first request so it picks up the fully loaded configuration.
|
|
5
6
|
*/
|
|
6
|
-
export declare const apiRateLimiter:
|
|
7
|
+
export declare const apiRateLimiter: RequestHandler;
|
|
7
8
|
//# sourceMappingURL=rateLimiter.d.ts.map
|