@jagilber-org/index-server 1.27.0 → 1.27.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/dist/dashboard/client/admin.html +57 -27
- package/dist/dashboard/client/css/admin.css +54 -0
- package/dist/dashboard/client/js/admin.config.js +3 -6
- package/dist/dashboard/client/js/admin.embeddings.js +63 -4
- package/dist/dashboard/client/js/admin.events.js +256 -0
- package/dist/dashboard/client/js/admin.maintenance.js +75 -32
- package/dist/dashboard/client/js/admin.sessions.js +1 -1
- package/dist/dashboard/server/AdminPanel.js +83 -6
- package/dist/dashboard/server/AdminPanelConfig.d.ts +11 -0
- package/dist/dashboard/server/AdminPanelConfig.js +47 -17
- package/dist/dashboard/server/DashboardServer.js +13 -0
- package/dist/dashboard/server/routes/admin.routes.js +143 -17
- package/dist/dashboard/server/routes/embeddings.routes.js +91 -1
- package/dist/server/sdkServer.js +12 -4
- package/dist/services/embeddingService.d.ts +2 -0
- package/dist/services/embeddingService.js +16 -4
- package/dist/services/embeddingTrigger.d.ts +33 -0
- package/dist/services/embeddingTrigger.js +86 -0
- package/dist/services/eventBuffer.d.ts +45 -0
- package/dist/services/eventBuffer.js +110 -0
- package/dist/services/handlers/instructions.import.js +71 -13
- package/dist/services/handlers.dashboardConfig.js +81 -0
- package/dist/services/indexContext.d.ts +18 -0
- package/dist/services/indexContext.js +133 -24
- package/dist/services/logger.js +9 -0
- package/dist/services/manifestManager.js +11 -1
- package/dist/services/seedBootstrap.js +5 -1
- package/dist/services/storage/factory.d.ts +2 -0
- package/dist/services/storage/factory.js +12 -1
- package/dist/services/tracing.js +3 -1
- package/package.json +12 -2
- package/server.json +3 -3
|
@@ -23,6 +23,8 @@ const AdminPanel_js_1 = require("../AdminPanel.js");
|
|
|
23
23
|
const WebSocketManager_js_1 = require("../WebSocketManager.js");
|
|
24
24
|
const featureFlags_js_1 = require("../../../services/featureFlags.js");
|
|
25
25
|
const handlers_dashboardConfig_js_1 = require("../../../services/handlers.dashboardConfig.js");
|
|
26
|
+
const runtimeConfig_js_1 = require("../../../config/runtimeConfig.js");
|
|
27
|
+
const eventBuffer_js_1 = require("../../../services/eventBuffer.js");
|
|
26
28
|
const registry_js_1 = require("../../../server/registry.js");
|
|
27
29
|
const adminAuth_js_1 = require("./adminAuth.js");
|
|
28
30
|
const logger_js_1 = require("../../../services/logger.js");
|
|
@@ -91,10 +93,35 @@ function createAdminRoutes(metricsCollector) {
|
|
|
91
93
|
try {
|
|
92
94
|
const updates = req.body;
|
|
93
95
|
const result = adminPanel.updateAdminConfig(updates);
|
|
94
|
-
// Feature flag persistence
|
|
96
|
+
// Feature flag persistence: split incoming flags into
|
|
97
|
+
// (a) namespace flags (e.g. response_envelope_v1) — persisted to flags.json
|
|
98
|
+
// (b) INDEX_SERVER_* boolean toggles — routed to process.env + reloadRuntimeConfig()
|
|
99
|
+
// so dashboard toggles actually take effect at runtime (issue #282 / fix #4).
|
|
95
100
|
if (updates.featureFlags && typeof updates.featureFlags === 'object') {
|
|
96
101
|
try {
|
|
97
|
-
(0,
|
|
102
|
+
const knownEnv = new Set((0, handlers_dashboardConfig_js_1.getFlagRegistrySnapshot)().filter(f => f.type === 'boolean').map(f => f.name));
|
|
103
|
+
const namespaceFlags = {};
|
|
104
|
+
let envChanged = 0;
|
|
105
|
+
const appliedEnv = [];
|
|
106
|
+
for (const [k, v] of Object.entries(updates.featureFlags)) {
|
|
107
|
+
if (typeof v !== 'boolean')
|
|
108
|
+
continue;
|
|
109
|
+
const upper = String(k).toUpperCase();
|
|
110
|
+
if (upper.startsWith('INDEX_SERVER_') && knownEnv.has(upper)) {
|
|
111
|
+
process.env[upper] = v ? '1' : '0';
|
|
112
|
+
appliedEnv.push(upper);
|
|
113
|
+
envChanged++;
|
|
114
|
+
}
|
|
115
|
+
else {
|
|
116
|
+
namespaceFlags[k] = v;
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
if (envChanged > 0) {
|
|
120
|
+
(0, runtimeConfig_js_1.reloadRuntimeConfig)();
|
|
121
|
+
(0, logger_js_1.logInfo)(`[admin] applied ${envChanged} INDEX_SERVER_* toggle(s): ${appliedEnv.join(', ')}`);
|
|
122
|
+
}
|
|
123
|
+
if (Object.keys(namespaceFlags).length > 0)
|
|
124
|
+
(0, featureFlags_js_1.updateFlags)(namespaceFlags);
|
|
98
125
|
}
|
|
99
126
|
catch (e) {
|
|
100
127
|
(0, logger_js_1.logWarn)('[API] feature flag update failed:', e instanceof Error ? e.message : e);
|
|
@@ -340,7 +367,9 @@ function createAdminRoutes(metricsCollector) {
|
|
|
340
367
|
router.post('/admin/maintenance/restore', (req, res) => {
|
|
341
368
|
try {
|
|
342
369
|
const { backupId } = req.body || {};
|
|
370
|
+
(0, logger_js_1.logInfo)('[admin] restore requested', { backupId });
|
|
343
371
|
const result = adminPanel.restoreBackup(backupId);
|
|
372
|
+
(0, logger_js_1.logInfo)('[admin] restore result', { backupId, success: result.success, restored: result.restored ?? 0, message: result.message });
|
|
344
373
|
if (result.success) {
|
|
345
374
|
res.json({ success: true, message: result.message, restored: result.restored, timestamp: Date.now() });
|
|
346
375
|
}
|
|
@@ -428,28 +457,75 @@ function createAdminRoutes(metricsCollector) {
|
|
|
428
457
|
/**
|
|
429
458
|
* POST /api/admin/maintenance/backup/import - Import backup from uploaded JSON bundle or zip archive
|
|
430
459
|
* body: { manifest?: object, files: { [filename]: content } } or raw zip bytes
|
|
460
|
+
* query: ?restore=1 - if set, immediately restore the imported backup (one-click "Restore from File")
|
|
431
461
|
*/
|
|
432
462
|
router.post('/admin/maintenance/backup/import', (0, express_1.raw)({ type: ['application/zip', 'application/octet-stream'], limit: '100mb' }), (req, res) => {
|
|
463
|
+
// Narrow req.query.restore safely — Express may return string | string[] | ParsedQs | ParsedQs[].
|
|
464
|
+
// Only treat a literal scalar string '1' / 'true' (case-insensitive) as opt-in; arrays/objects are rejected.
|
|
465
|
+
const restoreParam = req.query?.restore;
|
|
466
|
+
const wantRestore = typeof restoreParam === 'string' && (restoreParam === '1' || restoreParam.toLowerCase() === 'true');
|
|
467
|
+
const ctype = req.header('content-type') || '';
|
|
468
|
+
const rawBody = req.body;
|
|
469
|
+
// Refine req.body through typed locals so downstream uses don't rely on `as Buffer` casts.
|
|
470
|
+
// CodeQL js/type-confusion-through-parameter-tampering does NOT recognize Buffer.isBuffer
|
|
471
|
+
// as a sanitizer; use `instanceof Buffer` (native JS predicate, modeled by CodeQL).
|
|
472
|
+
// Array.isArray IS recognized. Avoid `.length` interpolation on tainted values.
|
|
473
|
+
const bodyBuffer = rawBody instanceof Buffer ? rawBody : null;
|
|
474
|
+
const bodyArray = Array.isArray(rawBody) ? rawBody : null;
|
|
475
|
+
const bodyObject = rawBody !== null
|
|
476
|
+
&& typeof rawBody === 'object'
|
|
477
|
+
&& !Array.isArray(rawBody)
|
|
478
|
+
&& !(rawBody instanceof Buffer)
|
|
479
|
+
? rawBody
|
|
480
|
+
: null;
|
|
481
|
+
const bodyKind = bodyBuffer !== null
|
|
482
|
+
? 'buffer'
|
|
483
|
+
: bodyArray !== null
|
|
484
|
+
? 'array'
|
|
485
|
+
: bodyObject !== null
|
|
486
|
+
? 'json'
|
|
487
|
+
: typeof rawBody;
|
|
488
|
+
(0, logger_js_1.logInfo)('[admin] backup/import received', { ctype, body: bodyKind, restore: wantRestore });
|
|
433
489
|
try {
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
const
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
}
|
|
440
|
-
return res.status(400).json({ success: false, error: result.message, timestamp: Date.now() });
|
|
490
|
+
let importResult;
|
|
491
|
+
if (bodyBuffer !== null && bodyBuffer.length > 0) {
|
|
492
|
+
const filenameHeader = req.header('x-backup-filename') || req.header('x-file-name');
|
|
493
|
+
const sourceName = typeof filenameHeader === 'string' ? filenameHeader : undefined;
|
|
494
|
+
importResult = adminPanel.importZipBackup(bodyBuffer, sourceName);
|
|
441
495
|
}
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
496
|
+
else {
|
|
497
|
+
if (bodyObject === null) {
|
|
498
|
+
(0, logger_js_1.logWarn)('[admin] backup/import rejected', { reason: 'body-not-json-object', ctype });
|
|
499
|
+
return res.status(400).json({ success: false, error: 'Request body must be a JSON object containing a "files" object', timestamp: Date.now() });
|
|
500
|
+
}
|
|
501
|
+
const bundle = bodyObject;
|
|
502
|
+
const files = bundle.files;
|
|
503
|
+
if (!files || typeof files !== 'object' || Array.isArray(files)) {
|
|
504
|
+
(0, logger_js_1.logWarn)('[admin] backup/import rejected', { reason: 'missing-or-invalid-files-object' });
|
|
505
|
+
return res.status(400).json({ success: false, error: 'Request body must contain a "files" object', timestamp: Date.now() });
|
|
506
|
+
}
|
|
507
|
+
importResult = adminPanel.importBackup({
|
|
508
|
+
manifest: typeof bundle.manifest === 'object' && bundle.manifest !== null && !Array.isArray(bundle.manifest)
|
|
509
|
+
? bundle.manifest
|
|
510
|
+
: undefined,
|
|
511
|
+
files: files,
|
|
512
|
+
});
|
|
445
513
|
}
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
res.json({ success:
|
|
514
|
+
if (!importResult.success) {
|
|
515
|
+
(0, logger_js_1.logWarn)('[admin] backup/import failed', { message: importResult.message });
|
|
516
|
+
return res.status(400).json({ success: false, error: importResult.message, timestamp: Date.now() });
|
|
449
517
|
}
|
|
450
|
-
|
|
451
|
-
|
|
518
|
+
(0, logger_js_1.logInfo)('[admin] backup/import ok', { backupId: importResult.backupId, files: importResult.files });
|
|
519
|
+
if (wantRestore && importResult.backupId) {
|
|
520
|
+
(0, logger_js_1.logInfo)('[admin] backup/import auto-restore start', { backupId: importResult.backupId });
|
|
521
|
+
const restoreResult = adminPanel.restoreBackup(importResult.backupId);
|
|
522
|
+
(0, logger_js_1.logInfo)('[admin] backup/import auto-restore result', { backupId: importResult.backupId, success: restoreResult.success, restored: restoreResult.restored ?? 0, message: restoreResult.message });
|
|
523
|
+
if (!restoreResult.success) {
|
|
524
|
+
return res.status(500).json({ success: false, error: `Imported as ${importResult.backupId} but restore failed: ${restoreResult.message}`, backupId: importResult.backupId, files: importResult.files, restored: 0, timestamp: Date.now() });
|
|
525
|
+
}
|
|
526
|
+
return res.json({ success: true, message: `Imported and restored ${importResult.backupId} (${restoreResult.restored ?? 0} files)`, backupId: importResult.backupId, files: importResult.files, restored: restoreResult.restored ?? 0, restored_applied: true, timestamp: Date.now() });
|
|
452
527
|
}
|
|
528
|
+
return res.json({ success: true, message: importResult.message, backupId: importResult.backupId, files: importResult.files, timestamp: Date.now() });
|
|
453
529
|
}
|
|
454
530
|
catch (error) {
|
|
455
531
|
(0, logger_js_1.logError)('[API] Import backup error:', error);
|
|
@@ -578,5 +654,55 @@ function createAdminRoutes(metricsCollector) {
|
|
|
578
654
|
});
|
|
579
655
|
}
|
|
580
656
|
});
|
|
657
|
+
/**
|
|
658
|
+
* GET /api/admin/events - Recent WARN/ERROR events from the in-memory ring buffer
|
|
659
|
+
* Query: ?since=<id>&level=WARN|ERROR&limit=<n>
|
|
660
|
+
*/
|
|
661
|
+
router.get('/admin/events', (req, res) => {
|
|
662
|
+
try {
|
|
663
|
+
const since = req.query.since !== undefined ? parseInt(String(req.query.since), 10) : undefined;
|
|
664
|
+
const limit = req.query.limit !== undefined ? parseInt(String(req.query.limit), 10) : undefined;
|
|
665
|
+
const levelRaw = String(req.query.level || '').toUpperCase();
|
|
666
|
+
const level = (levelRaw === 'WARN' || levelRaw === 'ERROR') ? levelRaw : undefined;
|
|
667
|
+
const events = (0, eventBuffer_js_1.listEvents)({
|
|
668
|
+
sinceId: Number.isFinite(since) ? since : undefined,
|
|
669
|
+
limit: Number.isFinite(limit) ? limit : undefined,
|
|
670
|
+
level,
|
|
671
|
+
});
|
|
672
|
+
const counts = (0, eventBuffer_js_1.eventCounts)(0);
|
|
673
|
+
res.json({ success: true, events, counts, timestamp: Date.now() });
|
|
674
|
+
}
|
|
675
|
+
catch (error) {
|
|
676
|
+
(0, logger_js_1.logError)('[API] Get events error:', error);
|
|
677
|
+
res.status(500).json({ success: false, error: 'Failed to read events' });
|
|
678
|
+
}
|
|
679
|
+
});
|
|
680
|
+
/**
|
|
681
|
+
* GET /api/admin/events/counts - Lightweight counts for nav-bubble polling
|
|
682
|
+
*/
|
|
683
|
+
router.get('/admin/events/counts', (req, res) => {
|
|
684
|
+
try {
|
|
685
|
+
const since = req.query.since !== undefined ? parseInt(String(req.query.since), 10) : 0;
|
|
686
|
+
const counts = (0, eventBuffer_js_1.eventCounts)(Number.isFinite(since) ? since : 0);
|
|
687
|
+
res.json({ success: true, counts, timestamp: Date.now() });
|
|
688
|
+
}
|
|
689
|
+
catch (error) {
|
|
690
|
+
(0, logger_js_1.logError)('[API] Get event counts error:', error);
|
|
691
|
+
res.status(500).json({ success: false, error: 'Failed to read event counts' });
|
|
692
|
+
}
|
|
693
|
+
});
|
|
694
|
+
/**
|
|
695
|
+
* DELETE /api/admin/events - Clear the buffer (also acts as "mark all read")
|
|
696
|
+
*/
|
|
697
|
+
router.delete('/admin/events', (_req, res) => {
|
|
698
|
+
try {
|
|
699
|
+
(0, eventBuffer_js_1.clearEvents)();
|
|
700
|
+
res.json({ success: true, message: 'Events cleared', timestamp: Date.now() });
|
|
701
|
+
}
|
|
702
|
+
catch (error) {
|
|
703
|
+
(0, logger_js_1.logError)('[API] Clear events error:', error);
|
|
704
|
+
res.status(500).json({ success: false, error: 'Failed to clear events' });
|
|
705
|
+
}
|
|
706
|
+
});
|
|
581
707
|
return router;
|
|
582
708
|
}
|
|
@@ -211,6 +211,85 @@ function createEmbeddingsRoutes(embeddingPathOverride, embeddingStore) {
|
|
|
211
211
|
});
|
|
212
212
|
}
|
|
213
213
|
});
|
|
214
|
+
// GET /embeddings/status — surface model + cache state for the Embeddings tab
|
|
215
|
+
router.get('/embeddings/status', (_req, res) => {
|
|
216
|
+
try {
|
|
217
|
+
const config = (0, runtimeConfig_js_1.getRuntimeConfig)();
|
|
218
|
+
const sem = config.semantic;
|
|
219
|
+
if (!sem.enabled) {
|
|
220
|
+
return res.json({
|
|
221
|
+
success: true,
|
|
222
|
+
enabled: false,
|
|
223
|
+
ready: false,
|
|
224
|
+
state: 'disabled',
|
|
225
|
+
message: 'Semantic embeddings are disabled. Set INDEX_SERVER_SEMANTIC_ENABLED=1 and restart to enable.',
|
|
226
|
+
});
|
|
227
|
+
}
|
|
228
|
+
const readiness = (0, embeddingService_js_1.checkModelReadiness)(sem.model, sem.cacheDir, sem.localOnly);
|
|
229
|
+
let embeddingsFileExists = false;
|
|
230
|
+
let embeddingsCount = 0;
|
|
231
|
+
let embeddingsModelName;
|
|
232
|
+
try {
|
|
233
|
+
if (embeddingStore) {
|
|
234
|
+
const loaded = embeddingStore.load();
|
|
235
|
+
if (loaded) {
|
|
236
|
+
embeddingsFileExists = true;
|
|
237
|
+
embeddingsCount = Object.keys(loaded.embeddings || {}).length;
|
|
238
|
+
embeddingsModelName = loaded.modelName ?? undefined;
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
else {
|
|
242
|
+
const file = embeddingPathOverride ?? sem.embeddingPath;
|
|
243
|
+
if (file && node_fs_1.default.existsSync(file)) {
|
|
244
|
+
embeddingsFileExists = true;
|
|
245
|
+
const raw = node_fs_1.default.readFileSync(file, 'utf8');
|
|
246
|
+
const parsed = JSON.parse(raw);
|
|
247
|
+
embeddingsCount = parsed.embeddings ? Object.keys(parsed.embeddings).length : 0;
|
|
248
|
+
embeddingsModelName = parsed.modelName ?? undefined;
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
catch { /* tolerate parse / read failures — surface as not present */ }
|
|
253
|
+
// State machine for the dashboard banner.
|
|
254
|
+
let state;
|
|
255
|
+
if (!readiness.ready) {
|
|
256
|
+
state = 'missing'; // localOnly=true and model not cached
|
|
257
|
+
}
|
|
258
|
+
else if (!readiness.cached) {
|
|
259
|
+
state = 'will-download';
|
|
260
|
+
}
|
|
261
|
+
else if (!embeddingsFileExists || embeddingsCount === 0) {
|
|
262
|
+
state = 'no-embeddings';
|
|
263
|
+
}
|
|
264
|
+
else {
|
|
265
|
+
state = 'ready';
|
|
266
|
+
}
|
|
267
|
+
return res.json({
|
|
268
|
+
success: true,
|
|
269
|
+
enabled: true,
|
|
270
|
+
ready: readiness.ready && embeddingsFileExists && embeddingsCount > 0,
|
|
271
|
+
state,
|
|
272
|
+
model: sem.model,
|
|
273
|
+
device: sem.device,
|
|
274
|
+
cacheDir: sem.cacheDir,
|
|
275
|
+
localOnly: sem.localOnly,
|
|
276
|
+
modelCached: readiness.cached,
|
|
277
|
+
modelPath: readiness.modelPath,
|
|
278
|
+
embeddingPath: sem.embeddingPath,
|
|
279
|
+
embeddingsFileExists,
|
|
280
|
+
embeddingsCount,
|
|
281
|
+
embeddingsModelName,
|
|
282
|
+
message: readiness.message,
|
|
283
|
+
});
|
|
284
|
+
}
|
|
285
|
+
catch (err) {
|
|
286
|
+
return res.status(500).json({
|
|
287
|
+
success: false,
|
|
288
|
+
error: 'Failed to compute embeddings status',
|
|
289
|
+
message: err.message,
|
|
290
|
+
});
|
|
291
|
+
}
|
|
292
|
+
});
|
|
214
293
|
// POST /embeddings/compute — trigger embedding computation for all instructions
|
|
215
294
|
router.post('/embeddings/compute', adminAuth_js_1.dashboardAdminAuth, async (_req, res) => {
|
|
216
295
|
try {
|
|
@@ -244,10 +323,21 @@ function createEmbeddingsRoutes(embeddingPathOverride, embeddingStore) {
|
|
|
244
323
|
});
|
|
245
324
|
}
|
|
246
325
|
catch (err) {
|
|
326
|
+
const msg = err.message ?? String(err);
|
|
327
|
+
// Detect the canonical "model not cached locally" signature from
|
|
328
|
+
// @huggingface/transformers and translate to an actionable hint so
|
|
329
|
+
// the dashboard can surface a useful banner.
|
|
330
|
+
const isLocalOnlyMiss = /local_files_only=true/i.test(msg) ||
|
|
331
|
+
/allowRemoteModels=false/i.test(msg) ||
|
|
332
|
+
/file was not found locally/i.test(msg);
|
|
247
333
|
return res.status(500).json({
|
|
248
334
|
success: false,
|
|
249
335
|
error: 'Failed to compute embeddings',
|
|
250
|
-
message:
|
|
336
|
+
message: msg,
|
|
337
|
+
hint: isLocalOnlyMiss
|
|
338
|
+
? 'The embedding model is not cached locally and remote downloads are disabled. ' +
|
|
339
|
+
'Set INDEX_SERVER_SEMANTIC_LOCAL_ONLY=0 and restart, or pre-stage the model in INDEX_SERVER_SEMANTIC_CACHE_DIR.'
|
|
340
|
+
: undefined,
|
|
251
341
|
});
|
|
252
342
|
}
|
|
253
343
|
});
|
package/dist/server/sdkServer.js
CHANGED
|
@@ -21,6 +21,7 @@ const registry_1 = require("./registry");
|
|
|
21
21
|
const zod_1 = require("zod");
|
|
22
22
|
const runtimeConfig_1 = require("../config/runtimeConfig");
|
|
23
23
|
const mcpLogBridge_1 = require("../services/mcpLogBridge");
|
|
24
|
+
const logger_1 = require("../services/logger");
|
|
24
25
|
const handshakeManager_1 = require("./handshakeManager");
|
|
25
26
|
const transportFactory_1 = require("./transportFactory");
|
|
26
27
|
const mcpReadOnlySurfaces_1 = require("./mcpReadOnlySurfaces");
|
|
@@ -205,6 +206,8 @@ function createSdkServer(ServerClass) {
|
|
|
205
206
|
const args = p.arguments || {};
|
|
206
207
|
if (name === 'health_check')
|
|
207
208
|
(0, handshakeManager_1.record)('tools_call_health');
|
|
209
|
+
const __callStart = Date.now();
|
|
210
|
+
(0, logger_1.logInfo)('[rpc] tools/call', { tool: name, id: req?.id ?? null });
|
|
208
211
|
try {
|
|
209
212
|
if ((0, runtimeConfig_1.getRuntimeConfig)().logging.verbose)
|
|
210
213
|
process.stderr.write(`[rpc] call method=tools/call tool=${name} id=${req?.id ?? 'n/a'}\n`);
|
|
@@ -216,9 +219,11 @@ function createSdkServer(ServerClass) {
|
|
|
216
219
|
}
|
|
217
220
|
try {
|
|
218
221
|
const result = await Promise.resolve(handler(args));
|
|
222
|
+
const bytes = Buffer.byteLength(JSON.stringify(result), 'utf8');
|
|
223
|
+
(0, logger_1.logInfo)('[rpc] tools/call ok', { tool: name, ms: Date.now() - __callStart, bytes });
|
|
219
224
|
try {
|
|
220
225
|
if ((0, runtimeConfig_1.getRuntimeConfig)().logging.verbose)
|
|
221
|
-
process.stderr.write(`[rpc] tool_result tool=${name} bytes=${
|
|
226
|
+
process.stderr.write(`[rpc] tool_result tool=${name} bytes=${bytes}\n`);
|
|
222
227
|
}
|
|
223
228
|
catch { /* ignore */ }
|
|
224
229
|
return { content: [{ type: 'text', text: JSON.stringify(result, null, 2) }] };
|
|
@@ -226,7 +231,10 @@ function createSdkServer(ServerClass) {
|
|
|
226
231
|
catch (e) {
|
|
227
232
|
const code = e?.code;
|
|
228
233
|
const sem = e?.__semantic === true;
|
|
234
|
+
const msgRaw = e instanceof Error ? e.message : String(e);
|
|
235
|
+
const stack = e instanceof Error ? e.stack : undefined;
|
|
229
236
|
if (Number.isSafeInteger(code)) {
|
|
237
|
+
(0, logger_1.logError)('[rpc] tools/call error', { tool: name, ms: Date.now() - __callStart, code, semantic: sem, message: msgRaw, stack });
|
|
230
238
|
try {
|
|
231
239
|
if ((0, runtimeConfig_1.getRuntimeConfig)().logging.verbose)
|
|
232
240
|
process.stderr.write(`[rpc] tool_error_passthru tool=${name} code=${code} semantic=${sem ? '1' : '0'} msg=${e?.message || ''}\n`);
|
|
@@ -234,13 +242,13 @@ function createSdkServer(ServerClass) {
|
|
|
234
242
|
catch { /* ignore */ }
|
|
235
243
|
throw e;
|
|
236
244
|
}
|
|
237
|
-
|
|
245
|
+
(0, logger_1.logError)('[rpc] tools/call error', { tool: name, ms: Date.now() - __callStart, code: code ?? null, message: msgRaw, stack });
|
|
238
246
|
try {
|
|
239
247
|
if ((0, runtimeConfig_1.getRuntimeConfig)().logging.verbose)
|
|
240
|
-
process.stderr.write(`[rpc] tool_error_wrap tool=${name} msg=${
|
|
248
|
+
process.stderr.write(`[rpc] tool_error_wrap tool=${name} msg=${msgRaw.replace(/\s+/g, ' ')} code=${code ?? 'n/a'}\n`);
|
|
241
249
|
}
|
|
242
250
|
catch { /* ignore */ }
|
|
243
|
-
throw { code: -32603, message: 'Tool execution failed', data: { message:
|
|
251
|
+
throw { code: -32603, message: 'Tool execution failed', data: { message: msgRaw, method: name } };
|
|
244
252
|
}
|
|
245
253
|
});
|
|
246
254
|
// Lightweight ping handler (simple reachability / latency measurement)
|
|
@@ -58,6 +58,8 @@ export declare function embedText(text: string, modelName: string, cacheDir: str
|
|
|
58
58
|
*/
|
|
59
59
|
export declare function checkModelReadiness(modelName: string, cacheDir: string, localOnly: boolean): {
|
|
60
60
|
ready: boolean;
|
|
61
|
+
cached: boolean;
|
|
62
|
+
modelPath: string;
|
|
61
63
|
message?: string;
|
|
62
64
|
};
|
|
63
65
|
/** Signature for the embed function (injectable for testing). */
|
|
@@ -187,22 +187,34 @@ async function embedText(text, modelName, cacheDir, device = 'cpu', localOnly =
|
|
|
187
187
|
* @returns Object with `ready` flag and optional remediation `message`.
|
|
188
188
|
*/
|
|
189
189
|
function checkModelReadiness(modelName, cacheDir, localOnly) {
|
|
190
|
-
if (!localOnly) {
|
|
191
|
-
return { ready: true }; // Model can be downloaded on demand
|
|
192
|
-
}
|
|
193
190
|
// HuggingFace transformers caches models as: models--<org>--<name>
|
|
194
191
|
const modelDirName = `models--${modelName.replace(/\//g, '--')}`;
|
|
195
192
|
const modelPath = path_1.default.join(cacheDir, modelDirName);
|
|
193
|
+
let cached = false;
|
|
196
194
|
try {
|
|
197
195
|
if (fs_1.default.existsSync(modelPath) && fs_1.default.readdirSync(modelPath).length > 0) {
|
|
198
|
-
|
|
196
|
+
cached = true;
|
|
199
197
|
}
|
|
200
198
|
}
|
|
201
199
|
catch {
|
|
202
200
|
// Directory doesn't exist or can't be read
|
|
203
201
|
}
|
|
202
|
+
if (cached) {
|
|
203
|
+
return { ready: true, cached: true, modelPath };
|
|
204
|
+
}
|
|
205
|
+
if (!localOnly) {
|
|
206
|
+
return {
|
|
207
|
+
ready: true,
|
|
208
|
+
cached: false,
|
|
209
|
+
modelPath,
|
|
210
|
+
message: `Embedding model '${modelName}' is not yet cached. ` +
|
|
211
|
+
`It will be downloaded to '${cacheDir}' on first compute (~25 MB).`,
|
|
212
|
+
};
|
|
213
|
+
}
|
|
204
214
|
return {
|
|
205
215
|
ready: false,
|
|
216
|
+
cached: false,
|
|
217
|
+
modelPath,
|
|
206
218
|
message: `Embedding model '${modelName}' not found in cache (${cacheDir}). ` +
|
|
207
219
|
`LOCAL_ONLY is enabled, so the model cannot be downloaded automatically. ` +
|
|
208
220
|
`To fix: set INDEX_SERVER_SEMANTIC_LOCAL_ONLY=0 to allow download, ` +
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Auto-compute embeddings after operations that mutate the loaded instruction set
|
|
3
|
+
* (zip import, restore, bulk migrations).
|
|
4
|
+
*
|
|
5
|
+
* Behaviour:
|
|
6
|
+
* - No-op when semantic is disabled (`INDEX_SERVER_SEMANTIC_ENABLED` falsy).
|
|
7
|
+
* - No-op when explicitly opted out via `INDEX_SERVER_AUTO_EMBED_ON_IMPORT=0`.
|
|
8
|
+
* - Runs asynchronously (fire-and-forget) so the caller's request returns promptly.
|
|
9
|
+
* - Coalesces concurrent triggers via the existing in-flight lock inside
|
|
10
|
+
* `getInstructionEmbeddings`.
|
|
11
|
+
*
|
|
12
|
+
* Logs success / failure at INFO / WARN so events surface in the events panel
|
|
13
|
+
* (constitution OB-3, OB-5).
|
|
14
|
+
*/
|
|
15
|
+
/** Whether auto-compute is enabled given current env / runtime config. */
|
|
16
|
+
export declare function autoEmbedEnabled(): boolean;
|
|
17
|
+
/**
|
|
18
|
+
* Trigger an embedding compute pass after an import / restore.
|
|
19
|
+
*
|
|
20
|
+
* @param reason - Free-form context (e.g. `import-zip`, `restore`) used in logs.
|
|
21
|
+
* @returns Promise resolving when compute finishes (or immediately if skipped).
|
|
22
|
+
*/
|
|
23
|
+
export declare function triggerEmbeddingComputeAfterImport(reason: string): Promise<{
|
|
24
|
+
triggered: boolean;
|
|
25
|
+
reason?: string;
|
|
26
|
+
entries?: number;
|
|
27
|
+
ms?: number;
|
|
28
|
+
}>;
|
|
29
|
+
/**
|
|
30
|
+
* Fire-and-forget variant — schedules the compute on the next tick and returns immediately.
|
|
31
|
+
* Use this from request handlers so HTTP responses are not blocked on model warm-up.
|
|
32
|
+
*/
|
|
33
|
+
export declare function scheduleEmbeddingComputeAfterImport(reason: string): void;
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Auto-compute embeddings after operations that mutate the loaded instruction set
|
|
4
|
+
* (zip import, restore, bulk migrations).
|
|
5
|
+
*
|
|
6
|
+
* Behaviour:
|
|
7
|
+
* - No-op when semantic is disabled (`INDEX_SERVER_SEMANTIC_ENABLED` falsy).
|
|
8
|
+
* - No-op when explicitly opted out via `INDEX_SERVER_AUTO_EMBED_ON_IMPORT=0`.
|
|
9
|
+
* - Runs asynchronously (fire-and-forget) so the caller's request returns promptly.
|
|
10
|
+
* - Coalesces concurrent triggers via the existing in-flight lock inside
|
|
11
|
+
* `getInstructionEmbeddings`.
|
|
12
|
+
*
|
|
13
|
+
* Logs success / failure at INFO / WARN so events surface in the events panel
|
|
14
|
+
* (constitution OB-3, OB-5).
|
|
15
|
+
*/
|
|
16
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
+
exports.autoEmbedEnabled = autoEmbedEnabled;
|
|
18
|
+
exports.triggerEmbeddingComputeAfterImport = triggerEmbeddingComputeAfterImport;
|
|
19
|
+
exports.scheduleEmbeddingComputeAfterImport = scheduleEmbeddingComputeAfterImport;
|
|
20
|
+
const runtimeConfig_1 = require("../config/runtimeConfig");
|
|
21
|
+
const indexContext_1 = require("./indexContext");
|
|
22
|
+
const embeddingService_1 = require("./embeddingService");
|
|
23
|
+
const logger_1 = require("./logger");
|
|
24
|
+
const envUtils_1 = require("../utils/envUtils");
|
|
25
|
+
let lastTriggerAt = 0;
|
|
26
|
+
/** Whether auto-compute is enabled given current env / runtime config. */
|
|
27
|
+
function autoEmbedEnabled() {
|
|
28
|
+
const cfg = (0, runtimeConfig_1.getRuntimeConfig)();
|
|
29
|
+
if (!cfg.semantic.enabled)
|
|
30
|
+
return false;
|
|
31
|
+
const raw = process.env.INDEX_SERVER_AUTO_EMBED_ON_IMPORT;
|
|
32
|
+
// Default ON when semantic is enabled; explicit '0' / 'false' opts out.
|
|
33
|
+
if (raw === undefined)
|
|
34
|
+
return true;
|
|
35
|
+
return (0, envUtils_1.getBooleanEnv)('INDEX_SERVER_AUTO_EMBED_ON_IMPORT', true);
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Trigger an embedding compute pass after an import / restore.
|
|
39
|
+
*
|
|
40
|
+
* @param reason - Free-form context (e.g. `import-zip`, `restore`) used in logs.
|
|
41
|
+
* @returns Promise resolving when compute finishes (or immediately if skipped).
|
|
42
|
+
*/
|
|
43
|
+
async function triggerEmbeddingComputeAfterImport(reason) {
|
|
44
|
+
if (!autoEmbedEnabled()) {
|
|
45
|
+
return { triggered: false, reason: 'auto-embed disabled or semantic disabled' };
|
|
46
|
+
}
|
|
47
|
+
// Light debounce — coalesce rapid back-to-back imports.
|
|
48
|
+
const now = Date.now();
|
|
49
|
+
if (now - lastTriggerAt < 1000) {
|
|
50
|
+
return { triggered: false, reason: 'debounced' };
|
|
51
|
+
}
|
|
52
|
+
lastTriggerAt = now;
|
|
53
|
+
const cfg = (0, runtimeConfig_1.getRuntimeConfig)();
|
|
54
|
+
const sem = cfg.semantic;
|
|
55
|
+
const start = Date.now();
|
|
56
|
+
try {
|
|
57
|
+
(0, indexContext_1.ensureLoaded)();
|
|
58
|
+
const state = (0, indexContext_1.getIndexState)();
|
|
59
|
+
if (!state.list || state.list.length === 0) {
|
|
60
|
+
(0, logger_1.logInfo)(`[embedding-trigger] Skipped (no instructions loaded) reason=${reason}`);
|
|
61
|
+
return { triggered: false, reason: 'no instructions loaded' };
|
|
62
|
+
}
|
|
63
|
+
(0, logger_1.logInfo)(`[embedding-trigger] Starting auto-compute reason=${reason} entries=${state.list.length}`);
|
|
64
|
+
await (0, embeddingService_1.getInstructionEmbeddings)(state.list, state.hash, sem.embeddingPath, sem.model, sem.cacheDir, sem.device, sem.localOnly);
|
|
65
|
+
const ms = Date.now() - start;
|
|
66
|
+
(0, logger_1.logInfo)(`[embedding-trigger] Auto-compute complete reason=${reason} entries=${state.list.length} ms=${ms}`);
|
|
67
|
+
return { triggered: true, entries: state.list.length, ms };
|
|
68
|
+
}
|
|
69
|
+
catch (err) {
|
|
70
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
71
|
+
// Use WARN (not ERROR) so we don't escalate transient model-load issues to ERROR-level paging.
|
|
72
|
+
(0, logger_1.logWarn)(`[embedding-trigger] Auto-compute failed reason=${reason}: ${msg}`);
|
|
73
|
+
return { triggered: false, reason: `failed: ${msg}` };
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* Fire-and-forget variant — schedules the compute on the next tick and returns immediately.
|
|
78
|
+
* Use this from request handlers so HTTP responses are not blocked on model warm-up.
|
|
79
|
+
*/
|
|
80
|
+
function scheduleEmbeddingComputeAfterImport(reason) {
|
|
81
|
+
if (!autoEmbedEnabled())
|
|
82
|
+
return;
|
|
83
|
+
setImmediate(() => {
|
|
84
|
+
triggerEmbeddingComputeAfterImport(reason).catch(() => { });
|
|
85
|
+
});
|
|
86
|
+
}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* In-process ring buffer of recent log events at WARN/ERROR severity.
|
|
3
|
+
*
|
|
4
|
+
* Surfaces operationally-meaningful events to the admin dashboard so operators
|
|
5
|
+
* do not have to tail server logs. Read-only, bounded; never persists to disk.
|
|
6
|
+
*
|
|
7
|
+
* Capacity defaults to 500. The logger emits records via `recordEvent()` only
|
|
8
|
+
* for WARN or ERROR levels so that volume on healthy systems is negligible.
|
|
9
|
+
*
|
|
10
|
+
* Constitution alignment: OB-1, OB-3, OB-4, OB-5 (structured, severity-visible).
|
|
11
|
+
*/
|
|
12
|
+
export type EventLevel = 'WARN' | 'ERROR';
|
|
13
|
+
export interface BufferedEvent {
|
|
14
|
+
/** Monotonically increasing per-process id (used for unread counts). */
|
|
15
|
+
id: number;
|
|
16
|
+
/** ISO 8601 timestamp. */
|
|
17
|
+
ts: string;
|
|
18
|
+
/** Severity. */
|
|
19
|
+
level: EventLevel;
|
|
20
|
+
/** Message; prefix `[module]` is preserved. */
|
|
21
|
+
msg: string;
|
|
22
|
+
/** Optional stack/detail snippet. */
|
|
23
|
+
detail?: string;
|
|
24
|
+
/** Process id of emitter. */
|
|
25
|
+
pid?: number;
|
|
26
|
+
}
|
|
27
|
+
/** Emit a WARN/ERROR event into the buffer. Called by the logger. */
|
|
28
|
+
export declare function recordEvent(level: EventLevel, msg: string, detail?: string, pid?: number): void;
|
|
29
|
+
/** List recent events (most recent last). */
|
|
30
|
+
export declare function listEvents(opts?: {
|
|
31
|
+
sinceId?: number;
|
|
32
|
+
level?: EventLevel;
|
|
33
|
+
limit?: number;
|
|
34
|
+
}): BufferedEvent[];
|
|
35
|
+
/** Compute new-event counts since the supplied id (used for the dashboard counter bubble). */
|
|
36
|
+
export declare function eventCounts(sinceId?: number): {
|
|
37
|
+
warn: number;
|
|
38
|
+
error: number;
|
|
39
|
+
total: number;
|
|
40
|
+
latestId: number;
|
|
41
|
+
};
|
|
42
|
+
/** Clear the buffer (used by `Mark all read` and tests). */
|
|
43
|
+
export declare function clearEvents(): void;
|
|
44
|
+
/** Test-only helper. */
|
|
45
|
+
export declare function _eventBufferSize(): number;
|