@jagilber-org/index-server 1.26.11 → 1.27.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +53 -1
- package/dist/config/dashboardConfig.d.ts +11 -4
- package/dist/config/dashboardConfig.js +1 -4
- package/dist/dashboard/client/admin.html +57 -27
- package/dist/dashboard/client/css/admin.css +54 -0
- package/dist/dashboard/client/js/admin.config.js +3 -6
- package/dist/dashboard/client/js/admin.embeddings.js +63 -4
- package/dist/dashboard/client/js/admin.events.js +256 -0
- package/dist/dashboard/client/js/admin.maintenance.js +75 -32
- package/dist/dashboard/client/js/admin.sessions.js +1 -1
- package/dist/dashboard/server/AdminPanel.js +83 -6
- package/dist/dashboard/server/AdminPanelConfig.d.ts +12 -2
- package/dist/dashboard/server/AdminPanelConfig.js +48 -19
- package/dist/dashboard/server/ApiRoutes.d.ts +5 -4
- package/dist/dashboard/server/ApiRoutes.js +40 -35
- package/dist/dashboard/server/DashboardServer.js +13 -0
- package/dist/dashboard/server/routes/admin.routes.js +143 -17
- package/dist/dashboard/server/routes/embeddings.routes.js +91 -1
- package/dist/dashboard/server/routes/index.js +11 -9
- package/dist/server/sdkServer.js +12 -4
- package/dist/services/embeddingService.d.ts +2 -0
- package/dist/services/embeddingService.js +16 -4
- package/dist/services/embeddingTrigger.d.ts +33 -0
- package/dist/services/embeddingTrigger.js +86 -0
- package/dist/services/eventBuffer.d.ts +45 -0
- package/dist/services/eventBuffer.js +110 -0
- package/dist/services/handlers/instructions.import.js +71 -13
- package/dist/services/handlers.dashboardConfig.js +82 -2
- package/dist/services/indexContext.d.ts +18 -0
- package/dist/services/indexContext.js +138 -30
- package/dist/services/logger.js +9 -0
- package/dist/services/manifestManager.js +11 -1
- package/dist/services/seedBootstrap.js +5 -1
- package/dist/services/storage/factory.d.ts +2 -0
- package/dist/services/storage/factory.js +12 -1
- package/dist/services/tracing.js +3 -1
- package/package.json +12 -2
- package/schemas/index-server.code-schema.json +7424 -1588
- package/schemas/manifest.json +3 -3
- package/server.json +3 -3
package/dist/server/sdkServer.js
CHANGED
|
@@ -21,6 +21,7 @@ const registry_1 = require("./registry");
|
|
|
21
21
|
const zod_1 = require("zod");
|
|
22
22
|
const runtimeConfig_1 = require("../config/runtimeConfig");
|
|
23
23
|
const mcpLogBridge_1 = require("../services/mcpLogBridge");
|
|
24
|
+
const logger_1 = require("../services/logger");
|
|
24
25
|
const handshakeManager_1 = require("./handshakeManager");
|
|
25
26
|
const transportFactory_1 = require("./transportFactory");
|
|
26
27
|
const mcpReadOnlySurfaces_1 = require("./mcpReadOnlySurfaces");
|
|
@@ -205,6 +206,8 @@ function createSdkServer(ServerClass) {
|
|
|
205
206
|
const args = p.arguments || {};
|
|
206
207
|
if (name === 'health_check')
|
|
207
208
|
(0, handshakeManager_1.record)('tools_call_health');
|
|
209
|
+
const __callStart = Date.now();
|
|
210
|
+
(0, logger_1.logInfo)('[rpc] tools/call', { tool: name, id: req?.id ?? null });
|
|
208
211
|
try {
|
|
209
212
|
if ((0, runtimeConfig_1.getRuntimeConfig)().logging.verbose)
|
|
210
213
|
process.stderr.write(`[rpc] call method=tools/call tool=${name} id=${req?.id ?? 'n/a'}\n`);
|
|
@@ -216,9 +219,11 @@ function createSdkServer(ServerClass) {
|
|
|
216
219
|
}
|
|
217
220
|
try {
|
|
218
221
|
const result = await Promise.resolve(handler(args));
|
|
222
|
+
const bytes = Buffer.byteLength(JSON.stringify(result), 'utf8');
|
|
223
|
+
(0, logger_1.logInfo)('[rpc] tools/call ok', { tool: name, ms: Date.now() - __callStart, bytes });
|
|
219
224
|
try {
|
|
220
225
|
if ((0, runtimeConfig_1.getRuntimeConfig)().logging.verbose)
|
|
221
|
-
process.stderr.write(`[rpc] tool_result tool=${name} bytes=${
|
|
226
|
+
process.stderr.write(`[rpc] tool_result tool=${name} bytes=${bytes}\n`);
|
|
222
227
|
}
|
|
223
228
|
catch { /* ignore */ }
|
|
224
229
|
return { content: [{ type: 'text', text: JSON.stringify(result, null, 2) }] };
|
|
@@ -226,7 +231,10 @@ function createSdkServer(ServerClass) {
|
|
|
226
231
|
catch (e) {
|
|
227
232
|
const code = e?.code;
|
|
228
233
|
const sem = e?.__semantic === true;
|
|
234
|
+
const msgRaw = e instanceof Error ? e.message : String(e);
|
|
235
|
+
const stack = e instanceof Error ? e.stack : undefined;
|
|
229
236
|
if (Number.isSafeInteger(code)) {
|
|
237
|
+
(0, logger_1.logError)('[rpc] tools/call error', { tool: name, ms: Date.now() - __callStart, code, semantic: sem, message: msgRaw, stack });
|
|
230
238
|
try {
|
|
231
239
|
if ((0, runtimeConfig_1.getRuntimeConfig)().logging.verbose)
|
|
232
240
|
process.stderr.write(`[rpc] tool_error_passthru tool=${name} code=${code} semantic=${sem ? '1' : '0'} msg=${e?.message || ''}\n`);
|
|
@@ -234,13 +242,13 @@ function createSdkServer(ServerClass) {
|
|
|
234
242
|
catch { /* ignore */ }
|
|
235
243
|
throw e;
|
|
236
244
|
}
|
|
237
|
-
|
|
245
|
+
(0, logger_1.logError)('[rpc] tools/call error', { tool: name, ms: Date.now() - __callStart, code: code ?? null, message: msgRaw, stack });
|
|
238
246
|
try {
|
|
239
247
|
if ((0, runtimeConfig_1.getRuntimeConfig)().logging.verbose)
|
|
240
|
-
process.stderr.write(`[rpc] tool_error_wrap tool=${name} msg=${
|
|
248
|
+
process.stderr.write(`[rpc] tool_error_wrap tool=${name} msg=${msgRaw.replace(/\s+/g, ' ')} code=${code ?? 'n/a'}\n`);
|
|
241
249
|
}
|
|
242
250
|
catch { /* ignore */ }
|
|
243
|
-
throw { code: -32603, message: 'Tool execution failed', data: { message:
|
|
251
|
+
throw { code: -32603, message: 'Tool execution failed', data: { message: msgRaw, method: name } };
|
|
244
252
|
}
|
|
245
253
|
});
|
|
246
254
|
// Lightweight ping handler (simple reachability / latency measurement)
|
|
@@ -58,6 +58,8 @@ export declare function embedText(text: string, modelName: string, cacheDir: str
|
|
|
58
58
|
*/
|
|
59
59
|
export declare function checkModelReadiness(modelName: string, cacheDir: string, localOnly: boolean): {
|
|
60
60
|
ready: boolean;
|
|
61
|
+
cached: boolean;
|
|
62
|
+
modelPath: string;
|
|
61
63
|
message?: string;
|
|
62
64
|
};
|
|
63
65
|
/** Signature for the embed function (injectable for testing). */
|
|
@@ -187,22 +187,34 @@ async function embedText(text, modelName, cacheDir, device = 'cpu', localOnly =
|
|
|
187
187
|
* @returns Object with `ready` flag and optional remediation `message`.
|
|
188
188
|
*/
|
|
189
189
|
function checkModelReadiness(modelName, cacheDir, localOnly) {
|
|
190
|
-
if (!localOnly) {
|
|
191
|
-
return { ready: true }; // Model can be downloaded on demand
|
|
192
|
-
}
|
|
193
190
|
// HuggingFace transformers caches models as: models--<org>--<name>
|
|
194
191
|
const modelDirName = `models--${modelName.replace(/\//g, '--')}`;
|
|
195
192
|
const modelPath = path_1.default.join(cacheDir, modelDirName);
|
|
193
|
+
let cached = false;
|
|
196
194
|
try {
|
|
197
195
|
if (fs_1.default.existsSync(modelPath) && fs_1.default.readdirSync(modelPath).length > 0) {
|
|
198
|
-
|
|
196
|
+
cached = true;
|
|
199
197
|
}
|
|
200
198
|
}
|
|
201
199
|
catch {
|
|
202
200
|
// Directory doesn't exist or can't be read
|
|
203
201
|
}
|
|
202
|
+
if (cached) {
|
|
203
|
+
return { ready: true, cached: true, modelPath };
|
|
204
|
+
}
|
|
205
|
+
if (!localOnly) {
|
|
206
|
+
return {
|
|
207
|
+
ready: true,
|
|
208
|
+
cached: false,
|
|
209
|
+
modelPath,
|
|
210
|
+
message: `Embedding model '${modelName}' is not yet cached. ` +
|
|
211
|
+
`It will be downloaded to '${cacheDir}' on first compute (~25 MB).`,
|
|
212
|
+
};
|
|
213
|
+
}
|
|
204
214
|
return {
|
|
205
215
|
ready: false,
|
|
216
|
+
cached: false,
|
|
217
|
+
modelPath,
|
|
206
218
|
message: `Embedding model '${modelName}' not found in cache (${cacheDir}). ` +
|
|
207
219
|
`LOCAL_ONLY is enabled, so the model cannot be downloaded automatically. ` +
|
|
208
220
|
`To fix: set INDEX_SERVER_SEMANTIC_LOCAL_ONLY=0 to allow download, ` +
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Auto-compute embeddings after operations that mutate the loaded instruction set
|
|
3
|
+
* (zip import, restore, bulk migrations).
|
|
4
|
+
*
|
|
5
|
+
* Behaviour:
|
|
6
|
+
* - No-op when semantic is disabled (`INDEX_SERVER_SEMANTIC_ENABLED` falsy).
|
|
7
|
+
* - No-op when explicitly opted out via `INDEX_SERVER_AUTO_EMBED_ON_IMPORT=0`.
|
|
8
|
+
* - Runs asynchronously (fire-and-forget) so the caller's request returns promptly.
|
|
9
|
+
* - Coalesces concurrent triggers via the existing in-flight lock inside
|
|
10
|
+
* `getInstructionEmbeddings`.
|
|
11
|
+
*
|
|
12
|
+
* Logs success / failure at INFO / WARN so events surface in the events panel
|
|
13
|
+
* (constitution OB-3, OB-5).
|
|
14
|
+
*/
|
|
15
|
+
/** Whether auto-compute is enabled given current env / runtime config. */
|
|
16
|
+
export declare function autoEmbedEnabled(): boolean;
|
|
17
|
+
/**
|
|
18
|
+
* Trigger an embedding compute pass after an import / restore.
|
|
19
|
+
*
|
|
20
|
+
* @param reason - Free-form context (e.g. `import-zip`, `restore`) used in logs.
|
|
21
|
+
* @returns Promise resolving when compute finishes (or immediately if skipped).
|
|
22
|
+
*/
|
|
23
|
+
export declare function triggerEmbeddingComputeAfterImport(reason: string): Promise<{
|
|
24
|
+
triggered: boolean;
|
|
25
|
+
reason?: string;
|
|
26
|
+
entries?: number;
|
|
27
|
+
ms?: number;
|
|
28
|
+
}>;
|
|
29
|
+
/**
|
|
30
|
+
* Fire-and-forget variant — schedules the compute on the next tick and returns immediately.
|
|
31
|
+
* Use this from request handlers so HTTP responses are not blocked on model warm-up.
|
|
32
|
+
*/
|
|
33
|
+
export declare function scheduleEmbeddingComputeAfterImport(reason: string): void;
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Auto-compute embeddings after operations that mutate the loaded instruction set
|
|
4
|
+
* (zip import, restore, bulk migrations).
|
|
5
|
+
*
|
|
6
|
+
* Behaviour:
|
|
7
|
+
* - No-op when semantic is disabled (`INDEX_SERVER_SEMANTIC_ENABLED` falsy).
|
|
8
|
+
* - No-op when explicitly opted out via `INDEX_SERVER_AUTO_EMBED_ON_IMPORT=0`.
|
|
9
|
+
* - Runs asynchronously (fire-and-forget) so the caller's request returns promptly.
|
|
10
|
+
* - Coalesces concurrent triggers via the existing in-flight lock inside
|
|
11
|
+
* `getInstructionEmbeddings`.
|
|
12
|
+
*
|
|
13
|
+
* Logs success / failure at INFO / WARN so events surface in the events panel
|
|
14
|
+
* (constitution OB-3, OB-5).
|
|
15
|
+
*/
|
|
16
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
+
exports.autoEmbedEnabled = autoEmbedEnabled;
|
|
18
|
+
exports.triggerEmbeddingComputeAfterImport = triggerEmbeddingComputeAfterImport;
|
|
19
|
+
exports.scheduleEmbeddingComputeAfterImport = scheduleEmbeddingComputeAfterImport;
|
|
20
|
+
const runtimeConfig_1 = require("../config/runtimeConfig");
|
|
21
|
+
const indexContext_1 = require("./indexContext");
|
|
22
|
+
const embeddingService_1 = require("./embeddingService");
|
|
23
|
+
const logger_1 = require("./logger");
|
|
24
|
+
const envUtils_1 = require("../utils/envUtils");
|
|
25
|
+
let lastTriggerAt = 0;
|
|
26
|
+
/** Whether auto-compute is enabled given current env / runtime config. */
|
|
27
|
+
function autoEmbedEnabled() {
|
|
28
|
+
const cfg = (0, runtimeConfig_1.getRuntimeConfig)();
|
|
29
|
+
if (!cfg.semantic.enabled)
|
|
30
|
+
return false;
|
|
31
|
+
const raw = process.env.INDEX_SERVER_AUTO_EMBED_ON_IMPORT;
|
|
32
|
+
// Default ON when semantic is enabled; explicit '0' / 'false' opts out.
|
|
33
|
+
if (raw === undefined)
|
|
34
|
+
return true;
|
|
35
|
+
return (0, envUtils_1.getBooleanEnv)('INDEX_SERVER_AUTO_EMBED_ON_IMPORT', true);
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Trigger an embedding compute pass after an import / restore.
|
|
39
|
+
*
|
|
40
|
+
* @param reason - Free-form context (e.g. `import-zip`, `restore`) used in logs.
|
|
41
|
+
* @returns Promise resolving when compute finishes (or immediately if skipped).
|
|
42
|
+
*/
|
|
43
|
+
async function triggerEmbeddingComputeAfterImport(reason) {
|
|
44
|
+
if (!autoEmbedEnabled()) {
|
|
45
|
+
return { triggered: false, reason: 'auto-embed disabled or semantic disabled' };
|
|
46
|
+
}
|
|
47
|
+
// Light debounce — coalesce rapid back-to-back imports.
|
|
48
|
+
const now = Date.now();
|
|
49
|
+
if (now - lastTriggerAt < 1000) {
|
|
50
|
+
return { triggered: false, reason: 'debounced' };
|
|
51
|
+
}
|
|
52
|
+
lastTriggerAt = now;
|
|
53
|
+
const cfg = (0, runtimeConfig_1.getRuntimeConfig)();
|
|
54
|
+
const sem = cfg.semantic;
|
|
55
|
+
const start = Date.now();
|
|
56
|
+
try {
|
|
57
|
+
(0, indexContext_1.ensureLoaded)();
|
|
58
|
+
const state = (0, indexContext_1.getIndexState)();
|
|
59
|
+
if (!state.list || state.list.length === 0) {
|
|
60
|
+
(0, logger_1.logInfo)(`[embedding-trigger] Skipped (no instructions loaded) reason=${reason}`);
|
|
61
|
+
return { triggered: false, reason: 'no instructions loaded' };
|
|
62
|
+
}
|
|
63
|
+
(0, logger_1.logInfo)(`[embedding-trigger] Starting auto-compute reason=${reason} entries=${state.list.length}`);
|
|
64
|
+
await (0, embeddingService_1.getInstructionEmbeddings)(state.list, state.hash, sem.embeddingPath, sem.model, sem.cacheDir, sem.device, sem.localOnly);
|
|
65
|
+
const ms = Date.now() - start;
|
|
66
|
+
(0, logger_1.logInfo)(`[embedding-trigger] Auto-compute complete reason=${reason} entries=${state.list.length} ms=${ms}`);
|
|
67
|
+
return { triggered: true, entries: state.list.length, ms };
|
|
68
|
+
}
|
|
69
|
+
catch (err) {
|
|
70
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
71
|
+
// Use WARN (not ERROR) so we don't escalate transient model-load issues to ERROR-level paging.
|
|
72
|
+
(0, logger_1.logWarn)(`[embedding-trigger] Auto-compute failed reason=${reason}: ${msg}`);
|
|
73
|
+
return { triggered: false, reason: `failed: ${msg}` };
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* Fire-and-forget variant — schedules the compute on the next tick and returns immediately.
|
|
78
|
+
* Use this from request handlers so HTTP responses are not blocked on model warm-up.
|
|
79
|
+
*/
|
|
80
|
+
function scheduleEmbeddingComputeAfterImport(reason) {
|
|
81
|
+
if (!autoEmbedEnabled())
|
|
82
|
+
return;
|
|
83
|
+
setImmediate(() => {
|
|
84
|
+
triggerEmbeddingComputeAfterImport(reason).catch(() => { });
|
|
85
|
+
});
|
|
86
|
+
}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* In-process ring buffer of recent log events at WARN/ERROR severity.
|
|
3
|
+
*
|
|
4
|
+
* Surfaces operationally-meaningful events to the admin dashboard so operators
|
|
5
|
+
* do not have to tail server logs. Read-only, bounded; never persists to disk.
|
|
6
|
+
*
|
|
7
|
+
* Capacity defaults to 500. The logger emits records via `recordEvent()` only
|
|
8
|
+
* for WARN or ERROR levels so that volume on healthy systems is negligible.
|
|
9
|
+
*
|
|
10
|
+
* Constitution alignment: OB-1, OB-3, OB-4, OB-5 (structured, severity-visible).
|
|
11
|
+
*/
|
|
12
|
+
export type EventLevel = 'WARN' | 'ERROR';
|
|
13
|
+
export interface BufferedEvent {
|
|
14
|
+
/** Monotonically increasing per-process id (used for unread counts). */
|
|
15
|
+
id: number;
|
|
16
|
+
/** ISO 8601 timestamp. */
|
|
17
|
+
ts: string;
|
|
18
|
+
/** Severity. */
|
|
19
|
+
level: EventLevel;
|
|
20
|
+
/** Message; prefix `[module]` is preserved. */
|
|
21
|
+
msg: string;
|
|
22
|
+
/** Optional stack/detail snippet. */
|
|
23
|
+
detail?: string;
|
|
24
|
+
/** Process id of emitter. */
|
|
25
|
+
pid?: number;
|
|
26
|
+
}
|
|
27
|
+
/** Emit a WARN/ERROR event into the buffer. Called by the logger. */
|
|
28
|
+
export declare function recordEvent(level: EventLevel, msg: string, detail?: string, pid?: number): void;
|
|
29
|
+
/** List recent events (most recent last). */
|
|
30
|
+
export declare function listEvents(opts?: {
|
|
31
|
+
sinceId?: number;
|
|
32
|
+
level?: EventLevel;
|
|
33
|
+
limit?: number;
|
|
34
|
+
}): BufferedEvent[];
|
|
35
|
+
/** Compute new-event counts since the supplied id (used for the dashboard counter bubble). */
|
|
36
|
+
export declare function eventCounts(sinceId?: number): {
|
|
37
|
+
warn: number;
|
|
38
|
+
error: number;
|
|
39
|
+
total: number;
|
|
40
|
+
latestId: number;
|
|
41
|
+
};
|
|
42
|
+
/** Clear the buffer (used by `Mark all read` and tests). */
|
|
43
|
+
export declare function clearEvents(): void;
|
|
44
|
+
/** Test-only helper. */
|
|
45
|
+
export declare function _eventBufferSize(): number;
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* In-process ring buffer of recent log events at WARN/ERROR severity.
|
|
4
|
+
*
|
|
5
|
+
* Surfaces operationally-meaningful events to the admin dashboard so operators
|
|
6
|
+
* do not have to tail server logs. Read-only, bounded; never persists to disk.
|
|
7
|
+
*
|
|
8
|
+
* Capacity defaults to 500. The logger emits records via `recordEvent()` only
|
|
9
|
+
* for WARN or ERROR levels so that volume on healthy systems is negligible.
|
|
10
|
+
*
|
|
11
|
+
* Constitution alignment: OB-1, OB-3, OB-4, OB-5 (structured, severity-visible).
|
|
12
|
+
*/
|
|
13
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
14
|
+
exports.recordEvent = recordEvent;
|
|
15
|
+
exports.listEvents = listEvents;
|
|
16
|
+
exports.eventCounts = eventCounts;
|
|
17
|
+
exports.clearEvents = clearEvents;
|
|
18
|
+
exports._eventBufferSize = _eventBufferSize;
|
|
19
|
+
const DEFAULT_CAPACITY = 500;
|
|
20
|
+
class EventRing {
|
|
21
|
+
buf = [];
|
|
22
|
+
nextId = 1;
|
|
23
|
+
capacity = DEFAULT_CAPACITY;
|
|
24
|
+
/** Read the configured capacity (env: INDEX_SERVER_EVENT_BUFFER_SIZE, min 50, max 5000). */
|
|
25
|
+
resolveCapacity() {
|
|
26
|
+
const raw = process.env.INDEX_SERVER_EVENT_BUFFER_SIZE;
|
|
27
|
+
if (!raw)
|
|
28
|
+
return DEFAULT_CAPACITY;
|
|
29
|
+
const n = parseInt(raw, 10);
|
|
30
|
+
if (!Number.isFinite(n) || n <= 0)
|
|
31
|
+
return DEFAULT_CAPACITY;
|
|
32
|
+
return Math.min(5000, Math.max(50, n));
|
|
33
|
+
}
|
|
34
|
+
add(level, msg, detail, pid) {
|
|
35
|
+
// Refresh capacity lazily so runtime changes take effect.
|
|
36
|
+
const cap = this.resolveCapacity();
|
|
37
|
+
if (cap !== this.capacity) {
|
|
38
|
+
this.capacity = cap;
|
|
39
|
+
if (this.buf.length > cap)
|
|
40
|
+
this.buf.splice(0, this.buf.length - cap);
|
|
41
|
+
}
|
|
42
|
+
const evt = {
|
|
43
|
+
id: this.nextId++,
|
|
44
|
+
ts: new Date().toISOString(),
|
|
45
|
+
level,
|
|
46
|
+
msg,
|
|
47
|
+
pid,
|
|
48
|
+
};
|
|
49
|
+
if (detail)
|
|
50
|
+
evt.detail = detail.length > 4096 ? detail.slice(0, 4096) + '…' : detail;
|
|
51
|
+
this.buf.push(evt);
|
|
52
|
+
if (this.buf.length > this.capacity)
|
|
53
|
+
this.buf.shift();
|
|
54
|
+
}
|
|
55
|
+
/** List events newer than `sinceId` (exclusive), optionally filtered by level. */
|
|
56
|
+
list(opts = {}) {
|
|
57
|
+
const sinceId = opts.sinceId ?? 0;
|
|
58
|
+
const limit = Math.min(1000, Math.max(1, opts.limit ?? 200));
|
|
59
|
+
const filtered = [];
|
|
60
|
+
for (let i = this.buf.length - 1; i >= 0 && filtered.length < limit; i--) {
|
|
61
|
+
const e = this.buf[i];
|
|
62
|
+
if (e.id <= sinceId)
|
|
63
|
+
break;
|
|
64
|
+
if (opts.level && e.level !== opts.level)
|
|
65
|
+
continue;
|
|
66
|
+
filtered.push(e);
|
|
67
|
+
}
|
|
68
|
+
return filtered.reverse();
|
|
69
|
+
}
|
|
70
|
+
/** Counts of WARN and ERROR events newer than `sinceId`. */
|
|
71
|
+
counts(sinceId = 0) {
|
|
72
|
+
let warn = 0, error = 0;
|
|
73
|
+
for (const e of this.buf) {
|
|
74
|
+
if (e.id <= sinceId)
|
|
75
|
+
continue;
|
|
76
|
+
if (e.level === 'WARN')
|
|
77
|
+
warn++;
|
|
78
|
+
else if (e.level === 'ERROR')
|
|
79
|
+
error++;
|
|
80
|
+
}
|
|
81
|
+
const latestId = this.buf.length ? this.buf[this.buf.length - 1].id : 0;
|
|
82
|
+
return { warn, error, total: warn + error, latestId };
|
|
83
|
+
}
|
|
84
|
+
clear() {
|
|
85
|
+
this.buf = [];
|
|
86
|
+
}
|
|
87
|
+
/** Test-only: current size. */
|
|
88
|
+
size() { return this.buf.length; }
|
|
89
|
+
}
|
|
90
|
+
const ring = new EventRing();
|
|
91
|
+
/** Emit a WARN/ERROR event into the buffer. Called by the logger. */
|
|
92
|
+
function recordEvent(level, msg, detail, pid) {
|
|
93
|
+
ring.add(level, msg, detail, pid);
|
|
94
|
+
}
|
|
95
|
+
/** List recent events (most recent last). */
|
|
96
|
+
function listEvents(opts) {
|
|
97
|
+
return ring.list(opts);
|
|
98
|
+
}
|
|
99
|
+
/** Compute new-event counts since the supplied id (used for the dashboard counter bubble). */
|
|
100
|
+
function eventCounts(sinceId) {
|
|
101
|
+
return ring.counts(sinceId);
|
|
102
|
+
}
|
|
103
|
+
/** Clear the buffer (used by `Mark all read` and tests). */
|
|
104
|
+
function clearEvents() {
|
|
105
|
+
ring.clear();
|
|
106
|
+
}
|
|
107
|
+
/** Test-only helper. */
|
|
108
|
+
function _eventBufferSize() {
|
|
109
|
+
return ring.size();
|
|
110
|
+
}
|
|
@@ -15,6 +15,12 @@ const ownershipService_1 = require("../ownershipService");
|
|
|
15
15
|
const instructionRecordValidation_1 = require("../instructionRecordValidation");
|
|
16
16
|
const instructionRecordValidation_2 = require("../instructionRecordValidation");
|
|
17
17
|
const auditLog_1 = require("../auditLog");
|
|
18
|
+
const logger_1 = require("../logger");
|
|
19
|
+
// Structured WARN without auto-attached call stack: the log-hygiene gate
|
|
20
|
+
// (scripts/crawl-logs.mjs) treats WARN-with-stack as a budget violation
|
|
21
|
+
// (max-stack-warn=5). Use log('WARN', ...) directly with serialized detail
|
|
22
|
+
// so per-entry import rejections stay structured but stackless.
|
|
23
|
+
const warnStruct = (msg, detail) => (0, logger_1.log)('WARN', msg, { detail: detail === undefined ? undefined : typeof detail === 'string' ? detail : JSON.stringify(detail) });
|
|
18
24
|
const runtimeConfig_1 = require("../../config/runtimeConfig");
|
|
19
25
|
const manifestManager_1 = require("../manifestManager");
|
|
20
26
|
const instructions_shared_1 = require("./instructions.shared");
|
|
@@ -44,48 +50,77 @@ function parseInlineEntries(rawEntries) {
|
|
|
44
50
|
(0, registry_1.registerHandler)('index_import', (0, instructions_shared_1.guard)('index_import', async (p) => {
|
|
45
51
|
let entries;
|
|
46
52
|
const mode = p.mode || 'skip';
|
|
53
|
+
// Source-type breadcrumb for observability: agents currently get a silent
|
|
54
|
+
// { error: ... } back on top-level failures (path-blocked, parse errors,
|
|
55
|
+
// missing files). Without these explicit WARN logs the only signal is the
|
|
56
|
+
// RPC response, which dashboards/tails never see (RCA 2026-05-01 dev 8687).
|
|
57
|
+
const sourceType = Array.isArray(p.entries)
|
|
58
|
+
? 'inline-array'
|
|
59
|
+
: typeof p.entries === 'string'
|
|
60
|
+
? 'inline-or-file'
|
|
61
|
+
: typeof p.source === 'string'
|
|
62
|
+
? 'directory'
|
|
63
|
+
: 'none';
|
|
64
|
+
const inlineCount = Array.isArray(p.entries) ? p.entries.length : undefined;
|
|
65
|
+
(0, logger_1.logInfo)('[import] start', { mode, sourceType, inlineCount, source: typeof p.source === 'string' ? p.source : undefined });
|
|
47
66
|
if (Array.isArray(p.entries)) {
|
|
48
67
|
entries = p.entries;
|
|
49
68
|
}
|
|
50
69
|
else if (typeof p.entries === 'string') {
|
|
51
70
|
const inlineEntries = parseInlineEntries(p.entries);
|
|
52
|
-
if (inlineEntries.error)
|
|
71
|
+
if (inlineEntries.error) {
|
|
72
|
+
warnStruct('[import] rejected', { reason: 'inline-parse-error', detail: inlineEntries.error });
|
|
53
73
|
return inlineEntries.error;
|
|
74
|
+
}
|
|
54
75
|
if (inlineEntries.entries) {
|
|
55
76
|
entries = inlineEntries.entries;
|
|
56
77
|
}
|
|
57
78
|
else {
|
|
58
79
|
const filePath = path_1.default.resolve(p.entries);
|
|
59
|
-
if (!isPathAllowed(filePath))
|
|
80
|
+
if (!isPathAllowed(filePath)) {
|
|
81
|
+
warnStruct('[import] rejected', { reason: 'path-not-allowed', path: filePath });
|
|
60
82
|
return { error: 'entries path is outside allowed directories', path: filePath };
|
|
61
|
-
|
|
83
|
+
}
|
|
84
|
+
if (!fs_1.default.existsSync(filePath)) {
|
|
85
|
+
warnStruct('[import] rejected', { reason: 'entries-file-not-found', path: filePath });
|
|
62
86
|
return { error: 'entries file not found', path: filePath };
|
|
87
|
+
}
|
|
63
88
|
try {
|
|
64
89
|
const raw = JSON.parse(fs_1.default.readFileSync(filePath, 'utf8'));
|
|
65
|
-
if (!Array.isArray(raw))
|
|
90
|
+
if (!Array.isArray(raw)) {
|
|
91
|
+
warnStruct('[import] rejected', { reason: 'entries-file-not-array', path: filePath });
|
|
66
92
|
return { error: 'entries file must contain a JSON array', path: filePath };
|
|
93
|
+
}
|
|
67
94
|
entries = raw;
|
|
68
95
|
}
|
|
69
96
|
catch (e) {
|
|
97
|
+
warnStruct('[import] rejected', { reason: 'entries-file-parse-error', path: filePath, detail: e.message });
|
|
70
98
|
return { error: 'entries file parse error', path: filePath, detail: e.message };
|
|
71
99
|
}
|
|
72
100
|
}
|
|
73
101
|
}
|
|
74
102
|
else if (typeof p.source === 'string') {
|
|
75
103
|
const dirPath = path_1.default.resolve(p.source);
|
|
76
|
-
if (!isPathAllowed(dirPath))
|
|
104
|
+
if (!isPathAllowed(dirPath)) {
|
|
105
|
+
warnStruct('[import] rejected', { reason: 'source-not-allowed', path: dirPath });
|
|
77
106
|
return { error: 'source path is outside allowed directories', path: dirPath };
|
|
78
|
-
|
|
107
|
+
}
|
|
108
|
+
if (!fs_1.default.existsSync(dirPath)) {
|
|
109
|
+
warnStruct('[import] rejected', { reason: 'source-not-found', path: dirPath });
|
|
79
110
|
return { error: 'source directory not found', path: dirPath };
|
|
111
|
+
}
|
|
80
112
|
let stat;
|
|
81
113
|
try {
|
|
82
114
|
stat = fs_1.default.statSync(dirPath);
|
|
83
115
|
}
|
|
84
116
|
catch (e) {
|
|
117
|
+
warnStruct('[import] rejected', { reason: 'source-inaccessible', path: dirPath, detail: e.message });
|
|
85
118
|
return { error: 'source path inaccessible', path: dirPath, detail: e.message };
|
|
86
119
|
}
|
|
87
|
-
if (!stat.isDirectory())
|
|
120
|
+
if (!stat.isDirectory()) {
|
|
121
|
+
warnStruct('[import] rejected', { reason: 'source-not-directory', path: dirPath });
|
|
88
122
|
return { error: 'source path is not a directory', path: dirPath };
|
|
123
|
+
}
|
|
89
124
|
const files = fs_1.default.readdirSync(dirPath).filter(f => f.endsWith('.json') && !f.startsWith('_'));
|
|
90
125
|
entries = [];
|
|
91
126
|
for (const fname of files) {
|
|
@@ -95,14 +130,18 @@ function parseInlineEntries(rawEntries) {
|
|
|
95
130
|
if (parsed && typeof parsed === 'object' && parsed.id)
|
|
96
131
|
entries.push(parsed);
|
|
97
132
|
}
|
|
98
|
-
catch {
|
|
133
|
+
catch (e) {
|
|
134
|
+
warnStruct('[import] file skipped (parse error)', { file: fname, detail: e.message });
|
|
135
|
+
}
|
|
99
136
|
}
|
|
100
137
|
}
|
|
101
138
|
else {
|
|
102
139
|
entries = [];
|
|
103
140
|
}
|
|
104
|
-
if (!entries.length)
|
|
141
|
+
if (!entries.length) {
|
|
142
|
+
warnStruct('[import] rejected', { reason: 'no-entries' });
|
|
105
143
|
return { error: 'no entries' };
|
|
144
|
+
}
|
|
106
145
|
const dir = (0, indexContext_1.getInstructionsDir)();
|
|
107
146
|
if (!fs_1.default.existsSync(dir))
|
|
108
147
|
fs_1.default.mkdirSync(dir, { recursive: true });
|
|
@@ -124,13 +163,16 @@ function parseInlineEntries(rawEntries) {
|
|
|
124
163
|
].filter((issue) => !!issue);
|
|
125
164
|
const surfaceValidation = e ? (0, instructionRecordValidation_1.validateInstructionInputSurface)(e) : { validationErrors: [], hints: [], schemaRef: 'index_add#input' };
|
|
126
165
|
if (!e || requiredFieldErrors.length || surfaceValidation.validationErrors.length) {
|
|
127
|
-
|
|
166
|
+
const errMsg = formatImportValidationError([...requiredFieldErrors, ...surfaceValidation.validationErrors]);
|
|
167
|
+
errors.push({ id, error: errMsg });
|
|
168
|
+
warnStruct('[import] entry rejected', { id, reason: 'invalid-input', error: errMsg });
|
|
128
169
|
continue;
|
|
129
170
|
}
|
|
130
171
|
const bodyTrimmed = typeof e.body === 'string' ? e.body.trim() : String(e.body);
|
|
131
172
|
const { bodyWarnLength: importBodyMax } = (0, runtimeConfig_1.getRuntimeConfig)().index;
|
|
132
173
|
if (bodyTrimmed.length > importBodyMax) {
|
|
133
174
|
errors.push({ id: e.id, error: `body_too_large: ${bodyTrimmed.length} chars exceeds ${importBodyMax} limit. Split into cross-linked instructions.` });
|
|
175
|
+
warnStruct('[import] entry rejected', { id: e.id, reason: 'body-too-large', length: bodyTrimmed.length, limit: importBodyMax });
|
|
134
176
|
continue;
|
|
135
177
|
}
|
|
136
178
|
const file = path_1.default.join(dir, `${e.id}.json`);
|
|
@@ -143,6 +185,7 @@ function parseInlineEntries(rawEntries) {
|
|
|
143
185
|
if (!categories.length) {
|
|
144
186
|
if (instructionsCfg.requireCategory) {
|
|
145
187
|
errors.push({ id: e.id, error: 'category_required' });
|
|
188
|
+
warnStruct('[import] entry rejected', { id: e.id, reason: 'category-required' });
|
|
146
189
|
continue;
|
|
147
190
|
}
|
|
148
191
|
categories = ['uncategorized'];
|
|
@@ -168,10 +211,12 @@ function parseInlineEntries(rawEntries) {
|
|
|
168
211
|
}
|
|
169
212
|
if (e.priorityTier === 'P1' && (!categories.length || !e.owner)) {
|
|
170
213
|
errors.push({ id: e.id, error: 'P1 requires category & owner' });
|
|
214
|
+
warnStruct('[import] entry rejected', { id: e.id, reason: 'p1-requires-category-and-owner' });
|
|
171
215
|
continue;
|
|
172
216
|
}
|
|
173
217
|
if ((e.requirement === 'mandatory' || e.requirement === 'critical') && !e.owner) {
|
|
174
218
|
errors.push({ id: e.id, error: 'mandatory/critical require owner' });
|
|
219
|
+
warnStruct('[import] entry rejected', { id: e.id, reason: 'mandatory-critical-require-owner', requirement: e.requirement });
|
|
175
220
|
continue;
|
|
176
221
|
}
|
|
177
222
|
if (fileExists && mode === 'skip') {
|
|
@@ -194,7 +239,9 @@ function parseInlineEntries(rawEntries) {
|
|
|
194
239
|
}
|
|
195
240
|
const recordValidation = (0, instructionRecordValidation_1.validateInstructionRecord)(record);
|
|
196
241
|
if (recordValidation.validationErrors.length) {
|
|
197
|
-
|
|
242
|
+
const errMsg = formatImportValidationError(recordValidation.validationErrors);
|
|
243
|
+
errors.push({ id: e.id, error: errMsg });
|
|
244
|
+
warnStruct('[import] entry rejected', { id: e.id, reason: 'record-validation-failed', error: errMsg });
|
|
198
245
|
continue;
|
|
199
246
|
}
|
|
200
247
|
try {
|
|
@@ -202,10 +249,14 @@ function parseInlineEntries(rawEntries) {
|
|
|
202
249
|
}
|
|
203
250
|
catch (err) {
|
|
204
251
|
if ((0, instructionRecordValidation_2.isInstructionValidationError)(err)) {
|
|
205
|
-
|
|
252
|
+
const errMsg = formatImportValidationError(err.validationErrors);
|
|
253
|
+
errors.push({ id: e.id, error: errMsg });
|
|
254
|
+
(0, logger_1.logError)('[import] entry write rejected', { id: e.id, reason: 'validation-failed-at-write', error: errMsg });
|
|
206
255
|
continue;
|
|
207
256
|
}
|
|
208
|
-
|
|
257
|
+
const writeMsg = err.message || 'unknown';
|
|
258
|
+
errors.push({ id: e.id, error: `write-failed: ${writeMsg}` });
|
|
259
|
+
(0, logger_1.logError)('[import] entry write failed', { id: e.id, error: writeMsg, stack: err.stack });
|
|
209
260
|
continue;
|
|
210
261
|
}
|
|
211
262
|
if (fileExists && mode === 'overwrite')
|
|
@@ -229,10 +280,17 @@ function parseInlineEntries(rawEntries) {
|
|
|
229
280
|
if (verificationErrors.length) {
|
|
230
281
|
errors.push(...verificationErrors);
|
|
231
282
|
(0, auditLog_1.logAudit)('import_verification', verificationErrors.map(v => v.id), { missingAfterReload: verificationErrors.length });
|
|
283
|
+
(0, logger_1.logError)('[import] verification failed', { missingAfterReload: verificationErrors.length, ids: verificationErrors.map(v => v.id) });
|
|
232
284
|
}
|
|
233
285
|
const verifiedCount = writtenIds.length - verificationErrors.length;
|
|
234
286
|
const summary = { hash: st.hash, imported, skipped, overwritten, total: entries.length, errors, verified: verificationErrors.length === 0, verifiedCount, verificationErrorCount: verificationErrors.length };
|
|
235
287
|
(0, auditLog_1.logAudit)('import', entries.map(e => e.id), { imported, skipped, overwritten, errors: errors.length, verified: verificationErrors.length === 0 });
|
|
288
|
+
if (errors.length) {
|
|
289
|
+
warnStruct('[import] complete with errors', { imported, skipped, overwritten, total: entries.length, errorCount: errors.length, verifiedCount, verificationErrorCount: verificationErrors.length, errorIds: errors.map(e => e.id) });
|
|
290
|
+
}
|
|
291
|
+
else {
|
|
292
|
+
(0, logger_1.logInfo)('[import] complete', { imported, skipped, overwritten, total: entries.length, verifiedCount });
|
|
293
|
+
}
|
|
236
294
|
(0, manifestManager_1.attemptManifestUpdate)();
|
|
237
295
|
return summary;
|
|
238
296
|
}));
|