@iris-eval/mcp-server 0.5.1 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +95 -33
- package/dist/config/index.d.ts +10 -0
- package/dist/config/index.js +33 -7
- package/dist/dashboard/assets/index-CshLgDRB.js +10 -0
- package/dist/dashboard/assets/{index-UffZ-aEJ.css → index-D0cFfBqn.css} +1 -1
- package/dist/dashboard/index.html +4 -3
- package/dist/dashboard/routes/health.js +10 -3
- package/dist/dashboard/routes/moments.js +1 -1
- package/dist/dashboard/routes/preferences.d.ts +1 -0
- package/dist/dashboard/routes/preferences.js +31 -3
- package/dist/dashboard/routes/rules.d.ts +18 -0
- package/dist/dashboard/routes/rules.js +160 -6
- package/dist/dashboard/routes/traces.js +21 -3
- package/dist/dashboard/seed-demo-data.js +11 -0
- package/dist/dashboard/server.js +13 -3
- package/dist/dashboard/session-auth.d.ts +8 -0
- package/dist/dashboard/session-auth.js +237 -0
- package/dist/dashboard/validation.d.ts +9 -3
- package/dist/dashboard/validation.js +69 -11
- package/dist/eval/engine.d.ts +62 -0
- package/dist/eval/engine.js +188 -82
- package/dist/eval/rules/safety.d.ts +8 -0
- package/dist/eval/rules/safety.js +43 -11
- package/dist/index.js +102 -16
- package/dist/middleware/rate-limit.d.ts +25 -0
- package/dist/middleware/rate-limit.js +54 -2
- package/dist/self-test.d.ts +14 -0
- package/dist/self-test.js +97 -13
- package/dist/storage/demo-guard.d.ts +8 -0
- package/dist/storage/demo-guard.js +53 -0
- package/dist/storage/sqlite-adapter.d.ts +6 -0
- package/dist/storage/sqlite-adapter.js +72 -1
- package/dist/tools/delete-rule.js +49 -11
- package/dist/tools/deploy-rule.d.ts +33 -0
- package/dist/tools/deploy-rule.js +130 -27
- package/dist/tools/evaluate-output.js +41 -22
- package/dist/tools/evaluate-with-llm-judge.js +10 -3
- package/dist/tools/get-traces.d.ts +27 -0
- package/dist/tools/get-traces.js +60 -8
- package/dist/tools/list-rules.js +2 -2
- package/dist/tools/log-trace.js +4 -3
- package/dist/tools/strict-input.d.ts +1 -0
- package/dist/tools/strict-input.js +25 -0
- package/dist/tools/trace-link.d.ts +7 -0
- package/dist/tools/trace-link.js +39 -0
- package/dist/tools/verify-citations.d.ts +19 -0
- package/dist/tools/verify-citations.js +41 -4
- package/dist/types/eval.d.ts +45 -1
- package/dist/types/index.d.ts +1 -1
- package/dist/types/query.d.ts +25 -0
- package/package.json +1 -1
- package/server.json +2 -2
- package/dist/dashboard/assets/index-VI_nbMfN.js +0 -10
package/dist/index.js
CHANGED
|
@@ -2,7 +2,9 @@
|
|
|
2
2
|
import { parseArgs } from 'node:util';
|
|
3
3
|
import { z } from 'zod';
|
|
4
4
|
import { loadConfig } from './config/index.js';
|
|
5
|
+
import { PKG_VERSION } from './config/defaults.js';
|
|
5
6
|
import { createStorage } from './storage/index.js';
|
|
7
|
+
import { withDemoIngestGuard } from './storage/demo-guard.js';
|
|
6
8
|
import { createIrisServer } from './server.js';
|
|
7
9
|
import { createStdioTransport } from './transport/stdio.js';
|
|
8
10
|
import { createHttpTransport } from './transport/http.js';
|
|
@@ -35,6 +37,8 @@ const CliSchema = z
|
|
|
35
37
|
demo: z.boolean().optional(),
|
|
36
38
|
'demo-clear': z.boolean().optional(),
|
|
37
39
|
'self-test': z.boolean().optional(),
|
|
40
|
+
purge: z.boolean().optional(),
|
|
41
|
+
version: z.boolean().optional(),
|
|
38
42
|
help: z.boolean().optional(),
|
|
39
43
|
})
|
|
40
44
|
.strict();
|
|
@@ -65,6 +69,8 @@ try {
|
|
|
65
69
|
demo: { type: 'boolean', default: false },
|
|
66
70
|
'demo-clear': { type: 'boolean', default: false },
|
|
67
71
|
'self-test': { type: 'boolean', default: false },
|
|
72
|
+
purge: { type: 'boolean', default: false },
|
|
73
|
+
version: { type: 'boolean', default: false },
|
|
68
74
|
help: { type: 'boolean', short: 'h', default: false },
|
|
69
75
|
},
|
|
70
76
|
strict: true,
|
|
@@ -74,6 +80,16 @@ catch (err) {
|
|
|
74
80
|
process.stderr.write(`iris-mcp: ${err.message}\nRun \`iris-mcp --help\` for usage.\n`);
|
|
75
81
|
process.exit(2);
|
|
76
82
|
}
|
|
83
|
+
/*
|
|
84
|
+
* --version answers on stdout, bare, before anything else: the README's
|
|
85
|
+
* "check your version" recipe pointed at a flag that did not exist and a
|
|
86
|
+
* --help banner that printed no version (#369). Bare so `iris-mcp
|
|
87
|
+
* --version` composes in scripts the way `npm --version` does.
|
|
88
|
+
*/
|
|
89
|
+
if (parsed.values.version) {
|
|
90
|
+
process.stdout.write(`${PKG_VERSION}\n`);
|
|
91
|
+
process.exit(0);
|
|
92
|
+
}
|
|
77
93
|
const validation = CliSchema.safeParse(parsed.values);
|
|
78
94
|
if (!validation.success) {
|
|
79
95
|
const issues = validation.error.issues
|
|
@@ -85,7 +101,7 @@ if (!validation.success) {
|
|
|
85
101
|
const values = validation.data;
|
|
86
102
|
if (values.help) {
|
|
87
103
|
process.stderr.write(`
|
|
88
|
-
Iris — MCP-Native Agent Eval Server
|
|
104
|
+
Iris — MCP-Native Agent Eval Server v${PKG_VERSION}
|
|
89
105
|
|
|
90
106
|
Usage: iris-mcp [options]
|
|
91
107
|
|
|
@@ -107,9 +123,16 @@ Options:
|
|
|
107
123
|
transport). Idempotent: re-running reuses the seeded data.
|
|
108
124
|
--demo-clear Delete the demo database (and its sidecar files), then exit.
|
|
109
125
|
Your real traces are not touched.
|
|
110
|
-
--self-test Run the offline install diagnostic and exit:
|
|
111
|
-
|
|
126
|
+
--self-test Run the offline install diagnostic and exit: the configured IRIS_HOME
|
|
127
|
+
is created and probed for writability, then storage round-trip,
|
|
128
|
+
deterministic evals, dashboard + rebinding guard run inside an
|
|
112
129
|
isolated temp home. Exit code 0 = healthy, 1 = a check failed.
|
|
130
|
+
--purge Delete EVERY stored trace, span and evaluation from the configured
|
|
131
|
+
database, compact the file and truncate the write-ahead log so the
|
|
132
|
+
deleted text does not linger on disk, then exit. Deployed rules, the
|
|
133
|
+
audit log and preferences are kept. Not reversible. Stop any running
|
|
134
|
+
Iris server first — the file is compacted in place.
|
|
135
|
+
--version Print the version and exit
|
|
113
136
|
-h, --help Show this help message
|
|
114
137
|
|
|
115
138
|
Environment variables (CLI flags take precedence):
|
|
@@ -139,11 +162,31 @@ Environment variables (CLI flags take precedence):
|
|
|
139
162
|
IRIS_OTEL_TIMEOUT_MS Per-export timeout (default: 15000)
|
|
140
163
|
RATE_LIMIT_SALT (waitlist API only — required when website is deployed)
|
|
141
164
|
|
|
142
|
-
Dashboard preferences (~/.iris/preferences.json):
|
|
165
|
+
Dashboard preferences ($IRIS_HOME/preferences.json, default ~/.iris/preferences.json):
|
|
143
166
|
Edit autoLaunch: false to permanently disable first-run dashboard auto-launch.
|
|
144
167
|
`);
|
|
145
168
|
process.exit(0);
|
|
146
169
|
}
|
|
170
|
+
/*
|
|
171
|
+
* The mode flags are mutually exclusive, and the check runs before any of
|
|
172
|
+
* them so a refused combination exits without touching the filesystem —
|
|
173
|
+
* `--self-test --purge` must not quietly run only the first one it sees.
|
|
174
|
+
*/
|
|
175
|
+
const modeFlags = ['demo', 'demo-clear', 'self-test', 'purge'].filter((flag) => values[flag]);
|
|
176
|
+
if (modeFlags.length > 1) {
|
|
177
|
+
process.stderr.write(`iris-mcp: ${modeFlags.map((flag) => `--${flag}`).join(' and ')} cannot be combined.\nRun \`iris-mcp --help\` for usage.\n`);
|
|
178
|
+
process.exit(2);
|
|
179
|
+
}
|
|
180
|
+
/*
|
|
181
|
+
* `--purge --dashboard` used to run the purge and drop the dashboard flag
|
|
182
|
+
* on the floor. A flag that changes nothing is the same fault as a
|
|
183
|
+
* misspelled tool argument: the caller asked for something and got no
|
|
184
|
+
* sign it was ignored. (v0.6.0 acceptance pass.)
|
|
185
|
+
*/
|
|
186
|
+
if (values.purge && values.dashboard) {
|
|
187
|
+
process.stderr.write('iris-mcp: --purge exits as soon as the purge finishes and cannot be combined with --dashboard (nothing would be served).\nRun `iris-mcp --help` for usage.\n');
|
|
188
|
+
process.exit(2);
|
|
189
|
+
}
|
|
147
190
|
/*
|
|
148
191
|
* --self-test exits BEFORE loadConfig() runs at module scope below —
|
|
149
192
|
* deliberately. The diagnostic builds its own isolated IRIS_HOME and
|
|
@@ -154,14 +197,6 @@ if (values['self-test']) {
|
|
|
154
197
|
const { runSelfTest } = await import('./self-test.js');
|
|
155
198
|
process.exit(await runSelfTest());
|
|
156
199
|
}
|
|
157
|
-
/*
|
|
158
|
-
* Demo-mode flag validation happens before loadConfig so a refused
|
|
159
|
-
* combination exits without touching the filesystem.
|
|
160
|
-
*/
|
|
161
|
-
if (values.demo && values['demo-clear']) {
|
|
162
|
-
process.stderr.write('iris-mcp: --demo and --demo-clear cannot be combined.\nRun `iris-mcp --help` for usage.\n');
|
|
163
|
-
process.exit(2);
|
|
164
|
-
}
|
|
165
200
|
if (values.demo && values['db-path']) {
|
|
166
201
|
process.stderr.write('iris-mcp: --demo always serves its own database (demo.db under your iris home) and cannot be combined with --db-path.\n' +
|
|
167
202
|
'Run `iris-mcp --demo` alone, or drop --demo to use your own database.\n');
|
|
@@ -193,6 +228,39 @@ const config = loadConfig({
|
|
|
193
228
|
dashboardHost: values['dashboard-host'],
|
|
194
229
|
});
|
|
195
230
|
const logger = createLogger(config);
|
|
231
|
+
/*
|
|
232
|
+
* --purge (#372): the retention sweep only ever trimmed by age, and
|
|
233
|
+
* deleting iris.db by hand left every row readable in iris.db-wal. This
|
|
234
|
+
* is the one-command answer to "remove everything Iris stored about my
|
|
235
|
+
* agents" — every trace, span and evaluation for the local tenant,
|
|
236
|
+
* followed by VACUUM + a TRUNCATE checkpoint so the text is gone from the
|
|
237
|
+
* main file and the write-ahead log alike. Rules, audit log and
|
|
238
|
+
* preferences are not storage rows and stay. Prints what it removed and
|
|
239
|
+
* exits 0; runs against the configured database, never the demo one
|
|
240
|
+
* (--demo-clear handles that).
|
|
241
|
+
*/
|
|
242
|
+
async function runPurge() {
|
|
243
|
+
const storage = createStorage(config);
|
|
244
|
+
await storage.initialize();
|
|
245
|
+
try {
|
|
246
|
+
const { traces, evalResults } = await storage.purge(LOCAL_TENANT);
|
|
247
|
+
process.stderr.write(`iris-mcp: purged ${traces} trace(s) and ${evalResults} evaluation(s) from "${config.storage.path}" ` +
|
|
248
|
+
'(database compacted, write-ahead log truncated). Deployed rules, audit log and preferences were kept.\n');
|
|
249
|
+
}
|
|
250
|
+
finally {
|
|
251
|
+
await storage.close();
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
if (values.purge) {
|
|
255
|
+
try {
|
|
256
|
+
await runPurge();
|
|
257
|
+
process.exit(0);
|
|
258
|
+
}
|
|
259
|
+
catch (err) {
|
|
260
|
+
process.stderr.write(`iris-mcp: purge failed: ${err instanceof Error ? err.message : String(err)}\n`);
|
|
261
|
+
process.exit(1);
|
|
262
|
+
}
|
|
263
|
+
}
|
|
196
264
|
async function main() {
|
|
197
265
|
logger.info(`Starting Iris MCP server v${config.server.version}`);
|
|
198
266
|
// F-006: fail fast on HTTP+dashboard port collision. See validatePortConfig.
|
|
@@ -231,9 +299,19 @@ async function main() {
|
|
|
231
299
|
// cleanly.
|
|
232
300
|
if (config.retention.days > 0) {
|
|
233
301
|
try {
|
|
234
|
-
const
|
|
235
|
-
|
|
236
|
-
|
|
302
|
+
const deletedTraces = await storage.deleteTracesOlderThan(LOCAL_TENANT, config.retention.days);
|
|
303
|
+
/*
|
|
304
|
+
* Evaluations too (#372). Deleting a trace only NULLs trace_id on
|
|
305
|
+
* its evaluations, so every eval row — output_text verbatim,
|
|
306
|
+
* including whatever no_pii flagged — used to outlive the retention
|
|
307
|
+
* window indefinitely while the traces around it were swept.
|
|
308
|
+
*/
|
|
309
|
+
const deletedEvals = await storage.deleteEvalResultsOlderThan(LOCAL_TENANT, config.retention.days);
|
|
310
|
+
if (deletedTraces + deletedEvals > 0) {
|
|
311
|
+
// Fold the WAL into the main file and truncate it, so the swept
|
|
312
|
+
// rows do not survive as readable text in iris.db-wal.
|
|
313
|
+
await storage.checkpoint();
|
|
314
|
+
logger.info(`Retention cleanup: deleted ${deletedTraces} trace(s) and ${deletedEvals} evaluation(s) older than ${config.retention.days} days`);
|
|
237
315
|
}
|
|
238
316
|
}
|
|
239
317
|
catch (err) {
|
|
@@ -332,6 +410,8 @@ ${line}
|
|
|
332
410
|
${counts}
|
|
333
411
|
Demo database: "${summary.dbPath}"
|
|
334
412
|
Your real trace database is untouched — demo data never mixes with it.
|
|
413
|
+
Trace ingest (POST /api/v1/traces) is refused in demo mode: start the
|
|
414
|
+
real server (iris-mcp --dashboard) to store your own traces.
|
|
335
415
|
|
|
336
416
|
Worth clicking into:
|
|
337
417
|
- a PII leak (a synthetic SSN in an agent reply) caught by the safety rules
|
|
@@ -367,7 +447,13 @@ async function runDemo() {
|
|
|
367
447
|
else {
|
|
368
448
|
logger.info(`Seeded demo database with ${seedSummary.traceCount} traces at ${seedSummary.dbPath}`);
|
|
369
449
|
}
|
|
370
|
-
|
|
450
|
+
/*
|
|
451
|
+
* Ingest is refused in demo mode: demo.db is disposable by design and
|
|
452
|
+
* --demo-clear deletes it wholesale, so a capture client pointed at the
|
|
453
|
+
* demo dashboard's port would have its real traces silently stored next
|
|
454
|
+
* to the fake ones and later destroyed (storage/demo-guard.ts).
|
|
455
|
+
*/
|
|
456
|
+
const storage = withDemoIngestGuard(createStorage(config));
|
|
371
457
|
await storage.initialize();
|
|
372
458
|
const customRuleStore = createCustomRuleStore({
|
|
373
459
|
pathFor: () => demoCustomRulesPath(),
|
|
@@ -1,3 +1,28 @@
|
|
|
1
1
|
import type { IrisConfig } from '../types/config.js';
|
|
2
2
|
export declare function createApiRateLimiter(config: Pick<IrisConfig, 'security'>): import("express-rate-limit").RateLimitRequestHandler;
|
|
3
|
+
/**
|
|
4
|
+
* Mounted directly in front of the session layer, so every request that
|
|
5
|
+
* is about to be AUTHORIZED — cookie check, Bearer check, the `?key=`
|
|
6
|
+
* exchange — has passed a per-IP ceiling first (CodeQL
|
|
7
|
+
* js/missing-rate-limiting on the session middleware). Same figure as the
|
|
8
|
+
* API limiter with its own counter; static assets share it, which a
|
|
9
|
+
* dashboard page load (a few dozen requests) never approaches.
|
|
10
|
+
*/
|
|
11
|
+
export declare function createAuthGateRateLimiter(config: Pick<IrisConfig, 'security'>): import("express-rate-limit").RateLimitRequestHandler;
|
|
12
|
+
/**
|
|
13
|
+
* JSON-RPC 2.0 application error code for "rate limited". The reserved
|
|
14
|
+
* server range is -32000..-32099; the MCP SDK uses -32000/-32001 for
|
|
15
|
+
* connection-closed and request-timeout, so this sits clear of both.
|
|
16
|
+
* Mirrors HTTP 429 in the low digits so a log line reads at a glance.
|
|
17
|
+
*/
|
|
18
|
+
export declare const JSON_RPC_RATE_LIMITED = -32029;
|
|
19
|
+
/**
|
|
20
|
+
* The MCP endpoint speaks JSON-RPC, so its 429 must too (#373). The stock
|
|
21
|
+
* express-rate-limit body — `{ "error": "Too many requests" }` — is not a
|
|
22
|
+
* JSON-RPC message: a strict client (the reference SDK included) fails to
|
|
23
|
+
* parse the response and surfaces a PROTOCOL error, and the one thing the
|
|
24
|
+
* caller needed to learn — wait, then retry — is exactly what got lost. The
|
|
25
|
+
* envelope below echoes the request id when the body carried one, names
|
|
26
|
+
* the limit and the wait, and points at the config key that raises it.
|
|
27
|
+
*/
|
|
3
28
|
export declare function createMcpRateLimiter(config: Pick<IrisConfig, 'security'>): import("express-rate-limit").RateLimitRequestHandler;
|
|
@@ -8,12 +8,64 @@ export function createApiRateLimiter(config) {
|
|
|
8
8
|
message: { error: 'Too many requests, please try again later' },
|
|
9
9
|
});
|
|
10
10
|
}
|
|
11
|
-
|
|
11
|
+
/**
|
|
12
|
+
* Mounted directly in front of the session layer, so every request that
|
|
13
|
+
* is about to be AUTHORIZED — cookie check, Bearer check, the `?key=`
|
|
14
|
+
* exchange — has passed a per-IP ceiling first (CodeQL
|
|
15
|
+
* js/missing-rate-limiting on the session middleware). Same figure as the
|
|
16
|
+
* API limiter with its own counter; static assets share it, which a
|
|
17
|
+
* dashboard page load (a few dozen requests) never approaches.
|
|
18
|
+
*/
|
|
19
|
+
export function createAuthGateRateLimiter(config) {
|
|
12
20
|
return rateLimit({
|
|
13
21
|
windowMs: 60_000,
|
|
14
|
-
limit: config.security.rateLimit.
|
|
22
|
+
limit: config.security.rateLimit.api,
|
|
15
23
|
standardHeaders: 'draft-7',
|
|
16
24
|
legacyHeaders: false,
|
|
17
25
|
message: { error: 'Too many requests, please try again later' },
|
|
18
26
|
});
|
|
19
27
|
}
|
|
28
|
+
/**
|
|
29
|
+
* JSON-RPC 2.0 application error code for "rate limited". The reserved
|
|
30
|
+
* server range is -32000..-32099; the MCP SDK uses -32000/-32001 for
|
|
31
|
+
* connection-closed and request-timeout, so this sits clear of both.
|
|
32
|
+
* Mirrors HTTP 429 in the low digits so a log line reads at a glance.
|
|
33
|
+
*/
|
|
34
|
+
export const JSON_RPC_RATE_LIMITED = -32029;
|
|
35
|
+
/**
|
|
36
|
+
* The MCP endpoint speaks JSON-RPC, so its 429 must too (#373). The stock
|
|
37
|
+
* express-rate-limit body — `{ "error": "Too many requests" }` — is not a
|
|
38
|
+
* JSON-RPC message: a strict client (the reference SDK included) fails to
|
|
39
|
+
* parse the response and surfaces a PROTOCOL error, and the one thing the
|
|
40
|
+
* caller needed to learn — wait, then retry — is exactly what got lost. The
|
|
41
|
+
* envelope below echoes the request id when the body carried one, names
|
|
42
|
+
* the limit and the wait, and points at the config key that raises it.
|
|
43
|
+
*/
|
|
44
|
+
export function createMcpRateLimiter(config) {
|
|
45
|
+
const limit = config.security.rateLimit.mcp;
|
|
46
|
+
return rateLimit({
|
|
47
|
+
windowMs: 60_000,
|
|
48
|
+
limit,
|
|
49
|
+
standardHeaders: 'draft-7',
|
|
50
|
+
legacyHeaders: false,
|
|
51
|
+
handler: (req, res) => {
|
|
52
|
+
const body = req.body;
|
|
53
|
+
const requestId = body !== null && typeof body === 'object' && !Array.isArray(body) && 'id' in body
|
|
54
|
+
? body.id
|
|
55
|
+
: null;
|
|
56
|
+
const id = typeof requestId === 'string' || typeof requestId === 'number' ? requestId : null;
|
|
57
|
+
const resetTime = req.rateLimit?.resetTime;
|
|
58
|
+
const retryAfterSeconds = resetTime instanceof Date ? Math.max(1, Math.ceil((resetTime.getTime() - Date.now()) / 1000)) : 60;
|
|
59
|
+
res.status(429).json({
|
|
60
|
+
jsonrpc: '2.0',
|
|
61
|
+
id,
|
|
62
|
+
error: {
|
|
63
|
+
code: JSON_RPC_RATE_LIMITED,
|
|
64
|
+
message: `Rate limit exceeded: this MCP endpoint allows ${limit} requests per minute. ` +
|
|
65
|
+
`Retry in ${retryAfterSeconds}s, or raise security.rateLimit.mcp in config.json for an interactive session.`,
|
|
66
|
+
data: { limit, windowMs: 60_000, retryAfterSeconds },
|
|
67
|
+
},
|
|
68
|
+
});
|
|
69
|
+
},
|
|
70
|
+
});
|
|
71
|
+
}
|
package/dist/self-test.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
export declare const SELF_TEST_STEPS: {
|
|
2
|
+
readonly configuredHome: "configured IRIS_HOME is writable";
|
|
2
3
|
readonly tempHome: "create isolated temp home";
|
|
3
4
|
readonly storage: "initialize storage";
|
|
4
5
|
readonly trace: "log a trace";
|
|
@@ -15,4 +16,17 @@ export declare const SELF_TEST_STEPS: {
|
|
|
15
16
|
export declare const SELF_TEST_PASS_VERDICT = "\u2713 PASS \u2014 this install works";
|
|
16
17
|
export declare const SELF_TEST_FAIL_VERDICT = "\u2717 FAIL";
|
|
17
18
|
export type WriteLine = (line: string) => void;
|
|
19
|
+
/**
|
|
20
|
+
* The configured-home probe (#371). Exercises the exact calls the real
|
|
21
|
+
* server makes at startup, in order: create IRIS_HOME (same helper and
|
|
22
|
+
* mode as loadConfig), create the database directory when IRIS_DB_PATH
|
|
23
|
+
* points elsewhere, write-and-unlink a probe file in each, and — only if
|
|
24
|
+
* the real database already exists — open it and take a write lock
|
|
25
|
+
* (BEGIN IMMEDIATE … ROLLBACK), which fails on a read-only file or a
|
|
26
|
+
* non-database exactly as the first INSERT would, without migrating or
|
|
27
|
+
* changing anything. A missing database is not created: the server
|
|
28
|
+
* creates it on first run, and the writable-directory probe is what
|
|
29
|
+
* proves that it can.
|
|
30
|
+
*/
|
|
31
|
+
export declare function probeConfiguredHome(home: string, dbPath: string): string;
|
|
18
32
|
export declare function runSelfTest(write?: WriteLine): Promise<number>;
|
package/dist/self-test.js
CHANGED
|
@@ -11,10 +11,20 @@
|
|
|
11
11
|
*
|
|
12
12
|
* Isolation is the load-bearing property. The diagnostic creates its own
|
|
13
13
|
* scratch IRIS_HOME and scrubs every IRIS_* env var that feeds
|
|
14
|
-
* loadConfig(), so it
|
|
15
|
-
* never reads their config.json, and never honours an
|
|
16
|
-
* would 401 its own probes. The scratch home is removed
|
|
17
|
-
* restored before returning — pass or fail.
|
|
14
|
+
* loadConfig(), so it never MIGRATES or writes rows into the user's real
|
|
15
|
+
* iris.db, never reads their config.json, and never honours an
|
|
16
|
+
* IRIS_API_KEY that would 401 its own probes. The scratch home is removed
|
|
17
|
+
* and the env restored before returning — pass or fail.
|
|
18
|
+
*
|
|
19
|
+
* Isolation is not the same as ignorance, though. The first check runs
|
|
20
|
+
* BEFORE the scrub, against the CONFIGURED home: it creates the directory
|
|
21
|
+
* the server would create, proves it can write there, and — when the real
|
|
22
|
+
* database already exists — opens it and takes (then releases) a write
|
|
23
|
+
* lock without changing a byte. #371: the diagnostic used to print PASS
|
|
24
|
+
* against an IRIS_HOME the server could not write, because every check
|
|
25
|
+
* ran in the temp home; the real server then died on startup with a raw
|
|
26
|
+
* EPERM stack. A diagnostic that cannot fail the way the product fails is
|
|
27
|
+
* not a diagnostic.
|
|
18
28
|
*
|
|
19
29
|
* Budget: everything is in-process or loopback. No LLM calls, no network
|
|
20
30
|
* beyond 127.0.0.1, and the whole sequence completes in well under the
|
|
@@ -23,11 +33,13 @@
|
|
|
23
33
|
* Exit contract: 0 = every check passed, 1 = any check failed. index.ts
|
|
24
34
|
* runs this BEFORE loadConfig() so the normal boot path never executes.
|
|
25
35
|
*/
|
|
26
|
-
import { mkdtempSync, rmSync } from 'node:fs';
|
|
36
|
+
import { existsSync, mkdtempSync, rmSync, unlinkSync, writeFileSync } from 'node:fs';
|
|
27
37
|
import { tmpdir } from 'node:os';
|
|
28
|
-
import { join } from 'node:path';
|
|
38
|
+
import { dirname, join } from 'node:path';
|
|
39
|
+
import { randomBytes } from 'node:crypto';
|
|
29
40
|
import { request as httpRequest } from 'node:http';
|
|
30
|
-
import
|
|
41
|
+
import Database from 'better-sqlite3';
|
|
42
|
+
import { ensureIrisDirectory, loadConfig } from './config/index.js';
|
|
31
43
|
import { PKG_VERSION } from './config/defaults.js';
|
|
32
44
|
import { createStorage } from './storage/index.js';
|
|
33
45
|
import { createDashboardServer } from './dashboard/server.js';
|
|
@@ -44,6 +56,7 @@ const CROSS = '✗';
|
|
|
44
56
|
* files, per the usual drift rule.
|
|
45
57
|
*/
|
|
46
58
|
export const SELF_TEST_STEPS = {
|
|
59
|
+
configuredHome: 'configured IRIS_HOME is writable',
|
|
47
60
|
tempHome: 'create isolated temp home',
|
|
48
61
|
storage: 'initialize storage',
|
|
49
62
|
trace: 'log a trace',
|
|
@@ -114,16 +127,77 @@ function probe(port, path, headers = {}) {
|
|
|
114
127
|
});
|
|
115
128
|
}
|
|
116
129
|
const stdoutLine = (line) => process.stdout.write(`${line}\n`);
|
|
130
|
+
function errorCode(err) {
|
|
131
|
+
const code = err?.code;
|
|
132
|
+
return typeof code === 'string' ? code : err instanceof Error ? err.message : String(err);
|
|
133
|
+
}
|
|
134
|
+
/**
|
|
135
|
+
* The configured-home probe (#371). Exercises the exact calls the real
|
|
136
|
+
* server makes at startup, in order: create IRIS_HOME (same helper and
|
|
137
|
+
* mode as loadConfig), create the database directory when IRIS_DB_PATH
|
|
138
|
+
* points elsewhere, write-and-unlink a probe file in each, and — only if
|
|
139
|
+
* the real database already exists — open it and take a write lock
|
|
140
|
+
* (BEGIN IMMEDIATE … ROLLBACK), which fails on a read-only file or a
|
|
141
|
+
* non-database exactly as the first INSERT would, without migrating or
|
|
142
|
+
* changing anything. A missing database is not created: the server
|
|
143
|
+
* creates it on first run, and the writable-directory probe is what
|
|
144
|
+
* proves that it can.
|
|
145
|
+
*/
|
|
146
|
+
export function probeConfiguredHome(home, dbPath) {
|
|
147
|
+
ensureIrisDirectory(home, 'IRIS_HOME');
|
|
148
|
+
probeWritable(home, 'IRIS_HOME');
|
|
149
|
+
const dbDir = dirname(dbPath);
|
|
150
|
+
if (dbDir !== home) {
|
|
151
|
+
ensureIrisDirectory(dbDir, 'the database directory (IRIS_DB_PATH / --db-path)');
|
|
152
|
+
probeWritable(dbDir, 'the database directory');
|
|
153
|
+
}
|
|
154
|
+
if (!existsSync(dbPath)) {
|
|
155
|
+
return `${home} (database ${dbPath} will be created on first run)`;
|
|
156
|
+
}
|
|
157
|
+
let db;
|
|
158
|
+
try {
|
|
159
|
+
db = new Database(dbPath, { fileMustExist: true });
|
|
160
|
+
db.exec('BEGIN IMMEDIATE');
|
|
161
|
+
db.exec('ROLLBACK');
|
|
162
|
+
}
|
|
163
|
+
catch (err) {
|
|
164
|
+
throw new Error(`database "${dbPath}" exists but cannot be opened for writing (${errorCode(err)}) — the server would fail ` +
|
|
165
|
+
'at startup with the same error. Fix the file permissions, or point IRIS_DB_PATH / --db-path at a writable location.');
|
|
166
|
+
}
|
|
167
|
+
finally {
|
|
168
|
+
db?.close();
|
|
169
|
+
}
|
|
170
|
+
return `${home} (database ${dbPath} opens for writing)`;
|
|
171
|
+
}
|
|
172
|
+
function probeWritable(dir, what) {
|
|
173
|
+
const probeFile = join(dir, `.iris-self-test-${randomBytes(4).toString('hex')}`);
|
|
174
|
+
try {
|
|
175
|
+
writeFileSync(probeFile, 'iris self-test write probe\n', { mode: 0o600 });
|
|
176
|
+
}
|
|
177
|
+
catch (err) {
|
|
178
|
+
throw new Error(`${what} "${dir}" is not writable (${errorCode(err)}) — the server would fail at startup with the same error. ` +
|
|
179
|
+
'Point IRIS_HOME at a directory this user can write, or fix the permissions on that path.');
|
|
180
|
+
}
|
|
181
|
+
try {
|
|
182
|
+
unlinkSync(probeFile);
|
|
183
|
+
}
|
|
184
|
+
catch {
|
|
185
|
+
// Written but not removable: unusual (sticky bit, AV lock). Not a
|
|
186
|
+
// startup blocker, so not a failure; the file is tiny and named for
|
|
187
|
+
// what it is.
|
|
188
|
+
}
|
|
189
|
+
}
|
|
117
190
|
export async function runSelfTest(write = stdoutLine) {
|
|
118
191
|
write(`Iris self-test v${PKG_VERSION}`);
|
|
119
192
|
write('');
|
|
120
193
|
/*
|
|
121
194
|
* Resolved BEFORE the env scrub: this is where a normal (non-self-test)
|
|
122
195
|
* run of this install would keep its data, which is the line the user
|
|
123
|
-
* actually wants from a diagnostic
|
|
124
|
-
*
|
|
196
|
+
* actually wants from a diagnostic — and the target of the configured-
|
|
197
|
+
* home probe below. The isolated checks never touch these paths.
|
|
125
198
|
*/
|
|
126
|
-
const
|
|
199
|
+
const userHome = irisHome();
|
|
200
|
+
const userStoragePath = process.env.IRIS_DB_PATH ?? join(userHome, 'iris.db');
|
|
127
201
|
const savedEnv = {};
|
|
128
202
|
for (const key of SCRUBBED_ENV_VARS) {
|
|
129
203
|
savedEnv[key] = process.env[key];
|
|
@@ -137,14 +211,18 @@ export async function runSelfTest(write = stdoutLine) {
|
|
|
137
211
|
let traceId = '';
|
|
138
212
|
const insertedIds = [];
|
|
139
213
|
const failedSteps = [];
|
|
214
|
+
let halted = false;
|
|
140
215
|
/*
|
|
141
216
|
* Steps run strictly in order and stop at the first failure — each one
|
|
142
217
|
* depends on the state the previous one built, so a cascade of
|
|
143
218
|
* follow-on crosses would only bury the real cause. Cleanup runs
|
|
144
|
-
* unconditionally afterwards.
|
|
219
|
+
* unconditionally afterwards. A step marked `independent` still fails
|
|
220
|
+
* the run but does not halt it: the configured-home probe has no
|
|
221
|
+
* successors that depend on it, and the user is better served by ALSO
|
|
222
|
+
* learning whether the install itself works.
|
|
145
223
|
*/
|
|
146
|
-
const step = async (label, fn) => {
|
|
147
|
-
if (
|
|
224
|
+
const step = async (label, fn, opts) => {
|
|
225
|
+
if (halted)
|
|
148
226
|
return;
|
|
149
227
|
try {
|
|
150
228
|
const detail = await fn();
|
|
@@ -152,9 +230,14 @@ export async function runSelfTest(write = stdoutLine) {
|
|
|
152
230
|
}
|
|
153
231
|
catch (err) {
|
|
154
232
|
failedSteps.push(label);
|
|
233
|
+
if (!opts?.independent)
|
|
234
|
+
halted = true;
|
|
155
235
|
write(`${CROSS} ${label} — ${err instanceof Error ? err.message : String(err)}`);
|
|
156
236
|
}
|
|
157
237
|
};
|
|
238
|
+
await step(SELF_TEST_STEPS.configuredHome, () => probeConfiguredHome(userHome, userStoragePath), {
|
|
239
|
+
independent: true,
|
|
240
|
+
});
|
|
158
241
|
await step(SELF_TEST_STEPS.tempHome, () => {
|
|
159
242
|
tempHome = mkdtempSync(join(tmpdir(), 'iris-self-test-'));
|
|
160
243
|
for (const key of SCRUBBED_ENV_VARS) {
|
|
@@ -321,6 +404,7 @@ export async function runSelfTest(write = stdoutLine) {
|
|
|
321
404
|
}
|
|
322
405
|
write('');
|
|
323
406
|
write(`version ${PKG_VERSION}`);
|
|
407
|
+
write(`home ${userHome}`);
|
|
324
408
|
write(`storage ${userStoragePath}`);
|
|
325
409
|
write(failedSteps.length === 0
|
|
326
410
|
? SELF_TEST_PASS_VERDICT
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { IStorageAdapter } from '../types/query.js';
|
|
2
|
+
export declare const DEMO_INGEST_REFUSED_MESSAGE: string;
|
|
3
|
+
export declare class DemoIngestRefusedError extends Error {
|
|
4
|
+
/** Read by the dashboard's error handler: a client fault, not a server one. */
|
|
5
|
+
readonly status = 403;
|
|
6
|
+
constructor();
|
|
7
|
+
}
|
|
8
|
+
export declare function withDemoIngestGuard(storage: IStorageAdapter): IStorageAdapter;
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Demo mode must not quietly become someone's production store.
|
|
3
|
+
*
|
|
4
|
+
* `--demo` serves the dashboard — and with it POST /api/v1/traces — against
|
|
5
|
+
* demo.db, a seeded, DISPOSABLE database that `--demo-clear` deletes
|
|
6
|
+
* outright. A reader following the README top to bottom could start the
|
|
7
|
+
* demo, point their capture client at the port it printed, watch real
|
|
8
|
+
* traces land beside the fake ones, and later lose all of them to the
|
|
9
|
+
* cleanup command the banner recommends. Nothing warned at any step.
|
|
10
|
+
*
|
|
11
|
+
* The guard wraps the demo store so every WRITE of trace or eval data is
|
|
12
|
+
* refused with a message that says what demo mode is and where real
|
|
13
|
+
* traces go. Reads and the demo's own seeded content are untouched — the
|
|
14
|
+
* dashboard keeps working exactly as before. Rule deploys are not storage
|
|
15
|
+
* writes (they go to the demo-scoped rule store) and stay allowed; they
|
|
16
|
+
* are part of what the demo exists to show.
|
|
17
|
+
*
|
|
18
|
+
* Implemented as a Proxy over the adapter rather than a subclass or a
|
|
19
|
+
* hand-written delegate: any method added to IStorageAdapter later
|
|
20
|
+
* delegates automatically instead of silently bypassing the guard.
|
|
21
|
+
*/
|
|
22
|
+
export const DEMO_INGEST_REFUSED_MESSAGE = 'Demo mode does not accept trace ingest. `--demo` serves a seeded, disposable database (demo.db) — ' +
|
|
23
|
+
'`--demo-clear` deletes everything in it, so real traces stored here would be lost. ' +
|
|
24
|
+
'Start the real server to store traces: `iris-mcp --dashboard` for HTTP ingest on POST /api/v1/traces, ' +
|
|
25
|
+
'or the MCP transport for log_trace.';
|
|
26
|
+
const REFUSED_METHODS = new Set([
|
|
27
|
+
'insertTrace',
|
|
28
|
+
'insertSpan',
|
|
29
|
+
'insertEvalResult',
|
|
30
|
+
]);
|
|
31
|
+
export class DemoIngestRefusedError extends Error {
|
|
32
|
+
/** Read by the dashboard's error handler: a client fault, not a server one. */
|
|
33
|
+
status = 403;
|
|
34
|
+
constructor() {
|
|
35
|
+
super(DEMO_INGEST_REFUSED_MESSAGE);
|
|
36
|
+
this.name = 'DemoIngestRefusedError';
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
export function withDemoIngestGuard(storage) {
|
|
40
|
+
return new Proxy(storage, {
|
|
41
|
+
get(target, prop, receiver) {
|
|
42
|
+
if (typeof prop === 'string' && REFUSED_METHODS.has(prop)) {
|
|
43
|
+
return async () => {
|
|
44
|
+
throw new DemoIngestRefusedError();
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
const value = Reflect.get(target, prop, receiver);
|
|
48
|
+
// Class methods live on the prototype and read private fields off
|
|
49
|
+
// `this`; bind them to the real adapter so `this.db` resolves.
|
|
50
|
+
return typeof value === 'function' ? value.bind(target) : value;
|
|
51
|
+
},
|
|
52
|
+
});
|
|
53
|
+
}
|
|
@@ -34,6 +34,12 @@ export declare class SqliteAdapter implements IStorageAdapter {
|
|
|
34
34
|
getEvalStatsRules(tenantId: TenantId, period: EvalStatsPeriod): Promise<EvalStatsRuleBreakdown[]>;
|
|
35
35
|
getEvalStatsFailures(tenantId: TenantId, period: EvalStatsPeriod, limit: number): Promise<EvalStatsFailure[]>;
|
|
36
36
|
deleteTracesOlderThan(tenantId: TenantId, days: number): Promise<number>;
|
|
37
|
+
deleteEvalResultsOlderThan(tenantId: TenantId, days: number): Promise<number>;
|
|
38
|
+
purge(tenantId: TenantId): Promise<{
|
|
39
|
+
traces: number;
|
|
40
|
+
evalResults: number;
|
|
41
|
+
}>;
|
|
42
|
+
checkpoint(): Promise<void>;
|
|
37
43
|
deleteTrace(tenantId: TenantId, traceId: string): Promise<boolean>;
|
|
38
44
|
getDistinctValues(tenantId: TenantId, column: string): Promise<string[]>;
|
|
39
45
|
private rowToTrace;
|