@iris-eval/mcp-server 0.4.4 → 0.4.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +55 -1
- package/dist/audit-log-reader.js +3 -3
- package/dist/config/defaults.d.ts +1 -0
- package/dist/config/defaults.js +6 -3
- package/dist/config/index.d.ts +1 -0
- package/dist/config/index.js +14 -5
- package/dist/custom-rule-store.js +190 -35
- package/dist/dashboard/assets/index-ChcHJDDJ.js +10 -0
- package/dist/dashboard/index.html +1 -1
- package/dist/dashboard/routes/rules.js +1 -1
- package/dist/dashboard/server.js +60 -5
- package/dist/dashboard/validation.d.ts +36 -62
- package/dist/eval/citation-verify/resolve.js +101 -15
- package/dist/eval/engine.js +25 -10
- package/dist/eval/rules/config-keys.d.ts +14 -0
- package/dist/eval/rules/config-keys.js +43 -0
- package/dist/eval/rules/custom.js +43 -14
- package/dist/eval/rules/regex-budget.d.ts +5 -0
- package/dist/eval/rules/regex-budget.js +0 -0
- package/dist/eval/rules/relevance.d.ts +1 -0
- package/dist/eval/rules/relevance.js +3 -1
- package/dist/eval/rules/safety.d.ts +5 -0
- package/dist/eval/rules/safety.js +43 -5
- package/dist/index.js +13 -2
- package/dist/middleware/error-handler.js +19 -1
- package/dist/middleware/rebinding-guard.d.ts +21 -0
- package/dist/middleware/rebinding-guard.js +77 -0
- package/dist/otel/mapper.js +2 -1
- package/dist/preferences.d.ts +43 -93
- package/dist/preferences.js +5 -10
- package/dist/storage/migrations/005-normalize-created-at.d.ts +3 -0
- package/dist/storage/migrations/005-normalize-created-at.js +34 -0
- package/dist/storage/migrations/index.js +8 -1
- package/dist/storage/sqlite-adapter.js +28 -4
- package/dist/tools/deploy-rule.js +2 -2
- package/dist/tools/evaluate-output.js +1 -1
- package/dist/tools/log-trace.js +3 -3
- package/dist/transport/http.js +68 -4
- package/dist/types/config.d.ts +7 -0
- package/dist/utils/iris-home.d.ts +1 -0
- package/dist/utils/iris-home.js +21 -0
- package/dist/utils/write-atomic.d.ts +1 -0
- package/dist/utils/write-atomic.js +64 -0
- package/package.json +10 -5
- package/server.json +2 -2
- package/dist/dashboard/assets/index-DNflCqmJ.js +0 -12
|
@@ -5,21 +5,59 @@
|
|
|
5
5
|
* patterns evaluate. Word-boundary anchors avoid matching inside larger
|
|
6
6
|
* strings where appropriate.
|
|
7
7
|
*/
|
|
8
|
-
|
|
8
|
+
/*
|
|
9
|
+
* Every pattern here runs against ATTACKER-CONTROLLED text — agent output is
|
|
10
|
+
* untrusted by definition (resolve.ts states this outright), and any agent
|
|
11
|
+
* that summarises a web page, reads email, or handles user tickets can be
|
|
12
|
+
* fed a crafted string straight into evaluate_output.
|
|
13
|
+
*
|
|
14
|
+
* So: no ambiguous quantifiers. The rule that bit us was `\s*[:.]?\s*` in
|
|
15
|
+
* DOB and Medical Record Number — two adjacent unbounded whitespace
|
|
16
|
+
* quantifiers give the engine N+1 ways to split a run of N spaces, each of
|
|
17
|
+
* which fails at the trailing character class. Cost was quadratic in the
|
|
18
|
+
* input: 'MRN' + N spaces + '!' measured 31ms at 4k, 118ms at 8k, 468ms at
|
|
19
|
+
* 16k, and did not finish at the 1MB body limit. Node is single-threaded,
|
|
20
|
+
* so one call wedged the whole server.
|
|
21
|
+
*
|
|
22
|
+
* Bounded quantifiers ({0,8}) keep the alternatives constant regardless of
|
|
23
|
+
* input length. When adding a pattern, check for: adjacent quantifiers over
|
|
24
|
+
* overlapping character classes, nested quantifiers, and a character that
|
|
25
|
+
* can match both inside a + and as the following literal.
|
|
26
|
+
*
|
|
27
|
+
* Exported so the claims drift test can assert .claims.json counts against
|
|
28
|
+
* the runtime truth (tests/claims-eval-rules-counts.test.ts).
|
|
29
|
+
*/
|
|
30
|
+
export const PII_PATTERNS = [
|
|
9
31
|
// Original v0.3.0 patterns
|
|
10
32
|
{ name: 'SSN', pattern: /\b\d{3}-\d{2}-\d{4}\b/ },
|
|
11
33
|
{ name: 'Credit Card', pattern: /\b(?:\d{4}[-\s]?){3}\d{4}\b/ },
|
|
12
34
|
{ name: 'Phone', pattern: /\b(?:\+1[-.\s]?)?\(?\d{3}\)?[-.\s]?\d{3}[-.\s]?\d{4}\b/ },
|
|
13
|
-
|
|
35
|
+
/*
|
|
36
|
+
* Every quantifier is bounded, at the RFC 5321 limits (local part 64,
|
|
37
|
+
* DNS label 63, TLD 24). Unbounded ones made this quadratic on text with
|
|
38
|
+
* no '@' in it: from EVERY starting position the local part consumed the
|
|
39
|
+
* rest of the string before failing, so N start positions each did O(N)
|
|
40
|
+
* work. 'a@' + 'a.'×32000 measured 3.5 seconds. Bounding the local part
|
|
41
|
+
* caps per-position work at a constant, which is what makes the whole
|
|
42
|
+
* scan linear.
|
|
43
|
+
*
|
|
44
|
+
* The domain is also written as explicit dot-separated labels rather than
|
|
45
|
+
* [A-Za-z0-9.-]+\. — that form lets '.' match both inside the + and as
|
|
46
|
+
* the following literal, which is its own source of splits to try.
|
|
47
|
+
*/
|
|
48
|
+
{
|
|
49
|
+
name: 'Email',
|
|
50
|
+
pattern: /\b[A-Za-z0-9._%+-]{1,64}@(?:[A-Za-z0-9-]{1,63}\.){1,8}[A-Z]{2,24}\b/i,
|
|
51
|
+
},
|
|
14
52
|
// v0.3.1 additions
|
|
15
53
|
// IBAN: 2 letters + 2 digits + 1-30 alphanumeric (international bank account number)
|
|
16
54
|
{ name: 'IBAN', pattern: /\b[A-Z]{2}\d{2}[A-Z0-9]{10,30}\b/ },
|
|
17
55
|
// US passport: 9 digits, optionally prefixed with letter (modern format C12345678)
|
|
18
56
|
{ name: 'Passport', pattern: /\b[A-Z]?\d{9}\b/ },
|
|
19
57
|
// Date of birth contextual — DOB or "Born:" / "Birthday:" + date
|
|
20
|
-
{ name: 'DOB', pattern: /\b(?:DOB|D\.O\.B\.|Date of Birth|Born|Birthday)\s
|
|
58
|
+
{ name: 'DOB', pattern: /\b(?:DOB|D\.O\.B\.|Date of Birth|Born|Birthday)\s{0,8}[:.]?\s{0,8}\d{1,2}[\/\-.]\d{1,2}[\/\-.](?:\d{2}|\d{4})\b/i },
|
|
21
59
|
// Medical record number — MRN: + alphanumeric (common format)
|
|
22
|
-
{ name: 'Medical Record Number', pattern: /\b(?:MRN|Medical Record (?:Number|No\.?|#))\s
|
|
60
|
+
{ name: 'Medical Record Number', pattern: /\b(?:MRN|Medical Record (?:Number|No\.?|#))\s{0,8}[:.]?\s{0,8}[A-Z0-9]{6,12}\b/i },
|
|
23
61
|
// IPv4 address
|
|
24
62
|
{ name: 'IP Address', pattern: /\b(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(?:\.(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}\b/ },
|
|
25
63
|
// API key heuristic — looks for sk-/pk-/api_/Bearer + long alphanumeric
|
|
@@ -79,7 +117,7 @@ export const noBlocklistWords = {
|
|
|
79
117
|
* leaks, or role-override acknowledgments). Input-side detection is the
|
|
80
118
|
* agent host's job; output-side is Iris's.
|
|
81
119
|
*/
|
|
82
|
-
const INJECTION_PATTERNS = [
|
|
120
|
+
export const INJECTION_PATTERNS = [
|
|
83
121
|
// Original v0.3.0 patterns
|
|
84
122
|
/ignore (?:all )?(?:previous|above|prior) (?:instructions|prompts)/i,
|
|
85
123
|
/you are now (?:a |in )/i,
|
package/dist/index.js
CHANGED
|
@@ -28,6 +28,7 @@ const CliSchema = z
|
|
|
28
28
|
'api-key': z.string().min(1).optional(),
|
|
29
29
|
dashboard: z.boolean().optional(),
|
|
30
30
|
'dashboard-port': PortSchema.optional(),
|
|
31
|
+
'dashboard-host': z.string().min(1).optional(),
|
|
31
32
|
help: z.boolean().optional(),
|
|
32
33
|
})
|
|
33
34
|
.strict();
|
|
@@ -42,6 +43,7 @@ try {
|
|
|
42
43
|
'api-key': { type: 'string' },
|
|
43
44
|
dashboard: { type: 'boolean', default: false },
|
|
44
45
|
'dashboard-port': { type: 'string' },
|
|
46
|
+
'dashboard-host': { type: 'string' },
|
|
45
47
|
help: { type: 'boolean', short: 'h', default: false },
|
|
46
48
|
},
|
|
47
49
|
strict: true,
|
|
@@ -74,18 +76,26 @@ Options:
|
|
|
74
76
|
--api-key <key> API key for HTTP authentication
|
|
75
77
|
--dashboard Enable web dashboard
|
|
76
78
|
--dashboard-port <port> Dashboard port 1-65535 (default: 6920)
|
|
79
|
+
--dashboard-host <host> Dashboard bind address (default: 127.0.0.1). The dashboard is
|
|
80
|
+
unauthenticated unless --api-key is set — binding it beyond
|
|
81
|
+
loopback exposes your full trace history to the network.
|
|
77
82
|
-h, --help Show this help message
|
|
78
83
|
|
|
79
84
|
Environment variables (CLI flags take precedence):
|
|
80
85
|
IRIS_TRANSPORT stdio | http
|
|
81
86
|
IRIS_HOST Bind address for HTTP transport (default: 127.0.0.1)
|
|
82
87
|
IRIS_PORT HTTP transport port (1-65535)
|
|
83
|
-
|
|
88
|
+
IRIS_HOME Directory for all per-user files: config.json, iris.db, custom-rules.json,
|
|
89
|
+
audit.log, preferences.json (default: ~/.iris)
|
|
90
|
+
IRIS_DB_PATH SQLite database path (overrides IRIS_HOME for the DB only)
|
|
84
91
|
IRIS_LOG_LEVEL debug | info | warn | error
|
|
85
92
|
IRIS_DASHBOARD true to enable web dashboard
|
|
86
93
|
IRIS_DASHBOARD_PORT Dashboard port (1-65535, default: 6920)
|
|
94
|
+
IRIS_DASHBOARD_HOST Dashboard bind address (default: 127.0.0.1)
|
|
87
95
|
IRIS_API_KEY API key for HTTP authentication
|
|
88
|
-
IRIS_ALLOWED_ORIGINS Comma-separated CORS
|
|
96
|
+
IRIS_ALLOWED_ORIGINS Comma-separated origin allowlist. Dashboard: CORS headers (supports globs, e.g. http://localhost:*).
|
|
97
|
+
HTTP transport: exact-match Origin allowlist for DNS-rebinding protection (globs ignored;
|
|
98
|
+
this server's own loopback origins are always allowed).
|
|
89
99
|
IRIS_NO_AUTO_LAUNCH Set to 1 to disable first-run dashboard auto-launch
|
|
90
100
|
IRIS_ANTHROPIC_API_KEY Required by evaluate_with_llm_judge + verify_citations (provider=anthropic)
|
|
91
101
|
IRIS_OPENAI_API_KEY Required by evaluate_with_llm_judge + verify_citations (provider=openai)
|
|
@@ -111,6 +121,7 @@ const config = loadConfig({
|
|
|
111
121
|
apiKey: values['api-key'],
|
|
112
122
|
dashboard: values.dashboard,
|
|
113
123
|
dashboardPort: values['dashboard-port'],
|
|
124
|
+
dashboardHost: values['dashboard-host'],
|
|
114
125
|
});
|
|
115
126
|
const logger = createLogger(config);
|
|
116
127
|
async function main() {
|
|
@@ -9,7 +9,25 @@ export function createErrorHandler(logger) {
|
|
|
9
9
|
return;
|
|
10
10
|
}
|
|
11
11
|
const status = err.status ?? err.statusCode ?? 500;
|
|
12
|
-
|
|
12
|
+
/*
|
|
13
|
+
* Never echo a Node system error to the client. Their messages embed
|
|
14
|
+
* absolute paths — an ENOENT from res.sendFile returned
|
|
15
|
+
* "ENOENT: no such file or directory, stat 'C:\...\dist\dashboard\index.html'"
|
|
16
|
+
* with a 404, disclosing the install path and OS user to anyone who
|
|
17
|
+
* could reach the dashboard (CWE-209). 5xx was already masked; the leak
|
|
18
|
+
* was in the 4xx branch, where echoing err.message is otherwise useful
|
|
19
|
+
* (body-parser's "request entity too large", Zod messages, etc.).
|
|
20
|
+
*
|
|
21
|
+
* Identify system errors by the shape Node gives them — a string `code`
|
|
22
|
+
* plus `syscall` — rather than by matching path-ish text, which would
|
|
23
|
+
* miss cases and mangle legitimate messages.
|
|
24
|
+
*/
|
|
25
|
+
const isSystemError = typeof err?.code === 'string' && typeof err?.syscall === 'string';
|
|
26
|
+
const message = status >= 500 || isSystemError
|
|
27
|
+
? status >= 500
|
|
28
|
+
? 'Internal server error'
|
|
29
|
+
: 'Not found'
|
|
30
|
+
: (err.message ?? 'Unknown error');
|
|
13
31
|
logger.error(`Request error: ${err.message}`, { status, stack: err.stack });
|
|
14
32
|
res.status(status).json({
|
|
15
33
|
error: message,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type { RequestHandler } from 'express';
|
|
2
|
+
export declare function isLoopbackHost(host: string): boolean;
|
|
3
|
+
/** Concrete origins/hosts this server answers to on `port`. */
|
|
4
|
+
export declare function loopbackOriginsFor(port: number): string[];
|
|
5
|
+
export declare function loopbackHostsFor(port: number): string[];
|
|
6
|
+
export interface RebindingGuardOptions {
|
|
7
|
+
/**
|
|
8
|
+
* Port the server is actually bound to. Accepts a resolver because the
|
|
9
|
+
* middleware is registered BEFORE listen() — and the configured port is
|
|
10
|
+
* 0 whenever the caller wants an ephemeral one (tests and embedders do
|
|
11
|
+
* this). Baking 0 into the allowlist would produce `http://localhost:0`
|
|
12
|
+
* and reject every real request with a 403 that looks exactly like an
|
|
13
|
+
* attack. Same trap the MCP transport documents at transport/http.ts.
|
|
14
|
+
*/
|
|
15
|
+
port: number | (() => number);
|
|
16
|
+
/** Bind address, used to decide whether Host validation applies. */
|
|
17
|
+
host: string;
|
|
18
|
+
/** Operator's configured origins; glob entries are ignored (see above). */
|
|
19
|
+
allowedOrigins?: string[];
|
|
20
|
+
}
|
|
21
|
+
export declare function createRebindingGuard(options: RebindingGuardOptions): RequestHandler;
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* DNS-rebinding protection for the dashboard HTTP API.
|
|
3
|
+
*
|
|
4
|
+
* v0.4.5 closed this hole on the MCP transport (/mcp) by handing
|
|
5
|
+
* allowedOrigins + allowedHosts to the SDK. The dashboard — same data,
|
|
6
|
+
* plus every mutating endpoint — never got the equivalent, and it starts
|
|
7
|
+
* implicitly alongside `--transport http`. So a browser on any page could
|
|
8
|
+
* POST a rule deployment to http://localhost:6920 and the server would
|
|
9
|
+
* execute it.
|
|
10
|
+
*
|
|
11
|
+
* CORS does not substitute, for the reason already written down in
|
|
12
|
+
* transport/http.ts: the browser withholds the RESPONSE, but the write has
|
|
13
|
+
* already happened. The request has to be REJECTED.
|
|
14
|
+
*
|
|
15
|
+
* Two checks, mirroring the SDK's semantics:
|
|
16
|
+
*
|
|
17
|
+
* Origin — enforced whenever the header is present. Absent means a
|
|
18
|
+
* non-browser client (curl, an MCP client, a health probe), which is not
|
|
19
|
+
* the threat model here; browsers always send it on cross-origin
|
|
20
|
+
* requests. Exact match only — glob patterns from the CORS allowlist are
|
|
21
|
+
* meaningless against a single concrete Origin and are dropped rather
|
|
22
|
+
* than left in the list looking effective.
|
|
23
|
+
*
|
|
24
|
+
* Host — enforced only when bound to loopback. A non-loopback bind is a
|
|
25
|
+
* deliberate network deployment, usually behind a proxy that rewrites
|
|
26
|
+
* Host, and an exact-match list would break it.
|
|
27
|
+
*/
|
|
28
|
+
export function isLoopbackHost(host) {
|
|
29
|
+
return host === '127.0.0.1' || host === 'localhost' || host === '::1' || host === '[::1]';
|
|
30
|
+
}
|
|
31
|
+
/** Concrete origins/hosts this server answers to on `port`. */
|
|
32
|
+
export function loopbackOriginsFor(port) {
|
|
33
|
+
return [`http://127.0.0.1:${port}`, `http://localhost:${port}`, `http://[::1]:${port}`];
|
|
34
|
+
}
|
|
35
|
+
export function loopbackHostsFor(port) {
|
|
36
|
+
return [`127.0.0.1:${port}`, `localhost:${port}`, `[::1]:${port}`];
|
|
37
|
+
}
|
|
38
|
+
export function createRebindingGuard(options) {
|
|
39
|
+
const { port, host, allowedOrigins = [] } = options;
|
|
40
|
+
const exactConfigured = allowedOrigins.filter((o) => !o.includes('*'));
|
|
41
|
+
const enforceHost = isLoopbackHost(host);
|
|
42
|
+
let cache;
|
|
43
|
+
function listsFor(resolvedPort) {
|
|
44
|
+
if (cache?.port !== resolvedPort) {
|
|
45
|
+
cache = {
|
|
46
|
+
port: resolvedPort,
|
|
47
|
+
origins: new Set([...loopbackOriginsFor(resolvedPort), ...exactConfigured]),
|
|
48
|
+
/*
|
|
49
|
+
* `[::1]:port` is the form Node actually puts in the Host header
|
|
50
|
+
* for an IPv6 loopback request — brackets included. A guard
|
|
51
|
+
* written against the bare '::1' would be inert, which is exactly
|
|
52
|
+
* how the citation-fetch SSRF guard was silently dead before
|
|
53
|
+
* v0.4.5 (URL.hostname returns '[::1]', never '::1').
|
|
54
|
+
*/
|
|
55
|
+
hosts: new Set(loopbackHostsFor(resolvedPort)),
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
return cache;
|
|
59
|
+
}
|
|
60
|
+
return (req, res, next) => {
|
|
61
|
+
const resolvedPort = typeof port === 'function' ? port() : port;
|
|
62
|
+
const { origins, hosts } = listsFor(resolvedPort);
|
|
63
|
+
const origin = req.headers.origin;
|
|
64
|
+
if (origin && !origins.has(origin)) {
|
|
65
|
+
res.status(403).json({ error: 'Forbidden: invalid Origin header' });
|
|
66
|
+
return;
|
|
67
|
+
}
|
|
68
|
+
if (enforceHost) {
|
|
69
|
+
const hostHeader = req.headers.host;
|
|
70
|
+
if (hostHeader && !hosts.has(hostHeader)) {
|
|
71
|
+
res.status(403).json({ error: 'Forbidden: invalid Host header' });
|
|
72
|
+
return;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
next();
|
|
76
|
+
};
|
|
77
|
+
}
|
package/dist/otel/mapper.js
CHANGED
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
//
|
|
10
10
|
// We intentionally do not depend on @opentelemetry/* — the exporter
|
|
11
11
|
// serializes these plain objects, and the payload shape is the OTLP spec.
|
|
12
|
+
import { PKG_VERSION } from '../config/defaults.js';
|
|
12
13
|
// OTel SpanKind enum values (from opentelemetry-proto/trace/v1/trace.proto).
|
|
13
14
|
const KIND_MAP = {
|
|
14
15
|
INTERNAL: 1,
|
|
@@ -145,7 +146,7 @@ export function buildExportPayload(traces, serviceName) {
|
|
|
145
146
|
{ key: 'service.name', value: { stringValue: serviceName } },
|
|
146
147
|
{ key: 'telemetry.sdk.name', value: { stringValue: 'iris-mcp' } },
|
|
147
148
|
{ key: 'telemetry.sdk.language', value: { stringValue: 'nodejs' } },
|
|
148
|
-
{ key: 'telemetry.sdk.version', value: { stringValue:
|
|
149
|
+
{ key: 'telemetry.sdk.version', value: { stringValue: PKG_VERSION } },
|
|
149
150
|
],
|
|
150
151
|
};
|
|
151
152
|
const spans = [];
|
package/dist/preferences.d.ts
CHANGED
|
@@ -1,108 +1,58 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
2
|
declare const MomentFiltersSchema: z.ZodObject<{
|
|
3
3
|
agentName: z.ZodOptional<z.ZodString>;
|
|
4
|
-
verdict: z.ZodOptional<z.ZodEnum<
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
significanceKind?: "safety-violation" | "cost-spike" | "first-failure" | "novel-pattern" | "rule-collision" | "normal-pass" | "normal-fail" | undefined;
|
|
10
|
-
}, {
|
|
11
|
-
verdict?: "pass" | "fail" | "partial" | "unevaluated" | undefined;
|
|
12
|
-
agentName?: string | undefined;
|
|
13
|
-
significanceKind?: "safety-violation" | "cost-spike" | "first-failure" | "novel-pattern" | "rule-collision" | "normal-pass" | "normal-fail" | undefined;
|
|
14
|
-
}>;
|
|
15
|
-
export declare const PreferencesSchema: z.ZodObject<{
|
|
16
|
-
autoLaunch: z.ZodDefault<z.ZodBoolean>;
|
|
17
|
-
firstSeen: z.ZodOptional<z.ZodString>;
|
|
18
|
-
dismissedBanners: z.ZodDefault<z.ZodArray<z.ZodString, "many">>;
|
|
19
|
-
/** Theme override; "system" defers to prefers-color-scheme. v0.4. */
|
|
20
|
-
theme: z.ZodDefault<z.ZodEnum<["dark", "light", "system"]>>;
|
|
21
|
-
/** Last filter set used on /moments — applied on next visit when the URL has no filter params. v0.4. */
|
|
22
|
-
momentFilters: z.ZodDefault<z.ZodObject<{
|
|
23
|
-
agentName: z.ZodOptional<z.ZodString>;
|
|
24
|
-
verdict: z.ZodOptional<z.ZodEnum<["pass", "fail", "partial", "unevaluated"]>>;
|
|
25
|
-
significanceKind: z.ZodOptional<z.ZodEnum<["safety-violation", "cost-spike", "first-failure", "novel-pattern", "rule-collision", "normal-pass", "normal-fail"]>>;
|
|
26
|
-
}, "strict", z.ZodTypeAny, {
|
|
27
|
-
verdict?: "pass" | "fail" | "partial" | "unevaluated" | undefined;
|
|
28
|
-
agentName?: string | undefined;
|
|
29
|
-
significanceKind?: "safety-violation" | "cost-spike" | "first-failure" | "novel-pattern" | "rule-collision" | "normal-pass" | "normal-fail" | undefined;
|
|
30
|
-
}, {
|
|
31
|
-
verdict?: "pass" | "fail" | "partial" | "unevaluated" | undefined;
|
|
32
|
-
agentName?: string | undefined;
|
|
33
|
-
significanceKind?: "safety-violation" | "cost-spike" | "first-failure" | "novel-pattern" | "rule-collision" | "normal-pass" | "normal-fail" | undefined;
|
|
4
|
+
verdict: z.ZodOptional<z.ZodEnum<{
|
|
5
|
+
pass: "pass";
|
|
6
|
+
fail: "fail";
|
|
7
|
+
partial: "partial";
|
|
8
|
+
unevaluated: "unevaluated";
|
|
34
9
|
}>>;
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
/** ISO timestamp of last notifications-popover opened — drives unread badge. */
|
|
44
|
-
notificationsLastSeen: z.ZodOptional<z.ZodString>;
|
|
45
|
-
}, "passthrough", z.ZodTypeAny, z.objectOutputType<{
|
|
46
|
-
autoLaunch: z.ZodDefault<z.ZodBoolean>;
|
|
47
|
-
firstSeen: z.ZodOptional<z.ZodString>;
|
|
48
|
-
dismissedBanners: z.ZodDefault<z.ZodArray<z.ZodString, "many">>;
|
|
49
|
-
/** Theme override; "system" defers to prefers-color-scheme. v0.4. */
|
|
50
|
-
theme: z.ZodDefault<z.ZodEnum<["dark", "light", "system"]>>;
|
|
51
|
-
/** Last filter set used on /moments — applied on next visit when the URL has no filter params. v0.4. */
|
|
52
|
-
momentFilters: z.ZodDefault<z.ZodObject<{
|
|
53
|
-
agentName: z.ZodOptional<z.ZodString>;
|
|
54
|
-
verdict: z.ZodOptional<z.ZodEnum<["pass", "fail", "partial", "unevaluated"]>>;
|
|
55
|
-
significanceKind: z.ZodOptional<z.ZodEnum<["safety-violation", "cost-spike", "first-failure", "novel-pattern", "rule-collision", "normal-pass", "normal-fail"]>>;
|
|
56
|
-
}, "strict", z.ZodTypeAny, {
|
|
57
|
-
verdict?: "pass" | "fail" | "partial" | "unevaluated" | undefined;
|
|
58
|
-
agentName?: string | undefined;
|
|
59
|
-
significanceKind?: "safety-violation" | "cost-spike" | "first-failure" | "novel-pattern" | "rule-collision" | "normal-pass" | "normal-fail" | undefined;
|
|
60
|
-
}, {
|
|
61
|
-
verdict?: "pass" | "fail" | "partial" | "unevaluated" | undefined;
|
|
62
|
-
agentName?: string | undefined;
|
|
63
|
-
significanceKind?: "safety-violation" | "cost-spike" | "first-failure" | "novel-pattern" | "rule-collision" | "normal-pass" | "normal-fail" | undefined;
|
|
10
|
+
significanceKind: z.ZodOptional<z.ZodEnum<{
|
|
11
|
+
"safety-violation": "safety-violation";
|
|
12
|
+
"cost-spike": "cost-spike";
|
|
13
|
+
"first-failure": "first-failure";
|
|
14
|
+
"novel-pattern": "novel-pattern";
|
|
15
|
+
"rule-collision": "rule-collision";
|
|
16
|
+
"normal-pass": "normal-pass";
|
|
17
|
+
"normal-fail": "normal-fail";
|
|
64
18
|
}>>;
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
/** Decision Moments hidden from the timeline by user action. v0.4. */
|
|
68
|
-
archivedMoments: z.ZodDefault<z.ZodArray<z.ZodString, "many">>;
|
|
69
|
-
/** Density mode for chrome (Design System v2.A). 'compact' default per R2.3. */
|
|
70
|
-
density: z.ZodDefault<z.ZodEnum<["compact", "comfortable"]>>;
|
|
71
|
-
/** Sidebar collapsed (icon-only at 64px) vs expanded (256px). Default expanded per R2.4. */
|
|
72
|
-
sidebarCollapsed: z.ZodDefault<z.ZodBoolean>;
|
|
73
|
-
/** ISO timestamp of last notifications-popover opened — drives unread badge. */
|
|
74
|
-
notificationsLastSeen: z.ZodOptional<z.ZodString>;
|
|
75
|
-
}, z.ZodTypeAny, "passthrough">, z.objectInputType<{
|
|
19
|
+
}, z.core.$strict>;
|
|
20
|
+
export declare const PreferencesSchema: z.ZodObject<{
|
|
76
21
|
autoLaunch: z.ZodDefault<z.ZodBoolean>;
|
|
77
22
|
firstSeen: z.ZodOptional<z.ZodString>;
|
|
78
|
-
dismissedBanners: z.ZodDefault<z.ZodArray<z.ZodString
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
23
|
+
dismissedBanners: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
24
|
+
theme: z.ZodDefault<z.ZodEnum<{
|
|
25
|
+
light: "light";
|
|
26
|
+
dark: "dark";
|
|
27
|
+
system: "system";
|
|
28
|
+
}>>;
|
|
82
29
|
momentFilters: z.ZodDefault<z.ZodObject<{
|
|
83
30
|
agentName: z.ZodOptional<z.ZodString>;
|
|
84
|
-
verdict: z.ZodOptional<z.ZodEnum<
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
31
|
+
verdict: z.ZodOptional<z.ZodEnum<{
|
|
32
|
+
pass: "pass";
|
|
33
|
+
fail: "fail";
|
|
34
|
+
partial: "partial";
|
|
35
|
+
unevaluated: "unevaluated";
|
|
36
|
+
}>>;
|
|
37
|
+
significanceKind: z.ZodOptional<z.ZodEnum<{
|
|
38
|
+
"safety-violation": "safety-violation";
|
|
39
|
+
"cost-spike": "cost-spike";
|
|
40
|
+
"first-failure": "first-failure";
|
|
41
|
+
"novel-pattern": "novel-pattern";
|
|
42
|
+
"rule-collision": "rule-collision";
|
|
43
|
+
"normal-pass": "normal-pass";
|
|
44
|
+
"normal-fail": "normal-fail";
|
|
45
|
+
}>>;
|
|
46
|
+
}, z.core.$strict>>;
|
|
47
|
+
dismissedTours: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
48
|
+
archivedMoments: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
49
|
+
density: z.ZodDefault<z.ZodEnum<{
|
|
50
|
+
compact: "compact";
|
|
51
|
+
comfortable: "comfortable";
|
|
94
52
|
}>>;
|
|
95
|
-
/** Tour ids the user has completed or dismissed. v0.4. */
|
|
96
|
-
dismissedTours: z.ZodDefault<z.ZodArray<z.ZodString, "many">>;
|
|
97
|
-
/** Decision Moments hidden from the timeline by user action. v0.4. */
|
|
98
|
-
archivedMoments: z.ZodDefault<z.ZodArray<z.ZodString, "many">>;
|
|
99
|
-
/** Density mode for chrome (Design System v2.A). 'compact' default per R2.3. */
|
|
100
|
-
density: z.ZodDefault<z.ZodEnum<["compact", "comfortable"]>>;
|
|
101
|
-
/** Sidebar collapsed (icon-only at 64px) vs expanded (256px). Default expanded per R2.4. */
|
|
102
53
|
sidebarCollapsed: z.ZodDefault<z.ZodBoolean>;
|
|
103
|
-
/** ISO timestamp of last notifications-popover opened — drives unread badge. */
|
|
104
54
|
notificationsLastSeen: z.ZodOptional<z.ZodString>;
|
|
105
|
-
}, z.
|
|
55
|
+
}, z.core.$loose>;
|
|
106
56
|
export type Preferences = z.infer<typeof PreferencesSchema>;
|
|
107
57
|
export type MomentFilters = z.infer<typeof MomentFiltersSchema>;
|
|
108
58
|
export interface PreferenceState {
|
package/dist/preferences.js
CHANGED
|
@@ -18,9 +18,10 @@
|
|
|
18
18
|
* /moments), dismissedTours, archivedMoments. These let the dashboard
|
|
19
19
|
* remember the user's last view across reloads + across iris-mcp restarts.
|
|
20
20
|
*/
|
|
21
|
-
import {
|
|
22
|
-
import {
|
|
23
|
-
import {
|
|
21
|
+
import { readFileSync, existsSync } from 'node:fs';
|
|
22
|
+
import { writeAtomic } from './utils/write-atomic.js';
|
|
23
|
+
import { irisHome } from './utils/iris-home.js';
|
|
24
|
+
import { join } from 'node:path';
|
|
24
25
|
import { z } from 'zod';
|
|
25
26
|
const MomentFiltersSchema = z
|
|
26
27
|
.object({
|
|
@@ -61,7 +62,7 @@ export const PreferencesSchema = z
|
|
|
61
62
|
})
|
|
62
63
|
.passthrough();
|
|
63
64
|
function defaultPreferencesPath() {
|
|
64
|
-
return join(
|
|
65
|
+
return join(irisHome(), 'preferences.json');
|
|
65
66
|
}
|
|
66
67
|
function freshPreferences() {
|
|
67
68
|
return PreferencesSchema.parse({
|
|
@@ -74,12 +75,6 @@ function freshPreferences() {
|
|
|
74
75
|
archivedMoments: [],
|
|
75
76
|
});
|
|
76
77
|
}
|
|
77
|
-
function writeAtomic(targetPath, contents) {
|
|
78
|
-
mkdirSync(dirname(targetPath), { recursive: true });
|
|
79
|
-
const tmp = `${targetPath}.tmp.${process.pid}`;
|
|
80
|
-
writeFileSync(tmp, contents, 'utf-8');
|
|
81
|
-
renameSync(tmp, targetPath);
|
|
82
|
-
}
|
|
83
78
|
export function loadOrInitPreferences(customPath) {
|
|
84
79
|
const path = customPath ?? defaultPreferencesPath();
|
|
85
80
|
if (!existsSync(path)) {
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
export const id = '005-normalize-created-at';
|
|
2
|
+
/*
|
|
3
|
+
* Normalize created_at to ISO-8601 UTC.
|
|
4
|
+
*
|
|
5
|
+
* The column's DEFAULT is `datetime('now')`, which SQLite renders as
|
|
6
|
+
* "2026-08-09 15:00:00" — space separator, no milliseconds, no Z. Nothing
|
|
7
|
+
* ever wrote the column explicitly, so every row carried that shape. But
|
|
8
|
+
* every query compares it against a JS `toISOString()` value
|
|
9
|
+
* ("2026-08-09T15:00:00.000Z") using plain string comparison.
|
|
10
|
+
*
|
|
11
|
+
* ' ' is 0x20 and 'T' is 0x54, so the stored value sorts BEFORE any
|
|
12
|
+
* same-date boundary. Result: every eval whose calendar date equalled the
|
|
13
|
+
* window boundary's date was silently dropped from the window. A 20-hour-old
|
|
14
|
+
* eval vanished from "last 24h"; at 01:00 UTC the 24h view showed only what
|
|
15
|
+
* had happened since midnight. Traces were unaffected — log-trace writes a
|
|
16
|
+
* real ISO string — which is why this presented as "my evals are missing but
|
|
17
|
+
* my traces aren't".
|
|
18
|
+
*
|
|
19
|
+
* Fix in two halves: the adapter now writes ISO explicitly (so the DEFAULT
|
|
20
|
+
* never fires), and this migration rewrites the rows already on disk.
|
|
21
|
+
* strftime with %f gives milliseconds; SQLite stores UTC, so the literal Z
|
|
22
|
+
* is accurate. Rows already in ISO form are left alone — the LIKE guard
|
|
23
|
+
* matches only the space-separated shape, which keeps this idempotent and
|
|
24
|
+
* safe to run against a partially-migrated DB.
|
|
25
|
+
*/
|
|
26
|
+
export function up(db) {
|
|
27
|
+
for (const table of ['traces', 'eval_results']) {
|
|
28
|
+
db.exec(`
|
|
29
|
+
UPDATE ${table}
|
|
30
|
+
SET created_at = strftime('%Y-%m-%dT%H:%M:%fZ', created_at)
|
|
31
|
+
WHERE created_at LIKE '____-__-__ __:__:__%'
|
|
32
|
+
`);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
@@ -2,7 +2,14 @@ import * as migration001 from './001-initial-schema.js';
|
|
|
2
2
|
import * as migration002 from './002-eval-skip-fields.js';
|
|
3
3
|
import * as migration003 from './003-eval-passed-index.js';
|
|
4
4
|
import * as migration004 from './004-tenant-id.js';
|
|
5
|
-
|
|
5
|
+
import * as migration005 from './005-normalize-created-at.js';
|
|
6
|
+
const migrations = [
|
|
7
|
+
migration001,
|
|
8
|
+
migration002,
|
|
9
|
+
migration003,
|
|
10
|
+
migration004,
|
|
11
|
+
migration005,
|
|
12
|
+
];
|
|
6
13
|
export function runMigrations(db) {
|
|
7
14
|
db.exec(`
|
|
8
15
|
CREATE TABLE IF NOT EXISTS _iris_migrations (
|
|
@@ -145,10 +145,18 @@ export class SqliteAdapter {
|
|
|
145
145
|
}
|
|
146
146
|
async insertEvalResult(tenantId, result) {
|
|
147
147
|
assertTenant(tenantId);
|
|
148
|
+
/*
|
|
149
|
+
* created_at is written EXPLICITLY as ISO-8601. Leaving it to the
|
|
150
|
+
* column DEFAULT (datetime('now')) stored "2026-08-09 15:00:00", which
|
|
151
|
+
* every period query then compared as a string against a JS
|
|
152
|
+
* toISOString() boundary — and ' ' sorts before 'T', so any eval whose
|
|
153
|
+
* calendar date matched the boundary's date was dropped from the
|
|
154
|
+
* window. Migration 005 rewrites rows written before this line existed.
|
|
155
|
+
*/
|
|
148
156
|
this.db.prepare(`
|
|
149
|
-
INSERT INTO eval_results (tenant_id, id, trace_id, eval_type, output_text, expected_text, score, passed, rule_results, suggestions, rules_evaluated, rules_skipped, insufficient_data)
|
|
150
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
151
|
-
`).run(tenantId, result.id, result.trace_id ?? null, result.eval_type, result.output_text, result.expected_text ?? null, result.score, result.passed ? 1 : 0, JSON.stringify(result.rule_results), JSON.stringify(result.suggestions), result.rules_evaluated ?? null, result.rules_skipped ?? null, result.insufficient_data ? 1 : 0);
|
|
157
|
+
INSERT INTO eval_results (tenant_id, id, trace_id, eval_type, output_text, expected_text, score, passed, rule_results, suggestions, rules_evaluated, rules_skipped, insufficient_data, created_at)
|
|
158
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
159
|
+
`).run(tenantId, result.id, result.trace_id ?? null, result.eval_type, result.output_text, result.expected_text ?? null, result.score, result.passed ? 1 : 0, JSON.stringify(result.rule_results), JSON.stringify(result.suggestions), result.rules_evaluated ?? null, result.rules_skipped ?? null, result.insufficient_data ? 1 : 0, new Date().toISOString());
|
|
152
160
|
}
|
|
153
161
|
async getEvalsByTraceId(tenantId, traceId) {
|
|
154
162
|
assertTenant(tenantId);
|
|
@@ -263,12 +271,28 @@ export class SqliteAdapter {
|
|
|
263
271
|
FROM traces
|
|
264
272
|
WHERE tenant_id = ? AND timestamp >= ?
|
|
265
273
|
`).get(tenantId, since);
|
|
274
|
+
/*
|
|
275
|
+
* No `AND passed = 0` here — deliberately.
|
|
276
|
+
*
|
|
277
|
+
* A safety eval's score is the average across its rules, so a single
|
|
278
|
+
* violation is routinely outvoted: output containing "Your SSN is
|
|
279
|
+
* 123-45-6789" fails no_pii (score 0) while the three other safety
|
|
280
|
+
* rules pass, giving 0.733 overall — above the 0.7 threshold, so
|
|
281
|
+
* passed = 1. Filtering to failed evals therefore reported
|
|
282
|
+
* {pii: 0, injection: 0, hallucination: 0} for a trace that leaked a
|
|
283
|
+
* social security number.
|
|
284
|
+
*
|
|
285
|
+
* For a product whose job is catching PII, injection and hallucination,
|
|
286
|
+
* that error ran in the direction that HIDES problems. The count is
|
|
287
|
+
* per-VIOLATION, not per-failed-eval; the per-rule loop below already
|
|
288
|
+
* skips rules that passed, so scanning every safety eval in the window
|
|
289
|
+
* is both correct and sufficient.
|
|
290
|
+
*/
|
|
266
291
|
const safetyRows = this.db.prepare(`
|
|
267
292
|
SELECT rule_results
|
|
268
293
|
FROM eval_results
|
|
269
294
|
WHERE tenant_id = ? AND created_at >= ?
|
|
270
295
|
AND eval_type = 'safety'
|
|
271
|
-
AND passed = 0
|
|
272
296
|
`).all(tenantId, since);
|
|
273
297
|
const violations = { pii: 0, injection: 0, hallucination: 0 };
|
|
274
298
|
for (const row of safetyRows) {
|
|
@@ -24,7 +24,7 @@ const CustomRuleDefinitionSchema = z.object({
|
|
|
24
24
|
'json_schema',
|
|
25
25
|
'cost_threshold',
|
|
26
26
|
]),
|
|
27
|
-
config: z.record(z.unknown()),
|
|
27
|
+
config: z.record(z.string(), z.unknown()),
|
|
28
28
|
weight: z.number().optional(),
|
|
29
29
|
});
|
|
30
30
|
const inputSchema = {
|
|
@@ -63,7 +63,7 @@ export function registerDeployRuleTool(server, customRuleStore) {
|
|
|
63
63
|
'',
|
|
64
64
|
"Don't use to VALIDATE a rule before committing — deploy writes immediately. Use the dashboard's preview endpoint (POST /api/v1/rules/custom/preview) for dry-run validation against sample output. Don't use to EDIT an existing rule — this call only creates; edits require a dedicated flow (coming in v0.5). To update a rule today: delete_rule then deploy_rule with the new definition.",
|
|
65
65
|
'',
|
|
66
|
-
'Parameters. name is 1-120 chars (Zod-enforced min/max); appears in eval_result rule_results so make it human-readable. description is optional, max 500 chars (used in dashboard tooltips). evalType determines WHEN the rule fires (must match the eval_type your evaluate_output calls use; e.g., a "completeness" rule fires on every evaluate_output where eval_type="completeness" OR eval_type="custom"). severity affects dashboard sort + audit log signal but does NOT affect scoring (scoring uses the rule\'s weight). definition.type and definition.config must match (e.g., regex_match needs config.pattern; cost_threshold needs config.
|
|
66
|
+
'Parameters. name is 1-120 chars (Zod-enforced min/max); appears in eval_result rule_results so make it human-readable. description is optional, max 500 chars (used in dashboard tooltips). evalType determines WHEN the rule fires (must match the eval_type your evaluate_output calls use; e.g., a "completeness" rule fires on every evaluate_output where eval_type="completeness" OR eval_type="custom"). severity affects dashboard sort + audit log signal but does NOT affect scoring (scoring uses the rule\'s weight). definition.type and definition.config must match (e.g., regex_match needs config.pattern; cost_threshold needs config.max_cost; min_length needs config.min_length; max_length needs config.max_length; contains_keywords/excludes_keywords need config.keywords). Invalid configs are now REJECTED at deploy time with the offending field named, instead of deploying and then failing every evaluation. sourceMomentId is optional but recommended (preserves workflow-inversion provenance from Make-This-A-Rule composer). Defaults: severity="medium".',
|
|
67
67
|
'',
|
|
68
68
|
"Error modes. Throws 400 on invalid definition (Zod rejects — e.g., regex that fails safe-regex2 ReDoS check, or length > 1000 chars). Throws 400 on empty `name`. Throws 400 if the eval category mismatches the definition type. Returns 429 when HTTP rate limit exceeded. File-write failures (disk full, read-only fs) propagate as 500; the audit log is best-effort and does not block deploy.",
|
|
69
69
|
].join('\n'),
|
|
@@ -6,7 +6,7 @@ const CustomRuleSchema = z.object({
|
|
|
6
6
|
'regex_match', 'regex_no_match', 'min_length', 'max_length',
|
|
7
7
|
'contains_keywords', 'excludes_keywords', 'json_schema', 'cost_threshold',
|
|
8
8
|
]),
|
|
9
|
-
config: z.record(z.unknown()),
|
|
9
|
+
config: z.record(z.string(), z.unknown()),
|
|
10
10
|
weight: z.number().optional(),
|
|
11
11
|
});
|
|
12
12
|
const inputSchema = {
|
package/dist/tools/log-trace.js
CHANGED
|
@@ -18,11 +18,11 @@ const SpanSchema = z.object({
|
|
|
18
18
|
status_message: z.string().optional(),
|
|
19
19
|
start_time: z.string(),
|
|
20
20
|
end_time: z.string().optional(),
|
|
21
|
-
attributes: z.record(z.unknown()).optional(),
|
|
21
|
+
attributes: z.record(z.string(), z.unknown()).optional(),
|
|
22
22
|
events: z.array(z.object({
|
|
23
23
|
name: z.string(),
|
|
24
24
|
timestamp: z.string(),
|
|
25
|
-
attributes: z.record(z.unknown()).optional(),
|
|
25
|
+
attributes: z.record(z.string(), z.unknown()).optional(),
|
|
26
26
|
})).optional(),
|
|
27
27
|
});
|
|
28
28
|
const TokenUsageSchema = z.object({
|
|
@@ -39,7 +39,7 @@ const inputSchema = {
|
|
|
39
39
|
latency_ms: z.number().optional().describe('Total execution time in milliseconds (end-to-end agent latency)'),
|
|
40
40
|
token_usage: TokenUsageSchema.optional().describe('Token usage breakdown (prompt/completion/total — used for cost analysis)'),
|
|
41
41
|
cost_usd: z.number().optional().describe('Total cost in USD — overrides per-span aggregation when provided (treated as authoritative)'),
|
|
42
|
-
metadata: z.record(z.unknown()).optional().describe('Opaque key-value tags (e.g. {requestId, userId, env}) — queryable in dashboard, not via get_traces filters'),
|
|
42
|
+
metadata: z.record(z.string(), z.unknown()).optional().describe('Opaque key-value tags (e.g. {requestId, userId, env}) — queryable in dashboard, not via get_traces filters'),
|
|
43
43
|
spans: z.array(SpanSchema).optional().describe('Detailed execution spans (hierarchical span tree with timings, attributes, events)'),
|
|
44
44
|
timestamp: z.string().optional().describe('Trace timestamp (ISO 8601); defaults to now() when omitted'),
|
|
45
45
|
};
|