@iris-eval/mcp-server 0.4.4 → 0.4.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +55 -1
- package/dist/audit-log-reader.js +1 -1
- package/dist/config/defaults.d.ts +1 -0
- package/dist/config/defaults.js +3 -0
- package/dist/custom-rule-store.js +109 -2
- package/dist/dashboard/assets/index-CIKsbEhq.js +10 -0
- package/dist/dashboard/index.html +1 -1
- package/dist/dashboard/routes/rules.js +1 -1
- package/dist/dashboard/server.js +16 -3
- package/dist/dashboard/validation.d.ts +36 -62
- package/dist/eval/citation-verify/resolve.js +101 -15
- package/dist/eval/rules/config-keys.d.ts +14 -0
- package/dist/eval/rules/config-keys.js +43 -0
- package/dist/eval/rules/custom.js +43 -14
- package/dist/eval/rules/relevance.d.ts +1 -0
- package/dist/eval/rules/relevance.js +3 -1
- package/dist/eval/rules/safety.d.ts +5 -0
- package/dist/eval/rules/safety.js +4 -2
- package/dist/index.js +3 -1
- package/dist/middleware/error-handler.js +19 -1
- package/dist/otel/mapper.js +2 -1
- package/dist/preferences.d.ts +43 -93
- package/dist/tools/deploy-rule.js +2 -2
- package/dist/tools/evaluate-output.js +1 -1
- package/dist/tools/log-trace.js +3 -3
- package/dist/transport/http.js +68 -4
- package/package.json +10 -5
- package/server.json +1 -1
- package/dist/dashboard/assets/index-DNflCqmJ.js +0 -12
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
6
6
|
<link rel="icon" type="image/svg+xml" href="/favicon.svg" />
|
|
7
7
|
<title>Iris — Agent Eval & Observability</title>
|
|
8
|
-
<script type="module" crossorigin src="/assets/index-
|
|
8
|
+
<script type="module" crossorigin src="/assets/index-CIKsbEhq.js"></script>
|
|
9
9
|
<link rel="stylesheet" crossorigin href="/assets/index-B4Aw6ozt.css">
|
|
10
10
|
</head>
|
|
11
11
|
<body>
|
|
@@ -16,7 +16,7 @@ const RuleTypeSchema = z.enum([
|
|
|
16
16
|
const DefinitionSchema = z.object({
|
|
17
17
|
name: z.string().min(1).max(80),
|
|
18
18
|
type: RuleTypeSchema,
|
|
19
|
-
config: z.record(z.unknown()),
|
|
19
|
+
config: z.record(z.string(), z.unknown()),
|
|
20
20
|
weight: z.number().positive().optional(),
|
|
21
21
|
});
|
|
22
22
|
const DeploySchema = z.object({
|
package/dist/dashboard/server.js
CHANGED
|
@@ -72,14 +72,27 @@ export function createDashboardServer(storage, config, logger, options) {
|
|
|
72
72
|
// when no custom rule store is provided (read-only access).
|
|
73
73
|
registerAuditRoutes(router, options?.customRuleStore);
|
|
74
74
|
app.use('/api/v1', router);
|
|
75
|
-
// Serve static dashboard files if built (rate limited)
|
|
75
|
+
// Serve static dashboard files if built (rate limited).
|
|
76
|
+
//
|
|
77
|
+
// Gate on index.html, not on the directory: `npm run build` compiles the
|
|
78
|
+
// dashboard SERVER into dist/dashboard (server.js, routes/) without the UI
|
|
79
|
+
// bundle, which is built separately by `cd dashboard && npm run build`. The
|
|
80
|
+
// directory therefore exists while index.html does not, so the SPA fallback
|
|
81
|
+
// was registered and every unmatched route hit res.sendFile on a missing
|
|
82
|
+
// file. The resulting ENOENT carries an absolute path, and the error
|
|
83
|
+
// handler returned it verbatim to the client:
|
|
84
|
+
// {"error":"ENOENT: ... stat 'C:\\...\\dist\\dashboard\\index.html'"}
|
|
85
|
+
// — leaking the install path (and the OS user) to anyone who can reach the
|
|
86
|
+
// dashboard. Without the UI built there is nothing to fall back TO, so the
|
|
87
|
+
// route simply should not exist, and unmatched paths get Express's own 404.
|
|
76
88
|
const currentDir = dirname(fileURLToPath(import.meta.url));
|
|
77
89
|
const staticDir = join(currentDir, '..', '..', 'dist', 'dashboard');
|
|
78
|
-
|
|
90
|
+
const indexHtml = join(staticDir, 'index.html');
|
|
91
|
+
if (existsSync(indexHtml)) {
|
|
79
92
|
app.use(createApiRateLimiter(config));
|
|
80
93
|
app.use(express.static(staticDir));
|
|
81
94
|
app.get('/{*path}', (_req, res) => {
|
|
82
|
-
res.sendFile(
|
|
95
|
+
res.sendFile(indexHtml);
|
|
83
96
|
});
|
|
84
97
|
}
|
|
85
98
|
// Error handler (must be last)
|
|
@@ -4,72 +4,46 @@ export declare const traceQuerySchema: z.ZodObject<{
|
|
|
4
4
|
framework: z.ZodOptional<z.ZodString>;
|
|
5
5
|
since: z.ZodOptional<z.ZodString>;
|
|
6
6
|
until: z.ZodOptional<z.ZodString>;
|
|
7
|
-
limit: z.ZodDefault<z.
|
|
8
|
-
offset: z.ZodDefault<z.
|
|
9
|
-
sort_by: z.ZodDefault<z.ZodEnum<
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
until?: string | undefined;
|
|
20
|
-
}, {
|
|
21
|
-
agent_name?: string | undefined;
|
|
22
|
-
framework?: string | undefined;
|
|
23
|
-
since?: string | undefined;
|
|
24
|
-
until?: string | undefined;
|
|
25
|
-
limit?: number | undefined;
|
|
26
|
-
offset?: number | undefined;
|
|
27
|
-
sort_by?: "timestamp" | "latency_ms" | "cost_usd" | undefined;
|
|
28
|
-
sort_order?: "asc" | "desc" | undefined;
|
|
29
|
-
}>;
|
|
7
|
+
limit: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
8
|
+
offset: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
9
|
+
sort_by: z.ZodDefault<z.ZodEnum<{
|
|
10
|
+
timestamp: "timestamp";
|
|
11
|
+
latency_ms: "latency_ms";
|
|
12
|
+
cost_usd: "cost_usd";
|
|
13
|
+
}>>;
|
|
14
|
+
sort_order: z.ZodDefault<z.ZodEnum<{
|
|
15
|
+
asc: "asc";
|
|
16
|
+
desc: "desc";
|
|
17
|
+
}>>;
|
|
18
|
+
}, z.core.$strip>;
|
|
30
19
|
export declare const evalQuerySchema: z.ZodObject<{
|
|
31
20
|
eval_type: z.ZodOptional<z.ZodString>;
|
|
32
|
-
passed: z.ZodOptional<z.
|
|
21
|
+
passed: z.ZodOptional<z.ZodPipe<z.ZodEnum<{
|
|
22
|
+
true: "true";
|
|
23
|
+
false: "false";
|
|
24
|
+
}>, z.ZodTransform<boolean, "true" | "false">>>;
|
|
33
25
|
since: z.ZodOptional<z.ZodString>;
|
|
34
26
|
until: z.ZodOptional<z.ZodString>;
|
|
35
|
-
limit: z.ZodDefault<z.
|
|
36
|
-
offset: z.ZodDefault<z.
|
|
37
|
-
},
|
|
38
|
-
limit: number;
|
|
39
|
-
offset: number;
|
|
40
|
-
since?: string | undefined;
|
|
41
|
-
until?: string | undefined;
|
|
42
|
-
eval_type?: string | undefined;
|
|
43
|
-
passed?: boolean | undefined;
|
|
44
|
-
}, {
|
|
45
|
-
since?: string | undefined;
|
|
46
|
-
until?: string | undefined;
|
|
47
|
-
limit?: number | undefined;
|
|
48
|
-
offset?: number | undefined;
|
|
49
|
-
eval_type?: string | undefined;
|
|
50
|
-
passed?: "true" | "false" | undefined;
|
|
51
|
-
}>;
|
|
27
|
+
limit: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
28
|
+
offset: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
29
|
+
}, z.core.$strip>;
|
|
52
30
|
export declare const summaryQuerySchema: z.ZodObject<{
|
|
53
|
-
hours: z.ZodDefault<z.
|
|
54
|
-
},
|
|
55
|
-
hours: number;
|
|
56
|
-
}, {
|
|
57
|
-
hours?: number | undefined;
|
|
58
|
-
}>;
|
|
31
|
+
hours: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
32
|
+
}, z.core.$strip>;
|
|
59
33
|
export declare const evalStatsPeriodSchema: z.ZodObject<{
|
|
60
|
-
period: z.ZodDefault<z.ZodEnum<
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
}
|
|
34
|
+
period: z.ZodDefault<z.ZodEnum<{
|
|
35
|
+
"24h": "24h";
|
|
36
|
+
"7d": "7d";
|
|
37
|
+
"30d": "30d";
|
|
38
|
+
all: "all";
|
|
39
|
+
}>>;
|
|
40
|
+
}, z.core.$strip>;
|
|
66
41
|
export declare const evalStatsFailuresSchema: z.ZodObject<{
|
|
67
|
-
period: z.ZodDefault<z.ZodEnum<
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
}
|
|
73
|
-
limit
|
|
74
|
-
|
|
75
|
-
}>;
|
|
42
|
+
period: z.ZodDefault<z.ZodEnum<{
|
|
43
|
+
"24h": "24h";
|
|
44
|
+
"7d": "7d";
|
|
45
|
+
"30d": "30d";
|
|
46
|
+
all: "all";
|
|
47
|
+
}>>;
|
|
48
|
+
limit: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
49
|
+
}, z.core.$strip>;
|
|
@@ -6,7 +6,10 @@
|
|
|
6
6
|
// Defense layers (in order):
|
|
7
7
|
// 1. Scheme allowlist — http/https only; refuse file:/javascript:/etc.
|
|
8
8
|
// 2. SSRF host check — refuse localhost, link-local, private ranges,
|
|
9
|
-
// and cloud metadata (AWS/GCP/Azure/DigitalOcean) IP literals.
|
|
9
|
+
// and cloud metadata (AWS/GCP/Azure/DigitalOcean) IP literals. IPv6
|
|
10
|
+
// literals are de-bracketed and canonicalized (incl. IPv4-mapped
|
|
11
|
+
// forms) before classification so `[::1]`, `[fd00::1]`, and
|
|
12
|
+
// `[::ffff:169.254.169.254]` cannot slip past the ^-anchored checks.
|
|
10
13
|
// 3. DNS pre-resolve — every public hostname is resolved via
|
|
11
14
|
// dns.lookup({all:true}) and EVERY returned IP is re-checked against
|
|
12
15
|
// the IP blocklist. Defeats DNS-rebinding via public records pointing
|
|
@@ -53,12 +56,6 @@ const BLOCKED_IPV4 = [
|
|
|
53
56
|
// This-network
|
|
54
57
|
/^0\./,
|
|
55
58
|
];
|
|
56
|
-
const BLOCKED_IPV6 = [
|
|
57
|
-
/^::1$/, // localhost
|
|
58
|
-
/^fc|^fd/i, // unique local
|
|
59
|
-
/^fe80/i, // link-local
|
|
60
|
-
/^::ffff:127\./i, // IPv4-mapped localhost
|
|
61
|
-
];
|
|
62
59
|
const BLOCKED_HOST_SUBSTRINGS = ['localhost', 'internal', '.local', 'metadata.google', 'metadata.azure'];
|
|
63
60
|
function isIpv4(host) {
|
|
64
61
|
return /^\d{1,3}(\.\d{1,3}){3}$/.test(host);
|
|
@@ -66,23 +63,112 @@ function isIpv4(host) {
|
|
|
66
63
|
function isIpv6(host) {
|
|
67
64
|
return host.includes(':');
|
|
68
65
|
}
|
|
66
|
+
// WHATWG URL parsing leaves IPv6 literals bracketed:
|
|
67
|
+
// `new URL('http://[::1]/').hostname === '[::1]'`. Every historical
|
|
68
|
+
// BLOCKED_IPV6 entry was `^`-anchored (`/^::1$/`, `/^fe80/`, …), so the
|
|
69
|
+
// leading `[` made ALL of them silently fail to match — the entire IPv6
|
|
70
|
+
// SSRF guard was inert for direct address literals (loopback, link-local,
|
|
71
|
+
// unique-local, and IPv4-mapped metadata all passed as "safe"). Strip the
|
|
72
|
+
// brackets before any IPv6 classification.
|
|
73
|
+
function stripIpv6Brackets(host) {
|
|
74
|
+
return host.length > 1 && host.startsWith('[') && host.endsWith(']')
|
|
75
|
+
? host.slice(1, -1)
|
|
76
|
+
: host;
|
|
77
|
+
}
|
|
78
|
+
// Expand a compressed / embedded-IPv4 IPv6 literal to exactly 8 zero-padded
|
|
79
|
+
// hextets. Returns null when `addr` is not a syntactically valid IPv6 literal.
|
|
80
|
+
// Canonicalizing to full form makes prefix classification reliable regardless
|
|
81
|
+
// of how the address was serialized (`::1`, `0:0:...:1`, `::ffff:a9fe:a9fe`).
|
|
82
|
+
function expandIpv6(addr) {
|
|
83
|
+
let a = addr.toLowerCase();
|
|
84
|
+
const zone = a.indexOf('%');
|
|
85
|
+
if (zone !== -1)
|
|
86
|
+
a = a.slice(0, zone); // drop scope/zone id
|
|
87
|
+
// Fold a trailing embedded IPv4 (`::ffff:1.2.3.4`, `::1.2.3.4`) into two hextets.
|
|
88
|
+
const v4 = a.match(/^(.*:)(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/);
|
|
89
|
+
if (v4) {
|
|
90
|
+
const octets = [v4[2], v4[3], v4[4], v4[5]].map(Number);
|
|
91
|
+
if (octets.some((n) => n > 255))
|
|
92
|
+
return null;
|
|
93
|
+
const hi = octets[0] * 256 + octets[1];
|
|
94
|
+
const lo = octets[2] * 256 + octets[3];
|
|
95
|
+
a = `${v4[1]}${hi.toString(16)}:${lo.toString(16)}`;
|
|
96
|
+
}
|
|
97
|
+
const halves = a.split('::');
|
|
98
|
+
if (halves.length > 2)
|
|
99
|
+
return null;
|
|
100
|
+
const head = halves[0] ? halves[0].split(':') : [];
|
|
101
|
+
const tail = halves.length === 2 && halves[1] ? halves[1].split(':') : [];
|
|
102
|
+
let groups;
|
|
103
|
+
if (halves.length === 2) {
|
|
104
|
+
const missing = 8 - head.length - tail.length;
|
|
105
|
+
if (missing < 0)
|
|
106
|
+
return null;
|
|
107
|
+
groups = [...head, ...Array(missing).fill('0'), ...tail];
|
|
108
|
+
}
|
|
109
|
+
else {
|
|
110
|
+
groups = head;
|
|
111
|
+
}
|
|
112
|
+
if (groups.length !== 8)
|
|
113
|
+
return null;
|
|
114
|
+
const out = [];
|
|
115
|
+
for (const g of groups) {
|
|
116
|
+
if (!/^[0-9a-f]{1,4}$/.test(g))
|
|
117
|
+
return null;
|
|
118
|
+
out.push(g.padStart(4, '0'));
|
|
119
|
+
}
|
|
120
|
+
return out;
|
|
121
|
+
}
|
|
122
|
+
function ipv4FromHextets(h6, h7) {
|
|
123
|
+
const hi = parseInt(h6, 16);
|
|
124
|
+
const lo = parseInt(h7, 16);
|
|
125
|
+
return `${Math.floor(hi / 256)}.${hi % 256}.${Math.floor(lo / 256)}.${lo % 256}`;
|
|
126
|
+
}
|
|
127
|
+
// True when an IPv6 literal resolves to a range we refuse: unspecified,
|
|
128
|
+
// loopback, link-local, unique-local, or an embedded IPv4 that itself hits
|
|
129
|
+
// the IPv4 blocklist (IPv4-mapped `::ffff:a.b.c.d` reaches the v4 endpoint on
|
|
130
|
+
// dual-stack hosts — e.g. `::ffff:169.254.169.254` == AWS IMDS). Fails closed
|
|
131
|
+
// on any colon-bearing host that does not parse as valid IPv6.
|
|
132
|
+
function isBlockedIpv6(addr) {
|
|
133
|
+
const g = expandIpv6(addr);
|
|
134
|
+
if (!g)
|
|
135
|
+
return true; // unparseable but IPv6-shaped (has a colon) — refuse
|
|
136
|
+
if (g.every((h) => h === '0000'))
|
|
137
|
+
return true; // :: unspecified
|
|
138
|
+
if (g.slice(0, 7).every((h) => h === '0000') && g[7] === '0001')
|
|
139
|
+
return true; // ::1 loopback
|
|
140
|
+
const first = g[0];
|
|
141
|
+
// fe80::/10 link-local (fe80–febf)
|
|
142
|
+
if (first === 'fe80' || /^fe[89ab]/.test(first))
|
|
143
|
+
return true;
|
|
144
|
+
// fc00::/7 unique-local (fc.. / fd..)
|
|
145
|
+
if (first.startsWith('fc') || first.startsWith('fd'))
|
|
146
|
+
return true;
|
|
147
|
+
// IPv4-mapped ::ffff:a.b.c.d and IPv4-compatible ::a.b.c.d (deprecated)
|
|
148
|
+
const mapped = g.slice(0, 5).every((h) => h === '0000') && g[5] === 'ffff';
|
|
149
|
+
const compat = g.slice(0, 6).every((h) => h === '0000') && !(g[6] === '0000' && g[7] === '0000');
|
|
150
|
+
if (mapped || compat) {
|
|
151
|
+
const embedded = ipv4FromHextets(g[6], g[7]);
|
|
152
|
+
return BLOCKED_IPV4.some((re) => re.test(embedded));
|
|
153
|
+
}
|
|
154
|
+
return false;
|
|
155
|
+
}
|
|
69
156
|
export function isSafeHost(host) {
|
|
70
|
-
const
|
|
157
|
+
const bare = stripIpv6Brackets(host);
|
|
158
|
+
const hostLower = bare.toLowerCase();
|
|
71
159
|
for (const sub of BLOCKED_HOST_SUBSTRINGS) {
|
|
72
160
|
if (hostLower === sub || hostLower.endsWith(sub))
|
|
73
161
|
return false;
|
|
74
162
|
}
|
|
75
|
-
if (isIpv4(
|
|
163
|
+
if (isIpv4(bare)) {
|
|
76
164
|
for (const re of BLOCKED_IPV4) {
|
|
77
|
-
if (re.test(
|
|
165
|
+
if (re.test(bare))
|
|
78
166
|
return false;
|
|
79
167
|
}
|
|
80
168
|
}
|
|
81
|
-
if (isIpv6(
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
return false;
|
|
85
|
-
}
|
|
169
|
+
if (isIpv6(bare)) {
|
|
170
|
+
if (isBlockedIpv6(bare))
|
|
171
|
+
return false;
|
|
86
172
|
}
|
|
87
173
|
return true;
|
|
88
174
|
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export declare const CUSTOM_RULE_CONFIG_KEYS: {
|
|
2
|
+
readonly min_length: readonly ["min_length", "length", "min"];
|
|
3
|
+
readonly max_length: readonly ["max_length", "length", "max"];
|
|
4
|
+
readonly cost_threshold: readonly ["max_cost", "max_usd"];
|
|
5
|
+
};
|
|
6
|
+
/** First key is the canonical one to document and teach. */
|
|
7
|
+
export declare function canonicalKey(type: keyof typeof CUSTOM_RULE_CONFIG_KEYS): string;
|
|
8
|
+
/**
|
|
9
|
+
* Read the first defined numeric value among a rule type's accepted keys.
|
|
10
|
+
* Returns undefined when none is present, so callers can raise a config error.
|
|
11
|
+
*/
|
|
12
|
+
export declare function readNumericConfig(config: Record<string, unknown>, type: keyof typeof CUSTOM_RULE_CONFIG_KEYS): number | undefined;
|
|
13
|
+
/** Human-readable "config.a (or config.b)" for error messages. */
|
|
14
|
+
export declare function describeKeys(type: keyof typeof CUSTOM_RULE_CONFIG_KEYS): string;
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Canonical config keys for custom rule types — the SINGLE source of truth.
|
|
3
|
+
*
|
|
4
|
+
* This module exists because the keys drifted across three surfaces and
|
|
5
|
+
* shipped broken: the evaluator read `config.min_length`, while the
|
|
6
|
+
* `deploy_rule` tool description (the text an LLM agent reads to construct
|
|
7
|
+
* its call) told agents to send `config.min`, and `docs/api-reference.md`
|
|
8
|
+
* showed `{ "min": 40 }`. Deploy-time validation accepted any object, so a
|
|
9
|
+
* rule built from our own documentation deployed cleanly and then failed on
|
|
10
|
+
* every evaluation forever.
|
|
11
|
+
*
|
|
12
|
+
* The evaluator, the deploy-time validator, and the tool description now all
|
|
13
|
+
* derive from this map, so a key cannot be correct in one place and wrong in
|
|
14
|
+
* another. The first entry of each list is CANONICAL (what docs should
|
|
15
|
+
* teach); the rest are accepted aliases kept for compatibility with configs
|
|
16
|
+
* created from earlier, incorrect documentation.
|
|
17
|
+
*/
|
|
18
|
+
export const CUSTOM_RULE_CONFIG_KEYS = {
|
|
19
|
+
min_length: ['min_length', 'length', 'min'],
|
|
20
|
+
max_length: ['max_length', 'length', 'max'],
|
|
21
|
+
cost_threshold: ['max_cost', 'max_usd'],
|
|
22
|
+
};
|
|
23
|
+
/** First key is the canonical one to document and teach. */
|
|
24
|
+
export function canonicalKey(type) {
|
|
25
|
+
return CUSTOM_RULE_CONFIG_KEYS[type][0];
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Read the first defined numeric value among a rule type's accepted keys.
|
|
29
|
+
* Returns undefined when none is present, so callers can raise a config error.
|
|
30
|
+
*/
|
|
31
|
+
export function readNumericConfig(config, type) {
|
|
32
|
+
for (const key of CUSTOM_RULE_CONFIG_KEYS[type]) {
|
|
33
|
+
const value = config[key];
|
|
34
|
+
if (typeof value === 'number' && Number.isFinite(value))
|
|
35
|
+
return value;
|
|
36
|
+
}
|
|
37
|
+
return undefined;
|
|
38
|
+
}
|
|
39
|
+
/** Human-readable "config.a (or config.b)" for error messages. */
|
|
40
|
+
export function describeKeys(type) {
|
|
41
|
+
const [first, ...rest] = CUSTOM_RULE_CONFIG_KEYS[type];
|
|
42
|
+
return rest.length ? `config.${first} (aliases: ${rest.map((k) => `config.${k}`).join(', ')})` : `config.${first}`;
|
|
43
|
+
}
|
|
@@ -1,7 +1,30 @@
|
|
|
1
1
|
import isSafeRegex from 'safe-regex2';
|
|
2
|
+
import { readNumericConfig, describeKeys } from './config-keys.js';
|
|
2
3
|
const MAX_PATTERN_LENGTH = 1000;
|
|
4
|
+
// A rule whose CONFIG is invalid has not evaluated the output — it could not
|
|
5
|
+
// run at all. Returning `passed:false, score:0` for that case conflates "your
|
|
6
|
+
// agent produced bad output" with "this rule is broken", and because the
|
|
7
|
+
// engine weights every non-skipped result, one misconfigured rule silently
|
|
8
|
+
// deflates every eval score for as long as it stays deployed (and gives no
|
|
9
|
+
// hint which of the two happened). Mark it skipped: the engine already
|
|
10
|
+
// excludes skipped rules from the weighted average and surfaces skipReason,
|
|
11
|
+
// the same contract `expected_coverage` uses when no expected output exists.
|
|
12
|
+
//
|
|
13
|
+
// Deploy-time validation in custom-rule-store.ts now rejects these configs
|
|
14
|
+
// outright; this path remains the safety net for rules already persisted in
|
|
15
|
+
// a user's ~/.iris/custom-rules.json from before that validation existed.
|
|
16
|
+
function configError(definition, message) {
|
|
17
|
+
return {
|
|
18
|
+
ruleName: definition.name,
|
|
19
|
+
passed: false,
|
|
20
|
+
score: 0,
|
|
21
|
+
message,
|
|
22
|
+
skipped: true,
|
|
23
|
+
skipReason: message,
|
|
24
|
+
};
|
|
25
|
+
}
|
|
3
26
|
function safeRegexResult(definition, message) {
|
|
4
|
-
return
|
|
27
|
+
return configError(definition, message);
|
|
5
28
|
}
|
|
6
29
|
function compileRegex(definition) {
|
|
7
30
|
let patternStr = definition.config.pattern;
|
|
@@ -19,15 +42,21 @@ function compileRegex(definition) {
|
|
|
19
42
|
if (patternStr.length > MAX_PATTERN_LENGTH) {
|
|
20
43
|
return safeRegexResult(definition, `Regex pattern too long (${patternStr.length} > ${MAX_PATTERN_LENGTH})`);
|
|
21
44
|
}
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
45
|
+
// Syntax BEFORE safety: safe-regex2 returns false for anything it cannot
|
|
46
|
+
// parse, so checking it first reports a plainly broken pattern like `(` as
|
|
47
|
+
// "catastrophic backtracking" — sending the author hunting a performance
|
|
48
|
+
// problem they do not have instead of the typo they do.
|
|
49
|
+
let compiled;
|
|
25
50
|
try {
|
|
26
|
-
|
|
51
|
+
compiled = new RegExp(patternStr, flags);
|
|
27
52
|
}
|
|
28
53
|
catch (e) {
|
|
29
54
|
return safeRegexResult(definition, `Invalid regex syntax: ${e instanceof Error ? e.message : 'unknown error'}`);
|
|
30
55
|
}
|
|
56
|
+
if (!isSafeRegex(patternStr)) {
|
|
57
|
+
return safeRegexResult(definition, 'Regex pattern rejected: potentially unsafe (catastrophic backtracking)');
|
|
58
|
+
}
|
|
59
|
+
return compiled;
|
|
31
60
|
}
|
|
32
61
|
export function createCustomRule(definition) {
|
|
33
62
|
return {
|
|
@@ -52,17 +81,17 @@ export function createCustomRule(definition) {
|
|
|
52
81
|
return { ruleName: definition.name, passed, score: passed ? 1 : 0, message: passed ? 'Forbidden pattern not found' : 'Forbidden pattern found in output' };
|
|
53
82
|
}
|
|
54
83
|
case 'min_length': {
|
|
55
|
-
const min = (definition.config
|
|
84
|
+
const min = readNumericConfig(definition.config, 'min_length');
|
|
56
85
|
if (min == null || min <= 0) {
|
|
57
|
-
return
|
|
86
|
+
return configError(definition, `min_length rule requires ${describeKeys('min_length')} (positive number)`);
|
|
58
87
|
}
|
|
59
88
|
const passed = context.output.length >= min;
|
|
60
89
|
return { ruleName: definition.name, passed, score: passed ? 1 : context.output.length / min, message: passed ? `Length (${context.output.length}) meets minimum (${min})` : `Length (${context.output.length}) below minimum (${min})` };
|
|
61
90
|
}
|
|
62
91
|
case 'max_length': {
|
|
63
|
-
const max = (definition.config
|
|
92
|
+
const max = readNumericConfig(definition.config, 'max_length');
|
|
64
93
|
if (max == null || max <= 0) {
|
|
65
|
-
return
|
|
94
|
+
return configError(definition, `max_length rule requires ${describeKeys('max_length')} (positive number)`);
|
|
66
95
|
}
|
|
67
96
|
const passed = context.output.length <= max;
|
|
68
97
|
return { ruleName: definition.name, passed, score: passed ? 1 : max / context.output.length, message: passed ? `Length (${context.output.length}) within maximum (${max})` : `Length (${context.output.length}) exceeds maximum (${max})` };
|
|
@@ -70,7 +99,7 @@ export function createCustomRule(definition) {
|
|
|
70
99
|
case 'contains_keywords': {
|
|
71
100
|
const keywords = definition.config.keywords;
|
|
72
101
|
if (!keywords || !Array.isArray(keywords) || keywords.length === 0) {
|
|
73
|
-
return
|
|
102
|
+
return configError(definition, 'contains_keywords rule requires config.keywords (non-empty string array)');
|
|
74
103
|
}
|
|
75
104
|
const lower = context.output.toLowerCase();
|
|
76
105
|
const found = keywords.filter((k) => lower.includes(k.toLowerCase()));
|
|
@@ -81,7 +110,7 @@ export function createCustomRule(definition) {
|
|
|
81
110
|
case 'excludes_keywords': {
|
|
82
111
|
const keywords = definition.config.keywords;
|
|
83
112
|
if (!keywords || !Array.isArray(keywords) || keywords.length === 0) {
|
|
84
|
-
return
|
|
113
|
+
return configError(definition, 'excludes_keywords rule requires config.keywords (non-empty string array)');
|
|
85
114
|
}
|
|
86
115
|
const lower = context.output.toLowerCase();
|
|
87
116
|
const found = keywords.filter((k) => lower.includes(k.toLowerCase()));
|
|
@@ -98,16 +127,16 @@ export function createCustomRule(definition) {
|
|
|
98
127
|
}
|
|
99
128
|
}
|
|
100
129
|
case 'cost_threshold': {
|
|
101
|
-
const max = definition.config
|
|
130
|
+
const max = readNumericConfig(definition.config, 'cost_threshold');
|
|
102
131
|
if (max == null || max < 0) {
|
|
103
|
-
return
|
|
132
|
+
return configError(definition, `cost_threshold rule requires ${describeKeys('cost_threshold')} (non-negative number)`);
|
|
104
133
|
}
|
|
105
134
|
const cost = context.costUsd ?? 0;
|
|
106
135
|
const passed = cost <= max;
|
|
107
136
|
return { ruleName: definition.name, passed, score: passed ? 1 : 0, message: passed ? `Cost ($${cost}) within threshold ($${max})` : `Cost ($${cost}) exceeds threshold ($${max})` };
|
|
108
137
|
}
|
|
109
138
|
default:
|
|
110
|
-
return
|
|
139
|
+
return configError(definition, `Unknown rule type: ${definition.type}`);
|
|
111
140
|
}
|
|
112
141
|
},
|
|
113
142
|
};
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { EvalRule } from '../../types/eval.js';
|
|
2
2
|
export declare const keywordOverlap: EvalRule;
|
|
3
|
+
export declare const HALLUCINATION_MARKERS: string[];
|
|
3
4
|
export declare const noHallucinationMarkers: EvalRule;
|
|
4
5
|
export declare const topicConsistency: EvalRule;
|
|
5
6
|
export declare const relevanceRules: EvalRule[];
|
|
@@ -28,7 +28,9 @@ export const keywordOverlap = {
|
|
|
28
28
|
};
|
|
29
29
|
},
|
|
30
30
|
};
|
|
31
|
-
|
|
31
|
+
// Exported so the claims drift test can assert .claims.json counts against
|
|
32
|
+
// the runtime truth (tests/claims-eval-rules-counts.test.ts).
|
|
33
|
+
export const HALLUCINATION_MARKERS = [
|
|
32
34
|
'as an ai',
|
|
33
35
|
'as a language model',
|
|
34
36
|
'i cannot',
|
|
@@ -1,6 +1,11 @@
|
|
|
1
1
|
import type { EvalRule } from '../../types/eval.js';
|
|
2
|
+
export declare const PII_PATTERNS: Array<{
|
|
3
|
+
name: string;
|
|
4
|
+
pattern: RegExp;
|
|
5
|
+
}>;
|
|
2
6
|
export declare const noPii: EvalRule;
|
|
3
7
|
export declare const noBlocklistWords: EvalRule;
|
|
8
|
+
export declare const INJECTION_PATTERNS: RegExp[];
|
|
4
9
|
export declare const noInjectionPatterns: EvalRule;
|
|
5
10
|
export declare const noStubOutput: EvalRule;
|
|
6
11
|
export declare const safetyRules: EvalRule[];
|
|
@@ -5,7 +5,9 @@
|
|
|
5
5
|
* patterns evaluate. Word-boundary anchors avoid matching inside larger
|
|
6
6
|
* strings where appropriate.
|
|
7
7
|
*/
|
|
8
|
-
|
|
8
|
+
// Exported so the claims drift test can assert .claims.json counts against
|
|
9
|
+
// the runtime truth (tests/claims-eval-rules-counts.test.ts).
|
|
10
|
+
export const PII_PATTERNS = [
|
|
9
11
|
// Original v0.3.0 patterns
|
|
10
12
|
{ name: 'SSN', pattern: /\b\d{3}-\d{2}-\d{4}\b/ },
|
|
11
13
|
{ name: 'Credit Card', pattern: /\b(?:\d{4}[-\s]?){3}\d{4}\b/ },
|
|
@@ -79,7 +81,7 @@ export const noBlocklistWords = {
|
|
|
79
81
|
* leaks, or role-override acknowledgments). Input-side detection is the
|
|
80
82
|
* agent host's job; output-side is Iris's.
|
|
81
83
|
*/
|
|
82
|
-
const INJECTION_PATTERNS = [
|
|
84
|
+
export const INJECTION_PATTERNS = [
|
|
83
85
|
// Original v0.3.0 patterns
|
|
84
86
|
/ignore (?:all )?(?:previous|above|prior) (?:instructions|prompts)/i,
|
|
85
87
|
/you are now (?:a |in )/i,
|
package/dist/index.js
CHANGED
|
@@ -85,7 +85,9 @@ Environment variables (CLI flags take precedence):
|
|
|
85
85
|
IRIS_DASHBOARD true to enable web dashboard
|
|
86
86
|
IRIS_DASHBOARD_PORT Dashboard port (1-65535, default: 6920)
|
|
87
87
|
IRIS_API_KEY API key for HTTP authentication
|
|
88
|
-
IRIS_ALLOWED_ORIGINS Comma-separated CORS
|
|
88
|
+
IRIS_ALLOWED_ORIGINS Comma-separated origin allowlist. Dashboard: CORS headers (supports globs, e.g. http://localhost:*).
|
|
89
|
+
HTTP transport: exact-match Origin allowlist for DNS-rebinding protection (globs ignored;
|
|
90
|
+
this server's own loopback origins are always allowed).
|
|
89
91
|
IRIS_NO_AUTO_LAUNCH Set to 1 to disable first-run dashboard auto-launch
|
|
90
92
|
IRIS_ANTHROPIC_API_KEY Required by evaluate_with_llm_judge + verify_citations (provider=anthropic)
|
|
91
93
|
IRIS_OPENAI_API_KEY Required by evaluate_with_llm_judge + verify_citations (provider=openai)
|
|
@@ -9,7 +9,25 @@ export function createErrorHandler(logger) {
|
|
|
9
9
|
return;
|
|
10
10
|
}
|
|
11
11
|
const status = err.status ?? err.statusCode ?? 500;
|
|
12
|
-
|
|
12
|
+
/*
|
|
13
|
+
* Never echo a Node system error to the client. Their messages embed
|
|
14
|
+
* absolute paths — an ENOENT from res.sendFile returned
|
|
15
|
+
* "ENOENT: no such file or directory, stat 'C:\...\dist\dashboard\index.html'"
|
|
16
|
+
* with a 404, disclosing the install path and OS user to anyone who
|
|
17
|
+
* could reach the dashboard (CWE-209). 5xx was already masked; the leak
|
|
18
|
+
* was in the 4xx branch, where echoing err.message is otherwise useful
|
|
19
|
+
* (body-parser's "request entity too large", Zod messages, etc.).
|
|
20
|
+
*
|
|
21
|
+
* Identify system errors by the shape Node gives them — a string `code`
|
|
22
|
+
* plus `syscall` — rather than by matching path-ish text, which would
|
|
23
|
+
* miss cases and mangle legitimate messages.
|
|
24
|
+
*/
|
|
25
|
+
const isSystemError = typeof err?.code === 'string' && typeof err?.syscall === 'string';
|
|
26
|
+
const message = status >= 500 || isSystemError
|
|
27
|
+
? status >= 500
|
|
28
|
+
? 'Internal server error'
|
|
29
|
+
: 'Not found'
|
|
30
|
+
: (err.message ?? 'Unknown error');
|
|
13
31
|
logger.error(`Request error: ${err.message}`, { status, stack: err.stack });
|
|
14
32
|
res.status(status).json({
|
|
15
33
|
error: message,
|
package/dist/otel/mapper.js
CHANGED
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
//
|
|
10
10
|
// We intentionally do not depend on @opentelemetry/* — the exporter
|
|
11
11
|
// serializes these plain objects, and the payload shape is the OTLP spec.
|
|
12
|
+
import { PKG_VERSION } from '../config/defaults.js';
|
|
12
13
|
// OTel SpanKind enum values (from opentelemetry-proto/trace/v1/trace.proto).
|
|
13
14
|
const KIND_MAP = {
|
|
14
15
|
INTERNAL: 1,
|
|
@@ -145,7 +146,7 @@ export function buildExportPayload(traces, serviceName) {
|
|
|
145
146
|
{ key: 'service.name', value: { stringValue: serviceName } },
|
|
146
147
|
{ key: 'telemetry.sdk.name', value: { stringValue: 'iris-mcp' } },
|
|
147
148
|
{ key: 'telemetry.sdk.language', value: { stringValue: 'nodejs' } },
|
|
148
|
-
{ key: 'telemetry.sdk.version', value: { stringValue:
|
|
149
|
+
{ key: 'telemetry.sdk.version', value: { stringValue: PKG_VERSION } },
|
|
149
150
|
],
|
|
150
151
|
};
|
|
151
152
|
const spans = [];
|