@iris-eval/mcp-server 0.4.4 → 0.4.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +55 -1
- package/dist/audit-log-reader.js +3 -3
- package/dist/config/defaults.d.ts +1 -0
- package/dist/config/defaults.js +6 -3
- package/dist/config/index.d.ts +1 -0
- package/dist/config/index.js +14 -5
- package/dist/custom-rule-store.js +190 -35
- package/dist/dashboard/assets/index-ChcHJDDJ.js +10 -0
- package/dist/dashboard/index.html +1 -1
- package/dist/dashboard/routes/rules.js +1 -1
- package/dist/dashboard/server.js +60 -5
- package/dist/dashboard/validation.d.ts +36 -62
- package/dist/eval/citation-verify/resolve.js +101 -15
- package/dist/eval/engine.js +25 -10
- package/dist/eval/rules/config-keys.d.ts +14 -0
- package/dist/eval/rules/config-keys.js +43 -0
- package/dist/eval/rules/custom.js +43 -14
- package/dist/eval/rules/regex-budget.d.ts +5 -0
- package/dist/eval/rules/regex-budget.js +0 -0
- package/dist/eval/rules/relevance.d.ts +1 -0
- package/dist/eval/rules/relevance.js +3 -1
- package/dist/eval/rules/safety.d.ts +5 -0
- package/dist/eval/rules/safety.js +43 -5
- package/dist/index.js +13 -2
- package/dist/middleware/error-handler.js +19 -1
- package/dist/middleware/rebinding-guard.d.ts +21 -0
- package/dist/middleware/rebinding-guard.js +77 -0
- package/dist/otel/mapper.js +2 -1
- package/dist/preferences.d.ts +43 -93
- package/dist/preferences.js +5 -10
- package/dist/storage/migrations/005-normalize-created-at.d.ts +3 -0
- package/dist/storage/migrations/005-normalize-created-at.js +34 -0
- package/dist/storage/migrations/index.js +8 -1
- package/dist/storage/sqlite-adapter.js +28 -4
- package/dist/tools/deploy-rule.js +2 -2
- package/dist/tools/evaluate-output.js +1 -1
- package/dist/tools/log-trace.js +3 -3
- package/dist/transport/http.js +68 -4
- package/dist/types/config.d.ts +7 -0
- package/dist/utils/iris-home.d.ts +1 -0
- package/dist/utils/iris-home.js +21 -0
- package/dist/utils/write-atomic.d.ts +1 -0
- package/dist/utils/write-atomic.js +64 -0
- package/package.json +10 -5
- package/server.json +2 -2
- package/dist/dashboard/assets/index-DNflCqmJ.js +0 -12
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
6
6
|
<link rel="icon" type="image/svg+xml" href="/favicon.svg" />
|
|
7
7
|
<title>Iris — Agent Eval & Observability</title>
|
|
8
|
-
<script type="module" crossorigin src="/assets/index-
|
|
8
|
+
<script type="module" crossorigin src="/assets/index-ChcHJDDJ.js"></script>
|
|
9
9
|
<link rel="stylesheet" crossorigin href="/assets/index-B4Aw6ozt.css">
|
|
10
10
|
</head>
|
|
11
11
|
<body>
|
|
@@ -16,7 +16,7 @@ const RuleTypeSchema = z.enum([
|
|
|
16
16
|
const DefinitionSchema = z.object({
|
|
17
17
|
name: z.string().min(1).max(80),
|
|
18
18
|
type: RuleTypeSchema,
|
|
19
|
-
config: z.record(z.unknown()),
|
|
19
|
+
config: z.record(z.string(), z.unknown()),
|
|
20
20
|
weight: z.number().positive().optional(),
|
|
21
21
|
});
|
|
22
22
|
const DeploySchema = z.object({
|
package/dist/dashboard/server.js
CHANGED
|
@@ -8,6 +8,7 @@ import { createCorsMiddleware } from '../middleware/cors.js';
|
|
|
8
8
|
import { createErrorHandler } from '../middleware/error-handler.js';
|
|
9
9
|
import { createApiRateLimiter } from '../middleware/rate-limit.js';
|
|
10
10
|
import { createTenantMiddleware } from '../middleware/tenant.js';
|
|
11
|
+
import { createRebindingGuard, isLoopbackHost } from '../middleware/rebinding-guard.js';
|
|
11
12
|
import { registerTraceRoutes } from './routes/traces.js';
|
|
12
13
|
import { registerSummaryRoutes } from './routes/summary.js';
|
|
13
14
|
import { registerEvaluationRoutes } from './routes/evaluations.js';
|
|
@@ -41,6 +42,18 @@ export function createDashboardServer(storage, config, logger, options) {
|
|
|
41
42
|
}));
|
|
42
43
|
// Body parser with size limit
|
|
43
44
|
app.use(express.json({ limit: config.security.requestSizeLimit }));
|
|
45
|
+
/*
|
|
46
|
+
* DNS-rebinding guard BEFORE anything that reads or writes state. CORS
|
|
47
|
+
* runs after it and only decorates responses the guard already allowed —
|
|
48
|
+
* on its own CORS cannot stop a rebound page, because the write executes
|
|
49
|
+
* before the browser withholds the reply.
|
|
50
|
+
*/
|
|
51
|
+
let boundPort;
|
|
52
|
+
app.use(createRebindingGuard({
|
|
53
|
+
port: () => boundPort ?? config.dashboard.port,
|
|
54
|
+
host: config.dashboard.host,
|
|
55
|
+
allowedOrigins: config.security.allowedOrigins,
|
|
56
|
+
}));
|
|
44
57
|
// CORS
|
|
45
58
|
app.use(createCorsMiddleware(config.security.allowedOrigins));
|
|
46
59
|
// Authentication
|
|
@@ -72,23 +85,65 @@ export function createDashboardServer(storage, config, logger, options) {
|
|
|
72
85
|
// when no custom rule store is provided (read-only access).
|
|
73
86
|
registerAuditRoutes(router, options?.customRuleStore);
|
|
74
87
|
app.use('/api/v1', router);
|
|
75
|
-
// Serve static dashboard files if built (rate limited)
|
|
88
|
+
// Serve static dashboard files if built (rate limited).
|
|
89
|
+
//
|
|
90
|
+
// Gate on index.html, not on the directory: `npm run build` compiles the
|
|
91
|
+
// dashboard SERVER into dist/dashboard (server.js, routes/) without the UI
|
|
92
|
+
// bundle, which is built separately by `cd dashboard && npm run build`. The
|
|
93
|
+
// directory therefore exists while index.html does not, so the SPA fallback
|
|
94
|
+
// was registered and every unmatched route hit res.sendFile on a missing
|
|
95
|
+
// file. The resulting ENOENT carries an absolute path, and the error
|
|
96
|
+
// handler returned it verbatim to the client:
|
|
97
|
+
// {"error":"ENOENT: ... stat 'C:\\...\\dist\\dashboard\\index.html'"}
|
|
98
|
+
// — leaking the install path (and the OS user) to anyone who can reach the
|
|
99
|
+
// dashboard. Without the UI built there is nothing to fall back TO, so the
|
|
100
|
+
// route simply should not exist, and unmatched paths get Express's own 404.
|
|
76
101
|
const currentDir = dirname(fileURLToPath(import.meta.url));
|
|
77
102
|
const staticDir = join(currentDir, '..', '..', 'dist', 'dashboard');
|
|
78
|
-
|
|
103
|
+
const indexHtml = join(staticDir, 'index.html');
|
|
104
|
+
if (existsSync(indexHtml)) {
|
|
79
105
|
app.use(createApiRateLimiter(config));
|
|
80
106
|
app.use(express.static(staticDir));
|
|
81
107
|
app.get('/{*path}', (_req, res) => {
|
|
82
|
-
res.sendFile(
|
|
108
|
+
res.sendFile(indexHtml);
|
|
83
109
|
});
|
|
84
110
|
}
|
|
111
|
+
else {
|
|
112
|
+
// Without this warning the server logs "Dashboard available at ..."
|
|
113
|
+
// while every page request 404s — an npm install always ships the
|
|
114
|
+
// bundle, so this only bites from-source runs, but when it bites the
|
|
115
|
+
// failure is opaque (before this line existed, a UI-less checkout
|
|
116
|
+
// failed the entire E2E suite with nothing but element-not-found
|
|
117
|
+
// timeouts).
|
|
118
|
+
logger.warn(`Dashboard UI bundle not found at ${indexHtml} — serving API only. ` +
|
|
119
|
+
`Build it with: cd dashboard && npm run build`);
|
|
120
|
+
}
|
|
85
121
|
// Error handler (must be last)
|
|
86
122
|
app.use(createErrorHandler(logger));
|
|
87
123
|
return {
|
|
88
124
|
app,
|
|
89
125
|
start() {
|
|
90
|
-
|
|
91
|
-
|
|
126
|
+
/*
|
|
127
|
+
* Bind to config.dashboard.host (loopback by default). Omitting the
|
|
128
|
+
* host argument makes Node listen on 0.0.0.0 AND [::], which put an
|
|
129
|
+
* unauthenticated API — full trace history plus rule deploy/delete —
|
|
130
|
+
* on every interface. That happened silently whenever `--transport
|
|
131
|
+
* http` started the dashboard implicitly, so binding the MCP
|
|
132
|
+
* transport to loopback still left a wide-open second server.
|
|
133
|
+
*/
|
|
134
|
+
const server = app.listen(config.dashboard.port, config.dashboard.host, () => {
|
|
135
|
+
// Record the port actually bound so the rebinding guard builds its
|
|
136
|
+
// allowlist from it rather than from a configured 0.
|
|
137
|
+
const addr = server.address();
|
|
138
|
+
if (typeof addr === 'object' && addr)
|
|
139
|
+
boundPort = addr.port;
|
|
140
|
+
const shown = isLoopbackHost(config.dashboard.host) ? 'localhost' : config.dashboard.host;
|
|
141
|
+
logger.info(`Dashboard available at http://${shown}:${boundPort ?? config.dashboard.port}`);
|
|
142
|
+
if (!isLoopbackHost(config.dashboard.host) && !config.security.apiKey) {
|
|
143
|
+
logger.warn(`Dashboard is bound to ${config.dashboard.host} with NO api key — the full trace ` +
|
|
144
|
+
`history and rule management are reachable by anyone who can route to this host. ` +
|
|
145
|
+
`Set --api-key / IRIS_API_KEY, or bind to 127.0.0.1.`);
|
|
146
|
+
}
|
|
92
147
|
});
|
|
93
148
|
/*
|
|
94
149
|
* F-006: surface listen() errors instead of swallowing them.
|
|
@@ -4,72 +4,46 @@ export declare const traceQuerySchema: z.ZodObject<{
|
|
|
4
4
|
framework: z.ZodOptional<z.ZodString>;
|
|
5
5
|
since: z.ZodOptional<z.ZodString>;
|
|
6
6
|
until: z.ZodOptional<z.ZodString>;
|
|
7
|
-
limit: z.ZodDefault<z.
|
|
8
|
-
offset: z.ZodDefault<z.
|
|
9
|
-
sort_by: z.ZodDefault<z.ZodEnum<
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
until?: string | undefined;
|
|
20
|
-
}, {
|
|
21
|
-
agent_name?: string | undefined;
|
|
22
|
-
framework?: string | undefined;
|
|
23
|
-
since?: string | undefined;
|
|
24
|
-
until?: string | undefined;
|
|
25
|
-
limit?: number | undefined;
|
|
26
|
-
offset?: number | undefined;
|
|
27
|
-
sort_by?: "timestamp" | "latency_ms" | "cost_usd" | undefined;
|
|
28
|
-
sort_order?: "asc" | "desc" | undefined;
|
|
29
|
-
}>;
|
|
7
|
+
limit: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
8
|
+
offset: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
9
|
+
sort_by: z.ZodDefault<z.ZodEnum<{
|
|
10
|
+
timestamp: "timestamp";
|
|
11
|
+
latency_ms: "latency_ms";
|
|
12
|
+
cost_usd: "cost_usd";
|
|
13
|
+
}>>;
|
|
14
|
+
sort_order: z.ZodDefault<z.ZodEnum<{
|
|
15
|
+
asc: "asc";
|
|
16
|
+
desc: "desc";
|
|
17
|
+
}>>;
|
|
18
|
+
}, z.core.$strip>;
|
|
30
19
|
export declare const evalQuerySchema: z.ZodObject<{
|
|
31
20
|
eval_type: z.ZodOptional<z.ZodString>;
|
|
32
|
-
passed: z.ZodOptional<z.
|
|
21
|
+
passed: z.ZodOptional<z.ZodPipe<z.ZodEnum<{
|
|
22
|
+
true: "true";
|
|
23
|
+
false: "false";
|
|
24
|
+
}>, z.ZodTransform<boolean, "true" | "false">>>;
|
|
33
25
|
since: z.ZodOptional<z.ZodString>;
|
|
34
26
|
until: z.ZodOptional<z.ZodString>;
|
|
35
|
-
limit: z.ZodDefault<z.
|
|
36
|
-
offset: z.ZodDefault<z.
|
|
37
|
-
},
|
|
38
|
-
limit: number;
|
|
39
|
-
offset: number;
|
|
40
|
-
since?: string | undefined;
|
|
41
|
-
until?: string | undefined;
|
|
42
|
-
eval_type?: string | undefined;
|
|
43
|
-
passed?: boolean | undefined;
|
|
44
|
-
}, {
|
|
45
|
-
since?: string | undefined;
|
|
46
|
-
until?: string | undefined;
|
|
47
|
-
limit?: number | undefined;
|
|
48
|
-
offset?: number | undefined;
|
|
49
|
-
eval_type?: string | undefined;
|
|
50
|
-
passed?: "true" | "false" | undefined;
|
|
51
|
-
}>;
|
|
27
|
+
limit: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
28
|
+
offset: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
29
|
+
}, z.core.$strip>;
|
|
52
30
|
export declare const summaryQuerySchema: z.ZodObject<{
|
|
53
|
-
hours: z.ZodDefault<z.
|
|
54
|
-
},
|
|
55
|
-
hours: number;
|
|
56
|
-
}, {
|
|
57
|
-
hours?: number | undefined;
|
|
58
|
-
}>;
|
|
31
|
+
hours: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
32
|
+
}, z.core.$strip>;
|
|
59
33
|
export declare const evalStatsPeriodSchema: z.ZodObject<{
|
|
60
|
-
period: z.ZodDefault<z.ZodEnum<
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
}
|
|
34
|
+
period: z.ZodDefault<z.ZodEnum<{
|
|
35
|
+
"24h": "24h";
|
|
36
|
+
"7d": "7d";
|
|
37
|
+
"30d": "30d";
|
|
38
|
+
all: "all";
|
|
39
|
+
}>>;
|
|
40
|
+
}, z.core.$strip>;
|
|
66
41
|
export declare const evalStatsFailuresSchema: z.ZodObject<{
|
|
67
|
-
period: z.ZodDefault<z.ZodEnum<
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
}
|
|
73
|
-
limit
|
|
74
|
-
|
|
75
|
-
}>;
|
|
42
|
+
period: z.ZodDefault<z.ZodEnum<{
|
|
43
|
+
"24h": "24h";
|
|
44
|
+
"7d": "7d";
|
|
45
|
+
"30d": "30d";
|
|
46
|
+
all: "all";
|
|
47
|
+
}>>;
|
|
48
|
+
limit: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
49
|
+
}, z.core.$strip>;
|
|
@@ -6,7 +6,10 @@
|
|
|
6
6
|
// Defense layers (in order):
|
|
7
7
|
// 1. Scheme allowlist — http/https only; refuse file:/javascript:/etc.
|
|
8
8
|
// 2. SSRF host check — refuse localhost, link-local, private ranges,
|
|
9
|
-
// and cloud metadata (AWS/GCP/Azure/DigitalOcean) IP literals.
|
|
9
|
+
// and cloud metadata (AWS/GCP/Azure/DigitalOcean) IP literals. IPv6
|
|
10
|
+
// literals are de-bracketed and canonicalized (incl. IPv4-mapped
|
|
11
|
+
// forms) before classification so `[::1]`, `[fd00::1]`, and
|
|
12
|
+
// `[::ffff:169.254.169.254]` cannot slip past the ^-anchored checks.
|
|
10
13
|
// 3. DNS pre-resolve — every public hostname is resolved via
|
|
11
14
|
// dns.lookup({all:true}) and EVERY returned IP is re-checked against
|
|
12
15
|
// the IP blocklist. Defeats DNS-rebinding via public records pointing
|
|
@@ -53,12 +56,6 @@ const BLOCKED_IPV4 = [
|
|
|
53
56
|
// This-network
|
|
54
57
|
/^0\./,
|
|
55
58
|
];
|
|
56
|
-
const BLOCKED_IPV6 = [
|
|
57
|
-
/^::1$/, // localhost
|
|
58
|
-
/^fc|^fd/i, // unique local
|
|
59
|
-
/^fe80/i, // link-local
|
|
60
|
-
/^::ffff:127\./i, // IPv4-mapped localhost
|
|
61
|
-
];
|
|
62
59
|
const BLOCKED_HOST_SUBSTRINGS = ['localhost', 'internal', '.local', 'metadata.google', 'metadata.azure'];
|
|
63
60
|
function isIpv4(host) {
|
|
64
61
|
return /^\d{1,3}(\.\d{1,3}){3}$/.test(host);
|
|
@@ -66,23 +63,112 @@ function isIpv4(host) {
|
|
|
66
63
|
function isIpv6(host) {
|
|
67
64
|
return host.includes(':');
|
|
68
65
|
}
|
|
66
|
+
// WHATWG URL parsing leaves IPv6 literals bracketed:
|
|
67
|
+
// `new URL('http://[::1]/').hostname === '[::1]'`. Every historical
|
|
68
|
+
// BLOCKED_IPV6 entry was `^`-anchored (`/^::1$/`, `/^fe80/`, …), so the
|
|
69
|
+
// leading `[` made ALL of them silently fail to match — the entire IPv6
|
|
70
|
+
// SSRF guard was inert for direct address literals (loopback, link-local,
|
|
71
|
+
// unique-local, and IPv4-mapped metadata all passed as "safe"). Strip the
|
|
72
|
+
// brackets before any IPv6 classification.
|
|
73
|
+
function stripIpv6Brackets(host) {
|
|
74
|
+
return host.length > 1 && host.startsWith('[') && host.endsWith(']')
|
|
75
|
+
? host.slice(1, -1)
|
|
76
|
+
: host;
|
|
77
|
+
}
|
|
78
|
+
// Expand a compressed / embedded-IPv4 IPv6 literal to exactly 8 zero-padded
|
|
79
|
+
// hextets. Returns null when `addr` is not a syntactically valid IPv6 literal.
|
|
80
|
+
// Canonicalizing to full form makes prefix classification reliable regardless
|
|
81
|
+
// of how the address was serialized (`::1`, `0:0:...:1`, `::ffff:a9fe:a9fe`).
|
|
82
|
+
function expandIpv6(addr) {
|
|
83
|
+
let a = addr.toLowerCase();
|
|
84
|
+
const zone = a.indexOf('%');
|
|
85
|
+
if (zone !== -1)
|
|
86
|
+
a = a.slice(0, zone); // drop scope/zone id
|
|
87
|
+
// Fold a trailing embedded IPv4 (`::ffff:1.2.3.4`, `::1.2.3.4`) into two hextets.
|
|
88
|
+
const v4 = a.match(/^(.*:)(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/);
|
|
89
|
+
if (v4) {
|
|
90
|
+
const octets = [v4[2], v4[3], v4[4], v4[5]].map(Number);
|
|
91
|
+
if (octets.some((n) => n > 255))
|
|
92
|
+
return null;
|
|
93
|
+
const hi = octets[0] * 256 + octets[1];
|
|
94
|
+
const lo = octets[2] * 256 + octets[3];
|
|
95
|
+
a = `${v4[1]}${hi.toString(16)}:${lo.toString(16)}`;
|
|
96
|
+
}
|
|
97
|
+
const halves = a.split('::');
|
|
98
|
+
if (halves.length > 2)
|
|
99
|
+
return null;
|
|
100
|
+
const head = halves[0] ? halves[0].split(':') : [];
|
|
101
|
+
const tail = halves.length === 2 && halves[1] ? halves[1].split(':') : [];
|
|
102
|
+
let groups;
|
|
103
|
+
if (halves.length === 2) {
|
|
104
|
+
const missing = 8 - head.length - tail.length;
|
|
105
|
+
if (missing < 0)
|
|
106
|
+
return null;
|
|
107
|
+
groups = [...head, ...Array(missing).fill('0'), ...tail];
|
|
108
|
+
}
|
|
109
|
+
else {
|
|
110
|
+
groups = head;
|
|
111
|
+
}
|
|
112
|
+
if (groups.length !== 8)
|
|
113
|
+
return null;
|
|
114
|
+
const out = [];
|
|
115
|
+
for (const g of groups) {
|
|
116
|
+
if (!/^[0-9a-f]{1,4}$/.test(g))
|
|
117
|
+
return null;
|
|
118
|
+
out.push(g.padStart(4, '0'));
|
|
119
|
+
}
|
|
120
|
+
return out;
|
|
121
|
+
}
|
|
122
|
+
function ipv4FromHextets(h6, h7) {
|
|
123
|
+
const hi = parseInt(h6, 16);
|
|
124
|
+
const lo = parseInt(h7, 16);
|
|
125
|
+
return `${Math.floor(hi / 256)}.${hi % 256}.${Math.floor(lo / 256)}.${lo % 256}`;
|
|
126
|
+
}
|
|
127
|
+
// True when an IPv6 literal resolves to a range we refuse: unspecified,
|
|
128
|
+
// loopback, link-local, unique-local, or an embedded IPv4 that itself hits
|
|
129
|
+
// the IPv4 blocklist (IPv4-mapped `::ffff:a.b.c.d` reaches the v4 endpoint on
|
|
130
|
+
// dual-stack hosts — e.g. `::ffff:169.254.169.254` == AWS IMDS). Fails closed
|
|
131
|
+
// on any colon-bearing host that does not parse as valid IPv6.
|
|
132
|
+
function isBlockedIpv6(addr) {
|
|
133
|
+
const g = expandIpv6(addr);
|
|
134
|
+
if (!g)
|
|
135
|
+
return true; // unparseable but IPv6-shaped (has a colon) — refuse
|
|
136
|
+
if (g.every((h) => h === '0000'))
|
|
137
|
+
return true; // :: unspecified
|
|
138
|
+
if (g.slice(0, 7).every((h) => h === '0000') && g[7] === '0001')
|
|
139
|
+
return true; // ::1 loopback
|
|
140
|
+
const first = g[0];
|
|
141
|
+
// fe80::/10 link-local (fe80–febf)
|
|
142
|
+
if (first === 'fe80' || /^fe[89ab]/.test(first))
|
|
143
|
+
return true;
|
|
144
|
+
// fc00::/7 unique-local (fc.. / fd..)
|
|
145
|
+
if (first.startsWith('fc') || first.startsWith('fd'))
|
|
146
|
+
return true;
|
|
147
|
+
// IPv4-mapped ::ffff:a.b.c.d and IPv4-compatible ::a.b.c.d (deprecated)
|
|
148
|
+
const mapped = g.slice(0, 5).every((h) => h === '0000') && g[5] === 'ffff';
|
|
149
|
+
const compat = g.slice(0, 6).every((h) => h === '0000') && !(g[6] === '0000' && g[7] === '0000');
|
|
150
|
+
if (mapped || compat) {
|
|
151
|
+
const embedded = ipv4FromHextets(g[6], g[7]);
|
|
152
|
+
return BLOCKED_IPV4.some((re) => re.test(embedded));
|
|
153
|
+
}
|
|
154
|
+
return false;
|
|
155
|
+
}
|
|
69
156
|
export function isSafeHost(host) {
|
|
70
|
-
const
|
|
157
|
+
const bare = stripIpv6Brackets(host);
|
|
158
|
+
const hostLower = bare.toLowerCase();
|
|
71
159
|
for (const sub of BLOCKED_HOST_SUBSTRINGS) {
|
|
72
160
|
if (hostLower === sub || hostLower.endsWith(sub))
|
|
73
161
|
return false;
|
|
74
162
|
}
|
|
75
|
-
if (isIpv4(
|
|
163
|
+
if (isIpv4(bare)) {
|
|
76
164
|
for (const re of BLOCKED_IPV4) {
|
|
77
|
-
if (re.test(
|
|
165
|
+
if (re.test(bare))
|
|
78
166
|
return false;
|
|
79
167
|
}
|
|
80
168
|
}
|
|
81
|
-
if (isIpv6(
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
return false;
|
|
85
|
-
}
|
|
169
|
+
if (isIpv6(bare)) {
|
|
170
|
+
if (isBlockedIpv6(bare))
|
|
171
|
+
return false;
|
|
86
172
|
}
|
|
87
173
|
return true;
|
|
88
174
|
}
|
package/dist/eval/engine.js
CHANGED
|
@@ -21,16 +21,31 @@ export class EvalEngine {
|
|
|
21
21
|
customConfig: { ...this.ruleThresholds, ...context.customConfig },
|
|
22
22
|
};
|
|
23
23
|
}
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
24
|
+
/*
|
|
25
|
+
* Inline custom_rules are ADDITIVE, which is what evaluate_output's
|
|
26
|
+
* description promises in two places: "fires REGARDLESS of eval_type"
|
|
27
|
+
* and "otherwise both your rules AND the eval_type bundle run together".
|
|
28
|
+
*
|
|
29
|
+
* The old branch did neither. `evalType === 'custom' && customRules`
|
|
30
|
+
* meant:
|
|
31
|
+
* - evaluate('safety', ctx, [myRule]) silently DISCARDED myRule and
|
|
32
|
+
* returned a plausible score that never applied it. An agent
|
|
33
|
+
* following the tool description got a wrong answer with no warning.
|
|
34
|
+
* - evaluate('custom', ctx, [myRule]) replaced the rule list entirely,
|
|
35
|
+
* EVICTING every rule the user had deployed and which the server
|
|
36
|
+
* registers at boot. Passing one ad-hoc rule disabled their whole
|
|
37
|
+
* library for that call.
|
|
38
|
+
*
|
|
39
|
+
* getRulesForType('custom') is [] (rules/index.ts), so eval_type="custom"
|
|
40
|
+
* still runs no built-in bundle — the documented "ONLY these" behaviour
|
|
41
|
+
* holds. What it now also includes is the caller's own deployed rules,
|
|
42
|
+
* which is the least surprising reading of having deployed them.
|
|
43
|
+
*/
|
|
44
|
+
const rules = [
|
|
45
|
+
...getRulesForType(evalType),
|
|
46
|
+
...(this.additionalRules.get(evalType) ?? []),
|
|
47
|
+
...(customRules ?? []).map((def) => createCustomRule(def)),
|
|
48
|
+
];
|
|
34
49
|
if (rules.length === 0) {
|
|
35
50
|
return {
|
|
36
51
|
id: generateEvalId(),
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export declare const CUSTOM_RULE_CONFIG_KEYS: {
|
|
2
|
+
readonly min_length: readonly ["min_length", "length", "min"];
|
|
3
|
+
readonly max_length: readonly ["max_length", "length", "max"];
|
|
4
|
+
readonly cost_threshold: readonly ["max_cost", "max_usd"];
|
|
5
|
+
};
|
|
6
|
+
/** First key is the canonical one to document and teach. */
|
|
7
|
+
export declare function canonicalKey(type: keyof typeof CUSTOM_RULE_CONFIG_KEYS): string;
|
|
8
|
+
/**
|
|
9
|
+
* Read the first defined numeric value among a rule type's accepted keys.
|
|
10
|
+
* Returns undefined when none is present, so callers can raise a config error.
|
|
11
|
+
*/
|
|
12
|
+
export declare function readNumericConfig(config: Record<string, unknown>, type: keyof typeof CUSTOM_RULE_CONFIG_KEYS): number | undefined;
|
|
13
|
+
/** Human-readable "config.a (or config.b)" for error messages. */
|
|
14
|
+
export declare function describeKeys(type: keyof typeof CUSTOM_RULE_CONFIG_KEYS): string;
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Canonical config keys for custom rule types — the SINGLE source of truth.
|
|
3
|
+
*
|
|
4
|
+
* This module exists because the keys drifted across three surfaces and
|
|
5
|
+
* shipped broken: the evaluator read `config.min_length`, while the
|
|
6
|
+
* `deploy_rule` tool description (the text an LLM agent reads to construct
|
|
7
|
+
* its call) told agents to send `config.min`, and `docs/api-reference.md`
|
|
8
|
+
* showed `{ "min": 40 }`. Deploy-time validation accepted any object, so a
|
|
9
|
+
* rule built from our own documentation deployed cleanly and then failed on
|
|
10
|
+
* every evaluation forever.
|
|
11
|
+
*
|
|
12
|
+
* The evaluator, the deploy-time validator, and the tool description now all
|
|
13
|
+
* derive from this map, so a key cannot be correct in one place and wrong in
|
|
14
|
+
* another. The first entry of each list is CANONICAL (what docs should
|
|
15
|
+
* teach); the rest are accepted aliases kept for compatibility with configs
|
|
16
|
+
* created from earlier, incorrect documentation.
|
|
17
|
+
*/
|
|
18
|
+
export const CUSTOM_RULE_CONFIG_KEYS = {
|
|
19
|
+
min_length: ['min_length', 'length', 'min'],
|
|
20
|
+
max_length: ['max_length', 'length', 'max'],
|
|
21
|
+
cost_threshold: ['max_cost', 'max_usd'],
|
|
22
|
+
};
|
|
23
|
+
/** First key is the canonical one to document and teach. */
|
|
24
|
+
export function canonicalKey(type) {
|
|
25
|
+
return CUSTOM_RULE_CONFIG_KEYS[type][0];
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Read the first defined numeric value among a rule type's accepted keys.
|
|
29
|
+
* Returns undefined when none is present, so callers can raise a config error.
|
|
30
|
+
*/
|
|
31
|
+
export function readNumericConfig(config, type) {
|
|
32
|
+
for (const key of CUSTOM_RULE_CONFIG_KEYS[type]) {
|
|
33
|
+
const value = config[key];
|
|
34
|
+
if (typeof value === 'number' && Number.isFinite(value))
|
|
35
|
+
return value;
|
|
36
|
+
}
|
|
37
|
+
return undefined;
|
|
38
|
+
}
|
|
39
|
+
/** Human-readable "config.a (or config.b)" for error messages. */
|
|
40
|
+
export function describeKeys(type) {
|
|
41
|
+
const [first, ...rest] = CUSTOM_RULE_CONFIG_KEYS[type];
|
|
42
|
+
return rest.length ? `config.${first} (aliases: ${rest.map((k) => `config.${k}`).join(', ')})` : `config.${first}`;
|
|
43
|
+
}
|
|
@@ -1,7 +1,30 @@
|
|
|
1
1
|
import isSafeRegex from 'safe-regex2';
|
|
2
|
+
import { readNumericConfig, describeKeys } from './config-keys.js';
|
|
2
3
|
const MAX_PATTERN_LENGTH = 1000;
|
|
4
|
+
// A rule whose CONFIG is invalid has not evaluated the output — it could not
|
|
5
|
+
// run at all. Returning `passed:false, score:0` for that case conflates "your
|
|
6
|
+
// agent produced bad output" with "this rule is broken", and because the
|
|
7
|
+
// engine weights every non-skipped result, one misconfigured rule silently
|
|
8
|
+
// deflates every eval score for as long as it stays deployed (and gives no
|
|
9
|
+
// hint which of the two happened). Mark it skipped: the engine already
|
|
10
|
+
// excludes skipped rules from the weighted average and surfaces skipReason,
|
|
11
|
+
// the same contract `expected_coverage` uses when no expected output exists.
|
|
12
|
+
//
|
|
13
|
+
// Deploy-time validation in custom-rule-store.ts now rejects these configs
|
|
14
|
+
// outright; this path remains the safety net for rules already persisted in
|
|
15
|
+
// a user's ~/.iris/custom-rules.json from before that validation existed.
|
|
16
|
+
function configError(definition, message) {
|
|
17
|
+
return {
|
|
18
|
+
ruleName: definition.name,
|
|
19
|
+
passed: false,
|
|
20
|
+
score: 0,
|
|
21
|
+
message,
|
|
22
|
+
skipped: true,
|
|
23
|
+
skipReason: message,
|
|
24
|
+
};
|
|
25
|
+
}
|
|
3
26
|
function safeRegexResult(definition, message) {
|
|
4
|
-
return
|
|
27
|
+
return configError(definition, message);
|
|
5
28
|
}
|
|
6
29
|
function compileRegex(definition) {
|
|
7
30
|
let patternStr = definition.config.pattern;
|
|
@@ -19,15 +42,21 @@ function compileRegex(definition) {
|
|
|
19
42
|
if (patternStr.length > MAX_PATTERN_LENGTH) {
|
|
20
43
|
return safeRegexResult(definition, `Regex pattern too long (${patternStr.length} > ${MAX_PATTERN_LENGTH})`);
|
|
21
44
|
}
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
45
|
+
// Syntax BEFORE safety: safe-regex2 returns false for anything it cannot
|
|
46
|
+
// parse, so checking it first reports a plainly broken pattern like `(` as
|
|
47
|
+
// "catastrophic backtracking" — sending the author hunting a performance
|
|
48
|
+
// problem they do not have instead of the typo they do.
|
|
49
|
+
let compiled;
|
|
25
50
|
try {
|
|
26
|
-
|
|
51
|
+
compiled = new RegExp(patternStr, flags);
|
|
27
52
|
}
|
|
28
53
|
catch (e) {
|
|
29
54
|
return safeRegexResult(definition, `Invalid regex syntax: ${e instanceof Error ? e.message : 'unknown error'}`);
|
|
30
55
|
}
|
|
56
|
+
if (!isSafeRegex(patternStr)) {
|
|
57
|
+
return safeRegexResult(definition, 'Regex pattern rejected: potentially unsafe (catastrophic backtracking)');
|
|
58
|
+
}
|
|
59
|
+
return compiled;
|
|
31
60
|
}
|
|
32
61
|
export function createCustomRule(definition) {
|
|
33
62
|
return {
|
|
@@ -52,17 +81,17 @@ export function createCustomRule(definition) {
|
|
|
52
81
|
return { ruleName: definition.name, passed, score: passed ? 1 : 0, message: passed ? 'Forbidden pattern not found' : 'Forbidden pattern found in output' };
|
|
53
82
|
}
|
|
54
83
|
case 'min_length': {
|
|
55
|
-
const min = (definition.config
|
|
84
|
+
const min = readNumericConfig(definition.config, 'min_length');
|
|
56
85
|
if (min == null || min <= 0) {
|
|
57
|
-
return
|
|
86
|
+
return configError(definition, `min_length rule requires ${describeKeys('min_length')} (positive number)`);
|
|
58
87
|
}
|
|
59
88
|
const passed = context.output.length >= min;
|
|
60
89
|
return { ruleName: definition.name, passed, score: passed ? 1 : context.output.length / min, message: passed ? `Length (${context.output.length}) meets minimum (${min})` : `Length (${context.output.length}) below minimum (${min})` };
|
|
61
90
|
}
|
|
62
91
|
case 'max_length': {
|
|
63
|
-
const max = (definition.config
|
|
92
|
+
const max = readNumericConfig(definition.config, 'max_length');
|
|
64
93
|
if (max == null || max <= 0) {
|
|
65
|
-
return
|
|
94
|
+
return configError(definition, `max_length rule requires ${describeKeys('max_length')} (positive number)`);
|
|
66
95
|
}
|
|
67
96
|
const passed = context.output.length <= max;
|
|
68
97
|
return { ruleName: definition.name, passed, score: passed ? 1 : max / context.output.length, message: passed ? `Length (${context.output.length}) within maximum (${max})` : `Length (${context.output.length}) exceeds maximum (${max})` };
|
|
@@ -70,7 +99,7 @@ export function createCustomRule(definition) {
|
|
|
70
99
|
case 'contains_keywords': {
|
|
71
100
|
const keywords = definition.config.keywords;
|
|
72
101
|
if (!keywords || !Array.isArray(keywords) || keywords.length === 0) {
|
|
73
|
-
return
|
|
102
|
+
return configError(definition, 'contains_keywords rule requires config.keywords (non-empty string array)');
|
|
74
103
|
}
|
|
75
104
|
const lower = context.output.toLowerCase();
|
|
76
105
|
const found = keywords.filter((k) => lower.includes(k.toLowerCase()));
|
|
@@ -81,7 +110,7 @@ export function createCustomRule(definition) {
|
|
|
81
110
|
case 'excludes_keywords': {
|
|
82
111
|
const keywords = definition.config.keywords;
|
|
83
112
|
if (!keywords || !Array.isArray(keywords) || keywords.length === 0) {
|
|
84
|
-
return
|
|
113
|
+
return configError(definition, 'excludes_keywords rule requires config.keywords (non-empty string array)');
|
|
85
114
|
}
|
|
86
115
|
const lower = context.output.toLowerCase();
|
|
87
116
|
const found = keywords.filter((k) => lower.includes(k.toLowerCase()));
|
|
@@ -98,16 +127,16 @@ export function createCustomRule(definition) {
|
|
|
98
127
|
}
|
|
99
128
|
}
|
|
100
129
|
case 'cost_threshold': {
|
|
101
|
-
const max = definition.config
|
|
130
|
+
const max = readNumericConfig(definition.config, 'cost_threshold');
|
|
102
131
|
if (max == null || max < 0) {
|
|
103
|
-
return
|
|
132
|
+
return configError(definition, `cost_threshold rule requires ${describeKeys('cost_threshold')} (non-negative number)`);
|
|
104
133
|
}
|
|
105
134
|
const cost = context.costUsd ?? 0;
|
|
106
135
|
const passed = cost <= max;
|
|
107
136
|
return { ruleName: definition.name, passed, score: passed ? 1 : 0, message: passed ? `Cost ($${cost}) within threshold ($${max})` : `Cost ($${cost}) exceeds threshold ($${max})` };
|
|
108
137
|
}
|
|
109
138
|
default:
|
|
110
|
-
return
|
|
139
|
+
return configError(definition, `Unknown rule type: ${definition.type}`);
|
|
111
140
|
}
|
|
112
141
|
},
|
|
113
142
|
};
|
|
Binary file
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { EvalRule } from '../../types/eval.js';
|
|
2
2
|
export declare const keywordOverlap: EvalRule;
|
|
3
|
+
export declare const HALLUCINATION_MARKERS: string[];
|
|
3
4
|
export declare const noHallucinationMarkers: EvalRule;
|
|
4
5
|
export declare const topicConsistency: EvalRule;
|
|
5
6
|
export declare const relevanceRules: EvalRule[];
|
|
@@ -28,7 +28,9 @@ export const keywordOverlap = {
|
|
|
28
28
|
};
|
|
29
29
|
},
|
|
30
30
|
};
|
|
31
|
-
|
|
31
|
+
// Exported so the claims drift test can assert .claims.json counts against
|
|
32
|
+
// the runtime truth (tests/claims-eval-rules-counts.test.ts).
|
|
33
|
+
export const HALLUCINATION_MARKERS = [
|
|
32
34
|
'as an ai',
|
|
33
35
|
'as a language model',
|
|
34
36
|
'i cannot',
|
|
@@ -1,6 +1,11 @@
|
|
|
1
1
|
import type { EvalRule } from '../../types/eval.js';
|
|
2
|
+
export declare const PII_PATTERNS: Array<{
|
|
3
|
+
name: string;
|
|
4
|
+
pattern: RegExp;
|
|
5
|
+
}>;
|
|
2
6
|
export declare const noPii: EvalRule;
|
|
3
7
|
export declare const noBlocklistWords: EvalRule;
|
|
8
|
+
export declare const INJECTION_PATTERNS: RegExp[];
|
|
4
9
|
export declare const noInjectionPatterns: EvalRule;
|
|
5
10
|
export declare const noStubOutput: EvalRule;
|
|
6
11
|
export declare const safetyRules: EvalRule[];
|