@iris-eval/mcp-server 0.4.5 → 0.5.0

package/dist/eval/failure-rank.js

@@ -0,0 +1,44 @@
+/*
+ * failure-rank — pure ranking logic for the failure-first landing list.
+ *
+ * The dashboard's default screen is a ranked list of recent failures
+ * ("what's new and bad"), not an aggregate. Ranking blends two signals:
+ *
+ *   severity — the significance classifier's 0-1 score (safety-violation
+ *              1.0 > cost-spike 0.9 > rule-collision 0.7 > normal-fail
+ *              0.5/0.4). See classifySignificance in decision-moment.ts.
+ *   recency  — exponential decay with a 24h half-life. A safety violation
+ *              from three days ago ranks below a plain fail from an hour
+ *              ago, which is the right call for a "since you last looked"
+ *              surface — old severity is history, not news.
+ *
+ * Kept as a pure module (no storage, no clock reads — `nowMs` is a
+ * parameter) so tests can pin time and assert exact orderings.
+ */
+/** Recency half-life: a failure loses half its rank weight every 24h. */
+export const FAILURE_RANK_HALF_LIFE_MS = 24 * 60 * 60 * 1000;
+/*
+ * Significance kinds that flag a moment for the failure list even when
+ * its verdict is not fail/partial. A cost spike on a passing trace is
+ * still something the builder should see on the landing screen.
+ */
+const FLAGGED_KINDS = new Set(['safety-violation', 'cost-spike']);
+/**
+ * Is this moment a failure (verdict fail/partial) or flagged
+ * (safety/cost significance regardless of verdict)?
+ */
+export function isFailureMoment(moment) {
+    if (moment.verdict === 'fail' || moment.verdict === 'partial')
+        return true;
+    return FLAGGED_KINDS.has(moment.significance.kind);
+}
+/**
+ * Rank score for a failure moment: significance × recency decay.
+ * Higher = shown first. Future timestamps (clock skew) clamp to age 0
+ * rather than inflating the score.
+ */
+export function rankFailureScore(moment, nowMs) {
+    const ageMs = Math.max(0, nowMs - new Date(moment.timestamp).getTime());
+    const recency = Math.pow(0.5, ageMs / FAILURE_RANK_HALF_LIFE_MS);
+    return moment.significance.score * recency;
+}

package/dist/eval/rules/custom.d.ts

@@ -1,2 +1,30 @@
 import type { EvalRule, CustomRuleDefinition } from '../../types/eval.js';
-export declare function createCustomRule(definition: CustomRuleDefinition): EvalRule;
+import type { RuleSeverity } from '../../types/custom-rule.js';
+/**
+ * Converts a leading inline flag group like `(?i)` or `(?im)` into a real
+ * flags argument. Node's RegExp engine does not support inline flag groups,
+ * and a user pasting `(?i)foo` from a regex tutorial would otherwise hit
+ * "Invalid group" with no clear recovery.
+ *
+ * Exported so deploy-time validation (custom-rule-store) probes the SAME
+ * pattern+flags pair the evaluator will actually run — the store used to
+ * strip the inline group but not merge its flags, probing `(?i)…` under
+ * different flags than evaluation used.
+ */
+export declare function normalizeRegexSource(patternStr: string, flags: string): {
+    pattern: string;
+    flags: string;
+};
+/**
+ * Builds a runnable EvalRule from a persisted/inline definition.
+ *
+ * `severity` comes from the DEPLOYED rule's metadata (deploy_rule / the
+ * dashboard composer). high/critical severities make the rule CRITICAL:
+ * a failing evaluation forces the overall eval to passed=false regardless
+ * of the weighted score. Before this, a rule-author could deploy a
+ * severity="critical" policy rule, watch it FAIL on a violating output,
+ * and still get passed:true (score 0.895) — severity affected nothing but
+ * dashboard sorting. Inline custom_rules (evaluate_output's per-call
+ * definitions) carry no severity and stay weight-only.
+ */
+export declare function createCustomRule(definition: CustomRuleDefinition, severity?: RuleSeverity): EvalRule;

package/dist/eval/rules/custom.js

@@ -1,5 +1,6 @@
 import isSafeRegex from 'safe-regex2';
 import { readNumericConfig, describeKeys } from './config-keys.js';
+import { sandboxedRegexTest, REGEX_MATCH_BUDGET_MS } from './regex-sandbox.js';
 const MAX_PATTERN_LENGTH = 1000;
 // A rule whose CONFIG is invalid has not evaluated the output — it could not
 // run at all. Returning `passed:false, score:0` for that case conflates "your
@@ -13,6 +14,11 @@ const MAX_PATTERN_LENGTH = 1000;
 // Deploy-time validation in custom-rule-store.ts now rejects these configs
 // outright; this path remains the safety net for rules already persisted in
 // a user's ~/.iris/custom-rules.json from before that validation existed.
+//
+// configInvalid distinguishes this skip from a legitimate one: config
+// errors depend only on the definition, never the input, so a caller that
+// holds the whole definition (the rule-preview endpoint) can reject it
+// as a 422 instead of reporting every trace as "would skip".
 function configError(definition, message) {
     return {
         ruleName: definition.name,
@@ -21,24 +27,45 @@ function configError(definition, message) {
         message,
         skipped: true,
         skipReason: message,
+        configInvalid: true,
     };
 }
 function safeRegexResult(definition, message) {
     return configError(definition, message);
 }
-function compileRegex(definition) {
-    let patternStr = definition.config.pattern;
-    let flags = definition.config.flags ?? '';
-    // Defensive UX: convert leading inline flag like `(?i)` or `(?im)` to a
-    // real flags arg. Node's RegExp engine does not support inline flag
-    // groups in older versions, and a user pasting `(?i)foo` from a regex
-    // tutorial would otherwise hit "Invalid group" with no clear recovery.
+/**
+ * Converts a leading inline flag group like `(?i)` or `(?im)` into a real
+ * flags argument. Node's RegExp engine does not support inline flag groups,
+ * and a user pasting `(?i)foo` from a regex tutorial would otherwise hit
+ * "Invalid group" with no clear recovery.
+ *
+ * Exported so deploy-time validation (custom-rule-store) probes the SAME
+ * pattern+flags pair the evaluator will actually run — the store used to
+ * strip the inline group but not merge its flags, probing `(?i)…` under
+ * different flags than evaluation used.
+ */
+export function normalizeRegexSource(patternStr, flags) {
     const inlineFlagMatch = patternStr.match(/^\(\?([imsugy]+)\)/);
     if (inlineFlagMatch) {
         const inlineFlags = inlineFlagMatch[1];
         flags = [...new Set((flags + inlineFlags).split(''))].join('');
         patternStr = patternStr.slice(inlineFlagMatch[0].length);
     }
+    return { pattern: patternStr, flags };
+}
+/*
+ * Validates a user pattern and returns the normalized {pattern, flags} pair —
+ * NOT a compiled RegExp, deliberately. The pattern is compiled here once for
+ * syntax validation (compilation does not backtrack), but matching happens in
+ * the sandbox worker (regex-sandbox.ts), which compiles its own copy. Nothing
+ * on the main thread may ever call `.test()`/`.exec()` on a user pattern: the
+ * static checks below are best-effort UX (fast rejection with a good message),
+ * not the safety boundary. safe-regex2 is star-height-only — `(a|a)*$` passes
+ * it and is exponential — and no static or probe-based check is sound in
+ * general. The sandbox's hard deadline is the boundary.
+ */
+function validateRegex(definition) {
+    const { pattern: patternStr, flags } = normalizeRegexSource(definition.config.pattern, definition.config.flags ?? '');
     if (patternStr.length > MAX_PATTERN_LENGTH) {
         return safeRegexResult(definition, `Regex pattern too long (${patternStr.length} > ${MAX_PATTERN_LENGTH})`);
     }
@@ -46,9 +73,8 @@ function compileRegex(definition) {
     // parse, so checking it first reports a plainly broken pattern like `(` as
     // "catastrophic backtracking" — sending the author hunting a performance
     // problem they do not have instead of the typo they do.
-    let compiled;
     try {
-        compiled = new RegExp(patternStr, flags);
+        new RegExp(patternStr, flags);
     }
     catch (e) {
         return safeRegexResult(definition, `Invalid regex syntax: ${e instanceof Error ? e.message : 'unknown error'}`);
@@ -56,28 +82,138 @@ function compileRegex(definition) {
     if (!isSafeRegex(patternStr)) {
         return safeRegexResult(definition, 'Regex pattern rejected: potentially unsafe (catastrophic backtracking)');
     }
-    return compiled;
+    return { pattern: patternStr, flags };
+}
+/*
+ * Budget breach is a property of pattern×input, not of the definition alone —
+ * the same pattern can be instant on one output and superlinear on the next
+ * (often one CRAFTED to stall it). So this is not configInvalid: the preview
+ * endpoint must not 422 a rule that merely met a hostile input. It follows the
+ * configError precedent instead: SKIPPED, because a rule whose match was
+ * killed mid-backtrack has not judged the output, and a skipped rule neither
+ * deflates the weighted score nor (for high/critical deployed rules) vetoes
+ * the eval on evidence it never gathered. The skipReason tells the author
+ * exactly what to fix, and the engine already surfaces it in suggestions.
+ */
+function budgetExceededResult(definition) {
+    const message = `Regex evaluation terminated: pattern exceeded the ${REGEX_MATCH_BUDGET_MS}ms matching ` +
+        `budget on this output (superlinear backtracking) and was killed in its sandbox worker. ` +
+        `The rule did NOT judge this output — a gate that must fail closed should treat ` +
+        `budgetExceeded skips as failures. Rewrite the pattern to avoid ambiguous repetition ` +
+        `— e.g. bound quantifiers (\\s{0,8} not \\s*) and remove overlapping alternatives.`;
+    return {
+        ruleName: definition.name,
+        passed: false,
+        score: 0,
+        message,
+        skipped: true,
+        skipReason: message,
+        budgetExceeded: true,
+    };
+}
+/**
+ * Per-evaluation cap on sandbox budget breaches. Each breach costs the
+ * request its budget PLUS a worker respawn (~190ms total measured), and the
+ * engine runs rules synchronously — so without a breaker, one request
+ * carrying N hostile regex rules stalls the server N × ~190ms (measured
+ * 9.3s at N=50). After this many breaches, remaining regex rules in the
+ * same evaluation skip WITHOUT running, bounding the whole request at
+ * roughly cap × 190ms regardless of rule count.
+ */
+const MAX_REGEX_BREACHES_PER_EVAL = 3;
+function circuitOpenResult(definition) {
+    const message = `Regex evaluation skipped: ${MAX_REGEX_BREACHES_PER_EVAL} earlier pattern(s) in this ` +
+        `evaluation already exhausted the ${REGEX_MATCH_BUDGET_MS}ms matching budget, so the ` +
+        `regex circuit breaker is open for the rest of this evaluation. The rule did NOT judge ` +
+        `this output — a gate that must fail closed should treat budgetExceeded skips as failures.`;
+    return {
+        ruleName: definition.name,
+        passed: false,
+        score: 0,
+        message,
+        skipped: true,
+        skipReason: message,
+        budgetExceeded: true,
+    };
+}
+/*
+ * A sandbox 'error' is NOT the author's fault and must not be reported as
+ * backtracking: it means the worker could not run the (pre-validated)
+ * pattern at all — in practice a worker that died between calls (postMessage
+ * to a terminated worker is a silent no-op). Accusing the pattern sends the
+ * author hunting a performance problem they do not have.
+ */
+function sandboxErrorResult(definition) {
+    const message = 'Regex evaluation skipped: internal sandbox error (the matching worker restarted). ' +
+        'The rule did not judge this output; the pattern itself is fine — retry the evaluation.';
+    return {
+        ruleName: definition.name,
+        passed: false,
+        score: 0,
+        message,
+        skipped: true,
+        skipReason: message,
+    };
+}
+/**
+ * Executes a validated user pattern through the sandbox with the
+ * per-evaluation circuit breaker. Shared by regex_match and regex_no_match.
+ */
+function runSandboxed(definition, pattern, flags, context) {
+    const budget = context.regexBudget;
+    if (budget && budget.breaches >= MAX_REGEX_BREACHES_PER_EVAL) {
+        return circuitOpenResult(definition);
+    }
+    const outcome = sandboxedRegexTest(pattern, flags, context.output);
+    if (outcome.kind === 'timeout') {
+        if (budget)
+            budget.breaches += 1;
+        return budgetExceededResult(definition);
+    }
+    if (outcome.kind === 'error') {
+        return sandboxErrorResult(definition);
+    }
+    return { matched: outcome.matched };
 }
-export function createCustomRule(definition) {
+/**
+ * Builds a runnable EvalRule from a persisted/inline definition.
+ *
+ * `severity` comes from the DEPLOYED rule's metadata (deploy_rule / the
+ * dashboard composer). high/critical severities make the rule CRITICAL:
+ * a failing evaluation forces the overall eval to passed=false regardless
+ * of the weighted score. Before this, a rule-author could deploy a
+ * severity="critical" policy rule, watch it FAIL on a violating output,
+ * and still get passed:true (score 0.895) — severity affected nothing but
+ * dashboard sorting. Inline custom_rules (evaluate_output's per-call
+ * definitions) carry no severity and stay weight-only.
+ */
+export function createCustomRule(definition, severity) {
     return {
         name: definition.name,
         description: `Custom rule: ${definition.name}`,
         evalType: 'custom',
         weight: definition.weight ?? 1,
+        critical: severity === 'high' || severity === 'critical',
         evaluate(context) {
             switch (definition.type) {
                 case 'regex_match': {
-                    const result = compileRegex(definition);
-                    if (!(result instanceof RegExp))
-                        return result;
-                    const passed = result.test(context.output);
+                    const validated = validateRegex(definition);
+                    if ('ruleName' in validated)
+                        return validated;
+                    const run = runSandboxed(definition, validated.pattern, validated.flags, context);
+                    if ('ruleName' in run)
+                        return run;
+                    const passed = run.matched;
                     return { ruleName: definition.name, passed, score: passed ? 1 : 0, message: passed ? 'Regex pattern matched' : 'Regex pattern did not match' };
                 }
                 case 'regex_no_match': {
-                    const result = compileRegex(definition);
-                    if (!(result instanceof RegExp))
-                        return result;
-                    const passed = !result.test(context.output);
+                    const validated = validateRegex(definition);
+                    if ('ruleName' in validated)
+                        return validated;
+                    const run = runSandboxed(definition, validated.pattern, validated.flags, context);
+                    if ('ruleName' in run)
+                        return run;
+                    const passed = !run.matched;
                     return { ruleName: definition.name, passed, score: passed ? 1 : 0, message: passed ? 'Forbidden pattern not found' : 'Forbidden pattern found in output' };
                 }
                 case 'min_length': {

package/dist/eval/rules/regex-budget.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Returns a human-readable reason when `source` shows superlinear
+ * backtracking, or null when it looks safe to deploy.
+ */
+export declare function regexBacktrackingBudgetExceeded(source: string, flags?: string): string | null;

package/dist/eval/rules/regex-budget.js

@@ -0,0 +1,100 @@
+/*
+ * Empirical backtracking probe for user-supplied regex patterns.
+ *
+ * safe-regex2 is a STATIC heuristic built on star height — it catches
+ * EXPONENTIAL blowup like `(a+)+$` and nothing else. Polynomial patterns
+ * sail through it: `a*a*a*a*a*b` is judged safe, and takes 156ms on 40
+ * characters, 237ms on 60, and effectively forever on a realistic agent
+ * output. Deployed rules are re-registered into the engine at every
+ * startup, so a pattern like that keeps wedging the server after a
+ * restart — a permanent, self-inflicted denial of service.
+ *
+ * Static analysis of backtracking is hard; actually running the pattern is
+ * not. This measures it against short adversarial payloads and rejects
+ * anything already slow at trivial sizes.
+ *
+ * The catch-22 — running an untrusted regex to find out whether it hangs —
+ * is handled twice over. First, probes escalate from a tiny payload upward
+ * and bail the moment the budget is exceeded. Second — and this is the part
+ * that actually holds — every probe executes in the sandbox worker
+ * (regex-sandbox.ts) under a hard deadline. The original version ran probes
+ * on the MAIN thread and checked Date.now() after each `.test()` returned:
+ * a synchronous call cannot be interrupted from behind, and a pattern the
+ * payload families did ignite blocked the probe itself for 43,380ms against
+ * this 50ms budget. Now the worker is terminated mid-backtrack instead.
+ *
+ * This probe remains a deploy-time UX courtesy (reject obviously dangerous
+ * patterns with a clear message before they are persisted), NOT the safety
+ * boundary. Probing depends on guessing an igniting payload, which is not
+ * possible in general — S79's fuel search failed to ignite `^(a|ab)+$` at
+ * all. The boundary is the same sandbox deadline applied at every
+ * evaluation in custom.ts.
+ */
+import { sandboxedRegexTest } from './regex-sandbox.js';
+/** Total match-execution time a candidate pattern may spend across all
+ * probes, as measured INSIDE the sandbox worker. Metering on worker-measured
+ * time (not wall-clock) matters: wall-clock includes OS scheduling, and on a
+ * busy host a 1ms match can take 60ms of wall time — the original wall-clock
+ * budget rejected perfectly ordinary patterns whenever the machine was loaded
+ * (every parallel test run reproduced it). */
+const BUDGET_MS = 50;
+/** Wall-clock ceiling per single probe call — the hang-killer, not the
+ * meter. Generous so scheduling noise can never trip it; a genuinely
+ * superlinear pattern burns through BUDGET_MS of measured time long before
+ * this fires. */
+const PROBE_WALL_DEADLINE_MS = 1000;
+const PROBE_SIZES = [16, 32, 64, 128];
+/** Appended to every probe payload to force a failed match (backtracking
+ * happens on failure). NUL beats a space here: space matches `\s` and
+ * several probe alphabets contain it, which would let the match succeed
+ * quickly instead of exploring alternatives. NOTE: this was previously a
+ * literal 0x00 byte inside the string — invisible in review and enough to
+ * make git treat the whole file as binary. Same behavior, now spelled out. */
+const TERMINATOR = '\0';
+/**
+ * Characters that tend to maximise backtracking pressure for a given
+ * pattern: the literals it mentions, plus generic filler. Feeding a
+ * pattern its own alphabet is what makes the engine explore alternatives
+ * rather than fail at the first character.
+ */
+function probeAlphabets(source) {
+    const literals = source.replace(/[^A-Za-z0-9 ._@-]/g, '');
+    const fromPattern = [...new Set(literals)].join('').slice(0, 4);
+    const alphabets = ['a', ' ', 'a.', 'ab'];
+    if (fromPattern.length > 0)
+        alphabets.unshift(fromPattern);
+    return alphabets;
+}
+/**
+ * Returns a human-readable reason when `source` shows superlinear
+ * backtracking, or null when it looks safe to deploy.
+ */
+export function regexBacktrackingBudgetExceeded(source, flags = '') {
+    try {
+        new RegExp(source, flags);
+    }
+    catch {
+        // Syntax is validated separately and reported with a better message.
+        return null;
+    }
+    let spentMs = 0;
+    for (const size of PROBE_SIZES) {
+        for (const alphabet of probeAlphabets(source)) {
+            const payload = alphabet.repeat(Math.ceil(size / alphabet.length)).slice(0, size) + TERMINATOR;
+            // The wall deadline is only the hang-killer; the budget meters on the
+            // worker-measured match duration, immune to scheduling noise.
+            const outcome = sandboxedRegexTest(source, flags, payload, PROBE_WALL_DEADLINE_MS);
+            if (outcome.kind === 'match')
+                spentMs += outcome.durationMs;
+            const elapsed = Math.round(spentMs);
+            if (outcome.kind === 'timeout' || elapsed > BUDGET_MS) {
+                return (`Regex pattern rejected: superlinear backtracking (still running after ${elapsed}ms ` +
+                    `on a ${size}-character input). safe-regex2 only catches exponential blowup, so ` +
+                    `polynomial patterns like a*a*a*a*a*b pass it while still hanging the server. ` +
+                    `Avoid adjacent unbounded quantifiers over overlapping character classes; bound them ` +
+                    `instead, e.g. \\s{0,8} rather than \\s*.`);
+            }
+        }
+    }
+    return null;
+}

package/dist/eval/rules/regex-sandbox.d.ts

@@ -0,0 +1,26 @@
+/** Wall-clock ceiling for a single `.test()` of a user pattern. Linear
+ * patterns stay in the low milliseconds even on megabyte inputs; only a
+ * superlinear pattern×input combination can approach this. */
+export declare const REGEX_MATCH_BUDGET_MS = 100;
+export type SandboxedRegexResult = {
+    kind: 'match';
+    matched: boolean;
+    durationMs: number;
+} | {
+    kind: 'timeout';
+} | {
+    kind: 'error';
+};
+/**
+ * Runs `new RegExp(pattern, flags).test(input)` in the sandbox worker,
+ * blocking the calling thread for at most `budgetMs`.
+ *
+ * `timeout` means the match was still backtracking at the deadline and the
+ * worker was killed mid-match — the pattern is superlinear on this input.
+ * `error` means the pattern failed to compile in the worker (callers
+ * pre-validate syntax, so this is unexpected).
+ */
+export declare function sandboxedRegexTest(pattern: string, flags: string, input: string, budgetMs?: number): SandboxedRegexResult;
+/** Test hook: kills the singleton so suites can assert respawn behavior and
+ * leave nothing running. Safe to call at any time. */
+export declare function shutdownRegexSandbox(): void;

package/dist/eval/rules/regex-sandbox.js

@@ -0,0 +1,131 @@
+import { Worker } from 'node:worker_threads';
+/*
+ * Hard-deadline execution for user-supplied regex patterns.
+ *
+ * Every prior guard on this path tried to PREDICT backtracking and lost:
+ * safe-regex2 is star-height-only (judges `(a|a)*$` safe; it is exponential),
+ * and the empirical deploy-time probe both ran the untrusted pattern on the
+ * main thread — a single synchronous `.test()` measured at 43,380ms against a
+ * 50ms budget, because `Date.now()` checks after a blocking call cannot
+ * interrupt it — and depended on guessing an igniting payload, which is not
+ * possible in general. A pattern that slipped past the probe hung the whole
+ * server for every concurrent client on a 34-character input.
+ *
+ * This module stops predicting and makes overrun physically impossible: the
+ * match runs in a worker thread while the calling thread blocks in
+ * `Atomics.wait` with a timeout. On breach the worker is terminated
+ * mid-backtrack and a fresh one is spawned for the next call. The API stays
+ * synchronous, which is what the eval engine requires.
+ *
+ * The worker is a singleton, spawned lazily on the first custom-regex
+ * evaluation and reused across calls (spawn costs ~20ms; a warm round-trip is
+ * sub-millisecond). Calls are strictly sequential — the caller blocks — so
+ * there is never more than one match in flight. `unref()` keeps the idle
+ * worker from holding the process open.
+ *
+ * The worker source is embedded as a string (`eval: true`) so the same code
+ * works from TS test context and from the built dist without bundler
+ * path gymnastics. It is CommonJS, which is what eval-mode workers run.
+ */
+/** Wall-clock ceiling for a single `.test()` of a user pattern. Linear
+ * patterns stay in the low milliseconds even on megabyte inputs; only a
+ * superlinear pattern×input combination can approach this. */
+export const REGEX_MATCH_BUDGET_MS = 100;
+/** How long a fresh worker may take to boot before we give up on it. */
+const WORKER_BOOT_TIMEOUT_MS = 5000;
+const WORKER_SOURCE = `
+const { parentPort, workerData } = require('worker_threads');
+parentPort.on('message', ({ flag, pattern, flags, input }) => {
+  const view = new Int32Array(flag);
+  let status;
+  const started = performance.now();
+  try {
+    status = new RegExp(pattern, flags).test(input) ? 1 : 2;
+  } catch {
+    status = 3;
+  }
+  // Slot 1: how long the match ITSELF ran, measured inside the worker.
+  // Callers meter budgets on this, not on wall-clock, so OS scheduling
+  // pressure on a busy host cannot masquerade as backtracking.
+  Atomics.store(view, 1, Math.ceil(performance.now() - started));
+  Atomics.store(view, 0, status);
+  Atomics.notify(view, 0);
+});
+// Ready handshake LAST: by the time the spawner unblocks, the message
+// listener above is installed and the first real match can be processed.
+{
+  const ready = new Int32Array(workerData);
+  Atomics.store(ready, 0, 1);
+  Atomics.notify(ready, 0);
+}
+`;
+let worker = null;
+function getWorker() {
+    if (worker === null) {
+        /*
+         * Spawn, then BLOCK until the worker signals ready. Without this, the
+         * ~20-60ms thread-boot cost lands inside the first caller's match
+         * budget: the deploy probe's 50ms allowance expired during boot, the
+         * still-booting worker was terminated as "backtracking", and the next
+         * call paid spawn again — every ordinary pattern got rejected in a
+         * spawn-kill loop. Boot happens once, outside any match budget.
+         */
+        const ready = new SharedArrayBuffer(4);
+        const spawned = new Worker(WORKER_SOURCE, { eval: true, workerData: ready });
+        // A crashed worker must not poison every later call: drop the handle so
+        // the next call respawns. 'exit' also fires after our own terminate().
+        spawned.on('error', () => {
+            if (worker === spawned)
+                worker = null;
+        });
+        spawned.on('exit', () => {
+            if (worker === spawned)
+                worker = null;
+        });
+        spawned.unref();
+        Atomics.wait(new Int32Array(ready), 0, 0, WORKER_BOOT_TIMEOUT_MS);
+        worker = spawned;
+    }
+    return worker;
+}
+/**
+ * Runs `new RegExp(pattern, flags).test(input)` in the sandbox worker,
+ * blocking the calling thread for at most `budgetMs`.
+ *
+ * `timeout` means the match was still backtracking at the deadline and the
+ * worker was killed mid-match — the pattern is superlinear on this input.
+ * `error` means the pattern failed to compile in the worker (callers
+ * pre-validate syntax, so this is unexpected).
+ */
+export function sandboxedRegexTest(pattern, flags, input, budgetMs = REGEX_MATCH_BUDGET_MS) {
+    // Fresh signal cells per call (slot 0 = status, slot 1 = worker-measured
+    // duration): a terminated worker can never write into a later call's cells.
+    const flag = new SharedArrayBuffer(8);
+    const view = new Int32Array(flag);
+    const w = getWorker();
+    w.postMessage({ flag, pattern, flags, input });
+    const outcome = Atomics.wait(view, 0, 0, budgetMs);
+    if (outcome === 'timed-out') {
+        // Still 0 → the worker is wedged inside .test(). Kill it mid-backtrack;
+        // the 'exit' handler clears the singleton so the next call respawns.
+        void w.terminate();
+        worker = null;
+        return { kind: 'timeout' };
+    }
+    // 'ok' (notified) or 'not-equal' (worker finished before we waited).
+    const status = Atomics.load(view, 0);
+    const durationMs = Atomics.load(view, 1);
+    if (status === 1)
+        return { kind: 'match', matched: true, durationMs };
+    if (status === 2)
+        return { kind: 'match', matched: false, durationMs };
+    return { kind: 'error' };
+}
+/** Test hook: kills the singleton so suites can assert respawn behavior and
+ * leave nothing running. Safe to call at any time. */
+export function shutdownRegexSandbox() {
+    if (worker !== null) {
+        void worker.terminate();
+        worker = null;
+    }
+}

package/dist/eval/rules/relevance.d.ts

@@ -1,6 +1,4 @@
 import type { EvalRule } from '../../types/eval.js';
 export declare const keywordOverlap: EvalRule;
-export declare const HALLUCINATION_MARKERS: string[];
-export declare const noHallucinationMarkers: EvalRule;
 export declare const topicConsistency: EvalRule;
 export declare const relevanceRules: EvalRule[];