@iris-eval/mcp-server 0.4.6 → 0.5.1

package/dist/eval/rules/custom.js

@@ -1,5 +1,6 @@
 import isSafeRegex from 'safe-regex2';
 import { readNumericConfig, describeKeys } from './config-keys.js';
+import { sandboxedRegexTest, REGEX_MATCH_BUDGET_MS } from './regex-sandbox.js';
 const MAX_PATTERN_LENGTH = 1000;
 // A rule whose CONFIG is invalid has not evaluated the output — it could not
 // run at all. Returning `passed:false, score:0` for that case conflates "your
@@ -13,6 +14,11 @@ const MAX_PATTERN_LENGTH = 1000;
 // Deploy-time validation in custom-rule-store.ts now rejects these configs
 // outright; this path remains the safety net for rules already persisted in
 // a user's ~/.iris/custom-rules.json from before that validation existed.
+//
+// configInvalid distinguishes this skip from a legitimate one: config
+// errors depend only on the definition, never the input, so a caller that
+// holds the whole definition (the rule-preview endpoint) can reject it
+// as a 422 instead of reporting every trace as "would skip".
 function configError(definition, message) {
     return {
         ruleName: definition.name,
@@ -21,24 +27,78 @@ function configError(definition, message) {
         message,
         skipped: true,
         skipReason: message,
+        configInvalid: true,
     };
 }
 function safeRegexResult(definition, message) {
     return configError(definition, message);
 }
-function compileRegex(definition) {
-    let patternStr = definition.config.pattern;
-    let flags = definition.config.flags ?? '';
-    // Defensive UX: convert leading inline flag like `(?i)` or `(?im)` to a
-    // real flags arg. Node's RegExp engine does not support inline flag
-    // groups in older versions, and a user pasting `(?i)foo` from a regex
-    // tutorial would otherwise hit "Invalid group" with no clear recovery.
+/**
+ * `config.keywords` as a non-empty array of strings, or undefined when it
+ * is anything else. Element types are checked at runtime because the
+ * inline schema accepts any config value: `keywords: [1, 2]` passed the
+ * old Array.isArray check and then threw from `.toLowerCase()` mid-eval.
+ */
+function readKeywordList(config) {
+    const value = config.keywords;
+    if (!Array.isArray(value) || value.length === 0)
+        return undefined;
+    if (!value.every((k) => typeof k === 'string'))
+        return undefined;
+    return value;
+}
+/**
+ * Converts a leading inline flag group like `(?i)` or `(?im)` into a real
+ * flags argument. Node's RegExp engine does not support inline flag groups,
+ * and a user pasting `(?i)foo` from a regex tutorial would otherwise hit
+ * "Invalid group" with no clear recovery.
+ *
+ * Exported so deploy-time validation (custom-rule-store) probes the SAME
+ * pattern+flags pair the evaluator will actually run — the store used to
+ * strip the inline group but not merge its flags, probing `(?i)…` under
+ * different flags than evaluation used.
+ */
+export function normalizeRegexSource(patternStr, flags) {
     const inlineFlagMatch = patternStr.match(/^\(\?([imsugy]+)\)/);
     if (inlineFlagMatch) {
         const inlineFlags = inlineFlagMatch[1];
         flags = [...new Set((flags + inlineFlags).split(''))].join('');
         patternStr = patternStr.slice(inlineFlagMatch[0].length);
     }
+    return { pattern: patternStr, flags };
+}
+/*
+ * Validates a user pattern and returns the normalized {pattern, flags} pair —
+ * NOT a compiled RegExp, deliberately. The pattern is compiled here once for
+ * syntax validation (compilation does not backtrack), but matching happens in
+ * the sandbox worker (regex-sandbox.ts), which compiles its own copy. Nothing
+ * on the main thread may ever call `.test()`/`.exec()` on a user pattern: the
+ * static checks below are best-effort UX (fast rejection with a good message),
+ * not the safety boundary. safe-regex2 is star-height-only — `(a|a)*$` passes
+ * it and is exponential — and no static or probe-based check is sound in
+ * general. The sandbox's hard deadline is the boundary.
+ */
+function validateRegex(definition) {
+    /*
+     * Runtime shape check, not just a compile-time cast. evaluate_output's
+     * inline custom_rules schema accepts any config record, so
+     * `{type: "regex_match", config: {}}` (or a null / numeric pattern)
+     * reaches this point; the old `as string` cast was a no-op at runtime
+     * and normalizeRegexSource threw a TypeError out of the engine — the
+     * whole evaluate_output call failed, contradicting its own description
+     * ("the eval itself never throws"). Deploy-time validation already
+     * rejects these; this is the same configError contract for the inline
+     * path and for rules persisted before that validation existed.
+     */
+    const rawPattern = definition.config.pattern;
+    if (typeof rawPattern !== 'string' || rawPattern.length === 0) {
+        return safeRegexResult(definition, `${definition.type} rule requires config.pattern (non-empty string)`);
+    }
+    const rawFlags = definition.config.flags;
+    if (rawFlags !== undefined && rawFlags !== null && typeof rawFlags !== 'string') {
+        return safeRegexResult(definition, `${definition.type} rule config.flags must be a string when present`);
+    }
+    const { pattern: patternStr, flags } = normalizeRegexSource(rawPattern, rawFlags ?? '');
     if (patternStr.length > MAX_PATTERN_LENGTH) {
         return safeRegexResult(definition, `Regex pattern too long (${patternStr.length} > ${MAX_PATTERN_LENGTH})`);
     }
@@ -46,9 +106,8 @@ function compileRegex(definition) {
     // parse, so checking it first reports a plainly broken pattern like `(` as
     // "catastrophic backtracking" — sending the author hunting a performance
     // problem they do not have instead of the typo they do.
-    let compiled;
     try {
-        compiled = new RegExp(patternStr, flags);
+        new RegExp(patternStr, flags);
     }
     catch (e) {
         return safeRegexResult(definition, `Invalid regex syntax: ${e instanceof Error ? e.message : 'unknown error'}`);
@@ -56,28 +115,138 @@ function compileRegex(definition) {
     if (!isSafeRegex(patternStr)) {
         return safeRegexResult(definition, 'Regex pattern rejected: potentially unsafe (catastrophic backtracking)');
     }
-    return compiled;
+    return { pattern: patternStr, flags };
 }
-export function createCustomRule(definition) {
+/*
+ * Budget breach is a property of pattern×input, not of the definition alone —
+ * the same pattern can be instant on one output and superlinear on the next
+ * (often one CRAFTED to stall it). So this is not configInvalid: the preview
+ * endpoint must not 422 a rule that merely met a hostile input. It follows the
+ * configError precedent instead: SKIPPED, because a rule whose match was
+ * killed mid-backtrack has not judged the output, and a skipped rule neither
+ * deflates the weighted score nor (for high/critical deployed rules) vetoes
+ * the eval on evidence it never gathered. The skipReason tells the author
+ * exactly what to fix, and the engine already surfaces it in suggestions.
+ */
+function budgetExceededResult(definition) {
+    const message = `Regex evaluation terminated: pattern exceeded the ${REGEX_MATCH_BUDGET_MS}ms matching ` +
+        `budget on this output (superlinear backtracking) and was killed in its sandbox worker. ` +
+        `The rule did NOT judge this output — a gate that must fail closed should treat ` +
+        `budgetExceeded skips as failures. Rewrite the pattern to avoid ambiguous repetition ` +
+        `— e.g. bound quantifiers (\\s{0,8} not \\s*) and remove overlapping alternatives.`;
+    return {
+        ruleName: definition.name,
+        passed: false,
+        score: 0,
+        message,
+        skipped: true,
+        skipReason: message,
+        budgetExceeded: true,
+    };
+}
+/**
+ * Per-evaluation cap on sandbox budget breaches. Each breach costs the
+ * request its budget PLUS a worker respawn (~190ms total measured), and the
+ * engine runs rules synchronously — so without a breaker, one request
+ * carrying N hostile regex rules stalls the server N × ~190ms (measured
+ * 9.3s at N=50). After this many breaches, remaining regex rules in the
+ * same evaluation skip WITHOUT running, bounding the whole request at
+ * roughly cap × 190ms regardless of rule count.
+ */
+const MAX_REGEX_BREACHES_PER_EVAL = 3;
+function circuitOpenResult(definition) {
+    const message = `Regex evaluation skipped: ${MAX_REGEX_BREACHES_PER_EVAL} earlier pattern(s) in this ` +
+        `evaluation already exhausted the ${REGEX_MATCH_BUDGET_MS}ms matching budget, so the ` +
+        `regex circuit breaker is open for the rest of this evaluation. The rule did NOT judge ` +
+        `this output — a gate that must fail closed should treat budgetExceeded skips as failures.`;
+    return {
+        ruleName: definition.name,
+        passed: false,
+        score: 0,
+        message,
+        skipped: true,
+        skipReason: message,
+        budgetExceeded: true,
+    };
+}
+/*
+ * A sandbox 'error' is NOT the author's fault and must not be reported as
+ * backtracking: it means the worker could not run the (pre-validated)
+ * pattern at all — in practice a worker that died between calls (postMessage
+ * to a terminated worker is a silent no-op). Accusing the pattern sends the
+ * author hunting a performance problem they do not have.
+ */
+function sandboxErrorResult(definition) {
+    const message = 'Regex evaluation skipped: internal sandbox error (the matching worker restarted). ' +
+        'The rule did not judge this output; the pattern itself is fine — retry the evaluation.';
+    return {
+        ruleName: definition.name,
+        passed: false,
+        score: 0,
+        message,
+        skipped: true,
+        skipReason: message,
+    };
+}
+/**
+ * Executes a validated user pattern through the sandbox with the
+ * per-evaluation circuit breaker. Shared by regex_match and regex_no_match.
+ */
+function runSandboxed(definition, pattern, flags, context) {
+    const budget = context.regexBudget;
+    if (budget && budget.breaches >= MAX_REGEX_BREACHES_PER_EVAL) {
+        return circuitOpenResult(definition);
+    }
+    const outcome = sandboxedRegexTest(pattern, flags, context.output);
+    if (outcome.kind === 'timeout') {
+        if (budget)
+            budget.breaches += 1;
+        return budgetExceededResult(definition);
+    }
+    if (outcome.kind === 'error') {
+        return sandboxErrorResult(definition);
+    }
+    return { matched: outcome.matched };
+}
+/**
+ * Builds a runnable EvalRule from a persisted/inline definition.
+ *
+ * `severity` comes from the DEPLOYED rule's metadata (deploy_rule / the
+ * dashboard composer). high/critical severities make the rule CRITICAL:
+ * a failing evaluation forces the overall eval to passed=false regardless
+ * of the weighted score. Before this, a rule-author could deploy a
+ * severity="critical" policy rule, watch it FAIL on a violating output,
+ * and still get passed:true (score 0.895) — severity affected nothing but
+ * dashboard sorting. Inline custom_rules (evaluate_output's per-call
+ * definitions) carry no severity and stay weight-only.
+ */
+export function createCustomRule(definition, severity) {
     return {
         name: definition.name,
         description: `Custom rule: ${definition.name}`,
         evalType: 'custom',
         weight: definition.weight ?? 1,
+        critical: severity === 'high' || severity === 'critical',
         evaluate(context) {
             switch (definition.type) {
                 case 'regex_match': {
-                    const result = compileRegex(definition);
-                    if (!(result instanceof RegExp))
-                        return result;
-                    const passed = result.test(context.output);
+                    const validated = validateRegex(definition);
+                    if ('ruleName' in validated)
+                        return validated;
+                    const run = runSandboxed(definition, validated.pattern, validated.flags, context);
+                    if ('ruleName' in run)
+                        return run;
+                    const passed = run.matched;
                     return { ruleName: definition.name, passed, score: passed ? 1 : 0, message: passed ? 'Regex pattern matched' : 'Regex pattern did not match' };
                 }
                 case 'regex_no_match': {
-                    const result = compileRegex(definition);
-                    if (!(result instanceof RegExp))
-                        return result;
-                    const passed = !result.test(context.output);
+                    const validated = validateRegex(definition);
+                    if ('ruleName' in validated)
+                        return validated;
+                    const run = runSandboxed(definition, validated.pattern, validated.flags, context);
+                    if ('ruleName' in run)
+                        return run;
+                    const passed = !run.matched;
                     return { ruleName: definition.name, passed, score: passed ? 1 : 0, message: passed ? 'Forbidden pattern not found' : 'Forbidden pattern found in output' };
                 }
                 case 'min_length': {
@@ -97,8 +266,8 @@ export function createCustomRule(definition) {
                     return { ruleName: definition.name, passed, score: passed ? 1 : max / context.output.length, message: passed ? `Length (${context.output.length}) within maximum (${max})` : `Length (${context.output.length}) exceeds maximum (${max})` };
                 }
                 case 'contains_keywords': {
-                    const keywords = definition.config.keywords;
-                    if (!keywords || !Array.isArray(keywords) || keywords.length === 0) {
+                    const keywords = readKeywordList(definition.config);
+                    if (!keywords) {
                         return configError(definition, 'contains_keywords rule requires config.keywords (non-empty string array)');
                     }
                     const lower = context.output.toLowerCase();
@@ -108,8 +277,8 @@ export function createCustomRule(definition) {
                     return { ruleName: definition.name, passed, score: ratio, message: `Found ${found.length}/${keywords.length} required keywords` };
                 }
                 case 'excludes_keywords': {
-                    const keywords = definition.config.keywords;
-                    if (!keywords || !Array.isArray(keywords) || keywords.length === 0) {
+                    const keywords = readKeywordList(definition.config);
+                    if (!keywords) {
                         return configError(definition, 'excludes_keywords rule requires config.keywords (non-empty string array)');
                     }
                     const lower = context.output.toLowerCase();
@@ -131,7 +300,27 @@ export function createCustomRule(definition) {
                     if (max == null || max < 0) {
                         return configError(definition, `cost_threshold rule requires ${describeKeys('cost_threshold')} (non-negative number)`);
                     }
-                    const cost = context.costUsd ?? 0;
+                    /*
+                     * No cost data → SKIP, exactly like the built-in
+                     * cost_under_threshold (cost.ts). The old `context.costUsd ?? 0`
+                     * read a missing cost as free, so a rule deployed at severity
+                     * critical to hard-fail evaluations over $0.50 reported
+                     * passed:true, score:1 on every evaluate_output call that simply
+                     * omitted cost_usd — the veto never fired on evidence it never
+                     * had. A skipped critical rule is reported in critical_skipped
+                     * instead, so a fail-closed gate can see the rule did not run.
+                     */
+                    if (context.costUsd === undefined || context.costUsd === null) {
+                        return {
+                            ruleName: definition.name,
+                            passed: false,
+                            score: 0,
+                            message: 'Cost data not provided',
+                            skipped: true,
+                            skipReason: 'context.costUsd not provided',
+                        };
+                    }
+                    const cost = context.costUsd;
                     const passed = cost <= max;
                     return { ruleName: definition.name, passed, score: passed ? 1 : 0, message: passed ? `Cost ($${cost}) within threshold ($${max})` : `Cost ($${cost}) exceeds threshold ($${max})` };
                 }

package/dist/eval/rules/regex-sandbox.d.ts

@@ -0,0 +1,26 @@
+/** Wall-clock ceiling for a single `.test()` of a user pattern. Linear
+ * patterns stay in the low milliseconds even on megabyte inputs; only a
+ * superlinear pattern×input combination can approach this. */
+export declare const REGEX_MATCH_BUDGET_MS = 100;
+export type SandboxedRegexResult = {
+    kind: 'match';
+    matched: boolean;
+    durationMs: number;
+} | {
+    kind: 'timeout';
+} | {
+    kind: 'error';
+};
+/**
+ * Runs `new RegExp(pattern, flags).test(input)` in the sandbox worker,
+ * blocking the calling thread for at most `budgetMs`.
+ *
+ * `timeout` means the match was still backtracking at the deadline and the
+ * worker was killed mid-match — the pattern is superlinear on this input.
+ * `error` means the pattern failed to compile in the worker (callers
+ * pre-validate syntax, so this is unexpected).
+ */
+export declare function sandboxedRegexTest(pattern: string, flags: string, input: string, budgetMs?: number): SandboxedRegexResult;
+/** Test hook: kills the singleton so suites can assert respawn behavior and
+ * leave nothing running. Safe to call at any time. */
+export declare function shutdownRegexSandbox(): void;

package/dist/eval/rules/regex-sandbox.js

@@ -0,0 +1,131 @@
+import { Worker } from 'node:worker_threads';
+/*
+ * Hard-deadline execution for user-supplied regex patterns.
+ *
+ * Every prior guard on this path tried to PREDICT backtracking and lost:
+ * safe-regex2 is star-height-only (judges `(a|a)*$` safe; it is exponential),
+ * and the empirical deploy-time probe both ran the untrusted pattern on the
+ * main thread — a single synchronous `.test()` measured at 43,380ms against a
+ * 50ms budget, because `Date.now()` checks after a blocking call cannot
+ * interrupt it — and depended on guessing an igniting payload, which is not
+ * possible in general. A pattern that slipped past the probe hung the whole
+ * server for every concurrent client on a 34-character input.
+ *
+ * This module stops predicting and makes overrun physically impossible: the
+ * match runs in a worker thread while the calling thread blocks in
+ * `Atomics.wait` with a timeout. On breach the worker is terminated
+ * mid-backtrack and a fresh one is spawned for the next call. The API stays
+ * synchronous, which is what the eval engine requires.
+ *
+ * The worker is a singleton, spawned lazily on the first custom-regex
+ * evaluation and reused across calls (spawn costs ~20ms; a warm round-trip is
+ * sub-millisecond). Calls are strictly sequential — the caller blocks — so
+ * there is never more than one match in flight. `unref()` keeps the idle
+ * worker from holding the process open.
+ *
+ * The worker source is embedded as a string (`eval: true`) so the same code
+ * works from TS test context and from the built dist without bundler
+ * path gymnastics. It is CommonJS, which is what eval-mode workers run.
+ */
+/** Wall-clock ceiling for a single `.test()` of a user pattern. Linear
+ * patterns stay in the low milliseconds even on megabyte inputs; only a
+ * superlinear pattern×input combination can approach this. */
+export const REGEX_MATCH_BUDGET_MS = 100;
+/** How long a fresh worker may take to boot before we give up on it. */
+const WORKER_BOOT_TIMEOUT_MS = 5000;
+const WORKER_SOURCE = `
+const { parentPort, workerData } = require('worker_threads');
+parentPort.on('message', ({ flag, pattern, flags, input }) => {
+  const view = new Int32Array(flag);
+  let status;
+  const started = performance.now();
+  try {
+    status = new RegExp(pattern, flags).test(input) ? 1 : 2;
+  } catch {
+    status = 3;
+  }
+  // Slot 1: how long the match ITSELF ran, measured inside the worker.
+  // Callers meter budgets on this, not on wall-clock, so OS scheduling
+  // pressure on a busy host cannot masquerade as backtracking.
+  Atomics.store(view, 1, Math.ceil(performance.now() - started));
+  Atomics.store(view, 0, status);
+  Atomics.notify(view, 0);
+});
+// Ready handshake LAST: by the time the spawner unblocks, the message
+// listener above is installed and the first real match can be processed.
+{
+  const ready = new Int32Array(workerData);
+  Atomics.store(ready, 0, 1);
+  Atomics.notify(ready, 0);
+}
+`;
+let worker = null;
+function getWorker() {
+    if (worker === null) {
+        /*
+         * Spawn, then BLOCK until the worker signals ready. Without this, the
+         * ~20-60ms thread-boot cost lands inside the first caller's match
+         * budget: the deploy probe's 50ms allowance expired during boot, the
+         * still-booting worker was terminated as "backtracking", and the next
+         * call paid spawn again — every ordinary pattern got rejected in a
+         * spawn-kill loop. Boot happens once, outside any match budget.
+         */
+        const ready = new SharedArrayBuffer(4);
+        const spawned = new Worker(WORKER_SOURCE, { eval: true, workerData: ready });
+        // A crashed worker must not poison every later call: drop the handle so
+        // the next call respawns. 'exit' also fires after our own terminate().
+        spawned.on('error', () => {
+            if (worker === spawned)
+                worker = null;
+        });
+        spawned.on('exit', () => {
+            if (worker === spawned)
+                worker = null;
+        });
+        spawned.unref();
+        Atomics.wait(new Int32Array(ready), 0, 0, WORKER_BOOT_TIMEOUT_MS);
+        worker = spawned;
+    }
+    return worker;
+}
+/**
+ * Runs `new RegExp(pattern, flags).test(input)` in the sandbox worker,
+ * blocking the calling thread for at most `budgetMs`.
+ *
+ * `timeout` means the match was still backtracking at the deadline and the
+ * worker was killed mid-match — the pattern is superlinear on this input.
+ * `error` means the pattern failed to compile in the worker (callers
+ * pre-validate syntax, so this is unexpected).
+ */
+export function sandboxedRegexTest(pattern, flags, input, budgetMs = REGEX_MATCH_BUDGET_MS) {
+    // Fresh signal cells per call (slot 0 = status, slot 1 = worker-measured
+    // duration): a terminated worker can never write into a later call's cells.
+    const flag = new SharedArrayBuffer(8);
+    const view = new Int32Array(flag);
+    const w = getWorker();
+    w.postMessage({ flag, pattern, flags, input });
+    const outcome = Atomics.wait(view, 0, 0, budgetMs);
+    if (outcome === 'timed-out') {
+        // Still 0 → the worker is wedged inside .test(). Kill it mid-backtrack;
+        // the 'exit' handler clears the singleton so the next call respawns.
+        void w.terminate();
+        worker = null;
+        return { kind: 'timeout' };
+    }
+    // 'ok' (notified) or 'not-equal' (worker finished before we waited).
+    const status = Atomics.load(view, 0);
+    const durationMs = Atomics.load(view, 1);
+    if (status === 1)
+        return { kind: 'match', matched: true, durationMs };
+    if (status === 2)
+        return { kind: 'match', matched: false, durationMs };
+    return { kind: 'error' };
+}
+/** Test hook: kills the singleton so suites can assert respawn behavior and
+ * leave nothing running. Safe to call at any time. */
+export function shutdownRegexSandbox() {
+    if (worker !== null) {
+        void worker.terminate();
+        worker = null;
+    }
+}

package/dist/eval/rules/relevance.d.ts

@@ -1,6 +1,4 @@
 import type { EvalRule } from '../../types/eval.js';
 export declare const keywordOverlap: EvalRule;
-export declare const HALLUCINATION_MARKERS: string[];
-export declare const noHallucinationMarkers: EvalRule;
 export declare const topicConsistency: EvalRule;
 export declare const relevanceRules: EvalRule[];

package/dist/eval/rules/relevance.js

@@ -28,75 +28,13 @@ export const keywordOverlap = {
         };
     },
 };
-// Exported so the claims drift test can assert .claims.json counts against
-// the runtime truth (tests/claims-eval-rules-counts.test.ts).
-export const HALLUCINATION_MARKERS = [
-    'as an ai',
-    'as a language model',
-    'i cannot',
-    'i don\'t have access',
-    'i apologize',
-    'i\'m not able to',
-    'i must clarify',
-    'it\'s important to note that i',
-    'i should mention that as',
-    'i\'m just an ai',
-    'i don\'t actually',
-    'i cannot provide',
-    'i\'m unable to',
-    'please note that i',
-    'as a digital assistant',
-    'i want to be transparent',
-    'i need to be honest',
-];
 /*
- * Heuristic for fabricated-citation patterns — added v0.3.1.
- *
- * Looks for the shape: numbered citation markers ([1], [2], etc.) appearing
- * 3+ times AND density of "Dr." / "Professor" / "according to" / "study by"
- * markers. Heuristic only — doesn't verify citations are real (that's v0.5
- * LLM-as-judge work). Catches the common pattern where an agent emits
- * confident-sounding citations to fabricated sources.
+ * no_hallucination_markers moved to the safety bundle (safety.ts) in
+ * v0.5.0 — its rewrite is context-grounded fabrication/contradiction
+ * detection, and the safety bundle is where the evaluate_output docs,
+ * the dashboard's safety-violations panel, and the storage adapter's
+ * violation counts have always placed it.
  */
-function looksLikeFabricatedCitations(output) {
-    const numberedCitations = (output.match(/\[\d+\]/g) ?? []).length;
-    if (numberedCitations < 3)
-        return false;
-    const expertMarkers = (output.match(/\b(?:Dr\.|Professor|according to|study by|research by|paper by)\b/gi) ?? []).length;
-    return expertMarkers >= 2;
-}
-export const noHallucinationMarkers = {
-    name: 'no_hallucination_markers',
-    description: 'Checks for AI hedging markers + heuristic fabricated-citation pattern',
-    evalType: 'relevance',
-    weight: 1,
-    evaluate(context) {
-        const lower = context.output.toLowerCase();
-        const foundMarkers = HALLUCINATION_MARKERS.filter((marker) => lower.includes(marker));
-        const fabricatedCitationPattern = looksLikeFabricatedCitations(context.output);
-        const totalIssues = foundMarkers.length + (fabricatedCitationPattern ? 1 : 0);
-        const passed = totalIssues === 0;
-        let message;
-        if (passed) {
-            message = 'No hallucination markers detected';
-        }
-        else if (fabricatedCitationPattern && foundMarkers.length === 0) {
-            message = 'Heuristic: fabricated-citation pattern detected (3+ numbered citations + expert markers)';
-        }
-        else if (fabricatedCitationPattern) {
-            message = `Markers: ${foundMarkers.join(', ')}; plus fabricated-citation heuristic`;
-        }
-        else {
-            message = `Found markers: ${foundMarkers.join(', ')}`;
-        }
-        return {
-            ruleName: 'no_hallucination_markers',
-            passed,
-            score: passed ? 1 : Math.max(0, 1 - totalIssues * 0.3),
-            message,
-        };
-    },
-};
 export const topicConsistency = {
     name: 'topic_consistency',
     description: 'Output stays on topic relative to input (skipped when output too brief for meaningful comparison)',
@@ -142,4 +80,4 @@ export const topicConsistency = {
         };
     },
 };
-export const relevanceRules = [keywordOverlap, noHallucinationMarkers, topicConsistency];
+export const relevanceRules = [keywordOverlap, topicConsistency];

package/dist/eval/rules/safety.d.ts

@@ -2,10 +2,20 @@ import type { EvalRule } from '../../types/eval.js';
 export declare const PII_PATTERNS: Array<{
     name: string;
     pattern: RegExp;
+    placeholders?: RegExp[];
 }>;
 export declare const noPii: EvalRule;
 export declare const noBlocklistWords: EvalRule;
 export declare const INJECTION_PATTERNS: RegExp[];
 export declare const noInjectionPatterns: EvalRule;
 export declare const noStubOutput: EvalRule;
+export interface HallucinationSignal {
+    /** Stable kebab-case identifier, reported in rule messages. */
+    name: string;
+    /** Context-grounded signals stay silent when the caller passes no input. */
+    requiresContext: boolean;
+    detect(output: string, input: string): string | null;
+}
+export declare const HALLUCINATION_MARKERS: ReadonlyArray<HallucinationSignal>;
+export declare const noHallucinationMarkers: EvalRule;
 export declare const safetyRules: EvalRule[];