@zanii/blackbox 0.3.0 → 0.4.0

package/dist/analysis/grounding.d.ts

@@ -0,0 +1,122 @@
+import { type Bodies, type Call } from "../reconcile/record.ts";
+import { canonicalNumber } from "./hallucination.ts";
+import { type ReferencePack } from "./reference.ts";
+export { canonicalNumber };
+import type { Finding } from "./index.ts";
+export interface Fact {
+    kind: "money" | "percent" | "quantity" | "date" | string;
+    /** Canonical: a decimal (`1250.5`) for money and percentages, normalised text otherwise. */
+    value: string;
+    at: number;
+    end: number;
+    currency?: string;
+}
+/** The facts an answer states: money, percentages, quantities, ISO dates, then identifiers (as H1's). */
+export declare function factsIn(text: string): Fact[];
+/** What the agent was given, to check facts against: the record lines it came from (§11 points at
+ * them), each normalised, with its numbers. */
+export declare class Evidence {
+    /** As given, for a grounding service (§9.1). */
+    readonly raw: string;
+    readonly numbers: Set<string>;
+    private readonly pieces;
+    private sums;
+    constructor(raw: string, pieces?: ReadonlyArray<{
+        seq: number;
+        text: string;
+    }>);
+    /** Whether the evidence holds the fact (§9). */
+    holds(f: Fact): boolean;
+    /**
+     * The seqs of the lines that hold a fact, or null. Money, percentages and quantities: the same
+     * number, the same in minor units, a percentage as a fraction, or (money) the sum of two of the
+     * evidence's numbers. Dates and identifiers: the normalised text.
+     */
+    support(f: Fact): number[] | null;
+    private pairSums;
+}
+/** The final answers on a record, each with the evidence it should rest on. */
+export declare function answersOf(lines: readonly string[], bodies: Bodies): Array<{
+    seq: number;
+    text: string;
+    evidence: Evidence;
+    call: Call;
+}>;
+/** What a grounding service is sent for one answer: the text, the exact tier's facts, and the
+ * context it should rest on (the newest 200,000 characters of it). */
+export declare function groundingRequest(sessionId: string, answer: {
+    seq: number;
+    text: string;
+    evidence: Evidence;
+}, packs?: readonly ReferencePack[]): {
+    session_id: string;
+    answer_seq: number;
+    answer: string;
+    facts: {
+        kind: string;
+        value: string;
+    }[];
+    context: string;
+    reference?: {
+        pack: string;
+        id: string;
+        text: string;
+        source?: string;
+    }[];
+};
+export interface GroundedClaim {
+    text: string;
+    status: "grounded" | "ungrounded" | "contradicted" | "no_fact";
+    score?: number;
+}
+/** A grounding service's answer, checked: `{claims, model?, version?}`, or null (an outage). */
+export declare function parseGroundingAnswer(v: unknown): {
+    claims: GroundedClaim[];
+    model?: string;
+    version?: string;
+} | null;
+/** §9.1: UNGROUNDED_CLAIM and CONTRADICTED_CLAIM from the verdicts a grounding service gave,
+ * recorded as `control {action: "grounding", via: "model"}` with the claims as the body. */
+export declare function modelGroundingFindings(lines: readonly string[], bodies: Bodies): Finding[];
+/** spec/findings.md §9-§10: for each fact in a final answer, CONTRADICTED_CLAIM when a reference
+ * pack says otherwise, else UNGROUNDED_CLAIM when nothing the agent was given holds it (at most 10
+ * per answer); then the grounding service's verdicts (§9.1). */
+export declare function groundingFindings(lines: readonly string[], bodies: Bodies, packs?: readonly ReferencePack[]): Finding[];
+/** What a judge is sent for one answer: the answer and the trajectory before it (its newest 100
+ * steps: tools called, whether they failed, earlier answers, the findings so far). */
+export declare function judgeRequest(sessionId: string, answer: {
+    seq: number;
+    text: string;
+}, lines: readonly string[]): {
+    session_id: string;
+    answer_seq: number;
+    answer: string;
+    steps: {
+        [key: string]: string | number | boolean;
+    }[];
+};
+/** A judge's answer, checked: `{steps: [{rubric, pass, reason?}], model?, version?}`, or null. */
+export declare function parseJudgeAnswer(v: unknown): {
+    steps: Array<{
+        rubric: string;
+        pass: boolean;
+        reason?: string;
+    }>;
+    model?: string;
+    version?: string;
+} | null;
+/** §12.1: JUDGE_FAILED for each rubric step a judge failed, from its recorded verdicts. */
+export declare function judgeFindings(lines: readonly string[], bodies: Bodies): Finding[];
+/** spec/findings.md §13: what can hold an answer (BLACKBOX_HOLD_ANSWERS). */
+export declare const HOLD_TRIGGERS: readonly ["contradicted", "money", "ids", "quantity", "percent", "date"];
+/** §13: why an answer should be held before it's delivered: each fact a pack contradicts (with
+ * `contradicted`), and each ungrounded fact whose kind is a trigger (`ids` covers the identifiers).
+ * Empty: deliver it. */
+export declare function answerRisk(answer: {
+    text: string;
+    evidence: Evidence;
+}, packs: readonly ReferencePack[], triggers: readonly string[]): Array<{
+    index: number;
+    kind: string;
+    status: "contradicted" | "ungrounded";
+}>;

package/dist/analysis/grounding.js

@@ -0,0 +1,445 @@
+// Facts against the record (spec/findings.md §9, H2): the amounts, percentages, dates and
+// identifiers in an agent's answer, each checked against what the agent was given (its request:
+// the person's words, the system prompt, the tool results it passed back) and the tool results on
+// the record. Exact, no model, Arabic and English. Mirrors
+// sdks/python/src/zanii_blackbox/analysis/grounding.py; pinned by spec/vectors/grounding.json.
+import { callsOf } from "../reconcile/record.js";
+import { canonical } from "../reconcile/shared.js";
+import { toolCallsOf } from "./detectors.js";
+import { answerText, canonicalNumber, identifierSpans, normaliseValue, western, } from "./hallucination.js";
+import { checkAgainstPacks, referenceContext } from "./reference.js";
+export { canonicalNumber };
+const decoder = new TextDecoder();
+/** Arabic decimal and thousands marks, and Arabic-Indic digits, as Western ones. */
+const westernText = (s) => western(s).replaceAll("٫", ".").replaceAll("٬", ",");
+const AMOUNT = "[0-9]{1,3}(?:,[0-9]{3})+(?:\\.[0-9]+)?|[0-9]+(?:\\.[0-9]+)?";
+// any currency: the ISO 4217 codes (in capitals), the usual words and symbols, English and Arabic
+const CURRENCY = "AED|AFN|AMD|ANG|AOA|ARS|AUD|AWG|AZN|BAM|BBD|BDT|BGN|BHD|BIF|BMD|BND|BOB|BRL|BSD|BTN|BWP|BYN|BZD|CAD|CDF|CHF|CLP|CNY|COP|CRC|CUP|CVE|CZK|DJF|DKK|DOP|DZD|EGP|ERN|ETB|EUR|FJD|FKP|GBP|GEL|GHS|GIP|GMD|GNF|GTQ|GYD|HKD|HNL|HTG|HUF|IDR|ILS|INR|IQD|IRR|ISK|JMD|JOD|JPY|KES|KGS|KHR|KMF|KPW|KRW|KWD|KYD|KZT|LAK|LBP|LKR|LRD|LSL|LYD|MAD|MDL|MGA|MKD|MMK|MNT|MOP|MRU|MUR|MVR|MWK|MXN|MYR|MZN|NAD|NGN|NIO|NOK|NPR|NZD|OMR|PAB|PEN|PGK|PHP|PKR|PLN|PYG|QAR|RON|RSD|RUB|RWF|SAR|SBD|SCR|SDG|SEK|SGD|SHP|SLE|SOS|SRD|SSP|STN|SVC|SYP|SZL|THB|TJS|TMT|TND|TRY|TTD|TWD|TZS|UAH|UGX|USD|UYU|UZS|VES|VND|VUV|WST|XAF|XCD|XOF|XPF|YER|ZAR|ZMW|ZWL|dirhams|dirham|Dirhams|Dirham|dollars|dollar|Dollars|Dollar|euros|euro|Euros|Euro|pounds|pound|Pounds|Pound|riyals|riyal|Riyals|Riyal|rials|rial|Rials|Rial|rupees|rupee|Rupees|Rupee|dinars|dinar|Dinars|Dinar|francs|franc|Francs|Franc|pesos|peso|Pesos|Peso|yen|Yen|yuan|Yuan|liras|lira|Liras|Lira|ringgits|ringgit|Ringgits|Ringgit|rand|Rand|nairas|naira|Nairas|Naira|shillings|shilling|Shillings|Shilling|taka|Taka|Dhs|Dh|\\$|€|£|¥|₹|₩|₽|₺|₪|฿|₫|₦|₱|د\\.إ|درهم|دراهم|ريال|دولار|دينار|جنيه|يورو";
+const MONEY = new RegExp(`(?<![A-Za-z])(${CURRENCY})\\s?(${AMOUNT})(?![0-9])|(?<![0-9.,])(${AMOUNT})\\s?(${CURRENCY})(?![A-Za-z])`, "g");
+const PERCENT = new RegExp(`(?<![0-9.,])(${AMOUNT})\\s?(?:%|٪|percent|per cent|بالمائة|في المائة)`, "gi");
+// any measured quantity: a dose, a weight, a distance, a duration ("within 14 days")
+const UNIT = "mg/dL|mmol/L|mcg|µg|mg|kg|g|mL|ml|L|km|cm|mm|m|mi|ft|lbs|lb|oz|°C|°F|kWh|MWh|kW|MW|IU|hours|hour|hrs|hr|h|minutes|minute|mins|min|seconds|second|secs|days|day|weeks|week|months|month|years|year|units|unit|items|pieces|tablets|doses|ملغ|مجم|غرام|كغ|كيلوغرام|مل|لتر|كم|متر|ساعة|ساعات|دقيقة|دقائق|يوم|أيام|أسبوع|أسابيع|شهر|أشهر|سنة|سنوات|وحدة|حبة|جرعة";
+const QUANTITY = new RegExp(`(?<![0-9.,])(${AMOUNT})\\s?(?:${UNIT})(?![A-Za-z\u0600-\u06FF])`, "g");
+const ISO_DATE = /(?<![0-9])[0-9]{4}-[0-9]{2}-[0-9]{2}(?![0-9])/g;
+const NUMBER = new RegExp(`(?<![0-9.,])(?:${AMOUNT})(?![0-9])`, "g");
+/** A canonical decimal as an integer of millionths, exactly (sums without floats). */
+function micros(c) {
+    const [w = "0", f = ""] = c.split(".");
+    return BigInt(w) * 1000000n + BigInt(f.slice(0, 6).padEnd(6, "0"));
+}
+const shift = (c, places) => {
+    // ×10^places for places ≥ 0, ÷10^-places otherwise, on the decimal string
+    const m = micros(c) * 10n ** BigInt(Math.max(0, places));
+    const v = places < 0 ? m / 10n ** BigInt(-places) : m;
+    const w = v / 1000000n;
+    const f = (v % 1000000n).toString().padStart(6, "0").replace(/0+$/, "");
+    return f ? `${w}.${f}` : `${w}`;
+};
+/** The facts an answer states: money, percentages, quantities, ISO dates, then identifiers (as H1's). */
+export function factsIn(text) {
+    const s = westernText(text);
+    const out = [];
+    const taken = [];
+    const add = (f) => {
+        if (taken.some(([a, b]) => f.at < b && f.end > a))
+            return;
+        taken.push([f.at, f.end]);
+        out.push(f);
+    };
+    for (const m of s.matchAll(MONEY)) {
+        const amount = (m[2] ?? m[3]);
+        const currency = (m[1] ?? m[4]);
+        const at = m.index ?? 0;
+        add({ kind: "money", value: canonicalNumber(amount), at, end: at + m[0].length, currency });
+    }
+    for (const m of s.matchAll(PERCENT)) {
+        const at = m.index ?? 0;
+        add({ kind: "percent", value: canonicalNumber(m[1]), at, end: at + m[0].length });
+    }
+    for (const m of s.matchAll(QUANTITY)) {
+        const at = m.index ?? 0;
+        add({ kind: "quantity", value: canonicalNumber(m[1]), at, end: at + m[0].length });
+    }
+    for (const m of s.matchAll(ISO_DATE)) {
+        const at = m.index ?? 0;
+        add({ kind: "date", value: normaliseValue(m[0]), at, end: at + m[0].length });
+    }
+    for (const x of identifierSpans(s, taken))
+        out.push({ kind: x.kind, value: x.value, at: x.at, end: x.end });
+    return out.sort((a, b) => a.at - b.at);
+}
+/** What the agent was given, to check facts against: the record lines it came from (§11 points at
+ * them), each normalised, with its numbers. */
+export class Evidence {
+    /** As given, for a grounding service (§9.1). */
+    raw;
+    numbers;
+    pieces;
+    sums = null;
+    constructor(raw, pieces) {
+        this.raw = raw;
+        this.pieces = (pieces ?? [{ seq: -1, text: raw }]).map((p) => {
+            const s = westernText(p.text);
+            return {
+                seq: p.seq,
+                text: normaliseValue(s),
+                numbers: new Set([...s.matchAll(NUMBER)].map((m) => canonicalNumber(m[0]))),
+            };
+        });
+        this.numbers = new Set(this.pieces.flatMap((p) => [...p.numbers]));
+    }
+    /** Whether the evidence holds the fact (§9). */
+    holds(f) {
+        return this.support(f) !== null;
+    }
+    /**
+     * The seqs of the lines that hold a fact, or null. Money, percentages and quantities: the same
+     * number, the same in minor units, a percentage as a fraction, or (money) the sum of two of the
+     * evidence's numbers. Dates and identifiers: the normalised text.
+     */
+    support(f) {
+        if (f.kind === "money" || f.kind === "percent" || f.kind === "quantity") {
+            const wanted = [
+                f.value,
+                shift(f.value, 2),
+                ...(f.kind === "percent" ? [shift(f.value, -2)] : []),
+            ];
+            const hit = this.pieces.find((p) => wanted.some((w) => p.numbers.has(w)));
+            if (hit)
+                return [hit.seq];
+            return f.kind === "money" ? (this.pairSums().get(f.value) ?? null) : null;
+        }
+        const hit = this.pieces.find((p) => p.text.includes(f.value));
+        return hit ? [hit.seq] : null;
+    }
+    // ponytail: pair sums only over the first 300 numbers, so a long record stays O(300²)
+    pairSums() {
+        if (this.sums)
+            return this.sums;
+        const first = new Map();
+        for (const p of this.pieces)
+            for (const n of p.numbers)
+                if (!first.has(n))
+                    first.set(n, p.seq);
+        const ns = [...first].slice(0, 300).map(([n, seq]) => [micros(n), seq]);
+        const out = new Map();
+        for (let i = 0; i < ns.length; i++)
+            for (let j = i + 1; j < ns.length; j++) {
+                const [a, sa] = ns[i];
+                const [b, sb] = ns[j];
+                const v = a + b;
+                const w = v / 1000000n;
+                const fr = (v % 1000000n).toString().padStart(6, "0").replace(/0+$/, "");
+                const key = fr ? `${w}.${fr}` : `${w}`;
+                if (!out.has(key))
+                    out.set(key, sa === sb ? [sa] : [sa, sb]);
+            }
+        this.sums = out;
+        return out;
+    }
+}
+/** The final answers on a record, each with the evidence it should rest on. */
+export function answersOf(lines, bodies) {
+    const calls = callsOf(lines, bodies);
+    const ends = new Set(toolCallsOf(calls).map((t) => t.seq));
+    // tool results on the record, by seq, to add those before each answer
+    const results = [];
+    // (each answer's evidence: its request, then the tool results before it, as record lines)
+    for (const l of lines) {
+        const e = JSON.parse(l);
+        if (e.kind === "tool.result" || (e.kind === "sdk.event" && e.meta.type === "tool.result")) {
+            const b = bodies(e.body_hash);
+            results.push({ seq: e.seq, text: b ? decoder.decode(b) : "" });
+        }
+    }
+    const out = [];
+    for (const c of [...calls].sort((a, b) => a.endSeq - b.endSeq)) {
+        if (!c.complete || !c.request || ends.has(c.endSeq))
+            continue;
+        const text = answerText(c);
+        if (!text)
+            continue;
+        const pieces = [
+            { seq: c.requestSeq, text: canonical(c.request) },
+            ...results.filter((r) => r.seq < c.endSeq),
+        ];
+        out.push({
+            seq: c.endSeq,
+            text,
+            evidence: new Evidence(pieces.map((p) => p.text).join("\n"), pieces),
+            call: c,
+        });
+    }
+    return out;
+}
+const MAX_PER_ANSWER = 10;
+const LABELS = {
+    money: "an amount",
+    percent: "a percentage",
+    quantity: "a quantity",
+    date: "a date",
+    email: "an email address",
+    iban: "an IBAN",
+    uuid: "an identifier",
+    reference: "a reference",
+    number: "a number",
+};
+// ---------------------------------------------------------------- the model tier (§9.1)
+const CONTEXT_CHARS = 200_000;
+const STATUSES = new Set(["grounded", "ungrounded", "contradicted", "no_fact"]);
+/** What a grounding service is sent for one answer: the text, the exact tier's facts, and the
+ * context it should rest on (the newest 200,000 characters of it). */
+export function groundingRequest(sessionId, answer, packs = []) {
+    const raw = answer.evidence.raw;
+    const reference = referenceContext(answer.text, packs);
+    return {
+        session_id: sessionId,
+        answer_seq: answer.seq,
+        answer: answer.text,
+        facts: factsIn(answer.text).map((f) => ({ kind: f.kind, value: f.value })),
+        context: raw.length > CONTEXT_CHARS ? raw.slice(raw.length - CONTEXT_CHARS) : raw,
+        // §10: what the organisation's own packs say about it, when they say anything
+        ...(reference.length ? { reference } : {}),
+    };
+}
+/** A grounding service's answer, checked: `{claims, model?, version?}`, or null (an outage). */
+export function parseGroundingAnswer(v) {
+    if (typeof v !== "object" || v === null || Array.isArray(v))
+        return null;
+    const a = v;
+    if (!Array.isArray(a.claims) || a.claims.length > 100)
+        return null;
+    const claims = [];
+    for (const c of a.claims) {
+        if (typeof c !== "object" || c === null)
+            return null;
+        const { text, status, score } = c;
+        if (typeof text !== "string" || text.length > 2000 || typeof status !== "string")
+            return null;
+        if (!STATUSES.has(status))
+            return null;
+        if (score !== undefined && (typeof score !== "number" || !(score >= 0 && score <= 1)))
+            return null;
+        claims.push({
+            text,
+            status: status,
+            ...(score !== undefined ? { score } : {}),
+        });
+    }
+    const short = (x) => typeof x === "string" && x.length > 0 && x.length <= 128;
+    return {
+        claims,
+        ...(short(a.model) ? { model: a.model } : {}),
+        ...(short(a.version) ? { version: a.version } : {}),
+    };
+}
+/** §9.1: UNGROUNDED_CLAIM and CONTRADICTED_CLAIM from the verdicts a grounding service gave,
+ * recorded as `control {action: "grounding", via: "model"}` with the claims as the body. */
+export function modelGroundingFindings(lines, bodies) {
+    const out = [];
+    for (const l of lines) {
+        const e = JSON.parse(l);
+        if (e.kind !== "control" || e.meta.action !== "grounding" || e.meta.via !== "model")
+            continue;
+        const seq = e.meta.answer_seq;
+        const b = bodies(e.body_hash);
+        if (typeof seq !== "number" || !b)
+            continue;
+        let parsed = null;
+        try {
+            parsed = parseGroundingAnswer(JSON.parse(decoder.decode(b)));
+        }
+        catch { }
+        for (const [index, c] of (parsed?.claims ?? []).entries()) {
+            if (c.status === "contradicted")
+                out.push({
+                    code: "CONTRADICTED_CLAIM",
+                    source: "hallucination",
+                    severity: "warning",
+                    ref: { seq, index, via: "model" },
+                    detail: "A claim in the answer is contradicted by what the agent was given, says the grounding service.",
+                });
+            else if (c.status === "ungrounded")
+                out.push({
+                    code: "UNGROUNDED_CLAIM",
+                    source: "hallucination",
+                    severity: "caution",
+                    ref: { seq, index, via: "model" },
+                    detail: "A claim in the answer has no support in what the agent was given, says the grounding service.",
+                });
+        }
+    }
+    return out;
+}
+/** spec/findings.md §9-§10: for each fact in a final answer, CONTRADICTED_CLAIM when a reference
+ * pack says otherwise, else UNGROUNDED_CLAIM when nothing the agent was given holds it (at most 10
+ * per answer); then the grounding service's verdicts (§9.1). */
+export function groundingFindings(lines, bodies, packs = []) {
+    return [...exactFindings(lines, bodies, packs), ...modelGroundingFindings(lines, bodies)];
+}
+function exactFindings(lines, bodies, packs) {
+    const out = [];
+    for (const a of answersOf(lines, bodies)) {
+        const facts = factsIn(a.text);
+        let n = 0;
+        for (const [index, f] of facts.entries()) {
+            if (n >= MAX_PER_ANSWER)
+                break;
+            // a pack is the organisation's own word: it comes before what a tool said
+            const said = checkAgainstPacks(a.text, f, packs);
+            if (said?.status === "contradicted") {
+                n++;
+                out.push({
+                    code: "CONTRADICTED_CLAIM",
+                    source: "hallucination",
+                    severity: "warning",
+                    ref: {
+                        seq: a.seq,
+                        kind: f.kind,
+                        index,
+                        via: "pack",
+                        pack: said.pack.id,
+                        fact: said.fact.id,
+                    },
+                    detail: `The answer states ${LABELS[f.kind] ?? "a value"} that the reference pack ${said.pack.id} contradicts (${said.fact.id}${said.fact.source ? `, ${said.fact.source}` : ""}).`,
+                });
+                continue;
+            }
+            if (said?.status === "grounded" || a.evidence.holds(f))
+                continue;
+            n++;
+            out.push({
+                code: "UNGROUNDED_CLAIM",
+                source: "hallucination",
+                severity: "caution",
+                ref: { seq: a.seq, kind: f.kind, index, via: "exact" },
+                detail: `The answer states ${LABELS[f.kind] ?? "a value"} that nothing the agent was given holds: it may be made up.`,
+            });
+        }
+    }
+    return out;
+}
+// ---------------------------------------------------------------- the judge (§12.1)
+const STEPS = 100;
+/** What a judge is sent for one answer: the answer and the trajectory before it (its newest 100
+ * steps: tools called, whether they failed, earlier answers, the findings so far). */
+export function judgeRequest(sessionId, answer, lines) {
+    const steps = [];
+    const callTools = new Map();
+    for (const l of lines) {
+        const e = JSON.parse(l);
+        if (e.seq >= answer.seq)
+            break;
+        const m = e.meta;
+        if (e.kind === "tool.call" && typeof m.tool === "string") {
+            callTools.set(e.seq, m.tool);
+            steps.push({ seq: e.seq, kind: "tool.call", tool: m.tool });
+        }
+        else if (e.kind === "tool.result" &&
+            typeof m.call_seq === "number" &&
+            callTools.has(m.call_seq))
+            steps.push({
+                seq: e.seq,
+                kind: "tool.result",
+                tool: callTools.get(m.call_seq),
+                error: m.is_error === true,
+            });
+        else if (e.kind === "sdk.event" &&
+            (m.type === "tool.call" || m.type === "tool.result") &&
+            typeof m.name === "string")
+            steps.push({ seq: e.seq, kind: m.type, tool: m.name });
+        else if (e.kind === "finding" && typeof m.code === "string")
+            steps.push({ seq: e.seq, kind: "finding", code: m.code });
+    }
+    return {
+        session_id: sessionId,
+        answer_seq: answer.seq,
+        answer: answer.text,
+        steps: steps.slice(-STEPS),
+    };
+}
+/** A judge's answer, checked: `{steps: [{rubric, pass, reason?}], model?, version?}`, or null. */
+export function parseJudgeAnswer(v) {
+    if (typeof v !== "object" || v === null || Array.isArray(v))
+        return null;
+    const a = v;
+    if (!Array.isArray(a.steps) || a.steps.length > 50)
+        return null;
+    const steps = [];
+    for (const s of a.steps) {
+        if (typeof s !== "object" || s === null)
+            return null;
+        const { rubric, pass, reason } = s;
+        if (typeof rubric !== "string" || rubric.length < 1 || rubric.length > 200)
+            return null;
+        if (typeof pass !== "boolean")
+            return null;
+        if (reason !== undefined && (typeof reason !== "string" || reason.length > 500))
+            return null;
+        steps.push({ rubric, pass, ...(reason !== undefined ? { reason } : {}) });
+    }
+    const short = (x) => typeof x === "string" && x.length > 0 && x.length <= 128;
+    return {
+        steps,
+        ...(short(a.model) ? { model: a.model } : {}),
+        ...(short(a.version) ? { version: a.version } : {}),
+    };
+}
+/** §12.1: JUDGE_FAILED for each rubric step a judge failed, from its recorded verdicts. */
+export function judgeFindings(lines, bodies) {
+    const out = [];
+    for (const l of lines) {
+        const e = JSON.parse(l);
+        if (e.kind !== "control" || e.meta.action !== "judge" || e.meta.via !== "model")
+            continue;
+        const seq = e.meta.answer_seq;
+        const b = bodies(e.body_hash);
+        if (typeof seq !== "number" || !b)
+            continue;
+        let parsed = null;
+        try {
+            parsed = parseJudgeAnswer(JSON.parse(decoder.decode(b)));
+        }
+        catch { }
+        for (const [index, s] of (parsed?.steps ?? []).entries())
+            if (!s.pass)
+                out.push({
+                    code: "JUDGE_FAILED",
+                    source: "hallucination",
+                    severity: "caution",
+                    ref: { seq, index, via: "judge" },
+                    detail: `The judge failed the answer on: ${s.rubric}`,
+                });
+    }
+    return out;
+}
+// ---------------------------------------------------------------- holding an answer (§13, H7)
+/** spec/findings.md §13: what can hold an answer (BLACKBOX_HOLD_ANSWERS). */
+export const HOLD_TRIGGERS = [
+    "contradicted",
+    "money",
+    "ids",
+    "quantity",
+    "percent",
+    "date",
+];
+const ID_KINDS = new Set(["email", "iban", "uuid", "reference", "number"]);
+/** §13: why an answer should be held before it's delivered: each fact a pack contradicts (with
+ * `contradicted`), and each ungrounded fact whose kind is a trigger (`ids` covers the identifiers).
+ * Empty: deliver it. */
+export function answerRisk(answer, packs, triggers) {
+    const out = [];
+    for (const [index, f] of factsIn(answer.text).entries()) {
+        const said = checkAgainstPacks(answer.text, f, packs);
+        if (said?.status === "contradicted") {
+            if (triggers.includes("contradicted"))
+                out.push({ index, kind: f.kind, status: "contradicted" });
+            continue;
+        }
+        if (said?.status === "grounded" || answer.evidence.holds(f))
+            continue;
+        if (triggers.includes(ID_KINDS.has(f.kind) ? "ids" : f.kind))
+            out.push({ index, kind: f.kind, status: "ungrounded" });
+    }
+    return out;
+}

package/dist/analysis/hallucination.d.ts

@@ -0,0 +1,32 @@
+import { type Bodies, type Call } from "../reconcile/record.ts";
+import type { Finding } from "./index.ts";
+/**
+ * The first way `value` breaks `schema`, or null. ponytail: the JSON Schema subset tools use
+ * (type, required, properties, additionalProperties: false, enum, items), not $ref or anyOf.
+ */
+export declare function schemaProblem(schema: unknown, value: unknown, path?: string): string | null;
+/** Arabic-Indic digits as Western ones. */
+export declare const western: (s: string) => string;
+/** Lower case, Western digits, and no separators, so `784-1990 1234567` matches `78419901234567`. */
+export declare function normaliseValue(s: string): string;
+/** The identifying values in a string (Western digits) and where they are, skipping `taken` spans. */
+export declare function identifierSpans(s: string, taken?: Array<[number, number]>): Array<{
+    kind: string;
+    value: string;
+    at: number;
+    end: number;
+}>;
+/** The identifying values in tool arguments' strings: emails, IBANs, UUIDs, references, long
+ * numbers. ponytail: JSON numbers are skipped (counts, sizes, epoch times), as are generated fields. */
+export declare function identifiersIn(args: unknown, path?: string): Array<{
+    kind: string;
+    path: string;
+    value: string;
+}>;
+/** `1,250.50` → `1250.5`: no separators, leading or trailing zeros. */
+export declare function canonicalNumber(s: string): string;
+/** The answer's text: Anthropic and Chat Completions text blocks, Responses message items. */
+export declare function answerText(c: Call): string;
+/** spec/findings.md §8: UNKNOWN_TOOL, TOOL_ARGS_INVALID, FABRICATED_VALUE, SUCCESS_AFTER_ERROR,
+ * PHANTOM_RESULT, in record order. */
+export declare function toolHallucinations(lines: readonly string[], bodies: Bodies): Finding[];