@sanity/ailf 4.5.0 → 5.0.0

package/dist/pipeline/compute-attribution.d.ts

@@ -0,0 +1,80 @@
+/**
+ * pipeline/compute-attribution.ts
+ *
+ * Pure three-signal attribution ensemble helper (v0).
+ *
+ * Computes per-document attribution scores for a single grader judgment
+ * using three signals:
+ *   - citation:   the grader's explicit doc citation with a role weight
+ *   - canonical:  whether the doc appears in the task's declared context.docs
+ *   - retrieved:  whether the doc slug was visited by the agent (agentic mode)
+ *
+ * No AppContext, no filesystem I/O — this is a pure function importable
+ * by calibration scripts and orchestration steps alike.
+ *
+ * Weights (v0): citation=0.55, canonical=0.30, retrieved=0.15.
+ * Embedding model: "none" (v0 — no embedding calls yet).
+ *
+ * @see docs/decisions/D0049-shared-confidence-contract.md
+ * @see docs/decisions/D0050-per-entry-attribution-layout.md
+ * @see docs/decisions/D0052-judgment-ref-granularity.md
+ * @see docs/design-docs/actionability-ladder/04-per-document-attribution-ensemble.md
+ */
+import type { Confidence, DocumentRef, GraderJudgment, JudgmentAttribution } from "../_vendor/ailf-core/index.d.ts";
+/**
+ * Widened structural type for ensemble weights. Calibration / grid-search
+ * callers pass dynamic `number`-valued tuples — they cannot satisfy the
+ * literal type produced by `typeof V0_WEIGHTS as const`.
+ */
+export interface EnsembleWeights {
+    citation: number;
+    canonical: number;
+    retrieved: number;
+}
+/** v0 ensemble weights. Callers may pass custom weights for calibration. */
+export declare const V0_WEIGHTS: EnsembleWeights;
+/**
+ * Compute the per-document attribution for a single grader judgment.
+ *
+ * The function is side-effect free except for the `reliability` accumulator —
+ * the caller owns the `GraderReliability`-shaped mutable object and should
+ * persist it after iterating over all judgments in a run.
+ *
+ * Hallucination short-circuit: when a candidate doc has a citation AND
+ * `doc.slug` is absent from `judgment.hallucinationCheckedAgainst`,
+ * the doc's attribution short-circuits to `score: 0`,
+ * `confidence.level: "low"`, `signalsPresent: 0`, and
+ * `reliability.hallucinationCount` is incremented. The function does NOT
+ * throw — the pipeline continues and the count is persisted by the caller.
+ *
+ * @param judgment       - The grader judgment being attributed.
+ * @param candidates     - DocumentRefs to compute attribution for. Callers
+ *                         control the set (contextDocs ∪ cited docs ∪
+ *                         retrieved docs).
+ * @param contextDocIds  - Set of documentIds declared in task.context.docs
+ *                         (the canonical signal source).
+ * @param retrievedSlugs - Set of doc slugs visited by the agent, or
+ *                         `undefined` when running in baseline mode (the
+ *                         retrieved signal is then dropped from the ensemble).
+ * @param reliability    - Mutable accumulator; `hallucinationCount` is
+ *                         incremented for each hallucinated citation.
+ * @param weights        - Ensemble weights; defaults to V0_WEIGHTS.
+ */
+export declare function computeJudgmentAttribution(judgment: GraderJudgment, candidates: DocumentRef[], contextDocIds: ReadonlySet<string>, retrievedSlugs: ReadonlySet<string> | undefined, reliability: {
+    hallucinationCount: number;
+}, weights?: EnsembleWeights): JudgmentAttribution;
+/**
+ * Derive a `Confidence` triple from the array of present signal values
+ * using standard-deviation-based agreement scoring.
+ *
+ * When `present.length <= 1`, the result is always `"low"` — a single
+ * signal cannot speak to agreement.
+ *
+ * The `0.5` normalization constant is the maximum standard deviation for
+ * a 2-or-3-point series whose values are in [0, 1] (achieved by
+ * [0, 1] or [0, 0, 1] patterns). Dividing the actual stdev by 0.5
+ * normalises agreement to [0, 1].
+ *
+ * Buckets: agreement > 0.7 → "high", > 0.4 → "medium", else → "low".
+ */
+export declare function deriveEnsembleConfidence(present: readonly number[]): Confidence;

package/dist/pipeline/compute-attribution.js

@@ -0,0 +1,196 @@
+/**
+ * pipeline/compute-attribution.ts
+ *
+ * Pure three-signal attribution ensemble helper (v0).
+ *
+ * Computes per-document attribution scores for a single grader judgment
+ * using three signals:
+ *   - citation:   the grader's explicit doc citation with a role weight
+ *   - canonical:  whether the doc appears in the task's declared context.docs
+ *   - retrieved:  whether the doc slug was visited by the agent (agentic mode)
+ *
+ * No AppContext, no filesystem I/O — this is a pure function importable
+ * by calibration scripts and orchestration steps alike.
+ *
+ * Weights (v0): citation=0.55, canonical=0.30, retrieved=0.15.
+ * Embedding model: "none" (v0 — no embedding calls yet).
+ *
+ * @see docs/decisions/D0049-shared-confidence-contract.md
+ * @see docs/decisions/D0050-per-entry-attribution-layout.md
+ * @see docs/decisions/D0052-judgment-ref-granularity.md
+ * @see docs/design-docs/actionability-ladder/04-per-document-attribution-ensemble.md
+ */
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+const ROLE_WEIGHTS = {
+    supports: 1.0,
+    contradicts: 0.8,
+    missing: 0.6,
+    irrelevant: 0.0,
+};
+/** v0 ensemble weights. Callers may pass custom weights for calibration. */
+export const V0_WEIGHTS = {
+    citation: 0.55,
+    canonical: 0.3,
+    retrieved: 0.15,
+};
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+/**
+ * Compute the per-document attribution for a single grader judgment.
+ *
+ * The function is side-effect free except for the `reliability` accumulator —
+ * the caller owns the `GraderReliability`-shaped mutable object and should
+ * persist it after iterating over all judgments in a run.
+ *
+ * Hallucination short-circuit: when a candidate doc has a citation AND
+ * `doc.slug` is absent from `judgment.hallucinationCheckedAgainst`,
+ * the doc's attribution short-circuits to `score: 0`,
+ * `confidence.level: "low"`, `signalsPresent: 0`, and
+ * `reliability.hallucinationCount` is incremented. The function does NOT
+ * throw — the pipeline continues and the count is persisted by the caller.
+ *
+ * @param judgment       - The grader judgment being attributed.
+ * @param candidates     - DocumentRefs to compute attribution for. Callers
+ *                         control the set (contextDocs ∪ cited docs ∪
+ *                         retrieved docs).
+ * @param contextDocIds  - Set of documentIds declared in task.context.docs
+ *                         (the canonical signal source).
+ * @param retrievedSlugs - Set of doc slugs visited by the agent, or
+ *                         `undefined` when running in baseline mode (the
+ *                         retrieved signal is then dropped from the ensemble).
+ * @param reliability    - Mutable accumulator; `hallucinationCount` is
+ *                         incremented for each hallucinated citation.
+ * @param weights        - Ensemble weights; defaults to V0_WEIGHTS.
+ */
+export function computeJudgmentAttribution(judgment, candidates, contextDocIds, retrievedSlugs, reliability, weights = V0_WEIGHTS) {
+    const resolvableSet = new Set(judgment.hallucinationCheckedAgainst);
+    // Build a map of citations by documentId for O(1) lookups (D0052).
+    const citationsByDocId = new Map();
+    for (const cit of judgment.docCitations) {
+        citationsByDocId.set(cit.documentId, cit);
+    }
+    const attributions = [];
+    // Defensive dedup by documentId. Callers may pass candidates that contain
+    // duplicate DocumentRef instances sharing a documentId (e.g. when
+    // contextSlugs has duplicate slug entries or two manifest slug aliases
+    // resolve to the same documentId). Without dedup, the hallucinationCount
+    // accumulator and the per-doc attribution array would double-count.
+    const seenDocIds = new Set();
+    for (const candidate of candidates) {
+        if (seenDocIds.has(candidate.documentId))
+            continue;
+        seenDocIds.add(candidate.documentId);
+        const cit = citationsByDocId.get(candidate.documentId);
+        // Hallucination short-circuit (Pitfall #11, T-04-01-01).
+        //
+        // A citation is hallucinated when:
+        //   (a) the candidate has a slug and that slug is not in resolvableSet, OR
+        //   (b) the candidate has no slug — we cannot verify resolvability, so
+        //       treat it conservatively as a hallucination (D0052 keys
+        //       resolvableSet by slug; no slug means no positive proof of
+        //       resolvability).
+        //
+        // Score is forced to 0 and reliability.hallucinationCount is incremented
+        // so downstream consumers can audit citation grounding without
+        // re-deriving the set.
+        if (cit && (!candidate.slug || !resolvableSet.has(candidate.slug))) {
+            reliability.hallucinationCount += 1;
+            attributions.push({
+                documentId: candidate.documentId,
+                ...(candidate.slug !== undefined ? { slug: candidate.slug } : {}),
+                score: 0,
+                signals: {},
+                confidence: {
+                    level: "low",
+                    signalsPresent: 0,
+                    derivation: "ensemble-stdev",
+                },
+            });
+            continue;
+        }
+        // Citation signal: role weight when cited; undefined when not cited at all.
+        // Do NOT coerce to 0 — undefined signals are dropped from signalsPresent.
+        const citation = cit
+            ? ROLE_WEIGHTS[cit.role]
+            : undefined;
+        // Canonical signal: binary presence in task's context.docs (always present).
+        const canonical = contextDocIds.has(candidate.documentId) ? 1 : 0;
+        // Retrieved signal: slug presence in agent retrieval set, or undefined
+        // when running in baseline mode (agentBehavior absent — Pitfall #4).
+        const retrieved = retrievedSlugs === undefined
+            ? undefined
+            : retrievedSlugs.has(candidate.slug ?? "")
+                ? 1
+                : 0;
+        // Collect present (non-undefined) signals for confidence derivation.
+        const present = [citation, canonical, retrieved].filter((s) => s !== undefined);
+        const score = clamp((citation ?? 0) * weights.citation +
+            canonical * weights.canonical +
+            (retrieved ?? 0) * weights.retrieved, 0, 1);
+        // Build the signals object omitting undefined members so JSON output is clean.
+        const signals = { canonical };
+        if (citation !== undefined)
+            signals.citation = citation;
+        if (retrieved !== undefined)
+            signals.retrieved = retrieved;
+        attributions.push({
+            documentId: candidate.documentId,
+            ...(candidate.slug !== undefined ? { slug: candidate.slug } : {}),
+            score,
+            signals,
+            confidence: deriveEnsembleConfidence(present),
+        });
+    }
+    const judgmentRef = `${judgment.taskId}--${judgment.modelId}--${judgment.dimension}`;
+    return {
+        judgmentRef,
+        taskId: judgment.taskId,
+        modelId: judgment.modelId,
+        dimension: judgment.dimension,
+        attributions,
+        hallucinationCheckedAgainst: judgment.hallucinationCheckedAgainst,
+    };
+}
+/**
+ * Derive a `Confidence` triple from the array of present signal values
+ * using standard-deviation-based agreement scoring.
+ *
+ * When `present.length <= 1`, the result is always `"low"` — a single
+ * signal cannot speak to agreement.
+ *
+ * The `0.5` normalization constant is the maximum standard deviation for
+ * a 2-or-3-point series whose values are in [0, 1] (achieved by
+ * [0, 1] or [0, 0, 1] patterns). Dividing the actual stdev by 0.5
+ * normalises agreement to [0, 1].
+ *
+ * Buckets: agreement > 0.7 → "high", > 0.4 → "medium", else → "low".
+ */
+export function deriveEnsembleConfidence(present) {
+    if (present.length <= 1) {
+        return {
+            level: "low",
+            signalsPresent: present.length,
+            derivation: "ensemble-stdev",
+        };
+    }
+    const mean = present.reduce((sum, v) => sum + v, 0) / present.length;
+    const variance = present.reduce((sum, v) => sum + (v - mean) ** 2, 0) / present.length;
+    const stdev = Math.sqrt(variance);
+    // Normalise: max stdev for [0,1]-bounded series ≈ 0.5
+    const agreement = 1 - stdev / 0.5;
+    const level = agreement > 0.7 ? "high" : agreement > 0.4 ? "medium" : "low";
+    return {
+        level,
+        signalsPresent: present.length,
+        derivation: "ensemble-stdev",
+    };
+}
+// ---------------------------------------------------------------------------
+// Internal helpers
+// ---------------------------------------------------------------------------
+function clamp(value, min, max) {
+    return Math.min(max, Math.max(min, value));
+}

package/dist/pipeline/failure-modes.d.ts

@@ -1,41 +1,76 @@
 /**
  * pipeline/failure-modes.ts
  *
- * Keyword-based failure mode classifier for grader reasoning text,
- * cross-referenced with ceiling decomposition data.
+ * Ceiling-cross-check failure-mode validator + report assembly.
  *
- * Phase 3a of the Scenario Matrix implementation.
+ * The grader emits `failureMode` directly under the per-dimension taxonomy
+ * (Plan 03-02 — `packages/eval/src/grader/`). This module consumes the
+ * grader's emission as the source of truth and uses the surviving ceiling
+ * decomposition (`classifyByCeiling`) only as a CONFIDENCE VALIDATOR — it
+ * cross-checks the emitted mode against structural score signals and emits
+ * a D0049 `Confidence` triple stamped with `derivation: "ceiling-cross-check"`.
  *
- * The classifier uses two signal sources:
- * 1. Keyword matching on grader reason text (primary)
- * 2. Ceiling decomposition structural signals (supplementary)
+ * The legacy keyword-pattern classifier (and its five regex pattern
+ * constants) was deleted in Plan 03-03 — its production coverage was ~1%
+ * (Doc 03 §"Where classification lives"), and resurrecting it as a fallback
+ * is explicitly out of scope.
  *
- * When both sources agree, confidence is boosted. When only ceiling
- * signals are available, they serve as a fallback for unclassified cases.
- *
- * @see docs/archive/exec-plans/scenario-matrix-implementation/phase-3-gap-analysis.md
+ * @see docs/decisions/D0005-grader-model-separation.md — single grader emits
+ *      failureMode under the per-dimension taxonomy
+ * @see docs/decisions/D0049-shared-confidence-contract.md — Confidence triple
+ *      shape and `ceiling-cross-check` derivation tag
  */
-import type { FailureMode, FailureModeReport, FeatureScore, GraderJudgment } from "./types.js";
+import type { Confidence } from "../_vendor/ailf-core/index.d.ts";
+import type { FailureModeReport, FeatureScore, GraderJudgment } from "./types.js";
 /**
  * Build a complete failure mode report from grader judgments and scores.
  *
+ * The grader-emitted `judgment.failureMode` is the source of truth (Plan
+ * 03-03 — keyword classifier deleted). `validateFailureMode` cross-checks
+ * the emission against ceiling decomposition and stamps a D0049 confidence.
+ *
+ * The `FailureMode` triple shape (`mode`, `confidence`, `source`) is
+ * preserved for backward compatibility with downstream consumers
+ * (gap-analysis, manifest emission) — the bucketed `confidence` enum maps
+ * 1:1 from `Confidence.level`, and `source` is always `"ceiling"` now that
+ * the keyword path is gone.
+ *
  * @param judgments - All grader judgments from the evaluation
  * @param scores - Per-area feature scores (for ceiling decomposition)
  * @returns Failure mode report with per-area breakdowns
  */
 export declare function buildFailureModeReport(judgments: GraderJudgment[], scores: FeatureScore[]): FailureModeReport;
 /**
- * Classify the failure mode of a low-scoring grader judgment.
+ * Cross-check a grader-emitted `failureMode` against ceiling decomposition
+ * and emit a D0049 `Confidence` triple stamped with
+ * `derivation: "ceiling-cross-check"`.
  *
- * Uses keyword matching on the reason text, then cross-references with
- * ceiling decomposition data for structural confirmation.
+ * Replaces the deleted keyword-pattern + ceiling combine classifier — the
+ * grader's emission is now the source of truth for the mode itself; this
+ * function only stamps confidence based on whether the structural ceiling
+ * signal agrees.
  *
- * @param judgment - The grader judgment to classify
+ *  - `level: "high"` (`signalsPresent: 2`) — grader emission and ceiling
+ *    decomposition agree on the same mode.
+ *  - `level: "medium"` (`signalsPresent: 2`) — both signals present but
+ *    disagree. The live pipeline increments
+ *    `GraderReliability.failureModeCalibration` ONLY on this branch — a
+ *    true calibration miss requires both signals to be present.
+ *  - `level: "low"` (`signalsPresent: 1`) — only the grader's emission;
+ *    ceiling decomposition produced no structural signal. Not a
+ *    calibration miss (we have nothing to cross-check against).
+ *  - `level: "low"` (`signalsPresent: 0`) — passing scores
+ *    (`>= CLASSIFICATION_THRESHOLD`) don't classify; emit absent.
+ *
+ * @param judgment - The grader judgment carrying `failureMode` + `score`
  * @param ceilingScore - The area's ceiling score (with-docs best case)
  * @param floorScore - The area's floor score (no-docs baseline)
- * @returns Classified failure mode with confidence level
+ * @returns D0049 Confidence triple stamped `derivation: "ceiling-cross-check"`
+ *
+ * @see docs/decisions/D0005-grader-model-separation.md
+ * @see docs/decisions/D0049-shared-confidence-contract.md
  */
-export declare function classifyFailureMode(judgment: GraderJudgment, ceilingScore: number, floorScore: number): FailureMode;
+export declare function validateFailureMode(judgment: GraderJudgment, ceilingScore: number, floorScore: number): Confidence;
 /**
  * Format a failure mode report for console output.
  */