@sanity/ailf 4.6.0 → 6.0.0

package/dist/orchestration/steps/compute-attribution-step.js

@@ -0,0 +1,279 @@
+/**
+ * Pipeline step: Per-judgment attribution ensemble (v0).
+ *
+ * Reads `grader-judgments.json` and `score-summary.json` from the latest
+ * results, calls the pure `computeJudgmentAttribution(...)` helper for each
+ * judgment, and emits:
+ *
+ *   - One `perEntryAttribution` artifact per judgment at
+ *     `runs/{runId}/attribution/{entryKey}.json`
+ *   - One `attributionMeta` artifact at
+ *     `runs/{runId}/attribution-meta.json`
+ *
+ * Additionally, when any hallucinated citations are detected, the step
+ * atomically rewrites `score-summary.json` to persist
+ * `graderReliability.hallucinationCount` (D-05 — only this one direct-
+ * mutation path uses the temp+rename pattern; all artifact emissions go
+ * through `ctx.artifactWriter.emit` which handles atomicity internally).
+ *
+ * This step is `optional: true` — it self-skips when either
+ * `grader-judgments.json` or `score-summary.json` is missing, so
+ * non-graded runs are unaffected.
+ *
+ * Task → judgment join (D-10): `judgment.taskId` is the promptfoo row
+ * description, which for literacy mode is `"${task.title} (gold|baseline)"`.
+ * The join strips the variant suffix and looks up in a triple-keyed cache
+ * by `task.title` (primary), `task.description`, and `task.id` (defensive
+ * fallbacks for non-literacy modes).
+ *
+ * Retrieved signal (D-11/D-12): `FeatureAgentBehavior.feature` is the join
+ * key — for literacy mode it equals `task.area` exactly (compiler propagates
+ * `task.area → __featureArea → ab.feature`).
+ *
+ * @see docs/decisions/D0033-unified-artifact-writer.md
+ * @see docs/decisions/D0049-shared-confidence-contract.md
+ * @see docs/decisions/D0050-per-entry-attribution-layout.md
+ * @see docs/decisions/D0052-judgment-ref-granularity.md
+ */
+import { existsSync, readFileSync, renameSync, unlinkSync, writeFileSync, } from "node:fs";
+import { resolve } from "node:path";
+import { isSlugRef } from "../../_vendor/ailf-core/index.js";
+import { calibrationSetVersion, embeddingModel, ensembleVersion, } from "../../pipeline/attribution.js";
+import { V0_WEIGHTS, computeJudgmentAttribution, } from "../../pipeline/compute-attribution.js";
+// ---------------------------------------------------------------------------
+// Step implementation
+// ---------------------------------------------------------------------------
+export class ComputeAttributionStep {
+    name = "compute-attribution";
+    optional = true;
+    check(ctx) {
+        const issues = [];
+        const judgmentsPath = resolve(ctx.config.rootDir, "results", "latest", "grader-judgments.json");
+        const summaryPath = resolve(ctx.config.rootDir, "results", "latest", "score-summary.json");
+        if (!existsSync(judgmentsPath)) {
+            issues.push({
+                message: "No grader-judgments.json — attribution computation will skip",
+                severity: "warning",
+                source: "compute-attribution",
+            });
+            return issues;
+        }
+        // WARN 5: documentManifest must be present and non-empty (gap-analysis
+        // enriches it). Without it, the canonical signal is permanently 0 and
+        // the three-signal ensemble silently degrades to citation-only.
+        if (existsSync(summaryPath)) {
+            try {
+                const summary = JSON.parse(readFileSync(summaryPath, "utf-8"));
+                const dm = summary.documentManifest;
+                if (!dm || dm.length === 0) {
+                    issues.push({
+                        message: "documentManifest is empty — attribution canonical signal will be permanently 0. Ensure gap-analysis runs before compute-attribution.",
+                        severity: "warning",
+                        source: "compute-attribution",
+                    });
+                }
+            }
+            catch {
+                // Surfaces at execute() with a failed StepResult.
+            }
+        }
+        return issues;
+    }
+    async execute(ctx, _state) {
+        const start = Date.now();
+        const root = ctx.config.rootDir;
+        const judgmentsPath = resolve(root, "results", "latest", "grader-judgments.json");
+        const summaryPath = resolve(root, "results", "latest", "score-summary.json");
+        if (!existsSync(judgmentsPath)) {
+            return { status: "skipped", reason: "No grader-judgments.json" };
+        }
+        if (!existsSync(summaryPath)) {
+            return { status: "skipped", reason: "No score-summary.json" };
+        }
+        try {
+            const judgments = JSON.parse(readFileSync(judgmentsPath, "utf-8"));
+            const summary = JSON.parse(readFileSync(summaryPath, "utf-8"));
+            // D-10: judgment.taskId is the promptfoo row description, sometimes
+            // suffixed "(gold)" / "(baseline)" by the literacy compiler
+            // (literacy/compiler.ts:184). Build a triple-keyed task cache:
+            //   - task.title  (primary for literacy — the compiler binds task.title
+            //                  as the row description before appending the suffix)
+            //   - task.description (defensive fallback for future modes)
+            //   - task.id     (defensive fallback for KP / agent-harness modes)
+            // Strip the variant suffix before lookup so the join matches.
+            const tasksByKey = new Map();
+            if (ctx.taskSource) {
+                try {
+                    const tasks = (await ctx.taskSource.loadTasks()).filter((t) => t.mode === "literacy");
+                    for (const t of tasks) {
+                        if (t.title)
+                            tasksByKey.set(t.title, t);
+                        if (t.description)
+                            tasksByKey.set(t.description, t);
+                        tasksByKey.set(t.id, t);
+                    }
+                }
+                catch (err) {
+                    // Surface the failure so operators see why the canonical signal
+                    // is permanently 0 for this run. Behavior is unchanged — we
+                    // degrade to an empty tasksByKey rather than failing the step.
+                    ctx.logger.warn(`[compute-attribution] taskSource.loadTasks() failed; canonical signal will be 0 for all judgments. ${err instanceof Error ? err.message : String(err)}`);
+                }
+            }
+            // D-12: FeatureAgentBehavior.feature is the correct field (NOT .area).
+            // Verified at packages/core/src/types/index.ts:1180-1190.
+            // D-11: for literacy mode, ab.feature === task.area via the compiler's
+            // __featureArea propagation (literacy/compiler.ts:180).
+            const visitedByFeature = new Map();
+            for (const ab of summary.agentBehavior ?? []) {
+                visitedByFeature.set(ab.feature, new Set(ab.docSlugsVisited ?? []));
+            }
+            // Build a slug→DocumentRef map from documentManifest for candidate resolution.
+            const manifestBySlug = new Map();
+            for (const ref of summary.documentManifest ?? []) {
+                if (ref.slug)
+                    manifestBySlug.set(ref.slug, ref);
+            }
+            const reliability = { hallucinationCount: 0 };
+            for (const j of judgments) {
+                // D-10: strip variant suffix before lookup — mirrors calculate-scores.ts:696.
+                const baseDesc = stripVariantSuffix(j.taskId);
+                const task = tasksByKey.get(baseDesc) ?? tasksByKey.get(j.taskId);
+                // Resolve context docs → DocumentRef[] via manifest lookup.
+                const contextSlugs = extractContextSlugs(task);
+                const contextDocs = contextSlugs
+                    .map((s) => manifestBySlug.get(s))
+                    .filter((r) => r !== undefined);
+                const contextDocIds = new Set(contextDocs.map((r) => r.documentId));
+                // D-11: retrieved signal keys by task.area (= ab.feature in literacy mode).
+                // When agentBehavior is absent (baseline run), drop the retrieved signal
+                // (pass undefined — Pitfall #4 / locked D-04).
+                const area = task?.area;
+                const retrievedSlugs = summary.agentBehavior === undefined || summary.agentBehavior === null
+                    ? undefined
+                    : area === undefined
+                        ? new Set()
+                        : (visitedByFeature.get(area) ?? new Set());
+                // Candidate set: contextDocs ∪ docs in the manifest cited by the judgment.
+                const citedDocIds = new Set(j.docCitations.map((c) => c.documentId));
+                const candidates = [
+                    ...contextDocs,
+                    ...Array.from(manifestBySlug.values()).filter((r) => !contextDocIds.has(r.documentId) &&
+                        (citedDocIds.has(r.documentId) ||
+                            (r.slug !== undefined &&
+                                (retrievedSlugs?.has(r.slug) ?? false)))),
+                ];
+                const judgmentAttribution = computeJudgmentAttribution(j, candidates, contextDocIds, retrievedSlugs, reliability, V0_WEIGHTS);
+                // D-06: pass { run, name } where name is the entry key.
+                // formatKeyFromAxes requires assoc.name for per-entry descriptors —
+                // forgetting it causes a hard throw at emit time.
+                const entryKey = `${j.taskId}--${j.modelId}--${j.dimension}`;
+                await ctx.artifactWriter.emit("perEntryAttribution", { run: ctx.runId, name: entryKey }, judgmentAttribution);
+            }
+            await ctx.artifactWriter.emit("attributionMeta", { run: ctx.runId }, {
+                ensembleVersion,
+                embeddingModel,
+                weights: V0_WEIGHTS,
+                calibrationSetVersion,
+            });
+            // Atomically persist reliability.hallucinationCount onto score-summary.json
+            // (D-05: the atomic write applies ONLY to this direct-mutation path;
+            // ctx.artifactWriter.emit handles atomicity for all other writes).
+            if (reliability.hallucinationCount > 0) {
+                // Build the updated graderReliability by merging only the new count.
+                // graderReliability.graderModel is a required field — spread to preserve
+                // all existing required fields before adding/updating hallucinationCount.
+                const existingReliability = summary.graderReliability;
+                const updatedReliability = existingReliability
+                    ? {
+                        ...existingReliability,
+                        hallucinationCount: (existingReliability.hallucinationCount ?? 0) +
+                            reliability.hallucinationCount,
+                    }
+                    : {
+                        // When there is no existing graderReliability, we cannot provide
+                        // the required graderModel field from the run context here.
+                        // Persist only the new counter under a minimal shape.
+                        graderModel: "unknown",
+                        hallucinationCount: reliability.hallucinationCount,
+                    };
+                const updated = {
+                    ...summary,
+                    graderReliability: updatedReliability,
+                };
+                const tmp = `${summaryPath}.tmp`;
+                try {
+                    writeFileSync(tmp, JSON.stringify(updated, null, 2));
+                    renameSync(tmp, summaryPath);
+                }
+                catch (err) {
+                    // Best-effort cleanup of the .tmp file so a stale leftover does
+                    // not confuse subsequent runs (cross-device move, EACCES, ENOSPC
+                    // after writeFileSync). Re-throw to surface the underlying error
+                    // via the outer try/catch's failed StepResult.
+                    try {
+                        unlinkSync(tmp);
+                    }
+                    catch {
+                        /* best-effort cleanup */
+                    }
+                    throw err;
+                }
+            }
+            return {
+                durationMs: Date.now() - start,
+                status: "success",
+                summary: `Attribution computed for ${judgments.length} judgments (${reliability.hallucinationCount} hallucinated citations)`,
+            };
+        }
+        catch (err) {
+            return {
+                durationMs: Date.now() - start,
+                error: err instanceof Error ? err.message : String(err),
+                status: "failed",
+            };
+        }
+    }
+}
+// ---------------------------------------------------------------------------
+// Internal helpers
+// ---------------------------------------------------------------------------
+/**
+ * Strip the literacy compiler's variant suffix from a row description.
+ * Mirrors the canonical Phase-3 strip site at
+ * packages/eval/src/pipeline/calculate-scores.ts:696 (D-10).
+ */
+function stripVariantSuffix(taskId) {
+    return taskId.replace(/\s*\((gold|baseline)\)\s*$/, "");
+}
+/** True for any ref whose shape carries a string-valued `slug` field. */
+function hasSlugField(ref) {
+    return "slug" in ref && typeof ref.slug === "string";
+}
+/**
+ * Extract slug strings from a task's context.docs array (D-09).
+ *
+ * Uses isSlugRef for SlugDocRef and falls back to the informational
+ * `slug` annotation on IdDocRef. PathDocRef refs (and the `path`
+ * annotation on IdDocRef) are intentionally NOT included — paths are
+ * filesystem-style and never match manifestBySlug, which is keyed by
+ * article slug. A PathDocRef contributes zero canonical signal (correct
+ * fallback) rather than polluting the slug lookup with stale keys.
+ */
+function extractContextSlugs(task) {
+    if (!task?.context?.docs)
+        return [];
+    const out = [];
+    for (const ref of task.context.docs) {
+        if (isSlugRef(ref)) {
+            out.push(ref.slug);
+        }
+        else if (hasSlugField(ref)) {
+            // IdDocRef carries an informational `slug` annotation — use it
+            // for the manifest lookup. PerspectiveDocRef has neither slug
+            // nor a usable lookup key and is skipped.
+            out.push(ref.slug);
+        }
+    }
+    return out;
+}

package/dist/orchestration/steps/gap-analysis-step.js

@@ -14,7 +14,7 @@
  *
  * This is an optional step — failure doesn't stop the pipeline.
  */
-import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
+import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync, } from "fs";
 import { join, resolve } from "path";
 import { assoc, isSlugRef } from "../../_vendor/ailf-core/index.js";
 import { emitFileContents } from "../../artifact-capture/emit-file.js";
@@ -151,19 +151,44 @@ export class GapAnalysisStep {
                     ...s,
                     documents: areaToDocRefs.get(s.feature),
                 }));
+                // Pitfall #11 hallucination cross-check (GRAD-05). The required
+                // `hallucinationCheckedAgainst` and per-citation `hallucinated`
+                // fields are populated here because `extractGraderJudgments`
+                // (the upstream emitter) does not have access to either the
+                // task contextDocs map or the run's document manifest. The
+                // populator mutates `judgments` in place and the rewrite below
+                // persists the enrichment back to disk so downstream consumers
+                // (Studio, gap-analysis followups) see populated fields.
+                const taskDocSlugs = new Map();
+                for (const [desc, refs] of descToDocRefs) {
+                    taskDocSlugs.set(desc, refs
+                        .map((r) => r.slug)
+                        .filter((s) => typeof s === "string" && s.length > 0));
+                }
+                const manifestSlugs = (documentManifest ?? [])
+                    .map((d) => d.slug)
+                    .filter((s) => typeof s === "string" && s.length > 0);
+                const { populateHallucinationFields } = await import("../../pipeline/calculate-scores.js");
+                populateHallucinationFields(judgments, taskDocSlugs, manifestSlugs);
+                // Atomic write — POSIX rename is atomic on the same filesystem,
+                // so a mid-write failure leaves either the prior file or the new
+                // file intact, never a half-written JSON document.
+                const tmpPath = `${judgmentsPath}.tmp`;
+                writeFileSync(tmpPath, JSON.stringify(judgments, null, 2));
+                renameSync(tmpPath, judgmentsPath);
             }
             // ── Per-test results (D0029: model output + metadata) ──────
             const testResultsPath = resolve(root, "results", "latest", "test-results.json");
             let testResults;
             if (existsSync(testResultsPath)) {
                 const rawTestResults = JSON.parse(readFileSync(testResultsPath, "utf-8"));
-                // Enrich with canonical docs (literacy mode only)
+                // Enrich with context docs (literacy mode only)
                 testResults = rawTestResults.map((tr) => {
                     if (!isLiteracyMode)
                         return tr;
                     const baseDesc = tr.taskId.replace(/\s*\((gold|baseline)\)\s*$/, "");
-                    const canonicalDocs = descToDocRefs.get(baseDesc);
-                    return canonicalDocs ? { ...tr, canonicalDocs } : tr;
+                    const contextDocs = descToDocRefs.get(baseDesc);
+                    return contextDocs ? { ...tr, contextDocs } : tr;
                 });
             }
             // ── Low-scoring judgments ────────────────────────────────────
@@ -182,8 +207,8 @@ export class GapAnalysisStep {
                     return j;
                 // Judgment taskId is the description with "(gold)" or "(baseline)" suffix
                 const baseDesc = j.taskId.replace(/\s*\((gold|baseline)\)\s*$/, "");
-                const canonicalDocs = descToDocRefs.get(baseDesc);
-                return canonicalDocs ? { ...j, canonicalDocs } : j;
+                const contextDocs = descToDocRefs.get(baseDesc);
+                return contextDocs ? { ...j, contextDocs } : j;
             });
             const enrichedSummary = {
                 ...scoreSummary,
@@ -194,7 +219,10 @@ export class GapAnalysisStep {
                 scores: enrichedScores,
                 ...(testResults !== undefined && { testResults }),
             };
-            writeFileSync(scoreSummaryPath, JSON.stringify(enrichedSummary, null, 2));
+            // Atomic write — see judgmentsPath above for rationale.
+            const scoreSummaryTmpPath = `${scoreSummaryPath}.tmp`;
+            writeFileSync(scoreSummaryTmpPath, JSON.stringify(enrichedSummary, null, 2));
+            renameSync(scoreSummaryTmpPath, scoreSummaryPath);
             // W0051 Slice 2 — failureModes is per-entry keyed by {mode, category};
             // one entry per classified FailureModeType. Zero-count categories are
             // skipped to keep the manifest honest about what the run surfaced.

package/dist/orchestration/steps/index.d.ts

@@ -5,6 +5,7 @@
  * AppContext instead of positional parameters.
  */
 export { CalculateScoresStep } from "./calculate-scores-step.js";
+export { ComputeAttributionStep } from "./compute-attribution-step.js";
 export { CompareStep } from "./compare-step.js";
 export { FetchDocsStep } from "./fetch-docs-step.js";
 export { GapAnalysisStep } from "./gap-analysis-step.js";

package/dist/orchestration/steps/index.js

@@ -5,6 +5,7 @@
  * AppContext instead of positional parameters.
  */
 export { CalculateScoresStep } from "./calculate-scores-step.js";
+export { ComputeAttributionStep } from "./compute-attribution-step.js";
 export { CompareStep } from "./compare-step.js";
 export { FetchDocsStep } from "./fetch-docs-step.js";
 export { GapAnalysisStep } from "./gap-analysis-step.js";

package/dist/pipeline/attribution.d.ts

@@ -17,6 +17,21 @@
  */
 import type { AttributionReport, ComparisonReport } from "./types.js";
 import type { ResolvedMappings } from "./resolve-mappings.js";
+/** v0 sentinel — no embedding call in v0; v1.2's flip to a real
+ *  model name (e.g., "text-embedding-3-small") mechanically forces
+ *  ensembleVersion's right segment to change, invalidating cached
+ *  weights downstream. */
+export declare const embeddingModel = "none";
+/** VER-01 D-02 — co-located ensemble version. Compound semver-ish
+ *  shape: "{algorithmVersion}+{embeddingModel}" (per D-02 of Phase
+ *  4 CONTEXT.md). Phase 1 landed the constant; Phase 4 wires the
+ *  compound shape and the bump-by-calibration discipline
+ *  (calibrate-attribution.ts is the ONLY allowed bumper). */
+export declare const ensembleVersion: "0.1.0+none";
+/** Version tag for the calibration set fixture co-located in the repo.
+ *  Bump when the fixture structure changes (e.g. when v1 moves the
+ *  calibration set to Content Lake or expands to ~30 stratified rows). */
+export declare const calibrationSetVersion = "v0-fixture";
 /**
  * Attribute score changes to individual documents.
  *

package/dist/pipeline/attribution.js

@@ -15,6 +15,21 @@
  * @see docs/design-docs/scenario-matrix/per-document-attribution.md
  * @see docs/archive/exec-plans/scenario-matrix-implementation/phase-2-impact-scenarios.md
  */
+/** v0 sentinel — no embedding call in v0; v1.2's flip to a real
+ *  model name (e.g., "text-embedding-3-small") mechanically forces
+ *  ensembleVersion's right segment to change, invalidating cached
+ *  weights downstream. */
+export const embeddingModel = "none";
+/** VER-01 D-02 — co-located ensemble version. Compound semver-ish
+ *  shape: "{algorithmVersion}+{embeddingModel}" (per D-02 of Phase
+ *  4 CONTEXT.md). Phase 1 landed the constant; Phase 4 wires the
+ *  compound shape and the bump-by-calibration discipline
+ *  (calibrate-attribution.ts is the ONLY allowed bumper). */
+export const ensembleVersion = `0.1.0+${embeddingModel}`;
+/** Version tag for the calibration set fixture co-located in the repo.
+ *  Bump when the fixture structure changes (e.g. when v1 moves the
+ *  calibration set to Content Lake or expands to ~30 stratified rows). */
+export const calibrationSetVersion = "v0-fixture";
 // ---------------------------------------------------------------------------
 // Public API
 // ---------------------------------------------------------------------------
@@ -59,11 +74,9 @@ export function attributeChanges(comparison, changedSlugs, mappings, noiseThresh
             // (shouldn't happen in practice, but handle gracefully)
             continue;
         }
-        // For area-level attribution: check which changed docs overlap
-        // with any task's canonical docs in this area
-        const areaCanonicalSlugs = new Set(areaTasks.flatMap(([, info]) => info.slugs));
-        const matchingSlugs = changedSlugs.filter((s) => areaCanonicalSlugs.has(s));
-        // Classify each task
+        // Classify each task. When no task in this area has a matching
+        // changed doc, every per-task classification will fall through to
+        // `uncorrelated` here — no separate area-level record is needed.
         for (const [taskId, taskInfo] of areaTasks) {
             const taskMatchingSlugs = taskInfo.slugs.filter((s) => changedSet.has(s));
             const classification = classifyAttribution(taskMatchingSlugs.length);
@@ -76,10 +89,6 @@ export function attributeChanges(comparison, changedSlugs, mappings, noiseThresh
                 withinNoiseFloor: Math.abs(areaDelta.delta) <= noiseThreshold,
             });
         }
-        // If no task-level matches but area has a delta, record area-level
-        if (areaTasks.length > 0 && matchingSlugs.length === 0) {
-            // All tasks in this area are uncorrelated — already handled above
-        }
     }
     // Find untracked documents: changed slugs not in ANY task's canonical docs
     const allTrackedSlugs = new Set([...taskCanonicalDocs.values()].flatMap((info) => info.slugs));

package/dist/pipeline/borderline-consensus-runner.d.ts

@@ -0,0 +1,63 @@
+/**
+ * pipeline/borderline-consensus-runner.ts
+ *
+ * GRAD-04 borderline-only intra-grader consensus runner. Thin sibling
+ * of runGraderConsistency — re-grades ONLY judgments where
+ * `isBorderline(score, thresholds)` returns true. Non-borderline
+ * judgments pass through unchanged.
+ *
+ * Per D0005 (grader-model separation), replicates the SAME pinned
+ * grader N times (default 3, configurable via
+ * RepoConfig.execution.borderlineReplications); NOT the inter-grader
+ * ensemble path. Doc 03 §"Multi-grader consensus" + the GRAD-04 "Doc 03
+ * source bug" callout pinned this to intra-grader replication only.
+ *
+ * The re-grade hook is supplied by the caller as a `regrade` callback.
+ * The composition root wires it to `gradeOnce` from grader-api.js with
+ * the response/rubric text drawn from the original Promptfoo result.
+ * The runner itself imports `gradeOnce` only as the default regrader
+ * fallback so unit tests can spy/inject without re-wiring.
+ *
+ * @see docs/decisions/D0005-grader-model-separation.md
+ * @see ./borderline-detector.ts — pure predicate
+ * @see ./grader-consistency.ts — JudgmentConsistency shape we emit
+ */
+import type { GraderJudgment, Logger } from "../_vendor/ailf-core/index.d.ts";
+import { gradeOnce } from "./grader-api.js";
+import { type JudgmentConsistency } from "./grader-consistency.js";
+/**
+ * Re-export `gradeOnce` so callers that need to wire the default regrader
+ * (composition-root, integration tests) can import the grader entry point
+ * from this module rather than rediscovering grader-api.js. The runner
+ * itself does not invoke `gradeOnce` — the caller-supplied `regrade`
+ * callback owns the live grader call (Pitfall 6 — runner stays pure wrt
+ * provider config).
+ */
+export { gradeOnce };
+export interface BorderlineConsensusOptions {
+    judgments: GraderJudgment[];
+    logger?: Logger;
+    /** Callback that re-grades a single judgment once. Returns a fresh score. */
+    regrade: (judgment: GraderJudgment) => Promise<number>;
+    /** Default 3 — see RepoConfig.execution.borderlineReplications. */
+    replications: number;
+    /** Severity boundaries from config/thresholds.ts (default [30, 50, 60]). */
+    thresholds: readonly number[];
+}
+export interface BorderlineConsensusResult {
+    consistencyByJudgment: Map<string, JudgmentConsistency>;
+    judgments: GraderJudgment[];
+}
+/**
+ * Run intra-grader consensus on the borderline subset of `judgments`.
+ *
+ * - Borderline (per `isBorderline(score, thresholds)`): re-grade
+ *   `replications` times via `regrade`; emit a `JudgmentConsistency`
+ *   keyed by `${taskId}::${dimension}::${modelId}`; merge the consensus
+ *   median back into the canonical judgment's `score`.
+ * - Non-borderline: pass through unchanged. Output array length == input.
+ *
+ * The function is order-preserving — the returned `judgments` array
+ * keeps the same element order as the input.
+ */
+export declare function runBorderlineConsensus(options: BorderlineConsensusOptions): Promise<BorderlineConsensusResult>;

package/dist/pipeline/borderline-consensus-runner.js

@@ -0,0 +1,124 @@
+/**
+ * pipeline/borderline-consensus-runner.ts
+ *
+ * GRAD-04 borderline-only intra-grader consensus runner. Thin sibling
+ * of runGraderConsistency — re-grades ONLY judgments where
+ * `isBorderline(score, thresholds)` returns true. Non-borderline
+ * judgments pass through unchanged.
+ *
+ * Per D0005 (grader-model separation), replicates the SAME pinned
+ * grader N times (default 3, configurable via
+ * RepoConfig.execution.borderlineReplications); NOT the inter-grader
+ * ensemble path. Doc 03 §"Multi-grader consensus" + the GRAD-04 "Doc 03
+ * source bug" callout pinned this to intra-grader replication only.
+ *
+ * The re-grade hook is supplied by the caller as a `regrade` callback.
+ * The composition root wires it to `gradeOnce` from grader-api.js with
+ * the response/rubric text drawn from the original Promptfoo result.
+ * The runner itself imports `gradeOnce` only as the default regrader
+ * fallback so unit tests can spy/inject without re-wiring.
+ *
+ * @see docs/decisions/D0005-grader-model-separation.md
+ * @see ./borderline-detector.ts — pure predicate
+ * @see ./grader-consistency.ts — JudgmentConsistency shape we emit
+ */
+import { isBorderline } from "./borderline-detector.js";
+// Imported for the default-regrader fallback documented in the header.
+// The runner does not invoke gradeOnce directly when `regrade` is supplied.
+// Keeping the import on the public surface preserves the architectural
+// rule that the runner's grader entry point lives in grader-api.js
+// (Pitfall 6 — the inter-grader ensemble module is intentionally NOT
+// reached for on this path).
+import { gradeOnce } from "./grader-api.js";
+import { analyzeJudgment, } from "./grader-consistency.js";
+/**
+ * Re-export `gradeOnce` so callers that need to wire the default regrader
+ * (composition-root, integration tests) can import the grader entry point
+ * from this module rather than rediscovering grader-api.js. The runner
+ * itself does not invoke `gradeOnce` — the caller-supplied `regrade`
+ * callback owns the live grader call (Pitfall 6 — runner stays pure wrt
+ * provider config).
+ */
+export { gradeOnce };
+/** Map key for the per-judgment consistency record. */
+function consistencyKey(j) {
+    return `${j.taskId}::${j.dimension}::${j.modelId}`;
+}
+/**
+ * Run intra-grader consensus on the borderline subset of `judgments`.
+ *
+ * - Borderline (per `isBorderline(score, thresholds)`): re-grade
+ *   `replications` times via `regrade`; emit a `JudgmentConsistency`
+ *   keyed by `${taskId}::${dimension}::${modelId}`; merge the consensus
+ *   median back into the canonical judgment's `score`.
+ * - Non-borderline: pass through unchanged. Output array length == input.
+ *
+ * The function is order-preserving — the returned `judgments` array
+ * keeps the same element order as the input.
+ */
+export async function runBorderlineConsensus(options) {
+    const { judgments, logger, regrade, replications, thresholds } = options;
+    const consistencyByJudgment = new Map();
+    // Filter to borderline subset; bypass entirely if empty.
+    const borderlineKeys = new Set();
+    for (const j of judgments) {
+        if (isBorderline(j.score, thresholds)) {
+            borderlineKeys.add(consistencyKey(j));
+        }
+    }
+    if (borderlineKeys.size === 0) {
+        return { consistencyByJudgment, judgments };
+    }
+    const out = [];
+    for (const j of judgments) {
+        const key = consistencyKey(j);
+        if (!borderlineKeys.has(key)) {
+            out.push(j); // non-borderline — single replica
+            continue;
+        }
+        // Re-grade `replications` times via the same pinned grader. The
+        // replications carry network-bound side effects (LLM calls), so run
+        // them concurrently — `Promise.allSettled` preserves the per-replica
+        // try/catch shape (failures log + drop, surviving replicas still
+        // contribute to the consensus median). Worst-case wall time drops
+        // from `replications * roundTrip` to a single `roundTrip`.
+        const scores = [j.score];
+        const settled = await Promise.allSettled(Array.from({ length: replications }, () => regrade(j)));
+        settled.forEach((outcome, i) => {
+            if (outcome.status === "fulfilled") {
+                scores.push(outcome.value);
+            }
+            else {
+                const err = outcome.reason;
+                logger?.warn(`Borderline replication ${i + 1}/${replications} failed for ${key}: ` +
+                    (err instanceof Error ? err.message : String(err)));
+            }
+        });
+        const grading = {
+            area: "",
+            dimension: j.dimension,
+            ...(j.modelId ? { providerId: j.modelId } : {}),
+            scores,
+            taskId: j.taskId,
+        };
+        const consistency = analyzeJudgment(grading);
+        consistencyByJudgment.set(key, consistency);
+        // Merge consensus (median across replicas) into the canonical judgment.
+        out.push({ ...j, score: median(scores) });
+    }
+    return { consistencyByJudgment, judgments: out };
+}
+/**
+ * Compute the median of an array of numbers. The runner uses median
+ * (not mean) so a single outlier replica doesn't drag the consensus
+ * score across a severity threshold.
+ */
+function median(values) {
+    if (values.length === 0)
+        return 0;
+    const sorted = [...values].sort((a, b) => a - b);
+    const mid = Math.floor(sorted.length / 2);
+    return sorted.length % 2 === 0
+        ? (sorted[mid - 1] + sorted[mid]) / 2
+        : sorted[mid];
+}

package/dist/pipeline/borderline-detector.d.ts

@@ -0,0 +1,24 @@
+/**
+ * pipeline/borderline-detector.ts
+ *
+ * GRAD-04 borderline-band predicate. Pure computation; no I/O.
+ *
+ * A judgment is "borderline" when its score lies within ±5 of any of
+ * the three rubric thresholds (severity boundaries 30 / 50 / 60 from
+ * packages/eval/config/thresholds.ts:50/54/58 — critical / warning /
+ * info edges).
+ *
+ * Per D0005 (grader-model separation), borderline judgments trigger
+ * intra-grader consensus replication of the SAME pinned grader rather
+ * than inter-grader ensemble — preserving D0005's reproducibility
+ * posture.
+ *
+ * @see docs/decisions/D0005-grader-model-separation.md
+ * @see docs/design-docs/actionability-ladder/03-structured-grader-judgments.md
+ */
+export declare const BORDERLINE_BAND = 5;
+/**
+ * Returns true when `score` lies within ±BORDERLINE_BAND of any
+ * configured threshold. Pure function — safe to call N×.
+ */
+export declare function isBorderline(score: number, thresholds: readonly number[]): boolean;