@sanity/ailf 4.6.0 → 5.0.0

package/dist/_vendor/ailf-core/types/branded-ids.js

@@ -84,11 +84,24 @@ export function generateRunId() {
         .toISOString()
         .replace(/[-:]/g, "")
         .replace(/\.\d{3}Z$/, "Z");
-    const bytes = crypto.getRandomValues(new Uint8Array(8));
-    let suffix = "";
-    for (const b of bytes) {
-        suffix += (b % 36).toString(36);
+    // Rejection-sample bytes against the largest multiple of 36 ≤ 256
+    // (252) before applying `% 36`. Naive `b % 36` over [0, 255] biases
+    // digits 0..3 (probability 8/256) over 4..35 (probability 7/256) by
+    // ~14% per character. Drawing fresh bytes whenever the buffer runs
+    // dry keeps the loop terminating with overwhelming probability
+    // (each byte is kept with probability 252/256 ≈ 98.4%).
+    const suffixChars = [];
+    while (suffixChars.length < 8) {
+        const buf = crypto.getRandomValues(new Uint8Array(8));
+        for (const b of buf) {
+            if (b >= 252)
+                continue; // reject biased range
+            suffixChars.push((b % 36).toString(36));
+            if (suffixChars.length === 8)
+                break;
+        }
     }
+    const suffix = suffixChars.join("");
     return `run_${ts}_${suffix}`;
 }
 /**
@@ -166,3 +179,66 @@ export function fixtureId(raw) {
     }
     return ok(raw);
 }
+/**
+ * Canonical shape for a `JudgmentId`.
+ *
+ * Two accepted forms:
+ *  - `judgment_<runId-suffix>_<sanitized-task>__<sanitized-model>__<dimension>`
+ *    — minted by `generateJudgmentId` for synthesized fall-back judgments
+ *    so dedup is per-run (a re-run of the same task produces a distinct id).
+ *  - `j_<alphanumeric>` — short form used by test fixtures and any caller
+ *    that wants a stable, opaque id without the structured composite.
+ *
+ * Inner segments may carry alphanumerics, hyphens, dots, and colons (the
+ * provider id surface is colon-separated). The full string is bounded to
+ * 256 characters to keep the id index-friendly downstream.
+ */
+const JUDGMENT_ID_RE = /^(?:judgment_[0-9a-z]{1,16}_[a-z0-9][a-z0-9.:_-]*__[a-z0-9][a-z0-9.:_-]*__[a-z0-9][a-z0-9-]*|j_[A-Za-z0-9_-]{4,})$/;
+/**
+ * Parse a raw string into a `JudgmentId`.
+ *
+ * See `JUDGMENT_ID_RE` for the accepted formats.
+ */
+export function judgmentId(raw) {
+    if (raw.length === 0 || raw.length > 256 || !JUDGMENT_ID_RE.test(raw)) {
+        return err({
+            code: "INVALID_JUDGMENT_ID",
+            raw,
+            message: `Invalid JudgmentId "${raw}": must match judgment_<runSalt>_<task>__<model>__<dimension> or j_<alnum>`,
+        });
+    }
+    return ok(raw);
+}
+/** Strip a value down to the alphanumeric+hyphen alphabet the id format allows. */
+function sanitizeJudgmentSegment(value) {
+    // Lowercase + replace runs of non-alphanumerics with a single hyphen,
+    // trim leading/trailing hyphens. Keeps dots and colons in `modelId`-like
+    // values (`openai:gpt-5.2`) since the regex permits them.
+    return value
+        .toLowerCase()
+        .replace(/[^a-z0-9.:_-]+/g, "-")
+        .replace(/^-+|-+$/g, "");
+}
+/**
+ * Generate a deterministic `JudgmentId` for a synthesized fall-back
+ * judgment. Salting with `runId` (when supplied) makes the id unique
+ * per-run so consumers' `(taskId, modelId, dimension)` dedup key
+ * doesn't collide across re-runs of the same task — every run writes
+ * fresh ids that still encode the natural composite key.
+ *
+ * When `runId` is absent the salt collapses to `nosalt`, preserving the
+ * legacy "deterministic across runs" shape for callers that explicitly
+ * want it (e.g. unit tests that assert the exact id string).
+ */
+export function generateJudgmentId(input) {
+    // Take the trailing 8 chars of the runId (the random base36 suffix on
+    // the canonical shape) so the salt stays compact while still cycling
+    // every run. Falls back to a constant marker when runId isn't passed.
+    const runSalt = input.runId
+        ? sanitizeJudgmentSegment(String(input.runId).slice(-8)) || "nosalt"
+        : "nosalt";
+    const task = sanitizeJudgmentSegment(input.taskId);
+    const model = sanitizeJudgmentSegment(input.modelId);
+    const dimension = sanitizeJudgmentSegment(input.dimension);
+    return `judgment_${runSalt}_${task}__${model}__${dimension}`;
+}

package/dist/_vendor/ailf-core/types/confidence.d.ts

@@ -19,7 +19,7 @@
  * is an open tag (see `ConfidenceDerivation`). The list is the
  * recommended starting set, not the universe.
  */
-export declare const CONVENTIONAL_DERIVATIONS: readonly ["ensemble-stdev", "ceiling-cross-check", "regression-gate", "card-type-specific"];
+export declare const CONVENTIONAL_DERIVATIONS: readonly ["ensemble-stdev", "ceiling-cross-check", "regression-gate", "card-type-specific", "synthesized-pre-cross-check"];
 /**
  * Tag identifying the formula used to derive `Confidence.level`.
  *

package/dist/_vendor/ailf-core/types/confidence.js

@@ -24,6 +24,13 @@ export const CONVENTIONAL_DERIVATIONS = [
     "ceiling-cross-check",
     "regression-gate",
     "card-type-specific",
+    // Sentinel placeholder used by the eval pipeline's
+    // `synthesizeUnparsedJudgment` fall-back. The validator
+    // (`validateGraderJudgmentsCalibration`) overwrites it with
+    // "ceiling-cross-check" before judgments leave the live pipeline,
+    // so it should not appear on stored reports — the literal is in
+    // this list so a leaked sentinel is greppable.
+    "synthesized-pre-cross-check",
 ];
 /**
  * Structural type guard for `Confidence`. Verifies the runtime shape

package/dist/_vendor/ailf-core/types/diagnosis.d.ts

@@ -0,0 +1,169 @@
+/**
+ * Diagnosis core domain types — canonical shapes for the post-run
+ * synthesis layer (Doc 05).
+ *
+ * `Diagnosis.inputs` is the four-version cache envelope (VER-01); any
+ * segment bump invalidates a cached Diagnosis. `DiagnosisCard` is an
+ * outer-`status` discriminated union with a nested `cardType`
+ * discriminator inside the `ready` variant.
+ *
+ * Phase 1 lands placeholder body shapes; Phase 5 enriches each per
+ * Doc 05 specs.
+ *
+ * @see docs/decisions/D0049-shared-confidence-contract.md
+ * @see docs/decisions/D0052-judgment-ref-granularity.md
+ * @see docs/decisions/D0050-artifact-registry-post-hoc-versioned-extension.md
+ */
+import type { RunId } from "./branded-ids.js";
+import type { ReportId } from "./index.js";
+/**
+ * The four-version cache envelope. Every cached `Diagnosis` carries the
+ * versions of the inputs that produced it; any bump in any segment
+ * invalidates the cache (cross-package contract test asserts this).
+ *
+ * Strings everywhere; semver convention by humans where authored
+ * manually. No branding, no tuples, no content-hash typing — keeps the
+ * envelope trivially serializable + greppable.
+ */
+export interface VersionedInputs {
+    graderJudgmentsVersion: string;
+    ensembleVersion: string;
+    diagnosisVersion: string;
+    cardVersion: string;
+}
+/**
+ * The 8 ready-card archetypes. Phase 5 cards register against these
+ * literals; the slim-shape boundary in Phase 7 reads them to render the
+ * Studio diagnosis renderer.
+ */
+export type CardType = "area-summary" | "failure-mode-summary" | "no-issues" | "top-recommendations" | "weakest-area" | "low-confidence-attribution" | "doc-attribution-spotlight" | "regression-vs-baseline";
+/**
+ * Per-card telemetry envelope. `cardVersion` here is the per-card
+ * version (e.g. `"area-summary@0.1.0"`), not the compound. Drives
+ * DIAG-06 cost telemetry in Phase 6.
+ */
+export interface CardMeta {
+    cardVersion: string;
+    tokenUsage?: {
+        input: number;
+        output: number;
+    };
+    latencyMs?: number;
+    /** ISO 8601 UTC timestamp. */
+    generatedAt: string;
+}
+/**
+ * A single actionable suggestion surfaced by a recommendations card.
+ * The full Phase 5 shape may add fields (per Doc 05 specs); Phase 1
+ * locks the minimum required surface.
+ */
+export interface ActionSuggestion {
+    title: string;
+    body: string;
+    priority: "high" | "medium" | "low";
+}
+/**
+ * Phase 1 body placeholders. Each shape is intentionally minimal; Phase 5
+ * card files enrich them per Doc 05 specs and assert
+ * `satisfies z.ZodType<Extract<DiagnosisCard, { status: "ready"; cardType: "X" }>["body"]>`
+ * against these declarations.
+ */
+export interface AreaSummaryBody {
+    summary: string;
+}
+export interface FailureModeSummaryBody {
+    summary: string;
+}
+export interface NoIssuesBody {
+    summary: string;
+}
+export interface TopRecommendationsBody {
+    summary: string;
+    suggestions: ActionSuggestion[];
+}
+export interface WeakestAreaBody {
+    summary: string;
+}
+export interface LowConfidenceAttributionBody {
+    summary: string;
+}
+export interface DocAttributionSpotlightBody {
+    summary: string;
+}
+export interface RegressionVsBaselineBody {
+    summary: string;
+}
+/**
+ * Outer-`status` discriminated union: 8 ready variants (one per
+ * `cardType`, each carrying its per-cardType body), plus a `degraded`
+ * variant (parse failed or downgraded by the runner) and a `missing`
+ * variant (card not produced for this run).
+ *
+ * No `not-yet-generated` variant — old-report fallback is a Phase 7
+ * concern at the slim-shape boundary, handled at fetch-time, not in
+ * `DiagnosisCard` itself.
+ */
+export type DiagnosisCard = {
+    status: "ready";
+    cardType: "area-summary";
+    body: AreaSummaryBody;
+    meta: CardMeta;
+} | {
+    status: "ready";
+    cardType: "failure-mode-summary";
+    body: FailureModeSummaryBody;
+    meta: CardMeta;
+} | {
+    status: "ready";
+    cardType: "no-issues";
+    body: NoIssuesBody;
+    meta: CardMeta;
+} | {
+    status: "ready";
+    cardType: "top-recommendations";
+    body: TopRecommendationsBody;
+    meta: CardMeta;
+} | {
+    status: "ready";
+    cardType: "weakest-area";
+    body: WeakestAreaBody;
+    meta: CardMeta;
+} | {
+    status: "ready";
+    cardType: "low-confidence-attribution";
+    body: LowConfidenceAttributionBody;
+    meta: CardMeta;
+} | {
+    status: "ready";
+    cardType: "doc-attribution-spotlight";
+    body: DocAttributionSpotlightBody;
+    meta: CardMeta;
+} | {
+    status: "ready";
+    cardType: "regression-vs-baseline";
+    body: RegressionVsBaselineBody;
+    meta: CardMeta;
+} | {
+    status: "degraded";
+    cardType: CardType;
+    reason: string;
+    parseFailed: boolean;
+    meta: CardMeta;
+} | {
+    status: "missing";
+    cardType: CardType;
+    reason: string;
+};
+/**
+ * The post-run synthesis aggregate. Consumed by Phase 5 (runner +
+ * cards), Phase 6 (CLI) and Phase 7 (Studio). Phase 1 lands the
+ * declarative shape; runtime construction lands in Phase 5.
+ */
+export interface Diagnosis {
+    runId: RunId;
+    reportId: ReportId;
+    inputs: VersionedInputs;
+    cards: DiagnosisCard[];
+    /** ISO 8601 UTC timestamp. */
+    generatedAt: string;
+}

package/dist/_vendor/ailf-core/types/diagnosis.js

@@ -0,0 +1,17 @@
+/**
+ * Diagnosis core domain types — canonical shapes for the post-run
+ * synthesis layer (Doc 05).
+ *
+ * `Diagnosis.inputs` is the four-version cache envelope (VER-01); any
+ * segment bump invalidates a cached Diagnosis. `DiagnosisCard` is an
+ * outer-`status` discriminated union with a nested `cardType`
+ * discriminator inside the `ready` variant.
+ *
+ * Phase 1 lands placeholder body shapes; Phase 5 enriches each per
+ * Doc 05 specs.
+ *
+ * @see docs/decisions/D0049-shared-confidence-contract.md
+ * @see docs/decisions/D0052-judgment-ref-granularity.md
+ * @see docs/decisions/D0050-artifact-registry-post-hoc-versioned-extension.md
+ */
+export {};

package/dist/_vendor/ailf-core/types/generalized-task.d.ts

@@ -53,11 +53,26 @@ export interface PerspectiveDocRef {
     perspective: string;
     reason?: string;
 }
+/**
+ * A single criterion within a templated llm-rubric assertion.
+ *
+ * The `id` is a stable, slug-formatted identifier — auto-derived from
+ * `text` in Studio (via slug `options.source: "text"`), or backfilled
+ * from Sanity's `_key` for pre-migration documents. Survives criterion
+ * text edits; downstream judgments and diagnosis cards reference by
+ * `id` per D0052 (judgment-ref granularity).
+ */
+export interface CriterionRef {
+    /** Stable per-criterion identifier — slug-format (`[a-z0-9][a-z0-9-]*`). */
+    id: string;
+    /** Author-facing criterion text (the original bullet). */
+    text: string;
+}
 /** A templated assertion referencing a rubric template */
 export interface GeneralizedTemplatedAssertion {
     type: "llm-rubric";
     template: string;
-    criteria: string[];
+    criteria: CriterionRef[];
     weight?: number;
 }
 /** A value-based assertion (contains, javascript, cost, latency, etc.) */

package/dist/_vendor/ailf-core/types/grader-judgment.d.ts

@@ -0,0 +1,125 @@
+/**
+ * GraderJudgment core domain types — canonical shapes for structured
+ * grader output (Doc 03, GRAD-02).
+ *
+ * Authored INDEPENDENTLY of any Zod schema (D0045 doctrine — Plan 02's
+ * `GraderJudgmentSchema` `satisfies` against this type, not the other
+ * way around). A tautological `satisfies z.ZodType<z.infer<typeof
+ * GraderJudgmentSchema>>` is forbidden.
+ *
+ * Phase 1 retained the existing pipeline core (`taskId`, `modelId`,
+ * `dimension`, `reason`, `score`, `outputFailure?`) as required for
+ * backward compat with Phase 0 callers (Doc 03 §"existing, unchanged")
+ * and added the GRAD-02 additive fields (`judgmentId`, `subJudgments`,
+ * `docCitations`, `failureMode`, `confidence`,
+ * `hallucinationCheckedAgainst`, `metadata`) as additive in Phase 1;
+ * required from Phase 3 GRAD-05.
+ *
+ * Phase 3 GRAD-05 has flipped the additive fields to required (this
+ * file) and the corresponding Zod schema in
+ * `packages/eval/src/adapters/grader-outputs/promptfoo-grader-output.ts`
+ * is `.strict()` with `graderJudgmentsVersion = "1.0.0"`. The
+ * read-only legacy parser at `…/legacy/promptfoo-grader-output-legacy.ts`
+ * (against `LegacyGraderJudgment`) is the named consumer for already-
+ * stored historical reports through GRAD-06 cutover.
+ *
+ * @see docs/decisions/D0049-shared-confidence-contract.md
+ * @see docs/decisions/D0052-judgment-ref-granularity.md
+ * @see docs/design-docs/actionability-ladder/03-structured-grader-judgments.md
+ */
+import type { JudgmentId } from "./branded-ids.js";
+import type { Confidence } from "./confidence.js";
+/**
+ * Role enum for doc citations attached to a grader judgment (GRAD-02).
+ * Closed string-literal union — Phase 3 may extend.
+ */
+export type DocCitationRole = "supports" | "contradicts" | "missing" | "irrelevant";
+/**
+ * A single doc the grader cited while reasoning. `documentId` is the
+ * canonical D0052 reference (id, not slug); `slug` is a human-readable
+ * annotation only. `hallucinated` is set true at adapter time when the
+ * `slug` does not resolve against the task's `contextDocs` set.
+ */
+export interface DocCitation {
+    /** Canonical D0052 document ref (id, not slug). */
+    documentId: string;
+    /** Optional human-readable annotation. Never the identity. */
+    slug?: string;
+    role: DocCitationRole;
+    /** True when `slug` is not in the resolvable-set. */
+    hallucinated?: boolean;
+}
+/**
+ * Per-criterion sub-judgment — one entry per task-criterion bullet
+ * (Doc 03 §"per-criterion sub-judgments"). The `criterionId` is the
+ * stable identifier declared on the task's `criteria` array (Phase 2
+ * GRAD-01 schema-sync), not synthesized at grade time.
+ */
+export interface CriterionSubJudgment {
+    /** Stable criterion identifier — matches `CriterionRef.id` from the task definition (D0052). */
+    criterionId: string;
+    met: boolean;
+    /** ≤280 chars — quote or paraphrase. */
+    evidence: string;
+    /** Grader self-confidence on this single criterion (D0049). */
+    confidence: Confidence;
+}
+/**
+ * The structured grader judgment — Phase 3 GRAD-05 shape.
+ *
+ * Existing pipeline core (Doc 03 §"existing, unchanged"): `taskId`,
+ * `modelId`, `dimension`, `reason`, `score`. The pre-existing
+ * `outputFailure?` remains optional. The
+ * `contextDocs? (legacy alias: canonicalDocs)` annotation (StoredJudgment
+ * extension) lives on the storage extension type, not here.
+ *
+ * Additive in Phase 1; required from Phase 3 GRAD-05: `judgmentId`,
+ * `subJudgments`, `docCitations`, `failureMode`, `confidence`,
+ * `hallucinationCheckedAgainst`, `metadata`. The corresponding Zod
+ * schema in `packages/eval/src/adapters/grader-outputs/promptfoo-grader-output.ts`
+ * is `.strict()` with `graderJudgmentsVersion = "1.0.0"`.
+ */
+export interface GraderJudgment {
+    /** Rubric template name (e.g. "task-completion", "code-correctness"). */
+    dimension: string;
+    /** The model that produced the response being graded. */
+    modelId: string;
+    /**
+     * True when the model failed to produce meaningful output (empty
+     * response, API error, or refusal). Distinguishes infrastructure
+     * failures from genuinely incorrect responses — a score of 0 from no
+     * output is fundamentally different from a score of 0 from wrong
+     * output.
+     */
+    outputFailure?: boolean;
+    /** The grader's natural-language reasoning. */
+    reason: string;
+    /** Numeric score in [0, 100] (normalized). */
+    score: number;
+    /** The task this judgment belongs to. */
+    taskId: string;
+    /**
+     * D0052 granular branded id. Required from Phase 3 GRAD-05 — every
+     * grader emission carries one.
+     */
+    judgmentId: JudgmentId;
+    /** Per-criterion sub-judgments. */
+    subJudgments: CriterionSubJudgment[];
+    /** Doc citations with role + hallucinated flag. */
+    docCitations: DocCitation[];
+    /**
+     * Per-dimension failure mode. Phase 3 GRAD-03 stamps the taxonomy
+     * literal at the runtime grader-prompt; the value is a free-form
+     * string for forward compat with future taxonomy extensions.
+     */
+    failureMode: string;
+    /** Grader self-confidence per D0049. */
+    confidence: Confidence;
+    /** Hallucination cross-check (Pitfall #11) — union of task.context.docs and run.documentManifest. */
+    hallucinationCheckedAgainst: string[];
+    /** Metadata about the grader run. */
+    metadata: {
+        graderModel: string;
+        graderJudgmentsVersion: string;
+    };
+}

package/dist/_vendor/ailf-core/types/grader-judgment.js

@@ -0,0 +1,30 @@
+/**
+ * GraderJudgment core domain types — canonical shapes for structured
+ * grader output (Doc 03, GRAD-02).
+ *
+ * Authored INDEPENDENTLY of any Zod schema (D0045 doctrine — Plan 02's
+ * `GraderJudgmentSchema` `satisfies` against this type, not the other
+ * way around). A tautological `satisfies z.ZodType<z.infer<typeof
+ * GraderJudgmentSchema>>` is forbidden.
+ *
+ * Phase 1 retained the existing pipeline core (`taskId`, `modelId`,
+ * `dimension`, `reason`, `score`, `outputFailure?`) as required for
+ * backward compat with Phase 0 callers (Doc 03 §"existing, unchanged")
+ * and added the GRAD-02 additive fields (`judgmentId`, `subJudgments`,
+ * `docCitations`, `failureMode`, `confidence`,
+ * `hallucinationCheckedAgainst`, `metadata`) as additive in Phase 1;
+ * required from Phase 3 GRAD-05.
+ *
+ * Phase 3 GRAD-05 has flipped the additive fields to required (this
+ * file) and the corresponding Zod schema in
+ * `packages/eval/src/adapters/grader-outputs/promptfoo-grader-output.ts`
+ * is `.strict()` with `graderJudgmentsVersion = "1.0.0"`. The
+ * read-only legacy parser at `…/legacy/promptfoo-grader-output-legacy.ts`
+ * (against `LegacyGraderJudgment`) is the named consumer for already-
+ * stored historical reports through GRAD-06 cutover.
+ *
+ * @see docs/decisions/D0049-shared-confidence-contract.md
+ * @see docs/decisions/D0052-judgment-ref-granularity.md
+ * @see docs/design-docs/actionability-ladder/03-structured-grader-judgments.md
+ */
+export {};

package/dist/_vendor/ailf-core/types/index.d.ts

@@ -13,6 +13,7 @@ import type { DocumentRef as _DocumentRef, EvalMode, RunContext } from "../../ai
 import type { ArtifactType } from "../artifact-registry.js";
 import type { SymbolPreflightReport } from "./symbol-preflight-report.js";
 import type { AssociationValues, RunId } from "./branded-ids.js";
+import type { GraderJudgment } from "./grader-judgment.js";
 export type { ActualScoreEntry, ComponentResult, TestResult, UrlMetadata, } from "./scoring-input.js";
 export type { DocumentRef, RunContext, RunTrigger } from "../../ailf-shared/index.d.ts";
 export type { StoredBaseline, StoredReport, StoredRun, StoredTaskResult, StoredTrace, SchemaVersioned, } from "./storage-schema.js";
@@ -32,9 +33,13 @@ export type { SymbolPreflightDeduction, SymbolPreflightFinding, SymbolPreflightR
 export { DEFAULT_PREFLIGHT_CODE_CORRECTNESS_WEIGHT, type PreflightRubricContext, type PreflightScoringConfig, } from "./preflight-scoring.js";
 export type { Confidence, ConfidenceDerivation } from "./confidence.js";
 export { CONVENTIONAL_DERIVATIONS, isConfidence } from "./confidence.js";
-export type { ArtifactId, AssociationAxis, AssociationValues, Brand, EntryKey, Err, FixtureId, IdValidationError, NewReportId, Ok, ProviderId, PromptId, Result, ResultId, RubricId, RunFingerprint, RunId, SuiteId, TaskId, TaskSlug, TraceId, } from "./branded-ids.js";
-export { err, fixtureId, generateRunId, ok, providerId, resultId, runId, suiteId, taskId, traceId, } from "./branded-ids.js";
-export type { AgentHarnessTaskDefinition, ContentLakeAuthorableMode, ContentLakeAuthorableTask, CustomTaskDefinition, GeneralizedAssertionDefinition, GeneralizedDocRef, GeneralizedTaskDefinition, GeneralizedTemplatedAssertion, GeneralizedValueAssertion, IdDocRef, KnowledgeProbeTaskDefinition, LiteracyTaskDefinition, MCPServerTaskDefinition, PromptVars, PathDocRef, PerspectiveDocRef, ReservedPromptVarKey, RubricRef, SlugDocRef, TaskCommonFields, TaskDifficulty, TaskOptions, TaskProviderConfig, TaskStatus, } from "./generalized-task.js";
+export type { ArtifactId, AssociationAxis, AssociationValues, Brand, EntryKey, Err, FixtureId, IdValidationError, JudgmentId, NewReportId, Ok, ProviderId, PromptId, Result, ResultId, RubricId, RunFingerprint, RunId, SuiteId, TaskId, TaskSlug, TraceId, } from "./branded-ids.js";
+export { err, fixtureId, generateJudgmentId, generateRunId, judgmentId, ok, providerId, resultId, runId, suiteId, taskId, traceId, } from "./branded-ids.js";
+export type { AgentHarnessTaskDefinition, ContentLakeAuthorableMode, ContentLakeAuthorableTask, CriterionRef, CustomTaskDefinition, GeneralizedAssertionDefinition, GeneralizedDocRef, GeneralizedTaskDefinition, GeneralizedTemplatedAssertion, GeneralizedValueAssertion, IdDocRef, KnowledgeProbeTaskDefinition, LiteracyTaskDefinition, MCPServerTaskDefinition, PromptVars, PathDocRef, PerspectiveDocRef, ReservedPromptVarKey, RubricRef, SlugDocRef, TaskCommonFields, TaskDifficulty, TaskOptions, TaskProviderConfig, TaskStatus, } from "./generalized-task.js";
+export type { ActionSuggestion, AreaSummaryBody, CardMeta, CardType, Diagnosis, DiagnosisCard, DocAttributionSpotlightBody, FailureModeSummaryBody, LowConfidenceAttributionBody, NoIssuesBody, RegressionVsBaselineBody, TopRecommendationsBody, VersionedInputs, WeakestAreaBody, } from "./diagnosis.js";
+export type { AttributionMeta, DocAttribution, JudgmentAttribution, } from "./attribution.js";
+export type { CriterionSubJudgment, DocCitation, DocCitationRole, GraderJudgment, } from "./grader-judgment.js";
+export type { LegacyGraderJudgment } from "./legacy-grader-judgment.js";
 type DocumentRef = _DocumentRef;
 /** Aggregated retrieval metrics for a feature area */
 export interface AreaRetrievalMetrics {
@@ -128,8 +133,31 @@ export interface FailureModeReport {
     /** Total judgments analyzed */
     totalJudgments: number;
 }
-/** Failure mode classification for a low-scoring judgment */
-export type FailureModeType = "api-error" | "incorrect-docs" | "missing-docs" | "model-limitation" | "outdated-docs" | "poor-structure" | "unclassified";
+/**
+ * Failure mode classification for a low-scoring judgment.
+ *
+ * Open-set string (Plan 03-02 per-dimension taxonomies introduce modes
+ * outside the original literacy enum: `false-floor`, `spec-mismatch`,
+ * `tool-misuse`, `factual-error`, `hallucination`, etc. — the grader
+ * is told these are legal answers via the rubric prompt). The legacy
+ * literacy enum survives as `LegacyFailureModeType` for the
+ * report-aggregation helpers that need stable bucket ordering and
+ * icon tables; consumers that only care about presence/absence treat
+ * `FailureModeType` as `string`.
+ */
+export type FailureModeType = string;
+/**
+ * Closed enum of the original literacy failure modes — used by the
+ * report formatters that iterate buckets in a stable order. Adding to
+ * this list is a deliberate extension; modes outside it still flow
+ * through the report (per-area `modes` record), just without a
+ * pre-allocated bucket in `summary`.
+ */
+export type LegacyFailureModeType = "api-error" | "incorrect-docs" | "missing-docs" | "model-limitation" | "outdated-docs" | "poor-structure" | "unclassified";
+/** Set of canonical legacy modes — exported for report-formatter use. */
+export declare const LEGACY_FAILURE_MODES: readonly LegacyFailureModeType[];
+/** Type guard for legacy modes. */
+export declare function isLegacyFailureMode(mode: string): mode is LegacyFailureModeType;
 /** Per-feature-area score breakdown */
 export interface FeatureScore {
     /**
@@ -261,30 +289,16 @@ export interface GapEstimate {
     /** Specific remediation description */
     remediation: string;
 }
-/** A single grader judgment — one per assertion per test */
-export interface GraderJudgment {
-    /** The rubric template used (task-completion, code-correctness, doc-coverage) */
-    dimension: string;
-    /** The model that produced the response being graded */
-    modelId: string;
+/** Enriched grader judgment with stored documentation context refs. */
+export interface StoredJudgment extends GraderJudgment {
     /**
-     * True when the model failed to produce meaningful output (empty response,
-     * API error, or refusal). Distinguishes infrastructure failures from
-     * genuinely incorrect responses — a score of 0 from no output is
-     * fundamentally different from a score of 0 from wrong output.
+     * Documentation context the task expected the model to use.
+     *
+     * Legacy alias `canonicalDocs` may appear on stored reports written
+     * before Phase 2 — readers should tolerate both. Writers (the pipeline)
+     * always emit `contextDocs`.
      */
-    outputFailure?: boolean;
-    /** The grader's natural language reasoning */
-    reason: string;
-    /** The numeric score (0–100) */
-    score: number;
-    /** The task this judgment belongs to */
-    taskId: string;
-}
-/** Enriched grader judgment with canonical doc references, stored in reports */
-export interface StoredJudgment extends GraderJudgment {
-    /** Canonical docs that the task expected the model to use */
-    canonicalDocs?: DocumentRef[];
+    contextDocs?: DocumentRef[];
 }
 /**
  * Per-test result stored in reports for drill-down and audit.
@@ -296,8 +310,11 @@ export interface StoredJudgment extends GraderJudgment {
 export interface StoredTestResult {
     /** Resolved feature area (from __featureArea or description) */
     area: string;
-    /** Canonical docs the task expected the model to use */
-    canonicalDocs?: DocumentRef[];
+    /**
+     * Documentation context the task expected the model to use.
+     * Legacy alias `canonicalDocs` may appear on pre-Phase-2 reports.
+     */
+    contextDocs?: DocumentRef[];
     /** Weighted composite score (gold variant only) */
     compositeScore?: number;
     /** Per-test cost (USD) */
@@ -349,6 +366,40 @@ export interface StoredTestResult {
 }
 /** Grader consistency diagnostics — does not affect scores, reported alongside */
 export interface GraderReliability {
+    /**
+     * Plan 03-03 — count of grader-emission vs ceiling cross-check disagreements.
+     *
+     * Incremented by the live pipeline when `validateFailureMode(...)` returns
+     * `level: "medium"` (the grader's emitted `failureMode` does not agree with
+     * the ceiling-decomposition mode). Surfaces calibration drift over time
+     * without affecting scores. Optional — undefined when the run did not
+     * exercise the failure-mode validator (e.g., grader-consistency-only paths).
+     *
+     * @see docs/decisions/D0049-shared-confidence-contract.md
+     */
+    failureModeCalibration?: number;
+    /**
+     * Plan 03-03 — count of strict-schema parse failures during grader-output
+     * extraction. Wired at the parse-fail branch in `extractGraderJudgments`;
+     * incremented when `GraderJudgmentSchema.safeParse` rejects a payload and
+     * the pipeline drops to the Phase 1 minimal-shape fallback.
+     *
+     * Plan 03-04 will tighten the strict schema (`.strict()` + GRAD-02 fields
+     * required) and graders will emit the structured wire format in earnest;
+     * this counter measures pre-hard-fail drift.
+     */
+    parseFailures?: number;
+    /**
+     * Phase 4 ATTR-01 — count of grader citations whose `slug` was not
+     * in the resolvable-set (`hallucinationCheckedAgainst`).
+     * Incremented by `computeJudgmentAttribution(...)` for every
+     * citation that fails the hallucination short-circuit (Success
+     * Criterion #5). A counter, not a ratio — consumers compute the
+     * rate by dividing by total-citations if needed.
+     *
+     * @see docs/decisions/D0049-shared-confidence-contract.md
+     */
+    hallucinationCount?: number;
     /** Inter-grader agreement (from multi-grader comparison) — Phase 3 */
     agreement?: {
         /** Models compared against the primary grader */

package/dist/_vendor/ailf-core/types/index.js

@@ -18,7 +18,21 @@ export { InMemoryPluginRegistry } from "./plugin-registry.js";
 export { evalModeType } from "./eval-mode-config.js";
 export { DEFAULT_PREFLIGHT_CODE_CORRECTNESS_WEIGHT, } from "./preflight-scoring.js";
 export { CONVENTIONAL_DERIVATIONS, isConfidence } from "./confidence.js";
-export { err, fixtureId, generateRunId, ok, providerId, resultId, runId, suiteId, taskId, traceId, } from "./branded-ids.js";
+export { err, fixtureId, generateJudgmentId, generateRunId, judgmentId, ok, providerId, resultId, runId, suiteId, taskId, traceId, } from "./branded-ids.js";
+/** Set of canonical legacy modes — exported for report-formatter use. */
+export const LEGACY_FAILURE_MODES = [
+    "api-error",
+    "incorrect-docs",
+    "missing-docs",
+    "model-limitation",
+    "outdated-docs",
+    "poor-structure",
+    "unclassified",
+];
+/** Type guard for legacy modes. */
+export function isLegacyFailureMode(mode) {
+    return LEGACY_FAILURE_MODES.includes(mode);
+}
 // ---------------------------------------------------------------------------
 // Comparison (Approach 2: structured comparison output)
 // ---------------------------------------------------------------------------