@sanity/ailf 0.1.0

package/dist/_vendor/ailf-core/types/index.d.ts

@@ -0,0 +1,1082 @@
+/**
+ * @sanity/ailf-core — Domain types
+ *
+ * Core types for the AI Literacy Framework evaluation pipeline.
+ * These define the contracts between pipeline steps, domain services,
+ * and port adapters.
+ *
+ * Extracted from packages/eval/src/pipeline/types.ts during the
+ * Ports & Adapters migration (Phase 0c). The original file is now a
+ * re-export barrel that preserves backward compatibility.
+ */
+import type { ConcreteEvalMode as _ConcreteEvalMode, DocumentRef as _DocumentRef, EvalMode as _EvalMode } from "../../ailf-shared/index.d.ts";
+export type { ActualScoreEntry, ComponentResult, TestResult, UrlMetadata, } from "./scoring-input.js";
+export type { DocumentRef } from "../../ailf-shared/index.d.ts";
+type DocumentRef = _DocumentRef;
+type EvalMode = _EvalMode;
+type ConcreteEvalMode = _ConcreteEvalMode;
+/** Aggregated retrieval metrics for a feature area */
+export interface AreaRetrievalMetrics {
+    area: string;
+    avgF1: number;
+    avgPrecision: number;
+    avgRecall: number;
+    /** Unique canonical slugs that were never retrieved across all tasks */
+    invisibleDocs: string[];
+    /** Number of tasks with retrieval data */
+    taskCount: number;
+    tasks: TaskRetrievalMetrics[];
+}
+/** A parsed task-to-canonical-docs mapping */
+export interface CanonicalMapping {
+    canonical_docs: {
+        slug: string;
+        reason: string;
+    }[];
+    description: string;
+    id: string;
+    reference_solution: string;
+}
+/** A document in a content release, classified by change type and evaluation tier */
+export interface ClassifiedReleaseDocument {
+    /** Feature areas whose tasks reference this document (via canonical_docs) */
+    affectedAreas: string[];
+    /** Whether this is a new, updated, or removed document */
+    changeType: "new" | "removed" | "updated";
+    /** The document slug */
+    slug: string;
+    /** For new docs: Tier A (area has tasks) or Tier B (no tasks exist) */
+    tier: "A" | "B" | "not-applicable";
+    /** Whether this document maps to any existing task's canonical_docs */
+    tracked: boolean;
+}
+/** Coverage audit report */
+export interface CoverageAuditReport {
+    /** Coverage percentage */
+    coveragePercent: number;
+    /** Covered features (with tasks) */
+    covered: (ProductFeature & {
+        actualTaskCount: number;
+    })[];
+    /** Timestamp of audit */
+    generatedAt: string;
+    /** Total features in registry */
+    totalFeatures: number;
+    /** Uncovered features (no tasks), sorted by priority */
+    uncovered: ProductFeature[];
+}
+/** Debug mode options for running a subset of tests */
+export interface DebugOptions {
+    /** Enable debug mode (run subset of tests) */
+    enabled: boolean;
+    /** Number of tests to run (default: 2) */
+    firstN?: number;
+    /** Regex pattern to filter tests by description */
+    pattern?: string;
+    /** Random sample of N tests */
+    sample?: number;
+}
+export type { DimensionName, EvalMode, ConcreteEvalMode, } from "../../ailf-shared/index.d.ts";
+export { FULL_MODE_SUBMODES } from "../../ailf-shared/index.d.ts";
+/** A classified failure mode with confidence level */
+export interface FailureMode {
+    /** How confident we are in this classification */
+    confidence: "high" | "low" | "medium";
+    /** The failure mode category */
+    mode: FailureModeType;
+    /** Source of the classification signal */
+    source: "ceiling" | "keyword" | "keyword+ceiling";
+}
+/** Summary report of failure modes across all judgments */
+export interface FailureModeReport {
+    /** Per-area failure mode distribution */
+    byArea: Record<string, {
+        area: string;
+        modes: Record<FailureModeType, number>;
+        topMode: FailureModeType;
+        totalJudgments: number;
+    }>;
+    /** Percentage of judgments classified as something other than unclassified */
+    classificationRate: number;
+    /** Judgments that were classified */
+    classifiedJudgments: {
+        classification: FailureMode;
+        judgment: GraderJudgment;
+    }[];
+    /** Summary counts by failure mode */
+    summary: Record<FailureModeType, number>;
+    /** Total judgments analyzed */
+    totalJudgments: number;
+}
+/** Failure mode classification for a low-scoring judgment */
+export type FailureModeType = "incorrect-docs" | "missing-docs" | "model-limitation" | "outdated-docs" | "poor-structure" | "unclassified";
+/** Per-feature-area score breakdown */
+export interface FeatureScore {
+    /**
+     * Actual score — agentic mode, model retrieves docs via tools.
+     * Only present when agentic evaluation data is available.
+     */
+    actualScore?: number;
+    /**
+     * Ceiling score — gold-standard docs injected directly.
+     * This is the theoretical maximum score for this area given the current docs.
+     */
+    ceilingScore: number;
+    codeCorrectness: number;
+    docCoverage: number;
+    /** Sanity documents used for this feature area's evaluation */
+    documents?: DocumentRef[];
+    /**
+     * Doc Lift — documentation quality contribution (ceiling − floor).
+     * Positive when docs help, negative when docs hurt (interference).
+     */
+    docLift: number;
+    /**
+     * Doc quality gap — room for documentation improvement (100 − ceiling).
+     * Lower is better.
+     */
+    docQualityGap: number;
+    feature: string;
+    /**
+     * Floor score — no docs, training data only.
+     * The model's inherent knowledge baseline.
+     */
+    floorScore: number;
+    /**
+     * Infrastructure efficiency — actual / ceiling (0.0–1.0).
+     * What fraction of documentation quality reaches agents through discovery?
+     * Null when ceiling ≤ 0, negative Doc Lift, or data is missing.
+     */
+    infrastructureEfficiency?: number | null;
+    /**
+     * Whether the retrieval gap is inverted — agents that can't find bad docs
+     * outperform those that can. Only true when Doc Lift is negative AND
+     * actual > ceiling. Signals a documentation quality problem being masked
+     * by retrieval failure. See evaluation-ceiling.md.
+     */
+    invertedRetrievalGap?: boolean;
+    /** Model identifier (e.g., "openai:gpt-4o"). Only present in per-model breakdowns. */
+    modelId?: string;
+    /**
+     * Whether Doc Lift is negative — docs hurt performance.
+     * When true, documentation is actively interfering with the model's
+     * training-data knowledge. See evaluation-ceiling.md.
+     */
+    negativeDocLift: boolean;
+    /**
+     * Retrieval gap — discovery/infrastructure loss (ceiling − actual).
+     * Positive means agents can't find all the good docs.
+     * Negative means agents outperform injected docs (inverted — see above).
+     * Only present when both ceiling and agentic data are available.
+     */
+    retrievalGap?: number;
+    taskCompletion: number;
+    testCount: number;
+    totalCost: number;
+    totalScore: number;
+}
+/** Scoping options to evaluate only specific areas or tasks */
+export interface FilterOptions {
+    /** Feature areas to include (filename stems, e.g., ["groq", "frameworks"]) */
+    areas?: string[];
+    /** Specific task IDs to include (e.g., ["groq-blog-queries"]) */
+    taskIds?: string[];
+}
+/** Full gap analysis report */
+export interface GapAnalysisReport {
+    /** All identified gaps, sorted by priority */
+    gaps: GapEstimate[];
+    /** Timestamp of analysis */
+    generatedAt: string;
+    /** Aggregate potential lift if all gaps were fixed */
+    totalPotentialLift: number;
+}
+/** A single gap with estimated impact if fixed */
+export interface GapEstimate {
+    /** The tasks affected by this gap */
+    affectedTaskIds: string[];
+    /** The area this gap affects */
+    area: string;
+    /** The dimension(s) bottlenecked by this gap */
+    bottleneckDimensions: string[];
+    /** Confidence in the estimate */
+    confidence: "high" | "low" | "medium";
+    /** Current score of the bottleneck dimension(s) */
+    currentDimensionScores: Record<string, number>;
+    /** Estimated composite score lift if this gap is fully closed */
+    estimatedLift: number;
+    /** The failure mode causing the gap */
+    failureMode: FailureModeType;
+    /** Priority score (lift × affectedTaskCount) */
+    priority: number;
+    /** Specific remediation description */
+    remediation: string;
+}
+/** A single grader judgment — one per assertion per test */
+export interface GraderJudgment {
+    /** The rubric template used (task-completion, code-correctness, doc-coverage) */
+    dimension: string;
+    /** The model that produced the response being graded */
+    modelId: string;
+    /** The grader's natural language reasoning */
+    reason: string;
+    /** The numeric score (0–100) */
+    score: number;
+    /** The task this judgment belongs to */
+    taskId: string;
+}
+/** Enriched grader judgment with canonical doc references, stored in reports */
+export interface StoredJudgment extends GraderJudgment {
+    /** Canonical docs that the task expected the model to use */
+    canonicalDocs?: DocumentRef[];
+}
+/** Grader consistency diagnostics — does not affect scores, reported alongside */
+export interface GraderReliability {
+    /** Inter-grader agreement (from multi-grader comparison) — Phase 3 */
+    agreement?: {
+        /** Models compared against the primary grader */
+        comparedModels: string[];
+        /** Pearson correlation with each compared model */
+        correlations: Record<string, number>;
+        /** Systematic bias (mean signed difference) per model */
+        bias: Record<string, number>;
+        /** Mean absolute difference per model */
+        meanAbsDiff: Record<string, number>;
+        /** Recommendation per candidate: comparable, divergent, or strongly-divergent */
+        recommendations: Record<string, string>;
+    };
+    /** Intra-grader consistency (from replicated grading runs) — Phase 1 */
+    consistency?: {
+        /** Average standard deviation across all grading judgments */
+        avgStdDev: number;
+        /** Max standard deviation observed (worst-case noise) */
+        maxStdDev: number;
+        /** Average score range (max - min) across judgments */
+        avgRange: number;
+        /** Number of replications per judgment */
+        replications: number;
+        /** Total judgments analyzed */
+        totalJudgments: number;
+        /** Recommended noise threshold for comparisons (2× max dimension σ) */
+        recommendedThreshold: number;
+        /** Per-dimension consistency */
+        perDimension: {
+            taskCompletion: {
+                avgStdDev: number;
+                maxStdDev: number;
+            };
+            codeCorrectness: {
+                avgStdDev: number;
+                maxStdDev: number;
+            };
+            docCoverage: {
+                avgStdDev: number;
+                maxStdDev: number;
+            };
+        };
+    };
+    /** Grader model used for this evaluation */
+    graderModel: string;
+    /** Sensitivity / discrimination power (from degradation testing) — Phase 4 */
+    sensitivity?: {
+        /** % of paired comparisons where grader ranked original higher */
+        concordanceRate: number;
+        /** Average score separation (original - degraded) */
+        avgSeparation: number;
+        /** Total paired comparisons analyzed */
+        totalPairs: number;
+        /** Per-dimension sensitivity */
+        perDimension: {
+            taskCompletion: {
+                concordanceRate: number;
+                avgSeparation: number;
+            };
+            codeCorrectness: {
+                concordanceRate: number;
+                avgSeparation: number;
+            };
+            docCoverage: {
+                concordanceRate: number;
+                avgSeparation: number;
+            };
+        };
+    };
+    /** Criterion validity (from human reference grades) — Phase 2 */
+    validity?: {
+        /** Mean Absolute Error vs. human grades */
+        mae: number;
+        /** Pearson correlation with human grades */
+        correlation: number;
+        /** Systematic bias (positive = grader scores higher than humans) */
+        bias: number;
+        /** Per-dimension correlation with human grades */
+        perDimension: {
+            taskCompletion: number;
+            codeCorrectness: number;
+            docCoverage: number;
+        };
+        /** Number of human-graded reference samples */
+        sampleSize: number;
+        /** Whether the grader passes the MAE threshold */
+        passesThreshold: boolean;
+    };
+}
+/** Cost estimate for targeted LOO before execution */
+export interface LOOCostEstimate {
+    /** Breakdown per ambiguous task */
+    perTask: {
+        /** Estimated cost for this task's LOO */
+        estimatedCost: number;
+        /** Number of documents to evaluate */
+        numDocuments: number;
+        /** Number of tests per document */
+        numTests: number;
+        /** Task ID */
+        taskId: string;
+    }[];
+    /** Total estimated additional cost */
+    totalEstimatedCost: number;
+}
+/** Result of targeted leave-one-out attribution for an ambiguous task */
+export interface LOOResult {
+    /** Additional eval cost incurred for this LOO analysis */
+    additionalCost: number;
+    /** Per-document marginal contributions */
+    contributions: {
+        /** Marginal score contribution of this document */
+        marginalContribution: number;
+        /** The document slug */
+        slug: string;
+        /** Whether the contribution is within grader noise floor */
+        withinNoiseFloor: boolean;
+    }[];
+    /** Whether contributions sum to approximately the total delta (±15%) */
+    sumMatchesTotal: boolean;
+    /** The task being analyzed */
+    taskId: string;
+}
+/** A parsed model entry from config/models.yaml */
+export interface ModelEntry {
+    config?: Record<string, unknown>;
+    /** Environment variable gate — model is skipped when this var is unset */
+    env?: string;
+    id: string;
+    label: string;
+    modes?: string[];
+}
+/** Parsed config/models.yaml structure */
+export interface ModelsConfig {
+    defaults: Record<string, unknown>;
+    grader: {
+        id: string;
+        label?: string;
+    };
+    /** Global max concurrency across all providers */
+    maxConcurrency?: number;
+    models: ModelEntry[];
+}
+/** Per-model score entry — one element per evaluated model */
+export interface PerModelEntry {
+    /** Model label (human-readable name) */
+    label: string;
+    /** Provider model ID (e.g., "openai:chat:gpt-4o") */
+    modelId: string;
+    /** Model-level aggregates */
+    overall: {
+        avgDocLift: number;
+        avgScore: number;
+        /** Total cost for this model's tests */
+        cost?: number;
+        /** Number of tests run for this model */
+        testCount: number;
+    };
+    /** Per-area scores for this model */
+    scores: FeatureScore[];
+}
+/** Per-model score breakdown within a single evaluation run */
+export type PerModelScores = PerModelEntry[];
+/** Options for a pipeline run */
+export interface PipelineOptions {
+    /** Debug mode — run a subset of tests for speed */
+    debug?: DebugOptions;
+    /** Run validation only, don't execute steps */
+    dryRun: boolean;
+    /** Scope evaluation to specific areas or tasks */
+    filter?: FilterOptions;
+    /** Which evaluation mode to run */
+    mode: EvalMode;
+    /** Bypass all pipeline caching (force re-execution of every step) */
+    noCache: boolean;
+    /** Skip the eval step (recalculate from existing results) */
+    skipEval: boolean;
+    /** Skip the documentation fetch step (reuse cached contexts) */
+    skipFetch: boolean;
+    /** Documentation source name (from config/sources.yaml) */
+    source?: string;
+}
+/** A Promptfoo share URL tagged with the evaluation mode that produced it. */
+export interface PromptfooUrlEntry {
+    mode: ConcreteEvalMode;
+    url: string;
+}
+/**
+ * Mutable state bag shared between pipeline steps during execution.
+ *
+ * Flows through the orchestrator loop — producer steps write to it,
+ * consumer steps read from it. This eliminates filesystem round-trips
+ * for small inter-step coordination values (e.g., a reportId generated
+ * by PublishReportStep and consumed by CallbackStep).
+ *
+ * Large artifacts (score-summary.json, comparison-report.json) still
+ * live on the filesystem — PipelineState is for ephemeral values that
+ * only matter during a single pipeline execution.
+ */
+export interface PipelineState {
+    /** Report ID generated by PublishReportStep, consumed by CallbackStep + orchestrator job update */
+    reportId?: string;
+    /** Eval fingerprint computed by RunEvalStep, consumed by PublishReportStep */
+    evalFingerprint?: string;
+    /** Promptfoo share URLs produced by RunEvalStep, consumed by PublishReportStep */
+    promptfooUrls?: PromptfooUrlEntry[];
+}
+/** Result of a full pipeline run */
+export interface PipelineResult {
+    /** Cache hit/miss statistics for this run */
+    cache?: {
+        hits: number;
+        misses: number;
+        skipped: number;
+        total: number;
+    };
+    /** Total duration in milliseconds */
+    durationMs: number;
+    /** @deprecated Use `promptfooUrls` — kept for backward compatibility */
+    promptfooUrl?: string;
+    /** Per-mode Promptfoo share URLs (one per sub-eval that produced a shareable link) */
+    promptfooUrls?: PromptfooUrlEntry[];
+    /** Results per step */
+    steps: Record<string, StepResult>;
+    /** Overall success (all non-skipped steps succeeded) */
+    success: boolean;
+    /** Validation issues found (if any) */
+    validation: ValidationResult;
+}
+/** Result of a Tier B probe (new feature, no tasks) */
+export interface ProbeResult {
+    /** API names the model extracted from the docs */
+    apiNamesExtracted: string[];
+    /** The documents evaluated */
+    documentSlugs: string[];
+    /** API names the model used that are NOT in the docs (hallucinations) */
+    hallucinatedApis: string[];
+    /** Whether the model produced any code output */
+    producedCode: boolean;
+    /** The model's raw output (for human review) */
+    rawOutput: string;
+    /** Suggested task descriptions based on document content analysis */
+    suggestedTasks: string[];
+    /** Qualitative assessment */
+    usability: "not-usable" | "partially-usable" | "usable";
+}
+/** A product feature in the registry */
+export interface ProductFeature {
+    /** Maps to tasks/<area>.yaml */
+    area?: string;
+    /** Unique feature identifier */
+    id: string;
+    /** Human-readable name */
+    name: string;
+    /** Priority for evaluation coverage */
+    priority: "critical" | "high" | "low" | "medium";
+    /** Product sections this feature belongs to */
+    sections: string[];
+    /** Coverage status */
+    status: "covered" | "out-of-scope" | "planned" | "uncovered";
+    /** Number of evaluation tasks (if covered) */
+    taskCount?: number;
+}
+/** Full classification of a content release for evaluation */
+export interface ReleaseClassification {
+    /** All classified documents in the release */
+    documents: ClassifiedReleaseDocument[];
+    /** Timestamp of classification */
+    generatedAt: string;
+    /** Summary counts */
+    summary: {
+        /** New documents in areas with existing tasks */
+        tierA: number;
+        /** New documents in areas without tasks */
+        tierB: number;
+        /** Total documents in the release */
+        total: number;
+        /** Documents tracked by at least one task */
+        tracked: number;
+        /** Updated documents (existing in canonical sets) */
+        updated: number;
+        /** Documents not tracked by any task */
+        untracked: number;
+    };
+}
+/** Cross-area release impact report */
+export interface ReleaseImpactReport {
+    /** Per-area impact summaries */
+    areas: {
+        /** Area name */
+        area: string;
+        /** Overall area score delta */
+        delta: number;
+        /** Whether the area regressed (delta < -threshold) */
+        regressed: boolean;
+        /** Per-task deltas within this area */
+        tasks: {
+            /** Documents attributed to this delta */
+            attributedDocs: string[];
+            /** Task score delta */
+            delta: number;
+            /** Task ID */
+            taskId: string;
+        }[];
+    }[];
+    /** Areas evaluated but with no score change (within noise) */
+    confirmedUnchanged: string[];
+    /** Classified documents from the release */
+    documents: ClassifiedReleaseDocument[];
+    /** Timestamp of report */
+    generatedAt: string;
+    /** Areas not evaluated (no release documents match) */
+    notEvaluated: string[];
+    /** Overall score delta across all affected areas */
+    overallDelta: number;
+    /** Probe results for Tier B documents (if any) */
+    probes: ProbeResult[];
+    /** Areas that regressed (delta < -threshold) */
+    regressions: string[];
+}
+/** Top-level retrieval metrics (added to ScoreSummary when agentic data exists) */
+export interface RetrievalMetrics {
+    /** Per-area retrieval metrics */
+    areas: AreaRetrievalMetrics[];
+    /** All canonical doc slugs that were never retrieved by any task */
+    globalInvisibleDocs: string[];
+    /** Overall averages across all areas */
+    overall: {
+        avgF1: number;
+        avgPrecision: number;
+        avgRecall: number;
+    };
+}
+/** Top-level score summary (the shape of score-summary.json) */
+export interface ScoreSummary {
+    belowCritical: string[];
+    /**
+     * All Sanity documents used across the entire evaluation.
+     * De-duplicated union of documents from all areas/tasks.
+     * Populated during the gap-analysis enrichment step from the document manifest.
+     */
+    documentManifest?: DocumentRef[];
+    /**
+     * Which evaluation modes contributed data to this summary.
+     * - `'full'`: both baseline and agentic data present
+     * - `'baseline'`: floor + ceiling only (no agentic data)
+     * - `'agentic'`: actual only (no floor/ceiling data)
+     * - `'observed'`: observed mode data
+     * Absent in legacy summaries — treat as `'baseline'` for backward compat.
+     */
+    evaluationMode?: EvalMode;
+    /** Failure mode analysis (Phase 3a) — diagnostic breakdown of why scores are low */
+    failureModes?: FailureModeReport;
+    /**
+     * Low-scoring grader judgments (normalized score < 70) — the raw "red text"
+     * from the grader explaining why specific tests scored poorly.
+     * Scores are normalized to 0–100 scale before filtering.
+     * Each judgment includes the canonical docs the task expected.
+     */
+    lowScoringJudgments?: StoredJudgment[];
+    /** Gap analysis recommendations (Phase 3b) — prioritized remediation plan */
+    recommendations?: GapAnalysisReport;
+    /** Grader reliability diagnostics (does not affect scores) */
+    graderReliability?: GraderReliability;
+    lowestArea: string;
+    lowestScore: number;
+    /** Areas with negative Doc Lift — docs hurt performance. Empty when all lifts are positive. */
+    negativeDocLiftAreas?: {
+        area: string;
+        docLift: number;
+    }[];
+    overall: {
+        /** Average actual (agentic) score across areas. Absent if no agentic data. */
+        avgActualScore?: number;
+        /** Average ceiling score across all areas */
+        avgCeilingScore: number;
+        avgScore: number;
+        avgDocLift: number;
+        /** Average doc quality gap across all areas (100 − ceiling) */
+        avgDocQualityGap: number;
+        /** Average floor score across all areas */
+        avgFloorScore: number;
+        /** Average infrastructure efficiency (actual / ceiling). Absent if no agentic data. */
+        avgInfrastructureEfficiency?: number;
+        /** Average retrieval gap (ceiling − actual). Absent if no agentic data. */
+        avgRetrievalGap?: number;
+        /** Count of areas with negative Doc Lift */
+        negativeDocLiftCount: number;
+        cost?: {
+            total: number;
+            perTest: number;
+            graderTotal: number;
+            graderModel?: string;
+            totalTokens: number;
+        };
+    };
+    /** Per-model score breakdowns (Tier 3 from metrics-design.md) */
+    perModel?: PerModelScores;
+    /** Retrieval metrics from agentic mode (Approach 4) */
+    retrievalMetrics?: RetrievalMetrics;
+    scores: FeatureScore[];
+    source?: {
+        name: string;
+        baseUrl: string;
+        dataset?: string;
+        /** Sanity release perspective ID (when evaluating content releases) */
+        perspective?: string;
+        projectId: string;
+    };
+    timestamp: string;
+}
+/** Result of a single pipeline step */
+export type StepResult = {
+    status: "failed";
+    durationMs: number;
+    error: string;
+} | {
+    status: "skipped";
+    reason: string;
+} | {
+    status: "success";
+    durationMs: number;
+    summary: string;
+};
+/** Per-task attribution result — which documents caused a score change */
+export interface TaskAttribution {
+    /** The feature area */
+    area: string;
+    /** Documents attributed (1 for unambiguous, N for ambiguous, 0 for uncorrelated) */
+    attributedDocs: string[];
+    /** Attribution classification */
+    classification: AttributionClass;
+    /** Score delta for this task (area-level delta used as proxy) */
+    delta: number;
+    /** The task ID */
+    taskId: string;
+    /** Whether delta is within grader noise floor */
+    withinNoiseFloor: boolean;
+}
+/** A parsed task entry from a tasks/*.yaml file (single-definition format) */
+export interface TaskEntry {
+    assert?: unknown[];
+    /** Baseline generation options (new format only). */
+    baseline?: {
+        enabled?: boolean;
+        rubric?: "abbreviated" | "full" | "none";
+    };
+    description: string;
+    /** Explicit task ID — determines the canonical context filename. */
+    id?: string;
+    transform?: string;
+    vars?: Record<string, unknown>;
+}
+/** Retrieval metrics for a single task */
+export interface TaskRetrievalMetrics {
+    /** Feature area (from task file) */
+    area: string;
+    /** Slugs expected (from canonical_docs) */
+    expected: string[];
+    /** F1 score: 2 * precision * recall / (precision + recall) */
+    f1: number;
+    /** Slugs in both retrieved and expected */
+    hits: string[];
+    /** Expected slugs the agent missed */
+    missed: string[];
+    /** Precision: |retrieved ∩ canonical| / |retrieved| */
+    precision: number;
+    /** Recall: |retrieved ∩ canonical| / |canonical| */
+    recall: number;
+    /** Slugs the agent actually retrieved */
+    retrieved: string[];
+    /** Task ID matching the task YAML definition */
+    taskId: string;
+    /** Retrieved slugs not in the expected set */
+    unexpected: string[];
+}
+/** A single validation finding — either an error (blocks execution) or a warning */
+export interface ValidationIssue {
+    /** What went wrong */
+    message: string;
+    /** Optional file path relevant to the issue */
+    path?: string;
+    /** Error = blocks execution, warning = informational */
+    severity: ValidationSeverity;
+    /** Which step or check produced this finding */
+    source: string;
+}
+/** Result of a validation pass */
+export interface ValidationResult {
+    /** All findings */
+    issues: ValidationIssue[];
+    /** True if there are no errors (warnings are OK) */
+    valid: boolean;
+}
+/** Severity of a validation finding */
+export type ValidationSeverity = "error" | "warning";
+/**
+ * Default noise threshold — re-exported from @sanity/ailf-shared.
+ * Named DEFAULT_NOISE_THRESHOLD here for backward compatibility with eval code
+ * that imports it from @sanity/ailf-core.
+ */
+export { NOISE_THRESHOLD as DEFAULT_NOISE_THRESHOLD } from "../../ailf-shared/index.d.ts";
+/** Per-area delta with dimension breakdowns */
+export interface AreaDelta {
+    /** Actual (agentic) score delta. Absent if either run lacks agentic data. */
+    actualDelta?: number;
+    area: string;
+    /** Baseline total score */
+    baseline: number;
+    /** Ceiling score delta (doc quality change). If ceiling moved, doc content changed. */
+    ceilingDelta: number;
+    /** Change classification based on noise threshold */
+    change: ChangeClass;
+    /** Cost delta (if cost data is available) */
+    costDelta?: number;
+    /** Overall score delta (experiment − baseline) */
+    delta: number;
+    /** Per-dimension deltas */
+    dimensions: {
+        taskCompletion: {
+            baseline: number;
+            experiment: number;
+            delta: number;
+        };
+        codeCorrectness: {
+            baseline: number;
+            experiment: number;
+            delta: number;
+        };
+        docCoverage: {
+            baseline: number;
+            experiment: number;
+            delta: number;
+        };
+    };
+    /** Doc Lift delta */
+    docLiftDelta: number;
+    /** Experiment total score */
+    experiment: number;
+    /** Floor score delta (model knowledge change — should be ~0 between runs) */
+    floorDelta: number;
+    /** Infrastructure efficiency delta. Absent if either run lacks full decomposition data. */
+    infrastructureEfficiencyDelta?: number;
+    /** Retrieval gap delta. Positive = gap grew (worse), negative = gap shrank (better). */
+    retrievalGapDelta?: number;
+}
+/** Attribution classification for a task delta */
+export type AttributionClass = "ambiguous" | "unambiguous" | "uncorrelated";
+/** Complete attribution report for a set of changed documents */
+export interface AttributionReport {
+    /** All task attributions */
+    attributions: TaskAttribution[];
+    /** Summary counts */
+    summary: {
+        ambiguous: number;
+        unambiguous: number;
+        uncorrelated: number;
+        withinNoise: number;
+    };
+    /** Changed documents that aren't in any task's canonical docs */
+    untrackedDocs: string[];
+}
+/** Classification of a feature area's score change */
+export type ChangeClass = "improved" | "regressed" | "unchanged";
+/** Options for the compare function */
+export interface CompareOptions {
+    /** Grader consistency data — if provided, used to compute empirical noise threshold */
+    graderConsistency?: GraderConsistencyData;
+    /**
+     * Deltas within ±threshold are classified as "unchanged".
+     * Defaults to DEFAULT_NOISE_THRESHOLD (2).
+     * Overridden by graderConsistency.recommendedThreshold when available.
+     */
+    noiseThreshold?: number;
+}
+/**
+ * Structured comparison between two evaluation runs.
+ *
+ * This is the core comparison primitive (see BP5 in the evaluation roadmap).
+ * Every comparison scenario uses the same shape — what varies is what
+ * produced each ScoreSummary:
+ *
+ * | Scenario            | Baseline            | Experiment         |
+ * |---------------------|---------------------|--------------------|
+ * | Doc improvement     | Before rewrite      | After rewrite      |
+ * | Model comparison    | Model A             | Model B            |
+ * | Model regression    | Previous version    | Current version    |
+ * | Branch validation   | Production          | Branch deploy      |
+ * | Infrastructure ROI  | Naive agent         | Optimized agent    |
+ * | Training freshness  | Without docs        | With docs (DocLift)|
+ */
+export interface ComparisonReport {
+    /** Detailed per-area breakdowns */
+    areas: AreaDelta[];
+    /** Per-document attribution (when changed docs are known) */
+    attribution?: AttributionReport;
+    /** The "before" or "control" summary */
+    baseline: ScoreSummary;
+    /** Aggregate deltas */
+    deltas: {
+        /** Overall score delta (experiment.avgScore − baseline.avgScore) */
+        overall: number;
+        /** Per-area total score deltas */
+        perArea: Record<string, number>;
+        /** Per-dimension average deltas (across all areas) */
+        perDimension: {
+            taskCompletion: number;
+            codeCorrectness: number;
+            docCoverage: number;
+        };
+        /** Doc Lift average delta */
+        docLift: number;
+        /** Cost delta (if both runs have cost data) */
+        cost?: number;
+        /** Per-model overall score deltas (when both summaries have per-model data) */
+        perModel?: {
+            delta: number;
+            modelId: string;
+        }[];
+    };
+    /** The "after" or "treatment" summary */
+    experiment: ScoreSummary;
+    /** When this comparison was generated */
+    generatedAt: string;
+    /** Areas that improved beyond the noise threshold */
+    improved: string[];
+    /** Areas present in only one of the two summaries */
+    mismatched: {
+        /** Areas in baseline but not experiment */
+        onlyInBaseline: string[];
+        /** Areas in experiment but not baseline */
+        onlyInExperiment: string[];
+    };
+    /** Noise threshold used for classification */
+    noiseThreshold: number;
+    /** Whether the noise threshold was derived from empirical grader consistency data */
+    noiseThresholdEmpirical: boolean;
+    /** Areas that regressed beyond the noise threshold */
+    regressed: string[];
+    /** Areas within the noise threshold */
+    unchanged: string[];
+}
+/** Confidence annotation for a single delta */
+export interface ConfidenceAnnotation {
+    /** 95% confidence interval half-width (±this value) */
+    ci95: number;
+    /** The measured delta */
+    delta: number;
+    /** The grader σ used to compute this CI */
+    graderSigma: number;
+    /** Whether the delta exceeds the confidence interval (statistically significant) */
+    significant: boolean;
+}
+/**
+ * Grader consistency data for empirical confidence intervals.
+ * This is the subset of GraderConsistency needed for comparison annotation.
+ * Matches the shape produced by pipeline/grader-consistency.ts.
+ */
+export interface GraderConsistencyData {
+    /** Per-dimension consistency metrics */
+    perDimension: {
+        codeCorrectness: {
+            avgStdDev: number;
+            maxStdDev: number;
+        };
+        docCoverage: {
+            avgStdDev: number;
+            maxStdDev: number;
+        };
+        taskCompletion: {
+            avgStdDev: number;
+            maxStdDev: number;
+        };
+    };
+    /** Recommended noise threshold for comparisons (2× max dimension σ) */
+    recommendedThreshold: number;
+}
+/** ISO 8601 timestamp string */
+export type ISOTimestamp = string & {
+    readonly __brand: "ISOTimestamp";
+};
+/** Query parameters for finding comparable baseline reports */
+export interface LineageQuery {
+    /** Match on feature areas (overlapping set) */
+    areas?: string[];
+    /** Only reports before this timestamp */
+    before?: ISOTimestamp;
+    /** Match on evaluation mode */
+    mode: EvalMode;
+    /** Match on models */
+    models?: string[];
+    /** Match on source */
+    source?: Partial<ReportProvenance["source"]>;
+}
+/** Result of publishing a report to all sinks */
+export interface PublishResult {
+    /** The auto-comparison (if a comparable baseline was found) */
+    comparison?: ComparisonReport;
+    /** The published report */
+    report: Report;
+    /** Per-sink results */
+    sinkResults: {
+        name: string;
+        result: SinkResult;
+    }[];
+}
+/** A published evaluation report — the atomic unit of the report store */
+export interface Report {
+    /** Optional auto-comparison against the most recent comparable report */
+    comparison?: ComparisonReport;
+    /** When the evaluation completed */
+    completedAt: ISOTimestamp;
+    /** How long the pipeline took */
+    durationMs: number;
+    /** Stable unique identifier (UUID v7 for time-sortability) */
+    id: ReportId;
+    /** What produced this report */
+    provenance: ReportProvenance;
+    /** The full score summary */
+    summary: ScoreSummary;
+    /** Optional human-supplied label */
+    tag?: string;
+}
+/** Branded type for report identifiers (UUID v7 for time-sortability) */
+export type ReportId = string & {
+    readonly __brand: "ReportId";
+};
+/** Full provenance metadata for an evaluation report */
+export interface ReportProvenance {
+    /** Which feature areas were evaluated */
+    areas: string[];
+    /** Content hash of the documentation context at eval time */
+    contextHash?: string;
+    /**
+     * Evaluation fingerprint — SHA-256 of all inputs that affect eval output.
+     * Used for cross-environment cache lookup (CI → Content Lake).
+     * @see docs/design-docs/content-lake-eval-caching.md
+     */
+    evalFingerprint?: string;
+    /** Git metadata (when run from CI) */
+    git?: {
+        branch: string;
+        prNumber?: number;
+        repo: string;
+        sha: string;
+    };
+    /** Grader model used for scoring */
+    graderModel: string;
+    /** Evaluation mode */
+    mode: EvalMode;
+    /** Models under evaluation */
+    models: {
+        id: string;
+        label: string;
+    }[];
+    /** @deprecated Use `promptfooUrls` — kept for backward compatibility */
+    promptfooUrl?: string;
+    /** Per-mode Promptfoo share URLs (one per sub-eval that produced a shareable link) */
+    promptfooUrls?: PromptfooUrlEntry[];
+    /** Documentation source configuration */
+    source: {
+        baseUrl: string;
+        dataset?: string;
+        name: string;
+        perspective?: string;
+        projectId?: string;
+    };
+    /** Sanity document IDs that were targeted (if using --sanity-document) */
+    targetDocuments?: string[];
+    /** Which specific task IDs were evaluated (if scoped) */
+    taskIds?: string[];
+    /** What initiated this evaluation */
+    trigger: ReportTrigger;
+}
+/** What triggered this evaluation */
+export type ReportTrigger = {
+    type: "ci";
+    runId: string;
+    workflow: string;
+} | {
+    type: "cross-repo";
+    callerRef?: string;
+    callerRepo: string;
+} | {
+    type: "manual";
+} | {
+    type: "scheduled";
+    schedule: string;
+} | {
+    type: "webhook";
+    documentId?: string;
+    source: string;
+};
+/** Health check result for a sink */
+export type SinkHealthStatus = {
+    healthy: false;
+    reason: string;
+} | {
+    healthy: true;
+};
+/** Payload delivered to sinks — extends Report with threshold context */
+export interface SinkPayload {
+    /** Comparison report (if available) */
+    comparison?: ComparisonReport;
+    /** Maximum severity across all violations */
+    maxSeverity: "none" | ThresholdSeverity;
+    /** The published report */
+    report: Report;
+    /** Threshold violations (empty if thresholds not configured) */
+    violations: ThresholdViolation[];
+}
+/** Result of a sink publish attempt */
+export type SinkResult = {
+    status: "failed";
+    error: string;
+} | {
+    status: "skipped";
+    reason: string;
+} | {
+    status: "success";
+    detail?: string;
+};
+/** Result of evaluating all thresholds against a score summary */
+export interface ThresholdEvaluation {
+    /** Maximum severity across all violations ("none" if no violations) */
+    maxSeverity: "critical" | "info" | "none" | "warning";
+    /** Whether all thresholds pass (no violations) */
+    pass: boolean;
+    /** All violations, sorted by severity (critical first) then by delta */
+    violations: ThresholdViolation[];
+}
+/** Severity level for threshold violations */
+export type ThresholdSeverity = "critical" | "info" | "warning";
+/** A single threshold violation — one dimension or metric exceeding its limit */
+export interface ThresholdViolation {
+    /** The actual measured value */
+    actual: number;
+    /** The area (or "overall" for global thresholds) */
+    area: string;
+    /** Human-readable description of what violated and why */
+    description: string;
+    /** The dimension or metric that violated (e.g., "composite", "task-completion", "doc-lift") */
+    metric: string;
+    /** Severity classification based on severity config rules */
+    severity: ThresholdSeverity;
+    /** The threshold value that was exceeded */
+    threshold: number;
+}