@sanity/ailf 0.1.0

package/dist/pipeline/targeted-loo.js

@@ -0,0 +1,203 @@
+/**
+ * pipeline/targeted-loo.ts
+ *
+ * Targeted leave-one-out (LOO) attribution for ambiguous cases.
+ *
+ * Phase 4d of the Scenario Matrix implementation.
+ *
+ * When correlation-based attribution (Phase 2c) identifies ambiguous tasks
+ * (2+ changed docs map to the same task), targeted LOO resolves the ambiguity
+ * by running per-document mini-evaluations to measure each document's
+ * marginal contribution.
+ *
+ * This module handles:
+ * - Cost estimation before execution (to enable user confirmation)
+ * - Result analysis after LOO runs complete
+ * - Integration with the existing AttributionReport
+ *
+ * The actual evaluation execution is handled by the pipeline orchestrator —
+ * this module is pure computation on inputs and outputs.
+ *
+ * @see docs/exec-plans/completed/scenario-matrix-implementation/phase-4-content-release-integration.md
+ * @see docs/design-docs/scenario-matrix/per-document-attribution.md
+ */
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+/** Default estimated cost per test (provider + grader combined) */
+const DEFAULT_COST_PER_TEST = 0.08;
+/** Sum tolerance — marginal contributions should sum to ~total delta (±15%) */
+const SUM_TOLERANCE = 0.15;
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+/**
+ * Analyze LOO evaluation results to compute marginal contributions.
+ *
+ * Given the full-release score and per-document revert scores,
+ * calculates each document's marginal contribution as:
+ *   marginal(doc) = fullReleaseScore - revertedScore(doc)
+ *
+ * @param taskId - The task being analyzed
+ * @param fullReleaseDelta - The total task delta from the full release
+ * @param revertResults - Per-document scores when that document is reverted
+ * @param noiseThreshold - Threshold for marking contributions as noise
+ * @param additionalCost - Actual cost incurred for the LOO evaluations
+ * @returns LOO result with per-document marginal contributions
+ */
+export function analyzeLOOResults(taskId, fullReleaseDelta, revertResults, noiseThreshold, additionalCost) {
+    const contributions = revertResults.map(({ revertedDelta, slug }) => {
+        // Marginal contribution = full delta - delta when this doc is reverted
+        const marginalContribution = fullReleaseDelta - revertedDelta;
+        return {
+            marginalContribution: Math.round(marginalContribution * 10) / 10,
+            slug,
+            withinNoiseFloor: Math.abs(marginalContribution) <= noiseThreshold,
+        };
+    });
+    // Check if contributions sum to approximately the total delta
+    const contributionSum = contributions.reduce((sum, c) => sum + c.marginalContribution, 0);
+    const sumMatchesTotal = fullReleaseDelta === 0 ||
+        Math.abs(contributionSum - fullReleaseDelta) / Math.abs(fullReleaseDelta) <=
+            SUM_TOLERANCE;
+    return {
+        additionalCost,
+        contributions: contributions.sort((a, b) => Math.abs(b.marginalContribution) - Math.abs(a.marginalContribution)),
+        sumMatchesTotal,
+        taskId,
+    };
+}
+/**
+ * Enrich an attribution report with LOO results for ambiguous tasks.
+ *
+ * Replaces the "ambiguous" classification with resolved per-document
+ * contributions for tasks that have LOO data.
+ *
+ * @param attribution - Original attribution report
+ * @param looResults - LOO results for ambiguous tasks
+ * @returns New attribution report with LOO data integrated
+ */
+export function enrichAttributionWithLOO(attribution, looResults) {
+    const looByTask = new Map();
+    for (const result of looResults) {
+        looByTask.set(result.taskId, result);
+    }
+    return {
+        ...attribution,
+        looResults,
+    };
+}
+/**
+ * Estimate the cost of running targeted LOO for ambiguous tasks.
+ *
+ * @param ambiguousTasks - Tasks identified as ambiguous
+ * @param testsPerTask - Number of tests per task (from task YAML)
+ * @param costPerTest - Estimated cost per test (default: $0.08)
+ * @returns Cost estimate with per-task breakdown
+ */
+export function estimateLOOCost(ambiguousTasks, testsPerTask, costPerTest = DEFAULT_COST_PER_TEST) {
+    const perTask = [];
+    for (const task of ambiguousTasks) {
+        const numDocuments = task.attributedDocs.length;
+        const numTests = testsPerTask[task.taskId] ?? 6; // Default estimate
+        const estimatedCost = numDocuments * numTests * costPerTest;
+        perTask.push({
+            estimatedCost: Math.round(estimatedCost * 100) / 100,
+            numDocuments,
+            numTests,
+            taskId: task.taskId,
+        });
+    }
+    const totalEstimatedCost = perTask.reduce((sum, t) => sum + t.estimatedCost, 0);
+    return {
+        perTask,
+        totalEstimatedCost: Math.round(totalEstimatedCost * 100) / 100,
+    };
+}
+/**
+ * Identify ambiguous tasks that would benefit from targeted LOO.
+ *
+ * Filters the attribution report to find tasks where 2+ changed documents
+ * are in the canonical set and the delta is outside the noise floor.
+ *
+ * @param attribution - Attribution report from Phase 2c
+ * @returns Ambiguous task attributions suitable for LOO
+ */
+export function findAmbiguousTasks(attribution) {
+    return attribution.attributions.filter((a) => a.classification === "ambiguous" && !a.withinNoiseFloor);
+}
+// ---------------------------------------------------------------------------
+// Formatting
+// ---------------------------------------------------------------------------
+/**
+ * Format a LOO cost estimate for console output (for user confirmation).
+ */
+export function formatLOOCostEstimate(estimate) {
+    const lines = [];
+    lines.push("💰 TARGETED LOO COST ESTIMATE");
+    lines.push("");
+    lines.push(`  Targeted LOO for ${estimate.perTask.length} ambiguous task(s):`);
+    lines.push("");
+    for (const task of estimate.perTask) {
+        lines.push(`    ${task.taskId}: ${task.numDocuments} documents × ~${task.numTests} tests × $${DEFAULT_COST_PER_TEST}/test = ~$${task.estimatedCost.toFixed(2)}`);
+    }
+    lines.push("");
+    lines.push(`  Total additional cost: ~$${estimate.totalEstimatedCost.toFixed(2)}`);
+    lines.push("");
+    return lines.join("\n");
+}
+/**
+ * Format LOO results for console output.
+ */
+export function formatLOOResultsConsole(results) {
+    const lines = [];
+    lines.push("🔬 TARGETED LOO RESULTS");
+    lines.push("");
+    for (const result of results) {
+        lines.push(`  ${result.taskId}:`);
+        for (const c of result.contributions) {
+            const sign = c.marginalContribution >= 0 ? "+" : "";
+            const noise = c.withinNoiseFloor ? " (within noise)" : "";
+            lines.push(`    ${c.slug}: ${sign}${c.marginalContribution.toFixed(1)}${noise}`);
+        }
+        if (!result.sumMatchesTotal) {
+            lines.push("    ⚠️  Contributions don't sum to total delta (interaction effects likely)");
+        }
+        lines.push(`    Additional cost: $${result.additionalCost.toFixed(2)}`);
+        lines.push("");
+    }
+    return lines.join("\n");
+}
+/**
+ * Format LOO results as markdown.
+ */
+export function formatLOOResultsMarkdown(results) {
+    const lines = [];
+    lines.push("#### 🔬 Targeted LOO Attribution");
+    lines.push("");
+    if (results.length === 0) {
+        lines.push("No ambiguous tasks required LOO resolution.");
+        lines.push("");
+        return lines.join("\n");
+    }
+    for (const result of results) {
+        lines.push(`**${result.taskId}:**`);
+        lines.push("");
+        for (const c of result.contributions) {
+            const sign = c.marginalContribution >= 0 ? "+" : "";
+            const pct = result.contributions.length > 0
+                ? Math.round((Math.abs(c.marginalContribution) /
+                    result.contributions.reduce((s, cc) => s + Math.abs(cc.marginalContribution), 0)) *
+                    100)
+                : 0;
+            const noise = c.withinNoiseFloor ? " ⚠️" : "";
+            lines.push(`- \`${c.slug}\`: ${sign}${c.marginalContribution.toFixed(1)} (${pct}%)${noise}`);
+        }
+        if (!result.sumMatchesTotal) {
+            lines.push("> ⚠️ Marginal contributions don't sum to total delta due to interaction effects.");
+        }
+        lines.push(`- Additional cost: $${result.additionalCost.toFixed(2)}`);
+        lines.push("");
+    }
+    return lines.join("\n");
+}

package/dist/pipeline/thresholds.d.ts

@@ -0,0 +1,27 @@
+/**
+ * pipeline/thresholds.ts
+ *
+ * Threshold evaluation engine — compares a ScoreSummary against configurable
+ * quality thresholds and produces a set of typed violations.
+ *
+ * Pure functions only: no I/O, no side effects. The caller (pipeline.ts,
+ * publish-report-step.ts) is responsible for loading the threshold config
+ * and score summary.
+ *
+ * @see docs/exec-plans/completed/scenario-matrix-implementation/phase-5-readiness-thresholds.md
+ */
+import type { ThresholdConfig } from "./schemas.js";
+import type { ComparisonReport, ScoreSummary, ThresholdEvaluation, ThresholdViolation } from "./types.js";
+/**
+ * Evaluate regression thresholds against a comparison report.
+ *
+ * Returns violations for regressions that exceed configured thresholds.
+ */
+export declare function evaluateRegressionThresholds(comparison: ComparisonReport, config: ThresholdConfig): ThresholdViolation[];
+/**
+ * Evaluate a score summary against threshold configuration.
+ *
+ * Returns all violations sorted by severity (critical first), then by
+ * the magnitude of the threshold breach (largest gap first).
+ */
+export declare function evaluateThresholds(scores: ScoreSummary, config: ThresholdConfig): ThresholdEvaluation;

package/dist/pipeline/thresholds.js

@@ -0,0 +1,245 @@
+/**
+ * pipeline/thresholds.ts
+ *
+ * Threshold evaluation engine — compares a ScoreSummary against configurable
+ * quality thresholds and produces a set of typed violations.
+ *
+ * Pure functions only: no I/O, no side effects. The caller (pipeline.ts,
+ * publish-report-step.ts) is responsible for loading the threshold config
+ * and score summary.
+ *
+ * @see docs/exec-plans/completed/scenario-matrix-implementation/phase-5-readiness-thresholds.md
+ */
+// ---------------------------------------------------------------------------
+// Severity priority for sorting (higher = more severe)
+// ---------------------------------------------------------------------------
+const SEVERITY_RANK = {
+    critical: 3,
+    info: 1,
+    warning: 2,
+};
+/**
+ * Evaluate regression thresholds against a comparison report.
+ *
+ * Returns violations for regressions that exceed configured thresholds.
+ */
+export function evaluateRegressionThresholds(comparison, config) {
+    const regression = config.regression;
+    if (!regression)
+        return [];
+    const violations = [];
+    // Overall composite regression
+    if (comparison.deltas.overall < regression.composite) {
+        violations.push({
+            actual: comparison.deltas.overall,
+            area: "overall",
+            description: `Overall score dropped by ${fmt(Math.abs(comparison.deltas.overall))} (threshold: ${regression.composite})`,
+            metric: "regression-composite",
+            severity: classifyRegressionSeverity(comparison.deltas.overall, config),
+            threshold: regression.composite,
+        });
+    }
+    // Per-area regressions
+    for (const areaDelta of comparison.areas) {
+        if (areaDelta.delta < regression["per-area"]) {
+            violations.push({
+                actual: areaDelta.delta,
+                area: areaDelta.area,
+                description: `${areaDelta.area} dropped by ${fmt(Math.abs(areaDelta.delta))} (threshold: ${regression["per-area"]})`,
+                metric: "regression-area",
+                severity: classifyRegressionSeverity(areaDelta.delta, config),
+                threshold: regression["per-area"],
+            });
+        }
+        // Per-dimension regressions
+        const dims = areaDelta.dimensions;
+        for (const [dimKey, dimData] of Object.entries(dims)) {
+            if (dimData.delta < regression["per-dimension"]) {
+                violations.push({
+                    actual: dimData.delta,
+                    area: areaDelta.area,
+                    description: `${areaDelta.area} ${dimKey} dropped by ${fmt(Math.abs(dimData.delta))} (threshold: ${regression["per-dimension"]})`,
+                    metric: `regression-${dimKey}`,
+                    severity: "warning",
+                    threshold: regression["per-dimension"],
+                });
+            }
+        }
+    }
+    // Sort by severity then magnitude
+    violations.sort((a, b) => {
+        const sevDiff = SEVERITY_RANK[b.severity] - SEVERITY_RANK[a.severity];
+        if (sevDiff !== 0)
+            return sevDiff;
+        return a.actual - b.actual;
+    });
+    return violations;
+}
+// ---------------------------------------------------------------------------
+// Per-area evaluation
+// ---------------------------------------------------------------------------
+/**
+ * Evaluate a score summary against threshold configuration.
+ *
+ * Returns all violations sorted by severity (critical first), then by
+ * the magnitude of the threshold breach (largest gap first).
+ */
+export function evaluateThresholds(scores, config) {
+    const violations = [];
+    // Evaluate overall composite score against defaults
+    const overallScore = scores.overall.avgScore;
+    if (overallScore < config.defaults.composite) {
+        violations.push({
+            actual: overallScore,
+            area: "overall",
+            description: `Overall composite score ${fmt(overallScore)} is below threshold ${config.defaults.composite}`,
+            metric: "composite",
+            severity: classifySeverity(overallScore, config),
+            threshold: config.defaults.composite,
+        });
+    }
+    // Evaluate per-area scores
+    for (const areaScore of scores.scores) {
+        const areaOverrides = config.areas?.[areaScore.feature];
+        const merged = mergeDefaults(config.defaults, areaOverrides);
+        evaluateArea(areaScore, merged, config, violations);
+    }
+    // Sort: critical first, then by delta magnitude (largest breach first)
+    violations.sort((a, b) => {
+        const sevDiff = SEVERITY_RANK[b.severity] - SEVERITY_RANK[a.severity];
+        if (sevDiff !== 0)
+            return sevDiff;
+        // Larger breach = more negative (actual - threshold)
+        return a.actual - a.threshold - (b.actual - b.threshold);
+    });
+    const maxSeverity = violations.length > 0 ? violations[0].severity : "none";
+    return {
+        maxSeverity,
+        pass: violations.length === 0,
+        violations,
+    };
+}
+/**
+ * Classify severity for a regression delta using the severity config.
+ */
+function classifyRegressionSeverity(delta, config) {
+    const sev = config.severity;
+    if (!sev)
+        return "warning";
+    if (sev.warning?.["regression-exceeds"] !== undefined &&
+        delta < sev.warning["regression-exceeds"]) {
+        return "warning";
+    }
+    return "info";
+}
+// ---------------------------------------------------------------------------
+// Severity classification
+// ---------------------------------------------------------------------------
+/**
+ * Classify the severity of a score-based violation using the severity config.
+ * Checks from most severe to least: critical → warning → info.
+ */
+function classifySeverity(score, config) {
+    const sev = config.severity;
+    if (!sev)
+        return "warning";
+    if (sev.critical?.["composite-below"] !== undefined &&
+        score < sev.critical["composite-below"]) {
+        return "critical";
+    }
+    if (sev.warning?.["composite-below"] !== undefined &&
+        score < sev.warning["composite-below"]) {
+        return "warning";
+    }
+    if (sev.info?.["composite-below"] !== undefined &&
+        score < sev.info["composite-below"]) {
+        return "info";
+    }
+    return "warning";
+}
+function evaluateArea(score, thresholds, config, violations) {
+    const area = score.feature;
+    // Composite score
+    if (score.totalScore < thresholds.composite) {
+        violations.push({
+            actual: score.totalScore,
+            area,
+            description: `${area} composite score ${fmt(score.totalScore)} is below threshold ${thresholds.composite}`,
+            metric: "composite",
+            severity: classifySeverity(score.totalScore, config),
+            threshold: thresholds.composite,
+        });
+    }
+    // Per-dimension thresholds
+    const dims = thresholds.dimensions;
+    if (dims) {
+        const dimMap = [
+            ["task-completion", score.taskCompletion, dims["task-completion"]],
+            ["code-correctness", score.codeCorrectness, dims["code-correctness"]],
+            ["doc-coverage", score.docCoverage, dims["doc-coverage"]],
+        ];
+        for (const [dimKey, actual, threshold] of dimMap) {
+            if (threshold !== undefined && actual < threshold) {
+                violations.push({
+                    actual,
+                    area,
+                    description: `${area} ${dimKey} score ${fmt(actual)} is below threshold ${threshold}`,
+                    metric: dimKey,
+                    severity: classifySeverity(actual, config),
+                    threshold,
+                });
+            }
+        }
+    }
+    // Doc Lift threshold (docs must not hurt)
+    const docLiftThreshold = thresholds["doc-lift"];
+    if (docLiftThreshold !== undefined && score.docLift < docLiftThreshold) {
+        const severity = score.negativeDocLift && config.severity?.critical?.["negative-doc-lift"]
+            ? "critical"
+            : "warning";
+        violations.push({
+            actual: score.docLift,
+            area,
+            description: `${area} Doc Lift ${fmt(score.docLift)} is below threshold ${docLiftThreshold}${score.negativeDocLift ? " — docs are hurting performance" : ""}`,
+            metric: "doc-lift",
+            severity,
+            threshold: docLiftThreshold,
+        });
+    }
+    // Ceiling threshold (doc quality floor)
+    const ceilingThreshold = thresholds.ceiling;
+    if (ceilingThreshold !== undefined && score.ceilingScore < ceilingThreshold) {
+        violations.push({
+            actual: score.ceilingScore,
+            area,
+            description: `${area} ceiling score ${fmt(score.ceilingScore)} is below threshold ${ceilingThreshold} — documentation quality is insufficient`,
+            metric: "ceiling",
+            severity: "warning",
+            threshold: ceilingThreshold,
+        });
+    }
+}
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+/** Format a score for display (round to nearest integer) */
+function fmt(n) {
+    return String(Math.round(n));
+}
+/**
+ * Merge per-area overrides with defaults. Area overrides take precedence.
+ */
+function mergeDefaults(defaults, overrides) {
+    if (!overrides) {
+        return { ...defaults };
+    }
+    return {
+        ceiling: overrides.ceiling ?? defaults.ceiling,
+        composite: overrides.composite ?? defaults.composite,
+        dimensions: {
+            ...defaults.dimensions,
+            ...overrides.dimensions,
+        },
+        "doc-lift": overrides["doc-lift"] ?? defaults["doc-lift"],
+    };
+}

package/dist/pipeline/types.d.ts

@@ -0,0 +1,10 @@
+/**
+ * pipeline/types.ts — Re-export barrel
+ *
+ * All domain types now live in @sanity/ailf-core. This file re-exports
+ * them for backward compatibility — existing imports throughout
+ * packages/eval continue to work unchanged.
+ *
+ * @see packages/core/src/types/index.ts (canonical source)
+ */
+export * from "../_vendor/ailf-core/index.d.ts";

package/dist/pipeline/types.js

@@ -0,0 +1,10 @@
+/**
+ * pipeline/types.ts — Re-export barrel
+ *
+ * All domain types now live in @sanity/ailf-core. This file re-exports
+ * them for backward compatibility — existing imports throughout
+ * packages/eval continue to work unchanged.
+ *
+ * @see packages/core/src/types/index.ts (canonical source)
+ */
+export * from "../_vendor/ailf-core/index.js";

package/dist/pipeline/validate.d.ts

@@ -0,0 +1,67 @@
+/**
+ * pipeline/validate.ts
+ *
+ * Configuration validation for the evaluation pipeline. Checks that all YAML
+ * config files are consistent: every task has a canonical mapping, every
+ * mapping has a reference solution file, required files exist, etc.
+ *
+ * All individual validators are exported so they can be tested independently.
+ */
+import type { ValidationIssue, ValidationResult } from "./types.js";
+/**
+ * Run all validation checks and return a combined result.
+ * `valid` is true only if there are zero error-severity issues.
+ */
+export declare function validateConfiguration(rootDir: string): ValidationResult;
+/**
+ * Check that canonical context files exist. These are the per-task
+ * gold-retrieval contexts actually referenced by task definitions.
+ *
+ * Contexts are generated by fetch-docs and may not exist yet —
+ * returns warnings, not errors.
+ */
+export declare function validateContexts(rootDir: string): ValidationIssue[];
+/**
+ * Check that config/features.yaml exists, parses, and conforms to the Zod schema.
+ * Also cross-references covered features against actual task files for consistency.
+ *
+ * Returns warnings (not errors) if the file is missing — the feature registry
+ * is optional and doesn't block evaluation.
+ */
+export declare function validateFeaturesYaml(rootDir: string): ValidationIssue[];
+/**
+ * Check that config/models.yaml exists, parses, has at least one model with an id
+ * and label, and has a grader defined.
+ */
+export declare function validateModelsYaml(rootDir: string): ValidationIssue[];
+/**
+ * Check that reference solution files exist on disk for every task
+ * that declares a reference_solution path. Reads from inline task
+ * definitions via resolveMappings.
+ *
+ * NOTE: When tasks come from the Content Lake, reference solutions also
+ * live there (as ailf.referenceSolution documents). This validator only
+ * applies to YAML-based tasks with local file paths.
+ */
+export declare function validateReferenceSolutions(rootDir: string): ValidationIssue[];
+/**
+ * Check that config/rubrics.yaml exists, parses, and conforms to the Zod schema.
+ * Returns the set of valid template keys for cross-referencing by task
+ * validation.
+ */
+export declare function validateRubricsYaml(rootDir: string): ValidationIssue[];
+/**
+ * Check that tasks/*.yaml files exist, parse, and conform to the Zod schema.
+ * Validates both the new single-definition format (with `id`) and the legacy
+ * paired format. Uses `TaskFileSchema` from schemas.ts for structural
+ * validation, plus cross-entry checks (duplicate IDs, docs path consistency).
+ */
+export declare function validateTaskFiles(rootDir: string): ValidationIssue[];
+/**
+ * Check that config/thresholds.yaml exists, parses, and conforms to the Zod schema.
+ *
+ * Returns warnings (not errors) if the file is missing — thresholds are
+ * optional and don't block evaluation. They only activate when
+ * `--readiness` or severity-aware sink routing is used.
+ */
+export declare function validateThresholdsYaml(rootDir: string): ValidationIssue[];