@sanity/ailf 0.1.0

package/dist/pipeline/gap-analysis.js

@@ -0,0 +1,231 @@
+/**
+ * pipeline/gap-analysis.ts
+ *
+ * Expected impact estimation for identified gaps.
+ *
+ * Phase 3b of the Scenario Matrix implementation.
+ *
+ * Given failure modes and scores, estimates the score lift that fixing
+ * each gap would produce. Gaps are prioritized by estimated lift × task count
+ * to produce an actionable remediation plan.
+ *
+ * The estimation model is conservative: it assumes fixing a gap raises the
+ * bottleneck dimension to the median of non-bottlenecked dimensions (not 100).
+ * This produces realistic estimates rather than theoretical maximums.
+ *
+ * @see docs/exec-plans/completed/scenario-matrix-implementation/phase-3-gap-analysis.md
+ */
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+/** Default dimension weights (must match rubrics.yaml) */
+const DEFAULT_WEIGHTS = {
+    "code-correctness": 0.25,
+    "doc-coverage": 0.25,
+    "task-completion": 0.5,
+};
+/** Map failure modes to the dimensions they typically bottleneck */
+const MODE_BOTTLENECKS = {
+    "incorrect-docs": ["code-correctness", "task-completion"],
+    "missing-docs": ["task-completion", "doc-coverage"],
+    "model-limitation": [], // Not a docs problem
+    "outdated-docs": ["code-correctness", "doc-coverage"],
+    "poor-structure": ["doc-coverage", "task-completion"],
+    unclassified: [],
+};
+/** Remediation descriptions by failure mode */
+const REMEDIATION_MAP = {
+    "incorrect-docs": "Fix factual errors in existing documentation",
+    "missing-docs": "Write new documentation for uncovered functionality",
+    "model-limitation": "Not a documentation problem — track for model improvement",
+    "outdated-docs": "Update documentation to reflect current API/patterns",
+    "poor-structure": "Restructure documentation for clarity and discoverability",
+    unclassified: "Flag for manual review — cause unclear",
+};
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+/**
+ * Build a complete gap analysis report.
+ *
+ * @param failureModeReport - Classified failure modes from Phase 3a
+ * @param scores - Per-area feature scores
+ * @param weights - Dimension weights
+ * @returns Gap analysis report with prioritized remediation plan
+ */
+export function buildGapAnalysisReport(failureModeReport, scores, weights) {
+    const gaps = estimateImpact(failureModeReport, scores, weights);
+    const totalPotentialLift = gaps.reduce((sum, g) => sum + g.estimatedLift, 0);
+    return {
+        gaps,
+        generatedAt: new Date().toISOString(),
+        totalPotentialLift: Math.round(totalPotentialLift * 10) / 10,
+    };
+}
+/**
+ * Estimate the impact of fixing each identified gap.
+ *
+ * For each area with failure modes, calculates the potential score lift
+ * from fixing the identified issues. Uses a conservative estimation model
+ * that targets the median of non-bottlenecked dimensions rather than 100.
+ *
+ * @param failureModeReport - Classified failure modes from Phase 3a
+ * @param scores - Per-area feature scores
+ * @param weights - Dimension weights (defaults to rubrics.yaml weights)
+ * @returns Gap estimates sorted by priority (highest first)
+ */
+export function estimateImpact(failureModeReport, scores, weights = DEFAULT_WEIGHTS) {
+    const gaps = [];
+    const scoreByArea = new Map();
+    for (const score of scores) {
+        scoreByArea.set(score.feature, score);
+    }
+    // Group failure modes by area + mode combination
+    for (const [area, areaData] of Object.entries(failureModeReport.byArea)) {
+        const areaScore = scoreByArea.get(area);
+        if (!areaScore)
+            continue;
+        // Get the tasks affected by this area
+        const areaTasks = failureModeReport.classifiedJudgments
+            .filter((cj) => cj.judgment.taskId.startsWith(area) &&
+            cj.classification.mode !== "unclassified" &&
+            cj.classification.mode !== "model-limitation")
+            .map((cj) => cj.judgment.taskId);
+        const uniqueTasks = [...new Set(areaTasks)];
+        // For each non-trivial failure mode in this area
+        for (const [modeStr, count] of Object.entries(areaData.modes)) {
+            const mode = modeStr;
+            if (count === 0)
+                continue;
+            if (mode === "unclassified" || mode === "model-limitation")
+                continue;
+            const bottleneckDimensions = MODE_BOTTLENECKS[mode];
+            if (bottleneckDimensions.length === 0)
+                continue;
+            const dimensionScores = getDimensionScores(areaScore);
+            const currentDimensionScores = {};
+            for (const dim of bottleneckDimensions) {
+                currentDimensionScores[dim] = dimensionScores[dim] ?? 0;
+            }
+            // Conservative estimation: target median of non-bottlenecked dimensions
+            const nonBottleneckScores = Object.entries(dimensionScores)
+                .filter(([dim]) => !bottleneckDimensions.includes(dim))
+                .map(([, score]) => score);
+            const targetScore = nonBottleneckScores.length > 0 ? median(nonBottleneckScores) : 70; // Fallback target
+            // Estimate composite lift
+            const estimatedLift = estimateCompositeLift(dimensionScores, bottleneckDimensions, targetScore, weights);
+            // Determine confidence based on failure mode confidence distribution
+            const modeJudgments = failureModeReport.classifiedJudgments.filter((cj) => cj.judgment.taskId.startsWith(area) && cj.classification.mode === mode);
+            const highConfCount = modeJudgments.filter((cj) => cj.classification.confidence === "high").length;
+            const confidence = highConfCount > modeJudgments.length / 2
+                ? "high"
+                : highConfCount > 0
+                    ? "medium"
+                    : "low";
+            gaps.push({
+                affectedTaskIds: uniqueTasks.length > 0 ? uniqueTasks : [area],
+                area,
+                bottleneckDimensions,
+                confidence,
+                currentDimensionScores,
+                estimatedLift: Math.round(estimatedLift * 10) / 10,
+                failureMode: mode,
+                priority: Math.round(estimatedLift * Math.max(uniqueTasks.length, 1) * 10) / 10,
+                remediation: REMEDIATION_MAP[mode],
+            });
+        }
+    }
+    // Sort by priority (highest first)
+    return gaps.sort((a, b) => b.priority - a.priority);
+}
+// ---------------------------------------------------------------------------
+// Formatting
+// ---------------------------------------------------------------------------
+/**
+ * Format a gap analysis report for console output.
+ */
+export function formatGapAnalysisConsole(report) {
+    const lines = [];
+    lines.push("📋 PRIORITIZED REMEDIATION PLAN");
+    lines.push("");
+    if (report.gaps.length === 0) {
+        lines.push("  No actionable gaps identified.");
+        lines.push("");
+        return lines.join("\n");
+    }
+    lines.push(`  Total potential lift: +${report.totalPotentialLift.toFixed(1)} points`);
+    lines.push("");
+    // Priority table
+    lines.push("  Priority  Area              Failure Mode     Est. Lift  Tasks  Action");
+    lines.push("  ────────  ────────────────  ───────────────  ─────────  ─────  ──────────────────────────────");
+    for (let i = 0; i < report.gaps.length; i++) {
+        const gap = report.gaps[i];
+        const liftStr = `+${gap.estimatedLift.toFixed(1)}`;
+        lines.push(`  #${(i + 1).toString().padEnd(6)}  ${gap.area.padEnd(16)}  ${gap.failureMode.padEnd(15)}  ${liftStr.padStart(9)}  ${gap.affectedTaskIds.length.toString().padStart(5)}  ${gap.remediation}`);
+    }
+    lines.push("");
+    return lines.join("\n");
+}
+/**
+ * Format a gap analysis report as markdown for PR comments.
+ */
+export function formatGapAnalysisMarkdown(report) {
+    const lines = [];
+    lines.push("### 📋 Prioritized Remediation Plan");
+    lines.push("");
+    if (report.gaps.length === 0) {
+        lines.push("No actionable gaps identified.");
+        lines.push("");
+        return lines.join("\n");
+    }
+    lines.push(`**Total potential lift: +${report.totalPotentialLift.toFixed(1)} points**`);
+    lines.push("");
+    lines.push("| Priority | Area | Failure Mode | Est. Lift | Tasks | Action |");
+    lines.push("|----------|------|--------------|-----------|-------|--------|");
+    for (let i = 0; i < report.gaps.length; i++) {
+        const gap = report.gaps[i];
+        const confIcon = gap.confidence === "high"
+            ? "🟢"
+            : gap.confidence === "medium"
+                ? "🟡"
+                : "🔴";
+        lines.push(`| #${i + 1} | ${gap.area} | ${confIcon} ${gap.failureMode} | +${gap.estimatedLift.toFixed(1)} | ${gap.affectedTaskIds.length} | ${gap.remediation} |`);
+    }
+    lines.push("");
+    return lines.join("\n");
+}
+// ---------------------------------------------------------------------------
+// Internal helpers
+// ---------------------------------------------------------------------------
+/**
+ * Estimate the composite score lift from raising bottleneck dimensions
+ * to the target score.
+ */
+function estimateCompositeLift(dimensionScores, bottleneckDimensions, targetScore, weights) {
+    let lift = 0;
+    for (const dim of bottleneckDimensions) {
+        const current = dimensionScores[dim] ?? 0;
+        const headroom = Math.max(0, targetScore - current);
+        const weight = weights[dim] ?? 0;
+        lift += headroom * weight;
+    }
+    return lift;
+}
+/** Extract dimension scores from a FeatureScore */
+function getDimensionScores(score) {
+    return {
+        "code-correctness": score.codeCorrectness,
+        "doc-coverage": score.docCoverage,
+        "task-completion": score.taskCompletion,
+    };
+}
+/** Calculate the median of an array of numbers */
+function median(values) {
+    if (values.length === 0)
+        return 0;
+    const sorted = [...values].sort((a, b) => a - b);
+    const mid = Math.floor(sorted.length / 2);
+    return sorted.length % 2 !== 0
+        ? sorted[mid]
+        : (sorted[mid - 1] + sorted[mid]) / 2;
+}

package/dist/pipeline/generate-configs.d.ts

@@ -0,0 +1,72 @@
+/**
+ * pipeline/generate-configs.ts
+ *
+ * Reads config/models.yaml (the central model registry) and generates all
+ * promptfoo config files with the correct provider entries.
+ *
+ * This keeps model definitions in one place — add a model to config/models.yaml
+ * and run `pnpm generate-configs` to propagate it to all eval modes.
+ *
+ * Generated configs:
+ *   - promptfooconfig.yaml           (baseline: with-docs vs without-docs)
+ *   - promptfooconfig.observed.yaml  (instrumented HTTP recording)
+ *   - promptfooconfig.agentic.yaml   (agentic tool-calling: naive vs optimized)
+ *
+ * All functions accept rootDir as a parameter — no module-level constants.
+ * No process.argv parsing. No env var fallbacks. Callers provide typed options.
+ *
+ * @see config/models.yaml — the central model registry
+ * @see docs/exec-plans/active/eliminate-lib-layer.md
+ */
+import { type TaskDefinition } from "../_vendor/ailf-core/index.d.ts";
+import type { FilterOptions } from "./types.js";
+import { type ResolvedSourceConfig } from "../sources.js";
+export { extractModelName, extractProvider, mergeConfig, modelMatchesMode, } from "../_vendor/ailf-core/index.d.ts";
+/** Auto-discover all task YAML files in the tasks/ directory. */
+export declare function discoverTaskFiles(rootDir: string): string[];
+interface LoadedPrompts {
+    agentic: {
+        id: string;
+        label: string;
+        raw: string;
+    };
+    withDocs: {
+        id: string;
+        label: string;
+        raw: string;
+    };
+    withoutDocs: {
+        id: string;
+        label: string;
+        raw: string;
+    };
+}
+/** Load prompt templates from config/prompts.yaml. Throws if missing or malformed. */
+export declare function loadPrompts(rootDir: string): LoadedPrompts;
+/** Options for the generateConfigs function. */
+export interface GenerateConfigsOptions {
+    /** Allowed origins for agentic mode (controls source isolation assertion) */
+    allowedOrigins?: string[];
+    /** Filter to specific feature areas or task IDs */
+    filter?: FilterOptions;
+    /** Pre-resolved source config (skips loadSource() call) */
+    resolvedSource?: ResolvedSourceConfig;
+    /** Root directory of the eval package (required) */
+    rootDir: string;
+    /** Search mode for agentic mode */
+    searchMode?: string;
+    /** Documentation source name (e.g., "branch", "local") */
+    source?: string;
+    /** Pre-loaded task definitions from a TaskSource adapter.
+     *  When provided, expandTaskDefinitions() is used instead of
+     *  loadAndExpandTasks() (which reads from tasks/*.yaml files). */
+    tasks?: TaskDefinition[];
+}
+/**
+ * Generate Promptfoo config files from models.yaml + task definitions.
+ *
+ * All parameters are passed via the typed options object — no process.argv
+ * parsing or env var fallbacks. Callers (command handlers, orchestration
+ * steps) are responsible for resolving options from their own context.
+ */
+export declare function generateConfigs(options: GenerateConfigsOptions): void;