@sanity/ailf 0.5.0 → 1.0.0

package/dist/pipeline/cache.d.ts

@@ -10,7 +10,7 @@
  *
  * Cache invalidation triggers:
  *   - Content change: any input file's content changes → hash changes → miss
- *   - Config change: config/models.yaml, config/sources.yaml, tasks/*.yaml changes → miss
+ *   - Config change: config/models, config/sources, tasks/*.yaml changes → miss
  *   - Manual bypass: --no-cache flag skips all cache lookups
  *   - Cache clear: delete results/cache/ to start fresh
  */

package/dist/pipeline/cache.js

@@ -10,7 +10,7 @@
  *
  * Cache invalidation triggers:
  *   - Content change: any input file's content changes → hash changes → miss
- *   - Config change: config/models.yaml, config/sources.yaml, tasks/*.yaml changes → miss
+ *   - Config change: config/models, config/sources, tasks/*.yaml changes → miss
  *   - Manual bypass: --no-cache flag skips all cache lookups
  *   - Cache clear: delete results/cache/ to start fresh
  */
@@ -18,6 +18,19 @@ import { createHash } from "crypto";
 import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync, } from "fs";
 import { join, resolve } from "path";
 // ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+/** Resolve first existing config file (matches loadConfigFile priority chain) */
+function resolveConfig(rootDir, name) {
+    const r = (f) => resolve(rootDir, f);
+    for (const ext of [".ts", ".js", ".yaml", ".yml", ".json"]) {
+        const p = r(`config/${name}${ext}`);
+        if (existsSync(p))
+            return p;
+    }
+    return undefined;
+}
+// ---------------------------------------------------------------------------
 // Constants
 // ---------------------------------------------------------------------------
 const CACHE_DIR_NAME = "cache";
@@ -79,7 +92,10 @@ export function getStepInputPaths(rootDir, step) {
             const isBaseline = step === "eval-baseline" || step === "eval";
             const isAgentic = step === "eval-agentic" || step === "eval";
             const isObserved = step === "eval-observed" || step === "eval";
-            const paths = [r("config/models.yaml")];
+            const paths = [];
+            const modelsPath = resolveConfig(rootDir, "models");
+            if (modelsPath)
+                paths.push(modelsPath);
             // Config files — only the relevant ones for this mode
             if (isBaseline) {
                 paths.push(r("promptfooconfig.yaml"));
@@ -130,25 +146,37 @@ export function getStepInputPaths(rootDir, step) {
             return paths;
         }
         case "fetch-docs": {
-            // Inputs: config/sources.yaml, config/models.yaml, task files (which contain inline mappings)
-            const paths = [r("config/sources.yaml"), r("config/models.yaml")];
+            // Inputs: config sources + models, task files
+            const paths = [];
+            const sourcesPath = resolveConfig(rootDir, "sources");
+            const modelsPath2 = resolveConfig(rootDir, "models");
+            if (sourcesPath)
+                paths.push(sourcesPath);
+            if (modelsPath2)
+                paths.push(modelsPath2);
             // Include all task files (they define feature areas)
             const tasksDir = r("tasks");
             if (existsSync(tasksDir)) {
                 const taskFiles = readdirSync(tasksDir)
-                    .filter((f) => f.endsWith(".yaml") || f.endsWith(".yml"))
+                    .filter((f) => /\.(yaml|yml|task\.ts|task\.js)$/.test(f))
                     .map((f) => join(tasksDir, f));
                 paths.push(...taskFiles);
             }
             return paths;
         }
         case "generate-configs": {
-            // Inputs: config/models.yaml, config/sources.yaml, all task files
-            const paths = [r("config/models.yaml"), r("config/sources.yaml")];
+            // Inputs: config models + sources, all task files
+            const paths = [];
+            const modelsPath3 = resolveConfig(rootDir, "models");
+            const sourcesPath2 = resolveConfig(rootDir, "sources");
+            if (modelsPath3)
+                paths.push(modelsPath3);
+            if (sourcesPath2)
+                paths.push(sourcesPath2);
             const tasksDir = r("tasks");
             if (existsSync(tasksDir)) {
                 const taskFiles = readdirSync(tasksDir)
-                    .filter((f) => f.endsWith(".yaml") || f.endsWith(".yml"))
+                    .filter((f) => /\.(yaml|yml|task\.ts|task\.js)$/.test(f))
                     .map((f) => join(tasksDir, f));
                 paths.push(...taskFiles);
             }

package/dist/pipeline/calculate-scores.d.ts

@@ -1,9 +1,7 @@
-import type { Logger, TestSummary } from "../_vendor/ailf-core/index.d.ts";
+import { type ActualScoreEntry, type ComponentResult, type Logger, type TestSummary } from "../_vendor/ailf-core/index.d.ts";
 import { type ResolvedSourceConfig } from "../sources.js";
-import { type ActualScoreEntry, type ComponentResult } from "../_vendor/ailf-core/index.d.ts";
 import type { GraderJudgment, PerModelEntry } from "./types.js";
-export { classifyRubric, detectFeatureArea, extractUrlMetadata, mergeScores, parseRubricScore, } from "../_vendor/ailf-core/index.d.ts";
-export type { ActualScoreEntry, ComponentResult, TestResult, UrlMetadata, } from "../_vendor/ailf-core/index.d.ts";
+export { classifyRubric, detectFeatureArea, extractUrlMetadata, mergeScores, parseRubricScore, type ActualScoreEntry, type ComponentResult, type TestResult, type UrlMetadata, } from "../_vendor/ailf-core/index.d.ts";
 export interface PromptfooResultsWrapper {
     results: RawTestResult[];
     stats: {

package/dist/pipeline/calculate-scores.js

@@ -9,7 +9,7 @@
  *   Doc Coverage       (0–100)  — Did docs provide the needed info?
  *
  * Dimensions are combined into a weighted composite (0–100) using named
- * scoring profiles from config/rubrics.yaml. Gold (with-docs) entries use
+ * scoring profiles from config/rubrics. Gold (with-docs) entries use
  * the "default" profile; baseline (without-docs) entries use "output-only"
  * which excludes doc-coverage (undefined without docs).
  * See docs/design-docs/named-scoring-profiles.md.
@@ -29,15 +29,17 @@
  */
 import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
 import { join } from "path";
+import { classifyRubric, detectFeatureArea, extractUrlMetadata, mergeScores, parseRubricScore, } from "../_vendor/ailf-core/index.js";
 import { calculateCost } from "../agent-observer/pricing.js";
 import { ConsoleLogger } from "../adapters/loggers/index.js";
+import { analyzeSourceIsolation, } from "../assertions/source-isolation.js";
 import { checkResultsExist } from "./checks.js";
-import { loadRubricTemplates } from "./expand-tasks.js";
+import { loadRubricTemplates } from "./rubric-loader.js";
 import { resolveProfile } from "./profile-resolution.js";
 import { loadSource } from "../sources.js";
-import { analyzeSourceIsolation, } from "../assertions/source-isolation.js";
-import { classifyRubric, detectFeatureArea, extractUrlMetadata, mergeScores, parseRubricScore, } from "../_vendor/ailf-core/index.js";
-// Re-export pure functions from core for backward compatibility.
+import { LiteracyVariant } from "./normalize-mode.js";
+import { scoreTestGroup } from "./compiler/scoring-bridge.js";
+// Re-export from core for backward compatibility.
 // Existing imports from this file continue to work unchanged.
 export { classifyRubric, detectFeatureArea, extractUrlMetadata, mergeScores, parseRubricScore, } from "../_vendor/ailf-core/index.js";
 /**
@@ -137,14 +139,8 @@ export function extractGraderJudgments(resultsPath) {
                     // Not JSON — use raw reason string
                 }
             }
-            // Map internal dimension names to hyphenated form
-            const dimensionMap = {
-                codeCorrectness: "code-correctness",
-                docCoverage: "doc-coverage",
-                taskCompletion: "task-completion",
-            };
             judgments.push({
-                dimension: dimensionMap[kind] ?? kind,
+                dimension: kind,
                 modelId,
                 reason,
                 score,
@@ -281,7 +277,7 @@ function aggregateUrlReferences(resultsPath) {
  * verification report.
  */
 function buildSourceVerification(root, source, verificationCtx) {
-    const mode = verificationCtx?.mode ?? "baseline";
+    const mode = verificationCtx?.mode ?? LiteracyVariant.STANDARD;
     const sourceUrl = source?.baseUrl ?? "default";
     const searchMode = verificationCtx?.searchMode;
     const allowedOrigins = verificationCtx?.allowedOrigins;
@@ -493,62 +489,6 @@ function readAndNormalizeResults(resultsPath, log) {
     }
     return valid;
 }
-/**
- * Accumulate raw dimension scores across an array of test results.
- * Dimension-agnostic: any dimension returned by classifyRubric() is tracked.
- */
-function accumulateDimensions(tests) {
-    const dimensions = {};
-    let totalCost = 0;
-    for (const test of tests) {
-        totalCost += test.cost;
-        for (const comp of test.gradingResult.componentResults) {
-            if (comp.assertion?.type !== "llm-rubric")
-                continue;
-            const score = parseRubricScore(comp);
-            const kind = classifyRubric(comp);
-            if (kind) {
-                dimensions[kind] = (dimensions[kind] ?? 0) + score;
-            }
-        }
-    }
-    return { dimensions, totalCost };
-}
-/**
- * Average accumulated dimension scores by a count.
- * Returns a dimension → average score map.
- */
-function averageDimensions(accumulated, count) {
-    const avg = {};
-    for (const [dim, total] of Object.entries(accumulated.dimensions)) {
-        avg[dim] = total / count;
-    }
-    return avg;
-}
-/**
- * Compute a weighted composite score from dimension averages and a profile.
- * Only dimensions present in the profile contribute to the composite.
- * Dimensions not in the profile are ignored (e.g., doc-coverage on baseline).
- *
- * The profile maps camelCase dimension names (as returned by classifyRubric)
- * to kebab-case keys (as used in rubrics.yaml). This function handles the
- * mapping internally.
- */
-function weightedComposite(dimensionAverages, profile) {
-    // Map profile keys (kebab-case: "task-completion") to classifyRubric
-    // output (camelCase: "taskCompletion")
-    const kebabToCamel = {
-        "code-correctness": "codeCorrectness",
-        "doc-coverage": "docCoverage",
-        "task-completion": "taskCompletion",
-    };
-    let total = 0;
-    for (const [profileKey, weight] of Object.entries(profile)) {
-        const dimKey = kebabToCamel[profileKey] ?? profileKey;
-        total += (dimensionAverages[dimKey] ?? 0) * weight;
-    }
-    return total;
-}
 /**
  * Core scoring logic: takes a pre-filtered array of TestResult and produces
  * FeatureScore[] grouped by feature area. This is the shared implementation
@@ -577,35 +517,28 @@ function scoreResults(results, goldProfile, baselineProfile, modelId) {
     }
     const scores = [];
     for (const [feature, data] of Object.entries(byFeature)) {
-        // --- With docs (gold / ceiling) ---
-        const goldDims = accumulateDimensions(data.withDocs);
-        let featureCost = goldDims.totalCost;
-        const countWithDocs = data.withDocs.length || 1;
-        const avgGold = averageDimensions(goldDims, countWithDocs);
-        const withDocsTotal = weightedComposite(avgGold, goldProfile);
+        // --- With docs (gold / ceiling) — scored via 4-tier engine ---
+        const gold = scoreTestGroup(data.withDocs, goldProfile, feature);
         // --- Without docs (baseline / floor) ---
         // Uses the baseline profile (e.g. "output-only") which may exclude
         // dimensions like doc-coverage that are undefined without docs.
         // See docs/design-docs/named-scoring-profiles.md.
-        const baselineDims = accumulateDimensions(data.withoutDocs);
-        featureCost += baselineDims.totalCost;
-        const countWithoutDocs = data.withoutDocs.length || 1;
-        const avgBaseline = averageDimensions(baselineDims, countWithoutDocs);
-        const withoutDocsScore = weightedComposite(avgBaseline, baselineProfile);
-        const ceilingScore = Math.round(withDocsTotal);
-        const floorScore = Math.round(withoutDocsScore);
+        const baseline = scoreTestGroup(data.withoutDocs, baselineProfile, feature);
+        const featureCost = gold.totalCost + baseline.totalCost;
+        const ceilingScore = gold.composite;
+        const floorScore = baseline.composite;
         const docLift = ceilingScore - floorScore;
         const featureScore = {
             ceilingScore,
-            codeCorrectness: Math.round(avgGold.codeCorrectness ?? 0),
-            docCoverage: Math.round(avgGold.docCoverage ?? 0),
+            codeCorrectness: gold.dimensions.codeCorrectness ?? 0,
+            docCoverage: gold.dimensions.docCoverage ?? 0,
             docLift,
             docQualityGap: 100 - ceilingScore,
             feature,
             floorScore,
             ...(modelId && { modelId }),
             negativeDocLift: docLift < 0,
-            taskCompletion: Math.round(avgGold.taskCompletion ?? 0),
+            taskCompletion: gold.dimensions.taskCompletion ?? 0,
             testCount: data.withDocs.length,
             totalCost: featureCost,
             totalScore: ceilingScore,
@@ -638,17 +571,14 @@ export function scoreAgenticResults(resultsPath, profile) {
     }
     const entries = {};
     for (const [feature, featureResults] of Object.entries(byFeature)) {
-        const count = featureResults.length || 1;
-        const accumulated = accumulateDimensions(featureResults);
-        const avg = averageDimensions(accumulated, count);
-        const actualScore = Math.round(weightedComposite(avg, profile));
+        const scored = scoreTestGroup(featureResults, profile, feature);
         entries[feature] = {
-            actualScore,
-            codeCorrectness: Math.round(avg.codeCorrectness ?? 0),
-            docCoverage: Math.round(avg.docCoverage ?? 0),
-            taskCompletion: Math.round(avg.taskCompletion ?? 0),
+            actualScore: scored.composite,
+            codeCorrectness: scored.dimensions.codeCorrectness ?? 0,
+            docCoverage: scored.dimensions.docCoverage ?? 0,
+            taskCompletion: scored.dimensions.taskCompletion ?? 0,
             testCount: featureResults.length,
-            totalCost: accumulated.totalCost,
+            totalCost: scored.totalCost,
         };
     }
     return entries;
@@ -677,17 +607,14 @@ export function scoreAgenticResultsPerModel(resultsPath, profile) {
     for (const [modelId, features] of Object.entries(byModel)) {
         perModel[modelId] = {};
         for (const [feature, featureResults] of Object.entries(features)) {
-            const count = featureResults.length || 1;
-            const accumulated = accumulateDimensions(featureResults);
-            const avg = averageDimensions(accumulated, count);
-            const actualScore = Math.round(weightedComposite(avg, profile));
+            const scored = scoreTestGroup(featureResults, profile, feature);
             perModel[modelId][feature] = {
-                actualScore,
-                codeCorrectness: Math.round(avg.codeCorrectness ?? 0),
-                docCoverage: Math.round(avg.docCoverage ?? 0),
-                taskCompletion: Math.round(avg.taskCompletion ?? 0),
+                actualScore: scored.composite,
+                codeCorrectness: scored.dimensions.codeCorrectness ?? 0,
+                docCoverage: scored.dimensions.docCoverage ?? 0,
+                taskCompletion: scored.dimensions.taskCompletion ?? 0,
                 testCount: featureResults.length,
-                totalCost: accumulated.totalCost,
+                totalCost: scored.totalCost,
             };
         }
     }
@@ -725,7 +652,7 @@ export function calculateAndWriteScores(options) {
         }
     }
     // Determine mode — controls which result files are read
-    const mode = options.mode ?? "baseline";
+    const mode = options.mode ?? LiteracyVariant.STANDARD;
     const baselineResultsPath = options.resultsPath ?? join(ROOT, "results", "latest", "eval-results.json");
     // Agentic results path (only used in full mode)
     const agenticResultsPath = join(ROOT, "results", "latest", "eval-results-agentic.json");
@@ -747,8 +674,8 @@ export function calculateAndWriteScores(options) {
     // Baseline (without-docs) entries use "output-only" (2 dimensions,
     // doc-coverage excluded). See docs/design-docs/named-scoring-profiles.md.
     const rubricConfig = loadRubricTemplates(ROOT);
-    const goldProfile = resolveProfile("baseline", "gold", rubricConfig);
-    const baselineProfileWeights = resolveProfile("baseline", "baseline", rubricConfig);
+    const goldProfile = resolveProfile("literacy", "gold", rubricConfig, LiteracyVariant.STANDARD);
+    const baselineProfileWeights = resolveProfile("literacy", LiteracyVariant.STANDARD, rubricConfig, LiteracyVariant.STANDARD);
     log.debug("Loaded scoring profiles", {
         gold: goldProfile,
         baseline: baselineProfileWeights,
@@ -776,9 +703,9 @@ export function calculateAndWriteScores(options) {
     let agentBehavior = null;
     let sourceIsolation = null;
     let evaluationMode;
-    if (mode === "full" && existsSync(agenticResultsPath)) {
+    if (mode === LiteracyVariant.FULL && existsSync(agenticResultsPath)) {
         log.info(`\nReading agentic results from: ${agenticResultsPath}`);
-        const agenticProfile = resolveProfile("agentic", "gold", rubricConfig);
+        const agenticProfile = resolveProfile("literacy", "gold", rubricConfig, LiteracyVariant.AGENTIC);
         const agenticScores = scoreAgenticResults(agenticResultsPath, agenticProfile);
         log.debug("Agentic scores calculated", {
             featureCount: Object.keys(agenticScores).length,
@@ -789,7 +716,7 @@ export function calculateAndWriteScores(options) {
             })),
         });
         scores = mergeScores(baselineScores, agenticScores);
-        evaluationMode = "full";
+        evaluationMode = LiteracyVariant.FULL;
         // Merge agentic actual scores into the per-model breakdown
         if (perModel) {
             const agenticPerModel = scoreAgenticResultsPerModel(agenticResultsPath, agenticProfile);
@@ -812,17 +739,20 @@ export function calculateAndWriteScores(options) {
             graderCost.completionTokens += agenticGraderCost.completionTokens;
         }
     }
-    else if (mode === "agentic") {
+    else if (mode === LiteracyVariant.AGENTIC) {
         scores = baselineScores;
         agentBehavior = aggregateAgentBehavior(baselineResultsPath);
         sourceIsolation = aggregateSourceIsolation(baselineResultsPath, options?.allowedOrigins);
-        evaluationMode = "agentic";
+        evaluationMode = LiteracyVariant.AGENTIC;
     }
     else {
         scores = baselineScores;
         agentBehavior = aggregateAgentBehavior(baselineResultsPath);
         sourceIsolation = aggregateSourceIsolation(baselineResultsPath, options?.allowedOrigins);
-        evaluationMode = mode === "observed" ? "observed" : "baseline";
+        evaluationMode =
+            mode === LiteracyVariant.OBSERVED
+                ? LiteracyVariant.OBSERVED
+                : LiteracyVariant.STANDARD;
     }
     const summary = printReport(scores, urlRefs, source, agentBehavior, graderCost, perModel, sourceIsolation, sourceVerification, evaluationMode, log);
     // Persist
@@ -833,7 +763,7 @@ export function calculateAndWriteScores(options) {
     // Extract and persist grader judgments (Phase 3a: failure mode extraction)
     const judgments = extractGraderJudgments(baselineResultsPath);
     // In full mode, also extract judgments from agentic results
-    if (mode === "full" && existsSync(agenticResultsPath)) {
+    if (mode === LiteracyVariant.FULL && existsSync(agenticResultsPath)) {
         const agenticJudgments = extractGraderJudgments(agenticResultsPath);
         judgments.push(...agenticJudgments);
     }

package/dist/pipeline/checks.js

@@ -117,7 +117,7 @@ export function checkGeneratedConfigsExist(rootDir) {
     const baselinePath = resolve(rootDir, "promptfooconfig.yaml");
     if (!existsSync(baselinePath)) {
         issues.push({
-            message: "Baseline config 'promptfooconfig.yaml' not found. Run 'pnpm generate-configs'.",
+            message: "Baseline config 'promptfooconfig.yaml' not found. Run the pipeline to generate it.",
             path: baselinePath,
             severity: "error",
             source: "checkGeneratedConfigsExist",
@@ -131,7 +131,7 @@ export function checkGeneratedConfigsExist(rootDir) {
         const configPath = resolve(rootDir, name);
         if (!existsSync(configPath)) {
             issues.push({
-                message: `Optional config \`${name}\` not found. Run \`pnpm generate-configs\` to create it.`,
+                message: `Optional config \`${name}\` not found. Run the pipeline to generate it.`,
                 path: configPath,
                 severity: "warning",
                 source: "checkGeneratedConfigsExist",

package/dist/pipeline/compare.js

@@ -79,14 +79,14 @@ export function compare(baseline, experiment, options) {
     // Per-dimension average deltas (only for areas present in both summaries)
     const commonAreas = areas.filter((a) => baselineAreas.has(a.area) && experimentAreas.has(a.area));
     const commonCount = commonAreas.length || 1;
-    const perDimension = {
-        codeCorrectness: commonAreas.reduce((s, a) => s + a.dimensions.codeCorrectness.delta, 0) /
-            commonCount,
-        docCoverage: commonAreas.reduce((s, a) => s + a.dimensions.docCoverage.delta, 0) /
-            commonCount,
-        taskCompletion: commonAreas.reduce((s, a) => s + a.dimensions.taskCompletion.delta, 0) /
-            commonCount,
-    };
+    // Collect all dimension keys from area deltas and average each
+    const allDimKeys = new Set(commonAreas.flatMap((a) => Object.keys(a.dimensions)));
+    const perDimension = {};
+    for (const dim of allDimKeys) {
+        perDimension[dim] =
+            commonAreas.reduce((s, a) => s + (a.dimensions[dim]?.delta ?? 0), 0) /
+                commonCount;
+    }
     // Doc Lift average delta (common areas only)
     const docLift = commonAreas.reduce((s, a) => s + a.docLiftDelta, 0) / commonCount;
     // Cost delta (if both summaries have cost data)

package/dist/pipeline/compiler/__tests__/agent-harness-handler.test.d.ts

@@ -0,0 +1,10 @@
+/**
+ * agent-harness-handler.test.ts — Tests for agent harness mode compilation.
+ *
+ * Tests validation, provider assembly, tool permission resolution,
+ * assertion mapping, sandbox config, lifecycle extensions, and
+ * end-to-end compilation of example tasks.
+ *
+ * Run: npx tsx --test src/pipeline/compiler/__tests__/agent-harness-handler.test.ts
+ */
+export {};