@sanity/ailf 0.1.0

package/dist/pipeline/cache.d.ts

@@ -0,0 +1,101 @@
+/**
+ * pipeline/cache.ts
+ *
+ * Content-aware caching for pipeline steps. Each step's cache key is derived
+ * from a hash of its inputs (config files, context files, task files, etc.).
+ * When inputs haven't changed between runs, cached outputs are reused.
+ *
+ * Cache storage: `results/cache/` directory with one JSON manifest per step.
+ * Each manifest maps a content hash to the step's output metadata.
+ *
+ * Cache invalidation triggers:
+ *   - Content change: any input file's content changes → hash changes → miss
+ *   - Config change: config/models.yaml, config/sources.yaml, tasks/*.yaml changes → miss
+ *   - Manual bypass: --no-cache flag skips all cache lookups
+ *   - Cache clear: delete results/cache/ to start fresh
+ */
+/** A single cache entry — stored per step per unique input hash */
+export interface CacheEntry {
+    /** Duration of the original execution in milliseconds */
+    durationMs: number;
+    /** SHA-256 hash of all input content */
+    inputHash: string;
+    /** Output file paths that were produced (relative to package root) */
+    outputPaths: string[];
+    /** Human-readable summary of what was cached */
+    summary: string;
+    /** When this entry was created */
+    timestamp: string;
+}
+/** Result of a cache lookup */
+export type CacheLookupResult = {
+    hit: false;
+    currentHash: string;
+} | {
+    hit: true;
+    entry: CacheEntry;
+};
+/** The cache manifest for a single pipeline step */
+export interface CacheManifest {
+    /** The most recent cache entry */
+    entry: CacheEntry | null;
+    /** Step name (e.g., "fetch-docs", "generate-configs") */
+    step: string;
+}
+/** Stats collected across all pipeline steps for reporting */
+export interface CacheStats {
+    /** Steps where cache was hit (skipped execution) */
+    hits: number;
+    /** Steps where cache was missed (executed normally) */
+    misses: number;
+    /** Steps that were skipped for other reasons (--skip-fetch, etc.) */
+    skipped: number;
+    /** Per-step detail */
+    steps: Record<string, "disabled" | "hit" | "miss" | "skipped">;
+    /** Total steps that participated in caching */
+    total: number;
+}
+/** Create an empty CacheStats object */
+export declare function createCacheStats(): CacheStats;
+/** Format cache stats as a human-readable summary line */
+export declare function formatCacheStats(stats: CacheStats): string;
+/**
+ * Collect all file paths that serve as inputs for a given pipeline step.
+ * This is the core of cache key computation — if any of these files change,
+ * the step must re-execute.
+ */
+export declare function getStepInputPaths(rootDir: string, step: string): string[];
+/**
+ * Compute a SHA-256 hash of the concatenated content of multiple files.
+ * Files are sorted by path for deterministic ordering. Missing files
+ * contribute a sentinel value so the hash changes if a file is deleted.
+ *
+ * Optional `context` strings are included in the hash so that non-file
+ * state (e.g., filter flags, environment variables) can also participate
+ * in cache key computation.
+ */
+export declare function hashFiles(paths: string[], context?: string[]): string;
+/**
+ * Look up the cache for a pipeline step.
+ *
+ * Computes the current input hash and compares it against the stored manifest.
+ * Returns a hit if the hashes match AND all expected output files still exist.
+ *
+ * Optional `context` strings are included in the hash so that non-file
+ * state (e.g., area/task filter flags) participates in cache key computation.
+ */
+export declare function lookupCache(rootDir: string, step: string, context?: string[]): CacheLookupResult;
+/**
+ * Read the cache manifest for a step.
+ * Returns null if no manifest exists or it's corrupt.
+ */
+export declare function readManifest(rootDir: string, step: string): CacheManifest | null;
+/**
+ * Record a cache entry after a successful step execution.
+ */
+export declare function recordCache(rootDir: string, step: string, inputHash: string, summary: string, durationMs: number, outputPaths: string[]): void;
+/**
+ * Write a cache manifest for a step.
+ * Creates the cache directory if it doesn't exist.
+ */
+export declare function writeManifest(rootDir: string, step: string, entry: CacheEntry): void;

package/dist/pipeline/cache.js

@@ -0,0 +1,283 @@
+/**
+ * pipeline/cache.ts
+ *
+ * Content-aware caching for pipeline steps. Each step's cache key is derived
+ * from a hash of its inputs (config files, context files, task files, etc.).
+ * When inputs haven't changed between runs, cached outputs are reused.
+ *
+ * Cache storage: `results/cache/` directory with one JSON manifest per step.
+ * Each manifest maps a content hash to the step's output metadata.
+ *
+ * Cache invalidation triggers:
+ *   - Content change: any input file's content changes → hash changes → miss
+ *   - Config change: config/models.yaml, config/sources.yaml, tasks/*.yaml changes → miss
+ *   - Manual bypass: --no-cache flag skips all cache lookups
+ *   - Cache clear: delete results/cache/ to start fresh
+ */
+import { createHash } from "crypto";
+import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync, } from "fs";
+import { join, resolve } from "path";
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+const CACHE_DIR_NAME = "cache";
+const CACHE_VERSION = 1;
+// ---------------------------------------------------------------------------
+// Hashing
+// ---------------------------------------------------------------------------
+/** Create an empty CacheStats object */
+export function createCacheStats() {
+    return {
+        hits: 0,
+        misses: 0,
+        skipped: 0,
+        steps: {},
+        total: 0,
+    };
+}
+/** Format cache stats as a human-readable summary line */
+export function formatCacheStats(stats) {
+    const { hits, misses, skipped, total } = stats;
+    const evaluated = misses;
+    const parts = [];
+    if (hits > 0)
+        parts.push(`${hits} cached`);
+    if (evaluated > 0)
+        parts.push(`${evaluated} evaluated`);
+    if (skipped > 0)
+        parts.push(`${skipped} skipped`);
+    return `${parts.join(", ")} (${total} total steps)`;
+}
+// ---------------------------------------------------------------------------
+// Cache store
+// ---------------------------------------------------------------------------
+/**
+ * Collect all file paths that serve as inputs for a given pipeline step.
+ * This is the core of cache key computation — if any of these files change,
+ * the step must re-execute.
+ */
+export function getStepInputPaths(rootDir, step) {
+    const r = (rel) => resolve(rootDir, rel);
+    switch (step) {
+        case "calculate-scores": {
+            // Inputs: eval results file + scoring logic (the script itself)
+            return [
+                r("results/latest/eval-results.json"),
+                r("results/latest/eval-results-agentic.json"),
+                r("results/latest/eval-results-observed.json"),
+            ].filter((p) => existsSync(p));
+        }
+        case "eval":
+        case "eval-baseline":
+        case "eval-agentic":
+        case "eval-observed": {
+            // Per-mode cache keys: eval-baseline and eval-agentic include only
+            // their own config + expanded YAML so changes to one mode's inputs
+            // don't invalidate the other's cache entry.
+            //
+            // The legacy "eval" key includes all configs for backward compat.
+            const isBaseline = step === "eval-baseline" || step === "eval";
+            const isAgentic = step === "eval-agentic" || step === "eval";
+            const isObserved = step === "eval-observed" || step === "eval";
+            const paths = [r("config/models.yaml")];
+            // Config files — only the relevant ones for this mode
+            if (isBaseline) {
+                paths.push(r("promptfooconfig.yaml"));
+                paths.push(r("tasks/.expanded.yaml"));
+            }
+            if (isAgentic) {
+                paths.push(r("promptfooconfig.agentic.yaml"));
+                paths.push(r("tasks/.expanded.agentic.yaml"));
+            }
+            if (isObserved) {
+                paths.push(r("promptfooconfig.observed.yaml"));
+            }
+            // Context files (shared across modes)
+            const contextsDir = r("contexts");
+            if (existsSync(contextsDir)) {
+                const contextFiles = readdirSync(contextsDir)
+                    .filter((f) => f.endsWith(".md"))
+                    .map((f) => join(contextsDir, f));
+                paths.push(...contextFiles);
+                // Canonical contexts
+                const canonicalDir = join(contextsDir, "canonical");
+                if (existsSync(canonicalDir)) {
+                    const canonicalFiles = readdirSync(canonicalDir)
+                        .filter((f) => f.endsWith(".md"))
+                        .map((f) => join(canonicalDir, f));
+                    paths.push(...canonicalFiles);
+                }
+            }
+            // Task files (contain assertions and test definitions).
+            // Exclude generated .expanded*.yaml files — those are already listed
+            // explicitly above per mode.
+            const tasksDir = r("tasks");
+            if (existsSync(tasksDir)) {
+                const taskFiles = readdirSync(tasksDir)
+                    .filter((f) => (f.endsWith(".yaml") || f.endsWith(".yml")) &&
+                    !f.startsWith(".expanded"))
+                    .map((f) => join(tasksDir, f));
+                paths.push(...taskFiles);
+            }
+            // Reference solutions (used by grader assertions)
+            const refDir = r("canonical/reference-solutions");
+            if (existsSync(refDir)) {
+                const refFiles = readdirSync(refDir)
+                    .map((f) => join(refDir, f))
+                    .filter((f) => statSync(f).isFile());
+                paths.push(...refFiles);
+            }
+            return paths;
+        }
+        case "fetch-docs": {
+            // Inputs: config/sources.yaml, config/models.yaml, task files (which contain inline mappings)
+            const paths = [r("config/sources.yaml"), r("config/models.yaml")];
+            // Include all task files (they define feature areas)
+            const tasksDir = r("tasks");
+            if (existsSync(tasksDir)) {
+                const taskFiles = readdirSync(tasksDir)
+                    .filter((f) => f.endsWith(".yaml") || f.endsWith(".yml"))
+                    .map((f) => join(tasksDir, f));
+                paths.push(...taskFiles);
+            }
+            return paths;
+        }
+        case "generate-configs": {
+            // Inputs: config/models.yaml, config/sources.yaml, all task files
+            const paths = [r("config/models.yaml"), r("config/sources.yaml")];
+            const tasksDir = r("tasks");
+            if (existsSync(tasksDir)) {
+                const taskFiles = readdirSync(tasksDir)
+                    .filter((f) => f.endsWith(".yaml") || f.endsWith(".yml"))
+                    .map((f) => join(tasksDir, f));
+                paths.push(...taskFiles);
+            }
+            return paths;
+        }
+        case "report": {
+            // Inputs: score summary
+            return [r("results/latest/score-summary.json")];
+        }
+        default:
+            return [];
+    }
+}
+/**
+ * Compute a SHA-256 hash of the concatenated content of multiple files.
+ * Files are sorted by path for deterministic ordering. Missing files
+ * contribute a sentinel value so the hash changes if a file is deleted.
+ *
+ * Optional `context` strings are included in the hash so that non-file
+ * state (e.g., filter flags, environment variables) can also participate
+ * in cache key computation.
+ */
+export function hashFiles(paths, context) {
+    const hash = createHash("sha256");
+    // Include cache version so format changes invalidate everything
+    hash.update(`cache-version:${CACHE_VERSION}\n`);
+    // Include non-file context (filter flags, env vars, etc.)
+    if (context && context.length > 0) {
+        for (const c of context) {
+            hash.update(`context:${c}\n`);
+        }
+    }
+    const sorted = [...paths].sort();
+    for (const filePath of sorted) {
+        hash.update(`path:${filePath}\n`);
+        if (existsSync(filePath)) {
+            const content = readFileSync(filePath);
+            hash.update(content);
+        }
+        else {
+            hash.update("__missing__\n");
+        }
+        hash.update("\n---\n");
+    }
+    return hash.digest("hex");
+}
+/**
+ * Look up the cache for a pipeline step.
+ *
+ * Computes the current input hash and compares it against the stored manifest.
+ * Returns a hit if the hashes match AND all expected output files still exist.
+ *
+ * Optional `context` strings are included in the hash so that non-file
+ * state (e.g., area/task filter flags) participates in cache key computation.
+ */
+export function lookupCache(rootDir, step, context) {
+    const inputPaths = getStepInputPaths(rootDir, step);
+    const currentHash = hashFiles(inputPaths, context);
+    const manifest = readManifest(rootDir, step);
+    if (!manifest?.entry) {
+        return { currentHash, hit: false };
+    }
+    // Hash must match
+    if (manifest.entry.inputHash !== currentHash) {
+        return { currentHash, hit: false };
+    }
+    // All output files must still exist (guard against manual deletion)
+    const outputsExist = manifest.entry.outputPaths.every((p) => existsSync(resolve(rootDir, p)));
+    if (!outputsExist) {
+        return { currentHash, hit: false };
+    }
+    return { entry: manifest.entry, hit: true };
+}
+/**
+ * Read the cache manifest for a step.
+ * Returns null if no manifest exists or it's corrupt.
+ */
+export function readManifest(rootDir, step) {
+    const manifestPath = getManifestPath(rootDir, step);
+    if (!existsSync(manifestPath))
+        return null;
+    try {
+        const raw = readFileSync(manifestPath, "utf-8");
+        const parsed = JSON.parse(raw);
+        if (parsed.step !== step)
+            return null;
+        return parsed;
+    }
+    catch {
+        return null;
+    }
+}
+// ---------------------------------------------------------------------------
+// Cache operations
+// ---------------------------------------------------------------------------
+/**
+ * Record a cache entry after a successful step execution.
+ */
+export function recordCache(rootDir, step, inputHash, summary, durationMs, outputPaths) {
+    const entry = {
+        durationMs,
+        inputHash,
+        outputPaths,
+        summary,
+        timestamp: new Date().toISOString(),
+    };
+    writeManifest(rootDir, step, entry);
+}
+/**
+ * Write a cache manifest for a step.
+ * Creates the cache directory if it doesn't exist.
+ */
+export function writeManifest(rootDir, step, entry) {
+    const cacheDir = getCacheDir(rootDir);
+    if (!existsSync(cacheDir)) {
+        mkdirSync(cacheDir, { recursive: true });
+    }
+    const manifest = { entry, step };
+    const manifestPath = getManifestPath(rootDir, step);
+    writeFileSync(manifestPath, JSON.stringify(manifest, null, 2));
+}
+// ---------------------------------------------------------------------------
+// Cache stats
+// ---------------------------------------------------------------------------
+/** Get the cache directory path */
+function getCacheDir(rootDir) {
+    return resolve(rootDir, "results", CACHE_DIR_NAME);
+}
+/** Get the manifest file path for a step */
+function getManifestPath(rootDir, step) {
+    return join(getCacheDir(rootDir), `${step}.json`);
+}

package/dist/pipeline/calculate-scores.d.ts

@@ -0,0 +1,102 @@
+import { type ResolvedSourceConfig } from "../sources.js";
+import { type ActualScoreEntry, type ComponentResult } from "../_vendor/ailf-core/index.d.ts";
+import type { GraderJudgment, PerModelEntry } from "./types.js";
+export { classifyRubric, detectFeatureArea, extractUrlMetadata, mergeScores, parseRubricScore, } from "../_vendor/ailf-core/index.d.ts";
+export type { ActualScoreEntry, ComponentResult, TestResult, UrlMetadata, } from "../_vendor/ailf-core/index.d.ts";
+export interface PromptfooResultsWrapper {
+    results: RawTestResult[];
+    stats: {
+        successes: number;
+        failures: number;
+        tokenUsage?: {
+            assertions?: {
+                completion: number;
+                prompt: number;
+                total: number;
+            };
+            completion: number;
+            prompt: number;
+            total: number;
+        };
+    };
+}
+export interface RawPromptfooFile {
+    [key: string]: unknown;
+    config?: {
+        defaultTest?: {
+            options?: {
+                provider?: string;
+                rubricProvider?: string;
+            };
+        };
+    };
+    results: PromptfooResultsWrapper;
+}
+export interface RawTestResult {
+    cost?: number;
+    error?: string;
+    gradingResult: null | {
+        componentResults: ComponentResult[];
+        pass: boolean;
+    };
+    metadata?: Record<string, unknown>;
+    provider?: {
+        id?: string;
+        label?: string;
+    };
+    response: {
+        output: string;
+    };
+    testCase?: {
+        description?: string;
+        vars?: Record<string, string>;
+    };
+    vars: Record<string, string>;
+}
+/**
+ * Calculate scores grouped by model. Each model gets its own FeatureScore[]
+ * and model-level aggregates.
+ *
+ * Uses the provider.id from Promptfoo results to identify models.
+ * Falls back to provider.label, then "unknown" if neither is available.
+ *
+ * @returns Record keyed by model ID, or null if only one model was used
+ *          (per-model breakdown is redundant when there's only one model).
+ */
+export declare function calculateScoresPerModel(resultsPath: string, weights: Record<string, number>): null | PerModelEntry[];
+/**
+ * Extract grader judgments (reason text + scores) from evaluation results.
+ *
+ * This preserves the grader's natural language reasoning for downstream
+ * analysis (failure mode classification, gap analysis). Each llm-rubric
+ * assertion produces one GraderJudgment entry.
+ *
+ * Phase 3a prerequisite: structured judgment data for failure mode extraction.
+ */
+export declare function extractGraderJudgments(resultsPath: string): GraderJudgment[];
+/**
+ * Score agentic evaluation results. In agentic mode, all test entries are
+ * gold-only (no baseline entries — the .expanded.agentic.yaml fix ensures this).
+ * The model retrieves docs via tools, so all results map to "actual" scores.
+ *
+ * Returns a record keyed by feature area with the composite actual score.
+ */
+export declare function scoreAgenticResults(resultsPath: string, weights: Record<string, number>): Record<string, ActualScoreEntry>;
+/** Options for the calculate-scores main() function. */
+export interface CalculateScoresOptions {
+    /** Allowed origins for source isolation reporting */
+    allowedOrigins?: string[];
+    /** Evaluation mode (controls which result files are read) */
+    mode?: string;
+    /** Pre-resolved source config (skips loadSource() call) */
+    resolvedSource?: ResolvedSourceConfig;
+    /** Path to baseline results file (default: results/latest/eval-results.json) */
+    resultsPath?: string;
+    /** Root directory of the eval package (required) */
+    rootDir: string;
+    /** Search mode for source verification metadata */
+    searchMode?: string;
+    /** Documentation source name */
+    source?: string;
+}
+export declare function calculateAndWriteScores(options: CalculateScoresOptions): void;