@sanity/ailf 2.0.0 → 2.0.2

package/dist/pipeline/retrieval-metrics.js

@@ -2,13 +2,14 @@
  * pipeline/retrieval-metrics.ts
  *
  * Computes retrieval precision and recall by comparing agent-retrieved
- * doc slugs against canonical_docs defined in task YAML files.
+ * doc slugs against canonical_docs defined in task definitions.
  *
- * This is a pure computation module — no file I/O beyond reading task YAMLs.
+ * This is a pure computation module — no file I/O beyond reading task files.
  */
-import { existsSync, readFileSync, readdirSync } from "fs";
-import { resolve } from "path";
-import { load } from "js-yaml";
+import { existsSync } from "fs";
+import { discoverTsTaskFiles, loadTsTaskFileSync, } from "../adapters/task-sources/task-file-loader.js";
+import { resolveVendoredSubdir } from "./compiler/config-loader.js";
+// --- Types for task definitions (just the canonical_docs part) ---
 /**
  * Compute retrieval metrics from agentic behavior data.
  *
@@ -107,28 +108,35 @@ export function computeTaskMetrics(taskId, area, retrieved, canonical) {
     };
 }
 /**
- * Load canonical_docs from all task YAML files.
+ * Load canonical docs from *.task.ts files in tasks/literacy/.
  * Returns a map of taskId → { slugs: Set<string>, area: string }.
  */
 export function loadCanonicalDocs(rootDir) {
-    const tasksDir = resolve(rootDir, "tasks");
+    const tasksDir = resolveVendoredSubdir(rootDir, "tasks/literacy");
     if (!existsSync(tasksDir))
         return new Map();
     const result = new Map();
-    const files = readdirSync(tasksDir).filter((f) => (f.endsWith(".yaml") || f.endsWith(".yml")) && !f.startsWith("."));
+    const files = discoverTsTaskFiles(tasksDir);
     for (const file of files) {
-        const area = file.replace(/\.ya?ml$/, "");
-        const raw = readFileSync(resolve(tasksDir, file), "utf-8");
-        const parsed = load(raw);
-        if (!Array.isArray(parsed))
-            continue;
-        for (const entry of parsed) {
-            const task = entry;
-            if (task.id && task.canonical_docs && task.canonical_docs.length > 0) {
-                result.set(task.id, {
-                    area,
-                    slugs: new Set(task.canonical_docs.map((d) => d.slug)),
-                });
+        const loaded = loadTsTaskFileSync(file);
+        for (const task of loaded.tasks) {
+            const t = task;
+            const id = typeof t.id === "string" ? t.id : undefined;
+            const area = typeof t.area === "string" ? t.area : undefined;
+            if (!id || !area)
+                continue;
+            // Extract slugs from context.docs
+            const context = t.context;
+            if (!context?.docs || !Array.isArray(context.docs))
+                continue;
+            const slugs = new Set();
+            for (const doc of context.docs) {
+                const d = doc;
+                if (typeof d.slug === "string")
+                    slugs.add(d.slug);
+            }
+            if (slugs.size > 0) {
+                result.set(id, { area, slugs });
             }
         }
     }

package/dist/pipeline/steps/calculate-scores-step.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Pipeline step: Calculate AI Literacy Scores from eval results.
+ *
+ * Preconditions: eval-results.json exists and is valid
+ * Postconditions: score-summary.json exists and is valid
+ *
+ * Cache key: eval results JSON file(s)
+ * Cache outputs: results/latest/score-summary.json
+ */
+import type { EvalMode, StepResult } from "../types.js";
+export declare function runCalculateScores(source?: string, mode?: EvalMode, noCache?: boolean): StepResult;

package/dist/pipeline/steps/calculate-scores-step.js

@@ -0,0 +1,89 @@
+/**
+ * Pipeline step: Calculate AI Literacy Scores from eval results.
+ *
+ * Preconditions: eval-results.json exists and is valid
+ * Postconditions: score-summary.json exists and is valid
+ *
+ * Cache key: eval results JSON file(s)
+ * Cache outputs: results/latest/score-summary.json
+ */
+import { execSync } from "child_process";
+import { dirname, resolve } from "path";
+import { fileURLToPath } from "url";
+import { getStepInputPaths, hashFiles, lookupCache, recordCache, } from "../cache.js";
+import { checkResultsExist, checkScoreSummaryValid } from "../checks.js";
+import { RESULTS_FILES } from "./eval-step.js";
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const ROOT = resolve(__dirname, "..", "..", "..");
+export function runCalculateScores(source, mode = "baseline", noCache = false) {
+    const start = Date.now();
+    // For full mode, use the baseline results file as the primary input
+    // (calculate-scores reads all available results files internally)
+    const primaryMode = mode === "full" ? "baseline" : mode;
+    const resultsFile = RESULTS_FILES[primaryMode];
+    const resultsIssues = checkResultsExist(ROOT, resultsFile);
+    const resultsErrors = resultsIssues.filter((i) => i.severity === "error");
+    if (resultsErrors.length > 0) {
+        return {
+            durationMs: Date.now() - start,
+            error: `Results missing: ${resultsErrors.map((e) => e.message).join("; ")}`,
+            status: "failed",
+        };
+    }
+    // Cache check
+    if (!noCache) {
+        const cacheResult = lookupCache(ROOT, "calculate-scores");
+        if (cacheResult.hit) {
+            return {
+                durationMs: Date.now() - start,
+                status: "success",
+                summary: `Skipped (cached) — ${cacheResult.entry.summary}`,
+            };
+        }
+    }
+    // Execute — note: calculate-scores exits 1 when areas are below critical,
+    // which is expected behavior, not an error
+    try {
+        const sourceArg = source ? ` --source ${source}` : "";
+        const resultsArg = primaryMode !== "baseline" ? ` ${resultsFile}` : "";
+        execSync(`tsx src/lib/calculate-scores.ts${resultsArg}${sourceArg}`, {
+            cwd: ROOT,
+            env: process.env,
+            stdio: "inherit",
+        });
+    }
+    catch (err) {
+        const code = err !== null && typeof err === "object" && "status" in err
+            ? err.status
+            : 1;
+        // Exit code 1 means "areas below critical" — that's expected
+        if (code !== 1) {
+            return {
+                durationMs: Date.now() - start,
+                error: `calculate-scores failed with exit code ${code}`,
+                status: "failed",
+            };
+        }
+    }
+    // Postcondition: score summary exists and is valid
+    const summaryIssues = checkScoreSummaryValid(ROOT);
+    const summaryErrors = summaryIssues.filter((i) => i.severity === "error");
+    if (summaryErrors.length > 0) {
+        return {
+            durationMs: Date.now() - start,
+            error: `Postcondition failed: ${summaryErrors.map((e) => e.message).join("; ")}`,
+            status: "failed",
+        };
+    }
+    const durationMs = Date.now() - start;
+    const summary = "Scores calculated and summary written";
+    // Record cache
+    if (!noCache) {
+        const inputPaths = getStepInputPaths(ROOT, "calculate-scores");
+        const inputHash = hashFiles(inputPaths);
+        recordCache(ROOT, "calculate-scores", inputHash, summary, durationMs, [
+            "results/latest/score-summary.json",
+        ]);
+    }
+    return { durationMs, status: "success", summary };
+}

package/dist/pipeline/steps/compare-step.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Pipeline step: Compare current scores against a baseline.
+ *
+ * Preconditions: score-summary.json exists
+ * Postconditions: comparison-report.json written to results/latest/
+ *
+ * This step is optional — it only runs when --compare is passed
+ * (or a baseline exists and auto-compare is enabled).
+ */
+import type { CompareOptions, StepResult } from "../types.js";
+/**
+ * Run comparison against a baseline.
+ *
+ * @param rootDir Package root directory
+ * @param baselinePath Explicit baseline file path (optional — uses latest if omitted)
+ * @param options Compare options (noise threshold, etc.)
+ */
+export declare function runCompare(rootDir: string, baselinePath?: string, options?: CompareOptions): StepResult;

package/dist/pipeline/steps/compare-step.js

@@ -0,0 +1,90 @@
+/**
+ * Pipeline step: Compare current scores against a baseline.
+ *
+ * Preconditions: score-summary.json exists
+ * Postconditions: comparison-report.json written to results/latest/
+ *
+ * This step is optional — it only runs when --compare is passed
+ * (or a baseline exists and auto-compare is enabled).
+ */
+import { existsSync, readFileSync, readdirSync, writeFileSync } from "fs";
+import { join, resolve } from "path";
+import { compare } from "../compare.js";
+/**
+ * Run comparison against a baseline.
+ *
+ * @param rootDir Package root directory
+ * @param baselinePath Explicit baseline file path (optional — uses latest if omitted)
+ * @param options Compare options (noise threshold, etc.)
+ */
+export function runCompare(rootDir, baselinePath, options) {
+    const start = Date.now();
+    const scoreSummaryPath = resolve(rootDir, "results", "latest", "score-summary.json");
+    if (!existsSync(scoreSummaryPath)) {
+        return {
+            durationMs: Date.now() - start,
+            error: "score-summary.json not found. Run calculate-scores first.",
+            status: "failed",
+        };
+    }
+    // Load experiment (current run)
+    const experiment = JSON.parse(readFileSync(scoreSummaryPath, "utf-8"));
+    // Resolve baseline
+    let resolvedBaselinePath;
+    if (baselinePath) {
+        resolvedBaselinePath = resolve(baselinePath);
+    }
+    else {
+        const baselinesDir = resolve(rootDir, "results", "baselines");
+        if (!existsSync(baselinesDir)) {
+            return {
+                reason: "No baselines directory found. Run 'pnpm baseline:save' first.",
+                status: "skipped",
+            };
+        }
+        const files = readdirSync(baselinesDir)
+            .filter((f) => f.endsWith(".json"))
+            .sort()
+            .reverse();
+        if (files.length === 0) {
+            return {
+                reason: "No baseline files found. Run 'pnpm baseline:save' first.",
+                status: "skipped",
+            };
+        }
+        resolvedBaselinePath = join(baselinesDir, files[0]);
+    }
+    if (!existsSync(resolvedBaselinePath)) {
+        return {
+            durationMs: Date.now() - start,
+            error: `Baseline file not found: ${resolvedBaselinePath}`,
+            status: "failed",
+        };
+    }
+    const baseline = JSON.parse(readFileSync(resolvedBaselinePath, "utf-8"));
+    // Run comparison
+    const report = compare(baseline, experiment, options);
+    // Write report
+    const reportPath = resolve(rootDir, "results", "latest", "comparison-report.json");
+    writeFileSync(reportPath, JSON.stringify(report, null, 2));
+    // Build summary
+    const improved = report.improved.length;
+    const regressed = report.regressed.length;
+    const unchanged = report.unchanged.length;
+    const overallDelta = report.deltas.overall;
+    const deltaStr = overallDelta > 0
+        ? `+${Math.round(overallDelta)}`
+        : String(Math.round(overallDelta));
+    const parts = [`Overall: ${deltaStr}`];
+    if (improved > 0)
+        parts.push(`${improved} improved`);
+    if (regressed > 0)
+        parts.push(`${regressed} regressed`);
+    if (unchanged > 0)
+        parts.push(`${unchanged} unchanged`);
+    return {
+        durationMs: Date.now() - start,
+        status: "success",
+        summary: parts.join(", "),
+    };
+}

package/dist/pipeline/steps/eval-step.d.ts

@@ -0,0 +1,53 @@
+/**
+ * Pipeline step: Run promptfoo evaluation.
+ *
+ * Preconditions: config files and context files exist
+ * Postconditions: eval-results.json exists and is valid
+ *
+ * Cache key: promptfooconfig*.yaml + contexts/*.md + tasks/*.yaml +
+ *            canonical contexts + reference solutions + config/models.yaml
+ * Cache outputs: results/latest/eval-results*.json
+ *
+ * Remote cache: When local cache misses and a Sanity token is available,
+ * the step queries the Content Lake for a report with a matching eval
+ * fingerprint. On a hit, the cached score-summary.json is written to disk
+ * and the eval + calculate-scores steps are skipped entirely.
+ *
+ * @see docs/design-docs/content-lake-eval-caching.md
+ */
+import type { ConcreteEvalMode, DebugOptions, FilterOptions, StepResult } from "../types.js";
+/** Each mode writes eval results to a different file (set in the config's outputPath) */
+export declare const RESULTS_FILES: Record<ConcreteEvalMode, string>;
+/** Extended step result that carries cache metadata for downstream steps */
+export interface EvalStepResult {
+    /** The computed eval fingerprint (for publishing in provenance) */
+    evalFingerprint?: string;
+    /** Whether this result came from a remote cache hit */
+    remoteCacheHit?: boolean;
+    /** The step result */
+    stepResult: StepResult;
+}
+/** Options for the remote cache (Content Lake fingerprint lookup) */
+export interface RemoteCacheOptions {
+    /** Whether this is a debug run (debug runs don't use remote cache) */
+    debug?: boolean;
+    /** Filter options used for fingerprint computation */
+    filter?: FilterOptions;
+    /** Grader model identifier from models.yaml */
+    graderModel: string;
+    /** Disable remote cache lookup (--no-remote-cache) */
+    noRemoteCache?: boolean;
+    /** Sanity API token for reading cached reports */
+    sanityToken?: string;
+}
+export declare function buildFilterFlags(debug?: DebugOptions): string;
+/**
+ * Extract the Promptfoo share URL from the eval results JSON.
+ *
+ * Promptfoo writes a `shareableUrl` field into the results file when
+ * `PROMPTFOO_API_KEY` is set. This replaces the previous approach of
+ * scraping the URL from a captured log file (which required piping
+ * through `tee` and broke TTY progress reporting).
+ */
+export declare function extractShareUrl(mode: ConcreteEvalMode): string | undefined;
+export declare function runEval(mode: ConcreteEvalMode, debug?: DebugOptions, concurrency?: number, noCache?: boolean, remoteCacheOpts?: RemoteCacheOptions): Promise<EvalStepResult>;