@sanity/ailf 2.0.1 → 2.1.0

package/dist/pipeline/steps/compare-step.js

@@ -1,90 +0,0 @@
-/**
- * Pipeline step: Compare current scores against a baseline.
- *
- * Preconditions: score-summary.json exists
- * Postconditions: comparison-report.json written to results/latest/
- *
- * This step is optional — it only runs when --compare is passed
- * (or a baseline exists and auto-compare is enabled).
- */
-import { existsSync, readFileSync, readdirSync, writeFileSync } from "fs";
-import { join, resolve } from "path";
-import { compare } from "../compare.js";
-/**
- * Run comparison against a baseline.
- *
- * @param rootDir Package root directory
- * @param baselinePath Explicit baseline file path (optional — uses latest if omitted)
- * @param options Compare options (noise threshold, etc.)
- */
-export function runCompare(rootDir, baselinePath, options) {
-    const start = Date.now();
-    const scoreSummaryPath = resolve(rootDir, "results", "latest", "score-summary.json");
-    if (!existsSync(scoreSummaryPath)) {
-        return {
-            durationMs: Date.now() - start,
-            error: "score-summary.json not found. Run calculate-scores first.",
-            status: "failed",
-        };
-    }
-    // Load experiment (current run)
-    const experiment = JSON.parse(readFileSync(scoreSummaryPath, "utf-8"));
-    // Resolve baseline
-    let resolvedBaselinePath;
-    if (baselinePath) {
-        resolvedBaselinePath = resolve(baselinePath);
-    }
-    else {
-        const baselinesDir = resolve(rootDir, "results", "baselines");
-        if (!existsSync(baselinesDir)) {
-            return {
-                reason: "No baselines directory found. Run 'pnpm baseline:save' first.",
-                status: "skipped",
-            };
-        }
-        const files = readdirSync(baselinesDir)
-            .filter((f) => f.endsWith(".json"))
-            .sort()
-            .reverse();
-        if (files.length === 0) {
-            return {
-                reason: "No baseline files found. Run 'pnpm baseline:save' first.",
-                status: "skipped",
-            };
-        }
-        resolvedBaselinePath = join(baselinesDir, files[0]);
-    }
-    if (!existsSync(resolvedBaselinePath)) {
-        return {
-            durationMs: Date.now() - start,
-            error: `Baseline file not found: ${resolvedBaselinePath}`,
-            status: "failed",
-        };
-    }
-    const baseline = JSON.parse(readFileSync(resolvedBaselinePath, "utf-8"));
-    // Run comparison
-    const report = compare(baseline, experiment, options);
-    // Write report
-    const reportPath = resolve(rootDir, "results", "latest", "comparison-report.json");
-    writeFileSync(reportPath, JSON.stringify(report, null, 2));
-    // Build summary
-    const improved = report.improved.length;
-    const regressed = report.regressed.length;
-    const unchanged = report.unchanged.length;
-    const overallDelta = report.deltas.overall;
-    const deltaStr = overallDelta > 0
-        ? `+${Math.round(overallDelta)}`
-        : String(Math.round(overallDelta));
-    const parts = [`Overall: ${deltaStr}`];
-    if (improved > 0)
-        parts.push(`${improved} improved`);
-    if (regressed > 0)
-        parts.push(`${regressed} regressed`);
-    if (unchanged > 0)
-        parts.push(`${unchanged} unchanged`);
-    return {
-        durationMs: Date.now() - start,
-        status: "success",
-        summary: parts.join(", "),
-    };
-}

package/dist/pipeline/steps/eval-step.d.ts

@@ -1,53 +0,0 @@
-/**
- * Pipeline step: Run promptfoo evaluation.
- *
- * Preconditions: config files and context files exist
- * Postconditions: eval-results.json exists and is valid
- *
- * Cache key: promptfooconfig*.yaml + contexts/*.md + tasks/*.yaml +
- *            canonical contexts + reference solutions + config/models.yaml
- * Cache outputs: results/latest/eval-results*.json
- *
- * Remote cache: When local cache misses and a Sanity token is available,
- * the step queries the Content Lake for a report with a matching eval
- * fingerprint. On a hit, the cached score-summary.json is written to disk
- * and the eval + calculate-scores steps are skipped entirely.
- *
- * @see docs/design-docs/content-lake-eval-caching.md
- */
-import type { ConcreteEvalMode, DebugOptions, FilterOptions, StepResult } from "../types.js";
-/** Each mode writes eval results to a different file (set in the config's outputPath) */
-export declare const RESULTS_FILES: Record<ConcreteEvalMode, string>;
-/** Extended step result that carries cache metadata for downstream steps */
-export interface EvalStepResult {
-    /** The computed eval fingerprint (for publishing in provenance) */
-    evalFingerprint?: string;
-    /** Whether this result came from a remote cache hit */
-    remoteCacheHit?: boolean;
-    /** The step result */
-    stepResult: StepResult;
-}
-/** Options for the remote cache (Content Lake fingerprint lookup) */
-export interface RemoteCacheOptions {
-    /** Whether this is a debug run (debug runs don't use remote cache) */
-    debug?: boolean;
-    /** Filter options used for fingerprint computation */
-    filter?: FilterOptions;
-    /** Grader model identifier from models.yaml */
-    graderModel: string;
-    /** Disable remote cache lookup (--no-remote-cache) */
-    noRemoteCache?: boolean;
-    /** Sanity API token for reading cached reports */
-    sanityToken?: string;
-}
-export declare function buildFilterFlags(debug?: DebugOptions): string;
-/**
- * Extract the Promptfoo share URL from the eval results JSON.
- *
- * Promptfoo writes a `shareableUrl` field into the results file when
- * `PROMPTFOO_API_KEY` is set. This replaces the previous approach of
- * scraping the URL from a captured log file (which required piping
- * through `tee` and broke TTY progress reporting).
- */
-export declare function extractShareUrl(mode: ConcreteEvalMode): string | undefined;
-export declare function runEval(mode: ConcreteEvalMode, debug?: DebugOptions, concurrency?: number, noCache?: boolean, remoteCacheOpts?: RemoteCacheOptions): Promise<EvalStepResult>;

package/dist/pipeline/steps/eval-step.js

@@ -1,347 +0,0 @@
-/**
- * Pipeline step: Run promptfoo evaluation.
- *
- * Preconditions: config files and context files exist
- * Postconditions: eval-results.json exists and is valid
- *
- * Cache key: promptfooconfig*.yaml + contexts/*.md + tasks/*.yaml +
- *            canonical contexts + reference solutions + config/models.yaml
- * Cache outputs: results/latest/eval-results*.json
- *
- * Remote cache: When local cache misses and a Sanity token is available,
- * the step queries the Content Lake for a report with a matching eval
- * fingerprint. On a hit, the cached score-summary.json is written to disk
- * and the eval + calculate-scores steps are skipped entirely.
- *
- * @see docs/design-docs/content-lake-eval-caching.md
- */
-import { execSync } from "child_process";
-import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
-import { dirname, resolve } from "path";
-import { fileURLToPath } from "url";
-import { getStepInputPaths, hashFiles, lookupCache, recordCache, } from "../cache.js";
-import { checkCanonicalContextsExist, checkGeneratedConfigsExist, checkResultsExist, } from "../checks.js";
-import { computeEvalFingerprint } from "../eval-fingerprint.js";
-import { resolveMappings } from "../resolve-mappings.js";
-const __dirname = dirname(fileURLToPath(import.meta.url));
-const ROOT = resolve(__dirname, "..", "..", "..");
-const CONFIG_FILES = {
-    agentic: "promptfooconfig.agentic.yaml",
-    baseline: "promptfooconfig.yaml",
-    observed: "promptfooconfig.observed.yaml",
-};
-/** Each mode writes eval results to a different file (set in the config's outputPath) */
-export const RESULTS_FILES = {
-    agentic: "results/latest/eval-results-agentic.json",
-    baseline: "results/latest/eval-results.json",
-    observed: "results/latest/eval-results-observed.json",
-};
-export function buildFilterFlags(debug) {
-    if (!debug?.enabled)
-        return "";
-    const flags = [];
-    if (debug.pattern) {
-        flags.push(`--filter-pattern '${debug.pattern}'`);
-    }
-    if (debug.sample) {
-        flags.push(`--filter-sample ${debug.sample}`);
-    }
-    if (debug.firstN) {
-        flags.push(`--filter-first-n ${debug.firstN}`);
-    }
-    // Default: first 2 tests when no other filters specified
-    if (flags.length === 0) {
-        flags.push("--filter-first-n 2");
-    }
-    return " " + flags.join(" ");
-}
-/**
- * Extract the Promptfoo share URL from the eval results JSON.
- *
- * Promptfoo writes a `shareableUrl` field into the results file when
- * `PROMPTFOO_API_KEY` is set. This replaces the previous approach of
- * scraping the URL from a captured log file (which required piping
- * through `tee` and broke TTY progress reporting).
- */
-export function extractShareUrl(mode) {
-    const resultsPath = resolve(ROOT, RESULTS_FILES[mode]);
-    if (!existsSync(resultsPath))
-        return undefined;
-    try {
-        const raw = readFileSync(resultsPath, "utf-8");
-        const data = JSON.parse(raw);
-        return data.shareableUrl ?? undefined;
-    }
-    catch {
-        return undefined;
-    }
-}
-// ---------------------------------------------------------------------------
-// Post-eval error scanning
-// ---------------------------------------------------------------------------
-export async function runEval(mode, debug, concurrency, noCache = false, remoteCacheOpts) {
-    const start = Date.now();
-    // Precondition: config file exists
-    const configIssues = checkGeneratedConfigsExist(ROOT);
-    const configErrors = configIssues.filter((i) => i.severity === "error");
-    if (configErrors.length > 0) {
-        return {
-            stepResult: {
-                durationMs: Date.now() - start,
-                error: `Config files missing: ${configErrors.map((e) => e.message).join("; ")}`,
-                status: "failed",
-            },
-        };
-    }
-    // Precondition: canonical context files exist for all mapped tasks
-    const mappings = resolveMappings(ROOT);
-    const taskIds = Object.values(mappings.feature_areas).flatMap((area) => area.tasks.map((t) => t.id));
-    const contextIssues = checkCanonicalContextsExist(ROOT, taskIds);
-    const contextErrors = contextIssues.filter((i) => i.severity === "error");
-    if (contextErrors.length > 0) {
-        return {
-            stepResult: {
-                durationMs: Date.now() - start,
-                error: `Context files missing. Run 'pnpm fetch-docs' first. ${contextErrors.map((e) => e.message).join("; ")}`,
-                status: "failed",
-            },
-        };
-    }
-    // -----------------------------------------------------------------------
-    // Compute eval fingerprint (used for both remote cache + provenance)
-    // Only for non-debug runs — debug runs use test subsets.
-    // -----------------------------------------------------------------------
-    let evalFingerprint;
-    if (!debug?.enabled && remoteCacheOpts?.graderModel) {
-        try {
-            evalFingerprint = computeEvalFingerprint({
-                filter: remoteCacheOpts.filter,
-                graderModel: remoteCacheOpts.graderModel,
-                mode,
-                rootDir: ROOT,
-            });
-        }
-        catch (err) {
-            console.warn(`  ⚠️  Could not compute eval fingerprint: ${err instanceof Error ? err.message : String(err)}`);
-        }
-    }
-    // -----------------------------------------------------------------------
-    // Cache check — local first, then remote
-    // -----------------------------------------------------------------------
-    // Local cache check — skip eval if inputs unchanged (biggest cost saver).
-    // Each mode gets its own cache key so that in `full` mode, a fresh agentic
-    // cache doesn't force baseline to re-run (or vice versa).
-    const cacheKey = `eval-${mode}`;
-    if (!noCache) {
-        const cacheResult = lookupCache(ROOT, cacheKey);
-        if (cacheResult.hit) {
-            return {
-                evalFingerprint,
-                stepResult: {
-                    durationMs: Date.now() - start,
-                    status: "success",
-                    summary: `Skipped (cached) — ${cacheResult.entry.summary}`,
-                },
-            };
-        }
-    }
-    // Remote cache check — query Content Lake for matching fingerprint
-    if (evalFingerprint &&
-        !noCache &&
-        !remoteCacheOpts?.noRemoteCache &&
-        remoteCacheOpts?.sanityToken) {
-        const remoteCacheResult = await checkRemoteCache(evalFingerprint, remoteCacheOpts.sanityToken);
-        if (remoteCacheResult) {
-            return {
-                evalFingerprint,
-                remoteCacheHit: true,
-                stepResult: {
-                    durationMs: Date.now() - start,
-                    status: "success",
-                    summary: `Skipped (remote cache hit) — reusing report ${remoteCacheResult.reportId} from ${remoteCacheResult.completedAt}`,
-                },
-            };
-        }
-    }
-    // Execute — run promptfoo directly with inherited stdio so the TTY
-    // progress bar works in interactive terminals and the CI progress
-    // reporter works in CI environments. Previously this was piped through
-    // `tee` to capture a log file for share-URL extraction, but `tee`
-    // destroyed TTY detection, disabling all progress output. The share URL
-    // is now read from the eval results JSON (`shareableUrl` field) instead.
-    //
-    // Sharing is enabled by default (via PROMPTFOO_API_KEY / cloud config).
-    // We set PROMPTFOO_DISABLE_SHARE_EMAIL_REQUEST=1 to prevent promptfoo's
-    // interactive email prompt from blocking the terminal in local TTY
-    // environments. In CI, isCI() already guards against the prompt, but
-    // the env var provides defense-in-depth for all execution contexts.
-    const configFile = CONFIG_FILES[mode];
-    const filterFlags = buildFilterFlags(debug);
-    const concurrencyFlag = concurrency ? ` --max-concurrency ${concurrency}` : "";
-    const noCacheFlag = noCache ? " --no-cache" : "";
-    const evalCmd = `dotenv -e ../../.env -o -- promptfoo eval --config ${configFile}${filterFlags}${concurrencyFlag}${noCacheFlag}`;
-    let exitCode = 0;
-    try {
-        execSync(evalCmd, {
-            cwd: ROOT,
-            env: {
-                ...process.env,
-                PROMPTFOO_DISABLE_SHARE_EMAIL_REQUEST: "1",
-            },
-            stdio: "inherit",
-        });
-    }
-    catch (err) {
-        // promptfoo exits 100 when assertions fail — that's expected, not an error
-        exitCode =
-            err !== null && typeof err === "object" && "status" in err
-                ? err.status
-                : 1;
-        if (exitCode !== 100) {
-            return {
-                evalFingerprint,
-                stepResult: {
-                    durationMs: Date.now() - start,
-                    error: `promptfoo eval failed with exit code ${exitCode}`,
-                    status: "failed",
-                },
-            };
-        }
-    }
-    // Postcondition: results file exists and is valid
-    const resultsIssues = checkResultsExist(ROOT, RESULTS_FILES[mode]);
-    const resultsErrors = resultsIssues.filter((i) => i.severity === "error");
-    if (resultsErrors.length > 0) {
-        return {
-            evalFingerprint,
-            stepResult: {
-                durationMs: Date.now() - start,
-                error: `Postcondition failed: ${resultsErrors.map((e) => e.message).join("; ")}`,
-                status: "failed",
-            },
-        };
-    }
-    // Scan results for errors and surface them clearly
-    const errorSummary = scanResultsForErrors(resolve(ROOT, RESULTS_FILES[mode]));
-    if (errorSummary) {
-        console.log();
-        console.log(errorSummary);
-    }
-    const durationMs = Date.now() - start;
-    const summary = `Evaluation complete (mode: ${mode}${debug?.enabled ? ", debug" : ""})`;
-    // Record cache — only for non-debug runs (debug uses a subset of tests).
-    // Uses per-mode cache key so baseline and agentic are independently cached.
-    if (!noCache && !debug?.enabled) {
-        const inputPaths = getStepInputPaths(ROOT, cacheKey);
-        const inputHash = hashFiles(inputPaths);
-        recordCache(ROOT, cacheKey, inputHash, summary, durationMs, [
-            RESULTS_FILES[mode],
-        ]);
-    }
-    return {
-        evalFingerprint,
-        stepResult: { durationMs, status: "success", summary },
-    };
-}
-// ---------------------------------------------------------------------------
-// Remote cache helpers
-// ---------------------------------------------------------------------------
-/**
- * Query the Sanity Content Lake for a report with a matching eval fingerprint.
- *
- * On a hit, writes the cached score-summary.json to results/latest/ so that
- * downstream steps (report, compare, publish) can proceed as if the eval
- * had just run.
- *
- * @returns The matched report metadata on hit, null on miss or error
- */
-async function checkRemoteCache(fingerprint, sanityToken) {
-    try {
-        const { ReportStore } = await import("../../report-store.js");
-        const store = new ReportStore({
-            dataset: process.env.AILF_REPORT_DATASET ?? undefined,
-            projectId: process.env.AILF_REPORT_PROJECT_ID ?? undefined,
-            token: sanityToken,
-        });
-        const startQuery = Date.now();
-        const cachedReport = await store.findByFingerprint(fingerprint);
-        const queryMs = Date.now() - startQuery;
-        if (!cachedReport) {
-            console.log(`  ℹ️  Remote cache miss — no report matches fingerprint (${queryMs}ms)`);
-            return null;
-        }
-        // Write the cached score summary to disk so downstream steps work
-        const outDir = resolve(ROOT, "results", "latest");
-        if (!existsSync(outDir)) {
-            mkdirSync(outDir, { recursive: true });
-        }
-        writeFileSync(resolve(outDir, "score-summary.json"), JSON.stringify(cachedReport.summary, null, 2));
-        console.log(`  ✅ Remote cache hit — reusing report ${cachedReport.id} from ${cachedReport.completedAt}`);
-        console.log(`  ℹ️  Fingerprint: ${fingerprint.slice(0, 16)}... (${queryMs}ms)`);
-        console.log("  ⚠️  Cached scores are statistically equivalent, not identical");
-        return {
-            completedAt: cachedReport.completedAt,
-            reportId: cachedReport.id,
-        };
-    }
-    catch (err) {
-        console.warn(`  ⚠️  Remote cache check failed: ${err instanceof Error ? err.message : String(err)}`);
-        return null;
-    }
-}
-/**
- * Read the eval results JSON and produce a human-readable summary of any
- * errored or failed tests. This surfaces API errors, timeouts, and other
- * issues that would otherwise be buried in the Promptfoo table output.
- *
- * Returns null if there are no errors/failures worth reporting.
- */
-function scanResultsForErrors(resultsPath) {
-    if (!existsSync(resultsPath))
-        return null;
-    let file;
-    try {
-        const raw = readFileSync(resultsPath, "utf-8");
-        file = JSON.parse(raw);
-    }
-    catch {
-        return null;
-    }
-    const results = file?.results?.results;
-    if (!Array.isArray(results))
-        return null;
-    const errored = [];
-    for (const r of results) {
-        if (r.gradingResult !== null)
-            continue;
-        const desc = r.testCase?.description ?? r.description ?? "unknown";
-        const provider = r.provider?.label ?? r.provider?.id ?? "unknown";
-        // No grading result = the provider errored before producing a response.
-        // This is the only case we surface — API 500s, timeouts, rate limits.
-        // Note: r.error may also be set for assertion failures, but those have
-        // a non-null gradingResult and are normal pass/fail outcomes.
-        const errorMsg = r.error
-            ? (typeof r.error === "string" ? r.error : JSON.stringify(r.error)).slice(0, 200)
-            : "Provider returned no scorable result";
-        errored.push({ description: desc, error: errorMsg, provider });
-    }
-    if (errored.length === 0)
-        return null;
-    const total = results.length;
-    const lines = [];
-    lines.push(`  ┌─────────────────────────────────────────────────────────────`);
-    lines.push(`  │ ⚠️  ${errored.length} of ${total} eval result(s) errored (no gradingResult)`);
-    lines.push(`  │`);
-    for (const e of errored) {
-        lines.push(`  │   ✗ [${e.provider}] ${e.description}`);
-        lines.push(`  │     → ${e.error}`);
-    }
-    const errorRate = Math.round((errored.length / total) * 100);
-    if (errorRate >= 25) {
-        lines.push(`  │`);
-        lines.push(`  │ 🔥 High error rate (${errorRate}%) — check API keys, rate limits,`);
-        lines.push(`  │   or model availability. Errored results are excluded from scoring.`);
-    }
-    lines.push(`  └─────────────────────────────────────────────────────────────`);
-    return lines.join("\n");
-}

package/dist/pipeline/steps/fetch-docs-step.d.ts

@@ -1,11 +0,0 @@
-/**
- * Pipeline step: Fetch documentation from Sanity CMS.
- *
- * Preconditions: tasks have inline canonical_docs
- * Postconditions: canonical context files exist for all mapped tasks
- *
- * Cache key: tasks/*.yaml + config/sources.yaml + config/models.yaml
- * Cache outputs: contexts/canonical/*.md files
- */
-import type { StepResult } from "../types.js";
-export declare function runFetchDocs(source?: string, noCache?: boolean): Promise<StepResult>;

package/dist/pipeline/steps/fetch-docs-step.js

@@ -1,84 +0,0 @@
-/**
- * Pipeline step: Fetch documentation from Sanity CMS.
- *
- * Preconditions: tasks have inline canonical_docs
- * Postconditions: canonical context files exist for all mapped tasks
- *
- * Cache key: tasks/*.yaml + config/sources.yaml + config/models.yaml
- * Cache outputs: contexts/canonical/*.md files
- */
-import { execSync } from "child_process";
-import { dirname, resolve } from "path";
-import { fileURLToPath } from "url";
-import { lookupCache, recordCache } from "../cache.js";
-import { checkCanonicalContextsExist } from "../checks.js";
-import { resolveMappings } from "../resolve-mappings.js";
-const __dirname = dirname(fileURLToPath(import.meta.url));
-const ROOT = resolve(__dirname, "..", "..", "..");
-export async function runFetchDocs(source, noCache = false) {
-    const start = Date.now();
-    // Precondition: at least one task has inline canonical mappings
-    const mappings = resolveMappings(ROOT);
-    const totalTasks = Object.values(mappings.feature_areas).reduce((sum, area) => sum + area.tasks.length, 0);
-    if (totalTasks === 0) {
-        return {
-            durationMs: Date.now() - start,
-            error: "No tasks with canonical_docs found in task files. Add canonical_docs to your task definitions.",
-            status: "failed",
-        };
-    }
-    // Cache check
-    if (!noCache) {
-        const cacheResult = lookupCache(ROOT, "fetch-docs");
-        if (cacheResult.hit) {
-            return {
-                durationMs: Date.now() - start,
-                status: "success",
-                summary: `Skipped (cached) — ${cacheResult.entry.summary}`,
-            };
-        }
-    }
-    // Execute
-    try {
-        const sourceArg = source ? ` --source ${source}` : "";
-        execSync(`pnpm fetch-docs${sourceArg}`, {
-            cwd: ROOT,
-            env: process.env,
-            stdio: "inherit",
-        });
-    }
-    catch (err) {
-        return {
-            durationMs: Date.now() - start,
-            error: `fetch-docs failed: ${err instanceof Error ? err.message : String(err)}`,
-            status: "failed",
-        };
-    }
-    // Postcondition: canonical context files exist for all mapped tasks
-    // Re-resolve in case fetch-docs modified things (unlikely but safe)
-    const postMappings = resolveMappings(ROOT);
-    const taskIds = Object.values(postMappings.feature_areas).flatMap((area) => area.tasks.map((t) => t.id));
-    const contextIssues = checkCanonicalContextsExist(ROOT, taskIds);
-    const contextErrors = contextIssues.filter((i) => i.severity === "error");
-    if (contextErrors.length > 0) {
-        return {
-            durationMs: Date.now() - start,
-            error: `Postcondition failed: ${contextErrors.map((e) => e.message).join("; ")}`,
-            status: "failed",
-        };
-    }
-    const durationMs = Date.now() - start;
-    const summary = `Fetched canonical contexts for ${taskIds.length} tasks`;
-    // Record cache
-    if (!noCache) {
-        const { getStepInputPaths, hashFiles } = await import("../cache.js");
-        const inputPaths = getStepInputPaths(ROOT, "fetch-docs");
-        const inputHash = hashFiles(inputPaths);
-        const outputPaths = [
-            ...taskIds.map((id) => `contexts/canonical/${id}.md`),
-            "contexts/document-manifest.json",
-        ];
-        recordCache(ROOT, "fetch-docs", inputHash, summary, durationMs, outputPaths);
-    }
-    return { durationMs, status: "success", summary };
-}

package/dist/pipeline/steps/generate-configs-step.d.ts

@@ -1,11 +0,0 @@
-/**
- * Pipeline step: Generate promptfoo config files from config/models.yaml.
- *
- * Preconditions: config/models.yaml is valid
- * Postconditions: promptfooconfig*.yaml files exist
- *
- * Cache key: config/models.yaml + config/sources.yaml + tasks/*.yaml
- * Cache outputs: promptfooconfig*.yaml files
- */
-import type { StepResult } from "../types.js";
-export declare function runGenerateConfigs(source?: string, noCache?: boolean): StepResult;