@sanity/ailf 0.1.0 → 0.1.2

package/dist/orchestration/env-bridge.js

@@ -1,66 +0,0 @@
-/**
- * Environment variable bridge — writes ResolvedConfig values to process.env
- * so that lib/*.ts modules (which still read process.env) work correctly.
- *
- * This replaces the former global applyEnvironment() with an explicit
- * per-step bridge. Each orchestration step calls this before invoking
- * its lib/*.ts main() function.
- *
- * Phase 9 will eliminate this file entirely by giving lib/*.ts main()
- * functions typed option parameters.
- *
- * @see docs/exec-plans/active/ports-and-adapters/phase-8-delete-legacy-step-layer.md
- */
-/**
- * Bridge ResolvedConfig values to process.env.
- *
- * Idempotent — safe to call multiple times. Only sets env vars for
- * config values that are defined (never deletes or resets).
- */
-export function bridgeConfigToEnv(config) {
-    // Mode
-    process.env.EVAL_MODE = config.mode;
-    // Search mode
-    if (config.searchMode !== "open") {
-        process.env.EVAL_SEARCH_MODE = config.searchMode;
-    }
-    // Source
-    if (config.source) {
-        process.env.DOC_SOURCE = config.source;
-    }
-    // URL-derived overrides
-    if (config.urls?.[0]) {
-        process.env.DOC_BASE_URL = config.urls[0];
-    }
-    // Sanity overrides
-    if (config.datasetOverride) {
-        process.env.SANITY_DATASET = config.datasetOverride;
-    }
-    if (config.projectIdOverride) {
-        process.env.SANITY_PROJECT_ID = config.projectIdOverride;
-    }
-    if (config.perspectiveOverride) {
-        process.env.SANITY_PERSPECTIVE = config.perspectiveOverride;
-    }
-    if (config.studioOriginOverride) {
-        process.env.SANITY_STUDIO_ORIGIN = config.studioOriginOverride;
-    }
-    if (config.sanityDocumentArgs?.length) {
-        process.env.SANITY_DOCUMENT_IDS = config.sanityDocumentArgs.join(",");
-    }
-    // Custom headers
-    if (config.headers) {
-        process.env.DOC_HEADERS = JSON.stringify(config.headers);
-    }
-    // Allowed origins
-    if (config.allowedOrigins?.length) {
-        process.env.DOC_ALLOWED_ORIGINS = config.allowedOrigins.join(",");
-    }
-    // Scoping filters
-    if (config.areas) {
-        process.env.EVAL_FILTER_AREAS = config.areas.join(",");
-    }
-    if (config.tasks) {
-        process.env.EVAL_FILTER_TASKS = config.tasks.join(",");
-    }
-}

package/dist/orchestration/steps/fetch-docs-shell.d.ts

@@ -1,17 +0,0 @@
-/**
- * Shell delegation for the fetch-docs step.
- *
- * Isolates the execSync call so it can be replaced when the pipeline
- * fully migrates to the DocFetcher port.
- */
-export interface ShellResult {
-    ok: boolean;
-    error?: string;
-}
-/**
- * Run `pnpm fetch-docs` via shell.
- *
- * Returns a result object instead of throwing so the step can
- * handle the failure uniformly.
- */
-export declare function runFetchDocsShell(rootDir: string, source?: string): ShellResult;

package/dist/orchestration/steps/fetch-docs-shell.js

@@ -1,30 +0,0 @@
-/**
- * Shell delegation for the fetch-docs step.
- *
- * Isolates the execSync call so it can be replaced when the pipeline
- * fully migrates to the DocFetcher port.
- */
-import { execSync } from "child_process";
-/**
- * Run `pnpm fetch-docs` via shell.
- *
- * Returns a result object instead of throwing so the step can
- * handle the failure uniformly.
- */
-export function runFetchDocsShell(rootDir, source) {
-    try {
-        const sourceArg = source ? ` --source ${source}` : "";
-        execSync(`pnpm fetch-docs${sourceArg}`, {
-            cwd: rootDir,
-            env: process.env,
-            stdio: "inherit",
-        });
-        return { ok: true };
-    }
-    catch (err) {
-        return {
-            ok: false,
-            error: err instanceof Error ? err.message : String(err),
-        };
-    }
-}

package/dist/pipeline/steps/calculate-scores-step.d.ts

@@ -1,11 +0,0 @@
-/**
- * Pipeline step: Calculate AI Literacy Scores from eval results.
- *
- * Preconditions: eval-results.json exists and is valid
- * Postconditions: score-summary.json exists and is valid
- *
- * Cache key: eval results JSON file(s)
- * Cache outputs: results/latest/score-summary.json
- */
-import type { EvalMode, StepResult } from "../types.js";
-export declare function runCalculateScores(source?: string, mode?: EvalMode, noCache?: boolean): StepResult;

package/dist/pipeline/steps/calculate-scores-step.js

@@ -1,89 +0,0 @@
-/**
- * Pipeline step: Calculate AI Literacy Scores from eval results.
- *
- * Preconditions: eval-results.json exists and is valid
- * Postconditions: score-summary.json exists and is valid
- *
- * Cache key: eval results JSON file(s)
- * Cache outputs: results/latest/score-summary.json
- */
-import { execSync } from "child_process";
-import { dirname, resolve } from "path";
-import { fileURLToPath } from "url";
-import { getStepInputPaths, hashFiles, lookupCache, recordCache, } from "../cache.js";
-import { checkResultsExist, checkScoreSummaryValid } from "../checks.js";
-import { RESULTS_FILES } from "./eval-step.js";
-const __dirname = dirname(fileURLToPath(import.meta.url));
-const ROOT = resolve(__dirname, "..", "..", "..");
-export function runCalculateScores(source, mode = "baseline", noCache = false) {
-    const start = Date.now();
-    // For full mode, use the baseline results file as the primary input
-    // (calculate-scores reads all available results files internally)
-    const primaryMode = mode === "full" ? "baseline" : mode;
-    const resultsFile = RESULTS_FILES[primaryMode];
-    const resultsIssues = checkResultsExist(ROOT, resultsFile);
-    const resultsErrors = resultsIssues.filter((i) => i.severity === "error");
-    if (resultsErrors.length > 0) {
-        return {
-            durationMs: Date.now() - start,
-            error: `Results missing: ${resultsErrors.map((e) => e.message).join("; ")}`,
-            status: "failed",
-        };
-    }
-    // Cache check
-    if (!noCache) {
-        const cacheResult = lookupCache(ROOT, "calculate-scores");
-        if (cacheResult.hit) {
-            return {
-                durationMs: Date.now() - start,
-                status: "success",
-                summary: `Skipped (cached) — ${cacheResult.entry.summary}`,
-            };
-        }
-    }
-    // Execute — note: calculate-scores exits 1 when areas are below critical,
-    // which is expected behavior, not an error
-    try {
-        const sourceArg = source ? ` --source ${source}` : "";
-        const resultsArg = primaryMode !== "baseline" ? ` ${resultsFile}` : "";
-        execSync(`tsx src/lib/calculate-scores.ts${resultsArg}${sourceArg}`, {
-            cwd: ROOT,
-            env: process.env,
-            stdio: "inherit",
-        });
-    }
-    catch (err) {
-        const code = err !== null && typeof err === "object" && "status" in err
-            ? err.status
-            : 1;
-        // Exit code 1 means "areas below critical" — that's expected
-        if (code !== 1) {
-            return {
-                durationMs: Date.now() - start,
-                error: `calculate-scores failed with exit code ${code}`,
-                status: "failed",
-            };
-        }
-    }
-    // Postcondition: score summary exists and is valid
-    const summaryIssues = checkScoreSummaryValid(ROOT);
-    const summaryErrors = summaryIssues.filter((i) => i.severity === "error");
-    if (summaryErrors.length > 0) {
-        return {
-            durationMs: Date.now() - start,
-            error: `Postcondition failed: ${summaryErrors.map((e) => e.message).join("; ")}`,
-            status: "failed",
-        };
-    }
-    const durationMs = Date.now() - start;
-    const summary = "Scores calculated and summary written";
-    // Record cache
-    if (!noCache) {
-        const inputPaths = getStepInputPaths(ROOT, "calculate-scores");
-        const inputHash = hashFiles(inputPaths);
-        recordCache(ROOT, "calculate-scores", inputHash, summary, durationMs, [
-            "results/latest/score-summary.json",
-        ]);
-    }
-    return { durationMs, status: "success", summary };
-}

package/dist/pipeline/steps/compare-step.d.ts

@@ -1,18 +0,0 @@
-/**
- * Pipeline step: Compare current scores against a baseline.
- *
- * Preconditions: score-summary.json exists
- * Postconditions: comparison-report.json written to results/latest/
- *
- * This step is optional — it only runs when --compare is passed
- * (or a baseline exists and auto-compare is enabled).
- */
-import type { CompareOptions, StepResult } from "../types.js";
-/**
- * Run comparison against a baseline.
- *
- * @param rootDir Package root directory
- * @param baselinePath Explicit baseline file path (optional — uses latest if omitted)
- * @param options Compare options (noise threshold, etc.)
- */
-export declare function runCompare(rootDir: string, baselinePath?: string, options?: CompareOptions): StepResult;

package/dist/pipeline/steps/compare-step.js

@@ -1,90 +0,0 @@
-/**
- * Pipeline step: Compare current scores against a baseline.
- *
- * Preconditions: score-summary.json exists
- * Postconditions: comparison-report.json written to results/latest/
- *
- * This step is optional — it only runs when --compare is passed
- * (or a baseline exists and auto-compare is enabled).
- */
-import { existsSync, readFileSync, readdirSync, writeFileSync } from "fs";
-import { join, resolve } from "path";
-import { compare } from "../compare.js";
-/**
- * Run comparison against a baseline.
- *
- * @param rootDir Package root directory
- * @param baselinePath Explicit baseline file path (optional — uses latest if omitted)
- * @param options Compare options (noise threshold, etc.)
- */
-export function runCompare(rootDir, baselinePath, options) {
-    const start = Date.now();
-    const scoreSummaryPath = resolve(rootDir, "results", "latest", "score-summary.json");
-    if (!existsSync(scoreSummaryPath)) {
-        return {
-            durationMs: Date.now() - start,
-            error: "score-summary.json not found. Run calculate-scores first.",
-            status: "failed",
-        };
-    }
-    // Load experiment (current run)
-    const experiment = JSON.parse(readFileSync(scoreSummaryPath, "utf-8"));
-    // Resolve baseline
-    let resolvedBaselinePath;
-    if (baselinePath) {
-        resolvedBaselinePath = resolve(baselinePath);
-    }
-    else {
-        const baselinesDir = resolve(rootDir, "results", "baselines");
-        if (!existsSync(baselinesDir)) {
-            return {
-                reason: "No baselines directory found. Run 'pnpm baseline:save' first.",
-                status: "skipped",
-            };
-        }
-        const files = readdirSync(baselinesDir)
-            .filter((f) => f.endsWith(".json"))
-            .sort()
-            .reverse();
-        if (files.length === 0) {
-            return {
-                reason: "No baseline files found. Run 'pnpm baseline:save' first.",
-                status: "skipped",
-            };
-        }
-        resolvedBaselinePath = join(baselinesDir, files[0]);
-    }
-    if (!existsSync(resolvedBaselinePath)) {
-        return {
-            durationMs: Date.now() - start,
-            error: `Baseline file not found: ${resolvedBaselinePath}`,
-            status: "failed",
-        };
-    }
-    const baseline = JSON.parse(readFileSync(resolvedBaselinePath, "utf-8"));
-    // Run comparison
-    const report = compare(baseline, experiment, options);
-    // Write report
-    const reportPath = resolve(rootDir, "results", "latest", "comparison-report.json");
-    writeFileSync(reportPath, JSON.stringify(report, null, 2));
-    // Build summary
-    const improved = report.improved.length;
-    const regressed = report.regressed.length;
-    const unchanged = report.unchanged.length;
-    const overallDelta = report.deltas.overall;
-    const deltaStr = overallDelta > 0
-        ? `+${Math.round(overallDelta)}`
-        : String(Math.round(overallDelta));
-    const parts = [`Overall: ${deltaStr}`];
-    if (improved > 0)
-        parts.push(`${improved} improved`);
-    if (regressed > 0)
-        parts.push(`${regressed} regressed`);
-    if (unchanged > 0)
-        parts.push(`${unchanged} unchanged`);
-    return {
-        durationMs: Date.now() - start,
-        status: "success",
-        summary: parts.join(", "),
-    };
-}

package/dist/pipeline/steps/eval-step.d.ts

@@ -1,53 +0,0 @@
-/**
- * Pipeline step: Run promptfoo evaluation.
- *
- * Preconditions: config files and context files exist
- * Postconditions: eval-results.json exists and is valid
- *
- * Cache key: promptfooconfig*.yaml + contexts/*.md + tasks/*.yaml +
- *            canonical contexts + reference solutions + config/models.yaml
- * Cache outputs: results/latest/eval-results*.json
- *
- * Remote cache: When local cache misses and a Sanity token is available,
- * the step queries the Content Lake for a report with a matching eval
- * fingerprint. On a hit, the cached score-summary.json is written to disk
- * and the eval + calculate-scores steps are skipped entirely.
- *
- * @see docs/design-docs/content-lake-eval-caching.md
- */
-import type { ConcreteEvalMode, DebugOptions, FilterOptions, StepResult } from "../types.js";
-/** Each mode writes eval results to a different file (set in the config's outputPath) */
-export declare const RESULTS_FILES: Record<ConcreteEvalMode, string>;
-/** Extended step result that carries cache metadata for downstream steps */
-export interface EvalStepResult {
-    /** The computed eval fingerprint (for publishing in provenance) */
-    evalFingerprint?: string;
-    /** Whether this result came from a remote cache hit */
-    remoteCacheHit?: boolean;
-    /** The step result */
-    stepResult: StepResult;
-}
-/** Options for the remote cache (Content Lake fingerprint lookup) */
-export interface RemoteCacheOptions {
-    /** Whether this is a debug run (debug runs don't use remote cache) */
-    debug?: boolean;
-    /** Filter options used for fingerprint computation */
-    filter?: FilterOptions;
-    /** Grader model identifier from models.yaml */
-    graderModel: string;
-    /** Disable remote cache lookup (--no-remote-cache) */
-    noRemoteCache?: boolean;
-    /** Sanity API token for reading cached reports */
-    sanityToken?: string;
-}
-export declare function buildFilterFlags(debug?: DebugOptions): string;
-/**
- * Extract the Promptfoo share URL from the eval results JSON.
- *
- * Promptfoo writes a `shareableUrl` field into the results file when
- * `PROMPTFOO_API_KEY` is set. This replaces the previous approach of
- * scraping the URL from a captured log file (which required piping
- * through `tee` and broke TTY progress reporting).
- */
-export declare function extractShareUrl(mode: ConcreteEvalMode): string | undefined;
-export declare function runEval(mode: ConcreteEvalMode, debug?: DebugOptions, concurrency?: number, noCache?: boolean, remoteCacheOpts?: RemoteCacheOptions): Promise<EvalStepResult>;