@sanity/ailf 0.1.0

package/dist/orchestration/pipeline-orchestrator.js

@@ -0,0 +1,153 @@
+/**
+ * Pipeline orchestrator — sequences steps and short-circuits on failure.
+ *
+ * This replaces the 1,672-line executePipeline() in pipeline-action.ts
+ * with declarative step sequencing. Each step is run through the
+ * StepRunner which provides uniform timing, precondition checking,
+ * error handling, and logging.
+ *
+ * When a jobId is present in the config (API-triggered evaluations),
+ * the orchestrator emits progress updates to the Content Lake after
+ * each step completes. This enables the GET /v1/jobs/:jobId polling
+ * endpoint to show real-time progress.
+ */
+import { runStep } from "./step-runner.js";
+// ---------------------------------------------------------------------------
+// Job progress reporter
+// ---------------------------------------------------------------------------
+/**
+ * Report job progress to the Content Lake via the report store's
+ * underlying Sanity client. Best-effort — failures are logged and
+ * never block the pipeline.
+ */
+async function reportJobProgress(ctx, stepName, completedSteps, totalSteps, status, errorInfo) {
+    const jobId = ctx.config.jobId;
+    if (!jobId)
+        return;
+    // Use the report store's write capability to patch the job document.
+    // The report store exposes a Sanity client — we access it through
+    // a best-effort PATCH via the same client infrastructure.
+    try {
+        // Dynamic import to avoid circular deps — the job store is a
+        // lightweight module that only needs a Sanity token.
+        const { JobStore } = await import("../job-store.js");
+        const store = new JobStore({
+            token: process.env.AILF_REPORT_SANITY_API_TOKEN ??
+                process.env.SANITY_API_TOKEN ??
+                undefined,
+        });
+        const update = {
+            status,
+            progress: {
+                currentStep: stepName,
+                completedSteps,
+                totalSteps,
+            },
+        };
+        if (status === "running" && completedSteps === 1) {
+            update.startedAt = new Date().toISOString();
+        }
+        if (errorInfo) {
+            update.error = errorInfo;
+            update.completedAt = new Date().toISOString();
+        }
+        await store.updateJob(jobId, update);
+    }
+    catch {
+        // Best effort — progress reporting should never block the pipeline
+        ctx.logger.warn(`Failed to report job progress for step "${stepName}" — continuing`);
+    }
+}
+// ---------------------------------------------------------------------------
+// Orchestrator
+// ---------------------------------------------------------------------------
+/**
+ * Run a sequence of pipeline steps, short-circuiting on required step failure.
+ *
+ * Optional steps (step.optional === true) may fail without stopping
+ * the pipeline. Required step failures cause an immediate abort.
+ *
+ * When ctx.config.jobId is set, emits progress updates to the Content
+ * Lake after each step completes.
+ */
+export async function orchestratePipeline(ctx, steps) {
+    const results = {};
+    const state = {};
+    const validation = { issues: [], valid: true };
+    const pipelineStart = Date.now();
+    const hasJob = !!ctx.config.jobId;
+    ctx.logger.section("ai-literacy-framework — Evaluation Pipeline");
+    // Report initial running status
+    if (hasJob) {
+        await reportJobProgress(ctx, steps[0]?.name ?? "init", 0, steps.length, "running");
+    }
+    for (let i = 0; i < steps.length; i++) {
+        const step = steps[i];
+        ctx.logger.section(step.name);
+        // Report current step progress
+        if (hasJob) {
+            await reportJobProgress(ctx, step.name, i, steps.length, "running");
+        }
+        const result = await runStep(step, ctx, state);
+        results[step.name] = result;
+        // Collect validation issues from the validate step
+        if (step.name === "validate" && result.status === "failed") {
+            validation.valid = false;
+        }
+        // Fail fast on required step failure
+        if (result.status === "failed" && !step.optional) {
+            ctx.logger.error(`Pipeline aborted: ${step.name} failed`);
+            // Report failure to job store
+            if (hasJob) {
+                const errorMsg = result.status === "failed" ? result.error : `${step.name} failed`;
+                await reportJobProgress(ctx, step.name, i + 1, steps.length, "failed", {
+                    message: errorMsg,
+                    step: step.name,
+                });
+            }
+            return {
+                durationMs: Date.now() - pipelineStart,
+                steps: results,
+                success: false,
+                validation,
+            };
+        }
+        // Report step completion
+        if (hasJob) {
+            await reportJobProgress(ctx, step.name, i + 1, steps.length, "running");
+        }
+    }
+    const durationMs = Date.now() - pipelineStart;
+    ctx.logger.section("Pipeline Complete");
+    ctx.logger.info(`All steps completed in ${durationMs}ms`);
+    // Report completion to job store (with reportId from state if available)
+    if (hasJob) {
+        try {
+            const { JobStore } = await import("../job-store.js");
+            const store = new JobStore({
+                token: process.env.AILF_REPORT_SANITY_API_TOKEN ??
+                    process.env.SANITY_API_TOKEN ??
+                    undefined,
+            });
+            await store.updateJob(ctx.config.jobId, {
+                status: "completed",
+                completedAt: new Date().toISOString(),
+                progress: {
+                    currentStep: "complete",
+                    completedSteps: steps.length,
+                    totalSteps: steps.length,
+                },
+                ...(state.reportId ? { reportId: state.reportId } : {}),
+            });
+        }
+        catch {
+            ctx.logger.warn("Failed to report job completion — continuing");
+        }
+    }
+    return {
+        durationMs,
+        steps: results,
+        success: true,
+        validation,
+    };
+}

package/dist/orchestration/step-runner.d.ts

@@ -0,0 +1,20 @@
+/**
+ * Uniform step execution with timing, logging, precondition checking,
+ * and cache integration.
+ *
+ * The StepRunner is the single place where cache lookup/record happens.
+ * Individual steps declare their cache inputs via cacheInputs(); the
+ * runner handles the rest.
+ */
+import type { AppContext, PipelineState, PipelineStep, StepResult } from "../_vendor/ailf-core/index.d.ts";
+/**
+ * Execute a single pipeline step with uniform middleware.
+ *
+ * Order of operations:
+ * 1. Precondition check → fail fast on errors
+ * 2. Cache lookup (if ctx.cache + step.cacheInputs + !noCache)
+ * 3. Execute step logic
+ * 4. Cache record (on success)
+ * 5. Log result
+ */
+export declare function runStep(step: PipelineStep, ctx: AppContext, state?: PipelineState): Promise<StepResult>;

package/dist/orchestration/step-runner.js

@@ -0,0 +1,88 @@
+/**
+ * Uniform step execution with timing, logging, precondition checking,
+ * and cache integration.
+ *
+ * The StepRunner is the single place where cache lookup/record happens.
+ * Individual steps declare their cache inputs via cacheInputs(); the
+ * runner handles the rest.
+ */
+/**
+ * Execute a single pipeline step with uniform middleware.
+ *
+ * Order of operations:
+ * 1. Precondition check → fail fast on errors
+ * 2. Cache lookup (if ctx.cache + step.cacheInputs + !noCache)
+ * 3. Execute step logic
+ * 4. Cache record (on success)
+ * 5. Log result
+ */
+export async function runStep(step, ctx, state = {}) {
+    const start = Date.now();
+    // 1. Precondition check
+    const issues = step.check(ctx);
+    const errors = issues.filter((i) => i.severity === "error");
+    if (errors.length > 0) {
+        const result = {
+            durationMs: Date.now() - start,
+            error: `Precondition failed: ${errors.map((e) => e.message).join("; ")}`,
+            status: "failed",
+        };
+        ctx.logger.step(step.name, result);
+        return result;
+    }
+    // 2. Cache lookup
+    const canCache = ctx.cache && !ctx.config.noCache && typeof step.cacheInputs === "function";
+    let cacheKey;
+    if (canCache) {
+        try {
+            const inputs = step.cacheInputs(ctx);
+            const key = await ctx.cache.computeKey(inputs);
+            cacheKey = key;
+            const cached = await ctx.cache.lookup(step.name, key);
+            if (cached.hit) {
+                const result = {
+                    durationMs: Date.now() - start,
+                    status: "success",
+                    summary: `Skipped (cached) — ${cached.entry.summary}`,
+                };
+                ctx.logger.step(step.name, result);
+                return result;
+            }
+        }
+        catch {
+            // Cache lookup failure is non-fatal — proceed to execute
+        }
+    }
+    // 3. Execute
+    try {
+        const result = await step.execute(ctx, state);
+        // Ensure timing is consistent (step may or may not track its own duration)
+        const timed = result.status === "skipped"
+            ? result
+            : { ...result, durationMs: Date.now() - start };
+        ctx.logger.step(step.name, timed);
+        // 4. Cache record (on success)
+        if (canCache && cacheKey && timed.status === "success") {
+            try {
+                await ctx.cache.record(step.name, cacheKey, {
+                    durationMs: timed.durationMs,
+                    outputPaths: [],
+                    summary: timed.status === "success" ? (timed.summary ?? step.name) : "",
+                });
+            }
+            catch {
+                // Cache record failure is non-fatal
+            }
+        }
+        return timed;
+    }
+    catch (err) {
+        const result = {
+            durationMs: Date.now() - start,
+            error: err instanceof Error ? err.message : String(err),
+            status: "failed",
+        };
+        ctx.logger.step(step.name, result);
+        return result;
+    }
+}

package/dist/orchestration/steps/calculate-scores-step.d.ts

@@ -0,0 +1,13 @@
+/**
+ * Pipeline step: Calculate AI Literacy Scores from eval results.
+ *
+ * Calls calculateAndWriteScores() from pipeline/calculate-scores.ts with
+ * typed options derived from AppContext. No env bridge needed.
+ */
+import type { AppContext, PipelineStep, StepResult, ValidationIssue } from "../../_vendor/ailf-core/index.d.ts";
+export declare class CalculateScoresStep implements PipelineStep {
+    readonly name = "calculate-scores";
+    check(): ValidationIssue[];
+    execute(ctx: AppContext): Promise<StepResult>;
+    cacheInputs(ctx: AppContext): string[];
+}

package/dist/orchestration/steps/calculate-scores-step.js

@@ -0,0 +1,95 @@
+/**
+ * Pipeline step: Calculate AI Literacy Scores from eval results.
+ *
+ * Calls calculateAndWriteScores() from pipeline/calculate-scores.ts with
+ * typed options derived from AppContext. No env bridge needed.
+ */
+import { join } from "path";
+import { getStepInputPaths } from "../../pipeline/cache.js";
+import { calculateAndWriteScores } from "../../pipeline/calculate-scores.js";
+import { checkResultsExist, checkScoreSummaryValid, } from "../../pipeline/checks.js";
+import { RESULTS_FILES } from "../../pipeline/eval-constants.js";
+import { loadSource } from "../../sources.js";
+import { configToSourceOverrides } from "../config-to-source-overrides.js";
+export class CalculateScoresStep {
+    name = "calculate-scores";
+    check() {
+        return [];
+    }
+    async execute(ctx) {
+        const start = Date.now();
+        const primaryMode = ctx.config.mode === "full"
+            ? "baseline"
+            : ctx.config.mode;
+        const resultsFile = RESULTS_FILES[primaryMode];
+        // Precondition: results file exists
+        const resultsIssues = checkResultsExist(ctx.config.rootDir, resultsFile);
+        const resultsErrors = resultsIssues.filter((i) => i.severity === "error");
+        if (resultsErrors.length > 0) {
+            return {
+                durationMs: Date.now() - start,
+                error: `Results missing: ${resultsErrors.map((e) => e.message).join("; ")}`,
+                status: "failed",
+            };
+        }
+        // Resolve source once with typed overrides
+        const overrides = configToSourceOverrides(ctx.config);
+        let resolvedSource;
+        try {
+            resolvedSource = loadSource(ctx.config.source, overrides);
+        }
+        catch {
+            // Non-fatal — proceed without source metadata
+        }
+        try {
+            calculateAndWriteScores({
+                allowedOrigins: ctx.config.allowedOrigins,
+                mode: ctx.config.mode,
+                resolvedSource,
+                resultsPath: primaryMode !== "baseline"
+                    ? join(ctx.config.rootDir, resultsFile)
+                    : undefined,
+                rootDir: ctx.config.rootDir,
+                searchMode: ctx.config.searchMode,
+                source: ctx.config.source,
+            });
+        }
+        catch (err) {
+            const code = err !== null && typeof err === "object" && "status" in err
+                ? err.status
+                : undefined;
+            if (code !== undefined && code !== 1) {
+                return {
+                    durationMs: Date.now() - start,
+                    error: `calculate-scores failed with exit code ${code}`,
+                    status: "failed",
+                };
+            }
+            if (code === undefined) {
+                return {
+                    durationMs: Date.now() - start,
+                    error: `calculate-scores failed: ${err instanceof Error ? err.message : String(err)}`,
+                    status: "failed",
+                };
+            }
+        }
+        // Postcondition: score summary exists and is valid
+        const summaryIssues = checkScoreSummaryValid(ctx.config.rootDir);
+        const summaryErrors = summaryIssues.filter((i) => i.severity === "error");
+        if (summaryErrors.length > 0) {
+            return {
+                durationMs: Date.now() - start,
+                error: `Postcondition failed: ${summaryErrors.map((e) => e.message).join("; ")}`,
+                status: "failed",
+            };
+        }
+        return {
+            durationMs: Date.now() - start,
+            status: "success",
+            summary: "Scores calculated and summary written",
+        };
+    }
+    cacheInputs(ctx) {
+        return getStepInputPaths(ctx.config.rootDir, "calculate-scores");
+    }
+}

package/dist/orchestration/steps/callback-step.d.ts

@@ -0,0 +1,24 @@
+/**
+ * Pipeline step: Deliver results to a callback URL.
+ *
+ * After the pipeline completes and the report is published, this step
+ * POSTs the results to the caller's callback URL. Used for API-triggered
+ * evaluations where the caller wants push-based result delivery.
+ *
+ * This step is always optional — callback failure never blocks the pipeline.
+ * The result is already in the Content Lake (system of record).
+ *
+ * @see packages/eval/src/pipeline/callback-delivery.ts
+ * @see docs/design-docs/api-service-gateway.md
+ */
+import type { AppContext, PipelineState, PipelineStep, StepResult, ValidationIssue } from "../../_vendor/ailf-core/index.d.ts";
+import { type CallbackConfig } from "../../pipeline/callback-delivery.js";
+export declare class CallbackStep implements PipelineStep {
+    private readonly callback;
+    private readonly jobId?;
+    readonly name = "callback-delivery";
+    readonly optional = true;
+    constructor(callback: CallbackConfig, jobId?: string | undefined);
+    check(): ValidationIssue[];
+    execute(ctx: AppContext, state: PipelineState): Promise<StepResult>;
+}

package/dist/orchestration/steps/callback-step.js

@@ -0,0 +1,76 @@
+/**
+ * Pipeline step: Deliver results to a callback URL.
+ *
+ * After the pipeline completes and the report is published, this step
+ * POSTs the results to the caller's callback URL. Used for API-triggered
+ * evaluations where the caller wants push-based result delivery.
+ *
+ * This step is always optional — callback failure never blocks the pipeline.
+ * The result is already in the Content Lake (system of record).
+ *
+ * @see packages/eval/src/pipeline/callback-delivery.ts
+ * @see docs/design-docs/api-service-gateway.md
+ */
+import { readFileSync } from "fs";
+import { resolve } from "path";
+import { deliverCallback, } from "../../pipeline/callback-delivery.js";
+export class CallbackStep {
+    callback;
+    jobId;
+    name = "callback-delivery";
+    optional = true;
+    constructor(callback, jobId) {
+        this.callback = callback;
+        this.jobId = jobId;
+    }
+    check() {
+        const issues = [];
+        if (!this.callback.url) {
+            issues.push({
+                message: "Callback URL is required",
+                severity: "error",
+                source: "callback-delivery",
+            });
+        }
+        return issues;
+    }
+    async execute(ctx, state) {
+        const start = Date.now();
+        const { rootDir } = ctx.config;
+        // Read score summary
+        let summary;
+        try {
+            const summaryPath = resolve(rootDir, "results", "latest", "score-summary.json");
+            summary = JSON.parse(readFileSync(summaryPath, "utf-8"));
+        }
+        catch (err) {
+            return {
+                durationMs: Date.now() - start,
+                error: `Failed to read score-summary.json for callback: ${err instanceof Error ? err.message : String(err)}`,
+                status: "failed",
+            };
+        }
+        // Deliver callback — read reportId from pipeline state (set by PublishReportStep)
+        ctx.logger.info(`Delivering results to ${this.callback.url}`);
+        const result = await deliverCallback(this.callback, {
+            deliveredAt: new Date().toISOString(),
+            jobId: this.jobId,
+            reportId: state.reportId,
+            summary,
+        });
+        if (result.ok) {
+            return {
+                durationMs: Date.now() - start,
+                status: "success",
+                summary: `Callback delivered to ${this.callback.url} (${result.attempts} attempt${result.attempts === 1 ? "" : "s"})`,
+            };
+        }
+        // Callback failure is not critical — log warning
+        ctx.logger.warn(`Callback delivery failed after ${result.attempts} attempts: ${result.error}`);
+        return {
+            durationMs: Date.now() - start,
+            error: `Callback delivery failed: ${result.error}`,
+            status: "failed",
+        };
+    }
+}

package/dist/orchestration/steps/compare-step.d.ts

@@ -0,0 +1,14 @@
+/**
+ * Pipeline step: Compare against baseline scores.
+ *
+ * This step is already pure (no execSync, no env vars) — the logic is
+ * inlined directly from the former pipeline/steps/compare-step.ts.
+ * This is an optional step — failure doesn't stop the pipeline.
+ */
+import type { AppContext, PipelineStep, StepResult, ValidationIssue } from "../../_vendor/ailf-core/index.d.ts";
+export declare class CompareStep implements PipelineStep {
+    readonly name = "compare";
+    readonly optional = true;
+    check(): ValidationIssue[];
+    execute(ctx: AppContext): Promise<StepResult>;
+}

package/dist/orchestration/steps/compare-step.js

@@ -0,0 +1,92 @@
+/**
+ * Pipeline step: Compare against baseline scores.
+ *
+ * This step is already pure (no execSync, no env vars) — the logic is
+ * inlined directly from the former pipeline/steps/compare-step.ts.
+ * This is an optional step — failure doesn't stop the pipeline.
+ */
+import { existsSync, readFileSync, readdirSync, writeFileSync } from "fs";
+import { join, resolve } from "path";
+import { compare } from "../../pipeline/compare.js";
+export class CompareStep {
+    name = "compare";
+    optional = true;
+    check() {
+        return [];
+    }
+    async execute(ctx) {
+        const start = Date.now();
+        const { rootDir } = ctx.config;
+        const scoreSummaryPath = resolve(rootDir, "results", "latest", "score-summary.json");
+        if (!existsSync(scoreSummaryPath)) {
+            return {
+                durationMs: Date.now() - start,
+                error: "score-summary.json not found. Run calculate-scores first.",
+                status: "failed",
+            };
+        }
+        // Load experiment (current run)
+        const experiment = JSON.parse(readFileSync(scoreSummaryPath, "utf-8"));
+        // Resolve baseline
+        let resolvedBaselinePath;
+        if (ctx.config.compareBaseline) {
+            resolvedBaselinePath = resolve(ctx.config.compareBaseline);
+        }
+        else {
+            const baselinesDir = resolve(rootDir, "results", "baselines");
+            if (!existsSync(baselinesDir)) {
+                return {
+                    reason: "No baselines directory found. Run 'pnpm baseline:save' first.",
+                    status: "skipped",
+                };
+            }
+            const files = readdirSync(baselinesDir)
+                .filter((f) => f.endsWith(".json"))
+                .sort()
+                .reverse();
+            if (files.length === 0) {
+                return {
+                    reason: "No baseline files found. Run 'pnpm baseline:save' first.",
+                    status: "skipped",
+                };
+            }
+            resolvedBaselinePath = join(baselinesDir, files[0]);
+        }
+        if (!existsSync(resolvedBaselinePath)) {
+            return {
+                durationMs: Date.now() - start,
+                error: `Baseline file not found: ${resolvedBaselinePath}`,
+                status: "failed",
+            };
+        }
+        const baseline = JSON.parse(readFileSync(resolvedBaselinePath, "utf-8"));
+        // Run comparison
+        const options = ctx.config.compareThreshold
+            ? { noiseThreshold: ctx.config.compareThreshold }
+            : undefined;
+        const report = compare(baseline, experiment, options);
+        // Write report
+        const reportPath = resolve(rootDir, "results", "latest", "comparison-report.json");
+        writeFileSync(reportPath, JSON.stringify(report, null, 2));
+        // Build summary
+        const improved = report.improved.length;
+        const regressed = report.regressed.length;
+        const unchanged = report.unchanged.length;
+        const overallDelta = report.deltas.overall;
+        const deltaStr = overallDelta > 0
+            ? `+${Math.round(overallDelta)}`
+            : String(Math.round(overallDelta));
+        const parts = [`Overall: ${deltaStr}`];
+        if (improved > 0)
+            parts.push(`${improved} improved`);
+        if (regressed > 0)
+            parts.push(`${regressed} regressed`);
+        if (unchanged > 0)
+            parts.push(`${unchanged} unchanged`);
+        return {
+            durationMs: Date.now() - start,
+            status: "success",
+            summary: parts.join(", "),
+        };
+    }
+}

package/dist/orchestration/steps/discovery-report-step.d.ts

@@ -0,0 +1,13 @@
+/**
+ * Pipeline step: Discovery report (agent discoverability analysis).
+ *
+ * Calls pure functions from pipeline/discovery-report.ts directly.
+ * Optional step — failure doesn't stop the pipeline.
+ */
+import type { AppContext, PipelineStep, StepResult, ValidationIssue } from "../../_vendor/ailf-core/index.d.ts";
+export declare class DiscoveryReportStep implements PipelineStep {
+    readonly name = "discovery-report";
+    readonly optional = true;
+    check(): ValidationIssue[];
+    execute(ctx: AppContext): Promise<StepResult>;
+}

package/dist/orchestration/steps/discovery-report-step.js

@@ -0,0 +1,55 @@
+/**
+ * Pipeline step: Discovery report (agent discoverability analysis).
+ *
+ * Calls pure functions from pipeline/discovery-report.ts directly.
+ * Optional step — failure doesn't stop the pipeline.
+ */
+import { existsSync, readFileSync, writeFileSync } from "fs";
+import { resolve } from "path";
+import { formatDiscoveryMarkdown, generateDiscoveryReport, } from "../../pipeline/discovery-report.js";
+export class DiscoveryReportStep {
+    name = "discovery-report";
+    optional = true;
+    check() {
+        return [];
+    }
+    async execute(ctx) {
+        const root = ctx.config.rootDir;
+        const start = Date.now();
+        try {
+            const scoreSummaryPath = resolve(root, "results", "latest", "score-summary.json");
+            if (!existsSync(scoreSummaryPath)) {
+                return {
+                    durationMs: Date.now() - start,
+                    error: "score-summary.json not found",
+                    status: "failed",
+                };
+            }
+            const scoreSummary = JSON.parse(readFileSync(scoreSummaryPath, "utf-8"));
+            if (!scoreSummary.retrievalMetrics) {
+                return {
+                    status: "skipped",
+                    reason: "No retrieval metrics in score summary — run an agentic evaluation first",
+                };
+            }
+            const report = generateDiscoveryReport(scoreSummary, ctx.config.areas);
+            const md = formatDiscoveryMarkdown(report);
+            writeFileSync(resolve(root, "results", "latest", "discovery-report.md"), md);
+            console.log(md);
+            const invisible = report.invisibleDocs.length;
+            const f1 = report.overall.avgF1.toFixed(2);
+            return {
+                durationMs: Date.now() - start,
+                status: "success",
+                summary: `F1=${f1}, ${invisible} invisible doc${invisible === 1 ? "" : "s"}, ${report.recommendations.length} recommendation${report.recommendations.length === 1 ? "" : "s"}`,
+            };
+        }
+        catch (err) {
+            return {
+                durationMs: Date.now() - start,
+                error: err instanceof Error ? err.message : String(err),
+                status: "failed",
+            };
+        }
+    }
+}

package/dist/orchestration/steps/fetch-docs-shell.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Shell delegation for the fetch-docs step.
+ *
+ * Isolates the execSync call so it can be replaced when the pipeline
+ * fully migrates to the DocFetcher port.
+ */
+export interface ShellResult {
+    ok: boolean;
+    error?: string;
+}
+/**
+ * Run `pnpm fetch-docs` via shell.
+ *
+ * Returns a result object instead of throwing so the step can
+ * handle the failure uniformly.
+ */
+export declare function runFetchDocsShell(rootDir: string, source?: string): ShellResult;