@sanity/ailf 0.1.0

package/dist/pipeline/callback-delivery.d.ts

@@ -0,0 +1,50 @@
+/**
+ * pipeline/callback-delivery.ts
+ *
+ * Delivers evaluation results to a callback URL with HMAC signature.
+ * Used by the API-triggered pipeline flow: after the report is published
+ * to the Content Lake (system of record), the result is also POSTed
+ * to the caller's callback URL.
+ *
+ * Fire-and-forget with 3 retries and exponential backoff. If callback
+ * fails, log a warning — the result is still in the Content Lake.
+ *
+ * @see docs/design-docs/api-service-gateway.md
+ */
+import type { CallbackPayload } from "../_vendor/ailf-core/index.d.ts";
+export type { CallbackPayload } from "../_vendor/ailf-core/index.d.ts";
+export interface CallbackConfig {
+    /** Optional custom headers to include in the callback request */
+    headers?: Record<string, string>;
+    /** The callback URL to POST results to */
+    url: string;
+}
+/**
+ * Generate an HMAC-SHA256 signature for the callback payload.
+ *
+ * The signature is computed over the raw JSON body using the callback URL
+ * as the signing key. This allows the receiver to verify the request
+ * originated from AILF without requiring a pre-shared secret.
+ *
+ * The receiver should:
+ * 1. Read the raw request body
+ * 2. Compute HMAC-SHA256(body, callbackUrl)
+ * 3. Compare with the X-AILF-Signature header
+ */
+export declare function computeSignature(body: string, signingKey: string): string;
+/**
+ * Deliver evaluation results to a callback URL.
+ *
+ * POSTs the payload as JSON with:
+ * - Content-Type: application/json
+ * - X-AILF-Signature: HMAC-SHA256 signature
+ * - Any custom headers from the callback config
+ *
+ * Retries up to 3 times with exponential backoff (1s → 2s → 4s).
+ * Returns success/failure status — never throws.
+ */
+export declare function deliverCallback(callback: CallbackConfig, payload: CallbackPayload): Promise<{
+    ok: boolean;
+    error?: string;
+    attempts: number;
+}>;

package/dist/pipeline/callback-delivery.js

@@ -0,0 +1,89 @@
+/**
+ * pipeline/callback-delivery.ts
+ *
+ * Delivers evaluation results to a callback URL with HMAC signature.
+ * Used by the API-triggered pipeline flow: after the report is published
+ * to the Content Lake (system of record), the result is also POSTed
+ * to the caller's callback URL.
+ *
+ * Fire-and-forget with 3 retries and exponential backoff. If callback
+ * fails, log a warning — the result is still in the Content Lake.
+ *
+ * @see docs/design-docs/api-service-gateway.md
+ */
+import { createHmac } from "crypto";
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+const MAX_RETRIES = 3;
+const BASE_DELAY_MS = 1000;
+const CALLBACK_TIMEOUT_MS = 10_000;
+// ---------------------------------------------------------------------------
+// HMAC Signature
+// ---------------------------------------------------------------------------
+/**
+ * Generate an HMAC-SHA256 signature for the callback payload.
+ *
+ * The signature is computed over the raw JSON body using the callback URL
+ * as the signing key. This allows the receiver to verify the request
+ * originated from AILF without requiring a pre-shared secret.
+ *
+ * The receiver should:
+ * 1. Read the raw request body
+ * 2. Compute HMAC-SHA256(body, callbackUrl)
+ * 3. Compare with the X-AILF-Signature header
+ */
+export function computeSignature(body, signingKey) {
+    return createHmac("sha256", signingKey).update(body).digest("hex");
+}
+// ---------------------------------------------------------------------------
+// Delivery
+// ---------------------------------------------------------------------------
+/**
+ * Deliver evaluation results to a callback URL.
+ *
+ * POSTs the payload as JSON with:
+ * - Content-Type: application/json
+ * - X-AILF-Signature: HMAC-SHA256 signature
+ * - Any custom headers from the callback config
+ *
+ * Retries up to 3 times with exponential backoff (1s → 2s → 4s).
+ * Returns success/failure status — never throws.
+ */
+export async function deliverCallback(callback, payload) {
+    const body = JSON.stringify(payload);
+    const signature = computeSignature(body, callback.url);
+    let lastError;
+    let attempts = 0;
+    for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
+        attempts = attempt + 1;
+        try {
+            const controller = new AbortController();
+            const timeout = setTimeout(() => controller.abort(), CALLBACK_TIMEOUT_MS);
+            const response = await fetch(callback.url, {
+                method: "POST",
+                headers: {
+                    "Content-Type": "application/json",
+                    "X-AILF-Signature": signature,
+                    ...(callback.headers ?? {}),
+                },
+                body,
+                signal: controller.signal,
+            });
+            clearTimeout(timeout);
+            if (response.ok) {
+                return { ok: true, attempts };
+            }
+            lastError = `HTTP ${response.status}: ${response.statusText}`;
+        }
+        catch (err) {
+            lastError = err instanceof Error ? err.message : String(err);
+        }
+        // Don't delay after the last attempt
+        if (attempt < MAX_RETRIES) {
+            const delayMs = BASE_DELAY_MS * Math.pow(2, attempt);
+            await new Promise((resolve) => setTimeout(resolve, delayMs));
+        }
+    }
+    return { ok: false, error: lastError, attempts };
+}

package/dist/pipeline/checks.d.ts

@@ -0,0 +1,39 @@
+/**
+ * pipeline/checks.ts
+ *
+ * Pre/postcondition checking for pipeline steps.
+ * Each function verifies filesystem or environment state and returns
+ * ValidationIssue[] — an empty array means all checks passed.
+ */
+import type { ValidationIssue } from "./types.js";
+/**
+ * Check that `contexts/canonical/<taskId>.md` exists and is non-empty
+ * for every task ID.
+ */
+export declare function checkCanonicalContextsExist(rootDir: string, taskIds: string[]): ValidationIssue[];
+/**
+ * Check that `contexts/<area>.md` exists and is non-empty for every area.
+ * This is the precondition for the eval step — it cannot run without docs.
+ */
+export declare function checkContextsExist(rootDir: string, areas: string[]): ValidationIssue[];
+/**
+ * Check that required environment variables are set.
+ * Loads the root `.env` file first (with override, matching the dotenv CLI
+ * `-o` flag used by other scripts), then checks for required keys.
+ */
+export declare function checkEnvironment(rootDir: string): ValidationIssue[];
+/**
+ * Check that the baseline `promptfooconfig.yaml` exists. Optionally check
+ * for `promptfooconfig.observed.yaml` and `promptfooconfig.agentic.yaml`.
+ */
+export declare function checkGeneratedConfigsExist(rootDir: string): ValidationIssue[];
+/**
+ * Check that the eval results JSON file exists, is valid JSON, and contains
+ * a `results` array.
+ */
+export declare function checkResultsExist(rootDir: string, resultsPath?: string): ValidationIssue[];
+/**
+ * Check that `results/latest/score-summary.json` exists, parses as JSON,
+ * contains feature area scores, and no scores are NaN or out of 0–100.
+ */
+export declare function checkScoreSummaryValid(rootDir: string): ValidationIssue[];

package/dist/pipeline/checks.js

@@ -0,0 +1,280 @@
+/**
+ * pipeline/checks.ts
+ *
+ * Pre/postcondition checking for pipeline steps.
+ * Each function verifies filesystem or environment state and returns
+ * ValidationIssue[] — an empty array means all checks passed.
+ */
+import { config as loadEnv } from "dotenv";
+import { existsSync, readFileSync, statSync } from "fs";
+import { join, resolve } from "path";
+// ---------------------------------------------------------------------------
+// Precondition: contexts exist for each feature area
+// ---------------------------------------------------------------------------
+/**
+ * Check that `contexts/canonical/<taskId>.md` exists and is non-empty
+ * for every task ID.
+ */
+export function checkCanonicalContextsExist(rootDir, taskIds) {
+    const issues = [];
+    const canonicalDir = resolve(rootDir, "contexts", "canonical");
+    for (const taskId of taskIds) {
+        const filePath = join(canonicalDir, `${taskId}.md`);
+        if (!existsSync(filePath)) {
+            issues.push({
+                message: `Missing canonical context for task "${taskId}". Run \`pnpm fetch-docs\` to generate it.`,
+                path: filePath,
+                severity: "error",
+                source: "checkCanonicalContextsExist",
+            });
+            continue;
+        }
+        const stat = statSync(filePath);
+        if (stat.size === 0) {
+            issues.push({
+                message: `Canonical context for task "${taskId}" is empty. Re-run \`pnpm fetch-docs\`.`,
+                path: filePath,
+                severity: "error",
+                source: "checkCanonicalContextsExist",
+            });
+        }
+    }
+    return issues;
+}
+// ---------------------------------------------------------------------------
+// Precondition: canonical context files exist per task
+// ---------------------------------------------------------------------------
+/**
+ * Check that `contexts/<area>.md` exists and is non-empty for every area.
+ * This is the precondition for the eval step — it cannot run without docs.
+ */
+export function checkContextsExist(rootDir, areas) {
+    const issues = [];
+    const contextsDir = resolve(rootDir, "contexts");
+    for (const area of areas) {
+        const filePath = join(contextsDir, `${area}.md`);
+        if (!existsSync(filePath)) {
+            issues.push({
+                message: `Missing context file for area "${area}". Run \`pnpm fetch-docs\` to generate it.`,
+                path: filePath,
+                severity: "error",
+                source: "checkContextsExist",
+            });
+            continue;
+        }
+        const stat = statSync(filePath);
+        if (stat.size === 0) {
+            issues.push({
+                message: `Context file for area "${area}" is empty. Re-run \`pnpm fetch-docs\`.`,
+                path: filePath,
+                severity: "error",
+                source: "checkContextsExist",
+            });
+        }
+    }
+    return issues;
+}
+// ---------------------------------------------------------------------------
+// Postcondition: eval results exist and are valid JSON
+// ---------------------------------------------------------------------------
+/**
+ * Check that required environment variables are set.
+ * Loads the root `.env` file first (with override, matching the dotenv CLI
+ * `-o` flag used by other scripts), then checks for required keys.
+ */
+export function checkEnvironment(rootDir) {
+    const issues = [];
+    // Load root .env so we see the same vars as dotenv -e ../../.env -o
+    const envPath = resolve(rootDir, "..", "..", ".env");
+    if (existsSync(envPath)) {
+        loadEnv({ override: true, path: envPath });
+    }
+    if (!process.env.OPENAI_API_KEY) {
+        issues.push({
+            message: "OPENAI_API_KEY is not set. Ensure it is defined in the root .env file.",
+            severity: "warning",
+            source: "checkEnvironment",
+        });
+    }
+    if (!process.env.SANITY_API_TOKEN) {
+        issues.push({
+            message: "SANITY_API_TOKEN is not set. It is needed for 'turbo fetch-docs'. Ensure it is defined in the root .env file.",
+            severity: "warning",
+            source: "checkEnvironment",
+        });
+    }
+    return issues;
+}
+// ---------------------------------------------------------------------------
+// Postcondition: score summary is valid
+// ---------------------------------------------------------------------------
+/**
+ * Check that the baseline `promptfooconfig.yaml` exists. Optionally check
+ * for `promptfooconfig.observed.yaml` and `promptfooconfig.agentic.yaml`.
+ */
+export function checkGeneratedConfigsExist(rootDir) {
+    const issues = [];
+    const baselinePath = resolve(rootDir, "promptfooconfig.yaml");
+    if (!existsSync(baselinePath)) {
+        issues.push({
+            message: "Baseline config 'promptfooconfig.yaml' not found. Run 'pnpm generate-configs'.",
+            path: baselinePath,
+            severity: "error",
+            source: "checkGeneratedConfigsExist",
+        });
+    }
+    const optionalConfigs = [
+        "promptfooconfig.observed.yaml",
+        "promptfooconfig.agentic.yaml",
+    ];
+    for (const name of optionalConfigs) {
+        const configPath = resolve(rootDir, name);
+        if (!existsSync(configPath)) {
+            issues.push({
+                message: `Optional config \`${name}\` not found. Run \`pnpm generate-configs\` to create it.`,
+                path: configPath,
+                severity: "warning",
+                source: "checkGeneratedConfigsExist",
+            });
+        }
+    }
+    return issues;
+}
+// ---------------------------------------------------------------------------
+// Precondition: generated promptfoo configs exist
+// ---------------------------------------------------------------------------
+/**
+ * Check that the eval results JSON file exists, is valid JSON, and contains
+ * a `results` array.
+ */
+export function checkResultsExist(rootDir, resultsPath) {
+    const issues = [];
+    const filePath = resolve(rootDir, resultsPath ?? join("results", "latest", "eval-results.json"));
+    if (!existsSync(filePath)) {
+        issues.push({
+            message: "Eval results file not found. Run 'pnpm eval' first.",
+            path: filePath,
+            severity: "error",
+            source: "checkResultsExist",
+        });
+        return issues;
+    }
+    let parsed;
+    try {
+        const raw = readFileSync(filePath, "utf-8");
+        parsed = JSON.parse(raw);
+    }
+    catch (err) {
+        issues.push({
+            message: `Eval results file is not valid JSON: ${err instanceof Error ? err.message : String(err)}`,
+            path: filePath,
+            severity: "error",
+            source: "checkResultsExist",
+        });
+        return issues;
+    }
+    if (typeof parsed !== "object" || parsed === null || !("results" in parsed)) {
+        issues.push({
+            message: 'Eval results file is missing the "results" key.',
+            path: filePath,
+            severity: "error",
+            source: "checkResultsExist",
+        });
+        return issues;
+    }
+    // Promptfoo wraps results in: { results: { results: [...], stats: {...} } }
+    const outer = parsed.results;
+    const resultsArray = Array.isArray(outer)
+        ? outer
+        : typeof outer === "object" &&
+            outer !== null &&
+            "results" in outer &&
+            Array.isArray(outer.results)
+            ? outer.results
+            : null;
+    if (!resultsArray) {
+        issues.push({
+            message: 'The "results" key in eval results is not an array (or results.results).',
+            path: filePath,
+            severity: "error",
+            source: "checkResultsExist",
+        });
+    }
+    return issues;
+}
+// ---------------------------------------------------------------------------
+// Environment variable checks
+// ---------------------------------------------------------------------------
+/**
+ * Check that `results/latest/score-summary.json` exists, parses as JSON,
+ * contains feature area scores, and no scores are NaN or out of 0–100.
+ */
+export function checkScoreSummaryValid(rootDir) {
+    const issues = [];
+    const filePath = resolve(rootDir, "results", "latest", "score-summary.json");
+    if (!existsSync(filePath)) {
+        issues.push({
+            message: "Score summary not found. Run 'pnpm report' to generate it.",
+            path: filePath,
+            severity: "error",
+            source: "checkScoreSummaryValid",
+        });
+        return issues;
+    }
+    let parsed;
+    try {
+        const raw = readFileSync(filePath, "utf-8");
+        parsed = JSON.parse(raw);
+    }
+    catch (err) {
+        issues.push({
+            message: `Score summary is not valid JSON: ${err instanceof Error ? err.message : String(err)}`,
+            path: filePath,
+            severity: "error",
+            source: "checkScoreSummaryValid",
+        });
+        return issues;
+    }
+    if (typeof parsed !== "object" || parsed === null) {
+        issues.push({
+            message: "Score summary is not a JSON object.",
+            path: filePath,
+            severity: "error",
+            source: "checkScoreSummaryValid",
+        });
+        return issues;
+    }
+    const record = parsed;
+    const scoreKeys = Object.keys(record);
+    if (scoreKeys.length === 0) {
+        issues.push({
+            message: "Score summary contains no feature area scores.",
+            path: filePath,
+            severity: "error",
+            source: "checkScoreSummaryValid",
+        });
+        return issues;
+    }
+    for (const key of scoreKeys) {
+        const value = record[key];
+        if (typeof value === "number") {
+            if (Number.isNaN(value)) {
+                issues.push({
+                    message: `Score for "${key}" is NaN.`,
+                    path: filePath,
+                    severity: "error",
+                    source: "checkScoreSummaryValid",
+                });
+            }
+            else if (value < 0 || value > 100) {
+                issues.push({
+                    message: `Score for "${key}" is ${value}, which is outside the 0–100 range.`,
+                    path: filePath,
+                    severity: "warning",
+                    source: "checkScoreSummaryValid",
+                });
+            }
+        }
+    }
+    return issues;
+}

package/dist/pipeline/classify-url.d.ts

@@ -0,0 +1,61 @@
+/**
+ * classify-url.ts
+ *
+ * Classifies URLs passed via --url/--urls into specific source types.
+ * Enables intelligent inference: a Sanity Studio release URL automatically
+ * sets the perspective, a Studio document URL extracts the document ID, etc.
+ *
+ * Classification rules (applied in order):
+ *   1. /releases/<id>         → Sanity Release (extract perspective)
+ *   2. /structure/...;<uuid>  → Sanity Studio Document (extract doc ID + optional perspective)
+ *   3. everything else        → Direct URL (fetch as documentation)
+ */
+/** Discriminated union of all classified URL types */
+export type ClassifiedUrl = DirectUrl | SanityDocumentUrl | SanityReleaseUrl;
+/** A URL classified as a direct documentation page (HTML, Markdown, etc.) */
+export interface DirectUrl {
+    type: "direct-url";
+    url: string;
+}
+/** A URL classified as a Sanity Studio document URL */
+export interface SanityDocumentUrl {
+    documentId: string;
+    perspectiveId?: string;
+    studioOrigin: string;
+    type: "sanity-document";
+}
+/** A URL classified as a Sanity Studio release URL */
+export interface SanityReleaseUrl {
+    perspectiveId: string;
+    studioOrigin: string;
+    type: "sanity-release";
+}
+/** Result of classifying multiple URLs */
+export interface ClassificationResult {
+    /** URLs classified as direct documentation pages */
+    directUrls: string[];
+    /** Sanity document IDs extracted from Studio document URLs */
+    documentIds: string[];
+    /** Perspective ID inferred from release or document URLs (first wins) */
+    inferredPerspective?: string;
+    /** Studio origin inferred from Sanity URLs (first wins) */
+    inferredStudioOrigin?: string;
+}
+/**
+ * Classify a single URL into one of the known source types.
+ *
+ * @param input - A URL string to classify
+ * @returns A discriminated union describing the URL type and extracted metadata
+ * @throws If the input is not a valid URL
+ */
+export declare function classifyUrl(input: string): ClassifiedUrl;
+/**
+ * Classify an array of URLs and aggregate the results.
+ *
+ * Collects direct URLs and document IDs into separate lists. Perspective
+ * and studio origin are inferred from the first Sanity URL that provides them.
+ *
+ * @param urls - Array of URL strings to classify
+ * @returns Aggregated classification result
+ */
+export declare function classifyUrls(urls: string[]): ClassificationResult;

package/dist/pipeline/classify-url.js

@@ -0,0 +1,93 @@
+/**
+ * classify-url.ts
+ *
+ * Classifies URLs passed via --url/--urls into specific source types.
+ * Enables intelligent inference: a Sanity Studio release URL automatically
+ * sets the perspective, a Studio document URL extracts the document ID, etc.
+ *
+ * Classification rules (applied in order):
+ *   1. /releases/<id>         → Sanity Release (extract perspective)
+ *   2. /structure/...;<uuid>  → Sanity Studio Document (extract doc ID + optional perspective)
+ *   3. everything else        → Direct URL (fetch as documentation)
+ */
+// ---------------------------------------------------------------------------
+// Patterns
+// ---------------------------------------------------------------------------
+/** UUID v4 pattern (case-insensitive) */
+const UUID_PATTERN = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/i;
+/** Matches /releases/<perspective-id> at the end of a pathname */
+const RELEASE_PATH = /\/releases\/([\w-]+)$/;
+/**
+ * Matches /structure/...;<last-segment> where the last semicolon-delimited
+ * segment is extracted. The UUID check is applied separately.
+ */
+const STRUCTURE_PATH = /\/structure\/.+;([^;?]+)/;
+// ---------------------------------------------------------------------------
+// Batch classification
+// ---------------------------------------------------------------------------
+/**
+ * Classify a single URL into one of the known source types.
+ *
+ * @param input - A URL string to classify
+ * @returns A discriminated union describing the URL type and extracted metadata
+ * @throws If the input is not a valid URL
+ */
+export function classifyUrl(input) {
+    const parsed = new URL(input);
+    // Rule 1: Sanity Studio release URL — /releases/<perspective-id>
+    const releaseMatch = parsed.pathname.match(RELEASE_PATH);
+    if (releaseMatch) {
+        return {
+            perspectiveId: releaseMatch[1],
+            studioOrigin: parsed.origin,
+            type: "sanity-release",
+        };
+    }
+    // Rule 2: Sanity Studio document URL — /structure/...;<uuid>
+    const structureMatch = parsed.pathname.match(STRUCTURE_PATH);
+    if (structureMatch && UUID_PATTERN.test(structureMatch[1])) {
+        return {
+            documentId: structureMatch[1],
+            perspectiveId: parsed.searchParams.get("perspective") ?? undefined,
+            studioOrigin: parsed.origin,
+            type: "sanity-document",
+        };
+    }
+    // Rule 3: Default — direct documentation URL
+    return { type: "direct-url", url: input };
+}
+/**
+ * Classify an array of URLs and aggregate the results.
+ *
+ * Collects direct URLs and document IDs into separate lists. Perspective
+ * and studio origin are inferred from the first Sanity URL that provides them.
+ *
+ * @param urls - Array of URL strings to classify
+ * @returns Aggregated classification result
+ */
+export function classifyUrls(urls) {
+    const result = {
+        directUrls: [],
+        documentIds: [],
+    };
+    for (const url of urls) {
+        const classified = classifyUrl(url);
+        switch (classified.type) {
+            case "direct-url":
+                result.directUrls.push(classified.url);
+                break;
+            case "sanity-document":
+                result.documentIds.push(classified.documentId);
+                if (classified.perspectiveId && !result.inferredPerspective) {
+                    result.inferredPerspective = classified.perspectiveId;
+                }
+                result.inferredStudioOrigin ??= classified.studioOrigin;
+                break;
+            case "sanity-release":
+                result.inferredPerspective ??= classified.perspectiveId;
+                result.inferredStudioOrigin ??= classified.studioOrigin;
+                break;
+        }
+    }
+    return result;
+}

package/dist/pipeline/compare.d.ts

@@ -0,0 +1,31 @@
+/**
+ * pipeline/compare.ts
+ *
+ * Core comparison primitive for the evaluation framework.
+ *
+ * Takes two ScoreSummary objects (baseline and experiment) and produces a
+ * structured ComparisonReport with overall, per-area, and per-dimension
+ * deltas, plus improved/regressed/unchanged classification.
+ *
+ * This is the single function that backs all comparison scenarios:
+ * doc improvement, model comparison, branch validation, etc.
+ * What varies is what produced each ScoreSummary — the comparison
+ * logic is always the same.
+ *
+ * @see docs/ideas/evaluation-roadmap.md — BP5: Make comparison a primitive
+ * @see docs/ideas/metrics-design.md — Tier 4: Comparison results
+ */
+import { type ChangeClass, type CompareOptions, type ComparisonReport, type ScoreSummary } from "./types.js";
+/** Classify a delta as improved, regressed, or unchanged given a threshold */
+export declare function classifyChange(delta: number, threshold: number): ChangeClass;
+/**
+ * Compare two evaluation score summaries and produce a structured report.
+ *
+ * This is a pure function — no side effects, no file I/O.
+ *
+ * @param baseline  The "before" or "control" score summary
+ * @param experiment  The "after" or "treatment" score summary
+ * @param options  Optional configuration (noise threshold, etc.)
+ * @returns A ComparisonReport with deltas, classifications, and breakdowns
+ */
+export declare function compare(baseline: ScoreSummary, experiment: ScoreSummary, options?: CompareOptions): ComparisonReport;