@sanity/ailf 0.1.0

package/dist/commands/validate.js

@@ -0,0 +1,73 @@
+/**
+ * validate command — checks pipeline configuration consistency.
+ *
+ * Validates all YAML files, task-to-mapping cross-references, reference
+ * solutions, context files, and environment variables.
+ *
+ */
+import { Command } from "commander";
+import { dirname, resolve } from "path";
+import { fileURLToPath } from "url";
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const ROOT = resolve(__dirname, "..", "..");
+export function createValidateCommand() {
+    return new Command("validate")
+        .description("Validate pipeline configuration")
+        .option("--strict", "Treat warnings as errors", false)
+        .option("--contexts", "Check that context files exist", false)
+        .action(async (opts) => {
+        const { validateConfiguration } = await import("../pipeline/validate.js");
+        const { checkContextsExist, checkEnvironment } = await import("../pipeline/checks.js");
+        console.log("=== ai-literacy-framework — Configuration Validator ===\n");
+        const result = validateConfiguration(ROOT);
+        // Optionally check contexts
+        if (opts.contexts) {
+            const { ALL_FEATURE_AREAS } = await import("../sanity/queries.js");
+            const contextIssues = checkContextsExist(ROOT, ALL_FEATURE_AREAS);
+            result.issues.push(...contextIssues);
+            result.valid =
+                result.valid && contextIssues.every((i) => i.severity !== "error");
+        }
+        // Check environment
+        const envIssues = checkEnvironment(ROOT);
+        result.issues.push(...envIssues);
+        // Report results
+        const errors = result.issues.filter((i) => i.severity === "error");
+        const warnings = result.issues.filter((i) => i.severity === "warning");
+        if (errors.length > 0) {
+            console.log(`❌ ${errors.length} error(s):\n`);
+            for (const issue of errors) {
+                console.log(`  ERROR [${issue.source}] ${issue.message}`);
+                if (issue.path)
+                    console.log(`        at ${issue.path}`);
+            }
+            console.log();
+        }
+        if (warnings.length > 0) {
+            console.log(`⚠️  ${warnings.length} warning(s):\n`);
+            for (const issue of warnings) {
+                console.log(`  WARN  [${issue.source}] ${issue.message}`);
+                if (issue.path)
+                    console.log(`        at ${issue.path}`);
+            }
+            console.log();
+        }
+        if (errors.length === 0 && warnings.length === 0) {
+            console.log("✅ All checks passed — configuration is valid.\n");
+        }
+        if (errors.length === 0 && warnings.length > 0) {
+            console.log(`✅ Configuration is valid (${warnings.length} warning(s)).\n`);
+        }
+        // In strict mode, warnings are treated as errors
+        const exitCode = opts.strict
+            ? result.issues.length > 0
+                ? 1
+                : 0
+            : errors.length > 0
+                ? 1
+                : 0;
+        if (exitCode !== 0) {
+            process.exit(exitCode);
+        }
+    });
+}

package/dist/commands/webhook-server.d.ts

@@ -0,0 +1,5 @@
+/**
+ * webhook-server command — local development server for webhook testing.
+ */
+import { Command } from "commander";
+export declare function createWebhookServerCommand(): Command;

package/dist/commands/webhook-server.js

@@ -0,0 +1,30 @@
+/**
+ * webhook-server command — local development server for webhook testing.
+ */
+import { dirname, resolve } from "path";
+import { fileURLToPath } from "url";
+import { Command } from "commander";
+import { startWebhookServer } from "../pipeline/webhook-server.js";
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const ROOT = resolve(__dirname, "..", "..");
+export function createWebhookServerCommand() {
+    return new Command("webhook-server")
+        .description("Start a local webhook development server for testing triggers")
+        .option("-p, --port <port>", "Server port", parseInt)
+        .action(async (opts) => {
+        try {
+            startWebhookServer({
+                dailyBudget: parseInt(process.env.WEBHOOK_DAILY_BUDGET ?? "20", 10),
+                debounceMs: parseInt(process.env.WEBHOOK_DEBOUNCE_MS ?? "10000", 10),
+                githubToken: process.env.GITHUB_TOKEN ?? "",
+                port: opts.port ?? parseInt(process.env.WEBHOOK_PORT ?? "3333", 10),
+                rootDir: ROOT,
+            });
+        }
+        catch (err) {
+            process.exitCode = 1;
+            if (err instanceof Error)
+                console.error(err.message);
+        }
+    });
+}

package/dist/commands/weekly-digest.d.ts

@@ -0,0 +1,10 @@
+/**
+ * weekly-digest command — generates and delivers a weekly evaluation digest.
+ *
+ * Queries the Sanity Content Lake for reports within the configured
+ * lookback window, computes trend analysis, and delivers the digest
+ * via Slack or stdout.
+ *
+ */
+import { Command } from "commander";
+export declare function createWeeklyDigestCommand(): Command;

package/dist/commands/weekly-digest.js

@@ -0,0 +1,104 @@
+/**
+ * weekly-digest command — generates and delivers a weekly evaluation digest.
+ *
+ * Queries the Sanity Content Lake for reports within the configured
+ * lookback window, computes trend analysis, and delivers the digest
+ * via Slack or stdout.
+ *
+ */
+import { Command } from "commander";
+export function createWeeklyDigestCommand() {
+    return new Command("weekly-digest")
+        .description("Generate and deliver a weekly evaluation digest")
+        .option("-n, --dry-run", "Print to stdout only, do not send to Slack", false)
+        .option("--lookback <days>", "Lookback window in days (default: from config or 7)", parseInt)
+        .option("--json", "Output raw JSON digest data", false)
+        .action(async (opts) => {
+        const { generateDigest } = await import("../schedules/digest.js");
+        const { getDigestConfig } = await import("../schedules/loader.js");
+        const { formatWeeklyDigest } = await import("../sinks/slack/format.js");
+        console.log();
+        console.log("=== AI Literacy Weekly Digest ===");
+        console.log();
+        // Load digest config
+        const digestConfig = getDigestConfig();
+        const lookbackDays = opts.lookback ?? digestConfig?.lookbackDays ?? 7;
+        console.log(`  Lookback: ${lookbackDays} days`);
+        console.log(`  Mode: ${opts.dryRun ? "dry run (stdout only)" : "live"}`);
+        console.log();
+        // Generate digest — uses AILF_REPORT_* env vars for report store access
+        const digest = await generateDigest({
+            dataset: process.env.AILF_REPORT_DATASET,
+            lookbackDays,
+            projectId: process.env.AILF_REPORT_PROJECT_ID,
+            token: process.env.AILF_REPORT_SANITY_API_TOKEN ??
+                process.env.SANITY_API_TOKEN,
+        });
+        if (!digest) {
+            console.log("  No reports found in the lookback window. Nothing to send.");
+            process.exit(0);
+        }
+        // Output summary
+        console.log(`  Reports found: ${digest.reportCount}`);
+        console.log(`  Overall: ${Math.round(digest.overallLatest)} (${digest.overallTrend})`);
+        console.log(`  Improved: ${digest.improved.join(", ") || "none"}`);
+        console.log(`  Regressed: ${digest.regressed.join(", ") || "none"}`);
+        console.log(`  Stable: ${digest.stable.join(", ") || "none"}`);
+        console.log();
+        if (opts.json) {
+            console.log(JSON.stringify(digest, null, 2));
+            return;
+        }
+        // Format for Slack
+        const message = formatWeeklyDigest(digest);
+        if (opts.dryRun) {
+            console.log("  --- Slack Message Preview ---");
+            console.log(`  Text: ${message.text}`);
+            console.log();
+            for (const block of message.blocks) {
+                if (block.text) {
+                    console.log(`  [${block.type}] ${block.text.text}`);
+                }
+                if (block.fields) {
+                    for (const field of block.fields) {
+                        console.log(`    [field] ${field.text}`);
+                    }
+                }
+                if (block.elements) {
+                    for (const el of block.elements) {
+                        console.log(`    [element] ${el.text}`);
+                    }
+                }
+            }
+            console.log();
+            return;
+        }
+        // Deliver via Slack
+        const webhookUrl = digestConfig?.slackWebhookUrl ?? process.env.SLACK_WEBHOOK_URL;
+        if (!webhookUrl) {
+            console.warn("  ⚠️  No Slack webhook URL configured. Set SLACK_WEBHOOK_URL or configure in schedules.yaml");
+            console.log("  Printing digest to stdout instead:");
+            console.log();
+            console.log(`  ${message.text}`);
+            return;
+        }
+        console.log("  Sending to Slack...");
+        try {
+            const response = await fetch(webhookUrl, {
+                body: JSON.stringify(message),
+                headers: { "Content-Type": "application/json" },
+                method: "POST",
+            });
+            if (response.ok) {
+                console.log("  ✅ Digest sent successfully");
+            }
+            else {
+                const text = await response.text();
+                console.warn(`  ⚠️  Slack delivery failed: ${response.status} ${text}`);
+            }
+        }
+        catch (error) {
+            console.warn(`  ⚠️  Slack delivery error: ${error instanceof Error ? error.message : String(error)}`);
+        }
+    });
+}

package/dist/composition-root.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Composition Root — the single place that knows about all concrete adapters.
+ *
+ * Creates a fully wired AppContext from resolved configuration.
+ * This function is called ONCE per CLI invocation, at the top of each
+ * command's action handler.
+ *
+ * Design principle: no DI container, no decorators, no reflection.
+ * Just a plain factory function that builds a plain object.
+ *
+ * Why this exists:
+ * - Before: each command constructs its own adapters (scattered coupling)
+ * - After: one factory, one place to change adapter wiring
+ *
+ * @see packages/core/src/ports/context.ts — AppContext interface
+ * @see docs/exec-plans/active/ports-and-adapters/phase-7-composition-root.md
+ */
+import type { AppContext, ResolvedConfig } from "./_vendor/ailf-core/index.d.ts";
+/**
+ * Create a fully wired AppContext from resolved configuration.
+ *
+ * Every adapter is constructed here and nowhere else (outside of tests).
+ * Swapping an adapter (e.g., YamlTaskSource → ContentLakeTaskSource)
+ * is a one-line change in this function.
+ */
+export declare function createAppContext(config: ResolvedConfig): AppContext;

package/dist/composition-root.js

@@ -0,0 +1,107 @@
+/**
+ * Composition Root — the single place that knows about all concrete adapters.
+ *
+ * Creates a fully wired AppContext from resolved configuration.
+ * This function is called ONCE per CLI invocation, at the top of each
+ * command's action handler.
+ *
+ * Design principle: no DI container, no decorators, no reflection.
+ * Just a plain factory function that builds a plain object.
+ *
+ * Why this exists:
+ * - Before: each command constructs its own adapters (scattered coupling)
+ * - After: one factory, one place to change adapter wiring
+ *
+ * @see packages/core/src/ports/context.ts — AppContext interface
+ * @see docs/exec-plans/active/ports-and-adapters/phase-7-composition-root.md
+ */
+import { ContentLakeCacheAdapter } from "./adapters/cache/content-lake-cache.js";
+import { FilesystemCache } from "./adapters/cache/filesystem-cache.js";
+import { SanityDocFetcher } from "./adapters/doc-fetchers/index.js";
+import { PromptfooEvalAdapter } from "./adapters/eval-runners/promptfoo-eval-adapter.js";
+import { ConsoleLogger, JsonLogger, QuietLogger, } from "./adapters/loggers/index.js";
+import { CompositeTaskSource, ContentLakeTaskSource, RepoTaskSource, YamlTaskSource, } from "./adapters/task-sources/index.js";
+import { getSanityClient } from "./sanity/client.js";
+import { ReportStore } from "./report-store.js";
+import { loadSinks } from "./sinks/index.js";
+/**
+ * Create a fully wired AppContext from resolved configuration.
+ *
+ * Every adapter is constructed here and nowhere else (outside of tests).
+ * Swapping an adapter (e.g., YamlTaskSource → ContentLakeTaskSource)
+ * is a one-line change in this function.
+ */
+export function createAppContext(config) {
+    // Logger — selected by env var preferences
+    const logger = createLogger();
+    // Cache — filesystem, optionally decorated with Content Lake fallback
+    const cache = config.noCache ? undefined : createCache(config);
+    // Task source — selected by config.taskSourceType
+    const taskSource = createTaskSource(config);
+    // Doc fetcher — Sanity Content Lake
+    const docFetcher = new SanityDocFetcher(config.rootDir);
+    // Eval runner — Promptfoo subprocess
+    const evalRunner = new PromptfooEvalAdapter(config.rootDir);
+    // Report store — Sanity Content Lake (for publish + auto-compare)
+    const reportStore = createReportStore();
+    // Sinks — loaded from config/sinks.yaml
+    const sinks = loadSinks();
+    return {
+        cache,
+        config,
+        docFetcher,
+        evalRunner,
+        logger,
+        reportStore,
+        sinks,
+        taskSource,
+    };
+}
+// ---------------------------------------------------------------------------
+// Sub-factories (extracted to keep createAppContext readable)
+// ---------------------------------------------------------------------------
+function createLogger() {
+    if (process.env.AILF_LOG_FORMAT === "json")
+        return new JsonLogger();
+    if (process.env.AILF_QUIET === "1")
+        return new QuietLogger();
+    return new ConsoleLogger({ verbose: process.env.AILF_VERBOSE === "1" });
+}
+function createCache(config) {
+    const local = new FilesystemCache(config.rootDir);
+    if (config.noRemoteCache)
+        return local;
+    // Only add Content Lake decorator when a report store token exists
+    const token = process.env.AILF_REPORT_SANITY_API_TOKEN ?? process.env.SANITY_API_TOKEN;
+    if (!token)
+        return local;
+    return new ContentLakeCacheAdapter(local, createReportStore());
+}
+function createTaskSource(config) {
+    // Primary source — selected by config.taskSourceType
+    // Content Lake tasks may require the report token for access.
+    const primary = config.taskSourceType === "yaml"
+        ? new YamlTaskSource(config.rootDir)
+        : new ContentLakeTaskSource(getSanityClient({
+            token: process.env.AILF_REPORT_SANITY_API_TOKEN ??
+                process.env.SANITY_API_TOKEN ??
+                undefined,
+        }));
+    // If repo tasks path is set, combine primary + repo sources
+    if (config.repoTasksPath) {
+        return new CompositeTaskSource([
+            primary,
+            new RepoTaskSource(config.repoTasksPath),
+        ]);
+    }
+    return primary;
+}
+function createReportStore() {
+    return new ReportStore({
+        dataset: process.env.AILF_REPORT_DATASET ?? undefined,
+        projectId: process.env.AILF_REPORT_PROJECT_ID ?? undefined,
+        token: process.env.AILF_REPORT_SANITY_API_TOKEN ??
+            process.env.SANITY_API_TOKEN ??
+            undefined,
+    });
+}

package/dist/interpolate.d.ts

@@ -0,0 +1,26 @@
+/**
+ * interpolate.ts
+ *
+ * Resolves ${{ VAR }} and ${{ VAR | default }} expressions in YAML-loaded
+ * configuration values. Used by the source loader to inject environment
+ * variables into config/sources.yaml at parse time.
+ *
+ * Design choices:
+ *   - ${{ }} syntax (not ${ }) to avoid collision with shell and JS templates
+ *   - Required by default — missing var with no default throws immediately
+ *   - Pipe | for defaults — familiar from GitHub Actions, Helm, etc.
+ *   - No nested logic — this is substitution, not a template engine
+ *   - Recursive on objects/arrays — env vars work at any depth
+ */
+/**
+ * Recursively interpolate all string values in a parsed YAML structure.
+ * Non-string values (numbers, booleans, null) pass through unchanged.
+ */
+export declare function interpolate<T>(value: T, env?: Record<string, string | undefined>): T;
+/**
+ * Interpolate a single string value, replacing all ${{ VAR }} expressions
+ * with values from the provided environment record.
+ *
+ * @throws Error if a required variable (no default) is not set
+ */
+export declare function interpolateString(value: string, env?: Record<string, string | undefined>): string;

package/dist/interpolate.js

@@ -0,0 +1,70 @@
+/**
+ * interpolate.ts
+ *
+ * Resolves ${{ VAR }} and ${{ VAR | default }} expressions in YAML-loaded
+ * configuration values. Used by the source loader to inject environment
+ * variables into config/sources.yaml at parse time.
+ *
+ * Design choices:
+ *   - ${{ }} syntax (not ${ }) to avoid collision with shell and JS templates
+ *   - Required by default — missing var with no default throws immediately
+ *   - Pipe | for defaults — familiar from GitHub Actions, Helm, etc.
+ *   - No nested logic — this is substitution, not a template engine
+ *   - Recursive on objects/arrays — env vars work at any depth
+ */
+// ---------------------------------------------------------------------------
+// Core interpolation
+// ---------------------------------------------------------------------------
+/**
+ * Pattern matching ${{ VAR }} and ${{ VAR | default_value }}.
+ * Captures:
+ *   group 1: variable name (trimmed)
+ *   group 2: default value (trimmed), or undefined if no pipe
+ */
+const INTERP_PATTERN = /\$\{\{\s*([^}|]+?)(?:\s*\|\s*([^}]*?))?\s*\}\}/g;
+/**
+ * Recursively interpolate all string values in a parsed YAML structure.
+ * Non-string values (numbers, booleans, null) pass through unchanged.
+ */
+export function interpolate(value, env = process.env) {
+    if (typeof value === "string") {
+        return interpolateString(value, env);
+    }
+    if (Array.isArray(value)) {
+        return value.map((item) => interpolate(item, env));
+    }
+    if (value !== null && typeof value === "object") {
+        const result = {};
+        for (const [key, val] of Object.entries(value)) {
+            result[key] = interpolate(val, env);
+        }
+        return result;
+    }
+    // Numbers, booleans, null — pass through
+    return value;
+}
+// ---------------------------------------------------------------------------
+// Recursive interpolation for parsed YAML structures
+// ---------------------------------------------------------------------------
+/**
+ * Interpolate a single string value, replacing all ${{ VAR }} expressions
+ * with values from the provided environment record.
+ *
+ * @throws Error if a required variable (no default) is not set
+ */
+export function interpolateString(value, env = process.env) {
+    return value.replace(INTERP_PATTERN, (match, varName, defaultValue) => {
+        const name = varName.trim();
+        const resolved = env[name];
+        if (resolved !== undefined && resolved !== "")
+            return resolved;
+        if (defaultValue !== undefined)
+            return defaultValue.trim();
+        throw new Error("Required environment variable " +
+            name +
+            " is not set and has no default. " +
+            "Set it in the environment file or pass it directly. " +
+            "Expression: " +
+            match);
+    });
+}

package/dist/job-store.d.ts

@@ -0,0 +1,104 @@
+/**
+ * job-store.ts
+ *
+ * Job tracking store backed by the Sanity Content Lake.
+ * Provides CRUD operations for `ailf.job` documents that track
+ * API-triggered pipeline evaluations.
+ *
+ * Job documents are created by the API gateway and updated by the
+ * CLI as the pipeline progresses through steps.
+ *
+ * @see docs/design-docs/api-service-gateway.md
+ * @see packages/studio/src/schema/job.ts — Sanity document schema
+ */
+import type { SanityClient } from "@sanity/client";
+export type JobStatus = "completed" | "failed" | "queued" | "received" | "running" | "timed-out";
+export interface JobDocument {
+    apiKeyPrefix?: string;
+    callback?: {
+        headers?: Record<string, string>;
+        url: string;
+    };
+    completedAt?: string;
+    config?: Record<string, unknown>;
+    createdAt: string;
+    deadlineAt?: string;
+    error?: {
+        message: string;
+        step?: string;
+    };
+    execution?: {
+        backend?: string;
+        runId?: string;
+        runUrl?: string;
+    };
+    jobId: string;
+    progress?: {
+        completedSteps?: number;
+        currentStep?: string;
+        totalSteps?: number;
+    };
+    reportId?: string;
+    startedAt?: string;
+    status: JobStatus;
+}
+export interface CreateJobOptions {
+    /** API key prefix — identifies which consumer created this job */
+    apiKeyPrefix: string;
+    /** Callback configuration (URL + optional headers) */
+    callback?: {
+        headers?: Record<string, string>;
+        url: string;
+    };
+    /** The pipeline request config (stored as JSON blob) */
+    config: Record<string, unknown>;
+    /** Custom deadline in milliseconds from now (default: 45 min) */
+    deadlineMs?: number;
+}
+export interface JobStoreOptions {
+    /** Override the Sanity client (for testing) */
+    client?: SanityClient;
+    /** Sanity dataset name */
+    dataset?: string;
+    /** Sanity project ID */
+    projectId?: string;
+    /** Sanity API token (required for writes) */
+    token?: string;
+}
+export declare class JobStore {
+    private readonly client;
+    constructor(options?: JobStoreOptions);
+    /**
+     * Create a new job document in the Content Lake.
+     *
+     * Generates a job ID (e.g. "job_a1b2c3d4e5f6") and writes the
+     * initial document with status "received". The job ID is used
+     * directly as the Sanity document `_id`.
+     *
+     * @returns The generated job ID, or null on failure
+     */
+    createJob(options: CreateJobOptions): Promise<string | null>;
+    /**
+     * Count jobs created today for a given API key prefix.
+     * Used for daily eval budget enforcement.
+     */
+    countTodayJobs(apiKeyPrefix: string): Promise<number>;
+    /**
+     * Count active (non-terminal) jobs for a given API key prefix.
+     * Used for concurrent job limit enforcement.
+     */
+    countActiveJobs(apiKeyPrefix: string): Promise<number>;
+    /**
+     * Find stale jobs that have exceeded their deadline.
+     * Used by the timeout cron job.
+     */
+    findStaleJobs(): Promise<JobDocument[]>;
+    /**
+     * Read a job by its job ID.
+     */
+    readJob(jobId: string): Promise<JobDocument | null>;
+    /**
+     * Update a job's status and optional associated data.
+     */
+    updateJob(jobId: string, update: Partial<Pick<JobDocument, "completedAt" | "error" | "execution" | "progress" | "reportId" | "startedAt" | "status">>): Promise<boolean>;
+}