@sanity/ailf 0.5.0 → 1.0.0

package/dist/pipeline/compiler/promptfoo-compiler.js

@@ -0,0 +1,230 @@
+/**
+ * PromptfooCompiler — compiles a TaskGraph into Promptfoo YAML configuration.
+ *
+ * The compiler is the core of the new architecture. It takes a validated
+ * TaskGraph and produces a Promptfoo config that can be executed via
+ * `promptfoo eval`.
+ *
+ * Compilation pipeline:
+ *   TaskGraph → resolve fixtures → resolve variables → map assertions
+ *             → assemble prompts → assemble providers → emit YAML
+ *
+ * This module exists alongside `generate-configs.ts` — it does NOT replace
+ * the existing codegen path. Phase 7 will swap callers over to the compiler.
+ *
+ * @see docs/exec-plans/architecture-overhaul/phase-2-config-compiler.md
+ */
+import { mapAssertions } from "./assertion-mapper.js";
+import { resolveTaskFixtures } from "./fixture-resolver.js";
+import { LiteracyVariant } from "../normalize-mode.js";
+import { resolveVariables } from "./variable-resolver.js";
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+/**
+ * Compile a TaskGraph into a Promptfoo configuration.
+ *
+ * Traverses the graph in topological order, resolves fixtures and
+ * variables for each node, maps assertions, and assembles the final
+ * Promptfoo config.
+ */
+export function compileToPromptfoo(graph, options) {
+    const warnings = [];
+    const tests = [];
+    // Sort nodes by priority (topological order)
+    const sortedNodes = [...graph.nodes.values()].sort((a, b) => a.priority - b.priority);
+    // Compile each node into test cases
+    for (const node of sortedNodes) {
+        const compiled = compileNode(node, graph, options, warnings);
+        tests.push(...compiled);
+    }
+    // Build providers list from model registry
+    const providers = buildProviders(options.models, options.mode);
+    // Prompt resolution: handler-owned → explicit override → built-in defaults
+    const prompts = resolvePrompts(options);
+    const config = {
+        description: `AILF evaluation — ${options.mode} mode (${tests.length} test cases)`,
+        prompts,
+        providers,
+        tests,
+        ...(options.outputPath ? { outputPath: options.outputPath } : {}),
+        ...(options.graderProvider
+            ? {
+                defaultTest: {
+                    options: {
+                        provider: options.graderProvider,
+                    },
+                },
+            }
+            : {}),
+    };
+    return {
+        config,
+        taskCount: sortedNodes.length,
+        testCaseCount: tests.length,
+        warnings,
+    };
+}
+// ---------------------------------------------------------------------------
+// Node compilation
+// ---------------------------------------------------------------------------
+function compileNode(node, graph, options, warnings) {
+    // Resolve fixtures using a minimal GeneralizedTaskDefinition stub.
+    // The fixture resolver needs task.id and context.docs — we use the
+    // node's mode (propagated from the original task definition) to
+    // construct the correct variant stub.
+    const nodeMode = node.mode ?? options.mode ?? "literacy";
+    const fixtureResult = resolveTaskFixtures({
+        mode: nodeMode,
+        id: node.taskId,
+        title: node.taskId,
+        prompt: { text: node.resolvedPrompt },
+    }, node.resolvedVariables, { rootDir: options.rootDir });
+    warnings.push(...fixtureResult.warnings);
+    // Resolve dynamic variables
+    const varResult = resolveVariables(fixtureResult.updatedVars);
+    warnings.push(...varResult.warnings);
+    // Map assertions from the node's metadata
+    // For now, nodes carry assertion data in resolvedVariables.values.__assertions
+    // (set by the TaskGraphBuilder when it has generalized task data)
+    const rawAssertions = varResult.envelope.values.__assertions ?? [];
+    const { mapped: assertions, warnings: assertionWarnings } = mapAssertions(rawAssertions, { mode: options.mode, graderProvider: options.graderProvider });
+    warnings.push(...assertionWarnings);
+    // Build test case vars (exclude internal __ prefixed keys)
+    const vars = {};
+    for (const [key, value] of Object.entries(varResult.envelope.values)) {
+        if (!key.startsWith("__")) {
+            vars[key] = value;
+        }
+    }
+    // Create the gold test case
+    const goldTest = {
+        description: node.taskId,
+        vars,
+        ...(assertions.length > 0 ? { assert: assertions } : {}),
+    };
+    const tests = [goldTest];
+    // For literacy/baseline mode, also create a baseline variant (no docs).
+    // Route on the node's mode (from the task definition) rather than
+    // the global options.mode, so heterogeneous graphs compile correctly.
+    if (nodeMode === LiteracyVariant.STANDARD || nodeMode === "literacy") {
+        const baselineVars = { ...vars, docs: "" };
+        const baselineTest = {
+            description: `${node.taskId} [baseline]`,
+            vars: baselineVars,
+            prompts: ["without-docs"],
+            ...(assertions.length > 0 ? { assert: assertions } : {}),
+        };
+        tests.push(baselineTest);
+    }
+    return tests;
+}
+// ---------------------------------------------------------------------------
+// Provider assembly
+// ---------------------------------------------------------------------------
+/**
+ * Build the Promptfoo providers list from the model registry.
+ */
+function buildProviders(models, mode) {
+    const providers = [];
+    for (const model of models.models) {
+        // Check mode compatibility
+        if (model.modes && model.modes.length > 0) {
+            if (!modelMatchesMode(model, mode))
+                continue;
+        }
+        // Check env gate
+        if (model.env && !process.env[model.env]) {
+            continue; // Skip models whose API key isn't set
+        }
+        providers.push({
+            id: model.id,
+            label: model.label,
+            config: {
+                ...models.defaults,
+                ...model.config,
+            },
+        });
+    }
+    return providers;
+}
+/**
+ * Check if a model entry matches the current evaluation mode.
+ *
+ * Literacy mode defaults to baseline model matching. Variant-specific
+ * provider filtering is handled by the provider-assembler and
+ * generate-configs-step, not here.
+ */
+function modelMatchesMode(model, mode) {
+    if (!model.modes || model.modes.length === 0)
+        return true;
+    switch (mode) {
+        case "literacy":
+            return model.modes.includes(LiteracyVariant.STANDARD);
+        default:
+            // Non-literacy modes accept all models by default
+            return true;
+    }
+}
+// ---------------------------------------------------------------------------
+// Prompt resolution
+// ---------------------------------------------------------------------------
+/**
+ * Resolve prompts with a three-level fallback chain:
+ *   1. handler.getPrompts() — mode-handler-owned templates
+ *   2. options.prompts — explicit caller-provided templates
+ *   3. buildDefaultPrompts() — built-in defaults per mode
+ */
+function resolvePrompts(options) {
+    // 1. Check handler-owned prompts
+    const handlerPrompts = options.handler?.getPrompts?.();
+    if (handlerPrompts && Object.keys(handlerPrompts).length > 0) {
+        return Object.values(handlerPrompts).map(promptTemplateToPromptfoo);
+    }
+    // 2. Check explicit override
+    if (options.prompts)
+        return options.prompts;
+    // 3. Built-in defaults
+    return buildDefaultPrompts(options.mode);
+}
+/**
+ * Convert a PromptTemplate (core port type) to a PromptfooPrompt (compiler type).
+ */
+function promptTemplateToPromptfoo(pt) {
+    return { id: pt.id, label: pt.label, raw: pt.template };
+}
+// ---------------------------------------------------------------------------
+// Default prompts
+// ---------------------------------------------------------------------------
+/**
+ * Build default prompt entries for a mode.
+ *
+ * Handler-owned prompts (via getPrompts()) take precedence over these
+ * built-in defaults. This fallback exists for modes that haven't yet
+ * migrated to handler-owned prompts.
+ */
+function buildDefaultPrompts(mode) {
+    switch (mode) {
+        case "literacy":
+            return [
+                {
+                    id: "with-docs",
+                    label: "With documentation context",
+                    raw: "{{task}}\n\nDocumentation context:\n{{docs}}",
+                },
+                {
+                    id: "without-docs",
+                    label: "Without documentation context",
+                    raw: "{{task}}",
+                },
+            ];
+        default:
+            return [
+                {
+                    id: "default",
+                    label: "Default prompt",
+                    raw: "{{task}}",
+                },
+            ];
+    }
+}

package/dist/pipeline/compiler/provider-assembler.d.ts

@@ -0,0 +1,39 @@
+/**
+ * provider-assembler.ts — Build per-mode provider arrays from models config.
+ *
+ * Replicates the provider-building logic from the legacy generate-configs.ts
+ * so the new compiler produces identical provider configurations.
+ *
+ * Separated into its own module so GenerateConfigsStep can import it
+ * without pulling in the full legacy generate-configs machinery.
+ */
+import { type ModelsConfig } from "../../_vendor/ailf-core/index.d.ts";
+import type { ResolvedSourceConfig } from "../../sources.js";
+/**
+ * Provider arrays grouped by literacy variant.
+ *
+ * These keys are literacy variant names (not EvalMode values). Each variant
+ * needs a different set of model providers with variant-specific config
+ * (e.g., agentic providers carry tool-use config, observed providers carry
+ * observer instrumentation).
+ */
+export interface LiteracyVariantProviders {
+    baseline: Record<string, unknown>[];
+    agentic: Record<string, unknown>[];
+    observed: Record<string, unknown>[];
+}
+/** @deprecated Use LiteracyVariantProviders — kept for backward compatibility */
+export type AssembledProviders = LiteracyVariantProviders;
+/** Result of loading models and assembling providers */
+export interface ModelsAndProviders {
+    models: ModelsConfig;
+    providers: LiteracyVariantProviders;
+}
+/**
+ * Load models config and assemble provider arrays for literacy variants.
+ *
+ * Returns provider arrays keyed by literacy variant name (baseline,
+ * agentic, observed). These are consumed by the YAML writer to produce
+ * the per-variant promptfoo config files.
+ */
+export declare function loadModelsAndProviders(rootDir: string, source?: ResolvedSourceConfig, searchMode?: string, allowedOrigins?: string[]): ModelsAndProviders;

package/dist/pipeline/compiler/provider-assembler.js

@@ -0,0 +1,137 @@
+/**
+ * provider-assembler.ts — Build per-mode provider arrays from models config.
+ *
+ * Replicates the provider-building logic from the legacy generate-configs.ts
+ * so the new compiler produces identical provider configurations.
+ *
+ * Separated into its own module so GenerateConfigsStep can import it
+ * without pulling in the full legacy generate-configs machinery.
+ */
+import { extractModelName, extractProvider, mergeConfig, modelMatchesMode, } from "../../_vendor/ailf-core/index.js";
+import { LiteracyVariant } from "../normalize-mode.js";
+import { loadConfigFile } from "./config-loader.js";
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+/**
+ * Load models config and assemble provider arrays for literacy variants.
+ *
+ * Returns provider arrays keyed by literacy variant name (baseline,
+ * agentic, observed). These are consumed by the YAML writer to produce
+ * the per-variant promptfoo config files.
+ */
+export function loadModelsAndProviders(rootDir, source, searchMode, allowedOrigins) {
+    const models = loadModelsYaml(rootDir);
+    return {
+        models,
+        providers: {
+            baseline: buildBaselineProviders(models),
+            agentic: buildAgenticProviders(models, source, searchMode, allowedOrigins),
+            observed: buildObservedProviders(models),
+        },
+    };
+}
+// ---------------------------------------------------------------------------
+// Baseline providers
+// ---------------------------------------------------------------------------
+function buildBaselineProviders(models) {
+    return models.models
+        .filter((m) => modelMatchesMode(m, LiteracyVariant.STANDARD))
+        .map((model) => ({
+        config: mergeConfig(models.defaults, model.config),
+        id: model.id,
+        label: model.label,
+    }));
+}
+// ---------------------------------------------------------------------------
+// Observed providers
+// ---------------------------------------------------------------------------
+function buildObservedProviders(models) {
+    return models.models
+        .filter((m) => modelMatchesMode(m, LiteracyVariant.OBSERVED))
+        .map((model) => {
+        const modelName = extractModelName(model.id);
+        return {
+            config: {
+                ...mergeConfig(models.defaults, model.config),
+                modelName,
+                observe: true,
+                recordOptions: models.defaults.observerOptions ?? {},
+            },
+            id: "file://dist/agent-observer/provider.js",
+            label: `${model.label} (Observed)`,
+        };
+    });
+}
+// ---------------------------------------------------------------------------
+// Agentic providers
+// ---------------------------------------------------------------------------
+function buildAgenticProviders(models, source, searchMode, _allowedOrigins) {
+    const naiveModels = models.models.filter((m) => modelMatchesMode(m, "agentic-naive"));
+    const optimizedModels = models.models.filter((m) => modelMatchesMode(m, "agentic-optimized"));
+    const resolvedSearchMode = searchMode ?? "open";
+    const sourceConfig = source
+        ? {
+            ...(source.allowedOrigins?.length
+                ? { allowedOrigins: source.allowedOrigins }
+                : {}),
+            docBaseUrl: source.baseUrl,
+            ...(source.headers && Object.keys(source.headers).length > 0
+                ? { customHeaders: source.headers }
+                : {}),
+            llmsTxtUrl: source.llmsTxt,
+            ...(source.priorityDomain
+                ? { priorityDomain: source.priorityDomain }
+                : {}),
+            ...(resolvedSearchMode !== "open"
+                ? { searchMode: resolvedSearchMode }
+                : {}),
+        }
+        : {};
+    const providers = [];
+    for (const model of naiveModels) {
+        const modelName = extractModelName(model.id);
+        const provider = extractProvider(model.id);
+        providers.push({
+            config: {
+                ...mergeConfig(models.defaults, model.config, {
+                    agentMode: "naive",
+                    maxToolRounds: models.defaults.maxToolRounds ?? 5,
+                    model: modelName,
+                    provider,
+                }),
+                ...sourceConfig,
+                observe: true,
+                observerOptions: models.defaults.observerOptions ?? {},
+            },
+            id: "file://dist/agent-observer/agentic-provider.js",
+            label: `${model.label} (Naive Agent)`,
+        });
+    }
+    for (const model of optimizedModels) {
+        const modelName = extractModelName(model.id);
+        const provider = extractProvider(model.id);
+        providers.push({
+            config: {
+                ...mergeConfig(models.defaults, model.config, {
+                    agentMode: "optimized",
+                    maxToolRounds: models.defaults.maxToolRounds ?? 5,
+                    model: modelName,
+                    provider,
+                }),
+                ...sourceConfig,
+                observe: true,
+                observerOptions: models.defaults.observerOptions ?? {},
+            },
+            id: "file://dist/agent-observer/agentic-provider.js",
+            label: `${model.label} (Optimized Agent)`,
+        });
+    }
+    return providers;
+}
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+function loadModelsYaml(rootDir) {
+    return loadConfigFile("models", rootDir).data;
+}

package/dist/pipeline/compiler/sandbox/docker-sandbox.d.ts

@@ -0,0 +1,21 @@
+/**
+ * DockerSandboxStrategy — full isolation via Docker containers.
+ *
+ * Provides deterministic filesystem, network control, and resource limits.
+ * Falls back to TempDirSandboxStrategy when Docker is unavailable.
+ *
+ * Docker interaction uses the `docker` CLI via `execFileSync` (array form,
+ * no shell) to prevent shell injection from task-supplied values like
+ * image names or task IDs.
+ *
+ * @see docs/exec-plans/architecture-overhaul/phase-4-agent-harness.md
+ */
+import type { SandboxArtifacts, SandboxInfo, SandboxProvisionOptions, SandboxStrategy } from "./sandbox-strategy.js";
+export declare class DockerSandboxStrategy implements SandboxStrategy {
+    readonly name = "Docker Container";
+    readonly type: "docker";
+    isAvailable(): Promise<boolean>;
+    provision(options: SandboxProvisionOptions): Promise<SandboxInfo>;
+    collectArtifacts(sandbox: SandboxInfo): Promise<SandboxArtifacts>;
+    teardown(sandbox: SandboxInfo): Promise<void>;
+}

package/dist/pipeline/compiler/sandbox/docker-sandbox.js

@@ -0,0 +1,136 @@
+/**
+ * DockerSandboxStrategy — full isolation via Docker containers.
+ *
+ * Provides deterministic filesystem, network control, and resource limits.
+ * Falls back to TempDirSandboxStrategy when Docker is unavailable.
+ *
+ * Docker interaction uses the `docker` CLI via `execFileSync` (array form,
+ * no shell) to prevent shell injection from task-supplied values like
+ * image names or task IDs.
+ *
+ * @see docs/exec-plans/architecture-overhaul/phase-4-agent-harness.md
+ */
+import { randomUUID } from "crypto";
+import { execFileSync } from "child_process";
+import { mkdirSync } from "fs";
+import { tmpdir } from "os";
+import { resolve } from "path";
+const DEFAULT_IMAGE = "node:22-slim";
+const DEFAULT_WORKDIR = "/workspace";
+/** Only allow official base images to prevent pulling from untrusted registries. */
+const ALLOWED_IMAGE_PATTERN = /^(node|python|ubuntu|alpine|debian|rust|golang|mcr\.microsoft\.com\/[a-z]+)(:[a-zA-Z0-9._-]+)?$/;
+function validateDockerImage(image) {
+    if (!ALLOWED_IMAGE_PATTERN.test(image)) {
+        throw new Error(`Docker image "${image}" is not in the allowlist. ` +
+            `Only official base images (node, python, ubuntu, alpine, debian, rust, golang) are permitted.`);
+    }
+}
+export class DockerSandboxStrategy {
+    name = "Docker Container";
+    type = "docker";
+    async isAvailable() {
+        try {
+            execFileSync("docker", ["info"], { stdio: "ignore", timeout: 5000 });
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    async provision(options) {
+        const image = options.image ?? DEFAULT_IMAGE;
+        validateDockerImage(image);
+        const id = `ailf-${randomUUID().slice(0, 12)}`;
+        // Create a local staging directory for fixture injection
+        const stagingDir = resolve(tmpdir(), `${id}-staging`);
+        mkdirSync(stagingDir, { recursive: true });
+        // Build docker create command as array (no shell, prevents injection)
+        const args = ["create", "--name", id, "--workdir", DEFAULT_WORKDIR];
+        // Security hardening — defense-in-depth against container escape
+        args.push("--cap-drop", "ALL");
+        args.push("--security-opt", "no-new-privileges");
+        args.push("--read-only");
+        args.push("--tmpfs", "/tmp:rw,noexec,nosuid,size=100m");
+        // Resource limits
+        if (options.limits) {
+            if (options.limits.cpus) {
+                args.push("--cpus", String(options.limits.cpus));
+            }
+            if (options.limits.memoryBytes) {
+                args.push("--memory", String(options.limits.memoryBytes));
+            }
+            if (options.limits.networkAccess === false) {
+                args.push("--network", "none");
+            }
+        }
+        // Bind mount staging directory
+        args.push("-v", `${stagingDir}:${DEFAULT_WORKDIR}`);
+        args.push(image);
+        args.push("sleep", "infinity"); // Keep container alive
+        try {
+            const containerId = execFileSync("docker", args, {
+                encoding: "utf-8",
+                timeout: 30_000,
+            }).trim();
+            // Start the container
+            execFileSync("docker", ["start", id], {
+                stdio: "ignore",
+                timeout: 10_000,
+            });
+            return {
+                id,
+                workingDir: stagingDir,
+                strategy: "docker",
+                containerId: containerId || id,
+                createdAt: new Date().toISOString(),
+            };
+        }
+        catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            throw new Error(`Failed to provision Docker sandbox "${id}": ${msg}`, {
+                cause: err,
+            });
+        }
+    }
+    async collectArtifacts(sandbox) {
+        const modifiedFiles = [];
+        try {
+            // Get list of modified files via docker diff (array form)
+            const diff = execFileSync("docker", ["diff", sandbox.id], {
+                encoding: "utf-8",
+                timeout: 10_000,
+            }).trim();
+            if (diff) {
+                for (const line of diff.split("\n")) {
+                    // docker diff output: C /workspace/file.ts (C=changed, A=added, D=deleted)
+                    const match = /^[ACD]\s+(.+)$/.exec(line.trim());
+                    if (match && match[1].startsWith(DEFAULT_WORKDIR)) {
+                        modifiedFiles.push(match[1].replace(`${DEFAULT_WORKDIR}/`, ""));
+                    }
+                }
+            }
+            return {
+                modifiedFiles,
+                diff: diff || undefined,
+                durationMs: Date.now() - new Date(sandbox.createdAt).getTime(),
+            };
+        }
+        catch {
+            return {
+                modifiedFiles,
+                durationMs: Date.now() - new Date(sandbox.createdAt).getTime(),
+            };
+        }
+    }
+    async teardown(sandbox) {
+        try {
+            execFileSync("docker", ["rm", "-f", sandbox.id], {
+                stdio: "ignore",
+                timeout: 10_000,
+            });
+        }
+        catch {
+            // Best-effort cleanup
+        }
+    }
+}

package/dist/pipeline/compiler/sandbox/fixture-provisioner.d.ts

@@ -0,0 +1,69 @@
+/**
+ * Fixture provisioner — five-stage pipeline for preparing sandbox state.
+ *
+ * Pipeline stages:
+ *   Resolve → Fetch → Cache → Transform → Inject
+ *
+ * Handles three URI schemes for v1:
+ * - file:// — local filesystem path (relative to task)
+ * - template:// — built-in project templates
+ * - sanity:// — Content Lake document by ID or query
+ *
+ * @see docs/design-docs/architecture-overhaul/fixtures-artifacts.md
+ * @see docs/exec-plans/architecture-overhaul/phase-4-agent-harness.md
+ */
+import type { SandboxInfo } from "./sandbox-strategy.js";
+/** A fixture reference from a task definition */
+export interface FixtureRef {
+    /** URI pointing to the fixture source */
+    uri: string;
+    /** Injection target */
+    inject: "provider_config" | "system_prompt" | "vars" | "working_dir";
+    /** Key name (for vars injection) or relative path (for working_dir) */
+    key?: string;
+    /** Content transform to apply before injection */
+    transform?: FixtureTransform;
+}
+/** Available fixture transforms */
+export type FixtureTransform = "extract-text" | "none" | "strip-html" | "truncate";
+/** A resolved and fetched fixture ready for injection */
+export interface ProvisionedFixture {
+    /** Original URI */
+    uri: string;
+    /** Resolved content */
+    content: string;
+    /** SHA-256 hash of the content */
+    contentHash: string;
+    /** Injection target */
+    inject: FixtureRef["inject"];
+    /** Key or path */
+    key?: string;
+}
+/** Result of the provisioning pipeline */
+export interface ProvisioningResult {
+    /** Successfully provisioned fixtures */
+    fixtures: ProvisionedFixture[];
+    /** Variable overrides from vars-injected fixtures */
+    vars: Record<string, unknown>;
+    /** Warnings (non-fatal issues) */
+    warnings: string[];
+    /** Fixture manifest for reproducibility */
+    manifest: Record<string, string>;
+}
+/** Options for the provisioning pipeline */
+export interface ProvisioningOptions {
+    /** Root directory for resolving relative paths */
+    rootDir: string;
+    /** Sandbox to inject working_dir fixtures into */
+    sandbox?: SandboxInfo;
+    /** Cache directory for content-addressable storage */
+    cacheDir?: string;
+}
+/**
+ * Run the five-stage fixture provisioning pipeline.
+ *
+ * @param refs - Fixture references from the task definition
+ * @param options - Provisioning configuration
+ * @returns Provisioned fixtures and injection metadata
+ */
+export declare function provisionFixtures(refs: FixtureRef[], options: ProvisioningOptions): Promise<ProvisioningResult>;