@sanity/ailf 0.5.0 → 2.0.0

package/dist/pipeline/compiler/provider-assembler.js

@@ -0,0 +1,137 @@
+/**
+ * provider-assembler.ts — Build per-mode provider arrays from models config.
+ *
+ * Replicates the provider-building logic from the legacy generate-configs.ts
+ * so the new compiler produces identical provider configurations.
+ *
+ * Separated into its own module so GenerateConfigsStep can import it
+ * without pulling in the full legacy generate-configs machinery.
+ */
+import { extractModelName, extractProvider, mergeConfig, modelMatchesMode, } from "../../_vendor/ailf-core/index.js";
+import { LiteracyVariant } from "../normalize-mode.js";
+import { loadConfigFile } from "./config-loader.js";
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+/**
+ * Load models config and assemble provider arrays for literacy variants.
+ *
+ * Returns provider arrays keyed by literacy variant name (baseline,
+ * agentic, observed). These are consumed by the YAML writer to produce
+ * the per-variant promptfoo config files.
+ */
+export function loadModelsAndProviders(rootDir, source, searchMode, allowedOrigins) {
+    const models = loadModelsYaml(rootDir);
+    return {
+        models,
+        providers: {
+            baseline: buildBaselineProviders(models),
+            agentic: buildAgenticProviders(models, source, searchMode, allowedOrigins),
+            observed: buildObservedProviders(models),
+        },
+    };
+}
+// ---------------------------------------------------------------------------
+// Baseline providers
+// ---------------------------------------------------------------------------
+function buildBaselineProviders(models) {
+    return models.models
+        .filter((m) => modelMatchesMode(m, LiteracyVariant.STANDARD))
+        .map((model) => ({
+        config: mergeConfig(models.defaults, model.config),
+        id: model.id,
+        label: model.label,
+    }));
+}
+// ---------------------------------------------------------------------------
+// Observed providers
+// ---------------------------------------------------------------------------
+function buildObservedProviders(models) {
+    return models.models
+        .filter((m) => modelMatchesMode(m, LiteracyVariant.OBSERVED))
+        .map((model) => {
+        const modelName = extractModelName(model.id);
+        return {
+            config: {
+                ...mergeConfig(models.defaults, model.config),
+                modelName,
+                observe: true,
+                recordOptions: models.defaults.observerOptions ?? {},
+            },
+            id: "file://dist/agent-observer/provider.js",
+            label: `${model.label} (Observed)`,
+        };
+    });
+}
+// ---------------------------------------------------------------------------
+// Agentic providers
+// ---------------------------------------------------------------------------
+function buildAgenticProviders(models, source, searchMode, _allowedOrigins) {
+    const naiveModels = models.models.filter((m) => modelMatchesMode(m, "agentic-naive"));
+    const optimizedModels = models.models.filter((m) => modelMatchesMode(m, "agentic-optimized"));
+    const resolvedSearchMode = searchMode ?? "open";
+    const sourceConfig = source
+        ? {
+            ...(source.allowedOrigins?.length
+                ? { allowedOrigins: source.allowedOrigins }
+                : {}),
+            docBaseUrl: source.baseUrl,
+            ...(source.headers && Object.keys(source.headers).length > 0
+                ? { customHeaders: source.headers }
+                : {}),
+            llmsTxtUrl: source.llmsTxt,
+            ...(source.priorityDomain
+                ? { priorityDomain: source.priorityDomain }
+                : {}),
+            ...(resolvedSearchMode !== "open"
+                ? { searchMode: resolvedSearchMode }
+                : {}),
+        }
+        : {};
+    const providers = [];
+    for (const model of naiveModels) {
+        const modelName = extractModelName(model.id);
+        const provider = extractProvider(model.id);
+        providers.push({
+            config: {
+                ...mergeConfig(models.defaults, model.config, {
+                    agentMode: "naive",
+                    maxToolRounds: models.defaults.maxToolRounds ?? 5,
+                    model: modelName,
+                    provider,
+                }),
+                ...sourceConfig,
+                observe: true,
+                observerOptions: models.defaults.observerOptions ?? {},
+            },
+            id: "file://dist/agent-observer/agentic-provider.js",
+            label: `${model.label} (Naive Agent)`,
+        });
+    }
+    for (const model of optimizedModels) {
+        const modelName = extractModelName(model.id);
+        const provider = extractProvider(model.id);
+        providers.push({
+            config: {
+                ...mergeConfig(models.defaults, model.config, {
+                    agentMode: "optimized",
+                    maxToolRounds: models.defaults.maxToolRounds ?? 5,
+                    model: modelName,
+                    provider,
+                }),
+                ...sourceConfig,
+                observe: true,
+                observerOptions: models.defaults.observerOptions ?? {},
+            },
+            id: "file://dist/agent-observer/agentic-provider.js",
+            label: `${model.label} (Optimized Agent)`,
+        });
+    }
+    return providers;
+}
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+function loadModelsYaml(rootDir) {
+    return loadConfigFile("models", rootDir).data;
+}

package/dist/pipeline/compiler/sandbox/docker-sandbox.d.ts

@@ -0,0 +1,21 @@
+/**
+ * DockerSandboxStrategy — full isolation via Docker containers.
+ *
+ * Provides deterministic filesystem, network control, and resource limits.
+ * Falls back to TempDirSandboxStrategy when Docker is unavailable.
+ *
+ * Docker interaction uses the `docker` CLI via `execFileSync` (array form,
+ * no shell) to prevent shell injection from task-supplied values like
+ * image names or task IDs.
+ *
+ * @see docs/exec-plans/architecture-overhaul/phase-4-agent-harness.md
+ */
+import type { SandboxArtifacts, SandboxInfo, SandboxProvisionOptions, SandboxStrategy } from "./sandbox-strategy.js";
+export declare class DockerSandboxStrategy implements SandboxStrategy {
+    readonly name = "Docker Container";
+    readonly type: "docker";
+    isAvailable(): Promise<boolean>;
+    provision(options: SandboxProvisionOptions): Promise<SandboxInfo>;
+    collectArtifacts(sandbox: SandboxInfo): Promise<SandboxArtifacts>;
+    teardown(sandbox: SandboxInfo): Promise<void>;
+}

package/dist/pipeline/compiler/sandbox/docker-sandbox.js

@@ -0,0 +1,136 @@
+/**
+ * DockerSandboxStrategy — full isolation via Docker containers.
+ *
+ * Provides deterministic filesystem, network control, and resource limits.
+ * Falls back to TempDirSandboxStrategy when Docker is unavailable.
+ *
+ * Docker interaction uses the `docker` CLI via `execFileSync` (array form,
+ * no shell) to prevent shell injection from task-supplied values like
+ * image names or task IDs.
+ *
+ * @see docs/exec-plans/architecture-overhaul/phase-4-agent-harness.md
+ */
+import { randomUUID } from "crypto";
+import { execFileSync } from "child_process";
+import { mkdirSync } from "fs";
+import { tmpdir } from "os";
+import { resolve } from "path";
+const DEFAULT_IMAGE = "node:22-slim";
+const DEFAULT_WORKDIR = "/workspace";
+/** Only allow official base images to prevent pulling from untrusted registries. */
+const ALLOWED_IMAGE_PATTERN = /^(node|python|ubuntu|alpine|debian|rust|golang|mcr\.microsoft\.com\/[a-z]+)(:[a-zA-Z0-9._-]+)?$/;
+function validateDockerImage(image) {
+    if (!ALLOWED_IMAGE_PATTERN.test(image)) {
+        throw new Error(`Docker image "${image}" is not in the allowlist. ` +
+            `Only official base images (node, python, ubuntu, alpine, debian, rust, golang) are permitted.`);
+    }
+}
+export class DockerSandboxStrategy {
+    name = "Docker Container";
+    type = "docker";
+    async isAvailable() {
+        try {
+            execFileSync("docker", ["info"], { stdio: "ignore", timeout: 5000 });
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    async provision(options) {
+        const image = options.image ?? DEFAULT_IMAGE;
+        validateDockerImage(image);
+        const id = `ailf-${randomUUID().slice(0, 12)}`;
+        // Create a local staging directory for fixture injection
+        const stagingDir = resolve(tmpdir(), `${id}-staging`);
+        mkdirSync(stagingDir, { recursive: true });
+        // Build docker create command as array (no shell, prevents injection)
+        const args = ["create", "--name", id, "--workdir", DEFAULT_WORKDIR];
+        // Security hardening — defense-in-depth against container escape
+        args.push("--cap-drop", "ALL");
+        args.push("--security-opt", "no-new-privileges");
+        args.push("--read-only");
+        args.push("--tmpfs", "/tmp:rw,noexec,nosuid,size=100m");
+        // Resource limits
+        if (options.limits) {
+            if (options.limits.cpus) {
+                args.push("--cpus", String(options.limits.cpus));
+            }
+            if (options.limits.memoryBytes) {
+                args.push("--memory", String(options.limits.memoryBytes));
+            }
+            if (options.limits.networkAccess === false) {
+                args.push("--network", "none");
+            }
+        }
+        // Bind mount staging directory
+        args.push("-v", `${stagingDir}:${DEFAULT_WORKDIR}`);
+        args.push(image);
+        args.push("sleep", "infinity"); // Keep container alive
+        try {
+            const containerId = execFileSync("docker", args, {
+                encoding: "utf-8",
+                timeout: 30_000,
+            }).trim();
+            // Start the container
+            execFileSync("docker", ["start", id], {
+                stdio: "ignore",
+                timeout: 10_000,
+            });
+            return {
+                id,
+                workingDir: stagingDir,
+                strategy: "docker",
+                containerId: containerId || id,
+                createdAt: new Date().toISOString(),
+            };
+        }
+        catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            throw new Error(`Failed to provision Docker sandbox "${id}": ${msg}`, {
+                cause: err,
+            });
+        }
+    }
+    async collectArtifacts(sandbox) {
+        const modifiedFiles = [];
+        try {
+            // Get list of modified files via docker diff (array form)
+            const diff = execFileSync("docker", ["diff", sandbox.id], {
+                encoding: "utf-8",
+                timeout: 10_000,
+            }).trim();
+            if (diff) {
+                for (const line of diff.split("\n")) {
+                    // docker diff output: C /workspace/file.ts (C=changed, A=added, D=deleted)
+                    const match = /^[ACD]\s+(.+)$/.exec(line.trim());
+                    if (match && match[1].startsWith(DEFAULT_WORKDIR)) {
+                        modifiedFiles.push(match[1].replace(`${DEFAULT_WORKDIR}/`, ""));
+                    }
+                }
+            }
+            return {
+                modifiedFiles,
+                diff: diff || undefined,
+                durationMs: Date.now() - new Date(sandbox.createdAt).getTime(),
+            };
+        }
+        catch {
+            return {
+                modifiedFiles,
+                durationMs: Date.now() - new Date(sandbox.createdAt).getTime(),
+            };
+        }
+    }
+    async teardown(sandbox) {
+        try {
+            execFileSync("docker", ["rm", "-f", sandbox.id], {
+                stdio: "ignore",
+                timeout: 10_000,
+            });
+        }
+        catch {
+            // Best-effort cleanup
+        }
+    }
+}

package/dist/pipeline/compiler/sandbox/fixture-provisioner.d.ts

@@ -0,0 +1,69 @@
+/**
+ * Fixture provisioner — five-stage pipeline for preparing sandbox state.
+ *
+ * Pipeline stages:
+ *   Resolve → Fetch → Cache → Transform → Inject
+ *
+ * Handles three URI schemes for v1:
+ * - file:// — local filesystem path (relative to task)
+ * - template:// — built-in project templates
+ * - sanity:// — Content Lake document by ID or query
+ *
+ * @see docs/design-docs/architecture-overhaul/fixtures-artifacts.md
+ * @see docs/exec-plans/architecture-overhaul/phase-4-agent-harness.md
+ */
+import type { SandboxInfo } from "./sandbox-strategy.js";
+/** A fixture reference from a task definition */
+export interface FixtureRef {
+    /** URI pointing to the fixture source */
+    uri: string;
+    /** Injection target */
+    inject: "provider_config" | "system_prompt" | "vars" | "working_dir";
+    /** Key name (for vars injection) or relative path (for working_dir) */
+    key?: string;
+    /** Content transform to apply before injection */
+    transform?: FixtureTransform;
+}
+/** Available fixture transforms */
+export type FixtureTransform = "extract-text" | "none" | "strip-html" | "truncate";
+/** A resolved and fetched fixture ready for injection */
+export interface ProvisionedFixture {
+    /** Original URI */
+    uri: string;
+    /** Resolved content */
+    content: string;
+    /** SHA-256 hash of the content */
+    contentHash: string;
+    /** Injection target */
+    inject: FixtureRef["inject"];
+    /** Key or path */
+    key?: string;
+}
+/** Result of the provisioning pipeline */
+export interface ProvisioningResult {
+    /** Successfully provisioned fixtures */
+    fixtures: ProvisionedFixture[];
+    /** Variable overrides from vars-injected fixtures */
+    vars: Record<string, unknown>;
+    /** Warnings (non-fatal issues) */
+    warnings: string[];
+    /** Fixture manifest for reproducibility */
+    manifest: Record<string, string>;
+}
+/** Options for the provisioning pipeline */
+export interface ProvisioningOptions {
+    /** Root directory for resolving relative paths */
+    rootDir: string;
+    /** Sandbox to inject working_dir fixtures into */
+    sandbox?: SandboxInfo;
+    /** Cache directory for content-addressable storage */
+    cacheDir?: string;
+}
+/**
+ * Run the five-stage fixture provisioning pipeline.
+ *
+ * @param refs - Fixture references from the task definition
+ * @param options - Provisioning configuration
+ * @returns Provisioned fixtures and injection metadata
+ */
+export declare function provisionFixtures(refs: FixtureRef[], options: ProvisioningOptions): Promise<ProvisioningResult>;

package/dist/pipeline/compiler/sandbox/fixture-provisioner.js

@@ -0,0 +1,189 @@
+/**
+ * Fixture provisioner — five-stage pipeline for preparing sandbox state.
+ *
+ * Pipeline stages:
+ *   Resolve → Fetch → Cache → Transform → Inject
+ *
+ * Handles three URI schemes for v1:
+ * - file:// — local filesystem path (relative to task)
+ * - template:// — built-in project templates
+ * - sanity:// — Content Lake document by ID or query
+ *
+ * @see docs/design-docs/architecture-overhaul/fixtures-artifacts.md
+ * @see docs/exec-plans/architecture-overhaul/phase-4-agent-harness.md
+ */
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
+import { createHash } from "crypto";
+import { basename, dirname, resolve } from "path";
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+/**
+ * Run the five-stage fixture provisioning pipeline.
+ *
+ * @param refs - Fixture references from the task definition
+ * @param options - Provisioning configuration
+ * @returns Provisioned fixtures and injection metadata
+ */
+export async function provisionFixtures(refs, options) {
+    const fixtures = [];
+    const vars = {};
+    const warnings = [];
+    const manifest = {};
+    for (const ref of refs) {
+        try {
+            // Stage 1: Resolve — parse URI, determine backend
+            const resolved = resolveFixtureURI(ref.uri, options.rootDir);
+            // Stage 2: Fetch — read content from source
+            const content = await fetchFixtureContent(resolved, warnings);
+            if (content === null)
+                continue;
+            // Stage 3: Cache — store in content-addressable cache
+            const contentHash = hashContent(content);
+            if (options.cacheDir) {
+                cacheFixture(options.cacheDir, contentHash, content);
+            }
+            manifest[ref.uri] = contentHash;
+            // Stage 4: Transform — apply preprocessing
+            const transformed = applyTransform(content, ref.transform);
+            // Stage 5: Inject — place into target
+            const provisioned = {
+                uri: ref.uri,
+                content: transformed,
+                contentHash,
+                inject: ref.inject,
+                key: ref.key,
+            };
+            fixtures.push(provisioned);
+            // Handle injection targets
+            switch (ref.inject) {
+                case "vars":
+                    if (ref.key) {
+                        vars[ref.key] = transformed;
+                    }
+                    break;
+                case "working_dir":
+                    if (options.sandbox) {
+                        injectIntoWorkingDir(options.sandbox.workingDir, ref.key ?? basename(ref.uri), transformed);
+                    }
+                    break;
+                case "system_prompt":
+                case "provider_config":
+                    // These are handled by the caller when assembling the Promptfoo config
+                    break;
+            }
+        }
+        catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            warnings.push(`Fixture "${ref.uri}": ${msg}`);
+        }
+    }
+    return { fixtures, vars, warnings, manifest };
+}
+function assertPathContained(absolutePath, rootDir) {
+    const normalizedBase = resolve(rootDir) + "/";
+    if (!absolutePath.startsWith(normalizedBase) &&
+        absolutePath !== resolve(rootDir)) {
+        throw new Error(`Path traversal detected: "${absolutePath}" resolves outside rootDir "${rootDir}"`);
+    }
+}
+function resolveFixtureURI(uri, rootDir) {
+    if (uri.startsWith("file://")) {
+        const relativePath = uri.slice(7);
+        const absolutePath = resolve(rootDir, relativePath);
+        assertPathContained(absolutePath, rootDir);
+        return { scheme: "file", path: relativePath, absolutePath };
+    }
+    if (uri.startsWith("template://")) {
+        return { scheme: "template", path: uri.slice(11) };
+    }
+    if (uri.startsWith("sanity://")) {
+        return { scheme: "sanity", path: uri.slice(9) };
+    }
+    // Bare path — treat as file
+    const absolutePath = resolve(rootDir, uri);
+    assertPathContained(absolutePath, rootDir);
+    return { scheme: "file", path: uri, absolutePath };
+}
+// ---------------------------------------------------------------------------
+// Stage 2: Fetch
+// ---------------------------------------------------------------------------
+async function fetchFixtureContent(resolved, warnings) {
+    switch (resolved.scheme) {
+        case "file": {
+            if (!resolved.absolutePath || !existsSync(resolved.absolutePath)) {
+                warnings.push(`Fixture file not found: ${resolved.absolutePath ?? resolved.path}`);
+                return null;
+            }
+            return readFileSync(resolved.absolutePath, "utf-8");
+        }
+        case "template": {
+            // Template fixtures are resolved from built-in templates
+            // For v1, return a placeholder — template registry is a future enhancement
+            warnings.push(`Template fixture "${resolved.path}" — template registry not yet implemented, ` +
+                "returning placeholder content");
+            return `<!-- Template: ${resolved.path} -->\n`;
+        }
+        case "sanity": {
+            // Sanity fixtures require the DocFetcher port (injected at eval time)
+            // At compile time, return a deferred marker
+            warnings.push(`Sanity fixture "${resolved.path}" — deferred to eval time ` +
+                "(requires DocFetcher port)");
+            return `<!-- Sanity document: ${resolved.path} -->\n`;
+        }
+        default:
+            warnings.push(`Unknown fixture scheme: ${resolved.scheme}`);
+            return null;
+    }
+}
+// ---------------------------------------------------------------------------
+// Stage 3: Cache
+// ---------------------------------------------------------------------------
+function hashContent(content) {
+    return createHash("sha256").update(content).digest("hex");
+}
+function cacheFixture(cacheDir, hash, content) {
+    const cachePath = resolve(cacheDir, hash);
+    if (!existsSync(cachePath)) {
+        mkdirSync(dirname(cachePath), { recursive: true });
+        writeFileSync(cachePath, content);
+    }
+}
+// ---------------------------------------------------------------------------
+// Stage 4: Transform
+// ---------------------------------------------------------------------------
+function applyTransform(content, transform) {
+    if (!transform || transform === "none")
+        return content;
+    switch (transform) {
+        case "strip-html":
+            return content.replace(/<[^>]*>/g, "").trim();
+        case "extract-text":
+            // Remove all markup, normalize whitespace
+            return content
+                .replace(/<[^>]*>/g, " ")
+                .replace(/\s+/g, " ")
+                .trim();
+        case "truncate":
+            // Default truncation: 10,000 characters
+            return content.length > 10_000
+                ? content.slice(0, 10_000) + "\n... (truncated)"
+                : content;
+        default:
+            return content;
+    }
+}
+// ---------------------------------------------------------------------------
+// Stage 5: Inject
+// ---------------------------------------------------------------------------
+function injectIntoWorkingDir(workingDir, targetPath, content) {
+    const fullPath = resolve(workingDir, targetPath);
+    const normalizedBase = resolve(workingDir) + "/";
+    // Path containment: prevent targetPath like "../../etc/cron.d/evil"
+    if (!fullPath.startsWith(normalizedBase) &&
+        fullPath !== resolve(workingDir)) {
+        throw new Error(`Path traversal detected: "${targetPath}" resolves outside sandbox "${workingDir}"`);
+    }
+    mkdirSync(dirname(fullPath), { recursive: true });
+    writeFileSync(fullPath, content);
+}

package/dist/pipeline/compiler/sandbox/git-worktree-sandbox.d.ts

@@ -0,0 +1,20 @@
+/**
+ * GitWorktreeSandboxStrategy — sandbox using `git worktree` for repo-based tasks.
+ *
+ * Creates a git worktree at a specific ref, providing a deterministic
+ * starting state for tasks that modify a git repository.
+ *
+ * All git CLI calls use `execFileSync` (array form, no shell) to prevent
+ * injection from task-supplied values like git refs or repo paths.
+ *
+ * @see docs/exec-plans/architecture-overhaul/phase-4-agent-harness.md
+ */
+import type { SandboxArtifacts, SandboxInfo, SandboxProvisionOptions, SandboxStrategy } from "./sandbox-strategy.js";
+export declare class GitWorktreeSandboxStrategy implements SandboxStrategy {
+    readonly name = "Git Worktree";
+    readonly type: "git-worktree";
+    isAvailable(): Promise<boolean>;
+    provision(options: SandboxProvisionOptions): Promise<SandboxInfo>;
+    collectArtifacts(sandbox: SandboxInfo): Promise<SandboxArtifacts>;
+    teardown(sandbox: SandboxInfo): Promise<void>;
+}