@sanity/ailf 0.5.0 → 2.0.0

package/dist/pipeline/compiler/presets/sanity-literacy.js

@@ -0,0 +1,208 @@
+/**
+ * sanity-literacy preset — Sanity-specific domain configuration for literacy evaluation.
+ *
+ * This is a domain preset that targets the `literacy` mode base. It provides
+ * Sanity-specific configuration:
+ * - Sanity doc source definitions (production, branch, local)
+ * - Product feature registry for coverage auditing
+ * - DocFetcher factory (SanityDocFetcher)
+ * - Sanity fixture resolver (sanity:// scheme)
+ *
+ * Evaluation methodology (rubrics, scoring, prompts) is inherited from the
+ * `literacy` mode base — see mode-bases/literacy.ts.
+ *
+ * @see docs/exec-plans/architecture-overhaul/phase-8-scoring-storage-presets.md
+ */
+import { env } from "../../../_vendor/ailf-core/index.js";
+import { SanityDocFetcher } from "../../../adapters/doc-fetchers/index.js";
+// ---------------------------------------------------------------------------
+// Factory
+// ---------------------------------------------------------------------------
+/**
+ * Create the sanity-literacy preset with all Sanity-specific configuration.
+ *
+ * The `rootDir` option controls where the SanityDocFetcher resolves file
+ * paths. When omitted it defaults to `process.cwd()`, which matches the
+ * composition root's existing behavior.
+ */
+export function createSanityLiteracyPreset(options) {
+    const rootDir = options?.rootDir ?? process.cwd();
+    return {
+        name: "sanity-literacy",
+        manifest: {
+            name: "sanity-literacy",
+            version: "1.0.0",
+            description: "Documentation literacy evaluation for Sanity.io — measures how " +
+                "effectively documentation enables AI coding tools to implement " +
+                "features correctly.",
+            pluginApiVersion: 1,
+        },
+        // ── Mode ──────────────────────────────────────────────────
+        // Targets the literacy mode base. Evaluation methodology (rubrics,
+        // scoring, prompts) is inherited from mode-bases/literacy.ts.
+        mode: "literacy",
+        // ── Fixture resolvers ────────────────────────────────────
+        fixtureResolvers: [
+            {
+                scheme: "sanity://",
+                handlerModule: "./fixture-resolver.js",
+            },
+        ],
+        // ── Doc fetcher factory ──────────────────────────────────
+        // Closure captures rootDir so the registry can instantiate
+        // the fetcher without knowing about Sanity internals.
+        docFetcher: () => new SanityDocFetcher(rootDir),
+        // ── Source definitions ────────────────────────────────────
+        // Mirrors config/sources.ts — env() calls are evaluated at
+        // preset creation time, matching the existing runtime behavior.
+        sourceDefs: [
+            {
+                name: "production",
+                baseUrl: "https://www.sanity.io/docs",
+                projectId: env("SANITY_PROJECT_ID", "3do82whm"),
+                dataset: env("SANITY_DATASET", "next"),
+            },
+            {
+                name: "branch",
+                baseUrl: env("DOC_BASE_URL", ""),
+                projectId: env("SANITY_PROJECT_ID", "3do82whm"),
+                dataset: env("SANITY_DATASET", "next"),
+            },
+            {
+                name: "local",
+                baseUrl: `http://localhost:${env("DOC_LOCAL_PORT", "3001")}/docs`,
+                projectId: env("SANITY_PROJECT_ID", "3do82whm"),
+                dataset: env("SANITY_DATASET", "next"),
+            },
+        ],
+        // ── Feature registry ─────────────────────────────────────
+        // Complete product feature set for coverage auditing.
+        featureDefs: {
+            features: [
+                // Currently covered (have evaluation tasks)
+                {
+                    id: "groq",
+                    name: "GROQ Query Language",
+                    sections: ["content-lake"],
+                    status: "covered",
+                    area: "groq",
+                    priority: "critical",
+                },
+                {
+                    id: "visual-editing",
+                    name: "Visual Editing",
+                    sections: ["visual-editing"],
+                    status: "covered",
+                    area: "visual-editing",
+                    priority: "critical",
+                },
+                {
+                    id: "nextjs-live",
+                    name: "Next.js Live Preview",
+                    sections: ["visual-editing"],
+                    status: "covered",
+                    area: "nextjs-live",
+                    priority: "high",
+                },
+                {
+                    id: "functions",
+                    name: "Sanity Functions",
+                    sections: ["compute-and-ai"],
+                    status: "covered",
+                    area: "functions",
+                    priority: "high",
+                },
+                {
+                    id: "studio-setup",
+                    name: "Studio Configuration",
+                    sections: ["studio"],
+                    status: "covered",
+                    area: "studio-setup",
+                    priority: "high",
+                },
+                {
+                    id: "frameworks",
+                    name: "Framework Integration",
+                    sections: ["developer-guides"],
+                    status: "covered",
+                    area: "frameworks",
+                    priority: "high",
+                },
+                // Uncovered (no evaluation tasks yet)
+                {
+                    id: "portable-text",
+                    name: "Portable Text",
+                    sections: ["content-lake", "studio"],
+                    status: "uncovered",
+                    priority: "high",
+                },
+                {
+                    id: "image-assets",
+                    name: "Image & Asset Handling",
+                    sections: ["content-lake", "apis-and-sdks"],
+                    status: "uncovered",
+                    priority: "high",
+                },
+                {
+                    id: "mutations",
+                    name: "Mutations & Transactions",
+                    sections: ["content-lake", "apis-and-sdks"],
+                    status: "uncovered",
+                    priority: "high",
+                },
+                {
+                    id: "schemas",
+                    name: "Schema Types & Validation",
+                    sections: ["studio", "content-lake"],
+                    status: "uncovered",
+                    priority: "medium",
+                },
+                {
+                    id: "authentication",
+                    name: "Authentication & Access Control",
+                    sections: ["apis-and-sdks"],
+                    status: "uncovered",
+                    priority: "medium",
+                },
+                {
+                    id: "webhooks",
+                    name: "Webhooks",
+                    sections: ["content-lake"],
+                    status: "uncovered",
+                    priority: "medium",
+                },
+                {
+                    id: "realtime",
+                    name: "Real-time Listeners",
+                    sections: ["apis-and-sdks"],
+                    status: "uncovered",
+                    priority: "low",
+                },
+                {
+                    id: "ai-assist",
+                    name: "AI Assist",
+                    sections: ["compute-and-ai", "studio"],
+                    status: "uncovered",
+                    priority: "medium",
+                },
+            ],
+        },
+    };
+}
+// ---------------------------------------------------------------------------
+// Convenience constant — preset with default options for backward compat
+// ---------------------------------------------------------------------------
+/**
+ * Pre-built preset instance using default options (cwd for rootDir).
+ *
+ * Prefer `createSanityLiteracyPreset()` when you need to control rootDir
+ * or other options. This constant exists for backward compatibility with
+ * code that imports `sanityLiteracyPreset` directly.
+ */
+export const sanityLiteracyPreset = createSanityLiteracyPreset();
+/**
+ * Register the sanity-literacy preset in a plugin registry.
+ */
+export function registerSanityLiteracyPreset(registry) {
+    registry.registerPreset(sanityLiteracyPreset);
+}

package/dist/pipeline/compiler/promptfoo-compiler.d.ts

@@ -0,0 +1,96 @@
+/**
+ * PromptfooCompiler — compiles a TaskGraph into Promptfoo YAML configuration.
+ *
+ * The compiler is the core of the new architecture. It takes a validated
+ * TaskGraph and produces a Promptfoo config that can be executed via
+ * `promptfoo eval`.
+ *
+ * Compilation pipeline:
+ *   TaskGraph → resolve fixtures → resolve variables → map assertions
+ *             → assemble prompts → assemble providers → emit YAML
+ *
+ * This module exists alongside `generate-configs.ts` — it does NOT replace
+ * the existing codegen path. Phase 7 will swap callers over to the compiler.
+ *
+ * @see docs/exec-plans/architecture-overhaul/phase-2-config-compiler.md
+ */
+import type { ModeHandler, ModelsConfig, TaskGraph } from "../../_vendor/ailf-core/index.d.ts";
+import type { EvalMode } from "../../_vendor/ailf-shared/index.d.ts";
+import { type PromptfooAssertion } from "./assertion-mapper.js";
+/** A single Promptfoo test case */
+export interface PromptfooTestCase {
+    assert?: PromptfooAssertion[];
+    description: string;
+    vars: Record<string, unknown>;
+    /** Prompt filter — which prompts this test case runs against */
+    prompts?: string[];
+}
+/** A compiled Promptfoo configuration */
+export interface CompiledPromptfooConfig {
+    /** Description header */
+    description: string;
+    /** Prompt entries (id, label, raw template) */
+    prompts: PromptfooPrompt[];
+    /** Provider entries */
+    providers: PromptfooProvider[];
+    /** Test cases */
+    tests: PromptfooTestCase[];
+    /** Default test configuration */
+    defaultTest?: {
+        options?: Record<string, unknown>;
+    };
+    /** Output path for results */
+    outputPath?: string;
+}
+/** A Promptfoo prompt entry */
+export interface PromptfooPrompt {
+    id: string;
+    label: string;
+    raw: string;
+}
+/** A Promptfoo provider entry */
+export interface PromptfooProvider {
+    id: string;
+    label?: string;
+    config?: Record<string, unknown>;
+}
+/** Options for the Promptfoo compiler */
+export interface PromptfooCompilerOptions {
+    /** The evaluation mode */
+    mode: EvalMode;
+    /** Model registry */
+    models: ModelsConfig;
+    /** Root directory for fixture resolution */
+    rootDir: string;
+    /** Output path for the generated config */
+    outputPath?: string;
+    /** Grader model provider ID */
+    graderProvider?: string;
+    /** Prompt templates to use (explicit override) */
+    prompts?: PromptfooPrompt[];
+    /**
+     * Mode handler — when provided, the compiler calls handler.getPrompts()
+     * and uses those templates before falling back to explicit prompts or
+     * built-in defaults.
+     */
+    handler?: ModeHandler;
+}
+/** Result of compilation */
+export interface CompilationResult {
+    /** The compiled Promptfoo config */
+    config: CompiledPromptfooConfig;
+    /** Warnings generated during compilation */
+    warnings: string[];
+    /** Number of tasks compiled */
+    taskCount: number;
+    /** Number of test cases generated */
+    testCaseCount: number;
+}
+/**
+ * Compile a TaskGraph into a Promptfoo configuration.
+ *
+ * Traverses the graph in topological order, resolves fixtures and
+ * variables for each node, maps assertions, and assembles the final
+ * Promptfoo config.
+ */
+export declare function compileToPromptfoo(graph: TaskGraph, options: PromptfooCompilerOptions): CompilationResult;

package/dist/pipeline/compiler/promptfoo-compiler.js

@@ -0,0 +1,230 @@
+/**
+ * PromptfooCompiler — compiles a TaskGraph into Promptfoo YAML configuration.
+ *
+ * The compiler is the core of the new architecture. It takes a validated
+ * TaskGraph and produces a Promptfoo config that can be executed via
+ * `promptfoo eval`.
+ *
+ * Compilation pipeline:
+ *   TaskGraph → resolve fixtures → resolve variables → map assertions
+ *             → assemble prompts → assemble providers → emit YAML
+ *
+ * This module exists alongside `generate-configs.ts` — it does NOT replace
+ * the existing codegen path. Phase 7 will swap callers over to the compiler.
+ *
+ * @see docs/exec-plans/architecture-overhaul/phase-2-config-compiler.md
+ */
+import { mapAssertions } from "./assertion-mapper.js";
+import { resolveTaskFixtures } from "./fixture-resolver.js";
+import { LiteracyVariant } from "../normalize-mode.js";
+import { resolveVariables } from "./variable-resolver.js";
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+/**
+ * Compile a TaskGraph into a Promptfoo configuration.
+ *
+ * Traverses the graph in topological order, resolves fixtures and
+ * variables for each node, maps assertions, and assembles the final
+ * Promptfoo config.
+ */
+export function compileToPromptfoo(graph, options) {
+    const warnings = [];
+    const tests = [];
+    // Sort nodes by priority (topological order)
+    const sortedNodes = [...graph.nodes.values()].sort((a, b) => a.priority - b.priority);
+    // Compile each node into test cases
+    for (const node of sortedNodes) {
+        const compiled = compileNode(node, graph, options, warnings);
+        tests.push(...compiled);
+    }
+    // Build providers list from model registry
+    const providers = buildProviders(options.models, options.mode);
+    // Prompt resolution: handler-owned → explicit override → built-in defaults
+    const prompts = resolvePrompts(options);
+    const config = {
+        description: `AILF evaluation — ${options.mode} mode (${tests.length} test cases)`,
+        prompts,
+        providers,
+        tests,
+        ...(options.outputPath ? { outputPath: options.outputPath } : {}),
+        ...(options.graderProvider
+            ? {
+                defaultTest: {
+                    options: {
+                        provider: options.graderProvider,
+                    },
+                },
+            }
+            : {}),
+    };
+    return {
+        config,
+        taskCount: sortedNodes.length,
+        testCaseCount: tests.length,
+        warnings,
+    };
+}
+// ---------------------------------------------------------------------------
+// Node compilation
+// ---------------------------------------------------------------------------
+function compileNode(node, graph, options, warnings) {
+    // Resolve fixtures using a minimal GeneralizedTaskDefinition stub.
+    // The fixture resolver needs task.id and context.docs — we use the
+    // node's mode (propagated from the original task definition) to
+    // construct the correct variant stub.
+    const nodeMode = node.mode ?? options.mode ?? "literacy";
+    const fixtureResult = resolveTaskFixtures({
+        mode: nodeMode,
+        id: node.taskId,
+        title: node.taskId,
+        prompt: { text: node.resolvedPrompt },
+    }, node.resolvedVariables, { rootDir: options.rootDir });
+    warnings.push(...fixtureResult.warnings);
+    // Resolve dynamic variables
+    const varResult = resolveVariables(fixtureResult.updatedVars);
+    warnings.push(...varResult.warnings);
+    // Map assertions from the node's metadata
+    // For now, nodes carry assertion data in resolvedVariables.values.__assertions
+    // (set by the TaskGraphBuilder when it has generalized task data)
+    const rawAssertions = varResult.envelope.values.__assertions ?? [];
+    const { mapped: assertions, warnings: assertionWarnings } = mapAssertions(rawAssertions, { mode: options.mode, graderProvider: options.graderProvider });
+    warnings.push(...assertionWarnings);
+    // Build test case vars (exclude internal __ prefixed keys)
+    const vars = {};
+    for (const [key, value] of Object.entries(varResult.envelope.values)) {
+        if (!key.startsWith("__")) {
+            vars[key] = value;
+        }
+    }
+    // Create the gold test case
+    const goldTest = {
+        description: node.taskId,
+        vars,
+        ...(assertions.length > 0 ? { assert: assertions } : {}),
+    };
+    const tests = [goldTest];
+    // For literacy/baseline mode, also create a baseline variant (no docs).
+    // Route on the node's mode (from the task definition) rather than
+    // the global options.mode, so heterogeneous graphs compile correctly.
+    if (nodeMode === LiteracyVariant.STANDARD || nodeMode === "literacy") {
+        const baselineVars = { ...vars, docs: "" };
+        const baselineTest = {
+            description: `${node.taskId} [baseline]`,
+            vars: baselineVars,
+            prompts: ["without-docs"],
+            ...(assertions.length > 0 ? { assert: assertions } : {}),
+        };
+        tests.push(baselineTest);
+    }
+    return tests;
+}
+// ---------------------------------------------------------------------------
+// Provider assembly
+// ---------------------------------------------------------------------------
+/**
+ * Build the Promptfoo providers list from the model registry.
+ */
+function buildProviders(models, mode) {
+    const providers = [];
+    for (const model of models.models) {
+        // Check mode compatibility
+        if (model.modes && model.modes.length > 0) {
+            if (!modelMatchesMode(model, mode))
+                continue;
+        }
+        // Check env gate
+        if (model.env && !process.env[model.env]) {
+            continue; // Skip models whose API key isn't set
+        }
+        providers.push({
+            id: model.id,
+            label: model.label,
+            config: {
+                ...models.defaults,
+                ...model.config,
+            },
+        });
+    }
+    return providers;
+}
+/**
+ * Check if a model entry matches the current evaluation mode.
+ *
+ * Literacy mode defaults to baseline model matching. Variant-specific
+ * provider filtering is handled by the provider-assembler and
+ * generate-configs-step, not here.
+ */
+function modelMatchesMode(model, mode) {
+    if (!model.modes || model.modes.length === 0)
+        return true;
+    switch (mode) {
+        case "literacy":
+            return model.modes.includes(LiteracyVariant.STANDARD);
+        default:
+            // Non-literacy modes accept all models by default
+            return true;
+    }
+}
+// ---------------------------------------------------------------------------
+// Prompt resolution
+// ---------------------------------------------------------------------------
+/**
+ * Resolve prompts with a three-level fallback chain:
+ *   1. handler.getPrompts() — mode-handler-owned templates
+ *   2. options.prompts — explicit caller-provided templates
+ *   3. buildDefaultPrompts() — built-in defaults per mode
+ */
+function resolvePrompts(options) {
+    // 1. Check handler-owned prompts
+    const handlerPrompts = options.handler?.getPrompts?.();
+    if (handlerPrompts && Object.keys(handlerPrompts).length > 0) {
+        return Object.values(handlerPrompts).map(promptTemplateToPromptfoo);
+    }
+    // 2. Check explicit override
+    if (options.prompts)
+        return options.prompts;
+    // 3. Built-in defaults
+    return buildDefaultPrompts(options.mode);
+}
+/**
+ * Convert a PromptTemplate (core port type) to a PromptfooPrompt (compiler type).
+ */
+function promptTemplateToPromptfoo(pt) {
+    return { id: pt.id, label: pt.label, raw: pt.template };
+}
+// ---------------------------------------------------------------------------
+// Default prompts
+// ---------------------------------------------------------------------------
+/**
+ * Build default prompt entries for a mode.
+ *
+ * Handler-owned prompts (via getPrompts()) take precedence over these
+ * built-in defaults. This fallback exists for modes that haven't yet
+ * migrated to handler-owned prompts.
+ */
+function buildDefaultPrompts(mode) {
+    switch (mode) {
+        case "literacy":
+            return [
+                {
+                    id: "with-docs",
+                    label: "With documentation context",
+                    raw: "{{task}}\n\nDocumentation context:\n{{docs}}",
+                },
+                {
+                    id: "without-docs",
+                    label: "Without documentation context",
+                    raw: "{{task}}",
+                },
+            ];
+        default:
+            return [
+                {
+                    id: "default",
+                    label: "Default prompt",
+                    raw: "{{task}}",
+                },
+            ];
+    }
+}

package/dist/pipeline/compiler/provider-assembler.d.ts

@@ -0,0 +1,39 @@
+/**
+ * provider-assembler.ts — Build per-mode provider arrays from models config.
+ *
+ * Replicates the provider-building logic from the legacy generate-configs.ts
+ * so the new compiler produces identical provider configurations.
+ *
+ * Separated into its own module so GenerateConfigsStep can import it
+ * without pulling in the full legacy generate-configs machinery.
+ */
+import { type ModelsConfig } from "../../_vendor/ailf-core/index.d.ts";
+import type { ResolvedSourceConfig } from "../../sources.js";
+/**
+ * Provider arrays grouped by literacy variant.
+ *
+ * These keys are literacy variant names (not EvalMode values). Each variant
+ * needs a different set of model providers with variant-specific config
+ * (e.g., agentic providers carry tool-use config, observed providers carry
+ * observer instrumentation).
+ */
+export interface LiteracyVariantProviders {
+    baseline: Record<string, unknown>[];
+    agentic: Record<string, unknown>[];
+    observed: Record<string, unknown>[];
+}
+/** @deprecated Use LiteracyVariantProviders — kept for backward compatibility */
+export type AssembledProviders = LiteracyVariantProviders;
+/** Result of loading models and assembling providers */
+export interface ModelsAndProviders {
+    models: ModelsConfig;
+    providers: LiteracyVariantProviders;
+}
+/**
+ * Load models config and assemble provider arrays for literacy variants.
+ *
+ * Returns provider arrays keyed by literacy variant name (baseline,
+ * agentic, observed). These are consumed by the YAML writer to produce
+ * the per-variant promptfoo config files.
+ */
+export declare function loadModelsAndProviders(rootDir: string, source?: ResolvedSourceConfig, searchMode?: string, allowedOrigins?: string[]): ModelsAndProviders;