@sanity/ailf 0.4.1 → 1.0.0

package/dist/pipeline/compiler/presets/sanity-literacy.d.ts

@@ -0,0 +1,45 @@
+/**
+ * sanity-literacy preset — the built-in documentation literacy evaluation.
+ *
+ * Packages ALL Sanity-specific configuration into a single preset:
+ * - Literacy mode handler registration
+ * - Prompt templates (with-docs, without-docs, agentic)
+ * - Rubric templates (task-completion, code-correctness, doc-coverage)
+ * - Scoring profiles (default, output-only)
+ * - Sanity doc source definitions (production, branch, local)
+ * - Product feature registry for coverage auditing
+ * - DocFetcher factory (SanityDocFetcher)
+ * - Standard assertions and fixture resolvers
+ *
+ * Use `createSanityLiteracyPreset()` to get a fully configured preset
+ * with a docFetcher factory bound to a specific rootDir.
+ *
+ * @see docs/exec-plans/architecture-overhaul/phase-8-scoring-storage-presets.md
+ */
+import { type PresetDefinition } from "../../../_vendor/ailf-core/index.d.ts";
+export interface SanityLiteracyPresetOptions {
+    /** Root directory for SanityDocFetcher file path resolution (defaults to cwd) */
+    rootDir?: string;
+}
+/**
+ * Create the sanity-literacy preset with all Sanity-specific configuration.
+ *
+ * The `rootDir` option controls where the SanityDocFetcher resolves file
+ * paths. When omitted it defaults to `process.cwd()`, which matches the
+ * composition root's existing behavior.
+ */
+export declare function createSanityLiteracyPreset(options?: SanityLiteracyPresetOptions): PresetDefinition;
+/**
+ * Pre-built preset instance using default options (cwd for rootDir).
+ *
+ * Prefer `createSanityLiteracyPreset()` when you need to control rootDir
+ * or other options. This constant exists for backward compatibility with
+ * code that imports `sanityLiteracyPreset` directly.
+ */
+export declare const sanityLiteracyPreset: PresetDefinition;
+/**
+ * Register the sanity-literacy preset in a plugin registry.
+ */
+export declare function registerSanityLiteracyPreset(registry: {
+    registerPreset(preset: PresetDefinition): void;
+}): void;

package/dist/pipeline/compiler/presets/sanity-literacy.js

@@ -0,0 +1,354 @@
+/**
+ * sanity-literacy preset — the built-in documentation literacy evaluation.
+ *
+ * Packages ALL Sanity-specific configuration into a single preset:
+ * - Literacy mode handler registration
+ * - Prompt templates (with-docs, without-docs, agentic)
+ * - Rubric templates (task-completion, code-correctness, doc-coverage)
+ * - Scoring profiles (default, output-only)
+ * - Sanity doc source definitions (production, branch, local)
+ * - Product feature registry for coverage auditing
+ * - DocFetcher factory (SanityDocFetcher)
+ * - Standard assertions and fixture resolvers
+ *
+ * Use `createSanityLiteracyPreset()` to get a fully configured preset
+ * with a docFetcher factory bound to a specific rootDir.
+ *
+ * @see docs/exec-plans/architecture-overhaul/phase-8-scoring-storage-presets.md
+ */
+import { env } from "../../../_vendor/ailf-core/index.js";
+import { SanityDocFetcher } from "../../../adapters/doc-fetchers/index.js";
+import { LITERACY_PROMPT_TEMPLATES } from "../mode-handlers/literacy-handler.js";
+// ---------------------------------------------------------------------------
+// Factory
+// ---------------------------------------------------------------------------
+/**
+ * Create the sanity-literacy preset with all Sanity-specific configuration.
+ *
+ * The `rootDir` option controls where the SanityDocFetcher resolves file
+ * paths. When omitted it defaults to `process.cwd()`, which matches the
+ * composition root's existing behavior.
+ */
+export function createSanityLiteracyPreset(options) {
+    const rootDir = options?.rootDir ?? process.cwd();
+    return {
+        name: "sanity-literacy",
+        manifest: {
+            name: "sanity-literacy",
+            version: "1.0.0",
+            description: "Documentation literacy evaluation for Sanity.io — measures how " +
+                "effectively documentation enables AI coding tools to implement " +
+                "features correctly.",
+            pluginApiVersion: 1,
+        },
+        // ── Mode handler ─────────────────────────────────────────
+        modes: [
+            {
+                id: "literacy",
+                label: "Documentation Literacy",
+                validProviderPatterns: ["^openai:", "^anthropic:", "^file://"],
+                rubricTemplateIds: [
+                    "task-completion",
+                    "code-correctness",
+                    "doc-coverage",
+                ],
+                handlerModule: "./mode-handlers/literacy-handler.js",
+            },
+        ],
+        // ── Assertions ───────────────────────────────────────────
+        assertions: [
+            {
+                type: "contains",
+                label: "Contains text",
+                compatibleModes: ["literacy", "knowledge-probe", "mcp-server"],
+                handlerModule: "promptfoo:builtin",
+            },
+            {
+                type: "contains-all",
+                label: "Contains all texts",
+                compatibleModes: ["literacy", "knowledge-probe", "mcp-server"],
+                handlerModule: "promptfoo:builtin",
+            },
+            {
+                type: "contains-any",
+                label: "Contains any text",
+                compatibleModes: ["literacy", "knowledge-probe", "mcp-server"],
+                handlerModule: "promptfoo:builtin",
+            },
+            {
+                type: "equals",
+                label: "Exact match",
+                compatibleModes: ["literacy"],
+                handlerModule: "promptfoo:builtin",
+            },
+            {
+                type: "regex",
+                label: "Regex match",
+                compatibleModes: ["literacy", "knowledge-probe"],
+                handlerModule: "promptfoo:builtin",
+            },
+            {
+                type: "is-json",
+                label: "Valid JSON",
+                compatibleModes: ["literacy", "mcp-server"],
+                handlerModule: "promptfoo:builtin",
+            },
+            {
+                type: "javascript",
+                label: "JavaScript assertion",
+                compatibleModes: [
+                    "literacy",
+                    "mcp-server",
+                    "agent-harness",
+                    "knowledge-probe",
+                    "custom",
+                ],
+                handlerModule: "promptfoo:builtin",
+            },
+            {
+                type: "llm-rubric",
+                label: "LLM-graded rubric",
+                compatibleModes: [
+                    "literacy",
+                    "mcp-server",
+                    "agent-harness",
+                    "knowledge-probe",
+                    "custom",
+                ],
+                handlerModule: "promptfoo:builtin",
+            },
+            {
+                type: "similar",
+                label: "Semantic similarity",
+                compatibleModes: ["literacy", "knowledge-probe"],
+                handlerModule: "promptfoo:builtin",
+            },
+        ],
+        // ── Rubric templates ─────────────────────────────────────
+        rubricTemplates: [
+            {
+                id: "task-completion",
+                dimension: "task-completion",
+                header: "Score task completion from 0 to 100:",
+                scale: [
+                    "0: Couldn't attempt — missing critical information",
+                    "20: Attempted but fundamentally wrong approach",
+                    "50: Partial implementation — major functional gaps",
+                    "80: Mostly complete — minor issues or missing edge cases",
+                    "100: Fully functional code — works as expected",
+                ],
+                criteriaLabel: "Must demonstrate:",
+            },
+            {
+                id: "code-correctness",
+                dimension: "code-correctness",
+                header: "Score code correctness from 0 to 100:",
+                scale: [
+                    "0: Broken code, syntax errors, or deprecated APIs",
+                    "30: Works but uses anti-patterns or inefficient approaches",
+                    "50: Works but not idiomatic",
+                    "80: Follows most best practices",
+                    "100: Follows all best practices, idiomatic implementation",
+                ],
+                criteriaLabel: "Check for:",
+            },
+            {
+                id: "doc-coverage",
+                dimension: "doc-coverage",
+                header: "Score documentation coverage from 0 to 100:",
+                scale: [
+                    "0: Had to hallucinate/guess most implementation details",
+                    "30: Significant gaps — filled with assumptions",
+                    "50: Some gaps — inferred from partial information",
+                    "80: Minor gaps — almost everything was documented",
+                    "100: Complete coverage — all necessary info was in docs",
+                ],
+            },
+        ],
+        // ── Fixture resolvers ────────────────────────────────────
+        fixtureResolvers: [
+            {
+                scheme: "sanity://",
+                handlerModule: "./fixture-resolver.js",
+            },
+        ],
+        // ── Prompt templates (from literacy handler) ─────────────
+        promptTemplates: LITERACY_PROMPT_TEMPLATES,
+        // ── Scoring profiles ─────────────────────────────────────
+        // Literacy-relevant profiles only; mode-specific profiles for
+        // mcp-server, knowledge-probe, etc. belong in their own presets.
+        scoringProfiles: {
+            default: {
+                "task-completion": 0.5,
+                "code-correctness": 0.25,
+                "doc-coverage": 0.25,
+            },
+            "output-only": {
+                "task-completion": 0.6,
+                "code-correctness": 0.4,
+            },
+        },
+        // ── Doc fetcher factory ──────────────────────────────────
+        // Closure captures rootDir so the registry can instantiate
+        // the fetcher without knowing about Sanity internals.
+        docFetcher: () => new SanityDocFetcher(rootDir),
+        // ── Source definitions ────────────────────────────────────
+        // Mirrors config/sources.ts — env() calls are evaluated at
+        // preset creation time, matching the existing runtime behavior.
+        sourceDefs: [
+            {
+                name: "production",
+                baseUrl: "https://www.sanity.io/docs",
+                projectId: env("SANITY_PROJECT_ID", "3do82whm"),
+                dataset: env("SANITY_DATASET", "next"),
+            },
+            {
+                name: "branch",
+                baseUrl: env("DOC_BASE_URL", ""),
+                projectId: env("SANITY_PROJECT_ID", "3do82whm"),
+                dataset: env("SANITY_DATASET", "next"),
+            },
+            {
+                name: "local",
+                baseUrl: `http://localhost:${env("DOC_LOCAL_PORT", "3001")}/docs`,
+                projectId: env("SANITY_PROJECT_ID", "3do82whm"),
+                dataset: env("SANITY_DATASET", "next"),
+            },
+        ],
+        // ── Feature registry ─────────────────────────────────────
+        // Complete product feature set for coverage auditing.
+        featureDefs: {
+            features: [
+                // Currently covered (have evaluation tasks)
+                {
+                    id: "groq",
+                    name: "GROQ Query Language",
+                    sections: ["content-lake"],
+                    status: "covered",
+                    area: "groq",
+                    priority: "critical",
+                    taskCount: 3,
+                },
+                {
+                    id: "visual-editing",
+                    name: "Visual Editing",
+                    sections: ["visual-editing"],
+                    status: "covered",
+                    area: "visual-editing",
+                    priority: "critical",
+                    taskCount: 1,
+                },
+                {
+                    id: "nextjs-live",
+                    name: "Next.js Live Preview",
+                    sections: ["visual-editing"],
+                    status: "covered",
+                    area: "nextjs-live",
+                    priority: "high",
+                    taskCount: 2,
+                },
+                {
+                    id: "functions",
+                    name: "Sanity Functions",
+                    sections: ["compute-and-ai"],
+                    status: "covered",
+                    area: "functions",
+                    priority: "high",
+                    taskCount: 2,
+                },
+                {
+                    id: "studio-setup",
+                    name: "Studio Configuration",
+                    sections: ["studio"],
+                    status: "covered",
+                    area: "studio-setup",
+                    priority: "high",
+                    taskCount: 1,
+                },
+                {
+                    id: "frameworks",
+                    name: "Framework Integration",
+                    sections: ["developer-guides"],
+                    status: "covered",
+                    area: "frameworks",
+                    priority: "high",
+                    taskCount: 2,
+                },
+                // Uncovered (no evaluation tasks yet)
+                {
+                    id: "portable-text",
+                    name: "Portable Text",
+                    sections: ["content-lake", "studio"],
+                    status: "uncovered",
+                    priority: "high",
+                },
+                {
+                    id: "image-assets",
+                    name: "Image & Asset Handling",
+                    sections: ["content-lake", "apis-and-sdks"],
+                    status: "uncovered",
+                    priority: "high",
+                },
+                {
+                    id: "mutations",
+                    name: "Mutations & Transactions",
+                    sections: ["content-lake", "apis-and-sdks"],
+                    status: "uncovered",
+                    priority: "high",
+                },
+                {
+                    id: "schemas",
+                    name: "Schema Types & Validation",
+                    sections: ["studio", "content-lake"],
+                    status: "uncovered",
+                    priority: "medium",
+                },
+                {
+                    id: "authentication",
+                    name: "Authentication & Access Control",
+                    sections: ["apis-and-sdks"],
+                    status: "uncovered",
+                    priority: "medium",
+                },
+                {
+                    id: "webhooks",
+                    name: "Webhooks",
+                    sections: ["content-lake"],
+                    status: "uncovered",
+                    priority: "medium",
+                },
+                {
+                    id: "realtime",
+                    name: "Real-time Listeners",
+                    sections: ["apis-and-sdks"],
+                    status: "uncovered",
+                    priority: "low",
+                },
+                {
+                    id: "ai-assist",
+                    name: "AI Assist",
+                    sections: ["compute-and-ai", "studio"],
+                    status: "uncovered",
+                    priority: "medium",
+                },
+            ],
+        },
+    };
+}
+// ---------------------------------------------------------------------------
+// Convenience constant — preset with default options for backward compat
+// ---------------------------------------------------------------------------
+/**
+ * Pre-built preset instance using default options (cwd for rootDir).
+ *
+ * Prefer `createSanityLiteracyPreset()` when you need to control rootDir
+ * or other options. This constant exists for backward compatibility with
+ * code that imports `sanityLiteracyPreset` directly.
+ */
+export const sanityLiteracyPreset = createSanityLiteracyPreset();
+/**
+ * Register the sanity-literacy preset in a plugin registry.
+ */
+export function registerSanityLiteracyPreset(registry) {
+    registry.registerPreset(sanityLiteracyPreset);
+}

package/dist/pipeline/compiler/promptfoo-compiler.d.ts

@@ -0,0 +1,96 @@
+/**
+ * PromptfooCompiler — compiles a TaskGraph into Promptfoo YAML configuration.
+ *
+ * The compiler is the core of the new architecture. It takes a validated
+ * TaskGraph and produces a Promptfoo config that can be executed via
+ * `promptfoo eval`.
+ *
+ * Compilation pipeline:
+ *   TaskGraph → resolve fixtures → resolve variables → map assertions
+ *             → assemble prompts → assemble providers → emit YAML
+ *
+ * This module exists alongside `generate-configs.ts` — it does NOT replace
+ * the existing codegen path. Phase 7 will swap callers over to the compiler.
+ *
+ * @see docs/exec-plans/architecture-overhaul/phase-2-config-compiler.md
+ */
+import type { ModeHandler, ModelsConfig, TaskGraph } from "../../_vendor/ailf-core/index.d.ts";
+import type { EvalMode } from "../../_vendor/ailf-shared/index.d.ts";
+import { type PromptfooAssertion } from "./assertion-mapper.js";
+/** A single Promptfoo test case */
+export interface PromptfooTestCase {
+    assert?: PromptfooAssertion[];
+    description: string;
+    vars: Record<string, unknown>;
+    /** Prompt filter — which prompts this test case runs against */
+    prompts?: string[];
+}
+/** A compiled Promptfoo configuration */
+export interface CompiledPromptfooConfig {
+    /** Description header */
+    description: string;
+    /** Prompt entries (id, label, raw template) */
+    prompts: PromptfooPrompt[];
+    /** Provider entries */
+    providers: PromptfooProvider[];
+    /** Test cases */
+    tests: PromptfooTestCase[];
+    /** Default test configuration */
+    defaultTest?: {
+        options?: Record<string, unknown>;
+    };
+    /** Output path for results */
+    outputPath?: string;
+}
+/** A Promptfoo prompt entry */
+export interface PromptfooPrompt {
+    id: string;
+    label: string;
+    raw: string;
+}
+/** A Promptfoo provider entry */
+export interface PromptfooProvider {
+    id: string;
+    label?: string;
+    config?: Record<string, unknown>;
+}
+/** Options for the Promptfoo compiler */
+export interface PromptfooCompilerOptions {
+    /** The evaluation mode */
+    mode: EvalMode;
+    /** Model registry */
+    models: ModelsConfig;
+    /** Root directory for fixture resolution */
+    rootDir: string;
+    /** Output path for the generated config */
+    outputPath?: string;
+    /** Grader model provider ID */
+    graderProvider?: string;
+    /** Prompt templates to use (explicit override) */
+    prompts?: PromptfooPrompt[];
+    /**
+     * Mode handler — when provided, the compiler calls handler.getPrompts()
+     * and uses those templates before falling back to explicit prompts or
+     * built-in defaults.
+     */
+    handler?: ModeHandler;
+}
+/** Result of compilation */
+export interface CompilationResult {
+    /** The compiled Promptfoo config */
+    config: CompiledPromptfooConfig;
+    /** Warnings generated during compilation */
+    warnings: string[];
+    /** Number of tasks compiled */
+    taskCount: number;
+    /** Number of test cases generated */
+    testCaseCount: number;
+}
+/**
+ * Compile a TaskGraph into a Promptfoo configuration.
+ *
+ * Traverses the graph in topological order, resolves fixtures and
+ * variables for each node, maps assertions, and assembles the final
+ * Promptfoo config.
+ */
+export declare function compileToPromptfoo(graph: TaskGraph, options: PromptfooCompilerOptions): CompilationResult;