@sanity/ailf 2.0.1 → 2.1.0

package/dist/_vendor/ailf-tasks/schemas.d.ts

@@ -1,198 +0,0 @@
-/**
- * schemas.ts — Zod schemas for repo-based task definitions.
- *
- * Validates .ailf/tasks/*.yaml task files from external repositories.
- * These schemas are the contract between external repos and the AILF eval
- * pipeline — they define exactly what fields are accepted, with friendly
- * error messages for authors writing task YAML by hand.
- *
- * This module is the single source of truth for task schemas. The eval
- * package re-exports from here to avoid duplication.
- *
- * @see docs/exec-plans/tasks-as-content/phase-4-repo-based-tasks.md
- */
-import { z } from "zod";
-/**
- * The set of assertion types allowed in repo-based task files.
- *
- * This is a curated subset of Promptfoo assertion types — we expose only the
- * types that are stable, well-documented, and useful for external authors.
- */
-export declare const CURATED_ASSERTION_TYPES: readonly ["llm-rubric", "contains", "contains-any", "contains-all", "not-contains", "icontains", "icontains-any", "regex", "javascript", "similar", "cost", "latency"];
-export type CuratedAssertionType = (typeof CURATED_ASSERTION_TYPES)[number];
-/**
- * Valid rubric template names — must match keys in config/rubrics.yaml.
- */
-export declare const RUBRIC_TEMPLATE_NAMES: readonly ["task-completion", "code-correctness", "doc-coverage"];
-export type RubricTemplateName = (typeof RUBRIC_TEMPLATE_NAMES)[number];
-/**
- * Zod schema for a single repo-based task definition.
- *
- * This is the external-author-facing contract. Field names are camelCase
- * to match the Content Lake document schema (ailf.task).
- */
-export declare const RepoTaskSchema: z.ZodObject<{
-    id: z.ZodString;
-    description: z.ZodString;
-    status: z.ZodDefault<z.ZodOptional<z.ZodEnum<{
-        active: "active";
-        draft: "draft";
-        paused: "paused";
-        archived: "archived";
-    }>>>;
-    featureArea: z.ZodString;
-    tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
-    canonicalDocs: z.ZodDefault<z.ZodOptional<z.ZodArray<z.ZodUnion<readonly [z.ZodObject<{
-        id: z.ZodString;
-        reason: z.ZodDefault<z.ZodOptional<z.ZodString>>;
-        slug: z.ZodOptional<z.ZodString>;
-        path: z.ZodOptional<z.ZodString>;
-    }, z.core.$strip>, z.ZodObject<{
-        slug: z.ZodString;
-        reason: z.ZodDefault<z.ZodOptional<z.ZodString>>;
-    }, z.core.$strip>, z.ZodObject<{
-        path: z.ZodString;
-        reason: z.ZodDefault<z.ZodOptional<z.ZodString>>;
-    }, z.core.$strip>, z.ZodObject<{
-        perspective: z.ZodString;
-        reason: z.ZodDefault<z.ZodOptional<z.ZodString>>;
-    }, z.core.$strip>]>>>>;
-    vars: z.ZodOptional<z.ZodObject<{
-        task: z.ZodString;
-    }, z.core.$loose>>;
-    assert: z.ZodArray<z.ZodUnion<readonly [z.ZodObject<{
-        type: z.ZodLiteral<"llm-rubric">;
-        template: z.ZodEnum<{
-            "task-completion": "task-completion";
-            "code-correctness": "code-correctness";
-            "doc-coverage": "doc-coverage";
-        }>;
-        criteria: z.ZodArray<z.ZodString>;
-        weight: z.ZodOptional<z.ZodNumber>;
-    }, z.core.$strip>, z.ZodObject<{
-        type: z.ZodEnum<{
-            "llm-rubric": "llm-rubric";
-            contains: "contains";
-            "contains-any": "contains-any";
-            "contains-all": "contains-all";
-            "not-contains": "not-contains";
-            icontains: "icontains";
-            "icontains-any": "icontains-any";
-            regex: "regex";
-            javascript: "javascript";
-            similar: "similar";
-            cost: "cost";
-            latency: "latency";
-        }>;
-        value: z.ZodOptional<z.ZodUnknown>;
-        threshold: z.ZodOptional<z.ZodNumber>;
-        weight: z.ZodOptional<z.ZodNumber>;
-    }, z.core.$loose>]>>;
-    baseline: z.ZodOptional<z.ZodObject<{
-        enabled: z.ZodOptional<z.ZodBoolean>;
-        rubric: z.ZodOptional<z.ZodEnum<{
-            abbreviated: "abbreviated";
-            full: "full";
-            none: "none";
-        }>>;
-    }, z.core.$strip>>;
-    docCoverage: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
-    referenceSolution: z.ZodOptional<z.ZodString>;
-    execution: z.ZodOptional<z.ZodObject<{
-        enabled: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
-        blocking: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
-        threshold: z.ZodOptional<z.ZodObject<{
-            score: z.ZodOptional<z.ZodNumber>;
-        }, z.core.$strip>>;
-        trigger: z.ZodOptional<z.ZodObject<{
-            branches: z.ZodOptional<z.ZodArray<z.ZodString>>;
-            paths: z.ZodOptional<z.ZodArray<z.ZodString>>;
-        }, z.core.$strip>>;
-        source: z.ZodOptional<z.ZodString>;
-    }, z.core.$strip>>;
-}, z.core.$strip>;
-export type RepoTask = z.infer<typeof RepoTaskSchema>;
-/**
- * Schema for an array of repo tasks — what a single .ailf/tasks/*.yaml file
- * contains. Each file must define at least one task.
- */
-export declare const RepoTaskFileSchema: z.ZodArray<z.ZodObject<{
-    id: z.ZodString;
-    description: z.ZodString;
-    status: z.ZodDefault<z.ZodOptional<z.ZodEnum<{
-        active: "active";
-        draft: "draft";
-        paused: "paused";
-        archived: "archived";
-    }>>>;
-    featureArea: z.ZodString;
-    tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
-    canonicalDocs: z.ZodDefault<z.ZodOptional<z.ZodArray<z.ZodUnion<readonly [z.ZodObject<{
-        id: z.ZodString;
-        reason: z.ZodDefault<z.ZodOptional<z.ZodString>>;
-        slug: z.ZodOptional<z.ZodString>;
-        path: z.ZodOptional<z.ZodString>;
-    }, z.core.$strip>, z.ZodObject<{
-        slug: z.ZodString;
-        reason: z.ZodDefault<z.ZodOptional<z.ZodString>>;
-    }, z.core.$strip>, z.ZodObject<{
-        path: z.ZodString;
-        reason: z.ZodDefault<z.ZodOptional<z.ZodString>>;
-    }, z.core.$strip>, z.ZodObject<{
-        perspective: z.ZodString;
-        reason: z.ZodDefault<z.ZodOptional<z.ZodString>>;
-    }, z.core.$strip>]>>>>;
-    vars: z.ZodOptional<z.ZodObject<{
-        task: z.ZodString;
-    }, z.core.$loose>>;
-    assert: z.ZodArray<z.ZodUnion<readonly [z.ZodObject<{
-        type: z.ZodLiteral<"llm-rubric">;
-        template: z.ZodEnum<{
-            "task-completion": "task-completion";
-            "code-correctness": "code-correctness";
-            "doc-coverage": "doc-coverage";
-        }>;
-        criteria: z.ZodArray<z.ZodString>;
-        weight: z.ZodOptional<z.ZodNumber>;
-    }, z.core.$strip>, z.ZodObject<{
-        type: z.ZodEnum<{
-            "llm-rubric": "llm-rubric";
-            contains: "contains";
-            "contains-any": "contains-any";
-            "contains-all": "contains-all";
-            "not-contains": "not-contains";
-            icontains: "icontains";
-            "icontains-any": "icontains-any";
-            regex: "regex";
-            javascript: "javascript";
-            similar: "similar";
-            cost: "cost";
-            latency: "latency";
-        }>;
-        value: z.ZodOptional<z.ZodUnknown>;
-        threshold: z.ZodOptional<z.ZodNumber>;
-        weight: z.ZodOptional<z.ZodNumber>;
-    }, z.core.$loose>]>>;
-    baseline: z.ZodOptional<z.ZodObject<{
-        enabled: z.ZodOptional<z.ZodBoolean>;
-        rubric: z.ZodOptional<z.ZodEnum<{
-            abbreviated: "abbreviated";
-            full: "full";
-            none: "none";
-        }>>;
-    }, z.core.$strip>>;
-    docCoverage: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
-    referenceSolution: z.ZodOptional<z.ZodString>;
-    execution: z.ZodOptional<z.ZodObject<{
-        enabled: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
-        blocking: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
-        threshold: z.ZodOptional<z.ZodObject<{
-            score: z.ZodOptional<z.ZodNumber>;
-        }, z.core.$strip>>;
-        trigger: z.ZodOptional<z.ZodObject<{
-            branches: z.ZodOptional<z.ZodArray<z.ZodString>>;
-            paths: z.ZodOptional<z.ZodArray<z.ZodString>>;
-        }, z.core.$strip>>;
-        source: z.ZodOptional<z.ZodString>;
-    }, z.core.$strip>>;
-}, z.core.$strip>>;

package/dist/_vendor/ailf-tasks/schemas.js

@@ -1,180 +0,0 @@
-/**
- * schemas.ts — Zod schemas for repo-based task definitions.
- *
- * Validates .ailf/tasks/*.yaml task files from external repositories.
- * These schemas are the contract between external repos and the AILF eval
- * pipeline — they define exactly what fields are accepted, with friendly
- * error messages for authors writing task YAML by hand.
- *
- * This module is the single source of truth for task schemas. The eval
- * package re-exports from here to avoid duplication.
- *
- * @see docs/exec-plans/tasks-as-content/phase-4-repo-based-tasks.md
- */
-import { z } from "zod";
-// ---------------------------------------------------------------------------
-// Constants — curated assertion types and rubric template names
-// ---------------------------------------------------------------------------
-/**
- * The set of assertion types allowed in repo-based task files.
- *
- * This is a curated subset of Promptfoo assertion types — we expose only the
- * types that are stable, well-documented, and useful for external authors.
- */
-export const CURATED_ASSERTION_TYPES = [
-    "llm-rubric",
-    "contains",
-    "contains-any",
-    "contains-all",
-    "not-contains",
-    "icontains",
-    "icontains-any",
-    "regex",
-    "javascript",
-    "similar",
-    "cost",
-    "latency",
-];
-/**
- * Valid rubric template names — must match keys in config/rubrics.yaml.
- */
-export const RUBRIC_TEMPLATE_NAMES = [
-    "task-completion",
-    "code-correctness",
-    "doc-coverage",
-];
-// ---------------------------------------------------------------------------
-// Assertion schemas
-// ---------------------------------------------------------------------------
-/**
- * Polymorphic canonical doc reference — discriminated by key presence.
- * Exactly one resolution key (slug, path, id, or perspective) must be present.
- *
- * @see docs/design-docs/canonical-doc-resolution.md
- */
-const SlugDocRefSchema = z.object({
-    slug: z.string().min(1),
-    reason: z.string().optional().default(""),
-});
-const PathDocRefSchema = z.object({
-    path: z.string().min(1),
-    reason: z.string().optional().default(""),
-});
-const IdDocRefSchema = z.object({
-    id: z.string().min(1),
-    reason: z.string().optional().default(""),
-    /** Human-readable slug annotation (not used for resolution) */
-    slug: z.string().optional(),
-    /** Human-readable path annotation (not used for resolution) */
-    path: z.string().optional(),
-});
-const PerspectiveDocRefSchema = z.object({
-    perspective: z.string().min(1),
-    reason: z.string().optional().default(""),
-});
-// Order matters: IdDocRefSchema first because it may also carry `slug`
-// and `path` as optional annotations. Zod tries schemas in order, so
-// entries like `{ id: "...", slug: "..." }` must match IdDocRefSchema
-// (not SlugDocRefSchema).
-const CanonicalDocRefSchema = z.union([
-    IdDocRefSchema,
-    SlugDocRefSchema,
-    PathDocRefSchema,
-    PerspectiveDocRefSchema,
-]);
-/**
- * A templated LLM-rubric assertion — uses one of the predefined rubric
- * templates with author-supplied criteria.
- */
-const TemplatedAssertionSchema = z.object({
-    type: z.literal("llm-rubric"),
-    template: z.enum(RUBRIC_TEMPLATE_NAMES),
-    criteria: z.array(z.string().min(1)).min(1),
-    weight: z.number().optional(),
-});
-/**
- * A value-based assertion (contains, regex, cost, etc.). Uses .passthrough()
- * to allow extra fields for future extension without schema breakage.
- */
-const ValueAssertionSchema = z
-    .object({
-    type: z.enum(CURATED_ASSERTION_TYPES),
-    value: z.unknown().optional(),
-    threshold: z.number().optional(),
-    weight: z.number().optional(),
-})
-    .passthrough();
-/** Union of all supported assertion shapes. */
-const AssertionSchema = z.union([
-    TemplatedAssertionSchema,
-    ValueAssertionSchema,
-]);
-// ---------------------------------------------------------------------------
-// Nested config schemas
-// ---------------------------------------------------------------------------
-const BaselineConfigSchema = z
-    .object({
-    enabled: z.boolean().optional(),
-    rubric: z.enum(["abbreviated", "full", "none"]).optional(),
-})
-    .optional();
-const ExecutionConfigSchema = z
-    .object({
-    enabled: z.boolean().optional().default(true),
-    blocking: z.boolean().optional().default(false),
-    threshold: z
-        .object({
-        score: z.number().min(0).max(100).optional(),
-    })
-        .optional(),
-    trigger: z
-        .object({
-        branches: z.array(z.string()).optional(),
-        paths: z.array(z.string()).optional(),
-    })
-        .optional(),
-    source: z.string().optional(),
-})
-    .optional();
-// ---------------------------------------------------------------------------
-// RepoTaskSchema — a single task definition from .ailf/tasks/*.yaml
-// ---------------------------------------------------------------------------
-/**
- * Zod schema for a single repo-based task definition.
- *
- * This is the external-author-facing contract. Field names are camelCase
- * to match the Content Lake document schema (ailf.task).
- */
-export const RepoTaskSchema = z.object({
-    id: z
-        .string()
-        .min(1)
-        .regex(/^[a-z0-9][a-z0-9-]*$/, "Task ID must be lowercase alphanumeric with hyphens"),
-    description: z.string().min(1),
-    status: z
-        .enum(["active", "draft", "paused", "archived"])
-        .optional()
-        .default("active"),
-    featureArea: z
-        .string()
-        .min(1)
-        .regex(/^[a-z0-9][a-z0-9-]*$/, "Feature area must be lowercase alphanumeric with hyphens"),
-    tags: z.array(z.string()).optional(),
-    canonicalDocs: z.array(CanonicalDocRefSchema).optional().default([]),
-    vars: z
-        .object({
-        task: z.string().min(1),
-    })
-        .passthrough()
-        .optional(),
-    assert: z.array(AssertionSchema).min(1),
-    baseline: BaselineConfigSchema,
-    docCoverage: z.boolean().optional().default(false),
-    referenceSolution: z.string().optional(),
-    execution: ExecutionConfigSchema,
-});
-/**
- * Schema for an array of repo tasks — what a single .ailf/tasks/*.yaml file
- * contains. Each file must define at least one task.
- */
-export const RepoTaskFileSchema = z.array(RepoTaskSchema).min(1);

package/dist/_vendor/ailf-tasks/validation.d.ts

@@ -1,47 +0,0 @@
-/**
- * validation.ts — Semantic validation for repo-based tasks.
- *
- * Checks that go beyond Zod schema parsing:
- * - Assertion types are in the curated set
- * - Rubric template names resolve to known templates
- * - Feature area strings are well-formed
- * - Canonical doc slugs look reasonable (slugs, not URLs)
- *
- * These produce warnings, not errors — the pipeline can still run
- * with imperfect tasks. Only structural failures (caught by Zod) block.
- */
-import { type RepoTask } from "./schemas.js";
-export interface ValidationResult {
-    valid: boolean;
-    errors: ValidationMessage[];
-    warnings: ValidationMessage[];
-}
-export interface ValidationMessage {
-    taskId: string;
-    field: string;
-    message: string;
-}
-/**
- * Run semantic validation on an array of parsed repo tasks.
- *
- * Returns warnings for issues that don't block execution (unknown feature
- * areas, unresolved slugs) and errors for issues that would cause pipeline
- * failures (completely missing required fields — though Zod catches most).
- */
-export declare function validateRepoTasks(tasks: RepoTask[]): ValidationResult;
-/**
- * Format validation results for console output.
- */
-export declare function formatValidationResult(result: ValidationResult): string;
-/**
- * Detect snake_case field names in raw task YAML data.
- *
- * This runs BEFORE Zod parsing to provide a user-friendly error message
- * when authors use framework-internal snake_case names instead of the
- * camelCase names expected in repo task files.
- *
- * @param raw - Raw parsed YAML (before Zod validation)
- * @param filename - Source filename for error messages
- * @returns Array of warning messages (empty if no issues)
- */
-export declare function detectSnakeCaseFields(raw: unknown, filename: string): string[];

package/dist/_vendor/ailf-tasks/validation.js

@@ -1,162 +0,0 @@
-/**
- * validation.ts — Semantic validation for repo-based tasks.
- *
- * Checks that go beyond Zod schema parsing:
- * - Assertion types are in the curated set
- * - Rubric template names resolve to known templates
- * - Feature area strings are well-formed
- * - Canonical doc slugs look reasonable (slugs, not URLs)
- *
- * These produce warnings, not errors — the pipeline can still run
- * with imperfect tasks. Only structural failures (caught by Zod) block.
- */
-import { CURATED_ASSERTION_TYPES, RUBRIC_TEMPLATE_NAMES, } from "./schemas.js";
-// ---------------------------------------------------------------------------
-// Public API
-// ---------------------------------------------------------------------------
-/**
- * Run semantic validation on an array of parsed repo tasks.
- *
- * Returns warnings for issues that don't block execution (unknown feature
- * areas, unresolved slugs) and errors for issues that would cause pipeline
- * failures (completely missing required fields — though Zod catches most).
- */
-export function validateRepoTasks(tasks) {
-    const errors = [];
-    const warnings = [];
-    // Check for duplicate IDs
-    const seenIds = new Set();
-    for (const task of tasks) {
-        if (seenIds.has(task.id)) {
-            errors.push({
-                taskId: task.id,
-                field: "id",
-                message: `Duplicate task ID "${task.id}"`,
-            });
-        }
-        seenIds.add(task.id);
-    }
-    for (const task of tasks) {
-        // Check assertion types
-        for (let i = 0; i < task.assert.length; i++) {
-            const assertion = task.assert[i];
-            if (!CURATED_ASSERTION_TYPES.includes(assertion.type)) {
-                warnings.push({
-                    taskId: task.id,
-                    field: `assert[${i}].type`,
-                    message: `Unknown assertion type "${assertion.type}". ` +
-                        `Valid types: ${CURATED_ASSERTION_TYPES.join(", ")}`,
-                });
-            }
-            // Check rubric template for llm-rubric assertions
-            if (assertion.type === "llm-rubric" && "template" in assertion) {
-                const template = assertion.template;
-                if (!RUBRIC_TEMPLATE_NAMES.includes(template)) {
-                    warnings.push({
-                        taskId: task.id,
-                        field: `assert[${i}].template`,
-                        message: `Unknown rubric template "${template}". ` +
-                            `Valid templates: ${RUBRIC_TEMPLATE_NAMES.join(", ")}`,
-                    });
-                }
-            }
-        }
-        // Check canonical doc refs look reasonable
-        for (let i = 0; i < (task.canonicalDocs?.length ?? 0); i++) {
-            const doc = task.canonicalDocs[i];
-            // Slug refs: warn if they look like URLs or paths
-            if ("slug" in doc && !("id" in doc) && typeof doc.slug === "string") {
-                if (doc.slug.includes("/") || doc.slug.includes("http")) {
-                    warnings.push({
-                        taskId: task.id,
-                        field: `canonicalDocs[${i}].slug`,
-                        message: `Slug "${doc.slug}" looks like a URL or path — use 'path' type for paths or 'slug' for document slugs (e.g., "groq-introduction")`,
-                    });
-                }
-            }
-        }
-        // Check task has at least one llm-rubric assertion (recommended but not required)
-        const hasLlmRubric = task.assert.some((a) => a.type === "llm-rubric");
-        if (!hasLlmRubric) {
-            warnings.push({
-                taskId: task.id,
-                field: "assert",
-                message: "No llm-rubric assertion found. Tasks should have at least one scored rubric for meaningful evaluation.",
-            });
-        }
-        // Check taskPrompt exists in vars (vars.task)
-        if (!task.vars?.task) {
-            warnings.push({
-                taskId: task.id,
-                field: "vars.task",
-                message: "No task prompt found in vars.task. The LLM will receive an empty implementation request.",
-            });
-        }
-    }
-    return {
-        valid: errors.length === 0,
-        errors,
-        warnings,
-    };
-}
-/**
- * Format validation results for console output.
- */
-export function formatValidationResult(result) {
-    const lines = [];
-    if (result.errors.length > 0) {
-        lines.push("❌ Errors:");
-        for (const e of result.errors) {
-            lines.push(`  [${e.taskId}] ${e.field}: ${e.message}`);
-        }
-    }
-    if (result.warnings.length > 0) {
-        lines.push("⚠️  Warnings:");
-        for (const w of result.warnings) {
-            lines.push(`  [${w.taskId}] ${w.field}: ${w.message}`);
-        }
-    }
-    if (result.valid && result.warnings.length === 0) {
-        lines.push("✅ All repo tasks pass validation");
-    }
-    return lines.join("\n");
-}
-// ---------------------------------------------------------------------------
-// Snake_case detection (pre-parse helper)
-// ---------------------------------------------------------------------------
-/** Known snake_case → camelCase field mappings for common errors */
-const SNAKE_TO_CAMEL = {
-    feature_area: "featureArea",
-    canonical_docs: "canonicalDocs",
-    doc_coverage: "docCoverage",
-    reference_solution: "referenceSolution",
-};
-/**
- * Detect snake_case field names in raw task YAML data.
- *
- * This runs BEFORE Zod parsing to provide a user-friendly error message
- * when authors use framework-internal snake_case names instead of the
- * camelCase names expected in repo task files.
- *
- * @param raw - Raw parsed YAML (before Zod validation)
- * @param filename - Source filename for error messages
- * @returns Array of warning messages (empty if no issues)
- */
-export function detectSnakeCaseFields(raw, filename) {
-    const warnings = [];
-    if (!Array.isArray(raw))
-        return warnings;
-    for (let i = 0; i < raw.length; i++) {
-        const entry = raw[i];
-        if (typeof entry !== "object" || entry === null)
-            continue;
-        const obj = entry;
-        const taskId = typeof obj.id === "string" ? obj.id : `task[${i}]`;
-        for (const [snake, camel] of Object.entries(SNAKE_TO_CAMEL)) {
-            if (snake in obj) {
-                warnings.push(`[${filename}] ${taskId}: Found "${snake}" — repo tasks use camelCase. Did you mean "${camel}"?`);
-            }
-        }
-    }
-    return warnings;
-}

package/dist/adapters/task-sources/yaml-task-source.d.ts

@@ -1,18 +0,0 @@
-/**
- * Adapter: Load task definitions from tasks/*.yaml files.
- *
- * This adapter reads the raw YAML task definitions (before Promptfoo
- * expansion) and maps them to GeneralizedTaskDefinition
- * (LiteracyTaskDefinition variant) from @sanity/ailf-core. It handles
- * area filtering (filename stem) and task ID filtering.
- *
- * Unlike loadAndExpandTasks() — which produces Promptfoo-specific
- * ExpandedTestEntry objects — this adapter produces domain-level
- * GeneralizedTaskDefinition objects suitable for the pipeline orchestrator.
- */
-import type { FilterOptions, GeneralizedTaskDefinition, TaskSource } from "../../_vendor/ailf-core/index.d.ts";
-export declare class YamlTaskSource implements TaskSource {
-    private readonly rootDir;
-    constructor(rootDir: string);
-    loadTasks(filter?: FilterOptions): Promise<GeneralizedTaskDefinition[]>;
-}