@sanity/ailf 0.1.0

package/dist/adapters/task-sources/repo-trigger.js

@@ -0,0 +1,153 @@
+/**
+ * Trigger processing for repo-based AILF tasks.
+ *
+ * Reads .ailf/config.yaml, determines the current execution context
+ * (PR, main merge, schedule, manual), and resolves which trigger
+ * config applies. This drives whether the pipeline runs in validate-only
+ * mode or full eval mode, and whether results are blocking.
+ *
+ * @see docs/exec-plans/completed/tasks-as-content/phase-4-repo-based-tasks.md
+ */
+import { existsSync, readFileSync } from "fs";
+import { resolve } from "path";
+import { load } from "js-yaml";
+import { parseRepoConfig } from "./repo-schemas.js";
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+/**
+ * Load and resolve the trigger configuration for the current context.
+ *
+ * @param repoRoot - Root of the external repo (contains .ailf/)
+ * @param context - The current execution context (PR, main, schedule, manual)
+ * @returns The resolved trigger config, or a default (eval, non-blocking)
+ */
+export function resolveTrigger(repoRoot, context) {
+    const configPath = resolve(repoRoot, ".ailf", "config.yaml");
+    if (!existsSync(configPath)) {
+        // No config file — use sensible defaults
+        return defaultTrigger(context);
+    }
+    const raw = readFileSync(configPath, "utf-8");
+    const parsed = load(raw);
+    const config = parseRepoConfig(parsed);
+    return matchTrigger(config, context);
+}
+/**
+ * Detect the current trigger context from environment variables.
+ *
+ * Reads GitHub Actions env vars to determine if we're in a PR,
+ * on the main branch, running on schedule, or manual dispatch.
+ */
+export function detectTriggerContext() {
+    const eventName = process.env.GITHUB_EVENT_NAME;
+    const ref = process.env.GITHUB_REF ?? "";
+    if (eventName === "schedule") {
+        return { type: "schedule" };
+    }
+    if (eventName === "workflow_dispatch") {
+        return { type: "manual" };
+    }
+    if (eventName === "pull_request" || eventName === "pull_request_target") {
+        const branch = process.env.GITHUB_HEAD_REF ?? ref.replace("refs/heads/", "");
+        // Changed files could come from GITHUB_EVENT_PATH or git diff
+        // For now, return without changedFiles — the caller can populate from git
+        return { type: "pr", branch };
+    }
+    // Default: check if we're on main/master
+    if (ref === "refs/heads/main" || ref === "refs/heads/master") {
+        return { type: "main" };
+    }
+    return { type: "manual" };
+}
+// ---------------------------------------------------------------------------
+// Internal matching
+// ---------------------------------------------------------------------------
+function matchTrigger(config, context) {
+    const triggers = config.triggers;
+    if (!triggers)
+        return defaultTrigger(context);
+    switch (context.type) {
+        case "pr": {
+            // Check pr-task-change first (more specific), then pr (general)
+            const prTaskChange = triggers["pr-task-change"];
+            if (prTaskChange &&
+                hasPathMatch(prTaskChange.paths, context.changedFiles)) {
+                return {
+                    matched: true,
+                    mode: prTaskChange.mode ?? "eval",
+                    blocking: prTaskChange.blocking ?? false,
+                    notify: prTaskChange.notify ?? true,
+                    compare: prTaskChange.compare ?? false,
+                    paths: prTaskChange.paths,
+                };
+            }
+            const pr = triggers.pr;
+            if (pr) {
+                return {
+                    matched: true,
+                    mode: pr.mode ?? "validate-only",
+                    blocking: pr.blocking ?? false,
+                    notify: pr.notify ?? true,
+                    compare: pr.compare ?? false,
+                    paths: pr.paths,
+                };
+            }
+            return defaultTrigger(context);
+        }
+        case "main": {
+            const main = triggers.main;
+            if (main) {
+                return {
+                    matched: true,
+                    mode: main.mode ?? "eval",
+                    blocking: main.blocking ?? false,
+                    notify: main.notify ?? true,
+                    compare: main.compare ?? false,
+                };
+            }
+            return defaultTrigger(context);
+        }
+        case "schedule": {
+            const schedule = triggers.schedule;
+            if (schedule) {
+                return {
+                    matched: true,
+                    mode: schedule.mode ?? "eval",
+                    blocking: false, // schedule is never blocking
+                    notify: schedule.notify ?? true,
+                    compare: schedule.compare ?? true,
+                    cron: schedule.cron,
+                };
+            }
+            return defaultTrigger(context);
+        }
+        case "manual":
+            return defaultTrigger(context);
+    }
+}
+/**
+ * Check if any changed files match the trigger's path patterns.
+ * Uses simple glob-like prefix matching (not full glob).
+ * Returns true if no paths are specified (match-all).
+ */
+function hasPathMatch(patterns, changedFiles) {
+    if (!patterns || patterns.length === 0)
+        return true;
+    if (!changedFiles || changedFiles.length === 0)
+        return false;
+    return changedFiles.some((file) => patterns.some((pattern) => {
+        // Simple glob: ".ailf/**" matches any path starting with ".ailf/"
+        const prefix = pattern.replace(/\*\*$/, "").replace(/\*$/, "");
+        return file.startsWith(prefix);
+    }));
+}
+function defaultTrigger(context) {
+    return {
+        matched: false,
+        mode: context.type === "pr" ? "validate-only" : "eval",
+        blocking: false,
+        notify: context.type !== "manual",
+        compare: context.type === "schedule",
+    };
+}

package/dist/adapters/task-sources/repo-validation.d.ts

@@ -0,0 +1,49 @@
+/**
+ * Semantic validation for repo-based tasks.
+ *
+ * Checks that go beyond Zod schema parsing:
+ * - Assertion types are in the curated set
+ * - Rubric template names resolve to known templates
+ * - Feature area strings are well-formed
+ * - Canonical doc slugs look reasonable (slugs, not URLs)
+ *
+ * These produce warnings, not errors — the pipeline can still run
+ * with imperfect tasks. Only structural failures (caught by Zod) block.
+ *
+ * @see packages/eval/src/adapters/task-sources/repo-schemas.ts
+ */
+import { type RepoTask } from "./repo-schemas.js";
+export interface ValidationResult {
+    valid: boolean;
+    errors: ValidationMessage[];
+    warnings: ValidationMessage[];
+}
+export interface ValidationMessage {
+    taskId: string;
+    field: string;
+    message: string;
+}
+/**
+ * Run semantic validation on an array of parsed repo tasks.
+ *
+ * Returns warnings for issues that don't block execution (unknown feature
+ * areas, unresolved slugs) and errors for issues that would cause pipeline
+ * failures (completely missing required fields — though Zod catches most).
+ */
+export declare function validateRepoTasks(tasks: RepoTask[]): ValidationResult;
+/**
+ * Format validation results for console output.
+ */
+export declare function formatValidationResult(result: ValidationResult): string;
+/**
+ * Detect snake_case field names in raw task YAML data.
+ *
+ * This runs BEFORE Zod parsing to provide a user-friendly error message
+ * when authors use framework-internal snake_case names instead of the
+ * camelCase names expected in repo task files.
+ *
+ * @param raw - Raw parsed YAML (before Zod validation)
+ * @param filename - Source filename for error messages
+ * @returns Array of warning messages (empty if no issues)
+ */
+export declare function detectSnakeCaseFields(raw: unknown, filename: string): string[];

package/dist/adapters/task-sources/repo-validation.js

@@ -0,0 +1,164 @@
+/**
+ * Semantic validation for repo-based tasks.
+ *
+ * Checks that go beyond Zod schema parsing:
+ * - Assertion types are in the curated set
+ * - Rubric template names resolve to known templates
+ * - Feature area strings are well-formed
+ * - Canonical doc slugs look reasonable (slugs, not URLs)
+ *
+ * These produce warnings, not errors — the pipeline can still run
+ * with imperfect tasks. Only structural failures (caught by Zod) block.
+ *
+ * @see packages/eval/src/adapters/task-sources/repo-schemas.ts
+ */
+import { CURATED_ASSERTION_TYPES, RUBRIC_TEMPLATE_NAMES, } from "./repo-schemas.js";
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+/**
+ * Run semantic validation on an array of parsed repo tasks.
+ *
+ * Returns warnings for issues that don't block execution (unknown feature
+ * areas, unresolved slugs) and errors for issues that would cause pipeline
+ * failures (completely missing required fields — though Zod catches most).
+ */
+export function validateRepoTasks(tasks) {
+    const errors = [];
+    const warnings = [];
+    // Check for duplicate IDs
+    const seenIds = new Set();
+    for (const task of tasks) {
+        if (seenIds.has(task.id)) {
+            errors.push({
+                taskId: task.id,
+                field: "id",
+                message: `Duplicate task ID "${task.id}"`,
+            });
+        }
+        seenIds.add(task.id);
+    }
+    for (const task of tasks) {
+        // Check assertion types
+        for (let i = 0; i < task.assert.length; i++) {
+            const assertion = task.assert[i];
+            if (!CURATED_ASSERTION_TYPES.includes(assertion.type)) {
+                warnings.push({
+                    taskId: task.id,
+                    field: `assert[${i}].type`,
+                    message: `Unknown assertion type "${assertion.type}". ` +
+                        `Valid types: ${CURATED_ASSERTION_TYPES.join(", ")}`,
+                });
+            }
+            // Check rubric template for llm-rubric assertions
+            if (assertion.type === "llm-rubric" && "template" in assertion) {
+                const template = assertion.template;
+                if (!RUBRIC_TEMPLATE_NAMES.includes(template)) {
+                    warnings.push({
+                        taskId: task.id,
+                        field: `assert[${i}].template`,
+                        message: `Unknown rubric template "${template}". ` +
+                            `Valid templates: ${RUBRIC_TEMPLATE_NAMES.join(", ")}`,
+                    });
+                }
+            }
+        }
+        // Check canonical doc refs look reasonable
+        for (let i = 0; i < (task.canonicalDocs?.length ?? 0); i++) {
+            const doc = task.canonicalDocs[i];
+            // Slug refs: warn if they look like URLs or paths
+            if ("slug" in doc && !("id" in doc) && typeof doc.slug === "string") {
+                if (doc.slug.includes("/") || doc.slug.includes("http")) {
+                    warnings.push({
+                        taskId: task.id,
+                        field: `canonicalDocs[${i}].slug`,
+                        message: `Slug "${doc.slug}" looks like a URL or path — use 'path' type for paths or 'slug' for document slugs (e.g., "groq-introduction")`,
+                    });
+                }
+            }
+        }
+        // Check task has at least one llm-rubric assertion (recommended but not required)
+        const hasLlmRubric = task.assert.some((a) => a.type === "llm-rubric");
+        if (!hasLlmRubric) {
+            warnings.push({
+                taskId: task.id,
+                field: "assert",
+                message: "No llm-rubric assertion found. Tasks should have at least one scored rubric for meaningful evaluation.",
+            });
+        }
+        // Check taskPrompt exists in vars (vars.task)
+        if (!task.vars?.task) {
+            warnings.push({
+                taskId: task.id,
+                field: "vars.task",
+                message: "No task prompt found in vars.task. The LLM will receive an empty implementation request.",
+            });
+        }
+    }
+    return {
+        valid: errors.length === 0,
+        errors,
+        warnings,
+    };
+}
+/**
+ * Format validation results for console output.
+ */
+export function formatValidationResult(result) {
+    const lines = [];
+    if (result.errors.length > 0) {
+        lines.push("❌ Errors:");
+        for (const e of result.errors) {
+            lines.push(`  [${e.taskId}] ${e.field}: ${e.message}`);
+        }
+    }
+    if (result.warnings.length > 0) {
+        lines.push("⚠️  Warnings:");
+        for (const w of result.warnings) {
+            lines.push(`  [${w.taskId}] ${w.field}: ${w.message}`);
+        }
+    }
+    if (result.valid && result.warnings.length === 0) {
+        lines.push("✅ All repo tasks pass validation");
+    }
+    return lines.join("\n");
+}
+// ---------------------------------------------------------------------------
+// Snake_case detection (pre-parse helper)
+// ---------------------------------------------------------------------------
+/** Known snake_case → camelCase field mappings for common errors */
+const SNAKE_TO_CAMEL = {
+    feature_area: "featureArea",
+    canonical_docs: "canonicalDocs",
+    doc_coverage: "docCoverage",
+    reference_solution: "referenceSolution",
+};
+/**
+ * Detect snake_case field names in raw task YAML data.
+ *
+ * This runs BEFORE Zod parsing to provide a user-friendly error message
+ * when authors use framework-internal snake_case names instead of the
+ * camelCase names expected in repo task files.
+ *
+ * @param raw - Raw parsed YAML (before Zod validation)
+ * @param filename - Source filename for error messages
+ * @returns Array of warning messages (empty if no issues)
+ */
+export function detectSnakeCaseFields(raw, filename) {
+    const warnings = [];
+    if (!Array.isArray(raw))
+        return warnings;
+    for (let i = 0; i < raw.length; i++) {
+        const entry = raw[i];
+        if (typeof entry !== "object" || entry === null)
+            continue;
+        const obj = entry;
+        const taskId = typeof obj.id === "string" ? obj.id : `task[${i}]`;
+        for (const [snake, camel] of Object.entries(SNAKE_TO_CAMEL)) {
+            if (snake in obj) {
+                warnings.push(`[${filename}] ${taskId}: Found "${snake}" — repo tasks use camelCase. Did you mean "${camel}"?`);
+            }
+        }
+    }
+    return warnings;
+}

package/dist/adapters/task-sources/yaml-task-source.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Adapter: Load task definitions from tasks/*.yaml files.
+ *
+ * This adapter reads the raw YAML task definitions (before Promptfoo
+ * expansion) and maps them to the canonical TaskDefinition type from
+ * @sanity/ailf-core. It handles area filtering (filename stem) and
+ * task ID filtering.
+ *
+ * Unlike loadAndExpandTasks() — which produces Promptfoo-specific
+ * ExpandedTestEntry objects — this adapter produces domain-level
+ * TaskDefinition objects suitable for the pipeline orchestrator.
+ */
+import type { FilterOptions, TaskDefinition, TaskSource } from "../../_vendor/ailf-core/index.d.ts";
+export declare class YamlTaskSource implements TaskSource {
+    private readonly rootDir;
+    constructor(rootDir: string);
+    loadTasks(filter?: FilterOptions): Promise<TaskDefinition[]>;
+}

package/dist/adapters/task-sources/yaml-task-source.js

@@ -0,0 +1,136 @@
+/**
+ * Adapter: Load task definitions from tasks/*.yaml files.
+ *
+ * This adapter reads the raw YAML task definitions (before Promptfoo
+ * expansion) and maps them to the canonical TaskDefinition type from
+ * @sanity/ailf-core. It handles area filtering (filename stem) and
+ * task ID filtering.
+ *
+ * Unlike loadAndExpandTasks() — which produces Promptfoo-specific
+ * ExpandedTestEntry objects — this adapter produces domain-level
+ * TaskDefinition objects suitable for the pipeline orchestrator.
+ */
+import { existsSync, readdirSync, readFileSync } from "fs";
+import { resolve } from "path";
+import { load } from "js-yaml";
+// ---------------------------------------------------------------------------
+// YamlTaskSource adapter
+// ---------------------------------------------------------------------------
+export class YamlTaskSource {
+    rootDir;
+    constructor(rootDir) {
+        this.rootDir = rootDir;
+    }
+    async loadTasks(filter) {
+        const tasksDir = resolve(this.rootDir, "tasks");
+        if (!existsSync(tasksDir)) {
+            throw new Error(`tasks/ directory not found at ${tasksDir}`);
+        }
+        let yamlFiles = readdirSync(tasksDir)
+            .filter((f) => (f.endsWith(".yaml") || f.endsWith(".yml")) && !f.startsWith("."))
+            .sort();
+        // Area filter — area name = filename stem (e.g., "groq" matches "groq.yaml")
+        if (filter?.areas && filter.areas.length > 0) {
+            const allowedAreas = new Set(filter.areas.map((a) => a.toLowerCase()));
+            yamlFiles = yamlFiles.filter((f) => {
+                const stem = f.replace(/\.ya?ml$/, "").toLowerCase();
+                return allowedAreas.has(stem);
+            });
+        }
+        const definitions = [];
+        for (const file of yamlFiles) {
+            const featureArea = file.replace(/\.ya?ml$/, "");
+            const filePath = resolve(tasksDir, file);
+            const raw = readFileSync(filePath, "utf-8");
+            const parsed = load(raw);
+            if (!Array.isArray(parsed)) {
+                throw new Error(`${file} did not parse to an array of tasks`);
+            }
+            for (const entry of parsed) {
+                if (!isRawYamlTask(entry))
+                    continue;
+                // Task ID filter
+                if (filter?.taskIds &&
+                    filter.taskIds.length > 0 &&
+                    !filter.taskIds.includes(entry.id)) {
+                    continue;
+                }
+                definitions.push(mapToTaskDefinition(entry, featureArea));
+            }
+        }
+        return definitions;
+    }
+}
+// ---------------------------------------------------------------------------
+// Mapping helpers
+// ---------------------------------------------------------------------------
+/**
+ * Map a raw YAML entry to a canonical TaskDefinition.
+ *
+ * Renames snake_case YAML keys to camelCase domain types and extracts
+ * the task prompt from `vars.task`. Additional vars beyond `task` and
+ * `docs` are collected into `extraVars`.
+ */
+function mapToTaskDefinition(raw, featureArea) {
+    const { task, docs: _docs, ...rest } = (raw.vars ?? {});
+    const canonicalDocs = (raw.canonical_docs ?? [])
+        .map(mapCanonicalDoc)
+        .filter((d) => d !== null);
+    const extraVars = Object.keys(rest).length > 0 ? rest : undefined;
+    return {
+        id: raw.id,
+        description: raw.description,
+        featureArea,
+        taskPrompt: typeof task === "string" ? task : "",
+        canonicalDocs,
+        referenceSolution: raw.reference_solution ?? "",
+        docCoverage: raw.doc_coverage ?? false,
+        assertions: (raw.assert ?? []),
+        ...(raw.baseline ? { baseline: raw.baseline } : {}),
+        ...(extraVars ? { extraVars } : {}),
+    };
+}
+// ---------------------------------------------------------------------------
+// Canonical doc mapping
+// ---------------------------------------------------------------------------
+/**
+ * Map a raw YAML canonical doc entry to the polymorphic CanonicalDocRef.
+ *
+ * Discriminates by key presence: slug, path, id, or perspective.
+ * Returns null (with a warning) if no valid resolution key is found.
+ */
+function mapCanonicalDoc(raw) {
+    const reason = raw.reason ?? "";
+    // Resolution priority: id > slug > path > perspective.
+    // When `id` is present it's always an IdDocRef — slug and path are carried
+    // as optional annotations for human readability (they are NOT used for
+    // resolution by the pipeline).
+    if (raw.id) {
+        return {
+            id: raw.id,
+            reason,
+            ...(raw.slug ? { slug: raw.slug } : {}),
+            ...(raw.path ? { path: raw.path } : {}),
+        };
+    }
+    if (raw.slug)
+        return { slug: raw.slug, reason };
+    if (raw.path)
+        return { path: raw.path, reason };
+    if (raw.perspective)
+        return { perspective: raw.perspective, reason };
+    console.warn("  [warn] Skipping canonical doc entry with no resolution key (id, slug, path, or perspective)");
+    return null;
+}
+// ---------------------------------------------------------------------------
+// Type guard
+// ---------------------------------------------------------------------------
+/** Check whether a parsed YAML value looks like a single task definition. */
+function isRawYamlTask(entry) {
+    return (typeof entry === "object" &&
+        entry !== null &&
+        "id" in entry &&
+        typeof entry.id === "string" &&
+        "description" in entry &&
+        typeof entry.description === "string");
+}

package/dist/agent-observer/agentic-provider.d.ts

@@ -0,0 +1,132 @@
+/**
+ * agentic-provider.ts
+ *
+ * An agentic Promptfoo provider that gives the model web_search and
+ * fetch_page tools, simulating how real AI agents behave when a user
+ * asks a development question.
+ *
+ * Supports two agent modes via the `agentMode` config:
+ *
+ *   - "naive"     — Simulates current agents (Claude Code, ChatGPT, Cursor):
+ *                    uses Jina Reader for search + page fetching because real
+ *                    agents can't render JavaScript-heavy SPAs server-side.
+ *
+ *   - "optimized" — Simulates an ideal agent that knows about Sanity's
+ *                    agent-friendly endpoints: fetches .md versions of doc
+ *                    pages directly, uses llms.txt for doc discovery, and
+ *                    falls back to Jina only for non-Sanity pages.
+ *
+ * All HTTP requests go through the RequestRecorder, so they're automatically
+ * classified as docPageVisits, searchQueries, etc.
+ *
+ * Promptfoo config usage:
+ *
+ *   providers:
+ *     - id: file://dist/agent-observer/agentic-provider.js
+ *       label: "GPT-4o (Naive Agent)"
+ *       config:
+ *         model: gpt-4o
+ *         agentMode: naive       # or "optimized"
+ *         maxToolRounds: 5
+ */
+import { RequestRecorder } from "./proxy.js";
+interface CallApiContextParams {
+    prompt?: {
+        raw: string;
+        label?: string;
+    };
+    vars?: Record<string, object | string>;
+}
+interface ProviderOptions {
+    config?: Record<string, unknown>;
+    id?: string;
+}
+interface ProviderResponse {
+    cached?: boolean;
+    cost?: number;
+    error?: string;
+    metadata?: Record<string, unknown>;
+    output?: object | string;
+    tokenUsage?: {
+        total?: number;
+        prompt?: number;
+        completion?: number;
+        cached?: number;
+    };
+}
+export default class AgenticProvider {
+    config: Record<string, unknown>;
+    protected providerId: string;
+    private agentMode;
+    private allowedOrigins;
+    private customHeaders;
+    private docBaseUrl;
+    private docsUrlPattern;
+    private llmsTxtUrl;
+    private priorityDomain;
+    private recorder;
+    private searchMode;
+    constructor(options: ProviderOptions);
+    /**
+     * Main Promptfoo provider entry point. Runs the full agentic loop.
+     */
+    callApi(prompt: string, context?: CallApiContextParams): Promise<ProviderResponse>;
+    /**
+     * Exposes the recorder for external integrations.
+     */
+    getRecorder(): RequestRecorder;
+    id(): string;
+    /**
+     * Build the system prompt based on agent mode and configured doc URLs.
+     */
+    private buildSystemPrompt;
+    /**
+     * Cleans Jina Reader markdown output by removing navigation boilerplate,
+     * cookie banners, and footer content.
+     */
+    private cleanJinaContent;
+    /**
+     * Detect which LLM provider to use based on config and model name.
+     * Reads the `provider` config field set by generate-configs, with
+     * fallback heuristics for backward compatibility.
+     */
+    private detectProvider;
+    private executeFetchPage;
+    private executeListDocs;
+    private executeTool;
+    private executeWebSearch;
+    /**
+     * Build the set of tools available to the agent based on search mode
+     * and agent mode. When searchMode is "off", web_search is excluded
+     * entirely — the model can't call what it can't see.
+     */
+    private getAvailableTools;
+    /**
+     * Merge custom headers into a request's headers.
+     * Custom headers are injected into doc-site requests only — never into
+     * external APIs (OpenAI, Jina, Google).
+     */
+    private mergeDocHeaders;
+    /**
+     * Parses search results from Jina Reader markdown output.
+     */
+    private parseSearchResults;
+    /**
+     * Runs the agentic tool-calling loop. Routes to OpenAI or Anthropic
+     * based on the `provider` config field.
+     */
+    private runAgenticLoop;
+    private runAnthropicLoop;
+    private runOpenAILoop;
+    /**
+     * Strips HTML tags and normalizes whitespace. Fallback for when
+     * neither .md endpoints nor Jina are available.
+     */
+    private stripHtml;
+    /**
+     * Convert our ToolDefinition[] to Anthropic's tool format.
+     * Anthropic uses `input_schema` instead of `parameters`.
+     */
+    private toAnthropicTools;
+}
+export {};