@sanity/ailf 2.0.0 → 2.0.1

package/dist/_vendor/ailf-tasks/validation.d.ts

@@ -0,0 +1,47 @@
+/**
+ * validation.ts — Semantic validation for repo-based tasks.
+ *
+ * Checks that go beyond Zod schema parsing:
+ * - Assertion types are in the curated set
+ * - Rubric template names resolve to known templates
+ * - Feature area strings are well-formed
+ * - Canonical doc slugs look reasonable (slugs, not URLs)
+ *
+ * These produce warnings, not errors — the pipeline can still run
+ * with imperfect tasks. Only structural failures (caught by Zod) block.
+ */
+import { type RepoTask } from "./schemas.js";
+export interface ValidationResult {
+    valid: boolean;
+    errors: ValidationMessage[];
+    warnings: ValidationMessage[];
+}
+export interface ValidationMessage {
+    taskId: string;
+    field: string;
+    message: string;
+}
+/**
+ * Run semantic validation on an array of parsed repo tasks.
+ *
+ * Returns warnings for issues that don't block execution (unknown feature
+ * areas, unresolved slugs) and errors for issues that would cause pipeline
+ * failures (completely missing required fields — though Zod catches most).
+ */
+export declare function validateRepoTasks(tasks: RepoTask[]): ValidationResult;
+/**
+ * Format validation results for console output.
+ */
+export declare function formatValidationResult(result: ValidationResult): string;
+/**
+ * Detect snake_case field names in raw task YAML data.
+ *
+ * This runs BEFORE Zod parsing to provide a user-friendly error message
+ * when authors use framework-internal snake_case names instead of the
+ * camelCase names expected in repo task files.
+ *
+ * @param raw - Raw parsed YAML (before Zod validation)
+ * @param filename - Source filename for error messages
+ * @returns Array of warning messages (empty if no issues)
+ */
+export declare function detectSnakeCaseFields(raw: unknown, filename: string): string[];

package/dist/_vendor/ailf-tasks/validation.js

@@ -0,0 +1,162 @@
+/**
+ * validation.ts — Semantic validation for repo-based tasks.
+ *
+ * Checks that go beyond Zod schema parsing:
+ * - Assertion types are in the curated set
+ * - Rubric template names resolve to known templates
+ * - Feature area strings are well-formed
+ * - Canonical doc slugs look reasonable (slugs, not URLs)
+ *
+ * These produce warnings, not errors — the pipeline can still run
+ * with imperfect tasks. Only structural failures (caught by Zod) block.
+ */
+import { CURATED_ASSERTION_TYPES, RUBRIC_TEMPLATE_NAMES, } from "./schemas.js";
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+/**
+ * Run semantic validation on an array of parsed repo tasks.
+ *
+ * Returns warnings for issues that don't block execution (unknown feature
+ * areas, unresolved slugs) and errors for issues that would cause pipeline
+ * failures (completely missing required fields — though Zod catches most).
+ */
+export function validateRepoTasks(tasks) {
+    const errors = [];
+    const warnings = [];
+    // Check for duplicate IDs
+    const seenIds = new Set();
+    for (const task of tasks) {
+        if (seenIds.has(task.id)) {
+            errors.push({
+                taskId: task.id,
+                field: "id",
+                message: `Duplicate task ID "${task.id}"`,
+            });
+        }
+        seenIds.add(task.id);
+    }
+    for (const task of tasks) {
+        // Check assertion types
+        for (let i = 0; i < task.assert.length; i++) {
+            const assertion = task.assert[i];
+            if (!CURATED_ASSERTION_TYPES.includes(assertion.type)) {
+                warnings.push({
+                    taskId: task.id,
+                    field: `assert[${i}].type`,
+                    message: `Unknown assertion type "${assertion.type}". ` +
+                        `Valid types: ${CURATED_ASSERTION_TYPES.join(", ")}`,
+                });
+            }
+            // Check rubric template for llm-rubric assertions
+            if (assertion.type === "llm-rubric" && "template" in assertion) {
+                const template = assertion.template;
+                if (!RUBRIC_TEMPLATE_NAMES.includes(template)) {
+                    warnings.push({
+                        taskId: task.id,
+                        field: `assert[${i}].template`,
+                        message: `Unknown rubric template "${template}". ` +
+                            `Valid templates: ${RUBRIC_TEMPLATE_NAMES.join(", ")}`,
+                    });
+                }
+            }
+        }
+        // Check canonical doc refs look reasonable
+        for (let i = 0; i < (task.canonicalDocs?.length ?? 0); i++) {
+            const doc = task.canonicalDocs[i];
+            // Slug refs: warn if they look like URLs or paths
+            if ("slug" in doc && !("id" in doc) && typeof doc.slug === "string") {
+                if (doc.slug.includes("/") || doc.slug.includes("http")) {
+                    warnings.push({
+                        taskId: task.id,
+                        field: `canonicalDocs[${i}].slug`,
+                        message: `Slug "${doc.slug}" looks like a URL or path — use 'path' type for paths or 'slug' for document slugs (e.g., "groq-introduction")`,
+                    });
+                }
+            }
+        }
+        // Check task has at least one llm-rubric assertion (recommended but not required)
+        const hasLlmRubric = task.assert.some((a) => a.type === "llm-rubric");
+        if (!hasLlmRubric) {
+            warnings.push({
+                taskId: task.id,
+                field: "assert",
+                message: "No llm-rubric assertion found. Tasks should have at least one scored rubric for meaningful evaluation.",
+            });
+        }
+        // Check taskPrompt exists in vars (vars.task)
+        if (!task.vars?.task) {
+            warnings.push({
+                taskId: task.id,
+                field: "vars.task",
+                message: "No task prompt found in vars.task. The LLM will receive an empty implementation request.",
+            });
+        }
+    }
+    return {
+        valid: errors.length === 0,
+        errors,
+        warnings,
+    };
+}
+/**
+ * Format validation results for console output.
+ */
+export function formatValidationResult(result) {
+    const lines = [];
+    if (result.errors.length > 0) {
+        lines.push("❌ Errors:");
+        for (const e of result.errors) {
+            lines.push(`  [${e.taskId}] ${e.field}: ${e.message}`);
+        }
+    }
+    if (result.warnings.length > 0) {
+        lines.push("⚠️  Warnings:");
+        for (const w of result.warnings) {
+            lines.push(`  [${w.taskId}] ${w.field}: ${w.message}`);
+        }
+    }
+    if (result.valid && result.warnings.length === 0) {
+        lines.push("✅ All repo tasks pass validation");
+    }
+    return lines.join("\n");
+}
+// ---------------------------------------------------------------------------
+// Snake_case detection (pre-parse helper)
+// ---------------------------------------------------------------------------
+/** Known snake_case → camelCase field mappings for common errors */
+const SNAKE_TO_CAMEL = {
+    feature_area: "featureArea",
+    canonical_docs: "canonicalDocs",
+    doc_coverage: "docCoverage",
+    reference_solution: "referenceSolution",
+};
+/**
+ * Detect snake_case field names in raw task YAML data.
+ *
+ * This runs BEFORE Zod parsing to provide a user-friendly error message
+ * when authors use framework-internal snake_case names instead of the
+ * camelCase names expected in repo task files.
+ *
+ * @param raw - Raw parsed YAML (before Zod validation)
+ * @param filename - Source filename for error messages
+ * @returns Array of warning messages (empty if no issues)
+ */
+export function detectSnakeCaseFields(raw, filename) {
+    const warnings = [];
+    if (!Array.isArray(raw))
+        return warnings;
+    for (let i = 0; i < raw.length; i++) {
+        const entry = raw[i];
+        if (typeof entry !== "object" || entry === null)
+            continue;
+        const obj = entry;
+        const taskId = typeof obj.id === "string" ? obj.id : `task[${i}]`;
+        for (const [snake, camel] of Object.entries(SNAKE_TO_CAMEL)) {
+            if (snake in obj) {
+                warnings.push(`[${filename}] ${taskId}: Found "${snake}" — repo tasks use camelCase. Did you mean "${camel}"?`);
+            }
+        }
+    }
+    return warnings;
+}

package/dist/adapters/api-client/remediation.js

@@ -17,7 +17,7 @@ const HINTS = [
             /no article found for slug/i.test(e.message),
         hint: "One or more `canonicalDocs` slugs in your task definitions don't match " +
             "any article in the documentation. Check the `slug` values in " +
-            "`.ailf/tasks/*.yaml` and ensure they correspond to real articles.\n" +
+            "`.ailf/tasks/` and ensure they correspond to real articles.\n" +
             "  Run `ailf validate` to check your task definitions locally.",
     },
     {
@@ -51,7 +51,7 @@ const HINTS = [
         hint: "The documentation fetch step completed but one or more tasks had " +
             "empty context. This usually means a `canonicalDocs` slug doesn't " +
             "match any article.\n" +
-            "  Check the slug values in `.ailf/tasks/*.yaml`.",
+            "  Check the slug values in `.ailf/tasks/`.",
     },
     {
         match: (e) => e.step === "dispatch" && /dispatch failed/i.test(e.message),

package/dist/adapters/config-sources/file-config-adapter.js

@@ -22,7 +22,7 @@
  * @see docs/design-docs/architecture-overhaul/typescript-configuration.md
  */
 import { readFileSync } from "fs";
-import { extname } from "path";
+import { extname, resolve } from "path";
 import { EvalConfigSchema, PipelineRequestSchema, } from "../../_vendor/ailf-core/index.js";
 import { mapRequestToConfig } from "../../pipeline/map-request-to-config.js";
 import { normalizeMode } from "../../pipeline/normalize-mode.js";
@@ -93,6 +93,7 @@ function mapEvalConfigToResolvedConfig(config, rootDir) {
     const normalized = normalizeMode(config.mode ?? "literacy");
     return {
         rootDir,
+        outputDir: resolve(rootDir, "results", "latest"),
         mode: normalized.mode,
         variant: normalized.variant,
         noAutoScope: config.noAutoScope ?? false,
@@ -119,6 +120,10 @@ function mapEvalConfigToResolvedConfig(config, rootDir) {
         allowedOrigins: config.allowedOrigins,
         searchMode: config.searchMode ?? "open",
         concurrency: config.concurrency,
+        captureEnabled: false,
+        captureDir: undefined,
+        captureCompress: true,
+        captureExtras: true,
         remote: false,
         apiUrl: "https://ailf-api.sanity.build",
         presets: config.presets,

package/dist/adapters/eval-runners/promptfoo-eval-adapter.js

@@ -33,6 +33,7 @@ export class PromptfooEvalAdapter {
                 cwd: this.rootDir,
                 env: { ...process.env, ...config.env },
                 stdio: "inherit",
+                ...(config.maxDurationMs ? { timeout: config.maxDurationMs } : {}),
             });
             return {
                 durationMs: Date.now() - start,
@@ -40,10 +41,15 @@ export class PromptfooEvalAdapter {
                 summary: `Evaluation complete (${config.configPath})`,
             };
         }
-        catch {
+        catch (err) {
+            const isTimeout = err instanceof Error &&
+                "killed" in err &&
+                err.killed === true;
             return {
                 durationMs: Date.now() - start,
-                error: `Promptfoo evaluation failed: ${config.configPath}`,
+                error: isTimeout
+                    ? `Eval subprocess killed after ${config.maxDurationMs}ms time budget`
+                    : `Promptfoo evaluation failed: ${config.configPath}`,
                 status: "failed",
             };
         }

package/dist/adapters/index.d.ts

@@ -9,4 +9,3 @@ export { SanityDocFetcher } from "./doc-fetchers/index.js";
 export { PromptfooEvalAdapter } from "./eval-runners/index.js";
 export { ConsoleLogger, type ConsoleLoggerOptions, JsonLogger, QuietLogger, } from "./loggers/index.js";
 export { CliConfigAdapter, FileConfigAdapter } from "./config-sources/index.js";
-export { YamlTaskSource } from "./task-sources/index.js";

package/dist/adapters/index.js

@@ -9,4 +9,3 @@ export { SanityDocFetcher } from "./doc-fetchers/index.js";
 export { PromptfooEvalAdapter } from "./eval-runners/index.js";
 export { ConsoleLogger, JsonLogger, QuietLogger, } from "./loggers/index.js";
 export { CliConfigAdapter, FileConfigAdapter } from "./config-sources/index.js";
-export { YamlTaskSource } from "./task-sources/index.js";

package/dist/adapters/task-sources/composite-task-source.d.ts

@@ -10,7 +10,7 @@
  * tasks in a single GeneralizedTaskDefinition[].
  *
  * @see packages/core/src/ports/task-source.ts — TaskSource port
- * @see docs/exec-plans/tasks-as-content/phase-4-repo-based-tasks.md
+ * @see docs/archive/exec-plans/tasks-as-content/phase-4-repo-based-tasks.md
  */
 import type { FilterOptions, GeneralizedTaskDefinition, TaskSource } from "../../_vendor/ailf-core/index.d.ts";
 export declare class CompositeTaskSource implements TaskSource {

package/dist/adapters/task-sources/composite-task-source.js

@@ -10,7 +10,7 @@
  * tasks in a single GeneralizedTaskDefinition[].
  *
  * @see packages/core/src/ports/task-source.ts — TaskSource port
- * @see docs/exec-plans/tasks-as-content/phase-4-repo-based-tasks.md
+ * @see docs/archive/exec-plans/tasks-as-content/phase-4-repo-based-tasks.md
  */
 export class CompositeTaskSource {
     sources;

package/dist/adapters/task-sources/content-lake-task-source.d.ts

@@ -2,15 +2,13 @@
  * Adapter: Load task definitions from the Sanity Content Lake.
  *
  * Fetches ailf.task documents via GROQ and maps them to
- * GeneralizedTaskDefinition (LiteracyTaskDefinition variant). This
- * adapter is the Content Lake counterpart of YamlTaskSource — both
- * produce identical GeneralizedTaskDefinition[] for the same logical
- * tasks. The pipeline never knows which adapter loaded the tasks.
+ * GeneralizedTaskDefinition (LiteracyTaskDefinition variant).
+ * The pipeline never knows which adapter loaded the tasks.
  *
- * Wired in the composition root when --task-source content-lake is set.
+ * Wired in the composition root as the default task source.
  *
  * @see packages/core/src/ports/task-source.ts — TaskSource port
- * @see docs/exec-plans/tasks-as-content/phase-2-pipeline-integration.md
+ * @see docs/archive/exec-plans/tasks-as-content/phase-2-pipeline-integration.md
  */
 import type { SanityClient } from "@sanity/client";
 import type { FilterOptions, GeneralizedTaskDefinition, TaskSource } from "../../_vendor/ailf-core/index.d.ts";

package/dist/adapters/task-sources/content-lake-task-source.js

@@ -2,15 +2,13 @@
  * Adapter: Load task definitions from the Sanity Content Lake.
  *
  * Fetches ailf.task documents via GROQ and maps them to
- * GeneralizedTaskDefinition (LiteracyTaskDefinition variant). This
- * adapter is the Content Lake counterpart of YamlTaskSource — both
- * produce identical GeneralizedTaskDefinition[] for the same logical
- * tasks. The pipeline never knows which adapter loaded the tasks.
+ * GeneralizedTaskDefinition (LiteracyTaskDefinition variant).
+ * The pipeline never knows which adapter loaded the tasks.
  *
- * Wired in the composition root when --task-source content-lake is set.
+ * Wired in the composition root as the default task source.
  *
  * @see packages/core/src/ports/task-source.ts — TaskSource port
- * @see docs/exec-plans/tasks-as-content/phase-2-pipeline-integration.md
+ * @see docs/archive/exec-plans/tasks-as-content/phase-2-pipeline-integration.md
  */
 // ---------------------------------------------------------------------------
 // GROQ query — fetches ailf.task documents with resolved references

package/dist/adapters/task-sources/index.d.ts

@@ -4,5 +4,4 @@ export { CanonicalTaskFileSchema, CanonicalTaskSchema, CURATED_ASSERTION_TYPES,
 export { RepoTaskSource } from "./repo-task-source.js";
 export { detectTriggerContext, resolveTrigger, type ResolvedTrigger, type TriggerContext, } from "./repo-trigger.js";
 export { formatValidationResult, validateCanonicalTasks, type ValidationMessage, type ValidationResult, } from "./repo-validation.js";
-export { discoverTsTaskFiles, loadAllTsTaskFiles, loadTsTaskFile, } from "./task-file-loader.js";
-export { YamlTaskSource } from "./yaml-task-source.js";
+export { discoverTsTaskFiles, loadAllTsTaskFiles, loadTsTaskFile, loadTsTaskFileSync, } from "./task-file-loader.js";

package/dist/adapters/task-sources/index.js

@@ -4,5 +4,4 @@ export { CanonicalTaskFileSchema, CanonicalTaskSchema, CURATED_ASSERTION_TYPES,
 export { RepoTaskSource } from "./repo-task-source.js";
 export { detectTriggerContext, resolveTrigger, } from "./repo-trigger.js";
 export { formatValidationResult, validateCanonicalTasks, } from "./repo-validation.js";
-export { discoverTsTaskFiles, loadAllTsTaskFiles, loadTsTaskFile, } from "./task-file-loader.js";
-export { YamlTaskSource } from "./yaml-task-source.js";
+export { discoverTsTaskFiles, loadAllTsTaskFiles, loadTsTaskFile, loadTsTaskFileSync, } from "./task-file-loader.js";

package/dist/adapters/task-sources/repo-schemas.d.ts

@@ -13,7 +13,7 @@
  * specific and remain here unchanged.
  *
  * @see packages/core/src/types/generalized-task.ts — canonical TypeScript types
- * @see docs/exec-plans/tasks-as-content/phase-4-repo-based-tasks.md
+ * @see docs/archive/exec-plans/tasks-as-content/phase-4-repo-based-tasks.md
  */
 import { z } from "zod";
 /**

package/dist/adapters/task-sources/repo-schemas.js

@@ -13,7 +13,7 @@
  * specific and remain here unchanged.
  *
  * @see packages/core/src/types/generalized-task.ts — canonical TypeScript types
- * @see docs/exec-plans/tasks-as-content/phase-4-repo-based-tasks.md
+ * @see docs/archive/exec-plans/tasks-as-content/phase-4-repo-based-tasks.md
  */
 import { z } from "zod";
 // ---------------------------------------------------------------------------
@@ -238,7 +238,7 @@ export function detectLegacyFieldNames(raw, filename) {
             if (legacy in obj) {
                 warnings.push(`[${filename}] ${taskId}: Found legacy field "${legacy}" — ` +
                     `use "${canonical}" instead. ` +
-                    "See CONTRIBUTING_TASKS.md for the canonical task format.");
+                    "See contributing-tasks.md for the canonical task format.");
             }
         }
     }

package/dist/adapters/task-sources/repo-task-source.js

@@ -64,7 +64,7 @@ export class RepoTaskSource {
                 throw new Error(`${file} uses legacy field names from @sanity/ailf-tasks.\n` +
                     "Task files must use canonical GeneralizedTaskDefinition field names.\n\n" +
                     legacyWarnings.join("\n") +
-                    "\n\nSee CONTRIBUTING_TASKS.md for the canonical task format.");
+                    "\n\nSee contributing-tasks.md for the canonical task format.");
             }
             // Validate through canonical Zod schema
             let validated;

package/dist/adapters/task-sources/repo-trigger.d.ts

@@ -6,7 +6,7 @@
  * config applies. This drives whether the pipeline runs in validate-only
  * mode or full eval mode, and whether results are blocking.
  *
- * @see docs/exec-plans/tasks-as-content/phase-4-repo-based-tasks.md
+ * @see docs/archive/exec-plans/tasks-as-content/phase-4-repo-based-tasks.md
  */
 export type TriggerContext = {
     type: "pr";

package/dist/adapters/task-sources/repo-trigger.js

@@ -6,7 +6,7 @@
  * config applies. This drives whether the pipeline runs in validate-only
  * mode or full eval mode, and whether results are blocking.
  *
- * @see docs/exec-plans/tasks-as-content/phase-4-repo-based-tasks.md
+ * @see docs/archive/exec-plans/tasks-as-content/phase-4-repo-based-tasks.md
  */
 import { existsSync, readFileSync } from "fs";
 import { resolve } from "path";

package/dist/adapters/task-sources/task-file-loader.d.ts

@@ -1,9 +1,8 @@
 /**
  * TaskFileLoader — loads task definitions from TypeScript files.
  *
- * Supplements the existing YAML-based task loading by supporting
- * `*.task.ts` and `*.task.js` files in task directories. Files are
- * loaded via jiti and expected to export GeneralizedTaskDefinition
+ * Supports `*.task.ts` and `*.task.js` files in task directories. Files
+ * are loaded via jiti and expected to export GeneralizedTaskDefinition
  * objects authored with `defineTask()`.
  *
  * TS task files export a single task or an array of tasks:
@@ -20,9 +19,6 @@
  * ]
  * ```
  *
- * The loader integrates into the existing RepoTaskSource adapter — TS
- * task files are discovered alongside YAML files in the same directory.
- *
  * @see docs/design-docs/architecture-overhaul/typescript-configuration.md
  */
 /** A raw task object loaded from a TS file (pre-validation) */
@@ -55,6 +51,13 @@ export declare function discoverTsTaskFiles(tasksDir: string): string[];
  * @returns The loaded task(s), or throws on load failure
  */
 export declare function loadTsTaskFile(filePath: string): Promise<RawTsTask>;
+/**
+ * Synchronously load task definitions from a single TS/JS task file.
+ *
+ * Uses jiti's synchronous require-style loading instead of async import().
+ * Needed by resolve-mappings.ts which is called from sync contexts.
+ */
+export declare function loadTsTaskFileSync(filePath: string): RawTsTask;
 /**
  * Load all TS task files from a directory.
  *

package/dist/adapters/task-sources/task-file-loader.js

@@ -1,9 +1,8 @@
 /**
  * TaskFileLoader — loads task definitions from TypeScript files.
  *
- * Supplements the existing YAML-based task loading by supporting
- * `*.task.ts` and `*.task.js` files in task directories. Files are
- * loaded via jiti and expected to export GeneralizedTaskDefinition
+ * Supports `*.task.ts` and `*.task.js` files in task directories. Files
+ * are loaded via jiti and expected to export GeneralizedTaskDefinition
  * objects authored with `defineTask()`.
  *
  * TS task files export a single task or an array of tasks:
@@ -20,13 +19,12 @@
  * ]
  * ```
  *
- * The loader integrates into the existing RepoTaskSource adapter — TS
- * task files are discovered alongside YAML files in the same directory.
- *
  * @see docs/design-docs/architecture-overhaul/typescript-configuration.md
  */
 import { existsSync, readdirSync } from "fs";
+import { pathToFileURL } from "node:url";
 import { resolve } from "path";
+import { createJiti } from "jiti";
 import { loadTsConfig } from "../config-sources/ts-config-loader.js";
 /**
  * Discover TS/JS task files in a directory.
@@ -67,6 +65,22 @@ export async function loadTsTaskFile(filePath) {
     const tasks = Array.isArray(value) ? value : [value];
     return { filePath, tasks };
 }
+/**
+ * Synchronously load task definitions from a single TS/JS task file.
+ *
+ * Uses jiti's synchronous require-style loading instead of async import().
+ * Needed by resolve-mappings.ts which is called from sync contexts.
+ */
+export function loadTsTaskFileSync(filePath) {
+    const jiti = createJiti(pathToFileURL(filePath).href, {
+        interopDefault: true,
+        requireCache: true,
+    });
+    const mod = jiti(filePath);
+    const value = mod && typeof mod === "object" && "default" in mod ? mod.default : mod;
+    const tasks = Array.isArray(value) ? value : [value];
+    return { filePath, tasks };
+}
 /**
  * Load all TS task files from a directory.
  *

package/dist/agent-observer/test-imports.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Final validation — ensures all agent-observer modules work together
+ * and the full data pipeline (record → classify → summarize) is correct.
+ *
+ * Run: tsx src/agent-observer/test-imports.ts
+ */
+export {};