@sanity/ailf 1.0.0 → 2.0.1

package/dist/_vendor/ailf-core/types/index.d.ts

@@ -9,12 +9,12 @@
  * Ports & Adapters migration (Phase 0c). The original file is now a
  * re-export barrel that preserves backward compatibility.
  */
-import type { DocumentRef as _DocumentRef, EvalMode as _EvalMode } from "../../ailf-shared/index.d.ts";
+import type { DocumentRef as _DocumentRef, EvalMode } from "../../ailf-shared/index.d.ts";
 export type { ActualScoreEntry, ComponentResult, TestResult, UrlMetadata, } from "./scoring-input.js";
 export type { DocumentRef } from "../../ailf-shared/index.d.ts";
 export type { StoredBaseline, StoredReport, StoredRun, StoredTaskResult, StoredTrace, SchemaVersioned, } from "./storage-schema.js";
 export { CURRENT_SCHEMA_VERSION, isSchemaVersioned, migrateDocument, } from "./storage-schema.js";
-export type { AssertionRegistration, FixtureResolverRegistration, ModeRegistration, PluginManifest, PluginRegistry, PresetDefinition, ReportSinkRegistration, RubricTemplateRegistration, } from "./plugin-registry.js";
+export type { AssertionRegistration, FixtureResolverRegistration, ModeBase, ModeRegistration, PluginManifest, PluginRegistry, PresetDefinition, ReportSinkRegistration, RubricTemplateRegistration, } from "./plugin-registry.js";
 export { InMemoryPluginRegistry } from "./plugin-registry.js";
 export type { AgentHarnessConfig, AgentHarnessModeConfig, CustomModeConfig, EvalModeConfig, EvalModeType, KnowledgeBaseRef, KnowledgeProbeModeConfig, LiteracyModeConfig, MCPServerConfig, MCPServerModeConfig, ProbeStrategy, SandboxConfig, ToolDef, } from "./eval-mode-config.js";
 export { evalModeType } from "./eval-mode-config.js";
@@ -25,7 +25,6 @@ export type { ArtifactId, Brand, Err, FixtureId, IdValidationError, NewReportId,
 export { err, fixtureId, ok, providerId, resultId, runId, suiteId, taskId, traceId, } from "./branded-ids.js";
 export type { AgentHarnessTaskDefinition, CustomTaskDefinition, GeneralizedAssertionDefinition, GeneralizedDocRef, GeneralizedTaskDefinition, GeneralizedTemplatedAssertion, GeneralizedValueAssertion, IdDocRef, KnowledgeProbeTaskDefinition, LiteracyTaskDefinition, MCPServerTaskDefinition, PathDocRef, PerspectiveDocRef, RubricRef, SlugDocRef, TaskCommonFields, TaskDifficulty, TaskOptions, TaskProviderConfig, TaskStatus, } from "./generalized-task.js";
 type DocumentRef = _DocumentRef;
-type EvalMode = _EvalMode;
 /** Aggregated retrieval metrics for a feature area */
 export interface AreaRetrievalMetrics {
     area: string;
@@ -119,7 +118,7 @@ export interface FailureModeReport {
     totalJudgments: number;
 }
 /** Failure mode classification for a low-scoring judgment */
-export type FailureModeType = "incorrect-docs" | "missing-docs" | "model-limitation" | "outdated-docs" | "poor-structure" | "unclassified";
+export type FailureModeType = "api-error" | "incorrect-docs" | "missing-docs" | "model-limitation" | "outdated-docs" | "poor-structure" | "unclassified";
 /** Per-feature-area score breakdown */
 export interface FeatureScore {
     /**
@@ -352,11 +351,40 @@ export interface ModelEntry {
     env?: string;
     id: string;
     label: string;
-    modes?: string[];
+    /**
+     * Which evaluation modes this model participates in.
+     *
+     * Values must be canonical eval mode names (e.g., "literacy", "mcp-server").
+     * When omitted, the model participates in all modes.
+     */
+    modes?: EvalMode[];
+    /**
+     * Per-provider timeout in ms. Emitted into Promptfoo provider config.
+     * Default: 300_000 (5 min, matching Promptfoo's built-in default).
+     */
+    timeoutMs?: number;
+    /**
+     * Per-mode variant whitelist. Keys are eval mode IDs, values are arrays
+     * of variant IDs to include for that mode.
+     *
+     * When a model enrolls in a mode (via `modes`) but does not specify
+     * variants for it here, ALL variants defined by the mode base are included.
+     *
+     * Only meaningful for modes that define variants (e.g., literacy has
+     * "baseline", "observed", "agentic-naive", "agentic-optimized").
+     * Ignored for modes without variants.
+     */
+    variants?: Partial<Record<EvalMode, string[]>>;
 }
 /** Parsed config/models.yaml structure */
 export interface ModelsConfig {
     defaults: Record<string, unknown>;
+    /**
+     * Maximum wall-clock time per eval step (all tests for one mode) in ms.
+     * When exceeded, the subprocess is killed and partial results are used.
+     * Default: no limit (backward compatible).
+     */
+    evalBudgetMs?: number;
     grader: {
         id: string;
         label?: string;
@@ -507,6 +535,21 @@ export interface TestSummary {
         task: string;
         error: string;
     }[];
+    /** Per-test timing statistics (when latencyMs is available from Promptfoo) */
+    timing?: {
+        /** Median test duration in ms */
+        medianMs: number;
+        /** 95th percentile test duration in ms */
+        p95Ms: number;
+        /** Maximum test duration in ms */
+        maxMs: number;
+        /** Tests that exceeded the slow threshold (2x median, min 60s) */
+        slowTests: {
+            task: string;
+            model: string;
+            durationMs: number;
+        }[];
+    };
 }
 /** Token usage and estimated cost for a pipeline run. */
 export interface PipelineUsage {
@@ -575,8 +618,6 @@ export interface ProductFeature {
     sections: string[];
     /** Coverage status */
     status: "covered" | "out-of-scope" | "planned" | "uncovered";
-    /** Number of evaluation tasks (if covered) */
-    taskCount?: number;
 }
 /** Full classification of a content release for evaluation */
 export interface ReleaseClassification {

package/dist/_vendor/ailf-core/types/plugin-registry.d.ts

@@ -1,12 +1,12 @@
 /**
  * Plugin registry — typed extension points for AILF evaluation capabilities.
  *
- * Twelve extension points: evaluation modes, providers, assertions,
- * rubric templates, fixture resolvers, report sinks, dashboard renderers,
- * prompt templates, scoring profiles, doc fetcher factory, source defs,
- * and feature defs.
- *
- * Presets bundle multiple extensions into a single installable unit.
+ * Three-tier architecture:
+ * - **Mode bases** define evaluation methodology (rubrics, scoring, prompts)
+ * - **Domain presets** target a mode base and add domain config (sources,
+ *   features, doc fetcher)
+ * - **Framework assertions** are generic evaluation primitives available to
+ *   all modes
  *
  * @see docs/design-docs/architecture-overhaul/extensibility-plugins.md
  */
@@ -14,6 +14,25 @@ import type { PromptTemplate } from "../ports/mode-handler.js";
 import type { DocFetcher } from "../ports/doc-fetcher.js";
 import type { SourceEntry } from "../config-helpers.js";
 import type { FeatureRegistry } from "../schemas/pipeline.js";
+/**
+ * A named variant within an evaluation mode.
+ *
+ * Modes can declare variants to represent different evaluation strategies
+ * that share the same methodology. For example, the literacy mode has
+ * "baseline", "observed", and "agentic-*" variants.
+ *
+ * Models opt into specific variants via `ModelEntry.variants`. When a model
+ * enrolls in a mode without specifying variants, all defined variants are
+ * included by default.
+ */
+export interface ModeVariantDefinition {
+    /** Variant identifier (e.g., "baseline", "agentic-naive") */
+    id: string;
+    /** Human-readable label (e.g., "Standard (baseline)") */
+    label: string;
+    /** Optional description for docs/CLI help */
+    description?: string;
+}
 /** A registered evaluation mode handler */
 export interface ModeRegistration {
     /** Unique mode identifier (e.g., "api-contract") */
@@ -26,6 +45,14 @@ export interface ModeRegistration {
     rubricTemplateIds: string[];
     /** Compile function module path (loaded at runtime) */
     handlerModule: string;
+    /**
+     * Variants this mode supports. Omit or empty for modes without variants.
+     *
+     * When defined, models can selectively opt into specific variants via
+     * `ModelEntry.variants`. Models that enroll in the mode without specifying
+     * variants participate in all defined variants.
+     */
+    variants?: ModeVariantDefinition[];
 }
 /** A registered assertion type */
 export interface AssertionRegistration {
@@ -33,8 +60,12 @@ export interface AssertionRegistration {
     type: string;
     /** Human-readable label */
     label: string;
-    /** Which modes this assertion is compatible with */
-    compatibleModes: string[];
+    /**
+     * Which modes this assertion is compatible with.
+     * When omitted, the assertion is compatible with all modes.
+     * When specified, acts as a whitelist of mode IDs.
+     */
+    compatibleModes?: string[];
     /** Assertion handler module path */
     handlerModule: string;
 }
@@ -65,6 +96,30 @@ export interface ReportSinkRegistration {
     /** Sink module path */
     handlerModule: string;
 }
+/**
+ * ModeBase — shared evaluation methodology for a mode.
+ *
+ * Defines HOW you evaluate (rubrics, scoring, prompts) independently of
+ * WHAT you're evaluating (sources, features, docs). Multiple domain presets
+ * can target the same mode base and inherit its defaults.
+ *
+ * Example: the "literacy" mode base defines rubric templates for
+ * task-completion, code-correctness, and doc-coverage. Both a Sanity docs
+ * preset and an external docs preset can target "literacy" and inherit
+ * these rubrics without redefining them.
+ */
+export interface ModeBase {
+    /** The mode registration (handler, provider patterns, rubric template IDs) */
+    mode: ModeRegistration;
+    /** Default rubric templates for this mode */
+    rubricTemplates?: RubricTemplateRegistration[];
+    /** Default scoring profiles for this mode (profile name → dimension weights) */
+    scoringProfiles?: Record<string, Record<string, number>>;
+    /** Default prompt templates for this mode (template name → template) */
+    promptTemplates?: Record<string, PromptTemplate>;
+    /** Mode-specific assertion types (beyond framework builtins) */
+    assertions?: AssertionRegistration[];
+}
 /** Plugin manifest describing a single plugin */
 export interface PluginManifest {
     /** Plugin name (npm package style) */
@@ -80,32 +135,49 @@ export interface PluginManifest {
     /** Dependencies on other plugins */
     requires?: string[];
 }
-/** A preset bundles multiple extensions into an installable unit */
+/**
+ * A domain preset targets a mode base and adds domain-specific configuration.
+ *
+ * The preset inherits evaluation methodology (rubrics, scoring, prompts) from
+ * its mode base. It can optionally override any inherited values.
+ */
 export interface PresetDefinition {
-    /** Preset name */
+    /** Preset name (unique identifier) */
     name: string;
     /** Plugin manifest */
     manifest: PluginManifest;
-    /** Evaluation modes to register */
-    modes?: ModeRegistration[];
-    /** Assertion types to register */
-    assertions?: AssertionRegistration[];
-    /** Rubric templates to register */
-    rubricTemplates?: RubricTemplateRegistration[];
-    /** Fixture resolvers to register */
+    /**
+     * Lifecycle status — mirrors task status semantics.
+     *   active:   registered and used in evaluations (default)
+     *   draft:    registered but skipped unless explicitly targeted
+     *   paused:   registered but skipped (can be resumed)
+     *   archived: not registered
+     */
+    status?: "active" | "archived" | "draft" | "paused";
+    /**
+     * Which mode this preset targets (by mode ID).
+     * Links to a registered ModeBase. The preset inherits rubrics,
+     * scoring profiles, and prompt templates from the base.
+     */
+    mode: string;
+    /** Fixture resolvers */
     fixtureResolvers?: FixtureResolverRegistration[];
-    /** Report sinks to register */
+    /** Report sinks */
     reportSinks?: ReportSinkRegistration[];
-    /** Prompt templates keyed by template name (e.g. "with-docs", "agentic") */
-    promptTemplates?: Record<string, PromptTemplate>;
-    /** Scoring profiles mapping profile name to dimension-weight pairs */
-    scoringProfiles?: Record<string, Record<string, number>>;
     /** Factory function that creates a DocFetcher instance */
     docFetcher?: () => DocFetcher;
     /** Documentation source definitions (production, branch, local, etc.) */
     sourceDefs?: SourceEntry[];
     /** Product feature registry for coverage tracking */
     featureDefs?: FeatureRegistry;
+    /** Override rubric templates (merged by ID with mode base) */
+    rubricTemplates?: RubricTemplateRegistration[];
+    /** Override scoring profiles (merged by name with mode base) */
+    scoringProfiles?: Record<string, Record<string, number>>;
+    /** Override prompt templates (merged by name with mode base) */
+    promptTemplates?: Record<string, PromptTemplate>;
+    /** Additional mode-specific assertions */
+    assertions?: AssertionRegistration[];
 }
 /**
  * PluginRegistry — central registry for all AILF extensions.
@@ -154,10 +226,16 @@ export interface PluginRegistry {
     registerSourceDefs(sources: SourceEntry[]): void;
     /** Get all registered source definitions */
     getSourceDefs(): SourceEntry[];
-    /** Register a feature registry (last-write-wins) */
+    /** Register a feature registry (merged by feature ID with existing) */
     registerFeatureDefs(features: FeatureRegistry): void;
     /** Get the registered feature registry, if any */
     getFeatureDefs(): FeatureRegistry | undefined;
+    /** Register a mode base (evaluation methodology) */
+    registerModeBase(base: ModeBase): void;
+    /** Get a mode base by mode ID */
+    getModeBase(modeId: string): ModeBase | undefined;
+    /** Get all registered mode bases */
+    getModeBases(): ModeBase[];
     /** Get all registered presets */
     getPresets(): PresetDefinition[];
 }
@@ -170,6 +248,7 @@ export declare class InMemoryPluginRegistry implements PluginRegistry {
     private readonly rubricTemplates_;
     private readonly fixtureResolvers_;
     private readonly reportSinks_;
+    private readonly modeBases_;
     private readonly presets_;
     private promptTemplates_;
     private scoringProfiles_;
@@ -199,4 +278,7 @@ export declare class InMemoryPluginRegistry implements PluginRegistry {
     getSourceDefs(): SourceEntry[];
     registerFeatureDefs(features: FeatureRegistry): void;
     getFeatureDefs(): FeatureRegistry | undefined;
+    registerModeBase(base: ModeBase): void;
+    getModeBase(modeId: string): ModeBase | undefined;
+    getModeBases(): ModeBase[];
 }

package/dist/_vendor/ailf-core/types/plugin-registry.js

@@ -1,12 +1,12 @@
 /**
  * Plugin registry — typed extension points for AILF evaluation capabilities.
  *
- * Twelve extension points: evaluation modes, providers, assertions,
- * rubric templates, fixture resolvers, report sinks, dashboard renderers,
- * prompt templates, scoring profiles, doc fetcher factory, source defs,
- * and feature defs.
- *
- * Presets bundle multiple extensions into a single installable unit.
+ * Three-tier architecture:
+ * - **Mode bases** define evaluation methodology (rubrics, scoring, prompts)
+ * - **Domain presets** target a mode base and add domain config (sources,
+ *   features, doc fetcher)
+ * - **Framework assertions** are generic evaluation primitives available to
+ *   all modes
  *
  * @see docs/design-docs/architecture-overhaul/extensibility-plugins.md
  */
@@ -19,6 +19,7 @@ export class InMemoryPluginRegistry {
     rubricTemplates_ = new Map();
     fixtureResolvers_ = new Map();
     reportSinks_ = new Map();
+    modeBases_ = new Map();
     presets_ = new Map();
     promptTemplates_ = {};
     scoringProfiles_ = {};
@@ -26,19 +27,56 @@ export class InMemoryPluginRegistry {
     sourceDefs_ = [];
     featureDefs_;
     registerPreset(preset) {
+        // Skip archived presets entirely
+        if (preset.status === "archived")
+            return;
+        // Store draft/paused presets in the map (for later activation via
+        // --preset flag) but skip all side-effect registrations. This prevents
+        // a draft preset from silently overwriting the doc fetcher, merging
+        // scoring profiles, etc.
         this.presets_.set(preset.name, preset);
-        if (preset.modes) {
-            for (const mode of preset.modes)
-                this.registerMode(mode);
+        if (preset.status === "draft" || preset.status === "paused")
+            return;
+        // Resolve mode base defaults
+        const base = this.modeBases_.get(preset.mode);
+        if (!base) {
+            throw new Error(`Preset "${preset.name}" targets mode "${preset.mode}" ` +
+                `but no mode base is registered for it. ` +
+                `Available mode bases: ${[...this.modeBases_.keys()].join(", ") || "(none)"}`);
+        }
+        // Mode is already registered by registerModeBase() — no need to re-register.
+        // Merge rubric templates: base defaults + preset overrides (by ID)
+        const baseRubrics = new Map((base.rubricTemplates ?? []).map((r) => [r.id, r]));
+        for (const r of preset.rubricTemplates ?? []) {
+            baseRubrics.set(r.id, r);
+        }
+        for (const r of baseRubrics.values()) {
+            this.registerRubricTemplate(r);
+        }
+        // Merge scoring profiles: base defaults + preset overrides (by name)
+        const profiles = {
+            ...base.scoringProfiles,
+            ...preset.scoringProfiles,
+        };
+        if (Object.keys(profiles).length > 0) {
+            this.registerScoringProfiles(profiles);
         }
+        // Merge prompt templates: base defaults + preset overrides (by name)
+        const prompts = {
+            ...base.promptTemplates,
+            ...preset.promptTemplates,
+        };
+        if (Object.keys(prompts).length > 0) {
+            this.registerPromptTemplates(prompts);
+        }
+        // Merge assertions: base + preset (preset overrides by type)
+        for (const a of base.assertions ?? [])
+            this.registerAssertion(a);
         if (preset.assertions) {
             for (const a of preset.assertions)
                 this.registerAssertion(a);
         }
-        if (preset.rubricTemplates) {
-            for (const t of preset.rubricTemplates)
-                this.registerRubricTemplate(t);
-        }
+        // Register domain-specific fields
         if (preset.fixtureResolvers) {
             for (const r of preset.fixtureResolvers)
                 this.registerFixtureResolver(r);
@@ -47,12 +85,6 @@ export class InMemoryPluginRegistry {
             for (const s of preset.reportSinks)
                 this.registerReportSink(s);
         }
-        if (preset.promptTemplates) {
-            this.registerPromptTemplates(preset.promptTemplates);
-        }
-        if (preset.scoringProfiles) {
-            this.registerScoringProfiles(preset.scoringProfiles);
-        }
         if (preset.docFetcher) {
             this.registerDocFetcherFactory(preset.docFetcher);
         }
@@ -124,9 +156,30 @@ export class InMemoryPluginRegistry {
         return this.sourceDefs_;
     }
     registerFeatureDefs(features) {
-        this.featureDefs_ = features;
+        if (!this.featureDefs_) {
+            this.featureDefs_ = features;
+            return;
+        }
+        // Merge by feature ID: new features override existing on ID collision,
+        // existing features not in new set are preserved.
+        const merged = new Map(this.featureDefs_.features.map((f) => [f.id, f]));
+        for (const f of features.features) {
+            merged.set(f.id, f);
+        }
+        this.featureDefs_ = { features: [...merged.values()] };
     }
     getFeatureDefs() {
         return this.featureDefs_;
     }
+    registerModeBase(base) {
+        this.modeBases_.set(base.mode.id, base);
+        // Also register the mode itself so getMode() works
+        this.registerMode(base.mode);
+    }
+    getModeBase(modeId) {
+        return this.modeBases_.get(modeId);
+    }
+    getModeBases() {
+        return [...this.modeBases_.values()];
+    }
 }

package/dist/_vendor/ailf-shared/eval-modes.d.ts

@@ -37,6 +37,21 @@ export type RawEvalMode = EvalMode | "agentic" | "baseline" | "full" | "observed
 export declare const CANONICAL_EVAL_MODES: readonly ["literacy", "mcp-server", "agent-harness", "knowledge-probe", "custom"];
 /** Legacy CLI aliases that map to `mode: "literacy"` + variant. */
 export declare const LEGACY_EVAL_MODE_ALIASES: readonly ["baseline", "agentic", "observed", "full"];
+/**
+ * Literacy mode variant names — each is a distinct evaluation strategy.
+ *
+ * These are the valid values for the `variant` field in PipelineRequest
+ * when `mode` is `"literacy"`. They match LEGACY_EVAL_MODE_ALIASES because
+ * variants were originally exposed as top-level mode names.
+ *
+ *   - `baseline` — with-docs / without-docs comparison (gold + floor)
+ *   - `agentic`  — model uses tools to find docs (gold only)
+ *   - `observed` — HTTP-instrumented behavior observation
+ *   - `full`     — combined baseline + agentic
+ */
+export declare const LITERACY_VARIANTS: readonly ["baseline", "agentic", "observed", "full"];
+/** Union of all literacy variant string values. */
+export type LiteracyVariant = (typeof LITERACY_VARIANTS)[number];
 /**
  * All accepted mode names for Zod enum construction.
  * Canonical modes first, then legacy aliases.

package/dist/_vendor/ailf-shared/eval-modes.js

@@ -22,6 +22,24 @@ export const LEGACY_EVAL_MODE_ALIASES = [
     "observed",
     "full",
 ];
+/**
+ * Literacy mode variant names — each is a distinct evaluation strategy.
+ *
+ * These are the valid values for the `variant` field in PipelineRequest
+ * when `mode` is `"literacy"`. They match LEGACY_EVAL_MODE_ALIASES because
+ * variants were originally exposed as top-level mode names.
+ *
+ *   - `baseline` — with-docs / without-docs comparison (gold + floor)
+ *   - `agentic`  — model uses tools to find docs (gold only)
+ *   - `observed` — HTTP-instrumented behavior observation
+ *   - `full`     — combined baseline + agentic
+ */
+export const LITERACY_VARIANTS = [
+    "baseline",
+    "agentic",
+    "observed",
+    "full",
+];
 /**
  * All accepted mode names for Zod enum construction.
  * Canonical modes first, then legacy aliases.

package/dist/adapters/api-client/remediation.js

@@ -17,7 +17,7 @@ const HINTS = [
             /no article found for slug/i.test(e.message),
         hint: "One or more `canonicalDocs` slugs in your task definitions don't match " +
             "any article in the documentation. Check the `slug` values in " +
-            "`.ailf/tasks/*.yaml` and ensure they correspond to real articles.\n" +
+            "`.ailf/tasks/` and ensure they correspond to real articles.\n" +
             "  Run `ailf validate` to check your task definitions locally.",
     },
     {
@@ -51,7 +51,7 @@ const HINTS = [
         hint: "The documentation fetch step completed but one or more tasks had " +
             "empty context. This usually means a `canonicalDocs` slug doesn't " +
             "match any article.\n" +
-            "  Check the slug values in `.ailf/tasks/*.yaml`.",
+            "  Check the slug values in `.ailf/tasks/`.",
     },
     {
         match: (e) => e.step === "dispatch" && /dispatch failed/i.test(e.message),

package/dist/adapters/config-sources/file-config-adapter.js

@@ -22,7 +22,7 @@
  * @see docs/design-docs/architecture-overhaul/typescript-configuration.md
  */
 import { readFileSync } from "fs";
-import { extname } from "path";
+import { extname, resolve } from "path";
 import { EvalConfigSchema, PipelineRequestSchema, } from "../../_vendor/ailf-core/index.js";
 import { mapRequestToConfig } from "../../pipeline/map-request-to-config.js";
 import { normalizeMode } from "../../pipeline/normalize-mode.js";
@@ -93,6 +93,7 @@ function mapEvalConfigToResolvedConfig(config, rootDir) {
     const normalized = normalizeMode(config.mode ?? "literacy");
     return {
         rootDir,
+        outputDir: resolve(rootDir, "results", "latest"),
         mode: normalized.mode,
         variant: normalized.variant,
         noAutoScope: config.noAutoScope ?? false,
@@ -119,7 +120,12 @@ function mapEvalConfigToResolvedConfig(config, rootDir) {
         allowedOrigins: config.allowedOrigins,
         searchMode: config.searchMode ?? "open",
         concurrency: config.concurrency,
+        captureEnabled: false,
+        captureDir: undefined,
+        captureCompress: true,
+        captureExtras: true,
         remote: false,
         apiUrl: "https://ailf-api.sanity.build",
+        presets: config.presets,
     };
 }

package/dist/adapters/config-sources/ts-config-loader.js

@@ -13,21 +13,29 @@
  * @see docs/design-docs/architecture-overhaul/typescript-configuration.md
  */
 import { existsSync } from "fs";
+import { pathToFileURL } from "node:url";
 import { createJiti } from "jiti";
 // ---------------------------------------------------------------------------
-// Singleton jiti instance — reused across all loads for caching
+// jiti instance factory — resolves imports relative to the loaded file
 // ---------------------------------------------------------------------------
-let _jiti = null;
-function getJiti() {
-    if (!_jiti) {
-        _jiti = createJiti(import.meta.url, {
-            // Interop: handle both `export default` and `module.exports`
-            interopDefault: true,
-            // Don't require file extensions in imports
-            requireCache: true,
-        });
-    }
-    return _jiti;
+/**
+ * Create a jiti instance that resolves bare-specifier imports relative to
+ * the given file path, not relative to this loader module.
+ *
+ * This is critical for pnpm workspaces: a task file at `.ailf/tasks/foo.task.ts`
+ * importing `@sanity/ailf` must resolve through the dependency graph
+ * visible from the task file's directory, not from deep inside packages/eval/.
+ *
+ * We pass a `file://` URL (not a bare path) so jiti uses ESM resolution,
+ * which matches the `"import"` condition in package.json exports maps.
+ */
+function createJitiForFile(filePath) {
+    return createJiti(pathToFileURL(filePath).href, {
+        // Interop: handle both `export default` and `module.exports`
+        interopDefault: true,
+        // Don't require file extensions in imports
+        requireCache: true,
+    });
 }
 /**
  * Load a TypeScript or JavaScript config file and return its default export.
@@ -43,7 +51,7 @@ export async function loadTsConfig(filePath) {
         return { ok: false, error: `File not found: ${filePath}`, path: filePath };
     }
     try {
-        const jiti = getJiti();
+        const jiti = createJitiForFile(filePath);
         const mod = await jiti.import(filePath);
         const value = extractDefault(mod);
         if (value === undefined || value === null) {

package/dist/adapters/eval-runners/promptfoo-eval-adapter.js

@@ -33,6 +33,7 @@ export class PromptfooEvalAdapter {
                 cwd: this.rootDir,
                 env: { ...process.env, ...config.env },
                 stdio: "inherit",
+                ...(config.maxDurationMs ? { timeout: config.maxDurationMs } : {}),
             });
             return {
                 durationMs: Date.now() - start,
@@ -40,10 +41,15 @@ export class PromptfooEvalAdapter {
                 summary: `Evaluation complete (${config.configPath})`,
             };
         }
-        catch {
+        catch (err) {
+            const isTimeout = err instanceof Error &&
+                "killed" in err &&
+                err.killed === true;
             return {
                 durationMs: Date.now() - start,
-                error: `Promptfoo evaluation failed: ${config.configPath}`,
+                error: isTimeout
+                    ? `Eval subprocess killed after ${config.maxDurationMs}ms time budget`
+                    : `Promptfoo evaluation failed: ${config.configPath}`,
                 status: "failed",
             };
         }

package/dist/adapters/index.d.ts

@@ -9,4 +9,3 @@ export { SanityDocFetcher } from "./doc-fetchers/index.js";
 export { PromptfooEvalAdapter } from "./eval-runners/index.js";
 export { ConsoleLogger, type ConsoleLoggerOptions, JsonLogger, QuietLogger, } from "./loggers/index.js";
 export { CliConfigAdapter, FileConfigAdapter } from "./config-sources/index.js";
-export { YamlTaskSource } from "./task-sources/index.js";

package/dist/adapters/index.js

@@ -9,4 +9,3 @@ export { SanityDocFetcher } from "./doc-fetchers/index.js";
 export { PromptfooEvalAdapter } from "./eval-runners/index.js";
 export { ConsoleLogger, JsonLogger, QuietLogger, } from "./loggers/index.js";
 export { CliConfigAdapter, FileConfigAdapter } from "./config-sources/index.js";
-export { YamlTaskSource } from "./task-sources/index.js";

package/dist/adapters/task-sources/composite-task-source.d.ts

@@ -10,7 +10,7 @@
  * tasks in a single GeneralizedTaskDefinition[].
  *
  * @see packages/core/src/ports/task-source.ts — TaskSource port
- * @see docs/exec-plans/tasks-as-content/phase-4-repo-based-tasks.md
+ * @see docs/archive/exec-plans/tasks-as-content/phase-4-repo-based-tasks.md
  */
 import type { FilterOptions, GeneralizedTaskDefinition, TaskSource } from "../../_vendor/ailf-core/index.d.ts";
 export declare class CompositeTaskSource implements TaskSource {

package/dist/adapters/task-sources/composite-task-source.js

@@ -10,7 +10,7 @@
  * tasks in a single GeneralizedTaskDefinition[].
  *
  * @see packages/core/src/ports/task-source.ts — TaskSource port
- * @see docs/exec-plans/tasks-as-content/phase-4-repo-based-tasks.md
+ * @see docs/archive/exec-plans/tasks-as-content/phase-4-repo-based-tasks.md
  */
 export class CompositeTaskSource {
     sources;