@sanity/ailf 2.0.0 → 2.0.2

package/dist/_vendor/ailf-core/index.js

@@ -20,3 +20,4 @@ export * from "./examples/index.js";
 // ---------------------------------------------------------------------------
 export { defineConfig, defineFeatures, defineModeBase, defineModels, definePricingTable, definePreset, definePrompts, defineRubrics, defineSchedules, defineSinks, defineSources, defineTask, defineThresholds, } from "./config-helpers.js";
 export { env } from "./env-helper.js";
+export { NoOpArtifactCollector } from "./artifact-capture/noop-collector.js";

package/dist/_vendor/ailf-core/ports/artifact-collector.d.ts

@@ -0,0 +1,94 @@
+/**
+ * Port: ArtifactCollector — captures pipeline artifacts during execution.
+ *
+ * Injected into AppContext. When capture is disabled (default), the
+ * composition root provides NoOpArtifactCollector. When --capture is
+ * set, provides FilesystemArtifactCollector.
+ *
+ * Design principles:
+ * - P1: Zero-cost when off (no-op stub)
+ * - P2: Capture, don't intercept (steps call capture() explicitly)
+ * - P5: Non-blocking (failures swallowed, never block the pipeline)
+ */
+/**
+ * The contract for artifact capture during pipeline execution.
+ *
+ * Steps call capture() for in-memory data and captureFile() for
+ * artifacts already on disk. The orchestrator calls flush() once
+ * at pipeline end to write everything to the configured destination.
+ */
+export interface ArtifactCollector {
+    /**
+     * Record an in-memory artifact produced during pipeline execution.
+     *
+     * Callers need not check `enabled` before calling — the NoOp
+     * implementation is zero-cost, so unconditional calls are safe.
+     *
+     * @param step - Pipeline step name (e.g., "run-eval")
+     * @param type - Artifact type identifier (e.g., "eval-results")
+     * @param data - Content to serialize (JSON or text)
+     * @param meta - Optional metadata (variant, model, etc.)
+     */
+    capture(step: string, type: string, data: unknown, meta?: Record<string, unknown>): void;
+    /**
+     * Record a file reference for an artifact already on disk.
+     * The file is copied into the capture directory on flush().
+     *
+     * @param step - Pipeline step name
+     * @param type - Artifact type identifier
+     * @param filePath - Absolute path to the existing file
+     * @param meta - Optional metadata
+     */
+    captureFile(step: string, type: string, filePath: string, meta?: Record<string, unknown>): void;
+    /**
+     * Flush all captured artifacts to the configured destination.
+     * Called once at pipeline end by the orchestrator.
+     */
+    flush(): Promise<CaptureFlushResult>;
+    /** Whether capture is active */
+    readonly enabled: boolean;
+    /** Whether mode-specific extras are being captured */
+    readonly extrasEnabled: boolean;
+}
+/** Result of flushing captured artifacts to the destination. */
+export interface CaptureFlushResult {
+    /** Total number of artifacts captured */
+    artifactCount: number;
+    /** Output path (directory or .tar.gz) */
+    destination: string;
+    /** Total bytes written (uncompressed) */
+    totalBytes: number;
+    /** Whether output was compressed */
+    compressed: boolean;
+}
+/** A single entry in the capture manifest. */
+export interface ArtifactManifestEntry {
+    /** Pipeline step that produced this artifact */
+    step: string;
+    /** Artifact type identifier */
+    type: string;
+    /** Relative path within the capture directory */
+    path: string;
+    /** ISO 8601 timestamp of when capture() was called */
+    capturedAt: string;
+    /** Byte size of the artifact */
+    bytes: number;
+    /** Content format */
+    format: "json" | "markdown" | "text";
+    /** Optional metadata */
+    meta?: Record<string, unknown>;
+}
+/** The manifest.json written to each capture directory. */
+export interface ArtifactManifest {
+    version: 1;
+    captureId: string;
+    startedAt: string;
+    completedAt: string;
+    pipeline: {
+        mode: string;
+        variant?: string;
+        source?: string;
+        areas?: string[];
+    };
+    artifacts: ArtifactManifestEntry[];
+}

package/dist/_vendor/ailf-core/ports/artifact-collector.js

@@ -0,0 +1,13 @@
+/**
+ * Port: ArtifactCollector — captures pipeline artifacts during execution.
+ *
+ * Injected into AppContext. When capture is disabled (default), the
+ * composition root provides NoOpArtifactCollector. When --capture is
+ * set, provides FilesystemArtifactCollector.
+ *
+ * Design principles:
+ * - P1: Zero-cost when off (no-op stub)
+ * - P2: Capture, don't intercept (steps call capture() explicitly)
+ * - P5: Non-blocking (failures swallowed, never block the pipeline)
+ */
+export {};

package/dist/_vendor/ailf-core/ports/capture-comparator.d.ts

@@ -0,0 +1,138 @@
+/**
+ * Types for cross-run capture comparison.
+ *
+ * The CaptureComparator reads two capture directories (baseline + experiment)
+ * and produces a CaptureDiffReport. Types are defined in core so external
+ * tooling can consume diff reports without depending on the eval package.
+ *
+ * Implementation lives in packages/eval/src/artifact-capture/comparator.ts.
+ */
+/** How deeply to compare artifacts. */
+export type ComparisonMode = "strict" | "structural" | "inventory";
+/** Configurable thresholds for comparison. */
+export interface ComparisonOptions {
+    /** Comparison depth: inventory (existence), structural (shape), strict (content) */
+    mode: ComparisonMode;
+    /** Score regression thresholds */
+    scoreThresholds?: {
+        /** Maximum allowed aggregate score delta (percentage points, default 5) */
+        aggregate: number;
+        /** Maximum allowed per-task score drop (points, default 10) */
+        perTask: number;
+    };
+    /** Timing regression thresholds */
+    timingThresholds?: {
+        /** Multiplier — flag steps exceeding this ratio (default 2.0) */
+        multiplier: number;
+        /** Per-step overrides (step name → custom multiplier) */
+        perStep?: Record<string, number>;
+    };
+    /** JSON structural diff depth (default 3) */
+    jsonDiffDepth?: number;
+    /** Additional ephemeral fields to ignore (merged with defaults) */
+    ephemeralFields?: string[];
+}
+/** Inventory diff — which artifacts exist in each capture. */
+export interface InventoryDiff {
+    /** Artifact types in experiment but not in baseline */
+    added: string[];
+    /** Artifact types in baseline but not in experiment */
+    removed: string[];
+    /** Artifact types present in both */
+    common: string[];
+}
+/** A single structural change in a JSON artifact. */
+export interface JsonDiffEntry {
+    /** JSON pointer path (e.g., "config.mode") */
+    path: string;
+    /** Value in baseline (undefined if key is added) */
+    baseline?: unknown;
+    /** Value in experiment (undefined if key is removed) */
+    experiment?: unknown;
+}
+/** Content diff for a single artifact. */
+export interface ArtifactContentDiff {
+    /** Artifact type identifier (step/type) */
+    artifactKey: string;
+    /** Content format */
+    format: "json" | "markdown" | "text";
+    /** Structural changes (JSON) or line diff summary (text/markdown) */
+    changes: JsonDiffEntry[] | {
+        addedLines: number;
+        removedLines: number;
+    };
+}
+/** Score comparison between two captures. */
+export interface ScoreComparison {
+    /** Baseline aggregate score */
+    baselineMean: number;
+    /** Experiment aggregate score */
+    currentMean: number;
+    /** Absolute delta (current - baseline) */
+    delta: number;
+    /** Per-task score deltas */
+    perTask: {
+        task: string;
+        baseline: number;
+        current: number;
+        delta: number;
+    }[];
+    /** Tasks that breached configured thresholds */
+    breaches: string[];
+}
+/** Timing comparison between two captures. */
+export interface TimingComparison {
+    /** Total pipeline duration delta in ms */
+    totalDeltaMs: number;
+    /** Per-step timing */
+    perStep: {
+        step: string;
+        baselineMs: number;
+        currentMs: number;
+        ratio: number;
+    }[];
+    /** Steps that breached the timing multiplier threshold */
+    breaches: string[];
+}
+/** Metadata comparison between two captures. */
+export interface MetadataComparison {
+    /** Whether pipeline modes match */
+    modeMatch: boolean;
+    /** Whether pipeline variants match */
+    variantMatch: boolean;
+    /** Config key differences */
+    configDiffs: JsonDiffEntry[];
+}
+/** Security scan results. */
+export interface SecurityScan {
+    /** Whether any potential secret leaks were found */
+    leaksFound: boolean;
+    /** Details of each violation */
+    violations: {
+        /** Relative artifact file path */
+        file: string;
+        /** Description of the finding */
+        detail: string;
+    }[];
+}
+/** The full diff report produced by CaptureComparator. */
+export interface CaptureDiffReport {
+    /** Are the two captures semantically equivalent? */
+    equivalent: boolean;
+    /** Human-readable summary (1-3 sentences) */
+    summary: string;
+    /** Comparison mode used */
+    mode: ComparisonMode;
+    /** Artifact inventory diff */
+    inventory: InventoryDiff;
+    /** Content diffs for common artifacts (structural/strict modes only) */
+    content?: ArtifactContentDiff[];
+    /** Score comparison (if score-summary exists in both captures) */
+    scores?: ScoreComparison;
+    /** Timing comparison (if pipeline-context exists in both captures) */
+    timing?: TimingComparison;
+    /** Metadata comparison */
+    metadata?: MetadataComparison;
+    /** Security scan results */
+    security: SecurityScan;
+}

package/dist/_vendor/ailf-core/ports/capture-comparator.js

@@ -0,0 +1,10 @@
+/**
+ * Types for cross-run capture comparison.
+ *
+ * The CaptureComparator reads two capture directories (baseline + experiment)
+ * and produces a CaptureDiffReport. Types are defined in core so external
+ * tooling can consume diff reports without depending on the eval package.
+ *
+ * Implementation lives in packages/eval/src/artifact-capture/comparator.ts.
+ */
+export {};

package/dist/_vendor/ailf-core/ports/context.d.ts

@@ -12,6 +12,7 @@
  * as downstream consumers are converted to use them.
  */
 import type { DebugOptions, EvalMode, PluginRegistry } from "../types/index.js";
+import type { ArtifactCollector } from "./artifact-collector.js";
 import type { CacheStore } from "./cache-store.js";
 import type { DocFetcher } from "./doc-fetcher.js";
 import type { EvalRunner } from "./eval-runner.js";
@@ -78,6 +79,8 @@ export interface ResolvedConfig {
     noRemoteCache: boolean;
     /** Grader replications for consistency measurement */
     graderReplications?: number;
+    /** Base directory for user-facing pipeline output artifacts. */
+    outputDir: string;
     /** Output path override */
     outputPath?: string;
     /** Doc source URL overrides */
@@ -90,6 +93,12 @@ export interface ResolvedConfig {
     searchMode: "off" | "open" | "origin-only";
     /** Eval concurrency */
     concurrency?: number;
+    /**
+     * Maximum wall-clock time per eval step in ms.
+     * When exceeded, the subprocess is killed and partial results are used.
+     * Sourced from models config `evalBudgetMs`.
+     */
+    evalBudgetMs?: number;
     /** Promptfoo URL from eval output */
     promptfooUrl?: string;
     /** Sanity dataset override */
@@ -109,7 +118,7 @@ export interface ResolvedConfig {
     /** Before option for comparison */
     beforeOption?: string;
     /** Task source adapter selection */
-    taskSourceType?: "content-lake" | "repo" | "yaml";
+    taskSourceType?: "content-lake" | "repo";
     /** Path to repo-based tasks directory (e.g., .ailf/tasks/) */
     repoTasksPath?: string;
     /** Report store project ID from .ailf/config.yaml reportStore block */
@@ -142,6 +151,14 @@ export interface ResolvedConfig {
     apiKey?: string;
     /** External preset file paths or npm package names to load */
     presets?: string[];
+    /** Whether artifact capture is enabled for this run (default: false) */
+    captureEnabled?: boolean;
+    /** Base directory for capture output (default: results/captures/) */
+    captureDir?: string;
+    /** Whether to compress capture output to tar.gz (default: true) */
+    captureCompress?: boolean;
+    /** Whether to include mode-specific extra artifacts (default: true) */
+    captureExtras?: boolean;
 }
 /**
  * Application context — the complete dependency carrier.
@@ -158,6 +175,8 @@ export interface ResolvedConfig {
 export interface AppContext {
     /** Evaluation caching (filesystem + optional Content Lake fallback) */
     readonly cache?: CacheStore;
+    /** Artifact capture collector (no-op when --capture is not set) */
+    readonly collector: ArtifactCollector;
     /** Resolved pipeline configuration */
     readonly config: ResolvedConfig;
     /** Documentation context fetcher */

package/dist/_vendor/ailf-core/ports/eval-runner.d.ts

@@ -15,6 +15,12 @@ export interface EvalRunConfig {
     concurrency?: number;
     /** Environment variables to pass to the eval process */
     env?: Record<string, string>;
+    /**
+     * Maximum wall-clock time for this eval subprocess in ms.
+     * When exceeded, the process is killed and partial results are used.
+     * Default: no limit (backward compatible).
+     */
+    maxDurationMs?: number;
 }
 export interface EvalRunner {
     /** Run an evaluation and return the step result */

package/dist/_vendor/ailf-core/ports/index.d.ts

@@ -4,6 +4,8 @@
  * Ports define the contracts between the domain kernel and the outside world.
  * Adapters (in packages/eval) implement these interfaces.
  */
+export type { ArtifactCollector, ArtifactManifest, ArtifactManifestEntry, CaptureFlushResult, } from "./artifact-collector.js";
+export type { ArtifactContentDiff, CaptureDiffReport, ComparisonMode, ComparisonOptions, InventoryDiff, JsonDiffEntry, MetadataComparison, ScoreComparison, SecurityScan, TimingComparison, } from "./capture-comparator.js";
 export type { CacheEntryMetadata, CacheKey, CacheLookupResult, CacheRecordInput, CacheStore, } from "./cache-store.js";
 export type { ConfigSource } from "./config-source.js";
 export type { AppContext, ReportSinkPort, ReportStorePort, ResolvedConfig, } from "./context.js";

package/dist/_vendor/ailf-core/ports/pipeline-step.d.ts

@@ -38,6 +38,17 @@ export interface PipelineStep {
      * When defined, the StepRunner computes a hash and checks the cache.
      */
     cacheInputs?(ctx: AppContext): string[];
+    /**
+     * Cache context strings — non-file state that participates in cache key
+     * computation (e.g., mode, variant, area/task/tag filters).
+     *
+     * Without these, two runs with different CLI flags but identical config
+     * files would share a cache entry, causing cross-mode or cross-area
+     * contamination.
+     *
+     * When undefined, only file content determines the cache key.
+     */
+    cacheContext?(ctx: AppContext): string[];
     /**
      * Whether this step is optional — a failure in an optional step
      * does not stop the pipeline.

package/dist/_vendor/ailf-core/ports/task-source.d.ts

@@ -2,9 +2,9 @@
  * Port: Where task definitions come from.
  *
  * Adapters:
- * - YamlTaskSource (current) — reads tasks/*.yaml files
- * - ContentLakeTaskSource (tasks-as-content Phase 2) — GROQ query
- * - RepoTaskSource (tasks-as-content Phase 4) — reads .ailf/tasks/
+ * - ContentLakeTaskSource — GROQ query against Sanity Content Lake
+ * - RepoTaskSource — reads .ailf/tasks/*.task.ts files
+ * - TsTaskFileLoader — reads tasks/{mode}/*.task.ts (eval package)
  *
  * The key invariant: the pipeline orchestrator and all downstream steps
  * work with GeneralizedTaskDefinition[] regardless of where they came from.

package/dist/_vendor/ailf-core/ports/task-source.js

@@ -2,9 +2,9 @@
  * Port: Where task definitions come from.
  *
  * Adapters:
- * - YamlTaskSource (current) — reads tasks/*.yaml files
- * - ContentLakeTaskSource (tasks-as-content Phase 2) — GROQ query
- * - RepoTaskSource (tasks-as-content Phase 4) — reads .ailf/tasks/
+ * - ContentLakeTaskSource — GROQ query against Sanity Content Lake
+ * - RepoTaskSource — reads .ailf/tasks/*.task.ts files
+ * - TsTaskFileLoader — reads tasks/{mode}/*.task.ts (eval package)
  *
  * The key invariant: the pipeline orchestrator and all downstream steps
  * work with GeneralizedTaskDefinition[] regardless of where they came from.

package/dist/_vendor/ailf-core/schemas/pipeline-request.d.ts

@@ -77,10 +77,15 @@ export declare const PipelineRequestSchema: z.ZodObject<{
     taskMode: z.ZodOptional<z.ZodEnum<{
         inline: "inline";
         "content-lake": "content-lake";
-        yaml: "yaml";
     }>>;
     tasks: z.ZodOptional<z.ZodArray<z.ZodString>>;
     urls: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    variant: z.ZodOptional<z.ZodEnum<{
+        baseline: "baseline";
+        agentic: "agentic";
+        observed: "observed";
+        full: "full";
+    }>>;
     presets: z.ZodOptional<z.ZodArray<z.ZodString>>;
 }, z.core.$strip>;
 /** Inferred TypeScript type for a pipeline request payload. */

package/dist/_vendor/ailf-core/schemas/pipeline-request.js

@@ -13,7 +13,7 @@
  * @see packages/eval/src/pipeline/map-request-to-config.ts — maps to ResolvedConfig
  */
 import { z } from "zod";
-import { RAW_EVAL_MODES } from "../../ailf-shared/index.js";
+import { LITERACY_VARIANTS, RAW_EVAL_MODES } from "../../ailf-shared/index.js";
 // ---------------------------------------------------------------------------
 // Debug options — boolean shorthand or structured object
 // ---------------------------------------------------------------------------
@@ -86,9 +86,21 @@ export const PipelineRequestSchema = z.object({
     searchMode: z.enum(["off", "open", "origin-only"]).optional(),
     source: z.string().optional(),
     sourceReportId: z.string().optional(),
-    taskMode: z.enum(["content-lake", "yaml", "inline"]).optional(),
+    taskMode: z.enum(["content-lake", "inline"]).optional(),
     tasks: z.array(z.string()).optional(),
     urls: z.array(z.string().url()).optional(),
+    /**
+     * Literacy variant — only meaningful when mode is "literacy".
+     *
+     * When provided with a canonical mode (`mode: "literacy"`), this field
+     * specifies the variant directly. When mode is a legacy alias (e.g.,
+     * `mode: "baseline"`), the variant is derived from the mode name and
+     * this field is ignored.
+     *
+     * Prefer explicit `mode: "literacy", variant: "baseline"` over the
+     * legacy `mode: "baseline"` form.
+     */
+    variant: z.enum(LITERACY_VARIANTS).optional(),
     /** External preset file paths or npm package names to load */
     presets: z.array(z.string()).optional(),
 });

package/dist/_vendor/ailf-core/services/config-helpers.d.ts

@@ -6,7 +6,9 @@
  * Extracted from packages/eval/src/lib/generate-configs.ts during
  * the Ports & Adapters migration (Phase 4e).
  */
+import type { EvalMode } from "../../ailf-shared/index.d.ts";
 import type { ModelEntry } from "../types/index.js";
+import type { ModeBase } from "../types/plugin-registry.js";
 /**
  * Extract the raw API model name from a Promptfoo provider ID.
  *
@@ -38,4 +40,17 @@ export declare function mergeConfig(defaults: Record<string, unknown>, modelConf
  *
  * Models without a `modes` field match all modes.
  */
-export declare function modelMatchesMode(model: ModelEntry, mode: string): boolean;
+export declare function modelMatchesMode(model: ModelEntry, mode: EvalMode): boolean;
+/**
+ * Resolve which variants a model participates in for a given mode.
+ *
+ * Resolution rules:
+ * - If the mode has no variants defined → returns `undefined` (no variant filtering)
+ * - If the model specifies variants for this mode → returns that whitelist
+ * - If the model omits variants for this mode → returns ALL mode variants (default)
+ *
+ * @param model  - The model entry from models config
+ * @param modeBase - The mode base (contains variant definitions)
+ * @returns Array of variant IDs, or `undefined` if the mode has no variants
+ */
+export declare function resolveModelVariants(model: ModelEntry, modeBase: ModeBase): string[] | undefined;

package/dist/_vendor/ailf-core/services/config-helpers.js

@@ -84,3 +84,24 @@ export function modelMatchesMode(model, mode) {
     }
     return model.modes.includes(mode);
 }
+/**
+ * Resolve which variants a model participates in for a given mode.
+ *
+ * Resolution rules:
+ * - If the mode has no variants defined → returns `undefined` (no variant filtering)
+ * - If the model specifies variants for this mode → returns that whitelist
+ * - If the model omits variants for this mode → returns ALL mode variants (default)
+ *
+ * @param model  - The model entry from models config
+ * @param modeBase - The mode base (contains variant definitions)
+ * @returns Array of variant IDs, or `undefined` if the mode has no variants
+ */
+export function resolveModelVariants(model, modeBase) {
+    const modeVariants = modeBase.mode.variants;
+    if (!modeVariants || modeVariants.length === 0)
+        return undefined;
+    const allVariantIds = modeVariants.map((v) => v.id);
+    const modeId = modeBase.mode.id;
+    const explicit = model.variants?.[modeId];
+    return explicit ?? allVariantIds;
+}

package/dist/_vendor/ailf-core/services/index.d.ts

@@ -10,4 +10,4 @@
 export { classifyRubric, detectFeatureArea, extractDimensions, extractUrlMetadata, mergeScores, parseRubricScore, } from "./scoring.js";
 export { formatComparisonMarkdown, formatComparisonTable, } from "./comparison-formatters.js";
 export { aggregateAreas, aggregateDimensions, computeEnsembleScore, computeTaskScore, normalizeScore, type AggregationStrategy, type AreaScore, type AssertionScore, type DimensionScore, type EnsembleGradingConfig, type GraderTransitionConfig, type TaskScore, type TaskScoreOptions, } from "./scoring-engine.js";
-export { extractModelName, extractProvider, mergeConfig, modelMatchesMode, } from "./config-helpers.js";
+export { extractModelName, extractProvider, mergeConfig, modelMatchesMode, resolveModelVariants, } from "./config-helpers.js";

package/dist/_vendor/ailf-core/services/index.js

@@ -10,4 +10,4 @@
 export { classifyRubric, detectFeatureArea, extractDimensions, extractUrlMetadata, mergeScores, parseRubricScore, } from "./scoring.js";
 export { formatComparisonMarkdown, formatComparisonTable, } from "./comparison-formatters.js";
 export { aggregateAreas, aggregateDimensions, computeEnsembleScore, computeTaskScore, normalizeScore, } from "./scoring-engine.js";
-export { extractModelName, extractProvider, mergeConfig, modelMatchesMode, } from "./config-helpers.js";
+export { extractModelName, extractProvider, mergeConfig, modelMatchesMode, resolveModelVariants, } from "./config-helpers.js";

package/dist/_vendor/ailf-core/services/scoring.js

@@ -65,6 +65,15 @@ export function classifyRubric(component) {
  */
 export function detectFeatureArea(description) {
     const desc = description.toLowerCase();
+    if (desc.includes("portable text")) {
+        return "portable-text";
+    }
+    if (desc.includes("content lake")) {
+        return "content-lake";
+    }
+    if (desc.includes("image handling") || desc.includes("image asset")) {
+        return "image-handling";
+    }
     if (desc.includes("studio")) {
         return "studio-setup";
     }

package/dist/_vendor/ailf-core/types/generalized-task.d.ts

@@ -8,7 +8,7 @@
  *
  * @see docs/design-docs/architecture-overhaul/domain-model.md (canonical)
  * @see docs/design-docs/architecture-overhaul/test-definition.md (authoring surfaces)
- * @see docs/exec-plans/architecture-overhaul/phase-0-foundation-types.md (task 0h)
+ * @see docs/archive/exec-plans/architecture-overhaul/phase-0-foundation-types.md (task 0h)
  */
 /** Difficulty level for a task */
 export type TaskDifficulty = "basic" | "intermediate" | "advanced";
@@ -178,6 +178,17 @@ export interface MCPServerTaskDefinition extends TaskCommonFields {
         url?: string;
         /** Environment variables for the server process */
         env?: Record<string, string>;
+        /**
+         * HTTP headers for remote transports (sse / streamable-http).
+         * Merged on top of any auth-derived headers, so explicit values
+         * here take precedence over `auth`-generated headers.
+         *
+         * Values support `{{env.VAR}}` template syntax for secrets.
+         *
+         * @example
+         * headers: { Authorization: "Bearer {{env.SANITY_API_TOKEN}}" }
+         */
+        headers?: Record<string, string>;
         /** Startup timeout in milliseconds */
         startupTimeoutMs?: number;
         /**

package/dist/_vendor/ailf-core/types/generalized-task.js

@@ -8,6 +8,6 @@
  *
  * @see docs/design-docs/architecture-overhaul/domain-model.md (canonical)
  * @see docs/design-docs/architecture-overhaul/test-definition.md (authoring surfaces)
- * @see docs/exec-plans/architecture-overhaul/phase-0-foundation-types.md (task 0h)
+ * @see docs/archive/exec-plans/architecture-overhaul/phase-0-foundation-types.md (task 0h)
  */
 export {};

package/dist/_vendor/ailf-core/types/index.d.ts

@@ -9,7 +9,7 @@
  * Ports & Adapters migration (Phase 0c). The original file is now a
  * re-export barrel that preserves backward compatibility.
  */
-import type { DocumentRef as _DocumentRef, EvalMode as _EvalMode } from "../../ailf-shared/index.d.ts";
+import type { DocumentRef as _DocumentRef, EvalMode } from "../../ailf-shared/index.d.ts";
 export type { ActualScoreEntry, ComponentResult, TestResult, UrlMetadata, } from "./scoring-input.js";
 export type { DocumentRef } from "../../ailf-shared/index.d.ts";
 export type { StoredBaseline, StoredReport, StoredRun, StoredTaskResult, StoredTrace, SchemaVersioned, } from "./storage-schema.js";
@@ -25,7 +25,6 @@ export type { ArtifactId, Brand, Err, FixtureId, IdValidationError, NewReportId,
 export { err, fixtureId, ok, providerId, resultId, runId, suiteId, taskId, traceId, } from "./branded-ids.js";
 export type { AgentHarnessTaskDefinition, CustomTaskDefinition, GeneralizedAssertionDefinition, GeneralizedDocRef, GeneralizedTaskDefinition, GeneralizedTemplatedAssertion, GeneralizedValueAssertion, IdDocRef, KnowledgeProbeTaskDefinition, LiteracyTaskDefinition, MCPServerTaskDefinition, PathDocRef, PerspectiveDocRef, RubricRef, SlugDocRef, TaskCommonFields, TaskDifficulty, TaskOptions, TaskProviderConfig, TaskStatus, } from "./generalized-task.js";
 type DocumentRef = _DocumentRef;
-type EvalMode = _EvalMode;
 /** Aggregated retrieval metrics for a feature area */
 export interface AreaRetrievalMetrics {
     area: string;
@@ -119,7 +118,7 @@ export interface FailureModeReport {
     totalJudgments: number;
 }
 /** Failure mode classification for a low-scoring judgment */
-export type FailureModeType = "incorrect-docs" | "missing-docs" | "model-limitation" | "outdated-docs" | "poor-structure" | "unclassified";
+export type FailureModeType = "api-error" | "incorrect-docs" | "missing-docs" | "model-limitation" | "outdated-docs" | "poor-structure" | "unclassified";
 /** Per-feature-area score breakdown */
 export interface FeatureScore {
     /**
@@ -352,11 +351,40 @@ export interface ModelEntry {
     env?: string;
     id: string;
     label: string;
-    modes?: string[];
+    /**
+     * Which evaluation modes this model participates in.
+     *
+     * Values must be canonical eval mode names (e.g., "literacy", "mcp-server").
+     * When omitted, the model participates in all modes.
+     */
+    modes?: EvalMode[];
+    /**
+     * Per-provider timeout in ms. Emitted into Promptfoo provider config.
+     * Default: 300_000 (5 min, matching Promptfoo's built-in default).
+     */
+    timeoutMs?: number;
+    /**
+     * Per-mode variant whitelist. Keys are eval mode IDs, values are arrays
+     * of variant IDs to include for that mode.
+     *
+     * When a model enrolls in a mode (via `modes`) but does not specify
+     * variants for it here, ALL variants defined by the mode base are included.
+     *
+     * Only meaningful for modes that define variants (e.g., literacy has
+     * "baseline", "observed", "agentic-naive", "agentic-optimized").
+     * Ignored for modes without variants.
+     */
+    variants?: Partial<Record<EvalMode, string[]>>;
 }
 /** Parsed config/models.yaml structure */
 export interface ModelsConfig {
     defaults: Record<string, unknown>;
+    /**
+     * Maximum wall-clock time per eval step (all tests for one mode) in ms.
+     * When exceeded, the subprocess is killed and partial results are used.
+     * Default: no limit (backward compatible).
+     */
+    evalBudgetMs?: number;
     grader: {
         id: string;
         label?: string;
@@ -507,6 +535,21 @@ export interface TestSummary {
         task: string;
         error: string;
     }[];
+    /** Per-test timing statistics (when latencyMs is available from Promptfoo) */
+    timing?: {
+        /** Median test duration in ms */
+        medianMs: number;
+        /** 95th percentile test duration in ms */
+        p95Ms: number;
+        /** Maximum test duration in ms */
+        maxMs: number;
+        /** Tests that exceeded the slow threshold (2x median, min 60s) */
+        slowTests: {
+            task: string;
+            model: string;
+            durationMs: number;
+        }[];
+    };
 }
 /** Token usage and estimated cost for a pipeline run. */
 export interface PipelineUsage {