@sanity/ailf 0.4.1 → 1.0.0

package/dist/_vendor/ailf-core/types/branded-ids.js

@@ -0,0 +1,136 @@
+/**
+ * Branded ID types — nominal typing for entity identifiers.
+ *
+ * All entity IDs use branded types to prevent accidental misuse.
+ * A `TaskId` cannot be passed where a `RunId` is expected, even
+ * though both are strings at runtime.
+ *
+ * Constructor functions validate format and return `Result<T, E>` —
+ * parse-don't-validate at the boundary, then pass branded values
+ * through the pipeline.
+ *
+ * The `Brand` utility and `Result` type are defined here as the
+ * foundation. Existing branded types in the codebase (`ReportId`,
+ * `ISOTimestamp`) use inline branding — those will be migrated to
+ * use this utility in Phase 7.
+ *
+ * @see docs/design-docs/architecture-overhaul/domain-model.md (canonical)
+ * @see docs/design-docs/parse-dont-validate.md (design principle)
+ */
+/** Construct a success result */
+export function ok(value) {
+    return { ok: true, value };
+}
+/** Construct a failure result */
+export function err(error) {
+    return { ok: false, error };
+}
+// ---------------------------------------------------------------------------
+// Constructor functions — parse-don't-validate
+// ---------------------------------------------------------------------------
+/**
+ * Parse a raw string into a `TaskId`.
+ *
+ * Valid format: alphanumeric + hyphens, 1–128 characters.
+ * Examples: `"groq-projection-basics"`, `"mcp-server-tools-list"`
+ */
+export function taskId(raw) {
+    if (!raw.match(/^[a-z0-9][a-z0-9-]{0,127}$/)) {
+        return err({
+            code: "INVALID_TASK_ID",
+            raw,
+            message: `Invalid TaskId "${raw}": must be 1–128 lowercase alphanumeric characters or hyphens, starting with a letter or digit`,
+        });
+    }
+    return ok(raw);
+}
+/**
+ * Parse a raw string into a `RunId`.
+ *
+ * Valid format: `run_` prefix followed by alphanumeric characters.
+ */
+export function runId(raw) {
+    if (!raw.match(/^run_[a-zA-Z0-9]{8,}$/)) {
+        return err({
+            code: "INVALID_RUN_ID",
+            raw,
+            message: `Invalid RunId "${raw}": must match run_[a-zA-Z0-9]{8,}`,
+        });
+    }
+    return ok(raw);
+}
+/**
+ * Parse a raw string into a `SuiteId`.
+ *
+ * Valid format: `suite_` prefix followed by alphanumeric characters.
+ */
+export function suiteId(raw) {
+    if (!raw.match(/^suite_[a-zA-Z0-9]{4,}$/)) {
+        return err({
+            code: "INVALID_SUITE_ID",
+            raw,
+            message: `Invalid SuiteId "${raw}": must match suite_[a-zA-Z0-9]{4,}`,
+        });
+    }
+    return ok(raw);
+}
+/**
+ * Parse a raw string into a `ResultId`.
+ *
+ * Valid format: `res_` prefix followed by alphanumeric characters.
+ */
+export function resultId(raw) {
+    if (!raw.match(/^res_[a-zA-Z0-9]{8,}$/)) {
+        return err({
+            code: "INVALID_RESULT_ID",
+            raw,
+            message: `Invalid ResultId "${raw}": must match res_[a-zA-Z0-9]{8,}`,
+        });
+    }
+    return ok(raw);
+}
+/**
+ * Parse a raw string into a `TraceId`.
+ *
+ * Valid format: `trace_` prefix followed by alphanumeric characters.
+ */
+export function traceId(raw) {
+    if (!raw.match(/^trace_[a-zA-Z0-9]{8,}$/)) {
+        return err({
+            code: "INVALID_TRACE_ID",
+            raw,
+            message: `Invalid TraceId "${raw}": must match trace_[a-zA-Z0-9]{8,}`,
+        });
+    }
+    return ok(raw);
+}
+/**
+ * Parse a raw string into a `ProviderId`.
+ *
+ * Valid format: colon-separated segments (e.g., `"openai:chat:gpt-4o"`).
+ */
+export function providerId(raw) {
+    if (!raw.match(/^[a-zA-Z0-9][a-zA-Z0-9:._-]{0,127}$/)) {
+        return err({
+            code: "INVALID_PROVIDER_ID",
+            raw,
+            message: `Invalid ProviderId "${raw}": must be 1–128 alphanumeric characters, colons, dots, underscores, or hyphens`,
+        });
+    }
+    return ok(raw);
+}
+/**
+ * Parse a raw string into a `FixtureId`.
+ *
+ * Valid format: alphanumeric + hyphens, 1–128 characters.
+ */
+export function fixtureId(raw) {
+    if (!raw.match(/^[a-z0-9][a-z0-9-]{0,127}$/)) {
+        return err({
+            code: "INVALID_FIXTURE_ID",
+            raw,
+            message: `Invalid FixtureId "${raw}": must be 1–128 lowercase alphanumeric characters or hyphens`,
+        });
+    }
+    return ok(raw);
+}

package/dist/_vendor/ailf-core/types/eval-mode-config.d.ts

@@ -0,0 +1,150 @@
+/**
+ * EvalModeConfig — Rich discriminated union for evaluation mode configuration.
+ *
+ * This is the NEW type system for evaluation modes, introduced in the
+ * architecture overhaul (Phase 0). It carries per-mode configuration
+ * payloads that the compiler and new evaluation modes use.
+ *
+ * The existing `EvalMode` string union in @sanity/ailf-shared remains
+ * unchanged — it is the lightweight identifier used throughout the
+ * existing pipeline. `EvalModeConfig` is the rich type that new code
+ * authors against.
+ *
+ * @see docs/design-docs/architecture-overhaul/domain-model.md (canonical)
+ */
+import type { EvalMode } from "../../ailf-shared/index.d.ts";
+/** Configuration for documentation source resolution in literacy mode */
+export interface DocSourceConfig {
+    /** Source name from config/sources.yaml */
+    name: string;
+    /** Override the base URL for doc fetching */
+    baseUrl?: string;
+    /** Sanity dataset to fetch from */
+    dataset?: string;
+    /** Sanity project ID */
+    projectId?: string;
+    /** Content release perspective */
+    perspective?: string;
+}
+/** Configuration for an MCP server under test */
+export interface MCPServerConfig {
+    /** Transport protocol */
+    transport: "stdio" | "sse" | "streamable-http";
+    /** Command to start the server (for stdio transport) */
+    command?: string;
+    /** Server URL (for sse/streamable-http transport) */
+    url?: string;
+    /** Environment variables to pass to the server process */
+    env?: Record<string, string>;
+    /** Startup timeout in milliseconds */
+    startupTimeoutMs?: number;
+}
+/** Configuration for an agent harness under test */
+export interface AgentHarnessConfig {
+    /** Runtime environment identifier */
+    runtime: string;
+    /** Entry point for the agent (file path or module specifier) */
+    entrypoint: string;
+    /** Maximum execution time per test case */
+    timeoutMs?: number;
+    /** Sandbox configuration */
+    sandbox?: SandboxConfig;
+}
+/** Sandbox configuration for isolated agent execution */
+export interface SandboxConfig {
+    /** Sandbox type */
+    type: "docker" | "git-worktree" | "none" | "nsjail" | "tempdir";
+    /** Docker image (when type is "docker") */
+    image?: string;
+    /** Resource limits */
+    limits?: {
+        /** CPU cores */
+        cpus?: number;
+        /** Memory in bytes */
+        memoryBytes?: number;
+        /** Disk space in bytes */
+        diskBytes?: number;
+        /** Network access allowed */
+        networkAccess?: boolean;
+    };
+}
+/** Tool definition for agent harness mode */
+export interface ToolDef {
+    /** Tool name */
+    name: string;
+    /** Tool description (for the agent's tool manifest) */
+    description: string;
+    /** JSON Schema for the tool's input parameters */
+    inputSchema?: Record<string, unknown>;
+}
+/** Reference to a knowledge base for knowledge-probe mode */
+export interface KnowledgeBaseRef {
+    /** Knowledge base type */
+    type: "sanity-dataset" | "embeddings-index" | "file-corpus";
+    /** Dataset or index name */
+    name: string;
+    /** Additional configuration */
+    config?: Record<string, unknown>;
+}
+/** Probing strategy for knowledge-probe mode */
+export type ProbeStrategy = "breadth-first" | "depth-first" | "random-sample" | "coverage-guided";
+/** Literacy mode — documentation quality evaluation (the original AILF mode) */
+export interface LiteracyModeConfig {
+    type: "literacy";
+    /** Documentation sources to evaluate against */
+    docSources: DocSourceConfig[];
+    /** Whether to compute retrieval metrics (precision/recall/F1) */
+    retrievalMetrics: boolean;
+}
+/** MCP server mode — evaluates an MCP server's tool/resource capabilities */
+export interface MCPServerModeConfig {
+    type: "mcp-server";
+    /** The MCP server under test */
+    serverConfig: MCPServerConfig;
+    /** Expected capabilities to verify (e.g., "tools/list", "resources/read") */
+    capabilities: string[];
+}
+/** Agent harness mode — evaluates an autonomous agent in a sandboxed environment */
+export interface AgentHarnessModeConfig {
+    type: "agent-harness";
+    /** Agent configuration */
+    agentConfig: AgentHarnessConfig;
+    /** Tools available to the agent */
+    toolManifest: ToolDef[];
+}
+/** Knowledge probe mode — measures knowledge coverage across a corpus */
+export interface KnowledgeProbeModeConfig {
+    type: "knowledge-probe";
+    /** The knowledge base to probe */
+    knowledgeBase: KnowledgeBaseRef;
+    /** How to traverse the knowledge space */
+    probeStrategy: ProbeStrategy;
+}
+/** Custom mode — user-provided evaluation handler */
+export interface CustomModeConfig {
+    type: "custom";
+    /** Module path to the custom handler implementing EvalModeHandler */
+    handler: string;
+    /** Freeform schema for mode-specific configuration */
+    schema: Record<string, unknown>;
+}
+/**
+ * Rich discriminated union for evaluation mode configuration.
+ *
+ * Discriminant: `type` field.
+ *
+ * New code should use `EvalModeConfig` for structured mode configuration.
+ * The string-level `EvalMode` type remains for pipeline options, report
+ * provenance, GROQ filters, and Studio UI.
+ */
+export type EvalModeConfig = LiteracyModeConfig | MCPServerModeConfig | AgentHarnessModeConfig | KnowledgeProbeModeConfig | CustomModeConfig;
+/** All valid mode type strings from EvalModeConfig */
+export type EvalModeType = EvalModeConfig["type"];
+/**
+ * Extract the string-level EvalMode from a rich EvalModeConfig.
+ *
+ * This bridges the new discriminated union with the existing string
+ * union used throughout the pipeline. The returned value is always
+ * a valid `EvalMode` string.
+ */
+export declare function evalModeType(config: EvalModeConfig): EvalMode;

package/dist/_vendor/ailf-core/types/eval-mode-config.js

@@ -0,0 +1,24 @@
+/**
+ * EvalModeConfig — Rich discriminated union for evaluation mode configuration.
+ *
+ * This is the NEW type system for evaluation modes, introduced in the
+ * architecture overhaul (Phase 0). It carries per-mode configuration
+ * payloads that the compiler and new evaluation modes use.
+ *
+ * The existing `EvalMode` string union in @sanity/ailf-shared remains
+ * unchanged — it is the lightweight identifier used throughout the
+ * existing pipeline. `EvalModeConfig` is the rich type that new code
+ * authors against.
+ *
+ * @see docs/design-docs/architecture-overhaul/domain-model.md (canonical)
+ */
+/**
+ * Extract the string-level EvalMode from a rich EvalModeConfig.
+ *
+ * This bridges the new discriminated union with the existing string
+ * union used throughout the pipeline. The returned value is always
+ * a valid `EvalMode` string.
+ */
+export function evalModeType(config) {
+    return config.type;
+}

package/dist/_vendor/ailf-core/types/generalized-task.d.ts

@@ -0,0 +1,319 @@
+/**
+ * GeneralizedTaskDefinition — Mode-discriminated task definitions.
+ *
+ * This is the canonical task type for the architecture overhaul. It supports
+ * all five evaluation modes via a discriminated union on the `mode`
+ * field. Common fields (id, title, description, area, etc.) are shared;
+ * mode-specific fields live only on the relevant variant.
+ *
+ * @see docs/design-docs/architecture-overhaul/domain-model.md (canonical)
+ * @see docs/design-docs/architecture-overhaul/test-definition.md (authoring surfaces)
+ * @see docs/exec-plans/architecture-overhaul/phase-0-foundation-types.md (task 0h)
+ */
+/** Difficulty level for a task */
+export type TaskDifficulty = "basic" | "intermediate" | "advanced";
+/** Lifecycle status for a task */
+export type TaskStatus = "active" | "draft" | "paused" | "archived";
+/**
+ * A canonical documentation reference — polymorphic resolution strategy.
+ *
+ * Each entry resolves docs through one of four strategies, discriminated
+ * by key presence (no explicit `type` field). All strategies carry an
+ * optional `reason` for context.
+ *
+ * @see docs/design-docs/canonical-doc-resolution.md
+ */
+export type GeneralizedDocRef = SlugDocRef | PathDocRef | IdDocRef | PerspectiveDocRef;
+/** Resolve by article slug field. Legacy — prefer `path` for uniqueness. */
+export interface SlugDocRef {
+    slug: string;
+    reason?: string;
+}
+/** Resolve by URL path (after /docs/). Unique across sections. */
+export interface PathDocRef {
+    path: string;
+    reason?: string;
+}
+/** Resolve by Sanity document `_id`. The primary resolution strategy.
+ *
+ * Optional `slug` and `path` provide human-readable context — they are
+ * NOT used for resolution (the `_id` is authoritative) but help YAML
+ * authors understand which document is being referenced.
+ */
+export interface IdDocRef {
+    id: string;
+    reason?: string;
+    /** Human-readable slug (informational only — not used for resolution) */
+    slug?: string;
+    /** Human-readable path (informational only — not used for resolution) */
+    path?: string;
+}
+/** Resolve all articles in a content release. One-to-many. */
+export interface PerspectiveDocRef {
+    perspective: string;
+    reason?: string;
+}
+/** A templated assertion referencing a rubric template */
+export interface GeneralizedTemplatedAssertion {
+    type: "llm-rubric";
+    template: string;
+    criteria: string[];
+    weight?: number;
+}
+/** A value-based assertion (contains, javascript, cost, latency, etc.) */
+export interface GeneralizedValueAssertion {
+    type: string;
+    value?: unknown;
+    weight?: number;
+    [key: string]: unknown;
+}
+/** Any assertion definition */
+export type GeneralizedAssertionDefinition = GeneralizedTemplatedAssertion | GeneralizedValueAssertion;
+/** Rubric reference — either a named template or an inline rubric */
+export type RubricRef = {
+    ref: string;
+} | {
+    inline: string;
+    dimensions?: {
+        key: string;
+        weight: number;
+    }[];
+};
+/** Provider override for a specific task */
+export interface TaskProviderConfig {
+    /** Provider ID (e.g., "openai:chat:gpt-4o", "anthropic:messages:claude-sonnet-4-6") */
+    id: string;
+    /** Provider-specific configuration overrides */
+    config?: Record<string, unknown>;
+}
+/** Task-level options */
+export interface TaskOptions {
+    /** Execution timeout in milliseconds */
+    timeout?: number;
+    /** Whether to cache results for this task */
+    cache?: boolean;
+    /** Output transform expression */
+    transformOutput?: string;
+    /** Arbitrary Promptfoo overrides (escape hatch) */
+    promptfooOverrides?: Record<string, unknown>;
+}
+/** Fields shared by all task modes */
+export interface TaskCommonFields {
+    /** Unique task identifier */
+    id: string;
+    /** Human-readable title */
+    title: string;
+    /** Detailed description of what this task evaluates */
+    description?: string;
+    /** Feature area this task belongs to (e.g., "groq", "studio", "mutations") */
+    area?: string;
+    /** Difficulty level */
+    difficulty?: TaskDifficulty;
+    /** Freeform labels for filtering and organization */
+    tags?: string[];
+    /** Lifecycle status — controls pipeline inclusion */
+    status?: TaskStatus;
+    /** Assertion definitions (rubric templates + value assertions) */
+    assertions?: GeneralizedAssertionDefinition[];
+    /** Rubric template or inline rubric for grading */
+    rubric?: RubricRef;
+    /** Provider overrides for this task */
+    providers?: TaskProviderConfig[];
+    /** Task-level execution options */
+    options?: TaskOptions;
+    /** Prompt configuration */
+    prompt?: {
+        /** Named prompt template */
+        template?: string;
+        /** Inline prompt text (mutually exclusive with template) */
+        text?: string;
+        /** System message override */
+        systemMessage?: string;
+        /** Variables for template interpolation */
+        vars?: Record<string, unknown>;
+    };
+    /** Arbitrary metadata */
+    metadata?: Record<string, unknown>;
+}
+/**
+ * Literacy mode — documentation quality evaluation.
+ *
+ * The original AILF evaluation mode. Tests whether AI coding tools
+ * can find the right docs and produce correct code.
+ */
+export interface LiteracyTaskDefinition extends TaskCommonFields {
+    mode: "literacy";
+    /** Documentation context configuration */
+    context?: {
+        /** Canonical doc references for this task */
+        docs?: GeneralizedDocRef[];
+        /** Fixture references */
+        fixtures?: string[];
+    };
+    /** Path to the reference solution (relative to eval package root) */
+    referenceSolution?: string;
+    /** Whether doc coverage rubric should be auto-generated */
+    docCoverage?: boolean;
+    /** Baseline variant configuration */
+    baseline?: {
+        enabled?: boolean;
+        rubric?: "abbreviated" | "full" | "none";
+    };
+}
+/**
+ * MCP server mode — evaluates an MCP server's capabilities.
+ *
+ * Tests whether an MCP server correctly implements tools, resources,
+ * and prompts according to the MCP specification.
+ */
+export interface MCPServerTaskDefinition extends TaskCommonFields {
+    mode: "mcp-server";
+    /** Server under test */
+    serverConfig?: {
+        /** Transport protocol */
+        transport: "stdio" | "sse" | "streamable-http";
+        /** Command to start the server (stdio) */
+        command?: string;
+        /** Server URL (sse/streamable-http) */
+        url?: string;
+        /** Environment variables for the server process */
+        env?: Record<string, string>;
+        /** Startup timeout in milliseconds */
+        startupTimeoutMs?: number;
+        /**
+         * Authentication config for the MCP server.
+         * Maps directly to Promptfoo's auth config.
+         * @see https://www.promptfoo.dev/docs/providers/mcp/#authentication
+         */
+        auth?: {
+            type: "bearer" | "basic" | "api_key" | "oauth";
+            /** Bearer token (for type: "bearer") */
+            token?: string;
+            /** Username (for type: "basic" or "oauth" password grant) */
+            username?: string;
+            /** Password (for type: "basic" or "oauth" password grant) */
+            password?: string;
+            /** API key value (for type: "api_key") */
+            value?: string;
+            /** Header/query param name for API key (default: "X-API-Key") */
+            keyName?: string;
+            /** "header" or "query" (for type: "api_key", default: "header") */
+            placement?: "header" | "query";
+            /** OAuth grant type */
+            grantType?: "client_credentials" | "password";
+            /** OAuth token endpoint URL */
+            tokenUrl?: string;
+            /** OAuth client ID */
+            clientId?: string;
+            /** OAuth client secret */
+            clientSecret?: string;
+            /** OAuth scopes */
+            scopes?: string[];
+        };
+    };
+    /** Expected capabilities to verify */
+    capabilities?: string[];
+    /** Documentation context (MCP servers can also be evaluated against docs) */
+    context?: {
+        docs?: GeneralizedDocRef[];
+        tools?: string[];
+        fixtures?: string[];
+    };
+    /** Multi-turn conversation definition */
+    multiTurn?: {
+        turns: {
+            role: "user" | "assistant";
+            content: string;
+        }[];
+    };
+}
+/**
+ * Agent harness mode — evaluates autonomous agents in a sandbox.
+ *
+ * Tests whether an AI agent can complete tasks correctly using a
+ * defined set of tools in an isolated environment.
+ */
+export interface AgentHarnessTaskDefinition extends TaskCommonFields {
+    mode: "agent-harness";
+    /** Sandbox configuration for isolated execution */
+    sandbox?: {
+        type: "docker" | "git-worktree" | "none" | "nsjail" | "tempdir";
+        image?: string;
+        limits?: {
+            cpus?: number;
+            memoryBytes?: number;
+            diskBytes?: number;
+            networkAccess?: boolean;
+        };
+    };
+    /** Tools available to the agent */
+    tools?: string[];
+    /** Fixture references for test data */
+    fixtures?: string[];
+    /** Documentation context */
+    context?: {
+        docs?: GeneralizedDocRef[];
+        fixtures?: string[];
+    };
+    /** Multi-turn conversation definition */
+    multiTurn?: {
+        turns: {
+            role: "user" | "assistant";
+            content: string;
+        }[];
+    };
+}
+/**
+ * Knowledge probe mode — measures knowledge coverage across a corpus.
+ *
+ * Tests breadth and depth of an LLM's knowledge about a specific
+ * domain by systematically probing different aspects.
+ */
+export interface KnowledgeProbeTaskDefinition extends TaskCommonFields {
+    mode: "knowledge-probe";
+    /** How to traverse the knowledge space */
+    probeStrategy?: "breadth-first" | "depth-first" | "random-sample" | "coverage-guided";
+    /** Knowledge base reference */
+    knowledgeBase?: {
+        type: "sanity-dataset" | "embeddings-index" | "file-corpus";
+        name: string;
+        config?: Record<string, unknown>;
+    };
+    /** Documentation context */
+    context?: {
+        docs?: GeneralizedDocRef[];
+        fixtures?: string[];
+    };
+}
+/**
+ * Custom mode — user-provided evaluation handler.
+ *
+ * The escape hatch for evaluation types not covered by the built-in
+ * modes. Users provide a handler module implementing `EvalModeHandler`.
+ */
+export interface CustomTaskDefinition extends TaskCommonFields {
+    mode: "custom";
+    /** Module path to the custom handler */
+    handler: string;
+    /** Freeform schema for mode-specific configuration */
+    schema?: Record<string, unknown>;
+    /** Documentation context */
+    context?: {
+        docs?: GeneralizedDocRef[];
+        fixtures?: string[];
+    };
+}
+/**
+ * Generalized task definition — a mode-discriminated union.
+ *
+ * The `mode` field determines which fields are available:
+ * - `"literacy"` → context.docs, referenceSolution, docCoverage, baseline
+ * - `"mcp-server"` → serverConfig, capabilities, multiTurn
+ * - `"agent-harness"` → sandbox, tools, fixtures, multiTurn
+ * - `"knowledge-probe"` → probeStrategy, knowledgeBase
+ * - `"custom"` → handler, schema
+ *
+ * Use `defineTask()` from `@sanity/ailf` for full IDE autocomplete
+ * when authoring tasks.
+ */
+export type GeneralizedTaskDefinition = LiteracyTaskDefinition | MCPServerTaskDefinition | AgentHarnessTaskDefinition | KnowledgeProbeTaskDefinition | CustomTaskDefinition;

package/dist/_vendor/ailf-core/types/generalized-task.js

@@ -0,0 +1,13 @@
+/**
+ * GeneralizedTaskDefinition — Mode-discriminated task definitions.
+ *
+ * This is the canonical task type for the architecture overhaul. It supports
+ * all five evaluation modes via a discriminated union on the `mode`
+ * field. Common fields (id, title, description, area, etc.) are shared;
+ * mode-specific fields live only on the relevant variant.
+ *
+ * @see docs/design-docs/architecture-overhaul/domain-model.md (canonical)
+ * @see docs/design-docs/architecture-overhaul/test-definition.md (authoring surfaces)
+ * @see docs/exec-plans/architecture-overhaul/phase-0-foundation-types.md (task 0h)
+ */
+export {};