@sanity/ailf 0.4.1 → 1.0.0

package/dist/_vendor/ailf-core/types/storage-schema.d.ts

@@ -0,0 +1,199 @@
+/**
+ * Storage schema types for Content Lake documents.
+ *
+ * All AILF document types include `schemaVersion` for forward-compatible
+ * migration. Full trace data goes to blob storage (GCP/local); only
+ * sanitized summaries are stored in Content Lake.
+ *
+ * SECURITY: The `next` dataset is publicly accessible. Never store
+ * sensitive data (API keys, full tool call arguments, raw outputs with
+ * PII) in Content Lake. Use blob storage + traceDataUri references.
+ *
+ * @see docs/design-docs/architecture-overhaul/storage-schema.md
+ */
+import type { VariableEnvelope } from "./variable-envelope.js";
+/** Current schema version for all AILF document types */
+export declare const CURRENT_SCHEMA_VERSION = 1;
+/** Schema-versioned document base */
+export interface SchemaVersioned {
+    /** Schema version for forward-compatible migration */
+    schemaVersion: number;
+}
+/** An evaluation run — one execution of the pipeline */
+export interface StoredRun extends SchemaVersioned {
+    _type: "ailf.run";
+    _id: string;
+    /** ISO timestamp when the run started */
+    startedAt: string;
+    /** ISO timestamp when the run completed */
+    completedAt?: string;
+    /** Run status */
+    status: "completed" | "failed" | "running";
+    /** Evaluation mode used */
+    mode: string;
+    /** Models evaluated */
+    models: {
+        id: string;
+        label: string;
+    }[];
+    /** Grader model used */
+    graderModel: string;
+    /** Number of tasks evaluated */
+    taskCount: number;
+    /** Run-level aggregate score (0-100) */
+    overallScore?: number;
+    /** Per-area scores */
+    areaScores?: {
+        areaId: string;
+        score: number;
+    }[];
+    /** Pipeline configuration fingerprint (for cache invalidation) */
+    configFingerprint?: string;
+    /** Git context at run time */
+    git?: {
+        branch: string;
+        sha: string;
+        repo: string;
+    };
+    /** Run metadata */
+    metadata?: Record<string, unknown>;
+}
+/** Result for one task × one model */
+export interface StoredTaskResult extends SchemaVersioned {
+    _type: "ailf.taskResult";
+    _id: string;
+    /** Reference to parent run */
+    runRef: {
+        _type: "reference";
+        _ref: string;
+    };
+    /** Reference to task definition */
+    taskRef: {
+        _type: "reference";
+        _ref: string;
+    };
+    /** Task ID (denormalized for querying) */
+    taskId: string;
+    /** Model that produced this result */
+    modelId: string;
+    /** Overall task score (0-100) */
+    score: number;
+    /** Per-dimension scores */
+    dimensions: {
+        dimensionId: string;
+        score: number;
+        weight: number;
+        assertionCount: number;
+        passCount: number;
+    }[];
+    /** Whether task passed its quality threshold */
+    passesThreshold: boolean;
+    /** Input variable envelope (provenance tracking) */
+    inputEnvelope?: VariableEnvelope;
+    /** Output variable envelope */
+    outputEnvelope?: VariableEnvelope;
+    /** URI to full trace payload in blob storage */
+    traceDataUri?: string;
+    /** Sanitized trace summary (safe for public dataset) */
+    traceSummary?: {
+        toolCallCount: number;
+        toolCallCategories: Record<string, number>;
+        totalTokens: number;
+        costEstimate: number;
+        durationMs: number;
+    };
+    /** Evaluation mode */
+    mode: string;
+    /** Mode-specific metadata */
+    modeMetadata?: Record<string, unknown>;
+}
+/** Aggregated report for a run */
+export interface StoredReport extends SchemaVersioned {
+    _type: "ailf.report";
+    _id: string;
+    /** Reference to parent run */
+    runRef: {
+        _type: "reference";
+        _ref: string;
+    };
+    /** ISO timestamp */
+    createdAt: string;
+    /** Report tag/label */
+    tag?: string;
+    /** Overall score */
+    overallScore: number;
+    /** Per-area breakdown */
+    areas: {
+        areaId: string;
+        score: number;
+        taskCount: number;
+        passingCount: number;
+    }[];
+    /** Per-model breakdown */
+    models: {
+        modelId: string;
+        score: number;
+    }[];
+    /** Human-readable markdown summary */
+    summary?: string;
+}
+/** Baseline snapshot for comparison */
+export interface StoredBaseline extends SchemaVersioned {
+    _type: "ailf.baseline";
+    _id: string;
+    /** ISO timestamp when baseline was saved */
+    savedAt: string;
+    /** Label for this baseline */
+    label?: string;
+    /** Reference to the run this baseline was taken from */
+    runRef: {
+        _type: "reference";
+        _ref: string;
+    };
+    /** Per-task scores at baseline time */
+    taskScores: {
+        taskId: string;
+        score: number;
+        dimensions: Record<string, number>;
+    }[];
+    /** Per-area scores at baseline time */
+    areaScores: {
+        areaId: string;
+        score: number;
+    }[];
+}
+/** Trace summary document in Content Lake */
+export interface StoredTrace extends SchemaVersioned {
+    _type: "ailf.trace";
+    _id: string;
+    /** Reference to parent run */
+    runRef: {
+        _type: "reference";
+        _ref: string;
+    };
+    /** Reference to task result */
+    taskResultRef: {
+        _type: "reference";
+        _ref: string;
+    };
+    /** URI to full trace payload in blob storage */
+    traceDataUri: string;
+    /** Task ID (denormalized) */
+    taskId: string;
+    /** Model ID */
+    modelId: string;
+    /** Tool call count */
+    toolCallCount: number;
+    /** Tool call category breakdown */
+    toolCallCategories: Record<string, number>;
+    /** Total tokens consumed */
+    totalTokens: number;
+    /** Cost estimate in USD */
+    costEstimate: number;
+    /** Duration in milliseconds */
+    durationMs: number;
+}
+/** Check if a document is schema-versioned */
+export declare function isSchemaVersioned(doc: unknown): doc is SchemaVersioned;
+/** Migrate a document to the current schema version (no-op for v1) */
+export declare function migrateDocument<T extends SchemaVersioned>(doc: T): T;

package/dist/_vendor/ailf-core/types/storage-schema.js

@@ -0,0 +1,39 @@
+/**
+ * Storage schema types for Content Lake documents.
+ *
+ * All AILF document types include `schemaVersion` for forward-compatible
+ * migration. Full trace data goes to blob storage (GCP/local); only
+ * sanitized summaries are stored in Content Lake.
+ *
+ * SECURITY: The `next` dataset is publicly accessible. Never store
+ * sensitive data (API keys, full tool call arguments, raw outputs with
+ * PII) in Content Lake. Use blob storage + traceDataUri references.
+ *
+ * @see docs/design-docs/architecture-overhaul/storage-schema.md
+ */
+// ---------------------------------------------------------------------------
+// Schema version
+// ---------------------------------------------------------------------------
+/** Current schema version for all AILF document types */
+export const CURRENT_SCHEMA_VERSION = 1;
+// ---------------------------------------------------------------------------
+// Type guards
+// ---------------------------------------------------------------------------
+/** Check if a document is schema-versioned */
+export function isSchemaVersioned(doc) {
+    return (typeof doc === "object" &&
+        doc !== null &&
+        "schemaVersion" in doc &&
+        typeof doc.schemaVersion === "number");
+}
+/** Migrate a document to the current schema version (no-op for v1) */
+export function migrateDocument(doc) {
+    if (doc.schemaVersion > CURRENT_SCHEMA_VERSION) {
+        throw new Error(`Document schema version ${doc.schemaVersion} is newer than supported version ${CURRENT_SCHEMA_VERSION}. ` +
+            `Please upgrade @sanity/ailf-core.`);
+    }
+    if (doc.schemaVersion === CURRENT_SCHEMA_VERSION)
+        return doc;
+    // When version 2 is introduced, add migration transforms here.
+    return { ...doc, schemaVersion: CURRENT_SCHEMA_VERSION };
+}

package/dist/_vendor/ailf-core/types/task-graph.d.ts

@@ -0,0 +1,86 @@
+/**
+ * TaskGraph — Intermediate Representation for the evaluation compiler.
+ *
+ * The TaskGraph is a directed acyclic graph (DAG) of tasks with dependency
+ * edges, shared fixtures, and compilation targets. It sits between the
+ * user-authored task definitions and the generated Promptfoo configs.
+ *
+ * Compilation pipeline:
+ *   GeneralizedTaskDefinitions → parse → TaskGraph → optimize → PromptfooConfig[]
+ *                                       ↘ validate ↗
+ *
+ * These types are introduced in Phase 0 but not consumed until Phase 2
+ * (the compiler). They establish the IR vocabulary early so all design
+ * discussions and subsequent phases use the same type language.
+ *
+ * Design decision: DAG-only (no cycles). See domain-model.md.
+ *
+ * @see docs/design-docs/architecture-overhaul/domain-model.md (canonical)
+ */
+import type { VariableEnvelope } from "./variable-envelope.js";
+/** A resolved fixture ready for injection into evaluations */
+export interface ResolvedFixture {
+    /** Fixture identifier */
+    id: string;
+    /** Human-readable name */
+    name: string;
+    /** Fixture type */
+    type: "static" | "fetched" | "generated";
+    /** Resolved content (null if generation failed or content is deferred) */
+    content: string | null;
+    /** Content hash for cache invalidation */
+    contentHash?: string;
+    /** Time-to-live in seconds (for fetched/generated fixtures) */
+    ttlSeconds?: number;
+}
+/**
+ * A single node in the task graph — one task with its resolved context.
+ */
+export interface TaskNode {
+    /** The task identifier */
+    taskId: string;
+    /** The evaluation mode this task belongs to (from GeneralizedTaskDefinition.mode) */
+    mode?: string;
+    /** Resolved prompt template (after variable interpolation) */
+    resolvedPrompt: string;
+    /** All variables available to this task, with provenance tracking */
+    resolvedVariables: VariableEnvelope;
+    /** Task IDs this node depends on (must complete before this node runs) */
+    dependsOn: string[];
+    /** Execution priority (lower = higher priority). Used for topological tie-breaking. */
+    priority: number;
+}
+/**
+ * A directed edge between two tasks in the graph.
+ *
+ * Edge types:
+ * - `data` — the downstream task consumes output from the upstream task
+ * - `ordering` — pure execution ordering constraint (no data flow)
+ * - `fixture` — both tasks share a fixture; the fixture must be resolved first
+ */
+export interface DependencyEdge {
+    /** Source task (upstream — must complete first) */
+    from: string;
+    /** Target task (downstream — depends on `from`) */
+    to: string;
+    /** Relationship type */
+    type: "data" | "ordering" | "fixture";
+}
+/**
+ * The complete task graph IR — the compilation unit for the eval pipeline.
+ *
+ * Invariants:
+ * - The graph is a DAG (no cycles). The compiler rejects cyclic graphs.
+ * - Every `taskId` in `edges` must exist as a key in `nodes`.
+ * - Every `fixtureId` referenced by a node must exist in `fixtures`.
+ */
+export interface TaskGraph {
+    /** All task nodes, keyed by task ID */
+    nodes: Map<string, TaskNode>;
+    /** Dependency edges between tasks */
+    edges: DependencyEdge[];
+    /** Resolved fixtures shared across tasks, keyed by fixture ID */
+    fixtures: Map<string, ResolvedFixture>;
+    /** The backend this graph compiles to */
+    compilationTarget: "promptfoo" | "custom";
+}

package/dist/_vendor/ailf-core/types/task-graph.js

@@ -0,0 +1,20 @@
+/**
+ * TaskGraph — Intermediate Representation for the evaluation compiler.
+ *
+ * The TaskGraph is a directed acyclic graph (DAG) of tasks with dependency
+ * edges, shared fixtures, and compilation targets. It sits between the
+ * user-authored task definitions and the generated Promptfoo configs.
+ *
+ * Compilation pipeline:
+ *   GeneralizedTaskDefinitions → parse → TaskGraph → optimize → PromptfooConfig[]
+ *                                       ↘ validate ↗
+ *
+ * These types are introduced in Phase 0 but not consumed until Phase 2
+ * (the compiler). They establish the IR vocabulary early so all design
+ * discussions and subsequent phases use the same type language.
+ *
+ * Design decision: DAG-only (no cycles). See domain-model.md.
+ *
+ * @see docs/design-docs/architecture-overhaul/domain-model.md (canonical)
+ */
+export {};

package/dist/_vendor/ailf-core/types/trace.d.ts

@@ -0,0 +1,118 @@
+/**
+ * EvalTrace — Observability types for evaluation execution.
+ *
+ * Captures the full request/response lifecycle for a single test case,
+ * including tool calls, URLs visited, search terms, token usage, and
+ * a chronological event log for replay and debugging.
+ *
+ * Canonical field names:
+ * - `operation` on TraceSpan (not `name` — storage adapts on serialization)
+ *
+ * Design decision: Start with simple per-result traces (flat span list
+ * per EvalResult). No depth limits for agent harness spans. OTel
+ * integration deferred.
+ *
+ * @see docs/design-docs/architecture-overhaul/observability-telemetry.md (canonical)
+ * @see docs/design-docs/architecture-overhaul/domain-model.md (Trace entity)
+ */
+/**
+ * Normalized tool call category for cross-model comparison.
+ *
+ * Raw tool names from providers are noisy and inconsistent (`WebSearch`
+ * vs `web_search` vs `Browser.search`). AILF normalizes every tool call
+ * into one of six categories.
+ */
+export type ToolCallCategory = "search" | "read" | "write" | "execute" | "navigate" | "communicate";
+/** A single tool call made during evaluation execution */
+export interface ToolCallRecord {
+    /** Tool name as reported by the provider (e.g., "WebSearch", "Bash") */
+    name: string;
+    /** Serialized input arguments */
+    input: Record<string, unknown>;
+    /** Serialized output (truncated to maxOutputBytes) */
+    output: unknown;
+    /** Wall-clock duration of the tool call in milliseconds */
+    durationMs: number;
+    /** Error message if the tool call failed */
+    error?: string;
+    /** Classification category */
+    category: ToolCallCategory;
+}
+/** A single span in the trace — one logical operation */
+export interface TraceSpan {
+    /** Unique identifier for this span */
+    spanId: string;
+    /** Parent span ID (null for root spans) */
+    parentSpanId: string | null;
+    /** Operation name (canonical field — storage may use `name`) */
+    operation: string;
+    /** Start time relative to trace start, in milliseconds */
+    startMs: number;
+    /** End time relative to trace start, in milliseconds */
+    endMs: number;
+    /** Arbitrary key-value attributes for this span */
+    attributes: Record<string, unknown>;
+}
+/** Timestamped event in the trace log */
+export interface TraceEvent {
+    /** ISO 8601 timestamp */
+    timestamp: string;
+    /** Event type */
+    type: "tool_call_start" | "tool_call_end" | "llm_request" | "llm_response" | "error" | "checkpoint" | "custom";
+    /** Event-specific payload */
+    data: Record<string, unknown>;
+}
+/** Token usage breakdown for a single evaluation step */
+export interface TraceTokenUsage {
+    /** Tokens in the prompt/input */
+    promptTokens: number;
+    /** Tokens in the completion/output */
+    completionTokens: number;
+    /** Total tokens (prompt + completion) */
+    totalTokens: number;
+    /** Tokens consumed by tool call inputs/outputs */
+    toolTokens?: number;
+}
+/**
+ * The complete trace for a single test case execution.
+ *
+ * Captures everything that happened during one task × one provider
+ * evaluation: tool calls, URLs visited, search terms, token usage,
+ * cost, and a chronological event log.
+ */
+export interface EvalTrace {
+    /** Unique identifier for this trace */
+    traceId: string;
+    /** Parent run this trace belongs to */
+    runId: string;
+    /** Task definition that produced this test case */
+    taskId: string;
+    /** Test case index within the task */
+    testCaseIndex: number;
+    /** Model under evaluation */
+    modelId: string;
+    /** Hierarchical spans for structured tracing */
+    spans: TraceSpan[];
+    /** Ordered list of tool calls made during execution */
+    toolCalls: ToolCallRecord[];
+    /** URLs fetched via WebFetch, WebSearch, or browser navigation */
+    urlsVisited: string[];
+    /** Search queries issued to WebSearch or semantic search */
+    searchTerms: string[];
+    /** Files read during execution (sandbox paths) */
+    filesRead: string[];
+    /** Files created or modified during execution */
+    filesWritten: string[];
+    /** Aggregate token usage across all LLM calls in this trace */
+    tokensUsed: TraceTokenUsage;
+    /** Estimated cost in USD */
+    costEstimate: number;
+    /** Total wall-clock execution time in milliseconds */
+    durationMs: number;
+    /** Chronological event log for replay / debugging */
+    events: TraceEvent[];
+    /** ISO 8601 start timestamp */
+    startedAt: string;
+    /** ISO 8601 end timestamp */
+    completedAt: string;
+}

package/dist/_vendor/ailf-core/types/trace.js

@@ -0,0 +1,18 @@
+/**
+ * EvalTrace — Observability types for evaluation execution.
+ *
+ * Captures the full request/response lifecycle for a single test case,
+ * including tool calls, URLs visited, search terms, token usage, and
+ * a chronological event log for replay and debugging.
+ *
+ * Canonical field names:
+ * - `operation` on TraceSpan (not `name` — storage adapts on serialization)
+ *
+ * Design decision: Start with simple per-result traces (flat span list
+ * per EvalResult). No depth limits for agent harness spans. OTel
+ * integration deferred.
+ *
+ * @see docs/design-docs/architecture-overhaul/observability-telemetry.md (canonical)
+ * @see docs/design-docs/architecture-overhaul/domain-model.md (Trace entity)
+ */
+export {};

package/dist/_vendor/ailf-core/types/variable-envelope.d.ts

@@ -0,0 +1,80 @@
+/**
+ * VariableEnvelope — Structured variable tracking with provenance.
+ *
+ * Replaces ad-hoc `Record<string, string>` variable bags with a
+ * structured, auditable format. Every variable carries metadata about
+ * where it came from, when it was resolved, and its content hash for
+ * cache invalidation.
+ *
+ * Canonical field names: `values` and `provenance`.
+ * The storage layer (Content Lake) may adapt these to different names
+ * on serialization (e.g., `vars`/`meta`), but the domain model always
+ * uses the canonical names defined here.
+ *
+ * @see docs/design-docs/architecture-overhaul/domain-model.md (canonical)
+ */
+/** Where a variable's value originated */
+export type VariableSource = {
+    type: "fixture";
+    fixtureId: string;
+} | {
+    type: "inline";
+    definedIn: "task" | "suite";
+} | {
+    type: "environment";
+    envVar: string;
+} | {
+    type: "derived";
+    expression: string;
+    inputs: string[];
+} | {
+    type: "fetched";
+    url: string;
+} | {
+    type: "previous-result";
+    resultId: string;
+    field: string;
+};
+/** Provenance metadata for a single variable */
+export interface VariableProvenance {
+    /** Where this variable's value came from */
+    source: VariableSource;
+    /** When the variable was resolved (ISO 8601 timestamp) */
+    resolvedAt: string;
+    /** Time-to-live in seconds (for fetched/cached variables) */
+    ttl?: number;
+    /** Content-addressable hash for cache invalidation */
+    hash: string;
+}
+/** Schema declaration for a variable — enables compile-time template validation */
+export interface VariableDeclaration {
+    /** Variable name (matches the key in `values`) */
+    name: string;
+    /** Expected type of the variable's value */
+    type: "string" | "number" | "boolean" | "json" | "markdown" | "unknown";
+    /** Whether the variable must be present for the task to run */
+    required: boolean;
+    /** Human-readable description */
+    description?: string;
+    /** Default value used when the variable is not provided */
+    default?: unknown;
+}
+/**
+ * A structured container for all variables flowing through an evaluation step.
+ *
+ * The envelope enables:
+ * - **Cache invalidation** — if a variable's content hash changes, downstream
+ *   results are invalidated.
+ * - **Audit trails** — every variable in a result can be traced back to its
+ *   origin.
+ * - **Type checking** — declarations allow compile-time validation of prompt
+ *   templates against available variables.
+ */
+export interface VariableEnvelope {
+    /** The resolved key-value pairs available to prompt templates */
+    values: Record<string, unknown>;
+    /** Provenance metadata for each variable (keyed by variable name) */
+    provenance: Record<string, VariableProvenance>;
+    /** Schema declarations for type-safe access */
+    declarations: VariableDeclaration[];
+}

package/dist/_vendor/ailf-core/types/variable-envelope.js

@@ -0,0 +1,16 @@
+/**
+ * VariableEnvelope — Structured variable tracking with provenance.
+ *
+ * Replaces ad-hoc `Record<string, string>` variable bags with a
+ * structured, auditable format. Every variable carries metadata about
+ * where it came from, when it was resolved, and its content hash for
+ * cache invalidation.
+ *
+ * Canonical field names: `values` and `provenance`.
+ * The storage layer (Content Lake) may adapt these to different names
+ * on serialization (e.g., `vars`/`meta`), but the domain model always
+ * uses the canonical names defined here.
+ *
+ * @see docs/design-docs/architecture-overhaul/domain-model.md (canonical)
+ */
+export {};

package/dist/_vendor/ailf-shared/dimension-names.d.ts

@@ -1,21 +1,8 @@
 /**
- * Grading dimension types and labels.
+ * @deprecated This file is intentionally empty.
  *
- * Shared between eval (scoring engine) and studio (display components).
- * Adding a dimension here ensures both packages show it correctly.
+ * Dimension names are now dynamic — derived at runtime from the scoring
+ * profile's dimension-weight map in packages/eval/config/rubrics.ts.
+ * Use `extractDimensions()` from @sanity/ailf-core to get dimension
+ * names for a given profile.
  */
-/** The three grading dimensions */
-export type DimensionName = "codeCorrectness" | "docCoverage" | "taskCompletion";
-/**
- * Human-readable display labels for dimensions.
- *
- * Note: The Sanity Content Lake stores dimension names in kebab-case
- * (e.g., "task-completion") while TypeScript uses camelCase. Both
- * mappings are provided for cross-system compatibility.
- */
-export declare const DIMENSION_LABELS: Record<DimensionName, string>;
-/**
- * Kebab-case dimension labels as stored in Sanity Content Lake.
- * Used by the studio package when rendering judgment data from GROQ.
- */
-export declare const DIMENSION_LABELS_KEBAB: Record<string, string>;

package/dist/_vendor/ailf-shared/dimension-names.js

@@ -1,27 +1,9 @@
+"use strict";
 /**
- * Grading dimension types and labels.
+ * @deprecated This file is intentionally empty.
  *
- * Shared between eval (scoring engine) and studio (display components).
- * Adding a dimension here ensures both packages show it correctly.
+ * Dimension names are now dynamic — derived at runtime from the scoring
+ * profile's dimension-weight map in packages/eval/config/rubrics.ts.
+ * Use `extractDimensions()` from @sanity/ailf-core to get dimension
+ * names for a given profile.
  */
-/**
- * Human-readable display labels for dimensions.
- *
- * Note: The Sanity Content Lake stores dimension names in kebab-case
- * (e.g., "task-completion") while TypeScript uses camelCase. Both
- * mappings are provided for cross-system compatibility.
- */
-export const DIMENSION_LABELS = {
-    codeCorrectness: "Code Correctness",
-    docCoverage: "Doc Coverage",
-    taskCompletion: "Task Completion",
-};
-/**
- * Kebab-case dimension labels as stored in Sanity Content Lake.
- * Used by the studio package when rendering judgment data from GROQ.
- */
-export const DIMENSION_LABELS_KEBAB = {
-    "code-correctness": "Code Correctness",
-    "doc-coverage": "Doc Coverage",
-    "task-completion": "Task Completion",
-};