@sanity/ailf 0.5.0 → 1.0.0

package/dist/pipeline/compiler/scoring-bridge.d.ts

@@ -0,0 +1,49 @@
+/**
+ * scoring-bridge.ts — Bridge between Promptfoo raw results and the
+ * 4-tier scoring engine.
+ *
+ * Converts Promptfoo `ComponentResult[]` (from test results) into the
+ * scoring engine's `AssertionScore[]` format, then delegates aggregation
+ * to `aggregateDimensions` and `computeTaskScore` from core.
+ *
+ * This bridge replaces the three legacy scoring primitives in
+ * `calculate-scores.ts`:
+ *   - `accumulateDimensions`  → `convertToAssertionScores` + `aggregateDimensions`
+ *   - `averageDimensions`     → (handled internally by `aggregateDimensions`)
+ *   - `weightedComposite`     → `computeTaskScore`
+ *
+ * The bridge preserves the existing 0–100 output scale. The 4-tier
+ * engine works in [0, 1]; this module handles the conversion at
+ * boundaries.
+ *
+ * @see packages/core/src/services/scoring-engine.ts — the 4-tier engine
+ * @see packages/eval/src/pipeline/calculate-scores.ts — the consumer
+ * @see docs/exec-plans/architecture-overhaul/phase-7-migrate-literacy.md
+ */
+import { type DimensionScore } from "../../_vendor/ailf-core/index.d.ts";
+import type { TestResult } from "../../_vendor/ailf-core/index.d.ts";
+/** Result of scoring a group of tests via the 4-tier engine */
+export interface BridgedScoreResult {
+    /** Per-dimension breakdown (0–100 scale) */
+    dimensions: Record<string, number>;
+    /** Weighted composite score (0–100 scale) */
+    composite: number;
+    /** Total cost across all tests */
+    totalCost: number;
+    /** Raw DimensionScore objects from the engine (0–1 scale) */
+    rawDimensions: DimensionScore[];
+}
+/**
+ * Score a group of test results using the 4-tier scoring engine.
+ *
+ * This replaces the legacy `accumulateDimensions → averageDimensions →
+ * weightedComposite` chain with the new engine's `aggregateDimensions →
+ * computeTaskScore` chain.
+ *
+ * @param tests    Pre-filtered test results (e.g., all gold or all baseline)
+ * @param profile  Weight profile mapping kebab-case dimension names to weights
+ *                 (e.g., `{ "task-completion": 0.4, "code-correctness": 0.35, "doc-coverage": 0.25 }`)
+ * @param taskId   Optional task identifier for traceability in TaskScore output
+ * @returns        Dimensions (0–100) and composite (0–100), matching legacy output format
+ */
+export declare function scoreTestGroup(tests: TestResult[], profile: Record<string, number>, taskId?: string): BridgedScoreResult;

package/dist/pipeline/compiler/scoring-bridge.js

@@ -0,0 +1,114 @@
+/**
+ * scoring-bridge.ts — Bridge between Promptfoo raw results and the
+ * 4-tier scoring engine.
+ *
+ * Converts Promptfoo `ComponentResult[]` (from test results) into the
+ * scoring engine's `AssertionScore[]` format, then delegates aggregation
+ * to `aggregateDimensions` and `computeTaskScore` from core.
+ *
+ * This bridge replaces the three legacy scoring primitives in
+ * `calculate-scores.ts`:
+ *   - `accumulateDimensions`  → `convertToAssertionScores` + `aggregateDimensions`
+ *   - `averageDimensions`     → (handled internally by `aggregateDimensions`)
+ *   - `weightedComposite`     → `computeTaskScore`
+ *
+ * The bridge preserves the existing 0–100 output scale. The 4-tier
+ * engine works in [0, 1]; this module handles the conversion at
+ * boundaries.
+ *
+ * @see packages/core/src/services/scoring-engine.ts — the 4-tier engine
+ * @see packages/eval/src/pipeline/calculate-scores.ts — the consumer
+ * @see docs/exec-plans/architecture-overhaul/phase-7-migrate-literacy.md
+ */
+import { aggregateDimensions, computeTaskScore, normalizeScore, } from "../../_vendor/ailf-core/index.js";
+import { classifyRubric, parseRubricScore } from "../../_vendor/ailf-core/index.js";
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+/**
+ * Score a group of test results using the 4-tier scoring engine.
+ *
+ * This replaces the legacy `accumulateDimensions → averageDimensions →
+ * weightedComposite` chain with the new engine's `aggregateDimensions →
+ * computeTaskScore` chain.
+ *
+ * @param tests    Pre-filtered test results (e.g., all gold or all baseline)
+ * @param profile  Weight profile mapping kebab-case dimension names to weights
+ *                 (e.g., `{ "task-completion": 0.4, "code-correctness": 0.35, "doc-coverage": 0.25 }`)
+ * @param taskId   Optional task identifier for traceability in TaskScore output
+ * @returns        Dimensions (0–100) and composite (0–100), matching legacy output format
+ */
+export function scoreTestGroup(tests, profile, taskId) {
+    let totalCost = 0;
+    // Step 1: Convert all ComponentResults into AssertionScore[] (0–1 scale)
+    const assertionScores = [];
+    for (const test of tests) {
+        totalCost += test.cost;
+        for (const comp of test.gradingResult.componentResults) {
+            if (comp.assertion?.type !== "llm-rubric")
+                continue;
+            const converted = componentToAssertionScore(comp);
+            if (converted)
+                assertionScores.push(converted);
+        }
+    }
+    // Step 2: Aggregate into DimensionScores (0–1 scale)
+    const dimensionLabels = {
+        "code-correctness": "Code Correctness",
+        "doc-coverage": "Doc Coverage",
+        "task-completion": "Task Completion",
+    };
+    const rawDimensions = aggregateDimensions(assertionScores, {
+        defaultAggregation: "mean",
+        dimensionLabels,
+    });
+    // Step 3: Compute weighted composite via TaskScore (0–1 scale)
+    const taskScoreResult = computeTaskScore(rawDimensions, {
+        taskId: taskId ?? "aggregate",
+        weights: profile,
+        weightSource: "scoring-bridge",
+    });
+    // Step 4: Convert back to 0–100 scale for legacy compatibility
+    const dimensions = {};
+    for (const dim of rawDimensions) {
+        // Map kebab-case dimension IDs to camelCase for legacy compatibility
+        const camelKey = kebabToCamel(dim.dimensionId);
+        dimensions[camelKey] = Math.round(dim.score * 100);
+    }
+    return {
+        composite: Math.round(taskScoreResult.score * 100),
+        dimensions,
+        rawDimensions,
+        totalCost,
+    };
+}
+// ---------------------------------------------------------------------------
+// Conversion helpers
+// ---------------------------------------------------------------------------
+/**
+ * Convert a single Promptfoo ComponentResult into the scoring engine's
+ * AssertionScore format.
+ *
+ * Returns null if the component doesn't map to a known dimension.
+ */
+function componentToAssertionScore(comp) {
+    const dim = classifyRubric(comp);
+    if (!dim)
+        return null;
+    // Parse the raw score (0–100 from the grader) and normalize to [0, 1]
+    const rawScore = parseRubricScore(comp);
+    const normalized = normalizeScore(rawScore, "llm-rubric");
+    return {
+        assertionType: comp.assertion?.type ?? "llm-rubric",
+        dimension: dim,
+        latencyMs: 0,
+        pass: comp.pass,
+        reason: comp.reason ?? "",
+        score: normalized,
+        weight: 1.0,
+    };
+}
+/** Convert kebab-case dimension key to camelCase (e.g., "task-completion" → "taskCompletion") */
+function kebabToCamel(kebab) {
+    return kebab.replace(/-([a-z])/g, (_, c) => c.toUpperCase());
+}

package/dist/pipeline/compiler/task-graph-builder.d.ts

@@ -0,0 +1,54 @@
+/**
+ * TaskGraphBuilder — converts task definitions into a TaskGraph IR.
+ *
+ * The builder is the first stage of the compilation pipeline:
+ *   GeneralizedTaskDefinitions → TaskGraphBuilder → TaskGraph → PromptfooCompiler → YAML
+ *
+ * Responsibilities:
+ * - Accept tasks from any source (TS, YAML, Content Lake)
+ * - Apply area/tag/mode filtering
+ * - Resolve inter-task dependencies into edges
+ * - Validate the graph is a DAG (reject cycles)
+ * - Assign execution priority via topological sort
+ *
+ * This module exists alongside `generate-configs.ts` — it does NOT replace
+ * the existing codegen path. Phase 7 will swap callers over to the compiler.
+ *
+ * @see packages/core/src/types/task-graph.ts — TaskGraph types
+ * @see docs/exec-plans/architecture-overhaul/phase-2-config-compiler.md
+ */
+import type { DependencyEdge, FilterOptions, GeneralizedTaskDefinition, TaskGraph, TaskNode } from "../../_vendor/ailf-core/index.d.ts";
+/** Options for building a task graph */
+export interface TaskGraphBuildOptions {
+    /** Task definitions from any source */
+    tasks: GeneralizedTaskDefinition[];
+    /** Optional filter to narrow task set */
+    filter?: FilterOptions;
+    /** Compilation target backend */
+    compilationTarget?: "custom" | "promptfoo";
+}
+/** Result of building a task graph */
+export interface TaskGraphBuildResult {
+    /** The built graph (null if no tasks survived filtering) */
+    graph: TaskGraph | null;
+    /** Warnings emitted during build (non-fatal) */
+    warnings: string[];
+    /** Tasks that were filtered out */
+    filteredOut: string[];
+}
+/**
+ * Build a TaskGraph from task definitions.
+ *
+ * 1. Filters tasks by area, tags, task IDs, and status
+ * 2. Creates TaskNodes with resolved variables
+ * 3. Discovers dependency edges from task metadata
+ * 4. Validates the graph is acyclic
+ * 5. Assigns topological priority
+ */
+export declare function buildTaskGraph(options: TaskGraphBuildOptions): TaskGraphBuildResult;
+/**
+ * Detect cycles in the task graph using Kahn's algorithm.
+ *
+ * @returns null if acyclic, or the cycle path as a string array
+ */
+export declare function detectCycle(nodes: Map<string, TaskNode>, edges: DependencyEdge[]): string[] | null;

package/dist/pipeline/compiler/task-graph-builder.js

@@ -0,0 +1,291 @@
+/**
+ * TaskGraphBuilder — converts task definitions into a TaskGraph IR.
+ *
+ * The builder is the first stage of the compilation pipeline:
+ *   GeneralizedTaskDefinitions → TaskGraphBuilder → TaskGraph → PromptfooCompiler → YAML
+ *
+ * Responsibilities:
+ * - Accept tasks from any source (TS, YAML, Content Lake)
+ * - Apply area/tag/mode filtering
+ * - Resolve inter-task dependencies into edges
+ * - Validate the graph is a DAG (reject cycles)
+ * - Assign execution priority via topological sort
+ *
+ * This module exists alongside `generate-configs.ts` — it does NOT replace
+ * the existing codegen path. Phase 7 will swap callers over to the compiler.
+ *
+ * @see packages/core/src/types/task-graph.ts — TaskGraph types
+ * @see docs/exec-plans/architecture-overhaul/phase-2-config-compiler.md
+ */
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+/**
+ * Build a TaskGraph from task definitions.
+ *
+ * 1. Filters tasks by area, tags, task IDs, and status
+ * 2. Creates TaskNodes with resolved variables
+ * 3. Discovers dependency edges from task metadata
+ * 4. Validates the graph is acyclic
+ * 5. Assigns topological priority
+ */
+export function buildTaskGraph(options) {
+    const warnings = [];
+    const filteredOut = [];
+    // Step 1: Filter tasks
+    const filtered = filterTasks(options.tasks, options.filter, filteredOut);
+    if (filtered.length === 0) {
+        return {
+            graph: null,
+            warnings: ["No tasks matched the filter criteria"],
+            filteredOut,
+        };
+    }
+    // Step 2: Create nodes
+    const nodes = new Map();
+    for (const task of filtered) {
+        const node = taskToNode(task);
+        if (nodes.has(node.taskId)) {
+            warnings.push(`Duplicate task ID "${node.taskId}" — later definition wins`);
+        }
+        nodes.set(node.taskId, node);
+    }
+    // Step 3: Discover edges from dependency metadata
+    const edges = discoverEdges(filtered, nodes, warnings);
+    // Step 4: Validate acyclicity
+    const cycleError = detectCycle(nodes, edges);
+    if (cycleError) {
+        throw new Error(`Task graph contains a cycle: ${cycleError.join(" → ")}. ` +
+            "Task graphs must be directed acyclic graphs (DAGs).");
+    }
+    // Step 5: Assign topological priority
+    assignPriority(nodes, edges);
+    // Step 6: Build fixture map (empty for now — Phase 2d fills this)
+    const fixtures = new Map();
+    const graph = {
+        compilationTarget: options.compilationTarget ?? "promptfoo",
+        edges,
+        fixtures,
+        nodes,
+    };
+    return { graph, warnings, filteredOut };
+}
+// ---------------------------------------------------------------------------
+// Filtering
+// ---------------------------------------------------------------------------
+function filterTasks(tasks, filter, filteredOut) {
+    return tasks.filter((task) => {
+        // Status filter — always applied (even without explicit filter options)
+        const status = task.status ?? "active";
+        const isTargetedById = filter?.taskIds && filter.taskIds.includes(task.id);
+        if (status === "archived") {
+            filteredOut.push(task.id);
+            return false;
+        }
+        if (status === "paused" && !isTargetedById) {
+            filteredOut.push(task.id);
+            return false;
+        }
+        if (status === "draft" && !isTargetedById && !filter?.includeDrafts) {
+            filteredOut.push(task.id);
+            return false;
+        }
+        // Remaining filters only apply when an explicit filter is provided
+        if (!filter)
+            return true;
+        // Area filter — GeneralizedTaskDefinition uses `area` (not `featureArea`)
+        const taskArea = task.area ?? "";
+        if (filter.areas &&
+            filter.areas.length > 0 &&
+            !filter.areas.map((a) => a.toLowerCase()).includes(taskArea.toLowerCase())) {
+            filteredOut.push(task.id);
+            return false;
+        }
+        // Task ID filter
+        if (filter.taskIds &&
+            filter.taskIds.length > 0 &&
+            !filter.taskIds.includes(task.id)) {
+            filteredOut.push(task.id);
+            return false;
+        }
+        // Tag filter
+        if (filter.tags &&
+            filter.tags.length > 0 &&
+            (!task.tags || !task.tags.some((t) => filter.tags.includes(t)))) {
+            filteredOut.push(task.id);
+            return false;
+        }
+        return true;
+    });
+}
+// ---------------------------------------------------------------------------
+// Node creation
+// ---------------------------------------------------------------------------
+function taskToNode(task) {
+    // GeneralizedTaskDefinition uses prompt.text/prompt.template instead of taskPrompt,
+    // and prompt.vars instead of extraVars
+    const promptText = task.prompt?.text ?? task.prompt?.template ?? "";
+    const promptVars = task.prompt?.vars ?? {};
+    const envelope = {
+        declarations: [],
+        provenance: {},
+        values: {
+            ...(promptText ? { task: promptText } : {}),
+            ...promptVars,
+        },
+    };
+    return {
+        dependsOn: [],
+        mode: task.mode,
+        priority: 0,
+        resolvedPrompt: promptText,
+        resolvedVariables: envelope,
+        taskId: task.id,
+    };
+}
+// ---------------------------------------------------------------------------
+// Edge discovery
+// ---------------------------------------------------------------------------
+/**
+ * Discover dependency edges from task metadata.
+ *
+ * Looks for explicit `dependsOn` arrays in prompt.vars (the generalized
+ * equivalent of the old extraVars convention).
+ * Future phases will add implicit deps from fixture sharing, data flow, etc.
+ */
+function discoverEdges(tasks, nodes, warnings) {
+    const edges = [];
+    for (const task of tasks) {
+        // Check for explicit dependencies in prompt.vars (was extraVars.dependsOn)
+        const deps = task.prompt?.vars?.dependsOn;
+        if (Array.isArray(deps)) {
+            for (const dep of deps) {
+                if (typeof dep !== "string")
+                    continue;
+                if (!nodes.has(dep)) {
+                    warnings.push(`Task "${task.id}" depends on "${dep}" which is not in the graph — ` +
+                        "dependency ignored (task may have been filtered out)");
+                    continue;
+                }
+                edges.push({ from: dep, to: task.id, type: "ordering" });
+                const node = nodes.get(task.id);
+                if (node && !node.dependsOn.includes(dep)) {
+                    node.dependsOn.push(dep);
+                }
+            }
+        }
+    }
+    return edges;
+}
+// ---------------------------------------------------------------------------
+// Cycle detection — Kahn's algorithm (topological sort)
+// ---------------------------------------------------------------------------
+/**
+ * Detect cycles in the task graph using Kahn's algorithm.
+ *
+ * @returns null if acyclic, or the cycle path as a string array
+ */
+export function detectCycle(nodes, edges) {
+    // Build in-degree map
+    const inDegree = new Map();
+    const adjacency = new Map();
+    for (const id of nodes.keys()) {
+        inDegree.set(id, 0);
+        adjacency.set(id, []);
+    }
+    for (const edge of edges) {
+        adjacency.get(edge.from).push(edge.to);
+        inDegree.set(edge.to, (inDegree.get(edge.to) ?? 0) + 1);
+    }
+    // Start with all zero-in-degree nodes
+    const queue = [];
+    for (const [id, deg] of inDegree) {
+        if (deg === 0)
+            queue.push(id);
+    }
+    let visited = 0;
+    while (queue.length > 0) {
+        const current = queue.shift();
+        visited++;
+        for (const neighbor of adjacency.get(current) ?? []) {
+            const newDeg = (inDegree.get(neighbor) ?? 1) - 1;
+            inDegree.set(neighbor, newDeg);
+            if (newDeg === 0)
+                queue.push(neighbor);
+        }
+    }
+    if (visited === nodes.size)
+        return null;
+    // Find cycle participants (nodes with remaining in-degree > 0)
+    const cycleNodes = [...inDegree.entries()]
+        .filter(([, deg]) => deg > 0)
+        .map(([id]) => id);
+    // Reconstruct a cycle path for the error message
+    return reconstructCyclePath(cycleNodes, adjacency);
+}
+/**
+ * Reconstruct a human-readable cycle path from cycle participants.
+ */
+function reconstructCyclePath(cycleNodes, adjacency) {
+    if (cycleNodes.length === 0)
+        return [];
+    const inCycle = new Set(cycleNodes);
+    const start = cycleNodes[0];
+    const path = [start];
+    const visited = new Set();
+    let current = start;
+    // Follow edges within the cycle to produce a readable path
+    while (true) {
+        visited.add(current);
+        const next = (adjacency.get(current) ?? []).find((n) => inCycle.has(n) && (!visited.has(n) || n === start));
+        if (!next)
+            break;
+        path.push(next);
+        if (next === start)
+            break; // Completed the cycle
+        current = next;
+    }
+    return path;
+}
+// ---------------------------------------------------------------------------
+// Topological priority assignment
+// ---------------------------------------------------------------------------
+/**
+ * Assign execution priority via topological order.
+ * Lower priority = earlier execution.
+ */
+function assignPriority(nodes, edges) {
+    const inDegree = new Map();
+    const adjacency = new Map();
+    for (const id of nodes.keys()) {
+        inDegree.set(id, 0);
+        adjacency.set(id, []);
+    }
+    for (const edge of edges) {
+        adjacency.get(edge.from).push(edge.to);
+        inDegree.set(edge.to, (inDegree.get(edge.to) ?? 0) + 1);
+    }
+    const queue = [];
+    for (const [id, deg] of inDegree) {
+        if (deg === 0)
+            queue.push(id);
+    }
+    let priority = 0;
+    while (queue.length > 0) {
+        // Process all nodes at the current level (same priority)
+        const levelSize = queue.length;
+        for (let i = 0; i < levelSize; i++) {
+            const current = queue.shift();
+            const node = nodes.get(current);
+            if (node)
+                node.priority = priority;
+            for (const neighbor of adjacency.get(current) ?? []) {
+                const newDeg = (inDegree.get(neighbor) ?? 1) - 1;
+                inDegree.set(neighbor, newDeg);
+                if (newDeg === 0)
+                    queue.push(neighbor);
+            }
+        }
+        priority++;
+    }
+}

package/dist/pipeline/compiler/telemetry/cost-tracker.d.ts

@@ -0,0 +1,90 @@
+/**
+ * Cost tracking — model pricing, pre-run estimation, and post-run actuals.
+ *
+ * Uses a pricing table (YAML config or TS `definePricingTable()`) to compute
+ * USD cost from token usage. Supports budget controls with warn/stop thresholds.
+ *
+ * @see docs/design-docs/architecture-overhaul/observability-telemetry.md
+ */
+import type { TraceTokenUsage } from "../../../_vendor/ailf-core/index.d.ts";
+/** Per-model pricing (USD per 1M tokens) */
+export interface ModelPricing {
+    /** Input tokens cost per 1M tokens */
+    input: number;
+    /** Output tokens cost per 1M tokens */
+    output: number;
+    /** Cached input tokens cost per 1M tokens (optional) */
+    cachedInput?: number;
+}
+/** Budget control thresholds (in USD) */
+export interface BudgetConfig {
+    perRun?: {
+        warn: number;
+        stop: number;
+    };
+    perTask?: {
+        warn: number;
+        stop: number;
+    };
+}
+/** Cost estimate for a pipeline run */
+export interface CostEstimate {
+    /** Estimated total cost in USD */
+    totalUSD: number;
+    /** Per-model breakdown */
+    perModel: {
+        modelId: string;
+        estimatedUSD: number;
+    }[];
+    /** Whether estimate exceeds budget warning threshold */
+    exceedsWarning: boolean;
+    /** Whether estimate exceeds budget stop threshold */
+    exceedsStop: boolean;
+}
+/** Actual cost computed from real token usage */
+export interface ActualCost {
+    /** Actual total cost in USD */
+    totalUSD: number;
+    /** Per-model actual cost */
+    perModel: {
+        modelId: string;
+        actualUSD: number;
+        tokens: TraceTokenUsage;
+    }[];
+}
+/** Budget check result */
+export interface BudgetCheckResult {
+    /** Whether to proceed */
+    proceed: boolean;
+    /** Warning message (if any) */
+    warning?: string;
+    /** Current spend in USD */
+    currentUSD: number;
+    /** Budget limit that was checked */
+    limitUSD?: number;
+}
+/**
+ * Compute actual cost from token usage and model pricing.
+ *
+ * @param usage - Token counts from provider response
+ * @param pricing - Per-model pricing (USD per 1M tokens)
+ * @returns Cost in USD
+ */
+export declare function computeCost(usage: TraceTokenUsage, pricing: ModelPricing): number;
+/**
+ * Look up pricing for a model ID.
+ *
+ * Tries exact match first, then falls back to prefix matching
+ * (e.g., "openai:chat:gpt-4o-2024-11-20" matches "openai:chat:gpt-4o").
+ */
+export declare function lookupPricing(modelId: string, customPricing?: Record<string, ModelPricing>): ModelPricing | undefined;
+/**
+ * Estimate cost for a pipeline run before execution.
+ *
+ * Uses task count, estimated tokens per task complexity, and model pricing.
+ */
+export declare function estimateRunCost(taskCount: number, modelIds: string[], budget?: BudgetConfig, customPricing?: Record<string, ModelPricing>): CostEstimate;
+/**
+ * Check if current spend exceeds budget thresholds.
+ */
+export declare function checkBudget(currentUSD: number, budget: BudgetConfig, level: "perRun" | "perTask"): BudgetCheckResult;