@sanity/ailf 0.4.1 → 1.0.0

package/dist/pipeline/compiler/telemetry/cost-tracker.js

@@ -0,0 +1,146 @@
+/**
+ * Cost tracking — model pricing, pre-run estimation, and post-run actuals.
+ *
+ * Uses a pricing table (YAML config or TS `definePricingTable()`) to compute
+ * USD cost from token usage. Supports budget controls with warn/stop thresholds.
+ *
+ * @see docs/design-docs/architecture-overhaul/observability-telemetry.md
+ */
+// ---------------------------------------------------------------------------
+// Pricing table
+// ---------------------------------------------------------------------------
+/** Default pricing table (updated periodically) */
+const DEFAULT_PRICING = {
+    "anthropic:messages:claude-opus-4-6": {
+        input: 15.0,
+        output: 75.0,
+        cachedInput: 1.5,
+    },
+    "anthropic:messages:claude-sonnet-4-6": {
+        input: 3.0,
+        output: 15.0,
+        cachedInput: 0.3,
+    },
+    "openai:chat:gpt-4.1": {
+        input: 2.0,
+        output: 8.0,
+        cachedInput: 0.5,
+    },
+    "openai:chat:gpt-4.1-mini": {
+        input: 0.4,
+        output: 1.6,
+        cachedInput: 0.1,
+    },
+    "openai:chat:gpt-4o": {
+        input: 2.5,
+        output: 10.0,
+        cachedInput: 1.25,
+    },
+    "openai:chat:gpt-5": {
+        input: 5.0,
+        output: 15.0,
+        cachedInput: 1.0,
+    },
+};
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+/**
+ * Compute actual cost from token usage and model pricing.
+ *
+ * @param usage - Token counts from provider response
+ * @param pricing - Per-model pricing (USD per 1M tokens)
+ * @returns Cost in USD
+ */
+export function computeCost(usage, pricing) {
+    const cached = usage.toolTokens ?? 0;
+    const uncachedPrompt = usage.promptTokens - cached;
+    const inputCost = (uncachedPrompt * pricing.input) / 1_000_000;
+    const cachedCost = pricing.cachedInput !== undefined
+        ? (cached * pricing.cachedInput) / 1_000_000
+        : (cached * pricing.input) / 1_000_000;
+    const outputCost = (usage.completionTokens * pricing.output) / 1_000_000;
+    return inputCost + cachedCost + outputCost;
+}
+/**
+ * Look up pricing for a model ID.
+ *
+ * Tries exact match first, then falls back to prefix matching
+ * (e.g., "openai:chat:gpt-4o-2024-11-20" matches "openai:chat:gpt-4o").
+ */
+export function lookupPricing(modelId, customPricing) {
+    // 1. Exact match in custom pricing
+    if (customPricing?.[modelId])
+        return customPricing[modelId];
+    // 2. Exact match in defaults
+    if (DEFAULT_PRICING[modelId])
+        return DEFAULT_PRICING[modelId];
+    // 3. Prefix match in custom pricing
+    if (customPricing) {
+        for (const [key, pricing] of Object.entries(customPricing)) {
+            if (modelId.startsWith(key))
+                return pricing;
+        }
+    }
+    // 4. Prefix match in defaults
+    for (const [key, pricing] of Object.entries(DEFAULT_PRICING)) {
+        if (modelId.startsWith(key))
+            return pricing;
+    }
+    return undefined;
+}
+/**
+ * Estimate cost for a pipeline run before execution.
+ *
+ * Uses task count, estimated tokens per task complexity, and model pricing.
+ */
+export function estimateRunCost(taskCount, modelIds, budget, customPricing) {
+    // Rough token estimates per task (empirical averages)
+    const AVG_PROMPT_TOKENS = 2000;
+    const AVG_COMPLETION_TOKENS = 1500;
+    const perModel = modelIds.map((modelId) => {
+        const pricing = lookupPricing(modelId, customPricing);
+        if (!pricing) {
+            return { modelId, estimatedUSD: 0 };
+        }
+        const estimatedUSD = computeCost({
+            promptTokens: AVG_PROMPT_TOKENS * taskCount,
+            completionTokens: AVG_COMPLETION_TOKENS * taskCount,
+            totalTokens: (AVG_PROMPT_TOKENS + AVG_COMPLETION_TOKENS) * taskCount,
+        }, pricing);
+        return { modelId, estimatedUSD };
+    });
+    const totalUSD = perModel.reduce((sum, m) => sum + m.estimatedUSD, 0);
+    return {
+        totalUSD,
+        perModel,
+        exceedsWarning: budget?.perRun ? totalUSD >= budget.perRun.warn : false,
+        exceedsStop: budget?.perRun ? totalUSD >= budget.perRun.stop : false,
+    };
+}
+/**
+ * Check if current spend exceeds budget thresholds.
+ */
+export function checkBudget(currentUSD, budget, level) {
+    const limits = budget[level];
+    if (!limits) {
+        return { proceed: true, currentUSD };
+    }
+    if (currentUSD >= limits.stop) {
+        return {
+            proceed: false,
+            warning: `Budget exceeded: $${currentUSD.toFixed(4)} >= $${limits.stop} (${level} stop limit)`,
+            currentUSD,
+            limitUSD: limits.stop,
+        };
+    }
+    if (currentUSD >= limits.warn) {
+        return {
+            proceed: true,
+            warning: `Budget warning: $${currentUSD.toFixed(4)} >= $${limits.warn} (${level} warn threshold)`,
+            currentUSD,
+            limitUSD: limits.warn,
+        };
+    }
+    return { proceed: true, currentUSD };
+}

package/dist/pipeline/compiler/telemetry/index.d.ts

@@ -0,0 +1,14 @@
+/**
+ * Telemetry — observability infrastructure for evaluation traces.
+ *
+ * Captures tool calls, token usage, cost, and timing for every evaluation.
+ * Full traces go to blob storage; sanitized summaries to Content Lake.
+ *
+ * @see docs/exec-plans/architecture-overhaul/phase-6-observability.md
+ * @see docs/design-docs/architecture-overhaul/observability-telemetry.md
+ */
+export { collectTrace, mergeTraces, type ProviderResponse, type RawToolCall, type TraceCollectorOptions, } from "./trace-collector.js";
+export { classifyToolCall, classifyToolCalls } from "./tool-classifier.js";
+export { checkBudget, computeCost, estimateRunCost, lookupPricing, type ActualCost, type BudgetCheckResult, type BudgetConfig, type CostEstimate, type ModelPricing, } from "./cost-tracker.js";
+export { extractTraceSummary, LocalTraceStore, type TraceSummary, type TraceStore, type TraceStoreResult, } from "./trace-store.js";
+export { createRedactionConfig, DEFAULT_REDACTION_RULES, redactTrace, type RedactionConfig, type RedactionResult, type RedactionRule, } from "./redactor.js";

package/dist/pipeline/compiler/telemetry/index.js

@@ -0,0 +1,19 @@
+/**
+ * Telemetry — observability infrastructure for evaluation traces.
+ *
+ * Captures tool calls, token usage, cost, and timing for every evaluation.
+ * Full traces go to blob storage; sanitized summaries to Content Lake.
+ *
+ * @see docs/exec-plans/architecture-overhaul/phase-6-observability.md
+ * @see docs/design-docs/architecture-overhaul/observability-telemetry.md
+ */
+// Trace collection
+export { collectTrace, mergeTraces, } from "./trace-collector.js";
+// Tool call classification
+export { classifyToolCall, classifyToolCalls } from "./tool-classifier.js";
+// Cost tracking
+export { checkBudget, computeCost, estimateRunCost, lookupPricing, } from "./cost-tracker.js";
+// Trace storage
+export { extractTraceSummary, LocalTraceStore, } from "./trace-store.js";
+// Redaction
+export { createRedactionConfig, DEFAULT_REDACTION_RULES, redactTrace, } from "./redactor.js";

package/dist/pipeline/compiler/telemetry/redactor.d.ts

@@ -0,0 +1,58 @@
+/**
+ * Redaction pipeline — strips sensitive data from traces before storage.
+ *
+ * Applied before ANY storage (both blob and Content Lake). Configurable
+ * patterns handle Bearer tokens, API keys, Sanity tokens, and other
+ * common secret formats.
+ *
+ * Principles:
+ * 1. Redact before store — sensitive data never reaches storage
+ * 2. Configurable patterns — teams can add project-specific rules
+ * 3. Truncation for cost — large outputs truncated to max bytes
+ * 4. No PII by default — tasks shouldn't contain PII, this is a safety net
+ *
+ * @see docs/design-docs/architecture-overhaul/observability-telemetry.md
+ */
+import type { EvalTrace } from "../../../_vendor/ailf-core/index.d.ts";
+/** A single redaction rule */
+export interface RedactionRule {
+    /** Rule name (for logging) */
+    name: string;
+    /** Regex pattern to match */
+    pattern: RegExp;
+    /** Replacement string (use $1, $2 for capture groups) */
+    replacement: string;
+}
+/** Redaction configuration */
+export interface RedactionConfig {
+    /** Regex-based substitution rules */
+    rules: RedactionRule[];
+    /** Fields to omit entirely from stored traces */
+    omitFields: string[];
+    /** Maximum tool call output size in bytes */
+    maxOutputBytes: number;
+}
+/** Result of redaction */
+export interface RedactionResult {
+    /** Redacted trace */
+    trace: EvalTrace;
+    /** Number of redactions applied */
+    redactionCount: number;
+    /** Which rules fired */
+    rulesApplied: string[];
+}
+/** Built-in redaction rules for common secret patterns */
+export declare const DEFAULT_REDACTION_RULES: RedactionRule[];
+/**
+ * Create a default redaction config.
+ *
+ * @param overrides - Custom rules or settings to merge
+ */
+export declare function createRedactionConfig(overrides?: Partial<RedactionConfig>): RedactionConfig;
+/**
+ * Apply redaction to an evaluation trace.
+ *
+ * Processes tool call inputs and outputs, event data, and search terms.
+ * Returns a new trace (does not mutate the original).
+ */
+export declare function redactTrace(trace: EvalTrace, config?: RedactionConfig): RedactionResult;

package/dist/pipeline/compiler/telemetry/redactor.js

@@ -0,0 +1,222 @@
+/**
+ * Redaction pipeline — strips sensitive data from traces before storage.
+ *
+ * Applied before ANY storage (both blob and Content Lake). Configurable
+ * patterns handle Bearer tokens, API keys, Sanity tokens, and other
+ * common secret formats.
+ *
+ * Principles:
+ * 1. Redact before store — sensitive data never reaches storage
+ * 2. Configurable patterns — teams can add project-specific rules
+ * 3. Truncation for cost — large outputs truncated to max bytes
+ * 4. No PII by default — tasks shouldn't contain PII, this is a safety net
+ *
+ * @see docs/design-docs/architecture-overhaul/observability-telemetry.md
+ */
+// ---------------------------------------------------------------------------
+// Default rules
+// ---------------------------------------------------------------------------
+/** Built-in redaction rules for common secret patterns */
+export const DEFAULT_REDACTION_RULES = [
+    {
+        name: "bearer_tokens",
+        pattern: /Bearer\s+[A-Za-z0-9._~+/=-]{10,}/g,
+        replacement: "Bearer [REDACTED]",
+    },
+    {
+        name: "sanity_tokens",
+        pattern: /sk[A-Za-z0-9]{30,}/g,
+        replacement: "[REDACTED_SANITY_TOKEN]",
+    },
+    {
+        name: "openai_keys",
+        pattern: /sk-[A-Za-z0-9_-]{20,}/g,
+        replacement: "[REDACTED_OPENAI_KEY]",
+    },
+    {
+        name: "api_key_values",
+        pattern: /((?:api[_-]?key|token|secret|password|authorization)\s*[:=]\s*)(["']?)(?!\[REDACTED)[^\s"']{8,}\2/gi,
+        replacement: "$1$2[REDACTED]$2",
+    },
+    {
+        name: "slack_tokens",
+        pattern: /xoxb-[A-Za-z0-9-]{20,}/g,
+        replacement: "[REDACTED_SLACK_TOKEN]",
+    },
+    {
+        name: "github_tokens",
+        pattern: /gh[ps]_[A-Za-z0-9]{30,}/g,
+        replacement: "[REDACTED_GITHUB_TOKEN]",
+    },
+    {
+        name: "anthropic_keys",
+        pattern: /sk-ant-[A-Za-z0-9_-]{20,}/g,
+        replacement: "[REDACTED_ANTHROPIC_KEY]",
+    },
+    {
+        name: "base64_credentials",
+        pattern: /Basic\s+[A-Za-z0-9+/=]{20,}/g,
+        replacement: "Basic [REDACTED]",
+    },
+];
+/** Default fields to omit entirely */
+const DEFAULT_OMIT_FIELDS = [
+    "toolCalls[*].input.headers.Authorization",
+    "toolCalls[*].input.headers.Cookie",
+    "toolCalls[*].input.headers.Set-Cookie",
+];
+const DEFAULT_MAX_OUTPUT_BYTES = 10_240;
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+/**
+ * Create a default redaction config.
+ *
+ * @param overrides - Custom rules or settings to merge
+ */
+export function createRedactionConfig(overrides) {
+    return {
+        rules: overrides?.rules
+            ? [...DEFAULT_REDACTION_RULES, ...overrides.rules]
+            : DEFAULT_REDACTION_RULES,
+        omitFields: overrides?.omitFields
+            ? [...DEFAULT_OMIT_FIELDS, ...overrides.omitFields]
+            : DEFAULT_OMIT_FIELDS,
+        maxOutputBytes: overrides?.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES,
+    };
+}
+/**
+ * Apply redaction to an evaluation trace.
+ *
+ * Processes tool call inputs and outputs, event data, and search terms.
+ * Returns a new trace (does not mutate the original).
+ */
+export function redactTrace(trace, config) {
+    const cfg = config ?? createRedactionConfig();
+    let redactionCount = 0;
+    const rulesApplied = new Set();
+    // Deep clone to avoid mutation
+    const redacted = JSON.parse(JSON.stringify(trace));
+    // Redact tool calls
+    redacted.toolCalls = redacted.toolCalls.map((call) => {
+        const result = redactToolCall(call, cfg);
+        redactionCount += result.count;
+        for (const rule of result.rules)
+            rulesApplied.add(rule);
+        return result.call;
+    });
+    // Redact events
+    redacted.events = redacted.events.map((event) => {
+        const dataStr = JSON.stringify(event.data);
+        const { text, count, rules } = applyRules(dataStr, cfg.rules);
+        redactionCount += count;
+        for (const rule of rules)
+            rulesApplied.add(rule);
+        return { ...event, data: JSON.parse(text) };
+    });
+    // Redact search terms (may contain embedded secrets)
+    redacted.searchTerms = redacted.searchTerms.map((term) => {
+        const { text, count, rules } = applyRules(term, cfg.rules);
+        redactionCount += count;
+        for (const rule of rules)
+            rulesApplied.add(rule);
+        return text;
+    });
+    return {
+        trace: redacted,
+        redactionCount,
+        rulesApplied: [...rulesApplied],
+    };
+}
+// ---------------------------------------------------------------------------
+// Tool call redaction
+// ---------------------------------------------------------------------------
+function redactToolCall(call, config) {
+    let count = 0;
+    const rules = [];
+    // Redact input
+    const inputStr = JSON.stringify(call.input);
+    const inputResult = applyRules(inputStr, config.rules);
+    count += inputResult.count;
+    rules.push(...inputResult.rules);
+    // Redact output
+    let outputStr = JSON.stringify(call.output);
+    // Truncate output if too large
+    if (outputStr.length > config.maxOutputBytes) {
+        outputStr = outputStr.slice(0, config.maxOutputBytes) + "... [truncated]";
+    }
+    const outputResult = applyRules(outputStr, config.rules);
+    count += outputResult.count;
+    rules.push(...outputResult.rules);
+    // Omit specific fields from input
+    let parsedInput = JSON.parse(inputResult.text);
+    parsedInput = omitFields(parsedInput, config.omitFields, "input");
+    return {
+        call: {
+            ...call,
+            input: parsedInput,
+            output: parseJsonSafe(outputResult.text),
+        },
+        count,
+        rules,
+    };
+}
+// ---------------------------------------------------------------------------
+// Rule application
+// ---------------------------------------------------------------------------
+function applyRules(text, rules) {
+    let result = text;
+    let count = 0;
+    const appliedRules = [];
+    for (const rule of rules) {
+        // Reset lastIndex before match() — global regexes are stateful
+        rule.pattern.lastIndex = 0;
+        const matches = result.match(rule.pattern);
+        if (matches && matches.length > 0) {
+            count += matches.length;
+            appliedRules.push(rule.name);
+            // Reset again before replace() — match() may leave lastIndex dirty
+            rule.pattern.lastIndex = 0;
+            result = result.replace(rule.pattern, rule.replacement);
+        }
+    }
+    return { text: result, count, rules: appliedRules };
+}
+// ---------------------------------------------------------------------------
+// Field omission
+// ---------------------------------------------------------------------------
+function omitFields(obj, patterns, context) {
+    for (const pattern of patterns) {
+        // Simple field path handling (not full JSONPath)
+        // Handles: "toolCalls[*].input.headers.Authorization" when context is "input"
+        if (pattern.includes(context)) {
+            const parts = pattern.split(".");
+            const fieldIndex = parts.indexOf(context);
+            if (fieldIndex >= 0) {
+                const remainingPath = parts.slice(fieldIndex + 1);
+                deleteNestedField(obj, remainingPath);
+            }
+        }
+    }
+    return obj;
+}
+function deleteNestedField(obj, path) {
+    if (path.length === 0)
+        return;
+    if (path.length === 1) {
+        delete obj[path[0]];
+        return;
+    }
+    const child = obj[path[0]];
+    if (child && typeof child === "object") {
+        deleteNestedField(child, path.slice(1));
+    }
+}
+function parseJsonSafe(text) {
+    try {
+        return JSON.parse(text);
+    }
+    catch {
+        return text;
+    }
+}

package/dist/pipeline/compiler/telemetry/tool-classifier.d.ts

@@ -0,0 +1,32 @@
+/**
+ * Tool call classification — maps raw provider tool names to categories.
+ *
+ * Raw tool names from providers are noisy and inconsistent (`WebSearch` vs
+ * `web_search` vs `Browser.search`). This module normalizes every tool call
+ * into one of six standard categories for cross-model comparison.
+ *
+ * @see docs/design-docs/architecture-overhaul/observability-telemetry.md
+ */
+import type { ToolCallCategory } from "../../../_vendor/ailf-core/index.d.ts";
+/**
+ * Classify a tool call by its raw name.
+ *
+ * Resolution order:
+ * 1. Exact match in custom overrides (if provided)
+ * 2. Exact match in default tool categories
+ * 3. Heuristic pattern matching on the name
+ * 4. Falls back to "execute" (safest default for unknown tools)
+ *
+ * @param name - Raw tool name from the provider
+ * @param customMappings - Optional custom tool → category overrides
+ * @returns The classified category
+ */
+export declare function classifyToolCall(name: string, customMappings?: Record<string, ToolCallCategory>): ToolCallCategory;
+/**
+ * Classify multiple tool calls, returning the category for each.
+ * Also tracks unrecognized names for the caller to log warnings.
+ */
+export declare function classifyToolCalls(names: string[], customMappings?: Record<string, ToolCallCategory>): {
+    categories: ToolCallCategory[];
+    unrecognized: string[];
+};

package/dist/pipeline/compiler/telemetry/tool-classifier.js

@@ -0,0 +1,120 @@
+/**
+ * Tool call classification — maps raw provider tool names to categories.
+ *
+ * Raw tool names from providers are noisy and inconsistent (`WebSearch` vs
+ * `web_search` vs `Browser.search`). This module normalizes every tool call
+ * into one of six standard categories for cross-model comparison.
+ *
+ * @see docs/design-docs/architecture-overhaul/observability-telemetry.md
+ */
+// ---------------------------------------------------------------------------
+// Default tool name → category mapping
+// ---------------------------------------------------------------------------
+const DEFAULT_TOOL_CATEGORIES = {
+    // Search tools
+    Grep: "search",
+    WebSearch: "search",
+    grep: "search",
+    search: "search",
+    semantic_search: "search",
+    web_search: "search",
+    // Read tools
+    Glob: "read",
+    Read: "read",
+    WebFetch: "read",
+    cat: "read",
+    curl: "read",
+    file_read: "read",
+    read_file: "read",
+    web_fetch: "read",
+    // Write tools
+    Edit: "write",
+    FileEdit: "write",
+    Write: "write",
+    file_write: "write",
+    patch: "write",
+    write_file: "write",
+    // Execute tools
+    Bash: "execute",
+    RunCode: "execute",
+    bash: "execute",
+    exec: "execute",
+    python: "execute",
+    run_code: "execute",
+    shell: "execute",
+    // Navigate tools
+    "Browser.navigate": "navigate",
+    FollowLink: "navigate",
+    browse: "navigate",
+    follow_link: "navigate",
+    navigate: "navigate",
+    open_url: "navigate",
+    // Communicate tools
+    AskUser: "communicate",
+    TodoRead: "communicate",
+    TodoWrite: "communicate",
+    ask_user: "communicate",
+    submit_response: "communicate",
+};
+// ---------------------------------------------------------------------------
+// Heuristic patterns (fallback when name not in lookup table)
+// ---------------------------------------------------------------------------
+const HEURISTIC_PATTERNS = [
+    [/search|find|query|lookup|grep/i, "search"],
+    [/read|fetch|get|load|cat|view/i, "read"],
+    [/write|create|edit|update|patch|save|put|post/i, "write"],
+    [/exec|run|bash|shell|python|code|command/i, "execute"],
+    [/navigate|browse|open|follow|link|url/i, "navigate"],
+    [/ask|user|chat|message|submit|todo|response/i, "communicate"],
+];
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+/**
+ * Classify a tool call by its raw name.
+ *
+ * Resolution order:
+ * 1. Exact match in custom overrides (if provided)
+ * 2. Exact match in default tool categories
+ * 3. Heuristic pattern matching on the name
+ * 4. Falls back to "execute" (safest default for unknown tools)
+ *
+ * @param name - Raw tool name from the provider
+ * @param customMappings - Optional custom tool → category overrides
+ * @returns The classified category
+ */
+export function classifyToolCall(name, customMappings) {
+    // 1. Custom overrides
+    if (customMappings?.[name]) {
+        return customMappings[name];
+    }
+    // 2. Default lookup
+    if (DEFAULT_TOOL_CATEGORIES[name]) {
+        return DEFAULT_TOOL_CATEGORIES[name];
+    }
+    // 3. Heuristic matching
+    for (const [pattern, category] of HEURISTIC_PATTERNS) {
+        if (pattern.test(name)) {
+            return category;
+        }
+    }
+    // 4. Unknown → execute (safest default)
+    return "execute";
+}
+/**
+ * Classify multiple tool calls, returning the category for each.
+ * Also tracks unrecognized names for the caller to log warnings.
+ */
+export function classifyToolCalls(names, customMappings) {
+    const categories = [];
+    const unrecognized = [];
+    for (const name of names) {
+        const category = classifyToolCall(name, customMappings);
+        categories.push(category);
+        // Track names that required heuristic or default fallback
+        if (!DEFAULT_TOOL_CATEGORIES[name] && !customMappings?.[name]) {
+            unrecognized.push(name);
+        }
+    }
+    return { categories, unrecognized };
+}

package/dist/pipeline/compiler/telemetry/trace-collector.d.ts

@@ -0,0 +1,75 @@
+/**
+ * TraceCollector — extracts structured trace data from provider responses.
+ *
+ * Parses tool calls, token usage, and timing data from Promptfoo result
+ * objects and normalizes them into the canonical `EvalTrace` shape.
+ *
+ * Works via inline extraction — parsing provider response metadata
+ * directly, without requiring additional infrastructure.
+ *
+ * @see docs/design-docs/architecture-overhaul/observability-telemetry.md
+ * @see packages/core/src/types/trace.ts — EvalTrace types
+ */
+import type { EvalTrace, ToolCallCategory } from "../../../_vendor/ailf-core/index.d.ts";
+/** Raw provider response shape (subset of Promptfoo's result object) */
+export interface ProviderResponse {
+    /** Raw text output */
+    output?: string;
+    /** Token usage (varies by provider) */
+    tokenUsage?: {
+        completion?: number;
+        prompt?: number;
+        total?: number;
+        cached?: number;
+    };
+    /** Provider-specific metadata (e.g., Claude's toolCalls) */
+    metadata?: {
+        toolCalls?: RawToolCall[];
+        [key: string]: unknown;
+    };
+    /** Response latency in milliseconds */
+    latencyMs?: number;
+}
+/** Raw tool call from a provider (pre-normalization) */
+export interface RawToolCall {
+    name?: string;
+    input?: Record<string, unknown>;
+    output?: unknown;
+    error?: string;
+    durationMs?: number;
+    /** Alternative field names used by some providers */
+    function?: {
+        name?: string;
+        arguments?: string;
+    };
+    type?: string;
+}
+/** Options for trace collection */
+export interface TraceCollectorOptions {
+    /** Run ID to associate with this trace */
+    runId: string;
+    /** Task ID that produced this test case */
+    taskId: string;
+    /** Test case index within the task */
+    testCaseIndex: number;
+    /** Model under evaluation */
+    modelId: string;
+    /** Custom tool → category mappings */
+    toolCategories?: Record<string, ToolCallCategory>;
+    /** Maximum output size per tool call (bytes) */
+    maxOutputBytes?: number;
+}
+/**
+ * Collect a trace from a single provider response.
+ *
+ * Extracts tool calls, token usage, timing, and builds the
+ * chronological event log.
+ */
+export declare function collectTrace(response: ProviderResponse, options: TraceCollectorOptions): EvalTrace;
+/**
+ * Merge multiple per-turn traces into a single test case trace.
+ *
+ * Each turn produces its own trace. This function combines them into
+ * a parent trace with per-turn spans.
+ */
+export declare function mergeTraces(turns: EvalTrace[], parentOptions: TraceCollectorOptions): EvalTrace;