@sanity/ailf 0.5.0 → 2.0.0

package/dist/pipeline/compiler/telemetry/redactor.js

@@ -0,0 +1,222 @@
+/**
+ * Redaction pipeline — strips sensitive data from traces before storage.
+ *
+ * Applied before ANY storage (both blob and Content Lake). Configurable
+ * patterns handle Bearer tokens, API keys, Sanity tokens, and other
+ * common secret formats.
+ *
+ * Principles:
+ * 1. Redact before store — sensitive data never reaches storage
+ * 2. Configurable patterns — teams can add project-specific rules
+ * 3. Truncation for cost — large outputs truncated to max bytes
+ * 4. No PII by default — tasks shouldn't contain PII, this is a safety net
+ *
+ * @see docs/design-docs/architecture-overhaul/observability-telemetry.md
+ */
+// ---------------------------------------------------------------------------
+// Default rules
+// ---------------------------------------------------------------------------
+/** Built-in redaction rules for common secret patterns */
+export const DEFAULT_REDACTION_RULES = [
+    {
+        name: "bearer_tokens",
+        pattern: /Bearer\s+[A-Za-z0-9._~+/=-]{10,}/g,
+        replacement: "Bearer [REDACTED]",
+    },
+    {
+        name: "sanity_tokens",
+        pattern: /sk[A-Za-z0-9]{30,}/g,
+        replacement: "[REDACTED_SANITY_TOKEN]",
+    },
+    {
+        name: "openai_keys",
+        pattern: /sk-[A-Za-z0-9_-]{20,}/g,
+        replacement: "[REDACTED_OPENAI_KEY]",
+    },
+    {
+        name: "api_key_values",
+        pattern: /((?:api[_-]?key|token|secret|password|authorization)\s*[:=]\s*)(["']?)(?!\[REDACTED)[^\s"']{8,}\2/gi,
+        replacement: "$1$2[REDACTED]$2",
+    },
+    {
+        name: "slack_tokens",
+        pattern: /xoxb-[A-Za-z0-9-]{20,}/g,
+        replacement: "[REDACTED_SLACK_TOKEN]",
+    },
+    {
+        name: "github_tokens",
+        pattern: /gh[ps]_[A-Za-z0-9]{30,}/g,
+        replacement: "[REDACTED_GITHUB_TOKEN]",
+    },
+    {
+        name: "anthropic_keys",
+        pattern: /sk-ant-[A-Za-z0-9_-]{20,}/g,
+        replacement: "[REDACTED_ANTHROPIC_KEY]",
+    },
+    {
+        name: "base64_credentials",
+        pattern: /Basic\s+[A-Za-z0-9+/=]{20,}/g,
+        replacement: "Basic [REDACTED]",
+    },
+];
+/** Default fields to omit entirely */
+const DEFAULT_OMIT_FIELDS = [
+    "toolCalls[*].input.headers.Authorization",
+    "toolCalls[*].input.headers.Cookie",
+    "toolCalls[*].input.headers.Set-Cookie",
+];
+const DEFAULT_MAX_OUTPUT_BYTES = 10_240;
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+/**
+ * Create a default redaction config.
+ *
+ * @param overrides - Custom rules or settings to merge
+ */
+export function createRedactionConfig(overrides) {
+    return {
+        rules: overrides?.rules
+            ? [...DEFAULT_REDACTION_RULES, ...overrides.rules]
+            : DEFAULT_REDACTION_RULES,
+        omitFields: overrides?.omitFields
+            ? [...DEFAULT_OMIT_FIELDS, ...overrides.omitFields]
+            : DEFAULT_OMIT_FIELDS,
+        maxOutputBytes: overrides?.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES,
+    };
+}
+/**
+ * Apply redaction to an evaluation trace.
+ *
+ * Processes tool call inputs and outputs, event data, and search terms.
+ * Returns a new trace (does not mutate the original).
+ */
+export function redactTrace(trace, config) {
+    const cfg = config ?? createRedactionConfig();
+    let redactionCount = 0;
+    const rulesApplied = new Set();
+    // Deep clone to avoid mutation
+    const redacted = JSON.parse(JSON.stringify(trace));
+    // Redact tool calls
+    redacted.toolCalls = redacted.toolCalls.map((call) => {
+        const result = redactToolCall(call, cfg);
+        redactionCount += result.count;
+        for (const rule of result.rules)
+            rulesApplied.add(rule);
+        return result.call;
+    });
+    // Redact events
+    redacted.events = redacted.events.map((event) => {
+        const dataStr = JSON.stringify(event.data);
+        const { text, count, rules } = applyRules(dataStr, cfg.rules);
+        redactionCount += count;
+        for (const rule of rules)
+            rulesApplied.add(rule);
+        return { ...event, data: JSON.parse(text) };
+    });
+    // Redact search terms (may contain embedded secrets)
+    redacted.searchTerms = redacted.searchTerms.map((term) => {
+        const { text, count, rules } = applyRules(term, cfg.rules);
+        redactionCount += count;
+        for (const rule of rules)
+            rulesApplied.add(rule);
+        return text;
+    });
+    return {
+        trace: redacted,
+        redactionCount,
+        rulesApplied: [...rulesApplied],
+    };
+}
+// ---------------------------------------------------------------------------
+// Tool call redaction
+// ---------------------------------------------------------------------------
+function redactToolCall(call, config) {
+    let count = 0;
+    const rules = [];
+    // Redact input
+    const inputStr = JSON.stringify(call.input);
+    const inputResult = applyRules(inputStr, config.rules);
+    count += inputResult.count;
+    rules.push(...inputResult.rules);
+    // Redact output
+    let outputStr = JSON.stringify(call.output);
+    // Truncate output if too large
+    if (outputStr.length > config.maxOutputBytes) {
+        outputStr = outputStr.slice(0, config.maxOutputBytes) + "... [truncated]";
+    }
+    const outputResult = applyRules(outputStr, config.rules);
+    count += outputResult.count;
+    rules.push(...outputResult.rules);
+    // Omit specific fields from input
+    let parsedInput = JSON.parse(inputResult.text);
+    parsedInput = omitFields(parsedInput, config.omitFields, "input");
+    return {
+        call: {
+            ...call,
+            input: parsedInput,
+            output: parseJsonSafe(outputResult.text),
+        },
+        count,
+        rules,
+    };
+}
+// ---------------------------------------------------------------------------
+// Rule application
+// ---------------------------------------------------------------------------
+function applyRules(text, rules) {
+    let result = text;
+    let count = 0;
+    const appliedRules = [];
+    for (const rule of rules) {
+        // Reset lastIndex before match() — global regexes are stateful
+        rule.pattern.lastIndex = 0;
+        const matches = result.match(rule.pattern);
+        if (matches && matches.length > 0) {
+            count += matches.length;
+            appliedRules.push(rule.name);
+            // Reset again before replace() — match() may leave lastIndex dirty
+            rule.pattern.lastIndex = 0;
+            result = result.replace(rule.pattern, rule.replacement);
+        }
+    }
+    return { text: result, count, rules: appliedRules };
+}
+// ---------------------------------------------------------------------------
+// Field omission
+// ---------------------------------------------------------------------------
+function omitFields(obj, patterns, context) {
+    for (const pattern of patterns) {
+        // Simple field path handling (not full JSONPath)
+        // Handles: "toolCalls[*].input.headers.Authorization" when context is "input"
+        if (pattern.includes(context)) {
+            const parts = pattern.split(".");
+            const fieldIndex = parts.indexOf(context);
+            if (fieldIndex >= 0) {
+                const remainingPath = parts.slice(fieldIndex + 1);
+                deleteNestedField(obj, remainingPath);
+            }
+        }
+    }
+    return obj;
+}
+function deleteNestedField(obj, path) {
+    if (path.length === 0)
+        return;
+    if (path.length === 1) {
+        delete obj[path[0]];
+        return;
+    }
+    const child = obj[path[0]];
+    if (child && typeof child === "object") {
+        deleteNestedField(child, path.slice(1));
+    }
+}
+function parseJsonSafe(text) {
+    try {
+        return JSON.parse(text);
+    }
+    catch {
+        return text;
+    }
+}

package/dist/pipeline/compiler/telemetry/tool-classifier.d.ts

@@ -0,0 +1,32 @@
+/**
+ * Tool call classification — maps raw provider tool names to categories.
+ *
+ * Raw tool names from providers are noisy and inconsistent (`WebSearch` vs
+ * `web_search` vs `Browser.search`). This module normalizes every tool call
+ * into one of six standard categories for cross-model comparison.
+ *
+ * @see docs/design-docs/architecture-overhaul/observability-telemetry.md
+ */
+import type { ToolCallCategory } from "../../../_vendor/ailf-core/index.d.ts";
+/**
+ * Classify a tool call by its raw name.
+ *
+ * Resolution order:
+ * 1. Exact match in custom overrides (if provided)
+ * 2. Exact match in default tool categories
+ * 3. Heuristic pattern matching on the name
+ * 4. Falls back to "execute" (safest default for unknown tools)
+ *
+ * @param name - Raw tool name from the provider
+ * @param customMappings - Optional custom tool → category overrides
+ * @returns The classified category
+ */
+export declare function classifyToolCall(name: string, customMappings?: Record<string, ToolCallCategory>): ToolCallCategory;
+/**
+ * Classify multiple tool calls, returning the category for each.
+ * Also tracks unrecognized names for the caller to log warnings.
+ */
+export declare function classifyToolCalls(names: string[], customMappings?: Record<string, ToolCallCategory>): {
+    categories: ToolCallCategory[];
+    unrecognized: string[];
+};

package/dist/pipeline/compiler/telemetry/tool-classifier.js

@@ -0,0 +1,120 @@
+/**
+ * Tool call classification — maps raw provider tool names to categories.
+ *
+ * Raw tool names from providers are noisy and inconsistent (`WebSearch` vs
+ * `web_search` vs `Browser.search`). This module normalizes every tool call
+ * into one of six standard categories for cross-model comparison.
+ *
+ * @see docs/design-docs/architecture-overhaul/observability-telemetry.md
+ */
+// ---------------------------------------------------------------------------
+// Default tool name → category mapping
+// ---------------------------------------------------------------------------
+const DEFAULT_TOOL_CATEGORIES = {
+    // Search tools
+    Grep: "search",
+    WebSearch: "search",
+    grep: "search",
+    search: "search",
+    semantic_search: "search",
+    web_search: "search",
+    // Read tools
+    Glob: "read",
+    Read: "read",
+    WebFetch: "read",
+    cat: "read",
+    curl: "read",
+    file_read: "read",
+    read_file: "read",
+    web_fetch: "read",
+    // Write tools
+    Edit: "write",
+    FileEdit: "write",
+    Write: "write",
+    file_write: "write",
+    patch: "write",
+    write_file: "write",
+    // Execute tools
+    Bash: "execute",
+    RunCode: "execute",
+    bash: "execute",
+    exec: "execute",
+    python: "execute",
+    run_code: "execute",
+    shell: "execute",
+    // Navigate tools
+    "Browser.navigate": "navigate",
+    FollowLink: "navigate",
+    browse: "navigate",
+    follow_link: "navigate",
+    navigate: "navigate",
+    open_url: "navigate",
+    // Communicate tools
+    AskUser: "communicate",
+    TodoRead: "communicate",
+    TodoWrite: "communicate",
+    ask_user: "communicate",
+    submit_response: "communicate",
+};
+// ---------------------------------------------------------------------------
+// Heuristic patterns (fallback when name not in lookup table)
+// ---------------------------------------------------------------------------
+const HEURISTIC_PATTERNS = [
+    [/search|find|query|lookup|grep/i, "search"],
+    [/read|fetch|get|load|cat|view/i, "read"],
+    [/write|create|edit|update|patch|save|put|post/i, "write"],
+    [/exec|run|bash|shell|python|code|command/i, "execute"],
+    [/navigate|browse|open|follow|link|url/i, "navigate"],
+    [/ask|user|chat|message|submit|todo|response/i, "communicate"],
+];
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+/**
+ * Classify a tool call by its raw name.
+ *
+ * Resolution order:
+ * 1. Exact match in custom overrides (if provided)
+ * 2. Exact match in default tool categories
+ * 3. Heuristic pattern matching on the name
+ * 4. Falls back to "execute" (safest default for unknown tools)
+ *
+ * @param name - Raw tool name from the provider
+ * @param customMappings - Optional custom tool → category overrides
+ * @returns The classified category
+ */
+export function classifyToolCall(name, customMappings) {
+    // 1. Custom overrides
+    if (customMappings?.[name]) {
+        return customMappings[name];
+    }
+    // 2. Default lookup
+    if (DEFAULT_TOOL_CATEGORIES[name]) {
+        return DEFAULT_TOOL_CATEGORIES[name];
+    }
+    // 3. Heuristic matching
+    for (const [pattern, category] of HEURISTIC_PATTERNS) {
+        if (pattern.test(name)) {
+            return category;
+        }
+    }
+    // 4. Unknown → execute (safest default)
+    return "execute";
+}
+/**
+ * Classify multiple tool calls, returning the category for each.
+ * Also tracks unrecognized names for the caller to log warnings.
+ */
+export function classifyToolCalls(names, customMappings) {
+    const categories = [];
+    const unrecognized = [];
+    for (const name of names) {
+        const category = classifyToolCall(name, customMappings);
+        categories.push(category);
+        // Track names that required heuristic or default fallback
+        if (!DEFAULT_TOOL_CATEGORIES[name] && !customMappings?.[name]) {
+            unrecognized.push(name);
+        }
+    }
+    return { categories, unrecognized };
+}

package/dist/pipeline/compiler/telemetry/trace-collector.d.ts

@@ -0,0 +1,75 @@
+/**
+ * TraceCollector — extracts structured trace data from provider responses.
+ *
+ * Parses tool calls, token usage, and timing data from Promptfoo result
+ * objects and normalizes them into the canonical `EvalTrace` shape.
+ *
+ * Works via inline extraction — parsing provider response metadata
+ * directly, without requiring additional infrastructure.
+ *
+ * @see docs/design-docs/architecture-overhaul/observability-telemetry.md
+ * @see packages/core/src/types/trace.ts — EvalTrace types
+ */
+import type { EvalTrace, ToolCallCategory } from "../../../_vendor/ailf-core/index.d.ts";
+/** Raw provider response shape (subset of Promptfoo's result object) */
+export interface ProviderResponse {
+    /** Raw text output */
+    output?: string;
+    /** Token usage (varies by provider) */
+    tokenUsage?: {
+        completion?: number;
+        prompt?: number;
+        total?: number;
+        cached?: number;
+    };
+    /** Provider-specific metadata (e.g., Claude's toolCalls) */
+    metadata?: {
+        toolCalls?: RawToolCall[];
+        [key: string]: unknown;
+    };
+    /** Response latency in milliseconds */
+    latencyMs?: number;
+}
+/** Raw tool call from a provider (pre-normalization) */
+export interface RawToolCall {
+    name?: string;
+    input?: Record<string, unknown>;
+    output?: unknown;
+    error?: string;
+    durationMs?: number;
+    /** Alternative field names used by some providers */
+    function?: {
+        name?: string;
+        arguments?: string;
+    };
+    type?: string;
+}
+/** Options for trace collection */
+export interface TraceCollectorOptions {
+    /** Run ID to associate with this trace */
+    runId: string;
+    /** Task ID that produced this test case */
+    taskId: string;
+    /** Test case index within the task */
+    testCaseIndex: number;
+    /** Model under evaluation */
+    modelId: string;
+    /** Custom tool → category mappings */
+    toolCategories?: Record<string, ToolCallCategory>;
+    /** Maximum output size per tool call (bytes) */
+    maxOutputBytes?: number;
+}
+/**
+ * Collect a trace from a single provider response.
+ *
+ * Extracts tool calls, token usage, timing, and builds the
+ * chronological event log.
+ */
+export declare function collectTrace(response: ProviderResponse, options: TraceCollectorOptions): EvalTrace;
+/**
+ * Merge multiple per-turn traces into a single test case trace.
+ *
+ * Each turn produces its own trace. This function combines them into
+ * a parent trace with per-turn spans.
+ */
+export declare function mergeTraces(turns: EvalTrace[], parentOptions: TraceCollectorOptions): EvalTrace;