@sanity/ailf 0.5.0 → 2.0.0

package/dist/pipeline/compiler/telemetry/trace-collector.js

@@ -0,0 +1,297 @@
+/**
+ * TraceCollector — extracts structured trace data from provider responses.
+ *
+ * Parses tool calls, token usage, and timing data from Promptfoo result
+ * objects and normalizes them into the canonical `EvalTrace` shape.
+ *
+ * Works via inline extraction — parsing provider response metadata
+ * directly, without requiring additional infrastructure.
+ *
+ * @see docs/design-docs/architecture-overhaul/observability-telemetry.md
+ * @see packages/core/src/types/trace.ts — EvalTrace types
+ */
+import { classifyToolCall } from "./tool-classifier.js";
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+/**
+ * Collect a trace from a single provider response.
+ *
+ * Extracts tool calls, token usage, timing, and builds the
+ * chronological event log.
+ */
+export function collectTrace(response, options) {
+    const now = new Date().toISOString();
+    const traceId = `trace-${options.runId}-${options.taskId}-${options.testCaseIndex}`;
+    // Extract tool calls
+    const toolCalls = extractToolCalls(response, options.toolCategories, options.maxOutputBytes ?? 10_240);
+    // Extract token usage
+    const tokensUsed = extractTokenUsage(response);
+    // Compute cost (caller provides pricing separately)
+    const durationMs = response.latencyMs ?? 0;
+    // Build event log
+    const events = buildEventLog(toolCalls, tokensUsed, durationMs, now);
+    // Build spans — one root span for the test case
+    const spans = buildSpans(options, durationMs);
+    // Extract URLs and search terms from tool calls
+    const { urlsVisited, searchTerms, filesRead, filesWritten } = extractAccessPatterns(toolCalls);
+    return {
+        traceId,
+        runId: options.runId,
+        taskId: options.taskId,
+        testCaseIndex: options.testCaseIndex,
+        modelId: options.modelId,
+        spans,
+        toolCalls,
+        urlsVisited,
+        searchTerms,
+        filesRead,
+        filesWritten,
+        tokensUsed,
+        costEstimate: 0, // Set by cost calculator separately
+        durationMs,
+        events,
+        startedAt: now,
+        completedAt: now,
+    };
+}
+/**
+ * Merge multiple per-turn traces into a single test case trace.
+ *
+ * Each turn produces its own trace. This function combines them into
+ * a parent trace with per-turn spans.
+ */
+export function mergeTraces(turns, parentOptions) {
+    if (turns.length === 0) {
+        return collectTrace({}, parentOptions);
+    }
+    const first = turns[0];
+    const last = turns[turns.length - 1];
+    const traceId = `trace-${parentOptions.runId}-${parentOptions.taskId}-${parentOptions.testCaseIndex}`;
+    // Merge all tool calls, events, and access patterns
+    const toolCalls = turns.flatMap((t) => t.toolCalls);
+    const events = turns.flatMap((t) => t.events);
+    const urlsVisited = [...new Set(turns.flatMap((t) => t.urlsVisited))];
+    const searchTerms = [...new Set(turns.flatMap((t) => t.searchTerms))];
+    const filesRead = [...new Set(turns.flatMap((t) => t.filesRead))];
+    const filesWritten = [...new Set(turns.flatMap((t) => t.filesWritten))];
+    // Aggregate token usage
+    const tokensUsed = {
+        promptTokens: turns.reduce((sum, t) => sum + t.tokensUsed.promptTokens, 0),
+        completionTokens: turns.reduce((sum, t) => sum + t.tokensUsed.completionTokens, 0),
+        totalTokens: turns.reduce((sum, t) => sum + t.tokensUsed.totalTokens, 0),
+        toolTokens: turns.reduce((sum, t) => sum + (t.tokensUsed.toolTokens ?? 0), 0),
+    };
+    // Build per-turn spans
+    const spans = [
+        {
+            spanId: `${traceId}-root`,
+            parentSpanId: null,
+            operation: "test-case",
+            startMs: 0,
+            endMs: turns.reduce((sum, t) => sum + t.durationMs, 0),
+            attributes: { turnCount: turns.length },
+        },
+        ...turns.map((turn, i) => ({
+            spanId: `${traceId}-turn-${i}`,
+            parentSpanId: `${traceId}-root`,
+            operation: `turn-${i}`,
+            startMs: turns.slice(0, i).reduce((sum, t) => sum + t.durationMs, 0),
+            endMs: turns.slice(0, i + 1).reduce((sum, t) => sum + t.durationMs, 0),
+            attributes: {
+                modelId: turn.modelId,
+                toolCallCount: turn.toolCalls.length,
+            },
+        })),
+    ];
+    return {
+        traceId,
+        runId: parentOptions.runId,
+        taskId: parentOptions.taskId,
+        testCaseIndex: parentOptions.testCaseIndex,
+        modelId: parentOptions.modelId,
+        spans,
+        toolCalls,
+        urlsVisited,
+        searchTerms,
+        filesRead,
+        filesWritten,
+        tokensUsed,
+        costEstimate: turns.reduce((sum, t) => sum + t.costEstimate, 0),
+        durationMs: turns.reduce((sum, t) => sum + t.durationMs, 0),
+        events,
+        startedAt: first.startedAt,
+        completedAt: last.completedAt,
+    };
+}
+// ---------------------------------------------------------------------------
+// Tool call extraction
+// ---------------------------------------------------------------------------
+function extractToolCalls(response, customCategories, maxOutputBytes) {
+    const rawCalls = response.metadata?.toolCalls ?? [];
+    return rawCalls.map((raw) => {
+        const name = raw.name ?? raw.function?.name ?? "unknown";
+        const input = raw.input ?? parseJsonSafe(raw.function?.arguments) ?? {};
+        let output = raw.output;
+        // Truncate output if too large
+        if (maxOutputBytes && output) {
+            const serialized = JSON.stringify(output);
+            if (serialized.length > maxOutputBytes) {
+                output = serialized.slice(0, maxOutputBytes) + "... [truncated]";
+            }
+        }
+        return {
+            name,
+            input: input,
+            output,
+            durationMs: raw.durationMs ?? 0,
+            error: raw.error,
+            category: classifyToolCall(name, customCategories),
+        };
+    });
+}
+// ---------------------------------------------------------------------------
+// Token usage extraction
+// ---------------------------------------------------------------------------
+function extractTokenUsage(response) {
+    const usage = response.tokenUsage;
+    if (!usage) {
+        return {
+            promptTokens: 0,
+            completionTokens: 0,
+            totalTokens: 0,
+        };
+    }
+    const prompt = usage.prompt ?? 0;
+    const completion = usage.completion ?? 0;
+    return {
+        promptTokens: prompt,
+        completionTokens: completion,
+        totalTokens: usage.total ?? prompt + completion,
+        ...(usage.cached ? { toolTokens: usage.cached } : {}),
+    };
+}
+// ---------------------------------------------------------------------------
+// Access pattern extraction
+// ---------------------------------------------------------------------------
+function extractAccessPatterns(toolCalls) {
+    const urlsVisited = new Set();
+    const searchTerms = new Set();
+    const filesRead = new Set();
+    const filesWritten = new Set();
+    for (const call of toolCalls) {
+        switch (call.category) {
+            case "navigate":
+            case "read": {
+                const url = extractString(call.input, "url");
+                if (url)
+                    urlsVisited.add(url);
+                const filePath = extractString(call.input, "path", "file", "filename");
+                if (filePath) {
+                    if (call.category === "read")
+                        filesRead.add(filePath);
+                }
+                break;
+            }
+            case "search": {
+                const query = extractString(call.input, "query", "term", "search");
+                if (query)
+                    searchTerms.add(query);
+                break;
+            }
+            case "write": {
+                const writePath = extractString(call.input, "path", "file", "filename");
+                if (writePath)
+                    filesWritten.add(writePath);
+                break;
+            }
+        }
+    }
+    return {
+        urlsVisited: [...urlsVisited],
+        searchTerms: [...searchTerms],
+        filesRead: [...filesRead],
+        filesWritten: [...filesWritten],
+    };
+}
+// ---------------------------------------------------------------------------
+// Event log
+// ---------------------------------------------------------------------------
+function buildEventLog(toolCalls, tokensUsed, durationMs, timestamp) {
+    const events = [];
+    // LLM request event
+    events.push({
+        timestamp,
+        type: "llm_request",
+        data: { promptTokens: tokensUsed.promptTokens },
+    });
+    // Tool call events
+    for (const call of toolCalls) {
+        events.push({
+            timestamp,
+            type: "tool_call_start",
+            data: { name: call.name, category: call.category },
+        });
+        events.push({
+            timestamp,
+            type: "tool_call_end",
+            data: {
+                name: call.name,
+                durationMs: call.durationMs,
+                ...(call.error ? { error: call.error } : {}),
+            },
+        });
+    }
+    // LLM response event
+    events.push({
+        timestamp,
+        type: "llm_response",
+        data: {
+            completionTokens: tokensUsed.completionTokens,
+            totalTokens: tokensUsed.totalTokens,
+            durationMs,
+        },
+    });
+    return events;
+}
+// ---------------------------------------------------------------------------
+// Span building
+// ---------------------------------------------------------------------------
+function buildSpans(options, durationMs) {
+    const traceId = `trace-${options.runId}-${options.taskId}-${options.testCaseIndex}`;
+    return [
+        {
+            spanId: `${traceId}-root`,
+            parentSpanId: null,
+            operation: "test-case",
+            startMs: 0,
+            endMs: durationMs,
+            attributes: {
+                taskId: options.taskId,
+                modelId: options.modelId,
+                testCaseIndex: options.testCaseIndex,
+            },
+        },
+    ];
+}
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+function parseJsonSafe(str) {
+    if (!str)
+        return null;
+    try {
+        return JSON.parse(str);
+    }
+    catch {
+        return null;
+    }
+}
+function extractString(input, ...keys) {
+    for (const key of keys) {
+        const val = input[key];
+        if (typeof val === "string" && val.length > 0)
+            return val;
+    }
+    return undefined;
+}

package/dist/pipeline/compiler/telemetry/trace-store.d.ts

@@ -0,0 +1,78 @@
+/**
+ * Trace storage — port interface for persisting evaluation traces.
+ *
+ * Full trace payloads go to blob storage (GCP Cloud Storage or local
+ * filesystem). Content Lake documents store only sanitized summaries
+ * with a `traceDataUri` pointing to the full payload.
+ *
+ * CRITICAL SECURITY: The `next` dataset is publicly accessible. Full
+ * trace data (which may contain tool call arguments with sensitive data)
+ * MUST go to blob storage. Only sanitized summaries in Content Lake.
+ *
+ * @see docs/design-docs/architecture-overhaul/observability-telemetry.md
+ * @see docs/design-docs/architecture-overhaul/storage-schema.md
+ */
+import type { EvalTrace } from "../../../_vendor/ailf-core/index.d.ts";
+/** Result of storing a trace */
+export interface TraceStoreResult {
+    /** URI pointing to the stored trace (blob:// or file://) */
+    uri: string;
+    /** Content hash for integrity verification */
+    contentHash: string;
+    /** Size in bytes */
+    sizeBytes: number;
+}
+/** Sanitized trace summary for Content Lake storage */
+export interface TraceSummary {
+    /** Trace ID */
+    traceId: string;
+    /** URI to the full trace payload */
+    traceDataUri: string;
+    /** Task ID */
+    taskId: string;
+    /** Model ID */
+    modelId: string;
+    /** Number of tool calls */
+    toolCallCount: number;
+    /** Tool call category breakdown */
+    toolCallCategories: Record<string, number>;
+    /** Total tokens used */
+    totalTokens: number;
+    /** Estimated cost in USD */
+    costEstimate: number;
+    /** Duration in milliseconds */
+    durationMs: number;
+    /** URLs visited count */
+    urlsVisitedCount: number;
+    /** Files read count */
+    filesReadCount: number;
+    /** Files written count */
+    filesWrittenCount: number;
+}
+/**
+ * Port interface for trace blob storage.
+ *
+ * Implementations:
+ * - LocalTraceStore — writes to local filesystem (development/CI)
+ * - GCPTraceStore — writes to GCP Cloud Storage (production)
+ */
+export interface TraceStore {
+    /** Store a full trace payload */
+    store(trace: EvalTrace): Promise<TraceStoreResult>;
+    /** Retrieve a trace by URI */
+    retrieve(uri: string): Promise<EvalTrace | null>;
+}
+/** Local filesystem trace store — for development and CI */
+export declare class LocalTraceStore implements TraceStore {
+    private readonly baseDir;
+    constructor(baseDir: string);
+    store(trace: EvalTrace): Promise<TraceStoreResult>;
+    retrieve(uri: string): Promise<EvalTrace | null>;
+}
+/**
+ * Extract a sanitized summary from a full trace.
+ *
+ * This summary is safe for Content Lake storage (no tool call arguments,
+ * no file contents, no potentially sensitive data).
+ */
+export declare function extractTraceSummary(trace: EvalTrace, traceDataUri: string): TraceSummary;

package/dist/pipeline/compiler/telemetry/trace-store.js

@@ -0,0 +1,85 @@
+/**
+ * Trace storage — port interface for persisting evaluation traces.
+ *
+ * Full trace payloads go to blob storage (GCP Cloud Storage or local
+ * filesystem). Content Lake documents store only sanitized summaries
+ * with a `traceDataUri` pointing to the full payload.
+ *
+ * CRITICAL SECURITY: The `next` dataset is publicly accessible. Full
+ * trace data (which may contain tool call arguments with sensitive data)
+ * MUST go to blob storage. Only sanitized summaries in Content Lake.
+ *
+ * @see docs/design-docs/architecture-overhaul/observability-telemetry.md
+ * @see docs/design-docs/architecture-overhaul/storage-schema.md
+ */
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
+import { resolve } from "path";
+import { simpleHash } from "../hash.js";
+// ---------------------------------------------------------------------------
+// Local filesystem implementation
+// ---------------------------------------------------------------------------
+/** Local filesystem trace store — for development and CI */
+export class LocalTraceStore {
+    baseDir;
+    constructor(baseDir) {
+        this.baseDir = baseDir;
+    }
+    async store(trace) {
+        mkdirSync(this.baseDir, { recursive: true });
+        const filename = `${trace.traceId}.json`;
+        const filepath = resolve(this.baseDir, filename);
+        const content = JSON.stringify(trace, null, 2);
+        writeFileSync(filepath, content);
+        return {
+            uri: `file://${filepath}`,
+            contentHash: simpleHash(content),
+            sizeBytes: Buffer.byteLength(content),
+        };
+    }
+    async retrieve(uri) {
+        const filepath = uri.startsWith("file://") ? uri.slice(7) : uri;
+        if (!existsSync(filepath))
+            return null;
+        try {
+            const content = readFileSync(filepath, "utf-8");
+            return JSON.parse(content);
+        }
+        catch {
+            return null;
+        }
+    }
+}
+// ---------------------------------------------------------------------------
+// Summary extraction
+// ---------------------------------------------------------------------------
+/**
+ * Extract a sanitized summary from a full trace.
+ *
+ * This summary is safe for Content Lake storage (no tool call arguments,
+ * no file contents, no potentially sensitive data).
+ */
+export function extractTraceSummary(trace, traceDataUri) {
+    // Count tool calls by category
+    const toolCallCategories = {};
+    for (const call of trace.toolCalls) {
+        toolCallCategories[call.category] =
+            (toolCallCategories[call.category] ?? 0) + 1;
+    }
+    return {
+        traceId: trace.traceId,
+        traceDataUri,
+        taskId: trace.taskId,
+        modelId: trace.modelId,
+        toolCallCount: trace.toolCalls.length,
+        toolCallCategories,
+        totalTokens: trace.tokensUsed.totalTokens,
+        costEstimate: trace.costEstimate,
+        durationMs: trace.durationMs,
+        urlsVisitedCount: trace.urlsVisited.length,
+        filesReadCount: trace.filesRead.length,
+        filesWrittenCount: trace.filesWritten.length,
+    };
+}
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------

package/dist/pipeline/compiler/variable-resolver.d.ts

@@ -0,0 +1,46 @@
+/**
+ * Variable resolver — handles dynamic variable interpolation.
+ *
+ * Resolves dynamic variables ($latest, $env, $now, etc.) in task
+ * configurations. All resolved values are recorded in the
+ * VariableEnvelope with provenance metadata for audit and cache
+ * invalidation.
+ *
+ * Design decision: dynamic variables are opt-in and logged.
+ * A task must explicitly use `$latest` or `$env(VAR)` syntax —
+ * there's no implicit resolution. Every dynamic resolution is
+ * recorded in provenance for reproducibility tracking.
+ *
+ * @see docs/design-docs/architecture-overhaul/domain-model.md (VariableEnvelope)
+ * @see docs/exec-plans/architecture-overhaul/phase-2-config-compiler.md
+ */
+import type { VariableDeclaration, VariableEnvelope } from "../../_vendor/ailf-core/index.d.ts";
+/** Options for variable resolution */
+export interface VariableResolverOptions {
+    /** Environment variables available for $env() resolution */
+    env?: Record<string, string | undefined>;
+    /** Log dynamic resolutions */
+    log?: (message: string) => void;
+}
+/** Result of resolving variables */
+export interface VariableResolutionResult {
+    /** Updated variable envelope with resolved values */
+    envelope: VariableEnvelope;
+    /** Resolution log entries */
+    resolutions: string[];
+    /** Warnings for unresolvable references */
+    warnings: string[];
+}
+/**
+ * Resolve dynamic variables in a VariableEnvelope.
+ *
+ * Scans all string values for dynamic patterns and replaces them
+ * with resolved values. Non-string values are passed through unchanged.
+ */
+export declare function resolveVariables(envelope: VariableEnvelope, options?: VariableResolverOptions): VariableResolutionResult;
+/**
+ * Create a VariableEnvelope from a plain key-value map.
+ *
+ * Convenience function for wrapping raw variables with empty provenance.
+ */
+export declare function createEnvelope(values: Record<string, unknown>, declarations?: VariableDeclaration[]): VariableEnvelope;

package/dist/pipeline/compiler/variable-resolver.js

@@ -0,0 +1,115 @@
+/**
+ * Variable resolver — handles dynamic variable interpolation.
+ *
+ * Resolves dynamic variables ($latest, $env, $now, etc.) in task
+ * configurations. All resolved values are recorded in the
+ * VariableEnvelope with provenance metadata for audit and cache
+ * invalidation.
+ *
+ * Design decision: dynamic variables are opt-in and logged.
+ * A task must explicitly use `$latest` or `$env(VAR)` syntax —
+ * there's no implicit resolution. Every dynamic resolution is
+ * recorded in provenance for reproducibility tracking.
+ *
+ * @see docs/design-docs/architecture-overhaul/domain-model.md (VariableEnvelope)
+ * @see docs/exec-plans/architecture-overhaul/phase-2-config-compiler.md
+ */
+import { simpleHash } from "./hash.js";
+// ---------------------------------------------------------------------------
+// Dynamic variable patterns
+// ---------------------------------------------------------------------------
+/** Pattern for $env(VAR_NAME) references */
+const ENV_PATTERN = /\$env\(([A-Z_][A-Z0-9_]*)\)/g;
+/** Pattern for $latest references (resolved to current timestamp) */
+const LATEST_PATTERN = /\$latest/g;
+/** Pattern for $now references (resolved to ISO timestamp) */
+const NOW_PATTERN = /\$now/g;
+/**
+ * Resolve dynamic variables in a VariableEnvelope.
+ *
+ * Scans all string values for dynamic patterns and replaces them
+ * with resolved values. Non-string values are passed through unchanged.
+ */
+export function resolveVariables(envelope, options) {
+    const resolutions = [];
+    const warnings = [];
+    const resolvedValues = {};
+    const resolvedProvenance = {
+        ...envelope.provenance,
+    };
+    const envVars = options?.env ?? process.env;
+    const log = options?.log ?? (() => { });
+    const now = new Date().toISOString();
+    for (const [key, value] of Object.entries(envelope.values)) {
+        if (typeof value !== "string") {
+            resolvedValues[key] = value;
+            continue;
+        }
+        let resolved = value;
+        let wasResolved = false;
+        // Resolve $env(VAR_NAME)
+        resolved = resolved.replace(ENV_PATTERN, (_match, varName) => {
+            const envValue = envVars[varName];
+            if (envValue === undefined) {
+                warnings.push(`Variable "${key}": $env(${varName}) could not be resolved — ` +
+                    `environment variable "${varName}" is not set`);
+                return _match; // Leave unresolved
+            }
+            wasResolved = true;
+            const msg = `Resolved $env(${varName}) in "${key}"`;
+            resolutions.push(msg);
+            log(msg);
+            return envValue;
+        });
+        // Resolve $latest — alias for $now (same ISO timestamp).
+        // Both exist for readability: $now in "run at $now" vs $latest in
+        // "compare against $latest baseline". Semantically identical.
+        resolved = resolved.replace(LATEST_PATTERN, () => {
+            wasResolved = true;
+            const msg = `Resolved $latest in "${key}" to ${now} (alias for $now)`;
+            resolutions.push(msg);
+            log(msg);
+            return now;
+        });
+        // Resolve $now — current ISO timestamp
+        resolved = resolved.replace(NOW_PATTERN, () => {
+            wasResolved = true;
+            const msg = `Resolved $now in "${key}" to ${now}`;
+            resolutions.push(msg);
+            log(msg);
+            return now;
+        });
+        resolvedValues[key] = resolved;
+        if (wasResolved) {
+            resolvedProvenance[key] = {
+                hash: simpleHash(resolved),
+                resolvedAt: now,
+                source: { expression: value, inputs: [key], type: "derived" },
+            };
+        }
+    }
+    return {
+        envelope: {
+            declarations: envelope.declarations,
+            provenance: resolvedProvenance,
+            values: resolvedValues,
+        },
+        resolutions,
+        warnings,
+    };
+}
+/**
+ * Create a VariableEnvelope from a plain key-value map.
+ *
+ * Convenience function for wrapping raw variables with empty provenance.
+ */
+export function createEnvelope(values, declarations) {
+    return {
+        declarations: declarations ?? [],
+        provenance: {},
+        values,
+    };
+}
+// ---------------------------------------------------------------------------
+// Hash helper (shared with fixture-resolver)
+// ---------------------------------------------------------------------------

package/dist/pipeline/coverage-audit.d.ts

@@ -2,14 +2,14 @@
  * coverage-audit.ts
  *
  * Pure computation functions for cross-referencing the product feature registry
- * (config/features.yaml) against actual task files (tasks/*.yaml)
+ * (config/features) against actual task files (tasks/*.yaml)
  * to produce a documentation coverage audit.
  *
  * Phase 3c of the Scenario Matrix implementation.
  *
  * @see docs/exec-plans/scenario-matrix-implementation/phase-3-gap-analysis.md
  */
-import type { Logger } from "../_vendor/ailf-core/index.d.ts";
+import type { Logger, PluginRegistry } from "../_vendor/ailf-core/index.d.ts";
 import type { CoverageAuditReport, ProductFeature } from "./types.js";
 /**
  * Count unique document slugs referenced across all tasks.
@@ -31,10 +31,20 @@ export declare function formatCoverageConsole(report: CoverageAuditReport): stri
  */
 export declare function formatCoverageMarkdown(report: CoverageAuditReport): string;
 /**
- * Load and validate the feature registry from config/features.yaml.
+ * Load and validate the feature registry.
+ *
+ * Resolution order:
+ *   1. config/features file (user overrides) — if non-empty, wins
+ *   2. Registry features (preset-provided) — fallback when config is empty
+ *   3. null — no features available
  */
-export declare function loadFeatureRegistry(rootDir: string, logger?: Logger): null | ProductFeature[];
+export declare function loadFeatureRegistry(rootDir: string, options?: {
+    logger?: Logger;
+    registry?: PluginRegistry;
+}): null | ProductFeature[];
 /**
  * Run the coverage audit and produce a structured report.
  */
-export declare function runCoverageAudit(rootDir: string): CoverageAuditReport | null;
+export declare function runCoverageAudit(rootDir: string, options?: {
+    registry?: PluginRegistry;
+}): CoverageAuditReport | null;