agent-eval-opencode 0.10.0

package/dist/lib/o11y/parsers/opencode.js

@@ -0,0 +1,320 @@
+/**
+ * Parser for OpenCode CLI transcript format.
+ * OpenCode outputs JSON events to stdout when run with --format json.
+ *
+ * Format reference (based on OpenCode CLI output):
+ * - Events have a "kind" field indicating the event type
+ * - Messages, tool calls, and results are separate events
+ */
+/**
+ * Map OpenCode tool names to canonical names.
+ */
+function normalizeToolName(name) {
+    const toolMap = {
+        // File operations
+        read: 'file_read',
+        read_file: 'file_read',
+        file_read: 'file_read',
+        write: 'file_write',
+        write_file: 'file_write',
+        file_write: 'file_write',
+        create: 'file_write',
+        edit: 'file_edit',
+        edit_file: 'file_edit',
+        file_edit: 'file_edit',
+        patch: 'file_edit',
+        // Shell
+        bash: 'shell',
+        shell: 'shell',
+        exec: 'shell',
+        execute: 'shell',
+        run: 'shell',
+        command: 'shell',
+        // Web
+        fetch: 'web_fetch',
+        http: 'web_fetch',
+        request: 'web_fetch',
+        web_fetch: 'web_fetch',
+        search: 'web_search',
+        web_search: 'web_search',
+        // Search/navigation
+        glob: 'glob',
+        find: 'glob',
+        list: 'glob',
+        grep: 'grep',
+        rg: 'grep',
+        ripgrep: 'grep',
+        ls: 'list_dir',
+        dir: 'list_dir',
+        list_dir: 'list_dir',
+    };
+    return toolMap[name.toLowerCase()] || 'unknown';
+}
+/**
+ * Extract file path from tool arguments.
+ */
+function extractFilePath(args) {
+    return (args.path || args.filePath || args.file || args.filename || args.target);
+}
+/**
+ * Extract URL from tool arguments.
+ */
+function extractUrl(args) {
+    return (args.url || args.uri || args.href || args.endpoint);
+}
+/**
+ * Extract command from tool arguments.
+ */
+function extractCommand(args) {
+    if (typeof args.command === 'string')
+        return args.command;
+    if (typeof args.cmd === 'string')
+        return args.cmd;
+    if (typeof args.script === 'string')
+        return args.script;
+    if (Array.isArray(args.args)) {
+        const program = args.program || args.bin || args.executable || '';
+        return `${program} ${args.args.join(' ')}`.trim();
+    }
+    return undefined;
+}
+/**
+ * Parse a single JSONL line from OpenCode transcript.
+ * Handles the real OpenCode format:
+ * - type: "tool_use" | "text" | "step_start" | "step_finish"
+ * - Tool info in part.tool, part.state.input, part.state.output
+ * - Text in part.text
+ */
+function parseOpenCodeLine(line) {
+    const events = [];
+    try {
+        const data = JSON.parse(line);
+        // OpenCode uses "type" for event type
+        const eventType = data.type || data.kind || data.event;
+        const part = data.part;
+        const state = part?.state;
+        switch (eventType) {
+            // Real OpenCode format: tool_use with part.tool
+            case 'tool_use': {
+                if (part && part.tool) {
+                    const name = part.tool;
+                    const args = state?.input || {};
+                    const output = state?.output;
+                    const status = state?.status;
+                    // Emit tool_call event
+                    events.push({
+                        timestamp: data.timestamp ? new Date(data.timestamp).toISOString() : undefined,
+                        type: 'tool_call',
+                        tool: {
+                            name: normalizeToolName(name),
+                            originalName: name,
+                            args,
+                        },
+                        raw: data,
+                    });
+                    // If completed, also emit tool_result
+                    if (status === 'completed' && output !== undefined) {
+                        const metadata = state?.metadata;
+                        const exitCode = metadata?.exit;
+                        // For shell commands, check exit code; for others, check status and error
+                        const isShellCommand = name === 'bash' || name === 'shell' || name === 'exec';
+                        const success = isShellCommand
+                            ? (exitCode === 0 || exitCode === undefined)
+                            : (status === 'completed' && !state?.error);
+                        events.push({
+                            timestamp: data.timestamp ? new Date(data.timestamp).toISOString() : undefined,
+                            type: 'tool_result',
+                            tool: {
+                                name: normalizeToolName(name),
+                                originalName: name,
+                                result: output,
+                                success,
+                            },
+                            raw: state,
+                        });
+                    }
+                }
+                break;
+            }
+            // Real OpenCode format: text with part.text
+            case 'text': {
+                const text = part?.text;
+                if (text && text.trim()) {
+                    events.push({
+                        timestamp: data.timestamp ? new Date(data.timestamp).toISOString() : undefined,
+                        type: 'message',
+                        role: 'assistant',
+                        content: text,
+                        raw: data,
+                    });
+                }
+                break;
+            }
+            // Step events - extract cost/token info if needed
+            case 'step_start':
+            case 'step_finish': {
+                // These are metadata events, skip for now
+                // Could extract token usage from step_finish if needed
+                break;
+            }
+            // Legacy/fallback formats
+            case 'message':
+            case 'response':
+            case 'assistant':
+            case 'user': {
+                const role = data.role || eventType === 'assistant'
+                    ? 'assistant'
+                    : eventType === 'user'
+                        ? 'user'
+                        : 'assistant';
+                const content = data.message?.content || data.content || data.text;
+                if (content) {
+                    events.push({
+                        timestamp: data.timestamp || data.time,
+                        type: 'message',
+                        role: role,
+                        content,
+                        raw: data,
+                    });
+                }
+                // Check for tool calls within the message
+                const toolCalls = data.message?.tool_calls || data.tool_calls || [];
+                for (const call of toolCalls) {
+                    const name = call.function?.name || call.name;
+                    const args = call.function?.arguments
+                        ? typeof call.function.arguments === 'string'
+                            ? JSON.parse(call.function.arguments)
+                            : call.function.arguments
+                        : call.arguments || call.input || {};
+                    events.push({
+                        timestamp: data.timestamp || data.time,
+                        type: 'tool_call',
+                        tool: {
+                            name: normalizeToolName(name),
+                            originalName: name,
+                            args,
+                        },
+                        raw: call,
+                    });
+                }
+                break;
+            }
+            case 'tool_call':
+            case 'function_call':
+            case 'action': {
+                const name = data.tool || data.function || data.name || data.action;
+                const args = data.input || data.arguments || data.params || {};
+                events.push({
+                    timestamp: data.timestamp || data.time,
+                    type: 'tool_call',
+                    tool: {
+                        name: normalizeToolName(name),
+                        originalName: name,
+                        args,
+                    },
+                    raw: data,
+                });
+                break;
+            }
+            case 'tool_result':
+            case 'function_result':
+            case 'action_result':
+            case 'result': {
+                events.push({
+                    timestamp: data.timestamp || data.time,
+                    type: 'tool_result',
+                    tool: {
+                        name: 'unknown',
+                        originalName: data.tool || data.function || 'unknown',
+                        result: data.output || data.result || data.content,
+                        success: data.success !== false && !data.error,
+                    },
+                    raw: data,
+                });
+                break;
+            }
+            case 'thinking':
+            case 'reasoning': {
+                events.push({
+                    timestamp: data.timestamp || data.time,
+                    type: 'thinking',
+                    content: data.content || data.text || data.thinking,
+                    raw: data,
+                });
+                break;
+            }
+            case 'error': {
+                events.push({
+                    timestamp: data.timestamp || data.time,
+                    type: 'error',
+                    content: data.error?.message || data.message || data.content,
+                    raw: data,
+                });
+                break;
+            }
+            default: {
+                // Try to infer from structure
+                if (data.message && typeof data.message === 'object') {
+                    const role = data.message.role || 'assistant';
+                    events.push({
+                        timestamp: data.timestamp || data.time,
+                        type: 'message',
+                        role: role,
+                        content: data.message.content,
+                        raw: data,
+                    });
+                }
+            }
+        }
+    }
+    catch {
+        // Skip unparseable lines
+    }
+    return events;
+}
+/**
+ * Parse OpenCode JSONL transcript into normalized events.
+ */
+export function parseOpenCodeTranscript(raw) {
+    const events = [];
+    const errors = [];
+    const lines = raw.split('\n').filter((line) => line.trim());
+    for (const line of lines) {
+        try {
+            const lineEvents = parseOpenCodeLine(line);
+            events.push(...lineEvents);
+        }
+        catch (e) {
+            errors.push(`Failed to parse line: ${e instanceof Error ? e.message : String(e)}`);
+        }
+    }
+    // Post-process to extract additional metadata
+    for (const event of events) {
+        if (event.type === 'tool_call' && event.tool) {
+            const args = event.tool.args || {};
+            // Extract file paths for file operations
+            if (['file_read', 'file_write', 'file_edit'].includes(event.tool.name)) {
+                const path = extractFilePath(args);
+                if (path) {
+                    event.tool.args = { ...args, _extractedPath: path };
+                }
+            }
+            // Extract URLs for web fetches
+            if (event.tool.name === 'web_fetch') {
+                const url = extractUrl(args);
+                if (url) {
+                    event.tool.args = { ...args, _extractedUrl: url };
+                }
+            }
+            // Extract commands for shell operations
+            if (event.tool.name === 'shell') {
+                const command = extractCommand(args);
+                if (command) {
+                    event.tool.args = { ...args, _extractedCommand: command };
+                }
+            }
+        }
+    }
+    return { events, errors };
+}
+//# sourceMappingURL=opencode.js.map

package/dist/lib/o11y/parsers/opencode.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"opencode.js","sourceRoot":"","sources":["../../../../src/lib/o11y/parsers/opencode.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAIH;;GAEG;AACH,SAAS,iBAAiB,CAAC,IAAY;IACrC,MAAM,OAAO,GAA6B;QACxC,kBAAkB;QAClB,IAAI,EAAE,WAAW;QACjB,SAAS,EAAE,WAAW;QACtB,SAAS,EAAE,WAAW;QACtB,KAAK,EAAE,YAAY;QACnB,UAAU,EAAE,YAAY;QACxB,UAAU,EAAE,YAAY;QACxB,MAAM,EAAE,YAAY;QACpB,IAAI,EAAE,WAAW;QACjB,SAAS,EAAE,WAAW;QACtB,SAAS,EAAE,WAAW;QACtB,KAAK,EAAE,WAAW;QAElB,QAAQ;QACR,IAAI,EAAE,OAAO;QACb,KAAK,EAAE,OAAO;QACd,IAAI,EAAE,OAAO;QACb,OAAO,EAAE,OAAO;QAChB,GAAG,EAAE,OAAO;QACZ,OAAO,EAAE,OAAO;QAEhB,MAAM;QACN,KAAK,EAAE,WAAW;QAClB,IAAI,EAAE,WAAW;QACjB,OAAO,EAAE,WAAW;QACpB,SAAS,EAAE,WAAW;QACtB,MAAM,EAAE,YAAY;QACpB,UAAU,EAAE,YAAY;QAExB,oBAAoB;QACpB,IAAI,EAAE,MAAM;QACZ,IAAI,EAAE,MAAM;QACZ,IAAI,EAAE,MAAM;QACZ,IAAI,EAAE,MAAM;QACZ,EAAE,EAAE,MAAM;QACV,OAAO,EAAE,MAAM;QACf,EAAE,EAAE,UAAU;QACd,GAAG,EAAE,UAAU;QACf,QAAQ,EAAE,UAAU;KACrB,CAAC;IAEF,OAAO,OAAO,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,IAAI,SAAS,CAAC;AAClD,CAAC;AAED;;GAEG;AACH,SAAS,eAAe,CAAC,IAA6B;IACpD,OAAO,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAuB,CAAC;AACzG,CAAC;AAED;;GAEG;AACH,SAAS,UAAU,CAAC,IAA6B;IAC/C,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,QAAQ,CAAuB,CAAC;AACpF,CAAC;AAED;;GAEG;AACH,SAAS,cAAc,CAAC,IAA6B;IACnD,IAAI,OAAO,IAAI,CAAC,OAAO,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC,OAAO,CAAC;IAC1D,IAAI,OAAO,IAAI,CAAC,GAAG,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC,GAAG,CAAC;IAClD,IAAI,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC,MAAM,CAAC;IACxD,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAC7B,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC;QAClE,OAAO,GAAG,OAAO,IAAK,IAAI,CAAC,IAAiB,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC;IAClE,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;;;;;GAMG;AACH,SAAS,iBAAiB,CAAC,IAAY;IACrC,MAAM,MAAM,GAAsB,EAAE,CAAC;IAErC,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAE9B,sCAAsC;QACtC,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC;QACvD,MAAM,IAAI,GAAG,IAAI,CAAC,IAA2C,CAAC;QAC9D,MAAM,KAAK,GAAG,IAAI,EAAE,KAA4C,CAAC;QAEjE,QAAQ,SAAS,EAAE,CAAC;YAClB,gDAAgD;YAChD,KAAK,UAAU,CAAC,CAAC,CAAC;gBAChB,IAAI,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;oBACtB,MAAM,IAAI,GAAG,IAAI,CAAC,IAAc,CAAC;oBACjC,MAAM,IAAI,GAAI,KAAK,EAAE,KAAiC,IAAI,EAAE,CAAC;oBAC7D,MAAM,MAAM,GAAG,KAAK,EAAE,MAAM,CAAC;oBAC7B,MAAM,MAAM,GAAG,KAAK,EAAE,MAA4B,CAAC;oBAEnD,uBAAuB;oBACvB,MAAM,CAAC,IAAI,CAAC;wBACV,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,SAAS;wBAC9E,IAAI,EAAE,WAAW;wBACjB,IAAI,EAAE;4BACJ,IAAI,EAAE,iBAAiB,CAAC,IAAI,CAAC;4BAC7B,YAAY,EAAE,IAAI;4BAClB,IAAI;yBACL;wBACD,GAAG,EAAE,IAAI;qBACV,CAAC,CAAC;oBAEH,sCAAsC;oBACtC,IAAI,MAAM,KAAK,WAAW,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;wBACnD,MAAM,QAAQ,GAAG,KAAK,EAAE,QAA+C,CAAC;wBACxE,MAAM,QAAQ,GAAG,QAAQ,EAAE,IAA0B,CAAC;wBACtD,0EAA0E;wBAC1E,MAAM,cAAc,GAAG,IAAI,KAAK,MAAM,IAAI,IAAI,KAAK,OAAO,IAAI,IAAI,KAAK,MAAM,CAAC;wBAC9E,MAAM,OAAO,GAAG,cAAc;4BAC5B,CAAC,CAAC,CAAC,QAAQ,KAAK,CAAC,IAAI,QAAQ,KAAK,SAAS,CAAC;4BAC5C,CAAC,CAAC,CAAC,MAAM,KAAK,WAAW,IAAI,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;wBAE9C,MAAM,CAAC,IAAI,CAAC;4BACV,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,SAAS;4BAC9E,IAAI,EAAE,aAAa;4BACnB,IAAI,EAAE;gCACJ,IAAI,EAAE,iBAAiB,CAAC,IAAI,CAAC;gCAC7B,YAAY,EAAE,IAAI;gCAClB,MAAM,EAAE,MAAM;gCACd,OAAO;6BACR;4BACD,GAAG,EAAE,KAAK;yBACX,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;gBACD,MAAM;YACR,CAAC;YAED,4CAA4C;YAC5C,KAAK,MAAM,CAAC,CAAC,CAAC;gBACZ,MAAM,IAAI,GAAG,IAAI,EAAE,IAA0B,CAAC;gBAC9C,IAAI,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;oBACxB,MAAM,CAAC,IAAI,CAAC;wBACV,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,SAAS;wBAC9E,IAAI,EAAE,SAAS;wBACf,IAAI,EAAE,WAAW;wBACjB,OAAO,EAAE,IAAI;wBACb,GAAG,EAAE,IAAI;qBACV,CAAC,CAAC;gBACL,CAAC;gBACD,MAAM;YACR,CAAC;YAED,kDAAkD;YAClD,KAAK,YAAY,CAAC;YAClB,KAAK,aAAa,CAAC,CAAC,CAAC;gBACnB,0CAA0C;gBAC1C,uDAAuD;gBACvD,MAAM;YACR,CAAC;YAED,0BAA0B;YAC1B,KAAK,SAAS,CAAC;YACf,KAAK,UAAU,CAAC;YAChB,KAAK,WAAW,CAAC;YACjB,KAAK,MAAM,CAAC,CAAC,CAAC;gBACZ,MAAM,IAAI,GACR,IAAI,CAAC,IAAI,IAAI,SAAS,KAAK,WAAW;oBACpC,CAAC,CAAC,WAAW;oBACb,CAAC,CAAC,SAAS,KAAK,MAAM;wBACpB,CAAC,CAAC,MAAM;wBACR,CAAC,CAAC,WAAW,CAAC;gBACpB,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,OAAO,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC;gBAEnE,IAAI,OAAO,EAAE,CAAC;oBACZ,MAAM,CAAC,IAAI,CAAC;wBACV,SAAS,EAAE,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,IAAI;wBACtC,IAAI,EAAE,SAAS;wBACf,IAAI,EAAE,IAAuC;wBAC7C,OAAO;wBACP,GAAG,EAAE,IAAI;qBACV,CAAC,CAAC;gBACL,CAAC;gBAED,0CAA0C;gBAC1C,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,EAAE,UAAU,IAAI,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC;gBACpE,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;oBAC7B,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,EAAE,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC;oBAC9C,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,EAAE,SAAS;wBACnC,CAAC,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,KAAK,QAAQ;4BAC3C,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC;4BACrC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS;wBAC3B,CAAC,CAAC,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;oBAEvC,MAAM,CAAC,IAAI,CAAC;wBACV,SAAS,EAAE,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,IAAI;wBACtC,IAAI,EAAE,WAAW;wBACjB,IAAI,EAAE;4BACJ,IAAI,EAAE,iBAAiB,CAAC,IAAI,CAAC;4BAC7B,YAAY,EAAE,IAAI;4BAClB,IAAI;yBACL;wBACD,GAAG,EAAE,IAAI;qBACV,CAAC,CAAC;gBACL,CAAC;gBACD,MAAM;YACR,CAAC;YAED,KAAK,WAAW,CAAC;YACjB,KAAK,eAAe,CAAC;YACrB,KAAK,QAAQ,CAAC,CAAC,CAAC;gBACd,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,CAAC;gBACpE,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC;gBAE/D,MAAM,CAAC,IAAI,CAAC;oBACV,SAAS,EAAE,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,IAAI;oBACtC,IAAI,EAAE,WAAW;oBACjB,IAAI,EAAE;wBACJ,IAAI,EAAE,iBAAiB,CAAC,IAAI,CAAC;wBAC7B,YAAY,EAAE,IAAI;wBAClB,IAAI;qBACL;oBACD,GAAG,EAAE,IAAI;iBACV,CAAC,CAAC;gBACH,MAAM;YACR,CAAC;YAED,KAAK,aAAa,CAAC;YACnB,KAAK,iBAAiB,CAAC;YACvB,KAAK,eAAe,CAAC;YACrB,KAAK,QAAQ,CAAC,CAAC,CAAC;gBACd,MAAM,CAAC,IAAI,CAAC;oBACV,SAAS,EAAE,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,IAAI;oBACtC,IAAI,EAAE,aAAa;oBACnB,IAAI,EAAE;wBACJ,IAAI,EAAE,SAAS;wBACf,YAAY,EAAE,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,QAAQ,IAAI,SAAS;wBACrD,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,OAAO;wBAClD,OAAO,EAAE,IAAI,CAAC,OAAO,KAAK,KAAK,IAAI,CAAC,IAAI,CAAC,KAAK;qBAC/C;oBACD,GAAG,EAAE,IAAI;iBACV,CAAC,CAAC;gBACH,MAAM;YACR,CAAC;YAED,KAAK,UAAU,CAAC;YAChB,KAAK,WAAW,CAAC,CAAC,CAAC;gBACjB,MAAM,CAAC,IAAI,CAAC;oBACV,SAAS,EAAE,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,IAAI;oBACtC,IAAI,EAAE,UAAU;oBAChB,OAAO,EAAE,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,QAAQ;oBACnD,GAAG,EAAE,IAAI;iBACV,CAAC,CAAC;gBACH,MAAM;YACR,CAAC;YAED,KAAK,OAAO,CAAC,CAAC,CAAC;gBACb,MAAM,CAAC,IAAI,CAAC;oBACV,SAAS,EAAE,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,IAAI;oBACtC,IAAI,EAAE,OAAO;oBACb,OAAO,EAAE,IAAI,CAAC,KAAK,EAAE,OAAO,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO;oBAC5D,GAAG,EAAE,IAAI;iBACV,CAAC,CAAC;gBACH,MAAM;YACR,CAAC;YAED,OAAO,CAAC,CAAC,CAAC;gBACR,8BAA8B;gBAC9B,IAAI,IAAI,CAAC,OAAO,IAAI,OAAO,IAAI,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;oBACrD,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,IAAI,WAAW,CAAC;oBAC9C,MAAM,CAAC,IAAI,CAAC;wBACV,SAAS,EAAE,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,IAAI;wBACtC,IAAI,EAAE,SAAS;wBACf,IAAI,EAAE,IAAuC;wBAC7C,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO;wBAC7B,GAAG,EAAE,IAAI;qBACV,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,yBAAyB;IAC3B,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,uBAAuB,CAAC,GAAW;IAIjD,MAAM,MAAM,GAAsB,EAAE,CAAC;IACrC,MAAM,MAAM,GAAa,EAAE,CAAC;IAE5B,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;IAE5D,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC;YAC3C,MAAM,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAC;QAC7B,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,CAAC,IAAI,CAAC,yBAAyB,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACrF,CAAC;IACH,CAAC;IAED,8CAA8C;IAC9C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,KAAK,CAAC,IAAI,KAAK,WAAW,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;YAC7C,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;YAEnC,yCAAyC;YACzC,IAAI,CAAC,WAAW,EAAE,YAAY,EAAE,WAAW,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBACvE,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;gBACnC,IAAI,IAAI,EAAE,CAAC;oBACT,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,EAAE,GAAG,IAAI,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC;gBACtD,CAAC;YACH,CAAC;YAED,+BAA+B;YAC/B,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;gBACpC,MAAM,GAAG,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;gBAC7B,IAAI,GAAG,EAAE,CAAC;oBACR,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,EAAE,GAAG,IAAI,EAAE,aAAa,EAAE,GAAG,EAAE,CAAC;gBACpD,CAAC;YACH,CAAC;YAED,wCAAwC;YACxC,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;gBAChC,MAAM,OAAO,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;gBACrC,IAAI,OAAO,EAAE,CAAC;oBACZ,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,EAAE,GAAG,IAAI,EAAE,iBAAiB,EAAE,OAAO,EAAE,CAAC;gBAC5D,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;AAC5B,CAAC"}

package/dist/lib/o11y/types.d.ts

@@ -0,0 +1,113 @@
+/**
+ * Observability types for cross-agent transcript analysis.
+ * Provides a unified schema regardless of which agent produced the transcript.
+ */
+/**
+ * Canonical tool names across agents.
+ * Maps agent-specific tool names to standardized names.
+ */
+export type ToolName = 'file_read' | 'file_write' | 'file_edit' | 'shell' | 'web_fetch' | 'web_search' | 'glob' | 'grep' | 'list_dir' | 'agent_task' | 'unknown';
+/**
+ * An event in the transcript.
+ */
+export interface TranscriptEvent {
+    /** ISO timestamp of the event */
+    timestamp?: string;
+    /** Event type */
+    type: 'message' | 'tool_call' | 'tool_result' | 'thinking' | 'error';
+    /** For message events: the role */
+    role?: 'user' | 'assistant' | 'system';
+    /** Text content (for messages, thinking, errors) */
+    content?: string;
+    /** For tool_call and tool_result events */
+    tool?: {
+        /** Canonical tool name */
+        name: ToolName;
+        /** Original tool name from the agent */
+        originalName: string;
+        /** Tool arguments */
+        args?: Record<string, unknown>;
+        /** Tool result (for tool_result events) */
+        result?: unknown;
+        /** Duration in milliseconds (if available) */
+        durationMs?: number;
+        /** Whether the tool call succeeded */
+        success?: boolean;
+    };
+    /** Raw event data from the agent (for debugging) */
+    raw?: unknown;
+}
+/**
+ * Web fetch information extracted from tool calls.
+ */
+export interface WebFetchInfo {
+    /** The URL that was fetched */
+    url: string;
+    /** HTTP method (if known) */
+    method?: string;
+    /** HTTP status code (if available) */
+    status?: number;
+    /** Whether the fetch succeeded */
+    success?: boolean;
+}
+/**
+ * File operation information.
+ */
+export interface FileOperationInfo {
+    /** File path */
+    path: string;
+    /** Operation type */
+    operation: 'read' | 'write' | 'edit';
+}
+/**
+ * Shell command information.
+ */
+export interface ShellCommandInfo {
+    /** The command that was run */
+    command: string;
+    /** Exit code (if available) */
+    exitCode?: number;
+    /** Whether the command succeeded */
+    success?: boolean;
+}
+/**
+ * Summary statistics derived from the transcript.
+ */
+export interface TranscriptSummary {
+    /** Total number of conversation turns */
+    totalTurns: number;
+    /** Count of each tool type used */
+    toolCalls: Record<ToolName, number>;
+    /** Total tool calls */
+    totalToolCalls: number;
+    /** Web fetches made during the run */
+    webFetches: WebFetchInfo[];
+    /** Files that were read */
+    filesRead: string[];
+    /** Files that were written or edited */
+    filesModified: string[];
+    /** Shell commands executed */
+    shellCommands: ShellCommandInfo[];
+    /** Errors encountered */
+    errors: string[];
+    /** Thinking/reasoning blocks (if available) */
+    thinkingBlocks: number;
+}
+/**
+ * A parsed transcript with events and summary.
+ */
+export interface Transcript {
+    /** Agent that produced this transcript */
+    agent: string;
+    /** Model used (if known) */
+    model?: string;
+    /** All events in order */
+    events: TranscriptEvent[];
+    /** Derived summary statistics */
+    summary: TranscriptSummary;
+    /** Whether parsing succeeded fully */
+    parseSuccess: boolean;
+    /** Any parsing warnings/errors */
+    parseErrors?: string[];
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/lib/o11y/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/lib/o11y/types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;GAGG;AACH,MAAM,MAAM,QAAQ,GAChB,WAAW,GACX,YAAY,GACZ,WAAW,GACX,OAAO,GACP,WAAW,GACX,YAAY,GACZ,MAAM,GACN,MAAM,GACN,UAAU,GACV,YAAY,GACZ,SAAS,CAAC;AAEd;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,iCAAiC;IACjC,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,iBAAiB;IACjB,IAAI,EAAE,SAAS,GAAG,WAAW,GAAG,aAAa,GAAG,UAAU,GAAG,OAAO,CAAC;IAErE,mCAAmC;IACnC,IAAI,CAAC,EAAE,MAAM,GAAG,WAAW,GAAG,QAAQ,CAAC;IAEvC,oDAAoD;IACpD,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,2CAA2C;IAC3C,IAAI,CAAC,EAAE;QACL,0BAA0B;QAC1B,IAAI,EAAE,QAAQ,CAAC;QACf,wCAAwC;QACxC,YAAY,EAAE,MAAM,CAAC;QACrB,qBAAqB;QACrB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAC/B,2CAA2C;QAC3C,MAAM,CAAC,EAAE,OAAO,CAAC;QACjB,8CAA8C;QAC9C,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,sCAAsC;QACtC,OAAO,CAAC,EAAE,OAAO,CAAC;KACnB,CAAC;IAEF,oDAAoD;IACpD,GAAG,CAAC,EAAE,OAAO,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,+BAA+B;IAC/B,GAAG,EAAE,MAAM,CAAC;IACZ,6BAA6B;IAC7B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,sCAAsC;IACtC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,kCAAkC;IAClC,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,gBAAgB;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,qBAAqB;IACrB,SAAS,EAAE,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC;CACtC;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,+BAA+B;IAC/B,OAAO,EAAE,MAAM,CAAC;IAChB,+BAA+B;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,oCAAoC;IACpC,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,yCAAyC;IACzC,UAAU,EAAE,MAAM,CAAC;IAEnB,mCAAmC;IACnC,SAAS,EAAE,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAEpC,uBAAuB;IACvB,cAAc,EAAE,MAAM,CAAC;IAEvB,sCAAsC;IACtC,UAAU,EAAE,YAAY,EAAE,CAAC;IAE3B,2BAA2B;IAC3B,SAAS,EAAE,MAAM,EAAE,CAAC;IAEpB,wCAAwC;IACxC,aAAa,EAAE,MAAM,EAAE,CAAC;IAExB,8BAA8B;IAC9B,aAAa,EAAE,gBAAgB,EAAE,CAAC;IAElC,yBAAyB;IACzB,MAAM,EAAE,MAAM,EAAE,CAAC;IAEjB,+CAA+C;IAC/C,cAAc,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,0CAA0C;IAC1C,KAAK,EAAE,MAAM,CAAC;IAEd,4BAA4B;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,0BAA0B;IAC1B,MAAM,EAAE,eAAe,EAAE,CAAC;IAE1B,iCAAiC;IACjC,OAAO,EAAE,iBAAiB,CAAC;IAE3B,sCAAsC;IACtC,YAAY,EAAE,OAAO,CAAC;IAEtB,kCAAkC;IAClC,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;CACxB"}

package/dist/lib/o11y/types.js

@@ -0,0 +1,6 @@
+/**
+ * Observability types for cross-agent transcript analysis.
+ * Provides a unified schema regardless of which agent produced the transcript.
+ */
+export {};
+//# sourceMappingURL=types.js.map

package/dist/lib/o11y/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/lib/o11y/types.ts"],"names":[],"mappings":"AAAA;;;GAGG"}

package/dist/lib/results.d.ts

@@ -0,0 +1,91 @@
+/**
+ * Results storage and reporting for eval experiments.
+ */
+import type { EvalRunResult, EvalRunData, EvalSummary, ExperimentResults, RunnableExperimentConfig } from './types.js';
+import type { AgentRunResult } from './agents/types.js';
+/**
+ * Convert AgentRunResult to EvalRunData (result + transcript).
+ */
+export declare function agentResultToEvalRunData(agentResult: AgentRunResult): EvalRunData;
+/**
+ * Create a summary from multiple run data.
+ */
+export declare function createEvalSummary(name: string, runData: EvalRunData[]): EvalSummary;
+/**
+ * Create experiment results from eval summaries.
+ */
+export declare function createExperimentResults(config: RunnableExperimentConfig, evals: EvalSummary[], startedAt: Date, completedAt: Date): ExperimentResults;
+/**
+ * Options for saving results.
+ */
+export interface SaveResultsOptions {
+    /** Base directory for results */
+    resultsDir: string;
+    /** Experiment name (used for subdirectory) */
+    experimentName: string;
+    /** Per-eval fingerprints (eval name -> fingerprint hash) */
+    fingerprints?: Record<string, string>;
+    /** Per-eval validity flags (eval name -> valid). Defaults to true. */
+    validity?: Record<string, boolean>;
+    /** Whether this is a smoke test run. Smoke results are excluded from reuse. */
+    smoke?: boolean;
+    /** Per-eval fixture paths (eval name -> absolute path). Required when copyFiles is 'all'. */
+    fixturePaths?: Record<string, string>;
+}
+/**
+ * Save experiment results to disk.
+ *
+ * Creates a directory structure per design:
+ * results/
+ *   experiment-name/
+ *     2024-01-26T12-00-00Z/
+ *       eval-1/
+ *         run-1/
+ *           result.json
+ *           transcript.json      (parsed/structured - primary format)
+ *           transcript-raw.jsonl (raw agent output - for debugging)
+ *           outputs/
+ *         summary.json
+ */
+export declare function saveResults(results: ExperimentResults, options: SaveResultsOptions): string;
+/**
+ * Format results for terminal display.
+ */
+export declare function formatResultsTable(results: ExperimentResults): string;
+/**
+ * Format a single eval result for terminal display (used during progress).
+ */
+export declare function formatRunResult(evalName: string, runNumber: number, totalRuns: number, result: EvalRunResult, context?: {
+    experimentName?: string;
+    model?: string;
+    agent?: string;
+}): string;
+/**
+ * Create a progress indicator for running evals.
+ */
+export declare function createProgressDisplay(evalName: string, runNumber: number, totalRuns: number, context?: {
+    experimentName?: string;
+    model?: string;
+    agent?: string;
+}): string;
+/**
+ * A reusable result found by the scanner.
+ */
+export interface ReusableResult {
+    evalName: string;
+    fingerprint: string;
+    passRate: string;
+    timestamp: string;
+}
+/**
+ * Scan existing results for an experiment to find reusable eval results.
+ *
+ * A result is reusable if:
+ * 1. Its fingerprint matches the current fingerprint
+ * 2. It is "valid" (not marked as invalid by the classifier)
+ * 3. It has passedRuns > 0 (successful result worth reusing)
+ *
+ * Scans all timestamps newest-first and returns the latest match per eval.
+ */
+export declare function scanReusableResults(resultsDir: string, experimentName: string, fingerprints: Record<string, string>): Map<string, ReusableResult>;
+//# sourceMappingURL=results.d.ts.map

package/dist/lib/results.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"results.d.ts","sourceRoot":"","sources":["../../src/lib/results.ts"],"names":[],"mappings":"AAAA;;GAEG;AAaH,OAAO,KAAK,EACX,aAAa,EACb,WAAW,EACX,WAAW,EACX,iBAAiB,EACjB,wBAAwB,EACxB,MAAM,YAAY,CAAC;AACpB,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAKxD;;GAEG;AACH,wBAAgB,wBAAwB,CACvC,WAAW,EAAE,cAAc,GACzB,WAAW,CAoCb;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAChC,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,WAAW,EAAE,GACpB,WAAW,CAab;AAED;;GAEG;AACH,wBAAgB,uBAAuB,CACtC,MAAM,EAAE,wBAAwB,EAChC,KAAK,EAAE,WAAW,EAAE,EACpB,SAAS,EAAE,IAAI,EACf,WAAW,EAAE,IAAI,GACf,iBAAiB,CAOnB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAClC,iCAAiC;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,8CAA8C;IAC9C,cAAc,EAAE,MAAM,CAAC;IACvB,4DAA4D;IAC5D,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtC,sEAAsE;IACtE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,+EAA+E;IAC/E,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,6FAA6F;IAC7F,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACtC;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,WAAW,CAC1B,OAAO,EAAE,iBAAiB,EAC1B,OAAO,EAAE,kBAAkB,GACzB,MAAM,CAwLR;AAED;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,iBAAiB,GAAG,MAAM,CA8DrE;AAED;;GAEG;AACH,wBAAgB,eAAe,CAC9B,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,aAAa,EACrB,OAAO,CAAC,EAAE;IAAE,cAAc,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,GACnE,MAAM,CAwBR;AAED;;GAEG;AACH,wBAAgB,qBAAqB,CACpC,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,EACjB,OAAO,CAAC,EAAE;IAAE,cAAc,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,GACnE,MAAM,CASR;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;CAClB;AAED;;;;;;;;;GASG;AACH,wBAAgB,mBAAmB,CAClC,UAAU,EAAE,MAAM,EAClB,cAAc,EAAE,MAAM,EACtB,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAClC,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,CA4E7B"}