model-test-bench 1.0.2

package/dist/src/server/services/transcript-formatter.js

@@ -0,0 +1,227 @@
+// ---------------------------------------------------------------------------
+// Transcript Formatter — converts SDK messages into evaluator-readable text
+// ---------------------------------------------------------------------------
+const MAX_TRANSCRIPT_CHARS = 100_000;
+const TRUNCATED_MARKER = '\n[transcript truncated]';
+const MAX_TOOL_INPUT_CHARS = 500;
+const MAX_TOOL_OUTPUT_CHARS = 500;
+/** Format SDK messages into a readable transcript and structured summary. */
+export function formatTranscript(messages) {
+    const lines = [];
+    const toolCalls = [];
+    const filesRead = new Set();
+    const filesModified = new Set();
+    const commandFailures = [];
+    const retryPatterns = [];
+    let askedClarifyingQuestions = false;
+    let totalChars = 0;
+    let truncated = false;
+    for (const record of messages) {
+        if (truncated)
+            break;
+        const msg = record.message;
+        const msgType = msg['type'];
+        if (msgType === 'text-delta') {
+            // Streaming text chunk — accumulate for transcript
+            const text = msg['text'];
+            if (text) {
+                const r = appendLine(lines, `[Text] ${text}`, totalChars);
+                totalChars = r.totalChars;
+                truncated = r.truncated;
+                if (looksLikeQuestion(text))
+                    askedClarifyingQuestions = true;
+            }
+        }
+        else if (msgType === 'reasoning-delta') {
+            const text = msg['text'];
+            if (text) {
+                const r = appendLine(lines, `[Thinking] ${text}`, totalChars);
+                totalChars = r.totalChars;
+                truncated = r.truncated;
+            }
+        }
+        else if (msgType === 'tool-call') {
+            const name = msg['toolName'] ?? 'unknown_tool';
+            const args = truncateStr(JSON.stringify(msg['args'] ?? ''), MAX_TOOL_INPUT_CHARS);
+            toolCalls.push(name);
+            trackFileAccess(name, msg, filesRead, filesModified);
+            const r = appendLine(lines, `[Tool Call] ${name}: ${args}`, totalChars);
+            totalChars = r.totalChars;
+            truncated = r.truncated;
+        }
+        else if (msgType === 'tool-result') {
+            const output = truncateStr(String(msg['result'] ?? ''), MAX_TOOL_OUTPUT_CHARS);
+            const r = appendLine(lines, `[Tool Result] ${output}`, totalChars);
+            totalChars = r.totalChars;
+            truncated = r.truncated;
+        }
+        else if (msgType === 'step') {
+            // AI SDK step format: { text, toolCalls, toolResults, usage }
+            const stepText = msg['text'];
+            if (stepText) {
+                const r = appendLine(lines, `[Assistant] ${stepText}`, totalChars);
+                totalChars = r.totalChars;
+                truncated = r.truncated;
+                if (looksLikeQuestion(stepText))
+                    askedClarifyingQuestions = true;
+            }
+            const calls = msg['toolCalls'];
+            if (Array.isArray(calls)) {
+                for (const call of calls) {
+                    if (truncated)
+                        break;
+                    const name = call['toolName'] ?? 'unknown_tool';
+                    const input = truncateStr(JSON.stringify(call['args'] ?? ''), MAX_TOOL_INPUT_CHARS);
+                    toolCalls.push(name);
+                    trackFileAccess(name, call, filesRead, filesModified);
+                    const r = appendLine(lines, `[Tool Call] ${name}: ${input}`, totalChars);
+                    totalChars = r.totalChars;
+                    truncated = r.truncated;
+                }
+            }
+            const results = msg['toolResults'];
+            if (Array.isArray(results)) {
+                for (const tr of results) {
+                    if (truncated)
+                        break;
+                    const output = truncateStr(String(tr['result'] ?? ''), MAX_TOOL_OUTPUT_CHARS);
+                    const r = appendLine(lines, `[Tool Result] ${output}`, totalChars);
+                    totalChars = r.totalChars;
+                    truncated = r.truncated;
+                }
+            }
+        }
+        else if (msgType === 'assistant') {
+            const formatted = formatAssistantMessage(msg);
+            if (formatted) {
+                const result = appendLine(lines, formatted, totalChars);
+                totalChars = result.totalChars;
+                truncated = result.truncated;
+                if (looksLikeQuestion(formatted)) {
+                    askedClarifyingQuestions = true;
+                }
+            }
+        }
+        else if (msgType === 'tool_use') {
+            const name = msg['name'] ?? 'unknown_tool';
+            const input = truncateStr(JSON.stringify(msg['input'] ?? ''), MAX_TOOL_INPUT_CHARS);
+            toolCalls.push(name);
+            trackFileAccess(name, msg, filesRead, filesModified);
+            const line = `[Tool Call] ${name}: ${input}`;
+            const result = appendLine(lines, line, totalChars);
+            totalChars = result.totalChars;
+            truncated = result.truncated;
+        }
+        else if (msgType === 'tool_result') {
+            const output = truncateStr(extractToolResultText(msg), MAX_TOOL_OUTPUT_CHARS);
+            const isError = msg['is_error'] === true;
+            if (isError) {
+                commandFailures.push(output);
+                detectRetryPattern(toolCalls, retryPatterns);
+            }
+            const prefix = isError ? '[Tool Error]' : '[Tool Result]';
+            const line = `${prefix} ${output}`;
+            const result = appendLine(lines, line, totalChars);
+            totalChars = result.totalChars;
+            truncated = result.truncated;
+        }
+        else if (msgType === 'result') {
+            const resultText = msg['result'] ?? '';
+            if (resultText) {
+                const line = `[Final Result] ${resultText}`;
+                const result = appendLine(lines, line, totalChars);
+                totalChars = result.totalChars;
+                truncated = result.truncated;
+            }
+        }
+    }
+    if (truncated) {
+        lines.push(TRUNCATED_MARKER);
+    }
+    return {
+        text: lines.join('\n'),
+        summary: {
+            toolCallSequence: toolCalls,
+            filesRead: [...filesRead],
+            filesModified: [...filesModified],
+            commandFailures,
+            retryPatterns,
+            askedClarifyingQuestions,
+        },
+    };
+}
+// ---------------------------------------------------------------------------
+// Internal helpers
+// ---------------------------------------------------------------------------
+function appendLine(lines, line, totalChars) {
+    const newTotal = totalChars + line.length + 1; // +1 for newline
+    if (newTotal > MAX_TRANSCRIPT_CHARS) {
+        return { totalChars, truncated: true };
+    }
+    lines.push(line);
+    return { totalChars: newTotal, truncated: false };
+}
+function formatAssistantMessage(msg) {
+    const content = msg['message'];
+    if (!content)
+        return undefined;
+    const contentBlocks = content['content'];
+    if (!Array.isArray(contentBlocks))
+        return undefined;
+    const textParts = [];
+    for (const block of contentBlocks) {
+        if (block['type'] === 'text' && typeof block['text'] === 'string') {
+            textParts.push(block['text']);
+        }
+    }
+    return textParts.length > 0 ? `[Assistant] ${textParts.join(' ')}` : undefined;
+}
+function extractToolResultText(msg) {
+    const content = msg['content'];
+    if (typeof content === 'string')
+        return content;
+    if (Array.isArray(content)) {
+        const parts = [];
+        for (const block of content) {
+            if (block['type'] === 'text' && typeof block['text'] === 'string') {
+                parts.push(block['text']);
+            }
+        }
+        return parts.join(' ');
+    }
+    return '';
+}
+function truncateStr(s, max) {
+    return s.length > max ? s.slice(0, max) + '...' : s;
+}
+function looksLikeQuestion(text) {
+    return /\?\s*$/.test(text.trim());
+}
+function trackFileAccess(toolName, msg, filesRead, filesModified) {
+    // Support both Claude SDK format (msg.input) and AI SDK format (msg.args)
+    const input = (msg['input'] ?? msg['args']);
+    if (!input)
+        return;
+    const filePath = (input['file_path'] ?? input['path'] ?? input['filename']);
+    if (!filePath)
+        return;
+    if (toolName === 'Read' || toolName === 'Glob' || toolName === 'Grep') {
+        filesRead.add(filePath);
+    }
+    else if (toolName === 'Edit' || toolName === 'Write' || toolName === 'NotebookEdit') {
+        filesModified.add(filePath);
+    }
+}
+function detectRetryPattern(toolCalls, retryPatterns) {
+    if (toolCalls.length < 2)
+        return;
+    const last = toolCalls[toolCalls.length - 1];
+    const prev = toolCalls[toolCalls.length - 2];
+    if (last === prev) {
+        const pattern = `Repeated ${last} after error`;
+        if (!retryPatterns.includes(pattern)) {
+            retryPatterns.push(pattern);
+        }
+    }
+}
+//# sourceMappingURL=transcript-formatter.js.map

package/dist/src/server/services/transcript-formatter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"transcript-formatter.js","sourceRoot":"","sources":["../../../../src/server/services/transcript-formatter.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,4EAA4E;AAC5E,8EAA8E;AAI9E,MAAM,oBAAoB,GAAG,OAAO,CAAC;AACrC,MAAM,gBAAgB,GAAG,0BAA0B,CAAC;AACpD,MAAM,oBAAoB,GAAG,GAAG,CAAC;AACjC,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAkBlC,6EAA6E;AAC7E,MAAM,UAAU,gBAAgB,CAAC,QAAqC;IACpE,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,MAAM,SAAS,GAAgB,IAAI,GAAG,EAAE,CAAC;IACzC,MAAM,aAAa,GAAgB,IAAI,GAAG,EAAE,CAAC;IAC7C,MAAM,eAAe,GAAa,EAAE,CAAC;IACrC,MAAM,aAAa,GAAa,EAAE,CAAC;IACnC,IAAI,wBAAwB,GAAG,KAAK,CAAC;IACrC,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,SAAS,GAAG,KAAK,CAAC;IAEtB,KAAK,MAAM,MAAM,IAAI,QAAQ,EAAE,CAAC;QAC9B,IAAI,SAAS;YAAE,MAAM;QACrB,MAAM,GAAG,GAAG,MAAM,CAAC,OAAO,CAAC;QAC3B,MAAM,OAAO,GAAG,GAAG,CAAC,MAAM,CAAuB,CAAC;QAElD,IAAI,OAAO,KAAK,YAAY,EAAE,CAAC;YAC7B,mDAAmD;YACnD,MAAM,IAAI,GAAG,GAAG,CAAC,MAAM,CAAuB,CAAC;YAC/C,IAAI,IAAI,EAAE,CAAC;gBACT,MAAM,CAAC,GAAG,UAAU,CAAC,KAAK,EAAE,UAAU,IAAI,EAAE,EAAE,UAAU,CAAC,CAAC;gBAC1D,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC;gBAC1B,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC;gBACxB,IAAI,iBAAiB,CAAC,IAAI,CAAC;oBAAE,wBAAwB,GAAG,IAAI,CAAC;YAC/D,CAAC;QACH,CAAC;aAAM,IAAI,OAAO,KAAK,iBAAiB,EAAE,CAAC;YACzC,MAAM,IAAI,GAAG,GAAG,CAAC,MAAM,CAAuB,CAAC;YAC/C,IAAI,IAAI,EAAE,CAAC;gBACT,MAAM,CAAC,GAAG,UAAU,CAAC,KAAK,EAAE,cAAc,IAAI,EAAE,EAAE,UAAU,CAAC,CAAC;gBAC9D,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC;gBAC1B,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC;YAC1B,CAAC;QACH,CAAC;aAAM,IAAI,OAAO,KAAK,WAAW,EAAE,CAAC;YACnC,MAAM,IAAI,GAAI,GAAG,CAAC,UAAU,CAAY,IAAI,cAAc,CAAC;YAC3D,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,EAAE,oBAAoB,CAAC,CAAC;YAClF,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACrB,eAAe,CAAC,IAAI,EAAE,GAAG,EAAE,SAAS,EAAE,aAAa,CAAC,CAAC;YACrD,MAAM,CAAC,GAAG,UAAU,CAAC,KAAK,EAAE,eAAe,IAAI,KAAK,IAAI,EAAE,EAAE,UAAU,CAAC,CAAC;YACxE,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC;YAC1B,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC;QAC1B,CAAC;aAAM,IAAI,OAAO,KAAK,aAAa,EAAE,CAAC;YACrC,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,EAAE,qBAAqB,CAAC,CAAC;YAC/E,MAAM,CAAC,GAAG,UAAU,CAAC,KAAK,EAAE,iBAAiB,MAAM,EAAE,EAAE,UAAU,CAAC,CAAC;YACnE,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC;YAC1B,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC;QAC1B,CAAC;aAAM,IAAI,OAAO,KAAK,MAAM,EAAE,CAAC;YAC9B,8DAA8D;YAC9D,MAAM,QAAQ,GAAG,GAAG,CAAC,MAAM,CAAuB,CAAC;YACnD,IAAI,QAAQ,EAAE,CAAC;gBACb,MAAM,CAAC,GAAG,UAAU,CAAC,KAAK,EAAE,eAAe,QAAQ,EAAE,EAAE,UAAU,CAAC,CAAC;gBACnE,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC;gBAC1B,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC;gBACxB,IAAI,iBAAiB,CAAC,QAAQ,CAAC;oBAAE,wBAAwB,GAAG,IAAI,CAAC;YACnE,CAAC;YACD,MAAM,KAAK,GAAG,GAAG,CAAC,WAAW,CAAuD,CAAC;YACrF,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;gBACzB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;oBACzB,IAAI,SAAS;wBAAE,MAAM;oBACrB,MAAM,IAAI,GAAI,IAAI,CAAC,UAAU,CAAY,IAAI,cAAc,CAAC;oBAC5D,MAAM,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,EAAE,oBAAoB,CAAC,CAAC;oBACpF,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBACrB,eAAe,CAAC,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,aAAa,CAAC,CAAC;oBACtD,MAAM,CAAC,GAAG,UAAU,CAAC,KAAK,EAAE,eAAe,IAAI,KAAK,KAAK,EAAE,EAAE,UAAU,CAAC,CAAC;oBACzE,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC;oBAC1B,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC;gBAC1B,CAAC;YACH,CAAC;YACD,MAAM,OAAO,GAAG,GAAG,CAAC,aAAa,CAAuD,CAAC;YACzF,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC3B,KAAK,MAAM,EAAE,IAAI,OAAO,EAAE,CAAC;oBACzB,IAAI,SAAS;wBAAE,MAAM;oBACrB,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,EAAE,qBAAqB,CAAC,CAAC;oBAC9E,MAAM,CAAC,GAAG,UAAU,CAAC,KAAK,EAAE,iBAAiB,MAAM,EAAE,EAAE,UAAU,CAAC,CAAC;oBACnE,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC;oBAC1B,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC;gBAC1B,CAAC;YACH,CAAC;QACH,CAAC;aAAM,IAAI,OAAO,KAAK,WAAW,EAAE,CAAC;YACnC,MAAM,SAAS,GAAG,sBAAsB,CAAC,GAAG,CAAC,CAAC;YAC9C,IAAI,SAAS,EAAE,CAAC;gBACd,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC;gBACxD,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC;gBAC/B,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;gBAC7B,IAAI,iBAAiB,CAAC,SAAS,CAAC,EAAE,CAAC;oBACjC,wBAAwB,GAAG,IAAI,CAAC;gBAClC,CAAC;YACH,CAAC;QACH,CAAC;aAAM,IAAI,OAAO,KAAK,UAAU,EAAE,CAAC;YAClC,MAAM,IAAI,GAAI,GAAG,CAAC,MAAM,CAAY,IAAI,cAAc,CAAC;YACvD,MAAM,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,EAAE,oBAAoB,CAAC,CAAC;YACpF,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACrB,eAAe,CAAC,IAAI,EAAE,GAAG,EAAE,SAAS,EAAE,aAAa,CAAC,CAAC;YACrD,MAAM,IAAI,GAAG,eAAe,IAAI,KAAK,KAAK,EAAE,CAAC;YAC7C,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;YACnD,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC;YAC/B,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;QAC/B,CAAC;aAAM,IAAI,OAAO,KAAK,aAAa,EAAE,CAAC;YACrC,MAAM,MAAM,GAAG,WAAW,CAAC,qBAAqB,CAAC,GAAG,CAAC,EAAE,qBAAqB,CAAC,CAAC;YAC9E,MAAM,OAAO,GAAG,GAAG,CAAC,UAAU,CAAC,KAAK,IAAI,CAAC;YACzC,IAAI,OAAO,EAAE,CAAC;gBACZ,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBAC7B,kBAAkB,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;YAC/C,CAAC;YACD,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,eAAe,CAAC;YAC1D,MAAM,IAAI,GAAG,GAAG,MAAM,IAAI,MAAM,EAAE,CAAC;YACnC,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;YACnD,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC;YAC/B,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;QAC/B,CAAC;aAAM,IAAI,OAAO,KAAK,QAAQ,EAAE,CAAC;YAChC,MAAM,UAAU,GAAI,GAAG,CAAC,QAAQ,CAAY,IAAI,EAAE,CAAC;YACnD,IAAI,UAAU,EAAE,CAAC;gBACf,MAAM,IAAI,GAAG,kBAAkB,UAAU,EAAE,CAAC;gBAC5C,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;gBACnD,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC;gBAC/B,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;YAC/B,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,SAAS,EAAE,CAAC;QACd,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;IAC/B,CAAC;IAED,OAAO;QACL,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;QACtB,OAAO,EAAE;YACP,gBAAgB,EAAE,SAAS;YAC3B,SAAS,EAAE,CAAC,GAAG,SAAS,CAAC;YACzB,aAAa,EAAE,CAAC,GAAG,aAAa,CAAC;YACjC,eAAe;YACf,aAAa;YACb,wBAAwB;SACzB;KACF,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,mBAAmB;AACnB,8EAA8E;AAE9E,SAAS,UAAU,CACjB,KAAe,EACf,IAAY,EACZ,UAAkB;IAElB,MAAM,QAAQ,GAAG,UAAU,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,iBAAiB;IAChE,IAAI,QAAQ,GAAG,oBAAoB,EAAE,CAAC;QACpC,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;IACzC,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACjB,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;AACpD,CAAC;AAED,SAAS,sBAAsB,CAAC,GAAsC;IACpE,MAAM,OAAO,GAAG,GAAG,CAAC,SAAS,CAAwC,CAAC;IACtE,IAAI,CAAC,OAAO;QAAE,OAAO,SAAS,CAAC;IAE/B,MAAM,aAAa,GAAG,OAAO,CAAC,SAAS,CAAuD,CAAC;IAC/F,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC;QAAE,OAAO,SAAS,CAAC;IAEpD,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,KAAK,MAAM,KAAK,IAAI,aAAa,EAAE,CAAC;QAClC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,MAAM,IAAI,OAAO,KAAK,CAAC,MAAM,CAAC,KAAK,QAAQ,EAAE,CAAC;YAClE,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;IACD,OAAO,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,eAAe,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;AACjF,CAAC;AAED,SAAS,qBAAqB,CAAC,GAAsC;IACnE,MAAM,OAAO,GAAG,GAAG,CAAC,SAAS,CAAgE,CAAC;IAC9F,IAAI,OAAO,OAAO,KAAK,QAAQ;QAAE,OAAO,OAAO,CAAC;IAChD,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,MAAM,IAAI,OAAO,KAAK,CAAC,MAAM,CAAC,KAAK,QAAQ,EAAE,CAAC;gBAClE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;YAC5B,CAAC;QACH,CAAC;QACD,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACzB,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,WAAW,CAAC,CAAS,EAAE,GAAW;IACzC,OAAO,CAAC,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AACtD,CAAC;AAED,SAAS,iBAAiB,CAAC,IAAY;IACrC,OAAO,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;AACpC,CAAC;AAED,SAAS,eAAe,CACtB,QAAgB,EAChB,GAAsC,EACtC,SAAsB,EACtB,aAA0B;IAE1B,0EAA0E;IAC1E,MAAM,KAAK,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,CAAwC,CAAC;IACnF,IAAI,CAAC,KAAK;QAAE,OAAO;IACnB,MAAM,QAAQ,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,CAAuB,CAAC;IAClG,IAAI,CAAC,QAAQ;QAAE,OAAO;IAEtB,IAAI,QAAQ,KAAK,MAAM,IAAI,QAAQ,KAAK,MAAM,IAAI,QAAQ,KAAK,MAAM,EAAE,CAAC;QACtE,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC1B,CAAC;SAAM,IAAI,QAAQ,KAAK,MAAM,IAAI,QAAQ,KAAK,OAAO,IAAI,QAAQ,KAAK,cAAc,EAAE,CAAC;QACtF,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC9B,CAAC;AACH,CAAC;AAED,SAAS,kBAAkB,CAAC,SAA4B,EAAE,aAAuB;IAC/E,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO;IACjC,MAAM,IAAI,GAAG,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC7C,MAAM,IAAI,GAAG,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC7C,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;QAClB,MAAM,OAAO,GAAG,YAAY,IAAI,cAAc,CAAC;QAC/C,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;YACrC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC9B,CAAC;IACH,CAAC;AACH,CAAC"}

package/dist/src/server/services/update-checker.d.ts

@@ -0,0 +1,3 @@
+export declare function isNewer(latest: string, current: string): boolean;
+export declare function checkForUpdate(currentVersion: string): void;
+//# sourceMappingURL=update-checker.d.ts.map

package/dist/src/server/services/update-checker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"update-checker.d.ts","sourceRoot":"","sources":["../../../../src/server/services/update-checker.ts"],"names":[],"mappings":"AAEA,wBAAgB,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAQhE;AAED,wBAAgB,cAAc,CAAC,cAAc,EAAE,MAAM,GAAG,IAAI,CAwB3D"}

package/dist/src/server/services/update-checker.js

@@ -0,0 +1,34 @@
+import https from 'node:https';
+export function isNewer(latest, current) {
+    const a = latest.split('.').map(Number);
+    const b = current.split('.').map(Number);
+    for (let i = 0; i < 3; i++) {
+        if ((a[i] ?? 0) > (b[i] ?? 0))
+            return true;
+        if ((a[i] ?? 0) < (b[i] ?? 0))
+            return false;
+    }
+    return false;
+}
+export function checkForUpdate(currentVersion) {
+    const req = https.get('https://registry.npmjs.org/model-test-bench/latest', { timeout: 3000, headers: { Accept: 'application/json' } }, (res) => {
+        if (res.statusCode !== 200)
+            return;
+        let body = '';
+        res.on('data', (chunk) => { body += chunk.toString(); });
+        res.on('end', () => {
+            try {
+                const data = JSON.parse(body);
+                const latest = data.version;
+                if (latest && latest !== currentVersion && isNewer(latest, currentVersion)) {
+                    console.log(`\n  Update available: ${currentVersion} → \x1b[32m${latest}\x1b[0m` +
+                        `\n  Run \x1b[36mnpm install -g model-test-bench\x1b[0m to update\n`);
+                }
+            }
+            catch { /* ignore parse errors */ }
+        });
+    });
+    req.on('error', () => { });
+    req.setTimeout(3000, () => req.destroy());
+}
+//# sourceMappingURL=update-checker.js.map

package/dist/src/server/services/update-checker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"update-checker.js","sourceRoot":"","sources":["../../../../src/server/services/update-checker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,YAAY,CAAC;AAE/B,MAAM,UAAU,OAAO,CAAC,MAAc,EAAE,OAAe;IACrD,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACxC,MAAM,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3B,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAAE,OAAO,IAAI,CAAC;QAC3C,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAAE,OAAO,KAAK,CAAC;IAC9C,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,cAAsB;IACnD,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CACnB,oDAAoD,EACpD,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,MAAM,EAAE,kBAAkB,EAAE,EAAE,EAC1D,CAAC,GAAG,EAAE,EAAE;QACN,IAAI,GAAG,CAAC,UAAU,KAAK,GAAG;YAAE,OAAO;QACnC,IAAI,IAAI,GAAG,EAAE,CAAC;QACd,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE,GAAG,IAAI,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACjE,GAAG,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;YACjB,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAyB,CAAC;gBACtD,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC;gBAC5B,IAAI,MAAM,IAAI,MAAM,KAAK,cAAc,IAAI,OAAO,CAAC,MAAM,EAAE,cAAc,CAAC,EAAE,CAAC;oBAC3E,OAAO,CAAC,GAAG,CACT,yBAAyB,cAAc,cAAc,MAAM,SAAS;wBACpE,oEAAoE,CACrE,CAAC;gBACJ,CAAC;YACH,CAAC;YAAC,MAAM,CAAC,CAAC,yBAAyB,CAAC,CAAC;QACvC,CAAC,CAAC,CAAC;IACL,CAAC,CACF,CAAC;IACF,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,GAA6C,CAAC,CAAC,CAAC;IACrE,GAAG,CAAC,UAAU,CAAC,IAAI,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;AAC5C,CAAC"}

package/dist/src/server/types/evaluation.d.ts

@@ -0,0 +1,94 @@
+/** Configuration for an evaluator agent. */
+export interface EvaluatorConfig {
+    readonly providerName: string;
+    readonly model: string;
+    readonly apiKey: string;
+    readonly baseUrl?: string;
+    readonly role: string;
+}
+/** Request to start an evaluation pipeline. */
+export interface EvaluationRequest {
+    readonly runId: string;
+    readonly evaluators: readonly EvaluatorConfig[];
+    readonly maxRounds: number;
+    readonly maxBudgetUsd?: number;
+}
+/** Lifecycle status of an evaluation. */
+export type EvaluationStatus = 'pending' | 'running' | 'completed' | 'failed';
+/** How well the answer matches the expected answer. */
+export interface AnswerComparison {
+    readonly matches: boolean;
+    readonly explanation: string;
+    readonly similarity: number;
+}
+/** Result for a single critical requirement. */
+export interface CriticalPartResult {
+    readonly requirement: string;
+    readonly met: boolean;
+    readonly evidence: string;
+}
+/** Per-dimension score from one evaluator. */
+export interface IndividualEvaluation {
+    readonly evaluatorRole: string;
+    readonly dimension: string;
+    readonly score: number;
+    readonly reasoning: string;
+}
+/** Did the agent follow the instructions in its system prompt? */
+export interface InstructionCompliance {
+    readonly followed: readonly string[];
+    readonly violated: readonly string[];
+    readonly notApplicable: readonly string[];
+    readonly overallCompliance: number;
+}
+/** Aggregated report on instruction compliance. */
+export interface SetupComplianceReport {
+    readonly instructionCompliance: InstructionCompliance;
+}
+/** Ledger tracking cost/usage per evaluator. */
+export interface EvaluatorLedger {
+    readonly evaluatorRole: string;
+    readonly totalCostUsd: number;
+    readonly totalTokensIn: number;
+    readonly totalTokensOut: number;
+    readonly roundsParticipated: number;
+}
+/** A single round of evaluation (there may be multiple rounds for consensus). */
+export interface EvaluationRound {
+    readonly roundNumber: number;
+    readonly evaluations: readonly IndividualEvaluation[];
+    readonly consensusReached: boolean;
+    readonly timestamp: string;
+}
+/** Synthesised final scores across evaluators and rounds. */
+export interface EvaluationSynthesis {
+    readonly dimensionScores: Readonly<Record<string, number>>;
+    readonly weightedTotal: number;
+    readonly confidence: number;
+    readonly dissenting: readonly string[];
+}
+/** Overall report on how effective a setup is across evaluations. */
+export interface SetupEffectivenessReport {
+    readonly setupId: string;
+    readonly averageScore: number;
+    readonly scenarioBreakdown: Readonly<Record<string, number>>;
+    readonly strengths: readonly string[];
+    readonly weaknesses: readonly string[];
+}
+/** The full evaluation record persisted after the pipeline completes. */
+export interface Evaluation {
+    readonly id: string;
+    readonly runId: string;
+    readonly status: EvaluationStatus;
+    readonly evaluators: readonly EvaluatorConfig[];
+    readonly rounds: readonly EvaluationRound[];
+    readonly answerComparison: AnswerComparison;
+    readonly criticalResults: readonly CriticalPartResult[];
+    readonly setupCompliance: SetupComplianceReport;
+    readonly synthesis: EvaluationSynthesis;
+    readonly ledger: readonly EvaluatorLedger[];
+    readonly totalCostUsd: number;
+    readonly createdAt: string;
+    readonly updatedAt: string;
+}
+//# sourceMappingURL=evaluation.d.ts.map

package/dist/src/server/types/evaluation.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluation.d.ts","sourceRoot":"","sources":["../../../../src/server/types/evaluation.ts"],"names":[],"mappings":"AAIA,4CAA4C;AAC5C,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACvB;AAED,+CAA+C;AAC/C,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,UAAU,EAAE,SAAS,eAAe,EAAE,CAAC;IAChD,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;CAChC;AAED,yCAAyC;AACzC,MAAM,MAAM,gBAAgB,GAAG,SAAS,GAAG,SAAS,GAAG,WAAW,GAAG,QAAQ,CAAC;AAE9E,uDAAuD;AACvD,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC7B;AAED,gDAAgD;AAChD,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC;IACtB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;CAC3B;AAED,8CAA8C;AAC9C,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B;AAED,kEAAkE;AAClE,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAC;IACrC,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAC;IACrC,QAAQ,CAAC,aAAa,EAAE,SAAS,MAAM,EAAE,CAAC;IAC1C,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;CACpC;AAED,mDAAmD;AACnD,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,qBAAqB,EAAE,qBAAqB,CAAC;CACvD;AAED,gDAAgD;AAChD,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;CACrC;AAED,iFAAiF;AACjF,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,WAAW,EAAE,SAAS,oBAAoB,EAAE,CAAC;IACtD,QAAQ,CAAC,gBAAgB,EAAE,OAAO,CAAC;IACnC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B;AAED,6DAA6D;AAC7D,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,eAAe,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAC3D,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,SAAS,MAAM,EAAE,CAAC;CACxC;AAED,qEAAqE;AACrE,MAAM,WAAW,wBAAwB;IACvC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,iBAAiB,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAC7D,QAAQ,CAAC,SAAS,EAAE,SAAS,MAAM,EAAE,CAAC;IACtC,QAAQ,CAAC,UAAU,EAAE,SAAS,MAAM,EAAE,CAAC;CACxC;AAED,yEAAyE;AACzE,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,gBAAgB,CAAC;IAClC,QAAQ,CAAC,UAAU,EAAE,SAAS,eAAe,EAAE,CAAC;IAChD,QAAQ,CAAC,MAAM,EAAE,SAAS,eAAe,EAAE,CAAC;IAC5C,QAAQ,CAAC,gBAAgB,EAAE,gBAAgB,CAAC;IAC5C,QAAQ,CAAC,eAAe,EAAE,SAAS,kBAAkB,EAAE,CAAC;IACxD,QAAQ,CAAC,eAAe,EAAE,qBAAqB,CAAC;IAChD,QAAQ,CAAC,SAAS,EAAE,mBAAmB,CAAC;IACxC,QAAQ,CAAC,MAAM,EAAE,SAAS,eAAe,EAAE,CAAC;IAC5C,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B"}

package/dist/src/server/types/evaluation.js

@@ -0,0 +1,5 @@
+// ---------------------------------------------------------------------------
+// Evaluation Types
+// ---------------------------------------------------------------------------
+export {};
+//# sourceMappingURL=evaluation.js.map

package/dist/src/server/types/evaluation.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluation.js","sourceRoot":"","sources":["../../../../src/server/types/evaluation.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,mBAAmB;AACnB,8EAA8E"}

package/dist/src/server/types/index.d.ts

@@ -0,0 +1,5 @@
+export type { Provider, ScoringDimension, } from './provider.js';
+export type { Scenario, ScenarioCategory } from './scenario.js';
+export type { Run, RunStatus, SDKMessageRecord } from './run.js';
+export type { AnswerComparison, CriticalPartResult, Evaluation, EvaluationRequest, EvaluationRound, EvaluationStatus, EvaluationSynthesis, EvaluatorConfig, EvaluatorLedger, IndividualEvaluation, InstructionCompliance, SetupComplianceReport, SetupEffectivenessReport, } from './evaluation.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/src/server/types/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/server/types/index.ts"],"names":[],"mappings":"AAIA,YAAY,EACV,QAAQ,EACR,gBAAgB,GACjB,MAAM,eAAe,CAAC;AAEvB,YAAY,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAEhE,YAAY,EAAE,GAAG,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAEjE,YAAY,EACV,gBAAgB,EAChB,kBAAkB,EAClB,UAAU,EACV,iBAAiB,EACjB,eAAe,EACf,gBAAgB,EAChB,mBAAmB,EACnB,eAAe,EACf,eAAe,EACf,oBAAoB,EACpB,qBAAqB,EACrB,qBAAqB,EACrB,wBAAwB,GACzB,MAAM,iBAAiB,CAAC"}

package/dist/src/server/types/index.js

@@ -0,0 +1,5 @@
+// ---------------------------------------------------------------------------
+// Central re-export for all shared types
+// ---------------------------------------------------------------------------
+export {};
+//# sourceMappingURL=index.js.map

package/dist/src/server/types/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/server/types/index.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,yCAAyC;AACzC,8EAA8E"}

package/dist/src/server/types/provider.d.ts

@@ -0,0 +1,23 @@
+/** Scoring dimension used to evaluate a run. */
+export interface ScoringDimension {
+    readonly name: string;
+    readonly weight: number;
+    readonly description: string;
+}
+/** A provider configuration that defines how to connect to an LLM provider. */
+export interface Provider {
+    readonly id: string;
+    readonly name: string;
+    readonly description: string;
+    readonly providerName: string;
+    readonly model: string;
+    readonly apiKey: string;
+    readonly baseUrl?: string;
+    readonly temperature?: number;
+    readonly maxTokens?: number;
+    readonly topP?: number;
+    readonly timeoutSeconds: number;
+    readonly createdAt: string;
+    readonly updatedAt: string;
+}
+//# sourceMappingURL=provider.d.ts.map

package/dist/src/server/types/provider.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"provider.d.ts","sourceRoot":"","sources":["../../../../src/server/types/provider.ts"],"names":[],"mappings":"AAIA,gDAAgD;AAChD,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAC9B;AAED,+EAA+E;AAC/E,MAAM,WAAW,QAAQ;IACvB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B"}

package/dist/src/server/types/provider.js

@@ -0,0 +1,5 @@
+// ---------------------------------------------------------------------------
+// Provider Types
+// ---------------------------------------------------------------------------
+export {};
+//# sourceMappingURL=provider.js.map

package/dist/src/server/types/provider.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"provider.js","sourceRoot":"","sources":["../../../../src/server/types/provider.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,iBAAiB;AACjB,8EAA8E"}

package/dist/src/server/types/run.d.ts

@@ -0,0 +1,31 @@
+import type { Provider } from './provider.js';
+import type { Scenario } from './scenario.js';
+/** A raw message captured during a run (provider-agnostic). */
+export interface SDKMessageRecord {
+    readonly timestamp: string;
+    readonly message: Readonly<Record<string, unknown>>;
+}
+/** Lifecycle status of a run. */
+export type RunStatus = 'pending' | 'running' | 'completed' | 'failed' | 'cancelled';
+/** A single run: one provider + one scenario + captured output. */
+export interface Run {
+    readonly id: string;
+    readonly providerId: string;
+    readonly scenarioId: string;
+    readonly status: RunStatus;
+    readonly providerSnapshot: Provider;
+    readonly scenarioSnapshot: Scenario;
+    readonly messages: readonly SDKMessageRecord[];
+    readonly resultText: string;
+    readonly totalCostUsd: number;
+    readonly durationMs: number;
+    readonly numTurns: number;
+    readonly error?: string;
+    readonly reviewerProviderIds?: readonly string[];
+    readonly reviewerProviderSnapshots?: readonly Provider[];
+    readonly maxEvalRounds?: number;
+    readonly evaluationId?: string;
+    readonly createdAt: string;
+    readonly updatedAt: string;
+}
+//# sourceMappingURL=run.d.ts.map

package/dist/src/server/types/run.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../../src/server/types/run.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAE9C,+DAA+D;AAC/D,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;CACrD;AAED,iCAAiC;AACjC,MAAM,MAAM,SAAS,GAAG,SAAS,GAAG,SAAS,GAAG,WAAW,GAAG,QAAQ,GAAG,WAAW,CAAC;AAErF,mEAAmE;AACnE,MAAM,WAAW,GAAG;IAClB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAC;IAC3B,QAAQ,CAAC,gBAAgB,EAAE,QAAQ,CAAC;IACpC,QAAQ,CAAC,gBAAgB,EAAE,QAAQ,CAAC;IACpC,QAAQ,CAAC,QAAQ,EAAE,SAAS,gBAAgB,EAAE,CAAC;IAC/C,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAExB,QAAQ,CAAC,mBAAmB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACjD,QAAQ,CAAC,yBAAyB,CAAC,EAAE,SAAS,QAAQ,EAAE,CAAC;IACzD,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B"}

package/dist/src/server/types/run.js

@@ -0,0 +1,5 @@
+// ---------------------------------------------------------------------------
+// Run Types
+// ---------------------------------------------------------------------------
+export {};
+//# sourceMappingURL=run.js.map

package/dist/src/server/types/run.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"run.js","sourceRoot":"","sources":["../../../../src/server/types/run.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,YAAY;AACZ,8EAA8E"}

package/dist/src/server/types/scenario.d.ts

@@ -0,0 +1,19 @@
+import type { ScoringDimension } from './provider.js';
+/** Built-in category labels for scenarios (behavior/planning focused). */
+export type ScenarioCategory = 'planning' | 'instruction-following' | 'reasoning' | 'tool-strategy' | 'error-handling' | 'ambiguity-handling' | 'scope-management' | 'custom';
+/** A scenario defines what to test and how to grade. */
+export interface Scenario {
+    readonly id: string;
+    readonly name: string;
+    readonly category: ScenarioCategory;
+    readonly prompt: string;
+    readonly systemPrompt: string;
+    readonly enabledTools: readonly string[];
+    readonly expectedAnswer: string;
+    readonly criticalRequirements: readonly string[];
+    readonly gradingGuidelines: string;
+    readonly scoringDimensions: readonly ScoringDimension[];
+    readonly createdAt: string;
+    readonly updatedAt: string;
+}
+//# sourceMappingURL=scenario.d.ts.map

package/dist/src/server/types/scenario.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scenario.d.ts","sourceRoot":"","sources":["../../../../src/server/types/scenario.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAEtD,0EAA0E;AAC1E,MAAM,MAAM,gBAAgB,GACxB,UAAU,GACV,uBAAuB,GACvB,WAAW,GACX,eAAe,GACf,gBAAgB,GAChB,oBAAoB,GACpB,kBAAkB,GAClB,QAAQ,CAAC;AAEb,wDAAwD;AACxD,MAAM,WAAW,QAAQ;IACvB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,QAAQ,EAAE,gBAAgB,CAAC;IAEpC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAE9B,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IAEzC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,oBAAoB,EAAE,SAAS,MAAM,EAAE,CAAC;IACjD,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,iBAAiB,EAAE,SAAS,gBAAgB,EAAE,CAAC;IACxD,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B"}

package/dist/src/server/types/scenario.js

@@ -0,0 +1,5 @@
+// ---------------------------------------------------------------------------
+// Scenario Types
+// ---------------------------------------------------------------------------
+export {};
+//# sourceMappingURL=scenario.js.map

package/dist/src/server/types/scenario.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scenario.js","sourceRoot":"","sources":["../../../../src/server/types/scenario.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,iBAAiB;AACjB,8EAA8E"}