@huydao/karrot 0.1.5 → 0.1.7

package/dist/executors/adapters/ag-ui-post.js

@@ -258,6 +258,35 @@ function parseSseBlock(block) {
         data: dataLines.join('\n'),
     };
 }
+function normalizeEventTimestamp(value) {
+    if (typeof value === 'number' && Number.isFinite(value)) {
+        return value > 10_000_000_000 ? value : value * 1000;
+    }
+    if (typeof value === 'string' && value.trim()) {
+        const numeric = Number(value);
+        if (Number.isFinite(numeric)) {
+            return numeric > 10_000_000_000 ? numeric : numeric * 1000;
+        }
+        const parsed = Date.parse(value);
+        return Number.isFinite(parsed) ? parsed : undefined;
+    }
+    return undefined;
+}
+function roundSeconds(startTimeMs, endTimeMs) {
+    if (typeof startTimeMs !== 'number' || typeof endTimeMs !== 'number' || endTimeMs < startTimeMs) {
+        return undefined;
+    }
+    return Number(((endTimeMs - startTimeMs) / 1000).toFixed(1));
+}
+function isAssistantTextEvent(event) {
+    return (event.type === 'TEXT_MESSAGE_CONTENT' ||
+        event.type === 'TEXT_MESSAGE_CHUNK' ||
+        (event.type === 'CUSTOM' && event.name === 'super-testing-agent.model_stream_chunk'));
+}
+function isToolStartEvent(event) {
+    return (event.type === 'TOOL_CALL_START' ||
+        (event.type === 'CUSTOM' && event.name === 'super-testing-agent.tool_started'));
+}
 function createConnectCollector(options) {
     const assistantFragments = [];
     const toolCalls = [];
@@ -266,6 +295,11 @@ function createConnectCollector(options) {
     let started = false;
     let finished = false;
     let sawAnyEvent = false;
+    let runStartedAt;
+    let firstTextAt;
+    let firstToolAt;
+    let runFinishedAt;
+    const eventTime = (event) => normalizeEventTimestamp(event.timestamp) ?? Date.now();
     const consume = (sseEvent) => {
         if (!sseEvent.data) {
             return false;
@@ -286,6 +320,7 @@ function createConnectCollector(options) {
         }
         if (parsed.type === 'RUN_STARTED' && parsed.runId === options.targetRunId) {
             started = true;
+            runStartedAt = eventTime(parsed);
             return false;
         }
         if (!started) {
@@ -300,7 +335,8 @@ function createConnectCollector(options) {
             });
         }
         const isAssistantMessage = parsed.role === 'assistant' || typeof parsed.role !== 'string';
-        if ((parsed.type === 'TEXT_MESSAGE_CHUNK' || parsed.type === 'TEXT_MESSAGE_CONTENT') && isAssistantMessage) {
+        if (isAssistantTextEvent(parsed) && isAssistantMessage) {
+            firstTextAt ??= eventTime(parsed);
             if (typeof parsed.content === 'string' && parsed.content.trim()) {
                 latestAssistantContent = parsed.content.trim();
             }
@@ -308,11 +344,15 @@ function createConnectCollector(options) {
                 assistantFragments.push(parsed.delta);
             }
         }
-        if (parsed.type === 'TOOL_CALL_START' && typeof parsed.toolCallName === 'string' && parsed.toolCallName.trim()) {
-            toolCalls.push(parsed.toolCallName.trim());
+        if (isToolStartEvent(parsed)) {
+            firstToolAt ??= eventTime(parsed);
+            if (typeof parsed.toolCallName === 'string' && parsed.toolCallName.trim()) {
+                toolCalls.push(parsed.toolCallName.trim());
+            }
         }
         if (parsed.type === 'RUN_FINISHED' && (!parsed.runId || parsed.runId === options.targetRunId)) {
             finished = true;
+            runFinishedAt = eventTime(parsed);
             return true;
         }
         return false;
@@ -320,12 +360,19 @@ function createConnectCollector(options) {
     return {
         consume,
         getResult() {
+            const turnCompleteSeconds = roundSeconds(runStartedAt, runFinishedAt);
             return {
                 output: latestAssistantContent?.trim() || assistantFragments.join('').trim(),
                 toolCalls: [...toolCalls],
                 threadId: resolvedThreadId,
                 finished,
                 sawAnyEvent,
+                metrics: {
+                    ttfTextSeconds: roundSeconds(runStartedAt, firstTextAt),
+                    ttfToolSeconds: roundSeconds(runStartedAt, firstToolAt),
+                    turnCompleteSeconds,
+                    totalSeconds: turnCompleteSeconds,
+                },
             };
         },
     };
@@ -337,6 +384,7 @@ async function postAndCaptureResponse(options) {
         : undefined;
     try {
         await promises_1.default.writeFile(options.outputPath, '', 'utf8');
+        const startedAtMs = Date.now();
         const response = await fetch(options.url, {
             method: 'POST',
             headers: {
@@ -369,6 +417,8 @@ async function postAndCaptureResponse(options) {
         return {
             status: response.status,
             rawContent,
+            startedAtMs,
+            finishedAtMs: Date.now(),
         };
     }
     catch (error) {
@@ -430,6 +480,7 @@ function startConnectStream(options) {
                     threadId: parsed.threadId,
                     finished: parsed.finished,
                     sawAnyEvent: parsed.sawAnyEvent,
+                    metrics: parsed.metrics,
                 };
             }
             const reader = response.body.getReader();
@@ -495,13 +546,17 @@ function startConnectStream(options) {
         result,
     };
 }
-function extractResultFromSse(rawContent, fallbackThreadId) {
+function extractResultFromSse(rawContent, fallbackThreadId, fallbackMetrics = {}) {
     const fragments = [];
     let latestContent;
     let resolvedThreadId = fallbackThreadId;
     const toolCalls = [];
     let finished = false;
     let sawAnyEvent = false;
+    let runStartedAt;
+    let firstTextAt;
+    let firstToolAt;
+    let runFinishedAt;
     for (const sseEvent of parseSseEvents(rawContent)) {
         if (!sseEvent.data) {
             continue;
@@ -515,7 +570,11 @@ function extractResultFromSse(rawContent, fallbackThreadId) {
             else if (typeof parsed.conversationId === 'string' && parsed.conversationId.trim()) {
                 resolvedThreadId = parsed.conversationId.trim();
             }
-            if (parsed.type === 'TEXT_MESSAGE_CONTENT') {
+            if (parsed.type === 'RUN_STARTED' && typeof runStartedAt !== 'number') {
+                runStartedAt = normalizeEventTimestamp(parsed.timestamp);
+            }
+            if (isAssistantTextEvent(parsed)) {
+                firstTextAt ??= normalizeEventTimestamp(parsed.timestamp);
                 if (typeof parsed.content === 'string' && parsed.content.trim()) {
                     latestContent = parsed.content.trim();
                 }
@@ -523,11 +582,15 @@ function extractResultFromSse(rawContent, fallbackThreadId) {
                     fragments.push(parsed.delta);
                 }
             }
-            if (parsed.type === 'TOOL_CALL_START' && typeof parsed.toolCallName === 'string' && parsed.toolCallName.trim()) {
-                toolCalls.push(parsed.toolCallName.trim());
+            if (isToolStartEvent(parsed)) {
+                firstToolAt ??= normalizeEventTimestamp(parsed.timestamp);
+                if (typeof parsed.toolCallName === 'string' && parsed.toolCallName.trim()) {
+                    toolCalls.push(parsed.toolCallName.trim());
+                }
             }
             if (parsed.type === 'RUN_FINISHED') {
                 finished = true;
+                runFinishedAt = normalizeEventTimestamp(parsed.timestamp);
             }
             if (parsed.type === 'RUN_ERROR') {
                 throw new run_result_1.MessageRunError(parsed.error?.trim() || 'Agent run failed.', {
@@ -544,12 +607,21 @@ function extractResultFromSse(rawContent, fallbackThreadId) {
             }
         }
     }
+    const effectiveStartedAt = runStartedAt ?? fallbackMetrics.startTimeMs;
+    const effectiveFinishedAt = runFinishedAt ?? fallbackMetrics.finishedTimeMs;
+    const turnCompleteSeconds = roundSeconds(effectiveStartedAt, effectiveFinishedAt);
     return {
         output: latestContent?.trim() || fragments.join('').trim(),
         toolCalls,
         threadId: resolvedThreadId,
         finished,
         sawAnyEvent,
+        metrics: {
+            ttfTextSeconds: roundSeconds(runStartedAt, firstTextAt),
+            ttfToolSeconds: roundSeconds(runStartedAt, firstToolAt),
+            turnCompleteSeconds,
+            totalSeconds: turnCompleteSeconds,
+        },
     };
 }
 async function runAgUiPostMessage(options) {
@@ -625,7 +697,7 @@ async function runAgUiPostMessage(options) {
                 note: `Run log: ${node_path_1.default.basename(runOutputPath)}`,
                 toolCallCount: connected.toolCalls.length,
                 toolCalls: connected.toolCalls,
-                metrics: {},
+                metrics: connected.metrics,
             };
         }
         const runResponse = await postAndCaptureResponse({
@@ -642,7 +714,10 @@ async function runAgUiPostMessage(options) {
                 output: runResponse.rawContent,
             });
         }
-        const parsed = extractResultFromSse(runResponse.rawContent, resolvedThreadId);
+        const parsed = extractResultFromSse(runResponse.rawContent, resolvedThreadId, {
+            startTimeMs: runResponse.startedAtMs,
+            finishedTimeMs: runResponse.finishedAtMs,
+        });
         if (options.observe) {
             await promises_1.default.writeFile(observePath, '', 'utf8');
             const observed = await waitForObservedCompletion({
@@ -659,7 +734,7 @@ async function runAgUiPostMessage(options) {
                 note: `Observe status: ${observed.status}. Observe log: ${node_path_1.default.basename(observePath)}`,
                 toolCallCount: parsed.toolCalls.length,
                 toolCalls: parsed.toolCalls,
-                metrics: {},
+                metrics: parsed.metrics,
             };
         }
         if (options.completionCheck) {
@@ -671,7 +746,7 @@ async function runAgUiPostMessage(options) {
                 note: `Completion status: ${completion.status}`,
                 toolCallCount: parsed.toolCalls.length,
                 toolCalls: parsed.toolCalls,
-                metrics: {},
+                metrics: parsed.metrics,
             };
         }
         if (!parsed.finished && !parsed.output) {
@@ -691,7 +766,7 @@ async function runAgUiPostMessage(options) {
             outputPath,
             toolCallCount: parsed.toolCalls.length,
             toolCalls: parsed.toolCalls,
-            metrics: {},
+            metrics: parsed.metrics,
         };
     }
     catch (error) {

package/dist/executors/adapters/ag-ui.js

@@ -9,8 +9,8 @@ exports.extractAppendedLog = extractAppendedLog;
 exports.runAgUiMessage = runAgUiMessage;
 const promises_1 = __importDefault(require("node:fs/promises"));
 const node_path_1 = __importDefault(require("node:path"));
+const node_crypto_1 = require("node:crypto");
 const stompjs_1 = require("@stomp/stompjs");
-const uuid_1 = require("uuid");
 const ws_1 = __importDefault(require("ws"));
 const run_result_1 = require("../run-result");
 Object.assign(globalThis, { WebSocket: ws_1.default });
@@ -169,6 +169,7 @@ function computeMetrics(state) {
     return {
         ttfToolSeconds,
         ttfTextSeconds,
+        turnCompleteSeconds: totalSeconds,
         totalSeconds,
         protocolUsedKb,
         protocolTotalKb,
@@ -179,6 +180,7 @@ function writeMetricsToStdout(metrics) {
     const parts = [
         metrics.ttfToolSeconds != null ? `TTF-Tool: ${metrics.ttfToolSeconds.toFixed(1)}s` : undefined,
         metrics.ttfTextSeconds != null ? `TTF-Text: ${metrics.ttfTextSeconds.toFixed(1)}s` : undefined,
+        metrics.turnCompleteSeconds != null ? `Turn complete: ${metrics.turnCompleteSeconds.toFixed(1)}s` : undefined,
         metrics.totalSeconds != null ? `Total: ${metrics.totalSeconds.toFixed(1)}s` : undefined,
         metrics.protocolUsedKb != null && metrics.protocolTotalKb != null && metrics.efficiencyPercent != null
             ? `Protocol efficiency: ${metrics.protocolUsedKb.toFixed(1)}KB/${metrics.protocolTotalKb.toFixed(1)}KB (${metrics.efficiencyPercent}%)`
@@ -430,8 +432,8 @@ async function connectAndRun(options) {
 async function runAgUiMessage(options) {
     await promises_1.default.mkdir(options.outputDirectory, { recursive: true });
     const config = parseAgUiEnv(options.env);
-    const threadId = options.threadId ?? options.threadIdFallback ?? (0, uuid_1.v7)();
-    const runId = (0, uuid_1.v7)();
+    const threadId = options.threadId ?? options.threadIdFallback ?? (0, node_crypto_1.randomUUID)();
+    const runId = (0, node_crypto_1.randomUUID)();
     const logPath = node_path_1.default.join(options.outputDirectory, `${threadId}.jsonl`);
     const previousLogContent = await readJsonl(logPath);
     const state = await connectAndRun({

package/dist/executors/executor.js

@@ -135,7 +135,8 @@ async function runSingleScenario(scenario, context, env, outputDirectory, deadli
                 if (assertionFailureNote) {
                     result.status = 'FAIL';
                     result.note = [result.note, assertionFailureNote].filter(Boolean).join(' ') || undefined;
-                    if (!scenario.continueOnAssertionFailure) {
+                    const shouldContinueOnAssertionFailure = turn.continueOnAssertionFailure ?? scenario.continueOnAssertionFailure ?? false;
+                    if (!shouldContinueOnAssertionFailure) {
                         throw new Error(assertionFailureNote);
                     }
                 }

package/dist/executors/run-result.d.ts

@@ -1,7 +1,10 @@
 export type TimingMetrics = {
     ttfToolSeconds?: number;
     ttfTextSeconds?: number;
+    turnCompleteSeconds?: number;
     totalSeconds?: number;
+    averageTtfTextSeconds?: number;
+    averageTurnCompleteSeconds?: number;
     protocolUsedKb?: number;
     protocolTotalKb?: number;
     efficiencyPercent?: number;

package/dist/reports/report.js

@@ -26,6 +26,16 @@ function sumNumbers(values) {
     }
     return Number(definedValues.reduce((total, value) => total + value, 0).toFixed(1));
 }
+function averageNumbers(values) {
+    const definedValues = values.filter((value) => typeof value === 'number');
+    if (definedValues.length === 0) {
+        return undefined;
+    }
+    return Number((definedValues.reduce((total, value) => total + value, 0) / definedValues.length).toFixed(1));
+}
+function turnCompleteSeconds(metrics) {
+    return metrics.turnCompleteSeconds ?? metrics.totalSeconds;
+}
 function summarizeScenarioMetrics(turns) {
     const protocolUsedKb = sumNumbers(turns.map((turn) => turn.metrics.protocolUsedKb));
     const protocolTotalKb = sumNumbers(turns.map((turn) => turn.metrics.protocolTotalKb));
@@ -35,7 +45,10 @@ function summarizeScenarioMetrics(turns) {
     return {
         ttfToolSeconds: sumNumbers(turns.map((turn) => turn.metrics.ttfToolSeconds)),
         ttfTextSeconds: sumNumbers(turns.map((turn) => turn.metrics.ttfTextSeconds)),
+        turnCompleteSeconds: sumNumbers(turns.map((turn) => turnCompleteSeconds(turn.metrics))),
         totalSeconds: sumNumbers(turns.map((turn) => turn.metrics.totalSeconds)),
+        averageTtfTextSeconds: averageNumbers(turns.map((turn) => turn.metrics.ttfTextSeconds)),
+        averageTurnCompleteSeconds: averageNumbers(turns.map((turn) => turnCompleteSeconds(turn.metrics))),
         protocolUsedKb,
         protocolTotalKb,
         efficiencyPercent,
@@ -112,6 +125,8 @@ function buildScenarioRunSummary(results) {
         failedAssertions: assertions.filter((assertion) => !assertion.passed).length,
         totalToolCalls: turns.reduce((total, turn) => total + turn.toolCallCount, 0),
         totalEvaluations: evaluations.length,
+        averageTtfTextSeconds: averageNumbers(turns.map((turn) => turn.metrics.ttfTextSeconds)),
+        averageTurnCompleteSeconds: averageNumbers(turns.map((turn) => turnCompleteSeconds(turn.metrics))),
         averageScoresByDimension,
         requestedEvalDimensions,
     };
@@ -155,7 +170,10 @@ function renderMetrics(metrics) {
     return [
         `TTF Tool: ${formatSeconds(metrics.ttfToolSeconds)}`,
         `TTF Text: ${formatSeconds(metrics.ttfTextSeconds)}`,
+        `Turn Complete: ${formatSeconds(turnCompleteSeconds(metrics))}`,
         `Total: ${formatSeconds(metrics.totalSeconds)}`,
+        `Avg TTF Text: ${formatSeconds(metrics.averageTtfTextSeconds)}`,
+        `Avg Complete: ${formatSeconds(metrics.averageTurnCompleteSeconds)}`,
         `Efficiency: ${formatPercent(metrics.efficiencyPercent)}`,
     ].join(' | ');
 }
@@ -279,60 +297,70 @@ function buildScenarioRunHtml(payload) {
         '<meta name="viewport" content="width=device-width, initial-scale=1" />',
         `<title>${escapeHtml(`${payload.projectName} AI Scenario Report`)}</title>`,
         '<style>',
-        'body{margin:0;font-family:ui-sans-serif,system-ui,-apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif;background:#f4f7fb;color:#142033;}',
-        '.page{max-width:1280px;margin:0 auto;padding:32px 24px 64px;}',
-        '.hero{background:linear-gradient(135deg,#0f172a,#1d4ed8);color:#fff;padding:28px;border-radius:20px;box-shadow:0 20px 50px rgba(15,23,42,.18);}',
-        '.hero h1{margin:0 0 8px;font-size:32px;}',
-        '.hero p{margin:4px 0;color:rgba(255,255,255,.88);}',
-        '.summary-grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(170px,1fr));gap:14px;margin:22px 0 28px;}',
-        '.summary-card,.panel,.turn-card,.scenario-card{background:#fff;border:1px solid #dbe4f0;border-radius:18px;box-shadow:0 10px 30px rgba(15,23,42,.06);}',
-        '.summary-card{padding:18px;}',
-        '.summary-card .label{display:block;font-size:12px;text-transform:uppercase;letter-spacing:.08em;color:#5b6b84;margin-bottom:8px;}',
-        '.summary-card .value{font-size:28px;font-weight:700;}',
-        '.summary-card .sub{font-size:13px;color:#61728d;}',
-        '.panels{display:grid;grid-template-columns:repeat(auto-fit,minmax(280px,1fr));gap:16px;margin-bottom:28px;}',
-        '.panel{padding:18px;}',
-        '.panel h2{margin:0 0 12px;font-size:18px;}',
-        '.panel pre{margin:0;white-space:pre-wrap;word-break:break-word;background:#f8fbff;border-radius:12px;padding:14px;font-size:13px;}',
+        ':root{--ink:#0b1220;--text:#273247;--muted:#6f7b91;--line:#e3ebf5;--panel:#fff;--panel-soft:#f8fafd;--page:#eaf1f8;--blue:#2f63e5;--blue-dark:#153f9f;--green:#3f9a8f;--red:#dc3d4d;--amber:#b7791f;--shadow:0 14px 34px rgba(21,45,85,.08);}',
+        '*{box-sizing:border-box;}',
+        'body{margin:0;font-family:"Avenir Next","Nunito Sans",ui-sans-serif,system-ui,-apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif;background:radial-gradient(circle at 8% -10%,rgba(47,99,229,.13),transparent 26rem),linear-gradient(180deg,#edf4fb 0%,var(--page) 100%);color:var(--text);font-size:15px;line-height:1.55;}',
+        '.page{max-width:1360px;margin:0 auto;padding:24px 18px 56px;}',
+        '.hero,.summary-card,.panel,.turn-card,.scenario-card{position:relative;background:var(--panel);border:1px solid #d8e3f0;border-radius:10px;box-shadow:var(--shadow);overflow:hidden;}',
+        '.hero::before,.summary-card::before,.panel::before,.turn-card::before,.scenario-card::before{content:"";position:absolute;inset:0 0 auto;height:4px;background:linear-gradient(90deg,#071326 0%,var(--blue) 78%,#4b7cff 100%);}',
+        '.hero{display:grid;grid-template-columns:minmax(0,1fr) auto;gap:18px;padding:26px 30px 24px;margin-bottom:22px;}',
+        '.hero h1{grid-column:1/-1;margin:0 0 2px;color:var(--ink);font-size:29px;line-height:1.16;font-weight:700;letter-spacing:-.02em;}',
+        '.hero p{margin:0;color:var(--muted);font-weight:700;}',
+        '.hero strong{color:var(--ink);font-weight:700;}',
+        '.summary-grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(190px,1fr));gap:16px;margin:0 0 18px;}',
+        '.summary-card{padding:22px 22px 18px;min-height:124px;}',
+        '.summary-card .label,.score-card .label,.meta-label{display:block;font-size:11px;text-transform:uppercase;letter-spacing:.06em;color:var(--muted);font-weight:700;margin-bottom:8px;}',
+        '.summary-card .value{font-size:31px;line-height:1.06;font-weight:700;color:var(--ink);letter-spacing:-.025em;}',
+        '.summary-card .sub{margin-top:8px;font-size:14px;color:var(--muted);font-weight:700;}',
+        '.panels{display:grid;grid-template-columns:repeat(auto-fit,minmax(300px,1fr));gap:16px;margin-bottom:18px;}',
+        '.panel{padding:22px;}',
+        '.panel h2{margin:0 0 14px;color:var(--ink);font-size:17px;line-height:1.22;font-weight:700;letter-spacing:-.01em;}',
+        '.panel pre{margin:0;white-space:pre-wrap;word-break:break-word;background:var(--panel-soft);border:1px solid #edf2f8;border-radius:6px;padding:14px;font-size:13px;color:#334155;}',
         '.scenario-list{display:grid;gap:18px;}',
-        '.scenario-card summary{list-style:none;display:flex;gap:12px;align-items:center;justify-content:space-between;padding:18px 20px;cursor:pointer;}',
+        '.scenario-card summary{list-style:none;display:grid;grid-template-columns:minmax(0,1fr) auto minmax(160px,36%);gap:14px;align-items:center;padding:20px 24px 18px;cursor:pointer;}',
         '.scenario-card summary::-webkit-details-marker{display:none;}',
-        '.scenario-title{font-weight:700;font-size:18px;flex:1;}',
-        '.summary-note{color:#61728d;font-size:14px;max-width:40%;text-align:right;}',
-        '.scenario-body{padding:0 20px 20px;}',
-        '.meta-grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(220px,1fr));gap:12px;margin:8px 0 18px;}',
-        '.meta-grid>div{background:#f8fbff;border-radius:12px;padding:12px;}',
-        '.meta-label{display:block;font-size:12px;color:#61728d;text-transform:uppercase;letter-spacing:.08em;margin-bottom:6px;}',
-        '.turn-card{padding:18px;margin-top:16px;}',
-        '.turn-card h4,.content-block h5{margin:0 0 10px;}',
+        '.scenario-title{font-weight:700;font-size:17px;color:var(--ink);letter-spacing:-.01em;overflow-wrap:anywhere;}',
+        '.summary-note{color:var(--muted);font-size:14px;font-weight:400;text-align:right;overflow-wrap:anywhere;}',
+        '.scenario-body{padding:0 24px 24px;}',
+        '.meta-grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(230px,1fr));gap:12px;margin:8px 0 18px;}',
+        '.meta-grid>div{background:var(--panel-soft);border:1px solid #edf2f8;border-radius:6px;padding:13px 14px;}',
+        '.turn-card{padding:22px;margin-top:16px;border-radius:8px;box-shadow:0 10px 24px rgba(21,45,85,.06);}',
+        '.turn-card h4{margin:0 0 14px;color:var(--ink);font-size:16px;font-weight:700;letter-spacing:0;}',
         '.content-block{margin-top:14px;}',
-        '.content-block pre{margin:0;white-space:pre-wrap;word-break:break-word;background:#f8fbff;border-radius:12px;padding:14px;max-height:420px;overflow:auto;}',
-        '.content-block p{margin:0;background:#f8fbff;border-radius:12px;padding:14px;}',
-        '.assertions,.evaluations{width:100%;border-collapse:collapse;font-size:14px;}',
+        '.content-block h5{margin:0 0 8px;color:var(--ink);font-size:12px;text-transform:uppercase;letter-spacing:.06em;font-weight:700;}',
+        '.content-block pre,.content-block p{margin:0;background:var(--panel-soft);border:1px solid #edf2f8;border-radius:6px;padding:14px;}',
+        '.content-block pre{white-space:pre-wrap;word-break:break-word;max-height:420px;overflow:auto;color:#29364b;}',
+        '.assertions,.evaluations{width:100%;border-collapse:separate;border-spacing:0 8px;font-size:14px;}',
         '.assertions{table-layout:fixed;}',
-        '.assertions th,.assertions td,.evaluations th,.evaluations td{padding:10px 12px;border-bottom:1px solid #e5edf7;vertical-align:top;text-align:left;}',
-        '.assertions th,.evaluations th{font-size:12px;text-transform:uppercase;letter-spacing:.08em;color:#61728d;}',
+        '.assertions th,.assertions td,.evaluations th,.evaluations td{padding:10px 12px;vertical-align:top;text-align:left;}',
+        '.assertions th,.evaluations th{font-size:11px;text-transform:uppercase;letter-spacing:.06em;color:var(--muted);font-weight:700;}',
+        '.assertions tbody tr,.evaluations tbody tr{background:var(--panel-soft);}',
+        '.assertions tbody td,.evaluations tbody td{border-top:1px solid #edf2f8;border-bottom:1px solid #edf2f8;}',
+        '.assertions tbody td:first-child,.evaluations tbody td:first-child{border-left:1px solid #edf2f8;border-radius:6px 0 0 6px;}',
+        '.assertions tbody td:last-child,.evaluations tbody td:last-child{border-right:1px solid #edf2f8;border-radius:0 6px 6px 0;}',
         '.assertions td{word-break:break-word;overflow-wrap:anywhere;}',
         '.assertions th:nth-child(1),.assertions td:nth-child(1){width:8%;}',
         '.assertions th:nth-child(2),.assertions td:nth-child(2){width:12%;}',
         '.assertions th:nth-child(3),.assertions td:nth-child(3){width:34%;}',
         '.assertions th:nth-child(4),.assertions td:nth-child(4){width:16%;}',
         '.assertions th:nth-child(5),.assertions td:nth-child(5){width:30%;}',
-        '.assertions .assertion-expected-object{margin:0;white-space:pre-wrap;word-break:break-word;overflow-wrap:anywhere;background:#f8fbff;border-radius:12px;padding:12px;font-size:12px;line-height:1.45;max-height:none;overflow:visible;}',
-        '.badge{display:inline-flex;align-items:center;justify-content:center;border-radius:999px;padding:5px 10px;font-size:12px;font-weight:700;min-width:56px;}',
-        '.badge.pass{background:#dcfce7;color:#166534;}',
-        '.badge.fail{background:#fee2e2;color:#991b1b;}',
-        '.badge.skip{background:#e2e8f0;color:#334155;}',
-        '.dimension-chip{display:inline-flex;align-items:center;justify-content:center;border-radius:999px;padding:4px 10px;margin:0 6px 6px 0;background:#eef2ff;color:#3730a3;font-size:12px;font-weight:600;}',
+        '.assertions .assertion-expected-object{margin:0;white-space:pre-wrap;word-break:break-word;overflow-wrap:anywhere;background:#fff;border:1px solid #e5edf7;border-radius:6px;padding:12px;font-size:12px;line-height:1.45;max-height:none;overflow:visible;}',
+        '.badge{display:inline-flex;align-items:center;justify-content:center;border-radius:999px;padding:5px 11px;font-size:12px;font-weight:900;min-width:58px;letter-spacing:.02em;}',
+        '.badge.pass{background:#e4f8ef;color:#157347;}',
+        '.badge.fail{background:#ffe8eb;color:#b42332;}',
+        '.badge.skip{background:#edf2f7;color:#4a5568;}',
+        '.scenario-card.pass::before{background:linear-gradient(90deg,#071326 0%,#22a06b 100%);}',
+        '.scenario-card.fail::before{background:linear-gradient(90deg,#071326 0%,var(--red) 100%);}',
+        '.scenario-card.skip::before{background:linear-gradient(90deg,#071326 0%,#8a94a6 100%);}',
+        '.dimension-chip{display:inline-flex;align-items:center;justify-content:center;border-radius:999px;padding:5px 11px;margin:0 6px 6px 0;background:#eef4ff;color:var(--blue);font-size:12px;font-weight:900;}',
         '.score-grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(160px,1fr));gap:12px;}',
-        '.score-card{background:#f8fbff;border-radius:12px;padding:14px;}',
-        '.score-card .label{display:block;font-size:12px;color:#61728d;text-transform:uppercase;letter-spacing:.08em;margin-bottom:8px;}',
-        '.score-card .value{font-size:24px;font-weight:700;}',
-        '.score-pill{display:inline-flex;align-items:center;justify-content:center;border-radius:999px;padding:4px 10px;background:#dbeafe;color:#1d4ed8;font-weight:700;min-width:58px;}',
-        '.muted{color:#7c8ba1;}',
+        '.score-card{background:var(--panel-soft);border:1px solid #edf2f8;border-radius:6px;padding:14px;}',
+        '.score-card .value{font-size:23px;font-weight:700;color:var(--ink);letter-spacing:-.015em;}',
+        '.score-pill{display:inline-flex;align-items:center;justify-content:center;border-radius:999px;padding:5px 11px;background:#e8f0ff;color:var(--blue);font-weight:900;min-width:58px;}',
+        '.muted{color:#8a95a8;}',
         'code{font-family:ui-monospace,SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono",monospace;font-size:12px;word-break:break-all;}',
-        '.footer{margin-top:28px;color:#61728d;font-size:13px;}',
-        '@media (max-width:900px){.scenario-card summary{flex-direction:column;align-items:flex-start;}.summary-note{max-width:none;text-align:left;}}',
+        '.footer{margin:24px 4px 0;color:var(--muted);font-size:13px;font-weight:700;}',
+        '@media (max-width:900px){.hero{display:block;padding:24px 20px;}.hero p{margin-top:6px;}.scenario-card summary{grid-template-columns:1fr;align-items:start;}.summary-note{text-align:left;}.page{padding:14px 10px 40px;}.assertions,.evaluations{display:block;overflow-x:auto;white-space:normal;}.summary-card .value{font-size:30px;}}',
         '</style>',
         '</head>',
         '<body>',
@@ -346,6 +374,8 @@ function buildScenarioRunHtml(payload) {
         '<section class="summary-grid">',
         `<article class="summary-card"><span class="label">Scenarios</span><div class="value">${payload.summary.totalScenarios}</div><div class="sub">${payload.summary.passedScenarios} pass / ${payload.summary.failedScenarios} fail / ${payload.summary.skippedScenarios} skip</div></article>`,
         `<article class="summary-card"><span class="label">Turns</span><div class="value">${payload.summary.totalTurns}</div><div class="sub">${payload.summary.totalToolCalls} tool calls total</div></article>`,
+        `<article class="summary-card"><span class="label">Avg TTF Text</span><div class="value">${formatSeconds(payload.summary.averageTtfTextSeconds)}</div><div class="sub">First assistant text per turn</div></article>`,
+        `<article class="summary-card"><span class="label">Avg Turn Complete</span><div class="value">${formatSeconds(payload.summary.averageTurnCompleteSeconds)}</div><div class="sub">Run started to run finished</div></article>`,
         `<article class="summary-card"><span class="label">Assertions</span><div class="value">${payload.summary.totalAssertions}</div><div class="sub">${payload.summary.passedAssertions} pass / ${payload.summary.failedAssertions} fail</div></article>`,
         `<article class="summary-card"><span class="label">Evaluations</span><div class="value">${payload.summary.totalEvaluations}</div><div class="sub">LLM-scored dimensions</div></article>`,
         '</section>',

package/dist/scenarios/scenario.d.ts

@@ -44,6 +44,7 @@ export type AiTurn<TContext extends BaseAiScenarioContext = BaseAiScenarioContex
     message: AiTurnMessage<TContext>;
     idleTimeoutMs?: number;
     processTimeoutMs?: number;
+    continueOnAssertionFailure?: boolean;
     assertions?: AiTurnAssertion[];
     eval?: AiTurnEvalDefinition[];
     onComplete?: (args: AiTurnCompletionArgs<TContext>) => void | Promise<void>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@huydao/karrot",
-  "version": "0.1.5",
+  "version": "0.1.7",
   "description": "Reusable AI scenario execution, assertion, evaluation, and reporting toolkit",
   "license": "ISC",
   "type": "commonjs",
@@ -126,12 +126,15 @@
   },
   "files": [
     "dist",
+    "site",
     "README.md",
     "GUIDE.md"
   ],
   "scripts": {
     "build": "rm -rf dist && tsc -p tsconfig.json && mkdir -p dist/prompts && cp prompts/*.md dist/prompts/",
-    "prepack": "npm run build"
+    "prepack": "npm run build",
+    "site:serve": "node site/serve.js",
+    "site:check": "node site/check.js"
   },
   "dependencies": {
     "@stomp/stompjs": "^7.3.0",