@huydao/karrot 0.1.4 → 0.1.5

package/dist/assertions/assertion.js

@@ -225,7 +225,7 @@ function evaluateToolCallWithContentAssertion(assertion, logContent) {
         .filter((value) => typeof value === 'string')
         .map((value) => value.trim())
         .filter(Boolean);
-    const matchingToolCallIds = new Set();
+    const toolCallNamesById = new Map();
     const eventPayloadsByToolCallId = new Map();
     const parsedArgumentsByToolCallId = new Map();
     for (const rawLine of logContent.split('\n')) {
@@ -238,25 +238,23 @@ function evaluateToolCallWithContentAssertion(assertion, logContent) {
             if (!event.toolCallId) {
                 continue;
             }
-            if (event.toolCallName === expected.name) {
-                matchingToolCallIds.add(event.toolCallId);
+            eventPayloadsByToolCallId.set(event.toolCallId, [
+                ...(eventPayloadsByToolCallId.get(event.toolCallId) ?? []),
+                line,
+            ]);
+            if (typeof event.toolCallName === 'string' && event.toolCallName.trim()) {
+                toolCallNamesById.set(event.toolCallId, event.toolCallName.trim());
             }
-            if (matchingToolCallIds.has(event.toolCallId)) {
-                eventPayloadsByToolCallId.set(event.toolCallId, [
-                    ...(eventPayloadsByToolCallId.get(event.toolCallId) ?? []),
-                    line,
-                ]);
-                if (typeof event.arguments === 'string' && event.arguments.trim()) {
-                    try {
-                        const parsedArguments = JSON.parse(event.arguments);
-                        parsedArgumentsByToolCallId.set(event.toolCallId, [
-                            ...(parsedArgumentsByToolCallId.get(event.toolCallId) ?? []),
-                            parsedArguments,
-                        ]);
-                    }
-                    catch {
-                        // Ignore unparsable arguments and fall back to raw text matching.
-                    }
+            if (typeof event.arguments === 'string' && event.arguments.trim()) {
+                try {
+                    const parsedArguments = JSON.parse(event.arguments);
+                    parsedArgumentsByToolCallId.set(event.toolCallId, [
+                        ...(parsedArgumentsByToolCallId.get(event.toolCallId) ?? []),
+                        parsedArguments,
+                    ]);
+                }
+                catch {
+                    // Ignore unparsable arguments and fall back to raw text matching.
                 }
             }
         }
@@ -264,7 +262,10 @@ function evaluateToolCallWithContentAssertion(assertion, logContent) {
             continue;
         }
     }
-    if (matchingToolCallIds.size === 0) {
+    const matchingToolCallIds = [...toolCallNamesById.entries()]
+        .filter(([, toolCallName]) => toolCallName === expected.name)
+        .map(([toolCallId]) => toolCallId);
+    if (matchingToolCallIds.length === 0) {
         return {
             kind: assertion.kind,
             matcher: assertion.matcher,
@@ -274,12 +275,12 @@ function evaluateToolCallWithContentAssertion(assertion, logContent) {
             reason: `${buildNormalizedAssertionReason(assertion)} failed. Tool call "${expected.name}" was not found in the run log.`,
         };
     }
-    const combinedPayload = [...matchingToolCallIds]
+    const combinedPayload = matchingToolCallIds
         .flatMap((toolCallId) => eventPayloadsByToolCallId.get(toolCallId) ?? [])
         .join('\n');
     const missingTexts = expectedTexts.filter((text) => !combinedPayload.includes(text));
     const hasPropertiesMatch = expected.hasProperties === undefined ||
-        [...matchingToolCallIds].some((toolCallId) => (parsedArgumentsByToolCallId.get(toolCallId) ?? []).some((parsedArguments) => matchesExpectedProperties(parsedArguments, expected.hasProperties)));
+        matchingToolCallIds.some((toolCallId) => (parsedArgumentsByToolCallId.get(toolCallId) ?? []).some((parsedArguments) => matchesExpectedProperties(parsedArguments, expected.hasProperties)));
     const passed = missingTexts.length === 0 && hasPropertiesMatch;
     return {
         kind: assertion.kind,

package/dist/executors/execute.d.ts

@@ -3,6 +3,9 @@ import { type ScenarioRunResult } from '../reports/report';
 type ExecuteOptions = {
     variables?: Record<string, unknown>;
     scenario: KarrotScenarioSelection;
+    execution?: {
+        concurrency?: number;
+    };
 };
 type ExecuteResult = {
     outputDirectory: string;

package/dist/executors/execute.js

@@ -122,6 +122,7 @@ async function execute(configOrPath, options) {
             ? createAgUiRunner(resolvedConfig)
             : createAgUiPostRunner(resolvedConfig),
         stopOnFailure: resolvedConfig.execution?.stopOnFailure ?? false,
+        concurrency: options.execution?.concurrency ?? resolvedConfig.execution?.concurrency,
     });
     let reportPaths;
     if (resolvedConfig.report?.enabled !== false && resolvedConfig.report) {

package/dist/executors/executor.d.ts

@@ -5,6 +5,7 @@ type BaseScenarioExecutionOptions = {
     env: NodeJS.ProcessEnv;
     outputDirectory: string;
     stopOnFailure?: boolean;
+    concurrency?: number;
     maxDurationMs?: number;
     initialThreadId?: string;
 };

package/dist/executors/executor.js

@@ -10,6 +10,12 @@ function readPositiveTimeoutMs(value) {
     const parsed = Number(value);
     return Number.isFinite(parsed) && parsed > 0 ? parsed : undefined;
 }
+function normalizeConcurrency(value, totalScenarios) {
+    if (!Number.isFinite(value) || (value ?? 0) <= 1) {
+        return 1;
+    }
+    return Math.min(Math.floor(value), Math.max(1, totalScenarios));
+}
 function resolveTurnProcessTimeoutMs(options) {
     const envOverrideMs = readPositiveTimeoutMs(options.env.AI_TURN_TIMEOUT_MS);
     const requestedMs = typeof envOverrideMs === 'number'
@@ -95,6 +101,7 @@ async function runSingleScenario(scenario, context, env, outputDirectory, deadli
                     toolCalls: run.toolCalls,
                     env: turnEnv,
                     outputDirectory,
+                    outputPath: run.outputPath,
                 });
                 const failedAssertions = assertionResults.filter((assertion) => !assertion.passed);
                 const assertionFailureNote = failedAssertions.length > 0
@@ -126,7 +133,11 @@ async function runSingleScenario(scenario, context, env, outputDirectory, deadli
                 result.turns.push(turnResult);
                 turnRecorded = true;
                 if (assertionFailureNote) {
-                    throw new Error(assertionFailureNote);
+                    result.status = 'FAIL';
+                    result.note = [result.note, assertionFailureNote].filter(Boolean).join(' ') || undefined;
+                    if (!scenario.continueOnAssertionFailure) {
+                        throw new Error(assertionFailureNote);
+                    }
                 }
             }
             catch (error) {
@@ -173,31 +184,61 @@ async function runSingleScenario(scenario, context, env, outputDirectory, deadli
 }
 async function runScenario(scenario, options) {
     const scenarios = Array.isArray(scenario) ? scenario : [scenario];
-    const results = [];
     const shouldStopOnFailure = options.stopOnFailure ?? true;
     const deadlineAt = typeof options.maxDurationMs === 'number' ? Date.now() + options.maxDurationMs : undefined;
-    for (const currentScenario of scenarios) {
+    const concurrency = normalizeConcurrency(options.concurrency, scenarios.length);
+    const runScenarioAtIndex = async (currentScenario) => {
         try {
-            results.push(await runSingleScenario(currentScenario, options.context, options.env, options.outputDirectory, deadlineAt, options.messageRunner, options.initialThreadId));
+            return await runSingleScenario(currentScenario, options.context, options.env, options.outputDirectory, deadlineAt, options.messageRunner, concurrency === 1 ? options.initialThreadId : undefined);
         }
         catch (error) {
             if (error instanceof report_1.ScenarioExecutionError) {
-                results.push(error.result);
-            }
-            else {
-                results.push({
-                    id: currentScenario.id,
-                    name: currentScenario.name,
-                    status: 'FAIL',
-                    note: error instanceof Error ? error.message : String(error),
-                    turns: [],
-                    metrics: {},
-                });
+                return error.result;
             }
-            if (shouldStopOnFailure) {
+            return {
+                id: currentScenario.id,
+                name: currentScenario.name,
+                status: 'FAIL',
+                note: error instanceof Error ? error.message : String(error),
+                turns: [],
+                metrics: {},
+            };
+        }
+    };
+    if (concurrency === 1) {
+        const results = [];
+        for (const currentScenario of scenarios) {
+            const result = await runScenarioAtIndex(currentScenario);
+            results.push(result);
+            if (shouldStopOnFailure && result.status === 'FAIL') {
                 break;
             }
         }
+        return Array.isArray(scenario) ? results : results[0];
     }
-    return Array.isArray(scenario) ? results : results[0];
+    const results = new Array(scenarios.length);
+    let nextIndex = 0;
+    let stopScheduling = false;
+    const worker = async () => {
+        while (true) {
+            if (shouldStopOnFailure && stopScheduling) {
+                return;
+            }
+            const currentIndex = nextIndex;
+            nextIndex += 1;
+            if (currentIndex >= scenarios.length) {
+                return;
+            }
+            const result = await runScenarioAtIndex(scenarios[currentIndex]);
+            results[currentIndex] = result;
+            if (shouldStopOnFailure && result.status === 'FAIL') {
+                stopScheduling = true;
+            }
+        }
+    };
+    await Promise.all(Array.from({ length: concurrency }, async () => {
+        await worker();
+    }));
+    const completedResults = results.filter((result) => result != null);
+    return Array.isArray(scenario) ? completedResults : completedResults[0];
 }

package/dist/reports/report.js

@@ -144,7 +144,7 @@ function renderAssertionExpected(expected) {
         return expected.length > 0 ? escapeHtml(expected.join(', ')) : '<span class="muted">[]</span>';
     }
     if (typeof expected === 'object' && expected !== null) {
-        return `<pre>${escapeHtml(stringifyAssertionExpected(expected))}</pre>`;
+        return `<pre class="assertion-expected-object">${escapeHtml(stringifyAssertionExpected(expected))}</pre>`;
     }
     return escapeHtml(expected);
 }
@@ -312,11 +312,13 @@ function buildScenarioRunHtml(payload) {
         '.assertions{table-layout:fixed;}',
         '.assertions th,.assertions td,.evaluations th,.evaluations td{padding:10px 12px;border-bottom:1px solid #e5edf7;vertical-align:top;text-align:left;}',
         '.assertions th,.evaluations th{font-size:12px;text-transform:uppercase;letter-spacing:.08em;color:#61728d;}',
+        '.assertions td{word-break:break-word;overflow-wrap:anywhere;}',
         '.assertions th:nth-child(1),.assertions td:nth-child(1){width:8%;}',
         '.assertions th:nth-child(2),.assertions td:nth-child(2){width:12%;}',
         '.assertions th:nth-child(3),.assertions td:nth-child(3){width:34%;}',
         '.assertions th:nth-child(4),.assertions td:nth-child(4){width:16%;}',
         '.assertions th:nth-child(5),.assertions td:nth-child(5){width:30%;}',
+        '.assertions .assertion-expected-object{margin:0;white-space:pre-wrap;word-break:break-word;overflow-wrap:anywhere;background:#f8fbff;border-radius:12px;padding:12px;font-size:12px;line-height:1.45;max-height:none;overflow:visible;}',
         '.badge{display:inline-flex;align-items:center;justify-content:center;border-radius:999px;padding:5px 10px;font-size:12px;font-weight:700;min-width:56px;}',
         '.badge.pass{background:#dcfce7;color:#166534;}',
         '.badge.fail{background:#fee2e2;color:#991b1b;}',

package/dist/scenarios/scenario.d.ts

@@ -51,6 +51,7 @@ export type AiTurn<TContext extends BaseAiScenarioContext = BaseAiScenarioContex
 export type AiScenario<TContext extends BaseAiScenarioContext = BaseAiScenarioContext> = {
     id: string;
     name: string;
+    continueOnAssertionFailure?: boolean;
     turns: AiTurn<TContext>[];
 };
 export declare class AiScenarioSet<TContext extends BaseAiScenarioContext = BaseAiScenarioContext> {

package/dist/utils/config.d.ts

@@ -79,6 +79,7 @@ export type KarrotConfig = {
     };
     execution?: {
         stopOnFailure?: boolean;
+        concurrency?: number;
     };
     evaluation?: {
         systemPromptPath?: string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@huydao/karrot",
-  "version": "0.1.4",
+  "version": "0.1.5",
   "description": "Reusable AI scenario execution, assertion, evaluation, and reporting toolkit",
   "license": "ISC",
   "type": "commonjs",