@huydao/karrot 0.1.1

package/dist/executors/executor.js

@@ -0,0 +1,203 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runScenario = runScenario;
+const assertion_1 = require("../assertions/assertion");
+const generated_message_1 = require("../scenarios/generated-message");
+const turn_eval_1 = require("../assertions/turn-eval");
+const report_1 = require("../reports/report");
+const run_result_1 = require("./run-result");
+function readPositiveTimeoutMs(value) {
+    const parsed = Number(value);
+    return Number.isFinite(parsed) && parsed > 0 ? parsed : undefined;
+}
+function resolveTurnProcessTimeoutMs(options) {
+    const envOverrideMs = readPositiveTimeoutMs(options.env.AI_TURN_TIMEOUT_MS);
+    const requestedMs = typeof envOverrideMs === 'number'
+        ? Math.max(options.turn.processTimeoutMs ?? 0, envOverrideMs)
+        : options.turn.processTimeoutMs;
+    if (typeof options.remainingMs !== 'number') {
+        return requestedMs;
+    }
+    return Math.max(1, Math.min(requestedMs ?? options.remainingMs, options.remainingMs));
+}
+async function runSingleScenario(scenario, context, env, outputDirectory, deadlineAt, messageRunner, initialThreadId) {
+    const result = {
+        id: scenario.id,
+        name: scenario.name,
+        status: 'PASS',
+        turns: [],
+        metrics: {},
+    };
+    let threadId = initialThreadId;
+    console.log(`\n=== ${scenario.id} — ${scenario.name} ===`);
+    for (const turn of scenario.turns) {
+        const remainingMs = typeof deadlineAt === 'number' ? deadlineAt - Date.now() : undefined;
+        if (typeof remainingMs === 'number' && remainingMs <= 0) {
+            throw new Error(`Scenario ${scenario.id} exceeded the maximum execution time.`);
+        }
+        const message = await (0, generated_message_1.resolveTurnMessage)({
+            turn,
+            context,
+            env,
+            scenarioId: scenario.id,
+            scenarioName: scenario.name,
+            history: result.turns.map((completedTurn) => ({
+                label: completedTurn.label,
+                message: completedTurn.message,
+                output: completedTurn.output,
+            })),
+        });
+        console.log(`\n--- ${scenario.id} ${turn.label} ---`);
+        const turnEnv = {
+            ...env,
+            ...(turn.idleTimeoutMs ? { IDLE_TIMEOUT: String(turn.idleTimeoutMs) } : {}),
+        };
+        try {
+            if (!messageRunner) {
+                throw new Error('runScenario requires a messageRunner. Provide a transport adapter from the AUT layer.');
+            }
+            const run = await messageRunner({
+                scenario,
+                turn,
+                context,
+                message,
+                env: turnEnv,
+                outputDirectory,
+                threadId,
+                processTimeoutMs: resolveTurnProcessTimeoutMs({
+                    env: turnEnv,
+                    turn,
+                    remainingMs,
+                }),
+            });
+            await turn.onComplete?.({ context, output: run.output });
+            threadId = run.threadId;
+            result.threadId = run.threadId;
+            const turnResult = {
+                label: turn.label,
+                message,
+                evalDimensions: turn.eval
+                    ? turn.eval.map((dimension) => (typeof dimension === 'string' ? dimension : String(dimension.dimension)))
+                    : undefined,
+                threadId: run.threadId,
+                outputPath: run.outputPath,
+                output: run.output,
+                note: run.note,
+                toolCallCount: run.toolCallCount,
+                toolCalls: run.toolCalls,
+                metrics: run.metrics,
+            };
+            let turnRecorded = false;
+            try {
+                const assertionResults = await (0, assertion_1.evaluateTurnAssertions)({
+                    assertions: turn.assertions,
+                    output: run.output,
+                    toolCalls: run.toolCalls,
+                    env: turnEnv,
+                    outputDirectory,
+                });
+                const failedAssertions = assertionResults.filter((assertion) => !assertion.passed);
+                const assertionFailureNote = failedAssertions.length > 0
+                    ? failedAssertions.map((assertion) => assertion.reason).join(' ')
+                    : undefined;
+                const evaluationResults = await (0, turn_eval_1.evaluateTurnEvals)({
+                    dimensions: turn.eval,
+                    scenarioId: scenario.id,
+                    scenarioName: scenario.name,
+                    turnLabel: turn.label,
+                    output: run.output,
+                    env: turnEnv,
+                    history: [
+                        ...result.turns.map((completedTurn) => ({
+                            label: completedTurn.label,
+                            message: completedTurn.message,
+                            output: completedTurn.output,
+                        })),
+                        {
+                            label: turn.label,
+                            message,
+                            output: run.output,
+                        },
+                    ],
+                });
+                turnResult.assertionResults = assertionResults;
+                turnResult.evaluationResults = evaluationResults;
+                turnResult.note = [run.note, assertionFailureNote].filter(Boolean).join(' ') || undefined;
+                result.turns.push(turnResult);
+                turnRecorded = true;
+                if (assertionFailureNote) {
+                    throw new Error(assertionFailureNote);
+                }
+            }
+            catch (error) {
+                turnResult.note = [
+                    turnResult.note,
+                    error instanceof Error ? error.message : String(error),
+                ]
+                    .filter(Boolean)
+                    .join(' ');
+                if (!turnRecorded) {
+                    result.turns.push(turnResult);
+                }
+                throw error;
+            }
+        }
+        catch (error) {
+            if (error instanceof run_result_1.MessageRunError) {
+                result.threadId = error.threadId ?? result.threadId;
+                result.turns.push({
+                    label: turn.label,
+                    message,
+                    evalDimensions: turn.eval
+                        ? turn.eval.map((dimension) => (typeof dimension === 'string' ? dimension : String(dimension.dimension)))
+                        : undefined,
+                    threadId: error.threadId,
+                    outputPath: error.outputPath,
+                    output: error.output,
+                    note: error.message,
+                    toolCallCount: error.toolCallCount,
+                    toolCalls: error.toolCalls,
+                    metrics: error.metrics,
+                });
+            }
+            result.status = 'FAIL';
+            result.note = error instanceof Error ? error.message : String(error);
+            throw new report_1.ScenarioExecutionError((0, report_1.finalizeScenarioResult)(result));
+        }
+    }
+    const turnNotes = result.turns.map((turn) => turn.note).filter(Boolean);
+    if (turnNotes.length > 0) {
+        result.note = turnNotes.join(' ');
+    }
+    return (0, report_1.finalizeScenarioResult)(result);
+}
+async function runScenario(scenario, options) {
+    const scenarios = Array.isArray(scenario) ? scenario : [scenario];
+    const results = [];
+    const shouldStopOnFailure = options.stopOnFailure ?? true;
+    const deadlineAt = typeof options.maxDurationMs === 'number' ? Date.now() + options.maxDurationMs : undefined;
+    for (const currentScenario of scenarios) {
+        try {
+            results.push(await runSingleScenario(currentScenario, options.context, options.env, options.outputDirectory, deadlineAt, options.messageRunner, options.initialThreadId));
+        }
+        catch (error) {
+            if (error instanceof report_1.ScenarioExecutionError) {
+                results.push(error.result);
+            }
+            else {
+                results.push({
+                    id: currentScenario.id,
+                    name: currentScenario.name,
+                    status: 'FAIL',
+                    note: error instanceof Error ? error.message : String(error),
+                    turns: [],
+                    metrics: {},
+                });
+            }
+            if (shouldStopOnFailure) {
+                break;
+            }
+        }
+    }
+    return Array.isArray(scenario) ? results : results[0];
+}

package/dist/executors/run-result.d.ts

@@ -0,0 +1,33 @@
+export type TimingMetrics = {
+    ttfToolSeconds?: number;
+    ttfTextSeconds?: number;
+    totalSeconds?: number;
+    protocolUsedKb?: number;
+    protocolTotalKb?: number;
+    efficiencyPercent?: number;
+};
+export type MessageRunResult = {
+    output: string;
+    threadId: string;
+    outputPath: string;
+    note?: string;
+    toolCallCount: number;
+    toolCalls: string[];
+    metrics: TimingMetrics;
+};
+export declare class MessageRunError extends Error {
+    threadId?: string;
+    outputPath?: string;
+    output: string;
+    metrics: TimingMetrics;
+    toolCallCount: number;
+    toolCalls: string[];
+    constructor(message: string, options?: {
+        threadId?: string;
+        outputPath?: string;
+        output?: string;
+        metrics?: TimingMetrics;
+        toolCallCount?: number;
+        toolCalls?: string[];
+    });
+}

package/dist/executors/run-result.js

@@ -0,0 +1,22 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MessageRunError = void 0;
+class MessageRunError extends Error {
+    threadId;
+    outputPath;
+    output;
+    metrics;
+    toolCallCount;
+    toolCalls;
+    constructor(message, options = {}) {
+        super(message);
+        this.name = 'MessageRunError';
+        this.threadId = options.threadId;
+        this.outputPath = options.outputPath;
+        this.output = options.output ?? '';
+        this.metrics = options.metrics ?? {};
+        this.toolCallCount = options.toolCallCount ?? 0;
+        this.toolCalls = options.toolCalls ?? [];
+    }
+}
+exports.MessageRunError = MessageRunError;

package/dist/index.d.ts

@@ -0,0 +1,12 @@
+export * from './assertions/assertion';
+export * from './assertions/turn-eval';
+export * from './executors/execute';
+export * from './executors/executor';
+export * from './executors/run-result';
+export * from './reports/report';
+export * from './scenarios/generated-message';
+export * from './scenarios/scenario';
+export * from './scenarios/scenario-loader';
+export * from './utils/artifact-files';
+export * from './utils/config';
+export * from './utils/openai-eval';

package/dist/index.js

@@ -0,0 +1,28 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./assertions/assertion"), exports);
+__exportStar(require("./assertions/turn-eval"), exports);
+__exportStar(require("./executors/execute"), exports);
+__exportStar(require("./executors/executor"), exports);
+__exportStar(require("./executors/run-result"), exports);
+__exportStar(require("./reports/report"), exports);
+__exportStar(require("./scenarios/generated-message"), exports);
+__exportStar(require("./scenarios/scenario"), exports);
+__exportStar(require("./scenarios/scenario-loader"), exports);
+__exportStar(require("./utils/artifact-files"), exports);
+__exportStar(require("./utils/config"), exports);
+__exportStar(require("./utils/openai-eval"), exports);

package/dist/prompts/turn-eval-system-prompt.md

@@ -0,0 +1,68 @@
+You are evaluating an AI assistant turn inside a multi-turn test scenario.
+
+Your job:
+- Score only the assistant response for the current turn, while using the full conversation history as context.
+- Evaluate only the requested dimensions.
+- Score each dimension as an integer from 1 to 10, where 10 is best.
+- Keep each explanation concise, no more than 3 sentences.
+- Keep each suggestion concise and actionable, no more than 2 sentences.
+
+General scoring guidance:
+- 9 to 10: excellent, only minor or no issues
+- 7 to 8: strong, useful response with some gaps
+- 5 to 6: acceptable but noticeably incomplete, weak, or uneven
+- 3 to 4: poor, major issues reduce usefulness
+- 1 to 2: very poor, misleading, unusable, or severely off-target
+
+Core dimension guidance:
+- correctness:
+  Judge factual accuracy, internal consistency, instruction-following, and whether the response avoids invented or contradictory details.
+  High score: requirements are interpreted correctly and outputs are logically valid.
+  Low score: contains wrong facts, wrong logic, contradictions, or unsupported assumptions.
+
+- coverage:
+  Judge whether the response addresses the important parts of the current request and relevant prior context.
+  High score: covers main cases, edge cases, constraints, and expected outputs at the right level.
+  Low score: omits important scenarios, ignores constraints, or responds too narrowly.
+
+- helpfulness:
+  Judge whether the response is useful for the user's goal, easy to act on, and presented clearly.
+  High score: well-structured, practical, readable, and directly usable.
+  Low score: vague, hard to use, rambling, or missing next-step value.
+
+Common optional dimensions:
+- clarity:
+  Judge readability, organization, and whether the wording is easy to understand.
+
+- completeness:
+  Judge whether the response feels sufficiently finished for the request, without major missing parts.
+
+- conciseness:
+  Judge whether the response is appropriately brief without losing needed substance.
+
+- relevance:
+  Judge whether the response stays on-topic and avoids unnecessary or distracting content.
+
+- actionability:
+  Judge whether the user can directly use the response to proceed, implement, or decide next steps.
+
+- structure:
+  Judge whether the response is organized into a form that is easy to scan and review.
+
+- consistency:
+  Judge whether the response aligns with earlier turns and remains internally coherent.
+
+- safety:
+  Judge whether the response avoids risky, misleading, or inappropriate guidance for the context.
+
+Evaluation habits:
+- Use the conversation history only as context. Score the current assistant response itself.
+- Do not reward style if the answer is wrong.
+- Do not punish brevity if the request is simple and the answer is still sufficient.
+- If a dimension is not strongly applicable, still score it based on the closest reasonable interpretation.
+
+Output rules:
+- Return only valid JSON.
+- Use this exact shape:
+{"evaluations":[{"dimension":"correctness","score":8,"explanation":"short reason","suggestion":"short improvement"}]}
+- Include one object for every requested dimension.

package/dist/prompts/turn-message-gen-system-prompt.md

@@ -0,0 +1,16 @@
+You generate exactly one user chat message for a scenario test runner.
+
+Rules:
+- Return only the user message text.
+- Do not include explanations, labels, quotes, JSON, or markdown fences.
+- Keep the message natural, specific, and useful.
+- Preserve the important intent, constraints, and domain details from the provided inputs.
+- If explicit guidance is provided, follow it closely.
+- If prior conversation history exists, make the message follow naturally from that history.
+- If source content is provided, convert it into a realistic user message instead of copying it mechanically when possible.
+- Keep the message concise unless the source content requires more detail.
+
+Quality bar:
+- The message should sound like something a real user would send next.
+- The message should move the conversation forward.
+- The message must not introduce unrelated facts.

package/dist/reports/report.d.ts

@@ -0,0 +1,68 @@
+import { type AssertionEvaluationResult } from '../assertions/assertion';
+import { type TurnEvaluationResult } from '../assertions/turn-eval';
+import { type TimingMetrics } from '../executors/run-result';
+export type TurnRunResult = {
+    label: string;
+    message: string;
+    evalDimensions?: string[];
+    threadId?: string;
+    outputPath?: string;
+    output?: string;
+    note?: string;
+    toolCallCount: number;
+    toolCalls?: string[];
+    metrics: TimingMetrics;
+    assertionResults?: AssertionEvaluationResult[];
+    evaluationResults?: TurnEvaluationResult[];
+};
+export type ScenarioRunResult = {
+    id: string;
+    name: string;
+    status: 'PASS' | 'FAIL' | 'SKIP';
+    note?: string;
+    threadId?: string;
+    turns: TurnRunResult[];
+    metrics: TimingMetrics;
+};
+type ScenarioRunSlackRow = {
+    id: string;
+    name: string;
+    status: 'PASS' | 'FAIL' | 'SKIP';
+    ttfTool: string;
+    ttfText: string;
+    total: string;
+    efficiency: string;
+    note?: string;
+};
+type ScenarioRuntimeSnapshot = {
+    agentUrl: string;
+    agentId: string;
+    wsUrl: string;
+    wsTopic: string;
+    accountId: string;
+    projectId: string;
+    appBaseUrl: string;
+};
+type WriteScenarioRunReportOptions = {
+    outputDirectory: string;
+    runtime: ScenarioRuntimeSnapshot;
+    environment: string;
+    projectName: string;
+    scenarioContext: Record<string, unknown>;
+    results: ScenarioRunResult[];
+};
+type ScenarioRunReportPaths = {
+    jsonPath: string;
+    htmlPath: string;
+};
+export declare class ScenarioExecutionError extends Error {
+    result: ScenarioRunResult;
+    constructor(result: ScenarioRunResult);
+}
+export declare function finalizeScenarioResult(result: ScenarioRunResult): ScenarioRunResult;
+export declare function getScenarioRunStatus(results: ScenarioRunResult[]): 'PASS' | 'FAIL';
+export declare function buildScenarioSlackRows(results: ScenarioRunResult[]): ScenarioRunSlackRow[];
+export declare function writeScenarioRunReport<TContext extends Record<string, unknown>>(options: Omit<WriteScenarioRunReportOptions, 'scenarioContext'> & {
+    scenarioContext: TContext;
+}): Promise<ScenarioRunReportPaths>;
+export {};