@huydao/karrot 0.1.1

package/dist/assertions/assertion.d.ts

@@ -0,0 +1,18 @@
+import { type AiTurnAssertion } from '../scenarios/scenario';
+export type AssertionEvaluationResult = {
+    kind: 'assert' | 'aiAssert';
+    matcher: 'hasText' | 'toolcall' | 'hasContent' | 'notHasContent';
+    expected: string | string[];
+    description?: string;
+    passed: boolean;
+    reason: string;
+};
+type EvaluateTurnAssertionsOptions = {
+    assertions?: AiTurnAssertion[];
+    output: string;
+    toolCalls?: string[];
+    env?: NodeJS.ProcessEnv;
+    outputDirectory?: string;
+};
+export declare function evaluateTurnAssertions(options: EvaluateTurnAssertionsOptions): Promise<AssertionEvaluationResult[]>;
+export {};

package/dist/assertions/assertion.js

@@ -0,0 +1,198 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.evaluateTurnAssertions = evaluateTurnAssertions;
+const openai_eval_1 = require("../utils/openai-eval");
+function normalizeAssertion(assertion) {
+    if ('assert' in assertion) {
+        if ('hasText' in assertion.assert) {
+            return {
+                kind: 'assert',
+                matcher: 'hasText',
+                expected: assertion.assert.hasText,
+                description: assertion.description,
+            };
+        }
+        return {
+            kind: 'assert',
+            matcher: 'toolcall',
+            expected: assertion.assert.toolcall,
+            description: assertion.description,
+        };
+    }
+    if ('hasContent' in assertion.aiAssert) {
+        return {
+            kind: 'aiAssert',
+            matcher: 'hasContent',
+            expected: assertion.aiAssert.hasContent,
+            description: assertion.description,
+        };
+    }
+    return {
+        kind: 'aiAssert',
+        matcher: 'notHasContent',
+        expected: assertion.aiAssert.notHasContent,
+        description: assertion.description,
+    };
+}
+function formatExpectedValue(expected) {
+    if (Array.isArray(expected)) {
+        return `[${expected.join(', ')}]`;
+    }
+    return `"${expected}"`;
+}
+function buildAssertionReason(result) {
+    const prefix = result.description ? `${result.description}: ` : '';
+    return `${prefix}${result.matcher}(${formatExpectedValue(result.expected)})`;
+}
+function buildNormalizedAssertionReason(assertion) {
+    return buildAssertionReason({
+        kind: assertion.kind,
+        matcher: assertion.matcher,
+        expected: assertion.expected,
+        description: assertion.description,
+        passed: false,
+        reason: '',
+    });
+}
+function extractJsonObject(text) {
+    const startIndex = text.indexOf('{');
+    const endIndex = text.lastIndexOf('}');
+    if (startIndex === -1 || endIndex === -1 || endIndex <= startIndex) {
+        return undefined;
+    }
+    return text.slice(startIndex, endIndex + 1);
+}
+function parseAiAssertionResult(output, fallback) {
+    const jsonText = extractJsonObject(output);
+    if (!jsonText) {
+        return {
+            ...fallback,
+            passed: false,
+            reason: 'AI assertion did not return parseable JSON.',
+        };
+    }
+    try {
+        const parsed = JSON.parse(jsonText);
+        return {
+            ...fallback,
+            passed: parsed.pass === true,
+            reason: typeof parsed.reason === 'string' && parsed.reason.trim() ? parsed.reason.trim() : 'No reason provided.',
+        };
+    }
+    catch {
+        return {
+            ...fallback,
+            passed: false,
+            reason: 'AI assertion returned invalid JSON.',
+        };
+    }
+}
+async function evaluateAiAssertion(assertion, output, env) {
+    const prompt = [
+        'You are evaluating an assistant response against an assertion.',
+        'Judge semantically, not only by exact string match.',
+        `Assertion type: ${assertion.matcher}`,
+        `Assertion text: ${assertion.expected}`,
+        'Assistant response:',
+        output,
+        'Return only compact JSON with this shape:',
+        '{"pass":true,"reason":"short explanation"}',
+    ].join('\n');
+    const response = await fetch(`${(0, openai_eval_1.getOpenAiBaseUrl)(env)}/responses`, {
+        method: 'POST',
+        headers: {
+            Authorization: `Bearer ${(0, openai_eval_1.getRequiredOpenAiApiKey)(env)}`,
+            'Content-Type': 'application/json',
+        },
+        body: JSON.stringify({
+            model: (0, openai_eval_1.getOpenAiEvalModel)(env),
+            input: [
+                {
+                    role: 'developer',
+                    content: 'Evaluate whether the assistant response satisfies the assertion. Return only JSON: {"pass":true|false,"reason":"short explanation"}',
+                },
+                {
+                    role: 'user',
+                    content: prompt,
+                },
+            ],
+        }),
+    });
+    if (!response.ok) {
+        throw new Error(`OpenAI aiAssert failed with HTTP ${response.status}: ${await response.text()}`);
+    }
+    const payload = (await response.json());
+    const responseText = (0, openai_eval_1.extractOutputText)(payload);
+    return parseAiAssertionResult(responseText, {
+        kind: assertion.kind,
+        matcher: assertion.matcher,
+        expected: assertion.expected,
+        description: assertion.description,
+    });
+}
+function evaluateToolCallAssertion(assertion, toolCalls) {
+    const expectedToolCalls = (Array.isArray(assertion.expected) ? assertion.expected : [])
+        .map((toolCall) => toolCall.trim())
+        .filter(Boolean);
+    const normalizedToolCallAssertion = {
+        ...assertion,
+        expected: expectedToolCalls,
+    };
+    const actualToolCalls = (toolCalls ?? []).map((toolCall) => toolCall.trim()).filter(Boolean);
+    const actualToolCallSet = new Set(actualToolCalls);
+    if (expectedToolCalls.length === 0) {
+        const passed = actualToolCalls.length === 0;
+        return {
+            kind: assertion.kind,
+            matcher: assertion.matcher,
+            expected: expectedToolCalls,
+            description: assertion.description,
+            passed,
+            reason: passed
+                ? `${buildNormalizedAssertionReason(normalizedToolCallAssertion)} passed.`
+                : `${buildNormalizedAssertionReason(normalizedToolCallAssertion)} failed. Observed: ${formatExpectedValue(actualToolCalls)}.`,
+        };
+    }
+    const missingToolCalls = expectedToolCalls.filter((toolCall) => !actualToolCallSet.has(toolCall));
+    const passed = missingToolCalls.length === 0;
+    return {
+        kind: assertion.kind,
+        matcher: assertion.matcher,
+        expected: expectedToolCalls,
+        description: assertion.description,
+        passed,
+        reason: passed
+            ? `${buildNormalizedAssertionReason(normalizedToolCallAssertion)} passed. Observed: ${formatExpectedValue(actualToolCalls)}.`
+            : `${buildNormalizedAssertionReason(normalizedToolCallAssertion)} failed. Missing: ${formatExpectedValue(missingToolCalls)}. Observed: ${formatExpectedValue(actualToolCalls)}.`,
+    };
+}
+async function evaluateTurnAssertions(options) {
+    if (!options.assertions?.length) {
+        return [];
+    }
+    const results = [];
+    for (const rawAssertion of options.assertions) {
+        const assertion = normalizeAssertion(rawAssertion);
+        if (assertion.kind === 'assert') {
+            if (assertion.matcher === 'toolcall') {
+                results.push(evaluateToolCallAssertion(assertion, options.toolCalls));
+                continue;
+            }
+            const expected = typeof assertion.expected === 'string' ? assertion.expected : assertion.expected.join(', ');
+            const passed = options.output.includes(expected);
+            results.push({
+                kind: assertion.kind,
+                matcher: assertion.matcher,
+                expected,
+                description: assertion.description,
+                passed,
+                reason: passed
+                    ? `${buildNormalizedAssertionReason({ ...assertion, expected })} passed.`
+                    : `${buildNormalizedAssertionReason({ ...assertion, expected })} failed.`,
+            });
+            continue;
+        }
+        results.push(await evaluateAiAssertion(assertion, options.output, options.env));
+    }
+    return results;
+}

package/dist/assertions/turn-eval.d.ts

@@ -0,0 +1,22 @@
+import { type AiTurnEvalDefinition } from '../scenarios/scenario';
+export type TurnEvaluationResult = {
+    dimension: string;
+    score: number;
+    explanation: string;
+    suggestion: string;
+};
+type EvaluateTurnEvalsOptions = {
+    dimensions?: AiTurnEvalDefinition[];
+    scenarioId: string;
+    scenarioName: string;
+    turnLabel: string;
+    output: string;
+    env?: NodeJS.ProcessEnv;
+    history: Array<{
+        label: string;
+        message: string;
+        output?: string;
+    }>;
+};
+export declare function evaluateTurnEvals(options: EvaluateTurnEvalsOptions): Promise<TurnEvaluationResult[]>;
+export {};

package/dist/assertions/turn-eval.js

@@ -0,0 +1,178 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.evaluateTurnEvals = evaluateTurnEvals;
+const node_path_1 = __importDefault(require("node:path"));
+const openai_eval_1 = require("../utils/openai-eval");
+const cachedSystemPrompts = new Map();
+const DEFAULT_SYSTEM_PROMPT_PATH = node_path_1.default.resolve(__dirname, '..', 'prompts', 'turn-eval-system-prompt.md');
+function resolveSystemPromptPath(env) {
+    const configuredPath = env?.AI_TURN_EVAL_SYSTEM_PROMPT_PATH?.trim();
+    if (!configuredPath) {
+        return DEFAULT_SYSTEM_PROMPT_PATH;
+    }
+    return node_path_1.default.isAbsolute(configuredPath) ? configuredPath : node_path_1.default.resolve(process.cwd(), configuredPath);
+}
+function resolvePromptDirectoryPath(env) {
+    const configuredPath = env?.AI_TURN_EVAL_PROMPT_DIRECTORY?.trim();
+    if (!configuredPath) {
+        return undefined;
+    }
+    return node_path_1.default.isAbsolute(configuredPath) ? configuredPath : node_path_1.default.resolve(process.cwd(), configuredPath);
+}
+async function readPromptDirectory(directoryPath) {
+    const cacheKey = `dir:${directoryPath}`;
+    const cachedPrompt = cachedSystemPrompts.get(cacheKey);
+    if (cachedPrompt) {
+        return cachedPrompt;
+    }
+    const { readdir, readFile } = await import('node:fs/promises');
+    const entries = await readdir(directoryPath, { withFileTypes: true });
+    const fileNames = entries
+        .filter((entry) => entry.isFile() && /\.(txt|md)$/i.test(entry.name))
+        .map((entry) => entry.name)
+        .sort((left, right) => left.localeCompare(right));
+    const fragments = await Promise.all(fileNames.map(async (fileName) => {
+        const filePath = node_path_1.default.join(directoryPath, fileName);
+        const content = (await readFile(filePath, 'utf8')).trim();
+        if (!content) {
+            return undefined;
+        }
+        return `File: ${fileName}\n${content}`;
+    }));
+    const joinedPrompt = fragments.filter(Boolean).join('\n\n');
+    cachedSystemPrompts.set(cacheKey, joinedPrompt);
+    return joinedPrompt;
+}
+async function readSystemPrompt(env) {
+    const systemPromptPath = resolveSystemPromptPath(env);
+    const promptDirectoryPath = resolvePromptDirectoryPath(env);
+    const cacheKey = `${systemPromptPath}::${promptDirectoryPath ?? ''}`;
+    const cachedCombinedPrompt = cachedSystemPrompts.get(cacheKey);
+    if (cachedCombinedPrompt) {
+        return cachedCombinedPrompt;
+    }
+    const { readFile } = await import('node:fs/promises');
+    const cachedSystemPrompt = cachedSystemPrompts.get(systemPromptPath);
+    const systemPrompt = cachedSystemPrompt ?? await readFile(systemPromptPath, 'utf8');
+    cachedSystemPrompts.set(systemPromptPath, systemPrompt);
+    if (!promptDirectoryPath) {
+        cachedSystemPrompts.set(cacheKey, systemPrompt);
+        return systemPrompt;
+    }
+    const directoryPrompt = await readPromptDirectory(promptDirectoryPath);
+    const combinedPrompt = directoryPrompt
+        ? `${systemPrompt.trim()}\n\nAdditional project-specific evaluation guidance:\n${directoryPrompt}`
+        : systemPrompt;
+    cachedSystemPrompts.set(cacheKey, combinedPrompt);
+    return combinedPrompt;
+}
+function extractJsonObject(text) {
+    const startIndex = text.indexOf('{');
+    const endIndex = text.lastIndexOf('}');
+    if (startIndex === -1 || endIndex === -1 || endIndex <= startIndex) {
+        return undefined;
+    }
+    return text.slice(startIndex, endIndex + 1);
+}
+function clampScore(value) {
+    const parsed = Number(value);
+    if (!Number.isFinite(parsed)) {
+        return 1;
+    }
+    return Math.max(1, Math.min(10, Math.round(parsed)));
+}
+function normalizeShortText(value, fallback) {
+    const text = typeof value === 'string' && value.trim() ? value.trim() : fallback;
+    const sentences = text.split(/(?<=[.!?])\s+/).filter(Boolean);
+    return sentences.slice(0, 3).join(' ').trim() || fallback;
+}
+function parseEvaluationResults(output, dimensions) {
+    const jsonText = extractJsonObject(output);
+    if (!jsonText) {
+        throw new Error('Turn evaluation did not return parseable JSON.');
+    }
+    const parsed = JSON.parse(jsonText);
+    const evaluations = Array.isArray(parsed.evaluations) ? parsed.evaluations : [];
+    return dimensions.map((dimension) => {
+        const matched = evaluations.find((evaluation) => typeof evaluation.dimension === 'string' && evaluation.dimension.trim().toLowerCase() === dimension.toLowerCase()) ?? {};
+        return {
+            dimension,
+            score: clampScore(matched.score),
+            explanation: normalizeShortText(matched.explanation, 'No explanation provided.'),
+            suggestion: normalizeShortText(matched.suggestion, 'No suggestion provided.'),
+        };
+    });
+}
+function normalizeEvalDefinitions(dimensions) {
+    return dimensions.map((definition) => typeof definition === 'string'
+        ? { dimension: definition }
+        : {
+            dimension: String(definition.dimension),
+            guidance: definition.guidance.trim(),
+        });
+}
+function buildHistoryText(history) {
+    return history
+        .map((entry, index) => [
+        `Turn ${index + 1}: ${entry.label}`,
+        `User message: ${entry.message}`,
+        `Assistant response: ${entry.output?.trim() || '[empty response]'}`,
+    ].join('\n'))
+        .join('\n\n');
+}
+async function evaluateTurnEvals(options) {
+    if (!options.dimensions?.length) {
+        return [];
+    }
+    const systemPrompt = await readSystemPrompt(options.env);
+    const dimensions = normalizeEvalDefinitions(options.dimensions);
+    const prompt = [
+        `Scenario ID: ${options.scenarioId}`,
+        `Scenario name: ${options.scenarioName}`,
+        `Current turn: ${options.turnLabel}`,
+        `Evaluation dimensions: ${dimensions.map((dimension) => dimension.dimension).join(', ')}`,
+        ...(dimensions.some((dimension) => dimension.guidance)
+            ? [
+                '',
+                'Custom evaluation guidance:',
+                ...dimensions
+                    .filter((dimension) => dimension.guidance)
+                    .map((dimension) => `- ${dimension.dimension}: ${dimension.guidance}`),
+            ]
+            : []),
+        '',
+        'Conversation history including the current turn:',
+        buildHistoryText(options.history),
+        '',
+        'Return only compact JSON with this shape:',
+        '{"evaluations":[{"dimension":"correctness","score":8,"explanation":"short reason","suggestion":"short improvement"}]}',
+    ].join('\n');
+    const response = await fetch(`${(0, openai_eval_1.getOpenAiBaseUrl)(options.env)}/responses`, {
+        method: 'POST',
+        headers: {
+            Authorization: `Bearer ${(0, openai_eval_1.getRequiredOpenAiApiKey)(options.env)}`,
+            'Content-Type': 'application/json',
+        },
+        body: JSON.stringify({
+            model: (0, openai_eval_1.getOpenAiEvalModel)(options.env),
+            input: [
+                {
+                    role: 'developer',
+                    content: systemPrompt,
+                },
+                {
+                    role: 'user',
+                    content: prompt,
+                },
+            ],
+        }),
+    });
+    if (!response.ok) {
+        throw new Error(`OpenAI turn eval failed with HTTP ${response.status}: ${await response.text()}`);
+    }
+    const payload = (await response.json());
+    return parseEvaluationResults((0, openai_eval_1.extractOutputText)(payload), dimensions.map((dimension) => dimension.dimension));
+}

package/dist/executors/adapters/ag-ui-post.d.ts

@@ -0,0 +1,55 @@
+import { type MessageRunResult } from '../run-result';
+export type RunAgUiPostMessageOptions = {
+    message: string;
+    outputDirectory: string;
+    threadId?: string;
+    processTimeoutMs?: number;
+    injectMessage?: boolean;
+    injectRunMetadata?: boolean;
+    run?: {
+        url: string;
+        headers?: Record<string, string>;
+        payload: Record<string, unknown>;
+    };
+    connect?: {
+        url: string;
+        headers?: Record<string, string>;
+        payload: Record<string, unknown>;
+        processTimeoutMs?: number;
+    };
+    observe?: {
+        type: 'http-poll';
+        status: {
+            url: string;
+            headers?: Record<string, string>;
+            statusPath: string;
+            successStatuses: string[];
+            failureStatuses?: string[];
+        };
+        progressEndpoints?: Array<{
+            url: string;
+            headers?: Record<string, string>;
+            itemsPath: string;
+            idPath?: string;
+            eventType?: string;
+        }>;
+        outputEndpoint?: {
+            url: string;
+            headers?: Record<string, string>;
+            textPath: string;
+        };
+        intervalMs?: number;
+        timeoutMs?: number;
+    };
+    completionCheck?: {
+        type: 'http-poll';
+        url: string;
+        headers?: Record<string, string>;
+        statusPath: string;
+        successStatuses: string[];
+        failureStatuses?: string[];
+        intervalMs?: number;
+        timeoutMs?: number;
+    };
+};
+export declare function runAgUiPostMessage(options: RunAgUiPostMessageOptions): Promise<MessageRunResult>;