@huydao/karrot 0.1.1

package/dist/scenarios/scenario.d.ts

@@ -0,0 +1,62 @@
+export type BaseAiScenarioContext = {
+    projectId: string;
+};
+type AiTurnCompletionArgs<TContext extends BaseAiScenarioContext> = {
+    context: TContext;
+    output: string;
+};
+export type AiTurnAssertion = {
+    assert: {
+        hasText: string;
+    } | {
+        toolcall: string[];
+    };
+    description?: string;
+} | {
+    aiAssert: {
+        hasContent: string;
+    } | {
+        notHasContent: string;
+    };
+    description?: string;
+};
+export type AiTurnEvalDimension = 'correctness' | 'coverage' | 'helpfulness' | (string & {});
+export type AiTurnEvalDefinition = AiTurnEvalDimension | {
+    dimension: AiTurnEvalDimension;
+    guidance: string;
+};
+export type AiGeneratedMessageDefinition = {
+    type: 'ai_gen_previous_context';
+} | {
+    type: 'ai_gen_guidance';
+    guidance: string;
+} | {
+    type: 'ai_gen_content';
+    content: string;
+};
+type AiTurnMessage<TContext extends BaseAiScenarioContext> = ((context: TContext) => string) | AiGeneratedMessageDefinition;
+export type AiTurn<TContext extends BaseAiScenarioContext> = {
+    label: string;
+    message: AiTurnMessage<TContext>;
+    idleTimeoutMs?: number;
+    processTimeoutMs?: number;
+    assertions?: AiTurnAssertion[];
+    eval?: AiTurnEvalDefinition[];
+    onComplete?: (args: AiTurnCompletionArgs<TContext>) => void | Promise<void>;
+};
+export type AiScenario<TContext extends BaseAiScenarioContext> = {
+    id: string;
+    name: string;
+    turns: AiTurn<TContext>[];
+};
+export declare class AiScenarioSet<TContext extends BaseAiScenarioContext> {
+    readonly items: AiScenario<TContext>[];
+    constructor(items: AiScenario<TContext>[]);
+    select(ids?: string[]): AiScenario<TContext>[];
+}
+export declare const aiGen: {
+    fromPreviousContext(): AiGeneratedMessageDefinition;
+    fromGuidance(guidance: string): AiGeneratedMessageDefinition;
+    fromContent(content: string): AiGeneratedMessageDefinition;
+};
+export {};

package/dist/scenarios/scenario.js

@@ -0,0 +1,35 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.aiGen = exports.AiScenarioSet = void 0;
+class AiScenarioSet {
+    items;
+    constructor(items) {
+        this.items = items;
+    }
+    select(ids) {
+        if (!ids?.length) {
+            return this.items;
+        }
+        return this.items.filter((item) => ids.includes(item.id));
+    }
+}
+exports.AiScenarioSet = AiScenarioSet;
+exports.aiGen = {
+    fromPreviousContext() {
+        return {
+            type: 'ai_gen_previous_context',
+        };
+    },
+    fromGuidance(guidance) {
+        return {
+            type: 'ai_gen_guidance',
+            guidance,
+        };
+    },
+    fromContent(content) {
+        return {
+            type: 'ai_gen_content',
+            content,
+        };
+    },
+};

package/dist/utils/artifact-files.d.ts

@@ -0,0 +1,3 @@
+export declare function getArtifactDirectory(baseDirectory?: string): string;
+export declare function ensureArtifactDirectory(directoryPath: string): Promise<void>;
+export declare function createRunArtifactDirectory(baseDirectory?: string): Promise<string>;

package/dist/utils/artifact-files.js

@@ -0,0 +1,22 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getArtifactDirectory = getArtifactDirectory;
+exports.ensureArtifactDirectory = ensureArtifactDirectory;
+exports.createRunArtifactDirectory = createRunArtifactDirectory;
+const promises_1 = __importDefault(require("node:fs/promises"));
+const node_path_1 = __importDefault(require("node:path"));
+function getArtifactDirectory(baseDirectory = process.cwd()) {
+    return node_path_1.default.join(baseDirectory, 'artifacts');
+}
+async function ensureArtifactDirectory(directoryPath) {
+    await promises_1.default.mkdir(directoryPath, { recursive: true });
+}
+async function createRunArtifactDirectory(baseDirectory = getArtifactDirectory()) {
+    await ensureArtifactDirectory(baseDirectory);
+    const runDirectory = node_path_1.default.join(baseDirectory, String(Date.now()));
+    await ensureArtifactDirectory(runDirectory);
+    return runDirectory;
+}

package/dist/utils/config.d.ts

@@ -0,0 +1,101 @@
+export type KarrotRuntimeSnapshot = {
+    agentUrl: string;
+    agentId: string;
+    wsUrl: string;
+    wsTopic: string;
+    accountId: string;
+    projectId: string;
+    appBaseUrl: string;
+};
+export type KarrotScenarioSelection = {
+    file: string;
+    ids?: string[];
+};
+export type KarrotConfig = {
+    version: 1;
+    transport: {
+        type: 'ag-ui-wss';
+        env: Record<string, string>;
+        processTimeoutMs?: number;
+        allowIdleTimeoutWithAssistantText?: boolean;
+    } | {
+        type: 'ag-ui-post';
+        injectMessage?: boolean;
+        injectRunMetadata?: boolean;
+        run?: {
+            url: string;
+            headers?: Record<string, string>;
+            payload: Record<string, unknown>;
+        };
+        connect?: {
+            url: string;
+            headers?: Record<string, string>;
+            payload: Record<string, unknown>;
+            processTimeoutMs?: number;
+        };
+        request?: {
+            url: string;
+            headers?: Record<string, string>;
+            payload: Record<string, unknown>;
+        };
+        observe?: {
+            type: 'http-poll';
+            status: {
+                url: string;
+                headers?: Record<string, string>;
+                statusPath: string;
+                successStatuses: string[];
+                failureStatuses?: string[];
+            };
+            progressEndpoints?: Array<{
+                url: string;
+                headers?: Record<string, string>;
+                itemsPath: string;
+                idPath?: string;
+                eventType?: string;
+            }>;
+            outputEndpoint?: {
+                url: string;
+                headers?: Record<string, string>;
+                textPath: string;
+            };
+            intervalMs?: number;
+            timeoutMs?: number;
+        };
+        completionCheck?: {
+            type: 'http-poll';
+            url: string;
+            headers?: Record<string, string>;
+            statusPath: string;
+            successStatuses: string[];
+            failureStatuses?: string[];
+            intervalMs?: number;
+            timeoutMs?: number;
+        };
+        processTimeoutMs?: number;
+    };
+    artifacts?: {
+        directory?: string;
+    };
+    execution?: {
+        stopOnFailure?: boolean;
+    };
+    evaluation?: {
+        systemPromptPath?: string;
+        promptDirectory?: string;
+    };
+    context: Record<string, unknown> & {
+        projectId: string;
+    };
+    report?: {
+        enabled?: boolean;
+        environment: string;
+        projectName: string;
+        runtime: KarrotRuntimeSnapshot;
+        scenarioContext?: Record<string, unknown>;
+    };
+};
+type VariableMap = Record<string, unknown>;
+export declare function loadConfig(configPath: string): Promise<KarrotConfig>;
+export declare function resolveVariables<T>(config: T, variables: VariableMap): T;
+export {};

package/dist/utils/config.js

@@ -0,0 +1,57 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.loadConfig = loadConfig;
+exports.resolveVariables = resolveVariables;
+const promises_1 = __importDefault(require("node:fs/promises"));
+const node_path_1 = __importDefault(require("node:path"));
+const yaml_1 = __importDefault(require("yaml"));
+function isRecord(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+function resolveConfigPath(configPath) {
+    return node_path_1.default.isAbsolute(configPath) ? configPath : node_path_1.default.resolve(process.cwd(), configPath);
+}
+async function loadConfig(configPath) {
+    const absolutePath = resolveConfigPath(configPath);
+    const content = await promises_1.default.readFile(absolutePath, 'utf8');
+    const extension = node_path_1.default.extname(absolutePath).toLowerCase();
+    const parsed = extension === '.json'
+        ? JSON.parse(content)
+        : yaml_1.default.parse(content);
+    return parsed;
+}
+function readVariable(name, variables) {
+    if (name in variables) {
+        return variables[name];
+    }
+    return process.env[name];
+}
+function resolveStringTemplate(template, variables) {
+    const exactMatch = template.match(/^\$\{([A-Z0-9_]+)\}$/i);
+    if (exactMatch) {
+        const exactValue = readVariable(exactMatch[1], variables);
+        return exactValue ?? template;
+    }
+    return template.replace(/\$\{([A-Z0-9_]+)\}/gi, (_match, variableName) => {
+        const resolved = readVariable(variableName, variables);
+        return resolved == null ? '' : String(resolved);
+    });
+}
+function resolveValue(value, variables) {
+    if (typeof value === 'string') {
+        return resolveStringTemplate(value, variables);
+    }
+    if (Array.isArray(value)) {
+        return value.map((item) => resolveValue(item, variables));
+    }
+    if (isRecord(value)) {
+        return Object.fromEntries(Object.entries(value).map(([key, nestedValue]) => [key, resolveValue(nestedValue, variables)]));
+    }
+    return value;
+}
+function resolveVariables(config, variables) {
+    return resolveValue(config, variables);
+}

package/dist/utils/openai-eval.d.ts

@@ -0,0 +1,5 @@
+export declare function getRequiredOpenAiApiKey(env?: NodeJS.ProcessEnv): string;
+export declare function getOpenAiEvalModel(env?: NodeJS.ProcessEnv): string;
+export declare function getOpenAiMessageGenModel(env?: NodeJS.ProcessEnv): string;
+export declare function getOpenAiBaseUrl(env?: NodeJS.ProcessEnv): string;
+export declare function extractOutputText(response: Record<string, unknown>): string;

package/dist/utils/openai-eval.js

@@ -0,0 +1,54 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getRequiredOpenAiApiKey = getRequiredOpenAiApiKey;
+exports.getOpenAiEvalModel = getOpenAiEvalModel;
+exports.getOpenAiMessageGenModel = getOpenAiMessageGenModel;
+exports.getOpenAiBaseUrl = getOpenAiBaseUrl;
+exports.extractOutputText = extractOutputText;
+function getEnv(name, env) {
+    const value = env?.[name] ?? process.env[name];
+    return value && value.trim() ? value.trim() : undefined;
+}
+function getRequiredOpenAiApiKey(env) {
+    const apiKey = getEnv('OPENAI_API_KEY', env);
+    if (!apiKey) {
+        throw new Error('Missing OPENAI_API_KEY for evaluation.');
+    }
+    return apiKey;
+}
+function getOpenAiEvalModel(env) {
+    return getEnv('OPENAI_EVAL_MODEL', env) ?? 'gpt-5.4';
+}
+function getOpenAiMessageGenModel(env) {
+    return getEnv('OPENAI_MESSAGE_GEN_MODEL', env) ?? getEnv('OPENAI_EVAL_MODEL', env) ?? 'gpt-5.4-mini';
+}
+function getOpenAiBaseUrl(env) {
+    return (getEnv('OPENAI_BASE_URL', env) ?? 'https://api.openai.com/v1').replace(/\/+$/, '');
+}
+function extractOutputText(response) {
+    if (typeof response.output_text === 'string' && response.output_text.trim()) {
+        return response.output_text;
+    }
+    const outputs = Array.isArray(response.output) ? response.output : [];
+    const parts = [];
+    for (const item of outputs) {
+        if (!item || typeof item !== 'object') {
+            continue;
+        }
+        const content = Array.isArray(item.content)
+            ? item.content
+            : [];
+        for (const entry of content) {
+            if (!entry || typeof entry !== 'object') {
+                continue;
+            }
+            const typedEntry = entry;
+            if ((typedEntry.type === 'output_text' || typedEntry.type === 'text') &&
+                typeof typedEntry.text === 'string' &&
+                typedEntry.text.trim()) {
+                parts.push(typedEntry.text);
+            }
+        }
+    }
+    return parts.join('\n').trim();
+}

package/package.json

@@ -0,0 +1,146 @@
+{
+  "name": "@huydao/karrot",
+  "version": "0.1.1",
+  "description": "Reusable AI scenario execution, assertion, evaluation, and reporting toolkit",
+  "license": "ISC",
+  "type": "commonjs",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "require": "./dist/index.js"
+    },
+    "./assertion": {
+      "types": "./dist/assertions/assertion.d.ts",
+      "require": "./dist/assertions/assertion.js"
+    },
+    "./artifact-files": {
+      "types": "./dist/utils/artifact-files.d.ts",
+      "require": "./dist/utils/artifact-files.js"
+    },
+    "./config": {
+      "types": "./dist/utils/config.d.ts",
+      "require": "./dist/utils/config.js"
+    },
+    "./execute": {
+      "types": "./dist/executors/execute.d.ts",
+      "require": "./dist/executors/execute.js"
+    },
+    "./executor": {
+      "types": "./dist/executors/executor.d.ts",
+      "require": "./dist/executors/executor.js"
+    },
+    "./generated-message": {
+      "types": "./dist/scenarios/generated-message.d.ts",
+      "require": "./dist/scenarios/generated-message.js"
+    },
+    "./openai-eval": {
+      "types": "./dist/utils/openai-eval.d.ts",
+      "require": "./dist/utils/openai-eval.js"
+    },
+    "./report": {
+      "types": "./dist/reports/report.d.ts",
+      "require": "./dist/reports/report.js"
+    },
+    "./run-result": {
+      "types": "./dist/executors/run-result.d.ts",
+      "require": "./dist/executors/run-result.js"
+    },
+    "./scenario": {
+      "types": "./dist/scenarios/scenario.d.ts",
+      "require": "./dist/scenarios/scenario.js"
+    },
+    "./scenario-loader": {
+      "types": "./dist/scenarios/scenario-loader.d.ts",
+      "require": "./dist/scenarios/scenario-loader.js"
+    },
+    "./turn-eval": {
+      "types": "./dist/assertions/turn-eval.d.ts",
+      "require": "./dist/assertions/turn-eval.js"
+    },
+    "./adapters/ag-ui": {
+      "types": "./dist/executors/adapters/ag-ui.d.ts",
+      "require": "./dist/executors/adapters/ag-ui.js"
+    },
+    "./adapters/ag-ui-post": {
+      "types": "./dist/executors/adapters/ag-ui-post.d.ts",
+      "require": "./dist/executors/adapters/ag-ui-post.js"
+    },
+    "./assertions/assertion": {
+      "types": "./dist/assertions/assertion.d.ts",
+      "require": "./dist/assertions/assertion.js"
+    },
+    "./assertions/turn-eval": {
+      "types": "./dist/assertions/turn-eval.d.ts",
+      "require": "./dist/assertions/turn-eval.js"
+    },
+    "./reports/report": {
+      "types": "./dist/reports/report.d.ts",
+      "require": "./dist/reports/report.js"
+    },
+    "./executors/execute": {
+      "types": "./dist/executors/execute.d.ts",
+      "require": "./dist/executors/execute.js"
+    },
+    "./executors/executor": {
+      "types": "./dist/executors/executor.d.ts",
+      "require": "./dist/executors/executor.js"
+    },
+    "./executors/run-result": {
+      "types": "./dist/executors/run-result.d.ts",
+      "require": "./dist/executors/run-result.js"
+    },
+    "./executors/adapters/ag-ui": {
+      "types": "./dist/executors/adapters/ag-ui.d.ts",
+      "require": "./dist/executors/adapters/ag-ui.js"
+    },
+    "./executors/adapters/ag-ui-post": {
+      "types": "./dist/executors/adapters/ag-ui-post.d.ts",
+      "require": "./dist/executors/adapters/ag-ui-post.js"
+    },
+    "./scenarios/scenario": {
+      "types": "./dist/scenarios/scenario.d.ts",
+      "require": "./dist/scenarios/scenario.js"
+    },
+    "./scenarios/scenario-loader": {
+      "types": "./dist/scenarios/scenario-loader.d.ts",
+      "require": "./dist/scenarios/scenario-loader.js"
+    },
+    "./scenarios/generated-message": {
+      "types": "./dist/scenarios/generated-message.d.ts",
+      "require": "./dist/scenarios/generated-message.js"
+    },
+    "./utils/artifact-files": {
+      "types": "./dist/utils/artifact-files.d.ts",
+      "require": "./dist/utils/artifact-files.js"
+    },
+    "./utils/config": {
+      "types": "./dist/utils/config.d.ts",
+      "require": "./dist/utils/config.js"
+    },
+    "./utils/openai-eval": {
+      "types": "./dist/utils/openai-eval.d.ts",
+      "require": "./dist/utils/openai-eval.js"
+    }
+  },
+  "files": [
+    "dist",
+    "README.md",
+    "GUIDE.md"
+  ],
+  "scripts": {
+    "build": "rm -rf dist && tsc -p tsconfig.json && mkdir -p dist/prompts && cp prompts/*.md dist/prompts/",
+    "prepack": "npm run build"
+  },
+  "dependencies": {
+    "ag-ui-wss": "file:vendor/ag-ui-wss",
+    "yaml": "^2.8.1"
+  },
+  "bundleDependencies": [
+    "ag-ui-wss"
+  ],
+  "publishConfig": {
+    "access": "public"
+  }
+}