@chanl/eval-cli 0.4.0

package/dist/commands/generate.js

@@ -0,0 +1,159 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.registerGenerateCommand = registerGenerateCommand;
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const chalk_1 = __importDefault(require("chalk"));
+const ora_1 = __importDefault(require("ora"));
+const client_1 = require("../client");
+const output_1 = require("../output");
+function registerGenerateCommand(program) {
+    program
+        .command('generate')
+        .description('Auto-generate test scenarios, personas, and scorecards from an agent system prompt')
+        .option('--from-prompt <prompt>', 'Agent system prompt text')
+        .option('--from-file <path>', 'Path to file containing agent system prompt')
+        .option('--count <number>', 'Number of scenarios to generate', '10')
+        .option('--difficulties <list>', 'Comma-separated difficulties: easy,medium,hard', 'easy,medium,hard')
+        .option('--adversarial', 'Include adversarial/red-team scenarios')
+        .option('--domain <domain>', 'Override auto-detected domain')
+        .option('--preview', 'Preview only — do not save to database')
+        .action(async (options) => {
+        try {
+            await generateAction(options);
+        }
+        catch (err) {
+            (0, output_1.printError)((0, client_1.formatError)(err));
+            process.exit(1);
+        }
+    });
+}
+async function generateAction(options) {
+    // Resolve system prompt
+    let systemPrompt = options.fromPrompt;
+    if (options.fromFile) {
+        const filePath = path.resolve(options.fromFile);
+        if (!fs.existsSync(filePath)) {
+            (0, output_1.printError)(`File not found: ${filePath}`);
+            process.exit(1);
+        }
+        systemPrompt = fs.readFileSync(filePath, 'utf-8');
+    }
+    if (!systemPrompt?.trim()) {
+        (0, output_1.printError)('System prompt required. Use --from-prompt "..." or --from-file <path>');
+        process.exit(1);
+    }
+    const count = parseInt(options.count || '10', 10);
+    const difficulties = (options.difficulties || 'easy,medium,hard')
+        .split(',')
+        .map((d) => d.trim());
+    const body = {
+        systemPrompt,
+        count,
+        difficulties,
+        includeAdversarial: Boolean(options.adversarial),
+        domain: options.domain,
+    };
+    if (options.preview) {
+        const spinner = (0, ora_1.default)('Generating test suite preview...').start();
+        try {
+            const result = await (0, client_1.post)('/generation/preview', body);
+            spinner.stop();
+            const suite = result.suite;
+            printSuitePreview(suite);
+        }
+        catch (err) {
+            spinner.fail('Generation failed');
+            throw err;
+        }
+    }
+    else {
+        const spinner = (0, ora_1.default)('Generating and saving test suite...').start();
+        try {
+            const result = await (0, client_1.post)('/generation/from-prompt', body);
+            spinner.stop();
+            const r = result.result;
+            (0, output_1.printSuccess)(`Generated ${r.scenarioIds.length} scenarios, ${r.personaIds.length} personas, ${r.scorecardId ? '1 scorecard' : '0 scorecards'}`);
+            console.log(chalk_1.default.dim(`  Domain: ${r.domain}`));
+            console.log(chalk_1.default.dim(`  ${r.summary}`));
+            console.log('');
+            console.log(chalk_1.default.dim('  View scenarios: chanl scenarios list'));
+            console.log(chalk_1.default.dim('  Run all:        chanl run --all'));
+        }
+        catch (err) {
+            spinner.fail('Generation failed');
+            throw err;
+        }
+    }
+}
+function printSuitePreview(suite) {
+    console.log('');
+    console.log(chalk_1.default.bold(`Test Suite Preview`));
+    console.log(chalk_1.default.dim(`Domain: ${suite.domain} | ${suite.summary}`));
+    console.log('');
+    // Scenarios
+    console.log(chalk_1.default.bold.underline(`Scenarios (${suite.scenarios?.length || 0})`));
+    for (const s of suite.scenarios || []) {
+        const diffColor = s.difficulty === 'easy' ? chalk_1.default.green : s.difficulty === 'hard' ? chalk_1.default.red : chalk_1.default.yellow;
+        console.log(`  ${diffColor(`[${s.difficulty}]`)} ${chalk_1.default.bold(s.name)}`);
+        console.log(`    ${chalk_1.default.dim(truncateText(s.prompt, 100))}`);
+    }
+    console.log('');
+    // Personas
+    console.log(chalk_1.default.bold.underline(`Personas (${suite.personas?.length || 0})`));
+    for (const p of suite.personas || []) {
+        console.log(`  ${chalk_1.default.bold(p.name)} — ${p.emotion}, ${p.behavior?.cooperationLevel || 'cooperative'}`);
+    }
+    console.log('');
+    // Scorecard
+    if (suite.scorecard) {
+        console.log(chalk_1.default.bold.underline(`Scorecard: ${suite.scorecard.name}`));
+        for (const c of suite.scorecard.criteria || []) {
+            console.log(`  ${chalk_1.default.cyan(c.type)} ${c.name} — ${chalk_1.default.dim(c.description || '')}`);
+        }
+    }
+    console.log('');
+    console.log(chalk_1.default.dim('Run without --preview to save to database.'));
+}
+function truncateText(text, maxLen) {
+    if (!text || text.length <= maxLen)
+        return text;
+    return text.slice(0, maxLen - 3) + '...';
+}
+//# sourceMappingURL=generate.js.map

package/dist/commands/generate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"generate.js","sourceRoot":"","sources":["../../src/commands/generate.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAQA,0DAmBC;AA1BD,uCAAyB;AACzB,2CAA6B;AAC7B,kDAA0B;AAC1B,8CAAsB;AACtB,sCAA8C;AAC9C,sCAAqD;AAErD,SAAgB,uBAAuB,CAAC,OAAgB;IACtD,OAAO;SACJ,OAAO,CAAC,UAAU,CAAC;SACnB,WAAW,CAAC,oFAAoF,CAAC;SACjG,MAAM,CAAC,wBAAwB,EAAE,0BAA0B,CAAC;SAC5D,MAAM,CAAC,oBAAoB,EAAE,6CAA6C,CAAC;SAC3E,MAAM,CAAC,kBAAkB,EAAE,iCAAiC,EAAE,IAAI,CAAC;SACnE,MAAM,CAAC,uBAAuB,EAAE,gDAAgD,EAAE,kBAAkB,CAAC;SACrG,MAAM,CAAC,eAAe,EAAE,wCAAwC,CAAC;SACjE,MAAM,CAAC,mBAAmB,EAAE,+BAA+B,CAAC;SAC5D,MAAM,CAAC,WAAW,EAAE,wCAAwC,CAAC;SAC7D,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;QACxB,IAAI,CAAC;YACH,MAAM,cAAc,CAAC,OAAO,CAAC,CAAC;QAChC,CAAC;QAAC,OAAO,GAAQ,EAAE,CAAC;YAClB,IAAA,mBAAU,EAAC,IAAA,oBAAW,EAAC,GAAG,CAAC,CAAC,CAAC;YAC7B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC,CAAC,CAAC;AACP,CAAC;AAED,KAAK,UAAU,cAAc,CAAC,OAQ7B;IACC,wBAAwB;IACxB,IAAI,YAAY,GAAuB,OAAO,CAAC,UAAU,CAAC;IAE1D,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;QACrB,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAChD,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC7B,IAAA,mBAAU,EAAC,mBAAmB,QAAQ,EAAE,CAAC,CAAC;YAC1C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QACD,YAAY,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACpD,CAAC;IAED,IAAI,CAAC,YAAY,EAAE,IAAI,EAAE,EAAE,CAAC;QAC1B,IAAA,mBAAU,EAAC,uEAAuE,CAAC,CAAC;QACpF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,KAAK,IAAI,IAAI,EAAE,EAAE,CAAC,CAAC;IAClD,MAAM,YAAY,GAAG,CAAC,OAAO,CAAC,YAAY,IAAI,kBAAkB,CAAC;SAC9D,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAmC,CAAC;IAE1D,MAAM,IAAI,GAAG;QACX,YAAY;QACZ,KAAK;QACL,YAAY;QACZ,kBAAkB,EAAE,OAAO,CAAC,OAAO,CAAC,WAAW,CAAC;QAChD,MAAM,EAAE,OAAO,CAAC,MAAM;KACvB,CAAC;IAEF,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,OAAO,GAAG,IAAA,aAAG,EAAC,kCAAkC,CAAC,CAAC,KAAK,EAAE,CAAC;QAChE,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAA,aAAI,EAAC,qBAAqB,EAAE,IAAI,CAAC,CAAC;YACvD,OAAO,CAAC,IAAI,EAAE,CAAC;YACf,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;YAC3B,iBAAiB,CAAC,KAAK,CAAC,CAAC;QAC3B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;YAClC,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;SAAM,CAAC;QACN,MAAM,OAAO,GAAG,IAAA,aAAG,EAAC,qCAAqC,CAAC,CAAC,KAAK,EAAE,CAAC;QACnE,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAA,aAAI,EAAC,yBAAyB,EAAE,IAAI,CAAC,CAAC;YAC3D,OAAO,CAAC,IAAI,EAAE,CAAC;YACf,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;YACxB,IAAA,qBAAY,EACV,aAAa,CAAC,CAAC,WAAW,CAAC,MAAM,eAAe,CAAC,CAAC,UAAU,CAAC,MAAM,cAAc,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,cAAc,EAAE,CAClI,CAAC;YACF,OAAO,CAAC,GAAG,CAAC,eAAK,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YAChD,OAAO,CAAC,GAAG,CAAC,eAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YACzC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAChB,OAAO,CAAC,GAAG,CAAC,eAAK,CAAC,GAAG,CAAC,wCAAwC,CAAC,CAAC,CAAC;YACjE,OAAO,CAAC,GAAG,CAAC,eAAK,CAAC,GAAG,CAAC,mCAAmC,CAAC,CAAC,CAAC;QAC9D,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;YAClC,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAU;IACnC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAChB,OAAO,CAAC,GAAG,CAAC,eAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC;IAC9C,OAAO,CAAC,GAAG,CAAC,eAAK,CAAC,GAAG,CAAC,WAAW,KAAK,CAAC,MAAM,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;IACrE,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAEhB,YAAY;IACZ,OAAO,CAAC,GAAG,CAAC,eAAK,CAAC,IAAI,CAAC,SAAS,CAAC,cAAc,KAAK,CAAC,SAAS,EAAE,MAAM,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IACjF,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,SAAS,IAAI,EAAE,EAAE,CAAC;QACtC,MAAM,SAAS,GACb,CAAC,CAAC,UAAU,KAAK,MAAM,CAAC,CAAC,CAAC,eAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,KAAK,MAAM,CAAC,CAAC,CAAC,eAAK,CAAC,GAAG,CAAC,CAAC,CAAC,eAAK,CAAC,MAAM,CAAC;QAC7F,OAAO,CAAC,GAAG,CAAC,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC,UAAU,GAAG,CAAC,IAAI,eAAK,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACzE,OAAO,CAAC,GAAG,CAAC,OAAO,eAAK,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;IAC/D,CAAC;IACD,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAEhB,WAAW;IACX,OAAO,CAAC,GAAG,CAAC,eAAK,CAAC,IAAI,CAAC,SAAS,CAAC,aAAa,KAAK,CAAC,QAAQ,EAAE,MAAM,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IAC/E,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,QAAQ,IAAI,EAAE,EAAE,CAAC;QACrC,OAAO,CAAC,GAAG,CACT,KAAK,eAAK,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,QAAQ,EAAE,gBAAgB,IAAI,aAAa,EAAE,CAC3F,CAAC;IACJ,CAAC;IACD,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAEhB,YAAY;IACZ,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;QACpB,OAAO,CAAC,GAAG,CAAC,eAAK,CAAC,IAAI,CAAC,SAAS,CAAC,cAAc,KAAK,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QACxE,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,SAAS,CAAC,QAAQ,IAAI,EAAE,EAAE,CAAC;YAC/C,OAAO,CAAC,GAAG,CAAC,KAAK,eAAK,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,MAAM,eAAK,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC;QACvF,CAAC;IACH,CAAC;IACD,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAChB,OAAO,CAAC,GAAG,CAAC,eAAK,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC,CAAC;AACvE,CAAC;AAED,SAAS,YAAY,CAAC,IAAY,EAAE,MAAc;IAChD,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,IAAI,MAAM;QAAE,OAAO,IAAI,CAAC;IAChD,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC;AAC3C,CAAC"}

package/dist/commands/init.d.ts

@@ -0,0 +1,29 @@
+import { Command } from 'commander';
+export interface InitOptions {
+    template?: string;
+}
+/**
+ * Scaffold a chanl-eval project directory.
+ *
+ * @param targetDir - Absolute path to the directory to scaffold into
+ * @param options   - Template selection
+ * @returns Object describing created files (for testing)
+ */
+export declare function scaffoldProject(targetDir: string, options?: InitOptions): {
+    files: string[];
+    dirs: string[];
+};
+interface TemplateInfo {
+    name: string;
+    description: string;
+    scenarioCount: number;
+    categories: string[];
+}
+/**
+ * Get metadata about all available templates.
+ */
+export declare function getTemplateList(): TemplateInfo[];
+export declare function registerInitCommand(program: Command): void;
+export declare function registerTemplatesCommand(program: Command): void;
+export {};
+//# sourceMappingURL=init.d.ts.map

package/dist/commands/init.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"init.d.ts","sourceRoot":"","sources":["../../src/commands/init.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AA2XpC,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;;;;GAMG;AACH,wBAAgB,eAAe,CAC7B,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE,WAAgB,GACxB;IAAE,KAAK,EAAE,MAAM,EAAE,CAAC;IAAC,IAAI,EAAE,MAAM,EAAE,CAAA;CAAE,CAwCrC;AAMD,UAAU,YAAY;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,EAAE,CAAC;CACtB;AAiBD;;GAEG;AACH,wBAAgB,eAAe,IAAI,YAAY,EAAE,CAEhD;AAMD,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAgB1D;AAED,wBAAgB,wBAAwB,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CA4B/D"}