@cliwatch/cli-bench 0.4.0

package/dist/help-loader.js

@@ -0,0 +1,135 @@
+/**
+ * Loads CLI help text from cached JSON files or live CLI execution.
+ *
+ * Cached mode reads from help_cache/<cli>.json, produced by
+ * `audit-worker --dry-run --output-help --output <file>`.
+ *
+ * Live mode shells out to the CLI's --help (requires CLI installed).
+ */
+import { readFile, readdir } from 'node:fs/promises';
+import { join } from 'node:path';
+import { execFile } from 'node:child_process';
+export async function loadHelpFromCache(cacheDir, cliName) {
+    try {
+        const filePath = join(cacheDir, `${cliName}.json`);
+        const raw = await readFile(filePath, 'utf-8');
+        const data = JSON.parse(raw);
+        // Support both formats: direct HelpCache or audit-worker output array
+        if (Array.isArray(data)) {
+            const entry = data.find((d) => d.cli_name === cliName);
+            if (entry?.help_texts) {
+                return {
+                    cli_name: entry.cli_name,
+                    help_texts: entry.help_texts,
+                    version: entry.version,
+                };
+            }
+            return null;
+        }
+        if (data.help_texts) {
+            return data;
+        }
+        return null;
+    }
+    catch {
+        return null;
+    }
+}
+export async function listAvailableCaches(cacheDir) {
+    try {
+        const files = await readdir(cacheDir);
+        return files
+            .filter((f) => f.endsWith('.json'))
+            .map((f) => f.replace(/\.json$/, ''));
+    }
+    catch {
+        return [];
+    }
+}
+export async function loadHelpLive(cliName, maxDepth = 2, maxCommands = 50) {
+    const helpTexts = {};
+    async function capture(commandParts, depth) {
+        if (depth > maxDepth || Object.keys(helpTexts).length >= maxCommands) {
+            return;
+        }
+        const pathKey = commandParts.slice(1).join(' ');
+        if (pathKey in helpTexts)
+            return;
+        const output = await runHelp(commandParts);
+        if (!output)
+            return;
+        helpTexts[pathKey] = output;
+        if (depth < maxDepth && Object.keys(helpTexts).length < maxCommands) {
+            const subcommands = parseSubcommands(output);
+            for (const sub of subcommands) {
+                if (Object.keys(helpTexts).length >= maxCommands)
+                    break;
+                await capture([...commandParts, sub], depth + 1);
+            }
+        }
+    }
+    await capture([cliName], 0);
+    return {
+        cli_name: cliName,
+        help_texts: helpTexts,
+    };
+}
+function runHelp(commandParts) {
+    return new Promise((resolve) => {
+        const [cmd, ...args] = commandParts;
+        execFile(cmd, [...args, '--help'], { timeout: 30_000 }, (err, stdout, stderr) => {
+            if (err) {
+                resolve(null);
+                return;
+            }
+            const output = (stdout || stderr).trim();
+            resolve(output || null);
+        });
+    });
+}
+/**
+ * Minimal subcommand parser — matches the audit-worker capturer's logic.
+ */
+function parseSubcommands(helpText) {
+    const subcommands = [];
+    const seen = new Set();
+    let inCommandsSection = false;
+    const COMMAND_SECTION_RE = /^[A-Z].*\bcommands?\b.*:?\s*(\(.*\))?\s*$/i;
+    const NON_COMMAND_SECTION_RE = /^(global\s+|cache\s+)?(flags|options|arguments?|usage|examples?|environment|help\s+topics|learn)\s*:?\s*$/i;
+    const SECTION_HEADER_RE = /^[A-Z][\w\s/&-]*(\(.*\)\s*)?:\s*$/;
+    const SKIP_WORDS = new Set([
+        'use', 'aliases', 'alias', 'see', 'for', 'more', 'additional',
+        'help', 'usage', 'learn', 'read', 'find',
+    ]);
+    for (const line of helpText.split('\n')) {
+        const stripped = line.trim();
+        if (!stripped)
+            continue;
+        // Only non-indented lines can be section headers
+        if (line === line.trimStart()) {
+            if (NON_COMMAND_SECTION_RE.test(stripped)) {
+                inCommandsSection = false;
+            }
+            else if (COMMAND_SECTION_RE.test(stripped)) {
+                inCommandsSection = true;
+            }
+            else if (SECTION_HEADER_RE.test(stripped) ||
+                (stripped === stripped.toUpperCase() && stripped.length > 3 && /^[A-Z]/.test(stripped))) {
+                inCommandsSection = false;
+            }
+            continue;
+        }
+        if (!inCommandsSection)
+            continue;
+        const match = line.match(/^\s{2,}([\w][\w-]*)\*?\s*:?\s/);
+        if (match) {
+            const cmd = match[1];
+            if (!SKIP_WORDS.has(cmd.toLowerCase()) && !seen.has(cmd)) {
+                seen.add(cmd);
+                subcommands.push(cmd);
+            }
+        }
+    }
+    return subcommands;
+}
+//# sourceMappingURL=help-loader.js.map

package/dist/help-loader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"help-loader.js","sourceRoot":"","sources":["../src/help-loader.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAG9C,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,QAAgB,EAChB,OAAe;IAEf,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,GAAG,OAAO,OAAO,CAAC,CAAC;QACnD,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC9C,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAE7B,sEAAsE;QACtE,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YACxB,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CACrB,CAAC,CAAwB,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,OAAO,CACrD,CAAC;YACF,IAAI,KAAK,EAAE,UAAU,EAAE,CAAC;gBACtB,OAAO;oBACL,QAAQ,EAAE,KAAK,CAAC,QAAQ;oBACxB,UAAU,EAAE,KAAK,CAAC,UAAU;oBAC5B,OAAO,EAAE,KAAK,CAAC,OAAO;iBACvB,CAAC;YACJ,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC;QAED,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,OAAO,IAAiB,CAAC;QAC3B,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,QAAgB;IACxD,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,CAAC;QACtC,OAAO,KAAK;aACT,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;aAClC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,CAAC;IAC1C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,OAAe,EACf,WAAmB,CAAC,EACpB,cAAsB,EAAE;IAExB,MAAM,SAAS,GAA2B,EAAE,CAAC;IAE7C,KAAK,UAAU,OAAO,CACpB,YAAsB,EACtB,KAAa;QAEb,IAAI,KAAK,GAAG,QAAQ,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,IAAI,WAAW,EAAE,CAAC;YACrE,OAAO;QACT,CAAC;QAED,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAChD,IAAI,OAAO,IAAI,SAAS;YAAE,OAAO;QAEjC,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,YAAY,CAAC,CAAC;QAC3C,IAAI,CAAC,MAAM;YAAE,OAAO;QAEpB,SAAS,CAAC,OAAO,CAAC,GAAG,MAAM,CAAC;QAE5B,IAAI,KAAK,GAAG,QAAQ,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,WAAW,EAAE,CAAC;YACpE,MAAM,WAAW,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC;YAC7C,KAAK,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;gBAC9B,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,IAAI,WAAW;oBAAE,MAAM;gBACxD,MAAM,OAAO,CAAC,CAAC,GAAG,YAAY,EAAE,GAAG,CAAC,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;YACnD,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,OAAO,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC;IAE5B,OAAO;QACL,QAAQ,EAAE,OAAO;QACjB,UAAU,EAAE,SAAS;KACtB,CAAC;AACJ,CAAC;AAED,SAAS,OAAO,CAAC,YAAsB;IACrC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC7B,MAAM,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,GAAG,YAAY,CAAC;QACpC,QAAQ,CAAC,GAAI,EAAE,CAAC,GAAG,IAAI,EAAE,QAAQ,CAAC,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE;YAC/E,IAAI,GAAG,EAAE,CAAC;gBACR,OAAO,CAAC,IAAI,CAAC,CAAC;gBACd,OAAO;YACT,CAAC;YACD,MAAM,MAAM,GAAG,CAAC,MAAM,IAAI,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;YACzC,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,SAAS,gBAAgB,CAAC,QAAgB;IACxC,MAAM,WAAW,GAAa,EAAE,CAAC;IACjC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,IAAI,iBAAiB,GAAG,KAAK,CAAC;IAE9B,MAAM,kBAAkB,GAAG,4CAA4C,CAAC;IACxE,MAAM,sBAAsB,GAC1B,4GAA4G,CAAC;IAC/G,MAAM,iBAAiB,GAAG,mCAAmC,CAAC;IAE9D,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC;QACzB,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,YAAY;QAC7D,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM;KACzC,CAAC,CAAC;IAEH,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QACxC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAC7B,IAAI,CAAC,QAAQ;YAAE,SAAS;QAExB,iDAAiD;QACjD,IAAI,IAAI,KAAK,IAAI,CAAC,SAAS,EAAE,EAAE,CAAC;YAC9B,IAAI,sBAAsB,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC1C,iBAAiB,GAAG,KAAK,CAAC;YAC5B,CAAC;iBAAM,IAAI,kBAAkB,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC7C,iBAAiB,GAAG,IAAI,CAAC;YAC3B,CAAC;iBAAM,IACL,iBAAiB,CAAC,IAAI,CAAC,QAAQ,CAAC;gBAChC,CAAC,QAAQ,KAAK,QAAQ,CAAC,WAAW,EAAE,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,EACvF,CAAC;gBACD,iBAAiB,GAAG,KAAK,CAAC;YAC5B,CAAC;YACD,SAAS;QACX,CAAC;QAED,IAAI,CAAC,iBAAiB;YAAE,SAAS;QAEjC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;QAC1D,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC;YACtB,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;gBACzD,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBACd,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,WAAW,CAAC;AACrB,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,14 @@
+#!/usr/bin/env node
+/**
+ * @cliwatch/cli-bench — LLM CLI Testing Engine
+ *
+ * Tests CLI agent-readiness by having LLMs execute tasks,
+ * then validating results with assertion-based checks.
+ *
+ * Dual-mode entry:
+ *   1. Config file mode: cli-bench.yaml found → load config → run grid
+ *   2. Legacy mode: no config → discover task_suites/ → run grid
+ *   3. Init mode: scaffold cli-bench.yaml
+ */
+export {};
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAEA;;;;;;;;;;GAUG"}

package/dist/index.js

@@ -0,0 +1,148 @@
+#!/usr/bin/env node
+/**
+ * @cliwatch/cli-bench — LLM CLI Testing Engine
+ *
+ * Tests CLI agent-readiness by having LLMs execute tasks,
+ * then validating results with assertion-based checks.
+ *
+ * Dual-mode entry:
+ *   1. Config file mode: cli-bench.yaml found → load config → run grid
+ *   2. Legacy mode: no config → discover task_suites/ → run grid
+ *   3. Init mode: scaffold cli-bench.yaml
+ */
+import { writeFile } from 'node:fs/promises';
+import { parseArgs } from './config.js';
+import { runGrid, uploadReport } from './runner.js';
+import { resolveConfigFile, loadProject } from './project.js';
+import { scaffoldProject } from './init.js';
+import { validateGatewayKey, resolveProviders } from './providers.js';
+import { checkThresholds, printThresholdResults } from './thresholds.js';
+async function main() {
+    const config = parseArgs(process.argv);
+    // Init mode — scaffold and exit
+    if (config.initMode) {
+        try {
+            const path = await scaffoldProject(process.cwd());
+            console.log(`Created ${path}`);
+            console.log('Edit cli-bench.yaml to configure your CLI tests, then run: npx @cliwatch/cli-bench');
+        }
+        catch (e) {
+            console.error(e instanceof Error ? e.message : e);
+            process.exit(1);
+        }
+        return;
+    }
+    console.log('@cliwatch/cli-bench v0.4.0');
+    // Try to find a config file
+    const configPath = await resolveConfigFile(config.configFile);
+    let reports;
+    let thresholdsConfig;
+    let thresholdFailed = false;
+    if (configPath) {
+        // Config file mode
+        console.log(`Config: ${configPath}`);
+        const { config: fileConfig, tasks } = await loadProject(configPath);
+        thresholdsConfig = fileConfig.thresholds;
+        // Merge CLI args with file config
+        const providers = config.models.length > 0
+            ? config.models
+            : fileConfig.providers ?? ['anthropic/claude-sonnet-4-20250514'];
+        const helpModes = fileConfig.help_modes
+            ? fileConfig.help_modes.filter((s) => ['injected', 'discoverable', 'none'].includes(s))
+            : config.helpModes;
+        const concurrency = fileConfig.concurrency ?? config.concurrency;
+        // Determine upload behavior
+        const uploadMode = fileConfig.upload ?? 'auto';
+        const shouldUpload = config.upload
+            || uploadMode === 'always'
+            || (uploadMode === 'auto' && !!config.apiKey);
+        console.log(`CLI: ${fileConfig.cli}`);
+        console.log(`Providers: ${providers.join(', ')}`);
+        console.log(`Tasks: ${tasks.length}`);
+        console.log(`Help modes: ${helpModes.join(', ')}`);
+        console.log(`Dry run: ${config.dryRun}`);
+        // Validate gateway key before running
+        if (!config.dryRun) {
+            validateGatewayKey();
+        }
+        const models = resolveProviders(providers);
+        const globalRepeat = config.repeat ?? fileConfig.repeat;
+        reports = await runGrid({
+            config: { ...config, concurrency, helpModes },
+            tasks,
+            cliName: fileConfig.cli,
+            models,
+            versionCommand: fileConfig.version_command,
+            workdir: fileConfig.workdir ?? config.workdir,
+            globalRepeat,
+        });
+        // Check thresholds before upload so results are included in the payload
+        if (thresholdsConfig && reports.length > 0 && !config.dryRun) {
+            const check = checkThresholds(reports.flatMap((r) => r.modelResults), thresholdsConfig);
+            printThresholdResults(check);
+            for (const report of reports) {
+                report.thresholdResults = check;
+            }
+            // Defer exit(1) until after upload so results still get posted
+            if (!check.allPassed && check.behavior === 'error') {
+                thresholdFailed = true;
+            }
+        }
+        // Upload if configured
+        if (shouldUpload && !config.dryRun) {
+            const backendUrl = fileConfig.backend_url ?? config.backendUrl;
+            for (const report of reports) {
+                try {
+                    await uploadReport(report, backendUrl, config.apiKey);
+                }
+                catch (e) {
+                    console.error(`Failed to upload report for ${report.cli}: ${e instanceof Error ? e.message : e}`);
+                }
+            }
+        }
+    }
+    else {
+        // Legacy task_suites/ discovery mode
+        console.log(`Filter: ${config.filter.length > 0 ? config.filter.join(', ') : 'all'}`);
+        console.log(`Models: ${config.models.length > 0 ? config.models.join(', ') : 'all'}`);
+        console.log(`Help modes: ${config.helpModes.join(', ')}`);
+        console.log(`Dry run: ${config.dryRun}`);
+        if (!config.dryRun && config.models.length > 0) {
+            validateGatewayKey();
+        }
+        reports = await runGrid({ config, globalRepeat: config.repeat });
+        if (config.upload) {
+            for (const report of reports) {
+                try {
+                    await uploadReport(report, config.backendUrl, config.apiKey);
+                }
+                catch (e) {
+                    console.error(`Failed to upload report for ${report.cli}: ${e instanceof Error ? e.message : e}`);
+                }
+            }
+        }
+    }
+    if (config.output && reports.length > 0) {
+        const output = JSON.stringify(reports.length === 1 ? reports[0] : reports, null, 2);
+        await writeFile(config.output, output, 'utf-8');
+        console.log(`\nResults written to ${config.output}`);
+    }
+    // Exit summary
+    if (reports.length > 0 && !config.dryRun) {
+        console.log('\n=== Final Summary ===');
+        for (const report of reports) {
+            for (const mr of report.modelResults) {
+                console.log(`${report.cli} x ${mr.displayName} [${mr.helpMode}]: ${(mr.passRate * 100).toFixed(0)}% pass, avgTurns=${mr.avgTurnsToSuccess.toFixed(1)}`);
+            }
+        }
+    }
+    // Exit with failure after upload + summary so results are still posted
+    if (thresholdFailed) {
+        process.exit(1);
+    }
+}
+main().catch((e) => {
+    console.error(e);
+    process.exit(1);
+});
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAEA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC7C,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AACpD,OAAO,EAAE,iBAAiB,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC9D,OAAO,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAC5C,OAAO,EAAE,kBAAkB,EAAE,gBAAgB,EAAgB,MAAM,gBAAgB,CAAC;AAEpF,OAAO,EAAE,eAAe,EAAE,qBAAqB,EAAE,MAAM,iBAAiB,CAAC;AAEzE,KAAK,UAAU,IAAI;IACjB,MAAM,MAAM,GAAG,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAEvC,gCAAgC;IAChC,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,eAAe,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;YAClD,OAAO,CAAC,GAAG,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC;YAC/B,OAAO,CAAC,GAAG,CAAC,oFAAoF,CAAC,CAAC;QACpG,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,CAAC,KAAK,CAAC,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAClD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QACD,OAAO;IACT,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,4BAA4B,CAAC,CAAC;IAE1C,4BAA4B;IAC5B,MAAM,UAAU,GAAG,MAAM,iBAAiB,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IAE9D,IAAI,OAAO,CAAC;IACZ,IAAI,gBAAoE,CAAC;IACzE,IAAI,eAAe,GAAG,KAAK,CAAC;IAE5B,IAAI,UAAU,EAAE,CAAC;QACf,mBAAmB;QACnB,OAAO,CAAC,GAAG,CAAC,WAAW,UAAU,EAAE,CAAC,CAAC;QACrC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,GAAG,MAAM,WAAW,CAAC,UAAU,CAAC,CAAC;QACpE,gBAAgB,GAAG,UAAU,CAAC,UAAU,CAAC;QAEzC,kCAAkC;QAClC,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC;YACxC,CAAC,CAAC,MAAM,CAAC,MAAM;YACf,CAAC,CAAC,UAAU,CAAC,SAAS,IAAI,CAAC,oCAAoC,CAAC,CAAC;QACnE,MAAM,SAAS,GAAG,UAAU,CAAC,UAAU;YACrC,CAAC,CAAC,UAAU,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAiB,EAAE,CAAC,CAAC,UAAU,EAAE,cAAc,EAAE,MAAM,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;YACtG,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC;QACrB,MAAM,WAAW,GAAG,UAAU,CAAC,WAAW,IAAI,MAAM,CAAC,WAAW,CAAC;QAEjE,4BAA4B;QAC5B,MAAM,UAAU,GAAG,UAAU,CAAC,MAAM,IAAI,MAAM,CAAC;QAC/C,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM;eAC7B,UAAU,KAAK,QAAQ;eACvB,CAAC,UAAU,KAAK,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAEhD,OAAO,CAAC,GAAG,CAAC,QAAQ,UAAU,CAAC,GAAG,EAAE,CAAC,CAAC;QACtC,OAAO,CAAC,GAAG,CAAC,cAAc,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAClD,OAAO,CAAC,GAAG,CAAC,UAAU,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;QACtC,OAAO,CAAC,GAAG,CAAC,eAAe,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACnD,OAAO,CAAC,GAAG,CAAC,YAAY,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QAEzC,sCAAsC;QACtC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACnB,kBAAkB,EAAE,CAAC;QACvB,CAAC;QAED,MAAM,MAAM,GAAG,gBAAgB,CAAC,SAAS,CAAC,CAAC;QAE3C,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,IAAI,UAAU,CAAC,MAAM,CAAC;QAExD,OAAO,GAAG,MAAM,OAAO,CAAC;YACtB,MAAM,EAAE,EAAE,GAAG,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE;YAC7C,KAAK;YACL,OAAO,EAAE,UAAU,CAAC,GAAG;YACvB,MAAM;YACN,cAAc,EAAE,UAAU,CAAC,eAAe;YAC1C,OAAO,EAAE,UAAU,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO;YAC7C,YAAY;SACb,CAAC,CAAC;QAEH,wEAAwE;QACxE,IAAI,gBAAgB,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YAC7D,MAAM,KAAK,GAAG,eAAe,CAC3B,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,EACtC,gBAAgB,CACjB,CAAC;YACF,qBAAqB,CAAC,KAAK,CAAC,CAAC;YAC7B,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC7B,MAAM,CAAC,gBAAgB,GAAG,KAAK,CAAC;YAClC,CAAC;YACD,+DAA+D;YAC/D,IAAI,CAAC,KAAK,CAAC,SAAS,IAAI,KAAK,CAAC,QAAQ,KAAK,OAAO,EAAE,CAAC;gBACnD,eAAe,GAAG,IAAI,CAAC;YACzB,CAAC;QACH,CAAC;QAED,uBAAuB;QACvB,IAAI,YAAY,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACnC,MAAM,UAAU,GAAG,UAAU,CAAC,WAAW,IAAI,MAAM,CAAC,UAAU,CAAC;YAC/D,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC7B,IAAI,CAAC;oBACH,MAAM,YAAY,CAAC,MAAM,EAAE,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;gBACxD,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACX,OAAO,CAAC,KAAK,CACX,+BAA+B,MAAM,CAAC,GAAG,KAAK,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CACnF,CAAC;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;SAAM,CAAC;QACN,qCAAqC;QACrC,OAAO,CAAC,GAAG,CAAC,WAAW,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;QACtF,OAAO,CAAC,GAAG,CAAC,WAAW,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;QACtF,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC1D,OAAO,CAAC,GAAG,CAAC,YAAY,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QAEzC,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/C,kBAAkB,EAAE,CAAC;QACvB,CAAC;QAED,OAAO,GAAG,MAAM,OAAO,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QAEjE,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;YAClB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC7B,IAAI,CAAC;oBACH,MAAM,YAAY,CAAC,MAAM,EAAE,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;gBAC/D,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACX,OAAO,CAAC,KAAK,CACX,+BAA+B,MAAM,CAAC,GAAG,KAAK,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CACnF,CAAC;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxC,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAC3B,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,EAC3C,IAAI,EACJ,CAAC,CACF,CAAC;QACF,MAAM,SAAS,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;QAChD,OAAO,CAAC,GAAG,CAAC,wBAAwB,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IACvD,CAAC;IAED,eAAe;IACf,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QACzC,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC;QACvC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,KAAK,MAAM,EAAE,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;gBACrC,OAAO,CAAC,GAAG,CACT,GAAG,MAAM,CAAC,GAAG,MAAM,EAAE,CAAC,WAAW,KAAK,EAAE,CAAC,QAAQ,MAAM,CAAC,EAAE,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,oBAAoB,EAAE,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAC3I,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED,uEAAuE;IACvE,IAAI,eAAe,EAAE,CAAC;QACpB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;IACjB,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACjB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}

package/dist/init.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Scaffolding — writes a starter cli-bench.yaml.
+ */
+export declare function scaffoldProject(cwd: string): Promise<string>;
+//# sourceMappingURL=init.d.ts.map

package/dist/init.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"init.d.ts","sourceRoot":"","sources":["../src/init.ts"],"names":[],"mappings":"AAAA;;GAEG;AAiDH,wBAAsB,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAYlE"}

package/dist/init.js

@@ -0,0 +1,62 @@
+/**
+ * Scaffolding — writes a starter cli-bench.yaml.
+ */
+import { writeFile, access } from 'node:fs/promises';
+import { join } from 'node:path';
+const TEMPLATE = `# cli-bench.yaml — @cliwatch/cli-bench configuration
+# Docs: https://github.com/anthropics/cliwatch
+
+# Required: the CLI to test (must be in PATH)
+cli: my-cli
+# version_command: "my-cli --version"
+
+# LLM providers to test against (requires corresponding API key env vars)
+providers:
+  - anthropic/claude-sonnet-4-20250514
+  # - openai/gpt-4o
+  # - google/gemini-2.5-pro
+
+# Optional settings
+# help_modes: [injected]       # injected | discoverable | none
+# concurrency: 3               # max concurrent API calls
+# workdir: ./workspace          # working directory for commands (default: temp dir)
+# upload: auto                  # auto | always | never (auto uploads if CLIWATCH_API_KEY is set)
+
+tasks:
+  - id: show-help
+    intent: "Show the help information for the CLI"
+    assert:
+      - ran: "my-cli.*--help"
+      - exit_code: 0
+
+  - id: example-task
+    intent: "Describe what the agent should do in natural language"
+    # setup:
+    #   - "mkdir -p /tmp/workspace"
+    assert:
+      - ran: "my-cli"
+      - exit_code: 0
+      # - output_contains: "expected output"
+      # - file_exists: "/tmp/workspace/output.txt"
+      # - verify:
+      #     run: "my-cli status --json"
+      #     output_contains: "ok"
+
+  # Split tasks across files for larger suites:
+  # - file://tasks/basics.yaml
+  # - file://tasks/advanced/*.yaml
+`;
+export async function scaffoldProject(cwd) {
+    const filePath = join(cwd, 'cli-bench.yaml');
+    try {
+        await access(filePath);
+        throw new Error(`cli-bench.yaml already exists in ${cwd}`);
+    }
+    catch (e) {
+        if (e instanceof Error && e.message.includes('already exists'))
+            throw e;
+    }
+    await writeFile(filePath, TEMPLATE, 'utf-8');
+    return filePath;
+}
+//# sourceMappingURL=init.js.map

package/dist/init.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"init.js","sourceRoot":"","sources":["../src/init.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,MAAM,QAAQ,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA0ChB,CAAC;AAEF,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,GAAW;IAC/C,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,gBAAgB,CAAC,CAAC;IAE7C,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,QAAQ,CAAC,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,oCAAoC,GAAG,EAAE,CAAC,CAAC;IAC7D,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,IAAI,CAAC,YAAY,KAAK,IAAI,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC;YAAE,MAAM,CAAC,CAAC;IAC1E,CAAC;IAED,MAAM,SAAS,CAAC,QAAQ,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC7C,OAAO,QAAQ,CAAC;AAClB,CAAC"}

package/dist/models.d.ts

@@ -0,0 +1,158 @@
+/**
+ * TypeScript types for @cliwatch/cli-bench.
+ *
+ * Assertion-based evaluation: tasks define assertions that are checked
+ * against the agent's execution trace.
+ */
+export type HelpMode = 'injected' | 'discoverable' | 'none';
+export type Assertion = {
+    output_contains: string;
+} | {
+    output_equals: string;
+} | {
+    error_contains: string;
+} | {
+    exit_code: number;
+} | {
+    file_exists: string;
+} | {
+    file_contains: {
+        path: string;
+        text: string;
+    };
+} | {
+    ran: string;
+} | {
+    not_ran: string;
+} | {
+    run_count: {
+        pattern: string;
+        min?: number;
+        max?: number;
+    };
+} | {
+    verify: {
+        run: string;
+        output_contains?: string;
+        output_equals?: string;
+    };
+};
+export interface AssertionResult {
+    assertion: Assertion;
+    passed: boolean;
+    actual?: string;
+    expected?: string;
+}
+export interface Task {
+    id: string;
+    intent: string;
+    assert: Assertion[];
+    setup?: string[];
+    max_turns?: number;
+    difficulty?: 'easy' | 'medium' | 'hard';
+    category?: string;
+    repeat?: number;
+}
+export interface TaskSuite {
+    cli: string;
+    version_command?: string;
+    tasks: Task[];
+}
+export type UploadMode = 'auto' | 'always' | 'never';
+export interface ThresholdsConfig {
+    default?: number;
+    models?: Record<string, number>;
+    tolerance?: number;
+    behavior?: 'error' | 'informational';
+}
+export interface ConfigFile {
+    cli: string;
+    version_command?: string;
+    providers?: string[];
+    help_modes?: string[];
+    concurrency?: number;
+    workdir?: string;
+    upload?: UploadMode;
+    backend_url?: string;
+    repeat?: number;
+    thresholds?: ThresholdsConfig;
+    tasks: (Task | string)[];
+}
+export type Provider = string;
+export interface ModelConfig {
+    provider: Provider;
+    modelId: string;
+    displayName: string;
+    temperature: number;
+    maxTokens: number;
+}
+export interface TaskEval {
+    taskId: string;
+    passed: boolean;
+    failureReason?: string;
+    assertionResults: AssertionResult[];
+    turnsUsed: number;
+    totalInputTokens: number;
+    totalOutputTokens: number;
+    totalLatencyMs: number;
+    commandsRun: string[];
+    difficulty?: 'easy' | 'medium' | 'hard';
+    category?: string;
+    conversationTrace?: unknown[];
+    taskDefinition?: Record<string, unknown>;
+    repeatIndex?: number;
+}
+export interface ModelResult {
+    provider: Provider;
+    modelId: string;
+    displayName: string;
+    helpMode: HelpMode;
+    taskResults: TaskEval[];
+    passRate: number;
+    avgTurnsToSuccess: number;
+    avgTotalTokens: number;
+    avgLatencyMs: number;
+}
+export interface ThresholdResult {
+    model: string;
+    passRate: number;
+    threshold: number;
+    tolerance: number;
+    effectiveMin: number;
+    passed: boolean;
+}
+export interface ThresholdCheckResult {
+    allPassed: boolean;
+    results: ThresholdResult[];
+    behavior: 'error' | 'informational';
+}
+export interface GridReport {
+    cli: string;
+    cliVersion?: string;
+    taskSuiteVersion: string;
+    gridVersion: string;
+    taskCount: number;
+    totalEvals?: number;
+    generatedAt: string;
+    modelResults: ModelResult[];
+    systemPrompt?: string;
+    gitSha?: string;
+    gitRef?: string;
+    gitBaseRef?: string;
+    commitMessage?: string;
+    gitAuthor?: string;
+    ciProvider?: string;
+    ciBuildUrl?: string;
+    ciEvent?: string;
+    prNumber?: number;
+    repository?: string;
+    tags?: string[];
+    taskSuiteHash?: string;
+    thresholdResults?: ThresholdCheckResult;
+}
+export interface HelpCache {
+    cli_name: string;
+    help_texts: Record<string, string>;
+    version?: string;
+}
+//# sourceMappingURL=models.d.ts.map

package/dist/models.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"models.d.ts","sourceRoot":"","sources":["../src/models.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAMH,MAAM,MAAM,QAAQ,GAAG,UAAU,GAAG,cAAc,GAAG,MAAM,CAAC;AAM5D,MAAM,MAAM,SAAS,GACjB;IAAE,eAAe,EAAE,MAAM,CAAA;CAAE,GAC3B;IAAE,aAAa,EAAE,MAAM,CAAA;CAAE,GACzB;IAAE,cAAc,EAAE,MAAM,CAAA;CAAE,GAC1B;IAAE,SAAS,EAAE,MAAM,CAAA;CAAE,GACrB;IAAE,WAAW,EAAE,MAAM,CAAA;CAAE,GACvB;IAAE,aAAa,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAA;CAAE,GACjD;IAAE,GAAG,EAAE,MAAM,CAAA;CAAE,GACf;IAAE,OAAO,EAAE,MAAM,CAAA;CAAE,GACnB;IAAE,SAAS,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,GAAG,CAAC,EAAE,MAAM,CAAC;QAAC,GAAG,CAAC,EAAE,MAAM,CAAA;KAAE,CAAA;CAAE,GAC9D;IAAE,MAAM,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,eAAe,CAAC,EAAE,MAAM,CAAC;QAAC,aAAa,CAAC,EAAE,MAAM,CAAA;KAAE,CAAA;CAAE,CAAC;AAElF,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,SAAS,CAAC;IACrB,MAAM,EAAE,OAAO,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,IAAI;IACnB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,SAAS,EAAE,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,MAAM,CAAC;IACxC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,SAAS;IACxB,GAAG,EAAE,MAAM,CAAC;IACZ,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,KAAK,EAAE,IAAI,EAAE,CAAC;CACf;AAMD,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,QAAQ,GAAG,OAAO,CAAC;AAErD,MAAM,WAAW,gBAAgB;IAC/B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,OAAO,GAAG,eAAe,CAAC;CACtC;AAED,MAAM,WAAW,UAAU;IACzB,GAAG,EAAE,MAAM,CAAC;IACZ,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,UAAU,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B,KAAK,EAAE,CAAC,IAAI,GAAG,MAAM,CAAC,EAAE,CAAC;CAC1B;AAMD,MAAM,MAAM,QAAQ,GAAG,MAAM,CAAC;AAE9B,MAAM,WAAW,WAAW;IAC1B,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;CACnB;AAMD,MAAM,WAAW,QAAQ;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,OAAO,CAAC;IAChB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,gBAAgB,EAAE,eAAe,EAAE,CAAC;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,UAAU,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,MAAM,CAAC;IACxC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,iBAAiB,CAAC,EAAE,OAAO,EAAE,CAAC;IAC9B,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACzC,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAMD,MAAM,WAAW,WAAW;IAC1B,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,QAAQ,CAAC;IACnB,WAAW,EAAE,QAAQ,EAAE,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,OAAO,CAAC;CACjB;AAED,MAAM,WAAW,oBAAoB;IACnC,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,EAAE,eAAe,EAAE,CAAC;IAC3B,QAAQ,EAAE,OAAO,GAAG,eAAe,CAAC;CACrC;AAED,MAAM,WAAW,UAAU;IACzB,GAAG,EAAE,MAAM,CAAC;IACZ,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,gBAAgB,EAAE,MAAM,CAAC;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,WAAW,EAAE,CAAC;IAC5B,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,gBAAgB,CAAC,EAAE,oBAAoB,CAAC;CACzC;AAMD,MAAM,WAAW,SAAS;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACnC,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB"}

package/dist/models.js

@@ -0,0 +1,8 @@
+/**
+ * TypeScript types for @cliwatch/cli-bench.
+ *
+ * Assertion-based evaluation: tasks define assertions that are checked
+ * against the agent's execution trace.
+ */
+export {};
+//# sourceMappingURL=models.js.map

package/dist/models.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"models.js","sourceRoot":"","sources":["../src/models.ts"],"names":[],"mappings":"AAAA;;;;;GAKG"}

package/dist/project.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Config file loader — discovers and parses cli-bench.yaml,
+ * resolves file:// task references with glob support.
+ */
+import type { Task, ConfigFile } from './models.js';
+/**
+ * Find cli-bench.yaml in the given directory (or CWD).
+ */
+export declare function resolveConfigFile(explicitPath?: string): Promise<string | null>;
+/**
+ * Parse and validate a cli-bench.yaml config file.
+ */
+export declare function loadConfigFile(path: string): Promise<ConfigFile>;
+/**
+ * Resolve file:// references and inline tasks into a flat task array.
+ * Deduplicates by task ID (first occurrence wins).
+ */
+export declare function resolveTaskRefs(tasks: (Task | string)[], baseDir: string): Promise<Task[]>;
+/**
+ * Load config file and resolve all task references.
+ */
+export declare function loadProject(configPath: string): Promise<{
+    config: ConfigFile;
+    tasks: Task[];
+}>;
+//# sourceMappingURL=project.d.ts.map

package/dist/project.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"project.d.ts","sourceRoot":"","sources":["../src/project.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAOH,OAAO,KAAK,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAIpD;;GAEG;AACH,wBAAsB,iBAAiB,CAAC,YAAY,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAqBrF;AAED;;GAEG;AACH,wBAAsB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAItE;AAED;;;GAGG;AACH,wBAAsB,eAAe,CACnC,KAAK,EAAE,CAAC,IAAI,GAAG,MAAM,CAAC,EAAE,EACxB,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,IAAI,EAAE,CAAC,CA6CjB;AAED;;GAEG;AACH,wBAAsB,WAAW,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC;IAAE,MAAM,EAAE,UAAU,CAAC;IAAC,KAAK,EAAE,IAAI,EAAE,CAAA;CAAE,CAAC,CAKpG"}