@cliwatch/cli-bench 0.6.3 → 0.7.1

package/dist/project.js

@@ -2,8 +2,8 @@
  * Config file loader — discovers and parses cli-bench.yaml,
  * resolves file:// task references with glob support.
  */
-import { readFile, access } from 'node:fs/promises';
-import { join, dirname, resolve } from 'node:path';
+import { readFile, readdir, access, stat, open } from 'node:fs/promises';
+import { join, dirname, resolve, relative } from 'node:path';
 import { glob } from 'node:fs/promises';
 import { parse as parseYaml, stringify as stringifyYaml } from 'yaml';
 import { ConfigFileSchema, TaskFileSchema, TaskSchema } from './schemas.js';
@@ -35,19 +35,23 @@ export async function resolveConfigFile(explicitPath) {
 }
 /**
  * Parse and validate a cli-bench.yaml config file.
+ * Returns the parsed config and the raw file content.
  */
 export async function loadConfigFile(path) {
-    const raw = await readFile(path, 'utf-8');
-    const parsed = parseYaml(raw);
-    return ConfigFileSchema.parse(parsed);
+    const rawContent = await readFile(path, 'utf-8');
+    const parsed = parseYaml(rawContent);
+    const config = ConfigFileSchema.parse(parsed);
+    return { config, rawContent };
 }
 /**
  * Resolve file:// references and inline tasks into a flat task array.
  * Deduplicates by task ID (first occurrence wins).
+ * Also tracks all loaded file paths and their raw contents.
  */
 export async function resolveTaskRefs(tasks, baseDir) {
     const resolved = [];
     const seenIds = new Set();
+    const loadedFiles = new Map();
     for (const entry of tasks) {
         if (typeof entry === 'string') {
             // file:// reference
@@ -68,6 +72,8 @@ export async function resolveTaskRefs(tasks, baseDir) {
             }
             for (const filePath of paths) {
                 const raw = await readFile(filePath, 'utf-8');
+                const relPath = relative(baseDir, filePath);
+                loadedFiles.set(relPath, raw);
                 const parsed = parseYaml(raw);
                 const fileTasks = TaskFileSchema.parse(parsed);
                 for (const task of fileTasks) {
@@ -87,16 +93,109 @@ export async function resolveTaskRefs(tasks, baseDir) {
             }
         }
     }
-    return resolved;
+    return { tasks: resolved, loadedFiles };
+}
+/**
+ * Assert that a path exists and is a directory.
+ */
+async function assertDirectory(absPath, label) {
+    let s;
+    try {
+        s = await stat(absPath);
+    }
+    catch {
+        throw new Error(`${label}: path does not exist: ${absPath}`);
+    }
+    if (!s.isDirectory()) {
+        throw new Error(`${label}: path is not a directory: ${absPath}`);
+    }
+}
+/**
+ * Recursively walk a directory, yielding absolute file paths.
+ */
+async function* walkDir(dir) {
+    const entries = await readdir(dir, { withFileTypes: true });
+    for (const entry of entries) {
+        const fullPath = join(dir, entry.name);
+        if (entry.isDirectory()) {
+            yield* walkDir(fullPath);
+        }
+        else if (entry.isFile()) {
+            yield fullPath;
+        }
+    }
+}
+/**
+ * Check if a file is likely a text file by reading the first 512 bytes
+ * and looking for null bytes.
+ */
+async function isTextFile(filePath) {
+    try {
+        const fh = await open(filePath, 'r');
+        try {
+            const buf = Buffer.alloc(512);
+            const { bytesRead } = await fh.read(buf, 0, 512, 0);
+            for (let i = 0; i < bytesRead; i++) {
+                if (buf[i] === 0)
+                    return false;
+            }
+            return true;
+        }
+        finally {
+            await fh.close();
+        }
+    }
+    catch {
+        return false;
+    }
 }
 /**
  * Load config file and resolve all task references.
  */
 export async function loadProject(configPath) {
-    const config = await loadConfigFile(configPath);
+    const { config, rawContent } = await loadConfigFile(configPath);
     const baseDir = dirname(configPath);
-    const tasks = await resolveTaskRefs(config.tasks, baseDir);
+    const { tasks, loadedFiles } = await resolveTaskRefs(config.tasks, baseDir);
+    // Serialize before resolving scaffold paths to avoid leaking absolute host paths
     const taskSuiteContent = stringifyYaml(tasks);
-    return { config, tasks, taskSuiteContent };
+    // Resolve scaffold paths to absolute and validate
+    if (config.scaffold) {
+        const abs = resolve(baseDir, config.scaffold);
+        await assertDirectory(abs, `config scaffold "${config.scaffold}"`);
+        config.scaffold = abs;
+    }
+    for (const task of tasks) {
+        if (typeof task.scaffold === 'string') {
+            const abs = resolve(baseDir, task.scaffold);
+            await assertDirectory(abs, `task "${task.id}" scaffold "${task.scaffold}"`);
+            task.scaffold = abs;
+        }
+    }
+    // Build projectFiles map: config file + all loaded task files
+    const configFileName = configPath.endsWith('.yml') ? 'cli-bench.yml' : 'cli-bench.yaml';
+    const projectFiles = {
+        [configFileName]: rawContent,
+    };
+    for (const [relPath, content] of loadedFiles) {
+        projectFiles[relPath] = content;
+    }
+    // Read all scaffold directories referenced by config or tasks
+    const scaffoldDirs = new Set();
+    if (config.scaffold)
+        scaffoldDirs.add(config.scaffold);
+    for (const task of tasks) {
+        if (typeof task.scaffold === 'string')
+            scaffoldDirs.add(task.scaffold);
+    }
+    for (const absDir of scaffoldDirs) {
+        const relDir = relative(baseDir, absDir);
+        for await (const entry of walkDir(absDir)) {
+            const relPath = join(relDir, relative(absDir, entry));
+            if (!(await isTextFile(entry)))
+                continue;
+            projectFiles[relPath] = await readFile(entry, 'utf-8');
+        }
+    }
+    return { config, tasks, taskSuiteContent, projectFiles };
 }
 //# sourceMappingURL=project.js.map

package/dist/project.js.map

@@ -1 +1 @@
-{"version":3,"file":"project.js","sourceRoot":"","sources":["../src/project.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AACxC,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,SAAS,IAAI,aAAa,EAAE,MAAM,MAAM,CAAC;AACtE,OAAO,EAAE,gBAAgB,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAG5E,MAAM,gBAAgB,GAAG,CAAC,gBAAgB,EAAE,eAAe,CAAC,CAAC;AAE7D;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,YAAqB;IAC3D,IAAI,YAAY,EAAE,CAAC;QACjB,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,YAAY,CAAC,CAAC;YAC3B,OAAO,OAAO,CAAC,YAAY,CAAC,CAAC;QAC/B,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,KAAK,MAAM,IAAI,IAAI,gBAAgB,EAAE,CAAC;QACpC,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;QAChC,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;YACxB,OAAO,SAAS,CAAC;QACnB,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,IAAY;IAC/C,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC1C,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;IAC9B,OAAO,gBAAgB,CAAC,KAAK,CAAC,MAAM,CAAe,CAAC;AACtD,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,KAAwB,EACxB,OAAe;IAEf,MAAM,QAAQ,GAAW,EAAE,CAAC;IAC5B,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAElC,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;QAC1B,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC9B,oBAAoB;YACpB,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;YAC5C,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;YAEnC,gCAAgC;YAChC,MAAM,KAAK,GAAa,EAAE,CAAC;YAC3B,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBACtB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;oBACxC,IAAI,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;wBACtD,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBACpB,CAAC;gBACH,CAAC;gBACD,KAAK,CAAC,IAAI,EAAE,CAAC;YACf,CAAC;iBAAM,CAAC;gBACN,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACtB,CAAC;YAED,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;gBAC7B,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;gBAC9C,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;gBAC9B,MAAM,SAAS,GAAG,cAAc,CAAC,KAAK,CAAC,MAAM,CAAW,CAAC;gBACzD,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;oBAC7B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;wBAC1B,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;wBACrB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBACtB,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;aAAM,CAAC;YACN,wCAAwC;YACxC,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,KAAK,CAAS,CAAC;YAC7C,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;gBAC1B,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBACrB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,UAAkB;IAClD,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,UAAU,CAAC,CAAC;IAChD,MAAM,OAAO,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;IACpC,MAAM,KAAK,GAAG,MAAM,eAAe,CAAC,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAC3D,MAAM,gBAAgB,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IAC9C,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,gBAAgB,EAAE,CAAC;AAC7C,CAAC"}
+{"version":3,"file":"project.js","sourceRoot":"","sources":["../src/project.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AACzE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAC7D,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AACxC,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,SAAS,IAAI,aAAa,EAAE,MAAM,MAAM,CAAC;AACtE,OAAO,EAAE,gBAAgB,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAG5E,MAAM,gBAAgB,GAAG,CAAC,gBAAgB,EAAE,eAAe,CAAC,CAAC;AAE7D;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,YAAqB;IAC3D,IAAI,YAAY,EAAE,CAAC;QACjB,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,YAAY,CAAC,CAAC;YAC3B,OAAO,OAAO,CAAC,YAAY,CAAC,CAAC;QAC/B,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,KAAK,MAAM,IAAI,IAAI,gBAAgB,EAAE,CAAC;QACpC,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;QAChC,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;YACxB,OAAO,SAAS,CAAC;QACnB,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,IAAY;IAC/C,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACjD,MAAM,MAAM,GAAG,SAAS,CAAC,UAAU,CAAC,CAAC;IACrC,MAAM,MAAM,GAAG,gBAAgB,CAAC,KAAK,CAAC,MAAM,CAAe,CAAC;IAC5D,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;AAChC,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,KAAwB,EACxB,OAAe;IAEf,MAAM,QAAQ,GAAW,EAAE,CAAC;IAC5B,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAClC,MAAM,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAC;IAE9C,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;QAC1B,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC9B,oBAAoB;YACpB,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;YAC5C,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;YAEnC,gCAAgC;YAChC,MAAM,KAAK,GAAa,EAAE,CAAC;YAC3B,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBACtB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;oBACxC,IAAI,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;wBACtD,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBACpB,CAAC;gBACH,CAAC;gBACD,KAAK,CAAC,IAAI,EAAE,CAAC;YACf,CAAC;iBAAM,CAAC;gBACN,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACtB,CAAC;YAED,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;gBAC7B,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;gBAC9C,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;gBAC5C,WAAW,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;gBAC9B,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;gBAC9B,MAAM,SAAS,GAAG,cAAc,CAAC,KAAK,CAAC,MAAM,CAAW,CAAC;gBACzD,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;oBAC7B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;wBAC1B,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;wBACrB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBACtB,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;aAAM,CAAC;YACN,wCAAwC;YACxC,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,KAAK,CAAS,CAAC;YAC7C,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;gBAC1B,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBACrB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,CAAC;AAC1C,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,eAAe,CAAC,OAAe,EAAE,KAAa;IAC3D,IAAI,CAAC,CAAC;IACN,IAAI,CAAC;QACH,CAAC,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,CAAC;IAC1B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,0BAA0B,OAAO,EAAE,CAAC,CAAC;IAC/D,CAAC;IACD,IAAI,CAAC,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC;QACrB,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,8BAA8B,OAAO,EAAE,CAAC,CAAC;IACnE,CAAC;AACH,CAAC;AAED;;GAEG;AACH,KAAK,SAAS,CAAC,CAAC,OAAO,CAAC,GAAW;IACjC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IAC5D,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QACvC,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;YACxB,KAAK,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC3B,CAAC;aAAM,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;YAC1B,MAAM,QAAQ,CAAC;QACjB,CAAC;IACH,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,UAAU,CAAC,QAAgB;IACxC,IAAI,CAAC;QACH,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QACrC,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAC9B,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;YACpD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;gBACnC,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC;oBAAE,OAAO,KAAK,CAAC;YACjC,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC;gBAAS,CAAC;YACT,MAAM,EAAE,CAAC,KAAK,EAAE,CAAC;QACnB,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,UAAkB;IAMlD,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,cAAc,CAAC,UAAU,CAAC,CAAC;IAChE,MAAM,OAAO,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;IACpC,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,GAAG,MAAM,eAAe,CAAC,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAE5E,iFAAiF;IACjF,MAAM,gBAAgB,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IAE9C,kDAAkD;IAClD,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QACpB,MAAM,GAAG,GAAG,OAAO,CAAC,OAAO,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC9C,MAAM,eAAe,CAAC,GAAG,EAAE,oBAAoB,MAAM,CAAC,QAAQ,GAAG,CAAC,CAAC;QACnE,MAAM,CAAC,QAAQ,GAAG,GAAG,CAAC;IACxB,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,OAAO,IAAI,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;YACtC,MAAM,GAAG,GAAG,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC5C,MAAM,eAAe,CAAC,GAAG,EAAE,SAAS,IAAI,CAAC,EAAE,eAAe,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YAC5E,IAAI,CAAC,QAAQ,GAAG,GAAG,CAAC;QACtB,CAAC;IACH,CAAC;IAED,8DAA8D;IAC9D,MAAM,cAAc,GAAG,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,gBAAgB,CAAC;IACxF,MAAM,YAAY,GAA2B;QAC3C,CAAC,cAAc,CAAC,EAAE,UAAU;KAC7B,CAAC;IACF,KAAK,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,IAAI,WAAW,EAAE,CAAC;QAC7C,YAAY,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC;IAClC,CAAC;IAED,8DAA8D;IAC9D,MAAM,YAAY,GAAG,IAAI,GAAG,EAAU,CAAC;IACvC,IAAI,MAAM,CAAC,QAAQ;QAAE,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IACvD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,OAAO,IAAI,CAAC,QAAQ,KAAK,QAAQ;YAAE,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACzE,CAAC;IACD,KAAK,MAAM,MAAM,IAAI,YAAY,EAAE,CAAC;QAClC,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QACzC,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YAC1C,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC;YACtD,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,KAAK,CAAC,CAAC;gBAAE,SAAS;YACzC,YAAY,CAAC,OAAO,CAAC,GAAG,MAAM,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QACzD,CAAC;IACH,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,CAAC;AAC3D,CAAC"}

package/dist/prompt.d.ts

@@ -1,13 +1,7 @@
 /**
  * Builds prompts for CLI benchmark agents.
- *
- * One unified system message for all context modes.
- * The user message varies by context mode:
- * - zero-shot: CLI name + task intent only
- * - help: CLI name + top-level --help output + task intent
- * - docs: CLI name + documentation contents + task intent
  */
-import type { ContextMode, Task } from './models.js';
+import type { Task } from './models.js';
 export declare function buildSystemMessage(customPrompt?: string): string;
-export declare function buildUserMessage(cliName: string, task: Task, contextMode: ContextMode, contextPayload?: string | null): string;
+export declare function buildUserMessage(cliName: string, task: Task): string;
 //# sourceMappingURL=prompt.d.ts.map

package/dist/prompt.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"prompt.d.ts","sourceRoot":"","sources":["../src/prompt.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AASrD,wBAAgB,kBAAkB,CAAC,YAAY,CAAC,EAAE,MAAM,GAAG,MAAM,CAKhE;AAED,wBAAgB,gBAAgB,CAC9B,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,IAAI,EACV,WAAW,EAAE,WAAW,EACxB,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,GAC7B,MAAM,CA+BR"}
+{"version":3,"file":"prompt.d.ts","sourceRoot":"","sources":["../src/prompt.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AASxC,wBAAgB,kBAAkB,CAAC,YAAY,CAAC,EAAE,MAAM,GAAG,MAAM,CAKhE;AAED,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,GAAG,MAAM,CAEpE"}

package/dist/prompt.js

@@ -1,11 +1,5 @@
 /**
  * Builds prompts for CLI benchmark agents.
- *
- * One unified system message for all context modes.
- * The user message varies by context mode:
- * - zero-shot: CLI name + task intent only
- * - help: CLI name + top-level --help output + task intent
- * - docs: CLI name + documentation contents + task intent
  */
 const AGENT_SYSTEM_MESSAGE = `You are a CLI expert. You have a run_command tool to execute shell commands. Use it to accomplish the user's task.
 
@@ -19,34 +13,7 @@ export function buildSystemMessage(customPrompt) {
     }
     return AGENT_SYSTEM_MESSAGE;
 }
-export function buildUserMessage(cliName, task, contextMode, contextPayload) {
-    switch (contextMode) {
-        case 'help':
-            if (contextPayload) {
-                return `CLI: ${cliName}
-
-Help text:
-$ ${cliName} --help
-${contextPayload}
-
-Task: ${task.intent}`;
-            }
-            // Fall through to zero-shot if no help available
-            return `CLI: ${cliName}\n\nTask: ${task.intent}`;
-        case 'docs':
-            if (contextPayload) {
-                return `CLI: ${cliName}
-
-Documentation:
-${contextPayload}
-
-Task: ${task.intent}`;
-            }
-            // Fall through to zero-shot if no docs available
-            return `CLI: ${cliName}\n\nTask: ${task.intent}`;
-        case 'zero-shot':
-        default:
-            return `CLI: ${cliName}\n\nTask: ${task.intent}`;
-    }
+export function buildUserMessage(cliName, task) {
+    return `CLI: ${cliName}\n\nTask: ${task.intent}`;
 }
 //# sourceMappingURL=prompt.js.map

package/dist/prompt.js.map

@@ -1 +1 @@
-{"version":3,"file":"prompt.js","sourceRoot":"","sources":["../src/prompt.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAIH,MAAM,oBAAoB,GAAG;;;;;gDAKmB,CAAC;AAEjD,MAAM,UAAU,kBAAkB,CAAC,YAAqB;IACtD,IAAI,YAAY,EAAE,CAAC;QACjB,OAAO,GAAG,oBAAoB,OAAO,YAAY,EAAE,CAAC;IACtD,CAAC;IACD,OAAO,oBAAoB,CAAC;AAC9B,CAAC;AAED,MAAM,UAAU,gBAAgB,CAC9B,OAAe,EACf,IAAU,EACV,WAAwB,EACxB,cAA8B;IAE9B,QAAQ,WAAW,EAAE,CAAC;QACpB,KAAK,MAAM;YACT,IAAI,cAAc,EAAE,CAAC;gBACnB,OAAO,QAAQ,OAAO;;;IAG1B,OAAO;EACT,cAAc;;QAER,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,CAAC;YACD,iDAAiD;YACjD,OAAO,QAAQ,OAAO,aAAa,IAAI,CAAC,MAAM,EAAE,CAAC;QAEnD,KAAK,MAAM;YACT,IAAI,cAAc,EAAE,CAAC;gBACnB,OAAO,QAAQ,OAAO;;;EAG5B,cAAc;;QAER,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,CAAC;YACD,iDAAiD;YACjD,OAAO,QAAQ,OAAO,aAAa,IAAI,CAAC,MAAM,EAAE,CAAC;QAEnD,KAAK,WAAW,CAAC;QACjB;YACE,OAAO,QAAQ,OAAO,aAAa,IAAI,CAAC,MAAM,EAAE,CAAC;IACrD,CAAC;AACH,CAAC"}
+{"version":3,"file":"prompt.js","sourceRoot":"","sources":["../src/prompt.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,MAAM,oBAAoB,GAAG;;;;;gDAKmB,CAAC;AAEjD,MAAM,UAAU,kBAAkB,CAAC,YAAqB;IACtD,IAAI,YAAY,EAAE,CAAC;QACjB,OAAO,GAAG,oBAAoB,OAAO,YAAY,EAAE,CAAC;IACtD,CAAC;IACD,OAAO,oBAAoB,CAAC;AAC9B,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,OAAe,EAAE,IAAU;IAC1D,OAAO,QAAQ,OAAO,aAAa,IAAI,CAAC,MAAM,EAAE,CAAC;AACnD,CAAC"}

package/dist/providers.d.ts

@@ -1,16 +1,17 @@
 /**
- * Model registry + AI Gateway configuration.
+ * Model registry + provider routing.
  *
- * Uses Vercel AI SDK v6 gateway() which is re-exported from 'ai'.
- * Model IDs use the gateway format: "provider/model-id".
+ * Google models use @ai-sdk/google directly when GOOGLE_GENERATIVE_AI_API_KEY
+ * is set, bypassing the gateway. All other models use the Vercel AI Gateway.
  */
+import { type LanguageModel } from 'ai';
 export interface ModelEntry {
     id: string;
     displayName: string;
     provider: string;
 }
 export declare const MODELS: readonly ModelEntry[];
-export declare function getModel(modelId: string): import("@ai-sdk/provider").LanguageModelV3;
+export declare function getModel(modelId: string): LanguageModel;
 export declare function getModelEntry(modelId: string): ModelEntry | undefined;
 export declare function filterModels(modelIds: string[]): ModelEntry[];
 /**
@@ -19,8 +20,9 @@ export declare function filterModels(modelIds: string[]): ModelEntry[];
  */
 export declare function resolveProviders(providerIds: string[]): ModelEntry[];
 /**
- * Validate that AI_GATEWAY_API_KEY is set.
- * All model calls go through the AI Gateway — no per-provider keys are needed.
+ * Validate that the required API keys are set for the given models.
+ * Google models can use either GOOGLE_GENERATIVE_AI_API_KEY (direct) or AI_GATEWAY_API_KEY.
+ * All other models require AI_GATEWAY_API_KEY.
  */
-export declare function validateGatewayKey(): void;
+export declare function validateApiKeys(modelIds: string[]): void;
 //# sourceMappingURL=providers.d.ts.map

package/dist/providers.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"providers.d.ts","sourceRoot":"","sources":["../src/providers.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,eAAO,MAAM,MAAM,EAAE,SAAS,UAAU,EAc9B,CAAC;AAEX,wBAAgB,QAAQ,CAAC,OAAO,EAAE,MAAM,8CAEvC;AAED,wBAAgB,aAAa,CAAC,OAAO,EAAE,MAAM,GAAG,UAAU,GAAG,SAAS,CAErE;AAED,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,UAAU,EAAE,CAG7D;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,WAAW,EAAE,MAAM,EAAE,GAAG,UAAU,EAAE,CAWpE;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,IAAI,IAAI,CAMzC"}
+{"version":3,"file":"providers.d.ts","sourceRoot":"","sources":["../src/providers.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAW,KAAK,aAAa,EAAE,MAAM,IAAI,CAAC;AAGjD,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,eAAO,MAAM,MAAM,EAAE,SAAS,UAAU,EAiB9B,CAAC;AAEX,wBAAgB,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,aAAa,CAQvD;AAED,wBAAgB,aAAa,CAAC,OAAO,EAAE,MAAM,GAAG,UAAU,GAAG,SAAS,CAErE;AAED,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,UAAU,EAAE,CAG7D;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,WAAW,EAAE,MAAM,EAAE,GAAG,UAAU,EAAE,CAWpE;AAED;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,IAAI,CAgBxD"}

package/dist/providers.js

@@ -1,10 +1,11 @@
 /**
- * Model registry + AI Gateway configuration.
+ * Model registry + provider routing.
  *
- * Uses Vercel AI SDK v6 gateway() which is re-exported from 'ai'.
- * Model IDs use the gateway format: "provider/model-id".
+ * Google models use @ai-sdk/google directly when GOOGLE_GENERATIVE_AI_API_KEY
+ * is set, bypassing the gateway. All other models use the Vercel AI Gateway.
  */
 import { gateway } from 'ai';
+import { createGoogleGenerativeAI } from '@ai-sdk/google';
 export const MODELS = [
     { id: 'anthropic/claude-opus-4.6', displayName: 'Claude Opus 4.6', provider: 'anthropic' },
     { id: 'anthropic/claude-sonnet-4.6', displayName: 'Claude Sonnet 4.6', provider: 'anthropic' },
@@ -14,13 +15,22 @@ export const MODELS = [
     { id: 'openai/gpt-5.2', displayName: 'GPT-5.2', provider: 'openai' },
     { id: 'openai/gpt-4o', displayName: 'GPT-4o', provider: 'openai' },
     { id: 'openai/gpt-4o-mini', displayName: 'GPT-4o Mini', provider: 'openai' },
+    { id: 'google/gemini-3-pro', displayName: 'Gemini 3 Pro', provider: 'google' },
     { id: 'google/gemini-3-flash', displayName: 'Gemini 3 Flash', provider: 'google' },
     { id: 'google/gemini-2.5-pro', displayName: 'Gemini 2.5 Pro', provider: 'google' },
     { id: 'google/gemini-2.5-flash', displayName: 'Gemini 2.5 Flash', provider: 'google' },
+    { id: 'google/gemini-2.5-flash-lite', displayName: 'Gemini 2.5 Flash Lite', provider: 'google' },
+    { id: 'google/gemini-2.0-flash', displayName: 'Gemini 2.0 Flash', provider: 'google' },
     { id: 'meta/llama-3.1-8b', displayName: 'Llama 3.1 8B', provider: 'meta' },
     { id: 'mistral/ministral-3b', displayName: 'Ministral 3B', provider: 'mistral' },
 ];
 export function getModel(modelId) {
+    // Direct Google API — takes priority when key is set
+    if (modelId.startsWith('google/') && process.env['GOOGLE_GENERATIVE_AI_API_KEY']) {
+        const google = createGoogleGenerativeAI();
+        return google(modelId.replace('google/', ''));
+    }
+    // All other models go through Vercel AI Gateway
     return gateway(modelId);
 }
 export function getModelEntry(modelId) {
@@ -49,12 +59,20 @@ export function resolveProviders(providerIds) {
     });
 }
 /**
- * Validate that AI_GATEWAY_API_KEY is set.
- * All model calls go through the AI Gateway — no per-provider keys are needed.
+ * Validate that the required API keys are set for the given models.
+ * Google models can use either GOOGLE_GENERATIVE_AI_API_KEY (direct) or AI_GATEWAY_API_KEY.
+ * All other models require AI_GATEWAY_API_KEY.
  */
-export function validateGatewayKey() {
-    if (!process.env['AI_GATEWAY_API_KEY']) {
-        throw new Error('Missing AI_GATEWAY_API_KEY. All model calls require the AI Gateway.\nSet AI_GATEWAY_API_KEY in your environment before running cli-bench.');
+export function validateApiKeys(modelIds) {
+    const hasGoogleKey = !!process.env['GOOGLE_GENERATIVE_AI_API_KEY'];
+    const hasGatewayKey = !!process.env['AI_GATEWAY_API_KEY'];
+    const googleModels = modelIds.filter((id) => id.startsWith('google/'));
+    const otherModels = modelIds.filter((id) => !id.startsWith('google/'));
+    if (googleModels.length > 0 && !hasGoogleKey && !hasGatewayKey) {
+        throw new Error('Google models require GOOGLE_GENERATIVE_AI_API_KEY or AI_GATEWAY_API_KEY.');
+    }
+    if (otherModels.length > 0 && !hasGatewayKey) {
+        throw new Error(`Non-Google models (${otherModels.join(', ')}) require AI_GATEWAY_API_KEY.`);
     }
 }
 //# sourceMappingURL=providers.js.map

package/dist/providers.js.map

@@ -1 +1 @@
-{"version":3,"file":"providers.js","sourceRoot":"","sources":["../src/providers.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,IAAI,CAAC;AAQ7B,MAAM,CAAC,MAAM,MAAM,GAA0B;IAC3C,EAAE,EAAE,EAAE,2BAA2B,EAAE,WAAW,EAAE,iBAAiB,EAAE,QAAQ,EAAE,WAAW,EAAE;IAC1F,EAAE,EAAE,EAAE,6BAA6B,EAAE,WAAW,EAAE,mBAAmB,EAAE,QAAQ,EAAE,WAAW,EAAE;IAC9F,EAAE,EAAE,EAAE,oCAAoC,EAAE,WAAW,EAAE,iBAAiB,EAAE,QAAQ,EAAE,WAAW,EAAE;IACnG,EAAE,EAAE,EAAE,4BAA4B,EAAE,WAAW,EAAE,kBAAkB,EAAE,QAAQ,EAAE,WAAW,EAAE;IAC5F,EAAE,EAAE,EAAE,qCAAqC,EAAE,WAAW,EAAE,kBAAkB,EAAE,QAAQ,EAAE,WAAW,EAAE;IACrG,EAAE,EAAE,EAAE,gBAAgB,EAAE,WAAW,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,EAAE;IACpE,EAAE,EAAE,EAAE,eAAe,EAAE,WAAW,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE;IAClE,EAAE,EAAE,EAAE,oBAAoB,EAAE,WAAW,EAAE,aAAa,EAAE,QAAQ,EAAE,QAAQ,EAAE;IAC5E,EAAE,EAAE,EAAE,uBAAuB,EAAE,WAAW,EAAE,gBAAgB,EAAE,QAAQ,EAAE,QAAQ,EAAE;IAClF,EAAE,EAAE,EAAE,uBAAuB,EAAE,WAAW,EAAE,gBAAgB,EAAE,QAAQ,EAAE,QAAQ,EAAE;IAClF,EAAE,EAAE,EAAE,yBAAyB,EAAE,WAAW,EAAE,kBAAkB,EAAE,QAAQ,EAAE,QAAQ,EAAE;IACtF,EAAE,EAAE,EAAE,mBAAmB,EAAE,WAAW,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,EAAE;IAC1E,EAAE,EAAE,EAAE,sBAAsB,EAAE,WAAW,EAAE,cAAc,EAAE,QAAQ,EAAE,SAAS,EAAE;CACxE,CAAC;AAEX,MAAM,UAAU,QAAQ,CAAC,OAAe;IACtC,OAAO,OAAO,CAAC,OAAO,CAAC,CAAC;AAC1B,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,OAAe;IAC3C,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,OAAO,CAAC,CAAC;AAC9C,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,QAAkB;IAC7C,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,GAAG,MAAM,CAAC,CAAC;IAC9C,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AACvD,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,WAAqB;IACpD,OAAO,WAAW,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;QAC5B,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;QACjD,IAAI,QAAQ;YAAE,OAAO,EAAE,GAAG,QAAQ,EAAE,CAAC;QACrC,MAAM,CAAC,QAAQ,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC1C,OAAO;YACL,EAAE;YACF,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;YAC3B,QAAQ,EAAE,QAAkC;SAC7C,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,kBAAkB;IAChC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,CAAC;QACvC,MAAM,IAAI,KAAK,CACb,2IAA2I,CAC5I,CAAC;IACJ,CAAC;AACH,CAAC"}
+{"version":3,"file":"providers.js","sourceRoot":"","sources":["../src/providers.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,OAAO,EAAsB,MAAM,IAAI,CAAC;AACjD,OAAO,EAAE,wBAAwB,EAAE,MAAM,gBAAgB,CAAC;AAQ1D,MAAM,CAAC,MAAM,MAAM,GAA0B;IAC3C,EAAE,EAAE,EAAE,2BAA2B,EAAE,WAAW,EAAE,iBAAiB,EAAE,QAAQ,EAAE,WAAW,EAAE;IAC1F,EAAE,EAAE,EAAE,6BAA6B,EAAE,WAAW,EAAE,mBAAmB,EAAE,QAAQ,EAAE,WAAW,EAAE;IAC9F,EAAE,EAAE,EAAE,oCAAoC,EAAE,WAAW,EAAE,iBAAiB,EAAE,QAAQ,EAAE,WAAW,EAAE;IACnG,EAAE,EAAE,EAAE,4BAA4B,EAAE,WAAW,EAAE,kBAAkB,EAAE,QAAQ,EAAE,WAAW,EAAE;IAC5F,EAAE,EAAE,EAAE,qCAAqC,EAAE,WAAW,EAAE,kBAAkB,EAAE,QAAQ,EAAE,WAAW,EAAE;IACrG,EAAE,EAAE,EAAE,gBAAgB,EAAE,WAAW,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,EAAE;IACpE,EAAE,EAAE,EAAE,eAAe,EAAE,WAAW,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE;IAClE,EAAE,EAAE,EAAE,oBAAoB,EAAE,WAAW,EAAE,aAAa,EAAE,QAAQ,EAAE,QAAQ,EAAE;IAC5E,EAAE,EAAE,EAAE,qBAAqB,EAAE,WAAW,EAAE,cAAc,EAAE,QAAQ,EAAE,QAAQ,EAAE;IAC9E,EAAE,EAAE,EAAE,uBAAuB,EAAE,WAAW,EAAE,gBAAgB,EAAE,QAAQ,EAAE,QAAQ,EAAE;IAClF,EAAE,EAAE,EAAE,uBAAuB,EAAE,WAAW,EAAE,gBAAgB,EAAE,QAAQ,EAAE,QAAQ,EAAE;IAClF,EAAE,EAAE,EAAE,yBAAyB,EAAE,WAAW,EAAE,kBAAkB,EAAE,QAAQ,EAAE,QAAQ,EAAE;IACtF,EAAE,EAAE,EAAE,8BAA8B,EAAE,WAAW,EAAE,uBAAuB,EAAE,QAAQ,EAAE,QAAQ,EAAE;IAChG,EAAE,EAAE,EAAE,yBAAyB,EAAE,WAAW,EAAE,kBAAkB,EAAE,QAAQ,EAAE,QAAQ,EAAE;IACtF,EAAE,EAAE,EAAE,mBAAmB,EAAE,WAAW,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,EAAE;IAC1E,EAAE,EAAE,EAAE,sBAAsB,EAAE,WAAW,EAAE,cAAc,EAAE,QAAQ,EAAE,SAAS,EAAE;CACxE,CAAC;AAEX,MAAM,UAAU,QAAQ,CAAC,OAAe;IACtC,qDAAqD;IACrD,IAAI,OAAO,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,OAAO,CAAC,GAAG,CAAC,8BAA8B,CAAC,EAAE,CAAC;QACjF,MAAM,MAAM,GAAG,wBAAwB,EAAE,CAAC;QAC1C,OAAO,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,CAAC;IAChD,CAAC;IACD,gDAAgD;IAChD,OAAO,OAAO,CAAC,OAAO,CAAC,CAAC;AAC1B,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,OAAe;IAC3C,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,OAAO,CAAC,CAAC;AAC9C,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,QAAkB;IAC7C,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,GAAG,MAAM,CAAC,CAAC;IAC9C,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AACvD,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,WAAqB;IACpD,OAAO,WAAW,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;QAC5B,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;QACjD,IAAI,QAAQ;YAAE,OAAO,EAAE,GAAG,QAAQ,EAAE,CAAC;QACrC,MAAM,CAAC,QAAQ,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC1C,OAAO;YACL,EAAE;YACF,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;YAC3B,QAAQ,EAAE,QAAkC;SAC7C,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAAC,QAAkB;IAChD,MAAM,YAAY,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,8BAA8B,CAAC,CAAC;IACnE,MAAM,aAAa,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;IAC1D,MAAM,YAAY,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC;IACvE,MAAM,WAAW,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC;IAEvE,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,YAAY,IAAI,CAAC,aAAa,EAAE,CAAC;QAC/D,MAAM,IAAI,KAAK,CACb,2EAA2E,CAC5E,CAAC;IACJ,CAAC;IACD,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;QAC7C,MAAM,IAAI,KAAK,CACb,sBAAsB,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,+BAA+B,CAC5E,CAAC;IACJ,CAAC;AACH,CAAC"}

package/dist/runner.d.ts

@@ -1,5 +1,5 @@
 /**
- * Orchestrates the task x model x contextMode matrix with tool-calling agents.
+ * Orchestrates the task x model matrix with tool-calling agents.
  *
  * For each task:
  *   - Create temp workdir (or use configured one)
@@ -9,11 +9,29 @@
  *   - After maxSteps or completion: run assertions
  *   - Aggregate into GridReport
  */
+import { type LanguageModel } from 'ai';
 import type { TaskSuite, Task, GridReport } from './models.js';
 import type { Config } from './config.js';
 import type { ModelEntry } from './providers.js';
 export declare function loadTaskSuite(filePath: string): Promise<TaskSuite>;
-export declare function discoverTaskSuites(suiteDir: string): Promise<Map<string, string>>;
+/**
+ * Replace {{workdir}} in a string with the actual path.
+ */
+export declare function resolveTemplate(s: string, vars: {
+    workdir: string;
+}): string;
+/**
+ * Resolve env var templates: replace {{workdir}} with actual path.
+ */
+export declare function resolveEnv(env: Record<string, string> | undefined, vars: {
+    workdir: string;
+}): Record<string, string> | undefined;
+/**
+ * Resolve {{workdir}} in an array of command strings.
+ */
+export declare function resolveCommands(commands: string[], vars: {
+    workdir: string;
+}): string[];
 export interface RunGridOptions {
     config: Config;
     /** Override tasks (from config file mode). */
@@ -37,12 +55,22 @@ export interface RunGridOptions {
     githubUrl?: string;
     /** Serialized YAML of the resolved task suite. */
     taskSuiteContent?: string;
-    /** Directory containing the config file (for resolving docs.md). */
-    configDir?: string;
     /** Env var names whose values should be redacted from results. */
     redactEnvVars?: string[];
     /** Regex patterns to redact from results (e.g. API-echoed account IDs). */
     redactPatterns?: string[];
+    /** Config-level env vars (merged with per-task env, supports {{workdir}}). */
+    fileEnv?: Record<string, string>;
+    /** Config-level setup commands (run before each task). */
+    fileSetup?: string[];
+    /** Config-level cleanup commands (run after each task). */
+    fileCleanup?: string[];
+    /** Config-level scaffold directory (copied into workdir before setup). */
+    fileScaffold?: string;
+    /** Original project files (config + task files) for upload. */
+    projectFiles?: Record<string, string>;
+    /** Override the language model (for testing with mock models). */
+    modelOverride?: LanguageModel;
 }
 export declare function runGrid(opts: RunGridOptions): Promise<GridReport[]>;
 export declare function uploadReport(report: GridReport, backendUrl: string, apiKey: string): Promise<void>;

package/dist/runner.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../src/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AASH,OAAO,KAAK,EACV,SAAS,EACT,IAAI,EACJ,UAAU,EAKX,MAAM,aAAa,CAAC;AACrB,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAUjD,wBAAsB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,CAKxE;AAED,wBAAsB,kBAAkB,CACtC,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAc9B;AAiOD,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,8CAA8C;IAC9C,KAAK,CAAC,EAAE,IAAI,EAAE,CAAC;IACf,iDAAiD;IACjD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,+CAA+C;IAC/C,MAAM,CAAC,EAAE,UAAU,EAAE,CAAC;IACtB,wDAAwD;IACxD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,0DAA0D;IAC1D,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,uDAAuD;IACvD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,oDAAoD;IACpD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,mDAAmD;IACnD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,kDAAkD;IAClD,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,oEAAoE;IACpE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,kEAAkE;IAClE,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IACzB,2EAA2E;IAC3E,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;CAC3B;AAED,wBAAsB,OAAO,CAAC,IAAI,EAAE,cAAc,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC,CAsEzE;AA+PD,wBAAsB,YAAY,CAChC,MAAM,EAAE,UAAU,EAClB,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,IAAI,CAAC,CAsBf"}
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../src/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAMH,OAAO,EAA6B,KAAK,aAAa,EAAE,MAAM,IAAI,CAAC;AAGnE,OAAO,KAAK,EACV,SAAS,EACT,IAAI,EACJ,UAAU,EAIX,MAAM,aAAa,CAAC;AACrB,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AASjD,wBAAsB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,CAKxE;AAGD;;GAEG;AACH,wBAAgB,eAAe,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE;IAAE,OAAO,EAAE,MAAM,CAAA;CAAE,GAAG,MAAM,CAE5E;AAED;;GAEG;AACH,wBAAgB,UAAU,CACxB,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,EACvC,IAAI,EAAE;IAAE,OAAO,EAAE,MAAM,CAAA;CAAE,GACxB,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,CAOpC;AAED;;GAEG;AACH,wBAAgB,eAAe,CAC7B,QAAQ,EAAE,MAAM,EAAE,EAClB,IAAI,EAAE;IAAE,OAAO,EAAE,MAAM,CAAA;CAAE,GACxB,MAAM,EAAE,CAEV;AAkOD,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,8CAA8C;IAC9C,KAAK,CAAC,EAAE,IAAI,EAAE,CAAC;IACf,iDAAiD;IACjD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,+CAA+C;IAC/C,MAAM,CAAC,EAAE,UAAU,EAAE,CAAC;IACtB,wDAAwD;IACxD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,0DAA0D;IAC1D,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,uDAAuD;IACvD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,oDAAoD;IACpD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,mDAAmD;IACnD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,kDAAkD;IAClD,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,kEAAkE;IAClE,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IACzB,2EAA2E;IAC3E,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,8EAA8E;IAC9E,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,0DAA0D;IAC1D,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,2DAA2D;IAC3D,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,0EAA0E;IAC1E,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,+DAA+D;IAC/D,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtC,kEAAkE;IAClE,aAAa,CAAC,EAAE,aAAa,CAAC;CAC/B;AAED,wBAAsB,OAAO,CAAC,IAAI,EAAE,cAAc,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC,CAmCzE;AA8RD,wBAAsB,YAAY,CAChC,MAAM,EAAE,UAAU,EAClB,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,IAAI,CAAC,CAsBf"}