@cliwatch/cli-bench 0.4.0

package/dist/project.js

@@ -0,0 +1,101 @@
+/**
+ * Config file loader — discovers and parses cli-bench.yaml,
+ * resolves file:// task references with glob support.
+ */
+import { readFile, access } from 'node:fs/promises';
+import { join, dirname, resolve } from 'node:path';
+import { glob } from 'node:fs/promises';
+import { parse as parseYaml } from 'yaml';
+import { ConfigFileSchema, TaskFileSchema, TaskSchema } from './schemas.js';
+const CONFIG_FILENAMES = ['cli-bench.yaml', 'cli-bench.yml'];
+/**
+ * Find cli-bench.yaml in the given directory (or CWD).
+ */
+export async function resolveConfigFile(explicitPath) {
+    if (explicitPath) {
+        try {
+            await access(explicitPath);
+            return resolve(explicitPath);
+        }
+        catch {
+            return null;
+        }
+    }
+    for (const name of CONFIG_FILENAMES) {
+        const candidate = resolve(name);
+        try {
+            await access(candidate);
+            return candidate;
+        }
+        catch {
+            continue;
+        }
+    }
+    return null;
+}
+/**
+ * Parse and validate a cli-bench.yaml config file.
+ */
+export async function loadConfigFile(path) {
+    const raw = await readFile(path, 'utf-8');
+    const parsed = parseYaml(raw);
+    return ConfigFileSchema.parse(parsed);
+}
+/**
+ * Resolve file:// references and inline tasks into a flat task array.
+ * Deduplicates by task ID (first occurrence wins).
+ */
+export async function resolveTaskRefs(tasks, baseDir) {
+    const resolved = [];
+    const seenIds = new Set();
+    for (const entry of tasks) {
+        if (typeof entry === 'string') {
+            // file:// reference
+            const ref = entry.replace(/^file:\/\//, '');
+            const pattern = join(baseDir, ref);
+            // Use glob to resolve wildcards
+            const paths = [];
+            if (ref.includes('*')) {
+                for await (const match of glob(pattern)) {
+                    if (match.endsWith('.yaml') || match.endsWith('.yml')) {
+                        paths.push(match);
+                    }
+                }
+                paths.sort();
+            }
+            else {
+                paths.push(pattern);
+            }
+            for (const filePath of paths) {
+                const raw = await readFile(filePath, 'utf-8');
+                const parsed = parseYaml(raw);
+                const fileTasks = TaskFileSchema.parse(parsed);
+                for (const task of fileTasks) {
+                    if (!seenIds.has(task.id)) {
+                        seenIds.add(task.id);
+                        resolved.push(task);
+                    }
+                }
+            }
+        }
+        else {
+            // Inline task — validate through schema
+            const task = TaskSchema.parse(entry);
+            if (!seenIds.has(task.id)) {
+                seenIds.add(task.id);
+                resolved.push(task);
+            }
+        }
+    }
+    return resolved;
+}
+/**
+ * Load config file and resolve all task references.
+ */
+export async function loadProject(configPath) {
+    const config = await loadConfigFile(configPath);
+    const baseDir = dirname(configPath);
+    const tasks = await resolveTaskRefs(config.tasks, baseDir);
+    return { config, tasks };
+}
+//# sourceMappingURL=project.js.map

package/dist/project.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"project.js","sourceRoot":"","sources":["../src/project.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AACxC,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,MAAM,CAAC;AAC1C,OAAO,EAAE,gBAAgB,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAG5E,MAAM,gBAAgB,GAAG,CAAC,gBAAgB,EAAE,eAAe,CAAC,CAAC;AAE7D;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,YAAqB;IAC3D,IAAI,YAAY,EAAE,CAAC;QACjB,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,YAAY,CAAC,CAAC;YAC3B,OAAO,OAAO,CAAC,YAAY,CAAC,CAAC;QAC/B,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,KAAK,MAAM,IAAI,IAAI,gBAAgB,EAAE,CAAC;QACpC,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;QAChC,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;YACxB,OAAO,SAAS,CAAC;QACnB,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,IAAY;IAC/C,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC1C,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;IAC9B,OAAO,gBAAgB,CAAC,KAAK,CAAC,MAAM,CAAe,CAAC;AACtD,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,KAAwB,EACxB,OAAe;IAEf,MAAM,QAAQ,GAAW,EAAE,CAAC;IAC5B,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAElC,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;QAC1B,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC9B,oBAAoB;YACpB,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;YAC5C,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;YAEnC,gCAAgC;YAChC,MAAM,KAAK,GAAa,EAAE,CAAC;YAC3B,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBACtB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;oBACxC,IAAI,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;wBACtD,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBACpB,CAAC;gBACH,CAAC;gBACD,KAAK,CAAC,IAAI,EAAE,CAAC;YACf,CAAC;iBAAM,CAAC;gBACN,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACtB,CAAC;YAED,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;gBAC7B,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;gBAC9C,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;gBAC9B,MAAM,SAAS,GAAG,cAAc,CAAC,KAAK,CAAC,MAAM,CAAW,CAAC;gBACzD,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;oBAC7B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;wBAC1B,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;wBACrB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBACtB,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;aAAM,CAAC;YACN,wCAAwC;YACxC,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,KAAK,CAAS,CAAC;YAC7C,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;gBAC1B,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBACrB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,UAAkB;IAClD,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,UAAU,CAAC,CAAC;IAChD,MAAM,OAAO,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;IACpC,MAAM,KAAK,GAAG,MAAM,eAAe,CAAC,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAC3D,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;AAC3B,CAAC"}

package/dist/prompt.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Builds prompts from help text + task definition.
+ *
+ * Three help modes:
+ * - injected: help text included in prompt (current behavior)
+ * - discoverable: agent must run --help to discover commands
+ * - none: agent relies on training knowledge only
+ */
+import type { HelpMode, Task } from './models.js';
+export declare function buildSystemMessage(helpMode: HelpMode): string;
+export declare function buildUserMessage(cliName: string, helpTexts: Record<string, string> | null, task: Task, helpMode: HelpMode): string;
+//# sourceMappingURL=prompt.d.ts.map

package/dist/prompt.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"prompt.d.ts","sourceRoot":"","sources":["../src/prompt.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AA6BlD,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,QAAQ,GAAG,MAAM,CAS7D;AAED,wBAAgB,gBAAgB,CAC9B,OAAO,EAAE,MAAM,EACf,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,EACxC,IAAI,EAAE,IAAI,EACV,QAAQ,EAAE,QAAQ,GACjB,MAAM,CAaR"}

package/dist/prompt.js

@@ -0,0 +1,88 @@
+/**
+ * Builds prompts from help text + task definition.
+ *
+ * Three help modes:
+ * - injected: help text included in prompt (current behavior)
+ * - discoverable: agent must run --help to discover commands
+ * - none: agent relies on training knowledge only
+ */
+const AGENT_SYSTEM_MESSAGE = `You are a CLI expert. You have a run_command tool to execute shell commands. Use it to accomplish the user's task.
+
+Rules:
+- Read the help text carefully to understand available subcommands and flags
+- Execute commands using the run_command tool
+- If a command fails, read the error and retry with corrected flags
+- Do NOT invent flags that don't exist in the help text
+- When the task is complete, stop calling tools`;
+const AGENT_SYSTEM_MESSAGE_DISCOVERABLE = `You are a CLI expert. You have a run_command tool to execute shell commands. Use it to accomplish the user's task.
+
+Rules:
+- Use <cli> --help and <cli> <subcommand> --help to discover available commands and flags
+- Execute commands using the run_command tool
+- If a command fails, read the error and retry with corrected flags
+- Do NOT invent flags — always check --help first
+- When the task is complete, stop calling tools`;
+const AGENT_SYSTEM_MESSAGE_NONE = `You are a CLI expert. You have a run_command tool to execute shell commands. Use it to accomplish the user's task.
+
+Rules:
+- Use your training knowledge of CLI tools to construct commands
+- Execute commands using the run_command tool
+- If a command fails, read the error and retry with corrected flags
+- Do NOT run --help commands — rely on your knowledge
+- When the task is complete, stop calling tools`;
+export function buildSystemMessage(helpMode) {
+    switch (helpMode) {
+        case 'injected':
+            return AGENT_SYSTEM_MESSAGE;
+        case 'discoverable':
+            return AGENT_SYSTEM_MESSAGE_DISCOVERABLE;
+        case 'none':
+            return AGENT_SYSTEM_MESSAGE_NONE;
+    }
+}
+export function buildUserMessage(cliName, helpTexts, task, helpMode) {
+    if (helpMode === 'injected' && helpTexts) {
+        const relevantHelp = selectRelevantHelp(cliName, helpTexts, task);
+        return `CLI: ${cliName}
+
+Help text:
+${relevantHelp}
+
+Task: ${task.intent}`;
+    }
+    return `CLI: ${cliName}
+
+Task: ${task.intent}`;
+}
+/**
+ * Select the most relevant help text sections for a task.
+ * Trims to stay within reasonable context limits (~4K chars).
+ */
+function selectRelevantHelp(cliName, helpTexts, task) {
+    const sections = [];
+    const maxChars = 4000;
+    let totalChars = 0;
+    // Always include root help
+    const rootHelp = helpTexts[''];
+    if (rootHelp) {
+        sections.push(`$ ${cliName} --help\n${rootHelp}`);
+        totalChars += rootHelp.length;
+    }
+    // Add help sections that match task keywords
+    if (totalChars < maxChars * 0.7) {
+        const taskWords = task.intent.toLowerCase().split(/\s+/);
+        for (const [key, help] of Object.entries(helpTexts)) {
+            if (key === '' || sections.some((s) => s.includes(`${cliName} ${key} --help`))) {
+                continue;
+            }
+            const keyWords = key.split(' ');
+            const matches = keyWords.some((kw) => taskWords.includes(kw));
+            if (matches && totalChars + help.length < maxChars) {
+                sections.push(`$ ${cliName} ${key} --help\n${help}`);
+                totalChars += help.length;
+            }
+        }
+    }
+    return sections.join('\n\n');
+}
+//# sourceMappingURL=prompt.js.map

package/dist/prompt.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"prompt.js","sourceRoot":"","sources":["../src/prompt.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAIH,MAAM,oBAAoB,GAAG;;;;;;;gDAOmB,CAAC;AAEjD,MAAM,iCAAiC,GAAG;;;;;;;gDAOM,CAAC;AAEjD,MAAM,yBAAyB,GAAG;;;;;;;gDAOc,CAAC;AAEjD,MAAM,UAAU,kBAAkB,CAAC,QAAkB;IACnD,QAAQ,QAAQ,EAAE,CAAC;QACjB,KAAK,UAAU;YACb,OAAO,oBAAoB,CAAC;QAC9B,KAAK,cAAc;YACjB,OAAO,iCAAiC,CAAC;QAC3C,KAAK,MAAM;YACT,OAAO,yBAAyB,CAAC;IACrC,CAAC;AACH,CAAC;AAED,MAAM,UAAU,gBAAgB,CAC9B,OAAe,EACf,SAAwC,EACxC,IAAU,EACV,QAAkB;IAElB,IAAI,QAAQ,KAAK,UAAU,IAAI,SAAS,EAAE,CAAC;QACzC,MAAM,YAAY,GAAG,kBAAkB,CAAC,OAAO,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;QAClE,OAAO,QAAQ,OAAO;;;EAGxB,YAAY;;QAEN,IAAI,CAAC,MAAM,EAAE,CAAC;IACpB,CAAC;IACD,OAAO,QAAQ,OAAO;;QAEhB,IAAI,CAAC,MAAM,EAAE,CAAC;AACtB,CAAC;AAED;;;GAGG;AACH,SAAS,kBAAkB,CACzB,OAAe,EACf,SAAiC,EACjC,IAAU;IAEV,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,MAAM,QAAQ,GAAG,IAAI,CAAC;IACtB,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,2BAA2B;IAC3B,MAAM,QAAQ,GAAG,SAAS,CAAC,EAAE,CAAC,CAAC;IAC/B,IAAI,QAAQ,EAAE,CAAC;QACb,QAAQ,CAAC,IAAI,CAAC,KAAK,OAAO,YAAY,QAAQ,EAAE,CAAC,CAAC;QAClD,UAAU,IAAI,QAAQ,CAAC,MAAM,CAAC;IAChC,CAAC;IAED,6CAA6C;IAC7C,IAAI,UAAU,GAAG,QAAQ,GAAG,GAAG,EAAE,CAAC;QAChC,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACzD,KAAK,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;YACpD,IAAI,GAAG,KAAK,EAAE,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,OAAO,IAAI,GAAG,SAAS,CAAC,CAAC,EAAE,CAAC;gBAC/E,SAAS;YACX,CAAC;YACD,MAAM,QAAQ,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAChC,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;YAC9D,IAAI,OAAO,IAAI,UAAU,GAAG,IAAI,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC;gBACnD,QAAQ,CAAC,IAAI,CAAC,KAAK,OAAO,IAAI,GAAG,YAAY,IAAI,EAAE,CAAC,CAAC;gBACrD,UAAU,IAAI,IAAI,CAAC,MAAM,CAAC;YAC5B,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC/B,CAAC"}

package/dist/providers.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Model registry + AI Gateway configuration.
+ *
+ * Uses Vercel AI SDK v6 gateway() which is re-exported from 'ai'.
+ * Model IDs use the gateway format: "provider/model-id".
+ */
+export interface ModelEntry {
+    id: string;
+    displayName: string;
+    provider: string;
+}
+export declare const MODELS: readonly ModelEntry[];
+export declare function getModel(modelId: string): import("@ai-sdk/provider").LanguageModelV3;
+export declare function getModelEntry(modelId: string): ModelEntry | undefined;
+export declare function filterModels(modelIds: string[]): ModelEntry[];
+/**
+ * Build ModelEntry objects from provider strings (e.g. "anthropic/claude-sonnet-4-20250514").
+ * For known models, returns the registry entry. For unknown models, creates a synthetic entry.
+ */
+export declare function resolveProviders(providerIds: string[]): ModelEntry[];
+/**
+ * Validate that AI_GATEWAY_API_KEY is set.
+ * All model calls go through the AI Gateway — no per-provider keys are needed.
+ */
+export declare function validateGatewayKey(): void;
+//# sourceMappingURL=providers.d.ts.map

package/dist/providers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"providers.d.ts","sourceRoot":"","sources":["../src/providers.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,eAAO,MAAM,MAAM,EAAE,SAAS,UAAU,EAS9B,CAAC;AAEX,wBAAgB,QAAQ,CAAC,OAAO,EAAE,MAAM,8CAEvC;AAED,wBAAgB,aAAa,CAAC,OAAO,EAAE,MAAM,GAAG,UAAU,GAAG,SAAS,CAErE;AAED,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,UAAU,EAAE,CAG7D;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,WAAW,EAAE,MAAM,EAAE,GAAG,UAAU,EAAE,CAWpE;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,IAAI,IAAI,CAMzC"}

package/dist/providers.js

@@ -0,0 +1,55 @@
+/**
+ * Model registry + AI Gateway configuration.
+ *
+ * Uses Vercel AI SDK v6 gateway() which is re-exported from 'ai'.
+ * Model IDs use the gateway format: "provider/model-id".
+ */
+import { gateway } from 'ai';
+export const MODELS = [
+    { id: 'anthropic/claude-sonnet-4-20250514', displayName: 'Claude Sonnet 4', provider: 'anthropic' },
+    { id: 'anthropic/claude-haiku-4-5-20251001', displayName: 'Claude Haiku 4.5', provider: 'anthropic' },
+    { id: 'openai/gpt-4o', displayName: 'GPT-4o', provider: 'openai' },
+    { id: 'openai/gpt-4o-mini', displayName: 'GPT-4o Mini', provider: 'openai' },
+    { id: 'google/gemini-2.5-pro', displayName: 'Gemini 2.5 Pro', provider: 'google' },
+    { id: 'google/gemini-2.5-flash', displayName: 'Gemini 2.5 Flash', provider: 'google' },
+    { id: 'meta/llama-3.1-8b', displayName: 'Llama 3.1 8B', provider: 'meta' },
+    { id: 'mistral/ministral-3b', displayName: 'Ministral 3B', provider: 'mistral' },
+];
+export function getModel(modelId) {
+    return gateway(modelId);
+}
+export function getModelEntry(modelId) {
+    return MODELS.find((m) => m.id === modelId);
+}
+export function filterModels(modelIds) {
+    if (modelIds.length === 0)
+        return [...MODELS];
+    return MODELS.filter((m) => modelIds.includes(m.id));
+}
+/**
+ * Build ModelEntry objects from provider strings (e.g. "anthropic/claude-sonnet-4-20250514").
+ * For known models, returns the registry entry. For unknown models, creates a synthetic entry.
+ */
+export function resolveProviders(providerIds) {
+    return providerIds.map((id) => {
+        const existing = MODELS.find((m) => m.id === id);
+        if (existing)
+            return { ...existing };
+        const [provider, ...rest] = id.split('/');
+        return {
+            id,
+            displayName: rest.join('/'),
+            provider: provider,
+        };
+    });
+}
+/**
+ * Validate that AI_GATEWAY_API_KEY is set.
+ * All model calls go through the AI Gateway — no per-provider keys are needed.
+ */
+export function validateGatewayKey() {
+    if (!process.env['AI_GATEWAY_API_KEY']) {
+        throw new Error('Missing AI_GATEWAY_API_KEY. All model calls require the AI Gateway.\nSet AI_GATEWAY_API_KEY in your environment before running cli-bench.');
+    }
+}
+//# sourceMappingURL=providers.js.map

package/dist/providers.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"providers.js","sourceRoot":"","sources":["../src/providers.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,IAAI,CAAC;AAQ7B,MAAM,CAAC,MAAM,MAAM,GAA0B;IAC3C,EAAE,EAAE,EAAE,oCAAoC,EAAE,WAAW,EAAE,iBAAiB,EAAE,QAAQ,EAAE,WAAW,EAAE;IACnG,EAAE,EAAE,EAAE,qCAAqC,EAAE,WAAW,EAAE,kBAAkB,EAAE,QAAQ,EAAE,WAAW,EAAE;IACrG,EAAE,EAAE,EAAE,eAAe,EAAE,WAAW,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE;IAClE,EAAE,EAAE,EAAE,oBAAoB,EAAE,WAAW,EAAE,aAAa,EAAE,QAAQ,EAAE,QAAQ,EAAE;IAC5E,EAAE,EAAE,EAAE,uBAAuB,EAAE,WAAW,EAAE,gBAAgB,EAAE,QAAQ,EAAE,QAAQ,EAAE;IAClF,EAAE,EAAE,EAAE,yBAAyB,EAAE,WAAW,EAAE,kBAAkB,EAAE,QAAQ,EAAE,QAAQ,EAAE;IACtF,EAAE,EAAE,EAAE,mBAAmB,EAAE,WAAW,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,EAAE;IAC1E,EAAE,EAAE,EAAE,sBAAsB,EAAE,WAAW,EAAE,cAAc,EAAE,QAAQ,EAAE,SAAS,EAAE;CACxE,CAAC;AAEX,MAAM,UAAU,QAAQ,CAAC,OAAe;IACtC,OAAO,OAAO,CAAC,OAAO,CAAC,CAAC;AAC1B,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,OAAe;IAC3C,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,OAAO,CAAC,CAAC;AAC9C,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,QAAkB;IAC7C,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,GAAG,MAAM,CAAC,CAAC;IAC9C,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AACvD,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,WAAqB;IACpD,OAAO,WAAW,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;QAC5B,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;QACjD,IAAI,QAAQ;YAAE,OAAO,EAAE,GAAG,QAAQ,EAAE,CAAC;QACrC,MAAM,CAAC,QAAQ,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC1C,OAAO;YACL,EAAE;YACF,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;YAC3B,QAAQ,EAAE,QAAkC;SAC7C,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,kBAAkB;IAChC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,CAAC;QACvC,MAAM,IAAI,KAAK,CACb,2IAA2I,CAC5I,CAAC;IACJ,CAAC;AACH,CAAC"}

package/dist/runner.d.ts

@@ -0,0 +1,34 @@
+/**
+ * Orchestrates the task x model x helpMode matrix with tool-calling agents.
+ *
+ * For each task:
+ *   - Create temp workdir (or use configured one)
+ *   - Run setup commands directly on the host
+ *   - Give LLM the run_command tool
+ *   - LLM calls tool -> execute on host -> return stdout/stderr/exit
+ *   - After maxSteps or completion: run assertions
+ *   - Aggregate into GridReport
+ */
+import type { TaskSuite, Task, GridReport } from './models.js';
+import type { Config } from './config.js';
+import type { ModelEntry } from './providers.js';
+export declare function loadTaskSuite(filePath: string): Promise<TaskSuite>;
+export declare function discoverTaskSuites(suiteDir: string): Promise<Map<string, string>>;
+export interface RunGridOptions {
+    config: Config;
+    /** Override tasks (from config file mode). */
+    tasks?: Task[];
+    /** Override CLI name (from config file mode). */
+    cliName?: string;
+    /** Override models (from config file mode). */
+    models?: ModelEntry[];
+    /** Override version command (from config file mode). */
+    versionCommand?: string;
+    /** Override working directory (from config file mode). */
+    workdir?: string;
+    /** Global repeat count — overrides per-task repeat. */
+    globalRepeat?: number;
+}
+export declare function runGrid(opts: RunGridOptions): Promise<GridReport[]>;
+export declare function uploadReport(report: GridReport, backendUrl: string, apiKey: string): Promise<void>;
+//# sourceMappingURL=runner.d.ts.map

package/dist/runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../src/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AASH,OAAO,KAAK,EACV,SAAS,EACT,IAAI,EACJ,UAAU,EAKX,MAAM,aAAa,CAAC;AACrB,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AASjD,wBAAsB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,CAKxE;AAED,wBAAsB,kBAAkB,CACtC,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAc9B;AAkND,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,8CAA8C;IAC9C,KAAK,CAAC,EAAE,IAAI,EAAE,CAAC;IACf,iDAAiD;IACjD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,+CAA+C;IAC/C,MAAM,CAAC,EAAE,UAAU,EAAE,CAAC;IACtB,wDAAwD;IACxD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,0DAA0D;IAC1D,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,uDAAuD;IACvD,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,wBAAsB,OAAO,CAAC,IAAI,EAAE,cAAc,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC,CAyDzE;AAmND,wBAAsB,YAAY,CAChC,MAAM,EAAE,UAAU,EAClB,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,IAAI,CAAC,CAsBf"}