@cliwatch/cli-bench 0.4.0

package/dist/schemas.d.ts

@@ -0,0 +1,256 @@
+/**
+ * Zod schemas for runtime validation of YAML config and task files.
+ */
+import { z } from 'zod';
+export declare const AssertionSchema: z.ZodUnion<readonly [z.ZodObject<{
+    output_contains: z.ZodString;
+}, z.core.$strip>, z.ZodObject<{
+    output_equals: z.ZodString;
+}, z.core.$strip>, z.ZodObject<{
+    error_contains: z.ZodString;
+}, z.core.$strip>, z.ZodObject<{
+    exit_code: z.ZodNumber;
+}, z.core.$strip>, z.ZodObject<{
+    file_exists: z.ZodString;
+}, z.core.$strip>, z.ZodObject<{
+    file_contains: z.ZodObject<{
+        path: z.ZodString;
+        text: z.ZodString;
+    }, z.core.$strip>;
+}, z.core.$strip>, z.ZodObject<{
+    ran: z.ZodString;
+}, z.core.$strip>, z.ZodObject<{
+    not_ran: z.ZodString;
+}, z.core.$strip>, z.ZodObject<{
+    run_count: z.ZodObject<{
+        pattern: z.ZodString;
+        min: z.ZodOptional<z.ZodNumber>;
+        max: z.ZodOptional<z.ZodNumber>;
+    }, z.core.$strip>;
+}, z.core.$strip>, z.ZodObject<{
+    verify: z.ZodObject<{
+        run: z.ZodString;
+        output_contains: z.ZodOptional<z.ZodString>;
+        output_equals: z.ZodOptional<z.ZodString>;
+    }, z.core.$strip>;
+}, z.core.$strip>]>;
+export declare const TaskSchema: z.ZodObject<{
+    id: z.ZodString;
+    intent: z.ZodString;
+    assert: z.ZodArray<z.ZodUnion<readonly [z.ZodObject<{
+        output_contains: z.ZodString;
+    }, z.core.$strip>, z.ZodObject<{
+        output_equals: z.ZodString;
+    }, z.core.$strip>, z.ZodObject<{
+        error_contains: z.ZodString;
+    }, z.core.$strip>, z.ZodObject<{
+        exit_code: z.ZodNumber;
+    }, z.core.$strip>, z.ZodObject<{
+        file_exists: z.ZodString;
+    }, z.core.$strip>, z.ZodObject<{
+        file_contains: z.ZodObject<{
+            path: z.ZodString;
+            text: z.ZodString;
+        }, z.core.$strip>;
+    }, z.core.$strip>, z.ZodObject<{
+        ran: z.ZodString;
+    }, z.core.$strip>, z.ZodObject<{
+        not_ran: z.ZodString;
+    }, z.core.$strip>, z.ZodObject<{
+        run_count: z.ZodObject<{
+            pattern: z.ZodString;
+            min: z.ZodOptional<z.ZodNumber>;
+            max: z.ZodOptional<z.ZodNumber>;
+        }, z.core.$strip>;
+    }, z.core.$strip>, z.ZodObject<{
+        verify: z.ZodObject<{
+            run: z.ZodString;
+            output_contains: z.ZodOptional<z.ZodString>;
+            output_equals: z.ZodOptional<z.ZodString>;
+        }, z.core.$strip>;
+    }, z.core.$strip>]>>;
+    setup: z.ZodDefault<z.ZodOptional<z.ZodArray<z.ZodString>>>;
+    max_turns: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
+    difficulty: z.ZodOptional<z.ZodEnum<{
+        easy: "easy";
+        medium: "medium";
+        hard: "hard";
+    }>>;
+    category: z.ZodOptional<z.ZodString>;
+    repeat: z.ZodOptional<z.ZodNumber>;
+}, z.core.$strip>;
+export declare const TaskSuiteSchema: z.ZodObject<{
+    cli: z.ZodString;
+    version_command: z.ZodOptional<z.ZodString>;
+    tasks: z.ZodArray<z.ZodObject<{
+        id: z.ZodString;
+        intent: z.ZodString;
+        assert: z.ZodArray<z.ZodUnion<readonly [z.ZodObject<{
+            output_contains: z.ZodString;
+        }, z.core.$strip>, z.ZodObject<{
+            output_equals: z.ZodString;
+        }, z.core.$strip>, z.ZodObject<{
+            error_contains: z.ZodString;
+        }, z.core.$strip>, z.ZodObject<{
+            exit_code: z.ZodNumber;
+        }, z.core.$strip>, z.ZodObject<{
+            file_exists: z.ZodString;
+        }, z.core.$strip>, z.ZodObject<{
+            file_contains: z.ZodObject<{
+                path: z.ZodString;
+                text: z.ZodString;
+            }, z.core.$strip>;
+        }, z.core.$strip>, z.ZodObject<{
+            ran: z.ZodString;
+        }, z.core.$strip>, z.ZodObject<{
+            not_ran: z.ZodString;
+        }, z.core.$strip>, z.ZodObject<{
+            run_count: z.ZodObject<{
+                pattern: z.ZodString;
+                min: z.ZodOptional<z.ZodNumber>;
+                max: z.ZodOptional<z.ZodNumber>;
+            }, z.core.$strip>;
+        }, z.core.$strip>, z.ZodObject<{
+            verify: z.ZodObject<{
+                run: z.ZodString;
+                output_contains: z.ZodOptional<z.ZodString>;
+                output_equals: z.ZodOptional<z.ZodString>;
+            }, z.core.$strip>;
+        }, z.core.$strip>]>>;
+        setup: z.ZodDefault<z.ZodOptional<z.ZodArray<z.ZodString>>>;
+        max_turns: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
+        difficulty: z.ZodOptional<z.ZodEnum<{
+            easy: "easy";
+            medium: "medium";
+            hard: "hard";
+        }>>;
+        category: z.ZodOptional<z.ZodString>;
+        repeat: z.ZodOptional<z.ZodNumber>;
+    }, z.core.$strip>>;
+}, z.core.$strip>;
+/** Schema for a task file referenced via file:// — plain array of tasks. */
+export declare const TaskFileSchema: z.ZodArray<z.ZodObject<{
+    id: z.ZodString;
+    intent: z.ZodString;
+    assert: z.ZodArray<z.ZodUnion<readonly [z.ZodObject<{
+        output_contains: z.ZodString;
+    }, z.core.$strip>, z.ZodObject<{
+        output_equals: z.ZodString;
+    }, z.core.$strip>, z.ZodObject<{
+        error_contains: z.ZodString;
+    }, z.core.$strip>, z.ZodObject<{
+        exit_code: z.ZodNumber;
+    }, z.core.$strip>, z.ZodObject<{
+        file_exists: z.ZodString;
+    }, z.core.$strip>, z.ZodObject<{
+        file_contains: z.ZodObject<{
+            path: z.ZodString;
+            text: z.ZodString;
+        }, z.core.$strip>;
+    }, z.core.$strip>, z.ZodObject<{
+        ran: z.ZodString;
+    }, z.core.$strip>, z.ZodObject<{
+        not_ran: z.ZodString;
+    }, z.core.$strip>, z.ZodObject<{
+        run_count: z.ZodObject<{
+            pattern: z.ZodString;
+            min: z.ZodOptional<z.ZodNumber>;
+            max: z.ZodOptional<z.ZodNumber>;
+        }, z.core.$strip>;
+    }, z.core.$strip>, z.ZodObject<{
+        verify: z.ZodObject<{
+            run: z.ZodString;
+            output_contains: z.ZodOptional<z.ZodString>;
+            output_equals: z.ZodOptional<z.ZodString>;
+        }, z.core.$strip>;
+    }, z.core.$strip>]>>;
+    setup: z.ZodDefault<z.ZodOptional<z.ZodArray<z.ZodString>>>;
+    max_turns: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
+    difficulty: z.ZodOptional<z.ZodEnum<{
+        easy: "easy";
+        medium: "medium";
+        hard: "hard";
+    }>>;
+    category: z.ZodOptional<z.ZodString>;
+    repeat: z.ZodOptional<z.ZodNumber>;
+}, z.core.$strip>>;
+export declare const ThresholdsSchema: z.ZodOptional<z.ZodObject<{
+    default: z.ZodOptional<z.ZodNumber>;
+    models: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodNumber>>;
+    tolerance: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
+    behavior: z.ZodDefault<z.ZodOptional<z.ZodEnum<{
+        error: "error";
+        informational: "informational";
+    }>>>;
+}, z.core.$strip>>;
+/** Schema for the top-level cli-bench.yaml config file. */
+export declare const ConfigFileSchema: z.ZodObject<{
+    cli: z.ZodString;
+    version_command: z.ZodOptional<z.ZodString>;
+    providers: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    help_modes: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    concurrency: z.ZodOptional<z.ZodNumber>;
+    workdir: z.ZodOptional<z.ZodString>;
+    upload: z.ZodOptional<z.ZodEnum<{
+        auto: "auto";
+        always: "always";
+        never: "never";
+    }>>;
+    backend_url: z.ZodOptional<z.ZodString>;
+    repeat: z.ZodOptional<z.ZodNumber>;
+    thresholds: z.ZodOptional<z.ZodObject<{
+        default: z.ZodOptional<z.ZodNumber>;
+        models: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodNumber>>;
+        tolerance: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
+        behavior: z.ZodDefault<z.ZodOptional<z.ZodEnum<{
+            error: "error";
+            informational: "informational";
+        }>>>;
+    }, z.core.$strip>>;
+    tasks: z.ZodArray<z.ZodUnion<readonly [z.ZodObject<{
+        id: z.ZodString;
+        intent: z.ZodString;
+        assert: z.ZodArray<z.ZodUnion<readonly [z.ZodObject<{
+            output_contains: z.ZodString;
+        }, z.core.$strip>, z.ZodObject<{
+            output_equals: z.ZodString;
+        }, z.core.$strip>, z.ZodObject<{
+            error_contains: z.ZodString;
+        }, z.core.$strip>, z.ZodObject<{
+            exit_code: z.ZodNumber;
+        }, z.core.$strip>, z.ZodObject<{
+            file_exists: z.ZodString;
+        }, z.core.$strip>, z.ZodObject<{
+            file_contains: z.ZodObject<{
+                path: z.ZodString;
+                text: z.ZodString;
+            }, z.core.$strip>;
+        }, z.core.$strip>, z.ZodObject<{
+            ran: z.ZodString;
+        }, z.core.$strip>, z.ZodObject<{
+            not_ran: z.ZodString;
+        }, z.core.$strip>, z.ZodObject<{
+            run_count: z.ZodObject<{
+                pattern: z.ZodString;
+                min: z.ZodOptional<z.ZodNumber>;
+                max: z.ZodOptional<z.ZodNumber>;
+            }, z.core.$strip>;
+        }, z.core.$strip>, z.ZodObject<{
+            verify: z.ZodObject<{
+                run: z.ZodString;
+                output_contains: z.ZodOptional<z.ZodString>;
+                output_equals: z.ZodOptional<z.ZodString>;
+            }, z.core.$strip>;
+        }, z.core.$strip>]>>;
+        setup: z.ZodDefault<z.ZodOptional<z.ZodArray<z.ZodString>>>;
+        max_turns: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
+        difficulty: z.ZodOptional<z.ZodEnum<{
+            easy: "easy";
+            medium: "medium";
+            hard: "hard";
+        }>>;
+        category: z.ZodOptional<z.ZodString>;
+        repeat: z.ZodOptional<z.ZodNumber>;
+    }, z.core.$strip>, z.ZodString]>>;
+}, z.core.$strip>;
+//# sourceMappingURL=schemas.d.ts.map

package/dist/schemas.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"schemas.d.ts","sourceRoot":"","sources":["../src/schemas.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;mBAW1B,CAAC;AAEH,eAAO,MAAM,UAAU;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBASrB,CAAC;AAEH,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAI1B,CAAC;AAEH,4EAA4E;AAC5E,eAAO,MAAM,cAAc;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAA6B,CAAC;AAEzD,eAAO,MAAM,gBAAgB;;;;;;;;kBAKhB,CAAC;AAEd,2DAA2D;AAC3D,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAmB3B,CAAC"}

package/dist/schemas.js

@@ -0,0 +1,59 @@
+/**
+ * Zod schemas for runtime validation of YAML config and task files.
+ */
+import { z } from 'zod';
+export const AssertionSchema = z.union([
+    z.object({ output_contains: z.string() }),
+    z.object({ output_equals: z.string() }),
+    z.object({ error_contains: z.string() }),
+    z.object({ exit_code: z.number() }),
+    z.object({ file_exists: z.string() }),
+    z.object({ file_contains: z.object({ path: z.string(), text: z.string() }) }),
+    z.object({ ran: z.string() }),
+    z.object({ not_ran: z.string() }),
+    z.object({ run_count: z.object({ pattern: z.string(), min: z.number().optional(), max: z.number().optional() }) }),
+    z.object({ verify: z.object({ run: z.string(), output_contains: z.string().optional(), output_equals: z.string().optional() }) }),
+]);
+export const TaskSchema = z.object({
+    id: z.string(),
+    intent: z.string(),
+    assert: z.array(AssertionSchema).min(1),
+    setup: z.array(z.string()).optional().default([]),
+    max_turns: z.number().int().min(1).max(20).optional().default(5),
+    difficulty: z.enum(['easy', 'medium', 'hard']).optional(),
+    category: z.string().optional(),
+    repeat: z.number().int().min(1).max(100).optional(),
+});
+export const TaskSuiteSchema = z.object({
+    cli: z.string(),
+    version_command: z.string().optional(),
+    tasks: z.array(TaskSchema).min(1),
+});
+/** Schema for a task file referenced via file:// — plain array of tasks. */
+export const TaskFileSchema = z.array(TaskSchema).min(1);
+export const ThresholdsSchema = z.object({
+    default: z.number().min(0).max(100).optional(),
+    models: z.record(z.string(), z.number().min(0).max(100)).optional(),
+    tolerance: z.number().min(0).max(100).optional().default(0),
+    behavior: z.enum(['error', 'informational']).optional().default('error'),
+}).optional();
+/** Schema for the top-level cli-bench.yaml config file. */
+export const ConfigFileSchema = z.object({
+    cli: z.string(),
+    version_command: z.string().optional(),
+    providers: z.array(z.string()).optional(),
+    help_modes: z.array(z.string()).optional(),
+    concurrency: z.number().int().min(1).optional(),
+    workdir: z.string().optional(),
+    upload: z.enum(['auto', 'always', 'never']).optional(),
+    backend_url: z.string().optional(),
+    repeat: z.number().int().min(1).max(100).optional(),
+    thresholds: ThresholdsSchema,
+    tasks: z.array(z.union([
+        TaskSchema,
+        z.string().refine((s) => s.startsWith('file://'), {
+            message: 'Task references must start with file://',
+        }),
+    ])).min(1),
+});
+//# sourceMappingURL=schemas.js.map

package/dist/schemas.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"schemas.js","sourceRoot":"","sources":["../src/schemas.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC;IACrC,CAAC,CAAC,MAAM,CAAC,EAAE,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;IACzC,CAAC,CAAC,MAAM,CAAC,EAAE,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;IACvC,CAAC,CAAC,MAAM,CAAC,EAAE,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;IACxC,CAAC,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;IACnC,CAAC,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;IACrC,CAAC,CAAC,MAAM,CAAC,EAAE,aAAa,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC;IAC7E,CAAC,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;IAC7B,CAAC,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;IACjC,CAAC,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,EAAE,CAAC;IAClH,CAAC,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,EAAE,CAAC;CAClI,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,CAAC;IACjC,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE;IACd,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;IAClB,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACvC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;IACjD,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IAChE,UAAU,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE;IACzD,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC/B,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE;CACpD,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC;IACtC,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE;IACf,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACtC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;CAClC,CAAC,CAAC;AAEH,4EAA4E;AAC5E,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;AAEzD,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC;IACvC,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE;IAC9C,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE;IACnE,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IAC3D,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC;CACzE,CAAC,CAAC,QAAQ,EAAE,CAAC;AAEd,2DAA2D;AAC3D,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC;IACvC,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE;IACf,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACtC,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IACzC,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAC1C,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC/C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC9B,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC,QAAQ,EAAE;IACtD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAClC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE;IACnD,UAAU,EAAE,gBAAgB;IAC5B,KAAK,EAAE,CAAC,CAAC,KAAK,CACZ,CAAC,CAAC,KAAK,CAAC;QACN,UAAU;QACV,CAAC,CAAC,MAAM,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE;YAChD,OAAO,EAAE,yCAAyC;SACnD,CAAC;KACH,CAAC,CACH,CAAC,GAAG,CAAC,CAAC,CAAC;CACT,CAAC,CAAC"}

package/dist/suite-generator.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Generates task suite YAML files for a given CLI using an LLM.
+ *
+ * Takes CLI name + help text and produces a task suite with
+ * assert-based validation.
+ */
+export declare function generateSuite(cliName: string, helpCacheDir: string): Promise<string>;
+//# sourceMappingURL=suite-generator.d.ts.map

package/dist/suite-generator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"suite-generator.d.ts","sourceRoot":"","sources":["../src/suite-generator.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAwCH,wBAAsB,aAAa,CACjC,OAAO,EAAE,MAAM,EACf,YAAY,EAAE,MAAM,GACnB,OAAO,CAAC,MAAM,CAAC,CAgEjB"}

package/dist/suite-generator.js

@@ -0,0 +1,100 @@
+/**
+ * Generates task suite YAML files for a given CLI using an LLM.
+ *
+ * Takes CLI name + help text and produces a task suite with
+ * assert-based validation.
+ */
+import { readFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { generateText } from 'ai';
+import { gateway } from 'ai';
+import { loadHelpFromCache, loadHelpLive } from './help-loader.js';
+const GENERATOR_MODEL = 'anthropic/claude-sonnet-4-20250514';
+const SYSTEM_PROMPT = `You are a CLI test suite generator. Given help text for a CLI tool and example task suites, generate a YAML task suite.
+
+Task format:
+- id: kebab-case identifier
+- intent: natural language description of what the agent should do
+- assert: array of assertions to validate the result
+- setup: (optional) commands to run before the task
+- max_turns: (optional) max tool calls, default 5
+- difficulty: easy | medium | hard
+- category: query | crud | output | config | workflow | auth
+
+Assertion types:
+- ran: regex pattern matched against commands the agent executed
+- not_ran: regex pattern that must NOT match any executed command
+- run_count: { pattern, min?, max? } — count regex matches
+- output_contains: string that must appear in last command stdout
+- output_equals: exact match on last command stdout
+- error_contains: string that must appear in last command stderr
+- exit_code: expected exit code of last command
+- file_exists: path that must exist after execution
+- file_contains: { path, text } — file must contain text
+- verify: { run, output_contains?, output_equals? } — run command to check state
+
+Rules:
+- Generate 10-15 tasks: 5 easy, 5 medium, 3-5 hard
+- Every task MUST be executable without authentication or network access
+- Focus on: local operations, file generation, formatting, config, help queries
+- Include setup commands to prepare the environment
+- Use realistic but safe values (no real credentials, no destructive operations)`;
+export async function generateSuite(cliName, helpCacheDir) {
+    // Load help text
+    let helpCache = await loadHelpFromCache(helpCacheDir, cliName);
+    if (!helpCache) {
+        try {
+            helpCache = await loadHelpLive(cliName);
+        }
+        catch {
+            throw new Error(`No help text available for ${cliName}`);
+        }
+    }
+    // Load example suite for few-shot
+    const suiteDir = join(new URL('.', import.meta.url).pathname.replace(/\/src\/$/, '').replace(/\/dist\/$/, ''), 'task_suites');
+    let dockerExample = '';
+    try {
+        dockerExample = await readFile(join(suiteDir, 'docker.yaml'), 'utf-8');
+    }
+    catch { /* ignore */ }
+    // Build help text summary (truncate to fit context)
+    const helpEntries = Object.entries(helpCache.help_texts);
+    let helpSummary = '';
+    for (const [key, text] of helpEntries) {
+        const label = key || '(root)';
+        const truncated = text.slice(0, 1500);
+        helpSummary += `\n--- ${cliName} ${label} --help ---\n${truncated}\n`;
+        if (helpSummary.length > 15000)
+            break;
+    }
+    const prompt = `Generate a task suite YAML for the "${cliName}" CLI.
+
+## Help Text
+
+${helpSummary}
+
+## Example Task Suite
+
+### docker.yaml
+\`\`\`yaml
+${dockerExample}
+\`\`\`
+
+## Output
+
+Generate ONLY the YAML content (no markdown fences). Start with \`cli: ${cliName}\`.`;
+    const result = await generateText({
+        model: gateway(GENERATOR_MODEL),
+        system: SYSTEM_PROMPT,
+        prompt,
+        temperature: 0.3,
+        maxOutputTokens: 4096,
+    });
+    // Clean up: remove any markdown fences the model might add
+    let yaml = result.text.trim();
+    if (yaml.startsWith('```')) {
+        yaml = yaml.replace(/^```\w*\n/, '').replace(/\n```$/, '');
+    }
+    return yaml;
+}
+//# sourceMappingURL=suite-generator.js.map

package/dist/suite-generator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"suite-generator.js","sourceRoot":"","sources":["../src/suite-generator.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,YAAY,EAAE,MAAM,IAAI,CAAC;AAClC,OAAO,EAAE,OAAO,EAAE,MAAM,IAAI,CAAC;AAC7B,OAAO,EAAE,iBAAiB,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAEnE,MAAM,eAAe,GAAG,oCAAoC,CAAC;AAE7D,MAAM,aAAa,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;iFA4B2D,CAAC;AAElF,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,OAAe,EACf,YAAoB;IAEpB,iBAAiB;IACjB,IAAI,SAAS,GAAG,MAAM,iBAAiB,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;IAC/D,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,IAAI,CAAC;YACH,SAAS,GAAG,MAAM,YAAY,CAAC,OAAO,CAAC,CAAC;QAC1C,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,IAAI,KAAK,CAAC,8BAA8B,OAAO,EAAE,CAAC,CAAC;QAC3D,CAAC;IACH,CAAC;IAED,kCAAkC;IAClC,MAAM,QAAQ,GAAG,IAAI,CACnB,IAAI,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC,EACvF,aAAa,CACd,CAAC;IAEF,IAAI,aAAa,GAAG,EAAE,CAAC;IACvB,IAAI,CAAC;QACH,aAAa,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,EAAE,OAAO,CAAC,CAAC;IACzE,CAAC;IAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;IAExB,oDAAoD;IACpD,MAAM,WAAW,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;IACzD,IAAI,WAAW,GAAG,EAAE,CAAC;IACrB,KAAK,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,WAAW,EAAE,CAAC;QACtC,MAAM,KAAK,GAAG,GAAG,IAAI,QAAQ,CAAC;QAC9B,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QACtC,WAAW,IAAI,SAAS,OAAO,IAAI,KAAK,gBAAgB,SAAS,IAAI,CAAC;QACtE,IAAI,WAAW,CAAC,MAAM,GAAG,KAAK;YAAE,MAAM;IACxC,CAAC;IAED,MAAM,MAAM,GAAG,uCAAuC,OAAO;;;;EAI7D,WAAW;;;;;;EAMX,aAAa;;;;;yEAK0D,OAAO,KAAK,CAAC;IAEpF,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;QAChC,KAAK,EAAE,OAAO,CAAC,eAAe,CAAC;QAC/B,MAAM,EAAE,aAAa;QACrB,MAAM;QACN,WAAW,EAAE,GAAG;QAChB,eAAe,EAAE,IAAI;KACtB,CAAC,CAAC;IAEH,2DAA2D;IAC3D,IAAI,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;IAC9B,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;QAC3B,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IAC7D,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC"}

package/dist/thresholds.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Threshold checking for CLI bench results.
+ *
+ * Compares actual model pass rates against configured thresholds
+ * with optional tolerance (wiggle room) and per-model overrides.
+ */
+import type { ModelResult, ThresholdsConfig, ThresholdCheckResult } from './models.js';
+export declare function checkThresholds(modelResults: ModelResult[], thresholds: ThresholdsConfig): ThresholdCheckResult;
+export declare function printThresholdResults(check: ThresholdCheckResult): void;
+//# sourceMappingURL=thresholds.d.ts.map

package/dist/thresholds.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"thresholds.d.ts","sourceRoot":"","sources":["../src/thresholds.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EACV,WAAW,EACX,gBAAgB,EAEhB,oBAAoB,EACrB,MAAM,aAAa,CAAC;AAErB,wBAAgB,eAAe,CAC7B,YAAY,EAAE,WAAW,EAAE,EAC3B,UAAU,EAAE,gBAAgB,GAC3B,oBAAoB,CAoCtB;AAED,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,oBAAoB,GAAG,IAAI,CAoBvE"}

package/dist/thresholds.js

@@ -0,0 +1,57 @@
+/**
+ * Threshold checking for CLI bench results.
+ *
+ * Compares actual model pass rates against configured thresholds
+ * with optional tolerance (wiggle room) and per-model overrides.
+ */
+export function checkThresholds(modelResults, thresholds) {
+    const tolerance = thresholds.tolerance ?? 0;
+    const behavior = thresholds.behavior ?? 'error';
+    const results = [];
+    for (const mr of modelResults) {
+        // Look up threshold: per-model → default → skip
+        const modelKey = `${mr.provider}/${mr.modelId}`;
+        const threshold = thresholds.models?.[modelKey]
+            ?? thresholds.models?.[mr.modelId]
+            ?? thresholds.default;
+        if (threshold === undefined) {
+            continue; // no threshold configured for this model
+        }
+        const passRate = mr.passRate * 100;
+        const effectiveMin = Math.max(0, threshold - tolerance);
+        const passed = passRate >= effectiveMin;
+        results.push({
+            model: mr.displayName,
+            passRate: Math.round(passRate * 100) / 100,
+            threshold,
+            tolerance,
+            effectiveMin,
+            passed,
+        });
+    }
+    return {
+        allPassed: results.every((r) => r.passed),
+        results,
+        behavior,
+    };
+}
+export function printThresholdResults(check) {
+    if (check.results.length === 0)
+        return;
+    console.log('\n=== Threshold Check ===');
+    for (const r of check.results) {
+        const icon = r.passed ? '✓' : '✗';
+        const cmp = r.passed ? '>=' : '<';
+        const tolStr = r.tolerance > 0 ? `, tolerance ${r.tolerance}%` : '';
+        console.log(`  ${icon} ${r.model}: ${r.passRate}% ${cmp} ${r.effectiveMin}% (threshold ${r.threshold}%${tolStr})`);
+    }
+    const failCount = check.results.filter((r) => !r.passed).length;
+    if (failCount > 0) {
+        const label = check.behavior === 'informational' ? 'WARNING' : 'FAILED';
+        console.log(`${label}: ${failCount} model(s) below threshold`);
+    }
+    else {
+        console.log('All models meet thresholds');
+    }
+}
+//# sourceMappingURL=thresholds.js.map

package/dist/thresholds.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"thresholds.js","sourceRoot":"","sources":["../src/thresholds.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AASH,MAAM,UAAU,eAAe,CAC7B,YAA2B,EAC3B,UAA4B;IAE5B,MAAM,SAAS,GAAG,UAAU,CAAC,SAAS,IAAI,CAAC,CAAC;IAC5C,MAAM,QAAQ,GAAG,UAAU,CAAC,QAAQ,IAAI,OAAO,CAAC;IAEhD,MAAM,OAAO,GAAsB,EAAE,CAAC;IAEtC,KAAK,MAAM,EAAE,IAAI,YAAY,EAAE,CAAC;QAC9B,gDAAgD;QAChD,MAAM,QAAQ,GAAG,GAAG,EAAE,CAAC,QAAQ,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;QAChD,MAAM,SAAS,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC;eAC1C,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,OAAO,CAAC;eAC/B,UAAU,CAAC,OAAO,CAAC;QAExB,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;YAC5B,SAAS,CAAC,yCAAyC;QACrD,CAAC;QAED,MAAM,QAAQ,GAAG,EAAE,CAAC,QAAQ,GAAG,GAAG,CAAC;QACnC,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,GAAG,SAAS,CAAC,CAAC;QACxD,MAAM,MAAM,GAAG,QAAQ,IAAI,YAAY,CAAC;QAExC,OAAO,CAAC,IAAI,CAAC;YACX,KAAK,EAAE,EAAE,CAAC,WAAW;YACrB,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,GAAG,CAAC,GAAG,GAAG;YAC1C,SAAS;YACT,SAAS;YACT,YAAY;YACZ,MAAM;SACP,CAAC,CAAC;IACL,CAAC;IAED,OAAO;QACL,SAAS,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;QACzC,OAAO;QACP,QAAQ;KACT,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,KAA2B;IAC/D,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IAEvC,OAAO,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAC;IACzC,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;QAC9B,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;QAClC,MAAM,GAAG,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC;QAClC,MAAM,MAAM,GAAG,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QACpE,OAAO,CAAC,GAAG,CACT,KAAK,IAAI,IAAI,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,QAAQ,KAAK,GAAG,IAAI,CAAC,CAAC,YAAY,gBAAgB,CAAC,CAAC,SAAS,IAAI,MAAM,GAAG,CACtG,CAAC;IACJ,CAAC;IAED,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAChE,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;QAClB,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,KAAK,eAAe,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC;QACxE,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,KAAK,SAAS,2BAA2B,CAAC,CAAC;IACjE,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CAAC,4BAA4B,CAAC,CAAC;IAC5C,CAAC;AACH,CAAC"}

package/package.json

@@ -0,0 +1,41 @@
+{
+  "name": "@cliwatch/cli-bench",
+  "version": "0.4.0",
+  "type": "module",
+  "bin": {
+    "cli-bench": "./dist/index.js"
+  },
+  "files": [
+    "dist",
+    "task_suites"
+  ],
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "generate-client": "npx @hey-api/openapi-ts",
+    "build": "tsc",
+    "start": "node dist/index.js",
+    "dev": "tsx src/index.ts",
+    "test": "vitest"
+  },
+  "dependencies": {
+    "@hey-api/client-fetch": "^0.13.1",
+    "ai": "^6.0.18",
+    "yaml": "^2.7.0",
+    "zod": "^4.1.12"
+  },
+  "devDependencies": {
+    "@hey-api/openapi-ts": "^0.91.1",
+    "@types/node": "^22.0.0",
+    "tsx": "^4.19.0",
+    "typescript": "^5.8.0",
+    "vitest": "^3.2.0"
+  }
+}