@cliwatch/cli-bench 0.6.2 → 0.7.0

package/dist/schemas.d.ts

@@ -70,6 +70,8 @@ export declare const TaskSchema: z.ZodObject<{
         }, z.core.$strip>;
     }, z.core.$strip>]>>;
     setup: z.ZodDefault<z.ZodOptional<z.ZodArray<z.ZodString>>>;
+    cleanup: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    env: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
     max_turns: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
     difficulty: z.ZodOptional<z.ZodEnum<{
         easy: "easy";
@@ -78,6 +80,7 @@ export declare const TaskSchema: z.ZodObject<{
     }>>;
     category: z.ZodOptional<z.ZodString>;
     repeat: z.ZodOptional<z.ZodNumber>;
+    tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
 }, z.core.$strip>;
 export declare const TaskSuiteSchema: z.ZodObject<{
     cli: z.ZodString;
@@ -122,6 +125,8 @@ export declare const TaskSuiteSchema: z.ZodObject<{
             }, z.core.$strip>;
         }, z.core.$strip>]>>;
         setup: z.ZodDefault<z.ZodOptional<z.ZodArray<z.ZodString>>>;
+        cleanup: z.ZodOptional<z.ZodArray<z.ZodString>>;
+        env: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
         max_turns: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
         difficulty: z.ZodOptional<z.ZodEnum<{
             easy: "easy";
@@ -130,6 +135,7 @@ export declare const TaskSuiteSchema: z.ZodObject<{
         }>>;
         category: z.ZodOptional<z.ZodString>;
         repeat: z.ZodOptional<z.ZodNumber>;
+        tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
     }, z.core.$strip>>;
 }, z.core.$strip>;
 /** Schema for a task file referenced via file:// — plain array of tasks. */
@@ -169,6 +175,8 @@ export declare const TaskFileSchema: z.ZodArray<z.ZodObject<{
         }, z.core.$strip>;
     }, z.core.$strip>]>>;
     setup: z.ZodDefault<z.ZodOptional<z.ZodArray<z.ZodString>>>;
+    cleanup: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    env: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
     max_turns: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
     difficulty: z.ZodOptional<z.ZodEnum<{
         easy: "easy";
@@ -177,6 +185,7 @@ export declare const TaskFileSchema: z.ZodArray<z.ZodObject<{
     }>>;
     category: z.ZodOptional<z.ZodString>;
     repeat: z.ZodOptional<z.ZodNumber>;
+    tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
 }, z.core.$strip>>;
 export declare const ThresholdsSchema: z.ZodOptional<z.ZodObject<{
     default: z.ZodOptional<z.ZodNumber>;
@@ -208,6 +217,7 @@ export declare const ConfigFileSchema: z.ZodObject<{
     backend_url: z.ZodOptional<z.ZodString>;
     repeat: z.ZodOptional<z.ZodNumber>;
     redact_env: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    redact_patterns: z.ZodOptional<z.ZodArray<z.ZodString>>;
     thresholds: z.ZodOptional<z.ZodObject<{
         default: z.ZodOptional<z.ZodNumber>;
         models: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodNumber>>;
@@ -217,6 +227,9 @@ export declare const ConfigFileSchema: z.ZodObject<{
             informational: "informational";
         }>>>;
     }, z.core.$strip>>;
+    env: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+    setup: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    cleanup: z.ZodOptional<z.ZodArray<z.ZodString>>;
     tasks: z.ZodArray<z.ZodUnion<readonly [z.ZodObject<{
         id: z.ZodString;
         intent: z.ZodString;
@@ -253,6 +266,8 @@ export declare const ConfigFileSchema: z.ZodObject<{
             }, z.core.$strip>;
         }, z.core.$strip>]>>;
         setup: z.ZodDefault<z.ZodOptional<z.ZodArray<z.ZodString>>>;
+        cleanup: z.ZodOptional<z.ZodArray<z.ZodString>>;
+        env: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
         max_turns: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
         difficulty: z.ZodOptional<z.ZodEnum<{
             easy: "easy";
@@ -261,6 +276,7 @@ export declare const ConfigFileSchema: z.ZodObject<{
         }>>;
         category: z.ZodOptional<z.ZodString>;
         repeat: z.ZodOptional<z.ZodNumber>;
+        tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
     }, z.core.$strip>, z.ZodString]>>;
 }, z.core.$strip>;
 //# sourceMappingURL=schemas.d.ts.map

package/dist/schemas.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"schemas.d.ts","sourceRoot":"","sources":["../src/schemas.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;mBAW1B,CAAC;AAEH,eAAO,MAAM,UAAU;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBASrB,CAAC;AAEH,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAQ1B,CAAC;AAEH,4EAA4E;AAC5E,eAAO,MAAM,cAAc;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAA6B,CAAC;AAEzD,eAAO,MAAM,gBAAgB;;;;;;;;kBAKhB,CAAC;AAEd,2DAA2D;AAC3D,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAyB3B,CAAC"}
+{"version":3,"file":"schemas.d.ts","sourceRoot":"","sources":["../src/schemas.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;mBAW1B,CAAC;AAEH,eAAO,MAAM,UAAU;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAYrB,CAAC;AAEH,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAQ1B,CAAC;AAEH,4EAA4E;AAC5E,eAAO,MAAM,cAAc;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAA6B,CAAC;AAEzD,eAAO,MAAM,gBAAgB;;;;;;;;kBAKhB,CAAC;AAEd,2DAA2D;AAC3D,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBA6B3B,CAAC"}

package/dist/schemas.js

@@ -19,10 +19,13 @@ export const TaskSchema = z.object({
     intent: z.string(),
     assert: z.array(AssertionSchema).min(1),
     setup: z.array(z.string()).optional().default([]),
+    cleanup: z.array(z.string()).optional(),
+    env: z.record(z.string(), z.string()).optional(),
     max_turns: z.number().int().min(1).max(20).optional().default(5),
     difficulty: z.enum(['easy', 'medium', 'hard']).optional(),
     category: z.string().optional(),
     repeat: z.number().int().min(1).max(100).optional(),
+    tags: z.array(z.string()).optional(),
 });
 export const TaskSuiteSchema = z.object({
     cli: z.string(),
@@ -58,7 +61,11 @@ export const ConfigFileSchema = z.object({
     backend_url: z.string().optional(),
     repeat: z.number().int().min(1).max(100).optional(),
     redact_env: z.array(z.string()).optional(),
+    redact_patterns: z.array(z.string()).optional(),
     thresholds: ThresholdsSchema,
+    env: z.record(z.string(), z.string()).optional(),
+    setup: z.array(z.string()).optional(),
+    cleanup: z.array(z.string()).optional(),
     tasks: z.array(z.union([
         TaskSchema,
         z.string().refine((s) => s.startsWith('file://'), {

package/dist/schemas.js.map

@@ -1 +1 @@
-{"version":3,"file":"schemas.js","sourceRoot":"","sources":["../src/schemas.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC;IACrC,CAAC,CAAC,MAAM,CAAC,EAAE,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;IACzC,CAAC,CAAC,MAAM,CAAC,EAAE,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;IACvC,CAAC,CAAC,MAAM,CAAC,EAAE,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;IACxC,CAAC,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;IACnC,CAAC,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;IACrC,CAAC,CAAC,MAAM,CAAC,EAAE,aAAa,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC;IAC7E,CAAC,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;IAC7B,CAAC,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;IACjC,CAAC,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,EAAE,CAAC;IAClH,CAAC,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,EAAE,CAAC;CAClI,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,CAAC;IACjC,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE;IACd,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;IAClB,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACvC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;IACjD,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IAChE,UAAU,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE;IACzD,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC/B,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE;CACpD,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC;IACtC,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE;IACf,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACtC,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACnC,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC/B,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAClC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACjC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;CAClC,CAAC,CAAC;AAEH,4EAA4E;AAC5E,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;AAEzD,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC;IACvC,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE;IAC9C,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE;IACnE,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IAC3D,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC;CACzE,CAAC,CAAC,QAAQ,EAAE,CAAC;AAEd,2DAA2D;AAC3D,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC;IACvC,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE;IACf,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACtC,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACnC,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC/B,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAClC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACjC,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IACzC,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IACvC,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACpC,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC/C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC9B,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC,QAAQ,EAAE;IACtD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAClC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE;IACnD,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAC1C,UAAU,EAAE,gBAAgB;IAC5B,KAAK,EAAE,CAAC,CAAC,KAAK,CACZ,CAAC,CAAC,KAAK,CAAC;QACN,UAAU;QACV,CAAC,CAAC,MAAM,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE;YAChD,OAAO,EAAE,yCAAyC;SACnD,CAAC;KACH,CAAC,CACH,CAAC,GAAG,CAAC,CAAC,CAAC;CACT,CAAC,CAAC"}
+{"version":3,"file":"schemas.js","sourceRoot":"","sources":["../src/schemas.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC;IACrC,CAAC,CAAC,MAAM,CAAC,EAAE,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;IACzC,CAAC,CAAC,MAAM,CAAC,EAAE,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;IACvC,CAAC,CAAC,MAAM,CAAC,EAAE,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;IACxC,CAAC,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;IACnC,CAAC,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;IACrC,CAAC,CAAC,MAAM,CAAC,EAAE,aAAa,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC;IAC7E,CAAC,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;IAC7B,CAAC,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;IACjC,CAAC,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,EAAE,CAAC;IAClH,CAAC,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,EAAE,CAAC;CAClI,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,CAAC;IACjC,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE;IACd,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;IAClB,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACvC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;IACjD,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IACvC,GAAG,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAChD,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IAChE,UAAU,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE;IACzD,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC/B,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE;IACnD,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;CACrC,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC;IACtC,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE;IACf,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACtC,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACnC,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC/B,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAClC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACjC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;CAClC,CAAC,CAAC;AAEH,4EAA4E;AAC5E,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;AAEzD,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC;IACvC,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE;IAC9C,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE;IACnE,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IAC3D,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC;CACzE,CAAC,CAAC,QAAQ,EAAE,CAAC;AAEd,2DAA2D;AAC3D,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC;IACvC,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE;IACf,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACtC,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACnC,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC/B,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAClC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACjC,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IACzC,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IACvC,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACpC,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC/C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC9B,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC,QAAQ,EAAE;IACtD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAClC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE;IACnD,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAC1C,eAAe,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAC/C,UAAU,EAAE,gBAAgB;IAC5B,GAAG,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAChD,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IACrC,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IACvC,KAAK,EAAE,CAAC,CAAC,KAAK,CACZ,CAAC,CAAC,KAAK,CAAC;QACN,UAAU;QACV,CAAC,CAAC,MAAM,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE;YAChD,OAAO,EAAE,yCAAyC;SACnD,CAAC;KACH,CAAC,CACH,CAAC,GAAG,CAAC,CAAC,CAAC;CACT,CAAC,CAAC"}

package/dist/suite-generator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"suite-generator.d.ts","sourceRoot":"","sources":["../src/suite-generator.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAwCH,wBAAsB,aAAa,CACjC,OAAO,EAAE,MAAM,EACf,YAAY,EAAE,MAAM,GACnB,OAAO,CAAC,MAAM,CAAC,CAiEjB"}
+{"version":3,"file":"suite-generator.d.ts","sourceRoot":"","sources":["../src/suite-generator.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAmGH,wBAAsB,aAAa,CACjC,OAAO,EAAE,MAAM,EACf,YAAY,EAAE,MAAM,GACnB,OAAO,CAAC,MAAM,CAAC,CAwDjB"}

package/dist/suite-generator.js

@@ -4,8 +4,6 @@
  * Takes CLI name + help text and produces a task suite with
  * assert-based validation.
  */
-import { readFile } from 'node:fs/promises';
-import { join } from 'node:path';
 import { generateText } from 'ai';
 import { gateway } from 'ai';
 import { loadHelpFromCache, loadTopLevelHelp } from './help-loader.js';
@@ -39,6 +37,66 @@ Rules:
 - Focus on: local operations, file generation, formatting, config, help queries
 - Include setup commands to prepare the environment
 - Use realistic but safe values (no real credentials, no destructive operations)`;
+const FEW_SHOT_EXAMPLE = `cli: docker
+version_command: "docker --version"
+
+providers:
+  - openai/gpt-5-nano
+  - google/gemini-2.5-flash-lite
+
+context:
+  - zero-shot
+
+concurrency: 3
+
+system_prompt: |
+  You are working in a temporary empty directory.
+  Docker is installed and the daemon is running.
+  Complete each task using docker commands.
+  Use unique names/tags to avoid conflicts (prefix with 'bench-').
+
+tasks:
+  - id: build-image
+    intent: "Create a Dockerfile for a simple Alpine-based image that prints 'hello from docker' when run. Build it with the tag 'bench-hello'."
+    difficulty: easy
+    category: build
+    max_turns: 5
+    assert:
+      - file_exists: "Dockerfile"
+      - ran: "docker build"
+      - verify:
+          run: "docker images bench-hello --format '{{.Repository}}'"
+          output_contains: "bench-hello"
+
+  - id: run-and-capture
+    intent: "Run the 'alpine' image with the command 'echo benchmark-test-output' and capture the output."
+    difficulty: easy
+    category: run
+    max_turns: 3
+    assert:
+      - ran: "docker run"
+      - output_contains: "benchmark-test-output"
+
+  - id: inspect-container
+    intent: "Run an alpine container named 'bench-inspect' in detached mode (sleep 300), then use docker inspect to show its IP address."
+    difficulty: medium
+    category: query
+    max_turns: 5
+    assert:
+      - ran: "docker run"
+      - ran: "docker inspect"
+      - exit_code: 0
+
+  - id: volume-mount
+    intent: "Create a file called 'data.txt' with content 'volume test'. Run an alpine container that mounts the current directory to /data and reads the file with 'cat /data/data.txt'."
+    difficulty: medium
+    category: volumes
+    max_turns: 5
+    setup:
+      - "echo 'volume test' > data.txt"
+    assert:
+      - ran: "docker run"
+      - output_contains: "volume test"`;
 export async function generateSuite(cliName, helpCacheDir) {
     // Load help text
     let helpCache = await loadHelpFromCache(helpCacheDir, cliName);
@@ -51,13 +109,7 @@ export async function generateSuite(cliName, helpCacheDir) {
             throw new Error(`No help text available for ${cliName}`);
         }
     }
-    // Load example suite for few-shot
-    const suiteDir = join(new URL('.', import.meta.url).pathname.replace(/\/src\/$/, '').replace(/\/dist\/$/, ''), 'task_suites');
-    let dockerExample = '';
-    try {
-        dockerExample = await readFile(join(suiteDir, 'docker.yaml'), 'utf-8');
-    }
-    catch { /* ignore */ }
+    const fewShotExample = FEW_SHOT_EXAMPLE;
     // Build help text summary (truncate to fit context)
     const helpEntries = Object.entries(helpCache.help_texts);
     let helpSummary = '';
@@ -76,9 +128,9 @@ ${helpSummary}
 
 ## Example Task Suite
 
-### docker.yaml
+### example cli-bench.yaml
 \`\`\`yaml
-${dockerExample}
+${fewShotExample}
 \`\`\`
 
 ## Output

package/dist/suite-generator.js.map

@@ -1 +1 @@
-{"version":3,"file":"suite-generator.js","sourceRoot":"","sources":["../src/suite-generator.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,YAAY,EAAE,MAAM,IAAI,CAAC;AAClC,OAAO,EAAE,OAAO,EAAE,MAAM,IAAI,CAAC;AAC7B,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAEvE,MAAM,eAAe,GAAG,oCAAoC,CAAC;AAE7D,MAAM,aAAa,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;iFA4B2D,CAAC;AAElF,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,OAAe,EACf,YAAoB;IAEpB,iBAAiB;IACjB,IAAI,SAAS,GAAG,MAAM,iBAAiB,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;IAC/D,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,MAAM,gBAAgB,CAAC,OAAO,CAAC,CAAC;QACrD,IAAI,YAAY,EAAE,CAAC;YACjB,SAAS,GAAG,EAAE,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,EAAE,EAAE,EAAE,YAAY,EAAE,EAAE,CAAC;QACtE,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,KAAK,CAAC,8BAA8B,OAAO,EAAE,CAAC,CAAC;QAC3D,CAAC;IACH,CAAC;IAED,kCAAkC;IAClC,MAAM,QAAQ,GAAG,IAAI,CACnB,IAAI,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC,EACvF,aAAa,CACd,CAAC;IAEF,IAAI,aAAa,GAAG,EAAE,CAAC;IACvB,IAAI,CAAC;QACH,aAAa,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,EAAE,OAAO,CAAC,CAAC;IACzE,CAAC;IAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;IAExB,oDAAoD;IACpD,MAAM,WAAW,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;IACzD,IAAI,WAAW,GAAG,EAAE,CAAC;IACrB,KAAK,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,WAAW,EAAE,CAAC;QACtC,MAAM,KAAK,GAAG,GAAG,IAAI,QAAQ,CAAC;QAC9B,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QACtC,WAAW,IAAI,SAAS,OAAO,IAAI,KAAK,gBAAgB,SAAS,IAAI,CAAC;QACtE,IAAI,WAAW,CAAC,MAAM,GAAG,KAAK;YAAE,MAAM;IACxC,CAAC;IAED,MAAM,MAAM,GAAG,uCAAuC,OAAO;;;;EAI7D,WAAW;;;;;;EAMX,aAAa;;;;;yEAK0D,OAAO,KAAK,CAAC;IAEpF,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;QAChC,KAAK,EAAE,OAAO,CAAC,eAAe,CAAC;QAC/B,MAAM,EAAE,aAAa;QACrB,MAAM;QACN,WAAW,EAAE,GAAG;QAChB,eAAe,EAAE,IAAI;KACtB,CAAC,CAAC;IAEH,2DAA2D;IAC3D,IAAI,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;IAC9B,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;QAC3B,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IAC7D,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC"}
+{"version":3,"file":"suite-generator.js","sourceRoot":"","sources":["../src/suite-generator.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,IAAI,CAAC;AAClC,OAAO,EAAE,OAAO,EAAE,MAAM,IAAI,CAAC;AAC7B,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAEvE,MAAM,eAAe,GAAG,oCAAoC,CAAC;AAE7D,MAAM,aAAa,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;iFA4B2D,CAAC;AAElF,MAAM,gBAAgB,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;uCA2Dc,CAAC;AAExC,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,OAAe,EACf,YAAoB;IAEpB,iBAAiB;IACjB,IAAI,SAAS,GAAG,MAAM,iBAAiB,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;IAC/D,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,MAAM,gBAAgB,CAAC,OAAO,CAAC,CAAC;QACrD,IAAI,YAAY,EAAE,CAAC;YACjB,SAAS,GAAG,EAAE,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,EAAE,EAAE,EAAE,YAAY,EAAE,EAAE,CAAC;QACtE,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,KAAK,CAAC,8BAA8B,OAAO,EAAE,CAAC,CAAC;QAC3D,CAAC;IACH,CAAC;IAED,MAAM,cAAc,GAAG,gBAAgB,CAAC;IAExC,oDAAoD;IACpD,MAAM,WAAW,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;IACzD,IAAI,WAAW,GAAG,EAAE,CAAC;IACrB,KAAK,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,WAAW,EAAE,CAAC;QACtC,MAAM,KAAK,GAAG,GAAG,IAAI,QAAQ,CAAC;QAC9B,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QACtC,WAAW,IAAI,SAAS,OAAO,IAAI,KAAK,gBAAgB,SAAS,IAAI,CAAC;QACtE,IAAI,WAAW,CAAC,MAAM,GAAG,KAAK;YAAE,MAAM;IACxC,CAAC;IAED,MAAM,MAAM,GAAG,uCAAuC,OAAO;;;;EAI7D,WAAW;;;;;;EAMX,cAAc;;;;;yEAKyD,OAAO,KAAK,CAAC;IAEpF,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;QAChC,KAAK,EAAE,OAAO,CAAC,eAAe,CAAC;QAC/B,MAAM,EAAE,aAAa;QACrB,MAAM;QACN,WAAW,EAAE,GAAG;QAChB,eAAe,EAAE,IAAI;KACtB,CAAC,CAAC;IAEH,2DAA2D;IAC3D,IAAI,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;IAC9B,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;QAC3B,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IAC7D,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cliwatch/cli-bench",
-  "version": "0.6.2",
+  "version": "0.7.0",
   "description": "LLM CLI agent testing framework — benchmark how well AI models use your CLI tool",
   "keywords": ["cli", "benchmark", "llm", "testing", "ai-agent", "cliwatch", "evaluation"],
   "license": "MIT",
@@ -14,7 +14,6 @@
   },
   "files": [
     "dist",
-    "task_suites",
     "LICENSE",
     "CHANGELOG.md"
   ],
@@ -35,6 +34,7 @@
     "test": "vitest"
   },
   "dependencies": {
+    "@ai-sdk/google": "^3.0.0",
     "@hey-api/client-fetch": "^0.13.1",
     "ai": "^6.0.18",
     "yaml": "^2.7.0",

package/task_suites/curl.yaml

@@ -1,138 +0,0 @@
-cli: curl
-version_command: "curl --version | head -1"
-
-tasks:
-  # -- Easy --------------------------------------------------------------------
-  - id: simple-get
-    intent: "Fetch the contents of https://httpbin.org/get"
-    difficulty: easy
-    category: query
-    max_turns: 2
-    assert:
-      - ran: "curl.*httpbin.org/get"
-      - exit_code: 0
-      - output_contains: "origin"
-
-  - id: head-request
-    intent: "Send a HEAD request to https://httpbin.org/get and show the response headers"
-    difficulty: easy
-    category: query
-    max_turns: 3
-    assert:
-      - ran: "curl"
-      - ran: "-I|--head"
-      - output_contains: "HTTP"
-
-  - id: download-file
-    intent: "Download https://httpbin.org/robots.txt and save it as robots.txt"
-    difficulty: easy
-    category: crud
-    max_turns: 3
-    assert:
-      - ran: "curl"
-      - ran: "-o|--output"
-      - file_exists: "robots.txt"
-
-  - id: follow-redirects
-    intent: "Fetch https://httpbin.org/redirect/2 and follow all redirects"
-    difficulty: easy
-    category: query
-    max_turns: 3
-    assert:
-      - ran: "curl"
-      - ran: "-L|--location"
-      - exit_code: 0
-
-  # -- Medium ------------------------------------------------------------------
-  - id: post-json
-    intent: "Send a POST request to https://httpbin.org/post with JSON body {\"name\": \"bench\", \"version\": 1} and set the Content-Type header to application/json"
-    difficulty: medium
-    category: crud
-    max_turns: 5
-    assert:
-      - ran: "curl"
-      - ran: "-X POST|--request POST|-d|--data"
-      - ran: "Content-Type.*application/json"
-      - output_contains: "bench"
-
-  - id: custom-headers
-    intent: "Send a GET request to https://httpbin.org/headers with custom headers X-Request-ID: abc123 and Accept: application/xml"
-    difficulty: medium
-    category: query
-    max_turns: 5
-    assert:
-      - ran: "curl"
-      - ran: "-H|--header"
-      - ran: "X-Request-ID"
-      - output_contains: "abc123"
-
-  - id: basic-auth
-    intent: "Send a GET request to https://httpbin.org/basic-auth/user/passwd using basic authentication with username 'user' and password 'passwd'"
-    difficulty: medium
-    category: auth
-    max_turns: 5
-    assert:
-      - ran: "curl"
-      - ran: "-u|--user|user:passwd"
-      - output_contains: "authenticated"
-
-  - id: verbose-timing
-    intent: "Fetch https://httpbin.org/get and show the total time taken for the request using curl's write-out feature"
-    difficulty: medium
-    category: output
-    max_turns: 5
-    assert:
-      - ran: "curl"
-      - ran: "-w|--write-out|time_total"
-
-  # -- Hard --------------------------------------------------------------------
-  - id: put-with-file
-    intent: "Upload the file /tmp/bench-workspace/data.json to https://httpbin.org/put using a PUT request with Content-Type application/json"
-    difficulty: hard
-    category: crud
-    setup:
-      - "mkdir -p /tmp/bench-workspace"
-      - "echo '{\"key\": \"value\"}' > /tmp/bench-workspace/data.json"
-    max_turns: 7
-    assert:
-      - ran: "curl"
-      - ran: "-X PUT|--request PUT|-T"
-      - ran: "data.json"
-      - output_contains: "key"
-
-  - id: retry-with-timeout
-    intent: "Fetch https://httpbin.org/delay/1 with a 5 second timeout, retry 3 times on failure, and save the response to response.json"
-    difficulty: hard
-    category: crud
-    max_turns: 7
-    assert:
-      - ran: "curl"
-      - ran: "--retry.*3"
-      - ran: "--max-time|--connect-timeout|-m"
-      - ran: "-o|--output"
-
-  - id: multipart-upload
-    intent: "Send a multipart/form-data POST to https://httpbin.org/post with a field 'username' set to 'admin' and a file field 'config' uploading /tmp/bench-workspace/app.conf"
-    difficulty: hard
-    category: crud
-    setup:
-      - "mkdir -p /tmp/bench-workspace"
-      - "echo 'port=8080' > /tmp/bench-workspace/app.conf"
-    max_turns: 7
-    assert:
-      - ran: "curl"
-      - ran: "-F|--form"
-      - ran: "username.*admin"
-      - ran: "config.*@.*app.conf"
-      - output_contains: "admin"
-
-  - id: conditional-request
-    intent: "Fetch https://httpbin.org/cache and use conditional headers: set If-None-Match to '12345' and If-Modified-Since to 'Thu, 01 Jan 2025 00:00:00 GMT'. Show the response status code."
-    difficulty: hard
-    category: query
-    max_turns: 7
-    assert:
-      - ran: "curl"
-      - ran: "If-None-Match"
-      - ran: "If-Modified-Since"
-      - ran: "-w|--write-out|-I|--head|-v|--verbose"

package/task_suites/docker.yaml

@@ -1,163 +0,0 @@
-cli: docker
-
-tasks:
-  # -- Easy --------------------------------------------------------------------
-  - id: list-containers
-    intent: "List all running containers"
-    difficulty: easy
-    category: query
-    max_turns: 3
-    assert:
-      - ran: "docker ps"
-      - exit_code: 0
-
-  - id: list-images
-    intent: "List all local Docker images"
-    difficulty: easy
-    category: query
-    max_turns: 3
-    assert:
-      - ran: "docker image"
-      - exit_code: 0
-
-  - id: pull-image
-    intent: "Pull the latest nginx image from Docker Hub"
-    difficulty: easy
-    category: crud
-    max_turns: 3
-    assert:
-      - ran: "docker pull.*nginx"
-      - verify:
-          run: "docker images nginx --format '{{.Repository}}'"
-          output_contains: "nginx"
-
-  - id: view-logs
-    intent: "Show the last 100 lines of logs from the container 'api' and follow new output"
-    difficulty: easy
-    category: query
-    setup:
-      - "docker run -d --name api alpine sh -c 'for i in $(seq 1 200); do echo line-$i; done; sleep 3600'"
-    max_turns: 3
-    assert:
-      - ran: "docker logs"
-      - ran: "--tail"
-
-  - id: stop-container
-    intent: "Stop the container named 'web-server'"
-    difficulty: easy
-    category: crud
-    setup:
-      - "docker run -d --name web-server alpine sleep 3600"
-    max_turns: 3
-    assert:
-      - ran: "docker stop.*web-server"
-      - verify:
-          run: "docker ps --filter name=web-server --format '{{.Names}}'"
-          output_equals: ""
-
-  # -- Medium ------------------------------------------------------------------
-  - id: run-detached
-    intent: "Run an nginx container in the background, mapping host port 8080 to container port 80, named 'web'"
-    difficulty: medium
-    category: crud
-    max_turns: 5
-    assert:
-      - ran: "docker run"
-      - ran: "-d"
-      - ran: "--name.*web"
-      - verify:
-          run: "docker ps --filter name=web --format '{{.Names}}'"
-          output_contains: "web"
-
-  - id: build-with-tag
-    intent: "Build a Docker image from ./Dockerfile and tag it as myapp:v2"
-    difficulty: medium
-    category: crud
-    setup:
-      - "mkdir -p /tmp/bench-workspace"
-      - "printf 'FROM alpine:latest\nRUN echo hello' > /tmp/bench-workspace/Dockerfile"
-    max_turns: 5
-    assert:
-      - ran: "docker build"
-      - ran: "myapp:v2"
-      - verify:
-          run: "docker images myapp:v2 --format '{{.Repository}}:{{.Tag}}'"
-          output_contains: "myapp:v2"
-
-  - id: exec-into-container
-    intent: "Run the command 'cat /etc/os-release' inside the running container 'web'"
-    difficulty: medium
-    category: crud
-    setup:
-      - "docker run -d --name web alpine sleep 3600"
-    max_turns: 5
-    assert:
-      - ran: "docker exec.*web.*cat /etc/os-release"
-      - exit_code: 0
-
-  - id: inspect-json
-    intent: "Get the IP address of the container 'web' using docker inspect with a format template"
-    difficulty: medium
-    category: output
-    setup:
-      - "docker run -d --name web alpine sleep 3600"
-    max_turns: 5
-    assert:
-      - ran: "docker inspect"
-      - ran: "--format"
-
-  - id: prune-all
-    intent: "Remove all stopped containers, unused networks, dangling images, and build cache without prompting for confirmation"
-    difficulty: medium
-    category: crud
-    max_turns: 5
-    assert:
-      - ran: "docker system prune"
-      - ran: "--force|-f"
-
-  # -- Hard --------------------------------------------------------------------
-  - id: run-complex
-    intent: "Run a postgres:16 container named 'db' in the background with environment variables POSTGRES_USER=admin and POSTGRES_PASSWORD=secret, mount a volume 'pgdata' to /var/lib/postgresql/data, and connect it to the network 'backend'"
-    difficulty: hard
-    category: crud
-    setup:
-      - "docker network create backend || true"
-    max_turns: 7
-    assert:
-      - ran: "docker run"
-      - ran: "--name.*db"
-      - ran: "POSTGRES_USER=admin"
-      - ran: "POSTGRES_PASSWORD=secret"
-      - verify:
-          run: "docker ps --filter name=db --format '{{.Names}}'"
-          output_contains: "db"
-
-  - id: compose-up
-    intent: "Start all services defined in docker-compose.yml in detached mode and rebuild any changed images"
-    difficulty: hard
-    category: workflow
-    setup:
-      - "mkdir -p /tmp/bench-workspace"
-      - "printf 'services:\n  web:\n    image: alpine\n    command: sleep 3600\n' > /tmp/bench-workspace/docker-compose.yml"
-    max_turns: 7
-    assert:
-      - ran: "docker compose.*up"
-      - ran: "-d|--detach"
-      - ran: "--build"
-
-  - id: multi-stage-debug
-    intent: "Build only the 'builder' stage from /tmp/bench-workspace/Dockerfile, tag it as 'myapp:debug', and don't use cache"
-    difficulty: hard
-    category: crud
-    setup:
-      - "mkdir -p /tmp/bench-workspace"
-      - "printf 'FROM alpine:latest AS builder\nRUN echo building\nFROM alpine:latest\nCOPY --from=builder / /\n' > /tmp/bench-workspace/Dockerfile"
-    max_turns: 7
-    assert:
-      - ran: "docker build"
-      - ran: "--target.*builder"
-      - ran: "myapp:debug"
-      - ran: "--no-cache"
-      - verify:
-          run: "docker images myapp:debug --format '{{.Repository}}:{{.Tag}}'"
-          output_contains: "myapp:debug"