@cliwatch/cli-bench 0.6.3 → 0.7.1

package/dist/schemas.js

@@ -19,10 +19,14 @@ export const TaskSchema = z.object({
     intent: z.string(),
     assert: z.array(AssertionSchema).min(1),
     setup: z.array(z.string()).optional().default([]),
+    cleanup: z.array(z.string()).optional(),
+    env: z.record(z.string(), z.string()).optional(),
     max_turns: z.number().int().min(1).max(20).optional().default(5),
     difficulty: z.enum(['easy', 'medium', 'hard']).optional(),
     category: z.string().optional(),
     repeat: z.number().int().min(1).max(100).optional(),
+    tags: z.array(z.string()).optional(),
+    scaffold: z.union([z.string(), z.literal(false)]).optional(),
 });
 export const TaskSuiteSchema = z.object({
     cli: z.string(),
@@ -50,7 +54,6 @@ export const ConfigFileSchema = z.object({
     website_url: z.string().optional(),
     github_url: z.string().optional(),
     providers: z.array(z.string()).optional(),
-    context: z.array(z.string()).optional(),
     system_prompt: z.string().optional(),
     concurrency: z.number().int().min(1).optional(),
     workdir: z.string().optional(),
@@ -60,6 +63,10 @@ export const ConfigFileSchema = z.object({
     redact_env: z.array(z.string()).optional(),
     redact_patterns: z.array(z.string()).optional(),
     thresholds: ThresholdsSchema,
+    env: z.record(z.string(), z.string()).optional(),
+    setup: z.array(z.string()).optional(),
+    scaffold: z.string().optional(),
+    cleanup: z.array(z.string()).optional(),
     tasks: z.array(z.union([
         TaskSchema,
         z.string().refine((s) => s.startsWith('file://'), {

package/dist/schemas.js.map

@@ -1 +1 @@
-{"version":3,"file":"schemas.js","sourceRoot":"","sources":["../src/schemas.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC;IACrC,CAAC,CAAC,MAAM,CAAC,EAAE,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;IACzC,CAAC,CAAC,MAAM,CAAC,EAAE,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;IACvC,CAAC,CAAC,MAAM,CAAC,EAAE,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;IACxC,CAAC,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;IACnC,CAAC,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;IACrC,CAAC,CAAC,MAAM,CAAC,EAAE,aAAa,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC;IAC7E,CAAC,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;IAC7B,CAAC,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;IACjC,CAAC,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,EAAE,CAAC;IAClH,CAAC,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,EAAE,CAAC;CAClI,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,CAAC;IACjC,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE;IACd,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;IAClB,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACvC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;IACjD,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IAChE,UAAU,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE;IACzD,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC/B,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE;CACpD,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC;IACtC,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE;IACf,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACtC,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACnC,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC/B,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAClC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACjC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;CAClC,CAAC,CAAC;AAEH,4EAA4E;AAC5E,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;AAEzD,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC;IACvC,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE;IAC9C,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE;IACnE,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IAC3D,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC;CACzE,CAAC,CAAC,QAAQ,EAAE,CAAC;AAEd,2DAA2D;AAC3D,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC;IACvC,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE;IACf,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACtC,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACnC,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC/B,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAClC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACjC,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IACzC,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IACvC,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACpC,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC/C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC9B,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC,QAAQ,EAAE;IACtD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAClC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE;IACnD,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAC1C,eAAe,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAC/C,UAAU,EAAE,gBAAgB;IAC5B,KAAK,EAAE,CAAC,CAAC,KAAK,CACZ,CAAC,CAAC,KAAK,CAAC;QACN,UAAU;QACV,CAAC,CAAC,MAAM,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE;YAChD,OAAO,EAAE,yCAAyC;SACnD,CAAC;KACH,CAAC,CACH,CAAC,GAAG,CAAC,CAAC,CAAC;CACT,CAAC,CAAC"}
+{"version":3,"file":"schemas.js","sourceRoot":"","sources":["../src/schemas.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC;IACrC,CAAC,CAAC,MAAM,CAAC,EAAE,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;IACzC,CAAC,CAAC,MAAM,CAAC,EAAE,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;IACvC,CAAC,CAAC,MAAM,CAAC,EAAE,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;IACxC,CAAC,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;IACnC,CAAC,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;IACrC,CAAC,CAAC,MAAM,CAAC,EAAE,aAAa,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC;IAC7E,CAAC,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;IAC7B,CAAC,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;IACjC,CAAC,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,EAAE,CAAC;IAClH,CAAC,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,EAAE,CAAC;CAClI,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,CAAC;IACjC,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE;IACd,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;IAClB,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACvC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;IACjD,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IACvC,GAAG,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAChD,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IAChE,UAAU,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE;IACzD,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC/B,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE;IACnD,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IACpC,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;CAC7D,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC;IACtC,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE;IACf,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACtC,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACnC,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC/B,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAClC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACjC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;CAClC,CAAC,CAAC;AAEH,4EAA4E;AAC5E,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;AAEzD,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC;IACvC,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE;IAC9C,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE;IACnE,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IAC3D,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC;CACzE,CAAC,CAAC,QAAQ,EAAE,CAAC;AAEd,2DAA2D;AAC3D,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC;IACvC,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE;IACf,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACtC,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACnC,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC/B,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAClC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACjC,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IACzC,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACpC,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC/C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC9B,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC,QAAQ,EAAE;IACtD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAClC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE;IACnD,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAC1C,eAAe,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAC/C,UAAU,EAAE,gBAAgB;IAC5B,GAAG,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAChD,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IACrC,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC/B,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IACvC,KAAK,EAAE,CAAC,CAAC,KAAK,CACZ,CAAC,CAAC,KAAK,CAAC;QACN,UAAU;QACV,CAAC,CAAC,MAAM,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE;YAChD,OAAO,EAAE,yCAAyC;SACnD,CAAC;KACH,CAAC,CACH,CAAC,GAAG,CAAC,CAAC,CAAC;CACT,CAAC,CAAC"}

package/dist/suite-generator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"suite-generator.d.ts","sourceRoot":"","sources":["../src/suite-generator.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAwCH,wBAAsB,aAAa,CACjC,OAAO,EAAE,MAAM,EACf,YAAY,EAAE,MAAM,GACnB,OAAO,CAAC,MAAM,CAAC,CAiEjB"}
+{"version":3,"file":"suite-generator.d.ts","sourceRoot":"","sources":["../src/suite-generator.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAuIH,wBAAsB,aAAa,CACjC,OAAO,EAAE,MAAM,EACf,YAAY,EAAE,MAAM,GACnB,OAAO,CAAC,MAAM,CAAC,CAwDjB"}

package/dist/suite-generator.js

@@ -5,10 +5,10 @@
  * assert-based validation.
  */
 import { readFile } from 'node:fs/promises';
+import { execFile } from 'node:child_process';
 import { join } from 'node:path';
 import { generateText } from 'ai';
 import { gateway } from 'ai';
-import { loadHelpFromCache, loadTopLevelHelp } from './help-loader.js';
 const GENERATOR_MODEL = 'anthropic/claude-sonnet-4-20250514';
 const SYSTEM_PROMPT = `You are a CLI test suite generator. Given help text for a CLI tool and example task suites, generate a YAML task suite.
 
@@ -39,6 +39,95 @@ Rules:
 - Focus on: local operations, file generation, formatting, config, help queries
 - Include setup commands to prepare the environment
 - Use realistic but safe values (no real credentials, no destructive operations)`;
+const FEW_SHOT_EXAMPLE = `cli: docker
+version_command: "docker --version"
+
+providers:
+  - openai/gpt-5-nano
+  - google/gemini-2.5-flash-lite
+
+concurrency: 3
+
+system_prompt: |
+  You are working in a temporary empty directory.
+  Docker is installed and the daemon is running.
+  Complete each task using docker commands.
+  Use unique names/tags to avoid conflicts (prefix with 'bench-').
+
+tasks:
+  - id: build-image
+    intent: "Create a Dockerfile for a simple Alpine-based image that prints 'hello from docker' when run. Build it with the tag 'bench-hello'."
+    difficulty: easy
+    category: build
+    max_turns: 5
+    assert:
+      - file_exists: "Dockerfile"
+      - ran: "docker build"
+      - verify:
+          run: "docker images bench-hello --format '{{.Repository}}'"
+          output_contains: "bench-hello"
+
+  - id: run-and-capture
+    intent: "Run the 'alpine' image with the command 'echo benchmark-test-output' and capture the output."
+    difficulty: easy
+    category: run
+    max_turns: 3
+    assert:
+      - ran: "docker run"
+      - output_contains: "benchmark-test-output"
+
+  - id: inspect-container
+    intent: "Run an alpine container named 'bench-inspect' in detached mode (sleep 300), then use docker inspect to show its IP address."
+    difficulty: medium
+    category: query
+    max_turns: 5
+    assert:
+      - ran: "docker run"
+      - ran: "docker inspect"
+      - exit_code: 0
+
+  - id: volume-mount
+    intent: "Create a file called 'data.txt' with content 'volume test'. Run an alpine container that mounts the current directory to /data and reads the file with 'cat /data/data.txt'."
+    difficulty: medium
+    category: volumes
+    max_turns: 5
+    setup:
+      - "echo 'volume test' > data.txt"
+    assert:
+      - ran: "docker run"
+      - output_contains: "volume test"`;
+async function loadHelpFromCache(cacheDir, cliName) {
+    try {
+        const filePath = join(cacheDir, `${cliName}.json`);
+        const raw = await readFile(filePath, 'utf-8');
+        const data = JSON.parse(raw);
+        if (Array.isArray(data)) {
+            const entry = data.find((d) => d.cli_name === cliName);
+            if (entry?.help_texts) {
+                return { cli_name: entry.cli_name, help_texts: entry.help_texts };
+            }
+            return null;
+        }
+        if (data.help_texts)
+            return data;
+        return null;
+    }
+    catch {
+        return null;
+    }
+}
+function loadTopLevelHelp(cliName) {
+    return new Promise((resolve) => {
+        execFile(cliName, ['--help'], { timeout: 30_000 }, (err, stdout, stderr) => {
+            if (err) {
+                resolve(null);
+                return;
+            }
+            const output = (stdout || stderr).trim();
+            resolve(output || null);
+        });
+    });
+}
 export async function generateSuite(cliName, helpCacheDir) {
     // Load help text
     let helpCache = await loadHelpFromCache(helpCacheDir, cliName);
@@ -51,13 +140,7 @@ export async function generateSuite(cliName, helpCacheDir) {
             throw new Error(`No help text available for ${cliName}`);
         }
     }
-    // Load example suite for few-shot
-    const suiteDir = join(new URL('.', import.meta.url).pathname.replace(/\/src\/$/, '').replace(/\/dist\/$/, ''), 'task_suites');
-    let dockerExample = '';
-    try {
-        dockerExample = await readFile(join(suiteDir, 'docker.yaml'), 'utf-8');
-    }
-    catch { /* ignore */ }
+    const fewShotExample = FEW_SHOT_EXAMPLE;
     // Build help text summary (truncate to fit context)
     const helpEntries = Object.entries(helpCache.help_texts);
     let helpSummary = '';
@@ -76,9 +159,9 @@ ${helpSummary}
 
 ## Example Task Suite
 
-### docker.yaml
+### example cli-bench.yaml
 \`\`\`yaml
-${dockerExample}
+${fewShotExample}
 \`\`\`
 
 ## Output

package/dist/suite-generator.js.map

@@ -1 +1 @@
-{"version":3,"file":"suite-generator.js","sourceRoot":"","sources":["../src/suite-generator.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,YAAY,EAAE,MAAM,IAAI,CAAC;AAClC,OAAO,EAAE,OAAO,EAAE,MAAM,IAAI,CAAC;AAC7B,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAEvE,MAAM,eAAe,GAAG,oCAAoC,CAAC;AAE7D,MAAM,aAAa,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;iFA4B2D,CAAC;AAElF,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,OAAe,EACf,YAAoB;IAEpB,iBAAiB;IACjB,IAAI,SAAS,GAAG,MAAM,iBAAiB,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;IAC/D,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,MAAM,gBAAgB,CAAC,OAAO,CAAC,CAAC;QACrD,IAAI,YAAY,EAAE,CAAC;YACjB,SAAS,GAAG,EAAE,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,EAAE,EAAE,EAAE,YAAY,EAAE,EAAE,CAAC;QACtE,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,KAAK,CAAC,8BAA8B,OAAO,EAAE,CAAC,CAAC;QAC3D,CAAC;IACH,CAAC;IAED,kCAAkC;IAClC,MAAM,QAAQ,GAAG,IAAI,CACnB,IAAI,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC,EACvF,aAAa,CACd,CAAC;IAEF,IAAI,aAAa,GAAG,EAAE,CAAC;IACvB,IAAI,CAAC;QACH,aAAa,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,EAAE,OAAO,CAAC,CAAC;IACzE,CAAC;IAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;IAExB,oDAAoD;IACpD,MAAM,WAAW,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;IACzD,IAAI,WAAW,GAAG,EAAE,CAAC;IACrB,KAAK,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,WAAW,EAAE,CAAC;QACtC,MAAM,KAAK,GAAG,GAAG,IAAI,QAAQ,CAAC;QAC9B,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QACtC,WAAW,IAAI,SAAS,OAAO,IAAI,KAAK,gBAAgB,SAAS,IAAI,CAAC;QACtE,IAAI,WAAW,CAAC,MAAM,GAAG,KAAK;YAAE,MAAM;IACxC,CAAC;IAED,MAAM,MAAM,GAAG,uCAAuC,OAAO;;;;EAI7D,WAAW;;;;;;EAMX,aAAa;;;;;yEAK0D,OAAO,KAAK,CAAC;IAEpF,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;QAChC,KAAK,EAAE,OAAO,CAAC,eAAe,CAAC;QAC/B,MAAM,EAAE,aAAa;QACrB,MAAM;QACN,WAAW,EAAE,GAAG;QAChB,eAAe,EAAE,IAAI;KACtB,CAAC,CAAC;IAEH,2DAA2D;IAC3D,IAAI,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;IAC9B,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;QAC3B,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IAC7D,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC"}
+{"version":3,"file":"suite-generator.js","sourceRoot":"","sources":["../src/suite-generator.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,YAAY,EAAE,MAAM,IAAI,CAAC;AAClC,OAAO,EAAE,OAAO,EAAE,MAAM,IAAI,CAAC;AAE7B,MAAM,eAAe,GAAG,oCAAoC,CAAC;AAE7D,MAAM,aAAa,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;iFA4B2D,CAAC;AAElF,MAAM,gBAAgB,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;uCAwDc,CAAC;AAOxC,KAAK,UAAU,iBAAiB,CAC9B,QAAgB,EAChB,OAAe;IAEf,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,GAAG,OAAO,OAAO,CAAC,CAAC;QACnD,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC9C,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC7B,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YACxB,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAwB,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC;YAC9E,IAAI,KAAK,EAAE,UAAU,EAAE,CAAC;gBACtB,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,UAAU,EAAE,KAAK,CAAC,UAAU,EAAE,CAAC;YACpE,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC;QACD,IAAI,IAAI,CAAC,UAAU;YAAE,OAAO,IAAiB,CAAC;QAC9C,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,gBAAgB,CAAC,OAAe;IACvC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC7B,QAAQ,CAAC,OAAO,EAAE,CAAC,QAAQ,CAAC,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE;YACzE,IAAI,GAAG,EAAE,CAAC;gBAAC,OAAO,CAAC,IAAI,CAAC,CAAC;gBAAC,OAAO;YAAC,CAAC;YACnC,MAAM,MAAM,GAAG,CAAC,MAAM,IAAI,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;YACzC,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,OAAe,EACf,YAAoB;IAEpB,iBAAiB;IACjB,IAAI,SAAS,GAAG,MAAM,iBAAiB,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;IAC/D,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,MAAM,gBAAgB,CAAC,OAAO,CAAC,CAAC;QACrD,IAAI,YAAY,EAAE,CAAC;YACjB,SAAS,GAAG,EAAE,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,EAAE,EAAE,EAAE,YAAY,EAAE,EAAE,CAAC;QACtE,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,KAAK,CAAC,8BAA8B,OAAO,EAAE,CAAC,CAAC;QAC3D,CAAC;IACH,CAAC;IAED,MAAM,cAAc,GAAG,gBAAgB,CAAC;IAExC,oDAAoD;IACpD,MAAM,WAAW,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;IACzD,IAAI,WAAW,GAAG,EAAE,CAAC;IACrB,KAAK,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,WAAW,EAAE,CAAC;QACtC,MAAM,KAAK,GAAG,GAAG,IAAI,QAAQ,CAAC;QAC9B,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QACtC,WAAW,IAAI,SAAS,OAAO,IAAI,KAAK,gBAAgB,SAAS,IAAI,CAAC;QACtE,IAAI,WAAW,CAAC,MAAM,GAAG,KAAK;YAAE,MAAM;IACxC,CAAC;IAED,MAAM,MAAM,GAAG,uCAAuC,OAAO;;;;EAI7D,WAAW;;;;;;EAMX,cAAc;;;;;yEAKyD,OAAO,KAAK,CAAC;IAEpF,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;QAChC,KAAK,EAAE,OAAO,CAAC,eAAe,CAAC;QAC/B,MAAM,EAAE,aAAa;QACrB,MAAM;QACN,WAAW,EAAE,GAAG;QAChB,eAAe,EAAE,IAAI;KACtB,CAAC,CAAC;IAEH,2DAA2D;IAC3D,IAAI,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;IAC9B,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;QAC3B,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IAC7D,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cliwatch/cli-bench",
-  "version": "0.6.3",
+  "version": "0.7.1",
   "description": "LLM CLI agent testing framework — benchmark how well AI models use your CLI tool",
   "keywords": ["cli", "benchmark", "llm", "testing", "ai-agent", "cliwatch", "evaluation"],
   "license": "MIT",
@@ -14,7 +14,6 @@
   },
   "files": [
     "dist",
-    "task_suites",
     "LICENSE",
     "CHANGELOG.md"
   ],
@@ -35,6 +34,7 @@
     "test": "vitest"
   },
   "dependencies": {
+    "@ai-sdk/google": "^3.0.0",
     "@hey-api/client-fetch": "^0.13.1",
     "ai": "^6.0.18",
     "yaml": "^2.7.0",

package/dist/help-loader.d.ts

@@ -1,17 +0,0 @@
-/**
- * Loads CLI help text from cached JSON files or live CLI execution.
- *
- * Cached mode reads from help_cache/<cli>.json, produced by
- * `audit-worker --dry-run --output-help --output <file>`.
- *
- * Live mode shells out to the CLI's --help.
- */
-import type { HelpCache } from './models.js';
-export declare function loadHelpFromCache(cacheDir: string, cliName: string): Promise<HelpCache | null>;
-export declare function listAvailableCaches(cacheDir: string): Promise<string[]>;
-/**
- * Run `cli --help` and return the output as a string.
- * Single top-level invocation — no subcommand crawling.
- */
-export declare function loadTopLevelHelp(cliName: string): Promise<string | null>;
-//# sourceMappingURL=help-loader.d.ts.map

package/dist/help-loader.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"help-loader.d.ts","sourceRoot":"","sources":["../src/help-loader.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAKH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAE7C,wBAAsB,iBAAiB,CACrC,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,CA6B3B;AAED,wBAAsB,mBAAmB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAS7E;AAED;;;GAGG;AACH,wBAAsB,gBAAgB,CACpC,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAWxB"}

package/dist/help-loader.js

@@ -1,65 +0,0 @@
-/**
- * Loads CLI help text from cached JSON files or live CLI execution.
- *
- * Cached mode reads from help_cache/<cli>.json, produced by
- * `audit-worker --dry-run --output-help --output <file>`.
- *
- * Live mode shells out to the CLI's --help.
- */
-import { readFile, readdir } from 'node:fs/promises';
-import { join } from 'node:path';
-import { execFile } from 'node:child_process';
-export async function loadHelpFromCache(cacheDir, cliName) {
-    try {
-        const filePath = join(cacheDir, `${cliName}.json`);
-        const raw = await readFile(filePath, 'utf-8');
-        const data = JSON.parse(raw);
-        // Support both formats: direct HelpCache or audit-worker output array
-        if (Array.isArray(data)) {
-            const entry = data.find((d) => d.cli_name === cliName);
-            if (entry?.help_texts) {
-                return {
-                    cli_name: entry.cli_name,
-                    help_texts: entry.help_texts,
-                    version: entry.version,
-                };
-            }
-            return null;
-        }
-        if (data.help_texts) {
-            return data;
-        }
-        return null;
-    }
-    catch {
-        return null;
-    }
-}
-export async function listAvailableCaches(cacheDir) {
-    try {
-        const files = await readdir(cacheDir);
-        return files
-            .filter((f) => f.endsWith('.json'))
-            .map((f) => f.replace(/\.json$/, ''));
-    }
-    catch {
-        return [];
-    }
-}
-/**
- * Run `cli --help` and return the output as a string.
- * Single top-level invocation — no subcommand crawling.
- */
-export async function loadTopLevelHelp(cliName) {
-    return new Promise((resolve) => {
-        execFile(cliName, ['--help'], { timeout: 30_000 }, (err, stdout, stderr) => {
-            if (err) {
-                resolve(null);
-                return;
-            }
-            const output = (stdout || stderr).trim();
-            resolve(output || null);
-        });
-    });
-}
-//# sourceMappingURL=help-loader.js.map

package/dist/help-loader.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"help-loader.js","sourceRoot":"","sources":["../src/help-loader.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAG9C,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,QAAgB,EAChB,OAAe;IAEf,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,GAAG,OAAO,OAAO,CAAC,CAAC;QACnD,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC9C,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAE7B,sEAAsE;QACtE,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YACxB,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CACrB,CAAC,CAAwB,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,OAAO,CACrD,CAAC;YACF,IAAI,KAAK,EAAE,UAAU,EAAE,CAAC;gBACtB,OAAO;oBACL,QAAQ,EAAE,KAAK,CAAC,QAAQ;oBACxB,UAAU,EAAE,KAAK,CAAC,UAAU;oBAC5B,OAAO,EAAE,KAAK,CAAC,OAAO;iBACvB,CAAC;YACJ,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC;QAED,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,OAAO,IAAiB,CAAC;QAC3B,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,QAAgB;IACxD,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,CAAC;QACtC,OAAO,KAAK;aACT,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;aAClC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,CAAC;IAC1C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,OAAe;IAEf,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC7B,QAAQ,CAAC,OAAO,EAAE,CAAC,QAAQ,CAAC,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE;YACzE,IAAI,GAAG,EAAE,CAAC;gBACR,OAAO,CAAC,IAAI,CAAC,CAAC;gBACd,OAAO;YACT,CAAC;YACD,MAAM,MAAM,GAAG,CAAC,MAAM,IAAI,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;YACzC,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC"}

package/task_suites/curl.yaml

@@ -1,138 +0,0 @@
-cli: curl
-version_command: "curl --version | head -1"
-
-tasks:
-  # -- Easy --------------------------------------------------------------------
-  - id: simple-get
-    intent: "Fetch the contents of https://httpbin.org/get"
-    difficulty: easy
-    category: query
-    max_turns: 2
-    assert:
-      - ran: "curl.*httpbin.org/get"
-      - exit_code: 0
-      - output_contains: "origin"
-
-  - id: head-request
-    intent: "Send a HEAD request to https://httpbin.org/get and show the response headers"
-    difficulty: easy
-    category: query
-    max_turns: 3
-    assert:
-      - ran: "curl"
-      - ran: "-I|--head"
-      - output_contains: "HTTP"
-
-  - id: download-file
-    intent: "Download https://httpbin.org/robots.txt and save it as robots.txt"
-    difficulty: easy
-    category: crud
-    max_turns: 3
-    assert:
-      - ran: "curl"
-      - ran: "-o|--output"
-      - file_exists: "robots.txt"
-
-  - id: follow-redirects
-    intent: "Fetch https://httpbin.org/redirect/2 and follow all redirects"
-    difficulty: easy
-    category: query
-    max_turns: 3
-    assert:
-      - ran: "curl"
-      - ran: "-L|--location"
-      - exit_code: 0
-
-  # -- Medium ------------------------------------------------------------------
-  - id: post-json
-    intent: "Send a POST request to https://httpbin.org/post with JSON body {\"name\": \"bench\", \"version\": 1} and set the Content-Type header to application/json"
-    difficulty: medium
-    category: crud
-    max_turns: 5
-    assert:
-      - ran: "curl"
-      - ran: "-X POST|--request POST|-d|--data"
-      - ran: "Content-Type.*application/json"
-      - output_contains: "bench"
-
-  - id: custom-headers
-    intent: "Send a GET request to https://httpbin.org/headers with custom headers X-Request-ID: abc123 and Accept: application/xml"
-    difficulty: medium
-    category: query
-    max_turns: 5
-    assert:
-      - ran: "curl"
-      - ran: "-H|--header"
-      - ran: "X-Request-ID"
-      - output_contains: "abc123"
-
-  - id: basic-auth
-    intent: "Send a GET request to https://httpbin.org/basic-auth/user/passwd using basic authentication with username 'user' and password 'passwd'"
-    difficulty: medium
-    category: auth
-    max_turns: 5
-    assert:
-      - ran: "curl"
-      - ran: "-u|--user|user:passwd"
-      - output_contains: "authenticated"
-
-  - id: verbose-timing
-    intent: "Fetch https://httpbin.org/get and show the total time taken for the request using curl's write-out feature"
-    difficulty: medium
-    category: output
-    max_turns: 5
-    assert:
-      - ran: "curl"
-      - ran: "-w|--write-out|time_total"
-
-  # -- Hard --------------------------------------------------------------------
-  - id: put-with-file
-    intent: "Upload the file /tmp/bench-workspace/data.json to https://httpbin.org/put using a PUT request with Content-Type application/json"
-    difficulty: hard
-    category: crud
-    setup:
-      - "mkdir -p /tmp/bench-workspace"
-      - "echo '{\"key\": \"value\"}' > /tmp/bench-workspace/data.json"
-    max_turns: 7
-    assert:
-      - ran: "curl"
-      - ran: "-X PUT|--request PUT|-T"
-      - ran: "data.json"
-      - output_contains: "key"
-
-  - id: retry-with-timeout
-    intent: "Fetch https://httpbin.org/delay/1 with a 5 second timeout, retry 3 times on failure, and save the response to response.json"
-    difficulty: hard
-    category: crud
-    max_turns: 7
-    assert:
-      - ran: "curl"
-      - ran: "--retry.*3"
-      - ran: "--max-time|--connect-timeout|-m"
-      - ran: "-o|--output"
-
-  - id: multipart-upload
-    intent: "Send a multipart/form-data POST to https://httpbin.org/post with a field 'username' set to 'admin' and a file field 'config' uploading /tmp/bench-workspace/app.conf"
-    difficulty: hard
-    category: crud
-    setup:
-      - "mkdir -p /tmp/bench-workspace"
-      - "echo 'port=8080' > /tmp/bench-workspace/app.conf"
-    max_turns: 7
-    assert:
-      - ran: "curl"
-      - ran: "-F|--form"
-      - ran: "username.*admin"
-      - ran: "config.*@.*app.conf"
-      - output_contains: "admin"
-
-  - id: conditional-request
-    intent: "Fetch https://httpbin.org/cache and use conditional headers: set If-None-Match to '12345' and If-Modified-Since to 'Thu, 01 Jan 2025 00:00:00 GMT'. Show the response status code."
-    difficulty: hard
-    category: query
-    max_turns: 7
-    assert:
-      - ran: "curl"
-      - ran: "If-None-Match"
-      - ran: "If-Modified-Since"
-      - ran: "-w|--write-out|-I|--head|-v|--verbose"

package/task_suites/docker.yaml

@@ -1,163 +0,0 @@
-cli: docker
-
-tasks:
-  # -- Easy --------------------------------------------------------------------
-  - id: list-containers
-    intent: "List all running containers"
-    difficulty: easy
-    category: query
-    max_turns: 3
-    assert:
-      - ran: "docker ps"
-      - exit_code: 0
-
-  - id: list-images
-    intent: "List all local Docker images"
-    difficulty: easy
-    category: query
-    max_turns: 3
-    assert:
-      - ran: "docker image"
-      - exit_code: 0
-
-  - id: pull-image
-    intent: "Pull the latest nginx image from Docker Hub"
-    difficulty: easy
-    category: crud
-    max_turns: 3
-    assert:
-      - ran: "docker pull.*nginx"
-      - verify:
-          run: "docker images nginx --format '{{.Repository}}'"
-          output_contains: "nginx"
-
-  - id: view-logs
-    intent: "Show the last 100 lines of logs from the container 'api' and follow new output"
-    difficulty: easy
-    category: query
-    setup:
-      - "docker run -d --name api alpine sh -c 'for i in $(seq 1 200); do echo line-$i; done; sleep 3600'"
-    max_turns: 3
-    assert:
-      - ran: "docker logs"
-      - ran: "--tail"
-
-  - id: stop-container
-    intent: "Stop the container named 'web-server'"
-    difficulty: easy
-    category: crud
-    setup:
-      - "docker run -d --name web-server alpine sleep 3600"
-    max_turns: 3
-    assert:
-      - ran: "docker stop.*web-server"
-      - verify:
-          run: "docker ps --filter name=web-server --format '{{.Names}}'"
-          output_equals: ""
-
-  # -- Medium ------------------------------------------------------------------
-  - id: run-detached
-    intent: "Run an nginx container in the background, mapping host port 8080 to container port 80, named 'web'"
-    difficulty: medium
-    category: crud
-    max_turns: 5
-    assert:
-      - ran: "docker run"
-      - ran: "-d"
-      - ran: "--name.*web"
-      - verify:
-          run: "docker ps --filter name=web --format '{{.Names}}'"
-          output_contains: "web"
-
-  - id: build-with-tag
-    intent: "Build a Docker image from ./Dockerfile and tag it as myapp:v2"
-    difficulty: medium
-    category: crud
-    setup:
-      - "mkdir -p /tmp/bench-workspace"
-      - "printf 'FROM alpine:latest\nRUN echo hello' > /tmp/bench-workspace/Dockerfile"
-    max_turns: 5
-    assert:
-      - ran: "docker build"
-      - ran: "myapp:v2"
-      - verify:
-          run: "docker images myapp:v2 --format '{{.Repository}}:{{.Tag}}'"
-          output_contains: "myapp:v2"
-
-  - id: exec-into-container
-    intent: "Run the command 'cat /etc/os-release' inside the running container 'web'"
-    difficulty: medium
-    category: crud
-    setup:
-      - "docker run -d --name web alpine sleep 3600"
-    max_turns: 5
-    assert:
-      - ran: "docker exec.*web.*cat /etc/os-release"
-      - exit_code: 0
-
-  - id: inspect-json
-    intent: "Get the IP address of the container 'web' using docker inspect with a format template"
-    difficulty: medium
-    category: output
-    setup:
-      - "docker run -d --name web alpine sleep 3600"
-    max_turns: 5
-    assert:
-      - ran: "docker inspect"
-      - ran: "--format"
-
-  - id: prune-all
-    intent: "Remove all stopped containers, unused networks, dangling images, and build cache without prompting for confirmation"
-    difficulty: medium
-    category: crud
-    max_turns: 5
-    assert:
-      - ran: "docker system prune"
-      - ran: "--force|-f"
-
-  # -- Hard --------------------------------------------------------------------
-  - id: run-complex
-    intent: "Run a postgres:16 container named 'db' in the background with environment variables POSTGRES_USER=admin and POSTGRES_PASSWORD=secret, mount a volume 'pgdata' to /var/lib/postgresql/data, and connect it to the network 'backend'"
-    difficulty: hard
-    category: crud
-    setup:
-      - "docker network create backend || true"
-    max_turns: 7
-    assert:
-      - ran: "docker run"
-      - ran: "--name.*db"
-      - ran: "POSTGRES_USER=admin"
-      - ran: "POSTGRES_PASSWORD=secret"
-      - verify:
-          run: "docker ps --filter name=db --format '{{.Names}}'"
-          output_contains: "db"
-
-  - id: compose-up
-    intent: "Start all services defined in docker-compose.yml in detached mode and rebuild any changed images"
-    difficulty: hard
-    category: workflow
-    setup:
-      - "mkdir -p /tmp/bench-workspace"
-      - "printf 'services:\n  web:\n    image: alpine\n    command: sleep 3600\n' > /tmp/bench-workspace/docker-compose.yml"
-    max_turns: 7
-    assert:
-      - ran: "docker compose.*up"
-      - ran: "-d|--detach"
-      - ran: "--build"
-
-  - id: multi-stage-debug
-    intent: "Build only the 'builder' stage from /tmp/bench-workspace/Dockerfile, tag it as 'myapp:debug', and don't use cache"
-    difficulty: hard
-    category: crud
-    setup:
-      - "mkdir -p /tmp/bench-workspace"
-      - "printf 'FROM alpine:latest AS builder\nRUN echo building\nFROM alpine:latest\nCOPY --from=builder / /\n' > /tmp/bench-workspace/Dockerfile"
-    max_turns: 7
-    assert:
-      - ran: "docker build"
-      - ran: "--target.*builder"
-      - ran: "myapp:debug"
-      - ran: "--no-cache"
-      - verify:
-          run: "docker images myapp:debug --format '{{.Repository}}:{{.Tag}}'"
-          output_contains: "myapp:debug"