@cliwatch/cli-bench 0.5.4 → 0.6.1

package/dist/models.d.ts

@@ -4,7 +4,8 @@
  * Assertion-based evaluation: tasks define assertions that are checked
  * against the agent's execution trace.
  */
-export type HelpMode = 'injected' | 'discoverable' | 'none';
+export type ContextMode = 'zero-shot' | 'help' | 'docs';
+export declare const CONTEXT_MODES: ContextMode[];
 export type Assertion = {
     output_contains: string;
 } | {
@@ -77,7 +78,7 @@ export interface ConfigFile {
     website_url?: string;
     github_url?: string;
     providers?: string[];
-    help_modes?: string[];
+    context?: string[];
     system_prompt?: string;
     concurrency?: number;
     workdir?: string;
@@ -115,7 +116,7 @@ export interface ModelResult {
     provider: Provider;
     modelId: string;
     displayName: string;
-    helpMode: HelpMode;
+    contextMode: ContextMode;
     taskResults: TaskEval[];
     passRate: number;
     avgTurnsToSuccess: number;
@@ -162,6 +163,7 @@ export interface GridReport {
     repository?: string;
     tags?: string[];
     taskSuiteHash?: string;
+    taskSuiteContent?: string;
     thresholdResults?: ThresholdCheckResult;
 }
 export interface HelpCache {

package/dist/models.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"models.d.ts","sourceRoot":"","sources":["../src/models.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAMH,MAAM,MAAM,QAAQ,GAAG,UAAU,GAAG,cAAc,GAAG,MAAM,CAAC;AAM5D,MAAM,MAAM,SAAS,GACjB;IAAE,eAAe,EAAE,MAAM,CAAA;CAAE,GAC3B;IAAE,aAAa,EAAE,MAAM,CAAA;CAAE,GACzB;IAAE,cAAc,EAAE,MAAM,CAAA;CAAE,GAC1B;IAAE,SAAS,EAAE,MAAM,CAAA;CAAE,GACrB;IAAE,WAAW,EAAE,MAAM,CAAA;CAAE,GACvB;IAAE,aAAa,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAA;CAAE,GACjD;IAAE,GAAG,EAAE,MAAM,CAAA;CAAE,GACf;IAAE,OAAO,EAAE,MAAM,CAAA;CAAE,GACnB;IAAE,SAAS,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,GAAG,CAAC,EAAE,MAAM,CAAC;QAAC,GAAG,CAAC,EAAE,MAAM,CAAA;KAAE,CAAA;CAAE,GAC9D;IAAE,MAAM,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,eAAe,CAAC,EAAE,MAAM,CAAC;QAAC,aAAa,CAAC,EAAE,MAAM,CAAA;KAAE,CAAA;CAAE,CAAC;AAElF,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,SAAS,CAAC;IACrB,MAAM,EAAE,OAAO,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,IAAI;IACnB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,SAAS,EAAE,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,MAAM,CAAC;IACxC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,SAAS;IACxB,GAAG,EAAE,MAAM,CAAC;IACZ,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,IAAI,EAAE,CAAC;CACf;AAMD,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,QAAQ,GAAG,OAAO,CAAC;AAErD,MAAM,WAAW,gBAAgB;IAC/B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,OAAO,GAAG,eAAe,CAAC;CACtC;AAED,MAAM,WAAW,UAAU;IACzB,GAAG,EAAE,MAAM,CAAC;IACZ,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,UAAU,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B,KAAK,EAAE,CAAC,IAAI,GAAG,MAAM,CAAC,EAAE,CAAC;CAC1B;AAMD,MAAM,MAAM,QAAQ,GAAG,MAAM,CAAC;AAE9B,MAAM,WAAW,WAAW;IAC1B,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;CACnB;AAMD,MAAM,WAAW,QAAQ;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,OAAO,CAAC;IAChB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,gBAAgB,EAAE,eAAe,EAAE,CAAC;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,UAAU,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,MAAM,CAAC;IACxC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,iBAAiB,CAAC,EAAE,OAAO,EAAE,CAAC;IAC9B,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACzC,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAMD,MAAM,WAAW,WAAW;IAC1B,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,QAAQ,CAAC;IACnB,WAAW,EAAE,QAAQ,EAAE,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,OAAO,CAAC;CACjB;AAED,MAAM,WAAW,oBAAoB;IACnC,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,EAAE,eAAe,EAAE,CAAC;IAC3B,QAAQ,EAAE,OAAO,GAAG,eAAe,CAAC;CACrC;AAED,MAAM,WAAW,UAAU;IACzB,GAAG,EAAE,MAAM,CAAC;IACZ,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,gBAAgB,EAAE,MAAM,CAAC;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,WAAW,EAAE,CAAC;IAC5B,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,gBAAgB,CAAC,EAAE,oBAAoB,CAAC;CACzC;AAMD,MAAM,WAAW,SAAS;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACnC,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB"}
+{"version":3,"file":"models.d.ts","sourceRoot":"","sources":["../src/models.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAMH,MAAM,MAAM,WAAW,GAAG,WAAW,GAAG,MAAM,GAAG,MAAM,CAAC;AACxD,eAAO,MAAM,aAAa,EAAE,WAAW,EAAkC,CAAC;AAM1E,MAAM,MAAM,SAAS,GACjB;IAAE,eAAe,EAAE,MAAM,CAAA;CAAE,GAC3B;IAAE,aAAa,EAAE,MAAM,CAAA;CAAE,GACzB;IAAE,cAAc,EAAE,MAAM,CAAA;CAAE,GAC1B;IAAE,SAAS,EAAE,MAAM,CAAA;CAAE,GACrB;IAAE,WAAW,EAAE,MAAM,CAAA;CAAE,GACvB;IAAE,aAAa,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAA;CAAE,GACjD;IAAE,GAAG,EAAE,MAAM,CAAA;CAAE,GACf;IAAE,OAAO,EAAE,MAAM,CAAA;CAAE,GACnB;IAAE,SAAS,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,GAAG,CAAC,EAAE,MAAM,CAAC;QAAC,GAAG,CAAC,EAAE,MAAM,CAAA;KAAE,CAAA;CAAE,GAC9D;IAAE,MAAM,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,eAAe,CAAC,EAAE,MAAM,CAAC;QAAC,aAAa,CAAC,EAAE,MAAM,CAAA;KAAE,CAAA;CAAE,CAAC;AAElF,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,SAAS,CAAC;IACrB,MAAM,EAAE,OAAO,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,IAAI;IACnB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,SAAS,EAAE,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,MAAM,CAAC;IACxC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,SAAS;IACxB,GAAG,EAAE,MAAM,CAAC;IACZ,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,IAAI,EAAE,CAAC;CACf;AAMD,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,QAAQ,GAAG,OAAO,CAAC;AAErD,MAAM,WAAW,gBAAgB;IAC/B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,OAAO,GAAG,eAAe,CAAC;CACtC;AAED,MAAM,WAAW,UAAU;IACzB,GAAG,EAAE,MAAM,CAAC;IACZ,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,UAAU,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B,KAAK,EAAE,CAAC,IAAI,GAAG,MAAM,CAAC,EAAE,CAAC;CAC1B;AAMD,MAAM,MAAM,QAAQ,GAAG,MAAM,CAAC;AAE9B,MAAM,WAAW,WAAW;IAC1B,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;CACnB;AAMD,MAAM,WAAW,QAAQ;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,OAAO,CAAC;IAChB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,gBAAgB,EAAE,eAAe,EAAE,CAAC;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,UAAU,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,MAAM,CAAC;IACxC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,iBAAiB,CAAC,EAAE,OAAO,EAAE,CAAC;IAC9B,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACzC,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAMD,MAAM,WAAW,WAAW;IAC1B,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,WAAW,CAAC;IACzB,WAAW,EAAE,QAAQ,EAAE,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,OAAO,CAAC;CACjB;AAED,MAAM,WAAW,oBAAoB;IACnC,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,EAAE,eAAe,EAAE,CAAC;IAC3B,QAAQ,EAAE,OAAO,GAAG,eAAe,CAAC;CACrC;AAED,MAAM,WAAW,UAAU;IACzB,GAAG,EAAE,MAAM,CAAC;IACZ,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,gBAAgB,EAAE,MAAM,CAAC;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,WAAW,EAAE,CAAC;IAC5B,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,gBAAgB,CAAC,EAAE,oBAAoB,CAAC;CACzC;AAMD,MAAM,WAAW,SAAS;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACnC,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB"}

package/dist/models.js

@@ -4,5 +4,5 @@
  * Assertion-based evaluation: tasks define assertions that are checked
  * against the agent's execution trace.
  */
-export {};
+export const CONTEXT_MODES = ['zero-shot', 'help', 'docs'];
 //# sourceMappingURL=models.js.map

package/dist/models.js.map

@@ -1 +1 @@
-{"version":3,"file":"models.js","sourceRoot":"","sources":["../src/models.ts"],"names":[],"mappings":"AAAA;;;;;GAKG"}
+{"version":3,"file":"models.js","sourceRoot":"","sources":["../src/models.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAOH,MAAM,CAAC,MAAM,aAAa,GAAkB,CAAC,WAAW,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC"}

package/dist/project.d.ts

@@ -22,5 +22,6 @@ export declare function resolveTaskRefs(tasks: (Task | string)[], baseDir: strin
 export declare function loadProject(configPath: string): Promise<{
     config: ConfigFile;
     tasks: Task[];
+    taskSuiteContent: string;
 }>;
 //# sourceMappingURL=project.d.ts.map

package/dist/project.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"project.d.ts","sourceRoot":"","sources":["../src/project.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAOH,OAAO,KAAK,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAIpD;;GAEG;AACH,wBAAsB,iBAAiB,CAAC,YAAY,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAqBrF;AAED;;GAEG;AACH,wBAAsB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAItE;AAED;;;GAGG;AACH,wBAAsB,eAAe,CACnC,KAAK,EAAE,CAAC,IAAI,GAAG,MAAM,CAAC,EAAE,EACxB,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,IAAI,EAAE,CAAC,CA6CjB;AAED;;GAEG;AACH,wBAAsB,WAAW,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC;IAAE,MAAM,EAAE,UAAU,CAAC;IAAC,KAAK,EAAE,IAAI,EAAE,CAAA;CAAE,CAAC,CAKpG"}
+{"version":3,"file":"project.d.ts","sourceRoot":"","sources":["../src/project.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAOH,OAAO,KAAK,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAIpD;;GAEG;AACH,wBAAsB,iBAAiB,CAAC,YAAY,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAqBrF;AAED;;GAEG;AACH,wBAAsB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAItE;AAED;;;GAGG;AACH,wBAAsB,eAAe,CACnC,KAAK,EAAE,CAAC,IAAI,GAAG,MAAM,CAAC,EAAE,EACxB,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,IAAI,EAAE,CAAC,CA6CjB;AAED;;GAEG;AACH,wBAAsB,WAAW,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC;IAAE,MAAM,EAAE,UAAU,CAAC;IAAC,KAAK,EAAE,IAAI,EAAE,CAAC;IAAC,gBAAgB,EAAE,MAAM,CAAA;CAAE,CAAC,CAM9H"}

package/dist/project.js

@@ -5,7 +5,7 @@
 import { readFile, access } from 'node:fs/promises';
 import { join, dirname, resolve } from 'node:path';
 import { glob } from 'node:fs/promises';
-import { parse as parseYaml } from 'yaml';
+import { parse as parseYaml, stringify as stringifyYaml } from 'yaml';
 import { ConfigFileSchema, TaskFileSchema, TaskSchema } from './schemas.js';
 const CONFIG_FILENAMES = ['cli-bench.yaml', 'cli-bench.yml'];
 /**
@@ -96,6 +96,7 @@ export async function loadProject(configPath) {
     const config = await loadConfigFile(configPath);
     const baseDir = dirname(configPath);
     const tasks = await resolveTaskRefs(config.tasks, baseDir);
-    return { config, tasks };
+    const taskSuiteContent = stringifyYaml(tasks);
+    return { config, tasks, taskSuiteContent };
 }
 //# sourceMappingURL=project.js.map

package/dist/project.js.map

@@ -1 +1 @@
-{"version":3,"file":"project.js","sourceRoot":"","sources":["../src/project.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AACxC,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,MAAM,CAAC;AAC1C,OAAO,EAAE,gBAAgB,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAG5E,MAAM,gBAAgB,GAAG,CAAC,gBAAgB,EAAE,eAAe,CAAC,CAAC;AAE7D;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,YAAqB;IAC3D,IAAI,YAAY,EAAE,CAAC;QACjB,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,YAAY,CAAC,CAAC;YAC3B,OAAO,OAAO,CAAC,YAAY,CAAC,CAAC;QAC/B,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,KAAK,MAAM,IAAI,IAAI,gBAAgB,EAAE,CAAC;QACpC,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;QAChC,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;YACxB,OAAO,SAAS,CAAC;QACnB,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,IAAY;IAC/C,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC1C,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;IAC9B,OAAO,gBAAgB,CAAC,KAAK,CAAC,MAAM,CAAe,CAAC;AACtD,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,KAAwB,EACxB,OAAe;IAEf,MAAM,QAAQ,GAAW,EAAE,CAAC;IAC5B,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAElC,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;QAC1B,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC9B,oBAAoB;YACpB,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;YAC5C,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;YAEnC,gCAAgC;YAChC,MAAM,KAAK,GAAa,EAAE,CAAC;YAC3B,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBACtB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;oBACxC,IAAI,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;wBACtD,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBACpB,CAAC;gBACH,CAAC;gBACD,KAAK,CAAC,IAAI,EAAE,CAAC;YACf,CAAC;iBAAM,CAAC;gBACN,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACtB,CAAC;YAED,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;gBAC7B,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;gBAC9C,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;gBAC9B,MAAM,SAAS,GAAG,cAAc,CAAC,KAAK,CAAC,MAAM,CAAW,CAAC;gBACzD,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;oBAC7B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;wBAC1B,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;wBACrB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBACtB,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;aAAM,CAAC;YACN,wCAAwC;YACxC,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,KAAK,CAAS,CAAC;YAC7C,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;gBAC1B,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBACrB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,UAAkB;IAClD,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,UAAU,CAAC,CAAC;IAChD,MAAM,OAAO,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;IACpC,MAAM,KAAK,GAAG,MAAM,eAAe,CAAC,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAC3D,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;AAC3B,CAAC"}
+{"version":3,"file":"project.js","sourceRoot":"","sources":["../src/project.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AACxC,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,SAAS,IAAI,aAAa,EAAE,MAAM,MAAM,CAAC;AACtE,OAAO,EAAE,gBAAgB,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAG5E,MAAM,gBAAgB,GAAG,CAAC,gBAAgB,EAAE,eAAe,CAAC,CAAC;AAE7D;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,YAAqB;IAC3D,IAAI,YAAY,EAAE,CAAC;QACjB,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,YAAY,CAAC,CAAC;YAC3B,OAAO,OAAO,CAAC,YAAY,CAAC,CAAC;QAC/B,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,KAAK,MAAM,IAAI,IAAI,gBAAgB,EAAE,CAAC;QACpC,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;QAChC,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;YACxB,OAAO,SAAS,CAAC;QACnB,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,IAAY;IAC/C,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC1C,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;IAC9B,OAAO,gBAAgB,CAAC,KAAK,CAAC,MAAM,CAAe,CAAC;AACtD,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,KAAwB,EACxB,OAAe;IAEf,MAAM,QAAQ,GAAW,EAAE,CAAC;IAC5B,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAElC,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;QAC1B,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC9B,oBAAoB;YACpB,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;YAC5C,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;YAEnC,gCAAgC;YAChC,MAAM,KAAK,GAAa,EAAE,CAAC;YAC3B,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBACtB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;oBACxC,IAAI,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;wBACtD,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBACpB,CAAC;gBACH,CAAC;gBACD,KAAK,CAAC,IAAI,EAAE,CAAC;YACf,CAAC;iBAAM,CAAC;gBACN,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACtB,CAAC;YAED,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;gBAC7B,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;gBAC9C,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;gBAC9B,MAAM,SAAS,GAAG,cAAc,CAAC,KAAK,CAAC,MAAM,CAAW,CAAC;gBACzD,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;oBAC7B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;wBAC1B,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;wBACrB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBACtB,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;aAAM,CAAC;YACN,wCAAwC;YACxC,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,KAAK,CAAS,CAAC;YAC7C,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;gBAC1B,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBACrB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,UAAkB;IAClD,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,UAAU,CAAC,CAAC;IAChD,MAAM,OAAO,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;IACpC,MAAM,KAAK,GAAG,MAAM,eAAe,CAAC,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAC3D,MAAM,gBAAgB,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IAC9C,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,gBAAgB,EAAE,CAAC;AAC7C,CAAC"}

package/dist/prompt.d.ts

@@ -1,12 +1,13 @@
 /**
- * Builds prompts from help text + task definition.
+ * Builds prompts for CLI benchmark agents.
  *
- * Three help modes:
- * - injected: help text included in prompt (current behavior)
- * - discoverable: agent must run --help to discover commands
- * - none: agent relies on training knowledge only
+ * One unified system message for all context modes.
+ * The user message varies by context mode:
+ * - zero-shot: CLI name + task intent only
+ * - help: CLI name + top-level --help output + task intent
+ * - docs: CLI name + documentation contents + task intent
  */
-import type { HelpMode, Task } from './models.js';
-export declare function buildSystemMessage(helpMode: HelpMode, customPrompt?: string): string;
-export declare function buildUserMessage(cliName: string, helpTexts: Record<string, string> | null, task: Task, helpMode: HelpMode): string;
+import type { ContextMode, Task } from './models.js';
+export declare function buildSystemMessage(customPrompt?: string): string;
+export declare function buildUserMessage(cliName: string, task: Task, contextMode: ContextMode, contextPayload?: string | null): string;
 //# sourceMappingURL=prompt.d.ts.map

package/dist/prompt.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"prompt.d.ts","sourceRoot":"","sources":["../src/prompt.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AA6BlD,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,QAAQ,EAAE,YAAY,CAAC,EAAE,MAAM,GAAG,MAAM,CAkBpF;AAED,wBAAgB,gBAAgB,CAC9B,OAAO,EAAE,MAAM,EACf,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,EACxC,IAAI,EAAE,IAAI,EACV,QAAQ,EAAE,QAAQ,GACjB,MAAM,CAaR"}
+{"version":3,"file":"prompt.d.ts","sourceRoot":"","sources":["../src/prompt.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AASrD,wBAAgB,kBAAkB,CAAC,YAAY,CAAC,EAAE,MAAM,GAAG,MAAM,CAKhE;AAED,wBAAgB,gBAAgB,CAC9B,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,IAAI,EACV,WAAW,EAAE,WAAW,EACxB,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,GAC7B,MAAM,CA+BR"}

package/dist/prompt.js

@@ -1,96 +1,52 @@
 /**
- * Builds prompts from help text + task definition.
+ * Builds prompts for CLI benchmark agents.
  *
- * Three help modes:
- * - injected: help text included in prompt (current behavior)
- * - discoverable: agent must run --help to discover commands
- * - none: agent relies on training knowledge only
+ * One unified system message for all context modes.
+ * The user message varies by context mode:
+ * - zero-shot: CLI name + task intent only
+ * - help: CLI name + top-level --help output + task intent
+ * - docs: CLI name + documentation contents + task intent
  */
 const AGENT_SYSTEM_MESSAGE = `You are a CLI expert. You have a run_command tool to execute shell commands. Use it to accomplish the user's task.
 
 Rules:
-- Read the help text carefully to understand available subcommands and flags
 - Execute commands using the run_command tool
 - If a command fails, read the error and retry with corrected flags
-- Do NOT invent flags that don't exist in the help text
 - When the task is complete, stop calling tools`;
-const AGENT_SYSTEM_MESSAGE_DISCOVERABLE = `You are a CLI expert. You have a run_command tool to execute shell commands. Use it to accomplish the user's task.
-
-Rules:
-- Use <cli> --help and <cli> <subcommand> --help to discover available commands and flags
-- Execute commands using the run_command tool
-- If a command fails, read the error and retry with corrected flags
-- Do NOT invent flags — always check --help first
-- When the task is complete, stop calling tools`;
-const AGENT_SYSTEM_MESSAGE_NONE = `You are a CLI expert. You have a run_command tool to execute shell commands. Use it to accomplish the user's task.
-
-Rules:
-- Use your training knowledge of CLI tools to construct commands
-- Execute commands using the run_command tool
-- If a command fails, read the error and retry with corrected flags
-- Do NOT run --help commands — rely on your knowledge
-- When the task is complete, stop calling tools`;
-export function buildSystemMessage(helpMode, customPrompt) {
-    let base;
-    switch (helpMode) {
-        case 'injected':
-            base = AGENT_SYSTEM_MESSAGE;
-            break;
-        case 'discoverable':
-            base = AGENT_SYSTEM_MESSAGE_DISCOVERABLE;
-            break;
-        case 'none':
-            base = AGENT_SYSTEM_MESSAGE_NONE;
-            break;
-    }
+export function buildSystemMessage(customPrompt) {
     if (customPrompt) {
-        return `${base}\n\n${customPrompt}`;
+        return `${AGENT_SYSTEM_MESSAGE}\n\n${customPrompt}`;
     }
-    return base;
+    return AGENT_SYSTEM_MESSAGE;
 }
-export function buildUserMessage(cliName, helpTexts, task, helpMode) {
-    if (helpMode === 'injected' && helpTexts) {
-        const relevantHelp = selectRelevantHelp(cliName, helpTexts, task);
-        return `CLI: ${cliName}
+export function buildUserMessage(cliName, task, contextMode, contextPayload) {
+    switch (contextMode) {
+        case 'help':
+            if (contextPayload) {
+                return `CLI: ${cliName}
 
 Help text:
-${relevantHelp}
+$ ${cliName} --help
+${contextPayload}
 
 Task: ${task.intent}`;
-    }
-    return `CLI: ${cliName}
+            }
+            // Fall through to zero-shot if no help available
+            return `CLI: ${cliName}\n\nTask: ${task.intent}`;
+        case 'docs':
+            if (contextPayload) {
+                return `CLI: ${cliName}
+
+Documentation:
+${contextPayload}
 
 Task: ${task.intent}`;
-}
-/**
- * Select the most relevant help text sections for a task.
- * Trims to stay within reasonable context limits (~4K chars).
- */
-function selectRelevantHelp(cliName, helpTexts, task) {
-    const sections = [];
-    const maxChars = 4000;
-    let totalChars = 0;
-    // Always include root help
-    const rootHelp = helpTexts[''];
-    if (rootHelp) {
-        sections.push(`$ ${cliName} --help\n${rootHelp}`);
-        totalChars += rootHelp.length;
-    }
-    // Add help sections that match task keywords
-    if (totalChars < maxChars * 0.7) {
-        const taskWords = task.intent.toLowerCase().split(/\s+/);
-        for (const [key, help] of Object.entries(helpTexts)) {
-            if (key === '' || sections.some((s) => s.includes(`${cliName} ${key} --help`))) {
-                continue;
-            }
-            const keyWords = key.split(' ');
-            const matches = keyWords.some((kw) => taskWords.includes(kw));
-            if (matches && totalChars + help.length < maxChars) {
-                sections.push(`$ ${cliName} ${key} --help\n${help}`);
-                totalChars += help.length;
             }
-        }
+            // Fall through to zero-shot if no docs available
+            return `CLI: ${cliName}\n\nTask: ${task.intent}`;
+        case 'zero-shot':
+        default:
+            return `CLI: ${cliName}\n\nTask: ${task.intent}`;
     }
-    return sections.join('\n\n');
 }
 //# sourceMappingURL=prompt.js.map

package/dist/prompt.js.map

@@ -1 +1 @@
-{"version":3,"file":"prompt.js","sourceRoot":"","sources":["../src/prompt.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAIH,MAAM,oBAAoB,GAAG;;;;;;;gDAOmB,CAAC;AAEjD,MAAM,iCAAiC,GAAG;;;;;;;gDAOM,CAAC;AAEjD,MAAM,yBAAyB,GAAG;;;;;;;gDAOc,CAAC;AAEjD,MAAM,UAAU,kBAAkB,CAAC,QAAkB,EAAE,YAAqB;IAC1E,IAAI,IAAY,CAAC;IACjB,QAAQ,QAAQ,EAAE,CAAC;QACjB,KAAK,UAAU;YACb,IAAI,GAAG,oBAAoB,CAAC;YAC5B,MAAM;QACR,KAAK,cAAc;YACjB,IAAI,GAAG,iCAAiC,CAAC;YACzC,MAAM;QACR,KAAK,MAAM;YACT,IAAI,GAAG,yBAAyB,CAAC;YACjC,MAAM;IACV,CAAC;IAED,IAAI,YAAY,EAAE,CAAC;QACjB,OAAO,GAAG,IAAI,OAAO,YAAY,EAAE,CAAC;IACtC,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,gBAAgB,CAC9B,OAAe,EACf,SAAwC,EACxC,IAAU,EACV,QAAkB;IAElB,IAAI,QAAQ,KAAK,UAAU,IAAI,SAAS,EAAE,CAAC;QACzC,MAAM,YAAY,GAAG,kBAAkB,CAAC,OAAO,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;QAClE,OAAO,QAAQ,OAAO;;;EAGxB,YAAY;;QAEN,IAAI,CAAC,MAAM,EAAE,CAAC;IACpB,CAAC;IACD,OAAO,QAAQ,OAAO;;QAEhB,IAAI,CAAC,MAAM,EAAE,CAAC;AACtB,CAAC;AAED;;;GAGG;AACH,SAAS,kBAAkB,CACzB,OAAe,EACf,SAAiC,EACjC,IAAU;IAEV,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,MAAM,QAAQ,GAAG,IAAI,CAAC;IACtB,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,2BAA2B;IAC3B,MAAM,QAAQ,GAAG,SAAS,CAAC,EAAE,CAAC,CAAC;IAC/B,IAAI,QAAQ,EAAE,CAAC;QACb,QAAQ,CAAC,IAAI,CAAC,KAAK,OAAO,YAAY,QAAQ,EAAE,CAAC,CAAC;QAClD,UAAU,IAAI,QAAQ,CAAC,MAAM,CAAC;IAChC,CAAC;IAED,6CAA6C;IAC7C,IAAI,UAAU,GAAG,QAAQ,GAAG,GAAG,EAAE,CAAC;QAChC,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACzD,KAAK,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;YACpD,IAAI,GAAG,KAAK,EAAE,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,OAAO,IAAI,GAAG,SAAS,CAAC,CAAC,EAAE,CAAC;gBAC/E,SAAS;YACX,CAAC;YACD,MAAM,QAAQ,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAChC,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;YAC9D,IAAI,OAAO,IAAI,UAAU,GAAG,IAAI,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC;gBACnD,QAAQ,CAAC,IAAI,CAAC,KAAK,OAAO,IAAI,GAAG,YAAY,IAAI,EAAE,CAAC,CAAC;gBACrD,UAAU,IAAI,IAAI,CAAC,MAAM,CAAC;YAC5B,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC/B,CAAC"}
+{"version":3,"file":"prompt.js","sourceRoot":"","sources":["../src/prompt.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAIH,MAAM,oBAAoB,GAAG;;;;;gDAKmB,CAAC;AAEjD,MAAM,UAAU,kBAAkB,CAAC,YAAqB;IACtD,IAAI,YAAY,EAAE,CAAC;QACjB,OAAO,GAAG,oBAAoB,OAAO,YAAY,EAAE,CAAC;IACtD,CAAC;IACD,OAAO,oBAAoB,CAAC;AAC9B,CAAC;AAED,MAAM,UAAU,gBAAgB,CAC9B,OAAe,EACf,IAAU,EACV,WAAwB,EACxB,cAA8B;IAE9B,QAAQ,WAAW,EAAE,CAAC;QACpB,KAAK,MAAM;YACT,IAAI,cAAc,EAAE,CAAC;gBACnB,OAAO,QAAQ,OAAO;;;IAG1B,OAAO;EACT,cAAc;;QAER,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,CAAC;YACD,iDAAiD;YACjD,OAAO,QAAQ,OAAO,aAAa,IAAI,CAAC,MAAM,EAAE,CAAC;QAEnD,KAAK,MAAM;YACT,IAAI,cAAc,EAAE,CAAC;gBACnB,OAAO,QAAQ,OAAO;;;EAG5B,cAAc;;QAER,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,CAAC;YACD,iDAAiD;YACjD,OAAO,QAAQ,OAAO,aAAa,IAAI,CAAC,MAAM,EAAE,CAAC;QAEnD,KAAK,WAAW,CAAC;QACjB;YACE,OAAO,QAAQ,OAAO,aAAa,IAAI,CAAC,MAAM,EAAE,CAAC;IACrD,CAAC;AACH,CAAC"}

package/dist/providers.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"providers.d.ts","sourceRoot":"","sources":["../src/providers.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,eAAO,MAAM,MAAM,EAAE,SAAS,UAAU,EAa9B,CAAC;AAEX,wBAAgB,QAAQ,CAAC,OAAO,EAAE,MAAM,8CAEvC;AAED,wBAAgB,aAAa,CAAC,OAAO,EAAE,MAAM,GAAG,UAAU,GAAG,SAAS,CAErE;AAED,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,UAAU,EAAE,CAG7D;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,WAAW,EAAE,MAAM,EAAE,GAAG,UAAU,EAAE,CAWpE;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,IAAI,IAAI,CAMzC"}
+{"version":3,"file":"providers.d.ts","sourceRoot":"","sources":["../src/providers.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,eAAO,MAAM,MAAM,EAAE,SAAS,UAAU,EAc9B,CAAC;AAEX,wBAAgB,QAAQ,CAAC,OAAO,EAAE,MAAM,8CAEvC;AAED,wBAAgB,aAAa,CAAC,OAAO,EAAE,MAAM,GAAG,UAAU,GAAG,SAAS,CAErE;AAED,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,UAAU,EAAE,CAG7D;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,WAAW,EAAE,MAAM,EAAE,GAAG,UAAU,EAAE,CAWpE;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,IAAI,IAAI,CAMzC"}

package/dist/providers.js

@@ -7,6 +7,7 @@
 import { gateway } from 'ai';
 export const MODELS = [
     { id: 'anthropic/claude-opus-4.6', displayName: 'Claude Opus 4.6', provider: 'anthropic' },
+    { id: 'anthropic/claude-sonnet-4.6', displayName: 'Claude Sonnet 4.6', provider: 'anthropic' },
     { id: 'anthropic/claude-sonnet-4-20250514', displayName: 'Claude Sonnet 4', provider: 'anthropic' },
     { id: 'anthropic/claude-haiku-4.5', displayName: 'Claude Haiku 4.5', provider: 'anthropic' },
     { id: 'anthropic/claude-haiku-4-5-20251001', displayName: 'Claude Haiku 4.5', provider: 'anthropic' },

package/dist/providers.js.map

@@ -1 +1 @@
-{"version":3,"file":"providers.js","sourceRoot":"","sources":["../src/providers.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,IAAI,CAAC;AAQ7B,MAAM,CAAC,MAAM,MAAM,GAA0B;IAC3C,EAAE,EAAE,EAAE,2BAA2B,EAAE,WAAW,EAAE,iBAAiB,EAAE,QAAQ,EAAE,WAAW,EAAE;IAC1F,EAAE,EAAE,EAAE,oCAAoC,EAAE,WAAW,EAAE,iBAAiB,EAAE,QAAQ,EAAE,WAAW,EAAE;IACnG,EAAE,EAAE,EAAE,4BAA4B,EAAE,WAAW,EAAE,kBAAkB,EAAE,QAAQ,EAAE,WAAW,EAAE;IAC5F,EAAE,EAAE,EAAE,qCAAqC,EAAE,WAAW,EAAE,kBAAkB,EAAE,QAAQ,EAAE,WAAW,EAAE;IACrG,EAAE,EAAE,EAAE,gBAAgB,EAAE,WAAW,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,EAAE;IACpE,EAAE,EAAE,EAAE,eAAe,EAAE,WAAW,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE;IAClE,EAAE,EAAE,EAAE,oBAAoB,EAAE,WAAW,EAAE,aAAa,EAAE,QAAQ,EAAE,QAAQ,EAAE;IAC5E,EAAE,EAAE,EAAE,uBAAuB,EAAE,WAAW,EAAE,gBAAgB,EAAE,QAAQ,EAAE,QAAQ,EAAE;IAClF,EAAE,EAAE,EAAE,uBAAuB,EAAE,WAAW,EAAE,gBAAgB,EAAE,QAAQ,EAAE,QAAQ,EAAE;IAClF,EAAE,EAAE,EAAE,yBAAyB,EAAE,WAAW,EAAE,kBAAkB,EAAE,QAAQ,EAAE,QAAQ,EAAE;IACtF,EAAE,EAAE,EAAE,mBAAmB,EAAE,WAAW,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,EAAE;IAC1E,EAAE,EAAE,EAAE,sBAAsB,EAAE,WAAW,EAAE,cAAc,EAAE,QAAQ,EAAE,SAAS,EAAE;CACxE,CAAC;AAEX,MAAM,UAAU,QAAQ,CAAC,OAAe;IACtC,OAAO,OAAO,CAAC,OAAO,CAAC,CAAC;AAC1B,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,OAAe;IAC3C,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,OAAO,CAAC,CAAC;AAC9C,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,QAAkB;IAC7C,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,GAAG,MAAM,CAAC,CAAC;IAC9C,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AACvD,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,WAAqB;IACpD,OAAO,WAAW,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;QAC5B,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;QACjD,IAAI,QAAQ;YAAE,OAAO,EAAE,GAAG,QAAQ,EAAE,CAAC;QACrC,MAAM,CAAC,QAAQ,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC1C,OAAO;YACL,EAAE;YACF,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;YAC3B,QAAQ,EAAE,QAAkC;SAC7C,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,kBAAkB;IAChC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,CAAC;QACvC,MAAM,IAAI,KAAK,CACb,2IAA2I,CAC5I,CAAC;IACJ,CAAC;AACH,CAAC"}
+{"version":3,"file":"providers.js","sourceRoot":"","sources":["../src/providers.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,IAAI,CAAC;AAQ7B,MAAM,CAAC,MAAM,MAAM,GAA0B;IAC3C,EAAE,EAAE,EAAE,2BAA2B,EAAE,WAAW,EAAE,iBAAiB,EAAE,QAAQ,EAAE,WAAW,EAAE;IAC1F,EAAE,EAAE,EAAE,6BAA6B,EAAE,WAAW,EAAE,mBAAmB,EAAE,QAAQ,EAAE,WAAW,EAAE;IAC9F,EAAE,EAAE,EAAE,oCAAoC,EAAE,WAAW,EAAE,iBAAiB,EAAE,QAAQ,EAAE,WAAW,EAAE;IACnG,EAAE,EAAE,EAAE,4BAA4B,EAAE,WAAW,EAAE,kBAAkB,EAAE,QAAQ,EAAE,WAAW,EAAE;IAC5F,EAAE,EAAE,EAAE,qCAAqC,EAAE,WAAW,EAAE,kBAAkB,EAAE,QAAQ,EAAE,WAAW,EAAE;IACrG,EAAE,EAAE,EAAE,gBAAgB,EAAE,WAAW,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,EAAE;IACpE,EAAE,EAAE,EAAE,eAAe,EAAE,WAAW,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE;IAClE,EAAE,EAAE,EAAE,oBAAoB,EAAE,WAAW,EAAE,aAAa,EAAE,QAAQ,EAAE,QAAQ,EAAE;IAC5E,EAAE,EAAE,EAAE,uBAAuB,EAAE,WAAW,EAAE,gBAAgB,EAAE,QAAQ,EAAE,QAAQ,EAAE;IAClF,EAAE,EAAE,EAAE,uBAAuB,EAAE,WAAW,EAAE,gBAAgB,EAAE,QAAQ,EAAE,QAAQ,EAAE;IAClF,EAAE,EAAE,EAAE,yBAAyB,EAAE,WAAW,EAAE,kBAAkB,EAAE,QAAQ,EAAE,QAAQ,EAAE;IACtF,EAAE,EAAE,EAAE,mBAAmB,EAAE,WAAW,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,EAAE;IAC1E,EAAE,EAAE,EAAE,sBAAsB,EAAE,WAAW,EAAE,cAAc,EAAE,QAAQ,EAAE,SAAS,EAAE;CACxE,CAAC;AAEX,MAAM,UAAU,QAAQ,CAAC,OAAe;IACtC,OAAO,OAAO,CAAC,OAAO,CAAC,CAAC;AAC1B,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,OAAe;IAC3C,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,OAAO,CAAC,CAAC;AAC9C,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,QAAkB;IAC7C,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,GAAG,MAAM,CAAC,CAAC;IAC9C,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AACvD,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,WAAqB;IACpD,OAAO,WAAW,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;QAC5B,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;QACjD,IAAI,QAAQ;YAAE,OAAO,EAAE,GAAG,QAAQ,EAAE,CAAC;QACrC,MAAM,CAAC,QAAQ,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC1C,OAAO;YACL,EAAE;YACF,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;YAC3B,QAAQ,EAAE,QAAkC;SAC7C,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,kBAAkB;IAChC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,CAAC;QACvC,MAAM,IAAI,KAAK,CACb,2IAA2I,CAC5I,CAAC;IACJ,CAAC;AACH,CAAC"}

package/dist/runner.d.ts

@@ -1,5 +1,5 @@
 /**
- * Orchestrates the task x model x helpMode matrix with tool-calling agents.
+ * Orchestrates the task x model x contextMode matrix with tool-calling agents.
  *
  * For each task:
  *   - Create temp workdir (or use configured one)
@@ -35,6 +35,10 @@ export interface RunGridOptions {
     category?: string;
     websiteUrl?: string;
     githubUrl?: string;
+    /** Serialized YAML of the resolved task suite. */
+    taskSuiteContent?: string;
+    /** Directory containing the config file (for resolving docs.md). */
+    configDir?: string;
 }
 export declare function runGrid(opts: RunGridOptions): Promise<GridReport[]>;
 export declare function uploadReport(report: GridReport, backendUrl: string, apiKey: string): Promise<void>;

package/dist/runner.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../src/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AASH,OAAO,KAAK,EACV,SAAS,EACT,IAAI,EACJ,UAAU,EAKX,MAAM,aAAa,CAAC;AACrB,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AASjD,wBAAsB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,CAKxE;AAED,wBAAsB,kBAAkB,CACtC,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAc9B;AAkND,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,8CAA8C;IAC9C,KAAK,CAAC,EAAE,IAAI,EAAE,CAAC;IACf,iDAAiD;IACjD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,+CAA+C;IAC/C,MAAM,CAAC,EAAE,UAAU,EAAE,CAAC;IACtB,wDAAwD;IACxD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,0DAA0D;IAC1D,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,uDAAuD;IACvD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,oDAAoD;IACpD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,mDAAmD;IACnD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,wBAAsB,OAAO,CAAC,IAAI,EAAE,cAAc,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC,CAkEzE;AA4ND,wBAAsB,YAAY,CAChC,MAAM,EAAE,UAAU,EAClB,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,IAAI,CAAC,CAsBf"}
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../src/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AASH,OAAO,KAAK,EACV,SAAS,EACT,IAAI,EACJ,UAAU,EAKX,MAAM,aAAa,CAAC;AACrB,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AASjD,wBAAsB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,CAKxE;AAED,wBAAsB,kBAAkB,CACtC,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAc9B;AAiOD,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,8CAA8C;IAC9C,KAAK,CAAC,EAAE,IAAI,EAAE,CAAC;IACf,iDAAiD;IACjD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,+CAA+C;IAC/C,MAAM,CAAC,EAAE,UAAU,EAAE,CAAC;IACtB,wDAAwD;IACxD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,0DAA0D;IAC1D,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,uDAAuD;IACvD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,oDAAoD;IACpD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,mDAAmD;IACnD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,kDAAkD;IAClD,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,oEAAoE;IACpE,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,wBAAsB,OAAO,CAAC,IAAI,EAAE,cAAc,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC,CAoEzE;AAkPD,wBAAsB,YAAY,CAChC,MAAM,EAAE,UAAU,EAClB,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,IAAI,CAAC,CAsBf"}

package/dist/runner.js

@@ -1,5 +1,5 @@
 /**
- * Orchestrates the task x model x helpMode matrix with tool-calling agents.
+ * Orchestrates the task x model x contextMode matrix with tool-calling agents.
  *
  * For each task:
  *   - Create temp workdir (or use configured one)
@@ -18,7 +18,7 @@ import { z } from 'zod';
 import { getModel, filterModels } from './providers.js';
 import { buildSystemMessage, buildUserMessage } from './prompt.js';
 import { runAssertions } from './assertions.js';
-import { loadHelpFromCache, loadHelpLive } from './help-loader.js';
+import { loadTopLevelHelp } from './help-loader.js';
 import { execCommand, runSetup } from './exec.js';
 import { TaskSuiteSchema } from './schemas.js';
 import { detectCI, computeTaskSuiteHash } from './ci.js';
@@ -143,10 +143,12 @@ async function runAgentTask(model, systemMessage, userMessage, task, workdir) {
             })),
             usage: step.usage,
         }));
+        // Cap turnsUsed at max_turns to guard against SDK retry anomalies
+        const maxTurns = task.max_turns ?? 5;
         return {
             passed: allPassed,
             assertionResults,
-            turnsUsed: commandsRun.length,
+            turnsUsed: Math.min(commandsRun.length, maxTurns),
             commandsRun,
             totalInputTokens,
             totalOutputTokens,
@@ -162,8 +164,8 @@ async function runAgentTask(model, systemMessage, userMessage, task, workdir) {
         return {
             passed: false,
             assertionResults: [],
-            turnsUsed: commandsRun.length,
-            commandsRun,
+            turnsUsed: 0,
+            commandsRun: [],
             totalInputTokens: 0,
             totalOutputTokens: 0,
             totalLatencyMs: Date.now() - start,
@@ -175,26 +177,36 @@ async function runAgentTask(model, systemMessage, userMessage, task, workdir) {
         };
     }
 }
-function aggregateModelResult(model, taskResults, helpMode) {
-    const passRate = taskResults.length > 0
-        ? taskResults.filter((t) => t.passed).length / taskResults.length
+/** An eval with 0 tokens is an infra error (gateway timeout, auth failure, etc.) — not a real task result. */
+function isInfraError(t) {
+    return t.totalInputTokens + t.totalOutputTokens === 0 && !t.passed;
+}
+function aggregateModelResult(model, taskResults, contextMode) {
+    // Exclude infra errors from pass rate — they're not real task failures
+    const scorable = taskResults.filter((t) => !isInfraError(t));
+    const infraErrors = taskResults.length - scorable.length;
+    if (infraErrors > 0) {
+        console.warn(`    ⚠ ${infraErrors} task(s) excluded from stats (infra error, 0 tokens)`);
+    }
+    const passRate = scorable.length > 0
+        ? scorable.filter((t) => t.passed).length / scorable.length
         : 0;
-    const avgLatencyMs = taskResults.length > 0
-        ? Math.round(taskResults.reduce((a, t) => a + t.totalLatencyMs, 0) / taskResults.length)
+    const avgLatencyMs = scorable.length > 0
+        ? Math.round(scorable.reduce((a, t) => a + t.totalLatencyMs, 0) / scorable.length)
         : 0;
-    const passedTasks = taskResults.filter((t) => t.passed);
+    const passedTasks = scorable.filter((t) => t.passed);
     const avgTurnsToSuccess = passedTasks.length > 0
         ? passedTasks.reduce((a, t) => a + t.turnsUsed, 0) / passedTasks.length
         : 0;
-    const avgTotalTokens = taskResults.length > 0
-        ? taskResults.reduce((a, t) => a + t.totalInputTokens + t.totalOutputTokens, 0) /
-            taskResults.length
+    const avgTotalTokens = scorable.length > 0
+        ? scorable.reduce((a, t) => a + t.totalInputTokens + t.totalOutputTokens, 0) /
+            scorable.length
         : 0;
     return {
         provider: model.provider,
         modelId: model.id,
         displayName: model.displayName,
-        helpMode,
+        contextMode,
         taskResults,
         passRate,
         avgTurnsToSuccess,
@@ -212,7 +224,7 @@ export async function runGrid(opts) {
             cliName: opts.cliName,
             tasks: opts.tasks,
             models: opts.models ?? filterModels(config.models),
-            helpModes: config.helpModes,
+            contextModes: config.contextModes,
             limiter,
             config,
             versionCommand: opts.versionCommand,
@@ -223,6 +235,8 @@ export async function runGrid(opts) {
             category: opts.category,
             websiteUrl: opts.websiteUrl,
             githubUrl: opts.githubUrl,
+            taskSuiteContent: opts.taskSuiteContent,
+            configDir: opts.configDir,
         });
         if (report)
             reports.push(report);
@@ -246,7 +260,7 @@ export async function runGrid(opts) {
             cliName,
             tasks: suite.tasks,
             models,
-            helpModes: config.helpModes,
+            contextModes: config.contextModes,
             limiter,
             config,
             versionCommand: suite.version_command,
@@ -260,32 +274,43 @@ export async function runGrid(opts) {
     }
     return reports;
 }
-async function runCliGrid(opts) {
-    const { cliName, tasks, models, helpModes, limiter, config, versionCommand, systemPrompt } = opts;
-    console.log(`\n=== ${cliName} ===`);
-    // Load help text (used for injected mode)
-    let helpTexts = null;
-    let cliVersion;
-    if (helpModes.includes('injected')) {
-        const helpCache = await loadHelpFromCache(config.helpCacheDir, cliName);
-        if (helpCache) {
-            helpTexts = helpCache.help_texts;
-            cliVersion = helpCache.version;
+/**
+ * Load context payload for a given mode.
+ */
+async function loadContextPayload(contextMode, cliName, configDir) {
+    switch (contextMode) {
+        case 'help': {
+            const helpText = await loadTopLevelHelp(cliName);
+            if (!helpText) {
+                console.warn(`  Warning: could not capture --help for ${cliName}, falling back to zero-shot`);
+            }
+            return helpText;
         }
-        else {
-            console.log(`  No cached help text, trying live capture...`);
+        case 'docs': {
+            if (!configDir) {
+                console.warn(`  Warning: no config directory for docs mode, falling back to zero-shot`);
+                return null;
+            }
             try {
-                const live = await loadHelpLive(cliName);
-                helpTexts = live.help_texts;
-                cliVersion = live.version;
+                const docsPath = join(configDir, 'docs.md');
+                return await readFile(docsPath, 'utf-8');
             }
             catch {
-                console.warn(`  Failed to capture help text for ${cliName} (injected mode may fail)`);
+                console.warn(`  Warning: docs.md not found in ${configDir}, falling back to zero-shot`);
+                return null;
             }
         }
+        case 'zero-shot':
+        default:
+            return null;
     }
+}
+async function runCliGrid(opts) {
+    const { cliName, tasks, models, contextModes, limiter, config, versionCommand, systemPrompt } = opts;
+    console.log(`\n=== ${cliName} ===`);
+    let cliVersion;
     // Detect CLI version
-    if (!cliVersion && versionCommand) {
+    if (versionCommand) {
         cliVersion = await detectCliVersion(versionCommand);
         if (cliVersion)
             console.log(`  CLI version: ${cliVersion}`);
@@ -294,11 +319,12 @@ async function runCliGrid(opts) {
     if (config.dryRun) {
         const firstTask = tasks[0];
         if (firstTask) {
-            for (const helpMode of helpModes) {
-                const systemMessage = buildSystemMessage(helpMode, systemPrompt);
-                const userMessage = buildUserMessage(cliName, helpTexts, firstTask, helpMode);
-                console.log(`\n--- Dry Run: ${cliName} / ${firstTask.id} (${helpMode}) ---`);
-                console.log(`System: ${systemMessage}\n`);
+            for (const contextMode of contextModes) {
+                const contextPayload = await loadContextPayload(contextMode, cliName, opts.configDir);
+                const sysMsg = buildSystemMessage(systemPrompt);
+                const userMessage = buildUserMessage(cliName, firstTask, contextMode, contextPayload);
+                console.log(`\n--- Dry Run: ${cliName} / ${firstTask.id} (${contextMode}) ---`);
+                console.log(`System: ${sysMsg}\n`);
                 console.log(`User:\n${userMessage}\n`);
                 console.log(`--- End Dry Run ---\n`);
             }
@@ -306,9 +332,10 @@ async function runCliGrid(opts) {
         return null;
     }
     const modelResults = [];
-    for (const helpMode of helpModes) {
-        console.log(`\n  Help mode: ${helpMode}`);
-        const systemMessage = buildSystemMessage(helpMode, systemPrompt);
+    for (const contextMode of contextModes) {
+        console.log(`\n  Context: ${contextMode}`);
+        const contextPayload = await loadContextPayload(contextMode, cliName, opts.configDir);
+        const sysMsg = buildSystemMessage(systemPrompt);
         for (const model of models) {
             console.log(`  Model: ${model.displayName}`);
             const taskResults = [];
@@ -324,11 +351,11 @@ async function runCliGrid(opts) {
                 const workdir = opts.workdir ?? config.workdir ?? await mkdtemp(join(tmpdir(), 'cli-bench-'));
                 const shouldCleanup = !opts.workdir && !config.workdir;
                 try {
-                    const userMessage = buildUserMessage(cliName, helpTexts, task, helpMode);
+                    const userMessage = buildUserMessage(cliName, task, contextMode, contextPayload);
                     if (task.setup && task.setup.length > 0) {
                         await runSetup(task.setup, { cwd: workdir });
                     }
-                    const agentResult = await runAgentTask(model, systemMessage, userMessage, task, workdir);
+                    const agentResult = await runAgentTask(model, sysMsg, userMessage, task, workdir);
                     const taskEval = {
                         taskId: task.id,
                         passed: agentResult.passed,
@@ -353,9 +380,10 @@ async function runCliGrid(opts) {
                         },
                         repeatIndex: repeatTotal > 1 ? repeatIndex : undefined,
                     };
-                    const icon = agentResult.passed ? '✓' : '✗';
+                    const totalTokens = agentResult.totalInputTokens + agentResult.totalOutputTokens;
+                    const icon = agentResult.passed ? '✓' : totalTokens === 0 ? '⚠' : '✗';
                     const repeatLabel = repeatTotal > 1 ? ` [${repeatIndex + 1}/${repeatTotal}]` : '';
-                    console.log(`    ${icon} ${task.id}${repeatLabel} (turns=${agentResult.turnsUsed}, tokens=${agentResult.totalInputTokens + agentResult.totalOutputTokens})`);
+                    console.log(`    ${icon} ${task.id}${repeatLabel} (turns=${agentResult.turnsUsed}, tokens=${totalTokens})`);
                     return taskEval;
                 }
                 finally {
@@ -368,7 +396,7 @@ async function runCliGrid(opts) {
             for (const taskEval of results) {
                 taskResults.push(taskEval);
             }
-            modelResults.push(aggregateModelResult(model, taskResults, helpMode));
+            modelResults.push(aggregateModelResult(model, taskResults, contextMode));
         }
     }
     const ciMetadata = detectCI();
@@ -391,13 +419,14 @@ async function runCliGrid(opts) {
         totalEvals,
         generatedAt: new Date().toISOString(),
         gridVersion: '0.4.0',
-        systemPrompt: buildSystemMessage(helpModes[0], systemPrompt),
+        systemPrompt: buildSystemMessage(systemPrompt),
         displayName: opts.displayName,
         category: opts.category,
         websiteUrl: opts.websiteUrl,
         githubUrl: opts.githubUrl,
         ...ciMetadata,
         taskSuiteHash,
+        taskSuiteContent: opts.taskSuiteContent,
     };
     printReportSummary(report);
     return report;
@@ -406,7 +435,7 @@ function printReportSummary(report) {
     console.log(`\n--- ${report.cli} Summary ---`);
     console.log(`Tasks: ${report.taskCount}${report.totalEvals ? `, Evals: ${report.totalEvals}` : ''}`);
     for (const mr of report.modelResults) {
-        console.log(`  ${mr.displayName} [${mr.helpMode}]: pass=${(mr.passRate * 100).toFixed(0)}% avgTurns=${mr.avgTurnsToSuccess.toFixed(1)} avgTokens=${Math.round(mr.avgTotalTokens)}`);
+        console.log(`  ${mr.displayName} [${mr.contextMode}]: pass=${(mr.passRate * 100).toFixed(0)}% avgTurns=${mr.avgTurnsToSuccess.toFixed(1)} avgTokens=${Math.round(mr.avgTotalTokens)}`);
         // Per-task breakdown when repeats are used
         if (report.totalEvals && report.totalEvals > report.taskCount) {
             const byTask = new Map();