@cliwatch/cli-bench 0.7.0 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/dist/ci.d.ts.map +1 -1
- package/dist/ci.js +14 -0
- package/dist/ci.js.map +1 -1
- package/dist/client/types.gen.d.ts +24 -12
- package/dist/client/types.gen.d.ts.map +1 -1
- package/dist/client/zod.gen.d.ts +22 -10
- package/dist/client/zod.gen.d.ts.map +1 -1
- package/dist/client/zod.gen.js +6 -6
- package/dist/client/zod.gen.js.map +1 -1
- package/dist/config.d.ts +0 -3
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +0 -16
- package/dist/config.js.map +1 -1
- package/dist/index.js +4 -10
- package/dist/index.js.map +1 -1
- package/dist/init.js +1 -1
- package/dist/models.d.ts +2 -9
- package/dist/models.d.ts.map +1 -1
- package/dist/models.js +1 -1
- package/dist/models.js.map +1 -1
- package/dist/project.d.ts.map +1 -1
- package/dist/project.js +86 -1
- package/dist/project.js.map +1 -1
- package/dist/prompt.d.ts +2 -8
- package/dist/prompt.d.ts.map +1 -1
- package/dist/prompt.js +2 -35
- package/dist/prompt.js.map +1 -1
- package/dist/runner.d.ts +3 -3
- package/dist/runner.d.ts.map +1 -1
- package/dist/runner.js +123 -120
- package/dist/runner.js.map +1 -1
- package/dist/schemas.d.ts +5 -1
- package/dist/schemas.d.ts.map +1 -1
- package/dist/schemas.js +2 -1
- package/dist/schemas.js.map +1 -1
- package/dist/suite-generator.d.ts.map +1 -1
- package/dist/suite-generator.js +35 -4
- package/dist/suite-generator.js.map +1 -1
- package/package.json +1 -1
- package/dist/help-loader.d.ts +0 -17
- package/dist/help-loader.d.ts.map +0 -1
- package/dist/help-loader.js +0 -65
- package/dist/help-loader.js.map +0 -1
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAEA;;;;;;;;;GASG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC7C,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAEA;;;;;;;;;GASG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC7C,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AACpD,OAAO,EAAE,iBAAiB,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC9D,OAAO,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAC5C,OAAO,EAAE,eAAe,EAAE,gBAAgB,EAAgB,MAAM,gBAAgB,CAAC;AACjF,OAAO,EAAE,eAAe,EAAE,qBAAqB,EAAE,MAAM,iBAAiB,CAAC;AACzE,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAEtD,KAAK,UAAU,IAAI;IACjB,MAAM,MAAM,GAAG,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAEvC,gCAAgC;IAChC,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,eAAe,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;YAClD,OAAO,CAAC,GAAG,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC;YAC/B,OAAO,CAAC,GAAG,CAAC,oFAAoF,CAAC,CAAC;QACpG,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,CAAC,KAAK,CAAC,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAClD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QACD,OAAO;IACT,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,4BAA4B,CAAC,CAAC;IAE1C,4BAA4B;IAC5B,MAAM,UAAU,GAAG,MAAM,iBAAiB,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IAE9D,IAAI,OAAO,CAAC;IACZ,IAAI,gBAAoE,CAAC;IACzE,IAAI,eAAe,GAAG,KAAK,CAAC;IAE5B,IAAI,UAAU,EAAE,CAAC;QACf,mBAAmB;QACnB,OAAO,CAAC,GAAG,CAAC,WAAW,UAAU,EAAE,CAAC,CAAC;QACrC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,QAAQ,EAAE,gBAAgB,EAAE,YAAY,EAAE,GAAG,MAAM,WAAW,CAAC,UAAU,CAAC,CAAC;QAC9G,gBAAgB,GAAG,UAAU,CAAC,UAAU,CAAC;QAEzC,8CAA8C;QAC9C,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC;YAClC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;YAC1E,CAAC,CAAC,QAAQ,CAAC;QACb,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,CAAC,KAAK,CAAC,wBAAwB,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAChE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QAED,kCAAkC;QAClC,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC;YACxC,CAAC,CAAC,MAAM,CAAC,MAAM;YACf,CAAC,CAAC,UAAU,CAAC,SAAS,IAAI,CAAC,oCAAoC,CAAC,CAAC;QACnE,MAAM,WAAW,GAAG,UAAU,CAAC,WAAW,IAAI,MAAM,CAAC,WAAW,CAAC;QAEjE,4BAA4B;QAC5B,MAAM,UAAU,GAAG,UAAU,CAAC,MAAM,IAAI,MAAM,CAAC;QAC/C,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM;eAC7B,UAAU,KAAK,QAAQ;eACvB,CAAC,UAAU,KAAK,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAEhD,OAAO,CAAC,GAAG,CAAC,QAAQ,UAAU,CAAC,GAAG,EAAE,CAAC,CAAC;QACtC,OAAO,CAAC,GAAG,CAAC,cAAc,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAClD,OAAO,CAAC,GAAG,CAAC,UAAU,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;QACtC,OAAO,CAAC,GAAG,CAAC,YAAY,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QAEzC,mCAAmC;QACnC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACnB,eAAe,CAAC,SAAS,CAAC,CAAC;QAC7B,CAAC;QAED,MAAM,MAAM,GAAG,gBAAgB,CAAC,SAAS,CAAC,CAAC;QAE3C,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,IAAI,UAAU,CAAC,MAAM,CAAC;QAExD,OAAO,GAAG,MAAM,OAAO,CAAC;YACtB,MAAM,EAAE,EAAE,GAAG,MAAM,EAAE,WAAW,EAAE;YAClC,KAAK;YACL,OAAO,EAAE,UAAU,CAAC,GAAG;YACvB,MAAM;YACN,cAAc,EAAE,UAAU,CAAC,eAAe;YAC1C,OAAO,EAAE,UAAU,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO;YAC7C,YAAY;YACZ,YAAY,EAAE,UAAU,CAAC,aAAa;YACtC,WAAW,EAAE,UAAU,CAAC,YAAY;YACpC,QAAQ,EAAE,UAAU,CAAC,QAAQ;YAC7B,UAAU,EAAE,UAAU,CAAC,WAAW;YAClC,SAAS,EAAE,UAAU,CAAC,UAAU;YAChC,gBAAgB;YAChB,aAAa,EAAE,UAAU,CAAC,UAAU;YACpC,cAAc,EAAE,UAAU,CAAC,eAAe;YAC1C,OAAO,EAAE,UAAU,CAAC,GAAG;YACvB,SAAS,EAAE,UAAU,CAAC,KAAK;YAC3B,WAAW,EAAE,UAAU,CAAC,OAAO;YAC/B,YAAY,EAAE,UAAU,CAAC,QAAQ;YACjC,YAAY;SACb,CAAC,CAAC;QAEH,wEAAwE;QACxE,IAAI,gBAAgB,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YAC7D,MAAM,KAAK,GAAG,eAAe,CAC3B,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,EACtC,gBAAgB,CACjB,CAAC;YACF,qBAAqB,CAAC,KAAK,CAAC,CAAC;YAC7B,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC7B,MAAM,CAAC,gBAAgB,GAAG,KAAK,CAAC;YAClC,CAAC;YACD,+DAA+D;YAC/D,IAAI,CAAC,KAAK,CAAC,SAAS,IAAI,KAAK,CAAC,QAAQ,KAAK,OAAO,EAAE,CAAC;gBACnD,eAAe,GAAG,IAAI,CAAC;YACzB,CAAC;QACH,CAAC;QAED,uBAAuB;QACvB,IAAI,YAAY,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACnC,MAAM,UAAU,GAAG,UAAU,CAAC,WAAW,IAAI,MAAM,CAAC,UAAU,CAAC;YAC/D,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC7B,IAAI,CAAC;oBACH,MAAM,YAAY,CAAC,MAAM,EAAE,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;gBACxD,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACX,OAAO,CAAC,KAAK,CACX,+BAA+B,MAAM,CAAC,GAAG,KAAK,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CACnF,CAAC;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;QACD,yCAAyC;QACzC,IAAI,MAAM,CAAC,iBAAiB,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACrE,MAAM,QAAQ,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC,CAAE,CAAC,CAAC;YAC9C,MAAM,SAAS,CAAC,MAAM,CAAC,iBAAiB,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;YAC7D,OAAO,CAAC,GAAG,CAAC,2BAA2B,MAAM,CAAC,iBAAiB,EAAE,CAAC,CAAC;QACrE,CAAC;IACH,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,KAAK,CAAC,8EAA8E,CAAC,CAAC;QAC9F,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxC,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAC3B,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,EAC3C,IAAI,EACJ,CAAC,CACF,CAAC;QACF,MAAM,SAAS,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;QAChD,OAAO,CAAC,GAAG,CAAC,wBAAwB,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IACvD,CAAC;IAED,eAAe;IACf,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QACzC,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC;QACvC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,KAAK,MAAM,EAAE,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;gBACrC,OAAO,CAAC,GAAG,CACT,GAAG,MAAM,CAAC,GAAG,MAAM,EAAE,CAAC,WAAW,KAAK,CAAC,EAAE,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,oBAAoB,EAAE,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAC1H,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED,uEAAuE;IACvE,IAAI,eAAe,EAAE,CAAC;QACpB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;IACjB,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACjB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}
|
package/dist/init.js
CHANGED
|
@@ -17,9 +17,9 @@ providers:
|
|
|
17
17
|
# - google/gemini-2.5-pro
|
|
18
18
|
|
|
19
19
|
# Optional settings
|
|
20
|
-
# context: [zero-shot] # zero-shot | help | docs
|
|
21
20
|
# concurrency: 3 # max concurrent API calls
|
|
22
21
|
# workdir: ./workspace # working directory for commands (default: temp dir)
|
|
22
|
+
# scaffold: scaffolds/my-project # directory copied into workdir before each task
|
|
23
23
|
# upload: auto # auto | always | never (auto uploads if CLIWATCH_API_KEY is set)
|
|
24
24
|
|
|
25
25
|
tasks:
|
package/dist/models.d.ts
CHANGED
|
@@ -4,8 +4,6 @@
|
|
|
4
4
|
* Assertion-based evaluation: tasks define assertions that are checked
|
|
5
5
|
* against the agent's execution trace.
|
|
6
6
|
*/
|
|
7
|
-
export type ContextMode = 'zero-shot' | 'help' | 'docs';
|
|
8
|
-
export declare const CONTEXT_MODES: ContextMode[];
|
|
9
7
|
export type Assertion = {
|
|
10
8
|
output_contains: string;
|
|
11
9
|
} | {
|
|
@@ -56,6 +54,7 @@ export interface Task {
|
|
|
56
54
|
category?: string;
|
|
57
55
|
repeat?: number;
|
|
58
56
|
tags?: string[];
|
|
57
|
+
scaffold?: string | false;
|
|
59
58
|
}
|
|
60
59
|
export interface TaskSuite {
|
|
61
60
|
cli: string;
|
|
@@ -81,7 +80,6 @@ export interface ConfigFile {
|
|
|
81
80
|
website_url?: string;
|
|
82
81
|
github_url?: string;
|
|
83
82
|
providers?: string[];
|
|
84
|
-
context?: string[];
|
|
85
83
|
system_prompt?: string;
|
|
86
84
|
concurrency?: number;
|
|
87
85
|
workdir?: string;
|
|
@@ -94,6 +92,7 @@ export interface ConfigFile {
|
|
|
94
92
|
env?: Record<string, string>;
|
|
95
93
|
setup?: string[];
|
|
96
94
|
cleanup?: string[];
|
|
95
|
+
scaffold?: string;
|
|
97
96
|
tasks: (Task | string)[];
|
|
98
97
|
}
|
|
99
98
|
export type Provider = string;
|
|
@@ -124,7 +123,6 @@ export interface ModelResult {
|
|
|
124
123
|
provider: Provider;
|
|
125
124
|
modelId: string;
|
|
126
125
|
displayName: string;
|
|
127
|
-
contextMode: ContextMode;
|
|
128
126
|
taskResults: TaskEval[];
|
|
129
127
|
passRate: number;
|
|
130
128
|
avgTurnsToSuccess: number;
|
|
@@ -175,9 +173,4 @@ export interface GridReport {
|
|
|
175
173
|
projectFiles?: Record<string, string>;
|
|
176
174
|
thresholdResults?: ThresholdCheckResult;
|
|
177
175
|
}
|
|
178
|
-
export interface HelpCache {
|
|
179
|
-
cli_name: string;
|
|
180
|
-
help_texts: Record<string, string>;
|
|
181
|
-
version?: string;
|
|
182
|
-
}
|
|
183
176
|
//# sourceMappingURL=models.d.ts.map
|
package/dist/models.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"models.d.ts","sourceRoot":"","sources":["../src/models.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAMH,MAAM,MAAM,
|
|
1
|
+
{"version":3,"file":"models.d.ts","sourceRoot":"","sources":["../src/models.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAMH,MAAM,MAAM,SAAS,GACjB;IAAE,eAAe,EAAE,MAAM,CAAA;CAAE,GAC3B;IAAE,aAAa,EAAE,MAAM,CAAA;CAAE,GACzB;IAAE,cAAc,EAAE,MAAM,CAAA;CAAE,GAC1B;IAAE,SAAS,EAAE,MAAM,CAAA;CAAE,GACrB;IAAE,WAAW,EAAE,MAAM,CAAA;CAAE,GACvB;IAAE,aAAa,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAA;CAAE,GACjD;IAAE,GAAG,EAAE,MAAM,CAAA;CAAE,GACf;IAAE,OAAO,EAAE,MAAM,CAAA;CAAE,GACnB;IAAE,SAAS,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,GAAG,CAAC,EAAE,MAAM,CAAC;QAAC,GAAG,CAAC,EAAE,MAAM,CAAA;KAAE,CAAA;CAAE,GAC9D;IAAE,MAAM,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,eAAe,CAAC,EAAE,MAAM,CAAC;QAAC,aAAa,CAAC,EAAE,MAAM,CAAA;KAAE,CAAA;CAAE,CAAC;AAElF,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,SAAS,CAAC;IACrB,MAAM,EAAE,OAAO,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,IAAI;IACnB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,SAAS,EAAE,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,MAAM,CAAC;IACxC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,GAAG,KAAK,CAAC;CAC3B;AAED,MAAM,WAAW,SAAS;IACxB,GAAG,EAAE,MAAM,CAAC;IACZ,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,IAAI,EAAE,CAAC;CACf;AAMD,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,QAAQ,GAAG,OAAO,CAAC;AAErD,MAAM,WAAW,gBAAgB;IAC/B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,OAAO,GAAG,eAAe,CAAC;CACtC;AAED,MAAM,WAAW,UAAU;IACzB,GAAG,EAAE,MAAM,CAAC;IACZ,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,UAAU,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7B,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,CAAC,IAAI,GAAG,MAAM,CAAC,EAAE,CAAC;CAC1B;AAMD,MAAM,MAAM,QAAQ,GAAG,MAAM,CAAC;AAE9B,MAAM,WAAW,WAAW;IAC1B,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;CACnB;AAMD,MAAM,WAAW,QAAQ;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,OAAO,CAAC;IAChB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,gBAAgB,EAAE,eAAe,EAAE,CAAC;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,UAAU,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,MAAM,CAAC;IACxC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,iBAAiB,CAAC,EAAE,OAAO,EAAE,CAAC;IAC9B,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACzC,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAMD,MAAM,WAAW,WAAW;IAC1B,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,QAAQ,EAAE,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,OAAO,CAAC;CACjB;AAED,MAAM,WAAW,oBAAoB;IACnC,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,EAAE,eAAe,EAAE,CAAC;IAC3B,QAAQ,EAAE,OAAO,GAAG,eAAe,CAAC;CACrC;AAED,MAAM,WAAW,UAAU;IACzB,GAAG,EAAE,MAAM,CAAC;IACZ,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,gBAAgB,EAAE,MAAM,CAAC;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,WAAW,EAAE,CAAC;IAC5B,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtC,gBAAgB,CAAC,EAAE,oBAAoB,CAAC;CACzC"}
|
package/dist/models.js
CHANGED
package/dist/models.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"models.js","sourceRoot":"","sources":["../src/models.ts"],"names":[],"mappings":"AAAA;;;;;GAKG
|
|
1
|
+
{"version":3,"file":"models.js","sourceRoot":"","sources":["../src/models.ts"],"names":[],"mappings":"AAAA;;;;;GAKG"}
|
package/dist/project.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"project.d.ts","sourceRoot":"","sources":["../src/project.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAOH,OAAO,KAAK,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAIpD;;GAEG;AACH,wBAAsB,iBAAiB,CAAC,YAAY,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAqBrF;AAED;;;GAGG;AACH,wBAAsB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC;IAAE,MAAM,EAAE,UAAU,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,CAAC,CAKtG;AAED;;;;GAIG;AACH,wBAAsB,eAAe,CACnC,KAAK,EAAE,CAAC,IAAI,GAAG,MAAM,CAAC,EAAE,EACxB,OAAO,EAAE,MAAM,GACd,OAAO,CAAC;IAAE,KAAK,EAAE,IAAI,EAAE,CAAC;IAAC,WAAW,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAAE,CAAC,CAgD9D;
|
|
1
|
+
{"version":3,"file":"project.d.ts","sourceRoot":"","sources":["../src/project.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAOH,OAAO,KAAK,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAIpD;;GAEG;AACH,wBAAsB,iBAAiB,CAAC,YAAY,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAqBrF;AAED;;;GAGG;AACH,wBAAsB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC;IAAE,MAAM,EAAE,UAAU,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,CAAC,CAKtG;AAED;;;;GAIG;AACH,wBAAsB,eAAe,CACnC,KAAK,EAAE,CAAC,IAAI,GAAG,MAAM,CAAC,EAAE,EACxB,OAAO,EAAE,MAAM,GACd,OAAO,CAAC;IAAE,KAAK,EAAE,IAAI,EAAE,CAAC;IAAC,WAAW,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAAE,CAAC,CAgD9D;AAsDD;;GAEG;AACH,wBAAsB,WAAW,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC;IAC7D,MAAM,EAAE,UAAU,CAAC;IACnB,KAAK,EAAE,IAAI,EAAE,CAAC;IACd,gBAAgB,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACtC,CAAC,CA+CD"}
|
package/dist/project.js
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* Config file loader — discovers and parses cli-bench.yaml,
|
|
3
3
|
* resolves file:// task references with glob support.
|
|
4
4
|
*/
|
|
5
|
-
import { readFile, access } from 'node:fs/promises';
|
|
5
|
+
import { readFile, readdir, access, stat, open } from 'node:fs/promises';
|
|
6
6
|
import { join, dirname, resolve, relative } from 'node:path';
|
|
7
7
|
import { glob } from 'node:fs/promises';
|
|
8
8
|
import { parse as parseYaml, stringify as stringifyYaml } from 'yaml';
|
|
@@ -95,6 +95,60 @@ export async function resolveTaskRefs(tasks, baseDir) {
|
|
|
95
95
|
}
|
|
96
96
|
return { tasks: resolved, loadedFiles };
|
|
97
97
|
}
|
|
98
|
+
/**
|
|
99
|
+
* Assert that a path exists and is a directory.
|
|
100
|
+
*/
|
|
101
|
+
async function assertDirectory(absPath, label) {
|
|
102
|
+
let s;
|
|
103
|
+
try {
|
|
104
|
+
s = await stat(absPath);
|
|
105
|
+
}
|
|
106
|
+
catch {
|
|
107
|
+
throw new Error(`${label}: path does not exist: ${absPath}`);
|
|
108
|
+
}
|
|
109
|
+
if (!s.isDirectory()) {
|
|
110
|
+
throw new Error(`${label}: path is not a directory: ${absPath}`);
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
/**
|
|
114
|
+
* Recursively walk a directory, yielding absolute file paths.
|
|
115
|
+
*/
|
|
116
|
+
async function* walkDir(dir) {
|
|
117
|
+
const entries = await readdir(dir, { withFileTypes: true });
|
|
118
|
+
for (const entry of entries) {
|
|
119
|
+
const fullPath = join(dir, entry.name);
|
|
120
|
+
if (entry.isDirectory()) {
|
|
121
|
+
yield* walkDir(fullPath);
|
|
122
|
+
}
|
|
123
|
+
else if (entry.isFile()) {
|
|
124
|
+
yield fullPath;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
/**
|
|
129
|
+
* Check if a file is likely a text file by reading the first 512 bytes
|
|
130
|
+
* and looking for null bytes.
|
|
131
|
+
*/
|
|
132
|
+
async function isTextFile(filePath) {
|
|
133
|
+
try {
|
|
134
|
+
const fh = await open(filePath, 'r');
|
|
135
|
+
try {
|
|
136
|
+
const buf = Buffer.alloc(512);
|
|
137
|
+
const { bytesRead } = await fh.read(buf, 0, 512, 0);
|
|
138
|
+
for (let i = 0; i < bytesRead; i++) {
|
|
139
|
+
if (buf[i] === 0)
|
|
140
|
+
return false;
|
|
141
|
+
}
|
|
142
|
+
return true;
|
|
143
|
+
}
|
|
144
|
+
finally {
|
|
145
|
+
await fh.close();
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
catch {
|
|
149
|
+
return false;
|
|
150
|
+
}
|
|
151
|
+
}
|
|
98
152
|
/**
|
|
99
153
|
* Load config file and resolve all task references.
|
|
100
154
|
*/
|
|
@@ -102,7 +156,21 @@ export async function loadProject(configPath) {
|
|
|
102
156
|
const { config, rawContent } = await loadConfigFile(configPath);
|
|
103
157
|
const baseDir = dirname(configPath);
|
|
104
158
|
const { tasks, loadedFiles } = await resolveTaskRefs(config.tasks, baseDir);
|
|
159
|
+
// Serialize before resolving scaffold paths to avoid leaking absolute host paths
|
|
105
160
|
const taskSuiteContent = stringifyYaml(tasks);
|
|
161
|
+
// Resolve scaffold paths to absolute and validate
|
|
162
|
+
if (config.scaffold) {
|
|
163
|
+
const abs = resolve(baseDir, config.scaffold);
|
|
164
|
+
await assertDirectory(abs, `config scaffold "${config.scaffold}"`);
|
|
165
|
+
config.scaffold = abs;
|
|
166
|
+
}
|
|
167
|
+
for (const task of tasks) {
|
|
168
|
+
if (typeof task.scaffold === 'string') {
|
|
169
|
+
const abs = resolve(baseDir, task.scaffold);
|
|
170
|
+
await assertDirectory(abs, `task "${task.id}" scaffold "${task.scaffold}"`);
|
|
171
|
+
task.scaffold = abs;
|
|
172
|
+
}
|
|
173
|
+
}
|
|
106
174
|
// Build projectFiles map: config file + all loaded task files
|
|
107
175
|
const configFileName = configPath.endsWith('.yml') ? 'cli-bench.yml' : 'cli-bench.yaml';
|
|
108
176
|
const projectFiles = {
|
|
@@ -111,6 +179,23 @@ export async function loadProject(configPath) {
|
|
|
111
179
|
for (const [relPath, content] of loadedFiles) {
|
|
112
180
|
projectFiles[relPath] = content;
|
|
113
181
|
}
|
|
182
|
+
// Read all scaffold directories referenced by config or tasks
|
|
183
|
+
const scaffoldDirs = new Set();
|
|
184
|
+
if (config.scaffold)
|
|
185
|
+
scaffoldDirs.add(config.scaffold);
|
|
186
|
+
for (const task of tasks) {
|
|
187
|
+
if (typeof task.scaffold === 'string')
|
|
188
|
+
scaffoldDirs.add(task.scaffold);
|
|
189
|
+
}
|
|
190
|
+
for (const absDir of scaffoldDirs) {
|
|
191
|
+
const relDir = relative(baseDir, absDir);
|
|
192
|
+
for await (const entry of walkDir(absDir)) {
|
|
193
|
+
const relPath = join(relDir, relative(absDir, entry));
|
|
194
|
+
if (!(await isTextFile(entry)))
|
|
195
|
+
continue;
|
|
196
|
+
projectFiles[relPath] = await readFile(entry, 'utf-8');
|
|
197
|
+
}
|
|
198
|
+
}
|
|
114
199
|
return { config, tasks, taskSuiteContent, projectFiles };
|
|
115
200
|
}
|
|
116
201
|
//# sourceMappingURL=project.js.map
|
package/dist/project.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"project.js","sourceRoot":"","sources":["../src/project.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"project.js","sourceRoot":"","sources":["../src/project.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AACzE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAC7D,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AACxC,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,SAAS,IAAI,aAAa,EAAE,MAAM,MAAM,CAAC;AACtE,OAAO,EAAE,gBAAgB,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAG5E,MAAM,gBAAgB,GAAG,CAAC,gBAAgB,EAAE,eAAe,CAAC,CAAC;AAE7D;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,YAAqB;IAC3D,IAAI,YAAY,EAAE,CAAC;QACjB,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,YAAY,CAAC,CAAC;YAC3B,OAAO,OAAO,CAAC,YAAY,CAAC,CAAC;QAC/B,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,KAAK,MAAM,IAAI,IAAI,gBAAgB,EAAE,CAAC;QACpC,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;QAChC,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;YACxB,OAAO,SAAS,CAAC;QACnB,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,IAAY;IAC/C,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACjD,MAAM,MAAM,GAAG,SAAS,CAAC,UAAU,CAAC,CAAC;IACrC,MAAM,MAAM,GAAG,gBAAgB,CAAC,KAAK,CAAC,MAAM,CAAe,CAAC;IAC5D,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;AAChC,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,KAAwB,EACxB,OAAe;IAEf,MAAM,QAAQ,GAAW,EAAE,CAAC;IAC5B,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAClC,MAAM,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAC;IAE9C,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;QAC1B,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC9B,oBAAoB;YACpB,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;YAC5C,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;YAEnC,gCAAgC;YAChC,MAAM,KAAK,GAAa,EAAE,CAAC;YAC3B,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBACtB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;oBACxC,IAAI,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;wBACtD,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBACpB,CAAC;gBACH,CAAC;gBACD,KAAK,CAAC,IAAI,EAAE,CAAC;YACf,CAAC;iBAAM,CAAC;gBACN,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACtB,CAAC;YAED,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;gBAC7B,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;gBAC9C,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;gBAC5C,WAAW,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;gBAC9B,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;gBAC9B,MAAM,SAAS,GAAG,cAAc,CAAC,KAAK,CAAC,MAAM,CAAW,CAAC;gBACzD,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;oBAC7B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;wBAC1B,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;wBACrB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBACtB,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;aAAM,CAAC;YACN,wCAAwC;YACxC,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,KAAK,CAAS,CAAC;YAC7C,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;gBAC1B,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBACrB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,CAAC;AAC1C,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,eAAe,CAAC,OAAe,EAAE,KAAa;IAC3D,IAAI,CAAC,CAAC;IACN,IAAI,CAAC;QACH,CAAC,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,CAAC;IAC1B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,0BAA0B,OAAO,EAAE,CAAC,CAAC;IAC/D,CAAC;IACD,IAAI,CAAC,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC;QACrB,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,8BAA8B,OAAO,EAAE,CAAC,CAAC;IACnE,CAAC;AACH,CAAC;AAED;;GAEG;AACH,KAAK,SAAS,CAAC,CAAC,OAAO,CAAC,GAAW;IACjC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IAC5D,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QACvC,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;YACxB,KAAK,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC3B,CAAC;aAAM,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;YAC1B,MAAM,QAAQ,CAAC;QACjB,CAAC;IACH,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,UAAU,CAAC,QAAgB;IACxC,IAAI,CAAC;QACH,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QACrC,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAC9B,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;YACpD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;gBACnC,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC;oBAAE,OAAO,KAAK,CAAC;YACjC,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC;gBAAS,CAAC;YACT,MAAM,EAAE,CAAC,KAAK,EAAE,CAAC;QACnB,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,UAAkB;IAMlD,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,cAAc,CAAC,UAAU,CAAC,CAAC;IAChE,MAAM,OAAO,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;IACpC,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,GAAG,MAAM,eAAe,CAAC,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAE5E,iFAAiF;IACjF,MAAM,gBAAgB,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IAE9C,kDAAkD;IAClD,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QACpB,MAAM,GAAG,GAAG,OAAO,CAAC,OAAO,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC9C,MAAM,eAAe,CAAC,GAAG,EAAE,oBAAoB,MAAM,CAAC,QAAQ,GAAG,CAAC,CAAC;QACnE,MAAM,CAAC,QAAQ,GAAG,GAAG,CAAC;IACxB,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,OAAO,IAAI,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;YACtC,MAAM,GAAG,GAAG,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC5C,MAAM,eAAe,CAAC,GAAG,EAAE,SAAS,IAAI,CAAC,EAAE,eAAe,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YAC5E,IAAI,CAAC,QAAQ,GAAG,GAAG,CAAC;QACtB,CAAC;IACH,CAAC;IAED,8DAA8D;IAC9D,MAAM,cAAc,GAAG,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,gBAAgB,CAAC;IACxF,MAAM,YAAY,GAA2B;QAC3C,CAAC,cAAc,CAAC,EAAE,UAAU;KAC7B,CAAC;IACF,KAAK,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,IAAI,WAAW,EAAE,CAAC;QAC7C,YAAY,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC;IAClC,CAAC;IAED,8DAA8D;IAC9D,MAAM,YAAY,GAAG,IAAI,GAAG,EAAU,CAAC;IACvC,IAAI,MAAM,CAAC,QAAQ;QAAE,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IACvD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,OAAO,IAAI,CAAC,QAAQ,KAAK,QAAQ;YAAE,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACzE,CAAC;IACD,KAAK,MAAM,MAAM,IAAI,YAAY,EAAE,CAAC;QAClC,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QACzC,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YAC1C,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC;YACtD,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,KAAK,CAAC,CAAC;gBAAE,SAAS;YACzC,YAAY,CAAC,OAAO,CAAC,GAAG,MAAM,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QACzD,CAAC;IACH,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,CAAC;AAC3D,CAAC"}
|
package/dist/prompt.d.ts
CHANGED
|
@@ -1,13 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Builds prompts for CLI benchmark agents.
|
|
3
|
-
*
|
|
4
|
-
* One unified system message for all context modes.
|
|
5
|
-
* The user message varies by context mode:
|
|
6
|
-
* - zero-shot: CLI name + task intent only
|
|
7
|
-
* - help: CLI name + top-level --help output + task intent
|
|
8
|
-
* - docs: CLI name + documentation contents + task intent
|
|
9
3
|
*/
|
|
10
|
-
import type {
|
|
4
|
+
import type { Task } from './models.js';
|
|
11
5
|
export declare function buildSystemMessage(customPrompt?: string): string;
|
|
12
|
-
export declare function buildUserMessage(cliName: string, task: Task
|
|
6
|
+
export declare function buildUserMessage(cliName: string, task: Task): string;
|
|
13
7
|
//# sourceMappingURL=prompt.d.ts.map
|
package/dist/prompt.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"prompt.d.ts","sourceRoot":"","sources":["../src/prompt.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"prompt.d.ts","sourceRoot":"","sources":["../src/prompt.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AASxC,wBAAgB,kBAAkB,CAAC,YAAY,CAAC,EAAE,MAAM,GAAG,MAAM,CAKhE;AAED,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,GAAG,MAAM,CAEpE"}
|
package/dist/prompt.js
CHANGED
|
@@ -1,11 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Builds prompts for CLI benchmark agents.
|
|
3
|
-
*
|
|
4
|
-
* One unified system message for all context modes.
|
|
5
|
-
* The user message varies by context mode:
|
|
6
|
-
* - zero-shot: CLI name + task intent only
|
|
7
|
-
* - help: CLI name + top-level --help output + task intent
|
|
8
|
-
* - docs: CLI name + documentation contents + task intent
|
|
9
3
|
*/
|
|
10
4
|
const AGENT_SYSTEM_MESSAGE = `You are a CLI expert. You have a run_command tool to execute shell commands. Use it to accomplish the user's task.
|
|
11
5
|
|
|
@@ -19,34 +13,7 @@ export function buildSystemMessage(customPrompt) {
|
|
|
19
13
|
}
|
|
20
14
|
return AGENT_SYSTEM_MESSAGE;
|
|
21
15
|
}
|
|
22
|
-
export function buildUserMessage(cliName, task
|
|
23
|
-
|
|
24
|
-
case 'help':
|
|
25
|
-
if (contextPayload) {
|
|
26
|
-
return `CLI: ${cliName}
|
|
27
|
-
|
|
28
|
-
Help text:
|
|
29
|
-
$ ${cliName} --help
|
|
30
|
-
${contextPayload}
|
|
31
|
-
|
|
32
|
-
Task: ${task.intent}`;
|
|
33
|
-
}
|
|
34
|
-
// Fall through to zero-shot if no help available
|
|
35
|
-
return `CLI: ${cliName}\n\nTask: ${task.intent}`;
|
|
36
|
-
case 'docs':
|
|
37
|
-
if (contextPayload) {
|
|
38
|
-
return `CLI: ${cliName}
|
|
39
|
-
|
|
40
|
-
Documentation:
|
|
41
|
-
${contextPayload}
|
|
42
|
-
|
|
43
|
-
Task: ${task.intent}`;
|
|
44
|
-
}
|
|
45
|
-
// Fall through to zero-shot if no docs available
|
|
46
|
-
return `CLI: ${cliName}\n\nTask: ${task.intent}`;
|
|
47
|
-
case 'zero-shot':
|
|
48
|
-
default:
|
|
49
|
-
return `CLI: ${cliName}\n\nTask: ${task.intent}`;
|
|
50
|
-
}
|
|
16
|
+
export function buildUserMessage(cliName, task) {
|
|
17
|
+
return `CLI: ${cliName}\n\nTask: ${task.intent}`;
|
|
51
18
|
}
|
|
52
19
|
//# sourceMappingURL=prompt.js.map
|
package/dist/prompt.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"prompt.js","sourceRoot":"","sources":["../src/prompt.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"prompt.js","sourceRoot":"","sources":["../src/prompt.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,MAAM,oBAAoB,GAAG;;;;;gDAKmB,CAAC;AAEjD,MAAM,UAAU,kBAAkB,CAAC,YAAqB;IACtD,IAAI,YAAY,EAAE,CAAC;QACjB,OAAO,GAAG,oBAAoB,OAAO,YAAY,EAAE,CAAC;IACtD,CAAC;IACD,OAAO,oBAAoB,CAAC;AAC9B,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,OAAe,EAAE,IAAU;IAC1D,OAAO,QAAQ,OAAO,aAAa,IAAI,CAAC,MAAM,EAAE,CAAC;AACnD,CAAC"}
|
package/dist/runner.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Orchestrates the task x model
|
|
2
|
+
* Orchestrates the task x model matrix with tool-calling agents.
|
|
3
3
|
*
|
|
4
4
|
* For each task:
|
|
5
5
|
* - Create temp workdir (or use configured one)
|
|
@@ -55,8 +55,6 @@ export interface RunGridOptions {
|
|
|
55
55
|
githubUrl?: string;
|
|
56
56
|
/** Serialized YAML of the resolved task suite. */
|
|
57
57
|
taskSuiteContent?: string;
|
|
58
|
-
/** Directory containing the config file (for resolving docs.md). */
|
|
59
|
-
configDir?: string;
|
|
60
58
|
/** Env var names whose values should be redacted from results. */
|
|
61
59
|
redactEnvVars?: string[];
|
|
62
60
|
/** Regex patterns to redact from results (e.g. API-echoed account IDs). */
|
|
@@ -67,6 +65,8 @@ export interface RunGridOptions {
|
|
|
67
65
|
fileSetup?: string[];
|
|
68
66
|
/** Config-level cleanup commands (run after each task). */
|
|
69
67
|
fileCleanup?: string[];
|
|
68
|
+
/** Config-level scaffold directory (copied into workdir before setup). */
|
|
69
|
+
fileScaffold?: string;
|
|
70
70
|
/** Original project files (config + task files) for upload. */
|
|
71
71
|
projectFiles?: Record<string, string>;
|
|
72
72
|
/** Override the language model (for testing with mock models). */
|
package/dist/runner.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../src/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAMH,OAAO,EAA6B,KAAK,aAAa,EAAE,MAAM,IAAI,CAAC;AAGnE,OAAO,KAAK,EACV,SAAS,EACT,IAAI,EACJ,UAAU,
|
|
1
|
+
{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../src/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAMH,OAAO,EAA6B,KAAK,aAAa,EAAE,MAAM,IAAI,CAAC;AAGnE,OAAO,KAAK,EACV,SAAS,EACT,IAAI,EACJ,UAAU,EAIX,MAAM,aAAa,CAAC;AACrB,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AASjD,wBAAsB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,CAKxE;AAGD;;GAEG;AACH,wBAAgB,eAAe,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE;IAAE,OAAO,EAAE,MAAM,CAAA;CAAE,GAAG,MAAM,CAE5E;AAED;;GAEG;AACH,wBAAgB,UAAU,CACxB,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,EACvC,IAAI,EAAE;IAAE,OAAO,EAAE,MAAM,CAAA;CAAE,GACxB,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,CAOpC;AAED;;GAEG;AACH,wBAAgB,eAAe,CAC7B,QAAQ,EAAE,MAAM,EAAE,EAClB,IAAI,EAAE;IAAE,OAAO,EAAE,MAAM,CAAA;CAAE,GACxB,MAAM,EAAE,CAEV;AAkOD,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,8CAA8C;IAC9C,KAAK,CAAC,EAAE,IAAI,EAAE,CAAC;IACf,iDAAiD;IACjD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,+CAA+C;IAC/C,MAAM,CAAC,EAAE,UAAU,EAAE,CAAC;IACtB,wDAAwD;IACxD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,0DAA0D;IAC1D,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,uDAAuD;IACvD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,oDAAoD;IACpD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,mDAAmD;IACnD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,kDAAkD;IAClD,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,kEAAkE;IAClE,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IACzB,2EAA2E;IAC3E,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,8EAA8E;IAC9E,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,0DAA0D;IAC1D,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,2DAA2D;IAC3D,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,0EAA0E;IAC1E,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,+DAA+D;IAC/D,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtC,kEAAkE;IAClE,aAAa,CAAC,EAAE,aAAa,CAAC;CAC/B;AAED,wBAAsB,OAAO,CAAC,IAAI,EAAE,cAAc,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC,CAmCzE;AA8RD,wBAAsB,YAAY,CAChC,MAAM,EAAE,UAAU,EAClB,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,IAAI,CAAC,CAsBf"}
|