@cliwatch/cli-bench 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +145 -0
- package/dist/assertions.d.ts +14 -0
- package/dist/assertions.d.ts.map +1 -0
- package/dist/assertions.js +161 -0
- package/dist/assertions.js.map +1 -0
- package/dist/ci.d.ts +29 -0
- package/dist/ci.d.ts.map +1 -0
- package/dist/ci.js +75 -0
- package/dist/ci.js.map +1 -0
- package/dist/client/client/client.gen.d.ts +3 -0
- package/dist/client/client/client.gen.d.ts.map +1 -0
- package/dist/client/client/client.gen.js +235 -0
- package/dist/client/client/client.gen.js.map +1 -0
- package/dist/client/client/index.d.ts +9 -0
- package/dist/client/client/index.d.ts.map +1 -0
- package/dist/client/client/index.js +7 -0
- package/dist/client/client/index.js.map +1 -0
- package/dist/client/client/types.gen.d.ts +118 -0
- package/dist/client/client/types.gen.d.ts.map +1 -0
- package/dist/client/client/types.gen.js +3 -0
- package/dist/client/client/types.gen.js.map +1 -0
- package/dist/client/client/utils.gen.d.ts +34 -0
- package/dist/client/client/utils.gen.d.ts.map +1 -0
- package/dist/client/client/utils.gen.js +229 -0
- package/dist/client/client/utils.gen.js.map +1 -0
- package/dist/client/client.gen.d.ts +13 -0
- package/dist/client/client.gen.d.ts.map +1 -0
- package/dist/client/client.gen.js +4 -0
- package/dist/client/client.gen.js.map +1 -0
- package/dist/client/core/auth.gen.d.ts +19 -0
- package/dist/client/core/auth.gen.d.ts.map +1 -0
- package/dist/client/core/auth.gen.js +15 -0
- package/dist/client/core/auth.gen.js.map +1 -0
- package/dist/client/core/bodySerializer.gen.d.ts +26 -0
- package/dist/client/core/bodySerializer.gen.d.ts.map +1 -0
- package/dist/client/core/bodySerializer.gen.js +58 -0
- package/dist/client/core/bodySerializer.gen.js.map +1 -0
- package/dist/client/core/params.gen.d.ts +44 -0
- package/dist/client/core/params.gen.d.ts.map +1 -0
- package/dist/client/core/params.gen.js +101 -0
- package/dist/client/core/params.gen.js.map +1 -0
- package/dist/client/core/pathSerializer.gen.d.ts +34 -0
- package/dist/client/core/pathSerializer.gen.d.ts.map +1 -0
- package/dist/client/core/pathSerializer.gen.js +107 -0
- package/dist/client/core/pathSerializer.gen.js.map +1 -0
- package/dist/client/core/queryKeySerializer.gen.d.ts +19 -0
- package/dist/client/core/queryKeySerializer.gen.d.ts.map +1 -0
- package/dist/client/core/queryKeySerializer.gen.js +93 -0
- package/dist/client/core/queryKeySerializer.gen.js.map +1 -0
- package/dist/client/core/serverSentEvents.gen.d.ts +72 -0
- package/dist/client/core/serverSentEvents.gen.d.ts.map +1 -0
- package/dist/client/core/serverSentEvents.gen.js +134 -0
- package/dist/client/core/serverSentEvents.gen.js.map +1 -0
- package/dist/client/core/types.gen.d.ts +79 -0
- package/dist/client/core/types.gen.d.ts.map +1 -0
- package/dist/client/core/types.gen.js +3 -0
- package/dist/client/core/types.gen.js.map +1 -0
- package/dist/client/core/utils.gen.d.ts +20 -0
- package/dist/client/core/utils.gen.d.ts.map +1 -0
- package/dist/client/core/utils.gen.js +88 -0
- package/dist/client/core/utils.gen.js.map +1 -0
- package/dist/client/index.d.ts +3 -0
- package/dist/client/index.d.ts.map +1 -0
- package/dist/client/index.js +3 -0
- package/dist/client/index.js.map +1 -0
- package/dist/client/sdk.gen.d.ts +45 -0
- package/dist/client/sdk.gen.d.ts.map +1 -0
- package/dist/client/sdk.gen.js +47 -0
- package/dist/client/sdk.gen.js.map +1 -0
- package/dist/client/types.gen.d.ts +694 -0
- package/dist/client/types.gen.d.ts.map +1 -0
- package/dist/client/types.gen.js +3 -0
- package/dist/client/types.gen.js.map +1 -0
- package/dist/client/zod.gen.d.ts +492 -0
- package/dist/client/zod.gen.d.ts.map +1 -0
- package/dist/client/zod.gen.js +413 -0
- package/dist/client/zod.gen.js.map +1 -0
- package/dist/config.d.ts +22 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +94 -0
- package/dist/config.js.map +1 -0
- package/dist/exec.d.ts +18 -0
- package/dist/exec.d.ts.map +1 -0
- package/dist/exec.js +30 -0
- package/dist/exec.js.map +1 -0
- package/dist/help-loader.d.ts +13 -0
- package/dist/help-loader.d.ts.map +1 -0
- package/dist/help-loader.js +135 -0
- package/dist/help-loader.js.map +1 -0
- package/dist/index.d.ts +14 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +148 -0
- package/dist/index.js.map +1 -0
- package/dist/init.d.ts +5 -0
- package/dist/init.d.ts.map +1 -0
- package/dist/init.js +62 -0
- package/dist/init.js.map +1 -0
- package/dist/models.d.ts +158 -0
- package/dist/models.d.ts.map +1 -0
- package/dist/models.js +8 -0
- package/dist/models.js.map +1 -0
- package/dist/project.d.ts +26 -0
- package/dist/project.d.ts.map +1 -0
- package/dist/project.js +101 -0
- package/dist/project.js.map +1 -0
- package/dist/prompt.d.ts +12 -0
- package/dist/prompt.d.ts.map +1 -0
- package/dist/prompt.js +88 -0
- package/dist/prompt.js.map +1 -0
- package/dist/providers.d.ts +26 -0
- package/dist/providers.d.ts.map +1 -0
- package/dist/providers.js +55 -0
- package/dist/providers.js.map +1 -0
- package/dist/runner.d.ts +34 -0
- package/dist/runner.d.ts.map +1 -0
- package/dist/runner.js +434 -0
- package/dist/runner.js.map +1 -0
- package/dist/schemas.d.ts +256 -0
- package/dist/schemas.d.ts.map +1 -0
- package/dist/schemas.js +59 -0
- package/dist/schemas.js.map +1 -0
- package/dist/suite-generator.d.ts +8 -0
- package/dist/suite-generator.d.ts.map +1 -0
- package/dist/suite-generator.js +100 -0
- package/dist/suite-generator.js.map +1 -0
- package/dist/thresholds.d.ts +10 -0
- package/dist/thresholds.d.ts.map +1 -0
- package/dist/thresholds.js +57 -0
- package/dist/thresholds.js.map +1 -0
- package/package.json +41 -0
- package/task_suites/curl.yaml +138 -0
- package/task_suites/docker.yaml +163 -0
- package/task_suites/gh.yaml +118 -0
- package/task_suites/jq.yaml +172 -0
- package/task_suites/kubectl.yaml +74 -0
package/dist/runner.js
ADDED
|
@@ -0,0 +1,434 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Orchestrates the task x model x helpMode matrix with tool-calling agents.
|
|
3
|
+
*
|
|
4
|
+
* For each task:
|
|
5
|
+
* - Create temp workdir (or use configured one)
|
|
6
|
+
* - Run setup commands directly on the host
|
|
7
|
+
* - Give LLM the run_command tool
|
|
8
|
+
* - LLM calls tool -> execute on host -> return stdout/stderr/exit
|
|
9
|
+
* - After maxSteps or completion: run assertions
|
|
10
|
+
* - Aggregate into GridReport
|
|
11
|
+
*/
|
|
12
|
+
import { readFile, readdir, mkdtemp, rm } from 'node:fs/promises';
|
|
13
|
+
import { join } from 'node:path';
|
|
14
|
+
import { tmpdir } from 'node:os';
|
|
15
|
+
import { parse as parseYaml } from 'yaml';
|
|
16
|
+
import { generateText, stepCountIs } from 'ai';
|
|
17
|
+
import { z } from 'zod';
|
|
18
|
+
import { getModel, filterModels } from './providers.js';
|
|
19
|
+
import { buildSystemMessage, buildUserMessage } from './prompt.js';
|
|
20
|
+
import { runAssertions } from './assertions.js';
|
|
21
|
+
import { loadHelpFromCache, loadHelpLive } from './help-loader.js';
|
|
22
|
+
import { execCommand, runSetup } from './exec.js';
|
|
23
|
+
import { TaskSuiteSchema } from './schemas.js';
|
|
24
|
+
import { detectCI, computeTaskSuiteHash } from './ci.js';
|
|
25
|
+
export async function loadTaskSuite(filePath) {
|
|
26
|
+
const raw = await readFile(filePath, 'utf-8');
|
|
27
|
+
const parsed = parseYaml(raw);
|
|
28
|
+
const data = TaskSuiteSchema.parse(parsed);
|
|
29
|
+
return data;
|
|
30
|
+
}
|
|
31
|
+
export async function discoverTaskSuites(suiteDir) {
|
|
32
|
+
const suites = new Map();
|
|
33
|
+
try {
|
|
34
|
+
const files = await readdir(suiteDir);
|
|
35
|
+
for (const f of files) {
|
|
36
|
+
if (f.endsWith('.yaml') || f.endsWith('.yml')) {
|
|
37
|
+
const name = f.replace(/\.ya?ml$/, '');
|
|
38
|
+
suites.set(name, join(suiteDir, f));
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
catch {
|
|
43
|
+
// No task suites dir
|
|
44
|
+
}
|
|
45
|
+
return suites;
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Simple concurrency limiter.
|
|
49
|
+
*/
|
|
50
|
+
function createLimiter(maxConcurrency) {
|
|
51
|
+
let active = 0;
|
|
52
|
+
const queue = [];
|
|
53
|
+
async function run(fn) {
|
|
54
|
+
while (active >= maxConcurrency) {
|
|
55
|
+
await new Promise((resolve) => queue.push(resolve));
|
|
56
|
+
}
|
|
57
|
+
active++;
|
|
58
|
+
try {
|
|
59
|
+
return await fn();
|
|
60
|
+
}
|
|
61
|
+
finally {
|
|
62
|
+
active--;
|
|
63
|
+
const next = queue.shift();
|
|
64
|
+
if (next)
|
|
65
|
+
next();
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
return { run };
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* Detect CLI version by running version_command directly.
|
|
72
|
+
*/
|
|
73
|
+
async function detectCliVersion(versionCommand) {
|
|
74
|
+
try {
|
|
75
|
+
const result = await execCommand(versionCommand);
|
|
76
|
+
if (result.exitCode === 0 && result.stdout.trim()) {
|
|
77
|
+
return result.stdout.trim().split('\n')[0];
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
catch {
|
|
81
|
+
// Ignore
|
|
82
|
+
}
|
|
83
|
+
return undefined;
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* Run a single task with tool-calling agent, then run assertions.
|
|
87
|
+
*/
|
|
88
|
+
async function runAgentTask(model, systemMessage, userMessage, task, workdir) {
|
|
89
|
+
const commandsRun = [];
|
|
90
|
+
let lastStdout = '';
|
|
91
|
+
let lastStderr = '';
|
|
92
|
+
let lastExitCode = -1;
|
|
93
|
+
const start = Date.now();
|
|
94
|
+
try {
|
|
95
|
+
const result = await generateText({
|
|
96
|
+
model: getModel(model.id),
|
|
97
|
+
system: systemMessage,
|
|
98
|
+
prompt: userMessage,
|
|
99
|
+
tools: {
|
|
100
|
+
run_command: {
|
|
101
|
+
description: 'Execute a shell command in the terminal',
|
|
102
|
+
inputSchema: z.object({
|
|
103
|
+
command: z.string().describe('The shell command to execute'),
|
|
104
|
+
}),
|
|
105
|
+
execute: async (input) => {
|
|
106
|
+
commandsRun.push(input.command);
|
|
107
|
+
const { stdout, stderr, exitCode } = await execCommand(input.command, { cwd: workdir });
|
|
108
|
+
lastStdout = stdout;
|
|
109
|
+
lastStderr = stderr;
|
|
110
|
+
lastExitCode = exitCode;
|
|
111
|
+
return { stdout, stderr, exitCode };
|
|
112
|
+
},
|
|
113
|
+
},
|
|
114
|
+
},
|
|
115
|
+
stopWhen: stepCountIs(task.max_turns ?? 5),
|
|
116
|
+
temperature: 0,
|
|
117
|
+
});
|
|
118
|
+
const totalLatencyMs = Date.now() - start;
|
|
119
|
+
let totalInputTokens = 0;
|
|
120
|
+
let totalOutputTokens = 0;
|
|
121
|
+
for (const step of result.steps) {
|
|
122
|
+
totalInputTokens += step.usage?.inputTokens ?? 0;
|
|
123
|
+
totalOutputTokens += step.usage?.outputTokens ?? 0;
|
|
124
|
+
}
|
|
125
|
+
const assertionResults = await runAssertions(task.assert, { commandsRun, lastStdout, lastStderr, lastExitCode }, workdir);
|
|
126
|
+
const allPassed = assertionResults.every((r) => r.passed);
|
|
127
|
+
const failures = assertionResults
|
|
128
|
+
.filter((r) => !r.passed)
|
|
129
|
+
.map((r) => {
|
|
130
|
+
const key = Object.keys(r.assertion)[0];
|
|
131
|
+
return `${key}: expected=${r.expected ?? '?'}, actual=${r.actual ?? '?'}`;
|
|
132
|
+
});
|
|
133
|
+
const conversationTrace = result.steps.map((step, i) => ({
|
|
134
|
+
step: i,
|
|
135
|
+
text: step.text,
|
|
136
|
+
toolCalls: step.toolCalls?.map((tc) => ({
|
|
137
|
+
name: tc['toolName'],
|
|
138
|
+
args: tc['input'],
|
|
139
|
+
})),
|
|
140
|
+
toolResults: step.toolResults?.map((tr) => ({
|
|
141
|
+
name: tr['toolName'],
|
|
142
|
+
result: tr['output'],
|
|
143
|
+
})),
|
|
144
|
+
usage: step.usage,
|
|
145
|
+
}));
|
|
146
|
+
return {
|
|
147
|
+
passed: allPassed,
|
|
148
|
+
assertionResults,
|
|
149
|
+
turnsUsed: commandsRun.length,
|
|
150
|
+
commandsRun,
|
|
151
|
+
totalInputTokens,
|
|
152
|
+
totalOutputTokens,
|
|
153
|
+
totalLatencyMs,
|
|
154
|
+
lastStdout,
|
|
155
|
+
lastStderr,
|
|
156
|
+
lastExitCode,
|
|
157
|
+
conversationTrace,
|
|
158
|
+
failureReason: failures.length > 0 ? failures.join('; ') : undefined,
|
|
159
|
+
};
|
|
160
|
+
}
|
|
161
|
+
catch (e) {
|
|
162
|
+
return {
|
|
163
|
+
passed: false,
|
|
164
|
+
assertionResults: [],
|
|
165
|
+
turnsUsed: commandsRun.length,
|
|
166
|
+
commandsRun,
|
|
167
|
+
totalInputTokens: 0,
|
|
168
|
+
totalOutputTokens: 0,
|
|
169
|
+
totalLatencyMs: Date.now() - start,
|
|
170
|
+
lastStdout,
|
|
171
|
+
lastStderr,
|
|
172
|
+
lastExitCode,
|
|
173
|
+
conversationTrace: [{ error: e instanceof Error ? e.message : String(e) }],
|
|
174
|
+
failureReason: e instanceof Error ? e.message : String(e),
|
|
175
|
+
};
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
function aggregateModelResult(model, taskResults, helpMode) {
|
|
179
|
+
const passRate = taskResults.length > 0
|
|
180
|
+
? taskResults.filter((t) => t.passed).length / taskResults.length
|
|
181
|
+
: 0;
|
|
182
|
+
const avgLatencyMs = taskResults.length > 0
|
|
183
|
+
? Math.round(taskResults.reduce((a, t) => a + t.totalLatencyMs, 0) / taskResults.length)
|
|
184
|
+
: 0;
|
|
185
|
+
const passedTasks = taskResults.filter((t) => t.passed);
|
|
186
|
+
const avgTurnsToSuccess = passedTasks.length > 0
|
|
187
|
+
? passedTasks.reduce((a, t) => a + t.turnsUsed, 0) / passedTasks.length
|
|
188
|
+
: 0;
|
|
189
|
+
const avgTotalTokens = taskResults.length > 0
|
|
190
|
+
? taskResults.reduce((a, t) => a + t.totalInputTokens + t.totalOutputTokens, 0) /
|
|
191
|
+
taskResults.length
|
|
192
|
+
: 0;
|
|
193
|
+
return {
|
|
194
|
+
provider: model.provider,
|
|
195
|
+
modelId: model.id,
|
|
196
|
+
displayName: model.displayName,
|
|
197
|
+
helpMode,
|
|
198
|
+
taskResults,
|
|
199
|
+
passRate,
|
|
200
|
+
avgTurnsToSuccess,
|
|
201
|
+
avgTotalTokens,
|
|
202
|
+
avgLatencyMs,
|
|
203
|
+
};
|
|
204
|
+
}
|
|
205
|
+
export async function runGrid(opts) {
|
|
206
|
+
const { config } = opts;
|
|
207
|
+
const reports = [];
|
|
208
|
+
const limiter = createLimiter(config.concurrency);
|
|
209
|
+
if (opts.tasks && opts.cliName) {
|
|
210
|
+
// Config file mode — single CLI run
|
|
211
|
+
const report = await runCliGrid({
|
|
212
|
+
cliName: opts.cliName,
|
|
213
|
+
tasks: opts.tasks,
|
|
214
|
+
models: opts.models ?? filterModels(config.models),
|
|
215
|
+
helpModes: config.helpModes,
|
|
216
|
+
limiter,
|
|
217
|
+
config,
|
|
218
|
+
versionCommand: opts.versionCommand,
|
|
219
|
+
workdir: opts.workdir,
|
|
220
|
+
globalRepeat: opts.globalRepeat,
|
|
221
|
+
});
|
|
222
|
+
if (report)
|
|
223
|
+
reports.push(report);
|
|
224
|
+
return reports;
|
|
225
|
+
}
|
|
226
|
+
// Legacy task_suites/ discovery mode
|
|
227
|
+
const suiteDir = join(new URL('.', import.meta.url).pathname.replace(/\/src\/$/, '').replace(/\/dist\/$/, ''), 'task_suites');
|
|
228
|
+
const suiteMap = await discoverTaskSuites(suiteDir);
|
|
229
|
+
const models = filterModels(config.models);
|
|
230
|
+
const cliFilter = config.filter.length > 0
|
|
231
|
+
? config.filter
|
|
232
|
+
: [...suiteMap.keys()];
|
|
233
|
+
for (const cliName of cliFilter) {
|
|
234
|
+
const suitePath = suiteMap.get(cliName);
|
|
235
|
+
if (!suitePath) {
|
|
236
|
+
console.error(`No task suite found for CLI: ${cliName}`);
|
|
237
|
+
continue;
|
|
238
|
+
}
|
|
239
|
+
const suite = await loadTaskSuite(suitePath);
|
|
240
|
+
const report = await runCliGrid({
|
|
241
|
+
cliName,
|
|
242
|
+
tasks: suite.tasks,
|
|
243
|
+
models,
|
|
244
|
+
helpModes: config.helpModes,
|
|
245
|
+
limiter,
|
|
246
|
+
config,
|
|
247
|
+
versionCommand: suite.version_command,
|
|
248
|
+
});
|
|
249
|
+
if (report)
|
|
250
|
+
reports.push(report);
|
|
251
|
+
}
|
|
252
|
+
return reports;
|
|
253
|
+
}
|
|
254
|
+
async function runCliGrid(opts) {
|
|
255
|
+
const { cliName, tasks, models, helpModes, limiter, config, versionCommand } = opts;
|
|
256
|
+
console.log(`\n=== ${cliName} ===`);
|
|
257
|
+
// Load help text (used for injected mode)
|
|
258
|
+
let helpTexts = null;
|
|
259
|
+
let cliVersion;
|
|
260
|
+
if (helpModes.includes('injected')) {
|
|
261
|
+
const helpCache = await loadHelpFromCache(config.helpCacheDir, cliName);
|
|
262
|
+
if (helpCache) {
|
|
263
|
+
helpTexts = helpCache.help_texts;
|
|
264
|
+
cliVersion = helpCache.version;
|
|
265
|
+
}
|
|
266
|
+
else {
|
|
267
|
+
console.log(` No cached help text, trying live capture...`);
|
|
268
|
+
try {
|
|
269
|
+
const live = await loadHelpLive(cliName);
|
|
270
|
+
helpTexts = live.help_texts;
|
|
271
|
+
cliVersion = live.version;
|
|
272
|
+
}
|
|
273
|
+
catch {
|
|
274
|
+
console.warn(` Failed to capture help text for ${cliName} (injected mode may fail)`);
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
// Detect CLI version
|
|
279
|
+
if (!cliVersion && versionCommand) {
|
|
280
|
+
cliVersion = await detectCliVersion(versionCommand);
|
|
281
|
+
if (cliVersion)
|
|
282
|
+
console.log(` CLI version: ${cliVersion}`);
|
|
283
|
+
}
|
|
284
|
+
// Dry run
|
|
285
|
+
if (config.dryRun) {
|
|
286
|
+
const firstTask = tasks[0];
|
|
287
|
+
if (firstTask) {
|
|
288
|
+
for (const helpMode of helpModes) {
|
|
289
|
+
const systemMessage = buildSystemMessage(helpMode);
|
|
290
|
+
const userMessage = buildUserMessage(cliName, helpTexts, firstTask, helpMode);
|
|
291
|
+
console.log(`\n--- Dry Run: ${cliName} / ${firstTask.id} (${helpMode}) ---`);
|
|
292
|
+
console.log(`System: ${systemMessage}\n`);
|
|
293
|
+
console.log(`User:\n${userMessage}\n`);
|
|
294
|
+
console.log(`--- End Dry Run ---\n`);
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
return null;
|
|
298
|
+
}
|
|
299
|
+
const modelResults = [];
|
|
300
|
+
for (const helpMode of helpModes) {
|
|
301
|
+
console.log(`\n Help mode: ${helpMode}`);
|
|
302
|
+
const systemMessage = buildSystemMessage(helpMode);
|
|
303
|
+
for (const model of models) {
|
|
304
|
+
console.log(` Model: ${model.displayName}`);
|
|
305
|
+
const taskResults = [];
|
|
306
|
+
const workItems = [];
|
|
307
|
+
for (const task of tasks) {
|
|
308
|
+
const repeatCount = opts.globalRepeat ?? task.repeat ?? 1;
|
|
309
|
+
for (let ri = 0; ri < repeatCount; ri++) {
|
|
310
|
+
workItems.push({ task, repeatIndex: ri, repeatTotal: repeatCount });
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
const taskPromises = workItems.map(({ task, repeatIndex, repeatTotal }) => limiter.run(async () => {
|
|
314
|
+
// Fresh temp workdir per repeat — critical for independence
|
|
315
|
+
const workdir = opts.workdir ?? config.workdir ?? await mkdtemp(join(tmpdir(), 'cli-bench-'));
|
|
316
|
+
const shouldCleanup = !opts.workdir && !config.workdir;
|
|
317
|
+
try {
|
|
318
|
+
const userMessage = buildUserMessage(cliName, helpTexts, task, helpMode);
|
|
319
|
+
if (task.setup && task.setup.length > 0) {
|
|
320
|
+
await runSetup(task.setup, { cwd: workdir });
|
|
321
|
+
}
|
|
322
|
+
const agentResult = await runAgentTask(model, systemMessage, userMessage, task, workdir);
|
|
323
|
+
const taskEval = {
|
|
324
|
+
taskId: task.id,
|
|
325
|
+
passed: agentResult.passed,
|
|
326
|
+
failureReason: agentResult.failureReason,
|
|
327
|
+
assertionResults: agentResult.assertionResults,
|
|
328
|
+
turnsUsed: agentResult.turnsUsed,
|
|
329
|
+
totalInputTokens: agentResult.totalInputTokens,
|
|
330
|
+
totalOutputTokens: agentResult.totalOutputTokens,
|
|
331
|
+
totalLatencyMs: agentResult.totalLatencyMs,
|
|
332
|
+
commandsRun: agentResult.commandsRun,
|
|
333
|
+
difficulty: task.difficulty,
|
|
334
|
+
category: task.category,
|
|
335
|
+
conversationTrace: agentResult.conversationTrace,
|
|
336
|
+
taskDefinition: {
|
|
337
|
+
id: task.id,
|
|
338
|
+
intent: task.intent,
|
|
339
|
+
assert: task.assert,
|
|
340
|
+
setup: task.setup,
|
|
341
|
+
max_turns: task.max_turns,
|
|
342
|
+
difficulty: task.difficulty,
|
|
343
|
+
category: task.category,
|
|
344
|
+
},
|
|
345
|
+
repeatIndex: repeatTotal > 1 ? repeatIndex : undefined,
|
|
346
|
+
};
|
|
347
|
+
const icon = agentResult.passed ? '✓' : '✗';
|
|
348
|
+
const repeatLabel = repeatTotal > 1 ? ` [${repeatIndex + 1}/${repeatTotal}]` : '';
|
|
349
|
+
console.log(` ${icon} ${task.id}${repeatLabel} (turns=${agentResult.turnsUsed}, tokens=${agentResult.totalInputTokens + agentResult.totalOutputTokens})`);
|
|
350
|
+
return taskEval;
|
|
351
|
+
}
|
|
352
|
+
finally {
|
|
353
|
+
if (shouldCleanup) {
|
|
354
|
+
await rm(workdir, { recursive: true, force: true }).catch(() => { });
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
}));
|
|
358
|
+
const results = await Promise.all(taskPromises);
|
|
359
|
+
for (const taskEval of results) {
|
|
360
|
+
taskResults.push(taskEval);
|
|
361
|
+
}
|
|
362
|
+
modelResults.push(aggregateModelResult(model, taskResults, helpMode));
|
|
363
|
+
}
|
|
364
|
+
}
|
|
365
|
+
const ciMetadata = detectCI();
|
|
366
|
+
const taskSuiteHash = computeTaskSuiteHash(tasks);
|
|
367
|
+
// Compute total evals (accounts for repeats)
|
|
368
|
+
const hasRepeats = opts.globalRepeat !== undefined || tasks.some((t) => t.repeat !== undefined);
|
|
369
|
+
let totalEvals;
|
|
370
|
+
if (hasRepeats) {
|
|
371
|
+
totalEvals = 0;
|
|
372
|
+
for (const task of tasks) {
|
|
373
|
+
totalEvals += opts.globalRepeat ?? task.repeat ?? 1;
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
const report = {
|
|
377
|
+
cli: cliName,
|
|
378
|
+
cliVersion,
|
|
379
|
+
taskSuiteVersion: '4',
|
|
380
|
+
modelResults,
|
|
381
|
+
taskCount: tasks.length,
|
|
382
|
+
totalEvals,
|
|
383
|
+
generatedAt: new Date().toISOString(),
|
|
384
|
+
gridVersion: '0.4.0',
|
|
385
|
+
systemPrompt: buildSystemMessage(helpModes[0]),
|
|
386
|
+
...ciMetadata,
|
|
387
|
+
taskSuiteHash,
|
|
388
|
+
};
|
|
389
|
+
printReportSummary(report);
|
|
390
|
+
return report;
|
|
391
|
+
}
|
|
392
|
+
function printReportSummary(report) {
|
|
393
|
+
console.log(`\n--- ${report.cli} Summary ---`);
|
|
394
|
+
console.log(`Tasks: ${report.taskCount}${report.totalEvals ? `, Evals: ${report.totalEvals}` : ''}`);
|
|
395
|
+
for (const mr of report.modelResults) {
|
|
396
|
+
console.log(` ${mr.displayName} [${mr.helpMode}]: pass=${(mr.passRate * 100).toFixed(0)}% avgTurns=${mr.avgTurnsToSuccess.toFixed(1)} avgTokens=${Math.round(mr.avgTotalTokens)}`);
|
|
397
|
+
// Per-task breakdown when repeats are used
|
|
398
|
+
if (report.totalEvals && report.totalEvals > report.taskCount) {
|
|
399
|
+
const byTask = new Map();
|
|
400
|
+
for (const tr of mr.taskResults) {
|
|
401
|
+
const entry = byTask.get(tr.taskId) ?? { passed: 0, total: 0 };
|
|
402
|
+
entry.total++;
|
|
403
|
+
if (tr.passed)
|
|
404
|
+
entry.passed++;
|
|
405
|
+
byTask.set(tr.taskId, entry);
|
|
406
|
+
}
|
|
407
|
+
for (const [taskId, { passed, total }] of byTask) {
|
|
408
|
+
if (total > 1) {
|
|
409
|
+
console.log(` ${taskId}: ${passed}/${total} (${Math.round((passed / total) * 100)}%)`);
|
|
410
|
+
}
|
|
411
|
+
}
|
|
412
|
+
}
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
export async function uploadReport(report, backendUrl, apiKey) {
|
|
416
|
+
const { createClient, createConfig } = await import('./client/client/index.js');
|
|
417
|
+
const { CliBench } = await import('./client/sdk.gen.js');
|
|
418
|
+
const uploadClient = createClient(createConfig({
|
|
419
|
+
baseUrl: backendUrl,
|
|
420
|
+
}));
|
|
421
|
+
const { error } = await CliBench.uploadCliBenchResults({
|
|
422
|
+
client: uploadClient,
|
|
423
|
+
headers: { authorization: `Bearer ${apiKey}` },
|
|
424
|
+
body: {
|
|
425
|
+
...report,
|
|
426
|
+
thresholdResults: report.thresholdResults,
|
|
427
|
+
},
|
|
428
|
+
});
|
|
429
|
+
if (error) {
|
|
430
|
+
throw new Error(`Upload failed: ${JSON.stringify(error)}`);
|
|
431
|
+
}
|
|
432
|
+
console.log(`Uploaded grid report for ${report.cli}`);
|
|
433
|
+
}
|
|
434
|
+
//# sourceMappingURL=runner.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"runner.js","sourceRoot":"","sources":["../src/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAC;AAClE,OAAO,EAAE,IAAI,EAAW,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,MAAM,CAAC;AAC1C,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,IAAI,CAAC;AAC/C,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAaxB,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AACxD,OAAO,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AACnE,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAChD,OAAO,EAAE,iBAAiB,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AACnE,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC/C,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,SAAS,CAAC;AAEzD,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,QAAgB;IAClD,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC9C,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;IAC9B,MAAM,IAAI,GAAG,eAAe,CAAC,KAAK,CAAC,MAAM,CAAc,CAAC;IACxD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,QAAgB;IAEhB,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,CAAC;QACtC,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;YACtB,IAAI,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC9C,MAAM,IAAI,GAAG,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;gBACvC,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC;YACtC,CAAC;QACH,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,qBAAqB;IACvB,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,SAAS,aAAa,CAAC,cAAsB;IAC3C,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,MAAM,KAAK,GAAsB,EAAE,CAAC;IAEpC,KAAK,UAAU,GAAG,CAAI,EAAoB;QACxC,OAAO,MAAM,IAAI,cAAc,EAAE,CAAC;YAChC,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;QAC5D,CAAC;QACD,MAAM,EAAE,CAAC;QACT,IAAI,CAAC;YACH,OAAO,MAAM,EAAE,EAAE,CAAC;QACpB,CAAC;gBAAS,CAAC;YACT,MAAM,EAAE,CAAC;YACT,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;YAC3B,IAAI,IAAI;gBAAE,IAAI,EAAE,CAAC;QACnB,CAAC;IACH,CAAC;IAED,OAAO,EAAE,GAAG,EAAE,CAAC;AACjB,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,gBAAgB,CAC7B,cAAsB;IAEtB,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,cAAc,CAAC,CAAC;QACjD,IAAI,MAAM,CAAC,QAAQ,KAAK,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC;YAClD,OAAO,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7C,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,SAAS;IACX,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAiBD;;GAEG;AACH,KAAK,UAAU,YAAY,CACzB,KAAiB,EACjB,aAAqB,EACrB,WAAmB,EACnB,IAAU,EACV,OAAe;IAEf,MAAM,WAAW,GAAa,EAAE,CAAC;IACjC,IAAI,UAAU,GAAG,EAAE,CAAC;IACpB,IAAI,UAAU,GAAG,EAAE,CAAC;IACpB,IAAI,YAAY,GAAG,CAAC,CAAC,CAAC;IACtB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAEzB,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;YAChC,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YACzB,MAAM,EAAE,aAAa;YACrB,MAAM,EAAE,WAAW;YACnB,KAAK,EAAE;gBACL,WAAW,EAAE;oBACX,WAAW,EAAE,yCAAyC;oBACtD,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;wBACpB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,8BAA8B,CAAC;qBAC7D,CAAC;oBACF,OAAO,EAAE,KAAK,EAAE,KAA0B,EAAE,EAAE;wBAC5C,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;wBAChC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,WAAW,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC;wBACxF,UAAU,GAAG,MAAM,CAAC;wBACpB,UAAU,GAAG,MAAM,CAAC;wBACpB,YAAY,GAAG,QAAQ,CAAC;wBACxB,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;oBACtC,CAAC;iBACF;aACF;YACD,QAAQ,EAAE,WAAW,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,CAAC;YAC1C,WAAW,EAAE,CAAC;SACf,CAAC,CAAC;QAEH,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;QAE1C,IAAI,gBAAgB,GAAG,CAAC,CAAC;QACzB,IAAI,iBAAiB,GAAG,CAAC,CAAC;QAC1B,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YAChC,gBAAgB,IAAI,IAAI,CAAC,KAAK,EAAE,WAAW,IAAI,CAAC,CAAC;YACjD,iBAAiB,IAAI,IAAI,CAAC,KAAK,EAAE,YAAY,IAAI,CAAC,CAAC;QACrD,CAAC;QAED,MAAM,gBAAgB,GAAG,MAAM,aAAa,CAC1C,IAAI,CAAC,MAAM,EACX,EAAE,WAAW,EAAE,UAAU,EAAE,UAAU,EAAE,YAAY,EAAE,EACrD,OAAO,CACR,CAAC;QAEF,MAAM,SAAS,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QAC1D,MAAM,QAAQ,GAAG,gBAAgB;aAC9B,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;aACxB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;YACT,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAE,CAAC;YACzC,OAAO,GAAG,GAAG,cAAc,CAAC,CAAC,QAAQ,IAAI,GAAG,YAAY,CAAC,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC;QAC5E,CAAC,CAAC,CAAC;QAEL,MAAM,iBAAiB,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YACvD,IAAI,EAAE,CAAC;YACP,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC,EAA2B,EAAE,EAAE,CAAC,CAAC;gBAC/D,IAAI,EAAE,EAAE,CAAC,UAAU,CAAC;gBACpB,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC;aAClB,CAAC,CAAC;YACH,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,EAA2B,EAAE,EAAE,CAAC,CAAC;gBACnE,IAAI,EAAE,EAAE,CAAC,UAAU,CAAC;gBACpB,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC;aACrB,CAAC,CAAC;YACH,KAAK,EAAE,IAAI,CAAC,KAAK;SAClB,CAAC,CAAC,CAAC;QAEJ,OAAO;YACL,MAAM,EAAE,SAAS;YACjB,gBAAgB;YAChB,SAAS,EAAE,WAAW,CAAC,MAAM;YAC7B,WAAW;YACX,gBAAgB;YAChB,iBAAiB;YACjB,cAAc;YACd,UAAU;YACV,UAAU;YACV,YAAY;YACZ,iBAAiB;YACjB,aAAa,EAAE,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS;SACrE,CAAC;IACJ,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO;YACL,MAAM,EAAE,KAAK;YACb,gBAAgB,EAAE,EAAE;YACpB,SAAS,EAAE,WAAW,CAAC,MAAM;YAC7B,WAAW;YACX,gBAAgB,EAAE,CAAC;YACnB,iBAAiB,EAAE,CAAC;YACpB,cAAc,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;YAClC,UAAU;YACV,UAAU;YACV,YAAY;YACZ,iBAAiB,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;YAC1E,aAAa,EAAE,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;SAC1D,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,oBAAoB,CAC3B,KAAiB,EACjB,WAAuB,EACvB,QAAkB;IAElB,MAAM,QAAQ,GACZ,WAAW,CAAC,MAAM,GAAG,CAAC;QACpB,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,WAAW,CAAC,MAAM;QACjE,CAAC,CAAC,CAAC,CAAC;IAER,MAAM,YAAY,GAChB,WAAW,CAAC,MAAM,GAAG,CAAC;QACpB,CAAC,CAAC,IAAI,CAAC,KAAK,CACR,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,cAAc,EAAE,CAAC,CAAC,GAAG,WAAW,CAAC,MAAM,CAC3E;QACH,CAAC,CAAC,CAAC,CAAC;IAER,MAAM,WAAW,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IACxD,MAAM,iBAAiB,GACrB,WAAW,CAAC,MAAM,GAAG,CAAC;QACpB,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,GAAG,WAAW,CAAC,MAAM;QACvE,CAAC,CAAC,CAAC,CAAC;IAER,MAAM,cAAc,GAClB,WAAW,CAAC,MAAM,GAAG,CAAC;QACpB,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,gBAAgB,GAAG,CAAC,CAAC,iBAAiB,EAAE,CAAC,CAAC;YAC7E,WAAW,CAAC,MAAM;QACpB,CAAC,CAAC,CAAC,CAAC;IAER,OAAO;QACL,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,OAAO,EAAE,KAAK,CAAC,EAAE;QACjB,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,QAAQ;QACR,WAAW;QACX,QAAQ;QACR,iBAAiB;QACjB,cAAc;QACd,YAAY;KACb,CAAC;AACJ,CAAC;AAkBD,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,IAAoB;IAChD,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;IACxB,MAAM,OAAO,GAAiB,EAAE,CAAC;IACjC,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;IAElD,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;QAC/B,oCAAoC;QACpC,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC;YAC9B,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,YAAY,CAAC,MAAM,CAAC,MAAM,CAAC;YAClD,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,OAAO;YACP,MAAM;YACN,cAAc,EAAE,IAAI,CAAC,cAAc;YACnC,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,YAAY,EAAE,IAAI,CAAC,YAAY;SAChC,CAAC,CAAC;QACH,IAAI,MAAM;YAAE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACjC,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,qCAAqC;IACrC,MAAM,QAAQ,GAAG,IAAI,CACnB,IAAI,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC,EACvF,aAAa,CACd,CAAC;IAEF,MAAM,QAAQ,GAAG,MAAM,kBAAkB,CAAC,QAAQ,CAAC,CAAC;IACpD,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAE3C,MAAM,SAAS,GACb,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC;QACtB,CAAC,CAAC,MAAM,CAAC,MAAM;QACf,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;IAE3B,KAAK,MAAM,OAAO,IAAI,SAAS,EAAE,CAAC;QAChC,MAAM,SAAS,GAAG,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACxC,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,gCAAgC,OAAO,EAAE,CAAC,CAAC;YACzD,SAAS;QACX,CAAC;QAED,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,SAAS,CAAC,CAAC;QAC7C,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC;YAC9B,OAAO;YACP,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,MAAM;YACN,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,OAAO;YACP,MAAM;YACN,cAAc,EAAE,KAAK,CAAC,eAAe;SACtC,CAAC,CAAC;QACH,IAAI,MAAM;YAAE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACnC,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAcD,KAAK,UAAU,UAAU,CAAC,IAAoB;IAC5C,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,GAAG,IAAI,CAAC;IAEpF,OAAO,CAAC,GAAG,CAAC,SAAS,OAAO,MAAM,CAAC,CAAC;IAEpC,0CAA0C;IAC1C,IAAI,SAAS,GAAkC,IAAI,CAAC;IACpD,IAAI,UAA8B,CAAC;IAEnC,IAAI,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;QACnC,MAAM,SAAS,GAAG,MAAM,iBAAiB,CAAC,MAAM,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;QACxE,IAAI,SAAS,EAAE,CAAC;YACd,SAAS,GAAG,SAAS,CAAC,UAAU,CAAC;YACjC,UAAU,GAAG,SAAS,CAAC,OAAO,CAAC;QACjC,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CAAC,+CAA+C,CAAC,CAAC;YAC7D,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,YAAY,CAAC,OAAO,CAAC,CAAC;gBACzC,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC;gBAC5B,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC;YAC5B,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,CAAC,IAAI,CAAC,qCAAqC,OAAO,2BAA2B,CAAC,CAAC;YACxF,CAAC;QACH,CAAC;IACH,CAAC;IAED,qBAAqB;IACrB,IAAI,CAAC,UAAU,IAAI,cAAc,EAAE,CAAC;QAClC,UAAU,GAAG,MAAM,gBAAgB,CAAC,cAAc,CAAC,CAAC;QACpD,IAAI,UAAU;YAAE,OAAO,CAAC,GAAG,CAAC,kBAAkB,UAAU,EAAE,CAAC,CAAC;IAC9D,CAAC;IAED,UAAU;IACV,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QAClB,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QAC3B,IAAI,SAAS,EAAE,CAAC;YACd,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;gBACjC,MAAM,aAAa,GAAG,kBAAkB,CAAC,QAAQ,CAAC,CAAC;gBACnD,MAAM,WAAW,GAAG,gBAAgB,CAAC,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;gBAC9E,OAAO,CAAC,GAAG,CAAC,kBAAkB,OAAO,MAAM,SAAS,CAAC,EAAE,KAAK,QAAQ,OAAO,CAAC,CAAC;gBAC7E,OAAO,CAAC,GAAG,CAAC,WAAW,aAAa,IAAI,CAAC,CAAC;gBAC1C,OAAO,CAAC,GAAG,CAAC,UAAU,WAAW,IAAI,CAAC,CAAC;gBACvC,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC;YACvC,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,YAAY,GAAkB,EAAE,CAAC;IAEvC,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,OAAO,CAAC,GAAG,CAAC,kBAAkB,QAAQ,EAAE,CAAC,CAAC;QAC1C,MAAM,aAAa,GAAG,kBAAkB,CAAC,QAAQ,CAAC,CAAC;QAEnD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,OAAO,CAAC,GAAG,CAAC,YAAY,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC;YAC7C,MAAM,WAAW,GAAe,EAAE,CAAC;YAInC,MAAM,SAAS,GAAe,EAAE,CAAC;YACjC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,MAAM,WAAW,GAAG,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC;gBAC1D,KAAK,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,WAAW,EAAE,EAAE,EAAE,EAAE,CAAC;oBACxC,SAAS,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,EAAE,EAAE,WAAW,EAAE,WAAW,EAAE,CAAC,CAAC;gBACtE,CAAC;YACH,CAAC;YAED,MAAM,YAAY,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,WAAW,EAAE,EAAE,EAAE,CACxE,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE;gBACrB,4DAA4D;gBAC5D,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,IAAI,MAAM,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,YAAY,CAAC,CAAC,CAAC;gBAC9F,MAAM,aAAa,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC;gBAEvD,IAAI,CAAC;oBACH,MAAM,WAAW,GAAG,gBAAgB,CAAC,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAC;oBAEzE,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBACxC,MAAM,QAAQ,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC;oBAC/C,CAAC;oBAED,MAAM,WAAW,GAAG,MAAM,YAAY,CACpC,KAAK,EACL,aAAa,EACb,WAAW,EACX,IAAI,EACJ,OAAO,CACR,CAAC;oBAEF,MAAM,QAAQ,GAAa;wBACzB,MAAM,EAAE,IAAI,CAAC,EAAE;wBACf,MAAM,EAAE,WAAW,CAAC,MAAM;wBAC1B,aAAa,EAAE,WAAW,CAAC,aAAa;wBACxC,gBAAgB,EAAE,WAAW,CAAC,gBAAgB;wBAC9C,SAAS,EAAE,WAAW,CAAC,SAAS;wBAChC,gBAAgB,EAAE,WAAW,CAAC,gBAAgB;wBAC9C,iBAAiB,EAAE,WAAW,CAAC,iBAAiB;wBAChD,cAAc,EAAE,WAAW,CAAC,cAAc;wBAC1C,WAAW,EAAE,WAAW,CAAC,WAAW;wBACpC,UAAU,EAAE,IAAI,CAAC,UAAU;wBAC3B,QAAQ,EAAE,IAAI,CAAC,QAAQ;wBACvB,iBAAiB,EAAE,WAAW,CAAC,iBAAiB;wBAChD,cAAc,EAAE;4BACd,EAAE,EAAE,IAAI,CAAC,EAAE;4BACX,MAAM,EAAE,IAAI,CAAC,MAAM;4BACnB,MAAM,EAAE,IAAI,CAAC,MAAM;4BACnB,KAAK,EAAE,IAAI,CAAC,KAAK;4BACjB,SAAS,EAAE,IAAI,CAAC,SAAS;4BACzB,UAAU,EAAE,IAAI,CAAC,UAAU;4BAC3B,QAAQ,EAAE,IAAI,CAAC,QAAQ;yBACxB;wBACD,WAAW,EAAE,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS;qBACvD,CAAC;oBAEF,MAAM,IAAI,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;oBAC5C,MAAM,WAAW,GAAG,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,WAAW,GAAG,CAAC,IAAI,WAAW,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;oBAClF,OAAO,CAAC,GAAG,CACT,OAAO,IAAI,IAAI,IAAI,CAAC,EAAE,GAAG,WAAW,WAAW,WAAW,CAAC,SAAS,YAAY,WAAW,CAAC,gBAAgB,GAAG,WAAW,CAAC,iBAAiB,GAAG,CAChJ,CAAC;oBAEF,OAAO,QAAQ,CAAC;gBAClB,CAAC;wBAAS,CAAC;oBACT,IAAI,aAAa,EAAE,CAAC;wBAClB,MAAM,EAAE,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;oBACtE,CAAC;gBACH,CAAC;YACH,CAAC,CAAC,CACH,CAAC;YAEF,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;YAChD,KAAK,MAAM,QAAQ,IAAI,OAAO,EAAE,CAAC;gBAC/B,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC7B,CAAC;YAED,YAAY,CAAC,IAAI,CACf,oBAAoB,CAAC,KAAK,EAAE,WAAW,EAAE,QAAQ,CAAC,CACnD,CAAC;QACJ,CAAC;IACH,CAAC;IAED,MAAM,UAAU,GAAG,QAAQ,EAAE,CAAC;IAC9B,MAAM,aAAa,GAAG,oBAAoB,CAAC,KAAK,CAAC,CAAC;IAElD,6CAA6C;IAC7C,MAAM,UAAU,GAAG,IAAI,CAAC,YAAY,KAAK,SAAS,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC;IAChG,IAAI,UAA8B,CAAC;IACnC,IAAI,UAAU,EAAE,CAAC;QACf,UAAU,GAAG,CAAC,CAAC;QACf,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,UAAU,IAAI,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC;QACtD,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAe;QACzB,GAAG,EAAE,OAAO;QACZ,UAAU;QACV,gBAAgB,EAAE,GAAG;QACrB,YAAY;QACZ,SAAS,EAAE,KAAK,CAAC,MAAM;QACvB,UAAU;QACV,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACrC,WAAW,EAAE,OAAO;QACpB,YAAY,EAAE,kBAAkB,CAAC,SAAS,CAAC,CAAC,CAAE,CAAC;QAC/C,GAAG,UAAU;QACb,aAAa;KACd,CAAC;IAEF,kBAAkB,CAAC,MAAM,CAAC,CAAC;IAC3B,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,kBAAkB,CAAC,MAAkB;IAC5C,OAAO,CAAC,GAAG,CAAC,SAAS,MAAM,CAAC,GAAG,cAAc,CAAC,CAAC;IAC/C,OAAO,CAAC,GAAG,CAAC,UAAU,MAAM,CAAC,SAAS,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,YAAY,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACrG,KAAK,MAAM,EAAE,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;QACrC,OAAO,CAAC,GAAG,CACT,KAAK,EAAE,CAAC,WAAW,KAAK,EAAE,CAAC,QAAQ,WAAW,CAAC,EAAE,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,cAAc,EAAE,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC,cAAc,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,cAAc,CAAC,EAAE,CACvK,CAAC;QAEF,2CAA2C;QAC3C,IAAI,MAAM,CAAC,UAAU,IAAI,MAAM,CAAC,UAAU,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;YAC9D,MAAM,MAAM,GAAG,IAAI,GAAG,EAA6C,CAAC;YACpE,KAAK,MAAM,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;gBAChC,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;gBAC/D,KAAK,CAAC,KAAK,EAAE,CAAC;gBACd,IAAI,EAAE,CAAC,MAAM;oBAAE,KAAK,CAAC,MAAM,EAAE,CAAC;gBAC9B,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;YAC/B,CAAC;YACD,KAAK,MAAM,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,IAAI,MAAM,EAAE,CAAC;gBACjD,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;oBACd,OAAO,CAAC,GAAG,CAAC,OAAO,MAAM,KAAK,MAAM,IAAI,KAAK,KAAK,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC;gBAC5F,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,MAAkB,EAClB,UAAkB,EAClB,MAAc;IAEd,MAAM,EAAE,YAAY,EAAE,YAAY,EAAE,GAAG,MAAM,MAAM,CAAC,0BAA0B,CAAC,CAAC;IAChF,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,qBAAqB,CAAC,CAAC;IAEzD,MAAM,YAAY,GAAG,YAAY,CAAC,YAAY,CAAC;QAC7C,OAAO,EAAE,UAAU;KACpB,CAAC,CAAC,CAAC;IAEJ,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,QAAQ,CAAC,qBAAqB,CAAC;QACrD,MAAM,EAAE,YAAY;QACpB,OAAO,EAAE,EAAE,aAAa,EAAE,UAAU,MAAM,EAAE,EAAE;QAC9C,IAAI,EAAE;YACJ,GAAG,MAAM;YACT,gBAAgB,EAAE,MAAM,CAAC,gBAAuD;SACjF;KACF,CAAC,CAAC;IAEH,IAAI,KAAK,EAAE,CAAC;QACV,MAAM,IAAI,KAAK,CAAC,kBAAkB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAC7D,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,4BAA4B,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC;AACxD,CAAC"}
|