@cogitator-ai/evals 0.13.2 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. package/README.md +94 -14
  2. package/dist/index.d.ts +2 -0
  3. package/dist/index.d.ts.map +1 -1
  4. package/dist/index.js +1 -0
  5. package/dist/index.js.map +1 -1
  6. package/dist/plugin-evals/execute.d.ts +45 -0
  7. package/dist/plugin-evals/execute.d.ts.map +1 -0
  8. package/dist/plugin-evals/execute.js +508 -0
  9. package/dist/plugin-evals/execute.js.map +1 -0
  10. package/dist/plugin-evals/files.d.ts +17 -0
  11. package/dist/plugin-evals/files.d.ts.map +1 -0
  12. package/dist/plugin-evals/files.js +46 -0
  13. package/dist/plugin-evals/files.js.map +1 -0
  14. package/dist/plugin-evals/glob.d.ts +8 -0
  15. package/dist/plugin-evals/glob.d.ts.map +1 -0
  16. package/dist/plugin-evals/glob.js +63 -0
  17. package/dist/plugin-evals/glob.js.map +1 -0
  18. package/dist/plugin-evals/graders.d.ts +21 -0
  19. package/dist/plugin-evals/graders.d.ts.map +1 -0
  20. package/dist/plugin-evals/graders.js +198 -0
  21. package/dist/plugin-evals/graders.js.map +1 -0
  22. package/dist/plugin-evals/index.d.ts +4 -0
  23. package/dist/plugin-evals/index.d.ts.map +1 -0
  24. package/dist/plugin-evals/index.js +3 -0
  25. package/dist/plugin-evals/index.js.map +1 -0
  26. package/dist/plugin-evals/judge.d.ts +33 -0
  27. package/dist/plugin-evals/judge.d.ts.map +1 -0
  28. package/dist/plugin-evals/judge.js +73 -0
  29. package/dist/plugin-evals/judge.js.map +1 -0
  30. package/dist/plugin-evals/mocks.d.ts +34 -0
  31. package/dist/plugin-evals/mocks.d.ts.map +1 -0
  32. package/dist/plugin-evals/mocks.js +373 -0
  33. package/dist/plugin-evals/mocks.js.map +1 -0
  34. package/dist/plugin-evals/report.d.ts +4 -0
  35. package/dist/plugin-evals/report.d.ts.map +1 -0
  36. package/dist/plugin-evals/report.js +177 -0
  37. package/dist/plugin-evals/report.js.map +1 -0
  38. package/dist/plugin-evals/runner.d.ts +23 -0
  39. package/dist/plugin-evals/runner.d.ts.map +1 -0
  40. package/dist/plugin-evals/runner.js +582 -0
  41. package/dist/plugin-evals/runner.js.map +1 -0
  42. package/dist/plugin-evals/schema.d.ts +179 -0
  43. package/dist/plugin-evals/schema.d.ts.map +1 -0
  44. package/dist/plugin-evals/schema.js +194 -0
  45. package/dist/plugin-evals/schema.js.map +1 -0
  46. package/dist/plugin-evals/suite.d.ts +40 -0
  47. package/dist/plugin-evals/suite.d.ts.map +1 -0
  48. package/dist/plugin-evals/suite.js +476 -0
  49. package/dist/plugin-evals/suite.js.map +1 -0
  50. package/dist/plugin-evals/tools.d.ts +42 -0
  51. package/dist/plugin-evals/tools.d.ts.map +1 -0
  52. package/dist/plugin-evals/tools.js +136 -0
  53. package/dist/plugin-evals/tools.js.map +1 -0
  54. package/dist/plugin-evals/transcript.d.ts +35 -0
  55. package/dist/plugin-evals/transcript.d.ts.map +1 -0
  56. package/dist/plugin-evals/transcript.js +149 -0
  57. package/dist/plugin-evals/transcript.js.map +1 -0
  58. package/dist/plugin-evals/types.d.ts +378 -0
  59. package/dist/plugin-evals/types.d.ts.map +1 -0
  60. package/dist/plugin-evals/types.js +2 -0
  61. package/dist/plugin-evals/types.js.map +1 -0
  62. package/dist/reporters/html-format.d.ts +5 -2
  63. package/dist/reporters/html-format.d.ts.map +1 -1
  64. package/dist/reporters/html-format.js +6 -3
  65. package/dist/reporters/html-format.js.map +1 -1
  66. package/package.json +5 -3
@@ -0,0 +1,508 @@
1
+ import { spawn } from 'node:child_process';
2
+ import { cpSync, existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, } from 'node:fs';
3
+ import { tmpdir } from 'node:os';
4
+ import { join, relative } from 'node:path';
5
+ import { z } from 'zod';
6
+ import { errorText, posixPath } from './files.js';
7
+ import { expectViolation, MockAbortError, renderMock } from './mocks.js';
8
+ import { isRecord } from './schema.js';
9
+ import { claudeNameOf, grantCovers, isGranted, WORKSPACE_TOOLS } from './tools.js';
10
+ import { parseTranscript } from './transcript.js';
11
+ /** Longest a scaffold script may run */
12
+ const SCAFFOLD_TIMEOUT_MS = 120_000;
13
+ const BASE_INSTRUCTIONS = [
14
+ 'You are an AI agent working on a request of the user in a workspace of your own.',
15
+ 'Use the tools you have to do the task well, then answer the user with the result.',
16
+ ].join('\n');
17
+ /** The subagent a granted `Agent` tool reaches in both arms, as Claude Code's built-in one */
18
+ const GENERAL_PURPOSE = {
19
+ description: 'General-purpose agent for researching questions, searching files and doing multi-step tasks. Use it when a task would take many steps and you only need its result.',
20
+ prompt: 'You do a task another agent handed you, with the tools you have. Work until it is done, then answer with a concise report of what you found or changed: the other agent reads only your final answer.',
21
+ };
22
+ /** Whether `server` is declared by a plugin loaded in this arm */
23
+ function declaredBy(spec, server) {
24
+ if (spec.arm !== 'with')
25
+ return undefined;
26
+ for (const [plugin, servers] of spec.pluginServers)
27
+ if (servers.has(server))
28
+ return plugin;
29
+ return undefined;
30
+ }
31
+ function mcpNamePart(name) {
32
+ return name.replace(/[^A-Za-z0-9_-]/g, '_');
33
+ }
34
+ /** The tool name of a mocked MCP tool: `mcp__plugin_<plugin>_<server>__<tool>` for a plugin's server */
35
+ export function mockToolName(plugin, server, tool) {
36
+ const serverPart = plugin === undefined ? server : `plugin_${plugin}_${server}`;
37
+ return `mcp__${mcpNamePart(serverPart)}__${mcpNamePart(tool)}`;
38
+ }
39
+ function listFiles(root) {
40
+ const found = [];
41
+ const walk = (dir) => {
42
+ for (const entry of readdirSync(dir, { withFileTypes: true })) {
43
+ const full = join(dir, entry.name);
44
+ if (entry.isDirectory())
45
+ walk(full);
46
+ else
47
+ found.push(posixPath(relative(root, full)));
48
+ }
49
+ };
50
+ if (existsSync(root))
51
+ walk(root);
52
+ return found.sort();
53
+ }
54
+ function scaffoldEnv(home, tmp) {
55
+ return {
56
+ PATH: process.env.PATH ?? '/usr/bin:/bin',
57
+ HOME: home,
58
+ TMPDIR: tmp,
59
+ TERM: 'dumb',
60
+ LANG: process.env.LANG ?? 'C.UTF-8',
61
+ };
62
+ }
63
+ /** Runs the scaffold script in the empty workspace, the reason it failed or undefined */
64
+ function runScaffold(script, cwd, home, tmp) {
65
+ return new Promise((resolve) => {
66
+ const child = spawn('bash', [script], {
67
+ cwd,
68
+ env: scaffoldEnv(home, tmp),
69
+ stdio: ['ignore', 'ignore', 'pipe'],
70
+ });
71
+ let stderr = '';
72
+ let timedOut = false;
73
+ const timer = setTimeout(() => {
74
+ timedOut = true;
75
+ child.kill('SIGKILL');
76
+ }, SCAFFOLD_TIMEOUT_MS);
77
+ child.stderr.on('data', (chunk) => {
78
+ stderr = (stderr + chunk.toString('utf8')).slice(-2000);
79
+ });
80
+ child.on('error', (error) => {
81
+ clearTimeout(timer);
82
+ resolve(`scaffold failed: ${error.message}`);
83
+ });
84
+ child.on('close', (code) => {
85
+ clearTimeout(timer);
86
+ if (timedOut)
87
+ resolve(`scaffold failed: it ran longer than ${SCAFFOLD_TIMEOUT_MS / 1000}s`);
88
+ else if (code !== 0) {
89
+ const tail = stderr.trim();
90
+ resolve(`scaffold failed: exit ${code}${tail === '' ? '' : `: ${tail}`}`);
91
+ }
92
+ else
93
+ resolve(undefined);
94
+ });
95
+ });
96
+ }
97
+ /** The tool names of a Claude Code transcript's calls, as the agent's tools are named */
98
+ const COGITATOR_NAME_OF = {
99
+ Read: 'read_file',
100
+ Write: 'write_file',
101
+ Edit: 'edit_file',
102
+ Glob: 'glob',
103
+ Grep: 'grep',
104
+ LS: 'list_files',
105
+ Bash: 'run_command',
106
+ Skill: 'use_skill',
107
+ TodoWrite: 'write_todos',
108
+ Agent: 'agent',
109
+ Task: 'agent',
110
+ WebFetch: 'http_request',
111
+ WebSearch: 'web_search',
112
+ };
113
+ function historyOf(file) {
114
+ const parsed = parseTranscript(readFileSync(file, 'utf8'));
115
+ if ('error' in parsed)
116
+ throw new Error(`history_file: ${parsed.error}`);
117
+ return parsed.map((entry) => {
118
+ switch (entry.role) {
119
+ case 'user':
120
+ return { role: 'user', content: entry.text };
121
+ case 'assistant':
122
+ return entry.toolCalls.length === 0
123
+ ? { role: 'assistant', content: entry.text }
124
+ : {
125
+ role: 'assistant',
126
+ content: entry.text,
127
+ toolCalls: entry.toolCalls.map((call) => ({
128
+ id: call.id,
129
+ name: COGITATOR_NAME_OF[call.name] ?? call.name,
130
+ arguments: call.input,
131
+ })),
132
+ };
133
+ case 'tool':
134
+ return {
135
+ role: 'tool',
136
+ content: entry.content,
137
+ toolCallId: entry.toolUseId,
138
+ name: COGITATOR_NAME_OF[entry.name] ?? entry.name,
139
+ };
140
+ }
141
+ });
142
+ }
143
+ function resultText(value) {
144
+ if (typeof value === 'string')
145
+ return value;
146
+ if (value === undefined)
147
+ return '';
148
+ try {
149
+ return JSON.stringify(value);
150
+ }
151
+ catch {
152
+ return String(value);
153
+ }
154
+ }
155
+ /** Whether `error` (or an error it wraps) is a model provider's, and whether it rejected the key */
156
+ function providerError(error) {
157
+ let current = error;
158
+ for (let depth = 0; depth < 5 && isRecord(current); depth++) {
159
+ const code = typeof current.code === 'string' ? current.code : '';
160
+ const details = isRecord(current.details) ? current.details : {};
161
+ const status = typeof details.statusCode === 'number' ? details.statusCode : undefined;
162
+ const kind = details.kind;
163
+ if (current.name === 'LLMError' || code.startsWith('LLM_')) {
164
+ return {
165
+ api: true,
166
+ auth: kind === 'authentication' || kind === 'permission' || status === 401 || status === 403,
167
+ };
168
+ }
169
+ current = current.cause;
170
+ }
171
+ return { api: false, auth: false };
172
+ }
173
+ /**
174
+ * The middleware of the eval's runtime, so every run gets it, the agents the agent under test
175
+ * delegates to included: it hides the tools the run was not granted from the model, refuses calls
176
+ * to them and lets granted calls run without asking, and records the tool calls under Claude
177
+ * Code's names for the graders (a delegated agent's with its name in `agent`), and the main run's
178
+ * messages for the transcript
179
+ */
180
+ function recorder(core, grants, mocked, place, recording) {
181
+ const root = place.projectRoot;
182
+ const offered = (name) => mocked.has(name) || isGranted(grants, name);
183
+ const pending = new Map();
184
+ return {
185
+ name: 'plugin-eval-grants',
186
+ wrapModelCall: (call, next) => {
187
+ const tools = call.request.tools?.filter((tool) => offered(tool.name));
188
+ return next({ ...call, request: { ...call.request, ...(tools && { tools }) } });
189
+ },
190
+ afterModel: (event, context) => {
191
+ if (context.parentRunId !== undefined)
192
+ return;
193
+ const calls = (event.response.toolCalls ?? []).map((call) => {
194
+ const recorded = {
195
+ id: call.id,
196
+ name: claudeNameOf(call.name),
197
+ input: call.arguments,
198
+ };
199
+ pending.set(call.id, recorded);
200
+ return recorded;
201
+ });
202
+ recording.entries.push({ role: 'assistant', text: event.response.content, toolCalls: calls });
203
+ if (event.response.content.trim() !== '')
204
+ recording.lastMessage = event.response.content;
205
+ },
206
+ beforeToolCall: async (event, context) => {
207
+ const { tool, toolCall } = event;
208
+ if (!mocked.has(tool.name)) {
209
+ const matching = grants.filter((grant) => grantCovers(grant.tool, tool.name));
210
+ if (matching.length === 0) {
211
+ return { decision: 'deny', reason: `${tool.name} is not granted in this eval run` };
212
+ }
213
+ if (!matching.some((grant) => grant.specifier === undefined)) {
214
+ let allowed = false;
215
+ for (const grant of matching) {
216
+ if (await core.ruleAllowsCall(grant.rule, tool, toolCall.arguments, place)) {
217
+ allowed = true;
218
+ break;
219
+ }
220
+ }
221
+ if (!allowed) {
222
+ return {
223
+ decision: 'deny',
224
+ reason: `this call is outside what the eval run grants: ${matching.map((g) => g.rule).join(', ')}`,
225
+ };
226
+ }
227
+ }
228
+ }
229
+ const input = core.claudeToolOf(tool)?.input(toolCall.arguments, root) ?? toolCall.arguments;
230
+ const name = tool.mcp ? tool.name : claudeNameOf(tool.name);
231
+ const delegated = context.parentRunId !== undefined;
232
+ recording.toolCalls.push({
233
+ id: toolCall.id,
234
+ name,
235
+ tool: tool.name,
236
+ input,
237
+ ...(delegated && { agent: context.agentName }),
238
+ });
239
+ const entry = delegated ? undefined : pending.get(toolCall.id);
240
+ if (entry)
241
+ entry.input = input;
242
+ return { decision: 'allow' };
243
+ },
244
+ wrapToolCall: async (call, next, context) => {
245
+ const result = await next(call);
246
+ if (context.parentRunId === undefined) {
247
+ recording.entries.push({
248
+ role: 'tool',
249
+ toolUseId: call.toolCall.id,
250
+ name: claudeNameOf(call.tool.name),
251
+ content: result.error ?? resultText(result.result),
252
+ isError: result.error !== undefined,
253
+ });
254
+ }
255
+ return result;
256
+ },
257
+ };
258
+ }
259
+ const MOCK_PARAMETERS = z.record(z.string(), z.unknown());
260
+ function mockTool(core, mock, name, origin, record, abort) {
261
+ const description = mock.description ?? `The ${mock.tool} tool of the ${mock.server} MCP server`;
262
+ const parameters = mock.inputSchema
263
+ ? core.toToolParameters({ ...mock.inputSchema, type: 'object' })
264
+ : { type: 'object', properties: {}, additionalProperties: true };
265
+ return {
266
+ name,
267
+ description,
268
+ parameters: MOCK_PARAMETERS,
269
+ mcp: { server: origin, tool: mock.tool },
270
+ execute: async (input) => {
271
+ const violation = expectViolation(mock, input);
272
+ if (violation)
273
+ abort({ server: mock.server, tool: mock.tool, reason: `expect: ${violation}` });
274
+ let output;
275
+ try {
276
+ output = renderMock(mock, input);
277
+ }
278
+ catch (error) {
279
+ record({
280
+ server: mock.server,
281
+ tool: mock.tool,
282
+ input,
283
+ output: errorText(error),
284
+ error: true,
285
+ });
286
+ throw error;
287
+ }
288
+ record({ server: mock.server, tool: mock.tool, input, output, error: mock.error });
289
+ if (mock.error)
290
+ throw new Error(output);
291
+ return output;
292
+ },
293
+ toJSON: () => ({ name, description, parameters }),
294
+ };
295
+ }
296
+ function instructionsOf(core, spec, workspaceTools, todos, subagents) {
297
+ const parts = [
298
+ BASE_INSTRUCTIONS,
299
+ core.harnessPrompt({ workspaceTools, todos, subagents, permissions: true }),
300
+ ];
301
+ const added = spec.case.context.addDirs;
302
+ if (added.length > 0) {
303
+ parts.push(`Reference directories of the workspace, read-only: ${added.map((dir) => `${dir}/`).join(', ')}`);
304
+ }
305
+ if (spec.case.appendSystemPrompt !== undefined)
306
+ parts.push(spec.case.appendSystemPrompt);
307
+ return parts.filter((part) => part.trim() !== '').join('\n\n');
308
+ }
309
+ /**
310
+ * Runs the case once in a fresh temporary workspace: the scaffold, the read-only fixture
311
+ * directories, an agent with the granted tools and, in the with-arm, the case's plugins. The run
312
+ * never stops to ask: what is not granted is not given
313
+ */
314
+ export async function executeRun(spec) {
315
+ const { core, case: evalCase } = spec;
316
+ const started = Date.now();
317
+ const tempDir = mkdtempSync(join(tmpdir(), 'cogitator-plugin-eval-'));
318
+ const root = join(tempDir, 'workspace');
319
+ const home = join(tempDir, 'home');
320
+ const tmp = join(tempDir, 'tmp');
321
+ for (const dir of [root, home, tmp])
322
+ mkdirSync(dir, { recursive: true });
323
+ const cleanup = () => {
324
+ if (!spec.keepTemp)
325
+ rmSync(tempDir, { recursive: true, force: true });
326
+ };
327
+ const seconds = () => (Date.now() - started) / 1000;
328
+ const ended = (fields) => ({
329
+ costUsd: 0,
330
+ durationSeconds: seconds(),
331
+ ...(spec.keepTemp && { tempDir }),
332
+ cleanup,
333
+ ...fields,
334
+ });
335
+ if (evalCase.context.scaffoldScript && spec.scaffold) {
336
+ const failed = await runScaffold(evalCase.context.scaffoldScript, root, home, tmp);
337
+ if (failed)
338
+ return ended({ error: failed });
339
+ }
340
+ for (const dir of evalCase.context.addDirs) {
341
+ const into = join(root, dir);
342
+ if (existsSync(into)) {
343
+ return ended({ error: `context.add_dirs ${dir} collides with a path the scaffold created` });
344
+ }
345
+ cpSync(join(evalCase.dir, dir), into, { recursive: true });
346
+ }
347
+ const before = new Set(listFiles(root));
348
+ const recording = {
349
+ entries: [{ role: 'user', text: evalCase.prompt }],
350
+ toolCalls: [],
351
+ mockCalls: [],
352
+ lastMessage: '',
353
+ };
354
+ const controller = new AbortController();
355
+ let aborted;
356
+ const abort = (reason) => {
357
+ aborted ??= reason;
358
+ const error = new MockAbortError(reason);
359
+ controller.abort(error);
360
+ throw error;
361
+ };
362
+ const record = (call) => recording.mockCalls.push(call);
363
+ const mockedNames = new Set();
364
+ const ownMocks = [];
365
+ const pluginMocks = new Map();
366
+ for (const [server, tools] of Object.entries(evalCase.mocks)) {
367
+ const plugin = declaredBy(spec, server);
368
+ if (plugin !== undefined) {
369
+ pluginMocks.set(`plugin:${plugin}:${server}`, tools);
370
+ for (const mock of tools)
371
+ mockedNames.add(mockToolName(plugin, server, mock.tool));
372
+ continue;
373
+ }
374
+ if ([...spec.pluginServers.values()].some((servers) => servers.has(server)))
375
+ continue;
376
+ for (const mock of tools) {
377
+ const name = mockToolName(undefined, server, mock.tool);
378
+ mockedNames.add(name);
379
+ ownMocks.push(mockTool(core, mock, name, server, record, abort));
380
+ }
381
+ }
382
+ const workspace = new core.LocalWorkspace({
383
+ root,
384
+ exec: { ...spec.exec, env: { ...evalCase.env, HOME: home, TMPDIR: tmp } },
385
+ });
386
+ const workspaceNames = WORKSPACE_TOOLS.filter((name) => isGranted(spec.grants, name));
387
+ const todos = isGranted(spec.grants, 'write_todos');
388
+ const subagents = isGranted(spec.grants, 'agent');
389
+ const tools = [
390
+ ...(workspaceNames.length > 0
391
+ ? core.workspaceTools(workspace, { tools: workspaceNames, requireApproval: () => false })
392
+ : []),
393
+ ...(isGranted(spec.grants, 'http_request') ? [core.httpRequest] : []),
394
+ ...(isGranted(spec.grants, 'web_search') ? [core.webSearch] : []),
395
+ ...ownMocks,
396
+ ];
397
+ const agent = new core.Agent({
398
+ name: 'plugin-eval',
399
+ ...(spec.model !== undefined && { model: spec.model }),
400
+ instructions: instructionsOf(core, spec, workspaceNames, todos, subagents),
401
+ tools,
402
+ workspace,
403
+ ...(todos && { todos: true }),
404
+ maxIterations: evalCase.maxTurns,
405
+ onIterationLimit: 'stop',
406
+ });
407
+ const connector = async (servers) => ({
408
+ tools: servers.flatMap(({ name }) => {
409
+ const [, plugin = '', server = ''] = /^plugin:([^:]+):(.+)$/.exec(name) ?? [];
410
+ return (pluginMocks.get(name) ?? []).map((mock) => mockTool(core, mock, mockToolName(plugin, server, mock.tool), name, record, abort));
411
+ }),
412
+ close: async () => undefined,
413
+ });
414
+ const cogitator = new core.Cogitator({
415
+ llm: spec.llm,
416
+ ...(spec.arm === 'with' && {
417
+ experimental_plugins: {
418
+ dirs: evalCase.plugins,
419
+ root,
420
+ installed: { root: join(tempDir, 'plugins') },
421
+ settings: false,
422
+ mcp: connector,
423
+ },
424
+ }),
425
+ ...(subagents && {
426
+ subagents: { files: false, agents: { 'general-purpose': GENERAL_PURPOSE } },
427
+ }),
428
+ middleware: [
429
+ recorder(core, spec.grants, mockedNames, { workspace, projectRoot: root }, recording),
430
+ ],
431
+ });
432
+ const timeoutMs = evalCase.timeoutSeconds * 1000;
433
+ let timedOut = false;
434
+ const timer = setTimeout(() => {
435
+ timedOut = true;
436
+ controller.abort(new Error(`timed out after ${evalCase.timeoutSeconds}s`));
437
+ }, timeoutMs);
438
+ const onInterrupt = () => controller.abort(new Error('interrupted'));
439
+ if (spec.signal.aborted)
440
+ onInterrupt();
441
+ else
442
+ spec.signal.addEventListener('abort', onInterrupt, { once: true });
443
+ let costUsd = 0;
444
+ let error = null;
445
+ let facts = { api: false, auth: false };
446
+ try {
447
+ const deny = evalCase.context.addDirs.flatMap((dir) => [`Edit(${dir}/**)`, `Write(${dir}/**)`]);
448
+ const result = await cogitator.run(agent, {
449
+ input: evalCase.prompt,
450
+ signal: controller.signal,
451
+ useMemory: false,
452
+ saveHistory: false,
453
+ permissions: { mode: 'dontAsk', ...(deny.length > 0 && { deny }) },
454
+ ...(evalCase.context.historyFile && { history: historyOf(evalCase.context.historyFile) }),
455
+ onRunError: (_error, _runId, usage) => {
456
+ costUsd = usage?.cost ?? costUsd;
457
+ },
458
+ });
459
+ costUsd = result.usage.cost;
460
+ if (result.output.trim() !== '')
461
+ recording.lastMessage = result.output;
462
+ if (result.iterationLimitReached)
463
+ error = `reached max_turns (${evalCase.maxTurns})`;
464
+ else if (result.status === 'paused')
465
+ error = 'the run paused waiting for an approval';
466
+ }
467
+ catch (thrown) {
468
+ if (aborted === undefined) {
469
+ if (timedOut)
470
+ error = `timed out after ${evalCase.timeoutSeconds}s`;
471
+ else if (spec.signal.aborted)
472
+ error = 'interrupted';
473
+ else {
474
+ facts = providerError(thrown);
475
+ error = facts.api ? `API error: ${errorText(thrown)}` : errorText(thrown);
476
+ }
477
+ }
478
+ }
479
+ finally {
480
+ clearTimeout(timer);
481
+ spec.signal.removeEventListener('abort', onInterrupt);
482
+ await cogitator.close().catch(() => undefined);
483
+ await workspace.dispose().catch(() => undefined);
484
+ }
485
+ if (aborted)
486
+ return ended({ error: null, aborted, costUsd });
487
+ if (error === 'interrupted')
488
+ return ended({ error, interrupted: true, costUsd });
489
+ if (facts.api) {
490
+ return ended({ error, apiError: true, ...(facts.auth && { authFailed: true }), costUsd });
491
+ }
492
+ const filesCreated = listFiles(root).filter((path) => !before.has(path));
493
+ return ended({
494
+ error,
495
+ costUsd,
496
+ capture: {
497
+ prompt: evalCase.prompt,
498
+ lastMessage: recording.lastMessage,
499
+ entries: recording.entries,
500
+ toolCalls: recording.toolCalls,
501
+ mockCalls: recording.mockCalls,
502
+ filesCreated,
503
+ workspaceRoot: root,
504
+ caseDir: evalCase.dir,
505
+ },
506
+ });
507
+ }
508
+ //# sourceMappingURL=execute.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"execute.js","sourceRoot":"","sources":["../../src/plugin-evals/execute.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EACL,MAAM,EACN,UAAU,EACV,SAAS,EACT,WAAW,EACX,WAAW,EACX,YAAY,EACZ,MAAM,GACP,MAAM,SAAS,CAAC;AACjB,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAC3C,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AASxB,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAE/C,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACtE,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AACpC,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,SAAS,EAAE,eAAe,EAAkB,MAAM,SAAS,CAAC;AAChG,OAAO,EAAE,eAAe,EAAiD,MAAM,cAAc,CAAC;AAa9F,wCAAwC;AACxC,MAAM,mBAAmB,GAAG,OAAO,CAAC;AAEpC,MAAM,iBAAiB,GAAG;IACxB,kFAAkF;IAClF,mFAAmF;CACpF,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAEb,8FAA8F;AAC9F,MAAM,eAAe,GAAG;IACtB,WAAW,EACT,qKAAqK;IACvK,MAAM,EACJ,uMAAuM;CAC1M,CAAC;AAkCF,kEAAkE;AAClE,SAAS,UAAU,CAAC,IAAa,EAAE,MAAc;IAC/C,IAAI,IAAI,CAAC,GAAG,KAAK,MAAM;QAAE,OAAO,SAAS,CAAC;IAC1C,KAAK,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,IAAI,CAAC,aAAa;QAAE,IAAI,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC;YAAE,OAAO,MAAM,CAAC;IAC3F,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,WAAW,CAAC,IAAY;IAC/B,OAAO,IAAI,CAAC,OAAO,CAAC,iBAAiB,EAAE,GAAG,CAAC,CAAC;AAC9C,CAAC;AAED,wGAAwG;AACxG,MAAM,UAAU,YAAY,CAAC,MAA0B,EAAE,MAAc,EAAE,IAAY;IACnF,MAAM,UAAU,GAAG,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,UAAU,MAAM,IAAI,MAAM,EAAE,CAAC;IAChF,OAAO,QAAQ,WAAW,CAAC,UAAU,CAAC,KAAK,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;AACjE,CAAC;AAED,SAAS,SAAS,CAAC,IAAY;IAC7B,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,IAAI,GAAG,CAAC,GAAW,EAAE,EAAE;QAC3B,KAAK,MAAM,KAAK,IAAI,WAAW,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;YAC9D,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YACnC,IAAI,KAAK,CAAC,WAAW,EAAE;gBAAE,IAAI,CAAC,IAAI,CAAC,CAAC;;gBAC/B,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;QACnD,CAAC;IACH,CAAC,CAAC;IACF,IAAI,UAAU,CAAC,IAAI,CAAC;QAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IACjC,OAAO,KAAK,CAAC,IAAI,EAAE,CAAC;AACtB,CAAC;AAED,SAAS,WAAW,CAAC,IAAY,EAAE,GAAW;IAC5C,OAAO;QACL,IAAI,EAAE,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,eAAe;QACzC,IAAI,EAAE,IAAI;QACV,MAAM,EAAE,GAAG;QACX,IAAI,EAAE,MAAM;QACZ,IAAI,EAAE,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,SAAS;KACpC,CAAC;AACJ,CAAC;AAED,yFAAyF;AACzF,SAAS,WAAW,CAClB,MAAc,EACd,GAAW,EACX,IAAY,EACZ,GAAW;IAEX,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC7B,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,MAAM,CAAC,EAAE;YACpC,GAAG;YACH,GAAG,EAAE,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC;YAC3B,KAAK,EAAE,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,CAAC;SACpC,CAAC,CAAC;QACH,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC5B,QAAQ,GAAG,IAAI,CAAC;YAChB,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACxB,CAAC,EAAE,mBAAmB,CAAC,CAAC;QACxB,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE;YACxC,MAAM,GAAG,CAAC,MAAM,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC;QAC1D,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;YAC1B,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,OAAO,CAAC,oBAAoB,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAC/C,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE;YACzB,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,IAAI,QAAQ;gBAAE,OAAO,CAAC,uCAAuC,mBAAmB,GAAG,IAAI,GAAG,CAAC,CAAC;iBACvF,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;gBACpB,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC;gBAC3B,OAAO,CAAC,yBAAyB,IAAI,GAAG,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC,CAAC;YAC5E,CAAC;;gBAAM,OAAO,CAAC,SAAS,CAAC,CAAC;QAC5B,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED,yFAAyF;AACzF,MAAM,iBAAiB,GAAqC;IAC1D,IAAI,EAAE,WAAW;IACjB,KAAK,EAAE,YAAY;IACnB,IAAI,EAAE,WAAW;IACjB,IAAI,EAAE,MAAM;IACZ,IAAI,EAAE,MAAM;IACZ,EAAE,EAAE,YAAY;IAChB,IAAI,EAAE,aAAa;IACnB,KAAK,EAAE,WAAW;IAClB,SAAS,EAAE,aAAa;IACxB,KAAK,EAAE,OAAO;IACd,IAAI,EAAE,OAAO;IACb,QAAQ,EAAE,cAAc;IACxB,SAAS,EAAE,YAAY;CACxB,CAAC;AAKF,SAAS,SAAS,CAAC,IAAY;IAC7B,MAAM,MAAM,GAAG,eAAe,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;IAC3D,IAAI,OAAO,IAAI,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,iBAAiB,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;IACxE,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAkB,EAAE;QAC1C,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;YACnB,KAAK,MAAM;gBACT,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC;YAC/C,KAAK,WAAW;gBACd,OAAO,KAAK,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC;oBACjC,CAAC,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,KAAK,CAAC,IAAI,EAAE;oBAC5C,CAAC,CAAC;wBACE,IAAI,EAAE,WAAW;wBACjB,OAAO,EAAE,KAAK,CAAC,IAAI;wBACnB,SAAS,EAAE,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;4BACxC,EAAE,EAAE,IAAI,CAAC,EAAE;4BACX,IAAI,EAAE,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI;4BAC/C,SAAS,EAAE,IAAI,CAAC,KAAK;yBACtB,CAAC,CAAC;qBACJ,CAAC;YACR,KAAK,MAAM;gBACT,OAAO;oBACL,IAAI,EAAE,MAAM;oBACZ,OAAO,EAAE,KAAK,CAAC,OAAO;oBACtB,UAAU,EAAE,KAAK,CAAC,SAAS;oBAC3B,IAAI,EAAE,iBAAiB,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,IAAI;iBAClD,CAAC;QACN,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,UAAU,CAAC,KAAc;IAChC,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC5C,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,EAAE,CAAC;IACnC,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IAC/B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;IACvB,CAAC;AACH,CAAC;AAOD,oGAAoG;AACpG,SAAS,aAAa,CAAC,KAAc;IACnC,IAAI,OAAO,GAAY,KAAK,CAAC;IAC7B,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,CAAC,IAAI,QAAQ,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC;QAC5D,MAAM,IAAI,GAAG,OAAO,OAAO,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;QAClE,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;QACjE,MAAM,MAAM,GAAG,OAAO,OAAO,CAAC,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC;QACvF,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QAC1B,IAAI,OAAO,CAAC,IAAI,KAAK,UAAU,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3D,OAAO;gBACL,GAAG,EAAE,IAAI;gBACT,IAAI,EACF,IAAI,KAAK,gBAAgB,IAAI,IAAI,KAAK,YAAY,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG;aACzF,CAAC;QACJ,CAAC;QACD,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC;IAC1B,CAAC;IACD,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;AACrC,CAAC;AAUD;;;;;;GAMG;AACH,SAAS,QAAQ,CACf,IAAU,EACV,MAA4B,EAC5B,MAA2B,EAC3B,KAAsE,EACtE,SAAoB;IAEpB,MAAM,IAAI,GAAG,KAAK,CAAC,WAAW,CAAC;IAC/B,MAAM,OAAO,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAC9E,MAAM,OAAO,GAAG,IAAI,GAAG,EAA8B,CAAC;IACtD,OAAO;QACL,IAAI,EAAE,oBAAoB;QAC1B,aAAa,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,EAAE;YAC5B,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,IAAgB,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;YACnF,OAAO,IAAI,CAAC,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,EAAE,GAAG,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QAClF,CAAC;QACD,UAAU,EAAE,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;YAC7B,IAAI,OAAO,CAAC,WAAW,KAAK,SAAS;gBAAE,OAAO;YAC9C,MAAM,KAAK,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,IAAc,EAAE,EAAE;gBACpE,MAAM,QAAQ,GAAuB;oBACnC,EAAE,EAAE,IAAI,CAAC,EAAE;oBACX,IAAI,EAAE,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC;oBAC7B,KAAK,EAAE,IAAI,CAAC,SAAS;iBACtB,CAAC;gBACF,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;gBAC/B,OAAO,QAAQ,CAAC;YAClB,CAAC,CAAC,CAAC;YACH,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,KAAK,CAAC,QAAQ,CAAC,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;YAC9F,IAAI,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,EAAE;gBAAE,SAAS,CAAC,WAAW,GAAG,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC;QAC3F,CAAC;QACD,cAAc,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;YACvC,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,KAAK,CAAC;YACjC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC3B,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,WAAW,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;gBAC9E,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBAC1B,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC,IAAI,kCAAkC,EAAE,CAAC;gBACtF,CAAC;gBACD,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,KAAK,SAAS,CAAC,EAAE,CAAC;oBAC7D,IAAI,OAAO,GAAG,KAAK,CAAC;oBACpB,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;wBAC7B,IAAI,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,QAAQ,CAAC,SAAS,EAAE,KAAK,CAAC,EAAE,CAAC;4BAC3E,OAAO,GAAG,IAAI,CAAC;4BACf,MAAM;wBACR,CAAC;oBACH,CAAC;oBACD,IAAI,CAAC,OAAO,EAAE,CAAC;wBACb,OAAO;4BACL,QAAQ,EAAE,MAAM;4BAChB,MAAM,EAAE,kDAAkD,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;yBACnG,CAAC;oBACJ,CAAC;gBACH,CAAC;YACH,CAAC;YACD,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,QAAQ,CAAC,SAAS,CAAC;YAC7F,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC5D,MAAM,SAAS,GAAG,OAAO,CAAC,WAAW,KAAK,SAAS,CAAC;YACpD,SAAS,CAAC,SAAS,CAAC,IAAI,CAAC;gBACvB,EAAE,EAAE,QAAQ,CAAC,EAAE;gBACf,IAAI;gBACJ,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,KAAK;gBACL,GAAG,CAAC,SAAS,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC;aAC/C,CAAC,CAAC;YACH,MAAM,KAAK,GAAG,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;YAC/D,IAAI,KAAK;gBAAE,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC;YAC/B,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC;QAC/B,CAAC;QACD,YAAY,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE;YAC1C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC;YAChC,IAAI,OAAO,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;gBACtC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC;oBACrB,IAAI,EAAE,MAAM;oBACZ,SAAS,EAAE,IAAI,CAAC,QAAQ,CAAC,EAAE;oBAC3B,IAAI,EAAE,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;oBAClC,OAAO,EAAE,MAAM,CAAC,KAAK,IAAI,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC;oBAClD,OAAO,EAAE,MAAM,CAAC,KAAK,KAAK,SAAS;iBACpC,CAAC,CAAC;YACL,CAAC;YACD,OAAO,MAAM,CAAC;QAChB,CAAC;KACF,CAAC;AACJ,CAAC;AAED,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;AAE1D,SAAS,QAAQ,CACf,IAAU,EACV,IAAwB,EACxB,IAAY,EACZ,MAAc,EACd,MAA0C,EAC1C,KAAwC;IAExC,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,IAAI,OAAO,IAAI,CAAC,IAAI,gBAAgB,IAAI,CAAC,MAAM,aAAa,CAAC;IACjG,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW;QACjC,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,EAAE,GAAG,IAAI,CAAC,WAAW,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;QAChE,CAAC,CAAC,EAAE,IAAI,EAAE,QAAiB,EAAE,UAAU,EAAE,EAAE,EAAE,oBAAoB,EAAE,IAAI,EAAE,CAAC;IAC5E,OAAO;QACL,IAAI;QACJ,WAAW;QACX,UAAU,EAAE,eAAe;QAC3B,GAAG,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE;QACxC,OAAO,EAAE,KAAK,EAAE,KAA8B,EAAE,EAAE;YAChD,MAAM,SAAS,GAAG,eAAe,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YAC/C,IAAI,SAAS;gBACX,KAAK,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,WAAW,SAAS,EAAE,EAAE,CAAC,CAAC;YAClF,IAAI,MAAc,CAAC;YACnB,IAAI,CAAC;gBACH,MAAM,GAAG,UAAU,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YACnC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC;oBACL,MAAM,EAAE,IAAI,CAAC,MAAM;oBACnB,IAAI,EAAE,IAAI,CAAC,IAAI;oBACf,KAAK;oBACL,MAAM,EAAE,SAAS,CAAC,KAAK,CAAC;oBACxB,KAAK,EAAE,IAAI;iBACZ,CAAC,CAAC;gBACH,MAAM,KAAK,CAAC;YACd,CAAC;YACD,MAAM,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;YACnF,IAAI,IAAI,CAAC,KAAK;gBAAE,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,CAAC;YACxC,OAAO,MAAM,CAAC;QAChB,CAAC;QACD,MAAM,EAAE,GAAe,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,UAAU,EAAE,CAAC;KAC9D,CAAC;AACJ,CAAC;AAED,SAAS,cAAc,CACrB,IAAU,EACV,IAAa,EACb,cAA2D,EAC3D,KAAc,EACd,SAAkB;IAElB,MAAM,KAAK,GAAG;QACZ,iBAAiB;QACjB,IAAI,CAAC,aAAa,CAAC,EAAE,cAAc,EAAE,KAAK,EAAE,SAAS,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;KAC5E,CAAC;IACF,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC;IACxC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrB,KAAK,CAAC,IAAI,CACR,sDAAsD,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACjG,CAAC;IACJ,CAAC;IACD,IAAI,IAAI,CAAC,IAAI,CAAC,kBAAkB,KAAK,SAAS;QAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;IACzF,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AACjE,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,IAAa;IAC5C,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC;IACtC,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC3B,MAAM,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,wBAAwB,CAAC,CAAC,CAAC;IACtE,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IACxC,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IACnC,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;IACjC,KAAK,MAAM,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,GAAG,CAAC;QAAE,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACzE,MAAM,OAAO,GAAG,GAAG,EAAE;QACnB,IAAI,CAAC,IAAI,CAAC,QAAQ;YAAE,MAAM,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACxE,CAAC,CAAC;IACF,MAAM,OAAO,GAAG,GAAG,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,GAAG,IAAI,CAAC;IACpD,MAAM,KAAK,GAAG,CAAC,MAAsD,EAAc,EAAE,CAAC,CAAC;QACrF,OAAO,EAAE,CAAC;QACV,eAAe,EAAE,OAAO,EAAE;QAC1B,GAAG,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,OAAO,EAAE,CAAC;QACjC,OAAO;QACP,GAAG,MAAM;KACV,CAAC,CAAC;IAEH,IAAI,QAAQ,CAAC,OAAO,CAAC,cAAc,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QACrD,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,QAAQ,CAAC,OAAO,CAAC,cAAc,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC;QACnF,IAAI,MAAM;YAAE,OAAO,KAAK,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;IAC9C,CAAC;IACD,KAAK,MAAM,GAAG,IAAI,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;QAC3C,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAC7B,IAAI,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YACrB,OAAO,KAAK,CAAC,EAAE,KAAK,EAAE,oBAAoB,GAAG,4CAA4C,EAAE,CAAC,CAAC;QAC/F,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC7D,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;IAExC,MAAM,SAAS,GAAc;QAC3B,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC;QAClD,SAAS,EAAE,EAAE;QACb,SAAS,EAAE,EAAE;QACb,WAAW,EAAE,EAAE;KAChB,CAAC;IACF,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,IAAI,OAAoC,CAAC;IACzC,MAAM,KAAK,GAAG,CAAC,MAAuB,EAAS,EAAE;QAC/C,OAAO,KAAK,MAAM,CAAC;QACnB,MAAM,KAAK,GAAG,IAAI,cAAc,CAAC,MAAM,CAAC,CAAC;QACzC,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACxB,MAAM,KAAK,CAAC;IACd,CAAC,CAAC;IACF,MAAM,MAAM,GAAG,CAAC,IAAwB,EAAE,EAAE,CAAC,SAAS,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE5E,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;IACtC,MAAM,QAAQ,GAAW,EAAE,CAAC;IAC5B,MAAM,WAAW,GAAG,IAAI,GAAG,EAAgC,CAAC;IAC5D,KAAK,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QAC7D,MAAM,MAAM,GAAG,UAAU,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACxC,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACzB,WAAW,CAAC,GAAG,CAAC,UAAU,MAAM,IAAI,MAAM,EAAE,EAAE,KAAK,CAAC,CAAC;YACrD,KAAK,MAAM,IAAI,IAAI,KAAK;gBAAE,WAAW,CAAC,GAAG,CAAC,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;YACnF,SAAS;QACX,CAAC;QACD,IAAI,CAAC,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YAAE,SAAS;QACtF,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,IAAI,GAAG,YAAY,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;YACxD,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACtB,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC;QACnE,CAAC;IACH,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,cAAc,CAAC;QACxC,IAAI;QACJ,IAAI,EAAE,EAAE,GAAG,IAAI,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE,GAAG,QAAQ,CAAC,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE;KAC1E,CAAC,CAAC;IACH,MAAM,cAAc,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC;IACtF,MAAM,KAAK,GAAG,SAAS,CAAC,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;IACpD,MAAM,SAAS,GAAG,SAAS,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAClD,MAAM,KAAK,GAAW;QACpB,GAAG,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC;YAC3B,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,cAAc,EAAE,eAAe,EAAE,GAAG,EAAE,CAAC,KAAK,EAAE,CAAC;YACzF,CAAC,CAAC,EAAE,CAAC;QACP,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACrE,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACjE,GAAG,QAAQ;KACZ,CAAC;IACF,MAAM,KAAK,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC;QAC3B,IAAI,EAAE,aAAa;QACnB,GAAG,CAAC,IAAI,CAAC,KAAK,KAAK,SAAS,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC;QACtD,YAAY,EAAE,cAAc,CAAC,IAAI,EAAE,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,SAAS,CAAC;QAC1E,KAAK;QACL,SAAS;QACT,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;QAC7B,aAAa,EAAE,QAAQ,CAAC,QAAQ;QAChC,gBAAgB,EAAE,MAAM;KACzB,CAAC,CAAC;IACH,MAAM,SAAS,GAEX,KAAK,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC;QACtB,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE;YAClC,MAAM,CAAC,EAAE,MAAM,GAAG,EAAE,EAAE,MAAM,GAAG,EAAE,CAAC,GAAG,uBAAuB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YAC9E,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAChD,QAAQ,CAAC,IAAI,EAAE,IAAI,EAAE,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,CACnF,CAAC;QACJ,CAAC,CAAC;QACF,KAAK,EAAE,KAAK,IAAI,EAAE,CAAC,SAAS;KAC7B,CAAC,CAAC;IACH,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC;QACnC,GAAG,EAAE,IAAI,CAAC,GAAG;QACb,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,MAAM,IAAI;YACzB,oBAAoB,EAAE;gBACpB,IAAI,EAAE,QAAQ,CAAC,OAAO;gBACtB,IAAI;gBACJ,SAAS,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,EAAE;gBAC7C,QAAQ,EAAE,KAAK;gBACf,GAAG,EAAE,SAAS;aACf;SACF,CAAC;QACF,GAAG,CAAC,SAAS,IAAI;YACf,SAAS,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,iBAAiB,EAAE,eAAe,EAAE,EAAE;SAC5E,CAAC;QACF,UAAU,EAAE;YACV,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,WAAW,EAAE,EAAE,SAAS,EAAE,WAAW,EAAE,IAAI,EAAE,EAAE,SAAS,CAAC;SACtF;KACF,CAAC,CAAC;IAEH,MAAM,SAAS,GAAG,QAAQ,CAAC,cAAc,GAAG,IAAI,CAAC;IACjD,IAAI,QAAQ,GAAG,KAAK,CAAC;IACrB,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;QAC5B,QAAQ,GAAG,IAAI,CAAC;QAChB,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,mBAAmB,QAAQ,CAAC,cAAc,GAAG,CAAC,CAAC,CAAC;IAC7E,CAAC,EAAE,SAAS,CAAC,CAAC;IACd,MAAM,WAAW,GAAG,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC;IACrE,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO;QAAE,WAAW,EAAE,CAAC;;QAClC,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,WAAW,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;IAExE,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,KAAK,GAAkB,IAAI,CAAC;IAChC,IAAI,KAAK,GAAe,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;IACpD,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,QAAQ,GAAG,MAAM,EAAE,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC;QAChG,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,KAAK,EAAE;YACxC,KAAK,EAAE,QAAQ,CAAC,MAAM;YACtB,MAAM,EAAE,UAAU,CAAC,MAAM;YACzB,SAAS,EAAE,KAAK;YAChB,WAAW,EAAE,KAAK;YAClB,WAAW,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,EAAE;YAClE,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,WAAW,IAAI,EAAE,OAAO,EAAE,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,CAAC;YACzF,UAAU,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE;gBACpC,OAAO,GAAG,KAAK,EAAE,IAAI,IAAI,OAAO,CAAC;YACnC,CAAC;SACF,CAAC,CAAC;QACH,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;QAC5B,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE;YAAE,SAAS,CAAC,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC;QACvE,IAAI,MAAM,CAAC,qBAAqB;YAAE,KAAK,GAAG,sBAAsB,QAAQ,CAAC,QAAQ,GAAG,CAAC;aAChF,IAAI,MAAM,CAAC,MAAM,KAAK,QAAQ;YAAE,KAAK,GAAG,wCAAwC,CAAC;IACxF,CAAC;IAAC,OAAO,MAAM,EAAE,CAAC;QAChB,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;YAC1B,IAAI,QAAQ;gBAAE,KAAK,GAAG,mBAAmB,QAAQ,CAAC,cAAc,GAAG,CAAC;iBAC/D,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO;gBAAE,KAAK,GAAG,aAAa,CAAC;iBAC/C,CAAC;gBACJ,KAAK,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;gBAC9B,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,cAAc,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YAC5E,CAAC;QACH,CAAC;IACH,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;QACpB,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;QACtD,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAC/C,MAAM,SAAS,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IACnD,CAAC;IAED,IAAI,OAAO;QAAE,OAAO,KAAK,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;IAC7D,IAAI,KAAK,KAAK,aAAa;QAAE,OAAO,KAAK,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;IACjF,IAAI,KAAK,CAAC,GAAG,EAAE,CAAC;QACd,OAAO,KAAK,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,GAAG,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC;IAC5F,CAAC;IACD,MAAM,YAAY,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;IACzE,OAAO,KAAK,CAAC;QACX,KAAK;QACL,OAAO;QACP,OAAO,EAAE;YACP,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,WAAW,EAAE,SAAS,CAAC,WAAW;YAClC,OAAO,EAAE,SAAS,CAAC,OAAO;YAC1B,SAAS,EAAE,SAAS,CAAC,SAAS;YAC9B,SAAS,EAAE,SAAS,CAAC,SAAS;YAC9B,YAAY;YACZ,aAAa,EAAE,IAAI;YACnB,OAAO,EAAE,QAAQ,CAAC,GAAG;SACtB;KACF,CAAC,CAAC;AACL,CAAC"}
@@ -0,0 +1,17 @@
1
+ /** Whether `path` is `root` or below it, both absolute */
2
+ export declare function isInside(root: string, path: string): boolean;
3
+ /**
4
+ * `path` as a relative path of plain directory names (`qa`, `./quality/evals`), with `.`
5
+ * segments dropped, or why it is not one: absolute, holding `..`, or empty
6
+ */
7
+ export declare function plainRelativePath(path: string): {
8
+ path: string;
9
+ } | {
10
+ problem: string;
11
+ };
12
+ export declare function isDirectory(path: string): boolean;
13
+ export declare function isFile(path: string): boolean;
14
+ export declare function errorText(error: unknown): string;
15
+ /** `path` with forward slashes, as graders and reports show workspace paths */
16
+ export declare function posixPath(path: string): string;
17
+ //# sourceMappingURL=files.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"files.d.ts","sourceRoot":"","sources":["../../src/plugin-evals/files.ts"],"names":[],"mappings":"AAGA,0DAA0D;AAC1D,wBAAgB,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAG5D;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG;IAAE,OAAO,EAAE,MAAM,CAAA;CAAE,CAQtF;AAED,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAMjD;AAED,wBAAgB,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAM5C;AAED,wBAAgB,SAAS,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAEhD;AAED,+EAA+E;AAC/E,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAE9C"}
@@ -0,0 +1,46 @@
1
+ import { statSync } from 'node:fs';
2
+ import { isAbsolute, relative, sep } from 'node:path';
3
+ /** Whether `path` is `root` or below it, both absolute */
4
+ export function isInside(root, path) {
5
+ const rel = relative(root, path);
6
+ return rel === '' || (rel !== '..' && !rel.startsWith(`..${sep}`) && !isAbsolute(rel));
7
+ }
8
+ /**
9
+ * `path` as a relative path of plain directory names (`qa`, `./quality/evals`), with `.`
10
+ * segments dropped, or why it is not one: absolute, holding `..`, or empty
11
+ */
12
+ export function plainRelativePath(path) {
13
+ if (isAbsolute(path) || /^[A-Za-z]:/.test(path) || /^[\\/]/.test(path)) {
14
+ return { problem: 'is absolute' };
15
+ }
16
+ const segments = path.split(/[\\/]/).filter((segment) => segment !== '' && segment !== '.');
17
+ if (segments.some((segment) => segment === '..'))
18
+ return { problem: 'contains ..' };
19
+ if (segments.length === 0)
20
+ return { problem: 'is empty' };
21
+ return { path: segments.join('/') };
22
+ }
23
+ export function isDirectory(path) {
24
+ try {
25
+ return statSync(path).isDirectory();
26
+ }
27
+ catch {
28
+ return false;
29
+ }
30
+ }
31
+ export function isFile(path) {
32
+ try {
33
+ return statSync(path).isFile();
34
+ }
35
+ catch {
36
+ return false;
37
+ }
38
+ }
39
+ export function errorText(error) {
40
+ return error instanceof Error ? error.message : String(error);
41
+ }
42
+ /** `path` with forward slashes, as graders and reports show workspace paths */
43
+ export function posixPath(path) {
44
+ return path.split(sep).join('/');
45
+ }
46
+ //# sourceMappingURL=files.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"files.js","sourceRoot":"","sources":["../../src/plugin-evals/files.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AACnC,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AAEtD,0DAA0D;AAC1D,MAAM,UAAU,QAAQ,CAAC,IAAY,EAAE,IAAY;IACjD,MAAM,GAAG,GAAG,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IACjC,OAAO,GAAG,KAAK,EAAE,IAAI,CAAC,GAAG,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,KAAK,GAAG,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;AACzF,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAAC,IAAY;IAC5C,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACvE,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC;IACpC,CAAC;IACD,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,KAAK,EAAE,IAAI,OAAO,KAAK,GAAG,CAAC,CAAC;IAC5F,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,KAAK,IAAI,CAAC;QAAE,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC;IACpF,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC;IAC1D,OAAO,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;AACtC,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,IAAY;IACtC,IAAI,CAAC;QACH,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;IACtC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,MAAM,UAAU,MAAM,CAAC,IAAY;IACjC,IAAI,CAAC;QACH,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;IACjC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,KAAc;IACtC,OAAO,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAChE,CAAC;AAED,+EAA+E;AAC/E,MAAM,UAAU,SAAS,CAAC,IAAY;IACpC,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACnC,CAAC"}
@@ -0,0 +1,8 @@
1
+ /**
2
+ * A glob as a regular expression over a whole string: `*` matches within a path segment, `**`
3
+ * across segments, `?` one character, `[...]` a class and `{a,b}` alternatives
4
+ */
5
+ export declare function globToRegExp(glob: string): RegExp;
6
+ /** Whether `value` matches `glob` as a whole */
7
+ export declare function globMatches(glob: string, value: string): boolean;
8
+ //# sourceMappingURL=glob.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"glob.d.ts","sourceRoot":"","sources":["../../src/plugin-evals/glob.ts"],"names":[],"mappings":"AAEA;;;GAGG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAoCjD;AAMD,gDAAgD;AAChD,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAEhE"}