@lesliechan721/agent-skill-eval 0.1.0-beta.1 → 0.1.0-beta.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (101) hide show
  1. package/dist/bin.js +167 -39
  2. package/dist/bin.js.map +1 -1
  3. package/dist/codex-eval-app-server.d.ts +199 -0
  4. package/dist/codex-eval-app-server.js +1364 -0
  5. package/dist/codex-eval-app-server.js.map +1 -0
  6. package/dist/codex-eval-cases.d.ts +43 -0
  7. package/dist/codex-eval-cases.js +327 -0
  8. package/dist/codex-eval-cases.js.map +1 -0
  9. package/dist/codex-eval-isolation.d.ts +79 -0
  10. package/dist/codex-eval-isolation.js +420 -0
  11. package/dist/codex-eval-isolation.js.map +1 -0
  12. package/dist/codex-eval-process-tree.d.ts +38 -0
  13. package/dist/codex-eval-process-tree.js +803 -0
  14. package/dist/codex-eval-process-tree.js.map +1 -0
  15. package/dist/codex-eval-runtime-config.d.ts +43 -0
  16. package/dist/codex-eval-runtime-config.js +607 -0
  17. package/dist/codex-eval-runtime-config.js.map +1 -0
  18. package/dist/codex-trigger-eval.d.ts +82 -0
  19. package/dist/codex-trigger-eval.js +790 -0
  20. package/dist/codex-trigger-eval.js.map +1 -0
  21. package/dist/codex-workflow.d.ts +191 -0
  22. package/dist/codex-workflow.js +1832 -0
  23. package/dist/codex-workflow.js.map +1 -0
  24. package/dist/credentials.d.ts +5 -0
  25. package/dist/credentials.js +30 -0
  26. package/dist/credentials.js.map +1 -0
  27. package/dist/custom-process.d.ts +60 -0
  28. package/dist/custom-process.js +423 -0
  29. package/dist/custom-process.js.map +1 -0
  30. package/dist/eval-audit.d.ts +38 -0
  31. package/dist/eval-audit.js +71 -0
  32. package/dist/eval-audit.js.map +1 -0
  33. package/dist/eval-file.d.ts +9 -0
  34. package/dist/eval-file.js +15 -0
  35. package/dist/eval-file.js.map +1 -0
  36. package/dist/eval-process-management.d.ts +21 -0
  37. package/dist/eval-process-management.js +84 -0
  38. package/dist/eval-process-management.js.map +1 -0
  39. package/dist/eval-result.d.ts +79 -0
  40. package/dist/eval-result.js +144 -0
  41. package/dist/eval-result.js.map +1 -0
  42. package/dist/eval-run.d.ts +27 -0
  43. package/dist/eval-run.js +564 -0
  44. package/dist/eval-run.js.map +1 -0
  45. package/dist/eval-spec.d.ts +57 -0
  46. package/dist/eval-spec.js +120 -0
  47. package/dist/eval-spec.js.map +1 -0
  48. package/dist/eval-suite-binding.d.ts +4 -0
  49. package/dist/eval-suite-binding.js +19 -0
  50. package/dist/eval-suite-binding.js.map +1 -0
  51. package/dist/index.d.ts +36 -9
  52. package/dist/index.js +19 -6
  53. package/dist/index.js.map +1 -1
  54. package/dist/json-schema.js +75 -5
  55. package/dist/json-schema.js.map +1 -1
  56. package/dist/output-path.js +2 -2
  57. package/dist/process-tree.d.ts +19 -1
  58. package/dist/process-tree.js +391 -77
  59. package/dist/process-tree.js.map +1 -1
  60. package/dist/process.d.ts +0 -2
  61. package/dist/process.js +76 -25
  62. package/dist/process.js.map +1 -1
  63. package/dist/provider-doctor.d.ts +45 -0
  64. package/dist/provider-doctor.js +112 -0
  65. package/dist/provider-doctor.js.map +1 -0
  66. package/dist/providers.d.ts +83 -0
  67. package/dist/providers.js +632 -0
  68. package/dist/providers.js.map +1 -0
  69. package/dist/run-paths.d.ts +8 -0
  70. package/dist/run-paths.js +71 -0
  71. package/dist/run-paths.js.map +1 -0
  72. package/dist/runtime-skill.d.ts +6 -0
  73. package/dist/runtime-skill.js +87 -3
  74. package/dist/runtime-skill.js.map +1 -1
  75. package/dist/schemas/digests.json +3 -3
  76. package/dist/schemas/eval-audit.schema.json +64 -0
  77. package/dist/schemas/eval-result.schema.json +232 -0
  78. package/dist/schemas/eval-spec.schema.json +109 -0
  79. package/dist/schemas.d.ts +1 -3
  80. package/dist/schemas.js +0 -17
  81. package/dist/schemas.js.map +1 -1
  82. package/dist/selected-cases.d.ts +30 -0
  83. package/dist/selected-cases.js +388 -0
  84. package/dist/selected-cases.js.map +1 -0
  85. package/dist/types.d.ts +2 -34
  86. package/package.json +8 -3
  87. package/dist/bundle.d.ts +0 -93
  88. package/dist/bundle.js +0 -216
  89. package/dist/bundle.js.map +0 -1
  90. package/dist/capabilities.d.ts +0 -2
  91. package/dist/capabilities.js +0 -38
  92. package/dist/capabilities.js.map +0 -1
  93. package/dist/plan.d.ts +0 -79
  94. package/dist/plan.js +0 -285
  95. package/dist/plan.js.map +0 -1
  96. package/dist/run.d.ts +0 -2
  97. package/dist/run.js +0 -592
  98. package/dist/run.js.map +0 -1
  99. package/dist/schemas/aw-skill-eval-execution-plan.schema.json +0 -220
  100. package/dist/schemas/eval-run-bundle.schema.json +0 -408
  101. package/dist/schemas/eval-runtime-capabilities.schema.json +0 -85
@@ -0,0 +1,564 @@
1
+ import { randomUUID } from 'node:crypto';
2
+ import { mkdir, readFile, rename, writeFile } from 'node:fs/promises';
3
+ import path from 'node:path';
4
+ import process from 'node:process';
5
+ import fs from 'fs-extra';
6
+ import { sha256, toPosixRelative } from './common.js';
7
+ import { credentialValues, redactSecrets } from './credentials.js';
8
+ import { CODEX_PROVIDER_ENVIRONMENT_ALLOWLIST, executeCustomProcessProvider, } from './custom-process.js';
9
+ import { validateEvalAuditFile } from './eval-audit.js';
10
+ import { computeEvalResultSummary, createEvalResultBinding, validateEvalResult, } from './eval-result.js';
11
+ import { resolveEvalProcessManagement, } from './eval-process-management.js';
12
+ import { resolveCodexEvalExecutable } from './codex-eval-isolation.js';
13
+ import { validateEvalSpecFile } from './eval-spec.js';
14
+ import { assertEvalProviderSuiteKind } from './eval-suite-binding.js';
15
+ import { runBuiltinEvalProvider, } from './providers.js';
16
+ import { resolveCodexEvalRuntimeConfig } from './codex-eval-runtime-config.js';
17
+ import { ProviderProcessCleanupError } from './process.js';
18
+ import { executeCodexWorkflowProvider, } from './codex-workflow.js';
19
+ import { computeCanonicalDirectoryDigest, computeSkillDigest } from './runtime-skill.js';
20
+ import { parseSelectedCases, } from './selected-cases.js';
21
+ class BoundInputChangedError extends Error {
22
+ }
23
+ function secretValues(sourceEnv) {
24
+ return credentialValues(sourceEnv, CODEX_PROVIDER_ENVIRONMENT_ALLOWLIST);
25
+ }
26
+ function redactText(value, secrets) {
27
+ return redactSecrets(value, secrets);
28
+ }
29
+ function redactValue(value, secrets) {
30
+ if (typeof value === 'string')
31
+ return redactText(value, secrets);
32
+ if (Array.isArray(value))
33
+ return value.map((entry) => redactValue(entry, secrets));
34
+ if (value && typeof value === 'object') {
35
+ return Object.fromEntries(Object.entries(value).map(([key, entry]) => [
36
+ redactText(key, secrets),
37
+ redactValue(entry, secrets),
38
+ ]));
39
+ }
40
+ return value;
41
+ }
42
+ async function inputDigests(calculated) {
43
+ return {
44
+ workspace: await computeCanonicalDirectoryDigest(calculated.paths.workspace),
45
+ target: await computeSkillDigest(calculated.paths.target),
46
+ cases: sha256(await readFile(calculated.paths.cases)),
47
+ previousTarget: calculated.paths.previousTarget
48
+ ? await computeSkillDigest(calculated.paths.previousTarget)
49
+ : null,
50
+ };
51
+ }
52
+ function sameDigests(left, right) {
53
+ return left.workspace === right.workspace &&
54
+ left.target === right.target &&
55
+ left.cases === right.cases &&
56
+ left.previousTarget === right.previousTarget;
57
+ }
58
+ async function assertInputsUnchanged(calculated, captured) {
59
+ let current;
60
+ try {
61
+ current = await inputDigests(calculated);
62
+ }
63
+ catch (error) {
64
+ throw new BoundInputChangedError(`Prepared Eval input changed: ${error.message}`);
65
+ }
66
+ if (!sameDigests(captured, current) || !digestsMatchSpec(calculated, current)) {
67
+ throw new BoundInputChangedError('Prepared Eval input digest changed or no longer matches EvalSpec digests.');
68
+ }
69
+ }
70
+ function digestsMatchSpec(calculated, digests) {
71
+ return digests.workspace === calculated.workspaceDigest
72
+ && digests.workspace === calculated.spec.workspace.digest
73
+ && digests.target === calculated.targetDigest
74
+ && digests.cases === calculated.caseDigest
75
+ && digests.previousTarget === calculated.previousTargetDigest;
76
+ }
77
+ function assertDigestsMatchSpec(calculated, digests) {
78
+ if (!digestsMatchSpec(calculated, digests)) {
79
+ throw new BoundInputChangedError('Prepared Eval input no longer matches EvalSpec digests.');
80
+ }
81
+ }
82
+ function triggerProviderCase(document, evalCase) {
83
+ if (!document.suite.providerSuite || !evalCase.providerCase) {
84
+ throw new Error('Selected trigger case has no canonical provider form.');
85
+ }
86
+ return {
87
+ suite: document.suite.providerSuite,
88
+ value: evalCase.providerCase,
89
+ };
90
+ }
91
+ export function adaptConversationProviderCase(document, evalCase) {
92
+ const participants = [...new Set([
93
+ document.suite.unitUnderTest,
94
+ ...(document.suite.participants ?? []),
95
+ ])];
96
+ const assertions = evalCase.assertions;
97
+ const input = evalCase.input && typeof evalCase.input === 'object' && !Array.isArray(evalCase.input)
98
+ ? evalCase.input
99
+ : {};
100
+ const expected = evalCase.expected && typeof evalCase.expected === 'object' && !Array.isArray(evalCase.expected)
101
+ ? evalCase.expected
102
+ : {};
103
+ const audit = evalCase.audit && typeof evalCase.audit === 'object' && !Array.isArray(evalCase.audit)
104
+ ? evalCase.audit
105
+ : null;
106
+ const workspaceUnchanged = expected.workspaceUnchanged === undefined
107
+ ? true
108
+ : expected.workspaceUnchanged;
109
+ const scriptedReplies = Array.isArray(input.scriptedReplies)
110
+ ? input.scriptedReplies.map((reply) => {
111
+ if (typeof reply === 'string')
112
+ return reply;
113
+ const row = reply;
114
+ return {
115
+ answers_by_index: row.answersByIndex,
116
+ ...(row.expectedQuestionCount === undefined
117
+ ? {}
118
+ : { expected_question_count: row.expectedQuestionCount }),
119
+ unexpected_question_answer: row.unexpectedQuestionAnswer,
120
+ };
121
+ })
122
+ : [];
123
+ const workspaceFiles = input.workspaceFiles && typeof input.workspaceFiles === 'object'
124
+ && !Array.isArray(input.workspaceFiles)
125
+ ? input.workspaceFiles
126
+ : {};
127
+ const allowedWorkspaceChanges = Array.isArray(expected.allowedWorkspaceChanges)
128
+ ? expected.allowedWorkspaceChanges
129
+ : [];
130
+ const artifacts = Array.isArray(expected.artifacts)
131
+ ? expected.artifacts.map((artifact) => {
132
+ const row = artifact;
133
+ const { notContains, ...providerArtifact } = row;
134
+ return {
135
+ ...providerArtifact,
136
+ ...(Object.hasOwn(row, 'notContains') ? { not_contains: notContains } : {}),
137
+ };
138
+ })
139
+ : [];
140
+ return {
141
+ value: {
142
+ id: evalCase.id,
143
+ prompt: evalCase.prompt,
144
+ scripted_replies: scriptedReplies,
145
+ expected_output: expected.output ?? 'Evaluate the response against the approved assertions.',
146
+ ...(Object.keys(workspaceFiles).length > 0 ? { workspace_files: workspaceFiles } : {}),
147
+ ...(Array.isArray(input.pathPrepend) && input.pathPrepend.length > 0
148
+ ? { environment: { pathPrepend: input.pathPrepend } }
149
+ : {}),
150
+ sandbox: workspaceUnchanged === false ? 'workspace-write' : 'read-only',
151
+ review: {
152
+ checks: assertions,
153
+ workspace_unchanged: workspaceUnchanged,
154
+ ...(allowedWorkspaceChanges.length > 0
155
+ ? { allowed_workspace_changes: allowedWorkspaceChanges }
156
+ : {}),
157
+ ...(artifacts.length > 0 ? { artifacts } : {}),
158
+ },
159
+ },
160
+ skillName: document.suite.unitUnderTest,
161
+ skillNames: participants,
162
+ isolation: document.suite.runtime?.isolation === 'strict' ? 'strict' : 'standard',
163
+ suitePathPrepend: document.suite.runtime?.pathPrepend ?? [],
164
+ assertions,
165
+ ...(audit && Object.hasOwn(audit, 'allowedVariance')
166
+ ? { allowedVariance: audit.allowedVariance }
167
+ : {}),
168
+ };
169
+ }
170
+ function providerCase(document, evalCase) {
171
+ if (document.suite.kind === 'trigger')
172
+ return triggerProviderCase(document, evalCase);
173
+ if (document.suite.kind === 'conversation')
174
+ return adaptConversationProviderCase(document, evalCase);
175
+ return evalCase;
176
+ }
177
+ async function prepareAttempt(options) {
178
+ const attemptRoot = path.join(options.calculated.paths.outputDir, 'runs', options.caseId, options.variant, `attempt-${options.attempt}`);
179
+ const workspacePath = path.join(attemptRoot, 'workspace', 'subject');
180
+ const fixturesPath = path.join(attemptRoot, 'workspace', 'eval');
181
+ const runOutput = path.join(attemptRoot, 'output');
182
+ await fs.ensureDir(path.dirname(attemptRoot));
183
+ await mkdir(attemptRoot);
184
+ try {
185
+ await mkdir(path.join(attemptRoot, 'workspace'));
186
+ await fs.copy(path.join(options.snapshot.workspace, 'subject'), workspacePath, {
187
+ dereference: false,
188
+ overwrite: false,
189
+ errorOnExist: true,
190
+ });
191
+ await fs.copy(path.join(options.snapshot.workspace, 'eval'), fixturesPath, {
192
+ dereference: false,
193
+ overwrite: false,
194
+ errorOnExist: true,
195
+ });
196
+ await mkdir(runOutput);
197
+ }
198
+ catch (error) {
199
+ await fs.remove(attemptRoot);
200
+ throw error;
201
+ }
202
+ const target = options.variant === 'current'
203
+ ? options.snapshot.currentTarget
204
+ : options.variant === 'previous-version'
205
+ ? options.snapshot.previousTarget
206
+ : null;
207
+ if (target) {
208
+ try {
209
+ const destination = path.join(workspacePath, '.agents', 'skills', options.document.suite.unitUnderTest);
210
+ if (await fs.pathExists(destination)) {
211
+ throw new Error('Prepared subject unexpectedly contains the unit-under-test Skill.');
212
+ }
213
+ await fs.ensureDir(path.dirname(destination));
214
+ await fs.copy(target, destination, {
215
+ dereference: false,
216
+ overwrite: false,
217
+ errorOnExist: true,
218
+ });
219
+ }
220
+ catch (error) {
221
+ await fs.remove(attemptRoot);
222
+ throw error;
223
+ }
224
+ }
225
+ return { attemptRoot, workspacePath, fixturesPath, runOutput };
226
+ }
227
+ async function createRunInputSnapshot(calculated, captured, casesBytes) {
228
+ const root = path.join(calculated.paths.outputDir, '.run-input');
229
+ const workspace = path.join(root, 'workspace');
230
+ const currentTarget = path.join(root, 'targets', 'current');
231
+ const previousTarget = calculated.paths.previousTarget
232
+ ? path.join(root, 'targets', 'previous')
233
+ : null;
234
+ const cases = path.join(root, 'cases.json');
235
+ await mkdir(root);
236
+ try {
237
+ await mkdir(workspace);
238
+ await mkdir(path.join(root, 'targets'));
239
+ await fs.copy(path.join(calculated.paths.workspace, 'subject'), path.join(workspace, 'subject'), {
240
+ dereference: false,
241
+ overwrite: false,
242
+ errorOnExist: true,
243
+ });
244
+ await fs.copy(path.join(calculated.paths.workspace, 'eval'), path.join(workspace, 'eval'), {
245
+ dereference: false,
246
+ overwrite: false,
247
+ errorOnExist: true,
248
+ });
249
+ await fs.copy(calculated.paths.target, currentTarget, {
250
+ dereference: false,
251
+ overwrite: false,
252
+ errorOnExist: true,
253
+ });
254
+ await writeFile(cases, casesBytes, { flag: 'wx', mode: 0o600 });
255
+ if (calculated.paths.previousTarget && previousTarget) {
256
+ await fs.copy(calculated.paths.previousTarget, previousTarget, {
257
+ dereference: false,
258
+ overwrite: false,
259
+ errorOnExist: true,
260
+ });
261
+ }
262
+ const snapshotDigests = {
263
+ workspace: await computeCanonicalDirectoryDigest(workspace),
264
+ target: await computeSkillDigest(currentTarget),
265
+ cases: sha256(await readFile(cases)),
266
+ previousTarget: previousTarget ? await computeSkillDigest(previousTarget) : null,
267
+ };
268
+ if (!sameDigests(captured, snapshotDigests)) {
269
+ throw new BoundInputChangedError('Run input snapshot does not match the prepared Eval input.');
270
+ }
271
+ assertDigestsMatchSpec(calculated, snapshotDigests);
272
+ return { root, workspace, currentTarget, previousTarget, cases };
273
+ }
274
+ catch (error) {
275
+ await fs.remove(root);
276
+ throw error;
277
+ }
278
+ }
279
+ async function assertSnapshotUnchanged(calculated, snapshot, captured) {
280
+ const current = {
281
+ workspace: await computeCanonicalDirectoryDigest(snapshot.workspace),
282
+ target: await computeSkillDigest(snapshot.currentTarget),
283
+ cases: sha256(await readFile(snapshot.cases)),
284
+ previousTarget: snapshot.previousTarget
285
+ ? await computeSkillDigest(snapshot.previousTarget)
286
+ : null,
287
+ };
288
+ if (!sameDigests(captured, current) || !digestsMatchSpec(calculated, current)) {
289
+ throw new BoundInputChangedError('Run input snapshot digest changed or no longer matches EvalSpec digests.');
290
+ }
291
+ }
292
+ function isTimeout(error) {
293
+ return /timed out|timeout/u.test(error instanceof Error ? error.message : String(error));
294
+ }
295
+ function isCleanupFailure(error) {
296
+ return error instanceof ProviderProcessCleanupError ||
297
+ /cleanup|termination|process-tree/u.test(error instanceof Error ? error.message : String(error));
298
+ }
299
+ function abortError() {
300
+ return new Error('Eval execution was aborted.');
301
+ }
302
+ async function writeResultSafely(resultPath, result, binding) {
303
+ const validation = await validateEvalResult(result, binding);
304
+ if (!validation.valid)
305
+ throw new Error(`Generated EvalResult is invalid: ${validation.errors.join('; ')}`);
306
+ if (await fs.pathExists(resultPath))
307
+ throw new Error('Eval result.json already exists.');
308
+ const bytes = Buffer.from(`${JSON.stringify(result, null, 2)}\n`);
309
+ const temporary = path.join(path.dirname(resultPath), `.result.${randomUUID()}.tmp`);
310
+ await writeFile(temporary, bytes, { flag: 'wx', mode: 0o600 });
311
+ try {
312
+ if (await fs.pathExists(resultPath))
313
+ throw new Error('Eval result.json appeared before commit.');
314
+ await rename(temporary, resultPath);
315
+ }
316
+ catch (error) {
317
+ await fs.remove(temporary).catch(() => undefined);
318
+ throw error;
319
+ }
320
+ return sha256(bytes);
321
+ }
322
+ export async function runEvalSpec(specPath, options) {
323
+ if (!options?.auditPath)
324
+ throw new Error('Eval execution requires auditPath.');
325
+ const specValidation = await validateEvalSpecFile(specPath);
326
+ if (!specValidation.valid || !specValidation.calculated) {
327
+ throw new Error(`EvalSpec is invalid: ${specValidation.errors.join('; ')}`);
328
+ }
329
+ const calculated = specValidation.calculated;
330
+ const specRoot = await fs.realpath(path.dirname(path.resolve(specPath)));
331
+ const casesBytes = await readFile(calculated.paths.cases);
332
+ if (sha256(casesBytes) !== calculated.caseDigest) {
333
+ throw new BoundInputChangedError('Selected cases changed after EvalSpec validation.');
334
+ }
335
+ const document = parseSelectedCases(JSON.parse(casesBytes.toString('utf8')));
336
+ if (document.suite.unitUnderTest !== calculated.spec.source.skill) {
337
+ throw new Error('Selected cases unitUnderTest does not match EvalSpec source.skill.');
338
+ }
339
+ assertEvalProviderSuiteKind(calculated.spec.provider, document.suite.kind);
340
+ const selectedCaseIds = document.cases.map((entry) => entry.id);
341
+ const auditValidation = await validateEvalAuditFile(options.auditPath, {
342
+ spec: calculated.spec,
343
+ selectedCaseIds,
344
+ });
345
+ if (!auditValidation.valid || !auditValidation.calculated) {
346
+ throw new Error(`EvalAudit does not admit execution: ${auditValidation.errors.join('; ')}`);
347
+ }
348
+ const resultBinding = createEvalResultBinding(calculated, auditValidation.calculated.auditDigest, document);
349
+ const captured = await inputDigests(calculated);
350
+ assertDigestsMatchSpec(calculated, captured);
351
+ await assertInputsUnchanged(calculated, captured);
352
+ const sourceEnv = options.sourceEnv
353
+ ?? options.builtinProviderDependencies?.sourceEnv
354
+ ?? process.env;
355
+ const requestedCodexBin = options.builtinProviderDependencies?.codexBin
356
+ ?? sourceEnv.AW_SKILL_EVAL_CODEX_BIN
357
+ ?? 'codex';
358
+ const processPlatform = options.processManagementDependencies?.platform
359
+ ?? options.processManagementDependencies?.processTree?.platform
360
+ ?? process.platform;
361
+ const resolvedCodexBin = await resolveCodexEvalExecutable({ executable: requestedCodexBin, sourceEnv, platform: processPlatform }, options.processManagementDependencies?.resolveCodexExecutable);
362
+ const codexBin = resolvedCodexBin ?? requestedCodexBin;
363
+ const processManagement = await resolveEvalProcessManagement({
364
+ allowUnmanaged: options.allowUnmanagedProcesses ?? false,
365
+ codexBin,
366
+ sourceEnv,
367
+ dependencies: {
368
+ ...options.processManagementDependencies,
369
+ resolveCodexExecutable: async () => resolvedCodexBin,
370
+ },
371
+ });
372
+ options.onProcessManagementResolved?.(processManagement);
373
+ const runtime = await resolveCodexEvalRuntimeConfig({
374
+ sourceEnv,
375
+ inheritUserConfig: true,
376
+ modelOverride: options.builtinProviderDependencies?.model
377
+ ?? sourceEnv.AW_SKILL_EVAL_CODEX_MODEL,
378
+ });
379
+ const secrets = () => [
380
+ ...secretValues(sourceEnv),
381
+ ...runtime.sensitiveValues,
382
+ ];
383
+ const builtinProviderDependencies = {
384
+ ...options.builtinProviderDependencies,
385
+ sourceEnv,
386
+ codexBin,
387
+ model: runtime.model ?? undefined,
388
+ codexRuntime: runtime,
389
+ processManagement,
390
+ };
391
+ const startedAt = new Date().toISOString();
392
+ const runs = [];
393
+ let stop = false;
394
+ await assertInputsUnchanged(calculated, captured);
395
+ await fs.ensureDir(path.dirname(calculated.paths.outputDir));
396
+ await mkdir(calculated.paths.outputDir);
397
+ let snapshot;
398
+ try {
399
+ snapshot = await createRunInputSnapshot(calculated, captured, casesBytes);
400
+ }
401
+ catch (error) {
402
+ await fs.remove(calculated.paths.outputDir);
403
+ throw error;
404
+ }
405
+ for (const evalCase of document.cases) {
406
+ if (stop)
407
+ break;
408
+ for (const variant of resultBinding.allowedVariants) {
409
+ if (stop)
410
+ break;
411
+ for (let attempt = 1; attempt <= calculated.spec.runs; attempt += 1) {
412
+ const attemptStarted = Date.now();
413
+ let status = 'failed';
414
+ let usage = null;
415
+ let grade = null;
416
+ let error = null;
417
+ let artifacts;
418
+ let runOutput = path.join(calculated.paths.outputDir, 'runs', evalCase.id, variant, `attempt-${attempt}`, 'output');
419
+ try {
420
+ if (options.signal?.aborted)
421
+ throw abortError();
422
+ await assertInputsUnchanged(calculated, captured);
423
+ await assertSnapshotUnchanged(calculated, snapshot, captured);
424
+ const prepared = await prepareAttempt({
425
+ calculated,
426
+ snapshot,
427
+ document,
428
+ caseId: evalCase.id,
429
+ variant,
430
+ attempt,
431
+ });
432
+ runOutput = prepared.runOutput;
433
+ const input = {
434
+ case: providerCase(document, evalCase),
435
+ variant,
436
+ attempt,
437
+ workspacePath: prepared.workspacePath,
438
+ fixturesPath: prepared.fixturesPath,
439
+ outputDir: prepared.runOutput,
440
+ timeoutMs: calculated.spec.timeoutMs,
441
+ ...(options.signal ? { signal: options.signal } : {}),
442
+ };
443
+ const providerResult = typeof calculated.spec.provider === 'string'
444
+ ? await (options.runBuiltinProvider ?? runBuiltinEvalProvider)(calculated.spec.provider, { ...input, evalRoot: prepared.attemptRoot }, builtinProviderDependencies)
445
+ : calculated.spec.provider.type === 'custom-process'
446
+ ? await (options.executeCustomProvider ?? executeCustomProcessProvider)({
447
+ command: calculated.spec.provider.command,
448
+ evalRoot: prepared.attemptRoot,
449
+ input,
450
+ sourceEnv,
451
+ codexRuntime: runtime,
452
+ processManagement,
453
+ })
454
+ : await (options.executeCodexWorkflowProvider ?? executeCodexWorkflowProvider)({
455
+ provider: {
456
+ ...calculated.spec.provider,
457
+ command: calculated.spec.provider.command,
458
+ },
459
+ evalRoot: prepared.attemptRoot,
460
+ input,
461
+ sourceEnv,
462
+ codexRuntime: runtime,
463
+ codexBin: builtinProviderDependencies.codexBin,
464
+ processManagement,
465
+ });
466
+ if (options.signal?.aborted)
467
+ throw abortError();
468
+ const sanitized = redactValue(providerResult, secrets());
469
+ usage = sanitized.usage === null ? null : { ...sanitized.usage };
470
+ grade = sanitized.grade;
471
+ artifacts = typeof calculated.spec.provider === 'object'
472
+ && calculated.spec.provider.type === 'codex-workflow'
473
+ && sanitized.artifacts
474
+ ? [
475
+ ...sanitized.artifacts.map((artifact) => ({ ...artifact })),
476
+ { path: 'provider-result.json', role: 'result' },
477
+ ]
478
+ : undefined;
479
+ await writeFile(path.join(prepared.runOutput, 'provider-result.json'), `${JSON.stringify(sanitized, null, 2)}\n`, { flag: 'wx' });
480
+ status = 'completed';
481
+ }
482
+ catch (caught) {
483
+ status = isTimeout(caught) ? 'timeout' : 'failed';
484
+ usage = null;
485
+ grade = null;
486
+ artifacts = undefined;
487
+ error = { message: redactText(caught instanceof Error ? caught.message : String(caught), secrets()) || 'Provider failed.' };
488
+ if (options.signal?.aborted
489
+ || caught instanceof BoundInputChangedError
490
+ || isCleanupFailure(caught)
491
+ || status === 'timeout'
492
+ || (typeof calculated.spec.provider === 'object'
493
+ && calculated.spec.provider.type === 'codex-workflow'))
494
+ stop = true;
495
+ }
496
+ try {
497
+ await assertInputsUnchanged(calculated, captured);
498
+ await assertSnapshotUnchanged(calculated, snapshot, captured);
499
+ }
500
+ catch (changed) {
501
+ status = 'failed';
502
+ usage = null;
503
+ grade = null;
504
+ artifacts = undefined;
505
+ error = { message: redactText(changed.message, secrets()) };
506
+ stop = true;
507
+ }
508
+ runs.push({
509
+ caseId: evalCase.id,
510
+ variant,
511
+ attempt,
512
+ status,
513
+ durationMs: Math.max(0, Date.now() - attemptStarted),
514
+ outputDir: toPosixRelative(specRoot, runOutput),
515
+ usage,
516
+ grade,
517
+ error,
518
+ ...(artifacts ? { artifacts } : {}),
519
+ });
520
+ if (stop)
521
+ break;
522
+ }
523
+ }
524
+ }
525
+ try {
526
+ await assertInputsUnchanged(calculated, captured);
527
+ await assertSnapshotUnchanged(calculated, snapshot, captured);
528
+ }
529
+ catch (changed) {
530
+ if (runs.length === 0)
531
+ throw changed;
532
+ const last = runs[runs.length - 1];
533
+ last.status = 'failed';
534
+ last.usage = null;
535
+ last.grade = null;
536
+ delete last.artifacts;
537
+ last.error = { message: redactText(changed.message, secrets()) };
538
+ }
539
+ if (options.signal?.aborted && runs.length > 0) {
540
+ const last = runs[runs.length - 1];
541
+ last.status = 'failed';
542
+ last.usage = null;
543
+ last.grade = null;
544
+ delete last.artifacts;
545
+ last.error = { message: abortError().message };
546
+ }
547
+ const result = {
548
+ version: 2,
549
+ specDigest: calculated.specDigest,
550
+ auditDigest: auditValidation.calculated.auditDigest,
551
+ targetDigest: calculated.targetDigest,
552
+ caseDigest: calculated.caseDigest,
553
+ provider: calculated.spec.provider,
554
+ processManagement: redactValue(processManagement, secrets()),
555
+ startedAt,
556
+ finishedAt: new Date().toISOString(),
557
+ runs,
558
+ summary: computeEvalResultSummary(runs),
559
+ };
560
+ const resultPath = path.join(calculated.paths.outputDir, 'result.json');
561
+ const resultDigest = await writeResultSafely(resultPath, result, resultBinding);
562
+ return { valid: true, resultPath, resultDigest, result };
563
+ }
564
+ //# sourceMappingURL=eval-run.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"eval-run.js","sourceRoot":"","sources":["../src/eval-run.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACrE,OAAO,IAAI,MAAM,WAAW,CAAA;AAC5B,OAAO,OAAO,MAAM,cAAc,CAAA;AAClC,OAAO,EAAE,MAAM,UAAU,CAAA;AACzB,OAAO,EAAE,MAAM,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AACrD,OAAO,EAAE,gBAAgB,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA;AAClE,OAAO,EACL,oCAAoC,EACpC,4BAA4B,GAK7B,MAAM,qBAAqB,CAAA;AAC5B,OAAO,EAAE,qBAAqB,EAAE,MAAM,iBAAiB,CAAA;AACvD,OAAO,EACL,wBAAwB,EACxB,uBAAuB,EACvB,kBAAkB,GAKnB,MAAM,kBAAkB,CAAA;AACzB,OAAO,EACL,4BAA4B,GAE7B,MAAM,8BAA8B,CAAA;AACrC,OAAO,EAAE,0BAA0B,EAAE,MAAM,2BAA2B,CAAA;AACtE,OAAO,EAAE,oBAAoB,EAA2B,MAAM,gBAAgB,CAAA;AAC9E,OAAO,EAAE,2BAA2B,EAAE,MAAM,yBAAyB,CAAA;AACrE,OAAO,EACL,sBAAsB,GAGvB,MAAM,gBAAgB,CAAA;AACvB,OAAO,EAAE,6BAA6B,EAAE,MAAM,gCAAgC,CAAA;AAC9E,OAAO,EAAE,2BAA2B,EAAE,MAAM,cAAc,CAAA;AAC1D,OAAO,EACL,4BAA4B,GAE7B,MAAM,qBAAqB,CAAA;AAC5B,OAAO,EAAE,+BAA+B,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAA;AACxF,OAAO,EACL,kBAAkB,GAGnB,MAAM,qBAAqB,CAAA;AA4C5B,MAAM,sBAAuB,SAAQ,KAAK;CAAG;AAE7C,SAAS,YAAY,CAAC,SAA4B;IAChD,OAAO,gBAAgB,CAAC,SAAS,EAAE,oCAAoC,CAAC,CAAA;AAC1E,CAAC;AAED,SAAS,UAAU,CAAC,KAAa,EAAE,OAA0B;IAC3D,OAAO,aAAa,CAAC,KAAK,EAAE,OAAO,CAAC,CAAA;AACtC,CAAC;AAED,SAAS,WAAW,CAAI,KAAQ,EAAE,OAA0B;IAC1D,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,UAAU,CAAC,KAAK,EAAE,OAAO,CAAM,CAAA;IACrE,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,WAAW,CAAC,KAAK,EAAE,OAAO,CAAC,CAAM,CAAA;IACvF,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QACvC,OAAO,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC;YACpE,UAAU,CAAC,GAAG,EAAE,OAAO,CAAC;YACxB,WAAW,CAAC,KAAK,EAAE,OAAO,CAAC;SAC5B,CAAC,CAAM,CAAA;IACV,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,UAA8B;IACxD,OAAO;QACL,SAAS,EAAE,MAAM,+BAA+B,CAAC,UAAU,CAAC,KAAK,CAAC,SAAS,CAAC;QAC5E,MAAM,EAAE,MAAM,kBAAkB,CAAC,UAAU,CAAC,KAAK,CAAC,MAAM,CAAC;QACzD,KAAK,EAAE,MAAM,CAAC,MAAM,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACrD,cAAc,EAAE,UAAU,CAAC,KAAK,CAAC,cAAc;YAC7C,CAAC,CAAC,MAAM,kBAAkB,CAAC,UAAU,CAAC,KAAK,CAAC,cAAc,CAAC;YAC3D,CAAC,CAAC,IAAI;KACT,CAAA;AACH,CAAC;AAED,SAAS,WAAW,CAAC,IAAuB,EAAE,KAAwB;IACpE,OAAO,IAAI,CAAC,SAAS,KAAK,KAAK,CAAC,SAAS;QACvC,IAAI,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM;QAC5B,IAAI,CAAC,KAAK,KAAK,KAAK,CAAC,KAAK;QAC1B,IAAI,CAAC,cAAc,KAAK,KAAK,CAAC,cAAc,CAAA;AAChD,CAAC;AAED,KAAK,UAAU,qBAAqB,CAClC,UAA8B,EAC9B,QAA2B;IAE3B,IAAI,OAA0B,CAAA;IAC9B,IAAI,CAAC;QACH,OAAO,GAAG,MAAM,YAAY,CAAC,UAAU,CAAC,CAAA;IAC1C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,sBAAsB,CAAC,gCAAiC,KAAe,CAAC,OAAO,EAAE,CAAC,CAAA;IAC9F,CAAC;IACD,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,OAAO,CAAC,EAAE,CAAC;QAC9E,MAAM,IAAI,sBAAsB,CAC9B,2EAA2E,CAC5E,CAAA;IACH,CAAC;AACH,CAAC;AAED,SAAS,gBAAgB,CACvB,UAA8B,EAC9B,OAA0B;IAE1B,OAAO,OAAO,CAAC,SAAS,KAAK,UAAU,CAAC,eAAe;WAClD,OAAO,CAAC,SAAS,KAAK,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM;WACtD,OAAO,CAAC,MAAM,KAAK,UAAU,CAAC,YAAY;WAC1C,OAAO,CAAC,KAAK,KAAK,UAAU,CAAC,UAAU;WACvC,OAAO,CAAC,cAAc,KAAK,UAAU,CAAC,oBAAoB,CAAA;AACjE,CAAC;AAED,SAAS,sBAAsB,CAC7B,UAA8B,EAC9B,OAA0B;IAE1B,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,OAAO,CAAC,EAAE,CAAC;QAC3C,MAAM,IAAI,sBAAsB,CAAC,yDAAyD,CAAC,CAAA;IAC7F,CAAC;AACH,CAAC;AAED,SAAS,mBAAmB,CAC1B,QAA+B,EAC/B,QAA0B;IAE1B,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,aAAa,IAAI,CAAC,QAAQ,CAAC,YAAY,EAAE,CAAC;QAC5D,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAA;IAC1E,CAAC;IACD,OAAO;QACL,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,aAAa;QACnC,KAAK,EAAE,QAAQ,CAAC,YAAY;KAC7B,CAAA;AACH,CAAC;AAED,MAAM,UAAU,6BAA6B,CAC3C,QAA+B,EAC/B,QAA0B;IAE1B,MAAM,YAAY,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC;YAC/B,QAAQ,CAAC,KAAK,CAAC,aAAa;YAC5B,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,YAAY,IAAI,EAAE,CAAC;SACvC,CAAC,CAAC,CAAA;IACH,MAAM,UAAU,GAAG,QAAQ,CAAC,UAAuB,CAAA;IACnD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,IAAI,OAAO,QAAQ,CAAC,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC;QAClG,CAAC,CAAC,QAAQ,CAAC,KAAgC;QAC3C,CAAC,CAAC,EAAE,CAAA;IACN,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,IAAI,OAAO,QAAQ,CAAC,QAAQ,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC;QAC9G,CAAC,CAAC,QAAQ,CAAC,QAAmC;QAC9C,CAAC,CAAC,EAAE,CAAA;IACN,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,IAAI,OAAO,QAAQ,CAAC,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC;QAClG,CAAC,CAAC,QAAQ,CAAC,KAAgC;QAC3C,CAAC,CAAC,IAAI,CAAA;IACR,MAAM,kBAAkB,GAAG,QAAQ,CAAC,kBAAkB,KAAK,SAAS;QAClE,CAAC,CAAC,IAAI;QACN,CAAC,CAAC,QAAQ,CAAC,kBAAkB,CAAA;IAC/B,MAAM,eAAe,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,eAAe,CAAC;QAC1D,CAAC,CAAC,KAAK,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;YAClC,IAAI,OAAO,KAAK,KAAK,QAAQ;gBAAE,OAAO,KAAK,CAAA;YAC3C,MAAM,GAAG,GAAG,KAAgC,CAAA;YAC5C,OAAO;gBACL,gBAAgB,EAAE,GAAG,CAAC,cAAc;gBACpC,GAAG,CAAC,GAAG,CAAC,qBAAqB,KAAK,SAAS;oBACzC,CAAC,CAAC,EAAE;oBACJ,CAAC,CAAC,EAAE,uBAAuB,EAAE,GAAG,CAAC,qBAAqB,EAAE,CAAC;gBAC3D,0BAA0B,EAAE,GAAG,CAAC,wBAAwB;aACzD,CAAA;QACH,CAAC,CAAC;QACJ,CAAC,CAAC,EAAE,CAAA;IACN,MAAM,cAAc,GAAG,KAAK,CAAC,cAAc,IAAI,OAAO,KAAK,CAAC,cAAc,KAAK,QAAQ;WAClF,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,cAAc,CAAC;QACvC,CAAC,CAAC,KAAK,CAAC,cAAyC;QACjD,CAAC,CAAC,EAAE,CAAA;IACN,MAAM,uBAAuB,GAAG,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,uBAAuB,CAAC;QAC7E,CAAC,CAAC,QAAQ,CAAC,uBAAuB;QAClC,CAAC,CAAC,EAAE,CAAA;IACN,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC;QACjD,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE;YAClC,MAAM,GAAG,GAAG,QAAmC,CAAA;YAC/C,MAAM,EAAE,WAAW,EAAE,GAAG,gBAAgB,EAAE,GAAG,GAAG,CAAA;YAChD,OAAO;gBACL,GAAG,gBAAgB;gBACnB,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC5E,CAAA;QACH,CAAC,CAAC;QACJ,CAAC,CAAC,EAAE,CAAA;IACN,OAAO;QACL,KAAK,EAAE;YACL,EAAE,EAAE,QAAQ,CAAC,EAAE;YACf,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,gBAAgB,EAAE,eAAe;YACjC,eAAe,EAAE,QAAQ,CAAC,MAAM,IAAI,wDAAwD;YAC5F,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACtF,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,KAAK,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC;gBAClE,CAAC,CAAC,EAAE,WAAW,EAAE,EAAE,WAAW,EAAE,KAAK,CAAC,WAAW,EAAE,EAAE;gBACrD,CAAC,CAAC,EAAE,CAAC;YACP,OAAO,EAAE,kBAAkB,KAAK,KAAK,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,WAAW;YACvE,MAAM,EAAE;gBACN,MAAM,EAAE,UAAU;gBAClB,mBAAmB,EAAE,kBAAkB;gBACvC,GAAG,CAAC,uBAAuB,CAAC,MAAM,GAAG,CAAC;oBACpC,CAAC,CAAC,EAAE,yBAAyB,EAAE,uBAAuB,EAAE;oBACxD,CAAC,CAAC,EAAE,CAAC;gBACP,GAAG,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC/C;SACF;QACD,SAAS,EAAE,QAAQ,CAAC,KAAK,CAAC,aAAa;QACvC,UAAU,EAAE,YAAY;QACxB,SAAS,EAAE,QAAQ,CAAC,KAAK,CAAC,OAAO,EAAE,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,UAAU;QACjF,gBAAgB,EAAE,QAAQ,CAAC,KAAK,CAAC,OAAO,EAAE,WAAW,IAAI,EAAE;QAC3D,UAAU;QACV,GAAG,CAAC,KAAK,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,iBAAiB,CAAC;YAClD,CAAC,CAAC,EAAE,eAAe,EAAE,KAAK,CAAC,eAAe,EAAE;YAC5C,CAAC,CAAC,EAAE,CAAC;KACR,CAAA;AACH,CAAC;AAED,SAAS,YAAY,CAAC,QAA+B,EAAE,QAA0B;IAC/E,IAAI,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,SAAS;QAAE,OAAO,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAA;IACrF,IAAI,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,cAAc;QAAE,OAAO,6BAA6B,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAA;IACpG,OAAO,QAAQ,CAAA;AACjB,CAAC;AAED,KAAK,UAAU,cAAc,CAAC,OAO7B;IAMC,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAC3B,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,SAAS,EAClC,MAAM,EACN,OAAO,CAAC,MAAM,EACd,OAAO,CAAC,OAAO,EACf,WAAW,OAAO,CAAC,OAAO,EAAE,CAC7B,CAAA;IACD,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,WAAW,EAAE,SAAS,CAAC,CAAA;IACpE,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,WAAW,EAAE,MAAM,CAAC,CAAA;IAChE,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAA;IAClD,MAAM,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAA;IAC7C,MAAM,KAAK,CAAC,WAAW,CAAC,CAAA;IACxB,IAAI,CAAC;QACH,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC,CAAA;QAChD,MAAM,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,SAAS,EAAE,SAAS,CAAC,EAAE,aAAa,EAAE;YAC7E,WAAW,EAAE,KAAK;YAClB,SAAS,EAAE,KAAK;YAChB,YAAY,EAAE,IAAI;SACnB,CAAC,CAAA;QACF,MAAM,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC,EAAE,YAAY,EAAE;YACzE,WAAW,EAAE,KAAK;YAClB,SAAS,EAAE,KAAK;YAChB,YAAY,EAAE,IAAI;SACnB,CAAC,CAAA;QACF,MAAM,KAAK,CAAC,SAAS,CAAC,CAAA;IACxB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA;QAC5B,MAAM,KAAK,CAAA;IACb,CAAC;IACD,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,KAAK,SAAS;QAC1C,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,aAAa;QAChC,CAAC,CAAC,OAAO,CAAC,OAAO,KAAK,kBAAkB;YACtC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,cAAc;YACjC,CAAC,CAAC,IAAI,CAAA;IACV,IAAI,MAAM,EAAE,CAAC;QACX,IAAI,CAAC;YACH,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAC3B,aAAa,EACb,SAAS,EACT,QAAQ,EACR,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,aAAa,CACrC,CAAA;YACD,IAAI,MAAM,EAAE,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;gBACrC,MAAM,IAAI,KAAK,CAAC,mEAAmE,CAAC,CAAA;YACtF,CAAC;YACD,MAAM,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAA;YAC7C,MAAM,EAAE,CAAC,IAAI,CAAC,MAAM,EAAE,WAAW,EAAE;gBACjC,WAAW,EAAE,KAAK;gBAClB,SAAS,EAAE,KAAK;gBAChB,YAAY,EAAE,IAAI;aACnB,CAAC,CAAA;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA;YAC5B,MAAM,KAAK,CAAA;QACb,CAAC;IACH,CAAC;IACD,OAAO,EAAE,WAAW,EAAE,aAAa,EAAE,YAAY,EAAE,SAAS,EAAE,CAAA;AAChE,CAAC;AAED,KAAK,UAAU,sBAAsB,CACnC,UAA8B,EAC9B,QAA2B,EAC3B,UAAkB;IAElB,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,SAAS,EAAE,YAAY,CAAC,CAAA;IAChE,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,WAAW,CAAC,CAAA;IAC9C,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,EAAE,SAAS,CAAC,CAAA;IAC3D,MAAM,cAAc,GAAG,UAAU,CAAC,KAAK,CAAC,cAAc;QACpD,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,CAAC;QACxC,CAAC,CAAC,IAAI,CAAA;IACR,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,YAAY,CAAC,CAAA;IAC3C,MAAM,KAAK,CAAC,IAAI,CAAC,CAAA;IACjB,IAAI,CAAC;QACH,MAAM,KAAK,CAAC,SAAS,CAAC,CAAA;QACtB,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC,CAAA;QACvC,MAAM,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,SAAS,EAAE,SAAS,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,EAAE;YAC/F,WAAW,EAAE,KAAK;YAClB,SAAS,EAAE,KAAK;YAChB,YAAY,EAAE,IAAI;SACnB,CAAC,CAAA;QACF,MAAM,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,SAAS,EAAE,MAAM,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,EAAE;YACzF,WAAW,EAAE,KAAK;YAClB,SAAS,EAAE,KAAK;YAChB,YAAY,EAAE,IAAI;SACnB,CAAC,CAAA;QACF,MAAM,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,MAAM,EAAE,aAAa,EAAE;YACpD,WAAW,EAAE,KAAK;YAClB,SAAS,EAAE,KAAK;YAChB,YAAY,EAAE,IAAI;SACnB,CAAC,CAAA;QACF,MAAM,SAAS,CAAC,KAAK,EAAE,UAAU,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAA;QAC/D,IAAI,UAAU,CAAC,KAAK,CAAC,cAAc,IAAI,cAAc,EAAE,CAAC;YACtD,MAAM,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,cAAc,EAAE,cAAc,EAAE;gBAC7D,WAAW,EAAE,KAAK;gBAClB,SAAS,EAAE,KAAK;gBAChB,YAAY,EAAE,IAAI;aACnB,CAAC,CAAA;QACJ,CAAC;QACD,MAAM,eAAe,GAAsB;YACzC,SAAS,EAAE,MAAM,+BAA+B,CAAC,SAAS,CAAC;YAC3D,MAAM,EAAE,MAAM,kBAAkB,CAAC,aAAa,CAAC;YAC/C,KAAK,EAAE,MAAM,CAAC,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAC;YACpC,cAAc,EAAE,cAAc,CAAC,CAAC,CAAC,MAAM,kBAAkB,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,IAAI;SACjF,CAAA;QACD,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,eAAe,CAAC,EAAE,CAAC;YAC5C,MAAM,IAAI,sBAAsB,CAAC,4DAA4D,CAAC,CAAA;QAChG,CAAC;QACD,sBAAsB,CAAC,UAAU,EAAE,eAAe,CAAC,CAAA;QACnD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,aAAa,EAAE,cAAc,EAAE,KAAK,EAAE,CAAA;IAClE,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;QACrB,MAAM,KAAK,CAAA;IACb,CAAC;AACH,CAAC;AAED,KAAK,UAAU,uBAAuB,CACpC,UAA8B,EAC9B,QAA0B,EAC1B,QAA2B;IAE3B,MAAM,OAAO,GAAsB;QACjC,SAAS,EAAE,MAAM,+BAA+B,CAAC,QAAQ,CAAC,SAAS,CAAC;QACpE,MAAM,EAAE,MAAM,kBAAkB,CAAC,QAAQ,CAAC,aAAa,CAAC;QACxD,KAAK,EAAE,MAAM,CAAC,MAAM,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QAC7C,cAAc,EAAE,QAAQ,CAAC,cAAc;YACrC,CAAC,CAAC,MAAM,kBAAkB,CAAC,QAAQ,CAAC,cAAc,CAAC;YACnD,CAAC,CAAC,IAAI;KACT,CAAA;IACD,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,OAAO,CAAC,EAAE,CAAC;QAC9E,MAAM,IAAI,sBAAsB,CAC9B,0EAA0E,CAC3E,CAAA;IACH,CAAC;AACH,CAAC;AAED,SAAS,SAAS,CAAC,KAAc;IAC/B,OAAO,oBAAoB,CAAC,IAAI,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAA;AAC1F,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAc;IACtC,OAAO,KAAK,YAAY,2BAA2B;QACjD,mCAAmC,CAAC,IAAI,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAA;AACpG,CAAC;AAED,SAAS,UAAU;IACjB,OAAO,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAA;AACjD,CAAC;AAED,KAAK,UAAU,iBAAiB,CAC9B,UAAkB,EAClB,MAAkB,EAClB,OAA0B;IAE1B,MAAM,UAAU,GAAG,MAAM,kBAAkB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IAC5D,IAAI,CAAC,UAAU,CAAC,KAAK;QAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAC1G,IAAI,MAAM,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAA;IACxF,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAA;IACjE,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,WAAW,UAAU,EAAE,MAAM,CAAC,CAAA;IACpF,MAAM,SAAS,CAAC,SAAS,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAA;IAC9D,IAAI,CAAC;QACH,IAAI,MAAM,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAA;QAChG,MAAM,MAAM,CAAC,SAAS,EAAE,UAAU,CAAC,CAAA;IACrC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAA;QACjD,MAAM,KAAK,CAAA;IACb,CAAC;IACD,OAAO,MAAM,CAAC,KAAK,CAAC,CAAA;AACtB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,QAAgB,EAChB,OAA2B;IAE3B,IAAI,CAAC,OAAO,EAAE,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAA;IAC9E,MAAM,cAAc,GAAG,MAAM,oBAAoB,CAAC,QAAQ,CAAC,CAAA;IAC3D,IAAI,CAAC,cAAc,CAAC,KAAK,IAAI,CAAC,cAAc,CAAC,UAAU,EAAE,CAAC;QACxD,MAAM,IAAI,KAAK,CAAC,wBAAwB,cAAc,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAC7E,CAAC;IACD,MAAM,UAAU,GAAG,cAAc,CAAC,UAAU,CAAA;IAC5C,MAAM,QAAQ,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAA;IACxE,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;IACzD,IAAI,MAAM,CAAC,UAAU,CAAC,KAAK,UAAU,CAAC,UAAU,EAAE,CAAC;QACjD,MAAM,IAAI,sBAAsB,CAAC,mDAAmD,CAAC,CAAA;IACvF,CAAC;IACD,MAAM,QAAQ,GAAG,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;IAC5E,IAAI,QAAQ,CAAC,KAAK,CAAC,aAAa,KAAK,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QAClE,MAAM,IAAI,KAAK,CAAC,oEAAoE,CAAC,CAAA;IACvF,CAAC;IACD,2BAA2B,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IAC1E,MAAM,eAAe,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;IAC/D,MAAM,eAAe,GAAG,MAAM,qBAAqB,CAAC,OAAO,CAAC,SAAS,EAAE;QACrE,IAAI,EAAE,UAAU,CAAC,IAAI;QACrB,eAAe;KAChB,CAAC,CAAA;IACF,IAAI,CAAC,eAAe,CAAC,KAAK,IAAI,CAAC,eAAe,CAAC,UAAU,EAAE,CAAC;QAC1D,MAAM,IAAI,KAAK,CAAC,uCAAuC,eAAe,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAC7F,CAAC;IACD,MAAM,aAAa,GAAG,uBAAuB,CAC3C,UAAU,EACV,eAAe,CAAC,UAAU,CAAC,WAAW,EACtC,QAAQ,CACT,CAAA;IACD,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,UAAU,CAAC,CAAA;IAC/C,sBAAsB,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAA;IAC5C,MAAM,qBAAqB,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAA;IACjD,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS;WAC9B,OAAO,CAAC,2BAA2B,EAAE,SAAS;WAC9C,OAAO,CAAC,GAAG,CAAA;IAChB,MAAM,iBAAiB,GAAG,OAAO,CAAC,2BAA2B,EAAE,QAAQ;WAClE,SAAS,CAAC,uBAAuB;WACjC,OAAO,CAAA;IACZ,MAAM,eAAe,GAAG,OAAO,CAAC,6BAA6B,EAAE,QAAQ;WAClE,OAAO,CAAC,6BAA6B,EAAE,WAAW,EAAE,QAAQ;WAC5D,OAAO,CAAC,QAAQ,CAAA;IACrB,MAAM,gBAAgB,GAAG,MAAM,0BAA0B,CACvD,EAAE,UAAU,EAAE,iBAAiB,EAAE,SAAS,EAAE,QAAQ,EAAE,eAAe,EAAE,EACvE,OAAO,CAAC,6BAA6B,EAAE,sBAAsB,CAC9D,CAAA;IACD,MAAM,QAAQ,GAAG,gBAAgB,IAAI,iBAAiB,CAAA;IACtD,MAAM,iBAAiB,GAAG,MAAM,4BAA4B,CAAC;QAC3D,cAAc,EAAE,OAAO,CAAC,uBAAuB,IAAI,KAAK;QACxD,QAAQ;QACR,SAAS;QACT,YAAY,EAAE;YACZ,GAAG,OAAO,CAAC,6BAA6B;YACxC,sBAAsB,EAAE,KAAK,IAAI,EAAE,CAAC,gBAAgB;SACrD;KACF,CAAC,CAAA;IACF,OAAO,CAAC,2BAA2B,EAAE,CAAC,iBAAiB,CAAC,CAAA;IACxD,MAAM,OAAO,GAAG,MAAM,6BAA6B,CAAC;QAClD,SAAS;QACT,iBAAiB,EAAE,IAAI;QACvB,aAAa,EAAE,OAAO,CAAC,2BAA2B,EAAE,KAAK;eACpD,SAAS,CAAC,yBAAyB;KACzC,CAAC,CAAA;IACF,MAAM,OAAO,GAAG,GAAa,EAAE,CAAC;QAC9B,GAAG,YAAY,CAAC,SAAS,CAAC;QAC1B,GAAG,OAAO,CAAC,eAAe;KAC3B,CAAA;IACD,MAAM,2BAA2B,GAAoC;QACnE,GAAG,OAAO,CAAC,2BAA2B;QACtC,SAAS;QACT,QAAQ;QACR,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,SAAS;QACjC,YAAY,EAAE,OAAO;QACrB,iBAAiB;KAClB,CAAA;IACD,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;IAC1C,MAAM,IAAI,GAAoB,EAAE,CAAA;IAChC,IAAI,IAAI,GAAG,KAAK,CAAA;IAChB,MAAM,qBAAqB,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAA;IACjD,MAAM,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAA;IAC5D,MAAM,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;IACvC,IAAI,QAA0B,CAAA;IAC9B,IAAI,CAAC;QACH,QAAQ,GAAG,MAAM,sBAAsB,CAAC,UAAU,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAA;IAC3E,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;QAC3C,MAAM,KAAK,CAAA;IACb,CAAC;IACD,KAAK,MAAM,QAAQ,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;QACtC,IAAI,IAAI;YAAE,MAAK;QACf,KAAK,MAAM,OAAO,IAAI,aAAa,CAAC,eAAe,EAAE,CAAC;YACpD,IAAI,IAAI;gBAAE,MAAK;YACf,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,IAAI,CAAC,EAAE,CAAC;gBACpE,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;gBACjC,IAAI,MAAM,GAA4B,QAAQ,CAAA;gBAC9C,IAAI,KAAK,GAA2B,IAAI,CAAA;gBACxC,IAAI,KAAK,GAA2B,IAAI,CAAA;gBACxC,IAAI,KAAK,GAA2B,IAAI,CAAA;gBACxC,IAAI,SAAqC,CAAA;gBACzC,IAAI,SAAS,GAAG,IAAI,CAAC,IAAI,CACvB,UAAU,CAAC,KAAK,CAAC,SAAS,EAC1B,MAAM,EACN,QAAQ,CAAC,EAAE,EACX,OAAO,EACP,WAAW,OAAO,EAAE,EACpB,QAAQ,CACT,CAAA;gBACD,IAAI,CAAC;oBACH,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO;wBAAE,MAAM,UAAU,EAAE,CAAA;oBAC/C,MAAM,qBAAqB,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAA;oBACjD,MAAM,uBAAuB,CAAC,UAAU,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAA;oBAC7D,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC;wBACpC,UAAU;wBACV,QAAQ;wBACR,QAAQ;wBACR,MAAM,EAAE,QAAQ,CAAC,EAAE;wBACnB,OAAO;wBACP,OAAO;qBACR,CAAC,CAAA;oBACF,SAAS,GAAG,QAAQ,CAAC,SAAS,CAAA;oBAC9B,MAAM,KAAK,GAAyB;wBAClC,IAAI,EAAE,YAAY,CAAC,QAAQ,EAAE,QAAQ,CAAC;wBACtC,OAAO;wBACP,OAAO;wBACP,aAAa,EAAE,QAAQ,CAAC,aAAa;wBACrC,YAAY,EAAE,QAAQ,CAAC,YAAY;wBACnC,SAAS,EAAE,QAAQ,CAAC,SAAS;wBAC7B,SAAS,EAAE,UAAU,CAAC,IAAI,CAAC,SAAS;wBACpC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;qBACtD,CAAA;oBACD,MAAM,cAAc,GAAG,OAAO,UAAU,CAAC,IAAI,CAAC,QAAQ,KAAK,QAAQ;wBACjE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,kBAAkB,IAAI,sBAAsB,CAAC,CAC1D,UAAU,CAAC,IAAI,CAAC,QAAQ,EACxB,EAAE,GAAG,KAAK,EAAE,QAAQ,EAAE,QAAQ,CAAC,WAAW,EAAE,EAC5C,2BAA2B,CAC5B;wBACH,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,KAAK,gBAAgB;4BAClD,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,qBAAqB,IAAI,4BAA4B,CAAC,CAAC;gCACpE,OAAO,EAAE,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAgC;gCAClE,QAAQ,EAAE,QAAQ,CAAC,WAAW;gCAC9B,KAAK;gCACL,SAAS;gCACT,YAAY,EAAE,OAAO;gCACrB,iBAAiB;6BAClB,CAAC;4BACJ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,4BAA4B,IAAI,4BAA4B,CAAC,CAAC;gCAC3E,QAAQ,EAAE;oCACR,GAAG,UAAU,CAAC,IAAI,CAAC,QAAQ;oCAC3B,OAAO,EAAE,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAgC;iCACnE;gCACD,QAAQ,EAAE,QAAQ,CAAC,WAAW;gCAC9B,KAAK;gCACL,SAAS;gCACT,YAAY,EAAE,OAAO;gCACrB,QAAQ,EAAE,2BAA2B,CAAC,QAAQ;gCAC9C,iBAAiB;6BAClB,CAAC,CAAA;oBACR,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO;wBAAE,MAAM,UAAU,EAAE,CAAA;oBAC/C,MAAM,SAAS,GAAG,WAAW,CAAC,cAAc,EAAE,OAAO,EAAE,CAAC,CAAA;oBACxD,KAAK,GAAG,SAAS,CAAC,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,SAAS,CAAC,KAAK,EAAE,CAAA;oBAChE,KAAK,GAAG,SAAS,CAAC,KAAK,CAAA;oBACvB,SAAS,GAAG,OAAO,UAAU,CAAC,IAAI,CAAC,QAAQ,KAAK,QAAQ;2BACnD,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,KAAK,gBAAgB;2BAClD,SAAS,CAAC,SAAS;wBACtB,CAAC,CAAC;4BACE,GAAG,SAAS,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,QAAQ,EAAE,CAAC,CAAC;4BAC3D,EAAE,IAAI,EAAE,sBAAsB,EAAE,IAAI,EAAE,QAAQ,EAAE;yBACjD;wBACH,CAAC,CAAC,SAAS,CAAA;oBACb,MAAM,SAAS,CACb,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,sBAAsB,CAAC,EACrD,GAAG,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EACzC,EAAE,IAAI,EAAE,IAAI,EAAE,CACf,CAAA;oBACD,MAAM,GAAG,WAAW,CAAA;gBACtB,CAAC;gBAAC,OAAO,MAAM,EAAE,CAAC;oBAChB,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAA;oBACjD,KAAK,GAAG,IAAI,CAAA;oBACZ,KAAK,GAAG,IAAI,CAAA;oBACZ,SAAS,GAAG,SAAS,CAAA;oBACrB,KAAK,GAAG,EAAE,OAAO,EAAE,UAAU,CAAC,MAAM,YAAY,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC,IAAI,kBAAkB,EAAE,CAAA;oBAC3H,IACE,OAAO,CAAC,MAAM,EAAE,OAAO;2BACpB,MAAM,YAAY,sBAAsB;2BACxC,gBAAgB,CAAC,MAAM,CAAC;2BACxB,MAAM,KAAK,SAAS;2BACpB,CACD,OAAO,UAAU,CAAC,IAAI,CAAC,QAAQ,KAAK,QAAQ;+BACzC,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,KAAK,gBAAgB,CACtD;wBACD,IAAI,GAAG,IAAI,CAAA;gBACf,CAAC;gBACD,IAAI,CAAC;oBACH,MAAM,qBAAqB,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAA;oBACjD,MAAM,uBAAuB,CAAC,UAAU,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAA;gBAC/D,CAAC;gBAAC,OAAO,OAAO,EAAE,CAAC;oBACjB,MAAM,GAAG,QAAQ,CAAA;oBACjB,KAAK,GAAG,IAAI,CAAA;oBACZ,KAAK,GAAG,IAAI,CAAA;oBACZ,SAAS,GAAG,SAAS,CAAA;oBACrB,KAAK,GAAG,EAAE,OAAO,EAAE,UAAU,CAAE,OAAiB,CAAC,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,CAAA;oBACtE,IAAI,GAAG,IAAI,CAAA;gBACb,CAAC;gBACD,IAAI,CAAC,IAAI,CAAC;oBACR,MAAM,EAAE,QAAQ,CAAC,EAAE;oBACnB,OAAO;oBACP,OAAO;oBACP,MAAM;oBACN,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,cAAc,CAAC;oBACpD,SAAS,EAAE,eAAe,CAAC,QAAQ,EAAE,SAAS,CAAC;oBAC/C,KAAK;oBACL,KAAK;oBACL,KAAK;oBACL,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBACpC,CAAC,CAAA;gBACF,IAAI,IAAI;oBAAE,MAAK;YACjB,CAAC;QACH,CAAC;IACH,CAAC;IACD,IAAI,CAAC;QACH,MAAM,qBAAqB,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAA;QACjD,MAAM,uBAAuB,CAAC,UAAU,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAA;IAC/D,CAAC;IAAC,OAAO,OAAO,EAAE,CAAC;QACjB,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM,OAAO,CAAA;QACpC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAE,CAAA;QACnC,IAAI,CAAC,MAAM,GAAG,QAAQ,CAAA;QACtB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAA;QACjB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAA;QACjB,OAAO,IAAI,CAAC,SAAS,CAAA;QACrB,IAAI,CAAC,KAAK,GAAG,EAAE,OAAO,EAAE,UAAU,CAAE,OAAiB,CAAC,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,CAAA;IAC7E,CAAC;IACD,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC/C,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAE,CAAA;QACnC,IAAI,CAAC,MAAM,GAAG,QAAQ,CAAA;QACtB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAA;QACjB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAA;QACjB,OAAO,IAAI,CAAC,SAAS,CAAA;QACrB,IAAI,CAAC,KAAK,GAAG,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC,OAAO,EAAE,CAAA;IAChD,CAAC;IACD,MAAM,MAAM,GAAe;QACzB,OAAO,EAAE,CAAC;QACV,UAAU,EAAE,UAAU,CAAC,UAAU;QACjC,WAAW,EAAE,eAAe,CAAC,UAAU,CAAC,WAAW;QACnD,YAAY,EAAE,UAAU,CAAC,YAAY;QACrC,UAAU,EAAE,UAAU,CAAC,UAAU;QACjC,QAAQ,EAAE,UAAU,CAAC,IAAI,CAAC,QAAQ;QAClC,iBAAiB,EAAE,WAAW,CAAC,iBAAiB,EAAE,OAAO,EAAE,CAAC;QAC5D,SAAS;QACT,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACpC,IAAI;QACJ,OAAO,EAAE,wBAAwB,CAAC,IAAI,CAAC;KACxC,CAAA;IACD,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,SAAS,EAAE,aAAa,CAAC,CAAA;IACvE,MAAM,YAAY,GAAG,MAAM,iBAAiB,CAAC,UAAU,EAAE,MAAM,EAAE,aAAa,CAAC,CAAA;IAC/E,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,EAAE,CAAA;AAC1D,CAAC"}
@@ -0,0 +1,57 @@
1
+ import type { Digest, ValidationResult } from './types.js';
2
+ export type EvalProvider = 'codex-trigger' | 'codex-conversation' | {
3
+ type: 'custom-process';
4
+ command: string[];
5
+ } | {
6
+ type: 'codex-workflow';
7
+ command: string[];
8
+ allowedModels: string[];
9
+ };
10
+ export interface EvalSpecPathDigest {
11
+ path: string;
12
+ digest: Digest;
13
+ }
14
+ export type EvalBaseline = {
15
+ type: 'none';
16
+ } | {
17
+ type: 'without-skill';
18
+ } | {
19
+ type: 'previous-version';
20
+ target: EvalSpecPathDigest;
21
+ };
22
+ export interface EvalSpec {
23
+ version: 1;
24
+ source: {
25
+ plugin: string;
26
+ skill: string;
27
+ suite: string;
28
+ profile: string;
29
+ };
30
+ workspace: EvalSpecPathDigest;
31
+ target: EvalSpecPathDigest;
32
+ cases: EvalSpecPathDigest;
33
+ provider: EvalProvider;
34
+ runs: number;
35
+ timeoutMs: number;
36
+ baseline: EvalBaseline;
37
+ outputDir: string;
38
+ }
39
+ export interface EvalSpecCalculated {
40
+ spec: EvalSpec;
41
+ specDigest: Digest;
42
+ workspaceDigest: Digest;
43
+ providerDigest: Digest;
44
+ targetDigest: Digest;
45
+ caseDigest: Digest;
46
+ previousTargetDigest: Digest | null;
47
+ paths: {
48
+ workspace: string;
49
+ target: string;
50
+ cases: string;
51
+ previousTarget: string | null;
52
+ outputDir: string;
53
+ };
54
+ }
55
+ export declare function validateEvalSpec(value: unknown): Promise<ValidationResult<EvalSpec>>;
56
+ export declare function parseEvalSpec(value: unknown): Promise<EvalSpec>;
57
+ export declare function validateEvalSpecFile(specPath: string): Promise<ValidationResult<EvalSpecCalculated>>;