@rudderhq/run-intelligence-core 0.1.0-canary.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. package/LICENSE +186 -0
  2. package/dist/cli/analyze.d.ts +2 -0
  3. package/dist/cli/analyze.d.ts.map +1 -0
  4. package/dist/cli/analyze.js +35 -0
  5. package/dist/cli/analyze.js.map +1 -0
  6. package/dist/cli/common.d.ts +9 -0
  7. package/dist/cli/common.d.ts.map +1 -0
  8. package/dist/cli/common.js +28 -0
  9. package/dist/cli/common.js.map +1 -0
  10. package/dist/cli/compare.d.ts +2 -0
  11. package/dist/cli/compare.d.ts.map +1 -0
  12. package/dist/cli/compare.js +24 -0
  13. package/dist/cli/compare.js.map +1 -0
  14. package/dist/cli/trace-entry.d.ts +2 -0
  15. package/dist/cli/trace-entry.d.ts.map +1 -0
  16. package/dist/cli/trace-entry.js +53 -0
  17. package/dist/cli/trace-entry.js.map +1 -0
  18. package/dist/cli/trace-outline.d.ts +2 -0
  19. package/dist/cli/trace-outline.d.ts.map +1 -0
  20. package/dist/cli/trace-outline.js +43 -0
  21. package/dist/cli/trace-outline.js.map +1 -0
  22. package/dist/create-agent-benchmark.d.ts +113 -0
  23. package/dist/create-agent-benchmark.d.ts.map +1 -0
  24. package/dist/create-agent-benchmark.js +451 -0
  25. package/dist/create-agent-benchmark.js.map +1 -0
  26. package/dist/create-agent-benchmark.test.d.ts +2 -0
  27. package/dist/create-agent-benchmark.test.d.ts.map +1 -0
  28. package/dist/create-agent-benchmark.test.js +289 -0
  29. package/dist/create-agent-benchmark.test.js.map +1 -0
  30. package/dist/diagnosis.d.ts +4 -0
  31. package/dist/diagnosis.d.ts.map +1 -0
  32. package/dist/diagnosis.js +360 -0
  33. package/dist/diagnosis.js.map +1 -0
  34. package/dist/diagnosis.test.d.ts +2 -0
  35. package/dist/diagnosis.test.d.ts.map +1 -0
  36. package/dist/diagnosis.test.js +85 -0
  37. package/dist/diagnosis.test.js.map +1 -0
  38. package/dist/index.d.ts +10 -0
  39. package/dist/index.d.ts.map +1 -0
  40. package/dist/index.js +10 -0
  41. package/dist/index.js.map +1 -0
  42. package/dist/langfuse-scores.d.ts +13 -0
  43. package/dist/langfuse-scores.d.ts.map +1 -0
  44. package/dist/langfuse-scores.js +95 -0
  45. package/dist/langfuse-scores.js.map +1 -0
  46. package/dist/langfuse-scores.test.d.ts +2 -0
  47. package/dist/langfuse-scores.test.d.ts.map +1 -0
  48. package/dist/langfuse-scores.test.js +121 -0
  49. package/dist/langfuse-scores.test.js.map +1 -0
  50. package/dist/loaders/filesystem.d.ts +26 -0
  51. package/dist/loaders/filesystem.d.ts.map +1 -0
  52. package/dist/loaders/filesystem.js +97 -0
  53. package/dist/loaders/filesystem.js.map +1 -0
  54. package/dist/loaders/rudder.d.ts +28 -0
  55. package/dist/loaders/rudder.d.ts.map +1 -0
  56. package/dist/loaders/rudder.js +81 -0
  57. package/dist/loaders/rudder.js.map +1 -0
  58. package/dist/parsers.d.ts +3 -0
  59. package/dist/parsers.d.ts.map +1 -0
  60. package/dist/parsers.js +23 -0
  61. package/dist/parsers.js.map +1 -0
  62. package/dist/trace.d.ts +42 -0
  63. package/dist/trace.d.ts.map +1 -0
  64. package/dist/trace.js +167 -0
  65. package/dist/trace.js.map +1 -0
  66. package/dist/trace.test.d.ts +2 -0
  67. package/dist/trace.test.d.ts.map +1 -0
  68. package/dist/trace.test.js +92 -0
  69. package/dist/trace.test.js.map +1 -0
  70. package/dist/transcript.d.ts +7 -0
  71. package/dist/transcript.d.ts.map +1 -0
  72. package/dist/transcript.js +70 -0
  73. package/dist/transcript.js.map +1 -0
  74. package/dist/types.d.ts +122 -0
  75. package/dist/types.d.ts.map +1 -0
  76. package/dist/types.js +2 -0
  77. package/dist/types.js.map +1 -0
  78. package/package.json +59 -0
@@ -0,0 +1,451 @@
1
+ import { buildObservedRunTrace, previewTextForTranscriptEntry } from "./trace.js";
2
+ export const CREATE_AGENT_BENCHMARK_MARKER = "rudder-create-agent-benchmark";
3
+ export const CREATE_AGENT_EVALUATION_VERSION = "create-agent-eval-v1";
4
+ export const CREATE_AGENT_LOCAL_JUDGE_VERSION = "local:create-agent-judge-v1";
5
+ const FALLBACK_PATTERNS = [
6
+ { pattern: /\bmkdir\b.*\.agents\//i, label: "mkdir .agents path" },
7
+ { pattern: /\b(create|write|save).*(SKILL\.md|COMPANY\.md|instructions? file)/i, label: "local instructions write" },
8
+ { pattern: /create agent director(y|ies)|local agent director(y|ies)/i, label: "local agent directory fallback" },
9
+ { pattern: /\.agents\/skills\/|server\/resources\/bundled-skills\/rudder-create-agent/i, label: "manual skill file browsing" },
10
+ ];
11
+ const TERMINAL_RUN_STATUSES = new Set(["succeeded", "failed", "cancelled", "timed_out"]);
12
+ function nonEmptyString(value) {
13
+ return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
14
+ }
15
+ function toStringArray(value) {
16
+ if (!Array.isArray(value))
17
+ return [];
18
+ return value.map((item) => nonEmptyString(item)).filter((item) => Boolean(item));
19
+ }
20
+ export function parseCreateAgentCase(raw) {
21
+ if (!raw || typeof raw !== "object") {
22
+ throw new Error("Create-agent benchmark case must be an object.");
23
+ }
24
+ const value = raw;
25
+ const id = nonEmptyString(value.id);
26
+ const prompt = nonEmptyString(value.prompt);
27
+ const expectedPath = nonEmptyString(value.expectedPath);
28
+ if (!id)
29
+ throw new Error("Create-agent benchmark case is missing a non-empty id.");
30
+ if (!prompt)
31
+ throw new Error(`Case ${id} is missing a non-empty prompt.`);
32
+ if (expectedPath !== "direct_create" && expectedPath !== "approval_required" && expectedPath !== "reject_or_escalate") {
33
+ throw new Error(`Case ${id} has invalid expectedPath: ${String(value.expectedPath)}`);
34
+ }
35
+ const expectedAgentShape = (value.expectedAgentShape ?? {});
36
+ if (!expectedAgentShape || typeof expectedAgentShape !== "object" || Array.isArray(expectedAgentShape)) {
37
+ throw new Error(`Case ${id} has invalid expectedAgentShape.`);
38
+ }
39
+ const fixtures = (value.fixtures ?? {});
40
+ if (fixtures && (typeof fixtures !== "object" || Array.isArray(fixtures))) {
41
+ throw new Error(`Case ${id} has invalid fixtures.`);
42
+ }
43
+ return {
44
+ id,
45
+ prompt,
46
+ expectedPath,
47
+ expectedAgentShape: {
48
+ name: nonEmptyString(expectedAgentShape.name) ?? undefined,
49
+ role: nonEmptyString(expectedAgentShape.role) ?? undefined,
50
+ title: expectedAgentShape.title === null ? null : nonEmptyString(expectedAgentShape.title) ?? undefined,
51
+ reportsTo: expectedAgentShape.reportsTo === null ? null : nonEmptyString(expectedAgentShape.reportsTo) ?? undefined,
52
+ reportsToFixture: expectedAgentShape.reportsToFixture === null
53
+ ? null
54
+ : nonEmptyString(expectedAgentShape.reportsToFixture) ?? undefined,
55
+ agentRuntimeType: nonEmptyString(expectedAgentShape.agentRuntimeType) ?? undefined,
56
+ desiredSkills: toStringArray(expectedAgentShape.desiredSkills),
57
+ sourceIssueRequired: typeof expectedAgentShape.sourceIssueRequired === "boolean"
58
+ ? expectedAgentShape.sourceIssueRequired
59
+ : undefined,
60
+ },
61
+ fixtures: {
62
+ requiredApproval: typeof fixtures.requiredApproval === "boolean" ? fixtures.requiredApproval : null,
63
+ requiredFixtureKeys: toStringArray(fixtures.requiredFixtureKeys),
64
+ },
65
+ judgeFocus: toStringArray(value.judgeFocus),
66
+ };
67
+ }
68
+ export function buildCreateAgentBenchmarkMetadata(input) {
69
+ return {
70
+ workflow: "create-agent",
71
+ benchmark: true,
72
+ benchmarkCaseId: input.testCase.id,
73
+ expectedPath: input.testCase.expectedPath,
74
+ requestedRole: input.testCase.expectedAgentShape.role ?? null,
75
+ requestedRuntimeType: input.testCase.expectedAgentShape.agentRuntimeType ?? null,
76
+ evaluationVersion: CREATE_AGENT_EVALUATION_VERSION,
77
+ judgeVersion: input.judgeVersion ?? null,
78
+ };
79
+ }
80
+ export function buildCreateAgentBenchmarkTags(metadata) {
81
+ return [
82
+ "workflow:create-agent",
83
+ "benchmark:true",
84
+ `benchmark-case:${metadata.benchmarkCaseId}`,
85
+ ];
86
+ }
87
+ export function encodeCreateAgentBenchmarkMetadataComment(metadata) {
88
+ return `<!-- ${CREATE_AGENT_BENCHMARK_MARKER}:${JSON.stringify(metadata)} -->`;
89
+ }
90
+ export function appendCreateAgentBenchmarkMetadata(description, metadata) {
91
+ const base = description.trim();
92
+ return `${base}\n\n${encodeCreateAgentBenchmarkMetadataComment(metadata)}\n`;
93
+ }
94
+ export function extractCreateAgentBenchmarkMetadata(text) {
95
+ if (!text)
96
+ return null;
97
+ const match = text.match(new RegExp(`<!--\\s*${CREATE_AGENT_BENCHMARK_MARKER}:(\\{[\\s\\S]*?\\})\\s*-->`, "i"));
98
+ if (!match?.[1])
99
+ return null;
100
+ try {
101
+ return coerceCreateAgentBenchmarkMetadata(JSON.parse(match[1]));
102
+ }
103
+ catch {
104
+ return null;
105
+ }
106
+ }
107
+ export function coerceCreateAgentBenchmarkMetadata(raw) {
108
+ if (!raw || typeof raw !== "object" || Array.isArray(raw))
109
+ return null;
110
+ const parsed = raw;
111
+ if (parsed.workflow !== "create-agent"
112
+ || parsed.benchmark !== true
113
+ || nonEmptyString(parsed.benchmarkCaseId) == null) {
114
+ return null;
115
+ }
116
+ const expectedPath = nonEmptyString(parsed.expectedPath);
117
+ if (expectedPath !== "direct_create" && expectedPath !== "approval_required" && expectedPath !== "reject_or_escalate") {
118
+ return null;
119
+ }
120
+ return {
121
+ workflow: "create-agent",
122
+ benchmark: true,
123
+ benchmarkCaseId: nonEmptyString(parsed.benchmarkCaseId),
124
+ expectedPath,
125
+ requestedRole: nonEmptyString(parsed.requestedRole),
126
+ requestedRuntimeType: nonEmptyString(parsed.requestedRuntimeType),
127
+ evaluationVersion: nonEmptyString(parsed.evaluationVersion) ?? CREATE_AGENT_EVALUATION_VERSION,
128
+ judgeVersion: nonEmptyString(parsed.judgeVersion),
129
+ };
130
+ }
131
+ function resolveExpectedReportsTo(testCase, fixtureRefs = {}) {
132
+ if (testCase.expectedAgentShape.reportsTo !== undefined)
133
+ return testCase.expectedAgentShape.reportsTo;
134
+ if (testCase.expectedAgentShape.reportsToFixture !== undefined) {
135
+ return fixtureRefs[testCase.expectedAgentShape.reportsToFixture ?? ""] ?? null;
136
+ }
137
+ return undefined;
138
+ }
139
+ function evaluateFieldEquality(actual, expected) {
140
+ if (expected === undefined) {
141
+ return { value: "not_applicable", comment: "No expected value defined for this case." };
142
+ }
143
+ if ((actual ?? null) === expected) {
144
+ return { value: "pass", comment: `Matched expected value ${expected ?? "null"}.` };
145
+ }
146
+ return {
147
+ value: "fail",
148
+ comment: `Expected ${expected ?? "null"} but observed ${actual ?? "null"}.`,
149
+ };
150
+ }
151
+ function findFilesystemFallbackMatches(detail) {
152
+ const haystacks = [
153
+ detail.logContent ?? "",
154
+ detail.run.stdoutExcerpt ?? "",
155
+ detail.run.stderrExcerpt ?? "",
156
+ ...detail.events.map((event) => event.message ?? ""),
157
+ ...detail.transcript.map((entry) => previewTextForTranscriptEntry(entry, 400)),
158
+ ];
159
+ const matches = new Set();
160
+ for (const haystack of haystacks) {
161
+ for (const { pattern, label } of FALLBACK_PATTERNS) {
162
+ if (pattern.test(haystack)) {
163
+ matches.add(label);
164
+ }
165
+ }
166
+ }
167
+ return [...matches];
168
+ }
169
+ function summarizeFinalOutput(detail) {
170
+ const trace = buildObservedRunTrace(detail);
171
+ const lastModelStep = [...trace.steps].reverse().find((step) => step.isModelEntry && step.preview);
172
+ if (lastModelStep?.preview)
173
+ return lastModelStep.preview;
174
+ const lastLoose = [...trace.steps].reverse().find((step) => step.preview);
175
+ return lastLoose?.preview ?? detail.run.error ?? "No final output summary available";
176
+ }
177
+ function findPrimaryAgent(capturedAgents, expectedPath) {
178
+ if (capturedAgents.length === 0)
179
+ return null;
180
+ if (expectedPath === "approval_required") {
181
+ return capturedAgents.find((item) => item.agent.status === "pending_approval") ?? capturedAgents[0] ?? null;
182
+ }
183
+ return capturedAgents[0] ?? null;
184
+ }
185
+ function diffSkillKeys(snapshot) {
186
+ if (!snapshot)
187
+ return [];
188
+ return [...snapshot.desiredSkills].sort();
189
+ }
190
+ function evaluateDesiredSkills(actualSkills, expectedSkills) {
191
+ if (!expectedSkills || expectedSkills.length === 0) {
192
+ return {
193
+ value: "not_applicable",
194
+ comment: "No expected desired skills defined for this case.",
195
+ };
196
+ }
197
+ const actual = [...actualSkills].sort();
198
+ const expected = [...expectedSkills].sort();
199
+ if (JSON.stringify(actual) === JSON.stringify(expected)) {
200
+ return {
201
+ value: "pass",
202
+ comment: `Matched desired skills (${expected.join(", ") || "none"}).`,
203
+ metadata: { expectedSkills: expected, actualSkills: actual },
204
+ };
205
+ }
206
+ return {
207
+ value: "fail",
208
+ comment: `Expected skills [${expected.join(", ")}] but observed [${actual.join(", ")}].`,
209
+ metadata: { expectedSkills: expected, actualSkills: actual },
210
+ };
211
+ }
212
+ function evaluateSourceIssueLink(input, primaryApproval) {
213
+ if (!input.testCase.expectedAgentShape.sourceIssueRequired) {
214
+ return {
215
+ value: "not_applicable",
216
+ comment: "Case does not require source issue linkage.",
217
+ };
218
+ }
219
+ if (!input.issueId) {
220
+ return {
221
+ value: "uncertain",
222
+ comment: "Run detail does not expose a linked issue id.",
223
+ };
224
+ }
225
+ if (!primaryApproval) {
226
+ return {
227
+ value: "uncertain",
228
+ comment: "No approval surface exists for direct-create linkage verification.",
229
+ };
230
+ }
231
+ if (primaryApproval.issueIds.includes(input.issueId)) {
232
+ return {
233
+ value: "pass",
234
+ comment: `Approval ${primaryApproval.approval.id} links benchmark issue ${input.issueId}.`,
235
+ };
236
+ }
237
+ return {
238
+ value: "fail",
239
+ comment: `Approval ${primaryApproval.approval.id} does not link benchmark issue ${input.issueId}.`,
240
+ metadata: { issueIds: primaryApproval.issueIds, expectedIssueId: input.issueId },
241
+ };
242
+ }
243
+ function aggregateOverallCheck(checks) {
244
+ if (checks.some((check) => check.value === "fail")) {
245
+ return {
246
+ value: "fail",
247
+ comment: "At least one deterministic correctness check failed.",
248
+ };
249
+ }
250
+ if (checks.some((check) => check.value === "uncertain")) {
251
+ return {
252
+ value: "uncertain",
253
+ comment: "No deterministic failures were found, but at least one required check is uncertain.",
254
+ };
255
+ }
256
+ return {
257
+ value: "pass",
258
+ comment: "All deterministic create-agent correctness checks passed.",
259
+ };
260
+ }
261
+ export function evaluateCreateAgentBenchmark(input) {
262
+ const primaryAgent = findPrimaryAgent(input.createdAgents, input.testCase.expectedPath);
263
+ const primaryApproval = input.createdApprovals[0] ?? null;
264
+ const expectedReportsTo = resolveExpectedReportsTo(input.testCase, input.fixtureRefs);
265
+ const filesystemFallbackMatches = findFilesystemFallbackMatches(input.runDetail);
266
+ const finalOutputSummary = summarizeFinalOutput(input.runDetail);
267
+ const runSucceeded = input.runDetail.run.status === "succeeded";
268
+ const runTerminal = TERMINAL_RUN_STATUSES.has(input.runDetail.run.status);
269
+ const sideEffectObserved = input.createdAgents.length > 0 || input.createdApprovals.length > 0;
270
+ const requestCompleted = input.testCase.expectedPath === "reject_or_escalate"
271
+ ? {
272
+ value: runSucceeded && input.createdAgents.length === 0 && input.createdApprovals.length === 0 ? "pass" : "fail",
273
+ comment: runSucceeded && input.createdAgents.length === 0 && input.createdApprovals.length === 0
274
+ ? "Run completed without creating agent or approval, matching the reject/escalate path."
275
+ : "Reject/escalate case still created agent/approval or failed the run.",
276
+ }
277
+ : {
278
+ value: sideEffectObserved ? "pass" : "fail",
279
+ comment: sideEffectObserved
280
+ ? runTerminal
281
+ ? "Run reached a create-agent side effect."
282
+ : `Run produced a create-agent side effect before reaching terminal status (${input.runDetail.run.status}).`
283
+ : "Run did not reach a create-agent side effect.",
284
+ };
285
+ let pathCorrect;
286
+ switch (input.testCase.expectedPath) {
287
+ case "approval_required":
288
+ pathCorrect = input.createdApprovals.length > 0 && primaryAgent?.agent.status === "pending_approval"
289
+ ? {
290
+ value: "pass",
291
+ comment: "Observed approval-required path with a pending approval agent.",
292
+ }
293
+ : {
294
+ value: "fail",
295
+ comment: "Expected approval-required path but did not observe a pending-approval agent plus approval.",
296
+ };
297
+ break;
298
+ case "direct_create":
299
+ pathCorrect = input.createdApprovals.length === 0 && primaryAgent != null && primaryAgent.agent.status !== "pending_approval"
300
+ ? {
301
+ value: "pass",
302
+ comment: "Observed direct create path without approval.",
303
+ }
304
+ : {
305
+ value: "fail",
306
+ comment: "Expected direct create path but observed approval or missing created agent.",
307
+ };
308
+ break;
309
+ case "reject_or_escalate":
310
+ pathCorrect = input.createdApprovals.length === 0 && input.createdAgents.length === 0
311
+ ? {
312
+ value: "pass",
313
+ comment: "Observed no created agent or approval for reject/escalate path.",
314
+ }
315
+ : {
316
+ value: "fail",
317
+ comment: "Reject/escalate path still created an agent or approval.",
318
+ };
319
+ break;
320
+ }
321
+ const payloadValid = primaryAgent
322
+ ? {
323
+ value: (!input.testCase.expectedAgentShape.name || primaryAgent.agent.name === input.testCase.expectedAgentShape.name)
324
+ && (!input.testCase.expectedAgentShape.role || primaryAgent.agent.role === input.testCase.expectedAgentShape.role)
325
+ && (input.testCase.expectedAgentShape.title === undefined || (primaryAgent.agent.title ?? null) === input.testCase.expectedAgentShape.title)
326
+ ? "pass"
327
+ : "fail",
328
+ comment: (!input.testCase.expectedAgentShape.name || primaryAgent.agent.name === input.testCase.expectedAgentShape.name)
329
+ && (!input.testCase.expectedAgentShape.role || primaryAgent.agent.role === input.testCase.expectedAgentShape.role)
330
+ && (input.testCase.expectedAgentShape.title === undefined || (primaryAgent.agent.title ?? null) === input.testCase.expectedAgentShape.title)
331
+ ? "Created agent payload matched expected identity fields."
332
+ : "Created agent payload does not match expected name/role/title fields.",
333
+ }
334
+ : input.testCase.expectedPath === "reject_or_escalate"
335
+ ? {
336
+ value: "not_applicable",
337
+ comment: "Reject/escalate cases do not create an agent payload.",
338
+ }
339
+ : {
340
+ value: "fail",
341
+ comment: "Expected a created agent payload but none was observed.",
342
+ };
343
+ const reportsToCheck = primaryAgent
344
+ ? evaluateFieldEquality(primaryAgent.agent.reportsTo, expectedReportsTo)
345
+ : {
346
+ value: input.testCase.expectedPath === "reject_or_escalate" ? "not_applicable" : "fail",
347
+ comment: input.testCase.expectedPath === "reject_or_escalate"
348
+ ? "Reject/escalate cases do not create an agent."
349
+ : "No created agent available for reportsTo validation.",
350
+ };
351
+ const runtimeCheck = primaryAgent
352
+ ? evaluateFieldEquality(primaryAgent.agent.agentRuntimeType, input.testCase.expectedAgentShape.agentRuntimeType)
353
+ : {
354
+ value: input.testCase.expectedPath === "reject_or_escalate" ? "not_applicable" : "fail",
355
+ comment: input.testCase.expectedPath === "reject_or_escalate"
356
+ ? "Reject/escalate cases do not create an agent."
357
+ : "No created agent available for runtime validation.",
358
+ };
359
+ const skillsCheck = primaryAgent
360
+ ? evaluateDesiredSkills(diffSkillKeys(primaryAgent.skills), input.testCase.expectedAgentShape.desiredSkills)
361
+ : {
362
+ value: input.testCase.expectedPath === "reject_or_escalate" ? "not_applicable" : "fail",
363
+ comment: input.testCase.expectedPath === "reject_or_escalate"
364
+ ? "Reject/escalate cases do not create an agent."
365
+ : "No created agent available for skill validation.",
366
+ };
367
+ const sourceIssueLinkCheck = evaluateSourceIssueLink(input, primaryApproval);
368
+ const noFilesystemFallback = filesystemFallbackMatches.length === 0
369
+ ? {
370
+ value: "pass",
371
+ comment: "No filesystem fallback signal was detected in transcript or logs.",
372
+ }
373
+ : {
374
+ value: "fail",
375
+ comment: `Detected filesystem fallback signals: ${filesystemFallbackMatches.join(", ")}`,
376
+ metadata: { matches: filesystemFallbackMatches },
377
+ };
378
+ const overallCorrectness = aggregateOverallCheck([
379
+ requestCompleted,
380
+ pathCorrect,
381
+ payloadValid,
382
+ reportsToCheck,
383
+ runtimeCheck,
384
+ skillsCheck,
385
+ sourceIssueLinkCheck,
386
+ noFilesystemFallback,
387
+ ]);
388
+ const reviewReasons = [];
389
+ if (overallCorrectness.value !== "pass") {
390
+ reviewReasons.push(`deterministic:${overallCorrectness.value}`);
391
+ }
392
+ if (!runTerminal) {
393
+ reviewReasons.push(`run_incomplete:${input.runDetail.run.status}`);
394
+ }
395
+ if (input.judge?.status === "completed") {
396
+ if ((input.judge.configQuality ?? 0) < 4)
397
+ reviewReasons.push("judge:config_quality_low");
398
+ if ((input.judge.reasoningQuality ?? 0) < 4)
399
+ reviewReasons.push("judge:reasoning_quality_low");
400
+ if ((input.judge.governanceJudgmentQuality ?? 0) < 4)
401
+ reviewReasons.push("judge:governance_quality_low");
402
+ }
403
+ else if (input.judge?.status === "failed") {
404
+ reviewReasons.push("judge:failed");
405
+ }
406
+ const finalClassification = overallCorrectness.value === "fail"
407
+ ? "fail"
408
+ : reviewReasons.length > 0 || overallCorrectness.value === "uncertain"
409
+ ? "needs_review"
410
+ : "pass";
411
+ return {
412
+ evaluationVersion: CREATE_AGENT_EVALUATION_VERSION,
413
+ benchmarkMetadata: input.benchmarkMetadata,
414
+ issueId: input.issueId,
415
+ runId: input.runDetail.run.id,
416
+ runStatus: input.runDetail.run.status,
417
+ checks: {
418
+ create_agent_request_completed: requestCompleted,
419
+ create_agent_path_correct: pathCorrect,
420
+ create_agent_payload_valid: payloadValid,
421
+ create_agent_reports_to_valid: reportsToCheck,
422
+ create_agent_runtime_valid: runtimeCheck,
423
+ create_agent_skills_valid: skillsCheck,
424
+ create_agent_source_issue_linked: sourceIssueLinkCheck,
425
+ create_agent_no_filesystem_fallback: noFilesystemFallback,
426
+ create_agent_overall_correctness: overallCorrectness,
427
+ },
428
+ judge: input.judge,
429
+ finalClassification,
430
+ reviewerStatus: reviewReasons.length > 0 ? "pending" : "not_required",
431
+ shouldQueueForReview: reviewReasons.length > 0,
432
+ reviewReasons,
433
+ finalOutputSummary,
434
+ filesystemFallbackMatches,
435
+ createdAgents: input.createdAgents,
436
+ createdApprovals: input.createdApprovals,
437
+ };
438
+ }
439
+ export function createAgentEvalCheckToScoreValue(check) {
440
+ switch (check.value) {
441
+ case "pass":
442
+ return true;
443
+ case "fail":
444
+ return false;
445
+ case "uncertain":
446
+ return "uncertain";
447
+ case "not_applicable":
448
+ return "not_applicable";
449
+ }
450
+ }
451
+ //# sourceMappingURL=create-agent-benchmark.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"create-agent-benchmark.js","sourceRoot":"","sources":["../src/create-agent-benchmark.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,qBAAqB,EAAE,6BAA6B,EAAE,MAAM,YAAY,CAAC;AAGlF,MAAM,CAAC,MAAM,6BAA6B,GAAG,+BAA+B,CAAC;AAC7E,MAAM,CAAC,MAAM,+BAA+B,GAAG,sBAAsB,CAAC;AACtE,MAAM,CAAC,MAAM,gCAAgC,GAAG,6BAA6B,CAAC;AAyG9E,MAAM,iBAAiB,GAA8C;IACnE,EAAE,OAAO,EAAE,wBAAwB,EAAE,KAAK,EAAE,oBAAoB,EAAE;IAClE,EAAE,OAAO,EAAE,oEAAoE,EAAE,KAAK,EAAE,0BAA0B,EAAE;IACpH,EAAE,OAAO,EAAE,2DAA2D,EAAE,KAAK,EAAE,gCAAgC,EAAE;IACjH,EAAE,OAAO,EAAE,4EAA4E,EAAE,KAAK,EAAE,4BAA4B,EAAE;CAC/H,CAAC;AACF,MAAM,qBAAqB,GAAG,IAAI,GAAG,CAAC,CAAC,WAAW,EAAE,QAAQ,EAAE,WAAW,EAAE,WAAW,CAAC,CAAC,CAAC;AAEzF,SAAS,cAAc,CAAC,KAAc;IACpC,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;AACpF,CAAC;AAED,SAAS,aAAa,CAAC,KAAc;IACnC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACrC,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAkB,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;AACnG,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,GAAY;IAC/C,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QACpC,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;IACpE,CAAC;IAED,MAAM,KAAK,GAAG,GAA8B,CAAC;IAC7C,MAAM,EAAE,GAAG,cAAc,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACpC,MAAM,MAAM,GAAG,cAAc,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAC5C,MAAM,YAAY,GAAG,cAAc,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;IAExD,IAAI,CAAC,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;IACnF,IAAI,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,QAAQ,EAAE,iCAAiC,CAAC,CAAC;IAC1E,IAAI,YAAY,KAAK,eAAe,IAAI,YAAY,KAAK,mBAAmB,IAAI,YAAY,KAAK,oBAAoB,EAAE,CAAC;QACtH,MAAM,IAAI,KAAK,CAAC,QAAQ,EAAE,8BAA8B,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;IACxF,CAAC;IAED,MAAM,kBAAkB,GAAG,CAAC,KAAK,CAAC,kBAAkB,IAAI,EAAE,CAA4B,CAAC;IACvF,IAAI,CAAC,kBAAkB,IAAI,OAAO,kBAAkB,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,kBAAkB,CAAC,EAAE,CAAC;QACvG,MAAM,IAAI,KAAK,CAAC,QAAQ,EAAE,kCAAkC,CAAC,CAAC;IAChE,CAAC;IAED,MAAM,QAAQ,GAAG,CAAC,KAAK,CAAC,QAAQ,IAAI,EAAE,CAA4B,CAAC;IACnE,IAAI,QAAQ,IAAI,CAAC,OAAO,QAAQ,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;QAC1E,MAAM,IAAI,KAAK,CAAC,QAAQ,EAAE,wBAAwB,CAAC,CAAC;IACtD,CAAC;IAED,OAAO;QACL,EAAE;QACF,MAAM;QACN,YAAY;QACZ,kBAAkB,EAAE;YAClB,IAAI,EAAE,cAAc,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,SAAS;YAC1D,IAAI,EAAE,cAAc,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,SAAS;YAC1D,KAAK,EAAE,kBAAkB,CAAC,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,cAAc,CAAC,kBAAkB,CAAC,KAAK,CAAC,IAAI,SAAS;YACvG,SAAS,EAAE,kBAAkB,CAAC,SAAS,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,cAAc,CAAC,kBAAkB,CAAC,SAAS,CAAC,IAAI,SAAS;YACnH,gBAAgB,EAAE,kBAAkB,CAAC,gBAAgB,KAAK,IAAI;gBAC5D,CAAC,CAAC,IAAI;gBACN,CAAC,CAAC,cAAc,CAAC,kBAAkB,CAAC,gBAAgB,CAAC,IAAI,SAAS;YACpE,gBAAgB,EAAE,cAAc,CAAC,kBAAkB,CAAC,gBAAgB,CAAC,IAAI,SAAS;YAClF,aAAa,EAAE,aAAa,CAAC,kBAAkB,CAAC,aAAa,CAAC;YAC9D,mBAAmB,EAAE,OAAO,kBAAkB,CAAC,mBAAmB,KAAK,SAAS;gBAC9E,CAAC,CAAC,kBAAkB,CAAC,mBAAmB;gBACxC,CAAC,CAAC,SAAS;SACd;QACD,QAAQ,EAAE;YACR,gBAAgB,EAAE,OAAO,QAAQ,CAAC,gBAAgB,KAAK,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI;YACnG,mBAAmB,EAAE,aAAa,CAAC,QAAQ,CAAC,mBAAmB,CAAC;SACjE;QACD,UAAU,EAAE,aAAa,CAAC,KAAK,CAAC,UAAU,CAAC;KAC5C,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,iCAAiC,CAAC,KAGjD;IACC,OAAO;QACL,QAAQ,EAAE,cAAc;QACxB,SAAS,EAAE,IAAI;QACf,eAAe,EAAE,KAAK,CAAC,QAAQ,CAAC,EAAE;QAClC,YAAY,EAAE,KAAK,CAAC,QAAQ,CAAC,YAAY;QACzC,aAAa,EAAE,KAAK,CAAC,QAAQ,CAAC,kBAAkB,CAAC,IAAI,IAAI,IAAI;QAC7D,oBAAoB,EAAE,KAAK,CAAC,QAAQ,CAAC,kBAAkB,CAAC,gBAAgB,IAAI,IAAI;QAChF,iBAAiB,EAAE,+BAA+B;QAClD,YAAY,EAAE,KAAK,CAAC,YAAY,IAAI,IAAI;KACzC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,6BAA6B,CAAC,QAAsC;IAClF,OAAO;QACL,uBAAuB;QACvB,gBAAgB;QAChB,kBAAkB,QAAQ,CAAC,eAAe,EAAE;KAC7C,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,yCAAyC,CAAC,QAAsC;IAC9F,OAAO,QAAQ,6BAA6B,IAAI,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC;AACjF,CAAC;AAED,MAAM,UAAU,kCAAkC,CAAC,WAAmB,EAAE,QAAsC;IAC5G,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC;IAChC,OAAO,GAAG,IAAI,OAAO,yCAAyC,CAAC,QAAQ,CAAC,IAAI,CAAC;AAC/E,CAAC;AAED,MAAM,UAAU,mCAAmC,CAAC,IAA+B;IACjF,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAC;IACvB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,WAAW,6BAA6B,4BAA4B,EAAE,GAAG,CAAC,CAAC,CAAC;IAChH,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IAC7B,IAAI,CAAC;QACH,OAAO,kCAAkC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAClE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,UAAU,kCAAkC,CAAC,GAAY;IAC7D,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC;QAAE,OAAO,IAAI,CAAC;IACvE,MAAM,MAAM,GAAG,GAA8B,CAAC;IAC9C,IACE,MAAM,CAAC,QAAQ,KAAK,cAAc;WAC/B,MAAM,CAAC,SAAS,KAAK,IAAI;WACzB,cAAc,CAAC,MAAM,CAAC,eAAe,CAAC,IAAI,IAAI,EACjD,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,YAAY,GAAG,cAAc,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;IACzD,IAAI,YAAY,KAAK,eAAe,IAAI,YAAY,KAAK,mBAAmB,IAAI,YAAY,KAAK,oBAAoB,EAAE,CAAC;QACtH,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO;QACL,QAAQ,EAAE,cAAc;QACxB,SAAS,EAAE,IAAI;QACf,eAAe,EAAE,cAAc,CAAC,MAAM,CAAC,eAAe,CAAE;QACxD,YAAY;QACZ,aAAa,EAAE,cAAc,CAAC,MAAM,CAAC,aAAa,CAAC;QACnD,oBAAoB,EAAE,cAAc,CAAC,MAAM,CAAC,oBAAoB,CAAC;QACjE,iBAAiB,EAAE,cAAc,CAAC,MAAM,CAAC,iBAAiB,CAAC,IAAI,+BAA+B;QAC9F,YAAY,EAAE,cAAc,CAAC,MAAM,CAAC,YAAY,CAAC;KAClD,CAAC;AACJ,CAAC;AAED,SAAS,wBAAwB,CAAC,QAAyB,EAAE,cAAkD,EAAE;IAC/G,IAAI,QAAQ,CAAC,kBAAkB,CAAC,SAAS,KAAK,SAAS;QAAE,OAAO,QAAQ,CAAC,kBAAkB,CAAC,SAAS,CAAC;IACtG,IAAI,QAAQ,CAAC,kBAAkB,CAAC,gBAAgB,KAAK,SAAS,EAAE,CAAC;QAC/D,OAAO,WAAW,CAAC,QAAQ,CAAC,kBAAkB,CAAC,gBAAgB,IAAI,EAAE,CAAC,IAAI,IAAI,CAAC;IACjF,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,qBAAqB,CAAC,MAAiC,EAAE,QAAmC;IACnG,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC3B,OAAO,EAAE,KAAK,EAAE,gBAAyB,EAAE,OAAO,EAAE,0CAA0C,EAAE,CAAC;IACnG,CAAC;IACD,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,KAAK,QAAQ,EAAE,CAAC;QAClC,OAAO,EAAE,KAAK,EAAE,MAAe,EAAE,OAAO,EAAE,0BAA0B,QAAQ,IAAI,MAAM,GAAG,EAAE,CAAC;IAC9F,CAAC;IACD,OAAO;QACL,KAAK,EAAE,MAAe;QACtB,OAAO,EAAE,YAAY,QAAQ,IAAI,MAAM,iBAAiB,MAAM,IAAI,MAAM,GAAG;KAC5E,CAAC;AACJ,CAAC;AAED,SAAS,6BAA6B,CAAC,MAAyB;IAC9D,MAAM,SAAS,GAAG;QAChB,MAAM,CAAC,UAAU,IAAI,EAAE;QACvB,MAAM,CAAC,GAAG,CAAC,aAAa,IAAI,EAAE;QAC9B,MAAM,CAAC,GAAG,CAAC,aAAa,IAAI,EAAE;QAC9B,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,IAAI,EAAE,CAAC;QACpD,GAAG,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,6BAA6B,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;KAC/E,CAAC;IACF,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAClC,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,KAAK,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,iBAAiB,EAAE,CAAC;YACnD,IAAI,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC3B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YACrB,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,CAAC,GAAG,OAAO,CAAC,CAAC;AACtB,CAAC;AAED,SAAS,oBAAoB,CAAC,MAAyB;IACrD,MAAM,KAAK,GAAG,qBAAqB,CAAC,MAAM,CAAC,CAAC;IAC5C,MAAM,aAAa,GAAG,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC;IACnG,IAAI,aAAa,EAAE,OAAO;QAAE,OAAO,aAAa,CAAC,OAAO,CAAC;IACzD,MAAM,SAAS,GAAG,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC1E,OAAO,SAAS,EAAE,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC,KAAK,IAAI,mCAAmC,CAAC;AACvF,CAAC;AAED,SAAS,gBAAgB,CAAC,cAA0C,EAAE,YAAqC;IACzG,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAC7C,IAAI,YAAY,KAAK,mBAAmB,EAAE,CAAC;QACzC,OAAO,cAAc,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,kBAAkB,CAAC,IAAI,cAAc,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;IAC9G,CAAC;IACD,OAAO,cAAc,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;AACnC,CAAC;AAED,SAAS,aAAa,CAAC,QAAmC;IACxD,IAAI,CAAC,QAAQ;QAAE,OAAO,EAAE,CAAC;IACzB,OAAO,CAAC,GAAG,QAAQ,CAAC,aAAa,CAAC,CAAC,IAAI,EAAE,CAAC;AAC5C,CAAC;AAED,SAAS,qBAAqB,CAAC,YAAsB,EAAE,cAAoC;IACzF,IAAI,CAAC,cAAc,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACnD,OAAO;YACL,KAAK,EAAE,gBAAgB;YACvB,OAAO,EAAE,mDAAmD;SAC7D,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,CAAC,GAAG,YAAY,CAAC,CAAC,IAAI,EAAE,CAAC;IACxC,MAAM,QAAQ,GAAG,CAAC,GAAG,cAAc,CAAC,CAAC,IAAI,EAAE,CAAC;IAC5C,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,CAAC;QACxD,OAAO;YACL,KAAK,EAAE,MAAM;YACb,OAAO,EAAE,2BAA2B,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,MAAM,IAAI;YACrE,QAAQ,EAAE,EAAE,cAAc,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,EAAE;SAC7D,CAAC;IACJ,CAAC;IACD,OAAO;QACL,KAAK,EAAE,MAAM;QACb,OAAO,EAAE,oBAAoB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAmB,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI;QACxF,QAAQ,EAAE,EAAE,cAAc,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,EAAE;KAC7D,CAAC;AACJ,CAAC;AAED,SAAS,uBAAuB,CAC9B,KAAwC,EACxC,eAAmD;IAEnD,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,kBAAkB,CAAC,mBAAmB,EAAE,CAAC;QAC3D,OAAO;YACL,KAAK,EAAE,gBAAgB;YACvB,OAAO,EAAE,6CAA6C;SACvD,CAAC;IACJ,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;QACnB,OAAO;YACL,KAAK,EAAE,WAAW;YAClB,OAAO,EAAE,+CAA+C;SACzD,CAAC;IACJ,CAAC;IACD,IAAI,CAAC,eAAe,EAAE,CAAC;QACrB,OAAO;YACL,KAAK,EAAE,WAAW;YAClB,OAAO,EAAE,oEAAoE;SAC9E,CAAC;IACJ,CAAC;IACD,IAAI,eAAe,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;QACrD,OAAO;YACL,KAAK,EAAE,MAAM;YACb,OAAO,EAAE,YAAY,eAAe,CAAC,QAAQ,CAAC,EAAE,0BAA0B,KAAK,CAAC,OAAO,GAAG;SAC3F,CAAC;IACJ,CAAC;IACD,OAAO;QACL,KAAK,EAAE,MAAM;QACb,OAAO,EAAE,YAAY,eAAe,CAAC,QAAQ,CAAC,EAAE,kCAAkC,KAAK,CAAC,OAAO,GAAG;QAClG,QAAQ,EAAE,EAAE,QAAQ,EAAE,eAAe,CAAC,QAAQ,EAAE,eAAe,EAAE,KAAK,CAAC,OAAO,EAAE;KACjF,CAAC;AACJ,CAAC;AAED,SAAS,qBAAqB,CAAC,MAAmC;IAChE,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,MAAM,CAAC,EAAE,CAAC;QACnD,OAAO;YACL,KAAK,EAAE,MAAM;YACb,OAAO,EAAE,sDAAsD;SAChE,CAAC;IACJ,CAAC;IACD,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,WAAW,CAAC,EAAE,CAAC;QACxD,OAAO;YACL,KAAK,EAAE,WAAW;YAClB,OAAO,EAAE,qFAAqF;SAC/F,CAAC;IACJ,CAAC;IACD,OAAO;QACL,KAAK,EAAE,MAAM;QACb,OAAO,EAAE,2DAA2D;KACrE,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,4BAA4B,CAC1C,KAA+F;IAE/F,MAAM,YAAY,GAAG,gBAAgB,CAAC,KAAK,CAAC,aAAa,EAAE,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;IACxF,MAAM,eAAe,GAAG,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;IAC1D,MAAM,iBAAiB,GAAG,wBAAwB,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC;IACtF,MAAM,yBAAyB,GAAG,6BAA6B,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IACjF,MAAM,kBAAkB,GAAG,oBAAoB,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IACjE,MAAM,YAAY,GAAG,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,KAAK,WAAW,CAAC;IAChE,MAAM,WAAW,GAAG,qBAAqB,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAC1E,MAAM,kBAAkB,GAAG,KAAK,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC;IAE/F,MAAM,gBAAgB,GAAyB,KAAK,CAAC,QAAQ,CAAC,YAAY,KAAK,oBAAoB;QACjG,CAAC,CAAC;YACA,KAAK,EAAE,YAAY,IAAI,KAAK,CAAC,aAAa,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,gBAAgB,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;YAChH,OAAO,EAAE,YAAY,IAAI,KAAK,CAAC,aAAa,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,gBAAgB,CAAC,MAAM,KAAK,CAAC;gBAC9F,CAAC,CAAC,sFAAsF;gBACxF,CAAC,CAAC,sEAAsE;SAC3E;QACD,CAAC,CAAC;YACA,KAAK,EAAE,kBAAkB,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;YAC3C,OAAO,EAAE,kBAAkB;gBACzB,CAAC,CAAC,WAAW;oBACX,CAAC,CAAC,yCAAyC;oBAC3C,CAAC,CAAC,4EAA4E,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,IAAI;gBAC9G,CAAC,CAAC,+CAA+C;SACpD,CAAC;IAEJ,IAAI,WAAiC,CAAC;IACtC,QAAQ,KAAK,CAAC,QAAQ,CAAC,YAAY,EAAE,CAAC;QACpC,KAAK,mBAAmB;YACtB,WAAW,GAAG,KAAK,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,IAAI,YAAY,EAAE,KAAK,CAAC,MAAM,KAAK,kBAAkB;gBAClG,CAAC,CAAC;oBACA,KAAK,EAAE,MAAM;oBACb,OAAO,EAAE,gEAAgE;iBAC1E;gBACD,CAAC,CAAC;oBACA,KAAK,EAAE,MAAM;oBACb,OAAO,EAAE,6FAA6F;iBACvG,CAAC;YACJ,MAAM;QACR,KAAK,eAAe;YAClB,WAAW,GAAG,KAAK,CAAC,gBAAgB,CAAC,MAAM,KAAK,CAAC,IAAI,YAAY,IAAI,IAAI,IAAI,YAAY,CAAC,KAAK,CAAC,MAAM,KAAK,kBAAkB;gBAC3H,CAAC,CAAC;oBACA,KAAK,EAAE,MAAM;oBACb,OAAO,EAAE,+CAA+C;iBACzD;gBACD,CAAC,CAAC;oBACA,KAAK,EAAE,MAAM;oBACb,OAAO,EAAE,6EAA6E;iBACvF,CAAC;YACJ,MAAM;QACR,KAAK,oBAAoB;YACvB,WAAW,GAAG,KAAK,CAAC,gBAAgB,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,aAAa,CAAC,MAAM,KAAK,CAAC;gBACnF,CAAC,CAAC;oBACA,KAAK,EAAE,MAAM;oBACb,OAAO,EAAE,iEAAiE;iBAC3E;gBACD,CAAC,CAAC;oBACA,KAAK,EAAE,MAAM;oBACb,OAAO,EAAE,0DAA0D;iBACpE,CAAC;YACJ,MAAM;IACV,CAAC;IAED,MAAM,YAAY,GAAyB,YAAY;QACrD,CAAC,CAAC;YACA,KAAK,EACH,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,kBAAkB,CAAC,IAAI,IAAI,YAAY,CAAC,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,QAAQ,CAAC,kBAAkB,CAAC,IAAI,CAAC;mBAC1G,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,kBAAkB,CAAC,IAAI,IAAI,YAAY,CAAC,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,QAAQ,CAAC,kBAAkB,CAAC,IAAI,CAAC;mBAC/G,CAAC,KAAK,CAAC,QAAQ,CAAC,kBAAkB,CAAC,KAAK,KAAK,SAAS,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,KAAK,CAAC,QAAQ,CAAC,kBAAkB,CAAC,KAAK,CAAC;gBAC5I,CAAC,CAAC,MAAM;gBACR,CAAC,CAAC,MAAM;YACZ,OAAO,EACL,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,kBAAkB,CAAC,IAAI,IAAI,YAAY,CAAC,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,QAAQ,CAAC,kBAAkB,CAAC,IAAI,CAAC;mBAC1G,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,kBAAkB,CAAC,IAAI,IAAI,YAAY,CAAC,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,QAAQ,CAAC,kBAAkB,CAAC,IAAI,CAAC;mBAC/G,CAAC,KAAK,CAAC,QAAQ,CAAC,kBAAkB,CAAC,KAAK,KAAK,SAAS,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,KAAK,CAAC,QAAQ,CAAC,kBAAkB,CAAC,KAAK,CAAC;gBAC5I,CAAC,CAAC,yDAAyD;gBAC3D,CAAC,CAAC,uEAAuE;SAC9E;QACD,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,YAAY,KAAK,oBAAoB;YACpD,CAAC,CAAC;gBACA,KAAK,EAAE,gBAAgB;gBACvB,OAAO,EAAE,uDAAuD;aACjE;YACD,CAAC,CAAC;gBACA,KAAK,EAAE,MAAM;gBACb,OAAO,EAAE,yDAAyD;aACnE,CAAC;IAEN,MAAM,cAAc,GAAyB,YAAY;QACvD,CAAC,CAAC,qBAAqB,CAAC,YAAY,CAAC,KAAK,CAAC,SAAS,EAAE,iBAAiB,CAAC;QACxE,CAAC,CAAC;YACA,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,YAAY,KAAK,oBAAoB,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,MAAM;YACvF,OAAO,EAAE,KAAK,CAAC,QAAQ,CAAC,YAAY,KAAK,oBAAoB;gBAC3D,CAAC,CAAC,+CAA+C;gBACjD,CAAC,CAAC,sDAAsD;SAC3D,CAAC;IAEJ,MAAM,YAAY,GAAyB,YAAY;QACrD,CAAC,CAAC,qBAAqB,CAAC,YAAY,CAAC,KAAK,CAAC,gBAAgB,EAAE,KAAK,CAAC,QAAQ,CAAC,kBAAkB,CAAC,gBAAgB,CAAC;QAChH,CAAC,CAAC;YACA,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,YAAY,KAAK,oBAAoB,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,MAAM;YACvF,OAAO,EAAE,KAAK,CAAC,QAAQ,CAAC,YAAY,KAAK,oBAAoB;gBAC3D,CAAC,CAAC,+CAA+C;gBACjD,CAAC,CAAC,oDAAoD;SACzD,CAAC;IAEJ,MAAM,WAAW,GAAyB,YAAY;QACpD,CAAC,CAAC,qBAAqB,CAAC,aAAa,CAAC,YAAY,CAAC,MAAM,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,kBAAkB,CAAC,aAAa,CAAC;QAC5G,CAAC,CAAC;YACA,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,YAAY,KAAK,oBAAoB,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,MAAM;YACvF,OAAO,EAAE,KAAK,CAAC,QAAQ,CAAC,YAAY,KAAK,oBAAoB;gBAC3D,CAAC,CAAC,+CAA+C;gBACjD,CAAC,CAAC,kDAAkD;SACvD,CAAC;IAEJ,MAAM,oBAAoB,GAAG,uBAAuB,CAAC,KAAK,EAAE,eAAe,CAAC,CAAC;IAC7E,MAAM,oBAAoB,GAAyB,yBAAyB,CAAC,MAAM,KAAK,CAAC;QACvF,CAAC,CAAC;YACA,KAAK,EAAE,MAAM;YACb,OAAO,EAAE,mEAAmE;SAC7E;QACD,CAAC,CAAC;YACA,KAAK,EAAE,MAAM;YACb,OAAO,EAAE,yCAAyC,yBAAyB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;YACxF,QAAQ,EAAE,EAAE,OAAO,EAAE,yBAAyB,EAAE;SACjD,CAAC;IAEJ,MAAM,kBAAkB,GAAG,qBAAqB,CAAC;QAC/C,gBAAgB;QAChB,WAAW;QACX,YAAY;QACZ,cAAc;QACd,YAAY;QACZ,WAAW;QACX,oBAAoB;QACpB,oBAAoB;KACrB,CAAC,CAAC;IAEH,MAAM,aAAa,GAAa,EAAE,CAAC;IACnC,IAAI,kBAAkB,CAAC,KAAK,KAAK,MAAM,EAAE,CAAC;QACxC,aAAa,CAAC,IAAI,CAAC,iBAAiB,kBAAkB,CAAC,KAAK,EAAE,CAAC,CAAC;IAClE,CAAC;IACD,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,aAAa,CAAC,IAAI,CAAC,kBAAkB,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;IACrE,CAAC;IACD,IAAI,KAAK,CAAC,KAAK,EAAE,MAAM,KAAK,WAAW,EAAE,CAAC;QACxC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,aAAa,IAAI,CAAC,CAAC,GAAG,CAAC;YAAE,aAAa,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QACzF,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,gBAAgB,IAAI,CAAC,CAAC,GAAG,CAAC;YAAE,aAAa,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;QAC/F,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,yBAAyB,IAAI,CAAC,CAAC,GAAG,CAAC;YAAE,aAAa,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;IAC3G,CAAC;SAAM,IAAI,KAAK,CAAC,KAAK,EAAE,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC5C,aAAa,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IACrC,CAAC;IAED,MAAM,mBAAmB,GACvB,kBAAkB,CAAC,KAAK,KAAK,MAAM;QACjC,CAAC,CAAC,MAAM;QACR,CAAC,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,IAAI,kBAAkB,CAAC,KAAK,KAAK,WAAW;YACpE,CAAC,CAAC,cAAc;YAChB,CAAC,CAAC,MAAM,CAAC;IAEf,OAAO;QACL,iBAAiB,EAAE,+BAA+B;QAClD,iBAAiB,EAAE,KAAK,CAAC,iBAAiB;QAC1C,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,KAAK,EAAE,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE;QAC7B,SAAS,EAAE,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM;QACrC,MAAM,EAAE;YACN,8BAA8B,EAAE,gBAAgB;YAChD,yBAAyB,EAAE,WAAW;YACtC,0BAA0B,EAAE,YAAY;YACxC,6BAA6B,EAAE,cAAc;YAC7C,0BAA0B,EAAE,YAAY;YACxC,yBAAyB,EAAE,WAAW;YACtC,gCAAgC,EAAE,oBAAoB;YACtD,mCAAmC,EAAE,oBAAoB;YACzD,gCAAgC,EAAE,kBAAkB;SACrD;QACD,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,mBAAmB;QACnB,cAAc,EAAE,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,cAAc;QACrE,oBAAoB,EAAE,aAAa,CAAC,MAAM,GAAG,CAAC;QAC9C,aAAa;QACb,kBAAkB;QAClB,yBAAyB;QACzB,aAAa,EAAE,KAAK,CAAC,aAAa;QAClC,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;KACzC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,gCAAgC,CAAC,KAA2B;IAC1E,QAAQ,KAAK,CAAC,KAAK,EAAE,CAAC;QACpB,KAAK,MAAM;YACT,OAAO,IAAI,CAAC;QACd,KAAK,MAAM;YACT,OAAO,KAAK,CAAC;QACf,KAAK,WAAW;YACd,OAAO,WAAW,CAAC;QACrB,KAAK,gBAAgB;YACnB,OAAO,gBAAgB,CAAC;IAC5B,CAAC;AACH,CAAC"}
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=create-agent-benchmark.test.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"create-agent-benchmark.test.d.ts","sourceRoot":"","sources":["../src/create-agent-benchmark.test.ts"],"names":[],"mappings":""}