@codemieai/code 0.15.5 → 0.15.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (118) hide show
  1. package/dist/agents/core/session/SessionStore.d.ts +6 -0
  2. package/dist/agents/core/session/SessionStore.d.ts.map +1 -1
  3. package/dist/agents/core/session/SessionStore.js +38 -0
  4. package/dist/agents/core/session/SessionStore.js.map +1 -1
  5. package/dist/agents/core/session/types.d.ts +2 -0
  6. package/dist/agents/core/session/types.d.ts.map +1 -1
  7. package/dist/agents/plugins/claude/claude.models.d.ts +18 -2
  8. package/dist/agents/plugins/claude/claude.models.d.ts.map +1 -1
  9. package/dist/agents/plugins/claude/claude.models.js +109 -61
  10. package/dist/agents/plugins/claude/claude.models.js.map +1 -1
  11. package/dist/agents/plugins/claude/claude.plugin.d.ts.map +1 -1
  12. package/dist/agents/plugins/claude/claude.plugin.js +25 -4
  13. package/dist/agents/plugins/claude/claude.plugin.js.map +1 -1
  14. package/dist/agents/plugins/claude/plugin/.claude-plugin/plugin.json +1 -1
  15. package/dist/agents/plugins/claude/plugin/skills/codemie-assistant-builder/SKILL.md +177 -0
  16. package/dist/agents/plugins/claude/plugin/skills/codemie-assistant-builder/agents/grader.md +38 -0
  17. package/dist/agents/plugins/claude/plugin/skills/codemie-assistant-builder/agents/improver.md +40 -0
  18. package/dist/agents/plugins/claude/plugin/skills/codemie-sdk/SKILL.md +10 -1
  19. package/dist/agents/plugins/claude/plugin/skills/codemie-sdk/examples/assistants.md +23 -0
  20. package/dist/agents/plugins/claude/plugin/statusline.bundle.mjs +57 -21
  21. package/dist/agents/plugins/claude/session/processors/claude.conversations-processor.d.ts +13 -0
  22. package/dist/agents/plugins/claude/session/processors/claude.conversations-processor.d.ts.map +1 -1
  23. package/dist/agents/plugins/claude/session/processors/claude.conversations-processor.js +27 -1
  24. package/dist/agents/plugins/claude/session/processors/claude.conversations-processor.js.map +1 -1
  25. package/dist/agents/plugins/claude/session/processors/claude.metrics-processor.d.ts +6 -0
  26. package/dist/agents/plugins/claude/session/processors/claude.metrics-processor.d.ts.map +1 -1
  27. package/dist/agents/plugins/claude/session/processors/claude.metrics-processor.js +21 -0
  28. package/dist/agents/plugins/claude/session/processors/claude.metrics-processor.js.map +1 -1
  29. package/dist/agents/plugins/claude/session/processors/messages-before.d.ts +11 -0
  30. package/dist/agents/plugins/claude/session/processors/messages-before.d.ts.map +1 -0
  31. package/dist/agents/plugins/claude/session/processors/messages-before.js +15 -0
  32. package/dist/agents/plugins/claude/session/processors/messages-before.js.map +1 -0
  33. package/dist/agents/plugins/claude/statusline-installer.d.ts +9 -0
  34. package/dist/agents/plugins/claude/statusline-installer.d.ts.map +1 -1
  35. package/dist/agents/plugins/claude/statusline-installer.js +42 -27
  36. package/dist/agents/plugins/claude/statusline-installer.js.map +1 -1
  37. package/dist/cli/commands/sdk/assistant-builder/checks.d.ts +4 -0
  38. package/dist/cli/commands/sdk/assistant-builder/checks.d.ts.map +1 -0
  39. package/dist/cli/commands/sdk/assistant-builder/checks.js +32 -0
  40. package/dist/cli/commands/sdk/assistant-builder/checks.js.map +1 -0
  41. package/dist/cli/commands/sdk/assistant-builder/commands.d.ts +6 -0
  42. package/dist/cli/commands/sdk/assistant-builder/commands.d.ts.map +1 -0
  43. package/dist/cli/commands/sdk/assistant-builder/commands.js +278 -0
  44. package/dist/cli/commands/sdk/assistant-builder/commands.js.map +1 -0
  45. package/dist/cli/commands/sdk/assistant-builder/conversations.d.ts +26 -0
  46. package/dist/cli/commands/sdk/assistant-builder/conversations.d.ts.map +1 -0
  47. package/dist/cli/commands/sdk/assistant-builder/conversations.js +45 -0
  48. package/dist/cli/commands/sdk/assistant-builder/conversations.js.map +1 -0
  49. package/dist/cli/commands/sdk/assistant-builder/normalize.d.ts +11 -0
  50. package/dist/cli/commands/sdk/assistant-builder/normalize.d.ts.map +1 -0
  51. package/dist/cli/commands/sdk/assistant-builder/normalize.js +43 -0
  52. package/dist/cli/commands/sdk/assistant-builder/normalize.js.map +1 -0
  53. package/dist/cli/commands/sdk/assistant-builder/output.d.ts +10 -0
  54. package/dist/cli/commands/sdk/assistant-builder/output.d.ts.map +1 -0
  55. package/dist/cli/commands/sdk/assistant-builder/output.js +29 -0
  56. package/dist/cli/commands/sdk/assistant-builder/output.js.map +1 -0
  57. package/dist/cli/commands/sdk/assistant-builder/report/compare.d.ts +23 -0
  58. package/dist/cli/commands/sdk/assistant-builder/report/compare.d.ts.map +1 -0
  59. package/dist/cli/commands/sdk/assistant-builder/report/compare.js +66 -0
  60. package/dist/cli/commands/sdk/assistant-builder/report/compare.js.map +1 -0
  61. package/dist/cli/commands/sdk/assistant-builder/report/html.d.ts +9 -0
  62. package/dist/cli/commands/sdk/assistant-builder/report/html.d.ts.map +1 -0
  63. package/dist/cli/commands/sdk/assistant-builder/report/html.js +87 -0
  64. package/dist/cli/commands/sdk/assistant-builder/report/html.js.map +1 -0
  65. package/dist/cli/commands/sdk/assistant-builder/report/summary.d.ts +31 -0
  66. package/dist/cli/commands/sdk/assistant-builder/report/summary.d.ts.map +1 -0
  67. package/dist/cli/commands/sdk/assistant-builder/report/summary.js +56 -0
  68. package/dist/cli/commands/sdk/assistant-builder/report/summary.js.map +1 -0
  69. package/dist/cli/commands/sdk/assistant-builder/report/workspace.d.ts +19 -0
  70. package/dist/cli/commands/sdk/assistant-builder/report/workspace.d.ts.map +1 -0
  71. package/dist/cli/commands/sdk/assistant-builder/report/workspace.js +112 -0
  72. package/dist/cli/commands/sdk/assistant-builder/report/workspace.js.map +1 -0
  73. package/dist/cli/commands/sdk/assistant-builder/runner.d.ts +32 -0
  74. package/dist/cli/commands/sdk/assistant-builder/runner.d.ts.map +1 -0
  75. package/dist/cli/commands/sdk/assistant-builder/runner.js +109 -0
  76. package/dist/cli/commands/sdk/assistant-builder/runner.js.map +1 -0
  77. package/dist/cli/commands/sdk/assistant-builder/scenarios.d.ts +5 -0
  78. package/dist/cli/commands/sdk/assistant-builder/scenarios.d.ts.map +1 -0
  79. package/dist/cli/commands/sdk/assistant-builder/scenarios.js +66 -0
  80. package/dist/cli/commands/sdk/assistant-builder/scenarios.js.map +1 -0
  81. package/dist/cli/commands/sdk/assistant-builder/types.d.ts +70 -0
  82. package/dist/cli/commands/sdk/assistant-builder/types.d.ts.map +1 -0
  83. package/dist/cli/commands/sdk/assistant-builder/types.js +2 -0
  84. package/dist/cli/commands/sdk/assistant-builder/types.js.map +1 -0
  85. package/dist/cli/commands/sdk/assistants.d.ts.map +1 -1
  86. package/dist/cli/commands/sdk/assistants.js +2 -0
  87. package/dist/cli/commands/sdk/assistants.js.map +1 -1
  88. package/dist/cli/commands/sdk/services/assistants.d.ts +20 -1
  89. package/dist/cli/commands/sdk/services/assistants.d.ts.map +1 -1
  90. package/dist/cli/commands/sdk/services/assistants.js +80 -2
  91. package/dist/cli/commands/sdk/services/assistants.js.map +1 -1
  92. package/dist/cli/commands/sdk/utils/cli-utils.js +1 -1
  93. package/dist/cli/commands/sdk/utils/cli-utils.js.map +1 -1
  94. package/dist/providers/plugins/sso/sso.http-client.d.ts +5 -0
  95. package/dist/providers/plugins/sso/sso.http-client.d.ts.map +1 -1
  96. package/dist/providers/plugins/sso/sso.http-client.js.map +1 -1
  97. package/dist/telemetry/clients/claude-desktop/ClaudeDesktopTelemetryAdapter.d.ts +3 -0
  98. package/dist/telemetry/clients/claude-desktop/ClaudeDesktopTelemetryAdapter.d.ts.map +1 -1
  99. package/dist/telemetry/clients/claude-desktop/ClaudeDesktopTelemetryAdapter.js +43 -2
  100. package/dist/telemetry/clients/claude-desktop/ClaudeDesktopTelemetryAdapter.js.map +1 -1
  101. package/dist/telemetry/clients/claude-desktop/claude-desktop.discovery.d.ts.map +1 -1
  102. package/dist/telemetry/clients/claude-desktop/claude-desktop.discovery.js +12 -10
  103. package/dist/telemetry/clients/claude-desktop/claude-desktop.discovery.js.map +1 -1
  104. package/dist/telemetry/runtime/DesktopRepositoryResolver.d.ts.map +1 -1
  105. package/dist/telemetry/runtime/DesktopRepositoryResolver.js +33 -6
  106. package/dist/telemetry/runtime/DesktopRepositoryResolver.js.map +1 -1
  107. package/dist/telemetry/runtime/DesktopTelemetryRuntime.d.ts +2 -0
  108. package/dist/telemetry/runtime/DesktopTelemetryRuntime.d.ts.map +1 -1
  109. package/dist/telemetry/runtime/DesktopTelemetryRuntime.js +61 -10
  110. package/dist/telemetry/runtime/DesktopTelemetryRuntime.js.map +1 -1
  111. package/dist/telemetry/runtime/checkpoints.d.ts +6 -0
  112. package/dist/telemetry/runtime/checkpoints.d.ts.map +1 -1
  113. package/dist/telemetry/runtime/checkpoints.js.map +1 -1
  114. package/dist/telemetry/runtime/types.d.ts +5 -0
  115. package/dist/telemetry/runtime/types.d.ts.map +1 -1
  116. package/dist/utils/pricing.json +9 -1
  117. package/package.json +2 -2
  118. package/scripts/bundle-statusline.mjs +14 -1
@@ -0,0 +1,87 @@
1
+ import { classifyIteration } from "./compare.js";
2
+ export function escapeHtml(value) {
3
+ return value
4
+ .replace(/&/g, "&")
5
+ .replace(/</g, "&lt;")
6
+ .replace(/>/g, "&gt;")
7
+ .replace(/"/g, "&quot;")
8
+ .replace(/'/g, "&#39;");
9
+ }
10
+ const STYLE = `
11
+ :root{--bg:#1A1A1A;--card:#151515;--elev:#212224;--border:#333436;--text:#FFFFFF;--muted:#BBBBBB;
12
+ --ok:#259F4C;--ok-bg:#1B271F;--err:#F9303C;--err-bg:#262121;--warn:#E8A33D;
13
+ --font:'Inter',-apple-system,BlinkMacSystemFont,'Segoe UI',Arial,sans-serif;--mono:'JetBrains Mono','Fira Code',monospace}
14
+ *{box-sizing:border-box}body{margin:0;background:var(--bg);color:var(--text);font:14px/1.5 var(--font)}
15
+ main{max-width:1100px;margin:0 auto;padding:24px 16px}h1{font-size:24px;margin:0 0 4px}h2{font-size:16px;color:var(--muted);margin:32px 0 12px}
16
+ .meta{color:var(--muted)}.cards{display:flex;flex-wrap:wrap;gap:12px;margin-top:16px}
17
+ .card{background:var(--card);border:1px solid var(--border);border-radius:8px;padding:12px 16px;min-width:140px}
18
+ .card b{display:block;font-size:20px}table{width:100%;border-collapse:collapse;background:var(--card);border:1px solid var(--border);border-radius:8px}
19
+ th,td{text-align:left;padding:8px 12px;border-bottom:1px solid var(--border);vertical-align:top}th{background:var(--elev);color:var(--muted);font-weight:500}
20
+ .pass{color:var(--ok)}.fail,.regressed{color:var(--err)}.error,.blocked,.not_graded{color:var(--warn)}
21
+ details{background:var(--card);border:1px solid var(--border);border-radius:8px;margin:8px 0;padding:8px 12px}
22
+ summary{cursor:pointer}pre{white-space:pre-wrap;word-break:break-word;font:12px/1.5 var(--mono);background:var(--elev);padding:8px;border-radius:6px}
23
+ .bar{display:inline-block;height:8px;background:var(--ok);border-radius:4px;vertical-align:middle}`;
24
+ const VERDICT_CLASSES = new Set(["pass", "fail", "error", "blocked"]);
25
+ function statusCell(check) {
26
+ const verdict = check.verdict && VERDICT_CLASSES.has(check.verdict) ? check.verdict : "not_graded";
27
+ return `<span class="${verdict}">${escapeHtml(verdict)}</span> <span class="meta">${escapeHtml(check.status.replace(/_/g, " "))}</span>`;
28
+ }
29
+ function roundsTable(summary, iterations) {
30
+ const rows = summary.rounds.map((round) => {
31
+ const change = iterations.find((i) => i.version === round.version)?.change ?? "";
32
+ const width = round.total ? Math.round((round.passed / round.total) * 120) : 0;
33
+ return `<tr><td>v${round.version}${round.version === summary.best_version ? " ★" : ""}</td>
34
+ <td>${round.graded ? `<span class="bar" style="width:${width}px"></span> ${round.passed}/${round.total}` : '<span class="not_graded">not graded</span>'}</td>
35
+ <td class="fail">${escapeHtml(round.regressed.join(", "))}</td><td class="pass">${escapeHtml(round.fixed.join(", "))}</td>
36
+ <td><pre>${escapeHtml(change.trim())}</pre></td></tr>`;
37
+ });
38
+ return `<table><tr><th>Version</th><th>Passing</th><th>Regressed</th><th>Fixed</th><th>What changed</th></tr>${rows.join("")}</table>`;
39
+ }
40
+ function scenarioPanels(latest, classified) {
41
+ return (latest.scenarios?.scenarios ?? [])
42
+ .map((scenario) => {
43
+ const result = latest.results.get(scenario.id);
44
+ const checks = classified.filter((c) => c.scenario_id === scenario.id);
45
+ const passed = checks.filter((c) => c.verdict === "pass").length;
46
+ const turns = (result?.turns ?? [])
47
+ .map((turn) => {
48
+ const tools = turn.tool_calls
49
+ .map((t) => `<li>${escapeHtml(t.name)}${t.error ? ' <span class="fail">error</span>' : ""}<pre>${escapeHtml(t.input)}\n→ ${escapeHtml(t.output_excerpt)}</pre></li>`)
50
+ .join("");
51
+ return `<p><b>User:</b></p><pre>${escapeHtml(turn.user)}</pre><p><b>Assistant</b> <span class="meta">${escapeHtml(String(turn.latency_ms))} ms · ${escapeHtml(String(turn.tokens ?? "?"))} tokens</span></p><pre>${escapeHtml(turn.assistant)}</pre>${tools ? `<ul>${tools}</ul>` : ""}`;
52
+ })
53
+ .join("");
54
+ const error = result?.status === "error" ? `<p class="error">Run error: ${escapeHtml(result.error_message ?? "unknown")}</p>` : "";
55
+ const checkRows = checks
56
+ .map((c) => `<tr><td>${escapeHtml(c.check_id)}</td><td>${escapeHtml(c.text)}</td><td>${statusCell(c)}</td><td>${escapeHtml(c.reason)}</td></tr>`)
57
+ .join("");
58
+ return `<details${passed < checks.length ? " open" : ""}><summary>${escapeHtml(scenario.title)} <span class="meta">(${escapeHtml(scenario.id)}) · ${passed}/${checks.length}</span></summary>
59
+ ${error}<table><tr><th>Check</th><th>Expectation</th><th>Verdict</th><th>Reason</th></tr>${checkRows}</table>${turns}</details>`;
60
+ })
61
+ .join("");
62
+ }
63
+ export function renderHtmlReport(input) {
64
+ const { summary, iterations } = input;
65
+ const latest = iterations[iterations.length - 1];
66
+ const previous = iterations.length > 1 ? iterations[iterations.length - 2] : undefined;
67
+ const classified = classifyIteration(latest, previous);
68
+ const observations = summary.observations.map((o) => `<li>${escapeHtml(o)}</li>`).join("");
69
+ return `<!DOCTYPE html>
70
+ <html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
71
+ <title>${escapeHtml(input.assistantName)} · Assistant Test Report</title><style>${STYLE}</style></head>
72
+ <body><main>
73
+ <h1>${escapeHtml(input.assistantName)}</h1>
74
+ <div class="meta">Latest v${summary.latest_version}${summary.best_version !== null ? ` · best v${summary.best_version}` : ""}</div>
75
+ <div class="cards">
76
+ <div class="card"><span class="meta">Passing</span><b>${summary.latest.passed}/${summary.latest.total}</b></div>
77
+ <div class="card"><span class="meta">Fixed</span><b class="pass">${summary.latest.fixed.length}</b></div>
78
+ <div class="card"><span class="meta">Regressed</span><b class="fail">${summary.latest.regressed.length}</b></div>
79
+ <div class="card"><span class="meta">Rounds</span><b>${summary.rounds.length}</b></div>
80
+ </div>
81
+ <h2>Rounds</h2>${roundsTable(summary, iterations)}
82
+ ${observations ? `<h2>Observations</h2><ul>${observations}</ul>` : ""}
83
+ <h2>Scenarios (v${latest.version})</h2>${scenarioPanels(latest, classified)}
84
+ </main></body></html>
85
+ `;
86
+ }
87
+ //# sourceMappingURL=html.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"html.js","sourceRoot":"","sources":["../../../../../../src/cli/commands/sdk/assistant-builder/report/html.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAwB,MAAM,cAAc,CAAC;AAIvE,MAAM,UAAU,UAAU,CAAC,KAAa;IACtC,OAAO,KAAK;SACT,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC;SACtB,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC;SACrB,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC;SACrB,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC;SACvB,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;AAC5B,CAAC;AAED,MAAM,KAAK,GAAG;;;;;;;;;;;;;mGAaqF,CAAC;AAEpG,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC;AAEtE,SAAS,UAAU,CAAC,KAAsB;IACxC,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,IAAI,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,YAAY,CAAC;IACnG,OAAO,gBAAgB,OAAO,KAAK,UAAU,CAAC,OAAO,CAAC,8BAA8B,UAAU,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,SAAS,CAAC;AAC3I,CAAC;AAED,SAAS,WAAW,CAAC,OAAsB,EAAE,UAA2B;IACtE,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QACxC,MAAM,MAAM,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,KAAK,CAAC,OAAO,CAAC,EAAE,MAAM,IAAI,EAAE,CAAC;QACjF,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/E,OAAO,YAAY,KAAK,CAAC,OAAO,GAAG,KAAK,CAAC,OAAO,KAAK,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE;MACnF,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,kCAAkC,KAAK,eAAe,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,4CAA4C;mBACpI,UAAU,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,yBAAyB,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;WACzG,UAAU,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,kBAAkB,CAAC;IACrD,CAAC,CAAC,CAAC;IACH,OAAO,wGAAwG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC;AACzI,CAAC;AAED,SAAS,cAAc,CAAC,MAAqB,EAAE,UAA6B;IAC1E,OAAO,CAAC,MAAM,CAAC,SAAS,EAAE,SAAS,IAAI,EAAE,CAAC;SACvC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE;QAChB,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;QAC/C,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,QAAQ,CAAC,EAAE,CAAC,CAAC;QACvE,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,MAAM,CAAC,CAAC,MAAM,CAAC;QACjE,MAAM,KAAK,GAAG,CAAC,MAAM,EAAE,KAAK,IAAI,EAAE,CAAC;aAChC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACZ,MAAM,KAAK,GAAG,IAAI,CAAC,UAAU;iBAC1B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,kCAAkC,CAAC,CAAC,CAAC,EAAE,QAAQ,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,UAAU,CAAC,CAAC,CAAC,cAAc,CAAC,aAAa,CAAC;iBACpK,IAAI,CAAC,EAAE,CAAC,CAAC;YACZ,OAAO,2BAA2B,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,gDAAgD,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,SAAS,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,IAAI,GAAG,CAAC,CAAC,0BAA0B,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,KAAK,CAAC,CAAC,CAAC,OAAO,KAAK,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;QAC3R,CAAC,CAAC;aACD,IAAI,CAAC,EAAE,CAAC,CAAC;QACZ,MAAM,KAAK,GAAG,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,+BAA+B,UAAU,CAAC,MAAM,CAAC,aAAa,IAAI,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;QACnI,MAAM,SAAS,GAAG,MAAM;aACrB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,YAAY,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,UAAU,CAAC,CAAC,CAAC,YAAY,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC;aAChJ,IAAI,CAAC,EAAE,CAAC,CAAC;QACZ,OAAO,WAAW,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,aAAa,UAAU,CAAC,QAAQ,CAAC,KAAK,CAAC,wBAAwB,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC,OAAO,MAAM,IAAI,MAAM,CAAC,MAAM;EAC/K,KAAK,oFAAoF,SAAS,WAAW,KAAK,YAAY,CAAC;IAC7H,CAAC,CAAC;SACD,IAAI,CAAC,EAAE,CAAC,CAAC;AACd,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,KAAqF;IACpH,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,GAAG,KAAK,CAAC;IACtC,MAAM,MAAM,GAAG,UAAU,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACjD,MAAM,QAAQ,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACvF,MAAM,UAAU,GAAG,iBAAiB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IACvD,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAE3F,OAAO;;SAEA,UAAU,CAAC,KAAK,CAAC,aAAa,CAAC,0CAA0C,KAAK;;MAEjF,UAAU,CAAC,KAAK,CAAC,aAAa,CAAC;4BACT,OAAO,CAAC,cAAc,GAAG,OAAO,CAAC,YAAY,KAAK,IAAI,CAAC,CAAC,CAAC,YAAY,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE;;wDAEpE,OAAO,CAAC,MAAM,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM,CAAC,KAAK;mEAClC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM;uEACvB,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM;uDAC/C,OAAO,CAAC,MAAM,CAAC,MAAM;;iBAE3D,WAAW,CAAC,OAAO,EAAE,UAAU,CAAC;EAC/C,YAAY,CAAC,CAAC,CAAC,4BAA4B,YAAY,OAAO,CAAC,CAAC,CAAC,EAAE;kBACnD,MAAM,CAAC,OAAO,SAAS,cAAc,CAAC,MAAM,EAAE,UAAU,CAAC;;CAE1E,CAAC;AACF,CAAC"}
@@ -0,0 +1,31 @@
1
+ import { type ClassifiedCheck, type RoundScore } from "./compare.js";
2
+ import { type IterationData } from "./workspace.js";
3
+ export interface VerdictCounts {
4
+ pass: number;
5
+ fail: number;
6
+ error: number;
7
+ blocked: number;
8
+ not_graded: number;
9
+ }
10
+ export interface ReportSummary {
11
+ latest_version: number;
12
+ best_version: number | null;
13
+ rounds: RoundScore[];
14
+ latest: {
15
+ passed: number;
16
+ total: number;
17
+ fixed: string[];
18
+ regressed: string[];
19
+ still_failing: string[];
20
+ not_graded: number;
21
+ counts: VerdictCounts;
22
+ };
23
+ confirmation: {
24
+ passed: number;
25
+ total: number;
26
+ } | null;
27
+ observations: string[];
28
+ }
29
+ export declare function buildReportSummary(iterations: IterationData[]): ReportSummary;
30
+ export declare function formatTerminalSummary(summary: ReportSummary, failures: ClassifiedCheck[]): string;
31
+ //# sourceMappingURL=summary.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"summary.d.ts","sourceRoot":"","sources":["../../../../../../src/cli/commands/sdk/assistant-builder/report/summary.ts"],"names":[],"mappings":"AAAA,OAAO,EAA+C,KAAK,eAAe,EAAE,KAAK,UAAU,EAAE,MAAM,cAAc,CAAC;AAClH,OAAO,EAAY,KAAK,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAE9D,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,aAAa;IAC5B,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,MAAM,EAAE,UAAU,EAAE,CAAC;IACrB,MAAM,EAAE;QACN,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,EAAE,MAAM,EAAE,CAAC;QAChB,SAAS,EAAE,MAAM,EAAE,CAAC;QACpB,aAAa,EAAE,MAAM,EAAE,CAAC;QACxB,UAAU,EAAE,MAAM,CAAC;QACnB,MAAM,EAAE,aAAa,CAAC;KACvB,CAAC;IACF,YAAY,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;IACvD,YAAY,EAAE,MAAM,EAAE,CAAC;CACxB;AAED,wBAAgB,kBAAkB,CAAC,UAAU,EAAE,aAAa,EAAE,GAAG,aAAa,CAkC7E;AAED,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,eAAe,EAAE,GAAG,MAAM,CAgBjG"}
@@ -0,0 +1,56 @@
1
+ import { bestVersion, classifyIteration, scoreRounds } from "./compare.js";
2
+ import { checkKey } from "./workspace.js";
3
+ export function buildReportSummary(iterations) {
4
+ const rounds = scoreRounds(iterations);
5
+ const latest = iterations[iterations.length - 1];
6
+ const previous = iterations.length > 1 ? iterations[iterations.length - 2] : undefined;
7
+ const classified = classifyIteration(latest, previous);
8
+ const keys = (filter) => classified.filter(filter).map((c) => checkKey(c.scenario_id, c.check_id));
9
+ const confirmVerdicts = latest.confirm ? [...latest.confirm.values()] : null;
10
+ return {
11
+ latest_version: latest.version,
12
+ best_version: bestVersion(rounds),
13
+ rounds,
14
+ latest: {
15
+ passed: classified.filter((c) => c.verdict === "pass").length,
16
+ total: classified.length,
17
+ fixed: keys((c) => c.status === "fixed"),
18
+ regressed: keys((c) => c.status === "regressed"),
19
+ still_failing: keys((c) => c.verdict !== null && c.verdict !== "pass" && c.status !== "regressed"),
20
+ not_graded: classified.filter((c) => c.status === "not_graded").length,
21
+ counts: {
22
+ pass: classified.filter((c) => c.verdict === "pass").length,
23
+ fail: classified.filter((c) => c.verdict === "fail").length,
24
+ error: classified.filter((c) => c.verdict === "error").length,
25
+ blocked: classified.filter((c) => c.verdict === "blocked").length,
26
+ not_graded: classified.filter((c) => c.verdict === null).length,
27
+ },
28
+ },
29
+ confirmation: confirmVerdicts
30
+ ? { passed: confirmVerdicts.filter((v) => v.verdict === "pass").length, total: confirmVerdicts.length }
31
+ : null,
32
+ observations: latest.observations,
33
+ };
34
+ }
35
+ export function formatTerminalSummary(summary, failures) {
36
+ const lines = [];
37
+ lines.push(`Rounds: ${summary.rounds.map((r) => `v${r.version} ${r.graded ? `${r.passed}/${r.total}` : "not graded"}`).join(" → ")}`);
38
+ lines.push(`Latest v${summary.latest_version}: ${summary.latest.passed}/${summary.latest.total} checks pass` +
39
+ (summary.best_version !== null ? ` · best v${summary.best_version}` : ""));
40
+ if (summary.confirmation) {
41
+ lines.push(`Confirmation run: ${summary.confirmation.passed}/${summary.confirmation.total}`);
42
+ }
43
+ if (summary.latest.fixed.length)
44
+ lines.push(`Fixed: ${summary.latest.fixed.join(", ")}`);
45
+ if (summary.latest.regressed.length)
46
+ lines.push(`Regressed: ${summary.latest.regressed.join(", ")}`);
47
+ if (summary.latest.still_failing.length)
48
+ lines.push(`Still failing: ${summary.latest.still_failing.join(", ")}`);
49
+ if (summary.latest.not_graded)
50
+ lines.push(`Not graded: ${summary.latest.not_graded}`);
51
+ for (const failure of failures.slice(0, 3)) {
52
+ lines.push(` ✗ ${failure.scenario_id}/${failure.check_id} — ${failure.text}: ${failure.reason}`);
53
+ }
54
+ return lines.join("\n");
55
+ }
56
+ //# sourceMappingURL=summary.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"summary.js","sourceRoot":"","sources":["../../../../../../src/cli/commands/sdk/assistant-builder/report/summary.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,iBAAiB,EAAE,WAAW,EAAyC,MAAM,cAAc,CAAC;AAClH,OAAO,EAAE,QAAQ,EAAsB,MAAM,gBAAgB,CAAC;AA2B9D,MAAM,UAAU,kBAAkB,CAAC,UAA2B;IAC5D,MAAM,MAAM,GAAG,WAAW,CAAC,UAAU,CAAC,CAAC;IACvC,MAAM,MAAM,GAAG,UAAU,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACjD,MAAM,QAAQ,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACvF,MAAM,UAAU,GAAG,iBAAiB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IACvD,MAAM,IAAI,GAAG,CAAC,MAAuC,EAAY,EAAE,CACjE,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;IAE5E,MAAM,eAAe,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAE7E,OAAO;QACL,cAAc,EAAE,MAAM,CAAC,OAAO;QAC9B,YAAY,EAAE,WAAW,CAAC,MAAM,CAAC;QACjC,MAAM;QACN,MAAM,EAAE;YACN,MAAM,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,MAAM,CAAC,CAAC,MAAM;YAC7D,KAAK,EAAE,UAAU,CAAC,MAAM;YACxB,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,OAAO,CAAC;YACxC,SAAS,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,WAAW,CAAC;YAChD,aAAa,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,IAAI,IAAI,CAAC,CAAC,OAAO,KAAK,MAAM,IAAI,CAAC,CAAC,MAAM,KAAK,WAAW,CAAC;YAClG,UAAU,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,YAAY,CAAC,CAAC,MAAM;YACtE,MAAM,EAAE;gBACN,IAAI,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,MAAM,CAAC,CAAC,MAAM;gBAC3D,IAAI,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,MAAM,CAAC,CAAC,MAAM;gBAC3D,KAAK,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,OAAO,CAAC,CAAC,MAAM;gBAC7D,OAAO,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,MAAM;gBACjE,UAAU,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,IAAI,CAAC,CAAC,MAAM;aAChE;SACF;QACD,YAAY,EAAE,eAAe;YAC3B,CAAC,CAAC,EAAE,MAAM,EAAE,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,MAAM,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,eAAe,CAAC,MAAM,EAAE;YACvG,CAAC,CAAC,IAAI;QACR,YAAY,EAAE,MAAM,CAAC,YAAY;KAClC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,OAAsB,EAAE,QAA2B;IACvF,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,CAAC,IAAI,CAAC,WAAW,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,YAAY,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACtI,KAAK,CAAC,IAAI,CAAC,WAAW,OAAO,CAAC,cAAc,KAAK,OAAO,CAAC,MAAM,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM,CAAC,KAAK,cAAc;QAC1G,CAAC,OAAO,CAAC,YAAY,KAAK,IAAI,CAAC,CAAC,CAAC,YAAY,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC7E,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;QACzB,KAAK,CAAC,IAAI,CAAC,qBAAqB,OAAO,CAAC,YAAY,CAAC,MAAM,IAAI,OAAO,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC,CAAC;IAC/F,CAAC;IACD,IAAI,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM;QAAE,KAAK,CAAC,IAAI,CAAC,UAAU,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACzF,IAAI,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM;QAAE,KAAK,CAAC,IAAI,CAAC,cAAc,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACrG,IAAI,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM;QAAE,KAAK,CAAC,IAAI,CAAC,kBAAkB,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjH,IAAI,OAAO,CAAC,MAAM,CAAC,UAAU;QAAE,KAAK,CAAC,IAAI,CAAC,eAAe,OAAO,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC;IACtF,KAAK,MAAM,OAAO,IAAI,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;QAC3C,KAAK,CAAC,IAAI,CAAC,OAAO,OAAO,CAAC,WAAW,IAAI,OAAO,CAAC,QAAQ,MAAM,OAAO,CAAC,IAAI,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IACpG,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC"}
@@ -0,0 +1,19 @@
1
+ import type { CheckVerdict, ScenarioFile, ScenarioResult } from "../types.js";
2
+ export interface IterationData {
3
+ version: number;
4
+ dir: string;
5
+ scenarios: ScenarioFile | null;
6
+ results: Map<string, ScenarioResult>;
7
+ verdicts: Map<string, CheckVerdict> | null;
8
+ observations: string[];
9
+ change: string | null;
10
+ confirm: Map<string, CheckVerdict> | null;
11
+ }
12
+ export declare function checkKey(scenarioId: string, checkId: string): string;
13
+ /**
14
+ * Deterministic tool checks win over the grader, except where the grader marked a check
15
+ * blocked or error — a missing integration or a failed run is not a prompt failure.
16
+ */
17
+ export declare function mergeVerdicts(deterministic: CheckVerdict[], graded: CheckVerdict[]): Map<string, CheckVerdict>;
18
+ export declare function loadWorkspace(dir: string): Promise<IterationData[]>;
19
+ //# sourceMappingURL=workspace.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"workspace.d.ts","sourceRoot":"","sources":["../../../../../../src/cli/commands/sdk/assistant-builder/report/workspace.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,YAAY,EAA2B,YAAY,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAEvG,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,YAAY,GAAG,IAAI,CAAC;IAC/B,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;IACrC,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,YAAY,CAAC,GAAG,IAAI,CAAC;IAC3C,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,YAAY,CAAC,GAAG,IAAI,CAAC;CAC3C;AAED,wBAAgB,QAAQ,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAEpE;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAAC,aAAa,EAAE,YAAY,EAAE,EAAE,MAAM,EAAE,YAAY,EAAE,GAAG,GAAG,CAAC,MAAM,EAAE,YAAY,CAAC,CAU9G;AAqDD,wBAAsB,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC,CAqCzE"}
@@ -0,0 +1,112 @@
1
+ import { readdir, readFile } from "node:fs/promises";
2
+ import { join } from "node:path";
3
+ import { ConfigurationError } from "../../../../../utils/errors.js";
4
+ export function checkKey(scenarioId, checkId) {
5
+ return `${scenarioId}/${checkId}`;
6
+ }
7
+ /**
8
+ * Deterministic tool checks win over the grader, except where the grader marked a check
9
+ * blocked or error — a missing integration or a failed run is not a prompt failure.
10
+ */
11
+ export function mergeVerdicts(deterministic, graded) {
12
+ const merged = new Map();
13
+ for (const verdict of graded)
14
+ merged.set(checkKey(verdict.scenario_id, verdict.check_id), verdict);
15
+ for (const verdict of deterministic) {
16
+ const key = checkKey(verdict.scenario_id, verdict.check_id);
17
+ const gradedVerdict = merged.get(key)?.verdict;
18
+ if (gradedVerdict === "blocked" || gradedVerdict === "error")
19
+ continue;
20
+ merged.set(key, verdict);
21
+ }
22
+ return merged;
23
+ }
24
+ /** A check only counts as passing when it also passed on the all-pass confirmation run. */
25
+ function applyConfirmation(verdicts, confirm) {
26
+ for (const [key, confirmed] of confirm) {
27
+ const original = verdicts.get(key);
28
+ if (original?.verdict === "pass" && confirmed.verdict !== "pass") {
29
+ verdicts.set(key, { ...confirmed, reason: `failed on confirmation run: ${confirmed.reason}` });
30
+ }
31
+ }
32
+ }
33
+ async function readJson(path) {
34
+ try {
35
+ return JSON.parse(await readFile(path, "utf-8"));
36
+ }
37
+ catch {
38
+ return null;
39
+ }
40
+ }
41
+ async function readText(path) {
42
+ try {
43
+ return await readFile(path, "utf-8");
44
+ }
45
+ catch {
46
+ return null;
47
+ }
48
+ }
49
+ async function loadVerdicts(dir) {
50
+ const grading = await readJson(join(dir, "grading.json"));
51
+ if (!grading)
52
+ return { verdicts: null, observations: [] };
53
+ const checks = await readJson(join(dir, "checks.json"));
54
+ return {
55
+ verdicts: mergeVerdicts(checks?.checks ?? [], grading.checks ?? []),
56
+ observations: grading.observations ?? [],
57
+ };
58
+ }
59
+ async function loadResults(dir) {
60
+ const results = new Map();
61
+ let files = [];
62
+ try {
63
+ files = await readdir(join(dir, "results"));
64
+ }
65
+ catch {
66
+ return results;
67
+ }
68
+ for (const file of files.filter((f) => f.endsWith(".json"))) {
69
+ const result = await readJson(join(dir, "results", file));
70
+ if (result)
71
+ results.set(result.scenario_id, result);
72
+ }
73
+ return results;
74
+ }
75
+ export async function loadWorkspace(dir) {
76
+ let entries = [];
77
+ try {
78
+ entries = await readdir(join(dir, "iterations"));
79
+ }
80
+ catch {
81
+ entries = [];
82
+ }
83
+ const versions = entries
84
+ .map((name) => /^v(\d+)$/.exec(name))
85
+ .filter((m) => m !== null)
86
+ .map((m) => Number(m[1]))
87
+ .sort((a, b) => a - b);
88
+ if (versions.length === 0) {
89
+ throw new ConfigurationError(`No iterations found under ${join(dir, "iterations")}`);
90
+ }
91
+ const iterations = [];
92
+ for (const version of versions) {
93
+ const iterDir = join(dir, "iterations", `v${version}`);
94
+ const { verdicts, observations } = await loadVerdicts(iterDir);
95
+ const confirm = await loadVerdicts(join(iterDir, "confirm"));
96
+ if (verdicts && confirm.verdicts) {
97
+ applyConfirmation(verdicts, confirm.verdicts);
98
+ }
99
+ iterations.push({
100
+ version,
101
+ dir: iterDir,
102
+ scenarios: await readJson(join(iterDir, "scenarios.json")),
103
+ results: await loadResults(iterDir),
104
+ verdicts,
105
+ observations,
106
+ change: await readText(join(iterDir, "change.md")),
107
+ confirm: confirm.verdicts,
108
+ });
109
+ }
110
+ return iterations;
111
+ }
112
+ //# sourceMappingURL=workspace.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"workspace.js","sourceRoot":"","sources":["../../../../../../src/cli/commands/sdk/assistant-builder/report/workspace.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAcvD,MAAM,UAAU,QAAQ,CAAC,UAAkB,EAAE,OAAe;IAC1D,OAAO,GAAG,UAAU,IAAI,OAAO,EAAE,CAAC;AACpC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,aAA6B,EAAE,MAAsB;IACjF,MAAM,MAAM,GAAG,IAAI,GAAG,EAAwB,CAAC;IAC/C,KAAK,MAAM,OAAO,IAAI,MAAM;QAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,WAAW,EAAE,OAAO,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAC,CAAC;IACnG,KAAK,MAAM,OAAO,IAAI,aAAa,EAAE,CAAC;QACpC,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,CAAC,WAAW,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC5D,MAAM,aAAa,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,OAAO,CAAC;QAC/C,IAAI,aAAa,KAAK,SAAS,IAAI,aAAa,KAAK,OAAO;YAAE,SAAS;QACvE,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAC3B,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,2FAA2F;AAC3F,SAAS,iBAAiB,CAAC,QAAmC,EAAE,OAAkC;IAChG,KAAK,MAAM,CAAC,GAAG,EAAE,SAAS,CAAC,IAAI,OAAO,EAAE,CAAC;QACvC,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACnC,IAAI,QAAQ,EAAE,OAAO,KAAK,MAAM,IAAI,SAAS,CAAC,OAAO,KAAK,MAAM,EAAE,CAAC;YACjE,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,GAAG,SAAS,EAAE,MAAM,EAAE,+BAA+B,SAAS,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACjG,CAAC;IACH,CAAC;AACH,CAAC;AAED,KAAK,UAAU,QAAQ,CAAI,IAAY;IACrC,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAM,CAAC;IACxD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,KAAK,UAAU,QAAQ,CAAC,IAAY;IAClC,IAAI,CAAC;QACH,OAAO,MAAM,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACvC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,GAAW;IACrC,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAc,IAAI,CAAC,GAAG,EAAE,cAAc,CAAC,CAAC,CAAC;IACvE,IAAI,CAAC,OAAO;QAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,YAAY,EAAE,EAAE,EAAE,CAAC;IAC1D,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAa,IAAI,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC,CAAC;IACpE,OAAO;QACL,QAAQ,EAAE,aAAa,CAAC,MAAM,EAAE,MAAM,IAAI,EAAE,EAAE,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC;QACnE,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,EAAE;KACzC,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,GAAW;IACpC,MAAM,OAAO,GAAG,IAAI,GAAG,EAA0B,CAAC;IAClD,IAAI,KAAK,GAAa,EAAE,CAAC;IACzB,IAAI,CAAC;QACH,KAAK,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC,CAAC;IAC9C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,OAAO,CAAC;IACjB,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC;QAC5D,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAiB,IAAI,CAAC,GAAG,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC,CAAC;QAC1E,IAAI,MAAM;YAAE,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;IACtD,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,GAAW;IAC7C,IAAI,OAAO,GAAa,EAAE,CAAC;IAC3B,IAAI,CAAC;QACH,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC,CAAC;IACnD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,GAAG,EAAE,CAAC;IACf,CAAC;IACD,MAAM,QAAQ,GAAG,OAAO;SACrB,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;SACpC,MAAM,CAAC,CAAC,CAAC,EAAwB,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC;SAC/C,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;SACxB,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAEzB,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,kBAAkB,CAAC,6BAA6B,IAAI,CAAC,GAAG,EAAE,YAAY,CAAC,EAAE,CAAC,CAAC;IACvF,CAAC;IAED,MAAM,UAAU,GAAoB,EAAE,CAAC;IACvC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,YAAY,EAAE,IAAI,OAAO,EAAE,CAAC,CAAC;QACvD,MAAM,EAAE,QAAQ,EAAE,YAAY,EAAE,GAAG,MAAM,YAAY,CAAC,OAAO,CAAC,CAAC;QAC/D,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC;QAC7D,IAAI,QAAQ,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;YACjC,iBAAiB,CAAC,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QAChD,CAAC;QACD,UAAU,CAAC,IAAI,CAAC;YACd,OAAO;YACP,GAAG,EAAE,OAAO;YACZ,SAAS,EAAE,MAAM,QAAQ,CAAe,IAAI,CAAC,OAAO,EAAE,gBAAgB,CAAC,CAAC;YACxE,OAAO,EAAE,MAAM,WAAW,CAAC,OAAO,CAAC;YACnC,QAAQ;YACR,YAAY;YACZ,MAAM,EAAE,MAAM,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;YAClD,OAAO,EAAE,OAAO,CAAC,QAAQ;SAC1B,CAAC,CAAC;IACL,CAAC;IACD,OAAO,UAAU,CAAC;AACpB,CAAC"}
@@ -0,0 +1,32 @@
1
+ import type { BaseModelResponse } from "codemie-sdk";
2
+ import { CodeMieError } from "../../../../utils/errors.js";
3
+ import type { ChatTurnInput } from "../services/assistants.js";
4
+ import type { HistoryEntry, NormalizedTurn, Scenario, ScenarioResult } from "./types.js";
5
+ export declare class TurnTimeoutError extends CodeMieError {
6
+ constructor(timeoutMs: number);
7
+ }
8
+ export interface RunnerDeps {
9
+ chat: (input: ChatTurnInput) => Promise<BaseModelResponse>;
10
+ now?: () => number;
11
+ sleep?: (ms: number) => Promise<void>;
12
+ }
13
+ export interface RunOptions {
14
+ timeoutMs: number;
15
+ concurrency: number;
16
+ version?: number;
17
+ retryDelayMs?: number;
18
+ }
19
+ export interface TurnOutcome {
20
+ turn: NormalizedTurn;
21
+ response: BaseModelResponse;
22
+ }
23
+ export interface ScenarioRun {
24
+ result: ScenarioResult;
25
+ rawThoughts: unknown[];
26
+ }
27
+ export declare function clampConcurrency(n: number): number;
28
+ export declare function isRetryable(error: unknown): boolean;
29
+ export declare function runTurn(deps: RunnerDeps, message: string, history: HistoryEntry[], opts: RunOptions): Promise<TurnOutcome>;
30
+ export declare function runScenario(scenario: Scenario, deps: RunnerDeps, opts: RunOptions): Promise<ScenarioRun>;
31
+ export declare function runScenarios(scenarios: Scenario[], deps: RunnerDeps, opts: RunOptions, onDone?: (run: ScenarioRun) => void): Promise<ScenarioRun[]>;
32
+ //# sourceMappingURL=runner.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../../../../src/cli/commands/sdk/assistant-builder/runner.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAEjD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAE/D,OAAO,KAAK,EAAE,YAAY,EAAE,cAAc,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAKzF,qBAAa,gBAAiB,SAAQ,YAAY;gBACpC,SAAS,EAAE,MAAM;CAI9B;AAED,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,CAAC,KAAK,EAAE,aAAa,KAAK,OAAO,CAAC,iBAAiB,CAAC,CAAC;IAC3D,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CACvC;AAED,MAAM,WAAW,UAAU;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,cAAc,CAAC;IACrB,QAAQ,EAAE,iBAAiB,CAAC;CAC7B;AAED,MAAM,WAAW,WAAW;IAC1B,MAAM,EAAE,cAAc,CAAC;IACvB,WAAW,EAAE,OAAO,EAAE,CAAC;CACxB;AAED,wBAAgB,gBAAgB,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAGlD;AAED,wBAAgB,WAAW,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAKnD;AAYD,wBAAsB,OAAO,CAC3B,IAAI,EAAE,UAAU,EAChB,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,YAAY,EAAE,EACvB,IAAI,EAAE,UAAU,GACf,OAAO,CAAC,WAAW,CAAC,CA0BtB;AAED,wBAAsB,WAAW,CAC/B,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,UAAU,EAChB,IAAI,EAAE,UAAU,GACf,OAAO,CAAC,WAAW,CAAC,CAmCtB;AAED,wBAAsB,YAAY,CAChC,SAAS,EAAE,QAAQ,EAAE,EACrB,IAAI,EAAE,UAAU,EAChB,IAAI,EAAE,UAAU,EAChB,MAAM,CAAC,EAAE,CAAC,GAAG,EAAE,WAAW,KAAK,IAAI,GAClC,OAAO,CAAC,WAAW,EAAE,CAAC,CAgBxB"}
@@ -0,0 +1,109 @@
1
+ import { CodeMieError } from "../../../../utils/errors.js";
2
+ import { logger } from "../../../../utils/logger.js";
3
+ import { generatedToText, thoughtsToToolCalls } from "./normalize.js";
4
+ const MAX_CONCURRENCY = 8;
5
+ const DEFAULT_RETRY_DELAY_MS = 2000;
6
+ export class TurnTimeoutError extends CodeMieError {
7
+ constructor(timeoutMs) {
8
+ super(`Assistant turn timed out after ${timeoutMs} ms`);
9
+ this.name = "TurnTimeoutError";
10
+ }
11
+ }
12
+ export function clampConcurrency(n) {
13
+ if (!Number.isFinite(n) || n < 1)
14
+ return 1;
15
+ return Math.min(Math.floor(n), MAX_CONCURRENCY);
16
+ }
17
+ export function isRetryable(error) {
18
+ if (!error || typeof error !== "object")
19
+ return false;
20
+ const record = error;
21
+ const status = typeof record.statusCode === "number" ? record.statusCode : record.status;
22
+ return typeof status === "number" && (status === 429 || status >= 500);
23
+ }
24
+ function withTimeout(promise, timeoutMs) {
25
+ let timer;
26
+ const timeout = new Promise((_, reject) => {
27
+ timer = setTimeout(() => reject(new TurnTimeoutError(timeoutMs)), timeoutMs);
28
+ });
29
+ return Promise.race([promise, timeout]).finally(() => clearTimeout(timer));
30
+ }
31
+ const defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
32
+ export async function runTurn(deps, message, history, opts) {
33
+ const now = deps.now ?? Date.now;
34
+ const sleep = deps.sleep ?? defaultSleep;
35
+ const input = { message, history, version: opts.version };
36
+ const started = now();
37
+ let response;
38
+ try {
39
+ response = await withTimeout(deps.chat(input), opts.timeoutMs);
40
+ }
41
+ catch (error) {
42
+ if (!isRetryable(error))
43
+ throw error;
44
+ logger.debug("Retrying assistant turn after retryable error", { message: String(error) });
45
+ await sleep(opts.retryDelayMs ?? DEFAULT_RETRY_DELAY_MS);
46
+ response = await withTimeout(deps.chat(input), opts.timeoutMs);
47
+ }
48
+ return {
49
+ response,
50
+ turn: {
51
+ user: message,
52
+ assistant: generatedToText(response.generated),
53
+ tool_calls: thoughtsToToolCalls(response.thoughts),
54
+ tokens: typeof response.tokens_used === "number" ? response.tokens_used : null,
55
+ latency_ms: now() - started,
56
+ },
57
+ };
58
+ }
59
+ export async function runScenario(scenario, deps, opts) {
60
+ const history = [];
61
+ const turns = [];
62
+ const rawThoughts = [];
63
+ const toolErrors = [];
64
+ const result = {
65
+ scenario_id: scenario.id,
66
+ status: "ok",
67
+ turns,
68
+ agent_error: null,
69
+ tool_errors: toolErrors,
70
+ raw_thoughts_file: `raw/${scenario.id}.json`,
71
+ };
72
+ for (const message of scenario.turns) {
73
+ try {
74
+ const { turn, response } = await runTurn(deps, message, [...history], opts);
75
+ turns.push(turn);
76
+ rawThoughts.push(response.thoughts ?? []);
77
+ toolErrors.push(...(response.tool_errors ?? []));
78
+ if (response.agent_error) {
79
+ result.status = "error";
80
+ result.agent_error = response.agent_error;
81
+ result.error_message = "Assistant reported an agent error";
82
+ break;
83
+ }
84
+ history.push({ role: "User", message }, { role: "Assistant", message: turn.assistant });
85
+ }
86
+ catch (error) {
87
+ result.status = "error";
88
+ result.error_message = error instanceof Error ? error.message : String(error);
89
+ break;
90
+ }
91
+ }
92
+ return { result, rawThoughts };
93
+ }
94
+ export async function runScenarios(scenarios, deps, opts, onDone) {
95
+ const runs = new Array(scenarios.length);
96
+ let next = 0;
97
+ const worker = async () => {
98
+ while (next < scenarios.length) {
99
+ const index = next++;
100
+ const run = await runScenario(scenarios[index], deps, opts);
101
+ runs[index] = run;
102
+ onDone?.(run);
103
+ }
104
+ };
105
+ const workers = Array.from({ length: Math.min(clampConcurrency(opts.concurrency), scenarios.length) }, worker);
106
+ await Promise.all(workers);
107
+ return runs;
108
+ }
109
+ //# sourceMappingURL=runner.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"runner.js","sourceRoot":"","sources":["../../../../../src/cli/commands/sdk/assistant-builder/runner.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACjD,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C,OAAO,EAAE,eAAe,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAGtE,MAAM,eAAe,GAAG,CAAC,CAAC;AAC1B,MAAM,sBAAsB,GAAG,IAAI,CAAC;AAEpC,MAAM,OAAO,gBAAiB,SAAQ,YAAY;IAChD,YAAY,SAAiB;QAC3B,KAAK,CAAC,kCAAkC,SAAS,KAAK,CAAC,CAAC;QACxD,IAAI,CAAC,IAAI,GAAG,kBAAkB,CAAC;IACjC,CAAC;CACF;AAyBD,MAAM,UAAU,gBAAgB,CAAC,CAAS;IACxC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC;IAC3C,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,eAAe,CAAC,CAAC;AAClD,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,KAAc;IACxC,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IACtD,MAAM,MAAM,GAAG,KAAmD,CAAC;IACnE,MAAM,MAAM,GAAG,OAAO,MAAM,CAAC,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC;IACzF,OAAO,OAAO,MAAM,KAAK,QAAQ,IAAI,CAAC,MAAM,KAAK,GAAG,IAAI,MAAM,IAAI,GAAG,CAAC,CAAC;AACzE,CAAC;AAED,SAAS,WAAW,CAAI,OAAmB,EAAE,SAAiB;IAC5D,IAAI,KAAiC,CAAC;IACtC,MAAM,OAAO,GAAG,IAAI,OAAO,CAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE;QAC/C,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,gBAAgB,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;IAC/E,CAAC,CAAC,CAAC;IACH,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC;AAC7E,CAAC;AAED,MAAM,YAAY,GAAG,CAAC,EAAU,EAAiB,EAAE,CAAC,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAEtG,MAAM,CAAC,KAAK,UAAU,OAAO,CAC3B,IAAgB,EAChB,OAAe,EACf,OAAuB,EACvB,IAAgB;IAEhB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC;IACjC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,YAAY,CAAC;IACzC,MAAM,KAAK,GAAkB,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC;IACzE,MAAM,OAAO,GAAG,GAAG,EAAE,CAAC;IAEtB,IAAI,QAA2B,CAAC;IAChC,IAAI,CAAC;QACH,QAAQ,GAAG,MAAM,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IACjE,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC;YAAE,MAAM,KAAK,CAAC;QACrC,MAAM,CAAC,KAAK,CAAC,+CAA+C,EAAE,EAAE,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC1F,MAAM,KAAK,CAAC,IAAI,CAAC,YAAY,IAAI,sBAAsB,CAAC,CAAC;QACzD,QAAQ,GAAG,MAAM,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IACjE,CAAC;IAED,OAAO;QACL,QAAQ;QACR,IAAI,EAAE;YACJ,IAAI,EAAE,OAAO;YACb,SAAS,EAAE,eAAe,CAAC,QAAQ,CAAC,SAAS,CAAC;YAC9C,UAAU,EAAE,mBAAmB,CAAC,QAAQ,CAAC,QAAQ,CAAC;YAClD,MAAM,EAAE,OAAO,QAAQ,CAAC,WAAW,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI;YAC9E,UAAU,EAAE,GAAG,EAAE,GAAG,OAAO;SAC5B;KACF,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,QAAkB,EAClB,IAAgB,EAChB,IAAgB;IAEhB,MAAM,OAAO,GAAmB,EAAE,CAAC;IACnC,MAAM,KAAK,GAAqB,EAAE,CAAC;IACnC,MAAM,WAAW,GAAc,EAAE,CAAC;IAClC,MAAM,UAAU,GAAc,EAAE,CAAC;IACjC,MAAM,MAAM,GAAmB;QAC7B,WAAW,EAAE,QAAQ,CAAC,EAAE;QACxB,MAAM,EAAE,IAAI;QACZ,KAAK;QACL,WAAW,EAAE,IAAI;QACjB,WAAW,EAAE,UAAU;QACvB,iBAAiB,EAAE,OAAO,QAAQ,CAAC,EAAE,OAAO;KAC7C,CAAC;IAEF,KAAK,MAAM,OAAO,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;QACrC,IAAI,CAAC;YACH,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,MAAM,OAAO,CAAC,IAAI,EAAE,OAAO,EAAE,CAAC,GAAG,OAAO,CAAC,EAAE,IAAI,CAAC,CAAC;YAC5E,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjB,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC;YAC1C,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,CAAC;YACjD,IAAI,QAAQ,CAAC,WAAW,EAAE,CAAC;gBACzB,MAAM,CAAC,MAAM,GAAG,OAAO,CAAC;gBACxB,MAAM,CAAC,WAAW,GAAG,QAAQ,CAAC,WAAW,CAAC;gBAC1C,MAAM,CAAC,aAAa,GAAG,mCAAmC,CAAC;gBAC3D,MAAM;YACR,CAAC;YACD,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;QAC1F,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,MAAM,GAAG,OAAO,CAAC;YACxB,MAAM,CAAC,aAAa,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAC9E,MAAM;QACR,CAAC;IACH,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;AACjC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,SAAqB,EACrB,IAAgB,EAChB,IAAgB,EAChB,MAAmC;IAEnC,MAAM,IAAI,GAAkB,IAAI,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IACxD,IAAI,IAAI,GAAG,CAAC,CAAC;IAEb,MAAM,MAAM,GAAG,KAAK,IAAmB,EAAE;QACvC,OAAO,IAAI,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC;YAC/B,MAAM,KAAK,GAAG,IAAI,EAAE,CAAC;YACrB,MAAM,GAAG,GAAG,MAAM,WAAW,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;YAC5D,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC;YAClB,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC;QAChB,CAAC;IACH,CAAC,CAAC;IAEF,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,gBAAgB,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,SAAS,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;IAC/G,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAC3B,OAAO,IAAI,CAAC;AACd,CAAC"}
@@ -0,0 +1,5 @@
1
+ import type { Scenario, ScenarioFile } from "./types.js";
2
+ export declare function parseScenarioFile(raw: unknown): ScenarioFile;
3
+ export declare function loadScenarioFile(path: string): Promise<ScenarioFile>;
4
+ export declare function filterScenarios(file: ScenarioFile, only?: string[]): Scenario[];
5
+ //# sourceMappingURL=scenarios.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"scenarios.d.ts","sourceRoot":"","sources":["../../../../../src/cli/commands/sdk/assistant-builder/scenarios.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAyBzD,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,OAAO,GAAG,YAAY,CAuB5D;AAED,wBAAsB,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAS1E;AAED,wBAAgB,eAAe,CAAC,IAAI,EAAE,YAAY,EAAE,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,QAAQ,EAAE,CAQ/E"}
@@ -0,0 +1,66 @@
1
+ import { readFile } from "node:fs/promises";
2
+ import z from "zod";
3
+ import { ConfigurationError } from "../../../../utils/errors.js";
4
+ const SAFE_ID = /^[a-z0-9][a-z0-9-]{0,63}$/;
5
+ const CheckSchema = z
6
+ .object({
7
+ id: z.string().regex(SAFE_ID),
8
+ text: z.string().min(1),
9
+ kind: z.enum(["tool_called", "tool_not_called"]).optional(),
10
+ tool: z.string().min(1).optional(),
11
+ })
12
+ .refine((c) => !c.kind || !!c.tool, { message: "tool is required when kind is set" });
13
+ const ScenarioSchema = z.object({
14
+ id: z.string().regex(SAFE_ID),
15
+ title: z.string().min(1),
16
+ turns: z.array(z.string().min(1)).min(1),
17
+ checks: z.array(CheckSchema).min(1),
18
+ added_in: z.number().int().optional(),
19
+ origin: z.string().optional(),
20
+ source_conversation: z.string().optional(),
21
+ });
22
+ const ScenarioFileSchema = z.object({ scenarios: z.array(ScenarioSchema).min(1) });
23
+ export function parseScenarioFile(raw) {
24
+ const parsed = ScenarioFileSchema.safeParse(raw);
25
+ if (!parsed.success) {
26
+ throw new ConfigurationError(`Invalid scenarios file:\n${z.prettifyError(parsed.error)}`);
27
+ }
28
+ const scenarioIds = new Set();
29
+ for (const scenario of parsed.data.scenarios) {
30
+ if (scenarioIds.has(scenario.id)) {
31
+ throw new ConfigurationError(`Duplicate scenario id: ${scenario.id}`);
32
+ }
33
+ scenarioIds.add(scenario.id);
34
+ const checkIds = new Set();
35
+ for (const check of scenario.checks) {
36
+ if (checkIds.has(check.id)) {
37
+ throw new ConfigurationError(`Duplicate check id ${check.id} in scenario ${scenario.id}`);
38
+ }
39
+ checkIds.add(check.id);
40
+ }
41
+ }
42
+ return parsed.data;
43
+ }
44
+ export async function loadScenarioFile(path) {
45
+ const content = await readFile(path, "utf-8");
46
+ let raw;
47
+ try {
48
+ raw = JSON.parse(content);
49
+ }
50
+ catch {
51
+ throw new ConfigurationError(`Scenarios file is not valid JSON: ${path}`);
52
+ }
53
+ return parseScenarioFile(raw);
54
+ }
55
+ export function filterScenarios(file, only) {
56
+ if (!only || only.length === 0)
57
+ return file.scenarios;
58
+ const known = new Set(file.scenarios.map((s) => s.id));
59
+ for (const id of only) {
60
+ if (!known.has(id))
61
+ throw new ConfigurationError(`Unknown scenario id: ${id}`);
62
+ }
63
+ const wanted = new Set(only);
64
+ return file.scenarios.filter((s) => wanted.has(s.id));
65
+ }
66
+ //# sourceMappingURL=scenarios.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"scenarios.js","sourceRoot":"","sources":["../../../../../src/cli/commands/sdk/assistant-builder/scenarios.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,CAAC,MAAM,KAAK,CAAC;AACpB,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAGvD,MAAM,OAAO,GAAG,2BAA2B,CAAC;AAE5C,MAAM,WAAW,GAAG,CAAC;KAClB,MAAM,CAAC;IACN,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC;IAC7B,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACvB,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,aAAa,EAAE,iBAAiB,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC3D,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;CACnC,CAAC;KACD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,mCAAmC,EAAE,CAAC,CAAC;AAExF,MAAM,cAAc,GAAG,CAAC,CAAC,MAAM,CAAC;IAC9B,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC;IAC7B,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACxB,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACxC,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACnC,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;IACrC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC7B,mBAAmB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CAC3C,CAAC,CAAC;AAEH,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;AAEnF,MAAM,UAAU,iBAAiB,CAAC,GAAY;IAC5C,MAAM,MAAM,GAAG,kBAAkB,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IACjD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,IAAI,kBAAkB,CAAC,4BAA4B,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAC5F,CAAC;IAED,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;IACtC,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;QAC7C,IAAI,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC;YACjC,MAAM,IAAI,kBAAkB,CAAC,0BAA0B,QAAQ,CAAC,EAAE,EAAE,CAAC,CAAC;QACxE,CAAC;QACD,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;QAE7B,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;QACnC,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;YACpC,IAAI,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,CAAC;gBAC3B,MAAM,IAAI,kBAAkB,CAAC,sBAAsB,KAAK,CAAC,EAAE,gBAAgB,QAAQ,CAAC,EAAE,EAAE,CAAC,CAAC;YAC5F,CAAC;YACD,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACzB,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC,IAAI,CAAC;AACrB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,IAAY;IACjD,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC9C,IAAI,GAAY,CAAC;IACjB,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC5B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,kBAAkB,CAAC,qCAAqC,IAAI,EAAE,CAAC,CAAC;IAC5E,CAAC;IACD,OAAO,iBAAiB,CAAC,GAAG,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,IAAkB,EAAE,IAAe;IACjE,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC,SAAS,CAAC;IACtD,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACvD,KAAK,MAAM,EAAE,IAAI,IAAI,EAAE,CAAC;QACtB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;YAAE,MAAM,IAAI,kBAAkB,CAAC,wBAAwB,EAAE,EAAE,CAAC,CAAC;IACjF,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC;IAC7B,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AACxD,CAAC"}