@post-print/agent-test 1.1.9 → 2.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (300) hide show
  1. package/README.md +18 -66
  2. package/dist/cli.d.ts +2 -64
  3. package/dist/cli.d.ts.map +1 -1
  4. package/dist/cli.js +26 -691
  5. package/dist/cli.js.map +1 -1
  6. package/dist/host-login.d.ts +1 -26
  7. package/dist/host-login.d.ts.map +1 -1
  8. package/dist/host-login.js +1 -92
  9. package/dist/host-login.js.map +1 -1
  10. package/dist/index.d.ts +6 -28
  11. package/dist/index.d.ts.map +1 -1
  12. package/dist/index.js +4 -26
  13. package/dist/index.js.map +1 -1
  14. package/dist/sdk/cli.d.ts +3 -0
  15. package/dist/sdk/cli.d.ts.map +1 -0
  16. package/dist/sdk/cli.js +55 -0
  17. package/dist/sdk/cli.js.map +1 -0
  18. package/dist/sdk/definitions.d.ts +22 -0
  19. package/dist/sdk/definitions.d.ts.map +1 -0
  20. package/dist/sdk/definitions.js +58 -0
  21. package/dist/sdk/definitions.js.map +1 -0
  22. package/dist/sdk/execution-reporter.d.ts +18 -0
  23. package/dist/sdk/execution-reporter.d.ts.map +1 -0
  24. package/dist/sdk/execution-reporter.js +171 -0
  25. package/dist/sdk/execution-reporter.js.map +1 -0
  26. package/dist/sdk/execution-runner.d.ts +14 -0
  27. package/dist/sdk/execution-runner.d.ts.map +1 -0
  28. package/dist/sdk/execution-runner.js +80 -0
  29. package/dist/sdk/execution-runner.js.map +1 -0
  30. package/dist/sdk/execution-store.d.ts +92 -0
  31. package/dist/sdk/execution-store.d.ts.map +1 -0
  32. package/dist/sdk/execution-store.js +357 -0
  33. package/dist/sdk/execution-store.js.map +1 -0
  34. package/dist/sdk/expect.d.ts +31 -0
  35. package/dist/sdk/expect.d.ts.map +1 -0
  36. package/dist/sdk/expect.js +96 -0
  37. package/dist/sdk/expect.js.map +1 -0
  38. package/dist/sdk/judge.d.ts +16 -0
  39. package/dist/sdk/judge.d.ts.map +1 -0
  40. package/dist/sdk/judge.js +82 -0
  41. package/dist/sdk/judge.js.map +1 -0
  42. package/dist/sdk/metrics.d.ts +3 -0
  43. package/dist/sdk/metrics.d.ts.map +1 -0
  44. package/dist/sdk/metrics.js +24 -0
  45. package/dist/sdk/metrics.js.map +1 -0
  46. package/dist/sdk/playwright-cli.d.ts +2 -0
  47. package/dist/sdk/playwright-cli.d.ts.map +1 -0
  48. package/dist/sdk/playwright-cli.js +6 -0
  49. package/dist/sdk/playwright-cli.js.map +1 -0
  50. package/dist/sdk/reporter.d.ts +12 -0
  51. package/dist/sdk/reporter.d.ts.map +1 -0
  52. package/dist/sdk/reporter.js +98 -0
  53. package/dist/sdk/reporter.js.map +1 -0
  54. package/dist/sdk/runtime.d.ts +33 -0
  55. package/dist/sdk/runtime.d.ts.map +1 -0
  56. package/dist/sdk/runtime.js +197 -0
  57. package/dist/sdk/runtime.js.map +1 -0
  58. package/dist/sdk/test.d.ts +36 -0
  59. package/dist/sdk/test.d.ts.map +1 -0
  60. package/dist/sdk/test.js +133 -0
  61. package/dist/sdk/test.js.map +1 -0
  62. package/dist/sdk/types.d.ts +76 -0
  63. package/dist/sdk/types.d.ts.map +1 -0
  64. package/dist/sdk/types.js.map +1 -0
  65. package/dist/sdk/usage.d.ts +10 -0
  66. package/dist/sdk/usage.d.ts.map +1 -0
  67. package/dist/sdk/usage.js +13 -0
  68. package/dist/sdk/usage.js.map +1 -0
  69. package/dist/sdk/viewer.d.ts +5 -0
  70. package/dist/sdk/viewer.d.ts.map +1 -0
  71. package/dist/sdk/viewer.js +74 -0
  72. package/dist/sdk/viewer.js.map +1 -0
  73. package/dist/sdk/workspace.d.ts +33 -0
  74. package/dist/sdk/workspace.d.ts.map +1 -0
  75. package/dist/sdk/workspace.js +100 -0
  76. package/dist/sdk/workspace.js.map +1 -0
  77. package/dist/viewer/bootstrap.d.ts +10 -0
  78. package/dist/viewer/bootstrap.d.ts.map +1 -0
  79. package/dist/viewer/bootstrap.js +2 -0
  80. package/dist/viewer/bootstrap.js.map +1 -0
  81. package/dist/viewer/client-entry.d.ts +2 -0
  82. package/dist/viewer/client-entry.d.ts.map +1 -0
  83. package/dist/viewer/client-entry.js +31 -0
  84. package/dist/viewer/client-entry.js.map +1 -0
  85. package/dist/viewer/generated-api.d.ts +15 -0
  86. package/dist/viewer/generated-api.d.ts.map +1 -0
  87. package/dist/viewer/generated-api.js +42 -0
  88. package/dist/viewer/generated-api.js.map +1 -0
  89. package/dist/viewer/generated-contracts.d.ts +13 -0
  90. package/dist/viewer/generated-contracts.d.ts.map +1 -0
  91. package/dist/viewer/generated-contracts.js +12 -0
  92. package/dist/viewer/generated-contracts.js.map +1 -0
  93. package/dist/viewer/main.js +64 -3
  94. package/dist/viewer/main.js.map +1 -1
  95. package/dist/viewer/page.d.ts +2 -4
  96. package/dist/viewer/page.d.ts.map +1 -1
  97. package/dist/viewer/page.js +18 -194
  98. package/dist/viewer/page.js.map +1 -1
  99. package/dist/viewer/presentation.d.ts +8 -0
  100. package/dist/viewer/presentation.d.ts.map +1 -0
  101. package/dist/viewer/presentation.js +45 -0
  102. package/dist/viewer/presentation.js.map +1 -0
  103. package/dist/viewer/protocol.d.ts +21 -0
  104. package/dist/viewer/protocol.d.ts.map +1 -0
  105. package/dist/viewer/protocol.js +6 -0
  106. package/dist/viewer/protocol.js.map +1 -0
  107. package/dist/viewer/server.d.ts +7 -11
  108. package/dist/viewer/server.d.ts.map +1 -1
  109. package/dist/viewer/server.js +314 -175
  110. package/dist/viewer/server.js.map +1 -1
  111. package/dist/viewer/styles.d.ts +2 -0
  112. package/dist/viewer/styles.d.ts.map +1 -0
  113. package/dist/viewer/styles.js +97 -0
  114. package/dist/viewer/styles.js.map +1 -0
  115. package/dist/viewer/stylex.css +1322 -0
  116. package/dist/viewer/test-catalog.d.ts +43 -0
  117. package/dist/viewer/test-catalog.d.ts.map +1 -0
  118. package/dist/viewer/test-catalog.js +27 -0
  119. package/dist/viewer/test-catalog.js.map +1 -0
  120. package/dist/viewer/v2-client.d.ts +7 -0
  121. package/dist/viewer/v2-client.d.ts.map +1 -0
  122. package/dist/viewer/v2-client.js +1862 -0
  123. package/dist/viewer/v2-client.js.map +1 -0
  124. package/package.json +19 -9
  125. package/dist/check.d.ts +0 -38
  126. package/dist/check.d.ts.map +0 -1
  127. package/dist/check.js +0 -169
  128. package/dist/check.js.map +0 -1
  129. package/dist/command-allowlist.d.ts +0 -9
  130. package/dist/command-allowlist.d.ts.map +0 -1
  131. package/dist/command-allowlist.js +0 -29
  132. package/dist/command-allowlist.js.map +0 -1
  133. package/dist/compare-scenario.d.ts +0 -65
  134. package/dist/compare-scenario.d.ts.map +0 -1
  135. package/dist/compare-scenario.js +0 -328
  136. package/dist/compare-scenario.js.map +0 -1
  137. package/dist/debug-bundle.d.ts +0 -81
  138. package/dist/debug-bundle.d.ts.map +0 -1
  139. package/dist/debug-bundle.js +0 -392
  140. package/dist/debug-bundle.js.map +0 -1
  141. package/dist/discover-suites.d.ts +0 -2
  142. package/dist/discover-suites.d.ts.map +0 -1
  143. package/dist/discover-suites.js +0 -19
  144. package/dist/discover-suites.js.map +0 -1
  145. package/dist/doctor.d.ts +0 -24
  146. package/dist/doctor.d.ts.map +0 -1
  147. package/dist/doctor.js +0 -272
  148. package/dist/doctor.js.map +0 -1
  149. package/dist/expect.d.ts +0 -59
  150. package/dist/expect.d.ts.map +0 -1
  151. package/dist/expect.js +0 -518
  152. package/dist/expect.js.map +0 -1
  153. package/dist/failures.d.ts +0 -4
  154. package/dist/failures.d.ts.map +0 -1
  155. package/dist/failures.js +0 -7
  156. package/dist/failures.js.map +0 -1
  157. package/dist/host-log.d.ts +0 -17
  158. package/dist/host-log.d.ts.map +0 -1
  159. package/dist/host-log.js +0 -154
  160. package/dist/host-log.js.map +0 -1
  161. package/dist/hosts.d.ts +0 -22
  162. package/dist/hosts.d.ts.map +0 -1
  163. package/dist/hosts.js +0 -84
  164. package/dist/hosts.js.map +0 -1
  165. package/dist/html-report.d.ts +0 -18
  166. package/dist/html-report.d.ts.map +0 -1
  167. package/dist/html-report.js +0 -1276
  168. package/dist/html-report.js.map +0 -1
  169. package/dist/live-isolation.d.ts +0 -82
  170. package/dist/live-isolation.d.ts.map +0 -1
  171. package/dist/live-isolation.js +0 -399
  172. package/dist/live-isolation.js.map +0 -1
  173. package/dist/live-timeout.d.ts +0 -11
  174. package/dist/live-timeout.d.ts.map +0 -1
  175. package/dist/live-timeout.js +0 -29
  176. package/dist/live-timeout.js.map +0 -1
  177. package/dist/load-adapters.d.ts +0 -13
  178. package/dist/load-adapters.d.ts.map +0 -1
  179. package/dist/load-adapters.js +0 -76
  180. package/dist/load-adapters.js.map +0 -1
  181. package/dist/load-suite.d.ts +0 -22
  182. package/dist/load-suite.d.ts.map +0 -1
  183. package/dist/load-suite.js +0 -154
  184. package/dist/load-suite.js.map +0 -1
  185. package/dist/mcp-config.d.ts +0 -13
  186. package/dist/mcp-config.d.ts.map +0 -1
  187. package/dist/mcp-config.js +0 -121
  188. package/dist/mcp-config.js.map +0 -1
  189. package/dist/preflight.d.ts +0 -3
  190. package/dist/preflight.d.ts.map +0 -1
  191. package/dist/preflight.js +0 -17
  192. package/dist/preflight.js.map +0 -1
  193. package/dist/progress.d.ts +0 -24
  194. package/dist/progress.d.ts.map +0 -1
  195. package/dist/progress.js +0 -111
  196. package/dist/progress.js.map +0 -1
  197. package/dist/record-trace.d.ts +0 -64
  198. package/dist/record-trace.d.ts.map +0 -1
  199. package/dist/record-trace.js +0 -124
  200. package/dist/record-trace.js.map +0 -1
  201. package/dist/report-preview-server.d.ts +0 -2
  202. package/dist/report-preview-server.d.ts.map +0 -1
  203. package/dist/report-preview-server.js +0 -24
  204. package/dist/report-preview-server.js.map +0 -1
  205. package/dist/report-preview.d.ts +0 -35
  206. package/dist/report-preview.d.ts.map +0 -1
  207. package/dist/report-preview.js +0 -205
  208. package/dist/report-preview.js.map +0 -1
  209. package/dist/run-suite.d.ts +0 -139
  210. package/dist/run-suite.d.ts.map +0 -1
  211. package/dist/run-suite.js +0 -1688
  212. package/dist/run-suite.js.map +0 -1
  213. package/dist/scenario-finalizer.d.ts +0 -27
  214. package/dist/scenario-finalizer.d.ts.map +0 -1
  215. package/dist/scenario-finalizer.js +0 -39
  216. package/dist/scenario-finalizer.js.map +0 -1
  217. package/dist/scenario-retry.d.ts +0 -15
  218. package/dist/scenario-retry.d.ts.map +0 -1
  219. package/dist/scenario-retry.js +0 -52
  220. package/dist/scenario-retry.js.map +0 -1
  221. package/dist/scenario-seed.d.ts +0 -13
  222. package/dist/scenario-seed.d.ts.map +0 -1
  223. package/dist/scenario-seed.js +0 -72
  224. package/dist/scenario-seed.js.map +0 -1
  225. package/dist/scenario-story.d.ts +0 -41
  226. package/dist/scenario-story.d.ts.map +0 -1
  227. package/dist/scenario-story.js +0 -459
  228. package/dist/scenario-story.js.map +0 -1
  229. package/dist/scenario-usage.d.ts +0 -15
  230. package/dist/scenario-usage.d.ts.map +0 -1
  231. package/dist/scenario-usage.js +0 -18
  232. package/dist/scenario-usage.js.map +0 -1
  233. package/dist/suite-summary.d.ts +0 -16
  234. package/dist/suite-summary.d.ts.map +0 -1
  235. package/dist/suite-summary.js +0 -218
  236. package/dist/suite-summary.js.map +0 -1
  237. package/dist/theme.d.ts +0 -96
  238. package/dist/theme.d.ts.map +0 -1
  239. package/dist/theme.js +0 -446
  240. package/dist/theme.js.map +0 -1
  241. package/dist/types.d.ts +0 -364
  242. package/dist/types.d.ts.map +0 -1
  243. package/dist/types.js.map +0 -1
  244. package/dist/validate-seeds.d.ts +0 -20
  245. package/dist/validate-seeds.d.ts.map +0 -1
  246. package/dist/validate-seeds.js +0 -72
  247. package/dist/validate-seeds.js.map +0 -1
  248. package/dist/validate-suite.d.ts +0 -24
  249. package/dist/validate-suite.d.ts.map +0 -1
  250. package/dist/validate-suite.js +0 -542
  251. package/dist/validate-suite.js.map +0 -1
  252. package/dist/viewer/catalog.d.ts +0 -72
  253. package/dist/viewer/catalog.d.ts.map +0 -1
  254. package/dist/viewer/catalog.js +0 -154
  255. package/dist/viewer/catalog.js.map +0 -1
  256. package/dist/viewer/cli.d.ts +0 -17
  257. package/dist/viewer/cli.d.ts.map +0 -1
  258. package/dist/viewer/cli.js +0 -38
  259. package/dist/viewer/cli.js.map +0 -1
  260. package/dist/viewer/client.d.ts +0 -7
  261. package/dist/viewer/client.d.ts.map +0 -1
  262. package/dist/viewer/client.js +0 -14
  263. package/dist/viewer/client.js.map +0 -1
  264. package/dist/viewer/compare-metrics.d.ts +0 -30
  265. package/dist/viewer/compare-metrics.d.ts.map +0 -1
  266. package/dist/viewer/compare-metrics.js +0 -101
  267. package/dist/viewer/compare-metrics.js.map +0 -1
  268. package/dist/viewer/context-files.d.ts +0 -8
  269. package/dist/viewer/context-files.d.ts.map +0 -1
  270. package/dist/viewer/context-files.js +0 -57
  271. package/dist/viewer/context-files.js.map +0 -1
  272. package/dist/viewer/emit.d.ts +0 -7
  273. package/dist/viewer/emit.d.ts.map +0 -1
  274. package/dist/viewer/emit.js +0 -25
  275. package/dist/viewer/emit.js.map +0 -1
  276. package/dist/viewer/events.d.ts +0 -118
  277. package/dist/viewer/events.d.ts.map +0 -1
  278. package/dist/viewer/events.js +0 -51
  279. package/dist/viewer/events.js.map +0 -1
  280. package/dist/viewer/live-runner.d.ts +0 -27
  281. package/dist/viewer/live-runner.d.ts.map +0 -1
  282. package/dist/viewer/live-runner.js +0 -166
  283. package/dist/viewer/live-runner.js.map +0 -1
  284. package/dist/viewer/run-controller.d.ts +0 -41
  285. package/dist/viewer/run-controller.d.ts.map +0 -1
  286. package/dist/viewer/run-controller.js +0 -426
  287. package/dist/viewer/run-controller.js.map +0 -1
  288. package/dist/viewer/session-server.d.ts +0 -2
  289. package/dist/viewer/session-server.d.ts.map +0 -1
  290. package/dist/viewer/session-server.js +0 -71
  291. package/dist/viewer/session-server.js.map +0 -1
  292. package/dist/warnings.d.ts +0 -3
  293. package/dist/warnings.d.ts.map +0 -1
  294. package/dist/warnings.js +0 -13
  295. package/dist/warnings.js.map +0 -1
  296. package/dist/worker-pool.d.ts +0 -8
  297. package/dist/worker-pool.d.ts.map +0 -1
  298. package/dist/worker-pool.js +0 -26
  299. package/dist/worker-pool.js.map +0 -1
  300. /package/dist/{types.js → sdk/types.js} +0 -0
@@ -0,0 +1,24 @@
1
+ export function statistics(values) {
2
+ const available = values.length > 0 &&
3
+ values.every((value) => typeof value === "number" && Number.isFinite(value));
4
+ const requireValues = () => {
5
+ if (!available)
6
+ throw new Error("Metric unavailable: every run must report this metric");
7
+ return values;
8
+ };
9
+ return {
10
+ available,
11
+ count: values.length,
12
+ get mean() {
13
+ const v = requireValues();
14
+ return v.reduce((a, b) => a + b, 0) / v.length;
15
+ },
16
+ get min() {
17
+ return Math.min(...requireValues());
18
+ },
19
+ get max() {
20
+ return Math.max(...requireValues());
21
+ },
22
+ };
23
+ }
24
+ //# sourceMappingURL=metrics.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"metrics.js","sourceRoot":"","sources":["../../src/sdk/metrics.ts"],"names":[],"mappings":"AACA,MAAM,UAAU,UAAU,CAAC,MAA8B;IACxD,MAAM,SAAS,GACd,MAAM,CAAC,MAAM,GAAG,CAAC;QACjB,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;IAC9E,MAAM,aAAa,GAAG,GAAG,EAAE;QAC1B,IAAI,CAAC,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAC;QACzF,OAAO,MAAkB,CAAC;IAC3B,CAAC,CAAC;IACF,OAAO;QACN,SAAS;QACT,KAAK,EAAE,MAAM,CAAC,MAAM;QACpB,IAAI,IAAI;YACP,MAAM,CAAC,GAAG,aAAa,EAAE,CAAC;YAC1B,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;QAChD,CAAC;QACD,IAAI,GAAG;YACN,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,aAAa,EAAE,CAAC,CAAC;QACrC,CAAC;QACD,IAAI,GAAG;YACN,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,aAAa,EAAE,CAAC,CAAC;QACrC,CAAC;KACD,CAAC;AACH,CAAC"}
@@ -0,0 +1,2 @@
1
+ export declare function playwrightCli(): string;
2
+ //# sourceMappingURL=playwright-cli.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"playwright-cli.d.ts","sourceRoot":"","sources":["../../src/sdk/playwright-cli.ts"],"names":[],"mappings":"AAIA,wBAAgB,aAAa,IAAI,MAAM,CAEtC"}
@@ -0,0 +1,6 @@
1
+ import { createRequire } from "node:module";
2
+ const require = createRequire(import.meta.url);
3
+ export function playwrightCli() {
4
+ return require.resolve("@playwright/test/cli");
5
+ }
6
+ //# sourceMappingURL=playwright-cli.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"playwright-cli.js","sourceRoot":"","sources":["../../src/sdk/playwright-cli.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5C,MAAM,OAAO,GAAG,aAAa,CAAC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC;AAE/C,MAAM,UAAU,aAAa;IAC5B,OAAO,OAAO,CAAC,OAAO,CAAC,sBAAsB,CAAC,CAAC;AAChD,CAAC"}
@@ -0,0 +1,12 @@
1
+ import type { FullConfig, Reporter, Suite, TestCase, TestResult } from "@playwright/test/reporter";
2
+ export default class AgentViewerReporter implements Reporter {
3
+ private buffers;
4
+ onBegin(_config: FullConfig, suite: Suite): void;
5
+ onStdOut(chunk: string | Buffer, test?: TestCase): void;
6
+ private emitLine;
7
+ onTestEnd(test: TestCase, result: TestResult): void;
8
+ onError(error: {
9
+ message?: string;
10
+ }): void;
11
+ }
12
+ //# sourceMappingURL=reporter.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"reporter.d.ts","sourceRoot":"","sources":["../../src/sdk/reporter.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;AAWnG,MAAM,CAAC,OAAO,OAAO,mBAAoB,YAAW,QAAQ;IAC3D,OAAO,CAAC,OAAO,CAA6B;IAC5C,OAAO,CAAC,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,KAAK,QAYxC;IACD,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,IAAI,CAAC,EAAE,QAAQ,QAK/C;IACD,OAAO,CAAC,QAAQ;IAYhB,SAAS,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,QAU3C;IACD,OAAO,CAAC,KAAK,EAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,QAElC;CACD"}
@@ -0,0 +1,98 @@
1
+ import { relative } from "node:path";
2
+ function sourceFile(test) {
3
+ let suite = test.parent;
4
+ while (suite && suite.type !== "file")
5
+ suite = suite.parent;
6
+ return suite?.location?.file ?? test.location.file;
7
+ }
8
+ const emit = (value) => process.stdout.write(`${JSON.stringify(value)}\n`);
9
+ export default class AgentViewerReporter {
10
+ buffers = new Map();
11
+ onBegin(_config, suite) {
12
+ emit({
13
+ type: "catalog",
14
+ tests: suite.allTests().map((test) => ({
15
+ ...discoveryMetadata(test),
16
+ id: test.id,
17
+ title: test.titlePath().slice(3).join(" › ") || test.title,
18
+ file: sourceFile(test),
19
+ listFile: relative(_config.rootDir, sourceFile(test)).split("\\").join("/"),
20
+ project: test.parent.project()?.name || "default",
21
+ })),
22
+ });
23
+ }
24
+ onStdOut(chunk, test) {
25
+ const id = test?.id ?? "global";
26
+ const lines = ((this.buffers.get(id) ?? "") + chunk.toString()).split("\n");
27
+ this.buffers.set(id, lines.pop() ?? "");
28
+ for (const line of lines)
29
+ this.emitLine(id, line);
30
+ }
31
+ emitLine(id, line) {
32
+ if (!line.startsWith("@@agent-test:")) {
33
+ if (line.trim())
34
+ emit({ type: "log", text: line });
35
+ return;
36
+ }
37
+ try {
38
+ emit({ type: "event", testId: id, event: JSON.parse(line.slice("@@agent-test:".length)) });
39
+ }
40
+ catch {
41
+ emit({ type: "log", text: line });
42
+ }
43
+ }
44
+ onTestEnd(test, result) {
45
+ emit({
46
+ type: "result",
47
+ testId: test.id,
48
+ passed: result.status === test.expectedStatus,
49
+ skipped: result.status === "skipped",
50
+ durationMs: result.duration,
51
+ errors: result.errors.map((error) => error.message ?? String(error)),
52
+ status: result.status,
53
+ });
54
+ }
55
+ onError(error) {
56
+ emit({ type: "error", message: error.message });
57
+ }
58
+ }
59
+ function discoveryMetadata(test) {
60
+ const use = test.parent.project()?.use;
61
+ const raw = test.annotations.find((entry) => entry.type === "agent-test.resources")?.description;
62
+ const resources = raw ? JSON.parse(raw) : [];
63
+ const criteria = parseCriteria(test.annotations.find((entry) => entry.type === "agent-test.criteria")?.description);
64
+ return {
65
+ description: test.annotations.find((entry) => entry.type === "description")?.description,
66
+ criteria,
67
+ workspace: use?.workspaceSource,
68
+ resources: resources.map((resource) => {
69
+ const definition = resource.kind === "judge" ? use?.judgeDefinition : use?.agentDefinition;
70
+ return resolveResource(resource, resource.host ? undefined : definition);
71
+ }),
72
+ };
73
+ }
74
+ function parseCriteria(raw) {
75
+ if (!raw)
76
+ return undefined;
77
+ try {
78
+ const parsed = JSON.parse(raw);
79
+ if (!Array.isArray(parsed) || !parsed.every((value) => typeof value === "string"))
80
+ return undefined;
81
+ return parsed;
82
+ }
83
+ catch {
84
+ return undefined;
85
+ }
86
+ }
87
+ function resolveResource(resource, definition) {
88
+ return {
89
+ ...resource,
90
+ skills: [...new Set([...(definition?.options.skills ?? []), ...resource.skills])],
91
+ mcpServers: [
92
+ ...new Set([...Object.keys(definition?.options.mcpServers ?? {}), ...resource.mcpServers]),
93
+ ],
94
+ host: resource.host ?? definition?.host ?? (definition?.adapter ? "custom" : undefined),
95
+ model: resource.model ?? definition?.options.model,
96
+ };
97
+ }
98
+ //# sourceMappingURL=reporter.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"reporter.js","sourceRoot":"","sources":["../../src/sdk/reporter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAMrC,SAAS,UAAU,CAAC,IAAc;IACjC,IAAI,KAAK,GAAsB,IAAI,CAAC,MAAM,CAAC;IAC3C,OAAO,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM;QAAE,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC;IAC5D,OAAO,KAAK,EAAE,QAAQ,EAAE,IAAI,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;AACpD,CAAC;AACD,MAAM,IAAI,GAAG,CAAC,KAAc,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;AACpF,MAAM,CAAC,OAAO,OAAO,mBAAmB;IAC/B,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC5C,OAAO,CAAC,OAAmB,EAAE,KAAY;QACxC,IAAI,CAAC;YACJ,IAAI,EAAE,SAAS;YACf,KAAK,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;gBACtC,GAAG,iBAAiB,CAAC,IAAI,CAAC;gBAC1B,EAAE,EAAE,IAAI,CAAC,EAAE;gBACX,KAAK,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK;gBAC1D,IAAI,EAAE,UAAU,CAAC,IAAI,CAAC;gBACtB,QAAQ,EAAE,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;gBAC3E,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,IAAI,IAAI,SAAS;aACjD,CAAC,CAAC;SACH,CAAC,CAAC;IACJ,CAAC;IACD,QAAQ,CAAC,KAAsB,EAAE,IAAe;QAC/C,MAAM,EAAE,GAAG,IAAI,EAAE,EAAE,IAAI,QAAQ,CAAC;QAChC,MAAM,KAAK,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC5E,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC;QACxC,KAAK,MAAM,IAAI,IAAI,KAAK;YAAE,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;IACnD,CAAC;IACO,QAAQ,CAAC,EAAU,EAAE,IAAY;QACxC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,eAAe,CAAC,EAAE,CAAC;YACvC,IAAI,IAAI,CAAC,IAAI,EAAE;gBAAE,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;YACnD,OAAO;QACR,CAAC;QACD,IAAI,CAAC;YACJ,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC;QAC5F,CAAC;QAAC,MAAM,CAAC;YACR,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QACnC,CAAC;IACF,CAAC;IAED,SAAS,CAAC,IAAc,EAAE,MAAkB;QAC3C,IAAI,CAAC;YACJ,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,IAAI,CAAC,EAAE;YACf,MAAM,EAAE,MAAM,CAAC,MAAM,KAAK,IAAI,CAAC,cAAc;YAC7C,OAAO,EAAE,MAAM,CAAC,MAAM,KAAK,SAAS;YACpC,UAAU,EAAE,MAAM,CAAC,QAAQ;YAC3B,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC;YACpE,MAAM,EAAE,MAAM,CAAC,MAAM;SACrB,CAAC,CAAC;IACJ,CAAC;IACD,OAAO,CAAC,KAA2B;QAClC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;IACjD,CAAC;CACD;AAED,SAAS,iBAAiB,CAAC,IAAc;IACxC,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,GAMvB,CAAC;IACb,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,sBAAsB,CAAC,EAAE,WAAW,CAAC;IACjG,MAAM,SAAS,GAAG,GAAG,CAAC,CAAC,CAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAoB,CAAC,CAAC,CAAC,EAAE,CAAC;IACjE,MAAM,QAAQ,GAAG,aAAa,CAC7B,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,qBAAqB,CAAC,EAAE,WAAW,CACnF,CAAC;IACF,OAAO;QACN,WAAW,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,aAAa,CAAC,EAAE,WAAW;QACxF,QAAQ;QACR,SAAS,EAAE,GAAG,EAAE,eAAe;QAC/B,SAAS,EAAE,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE;YACrC,MAAM,UAAU,GAAG,QAAQ,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,eAAe,CAAC,CAAC,CAAC,GAAG,EAAE,eAAe,CAAC;YAC3F,OAAO,eAAe,CAAC,QAAQ,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;QAC1E,CAAC,CAAC;KACF,CAAC;AACH,CAAC;AAED,SAAS,aAAa,CAAC,GAAY;IAClC,IAAI,CAAC,GAAG;QAAE,OAAO,SAAS,CAAC;IAC3B,IAAI,CAAC;QACJ,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACxC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC;YAChF,OAAO,SAAS,CAAC;QAClB,OAAO,MAAM,CAAC;IACf,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,SAAS,CAAC;IAClB,CAAC;AACF,CAAC;AAED,SAAS,eAAe,CAAC,QAAsB,EAAE,UAA4B;IAC5E,OAAO;QACN,GAAG,QAAQ;QACX,MAAM,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,UAAU,EAAE,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC,EAAE,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;QACjF,UAAU,EAAE;YACX,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE,OAAO,CAAC,UAAU,IAAI,EAAE,CAAC,EAAE,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAC;SAC1F;QACD,IAAI,EAAE,QAAQ,CAAC,IAAI,IAAI,UAAU,EAAE,IAAI,IAAI,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC;QACvF,KAAK,EAAE,QAAQ,CAAC,KAAK,IAAI,UAAU,EAAE,OAAO,CAAC,KAAK;KAClD,CAAC;AACH,CAAC"}
@@ -0,0 +1,33 @@
1
+ import { type AgentDefinition } from "@post-print/agent-harness";
2
+ import { z } from "zod/v4";
3
+ import type { AgentFixture, AgentSettings, JudgeFixture, JudgeSettings, WorkspaceSetup } from "./types.js";
4
+ export interface RuntimeOptions {
5
+ baseDir: string;
6
+ outputDir: string;
7
+ agent?: AgentDefinition;
8
+ judge?: AgentDefinition;
9
+ workspace: string;
10
+ signal: AbortSignal;
11
+ onEvent?: (event: {
12
+ runId: string;
13
+ type: string;
14
+ value: unknown;
15
+ }) => void;
16
+ }
17
+ export declare class TestRuntime {
18
+ readonly options: RuntimeOptions;
19
+ private readonly resources;
20
+ private readonly pending;
21
+ private readonly controller;
22
+ private closed;
23
+ constructor(options: RuntimeOptions);
24
+ private get signal();
25
+ private track;
26
+ agent(name: string, settings: AgentSettings, preparation?: readonly WorkspaceSetup[]): AgentFixture;
27
+ judge<S extends z.ZodType>(name: string, settings: JudgeSettings<S>): JudgeFixture<z.output<S>>;
28
+ private start;
29
+ private conversation;
30
+ private run;
31
+ close(): Promise<void>;
32
+ }
33
+ //# sourceMappingURL=runtime.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"runtime.d.ts","sourceRoot":"","sources":["../../src/sdk/runtime.ts"],"names":[],"mappings":"AAEA,OAAO,EACN,KAAK,eAAe,EAKpB,MAAM,2BAA2B,CAAC;AACnC,OAAO,EAAE,CAAC,EAAE,MAAM,QAAQ,CAAC;AAG3B,OAAO,KAAK,EACX,YAAY,EACZ,aAAa,EACb,YAAY,EACZ,aAAa,EAEb,cAAc,EACd,MAAM,YAAY,CAAC;AAYpB,MAAM,WAAW,cAAc;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,eAAe,CAAC;IACxB,KAAK,CAAC,EAAE,eAAe,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,WAAW,CAAC;IACpB,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,OAAO,CAAA;KAAE,KAAK,IAAI,CAAC;CAC3E;AA2DD,qBAAa,WAAW;IAKX,QAAQ,CAAC,OAAO,EAAE,cAAc;IAJ5C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAgE;IAC1F,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA+B;IACvD,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAyB;IACpD,OAAO,CAAC,MAAM,CAAS;IACvB,YAAqB,OAAO,EAAE,cAAc,EAAI;IAChD,OAAO,KAAK,MAAM,GAEjB;IACD,OAAO,CAAC,KAAK;IAUb,KAAK,CACJ,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,aAAa,EACvB,WAAW,GAAE,SAAS,cAAc,EAAO,GACzC,YAAY,CAQd;IACD,KAAK,CAAC,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,aAAa,CAAC,CAAC,CAAC,GAAG,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CA0B9F;YACa,KAAK;IAoCnB,OAAO,CAAC,YAAY;YAaN,GAAG;IAuCX,KAAK,kBAkBV;CACD"}
@@ -0,0 +1,197 @@
1
+ import { mkdir } from "node:fs/promises";
2
+ import { join } from "node:path";
3
+ import { createAgentSession, toolPathsOutsideWorkspace, } from "@post-print/agent-harness";
4
+ import { z } from "zod/v4";
5
+ import { configuredAgent, mergeSettings, validateSkills } from "./definitions.js";
6
+ import { evaluate } from "./judge.js";
7
+ import { runUsage } from "./usage.js";
8
+ import { changedPaths, prepareAgent, prepareWorkspace, snapshot, withContext, writeJson, } from "./workspace.js";
9
+ function captureRun(input) {
10
+ const { id, prompt, trace, history, durationMs, context, session, workspace, initial, final } = input;
11
+ history.messages.push(...trace.messages);
12
+ history.toolCalls.push(...trace.toolCalls);
13
+ history.shellCommands.push(...trace.shellCommands);
14
+ return {
15
+ id,
16
+ name: input.name,
17
+ artifact: input.artifact,
18
+ prompt,
19
+ trace,
20
+ conversation: structuredClone(history),
21
+ output: finalAssistantMessage(trace),
22
+ toolCalls: trace.toolCalls,
23
+ durationMs,
24
+ usage: runUsage(trace.usage),
25
+ capabilities: session.capabilities,
26
+ startingContext: context,
27
+ workspace: {
28
+ root: workspace.path,
29
+ initial,
30
+ final,
31
+ changedPaths: changedPaths(initial, final),
32
+ },
33
+ };
34
+ }
35
+ function finalAssistantMessage(trace) {
36
+ return trace.messages.filter((message) => message.role === "assistant").at(-1)?.content ?? "";
37
+ }
38
+ function assertWorkspacePaths(trace, workspace) {
39
+ const escaped = toolPathsOutsideWorkspace(trace, workspace);
40
+ if (escaped.length)
41
+ throw new Error(`Agent used paths outside the isolated workspace: ${escaped.join(", ")}`);
42
+ }
43
+ export class TestRuntime {
44
+ options;
45
+ resources = [];
46
+ pending = new Set();
47
+ controller = new AbortController();
48
+ closed = false;
49
+ constructor(options) {
50
+ this.options = options;
51
+ }
52
+ get signal() {
53
+ return AbortSignal.any([this.options.signal, this.controller.signal]);
54
+ }
55
+ track(operation) {
56
+ if (this.closed)
57
+ return Promise.reject(new Error("Test runtime is closed"));
58
+ const promise = Promise.resolve().then(operation);
59
+ this.pending.add(promise);
60
+ void promise.then(() => this.pending.delete(promise), () => this.pending.delete(promise));
61
+ return promise;
62
+ }
63
+ agent(name, settings, preparation = []) {
64
+ return {
65
+ setup: (prepare) => this.agent(name, settings, [...preparation, prepare]),
66
+ run: ({ prompt, ...options }) => this.track(() => this.start({ name, preparation, prompt }, mergeSettings(settings, options))),
67
+ };
68
+ }
69
+ judge(name, settings) {
70
+ const { prompt, schema, ...agentSettings } = settings;
71
+ return {
72
+ run: ({ input }) => this.track(async () => {
73
+ const definition = configuredAgent(this.options.judge, agentSettings);
74
+ const result = await evaluate({
75
+ name,
76
+ definition,
77
+ settings: { ...agentSettings, prompt, schema },
78
+ input,
79
+ ...this.options,
80
+ signal: this.signal,
81
+ });
82
+ this.options.onEvent?.({
83
+ runId: result.id,
84
+ type: "evaluation",
85
+ value: {
86
+ ...result,
87
+ input,
88
+ evaluation: { prompt, schema: z.toJSONSchema(schema) },
89
+ },
90
+ });
91
+ return result;
92
+ }),
93
+ };
94
+ }
95
+ async start(task, settings) {
96
+ const { name, prompt, preparation } = task;
97
+ this.signal.throwIfAborted();
98
+ if (typeof prompt !== "string" || !prompt.trim())
99
+ throw new Error("agent.run requires a nonempty prompt");
100
+ const definition = configuredAgent(this.options.agent, settings);
101
+ validateSkills(definition.options.skills ?? [], this.options.baseDir);
102
+ const workspace = await prepareWorkspace(this.options.baseDir, settings.workspace ?? this.options.workspace);
103
+ const resource = {
104
+ session: undefined,
105
+ cleanup: workspace.cleanup,
106
+ };
107
+ this.resources.push(resource);
108
+ for (const prepare of preparation)
109
+ await prepare(workspace);
110
+ const context = await prepareAgent(definition, this.options.baseDir, workspace.path);
111
+ this.signal.throwIfAborted();
112
+ resource.session = await createAgentSession({
113
+ agent: definition,
114
+ workspace: workspace.path,
115
+ signal: this.signal,
116
+ });
117
+ const history = { messages: [], toolCalls: [], shellCommands: [], artifacts: {} };
118
+ return this.conversation(name, {
119
+ session: resource.session,
120
+ workspace,
121
+ context,
122
+ history,
123
+ prompt,
124
+ });
125
+ }
126
+ conversation(name, input) {
127
+ let busy = false;
128
+ const next = async (prompt) => {
129
+ if (busy)
130
+ throw new Error("Overlapping continuations of one task are not supported");
131
+ busy = true;
132
+ try {
133
+ return await this.run(name, { ...input, prompt }, (value) => this.track(() => next(value)));
134
+ }
135
+ finally {
136
+ busy = false;
137
+ }
138
+ };
139
+ return next(input.prompt);
140
+ }
141
+ async run(name, input, continuation) {
142
+ const { session, workspace, prompt, context } = input;
143
+ const id = crypto.randomUUID(), directory = join(this.options.outputDir, id);
144
+ await mkdir(directory, { recursive: true });
145
+ const initial = await snapshot(workspace.path, join(directory, "initial"));
146
+ const start = performance.now();
147
+ this.options.onEvent?.({ runId: id, type: "start", value: { prompt, name } });
148
+ try {
149
+ const trace = await session.run(withContext(prompt, context), (event) => this.options.onEvent?.({ runId: id, type: "agent", value: event }));
150
+ assertWorkspacePaths(trace, workspace.path);
151
+ const final = await snapshot(workspace.path, join(directory, "final"));
152
+ const result = {
153
+ ...captureRun({
154
+ ...input,
155
+ name,
156
+ artifact: directory,
157
+ id,
158
+ trace,
159
+ initial,
160
+ final,
161
+ durationMs: Math.round(performance.now() - start),
162
+ }),
163
+ continue: (options) => continuation(options.prompt),
164
+ };
165
+ await writeJson(join(directory, "run.json"), result);
166
+ this.options.onEvent?.({ runId: id, type: "complete", value: result });
167
+ return result;
168
+ }
169
+ catch (error) {
170
+ await writeJson(join(directory, "error.json"), { message: String(error) });
171
+ throw error;
172
+ }
173
+ }
174
+ async close() {
175
+ this.closed = true;
176
+ this.controller.abort();
177
+ await Promise.allSettled([...this.pending]);
178
+ const errors = [];
179
+ for (const resource of this.resources.splice(0).reverse()) {
180
+ try {
181
+ await resource.session?.close();
182
+ }
183
+ catch (error) {
184
+ errors.push(error);
185
+ }
186
+ try {
187
+ await resource.cleanup();
188
+ }
189
+ catch (error) {
190
+ errors.push(error);
191
+ }
192
+ }
193
+ if (errors.length)
194
+ throw new AggregateError(errors, "Agent fixture cleanup failed");
195
+ }
196
+ }
197
+ //# sourceMappingURL=runtime.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"runtime.js","sourceRoot":"","sources":["../../src/sdk/runtime.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACzC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAGN,kBAAkB,EAElB,yBAAyB,GACzB,MAAM,2BAA2B,CAAC;AACnC,OAAO,EAAE,CAAC,EAAE,MAAM,QAAQ,CAAC;AAC3B,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAClF,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAStC,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,EACN,YAAY,EACZ,YAAY,EACZ,gBAAgB,EAEhB,QAAQ,EAER,WAAW,EACX,SAAS,GACT,MAAM,gBAAgB,CAAC;AA+BxB,SAAS,UAAU,CAAC,KAAmB;IACtC,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,GAC5F,KAAK,CAAC;IACP,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC;IACzC,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC;IAC3C,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,aAAa,CAAC,CAAC;IACnD,OAAO;QACN,EAAE;QACF,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,MAAM;QACN,KAAK;QACL,YAAY,EAAE,eAAe,CAAC,OAAO,CAAC;QACtC,MAAM,EAAE,qBAAqB,CAAC,KAAK,CAAC;QACpC,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,UAAU;QACV,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC;QAC5B,YAAY,EAAE,OAAO,CAAC,YAAY;QAClC,eAAe,EAAE,OAAO;QACxB,SAAS,EAAE;YACV,IAAI,EAAE,SAAS,CAAC,IAAI;YACpB,OAAO;YACP,KAAK;YACL,YAAY,EAAE,YAAY,CAAC,OAAO,EAAE,KAAK,CAAC;SAC1C;KACD,CAAC;AACH,CAAC;AAED,SAAS,qBAAqB,CAAC,KAAiB;IAC/C,OAAO,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,IAAI,EAAE,CAAC;AAC/F,CAAC;AACD,SAAS,oBAAoB,CAAC,KAAiB,EAAE,SAAiB;IACjE,MAAM,OAAO,GAAG,yBAAyB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IAC5D,IAAI,OAAO,CAAC,MAAM;QACjB,MAAM,IAAI,KAAK,CAAC,oDAAoD,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;AAC5F,CAAC;AAED,MAAM,OAAO,WAAW;IAKF,OAAO;IAJX,SAAS,GAA6D,EAAE,CAAC;IACzE,OAAO,GAAG,IAAI,GAAG,EAAoB,CAAC;IACtC,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IAC5C,MAAM,GAAG,KAAK,CAAC;IACvB,YAAqB,OAAuB;uBAAvB,OAAO;IAAmB,CAAC;IAChD,IAAY,MAAM;QACjB,OAAO,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC;IACvE,CAAC;IACO,KAAK,CAAI,SAA2B;QAC3C,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC,CAAC;QAC5E,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAClD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAC1B,KAAK,OAAO,CAAC,IAAI,CAChB,GAAG,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,EAClC,GAAG,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAClC,CAAC;QACF,OAAO,OAAO,CAAC;IAChB,CAAC;IACD,KAAK,CACJ,IAAY,EACZ,QAAuB,EACvB,WAAW,GAA8B,EAAE;QAE3C,OAAO;YACN,KAAK,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC,GAAG,WAAW,EAAE,OAAO,CAAC,CAAC;YACzE,GAAG,EAAE,CAAC,EAAE,MAAM,EAAE,GAAG,OAAO,EAAE,EAAE,EAAE,CAC/B,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,CACf,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,EAAE,EAAE,aAAa,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAC3E;SACF,CAAC;IACH,CAAC;IACD,KAAK,CAAsB,IAAY,EAAE,QAA0B;QAClE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,aAAa,EAAE,GAAG,QAAQ,CAAC;QACtD,OAAO;YACN,GAAG,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAClB,IAAI,CAAC,KAAK,CAAC,KAAK,IAAI,EAAE;gBACrB,MAAM,UAAU,GAAG,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,aAAa,CAAC,CAAC;gBACtE,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC;oBAC7B,IAAI;oBACJ,UAAU;oBACV,QAAQ,EAAE,EAAE,GAAG,aAAa,EAAE,MAAM,EAAE,MAAM,EAAE;oBAC9C,KAAK;oBACL,GAAG,IAAI,CAAC,OAAO;oBACf,MAAM,EAAE,IAAI,CAAC,MAAM;iBACnB,CAAC,CAAC;gBACH,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;oBACtB,KAAK,EAAE,MAAM,CAAC,EAAE;oBAChB,IAAI,EAAE,YAAY;oBAClB,KAAK,EAAE;wBACN,GAAG,MAAM;wBACT,KAAK;wBACL,UAAU,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,EAAE;qBACtD;iBACD,CAAC,CAAC;gBACH,OAAO,MAAM,CAAC;YACf,CAAC,CAAC;SACH,CAAC;IACH,CAAC;IACO,KAAK,CAAC,KAAK,CAClB,IAA8E,EAC9E,QAAuB;QAEvB,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,IAAI,CAAC;QAC3C,IAAI,CAAC,MAAM,CAAC,cAAc,EAAE,CAAC;QAC7B,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE;YAC/C,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;QACzD,MAAM,UAAU,GAAG,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QACjE,cAAc,CAAC,UAAU,CAAC,OAAO,CAAC,MAAM,IAAI,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QACtE,MAAM,SAAS,GAAG,MAAM,gBAAgB,CACvC,IAAI,CAAC,OAAO,CAAC,OAAO,EACpB,QAAQ,CAAC,SAAS,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,CAC5C,CAAC;QACF,MAAM,QAAQ,GAAG;YAChB,OAAO,EAAE,SAAuC;YAChD,OAAO,EAAE,SAAS,CAAC,OAAO;SAC1B,CAAC;QACF,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC9B,KAAK,MAAM,OAAO,IAAI,WAAW;YAAE,MAAM,OAAO,CAAC,SAAS,CAAC,CAAC;QAC5D,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC;QACrF,IAAI,CAAC,MAAM,CAAC,cAAc,EAAE,CAAC;QAC7B,QAAQ,CAAC,OAAO,GAAG,MAAM,kBAAkB,CAAC;YAC3C,KAAK,EAAE,UAAU;YACjB,SAAS,EAAE,SAAS,CAAC,IAAI;YACzB,MAAM,EAAE,IAAI,CAAC,MAAM;SACnB,CAAC,CAAC;QACH,MAAM,OAAO,GAAe,EAAE,QAAQ,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,aAAa,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC;QAC9F,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE;YAC9B,OAAO,EAAE,QAAQ,CAAC,OAAO;YACzB,SAAS;YACT,OAAO;YACP,OAAO;YACP,MAAM;SACN,CAAC,CAAC;IACJ,CAAC;IACO,YAAY,CAAC,IAAY,EAAE,KAAe;QACjD,IAAI,IAAI,GAAG,KAAK,CAAC;QACjB,MAAM,IAAI,GAAG,KAAK,EAAE,MAAc,EAAgB,EAAE;YACnD,IAAI,IAAI;gBAAE,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;YACrF,IAAI,GAAG,IAAI,CAAC;YACZ,IAAI,CAAC;gBACJ,OAAO,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,GAAG,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAC7F,CAAC;oBAAS,CAAC;gBACV,IAAI,GAAG,KAAK,CAAC;YACd,CAAC;QACF,CAAC,CAAC;QACF,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAC3B,CAAC;IACO,KAAK,CAAC,GAAG,CAChB,IAAY,EACZ,KAAe,EACf,YAA8C;QAE9C,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,KAAK,CAAC;QACtD,MAAM,EAAE,GAAG,MAAM,CAAC,UAAU,EAAE,EAC7B,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QAC9C,MAAM,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5C,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC,CAAC;QAC3E,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAChC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;QAC9E,IAAI,CAAC;YACJ,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,EAAE,CACvE,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAClE,CAAC;YACF,oBAAoB,CAAC,KAAK,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC;YAC5C,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC;YACvE,MAAM,MAAM,GAAQ;gBACnB,GAAG,UAAU,CAAC;oBACb,GAAG,KAAK;oBACR,IAAI;oBACJ,QAAQ,EAAE,SAAS;oBACnB,EAAE;oBACF,KAAK;oBACL,OAAO;oBACP,KAAK;oBACL,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;iBACjD,CAAC;gBACF,QAAQ,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,YAAY,CAAC,OAAO,CAAC,MAAM,CAAC;aACnD,CAAC;YACF,MAAM,SAAS,CAAC,IAAI,CAAC,SAAS,EAAE,UAAU,CAAC,EAAE,MAAM,CAAC,CAAC;YACrD,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;YACvE,OAAO,MAAM,CAAC;QACf,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,SAAS,CAAC,IAAI,CAAC,SAAS,EAAE,YAAY,CAAC,EAAE,EAAE,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAC3E,MAAM,KAAK,CAAC;QACb,CAAC;IACF,CAAC;IACD,KAAK,CAAC,KAAK;QACV,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;QACxB,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;QAC5C,MAAM,MAAM,GAAc,EAAE,CAAC;QAC7B,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC;YAC3D,IAAI,CAAC;gBACJ,MAAM,QAAQ,CAAC,OAAO,EAAE,KAAK,EAAE,CAAC;YACjC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAChB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACpB,CAAC;YACD,IAAI,CAAC;gBACJ,MAAM,QAAQ,CAAC,OAAO,EAAE,CAAC;YAC1B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAChB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACpB,CAAC;QACF,CAAC;QACD,IAAI,MAAM,CAAC,MAAM;YAAE,MAAM,IAAI,cAAc,CAAC,MAAM,EAAE,8BAA8B,CAAC,CAAC;IACrF,CAAC;CACD"}
@@ -0,0 +1,36 @@
1
+ import { type PlaywrightTestConfig, type TestInfo } from "@playwright/test";
2
+ import type { AgentDefinition } from "@post-print/agent-harness";
3
+ import type { z } from "zod/v4";
4
+ import { type AgentResource, factories, type JudgeResource, type Resource } from "./definitions.js";
5
+ import type { AgentFixture, JudgeFixture } from "./types.js";
6
+ interface Defaults {
7
+ agentDefinition?: AgentDefinition;
8
+ judgeDefinition?: AgentDefinition;
9
+ workspaceSource: string;
10
+ }
11
+ export type Resources = Record<string, Resource>;
12
+ export type Fixtures<R extends Resources> = {
13
+ [K in keyof R]: R[K] extends JudgeResource<infer S> ? JudgeFixture<z.output<S>> : R[K] extends AgentResource ? AgentFixture : never;
14
+ };
15
+ type TestBody<R extends Resources> = (fixtures: Fixtures<R>, info: TestInfo) => Promise<void> | void;
16
+ export type TestDetails<R extends Resources = Resources> = {
17
+ description: string;
18
+ criteria?: readonly string[];
19
+ resources?: readonly Extract<keyof R, string>[];
20
+ };
21
+ export interface SuiteTest<R extends Resources> {
22
+ (title: string, body: TestBody<R>): void;
23
+ (title: string, details: TestDetails<R>, body: TestBody<R>): void;
24
+ }
25
+ export declare function describe<R extends Resources>(name: string, define: (builders: typeof factories) => R): SuiteTest<R>;
26
+ interface Configuration {
27
+ agent?: AgentDefinition;
28
+ judge?: AgentDefinition;
29
+ workspace?: string;
30
+ }
31
+ export type AgentTestConfig = Omit<PlaywrightTestConfig<Defaults>, "use" | "projects"> & Configuration & {
32
+ projects?: (Omit<NonNullable<PlaywrightTestConfig<Defaults>["projects"]>[number], "use"> & Configuration)[];
33
+ };
34
+ export declare function defineConfig(config: AgentTestConfig): PlaywrightTestConfig<Defaults>;
35
+ export {};
36
+ //# sourceMappingURL=test.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"test.d.ts","sourceRoot":"","sources":["../../src/sdk/test.ts"],"names":[],"mappings":"AAEA,OAAO,EAEN,KAAK,oBAAoB,EAEzB,KAAK,QAAQ,EACb,MAAM,kBAAkB,CAAC;AAC1B,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,2BAA2B,CAAC;AACjE,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,QAAQ,CAAC;AAChC,OAAO,EAAE,KAAK,aAAa,EAAE,SAAS,EAAE,KAAK,aAAa,EAAE,KAAK,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAEpG,OAAO,KAAK,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAE7D,UAAU,QAAQ;IACjB,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,eAAe,EAAE,MAAM,CAAC;CACxB;AA6CD,MAAM,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;AACjD,MAAM,MAAM,QAAQ,CAAC,CAAC,SAAS,SAAS,IAAI;KAC1C,CAAC,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,aAAa,CAAC,MAAM,CAAC,CAAC,GAChD,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GACzB,CAAC,CAAC,CAAC,CAAC,SAAS,aAAa,GACzB,YAAY,GACZ,KAAK;CACT,CAAC;AACF,KAAK,QAAQ,CAAC,CAAC,SAAS,SAAS,IAAI,CACpC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC,EACrB,IAAI,EAAE,QAAQ,KACV,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;AAC1B,MAAM,MAAM,WAAW,CAAC,CAAC,SAAS,SAAS,GAAG,SAAS,IAAI;IAC1D,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAC7B,SAAS,CAAC,EAAE,SAAS,OAAO,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,EAAE,CAAC;CAChD,CAAC;AACF,MAAM,WAAW,SAAS,CAAC,CAAC,SAAS,SAAS;IAC7C,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;IACzC,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;CAClE;AAWD,wBAAgB,QAAQ,CAAC,CAAC,SAAS,SAAS,EAC3C,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,CAAC,QAAQ,EAAE,OAAO,SAAS,KAAK,CAAC,GACvC,SAAS,CAAC,CAAC,CAAC,CAoBd;AAcD,UAAU,aAAa;IACtB,KAAK,CAAC,EAAE,eAAe,CAAC;IACxB,KAAK,CAAC,EAAE,eAAe,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;CACnB;AACD,MAAM,MAAM,eAAe,GAAG,IAAI,CAAC,oBAAoB,CAAC,QAAQ,CAAC,EAAE,KAAK,GAAG,UAAU,CAAC,GACrF,aAAa,GAAG;IACf,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,oBAAoB,CAAC,QAAQ,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,KAAK,CAAC,GACvF,aAAa,CAAC,EAAE,CAAC;CAClB,CAAC;AAWH,wBAAgB,YAAY,CAAC,MAAM,EAAE,eAAe,GAAG,oBAAoB,CAAC,QAAQ,CAAC,CAUpF"}
@@ -0,0 +1,133 @@
1
+ import { appendFile, mkdir } from "node:fs/promises";
2
+ import { dirname, join } from "node:path";
3
+ import { test as base, defineConfig as playwrightConfig, } from "@playwright/test";
4
+ import { factories } from "./definitions.js";
5
+ import { TestRuntime } from "./runtime.js";
6
+ const runner = base.extend({
7
+ agentDefinition: [undefined, { option: true }],
8
+ judgeDefinition: [undefined, { option: true }],
9
+ workspaceSource: [".", { option: true }],
10
+ runtime: async ({ agentDefinition, judgeDefinition, workspaceSource }, use, info) => {
11
+ const owner = await runtimeOwner({ agentDefinition, judgeDefinition, workspaceSource }, info);
12
+ try {
13
+ await use(owner.runtime);
14
+ }
15
+ finally {
16
+ await owner.close();
17
+ }
18
+ },
19
+ });
20
+ async function runtimeOwner(defaults, info) {
21
+ const outputDir = info.outputPath("agent-runs"), path = join(outputDir, "events.ndjson");
22
+ await mkdir(outputDir, { recursive: true });
23
+ await appendFile(path, "");
24
+ let writes = Promise.resolve();
25
+ const runtime = new TestRuntime({
26
+ baseDir: dirname(info.config.configFile || join(process.cwd(), "agent-test.config.ts")),
27
+ outputDir,
28
+ agent: defaults.agentDefinition,
29
+ judge: defaults.judgeDefinition,
30
+ workspace: defaults.workspaceSource,
31
+ signal: new AbortController().signal,
32
+ onEvent: (event) => {
33
+ writes = writes.then(() => appendFile(path, `${JSON.stringify(event)}\n`));
34
+ if (process.env.AGENT_TEST_VIEWER_EVENTS === "1" || process.env.AGENT_TEST_RECORDER === "1")
35
+ process.stdout.write(`@@agent-test:${JSON.stringify(event)}\n`);
36
+ },
37
+ });
38
+ return {
39
+ runtime,
40
+ close: async () => {
41
+ try {
42
+ await runtime.close();
43
+ }
44
+ finally {
45
+ await writes;
46
+ await info.attach("agent-events", { path, contentType: "application/x-ndjson" });
47
+ }
48
+ },
49
+ };
50
+ }
51
+ function bind(resources, runtime) {
52
+ return Object.fromEntries(Object.entries(resources).map(([name, resource]) => [
53
+ name,
54
+ resource.kind === "agent"
55
+ ? runtime.agent(name, resource.settings, resource.preparation)
56
+ : runtime.judge(name, resource.settings),
57
+ ]));
58
+ }
59
+ export function describe(name, define) {
60
+ const resources = define(factories);
61
+ if (!resources ||
62
+ Object.values(resources).some((resource) => !resource || !["agent", "judge"].includes(resource.kind)))
63
+ throw new Error("describe must return named agent() and judge() resources");
64
+ return (title, detailsOrBody, body) => {
65
+ const callback = typeof detailsOrBody === "function" ? detailsOrBody : body;
66
+ if (!callback)
67
+ throw new Error("Test body is required");
68
+ const details = typeof detailsOrBody === "function" ? undefined : detailsOrBody;
69
+ const annotation = testAnnotations(resources, details);
70
+ runner.describe(name, () => {
71
+ runner(title, { annotation }, async ({ runtime }, info) => {
72
+ await callback(bind(resources, runtime), info);
73
+ });
74
+ });
75
+ };
76
+ }
77
+ function testAnnotations(resources, details) {
78
+ return [
79
+ {
80
+ type: "agent-test.resources",
81
+ description: JSON.stringify(resourceMetadata(resources, details?.resources)),
82
+ },
83
+ ...(details?.description ? [{ type: "description", description: details.description }] : []),
84
+ ...(details?.criteria?.length
85
+ ? [{ type: "agent-test.criteria", description: JSON.stringify(details.criteria) }]
86
+ : []),
87
+ ];
88
+ }
89
+ function translate({ agent, judge, workspace, ...rest }) {
90
+ return {
91
+ ...rest,
92
+ use: {
93
+ ...(agent ? { agentDefinition: agent } : {}),
94
+ ...(judge ? { judgeDefinition: judge } : {}),
95
+ ...(workspace ? { workspaceSource: workspace } : {}),
96
+ },
97
+ };
98
+ }
99
+ export function defineConfig(config) {
100
+ const { projects, ...root } = config;
101
+ return playwrightConfig({
102
+ timeout: 180_000,
103
+ retries: 0,
104
+ workers: 1,
105
+ fullyParallel: true,
106
+ ...translate(root),
107
+ projects: projects?.map(translate),
108
+ });
109
+ }
110
+ function resourceMetadata(resources, selected) {
111
+ const names = selected ?? Object.keys(resources);
112
+ return names.map((name) => {
113
+ const resource = resources[name];
114
+ if (!resource)
115
+ throw new Error(`Unknown test resource: ${name}`);
116
+ return resourceMetadataEntry(name, resource);
117
+ });
118
+ }
119
+ function resourceMetadataEntry(name, resource) {
120
+ const { settings } = resource;
121
+ return {
122
+ name,
123
+ kind: resource.kind,
124
+ description: settings.description,
125
+ host: settings.agent?.host ?? (settings.agent?.adapter ? "custom" : undefined),
126
+ model: settings.model ?? settings.agent?.options.model,
127
+ skills: [...(settings.agent?.options.skills ?? []), ...(settings.skills ?? [])],
128
+ mcpServers: Object.keys({ ...settings.agent?.options.mcpServers, ...settings.mcpServers }),
129
+ workspace: resource.kind === "agent" ? resource.settings.workspace : undefined,
130
+ prompt: resource.kind === "judge" ? resource.settings.prompt : undefined,
131
+ };
132
+ }
133
+ //# sourceMappingURL=test.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"test.js","sourceRoot":"","sources":["../../src/sdk/test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EACN,IAAI,IAAI,IAAI,EAEZ,YAAY,IAAI,gBAAgB,GAEhC,MAAM,kBAAkB,CAAC;AAG1B,OAAO,EAAsB,SAAS,EAAqC,MAAM,kBAAkB,CAAC;AACpG,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAQ3C,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAsC;IAC/D,eAAe,EAAE,CAAC,SAAS,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IAC9C,eAAe,EAAE,CAAC,SAAS,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IAC9C,eAAe,EAAE,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IACxC,OAAO,EAAE,KAAK,EAAE,EAAE,eAAe,EAAE,eAAe,EAAE,eAAe,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE;QACnF,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,EAAE,eAAe,EAAE,eAAe,EAAE,eAAe,EAAE,EAAE,IAAI,CAAC,CAAC;QAC9F,IAAI,CAAC;YACJ,MAAM,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC1B,CAAC;gBAAS,CAAC;YACV,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC;IACF,CAAC;CACD,CAAC,CAAC;AACH,KAAK,UAAU,YAAY,CAAC,QAAkB,EAAE,IAAc;IAC7D,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,EAC9C,IAAI,GAAG,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;IACzC,MAAM,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC5C,MAAM,UAAU,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IAC3B,IAAI,MAAM,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IAC/B,MAAM,OAAO,GAAG,IAAI,WAAW,CAAC;QAC/B,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,sBAAsB,CAAC,CAAC;QACvF,SAAS;QACT,KAAK,EAAE,QAAQ,CAAC,eAAe;QAC/B,KAAK,EAAE,QAAQ,CAAC,eAAe;QAC/B,SAAS,EAAE,QAAQ,CAAC,eAAe;QACnC,MAAM,EAAE,IAAI,eAAe,EAAE,CAAC,MAAM;QACpC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;YAClB,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;YAC3E,IAAI,OAAO,CAAC,GAAG,CAAC,wBAAwB,KAAK,GAAG,IAAI,OAAO,CAAC,GAAG,CAAC,mBAAmB,KAAK,GAAG;gBAC1F,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,gBAAgB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAClE,CAAC;KACD,CAAC,CAAC;IACH,OAAO;QACN,OAAO;QACP,KAAK,EAAE,KAAK,IAAI,EAAE;YACjB,IAAI,CAAC;gBACJ,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;YACvB,CAAC;oBAAS,CAAC;gBACV,MAAM,MAAM,CAAC;gBACb,MAAM,IAAI,CAAC,MAAM,CAAC,cAAc,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,sBAAsB,EAAE,CAAC,CAAC;YAClF,CAAC;QACF,CAAC;KACD,CAAC;AACH,CAAC;AAsBD,SAAS,IAAI,CAAsB,SAAY,EAAE,OAAoB;IACpE,OAAO,MAAM,CAAC,WAAW,CACxB,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,QAAQ,CAAC,EAAE,EAAE,CAAC;QACnD,IAAI;QACJ,QAAQ,CAAC,IAAI,KAAK,OAAO;YACxB,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,WAAW,CAAC;YAC9D,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,QAAQ,CAAC;KACzC,CAAC,CACa,CAAC;AAClB,CAAC;AACD,MAAM,UAAU,QAAQ,CACvB,IAAY,EACZ,MAAyC;IAEzC,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC;IACpC,IACC,CAAC,SAAS;QACV,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,IAAI,CAC5B,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CACtE;QAED,MAAM,IAAI,KAAK,CAAC,0DAA0D,CAAC,CAAC;IAC7E,OAAO,CAAC,KAAK,EAAE,aAA2C,EAAE,IAAkB,EAAE,EAAE;QACjF,MAAM,QAAQ,GAAG,OAAO,aAAa,KAAK,UAAU,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC;QAC5E,IAAI,CAAC,QAAQ;YAAE,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;QACxD,MAAM,OAAO,GAAG,OAAO,aAAa,KAAK,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,aAAa,CAAC;QAChF,MAAM,UAAU,GAAG,eAAe,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QACvD,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,GAAG,EAAE;YAC1B,MAAM,CAAC,KAAK,EAAE,EAAE,UAAU,EAAE,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,EAAE;gBACzD,MAAM,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,EAAE,IAAI,CAAC,CAAC;YAChD,CAAC,CAAC,CAAC;QACJ,CAAC,CAAC,CAAC;IACJ,CAAC,CAAC;AACH,CAAC;AAED,SAAS,eAAe,CAAsB,SAAY,EAAE,OAAwB;IACnF,OAAO;QACN;YACC,IAAI,EAAE,sBAAsB;YAC5B,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,gBAAgB,CAAC,SAAS,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;SAC5E;QACD,GAAG,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,WAAW,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5F,GAAG,CAAC,OAAO,EAAE,QAAQ,EAAE,MAAM;YAC5B,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,qBAAqB,EAAE,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;YAClF,CAAC,CAAC,EAAE,CAAC;KACN,CAAC;AACH,CAAC;AAWD,SAAS,SAAS,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,IAAI,EAAiB;IACrE,OAAO;QACN,GAAG,IAAI;QACP,GAAG,EAAE;YACJ,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC5C,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC5C,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACpD;KACD,CAAC;AACH,CAAC;AACD,MAAM,UAAU,YAAY,CAAC,MAAuB;IACnD,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,EAAE,GAAG,MAAM,CAAC;IACrC,OAAO,gBAAgB,CAAC;QACvB,OAAO,EAAE,OAAO;QAChB,OAAO,EAAE,CAAC;QACV,OAAO,EAAE,CAAC;QACV,aAAa,EAAE,IAAI;QACnB,GAAG,SAAS,CAAC,IAAI,CAAC;QAClB,QAAQ,EAAE,QAAQ,EAAE,GAAG,CAAC,SAAS,CAAC;KAClC,CAAC,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CACxB,SAAY,EACZ,QAA8C;IAE9C,MAAM,KAAK,GAAG,QAAQ,IAAK,MAAM,CAAC,IAAI,CAAC,SAAS,CAAgC,CAAC;IACjF,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACzB,MAAM,QAAQ,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;QACjC,IAAI,CAAC,QAAQ;YAAE,MAAM,IAAI,KAAK,CAAC,0BAA0B,IAAI,EAAE,CAAC,CAAC;QACjE,OAAO,qBAAqB,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAC9C,CAAC,CAAC,CAAC;AACJ,CAAC;AACD,SAAS,qBAAqB,CAAC,IAAY,EAAE,QAAkB;IAC9D,MAAM,EAAE,QAAQ,EAAE,GAAG,QAAQ,CAAC;IAC9B,OAAO;QACN,IAAI;QACJ,IAAI,EAAE,QAAQ,CAAC,IAAI;QACnB,WAAW,EAAE,QAAQ,CAAC,WAAW;QACjC,IAAI,EAAE,QAAQ,CAAC,KAAK,EAAE,IAAI,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC;QAC9E,KAAK,EAAE,QAAQ,CAAC,KAAK,IAAI,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK;QACtD,MAAM,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,QAAQ,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC;QAC/E,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,UAAU,EAAE,GAAG,QAAQ,CAAC,UAAU,EAAE,CAAC;QAC1F,SAAS,EAAE,QAAQ,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;QAC9E,MAAM,EAAE,QAAQ,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;KACxE,CAAC;AACH,CAAC"}