@xoxoai/checkmate 0.4.27 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (397) hide show
  1. package/README.md +259 -224
  2. package/bin/checkmate.js +1 -1
  3. package/dist/ai/client.d.ts +45 -30
  4. package/dist/ai/client.d.ts.map +1 -1
  5. package/dist/ai/client.js +198 -129
  6. package/dist/ai/client.js.map +1 -1
  7. package/dist/ai/message-handler.d.ts +5 -7
  8. package/dist/ai/message-handler.d.ts.map +1 -1
  9. package/dist/ai/message-handler.js +38 -21
  10. package/dist/ai/message-handler.js.map +1 -1
  11. package/dist/ai/message-history.d.ts +4 -6
  12. package/dist/ai/message-history.d.ts.map +1 -1
  13. package/dist/ai/message-history.js +18 -21
  14. package/dist/ai/message-history.js.map +1 -1
  15. package/dist/ai/rate-limit-policy.d.ts +5 -3
  16. package/dist/ai/rate-limit-policy.d.ts.map +1 -1
  17. package/dist/ai/rate-limit-policy.js +24 -7
  18. package/dist/ai/rate-limit-policy.js.map +1 -1
  19. package/dist/ai/tool-response-handler.d.ts +11 -16
  20. package/dist/ai/tool-response-handler.d.ts.map +1 -1
  21. package/dist/ai/tool-response-handler.js +59 -28
  22. package/dist/ai/tool-response-handler.js.map +1 -1
  23. package/dist/ai/turn-processor.d.ts +34 -0
  24. package/dist/ai/turn-processor.d.ts.map +1 -0
  25. package/dist/ai/turn-processor.js +107 -0
  26. package/dist/ai/turn-processor.js.map +1 -0
  27. package/dist/api/allocate-run-identity.d.ts +8 -0
  28. package/dist/api/allocate-run-identity.d.ts.map +1 -0
  29. package/dist/api/allocate-run-identity.js +10 -0
  30. package/dist/api/allocate-run-identity.js.map +1 -0
  31. package/dist/api/execute-prepared-run.d.ts +22 -0
  32. package/dist/api/execute-prepared-run.d.ts.map +1 -0
  33. package/dist/api/execute-prepared-run.js +160 -0
  34. package/dist/api/execute-prepared-run.js.map +1 -0
  35. package/dist/api/index.d.ts +25 -0
  36. package/dist/api/index.d.ts.map +1 -0
  37. package/dist/api/index.js +69 -0
  38. package/dist/api/index.js.map +1 -0
  39. package/dist/api/prepare-run.d.ts +44 -0
  40. package/dist/api/prepare-run.d.ts.map +1 -0
  41. package/dist/api/prepare-run.js +261 -0
  42. package/dist/api/prepare-run.js.map +1 -0
  43. package/dist/cli/diagnostic-relay.d.ts +6 -0
  44. package/dist/cli/diagnostic-relay.d.ts.map +1 -0
  45. package/dist/cli/diagnostic-relay.js +6 -0
  46. package/dist/cli/diagnostic-relay.js.map +1 -0
  47. package/dist/cli/main.d.ts +21 -0
  48. package/dist/cli/main.d.ts.map +1 -0
  49. package/dist/cli/main.js +166 -0
  50. package/dist/cli/main.js.map +1 -0
  51. package/dist/cli/protocol.d.ts +57 -0
  52. package/dist/cli/protocol.d.ts.map +1 -0
  53. package/dist/cli/protocol.js +116 -0
  54. package/dist/cli/protocol.js.map +1 -0
  55. package/dist/cli/reconcile-result.d.ts +13 -0
  56. package/dist/cli/reconcile-result.d.ts.map +1 -0
  57. package/dist/cli/reconcile-result.js +300 -0
  58. package/dist/cli/reconcile-result.js.map +1 -0
  59. package/dist/cli/run-parent.d.ts +37 -0
  60. package/dist/cli/run-parent.d.ts.map +1 -0
  61. package/dist/cli/run-parent.js +491 -0
  62. package/dist/cli/run-parent.js.map +1 -0
  63. package/dist/cli/signals.d.ts +36 -0
  64. package/dist/cli/signals.d.ts.map +1 -0
  65. package/dist/cli/signals.js +85 -0
  66. package/dist/cli/signals.js.map +1 -0
  67. package/dist/cli/static-commands.d.ts +11 -0
  68. package/dist/cli/static-commands.d.ts.map +1 -0
  69. package/dist/cli/static-commands.js +83 -0
  70. package/dist/cli/static-commands.js.map +1 -0
  71. package/dist/cli/worker-stream-guard.d.ts +20 -0
  72. package/dist/cli/worker-stream-guard.d.ts.map +1 -0
  73. package/dist/cli/worker-stream-guard.js +41 -0
  74. package/dist/cli/worker-stream-guard.js.map +1 -0
  75. package/dist/cli/worker.d.ts +3 -0
  76. package/dist/cli/worker.d.ts.map +1 -0
  77. package/dist/cli/worker.js +111 -0
  78. package/dist/cli/worker.js.map +1 -0
  79. package/dist/config/ingestion.d.ts +58 -0
  80. package/dist/config/ingestion.d.ts.map +1 -0
  81. package/dist/config/ingestion.js +283 -0
  82. package/dist/config/ingestion.js.map +1 -0
  83. package/dist/config/manifest.d.ts +20 -0
  84. package/dist/config/manifest.d.ts.map +1 -0
  85. package/dist/config/manifest.js +91 -0
  86. package/dist/config/manifest.js.map +1 -0
  87. package/dist/config/model-egress.d.ts +10 -0
  88. package/dist/config/model-egress.d.ts.map +1 -0
  89. package/dist/config/model-egress.js +62 -0
  90. package/dist/config/model-egress.js.map +1 -0
  91. package/dist/config/package-resolution.d.ts +4 -0
  92. package/dist/config/package-resolution.d.ts.map +1 -0
  93. package/dist/config/package-resolution.js +24 -0
  94. package/dist/config/package-resolution.js.map +1 -0
  95. package/dist/config/policy.d.ts +44 -0
  96. package/dist/config/policy.d.ts.map +1 -0
  97. package/dist/config/policy.js +141 -0
  98. package/dist/config/policy.js.map +1 -0
  99. package/dist/config/record.d.ts +4 -0
  100. package/dist/config/record.d.ts.map +1 -0
  101. package/dist/config/record.js +10 -0
  102. package/dist/config/record.js.map +1 -0
  103. package/dist/config/secrets.d.ts +11 -0
  104. package/dist/config/secrets.d.ts.map +1 -0
  105. package/dist/config/secrets.js +42 -0
  106. package/dist/config/secrets.js.map +1 -0
  107. package/dist/contracts/diagnostics.d.ts +4 -0
  108. package/dist/contracts/diagnostics.d.ts.map +1 -0
  109. package/dist/contracts/diagnostics.js +31 -0
  110. package/dist/contracts/diagnostics.js.map +1 -0
  111. package/dist/contracts/serialize.d.ts +2 -0
  112. package/dist/contracts/serialize.d.ts.map +1 -0
  113. package/dist/contracts/serialize.js +90 -0
  114. package/dist/contracts/serialize.js.map +1 -0
  115. package/dist/contracts/types.d.ts +316 -0
  116. package/dist/contracts/types.d.ts.map +1 -0
  117. package/dist/contracts/types.js +74 -0
  118. package/dist/contracts/types.js.map +1 -0
  119. package/dist/contracts/validator.d.ts +8 -0
  120. package/dist/contracts/validator.d.ts.map +1 -0
  121. package/dist/contracts/validator.js +71 -0
  122. package/dist/contracts/validator.js.map +1 -0
  123. package/dist/driver.d.ts +97 -0
  124. package/dist/driver.d.ts.map +1 -0
  125. package/dist/driver.js +24 -0
  126. package/dist/driver.js.map +1 -0
  127. package/dist/drivers/descriptor.d.ts +15 -0
  128. package/dist/drivers/descriptor.d.ts.map +1 -0
  129. package/dist/drivers/descriptor.js +122 -0
  130. package/dist/drivers/descriptor.js.map +1 -0
  131. package/dist/drivers/loader.d.ts +18 -0
  132. package/dist/drivers/loader.d.ts.map +1 -0
  133. package/dist/drivers/loader.js +97 -0
  134. package/dist/drivers/loader.js.map +1 -0
  135. package/dist/drivers/web/checkmate-driver.json +44 -0
  136. package/dist/drivers/web/index.d.ts +6 -0
  137. package/dist/drivers/web/index.d.ts.map +1 -0
  138. package/dist/drivers/web/index.js +27 -0
  139. package/dist/drivers/web/index.js.map +1 -0
  140. package/dist/drivers/web/session.d.ts +22 -0
  141. package/dist/drivers/web/session.d.ts.map +1 -0
  142. package/dist/drivers/web/session.js +130 -0
  143. package/dist/drivers/web/session.js.map +1 -0
  144. package/dist/drivers/web/tools/network-request-recorder.d.ts +46 -0
  145. package/dist/drivers/web/tools/network-request-recorder.d.ts.map +1 -0
  146. package/dist/drivers/web/tools/network-request-recorder.js +164 -0
  147. package/dist/drivers/web/tools/network-request-recorder.js.map +1 -0
  148. package/dist/{tools/browser → drivers/web/tools}/screenshot-service.d.ts +1 -1
  149. package/dist/drivers/web/tools/screenshot-service.d.ts.map +1 -0
  150. package/dist/drivers/web/tools/screenshot-service.js.map +1 -0
  151. package/dist/drivers/web/tools/snapshot-filter/index.d.ts.map +1 -0
  152. package/dist/drivers/web/tools/snapshot-filter/index.js.map +1 -0
  153. package/dist/drivers/web/tools/snapshot-filter/semantic-scorer.d.ts.map +1 -0
  154. package/dist/drivers/web/tools/snapshot-filter/semantic-scorer.js.map +1 -0
  155. package/dist/drivers/web/tools/snapshot-filter/snapshot-filter.d.ts +5 -0
  156. package/dist/drivers/web/tools/snapshot-filter/snapshot-filter.d.ts.map +1 -0
  157. package/dist/drivers/web/tools/snapshot-filter/snapshot-filter.js +56 -0
  158. package/dist/drivers/web/tools/snapshot-filter/snapshot-filter.js.map +1 -0
  159. package/dist/drivers/web/tools/snapshot-filter/tree-reconstructor.d.ts.map +1 -0
  160. package/dist/drivers/web/tools/snapshot-filter/tree-reconstructor.js.map +1 -0
  161. package/dist/drivers/web/tools/snapshot-service.d.ts +31 -0
  162. package/dist/drivers/web/tools/snapshot-service.d.ts.map +1 -0
  163. package/dist/{tools/browser → drivers/web/tools}/snapshot-service.js +36 -7
  164. package/dist/drivers/web/tools/snapshot-service.js.map +1 -0
  165. package/dist/drivers/web/tools/tool.d.ts +76 -0
  166. package/dist/drivers/web/tools/tool.d.ts.map +1 -0
  167. package/dist/drivers/web/tools/tool.js +562 -0
  168. package/dist/drivers/web/tools/tool.js.map +1 -0
  169. package/dist/drivers/web/tools/transient-state-tracker.d.ts +26 -0
  170. package/dist/drivers/web/tools/transient-state-tracker.d.ts.map +1 -0
  171. package/dist/{tools/browser → drivers/web/tools}/transient-state-tracker.js +54 -19
  172. package/dist/drivers/web/tools/transient-state-tracker.js.map +1 -0
  173. package/dist/drivers/web/tools/types.d.ts +11 -0
  174. package/dist/drivers/web/tools/types.d.ts.map +1 -0
  175. package/dist/drivers/web/tools/types.js +2 -0
  176. package/dist/drivers/web/tools/types.js.map +1 -0
  177. package/dist/evidence/atomic-file.d.ts +23 -0
  178. package/dist/evidence/atomic-file.d.ts.map +1 -0
  179. package/dist/evidence/atomic-file.js +104 -0
  180. package/dist/evidence/atomic-file.js.map +1 -0
  181. package/dist/evidence/layout.d.ts +30 -0
  182. package/dist/evidence/layout.d.ts.map +1 -0
  183. package/dist/evidence/layout.js +152 -0
  184. package/dist/evidence/layout.js.map +1 -0
  185. package/dist/evidence/retention.d.ts +5 -0
  186. package/dist/evidence/retention.d.ts.map +1 -0
  187. package/dist/evidence/retention.js +10 -0
  188. package/dist/evidence/retention.js.map +1 -0
  189. package/dist/evidence/store.d.ts +101 -0
  190. package/dist/evidence/store.d.ts.map +1 -0
  191. package/dist/evidence/store.js +375 -0
  192. package/dist/evidence/store.js.map +1 -0
  193. package/dist/evidence/terminal-finalizer.d.ts +19 -0
  194. package/dist/evidence/terminal-finalizer.d.ts.map +1 -0
  195. package/dist/evidence/terminal-finalizer.js +56 -0
  196. package/dist/evidence/terminal-finalizer.js.map +1 -0
  197. package/dist/index.d.ts +8 -1
  198. package/dist/index.d.ts.map +1 -1
  199. package/dist/index.js +11 -1
  200. package/dist/index.js.map +1 -1
  201. package/dist/logging/invocation-logger.d.ts +4 -0
  202. package/dist/logging/invocation-logger.d.ts.map +1 -0
  203. package/dist/logging/invocation-logger.js +25 -0
  204. package/dist/logging/invocation-logger.js.map +1 -0
  205. package/dist/logging/types.d.ts +8 -0
  206. package/dist/logging/types.d.ts.map +1 -0
  207. package/dist/logging/types.js +7 -0
  208. package/dist/logging/types.js.map +1 -0
  209. package/dist/redaction/diagnostic-sanitizer.d.ts +7 -0
  210. package/dist/redaction/diagnostic-sanitizer.d.ts.map +1 -0
  211. package/dist/redaction/diagnostic-sanitizer.js +42 -0
  212. package/dist/redaction/diagnostic-sanitizer.js.map +1 -0
  213. package/dist/redaction/redactor.d.ts +40 -0
  214. package/dist/redaction/redactor.d.ts.map +1 -0
  215. package/dist/redaction/redactor.js +176 -0
  216. package/dist/redaction/redactor.js.map +1 -0
  217. package/dist/redaction/scrub.d.ts +29 -0
  218. package/dist/redaction/scrub.d.ts.map +1 -0
  219. package/dist/redaction/scrub.js +54 -0
  220. package/dist/redaction/scrub.js.map +1 -0
  221. package/dist/runtime/config.d.ts +15 -0
  222. package/dist/runtime/config.d.ts.map +1 -0
  223. package/dist/runtime/config.js +2 -0
  224. package/dist/runtime/config.js.map +1 -0
  225. package/dist/runtime/driver-boundary.d.ts +16 -0
  226. package/dist/runtime/driver-boundary.d.ts.map +1 -0
  227. package/dist/runtime/driver-boundary.js +71 -0
  228. package/dist/runtime/driver-boundary.js.map +1 -0
  229. package/dist/runtime/driver-session.d.ts +9 -0
  230. package/dist/runtime/driver-session.d.ts.map +1 -0
  231. package/dist/runtime/driver-session.js +64 -0
  232. package/dist/runtime/driver-session.js.map +1 -0
  233. package/dist/runtime/internal-step-evidence.d.ts +30 -0
  234. package/dist/runtime/internal-step-evidence.d.ts.map +1 -0
  235. package/dist/runtime/internal-step-evidence.js +82 -0
  236. package/dist/runtime/internal-step-evidence.js.map +1 -0
  237. package/dist/runtime/result-adapter.d.ts +9 -0
  238. package/dist/runtime/result-adapter.d.ts.map +1 -0
  239. package/dist/runtime/result-adapter.js +77 -0
  240. package/dist/runtime/result-adapter.js.map +1 -0
  241. package/dist/runtime/runner.d.ts +35 -88
  242. package/dist/runtime/runner.d.ts.map +1 -1
  243. package/dist/runtime/runner.js +60 -74
  244. package/dist/runtime/runner.js.map +1 -1
  245. package/dist/runtime/scenario-control.d.ts +40 -0
  246. package/dist/runtime/scenario-control.d.ts.map +1 -0
  247. package/dist/runtime/scenario-control.js +94 -0
  248. package/dist/runtime/scenario-control.js.map +1 -0
  249. package/dist/runtime/scenario-runner.d.ts +39 -0
  250. package/dist/runtime/scenario-runner.d.ts.map +1 -0
  251. package/dist/runtime/scenario-runner.js +273 -0
  252. package/dist/runtime/scenario-runner.js.map +1 -0
  253. package/dist/runtime/scenario-state.d.ts +55 -0
  254. package/dist/runtime/scenario-state.d.ts.map +1 -0
  255. package/dist/runtime/scenario-state.js +128 -0
  256. package/dist/runtime/scenario-state.js.map +1 -0
  257. package/dist/runtime/step-execution.d.ts +24 -14
  258. package/dist/runtime/step-execution.d.ts.map +1 -1
  259. package/dist/runtime/step-execution.js +130 -36
  260. package/dist/runtime/step-execution.js.map +1 -1
  261. package/dist/runtime/text.d.ts +10 -0
  262. package/dist/runtime/text.d.ts.map +1 -0
  263. package/dist/runtime/text.js +25 -0
  264. package/dist/runtime/text.js.map +1 -0
  265. package/dist/runtime/types.d.ts +50 -76
  266. package/dist/runtime/types.d.ts.map +1 -1
  267. package/dist/runtime/usage-tracker.d.ts +37 -0
  268. package/dist/runtime/usage-tracker.d.ts.map +1 -0
  269. package/dist/runtime/usage-tracker.js +110 -0
  270. package/dist/runtime/usage-tracker.js.map +1 -0
  271. package/dist/tools/define-agent-tool.d.ts +3 -28
  272. package/dist/tools/define-agent-tool.d.ts.map +1 -1
  273. package/dist/tools/define-agent-tool.js +3 -28
  274. package/dist/tools/define-agent-tool.js.map +1 -1
  275. package/dist/tools/dispatcher.d.ts +11 -3
  276. package/dist/tools/dispatcher.d.ts.map +1 -1
  277. package/dist/tools/dispatcher.js +50 -8
  278. package/dist/tools/dispatcher.js.map +1 -1
  279. package/dist/tools/registry.d.ts +8 -10
  280. package/dist/tools/registry.d.ts.map +1 -1
  281. package/dist/tools/registry.js +19 -11
  282. package/dist/tools/registry.js.map +1 -1
  283. package/dist/tools/step/result-tool.d.ts.map +1 -1
  284. package/dist/tools/step/result-tool.js +8 -6
  285. package/dist/tools/step/result-tool.js.map +1 -1
  286. package/dist/tools/tool-contract.d.ts +1 -1
  287. package/dist/tools/tool-contract.d.ts.map +1 -1
  288. package/dist/tools/tool-contract.js.map +1 -1
  289. package/dist/tools/types.d.ts +77 -6
  290. package/dist/tools/types.d.ts.map +1 -1
  291. package/dist/tools/types.js.map +1 -1
  292. package/docs/CLI.md +59 -0
  293. package/docs/CONFIGURATION.md +91 -0
  294. package/docs/DRIVERS.md +75 -0
  295. package/docs/EVIDENCE.md +62 -0
  296. package/package.json +51 -59
  297. package/schemas/checkmate-config.v1.json +176 -0
  298. package/schemas/describe-result.v1.json +414 -0
  299. package/schemas/driver-descriptor.v1.json +66 -0
  300. package/schemas/run-request.v1.json +67 -0
  301. package/schemas/run-result.v1.json +333 -0
  302. package/schemas/validation-result.v1.json +86 -0
  303. package/dist/ai/response-processor.d.ts +0 -20
  304. package/dist/ai/response-processor.d.ts.map +0 -1
  305. package/dist/ai/response-processor.js +0 -62
  306. package/dist/ai/response-processor.js.map +0 -1
  307. package/dist/ai/token-pricing.d.ts +0 -13
  308. package/dist/ai/token-pricing.d.ts.map +0 -1
  309. package/dist/ai/token-pricing.js +0 -348
  310. package/dist/ai/token-pricing.js.map +0 -1
  311. package/dist/ai/token-tracker.d.ts +0 -33
  312. package/dist/ai/token-tracker.d.ts.map +0 -1
  313. package/dist/ai/token-tracker.js +0 -138
  314. package/dist/ai/token-tracker.js.map +0 -1
  315. package/dist/cli/create-examples.d.ts +0 -21
  316. package/dist/cli/create-examples.d.ts.map +0 -1
  317. package/dist/cli/create-examples.js +0 -116
  318. package/dist/cli/create-examples.js.map +0 -1
  319. package/dist/cli.d.ts +0 -15
  320. package/dist/cli.d.ts.map +0 -1
  321. package/dist/cli.js +0 -69
  322. package/dist/cli.js.map +0 -1
  323. package/dist/config/runtime-config.d.ts +0 -21
  324. package/dist/config/runtime-config.d.ts.map +0 -1
  325. package/dist/config/runtime-config.js +0 -98
  326. package/dist/config/runtime-config.js.map +0 -1
  327. package/dist/core.d.ts +0 -8
  328. package/dist/core.d.ts.map +0 -1
  329. package/dist/core.js +0 -4
  330. package/dist/core.js.map +0 -1
  331. package/dist/integrations/salesforce/authenticator.d.ts +0 -27
  332. package/dist/integrations/salesforce/authenticator.d.ts.map +0 -1
  333. package/dist/integrations/salesforce/authenticator.js +0 -27
  334. package/dist/integrations/salesforce/authenticator.js.map +0 -1
  335. package/dist/integrations/salesforce/cli-handler.d.ts +0 -14
  336. package/dist/integrations/salesforce/cli-handler.d.ts.map +0 -1
  337. package/dist/integrations/salesforce/cli-handler.js +0 -50
  338. package/dist/integrations/salesforce/cli-handler.js.map +0 -1
  339. package/dist/logging/index.d.ts +0 -2
  340. package/dist/logging/index.d.ts.map +0 -1
  341. package/dist/logging/index.js +0 -4
  342. package/dist/logging/index.js.map +0 -1
  343. package/dist/logging/logger.d.ts +0 -5
  344. package/dist/logging/logger.d.ts.map +0 -1
  345. package/dist/logging/logger.js +0 -15
  346. package/dist/logging/logger.js.map +0 -1
  347. package/dist/playwright.d.ts +0 -102
  348. package/dist/playwright.d.ts.map +0 -1
  349. package/dist/playwright.js +0 -116
  350. package/dist/playwright.js.map +0 -1
  351. package/dist/runtime/extension.d.ts +0 -274
  352. package/dist/runtime/extension.d.ts.map +0 -1
  353. package/dist/runtime/extension.js +0 -171
  354. package/dist/runtime/extension.js.map +0 -1
  355. package/dist/salesforce.d.ts +0 -71
  356. package/dist/salesforce.d.ts.map +0 -1
  357. package/dist/salesforce.js +0 -73
  358. package/dist/salesforce.js.map +0 -1
  359. package/dist/tools/browser/screenshot-service.d.ts.map +0 -1
  360. package/dist/tools/browser/screenshot-service.js.map +0 -1
  361. package/dist/tools/browser/snapshot-filter/index.d.ts.map +0 -1
  362. package/dist/tools/browser/snapshot-filter/index.js.map +0 -1
  363. package/dist/tools/browser/snapshot-filter/semantic-scorer.d.ts.map +0 -1
  364. package/dist/tools/browser/snapshot-filter/semantic-scorer.js.map +0 -1
  365. package/dist/tools/browser/snapshot-filter/snapshot-filter.d.ts +0 -4
  366. package/dist/tools/browser/snapshot-filter/snapshot-filter.d.ts.map +0 -1
  367. package/dist/tools/browser/snapshot-filter/snapshot-filter.js +0 -57
  368. package/dist/tools/browser/snapshot-filter/snapshot-filter.js.map +0 -1
  369. package/dist/tools/browser/snapshot-filter/tree-reconstructor.d.ts.map +0 -1
  370. package/dist/tools/browser/snapshot-filter/tree-reconstructor.js.map +0 -1
  371. package/dist/tools/browser/snapshot-service.d.ts +0 -19
  372. package/dist/tools/browser/snapshot-service.d.ts.map +0 -1
  373. package/dist/tools/browser/snapshot-service.js.map +0 -1
  374. package/dist/tools/browser/tool.d.ts +0 -34
  375. package/dist/tools/browser/tool.d.ts.map +0 -1
  376. package/dist/tools/browser/tool.js +0 -226
  377. package/dist/tools/browser/tool.js.map +0 -1
  378. package/dist/tools/browser/transient-state-tracker.d.ts +0 -16
  379. package/dist/tools/browser/transient-state-tracker.d.ts.map +0 -1
  380. package/dist/tools/browser/transient-state-tracker.js.map +0 -1
  381. package/dist/tools/salesforce/login-tool.d.ts +0 -7
  382. package/dist/tools/salesforce/login-tool.d.ts.map +0 -1
  383. package/dist/tools/salesforce/login-tool.js +0 -25
  384. package/dist/tools/salesforce/login-tool.js.map +0 -1
  385. package/docs/EXTENSIONS.md +0 -232
  386. package/docs/GUIDE.md +0 -465
  387. package/docs/ROADMAP.md +0 -47
  388. package/playwright.config.ts +0 -39
  389. package/test/examples/salesforce/trial-dev-org.spec.ts +0 -82
  390. package/test/examples/web/website-testing.spec.ts +0 -281
  391. /package/dist/{tools/browser → drivers/web/tools}/screenshot-service.js +0 -0
  392. /package/dist/{tools/browser → drivers/web/tools}/snapshot-filter/index.d.ts +0 -0
  393. /package/dist/{tools/browser → drivers/web/tools}/snapshot-filter/index.js +0 -0
  394. /package/dist/{tools/browser → drivers/web/tools}/snapshot-filter/semantic-scorer.d.ts +0 -0
  395. /package/dist/{tools/browser → drivers/web/tools}/snapshot-filter/semantic-scorer.js +0 -0
  396. /package/dist/{tools/browser → drivers/web/tools}/snapshot-filter/tree-reconstructor.d.ts +0 -0
  397. /package/dist/{tools/browser → drivers/web/tools}/snapshot-filter/tree-reconstructor.js +0 -0
@@ -0,0 +1,64 @@
1
+ import { awaitStepDriverBoundary } from './driver-boundary.js';
2
+ export function adaptDriverTool(driverId, tool) {
3
+ return {
4
+ driverId,
5
+ definition: { ...tool.definition, parameters: structuredClone(tool.definition.parameters) },
6
+ execute: async (args, context) => {
7
+ if (!context.control || context.turn === undefined || !context.signal) {
8
+ throw new Error(`Driver tool '${tool.definition.name}' requires an active step control`);
9
+ }
10
+ return awaitStepDriverBoundary(`tool:${tool.definition.name}`, context.control, (signal) => tool.execute(args, {
11
+ step: toStepIntent(context.step),
12
+ turn: context.turn,
13
+ signal,
14
+ }));
15
+ },
16
+ };
17
+ }
18
+ export async function buildInitialDriverContext(session, step, control) {
19
+ const context = await awaitStepDriverBoundary('build-initial-context', control, (signal) => session.buildInitialContext({ step, signal }));
20
+ return context.map(toProviderMessage);
21
+ }
22
+ export async function buildPostToolDriverContext(session, step, turn, toolResponses, control) {
23
+ const context = await awaitStepDriverBoundary('handle-tool-responses', control, (signal) => session.handleToolResponses({
24
+ step,
25
+ turn,
26
+ toolResponses: toolResponses.map(toDriverToolExecution),
27
+ signal,
28
+ }));
29
+ return context.map(toProviderMessage);
30
+ }
31
+ export function isEphemeralDriverMessage(message) {
32
+ return Boolean(message.__checkmateEphemeral);
33
+ }
34
+ function toProviderMessage(context) {
35
+ const content = typeof context.content === 'string'
36
+ ? context.content
37
+ : context.content.map((part) => part.type === 'text'
38
+ ? { type: 'text', text: part.text }
39
+ : {
40
+ type: 'image_url',
41
+ image_url: {
42
+ url: `data:${part.mediaType};base64,${part.data}`,
43
+ detail: 'high',
44
+ },
45
+ });
46
+ const message = { role: 'user', content };
47
+ if (context.ephemeral) {
48
+ Object.defineProperty(message, '__checkmateEphemeral', { value: true, enumerable: false });
49
+ }
50
+ return message;
51
+ }
52
+ function toDriverToolExecution(execution) {
53
+ return {
54
+ toolCallId: execution.toolCallId,
55
+ name: execution.toolCall.name,
56
+ arguments: structuredClone(execution.toolCall.arguments ?? {}),
57
+ response: execution.toolResponse.response,
58
+ status: execution.toolResponse.status,
59
+ };
60
+ }
61
+ function toStepIntent(step) {
62
+ return { id: step.id ?? 'legacy-step', action: step.action, expect: step.expect };
63
+ }
64
+ //# sourceMappingURL=driver-session.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"driver-session.js","sourceRoot":"","sources":["../../src/runtime/driver-session.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAA;AAG9D,MAAM,UAAU,eAAe,CAAC,QAAgB,EAAE,IAAgB;IACjE,OAAO;QACN,QAAQ;QACR,UAAU,EAAE,EAAE,GAAG,IAAI,CAAC,UAAU,EAAE,UAAU,EAAE,eAAe,CAAC,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE;QAC3F,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE;YAChC,IAAI,CAAC,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;gBACvE,MAAM,IAAI,KAAK,CAAC,gBAAgB,IAAI,CAAC,UAAU,CAAC,IAAI,mCAAmC,CAAC,CAAA;YACzF,CAAC;YACD,OAAO,uBAAuB,CAAC,QAAQ,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,MAAM,EAAE,EAAE,CAC1F,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE;gBAClB,IAAI,EAAE,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC;gBAChC,IAAI,EAAE,OAAO,CAAC,IAAK;gBACnB,MAAM;aACN,CAAC,CACF,CAAA;QACF,CAAC;KACD,CAAA;AACF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC9C,OAAsB,EACtB,IAAgB,EAChB,OAAoB;IAEpB,MAAM,OAAO,GAAG,MAAM,uBAAuB,CAAC,uBAAuB,EAAE,OAAO,EAAE,CAAC,MAAM,EAAE,EAAE,CAC1F,OAAO,CAAC,mBAAmB,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAC7C,CAAA;IACD,OAAO,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAA;AACtC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAC/C,OAAsB,EACtB,IAAgB,EAChB,IAAY,EACZ,aAAuC,EACvC,OAAoB;IAEpB,MAAM,OAAO,GAAG,MAAM,uBAAuB,CAAC,uBAAuB,EAAE,OAAO,EAAE,CAAC,MAAM,EAAE,EAAE,CAC1F,OAAO,CAAC,mBAAmB,CAAC;QAC3B,IAAI;QACJ,IAAI;QACJ,aAAa,EAAE,aAAa,CAAC,GAAG,CAAC,qBAAqB,CAAC;QACvD,MAAM;KACN,CAAC,CACF,CAAA;IACD,OAAO,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAA;AACtC,CAAC;AAED,MAAM,UAAU,wBAAwB,CAAC,OAAmC;IAC3E,OAAO,OAAO,CAAE,OAA2E,CAAC,oBAAoB,CAAC,CAAA;AAClH,CAAC;AAED,SAAS,iBAAiB,CAAC,OAA6B;IACvD,MAAM,OAAO,GACZ,OAAO,OAAO,CAAC,OAAO,KAAK,QAAQ;QAClC,CAAC,CAAC,OAAO,CAAC,OAAO;QACjB,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAC7B,IAAI,CAAC,IAAI,KAAK,MAAM;YACnB,CAAC,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE;YAC5C,CAAC,CAAC;gBACA,IAAI,EAAE,WAAoB;gBAC1B,SAAS,EAAE;oBACV,GAAG,EAAE,QAAQ,IAAI,CAAC,SAAS,WAAW,IAAI,CAAC,IAAI,EAAE;oBACjD,MAAM,EAAE,MAAe;iBACvB;aACD,CACH,CAAA;IACJ,MAAM,OAAO,GAAG,EAAE,IAAI,EAAE,MAAe,EAAE,OAAO,EAE/C,CAAA;IACD,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;QACvB,MAAM,CAAC,cAAc,CAAC,OAAO,EAAE,sBAAsB,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,CAAA;IAC3F,CAAC;IACD,OAAO,OAAO,CAAA;AACf,CAAC;AAED,SAAS,qBAAqB,CAAC,SAAwB;IACtD,OAAO;QACN,UAAU,EAAE,SAAS,CAAC,UAAU;QAChC,IAAI,EAAE,SAAS,CAAC,QAAQ,CAAC,IAAI;QAC7B,SAAS,EAAE,eAAe,CAAC,SAAS,CAAC,QAAQ,CAAC,SAAS,IAAI,EAAE,CAAC;QAC9D,QAAQ,EAAE,SAAS,CAAC,YAAY,CAAC,QAAQ;QACzC,MAAM,EAAE,SAAS,CAAC,YAAY,CAAC,MAAM;KACrC,CAAA;AACF,CAAC;AAED,SAAS,YAAY,CAAC,IAAqD;IAC1E,OAAO,EAAE,EAAE,EAAE,IAAI,CAAC,EAAE,IAAI,aAAa,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAA;AAClF,CAAC"}
@@ -0,0 +1,30 @@
1
+ import type { Diagnostic, Usage } from '../contracts/types.js';
2
+ import type { StepIntent } from '../driver.js';
3
+ import { DiagnosticSanitizer } from '../redaction/diagnostic-sanitizer.js';
4
+ import type { ToolCall, ToolResponse } from '../tools/types.js';
5
+ import type { InternalStepReport, InternalTerminationReason } from './types.js';
6
+ export type InternalStepTermination = Readonly<{
7
+ outcome: 'passed' | 'failed';
8
+ reason: InternalTerminationReason;
9
+ actual?: string;
10
+ turns: number;
11
+ }>;
12
+ export declare class InternalStepEvidence {
13
+ private readonly step;
14
+ private readonly driverId;
15
+ private readonly redact;
16
+ private readonly diagnosticSanitizer;
17
+ private readonly now;
18
+ private readonly startedAt;
19
+ private readonly toolCalls;
20
+ private readonly transcript;
21
+ private readonly diagnostics;
22
+ constructor(step: StepIntent, driverId: string, redact: boolean, diagnosticSanitizer: DiagnosticSanitizer, now?: () => number);
23
+ recordAssistantMessage(turn: number, content: string): void;
24
+ recordToolCall(turn: number, toolCall: ToolCall, toolResponse: ToolResponse): void;
25
+ recordDiagnostic(diagnostic: Diagnostic): void;
26
+ buildReport(termination: InternalStepTermination, usage: Usage): InternalStepReport;
27
+ private sanitize;
28
+ private sanitizeArguments;
29
+ }
30
+ //# sourceMappingURL=internal-step-evidence.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"internal-step-evidence.d.ts","sourceRoot":"","sources":["../../src/runtime/internal-step-evidence.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAA;AAC9D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,cAAc,CAAA;AAC9C,OAAO,EAAE,mBAAmB,EAAE,MAAM,sCAAsC,CAAA;AAE1E,OAAO,KAAK,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAA;AAC/D,OAAO,KAAK,EACX,kBAAkB,EAElB,yBAAyB,EAGzB,MAAM,YAAY,CAAA;AAkBnB,MAAM,MAAM,uBAAuB,GAAG,QAAQ,CAAC;IAC9C,OAAO,EAAE,QAAQ,GAAG,QAAQ,CAAA;IAC5B,MAAM,EAAE,yBAAyB,CAAA;IACjC,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,KAAK,EAAE,MAAM,CAAA;CACb,CAAC,CAAA;AAEF,qBAAa,oBAAoB;IAO/B,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,QAAQ;IACzB,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,mBAAmB;IACpC,OAAO,CAAC,QAAQ,CAAC,GAAG;IAVrB,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAQ;IAClC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAA6B;IACvD,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAwB;IACnD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAmB;gBAG7B,IAAI,EAAE,UAAU,EAChB,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,OAAO,EACf,mBAAmB,EAAE,mBAAmB,EACxC,GAAG,GAAE,MAAM,MAAiB;IAK9C,sBAAsB,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI;IAI3D,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,YAAY,EAAE,YAAY,GAAG,IAAI;IAelF,gBAAgB,CAAC,UAAU,EAAE,UAAU,GAAG,IAAI;IAI9C,WAAW,CAAC,WAAW,EAAE,uBAAuB,EAAE,KAAK,EAAE,KAAK,GAAG,kBAAkB;IAgBnF,OAAO,CAAC,QAAQ;IAIhB,OAAO,CAAC,iBAAiB;CAGzB"}
@@ -0,0 +1,82 @@
1
+ import { scrub, scrubValue } from '../redaction/scrub.js';
2
+ const TRANSCRIPT_CONTENT_LIMIT = 2_000;
3
+ const CATEGORY_BY_REASON = {
4
+ 'met-expectation': 'app',
5
+ 'failed-expectation': 'app',
6
+ 'loop-detected': 'model',
7
+ 'turn-cap-exceeded': 'model',
8
+ 'step-timeout': 'model',
9
+ 'scenario-timeout': 'infra',
10
+ 'tool-error': 'infra',
11
+ 'provider-error': 'infra',
12
+ 'token-budget-exceeded': 'infra',
13
+ interrupted: 'infra',
14
+ 'internal-error': 'infra',
15
+ };
16
+ export class InternalStepEvidence {
17
+ step;
18
+ driverId;
19
+ redact;
20
+ diagnosticSanitizer;
21
+ now;
22
+ startedAt;
23
+ toolCalls = [];
24
+ transcript = [];
25
+ diagnostics = [];
26
+ constructor(step, driverId, redact, diagnosticSanitizer, now = Date.now) {
27
+ this.step = step;
28
+ this.driverId = driverId;
29
+ this.redact = redact;
30
+ this.diagnosticSanitizer = diagnosticSanitizer;
31
+ this.now = now;
32
+ this.startedAt = now();
33
+ }
34
+ recordAssistantMessage(turn, content) {
35
+ this.transcript.push({ turn, role: 'assistant', content: truncate(this.sanitize(content)) });
36
+ }
37
+ recordToolCall(turn, toolCall, toolResponse) {
38
+ this.toolCalls.push({
39
+ turn,
40
+ driverId: isHarnessTool(toolCall.name) ? 'harness' : this.driverId,
41
+ name: toolCall.name,
42
+ arguments: this.sanitizeArguments(toolCall.arguments ?? {}),
43
+ status: toolResponse.status === 'error' ? 'error' : 'ok',
44
+ });
45
+ this.transcript.push({
46
+ turn,
47
+ role: 'tool',
48
+ content: truncate(this.sanitize(`${toolCall.name} -> ${toolResponse.response}`)),
49
+ });
50
+ }
51
+ recordDiagnostic(diagnostic) {
52
+ this.diagnostics.push({ ...diagnostic, message: this.diagnosticSanitizer.text(diagnostic.message) });
53
+ }
54
+ buildReport(termination, usage) {
55
+ return {
56
+ step: { ...this.step },
57
+ outcome: termination.outcome,
58
+ category: CATEGORY_BY_REASON[termination.reason],
59
+ reason: termination.reason,
60
+ ...(termination.actual === undefined ? {} : { actual: this.sanitize(termination.actual) }),
61
+ turns: termination.turns,
62
+ durationMs: this.now() - this.startedAt,
63
+ usage: { ...usage },
64
+ toolCalls: this.toolCalls.map((call) => ({ ...call })),
65
+ transcript: this.transcript.map((entry) => ({ ...entry })),
66
+ diagnostics: this.diagnostics.map((diagnostic) => ({ ...diagnostic })),
67
+ };
68
+ }
69
+ sanitize(value) {
70
+ return this.redact ? scrub(value) : value;
71
+ }
72
+ sanitizeArguments(value) {
73
+ return this.redact ? scrubValue(value) : structuredClone(value);
74
+ }
75
+ }
76
+ function isHarnessTool(name) {
77
+ return name === 'pass_test_step' || name === 'fail_test_step';
78
+ }
79
+ function truncate(value) {
80
+ return value.length <= TRANSCRIPT_CONTENT_LIMIT ? value : `${value.slice(0, TRANSCRIPT_CONTENT_LIMIT - 3)}...`;
81
+ }
82
+ //# sourceMappingURL=internal-step-evidence.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"internal-step-evidence.js","sourceRoot":"","sources":["../../src/runtime/internal-step-evidence.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAUzD,MAAM,wBAAwB,GAAG,KAAK,CAAA;AAEtC,MAAM,kBAAkB,GAAoD;IAC3E,iBAAiB,EAAE,KAAK;IACxB,oBAAoB,EAAE,KAAK;IAC3B,eAAe,EAAE,OAAO;IACxB,mBAAmB,EAAE,OAAO;IAC5B,cAAc,EAAE,OAAO;IACvB,kBAAkB,EAAE,OAAO;IAC3B,YAAY,EAAE,OAAO;IACrB,gBAAgB,EAAE,OAAO;IACzB,uBAAuB,EAAE,OAAO;IAChC,WAAW,EAAE,OAAO;IACpB,gBAAgB,EAAE,OAAO;CACzB,CAAA;AASD,MAAM,OAAO,oBAAoB;IAOd;IACA;IACA;IACA;IACA;IAVD,SAAS,CAAQ;IACjB,SAAS,GAA2B,EAAE,CAAA;IACtC,UAAU,GAAsB,EAAE,CAAA;IAClC,WAAW,GAAiB,EAAE,CAAA;IAE/C,YACkB,IAAgB,EAChB,QAAgB,EAChB,MAAe,EACf,mBAAwC,EACxC,MAAoB,IAAI,CAAC,GAAG;QAJ5B,SAAI,GAAJ,IAAI,CAAY;QAChB,aAAQ,GAAR,QAAQ,CAAQ;QAChB,WAAM,GAAN,MAAM,CAAS;QACf,wBAAmB,GAAnB,mBAAmB,CAAqB;QACxC,QAAG,GAAH,GAAG,CAAyB;QAE7C,IAAI,CAAC,SAAS,GAAG,GAAG,EAAE,CAAA;IACvB,CAAC;IAED,sBAAsB,CAAC,IAAY,EAAE,OAAe;QACnD,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC,CAAA;IAC7F,CAAC;IAED,cAAc,CAAC,IAAY,EAAE,QAAkB,EAAE,YAA0B;QAC1E,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;YACnB,IAAI;YACJ,QAAQ,EAAE,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ;YAClE,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,SAAS,EAAE,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,SAAS,IAAI,EAAE,CAAC;YAC3D,MAAM,EAAE,YAAY,CAAC,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI;SACxD,CAAC,CAAA;QACF,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;YACpB,IAAI;YACJ,IAAI,EAAE,MAAM;YACZ,OAAO,EAAE,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,QAAQ,CAAC,IAAI,OAAO,YAAY,CAAC,QAAQ,EAAE,CAAC,CAAC;SAChF,CAAC,CAAA;IACH,CAAC;IAED,gBAAgB,CAAC,UAAsB;QACtC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,GAAG,UAAU,EAAE,OAAO,EAAE,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;IACrG,CAAC;IAED,WAAW,CAAC,WAAoC,EAAE,KAAY;QAC7D,OAAO;YACN,IAAI,EAAE,EAAE,GAAG,IAAI,CAAC,IAAI,EAAE;YACtB,OAAO,EAAE,WAAW,CAAC,OAAO;YAC5B,QAAQ,EAAE,kBAAkB,CAAC,WAAW,CAAC,MAAM,CAAC;YAChD,MAAM,EAAE,WAAW,CAAC,MAAM;YAC1B,GAAG,CAAC,WAAW,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC;YAC1F,KAAK,EAAE,WAAW,CAAC,KAAK;YACxB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,SAAS;YACvC,KAAK,EAAE,EAAE,GAAG,KAAK,EAAE;YACnB,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,CAAC,CAAC;YACtD,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,KAAK,EAAE,CAAC,CAAC;YAC1D,WAAW,EAAE,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,UAAU,EAAE,CAAC,CAAC;SACtE,CAAA;IACF,CAAC;IAEO,QAAQ,CAAC,KAAa;QAC7B,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAA;IAC1C,CAAC;IAEO,iBAAiB,CAAC,KAAc;QACvC,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,KAAK,CAAC,CAAA;IAChE,CAAC;CACD;AAED,SAAS,aAAa,CAAC,IAAY;IAClC,OAAO,IAAI,KAAK,gBAAgB,IAAI,IAAI,KAAK,gBAAgB,CAAA;AAC9D,CAAC;AAED,SAAS,QAAQ,CAAC,KAAa;IAC9B,OAAO,KAAK,CAAC,MAAM,IAAI,wBAAwB,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,wBAAwB,GAAG,CAAC,CAAC,KAAK,CAAA;AAC/G,CAAC"}
@@ -0,0 +1,9 @@
1
+ import type { ExecutedStepResult } from '../contracts/types.js';
2
+ import type { HarnessEvidenceCandidate } from '../evidence/store.js';
3
+ import type { InternalStepReport } from './types.js';
4
+ export type AdaptedStepResult = Readonly<{
5
+ result: ExecutedStepResult;
6
+ evidence: readonly HarnessEvidenceCandidate[];
7
+ }>;
8
+ export declare function adaptStepResult(report: InternalStepReport): AdaptedStepResult;
9
+ //# sourceMappingURL=result-adapter.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"result-adapter.d.ts","sourceRoot":"","sources":["../../src/runtime/result-adapter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAyB,MAAM,uBAAuB,CAAA;AAEtF,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,sBAAsB,CAAA;AACpE,OAAO,KAAK,EAAE,kBAAkB,EAA2C,MAAM,YAAY,CAAA;AA8B7F,MAAM,MAAM,iBAAiB,GAAG,QAAQ,CAAC;IACxC,MAAM,EAAE,kBAAkB,CAAA;IAC1B,QAAQ,EAAE,SAAS,wBAAwB,EAAE,CAAA;CAC7C,CAAC,CAAA;AAEF,wBAAgB,eAAe,CAAC,MAAM,EAAE,kBAAkB,GAAG,iBAAiB,CAkC7E"}
@@ -0,0 +1,77 @@
1
+ import { serializeJson } from '../contracts/serialize.js';
2
+ const PUBLIC_REASON_BY_INTERNAL = {
3
+ 'met-expectation': 'met-expectation',
4
+ 'failed-expectation': 'failed-expectation',
5
+ 'loop-detected': 'loop-detected',
6
+ 'turn-cap-exceeded': 'turn-cap-exceeded',
7
+ 'step-timeout': 'step-timeout',
8
+ 'scenario-timeout': 'scenario-timeout',
9
+ 'tool-error': 'tool-error',
10
+ 'provider-error': 'provider-error',
11
+ 'token-budget-exceeded': 'token-budget-exceeded',
12
+ interrupted: 'interrupted',
13
+ 'internal-error': 'internal-error',
14
+ };
15
+ const CATEGORY_BY_INTERNAL = {
16
+ 'met-expectation': 'app',
17
+ 'failed-expectation': 'app',
18
+ 'loop-detected': 'model',
19
+ 'turn-cap-exceeded': 'model',
20
+ 'step-timeout': 'model',
21
+ 'scenario-timeout': 'infra',
22
+ 'tool-error': 'infra',
23
+ 'provider-error': 'infra',
24
+ 'token-budget-exceeded': 'infra',
25
+ interrupted: 'infra',
26
+ 'internal-error': 'infra',
27
+ };
28
+ export function adaptStepResult(report) {
29
+ const reason = PUBLIC_REASON_BY_INTERNAL[report.reason];
30
+ const status = report.reason === 'met-expectation' ? 'passed' : 'failed';
31
+ if (report.outcome !== status) {
32
+ throw new Error(`Internal step '${report.step.id}' has inconsistent outcome and reason`);
33
+ }
34
+ const result = {
35
+ id: report.step.id,
36
+ status,
37
+ category: CATEGORY_BY_INTERNAL[report.reason],
38
+ reason,
39
+ ...(report.actual === undefined ? {} : { actual: report.actual }),
40
+ turns: report.turns,
41
+ durationMs: report.durationMs,
42
+ usage: { ...report.usage },
43
+ toolCalls: report.toolCalls.map((call) => ({
44
+ turn: call.turn,
45
+ driverId: call.driverId,
46
+ name: call.name,
47
+ arguments: copyJson(call.arguments),
48
+ status: call.status,
49
+ })),
50
+ };
51
+ const evidence = [];
52
+ if (report.transcript.length > 0) {
53
+ evidence.push({
54
+ stepId: report.step.id,
55
+ kind: 'transcript',
56
+ mediaType: 'text/markdown',
57
+ content: renderTranscript(report),
58
+ });
59
+ }
60
+ return { result, evidence };
61
+ }
62
+ function copyJson(value) {
63
+ serializeJson(value);
64
+ return structuredClone(value);
65
+ }
66
+ function renderTranscript(report) {
67
+ const lines = [
68
+ `# ${report.step.action}`,
69
+ '',
70
+ `**Outcome:** ${report.outcome} - ${CATEGORY_BY_INTERNAL[report.reason]} / ${PUBLIC_REASON_BY_INTERNAL[report.reason]}`,
71
+ '',
72
+ ];
73
+ for (const entry of report.transcript)
74
+ lines.push(`### Turn ${entry.turn} - ${entry.role}`, '', entry.content, '');
75
+ return lines.join('\n');
76
+ }
77
+ //# sourceMappingURL=result-adapter.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"result-adapter.js","sourceRoot":"","sources":["../../src/runtime/result-adapter.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAA;AAIzD,MAAM,yBAAyB,GAAkD;IAChF,iBAAiB,EAAE,iBAAiB;IACpC,oBAAoB,EAAE,oBAAoB;IAC1C,eAAe,EAAE,eAAe;IAChC,mBAAmB,EAAE,mBAAmB;IACxC,cAAc,EAAE,cAAc;IAC9B,kBAAkB,EAAE,kBAAkB;IACtC,YAAY,EAAE,YAAY;IAC1B,gBAAgB,EAAE,gBAAgB;IAClC,uBAAuB,EAAE,uBAAuB;IAChD,WAAW,EAAE,aAAa;IAC1B,gBAAgB,EAAE,gBAAgB;CAClC,CAAA;AAED,MAAM,oBAAoB,GAAoD;IAC7E,iBAAiB,EAAE,KAAK;IACxB,oBAAoB,EAAE,KAAK;IAC3B,eAAe,EAAE,OAAO;IACxB,mBAAmB,EAAE,OAAO;IAC5B,cAAc,EAAE,OAAO;IACvB,kBAAkB,EAAE,OAAO;IAC3B,YAAY,EAAE,OAAO;IACrB,gBAAgB,EAAE,OAAO;IACzB,uBAAuB,EAAE,OAAO;IAChC,WAAW,EAAE,OAAO;IACpB,gBAAgB,EAAE,OAAO;CACzB,CAAA;AAOD,MAAM,UAAU,eAAe,CAAC,MAA0B;IACzD,MAAM,MAAM,GAAG,yBAAyB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;IACvD,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,KAAK,iBAAiB,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAA;IACxE,IAAI,MAAM,CAAC,OAAO,KAAK,MAAM,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CAAC,kBAAkB,MAAM,CAAC,IAAI,CAAC,EAAE,uCAAuC,CAAC,CAAA;IACzF,CAAC;IAED,MAAM,MAAM,GAAuB;QAClC,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE;QAClB,MAAM;QACN,QAAQ,EAAE,oBAAoB,CAAC,MAAM,CAAC,MAAM,CAAC;QAC7C,MAAM;QACN,GAAG,CAAC,MAAM,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;QACjE,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,KAAK,EAAE,EAAE,GAAG,MAAM,CAAC,KAAK,EAAE;QAC1B,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YAC1C,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,SAAS,EAAE,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC;YACnC,MAAM,EAAE,IAAI,CAAC,MAAM;SACnB,CAAC,CAAC;KACH,CAAA;IACD,MAAM,QAAQ,GAA+B,EAAE,CAAA;IAC/C,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClC,QAAQ,CAAC,IAAI,CAAC;YACb,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE;YACtB,IAAI,EAAE,YAAY;YAClB,SAAS,EAAE,eAAe;YAC1B,OAAO,EAAE,gBAAgB,CAAC,MAAM,CAAC;SACjC,CAAC,CAAA;IACH,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAA;AAC5B,CAAC;AAED,SAAS,QAAQ,CAAC,KAAc;IAC/B,aAAa,CAAC,KAAK,CAAC,CAAA;IACpB,OAAO,eAAe,CAAC,KAAK,CAAc,CAAA;AAC3C,CAAC;AAED,SAAS,gBAAgB,CAAC,MAA0B;IACnD,MAAM,KAAK,GAAG;QACb,KAAK,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE;QACzB,EAAE;QACF,gBAAgB,MAAM,CAAC,OAAO,MAAM,oBAAoB,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,yBAAyB,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE;QACvH,EAAE;KACF,CAAA;IACD,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,UAAU;QAAE,KAAK,CAAC,IAAI,CAAC,YAAY,KAAK,CAAC,IAAI,MAAM,KAAK,CAAC,IAAI,EAAE,EAAE,EAAE,EAAE,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC,CAAA;IAClH,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AACxB,CAAC"}
@@ -1,94 +1,41 @@
1
- import { RuntimeConfig } from '../config/runtime-config.js';
2
- import { Step } from './types.js';
3
- import { CheckmateExtension } from './extension.js';
4
- /**
5
- * Options for creating a Checkmate runner.
6
- *
7
- * @example
8
- * ```ts
9
- * import { createRunner } from '@xoxoai/checkmate/core'
10
- * import { web } from '@xoxoai/checkmate/playwright'
11
- *
12
- * const ai = createRunner({
13
- * extensions: [web({ page })],
14
- * })
15
- * ```
16
- */
17
- export type CheckmateRunnerOptions = {
18
- /**
19
- * Extensions to compose into the runner.
20
- */
21
- extensions?: CheckmateExtension[];
22
- /**
23
- * Advanced: provide a custom runtime config instance.
24
- */
25
- runtimeConfig?: RuntimeConfig;
1
+ import type { DriverDescriptorV1, ModelEgressPolicyV1 } from '../contracts/types.js';
2
+ import type { DriverSession, StepIntent } from '../driver.js';
3
+ import type { StepControl } from './scenario-control.js';
4
+ import type { InternalStepReport } from './types.js';
5
+ import { AiClient } from '../ai/client.js';
6
+ import { ScenarioUsageTracker } from './usage-tracker.js';
7
+ import { type RuntimeLogger } from '../logging/types.js';
8
+ export type DriverCheckmateRunnerOptions = {
9
+ driverId: string;
10
+ descriptor: DriverDescriptorV1;
11
+ session: DriverSession;
12
+ allowedTools: '*' | readonly string[];
13
+ modelEgress: ModelEgressPolicyV1;
14
+ limits: {
15
+ turnsPerStep: number;
16
+ stepTimeoutMs: number;
17
+ requestTimeoutMs: number;
18
+ maxRetries: number;
19
+ loopMaxRepetitions: number;
20
+ budgetTokens?: number;
21
+ };
22
+ apiKey: string;
23
+ exactSecrets?: Iterable<string>;
24
+ logger?: RuntimeLogger;
25
+ usageTracker?: ScenarioUsageTracker;
26
+ aiClient?: AiClient;
26
27
  };
27
- /**
28
- * Public runtime entry point for executing natural-language steps with Checkmate.
29
- *
30
- * @example
31
- * ```ts
32
- * const runner = new CheckmateRunner()
33
- * await runner.run({
34
- * action: 'Open the pricing page',
35
- * expect: 'Pricing details are visible',
36
- * })
37
- * ```
38
- */
39
28
  export declare class CheckmateRunner {
29
+ private readonly options;
30
+ private readonly config;
31
+ private readonly toolRegistry;
40
32
  private readonly aiClient;
41
- private readonly extensionHost;
42
- /**
43
- * Creates a new runner composed from extensions.
44
- *
45
- * @example
46
- * ```ts
47
- * const ai = new CheckmateRunner({
48
- * extensions: [web({ page })],
49
- * })
50
- * ```
51
- */
52
- constructor(options?: CheckmateRunnerOptions);
53
- /**
54
- * Releases any runner-owned resources.
55
- *
56
- * @example
57
- * ```ts
58
- * await ai.teardown()
59
- * ```
60
- */
33
+ private readonly usageTracker;
34
+ private readonly diagnosticSanitizer;
35
+ private readonly runtimeLogger;
36
+ constructor(options: DriverCheckmateRunnerOptions);
61
37
  teardown(): Promise<void>;
62
- /**
63
- * Executes one natural-language test step.
64
- *
65
- * @param step - The step definition to execute.
66
- *
67
- * @example
68
- * ```ts
69
- * await runner.run({
70
- * action: 'Search for qwen3-vl',
71
- * expect: 'The qwen3-vl model page is displayed',
72
- * topPercent: 10,
73
- * })
74
- * ```
75
- */
76
- run(step: Step): Promise<void>;
38
+ run(step: StepIntent, control: StepControl): Promise<InternalStepReport>;
77
39
  }
78
- /**
79
- * Creates a runner from the provided extensions.
80
- *
81
- * This is the main programmatic entry point from `@xoxoai/checkmate/core`.
82
- *
83
- * @example
84
- * ```ts
85
- * import { createRunner } from '@xoxoai/checkmate/core'
86
- * import { web } from '@xoxoai/checkmate/playwright'
87
- *
88
- * const ai = createRunner({
89
- * extensions: [web({ page })],
90
- * })
91
- * ```
92
- */
93
- export declare function createRunner(options?: CheckmateRunnerOptions): CheckmateRunner;
40
+ export declare function createDriverRunner(options: DriverCheckmateRunnerOptions): CheckmateRunner;
94
41
  //# sourceMappingURL=runner.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/runtime/runner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAA;AAC3D,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAA;AAKjC,OAAO,EAAE,kBAAkB,EAAiB,MAAM,gBAAgB,CAAA;AAElE;;;;;;;;;;;;GAYG;AACH,MAAM,MAAM,sBAAsB,GAAG;IACpC;;OAEG;IACH,UAAU,CAAC,EAAE,kBAAkB,EAAE,CAAA;IAEjC;;OAEG;IACH,aAAa,CAAC,EAAE,aAAa,CAAA;CAC7B,CAAA;AAED;;;;;;;;;;;GAWG;AACH,qBAAa,eAAe;IAC3B,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAU;IACnC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAe;IAE7C;;;;;;;;;OASG;gBACS,OAAO,GAAE,sBAA2B;IAQhD;;;;;;;OAOG;IACG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAI/B;;;;;;;;;;;;;OAaG;IACG,GAAG,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;CAGpC;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,YAAY,CAAC,OAAO,GAAE,sBAA2B,GAAG,eAAe,CAElF"}
1
+ {"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/runtime/runner.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAA;AACpF,OAAO,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,MAAM,cAAc,CAAA;AAG7D,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAA;AACxD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAA;AAIpD,OAAO,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAA;AAC1C,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAA;AAGzD,OAAO,EAAgB,KAAK,aAAa,EAAE,MAAM,qBAAqB,CAAA;AAGtE,MAAM,MAAM,4BAA4B,GAAG;IAC1C,QAAQ,EAAE,MAAM,CAAA;IAChB,UAAU,EAAE,kBAAkB,CAAA;IAC9B,OAAO,EAAE,aAAa,CAAA;IACtB,YAAY,EAAE,GAAG,GAAG,SAAS,MAAM,EAAE,CAAA;IACrC,WAAW,EAAE,mBAAmB,CAAA;IAChC,MAAM,EAAE;QACP,YAAY,EAAE,MAAM,CAAA;QACpB,aAAa,EAAE,MAAM,CAAA;QACrB,gBAAgB,EAAE,MAAM,CAAA;QACxB,UAAU,EAAE,MAAM,CAAA;QAClB,kBAAkB,EAAE,MAAM,CAAA;QAC1B,YAAY,CAAC,EAAE,MAAM,CAAA;KACrB,CAAA;IACD,MAAM,EAAE,MAAM,CAAA;IACd,YAAY,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAA;IAC/B,MAAM,CAAC,EAAE,aAAa,CAAA;IACtB,YAAY,CAAC,EAAE,oBAAoB,CAAA;IACnC,QAAQ,CAAC,EAAE,QAAQ,CAAA;CACnB,CAAA;AAED,qBAAa,eAAe;IAQf,OAAO,CAAC,QAAQ,CAAC,OAAO;IAPpC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAe;IACtC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAc;IAC3C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAU;IACnC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAsB;IACnD,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAqB;IACzD,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAe;gBAEhB,OAAO,EAAE,4BAA4B;IAmC5D,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAEzB,GAAG,CAAC,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,kBAAkB,CAAC;CAa9E;AAED,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,4BAA4B,GAAG,eAAe,CAEzF"}
@@ -1,86 +1,72 @@
1
- import { RuntimeConfig } from '../config/runtime-config.js';
1
+ import { validateDriverSession } from '../drivers/loader.js';
2
+ import { adaptDriverTool } from './driver-session.js';
2
3
  import { createStepResultTools } from '../tools/step/result-tool.js';
3
4
  import { ToolRegistry } from '../tools/registry.js';
4
5
  import { AiClient } from '../ai/client.js';
6
+ import { ScenarioUsageTracker } from './usage-tracker.js';
5
7
  import { StepExecution } from './step-execution.js';
6
- import { ExtensionHost } from './extension.js';
7
- /**
8
- * Public runtime entry point for executing natural-language steps with Checkmate.
9
- *
10
- * @example
11
- * ```ts
12
- * const runner = new CheckmateRunner()
13
- * await runner.run({
14
- * action: 'Open the pricing page',
15
- * expect: 'Pricing details are visible',
16
- * })
17
- * ```
18
- */
8
+ import { createInvocationLogger } from '../logging/invocation-logger.js';
9
+ import { silentLogger } from '../logging/types.js';
10
+ import { DiagnosticSanitizer } from '../redaction/diagnostic-sanitizer.js';
19
11
  export class CheckmateRunner {
12
+ options;
13
+ config;
14
+ toolRegistry;
20
15
  aiClient;
21
- extensionHost;
22
- /**
23
- * Creates a new runner composed from extensions.
24
- *
25
- * @example
26
- * ```ts
27
- * const ai = new CheckmateRunner({
28
- * extensions: [web({ page })],
29
- * })
30
- * ```
31
- */
32
- constructor(options = {}) {
33
- const runtimeConfig = options.runtimeConfig ?? new RuntimeConfig();
34
- const toolRegistry = new ToolRegistry(runtimeConfig);
35
- toolRegistry.register(createStepResultTools());
36
- this.extensionHost = new ExtensionHost(runtimeConfig, toolRegistry, options.extensions ?? []);
37
- this.aiClient = new AiClient({ runtimeConfig, toolRegistry, extensionHost: this.extensionHost });
16
+ usageTracker;
17
+ diagnosticSanitizer;
18
+ runtimeLogger;
19
+ constructor(options) {
20
+ this.options = options;
21
+ const exactSecrets = [options.apiKey, ...(options.exactSecrets ?? [])];
22
+ this.diagnosticSanitizer = new DiagnosticSanitizer(exactSecrets);
23
+ this.runtimeLogger = createInvocationLogger(options.logger ?? silentLogger, this.diagnosticSanitizer);
24
+ this.config = {
25
+ model: options.modelEgress.provider.model,
26
+ baseUrl: options.modelEgress.provider.baseUrl,
27
+ reasoningEffort: options.modelEgress.provider.reasoningEffort,
28
+ temperature: options.modelEgress.provider.temperature ?? 0,
29
+ turnCap: options.limits.turnsPerStep,
30
+ redact: options.modelEgress.textRedaction === 'on',
31
+ toolChoice: 'required',
32
+ maxRetries: options.limits.maxRetries,
33
+ requestTimeout: options.limits.requestTimeoutMs,
34
+ loopMaxRepetitions: options.limits.loopMaxRepetitions,
35
+ rateLimitDelay: 0,
36
+ logLevel: 'off',
37
+ };
38
+ this.usageTracker = options.usageTracker ?? new ScenarioUsageTracker(options.limits.budgetTokens);
39
+ this.toolRegistry = new ToolRegistry({ allowedTools: options.allowedTools });
40
+ this.toolRegistry.register(createStepResultTools());
41
+ const tools = validateDriverSession(options.session, options.descriptor, options.allowedTools);
42
+ this.toolRegistry.register(tools.map((tool) => adaptDriverTool(options.driverId, tool)));
43
+ this.aiClient =
44
+ options.aiClient ??
45
+ new AiClient({
46
+ config: this.config,
47
+ toolRegistry: this.toolRegistry,
48
+ apiKey: options.apiKey,
49
+ modelEgress: options.modelEgress,
50
+ exactSecrets,
51
+ logger: this.runtimeLogger,
52
+ });
38
53
  }
39
- /**
40
- * Releases any runner-owned resources.
41
- *
42
- * @example
43
- * ```ts
44
- * await ai.teardown()
45
- * ```
46
- */
47
- async teardown() {
48
- await this.extensionHost.teardown();
49
- }
50
- /**
51
- * Executes one natural-language test step.
52
- *
53
- * @param step - The step definition to execute.
54
- *
55
- * @example
56
- * ```ts
57
- * await runner.run({
58
- * action: 'Search for qwen3-vl',
59
- * expect: 'The qwen3-vl model page is displayed',
60
- * topPercent: 10,
61
- * })
62
- * ```
63
- */
64
- async run(step) {
65
- await new StepExecution(this.aiClient, this.extensionHost).run(step);
54
+ async teardown() { }
55
+ async run(step, control) {
56
+ return new StepExecution({
57
+ config: this.config,
58
+ aiClient: this.aiClient,
59
+ toolRegistry: this.toolRegistry,
60
+ driverSession: this.options.session,
61
+ usageTracker: this.usageTracker,
62
+ driverId: this.options.driverId,
63
+ redact: this.config.redact,
64
+ diagnosticSanitizer: this.diagnosticSanitizer,
65
+ logger: this.runtimeLogger,
66
+ }).run(step, control);
66
67
  }
67
68
  }
68
- /**
69
- * Creates a runner from the provided extensions.
70
- *
71
- * This is the main programmatic entry point from `@xoxoai/checkmate/core`.
72
- *
73
- * @example
74
- * ```ts
75
- * import { createRunner } from '@xoxoai/checkmate/core'
76
- * import { web } from '@xoxoai/checkmate/playwright'
77
- *
78
- * const ai = createRunner({
79
- * extensions: [web({ page })],
80
- * })
81
- * ```
82
- */
83
- export function createRunner(options = {}) {
69
+ export function createDriverRunner(options) {
84
70
  return new CheckmateRunner(options);
85
71
  }
86
72
  //# sourceMappingURL=runner.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"runner.js","sourceRoot":"","sources":["../../src/runtime/runner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAA;AAE3D,OAAO,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAA;AACpE,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAA;AACnD,OAAO,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAA;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAA;AACnD,OAAO,EAAsB,aAAa,EAAE,MAAM,gBAAgB,CAAA;AA2BlE;;;;;;;;;;;GAWG;AACH,MAAM,OAAO,eAAe;IACV,QAAQ,CAAU;IAClB,aAAa,CAAe;IAE7C;;;;;;;;;OASG;IACH,YAAY,UAAkC,EAAE;QAC/C,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,IAAI,aAAa,EAAE,CAAA;QAClE,MAAM,YAAY,GAAG,IAAI,YAAY,CAAC,aAAa,CAAC,CAAA;QACpD,YAAY,CAAC,QAAQ,CAAC,qBAAqB,EAAE,CAAC,CAAA;QAC9C,IAAI,CAAC,aAAa,GAAG,IAAI,aAAa,CAAC,aAAa,EAAE,YAAY,EAAE,OAAO,CAAC,UAAU,IAAI,EAAE,CAAC,CAAA;QAC7F,IAAI,CAAC,QAAQ,GAAG,IAAI,QAAQ,CAAC,EAAE,aAAa,EAAE,YAAY,EAAE,aAAa,EAAE,IAAI,CAAC,aAAa,EAAE,CAAC,CAAA;IACjG,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,QAAQ;QACb,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,CAAA;IACpC,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,KAAK,CAAC,GAAG,CAAC,IAAU;QACnB,MAAM,IAAI,aAAa,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IACrE,CAAC;CACD;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,YAAY,CAAC,UAAkC,EAAE;IAChE,OAAO,IAAI,eAAe,CAAC,OAAO,CAAC,CAAA;AACpC,CAAC"}
1
+ {"version":3,"file":"runner.js","sourceRoot":"","sources":["../../src/runtime/runner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAA;AAC5D,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAA;AAIrD,OAAO,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAA;AACpE,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAA;AACnD,OAAO,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAA;AAC1C,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAA;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAA;AACnD,OAAO,EAAE,sBAAsB,EAAE,MAAM,iCAAiC,CAAA;AACxE,OAAO,EAAE,YAAY,EAAsB,MAAM,qBAAqB,CAAA;AACtE,OAAO,EAAE,mBAAmB,EAAE,MAAM,sCAAsC,CAAA;AAuB1E,MAAM,OAAO,eAAe;IAQE;IAPZ,MAAM,CAAe;IACrB,YAAY,CAAc;IAC1B,QAAQ,CAAU;IAClB,YAAY,CAAsB;IAClC,mBAAmB,CAAqB;IACxC,aAAa,CAAe;IAE7C,YAA6B,OAAqC;QAArC,YAAO,GAAP,OAAO,CAA8B;QACjE,MAAM,YAAY,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,OAAO,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC,CAAA;QACtE,IAAI,CAAC,mBAAmB,GAAG,IAAI,mBAAmB,CAAC,YAAY,CAAC,CAAA;QAChE,IAAI,CAAC,aAAa,GAAG,sBAAsB,CAAC,OAAO,CAAC,MAAM,IAAI,YAAY,EAAE,IAAI,CAAC,mBAAmB,CAAC,CAAA;QACrG,IAAI,CAAC,MAAM,GAAG;YACb,KAAK,EAAE,OAAO,CAAC,WAAW,CAAC,QAAQ,CAAC,KAAK;YACzC,OAAO,EAAE,OAAO,CAAC,WAAW,CAAC,QAAQ,CAAC,OAAO;YAC7C,eAAe,EAAE,OAAO,CAAC,WAAW,CAAC,QAAQ,CAAC,eAAe;YAC7D,WAAW,EAAE,OAAO,CAAC,WAAW,CAAC,QAAQ,CAAC,WAAW,IAAI,CAAC;YAC1D,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,YAAY;YACpC,MAAM,EAAE,OAAO,CAAC,WAAW,CAAC,aAAa,KAAK,IAAI;YAClD,UAAU,EAAE,UAAU;YACtB,UAAU,EAAE,OAAO,CAAC,MAAM,CAAC,UAAU;YACrC,cAAc,EAAE,OAAO,CAAC,MAAM,CAAC,gBAAgB;YAC/C,kBAAkB,EAAE,OAAO,CAAC,MAAM,CAAC,kBAAkB;YACrD,cAAc,EAAE,CAAC;YACjB,QAAQ,EAAE,KAAK;SACf,CAAA;QACD,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,IAAI,oBAAoB,CAAC,OAAO,CAAC,MAAM,CAAC,YAAY,CAAC,CAAA;QACjG,IAAI,CAAC,YAAY,GAAG,IAAI,YAAY,CAAC,EAAE,YAAY,EAAE,OAAO,CAAC,YAAY,EAAE,CAAC,CAAA;QAC5E,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,qBAAqB,EAAE,CAAC,CAAA;QACnD,MAAM,KAAK,GAAG,qBAAqB,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,UAAU,EAAE,OAAO,CAAC,YAAY,CAAC,CAAA;QAC9F,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,eAAe,CAAC,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC,CAAA;QACxF,IAAI,CAAC,QAAQ;YACZ,OAAO,CAAC,QAAQ;gBAChB,IAAI,QAAQ,CAAC;oBACZ,MAAM,EAAE,IAAI,CAAC,MAAM;oBACnB,YAAY,EAAE,IAAI,CAAC,YAAY;oBAC/B,MAAM,EAAE,OAAO,CAAC,MAAM;oBACtB,WAAW,EAAE,OAAO,CAAC,WAAW;oBAChC,YAAY;oBACZ,MAAM,EAAE,IAAI,CAAC,aAAa;iBAC1B,CAAC,CAAA;IACJ,CAAC;IAED,KAAK,CAAC,QAAQ,KAAmB,CAAC;IAElC,KAAK,CAAC,GAAG,CAAC,IAAgB,EAAE,OAAoB;QAC/C,OAAO,IAAI,aAAa,CAAC;YACxB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,YAAY,EAAE,IAAI,CAAC,YAAY;YAC/B,aAAa,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO;YACnC,YAAY,EAAE,IAAI,CAAC,YAAY;YAC/B,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ;YAC/B,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM;YAC1B,mBAAmB,EAAE,IAAI,CAAC,mBAAmB;YAC7C,MAAM,EAAE,IAAI,CAAC,aAAa;SAC1B,CAAC,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;IACtB,CAAC;CACD;AAED,MAAM,UAAU,kBAAkB,CAAC,OAAqC;IACvE,OAAO,IAAI,eAAe,CAAC,OAAO,CAAC,CAAA;AACpC,CAAC"}
@@ -0,0 +1,40 @@
1
+ export type ControlExpirationReason = 'step-timeout' | 'scenario-timeout' | 'interrupted';
2
+ export type ControlPoll = {
3
+ expired: false;
4
+ } | {
5
+ expired: true;
6
+ reason: ControlExpirationReason;
7
+ };
8
+ export type StepControl = Readonly<{
9
+ signal: AbortSignal;
10
+ deadline: number;
11
+ now(): number;
12
+ poll(): ControlPoll;
13
+ dispose(): void;
14
+ }>;
15
+ export type ScenarioControlOptions = Readonly<{
16
+ timeoutMs: number;
17
+ signal?: AbortSignal;
18
+ now?: () => number;
19
+ setTimer?: typeof setTimeout;
20
+ clearTimer?: typeof clearTimeout;
21
+ }>;
22
+ export declare class ScenarioControl {
23
+ readonly signal: AbortSignal;
24
+ readonly interruptionSignal: AbortSignal;
25
+ readonly deadline: number;
26
+ private readonly controller;
27
+ private readonly interruptionController;
28
+ private readonly now;
29
+ private readonly setTimer;
30
+ private readonly clearTimer;
31
+ private readonly timer;
32
+ private readonly externalSignal?;
33
+ private disposed;
34
+ constructor(options: ScenarioControlOptions);
35
+ poll(): ControlPoll;
36
+ createStepControl(stepTimeoutMs: number): StepControl;
37
+ dispose(): void;
38
+ private readonly handleExternalAbort;
39
+ }
40
+ //# sourceMappingURL=scenario-control.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"scenario-control.d.ts","sourceRoot":"","sources":["../../src/runtime/scenario-control.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,uBAAuB,GAAG,cAAc,GAAG,kBAAkB,GAAG,aAAa,CAAA;AAEzF,MAAM,MAAM,WAAW,GAAG;IAAE,OAAO,EAAE,KAAK,CAAA;CAAE,GAAG;IAAE,OAAO,EAAE,IAAI,CAAC;IAAC,MAAM,EAAE,uBAAuB,CAAA;CAAE,CAAA;AAEjG,MAAM,MAAM,WAAW,GAAG,QAAQ,CAAC;IAClC,MAAM,EAAE,WAAW,CAAA;IACnB,QAAQ,EAAE,MAAM,CAAA;IAChB,GAAG,IAAI,MAAM,CAAA;IACb,IAAI,IAAI,WAAW,CAAA;IACnB,OAAO,IAAI,IAAI,CAAA;CACf,CAAC,CAAA;AAEF,MAAM,MAAM,sBAAsB,GAAG,QAAQ,CAAC;IAC7C,SAAS,EAAE,MAAM,CAAA;IACjB,MAAM,CAAC,EAAE,WAAW,CAAA;IACpB,GAAG,CAAC,EAAE,MAAM,MAAM,CAAA;IAClB,QAAQ,CAAC,EAAE,OAAO,UAAU,CAAA;IAC5B,UAAU,CAAC,EAAE,OAAO,YAAY,CAAA;CAChC,CAAC,CAAA;AAEF,qBAAa,eAAe;IAC3B,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAA;IAC5B,QAAQ,CAAC,kBAAkB,EAAE,WAAW,CAAA;IACxC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAwB;IACnD,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAwB;IAC/D,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAc;IAClC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAmB;IAC5C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAqB;IAChD,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA+B;IACrD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAa;IAC7C,OAAO,CAAC,QAAQ,CAAQ;gBAEZ,OAAO,EAAE,sBAAsB;IAgB3C,IAAI,IAAI,WAAW;IAYnB,iBAAiB,CAAC,aAAa,EAAE,MAAM,GAAG,WAAW;IAuCrD,OAAO,IAAI,IAAI;IAOf,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAGnC;CACD"}