@rpamis/comet 0.4.0-beta.2 → 0.4.0-beta.20

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (1732) hide show
  1. package/README.md +361 -136
  2. package/assets/eval-reports/comet-native-vs-beta16-beta17-20260810/native-benchmark-report.html +8 -0
  3. package/assets/eval-reports/comet-native-vs-beta16-beta17-20260810/native-benchmark-report.json +286 -0
  4. package/assets/manifest.json +53 -16
  5. package/assets/skills/comet/SKILL.md +16 -256
  6. package/assets/skills/comet/agents/openai.yaml +3 -0
  7. package/assets/skills/comet/rules/comet-phase-guard.en.md +39 -35
  8. package/assets/skills/comet/rules/comet-phase-guard.md +38 -35
  9. package/assets/skills/comet/rules/comet-workflow-guard.en.md +33 -0
  10. package/assets/skills/comet/rules/comet-workflow-guard.md +33 -0
  11. package/assets/skills/comet/scripts/comet-archive.mjs +165 -2
  12. package/assets/skills/comet/scripts/comet-entry-runtime.mjs +149 -0
  13. package/assets/skills/comet/scripts/comet-guard.mjs +207 -2
  14. package/assets/skills/comet/scripts/comet-handoff.mjs +165 -2
  15. package/assets/skills/comet/scripts/comet-hook-guard.mjs +153 -2
  16. package/assets/skills/comet/scripts/comet-hook-router.mjs +168 -0
  17. package/assets/skills/comet/scripts/comet-intent.mjs +10 -2
  18. package/assets/skills/comet/scripts/comet-resume-probe.mjs +149 -0
  19. package/assets/skills/comet/scripts/comet-runtime.mjs +293 -12935
  20. package/assets/skills/comet/scripts/comet-state.mjs +165 -2
  21. package/assets/skills/comet/scripts/comet-yaml-validate.mjs +149 -2
  22. package/assets/skills/comet-any/SKILL.md +7 -6
  23. package/assets/skills/comet-any/agents/openai.yaml +5 -0
  24. package/assets/skills/comet-any/reference/authored-zone-example.md +10 -5
  25. package/assets/skills/comet-any/reference/authoring-subagents.md +4 -12
  26. package/assets/skills/comet-any/reference/bundle-authoring.md +2 -2
  27. package/assets/skills/comet-any/reference/subagents/pause-points-author.md +15 -15
  28. package/assets/skills/comet-any/reference/subagents/reference-author.md +2 -5
  29. package/assets/skills/comet-any/reference/subagents/script-author.md +2 -5
  30. package/assets/skills/comet-any/reference/subagents/skill-core-author.md +2 -5
  31. package/assets/skills/comet-any/reference/subagents/skill-reviewer.md +5 -6
  32. package/assets/skills/comet-any/reference/subagents/workflow-entry-author.md +9 -11
  33. package/assets/skills/comet-archive/SKILL.md +69 -26
  34. package/assets/skills/comet-archive/agents/openai.yaml +5 -0
  35. package/assets/skills/comet-build/SKILL.md +94 -102
  36. package/assets/skills/comet-build/agents/openai.yaml +5 -0
  37. package/assets/skills/comet-classic/SKILL.md +301 -0
  38. package/assets/skills/comet-classic/agents/openai.yaml +3 -0
  39. package/assets/skills/comet-classic/reference/auto-transition.md +27 -0
  40. package/assets/skills/comet-classic/reference/classic-layout.md +34 -0
  41. package/assets/skills/comet-classic/reference/comet-yaml-fields.md +81 -0
  42. package/assets/skills/comet-classic/reference/context-recovery.md +55 -0
  43. package/assets/skills/{comet → comet-classic}/reference/debug-gate.md +2 -2
  44. package/assets/skills/comet-classic/reference/decision-point.md +41 -0
  45. package/assets/skills/comet-classic/reference/dirty-worktree.md +59 -0
  46. package/assets/skills/comet-classic/reference/file-structure.md +28 -0
  47. package/assets/skills/{comet → comet-classic}/reference/intent-frame.md +4 -4
  48. package/assets/skills/comet-classic/reference/scripts.md +73 -0
  49. package/assets/skills/{comet → comet-classic}/reference/subagent-dispatch.md +13 -14
  50. package/assets/skills/comet-classic/reference/workspace.md +23 -0
  51. package/assets/skills/comet-design/SKILL.md +46 -42
  52. package/assets/skills/comet-design/agents/openai.yaml +5 -0
  53. package/assets/skills/comet-hotfix/SKILL.md +62 -40
  54. package/assets/skills/comet-hotfix/agents/openai.yaml +5 -0
  55. package/assets/skills/comet-native/SKILL.md +108 -0
  56. package/assets/skills/comet-native/agents/openai.yaml +3 -0
  57. package/assets/skills/comet-native/reference/artifacts.md +71 -0
  58. package/assets/skills/comet-native/reference/clarification.md +79 -0
  59. package/assets/skills/comet-native/reference/commands.md +46 -0
  60. package/assets/skills/comet-native/reference/recovery.md +67 -0
  61. package/assets/skills/comet-native/reference/workspace.md +23 -0
  62. package/assets/skills/comet-native/scripts/comet-native-archive.mjs +217 -0
  63. package/assets/skills/comet-native/scripts/comet-native-doctor.mjs +263 -0
  64. package/assets/skills/comet-native/scripts/comet-native-hook-guard.mjs +167 -0
  65. package/assets/skills/comet-native/scripts/comet-native-init.mjs +181 -0
  66. package/assets/skills/comet-native/scripts/comet-native-new.mjs +207 -0
  67. package/assets/skills/comet-native/scripts/comet-native-next.mjs +213 -0
  68. package/assets/skills/comet-native/scripts/comet-native-root.mjs +190 -0
  69. package/assets/skills/comet-native/scripts/comet-native-runtime.mjs +299 -0
  70. package/assets/skills/comet-native/scripts/comet-native-select.mjs +185 -0
  71. package/assets/skills/comet-native/scripts/comet-native-show.mjs +158 -0
  72. package/assets/skills/comet-native/scripts/comet-native-spec.mjs +163 -0
  73. package/assets/skills/comet-native/scripts/comet-native-status.mjs +182 -0
  74. package/assets/skills/comet-open/SKILL.md +140 -61
  75. package/assets/skills/comet-open/agents/openai.yaml +5 -0
  76. package/assets/skills/comet-review/SKILL.md +121 -0
  77. package/assets/skills/comet-review/agents/openai.yaml +5 -0
  78. package/assets/skills/comet-tweak/SKILL.md +57 -32
  79. package/assets/skills/comet-tweak/agents/openai.yaml +5 -0
  80. package/assets/skills/comet-verify/SKILL.md +69 -77
  81. package/assets/skills/comet-verify/agents/openai.yaml +5 -0
  82. package/assets/skills-zh/comet/SKILL.md +15 -244
  83. package/assets/skills-zh/comet/agents/openai.yaml +3 -0
  84. package/assets/skills-zh/comet-any/SKILL.md +7 -6
  85. package/assets/skills-zh/comet-any/agents/openai.yaml +5 -0
  86. package/assets/skills-zh/comet-any/reference/authored-zone-example.md +10 -5
  87. package/assets/skills-zh/comet-any/reference/authoring-subagents.md +4 -15
  88. package/assets/skills-zh/comet-any/reference/bundle-authoring.md +2 -2
  89. package/assets/skills-zh/comet-any/reference/subagents/pause-points-author.md +15 -16
  90. package/assets/skills-zh/comet-any/reference/subagents/reference-author.md +2 -5
  91. package/assets/skills-zh/comet-any/reference/subagents/script-author.md +2 -5
  92. package/assets/skills-zh/comet-any/reference/subagents/skill-core-author.md +2 -5
  93. package/assets/skills-zh/comet-any/reference/subagents/skill-reviewer.md +5 -6
  94. package/assets/skills-zh/comet-any/reference/subagents/workflow-entry-author.md +9 -11
  95. package/assets/skills-zh/comet-archive/SKILL.md +69 -26
  96. package/assets/skills-zh/comet-archive/agents/openai.yaml +5 -0
  97. package/assets/skills-zh/comet-build/SKILL.md +94 -102
  98. package/assets/skills-zh/comet-build/agents/openai.yaml +5 -0
  99. package/assets/skills-zh/comet-classic/SKILL.md +289 -0
  100. package/assets/skills-zh/comet-classic/agents/openai.yaml +3 -0
  101. package/assets/skills-zh/comet-classic/reference/auto-transition.md +27 -0
  102. package/assets/skills-zh/comet-classic/reference/classic-layout.md +34 -0
  103. package/assets/skills-zh/comet-classic/reference/comet-yaml-fields.md +80 -0
  104. package/assets/skills-zh/comet-classic/reference/context-recovery.md +55 -0
  105. package/assets/skills-zh/comet-classic/reference/debug-gate.md +33 -0
  106. package/assets/skills-zh/comet-classic/reference/decision-point.md +41 -0
  107. package/assets/skills-zh/comet-classic/reference/dirty-worktree.md +59 -0
  108. package/assets/skills-zh/comet-classic/reference/file-structure.md +28 -0
  109. package/assets/skills-zh/comet-classic/reference/intent-frame.md +74 -0
  110. package/assets/skills-zh/comet-classic/reference/scripts.md +73 -0
  111. package/assets/skills-zh/comet-classic/reference/subagent-dispatch.md +168 -0
  112. package/assets/skills-zh/comet-classic/reference/workspace.md +23 -0
  113. package/assets/skills-zh/comet-design/SKILL.md +46 -42
  114. package/assets/skills-zh/comet-design/agents/openai.yaml +5 -0
  115. package/assets/skills-zh/comet-hotfix/SKILL.md +61 -39
  116. package/assets/skills-zh/comet-hotfix/agents/openai.yaml +5 -0
  117. package/assets/skills-zh/comet-native/SKILL.md +108 -0
  118. package/assets/skills-zh/comet-native/agents/openai.yaml +3 -0
  119. package/assets/skills-zh/comet-native/reference/artifacts.md +71 -0
  120. package/assets/skills-zh/comet-native/reference/clarification.md +79 -0
  121. package/assets/skills-zh/comet-native/reference/commands.md +46 -0
  122. package/assets/skills-zh/comet-native/reference/recovery.md +67 -0
  123. package/assets/skills-zh/comet-native/reference/workspace.md +23 -0
  124. package/assets/skills-zh/comet-open/SKILL.md +139 -60
  125. package/assets/skills-zh/comet-open/agents/openai.yaml +5 -0
  126. package/assets/skills-zh/comet-review/SKILL.md +121 -0
  127. package/assets/skills-zh/comet-review/agents/openai.yaml +5 -0
  128. package/assets/skills-zh/comet-tweak/SKILL.md +56 -31
  129. package/assets/skills-zh/comet-tweak/agents/openai.yaml +5 -0
  130. package/assets/skills-zh/comet-verify/SKILL.md +69 -77
  131. package/assets/skills-zh/comet-verify/agents/openai.yaml +5 -0
  132. package/bin/comet.js +5 -1
  133. package/bin/fast-runtime-router.js +79 -0
  134. package/dist/app/cli/comet-banner.d.ts +25 -0
  135. package/dist/app/cli/comet-banner.d.ts.map +1 -0
  136. package/dist/app/cli/comet-banner.js +265 -0
  137. package/dist/app/cli/comet-banner.js.map +1 -0
  138. package/dist/app/cli/index.js +184 -28
  139. package/dist/app/cli/index.js.map +1 -1
  140. package/dist/app/commands/classic.d.ts +5 -0
  141. package/dist/app/commands/classic.d.ts.map +1 -0
  142. package/dist/app/commands/classic.js +34 -0
  143. package/dist/app/commands/classic.js.map +1 -0
  144. package/dist/app/commands/command-result.d.ts +6 -0
  145. package/dist/app/commands/command-result.d.ts.map +1 -0
  146. package/dist/app/commands/command-result.js +4 -0
  147. package/dist/app/commands/command-result.js.map +1 -0
  148. package/dist/app/commands/dashboard.d.ts.map +1 -1
  149. package/dist/app/commands/dashboard.js +2 -1
  150. package/dist/app/commands/dashboard.js.map +1 -1
  151. package/dist/app/commands/doctor.d.ts +4 -0
  152. package/dist/app/commands/doctor.d.ts.map +1 -1
  153. package/dist/app/commands/doctor.js +1066 -95
  154. package/dist/app/commands/doctor.js.map +1 -1
  155. package/dist/app/commands/eval.d.ts +10 -1
  156. package/dist/app/commands/eval.d.ts.map +1 -1
  157. package/dist/app/commands/eval.js +354 -51
  158. package/dist/app/commands/eval.js.map +1 -1
  159. package/dist/app/commands/i18n.d.ts +1 -1
  160. package/dist/app/commands/i18n.d.ts.map +1 -1
  161. package/dist/app/commands/i18n.js +118 -6
  162. package/dist/app/commands/i18n.js.map +1 -1
  163. package/dist/app/commands/init.d.ts +13 -3
  164. package/dist/app/commands/init.d.ts.map +1 -1
  165. package/dist/app/commands/init.js +627 -82
  166. package/dist/app/commands/init.js.map +1 -1
  167. package/dist/app/commands/native.d.ts +2 -0
  168. package/dist/app/commands/native.d.ts.map +1 -0
  169. package/dist/app/commands/native.js +10 -0
  170. package/dist/app/commands/native.js.map +1 -0
  171. package/dist/app/commands/project-scope-selection.d.ts +14 -0
  172. package/dist/app/commands/project-scope-selection.d.ts.map +1 -0
  173. package/dist/app/commands/project-scope-selection.js +33 -0
  174. package/dist/app/commands/project-scope-selection.js.map +1 -0
  175. package/dist/app/commands/resume-probe.d.ts +12 -0
  176. package/dist/app/commands/resume-probe.d.ts.map +1 -0
  177. package/dist/app/commands/resume-probe.js +68 -0
  178. package/dist/app/commands/resume-probe.js.map +1 -0
  179. package/dist/app/commands/status.d.ts.map +1 -1
  180. package/dist/app/commands/status.js +70 -104
  181. package/dist/app/commands/status.js.map +1 -1
  182. package/dist/app/commands/uninstall.d.ts +11 -0
  183. package/dist/app/commands/uninstall.d.ts.map +1 -1
  184. package/dist/app/commands/uninstall.js +526 -86
  185. package/dist/app/commands/uninstall.js.map +1 -1
  186. package/dist/app/commands/update.d.ts +30 -5
  187. package/dist/app/commands/update.d.ts.map +1 -1
  188. package/dist/app/commands/update.js +1641 -123
  189. package/dist/app/commands/update.js.map +1 -1
  190. package/dist/app/commands/workflow.d.ts +7 -0
  191. package/dist/app/commands/workflow.d.ts.map +1 -0
  192. package/dist/app/commands/workflow.js +18 -0
  193. package/dist/app/commands/workflow.js.map +1 -0
  194. package/dist/config/repository-layout.json +48 -3
  195. package/dist/domains/bundle/bundle-platform.d.ts +1 -0
  196. package/dist/domains/bundle/bundle-platform.d.ts.map +1 -1
  197. package/dist/domains/bundle/bundle-platform.js +18 -4
  198. package/dist/domains/bundle/bundle-platform.js.map +1 -1
  199. package/dist/domains/bundle/eval-manifest-runtime.d.ts +8 -0
  200. package/dist/domains/bundle/eval-manifest-runtime.d.ts.map +1 -0
  201. package/dist/domains/bundle/eval-manifest-runtime.js +118 -0
  202. package/dist/domains/bundle/eval-manifest-runtime.js.map +1 -0
  203. package/dist/domains/bundle/eval-run-result.d.ts +22 -0
  204. package/dist/domains/bundle/eval-run-result.d.ts.map +1 -0
  205. package/dist/domains/bundle/eval-run-result.js +207 -0
  206. package/dist/domains/bundle/eval-run-result.js.map +1 -0
  207. package/dist/domains/bundle/eval.d.ts.map +1 -1
  208. package/dist/domains/bundle/eval.js +21 -24
  209. package/dist/domains/bundle/eval.js.map +1 -1
  210. package/dist/domains/bundle/factory.d.ts.map +1 -1
  211. package/dist/domains/bundle/factory.js +2 -9
  212. package/dist/domains/bundle/factory.js.map +1 -1
  213. package/dist/domains/bundle/review-summary.js +1 -1
  214. package/dist/domains/bundle/review-summary.js.map +1 -1
  215. package/dist/domains/bundle/state.d.ts.map +1 -1
  216. package/dist/domains/bundle/state.js +117 -2
  217. package/dist/domains/bundle/state.js.map +1 -1
  218. package/dist/domains/bundle/types.d.ts +1 -1
  219. package/dist/domains/bundle/types.d.ts.map +1 -1
  220. package/dist/domains/comet-classic/classic-archive-entry.js +1 -1
  221. package/dist/domains/comet-classic/classic-archive-entry.js.map +1 -1
  222. package/dist/domains/comet-classic/classic-archive-pointer.d.ts +8 -0
  223. package/dist/domains/comet-classic/classic-archive-pointer.d.ts.map +1 -0
  224. package/dist/domains/comet-classic/classic-archive-pointer.js +66 -0
  225. package/dist/domains/comet-classic/classic-archive-pointer.js.map +1 -0
  226. package/dist/domains/comet-classic/classic-archive.d.ts +1 -0
  227. package/dist/domains/comet-classic/classic-archive.d.ts.map +1 -1
  228. package/dist/domains/comet-classic/classic-archive.js +205 -114
  229. package/dist/domains/comet-classic/classic-archive.js.map +1 -1
  230. package/dist/domains/comet-classic/classic-branch-binding.d.ts +51 -0
  231. package/dist/domains/comet-classic/classic-branch-binding.d.ts.map +1 -0
  232. package/dist/domains/comet-classic/classic-branch-binding.js +113 -0
  233. package/dist/domains/comet-classic/classic-branch-binding.js.map +1 -0
  234. package/dist/domains/comet-classic/classic-cli.d.ts +3 -1
  235. package/dist/domains/comet-classic/classic-cli.d.ts.map +1 -1
  236. package/dist/domains/comet-classic/classic-cli.js +15 -3
  237. package/dist/domains/comet-classic/classic-cli.js.map +1 -1
  238. package/dist/domains/comet-classic/classic-command-checks.d.ts +20 -0
  239. package/dist/domains/comet-classic/classic-command-checks.d.ts.map +1 -0
  240. package/dist/domains/comet-classic/classic-command-checks.js +94 -0
  241. package/dist/domains/comet-classic/classic-command-checks.js.map +1 -0
  242. package/dist/domains/comet-classic/classic-command-context.d.ts +21 -0
  243. package/dist/domains/comet-classic/classic-command-context.d.ts.map +1 -0
  244. package/dist/domains/comet-classic/classic-command-context.js +52 -0
  245. package/dist/domains/comet-classic/classic-command-context.js.map +1 -0
  246. package/dist/domains/comet-classic/classic-current-change.d.ts +17 -0
  247. package/dist/domains/comet-classic/classic-current-change.d.ts.map +1 -0
  248. package/dist/domains/comet-classic/classic-current-change.js +143 -0
  249. package/dist/domains/comet-classic/classic-current-change.js.map +1 -0
  250. package/dist/domains/comet-classic/classic-diagnostics.d.ts +2 -1
  251. package/dist/domains/comet-classic/classic-diagnostics.d.ts.map +1 -1
  252. package/dist/domains/comet-classic/classic-diagnostics.js +54 -1
  253. package/dist/domains/comet-classic/classic-diagnostics.js.map +1 -1
  254. package/dist/domains/comet-classic/classic-evidence.d.ts +1 -0
  255. package/dist/domains/comet-classic/classic-evidence.d.ts.map +1 -1
  256. package/dist/domains/comet-classic/classic-evidence.js +140 -40
  257. package/dist/domains/comet-classic/classic-evidence.js.map +1 -1
  258. package/dist/domains/comet-classic/classic-guard-entry.js +1 -1
  259. package/dist/domains/comet-classic/classic-guard-entry.js.map +1 -1
  260. package/dist/domains/comet-classic/classic-guard.d.ts.map +1 -1
  261. package/dist/domains/comet-classic/classic-guard.js +230 -209
  262. package/dist/domains/comet-classic/classic-guard.js.map +1 -1
  263. package/dist/domains/comet-classic/classic-handoff-entry.js +1 -1
  264. package/dist/domains/comet-classic/classic-handoff-entry.js.map +1 -1
  265. package/dist/domains/comet-classic/classic-handoff.d.ts.map +1 -1
  266. package/dist/domains/comet-classic/classic-handoff.js +195 -109
  267. package/dist/domains/comet-classic/classic-handoff.js.map +1 -1
  268. package/dist/domains/comet-classic/classic-hook-guard-entry.js +1 -1
  269. package/dist/domains/comet-classic/classic-hook-guard-entry.js.map +1 -1
  270. package/dist/domains/comet-classic/classic-hook-guard.d.ts +13 -0
  271. package/dist/domains/comet-classic/classic-hook-guard.d.ts.map +1 -1
  272. package/dist/domains/comet-classic/classic-hook-guard.js +448 -75
  273. package/dist/domains/comet-classic/classic-hook-guard.js.map +1 -1
  274. package/dist/domains/comet-classic/classic-intent-command.d.ts.map +1 -1
  275. package/dist/domains/comet-classic/classic-intent-command.js +2 -1
  276. package/dist/domains/comet-classic/classic-intent-command.js.map +1 -1
  277. package/dist/domains/comet-classic/classic-intent-entry.js +1 -1
  278. package/dist/domains/comet-classic/classic-intent-entry.js.map +1 -1
  279. package/dist/domains/comet-classic/classic-layout-initialization.d.ts +87 -0
  280. package/dist/domains/comet-classic/classic-layout-initialization.d.ts.map +1 -0
  281. package/dist/domains/comet-classic/classic-layout-initialization.js +828 -0
  282. package/dist/domains/comet-classic/classic-layout-initialization.js.map +1 -0
  283. package/dist/domains/comet-classic/classic-layout.d.ts +41 -0
  284. package/dist/domains/comet-classic/classic-layout.d.ts.map +1 -0
  285. package/dist/domains/comet-classic/classic-layout.js +222 -0
  286. package/dist/domains/comet-classic/classic-layout.js.map +1 -0
  287. package/dist/domains/comet-classic/classic-migrate.d.ts +4 -0
  288. package/dist/domains/comet-classic/classic-migrate.d.ts.map +1 -1
  289. package/dist/domains/comet-classic/classic-migrate.js +8 -12
  290. package/dist/domains/comet-classic/classic-migrate.js.map +1 -1
  291. package/dist/domains/comet-classic/classic-openspec-command.d.ts +4 -0
  292. package/dist/domains/comet-classic/classic-openspec-command.d.ts.map +1 -0
  293. package/dist/domains/comet-classic/classic-openspec-command.js +45 -0
  294. package/dist/domains/comet-classic/classic-openspec-command.js.map +1 -0
  295. package/dist/domains/comet-classic/classic-openspec-root.d.ts +13 -0
  296. package/dist/domains/comet-classic/classic-openspec-root.d.ts.map +1 -0
  297. package/dist/domains/comet-classic/classic-openspec-root.js +47 -0
  298. package/dist/domains/comet-classic/classic-openspec-root.js.map +1 -0
  299. package/dist/domains/comet-classic/classic-paths.d.ts +11 -1
  300. package/dist/domains/comet-classic/classic-paths.d.ts.map +1 -1
  301. package/dist/domains/comet-classic/classic-paths.js +87 -35
  302. package/dist/domains/comet-classic/classic-paths.js.map +1 -1
  303. package/dist/domains/comet-classic/classic-plan-readiness.d.ts +12 -0
  304. package/dist/domains/comet-classic/classic-plan-readiness.d.ts.map +1 -0
  305. package/dist/domains/comet-classic/classic-plan-readiness.js +26 -0
  306. package/dist/domains/comet-classic/classic-plan-readiness.js.map +1 -0
  307. package/dist/domains/comet-classic/classic-project-config.d.ts +17 -0
  308. package/dist/domains/comet-classic/classic-project-config.d.ts.map +1 -0
  309. package/dist/domains/comet-classic/classic-project-config.js +42 -0
  310. package/dist/domains/comet-classic/classic-project-config.js.map +1 -0
  311. package/dist/domains/comet-classic/classic-protected-path.d.ts +39 -0
  312. package/dist/domains/comet-classic/classic-protected-path.d.ts.map +1 -0
  313. package/dist/domains/comet-classic/classic-protected-path.js +69 -0
  314. package/dist/domains/comet-classic/classic-protected-path.js.map +1 -0
  315. package/dist/domains/comet-classic/classic-resolver.d.ts.map +1 -1
  316. package/dist/domains/comet-classic/classic-resolver.js +4 -4
  317. package/dist/domains/comet-classic/classic-resolver.js.map +1 -1
  318. package/dist/domains/comet-classic/classic-resume-probe-command.d.ts +3 -0
  319. package/dist/domains/comet-classic/classic-resume-probe-command.d.ts.map +1 -0
  320. package/dist/domains/comet-classic/classic-resume-probe-command.js +60 -0
  321. package/dist/domains/comet-classic/classic-resume-probe-command.js.map +1 -0
  322. package/dist/domains/comet-classic/classic-resume-probe-entry.d.ts +2 -0
  323. package/dist/domains/comet-classic/classic-resume-probe-entry.d.ts.map +1 -0
  324. package/dist/domains/comet-classic/classic-resume-probe-entry.js +4 -0
  325. package/dist/domains/comet-classic/classic-resume-probe-entry.js.map +1 -0
  326. package/dist/domains/comet-classic/classic-resume-probe.d.ts +29 -0
  327. package/dist/domains/comet-classic/classic-resume-probe.d.ts.map +1 -0
  328. package/dist/domains/comet-classic/classic-resume-probe.js +406 -0
  329. package/dist/domains/comet-classic/classic-resume-probe.js.map +1 -0
  330. package/dist/domains/comet-classic/classic-root-command.d.ts +8 -0
  331. package/dist/domains/comet-classic/classic-root-command.d.ts.map +1 -0
  332. package/dist/domains/comet-classic/classic-root-command.js +199 -0
  333. package/dist/domains/comet-classic/classic-root-command.js.map +1 -0
  334. package/dist/domains/comet-classic/classic-root-move.d.ts +118 -0
  335. package/dist/domains/comet-classic/classic-root-move.d.ts.map +1 -0
  336. package/dist/domains/comet-classic/classic-root-move.js +1640 -0
  337. package/dist/domains/comet-classic/classic-root-move.js.map +1 -0
  338. package/dist/domains/comet-classic/classic-runtime-run.d.ts +8 -1
  339. package/dist/domains/comet-classic/classic-runtime-run.d.ts.map +1 -1
  340. package/dist/domains/comet-classic/classic-runtime-run.js +327 -5
  341. package/dist/domains/comet-classic/classic-runtime-run.js.map +1 -1
  342. package/dist/domains/comet-classic/classic-script-entry.d.ts +2 -2
  343. package/dist/domains/comet-classic/classic-script-entry.d.ts.map +1 -1
  344. package/dist/domains/comet-classic/classic-script-entry.js +4 -3
  345. package/dist/domains/comet-classic/classic-script-entry.js.map +1 -1
  346. package/dist/domains/comet-classic/classic-state-command.d.ts.map +1 -1
  347. package/dist/domains/comet-classic/classic-state-command.js +431 -99
  348. package/dist/domains/comet-classic/classic-state-command.js.map +1 -1
  349. package/dist/domains/comet-classic/classic-state-entry.js +1 -1
  350. package/dist/domains/comet-classic/classic-state-entry.js.map +1 -1
  351. package/dist/domains/comet-classic/classic-state-events.d.ts +1 -1
  352. package/dist/domains/comet-classic/classic-state-events.d.ts.map +1 -1
  353. package/dist/domains/comet-classic/classic-state-events.js +3 -4
  354. package/dist/domains/comet-classic/classic-state-events.js.map +1 -1
  355. package/dist/domains/comet-classic/classic-state.d.ts +6 -4
  356. package/dist/domains/comet-classic/classic-state.d.ts.map +1 -1
  357. package/dist/domains/comet-classic/classic-state.js +20 -7
  358. package/dist/domains/comet-classic/classic-state.js.map +1 -1
  359. package/dist/domains/comet-classic/classic-store.d.ts +6 -1
  360. package/dist/domains/comet-classic/classic-store.d.ts.map +1 -1
  361. package/dist/domains/comet-classic/classic-store.js +36 -23
  362. package/dist/domains/comet-classic/classic-store.js.map +1 -1
  363. package/dist/domains/comet-classic/classic-transitions.d.ts +1 -1
  364. package/dist/domains/comet-classic/classic-transitions.d.ts.map +1 -1
  365. package/dist/domains/comet-classic/classic-transitions.js +37 -3
  366. package/dist/domains/comet-classic/classic-transitions.js.map +1 -1
  367. package/dist/domains/comet-classic/classic-validate-command.d.ts.map +1 -1
  368. package/dist/domains/comet-classic/classic-validate-command.js +40 -20
  369. package/dist/domains/comet-classic/classic-validate-command.js.map +1 -1
  370. package/dist/domains/comet-classic/classic-validate-entry.js +1 -1
  371. package/dist/domains/comet-classic/classic-validate-entry.js.map +1 -1
  372. package/dist/domains/comet-classic/classic-workspace-command.d.ts +3 -0
  373. package/dist/domains/comet-classic/classic-workspace-command.d.ts.map +1 -0
  374. package/dist/domains/comet-classic/classic-workspace-command.js +45 -0
  375. package/dist/domains/comet-classic/classic-workspace-command.js.map +1 -0
  376. package/dist/domains/comet-classic/classic-workspace.d.ts +38 -0
  377. package/dist/domains/comet-classic/classic-workspace.d.ts.map +1 -0
  378. package/dist/domains/comet-classic/classic-workspace.js +315 -0
  379. package/dist/domains/comet-classic/classic-workspace.js.map +1 -0
  380. package/dist/domains/comet-classic/index.d.ts +3 -0
  381. package/dist/domains/comet-classic/index.d.ts.map +1 -1
  382. package/dist/domains/comet-classic/index.js +3 -0
  383. package/dist/domains/comet-classic/index.js.map +1 -1
  384. package/dist/domains/comet-entry/current-selection-repair.d.ts +17 -0
  385. package/dist/domains/comet-entry/current-selection-repair.d.ts.map +1 -0
  386. package/dist/domains/comet-entry/current-selection-repair.js +19 -0
  387. package/dist/domains/comet-entry/current-selection-repair.js.map +1 -0
  388. package/dist/domains/comet-entry/current-selection.d.ts +28 -0
  389. package/dist/domains/comet-entry/current-selection.d.ts.map +1 -0
  390. package/dist/domains/comet-entry/current-selection.js +114 -0
  391. package/dist/domains/comet-entry/current-selection.js.map +1 -0
  392. package/dist/domains/comet-entry/entry-reads.d.ts +17 -0
  393. package/dist/domains/comet-entry/entry-reads.d.ts.map +1 -0
  394. package/dist/domains/comet-entry/entry-reads.js +20 -0
  395. package/dist/domains/comet-entry/entry-reads.js.map +1 -0
  396. package/dist/domains/comet-entry/entry-runtime-entry.d.ts +2 -0
  397. package/dist/domains/comet-entry/entry-runtime-entry.d.ts.map +1 -0
  398. package/dist/domains/comet-entry/entry-runtime-entry.js +3 -0
  399. package/dist/domains/comet-entry/entry-runtime-entry.js.map +1 -0
  400. package/dist/domains/comet-entry/entry-runtime.d.ts +7 -0
  401. package/dist/domains/comet-entry/entry-runtime.d.ts.map +1 -0
  402. package/dist/domains/comet-entry/entry-runtime.js +55 -0
  403. package/dist/domains/comet-entry/entry-runtime.js.map +1 -0
  404. package/dist/domains/comet-entry/hook-adapter.d.ts +6 -0
  405. package/dist/domains/comet-entry/hook-adapter.d.ts.map +1 -0
  406. package/dist/domains/comet-entry/hook-adapter.js +225 -0
  407. package/dist/domains/comet-entry/hook-adapter.js.map +1 -0
  408. package/dist/domains/comet-entry/hook-project-root.d.ts +3 -0
  409. package/dist/domains/comet-entry/hook-project-root.d.ts.map +1 -0
  410. package/dist/domains/comet-entry/hook-project-root.js +62 -0
  411. package/dist/domains/comet-entry/hook-project-root.js.map +1 -0
  412. package/dist/domains/comet-entry/hook-router-entry.d.ts +11 -0
  413. package/dist/domains/comet-entry/hook-router-entry.d.ts.map +1 -0
  414. package/dist/domains/comet-entry/hook-router-entry.js +121 -0
  415. package/dist/domains/comet-entry/hook-router-entry.js.map +1 -0
  416. package/dist/domains/comet-entry/hook-router.d.ts +44 -0
  417. package/dist/domains/comet-entry/hook-router.d.ts.map +1 -0
  418. package/dist/domains/comet-entry/hook-router.js +200 -0
  419. package/dist/domains/comet-entry/hook-router.js.map +1 -0
  420. package/dist/domains/comet-entry/hook-types.d.ts +21 -0
  421. package/dist/domains/comet-entry/hook-types.d.ts.map +1 -0
  422. package/dist/domains/comet-entry/hook-types.js +2 -0
  423. package/dist/domains/comet-entry/hook-types.js.map +1 -0
  424. package/dist/domains/comet-entry/init-workflow.d.ts +19 -0
  425. package/dist/domains/comet-entry/init-workflow.d.ts.map +1 -0
  426. package/dist/domains/comet-entry/init-workflow.js +138 -0
  427. package/dist/domains/comet-entry/init-workflow.js.map +1 -0
  428. package/dist/domains/comet-entry/project-activation.d.ts +12 -0
  429. package/dist/domains/comet-entry/project-activation.d.ts.map +1 -0
  430. package/dist/domains/comet-entry/project-activation.js +110 -0
  431. package/dist/domains/comet-entry/project-activation.js.map +1 -0
  432. package/dist/domains/comet-entry/project-status.d.ts +3 -0
  433. package/dist/domains/comet-entry/project-status.d.ts.map +1 -0
  434. package/dist/domains/comet-entry/project-status.js +291 -0
  435. package/dist/domains/comet-entry/project-status.js.map +1 -0
  436. package/dist/domains/comet-entry/resolve-entry.d.ts +3 -0
  437. package/dist/domains/comet-entry/resolve-entry.d.ts.map +1 -0
  438. package/dist/domains/comet-entry/resolve-entry.js +17 -0
  439. package/dist/domains/comet-entry/resolve-entry.js.map +1 -0
  440. package/dist/domains/comet-entry/resume-probe.d.ts +34 -0
  441. package/dist/domains/comet-entry/resume-probe.d.ts.map +1 -0
  442. package/dist/domains/comet-entry/resume-probe.js +538 -0
  443. package/dist/domains/comet-entry/resume-probe.js.map +1 -0
  444. package/dist/domains/comet-entry/types.d.ts +60 -0
  445. package/dist/domains/comet-entry/types.d.ts.map +1 -0
  446. package/dist/domains/comet-entry/types.js +2 -0
  447. package/dist/domains/comet-entry/types.js.map +1 -0
  448. package/dist/domains/comet-entry/workflow-resolution.d.ts +8 -0
  449. package/dist/domains/comet-entry/workflow-resolution.d.ts.map +1 -0
  450. package/dist/domains/comet-entry/workflow-resolution.js +16 -0
  451. package/dist/domains/comet-entry/workflow-resolution.js.map +1 -0
  452. package/dist/domains/comet-native/index.d.ts +36 -0
  453. package/dist/domains/comet-native/index.d.ts.map +1 -0
  454. package/dist/domains/comet-native/index.js +36 -0
  455. package/dist/domains/comet-native/index.js.map +1 -0
  456. package/dist/domains/comet-native/native-acceptance.d.ts +53 -0
  457. package/dist/domains/comet-native/native-acceptance.d.ts.map +1 -0
  458. package/dist/domains/comet-native/native-acceptance.js +639 -0
  459. package/dist/domains/comet-native/native-acceptance.js.map +1 -0
  460. package/dist/domains/comet-native/native-archive-command.d.ts +3 -0
  461. package/dist/domains/comet-native/native-archive-command.d.ts.map +1 -0
  462. package/dist/domains/comet-native/native-archive-command.js +236 -0
  463. package/dist/domains/comet-native/native-archive-command.js.map +1 -0
  464. package/dist/domains/comet-native/native-archive-content.d.ts +26 -0
  465. package/dist/domains/comet-native/native-archive-content.d.ts.map +1 -0
  466. package/dist/domains/comet-native/native-archive-content.js +159 -0
  467. package/dist/domains/comet-native/native-archive-content.js.map +1 -0
  468. package/dist/domains/comet-native/native-archive-entry.d.ts +2 -0
  469. package/dist/domains/comet-native/native-archive-entry.d.ts.map +1 -0
  470. package/dist/domains/comet-native/native-archive-entry.js +4 -0
  471. package/dist/domains/comet-native/native-archive-entry.js.map +1 -0
  472. package/dist/domains/comet-native/native-archive-inspection.d.ts +9 -0
  473. package/dist/domains/comet-native/native-archive-inspection.d.ts.map +1 -0
  474. package/dist/domains/comet-native/native-archive-inspection.js +108 -0
  475. package/dist/domains/comet-native/native-archive-inspection.js.map +1 -0
  476. package/dist/domains/comet-native/native-archive-preflight.d.ts +64 -0
  477. package/dist/domains/comet-native/native-archive-preflight.d.ts.map +1 -0
  478. package/dist/domains/comet-native/native-archive-preflight.js +266 -0
  479. package/dist/domains/comet-native/native-archive-preflight.js.map +1 -0
  480. package/dist/domains/comet-native/native-archive-transaction.d.ts +20 -0
  481. package/dist/domains/comet-native/native-archive-transaction.d.ts.map +1 -0
  482. package/dist/domains/comet-native/native-archive-transaction.js +1188 -0
  483. package/dist/domains/comet-native/native-archive-transaction.js.map +1 -0
  484. package/dist/domains/comet-native/native-archive.d.ts +37 -0
  485. package/dist/domains/comet-native/native-archive.d.ts.map +1 -0
  486. package/dist/domains/comet-native/native-archive.js +466 -0
  487. package/dist/domains/comet-native/native-archive.js.map +1 -0
  488. package/dist/domains/comet-native/native-artifacts.d.ts +10 -0
  489. package/dist/domains/comet-native/native-artifacts.d.ts.map +1 -0
  490. package/dist/domains/comet-native/native-artifacts.js +204 -0
  491. package/dist/domains/comet-native/native-artifacts.js.map +1 -0
  492. package/dist/domains/comet-native/native-atomic-file.d.ts +10 -0
  493. package/dist/domains/comet-native/native-atomic-file.d.ts.map +1 -0
  494. package/dist/domains/comet-native/native-atomic-file.js +186 -0
  495. package/dist/domains/comet-native/native-atomic-file.js.map +1 -0
  496. package/dist/domains/comet-native/native-bounded-file.d.ts +49 -0
  497. package/dist/domains/comet-native/native-bounded-file.d.ts.map +1 -0
  498. package/dist/domains/comet-native/native-bounded-file.js +265 -0
  499. package/dist/domains/comet-native/native-bounded-file.js.map +1 -0
  500. package/dist/domains/comet-native/native-build-evidence.d.ts +42 -0
  501. package/dist/domains/comet-native/native-build-evidence.d.ts.map +1 -0
  502. package/dist/domains/comet-native/native-build-evidence.js +279 -0
  503. package/dist/domains/comet-native/native-build-evidence.js.map +1 -0
  504. package/dist/domains/comet-native/native-canonical-hash.d.ts +10 -0
  505. package/dist/domains/comet-native/native-canonical-hash.d.ts.map +1 -0
  506. package/dist/domains/comet-native/native-canonical-hash.js +100 -0
  507. package/dist/domains/comet-native/native-canonical-hash.js.map +1 -0
  508. package/dist/domains/comet-native/native-change-recovery.d.ts +11 -0
  509. package/dist/domains/comet-native/native-change-recovery.d.ts.map +1 -0
  510. package/dist/domains/comet-native/native-change-recovery.js +15 -0
  511. package/dist/domains/comet-native/native-change-recovery.js.map +1 -0
  512. package/dist/domains/comet-native/native-change.d.ts +81 -0
  513. package/dist/domains/comet-native/native-change.d.ts.map +1 -0
  514. package/dist/domains/comet-native/native-change.js +895 -0
  515. package/dist/domains/comet-native/native-change.js.map +1 -0
  516. package/dist/domains/comet-native/native-check-command.d.ts +3 -0
  517. package/dist/domains/comet-native/native-check-command.d.ts.map +1 -0
  518. package/dist/domains/comet-native/native-check-command.js +30 -0
  519. package/dist/domains/comet-native/native-check-command.js.map +1 -0
  520. package/dist/domains/comet-native/native-check-executor.d.ts +36 -0
  521. package/dist/domains/comet-native/native-check-executor.d.ts.map +1 -0
  522. package/dist/domains/comet-native/native-check-executor.js +132 -0
  523. package/dist/domains/comet-native/native-check-executor.js.map +1 -0
  524. package/dist/domains/comet-native/native-check-receipt-model.d.ts +72 -0
  525. package/dist/domains/comet-native/native-check-receipt-model.d.ts.map +1 -0
  526. package/dist/domains/comet-native/native-check-receipt-model.js +414 -0
  527. package/dist/domains/comet-native/native-check-receipt-model.js.map +1 -0
  528. package/dist/domains/comet-native/native-check-receipt-storage.d.ts +10 -0
  529. package/dist/domains/comet-native/native-check-receipt-storage.d.ts.map +1 -0
  530. package/dist/domains/comet-native/native-check-receipt-storage.js +201 -0
  531. package/dist/domains/comet-native/native-check-receipt-storage.js.map +1 -0
  532. package/dist/domains/comet-native/native-check-receipt.d.ts +19 -0
  533. package/dist/domains/comet-native/native-check-receipt.d.ts.map +1 -0
  534. package/dist/domains/comet-native/native-check-receipt.js +516 -0
  535. package/dist/domains/comet-native/native-check-receipt.js.map +1 -0
  536. package/dist/domains/comet-native/native-check.d.ts +24 -0
  537. package/dist/domains/comet-native/native-check.d.ts.map +1 -0
  538. package/dist/domains/comet-native/native-check.js +50 -0
  539. package/dist/domains/comet-native/native-check.js.map +1 -0
  540. package/dist/domains/comet-native/native-checkpoint-command.d.ts +3 -0
  541. package/dist/domains/comet-native/native-checkpoint-command.d.ts.map +1 -0
  542. package/dist/domains/comet-native/native-checkpoint-command.js +33 -0
  543. package/dist/domains/comet-native/native-checkpoint-command.js.map +1 -0
  544. package/dist/domains/comet-native/native-checkpoint-journal.d.ts +13 -0
  545. package/dist/domains/comet-native/native-checkpoint-journal.d.ts.map +1 -0
  546. package/dist/domains/comet-native/native-checkpoint-journal.js +53 -0
  547. package/dist/domains/comet-native/native-checkpoint-journal.js.map +1 -0
  548. package/dist/domains/comet-native/native-checkpoint-storage.d.ts +43 -0
  549. package/dist/domains/comet-native/native-checkpoint-storage.d.ts.map +1 -0
  550. package/dist/domains/comet-native/native-checkpoint-storage.js +495 -0
  551. package/dist/domains/comet-native/native-checkpoint-storage.js.map +1 -0
  552. package/dist/domains/comet-native/native-children.d.ts +65 -0
  553. package/dist/domains/comet-native/native-children.d.ts.map +1 -0
  554. package/dist/domains/comet-native/native-children.js +453 -0
  555. package/dist/domains/comet-native/native-children.js.map +1 -0
  556. package/dist/domains/comet-native/native-cli-entry.d.ts +2 -0
  557. package/dist/domains/comet-native/native-cli-entry.d.ts.map +1 -0
  558. package/dist/domains/comet-native/native-cli-entry.js +18 -0
  559. package/dist/domains/comet-native/native-cli-entry.js.map +1 -0
  560. package/dist/domains/comet-native/native-cli-help.d.ts +6 -0
  561. package/dist/domains/comet-native/native-cli-help.d.ts.map +1 -0
  562. package/dist/domains/comet-native/native-cli-help.js +198 -0
  563. package/dist/domains/comet-native/native-cli-help.js.map +1 -0
  564. package/dist/domains/comet-native/native-cli-shared.d.ts +40 -0
  565. package/dist/domains/comet-native/native-cli-shared.d.ts.map +1 -0
  566. package/dist/domains/comet-native/native-cli-shared.js +328 -0
  567. package/dist/domains/comet-native/native-cli-shared.js.map +1 -0
  568. package/dist/domains/comet-native/native-cli.d.ts +4 -0
  569. package/dist/domains/comet-native/native-cli.d.ts.map +1 -0
  570. package/dist/domains/comet-native/native-cli.js +72 -0
  571. package/dist/domains/comet-native/native-cli.js.map +1 -0
  572. package/dist/domains/comet-native/native-config.d.ts +24 -0
  573. package/dist/domains/comet-native/native-config.d.ts.map +1 -0
  574. package/dist/domains/comet-native/native-config.js +61 -0
  575. package/dist/domains/comet-native/native-config.js.map +1 -0
  576. package/dist/domains/comet-native/native-conflict-inspection.d.ts +19 -0
  577. package/dist/domains/comet-native/native-conflict-inspection.d.ts.map +1 -0
  578. package/dist/domains/comet-native/native-conflict-inspection.js +100 -0
  579. package/dist/domains/comet-native/native-conflict-inspection.js.map +1 -0
  580. package/dist/domains/comet-native/native-conflict-radar.d.ts +75 -0
  581. package/dist/domains/comet-native/native-conflict-radar.d.ts.map +1 -0
  582. package/dist/domains/comet-native/native-conflict-radar.js +326 -0
  583. package/dist/domains/comet-native/native-conflict-radar.js.map +1 -0
  584. package/dist/domains/comet-native/native-continuation.d.ts +12 -0
  585. package/dist/domains/comet-native/native-continuation.d.ts.map +1 -0
  586. package/dist/domains/comet-native/native-continuation.js +274 -0
  587. package/dist/domains/comet-native/native-continuation.js.map +1 -0
  588. package/dist/domains/comet-native/native-contract-files.d.ts +24 -0
  589. package/dist/domains/comet-native/native-contract-files.d.ts.map +1 -0
  590. package/dist/domains/comet-native/native-contract-files.js +66 -0
  591. package/dist/domains/comet-native/native-contract-files.js.map +1 -0
  592. package/dist/domains/comet-native/native-contract.d.ts +41 -0
  593. package/dist/domains/comet-native/native-contract.d.ts.map +1 -0
  594. package/dist/domains/comet-native/native-contract.js +142 -0
  595. package/dist/domains/comet-native/native-contract.js.map +1 -0
  596. package/dist/domains/comet-native/native-diagnostics.d.ts +27 -0
  597. package/dist/domains/comet-native/native-diagnostics.d.ts.map +1 -0
  598. package/dist/domains/comet-native/native-diagnostics.js +645 -0
  599. package/dist/domains/comet-native/native-diagnostics.js.map +1 -0
  600. package/dist/domains/comet-native/native-doctor-command.d.ts +3 -0
  601. package/dist/domains/comet-native/native-doctor-command.d.ts.map +1 -0
  602. package/dist/domains/comet-native/native-doctor-command.js +361 -0
  603. package/dist/domains/comet-native/native-doctor-command.js.map +1 -0
  604. package/dist/domains/comet-native/native-doctor-entry.d.ts +2 -0
  605. package/dist/domains/comet-native/native-doctor-entry.d.ts.map +1 -0
  606. package/dist/domains/comet-native/native-doctor-entry.js +4 -0
  607. package/dist/domains/comet-native/native-doctor-entry.js.map +1 -0
  608. package/dist/domains/comet-native/native-doctor.d.ts +13 -0
  609. package/dist/domains/comet-native/native-doctor.d.ts.map +1 -0
  610. package/dist/domains/comet-native/native-doctor.js +865 -0
  611. package/dist/domains/comet-native/native-doctor.js.map +1 -0
  612. package/dist/domains/comet-native/native-evidence-command.d.ts +3 -0
  613. package/dist/domains/comet-native/native-evidence-command.d.ts.map +1 -0
  614. package/dist/domains/comet-native/native-evidence-command.js +35 -0
  615. package/dist/domains/comet-native/native-evidence-command.js.map +1 -0
  616. package/dist/domains/comet-native/native-evidence-projection.d.ts +50 -0
  617. package/dist/domains/comet-native/native-evidence-projection.d.ts.map +1 -0
  618. package/dist/domains/comet-native/native-evidence-projection.js +284 -0
  619. package/dist/domains/comet-native/native-evidence-projection.js.map +1 -0
  620. package/dist/domains/comet-native/native-evidence-retention.d.ts +37 -0
  621. package/dist/domains/comet-native/native-evidence-retention.d.ts.map +1 -0
  622. package/dist/domains/comet-native/native-evidence-retention.js +902 -0
  623. package/dist/domains/comet-native/native-evidence-retention.js.map +1 -0
  624. package/dist/domains/comet-native/native-evidence-storage.d.ts +65 -0
  625. package/dist/domains/comet-native/native-evidence-storage.d.ts.map +1 -0
  626. package/dist/domains/comet-native/native-evidence-storage.js +473 -0
  627. package/dist/domains/comet-native/native-evidence-storage.js.map +1 -0
  628. package/dist/domains/comet-native/native-file-identity.d.ts +3 -0
  629. package/dist/domains/comet-native/native-file-identity.d.ts.map +1 -0
  630. package/dist/domains/comet-native/native-file-identity.js +2 -0
  631. package/dist/domains/comet-native/native-file-identity.js.map +1 -0
  632. package/dist/domains/comet-native/native-findings.d.ts +8 -0
  633. package/dist/domains/comet-native/native-findings.d.ts.map +1 -0
  634. package/dist/domains/comet-native/native-findings.js +328 -0
  635. package/dist/domains/comet-native/native-findings.js.map +1 -0
  636. package/dist/domains/comet-native/native-git-provenance.d.ts +16 -0
  637. package/dist/domains/comet-native/native-git-provenance.d.ts.map +1 -0
  638. package/dist/domains/comet-native/native-git-provenance.js +171 -0
  639. package/dist/domains/comet-native/native-git-provenance.js.map +1 -0
  640. package/dist/domains/comet-native/native-guards.d.ts +8 -0
  641. package/dist/domains/comet-native/native-guards.d.ts.map +1 -0
  642. package/dist/domains/comet-native/native-guards.js +140 -0
  643. package/dist/domains/comet-native/native-guards.js.map +1 -0
  644. package/dist/domains/comet-native/native-hash.d.ts +3 -0
  645. package/dist/domains/comet-native/native-hash.d.ts.map +1 -0
  646. package/dist/domains/comet-native/native-hash.js +12 -0
  647. package/dist/domains/comet-native/native-hash.js.map +1 -0
  648. package/dist/domains/comet-native/native-hook-guard-command.d.ts +3 -0
  649. package/dist/domains/comet-native/native-hook-guard-command.d.ts.map +1 -0
  650. package/dist/domains/comet-native/native-hook-guard-command.js +32 -0
  651. package/dist/domains/comet-native/native-hook-guard-command.js.map +1 -0
  652. package/dist/domains/comet-native/native-hook-guard-entry.d.ts +2 -0
  653. package/dist/domains/comet-native/native-hook-guard-entry.d.ts.map +1 -0
  654. package/dist/domains/comet-native/native-hook-guard-entry.js +4 -0
  655. package/dist/domains/comet-native/native-hook-guard-entry.js.map +1 -0
  656. package/dist/domains/comet-native/native-hook-guard.d.ts +17 -0
  657. package/dist/domains/comet-native/native-hook-guard.d.ts.map +1 -0
  658. package/dist/domains/comet-native/native-hook-guard.js +353 -0
  659. package/dist/domains/comet-native/native-hook-guard.js.map +1 -0
  660. package/dist/domains/comet-native/native-init-command.d.ts +3 -0
  661. package/dist/domains/comet-native/native-init-command.d.ts.map +1 -0
  662. package/dist/domains/comet-native/native-init-command.js +41 -0
  663. package/dist/domains/comet-native/native-init-command.js.map +1 -0
  664. package/dist/domains/comet-native/native-init-entry.d.ts +2 -0
  665. package/dist/domains/comet-native/native-init-entry.d.ts.map +1 -0
  666. package/dist/domains/comet-native/native-init-entry.js +4 -0
  667. package/dist/domains/comet-native/native-init-entry.js.map +1 -0
  668. package/dist/domains/comet-native/native-local-execution.d.ts +25 -0
  669. package/dist/domains/comet-native/native-local-execution.d.ts.map +1 -0
  670. package/dist/domains/comet-native/native-local-execution.js +241 -0
  671. package/dist/domains/comet-native/native-local-execution.js.map +1 -0
  672. package/dist/domains/comet-native/native-lock.d.ts +47 -0
  673. package/dist/domains/comet-native/native-lock.d.ts.map +1 -0
  674. package/dist/domains/comet-native/native-lock.js +390 -0
  675. package/dist/domains/comet-native/native-lock.js.map +1 -0
  676. package/dist/domains/comet-native/native-loop-runtime.d.ts +63 -0
  677. package/dist/domains/comet-native/native-loop-runtime.d.ts.map +1 -0
  678. package/dist/domains/comet-native/native-loop-runtime.js +580 -0
  679. package/dist/domains/comet-native/native-loop-runtime.js.map +1 -0
  680. package/dist/domains/comet-native/native-mutation-lock.d.ts +7 -0
  681. package/dist/domains/comet-native/native-mutation-lock.d.ts.map +1 -0
  682. package/dist/domains/comet-native/native-mutation-lock.js +87 -0
  683. package/dist/domains/comet-native/native-mutation-lock.js.map +1 -0
  684. package/dist/domains/comet-native/native-new-command.d.ts +3 -0
  685. package/dist/domains/comet-native/native-new-command.d.ts.map +1 -0
  686. package/dist/domains/comet-native/native-new-command.js +57 -0
  687. package/dist/domains/comet-native/native-new-command.js.map +1 -0
  688. package/dist/domains/comet-native/native-new-entry.d.ts +2 -0
  689. package/dist/domains/comet-native/native-new-entry.d.ts.map +1 -0
  690. package/dist/domains/comet-native/native-new-entry.js +4 -0
  691. package/dist/domains/comet-native/native-new-entry.js.map +1 -0
  692. package/dist/domains/comet-native/native-next-command.d.ts +3 -0
  693. package/dist/domains/comet-native/native-next-command.d.ts.map +1 -0
  694. package/dist/domains/comet-native/native-next-command.js +292 -0
  695. package/dist/domains/comet-native/native-next-command.js.map +1 -0
  696. package/dist/domains/comet-native/native-next-entry.d.ts +2 -0
  697. package/dist/domains/comet-native/native-next-entry.d.ts.map +1 -0
  698. package/dist/domains/comet-native/native-next-entry.js +4 -0
  699. package/dist/domains/comet-native/native-next-entry.js.map +1 -0
  700. package/dist/domains/comet-native/native-paths.d.ts +28 -0
  701. package/dist/domains/comet-native/native-paths.d.ts.map +1 -0
  702. package/dist/domains/comet-native/native-paths.js +272 -0
  703. package/dist/domains/comet-native/native-paths.js.map +1 -0
  704. package/dist/domains/comet-native/native-portable-acceptance.d.ts +28 -0
  705. package/dist/domains/comet-native/native-portable-acceptance.d.ts.map +1 -0
  706. package/dist/domains/comet-native/native-portable-acceptance.js +73 -0
  707. package/dist/domains/comet-native/native-portable-acceptance.js.map +1 -0
  708. package/dist/domains/comet-native/native-portable-archive.d.ts +37 -0
  709. package/dist/domains/comet-native/native-portable-archive.d.ts.map +1 -0
  710. package/dist/domains/comet-native/native-portable-archive.js +492 -0
  711. package/dist/domains/comet-native/native-portable-archive.js.map +1 -0
  712. package/dist/domains/comet-native/native-portable-continuation.d.ts +42 -0
  713. package/dist/domains/comet-native/native-portable-continuation.d.ts.map +1 -0
  714. package/dist/domains/comet-native/native-portable-continuation.js +469 -0
  715. package/dist/domains/comet-native/native-portable-continuation.js.map +1 -0
  716. package/dist/domains/comet-native/native-portable-migration-runtime.d.ts +9 -0
  717. package/dist/domains/comet-native/native-portable-migration-runtime.d.ts.map +1 -0
  718. package/dist/domains/comet-native/native-portable-migration-runtime.js +218 -0
  719. package/dist/domains/comet-native/native-portable-migration-runtime.js.map +1 -0
  720. package/dist/domains/comet-native/native-portable-migration.d.ts +42 -0
  721. package/dist/domains/comet-native/native-portable-migration.d.ts.map +1 -0
  722. package/dist/domains/comet-native/native-portable-migration.js +212 -0
  723. package/dist/domains/comet-native/native-portable-migration.js.map +1 -0
  724. package/dist/domains/comet-native/native-portable-recovery.d.ts +15 -0
  725. package/dist/domains/comet-native/native-portable-recovery.d.ts.map +1 -0
  726. package/dist/domains/comet-native/native-portable-recovery.js +203 -0
  727. package/dist/domains/comet-native/native-portable-recovery.js.map +1 -0
  728. package/dist/domains/comet-native/native-portable-runtime.d.ts +168 -0
  729. package/dist/domains/comet-native/native-portable-runtime.d.ts.map +1 -0
  730. package/dist/domains/comet-native/native-portable-runtime.js +1608 -0
  731. package/dist/domains/comet-native/native-portable-runtime.js.map +1 -0
  732. package/dist/domains/comet-native/native-portable-state.d.ts +30 -0
  733. package/dist/domains/comet-native/native-portable-state.d.ts.map +1 -0
  734. package/dist/domains/comet-native/native-portable-state.js +701 -0
  735. package/dist/domains/comet-native/native-portable-state.js.map +1 -0
  736. package/dist/domains/comet-native/native-portable-status.d.ts +65 -0
  737. package/dist/domains/comet-native/native-portable-status.d.ts.map +1 -0
  738. package/dist/domains/comet-native/native-portable-status.js +133 -0
  739. package/dist/domains/comet-native/native-portable-status.js.map +1 -0
  740. package/dist/domains/comet-native/native-portable-text.d.ts +8 -0
  741. package/dist/domains/comet-native/native-portable-text.d.ts.map +1 -0
  742. package/dist/domains/comet-native/native-portable-text.js +28 -0
  743. package/dist/domains/comet-native/native-portable-text.js.map +1 -0
  744. package/dist/domains/comet-native/native-portable-transactions.d.ts +41 -0
  745. package/dist/domains/comet-native/native-portable-transactions.d.ts.map +1 -0
  746. package/dist/domains/comet-native/native-portable-transactions.js +204 -0
  747. package/dist/domains/comet-native/native-portable-transactions.js.map +1 -0
  748. package/dist/domains/comet-native/native-portable-types.d.ts +166 -0
  749. package/dist/domains/comet-native/native-portable-types.d.ts.map +1 -0
  750. package/dist/domains/comet-native/native-portable-types.js +18 -0
  751. package/dist/domains/comet-native/native-portable-types.js.map +1 -0
  752. package/dist/domains/comet-native/native-progress-checkpoint.d.ts +13 -0
  753. package/dist/domains/comet-native/native-progress-checkpoint.d.ts.map +1 -0
  754. package/dist/domains/comet-native/native-progress-checkpoint.js +104 -0
  755. package/dist/domains/comet-native/native-progress-checkpoint.js.map +1 -0
  756. package/dist/domains/comet-native/native-protected-file.d.ts +113 -0
  757. package/dist/domains/comet-native/native-protected-file.d.ts.map +1 -0
  758. package/dist/domains/comet-native/native-protected-file.js +510 -0
  759. package/dist/domains/comet-native/native-protected-file.js.map +1 -0
  760. package/dist/domains/comet-native/native-pull-request-finish.d.ts +35 -0
  761. package/dist/domains/comet-native/native-pull-request-finish.d.ts.map +1 -0
  762. package/dist/domains/comet-native/native-pull-request-finish.js +282 -0
  763. package/dist/domains/comet-native/native-pull-request-finish.js.map +1 -0
  764. package/dist/domains/comet-native/native-receipt-command.d.ts +3 -0
  765. package/dist/domains/comet-native/native-receipt-command.d.ts.map +1 -0
  766. package/dist/domains/comet-native/native-receipt-command.js +120 -0
  767. package/dist/domains/comet-native/native-receipt-command.js.map +1 -0
  768. package/dist/domains/comet-native/native-receipt-errors.d.ts +23 -0
  769. package/dist/domains/comet-native/native-receipt-errors.d.ts.map +1 -0
  770. package/dist/domains/comet-native/native-receipt-errors.js +9 -0
  771. package/dist/domains/comet-native/native-receipt-errors.js.map +1 -0
  772. package/dist/domains/comet-native/native-receipt-refresh.d.ts +89 -0
  773. package/dist/domains/comet-native/native-receipt-refresh.d.ts.map +1 -0
  774. package/dist/domains/comet-native/native-receipt-refresh.js +243 -0
  775. package/dist/domains/comet-native/native-receipt-refresh.js.map +1 -0
  776. package/dist/domains/comet-native/native-redaction.d.ts +2 -0
  777. package/dist/domains/comet-native/native-redaction.d.ts.map +1 -0
  778. package/dist/domains/comet-native/native-redaction.js +18 -0
  779. package/dist/domains/comet-native/native-redaction.js.map +1 -0
  780. package/dist/domains/comet-native/native-repair-integration.d.ts +44 -0
  781. package/dist/domains/comet-native/native-repair-integration.d.ts.map +1 -0
  782. package/dist/domains/comet-native/native-repair-integration.js +232 -0
  783. package/dist/domains/comet-native/native-repair-integration.js.map +1 -0
  784. package/dist/domains/comet-native/native-repair-runtime.d.ts +77 -0
  785. package/dist/domains/comet-native/native-repair-runtime.d.ts.map +1 -0
  786. package/dist/domains/comet-native/native-repair-runtime.js +586 -0
  787. package/dist/domains/comet-native/native-repair-runtime.js.map +1 -0
  788. package/dist/domains/comet-native/native-repair-stagnation.d.ts +81 -0
  789. package/dist/domains/comet-native/native-repair-stagnation.d.ts.map +1 -0
  790. package/dist/domains/comet-native/native-repair-stagnation.js +298 -0
  791. package/dist/domains/comet-native/native-repair-stagnation.js.map +1 -0
  792. package/dist/domains/comet-native/native-resume-view.d.ts +14 -0
  793. package/dist/domains/comet-native/native-resume-view.d.ts.map +1 -0
  794. package/dist/domains/comet-native/native-resume-view.js +101 -0
  795. package/dist/domains/comet-native/native-resume-view.js.map +1 -0
  796. package/dist/domains/comet-native/native-revision.d.ts +20 -0
  797. package/dist/domains/comet-native/native-revision.d.ts.map +1 -0
  798. package/dist/domains/comet-native/native-revision.js +32 -0
  799. package/dist/domains/comet-native/native-revision.js.map +1 -0
  800. package/dist/domains/comet-native/native-root-command.d.ts +3 -0
  801. package/dist/domains/comet-native/native-root-command.d.ts.map +1 -0
  802. package/dist/domains/comet-native/native-root-command.js +29 -0
  803. package/dist/domains/comet-native/native-root-command.js.map +1 -0
  804. package/dist/domains/comet-native/native-root-entry.d.ts +2 -0
  805. package/dist/domains/comet-native/native-root-entry.d.ts.map +1 -0
  806. package/dist/domains/comet-native/native-root-entry.js +4 -0
  807. package/dist/domains/comet-native/native-root-entry.js.map +1 -0
  808. package/dist/domains/comet-native/native-root-move.d.ts +20 -0
  809. package/dist/domains/comet-native/native-root-move.d.ts.map +1 -0
  810. package/dist/domains/comet-native/native-root-move.js +855 -0
  811. package/dist/domains/comet-native/native-root-move.js.map +1 -0
  812. package/dist/domains/comet-native/native-run-consistency.d.ts +3 -0
  813. package/dist/domains/comet-native/native-run-consistency.d.ts.map +1 -0
  814. package/dist/domains/comet-native/native-run-consistency.js +132 -0
  815. package/dist/domains/comet-native/native-run-consistency.js.map +1 -0
  816. package/dist/domains/comet-native/native-run-store.d.ts +41 -0
  817. package/dist/domains/comet-native/native-run-store.d.ts.map +1 -0
  818. package/dist/domains/comet-native/native-run-store.js +452 -0
  819. package/dist/domains/comet-native/native-run-store.js.map +1 -0
  820. package/dist/domains/comet-native/native-runner-input.d.ts +160 -0
  821. package/dist/domains/comet-native/native-runner-input.d.ts.map +1 -0
  822. package/dist/domains/comet-native/native-runner-input.js +347 -0
  823. package/dist/domains/comet-native/native-runner-input.js.map +1 -0
  824. package/dist/domains/comet-native/native-runner-protocol.d.ts +40 -0
  825. package/dist/domains/comet-native/native-runner-protocol.d.ts.map +1 -0
  826. package/dist/domains/comet-native/native-runner-protocol.js +55 -0
  827. package/dist/domains/comet-native/native-runner-protocol.js.map +1 -0
  828. package/dist/domains/comet-native/native-runtime-package.d.ts +19 -0
  829. package/dist/domains/comet-native/native-runtime-package.d.ts.map +1 -0
  830. package/dist/domains/comet-native/native-runtime-package.js +76 -0
  831. package/dist/domains/comet-native/native-runtime-package.js.map +1 -0
  832. package/dist/domains/comet-native/native-schema-migration.d.ts +11 -0
  833. package/dist/domains/comet-native/native-schema-migration.d.ts.map +1 -0
  834. package/dist/domains/comet-native/native-schema-migration.js +903 -0
  835. package/dist/domains/comet-native/native-schema-migration.js.map +1 -0
  836. package/dist/domains/comet-native/native-script-entry.d.ts +12 -0
  837. package/dist/domains/comet-native/native-script-entry.d.ts.map +1 -0
  838. package/dist/domains/comet-native/native-script-entry.js +44 -0
  839. package/dist/domains/comet-native/native-script-entry.js.map +1 -0
  840. package/dist/domains/comet-native/native-select-command.d.ts +3 -0
  841. package/dist/domains/comet-native/native-select-command.d.ts.map +1 -0
  842. package/dist/domains/comet-native/native-select-command.js +19 -0
  843. package/dist/domains/comet-native/native-select-command.js.map +1 -0
  844. package/dist/domains/comet-native/native-select-entry.d.ts +2 -0
  845. package/dist/domains/comet-native/native-select-entry.d.ts.map +1 -0
  846. package/dist/domains/comet-native/native-select-entry.js +4 -0
  847. package/dist/domains/comet-native/native-select-entry.js.map +1 -0
  848. package/dist/domains/comet-native/native-selection.d.ts +12 -0
  849. package/dist/domains/comet-native/native-selection.d.ts.map +1 -0
  850. package/dist/domains/comet-native/native-selection.js +61 -0
  851. package/dist/domains/comet-native/native-selection.js.map +1 -0
  852. package/dist/domains/comet-native/native-sensitive-paths.d.ts +7 -0
  853. package/dist/domains/comet-native/native-sensitive-paths.d.ts.map +1 -0
  854. package/dist/domains/comet-native/native-sensitive-paths.js +69 -0
  855. package/dist/domains/comet-native/native-sensitive-paths.js.map +1 -0
  856. package/dist/domains/comet-native/native-show-command.d.ts +3 -0
  857. package/dist/domains/comet-native/native-show-command.d.ts.map +1 -0
  858. package/dist/domains/comet-native/native-show-command.js +78 -0
  859. package/dist/domains/comet-native/native-show-command.js.map +1 -0
  860. package/dist/domains/comet-native/native-show-entry.d.ts +2 -0
  861. package/dist/domains/comet-native/native-show-entry.d.ts.map +1 -0
  862. package/dist/domains/comet-native/native-show-entry.js +4 -0
  863. package/dist/domains/comet-native/native-show-entry.js.map +1 -0
  864. package/dist/domains/comet-native/native-snapshot.d.ts +87 -0
  865. package/dist/domains/comet-native/native-snapshot.d.ts.map +1 -0
  866. package/dist/domains/comet-native/native-snapshot.js +2728 -0
  867. package/dist/domains/comet-native/native-snapshot.js.map +1 -0
  868. package/dist/domains/comet-native/native-spec-command.d.ts +3 -0
  869. package/dist/domains/comet-native/native-spec-command.d.ts.map +1 -0
  870. package/dist/domains/comet-native/native-spec-command.js +20 -0
  871. package/dist/domains/comet-native/native-spec-command.js.map +1 -0
  872. package/dist/domains/comet-native/native-spec-entry.d.ts +2 -0
  873. package/dist/domains/comet-native/native-spec-entry.d.ts.map +1 -0
  874. package/dist/domains/comet-native/native-spec-entry.js +4 -0
  875. package/dist/domains/comet-native/native-spec-entry.js.map +1 -0
  876. package/dist/domains/comet-native/native-specs.d.ts +12 -0
  877. package/dist/domains/comet-native/native-specs.d.ts.map +1 -0
  878. package/dist/domains/comet-native/native-specs.js +270 -0
  879. package/dist/domains/comet-native/native-specs.js.map +1 -0
  880. package/dist/domains/comet-native/native-status-command.d.ts +3 -0
  881. package/dist/domains/comet-native/native-status-command.d.ts.map +1 -0
  882. package/dist/domains/comet-native/native-status-command.js +26 -0
  883. package/dist/domains/comet-native/native-status-command.js.map +1 -0
  884. package/dist/domains/comet-native/native-status-discovery.d.ts +51 -0
  885. package/dist/domains/comet-native/native-status-discovery.d.ts.map +1 -0
  886. package/dist/domains/comet-native/native-status-discovery.js +317 -0
  887. package/dist/domains/comet-native/native-status-discovery.js.map +1 -0
  888. package/dist/domains/comet-native/native-status-entry.d.ts +2 -0
  889. package/dist/domains/comet-native/native-status-entry.d.ts.map +1 -0
  890. package/dist/domains/comet-native/native-status-entry.js +4 -0
  891. package/dist/domains/comet-native/native-status-entry.js.map +1 -0
  892. package/dist/domains/comet-native/native-trajectory-limits.d.ts +7 -0
  893. package/dist/domains/comet-native/native-trajectory-limits.d.ts.map +1 -0
  894. package/dist/domains/comet-native/native-trajectory-limits.js +13 -0
  895. package/dist/domains/comet-native/native-trajectory-limits.js.map +1 -0
  896. package/dist/domains/comet-native/native-trajectory-recovery.d.ts +37 -0
  897. package/dist/domains/comet-native/native-trajectory-recovery.d.ts.map +1 -0
  898. package/dist/domains/comet-native/native-trajectory-recovery.js +130 -0
  899. package/dist/domains/comet-native/native-trajectory-recovery.js.map +1 -0
  900. package/dist/domains/comet-native/native-trajectory.d.ts +16 -0
  901. package/dist/domains/comet-native/native-trajectory.d.ts.map +1 -0
  902. package/dist/domains/comet-native/native-trajectory.js +27 -0
  903. package/dist/domains/comet-native/native-trajectory.js.map +1 -0
  904. package/dist/domains/comet-native/native-transaction.d.ts +45 -0
  905. package/dist/domains/comet-native/native-transaction.d.ts.map +1 -0
  906. package/dist/domains/comet-native/native-transaction.js +857 -0
  907. package/dist/domains/comet-native/native-transaction.js.map +1 -0
  908. package/dist/domains/comet-native/native-transition-evidence.d.ts +4 -0
  909. package/dist/domains/comet-native/native-transition-evidence.d.ts.map +1 -0
  910. package/dist/domains/comet-native/native-transition-evidence.js +18 -0
  911. package/dist/domains/comet-native/native-transition-evidence.js.map +1 -0
  912. package/dist/domains/comet-native/native-transition-journal.d.ts +30 -0
  913. package/dist/domains/comet-native/native-transition-journal.d.ts.map +1 -0
  914. package/dist/domains/comet-native/native-transition-journal.js +868 -0
  915. package/dist/domains/comet-native/native-transition-journal.js.map +1 -0
  916. package/dist/domains/comet-native/native-transitions.d.ts +19 -0
  917. package/dist/domains/comet-native/native-transitions.d.ts.map +1 -0
  918. package/dist/domains/comet-native/native-transitions.js +863 -0
  919. package/dist/domains/comet-native/native-transitions.js.map +1 -0
  920. package/dist/domains/comet-native/native-types.d.ts +584 -0
  921. package/dist/domains/comet-native/native-types.d.ts.map +1 -0
  922. package/dist/domains/comet-native/native-types.js +8 -0
  923. package/dist/domains/comet-native/native-types.js.map +1 -0
  924. package/dist/domains/comet-native/native-verification-evidence.d.ts +90 -0
  925. package/dist/domains/comet-native/native-verification-evidence.d.ts.map +1 -0
  926. package/dist/domains/comet-native/native-verification-evidence.js +536 -0
  927. package/dist/domains/comet-native/native-verification-evidence.js.map +1 -0
  928. package/dist/domains/comet-native/native-verification-receipt-runtime.d.ts +99 -0
  929. package/dist/domains/comet-native/native-verification-receipt-runtime.d.ts.map +1 -0
  930. package/dist/domains/comet-native/native-verification-receipt-runtime.js +597 -0
  931. package/dist/domains/comet-native/native-verification-receipt-runtime.js.map +1 -0
  932. package/dist/domains/comet-native/native-verification-receipt.d.ts +79 -0
  933. package/dist/domains/comet-native/native-verification-receipt.d.ts.map +1 -0
  934. package/dist/domains/comet-native/native-verification-receipt.js +305 -0
  935. package/dist/domains/comet-native/native-verification-receipt.js.map +1 -0
  936. package/dist/domains/comet-native/native-verification-report-v2.d.ts +12 -0
  937. package/dist/domains/comet-native/native-verification-report-v2.d.ts.map +1 -0
  938. package/dist/domains/comet-native/native-verification-report-v2.js +141 -0
  939. package/dist/domains/comet-native/native-verification-report-v2.js.map +1 -0
  940. package/dist/domains/comet-native/native-verification-runtime.d.ts +111 -0
  941. package/dist/domains/comet-native/native-verification-runtime.d.ts.map +1 -0
  942. package/dist/domains/comet-native/native-verification-runtime.js +520 -0
  943. package/dist/domains/comet-native/native-verification-runtime.js.map +1 -0
  944. package/dist/domains/comet-native/native-verification-scope.d.ts +141 -0
  945. package/dist/domains/comet-native/native-verification-scope.d.ts.map +1 -0
  946. package/dist/domains/comet-native/native-verification-scope.js +1132 -0
  947. package/dist/domains/comet-native/native-verification-scope.js.map +1 -0
  948. package/dist/domains/comet-native/native-verifier-protocol.d.ts +49 -0
  949. package/dist/domains/comet-native/native-verifier-protocol.d.ts.map +1 -0
  950. package/dist/domains/comet-native/native-verifier-protocol.js +157 -0
  951. package/dist/domains/comet-native/native-verifier-protocol.js.map +1 -0
  952. package/dist/domains/comet-native/native-workspace-finish.d.ts +67 -0
  953. package/dist/domains/comet-native/native-workspace-finish.d.ts.map +1 -0
  954. package/dist/domains/comet-native/native-workspace-finish.js +383 -0
  955. package/dist/domains/comet-native/native-workspace-finish.js.map +1 -0
  956. package/dist/domains/comet-native/native-workspace-preparation.d.ts +32 -0
  957. package/dist/domains/comet-native/native-workspace-preparation.d.ts.map +1 -0
  958. package/dist/domains/comet-native/native-workspace-preparation.js +324 -0
  959. package/dist/domains/comet-native/native-workspace-preparation.js.map +1 -0
  960. package/dist/domains/comet-native/native-workspace.d.ts +90 -0
  961. package/dist/domains/comet-native/native-workspace.d.ts.map +1 -0
  962. package/dist/domains/comet-native/native-workspace.js +566 -0
  963. package/dist/domains/comet-native/native-workspace.js.map +1 -0
  964. package/dist/domains/dashboard/collector.d.ts +26 -1
  965. package/dist/domains/dashboard/collector.d.ts.map +1 -1
  966. package/dist/domains/dashboard/collector.js +609 -95
  967. package/dist/domains/dashboard/collector.js.map +1 -1
  968. package/dist/domains/dashboard/native-adapter.d.ts +201 -0
  969. package/dist/domains/dashboard/native-adapter.d.ts.map +1 -0
  970. package/dist/domains/dashboard/native-adapter.js +199 -0
  971. package/dist/domains/dashboard/native-adapter.js.map +1 -0
  972. package/dist/domains/dashboard/native-collector.d.ts +32 -0
  973. package/dist/domains/dashboard/native-collector.d.ts.map +1 -0
  974. package/dist/domains/dashboard/native-collector.js +645 -0
  975. package/dist/domains/dashboard/native-collector.js.map +1 -0
  976. package/dist/domains/dashboard/native-legacy-adapter.d.ts +37 -0
  977. package/dist/domains/dashboard/native-legacy-adapter.d.ts.map +1 -0
  978. package/dist/domains/dashboard/native-legacy-adapter.js +127 -0
  979. package/dist/domains/dashboard/native-legacy-adapter.js.map +1 -0
  980. package/dist/domains/dashboard/project-directory.d.ts +20 -0
  981. package/dist/domains/dashboard/project-directory.d.ts.map +1 -0
  982. package/dist/domains/dashboard/project-directory.js +87 -0
  983. package/dist/domains/dashboard/project-directory.js.map +1 -0
  984. package/dist/domains/dashboard/project-root.d.ts +8 -0
  985. package/dist/domains/dashboard/project-root.d.ts.map +1 -0
  986. package/dist/domains/dashboard/project-root.js +68 -0
  987. package/dist/domains/dashboard/project-root.js.map +1 -0
  988. package/dist/domains/dashboard/server.d.ts +1 -0
  989. package/dist/domains/dashboard/server.d.ts.map +1 -1
  990. package/dist/domains/dashboard/server.js +142 -11
  991. package/dist/domains/dashboard/server.js.map +1 -1
  992. package/dist/domains/dashboard/types.d.ts +55 -0
  993. package/dist/domains/dashboard/types.d.ts.map +1 -1
  994. package/dist/domains/dashboard/verify-parser.d.ts +1 -0
  995. package/dist/domains/dashboard/verify-parser.d.ts.map +1 -1
  996. package/dist/domains/dashboard/verify-parser.js +41 -31
  997. package/dist/domains/dashboard/verify-parser.js.map +1 -1
  998. package/dist/domains/dashboard/web/assets/abnfDiagram-N423BO3Z-BYya6ofk.js +1 -0
  999. package/dist/domains/dashboard/web/assets/arc-DJbo4TD3.js +1 -0
  1000. package/dist/domains/dashboard/web/assets/architectureDiagram-T3A2C74G-D8uZ2IB-.js +36 -0
  1001. package/dist/domains/dashboard/web/assets/blockDiagram-VBNYF7ZC-B16BVj-r.js +132 -0
  1002. package/dist/domains/dashboard/web/assets/c4Diagram-5PPSVZJV-ayV208_D.js +10 -0
  1003. package/dist/domains/dashboard/web/assets/channel-CEsrvmig.js +1 -0
  1004. package/dist/domains/dashboard/web/assets/chunk-2GRJ4B5K-Bn5o8p3H.js +1 -0
  1005. package/dist/domains/dashboard/web/assets/chunk-2Q5K7J3B-BnF1SI6B.js +1 -0
  1006. package/dist/domains/dashboard/web/assets/chunk-5RXB4S5H-ETnma053.js +231 -0
  1007. package/dist/domains/dashboard/web/assets/chunk-5VM5RSS4-Cuvy639H.js +15 -0
  1008. package/dist/domains/dashboard/web/assets/chunk-6Q2QTUOP-DOXBzl73.js +88 -0
  1009. package/dist/domains/dashboard/web/assets/chunk-GF5L2VYU-CfREXlEl.js +206 -0
  1010. package/dist/domains/dashboard/web/assets/chunk-JWPE2WC7-DWIdpYTb.js +1 -0
  1011. package/dist/domains/dashboard/web/assets/chunk-KBJHAD2P-BnqPcfZB.js +1 -0
  1012. package/dist/domains/dashboard/web/assets/chunk-RYQCIY6F-DdJ00fZz.js +1 -0
  1013. package/dist/domains/dashboard/web/assets/chunk-XXDRQBXY-BVDhdJyW.js +1 -0
  1014. package/dist/domains/dashboard/web/assets/classDiagram-JCYQIIEL-CKFe0hg4.js +1 -0
  1015. package/dist/domains/dashboard/web/assets/classDiagram-v2-OCEON4UE-CKFe0hg4.js +1 -0
  1016. package/dist/domains/dashboard/web/assets/cose-bilkent-JH36ORCC-C-nglwHC.js +1 -0
  1017. package/dist/domains/dashboard/web/assets/cynefin-VYW2F7L2-C8TMmrbP.js +166 -0
  1018. package/dist/domains/dashboard/web/assets/cynefinDiagram-MW4NZA55-C3pnQnjg.js +62 -0
  1019. package/dist/domains/dashboard/web/assets/cytoscape.esm-D3_iZ_3b.js +321 -0
  1020. package/dist/domains/dashboard/web/assets/dagre-VZM6K2ZE-DY7UWiPI.js +4 -0
  1021. package/dist/domains/dashboard/web/assets/defaultLocale-DX6XiGOO.js +1 -0
  1022. package/dist/domains/dashboard/web/assets/demo-BfLsKrIL.js +45 -0
  1023. package/dist/domains/dashboard/web/assets/diagram-7IWD3JNH-voTyDkQ5.js +30 -0
  1024. package/dist/domains/dashboard/web/assets/diagram-B4RE2ZJO-B3VkUMck.js +3 -0
  1025. package/dist/domains/dashboard/web/assets/diagram-LBJQPF4R-BdJzPkv8.js +24 -0
  1026. package/dist/domains/dashboard/web/assets/diagram-Q27KOJAE-Bsq0anZx.js +24 -0
  1027. package/dist/domains/dashboard/web/assets/diagram-UB23O5K3-CBaKRbcu.js +41 -0
  1028. package/dist/domains/dashboard/web/assets/ebnfDiagram-BXEA7PRR-Bl9Bm5wa.js +1 -0
  1029. package/dist/domains/dashboard/web/assets/erDiagram-JOGREHBK-DjOZzfKI.js +85 -0
  1030. package/dist/domains/dashboard/web/assets/flowDiagram-UKHOOZJN-C21e4siK.js +156 -0
  1031. package/dist/domains/dashboard/web/assets/ganttDiagram-PKOTCBZU-Bs62uy6-.js +292 -0
  1032. package/dist/domains/dashboard/web/assets/gitGraphDiagram-DS77QQ5N-DQY_Ba00.js +106 -0
  1033. package/dist/domains/dashboard/web/assets/graph-DOmOIIwC.js +1 -0
  1034. package/dist/domains/dashboard/web/assets/index-BLv71GRj.js +10 -0
  1035. package/dist/domains/dashboard/web/assets/index-Be4r99N4.js +257 -0
  1036. package/dist/domains/dashboard/web/assets/index-ByVYiA6g.css +1 -0
  1037. package/dist/domains/dashboard/web/assets/index-DlSYK56b.js +139 -0
  1038. package/dist/domains/dashboard/web/assets/infoDiagram-6WML65LV-CNgNNG9l.js +2 -0
  1039. package/dist/domains/dashboard/web/assets/init-Gi6I4Gst.js +1 -0
  1040. package/dist/domains/dashboard/web/assets/ishikawaDiagram-WSZJBQD7-D9k1fTO1.js +70 -0
  1041. package/dist/domains/dashboard/web/assets/journeyDiagram-NVQOT4AX-CHijUnPl.js +139 -0
  1042. package/dist/domains/dashboard/web/assets/kanban-definition-27J2QSJJ-DfWlntDq.js +89 -0
  1043. package/dist/domains/dashboard/web/assets/katex-HP8lGamR.js +257 -0
  1044. package/dist/domains/dashboard/web/assets/layout-D-LzfAck.js +1 -0
  1045. package/dist/domains/dashboard/web/assets/linear-o6hemX-z.js +1 -0
  1046. package/dist/domains/dashboard/web/assets/map-DxJ2ADlA.js +1 -0
  1047. package/dist/domains/dashboard/web/assets/marked.esm-DECB3bxB.js +64 -0
  1048. package/dist/domains/dashboard/web/assets/mermaid.core-B1DoOsPD.js +306 -0
  1049. package/dist/domains/dashboard/web/assets/mindmap-definition-FAOFIHXS-COMQF9Yy.js +96 -0
  1050. package/dist/domains/dashboard/web/assets/ordinal-Cboi1Yqb.js +1 -0
  1051. package/dist/domains/dashboard/web/assets/pegDiagram-VL7TDLO6-C_21n7Ps.js +1 -0
  1052. package/dist/domains/dashboard/web/assets/pieDiagram-7S7Q4E2Y-D3EmoYDY.js +39 -0
  1053. package/dist/domains/dashboard/web/assets/purify.es-Cz4mVeUR.js +3 -0
  1054. package/dist/domains/dashboard/web/assets/quadrantDiagram-CIZ2JOQS-C3fGdiMa.js +7 -0
  1055. package/dist/domains/dashboard/web/assets/railroadDiagram-AXF67PYL-mIsjemD-.js +1 -0
  1056. package/dist/domains/dashboard/web/assets/requirementDiagram-LRYGKXZP-D_UMpS5e.js +84 -0
  1057. package/dist/domains/dashboard/web/assets/sankeyDiagram-W5VNT64P-D1MwyMF6.js +40 -0
  1058. package/dist/domains/dashboard/web/assets/sequenceDiagram-SI44F4Z6-B8jp1RxX.js +162 -0
  1059. package/dist/domains/dashboard/web/assets/sizeCapture-X5ZJPWSS-BKioI1Ql.js +1 -0
  1060. package/dist/domains/dashboard/web/assets/stateDiagram-OKZ733FA-B7l8_CDo.js +1 -0
  1061. package/dist/domains/dashboard/web/assets/stateDiagram-v2-UEYNNEHI-BIxuwbeV.js +1 -0
  1062. package/dist/domains/dashboard/web/assets/swimlanes-SLNWSIFB-951PRLYp.js +2 -0
  1063. package/dist/domains/dashboard/web/assets/swimlanesDiagram-ULZ7WXOC-CEo6cGsy.js +8 -0
  1064. package/dist/domains/dashboard/web/assets/timeline-definition-Z64GVDOM-CeyA2KbI.js +120 -0
  1065. package/dist/domains/dashboard/web/assets/vennDiagram-T6HMQDX7-BHe0TZJV.js +34 -0
  1066. package/dist/domains/dashboard/web/assets/wardleyDiagram-T6FBY63Y-CWqdSzJ-.js +78 -0
  1067. package/dist/domains/dashboard/web/assets/xychartDiagram-ELKLHX3M-DCKiZVPV.js +7 -0
  1068. package/dist/domains/dashboard/web/index.html +16 -16
  1069. package/dist/domains/dashboard/workspace.d.ts +27 -0
  1070. package/dist/domains/dashboard/workspace.d.ts.map +1 -0
  1071. package/dist/domains/dashboard/workspace.js +108 -0
  1072. package/dist/domains/dashboard/workspace.js.map +1 -0
  1073. package/dist/domains/engine/protected-run-file.d.ts +15 -0
  1074. package/dist/domains/engine/protected-run-file.d.ts.map +1 -0
  1075. package/dist/domains/engine/protected-run-file.js +76 -0
  1076. package/dist/domains/engine/protected-run-file.js.map +1 -0
  1077. package/dist/domains/engine/run-store.d.ts +12 -11
  1078. package/dist/domains/engine/run-store.d.ts.map +1 -1
  1079. package/dist/domains/engine/run-store.js +46 -65
  1080. package/dist/domains/engine/run-store.js.map +1 -1
  1081. package/dist/domains/engine/state.d.ts +4 -3
  1082. package/dist/domains/engine/state.d.ts.map +1 -1
  1083. package/dist/domains/engine/state.js +10 -21
  1084. package/dist/domains/engine/state.js.map +1 -1
  1085. package/dist/domains/engine/storage-layout.d.ts +14 -0
  1086. package/dist/domains/engine/storage-layout.d.ts.map +1 -0
  1087. package/dist/domains/engine/storage-layout.js +32 -0
  1088. package/dist/domains/engine/storage-layout.js.map +1 -0
  1089. package/dist/domains/engine/storage-run.d.ts +9 -0
  1090. package/dist/domains/engine/storage-run.d.ts.map +1 -0
  1091. package/dist/domains/engine/storage-run.js +116 -0
  1092. package/dist/domains/engine/storage-run.js.map +1 -0
  1093. package/dist/domains/engine/types.d.ts +1 -1
  1094. package/dist/domains/engine/types.d.ts.map +1 -1
  1095. package/dist/domains/eval/index.d.ts +3 -0
  1096. package/dist/domains/eval/index.d.ts.map +1 -1
  1097. package/dist/domains/eval/index.js +3 -0
  1098. package/dist/domains/eval/index.js.map +1 -1
  1099. package/dist/domains/eval/standalone-context.d.ts +19 -0
  1100. package/dist/domains/eval/standalone-context.d.ts.map +1 -0
  1101. package/dist/domains/eval/standalone-context.js +107 -0
  1102. package/dist/domains/eval/standalone-context.js.map +1 -0
  1103. package/dist/domains/eval/standalone-static-collect.d.ts +22 -0
  1104. package/dist/domains/eval/standalone-static-collect.d.ts.map +1 -0
  1105. package/dist/domains/eval/standalone-static-collect.js +779 -0
  1106. package/dist/domains/eval/standalone-static-collect.js.map +1 -0
  1107. package/dist/domains/eval/user-environment.d.ts +6 -0
  1108. package/dist/domains/eval/user-environment.d.ts.map +1 -0
  1109. package/dist/domains/eval/user-environment.js +73 -0
  1110. package/dist/domains/eval/user-environment.js.map +1 -0
  1111. package/dist/domains/factory/package.d.ts.map +1 -1
  1112. package/dist/domains/factory/package.js +547 -91
  1113. package/dist/domains/factory/package.js.map +1 -1
  1114. package/dist/domains/integrations/codegraph.d.ts +35 -2
  1115. package/dist/domains/integrations/codegraph.d.ts.map +1 -1
  1116. package/dist/domains/integrations/codegraph.js +466 -12
  1117. package/dist/domains/integrations/codegraph.js.map +1 -1
  1118. package/dist/domains/integrations/openspec.d.ts +13 -2
  1119. package/dist/domains/integrations/openspec.d.ts.map +1 -1
  1120. package/dist/domains/integrations/openspec.js +316 -55
  1121. package/dist/domains/integrations/openspec.js.map +1 -1
  1122. package/dist/domains/integrations/superpowers.d.ts +14 -1
  1123. package/dist/domains/integrations/superpowers.d.ts.map +1 -1
  1124. package/dist/domains/integrations/superpowers.js +93 -5
  1125. package/dist/domains/integrations/superpowers.js.map +1 -1
  1126. package/dist/domains/skill/find.js +8 -7
  1127. package/dist/domains/skill/find.js.map +1 -1
  1128. package/dist/domains/skill/hook-lifecycle.d.ts +9 -0
  1129. package/dist/domains/skill/hook-lifecycle.d.ts.map +1 -0
  1130. package/dist/domains/skill/hook-lifecycle.js +48 -0
  1131. package/dist/domains/skill/hook-lifecycle.js.map +1 -0
  1132. package/dist/domains/skill/json-object.d.ts +13 -0
  1133. package/dist/domains/skill/json-object.d.ts.map +1 -0
  1134. package/dist/domains/skill/json-object.js +24 -0
  1135. package/dist/domains/skill/json-object.js.map +1 -0
  1136. package/dist/domains/skill/languages.d.ts +2 -1
  1137. package/dist/domains/skill/languages.d.ts.map +1 -1
  1138. package/dist/domains/skill/languages.js +4 -1
  1139. package/dist/domains/skill/languages.js.map +1 -1
  1140. package/dist/domains/skill/managed-markdown.d.ts +14 -0
  1141. package/dist/domains/skill/managed-markdown.d.ts.map +1 -0
  1142. package/dist/domains/skill/managed-markdown.js +126 -0
  1143. package/dist/domains/skill/managed-markdown.js.map +1 -0
  1144. package/dist/domains/skill/platform-inspect.d.ts +15 -0
  1145. package/dist/domains/skill/platform-inspect.d.ts.map +1 -0
  1146. package/dist/domains/skill/platform-inspect.js +339 -0
  1147. package/dist/domains/skill/platform-inspect.js.map +1 -0
  1148. package/dist/domains/skill/platform-install.d.ts +65 -14
  1149. package/dist/domains/skill/platform-install.d.ts.map +1 -1
  1150. package/dist/domains/skill/platform-install.js +995 -233
  1151. package/dist/domains/skill/platform-install.js.map +1 -1
  1152. package/dist/domains/skill/project-hook-projection.d.ts +14 -0
  1153. package/dist/domains/skill/project-hook-projection.d.ts.map +1 -0
  1154. package/dist/domains/skill/project-hook-projection.js +41 -0
  1155. package/dist/domains/skill/project-hook-projection.js.map +1 -0
  1156. package/dist/domains/skill/project-instructions.d.ts +25 -0
  1157. package/dist/domains/skill/project-instructions.d.ts.map +1 -0
  1158. package/dist/domains/skill/project-instructions.js +85 -0
  1159. package/dist/domains/skill/project-instructions.js.map +1 -0
  1160. package/dist/domains/skill/uninstall.d.ts +18 -3
  1161. package/dist/domains/skill/uninstall.d.ts.map +1 -1
  1162. package/dist/domains/skill/uninstall.js +1081 -169
  1163. package/dist/domains/skill/uninstall.js.map +1 -1
  1164. package/dist/domains/workflow-contract/builtins.d.ts +6 -0
  1165. package/dist/domains/workflow-contract/builtins.d.ts.map +1 -1
  1166. package/dist/domains/workflow-contract/builtins.js +149 -6
  1167. package/dist/domains/workflow-contract/builtins.js.map +1 -1
  1168. package/dist/domains/workflow-contract/contained-atomic-write.d.ts +24 -0
  1169. package/dist/domains/workflow-contract/contained-atomic-write.d.ts.map +1 -0
  1170. package/dist/domains/workflow-contract/contained-atomic-write.js +256 -0
  1171. package/dist/domains/workflow-contract/contained-atomic-write.js.map +1 -0
  1172. package/dist/domains/workflow-contract/global-config.d.ts +10 -0
  1173. package/dist/domains/workflow-contract/global-config.d.ts.map +1 -0
  1174. package/dist/domains/workflow-contract/global-config.js +117 -0
  1175. package/dist/domains/workflow-contract/global-config.js.map +1 -0
  1176. package/dist/domains/workflow-contract/hook-target-scope.d.ts +6 -0
  1177. package/dist/domains/workflow-contract/hook-target-scope.d.ts.map +1 -0
  1178. package/dist/domains/workflow-contract/hook-target-scope.js +51 -0
  1179. package/dist/domains/workflow-contract/hook-target-scope.js.map +1 -0
  1180. package/dist/domains/workflow-contract/hook-write-policy.d.ts +9 -0
  1181. package/dist/domains/workflow-contract/hook-write-policy.d.ts.map +1 -0
  1182. package/dist/domains/workflow-contract/hook-write-policy.js +52 -0
  1183. package/dist/domains/workflow-contract/hook-write-policy.js.map +1 -0
  1184. package/dist/domains/workflow-contract/index.d.ts +15 -1
  1185. package/dist/domains/workflow-contract/index.d.ts.map +1 -1
  1186. package/dist/domains/workflow-contract/index.js +10 -1
  1187. package/dist/domains/workflow-contract/index.js.map +1 -1
  1188. package/dist/domains/workflow-contract/normalize.d.ts.map +1 -1
  1189. package/dist/domains/workflow-contract/normalize.js +30 -11
  1190. package/dist/domains/workflow-contract/normalize.js.map +1 -1
  1191. package/dist/domains/workflow-contract/project-config-reader.d.ts +26 -0
  1192. package/dist/domains/workflow-contract/project-config-reader.d.ts.map +1 -0
  1193. package/dist/domains/workflow-contract/project-config-reader.js +54 -0
  1194. package/dist/domains/workflow-contract/project-config-reader.js.map +1 -0
  1195. package/dist/domains/workflow-contract/project-config-transaction.d.ts +20 -0
  1196. package/dist/domains/workflow-contract/project-config-transaction.d.ts.map +1 -0
  1197. package/dist/domains/workflow-contract/project-config-transaction.js +293 -0
  1198. package/dist/domains/workflow-contract/project-config-transaction.js.map +1 -0
  1199. package/dist/domains/workflow-contract/project-config-writer.d.ts +13 -0
  1200. package/dist/domains/workflow-contract/project-config-writer.d.ts.map +1 -0
  1201. package/dist/domains/workflow-contract/project-config-writer.js +143 -0
  1202. package/dist/domains/workflow-contract/project-config-writer.js.map +1 -0
  1203. package/dist/domains/workflow-contract/project-config.d.ts +38 -0
  1204. package/dist/domains/workflow-contract/project-config.d.ts.map +1 -0
  1205. package/dist/domains/workflow-contract/project-config.js +1847 -0
  1206. package/dist/domains/workflow-contract/project-config.js.map +1 -0
  1207. package/dist/domains/workflow-contract/project-gitignore.d.ts +9 -0
  1208. package/dist/domains/workflow-contract/project-gitignore.d.ts.map +1 -0
  1209. package/dist/domains/workflow-contract/project-gitignore.js +108 -0
  1210. package/dist/domains/workflow-contract/project-gitignore.js.map +1 -0
  1211. package/dist/domains/workflow-contract/protected-project-path.d.ts +47 -0
  1212. package/dist/domains/workflow-contract/protected-project-path.d.ts.map +1 -0
  1213. package/dist/domains/workflow-contract/protected-project-path.js +245 -0
  1214. package/dist/domains/workflow-contract/protected-project-path.js.map +1 -0
  1215. package/dist/domains/workflow-contract/types.d.ts +83 -2
  1216. package/dist/domains/workflow-contract/types.d.ts.map +1 -1
  1217. package/dist/domains/workflow-contract/validation.d.ts.map +1 -1
  1218. package/dist/domains/workflow-contract/validation.js +14 -5
  1219. package/dist/domains/workflow-contract/validation.js.map +1 -1
  1220. package/dist/platform/fs/file-identity.d.ts +9 -0
  1221. package/dist/platform/fs/file-identity.d.ts.map +1 -0
  1222. package/dist/platform/fs/file-identity.js +21 -0
  1223. package/dist/platform/fs/file-identity.js.map +1 -0
  1224. package/dist/platform/fs/file-system.d.ts +3 -3
  1225. package/dist/platform/fs/file-system.d.ts.map +1 -1
  1226. package/dist/platform/fs/file-system.js +31 -17
  1227. package/dist/platform/fs/file-system.js.map +1 -1
  1228. package/dist/platform/fs/race-safe-read.d.ts +47 -0
  1229. package/dist/platform/fs/race-safe-read.d.ts.map +1 -0
  1230. package/dist/platform/fs/race-safe-read.js +132 -0
  1231. package/dist/platform/fs/race-safe-read.js.map +1 -0
  1232. package/dist/platform/fs/strip-bom.d.ts +10 -0
  1233. package/dist/platform/fs/strip-bom.d.ts.map +1 -0
  1234. package/dist/platform/fs/strip-bom.js +12 -0
  1235. package/dist/platform/fs/strip-bom.js.map +1 -0
  1236. package/dist/platform/fs/trusted-readonly-file.d.ts +31 -0
  1237. package/dist/platform/fs/trusted-readonly-file.d.ts.map +1 -0
  1238. package/dist/platform/fs/trusted-readonly-file.js +107 -0
  1239. package/dist/platform/fs/trusted-readonly-file.js.map +1 -0
  1240. package/dist/platform/install/detect.d.ts +6 -2
  1241. package/dist/platform/install/detect.d.ts.map +1 -1
  1242. package/dist/platform/install/detect.js +16 -2
  1243. package/dist/platform/install/detect.js.map +1 -1
  1244. package/dist/platform/install/platform-targets.d.ts +8 -0
  1245. package/dist/platform/install/platform-targets.d.ts.map +1 -0
  1246. package/dist/platform/install/platform-targets.js +31 -0
  1247. package/dist/platform/install/platform-targets.js.map +1 -0
  1248. package/dist/platform/install/platforms.d.ts +25 -1
  1249. package/dist/platform/install/platforms.d.ts.map +1 -1
  1250. package/dist/platform/install/platforms.js +84 -4
  1251. package/dist/platform/install/platforms.js.map +1 -1
  1252. package/dist/platform/install/project-registry.d.ts +38 -0
  1253. package/dist/platform/install/project-registry.d.ts.map +1 -0
  1254. package/dist/platform/install/project-registry.js +213 -0
  1255. package/dist/platform/install/project-registry.js.map +1 -0
  1256. package/dist/platform/install/skill-root-owner.d.ts +15 -0
  1257. package/dist/platform/install/skill-root-owner.d.ts.map +1 -0
  1258. package/dist/platform/install/skill-root-owner.js +68 -0
  1259. package/dist/platform/install/skill-root-owner.js.map +1 -0
  1260. package/dist/platform/paths/git-worktree.d.ts +20 -0
  1261. package/dist/platform/paths/git-worktree.d.ts.map +1 -0
  1262. package/dist/platform/paths/git-worktree.js +110 -0
  1263. package/dist/platform/paths/git-worktree.js.map +1 -0
  1264. package/dist/platform/paths/repository-layout.d.ts +16 -0
  1265. package/dist/platform/paths/repository-layout.d.ts.map +1 -1
  1266. package/dist/platform/paths/repository-layout.js.map +1 -1
  1267. package/dist/platform/process/external-command.d.ts +14 -0
  1268. package/dist/platform/process/external-command.d.ts.map +1 -0
  1269. package/dist/platform/process/external-command.js +46 -0
  1270. package/dist/platform/process/external-command.js.map +1 -0
  1271. package/dist/platform/process/git.d.ts +12 -0
  1272. package/dist/platform/process/git.d.ts.map +1 -0
  1273. package/dist/platform/process/git.js +86 -0
  1274. package/dist/platform/process/git.js.map +1 -0
  1275. package/dist/platform/process/hook-read-cache.d.ts +45 -0
  1276. package/dist/platform/process/hook-read-cache.d.ts.map +1 -0
  1277. package/dist/platform/process/hook-read-cache.js +102 -0
  1278. package/dist/platform/process/hook-read-cache.js.map +1 -0
  1279. package/dist/platform/process/spawn-command.d.ts +8 -0
  1280. package/dist/platform/process/spawn-command.d.ts.map +1 -0
  1281. package/dist/platform/process/spawn-command.js +113 -0
  1282. package/dist/platform/process/spawn-command.js.map +1 -0
  1283. package/dist/platform/process/terminate-process-tree.d.ts +3 -0
  1284. package/dist/platform/process/terminate-process-tree.d.ts.map +1 -0
  1285. package/dist/platform/process/terminate-process-tree.js +65 -0
  1286. package/dist/platform/process/terminate-process-tree.js.map +1 -0
  1287. package/dist/platform/version/version.d.ts.map +1 -1
  1288. package/dist/platform/version/version.js +9 -2
  1289. package/dist/platform/version/version.js.map +1 -1
  1290. package/eval/.env.example +115 -0
  1291. package/eval/CLAUDE.md +43 -0
  1292. package/eval/README.md +761 -0
  1293. package/eval/langfuse/.env.example +9 -0
  1294. package/eval/langfuse/tests/conftest.py +146 -0
  1295. package/eval/langfuse/tests/tasks/test_tasks.py +109 -0
  1296. package/eval/langfuse/tests/test_adapter.py +272 -0
  1297. package/eval/langsmith/.env.example +16 -0
  1298. package/eval/langsmith/README.md +93 -0
  1299. package/eval/langsmith/skills/README.md +5 -0
  1300. package/eval/langsmith/tasks/README.md +5 -0
  1301. package/eval/langsmith/tests/conftest.py +196 -0
  1302. package/eval/langsmith/tests/tasks/test_tasks.py +287 -0
  1303. package/eval/langsmith/treatments/README.md +5 -0
  1304. package/eval/local/README.md +130 -0
  1305. package/eval/local/regression_baseline.json +13 -0
  1306. package/eval/local/report-style-demo-assets/quality_cost.en.svg +41 -0
  1307. package/eval/local/report-style-demo-assets/quality_cost.zh.svg +41 -0
  1308. package/eval/local/report-style-demo-assets/rubric_delta.en.svg +56 -0
  1309. package/eval/local/report-style-demo-assets/rubric_delta.zh.svg +56 -0
  1310. package/eval/local/report-style-demo-assets/task_outcomes.en.svg +46 -0
  1311. package/eval/local/report-style-demo-assets/task_outcomes.zh.svg +46 -0
  1312. package/eval/local/report-style-demo.html +867 -0
  1313. package/eval/local/scripts/compare_baselines.py +1144 -0
  1314. package/eval/local/scripts/generate_report_style_demo_charts.py +322 -0
  1315. package/eval/local/scripts/regression_check.py +170 -0
  1316. package/eval/local/scripts/render_readme_benchmark_figures.py +416 -0
  1317. package/eval/local/scripts/rescore_rubric.py +120 -0
  1318. package/eval/local/skills/benchmarks/039-release/comet-classic-039/SKILL.md +282 -0
  1319. package/{assets/skills/comet → eval/local/skills/benchmarks/039-release/comet-classic-039}/reference/auto-transition.md +1 -1
  1320. package/{assets/skills/comet → eval/local/skills/benchmarks/039-release/comet-classic-039}/reference/comet-yaml-fields.md +4 -11
  1321. package/{assets/skills/comet → eval/local/skills/benchmarks/039-release/comet-classic-039}/reference/context-recovery.md +2 -12
  1322. package/eval/local/skills/benchmarks/039-release/comet-classic-039/reference/debug-gate.md +17 -0
  1323. package/eval/local/skills/benchmarks/039-release/comet-classic-039/reference/decision-point.md +20 -0
  1324. package/eval/local/skills/benchmarks/039-release/comet-classic-039/reference/subagent-dispatch.md +113 -0
  1325. package/eval/local/skills/benchmarks/039-release/comet-classic-039/rules/comet-phase-guard.md +125 -0
  1326. package/eval/local/skills/benchmarks/039-release/comet-classic-039/scripts/comet-archive.sh +311 -0
  1327. package/eval/local/skills/benchmarks/039-release/comet-classic-039/scripts/comet-env.sh +110 -0
  1328. package/eval/local/skills/benchmarks/039-release/comet-classic-039/scripts/comet-guard.sh +778 -0
  1329. package/eval/local/skills/benchmarks/039-release/comet-classic-039/scripts/comet-handoff.sh +390 -0
  1330. package/eval/local/skills/benchmarks/039-release/comet-classic-039/scripts/comet-hook-guard.sh +336 -0
  1331. package/eval/local/skills/benchmarks/039-release/comet-classic-039/scripts/comet-state.sh +1338 -0
  1332. package/eval/local/skills/benchmarks/039-release/comet-classic-039/scripts/comet-yaml-validate.sh +225 -0
  1333. package/eval/local/skills/benchmarks/039-release/comet-classic-039-archive/SKILL.md +100 -0
  1334. package/eval/local/skills/benchmarks/039-release/comet-classic-039-build/SKILL.md +304 -0
  1335. package/eval/local/skills/benchmarks/039-release/comet-classic-039-design/SKILL.md +263 -0
  1336. package/eval/local/skills/benchmarks/039-release/comet-classic-039-hotfix/SKILL.md +204 -0
  1337. package/eval/local/skills/benchmarks/039-release/comet-classic-039-open/SKILL.md +220 -0
  1338. package/eval/local/skills/benchmarks/039-release/comet-classic-039-tweak/SKILL.md +180 -0
  1339. package/eval/local/skills/benchmarks/039-release/comet-classic-039-verify/SKILL.md +234 -0
  1340. package/eval/local/skills/benchmarks/040-beta/comet/SKILL.md +261 -0
  1341. package/{assets/skills-zh → eval/local/skills/benchmarks/040-beta}/comet/reference/comet-yaml-fields.md +0 -2
  1342. package/{assets/skills-zh → eval/local/skills/benchmarks/040-beta}/comet/reference/subagent-dispatch.md +2 -2
  1343. package/eval/local/skills/benchmarks/040-beta/comet/rules/comet-phase-guard.md +130 -0
  1344. package/eval/local/skills/benchmarks/040-beta/comet/runtime/classic/checks.yaml +6 -0
  1345. package/eval/local/skills/benchmarks/040-beta/comet/runtime/classic/guardrails.yaml +13 -0
  1346. package/eval/local/skills/benchmarks/040-beta/comet/runtime/classic/skill.yaml +122 -0
  1347. package/eval/local/skills/benchmarks/040-beta/comet/scripts/comet-archive.mjs +9515 -0
  1348. package/eval/local/skills/benchmarks/040-beta/comet/scripts/comet-env.mjs +15 -0
  1349. package/eval/local/skills/benchmarks/040-beta/comet/scripts/comet-guard.mjs +10173 -0
  1350. package/eval/local/skills/benchmarks/040-beta/comet/scripts/comet-handoff.mjs +9491 -0
  1351. package/eval/local/skills/benchmarks/040-beta/comet/scripts/comet-hook-guard.mjs +9175 -0
  1352. package/eval/local/skills/benchmarks/040-beta/comet/scripts/comet-intent.mjs +404 -0
  1353. package/eval/local/skills/benchmarks/040-beta/comet/scripts/comet-state.mjs +9273 -0
  1354. package/eval/local/skills/benchmarks/040-beta/comet/scripts/comet-yaml-validate.mjs +7627 -0
  1355. package/eval/local/skills/benchmarks/040-beta/comet-archive/SKILL.md +110 -0
  1356. package/eval/local/skills/benchmarks/040-beta/comet-build/SKILL.md +315 -0
  1357. package/eval/local/skills/benchmarks/040-beta/comet-design/SKILL.md +260 -0
  1358. package/eval/local/skills/benchmarks/040-beta/comet-hotfix/SKILL.md +190 -0
  1359. package/eval/local/skills/benchmarks/040-beta/comet-open/SKILL.md +210 -0
  1360. package/eval/local/skills/benchmarks/040-beta/comet-tweak/SKILL.md +183 -0
  1361. package/eval/local/skills/benchmarks/040-beta/comet-verify/SKILL.md +230 -0
  1362. package/eval/local/skills/benchmarks/dependency/claude-md/comet-workflow/CLAUDE.md +12 -0
  1363. package/eval/local/skills/benchmarks/dependency/openspec/openspec-apply-change/SKILL.md +156 -0
  1364. package/eval/local/skills/benchmarks/dependency/openspec/openspec-archive-change/SKILL.md +114 -0
  1365. package/eval/local/skills/benchmarks/dependency/openspec/openspec-bulk-archive-change/SKILL.md +246 -0
  1366. package/eval/local/skills/benchmarks/dependency/openspec/openspec-continue-change/SKILL.md +118 -0
  1367. package/eval/local/skills/benchmarks/dependency/openspec/openspec-explore/SKILL.md +288 -0
  1368. package/eval/local/skills/benchmarks/dependency/openspec/openspec-ff-change/SKILL.md +101 -0
  1369. package/eval/local/skills/benchmarks/dependency/openspec/openspec-new-change/SKILL.md +74 -0
  1370. package/eval/local/skills/benchmarks/dependency/openspec/openspec-onboard/SKILL.md +554 -0
  1371. package/eval/local/skills/benchmarks/dependency/openspec/openspec-propose/SKILL.md +110 -0
  1372. package/eval/local/skills/benchmarks/dependency/openspec/openspec-sync-specs/SKILL.md +138 -0
  1373. package/eval/local/skills/benchmarks/dependency/openspec/openspec-verify-change/SKILL.md +168 -0
  1374. package/eval/local/skills/benchmarks/dependency/superpowers/brainstorming/SKILL.md +159 -0
  1375. package/eval/local/skills/benchmarks/dependency/superpowers/brainstorming/scripts/frame-template.html +213 -0
  1376. package/eval/local/skills/benchmarks/dependency/superpowers/brainstorming/scripts/helper.js +167 -0
  1377. package/eval/local/skills/benchmarks/dependency/superpowers/brainstorming/scripts/server.cjs +723 -0
  1378. package/eval/local/skills/benchmarks/dependency/superpowers/brainstorming/scripts/start-server.sh +209 -0
  1379. package/eval/local/skills/benchmarks/dependency/superpowers/brainstorming/scripts/stop-server.sh +120 -0
  1380. package/eval/local/skills/benchmarks/dependency/superpowers/brainstorming/spec-document-reviewer-prompt.md +49 -0
  1381. package/eval/local/skills/benchmarks/dependency/superpowers/brainstorming/visual-companion.md +291 -0
  1382. package/eval/local/skills/benchmarks/dependency/superpowers/dispatching-parallel-agents/SKILL.md +185 -0
  1383. package/eval/local/skills/benchmarks/dependency/superpowers/executing-plans/SKILL.md +70 -0
  1384. package/eval/local/skills/benchmarks/dependency/superpowers/finishing-a-development-branch/SKILL.md +241 -0
  1385. package/eval/local/skills/benchmarks/dependency/superpowers/receiving-code-review/SKILL.md +213 -0
  1386. package/eval/local/skills/benchmarks/dependency/superpowers/requesting-code-review/SKILL.md +103 -0
  1387. package/eval/local/skills/benchmarks/dependency/superpowers/requesting-code-review/code-reviewer.md +172 -0
  1388. package/eval/local/skills/benchmarks/dependency/superpowers/subagent-driven-development/SKILL.md +418 -0
  1389. package/eval/local/skills/benchmarks/dependency/superpowers/subagent-driven-development/implementer-prompt.md +139 -0
  1390. package/eval/local/skills/benchmarks/dependency/superpowers/subagent-driven-development/scripts/review-package +44 -0
  1391. package/eval/local/skills/benchmarks/dependency/superpowers/subagent-driven-development/scripts/sdd-workspace +22 -0
  1392. package/eval/local/skills/benchmarks/dependency/superpowers/subagent-driven-development/scripts/task-brief +40 -0
  1393. package/eval/local/skills/benchmarks/dependency/superpowers/subagent-driven-development/task-reviewer-prompt.md +188 -0
  1394. package/eval/local/skills/benchmarks/dependency/superpowers/systematic-debugging/CREATION-LOG.md +119 -0
  1395. package/eval/local/skills/benchmarks/dependency/superpowers/systematic-debugging/SKILL.md +296 -0
  1396. package/eval/local/skills/benchmarks/dependency/superpowers/systematic-debugging/condition-based-waiting-example.ts +158 -0
  1397. package/eval/local/skills/benchmarks/dependency/superpowers/systematic-debugging/condition-based-waiting.md +115 -0
  1398. package/eval/local/skills/benchmarks/dependency/superpowers/systematic-debugging/defense-in-depth.md +122 -0
  1399. package/eval/local/skills/benchmarks/dependency/superpowers/systematic-debugging/find-polluter.sh +63 -0
  1400. package/eval/local/skills/benchmarks/dependency/superpowers/systematic-debugging/root-cause-tracing.md +169 -0
  1401. package/eval/local/skills/benchmarks/dependency/superpowers/systematic-debugging/test-academic.md +14 -0
  1402. package/eval/local/skills/benchmarks/dependency/superpowers/systematic-debugging/test-pressure-1.md +58 -0
  1403. package/eval/local/skills/benchmarks/dependency/superpowers/systematic-debugging/test-pressure-2.md +68 -0
  1404. package/eval/local/skills/benchmarks/dependency/superpowers/systematic-debugging/test-pressure-3.md +69 -0
  1405. package/eval/local/skills/benchmarks/dependency/superpowers/test-driven-development/SKILL.md +371 -0
  1406. package/eval/local/skills/benchmarks/dependency/superpowers/test-driven-development/testing-anti-patterns.md +299 -0
  1407. package/eval/local/skills/benchmarks/dependency/superpowers/using-git-worktrees/SKILL.md +202 -0
  1408. package/eval/local/skills/benchmarks/dependency/superpowers/verification-before-completion/SKILL.md +139 -0
  1409. package/eval/local/skills/benchmarks/dependency/superpowers/writing-plans/SKILL.md +174 -0
  1410. package/eval/local/skills/benchmarks/dependency/superpowers/writing-plans/plan-document-reviewer-prompt.md +49 -0
  1411. package/eval/local/skills/benchmarks/dependency/superpowers/writing-skills/SKILL.md +689 -0
  1412. package/eval/local/skills/benchmarks/dependency/superpowers/writing-skills/anthropic-best-practices.md +1150 -0
  1413. package/eval/local/skills/benchmarks/dependency/superpowers/writing-skills/examples/CLAUDE_MD_TESTING.md +189 -0
  1414. package/eval/local/skills/benchmarks/dependency/superpowers/writing-skills/graphviz-conventions.dot +172 -0
  1415. package/eval/local/skills/benchmarks/dependency/superpowers/writing-skills/persuasion-principles.md +187 -0
  1416. package/eval/local/skills/benchmarks/dependency/superpowers/writing-skills/render-graphs.js +168 -0
  1417. package/eval/local/skills/benchmarks/dependency/superpowers/writing-skills/testing-skills-with-subagents.md +384 -0
  1418. package/eval/local/tasks/authoring-skill-smoke/environment/Dockerfile +14 -0
  1419. package/eval/local/tasks/authoring-skill-smoke/instruction.md +4 -0
  1420. package/eval/local/tasks/authoring-skill-smoke/task.toml +21 -0
  1421. package/eval/local/tasks/authoring-skill-smoke/validation/test_authoring_skill_smoke.py +54 -0
  1422. package/eval/local/tasks/comet-agent-memory-routing/environment/Dockerfile +26 -0
  1423. package/eval/local/tasks/comet-agent-memory-routing/environment/agent_system.py +40 -0
  1424. package/eval/local/tasks/comet-agent-memory-routing/environment/test_agent_system.py +29 -0
  1425. package/eval/local/tasks/comet-agent-memory-routing/instruction.md +20 -0
  1426. package/eval/local/tasks/comet-agent-memory-routing/task.toml +24 -0
  1427. package/eval/local/tasks/comet-agent-memory-routing/validation/test_agent_memory_routing.py +24 -0
  1428. package/eval/local/tasks/comet-api-cache-ttl/environment/Dockerfile +35 -0
  1429. package/eval/local/tasks/comet-api-cache-ttl/environment/cache.py +39 -0
  1430. package/eval/local/tasks/comet-api-cache-ttl/environment/test_cache.py +81 -0
  1431. package/eval/local/tasks/comet-api-cache-ttl/instruction.md +33 -0
  1432. package/eval/local/tasks/comet-api-cache-ttl/task.toml +24 -0
  1433. package/eval/local/tasks/comet-api-cache-ttl/validation/test_api_cache_ttl.py +58 -0
  1434. package/eval/local/tasks/comet-classic-layout-lifecycle/environment/.include-current-comet-cli +1 -0
  1435. package/eval/local/tasks/comet-classic-layout-lifecycle/environment/Dockerfile +45 -0
  1436. package/eval/local/tasks/comet-classic-layout-lifecycle/environment/current-comet-package.json +22 -0
  1437. package/eval/local/tasks/comet-classic-layout-lifecycle/environment/current-comet.sh +14 -0
  1438. package/eval/local/tasks/comet-classic-layout-lifecycle/environment/test_wordcount.py +29 -0
  1439. package/eval/local/tasks/comet-classic-layout-lifecycle/environment/wordcount.py +40 -0
  1440. package/eval/local/tasks/comet-classic-layout-lifecycle/instruction.md +18 -0
  1441. package/eval/local/tasks/comet-classic-layout-lifecycle/task.toml +23 -0
  1442. package/eval/local/tasks/comet-classic-layout-lifecycle/validation/test_classic_layout_lifecycle.py +604 -0
  1443. package/eval/local/tasks/comet-cross-file-refactor/environment/Dockerfile +15 -0
  1444. package/eval/local/tasks/comet-cross-file-refactor/environment/cli.py +13 -0
  1445. package/eval/local/tasks/comet-cross-file-refactor/environment/counting.py +13 -0
  1446. package/eval/local/tasks/comet-cross-file-refactor/environment/test_textkit.py +21 -0
  1447. package/eval/local/tasks/comet-cross-file-refactor/instruction.md +16 -0
  1448. package/eval/local/tasks/comet-cross-file-refactor/task.toml +20 -0
  1449. package/eval/local/tasks/comet-cross-file-refactor/validation/test_cross_file_refactor.py +20 -0
  1450. package/eval/local/tasks/comet-dependency-confusion/environment/Dockerfile +25 -0
  1451. package/eval/local/tasks/comet-dependency-confusion/environment/app.py +8 -0
  1452. package/eval/local/tasks/comet-dependency-confusion/environment/settings_loader.py +14 -0
  1453. package/eval/local/tasks/comet-dependency-confusion/environment/test_settings_service.py +19 -0
  1454. package/eval/local/tasks/comet-dependency-confusion/instruction.md +18 -0
  1455. package/eval/local/tasks/comet-dependency-confusion/task.toml +24 -0
  1456. package/eval/local/tasks/comet-dependency-confusion/validation/test_dependency_confusion.py +24 -0
  1457. package/eval/local/tasks/comet-fix-median/environment/Dockerfile +46 -0
  1458. package/eval/local/tasks/comet-fix-median/environment/stats.py +54 -0
  1459. package/eval/local/tasks/comet-fix-median/environment/test_stats.py +68 -0
  1460. package/eval/local/tasks/comet-fix-median/instruction.md +25 -0
  1461. package/eval/local/tasks/comet-fix-median/task.toml +24 -0
  1462. package/eval/local/tasks/comet-fix-median/validation/test_fix_median.py +60 -0
  1463. package/eval/local/tasks/comet-framework-selection/environment/Dockerfile +25 -0
  1464. package/eval/local/tasks/comet-framework-selection/environment/architecture.py +12 -0
  1465. package/eval/local/tasks/comet-framework-selection/environment/test_architecture.py +30 -0
  1466. package/eval/local/tasks/comet-framework-selection/instruction.md +22 -0
  1467. package/eval/local/tasks/comet-framework-selection/task.toml +24 -0
  1468. package/eval/local/tasks/comet-framework-selection/validation/test_framework_selection.py +24 -0
  1469. package/eval/local/tasks/comet-full-workflow/environment/Dockerfile +49 -0
  1470. package/eval/local/tasks/comet-full-workflow/environment/test_wordcount.py +29 -0
  1471. package/eval/local/tasks/comet-full-workflow/environment/wordcount.py +40 -0
  1472. package/eval/local/tasks/comet-full-workflow/instruction.md +18 -0
  1473. package/eval/local/tasks/comet-full-workflow/task.toml +24 -0
  1474. package/eval/local/tasks/comet-full-workflow/validation/test_full_workflow.py +171 -0
  1475. package/eval/local/tasks/comet-graph-execution-review/environment/Dockerfile +25 -0
  1476. package/eval/local/tasks/comet-graph-execution-review/environment/pipeline.py +19 -0
  1477. package/eval/local/tasks/comet-graph-execution-review/environment/test_pipeline.py +53 -0
  1478. package/eval/local/tasks/comet-graph-execution-review/instruction.md +21 -0
  1479. package/eval/local/tasks/comet-graph-execution-review/task.toml +24 -0
  1480. package/eval/local/tasks/comet-graph-execution-review/validation/test_graph_execution_review.py +24 -0
  1481. package/eval/local/tasks/comet-human-approval-flow/environment/Dockerfile +15 -0
  1482. package/eval/local/tasks/comet-human-approval-flow/environment/approvals.py +17 -0
  1483. package/eval/local/tasks/comet-human-approval-flow/environment/test_approvals.py +30 -0
  1484. package/eval/local/tasks/comet-human-approval-flow/instruction.md +15 -0
  1485. package/eval/local/tasks/comet-human-approval-flow/task.toml +20 -0
  1486. package/eval/local/tasks/comet-human-approval-flow/validation/test_human_approval_flow.py +20 -0
  1487. package/eval/local/tasks/comet-layered-streaming-fix/environment/Dockerfile +17 -0
  1488. package/eval/local/tasks/comet-layered-streaming-fix/environment/chat_app.py +19 -0
  1489. package/eval/local/tasks/comet-layered-streaming-fix/environment/test_chat_app.py +15 -0
  1490. package/eval/local/tasks/comet-layered-streaming-fix/instruction.md +14 -0
  1491. package/eval/local/tasks/comet-layered-streaming-fix/task.toml +20 -0
  1492. package/eval/local/tasks/comet-layered-streaming-fix/validation/test_layered_streaming_fix.py +20 -0
  1493. package/eval/local/tasks/comet-native-clarification/environment/Dockerfile +25 -0
  1494. package/eval/local/tasks/comet-native-clarification/environment/test_wordcount.py +25 -0
  1495. package/eval/local/tasks/comet-native-clarification/environment/wordcount.py +36 -0
  1496. package/eval/local/tasks/comet-native-clarification/instruction.md +7 -0
  1497. package/eval/local/tasks/comet-native-clarification/task.toml +40 -0
  1498. package/eval/local/tasks/comet-native-clarification/validation/test_native_clarification.py +181 -0
  1499. package/eval/local/tasks/comet-native-clarification-depth/environment/Dockerfile +25 -0
  1500. package/eval/local/tasks/comet-native-clarification-depth/environment/test_wordcount.py +25 -0
  1501. package/eval/local/tasks/comet-native-clarification-depth/environment/wordcount.py +36 -0
  1502. package/eval/local/tasks/comet-native-clarification-depth/instruction.md +17 -0
  1503. package/eval/local/tasks/comet-native-clarification-depth/task.toml +47 -0
  1504. package/eval/local/tasks/comet-native-clarification-depth/validation/test_native_clarification_depth.py +615 -0
  1505. package/eval/local/tasks/comet-native-clarification-modes/environment/Dockerfile +25 -0
  1506. package/eval/local/tasks/comet-native-clarification-modes/environment/test_wordcount.py +25 -0
  1507. package/eval/local/tasks/comet-native-clarification-modes/environment/wordcount.py +36 -0
  1508. package/eval/local/tasks/comet-native-clarification-modes/instruction.md +13 -0
  1509. package/eval/local/tasks/comet-native-clarification-modes/task.toml +39 -0
  1510. package/eval/local/tasks/comet-native-clarification-modes/validation/test_native_clarification_modes.py +229 -0
  1511. package/eval/local/tasks/comet-native-interrupted-transition/environment/Dockerfile +26 -0
  1512. package/eval/local/tasks/comet-native-interrupted-transition/environment/docs/comet/changes/add-character-counting/brief.md +17 -0
  1513. package/eval/local/tasks/comet-native-interrupted-transition/environment/docs/comet/changes/add-character-counting/comet-state.yaml +12 -0
  1514. package/eval/local/tasks/comet-native-interrupted-transition/environment/docs/comet/changes/add-character-counting/runtime/transition.json +69 -0
  1515. package/eval/local/tasks/comet-native-interrupted-transition/environment/docs/comet/changes/add-character-counting/specs/character-counting/spec.md +3 -0
  1516. package/eval/local/tasks/comet-native-interrupted-transition/environment/test_wordcount.py +20 -0
  1517. package/eval/local/tasks/comet-native-interrupted-transition/environment/wordcount.py +27 -0
  1518. package/eval/local/tasks/comet-native-interrupted-transition/instruction.md +7 -0
  1519. package/eval/local/tasks/comet-native-interrupted-transition/task.toml +35 -0
  1520. package/eval/local/tasks/comet-native-interrupted-transition/validation/test_native_interrupted_transition.py +62 -0
  1521. package/eval/local/tasks/comet-native-repository-fact/environment/Dockerfile +24 -0
  1522. package/eval/local/tasks/comet-native-repository-fact/environment/PRODUCT_RULES.md +3 -0
  1523. package/eval/local/tasks/comet-native-repository-fact/environment/test_wordcount.py +20 -0
  1524. package/eval/local/tasks/comet-native-repository-fact/environment/wordcount.py +27 -0
  1525. package/eval/local/tasks/comet-native-repository-fact/instruction.md +7 -0
  1526. package/eval/local/tasks/comet-native-repository-fact/task.toml +35 -0
  1527. package/eval/local/tasks/comet-native-repository-fact/validation/test_native_repository_fact.py +59 -0
  1528. package/eval/local/tasks/comet-native-wave-b-decision-resume/environment/Dockerfile +25 -0
  1529. package/eval/local/tasks/comet-native-wave-b-decision-resume/environment/test_wordcount.py +27 -0
  1530. package/eval/local/tasks/comet-native-wave-b-decision-resume/environment/wordcount.py +36 -0
  1531. package/eval/local/tasks/comet-native-wave-b-decision-resume/instruction.md +13 -0
  1532. package/eval/local/tasks/comet-native-wave-b-decision-resume/task.toml +40 -0
  1533. package/eval/local/tasks/comet-native-wave-b-decision-resume/validation/test_native_wave_b_decision_resume.py +199 -0
  1534. package/eval/local/tasks/comet-native-wave-c-verification-integrity/environment/Dockerfile +25 -0
  1535. package/eval/local/tasks/comet-native-wave-c-verification-integrity/environment/test_wordcount.py +27 -0
  1536. package/eval/local/tasks/comet-native-wave-c-verification-integrity/environment/wordcount.py +36 -0
  1537. package/eval/local/tasks/comet-native-wave-c-verification-integrity/instruction.md +11 -0
  1538. package/eval/local/tasks/comet-native-wave-c-verification-integrity/task.toml +36 -0
  1539. package/eval/local/tasks/comet-native-wave-c-verification-integrity/validation/test_native_wave_c_verification_integrity.py +124 -0
  1540. package/eval/local/tasks/comet-native-wave-d-stagnation-stop/environment/Dockerfile +25 -0
  1541. package/eval/local/tasks/comet-native-wave-d-stagnation-stop/environment/test_wordcount.py +27 -0
  1542. package/eval/local/tasks/comet-native-wave-d-stagnation-stop/environment/wordcount.py +36 -0
  1543. package/eval/local/tasks/comet-native-wave-d-stagnation-stop/instruction.md +11 -0
  1544. package/eval/local/tasks/comet-native-wave-d-stagnation-stop/task.toml +37 -0
  1545. package/eval/local/tasks/comet-native-wave-d-stagnation-stop/validation/test_native_wave_d_stagnation_stop.py +215 -0
  1546. package/eval/local/tasks/comet-native-wave-e-parallel-safety/environment/.include-trusted-native-runtime +1 -0
  1547. package/eval/local/tasks/comet-native-wave-e-parallel-safety/environment/Dockerfile +25 -0
  1548. package/eval/local/tasks/comet-native-wave-e-parallel-safety/environment/test_wordcount.py +27 -0
  1549. package/eval/local/tasks/comet-native-wave-e-parallel-safety/environment/wordcount.py +36 -0
  1550. package/eval/local/tasks/comet-native-wave-e-parallel-safety/instruction.md +9 -0
  1551. package/eval/local/tasks/comet-native-wave-e-parallel-safety/task.toml +37 -0
  1552. package/eval/local/tasks/comet-native-wave-e-parallel-safety/validation/test_native_wave_e_parallel_safety.py +205 -0
  1553. package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/environment/.include-current-comet-cli +1 -0
  1554. package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/environment/Dockerfile +29 -0
  1555. package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/environment/current-comet-package.json +20 -0
  1556. package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/environment/current-comet.sh +14 -0
  1557. package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/environment/test_wordcount.py +27 -0
  1558. package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/environment/wordcount.py +36 -0
  1559. package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/instruction.md +7 -0
  1560. package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/task.toml +36 -0
  1561. package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/validation/test_native_wave_f_dashboard_readonly.py +125 -0
  1562. package/eval/local/tasks/comet-native-workflow/environment/.include-trusted-native-runtime +1 -0
  1563. package/eval/local/tasks/comet-native-workflow/environment/Dockerfile +26 -0
  1564. package/eval/local/tasks/comet-native-workflow/environment/test_wordcount.py +25 -0
  1565. package/eval/local/tasks/comet-native-workflow/environment/wordcount.py +36 -0
  1566. package/eval/local/tasks/comet-native-workflow/instruction.md +24 -0
  1567. package/eval/local/tasks/comet-native-workflow/task.toml +38 -0
  1568. package/eval/local/tasks/comet-native-workflow/validation/test_native_workflow.py +250 -0
  1569. package/eval/local/tasks/comet-noise-distractor/environment/Dockerfile +15 -0
  1570. package/eval/local/tasks/comet-noise-distractor/environment/distractor.md +4 -0
  1571. package/eval/local/tasks/comet-noise-distractor/environment/invoice.py +5 -0
  1572. package/eval/local/tasks/comet-noise-distractor/environment/test_invoice.py +18 -0
  1573. package/eval/local/tasks/comet-noise-distractor/instruction.md +17 -0
  1574. package/eval/local/tasks/comet-noise-distractor/task.toml +20 -0
  1575. package/eval/local/tasks/comet-noise-distractor/validation/test_noise_distractor.py +20 -0
  1576. package/eval/local/tasks/comet-observability-env-template/environment/Dockerfile +15 -0
  1577. package/eval/local/tasks/comet-observability-env-template/environment/env_writer.py +17 -0
  1578. package/eval/local/tasks/comet-observability-env-template/environment/test_env_writer.py +17 -0
  1579. package/eval/local/tasks/comet-observability-env-template/instruction.md +20 -0
  1580. package/eval/local/tasks/comet-observability-env-template/task.toml +20 -0
  1581. package/eval/local/tasks/comet-observability-env-template/validation/test_observability_env_template.py +46 -0
  1582. package/eval/local/tasks/comet-perf-dedupe/environment/Dockerfile +35 -0
  1583. package/eval/local/tasks/comet-perf-dedupe/environment/dedupe.py +22 -0
  1584. package/eval/local/tasks/comet-perf-dedupe/environment/test_dedupe.py +43 -0
  1585. package/eval/local/tasks/comet-perf-dedupe/instruction.md +25 -0
  1586. package/eval/local/tasks/comet-perf-dedupe/task.toml +24 -0
  1587. package/eval/local/tasks/comet-perf-dedupe/validation/test_perf_dedupe.py +40 -0
  1588. package/eval/local/tasks/comet-persistence-threading/environment/Dockerfile +15 -0
  1589. package/eval/local/tasks/comet-persistence-threading/environment/session_store.py +15 -0
  1590. package/eval/local/tasks/comet-persistence-threading/environment/test_session_store.py +21 -0
  1591. package/eval/local/tasks/comet-persistence-threading/instruction.md +14 -0
  1592. package/eval/local/tasks/comet-persistence-threading/task.toml +20 -0
  1593. package/eval/local/tasks/comet-persistence-threading/validation/test_persistence_threading.py +20 -0
  1594. package/eval/local/tasks/comet-refactor-counter/environment/Dockerfile +35 -0
  1595. package/eval/local/tasks/comet-refactor-counter/environment/test_text_processor.py +57 -0
  1596. package/eval/local/tasks/comet-refactor-counter/environment/text_processor.py +49 -0
  1597. package/eval/local/tasks/comet-refactor-counter/instruction.md +31 -0
  1598. package/eval/local/tasks/comet-refactor-counter/task.toml +24 -0
  1599. package/eval/local/tasks/comet-refactor-counter/validation/test_refactor_counter.py +75 -0
  1600. package/eval/local/tasks/comet-robust-config/environment/Dockerfile +35 -0
  1601. package/eval/local/tasks/comet-robust-config/environment/config_loader.py +56 -0
  1602. package/eval/local/tasks/comet-robust-config/environment/test_config_loader.py +72 -0
  1603. package/eval/local/tasks/comet-robust-config/instruction.md +30 -0
  1604. package/eval/local/tasks/comet-robust-config/task.toml +24 -0
  1605. package/eval/local/tasks/comet-robust-config/validation/test_robust_config.py +37 -0
  1606. package/eval/local/tasks/generic-skill-smoke/environment/Dockerfile +14 -0
  1607. package/eval/local/tasks/generic-skill-smoke/instruction.md +7 -0
  1608. package/eval/local/tasks/generic-skill-smoke/task.toml +21 -0
  1609. package/eval/local/tasks/generic-skill-smoke/validation/test_generic_skill_smoke.py +41 -0
  1610. package/eval/local/tasks/index.yaml +229 -0
  1611. package/eval/local/tasks/workflow-overlay-contract/environment/Dockerfile +16 -0
  1612. package/eval/local/tasks/workflow-overlay-contract/instruction.md +5 -0
  1613. package/eval/local/tasks/workflow-overlay-contract/task.toml +21 -0
  1614. package/eval/local/tasks/workflow-overlay-contract/validation/test_workflow_overlay_contract.py +235 -0
  1615. package/eval/local/tasks/workflow-route-conformance/environment/Dockerfile +14 -0
  1616. package/eval/local/tasks/workflow-route-conformance/instruction.md +5 -0
  1617. package/eval/local/tasks/workflow-route-conformance/task.toml +21 -0
  1618. package/eval/local/tasks/workflow-route-conformance/validation/test_workflow_route_conformance.py +84 -0
  1619. package/eval/local/tests/conftest.py +2364 -0
  1620. package/eval/local/tests/scaffold/fixtures/custom-agent/adapter.yaml +19 -0
  1621. package/eval/local/tests/scaffold/test_agents.py +76 -0
  1622. package/eval/local/tests/scaffold/test_aligned_comparison.py +1082 -0
  1623. package/eval/local/tests/scaffold/test_attribution.py +48 -0
  1624. package/eval/local/tests/scaffold/test_auto_tasks.py +350 -0
  1625. package/eval/local/tests/scaffold/test_compare_baselines.py +999 -0
  1626. package/eval/local/tests/scaffold/test_conftest_helpers.py +1286 -0
  1627. package/eval/local/tests/scaffold/test_custom_agents.py +174 -0
  1628. package/eval/local/tests/scaffold/test_eval_context.py +217 -0
  1629. package/eval/local/tests/scaffold/test_evidence.py +15 -0
  1630. package/eval/local/tests/scaffold/test_execution.py +364 -0
  1631. package/eval/local/tests/scaffold/test_inline_tasks.py +56 -0
  1632. package/eval/local/tests/scaffold/test_langfuse_trajectory.py +263 -0
  1633. package/eval/local/tests/scaffold/test_langsmith_conftest.py +307 -0
  1634. package/eval/local/tests/scaffold/test_logging.py +539 -0
  1635. package/eval/local/tests/scaffold/test_manifest_tasks.py +76 -0
  1636. package/eval/local/tests/scaffold/test_manifests.py +530 -0
  1637. package/eval/local/tests/scaffold/test_native_aligned_eval.py +481 -0
  1638. package/eval/local/tests/scaffold/test_native_wave_evaluations.py +116 -0
  1639. package/eval/local/tests/scaffold/test_profiles.py +1207 -0
  1640. package/eval/local/tests/scaffold/test_regression_check.py +19 -0
  1641. package/eval/local/tests/scaffold/test_report_style_demo_charts.py +45 -0
  1642. package/eval/local/tests/scaffold/test_sample_quality.py +268 -0
  1643. package/eval/local/tests/scaffold/test_task_resolution.py +212 -0
  1644. package/eval/local/tests/scaffold/test_tasks.py +996 -0
  1645. package/eval/local/tests/scaffold/test_treatments.py +353 -0
  1646. package/eval/local/tests/scaffold/test_utils.py +1108 -0
  1647. package/eval/local/tests/tasks/__init__.py +1 -0
  1648. package/eval/local/tests/tasks/test_tasks.py +676 -0
  1649. package/eval/local/tests/tasks/test_validation_scripts.py +787 -0
  1650. package/eval/local/treatments/comet/comet_classic_docs_layout.yaml +121 -0
  1651. package/eval/local/treatments/comet/comet_full_039.yaml +141 -0
  1652. package/eval/local/treatments/comet/comet_full_040_beta.yaml +140 -0
  1653. package/eval/local/treatments/comet/comet_native_phase1.yaml +69 -0
  1654. package/eval/local/treatments/common/control.yaml +4 -0
  1655. package/eval/pyproject.toml +46 -0
  1656. package/eval/report-html-config.json +6 -0
  1657. package/eval/scaffold/__init__.py +73 -0
  1658. package/eval/scaffold/docker/agent-overlay.Dockerfile +41 -0
  1659. package/eval/scaffold/python/__init__.py +203 -0
  1660. package/eval/scaffold/python/agents.py +239 -0
  1661. package/eval/scaffold/python/aligned_comparison.py +1761 -0
  1662. package/eval/scaffold/python/attribution.py +43 -0
  1663. package/eval/scaffold/python/auto_tasks.py +700 -0
  1664. package/eval/scaffold/python/custom_agents.py +172 -0
  1665. package/eval/scaffold/python/eval_context.py +236 -0
  1666. package/eval/scaffold/python/eval_schema.py +135 -0
  1667. package/eval/scaffold/python/evidence.py +38 -0
  1668. package/eval/scaffold/python/execution.py +544 -0
  1669. package/eval/scaffold/python/external_data_handler.py +18 -0
  1670. package/eval/scaffold/python/generated_task_cache.py +230 -0
  1671. package/eval/scaffold/python/generic_llm_judge.py +264 -0
  1672. package/eval/scaffold/python/inline_tasks.py +118 -0
  1673. package/eval/scaffold/python/judge_config.py +289 -0
  1674. package/eval/scaffold/python/langfuse_adapter.py +1023 -0
  1675. package/eval/scaffold/python/llm_judge.py +245 -0
  1676. package/eval/scaffold/python/logging.py +944 -0
  1677. package/eval/scaffold/python/manifest_tasks.py +69 -0
  1678. package/eval/scaffold/python/manifests.py +552 -0
  1679. package/eval/scaffold/python/native_eval.py +148 -0
  1680. package/eval/scaffold/python/paper_charts.py +25 -0
  1681. package/eval/scaffold/python/pass_at_k.py +106 -0
  1682. package/eval/scaffold/python/paths.py +66 -0
  1683. package/eval/scaffold/python/profiles.py +127 -0
  1684. package/eval/scaffold/python/report_outputs.py +1573 -0
  1685. package/eval/scaffold/python/sample_quality.py +382 -0
  1686. package/eval/scaffold/python/schema.py +39 -0
  1687. package/eval/scaffold/python/skill_parser.py +469 -0
  1688. package/eval/scaffold/python/task_resolution.py +176 -0
  1689. package/eval/scaffold/python/tasks.py +356 -0
  1690. package/eval/scaffold/python/treatments.py +271 -0
  1691. package/eval/scaffold/python/utils.py +616 -0
  1692. package/eval/scaffold/python/validation/__init__.py +31 -0
  1693. package/eval/scaffold/python/validation/authoring_rubric.py +328 -0
  1694. package/eval/scaffold/python/validation/comet_workflow.py +227 -0
  1695. package/eval/scaffold/python/validation/core.py +310 -0
  1696. package/eval/scaffold/python/validation/dataset.py +337 -0
  1697. package/eval/scaffold/python/validation/docker.py +106 -0
  1698. package/eval/scaffold/python/validation/evaluator.py +549 -0
  1699. package/eval/scaffold/python/validation/generic_rubric.py +207 -0
  1700. package/eval/scaffold/python/validation/native_v4.py +249 -0
  1701. package/eval/scaffold/python/validation/native_wave.py +2153 -0
  1702. package/eval/scaffold/python/validation/native_workflow.py +353 -0
  1703. package/eval/scaffold/python/validation/rubric.py +908 -0
  1704. package/eval/scaffold/python/validation/runner.py +237 -0
  1705. package/eval/scaffold/python/validation/scripts.py +58 -0
  1706. package/eval/scaffold/python/validation/tracing.py +313 -0
  1707. package/eval/scaffold/shell/agent-runtime-config.sh +88 -0
  1708. package/eval/scaffold/shell/common.sh +126 -0
  1709. package/eval/scaffold/shell/completion-point.sh +18 -0
  1710. package/eval/scaffold/shell/decision-point.sh +35 -0
  1711. package/eval/scaffold/shell/docker.sh +1129 -0
  1712. package/eval/scaffold/shell/run-agent-runtime.sh +49 -0
  1713. package/eval/scaffold/shell/run-claude-loop.sh +373 -0
  1714. package/eval/scaffold/shell/setup.sh +259 -0
  1715. package/eval/schemas/comet.eval/v1alpha1.schema.json +244 -0
  1716. package/eval/simulator-instruction.md +9 -0
  1717. package/package.json +59 -8
  1718. package/assets/skills/comet/reference/decision-point.md +0 -31
  1719. package/assets/skills/comet/reference/scripts.md +0 -69
  1720. package/dist/domains/dashboard/web/assets/demo-DQyfozbi.js +0 -1
  1721. package/dist/domains/dashboard/web/assets/index-BcO3_QlT.css +0 -1
  1722. package/dist/domains/dashboard/web/assets/index-CjU3VXVu.js +0 -22
  1723. /package/{assets/skills/comet → eval/local/skills/benchmarks/039-release/comet-classic-039}/reference/dirty-worktree.md +0 -0
  1724. /package/{assets/skills/comet → eval/local/skills/benchmarks/039-release/comet-classic-039}/reference/file-structure.md +0 -0
  1725. /package/{assets/skills-zh → eval/local/skills/benchmarks/040-beta}/comet/reference/auto-transition.md +0 -0
  1726. /package/{assets/skills-zh → eval/local/skills/benchmarks/040-beta}/comet/reference/context-recovery.md +0 -0
  1727. /package/{assets/skills-zh → eval/local/skills/benchmarks/040-beta}/comet/reference/debug-gate.md +0 -0
  1728. /package/{assets/skills-zh → eval/local/skills/benchmarks/040-beta}/comet/reference/decision-point.md +0 -0
  1729. /package/{assets/skills-zh → eval/local/skills/benchmarks/040-beta}/comet/reference/dirty-worktree.md +0 -0
  1730. /package/{assets/skills-zh → eval/local/skills/benchmarks/040-beta}/comet/reference/file-structure.md +0 -0
  1731. /package/{assets/skills-zh → eval/local/skills/benchmarks/040-beta}/comet/reference/intent-frame.md +0 -0
  1732. /package/{assets/skills-zh → eval/local/skills/benchmarks/040-beta}/comet/reference/scripts.md +0 -0
@@ -0,0 +1,2364 @@
1
+ """Shared pytest fixtures and experiment logging plugin.
2
+
3
+ Generates rich experiment logs in logs/experiments/ including:
4
+ - summary.md: Full markdown report with tables and details
5
+ - events/: Parsed events from each test run
6
+ - raw/: Raw selected Agent CLI output
7
+ - reports/: Per-run validation reports
8
+ - artifacts/: Files selected Agent generated and their execution output
9
+ - metadata.json: Experiment metadata
10
+
11
+ Supports pytest-xdist parallel execution via worker coordination.
12
+ """
13
+
14
+ import json
15
+ import hashlib
16
+ import os
17
+ import re
18
+ import shutil
19
+ import subprocess
20
+ import tempfile
21
+ import time
22
+ from contextlib import contextmanager
23
+ from datetime import datetime
24
+ from pathlib import Path
25
+ from types import SimpleNamespace
26
+ from typing import Any
27
+
28
+ import pytest
29
+
30
+ from scaffold import run_agent_in_docker, run_python_in_docker, run_shell
31
+ from scaffold.python import (
32
+ ExperimentLogger,
33
+ TreatmentResult,
34
+ build_eval_artifact_references,
35
+ classify_failures,
36
+ get_profile,
37
+ load_report_output_config,
38
+ save_events,
39
+ save_raw,
40
+ save_report,
41
+ strip_ansi,
42
+ )
43
+ from scaffold.python.agents import get_agent_adapter, resolve_agent, validate_agent_capabilities
44
+ from scaffold.python.auto_tasks import AutoTaskError, ensure_generated_manifest, find_project_root
45
+ from scaffold.python.eval_context import (
46
+ EvalContextError,
47
+ context_from_environment,
48
+ resolve_eval_context,
49
+ resolve_managed_path,
50
+ )
51
+ from scaffold.python.execution import (
52
+ ResolvedExecution,
53
+ ResolvedJudge,
54
+ build_agent_environment,
55
+ missing_credentials,
56
+ preflight_credentials,
57
+ redact_sensitive,
58
+ resolve_execution,
59
+ resolve_judge,
60
+ validate_base_url,
61
+ )
62
+ from scaffold.python.generated_task_cache import (
63
+ selected_agent_model as _legacy_selected_agent_model,
64
+ )
65
+ from scaffold.python.langfuse_adapter import (
66
+ install_codex_plugin_workspace,
67
+ install_transcript_hook,
68
+ provision_trajectory_plugin,
69
+ )
70
+ from scaffold.python.sample_quality import infer_sample_quality
71
+ from scaffold.python.skill_parser import SCRIPT_EXTENSIONS
72
+ from scaffold.python.utils import (
73
+ check_docker_available,
74
+ run_agent_loop_in_docker,
75
+ run_claude_loop_in_docker,
76
+ )
77
+ from scaffold.python.aligned_comparison import (
78
+ EXPECTED_CASE_MATRIX_FILENAME,
79
+ build_execution_identity,
80
+ expected_case_matrix_payload,
81
+ parse_expected_case_matrix,
82
+ )
83
+
84
+
85
+ def _extract_loop_turns(stderr: str | None) -> int | None:
86
+ match = re.search(r"\[loop\] finished after (\d+) turns", stderr or "")
87
+ return int(match.group(1)) if match else None
88
+
89
+
90
+ def _extract_loop_interaction(stderr: str | None) -> dict[str, Any]:
91
+ source = stderr or ""
92
+ role_sessions = {"subject": [], "simulator": [], "judge": []}
93
+ for role, session_id in re.findall(
94
+ r"\[loop\] role-session (subject|simulator|judge) ([^\s]+)", source
95
+ ):
96
+ if session_id not in role_sessions[role]:
97
+ role_sessions[role].append(session_id)
98
+ return {
99
+ "actual_turns": _extract_loop_turns(source),
100
+ "decision_points": source.count("[loop] decision point detected"),
101
+ "deterministic_replies": source.count("[loop] deterministic decision reply applied"),
102
+ "completion_signals": source.count("[loop] workflow completion detected"),
103
+ "fresh_resume_boundaries": source.count("[loop] fresh resume boundary detected"),
104
+ "role_sessions": role_sessions,
105
+ }
106
+
107
+
108
+ def _bounded_command_evidence(command: str, matches: list[re.Match[str]]) -> str:
109
+ chunks: list[str] = []
110
+ remaining = 4000
111
+ last_end = -1
112
+ for match in matches:
113
+ start = max(0, match.start() - 300)
114
+ end = min(len(command), match.end() + 300)
115
+ if start <= last_end and chunks:
116
+ overlap = max(0, last_end - start)
117
+ addition = command[start + overlap : end]
118
+ addition = addition[:remaining]
119
+ chunks[-1] += addition
120
+ remaining -= len(addition)
121
+ last_end = max(last_end, end)
122
+ else:
123
+ chunk = command[start:end][:remaining]
124
+ if chunks:
125
+ separator = "\n...[truncated]...\n"
126
+ if len(separator) > remaining:
127
+ break
128
+ chunks.append(separator)
129
+ remaining -= len(separator)
130
+ chunks.append(chunk)
131
+ remaining -= len(chunk)
132
+ last_end = end
133
+ if remaining <= 0:
134
+ break
135
+ excerpt = "".join(chunks)
136
+ return re.sub(
137
+ r"(?i)\b(api[_-]?key|auth[_-]?token|password|secret)\s*=\s*"
138
+ r"(?:\"[^\"]*\"|'[^']*'|[^\s;]+)",
139
+ r"\1=[REDACTED]",
140
+ excerpt,
141
+ )
142
+
143
+
144
+ def _tool_result_succeeded(item: dict[str, Any]) -> bool:
145
+ is_error = item.get("is_error")
146
+ if is_error is True or str(is_error).strip().lower() == "true" or item.get("error"):
147
+ return False
148
+ status = str(item.get("status") or "").strip().lower()
149
+ if status in {"error", "failed", "failure", "cancelled", "canceled"}:
150
+ return False
151
+ content = item.get("content", "")
152
+ if isinstance(content, list):
153
+ content = " ".join(
154
+ str(block.get("text") or "") if isinstance(block, dict) else str(block)
155
+ for block in content
156
+ )
157
+ return not bool(
158
+ re.search(
159
+ r"(?i)^\s*(?:error|failed|failure|tool[_ -]?error)\b|"
160
+ r"\b(?:exit|exited with)(?:\s+code)?\s*[1-9]\d*\b",
161
+ str(content),
162
+ )
163
+ )
164
+
165
+
166
+ def _extract_subject_turn_evidence(stdout: str | None) -> list[dict[str, Any]]:
167
+ """Group safe assistant result text and bounded tool evidence by subject turn."""
168
+ turns: list[dict[str, Any]] = []
169
+ tool_calls: list[dict[str, Any]] = []
170
+ tool_calls_by_id: dict[str, dict[str, Any]] = {}
171
+ last_assistant_text = ""
172
+ for line in (stdout or "").splitlines():
173
+ try:
174
+ event = json.loads(line)
175
+ except (json.JSONDecodeError, TypeError):
176
+ continue
177
+ if not isinstance(event, dict):
178
+ continue
179
+ if event.get("type") == "assistant":
180
+ content = (event.get("message") or {}).get("content") or []
181
+ assistant_text = " ".join(
182
+ str(block.get("text") or "")
183
+ for block in content
184
+ if isinstance(block, dict) and block.get("type") == "text"
185
+ ).strip()
186
+ if assistant_text:
187
+ last_assistant_text = assistant_text
188
+ for block in content:
189
+ if isinstance(block, dict) and block.get("type") == "tool_use":
190
+ name = block.get("name")
191
+ if isinstance(name, str) and name:
192
+ evidence: dict[str, Any] = {"name": name, "success": False}
193
+ tool_input = block.get("input")
194
+ if isinstance(tool_input, dict):
195
+ for key in ("file_path", "path", "notebook_path"):
196
+ value = tool_input.get(key)
197
+ if isinstance(value, str) and value:
198
+ evidence["path"] = value[:500]
199
+ break
200
+ command = tool_input.get("command")
201
+ if isinstance(command, str) and command:
202
+ target_matches = list(
203
+ re.finditer(
204
+ r"(?i)(?:[a-z]:)?[./\\\w-]*\.py\b|"
205
+ r"\bbrief\.md\b|"
206
+ r"\bspec\.md\b",
207
+ command,
208
+ )
209
+ )
210
+ if target_matches:
211
+ evidence["command"] = _bounded_command_evidence(
212
+ command, target_matches
213
+ )
214
+ tool_calls.append(evidence)
215
+ tool_id = block.get("id")
216
+ if isinstance(tool_id, str) and tool_id:
217
+ tool_calls_by_id[tool_id] = evidence
218
+ continue
219
+ if event.get("type") == "user":
220
+ content = (event.get("message") or {}).get("content") or []
221
+ for block in content:
222
+ if not isinstance(block, dict) or block.get("type") != "tool_result":
223
+ continue
224
+ tool_use_id = block.get("tool_use_id")
225
+ if isinstance(tool_use_id, str) and tool_use_id in tool_calls_by_id:
226
+ tool_calls_by_id[tool_use_id]["success"] = _tool_result_succeeded(block)
227
+ continue
228
+ if event.get("type") != "result":
229
+ continue
230
+ result = event.get("result")
231
+ turns.append(
232
+ {
233
+ "turn": len(turns) + 1,
234
+ "result": result if isinstance(result, str) and result else last_assistant_text,
235
+ "tool_calls": tool_calls,
236
+ }
237
+ )
238
+ tool_calls = []
239
+ tool_calls_by_id = {}
240
+ last_assistant_text = ""
241
+ return turns
242
+
243
+
244
+ # =============================================================================
245
+ # CONSTANTS
246
+ # =============================================================================
247
+
248
+ PROJECT_ROOT = Path(__file__).parent.parent
249
+ EVAL_ROOT = PROJECT_ROOT.parent
250
+ REPOSITORY_ROOT = EVAL_ROOT.parent
251
+
252
+ # Shared files for xdist worker coordination
253
+ _RUNTIME_CONTEXT = context_from_environment()
254
+ _CONTEXT_LOCK_ROOT = (
255
+ resolve_managed_path(_RUNTIME_CONTEXT, "locks") if _RUNTIME_CONTEXT else PROJECT_ROOT
256
+ )
257
+ XDIST_EXPERIMENT_FILE = _CONTEXT_LOCK_ROOT / ".pytest_experiment_id"
258
+ DOCKER_BUILD_LOCK = _CONTEXT_LOCK_ROOT / ".pytest_docker_build.lock"
259
+ EXPERIMENT_ID_ENV = "COMET_EVAL_EXPERIMENT_ID"
260
+ EXPERIMENT_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
261
+
262
+ # Global plugin instance (set during pytest_configure)
263
+ _plugin: "ExperimentPlugin | None" = None
264
+
265
+ # Cache discovered scripts (computed once on first call)
266
+ _KNOWN_SCRIPTS: list[str] | None = None
267
+
268
+ COMET_WORKFLOW_CLAUDE_MD_PATH = (
269
+ PROJECT_ROOT
270
+ / "skills"
271
+ / "benchmarks"
272
+ / "dependency"
273
+ / "claude-md"
274
+ / "comet-workflow"
275
+ / "CLAUDE.md"
276
+ )
277
+
278
+ CURRENT_COMET_CLI_MARKER = ".include-current-comet-cli"
279
+ CURRENT_COMET_BUILD_SCHEMA = "comet.eval.current-comet-build.v1"
280
+ TRUSTED_NATIVE_RUNTIME_MARKER = ".include-trusted-native-runtime"
281
+ TRUSTED_NATIVE_RUNTIME_SCHEMA = "comet.eval.trusted-native-runtime.v1"
282
+ MODEL_EXECUTION_ENV_KEYS = (
283
+ "ANTHROPIC_BASE_URL",
284
+ "ANTHROPIC_MODEL",
285
+ "ANTHROPIC_DEFAULT_HAIKU_MODEL",
286
+ "ANTHROPIC_DEFAULT_SONNET_MODEL",
287
+ "ANTHROPIC_DEFAULT_OPUS_MODEL",
288
+ "ANTHROPIC_DEFAULT_OPUS_MODEL_NAME",
289
+ "ANTHROPIC_DEFAULT_SONNET_MODEL_NAME",
290
+ "CLAUDE_CODE_SUBAGENT_MODEL",
291
+ "CODEBUDDY_BASE_URL",
292
+ "CODEBUDDY_MODEL",
293
+ "CODEBUDDY_SMALL_FAST_MODEL",
294
+ "CODEBUDDY_BIG_SLOW_MODEL",
295
+ "CODEBUDDY_CODE_SUBAGENT_MODEL",
296
+ "BENCH_CODEBUDDY_MODEL",
297
+ "OPENAI_BASE_URL",
298
+ "OPENAI_MODEL",
299
+ "CODEX_BASE_URL",
300
+ "CODEX_MODEL",
301
+ "QODER_BASE_URL",
302
+ "QODER_MODEL",
303
+ )
304
+
305
+
306
+ def _getoption(config, name: str, default=None):
307
+ try:
308
+ value = config.getoption(name)
309
+ except (AttributeError, KeyError, ValueError):
310
+ return default
311
+ return default if value is None else value
312
+
313
+
314
+ def _manifest_for_config(config):
315
+ manifest_path = _getoption(config, "--eval-manifest")
316
+ if not manifest_path:
317
+ return None
318
+ from scaffold.python.manifests import load_eval_manifest
319
+
320
+ return load_eval_manifest(manifest_path)
321
+
322
+
323
+ def _resolve_eval_execution(config) -> ResolvedExecution:
324
+ manifest = _manifest_for_config(config)
325
+ manifest_execution = manifest.execution if manifest is not None else None
326
+ return resolve_execution(
327
+ cli_agent=_getoption(config, "--agent"),
328
+ cli_model=_getoption(config, "--model"),
329
+ cli_base_url=_getoption(config, "--base-url"),
330
+ manifest=manifest_execution,
331
+ )
332
+
333
+
334
+ def _resolve_eval_judge(config) -> ResolvedJudge | None:
335
+ manifest = _manifest_for_config(config)
336
+ manifest_judge = manifest.judge if manifest is not None else None
337
+ return resolve_judge(
338
+ main=_resolve_eval_execution(config),
339
+ cli_agent=_getoption(config, "--judge-agent"),
340
+ cli_model=_getoption(config, "--judge-model"),
341
+ cli_base_url=_getoption(config, "--judge-base-url"),
342
+ manifest=manifest_judge,
343
+ )
344
+
345
+
346
+ def _selected_agent_model(agent: str) -> str | None:
347
+ if _plugin is not None:
348
+ return _resolve_eval_execution(_plugin.config).model
349
+ return _legacy_selected_agent_model(agent)
350
+
351
+
352
+ def _resolve_eval_agent(config):
353
+ execution = _resolve_eval_execution(config)
354
+ return resolve_agent(execution.agent, None)
355
+
356
+
357
+ def _require_agent_credentials(agent: str, source_env: dict[str, str] | None = None) -> None:
358
+ """Fail setup before workloads when the isolated Agent cannot authenticate."""
359
+ environment = source_env if source_env is not None else os.environ
360
+ missing = missing_credentials(agent, source_env=environment)
361
+ if not missing:
362
+ return
363
+ adapter = get_agent_adapter(agent)
364
+ separator = ", " if adapter.custom else " or "
365
+ raise pytest.UsageError(
366
+ f"Credentials for {agent} not set: {separator.join(missing)}"
367
+ )
368
+
369
+
370
+ def _normal_run_preflight(config) -> None:
371
+ """Validate normal-run dependencies before collection can generate or run tasks."""
372
+
373
+ if _is_unit_tests_only(config) or getattr(config.option, "collectonly", False):
374
+ return
375
+ from scaffold.python.utils import load_eval_environment
376
+
377
+ load_eval_environment()
378
+ main = _resolve_eval_execution(config)
379
+ judge = _resolve_eval_judge(config)
380
+ suite = next(
381
+ (name for name in ("local", "langsmith", "langfuse") if f"{name}/tests/" in " ".join(config.args)),
382
+ "local",
383
+ )
384
+ required_harness = (
385
+ EVAL_ROOT / "pyproject.toml",
386
+ EVAL_ROOT / "schemas" / "comet.eval" / "v1alpha1.schema.json",
387
+ EVAL_ROOT / suite / "tests" / "tasks" / "test_tasks.py",
388
+ )
389
+ missing_harness = [str(path) for path in required_harness if not path.is_file()]
390
+ if missing_harness:
391
+ raise pytest.UsageError(f"Packaged eval harness is incomplete: {', '.join(missing_harness)}")
392
+ manifest = _manifest_for_config(config)
393
+ if manifest is not None and manifest.reporting is not None:
394
+ selected_report_suite = manifest.reporting.suite
395
+ if selected_report_suite and selected_report_suite != suite:
396
+ raise pytest.UsageError(
397
+ f"reporting.suite is {selected_report_suite!r}, but the selected suite is {suite!r}"
398
+ )
399
+ report_config = manifest.reporting.config
400
+ if report_config and not _getoption(config, "--report-config"):
401
+ report_path = Path(report_config)
402
+ if not report_path.is_absolute():
403
+ report_path = manifest.path.parent / report_path
404
+ config.option.report_config = str(report_path.resolve())
405
+ try:
406
+ load_report_output_config(_getoption(config, "--report-config"))
407
+ except (OSError, ValueError) as exc:
408
+ raise pytest.UsageError(f"Eval report configuration is invalid: {exc}") from exc
409
+ try:
410
+ validate_agent_capabilities(get_agent_adapter(main.agent))
411
+ if judge is not None:
412
+ validate_agent_capabilities(get_agent_adapter(judge.agent))
413
+ except ValueError as exc:
414
+ raise pytest.UsageError(str(exc)) from exc
415
+ if main.base_url:
416
+ validate_base_url(main.base_url, field="execution.baseUrl")
417
+ if judge is not None and judge.base_url:
418
+ validate_base_url(judge.base_url, field="judge.baseUrl")
419
+ credential_errors = preflight_credentials(main, judge)
420
+ if credential_errors:
421
+ raise pytest.UsageError("; ".join(credential_errors))
422
+ if shutil.which("uv") is None:
423
+ raise pytest.UsageError("uv is not installed or not in PATH")
424
+ if not check_docker_available():
425
+ raise pytest.UsageError("Docker CLI or daemon is unavailable")
426
+ context = getattr(config, "_comet_eval_context", None)
427
+ if context is None:
428
+ raise pytest.UsageError("Eval context is missing before normal-run preflight")
429
+ artifact_root = Path(context.artifact_root)
430
+ artifact_root.mkdir(parents=True, exist_ok=True)
431
+ try:
432
+ with tempfile.NamedTemporaryFile(dir=artifact_root, prefix=".preflight-", delete=True):
433
+ pass
434
+ except OSError as exc:
435
+ raise pytest.UsageError(f"Eval artifact root is not writable: {artifact_root}") from exc
436
+ config._comet_execution_env = build_agent_environment(main)
437
+ config._comet_execution = main
438
+ config._comet_judge = judge
439
+ os.environ["COMET_EVAL_MAIN_CREDENTIALS"] = ",".join(
440
+ get_agent_adapter(main.agent).required_credentials
441
+ )
442
+ if judge is not None:
443
+ os.environ["BENCH_LLM_JUDGE"] = "1"
444
+ os.environ["BENCH_JUDGE_MODEL"] = judge.model
445
+ os.environ["BENCH_JUDGE_AGENT"] = judge.agent
446
+ if judge.base_url:
447
+ os.environ["BENCH_JUDGE_BASE_URL"] = judge.base_url
448
+ else:
449
+ os.environ.pop("BENCH_JUDGE_BASE_URL", None)
450
+
451
+
452
+ def _agent_project_root(agent: str) -> str:
453
+ return {
454
+ "claude-code": ".claude",
455
+ "codex": ".agents",
456
+ "qoder": ".qoder",
457
+ "codebuddy": ".codebuddy",
458
+ }.get(agent, ".comet-agent")
459
+
460
+
461
+ def _ensure_langfuse_trajectory_support(test_dir: Path, agent: str) -> None:
462
+ """Install only isolated, project-local trajectory support for Langfuse runs."""
463
+ if os.environ.get("TRACE_TO_LANGFUSE", "").lower() != "true":
464
+ return
465
+ if agent in {"qoder", "codebuddy"}:
466
+ try:
467
+ install_transcript_hook(test_dir, agent)
468
+ except Exception as exc:
469
+ if os.environ.get("LANGFUSE_TRANSCRIPT_SETUP_WARNING_EMITTED") != "true":
470
+ print(
471
+ f"[langfuse] optional {agent} transcript setup skipped: {exc}",
472
+ file=os.sys.stderr,
473
+ )
474
+ os.environ["LANGFUSE_TRANSCRIPT_SETUP_WARNING_EMITTED"] = "true"
475
+ return
476
+ if agent != "codex":
477
+ return
478
+ cache_root = os.environ.get("LANGFUSE_TRAJECTORY_CACHE_DIR")
479
+ if not cache_root:
480
+ return
481
+ try:
482
+ provision = provision_trajectory_plugin(Path(cache_root), agent)
483
+ install_codex_plugin_workspace(test_dir, provision)
484
+ except Exception as exc:
485
+ if os.environ.get("LANGFUSE_TRAJECTORY_SETUP_WARNING_EMITTED") != "true":
486
+ print(f"[langfuse] optional Codex trajectory setup skipped: {exc}", file=os.sys.stderr)
487
+ os.environ["LANGFUSE_TRAJECTORY_SETUP_WARNING_EMITTED"] = "true"
488
+
489
+
490
+ def _ensure_auto_generated_manifest(config, task_filter: str | None = None) -> None:
491
+ """Freeze generated descriptors once while retaining the original manifest authority."""
492
+ if getattr(config, "_comet_frozen_task_set", None) is not None:
493
+ return
494
+ if config.getoption("--task") or task_filter or config.getoption("--quick"):
495
+ return
496
+
497
+ manifest_path = config.getoption("--eval-manifest")
498
+ skill_path = config.getoption("--skill-path")
499
+ manifest = None
500
+ if manifest_path:
501
+ from scaffold.python.manifests import load_eval_manifest
502
+
503
+ manifest = load_eval_manifest(manifest_path)
504
+ if manifest.tasks or manifest.recommended_tasks:
505
+ return
506
+ skill_path = str(manifest.skill_path)
507
+ if not skill_path:
508
+ return
509
+
510
+ if getattr(getattr(config, "option", None), "collectonly", False):
511
+ from scaffold.python.manifests import SkillEvalManifest
512
+ from scaffold.python.task_resolution import ResolvedTaskSet
513
+
514
+ if manifest is None:
515
+ resolved_skill_root = Path(skill_path).expanduser().resolve()
516
+ if resolved_skill_root.is_file() and resolved_skill_root.name == "SKILL.md":
517
+ resolved_skill_root = resolved_skill_root.parent
518
+ manifest = SkillEvalManifest(
519
+ path=resolved_skill_root / "comet" / "eval.yaml",
520
+ name=resolved_skill_root.name,
521
+ description="",
522
+ skill_name=getattr(config.option, "skill_name", None) or resolved_skill_root.name,
523
+ skill_path=resolved_skill_root,
524
+ profile=config.getoption("--profile"),
525
+ )
526
+ config._comet_resolution_manifest = manifest
527
+ config._comet_resolved_tasks = {}
528
+ config._comet_frozen_task_set = ResolvedTaskSet("pending-generation", (), None)
529
+ return
530
+
531
+ # Task generation happens during collection, before session fixtures run.
532
+ # Load the documented local environment now so credentials and the model
533
+ # participating in the cache key match the later execution environment.
534
+ from scaffold.python.utils import load_eval_environment
535
+
536
+ load_eval_environment()
537
+ execution = _resolve_eval_execution(config)
538
+ profile = config.getoption("--profile") or (manifest.profile if manifest else None) or "generic"
539
+ interaction = vars(manifest.interaction) if manifest else {"mode": "none", "max_turns": 12}
540
+ context = getattr(config, "_comet_eval_context", None)
541
+ project_root = (
542
+ context.artifact_owner_root
543
+ if context is not None
544
+ else config.getoption("--project-root") or find_project_root(skill_path)
545
+ )
546
+ try:
547
+ generated = ensure_generated_manifest(
548
+ skill_path,
549
+ project_root,
550
+ agent=execution.agent,
551
+ model=execution.model,
552
+ base_url=execution.base_url,
553
+ environment=build_agent_environment(execution),
554
+ profile=profile,
555
+ interaction=interaction,
556
+ collect_only=bool(getattr(getattr(config, "option", None), "collectonly", False)),
557
+ )
558
+ except AutoTaskError as exc:
559
+ raise pytest.UsageError(str(exc)) from exc
560
+ from scaffold.python.manifest_tasks import load_manifest_tasks
561
+ from scaffold.python.manifests import SkillEvalManifest, load_eval_manifest
562
+ from scaffold.python.task_resolution import ResolvedTask, ResolvedTaskSet
563
+
564
+ generated_manifest = load_eval_manifest(generated.manifest_path)
565
+ generated_tasks = tuple(
566
+ ResolvedTask(task.name, "generated", task)
567
+ for task in load_manifest_tasks(generated_manifest)
568
+ )
569
+ if manifest is None:
570
+ resolved_skill_root = Path(skill_path).expanduser().resolve()
571
+ if resolved_skill_root.is_file() and resolved_skill_root.name == "SKILL.md":
572
+ resolved_skill_root = resolved_skill_root.parent
573
+ manifest = SkillEvalManifest(
574
+ path=resolved_skill_root / "comet" / "eval.yaml",
575
+ name=resolved_skill_root.name,
576
+ description="",
577
+ skill_name=getattr(config.option, "skill_name", None) or resolved_skill_root.name,
578
+ skill_path=resolved_skill_root,
579
+ profile=config.getoption("--profile"),
580
+ )
581
+ frozen = ResolvedTaskSet(
582
+ "generated-cache",
583
+ generated_tasks,
584
+ None,
585
+ )
586
+ config._comet_resolution_manifest = manifest
587
+ config._comet_resolved_tasks = {item.name: item.task for item in frozen.tasks}
588
+ config._comet_frozen_task_set = frozen
589
+ config._comet_generated_manifest = generated
590
+
591
+
592
+ def _model_execution_config() -> dict[str, str | None]:
593
+ """Return only model-routing values; credentials are deliberately excluded."""
594
+ return {key: os.environ.get(key) for key in MODEL_EXECUTION_ENV_KEYS}
595
+
596
+
597
+ def _capture_execution_identity(test_dir: Path, *, model: str | None, interaction):
598
+ """Capture an immutable image/tool identity and a safe report payload."""
599
+ result = run_shell(
600
+ "docker.sh",
601
+ "execution-identity",
602
+ str(test_dir),
603
+ "--agent",
604
+ _resolve_eval_agent(_plugin.config).agent if _plugin else "claude-code",
605
+ timeout=300,
606
+ check=False,
607
+ )
608
+ if result.returncode != 0:
609
+ raise RuntimeError("Could not verify eval Docker/agent execution identity")
610
+ try:
611
+ raw = json.loads(result.stdout)
612
+ except (TypeError, json.JSONDecodeError) as error:
613
+ raise RuntimeError("Eval execution identity output is not valid JSON") from error
614
+ report_identity = build_execution_identity(
615
+ raw,
616
+ model=model,
617
+ model_config=_model_execution_config(),
618
+ interaction=interaction,
619
+ )
620
+ return SimpleNamespace(
621
+ runtime_image_id=raw["runtime_image_id"],
622
+ report_identity=report_identity,
623
+ )
624
+
625
+
626
+ def _regular_tree_files(root: Path) -> list[Path]:
627
+ files: list[Path] = []
628
+ for path in sorted(root.rglob("*")):
629
+ if path.is_symlink():
630
+ raise ValueError(f"Current Comet snapshot source contains a symbolic link: {path}")
631
+ if path.is_file():
632
+ files.append(path)
633
+ return files
634
+
635
+
636
+ def _tree_digest(root: Path, files: list[Path] | None = None) -> tuple[str, int]:
637
+ digest = hashlib.sha256()
638
+ selected = files if files is not None else _regular_tree_files(root)
639
+ for path in selected:
640
+ relative = path.relative_to(root).as_posix()
641
+ payload = path.read_bytes()
642
+ digest.update(relative.encode("utf-8"))
643
+ digest.update(b"\0")
644
+ digest.update(str(len(payload)).encode("ascii"))
645
+ digest.update(b"\0")
646
+ digest.update(hashlib.sha256(payload).hexdigest().encode("ascii"))
647
+ digest.update(b"\n")
648
+ return digest.hexdigest(), len(selected)
649
+
650
+
651
+ def _build_current_comet_dist(checkout: Path, output: Path) -> str:
652
+ """Compile the checkout's TypeScript into an isolated, timestamp-free dist tree."""
653
+ node = shutil.which("node")
654
+ tsc = checkout / "node_modules/typescript/bin/tsc"
655
+ compiler_package = checkout / "node_modules/typescript/package.json"
656
+ if not node or not tsc.is_file() or not compiler_package.is_file():
657
+ raise FileNotFoundError("Current Comet source build requires local Node.js and TypeScript")
658
+ compiler = json.loads(compiler_package.read_text(encoding="utf-8"))
659
+ version = compiler.get("version")
660
+ if not isinstance(version, str) or not version:
661
+ raise ValueError("Current Comet TypeScript compiler version is invalid")
662
+ result = subprocess.run(
663
+ [
664
+ node,
665
+ str(tsc),
666
+ "--pretty",
667
+ "false",
668
+ "--outDir",
669
+ str(output),
670
+ "--declaration",
671
+ "false",
672
+ "--declarationMap",
673
+ "false",
674
+ "--sourceMap",
675
+ "false",
676
+ "--incremental",
677
+ "false",
678
+ ],
679
+ cwd=checkout,
680
+ capture_output=True,
681
+ text=True,
682
+ timeout=180,
683
+ check=False,
684
+ )
685
+ if result.returncode != 0:
686
+ detail = (result.stdout + "\n" + result.stderr).strip()
687
+ raise RuntimeError(f"Current Comet source build failed: {detail[-4000:]}")
688
+ required = [output / "app/cli/index.js", output / "domains/dashboard/native-adapter.js"]
689
+ missing = [str(path) for path in required if not path.is_file()]
690
+ if missing:
691
+ raise FileNotFoundError(f"Current Comet source build omitted required output: {missing}")
692
+ return version
693
+
694
+
695
+ def _copy_current_comet_cli_snapshot(environment_dir: Path, test_dir: Path) -> None:
696
+ """Expose the current checkout's built CLI to task containers on request.
697
+
698
+ The task workspace is the only host directory mounted into Docker. A task
699
+ carrying ``CURRENT_COMET_CLI_MARKER`` therefore needs an explicit snapshot
700
+ of this checkout's bin/dist package rather than a published npm version.
701
+ """
702
+ if not (environment_dir / CURRENT_COMET_CLI_MARKER).is_file():
703
+ return
704
+
705
+ target = test_dir / "_eval_current_comet"
706
+ if target.exists():
707
+ shutil.rmtree(target)
708
+ target.mkdir(parents=True)
709
+ package_file = REPOSITORY_ROOT / "package.json"
710
+ bin_dir = REPOSITORY_ROOT / "bin"
711
+ assets_dir = REPOSITORY_ROOT / "assets"
712
+ assets_manifest = assets_dir / "manifest.json"
713
+ source_roots = [
714
+ REPOSITORY_ROOT / "app",
715
+ REPOSITORY_ROOT / "domains",
716
+ REPOSITORY_ROOT / "platform",
717
+ ]
718
+ source_files = [
719
+ path
720
+ for root in source_roots
721
+ for path in _regular_tree_files(root)
722
+ if path.suffix in {".ts", ".tsx", ".json"}
723
+ ]
724
+ source_files.extend(
725
+ path
726
+ for path in (
727
+ REPOSITORY_ROOT / "tsconfig.json",
728
+ package_file,
729
+ REPOSITORY_ROOT / "bin/comet.js",
730
+ )
731
+ if path.is_file()
732
+ )
733
+ asset_files = _regular_tree_files(assets_dir) if assets_dir.is_dir() else []
734
+ source_files.extend(asset_files)
735
+ if (
736
+ not package_file.is_file()
737
+ or not bin_dir.is_dir()
738
+ or not assets_manifest.is_file()
739
+ or not asset_files
740
+ or not source_files
741
+ ):
742
+ raise FileNotFoundError("Current Comet source snapshot is incomplete")
743
+ source_hash, source_count = _tree_digest(REPOSITORY_ROOT, sorted(set(source_files)))
744
+ with tempfile.TemporaryDirectory(prefix="comet-eval-source-build-") as temporary:
745
+ built_dist = Path(temporary) / "dist"
746
+ compiler_version = _build_current_comet_dist(REPOSITORY_ROOT, built_dist)
747
+ shutil.copytree(built_dist, target / "dist")
748
+ shutil.copytree(bin_dir, target / "bin")
749
+ shutil.copytree(assets_dir, target / "assets")
750
+ shutil.copy2(package_file, target / "package.json")
751
+ snapshot_files = [
752
+ path
753
+ for relative in ("assets", "bin", "dist")
754
+ for path in _regular_tree_files(target / relative)
755
+ ] + [target / "package.json"]
756
+ snapshot_hash, snapshot_count = _tree_digest(target, sorted(snapshot_files))
757
+ identity = {
758
+ "schema": CURRENT_COMET_BUILD_SCHEMA,
759
+ "sourceHash": source_hash,
760
+ "sourceFileCount": source_count,
761
+ "snapshotHash": snapshot_hash,
762
+ "snapshotFileCount": snapshot_count,
763
+ "packageHash": hashlib.sha256((target / "package.json").read_bytes()).hexdigest(),
764
+ "assetsHash": _tree_digest(target, _regular_tree_files(target / "assets"))[0],
765
+ "assetsFileCount": len(_regular_tree_files(target / "assets")),
766
+ "manifestHash": hashlib.sha256((target / "assets/manifest.json").read_bytes()).hexdigest(),
767
+ "entryHash": hashlib.sha256((target / "dist/app/cli/index.js").read_bytes()).hexdigest(),
768
+ "nativeAdapterHash": hashlib.sha256(
769
+ (target / "dist/domains/dashboard/native-adapter.js").read_bytes()
770
+ ).hexdigest(),
771
+ "compilerVersion": compiler_version,
772
+ }
773
+ (target / "build-identity.json").write_text(
774
+ json.dumps(identity, indent=2) + "\n", encoding="utf-8"
775
+ )
776
+
777
+
778
+ def _copy_trusted_native_runtime_snapshot(
779
+ environment_dir: Path,
780
+ test_dir: Path,
781
+ skills: dict[str, Any] | None,
782
+ ) -> None:
783
+ """Snapshot the controller-selected Native runtime before the Agent starts."""
784
+ if not (environment_dir / TRUSTED_NATIVE_RUNTIME_MARKER).is_file():
785
+ return
786
+ config = (skills or {}).get("comet-native")
787
+ if not isinstance(config, dict):
788
+ raise ValueError("Trusted Native oracle requires the comet-native Skill source")
789
+ source_dir = config.get("source_dir")
790
+ scripts_dir = config.get("scripts_dir")
791
+ candidates = []
792
+ if source_dir:
793
+ candidates.append(Path(source_dir) / "scripts/comet-native-runtime.mjs")
794
+ if scripts_dir:
795
+ candidates.append(Path(scripts_dir) / "comet-native-runtime.mjs")
796
+ source = next(
797
+ (
798
+ candidate
799
+ for candidate in candidates
800
+ if candidate.is_file() and not candidate.is_symlink()
801
+ ),
802
+ None,
803
+ )
804
+ if source is None:
805
+ raise FileNotFoundError("Controller-selected Comet Native runtime is unavailable")
806
+
807
+ target_root = test_dir / "_eval_trusted_oracles"
808
+ if target_root.exists():
809
+ shutil.rmtree(target_root)
810
+ target_root.mkdir(parents=True)
811
+ target = target_root / "comet-native-runtime.mjs"
812
+ shutil.copyfile(source, target)
813
+ identity = {
814
+ "schema": TRUSTED_NATIVE_RUNTIME_SCHEMA,
815
+ "runtimeFile": target.name,
816
+ "runtimeHash": hashlib.sha256(target.read_bytes()).hexdigest(),
817
+ }
818
+ (target_root / "native-runtime-identity.json").write_text(
819
+ json.dumps(identity, indent=2) + "\n", encoding="utf-8"
820
+ )
821
+
822
+
823
+ # PYTEST HOOKS
824
+ # =============================================================================
825
+
826
+
827
+ def pytest_addoption(parser):
828
+ """Add CLI options for task and treatment selection."""
829
+ parser.addoption(
830
+ "--task",
831
+ action="store",
832
+ default=None,
833
+ help="Run specific task (e.g., --task=comet-full-workflow)",
834
+ )
835
+ parser.addoption(
836
+ "--treatment",
837
+ action="store",
838
+ default=None,
839
+ help="Run specific treatment (e.g., --treatment=COMET_FULL_040_BETA)",
840
+ )
841
+ parser.addoption(
842
+ "--count",
843
+ action="store",
844
+ type=int,
845
+ default=1,
846
+ help="Repeat each task/treatment combination N times for distribution stats (default: 1)",
847
+ )
848
+ parser.addoption(
849
+ "--skill-path",
850
+ action="store",
851
+ default=None,
852
+ help="Local Skill directory or SKILL.md to evaluate",
853
+ )
854
+ parser.addoption(
855
+ "--skill-name",
856
+ action="store",
857
+ default=None,
858
+ help="Skill name to inject for --skill-path",
859
+ )
860
+ parser.addoption(
861
+ "--profile",
862
+ action="store",
863
+ default=None,
864
+ help="Eval profile override",
865
+ )
866
+ parser.addoption(
867
+ "--eval-manifest",
868
+ action="store",
869
+ default=None,
870
+ help="Path to comet/eval.yaml",
871
+ )
872
+ parser.addoption(
873
+ "--agent",
874
+ action="store",
875
+ default=None,
876
+ help="Evaluation agent CLI (built-in or explicitly installed custom adapter)",
877
+ )
878
+ parser.addoption(
879
+ "--model",
880
+ action="store",
881
+ default=None,
882
+ help="Main evaluation model override",
883
+ )
884
+ parser.addoption(
885
+ "--base-url",
886
+ action="store",
887
+ default=None,
888
+ help="Main evaluation API base URL override",
889
+ )
890
+ parser.addoption(
891
+ "--judge-agent",
892
+ action="store",
893
+ default=None,
894
+ help="Independent Judge agent (built-in or explicitly installed custom adapter)",
895
+ )
896
+ parser.addoption(
897
+ "--judge-model",
898
+ action="store",
899
+ default=None,
900
+ help="Independent Judge model",
901
+ )
902
+ parser.addoption(
903
+ "--judge-base-url",
904
+ action="store",
905
+ default=None,
906
+ help="Independent Judge API base URL",
907
+ )
908
+ parser.addoption(
909
+ "--quick",
910
+ action="store_true",
911
+ default=False,
912
+ help="Use generic-skill-smoke and bypass automatic task generation",
913
+ )
914
+ parser.addoption(
915
+ "--project-root",
916
+ action="store",
917
+ default=None,
918
+ help="Project root used for generated eval cache",
919
+ )
920
+ parser.addoption(
921
+ "--interaction-mode",
922
+ action="store",
923
+ default=None,
924
+ help="Override interaction mode (e.g., none, auto_user)",
925
+ )
926
+ parser.addoption(
927
+ "--max-turns",
928
+ action="store",
929
+ default=None,
930
+ help="Override max interaction turns for auto_user loops",
931
+ )
932
+ parser.addoption(
933
+ "--simulator-prompt",
934
+ action="store",
935
+ default=None,
936
+ help="Override user simulator prompt for auto_user loops",
937
+ )
938
+ parser.addoption(
939
+ "--report-config",
940
+ action="store",
941
+ default=None,
942
+ help="JSON/YAML config for eval report outputs",
943
+ )
944
+
945
+
946
+ def pytest_configure(config):
947
+ """Register experiment plugin (decision deferred to sessionstart)."""
948
+ config.addinivalue_line(
949
+ "markers",
950
+ "eval_case(repetition): controller-owned task/treatment/repetition identity",
951
+ )
952
+ try:
953
+ context = context_from_environment()
954
+ if context is None and (config.getoption("--eval-manifest") or config.getoption("--skill-path")):
955
+ context = resolve_eval_context(
956
+ skill_path=config.getoption("--skill-path"),
957
+ manifest_path=config.getoption("--eval-manifest"),
958
+ project_root=config.getoption("--project-root"),
959
+ )
960
+ except EvalContextError as exc:
961
+ raise pytest.UsageError(str(exc)) from exc
962
+ if context is not None:
963
+ config._comet_eval_context = context
964
+ config.option.project_root = str(context.artifact_owner_root)
965
+ if context.manifest_path is not None:
966
+ config.option.eval_manifest = str(context.manifest_path)
967
+ else:
968
+ config.option.skill_path = str(context.skill_root)
969
+ config._comet_execution = _resolve_eval_execution(config)
970
+ config._comet_judge = _resolve_eval_judge(config)
971
+ _normal_run_preflight(config)
972
+ _resolve_eval_agent(config)
973
+ global _plugin
974
+ _plugin = ExperimentPlugin(config)
975
+ config.pluginmanager.register(_plugin, "experiment_plugin")
976
+
977
+
978
+ # =============================================================================
979
+ # EXPERIMENT PLUGIN
980
+ # =============================================================================
981
+
982
+
983
+ class ExperimentPlugin:
984
+ """Pytest plugin that generates rich experiment logs in logs/experiments/."""
985
+
986
+ def __init__(self, config):
987
+ self.config = config
988
+ self.logger: ExperimentLogger | None = None
989
+ self.start_time = None
990
+ self.run_counter: dict[str, int] = {}
991
+ self.is_xdist_worker = hasattr(config, "workerinput")
992
+ self.is_xdist_master = (
993
+ not hasattr(config, "workerinput")
994
+ and (getattr(config.option, "numprocesses", None) or 0) > 0
995
+ )
996
+ self.worker_id = (
997
+ config.workerinput.get("workerid", "master") if self.is_xdist_worker else "master"
998
+ )
999
+
1000
+ def pytest_sessionstart(self, session):
1001
+ """Create or join experiment logger at session start."""
1002
+ if _is_unit_tests_only(self.config):
1003
+ return
1004
+
1005
+ name = _get_experiment_name(session)
1006
+ use_coordination = self.is_xdist_worker or self.is_xdist_master
1007
+ experiment_id = _get_or_create_experiment_id(name, use_coordination)
1008
+
1009
+ report_outputs = load_report_output_config(self.config.getoption("--report-config"))
1010
+ self.logger = ExperimentLogger(
1011
+ experiment_name=name,
1012
+ experiment_id=experiment_id,
1013
+ report_outputs=report_outputs,
1014
+ )
1015
+ self.start_time = time.time()
1016
+
1017
+ print(f"\n{'=' * 60}")
1018
+ print(f"EXPERIMENT: {self.logger.experiment_id}")
1019
+ print(f"Logging to: {self.logger.base_dir}")
1020
+ print(f"{'=' * 60}\n")
1021
+
1022
+ def pytest_collection_finish(self, session):
1023
+ """Persist the full collected case matrix before any model run starts."""
1024
+ if not self.logger:
1025
+ return
1026
+ cases = _expected_cases_from_items(session.items)
1027
+ if not cases:
1028
+ return
1029
+ payload = expected_case_matrix_payload(cases)
1030
+ _persist_expected_case_matrix(self.logger.base_dir, payload)
1031
+ self.logger.metadata["expected_case_matrix"] = {
1032
+ "schema": payload["schema"],
1033
+ "matrix_hash": payload["matrix_hash"],
1034
+ "case_count": len(payload["cases"]),
1035
+ "path": EXPECTED_CASE_MATRIX_FILENAME,
1036
+ }
1037
+
1038
+ def pytest_sessionfinish(self, session, exitstatus):
1039
+ """Generate and save summary at session end."""
1040
+ if not self.logger:
1041
+ return
1042
+
1043
+ if self.is_xdist_worker:
1044
+ return
1045
+
1046
+ if self.is_xdist_master:
1047
+ time.sleep(1)
1048
+
1049
+ self._reload_expected_case_matrix_metadata()
1050
+ self._reload_results_from_reports()
1051
+
1052
+ if self.logger.results:
1053
+ self.logger.finalize()
1054
+ self._print_summary()
1055
+
1056
+ _cleanup_experiment_coordination()
1057
+
1058
+ def _reload_expected_case_matrix_metadata(self):
1059
+ """Attach the worker-written matrix to controller-finalized metadata."""
1060
+ path = self.logger.base_dir / EXPECTED_CASE_MATRIX_FILENAME
1061
+ if not path.is_file():
1062
+ return
1063
+ try:
1064
+ payload = json.loads(path.read_text(encoding="utf-8"))
1065
+ matrix = parse_expected_case_matrix(payload)
1066
+ except (OSError, json.JSONDecodeError, ValueError) as error:
1067
+ raise RuntimeError("Expected case matrix became invalid before finalize") from error
1068
+ self.logger.metadata["expected_case_matrix"] = {
1069
+ "schema": payload["schema"],
1070
+ "matrix_hash": matrix.matrix_hash,
1071
+ "case_count": len(matrix.cases),
1072
+ "path": EXPECTED_CASE_MATRIX_FILENAME,
1073
+ }
1074
+
1075
+ def get_rep_number(self, treatment_name: str) -> int:
1076
+ """Get the next repetition number for a treatment."""
1077
+ if treatment_name not in self.run_counter:
1078
+ self.run_counter[treatment_name] = 0
1079
+ self.run_counter[treatment_name] += 1
1080
+ return self.run_counter[treatment_name]
1081
+
1082
+ def _reload_results_from_reports(self):
1083
+ """Reload results from saved report files (aggregates all workers)."""
1084
+ reports_dir = self.logger.reports_dir
1085
+ if not reports_dir.exists():
1086
+ return
1087
+
1088
+ self.logger.results.clear()
1089
+
1090
+ for report_file in sorted(reports_dir.glob("*.json")):
1091
+ try:
1092
+ report = json.loads(report_file.read_text())
1093
+ treatment_name = report.get("name", "unknown")
1094
+ result = TreatmentResult(
1095
+ name=treatment_name,
1096
+ passed=report.get("passed", False),
1097
+ checks_passed=report.get("checks_passed", []),
1098
+ checks_failed=report.get("checks_failed", []),
1099
+ events_summary=report.get("events_summary", {}),
1100
+ run_id=report.get("run_id", ""),
1101
+ )
1102
+ if treatment_name not in self.logger.results:
1103
+ self.logger.results[treatment_name] = []
1104
+ self.logger.results[treatment_name].append(result)
1105
+ except Exception as e:
1106
+ import sys
1107
+
1108
+ print(f"Warning: failed to reload report {report_file.name}: {e}", file=sys.stderr)
1109
+
1110
+ def _print_summary(self):
1111
+ """Print summary to console."""
1112
+ print(f"\n{'=' * 120}")
1113
+ print(" RESULTS")
1114
+ print(f"{'=' * 120}\n")
1115
+
1116
+ print(
1117
+ f"{'Treatment':<25} {'Checks':<15} {'Turns':<8} {'Dur':<8} {'Tokens':<12} {'Cost':<10} {'Skills':<40}"
1118
+ )
1119
+ print("-" * 120)
1120
+
1121
+ for treatment, runs in self.logger.results.items():
1122
+ for r in runs:
1123
+ checks_passed = len(r.checks_passed)
1124
+ checks_total = checks_passed + len(r.checks_failed)
1125
+ check_pct = (checks_passed / checks_total * 100) if checks_total > 0 else 0
1126
+ checks_str = f"{checks_passed}/{checks_total} ({check_pct:.0f}%)"
1127
+ turns = str(r.turns) if r.turns else "?"
1128
+ dur = f"{r.duration:.0f}s" if r.duration else "?"
1129
+ tokens = f"{r.total_tokens:,}" if r.total_tokens is not None else "?"
1130
+ cost = f"${r.total_cost_usd:.4f}" if r.total_cost_usd is not None else "?"
1131
+ skills = r.events_summary.get("skills_invoked", [])
1132
+ skills_str = ", ".join(skills) if skills else "none"
1133
+ if len(skills_str) > 38:
1134
+ skills_str = skills_str[:35] + "..."
1135
+ print(
1136
+ f"{treatment:<25} {checks_str:<15} {turns:<8} {dur:<8} {tokens:<12} {cost:<10} {skills_str:<40}"
1137
+ )
1138
+
1139
+ print("-" * 120)
1140
+ total_passed = sum(
1141
+ sum(len(r.checks_passed) for r in runs) for runs in self.logger.results.values()
1142
+ )
1143
+ total_checks = sum(
1144
+ sum(len(r.checks_passed) + len(r.checks_failed) for r in runs)
1145
+ for runs in self.logger.results.values()
1146
+ )
1147
+ if total_checks:
1148
+ print(
1149
+ f"Total: {total_passed}/{total_checks} checks passed ({total_passed / total_checks * 100:.1f}%)"
1150
+ )
1151
+ print(f"{'=' * 120}")
1152
+
1153
+
1154
+ # =============================================================================
1155
+ # EXPERIMENT PLUGIN HELPERS
1156
+ # =============================================================================
1157
+
1158
+
1159
+ def _is_unit_tests_only(config) -> bool:
1160
+ """Check if running ONLY unit tests (scaffold/scripts - don't need experiment logs)."""
1161
+ args = [a for a in (config.args or []) if not a.startswith("-")]
1162
+ if not args:
1163
+ return False
1164
+ return all("scripts" in arg or "scaffold" in arg for arg in args)
1165
+
1166
+
1167
+ def _get_experiment_name(session) -> str:
1168
+ """Determine experiment name from task name parameter."""
1169
+ items = getattr(session, "items", None)
1170
+ if not items:
1171
+ return "experiment"
1172
+
1173
+ first_item = items[0]
1174
+ if hasattr(first_item, "callspec") and "task_name" in first_item.callspec.params:
1175
+ return first_item.callspec.params["task_name"].replace("-", "_")
1176
+
1177
+ return "experiment"
1178
+
1179
+
1180
+ def _get_dynamic_treatment_config(config):
1181
+ _ensure_auto_generated_manifest(config)
1182
+
1183
+ def generation_hints(manifest):
1184
+ generation_metadata = {}
1185
+ generation_metadata_hash = None
1186
+ generated = getattr(config, "_comet_generated_manifest", None)
1187
+ generation_metadata_path = (
1188
+ generated.metadata_path if generated is not None else manifest.generation_metadata_path
1189
+ )
1190
+ if generation_metadata_path and generation_metadata_path.is_file():
1191
+ generation_metadata_bytes = generation_metadata_path.read_bytes()
1192
+ generation_metadata = json.loads(generation_metadata_bytes)
1193
+ generation_metadata_hash = (
1194
+ "sha256:" + hashlib.sha256(generation_metadata_bytes).hexdigest()
1195
+ )
1196
+ return {
1197
+ "generation_hash": (
1198
+ generated.generation_hash if generated is not None else manifest.generation_hash
1199
+ ),
1200
+ "generation_metadata_path": (
1201
+ str(generation_metadata_path) if generation_metadata_path else None
1202
+ ),
1203
+ "generation_manifest_hash": generation_metadata.get("manifest_hash"),
1204
+ "generation_metadata_hash": generation_metadata_hash,
1205
+ "generation_overhead": generation_metadata.get("generation_overhead"),
1206
+ }
1207
+
1208
+ manifest_path = config.getoption("--eval-manifest")
1209
+ if manifest_path:
1210
+ from scaffold.python.manifests import load_eval_manifest
1211
+ from scaffold.python.treatments import TreatmentConfig
1212
+
1213
+ manifest = load_eval_manifest(manifest_path)
1214
+ generation = generation_hints(manifest)
1215
+ node_skills = []
1216
+ for node_skill in manifest.generated_node_skills:
1217
+ node_path = manifest.skill_path.parent / node_skill
1218
+ if (node_path / "SKILL.md").exists():
1219
+ node_skills.append(
1220
+ {
1221
+ "name": node_skill,
1222
+ "source": "path",
1223
+ "path": str(node_path),
1224
+ }
1225
+ )
1226
+ return TreatmentConfig(
1227
+ name="DYNAMIC_SKILL",
1228
+ description=f"Dynamic Skill target: {manifest.skill_name}",
1229
+ skills=[
1230
+ {
1231
+ "name": manifest.skill_name,
1232
+ "source": "path",
1233
+ "path": str(manifest.skill_path),
1234
+ "profile": manifest.profile,
1235
+ "manifest": str(manifest.path),
1236
+ "execution_agent": manifest.execution_agent,
1237
+ "baseline_treatments": manifest.baseline_treatments,
1238
+ "quality_gates": manifest.quality_gates,
1239
+ "required_output_schemas": manifest.required_output_schemas,
1240
+ "expected_evidence": manifest.expected_evidence,
1241
+ "draft_hash": manifest.draft_hash,
1242
+ **generation,
1243
+ "required_skills": manifest.required_skills,
1244
+ "expected_artifacts": manifest.expected_artifacts,
1245
+ "generated_node_skills": manifest.generated_node_skills,
1246
+ "route_conformance_task": manifest.route_conformance_task,
1247
+ "route_conformance_expected_node_order": (
1248
+ manifest.route_conformance_expected_node_order
1249
+ ),
1250
+ }
1251
+ ]
1252
+ + node_skills,
1253
+ )
1254
+
1255
+ skill_path = config.getoption("--skill-path")
1256
+ if not skill_path:
1257
+ return None
1258
+ skill_name = config.getoption("--skill-name") or Path(skill_path).resolve().parent.name
1259
+ manifest = getattr(config, "_comet_resolution_manifest", None)
1260
+ profile = config.getoption("--profile") or (manifest.profile if manifest else None)
1261
+ skill_cfg = {
1262
+ "name": skill_name,
1263
+ "source": "path",
1264
+ "path": skill_path,
1265
+ }
1266
+ if profile:
1267
+ skill_cfg["profile"] = profile
1268
+ if manifest:
1269
+ skill_cfg["manifest"] = str(manifest.path)
1270
+ skill_cfg.update(generation_hints(manifest))
1271
+ from scaffold.python.treatments import TreatmentConfig
1272
+
1273
+ return TreatmentConfig(
1274
+ name="DYNAMIC_SKILL",
1275
+ description=f"Dynamic Skill target: {skill_name}",
1276
+ skills=[skill_cfg],
1277
+ )
1278
+
1279
+
1280
+ def _resolve_interaction_config(task, profile_name: str, config):
1281
+ profile_default = get_profile(profile_name).default_interaction
1282
+ task_interaction = task.config.interaction
1283
+
1284
+ mode = task_interaction.mode or profile_default.mode
1285
+ max_turns = task_interaction.max_turns or profile_default.max_turns
1286
+ simulator_prompt = task_interaction.simulator_prompt or profile_default.simulator_prompt
1287
+ decision_patterns = list(
1288
+ task_interaction.decision_patterns or profile_default.decision_patterns
1289
+ )
1290
+ decision_reply = task_interaction.decision_reply or profile_default.decision_reply
1291
+ decision_replies = list(task_interaction.decision_replies or profile_default.decision_replies)
1292
+ continue_prompt = task_interaction.continue_prompt or profile_default.continue_prompt
1293
+ fresh_resume_marker = task_interaction.fresh_resume_marker
1294
+
1295
+ mode_override = config.getoption("--interaction-mode")
1296
+ if mode_override:
1297
+ mode = mode_override
1298
+
1299
+ max_turns_override = config.getoption("--max-turns")
1300
+ if max_turns_override not in (None, ""):
1301
+ max_turns = int(max_turns_override)
1302
+
1303
+ simulator_prompt_override = config.getoption("--simulator-prompt")
1304
+
1305
+ prompt_file = os.environ.get("BENCH_SIMULATOR_PROMPT_FILE")
1306
+ prompt_path = Path(prompt_file) if prompt_file else (EVAL_ROOT / "simulator-instruction.md")
1307
+ if not prompt_path.is_absolute():
1308
+ prompt_path = EVAL_ROOT / prompt_path
1309
+ if prompt_path.exists() and (prompt_file or not task_interaction.simulator_prompt):
1310
+ simulator_prompt = prompt_path.read_text(encoding="utf-8")
1311
+
1312
+ if simulator_prompt_override:
1313
+ simulator_prompt = simulator_prompt_override
1314
+
1315
+ return task_interaction.__class__(
1316
+ mode=mode,
1317
+ max_turns=max_turns,
1318
+ simulator_prompt=simulator_prompt,
1319
+ decision_patterns=decision_patterns,
1320
+ decision_reply=decision_reply,
1321
+ decision_replies=decision_replies,
1322
+ continue_prompt=continue_prompt,
1323
+ fresh_resume_marker=fresh_resume_marker,
1324
+ )
1325
+
1326
+
1327
+ def _read_required_text(path: Path) -> str:
1328
+ if not path.exists():
1329
+ raise FileNotFoundError(f"Required eval instruction file not found: {path}")
1330
+ return path.read_text(encoding="utf-8")
1331
+
1332
+
1333
+ def _build_eval_claude_md(profile_name: str, treatment_claude_md: str | None = None) -> str | None:
1334
+ sections: list[str] = []
1335
+ if profile_name == "comet-workflow":
1336
+ sections.append(_read_required_text(COMET_WORKFLOW_CLAUDE_MD_PATH))
1337
+ if treatment_claude_md:
1338
+ sections.append(treatment_claude_md.strip())
1339
+ return "\n\n".join(section for section in sections if section.strip()) or None
1340
+
1341
+
1342
+ def _comet_hook_command(test_dir: Path, agent: str = "claude-code") -> str | None:
1343
+ project_root = _agent_project_root(agent)
1344
+ scripts_dir = test_dir / project_root / "skills" / "comet" / "scripts"
1345
+ mjs_hook = scripts_dir / "comet-hook-guard.mjs"
1346
+ shell_hook = scripts_dir / "comet-hook-guard.sh"
1347
+ if mjs_hook.exists():
1348
+ return f"node /workspace/{project_root}/skills/comet/scripts/comet-hook-guard.mjs"
1349
+ if shell_hook.exists():
1350
+ return f"bash /workspace/{project_root}/skills/comet/scripts/comet-hook-guard.sh"
1351
+ return None
1352
+
1353
+
1354
+ def _ensure_claude_pre_tool_hook(test_dir: Path, command: str | None) -> None:
1355
+ if not command:
1356
+ return
1357
+ settings_path = test_dir / ".claude" / "settings.json"
1358
+ settings_path.parent.mkdir(parents=True, exist_ok=True)
1359
+ if settings_path.exists():
1360
+ settings = json.loads(settings_path.read_text(encoding="utf-8"))
1361
+ else:
1362
+ settings = {}
1363
+ hooks = settings.setdefault("hooks", {})
1364
+ pre_tool_use = hooks.setdefault("PreToolUse", [])
1365
+ hook_entry = {
1366
+ "matcher": "Write|Edit|MultiEdit",
1367
+ "hooks": [{"type": "command", "command": command}],
1368
+ }
1369
+ if not any(entry == hook_entry for entry in pre_tool_use):
1370
+ pre_tool_use.append(hook_entry)
1371
+ settings_path.write_text(json.dumps(settings, indent=2) + "\n", encoding="utf-8")
1372
+
1373
+
1374
+ def _ensure_agent_pre_tool_hook(test_dir: Path, agent: str, command: str | None) -> None:
1375
+ if agent == "claude-code":
1376
+ _ensure_claude_pre_tool_hook(test_dir, command)
1377
+ return
1378
+ if not command:
1379
+ return
1380
+
1381
+ project_root = ".codex" if agent == "codex" else _agent_project_root(agent)
1382
+ settings_name = "hooks.json" if agent == "codex" else "settings.json"
1383
+ settings_path = test_dir / project_root / settings_name
1384
+ if settings_path.exists():
1385
+ settings = json.loads(settings_path.read_text(encoding="utf-8"))
1386
+ else:
1387
+ settings = {}
1388
+ hooks = settings.setdefault("hooks", {})
1389
+ pre_tool_use = hooks.setdefault("PreToolUse", [])
1390
+ hook_entry = {
1391
+ "matcher": "Write|Edit|MultiEdit",
1392
+ "hooks": [{"type": "command", "command": command}],
1393
+ }
1394
+ if agent in {"qoder", "codebuddy"}:
1395
+ hook_entry["hooks"][0]["description"] = "Comet workflow guard"
1396
+ if not any(entry == hook_entry for entry in pre_tool_use):
1397
+ pre_tool_use.append(hook_entry)
1398
+ settings_path.parent.mkdir(parents=True, exist_ok=True)
1399
+ settings_path.write_text(json.dumps(settings, indent=2) + "\n", encoding="utf-8")
1400
+
1401
+
1402
+ def _get_or_create_experiment_id(name: str, use_coordination: bool) -> str:
1403
+ """Get shared experiment ID or create new one."""
1404
+ requested = os.environ.get(EXPERIMENT_ID_ENV)
1405
+ if requested is not None:
1406
+ if not EXPERIMENT_ID_RE.fullmatch(requested):
1407
+ raise ValueError(
1408
+ f"{EXPERIMENT_ID_ENV} must contain only letters, digits, dot, underscore, or hyphen"
1409
+ )
1410
+ return requested
1411
+
1412
+ if not use_coordination:
1413
+ timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
1414
+ return f"{name}_{timestamp}"
1415
+
1416
+ context = context_from_environment()
1417
+ experiment_file = (
1418
+ resolve_managed_path(context, "locks", ".pytest_experiment_id")
1419
+ if context is not None
1420
+ else XDIST_EXPERIMENT_FILE
1421
+ )
1422
+ lock_file = experiment_file.with_suffix(".lock")
1423
+
1424
+ with file_lock(lock_file):
1425
+ if experiment_file.exists():
1426
+ data = json.loads(experiment_file.read_text())
1427
+ return data["experiment_id"]
1428
+
1429
+ timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
1430
+ experiment_id = f"{name}_{timestamp}"
1431
+ experiment_file.write_text(
1432
+ json.dumps(
1433
+ {
1434
+ "experiment_id": experiment_id,
1435
+ "created_at": datetime.now().isoformat(),
1436
+ }
1437
+ )
1438
+ )
1439
+ return experiment_id
1440
+
1441
+
1442
+ def _expected_cases_from_items(items) -> list[tuple[str, str, int]]:
1443
+ """Read explicit eval-case marks from the complete pytest collection."""
1444
+ cases: list[tuple[str, str, int]] = []
1445
+ for item in items:
1446
+ callspec = getattr(item, "callspec", None)
1447
+ marker = item.get_closest_marker("eval_case")
1448
+ if callspec is None or marker is None:
1449
+ continue
1450
+ task = callspec.params.get("task_name")
1451
+ treatment = callspec.params.get("treatment_name")
1452
+ repetition = marker.kwargs.get("repetition")
1453
+ if not isinstance(task, str) or not isinstance(treatment, str):
1454
+ raise pytest.UsageError("eval_case is missing task/treatment parameters")
1455
+ if isinstance(repetition, bool) or not isinstance(repetition, int) or repetition < 1:
1456
+ raise pytest.UsageError("eval_case repetition must be a positive integer")
1457
+ cases.append((task, treatment, repetition))
1458
+ return cases
1459
+
1460
+
1461
+ def _persist_expected_case_matrix(base_dir: Path, payload: dict[str, Any]) -> None:
1462
+ """Write one xdist-shared matrix; workers must agree byte-for-byte."""
1463
+ canonical = parse_expected_case_matrix(payload)
1464
+ path = base_dir / EXPECTED_CASE_MATRIX_FILENAME
1465
+ lock_path = base_dir / f".{EXPECTED_CASE_MATRIX_FILENAME}.lock"
1466
+ serialized = json.dumps(payload, indent=2, sort_keys=True) + "\n"
1467
+ with file_lock(lock_path):
1468
+ if path.exists():
1469
+ try:
1470
+ existing = parse_expected_case_matrix(json.loads(path.read_text(encoding="utf-8")))
1471
+ except (OSError, json.JSONDecodeError, ValueError) as error:
1472
+ raise RuntimeError("Existing expected case matrix is invalid") from error
1473
+ if existing != canonical:
1474
+ raise RuntimeError("pytest workers collected different expected case matrices")
1475
+ return
1476
+ temporary = path.with_name(f".{path.name}.{os.getpid()}.{time.time_ns()}.tmp")
1477
+ try:
1478
+ temporary.write_text(serialized, encoding="utf-8")
1479
+ os.replace(temporary, path)
1480
+ finally:
1481
+ temporary.unlink(missing_ok=True)
1482
+
1483
+
1484
+ def _snapshot_dynamic_skill_package(test_dir: Path, skill_hints: dict[str, Any]) -> str | None:
1485
+ """Copy a manifest target package and generated Node Skills into the workspace.
1486
+
1487
+ Validation scripts run inside Docker with ``test_dir`` mounted as /workspace,
1488
+ so they cannot inspect arbitrary host paths from the original manifest. Keep
1489
+ a package-shaped snapshot under the workspace and pass a relative path.
1490
+ """
1491
+ raw_path = skill_hints.get("path")
1492
+ if not raw_path:
1493
+ return None
1494
+ source = Path(raw_path).expanduser().resolve()
1495
+ package_dir = source.parent if source.is_file() else source
1496
+ if not package_dir.exists():
1497
+ return None
1498
+
1499
+ snapshot_root = test_dir / "_eval_target_skills"
1500
+ snapshot_root.mkdir(parents=True, exist_ok=True)
1501
+ package_dest = snapshot_root / package_dir.name
1502
+ shutil.copytree(
1503
+ package_dir,
1504
+ package_dest,
1505
+ dirs_exist_ok=True,
1506
+ ignore=shutil.ignore_patterns(".comet"),
1507
+ )
1508
+
1509
+ for node_skill in skill_hints.get("generated_node_skills") or []:
1510
+ node_source = package_dir.parent / node_skill
1511
+ if not (node_source / "SKILL.md").exists():
1512
+ continue
1513
+ shutil.copytree(
1514
+ node_source,
1515
+ snapshot_root / node_skill,
1516
+ dirs_exist_ok=True,
1517
+ ignore=shutil.ignore_patterns(".comet"),
1518
+ )
1519
+
1520
+ return str(package_dest.relative_to(test_dir)).replace("\\", "/")
1521
+
1522
+
1523
+ def _cleanup_experiment_coordination():
1524
+ """Remove coordination files after experiment."""
1525
+ import sys
1526
+
1527
+ context = context_from_environment()
1528
+ experiment_file = (
1529
+ resolve_managed_path(context, "locks", ".pytest_experiment_id")
1530
+ if context is not None
1531
+ else XDIST_EXPERIMENT_FILE
1532
+ )
1533
+ for f in [experiment_file, experiment_file.with_suffix(".lock")]:
1534
+ try:
1535
+ f.unlink(missing_ok=True)
1536
+ except Exception as e:
1537
+ print(f"Warning: failed to clean up {f.name}: {e}", file=sys.stderr)
1538
+
1539
+
1540
+ # =============================================================================
1541
+ # SESSION-SCOPED FIXTURES
1542
+ # =============================================================================
1543
+
1544
+
1545
+ @pytest.fixture(scope="session")
1546
+ def project_root():
1547
+ """Project root directory."""
1548
+ return PROJECT_ROOT
1549
+
1550
+
1551
+ @pytest.fixture(scope="session")
1552
+ def worker_id(request):
1553
+ """Get pytest-xdist worker ID, or 'master' if not using xdist."""
1554
+ if hasattr(request.config, "workerinput"):
1555
+ return request.config.workerinput["workerid"]
1556
+ return "master"
1557
+
1558
+
1559
+ @pytest.fixture(scope="session", autouse=True)
1560
+ def verify_environment(project_root, request):
1561
+ """Verify the selected agent's Docker execution prerequisites."""
1562
+ if _is_unit_tests_only(request.config) or request.config.option.collectonly:
1563
+ return
1564
+
1565
+ from scaffold.python.utils import load_eval_environment
1566
+
1567
+ load_eval_environment()
1568
+
1569
+ # Check uv (Python package manager)
1570
+ if shutil.which("uv") is None:
1571
+ pytest.skip(
1572
+ "uv is not installed or not in PATH.\n"
1573
+ "Install it: https://docs.astral.sh/uv/getting-started/installation/"
1574
+ )
1575
+
1576
+ # Check bash (required for MSYS shell scripts on Windows)
1577
+ from scaffold.python.utils import BASH_EXEC
1578
+
1579
+ if os.name == "nt" and BASH_EXEC == "bash":
1580
+ # _resolve_bash() fell back to bare "bash" — verify it actually works
1581
+ try:
1582
+ bash_check = subprocess.run(["bash", "--version"], capture_output=True, timeout=5)
1583
+ if bash_check.returncode != 0:
1584
+ raise FileNotFoundError
1585
+ except (FileNotFoundError, subprocess.TimeoutExpired):
1586
+ pytest.skip(
1587
+ "Git Bash not found. Install Git for Windows: https://git-scm.com/download/win\n"
1588
+ "Or set GIT_BASH env var to the full path of bash.exe"
1589
+ )
1590
+
1591
+ result = run_shell("docker.sh", "check", check=False)
1592
+ if result.returncode != 0:
1593
+ pytest.skip("Docker not available")
1594
+
1595
+ selection = _resolve_eval_agent(request.config)
1596
+ _require_agent_credentials(selection.agent)
1597
+
1598
+
1599
+ def _docker_environment_dirs_for_request(request, tasks_dir: Path) -> list[Path]:
1600
+ """Return only task environments selected by the current eval invocation."""
1601
+ frozen = getattr(request.config, "_comet_frozen_task_set", None)
1602
+ if frozen is None:
1603
+ raise pytest.UsageError("Eval task collection did not produce a frozen task set")
1604
+ selected: list[Path] = []
1605
+ for item in frozen.tasks:
1606
+ environment_dir = item.task.environment_dir
1607
+ if (
1608
+ environment_dir.is_dir()
1609
+ and (environment_dir / "Dockerfile").is_file()
1610
+ and environment_dir not in selected
1611
+ ):
1612
+ selected.append(environment_dir)
1613
+ return selected
1614
+
1615
+
1616
+ @pytest.fixture(scope="session", autouse=True)
1617
+ def prebuild_docker_image(request):
1618
+ """Pre-build Docker image once per session to avoid race conditions."""
1619
+ if _is_unit_tests_only(request.config) or request.config.option.collectonly:
1620
+ yield
1621
+ return
1622
+
1623
+ tasks_dir = PROJECT_ROOT / "tasks"
1624
+ for env_dir in _docker_environment_dirs_for_request(request, tasks_dir):
1625
+ selected_agent = _resolve_eval_agent(request.config).agent
1626
+ image = _build_docker_image_with_lock(
1627
+ env_dir,
1628
+ selected_agent if selected_agent != "claude-code" else None,
1629
+ environment=getattr(request.config, "_comet_execution_env", None),
1630
+ )
1631
+ if image:
1632
+ print(f"\nPre-built Docker image: {image}")
1633
+
1634
+ yield
1635
+
1636
+ try:
1637
+ context = context_from_environment()
1638
+ lock_path = (
1639
+ resolve_managed_path(context, "locks", ".pytest_docker_build.lock")
1640
+ if context is not None
1641
+ else DOCKER_BUILD_LOCK
1642
+ )
1643
+ lock_path.unlink(missing_ok=True)
1644
+ except Exception:
1645
+ pass
1646
+
1647
+
1648
+ # =============================================================================
1649
+ # FUNCTION-SCOPED FIXTURES
1650
+ # =============================================================================
1651
+
1652
+
1653
+ @pytest.fixture
1654
+ def test_dir(tmp_path):
1655
+ """Create isolated test directory (pytest manages cleanup)."""
1656
+ return tmp_path
1657
+
1658
+
1659
+ @pytest.fixture
1660
+ def experiment_logger():
1661
+ """Get the experiment logger for the current session."""
1662
+ return _plugin.logger if _plugin else None
1663
+
1664
+
1665
+ @pytest.fixture
1666
+ def setup_test_context(test_dir):
1667
+ """Factory fixture to set up test context with skills and CLAUDE.md."""
1668
+
1669
+ def _write_skill(
1670
+ skill_name: str,
1671
+ skill_file: str,
1672
+ scripts_dir: Path | None = None,
1673
+ source_dir: Path | None = None,
1674
+ ) -> None:
1675
+ agent = _resolve_eval_agent(_plugin.config).agent if _plugin else "claude-code"
1676
+ skill_dir = test_dir / _agent_project_root(agent) / "skills" / skill_name
1677
+ skill_dir.mkdir(parents=True, exist_ok=True)
1678
+
1679
+ if source_dir and source_dir.is_dir():
1680
+ shutil.copytree(
1681
+ source_dir,
1682
+ skill_dir,
1683
+ dirs_exist_ok=True,
1684
+ ignore=shutil.ignore_patterns(".comet"),
1685
+ )
1686
+
1687
+ shutil.copyfile(skill_file, skill_dir / "SKILL.md")
1688
+
1689
+ if scripts_dir and scripts_dir.is_dir():
1690
+ scripts_dest = skill_dir / "scripts"
1691
+ shutil.rmtree(scripts_dest, ignore_errors=True)
1692
+ shutil.copytree(scripts_dir, scripts_dest, dirs_exist_ok=True)
1693
+
1694
+ def _copy_environment(environment_dir: Path) -> None:
1695
+ for item in environment_dir.iterdir():
1696
+ if item.name in {
1697
+ CURRENT_COMET_CLI_MARKER,
1698
+ TRUSTED_NATIVE_RUNTIME_MARKER,
1699
+ }:
1700
+ continue
1701
+ dest = test_dir / item.name
1702
+ if item.is_dir():
1703
+ if dest.exists() and dest.is_dir():
1704
+ shutil.copytree(item, dest, dirs_exist_ok=True)
1705
+ else:
1706
+ shutil.copytree(item, dest)
1707
+ else:
1708
+ dest.parent.mkdir(parents=True, exist_ok=True)
1709
+ shutil.copy2(item, dest)
1710
+ _copy_current_comet_cli_snapshot(environment_dir, test_dir)
1711
+
1712
+ def _copy_workspace(workspace_dir: Path) -> None:
1713
+ """Copy a bounded project fixture without replacing the task Dockerfile."""
1714
+ for item in workspace_dir.iterdir():
1715
+ if item.name in {"Dockerfile", CURRENT_COMET_CLI_MARKER, TRUSTED_NATIVE_RUNTIME_MARKER}:
1716
+ continue
1717
+ dest = test_dir / item.name
1718
+ if item.is_dir():
1719
+ shutil.copytree(item, dest, dirs_exist_ok=True)
1720
+ else:
1721
+ dest.parent.mkdir(parents=True, exist_ok=True)
1722
+ shutil.copy2(item, dest)
1723
+
1724
+ def _write_claude_md(content_file: str) -> None:
1725
+ agent = _resolve_eval_agent(_plugin.config).agent if _plugin else "claude-code"
1726
+ if agent == "claude-code":
1727
+ claude_dir = test_dir / ".claude"
1728
+ claude_dir.mkdir(parents=True, exist_ok=True)
1729
+ shutil.copyfile(content_file, claude_dir / "CLAUDE.md")
1730
+ shutil.copyfile(content_file, test_dir / "CLAUDE.md")
1731
+ elif agent == "codebuddy":
1732
+ codebuddy_dir = test_dir / ".codebuddy"
1733
+ codebuddy_dir.mkdir(parents=True, exist_ok=True)
1734
+ shutil.copyfile(content_file, codebuddy_dir / "CODEBUDDY.md")
1735
+ else:
1736
+ shutil.copyfile(content_file, test_dir / "AGENTS.md")
1737
+
1738
+ def _setup(
1739
+ skills: dict = None,
1740
+ claude_md: str = None,
1741
+ environment_dir: Path = None,
1742
+ workspace_dir: Path = None,
1743
+ ):
1744
+ for skill_name, cfg in (skills or {}).items():
1745
+ if not cfg:
1746
+ continue
1747
+
1748
+ if isinstance(cfg, dict):
1749
+ sections = cfg.get("sections") or cfg.get("all", [])
1750
+ scripts_dir = cfg.get("scripts_dir")
1751
+ script_filter = cfg.get("script_filter")
1752
+ source_dir = cfg.get("source_dir")
1753
+ else:
1754
+ sections, scripts_dir, script_filter, source_dir = cfg, None, None, None
1755
+
1756
+ if not sections:
1757
+ continue
1758
+
1759
+ content = "\n\n".join(s for s in sections if s and s.strip())
1760
+ with tempfile.NamedTemporaryFile(
1761
+ mode="w", suffix=".md", delete=False, encoding="utf-8"
1762
+ ) as f:
1763
+ f.write(content)
1764
+ skill_file = f.name
1765
+
1766
+ filtered_dir = _filter_scripts(scripts_dir, script_filter)
1767
+ is_temp_dir = filtered_dir and filtered_dir != scripts_dir
1768
+
1769
+ try:
1770
+ _write_skill(skill_name, skill_file, filtered_dir, source_dir)
1771
+ finally:
1772
+ os.unlink(skill_file)
1773
+ if is_temp_dir and filtered_dir.exists():
1774
+ shutil.rmtree(filtered_dir)
1775
+
1776
+ if workspace_dir and workspace_dir.exists():
1777
+ _copy_workspace(workspace_dir)
1778
+
1779
+ if environment_dir and environment_dir.exists():
1780
+ _copy_environment(environment_dir)
1781
+ _copy_trusted_native_runtime_snapshot(environment_dir, test_dir, skills)
1782
+
1783
+ if claude_md:
1784
+ with tempfile.NamedTemporaryFile(
1785
+ mode="w", suffix=".md", delete=False, encoding="utf-8"
1786
+ ) as f:
1787
+ f.write(claude_md)
1788
+ temp_file = f.name
1789
+ try:
1790
+ _write_claude_md(temp_file)
1791
+ finally:
1792
+ os.unlink(temp_file)
1793
+
1794
+ agent = _resolve_eval_agent(_plugin.config).agent if _plugin else "claude-code"
1795
+ _ensure_langfuse_trajectory_support(test_dir, agent)
1796
+ _ensure_agent_pre_tool_hook(test_dir, agent, _comet_hook_command(test_dir, agent))
1797
+
1798
+ return test_dir
1799
+
1800
+ return _setup
1801
+
1802
+
1803
+ @pytest.fixture
1804
+ def run_claude(test_dir, experiment_logger, request):
1805
+ """Factory fixture to run the selected evaluation agent in Docker.
1806
+
1807
+ For tasks using ``interaction.mode=auto_user`` the single-shot ``run-claude``
1808
+ is replaced by the multi-turn ``run-claude-loop`` driver, which simulates a
1809
+ user replying at the workflow's decision points.
1810
+ """
1811
+ execution = _resolve_eval_execution(request.config)
1812
+ selected_agent = execution.agent
1813
+ default_model = execution.model
1814
+
1815
+ def _run(
1816
+ prompt: str,
1817
+ timeout: int = 600,
1818
+ model: str = None,
1819
+ interaction=None,
1820
+ image_id: str | None = None,
1821
+ ):
1822
+ mdl = model or default_model
1823
+ run_execution = ResolvedExecution(
1824
+ selected_agent,
1825
+ mdl,
1826
+ execution.base_url,
1827
+ execution.sources,
1828
+ )
1829
+ run_environment = build_agent_environment(run_execution)
1830
+ if os.environ.get("TRACE_TO_LANGSMITH", "").lower() == "true" and not os.environ.get(
1831
+ "CC_LANGSMITH_LOG_FILE"
1832
+ ):
1833
+ os.environ["CC_LANGSMITH_LOG_FILE"] = "/workspace/langsmith-hook.log"
1834
+ use_loop = interaction is not None and interaction.mode == "auto_user"
1835
+ if not use_loop:
1836
+ result = run_agent_in_docker(
1837
+ test_dir,
1838
+ prompt,
1839
+ agent=selected_agent,
1840
+ timeout=timeout,
1841
+ model=mdl,
1842
+ base_url=execution.base_url,
1843
+ image_id=image_id,
1844
+ environment=run_environment,
1845
+ )
1846
+ else:
1847
+ task_prompt_file = test_dir / ".eval-task-prompt.txt"
1848
+ task_prompt_file.write_text(prompt, encoding="utf-8")
1849
+ loop_args = [
1850
+ "run-claude-loop",
1851
+ test_dir,
1852
+ "@//workspace/.eval-task-prompt.txt",
1853
+ "--max-turns",
1854
+ str(interaction.max_turns),
1855
+ ]
1856
+ if mdl:
1857
+ loop_args += ["--model", mdl]
1858
+ if image_id:
1859
+ loop_args += ["--image-id", image_id]
1860
+ if interaction.continue_prompt:
1861
+ loop_args += ["--continue-prompt", interaction.continue_prompt]
1862
+ for pattern in interaction.decision_patterns:
1863
+ loop_args += ["--decision-pattern", pattern]
1864
+ if interaction.decision_reply:
1865
+ loop_args += ["--decision-reply", interaction.decision_reply]
1866
+ for decision_reply_step in interaction.decision_replies:
1867
+ loop_args += ["--decision-reply-step", decision_reply_step]
1868
+ if interaction.fresh_resume_marker:
1869
+ loop_args += ["--fresh-resume-marker", interaction.fresh_resume_marker]
1870
+
1871
+ prompt_file = None
1872
+ try:
1873
+ if (
1874
+ interaction.simulator_prompt
1875
+ and not interaction.decision_reply
1876
+ and not interaction.decision_replies
1877
+ ):
1878
+ prompt_file = test_dir / ".eval-simulator-prompt.txt"
1879
+ prompt_file.write_text(interaction.simulator_prompt, encoding="utf-8")
1880
+ loop_args += [
1881
+ "--simulator-prompt-file",
1882
+ "//workspace/.eval-simulator-prompt.txt",
1883
+ ]
1884
+ if selected_agent == "claude-code":
1885
+ result = run_claude_loop_in_docker(
1886
+ test_dir,
1887
+ loop_args[2:],
1888
+ timeout=timeout + 60,
1889
+ environment=run_environment,
1890
+ )
1891
+ else:
1892
+ result = run_agent_loop_in_docker(
1893
+ test_dir,
1894
+ [
1895
+ loop_args[2],
1896
+ "--agent",
1897
+ selected_agent,
1898
+ *loop_args[3:],
1899
+ ],
1900
+ timeout=timeout + 60,
1901
+ environment=run_environment,
1902
+ )
1903
+ finally:
1904
+ task_prompt_file.unlink(missing_ok=True)
1905
+ if prompt_file and prompt_file.exists():
1906
+ prompt_file.unlink()
1907
+
1908
+ if experiment_logger and hasattr(request, "node"):
1909
+ treatment_name = _get_treatment_name(request.node)
1910
+ rep = _plugin.get_rep_number(treatment_name) if _plugin else 1
1911
+ save_raw(
1912
+ experiment_logger.base_dir,
1913
+ treatment_name,
1914
+ rep,
1915
+ result.stdout,
1916
+ result.stderr,
1917
+ )
1918
+
1919
+ return result
1920
+
1921
+ return _run
1922
+
1923
+
1924
+ @pytest.fixture
1925
+ def record_result(test_dir, experiment_logger, request):
1926
+ """Factory fixture to record validation results and save artifacts."""
1927
+
1928
+ def _record(
1929
+ events: dict[str, Any],
1930
+ passed: list[str],
1931
+ failed: list[str],
1932
+ run_id: str = "",
1933
+ returncode: int | None = None,
1934
+ stdout: str | None = None,
1935
+ stderr: str | None = None,
1936
+ ):
1937
+ if not experiment_logger:
1938
+ return
1939
+
1940
+ if _plugin is not None:
1941
+ _plugin.last_test_dir = test_dir
1942
+
1943
+ treatment_name = _get_treatment_name(request.node)
1944
+ rep = _plugin.run_counter.get(treatment_name, 1) if _plugin else 1
1945
+ events["sample"] = rep
1946
+ base_dir = experiment_logger.base_dir
1947
+
1948
+ save_events(base_dir, treatment_name, rep, events)
1949
+ _save_artifacts(
1950
+ base_dir,
1951
+ treatment_name,
1952
+ rep,
1953
+ test_dir,
1954
+ agent=events.get("agent", "claude-code"),
1955
+ )
1956
+ artifact_references = build_eval_artifact_references(base_dir, treatment_name, rep)
1957
+
1958
+ scripts_used = _extract_scripts_used(events)
1959
+ failure_attribution = classify_failures(
1960
+ failed,
1961
+ events,
1962
+ events.get("profile"),
1963
+ )
1964
+
1965
+ report = _build_report_payload(
1966
+ treatment_name=treatment_name,
1967
+ rep=rep,
1968
+ run_id=run_id,
1969
+ events=events,
1970
+ passed=passed,
1971
+ failed=failed,
1972
+ scripts_used=scripts_used,
1973
+ artifact_references=artifact_references,
1974
+ failure_attribution=failure_attribution,
1975
+ returncode=returncode,
1976
+ stdout=stdout,
1977
+ stderr=stderr,
1978
+ )
1979
+ save_report(base_dir, treatment_name, rep, report)
1980
+
1981
+ experiment_logger.add_result(
1982
+ treatment_name,
1983
+ TreatmentResult(
1984
+ name=treatment_name,
1985
+ passed=len(failed) == 0,
1986
+ checks_passed=passed,
1987
+ checks_failed=failed,
1988
+ events_summary={
1989
+ "num_turns": events.get("num_turns"),
1990
+ "duration_seconds": events.get("duration_seconds"),
1991
+ "tool_calls": len(events.get("tool_calls", [])),
1992
+ "input_tokens": events.get("input_tokens"),
1993
+ "output_tokens": events.get("output_tokens"),
1994
+ "cache_read_input_tokens": events.get("cache_read_input_tokens"),
1995
+ "cache_creation_input_tokens": events.get("cache_creation_input_tokens"),
1996
+ "total_tokens": events.get("total_tokens"),
1997
+ "total_cost_usd": events.get("total_cost_usd"),
1998
+ "model_usage": events.get("model_usage", {}),
1999
+ "skills_invoked": events.get("skills_invoked", []),
2000
+ "scripts_used": scripts_used,
2001
+ "profile": events.get("profile"),
2002
+ "skill_sources": events.get("skill_sources", []),
2003
+ "eval_manifest": events.get("eval_manifest"),
2004
+ "case_manifest": events.get("case_manifest"),
2005
+ "eval_generation": events.get("eval_generation"),
2006
+ "interaction": events.get("interaction", {}),
2007
+ "artifact_references": artifact_references,
2008
+ "failure_attribution": failure_attribution,
2009
+ "agent": events.get("agent", "claude-code"),
2010
+ "role_models": events.get("role_models", {}),
2011
+ "role_agents": events.get("role_agents", {}),
2012
+ "telemetry_status": events.get("telemetry_status", "N/A"),
2013
+ "role_sessions": events.get("role_sessions", {}),
2014
+ "task": events.get("task"),
2015
+ "treatment": events.get("treatment", treatment_name),
2016
+ "sample": events.get("sample", rep),
2017
+ "prompt": events.get("prompt"),
2018
+ "skill": events.get("skill"),
2019
+ "final_response": events.get("final_response"),
2020
+ "quality_gates": events.get("quality_gates", {}),
2021
+ "execution_identity": events.get("execution_identity"),
2022
+ },
2023
+ run_id=run_id,
2024
+ ),
2025
+ )
2026
+
2027
+ return _record
2028
+
2029
+
2030
+ def _build_report_payload(
2031
+ *,
2032
+ treatment_name: str,
2033
+ rep: int,
2034
+ run_id: str,
2035
+ events: dict[str, Any],
2036
+ passed: list[str],
2037
+ failed: list[str],
2038
+ scripts_used: list[str],
2039
+ artifact_references: dict[str, str],
2040
+ failure_attribution: list[dict[str, str]],
2041
+ returncode: int | None = None,
2042
+ stdout: str | None = None,
2043
+ stderr: str | None = None,
2044
+ ) -> dict[str, Any]:
2045
+ sample_quality = infer_sample_quality(
2046
+ events=events,
2047
+ checks_failed=failed,
2048
+ failure_attribution=failure_attribution,
2049
+ stdout=stdout,
2050
+ stderr=stderr,
2051
+ returncode=returncode,
2052
+ ).to_dict()
2053
+
2054
+ return {
2055
+ "name": treatment_name,
2056
+ "rep": rep,
2057
+ "passed": len(failed) == 0,
2058
+ "run_id": run_id,
2059
+ "checks_passed": passed,
2060
+ "checks_failed": failed,
2061
+ "sample_quality": sample_quality,
2062
+ "events_summary": {
2063
+ "duration_seconds": events.get("duration_seconds"),
2064
+ "num_turns": events.get("num_turns"),
2065
+ "tool_calls": len(events.get("tool_calls", [])),
2066
+ "input_tokens": events.get("input_tokens"),
2067
+ "output_tokens": events.get("output_tokens"),
2068
+ "cache_read_input_tokens": events.get("cache_read_input_tokens"),
2069
+ "cache_creation_input_tokens": events.get("cache_creation_input_tokens"),
2070
+ "total_tokens": events.get("total_tokens"),
2071
+ "total_cost_usd": events.get("total_cost_usd"),
2072
+ "model_usage": events.get("model_usage", {}),
2073
+ "files_created": events.get("files_created", []),
2074
+ "skills_invoked": events.get("skills_invoked", []),
2075
+ "skill_invocations": events.get("skill_invocations", []),
2076
+ "scripts_used": scripts_used,
2077
+ "profile": events.get("profile"),
2078
+ "skill_sources": events.get("skill_sources", []),
2079
+ "eval_manifest": events.get("eval_manifest"),
2080
+ "case_manifest": events.get("case_manifest"),
2081
+ "eval_generation": events.get("eval_generation"),
2082
+ "interaction": events.get("interaction", {}),
2083
+ "artifact_references": artifact_references,
2084
+ "failure_attribution": failure_attribution,
2085
+ "agent": events.get("agent", "claude-code"),
2086
+ "role_models": events.get("role_models", {}),
2087
+ "role_agents": events.get("role_agents", {}),
2088
+ "telemetry_status": events.get("telemetry_status", "N/A"),
2089
+ "role_sessions": events.get("role_sessions", {}),
2090
+ "task": events.get("task"),
2091
+ "treatment": events.get("treatment", treatment_name),
2092
+ "sample": events.get("sample", rep),
2093
+ "prompt": events.get("prompt"),
2094
+ "skill": events.get("skill"),
2095
+ "final_response": events.get("final_response"),
2096
+ "quality_gates": events.get("quality_gates", {}),
2097
+ "execution_identity": events.get("execution_identity"),
2098
+ },
2099
+ "timestamp": datetime.now().isoformat(),
2100
+ }
2101
+
2102
+
2103
+ # Fixture bundle accessor
2104
+ _current_fixtures: SimpleNamespace | None = None
2105
+
2106
+
2107
+ def get_fixtures() -> SimpleNamespace:
2108
+ """Get the current test's fixtures bundle."""
2109
+ if _current_fixtures is None:
2110
+ raise RuntimeError("get_fixtures() called outside of test context")
2111
+ return _current_fixtures
2112
+
2113
+
2114
+ @pytest.fixture(scope="function", autouse=True)
2115
+ def fixtures(
2116
+ verify_environment,
2117
+ test_dir,
2118
+ setup_test_context,
2119
+ run_claude,
2120
+ record_result,
2121
+ request,
2122
+ ):
2123
+ """Bundle test fixtures and make them accessible via get_fixtures()."""
2124
+ global _current_fixtures
2125
+ _current_fixtures = SimpleNamespace(
2126
+ test_dir=test_dir,
2127
+ setup_test_context=setup_test_context,
2128
+ run_claude=run_claude,
2129
+ record_result=record_result,
2130
+ request_config=request.config,
2131
+ )
2132
+ yield _current_fixtures
2133
+ _current_fixtures = None
2134
+
2135
+
2136
+ # =============================================================================
2137
+ # FIXTURE HELPERS
2138
+ # =============================================================================
2139
+
2140
+
2141
+ def _get_treatment_name(node) -> str:
2142
+ """Extract treatment name from pytest node."""
2143
+ nodeid = node.nodeid
2144
+ if "[" in nodeid:
2145
+ return nodeid.split("[")[1].rstrip("]")
2146
+ return nodeid.split("::")[-1]
2147
+
2148
+
2149
+ def _filter_scripts(scripts_dir: Path, script_filter: str) -> Path | None:
2150
+ """Filter scripts by extension and return a temp dir with filtered scripts."""
2151
+ if not scripts_dir or not scripts_dir.exists():
2152
+ return None
2153
+
2154
+ if script_filter is None or script_filter == "all":
2155
+ return scripts_dir
2156
+
2157
+ extensions = SCRIPT_EXTENSIONS.get(script_filter)
2158
+ if extensions is None:
2159
+ return scripts_dir
2160
+
2161
+ temp_dir = Path(tempfile.mkdtemp(prefix="scripts_"))
2162
+ copied_any = False
2163
+
2164
+ for script in scripts_dir.iterdir():
2165
+ if script.is_file() and script.suffix in extensions:
2166
+ shutil.copy2(script, temp_dir / script.name)
2167
+ copied_any = True
2168
+
2169
+ if not copied_any:
2170
+ shutil.rmtree(temp_dir)
2171
+ return None
2172
+
2173
+ return temp_dir
2174
+
2175
+
2176
+ def _build_docker_image_with_lock(
2177
+ environment_dir: Path,
2178
+ agent: str | None = None,
2179
+ *,
2180
+ environment: dict[str, str] | None = None,
2181
+ ) -> str | None:
2182
+ """Build Docker image with file locking to prevent race conditions."""
2183
+ if not environment_dir or not (environment_dir / "Dockerfile").exists():
2184
+ return None
2185
+
2186
+ context = context_from_environment()
2187
+ lock_path = (
2188
+ resolve_managed_path(context, "locks", ".pytest_docker_build.lock")
2189
+ if context is not None
2190
+ else DOCKER_BUILD_LOCK
2191
+ )
2192
+ with file_lock(lock_path):
2193
+ # A cold image build downloads Debian packages and installs the selected CLI.
2194
+ # Five minutes is insufficient after cache cleanup or on a proxied connection.
2195
+ command = ["docker.sh", "build", str(environment_dir)]
2196
+ if agent:
2197
+ command.extend(["--agent", agent])
2198
+ run_options = {"timeout": 900, "check": False}
2199
+ if environment is not None:
2200
+ run_options["env"] = environment
2201
+ result = run_shell(*command, **run_options)
2202
+ if result.returncode == 0:
2203
+ return result.stdout.strip()
2204
+ return None
2205
+
2206
+
2207
+ @contextmanager
2208
+ def file_lock(path: Path, timeout: float = 600):
2209
+ """Cross-platform exclusive file lock for pytest-xdist coordination."""
2210
+ path.parent.mkdir(parents=True, exist_ok=True)
2211
+ with open(path, "a+b") as lock_file:
2212
+ if os.name == "nt":
2213
+ import msvcrt
2214
+
2215
+ deadline = time.monotonic() + timeout
2216
+ while True:
2217
+ lock_file.seek(0)
2218
+ try:
2219
+ msvcrt.locking(lock_file.fileno(), msvcrt.LK_NBLCK, 1)
2220
+ break
2221
+ except OSError as error:
2222
+ if error.errno not in {13, 36}:
2223
+ raise
2224
+ if time.monotonic() >= deadline:
2225
+ raise
2226
+ time.sleep(0.05)
2227
+ try:
2228
+ yield
2229
+ finally:
2230
+ lock_file.seek(0)
2231
+ msvcrt.locking(lock_file.fileno(), msvcrt.LK_UNLCK, 1)
2232
+ else:
2233
+ import fcntl
2234
+
2235
+ fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX)
2236
+ try:
2237
+ yield
2238
+ finally:
2239
+ fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN)
2240
+
2241
+
2242
+ def _discover_skill_scripts() -> list[str]:
2243
+ """Dynamically discover all script files from skills directories."""
2244
+ scripts = set()
2245
+ skills_dir = PROJECT_ROOT / "skills"
2246
+
2247
+ if not skills_dir.exists():
2248
+ return []
2249
+
2250
+ for scripts_dir in skills_dir.rglob("scripts"):
2251
+ if scripts_dir.is_dir():
2252
+ for script in scripts_dir.iterdir():
2253
+ if script.is_file() and script.suffix in {".py", ".ts", ".js"}:
2254
+ scripts.add(script.name)
2255
+
2256
+ return sorted(scripts)
2257
+
2258
+
2259
+ def _get_known_scripts() -> list[str]:
2260
+ """Get known scripts, discovering them on first call."""
2261
+ global _KNOWN_SCRIPTS
2262
+ if _KNOWN_SCRIPTS is None:
2263
+ _KNOWN_SCRIPTS = _discover_skill_scripts()
2264
+ return _KNOWN_SCRIPTS
2265
+
2266
+
2267
+ def _extract_scripts_used(events: dict) -> list[str]:
2268
+ """Extract which skill scripts were used from events."""
2269
+ commands = " ".join(events.get("commands_run", [])).lower()
2270
+ files_read = " ".join(events.get("files_read", [])).lower()
2271
+ all_activity = commands + " " + files_read
2272
+
2273
+ return [s for s in _get_known_scripts() if s.lower() in all_activity]
2274
+
2275
+
2276
+ def _save_artifacts(
2277
+ base_dir: Path,
2278
+ treatment_name: str,
2279
+ rep: int,
2280
+ test_dir: Path,
2281
+ *,
2282
+ agent: str = "claude-code",
2283
+ ):
2284
+ """Save the selected Agent's generated files as artifacts."""
2285
+ artifacts_dir = base_dir / "artifacts" / f"{treatment_name.lower()}_rep{rep}"
2286
+ agent_dir_name = {
2287
+ "claude-code": "claude",
2288
+ "codex": "codex",
2289
+ "qoder": "qoder",
2290
+ "codebuddy": "codebuddy",
2291
+ }.get(agent, agent)
2292
+ agent_dir = artifacts_dir / agent_dir_name
2293
+ execution_dir = artifacts_dir / "execution"
2294
+ agent_dir.mkdir(parents=True, exist_ok=True)
2295
+ execution_dir.mkdir(parents=True, exist_ok=True)
2296
+
2297
+ from scaffold.python.utils import TEST_CONTEXT_FILE, TEST_RESULTS_FILE
2298
+
2299
+ exclude_dirs = {
2300
+ ".claude",
2301
+ ".agents",
2302
+ ".qoder",
2303
+ ".codebuddy",
2304
+ ".git",
2305
+ "_eval_current_comet",
2306
+ "node_modules",
2307
+ "__pycache__",
2308
+ "scaffold",
2309
+ "validation",
2310
+ "data",
2311
+ }
2312
+ exclude_files = {
2313
+ "CLAUDE.md",
2314
+ "Dockerfile",
2315
+ "requirements.txt",
2316
+ "package.json",
2317
+ "package-lock.json",
2318
+ "tsconfig.json",
2319
+ TEST_CONTEXT_FILE,
2320
+ TEST_RESULTS_FILE,
2321
+ }
2322
+
2323
+ claude_files = []
2324
+ for item in test_dir.rglob("*"):
2325
+ if not item.is_file():
2326
+ continue
2327
+ if item.name.startswith("."):
2328
+ continue
2329
+ if item.name in exclude_files:
2330
+ continue
2331
+ if any(excl in item.parts for excl in exclude_dirs):
2332
+ continue
2333
+ try:
2334
+ rel_path = item.relative_to(test_dir)
2335
+ dest = agent_dir / rel_path
2336
+ dest.parent.mkdir(parents=True, exist_ok=True)
2337
+ try:
2338
+ content = item.read_text(encoding="utf-8")
2339
+ except UnicodeDecodeError:
2340
+ import sys
2341
+
2342
+ print(
2343
+ f"Warning: skipped non-text artifact {rel_path}; binary content is not persisted to protect credentials",
2344
+ file=sys.stderr,
2345
+ )
2346
+ continue
2347
+ else:
2348
+ dest.write_text(redact_sensitive(content), encoding="utf-8")
2349
+ claude_files.append(item)
2350
+ except Exception as e:
2351
+ import sys
2352
+
2353
+ print(f"Warning: failed to save artifact {item.name}: {e}", file=sys.stderr)
2354
+
2355
+ for py_file in claude_files:
2356
+ if py_file.suffix == ".py" and py_file.parent == test_dir:
2357
+ try:
2358
+ success, output = run_python_in_docker(test_dir, py_file.name, timeout=300)
2359
+ status = "success" if success else "error"
2360
+ output_file = execution_dir / f"{py_file.stem}_{status}.txt"
2361
+ output_file.write_text(redact_sensitive(strip_ansi(output)), encoding="utf-8")
2362
+ except Exception as e:
2363
+ error_file = execution_dir / f"{py_file.stem}_error.txt"
2364
+ error_file.write_text(redact_sensitive(str(e)), encoding="utf-8")