@smoothbricks/nx-plugin 0.5.7 → 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (446) hide show
  1. package/README.md +1097 -360
  2. package/dist/.build-inputs.json +141 -0
  3. package/dist/bin/smoo-nx-cargo-toolchain.d.ts +3 -0
  4. package/dist/bin/smoo-nx-cargo-toolchain.d.ts.map +1 -0
  5. package/dist/bin/smoo-nx-cargo-toolchain.js +8 -0
  6. package/dist/bin/smoo-nx-forbidden-spellings.d.ts +3 -0
  7. package/dist/bin/smoo-nx-forbidden-spellings.d.ts.map +1 -0
  8. package/dist/bin/smoo-nx-forbidden-spellings.js +32 -0
  9. package/dist/bin/smoo-nx-nextest-timings.d.ts +3 -0
  10. package/dist/bin/smoo-nx-nextest-timings.d.ts.map +1 -0
  11. package/dist/bin/smoo-nx-nextest-timings.js +28 -0
  12. package/dist/bounded-test-policy.d.ts +1 -1
  13. package/dist/bounded-test-policy.d.ts.map +1 -1
  14. package/dist/bounded-test-policy.js +14 -15
  15. package/dist/build-state-output-policy.d.ts +95 -1
  16. package/dist/build-state-output-policy.d.ts.map +1 -1
  17. package/dist/build-state-output-policy.js +368 -30
  18. package/dist/bun-lock-scope.d.ts +44 -0
  19. package/dist/bun-lock-scope.d.ts.map +1 -0
  20. package/dist/bun-lock-scope.js +97 -0
  21. package/dist/cargo-build-script-env.d.ts +49 -0
  22. package/dist/cargo-build-script-env.d.ts.map +1 -0
  23. package/dist/cargo-build-script-env.js +324 -0
  24. package/dist/cargo-closure-input.d.ts +31 -4
  25. package/dist/cargo-closure-input.d.ts.map +1 -1
  26. package/dist/cargo-closure-input.js +66 -6
  27. package/dist/cargo-retention-hook.d.ts +17 -0
  28. package/dist/cargo-retention-hook.d.ts.map +1 -0
  29. package/dist/cargo-retention-hook.js +85 -0
  30. package/dist/cargo-source-hash.d.ts +25 -7
  31. package/dist/cargo-source-hash.d.ts.map +1 -1
  32. package/dist/cargo-source-hash.js +108 -43
  33. package/dist/cargo-toolchain-paths.d.ts +40 -0
  34. package/dist/cargo-toolchain-paths.d.ts.map +1 -0
  35. package/dist/cargo-toolchain-paths.js +84 -0
  36. package/dist/cargo-toolchain-policy.d.ts +25 -8
  37. package/dist/cargo-toolchain-policy.d.ts.map +1 -1
  38. package/dist/cargo-toolchain-policy.js +39 -5
  39. package/dist/cargo-workspace.d.ts +10 -4
  40. package/dist/cargo-workspace.d.ts.map +1 -1
  41. package/dist/cargo-workspace.js +65 -95
  42. package/dist/cross-check-policy.d.ts +9 -9
  43. package/dist/cross-check-policy.d.ts.map +1 -1
  44. package/dist/cross-check-policy.js +11 -11
  45. package/dist/disk-command-policy.d.ts.map +1 -1
  46. package/dist/disk-command-policy.js +1 -0
  47. package/dist/ensure-built-receipt.d.ts +22 -5
  48. package/dist/ensure-built-receipt.d.ts.map +1 -1
  49. package/dist/ensure-built-receipt.js +168 -48
  50. package/dist/ensure-built-witness.d.ts +27 -8
  51. package/dist/ensure-built-witness.d.ts.map +1 -1
  52. package/dist/ensure-built-witness.js +45 -15
  53. package/dist/ensure-built.d.ts.map +1 -1
  54. package/dist/ensure-built.js +9 -1
  55. package/dist/eslint-config-closure.d.ts +7 -0
  56. package/dist/eslint-config-closure.d.ts.map +1 -0
  57. package/dist/eslint-config-closure.js +346 -0
  58. package/dist/executors/bounded-exec/bounds.d.ts +56 -0
  59. package/dist/executors/bounded-exec/bounds.d.ts.map +1 -0
  60. package/dist/executors/bounded-exec/bounds.js +43 -0
  61. package/dist/executors/bounded-exec/cpu-pool-thread.d.ts +28 -0
  62. package/dist/executors/bounded-exec/cpu-pool-thread.d.ts.map +1 -0
  63. package/dist/executors/bounded-exec/cpu-pool-thread.js +20 -0
  64. package/dist/executors/bounded-exec/cpu-pool.d.ts +35 -38
  65. package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
  66. package/dist/executors/bounded-exec/cpu-pool.js +56 -120
  67. package/dist/executors/bounded-exec/executor.d.ts +19 -12
  68. package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
  69. package/dist/executors/bounded-exec/executor.js +389 -239
  70. package/dist/executors/bounded-exec/gateway-lease.d.ts +23 -19
  71. package/dist/executors/bounded-exec/gateway-lease.d.ts.map +1 -1
  72. package/dist/executors/bounded-exec/gateway-lease.js +102 -29
  73. package/dist/executors/bounded-exec/hang-diagnostics.d.ts +18 -1
  74. package/dist/executors/bounded-exec/hang-diagnostics.d.ts.map +1 -1
  75. package/dist/executors/bounded-exec/hang-diagnostics.js +148 -30
  76. package/dist/executors/bounded-exec/lease-wait.d.ts +64 -0
  77. package/dist/executors/bounded-exec/lease-wait.d.ts.map +1 -0
  78. package/dist/executors/bounded-exec/lease-wait.js +72 -0
  79. package/dist/executors/bounded-exec/lock-wait.d.ts +51 -0
  80. package/dist/executors/bounded-exec/lock-wait.d.ts.map +1 -0
  81. package/dist/executors/bounded-exec/lock-wait.js +108 -0
  82. package/dist/executors/bounded-exec/mbx-pool.d.ts +31 -0
  83. package/dist/executors/bounded-exec/mbx-pool.d.ts.map +1 -0
  84. package/dist/executors/bounded-exec/mbx-pool.js +24 -0
  85. package/dist/executors/bounded-exec/owned-subtree.d.ts +84 -0
  86. package/dist/executors/bounded-exec/owned-subtree.d.ts.map +1 -0
  87. package/dist/executors/bounded-exec/owned-subtree.js +178 -0
  88. package/dist/executors/bounded-exec/progress.d.ts +4 -1
  89. package/dist/executors/bounded-exec/progress.d.ts.map +1 -1
  90. package/dist/executors/bounded-exec/progress.js +5 -2
  91. package/dist/executors/bounded-exec/ram-attach.d.ts +19 -0
  92. package/dist/executors/bounded-exec/ram-attach.d.ts.map +1 -0
  93. package/dist/executors/bounded-exec/ram-attach.js +207 -0
  94. package/dist/executors/bounded-exec/ram-detach.d.ts +25 -0
  95. package/dist/executors/bounded-exec/ram-detach.d.ts.map +1 -0
  96. package/dist/executors/bounded-exec/ram-detach.js +207 -0
  97. package/dist/executors/bounded-exec/ram-temp-fixtures.d.ts +76 -0
  98. package/dist/executors/bounded-exec/ram-temp-fixtures.d.ts.map +1 -1
  99. package/dist/executors/bounded-exec/ram-temp-fixtures.js +296 -0
  100. package/dist/executors/bounded-exec/ram-temp.d.ts +388 -88
  101. package/dist/executors/bounded-exec/ram-temp.d.ts.map +1 -1
  102. package/dist/executors/bounded-exec/ram-temp.js +1213 -399
  103. package/dist/executors/bounded-exec/run-cpu-pool.d.ts +10 -4
  104. package/dist/executors/bounded-exec/run-cpu-pool.d.ts.map +1 -1
  105. package/dist/executors/bounded-exec/run-cpu-pool.js +122 -35
  106. package/dist/executors/bounded-exec/runner-hook.js +14 -5
  107. package/dist/executors/bounded-exec/runner-lifecycle.d.ts +14 -1
  108. package/dist/executors/bounded-exec/runner-lifecycle.d.ts.map +1 -1
  109. package/dist/executors/bounded-exec/runner-lifecycle.js +27 -6
  110. package/dist/executors/bounded-exec/{cpu-tokens.d.ts → runner-sizing.d.ts} +14 -42
  111. package/dist/executors/bounded-exec/runner-sizing.d.ts.map +1 -0
  112. package/dist/executors/bounded-exec/{cpu-tokens.js → runner-sizing.js} +5 -35
  113. package/dist/executors/bounded-exec/runtime-pin.d.ts +73 -0
  114. package/dist/executors/bounded-exec/runtime-pin.d.ts.map +1 -0
  115. package/dist/executors/bounded-exec/runtime-pin.js +167 -0
  116. package/dist/executors/bounded-exec/schema.d.ts +18 -1
  117. package/dist/executors/bounded-exec/schema.d.ts.map +1 -1
  118. package/dist/executors/bounded-exec/verdict.d.ts +48 -11
  119. package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
  120. package/dist/executors/bounded-exec/verdict.js +89 -20
  121. package/dist/executors/cargo-zigbuild/executor.d.ts +21 -0
  122. package/dist/executors/cargo-zigbuild/executor.d.ts.map +1 -1
  123. package/dist/executors/cargo-zigbuild/executor.js +67 -12
  124. package/dist/executors/clean-outputs/executor.js +1 -0
  125. package/dist/executors/scriptc-native/executor.d.ts +79 -0
  126. package/dist/executors/scriptc-native/executor.d.ts.map +1 -0
  127. package/dist/executors/scriptc-native/executor.js +367 -0
  128. package/dist/executors/scriptc-native/ffi-paths.d.ts +9 -0
  129. package/dist/executors/scriptc-native/ffi-paths.d.ts.map +1 -0
  130. package/dist/executors/scriptc-native/ffi-paths.js +25 -0
  131. package/dist/executors/scriptc-native/schema.d.ts +34 -0
  132. package/dist/executors/scriptc-native/schema.d.ts.map +1 -0
  133. package/dist/executors/scriptc-native/schema.js +1 -0
  134. package/dist/executors/typescript-emit/emit-overrides.d.ts +14 -0
  135. package/dist/executors/typescript-emit/emit-overrides.d.ts.map +1 -0
  136. package/dist/executors/typescript-emit/emit-overrides.js +24 -0
  137. package/dist/executors/typescript-emit/executor.d.ts +3 -1
  138. package/dist/executors/typescript-emit/executor.d.ts.map +1 -1
  139. package/dist/executors/typescript-emit/executor.js +146 -33
  140. package/dist/forbidden-spelling-policy.d.ts +56 -0
  141. package/dist/forbidden-spelling-policy.d.ts.map +1 -0
  142. package/dist/forbidden-spelling-policy.js +176 -0
  143. package/dist/generators/create-package/generator.d.ts.map +1 -1
  144. package/dist/generators/create-package/generator.js +3 -4
  145. package/dist/graph-input-contract.d.ts +14 -0
  146. package/dist/graph-input-contract.d.ts.map +1 -1
  147. package/dist/graph-input-contract.js +67 -2
  148. package/dist/index.d.ts +23 -6
  149. package/dist/index.d.ts.map +1 -1
  150. package/dist/index.js +672 -214
  151. package/dist/managed-assets.d.ts +2 -0
  152. package/dist/managed-assets.d.ts.map +1 -1
  153. package/dist/managed-files/files.d.ts +3 -3
  154. package/dist/managed-files/files.d.ts.map +1 -1
  155. package/dist/managed-files/files.js +28 -3
  156. package/dist/managed-files/managed-format.d.ts.map +1 -1
  157. package/dist/managed-files/managed-format.js +1 -0
  158. package/dist/managed-files/publish-workflow.js +14 -11
  159. package/dist/nextest-code-bound.js +1 -0
  160. package/dist/nextest-timings.d.ts +35 -0
  161. package/dist/nextest-timings.d.ts.map +1 -0
  162. package/dist/nextest-timings.js +168 -0
  163. package/dist/owned-installation.d.ts +6 -5
  164. package/dist/owned-installation.d.ts.map +1 -1
  165. package/dist/owned-installation.js +150 -26
  166. package/dist/package-target-policy.d.ts.map +1 -1
  167. package/dist/package-target-policy.js +2 -3
  168. package/dist/process-children.d.ts +7 -0
  169. package/dist/process-children.d.ts.map +1 -0
  170. package/dist/process-children.darwin.d.ts +7 -0
  171. package/dist/process-children.darwin.d.ts.map +1 -0
  172. package/dist/process-children.darwin.js +13 -0
  173. package/dist/process-children.js +14 -0
  174. package/dist/process-children.linux.d.ts +8 -0
  175. package/dist/process-children.linux.d.ts.map +1 -0
  176. package/dist/process-children.linux.js +25 -0
  177. package/dist/process-group.d.ts +108 -0
  178. package/dist/process-group.d.ts.map +1 -0
  179. package/dist/process-group.js +171 -0
  180. package/dist/process-listing.d.ts +7 -0
  181. package/dist/process-listing.d.ts.map +1 -0
  182. package/dist/process-listing.js +51 -0
  183. package/dist/producer-receipts.d.ts +8 -2
  184. package/dist/producer-receipts.d.ts.map +1 -1
  185. package/dist/producer-receipts.js +514 -47
  186. package/dist/task-graph-policy.d.ts +21 -0
  187. package/dist/task-graph-policy.d.ts.map +1 -0
  188. package/dist/task-graph-policy.js +91 -0
  189. package/dist/test-timings.js +4 -4
  190. package/dist/test-walk-policy.d.ts +14 -5
  191. package/dist/test-walk-policy.d.ts.map +1 -1
  192. package/dist/test-walk-policy.js +8 -2
  193. package/dist/testing.d.ts +73 -28
  194. package/dist/testing.d.ts.map +1 -1
  195. package/dist/testing.js +217 -223
  196. package/dist/tool-bins.d.ts +33 -0
  197. package/dist/tool-bins.d.ts.map +1 -0
  198. package/dist/tool-bins.js +72 -0
  199. package/dist/typecheck-test-policy.d.ts.map +1 -1
  200. package/dist/typecheck-test-policy.js +3 -4
  201. package/dist/wait-why-policy.d.ts +28 -0
  202. package/dist/wait-why-policy.d.ts.map +1 -0
  203. package/dist/wait-why-policy.js +223 -0
  204. package/dist/workspace-config-policy.d.ts +14 -8
  205. package/dist/workspace-config-policy.d.ts.map +1 -1
  206. package/dist/workspace-config-policy.js +60 -16
  207. package/dist/workspace-manifest.d.ts +41 -6
  208. package/dist/workspace-manifest.d.ts.map +1 -1
  209. package/dist/workspace-manifest.js +57 -0
  210. package/executors.json +16 -1
  211. package/loader.js +2 -0
  212. package/managed/raw/envrc +11 -0
  213. package/managed/raw/patches/nx@23.2.1.patch +2840 -199
  214. package/managed/raw/patches/ttsc@0.34.1.patch +119 -0
  215. package/managed/raw/tooling/direnv/cargo-fetch-bound.ts +245 -0
  216. package/managed/raw/tooling/direnv/cargo-fetch-run.ts +299 -0
  217. package/managed/raw/tooling/direnv/compiler-variables.sh +42 -0
  218. package/managed/raw/tooling/direnv/developer-links.ts +464 -25
  219. package/managed/raw/tooling/direnv/devenv.smoo.nix +78 -38
  220. package/managed/raw/tooling/direnv/github-actions-bootstrap.sh +14 -12
  221. package/managed/raw/tooling/direnv/installed-bun-links.ts +308 -0
  222. package/managed/raw/tooling/direnv/linked-snapshots.ts +62 -3
  223. package/managed/raw/tooling/direnv/nx-socket-dir.sh +56 -17
  224. package/managed/raw/tooling/direnv/repo-path +68 -6
  225. package/managed/raw/tooling/direnv/secret-references.ts +13 -4
  226. package/managed/raw/tooling/direnv/setup-environment.ts +128 -537
  227. package/managed/raw/tooling/direnv/toolchain-stamp.ts +4 -2
  228. package/managed/raw/tooling/direnv/ttsc-env.sh +9 -0
  229. package/managed/raw/tooling/direnv/uv-relocation.ts +227 -0
  230. package/managed/raw/tooling/git-hooks/commit-msg.sh +2 -1
  231. package/managed/raw/tooling/git-hooks/pre-commit.sh +2 -1
  232. package/managed/raw/tooling/git-hooks/pre-push.sh +2 -1
  233. package/managed/templates/github/actions/cache-ttsc-plugins/action.yml +85 -7
  234. package/managed/templates/github/actions/save-nix-devenv/action.yml +12 -2
  235. package/managed/templates/github/actions/setup-devenv/action.yml +2 -0
  236. package/package.json +95 -14
  237. package/src/bin/smoo-nx-cargo-toolchain.ts +11 -0
  238. package/src/bin/smoo-nx-forbidden-spellings.ts +33 -0
  239. package/src/bin/smoo-nx-nextest-timings.ts +31 -0
  240. package/src/bounded-test-policy.ts +12 -14
  241. package/src/build-state-output-policy.ts +458 -34
  242. package/src/bun-lock-scope.ts +108 -0
  243. package/src/cargo-build-script-env.ts +265 -0
  244. package/src/cargo-closure-input.ts +74 -7
  245. package/src/cargo-retention-hook.ts +94 -0
  246. package/src/cargo-source-hash.ts +138 -41
  247. package/src/cargo-toolchain-paths.ts +95 -0
  248. package/src/cargo-toolchain-policy.ts +49 -5
  249. package/src/cargo-workspace.ts +72 -123
  250. package/src/cross-check-policy.ts +12 -10
  251. package/src/disk-command-policy.ts +1 -0
  252. package/src/ensure-built-receipt.ts +173 -39
  253. package/src/ensure-built-witness.ts +62 -10
  254. package/src/ensure-built.ts +11 -1
  255. package/src/eslint-config-closure.ts +357 -0
  256. package/src/executors/bounded-exec/bounds.ts +76 -0
  257. package/src/executors/bounded-exec/cpu-pool-thread.ts +43 -0
  258. package/src/executors/bounded-exec/cpu-pool.ts +55 -120
  259. package/src/executors/bounded-exec/executor.ts +442 -260
  260. package/src/executors/bounded-exec/gateway-lease.ts +94 -35
  261. package/src/executors/bounded-exec/hang-diagnostics.ts +173 -31
  262. package/src/executors/bounded-exec/lease-wait.ts +116 -0
  263. package/src/executors/bounded-exec/lock-wait.ts +141 -0
  264. package/src/executors/bounded-exec/mbx-pool.ts +48 -0
  265. package/src/executors/bounded-exec/owned-subtree.ts +235 -0
  266. package/src/executors/bounded-exec/progress.ts +9 -3
  267. package/src/executors/bounded-exec/ram-attach.ts +238 -0
  268. package/src/executors/bounded-exec/ram-detach.ts +226 -0
  269. package/src/executors/bounded-exec/ram-temp-fixtures.ts +358 -0
  270. package/src/executors/bounded-exec/ram-temp-registry.txt +440 -0
  271. package/src/executors/bounded-exec/ram-temp.ts +1497 -460
  272. package/src/executors/bounded-exec/run-cpu-pool.ts +129 -35
  273. package/src/executors/bounded-exec/runner-hook.ts +14 -6
  274. package/src/executors/bounded-exec/runner-lifecycle.ts +31 -8
  275. package/src/executors/bounded-exec/{cpu-tokens.ts → runner-sizing.ts} +13 -59
  276. package/src/executors/bounded-exec/runtime-pin.ts +232 -0
  277. package/src/executors/bounded-exec/schema.json +11 -2
  278. package/src/executors/bounded-exec/schema.ts +19 -1
  279. package/src/executors/bounded-exec/verdict.ts +145 -35
  280. package/src/executors/cargo-zigbuild/executor.ts +82 -8
  281. package/src/executors/clean-outputs/executor.ts +1 -0
  282. package/src/executors/scriptc-native/executor.ts +394 -0
  283. package/src/executors/scriptc-native/ffi-paths.ts +29 -0
  284. package/src/executors/scriptc-native/schema.json +60 -0
  285. package/src/executors/scriptc-native/schema.ts +34 -0
  286. package/src/executors/typescript-emit/emit-overrides.ts +24 -0
  287. package/src/executors/typescript-emit/executor.ts +186 -38
  288. package/src/forbidden-spelling-policy.ts +226 -0
  289. package/src/generators/create-package/generator.ts +7 -7
  290. package/src/graph-input-contract.ts +61 -2
  291. package/src/index.ts +788 -224
  292. package/src/managed-assets.ts +3 -0
  293. package/src/managed-files/files.ts +28 -3
  294. package/src/managed-files/managed-format.ts +1 -0
  295. package/src/managed-files/publish-workflow.ts +20 -15
  296. package/src/nextest-code-bound.ts +1 -0
  297. package/src/nextest-timings.ts +106 -0
  298. package/src/owned-installation.ts +150 -23
  299. package/src/package-target-policy.ts +2 -3
  300. package/src/process-children.darwin.ts +14 -0
  301. package/src/process-children.linux.ts +33 -0
  302. package/src/process-children.ts +22 -0
  303. package/src/process-group.ts +209 -0
  304. package/src/process-listing.ts +54 -0
  305. package/src/producer-receipts.ts +92 -41
  306. package/src/task-graph-policy.ts +98 -0
  307. package/src/test-walk-policy.ts +23 -8
  308. package/src/testing.ts +240 -232
  309. package/src/tool-bins.ts +108 -0
  310. package/src/typecheck-test-policy.ts +2 -3
  311. package/src/wait-why-policy.ts +226 -0
  312. package/src/workspace-config-policy.ts +63 -15
  313. package/src/workspace-manifest.ts +89 -6
  314. package/test-timings.json +124 -99
  315. package/dist/bin/smoo-nx-failures.d.ts +0 -3
  316. package/dist/bin/smoo-nx-failures.d.ts.map +0 -1
  317. package/dist/bin/smoo-nx-failures.js +0 -3
  318. package/dist/bin/smoo-nx-nextest-extract.d.ts +0 -3
  319. package/dist/bin/smoo-nx-nextest-extract.d.ts.map +0 -1
  320. package/dist/bin/smoo-nx-nextest-extract.js +0 -42
  321. package/dist/executors/bounded-exec/cpu-tokens.d.ts.map +0 -1
  322. package/dist/failure-summary.d.ts +0 -5
  323. package/dist/failure-summary.d.ts.map +0 -1
  324. package/dist/failure-summary.js +0 -118
  325. package/dist/nextest-extraction.d.ts +0 -117
  326. package/dist/nextest-extraction.d.ts.map +0 -1
  327. package/dist/nextest-extraction.js +0 -303
  328. package/managed/raw/patches/ttsc@0.30.4.patch +0 -2255
  329. package/src/__tests__/cargo-closure-filesets.ts +0 -94
  330. package/src/__tests__/cargo-closure-fixture.ts +0 -113
  331. package/src/__tests__/counted-cargo-process.ts +0 -286
  332. package/src/__tests__/counted-cargo.ts +0 -713
  333. package/src/__tests__/directory-change.ts +0 -27
  334. package/src/__tests__/fixture-cargo-home.ts +0 -36
  335. package/src/__tests__/fixture-nx-env.ts +0 -193
  336. package/src/__tests__/owned-installation.ts +0 -13
  337. package/src/__tests__/past-tick.ts +0 -12
  338. package/src/__tests__/progress-clock.ts +0 -43
  339. package/src/__tests__/runtime-input.ts +0 -23
  340. package/src/bin/smoo-nx-bound-failures.test.ts +0 -127
  341. package/src/bin/smoo-nx-failures.ts +0 -4
  342. package/src/bin/smoo-nx-nextest-extract.ts +0 -44
  343. package/src/bounded-test-policy.test.ts +0 -1029
  344. package/src/build-state-output-policy.test.ts +0 -115
  345. package/src/bun-bootstrap.test.ts +0 -102
  346. package/src/bun-file-packages.test.ts +0 -47
  347. package/src/cargo-closure-filesets.test.ts +0 -63
  348. package/src/cargo-closure-input.property.test.ts +0 -50
  349. package/src/cargo-closure-input.test.ts +0 -506
  350. package/src/cargo-closure-key.test.ts +0 -240
  351. package/src/cargo-resolve-cache.test.ts +0 -1181
  352. package/src/cargo-source-hash.test.ts +0 -707
  353. package/src/cargo-workspace.test.ts +0 -615
  354. package/src/cross-check-policy.test.ts +0 -316
  355. package/src/disk-command-policy.test.ts +0 -61
  356. package/src/ensure-built-receipt.test.ts +0 -279
  357. package/src/ensure-built-witness.test.ts +0 -688
  358. package/src/ensure-built.test.ts +0 -1462
  359. package/src/executors/all-verdicts/executor.test.ts +0 -135
  360. package/src/executors/bounded-exec/cargo-progress-host.test.ts +0 -332
  361. package/src/executors/bounded-exec/cpu-pool.test.ts +0 -684
  362. package/src/executors/bounded-exec/cpu-tokens.test.ts +0 -519
  363. package/src/executors/bounded-exec/disk-lease.test.ts +0 -219
  364. package/src/executors/bounded-exec/executor-host.test.ts +0 -800
  365. package/src/executors/bounded-exec/executor.test.ts +0 -1819
  366. package/src/executors/bounded-exec/go-progress-host.test.ts +0 -171
  367. package/src/executors/bounded-exec/hang-diagnostics.test.ts +0 -291
  368. package/src/executors/bounded-exec/progress.test.ts +0 -139
  369. package/src/executors/bounded-exec/ram-temp-held.test.ts +0 -264
  370. package/src/executors/bounded-exec/ram-temp-host.test.ts +0 -635
  371. package/src/executors/bounded-exec/ram-temp.test.ts +0 -750
  372. package/src/executors/bounded-exec/run-cpu-pool.test.ts +0 -79
  373. package/src/executors/bounded-exec/runner-hook.test.ts +0 -193
  374. package/src/executors/bounded-exec/runner-lifecycle.test.ts +0 -82
  375. package/src/executors/bounded-exec/verdict.test.ts +0 -545
  376. package/src/executors/cargo-zigbuild/executor.test.ts +0 -129
  377. package/src/executors/clean-outputs/executor.test.ts +0 -20
  378. package/src/executors/typescript-emit/executor.test.ts +0 -83
  379. package/src/failure-summary.ts +0 -119
  380. package/src/generators/bounded-test-targets/generator.test.ts +0 -138
  381. package/src/generators/bun-test-tracing/generator.test.ts +0 -250
  382. package/src/generators/create-package/generator.test.ts +0 -190
  383. package/src/generators/make-public/generator.test.ts +0 -137
  384. package/src/graph-input-contract.test.ts +0 -379
  385. package/src/host-controller-wiring.test.ts +0 -31
  386. package/src/index.test.ts +0 -3321
  387. package/src/linked-snapshots.nx.test.ts +0 -254
  388. package/src/managed-files/forgejo-artifact.test.ts +0 -395
  389. package/src/managed-files/generator.test.ts +0 -243
  390. package/src/managed-files/github-host-cache-isolation.test.ts +0 -86
  391. package/src/managed-files/json-edit.test.ts +0 -216
  392. package/src/managed-files/managed-content.test.ts +0 -166
  393. package/src/managed-files/managed-format-json.test.ts +0 -308
  394. package/src/managed-files/repo-path.test.ts +0 -172
  395. package/src/managed-files/setup-devenv-free-disk.test.ts +0 -95
  396. package/src/managed-files/trace-db-artifact.test.ts +0 -7
  397. package/src/managed-files/tree-permissions.test.ts +0 -109
  398. package/src/managed-files/tree.test.ts +0 -123
  399. package/src/manifest-hash.test.ts +0 -219
  400. package/src/nextest-code-bound.test.ts +0 -161
  401. package/src/nextest-extraction.test.ts +0 -216
  402. package/src/nextest-extraction.ts +0 -371
  403. package/src/nextest-partitions.test.ts +0 -501
  404. package/src/nx-cache-bound.test.ts +0 -109
  405. package/src/nx-cache-restore-mtime.test.ts +0 -170
  406. package/src/nx-daemon-claim.test.ts +0 -279
  407. package/src/nx-daemon-idle-cpu.test.ts +0 -80
  408. package/src/nx-daemon-idle-policy.test.ts +0 -111
  409. package/src/nx-daemon-ignore-files.test.ts +0 -210
  410. package/src/nx-daemon-root-normalization.test.ts +0 -105
  411. package/src/nx-daemon-start-never.test.ts +0 -292
  412. package/src/nx-daemon-validity.test.ts +0 -1055
  413. package/src/nx-dot-input-policy.test.ts +0 -353
  414. package/src/nx-failure-life-cycles.test.ts +0 -207
  415. package/src/nx-files-archive-racy.test.ts +0 -97
  416. package/src/nx-foreign-workspace-env.test.ts +0 -329
  417. package/src/nx-incomplete-run-summary.test.ts +0 -70
  418. package/src/nx-plugin-load-errors.test.ts +0 -120
  419. package/src/nx-project-graph-errors.test.ts +0 -70
  420. package/src/nx-project-resolution-cache.test.ts +0 -176
  421. package/src/nx-raw-inputs.test.ts +0 -166
  422. package/src/nx-resident-graph-workers.test.ts +0 -112
  423. package/src/nx-run-information.test.ts +0 -118
  424. package/src/nx-skipped-task-exit.test.ts +0 -180
  425. package/src/nx-state-location.test.ts +0 -63
  426. package/src/nx-store-resolution.test.ts +0 -81
  427. package/src/nx-task-graph-cycles.test.ts +0 -148
  428. package/src/nx-task-output-lease.test.ts +0 -550
  429. package/src/nx-use-local.test.ts +0 -105
  430. package/src/owned-installation.test.ts +0 -105
  431. package/src/package-target-policy.test.ts +0 -1014
  432. package/src/producer-receipts.test.ts +0 -285
  433. package/src/release-config-policy.test.ts +0 -311
  434. package/src/runtime-input.test.ts +0 -393
  435. package/src/serial-group-wiring.test.ts +0 -326
  436. package/src/task-history-namespace.test.ts +0 -181
  437. package/src/task-results.test.ts +0 -228
  438. package/src/test-clock-policy.test.ts +0 -338
  439. package/src/test-program.test.ts +0 -303
  440. package/src/test-timings.test.ts +0 -240
  441. package/src/test-walk-policy.test.ts +0 -620
  442. package/src/testing.test.ts +0 -509
  443. package/src/typecheck-test-policy.test.ts +0 -1468
  444. package/src/typescript-resolution.test.ts +0 -76
  445. package/src/version-actions.test.ts +0 -80
  446. package/src/workspace-config-policy.test.ts +0 -737
@@ -1,17 +1,24 @@
1
1
  import { spawn } from 'node:child_process';
2
2
  import { accessSync, constants } from 'node:fs';
3
+ import { mkdtemp, rm } from 'node:fs/promises';
3
4
  import { availableParallelism, userInfo } from 'node:os';
4
5
  import { delimiter, isAbsolute, join, resolve as resolvePath } from 'node:path';
5
6
  import { createInterface } from 'node:readline';
6
- import { Duplex } from 'node:stream';
7
+ import { Duplex, Readable } from 'node:stream';
7
8
  import { workspaceDataDirectoryForWorkspace } from 'nx/src/utils/cache-directory.js';
9
+ import { DEADLINE_PATIENCE, describeMembersLeft, killProcessGroup, processGroupHasMembers, signalProcessGroup, } from '../../process-group.js';
10
+ import { earliestBound, stepBounds, UNARMED } from './bounds.js';
8
11
  import { CPU_POOL_ENV, createCpuPool, readInheritedCpuPool, takeParentCpuTokens, } from './cpu-pool.js';
9
- import { CPU_TOKENS_ENV, gatewayCpuBudget, runnerDemand, sizeRunner, } from './cpu-tokens.js';
12
+ import { GATEWAY_SOCKET } from './gateway-lease.js';
10
13
  import { captureHangDiagnostics, HANG_CAPTURE_MS } from './hang-diagnostics.js';
14
+ import { mbxLeaf, mbxPool, withMbxLeaf } from './mbx-pool.js';
11
15
  import { createNoSpaceWatch } from './no-space.js';
16
+ import { COWSHED_EXECUTABLE_ENV, cowshedSubtreeOwner, HANDLE_FD, readAdmission, SUPERVISOR_SOCKET_ENV, } from './owned-subtree.js';
12
17
  import { createRunProgress, withProgressReport } from './progress.js';
13
- import { COWSHED_EXECUTABLE_ENV, createHostCommands, describeRamTempError, RAM_TEMP_CAPACITY_BYTES, RamTempVolume, ramTempPaths, } from './ram-temp.js';
14
- import { createRunnerLifecycle, withRunnerLifecycle } from './runner-lifecycle.js';
18
+ import { createHostCommands, describeCoverageGap, describeRamTempError, describeSignalled, LEFTOVER_GRACE_MS, PRODUCTION, RAM_TEMP_CAPACITY_BYTES, RamTempVolume, ramTempPaths, } from './ram-temp.js';
19
+ import { createRunnerLifecycle, withFixtureLifecycle, withRunnerLifecycle } from './runner-lifecycle.js';
20
+ import { CPU_TOKENS_ENV, runnerDemand, sizeRunner } from './runner-sizing.js';
21
+ import { checkRuntimePin, describeRuntimePinRefusal } from './runtime-pin.js';
15
22
  import { clearTaskDirectory, describeNoSpace, JUNIT_FILE, JUNIT_REPORT_ENV, judgeOperationalFailure, judgeRun, readJunitReport, taskDirectory, withJunitReport, writeRecord, } from './verdict.js';
16
23
  const DEFAULT_KILL_AFTER_MS = 10_000;
17
24
  const REAP_AFTER_FORCE_KILL_MS = 2_000;
@@ -23,9 +30,9 @@ const EXIT_CODE_BY_SIGNAL = {
23
30
  /** The pool descriptor: only a grant may give it, so neither the target's nor the inherited environment passes it on. */
24
31
  const POOL_ENV = [CPU_TOKENS_ENV, CPU_POOL_ENV];
25
32
  /**
26
- * The variables the CPU budget sets for the command it grants: no target's environment may set them. A jobserver the
27
- * executor itself inherits, a parent make's, still reaches an ungranted command by Cargo's own rule; a granted one's
28
- * sub-pool names its own fifo over it.
33
+ * The variables a grant sets for the command it sizes: no target's environment may set them. A jobserver the executor
34
+ * itself inherits, a parent make's or its cowshed job's, still reaches an ungranted command by Cargo's own rule; a
35
+ * granted one's sub-pool names its own fifo over it.
29
36
  */
30
37
  const BUDGET_ENV = [...POOL_ENV, 'CARGO_MAKEFLAGS', 'MAKEFLAGS', 'MFLAGS'];
31
38
  /** Workspace-provided cowshed launchers, in the order a workspace that provides one names them. */
@@ -73,22 +80,36 @@ export function resolveCowshedExecutable(root, named, inherited, pathValue, cwd,
73
80
  export default function boundedExecExecutor(options, context) {
74
81
  return runDefaultBoundedExec(options, recordContext(context));
75
82
  }
76
- /** Runs with the normal process, RAM lease and CPU-budget owners, with an optional stdout sink. */
83
+ /**
84
+ * Runs with the normal process, RAM lease and CPU pool owners, with an optional stdout sink, on the user's RAM temp
85
+ * volume where there is one, through the host's own gateway: the Nx executor's runs, never a test's.
86
+ */
77
87
  export function runDefaultBoundedExec(options, context) {
88
+ // Only macOS has `hdiutil ram://`; elsewhere the command keeps the inherited TMPDIR.
89
+ return runBoundedExecWith(options, context, (cowshed) => process.platform === 'darwin'
90
+ ? new RamTempVolume(ramTempPaths(userInfo(), PRODUCTION), RAM_TEMP_CAPACITY_BYTES, createHostCommands(GATEWAY_SOCKET, undefined, undefined, cowshed), undefined, undefined, 'optional')
91
+ : null);
92
+ }
93
+ /**
94
+ * The default runner with every owner but its RAM volume, which its caller makes: `volume` makes the volume for the
95
+ * cowshed the run's disk commands go through. The Nx executor makes the user's; a host proof passes a fixture volume
96
+ * (`fixtureRamVolume`), so no test leases, creates or detaches the user's volume.
97
+ */
98
+ export function runBoundedExecWith(options, context, volume) {
78
99
  // A target that names the cowshed its disk commands run through names it for this run's own RAM volume work and
79
100
  // for its command alike. Nx resolves `{workspaceRoot}/…` in an option to a workspace-relative path, which the
80
101
  // command, starting in another directory, could not find: it gets the absolute one.
81
102
  const named = options.env?.[COWSHED_EXECUTABLE_ENV];
82
103
  const cowshed = resolveCowshedExecutable(context.root, named, process.env[COWSHED_EXECUTABLE_ENV], process.env.PATH, process.cwd());
83
104
  const run = cowshed === undefined ? options : { ...options, env: { ...options.env, [COWSHED_EXECUTABLE_ENV]: cowshed } };
84
- // Only macOS has `hdiutil ram://`; elsewhere the command keeps the inherited TMPDIR.
85
- const tempVolume = process.platform === 'darwin'
86
- ? new RamTempVolume(ramTempPaths(userInfo()), RAM_TEMP_CAPACITY_BYTES, createHostCommands(undefined, undefined, undefined, cowshed))
87
- : null;
105
+ const tempVolume = volume(cowshed);
106
+ // The command's processes are owned by the supervisor that serves this job, when one does
107
+ // (specs/cowshed/07_api.md, `JobSubtreeHandle`); with none, nothing here may end what it leaves.
108
+ const owner = process.env[SUPERVISOR_SOCKET_ENV] === undefined ? null : cowshedSubtreeOwner(cowshed ?? 'cowshed');
88
109
  // A command a parent's CPU pool started draws its tokens from that pool. The pool is named by the environment the
89
110
  // parent set for this command; a run no parent started has none.
90
111
  const inherited = readInheritedCpuPool(process.env);
91
- return runBoundedExec(run, context, createProcessTreeKiller(), tempVolume, gatewayCpuBudget(), inherited);
112
+ return runBoundedExec(run, context, createProcessTreeKiller(), tempVolume, inherited, captureHangDiagnostics, owner);
92
113
  }
93
114
  /**
94
115
  * Nx gives an executor the task's project, target and configuration, not its id; the id is the
@@ -113,17 +134,15 @@ export function recordContext(context) {
113
134
  };
114
135
  }
115
136
  /**
116
- * Run the command once its runner holds CPU tokens from `budget` (`cpu-tokens.ts`), sized to the
117
- * grant; `null`, or a budget nobody serves, runs it as asked. Both bounds start once it is
118
- * granted: a wait for the host is not the command's time.
137
+ * Run the command, sized to the tokens its parent's CPU pool (`cpu-pool.ts`) lends it; a run that inherits no pool
138
+ * runs it as asked. Both bounds start once it is granted: a wait for the pool is not the command's time.
119
139
  *
120
- * `inherited` is the parent's CPU pool (`cpu-pool.ts`) that lends this run its tokens. The run asks that pool first
121
- * and the host budget only when the pool refuses. Whatever it is granted, the run serves its own sub-pool to its
122
- * descendants, closed before the grant returns. A grant whose connection closes while the command runs (a gateway
123
- * restart, or the lending parent's exit) ends its accounting, never the command: the command runs on over budget to
124
- * its own exit, and the run says so.
140
+ * `inherited` is the parent's pool: an enclosing bounded-exec's sub-pool or the Nx run's own (`run-cpu-pool.ts`). A
141
+ * granted run serves a sub-pool of its grant to its own descendants, closed before the grant returns. A grant whose
142
+ * connection closes while the command runs (the lending parent's exit) ends its accounting, never the command: the
143
+ * command runs on over budget to its own exit, and the run says so.
125
144
  */
126
- export async function runBoundedExec(options, context, killer, tempVolume, budget, inherited = null, captureDiagnostics = captureHangDiagnostics) {
145
+ export async function runBoundedExec(options, context, killer, tempVolume, inherited = null, captureDiagnostics = captureHangDiagnostics, owner = null) {
127
146
  const cwd = resolveCwd(options.cwd, context.root);
128
147
  const directory = context.record?.directory ?? null;
129
148
  // A previous run's verdict and report go first, so neither can be read as this run's.
@@ -134,13 +153,18 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
134
153
  const timeoutMs = options.timeoutMs;
135
154
  const idleTimeoutMs = options.idleTimeoutMs;
136
155
  const killAfterMs = options.killAfterMs ?? DEFAULT_KILL_AFTER_MS;
156
+ const fixture = options.fixtureCommand !== undefined && options.fixtureTimeoutMs !== undefined
157
+ ? { command: options.fixtureCommand, timeoutMs: options.fixtureTimeoutMs }
158
+ : null;
137
159
  const outputChunks = [];
138
160
  const state = {
139
161
  settled: false,
140
162
  expiry: null,
141
163
  forceKillNeeded: false,
142
164
  poolCloseError: null,
165
+ retireError: null,
143
166
  noSpace: null,
167
+ lockWaits: [],
144
168
  };
145
169
  const appendStdout = (chunk) => {
146
170
  const text = chunk.toString();
@@ -157,9 +181,55 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
157
181
  outputChunks.push(text);
158
182
  process.stderr.write(text);
159
183
  };
184
+ // Told, never thrown, wherever the command is being ended: a diagnostic must neither replace the failure it
185
+ // accompanies nor skip the cleanup that follows (the signal handlers, the CPU grant, the RAM lease, the run's record).
186
+ const tell = (text) => {
187
+ try {
188
+ appendStderr(text);
189
+ }
190
+ catch {
191
+ // Nobody is left to tell.
192
+ }
193
+ };
194
+ // The fixture pair names one segment: a fixture with no build budget, or a budget with no fixture, is a
195
+ // target that does not say what it runs, so it does not run.
196
+ if ((options.fixtureCommand === undefined) !== (options.fixtureTimeoutMs === undefined) ||
197
+ (fixture !== null && !(fixture.timeoutMs > 0))) {
198
+ appendStderr('bounded-exec: fixtureCommand and a positive fixtureTimeoutMs must be named together; refusing to run the command\n');
199
+ if (context.record !== undefined) {
200
+ const { task, directory: recordDirectory } = context.record;
201
+ await writeRecord(recordDirectory, { task: task.id, hash: task.hash, verdict: judgeOperationalFailure(1, null) });
202
+ }
203
+ return { success: false, terminalOutput: outputChunks.join('') };
204
+ }
205
+ // A leg that runs bun runs the Bun its workspace declares, or it does not run. This comes before the RAM lease and the
206
+ // CPU tokens: a refusal that waited in its pool's queue first would say nothing for minutes.
207
+ const runtime = await checkRuntimePin({
208
+ root: context.root,
209
+ command: fixture === null ? requested : `${fixture.command} && ${requested}`,
210
+ env: { ...withoutPoolEnv(process.env), ...options.env },
211
+ cwd,
212
+ pathSource: options.env?.PATH === undefined ? 'the inherited environment' : "the target's env",
213
+ });
214
+ if (runtime !== null) {
215
+ appendStderr(describeRuntimePinRefusal(runtime, context.record?.task.id));
216
+ if (context.record !== undefined) {
217
+ const { task, directory: recordDirectory } = context.record;
218
+ await writeRecord(recordDirectory, { task: task.id, hash: task.hash, verdict: judgeOperationalFailure(1, null) });
219
+ }
220
+ return { success: false, terminalOutput: outputChunks.join('') };
221
+ }
160
222
  let lease = null;
161
- if (tempVolume !== null) {
162
- const acquired = await tempVolume.acquire();
223
+ let ordinaryTemp = null;
224
+ // Before spawn there is nothing to retire. Once started, only the owner's terminal retirement permits deletion.
225
+ let subtreeRetired = true;
226
+ // A lease is removed when its command ends, so it is taken only where whatever the command leaves
227
+ // in it can be ended by its owner first: the supervisor's subtree retirement, never a census.
228
+ if (tempVolume !== null && owner === null) {
229
+ appendStderr(`RAM temp volume not leased: no cowshed supervisor owns this command's processes (${SUPERVISOR_SOCKET_ENV} is unset); TMPDIR stays ${process.env.TMPDIR ?? '(unset)'}\n`);
230
+ }
231
+ else if (tempVolume !== null && owner !== null) {
232
+ const acquired = await tempVolume.acquire(owner);
163
233
  if (!acquired.ok) {
164
234
  const message = `${describeRamTempError(acquired.error)}\n`;
165
235
  process.stderr.write(message);
@@ -171,78 +241,149 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
171
241
  appendStderr(`RAM temp volume: dead lease ${held}\n`);
172
242
  }
173
243
  for (const reaped of acquired.value.reaped) {
174
- appendStderr(`RAM temp volume: stopped ${reaped}, which a dead task left working in its lease\n`);
244
+ appendStderr(`RAM temp volume: the supervisor stopped ${reaped}, which a dead task's command left running\n`);
245
+ }
246
+ for (const gap of acquired.value.gaps) {
247
+ appendStderr(`RAM temp volume: the supervisor's view of a dead task's command has a gap: ${gap}\n`);
248
+ }
249
+ for (const retained of acquired.value.retained) {
250
+ appendStderr(`RAM temp volume: dead lease ${retained}\n`);
251
+ }
252
+ }
253
+ else if (acquired.value.kind === 'unavailable') {
254
+ const unavailable = `RAM temp volume unavailable (${acquired.value.reason}: ${acquired.value.detail})`;
255
+ if (options.env?.TMPDIR !== undefined) {
256
+ appendStderr(`${unavailable}; using target-provided TMPDIR ${options.env.TMPDIR} (medium not inferred)\n`);
257
+ }
258
+ else {
259
+ try {
260
+ ordinaryTemp = await mkdtemp(join(tempVolume.ordinaryParent, `smoo-bounded-exec-${process.pid}-`));
261
+ }
262
+ catch (error) {
263
+ appendStderr(`${unavailable}; could not create ordinary scratch under ${tempVolume.ordinaryParent}: ${error instanceof Error ? error.message : String(error)}\n`);
264
+ return { success: false, terminalOutput: outputChunks.join('') };
265
+ }
266
+ appendStderr(`${unavailable}; using owned ordinary TMPDIR ${ordinaryTemp}\n`);
175
267
  }
176
268
  }
177
269
  else {
178
270
  appendStderr(`RAM temp volume unavailable in this sandbox (${acquired.value.detail}); TMPDIR stays ${process.env.TMPDIR ?? '(unset)'}\n`);
179
271
  }
180
272
  }
181
- const releaseLease = async () => {
182
- if (lease === null) {
273
+ // The supervisor's answer to the command's own admission, once it is spawned as an owned subtree root, and once
274
+ // any lease has recorded the handle for whoever settles that lease should this task die first.
275
+ let admission = null;
276
+ const retireSubtree = async () => {
277
+ if (admission === null || owner === null) {
183
278
  return;
184
279
  }
185
- const held = lease;
186
- lease = null;
187
- // What the command left working in its lease is a leak, whichever way the command ended; it
188
- // is stopped before the lease goes, so the volume can detach, and said, so its owner fixes it.
189
- const reaped = await tempVolume?.reap(held);
190
- if (reaped?.ok) {
191
- for (const leftover of reaped.value) {
192
- appendStderr(`RAM temp volume: stopped ${leftover}, which the command left working in its lease\n`);
193
- }
194
- }
195
- else if (reaped) {
196
- appendStderr(`${describeRamTempError(reaped.error)}\n`);
280
+ const answer = await admission;
281
+ admission = null;
282
+ if (answer.kind !== 'admitted') {
283
+ appendStderr(`cowshed: the command ran without an owned subtree (${answer.kind === 'refused' ? `supervisor: ${answer.reason}` : answer.detail}); nothing retires what it left running\n`);
284
+ return;
197
285
  }
198
- const released = await tempVolume?.release(held);
199
- if (released && !released.ok) {
200
- appendStderr(`${describeRamTempError(released.error)}\n`);
286
+ // What the command left running is a leak, whichever way the command ended: its supervisor ends exactly the
287
+ // processes its tree records under the command, before any lease goes, and each is said so its owner fixes it.
288
+ const retired = await owner.retire(answer.handle, LEFTOVER_GRACE_MS);
289
+ switch (retired.kind) {
290
+ case 'retired':
291
+ subtreeRetired = true;
292
+ for (const entry of retired.signalled) {
293
+ appendStderr(`cowshed: the supervisor stopped ${describeSignalled(entry)}, which the command left running\n`);
294
+ }
295
+ if (retired.signalledIncomplete) {
296
+ appendStderr("cowshed: the supervisor may have stopped processes of the command's that it could not name\n");
297
+ }
298
+ for (const gap of retired.gaps) {
299
+ appendStderr(`cowshed: the supervisor's view of the command's processes has a gap: ${describeCoverageGap(gap)}\n`);
300
+ }
301
+ return;
302
+ case 'refused':
303
+ appendStderr(`cowshed: the supervisor refused to retire the command's processes (${retired.reason})\n`);
304
+ return;
305
+ case 'failed':
306
+ state.retireError = retired.detail;
307
+ appendStderr(`cowshed: could not retire the command's processes: ${retired.detail}\n`);
308
+ return;
201
309
  }
202
310
  };
203
- // The wait is one span line, as cowshed's own spans read. On a host without a gateway the Nx run serves its own
204
- // pool (`run-cpu-pool.ts`), taken below as a parent's; a command no pool and no gateway answers runs exactly as
311
+ // Failed scratch remains evidence: RAM leases are archived; ordinary scratch is retained at its reported path.
312
+ let tempReleased = null;
313
+ const releaseLease = (keep) => {
314
+ tempReleased ??= (async () => {
315
+ await retireSubtree();
316
+ if (ordinaryTemp !== null) {
317
+ const held = ordinaryTemp;
318
+ ordinaryTemp = null;
319
+ if (keep || state.retireError !== null || !subtreeRetired) {
320
+ appendStderr(`Ordinary temp directory retained at ${held}: ${!subtreeRetired ? 'the command subtree was not retired' : 'the run did not pass'}\n`);
321
+ }
322
+ else {
323
+ try {
324
+ await rm(held, { recursive: true, force: true });
325
+ }
326
+ catch (error) {
327
+ appendStderr(`Ordinary temp directory retained at ${held}: cleanup failed: ${error instanceof Error ? error.message : String(error)}\n`);
328
+ }
329
+ }
330
+ }
331
+ if (lease === null || owner === null) {
332
+ return;
333
+ }
334
+ const held = lease;
335
+ lease = null;
336
+ const released = await tempVolume?.release(held, keep || state.retireError !== null, owner);
337
+ if (released?.ok) {
338
+ for (const retained of released.value) {
339
+ appendStderr(`RAM temp volume: lease ${retained}\n`);
340
+ }
341
+ }
342
+ else if (released) {
343
+ appendStderr(`${describeRamTempError(released.error)}\n`);
344
+ }
345
+ })();
346
+ return tempReleased;
347
+ };
348
+ // The wait is one span line, as cowshed's own spans read. Every Nx run that inherits no pool serves its own
349
+ // (`run-cpu-pool.ts`), so a task takes its tokens from a parent's pool; a command that inherits none runs exactly as
205
350
  // asked and nothing is said.
206
351
  const cores = availableParallelism();
207
352
  const demand = runnerDemand(requested, options.parallelism, cores, process.env, options.env);
208
353
  const asked = Date.now();
209
- // A run a parent's CPU pool serves takes its tokens there. Only an absent pool leaves the host budget to ask.
210
- // A pool that answered and refused is a live budget of its own -- a lent grant, or a gate's or a resident's
211
- // relay to the host -- so asking the host beside it would count the run twice; the run fails instead.
354
+ // A pool that answered and refused is a live budget, and running beside it would count the run twice: the run fails
355
+ // instead. A pool whose endpoint is gone (its holder exited) lends nothing, and the run goes on as asked.
212
356
  let grant = null;
213
357
  let parentRefusal = null;
358
+ let parentWait;
214
359
  if (inherited !== null) {
215
360
  const fromParent = await takeParentCpuTokens(inherited, demand.want);
216
361
  if (fromParent.granted) {
217
362
  grant = fromParent;
218
363
  }
219
364
  else if (fromParent.cause === 'absent') {
220
- appendStderr(`cowshed: cpu-tokens parent pool refused: ${fromParent.reason}; asking the host budget\n`);
365
+ appendStderr(`cowshed: cpu-pool parent pool absent: ${fromParent.reason}; running the command as asked\n`);
221
366
  }
222
367
  else {
223
368
  parentRefusal = fromParent.reason;
224
- }
225
- }
226
- let unbudgeted = null;
227
- if (grant === null && parentRefusal === null && budget !== null) {
228
- const fromHost = await budget.take(demand.want, context.root, requested);
229
- if (fromHost.granted) {
230
- grant = fromHost;
231
- }
232
- else if (fromHost.cause !== 'absent') {
233
- unbudgeted = fromHost.reason;
369
+ parentWait = fromParent.wait;
234
370
  }
235
371
  }
236
372
  const waitedMs = Date.now() - asked;
237
373
  let cpu;
238
374
  let sized = { command: requested, env: {} };
375
+ // With mbx the leaf's admission is mbx's one pool, not a parent's tokens: its runner is sized to the job's count,
376
+ // and a non-Cargo leaf holds that much of the pool for its run (specs/cowshed/05_gateway.md, "Parallelism inside a
377
+ // job"). No grant and no sub-pool beside it.
378
+ const mbx = mbxPool(process.env);
379
+ const leaf = mbx === null ? null : mbxLeaf(mbx, demand.kind, demand.want);
239
380
  if (grant !== null) {
240
381
  sized = sizeRunner(demand, requested, grant.tokens);
241
- cpu = { tokens: grant.tokens, want: demand.want, waitedMs, ...(grant.inherited ? { inherited: true } : {}) };
242
- appendStderr(`cowshed: cpu-tokens wait done elapsed=${waitedMs}ms status=ok ${grant.inherited ? 'inherited=true ' : ''}tokens=${grant.tokens}/${demand.want} runner=${demand.kind}\n`);
382
+ cpu = { tokens: grant.tokens, want: demand.want, waitedMs };
383
+ appendStderr(`cowshed: cpu-pool wait done elapsed=${waitedMs}ms status=ok tokens=${grant.tokens}/${demand.want} runner=${demand.kind}\n`);
243
384
  }
244
- else if (unbudgeted !== null) {
245
- appendStderr(`cowshed: cpu-tokens wait done elapsed=${waitedMs}ms status=err runner=${demand.kind} runs unbudgeted: ${unbudgeted}\n`);
385
+ else if (leaf !== null) {
386
+ sized = sizeRunner(demand, requested, demand.want);
246
387
  }
247
388
  // The run's sub-pool and grant are given back exactly once, however the run ends. The sub-pool closes first, so no
248
389
  // descendant still draws on tokens the grant has returned.
@@ -264,9 +405,9 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
264
405
  return cpuReleased;
265
406
  };
266
407
  // Records the run's verdict when it has a task to record as. An `expiry` is set only when a bound ended the run.
267
- // An `operational` run failed on its CPU budget (a sub-pool that did not close): an ordinary failure, whatever
268
- // bound also fired, so it is never judged as a bound.
269
- const recordRun = async (exitCode, expiry, elapsedMs, operational, noSpace) => {
408
+ // An `operational` run failed on its CPU pool (a sub-pool that did not close): an ordinary failure, whatever bound also
409
+ // fired, so it is never judged as a bound.
410
+ const recordRun = async (exitCode, expiry, elapsedMs, operational, noSpace, lockWaits) => {
270
411
  if (context.record === undefined) {
271
412
  return;
272
413
  }
@@ -274,44 +415,34 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
274
415
  const report = await readJunitReport(recordDirectory);
275
416
  const verdict = operational
276
417
  ? judgeOperationalFailure(exitCode, report)
277
- : judgeRun({ exitCode, elapsedMs, expiry, report, noSpace });
418
+ : judgeRun({ exitCode, elapsedMs, expiry, report, noSpace, lockWaits });
278
419
  if (verdict.outcome === 'bound' && verdict.bound === 'test') {
279
420
  appendStderr(`Every failing test failed only on its per-test timeout: ${verdict.tests.join(', ')}\n`);
280
421
  }
281
- await writeRecord(recordDirectory, { task: task.id, hash: task.hash, verdict, ...(cpu ? { cpu } : {}) });
422
+ await writeRecord(recordDirectory, {
423
+ task: task.id,
424
+ hash: task.hash,
425
+ verdict,
426
+ ...(cpu ? { cpu } : {}),
427
+ ...(parentWait ? { cpuWait: parentWait } : {}),
428
+ });
282
429
  };
283
430
  // A run refused before its command starts says why, gives back its grant and RAM, and records the failure.
284
431
  const refuse = async (message) => {
285
432
  appendStderr(message);
286
433
  await releaseCpu();
287
- await releaseLease();
288
- await recordRun(1, null, Date.now() - asked, false, null);
434
+ // The command never started, so its lease holds nothing of its own to keep.
435
+ await releaseLease(false);
436
+ await recordRun(1, null, Date.now() - asked, false, null, []);
289
437
  return { success: false, terminalOutput: outputChunks.join('') };
290
438
  };
291
439
  if (parentRefusal !== null) {
292
- return refuse(`cowshed: cpu-tokens parent pool refused: ${parentRefusal}; refusing to run the command without asking the host budget\n`);
293
- }
294
- // Descendant work queued for host CPU tokens, as its pool hears it (`host-wait`): while any is held the idle clock
295
- // is stopped, because a queued job is not an idle one (specs/cowshed/05_gateway.md, "The bounds start at the
296
- // grant"). The clock it stops is made with the command below; a wait held before then is counted and honoured there.
297
- let heldWaits = 0;
298
- let idleClock = null;
299
- const hostWaits = {
300
- begin: () => {
301
- heldWaits += 1;
302
- if (heldWaits === 1)
303
- idleClock?.stop();
304
- },
305
- end: () => {
306
- heldWaits -= 1;
307
- if (heldWaits === 0)
308
- idleClock?.restart();
309
- },
310
- };
440
+ return refuse(`cowshed: cpu-pool parent pool refused: ${parentRefusal}; refusing to run the command\n`);
441
+ }
311
442
  // A granted run serves a sub-pool of its own grant, so its descendants draw from that and never from above it.
312
- // A pool that cannot be set up refuses the run: no command runs unbudgeted.
443
+ // A pool that cannot be set up refuses the run: no granted command runs beside its grant.
313
444
  if (grant !== null) {
314
- const created = await createCpuPool(grant.tokens, grant.closed, hostWaits);
445
+ const created = await createCpuPool(grant.tokens, grant.closed);
315
446
  if (!created.ok) {
316
447
  return refuse(`cowshed: cpu pool unavailable: ${created.error}; refusing to run the command\n`);
317
448
  }
@@ -330,26 +461,62 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
330
461
  ...sized.env,
331
462
  ...(pool?.environment ?? {}),
332
463
  };
333
- const hooked = process.platform === 'win32' ? null : withRunnerLifecycle(command);
464
+ // A fixture rides as segment 0 of the lifecycle. Without the channel (Windows) it is plain shell text ahead
465
+ // of the command: the progress bound runs during it and its own ceiling does not.
466
+ const hooked = process.platform === 'win32'
467
+ ? null
468
+ : fixture === null
469
+ ? withRunnerLifecycle(command)
470
+ : withFixtureLifecycle(fixture.command, command);
334
471
  const lifecycle = hooked === null ? null : createRunnerLifecycle(hooked.runners);
335
472
  let lifecycleFailure = null;
336
473
  const isTesting = () => lifecycleFailure !== null || (lifecycle?.isTesting() ?? true);
337
474
  const startedAt = Date.now();
475
+ // With a supervisor, `cowshed subtree-root` asks it for a handle on itself, writes the answer to the handle
476
+ // descriptor, and execs `/bin/sh -c command` in the same process: the command's pid, group and life are unchanged.
477
+ const unadmitted = hooked?.command ?? (fixture === null ? command : `${fixture.command} && ${command}`);
478
+ const shellCommand = leaf === null ? unadmitted : withMbxLeaf(unadmitted, leaf);
479
+ const root = owner?.rootArgv(shellCommand);
338
480
  let child;
339
481
  try {
340
- child = spawn(hooked?.command ?? command, [], {
341
- cwd,
342
- env: mergeEnv(lease, env),
343
- shell: true,
344
- stdio: hooked === null ? 'pipe' : ['pipe', 'pipe', 'pipe', 'pipe'],
345
- detached: process.platform !== 'win32',
346
- windowsHide: true,
347
- });
482
+ // fd 3 is the runner lifecycle's duplex socket when the command is hooked; the subtree admission takes fd 4, which
483
+ // `cowshed subtree-root` closes before it execs, so fd 3 reaches the shell unchanged.
484
+ child =
485
+ root === undefined
486
+ ? spawn(shellCommand, [], {
487
+ cwd,
488
+ env: mergeEnv(lease?.directory ?? ordinaryTemp, env),
489
+ shell: true,
490
+ stdio: hooked === null ? 'pipe' : ['pipe', 'pipe', 'pipe', 'pipe'],
491
+ detached: process.platform !== 'win32',
492
+ windowsHide: true,
493
+ })
494
+ : spawn(root.file, root.args, {
495
+ cwd,
496
+ env: mergeEnv(lease?.directory ?? ordinaryTemp, env),
497
+ stdio: ['pipe', 'pipe', 'pipe', hooked === null ? 'ignore' : 'pipe', 'pipe'],
498
+ detached: process.platform !== 'win32',
499
+ windowsHide: true,
500
+ });
348
501
  }
349
502
  catch (error) {
350
503
  // Nothing started, so nothing runs under the grant: it goes back here, with the RAM lease.
351
504
  return refuse(`cowshed: could not start the command: ${error instanceof Error ? error.message : String(error)}\n`);
352
505
  }
506
+ subtreeRetired = false;
507
+ if (root !== undefined) {
508
+ const channel = child.stdio[HANDLE_FD];
509
+ const leased = lease;
510
+ admission = readAdmission(channel instanceof Readable ? channel : null).then(async (answer) => {
511
+ if (answer.kind === 'admitted' && leased !== null && tempVolume !== null) {
512
+ const recorded = await tempVolume.record(leased, answer.handle);
513
+ if (!recorded.ok) {
514
+ appendStderr(`${describeRamTempError(recorded.error)}\n`);
515
+ }
516
+ }
517
+ return answer;
518
+ });
519
+ }
353
520
  // Progress is a finished test, not output: a runner's SLOW warnings and a hung test's heartbeat keep printing.
354
521
  const progress = createRunProgress(command);
355
522
  // A write that found no space is said once, in the output, by whatever made it; the volume is read when it is said,
@@ -359,24 +526,22 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
359
526
  const held = lease;
360
527
  return { line, volume: held === null ? null : ((await tempVolume?.space(held)) ?? null) };
361
528
  };
362
- let idleTimer;
529
+ // Which bound is armed, and when it expires, is bounds.ts's decision; one timer waits for the earliest of them.
530
+ let bounds = UNARMED;
531
+ let boundTimer;
532
+ // The fixture's own output is compilation, not the command's progress: it is not scanned for finished tests.
533
+ let fixtureDone = fixture === null || lifecycle === null;
534
+ // Whether a segment past the fixture ever announced: without one, the shell's exit is the fixture's.
535
+ let mainStarted = fixture === null || lifecycle === null;
363
536
  let captureTimer;
364
537
  let idleCapture;
365
538
  const cancelCapture = () => {
366
539
  clearTimeout(captureTimer);
367
540
  idleCapture?.controller.abort();
368
541
  };
369
- // What is left of the current no-progress interval. A finished test sets it back to `idleTimeoutMs`; a held host
370
- // wait keeps it unspent: the clock is stopped while descendant work is queued for CPU tokens, and runs on from where
371
- // it stopped once the last wait ends. `queuedMs` is the queued time this interval has not counted, for the report.
372
- let idleLeftMs = idleTimeoutMs ?? 0;
373
- let queuedMs = 0;
374
- let queuedSinceMs = null;
375
- const scheduleIdle = (limitMs, leftMs) => {
376
- clearTimeout(idleTimer);
377
- cancelCapture();
542
+ const scheduleCapture = (expiresAtMs) => {
378
543
  const capture = {
379
- expiresAtMs: Date.now() + leftMs,
544
+ expiresAtMs,
380
545
  controller: new AbortController(),
381
546
  result: { kind: 'pending', startedAtMs: null },
382
547
  };
@@ -409,65 +574,42 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
409
574
  };
410
575
  }
411
576
  });
412
- }, Math.max(0, leftMs - HANG_CAPTURE_MS));
413
- idleTimer = setTimeout(() => expire({ kind: 'idle', limitMs, tests: progress.inFlight() }, queuedMs), leftMs);
577
+ }, Math.max(0, expiresAtMs - Date.now() - HANG_CAPTURE_MS));
414
578
  };
415
- const armIdleTimer = () => {
416
- if (idleTimeoutMs === undefined || state.settled || state.expiry !== null || !isTesting()) {
417
- return;
418
- }
419
- idleLeftMs = idleTimeoutMs;
420
- queuedMs = 0;
421
- if (queuedSinceMs !== null) {
422
- queuedSinceMs = Date.now();
579
+ const expiredAs = (kind) => kind === 'idle'
580
+ ? { kind, limitMs: idleTimeoutMs ?? 0, tests: progress.inFlight() }
581
+ : kind === 'fixture'
582
+ ? { kind, limitMs: fixture?.timeoutMs ?? 0 }
583
+ : { kind, limitMs: timeoutMs ?? 0 };
584
+ const stepTo = (event) => {
585
+ if (state.settled || state.expiry !== null)
423
586
  return;
424
- }
425
- scheduleIdle(idleTimeoutMs, idleLeftMs);
426
- };
427
- idleClock = {
428
- stop: () => {
429
- if (idleTimeoutMs === undefined || state.settled || state.expiry !== null || queuedSinceMs !== null) {
430
- return;
431
- }
432
- const now = Date.now();
433
- queuedSinceMs = now;
434
- if (idleCapture !== undefined) {
435
- idleLeftMs = Math.max(0, idleCapture.expiresAtMs - now);
436
- }
437
- clearTimeout(idleTimer);
587
+ const idleBefore = bounds.idle;
588
+ bounds = stepBounds(bounds, {
589
+ ...(idleTimeoutMs === undefined ? {} : { idleMs: idleTimeoutMs }),
590
+ ...(fixture === null || lifecycle === null ? {} : { fixtureMs: fixture.timeoutMs }),
591
+ ...(timeoutMs === undefined ? {} : { totalMs: timeoutMs }),
592
+ }, event);
593
+ if (bounds.idle !== idleBefore) {
438
594
  cancelCapture();
439
595
  idleCapture = undefined;
440
- appendStderr(`bounded-exec: idle clock stopped: a descendant's work is queued for host CPU tokens; ${idleLeftMs}ms of idleTimeoutMs=${idleTimeoutMs} left\n`);
441
- },
442
- restart: () => {
443
- if (idleTimeoutMs === undefined || state.settled || state.expiry !== null || queuedSinceMs === null) {
444
- return;
445
- }
446
- const waitedMs = Date.now() - queuedSinceMs;
447
- queuedMs += waitedMs;
448
- queuedSinceMs = null;
449
- if (!isTesting()) {
450
- appendStderr(`bounded-exec: host CPU wait ended after ${waitedMs}ms; no test runner is active\n`);
451
- return;
452
- }
453
- appendStderr(`bounded-exec: idle clock restarted: no descendant's work is queued for host CPU tokens after ${waitedMs}ms; ${idleLeftMs}ms of idleTimeoutMs=${idleTimeoutMs} left\n`);
454
- scheduleIdle(idleTimeoutMs, idleLeftMs);
455
- },
596
+ if (bounds.idle !== null)
597
+ scheduleCapture(bounds.idle);
598
+ }
599
+ clearTimeout(boundTimer);
600
+ const next = earliestBound(bounds);
601
+ if (next !== null)
602
+ boundTimer = setTimeout(() => expire(expiredAs(next.kind)), Math.max(0, next.at - Date.now()));
456
603
  };
457
604
  // Only the CHILD's own output is read for progress. The diagnostics below go
458
605
  // through appendStderr as well, and re-arming on those would let a teardown
459
606
  // message extend the very bound that just fired.
460
607
  const updateIdle = (finished, wasTesting) => {
461
- if (!isTesting()) {
462
- clearTimeout(idleTimer);
463
- cancelCapture();
464
- idleCapture = undefined;
465
- }
466
- else if (finished || !wasTesting) {
467
- armIdleTimer();
468
- }
608
+ stepTo({ kind: 'progress', at: Date.now(), testing: isTesting(), wasTesting, finished });
469
609
  };
470
610
  const scanProgress = (stream, text) => {
611
+ if (!fixtureDone)
612
+ return;
471
613
  updateIdle(progress.scan(stream, text), isTesting());
472
614
  };
473
615
  if (lifecycle !== null) {
@@ -481,7 +623,8 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
481
623
  appendStderr(`bounded-exec: runner lifecycle failed: ${detail}\n`);
482
624
  updateIdle(false, wasTesting);
483
625
  };
484
- createInterface({ input: channel }).on('line', (line) => {
626
+ const lines = createInterface({ input: channel });
627
+ lines.on('line', (line) => {
485
628
  const wasTesting = isTesting();
486
629
  const event = lifecycle.accept(line);
487
630
  if (event === null) {
@@ -489,9 +632,18 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
489
632
  return;
490
633
  }
491
634
  updateIdle(false, wasTesting);
635
+ if (event.id === 0 && event.phase === 'terminal' && fixture !== null) {
636
+ fixtureDone = true;
637
+ stepTo({ kind: 'fixture-done' });
638
+ }
639
+ if (event.id > 0)
640
+ mainStarted = true;
492
641
  // Tool startup and native lifecycle-bearing stdout wait for their phase acknowledgement.
493
642
  channel.write(`${JSON.stringify(event)}\n`);
494
643
  });
644
+ // readline re-emits its input's errors on the interface, where one with no listener is thrown: a
645
+ // runner killed with unread lifecycle bytes resets the channel (ECONNRESET on Linux).
646
+ lines.on('error', (error) => failLifecycle(error.message));
495
647
  channel.on('error', (error) => failLifecycle(error.message));
496
648
  channel.on('end', () => {
497
649
  if (!lifecycle.finished())
@@ -514,22 +666,30 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
514
666
  state.noSpace = sight(said);
515
667
  appendStderr(text);
516
668
  });
517
- const killChildTree = async (force) => {
669
+ // The graceful pass: SIGTERM to the whole group. A signal that fails is told, not thrown: every caller has more to do
670
+ // after it, `terminate` the force-kill and the signal handler its releases and the signal it gives back.
671
+ const killChildTree = async () => {
518
672
  const pid = child.pid;
519
673
  if (!pid) {
520
674
  return;
521
675
  }
522
- await killer.kill(pid, force ? 'SIGKILL' : 'SIGTERM');
676
+ try {
677
+ await killer.kill(pid, 'SIGTERM');
678
+ }
679
+ catch (error) {
680
+ const reason = error instanceof Error ? error.message : String(error);
681
+ tell(`Terminating process group ${pid} failed: ${reason}\n`);
682
+ }
523
683
  };
524
684
  const onProcessExit = () => {
525
- void killChildTree(false);
685
+ void killChildTree();
526
686
  };
527
687
  const onTerminationSignal = (signal) => {
528
688
  removeSignalHandlers();
529
689
  cancelCapture();
530
- void killChildTree(false)
690
+ void killChildTree()
531
691
  .finally(releaseCpu)
532
- .finally(releaseLease)
692
+ .finally(() => releaseLease(true))
533
693
  .finally(() => process.kill(process.pid, signal));
534
694
  };
535
695
  const onSigint = () => onTerminationSignal('SIGINT');
@@ -565,22 +725,23 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
565
725
  // the reader unable to tell a wedged toolchain from a loaded machine — the
566
726
  // two have opposite fixes.
567
727
  let termination = Promise.resolve();
568
- // `queuedMs` is the time the idle interval did not count because descendant work was queued for host CPU tokens.
569
- const expire = (bound, queuedMs = 0) => {
728
+ const expire = (bound) => {
570
729
  if (state.settled || state.expiry !== null) {
571
730
  return;
572
731
  }
573
732
  state.expiry = bound;
574
- clearTimeout(totalTimer);
575
- clearTimeout(idleTimer);
733
+ clearTimeout(boundTimer);
576
734
  const elapsedMs = Date.now() - startedAt;
577
735
  appendStderr(bound.kind === 'idle'
578
- ? `\nCommand made no progress: no test finished for ${bound.limitMs}ms (idleTimeoutMs=${bound.limitMs})${queuedMs > 0 ? `, not counting ${queuedMs}ms queued for host CPU tokens,` : ''} after ${elapsedMs}ms of runtime; ${bound.tests.length > 0
736
+ ? `\nCommand made no progress: no test finished for ${bound.limitMs}ms (idleTimeoutMs=${bound.limitMs}) after ${elapsedMs}ms of runtime; ${bound.tests.length > 0
579
737
  ? `still running: ${bound.tests.join(', ')}`
580
738
  : 'the runner named no test still running'} (cwd=${cwd}): ${command}\n`
581
- : `\nCommand timed out after ${elapsedMs}ms (timeoutMs=${bound.limitMs}, cwd=${cwd}): ${command}\n`);
739
+ : bound.kind === 'fixture'
740
+ ? `\nFixture command timed out after ${elapsedMs}ms (fixtureTimeoutMs=${bound.limitMs}, cwd=${cwd}): ${fixture?.command ?? command}\n`
741
+ : `\nCommand timed out after ${elapsedMs}ms (timeoutMs=${bound.limitMs}, cwd=${cwd}): ${command}\n`);
582
742
  if (bound.kind === 'idle') {
583
743
  const observed = idleCapture?.result;
744
+ state.lockWaits = observed?.kind === 'captured' ? observed.snapshot.lockWaits : [];
584
745
  appendStderr(`Hang diagnostics: runner pid=${child.pid ?? 'unavailable'} pgid=${process.platform === 'win32' ? 'unavailable' : (child.pid ?? 'unavailable')} expiryAtMs=${Date.now()} scheduledExpiryAtMs=${idleCapture?.expiresAtMs ?? 'unavailable'}\n`);
585
746
  if (observed?.kind === 'captured') {
586
747
  appendStderr(`Hang diagnostics: outcome=${observed.snapshot.outcome} capturedAtMs=${observed.snapshot.capturedAtMs} actualExpiryOffsetMs=${Date.now() - observed.snapshot.capturedAtMs}\n${observed.snapshot.text}`);
@@ -594,55 +755,83 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
594
755
  cancelCapture();
595
756
  termination = terminate();
596
757
  };
758
+ // Whether the group still lists members. A probe that fails proves nothing about the group, so it is told, and the
759
+ // group is taken as listed: a force-kill that was not needed costs nothing, and one that was is not skipped.
760
+ const groupListed = (pid) => {
761
+ try {
762
+ return processGroupHasMembers(pid);
763
+ }
764
+ catch (error) {
765
+ const reason = error instanceof Error ? error.message : String(error);
766
+ tell(`Probing process group ${pid} for members failed: ${reason}; taking it as still listing members\n`);
767
+ return true;
768
+ }
769
+ };
597
770
  // A timed-out run ends with its whole process group, not with its leader.
598
771
  // The graceful signal reaches every member that exists when it is sent, but
599
772
  // a member that ignores SIGTERM, or one forked while the signal goes out
600
773
  // (macOS does not hold a fork back for a group signal in flight), outlives
601
774
  // a leader that exits on it. The grace ends when the leader exits or
602
775
  // killAfterMs passes, whichever comes first; whatever is left then is killed.
776
+ // SIGKILL is no different: a member forked while it goes out survives it too,
777
+ // so the force-kill is repeated while the group lists members. What the repeats
778
+ // cannot end is said, and a failure of the probe or of a signal is shown.
779
+ const forceKill = async (pid) => {
780
+ try {
781
+ if (await killProcessGroup(pid, DEADLINE_PATIENCE, (group) => killer.kill(group, 'SIGKILL'))) {
782
+ tell(`${describeMembersLeft(pid)}\n`);
783
+ }
784
+ }
785
+ catch (error) {
786
+ const reason = error instanceof Error ? error.message : String(error);
787
+ tell(`Force-kill of process group ${pid} failed: ${reason}\n`);
788
+ }
789
+ };
603
790
  const terminate = async () => {
604
- await ignoreKillError(killChildTree(false));
791
+ await killChildTree();
605
792
  if (!state.settled && killAfterMs > 0) {
606
793
  await settledWithin(exitPromise, killAfterMs);
607
794
  }
608
795
  if (state.settled) {
609
- if (child.pid !== undefined && processGroupHasMembers(child.pid)) {
796
+ if (child.pid !== undefined && groupListed(child.pid)) {
610
797
  state.forceKillNeeded = true;
611
798
  appendStderr(`Force-killing what the ${state.expiry === null ? 'command' : 'timed-out command'} left in its process group: ${command}\n`);
612
- await ignoreKillError(killChildTree(true));
799
+ await forceKill(child.pid);
613
800
  }
614
801
  return;
615
802
  }
616
803
  state.forceKillNeeded = true;
617
804
  appendStderr(`Force-killing ${state.expiry === null ? 'command' : 'timed out command'} after killAfterMs=${killAfterMs}: ${command}\n`);
618
- await ignoreKillError(killChildTree(true));
619
- await settledWithin(exitPromise, REAP_AFTER_FORCE_KILL_MS);
805
+ if (child.pid !== undefined)
806
+ await forceKill(child.pid);
807
+ // The repeats wait for the group to empty, the leader with it. Windows has no process groups, so there they have
808
+ // nothing to wait on once the one signal is sent, and the leader's exit is awaited as it always was.
809
+ if (process.platform === 'win32' || child.pid === undefined) {
810
+ await settledWithin(exitPromise, REAP_AFTER_FORCE_KILL_MS);
811
+ }
620
812
  if (!state.settled) {
621
813
  resolveExit({ code: 1, signal: 'SIGKILL' });
622
814
  }
623
815
  };
624
- const totalTimer = timeoutMs === undefined ? undefined : setTimeout(() => expire({ kind: 'total', limitMs: timeoutMs }), timeoutMs);
625
- armIdleTimer();
626
- // A wait its pool heard before the command started stops the clock it was not yet there to stop.
627
- if (heldWaits > 0)
628
- idleClock.stop();
629
- // A grant's connection is a ledger entry, not the command's owner (05_gateway.md "Host CPU budget": a gateway
630
- // restart forgets every grant, and what runs under it runs on over budget until it finishes). Its close, while the
631
- // command runs, is said and changes nothing else; only a bound or a signal stops the command. Our own release after
632
- // the command settled is not a loss; `cpuReleased` marks it.
816
+ // The fixture's ceiling lives and dies with its segment: the segment's terminal clears it, so it can never
817
+ // fire on a command the fixture already handed over to.
818
+ stepTo({ kind: 'spawn', at: Date.now(), testing: isTesting(), fixture: fixture !== null && lifecycle !== null });
819
+ // A grant's connection is a ledger entry, not the command's owner: the lending parent's exit ends it, and what runs
820
+ // under it runs on over budget until it finishes. Its close, while the command runs, is said and changes nothing
821
+ // else; only a bound or a signal stops the command. Our own release after the command settled is not a loss;
822
+ // `cpuReleased` marks it.
633
823
  if (grant !== null) {
634
824
  const held = grant;
635
825
  void held.closed.then(() => {
636
826
  if (state.settled || cpuReleased !== null) {
637
827
  return;
638
828
  }
639
- appendStderr(`cowshed: cpu-tokens grant ended: the ${held.inherited ? 'parent pool' : 'gateway'} closed its ${held.tokens}-token grant while the command ran; it runs on over budget until it ends: ${command}\n`);
829
+ appendStderr(`cowshed: cpu-pool grant ended: the parent pool closed its ${held.tokens}-token grant while the command ran; it runs on over budget until it ends: ${command}\n`);
640
830
  });
641
831
  }
642
832
  const { code: exitCode, signal: exitSignal } = await exitPromise;
643
833
  await termination;
644
- clearTimeout(totalTimer);
645
- clearTimeout(idleTimer);
834
+ clearTimeout(boundTimer);
646
835
  cancelCapture();
647
836
  removeSignalHandlers();
648
837
  // The runner is gone. Its sub-pool closes first, so no descendant still draws on tokens the grant returns;
@@ -650,7 +839,9 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
650
839
  await releaseCpu();
651
840
  const code = exitCode ?? signalToExitCode(exitSignal);
652
841
  if (code !== 0 && state.expiry === null) {
653
- appendStderr(`Command exited with status ${code}: ${command}\n`);
842
+ appendStderr(fixture !== null && !mainStarted
843
+ ? `Fixture command exited with status ${code}: ${fixture.command}\n`
844
+ : `Command exited with status ${code}: ${command}\n`);
654
845
  }
655
846
  if (state.expiry !== null && !state.forceKillNeeded) {
656
847
  appendStderr(`Timed out command exited after graceful termination: ${command}\n`);
@@ -668,11 +859,11 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
668
859
  appendStderr(`${describeRamTempError(full)}\n`);
669
860
  }
670
861
  }
671
- await releaseLease();
672
- // Broken resource ownership or a lost native lifecycle fails even if a shell branch later exited 0:
673
- // its verdict is an ordinary operational failure, never a successful or silently unbounded run.
674
- const operationalFailure = state.poolCloseError !== null || lifecycleFailure !== null;
675
- await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure, seen);
862
+ await releaseLease(code !== 0 || state.expiry !== null || state.poolCloseError !== null || lifecycleFailure !== null);
863
+ // Broken resource ownership, a lost native lifecycle, or leftovers that could not be retired fail even if a shell
864
+ // branch later exited 0: its verdict is an ordinary operational failure, never a successful or silently unbounded run.
865
+ const operationalFailure = state.poolCloseError !== null || lifecycleFailure !== null || state.retireError !== null;
866
+ await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure, seen, state.lockWaits);
676
867
  return {
677
868
  success: !operationalFailure && state.expiry === null && code === 0,
678
869
  terminalOutput: outputChunks.join(''),
@@ -680,44 +871,11 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
680
871
  }
681
872
  export function createProcessTreeKiller() {
682
873
  return {
683
- kill(pid, signal) {
684
- return killProcessGroup(pid, signal);
874
+ async kill(pid, signal) {
875
+ signalProcessGroup(pid, signal);
685
876
  },
686
877
  };
687
878
  }
688
- function killProcessGroup(pid, signal) {
689
- try {
690
- // On POSIX the executor spawns with detached: true, so the child leads a
691
- // process group of its own and -pid signals all of it at once, including
692
- // grandchildren that tree-walk libraries miss when parents die and
693
- // children get reparented to init. Only the group is named: the leader may
694
- // already be reaped, and its pid reused by an unrelated process, while the
695
- // group lives on in its other members. Windows has no process groups.
696
- process.kill(process.platform === 'win32' ? pid : -pid, signal);
697
- }
698
- catch {
699
- // ESRCH: the group (or the Windows process) has already exited.
700
- }
701
- return Promise.resolve();
702
- }
703
- /**
704
- * Whether any process is left in the group `pgid` leads. A group outlives its
705
- * reaped leader for as long as one member does, and no process can take its id
706
- * meanwhile. Windows has no process groups, so there is nothing left to name.
707
- */
708
- function processGroupHasMembers(pgid) {
709
- if (process.platform === 'win32') {
710
- return false;
711
- }
712
- try {
713
- process.kill(-pgid, 0);
714
- return true;
715
- }
716
- catch (error) {
717
- // EPERM: a member exists but runs as another user.
718
- return error instanceof Error && 'code' in error && error.code === 'EPERM';
719
- }
720
- }
721
879
  function resolveCwd(cwd, root) {
722
880
  if (!cwd) {
723
881
  return root;
@@ -737,10 +895,10 @@ function buildCommand(options) {
737
895
  }
738
896
  return parts.join(' ');
739
897
  }
740
- /** The lease's directory is the command's TMPDIR unless the target names one itself. */
741
- function mergeEnv(lease, env) {
898
+ /** Owned scratch is the command's TMPDIR unless the target names one itself. */
899
+ function mergeEnv(directory, env) {
742
900
  const inherited = withoutPoolEnv(process.env);
743
- return lease === null ? { ...inherited, ...env } : { ...inherited, TMPDIR: lease.directory, ...env };
901
+ return directory === null ? { ...inherited, ...env } : { ...inherited, TMPDIR: directory, ...env };
744
902
  }
745
903
  /** The process's environment without the CPU pool's descriptor: a pool and its count only a grant may give. */
746
904
  function withoutPoolEnv(env) {
@@ -770,14 +928,6 @@ function signalToExitCode(signal) {
770
928
  }
771
929
  return EXIT_CODE_BY_SIGNAL[signal] ?? 1;
772
930
  }
773
- async function ignoreKillError(promise) {
774
- try {
775
- await promise;
776
- }
777
- catch {
778
- // The process tree may have already exited between timeout and kill.
779
- }
780
- }
781
931
  /** Resolves when `promise` settles or `ms` passes, whichever is first, leaving no timer behind. */
782
932
  async function settledWithin(promise, ms) {
783
933
  let timer;