@smoothbricks/nx-plugin 0.5.6 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (440) hide show
  1. package/README.md +1095 -360
  2. package/dist/.build-inputs.json +138 -0
  3. package/dist/bin/smoo-nx-cargo-toolchain.d.ts +3 -0
  4. package/dist/bin/smoo-nx-cargo-toolchain.d.ts.map +1 -0
  5. package/dist/bin/smoo-nx-cargo-toolchain.js +8 -0
  6. package/dist/bin/smoo-nx-forbidden-spellings.d.ts +3 -0
  7. package/dist/bin/smoo-nx-forbidden-spellings.d.ts.map +1 -0
  8. package/dist/bin/smoo-nx-forbidden-spellings.js +32 -0
  9. package/dist/bin/smoo-nx-nextest-timings.d.ts +3 -0
  10. package/dist/bin/smoo-nx-nextest-timings.d.ts.map +1 -0
  11. package/dist/bin/smoo-nx-nextest-timings.js +28 -0
  12. package/dist/bounded-test-policy.d.ts +1 -1
  13. package/dist/bounded-test-policy.d.ts.map +1 -1
  14. package/dist/bounded-test-policy.js +14 -15
  15. package/dist/build-state-output-policy.d.ts +95 -1
  16. package/dist/build-state-output-policy.d.ts.map +1 -1
  17. package/dist/build-state-output-policy.js +368 -30
  18. package/dist/bun-lock-scope.d.ts +44 -0
  19. package/dist/bun-lock-scope.d.ts.map +1 -0
  20. package/dist/bun-lock-scope.js +97 -0
  21. package/dist/cargo-build-script-env.d.ts +49 -0
  22. package/dist/cargo-build-script-env.d.ts.map +1 -0
  23. package/dist/cargo-build-script-env.js +324 -0
  24. package/dist/cargo-closure-input.d.ts +31 -4
  25. package/dist/cargo-closure-input.d.ts.map +1 -1
  26. package/dist/cargo-closure-input.js +66 -6
  27. package/dist/cargo-retention-hook.d.ts +17 -0
  28. package/dist/cargo-retention-hook.d.ts.map +1 -0
  29. package/dist/cargo-retention-hook.js +85 -0
  30. package/dist/cargo-source-hash.d.ts +25 -7
  31. package/dist/cargo-source-hash.d.ts.map +1 -1
  32. package/dist/cargo-source-hash.js +108 -43
  33. package/dist/cargo-toolchain-paths.d.ts +40 -0
  34. package/dist/cargo-toolchain-paths.d.ts.map +1 -0
  35. package/dist/cargo-toolchain-paths.js +84 -0
  36. package/dist/cargo-toolchain-policy.d.ts +25 -8
  37. package/dist/cargo-toolchain-policy.d.ts.map +1 -1
  38. package/dist/cargo-toolchain-policy.js +39 -5
  39. package/dist/cargo-workspace.d.ts +10 -4
  40. package/dist/cargo-workspace.d.ts.map +1 -1
  41. package/dist/cargo-workspace.js +65 -95
  42. package/dist/cross-check-policy.d.ts +9 -9
  43. package/dist/cross-check-policy.d.ts.map +1 -1
  44. package/dist/cross-check-policy.js +11 -11
  45. package/dist/disk-command-policy.d.ts.map +1 -1
  46. package/dist/disk-command-policy.js +1 -0
  47. package/dist/ensure-built-receipt.d.ts +13 -5
  48. package/dist/ensure-built-receipt.d.ts.map +1 -1
  49. package/dist/ensure-built-receipt.js +126 -47
  50. package/dist/ensure-built-witness.d.ts +27 -8
  51. package/dist/ensure-built-witness.d.ts.map +1 -1
  52. package/dist/ensure-built-witness.js +45 -15
  53. package/dist/ensure-built.d.ts.map +1 -1
  54. package/dist/ensure-built.js +9 -1
  55. package/dist/eslint-config-closure.d.ts +7 -0
  56. package/dist/eslint-config-closure.d.ts.map +1 -0
  57. package/dist/eslint-config-closure.js +346 -0
  58. package/dist/executors/all-verdicts/executor.d.ts +35 -0
  59. package/dist/executors/all-verdicts/executor.d.ts.map +1 -0
  60. package/dist/executors/all-verdicts/executor.js +133 -0
  61. package/dist/executors/all-verdicts/schema.d.ts +10 -0
  62. package/dist/executors/all-verdicts/schema.d.ts.map +1 -0
  63. package/dist/executors/all-verdicts/schema.js +2 -0
  64. package/dist/executors/bounded-exec/bounds.d.ts +56 -0
  65. package/dist/executors/bounded-exec/bounds.d.ts.map +1 -0
  66. package/dist/executors/bounded-exec/bounds.js +43 -0
  67. package/dist/executors/bounded-exec/cpu-pool.d.ts +33 -36
  68. package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
  69. package/dist/executors/bounded-exec/cpu-pool.js +54 -119
  70. package/dist/executors/bounded-exec/executor.d.ts +19 -12
  71. package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
  72. package/dist/executors/bounded-exec/executor.js +371 -239
  73. package/dist/executors/bounded-exec/gateway-lease.d.ts +22 -19
  74. package/dist/executors/bounded-exec/gateway-lease.d.ts.map +1 -1
  75. package/dist/executors/bounded-exec/gateway-lease.js +81 -28
  76. package/dist/executors/bounded-exec/hang-diagnostics.d.ts +18 -1
  77. package/dist/executors/bounded-exec/hang-diagnostics.d.ts.map +1 -1
  78. package/dist/executors/bounded-exec/hang-diagnostics.js +148 -30
  79. package/dist/executors/bounded-exec/lease-wait.d.ts +43 -0
  80. package/dist/executors/bounded-exec/lease-wait.d.ts.map +1 -0
  81. package/dist/executors/bounded-exec/lease-wait.js +68 -0
  82. package/dist/executors/bounded-exec/lock-wait.d.ts +51 -0
  83. package/dist/executors/bounded-exec/lock-wait.d.ts.map +1 -0
  84. package/dist/executors/bounded-exec/lock-wait.js +108 -0
  85. package/dist/executors/bounded-exec/owned-subtree.d.ts +84 -0
  86. package/dist/executors/bounded-exec/owned-subtree.d.ts.map +1 -0
  87. package/dist/executors/bounded-exec/owned-subtree.js +178 -0
  88. package/dist/executors/bounded-exec/progress.d.ts +4 -1
  89. package/dist/executors/bounded-exec/progress.d.ts.map +1 -1
  90. package/dist/executors/bounded-exec/progress.js +5 -2
  91. package/dist/executors/bounded-exec/ram-attach.d.ts +19 -0
  92. package/dist/executors/bounded-exec/ram-attach.d.ts.map +1 -0
  93. package/dist/executors/bounded-exec/ram-attach.js +207 -0
  94. package/dist/executors/bounded-exec/ram-detach.d.ts +25 -0
  95. package/dist/executors/bounded-exec/ram-detach.d.ts.map +1 -0
  96. package/dist/executors/bounded-exec/ram-detach.js +207 -0
  97. package/dist/executors/bounded-exec/ram-temp-fixtures.d.ts +76 -0
  98. package/dist/executors/bounded-exec/ram-temp-fixtures.d.ts.map +1 -1
  99. package/dist/executors/bounded-exec/ram-temp-fixtures.js +296 -0
  100. package/dist/executors/bounded-exec/ram-temp.d.ts +388 -88
  101. package/dist/executors/bounded-exec/ram-temp.d.ts.map +1 -1
  102. package/dist/executors/bounded-exec/ram-temp.js +1213 -399
  103. package/dist/executors/bounded-exec/run-cpu-pool.d.ts +4 -4
  104. package/dist/executors/bounded-exec/run-cpu-pool.d.ts.map +1 -1
  105. package/dist/executors/bounded-exec/run-cpu-pool.js +20 -30
  106. package/dist/executors/bounded-exec/runner-hook.js +14 -5
  107. package/dist/executors/bounded-exec/runner-lifecycle.d.ts +14 -1
  108. package/dist/executors/bounded-exec/runner-lifecycle.d.ts.map +1 -1
  109. package/dist/executors/bounded-exec/runner-lifecycle.js +27 -6
  110. package/dist/executors/bounded-exec/{cpu-tokens.d.ts → runner-sizing.d.ts} +14 -42
  111. package/dist/executors/bounded-exec/runner-sizing.d.ts.map +1 -0
  112. package/dist/executors/bounded-exec/{cpu-tokens.js → runner-sizing.js} +5 -35
  113. package/dist/executors/bounded-exec/runtime-pin.d.ts +73 -0
  114. package/dist/executors/bounded-exec/runtime-pin.d.ts.map +1 -0
  115. package/dist/executors/bounded-exec/runtime-pin.js +167 -0
  116. package/dist/executors/bounded-exec/schema.d.ts +18 -1
  117. package/dist/executors/bounded-exec/schema.d.ts.map +1 -1
  118. package/dist/executors/bounded-exec/verdict.d.ts +40 -11
  119. package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
  120. package/dist/executors/bounded-exec/verdict.js +88 -19
  121. package/dist/executors/cargo-zigbuild/executor.d.ts +21 -0
  122. package/dist/executors/cargo-zigbuild/executor.d.ts.map +1 -1
  123. package/dist/executors/cargo-zigbuild/executor.js +67 -12
  124. package/dist/executors/clean-outputs/executor.js +1 -0
  125. package/dist/executors/scriptc-native/executor.d.ts +79 -0
  126. package/dist/executors/scriptc-native/executor.d.ts.map +1 -0
  127. package/dist/executors/scriptc-native/executor.js +367 -0
  128. package/dist/executors/scriptc-native/ffi-paths.d.ts +9 -0
  129. package/dist/executors/scriptc-native/ffi-paths.d.ts.map +1 -0
  130. package/dist/executors/scriptc-native/ffi-paths.js +25 -0
  131. package/dist/executors/scriptc-native/schema.d.ts +34 -0
  132. package/dist/executors/scriptc-native/schema.d.ts.map +1 -0
  133. package/dist/executors/scriptc-native/schema.js +1 -0
  134. package/dist/executors/typescript-emit/executor.d.ts +3 -1
  135. package/dist/executors/typescript-emit/executor.d.ts.map +1 -1
  136. package/dist/executors/typescript-emit/executor.js +164 -33
  137. package/dist/forbidden-spelling-policy.d.ts +56 -0
  138. package/dist/forbidden-spelling-policy.d.ts.map +1 -0
  139. package/dist/forbidden-spelling-policy.js +176 -0
  140. package/dist/generators/create-package/generator.d.ts.map +1 -1
  141. package/dist/generators/create-package/generator.js +3 -4
  142. package/dist/graph-input-contract.d.ts +14 -0
  143. package/dist/graph-input-contract.d.ts.map +1 -1
  144. package/dist/graph-input-contract.js +67 -2
  145. package/dist/index.d.ts +23 -6
  146. package/dist/index.d.ts.map +1 -1
  147. package/dist/index.js +676 -215
  148. package/dist/managed-assets.d.ts +2 -0
  149. package/dist/managed-assets.d.ts.map +1 -1
  150. package/dist/managed-files/files.d.ts +3 -3
  151. package/dist/managed-files/files.d.ts.map +1 -1
  152. package/dist/managed-files/files.js +28 -3
  153. package/dist/managed-files/managed-format.d.ts.map +1 -1
  154. package/dist/managed-files/managed-format.js +1 -0
  155. package/dist/nextest-code-bound.js +1 -0
  156. package/dist/nextest-timings.d.ts +35 -0
  157. package/dist/nextest-timings.d.ts.map +1 -0
  158. package/dist/nextest-timings.js +168 -0
  159. package/dist/owned-installation.d.ts +6 -5
  160. package/dist/owned-installation.d.ts.map +1 -1
  161. package/dist/owned-installation.js +150 -26
  162. package/dist/package-target-policy.d.ts.map +1 -1
  163. package/dist/package-target-policy.js +2 -3
  164. package/dist/process-children.d.ts +7 -0
  165. package/dist/process-children.d.ts.map +1 -0
  166. package/dist/process-children.darwin.d.ts +7 -0
  167. package/dist/process-children.darwin.d.ts.map +1 -0
  168. package/dist/process-children.darwin.js +13 -0
  169. package/dist/process-children.js +14 -0
  170. package/dist/process-children.linux.d.ts +8 -0
  171. package/dist/process-children.linux.d.ts.map +1 -0
  172. package/dist/process-children.linux.js +25 -0
  173. package/dist/process-group.d.ts +108 -0
  174. package/dist/process-group.d.ts.map +1 -0
  175. package/dist/process-group.js +171 -0
  176. package/dist/process-listing.d.ts +7 -0
  177. package/dist/process-listing.d.ts.map +1 -0
  178. package/dist/process-listing.js +51 -0
  179. package/dist/producer-receipts.d.ts +8 -2
  180. package/dist/producer-receipts.d.ts.map +1 -1
  181. package/dist/producer-receipts.js +75 -48
  182. package/dist/task-graph-policy.d.ts +21 -0
  183. package/dist/task-graph-policy.d.ts.map +1 -0
  184. package/dist/task-graph-policy.js +91 -0
  185. package/dist/test-timings.js +4 -4
  186. package/dist/test-walk-policy.d.ts +14 -5
  187. package/dist/test-walk-policy.d.ts.map +1 -1
  188. package/dist/test-walk-policy.js +8 -2
  189. package/dist/testing.d.ts +73 -28
  190. package/dist/testing.d.ts.map +1 -1
  191. package/dist/testing.js +217 -223
  192. package/dist/tool-bins.d.ts +33 -0
  193. package/dist/tool-bins.d.ts.map +1 -0
  194. package/dist/tool-bins.js +72 -0
  195. package/dist/typecheck-test-policy.d.ts.map +1 -1
  196. package/dist/typecheck-test-policy.js +3 -4
  197. package/dist/wait-why-policy.d.ts +28 -0
  198. package/dist/wait-why-policy.d.ts.map +1 -0
  199. package/dist/wait-why-policy.js +223 -0
  200. package/dist/workspace-config-policy.d.ts +14 -8
  201. package/dist/workspace-config-policy.d.ts.map +1 -1
  202. package/dist/workspace-config-policy.js +60 -16
  203. package/dist/workspace-manifest.d.ts +41 -6
  204. package/dist/workspace-manifest.d.ts.map +1 -1
  205. package/dist/workspace-manifest.js +57 -0
  206. package/executors.json +21 -1
  207. package/loader.js +2 -0
  208. package/managed/raw/envrc +11 -0
  209. package/managed/raw/patches/nx@23.2.1.patch +2840 -199
  210. package/managed/raw/patches/ttsc@0.34.1.patch +119 -0
  211. package/managed/raw/tooling/direnv/cargo-fetch-bound.ts +245 -0
  212. package/managed/raw/tooling/direnv/cargo-fetch-run.ts +299 -0
  213. package/managed/raw/tooling/direnv/compiler-variables.sh +42 -0
  214. package/managed/raw/tooling/direnv/developer-links.ts +464 -25
  215. package/managed/raw/tooling/direnv/devenv.smoo.nix +78 -38
  216. package/managed/raw/tooling/direnv/github-actions-bootstrap.sh +14 -12
  217. package/managed/raw/tooling/direnv/installed-bun-links.ts +308 -0
  218. package/managed/raw/tooling/direnv/linked-snapshots.ts +62 -3
  219. package/managed/raw/tooling/direnv/nx-socket-dir.sh +56 -17
  220. package/managed/raw/tooling/direnv/repo-path +68 -6
  221. package/managed/raw/tooling/direnv/secret-references.ts +13 -4
  222. package/managed/raw/tooling/direnv/setup-environment.ts +128 -537
  223. package/managed/raw/tooling/direnv/toolchain-stamp.ts +4 -2
  224. package/managed/raw/tooling/direnv/ttsc-env.sh +9 -0
  225. package/managed/raw/tooling/direnv/uv-relocation.ts +227 -0
  226. package/managed/raw/tooling/git-hooks/commit-msg.sh +2 -1
  227. package/managed/raw/tooling/git-hooks/pre-commit.sh +2 -1
  228. package/managed/raw/tooling/git-hooks/pre-push.sh +2 -1
  229. package/managed/templates/github/actions/cache-ttsc-plugins/action.yml +85 -7
  230. package/managed/templates/github/actions/save-nix-devenv/action.yml +12 -2
  231. package/managed/templates/github/actions/setup-devenv/action.yml +2 -0
  232. package/package.json +77 -13
  233. package/src/bin/smoo-nx-cargo-toolchain.ts +11 -0
  234. package/src/bin/smoo-nx-forbidden-spellings.ts +33 -0
  235. package/src/bin/smoo-nx-nextest-timings.ts +31 -0
  236. package/src/bounded-test-policy.ts +12 -14
  237. package/src/build-state-output-policy.ts +458 -34
  238. package/src/bun-lock-scope.ts +108 -0
  239. package/src/cargo-build-script-env.ts +265 -0
  240. package/src/cargo-closure-input.ts +74 -7
  241. package/src/cargo-retention-hook.ts +94 -0
  242. package/src/cargo-source-hash.ts +138 -41
  243. package/src/cargo-toolchain-paths.ts +95 -0
  244. package/src/cargo-toolchain-policy.ts +49 -5
  245. package/src/cargo-workspace.ts +72 -123
  246. package/src/cross-check-policy.ts +12 -10
  247. package/src/disk-command-policy.ts +1 -0
  248. package/src/ensure-built-receipt.ts +132 -39
  249. package/src/ensure-built-witness.ts +62 -10
  250. package/src/ensure-built.ts +11 -1
  251. package/src/eslint-config-closure.ts +357 -0
  252. package/src/executors/all-verdicts/executor.ts +164 -0
  253. package/src/executors/all-verdicts/schema.json +22 -0
  254. package/src/executors/all-verdicts/schema.ts +10 -0
  255. package/src/executors/bounded-exec/bounds.ts +76 -0
  256. package/src/executors/bounded-exec/cpu-pool.ts +51 -117
  257. package/src/executors/bounded-exec/executor.ts +423 -259
  258. package/src/executors/bounded-exec/gateway-lease.ts +82 -33
  259. package/src/executors/bounded-exec/hang-diagnostics.ts +173 -31
  260. package/src/executors/bounded-exec/lease-wait.ts +89 -0
  261. package/src/executors/bounded-exec/lock-wait.ts +141 -0
  262. package/src/executors/bounded-exec/owned-subtree.ts +235 -0
  263. package/src/executors/bounded-exec/progress.ts +9 -3
  264. package/src/executors/bounded-exec/ram-attach.ts +238 -0
  265. package/src/executors/bounded-exec/ram-detach.ts +226 -0
  266. package/src/executors/bounded-exec/ram-temp-fixtures.ts +358 -0
  267. package/src/executors/bounded-exec/ram-temp-registry.txt +440 -0
  268. package/src/executors/bounded-exec/ram-temp.ts +1497 -460
  269. package/src/executors/bounded-exec/run-cpu-pool.ts +22 -30
  270. package/src/executors/bounded-exec/runner-hook.ts +14 -6
  271. package/src/executors/bounded-exec/runner-lifecycle.ts +31 -8
  272. package/src/executors/bounded-exec/{cpu-tokens.ts → runner-sizing.ts} +13 -59
  273. package/src/executors/bounded-exec/runtime-pin.ts +232 -0
  274. package/src/executors/bounded-exec/schema.json +11 -2
  275. package/src/executors/bounded-exec/schema.ts +19 -1
  276. package/src/executors/bounded-exec/verdict.ts +137 -34
  277. package/src/executors/cargo-zigbuild/executor.ts +82 -8
  278. package/src/executors/clean-outputs/executor.ts +1 -0
  279. package/src/executors/scriptc-native/executor.ts +394 -0
  280. package/src/executors/scriptc-native/ffi-paths.ts +29 -0
  281. package/src/executors/scriptc-native/schema.json +60 -0
  282. package/src/executors/scriptc-native/schema.ts +34 -0
  283. package/src/executors/typescript-emit/executor.ts +205 -38
  284. package/src/forbidden-spelling-policy.ts +226 -0
  285. package/src/generators/create-package/generator.ts +7 -7
  286. package/src/graph-input-contract.ts +61 -2
  287. package/src/index.ts +792 -225
  288. package/src/managed-assets.ts +3 -0
  289. package/src/managed-files/files.ts +28 -3
  290. package/src/managed-files/managed-format.ts +1 -0
  291. package/src/nextest-code-bound.ts +1 -0
  292. package/src/nextest-timings.ts +106 -0
  293. package/src/owned-installation.ts +150 -23
  294. package/src/package-target-policy.ts +2 -3
  295. package/src/process-children.darwin.ts +14 -0
  296. package/src/process-children.linux.ts +33 -0
  297. package/src/process-children.ts +22 -0
  298. package/src/process-group.ts +209 -0
  299. package/src/process-listing.ts +54 -0
  300. package/src/producer-receipts.ts +68 -45
  301. package/src/task-graph-policy.ts +98 -0
  302. package/src/test-walk-policy.ts +23 -8
  303. package/src/testing.ts +240 -232
  304. package/src/tool-bins.ts +108 -0
  305. package/src/typecheck-test-policy.ts +2 -3
  306. package/src/wait-why-policy.ts +226 -0
  307. package/src/workspace-config-policy.ts +63 -15
  308. package/src/workspace-manifest.ts +89 -6
  309. package/test-timings.json +123 -98
  310. package/dist/bin/smoo-nx-failures.d.ts +0 -3
  311. package/dist/bin/smoo-nx-failures.d.ts.map +0 -1
  312. package/dist/bin/smoo-nx-failures.js +0 -3
  313. package/dist/bin/smoo-nx-nextest-extract.d.ts +0 -3
  314. package/dist/bin/smoo-nx-nextest-extract.d.ts.map +0 -1
  315. package/dist/bin/smoo-nx-nextest-extract.js +0 -42
  316. package/dist/executors/bounded-exec/cpu-tokens.d.ts.map +0 -1
  317. package/dist/failure-summary.d.ts +0 -5
  318. package/dist/failure-summary.d.ts.map +0 -1
  319. package/dist/failure-summary.js +0 -118
  320. package/dist/nextest-extraction.d.ts +0 -117
  321. package/dist/nextest-extraction.d.ts.map +0 -1
  322. package/dist/nextest-extraction.js +0 -303
  323. package/managed/raw/patches/ttsc@0.30.4.patch +0 -2255
  324. package/src/__tests__/cargo-closure-filesets.ts +0 -94
  325. package/src/__tests__/cargo-closure-fixture.ts +0 -113
  326. package/src/__tests__/counted-cargo-process.ts +0 -286
  327. package/src/__tests__/counted-cargo.ts +0 -713
  328. package/src/__tests__/directory-change.ts +0 -27
  329. package/src/__tests__/fixture-cargo-home.ts +0 -36
  330. package/src/__tests__/fixture-nx-env.ts +0 -193
  331. package/src/__tests__/owned-installation.ts +0 -13
  332. package/src/__tests__/past-tick.ts +0 -12
  333. package/src/__tests__/progress-clock.ts +0 -43
  334. package/src/__tests__/runtime-input.ts +0 -23
  335. package/src/bin/smoo-nx-bound-failures.test.ts +0 -127
  336. package/src/bin/smoo-nx-failures.ts +0 -4
  337. package/src/bin/smoo-nx-nextest-extract.ts +0 -44
  338. package/src/bounded-test-policy.test.ts +0 -1029
  339. package/src/build-state-output-policy.test.ts +0 -115
  340. package/src/bun-bootstrap.test.ts +0 -102
  341. package/src/bun-file-packages.test.ts +0 -47
  342. package/src/cargo-closure-filesets.test.ts +0 -63
  343. package/src/cargo-closure-input.property.test.ts +0 -50
  344. package/src/cargo-closure-input.test.ts +0 -506
  345. package/src/cargo-closure-key.test.ts +0 -240
  346. package/src/cargo-resolve-cache.test.ts +0 -1181
  347. package/src/cargo-source-hash.test.ts +0 -707
  348. package/src/cargo-workspace.test.ts +0 -615
  349. package/src/cross-check-policy.test.ts +0 -316
  350. package/src/disk-command-policy.test.ts +0 -61
  351. package/src/ensure-built-receipt.test.ts +0 -279
  352. package/src/ensure-built-witness.test.ts +0 -688
  353. package/src/ensure-built.test.ts +0 -1462
  354. package/src/executors/bounded-exec/cargo-progress-host.test.ts +0 -332
  355. package/src/executors/bounded-exec/cpu-pool.test.ts +0 -684
  356. package/src/executors/bounded-exec/cpu-tokens.test.ts +0 -519
  357. package/src/executors/bounded-exec/disk-lease.test.ts +0 -219
  358. package/src/executors/bounded-exec/executor-host.test.ts +0 -800
  359. package/src/executors/bounded-exec/executor.test.ts +0 -1819
  360. package/src/executors/bounded-exec/go-progress-host.test.ts +0 -171
  361. package/src/executors/bounded-exec/hang-diagnostics.test.ts +0 -291
  362. package/src/executors/bounded-exec/progress.test.ts +0 -139
  363. package/src/executors/bounded-exec/ram-temp-held.test.ts +0 -264
  364. package/src/executors/bounded-exec/ram-temp-host.test.ts +0 -635
  365. package/src/executors/bounded-exec/ram-temp.test.ts +0 -750
  366. package/src/executors/bounded-exec/run-cpu-pool.test.ts +0 -79
  367. package/src/executors/bounded-exec/runner-hook.test.ts +0 -193
  368. package/src/executors/bounded-exec/runner-lifecycle.test.ts +0 -82
  369. package/src/executors/bounded-exec/verdict.test.ts +0 -545
  370. package/src/executors/cargo-zigbuild/executor.test.ts +0 -129
  371. package/src/executors/clean-outputs/executor.test.ts +0 -20
  372. package/src/executors/typescript-emit/executor.test.ts +0 -83
  373. package/src/failure-summary.ts +0 -119
  374. package/src/generators/bounded-test-targets/generator.test.ts +0 -138
  375. package/src/generators/bun-test-tracing/generator.test.ts +0 -250
  376. package/src/generators/create-package/generator.test.ts +0 -190
  377. package/src/generators/make-public/generator.test.ts +0 -137
  378. package/src/graph-input-contract.test.ts +0 -379
  379. package/src/host-controller-wiring.test.ts +0 -31
  380. package/src/index.test.ts +0 -3319
  381. package/src/linked-snapshots.nx.test.ts +0 -254
  382. package/src/managed-files/forgejo-artifact.test.ts +0 -395
  383. package/src/managed-files/generator.test.ts +0 -243
  384. package/src/managed-files/github-host-cache-isolation.test.ts +0 -86
  385. package/src/managed-files/json-edit.test.ts +0 -216
  386. package/src/managed-files/managed-content.test.ts +0 -166
  387. package/src/managed-files/managed-format-json.test.ts +0 -308
  388. package/src/managed-files/repo-path.test.ts +0 -172
  389. package/src/managed-files/setup-devenv-free-disk.test.ts +0 -95
  390. package/src/managed-files/trace-db-artifact.test.ts +0 -7
  391. package/src/managed-files/tree-permissions.test.ts +0 -109
  392. package/src/managed-files/tree.test.ts +0 -123
  393. package/src/manifest-hash.test.ts +0 -219
  394. package/src/nextest-code-bound.test.ts +0 -161
  395. package/src/nextest-extraction.test.ts +0 -216
  396. package/src/nextest-extraction.ts +0 -371
  397. package/src/nextest-partitions.test.ts +0 -501
  398. package/src/nx-cache-bound.test.ts +0 -109
  399. package/src/nx-cache-restore-mtime.test.ts +0 -170
  400. package/src/nx-daemon-claim.test.ts +0 -279
  401. package/src/nx-daemon-idle-cpu.test.ts +0 -80
  402. package/src/nx-daemon-idle-policy.test.ts +0 -111
  403. package/src/nx-daemon-ignore-files.test.ts +0 -210
  404. package/src/nx-daemon-root-normalization.test.ts +0 -105
  405. package/src/nx-daemon-start-never.test.ts +0 -292
  406. package/src/nx-daemon-validity.test.ts +0 -1055
  407. package/src/nx-dot-input-policy.test.ts +0 -353
  408. package/src/nx-failure-life-cycles.test.ts +0 -207
  409. package/src/nx-files-archive-racy.test.ts +0 -97
  410. package/src/nx-foreign-workspace-env.test.ts +0 -329
  411. package/src/nx-incomplete-run-summary.test.ts +0 -70
  412. package/src/nx-plugin-load-errors.test.ts +0 -120
  413. package/src/nx-project-graph-errors.test.ts +0 -70
  414. package/src/nx-project-resolution-cache.test.ts +0 -176
  415. package/src/nx-raw-inputs.test.ts +0 -166
  416. package/src/nx-resident-graph-workers.test.ts +0 -112
  417. package/src/nx-run-information.test.ts +0 -118
  418. package/src/nx-skipped-task-exit.test.ts +0 -180
  419. package/src/nx-state-location.test.ts +0 -63
  420. package/src/nx-store-resolution.test.ts +0 -81
  421. package/src/nx-task-graph-cycles.test.ts +0 -148
  422. package/src/nx-task-output-lease.test.ts +0 -550
  423. package/src/nx-use-local.test.ts +0 -105
  424. package/src/owned-installation.test.ts +0 -105
  425. package/src/package-target-policy.test.ts +0 -1014
  426. package/src/producer-receipts.test.ts +0 -285
  427. package/src/release-config-policy.test.ts +0 -311
  428. package/src/runtime-input.test.ts +0 -393
  429. package/src/serial-group-wiring.test.ts +0 -326
  430. package/src/task-history-namespace.test.ts +0 -181
  431. package/src/task-results.test.ts +0 -228
  432. package/src/test-clock-policy.test.ts +0 -338
  433. package/src/test-program.test.ts +0 -303
  434. package/src/test-timings.test.ts +0 -240
  435. package/src/test-walk-policy.test.ts +0 -620
  436. package/src/testing.test.ts +0 -509
  437. package/src/typecheck-test-policy.test.ts +0 -1468
  438. package/src/typescript-resolution.test.ts +0 -76
  439. package/src/version-actions.test.ts +0 -80
  440. package/src/workspace-config-policy.test.ts +0 -737
@@ -1,42 +1,58 @@
1
1
  import { type ChildProcess, spawn } from 'node:child_process';
2
2
  import { accessSync, constants } from 'node:fs';
3
+ import { mkdtemp, rm } from 'node:fs/promises';
3
4
  import { availableParallelism, userInfo } from 'node:os';
4
5
  import { delimiter, isAbsolute, join, resolve as resolvePath } from 'node:path';
5
6
  import { createInterface } from 'node:readline';
6
- import { Duplex } from 'node:stream';
7
+ import { Duplex, Readable } from 'node:stream';
7
8
  import { workspaceDataDirectoryForWorkspace } from 'nx/src/utils/cache-directory.js';
8
-
9
+ import {
10
+ DEADLINE_PATIENCE,
11
+ describeMembersLeft,
12
+ killProcessGroup,
13
+ processGroupHasMembers,
14
+ signalProcessGroup,
15
+ } from '../../process-group.js';
16
+ import { type BoundEvent, type BoundKind, type Bounds, earliestBound, stepBounds, UNARMED } from './bounds.js';
9
17
  import {
10
18
  CPU_POOL_ENV,
11
19
  type CpuPool,
12
20
  createCpuPool,
13
- type HostWaits,
14
21
  type InheritedCpuPool,
15
22
  readInheritedCpuPool,
23
+ type TokenGrant,
16
24
  takeParentCpuTokens,
17
25
  } from './cpu-pool.js';
18
- import {
19
- CPU_TOKENS_ENV,
20
- type CpuBudget,
21
- gatewayCpuBudget,
22
- runnerDemand,
23
- sizeRunner,
24
- type TokenGrant,
25
- } from './cpu-tokens.js';
26
+ import { GATEWAY_SOCKET } from './gateway-lease.js';
26
27
  import type { HangDiagnostics } from './hang-diagnostics.js';
27
28
  import { captureHangDiagnostics, HANG_CAPTURE_MS } from './hang-diagnostics.js';
29
+ import type { LockWait } from './lock-wait.js';
28
30
  import { createNoSpaceWatch } from './no-space.js';
29
- import { createRunProgress, withProgressReport } from './progress.js';
30
31
  import {
31
32
  COWSHED_EXECUTABLE_ENV,
33
+ cowshedSubtreeOwner,
34
+ HANDLE_FD,
35
+ readAdmission,
36
+ SUPERVISOR_SOCKET_ENV,
37
+ type SubtreeAdmission,
38
+ type SubtreeOwner,
39
+ } from './owned-subtree.js';
40
+ import { createRunProgress, withProgressReport } from './progress.js';
41
+ import {
32
42
  createHostCommands,
43
+ describeCoverageGap,
33
44
  describeRamTempError,
45
+ describeSignalled,
46
+ LEFTOVER_GRACE_MS,
47
+ PRODUCTION,
34
48
  RAM_TEMP_CAPACITY_BYTES,
35
49
  type RamTempLease,
36
50
  RamTempVolume,
37
51
  ramTempPaths,
38
52
  } from './ram-temp.js';
39
- import { createRunnerLifecycle, withRunnerLifecycle } from './runner-lifecycle.js';
53
+ import { createRunnerLifecycle, withFixtureLifecycle, withRunnerLifecycle } from './runner-lifecycle.js';
54
+ import { CPU_TOKENS_ENV, runnerDemand, sizeRunner } from './runner-sizing.js';
55
+ import { checkRuntimePin, describeRuntimePinRefusal } from './runtime-pin.js';
40
56
  import type { BoundedExecOptions } from './schema.js';
41
57
  import {
42
58
  type BoundedExecRecord,
@@ -70,9 +86,9 @@ const EXIT_CODE_BY_SIGNAL: Partial<Record<NodeJS.Signals, number>> = {
70
86
  /** The pool descriptor: only a grant may give it, so neither the target's nor the inherited environment passes it on. */
71
87
  const POOL_ENV = [CPU_TOKENS_ENV, CPU_POOL_ENV];
72
88
  /**
73
- * The variables the CPU budget sets for the command it grants: no target's environment may set them. A jobserver the
74
- * executor itself inherits, a parent make's, still reaches an ungranted command by Cargo's own rule; a granted one's
75
- * sub-pool names its own fifo over it.
89
+ * The variables a grant sets for the command it sizes: no target's environment may set them. A jobserver the executor
90
+ * itself inherits, a parent make's or its cowshed job's, still reaches an ungranted command by Cargo's own rule; a
91
+ * granted one's sub-pool names its own fifo over it.
76
92
  */
77
93
  const BUDGET_ENV = [...POOL_ENV, 'CARGO_MAKEFLAGS', 'MAKEFLAGS', 'MFLAGS'];
78
94
 
@@ -104,8 +120,12 @@ interface RunState {
104
120
  forceKillNeeded: boolean;
105
121
  /** The sub-pool's close error when it did not close cleanly: the run fails. */
106
122
  poolCloseError: string | null;
123
+ /** Why the command's subtree could not be retired, when the supervisor's client failed: the run fails. */
124
+ retireError: string | null;
107
125
  /** The first output line that said ENOSPC, with the task's volume read when it arrived. */
108
126
  noSpace: Promise<NoSpaceSighting> | null;
127
+ /** The lock waits the expiry's snapshot named: empty unless that capture completed. Judged with the expiry. */
128
+ lockWaits: readonly LockWait[];
109
129
  }
110
130
 
111
131
  type GrantedTokens = Extract<TokenGrant, { granted: true }>;
@@ -115,7 +135,10 @@ export interface ProcessTreeKiller {
115
135
  }
116
136
 
117
137
  /** Where the command's TMPDIR comes from; `null` leaves the inherited one. */
118
- export type TempVolume = Pick<RamTempVolume, 'acquire' | 'release' | 'fullness' | 'space' | 'reap'>;
138
+ export type TempVolume = Pick<
139
+ RamTempVolume,
140
+ 'acquire' | 'release' | 'fullness' | 'space' | 'record' | 'ordinaryParent'
141
+ >;
119
142
 
120
143
  /** Workspace-provided cowshed launchers, in the order a workspace that provides one names them. */
121
144
  const WORKSPACE_COWSHED_LAUNCHERS = [
@@ -170,10 +193,38 @@ export default function boundedExecExecutor(
170
193
  return runDefaultBoundedExec(options, recordContext(context));
171
194
  }
172
195
 
173
- /** Runs with the normal process, RAM lease and CPU-budget owners, with an optional stdout sink. */
196
+ /**
197
+ * Runs with the normal process, RAM lease and CPU pool owners, with an optional stdout sink, on the user's RAM temp
198
+ * volume where there is one, through the host's own gateway: the Nx executor's runs, never a test's.
199
+ */
174
200
  export function runDefaultBoundedExec(
175
201
  options: BoundedExecOptions,
176
202
  context: BoundedExecContext,
203
+ ): Promise<BoundedExecResult> {
204
+ // Only macOS has `hdiutil ram://`; elsewhere the command keeps the inherited TMPDIR.
205
+ return runBoundedExecWith(options, context, (cowshed) =>
206
+ process.platform === 'darwin'
207
+ ? new RamTempVolume(
208
+ ramTempPaths(userInfo(), PRODUCTION),
209
+ RAM_TEMP_CAPACITY_BYTES,
210
+ createHostCommands(GATEWAY_SOCKET, undefined, undefined, cowshed),
211
+ undefined,
212
+ undefined,
213
+ 'optional',
214
+ )
215
+ : null,
216
+ );
217
+ }
218
+
219
+ /**
220
+ * The default runner with every owner but its RAM volume, which its caller makes: `volume` makes the volume for the
221
+ * cowshed the run's disk commands go through. The Nx executor makes the user's; a host proof passes a fixture volume
222
+ * (`fixtureRamVolume`), so no test leases, creates or detaches the user's volume.
223
+ */
224
+ export function runBoundedExecWith(
225
+ options: BoundedExecOptions,
226
+ context: BoundedExecContext,
227
+ volume: (cowshed: string | undefined) => TempVolume | null,
177
228
  ): Promise<BoundedExecResult> {
178
229
  // A target that names the cowshed its disk commands run through names it for this run's own RAM volume work and
179
230
  // for its command alike. Nx resolves `{workspaceRoot}/…` in an option to a workspace-relative path, which the
@@ -188,19 +239,14 @@ export function runDefaultBoundedExec(
188
239
  );
189
240
  const run: BoundedExecOptions =
190
241
  cowshed === undefined ? options : { ...options, env: { ...options.env, [COWSHED_EXECUTABLE_ENV]: cowshed } };
191
- // Only macOS has `hdiutil ram://`; elsewhere the command keeps the inherited TMPDIR.
192
- const tempVolume =
193
- process.platform === 'darwin'
194
- ? new RamTempVolume(
195
- ramTempPaths(userInfo()),
196
- RAM_TEMP_CAPACITY_BYTES,
197
- createHostCommands(undefined, undefined, undefined, cowshed),
198
- )
199
- : null;
242
+ const tempVolume = volume(cowshed);
243
+ // The command's processes are owned by the supervisor that serves this job, when one does
244
+ // (specs/cowshed/07_api.md, `JobSubtreeHandle`); with none, nothing here may end what it leaves.
245
+ const owner = process.env[SUPERVISOR_SOCKET_ENV] === undefined ? null : cowshedSubtreeOwner(cowshed ?? 'cowshed');
200
246
  // A command a parent's CPU pool started draws its tokens from that pool. The pool is named by the environment the
201
247
  // parent set for this command; a run no parent started has none.
202
248
  const inherited = readInheritedCpuPool(process.env);
203
- return runBoundedExec(run, context, createProcessTreeKiller(), tempVolume, gatewayCpuBudget(), inherited);
249
+ return runBoundedExec(run, context, createProcessTreeKiller(), tempVolume, inherited, captureHangDiagnostics, owner);
204
250
  }
205
251
 
206
252
  /**
@@ -227,24 +273,22 @@ export function recordContext(context: NxTaskContext): BoundedExecContext {
227
273
  }
228
274
 
229
275
  /**
230
- * Run the command once its runner holds CPU tokens from `budget` (`cpu-tokens.ts`), sized to the
231
- * grant; `null`, or a budget nobody serves, runs it as asked. Both bounds start once it is
232
- * granted: a wait for the host is not the command's time.
276
+ * Run the command, sized to the tokens its parent's CPU pool (`cpu-pool.ts`) lends it; a run that inherits no pool
277
+ * runs it as asked. Both bounds start once it is granted: a wait for the pool is not the command's time.
233
278
  *
234
- * `inherited` is the parent's CPU pool (`cpu-pool.ts`) that lends this run its tokens. The run asks that pool first
235
- * and the host budget only when the pool refuses. Whatever it is granted, the run serves its own sub-pool to its
236
- * descendants, closed before the grant returns. A grant whose connection closes while the command runs (a gateway
237
- * restart, or the lending parent's exit) ends its accounting, never the command: the command runs on over budget to
238
- * its own exit, and the run says so.
279
+ * `inherited` is the parent's pool: an enclosing bounded-exec's sub-pool or the Nx run's own (`run-cpu-pool.ts`). A
280
+ * granted run serves a sub-pool of its grant to its own descendants, closed before the grant returns. A grant whose
281
+ * connection closes while the command runs (the lending parent's exit) ends its accounting, never the command: the
282
+ * command runs on over budget to its own exit, and the run says so.
239
283
  */
240
284
  export async function runBoundedExec(
241
285
  options: BoundedExecOptions,
242
286
  context: BoundedExecContext,
243
287
  killer: ProcessTreeKiller,
244
288
  tempVolume: TempVolume | null,
245
- budget: CpuBudget | null,
246
289
  inherited: InheritedCpuPool | null = null,
247
290
  captureDiagnostics: typeof captureHangDiagnostics = captureHangDiagnostics,
291
+ owner: SubtreeOwner | null = null,
248
292
  ): Promise<BoundedExecResult> {
249
293
  const cwd = resolveCwd(options.cwd, context.root);
250
294
  const directory = context.record?.directory ?? null;
@@ -256,13 +300,19 @@ export async function runBoundedExec(
256
300
  const timeoutMs = options.timeoutMs;
257
301
  const idleTimeoutMs = options.idleTimeoutMs;
258
302
  const killAfterMs = options.killAfterMs ?? DEFAULT_KILL_AFTER_MS;
303
+ const fixture: { command: string; timeoutMs: number } | null =
304
+ options.fixtureCommand !== undefined && options.fixtureTimeoutMs !== undefined
305
+ ? { command: options.fixtureCommand, timeoutMs: options.fixtureTimeoutMs }
306
+ : null;
259
307
  const outputChunks: string[] = [];
260
308
  const state: RunState = {
261
309
  settled: false,
262
310
  expiry: null,
263
311
  forceKillNeeded: false,
264
312
  poolCloseError: null,
313
+ retireError: null,
265
314
  noSpace: null,
315
+ lockWaits: [],
266
316
  };
267
317
  const appendStdout = (chunk: Buffer | string): void => {
268
318
  const text = chunk.toString();
@@ -278,10 +328,62 @@ export async function runBoundedExec(
278
328
  outputChunks.push(text);
279
329
  process.stderr.write(text);
280
330
  };
331
+ // Told, never thrown, wherever the command is being ended: a diagnostic must neither replace the failure it
332
+ // accompanies nor skip the cleanup that follows (the signal handlers, the CPU grant, the RAM lease, the run's record).
333
+ const tell = (text: string): void => {
334
+ try {
335
+ appendStderr(text);
336
+ } catch {
337
+ // Nobody is left to tell.
338
+ }
339
+ };
340
+
341
+ // The fixture pair names one segment: a fixture with no build budget, or a budget with no fixture, is a
342
+ // target that does not say what it runs, so it does not run.
343
+ if (
344
+ (options.fixtureCommand === undefined) !== (options.fixtureTimeoutMs === undefined) ||
345
+ (fixture !== null && !(fixture.timeoutMs > 0))
346
+ ) {
347
+ appendStderr(
348
+ 'bounded-exec: fixtureCommand and a positive fixtureTimeoutMs must be named together; refusing to run the command\n',
349
+ );
350
+ if (context.record !== undefined) {
351
+ const { task, directory: recordDirectory } = context.record;
352
+ await writeRecord(recordDirectory, { task: task.id, hash: task.hash, verdict: judgeOperationalFailure(1, null) });
353
+ }
354
+ return { success: false, terminalOutput: outputChunks.join('') };
355
+ }
356
+
357
+ // A leg that runs bun runs the Bun its workspace declares, or it does not run. This comes before the RAM lease and the
358
+ // CPU tokens: a refusal that waited in its pool's queue first would say nothing for minutes.
359
+ const runtime = await checkRuntimePin({
360
+ root: context.root,
361
+ command: fixture === null ? requested : `${fixture.command} && ${requested}`,
362
+ env: { ...withoutPoolEnv(process.env), ...options.env },
363
+ cwd,
364
+ pathSource: options.env?.PATH === undefined ? 'the inherited environment' : "the target's env",
365
+ });
366
+ if (runtime !== null) {
367
+ appendStderr(describeRuntimePinRefusal(runtime, context.record?.task.id));
368
+ if (context.record !== undefined) {
369
+ const { task, directory: recordDirectory } = context.record;
370
+ await writeRecord(recordDirectory, { task: task.id, hash: task.hash, verdict: judgeOperationalFailure(1, null) });
371
+ }
372
+ return { success: false, terminalOutput: outputChunks.join('') };
373
+ }
281
374
 
282
375
  let lease: RamTempLease | null = null;
283
- if (tempVolume !== null) {
284
- const acquired = await tempVolume.acquire();
376
+ let ordinaryTemp: string | null = null;
377
+ // Before spawn there is nothing to retire. Once started, only the owner's terminal retirement permits deletion.
378
+ let subtreeRetired = true;
379
+ // A lease is removed when its command ends, so it is taken only where whatever the command leaves
380
+ // in it can be ended by its owner first: the supervisor's subtree retirement, never a census.
381
+ if (tempVolume !== null && owner === null) {
382
+ appendStderr(
383
+ `RAM temp volume not leased: no cowshed supervisor owns this command's processes (${SUPERVISOR_SOCKET_ENV} is unset); TMPDIR stays ${process.env.TMPDIR ?? '(unset)'}\n`,
384
+ );
385
+ } else if (tempVolume !== null && owner !== null) {
386
+ const acquired = await tempVolume.acquire(owner);
285
387
  if (!acquired.ok) {
286
388
  const message = `${describeRamTempError(acquired.error)}\n`;
287
389
  process.stderr.write(message);
@@ -293,7 +395,28 @@ export async function runBoundedExec(
293
395
  appendStderr(`RAM temp volume: dead lease ${held}\n`);
294
396
  }
295
397
  for (const reaped of acquired.value.reaped) {
296
- appendStderr(`RAM temp volume: stopped ${reaped}, which a dead task left working in its lease\n`);
398
+ appendStderr(`RAM temp volume: the supervisor stopped ${reaped}, which a dead task's command left running\n`);
399
+ }
400
+ for (const gap of acquired.value.gaps) {
401
+ appendStderr(`RAM temp volume: the supervisor's view of a dead task's command has a gap: ${gap}\n`);
402
+ }
403
+ for (const retained of acquired.value.retained) {
404
+ appendStderr(`RAM temp volume: dead lease ${retained}\n`);
405
+ }
406
+ } else if (acquired.value.kind === 'unavailable') {
407
+ const unavailable = `RAM temp volume unavailable (${acquired.value.reason}: ${acquired.value.detail})`;
408
+ if (options.env?.TMPDIR !== undefined) {
409
+ appendStderr(`${unavailable}; using target-provided TMPDIR ${options.env.TMPDIR} (medium not inferred)\n`);
410
+ } else {
411
+ try {
412
+ ordinaryTemp = await mkdtemp(join(tempVolume.ordinaryParent, `smoo-bounded-exec-${process.pid}-`));
413
+ } catch (error) {
414
+ appendStderr(
415
+ `${unavailable}; could not create ordinary scratch under ${tempVolume.ordinaryParent}: ${error instanceof Error ? error.message : String(error)}\n`,
416
+ );
417
+ return { success: false, terminalOutput: outputChunks.join('') };
418
+ }
419
+ appendStderr(`${unavailable}; using owned ordinary TMPDIR ${ordinaryTemp}\n`);
297
420
  }
298
421
  } else {
299
422
  appendStderr(
@@ -301,37 +424,95 @@ export async function runBoundedExec(
301
424
  );
302
425
  }
303
426
  }
304
- const releaseLease = async (): Promise<void> => {
305
- if (lease === null) {
427
+ // The supervisor's answer to the command's own admission, once it is spawned as an owned subtree root, and once
428
+ // any lease has recorded the handle for whoever settles that lease should this task die first.
429
+ let admission: Promise<SubtreeAdmission> | null = null;
430
+ const retireSubtree = async (): Promise<void> => {
431
+ if (admission === null || owner === null) {
306
432
  return;
307
433
  }
308
- const held = lease;
309
- lease = null;
310
- // What the command left working in its lease is a leak, whichever way the command ended; it
311
- // is stopped before the lease goes, so the volume can detach, and said, so its owner fixes it.
312
- const reaped = await tempVolume?.reap(held);
313
- if (reaped?.ok) {
314
- for (const leftover of reaped.value) {
315
- appendStderr(`RAM temp volume: stopped ${leftover}, which the command left working in its lease\n`);
316
- }
317
- } else if (reaped) {
318
- appendStderr(`${describeRamTempError(reaped.error)}\n`);
434
+ const answer = await admission;
435
+ admission = null;
436
+ if (answer.kind !== 'admitted') {
437
+ appendStderr(
438
+ `cowshed: the command ran without an owned subtree (${answer.kind === 'refused' ? `supervisor: ${answer.reason}` : answer.detail}); nothing retires what it left running\n`,
439
+ );
440
+ return;
319
441
  }
320
- const released = await tempVolume?.release(held);
321
- if (released && !released.ok) {
322
- appendStderr(`${describeRamTempError(released.error)}\n`);
442
+ // What the command left running is a leak, whichever way the command ended: its supervisor ends exactly the
443
+ // processes its tree records under the command, before any lease goes, and each is said so its owner fixes it.
444
+ const retired = await owner.retire(answer.handle, LEFTOVER_GRACE_MS);
445
+ switch (retired.kind) {
446
+ case 'retired':
447
+ subtreeRetired = true;
448
+ for (const entry of retired.signalled) {
449
+ appendStderr(`cowshed: the supervisor stopped ${describeSignalled(entry)}, which the command left running\n`);
450
+ }
451
+ if (retired.signalledIncomplete) {
452
+ appendStderr("cowshed: the supervisor may have stopped processes of the command's that it could not name\n");
453
+ }
454
+ for (const gap of retired.gaps) {
455
+ appendStderr(
456
+ `cowshed: the supervisor's view of the command's processes has a gap: ${describeCoverageGap(gap)}\n`,
457
+ );
458
+ }
459
+ return;
460
+ case 'refused':
461
+ appendStderr(`cowshed: the supervisor refused to retire the command's processes (${retired.reason})\n`);
462
+ return;
463
+ case 'failed':
464
+ state.retireError = retired.detail;
465
+ appendStderr(`cowshed: could not retire the command's processes: ${retired.detail}\n`);
466
+ return;
323
467
  }
324
468
  };
469
+ // Failed scratch remains evidence: RAM leases are archived; ordinary scratch is retained at its reported path.
470
+ let tempReleased: Promise<void> | null = null;
471
+ const releaseLease = (keep: boolean): Promise<void> => {
472
+ tempReleased ??= (async () => {
473
+ await retireSubtree();
474
+ if (ordinaryTemp !== null) {
475
+ const held = ordinaryTemp;
476
+ ordinaryTemp = null;
477
+ if (keep || state.retireError !== null || !subtreeRetired) {
478
+ appendStderr(
479
+ `Ordinary temp directory retained at ${held}: ${!subtreeRetired ? 'the command subtree was not retired' : 'the run did not pass'}\n`,
480
+ );
481
+ } else {
482
+ try {
483
+ await rm(held, { recursive: true, force: true });
484
+ } catch (error) {
485
+ appendStderr(
486
+ `Ordinary temp directory retained at ${held}: cleanup failed: ${error instanceof Error ? error.message : String(error)}\n`,
487
+ );
488
+ }
489
+ }
490
+ }
491
+ if (lease === null || owner === null) {
492
+ return;
493
+ }
494
+ const held = lease;
495
+ lease = null;
496
+ const released = await tempVolume?.release(held, keep || state.retireError !== null, owner);
497
+ if (released?.ok) {
498
+ for (const retained of released.value) {
499
+ appendStderr(`RAM temp volume: lease ${retained}\n`);
500
+ }
501
+ } else if (released) {
502
+ appendStderr(`${describeRamTempError(released.error)}\n`);
503
+ }
504
+ })();
505
+ return tempReleased;
506
+ };
325
507
 
326
- // The wait is one span line, as cowshed's own spans read. On a host without a gateway the Nx run serves its own
327
- // pool (`run-cpu-pool.ts`), taken below as a parent's; a command no pool and no gateway answers runs exactly as
508
+ // The wait is one span line, as cowshed's own spans read. Every Nx run that inherits no pool serves its own
509
+ // (`run-cpu-pool.ts`), so a task takes its tokens from a parent's pool; a command that inherits none runs exactly as
328
510
  // asked and nothing is said.
329
511
  const cores = availableParallelism();
330
512
  const demand = runnerDemand(requested, options.parallelism, cores, process.env, options.env);
331
513
  const asked = Date.now();
332
- // A run a parent's CPU pool serves takes its tokens there. Only an absent pool leaves the host budget to ask.
333
- // A pool that answered and refused is a live budget of its own -- a lent grant, or a gate's or a resident's
334
- // relay to the host -- so asking the host beside it would count the run twice; the run fails instead.
514
+ // A pool that answered and refused is a live budget, and running beside it would count the run twice: the run fails
515
+ // instead. A pool whose endpoint is gone (its holder exited) lends nothing, and the run goes on as asked.
335
516
  let grant: GrantedTokens | null = null;
336
517
  let parentRefusal: string | null = null;
337
518
  if (inherited !== null) {
@@ -339,32 +520,19 @@ export async function runBoundedExec(
339
520
  if (fromParent.granted) {
340
521
  grant = fromParent;
341
522
  } else if (fromParent.cause === 'absent') {
342
- appendStderr(`cowshed: cpu-tokens parent pool refused: ${fromParent.reason}; asking the host budget\n`);
523
+ appendStderr(`cowshed: cpu-pool parent pool absent: ${fromParent.reason}; running the command as asked\n`);
343
524
  } else {
344
525
  parentRefusal = fromParent.reason;
345
526
  }
346
527
  }
347
- let unbudgeted: string | null = null;
348
- if (grant === null && parentRefusal === null && budget !== null) {
349
- const fromHost = await budget.take(demand.want, context.root, requested);
350
- if (fromHost.granted) {
351
- grant = fromHost;
352
- } else if (fromHost.cause !== 'absent') {
353
- unbudgeted = fromHost.reason;
354
- }
355
- }
356
528
  const waitedMs = Date.now() - asked;
357
529
  let cpu: BoundedExecRecord['cpu'];
358
530
  let sized: { command: string; env: Record<string, string> } = { command: requested, env: {} };
359
531
  if (grant !== null) {
360
532
  sized = sizeRunner(demand, requested, grant.tokens);
361
- cpu = { tokens: grant.tokens, want: demand.want, waitedMs, ...(grant.inherited ? { inherited: true } : {}) };
362
- appendStderr(
363
- `cowshed: cpu-tokens wait done elapsed=${waitedMs}ms status=ok ${grant.inherited ? 'inherited=true ' : ''}tokens=${grant.tokens}/${demand.want} runner=${demand.kind}\n`,
364
- );
365
- } else if (unbudgeted !== null) {
533
+ cpu = { tokens: grant.tokens, want: demand.want, waitedMs };
366
534
  appendStderr(
367
- `cowshed: cpu-tokens wait done elapsed=${waitedMs}ms status=err runner=${demand.kind} runs unbudgeted: ${unbudgeted}\n`,
535
+ `cowshed: cpu-pool wait done elapsed=${waitedMs}ms status=ok tokens=${grant.tokens}/${demand.want} runner=${demand.kind}\n`,
368
536
  );
369
537
  }
370
538
  // The run's sub-pool and grant are given back exactly once, however the run ends. The sub-pool closes first, so no
@@ -386,14 +554,15 @@ export async function runBoundedExec(
386
554
  return cpuReleased;
387
555
  };
388
556
  // Records the run's verdict when it has a task to record as. An `expiry` is set only when a bound ended the run.
389
- // An `operational` run failed on its CPU budget (a sub-pool that did not close): an ordinary failure, whatever
390
- // bound also fired, so it is never judged as a bound.
557
+ // An `operational` run failed on its CPU pool (a sub-pool that did not close): an ordinary failure, whatever bound also
558
+ // fired, so it is never judged as a bound.
391
559
  const recordRun = async (
392
560
  exitCode: number,
393
561
  expiry: ExpiredBound | null,
394
562
  elapsedMs: number,
395
563
  operational: boolean,
396
564
  noSpace: NoSpaceSighting | null,
565
+ lockWaits: readonly LockWait[],
397
566
  ): Promise<void> => {
398
567
  if (context.record === undefined) {
399
568
  return;
@@ -402,7 +571,7 @@ export async function runBoundedExec(
402
571
  const report = await readJunitReport(recordDirectory);
403
572
  const verdict = operational
404
573
  ? judgeOperationalFailure(exitCode, report)
405
- : judgeRun({ exitCode, elapsedMs, expiry, report, noSpace });
574
+ : judgeRun({ exitCode, elapsedMs, expiry, report, noSpace, lockWaits });
406
575
  if (verdict.outcome === 'bound' && verdict.bound === 'test') {
407
576
  appendStderr(`Every failing test failed only on its per-test timeout: ${verdict.tests.join(', ')}\n`);
408
577
  }
@@ -412,36 +581,19 @@ export async function runBoundedExec(
412
581
  const refuse = async (message: string): Promise<BoundedExecResult> => {
413
582
  appendStderr(message);
414
583
  await releaseCpu();
415
- await releaseLease();
416
- await recordRun(1, null, Date.now() - asked, false, null);
584
+ // The command never started, so its lease holds nothing of its own to keep.
585
+ await releaseLease(false);
586
+ await recordRun(1, null, Date.now() - asked, false, null, []);
417
587
  return { success: false, terminalOutput: outputChunks.join('') };
418
588
  };
419
589
  if (parentRefusal !== null) {
420
- return refuse(
421
- `cowshed: cpu-tokens parent pool refused: ${parentRefusal}; refusing to run the command without asking the host budget\n`,
422
- );
590
+ return refuse(`cowshed: cpu-pool parent pool refused: ${parentRefusal}; refusing to run the command\n`);
423
591
  }
424
592
 
425
- // Descendant work queued for host CPU tokens, as its pool hears it (`host-wait`): while any is held the idle clock
426
- // is stopped, because a queued job is not an idle one (specs/cowshed/05_gateway.md, "The bounds start at the
427
- // grant"). The clock it stops is made with the command below; a wait held before then is counted and honoured there.
428
- let heldWaits = 0;
429
- let idleClock: { stop(): void; restart(): void } | null = null;
430
- const hostWaits: HostWaits = {
431
- begin: () => {
432
- heldWaits += 1;
433
- if (heldWaits === 1) idleClock?.stop();
434
- },
435
- end: () => {
436
- heldWaits -= 1;
437
- if (heldWaits === 0) idleClock?.restart();
438
- },
439
- };
440
-
441
593
  // A granted run serves a sub-pool of its own grant, so its descendants draw from that and never from above it.
442
- // A pool that cannot be set up refuses the run: no command runs unbudgeted.
594
+ // A pool that cannot be set up refuses the run: no granted command runs beside its grant.
443
595
  if (grant !== null) {
444
- const created = await createCpuPool(grant.tokens, grant.closed, hostWaits);
596
+ const created = await createCpuPool(grant.tokens, grant.closed);
445
597
  if (!created.ok) {
446
598
  return refuse(`cowshed: cpu pool unavailable: ${created.error}; refusing to run the command\n`);
447
599
  }
@@ -461,25 +613,61 @@ export async function runBoundedExec(
461
613
  ...sized.env,
462
614
  ...(pool?.environment ?? {}),
463
615
  };
464
- const hooked = process.platform === 'win32' ? null : withRunnerLifecycle(command);
616
+ // A fixture rides as segment 0 of the lifecycle. Without the channel (Windows) it is plain shell text ahead
617
+ // of the command: the progress bound runs during it and its own ceiling does not.
618
+ const hooked =
619
+ process.platform === 'win32'
620
+ ? null
621
+ : fixture === null
622
+ ? withRunnerLifecycle(command)
623
+ : withFixtureLifecycle(fixture.command, command);
465
624
  const lifecycle = hooked === null ? null : createRunnerLifecycle(hooked.runners);
466
625
  let lifecycleFailure: string | null = null;
467
626
  const isTesting = (): boolean => lifecycleFailure !== null || (lifecycle?.isTesting() ?? true);
468
627
  const startedAt = Date.now();
628
+ // With a supervisor, `cowshed subtree-root` asks it for a handle on itself, writes the answer to the handle
629
+ // descriptor, and execs `/bin/sh -c command` in the same process: the command's pid, group and life are unchanged.
630
+ const shellCommand = hooked?.command ?? (fixture === null ? command : `${fixture.command} && ${command}`);
631
+ const root = owner?.rootArgv(shellCommand);
469
632
  let child: ChildProcess;
470
633
  try {
471
- child = spawn(hooked?.command ?? command, [], {
472
- cwd,
473
- env: mergeEnv(lease, env),
474
- shell: true,
475
- stdio: hooked === null ? 'pipe' : ['pipe', 'pipe', 'pipe', 'pipe'],
476
- detached: process.platform !== 'win32',
477
- windowsHide: true,
478
- });
634
+ // fd 3 is the runner lifecycle's duplex socket when the command is hooked; the subtree admission takes fd 4, which
635
+ // `cowshed subtree-root` closes before it execs, so fd 3 reaches the shell unchanged.
636
+ child =
637
+ root === undefined
638
+ ? spawn(shellCommand, [], {
639
+ cwd,
640
+ env: mergeEnv(lease?.directory ?? ordinaryTemp, env),
641
+ shell: true,
642
+ stdio: hooked === null ? 'pipe' : ['pipe', 'pipe', 'pipe', 'pipe'],
643
+ detached: process.platform !== 'win32',
644
+ windowsHide: true,
645
+ })
646
+ : spawn(root.file, root.args, {
647
+ cwd,
648
+ env: mergeEnv(lease?.directory ?? ordinaryTemp, env),
649
+ stdio: ['pipe', 'pipe', 'pipe', hooked === null ? 'ignore' : 'pipe', 'pipe'],
650
+ detached: process.platform !== 'win32',
651
+ windowsHide: true,
652
+ });
479
653
  } catch (error) {
480
654
  // Nothing started, so nothing runs under the grant: it goes back here, with the RAM lease.
481
655
  return refuse(`cowshed: could not start the command: ${error instanceof Error ? error.message : String(error)}\n`);
482
656
  }
657
+ subtreeRetired = false;
658
+ if (root !== undefined) {
659
+ const channel = child.stdio[HANDLE_FD];
660
+ const leased = lease;
661
+ admission = readAdmission(channel instanceof Readable ? channel : null).then(async (answer) => {
662
+ if (answer.kind === 'admitted' && leased !== null && tempVolume !== null) {
663
+ const recorded = await tempVolume.record(leased, answer.handle);
664
+ if (!recorded.ok) {
665
+ appendStderr(`${describeRamTempError(recorded.error)}\n`);
666
+ }
667
+ }
668
+ return answer;
669
+ });
670
+ }
483
671
 
484
672
  // Progress is a finished test, not output: a runner's SLOW warnings and a hung test's heartbeat keep printing.
485
673
  const progress = createRunProgress(command);
@@ -490,24 +678,22 @@ export async function runBoundedExec(
490
678
  const held = lease;
491
679
  return { line, volume: held === null ? null : ((await tempVolume?.space(held)) ?? null) };
492
680
  };
493
- let idleTimer: NodeJS.Timeout | undefined;
681
+ // Which bound is armed, and when it expires, is bounds.ts's decision; one timer waits for the earliest of them.
682
+ let bounds: Bounds = UNARMED;
683
+ let boundTimer: NodeJS.Timeout | undefined;
684
+ // The fixture's own output is compilation, not the command's progress: it is not scanned for finished tests.
685
+ let fixtureDone = fixture === null || lifecycle === null;
686
+ // Whether a segment past the fixture ever announced: without one, the shell's exit is the fixture's.
687
+ let mainStarted = fixture === null || lifecycle === null;
494
688
  let captureTimer: NodeJS.Timeout | undefined;
495
689
  let idleCapture: { expiresAtMs: number; controller: AbortController; result: IdleDiagnosticResult } | undefined;
496
690
  const cancelCapture = (): void => {
497
691
  clearTimeout(captureTimer);
498
692
  idleCapture?.controller.abort();
499
693
  };
500
- // What is left of the current no-progress interval. A finished test sets it back to `idleTimeoutMs`; a held host
501
- // wait keeps it unspent: the clock is stopped while descendant work is queued for CPU tokens, and runs on from where
502
- // it stopped once the last wait ends. `queuedMs` is the queued time this interval has not counted, for the report.
503
- let idleLeftMs = idleTimeoutMs ?? 0;
504
- let queuedMs = 0;
505
- let queuedSinceMs: number | null = null;
506
- const scheduleIdle = (limitMs: number, leftMs: number): void => {
507
- clearTimeout(idleTimer);
508
- cancelCapture();
694
+ const scheduleCapture = (expiresAtMs: number): void => {
509
695
  const capture: NonNullable<typeof idleCapture> = {
510
- expiresAtMs: Date.now() + leftMs,
696
+ expiresAtMs,
511
697
  controller: new AbortController(),
512
698
  result: { kind: 'pending', startedAtMs: null },
513
699
  };
@@ -545,70 +731,45 @@ export async function runBoundedExec(
545
731
  },
546
732
  );
547
733
  },
548
- Math.max(0, leftMs - HANG_CAPTURE_MS),
734
+ Math.max(0, expiresAtMs - Date.now() - HANG_CAPTURE_MS),
549
735
  );
550
- idleTimer = setTimeout(() => expire({ kind: 'idle', limitMs, tests: progress.inFlight() }, queuedMs), leftMs);
551
736
  };
552
- const armIdleTimer = (): void => {
553
- if (idleTimeoutMs === undefined || state.settled || state.expiry !== null || !isTesting()) {
554
- return;
555
- }
556
- idleLeftMs = idleTimeoutMs;
557
- queuedMs = 0;
558
- if (queuedSinceMs !== null) {
559
- queuedSinceMs = Date.now();
560
- return;
561
- }
562
- scheduleIdle(idleTimeoutMs, idleLeftMs);
563
- };
564
- idleClock = {
565
- stop: () => {
566
- if (idleTimeoutMs === undefined || state.settled || state.expiry !== null || queuedSinceMs !== null) {
567
- return;
568
- }
569
- const now = Date.now();
570
- queuedSinceMs = now;
571
- if (idleCapture !== undefined) {
572
- idleLeftMs = Math.max(0, idleCapture.expiresAtMs - now);
573
- }
574
- clearTimeout(idleTimer);
737
+ const expiredAs = (kind: BoundKind): ExpiredBound =>
738
+ kind === 'idle'
739
+ ? { kind, limitMs: idleTimeoutMs ?? 0, tests: progress.inFlight() }
740
+ : kind === 'fixture'
741
+ ? { kind, limitMs: fixture?.timeoutMs ?? 0 }
742
+ : { kind, limitMs: timeoutMs ?? 0 };
743
+ const stepTo = (event: BoundEvent): void => {
744
+ if (state.settled || state.expiry !== null) return;
745
+ const idleBefore = bounds.idle;
746
+ bounds = stepBounds(
747
+ bounds,
748
+ {
749
+ ...(idleTimeoutMs === undefined ? {} : { idleMs: idleTimeoutMs }),
750
+ ...(fixture === null || lifecycle === null ? {} : { fixtureMs: fixture.timeoutMs }),
751
+ ...(timeoutMs === undefined ? {} : { totalMs: timeoutMs }),
752
+ },
753
+ event,
754
+ );
755
+ if (bounds.idle !== idleBefore) {
575
756
  cancelCapture();
576
757
  idleCapture = undefined;
577
- appendStderr(
578
- `bounded-exec: idle clock stopped: a descendant's work is queued for host CPU tokens; ${idleLeftMs}ms of idleTimeoutMs=${idleTimeoutMs} left\n`,
579
- );
580
- },
581
- restart: () => {
582
- if (idleTimeoutMs === undefined || state.settled || state.expiry !== null || queuedSinceMs === null) {
583
- return;
584
- }
585
- const waitedMs = Date.now() - queuedSinceMs;
586
- queuedMs += waitedMs;
587
- queuedSinceMs = null;
588
- if (!isTesting()) {
589
- appendStderr(`bounded-exec: host CPU wait ended after ${waitedMs}ms; no test runner is active\n`);
590
- return;
591
- }
592
- appendStderr(
593
- `bounded-exec: idle clock restarted: no descendant's work is queued for host CPU tokens after ${waitedMs}ms; ${idleLeftMs}ms of idleTimeoutMs=${idleTimeoutMs} left\n`,
594
- );
595
- scheduleIdle(idleTimeoutMs, idleLeftMs);
596
- },
758
+ if (bounds.idle !== null) scheduleCapture(bounds.idle);
759
+ }
760
+ clearTimeout(boundTimer);
761
+ const next = earliestBound(bounds);
762
+ if (next !== null) boundTimer = setTimeout(() => expire(expiredAs(next.kind)), Math.max(0, next.at - Date.now()));
597
763
  };
598
764
 
599
765
  // Only the CHILD's own output is read for progress. The diagnostics below go
600
766
  // through appendStderr as well, and re-arming on those would let a teardown
601
767
  // message extend the very bound that just fired.
602
768
  const updateIdle = (finished: boolean, wasTesting: boolean): void => {
603
- if (!isTesting()) {
604
- clearTimeout(idleTimer);
605
- cancelCapture();
606
- idleCapture = undefined;
607
- } else if (finished || !wasTesting) {
608
- armIdleTimer();
609
- }
769
+ stepTo({ kind: 'progress', at: Date.now(), testing: isTesting(), wasTesting, finished });
610
770
  };
611
771
  const scanProgress = (stream: 'stdout' | 'stderr', text: string): void => {
772
+ if (!fixtureDone) return;
612
773
  updateIdle(progress.scan(stream, text), isTesting());
613
774
  };
614
775
  if (lifecycle !== null) {
@@ -620,7 +781,8 @@ export async function runBoundedExec(
620
781
  appendStderr(`bounded-exec: runner lifecycle failed: ${detail}\n`);
621
782
  updateIdle(false, wasTesting);
622
783
  };
623
- createInterface({ input: channel }).on('line', (line) => {
784
+ const lines = createInterface({ input: channel });
785
+ lines.on('line', (line) => {
624
786
  const wasTesting = isTesting();
625
787
  const event = lifecycle.accept(line);
626
788
  if (event === null) {
@@ -628,9 +790,17 @@ export async function runBoundedExec(
628
790
  return;
629
791
  }
630
792
  updateIdle(false, wasTesting);
793
+ if (event.id === 0 && event.phase === 'terminal' && fixture !== null) {
794
+ fixtureDone = true;
795
+ stepTo({ kind: 'fixture-done' });
796
+ }
797
+ if (event.id > 0) mainStarted = true;
631
798
  // Tool startup and native lifecycle-bearing stdout wait for their phase acknowledgement.
632
799
  channel.write(`${JSON.stringify(event)}\n`);
633
800
  });
801
+ // readline re-emits its input's errors on the interface, where one with no listener is thrown: a
802
+ // runner killed with unread lifecycle bytes resets the channel (ECONNRESET on Linux).
803
+ lines.on('error', (error) => failLifecycle(error.message));
634
804
  channel.on('error', (error) => failLifecycle(error.message));
635
805
  channel.on('end', () => {
636
806
  if (!lifecycle.finished()) failLifecycle('supervisor channel ended without an owned terminal event');
@@ -651,23 +821,30 @@ export async function runBoundedExec(
651
821
  appendStderr(text);
652
822
  });
653
823
 
654
- const killChildTree = async (force: boolean): Promise<void> => {
824
+ // The graceful pass: SIGTERM to the whole group. A signal that fails is told, not thrown: every caller has more to do
825
+ // after it, `terminate` the force-kill and the signal handler its releases and the signal it gives back.
826
+ const killChildTree = async (): Promise<void> => {
655
827
  const pid = child.pid;
656
828
  if (!pid) {
657
829
  return;
658
830
  }
659
- await killer.kill(pid, force ? 'SIGKILL' : 'SIGTERM');
831
+ try {
832
+ await killer.kill(pid, 'SIGTERM');
833
+ } catch (error) {
834
+ const reason = error instanceof Error ? error.message : String(error);
835
+ tell(`Terminating process group ${pid} failed: ${reason}\n`);
836
+ }
660
837
  };
661
838
 
662
839
  const onProcessExit = (): void => {
663
- void killChildTree(false);
840
+ void killChildTree();
664
841
  };
665
842
  const onTerminationSignal = (signal: NodeJS.Signals): void => {
666
843
  removeSignalHandlers();
667
844
  cancelCapture();
668
- void killChildTree(false)
845
+ void killChildTree()
669
846
  .finally(releaseCpu)
670
- .finally(releaseLease)
847
+ .finally(() => releaseLease(true))
671
848
  .finally(() => process.kill(process.pid, signal));
672
849
  };
673
850
  const onSigint = (): void => onTerminationSignal('SIGINT');
@@ -706,28 +883,27 @@ export async function runBoundedExec(
706
883
  // the reader unable to tell a wedged toolchain from a loaded machine — the
707
884
  // two have opposite fixes.
708
885
  let termination: Promise<void> = Promise.resolve();
709
- // `queuedMs` is the time the idle interval did not count because descendant work was queued for host CPU tokens.
710
- const expire = (bound: ExpiredBound, queuedMs = 0): void => {
886
+ const expire = (bound: ExpiredBound): void => {
711
887
  if (state.settled || state.expiry !== null) {
712
888
  return;
713
889
  }
714
890
  state.expiry = bound;
715
- clearTimeout(totalTimer);
716
- clearTimeout(idleTimer);
891
+ clearTimeout(boundTimer);
717
892
  const elapsedMs = Date.now() - startedAt;
718
893
  appendStderr(
719
894
  bound.kind === 'idle'
720
- ? `\nCommand made no progress: no test finished for ${bound.limitMs}ms (idleTimeoutMs=${bound.limitMs})${
721
- queuedMs > 0 ? `, not counting ${queuedMs}ms queued for host CPU tokens,` : ''
722
- } after ${elapsedMs}ms of runtime; ${
895
+ ? `\nCommand made no progress: no test finished for ${bound.limitMs}ms (idleTimeoutMs=${bound.limitMs}) after ${elapsedMs}ms of runtime; ${
723
896
  bound.tests.length > 0
724
897
  ? `still running: ${bound.tests.join(', ')}`
725
898
  : 'the runner named no test still running'
726
899
  } (cwd=${cwd}): ${command}\n`
727
- : `\nCommand timed out after ${elapsedMs}ms (timeoutMs=${bound.limitMs}, cwd=${cwd}): ${command}\n`,
900
+ : bound.kind === 'fixture'
901
+ ? `\nFixture command timed out after ${elapsedMs}ms (fixtureTimeoutMs=${bound.limitMs}, cwd=${cwd}): ${fixture?.command ?? command}\n`
902
+ : `\nCommand timed out after ${elapsedMs}ms (timeoutMs=${bound.limitMs}, cwd=${cwd}): ${command}\n`,
728
903
  );
729
904
  if (bound.kind === 'idle') {
730
905
  const observed = idleCapture?.result;
906
+ state.lockWaits = observed?.kind === 'captured' ? observed.snapshot.lockWaits : [];
731
907
  appendStderr(
732
908
  `Hang diagnostics: runner pid=${child.pid ?? 'unavailable'} pgid=${
733
909
  process.platform === 'win32' ? 'unavailable' : (child.pid ?? 'unavailable')
@@ -753,24 +929,49 @@ export async function runBoundedExec(
753
929
  termination = terminate();
754
930
  };
755
931
 
932
+ // Whether the group still lists members. A probe that fails proves nothing about the group, so it is told, and the
933
+ // group is taken as listed: a force-kill that was not needed costs nothing, and one that was is not skipped.
934
+ const groupListed = (pid: number): boolean => {
935
+ try {
936
+ return processGroupHasMembers(pid);
937
+ } catch (error) {
938
+ const reason = error instanceof Error ? error.message : String(error);
939
+ tell(`Probing process group ${pid} for members failed: ${reason}; taking it as still listing members\n`);
940
+ return true;
941
+ }
942
+ };
943
+
756
944
  // A timed-out run ends with its whole process group, not with its leader.
757
945
  // The graceful signal reaches every member that exists when it is sent, but
758
946
  // a member that ignores SIGTERM, or one forked while the signal goes out
759
947
  // (macOS does not hold a fork back for a group signal in flight), outlives
760
948
  // a leader that exits on it. The grace ends when the leader exits or
761
949
  // killAfterMs passes, whichever comes first; whatever is left then is killed.
950
+ // SIGKILL is no different: a member forked while it goes out survives it too,
951
+ // so the force-kill is repeated while the group lists members. What the repeats
952
+ // cannot end is said, and a failure of the probe or of a signal is shown.
953
+ const forceKill = async (pid: number): Promise<void> => {
954
+ try {
955
+ if (await killProcessGroup(pid, DEADLINE_PATIENCE, (group) => killer.kill(group, 'SIGKILL'))) {
956
+ tell(`${describeMembersLeft(pid)}\n`);
957
+ }
958
+ } catch (error) {
959
+ const reason = error instanceof Error ? error.message : String(error);
960
+ tell(`Force-kill of process group ${pid} failed: ${reason}\n`);
961
+ }
962
+ };
762
963
  const terminate = async (): Promise<void> => {
763
- await ignoreKillError(killChildTree(false));
964
+ await killChildTree();
764
965
  if (!state.settled && killAfterMs > 0) {
765
966
  await settledWithin(exitPromise, killAfterMs);
766
967
  }
767
968
  if (state.settled) {
768
- if (child.pid !== undefined && processGroupHasMembers(child.pid)) {
969
+ if (child.pid !== undefined && groupListed(child.pid)) {
769
970
  state.forceKillNeeded = true;
770
971
  appendStderr(
771
972
  `Force-killing what the ${state.expiry === null ? 'command' : 'timed-out command'} left in its process group: ${command}\n`,
772
973
  );
773
- await ignoreKillError(killChildTree(true));
974
+ await forceKill(child.pid);
774
975
  }
775
976
  return;
776
977
  }
@@ -778,23 +979,25 @@ export async function runBoundedExec(
778
979
  appendStderr(
779
980
  `Force-killing ${state.expiry === null ? 'command' : 'timed out command'} after killAfterMs=${killAfterMs}: ${command}\n`,
780
981
  );
781
- await ignoreKillError(killChildTree(true));
782
- await settledWithin(exitPromise, REAP_AFTER_FORCE_KILL_MS);
982
+ if (child.pid !== undefined) await forceKill(child.pid);
983
+ // The repeats wait for the group to empty, the leader with it. Windows has no process groups, so there they have
984
+ // nothing to wait on once the one signal is sent, and the leader's exit is awaited as it always was.
985
+ if (process.platform === 'win32' || child.pid === undefined) {
986
+ await settledWithin(exitPromise, REAP_AFTER_FORCE_KILL_MS);
987
+ }
783
988
  if (!state.settled) {
784
989
  resolveExit({ code: 1, signal: 'SIGKILL' });
785
990
  }
786
991
  };
787
992
 
788
- const totalTimer =
789
- timeoutMs === undefined ? undefined : setTimeout(() => expire({ kind: 'total', limitMs: timeoutMs }), timeoutMs);
790
- armIdleTimer();
791
- // A wait its pool heard before the command started stops the clock it was not yet there to stop.
792
- if (heldWaits > 0) idleClock.stop();
993
+ // The fixture's ceiling lives and dies with its segment: the segment's terminal clears it, so it can never
994
+ // fire on a command the fixture already handed over to.
995
+ stepTo({ kind: 'spawn', at: Date.now(), testing: isTesting(), fixture: fixture !== null && lifecycle !== null });
793
996
 
794
- // A grant's connection is a ledger entry, not the command's owner (05_gateway.md "Host CPU budget": a gateway
795
- // restart forgets every grant, and what runs under it runs on over budget until it finishes). Its close, while the
796
- // command runs, is said and changes nothing else; only a bound or a signal stops the command. Our own release after
797
- // the command settled is not a loss; `cpuReleased` marks it.
997
+ // A grant's connection is a ledger entry, not the command's owner: the lending parent's exit ends it, and what runs
998
+ // under it runs on over budget until it finishes. Its close, while the command runs, is said and changes nothing
999
+ // else; only a bound or a signal stops the command. Our own release after the command settled is not a loss;
1000
+ // `cpuReleased` marks it.
798
1001
  if (grant !== null) {
799
1002
  const held = grant;
800
1003
  void held.closed.then(() => {
@@ -802,15 +1005,14 @@ export async function runBoundedExec(
802
1005
  return;
803
1006
  }
804
1007
  appendStderr(
805
- `cowshed: cpu-tokens grant ended: the ${held.inherited ? 'parent pool' : 'gateway'} closed its ${held.tokens}-token grant while the command ran; it runs on over budget until it ends: ${command}\n`,
1008
+ `cowshed: cpu-pool grant ended: the parent pool closed its ${held.tokens}-token grant while the command ran; it runs on over budget until it ends: ${command}\n`,
806
1009
  );
807
1010
  });
808
1011
  }
809
1012
 
810
1013
  const { code: exitCode, signal: exitSignal } = await exitPromise;
811
1014
  await termination;
812
- clearTimeout(totalTimer);
813
- clearTimeout(idleTimer);
1015
+ clearTimeout(boundTimer);
814
1016
  cancelCapture();
815
1017
  removeSignalHandlers();
816
1018
  // The runner is gone. Its sub-pool closes first, so no descendant still draws on tokens the grant returns;
@@ -819,7 +1021,11 @@ export async function runBoundedExec(
819
1021
 
820
1022
  const code = exitCode ?? signalToExitCode(exitSignal);
821
1023
  if (code !== 0 && state.expiry === null) {
822
- appendStderr(`Command exited with status ${code}: ${command}\n`);
1024
+ appendStderr(
1025
+ fixture !== null && !mainStarted
1026
+ ? `Fixture command exited with status ${code}: ${fixture.command}\n`
1027
+ : `Command exited with status ${code}: ${command}\n`,
1028
+ );
823
1029
  }
824
1030
 
825
1031
  if (state.expiry !== null && !state.forceKillNeeded) {
@@ -838,13 +1044,12 @@ export async function runBoundedExec(
838
1044
  appendStderr(`${describeRamTempError(full)}\n`);
839
1045
  }
840
1046
  }
841
- await releaseLease();
842
-
843
- // Broken resource ownership or a lost native lifecycle fails even if a shell branch later exited 0:
844
- // its verdict is an ordinary operational failure, never a successful or silently unbounded run.
845
- const operationalFailure = state.poolCloseError !== null || lifecycleFailure !== null;
846
- await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure, seen);
1047
+ await releaseLease(code !== 0 || state.expiry !== null || state.poolCloseError !== null || lifecycleFailure !== null);
847
1048
 
1049
+ // Broken resource ownership, a lost native lifecycle, or leftovers that could not be retired fail even if a shell
1050
+ // branch later exited 0: its verdict is an ordinary operational failure, never a successful or silently unbounded run.
1051
+ const operationalFailure = state.poolCloseError !== null || lifecycleFailure !== null || state.retireError !== null;
1052
+ await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure, seen, state.lockWaits);
848
1053
  return {
849
1054
  success: !operationalFailure && state.expiry === null && code === 0,
850
1055
  terminalOutput: outputChunks.join(''),
@@ -853,45 +1058,12 @@ export async function runBoundedExec(
853
1058
 
854
1059
  export function createProcessTreeKiller(): ProcessTreeKiller {
855
1060
  return {
856
- kill(pid, signal) {
857
- return killProcessGroup(pid, signal);
1061
+ async kill(pid, signal) {
1062
+ signalProcessGroup(pid, signal);
858
1063
  },
859
1064
  };
860
1065
  }
861
1066
 
862
- function killProcessGroup(pid: number, signal: NodeJS.Signals): Promise<void> {
863
- try {
864
- // On POSIX the executor spawns with detached: true, so the child leads a
865
- // process group of its own and -pid signals all of it at once, including
866
- // grandchildren that tree-walk libraries miss when parents die and
867
- // children get reparented to init. Only the group is named: the leader may
868
- // already be reaped, and its pid reused by an unrelated process, while the
869
- // group lives on in its other members. Windows has no process groups.
870
- process.kill(process.platform === 'win32' ? pid : -pid, signal);
871
- } catch {
872
- // ESRCH: the group (or the Windows process) has already exited.
873
- }
874
- return Promise.resolve();
875
- }
876
-
877
- /**
878
- * Whether any process is left in the group `pgid` leads. A group outlives its
879
- * reaped leader for as long as one member does, and no process can take its id
880
- * meanwhile. Windows has no process groups, so there is nothing left to name.
881
- */
882
- function processGroupHasMembers(pgid: number): boolean {
883
- if (process.platform === 'win32') {
884
- return false;
885
- }
886
- try {
887
- process.kill(-pgid, 0);
888
- return true;
889
- } catch (error) {
890
- // EPERM: a member exists but runs as another user.
891
- return error instanceof Error && 'code' in error && error.code === 'EPERM';
892
- }
893
- }
894
-
895
1067
  function resolveCwd(cwd: string | undefined, root: string): string {
896
1068
  if (!cwd) {
897
1069
  return root;
@@ -912,10 +1084,10 @@ function buildCommand(options: BoundedExecOptions): string {
912
1084
  return parts.join(' ');
913
1085
  }
914
1086
 
915
- /** The lease's directory is the command's TMPDIR unless the target names one itself. */
916
- function mergeEnv(lease: RamTempLease | null, env: Record<string, string>): NodeJS.ProcessEnv {
1087
+ /** Owned scratch is the command's TMPDIR unless the target names one itself. */
1088
+ function mergeEnv(directory: string | null, env: Record<string, string>): NodeJS.ProcessEnv {
917
1089
  const inherited = withoutPoolEnv(process.env);
918
- return lease === null ? { ...inherited, ...env } : { ...inherited, TMPDIR: lease.directory, ...env };
1090
+ return directory === null ? { ...inherited, ...env } : { ...inherited, TMPDIR: directory, ...env };
919
1091
  }
920
1092
 
921
1093
  /** The process's environment without the CPU pool's descriptor: a pool and its count only a grant may give. */
@@ -949,14 +1121,6 @@ function signalToExitCode(signal: NodeJS.Signals | null): number {
949
1121
  return EXIT_CODE_BY_SIGNAL[signal] ?? 1;
950
1122
  }
951
1123
 
952
- async function ignoreKillError(promise: Promise<void>): Promise<void> {
953
- try {
954
- await promise;
955
- } catch {
956
- // The process tree may have already exited between timeout and kill.
957
- }
958
- }
959
-
960
1124
  /** Resolves when `promise` settles or `ms` passes, whichever is first, leaving no timer behind. */
961
1125
  async function settledWithin(promise: Promise<unknown>, ms: number): Promise<void> {
962
1126
  let timer: NodeJS.Timeout | undefined;