@smoothbricks/nx-plugin 0.5.7 → 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (446) hide show
  1. package/README.md +1097 -360
  2. package/dist/.build-inputs.json +141 -0
  3. package/dist/bin/smoo-nx-cargo-toolchain.d.ts +3 -0
  4. package/dist/bin/smoo-nx-cargo-toolchain.d.ts.map +1 -0
  5. package/dist/bin/smoo-nx-cargo-toolchain.js +8 -0
  6. package/dist/bin/smoo-nx-forbidden-spellings.d.ts +3 -0
  7. package/dist/bin/smoo-nx-forbidden-spellings.d.ts.map +1 -0
  8. package/dist/bin/smoo-nx-forbidden-spellings.js +32 -0
  9. package/dist/bin/smoo-nx-nextest-timings.d.ts +3 -0
  10. package/dist/bin/smoo-nx-nextest-timings.d.ts.map +1 -0
  11. package/dist/bin/smoo-nx-nextest-timings.js +28 -0
  12. package/dist/bounded-test-policy.d.ts +1 -1
  13. package/dist/bounded-test-policy.d.ts.map +1 -1
  14. package/dist/bounded-test-policy.js +14 -15
  15. package/dist/build-state-output-policy.d.ts +95 -1
  16. package/dist/build-state-output-policy.d.ts.map +1 -1
  17. package/dist/build-state-output-policy.js +368 -30
  18. package/dist/bun-lock-scope.d.ts +44 -0
  19. package/dist/bun-lock-scope.d.ts.map +1 -0
  20. package/dist/bun-lock-scope.js +97 -0
  21. package/dist/cargo-build-script-env.d.ts +49 -0
  22. package/dist/cargo-build-script-env.d.ts.map +1 -0
  23. package/dist/cargo-build-script-env.js +324 -0
  24. package/dist/cargo-closure-input.d.ts +31 -4
  25. package/dist/cargo-closure-input.d.ts.map +1 -1
  26. package/dist/cargo-closure-input.js +66 -6
  27. package/dist/cargo-retention-hook.d.ts +17 -0
  28. package/dist/cargo-retention-hook.d.ts.map +1 -0
  29. package/dist/cargo-retention-hook.js +85 -0
  30. package/dist/cargo-source-hash.d.ts +25 -7
  31. package/dist/cargo-source-hash.d.ts.map +1 -1
  32. package/dist/cargo-source-hash.js +108 -43
  33. package/dist/cargo-toolchain-paths.d.ts +40 -0
  34. package/dist/cargo-toolchain-paths.d.ts.map +1 -0
  35. package/dist/cargo-toolchain-paths.js +84 -0
  36. package/dist/cargo-toolchain-policy.d.ts +25 -8
  37. package/dist/cargo-toolchain-policy.d.ts.map +1 -1
  38. package/dist/cargo-toolchain-policy.js +39 -5
  39. package/dist/cargo-workspace.d.ts +10 -4
  40. package/dist/cargo-workspace.d.ts.map +1 -1
  41. package/dist/cargo-workspace.js +65 -95
  42. package/dist/cross-check-policy.d.ts +9 -9
  43. package/dist/cross-check-policy.d.ts.map +1 -1
  44. package/dist/cross-check-policy.js +11 -11
  45. package/dist/disk-command-policy.d.ts.map +1 -1
  46. package/dist/disk-command-policy.js +1 -0
  47. package/dist/ensure-built-receipt.d.ts +22 -5
  48. package/dist/ensure-built-receipt.d.ts.map +1 -1
  49. package/dist/ensure-built-receipt.js +168 -48
  50. package/dist/ensure-built-witness.d.ts +27 -8
  51. package/dist/ensure-built-witness.d.ts.map +1 -1
  52. package/dist/ensure-built-witness.js +45 -15
  53. package/dist/ensure-built.d.ts.map +1 -1
  54. package/dist/ensure-built.js +9 -1
  55. package/dist/eslint-config-closure.d.ts +7 -0
  56. package/dist/eslint-config-closure.d.ts.map +1 -0
  57. package/dist/eslint-config-closure.js +346 -0
  58. package/dist/executors/bounded-exec/bounds.d.ts +56 -0
  59. package/dist/executors/bounded-exec/bounds.d.ts.map +1 -0
  60. package/dist/executors/bounded-exec/bounds.js +43 -0
  61. package/dist/executors/bounded-exec/cpu-pool-thread.d.ts +28 -0
  62. package/dist/executors/bounded-exec/cpu-pool-thread.d.ts.map +1 -0
  63. package/dist/executors/bounded-exec/cpu-pool-thread.js +20 -0
  64. package/dist/executors/bounded-exec/cpu-pool.d.ts +35 -38
  65. package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
  66. package/dist/executors/bounded-exec/cpu-pool.js +56 -120
  67. package/dist/executors/bounded-exec/executor.d.ts +19 -12
  68. package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
  69. package/dist/executors/bounded-exec/executor.js +389 -239
  70. package/dist/executors/bounded-exec/gateway-lease.d.ts +23 -19
  71. package/dist/executors/bounded-exec/gateway-lease.d.ts.map +1 -1
  72. package/dist/executors/bounded-exec/gateway-lease.js +102 -29
  73. package/dist/executors/bounded-exec/hang-diagnostics.d.ts +18 -1
  74. package/dist/executors/bounded-exec/hang-diagnostics.d.ts.map +1 -1
  75. package/dist/executors/bounded-exec/hang-diagnostics.js +148 -30
  76. package/dist/executors/bounded-exec/lease-wait.d.ts +64 -0
  77. package/dist/executors/bounded-exec/lease-wait.d.ts.map +1 -0
  78. package/dist/executors/bounded-exec/lease-wait.js +72 -0
  79. package/dist/executors/bounded-exec/lock-wait.d.ts +51 -0
  80. package/dist/executors/bounded-exec/lock-wait.d.ts.map +1 -0
  81. package/dist/executors/bounded-exec/lock-wait.js +108 -0
  82. package/dist/executors/bounded-exec/mbx-pool.d.ts +31 -0
  83. package/dist/executors/bounded-exec/mbx-pool.d.ts.map +1 -0
  84. package/dist/executors/bounded-exec/mbx-pool.js +24 -0
  85. package/dist/executors/bounded-exec/owned-subtree.d.ts +84 -0
  86. package/dist/executors/bounded-exec/owned-subtree.d.ts.map +1 -0
  87. package/dist/executors/bounded-exec/owned-subtree.js +178 -0
  88. package/dist/executors/bounded-exec/progress.d.ts +4 -1
  89. package/dist/executors/bounded-exec/progress.d.ts.map +1 -1
  90. package/dist/executors/bounded-exec/progress.js +5 -2
  91. package/dist/executors/bounded-exec/ram-attach.d.ts +19 -0
  92. package/dist/executors/bounded-exec/ram-attach.d.ts.map +1 -0
  93. package/dist/executors/bounded-exec/ram-attach.js +207 -0
  94. package/dist/executors/bounded-exec/ram-detach.d.ts +25 -0
  95. package/dist/executors/bounded-exec/ram-detach.d.ts.map +1 -0
  96. package/dist/executors/bounded-exec/ram-detach.js +207 -0
  97. package/dist/executors/bounded-exec/ram-temp-fixtures.d.ts +76 -0
  98. package/dist/executors/bounded-exec/ram-temp-fixtures.d.ts.map +1 -1
  99. package/dist/executors/bounded-exec/ram-temp-fixtures.js +296 -0
  100. package/dist/executors/bounded-exec/ram-temp.d.ts +388 -88
  101. package/dist/executors/bounded-exec/ram-temp.d.ts.map +1 -1
  102. package/dist/executors/bounded-exec/ram-temp.js +1213 -399
  103. package/dist/executors/bounded-exec/run-cpu-pool.d.ts +10 -4
  104. package/dist/executors/bounded-exec/run-cpu-pool.d.ts.map +1 -1
  105. package/dist/executors/bounded-exec/run-cpu-pool.js +122 -35
  106. package/dist/executors/bounded-exec/runner-hook.js +14 -5
  107. package/dist/executors/bounded-exec/runner-lifecycle.d.ts +14 -1
  108. package/dist/executors/bounded-exec/runner-lifecycle.d.ts.map +1 -1
  109. package/dist/executors/bounded-exec/runner-lifecycle.js +27 -6
  110. package/dist/executors/bounded-exec/{cpu-tokens.d.ts → runner-sizing.d.ts} +14 -42
  111. package/dist/executors/bounded-exec/runner-sizing.d.ts.map +1 -0
  112. package/dist/executors/bounded-exec/{cpu-tokens.js → runner-sizing.js} +5 -35
  113. package/dist/executors/bounded-exec/runtime-pin.d.ts +73 -0
  114. package/dist/executors/bounded-exec/runtime-pin.d.ts.map +1 -0
  115. package/dist/executors/bounded-exec/runtime-pin.js +167 -0
  116. package/dist/executors/bounded-exec/schema.d.ts +18 -1
  117. package/dist/executors/bounded-exec/schema.d.ts.map +1 -1
  118. package/dist/executors/bounded-exec/verdict.d.ts +48 -11
  119. package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
  120. package/dist/executors/bounded-exec/verdict.js +89 -20
  121. package/dist/executors/cargo-zigbuild/executor.d.ts +21 -0
  122. package/dist/executors/cargo-zigbuild/executor.d.ts.map +1 -1
  123. package/dist/executors/cargo-zigbuild/executor.js +67 -12
  124. package/dist/executors/clean-outputs/executor.js +1 -0
  125. package/dist/executors/scriptc-native/executor.d.ts +79 -0
  126. package/dist/executors/scriptc-native/executor.d.ts.map +1 -0
  127. package/dist/executors/scriptc-native/executor.js +367 -0
  128. package/dist/executors/scriptc-native/ffi-paths.d.ts +9 -0
  129. package/dist/executors/scriptc-native/ffi-paths.d.ts.map +1 -0
  130. package/dist/executors/scriptc-native/ffi-paths.js +25 -0
  131. package/dist/executors/scriptc-native/schema.d.ts +34 -0
  132. package/dist/executors/scriptc-native/schema.d.ts.map +1 -0
  133. package/dist/executors/scriptc-native/schema.js +1 -0
  134. package/dist/executors/typescript-emit/emit-overrides.d.ts +14 -0
  135. package/dist/executors/typescript-emit/emit-overrides.d.ts.map +1 -0
  136. package/dist/executors/typescript-emit/emit-overrides.js +24 -0
  137. package/dist/executors/typescript-emit/executor.d.ts +3 -1
  138. package/dist/executors/typescript-emit/executor.d.ts.map +1 -1
  139. package/dist/executors/typescript-emit/executor.js +146 -33
  140. package/dist/forbidden-spelling-policy.d.ts +56 -0
  141. package/dist/forbidden-spelling-policy.d.ts.map +1 -0
  142. package/dist/forbidden-spelling-policy.js +176 -0
  143. package/dist/generators/create-package/generator.d.ts.map +1 -1
  144. package/dist/generators/create-package/generator.js +3 -4
  145. package/dist/graph-input-contract.d.ts +14 -0
  146. package/dist/graph-input-contract.d.ts.map +1 -1
  147. package/dist/graph-input-contract.js +67 -2
  148. package/dist/index.d.ts +23 -6
  149. package/dist/index.d.ts.map +1 -1
  150. package/dist/index.js +672 -214
  151. package/dist/managed-assets.d.ts +2 -0
  152. package/dist/managed-assets.d.ts.map +1 -1
  153. package/dist/managed-files/files.d.ts +3 -3
  154. package/dist/managed-files/files.d.ts.map +1 -1
  155. package/dist/managed-files/files.js +28 -3
  156. package/dist/managed-files/managed-format.d.ts.map +1 -1
  157. package/dist/managed-files/managed-format.js +1 -0
  158. package/dist/managed-files/publish-workflow.js +14 -11
  159. package/dist/nextest-code-bound.js +1 -0
  160. package/dist/nextest-timings.d.ts +35 -0
  161. package/dist/nextest-timings.d.ts.map +1 -0
  162. package/dist/nextest-timings.js +168 -0
  163. package/dist/owned-installation.d.ts +6 -5
  164. package/dist/owned-installation.d.ts.map +1 -1
  165. package/dist/owned-installation.js +150 -26
  166. package/dist/package-target-policy.d.ts.map +1 -1
  167. package/dist/package-target-policy.js +2 -3
  168. package/dist/process-children.d.ts +7 -0
  169. package/dist/process-children.d.ts.map +1 -0
  170. package/dist/process-children.darwin.d.ts +7 -0
  171. package/dist/process-children.darwin.d.ts.map +1 -0
  172. package/dist/process-children.darwin.js +13 -0
  173. package/dist/process-children.js +14 -0
  174. package/dist/process-children.linux.d.ts +8 -0
  175. package/dist/process-children.linux.d.ts.map +1 -0
  176. package/dist/process-children.linux.js +25 -0
  177. package/dist/process-group.d.ts +108 -0
  178. package/dist/process-group.d.ts.map +1 -0
  179. package/dist/process-group.js +171 -0
  180. package/dist/process-listing.d.ts +7 -0
  181. package/dist/process-listing.d.ts.map +1 -0
  182. package/dist/process-listing.js +51 -0
  183. package/dist/producer-receipts.d.ts +8 -2
  184. package/dist/producer-receipts.d.ts.map +1 -1
  185. package/dist/producer-receipts.js +514 -47
  186. package/dist/task-graph-policy.d.ts +21 -0
  187. package/dist/task-graph-policy.d.ts.map +1 -0
  188. package/dist/task-graph-policy.js +91 -0
  189. package/dist/test-timings.js +4 -4
  190. package/dist/test-walk-policy.d.ts +14 -5
  191. package/dist/test-walk-policy.d.ts.map +1 -1
  192. package/dist/test-walk-policy.js +8 -2
  193. package/dist/testing.d.ts +73 -28
  194. package/dist/testing.d.ts.map +1 -1
  195. package/dist/testing.js +217 -223
  196. package/dist/tool-bins.d.ts +33 -0
  197. package/dist/tool-bins.d.ts.map +1 -0
  198. package/dist/tool-bins.js +72 -0
  199. package/dist/typecheck-test-policy.d.ts.map +1 -1
  200. package/dist/typecheck-test-policy.js +3 -4
  201. package/dist/wait-why-policy.d.ts +28 -0
  202. package/dist/wait-why-policy.d.ts.map +1 -0
  203. package/dist/wait-why-policy.js +223 -0
  204. package/dist/workspace-config-policy.d.ts +14 -8
  205. package/dist/workspace-config-policy.d.ts.map +1 -1
  206. package/dist/workspace-config-policy.js +60 -16
  207. package/dist/workspace-manifest.d.ts +41 -6
  208. package/dist/workspace-manifest.d.ts.map +1 -1
  209. package/dist/workspace-manifest.js +57 -0
  210. package/executors.json +16 -1
  211. package/loader.js +2 -0
  212. package/managed/raw/envrc +11 -0
  213. package/managed/raw/patches/nx@23.2.1.patch +2840 -199
  214. package/managed/raw/patches/ttsc@0.34.1.patch +119 -0
  215. package/managed/raw/tooling/direnv/cargo-fetch-bound.ts +245 -0
  216. package/managed/raw/tooling/direnv/cargo-fetch-run.ts +299 -0
  217. package/managed/raw/tooling/direnv/compiler-variables.sh +42 -0
  218. package/managed/raw/tooling/direnv/developer-links.ts +464 -25
  219. package/managed/raw/tooling/direnv/devenv.smoo.nix +78 -38
  220. package/managed/raw/tooling/direnv/github-actions-bootstrap.sh +14 -12
  221. package/managed/raw/tooling/direnv/installed-bun-links.ts +308 -0
  222. package/managed/raw/tooling/direnv/linked-snapshots.ts +62 -3
  223. package/managed/raw/tooling/direnv/nx-socket-dir.sh +56 -17
  224. package/managed/raw/tooling/direnv/repo-path +68 -6
  225. package/managed/raw/tooling/direnv/secret-references.ts +13 -4
  226. package/managed/raw/tooling/direnv/setup-environment.ts +128 -537
  227. package/managed/raw/tooling/direnv/toolchain-stamp.ts +4 -2
  228. package/managed/raw/tooling/direnv/ttsc-env.sh +9 -0
  229. package/managed/raw/tooling/direnv/uv-relocation.ts +227 -0
  230. package/managed/raw/tooling/git-hooks/commit-msg.sh +2 -1
  231. package/managed/raw/tooling/git-hooks/pre-commit.sh +2 -1
  232. package/managed/raw/tooling/git-hooks/pre-push.sh +2 -1
  233. package/managed/templates/github/actions/cache-ttsc-plugins/action.yml +85 -7
  234. package/managed/templates/github/actions/save-nix-devenv/action.yml +12 -2
  235. package/managed/templates/github/actions/setup-devenv/action.yml +2 -0
  236. package/package.json +95 -14
  237. package/src/bin/smoo-nx-cargo-toolchain.ts +11 -0
  238. package/src/bin/smoo-nx-forbidden-spellings.ts +33 -0
  239. package/src/bin/smoo-nx-nextest-timings.ts +31 -0
  240. package/src/bounded-test-policy.ts +12 -14
  241. package/src/build-state-output-policy.ts +458 -34
  242. package/src/bun-lock-scope.ts +108 -0
  243. package/src/cargo-build-script-env.ts +265 -0
  244. package/src/cargo-closure-input.ts +74 -7
  245. package/src/cargo-retention-hook.ts +94 -0
  246. package/src/cargo-source-hash.ts +138 -41
  247. package/src/cargo-toolchain-paths.ts +95 -0
  248. package/src/cargo-toolchain-policy.ts +49 -5
  249. package/src/cargo-workspace.ts +72 -123
  250. package/src/cross-check-policy.ts +12 -10
  251. package/src/disk-command-policy.ts +1 -0
  252. package/src/ensure-built-receipt.ts +173 -39
  253. package/src/ensure-built-witness.ts +62 -10
  254. package/src/ensure-built.ts +11 -1
  255. package/src/eslint-config-closure.ts +357 -0
  256. package/src/executors/bounded-exec/bounds.ts +76 -0
  257. package/src/executors/bounded-exec/cpu-pool-thread.ts +43 -0
  258. package/src/executors/bounded-exec/cpu-pool.ts +55 -120
  259. package/src/executors/bounded-exec/executor.ts +442 -260
  260. package/src/executors/bounded-exec/gateway-lease.ts +94 -35
  261. package/src/executors/bounded-exec/hang-diagnostics.ts +173 -31
  262. package/src/executors/bounded-exec/lease-wait.ts +116 -0
  263. package/src/executors/bounded-exec/lock-wait.ts +141 -0
  264. package/src/executors/bounded-exec/mbx-pool.ts +48 -0
  265. package/src/executors/bounded-exec/owned-subtree.ts +235 -0
  266. package/src/executors/bounded-exec/progress.ts +9 -3
  267. package/src/executors/bounded-exec/ram-attach.ts +238 -0
  268. package/src/executors/bounded-exec/ram-detach.ts +226 -0
  269. package/src/executors/bounded-exec/ram-temp-fixtures.ts +358 -0
  270. package/src/executors/bounded-exec/ram-temp-registry.txt +440 -0
  271. package/src/executors/bounded-exec/ram-temp.ts +1497 -460
  272. package/src/executors/bounded-exec/run-cpu-pool.ts +129 -35
  273. package/src/executors/bounded-exec/runner-hook.ts +14 -6
  274. package/src/executors/bounded-exec/runner-lifecycle.ts +31 -8
  275. package/src/executors/bounded-exec/{cpu-tokens.ts → runner-sizing.ts} +13 -59
  276. package/src/executors/bounded-exec/runtime-pin.ts +232 -0
  277. package/src/executors/bounded-exec/schema.json +11 -2
  278. package/src/executors/bounded-exec/schema.ts +19 -1
  279. package/src/executors/bounded-exec/verdict.ts +145 -35
  280. package/src/executors/cargo-zigbuild/executor.ts +82 -8
  281. package/src/executors/clean-outputs/executor.ts +1 -0
  282. package/src/executors/scriptc-native/executor.ts +394 -0
  283. package/src/executors/scriptc-native/ffi-paths.ts +29 -0
  284. package/src/executors/scriptc-native/schema.json +60 -0
  285. package/src/executors/scriptc-native/schema.ts +34 -0
  286. package/src/executors/typescript-emit/emit-overrides.ts +24 -0
  287. package/src/executors/typescript-emit/executor.ts +186 -38
  288. package/src/forbidden-spelling-policy.ts +226 -0
  289. package/src/generators/create-package/generator.ts +7 -7
  290. package/src/graph-input-contract.ts +61 -2
  291. package/src/index.ts +788 -224
  292. package/src/managed-assets.ts +3 -0
  293. package/src/managed-files/files.ts +28 -3
  294. package/src/managed-files/managed-format.ts +1 -0
  295. package/src/managed-files/publish-workflow.ts +20 -15
  296. package/src/nextest-code-bound.ts +1 -0
  297. package/src/nextest-timings.ts +106 -0
  298. package/src/owned-installation.ts +150 -23
  299. package/src/package-target-policy.ts +2 -3
  300. package/src/process-children.darwin.ts +14 -0
  301. package/src/process-children.linux.ts +33 -0
  302. package/src/process-children.ts +22 -0
  303. package/src/process-group.ts +209 -0
  304. package/src/process-listing.ts +54 -0
  305. package/src/producer-receipts.ts +92 -41
  306. package/src/task-graph-policy.ts +98 -0
  307. package/src/test-walk-policy.ts +23 -8
  308. package/src/testing.ts +240 -232
  309. package/src/tool-bins.ts +108 -0
  310. package/src/typecheck-test-policy.ts +2 -3
  311. package/src/wait-why-policy.ts +226 -0
  312. package/src/workspace-config-policy.ts +63 -15
  313. package/src/workspace-manifest.ts +89 -6
  314. package/test-timings.json +124 -99
  315. package/dist/bin/smoo-nx-failures.d.ts +0 -3
  316. package/dist/bin/smoo-nx-failures.d.ts.map +0 -1
  317. package/dist/bin/smoo-nx-failures.js +0 -3
  318. package/dist/bin/smoo-nx-nextest-extract.d.ts +0 -3
  319. package/dist/bin/smoo-nx-nextest-extract.d.ts.map +0 -1
  320. package/dist/bin/smoo-nx-nextest-extract.js +0 -42
  321. package/dist/executors/bounded-exec/cpu-tokens.d.ts.map +0 -1
  322. package/dist/failure-summary.d.ts +0 -5
  323. package/dist/failure-summary.d.ts.map +0 -1
  324. package/dist/failure-summary.js +0 -118
  325. package/dist/nextest-extraction.d.ts +0 -117
  326. package/dist/nextest-extraction.d.ts.map +0 -1
  327. package/dist/nextest-extraction.js +0 -303
  328. package/managed/raw/patches/ttsc@0.30.4.patch +0 -2255
  329. package/src/__tests__/cargo-closure-filesets.ts +0 -94
  330. package/src/__tests__/cargo-closure-fixture.ts +0 -113
  331. package/src/__tests__/counted-cargo-process.ts +0 -286
  332. package/src/__tests__/counted-cargo.ts +0 -713
  333. package/src/__tests__/directory-change.ts +0 -27
  334. package/src/__tests__/fixture-cargo-home.ts +0 -36
  335. package/src/__tests__/fixture-nx-env.ts +0 -193
  336. package/src/__tests__/owned-installation.ts +0 -13
  337. package/src/__tests__/past-tick.ts +0 -12
  338. package/src/__tests__/progress-clock.ts +0 -43
  339. package/src/__tests__/runtime-input.ts +0 -23
  340. package/src/bin/smoo-nx-bound-failures.test.ts +0 -127
  341. package/src/bin/smoo-nx-failures.ts +0 -4
  342. package/src/bin/smoo-nx-nextest-extract.ts +0 -44
  343. package/src/bounded-test-policy.test.ts +0 -1029
  344. package/src/build-state-output-policy.test.ts +0 -115
  345. package/src/bun-bootstrap.test.ts +0 -102
  346. package/src/bun-file-packages.test.ts +0 -47
  347. package/src/cargo-closure-filesets.test.ts +0 -63
  348. package/src/cargo-closure-input.property.test.ts +0 -50
  349. package/src/cargo-closure-input.test.ts +0 -506
  350. package/src/cargo-closure-key.test.ts +0 -240
  351. package/src/cargo-resolve-cache.test.ts +0 -1181
  352. package/src/cargo-source-hash.test.ts +0 -707
  353. package/src/cargo-workspace.test.ts +0 -615
  354. package/src/cross-check-policy.test.ts +0 -316
  355. package/src/disk-command-policy.test.ts +0 -61
  356. package/src/ensure-built-receipt.test.ts +0 -279
  357. package/src/ensure-built-witness.test.ts +0 -688
  358. package/src/ensure-built.test.ts +0 -1462
  359. package/src/executors/all-verdicts/executor.test.ts +0 -135
  360. package/src/executors/bounded-exec/cargo-progress-host.test.ts +0 -332
  361. package/src/executors/bounded-exec/cpu-pool.test.ts +0 -684
  362. package/src/executors/bounded-exec/cpu-tokens.test.ts +0 -519
  363. package/src/executors/bounded-exec/disk-lease.test.ts +0 -219
  364. package/src/executors/bounded-exec/executor-host.test.ts +0 -800
  365. package/src/executors/bounded-exec/executor.test.ts +0 -1819
  366. package/src/executors/bounded-exec/go-progress-host.test.ts +0 -171
  367. package/src/executors/bounded-exec/hang-diagnostics.test.ts +0 -291
  368. package/src/executors/bounded-exec/progress.test.ts +0 -139
  369. package/src/executors/bounded-exec/ram-temp-held.test.ts +0 -264
  370. package/src/executors/bounded-exec/ram-temp-host.test.ts +0 -635
  371. package/src/executors/bounded-exec/ram-temp.test.ts +0 -750
  372. package/src/executors/bounded-exec/run-cpu-pool.test.ts +0 -79
  373. package/src/executors/bounded-exec/runner-hook.test.ts +0 -193
  374. package/src/executors/bounded-exec/runner-lifecycle.test.ts +0 -82
  375. package/src/executors/bounded-exec/verdict.test.ts +0 -545
  376. package/src/executors/cargo-zigbuild/executor.test.ts +0 -129
  377. package/src/executors/clean-outputs/executor.test.ts +0 -20
  378. package/src/executors/typescript-emit/executor.test.ts +0 -83
  379. package/src/failure-summary.ts +0 -119
  380. package/src/generators/bounded-test-targets/generator.test.ts +0 -138
  381. package/src/generators/bun-test-tracing/generator.test.ts +0 -250
  382. package/src/generators/create-package/generator.test.ts +0 -190
  383. package/src/generators/make-public/generator.test.ts +0 -137
  384. package/src/graph-input-contract.test.ts +0 -379
  385. package/src/host-controller-wiring.test.ts +0 -31
  386. package/src/index.test.ts +0 -3321
  387. package/src/linked-snapshots.nx.test.ts +0 -254
  388. package/src/managed-files/forgejo-artifact.test.ts +0 -395
  389. package/src/managed-files/generator.test.ts +0 -243
  390. package/src/managed-files/github-host-cache-isolation.test.ts +0 -86
  391. package/src/managed-files/json-edit.test.ts +0 -216
  392. package/src/managed-files/managed-content.test.ts +0 -166
  393. package/src/managed-files/managed-format-json.test.ts +0 -308
  394. package/src/managed-files/repo-path.test.ts +0 -172
  395. package/src/managed-files/setup-devenv-free-disk.test.ts +0 -95
  396. package/src/managed-files/trace-db-artifact.test.ts +0 -7
  397. package/src/managed-files/tree-permissions.test.ts +0 -109
  398. package/src/managed-files/tree.test.ts +0 -123
  399. package/src/manifest-hash.test.ts +0 -219
  400. package/src/nextest-code-bound.test.ts +0 -161
  401. package/src/nextest-extraction.test.ts +0 -216
  402. package/src/nextest-extraction.ts +0 -371
  403. package/src/nextest-partitions.test.ts +0 -501
  404. package/src/nx-cache-bound.test.ts +0 -109
  405. package/src/nx-cache-restore-mtime.test.ts +0 -170
  406. package/src/nx-daemon-claim.test.ts +0 -279
  407. package/src/nx-daemon-idle-cpu.test.ts +0 -80
  408. package/src/nx-daemon-idle-policy.test.ts +0 -111
  409. package/src/nx-daemon-ignore-files.test.ts +0 -210
  410. package/src/nx-daemon-root-normalization.test.ts +0 -105
  411. package/src/nx-daemon-start-never.test.ts +0 -292
  412. package/src/nx-daemon-validity.test.ts +0 -1055
  413. package/src/nx-dot-input-policy.test.ts +0 -353
  414. package/src/nx-failure-life-cycles.test.ts +0 -207
  415. package/src/nx-files-archive-racy.test.ts +0 -97
  416. package/src/nx-foreign-workspace-env.test.ts +0 -329
  417. package/src/nx-incomplete-run-summary.test.ts +0 -70
  418. package/src/nx-plugin-load-errors.test.ts +0 -120
  419. package/src/nx-project-graph-errors.test.ts +0 -70
  420. package/src/nx-project-resolution-cache.test.ts +0 -176
  421. package/src/nx-raw-inputs.test.ts +0 -166
  422. package/src/nx-resident-graph-workers.test.ts +0 -112
  423. package/src/nx-run-information.test.ts +0 -118
  424. package/src/nx-skipped-task-exit.test.ts +0 -180
  425. package/src/nx-state-location.test.ts +0 -63
  426. package/src/nx-store-resolution.test.ts +0 -81
  427. package/src/nx-task-graph-cycles.test.ts +0 -148
  428. package/src/nx-task-output-lease.test.ts +0 -550
  429. package/src/nx-use-local.test.ts +0 -105
  430. package/src/owned-installation.test.ts +0 -105
  431. package/src/package-target-policy.test.ts +0 -1014
  432. package/src/producer-receipts.test.ts +0 -285
  433. package/src/release-config-policy.test.ts +0 -311
  434. package/src/runtime-input.test.ts +0 -393
  435. package/src/serial-group-wiring.test.ts +0 -326
  436. package/src/task-history-namespace.test.ts +0 -181
  437. package/src/task-results.test.ts +0 -228
  438. package/src/test-clock-policy.test.ts +0 -338
  439. package/src/test-program.test.ts +0 -303
  440. package/src/test-timings.test.ts +0 -240
  441. package/src/test-walk-policy.test.ts +0 -620
  442. package/src/testing.test.ts +0 -509
  443. package/src/typecheck-test-policy.test.ts +0 -1468
  444. package/src/typescript-resolution.test.ts +0 -76
  445. package/src/version-actions.test.ts +0 -80
  446. package/src/workspace-config-policy.test.ts +0 -737
@@ -1,42 +1,60 @@
1
1
  import { type ChildProcess, spawn } from 'node:child_process';
2
2
  import { accessSync, constants } from 'node:fs';
3
+ import { mkdtemp, rm } from 'node:fs/promises';
3
4
  import { availableParallelism, userInfo } from 'node:os';
4
5
  import { delimiter, isAbsolute, join, resolve as resolvePath } from 'node:path';
5
6
  import { createInterface } from 'node:readline';
6
- import { Duplex } from 'node:stream';
7
+ import { Duplex, Readable } from 'node:stream';
7
8
  import { workspaceDataDirectoryForWorkspace } from 'nx/src/utils/cache-directory.js';
8
-
9
+ import {
10
+ DEADLINE_PATIENCE,
11
+ describeMembersLeft,
12
+ killProcessGroup,
13
+ processGroupHasMembers,
14
+ signalProcessGroup,
15
+ } from '../../process-group.js';
16
+ import { type BoundEvent, type BoundKind, type Bounds, earliestBound, stepBounds, UNARMED } from './bounds.js';
9
17
  import {
10
18
  CPU_POOL_ENV,
11
19
  type CpuPool,
12
20
  createCpuPool,
13
- type HostWaits,
14
21
  type InheritedCpuPool,
15
22
  readInheritedCpuPool,
23
+ type TokenGrant,
16
24
  takeParentCpuTokens,
17
25
  } from './cpu-pool.js';
18
- import {
19
- CPU_TOKENS_ENV,
20
- type CpuBudget,
21
- gatewayCpuBudget,
22
- runnerDemand,
23
- sizeRunner,
24
- type TokenGrant,
25
- } from './cpu-tokens.js';
26
+ import { GATEWAY_SOCKET } from './gateway-lease.js';
26
27
  import type { HangDiagnostics } from './hang-diagnostics.js';
27
28
  import { captureHangDiagnostics, HANG_CAPTURE_MS } from './hang-diagnostics.js';
29
+ import type { CpuWaitRow } from './lease-wait.js';
30
+ import type { LockWait } from './lock-wait.js';
31
+ import { mbxLeaf, mbxPool, withMbxLeaf } from './mbx-pool.js';
28
32
  import { createNoSpaceWatch } from './no-space.js';
29
- import { createRunProgress, withProgressReport } from './progress.js';
30
33
  import {
31
34
  COWSHED_EXECUTABLE_ENV,
35
+ cowshedSubtreeOwner,
36
+ HANDLE_FD,
37
+ readAdmission,
38
+ SUPERVISOR_SOCKET_ENV,
39
+ type SubtreeAdmission,
40
+ type SubtreeOwner,
41
+ } from './owned-subtree.js';
42
+ import { createRunProgress, withProgressReport } from './progress.js';
43
+ import {
32
44
  createHostCommands,
45
+ describeCoverageGap,
33
46
  describeRamTempError,
47
+ describeSignalled,
48
+ LEFTOVER_GRACE_MS,
49
+ PRODUCTION,
34
50
  RAM_TEMP_CAPACITY_BYTES,
35
51
  type RamTempLease,
36
52
  RamTempVolume,
37
53
  ramTempPaths,
38
54
  } from './ram-temp.js';
39
- import { createRunnerLifecycle, withRunnerLifecycle } from './runner-lifecycle.js';
55
+ import { createRunnerLifecycle, withFixtureLifecycle, withRunnerLifecycle } from './runner-lifecycle.js';
56
+ import { CPU_TOKENS_ENV, runnerDemand, sizeRunner } from './runner-sizing.js';
57
+ import { checkRuntimePin, describeRuntimePinRefusal } from './runtime-pin.js';
40
58
  import type { BoundedExecOptions } from './schema.js';
41
59
  import {
42
60
  type BoundedExecRecord,
@@ -70,9 +88,9 @@ const EXIT_CODE_BY_SIGNAL: Partial<Record<NodeJS.Signals, number>> = {
70
88
  /** The pool descriptor: only a grant may give it, so neither the target's nor the inherited environment passes it on. */
71
89
  const POOL_ENV = [CPU_TOKENS_ENV, CPU_POOL_ENV];
72
90
  /**
73
- * The variables the CPU budget sets for the command it grants: no target's environment may set them. A jobserver the
74
- * executor itself inherits, a parent make's, still reaches an ungranted command by Cargo's own rule; a granted one's
75
- * sub-pool names its own fifo over it.
91
+ * The variables a grant sets for the command it sizes: no target's environment may set them. A jobserver the executor
92
+ * itself inherits, a parent make's or its cowshed job's, still reaches an ungranted command by Cargo's own rule; a
93
+ * granted one's sub-pool names its own fifo over it.
76
94
  */
77
95
  const BUDGET_ENV = [...POOL_ENV, 'CARGO_MAKEFLAGS', 'MAKEFLAGS', 'MFLAGS'];
78
96
 
@@ -104,8 +122,12 @@ interface RunState {
104
122
  forceKillNeeded: boolean;
105
123
  /** The sub-pool's close error when it did not close cleanly: the run fails. */
106
124
  poolCloseError: string | null;
125
+ /** Why the command's subtree could not be retired, when the supervisor's client failed: the run fails. */
126
+ retireError: string | null;
107
127
  /** The first output line that said ENOSPC, with the task's volume read when it arrived. */
108
128
  noSpace: Promise<NoSpaceSighting> | null;
129
+ /** The lock waits the expiry's snapshot named: empty unless that capture completed. Judged with the expiry. */
130
+ lockWaits: readonly LockWait[];
109
131
  }
110
132
 
111
133
  type GrantedTokens = Extract<TokenGrant, { granted: true }>;
@@ -115,7 +137,10 @@ export interface ProcessTreeKiller {
115
137
  }
116
138
 
117
139
  /** Where the command's TMPDIR comes from; `null` leaves the inherited one. */
118
- export type TempVolume = Pick<RamTempVolume, 'acquire' | 'release' | 'fullness' | 'space' | 'reap'>;
140
+ export type TempVolume = Pick<
141
+ RamTempVolume,
142
+ 'acquire' | 'release' | 'fullness' | 'space' | 'record' | 'ordinaryParent'
143
+ >;
119
144
 
120
145
  /** Workspace-provided cowshed launchers, in the order a workspace that provides one names them. */
121
146
  const WORKSPACE_COWSHED_LAUNCHERS = [
@@ -170,10 +195,38 @@ export default function boundedExecExecutor(
170
195
  return runDefaultBoundedExec(options, recordContext(context));
171
196
  }
172
197
 
173
- /** Runs with the normal process, RAM lease and CPU-budget owners, with an optional stdout sink. */
198
+ /**
199
+ * Runs with the normal process, RAM lease and CPU pool owners, with an optional stdout sink, on the user's RAM temp
200
+ * volume where there is one, through the host's own gateway: the Nx executor's runs, never a test's.
201
+ */
174
202
  export function runDefaultBoundedExec(
175
203
  options: BoundedExecOptions,
176
204
  context: BoundedExecContext,
205
+ ): Promise<BoundedExecResult> {
206
+ // Only macOS has `hdiutil ram://`; elsewhere the command keeps the inherited TMPDIR.
207
+ return runBoundedExecWith(options, context, (cowshed) =>
208
+ process.platform === 'darwin'
209
+ ? new RamTempVolume(
210
+ ramTempPaths(userInfo(), PRODUCTION),
211
+ RAM_TEMP_CAPACITY_BYTES,
212
+ createHostCommands(GATEWAY_SOCKET, undefined, undefined, cowshed),
213
+ undefined,
214
+ undefined,
215
+ 'optional',
216
+ )
217
+ : null,
218
+ );
219
+ }
220
+
221
+ /**
222
+ * The default runner with every owner but its RAM volume, which its caller makes: `volume` makes the volume for the
223
+ * cowshed the run's disk commands go through. The Nx executor makes the user's; a host proof passes a fixture volume
224
+ * (`fixtureRamVolume`), so no test leases, creates or detaches the user's volume.
225
+ */
226
+ export function runBoundedExecWith(
227
+ options: BoundedExecOptions,
228
+ context: BoundedExecContext,
229
+ volume: (cowshed: string | undefined) => TempVolume | null,
177
230
  ): Promise<BoundedExecResult> {
178
231
  // A target that names the cowshed its disk commands run through names it for this run's own RAM volume work and
179
232
  // for its command alike. Nx resolves `{workspaceRoot}/…` in an option to a workspace-relative path, which the
@@ -188,19 +241,14 @@ export function runDefaultBoundedExec(
188
241
  );
189
242
  const run: BoundedExecOptions =
190
243
  cowshed === undefined ? options : { ...options, env: { ...options.env, [COWSHED_EXECUTABLE_ENV]: cowshed } };
191
- // Only macOS has `hdiutil ram://`; elsewhere the command keeps the inherited TMPDIR.
192
- const tempVolume =
193
- process.platform === 'darwin'
194
- ? new RamTempVolume(
195
- ramTempPaths(userInfo()),
196
- RAM_TEMP_CAPACITY_BYTES,
197
- createHostCommands(undefined, undefined, undefined, cowshed),
198
- )
199
- : null;
244
+ const tempVolume = volume(cowshed);
245
+ // The command's processes are owned by the supervisor that serves this job, when one does
246
+ // (specs/cowshed/07_api.md, `JobSubtreeHandle`); with none, nothing here may end what it leaves.
247
+ const owner = process.env[SUPERVISOR_SOCKET_ENV] === undefined ? null : cowshedSubtreeOwner(cowshed ?? 'cowshed');
200
248
  // A command a parent's CPU pool started draws its tokens from that pool. The pool is named by the environment the
201
249
  // parent set for this command; a run no parent started has none.
202
250
  const inherited = readInheritedCpuPool(process.env);
203
- return runBoundedExec(run, context, createProcessTreeKiller(), tempVolume, gatewayCpuBudget(), inherited);
251
+ return runBoundedExec(run, context, createProcessTreeKiller(), tempVolume, inherited, captureHangDiagnostics, owner);
204
252
  }
205
253
 
206
254
  /**
@@ -227,24 +275,22 @@ export function recordContext(context: NxTaskContext): BoundedExecContext {
227
275
  }
228
276
 
229
277
  /**
230
- * Run the command once its runner holds CPU tokens from `budget` (`cpu-tokens.ts`), sized to the
231
- * grant; `null`, or a budget nobody serves, runs it as asked. Both bounds start once it is
232
- * granted: a wait for the host is not the command's time.
278
+ * Run the command, sized to the tokens its parent's CPU pool (`cpu-pool.ts`) lends it; a run that inherits no pool
279
+ * runs it as asked. Both bounds start once it is granted: a wait for the pool is not the command's time.
233
280
  *
234
- * `inherited` is the parent's CPU pool (`cpu-pool.ts`) that lends this run its tokens. The run asks that pool first
235
- * and the host budget only when the pool refuses. Whatever it is granted, the run serves its own sub-pool to its
236
- * descendants, closed before the grant returns. A grant whose connection closes while the command runs (a gateway
237
- * restart, or the lending parent's exit) ends its accounting, never the command: the command runs on over budget to
238
- * its own exit, and the run says so.
281
+ * `inherited` is the parent's pool: an enclosing bounded-exec's sub-pool or the Nx run's own (`run-cpu-pool.ts`). A
282
+ * granted run serves a sub-pool of its grant to its own descendants, closed before the grant returns. A grant whose
283
+ * connection closes while the command runs (the lending parent's exit) ends its accounting, never the command: the
284
+ * command runs on over budget to its own exit, and the run says so.
239
285
  */
240
286
  export async function runBoundedExec(
241
287
  options: BoundedExecOptions,
242
288
  context: BoundedExecContext,
243
289
  killer: ProcessTreeKiller,
244
290
  tempVolume: TempVolume | null,
245
- budget: CpuBudget | null,
246
291
  inherited: InheritedCpuPool | null = null,
247
292
  captureDiagnostics: typeof captureHangDiagnostics = captureHangDiagnostics,
293
+ owner: SubtreeOwner | null = null,
248
294
  ): Promise<BoundedExecResult> {
249
295
  const cwd = resolveCwd(options.cwd, context.root);
250
296
  const directory = context.record?.directory ?? null;
@@ -256,13 +302,19 @@ export async function runBoundedExec(
256
302
  const timeoutMs = options.timeoutMs;
257
303
  const idleTimeoutMs = options.idleTimeoutMs;
258
304
  const killAfterMs = options.killAfterMs ?? DEFAULT_KILL_AFTER_MS;
305
+ const fixture: { command: string; timeoutMs: number } | null =
306
+ options.fixtureCommand !== undefined && options.fixtureTimeoutMs !== undefined
307
+ ? { command: options.fixtureCommand, timeoutMs: options.fixtureTimeoutMs }
308
+ : null;
259
309
  const outputChunks: string[] = [];
260
310
  const state: RunState = {
261
311
  settled: false,
262
312
  expiry: null,
263
313
  forceKillNeeded: false,
264
314
  poolCloseError: null,
315
+ retireError: null,
265
316
  noSpace: null,
317
+ lockWaits: [],
266
318
  };
267
319
  const appendStdout = (chunk: Buffer | string): void => {
268
320
  const text = chunk.toString();
@@ -278,10 +330,62 @@ export async function runBoundedExec(
278
330
  outputChunks.push(text);
279
331
  process.stderr.write(text);
280
332
  };
333
+ // Told, never thrown, wherever the command is being ended: a diagnostic must neither replace the failure it
334
+ // accompanies nor skip the cleanup that follows (the signal handlers, the CPU grant, the RAM lease, the run's record).
335
+ const tell = (text: string): void => {
336
+ try {
337
+ appendStderr(text);
338
+ } catch {
339
+ // Nobody is left to tell.
340
+ }
341
+ };
342
+
343
+ // The fixture pair names one segment: a fixture with no build budget, or a budget with no fixture, is a
344
+ // target that does not say what it runs, so it does not run.
345
+ if (
346
+ (options.fixtureCommand === undefined) !== (options.fixtureTimeoutMs === undefined) ||
347
+ (fixture !== null && !(fixture.timeoutMs > 0))
348
+ ) {
349
+ appendStderr(
350
+ 'bounded-exec: fixtureCommand and a positive fixtureTimeoutMs must be named together; refusing to run the command\n',
351
+ );
352
+ if (context.record !== undefined) {
353
+ const { task, directory: recordDirectory } = context.record;
354
+ await writeRecord(recordDirectory, { task: task.id, hash: task.hash, verdict: judgeOperationalFailure(1, null) });
355
+ }
356
+ return { success: false, terminalOutput: outputChunks.join('') };
357
+ }
358
+
359
+ // A leg that runs bun runs the Bun its workspace declares, or it does not run. This comes before the RAM lease and the
360
+ // CPU tokens: a refusal that waited in its pool's queue first would say nothing for minutes.
361
+ const runtime = await checkRuntimePin({
362
+ root: context.root,
363
+ command: fixture === null ? requested : `${fixture.command} && ${requested}`,
364
+ env: { ...withoutPoolEnv(process.env), ...options.env },
365
+ cwd,
366
+ pathSource: options.env?.PATH === undefined ? 'the inherited environment' : "the target's env",
367
+ });
368
+ if (runtime !== null) {
369
+ appendStderr(describeRuntimePinRefusal(runtime, context.record?.task.id));
370
+ if (context.record !== undefined) {
371
+ const { task, directory: recordDirectory } = context.record;
372
+ await writeRecord(recordDirectory, { task: task.id, hash: task.hash, verdict: judgeOperationalFailure(1, null) });
373
+ }
374
+ return { success: false, terminalOutput: outputChunks.join('') };
375
+ }
281
376
 
282
377
  let lease: RamTempLease | null = null;
283
- if (tempVolume !== null) {
284
- const acquired = await tempVolume.acquire();
378
+ let ordinaryTemp: string | null = null;
379
+ // Before spawn there is nothing to retire. Once started, only the owner's terminal retirement permits deletion.
380
+ let subtreeRetired = true;
381
+ // A lease is removed when its command ends, so it is taken only where whatever the command leaves
382
+ // in it can be ended by its owner first: the supervisor's subtree retirement, never a census.
383
+ if (tempVolume !== null && owner === null) {
384
+ appendStderr(
385
+ `RAM temp volume not leased: no cowshed supervisor owns this command's processes (${SUPERVISOR_SOCKET_ENV} is unset); TMPDIR stays ${process.env.TMPDIR ?? '(unset)'}\n`,
386
+ );
387
+ } else if (tempVolume !== null && owner !== null) {
388
+ const acquired = await tempVolume.acquire(owner);
285
389
  if (!acquired.ok) {
286
390
  const message = `${describeRamTempError(acquired.error)}\n`;
287
391
  process.stderr.write(message);
@@ -293,7 +397,28 @@ export async function runBoundedExec(
293
397
  appendStderr(`RAM temp volume: dead lease ${held}\n`);
294
398
  }
295
399
  for (const reaped of acquired.value.reaped) {
296
- appendStderr(`RAM temp volume: stopped ${reaped}, which a dead task left working in its lease\n`);
400
+ appendStderr(`RAM temp volume: the supervisor stopped ${reaped}, which a dead task's command left running\n`);
401
+ }
402
+ for (const gap of acquired.value.gaps) {
403
+ appendStderr(`RAM temp volume: the supervisor's view of a dead task's command has a gap: ${gap}\n`);
404
+ }
405
+ for (const retained of acquired.value.retained) {
406
+ appendStderr(`RAM temp volume: dead lease ${retained}\n`);
407
+ }
408
+ } else if (acquired.value.kind === 'unavailable') {
409
+ const unavailable = `RAM temp volume unavailable (${acquired.value.reason}: ${acquired.value.detail})`;
410
+ if (options.env?.TMPDIR !== undefined) {
411
+ appendStderr(`${unavailable}; using target-provided TMPDIR ${options.env.TMPDIR} (medium not inferred)\n`);
412
+ } else {
413
+ try {
414
+ ordinaryTemp = await mkdtemp(join(tempVolume.ordinaryParent, `smoo-bounded-exec-${process.pid}-`));
415
+ } catch (error) {
416
+ appendStderr(
417
+ `${unavailable}; could not create ordinary scratch under ${tempVolume.ordinaryParent}: ${error instanceof Error ? error.message : String(error)}\n`,
418
+ );
419
+ return { success: false, terminalOutput: outputChunks.join('') };
420
+ }
421
+ appendStderr(`${unavailable}; using owned ordinary TMPDIR ${ordinaryTemp}\n`);
297
422
  }
298
423
  } else {
299
424
  appendStderr(
@@ -301,71 +426,125 @@ export async function runBoundedExec(
301
426
  );
302
427
  }
303
428
  }
304
- const releaseLease = async (): Promise<void> => {
305
- if (lease === null) {
429
+ // The supervisor's answer to the command's own admission, once it is spawned as an owned subtree root, and once
430
+ // any lease has recorded the handle for whoever settles that lease should this task die first.
431
+ let admission: Promise<SubtreeAdmission> | null = null;
432
+ const retireSubtree = async (): Promise<void> => {
433
+ if (admission === null || owner === null) {
306
434
  return;
307
435
  }
308
- const held = lease;
309
- lease = null;
310
- // What the command left working in its lease is a leak, whichever way the command ended; it
311
- // is stopped before the lease goes, so the volume can detach, and said, so its owner fixes it.
312
- const reaped = await tempVolume?.reap(held);
313
- if (reaped?.ok) {
314
- for (const leftover of reaped.value) {
315
- appendStderr(`RAM temp volume: stopped ${leftover}, which the command left working in its lease\n`);
316
- }
317
- } else if (reaped) {
318
- appendStderr(`${describeRamTempError(reaped.error)}\n`);
436
+ const answer = await admission;
437
+ admission = null;
438
+ if (answer.kind !== 'admitted') {
439
+ appendStderr(
440
+ `cowshed: the command ran without an owned subtree (${answer.kind === 'refused' ? `supervisor: ${answer.reason}` : answer.detail}); nothing retires what it left running\n`,
441
+ );
442
+ return;
319
443
  }
320
- const released = await tempVolume?.release(held);
321
- if (released && !released.ok) {
322
- appendStderr(`${describeRamTempError(released.error)}\n`);
444
+ // What the command left running is a leak, whichever way the command ended: its supervisor ends exactly the
445
+ // processes its tree records under the command, before any lease goes, and each is said so its owner fixes it.
446
+ const retired = await owner.retire(answer.handle, LEFTOVER_GRACE_MS);
447
+ switch (retired.kind) {
448
+ case 'retired':
449
+ subtreeRetired = true;
450
+ for (const entry of retired.signalled) {
451
+ appendStderr(`cowshed: the supervisor stopped ${describeSignalled(entry)}, which the command left running\n`);
452
+ }
453
+ if (retired.signalledIncomplete) {
454
+ appendStderr("cowshed: the supervisor may have stopped processes of the command's that it could not name\n");
455
+ }
456
+ for (const gap of retired.gaps) {
457
+ appendStderr(
458
+ `cowshed: the supervisor's view of the command's processes has a gap: ${describeCoverageGap(gap)}\n`,
459
+ );
460
+ }
461
+ return;
462
+ case 'refused':
463
+ appendStderr(`cowshed: the supervisor refused to retire the command's processes (${retired.reason})\n`);
464
+ return;
465
+ case 'failed':
466
+ state.retireError = retired.detail;
467
+ appendStderr(`cowshed: could not retire the command's processes: ${retired.detail}\n`);
468
+ return;
323
469
  }
324
470
  };
471
+ // Failed scratch remains evidence: RAM leases are archived; ordinary scratch is retained at its reported path.
472
+ let tempReleased: Promise<void> | null = null;
473
+ const releaseLease = (keep: boolean): Promise<void> => {
474
+ tempReleased ??= (async () => {
475
+ await retireSubtree();
476
+ if (ordinaryTemp !== null) {
477
+ const held = ordinaryTemp;
478
+ ordinaryTemp = null;
479
+ if (keep || state.retireError !== null || !subtreeRetired) {
480
+ appendStderr(
481
+ `Ordinary temp directory retained at ${held}: ${!subtreeRetired ? 'the command subtree was not retired' : 'the run did not pass'}\n`,
482
+ );
483
+ } else {
484
+ try {
485
+ await rm(held, { recursive: true, force: true });
486
+ } catch (error) {
487
+ appendStderr(
488
+ `Ordinary temp directory retained at ${held}: cleanup failed: ${error instanceof Error ? error.message : String(error)}\n`,
489
+ );
490
+ }
491
+ }
492
+ }
493
+ if (lease === null || owner === null) {
494
+ return;
495
+ }
496
+ const held = lease;
497
+ lease = null;
498
+ const released = await tempVolume?.release(held, keep || state.retireError !== null, owner);
499
+ if (released?.ok) {
500
+ for (const retained of released.value) {
501
+ appendStderr(`RAM temp volume: lease ${retained}\n`);
502
+ }
503
+ } else if (released) {
504
+ appendStderr(`${describeRamTempError(released.error)}\n`);
505
+ }
506
+ })();
507
+ return tempReleased;
508
+ };
325
509
 
326
- // The wait is one span line, as cowshed's own spans read. On a host without a gateway the Nx run serves its own
327
- // pool (`run-cpu-pool.ts`), taken below as a parent's; a command no pool and no gateway answers runs exactly as
510
+ // The wait is one span line, as cowshed's own spans read. Every Nx run that inherits no pool serves its own
511
+ // (`run-cpu-pool.ts`), so a task takes its tokens from a parent's pool; a command that inherits none runs exactly as
328
512
  // asked and nothing is said.
329
513
  const cores = availableParallelism();
330
514
  const demand = runnerDemand(requested, options.parallelism, cores, process.env, options.env);
331
515
  const asked = Date.now();
332
- // A run a parent's CPU pool serves takes its tokens there. Only an absent pool leaves the host budget to ask.
333
- // A pool that answered and refused is a live budget of its own -- a lent grant, or a gate's or a resident's
334
- // relay to the host -- so asking the host beside it would count the run twice; the run fails instead.
516
+ // A pool that answered and refused is a live budget, and running beside it would count the run twice: the run fails
517
+ // instead. A pool whose endpoint is gone (its holder exited) lends nothing, and the run goes on as asked.
335
518
  let grant: GrantedTokens | null = null;
336
519
  let parentRefusal: string | null = null;
520
+ let parentWait: CpuWaitRow | undefined;
337
521
  if (inherited !== null) {
338
522
  const fromParent = await takeParentCpuTokens(inherited, demand.want);
339
523
  if (fromParent.granted) {
340
524
  grant = fromParent;
341
525
  } else if (fromParent.cause === 'absent') {
342
- appendStderr(`cowshed: cpu-tokens parent pool refused: ${fromParent.reason}; asking the host budget\n`);
526
+ appendStderr(`cowshed: cpu-pool parent pool absent: ${fromParent.reason}; running the command as asked\n`);
343
527
  } else {
344
528
  parentRefusal = fromParent.reason;
345
- }
346
- }
347
- let unbudgeted: string | null = null;
348
- if (grant === null && parentRefusal === null && budget !== null) {
349
- const fromHost = await budget.take(demand.want, context.root, requested);
350
- if (fromHost.granted) {
351
- grant = fromHost;
352
- } else if (fromHost.cause !== 'absent') {
353
- unbudgeted = fromHost.reason;
529
+ parentWait = fromParent.wait;
354
530
  }
355
531
  }
356
532
  const waitedMs = Date.now() - asked;
357
533
  let cpu: BoundedExecRecord['cpu'];
358
534
  let sized: { command: string; env: Record<string, string> } = { command: requested, env: {} };
535
+ // With mbx the leaf's admission is mbx's one pool, not a parent's tokens: its runner is sized to the job's count,
536
+ // and a non-Cargo leaf holds that much of the pool for its run (specs/cowshed/05_gateway.md, "Parallelism inside a
537
+ // job"). No grant and no sub-pool beside it.
538
+ const mbx = mbxPool(process.env);
539
+ const leaf = mbx === null ? null : mbxLeaf(mbx, demand.kind, demand.want);
359
540
  if (grant !== null) {
360
541
  sized = sizeRunner(demand, requested, grant.tokens);
361
- cpu = { tokens: grant.tokens, want: demand.want, waitedMs, ...(grant.inherited ? { inherited: true } : {}) };
362
- appendStderr(
363
- `cowshed: cpu-tokens wait done elapsed=${waitedMs}ms status=ok ${grant.inherited ? 'inherited=true ' : ''}tokens=${grant.tokens}/${demand.want} runner=${demand.kind}\n`,
364
- );
365
- } else if (unbudgeted !== null) {
542
+ cpu = { tokens: grant.tokens, want: demand.want, waitedMs };
366
543
  appendStderr(
367
- `cowshed: cpu-tokens wait done elapsed=${waitedMs}ms status=err runner=${demand.kind} runs unbudgeted: ${unbudgeted}\n`,
544
+ `cowshed: cpu-pool wait done elapsed=${waitedMs}ms status=ok tokens=${grant.tokens}/${demand.want} runner=${demand.kind}\n`,
368
545
  );
546
+ } else if (leaf !== null) {
547
+ sized = sizeRunner(demand, requested, demand.want);
369
548
  }
370
549
  // The run's sub-pool and grant are given back exactly once, however the run ends. The sub-pool closes first, so no
371
550
  // descendant still draws on tokens the grant has returned.
@@ -386,14 +565,15 @@ export async function runBoundedExec(
386
565
  return cpuReleased;
387
566
  };
388
567
  // Records the run's verdict when it has a task to record as. An `expiry` is set only when a bound ended the run.
389
- // An `operational` run failed on its CPU budget (a sub-pool that did not close): an ordinary failure, whatever
390
- // bound also fired, so it is never judged as a bound.
568
+ // An `operational` run failed on its CPU pool (a sub-pool that did not close): an ordinary failure, whatever bound also
569
+ // fired, so it is never judged as a bound.
391
570
  const recordRun = async (
392
571
  exitCode: number,
393
572
  expiry: ExpiredBound | null,
394
573
  elapsedMs: number,
395
574
  operational: boolean,
396
575
  noSpace: NoSpaceSighting | null,
576
+ lockWaits: readonly LockWait[],
397
577
  ): Promise<void> => {
398
578
  if (context.record === undefined) {
399
579
  return;
@@ -402,46 +582,35 @@ export async function runBoundedExec(
402
582
  const report = await readJunitReport(recordDirectory);
403
583
  const verdict = operational
404
584
  ? judgeOperationalFailure(exitCode, report)
405
- : judgeRun({ exitCode, elapsedMs, expiry, report, noSpace });
585
+ : judgeRun({ exitCode, elapsedMs, expiry, report, noSpace, lockWaits });
406
586
  if (verdict.outcome === 'bound' && verdict.bound === 'test') {
407
587
  appendStderr(`Every failing test failed only on its per-test timeout: ${verdict.tests.join(', ')}\n`);
408
588
  }
409
- await writeRecord(recordDirectory, { task: task.id, hash: task.hash, verdict, ...(cpu ? { cpu } : {}) });
589
+ await writeRecord(recordDirectory, {
590
+ task: task.id,
591
+ hash: task.hash,
592
+ verdict,
593
+ ...(cpu ? { cpu } : {}),
594
+ ...(parentWait ? { cpuWait: parentWait } : {}),
595
+ });
410
596
  };
411
597
  // A run refused before its command starts says why, gives back its grant and RAM, and records the failure.
412
598
  const refuse = async (message: string): Promise<BoundedExecResult> => {
413
599
  appendStderr(message);
414
600
  await releaseCpu();
415
- await releaseLease();
416
- await recordRun(1, null, Date.now() - asked, false, null);
601
+ // The command never started, so its lease holds nothing of its own to keep.
602
+ await releaseLease(false);
603
+ await recordRun(1, null, Date.now() - asked, false, null, []);
417
604
  return { success: false, terminalOutput: outputChunks.join('') };
418
605
  };
419
606
  if (parentRefusal !== null) {
420
- return refuse(
421
- `cowshed: cpu-tokens parent pool refused: ${parentRefusal}; refusing to run the command without asking the host budget\n`,
422
- );
607
+ return refuse(`cowshed: cpu-pool parent pool refused: ${parentRefusal}; refusing to run the command\n`);
423
608
  }
424
609
 
425
- // Descendant work queued for host CPU tokens, as its pool hears it (`host-wait`): while any is held the idle clock
426
- // is stopped, because a queued job is not an idle one (specs/cowshed/05_gateway.md, "The bounds start at the
427
- // grant"). The clock it stops is made with the command below; a wait held before then is counted and honoured there.
428
- let heldWaits = 0;
429
- let idleClock: { stop(): void; restart(): void } | null = null;
430
- const hostWaits: HostWaits = {
431
- begin: () => {
432
- heldWaits += 1;
433
- if (heldWaits === 1) idleClock?.stop();
434
- },
435
- end: () => {
436
- heldWaits -= 1;
437
- if (heldWaits === 0) idleClock?.restart();
438
- },
439
- };
440
-
441
610
  // A granted run serves a sub-pool of its own grant, so its descendants draw from that and never from above it.
442
- // A pool that cannot be set up refuses the run: no command runs unbudgeted.
611
+ // A pool that cannot be set up refuses the run: no granted command runs beside its grant.
443
612
  if (grant !== null) {
444
- const created = await createCpuPool(grant.tokens, grant.closed, hostWaits);
613
+ const created = await createCpuPool(grant.tokens, grant.closed);
445
614
  if (!created.ok) {
446
615
  return refuse(`cowshed: cpu pool unavailable: ${created.error}; refusing to run the command\n`);
447
616
  }
@@ -461,25 +630,62 @@ export async function runBoundedExec(
461
630
  ...sized.env,
462
631
  ...(pool?.environment ?? {}),
463
632
  };
464
- const hooked = process.platform === 'win32' ? null : withRunnerLifecycle(command);
633
+ // A fixture rides as segment 0 of the lifecycle. Without the channel (Windows) it is plain shell text ahead
634
+ // of the command: the progress bound runs during it and its own ceiling does not.
635
+ const hooked =
636
+ process.platform === 'win32'
637
+ ? null
638
+ : fixture === null
639
+ ? withRunnerLifecycle(command)
640
+ : withFixtureLifecycle(fixture.command, command);
465
641
  const lifecycle = hooked === null ? null : createRunnerLifecycle(hooked.runners);
466
642
  let lifecycleFailure: string | null = null;
467
643
  const isTesting = (): boolean => lifecycleFailure !== null || (lifecycle?.isTesting() ?? true);
468
644
  const startedAt = Date.now();
645
+ // With a supervisor, `cowshed subtree-root` asks it for a handle on itself, writes the answer to the handle
646
+ // descriptor, and execs `/bin/sh -c command` in the same process: the command's pid, group and life are unchanged.
647
+ const unadmitted = hooked?.command ?? (fixture === null ? command : `${fixture.command} && ${command}`);
648
+ const shellCommand = leaf === null ? unadmitted : withMbxLeaf(unadmitted, leaf);
649
+ const root = owner?.rootArgv(shellCommand);
469
650
  let child: ChildProcess;
470
651
  try {
471
- child = spawn(hooked?.command ?? command, [], {
472
- cwd,
473
- env: mergeEnv(lease, env),
474
- shell: true,
475
- stdio: hooked === null ? 'pipe' : ['pipe', 'pipe', 'pipe', 'pipe'],
476
- detached: process.platform !== 'win32',
477
- windowsHide: true,
478
- });
652
+ // fd 3 is the runner lifecycle's duplex socket when the command is hooked; the subtree admission takes fd 4, which
653
+ // `cowshed subtree-root` closes before it execs, so fd 3 reaches the shell unchanged.
654
+ child =
655
+ root === undefined
656
+ ? spawn(shellCommand, [], {
657
+ cwd,
658
+ env: mergeEnv(lease?.directory ?? ordinaryTemp, env),
659
+ shell: true,
660
+ stdio: hooked === null ? 'pipe' : ['pipe', 'pipe', 'pipe', 'pipe'],
661
+ detached: process.platform !== 'win32',
662
+ windowsHide: true,
663
+ })
664
+ : spawn(root.file, root.args, {
665
+ cwd,
666
+ env: mergeEnv(lease?.directory ?? ordinaryTemp, env),
667
+ stdio: ['pipe', 'pipe', 'pipe', hooked === null ? 'ignore' : 'pipe', 'pipe'],
668
+ detached: process.platform !== 'win32',
669
+ windowsHide: true,
670
+ });
479
671
  } catch (error) {
480
672
  // Nothing started, so nothing runs under the grant: it goes back here, with the RAM lease.
481
673
  return refuse(`cowshed: could not start the command: ${error instanceof Error ? error.message : String(error)}\n`);
482
674
  }
675
+ subtreeRetired = false;
676
+ if (root !== undefined) {
677
+ const channel = child.stdio[HANDLE_FD];
678
+ const leased = lease;
679
+ admission = readAdmission(channel instanceof Readable ? channel : null).then(async (answer) => {
680
+ if (answer.kind === 'admitted' && leased !== null && tempVolume !== null) {
681
+ const recorded = await tempVolume.record(leased, answer.handle);
682
+ if (!recorded.ok) {
683
+ appendStderr(`${describeRamTempError(recorded.error)}\n`);
684
+ }
685
+ }
686
+ return answer;
687
+ });
688
+ }
483
689
 
484
690
  // Progress is a finished test, not output: a runner's SLOW warnings and a hung test's heartbeat keep printing.
485
691
  const progress = createRunProgress(command);
@@ -490,24 +696,22 @@ export async function runBoundedExec(
490
696
  const held = lease;
491
697
  return { line, volume: held === null ? null : ((await tempVolume?.space(held)) ?? null) };
492
698
  };
493
- let idleTimer: NodeJS.Timeout | undefined;
699
+ // Which bound is armed, and when it expires, is bounds.ts's decision; one timer waits for the earliest of them.
700
+ let bounds: Bounds = UNARMED;
701
+ let boundTimer: NodeJS.Timeout | undefined;
702
+ // The fixture's own output is compilation, not the command's progress: it is not scanned for finished tests.
703
+ let fixtureDone = fixture === null || lifecycle === null;
704
+ // Whether a segment past the fixture ever announced: without one, the shell's exit is the fixture's.
705
+ let mainStarted = fixture === null || lifecycle === null;
494
706
  let captureTimer: NodeJS.Timeout | undefined;
495
707
  let idleCapture: { expiresAtMs: number; controller: AbortController; result: IdleDiagnosticResult } | undefined;
496
708
  const cancelCapture = (): void => {
497
709
  clearTimeout(captureTimer);
498
710
  idleCapture?.controller.abort();
499
711
  };
500
- // What is left of the current no-progress interval. A finished test sets it back to `idleTimeoutMs`; a held host
501
- // wait keeps it unspent: the clock is stopped while descendant work is queued for CPU tokens, and runs on from where
502
- // it stopped once the last wait ends. `queuedMs` is the queued time this interval has not counted, for the report.
503
- let idleLeftMs = idleTimeoutMs ?? 0;
504
- let queuedMs = 0;
505
- let queuedSinceMs: number | null = null;
506
- const scheduleIdle = (limitMs: number, leftMs: number): void => {
507
- clearTimeout(idleTimer);
508
- cancelCapture();
712
+ const scheduleCapture = (expiresAtMs: number): void => {
509
713
  const capture: NonNullable<typeof idleCapture> = {
510
- expiresAtMs: Date.now() + leftMs,
714
+ expiresAtMs,
511
715
  controller: new AbortController(),
512
716
  result: { kind: 'pending', startedAtMs: null },
513
717
  };
@@ -545,70 +749,45 @@ export async function runBoundedExec(
545
749
  },
546
750
  );
547
751
  },
548
- Math.max(0, leftMs - HANG_CAPTURE_MS),
752
+ Math.max(0, expiresAtMs - Date.now() - HANG_CAPTURE_MS),
549
753
  );
550
- idleTimer = setTimeout(() => expire({ kind: 'idle', limitMs, tests: progress.inFlight() }, queuedMs), leftMs);
551
754
  };
552
- const armIdleTimer = (): void => {
553
- if (idleTimeoutMs === undefined || state.settled || state.expiry !== null || !isTesting()) {
554
- return;
555
- }
556
- idleLeftMs = idleTimeoutMs;
557
- queuedMs = 0;
558
- if (queuedSinceMs !== null) {
559
- queuedSinceMs = Date.now();
560
- return;
561
- }
562
- scheduleIdle(idleTimeoutMs, idleLeftMs);
563
- };
564
- idleClock = {
565
- stop: () => {
566
- if (idleTimeoutMs === undefined || state.settled || state.expiry !== null || queuedSinceMs !== null) {
567
- return;
568
- }
569
- const now = Date.now();
570
- queuedSinceMs = now;
571
- if (idleCapture !== undefined) {
572
- idleLeftMs = Math.max(0, idleCapture.expiresAtMs - now);
573
- }
574
- clearTimeout(idleTimer);
755
+ const expiredAs = (kind: BoundKind): ExpiredBound =>
756
+ kind === 'idle'
757
+ ? { kind, limitMs: idleTimeoutMs ?? 0, tests: progress.inFlight() }
758
+ : kind === 'fixture'
759
+ ? { kind, limitMs: fixture?.timeoutMs ?? 0 }
760
+ : { kind, limitMs: timeoutMs ?? 0 };
761
+ const stepTo = (event: BoundEvent): void => {
762
+ if (state.settled || state.expiry !== null) return;
763
+ const idleBefore = bounds.idle;
764
+ bounds = stepBounds(
765
+ bounds,
766
+ {
767
+ ...(idleTimeoutMs === undefined ? {} : { idleMs: idleTimeoutMs }),
768
+ ...(fixture === null || lifecycle === null ? {} : { fixtureMs: fixture.timeoutMs }),
769
+ ...(timeoutMs === undefined ? {} : { totalMs: timeoutMs }),
770
+ },
771
+ event,
772
+ );
773
+ if (bounds.idle !== idleBefore) {
575
774
  cancelCapture();
576
775
  idleCapture = undefined;
577
- appendStderr(
578
- `bounded-exec: idle clock stopped: a descendant's work is queued for host CPU tokens; ${idleLeftMs}ms of idleTimeoutMs=${idleTimeoutMs} left\n`,
579
- );
580
- },
581
- restart: () => {
582
- if (idleTimeoutMs === undefined || state.settled || state.expiry !== null || queuedSinceMs === null) {
583
- return;
584
- }
585
- const waitedMs = Date.now() - queuedSinceMs;
586
- queuedMs += waitedMs;
587
- queuedSinceMs = null;
588
- if (!isTesting()) {
589
- appendStderr(`bounded-exec: host CPU wait ended after ${waitedMs}ms; no test runner is active\n`);
590
- return;
591
- }
592
- appendStderr(
593
- `bounded-exec: idle clock restarted: no descendant's work is queued for host CPU tokens after ${waitedMs}ms; ${idleLeftMs}ms of idleTimeoutMs=${idleTimeoutMs} left\n`,
594
- );
595
- scheduleIdle(idleTimeoutMs, idleLeftMs);
596
- },
776
+ if (bounds.idle !== null) scheduleCapture(bounds.idle);
777
+ }
778
+ clearTimeout(boundTimer);
779
+ const next = earliestBound(bounds);
780
+ if (next !== null) boundTimer = setTimeout(() => expire(expiredAs(next.kind)), Math.max(0, next.at - Date.now()));
597
781
  };
598
782
 
599
783
  // Only the CHILD's own output is read for progress. The diagnostics below go
600
784
  // through appendStderr as well, and re-arming on those would let a teardown
601
785
  // message extend the very bound that just fired.
602
786
  const updateIdle = (finished: boolean, wasTesting: boolean): void => {
603
- if (!isTesting()) {
604
- clearTimeout(idleTimer);
605
- cancelCapture();
606
- idleCapture = undefined;
607
- } else if (finished || !wasTesting) {
608
- armIdleTimer();
609
- }
787
+ stepTo({ kind: 'progress', at: Date.now(), testing: isTesting(), wasTesting, finished });
610
788
  };
611
789
  const scanProgress = (stream: 'stdout' | 'stderr', text: string): void => {
790
+ if (!fixtureDone) return;
612
791
  updateIdle(progress.scan(stream, text), isTesting());
613
792
  };
614
793
  if (lifecycle !== null) {
@@ -620,7 +799,8 @@ export async function runBoundedExec(
620
799
  appendStderr(`bounded-exec: runner lifecycle failed: ${detail}\n`);
621
800
  updateIdle(false, wasTesting);
622
801
  };
623
- createInterface({ input: channel }).on('line', (line) => {
802
+ const lines = createInterface({ input: channel });
803
+ lines.on('line', (line) => {
624
804
  const wasTesting = isTesting();
625
805
  const event = lifecycle.accept(line);
626
806
  if (event === null) {
@@ -628,9 +808,17 @@ export async function runBoundedExec(
628
808
  return;
629
809
  }
630
810
  updateIdle(false, wasTesting);
811
+ if (event.id === 0 && event.phase === 'terminal' && fixture !== null) {
812
+ fixtureDone = true;
813
+ stepTo({ kind: 'fixture-done' });
814
+ }
815
+ if (event.id > 0) mainStarted = true;
631
816
  // Tool startup and native lifecycle-bearing stdout wait for their phase acknowledgement.
632
817
  channel.write(`${JSON.stringify(event)}\n`);
633
818
  });
819
+ // readline re-emits its input's errors on the interface, where one with no listener is thrown: a
820
+ // runner killed with unread lifecycle bytes resets the channel (ECONNRESET on Linux).
821
+ lines.on('error', (error) => failLifecycle(error.message));
634
822
  channel.on('error', (error) => failLifecycle(error.message));
635
823
  channel.on('end', () => {
636
824
  if (!lifecycle.finished()) failLifecycle('supervisor channel ended without an owned terminal event');
@@ -651,23 +839,30 @@ export async function runBoundedExec(
651
839
  appendStderr(text);
652
840
  });
653
841
 
654
- const killChildTree = async (force: boolean): Promise<void> => {
842
+ // The graceful pass: SIGTERM to the whole group. A signal that fails is told, not thrown: every caller has more to do
843
+ // after it, `terminate` the force-kill and the signal handler its releases and the signal it gives back.
844
+ const killChildTree = async (): Promise<void> => {
655
845
  const pid = child.pid;
656
846
  if (!pid) {
657
847
  return;
658
848
  }
659
- await killer.kill(pid, force ? 'SIGKILL' : 'SIGTERM');
849
+ try {
850
+ await killer.kill(pid, 'SIGTERM');
851
+ } catch (error) {
852
+ const reason = error instanceof Error ? error.message : String(error);
853
+ tell(`Terminating process group ${pid} failed: ${reason}\n`);
854
+ }
660
855
  };
661
856
 
662
857
  const onProcessExit = (): void => {
663
- void killChildTree(false);
858
+ void killChildTree();
664
859
  };
665
860
  const onTerminationSignal = (signal: NodeJS.Signals): void => {
666
861
  removeSignalHandlers();
667
862
  cancelCapture();
668
- void killChildTree(false)
863
+ void killChildTree()
669
864
  .finally(releaseCpu)
670
- .finally(releaseLease)
865
+ .finally(() => releaseLease(true))
671
866
  .finally(() => process.kill(process.pid, signal));
672
867
  };
673
868
  const onSigint = (): void => onTerminationSignal('SIGINT');
@@ -706,28 +901,27 @@ export async function runBoundedExec(
706
901
  // the reader unable to tell a wedged toolchain from a loaded machine — the
707
902
  // two have opposite fixes.
708
903
  let termination: Promise<void> = Promise.resolve();
709
- // `queuedMs` is the time the idle interval did not count because descendant work was queued for host CPU tokens.
710
- const expire = (bound: ExpiredBound, queuedMs = 0): void => {
904
+ const expire = (bound: ExpiredBound): void => {
711
905
  if (state.settled || state.expiry !== null) {
712
906
  return;
713
907
  }
714
908
  state.expiry = bound;
715
- clearTimeout(totalTimer);
716
- clearTimeout(idleTimer);
909
+ clearTimeout(boundTimer);
717
910
  const elapsedMs = Date.now() - startedAt;
718
911
  appendStderr(
719
912
  bound.kind === 'idle'
720
- ? `\nCommand made no progress: no test finished for ${bound.limitMs}ms (idleTimeoutMs=${bound.limitMs})${
721
- queuedMs > 0 ? `, not counting ${queuedMs}ms queued for host CPU tokens,` : ''
722
- } after ${elapsedMs}ms of runtime; ${
913
+ ? `\nCommand made no progress: no test finished for ${bound.limitMs}ms (idleTimeoutMs=${bound.limitMs}) after ${elapsedMs}ms of runtime; ${
723
914
  bound.tests.length > 0
724
915
  ? `still running: ${bound.tests.join(', ')}`
725
916
  : 'the runner named no test still running'
726
917
  } (cwd=${cwd}): ${command}\n`
727
- : `\nCommand timed out after ${elapsedMs}ms (timeoutMs=${bound.limitMs}, cwd=${cwd}): ${command}\n`,
918
+ : bound.kind === 'fixture'
919
+ ? `\nFixture command timed out after ${elapsedMs}ms (fixtureTimeoutMs=${bound.limitMs}, cwd=${cwd}): ${fixture?.command ?? command}\n`
920
+ : `\nCommand timed out after ${elapsedMs}ms (timeoutMs=${bound.limitMs}, cwd=${cwd}): ${command}\n`,
728
921
  );
729
922
  if (bound.kind === 'idle') {
730
923
  const observed = idleCapture?.result;
924
+ state.lockWaits = observed?.kind === 'captured' ? observed.snapshot.lockWaits : [];
731
925
  appendStderr(
732
926
  `Hang diagnostics: runner pid=${child.pid ?? 'unavailable'} pgid=${
733
927
  process.platform === 'win32' ? 'unavailable' : (child.pid ?? 'unavailable')
@@ -753,24 +947,49 @@ export async function runBoundedExec(
753
947
  termination = terminate();
754
948
  };
755
949
 
950
+ // Whether the group still lists members. A probe that fails proves nothing about the group, so it is told, and the
951
+ // group is taken as listed: a force-kill that was not needed costs nothing, and one that was is not skipped.
952
+ const groupListed = (pid: number): boolean => {
953
+ try {
954
+ return processGroupHasMembers(pid);
955
+ } catch (error) {
956
+ const reason = error instanceof Error ? error.message : String(error);
957
+ tell(`Probing process group ${pid} for members failed: ${reason}; taking it as still listing members\n`);
958
+ return true;
959
+ }
960
+ };
961
+
756
962
  // A timed-out run ends with its whole process group, not with its leader.
757
963
  // The graceful signal reaches every member that exists when it is sent, but
758
964
  // a member that ignores SIGTERM, or one forked while the signal goes out
759
965
  // (macOS does not hold a fork back for a group signal in flight), outlives
760
966
  // a leader that exits on it. The grace ends when the leader exits or
761
967
  // killAfterMs passes, whichever comes first; whatever is left then is killed.
968
+ // SIGKILL is no different: a member forked while it goes out survives it too,
969
+ // so the force-kill is repeated while the group lists members. What the repeats
970
+ // cannot end is said, and a failure of the probe or of a signal is shown.
971
+ const forceKill = async (pid: number): Promise<void> => {
972
+ try {
973
+ if (await killProcessGroup(pid, DEADLINE_PATIENCE, (group) => killer.kill(group, 'SIGKILL'))) {
974
+ tell(`${describeMembersLeft(pid)}\n`);
975
+ }
976
+ } catch (error) {
977
+ const reason = error instanceof Error ? error.message : String(error);
978
+ tell(`Force-kill of process group ${pid} failed: ${reason}\n`);
979
+ }
980
+ };
762
981
  const terminate = async (): Promise<void> => {
763
- await ignoreKillError(killChildTree(false));
982
+ await killChildTree();
764
983
  if (!state.settled && killAfterMs > 0) {
765
984
  await settledWithin(exitPromise, killAfterMs);
766
985
  }
767
986
  if (state.settled) {
768
- if (child.pid !== undefined && processGroupHasMembers(child.pid)) {
987
+ if (child.pid !== undefined && groupListed(child.pid)) {
769
988
  state.forceKillNeeded = true;
770
989
  appendStderr(
771
990
  `Force-killing what the ${state.expiry === null ? 'command' : 'timed-out command'} left in its process group: ${command}\n`,
772
991
  );
773
- await ignoreKillError(killChildTree(true));
992
+ await forceKill(child.pid);
774
993
  }
775
994
  return;
776
995
  }
@@ -778,23 +997,25 @@ export async function runBoundedExec(
778
997
  appendStderr(
779
998
  `Force-killing ${state.expiry === null ? 'command' : 'timed out command'} after killAfterMs=${killAfterMs}: ${command}\n`,
780
999
  );
781
- await ignoreKillError(killChildTree(true));
782
- await settledWithin(exitPromise, REAP_AFTER_FORCE_KILL_MS);
1000
+ if (child.pid !== undefined) await forceKill(child.pid);
1001
+ // The repeats wait for the group to empty, the leader with it. Windows has no process groups, so there they have
1002
+ // nothing to wait on once the one signal is sent, and the leader's exit is awaited as it always was.
1003
+ if (process.platform === 'win32' || child.pid === undefined) {
1004
+ await settledWithin(exitPromise, REAP_AFTER_FORCE_KILL_MS);
1005
+ }
783
1006
  if (!state.settled) {
784
1007
  resolveExit({ code: 1, signal: 'SIGKILL' });
785
1008
  }
786
1009
  };
787
1010
 
788
- const totalTimer =
789
- timeoutMs === undefined ? undefined : setTimeout(() => expire({ kind: 'total', limitMs: timeoutMs }), timeoutMs);
790
- armIdleTimer();
791
- // A wait its pool heard before the command started stops the clock it was not yet there to stop.
792
- if (heldWaits > 0) idleClock.stop();
1011
+ // The fixture's ceiling lives and dies with its segment: the segment's terminal clears it, so it can never
1012
+ // fire on a command the fixture already handed over to.
1013
+ stepTo({ kind: 'spawn', at: Date.now(), testing: isTesting(), fixture: fixture !== null && lifecycle !== null });
793
1014
 
794
- // A grant's connection is a ledger entry, not the command's owner (05_gateway.md "Host CPU budget": a gateway
795
- // restart forgets every grant, and what runs under it runs on over budget until it finishes). Its close, while the
796
- // command runs, is said and changes nothing else; only a bound or a signal stops the command. Our own release after
797
- // the command settled is not a loss; `cpuReleased` marks it.
1015
+ // A grant's connection is a ledger entry, not the command's owner: the lending parent's exit ends it, and what runs
1016
+ // under it runs on over budget until it finishes. Its close, while the command runs, is said and changes nothing
1017
+ // else; only a bound or a signal stops the command. Our own release after the command settled is not a loss;
1018
+ // `cpuReleased` marks it.
798
1019
  if (grant !== null) {
799
1020
  const held = grant;
800
1021
  void held.closed.then(() => {
@@ -802,15 +1023,14 @@ export async function runBoundedExec(
802
1023
  return;
803
1024
  }
804
1025
  appendStderr(
805
- `cowshed: cpu-tokens grant ended: the ${held.inherited ? 'parent pool' : 'gateway'} closed its ${held.tokens}-token grant while the command ran; it runs on over budget until it ends: ${command}\n`,
1026
+ `cowshed: cpu-pool grant ended: the parent pool closed its ${held.tokens}-token grant while the command ran; it runs on over budget until it ends: ${command}\n`,
806
1027
  );
807
1028
  });
808
1029
  }
809
1030
 
810
1031
  const { code: exitCode, signal: exitSignal } = await exitPromise;
811
1032
  await termination;
812
- clearTimeout(totalTimer);
813
- clearTimeout(idleTimer);
1033
+ clearTimeout(boundTimer);
814
1034
  cancelCapture();
815
1035
  removeSignalHandlers();
816
1036
  // The runner is gone. Its sub-pool closes first, so no descendant still draws on tokens the grant returns;
@@ -819,7 +1039,11 @@ export async function runBoundedExec(
819
1039
 
820
1040
  const code = exitCode ?? signalToExitCode(exitSignal);
821
1041
  if (code !== 0 && state.expiry === null) {
822
- appendStderr(`Command exited with status ${code}: ${command}\n`);
1042
+ appendStderr(
1043
+ fixture !== null && !mainStarted
1044
+ ? `Fixture command exited with status ${code}: ${fixture.command}\n`
1045
+ : `Command exited with status ${code}: ${command}\n`,
1046
+ );
823
1047
  }
824
1048
 
825
1049
  if (state.expiry !== null && !state.forceKillNeeded) {
@@ -838,13 +1062,12 @@ export async function runBoundedExec(
838
1062
  appendStderr(`${describeRamTempError(full)}\n`);
839
1063
  }
840
1064
  }
841
- await releaseLease();
842
-
843
- // Broken resource ownership or a lost native lifecycle fails even if a shell branch later exited 0:
844
- // its verdict is an ordinary operational failure, never a successful or silently unbounded run.
845
- const operationalFailure = state.poolCloseError !== null || lifecycleFailure !== null;
846
- await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure, seen);
1065
+ await releaseLease(code !== 0 || state.expiry !== null || state.poolCloseError !== null || lifecycleFailure !== null);
847
1066
 
1067
+ // Broken resource ownership, a lost native lifecycle, or leftovers that could not be retired fail even if a shell
1068
+ // branch later exited 0: its verdict is an ordinary operational failure, never a successful or silently unbounded run.
1069
+ const operationalFailure = state.poolCloseError !== null || lifecycleFailure !== null || state.retireError !== null;
1070
+ await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure, seen, state.lockWaits);
848
1071
  return {
849
1072
  success: !operationalFailure && state.expiry === null && code === 0,
850
1073
  terminalOutput: outputChunks.join(''),
@@ -853,45 +1076,12 @@ export async function runBoundedExec(
853
1076
 
854
1077
  export function createProcessTreeKiller(): ProcessTreeKiller {
855
1078
  return {
856
- kill(pid, signal) {
857
- return killProcessGroup(pid, signal);
1079
+ async kill(pid, signal) {
1080
+ signalProcessGroup(pid, signal);
858
1081
  },
859
1082
  };
860
1083
  }
861
1084
 
862
- function killProcessGroup(pid: number, signal: NodeJS.Signals): Promise<void> {
863
- try {
864
- // On POSIX the executor spawns with detached: true, so the child leads a
865
- // process group of its own and -pid signals all of it at once, including
866
- // grandchildren that tree-walk libraries miss when parents die and
867
- // children get reparented to init. Only the group is named: the leader may
868
- // already be reaped, and its pid reused by an unrelated process, while the
869
- // group lives on in its other members. Windows has no process groups.
870
- process.kill(process.platform === 'win32' ? pid : -pid, signal);
871
- } catch {
872
- // ESRCH: the group (or the Windows process) has already exited.
873
- }
874
- return Promise.resolve();
875
- }
876
-
877
- /**
878
- * Whether any process is left in the group `pgid` leads. A group outlives its
879
- * reaped leader for as long as one member does, and no process can take its id
880
- * meanwhile. Windows has no process groups, so there is nothing left to name.
881
- */
882
- function processGroupHasMembers(pgid: number): boolean {
883
- if (process.platform === 'win32') {
884
- return false;
885
- }
886
- try {
887
- process.kill(-pgid, 0);
888
- return true;
889
- } catch (error) {
890
- // EPERM: a member exists but runs as another user.
891
- return error instanceof Error && 'code' in error && error.code === 'EPERM';
892
- }
893
- }
894
-
895
1085
  function resolveCwd(cwd: string | undefined, root: string): string {
896
1086
  if (!cwd) {
897
1087
  return root;
@@ -912,10 +1102,10 @@ function buildCommand(options: BoundedExecOptions): string {
912
1102
  return parts.join(' ');
913
1103
  }
914
1104
 
915
- /** The lease's directory is the command's TMPDIR unless the target names one itself. */
916
- function mergeEnv(lease: RamTempLease | null, env: Record<string, string>): NodeJS.ProcessEnv {
1105
+ /** Owned scratch is the command's TMPDIR unless the target names one itself. */
1106
+ function mergeEnv(directory: string | null, env: Record<string, string>): NodeJS.ProcessEnv {
917
1107
  const inherited = withoutPoolEnv(process.env);
918
- return lease === null ? { ...inherited, ...env } : { ...inherited, TMPDIR: lease.directory, ...env };
1108
+ return directory === null ? { ...inherited, ...env } : { ...inherited, TMPDIR: directory, ...env };
919
1109
  }
920
1110
 
921
1111
  /** The process's environment without the CPU pool's descriptor: a pool and its count only a grant may give. */
@@ -949,14 +1139,6 @@ function signalToExitCode(signal: NodeJS.Signals | null): number {
949
1139
  return EXIT_CODE_BY_SIGNAL[signal] ?? 1;
950
1140
  }
951
1141
 
952
- async function ignoreKillError(promise: Promise<void>): Promise<void> {
953
- try {
954
- await promise;
955
- } catch {
956
- // The process tree may have already exited between timeout and kill.
957
- }
958
- }
959
-
960
1142
  /** Resolves when `promise` settles or `ms` passes, whichever is first, leaving no timer behind. */
961
1143
  async function settledWithin(promise: Promise<unknown>, ms: number): Promise<void> {
962
1144
  let timer: NodeJS.Timeout | undefined;