@smoothbricks/nx-plugin 0.4.25 → 0.4.27

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (348) hide show
  1. package/README.md +1078 -98
  2. package/dist/bin/smoo-nx-bound-failures.d.ts +3 -0
  3. package/dist/bin/smoo-nx-bound-failures.d.ts.map +1 -0
  4. package/dist/bin/smoo-nx-bound-failures.js +31 -0
  5. package/dist/bin/smoo-nx-cargo-hash.js +28 -3
  6. package/dist/bin/smoo-nx-exec.js +16 -85
  7. package/dist/bin/smoo-nx-failures.d.ts +3 -0
  8. package/dist/bin/smoo-nx-failures.d.ts.map +1 -0
  9. package/dist/bin/smoo-nx-failures.js +3 -0
  10. package/dist/bin/smoo-nx-nextest-extract.js +3 -2
  11. package/dist/bin/smoo-nx-test-walks.d.ts +3 -0
  12. package/dist/bin/smoo-nx-test-walks.d.ts.map +1 -0
  13. package/dist/bin/smoo-nx-test-walks.js +34 -0
  14. package/dist/bounded-test-policy.d.ts +7 -1
  15. package/dist/bounded-test-policy.d.ts.map +1 -1
  16. package/dist/bounded-test-policy.js +101 -40
  17. package/dist/build-state-output-policy.d.ts +25 -0
  18. package/dist/build-state-output-policy.d.ts.map +1 -0
  19. package/dist/build-state-output-policy.js +64 -0
  20. package/dist/cargo-closure-input.d.ts +16 -17
  21. package/dist/cargo-closure-input.d.ts.map +1 -1
  22. package/dist/cargo-closure-input.js +37 -44
  23. package/dist/cargo-dev-profile.d.ts +37 -0
  24. package/dist/cargo-dev-profile.d.ts.map +1 -0
  25. package/dist/cargo-dev-profile.js +108 -0
  26. package/dist/cargo-source-hash.d.ts +50 -8
  27. package/dist/cargo-source-hash.d.ts.map +1 -1
  28. package/dist/cargo-source-hash.js +691 -71
  29. package/dist/cargo-toolchain-policy.d.ts +4 -0
  30. package/dist/cargo-toolchain-policy.d.ts.map +1 -1
  31. package/dist/cargo-toolchain-policy.js +5 -0
  32. package/dist/cargo-workspace.d.ts +108 -12
  33. package/dist/cargo-workspace.d.ts.map +1 -1
  34. package/dist/cargo-workspace.js +299 -18
  35. package/dist/cross-check-policy.d.ts +21 -3
  36. package/dist/cross-check-policy.d.ts.map +1 -1
  37. package/dist/cross-check-policy.js +24 -6
  38. package/dist/disk-command-policy.d.ts +7 -0
  39. package/dist/disk-command-policy.d.ts.map +1 -0
  40. package/dist/disk-command-policy.js +98 -0
  41. package/dist/ensure-built-receipt.d.ts +25 -0
  42. package/dist/ensure-built-receipt.d.ts.map +1 -0
  43. package/dist/ensure-built-receipt.js +261 -0
  44. package/dist/ensure-built-witness.d.ts +57 -0
  45. package/dist/ensure-built-witness.d.ts.map +1 -0
  46. package/dist/ensure-built-witness.js +412 -0
  47. package/dist/ensure-built.d.ts +213 -12
  48. package/dist/ensure-built.d.ts.map +1 -1
  49. package/dist/ensure-built.js +624 -177
  50. package/dist/eslint-file-set.d.ts +7 -0
  51. package/dist/eslint-file-set.d.ts.map +1 -0
  52. package/dist/eslint-file-set.js +12 -0
  53. package/dist/executors/bounded-exec/cpu-pool.d.ts +63 -0
  54. package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -0
  55. package/dist/executors/bounded-exec/cpu-pool.js +379 -0
  56. package/dist/executors/bounded-exec/cpu-tokens.d.ts +82 -0
  57. package/dist/executors/bounded-exec/cpu-tokens.d.ts.map +1 -0
  58. package/dist/executors/bounded-exec/cpu-tokens.js +114 -0
  59. package/dist/executors/bounded-exec/disk-lease.d.ts +61 -0
  60. package/dist/executors/bounded-exec/disk-lease.d.ts.map +1 -0
  61. package/dist/executors/bounded-exec/disk-lease.js +91 -0
  62. package/dist/executors/bounded-exec/executor.d.ts +45 -2
  63. package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
  64. package/dist/executors/bounded-exec/executor.js +447 -52
  65. package/dist/executors/bounded-exec/gateway-lease.d.ts +52 -0
  66. package/dist/executors/bounded-exec/gateway-lease.d.ts.map +1 -0
  67. package/dist/executors/bounded-exec/gateway-lease.js +143 -0
  68. package/dist/executors/bounded-exec/hang-diagnostics.d.ts +35 -0
  69. package/dist/executors/bounded-exec/hang-diagnostics.d.ts.map +1 -0
  70. package/dist/executors/bounded-exec/hang-diagnostics.js +231 -0
  71. package/dist/executors/bounded-exec/progress.d.ts +35 -0
  72. package/dist/executors/bounded-exec/progress.d.ts.map +1 -0
  73. package/dist/executors/bounded-exec/progress.js +79 -0
  74. package/dist/executors/bounded-exec/ram-temp-fixtures.d.ts +3 -0
  75. package/dist/executors/bounded-exec/ram-temp-fixtures.d.ts.map +1 -0
  76. package/dist/executors/bounded-exec/ram-temp-fixtures.js +12 -0
  77. package/dist/executors/bounded-exec/ram-temp-held.d.ts +91 -0
  78. package/dist/executors/bounded-exec/ram-temp-held.d.ts.map +1 -0
  79. package/dist/executors/bounded-exec/ram-temp-held.js +152 -0
  80. package/dist/executors/bounded-exec/ram-temp.d.ts +223 -0
  81. package/dist/executors/bounded-exec/ram-temp.d.ts.map +1 -0
  82. package/dist/executors/bounded-exec/ram-temp.js +1528 -0
  83. package/dist/executors/bounded-exec/schema.d.ts +29 -11
  84. package/dist/executors/bounded-exec/schema.d.ts.map +1 -1
  85. package/dist/executors/bounded-exec/verdict.d.ts +162 -0
  86. package/dist/executors/bounded-exec/verdict.d.ts.map +1 -0
  87. package/dist/executors/bounded-exec/verdict.js +318 -0
  88. package/dist/executors/shard-timings/executor.d.ts +14 -0
  89. package/dist/executors/shard-timings/executor.d.ts.map +1 -0
  90. package/dist/executors/shard-timings/executor.js +25 -0
  91. package/dist/executors/shard-timings/schema.d.ts +4 -0
  92. package/dist/executors/shard-timings/schema.d.ts.map +1 -0
  93. package/dist/executors/shard-timings/schema.js +1 -0
  94. package/dist/failure-summary.d.ts +5 -0
  95. package/dist/failure-summary.d.ts.map +1 -0
  96. package/dist/failure-summary.js +118 -0
  97. package/dist/generators/create-package/generator.d.ts.map +1 -1
  98. package/dist/generators/create-package/generator.js +6 -2
  99. package/dist/index.d.ts +14 -2
  100. package/dist/index.d.ts.map +1 -1
  101. package/dist/index.js +435 -230
  102. package/dist/managed-assets.d.ts +6 -0
  103. package/dist/managed-assets.d.ts.map +1 -1
  104. package/dist/managed-files/ci-workflow.d.ts +13 -0
  105. package/dist/managed-files/ci-workflow.d.ts.map +1 -1
  106. package/dist/managed-files/ci-workflow.js +44 -13
  107. package/dist/managed-files/files.d.ts +7 -1
  108. package/dist/managed-files/files.d.ts.map +1 -1
  109. package/dist/managed-files/files.js +62 -8
  110. package/dist/managed-files/json-edit.d.ts +21 -0
  111. package/dist/managed-files/json-edit.d.ts.map +1 -0
  112. package/dist/managed-files/json-edit.js +295 -0
  113. package/dist/managed-files/managed-format.d.ts +33 -1
  114. package/dist/managed-files/managed-format.d.ts.map +1 -1
  115. package/dist/managed-files/managed-format.js +82 -2
  116. package/dist/managed-files/pr-preview-cleanup-workflow.d.ts +7 -0
  117. package/dist/managed-files/pr-preview-cleanup-workflow.d.ts.map +1 -1
  118. package/dist/managed-files/pr-preview-cleanup-workflow.js +68 -4
  119. package/dist/managed-files/publish-workflow.d.ts.map +1 -1
  120. package/dist/managed-files/publish-workflow.js +231 -48
  121. package/dist/managed-files/tree.d.ts +5 -0
  122. package/dist/managed-files/tree.d.ts.map +1 -1
  123. package/dist/managed-files/tree.js +5 -3
  124. package/dist/manifest-hash.js +3 -2
  125. package/dist/nextest-code-bound.d.ts +62 -0
  126. package/dist/nextest-code-bound.d.ts.map +1 -0
  127. package/dist/nextest-code-bound.js +154 -0
  128. package/dist/nextest-extraction.d.ts +24 -8
  129. package/dist/nextest-extraction.d.ts.map +1 -1
  130. package/dist/nextest-extraction.js +42 -7
  131. package/dist/package-target-policy.d.ts.map +1 -1
  132. package/dist/package-target-policy.js +3 -5
  133. package/dist/release-config-policy.d.ts.map +1 -1
  134. package/dist/release-config-policy.js +3 -5
  135. package/dist/runtime-input.d.ts +14 -0
  136. package/dist/runtime-input.d.ts.map +1 -0
  137. package/dist/runtime-input.js +15 -0
  138. package/dist/rust-tokens.d.ts +11 -0
  139. package/dist/rust-tokens.d.ts.map +1 -0
  140. package/dist/rust-tokens.js +81 -0
  141. package/dist/task-results.d.ts +6 -0
  142. package/dist/task-results.d.ts.map +1 -0
  143. package/dist/task-results.js +61 -0
  144. package/dist/test-program.d.ts +54 -0
  145. package/dist/test-program.d.ts.map +1 -0
  146. package/dist/test-program.js +292 -0
  147. package/dist/test-timings.d.ts +71 -0
  148. package/dist/test-timings.d.ts.map +1 -0
  149. package/dist/test-timings.js +393 -0
  150. package/dist/test-walk-policy.d.ts +103 -0
  151. package/dist/test-walk-policy.d.ts.map +1 -0
  152. package/dist/test-walk-policy.js +657 -0
  153. package/dist/testing.d.ts +177 -0
  154. package/dist/testing.d.ts.map +1 -0
  155. package/dist/testing.js +824 -0
  156. package/dist/type-guards.d.ts +3 -0
  157. package/dist/type-guards.d.ts.map +1 -0
  158. package/dist/type-guards.js +4 -0
  159. package/dist/typecheck-test-policy.d.ts +8 -5
  160. package/dist/typecheck-test-policy.d.ts.map +1 -1
  161. package/dist/typecheck-test-policy.js +113 -100
  162. package/dist/workspace-config-policy.d.ts +8 -0
  163. package/dist/workspace-config-policy.d.ts.map +1 -1
  164. package/dist/workspace-config-policy.js +175 -63
  165. package/dist/workspace-index.d.ts +12 -0
  166. package/dist/workspace-index.d.ts.map +1 -0
  167. package/dist/workspace-index.js +17 -0
  168. package/dist/workspace-manifest.d.ts +37 -1
  169. package/dist/workspace-manifest.d.ts.map +1 -1
  170. package/executors.json +5 -0
  171. package/loader.js +13 -11
  172. package/managed/raw/envrc +33 -62
  173. package/managed/raw/git-format-staged.yml +30 -35
  174. package/managed/raw/patches/nx@23.2.1.patch +3484 -0
  175. package/managed/raw/tooling/direnv/apple-developer.sh +83 -0
  176. package/managed/raw/tooling/direnv/developer-links.ts +607 -17
  177. package/managed/raw/tooling/direnv/devenv.smoo.nix +314 -92
  178. package/managed/raw/tooling/direnv/github-actions-bootstrap.sh +20 -11
  179. package/managed/raw/tooling/direnv/linked-snapshots.ts +845 -0
  180. package/managed/raw/tooling/direnv/nx-socket-dir.sh +82 -0
  181. package/managed/raw/tooling/direnv/repo-path +14 -2
  182. package/managed/raw/tooling/direnv/setup-environment.ts +1009 -118
  183. package/managed/raw/tooling/direnv/toolchain-stamp.ts +45 -3
  184. package/managed/raw/tooling/direnv/ttsc-env.sh +39 -0
  185. package/managed/raw/tooling/git-hooks/commit-msg.sh +5 -6
  186. package/managed/raw/tooling/git-hooks/pre-commit.sh +7 -6
  187. package/managed/raw/tooling/git-hooks/pre-push.sh +20 -15
  188. package/managed/templates/github/actions/cache-nix-devenv/action.yml +6 -6
  189. package/managed/templates/github/actions/cache-node-modules/action.yml +6 -4
  190. package/managed/templates/github/actions/cache-nx/action.yml +1 -1
  191. package/managed/templates/github/actions/cache-ttsc-plugins/action.yml +1 -1
  192. package/managed/templates/github/actions/forgejo-artifact.cjs +99 -0
  193. package/managed/templates/github/actions/forgejo-download-artifact/action.yml +49 -0
  194. package/managed/templates/github/actions/forgejo-download-artifact/index.cjs +7 -0
  195. package/managed/templates/github/actions/forgejo-upload-artifact/action.yml +65 -0
  196. package/managed/templates/github/actions/forgejo-upload-artifact/index.cjs +7 -0
  197. package/managed/templates/github/actions/setup-devenv/action.yml +21 -3
  198. package/managed/templates/github/workflows/managed-files.yml +1 -2
  199. package/nextest.toml +8 -75
  200. package/package.json +75 -13
  201. package/runtime-input.sh +81 -0
  202. package/src/__tests__/counted-cargo-process.ts +286 -0
  203. package/src/__tests__/counted-cargo.ts +730 -0
  204. package/src/__tests__/fixture-cargo-home.ts +36 -0
  205. package/src/__tests__/fixture-nx-env.ts +157 -2
  206. package/src/__tests__/progress-clock.ts +43 -0
  207. package/src/__tests__/runtime-input.ts +23 -0
  208. package/src/bin/smoo-nx-bound-failures.test.ts +127 -0
  209. package/src/bin/smoo-nx-bound-failures.ts +33 -0
  210. package/src/bin/smoo-nx-cargo-hash.ts +28 -3
  211. package/src/bin/smoo-nx-exec.ts +17 -86
  212. package/src/bin/smoo-nx-failures.ts +4 -0
  213. package/src/bin/smoo-nx-nextest-extract.ts +3 -2
  214. package/src/bin/smoo-nx-test-walks.ts +42 -0
  215. package/src/bounded-test-policy.test.ts +268 -22
  216. package/src/bounded-test-policy.ts +114 -42
  217. package/src/build-state-output-policy.test.ts +115 -0
  218. package/src/build-state-output-policy.ts +90 -0
  219. package/src/bun-bootstrap.test.ts +57 -3
  220. package/src/bun-file-packages.test.ts +47 -0
  221. package/src/cargo-closure-input.test.ts +432 -62
  222. package/src/cargo-closure-input.ts +51 -57
  223. package/src/cargo-dev-profile.ts +137 -0
  224. package/src/cargo-resolve-cache.test.ts +1044 -0
  225. package/src/cargo-source-hash.test.ts +203 -93
  226. package/src/cargo-source-hash.ts +762 -75
  227. package/src/cargo-toolchain-policy.ts +6 -0
  228. package/src/cargo-workspace.test.ts +320 -0
  229. package/src/cargo-workspace.ts +344 -18
  230. package/src/cross-check-policy.test.ts +59 -1
  231. package/src/cross-check-policy.ts +25 -6
  232. package/src/disk-command-policy.test.ts +61 -0
  233. package/src/disk-command-policy.ts +107 -0
  234. package/src/ensure-built-receipt.test.ts +184 -0
  235. package/src/ensure-built-receipt.ts +263 -0
  236. package/src/ensure-built-witness.test.ts +344 -0
  237. package/src/ensure-built-witness.ts +252 -0
  238. package/src/ensure-built.test.ts +1043 -483
  239. package/src/ensure-built.ts +870 -197
  240. package/src/eslint-file-set.ts +9 -0
  241. package/src/executors/bounded-exec/cpu-pool.test.ts +454 -0
  242. package/src/executors/bounded-exec/cpu-pool.ts +417 -0
  243. package/src/executors/bounded-exec/cpu-tokens.test.ts +269 -0
  244. package/src/executors/bounded-exec/cpu-tokens.ts +177 -0
  245. package/src/executors/bounded-exec/disk-lease.test.ts +185 -0
  246. package/src/executors/bounded-exec/disk-lease.ts +139 -0
  247. package/src/executors/bounded-exec/executor-host.test.ts +778 -0
  248. package/src/executors/bounded-exec/executor.test.ts +1116 -63
  249. package/src/executors/bounded-exec/executor.ts +565 -67
  250. package/src/executors/bounded-exec/gateway-lease.ts +176 -0
  251. package/src/executors/bounded-exec/hang-diagnostics.test.ts +285 -0
  252. package/src/executors/bounded-exec/hang-diagnostics.ts +283 -0
  253. package/src/executors/bounded-exec/progress.test.ts +79 -0
  254. package/src/executors/bounded-exec/progress.ts +118 -0
  255. package/src/executors/bounded-exec/ram-temp-fixtures.ts +12 -0
  256. package/src/executors/bounded-exec/ram-temp-held.test.ts +264 -0
  257. package/src/executors/bounded-exec/ram-temp-held.ts +242 -0
  258. package/src/executors/bounded-exec/ram-temp-host.test.ts +605 -0
  259. package/src/executors/bounded-exec/ram-temp.test.ts +674 -0
  260. package/src/executors/bounded-exec/ram-temp.ts +1918 -0
  261. package/src/executors/bounded-exec/schema.json +11 -5
  262. package/src/executors/bounded-exec/schema.ts +27 -11
  263. package/src/executors/bounded-exec/verdict.test.ts +440 -0
  264. package/src/executors/bounded-exec/verdict.ts +433 -0
  265. package/src/executors/shard-timings/executor.ts +36 -0
  266. package/src/executors/shard-timings/schema.json +21 -0
  267. package/src/executors/shard-timings/schema.ts +4 -0
  268. package/src/failure-summary.ts +119 -0
  269. package/src/generators/bounded-test-targets/generator.test.ts +4 -4
  270. package/src/generators/bun-test-tracing/generator.test.ts +2 -2
  271. package/src/generators/create-package/generator.test.ts +6 -0
  272. package/src/generators/create-package/generator.ts +6 -2
  273. package/src/host-controller-wiring.test.ts +31 -0
  274. package/src/index.test.ts +1070 -360
  275. package/src/index.ts +524 -255
  276. package/src/linked-snapshots.nx.test.ts +172 -0
  277. package/src/managed-assets.ts +12 -0
  278. package/src/managed-files/ci-workflow.ts +54 -13
  279. package/src/managed-files/files.ts +65 -9
  280. package/src/managed-files/forgejo-artifact.test.ts +395 -0
  281. package/src/managed-files/generator.test.ts +60 -1
  282. package/src/managed-files/json-edit.test.ts +216 -0
  283. package/src/managed-files/json-edit.ts +290 -0
  284. package/src/managed-files/managed-format-json.test.ts +308 -0
  285. package/src/managed-files/managed-format.ts +90 -2
  286. package/src/managed-files/pr-preview-cleanup-workflow.ts +69 -3
  287. package/src/managed-files/publish-workflow.ts +285 -67
  288. package/src/managed-files/repo-path.test.ts +129 -0
  289. package/src/managed-files/trace-db-artifact.test.ts +7 -0
  290. package/src/managed-files/tree.ts +11 -3
  291. package/src/manifest-hash.ts +3 -2
  292. package/src/nextest-code-bound.test.ts +155 -0
  293. package/src/nextest-code-bound.ts +195 -0
  294. package/src/nextest-extraction.test.ts +37 -5
  295. package/src/nextest-extraction.ts +51 -9
  296. package/src/nextest-partitions.test.ts +495 -0
  297. package/src/nx-cache-bound.test.ts +109 -0
  298. package/src/nx-cache-restore-mtime.test.ts +170 -0
  299. package/src/nx-daemon-claim.test.ts +277 -0
  300. package/src/nx-daemon-idle-cpu.test.ts +81 -0
  301. package/src/nx-daemon-idle-policy.test.ts +51 -0
  302. package/src/nx-daemon-ignore-files.test.ts +210 -0
  303. package/src/nx-daemon-root-normalization.test.ts +105 -0
  304. package/src/nx-daemon-validity.test.ts +925 -0
  305. package/src/nx-dot-input-policy.test.ts +353 -0
  306. package/src/nx-failure-life-cycles.test.ts +207 -0
  307. package/src/nx-files-archive-racy.test.ts +97 -0
  308. package/src/nx-foreign-workspace-env.test.ts +329 -0
  309. package/src/nx-incomplete-run-summary.test.ts +70 -0
  310. package/src/nx-plugin-load-errors.test.ts +120 -0
  311. package/src/nx-project-resolution-cache.test.ts +176 -0
  312. package/src/nx-raw-inputs.test.ts +166 -0
  313. package/src/nx-resident-graph-workers.test.ts +112 -0
  314. package/src/nx-run-information.test.ts +118 -0
  315. package/src/nx-skipped-task-exit.test.ts +180 -0
  316. package/src/nx-state-location.test.ts +63 -0
  317. package/src/nx-store-resolution.test.ts +99 -0
  318. package/src/nx-task-graph-cycles.test.ts +148 -0
  319. package/src/nx-task-output-lease.test.ts +414 -0
  320. package/src/nx-use-local.test.ts +103 -0
  321. package/src/package-target-policy.ts +3 -6
  322. package/src/release-config-policy.ts +3 -6
  323. package/src/runtime-input.test.ts +215 -0
  324. package/src/runtime-input.ts +17 -0
  325. package/src/rust-tokens.ts +78 -0
  326. package/src/serial-group-wiring.test.ts +326 -0
  327. package/src/task-history-namespace.test.ts +181 -0
  328. package/src/task-results.test.ts +226 -0
  329. package/src/task-results.ts +75 -0
  330. package/src/test-program.test.ts +303 -0
  331. package/src/test-program.ts +341 -0
  332. package/src/test-timings.test.ts +240 -0
  333. package/src/test-timings.ts +311 -0
  334. package/src/test-walk-policy.test.ts +585 -0
  335. package/src/test-walk-policy.ts +731 -0
  336. package/src/testing.test.ts +509 -0
  337. package/src/testing.ts +922 -0
  338. package/src/type-guards.ts +4 -0
  339. package/src/typecheck-test-policy.test.ts +496 -11
  340. package/src/typecheck-test-policy.ts +137 -103
  341. package/src/workspace-config-policy.test.ts +212 -13
  342. package/src/workspace-config-policy.ts +101 -73
  343. package/src/workspace-index.ts +18 -0
  344. package/src/workspace-manifest.ts +41 -1
  345. package/test-timings.json +85 -28
  346. package/managed/raw/tooling/direnv/inherited-devenv.ts +0 -615
  347. package/managed/raw/tooling/direnv/shared-caches.sh +0 -53
  348. package/src/__tests__/in-process-nx-index.ts +0 -9
@@ -1,18 +1,82 @@
1
- import { spawn } from 'node:child_process';
2
- import { isAbsolute, join } from 'node:path';
1
+ import { type ChildProcess, spawn } from 'node:child_process';
2
+ import { availableParallelism, userInfo } from 'node:os';
3
+ import { isAbsolute, join, resolve as resolvePath } from 'node:path';
4
+ import { workspaceDataDirectoryForWorkspace } from 'nx/src/utils/cache-directory.js';
3
5
 
6
+ import {
7
+ CPU_POOL_ENV,
8
+ type CpuPool,
9
+ createCpuPool,
10
+ type InheritedCpuPool,
11
+ readInheritedCpuPool,
12
+ takeParentCpuTokens,
13
+ } from './cpu-pool.js';
14
+ import {
15
+ CPU_TOKENS_ENV,
16
+ type CpuBudget,
17
+ gatewayCpuBudget,
18
+ runnerDemand,
19
+ sizeRunner,
20
+ type TokenGrant,
21
+ } from './cpu-tokens.js';
22
+ import type { HangDiagnostics } from './hang-diagnostics.js';
23
+ import { captureHangDiagnostics, HANG_CAPTURE_MS } from './hang-diagnostics.js';
24
+ import { createRunProgress, withProgressReport } from './progress.js';
25
+ import {
26
+ COWSHED_EXECUTABLE_ENV,
27
+ createHostCommands,
28
+ describeRamTempError,
29
+ RAM_TEMP_CAPACITY_BYTES,
30
+ type RamTempLease,
31
+ RamTempVolume,
32
+ ramTempPaths,
33
+ } from './ram-temp.js';
4
34
  import type { BoundedExecOptions } from './schema.js';
35
+ import {
36
+ type BoundedExecRecord,
37
+ type BoundedExecTask,
38
+ clearTaskDirectory,
39
+ type ExpiredBound,
40
+ JUNIT_FILE,
41
+ JUNIT_REPORT_ENV,
42
+ judgeOperationalFailure,
43
+ judgeRun,
44
+ readJunitReport,
45
+ taskDirectory,
46
+ withJunitReport,
47
+ writeRecord,
48
+ } from './verdict.js';
5
49
 
6
50
  const DEFAULT_KILL_AFTER_MS = 10_000;
7
51
  const REAP_AFTER_FORCE_KILL_MS = 2_000;
52
+ type IdleDiagnosticResult =
53
+ | { kind: 'pending'; startedAtMs: number | null }
54
+ | { kind: 'captured'; snapshot: HangDiagnostics }
55
+ | { kind: 'failed'; detail: string };
8
56
  const EXIT_CODE_BY_SIGNAL: Partial<Record<NodeJS.Signals, number>> = {
9
57
  SIGHUP: 129,
10
58
  SIGINT: 130,
11
59
  SIGTERM: 143,
12
60
  };
13
61
 
62
+ /** The variables the CPU budget sets for the command it grants: no target's environment may set them. */
63
+ const BUDGET_ENV = [CPU_TOKENS_ENV, CPU_POOL_ENV];
64
+
14
65
  export interface BoundedExecContext {
15
66
  root: string;
67
+ /** The Nx task this run executes as, and the directory its verdict and report go to. */
68
+ record?: { task: BoundedExecTask; directory: string };
69
+ /** Owns child stdout when present; otherwise it is retained and forwarded normally. */
70
+ stdout?: (chunk: string) => void;
71
+ }
72
+
73
+ /** The part of Nx's executor context that names the running task. */
74
+ export interface NxTaskContext {
75
+ root: string;
76
+ projectName?: string;
77
+ targetName?: string;
78
+ configurationName?: string;
79
+ taskGraph?: { tasks: Record<string, { hash?: string }> };
16
80
  }
17
81
 
18
82
  export interface BoundedExecResult {
@@ -20,62 +84,127 @@ export interface BoundedExecResult {
20
84
  terminalOutput: string;
21
85
  }
22
86
 
23
- /**
24
- * Which bound ended the run.
25
- *
26
- * `total` is a wall-clock ceiling: it answers "is this run unbounded?".
27
- * `idle` is a no-progress bound: it answers "is this run wedged?".
28
- *
29
- * These are different questions and one number cannot answer both. A ceiling
30
- * tight enough to catch a wedge promptly is also tight enough to fail a correct
31
- * run on a busy machine; a ceiling loose enough never to do that cannot catch a
32
- * wedge promptly. Keeping the two bounds separate is what makes each honest.
33
- */
34
- type ExpiredBound = { kind: 'total' | 'idle'; limitMs: number };
35
-
36
87
  interface RunState {
37
88
  settled: boolean;
38
89
  expiry: ExpiredBound | null;
39
90
  forceKillNeeded: boolean;
91
+ /** Set when a borrowed grant closed while the command ran, and the command was stopped as a lost grant. */
92
+ grantLost: boolean;
93
+ /** The sub-pool's close error when it did not close cleanly: the run fails. */
94
+ poolCloseError: string | null;
40
95
  }
41
96
 
97
+ type GrantedTokens = Extract<TokenGrant, { granted: true }>;
98
+
42
99
  export interface ProcessTreeKiller {
43
100
  kill(pid: number, signal: NodeJS.Signals): Promise<void>;
44
101
  }
45
102
 
103
+ /** Where the command's TMPDIR comes from; `null` leaves the inherited one. */
104
+ export type TempVolume = Pick<RamTempVolume, 'acquire' | 'release' | 'fullness' | 'reap'>;
105
+
46
106
  export default function boundedExecExecutor(
107
+ options: BoundedExecOptions,
108
+ context: NxTaskContext,
109
+ ): Promise<BoundedExecResult> {
110
+ return runDefaultBoundedExec(options, recordContext(context));
111
+ }
112
+
113
+ /** Runs with the normal process, RAM lease and CPU-budget owners, with an optional stdout sink. */
114
+ export function runDefaultBoundedExec(
47
115
  options: BoundedExecOptions,
48
116
  context: BoundedExecContext,
49
117
  ): Promise<BoundedExecResult> {
50
- return runBoundedExec(options, context, createProcessTreeKiller());
118
+ // A target that names the cowshed its disk commands run through names it for this run's own RAM volume work and
119
+ // for its command alike. Nx resolves `{workspaceRoot}/…` in an option to a workspace-relative path, which the
120
+ // command, starting in another directory, could not find: it gets the absolute one.
121
+ const named = options.env?.[COWSHED_EXECUTABLE_ENV];
122
+ const cowshed = named === undefined ? undefined : resolvePath(context.root, named);
123
+ const run: BoundedExecOptions =
124
+ cowshed === undefined ? options : { ...options, env: { ...options.env, [COWSHED_EXECUTABLE_ENV]: cowshed } };
125
+ // Only macOS has `hdiutil ram://`; elsewhere the command keeps the inherited TMPDIR.
126
+ const tempVolume =
127
+ process.platform === 'darwin'
128
+ ? new RamTempVolume(
129
+ ramTempPaths(userInfo()),
130
+ RAM_TEMP_CAPACITY_BYTES,
131
+ createHostCommands(undefined, undefined, undefined, cowshed),
132
+ )
133
+ : null;
134
+ // A command a parent's CPU pool started draws its tokens from that pool. The pool is named by the environment the
135
+ // parent set for this command; a run no parent started has none.
136
+ const inherited = readInheritedCpuPool(process.env);
137
+ return runBoundedExec(run, context, createProcessTreeKiller(), tempVolume, gatewayCpuBudget(), inherited);
51
138
  }
52
139
 
140
+ /**
141
+ * Nx gives an executor the task's project, target and configuration, not its id; the id is the
142
+ * key of the task graph entry they name. Outside a task graph the run records nothing.
143
+ */
144
+ export function recordContext(context: NxTaskContext): BoundedExecContext {
145
+ const { root, projectName, targetName, configurationName, taskGraph } = context;
146
+ if (!projectName || !targetName) {
147
+ return { root };
148
+ }
149
+ const id = `${projectName}:${targetName}${configurationName ? `:${configurationName}` : ''}`;
150
+ const task = taskGraph?.tasks[id];
151
+ if (task === undefined) {
152
+ return { root };
153
+ }
154
+ return {
155
+ root,
156
+ record: {
157
+ task: { id, hash: task.hash ?? null },
158
+ directory: taskDirectory(workspaceDataDirectoryForWorkspace(root), id),
159
+ },
160
+ };
161
+ }
162
+
163
+ /**
164
+ * Run the command once its runner holds CPU tokens from `budget` (`cpu-tokens.ts`), sized to the
165
+ * grant; `null`, or a budget nobody serves, runs it as asked. Both bounds start once it is
166
+ * granted: a wait for the host is not the command's time.
167
+ *
168
+ * `inherited` is the parent's CPU pool (`cpu-pool.ts`) that lends this run its tokens. The run asks that pool first
169
+ * and the host budget only when the pool refuses. Whatever it is granted, the run serves its own sub-pool to its
170
+ * descendants, closed before the grant returns. A borrowed grant that closes while the command runs stops the command
171
+ * as a lost grant, which is a failure.
172
+ */
53
173
  export async function runBoundedExec(
54
174
  options: BoundedExecOptions,
55
175
  context: BoundedExecContext,
56
176
  killer: ProcessTreeKiller,
177
+ tempVolume: TempVolume | null,
178
+ budget: CpuBudget | null,
179
+ inherited: InheritedCpuPool | null = null,
180
+ captureDiagnostics: typeof captureHangDiagnostics = captureHangDiagnostics,
57
181
  ): Promise<BoundedExecResult> {
58
182
  const cwd = resolveCwd(options.cwd, context.root);
59
- const command = buildCommand(options);
183
+ const directory = context.record?.directory ?? null;
184
+ // A previous run's verdict and report go first, so neither can be read as this run's.
185
+ if (directory !== null) {
186
+ await clearTaskDirectory(directory);
187
+ }
188
+ const requested = buildCommand(options);
60
189
  const timeoutMs = options.timeoutMs;
61
190
  const idleTimeoutMs = options.idleTimeoutMs;
62
191
  const killAfterMs = options.killAfterMs ?? DEFAULT_KILL_AFTER_MS;
63
- const startedAt = Date.now();
64
192
  const outputChunks: string[] = [];
65
- const state: RunState = { settled: false, expiry: null, forceKillNeeded: false };
66
-
67
- const child = spawn(command, [], {
68
- cwd,
69
- env: mergeEnv(options.env),
70
- shell: true,
71
- detached: process.platform !== 'win32',
72
- windowsHide: true,
73
- });
74
-
193
+ const state: RunState = {
194
+ settled: false,
195
+ expiry: null,
196
+ forceKillNeeded: false,
197
+ grantLost: false,
198
+ poolCloseError: null,
199
+ };
75
200
  const appendStdout = (chunk: Buffer | string): void => {
76
201
  const text = chunk.toString();
77
- outputChunks.push(text);
78
- process.stdout.write(text);
202
+ if (context.stdout !== undefined) {
203
+ context.stdout(text);
204
+ } else {
205
+ outputChunks.push(text);
206
+ process.stdout.write(text);
207
+ }
79
208
  };
80
209
  const appendStderr = (chunk: Buffer | string): void => {
81
210
  const text = chunk.toString();
@@ -83,25 +212,259 @@ export async function runBoundedExec(
83
212
  process.stderr.write(text);
84
213
  };
85
214
 
215
+ let lease: RamTempLease | null = null;
216
+ if (tempVolume !== null) {
217
+ const acquired = await tempVolume.acquire();
218
+ if (!acquired.ok) {
219
+ const message = `${describeRamTempError(acquired.error)}\n`;
220
+ process.stderr.write(message);
221
+ return { success: false, terminalOutput: message };
222
+ }
223
+ if (acquired.value.kind === 'leased') {
224
+ lease = acquired.value.lease;
225
+ for (const held of acquired.value.held) {
226
+ appendStderr(`RAM temp volume: dead lease ${held}\n`);
227
+ }
228
+ for (const reaped of acquired.value.reaped) {
229
+ appendStderr(`RAM temp volume: stopped ${reaped}, which a dead task left working in its lease\n`);
230
+ }
231
+ } else {
232
+ appendStderr(
233
+ `RAM temp volume unavailable in this sandbox (${acquired.value.detail}); TMPDIR stays ${process.env.TMPDIR ?? '(unset)'}\n`,
234
+ );
235
+ }
236
+ }
237
+ const releaseLease = async (): Promise<void> => {
238
+ if (lease === null) {
239
+ return;
240
+ }
241
+ const held = lease;
242
+ lease = null;
243
+ // What the command left working in its lease is a leak, whichever way the command ended; it
244
+ // is stopped before the lease goes, so the volume can detach, and said, so its owner fixes it.
245
+ const reaped = await tempVolume?.reap(held);
246
+ if (reaped?.ok) {
247
+ for (const leftover of reaped.value) {
248
+ appendStderr(`RAM temp volume: stopped ${leftover}, which the command left working in its lease\n`);
249
+ }
250
+ } else if (reaped) {
251
+ appendStderr(`${describeRamTempError(reaped.error)}\n`);
252
+ }
253
+ const released = await tempVolume?.release(held);
254
+ if (released && !released.ok) {
255
+ appendStderr(`${describeRamTempError(released.error)}\n`);
256
+ }
257
+ };
258
+
259
+ // The wait is one span line, as cowshed's own spans read. With no gateway listening, as on any
260
+ // host without cowshed, the command runs exactly as asked and nothing is said.
261
+ const cores = availableParallelism();
262
+ const demand = runnerDemand(requested, options.parallelism, cores, process.env, options.env);
263
+ const asked = Date.now();
264
+ // A run a parent's CPU pool lent tokens to takes them there. Only an absent pool leaves the host budget to ask.
265
+ // A pool that answered and refused still holds tokens of its own: asking the host then would hold them while waiting
266
+ // on another budget, so the run fails instead.
267
+ let grant: GrantedTokens | null = null;
268
+ let parentRefusal: string | null = null;
269
+ if (inherited !== null) {
270
+ const fromParent = await takeParentCpuTokens(inherited, demand.want, cores);
271
+ if (fromParent.granted) {
272
+ grant = fromParent;
273
+ } else if (fromParent.cause === 'absent') {
274
+ appendStderr(`cowshed: cpu-tokens parent pool refused: ${fromParent.reason}; asking the host budget\n`);
275
+ } else {
276
+ parentRefusal = fromParent.reason;
277
+ }
278
+ }
279
+ let unbudgeted: string | null = null;
280
+ if (grant === null && parentRefusal === null && budget !== null) {
281
+ const fromHost = await budget.take(demand.want, context.root, requested);
282
+ if (fromHost.granted) {
283
+ grant = fromHost;
284
+ } else if (fromHost.cause !== 'absent') {
285
+ unbudgeted = fromHost.reason;
286
+ }
287
+ }
288
+ const waitedMs = Date.now() - asked;
289
+ let cpu: BoundedExecRecord['cpu'];
290
+ let sized: { command: string; env: Record<string, string> } = { command: requested, env: {} };
291
+ if (grant !== null) {
292
+ sized = sizeRunner(demand, requested, grant.tokens);
293
+ cpu = { tokens: grant.tokens, want: demand.want, waitedMs, ...(grant.inherited ? { inherited: true } : {}) };
294
+ appendStderr(
295
+ `cowshed: cpu-tokens wait done elapsed=${waitedMs}ms status=ok ${grant.inherited ? 'inherited=true ' : ''}tokens=${grant.tokens}/${demand.want} runner=${demand.kind}\n`,
296
+ );
297
+ } else if (unbudgeted !== null) {
298
+ appendStderr(
299
+ `cowshed: cpu-tokens wait done elapsed=${waitedMs}ms status=err runner=${demand.kind} runs unbudgeted: ${unbudgeted}\n`,
300
+ );
301
+ }
302
+ // The run's sub-pool and grant are given back exactly once, however the run ends. The sub-pool closes first, so no
303
+ // descendant still draws on tokens the grant has returned.
304
+ let pool: CpuPool | null = null;
305
+ let cpuReleased: Promise<void> | null = null;
306
+ const releaseCpu = (): Promise<void> => {
307
+ cpuReleased ??= (async () => {
308
+ try {
309
+ const outcome = await pool?.close();
310
+ if (outcome !== undefined && !outcome.ok) {
311
+ state.poolCloseError = outcome.error;
312
+ appendStderr(`cowshed: cpu pool did not close cleanly: ${outcome.error}\n`);
313
+ }
314
+ } finally {
315
+ grant?.release();
316
+ }
317
+ })();
318
+ return cpuReleased;
319
+ };
320
+ // Records the run's verdict when it has a task to record as. An `expiry` is set only when a bound ended the run.
321
+ // An `operational` run failed on its CPU budget (a lost grant, or a sub-pool that did not close): an ordinary
322
+ // failure, whatever bound also fired, so it is never judged as a bound.
323
+ const recordRun = async (
324
+ exitCode: number,
325
+ expiry: ExpiredBound | null,
326
+ elapsedMs: number,
327
+ operational: boolean,
328
+ ): Promise<void> => {
329
+ if (context.record === undefined) {
330
+ return;
331
+ }
332
+ const { task, directory: recordDirectory } = context.record;
333
+ const report = await readJunitReport(recordDirectory);
334
+ const verdict = operational
335
+ ? judgeOperationalFailure(exitCode, report)
336
+ : judgeRun({ exitCode, elapsedMs, expiry, report });
337
+ if (verdict.outcome === 'bound' && verdict.bound === 'test') {
338
+ appendStderr(`Every failing test failed only on its per-test timeout: ${verdict.tests.join(', ')}\n`);
339
+ }
340
+ await writeRecord(recordDirectory, { task: task.id, hash: task.hash, verdict, ...(cpu ? { cpu } : {}) });
341
+ };
342
+ // A run refused before its command starts says why, gives back its grant and RAM, and records the failure.
343
+ const refuse = async (message: string): Promise<BoundedExecResult> => {
344
+ appendStderr(message);
345
+ await releaseCpu();
346
+ await releaseLease();
347
+ await recordRun(1, null, Date.now() - asked, false);
348
+ return { success: false, terminalOutput: outputChunks.join('') };
349
+ };
350
+ if (parentRefusal !== null) {
351
+ return refuse(
352
+ `cowshed: cpu-tokens parent pool refused: ${parentRefusal}; refusing to run the command without asking the host budget\n`,
353
+ );
354
+ }
355
+
356
+ // A granted run serves a sub-pool of its own grant, so its descendants draw from that and never from above it.
357
+ // A pool that cannot be set up refuses the run: no command runs unbudgeted.
358
+ if (grant !== null) {
359
+ const created = await createCpuPool(grant.tokens, grant.closed);
360
+ if (!created.ok) {
361
+ return refuse(`cowshed: cpu pool unavailable: ${created.error}; refusing to run the command\n`);
362
+ }
363
+ pool = created.pool;
364
+ }
365
+ const command =
366
+ directory === null
367
+ ? withProgressReport(sized.command)
368
+ : await withJunitReport(withProgressReport(sized.command), directory);
369
+ // A runner bounded-exec cannot see in the command (a script that spawns `bun test`) writes its JUnit report to this
370
+ // path when it is set. The grant's sizing outranks the target's own environment: the runner may use what it was
371
+ // granted, not what it was configured with. The sub-pool's environment comes last: a descendant reads only the pool
372
+ // this run serves.
373
+ const env: Record<string, string> = {
374
+ ...(directory === null ? {} : { [JUNIT_REPORT_ENV]: join(directory, JUNIT_FILE) }),
375
+ ...targetEnv(options.env, appendStderr),
376
+ ...sized.env,
377
+ ...(pool?.environment ?? {}),
378
+ };
379
+ const startedAt = Date.now();
380
+ let child: ChildProcess;
381
+ try {
382
+ child = spawn(command, [], {
383
+ cwd,
384
+ env: mergeEnv(lease, env),
385
+ shell: true,
386
+ detached: process.platform !== 'win32',
387
+ windowsHide: true,
388
+ });
389
+ } catch (error) {
390
+ // Nothing started, so nothing runs under the grant: it goes back here, with the RAM lease.
391
+ return refuse(`cowshed: could not start the command: ${error instanceof Error ? error.message : String(error)}\n`);
392
+ }
393
+
394
+ // Progress is a finished test, not output: a runner's SLOW warnings and a hung test's heartbeat keep printing.
395
+ const progress = createRunProgress();
86
396
  let idleTimer: NodeJS.Timeout | undefined;
397
+ let captureTimer: NodeJS.Timeout | undefined;
398
+ let idleCapture: { expiresAtMs: number; controller: AbortController; result: IdleDiagnosticResult } | undefined;
399
+ const cancelCapture = (): void => {
400
+ clearTimeout(captureTimer);
401
+ idleCapture?.controller.abort();
402
+ };
87
403
  const armIdleTimer = (): void => {
88
404
  if (idleTimeoutMs === undefined || state.settled || state.expiry !== null) {
89
405
  return;
90
406
  }
91
407
  clearTimeout(idleTimer);
92
- idleTimer = setTimeout(() => expire({ kind: 'idle', limitMs: idleTimeoutMs }), idleTimeoutMs);
408
+ cancelCapture();
409
+ const capture: NonNullable<typeof idleCapture> = {
410
+ expiresAtMs: Date.now() + idleTimeoutMs,
411
+ controller: new AbortController(),
412
+ result: { kind: 'pending', startedAtMs: null },
413
+ };
414
+ idleCapture = capture;
415
+ captureTimer = setTimeout(
416
+ () => {
417
+ if (state.settled || state.expiry !== null || idleCapture !== capture) return;
418
+ const pid = child.pid;
419
+ if (pid === undefined) {
420
+ idleCapture.result = { kind: 'failed', detail: 'runner PID unavailable; no holder conclusion' };
421
+ return;
422
+ }
423
+ idleCapture.result = { kind: 'pending', startedAtMs: Date.now() };
424
+ void Promise.resolve()
425
+ .then(() =>
426
+ captureDiagnostics({
427
+ pid,
428
+ pgid: process.platform === 'win32' ? null : pid,
429
+ expiresAtMs: capture.expiresAtMs,
430
+ deadlineMs: Math.max(0, Math.min(HANG_CAPTURE_MS, capture.expiresAtMs - Date.now())),
431
+ signal: capture.controller.signal,
432
+ }),
433
+ )
434
+ .then(
435
+ (snapshot) => {
436
+ if (idleCapture === capture) idleCapture.result = { kind: 'captured', snapshot };
437
+ },
438
+ (error: unknown) => {
439
+ if (idleCapture === capture) {
440
+ idleCapture.result = {
441
+ kind: 'failed',
442
+ detail: error instanceof Error ? error.message : String(error),
443
+ };
444
+ }
445
+ },
446
+ );
447
+ },
448
+ Math.max(0, idleTimeoutMs - HANG_CAPTURE_MS),
449
+ );
450
+ idleTimer = setTimeout(
451
+ () => expire({ kind: 'idle', limitMs: idleTimeoutMs, tests: progress.inFlight() }),
452
+ idleTimeoutMs,
453
+ );
93
454
  };
94
455
 
95
- // Only the CHILD's own output counts as progress. The diagnostics below go
456
+ // Only the CHILD's own output is read for progress. The diagnostics below go
96
457
  // through appendStderr as well, and re-arming on those would let a teardown
97
458
  // message extend the very bound that just fired.
98
459
  child.stdout?.on('data', (chunk: Buffer | string) => {
99
- armIdleTimer();
100
- appendStdout(chunk);
460
+ const text = chunk.toString();
461
+ if (progress.scan('stdout', text)) armIdleTimer();
462
+ appendStdout(text);
101
463
  });
102
464
  child.stderr?.on('data', (chunk: Buffer | string) => {
103
- armIdleTimer();
104
- appendStderr(chunk);
465
+ const text = chunk.toString();
466
+ if (progress.scan('stderr', text)) armIdleTimer();
467
+ appendStderr(text);
105
468
  });
106
469
 
107
470
  const killChildTree = async (force: boolean): Promise<void> => {
@@ -117,7 +480,11 @@ export async function runBoundedExec(
117
480
  };
118
481
  const onTerminationSignal = (signal: NodeJS.Signals): void => {
119
482
  removeSignalHandlers();
120
- void killChildTree(false).finally(() => process.kill(process.pid, signal));
483
+ cancelCapture();
484
+ void killChildTree(false)
485
+ .finally(releaseCpu)
486
+ .finally(releaseLease)
487
+ .finally(() => process.kill(process.pid, signal));
121
488
  };
122
489
  const onSigint = (): void => onTerminationSignal('SIGINT');
123
490
  const onSigterm = (): void => onTerminationSignal('SIGTERM');
@@ -154,8 +521,9 @@ export async function runBoundedExec(
154
521
  // the report always names which bound did it. A bare "timed out" would leave
155
522
  // the reader unable to tell a wedged toolchain from a loaded machine — the
156
523
  // two have opposite fixes.
524
+ let termination: Promise<void> = Promise.resolve();
157
525
  const expire = (bound: ExpiredBound): void => {
158
- if (state.settled || state.expiry !== null) {
526
+ if (state.settled || state.expiry !== null || state.grantLost) {
159
527
  return;
160
528
  }
161
529
  state.expiry = bound;
@@ -164,33 +532,105 @@ export async function runBoundedExec(
164
532
  const elapsedMs = Date.now() - startedAt;
165
533
  appendStderr(
166
534
  bound.kind === 'idle'
167
- ? `\nCommand made no progress: no output for ${bound.limitMs}ms (idleTimeoutMs=${bound.limitMs}) after ${elapsedMs}ms of runtime (cwd=${cwd}): ${command}\n`
535
+ ? `\nCommand made no progress: no test finished for ${bound.limitMs}ms (idleTimeoutMs=${bound.limitMs}) after ${elapsedMs}ms of runtime; ${
536
+ bound.tests.length > 0
537
+ ? `still running: ${bound.tests.join(', ')}`
538
+ : 'the runner named no test still running'
539
+ } (cwd=${cwd}): ${command}\n`
168
540
  : `\nCommand timed out after ${elapsedMs}ms (timeoutMs=${bound.limitMs}, cwd=${cwd}): ${command}\n`,
169
541
  );
170
- void (async () => {
171
- await ignoreKillError(killChildTree(false));
172
- if (!state.settled && killAfterMs > 0) {
173
- await delay(killAfterMs);
542
+ if (bound.kind === 'idle') {
543
+ const observed = idleCapture?.result;
544
+ appendStderr(
545
+ `Hang diagnostics: runner pid=${child.pid ?? 'unavailable'} pgid=${
546
+ process.platform === 'win32' ? 'unavailable' : (child.pid ?? 'unavailable')
547
+ } expiryAtMs=${Date.now()} scheduledExpiryAtMs=${idleCapture?.expiresAtMs ?? 'unavailable'}\n`,
548
+ );
549
+ if (observed?.kind === 'captured') {
550
+ appendStderr(
551
+ `Hang diagnostics: outcome=${observed.snapshot.outcome} capturedAtMs=${observed.snapshot.capturedAtMs} actualExpiryOffsetMs=${
552
+ Date.now() - observed.snapshot.capturedAtMs
553
+ }\n${observed.snapshot.text}`,
554
+ );
555
+ } else {
556
+ appendStderr(
557
+ `Hang diagnostics: outcome=incomplete; ${
558
+ observed?.kind === 'failed'
559
+ ? `capture failed: ${observed.detail}`
560
+ : `capture incomplete at expiry (startedAtMs=${observed?.startedAtMs ?? 'not started'})`
561
+ }; no empty-holder conclusion\n`,
562
+ );
174
563
  }
175
- if (!state.settled) {
564
+ }
565
+ cancelCapture();
566
+ termination = terminate();
567
+ };
568
+
569
+ // A timed-out run ends with its whole process group, not with its leader.
570
+ // The graceful signal reaches every member that exists when it is sent, but
571
+ // a member that ignores SIGTERM, or one forked while the signal goes out
572
+ // (macOS does not hold a fork back for a group signal in flight), outlives
573
+ // a leader that exits on it. The grace ends when the leader exits or
574
+ // killAfterMs passes, whichever comes first; whatever is left then is killed.
575
+ const terminate = async (): Promise<void> => {
576
+ await ignoreKillError(killChildTree(false));
577
+ if (!state.settled && killAfterMs > 0) {
578
+ await settledWithin(exitPromise, killAfterMs);
579
+ }
580
+ if (state.settled) {
581
+ if (child.pid !== undefined && processGroupHasMembers(child.pid)) {
176
582
  state.forceKillNeeded = true;
177
- appendStderr(`Force-killing timed out command after killAfterMs=${killAfterMs}: ${command}\n`);
583
+ appendStderr(
584
+ `Force-killing what the ${state.expiry === null ? 'command' : 'timed-out command'} left in its process group: ${command}\n`,
585
+ );
178
586
  await ignoreKillError(killChildTree(true));
179
- await delay(REAP_AFTER_FORCE_KILL_MS);
180
587
  }
181
- if (!state.settled) {
182
- resolveExit({ code: 1, signal: 'SIGKILL' });
183
- }
184
- })();
588
+ return;
589
+ }
590
+ state.forceKillNeeded = true;
591
+ appendStderr(
592
+ `Force-killing ${state.expiry === null ? 'command' : 'timed out command'} after killAfterMs=${killAfterMs}: ${command}\n`,
593
+ );
594
+ await ignoreKillError(killChildTree(true));
595
+ await settledWithin(exitPromise, REAP_AFTER_FORCE_KILL_MS);
596
+ if (!state.settled) {
597
+ resolveExit({ code: 1, signal: 'SIGKILL' });
598
+ }
185
599
  };
186
600
 
187
- const totalTimer = setTimeout(() => expire({ kind: 'total', limitMs: timeoutMs }), timeoutMs);
601
+ const totalTimer =
602
+ timeoutMs === undefined ? undefined : setTimeout(() => expire({ kind: 'total', limitMs: timeoutMs }), timeoutMs);
188
603
  armIdleTimer();
189
604
 
605
+ // A borrowed grant ends with the ancestor that lent it, and the command runs on tokens that ancestor takes back
606
+ // then. Stopping the command is a lost grant: a failure, not a bound. Our own release after the command settled
607
+ // is not a loss; `cpuReleased` marks it.
608
+ const onGrantLost = (): void => {
609
+ if (state.settled || state.grantLost || cpuReleased !== null) {
610
+ return;
611
+ }
612
+ state.grantLost = true;
613
+ cancelCapture();
614
+ appendStderr(
615
+ `cowshed: cpu-tokens grant lost: its parent pool closed while the command ran; stopping it: ${command}\n`,
616
+ );
617
+ // A bound may already be terminating the still-running command. The loss remains an ordinary
618
+ // operational failure, but that existing termination owns the signals and grace.
619
+ if (state.expiry === null) termination = terminate();
620
+ };
621
+ if (grant?.inherited) {
622
+ void grant.closed.then(onGrantLost);
623
+ }
624
+
190
625
  const { code: exitCode, signal: exitSignal } = await exitPromise;
626
+ await termination;
191
627
  clearTimeout(totalTimer);
192
628
  clearTimeout(idleTimer);
629
+ cancelCapture();
193
630
  removeSignalHandlers();
631
+ // The runner is gone. Its sub-pool closes first, so no descendant still draws on tokens the grant returns;
632
+ // then the grant goes back to its parent or the host.
633
+ await releaseCpu();
194
634
 
195
635
  const code = exitCode ?? signalToExitCode(exitSignal);
196
636
  if (code !== 0 && state.expiry === null) {
@@ -201,8 +641,20 @@ export async function runBoundedExec(
201
641
  appendStderr(`Timed out command exited after graceful termination: ${command}\n`);
202
642
  }
203
643
 
644
+ // A test that dies of ENOSPC rarely says where it was writing; the volume can.
645
+ const full = code !== 0 && lease !== null ? await tempVolume?.fullness(lease) : null;
646
+ if (full) {
647
+ appendStderr(`${describeRamTempError(full)}\n`);
648
+ }
649
+ await releaseLease();
650
+
651
+ // A run whose grant was lost, or whose sub-pool did not close cleanly, fails even when the command exited 0, and
652
+ // whatever bound also fired: its verdict is an ordinary failure, never a bound.
653
+ const operationalFailure = state.grantLost || state.poolCloseError !== null;
654
+ await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure);
655
+
204
656
  return {
205
- success: state.expiry === null && code === 0,
657
+ success: !operationalFailure && state.expiry === null && code === 0,
206
658
  terminalOutput: outputChunks.join(''),
207
659
  };
208
660
  }
@@ -217,21 +669,35 @@ export function createProcessTreeKiller(): ProcessTreeKiller {
217
669
 
218
670
  function killProcessGroup(pid: number, signal: NodeJS.Signals): Promise<void> {
219
671
  try {
220
- // Send signal to the entire process group (negative PID).
221
- // The executor spawns with detached: true on POSIX, which creates a
222
- // dedicated process group. Signaling -pid is atomic and catches all
223
- // descendants, including grandchildren that tree-walk libraries miss
224
- // when parents die and children get reparented to init.
225
- process.kill(-pid, signal);
672
+ // On POSIX the executor spawns with detached: true, so the child leads a
673
+ // process group of its own and -pid signals all of it at once, including
674
+ // grandchildren that tree-walk libraries miss when parents die and
675
+ // children get reparented to init. Only the group is named: the leader may
676
+ // already be reaped, and its pid reused by an unrelated process, while the
677
+ // group lives on in its other members. Windows has no process groups.
678
+ process.kill(process.platform === 'win32' ? pid : -pid, signal);
226
679
  } catch {
227
- // ESRCH: process group already exited, or this pid is not a group leader.
680
+ // ESRCH: the group (or the Windows process) has already exited.
681
+ }
682
+ return Promise.resolve();
683
+ }
684
+
685
+ /**
686
+ * Whether any process is left in the group `pgid` leads. A group outlives its
687
+ * reaped leader for as long as one member does, and no process can take its id
688
+ * meanwhile. Windows has no process groups, so there is nothing left to name.
689
+ */
690
+ function processGroupHasMembers(pgid: number): boolean {
691
+ if (process.platform === 'win32') {
692
+ return false;
228
693
  }
229
694
  try {
230
- process.kill(pid, signal);
231
- } catch {
232
- // Already reaped.
695
+ process.kill(-pgid, 0);
696
+ return true;
697
+ } catch (error) {
698
+ // EPERM: a member exists but runs as another user.
699
+ return error instanceof Error && 'code' in error && error.code === 'EPERM';
233
700
  }
234
- return Promise.resolve();
235
701
  }
236
702
 
237
703
  function resolveCwd(cwd: string | undefined, root: string): string {
@@ -254,8 +720,34 @@ function buildCommand(options: BoundedExecOptions): string {
254
720
  return parts.join(' ');
255
721
  }
256
722
 
257
- function mergeEnv(env: Record<string, string> | undefined): NodeJS.ProcessEnv {
258
- return env ? { ...process.env, ...env } : process.env;
723
+ /** The lease's directory is the command's TMPDIR unless the target names one itself. */
724
+ function mergeEnv(lease: RamTempLease | null, env: Record<string, string>): NodeJS.ProcessEnv {
725
+ const inherited = withoutBudgetEnv(process.env);
726
+ return lease === null ? { ...inherited, ...env } : { ...inherited, TMPDIR: lease.directory, ...env };
727
+ }
728
+
729
+ /** The process's environment without the CPU budget's variables: a pool and its count only a grant may give. */
730
+ function withoutBudgetEnv(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv {
731
+ const kept = { ...env };
732
+ for (const name of BUDGET_ENV) {
733
+ delete kept[name];
734
+ }
735
+ return kept;
736
+ }
737
+
738
+ /**
739
+ * The target's environment without the CPU budget's variables: the budget sets those for the command it grants,
740
+ * so a value the target names is ignored, and `say` tells the reader so.
741
+ */
742
+ function targetEnv(env: Record<string, string> | undefined, say: (text: string) => void): Record<string, string> {
743
+ const kept = { ...env };
744
+ for (const name of BUDGET_ENV) {
745
+ if (name in kept) {
746
+ say(`cowshed: ${name} in the target's env is ignored: the CPU budget sets it\n`);
747
+ delete kept[name];
748
+ }
749
+ }
750
+ return kept;
259
751
  }
260
752
 
261
753
  function signalToExitCode(signal: NodeJS.Signals | null): number {
@@ -273,6 +765,12 @@ async function ignoreKillError(promise: Promise<void>): Promise<void> {
273
765
  }
274
766
  }
275
767
 
276
- function delay(ms: number): Promise<void> {
277
- return new Promise((resolve) => setTimeout(resolve, ms));
768
+ /** Resolves when `promise` settles or `ms` passes, whichever is first, leaving no timer behind. */
769
+ async function settledWithin(promise: Promise<unknown>, ms: number): Promise<void> {
770
+ let timer: NodeJS.Timeout | undefined;
771
+ try {
772
+ await Promise.race([promise, new Promise<void>((resolve) => (timer = setTimeout(resolve, ms)))]);
773
+ } finally {
774
+ clearTimeout(timer);
775
+ }
278
776
  }