@smoothbricks/nx-plugin 0.5.7 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1097 -360
- package/dist/.build-inputs.json +141 -0
- package/dist/bin/smoo-nx-cargo-toolchain.d.ts +3 -0
- package/dist/bin/smoo-nx-cargo-toolchain.d.ts.map +1 -0
- package/dist/bin/smoo-nx-cargo-toolchain.js +8 -0
- package/dist/bin/smoo-nx-forbidden-spellings.d.ts +3 -0
- package/dist/bin/smoo-nx-forbidden-spellings.d.ts.map +1 -0
- package/dist/bin/smoo-nx-forbidden-spellings.js +32 -0
- package/dist/bin/smoo-nx-nextest-timings.d.ts +3 -0
- package/dist/bin/smoo-nx-nextest-timings.d.ts.map +1 -0
- package/dist/bin/smoo-nx-nextest-timings.js +28 -0
- package/dist/bounded-test-policy.d.ts +1 -1
- package/dist/bounded-test-policy.d.ts.map +1 -1
- package/dist/bounded-test-policy.js +14 -15
- package/dist/build-state-output-policy.d.ts +95 -1
- package/dist/build-state-output-policy.d.ts.map +1 -1
- package/dist/build-state-output-policy.js +368 -30
- package/dist/bun-lock-scope.d.ts +44 -0
- package/dist/bun-lock-scope.d.ts.map +1 -0
- package/dist/bun-lock-scope.js +97 -0
- package/dist/cargo-build-script-env.d.ts +49 -0
- package/dist/cargo-build-script-env.d.ts.map +1 -0
- package/dist/cargo-build-script-env.js +324 -0
- package/dist/cargo-closure-input.d.ts +31 -4
- package/dist/cargo-closure-input.d.ts.map +1 -1
- package/dist/cargo-closure-input.js +66 -6
- package/dist/cargo-retention-hook.d.ts +17 -0
- package/dist/cargo-retention-hook.d.ts.map +1 -0
- package/dist/cargo-retention-hook.js +85 -0
- package/dist/cargo-source-hash.d.ts +25 -7
- package/dist/cargo-source-hash.d.ts.map +1 -1
- package/dist/cargo-source-hash.js +108 -43
- package/dist/cargo-toolchain-paths.d.ts +40 -0
- package/dist/cargo-toolchain-paths.d.ts.map +1 -0
- package/dist/cargo-toolchain-paths.js +84 -0
- package/dist/cargo-toolchain-policy.d.ts +25 -8
- package/dist/cargo-toolchain-policy.d.ts.map +1 -1
- package/dist/cargo-toolchain-policy.js +39 -5
- package/dist/cargo-workspace.d.ts +10 -4
- package/dist/cargo-workspace.d.ts.map +1 -1
- package/dist/cargo-workspace.js +65 -95
- package/dist/cross-check-policy.d.ts +9 -9
- package/dist/cross-check-policy.d.ts.map +1 -1
- package/dist/cross-check-policy.js +11 -11
- package/dist/disk-command-policy.d.ts.map +1 -1
- package/dist/disk-command-policy.js +1 -0
- package/dist/ensure-built-receipt.d.ts +22 -5
- package/dist/ensure-built-receipt.d.ts.map +1 -1
- package/dist/ensure-built-receipt.js +168 -48
- package/dist/ensure-built-witness.d.ts +27 -8
- package/dist/ensure-built-witness.d.ts.map +1 -1
- package/dist/ensure-built-witness.js +45 -15
- package/dist/ensure-built.d.ts.map +1 -1
- package/dist/ensure-built.js +9 -1
- package/dist/eslint-config-closure.d.ts +7 -0
- package/dist/eslint-config-closure.d.ts.map +1 -0
- package/dist/eslint-config-closure.js +346 -0
- package/dist/executors/bounded-exec/bounds.d.ts +56 -0
- package/dist/executors/bounded-exec/bounds.d.ts.map +1 -0
- package/dist/executors/bounded-exec/bounds.js +43 -0
- package/dist/executors/bounded-exec/cpu-pool-thread.d.ts +28 -0
- package/dist/executors/bounded-exec/cpu-pool-thread.d.ts.map +1 -0
- package/dist/executors/bounded-exec/cpu-pool-thread.js +20 -0
- package/dist/executors/bounded-exec/cpu-pool.d.ts +35 -38
- package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool.js +56 -120
- package/dist/executors/bounded-exec/executor.d.ts +19 -12
- package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
- package/dist/executors/bounded-exec/executor.js +389 -239
- package/dist/executors/bounded-exec/gateway-lease.d.ts +23 -19
- package/dist/executors/bounded-exec/gateway-lease.d.ts.map +1 -1
- package/dist/executors/bounded-exec/gateway-lease.js +102 -29
- package/dist/executors/bounded-exec/hang-diagnostics.d.ts +18 -1
- package/dist/executors/bounded-exec/hang-diagnostics.d.ts.map +1 -1
- package/dist/executors/bounded-exec/hang-diagnostics.js +148 -30
- package/dist/executors/bounded-exec/lease-wait.d.ts +64 -0
- package/dist/executors/bounded-exec/lease-wait.d.ts.map +1 -0
- package/dist/executors/bounded-exec/lease-wait.js +72 -0
- package/dist/executors/bounded-exec/lock-wait.d.ts +51 -0
- package/dist/executors/bounded-exec/lock-wait.d.ts.map +1 -0
- package/dist/executors/bounded-exec/lock-wait.js +108 -0
- package/dist/executors/bounded-exec/mbx-pool.d.ts +31 -0
- package/dist/executors/bounded-exec/mbx-pool.d.ts.map +1 -0
- package/dist/executors/bounded-exec/mbx-pool.js +24 -0
- package/dist/executors/bounded-exec/owned-subtree.d.ts +84 -0
- package/dist/executors/bounded-exec/owned-subtree.d.ts.map +1 -0
- package/dist/executors/bounded-exec/owned-subtree.js +178 -0
- package/dist/executors/bounded-exec/progress.d.ts +4 -1
- package/dist/executors/bounded-exec/progress.d.ts.map +1 -1
- package/dist/executors/bounded-exec/progress.js +5 -2
- package/dist/executors/bounded-exec/ram-attach.d.ts +19 -0
- package/dist/executors/bounded-exec/ram-attach.d.ts.map +1 -0
- package/dist/executors/bounded-exec/ram-attach.js +207 -0
- package/dist/executors/bounded-exec/ram-detach.d.ts +25 -0
- package/dist/executors/bounded-exec/ram-detach.d.ts.map +1 -0
- package/dist/executors/bounded-exec/ram-detach.js +207 -0
- package/dist/executors/bounded-exec/ram-temp-fixtures.d.ts +76 -0
- package/dist/executors/bounded-exec/ram-temp-fixtures.d.ts.map +1 -1
- package/dist/executors/bounded-exec/ram-temp-fixtures.js +296 -0
- package/dist/executors/bounded-exec/ram-temp.d.ts +388 -88
- package/dist/executors/bounded-exec/ram-temp.d.ts.map +1 -1
- package/dist/executors/bounded-exec/ram-temp.js +1213 -399
- package/dist/executors/bounded-exec/run-cpu-pool.d.ts +10 -4
- package/dist/executors/bounded-exec/run-cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/run-cpu-pool.js +122 -35
- package/dist/executors/bounded-exec/runner-hook.js +14 -5
- package/dist/executors/bounded-exec/runner-lifecycle.d.ts +14 -1
- package/dist/executors/bounded-exec/runner-lifecycle.d.ts.map +1 -1
- package/dist/executors/bounded-exec/runner-lifecycle.js +27 -6
- package/dist/executors/bounded-exec/{cpu-tokens.d.ts → runner-sizing.d.ts} +14 -42
- package/dist/executors/bounded-exec/runner-sizing.d.ts.map +1 -0
- package/dist/executors/bounded-exec/{cpu-tokens.js → runner-sizing.js} +5 -35
- package/dist/executors/bounded-exec/runtime-pin.d.ts +73 -0
- package/dist/executors/bounded-exec/runtime-pin.d.ts.map +1 -0
- package/dist/executors/bounded-exec/runtime-pin.js +167 -0
- package/dist/executors/bounded-exec/schema.d.ts +18 -1
- package/dist/executors/bounded-exec/schema.d.ts.map +1 -1
- package/dist/executors/bounded-exec/verdict.d.ts +48 -11
- package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
- package/dist/executors/bounded-exec/verdict.js +89 -20
- package/dist/executors/cargo-zigbuild/executor.d.ts +21 -0
- package/dist/executors/cargo-zigbuild/executor.d.ts.map +1 -1
- package/dist/executors/cargo-zigbuild/executor.js +67 -12
- package/dist/executors/clean-outputs/executor.js +1 -0
- package/dist/executors/scriptc-native/executor.d.ts +79 -0
- package/dist/executors/scriptc-native/executor.d.ts.map +1 -0
- package/dist/executors/scriptc-native/executor.js +367 -0
- package/dist/executors/scriptc-native/ffi-paths.d.ts +9 -0
- package/dist/executors/scriptc-native/ffi-paths.d.ts.map +1 -0
- package/dist/executors/scriptc-native/ffi-paths.js +25 -0
- package/dist/executors/scriptc-native/schema.d.ts +34 -0
- package/dist/executors/scriptc-native/schema.d.ts.map +1 -0
- package/dist/executors/scriptc-native/schema.js +1 -0
- package/dist/executors/typescript-emit/emit-overrides.d.ts +14 -0
- package/dist/executors/typescript-emit/emit-overrides.d.ts.map +1 -0
- package/dist/executors/typescript-emit/emit-overrides.js +24 -0
- package/dist/executors/typescript-emit/executor.d.ts +3 -1
- package/dist/executors/typescript-emit/executor.d.ts.map +1 -1
- package/dist/executors/typescript-emit/executor.js +146 -33
- package/dist/forbidden-spelling-policy.d.ts +56 -0
- package/dist/forbidden-spelling-policy.d.ts.map +1 -0
- package/dist/forbidden-spelling-policy.js +176 -0
- package/dist/generators/create-package/generator.d.ts.map +1 -1
- package/dist/generators/create-package/generator.js +3 -4
- package/dist/graph-input-contract.d.ts +14 -0
- package/dist/graph-input-contract.d.ts.map +1 -1
- package/dist/graph-input-contract.js +67 -2
- package/dist/index.d.ts +23 -6
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +672 -214
- package/dist/managed-assets.d.ts +2 -0
- package/dist/managed-assets.d.ts.map +1 -1
- package/dist/managed-files/files.d.ts +3 -3
- package/dist/managed-files/files.d.ts.map +1 -1
- package/dist/managed-files/files.js +28 -3
- package/dist/managed-files/managed-format.d.ts.map +1 -1
- package/dist/managed-files/managed-format.js +1 -0
- package/dist/managed-files/publish-workflow.js +14 -11
- package/dist/nextest-code-bound.js +1 -0
- package/dist/nextest-timings.d.ts +35 -0
- package/dist/nextest-timings.d.ts.map +1 -0
- package/dist/nextest-timings.js +168 -0
- package/dist/owned-installation.d.ts +6 -5
- package/dist/owned-installation.d.ts.map +1 -1
- package/dist/owned-installation.js +150 -26
- package/dist/package-target-policy.d.ts.map +1 -1
- package/dist/package-target-policy.js +2 -3
- package/dist/process-children.d.ts +7 -0
- package/dist/process-children.d.ts.map +1 -0
- package/dist/process-children.darwin.d.ts +7 -0
- package/dist/process-children.darwin.d.ts.map +1 -0
- package/dist/process-children.darwin.js +13 -0
- package/dist/process-children.js +14 -0
- package/dist/process-children.linux.d.ts +8 -0
- package/dist/process-children.linux.d.ts.map +1 -0
- package/dist/process-children.linux.js +25 -0
- package/dist/process-group.d.ts +108 -0
- package/dist/process-group.d.ts.map +1 -0
- package/dist/process-group.js +171 -0
- package/dist/process-listing.d.ts +7 -0
- package/dist/process-listing.d.ts.map +1 -0
- package/dist/process-listing.js +51 -0
- package/dist/producer-receipts.d.ts +8 -2
- package/dist/producer-receipts.d.ts.map +1 -1
- package/dist/producer-receipts.js +514 -47
- package/dist/task-graph-policy.d.ts +21 -0
- package/dist/task-graph-policy.d.ts.map +1 -0
- package/dist/task-graph-policy.js +91 -0
- package/dist/test-timings.js +4 -4
- package/dist/test-walk-policy.d.ts +14 -5
- package/dist/test-walk-policy.d.ts.map +1 -1
- package/dist/test-walk-policy.js +8 -2
- package/dist/testing.d.ts +73 -28
- package/dist/testing.d.ts.map +1 -1
- package/dist/testing.js +217 -223
- package/dist/tool-bins.d.ts +33 -0
- package/dist/tool-bins.d.ts.map +1 -0
- package/dist/tool-bins.js +72 -0
- package/dist/typecheck-test-policy.d.ts.map +1 -1
- package/dist/typecheck-test-policy.js +3 -4
- package/dist/wait-why-policy.d.ts +28 -0
- package/dist/wait-why-policy.d.ts.map +1 -0
- package/dist/wait-why-policy.js +223 -0
- package/dist/workspace-config-policy.d.ts +14 -8
- package/dist/workspace-config-policy.d.ts.map +1 -1
- package/dist/workspace-config-policy.js +60 -16
- package/dist/workspace-manifest.d.ts +41 -6
- package/dist/workspace-manifest.d.ts.map +1 -1
- package/dist/workspace-manifest.js +57 -0
- package/executors.json +16 -1
- package/loader.js +2 -0
- package/managed/raw/envrc +11 -0
- package/managed/raw/patches/nx@23.2.1.patch +2840 -199
- package/managed/raw/patches/ttsc@0.34.1.patch +119 -0
- package/managed/raw/tooling/direnv/cargo-fetch-bound.ts +245 -0
- package/managed/raw/tooling/direnv/cargo-fetch-run.ts +299 -0
- package/managed/raw/tooling/direnv/compiler-variables.sh +42 -0
- package/managed/raw/tooling/direnv/developer-links.ts +464 -25
- package/managed/raw/tooling/direnv/devenv.smoo.nix +78 -38
- package/managed/raw/tooling/direnv/github-actions-bootstrap.sh +14 -12
- package/managed/raw/tooling/direnv/installed-bun-links.ts +308 -0
- package/managed/raw/tooling/direnv/linked-snapshots.ts +62 -3
- package/managed/raw/tooling/direnv/nx-socket-dir.sh +56 -17
- package/managed/raw/tooling/direnv/repo-path +68 -6
- package/managed/raw/tooling/direnv/secret-references.ts +13 -4
- package/managed/raw/tooling/direnv/setup-environment.ts +128 -537
- package/managed/raw/tooling/direnv/toolchain-stamp.ts +4 -2
- package/managed/raw/tooling/direnv/ttsc-env.sh +9 -0
- package/managed/raw/tooling/direnv/uv-relocation.ts +227 -0
- package/managed/raw/tooling/git-hooks/commit-msg.sh +2 -1
- package/managed/raw/tooling/git-hooks/pre-commit.sh +2 -1
- package/managed/raw/tooling/git-hooks/pre-push.sh +2 -1
- package/managed/templates/github/actions/cache-ttsc-plugins/action.yml +85 -7
- package/managed/templates/github/actions/save-nix-devenv/action.yml +12 -2
- package/managed/templates/github/actions/setup-devenv/action.yml +2 -0
- package/package.json +95 -14
- package/src/bin/smoo-nx-cargo-toolchain.ts +11 -0
- package/src/bin/smoo-nx-forbidden-spellings.ts +33 -0
- package/src/bin/smoo-nx-nextest-timings.ts +31 -0
- package/src/bounded-test-policy.ts +12 -14
- package/src/build-state-output-policy.ts +458 -34
- package/src/bun-lock-scope.ts +108 -0
- package/src/cargo-build-script-env.ts +265 -0
- package/src/cargo-closure-input.ts +74 -7
- package/src/cargo-retention-hook.ts +94 -0
- package/src/cargo-source-hash.ts +138 -41
- package/src/cargo-toolchain-paths.ts +95 -0
- package/src/cargo-toolchain-policy.ts +49 -5
- package/src/cargo-workspace.ts +72 -123
- package/src/cross-check-policy.ts +12 -10
- package/src/disk-command-policy.ts +1 -0
- package/src/ensure-built-receipt.ts +173 -39
- package/src/ensure-built-witness.ts +62 -10
- package/src/ensure-built.ts +11 -1
- package/src/eslint-config-closure.ts +357 -0
- package/src/executors/bounded-exec/bounds.ts +76 -0
- package/src/executors/bounded-exec/cpu-pool-thread.ts +43 -0
- package/src/executors/bounded-exec/cpu-pool.ts +55 -120
- package/src/executors/bounded-exec/executor.ts +442 -260
- package/src/executors/bounded-exec/gateway-lease.ts +94 -35
- package/src/executors/bounded-exec/hang-diagnostics.ts +173 -31
- package/src/executors/bounded-exec/lease-wait.ts +116 -0
- package/src/executors/bounded-exec/lock-wait.ts +141 -0
- package/src/executors/bounded-exec/mbx-pool.ts +48 -0
- package/src/executors/bounded-exec/owned-subtree.ts +235 -0
- package/src/executors/bounded-exec/progress.ts +9 -3
- package/src/executors/bounded-exec/ram-attach.ts +238 -0
- package/src/executors/bounded-exec/ram-detach.ts +226 -0
- package/src/executors/bounded-exec/ram-temp-fixtures.ts +358 -0
- package/src/executors/bounded-exec/ram-temp-registry.txt +440 -0
- package/src/executors/bounded-exec/ram-temp.ts +1497 -460
- package/src/executors/bounded-exec/run-cpu-pool.ts +129 -35
- package/src/executors/bounded-exec/runner-hook.ts +14 -6
- package/src/executors/bounded-exec/runner-lifecycle.ts +31 -8
- package/src/executors/bounded-exec/{cpu-tokens.ts → runner-sizing.ts} +13 -59
- package/src/executors/bounded-exec/runtime-pin.ts +232 -0
- package/src/executors/bounded-exec/schema.json +11 -2
- package/src/executors/bounded-exec/schema.ts +19 -1
- package/src/executors/bounded-exec/verdict.ts +145 -35
- package/src/executors/cargo-zigbuild/executor.ts +82 -8
- package/src/executors/clean-outputs/executor.ts +1 -0
- package/src/executors/scriptc-native/executor.ts +394 -0
- package/src/executors/scriptc-native/ffi-paths.ts +29 -0
- package/src/executors/scriptc-native/schema.json +60 -0
- package/src/executors/scriptc-native/schema.ts +34 -0
- package/src/executors/typescript-emit/emit-overrides.ts +24 -0
- package/src/executors/typescript-emit/executor.ts +186 -38
- package/src/forbidden-spelling-policy.ts +226 -0
- package/src/generators/create-package/generator.ts +7 -7
- package/src/graph-input-contract.ts +61 -2
- package/src/index.ts +788 -224
- package/src/managed-assets.ts +3 -0
- package/src/managed-files/files.ts +28 -3
- package/src/managed-files/managed-format.ts +1 -0
- package/src/managed-files/publish-workflow.ts +20 -15
- package/src/nextest-code-bound.ts +1 -0
- package/src/nextest-timings.ts +106 -0
- package/src/owned-installation.ts +150 -23
- package/src/package-target-policy.ts +2 -3
- package/src/process-children.darwin.ts +14 -0
- package/src/process-children.linux.ts +33 -0
- package/src/process-children.ts +22 -0
- package/src/process-group.ts +209 -0
- package/src/process-listing.ts +54 -0
- package/src/producer-receipts.ts +92 -41
- package/src/task-graph-policy.ts +98 -0
- package/src/test-walk-policy.ts +23 -8
- package/src/testing.ts +240 -232
- package/src/tool-bins.ts +108 -0
- package/src/typecheck-test-policy.ts +2 -3
- package/src/wait-why-policy.ts +226 -0
- package/src/workspace-config-policy.ts +63 -15
- package/src/workspace-manifest.ts +89 -6
- package/test-timings.json +124 -99
- package/dist/bin/smoo-nx-failures.d.ts +0 -3
- package/dist/bin/smoo-nx-failures.d.ts.map +0 -1
- package/dist/bin/smoo-nx-failures.js +0 -3
- package/dist/bin/smoo-nx-nextest-extract.d.ts +0 -3
- package/dist/bin/smoo-nx-nextest-extract.d.ts.map +0 -1
- package/dist/bin/smoo-nx-nextest-extract.js +0 -42
- package/dist/executors/bounded-exec/cpu-tokens.d.ts.map +0 -1
- package/dist/failure-summary.d.ts +0 -5
- package/dist/failure-summary.d.ts.map +0 -1
- package/dist/failure-summary.js +0 -118
- package/dist/nextest-extraction.d.ts +0 -117
- package/dist/nextest-extraction.d.ts.map +0 -1
- package/dist/nextest-extraction.js +0 -303
- package/managed/raw/patches/ttsc@0.30.4.patch +0 -2255
- package/src/__tests__/cargo-closure-filesets.ts +0 -94
- package/src/__tests__/cargo-closure-fixture.ts +0 -113
- package/src/__tests__/counted-cargo-process.ts +0 -286
- package/src/__tests__/counted-cargo.ts +0 -713
- package/src/__tests__/directory-change.ts +0 -27
- package/src/__tests__/fixture-cargo-home.ts +0 -36
- package/src/__tests__/fixture-nx-env.ts +0 -193
- package/src/__tests__/owned-installation.ts +0 -13
- package/src/__tests__/past-tick.ts +0 -12
- package/src/__tests__/progress-clock.ts +0 -43
- package/src/__tests__/runtime-input.ts +0 -23
- package/src/bin/smoo-nx-bound-failures.test.ts +0 -127
- package/src/bin/smoo-nx-failures.ts +0 -4
- package/src/bin/smoo-nx-nextest-extract.ts +0 -44
- package/src/bounded-test-policy.test.ts +0 -1029
- package/src/build-state-output-policy.test.ts +0 -115
- package/src/bun-bootstrap.test.ts +0 -102
- package/src/bun-file-packages.test.ts +0 -47
- package/src/cargo-closure-filesets.test.ts +0 -63
- package/src/cargo-closure-input.property.test.ts +0 -50
- package/src/cargo-closure-input.test.ts +0 -506
- package/src/cargo-closure-key.test.ts +0 -240
- package/src/cargo-resolve-cache.test.ts +0 -1181
- package/src/cargo-source-hash.test.ts +0 -707
- package/src/cargo-workspace.test.ts +0 -615
- package/src/cross-check-policy.test.ts +0 -316
- package/src/disk-command-policy.test.ts +0 -61
- package/src/ensure-built-receipt.test.ts +0 -279
- package/src/ensure-built-witness.test.ts +0 -688
- package/src/ensure-built.test.ts +0 -1462
- package/src/executors/all-verdicts/executor.test.ts +0 -135
- package/src/executors/bounded-exec/cargo-progress-host.test.ts +0 -332
- package/src/executors/bounded-exec/cpu-pool.test.ts +0 -684
- package/src/executors/bounded-exec/cpu-tokens.test.ts +0 -519
- package/src/executors/bounded-exec/disk-lease.test.ts +0 -219
- package/src/executors/bounded-exec/executor-host.test.ts +0 -800
- package/src/executors/bounded-exec/executor.test.ts +0 -1819
- package/src/executors/bounded-exec/go-progress-host.test.ts +0 -171
- package/src/executors/bounded-exec/hang-diagnostics.test.ts +0 -291
- package/src/executors/bounded-exec/progress.test.ts +0 -139
- package/src/executors/bounded-exec/ram-temp-held.test.ts +0 -264
- package/src/executors/bounded-exec/ram-temp-host.test.ts +0 -635
- package/src/executors/bounded-exec/ram-temp.test.ts +0 -750
- package/src/executors/bounded-exec/run-cpu-pool.test.ts +0 -79
- package/src/executors/bounded-exec/runner-hook.test.ts +0 -193
- package/src/executors/bounded-exec/runner-lifecycle.test.ts +0 -82
- package/src/executors/bounded-exec/verdict.test.ts +0 -545
- package/src/executors/cargo-zigbuild/executor.test.ts +0 -129
- package/src/executors/clean-outputs/executor.test.ts +0 -20
- package/src/executors/typescript-emit/executor.test.ts +0 -83
- package/src/failure-summary.ts +0 -119
- package/src/generators/bounded-test-targets/generator.test.ts +0 -138
- package/src/generators/bun-test-tracing/generator.test.ts +0 -250
- package/src/generators/create-package/generator.test.ts +0 -190
- package/src/generators/make-public/generator.test.ts +0 -137
- package/src/graph-input-contract.test.ts +0 -379
- package/src/host-controller-wiring.test.ts +0 -31
- package/src/index.test.ts +0 -3321
- package/src/linked-snapshots.nx.test.ts +0 -254
- package/src/managed-files/forgejo-artifact.test.ts +0 -395
- package/src/managed-files/generator.test.ts +0 -243
- package/src/managed-files/github-host-cache-isolation.test.ts +0 -86
- package/src/managed-files/json-edit.test.ts +0 -216
- package/src/managed-files/managed-content.test.ts +0 -166
- package/src/managed-files/managed-format-json.test.ts +0 -308
- package/src/managed-files/repo-path.test.ts +0 -172
- package/src/managed-files/setup-devenv-free-disk.test.ts +0 -95
- package/src/managed-files/trace-db-artifact.test.ts +0 -7
- package/src/managed-files/tree-permissions.test.ts +0 -109
- package/src/managed-files/tree.test.ts +0 -123
- package/src/manifest-hash.test.ts +0 -219
- package/src/nextest-code-bound.test.ts +0 -161
- package/src/nextest-extraction.test.ts +0 -216
- package/src/nextest-extraction.ts +0 -371
- package/src/nextest-partitions.test.ts +0 -501
- package/src/nx-cache-bound.test.ts +0 -109
- package/src/nx-cache-restore-mtime.test.ts +0 -170
- package/src/nx-daemon-claim.test.ts +0 -279
- package/src/nx-daemon-idle-cpu.test.ts +0 -80
- package/src/nx-daemon-idle-policy.test.ts +0 -111
- package/src/nx-daemon-ignore-files.test.ts +0 -210
- package/src/nx-daemon-root-normalization.test.ts +0 -105
- package/src/nx-daemon-start-never.test.ts +0 -292
- package/src/nx-daemon-validity.test.ts +0 -1055
- package/src/nx-dot-input-policy.test.ts +0 -353
- package/src/nx-failure-life-cycles.test.ts +0 -207
- package/src/nx-files-archive-racy.test.ts +0 -97
- package/src/nx-foreign-workspace-env.test.ts +0 -329
- package/src/nx-incomplete-run-summary.test.ts +0 -70
- package/src/nx-plugin-load-errors.test.ts +0 -120
- package/src/nx-project-graph-errors.test.ts +0 -70
- package/src/nx-project-resolution-cache.test.ts +0 -176
- package/src/nx-raw-inputs.test.ts +0 -166
- package/src/nx-resident-graph-workers.test.ts +0 -112
- package/src/nx-run-information.test.ts +0 -118
- package/src/nx-skipped-task-exit.test.ts +0 -180
- package/src/nx-state-location.test.ts +0 -63
- package/src/nx-store-resolution.test.ts +0 -81
- package/src/nx-task-graph-cycles.test.ts +0 -148
- package/src/nx-task-output-lease.test.ts +0 -550
- package/src/nx-use-local.test.ts +0 -105
- package/src/owned-installation.test.ts +0 -105
- package/src/package-target-policy.test.ts +0 -1014
- package/src/producer-receipts.test.ts +0 -285
- package/src/release-config-policy.test.ts +0 -311
- package/src/runtime-input.test.ts +0 -393
- package/src/serial-group-wiring.test.ts +0 -326
- package/src/task-history-namespace.test.ts +0 -181
- package/src/task-results.test.ts +0 -228
- package/src/test-clock-policy.test.ts +0 -338
- package/src/test-program.test.ts +0 -303
- package/src/test-timings.test.ts +0 -240
- package/src/test-walk-policy.test.ts +0 -620
- package/src/testing.test.ts +0 -509
- package/src/typecheck-test-policy.test.ts +0 -1468
- package/src/typescript-resolution.test.ts +0 -76
- package/src/version-actions.test.ts +0 -80
- package/src/workspace-config-policy.test.ts +0 -737
|
@@ -1,42 +1,60 @@
|
|
|
1
1
|
import { type ChildProcess, spawn } from 'node:child_process';
|
|
2
2
|
import { accessSync, constants } from 'node:fs';
|
|
3
|
+
import { mkdtemp, rm } from 'node:fs/promises';
|
|
3
4
|
import { availableParallelism, userInfo } from 'node:os';
|
|
4
5
|
import { delimiter, isAbsolute, join, resolve as resolvePath } from 'node:path';
|
|
5
6
|
import { createInterface } from 'node:readline';
|
|
6
|
-
import { Duplex } from 'node:stream';
|
|
7
|
+
import { Duplex, Readable } from 'node:stream';
|
|
7
8
|
import { workspaceDataDirectoryForWorkspace } from 'nx/src/utils/cache-directory.js';
|
|
8
|
-
|
|
9
|
+
import {
|
|
10
|
+
DEADLINE_PATIENCE,
|
|
11
|
+
describeMembersLeft,
|
|
12
|
+
killProcessGroup,
|
|
13
|
+
processGroupHasMembers,
|
|
14
|
+
signalProcessGroup,
|
|
15
|
+
} from '../../process-group.js';
|
|
16
|
+
import { type BoundEvent, type BoundKind, type Bounds, earliestBound, stepBounds, UNARMED } from './bounds.js';
|
|
9
17
|
import {
|
|
10
18
|
CPU_POOL_ENV,
|
|
11
19
|
type CpuPool,
|
|
12
20
|
createCpuPool,
|
|
13
|
-
type HostWaits,
|
|
14
21
|
type InheritedCpuPool,
|
|
15
22
|
readInheritedCpuPool,
|
|
23
|
+
type TokenGrant,
|
|
16
24
|
takeParentCpuTokens,
|
|
17
25
|
} from './cpu-pool.js';
|
|
18
|
-
import {
|
|
19
|
-
CPU_TOKENS_ENV,
|
|
20
|
-
type CpuBudget,
|
|
21
|
-
gatewayCpuBudget,
|
|
22
|
-
runnerDemand,
|
|
23
|
-
sizeRunner,
|
|
24
|
-
type TokenGrant,
|
|
25
|
-
} from './cpu-tokens.js';
|
|
26
|
+
import { GATEWAY_SOCKET } from './gateway-lease.js';
|
|
26
27
|
import type { HangDiagnostics } from './hang-diagnostics.js';
|
|
27
28
|
import { captureHangDiagnostics, HANG_CAPTURE_MS } from './hang-diagnostics.js';
|
|
29
|
+
import type { CpuWaitRow } from './lease-wait.js';
|
|
30
|
+
import type { LockWait } from './lock-wait.js';
|
|
31
|
+
import { mbxLeaf, mbxPool, withMbxLeaf } from './mbx-pool.js';
|
|
28
32
|
import { createNoSpaceWatch } from './no-space.js';
|
|
29
|
-
import { createRunProgress, withProgressReport } from './progress.js';
|
|
30
33
|
import {
|
|
31
34
|
COWSHED_EXECUTABLE_ENV,
|
|
35
|
+
cowshedSubtreeOwner,
|
|
36
|
+
HANDLE_FD,
|
|
37
|
+
readAdmission,
|
|
38
|
+
SUPERVISOR_SOCKET_ENV,
|
|
39
|
+
type SubtreeAdmission,
|
|
40
|
+
type SubtreeOwner,
|
|
41
|
+
} from './owned-subtree.js';
|
|
42
|
+
import { createRunProgress, withProgressReport } from './progress.js';
|
|
43
|
+
import {
|
|
32
44
|
createHostCommands,
|
|
45
|
+
describeCoverageGap,
|
|
33
46
|
describeRamTempError,
|
|
47
|
+
describeSignalled,
|
|
48
|
+
LEFTOVER_GRACE_MS,
|
|
49
|
+
PRODUCTION,
|
|
34
50
|
RAM_TEMP_CAPACITY_BYTES,
|
|
35
51
|
type RamTempLease,
|
|
36
52
|
RamTempVolume,
|
|
37
53
|
ramTempPaths,
|
|
38
54
|
} from './ram-temp.js';
|
|
39
|
-
import { createRunnerLifecycle, withRunnerLifecycle } from './runner-lifecycle.js';
|
|
55
|
+
import { createRunnerLifecycle, withFixtureLifecycle, withRunnerLifecycle } from './runner-lifecycle.js';
|
|
56
|
+
import { CPU_TOKENS_ENV, runnerDemand, sizeRunner } from './runner-sizing.js';
|
|
57
|
+
import { checkRuntimePin, describeRuntimePinRefusal } from './runtime-pin.js';
|
|
40
58
|
import type { BoundedExecOptions } from './schema.js';
|
|
41
59
|
import {
|
|
42
60
|
type BoundedExecRecord,
|
|
@@ -70,9 +88,9 @@ const EXIT_CODE_BY_SIGNAL: Partial<Record<NodeJS.Signals, number>> = {
|
|
|
70
88
|
/** The pool descriptor: only a grant may give it, so neither the target's nor the inherited environment passes it on. */
|
|
71
89
|
const POOL_ENV = [CPU_TOKENS_ENV, CPU_POOL_ENV];
|
|
72
90
|
/**
|
|
73
|
-
* The variables
|
|
74
|
-
*
|
|
75
|
-
* sub-pool names its own fifo over it.
|
|
91
|
+
* The variables a grant sets for the command it sizes: no target's environment may set them. A jobserver the executor
|
|
92
|
+
* itself inherits, a parent make's or its cowshed job's, still reaches an ungranted command by Cargo's own rule; a
|
|
93
|
+
* granted one's sub-pool names its own fifo over it.
|
|
76
94
|
*/
|
|
77
95
|
const BUDGET_ENV = [...POOL_ENV, 'CARGO_MAKEFLAGS', 'MAKEFLAGS', 'MFLAGS'];
|
|
78
96
|
|
|
@@ -104,8 +122,12 @@ interface RunState {
|
|
|
104
122
|
forceKillNeeded: boolean;
|
|
105
123
|
/** The sub-pool's close error when it did not close cleanly: the run fails. */
|
|
106
124
|
poolCloseError: string | null;
|
|
125
|
+
/** Why the command's subtree could not be retired, when the supervisor's client failed: the run fails. */
|
|
126
|
+
retireError: string | null;
|
|
107
127
|
/** The first output line that said ENOSPC, with the task's volume read when it arrived. */
|
|
108
128
|
noSpace: Promise<NoSpaceSighting> | null;
|
|
129
|
+
/** The lock waits the expiry's snapshot named: empty unless that capture completed. Judged with the expiry. */
|
|
130
|
+
lockWaits: readonly LockWait[];
|
|
109
131
|
}
|
|
110
132
|
|
|
111
133
|
type GrantedTokens = Extract<TokenGrant, { granted: true }>;
|
|
@@ -115,7 +137,10 @@ export interface ProcessTreeKiller {
|
|
|
115
137
|
}
|
|
116
138
|
|
|
117
139
|
/** Where the command's TMPDIR comes from; `null` leaves the inherited one. */
|
|
118
|
-
export type TempVolume = Pick<
|
|
140
|
+
export type TempVolume = Pick<
|
|
141
|
+
RamTempVolume,
|
|
142
|
+
'acquire' | 'release' | 'fullness' | 'space' | 'record' | 'ordinaryParent'
|
|
143
|
+
>;
|
|
119
144
|
|
|
120
145
|
/** Workspace-provided cowshed launchers, in the order a workspace that provides one names them. */
|
|
121
146
|
const WORKSPACE_COWSHED_LAUNCHERS = [
|
|
@@ -170,10 +195,38 @@ export default function boundedExecExecutor(
|
|
|
170
195
|
return runDefaultBoundedExec(options, recordContext(context));
|
|
171
196
|
}
|
|
172
197
|
|
|
173
|
-
/**
|
|
198
|
+
/**
|
|
199
|
+
* Runs with the normal process, RAM lease and CPU pool owners, with an optional stdout sink, on the user's RAM temp
|
|
200
|
+
* volume where there is one, through the host's own gateway: the Nx executor's runs, never a test's.
|
|
201
|
+
*/
|
|
174
202
|
export function runDefaultBoundedExec(
|
|
175
203
|
options: BoundedExecOptions,
|
|
176
204
|
context: BoundedExecContext,
|
|
205
|
+
): Promise<BoundedExecResult> {
|
|
206
|
+
// Only macOS has `hdiutil ram://`; elsewhere the command keeps the inherited TMPDIR.
|
|
207
|
+
return runBoundedExecWith(options, context, (cowshed) =>
|
|
208
|
+
process.platform === 'darwin'
|
|
209
|
+
? new RamTempVolume(
|
|
210
|
+
ramTempPaths(userInfo(), PRODUCTION),
|
|
211
|
+
RAM_TEMP_CAPACITY_BYTES,
|
|
212
|
+
createHostCommands(GATEWAY_SOCKET, undefined, undefined, cowshed),
|
|
213
|
+
undefined,
|
|
214
|
+
undefined,
|
|
215
|
+
'optional',
|
|
216
|
+
)
|
|
217
|
+
: null,
|
|
218
|
+
);
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
/**
|
|
222
|
+
* The default runner with every owner but its RAM volume, which its caller makes: `volume` makes the volume for the
|
|
223
|
+
* cowshed the run's disk commands go through. The Nx executor makes the user's; a host proof passes a fixture volume
|
|
224
|
+
* (`fixtureRamVolume`), so no test leases, creates or detaches the user's volume.
|
|
225
|
+
*/
|
|
226
|
+
export function runBoundedExecWith(
|
|
227
|
+
options: BoundedExecOptions,
|
|
228
|
+
context: BoundedExecContext,
|
|
229
|
+
volume: (cowshed: string | undefined) => TempVolume | null,
|
|
177
230
|
): Promise<BoundedExecResult> {
|
|
178
231
|
// A target that names the cowshed its disk commands run through names it for this run's own RAM volume work and
|
|
179
232
|
// for its command alike. Nx resolves `{workspaceRoot}/…` in an option to a workspace-relative path, which the
|
|
@@ -188,19 +241,14 @@ export function runDefaultBoundedExec(
|
|
|
188
241
|
);
|
|
189
242
|
const run: BoundedExecOptions =
|
|
190
243
|
cowshed === undefined ? options : { ...options, env: { ...options.env, [COWSHED_EXECUTABLE_ENV]: cowshed } };
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
ramTempPaths(userInfo()),
|
|
196
|
-
RAM_TEMP_CAPACITY_BYTES,
|
|
197
|
-
createHostCommands(undefined, undefined, undefined, cowshed),
|
|
198
|
-
)
|
|
199
|
-
: null;
|
|
244
|
+
const tempVolume = volume(cowshed);
|
|
245
|
+
// The command's processes are owned by the supervisor that serves this job, when one does
|
|
246
|
+
// (specs/cowshed/07_api.md, `JobSubtreeHandle`); with none, nothing here may end what it leaves.
|
|
247
|
+
const owner = process.env[SUPERVISOR_SOCKET_ENV] === undefined ? null : cowshedSubtreeOwner(cowshed ?? 'cowshed');
|
|
200
248
|
// A command a parent's CPU pool started draws its tokens from that pool. The pool is named by the environment the
|
|
201
249
|
// parent set for this command; a run no parent started has none.
|
|
202
250
|
const inherited = readInheritedCpuPool(process.env);
|
|
203
|
-
return runBoundedExec(run, context, createProcessTreeKiller(), tempVolume,
|
|
251
|
+
return runBoundedExec(run, context, createProcessTreeKiller(), tempVolume, inherited, captureHangDiagnostics, owner);
|
|
204
252
|
}
|
|
205
253
|
|
|
206
254
|
/**
|
|
@@ -227,24 +275,22 @@ export function recordContext(context: NxTaskContext): BoundedExecContext {
|
|
|
227
275
|
}
|
|
228
276
|
|
|
229
277
|
/**
|
|
230
|
-
* Run the command
|
|
231
|
-
*
|
|
232
|
-
* granted: a wait for the host is not the command's time.
|
|
278
|
+
* Run the command, sized to the tokens its parent's CPU pool (`cpu-pool.ts`) lends it; a run that inherits no pool
|
|
279
|
+
* runs it as asked. Both bounds start once it is granted: a wait for the pool is not the command's time.
|
|
233
280
|
*
|
|
234
|
-
* `inherited` is the parent's
|
|
235
|
-
*
|
|
236
|
-
*
|
|
237
|
-
*
|
|
238
|
-
* its own exit, and the run says so.
|
|
281
|
+
* `inherited` is the parent's pool: an enclosing bounded-exec's sub-pool or the Nx run's own (`run-cpu-pool.ts`). A
|
|
282
|
+
* granted run serves a sub-pool of its grant to its own descendants, closed before the grant returns. A grant whose
|
|
283
|
+
* connection closes while the command runs (the lending parent's exit) ends its accounting, never the command: the
|
|
284
|
+
* command runs on over budget to its own exit, and the run says so.
|
|
239
285
|
*/
|
|
240
286
|
export async function runBoundedExec(
|
|
241
287
|
options: BoundedExecOptions,
|
|
242
288
|
context: BoundedExecContext,
|
|
243
289
|
killer: ProcessTreeKiller,
|
|
244
290
|
tempVolume: TempVolume | null,
|
|
245
|
-
budget: CpuBudget | null,
|
|
246
291
|
inherited: InheritedCpuPool | null = null,
|
|
247
292
|
captureDiagnostics: typeof captureHangDiagnostics = captureHangDiagnostics,
|
|
293
|
+
owner: SubtreeOwner | null = null,
|
|
248
294
|
): Promise<BoundedExecResult> {
|
|
249
295
|
const cwd = resolveCwd(options.cwd, context.root);
|
|
250
296
|
const directory = context.record?.directory ?? null;
|
|
@@ -256,13 +302,19 @@ export async function runBoundedExec(
|
|
|
256
302
|
const timeoutMs = options.timeoutMs;
|
|
257
303
|
const idleTimeoutMs = options.idleTimeoutMs;
|
|
258
304
|
const killAfterMs = options.killAfterMs ?? DEFAULT_KILL_AFTER_MS;
|
|
305
|
+
const fixture: { command: string; timeoutMs: number } | null =
|
|
306
|
+
options.fixtureCommand !== undefined && options.fixtureTimeoutMs !== undefined
|
|
307
|
+
? { command: options.fixtureCommand, timeoutMs: options.fixtureTimeoutMs }
|
|
308
|
+
: null;
|
|
259
309
|
const outputChunks: string[] = [];
|
|
260
310
|
const state: RunState = {
|
|
261
311
|
settled: false,
|
|
262
312
|
expiry: null,
|
|
263
313
|
forceKillNeeded: false,
|
|
264
314
|
poolCloseError: null,
|
|
315
|
+
retireError: null,
|
|
265
316
|
noSpace: null,
|
|
317
|
+
lockWaits: [],
|
|
266
318
|
};
|
|
267
319
|
const appendStdout = (chunk: Buffer | string): void => {
|
|
268
320
|
const text = chunk.toString();
|
|
@@ -278,10 +330,62 @@ export async function runBoundedExec(
|
|
|
278
330
|
outputChunks.push(text);
|
|
279
331
|
process.stderr.write(text);
|
|
280
332
|
};
|
|
333
|
+
// Told, never thrown, wherever the command is being ended: a diagnostic must neither replace the failure it
|
|
334
|
+
// accompanies nor skip the cleanup that follows (the signal handlers, the CPU grant, the RAM lease, the run's record).
|
|
335
|
+
const tell = (text: string): void => {
|
|
336
|
+
try {
|
|
337
|
+
appendStderr(text);
|
|
338
|
+
} catch {
|
|
339
|
+
// Nobody is left to tell.
|
|
340
|
+
}
|
|
341
|
+
};
|
|
342
|
+
|
|
343
|
+
// The fixture pair names one segment: a fixture with no build budget, or a budget with no fixture, is a
|
|
344
|
+
// target that does not say what it runs, so it does not run.
|
|
345
|
+
if (
|
|
346
|
+
(options.fixtureCommand === undefined) !== (options.fixtureTimeoutMs === undefined) ||
|
|
347
|
+
(fixture !== null && !(fixture.timeoutMs > 0))
|
|
348
|
+
) {
|
|
349
|
+
appendStderr(
|
|
350
|
+
'bounded-exec: fixtureCommand and a positive fixtureTimeoutMs must be named together; refusing to run the command\n',
|
|
351
|
+
);
|
|
352
|
+
if (context.record !== undefined) {
|
|
353
|
+
const { task, directory: recordDirectory } = context.record;
|
|
354
|
+
await writeRecord(recordDirectory, { task: task.id, hash: task.hash, verdict: judgeOperationalFailure(1, null) });
|
|
355
|
+
}
|
|
356
|
+
return { success: false, terminalOutput: outputChunks.join('') };
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
// A leg that runs bun runs the Bun its workspace declares, or it does not run. This comes before the RAM lease and the
|
|
360
|
+
// CPU tokens: a refusal that waited in its pool's queue first would say nothing for minutes.
|
|
361
|
+
const runtime = await checkRuntimePin({
|
|
362
|
+
root: context.root,
|
|
363
|
+
command: fixture === null ? requested : `${fixture.command} && ${requested}`,
|
|
364
|
+
env: { ...withoutPoolEnv(process.env), ...options.env },
|
|
365
|
+
cwd,
|
|
366
|
+
pathSource: options.env?.PATH === undefined ? 'the inherited environment' : "the target's env",
|
|
367
|
+
});
|
|
368
|
+
if (runtime !== null) {
|
|
369
|
+
appendStderr(describeRuntimePinRefusal(runtime, context.record?.task.id));
|
|
370
|
+
if (context.record !== undefined) {
|
|
371
|
+
const { task, directory: recordDirectory } = context.record;
|
|
372
|
+
await writeRecord(recordDirectory, { task: task.id, hash: task.hash, verdict: judgeOperationalFailure(1, null) });
|
|
373
|
+
}
|
|
374
|
+
return { success: false, terminalOutput: outputChunks.join('') };
|
|
375
|
+
}
|
|
281
376
|
|
|
282
377
|
let lease: RamTempLease | null = null;
|
|
283
|
-
|
|
284
|
-
|
|
378
|
+
let ordinaryTemp: string | null = null;
|
|
379
|
+
// Before spawn there is nothing to retire. Once started, only the owner's terminal retirement permits deletion.
|
|
380
|
+
let subtreeRetired = true;
|
|
381
|
+
// A lease is removed when its command ends, so it is taken only where whatever the command leaves
|
|
382
|
+
// in it can be ended by its owner first: the supervisor's subtree retirement, never a census.
|
|
383
|
+
if (tempVolume !== null && owner === null) {
|
|
384
|
+
appendStderr(
|
|
385
|
+
`RAM temp volume not leased: no cowshed supervisor owns this command's processes (${SUPERVISOR_SOCKET_ENV} is unset); TMPDIR stays ${process.env.TMPDIR ?? '(unset)'}\n`,
|
|
386
|
+
);
|
|
387
|
+
} else if (tempVolume !== null && owner !== null) {
|
|
388
|
+
const acquired = await tempVolume.acquire(owner);
|
|
285
389
|
if (!acquired.ok) {
|
|
286
390
|
const message = `${describeRamTempError(acquired.error)}\n`;
|
|
287
391
|
process.stderr.write(message);
|
|
@@ -293,7 +397,28 @@ export async function runBoundedExec(
|
|
|
293
397
|
appendStderr(`RAM temp volume: dead lease ${held}\n`);
|
|
294
398
|
}
|
|
295
399
|
for (const reaped of acquired.value.reaped) {
|
|
296
|
-
appendStderr(`RAM temp volume: stopped ${reaped}, which a dead task left
|
|
400
|
+
appendStderr(`RAM temp volume: the supervisor stopped ${reaped}, which a dead task's command left running\n`);
|
|
401
|
+
}
|
|
402
|
+
for (const gap of acquired.value.gaps) {
|
|
403
|
+
appendStderr(`RAM temp volume: the supervisor's view of a dead task's command has a gap: ${gap}\n`);
|
|
404
|
+
}
|
|
405
|
+
for (const retained of acquired.value.retained) {
|
|
406
|
+
appendStderr(`RAM temp volume: dead lease ${retained}\n`);
|
|
407
|
+
}
|
|
408
|
+
} else if (acquired.value.kind === 'unavailable') {
|
|
409
|
+
const unavailable = `RAM temp volume unavailable (${acquired.value.reason}: ${acquired.value.detail})`;
|
|
410
|
+
if (options.env?.TMPDIR !== undefined) {
|
|
411
|
+
appendStderr(`${unavailable}; using target-provided TMPDIR ${options.env.TMPDIR} (medium not inferred)\n`);
|
|
412
|
+
} else {
|
|
413
|
+
try {
|
|
414
|
+
ordinaryTemp = await mkdtemp(join(tempVolume.ordinaryParent, `smoo-bounded-exec-${process.pid}-`));
|
|
415
|
+
} catch (error) {
|
|
416
|
+
appendStderr(
|
|
417
|
+
`${unavailable}; could not create ordinary scratch under ${tempVolume.ordinaryParent}: ${error instanceof Error ? error.message : String(error)}\n`,
|
|
418
|
+
);
|
|
419
|
+
return { success: false, terminalOutput: outputChunks.join('') };
|
|
420
|
+
}
|
|
421
|
+
appendStderr(`${unavailable}; using owned ordinary TMPDIR ${ordinaryTemp}\n`);
|
|
297
422
|
}
|
|
298
423
|
} else {
|
|
299
424
|
appendStderr(
|
|
@@ -301,71 +426,125 @@ export async function runBoundedExec(
|
|
|
301
426
|
);
|
|
302
427
|
}
|
|
303
428
|
}
|
|
304
|
-
|
|
305
|
-
|
|
429
|
+
// The supervisor's answer to the command's own admission, once it is spawned as an owned subtree root, and once
|
|
430
|
+
// any lease has recorded the handle for whoever settles that lease should this task die first.
|
|
431
|
+
let admission: Promise<SubtreeAdmission> | null = null;
|
|
432
|
+
const retireSubtree = async (): Promise<void> => {
|
|
433
|
+
if (admission === null || owner === null) {
|
|
306
434
|
return;
|
|
307
435
|
}
|
|
308
|
-
const
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
appendStderr(`RAM temp volume: stopped ${leftover}, which the command left working in its lease\n`);
|
|
316
|
-
}
|
|
317
|
-
} else if (reaped) {
|
|
318
|
-
appendStderr(`${describeRamTempError(reaped.error)}\n`);
|
|
436
|
+
const answer = await admission;
|
|
437
|
+
admission = null;
|
|
438
|
+
if (answer.kind !== 'admitted') {
|
|
439
|
+
appendStderr(
|
|
440
|
+
`cowshed: the command ran without an owned subtree (${answer.kind === 'refused' ? `supervisor: ${answer.reason}` : answer.detail}); nothing retires what it left running\n`,
|
|
441
|
+
);
|
|
442
|
+
return;
|
|
319
443
|
}
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
444
|
+
// What the command left running is a leak, whichever way the command ended: its supervisor ends exactly the
|
|
445
|
+
// processes its tree records under the command, before any lease goes, and each is said so its owner fixes it.
|
|
446
|
+
const retired = await owner.retire(answer.handle, LEFTOVER_GRACE_MS);
|
|
447
|
+
switch (retired.kind) {
|
|
448
|
+
case 'retired':
|
|
449
|
+
subtreeRetired = true;
|
|
450
|
+
for (const entry of retired.signalled) {
|
|
451
|
+
appendStderr(`cowshed: the supervisor stopped ${describeSignalled(entry)}, which the command left running\n`);
|
|
452
|
+
}
|
|
453
|
+
if (retired.signalledIncomplete) {
|
|
454
|
+
appendStderr("cowshed: the supervisor may have stopped processes of the command's that it could not name\n");
|
|
455
|
+
}
|
|
456
|
+
for (const gap of retired.gaps) {
|
|
457
|
+
appendStderr(
|
|
458
|
+
`cowshed: the supervisor's view of the command's processes has a gap: ${describeCoverageGap(gap)}\n`,
|
|
459
|
+
);
|
|
460
|
+
}
|
|
461
|
+
return;
|
|
462
|
+
case 'refused':
|
|
463
|
+
appendStderr(`cowshed: the supervisor refused to retire the command's processes (${retired.reason})\n`);
|
|
464
|
+
return;
|
|
465
|
+
case 'failed':
|
|
466
|
+
state.retireError = retired.detail;
|
|
467
|
+
appendStderr(`cowshed: could not retire the command's processes: ${retired.detail}\n`);
|
|
468
|
+
return;
|
|
323
469
|
}
|
|
324
470
|
};
|
|
471
|
+
// Failed scratch remains evidence: RAM leases are archived; ordinary scratch is retained at its reported path.
|
|
472
|
+
let tempReleased: Promise<void> | null = null;
|
|
473
|
+
const releaseLease = (keep: boolean): Promise<void> => {
|
|
474
|
+
tempReleased ??= (async () => {
|
|
475
|
+
await retireSubtree();
|
|
476
|
+
if (ordinaryTemp !== null) {
|
|
477
|
+
const held = ordinaryTemp;
|
|
478
|
+
ordinaryTemp = null;
|
|
479
|
+
if (keep || state.retireError !== null || !subtreeRetired) {
|
|
480
|
+
appendStderr(
|
|
481
|
+
`Ordinary temp directory retained at ${held}: ${!subtreeRetired ? 'the command subtree was not retired' : 'the run did not pass'}\n`,
|
|
482
|
+
);
|
|
483
|
+
} else {
|
|
484
|
+
try {
|
|
485
|
+
await rm(held, { recursive: true, force: true });
|
|
486
|
+
} catch (error) {
|
|
487
|
+
appendStderr(
|
|
488
|
+
`Ordinary temp directory retained at ${held}: cleanup failed: ${error instanceof Error ? error.message : String(error)}\n`,
|
|
489
|
+
);
|
|
490
|
+
}
|
|
491
|
+
}
|
|
492
|
+
}
|
|
493
|
+
if (lease === null || owner === null) {
|
|
494
|
+
return;
|
|
495
|
+
}
|
|
496
|
+
const held = lease;
|
|
497
|
+
lease = null;
|
|
498
|
+
const released = await tempVolume?.release(held, keep || state.retireError !== null, owner);
|
|
499
|
+
if (released?.ok) {
|
|
500
|
+
for (const retained of released.value) {
|
|
501
|
+
appendStderr(`RAM temp volume: lease ${retained}\n`);
|
|
502
|
+
}
|
|
503
|
+
} else if (released) {
|
|
504
|
+
appendStderr(`${describeRamTempError(released.error)}\n`);
|
|
505
|
+
}
|
|
506
|
+
})();
|
|
507
|
+
return tempReleased;
|
|
508
|
+
};
|
|
325
509
|
|
|
326
|
-
// The wait is one span line, as cowshed's own spans read.
|
|
327
|
-
//
|
|
510
|
+
// The wait is one span line, as cowshed's own spans read. Every Nx run that inherits no pool serves its own
|
|
511
|
+
// (`run-cpu-pool.ts`), so a task takes its tokens from a parent's pool; a command that inherits none runs exactly as
|
|
328
512
|
// asked and nothing is said.
|
|
329
513
|
const cores = availableParallelism();
|
|
330
514
|
const demand = runnerDemand(requested, options.parallelism, cores, process.env, options.env);
|
|
331
515
|
const asked = Date.now();
|
|
332
|
-
// A
|
|
333
|
-
// A pool
|
|
334
|
-
// relay to the host -- so asking the host beside it would count the run twice; the run fails instead.
|
|
516
|
+
// A pool that answered and refused is a live budget, and running beside it would count the run twice: the run fails
|
|
517
|
+
// instead. A pool whose endpoint is gone (its holder exited) lends nothing, and the run goes on as asked.
|
|
335
518
|
let grant: GrantedTokens | null = null;
|
|
336
519
|
let parentRefusal: string | null = null;
|
|
520
|
+
let parentWait: CpuWaitRow | undefined;
|
|
337
521
|
if (inherited !== null) {
|
|
338
522
|
const fromParent = await takeParentCpuTokens(inherited, demand.want);
|
|
339
523
|
if (fromParent.granted) {
|
|
340
524
|
grant = fromParent;
|
|
341
525
|
} else if (fromParent.cause === 'absent') {
|
|
342
|
-
appendStderr(`cowshed: cpu-
|
|
526
|
+
appendStderr(`cowshed: cpu-pool parent pool absent: ${fromParent.reason}; running the command as asked\n`);
|
|
343
527
|
} else {
|
|
344
528
|
parentRefusal = fromParent.reason;
|
|
345
|
-
|
|
346
|
-
}
|
|
347
|
-
let unbudgeted: string | null = null;
|
|
348
|
-
if (grant === null && parentRefusal === null && budget !== null) {
|
|
349
|
-
const fromHost = await budget.take(demand.want, context.root, requested);
|
|
350
|
-
if (fromHost.granted) {
|
|
351
|
-
grant = fromHost;
|
|
352
|
-
} else if (fromHost.cause !== 'absent') {
|
|
353
|
-
unbudgeted = fromHost.reason;
|
|
529
|
+
parentWait = fromParent.wait;
|
|
354
530
|
}
|
|
355
531
|
}
|
|
356
532
|
const waitedMs = Date.now() - asked;
|
|
357
533
|
let cpu: BoundedExecRecord['cpu'];
|
|
358
534
|
let sized: { command: string; env: Record<string, string> } = { command: requested, env: {} };
|
|
535
|
+
// With mbx the leaf's admission is mbx's one pool, not a parent's tokens: its runner is sized to the job's count,
|
|
536
|
+
// and a non-Cargo leaf holds that much of the pool for its run (specs/cowshed/05_gateway.md, "Parallelism inside a
|
|
537
|
+
// job"). No grant and no sub-pool beside it.
|
|
538
|
+
const mbx = mbxPool(process.env);
|
|
539
|
+
const leaf = mbx === null ? null : mbxLeaf(mbx, demand.kind, demand.want);
|
|
359
540
|
if (grant !== null) {
|
|
360
541
|
sized = sizeRunner(demand, requested, grant.tokens);
|
|
361
|
-
cpu = { tokens: grant.tokens, want: demand.want, waitedMs
|
|
362
|
-
appendStderr(
|
|
363
|
-
`cowshed: cpu-tokens wait done elapsed=${waitedMs}ms status=ok ${grant.inherited ? 'inherited=true ' : ''}tokens=${grant.tokens}/${demand.want} runner=${demand.kind}\n`,
|
|
364
|
-
);
|
|
365
|
-
} else if (unbudgeted !== null) {
|
|
542
|
+
cpu = { tokens: grant.tokens, want: demand.want, waitedMs };
|
|
366
543
|
appendStderr(
|
|
367
|
-
`cowshed: cpu-
|
|
544
|
+
`cowshed: cpu-pool wait done elapsed=${waitedMs}ms status=ok tokens=${grant.tokens}/${demand.want} runner=${demand.kind}\n`,
|
|
368
545
|
);
|
|
546
|
+
} else if (leaf !== null) {
|
|
547
|
+
sized = sizeRunner(demand, requested, demand.want);
|
|
369
548
|
}
|
|
370
549
|
// The run's sub-pool and grant are given back exactly once, however the run ends. The sub-pool closes first, so no
|
|
371
550
|
// descendant still draws on tokens the grant has returned.
|
|
@@ -386,14 +565,15 @@ export async function runBoundedExec(
|
|
|
386
565
|
return cpuReleased;
|
|
387
566
|
};
|
|
388
567
|
// Records the run's verdict when it has a task to record as. An `expiry` is set only when a bound ended the run.
|
|
389
|
-
// An `operational` run failed on its CPU
|
|
390
|
-
//
|
|
568
|
+
// An `operational` run failed on its CPU pool (a sub-pool that did not close): an ordinary failure, whatever bound also
|
|
569
|
+
// fired, so it is never judged as a bound.
|
|
391
570
|
const recordRun = async (
|
|
392
571
|
exitCode: number,
|
|
393
572
|
expiry: ExpiredBound | null,
|
|
394
573
|
elapsedMs: number,
|
|
395
574
|
operational: boolean,
|
|
396
575
|
noSpace: NoSpaceSighting | null,
|
|
576
|
+
lockWaits: readonly LockWait[],
|
|
397
577
|
): Promise<void> => {
|
|
398
578
|
if (context.record === undefined) {
|
|
399
579
|
return;
|
|
@@ -402,46 +582,35 @@ export async function runBoundedExec(
|
|
|
402
582
|
const report = await readJunitReport(recordDirectory);
|
|
403
583
|
const verdict = operational
|
|
404
584
|
? judgeOperationalFailure(exitCode, report)
|
|
405
|
-
: judgeRun({ exitCode, elapsedMs, expiry, report, noSpace });
|
|
585
|
+
: judgeRun({ exitCode, elapsedMs, expiry, report, noSpace, lockWaits });
|
|
406
586
|
if (verdict.outcome === 'bound' && verdict.bound === 'test') {
|
|
407
587
|
appendStderr(`Every failing test failed only on its per-test timeout: ${verdict.tests.join(', ')}\n`);
|
|
408
588
|
}
|
|
409
|
-
await writeRecord(recordDirectory, {
|
|
589
|
+
await writeRecord(recordDirectory, {
|
|
590
|
+
task: task.id,
|
|
591
|
+
hash: task.hash,
|
|
592
|
+
verdict,
|
|
593
|
+
...(cpu ? { cpu } : {}),
|
|
594
|
+
...(parentWait ? { cpuWait: parentWait } : {}),
|
|
595
|
+
});
|
|
410
596
|
};
|
|
411
597
|
// A run refused before its command starts says why, gives back its grant and RAM, and records the failure.
|
|
412
598
|
const refuse = async (message: string): Promise<BoundedExecResult> => {
|
|
413
599
|
appendStderr(message);
|
|
414
600
|
await releaseCpu();
|
|
415
|
-
|
|
416
|
-
await
|
|
601
|
+
// The command never started, so its lease holds nothing of its own to keep.
|
|
602
|
+
await releaseLease(false);
|
|
603
|
+
await recordRun(1, null, Date.now() - asked, false, null, []);
|
|
417
604
|
return { success: false, terminalOutput: outputChunks.join('') };
|
|
418
605
|
};
|
|
419
606
|
if (parentRefusal !== null) {
|
|
420
|
-
return refuse(
|
|
421
|
-
`cowshed: cpu-tokens parent pool refused: ${parentRefusal}; refusing to run the command without asking the host budget\n`,
|
|
422
|
-
);
|
|
607
|
+
return refuse(`cowshed: cpu-pool parent pool refused: ${parentRefusal}; refusing to run the command\n`);
|
|
423
608
|
}
|
|
424
609
|
|
|
425
|
-
// Descendant work queued for host CPU tokens, as its pool hears it (`host-wait`): while any is held the idle clock
|
|
426
|
-
// is stopped, because a queued job is not an idle one (specs/cowshed/05_gateway.md, "The bounds start at the
|
|
427
|
-
// grant"). The clock it stops is made with the command below; a wait held before then is counted and honoured there.
|
|
428
|
-
let heldWaits = 0;
|
|
429
|
-
let idleClock: { stop(): void; restart(): void } | null = null;
|
|
430
|
-
const hostWaits: HostWaits = {
|
|
431
|
-
begin: () => {
|
|
432
|
-
heldWaits += 1;
|
|
433
|
-
if (heldWaits === 1) idleClock?.stop();
|
|
434
|
-
},
|
|
435
|
-
end: () => {
|
|
436
|
-
heldWaits -= 1;
|
|
437
|
-
if (heldWaits === 0) idleClock?.restart();
|
|
438
|
-
},
|
|
439
|
-
};
|
|
440
|
-
|
|
441
610
|
// A granted run serves a sub-pool of its own grant, so its descendants draw from that and never from above it.
|
|
442
|
-
// A pool that cannot be set up refuses the run: no command runs
|
|
611
|
+
// A pool that cannot be set up refuses the run: no granted command runs beside its grant.
|
|
443
612
|
if (grant !== null) {
|
|
444
|
-
const created = await createCpuPool(grant.tokens, grant.closed
|
|
613
|
+
const created = await createCpuPool(grant.tokens, grant.closed);
|
|
445
614
|
if (!created.ok) {
|
|
446
615
|
return refuse(`cowshed: cpu pool unavailable: ${created.error}; refusing to run the command\n`);
|
|
447
616
|
}
|
|
@@ -461,25 +630,62 @@ export async function runBoundedExec(
|
|
|
461
630
|
...sized.env,
|
|
462
631
|
...(pool?.environment ?? {}),
|
|
463
632
|
};
|
|
464
|
-
|
|
633
|
+
// A fixture rides as segment 0 of the lifecycle. Without the channel (Windows) it is plain shell text ahead
|
|
634
|
+
// of the command: the progress bound runs during it and its own ceiling does not.
|
|
635
|
+
const hooked =
|
|
636
|
+
process.platform === 'win32'
|
|
637
|
+
? null
|
|
638
|
+
: fixture === null
|
|
639
|
+
? withRunnerLifecycle(command)
|
|
640
|
+
: withFixtureLifecycle(fixture.command, command);
|
|
465
641
|
const lifecycle = hooked === null ? null : createRunnerLifecycle(hooked.runners);
|
|
466
642
|
let lifecycleFailure: string | null = null;
|
|
467
643
|
const isTesting = (): boolean => lifecycleFailure !== null || (lifecycle?.isTesting() ?? true);
|
|
468
644
|
const startedAt = Date.now();
|
|
645
|
+
// With a supervisor, `cowshed subtree-root` asks it for a handle on itself, writes the answer to the handle
|
|
646
|
+
// descriptor, and execs `/bin/sh -c command` in the same process: the command's pid, group and life are unchanged.
|
|
647
|
+
const unadmitted = hooked?.command ?? (fixture === null ? command : `${fixture.command} && ${command}`);
|
|
648
|
+
const shellCommand = leaf === null ? unadmitted : withMbxLeaf(unadmitted, leaf);
|
|
649
|
+
const root = owner?.rootArgv(shellCommand);
|
|
469
650
|
let child: ChildProcess;
|
|
470
651
|
try {
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
652
|
+
// fd 3 is the runner lifecycle's duplex socket when the command is hooked; the subtree admission takes fd 4, which
|
|
653
|
+
// `cowshed subtree-root` closes before it execs, so fd 3 reaches the shell unchanged.
|
|
654
|
+
child =
|
|
655
|
+
root === undefined
|
|
656
|
+
? spawn(shellCommand, [], {
|
|
657
|
+
cwd,
|
|
658
|
+
env: mergeEnv(lease?.directory ?? ordinaryTemp, env),
|
|
659
|
+
shell: true,
|
|
660
|
+
stdio: hooked === null ? 'pipe' : ['pipe', 'pipe', 'pipe', 'pipe'],
|
|
661
|
+
detached: process.platform !== 'win32',
|
|
662
|
+
windowsHide: true,
|
|
663
|
+
})
|
|
664
|
+
: spawn(root.file, root.args, {
|
|
665
|
+
cwd,
|
|
666
|
+
env: mergeEnv(lease?.directory ?? ordinaryTemp, env),
|
|
667
|
+
stdio: ['pipe', 'pipe', 'pipe', hooked === null ? 'ignore' : 'pipe', 'pipe'],
|
|
668
|
+
detached: process.platform !== 'win32',
|
|
669
|
+
windowsHide: true,
|
|
670
|
+
});
|
|
479
671
|
} catch (error) {
|
|
480
672
|
// Nothing started, so nothing runs under the grant: it goes back here, with the RAM lease.
|
|
481
673
|
return refuse(`cowshed: could not start the command: ${error instanceof Error ? error.message : String(error)}\n`);
|
|
482
674
|
}
|
|
675
|
+
subtreeRetired = false;
|
|
676
|
+
if (root !== undefined) {
|
|
677
|
+
const channel = child.stdio[HANDLE_FD];
|
|
678
|
+
const leased = lease;
|
|
679
|
+
admission = readAdmission(channel instanceof Readable ? channel : null).then(async (answer) => {
|
|
680
|
+
if (answer.kind === 'admitted' && leased !== null && tempVolume !== null) {
|
|
681
|
+
const recorded = await tempVolume.record(leased, answer.handle);
|
|
682
|
+
if (!recorded.ok) {
|
|
683
|
+
appendStderr(`${describeRamTempError(recorded.error)}\n`);
|
|
684
|
+
}
|
|
685
|
+
}
|
|
686
|
+
return answer;
|
|
687
|
+
});
|
|
688
|
+
}
|
|
483
689
|
|
|
484
690
|
// Progress is a finished test, not output: a runner's SLOW warnings and a hung test's heartbeat keep printing.
|
|
485
691
|
const progress = createRunProgress(command);
|
|
@@ -490,24 +696,22 @@ export async function runBoundedExec(
|
|
|
490
696
|
const held = lease;
|
|
491
697
|
return { line, volume: held === null ? null : ((await tempVolume?.space(held)) ?? null) };
|
|
492
698
|
};
|
|
493
|
-
|
|
699
|
+
// Which bound is armed, and when it expires, is bounds.ts's decision; one timer waits for the earliest of them.
|
|
700
|
+
let bounds: Bounds = UNARMED;
|
|
701
|
+
let boundTimer: NodeJS.Timeout | undefined;
|
|
702
|
+
// The fixture's own output is compilation, not the command's progress: it is not scanned for finished tests.
|
|
703
|
+
let fixtureDone = fixture === null || lifecycle === null;
|
|
704
|
+
// Whether a segment past the fixture ever announced: without one, the shell's exit is the fixture's.
|
|
705
|
+
let mainStarted = fixture === null || lifecycle === null;
|
|
494
706
|
let captureTimer: NodeJS.Timeout | undefined;
|
|
495
707
|
let idleCapture: { expiresAtMs: number; controller: AbortController; result: IdleDiagnosticResult } | undefined;
|
|
496
708
|
const cancelCapture = (): void => {
|
|
497
709
|
clearTimeout(captureTimer);
|
|
498
710
|
idleCapture?.controller.abort();
|
|
499
711
|
};
|
|
500
|
-
|
|
501
|
-
// wait keeps it unspent: the clock is stopped while descendant work is queued for CPU tokens, and runs on from where
|
|
502
|
-
// it stopped once the last wait ends. `queuedMs` is the queued time this interval has not counted, for the report.
|
|
503
|
-
let idleLeftMs = idleTimeoutMs ?? 0;
|
|
504
|
-
let queuedMs = 0;
|
|
505
|
-
let queuedSinceMs: number | null = null;
|
|
506
|
-
const scheduleIdle = (limitMs: number, leftMs: number): void => {
|
|
507
|
-
clearTimeout(idleTimer);
|
|
508
|
-
cancelCapture();
|
|
712
|
+
const scheduleCapture = (expiresAtMs: number): void => {
|
|
509
713
|
const capture: NonNullable<typeof idleCapture> = {
|
|
510
|
-
expiresAtMs
|
|
714
|
+
expiresAtMs,
|
|
511
715
|
controller: new AbortController(),
|
|
512
716
|
result: { kind: 'pending', startedAtMs: null },
|
|
513
717
|
};
|
|
@@ -545,70 +749,45 @@ export async function runBoundedExec(
|
|
|
545
749
|
},
|
|
546
750
|
);
|
|
547
751
|
},
|
|
548
|
-
Math.max(0,
|
|
752
|
+
Math.max(0, expiresAtMs - Date.now() - HANG_CAPTURE_MS),
|
|
549
753
|
);
|
|
550
|
-
idleTimer = setTimeout(() => expire({ kind: 'idle', limitMs, tests: progress.inFlight() }, queuedMs), leftMs);
|
|
551
754
|
};
|
|
552
|
-
const
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
if (idleCapture !== undefined) {
|
|
572
|
-
idleLeftMs = Math.max(0, idleCapture.expiresAtMs - now);
|
|
573
|
-
}
|
|
574
|
-
clearTimeout(idleTimer);
|
|
755
|
+
const expiredAs = (kind: BoundKind): ExpiredBound =>
|
|
756
|
+
kind === 'idle'
|
|
757
|
+
? { kind, limitMs: idleTimeoutMs ?? 0, tests: progress.inFlight() }
|
|
758
|
+
: kind === 'fixture'
|
|
759
|
+
? { kind, limitMs: fixture?.timeoutMs ?? 0 }
|
|
760
|
+
: { kind, limitMs: timeoutMs ?? 0 };
|
|
761
|
+
const stepTo = (event: BoundEvent): void => {
|
|
762
|
+
if (state.settled || state.expiry !== null) return;
|
|
763
|
+
const idleBefore = bounds.idle;
|
|
764
|
+
bounds = stepBounds(
|
|
765
|
+
bounds,
|
|
766
|
+
{
|
|
767
|
+
...(idleTimeoutMs === undefined ? {} : { idleMs: idleTimeoutMs }),
|
|
768
|
+
...(fixture === null || lifecycle === null ? {} : { fixtureMs: fixture.timeoutMs }),
|
|
769
|
+
...(timeoutMs === undefined ? {} : { totalMs: timeoutMs }),
|
|
770
|
+
},
|
|
771
|
+
event,
|
|
772
|
+
);
|
|
773
|
+
if (bounds.idle !== idleBefore) {
|
|
575
774
|
cancelCapture();
|
|
576
775
|
idleCapture = undefined;
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
if (idleTimeoutMs === undefined || state.settled || state.expiry !== null || queuedSinceMs === null) {
|
|
583
|
-
return;
|
|
584
|
-
}
|
|
585
|
-
const waitedMs = Date.now() - queuedSinceMs;
|
|
586
|
-
queuedMs += waitedMs;
|
|
587
|
-
queuedSinceMs = null;
|
|
588
|
-
if (!isTesting()) {
|
|
589
|
-
appendStderr(`bounded-exec: host CPU wait ended after ${waitedMs}ms; no test runner is active\n`);
|
|
590
|
-
return;
|
|
591
|
-
}
|
|
592
|
-
appendStderr(
|
|
593
|
-
`bounded-exec: idle clock restarted: no descendant's work is queued for host CPU tokens after ${waitedMs}ms; ${idleLeftMs}ms of idleTimeoutMs=${idleTimeoutMs} left\n`,
|
|
594
|
-
);
|
|
595
|
-
scheduleIdle(idleTimeoutMs, idleLeftMs);
|
|
596
|
-
},
|
|
776
|
+
if (bounds.idle !== null) scheduleCapture(bounds.idle);
|
|
777
|
+
}
|
|
778
|
+
clearTimeout(boundTimer);
|
|
779
|
+
const next = earliestBound(bounds);
|
|
780
|
+
if (next !== null) boundTimer = setTimeout(() => expire(expiredAs(next.kind)), Math.max(0, next.at - Date.now()));
|
|
597
781
|
};
|
|
598
782
|
|
|
599
783
|
// Only the CHILD's own output is read for progress. The diagnostics below go
|
|
600
784
|
// through appendStderr as well, and re-arming on those would let a teardown
|
|
601
785
|
// message extend the very bound that just fired.
|
|
602
786
|
const updateIdle = (finished: boolean, wasTesting: boolean): void => {
|
|
603
|
-
|
|
604
|
-
clearTimeout(idleTimer);
|
|
605
|
-
cancelCapture();
|
|
606
|
-
idleCapture = undefined;
|
|
607
|
-
} else if (finished || !wasTesting) {
|
|
608
|
-
armIdleTimer();
|
|
609
|
-
}
|
|
787
|
+
stepTo({ kind: 'progress', at: Date.now(), testing: isTesting(), wasTesting, finished });
|
|
610
788
|
};
|
|
611
789
|
const scanProgress = (stream: 'stdout' | 'stderr', text: string): void => {
|
|
790
|
+
if (!fixtureDone) return;
|
|
612
791
|
updateIdle(progress.scan(stream, text), isTesting());
|
|
613
792
|
};
|
|
614
793
|
if (lifecycle !== null) {
|
|
@@ -620,7 +799,8 @@ export async function runBoundedExec(
|
|
|
620
799
|
appendStderr(`bounded-exec: runner lifecycle failed: ${detail}\n`);
|
|
621
800
|
updateIdle(false, wasTesting);
|
|
622
801
|
};
|
|
623
|
-
createInterface({ input: channel })
|
|
802
|
+
const lines = createInterface({ input: channel });
|
|
803
|
+
lines.on('line', (line) => {
|
|
624
804
|
const wasTesting = isTesting();
|
|
625
805
|
const event = lifecycle.accept(line);
|
|
626
806
|
if (event === null) {
|
|
@@ -628,9 +808,17 @@ export async function runBoundedExec(
|
|
|
628
808
|
return;
|
|
629
809
|
}
|
|
630
810
|
updateIdle(false, wasTesting);
|
|
811
|
+
if (event.id === 0 && event.phase === 'terminal' && fixture !== null) {
|
|
812
|
+
fixtureDone = true;
|
|
813
|
+
stepTo({ kind: 'fixture-done' });
|
|
814
|
+
}
|
|
815
|
+
if (event.id > 0) mainStarted = true;
|
|
631
816
|
// Tool startup and native lifecycle-bearing stdout wait for their phase acknowledgement.
|
|
632
817
|
channel.write(`${JSON.stringify(event)}\n`);
|
|
633
818
|
});
|
|
819
|
+
// readline re-emits its input's errors on the interface, where one with no listener is thrown: a
|
|
820
|
+
// runner killed with unread lifecycle bytes resets the channel (ECONNRESET on Linux).
|
|
821
|
+
lines.on('error', (error) => failLifecycle(error.message));
|
|
634
822
|
channel.on('error', (error) => failLifecycle(error.message));
|
|
635
823
|
channel.on('end', () => {
|
|
636
824
|
if (!lifecycle.finished()) failLifecycle('supervisor channel ended without an owned terminal event');
|
|
@@ -651,23 +839,30 @@ export async function runBoundedExec(
|
|
|
651
839
|
appendStderr(text);
|
|
652
840
|
});
|
|
653
841
|
|
|
654
|
-
|
|
842
|
+
// The graceful pass: SIGTERM to the whole group. A signal that fails is told, not thrown: every caller has more to do
|
|
843
|
+
// after it, `terminate` the force-kill and the signal handler its releases and the signal it gives back.
|
|
844
|
+
const killChildTree = async (): Promise<void> => {
|
|
655
845
|
const pid = child.pid;
|
|
656
846
|
if (!pid) {
|
|
657
847
|
return;
|
|
658
848
|
}
|
|
659
|
-
|
|
849
|
+
try {
|
|
850
|
+
await killer.kill(pid, 'SIGTERM');
|
|
851
|
+
} catch (error) {
|
|
852
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
853
|
+
tell(`Terminating process group ${pid} failed: ${reason}\n`);
|
|
854
|
+
}
|
|
660
855
|
};
|
|
661
856
|
|
|
662
857
|
const onProcessExit = (): void => {
|
|
663
|
-
void killChildTree(
|
|
858
|
+
void killChildTree();
|
|
664
859
|
};
|
|
665
860
|
const onTerminationSignal = (signal: NodeJS.Signals): void => {
|
|
666
861
|
removeSignalHandlers();
|
|
667
862
|
cancelCapture();
|
|
668
|
-
void killChildTree(
|
|
863
|
+
void killChildTree()
|
|
669
864
|
.finally(releaseCpu)
|
|
670
|
-
.finally(releaseLease)
|
|
865
|
+
.finally(() => releaseLease(true))
|
|
671
866
|
.finally(() => process.kill(process.pid, signal));
|
|
672
867
|
};
|
|
673
868
|
const onSigint = (): void => onTerminationSignal('SIGINT');
|
|
@@ -706,28 +901,27 @@ export async function runBoundedExec(
|
|
|
706
901
|
// the reader unable to tell a wedged toolchain from a loaded machine — the
|
|
707
902
|
// two have opposite fixes.
|
|
708
903
|
let termination: Promise<void> = Promise.resolve();
|
|
709
|
-
|
|
710
|
-
const expire = (bound: ExpiredBound, queuedMs = 0): void => {
|
|
904
|
+
const expire = (bound: ExpiredBound): void => {
|
|
711
905
|
if (state.settled || state.expiry !== null) {
|
|
712
906
|
return;
|
|
713
907
|
}
|
|
714
908
|
state.expiry = bound;
|
|
715
|
-
clearTimeout(
|
|
716
|
-
clearTimeout(idleTimer);
|
|
909
|
+
clearTimeout(boundTimer);
|
|
717
910
|
const elapsedMs = Date.now() - startedAt;
|
|
718
911
|
appendStderr(
|
|
719
912
|
bound.kind === 'idle'
|
|
720
|
-
? `\nCommand made no progress: no test finished for ${bound.limitMs}ms (idleTimeoutMs=${bound.limitMs})${
|
|
721
|
-
queuedMs > 0 ? `, not counting ${queuedMs}ms queued for host CPU tokens,` : ''
|
|
722
|
-
} after ${elapsedMs}ms of runtime; ${
|
|
913
|
+
? `\nCommand made no progress: no test finished for ${bound.limitMs}ms (idleTimeoutMs=${bound.limitMs}) after ${elapsedMs}ms of runtime; ${
|
|
723
914
|
bound.tests.length > 0
|
|
724
915
|
? `still running: ${bound.tests.join(', ')}`
|
|
725
916
|
: 'the runner named no test still running'
|
|
726
917
|
} (cwd=${cwd}): ${command}\n`
|
|
727
|
-
:
|
|
918
|
+
: bound.kind === 'fixture'
|
|
919
|
+
? `\nFixture command timed out after ${elapsedMs}ms (fixtureTimeoutMs=${bound.limitMs}, cwd=${cwd}): ${fixture?.command ?? command}\n`
|
|
920
|
+
: `\nCommand timed out after ${elapsedMs}ms (timeoutMs=${bound.limitMs}, cwd=${cwd}): ${command}\n`,
|
|
728
921
|
);
|
|
729
922
|
if (bound.kind === 'idle') {
|
|
730
923
|
const observed = idleCapture?.result;
|
|
924
|
+
state.lockWaits = observed?.kind === 'captured' ? observed.snapshot.lockWaits : [];
|
|
731
925
|
appendStderr(
|
|
732
926
|
`Hang diagnostics: runner pid=${child.pid ?? 'unavailable'} pgid=${
|
|
733
927
|
process.platform === 'win32' ? 'unavailable' : (child.pid ?? 'unavailable')
|
|
@@ -753,24 +947,49 @@ export async function runBoundedExec(
|
|
|
753
947
|
termination = terminate();
|
|
754
948
|
};
|
|
755
949
|
|
|
950
|
+
// Whether the group still lists members. A probe that fails proves nothing about the group, so it is told, and the
|
|
951
|
+
// group is taken as listed: a force-kill that was not needed costs nothing, and one that was is not skipped.
|
|
952
|
+
const groupListed = (pid: number): boolean => {
|
|
953
|
+
try {
|
|
954
|
+
return processGroupHasMembers(pid);
|
|
955
|
+
} catch (error) {
|
|
956
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
957
|
+
tell(`Probing process group ${pid} for members failed: ${reason}; taking it as still listing members\n`);
|
|
958
|
+
return true;
|
|
959
|
+
}
|
|
960
|
+
};
|
|
961
|
+
|
|
756
962
|
// A timed-out run ends with its whole process group, not with its leader.
|
|
757
963
|
// The graceful signal reaches every member that exists when it is sent, but
|
|
758
964
|
// a member that ignores SIGTERM, or one forked while the signal goes out
|
|
759
965
|
// (macOS does not hold a fork back for a group signal in flight), outlives
|
|
760
966
|
// a leader that exits on it. The grace ends when the leader exits or
|
|
761
967
|
// killAfterMs passes, whichever comes first; whatever is left then is killed.
|
|
968
|
+
// SIGKILL is no different: a member forked while it goes out survives it too,
|
|
969
|
+
// so the force-kill is repeated while the group lists members. What the repeats
|
|
970
|
+
// cannot end is said, and a failure of the probe or of a signal is shown.
|
|
971
|
+
const forceKill = async (pid: number): Promise<void> => {
|
|
972
|
+
try {
|
|
973
|
+
if (await killProcessGroup(pid, DEADLINE_PATIENCE, (group) => killer.kill(group, 'SIGKILL'))) {
|
|
974
|
+
tell(`${describeMembersLeft(pid)}\n`);
|
|
975
|
+
}
|
|
976
|
+
} catch (error) {
|
|
977
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
978
|
+
tell(`Force-kill of process group ${pid} failed: ${reason}\n`);
|
|
979
|
+
}
|
|
980
|
+
};
|
|
762
981
|
const terminate = async (): Promise<void> => {
|
|
763
|
-
await
|
|
982
|
+
await killChildTree();
|
|
764
983
|
if (!state.settled && killAfterMs > 0) {
|
|
765
984
|
await settledWithin(exitPromise, killAfterMs);
|
|
766
985
|
}
|
|
767
986
|
if (state.settled) {
|
|
768
|
-
if (child.pid !== undefined &&
|
|
987
|
+
if (child.pid !== undefined && groupListed(child.pid)) {
|
|
769
988
|
state.forceKillNeeded = true;
|
|
770
989
|
appendStderr(
|
|
771
990
|
`Force-killing what the ${state.expiry === null ? 'command' : 'timed-out command'} left in its process group: ${command}\n`,
|
|
772
991
|
);
|
|
773
|
-
await
|
|
992
|
+
await forceKill(child.pid);
|
|
774
993
|
}
|
|
775
994
|
return;
|
|
776
995
|
}
|
|
@@ -778,23 +997,25 @@ export async function runBoundedExec(
|
|
|
778
997
|
appendStderr(
|
|
779
998
|
`Force-killing ${state.expiry === null ? 'command' : 'timed out command'} after killAfterMs=${killAfterMs}: ${command}\n`,
|
|
780
999
|
);
|
|
781
|
-
await
|
|
782
|
-
|
|
1000
|
+
if (child.pid !== undefined) await forceKill(child.pid);
|
|
1001
|
+
// The repeats wait for the group to empty, the leader with it. Windows has no process groups, so there they have
|
|
1002
|
+
// nothing to wait on once the one signal is sent, and the leader's exit is awaited as it always was.
|
|
1003
|
+
if (process.platform === 'win32' || child.pid === undefined) {
|
|
1004
|
+
await settledWithin(exitPromise, REAP_AFTER_FORCE_KILL_MS);
|
|
1005
|
+
}
|
|
783
1006
|
if (!state.settled) {
|
|
784
1007
|
resolveExit({ code: 1, signal: 'SIGKILL' });
|
|
785
1008
|
}
|
|
786
1009
|
};
|
|
787
1010
|
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
// A wait its pool heard before the command started stops the clock it was not yet there to stop.
|
|
792
|
-
if (heldWaits > 0) idleClock.stop();
|
|
1011
|
+
// The fixture's ceiling lives and dies with its segment: the segment's terminal clears it, so it can never
|
|
1012
|
+
// fire on a command the fixture already handed over to.
|
|
1013
|
+
stepTo({ kind: 'spawn', at: Date.now(), testing: isTesting(), fixture: fixture !== null && lifecycle !== null });
|
|
793
1014
|
|
|
794
|
-
// A grant's connection is a ledger entry, not the command's owner
|
|
795
|
-
//
|
|
796
|
-
//
|
|
797
|
-
//
|
|
1015
|
+
// A grant's connection is a ledger entry, not the command's owner: the lending parent's exit ends it, and what runs
|
|
1016
|
+
// under it runs on over budget until it finishes. Its close, while the command runs, is said and changes nothing
|
|
1017
|
+
// else; only a bound or a signal stops the command. Our own release after the command settled is not a loss;
|
|
1018
|
+
// `cpuReleased` marks it.
|
|
798
1019
|
if (grant !== null) {
|
|
799
1020
|
const held = grant;
|
|
800
1021
|
void held.closed.then(() => {
|
|
@@ -802,15 +1023,14 @@ export async function runBoundedExec(
|
|
|
802
1023
|
return;
|
|
803
1024
|
}
|
|
804
1025
|
appendStderr(
|
|
805
|
-
`cowshed: cpu-
|
|
1026
|
+
`cowshed: cpu-pool grant ended: the parent pool closed its ${held.tokens}-token grant while the command ran; it runs on over budget until it ends: ${command}\n`,
|
|
806
1027
|
);
|
|
807
1028
|
});
|
|
808
1029
|
}
|
|
809
1030
|
|
|
810
1031
|
const { code: exitCode, signal: exitSignal } = await exitPromise;
|
|
811
1032
|
await termination;
|
|
812
|
-
clearTimeout(
|
|
813
|
-
clearTimeout(idleTimer);
|
|
1033
|
+
clearTimeout(boundTimer);
|
|
814
1034
|
cancelCapture();
|
|
815
1035
|
removeSignalHandlers();
|
|
816
1036
|
// The runner is gone. Its sub-pool closes first, so no descendant still draws on tokens the grant returns;
|
|
@@ -819,7 +1039,11 @@ export async function runBoundedExec(
|
|
|
819
1039
|
|
|
820
1040
|
const code = exitCode ?? signalToExitCode(exitSignal);
|
|
821
1041
|
if (code !== 0 && state.expiry === null) {
|
|
822
|
-
appendStderr(
|
|
1042
|
+
appendStderr(
|
|
1043
|
+
fixture !== null && !mainStarted
|
|
1044
|
+
? `Fixture command exited with status ${code}: ${fixture.command}\n`
|
|
1045
|
+
: `Command exited with status ${code}: ${command}\n`,
|
|
1046
|
+
);
|
|
823
1047
|
}
|
|
824
1048
|
|
|
825
1049
|
if (state.expiry !== null && !state.forceKillNeeded) {
|
|
@@ -838,13 +1062,12 @@ export async function runBoundedExec(
|
|
|
838
1062
|
appendStderr(`${describeRamTempError(full)}\n`);
|
|
839
1063
|
}
|
|
840
1064
|
}
|
|
841
|
-
await releaseLease();
|
|
842
|
-
|
|
843
|
-
// Broken resource ownership or a lost native lifecycle fails even if a shell branch later exited 0:
|
|
844
|
-
// its verdict is an ordinary operational failure, never a successful or silently unbounded run.
|
|
845
|
-
const operationalFailure = state.poolCloseError !== null || lifecycleFailure !== null;
|
|
846
|
-
await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure, seen);
|
|
1065
|
+
await releaseLease(code !== 0 || state.expiry !== null || state.poolCloseError !== null || lifecycleFailure !== null);
|
|
847
1066
|
|
|
1067
|
+
// Broken resource ownership, a lost native lifecycle, or leftovers that could not be retired fail even if a shell
|
|
1068
|
+
// branch later exited 0: its verdict is an ordinary operational failure, never a successful or silently unbounded run.
|
|
1069
|
+
const operationalFailure = state.poolCloseError !== null || lifecycleFailure !== null || state.retireError !== null;
|
|
1070
|
+
await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure, seen, state.lockWaits);
|
|
848
1071
|
return {
|
|
849
1072
|
success: !operationalFailure && state.expiry === null && code === 0,
|
|
850
1073
|
terminalOutput: outputChunks.join(''),
|
|
@@ -853,45 +1076,12 @@ export async function runBoundedExec(
|
|
|
853
1076
|
|
|
854
1077
|
export function createProcessTreeKiller(): ProcessTreeKiller {
|
|
855
1078
|
return {
|
|
856
|
-
kill(pid, signal) {
|
|
857
|
-
|
|
1079
|
+
async kill(pid, signal) {
|
|
1080
|
+
signalProcessGroup(pid, signal);
|
|
858
1081
|
},
|
|
859
1082
|
};
|
|
860
1083
|
}
|
|
861
1084
|
|
|
862
|
-
function killProcessGroup(pid: number, signal: NodeJS.Signals): Promise<void> {
|
|
863
|
-
try {
|
|
864
|
-
// On POSIX the executor spawns with detached: true, so the child leads a
|
|
865
|
-
// process group of its own and -pid signals all of it at once, including
|
|
866
|
-
// grandchildren that tree-walk libraries miss when parents die and
|
|
867
|
-
// children get reparented to init. Only the group is named: the leader may
|
|
868
|
-
// already be reaped, and its pid reused by an unrelated process, while the
|
|
869
|
-
// group lives on in its other members. Windows has no process groups.
|
|
870
|
-
process.kill(process.platform === 'win32' ? pid : -pid, signal);
|
|
871
|
-
} catch {
|
|
872
|
-
// ESRCH: the group (or the Windows process) has already exited.
|
|
873
|
-
}
|
|
874
|
-
return Promise.resolve();
|
|
875
|
-
}
|
|
876
|
-
|
|
877
|
-
/**
|
|
878
|
-
* Whether any process is left in the group `pgid` leads. A group outlives its
|
|
879
|
-
* reaped leader for as long as one member does, and no process can take its id
|
|
880
|
-
* meanwhile. Windows has no process groups, so there is nothing left to name.
|
|
881
|
-
*/
|
|
882
|
-
function processGroupHasMembers(pgid: number): boolean {
|
|
883
|
-
if (process.platform === 'win32') {
|
|
884
|
-
return false;
|
|
885
|
-
}
|
|
886
|
-
try {
|
|
887
|
-
process.kill(-pgid, 0);
|
|
888
|
-
return true;
|
|
889
|
-
} catch (error) {
|
|
890
|
-
// EPERM: a member exists but runs as another user.
|
|
891
|
-
return error instanceof Error && 'code' in error && error.code === 'EPERM';
|
|
892
|
-
}
|
|
893
|
-
}
|
|
894
|
-
|
|
895
1085
|
function resolveCwd(cwd: string | undefined, root: string): string {
|
|
896
1086
|
if (!cwd) {
|
|
897
1087
|
return root;
|
|
@@ -912,10 +1102,10 @@ function buildCommand(options: BoundedExecOptions): string {
|
|
|
912
1102
|
return parts.join(' ');
|
|
913
1103
|
}
|
|
914
1104
|
|
|
915
|
-
/**
|
|
916
|
-
function mergeEnv(
|
|
1105
|
+
/** Owned scratch is the command's TMPDIR unless the target names one itself. */
|
|
1106
|
+
function mergeEnv(directory: string | null, env: Record<string, string>): NodeJS.ProcessEnv {
|
|
917
1107
|
const inherited = withoutPoolEnv(process.env);
|
|
918
|
-
return
|
|
1108
|
+
return directory === null ? { ...inherited, ...env } : { ...inherited, TMPDIR: directory, ...env };
|
|
919
1109
|
}
|
|
920
1110
|
|
|
921
1111
|
/** The process's environment without the CPU pool's descriptor: a pool and its count only a grant may give. */
|
|
@@ -949,14 +1139,6 @@ function signalToExitCode(signal: NodeJS.Signals | null): number {
|
|
|
949
1139
|
return EXIT_CODE_BY_SIGNAL[signal] ?? 1;
|
|
950
1140
|
}
|
|
951
1141
|
|
|
952
|
-
async function ignoreKillError(promise: Promise<void>): Promise<void> {
|
|
953
|
-
try {
|
|
954
|
-
await promise;
|
|
955
|
-
} catch {
|
|
956
|
-
// The process tree may have already exited between timeout and kill.
|
|
957
|
-
}
|
|
958
|
-
}
|
|
959
|
-
|
|
960
1142
|
/** Resolves when `promise` settles or `ms` passes, whichever is first, leaving no timer behind. */
|
|
961
1143
|
async function settledWithin(promise: Promise<unknown>, ms: number): Promise<void> {
|
|
962
1144
|
let timer: NodeJS.Timeout | undefined;
|