@smoothbricks/nx-plugin 0.5.7 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1086 -360
- package/dist/.build-inputs.json +138 -0
- package/dist/bin/smoo-nx-cargo-toolchain.d.ts +3 -0
- package/dist/bin/smoo-nx-cargo-toolchain.d.ts.map +1 -0
- package/dist/bin/smoo-nx-cargo-toolchain.js +8 -0
- package/dist/bin/smoo-nx-forbidden-spellings.d.ts +3 -0
- package/dist/bin/smoo-nx-forbidden-spellings.d.ts.map +1 -0
- package/dist/bin/smoo-nx-forbidden-spellings.js +32 -0
- package/dist/bin/smoo-nx-nextest-timings.d.ts +3 -0
- package/dist/bin/smoo-nx-nextest-timings.d.ts.map +1 -0
- package/dist/bin/smoo-nx-nextest-timings.js +28 -0
- package/dist/bounded-test-policy.d.ts +1 -1
- package/dist/bounded-test-policy.d.ts.map +1 -1
- package/dist/bounded-test-policy.js +14 -15
- package/dist/build-state-output-policy.d.ts +95 -1
- package/dist/build-state-output-policy.d.ts.map +1 -1
- package/dist/build-state-output-policy.js +368 -30
- package/dist/bun-lock-scope.d.ts +44 -0
- package/dist/bun-lock-scope.d.ts.map +1 -0
- package/dist/bun-lock-scope.js +97 -0
- package/dist/cargo-build-script-env.d.ts +49 -0
- package/dist/cargo-build-script-env.d.ts.map +1 -0
- package/dist/cargo-build-script-env.js +324 -0
- package/dist/cargo-closure-input.d.ts +31 -4
- package/dist/cargo-closure-input.d.ts.map +1 -1
- package/dist/cargo-closure-input.js +66 -6
- package/dist/cargo-retention-hook.d.ts +17 -0
- package/dist/cargo-retention-hook.d.ts.map +1 -0
- package/dist/cargo-retention-hook.js +85 -0
- package/dist/cargo-source-hash.d.ts +25 -7
- package/dist/cargo-source-hash.d.ts.map +1 -1
- package/dist/cargo-source-hash.js +108 -43
- package/dist/cargo-toolchain-paths.d.ts +40 -0
- package/dist/cargo-toolchain-paths.d.ts.map +1 -0
- package/dist/cargo-toolchain-paths.js +84 -0
- package/dist/cargo-toolchain-policy.d.ts +25 -8
- package/dist/cargo-toolchain-policy.d.ts.map +1 -1
- package/dist/cargo-toolchain-policy.js +39 -5
- package/dist/cargo-workspace.d.ts +10 -4
- package/dist/cargo-workspace.d.ts.map +1 -1
- package/dist/cargo-workspace.js +65 -95
- package/dist/cross-check-policy.d.ts +9 -9
- package/dist/cross-check-policy.d.ts.map +1 -1
- package/dist/cross-check-policy.js +11 -11
- package/dist/disk-command-policy.d.ts.map +1 -1
- package/dist/disk-command-policy.js +1 -0
- package/dist/ensure-built-receipt.d.ts +13 -5
- package/dist/ensure-built-receipt.d.ts.map +1 -1
- package/dist/ensure-built-receipt.js +126 -47
- package/dist/ensure-built-witness.d.ts +27 -8
- package/dist/ensure-built-witness.d.ts.map +1 -1
- package/dist/ensure-built-witness.js +45 -15
- package/dist/ensure-built.d.ts.map +1 -1
- package/dist/ensure-built.js +9 -1
- package/dist/eslint-config-closure.d.ts +7 -0
- package/dist/eslint-config-closure.d.ts.map +1 -0
- package/dist/eslint-config-closure.js +346 -0
- package/dist/executors/bounded-exec/bounds.d.ts +56 -0
- package/dist/executors/bounded-exec/bounds.d.ts.map +1 -0
- package/dist/executors/bounded-exec/bounds.js +43 -0
- package/dist/executors/bounded-exec/cpu-pool.d.ts +33 -36
- package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool.js +54 -119
- package/dist/executors/bounded-exec/executor.d.ts +19 -12
- package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
- package/dist/executors/bounded-exec/executor.js +371 -239
- package/dist/executors/bounded-exec/gateway-lease.d.ts +22 -19
- package/dist/executors/bounded-exec/gateway-lease.d.ts.map +1 -1
- package/dist/executors/bounded-exec/gateway-lease.js +81 -28
- package/dist/executors/bounded-exec/hang-diagnostics.d.ts +18 -1
- package/dist/executors/bounded-exec/hang-diagnostics.d.ts.map +1 -1
- package/dist/executors/bounded-exec/hang-diagnostics.js +148 -30
- package/dist/executors/bounded-exec/lease-wait.d.ts +43 -0
- package/dist/executors/bounded-exec/lease-wait.d.ts.map +1 -0
- package/dist/executors/bounded-exec/lease-wait.js +68 -0
- package/dist/executors/bounded-exec/lock-wait.d.ts +51 -0
- package/dist/executors/bounded-exec/lock-wait.d.ts.map +1 -0
- package/dist/executors/bounded-exec/lock-wait.js +108 -0
- package/dist/executors/bounded-exec/owned-subtree.d.ts +84 -0
- package/dist/executors/bounded-exec/owned-subtree.d.ts.map +1 -0
- package/dist/executors/bounded-exec/owned-subtree.js +178 -0
- package/dist/executors/bounded-exec/progress.d.ts +4 -1
- package/dist/executors/bounded-exec/progress.d.ts.map +1 -1
- package/dist/executors/bounded-exec/progress.js +5 -2
- package/dist/executors/bounded-exec/ram-attach.d.ts +19 -0
- package/dist/executors/bounded-exec/ram-attach.d.ts.map +1 -0
- package/dist/executors/bounded-exec/ram-attach.js +207 -0
- package/dist/executors/bounded-exec/ram-detach.d.ts +25 -0
- package/dist/executors/bounded-exec/ram-detach.d.ts.map +1 -0
- package/dist/executors/bounded-exec/ram-detach.js +207 -0
- package/dist/executors/bounded-exec/ram-temp-fixtures.d.ts +76 -0
- package/dist/executors/bounded-exec/ram-temp-fixtures.d.ts.map +1 -1
- package/dist/executors/bounded-exec/ram-temp-fixtures.js +296 -0
- package/dist/executors/bounded-exec/ram-temp.d.ts +388 -88
- package/dist/executors/bounded-exec/ram-temp.d.ts.map +1 -1
- package/dist/executors/bounded-exec/ram-temp.js +1213 -399
- package/dist/executors/bounded-exec/run-cpu-pool.d.ts +4 -4
- package/dist/executors/bounded-exec/run-cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/run-cpu-pool.js +20 -30
- package/dist/executors/bounded-exec/runner-hook.js +14 -5
- package/dist/executors/bounded-exec/runner-lifecycle.d.ts +14 -1
- package/dist/executors/bounded-exec/runner-lifecycle.d.ts.map +1 -1
- package/dist/executors/bounded-exec/runner-lifecycle.js +27 -6
- package/dist/executors/bounded-exec/{cpu-tokens.d.ts → runner-sizing.d.ts} +14 -42
- package/dist/executors/bounded-exec/runner-sizing.d.ts.map +1 -0
- package/dist/executors/bounded-exec/{cpu-tokens.js → runner-sizing.js} +5 -35
- package/dist/executors/bounded-exec/runtime-pin.d.ts +73 -0
- package/dist/executors/bounded-exec/runtime-pin.d.ts.map +1 -0
- package/dist/executors/bounded-exec/runtime-pin.js +167 -0
- package/dist/executors/bounded-exec/schema.d.ts +18 -1
- package/dist/executors/bounded-exec/schema.d.ts.map +1 -1
- package/dist/executors/bounded-exec/verdict.d.ts +40 -11
- package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
- package/dist/executors/bounded-exec/verdict.js +88 -19
- package/dist/executors/cargo-zigbuild/executor.d.ts +21 -0
- package/dist/executors/cargo-zigbuild/executor.d.ts.map +1 -1
- package/dist/executors/cargo-zigbuild/executor.js +67 -12
- package/dist/executors/clean-outputs/executor.js +1 -0
- package/dist/executors/scriptc-native/executor.d.ts +79 -0
- package/dist/executors/scriptc-native/executor.d.ts.map +1 -0
- package/dist/executors/scriptc-native/executor.js +367 -0
- package/dist/executors/scriptc-native/ffi-paths.d.ts +9 -0
- package/dist/executors/scriptc-native/ffi-paths.d.ts.map +1 -0
- package/dist/executors/scriptc-native/ffi-paths.js +25 -0
- package/dist/executors/scriptc-native/schema.d.ts +34 -0
- package/dist/executors/scriptc-native/schema.d.ts.map +1 -0
- package/dist/executors/scriptc-native/schema.js +1 -0
- package/dist/executors/typescript-emit/executor.d.ts +3 -1
- package/dist/executors/typescript-emit/executor.d.ts.map +1 -1
- package/dist/executors/typescript-emit/executor.js +164 -33
- package/dist/forbidden-spelling-policy.d.ts +56 -0
- package/dist/forbidden-spelling-policy.d.ts.map +1 -0
- package/dist/forbidden-spelling-policy.js +176 -0
- package/dist/generators/create-package/generator.d.ts.map +1 -1
- package/dist/generators/create-package/generator.js +3 -4
- package/dist/graph-input-contract.d.ts +14 -0
- package/dist/graph-input-contract.d.ts.map +1 -1
- package/dist/graph-input-contract.js +67 -2
- package/dist/index.d.ts +23 -6
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +672 -214
- package/dist/managed-assets.d.ts +2 -0
- package/dist/managed-assets.d.ts.map +1 -1
- package/dist/managed-files/files.d.ts +3 -3
- package/dist/managed-files/files.d.ts.map +1 -1
- package/dist/managed-files/files.js +28 -3
- package/dist/managed-files/managed-format.d.ts.map +1 -1
- package/dist/managed-files/managed-format.js +1 -0
- package/dist/nextest-code-bound.js +1 -0
- package/dist/nextest-timings.d.ts +35 -0
- package/dist/nextest-timings.d.ts.map +1 -0
- package/dist/nextest-timings.js +168 -0
- package/dist/owned-installation.d.ts +6 -5
- package/dist/owned-installation.d.ts.map +1 -1
- package/dist/owned-installation.js +150 -26
- package/dist/package-target-policy.d.ts.map +1 -1
- package/dist/package-target-policy.js +2 -3
- package/dist/process-children.d.ts +7 -0
- package/dist/process-children.d.ts.map +1 -0
- package/dist/process-children.darwin.d.ts +7 -0
- package/dist/process-children.darwin.d.ts.map +1 -0
- package/dist/process-children.darwin.js +13 -0
- package/dist/process-children.js +14 -0
- package/dist/process-children.linux.d.ts +8 -0
- package/dist/process-children.linux.d.ts.map +1 -0
- package/dist/process-children.linux.js +25 -0
- package/dist/process-group.d.ts +108 -0
- package/dist/process-group.d.ts.map +1 -0
- package/dist/process-group.js +171 -0
- package/dist/process-listing.d.ts +7 -0
- package/dist/process-listing.d.ts.map +1 -0
- package/dist/process-listing.js +51 -0
- package/dist/producer-receipts.d.ts +8 -2
- package/dist/producer-receipts.d.ts.map +1 -1
- package/dist/producer-receipts.js +75 -48
- package/dist/task-graph-policy.d.ts +21 -0
- package/dist/task-graph-policy.d.ts.map +1 -0
- package/dist/task-graph-policy.js +91 -0
- package/dist/test-timings.js +4 -4
- package/dist/test-walk-policy.d.ts +14 -5
- package/dist/test-walk-policy.d.ts.map +1 -1
- package/dist/test-walk-policy.js +8 -2
- package/dist/testing.d.ts +73 -28
- package/dist/testing.d.ts.map +1 -1
- package/dist/testing.js +217 -223
- package/dist/tool-bins.d.ts +33 -0
- package/dist/tool-bins.d.ts.map +1 -0
- package/dist/tool-bins.js +72 -0
- package/dist/typecheck-test-policy.d.ts.map +1 -1
- package/dist/typecheck-test-policy.js +3 -4
- package/dist/wait-why-policy.d.ts +28 -0
- package/dist/wait-why-policy.d.ts.map +1 -0
- package/dist/wait-why-policy.js +223 -0
- package/dist/workspace-config-policy.d.ts +14 -8
- package/dist/workspace-config-policy.d.ts.map +1 -1
- package/dist/workspace-config-policy.js +60 -16
- package/dist/workspace-manifest.d.ts +41 -6
- package/dist/workspace-manifest.d.ts.map +1 -1
- package/dist/workspace-manifest.js +57 -0
- package/executors.json +16 -1
- package/loader.js +2 -0
- package/managed/raw/envrc +11 -0
- package/managed/raw/patches/nx@23.2.1.patch +2840 -199
- package/managed/raw/patches/ttsc@0.34.1.patch +119 -0
- package/managed/raw/tooling/direnv/cargo-fetch-bound.ts +245 -0
- package/managed/raw/tooling/direnv/cargo-fetch-run.ts +299 -0
- package/managed/raw/tooling/direnv/compiler-variables.sh +42 -0
- package/managed/raw/tooling/direnv/developer-links.ts +464 -25
- package/managed/raw/tooling/direnv/devenv.smoo.nix +78 -38
- package/managed/raw/tooling/direnv/github-actions-bootstrap.sh +14 -12
- package/managed/raw/tooling/direnv/installed-bun-links.ts +308 -0
- package/managed/raw/tooling/direnv/linked-snapshots.ts +62 -3
- package/managed/raw/tooling/direnv/nx-socket-dir.sh +56 -17
- package/managed/raw/tooling/direnv/repo-path +68 -6
- package/managed/raw/tooling/direnv/secret-references.ts +13 -4
- package/managed/raw/tooling/direnv/setup-environment.ts +128 -537
- package/managed/raw/tooling/direnv/toolchain-stamp.ts +4 -2
- package/managed/raw/tooling/direnv/ttsc-env.sh +9 -0
- package/managed/raw/tooling/direnv/uv-relocation.ts +227 -0
- package/managed/raw/tooling/git-hooks/commit-msg.sh +2 -1
- package/managed/raw/tooling/git-hooks/pre-commit.sh +2 -1
- package/managed/raw/tooling/git-hooks/pre-push.sh +2 -1
- package/managed/templates/github/actions/cache-ttsc-plugins/action.yml +85 -7
- package/managed/templates/github/actions/save-nix-devenv/action.yml +12 -2
- package/managed/templates/github/actions/setup-devenv/action.yml +2 -0
- package/package.json +77 -13
- package/src/bin/smoo-nx-cargo-toolchain.ts +11 -0
- package/src/bin/smoo-nx-forbidden-spellings.ts +33 -0
- package/src/bin/smoo-nx-nextest-timings.ts +31 -0
- package/src/bounded-test-policy.ts +12 -14
- package/src/build-state-output-policy.ts +458 -34
- package/src/bun-lock-scope.ts +108 -0
- package/src/cargo-build-script-env.ts +265 -0
- package/src/cargo-closure-input.ts +74 -7
- package/src/cargo-retention-hook.ts +94 -0
- package/src/cargo-source-hash.ts +138 -41
- package/src/cargo-toolchain-paths.ts +95 -0
- package/src/cargo-toolchain-policy.ts +49 -5
- package/src/cargo-workspace.ts +72 -123
- package/src/cross-check-policy.ts +12 -10
- package/src/disk-command-policy.ts +1 -0
- package/src/ensure-built-receipt.ts +132 -39
- package/src/ensure-built-witness.ts +62 -10
- package/src/ensure-built.ts +11 -1
- package/src/eslint-config-closure.ts +357 -0
- package/src/executors/bounded-exec/bounds.ts +76 -0
- package/src/executors/bounded-exec/cpu-pool.ts +51 -117
- package/src/executors/bounded-exec/executor.ts +423 -259
- package/src/executors/bounded-exec/gateway-lease.ts +82 -33
- package/src/executors/bounded-exec/hang-diagnostics.ts +173 -31
- package/src/executors/bounded-exec/lease-wait.ts +89 -0
- package/src/executors/bounded-exec/lock-wait.ts +141 -0
- package/src/executors/bounded-exec/owned-subtree.ts +235 -0
- package/src/executors/bounded-exec/progress.ts +9 -3
- package/src/executors/bounded-exec/ram-attach.ts +238 -0
- package/src/executors/bounded-exec/ram-detach.ts +226 -0
- package/src/executors/bounded-exec/ram-temp-fixtures.ts +358 -0
- package/src/executors/bounded-exec/ram-temp-registry.txt +440 -0
- package/src/executors/bounded-exec/ram-temp.ts +1497 -460
- package/src/executors/bounded-exec/run-cpu-pool.ts +22 -30
- package/src/executors/bounded-exec/runner-hook.ts +14 -6
- package/src/executors/bounded-exec/runner-lifecycle.ts +31 -8
- package/src/executors/bounded-exec/{cpu-tokens.ts → runner-sizing.ts} +13 -59
- package/src/executors/bounded-exec/runtime-pin.ts +232 -0
- package/src/executors/bounded-exec/schema.json +11 -2
- package/src/executors/bounded-exec/schema.ts +19 -1
- package/src/executors/bounded-exec/verdict.ts +137 -34
- package/src/executors/cargo-zigbuild/executor.ts +82 -8
- package/src/executors/clean-outputs/executor.ts +1 -0
- package/src/executors/scriptc-native/executor.ts +394 -0
- package/src/executors/scriptc-native/ffi-paths.ts +29 -0
- package/src/executors/scriptc-native/schema.json +60 -0
- package/src/executors/scriptc-native/schema.ts +34 -0
- package/src/executors/typescript-emit/executor.ts +205 -38
- package/src/forbidden-spelling-policy.ts +226 -0
- package/src/generators/create-package/generator.ts +7 -7
- package/src/graph-input-contract.ts +61 -2
- package/src/index.ts +788 -224
- package/src/managed-assets.ts +3 -0
- package/src/managed-files/files.ts +28 -3
- package/src/managed-files/managed-format.ts +1 -0
- package/src/nextest-code-bound.ts +1 -0
- package/src/nextest-timings.ts +106 -0
- package/src/owned-installation.ts +150 -23
- package/src/package-target-policy.ts +2 -3
- package/src/process-children.darwin.ts +14 -0
- package/src/process-children.linux.ts +33 -0
- package/src/process-children.ts +22 -0
- package/src/process-group.ts +209 -0
- package/src/process-listing.ts +54 -0
- package/src/producer-receipts.ts +68 -45
- package/src/task-graph-policy.ts +98 -0
- package/src/test-walk-policy.ts +23 -8
- package/src/testing.ts +240 -232
- package/src/tool-bins.ts +108 -0
- package/src/typecheck-test-policy.ts +2 -3
- package/src/wait-why-policy.ts +226 -0
- package/src/workspace-config-policy.ts +63 -15
- package/src/workspace-manifest.ts +89 -6
- package/test-timings.json +122 -98
- package/dist/bin/smoo-nx-failures.d.ts +0 -3
- package/dist/bin/smoo-nx-failures.d.ts.map +0 -1
- package/dist/bin/smoo-nx-failures.js +0 -3
- package/dist/bin/smoo-nx-nextest-extract.d.ts +0 -3
- package/dist/bin/smoo-nx-nextest-extract.d.ts.map +0 -1
- package/dist/bin/smoo-nx-nextest-extract.js +0 -42
- package/dist/executors/bounded-exec/cpu-tokens.d.ts.map +0 -1
- package/dist/failure-summary.d.ts +0 -5
- package/dist/failure-summary.d.ts.map +0 -1
- package/dist/failure-summary.js +0 -118
- package/dist/nextest-extraction.d.ts +0 -117
- package/dist/nextest-extraction.d.ts.map +0 -1
- package/dist/nextest-extraction.js +0 -303
- package/managed/raw/patches/ttsc@0.30.4.patch +0 -2255
- package/src/__tests__/cargo-closure-filesets.ts +0 -94
- package/src/__tests__/cargo-closure-fixture.ts +0 -113
- package/src/__tests__/counted-cargo-process.ts +0 -286
- package/src/__tests__/counted-cargo.ts +0 -713
- package/src/__tests__/directory-change.ts +0 -27
- package/src/__tests__/fixture-cargo-home.ts +0 -36
- package/src/__tests__/fixture-nx-env.ts +0 -193
- package/src/__tests__/owned-installation.ts +0 -13
- package/src/__tests__/past-tick.ts +0 -12
- package/src/__tests__/progress-clock.ts +0 -43
- package/src/__tests__/runtime-input.ts +0 -23
- package/src/bin/smoo-nx-bound-failures.test.ts +0 -127
- package/src/bin/smoo-nx-failures.ts +0 -4
- package/src/bin/smoo-nx-nextest-extract.ts +0 -44
- package/src/bounded-test-policy.test.ts +0 -1029
- package/src/build-state-output-policy.test.ts +0 -115
- package/src/bun-bootstrap.test.ts +0 -102
- package/src/bun-file-packages.test.ts +0 -47
- package/src/cargo-closure-filesets.test.ts +0 -63
- package/src/cargo-closure-input.property.test.ts +0 -50
- package/src/cargo-closure-input.test.ts +0 -506
- package/src/cargo-closure-key.test.ts +0 -240
- package/src/cargo-resolve-cache.test.ts +0 -1181
- package/src/cargo-source-hash.test.ts +0 -707
- package/src/cargo-workspace.test.ts +0 -615
- package/src/cross-check-policy.test.ts +0 -316
- package/src/disk-command-policy.test.ts +0 -61
- package/src/ensure-built-receipt.test.ts +0 -279
- package/src/ensure-built-witness.test.ts +0 -688
- package/src/ensure-built.test.ts +0 -1462
- package/src/executors/all-verdicts/executor.test.ts +0 -135
- package/src/executors/bounded-exec/cargo-progress-host.test.ts +0 -332
- package/src/executors/bounded-exec/cpu-pool.test.ts +0 -684
- package/src/executors/bounded-exec/cpu-tokens.test.ts +0 -519
- package/src/executors/bounded-exec/disk-lease.test.ts +0 -219
- package/src/executors/bounded-exec/executor-host.test.ts +0 -800
- package/src/executors/bounded-exec/executor.test.ts +0 -1819
- package/src/executors/bounded-exec/go-progress-host.test.ts +0 -171
- package/src/executors/bounded-exec/hang-diagnostics.test.ts +0 -291
- package/src/executors/bounded-exec/progress.test.ts +0 -139
- package/src/executors/bounded-exec/ram-temp-held.test.ts +0 -264
- package/src/executors/bounded-exec/ram-temp-host.test.ts +0 -635
- package/src/executors/bounded-exec/ram-temp.test.ts +0 -750
- package/src/executors/bounded-exec/run-cpu-pool.test.ts +0 -79
- package/src/executors/bounded-exec/runner-hook.test.ts +0 -193
- package/src/executors/bounded-exec/runner-lifecycle.test.ts +0 -82
- package/src/executors/bounded-exec/verdict.test.ts +0 -545
- package/src/executors/cargo-zigbuild/executor.test.ts +0 -129
- package/src/executors/clean-outputs/executor.test.ts +0 -20
- package/src/executors/typescript-emit/executor.test.ts +0 -83
- package/src/failure-summary.ts +0 -119
- package/src/generators/bounded-test-targets/generator.test.ts +0 -138
- package/src/generators/bun-test-tracing/generator.test.ts +0 -250
- package/src/generators/create-package/generator.test.ts +0 -190
- package/src/generators/make-public/generator.test.ts +0 -137
- package/src/graph-input-contract.test.ts +0 -379
- package/src/host-controller-wiring.test.ts +0 -31
- package/src/index.test.ts +0 -3321
- package/src/linked-snapshots.nx.test.ts +0 -254
- package/src/managed-files/forgejo-artifact.test.ts +0 -395
- package/src/managed-files/generator.test.ts +0 -243
- package/src/managed-files/github-host-cache-isolation.test.ts +0 -86
- package/src/managed-files/json-edit.test.ts +0 -216
- package/src/managed-files/managed-content.test.ts +0 -166
- package/src/managed-files/managed-format-json.test.ts +0 -308
- package/src/managed-files/repo-path.test.ts +0 -172
- package/src/managed-files/setup-devenv-free-disk.test.ts +0 -95
- package/src/managed-files/trace-db-artifact.test.ts +0 -7
- package/src/managed-files/tree-permissions.test.ts +0 -109
- package/src/managed-files/tree.test.ts +0 -123
- package/src/manifest-hash.test.ts +0 -219
- package/src/nextest-code-bound.test.ts +0 -161
- package/src/nextest-extraction.test.ts +0 -216
- package/src/nextest-extraction.ts +0 -371
- package/src/nextest-partitions.test.ts +0 -501
- package/src/nx-cache-bound.test.ts +0 -109
- package/src/nx-cache-restore-mtime.test.ts +0 -170
- package/src/nx-daemon-claim.test.ts +0 -279
- package/src/nx-daemon-idle-cpu.test.ts +0 -80
- package/src/nx-daemon-idle-policy.test.ts +0 -111
- package/src/nx-daemon-ignore-files.test.ts +0 -210
- package/src/nx-daemon-root-normalization.test.ts +0 -105
- package/src/nx-daemon-start-never.test.ts +0 -292
- package/src/nx-daemon-validity.test.ts +0 -1055
- package/src/nx-dot-input-policy.test.ts +0 -353
- package/src/nx-failure-life-cycles.test.ts +0 -207
- package/src/nx-files-archive-racy.test.ts +0 -97
- package/src/nx-foreign-workspace-env.test.ts +0 -329
- package/src/nx-incomplete-run-summary.test.ts +0 -70
- package/src/nx-plugin-load-errors.test.ts +0 -120
- package/src/nx-project-graph-errors.test.ts +0 -70
- package/src/nx-project-resolution-cache.test.ts +0 -176
- package/src/nx-raw-inputs.test.ts +0 -166
- package/src/nx-resident-graph-workers.test.ts +0 -112
- package/src/nx-run-information.test.ts +0 -118
- package/src/nx-skipped-task-exit.test.ts +0 -180
- package/src/nx-state-location.test.ts +0 -63
- package/src/nx-store-resolution.test.ts +0 -81
- package/src/nx-task-graph-cycles.test.ts +0 -148
- package/src/nx-task-output-lease.test.ts +0 -550
- package/src/nx-use-local.test.ts +0 -105
- package/src/owned-installation.test.ts +0 -105
- package/src/package-target-policy.test.ts +0 -1014
- package/src/producer-receipts.test.ts +0 -285
- package/src/release-config-policy.test.ts +0 -311
- package/src/runtime-input.test.ts +0 -393
- package/src/serial-group-wiring.test.ts +0 -326
- package/src/task-history-namespace.test.ts +0 -181
- package/src/task-results.test.ts +0 -228
- package/src/test-clock-policy.test.ts +0 -338
- package/src/test-program.test.ts +0 -303
- package/src/test-timings.test.ts +0 -240
- package/src/test-walk-policy.test.ts +0 -620
- package/src/testing.test.ts +0 -509
- package/src/typecheck-test-policy.test.ts +0 -1468
- package/src/typescript-resolution.test.ts +0 -76
- package/src/version-actions.test.ts +0 -80
- package/src/workspace-config-policy.test.ts +0 -737
|
@@ -1,17 +1,23 @@
|
|
|
1
1
|
import { spawn } from 'node:child_process';
|
|
2
2
|
import { accessSync, constants } from 'node:fs';
|
|
3
|
+
import { mkdtemp, rm } from 'node:fs/promises';
|
|
3
4
|
import { availableParallelism, userInfo } from 'node:os';
|
|
4
5
|
import { delimiter, isAbsolute, join, resolve as resolvePath } from 'node:path';
|
|
5
6
|
import { createInterface } from 'node:readline';
|
|
6
|
-
import { Duplex } from 'node:stream';
|
|
7
|
+
import { Duplex, Readable } from 'node:stream';
|
|
7
8
|
import { workspaceDataDirectoryForWorkspace } from 'nx/src/utils/cache-directory.js';
|
|
9
|
+
import { DEADLINE_PATIENCE, describeMembersLeft, killProcessGroup, processGroupHasMembers, signalProcessGroup, } from '../../process-group.js';
|
|
10
|
+
import { earliestBound, stepBounds, UNARMED } from './bounds.js';
|
|
8
11
|
import { CPU_POOL_ENV, createCpuPool, readInheritedCpuPool, takeParentCpuTokens, } from './cpu-pool.js';
|
|
9
|
-
import {
|
|
12
|
+
import { GATEWAY_SOCKET } from './gateway-lease.js';
|
|
10
13
|
import { captureHangDiagnostics, HANG_CAPTURE_MS } from './hang-diagnostics.js';
|
|
11
14
|
import { createNoSpaceWatch } from './no-space.js';
|
|
15
|
+
import { COWSHED_EXECUTABLE_ENV, cowshedSubtreeOwner, HANDLE_FD, readAdmission, SUPERVISOR_SOCKET_ENV, } from './owned-subtree.js';
|
|
12
16
|
import { createRunProgress, withProgressReport } from './progress.js';
|
|
13
|
-
import {
|
|
14
|
-
import { createRunnerLifecycle, withRunnerLifecycle } from './runner-lifecycle.js';
|
|
17
|
+
import { createHostCommands, describeCoverageGap, describeRamTempError, describeSignalled, LEFTOVER_GRACE_MS, PRODUCTION, RAM_TEMP_CAPACITY_BYTES, RamTempVolume, ramTempPaths, } from './ram-temp.js';
|
|
18
|
+
import { createRunnerLifecycle, withFixtureLifecycle, withRunnerLifecycle } from './runner-lifecycle.js';
|
|
19
|
+
import { CPU_TOKENS_ENV, runnerDemand, sizeRunner } from './runner-sizing.js';
|
|
20
|
+
import { checkRuntimePin, describeRuntimePinRefusal } from './runtime-pin.js';
|
|
15
21
|
import { clearTaskDirectory, describeNoSpace, JUNIT_FILE, JUNIT_REPORT_ENV, judgeOperationalFailure, judgeRun, readJunitReport, taskDirectory, withJunitReport, writeRecord, } from './verdict.js';
|
|
16
22
|
const DEFAULT_KILL_AFTER_MS = 10_000;
|
|
17
23
|
const REAP_AFTER_FORCE_KILL_MS = 2_000;
|
|
@@ -23,9 +29,9 @@ const EXIT_CODE_BY_SIGNAL = {
|
|
|
23
29
|
/** The pool descriptor: only a grant may give it, so neither the target's nor the inherited environment passes it on. */
|
|
24
30
|
const POOL_ENV = [CPU_TOKENS_ENV, CPU_POOL_ENV];
|
|
25
31
|
/**
|
|
26
|
-
* The variables
|
|
27
|
-
*
|
|
28
|
-
* sub-pool names its own fifo over it.
|
|
32
|
+
* The variables a grant sets for the command it sizes: no target's environment may set them. A jobserver the executor
|
|
33
|
+
* itself inherits, a parent make's or its cowshed job's, still reaches an ungranted command by Cargo's own rule; a
|
|
34
|
+
* granted one's sub-pool names its own fifo over it.
|
|
29
35
|
*/
|
|
30
36
|
const BUDGET_ENV = [...POOL_ENV, 'CARGO_MAKEFLAGS', 'MAKEFLAGS', 'MFLAGS'];
|
|
31
37
|
/** Workspace-provided cowshed launchers, in the order a workspace that provides one names them. */
|
|
@@ -73,22 +79,36 @@ export function resolveCowshedExecutable(root, named, inherited, pathValue, cwd,
|
|
|
73
79
|
export default function boundedExecExecutor(options, context) {
|
|
74
80
|
return runDefaultBoundedExec(options, recordContext(context));
|
|
75
81
|
}
|
|
76
|
-
/**
|
|
82
|
+
/**
|
|
83
|
+
* Runs with the normal process, RAM lease and CPU pool owners, with an optional stdout sink, on the user's RAM temp
|
|
84
|
+
* volume where there is one, through the host's own gateway: the Nx executor's runs, never a test's.
|
|
85
|
+
*/
|
|
77
86
|
export function runDefaultBoundedExec(options, context) {
|
|
87
|
+
// Only macOS has `hdiutil ram://`; elsewhere the command keeps the inherited TMPDIR.
|
|
88
|
+
return runBoundedExecWith(options, context, (cowshed) => process.platform === 'darwin'
|
|
89
|
+
? new RamTempVolume(ramTempPaths(userInfo(), PRODUCTION), RAM_TEMP_CAPACITY_BYTES, createHostCommands(GATEWAY_SOCKET, undefined, undefined, cowshed), undefined, undefined, 'optional')
|
|
90
|
+
: null);
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* The default runner with every owner but its RAM volume, which its caller makes: `volume` makes the volume for the
|
|
94
|
+
* cowshed the run's disk commands go through. The Nx executor makes the user's; a host proof passes a fixture volume
|
|
95
|
+
* (`fixtureRamVolume`), so no test leases, creates or detaches the user's volume.
|
|
96
|
+
*/
|
|
97
|
+
export function runBoundedExecWith(options, context, volume) {
|
|
78
98
|
// A target that names the cowshed its disk commands run through names it for this run's own RAM volume work and
|
|
79
99
|
// for its command alike. Nx resolves `{workspaceRoot}/…` in an option to a workspace-relative path, which the
|
|
80
100
|
// command, starting in another directory, could not find: it gets the absolute one.
|
|
81
101
|
const named = options.env?.[COWSHED_EXECUTABLE_ENV];
|
|
82
102
|
const cowshed = resolveCowshedExecutable(context.root, named, process.env[COWSHED_EXECUTABLE_ENV], process.env.PATH, process.cwd());
|
|
83
103
|
const run = cowshed === undefined ? options : { ...options, env: { ...options.env, [COWSHED_EXECUTABLE_ENV]: cowshed } };
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
104
|
+
const tempVolume = volume(cowshed);
|
|
105
|
+
// The command's processes are owned by the supervisor that serves this job, when one does
|
|
106
|
+
// (specs/cowshed/07_api.md, `JobSubtreeHandle`); with none, nothing here may end what it leaves.
|
|
107
|
+
const owner = process.env[SUPERVISOR_SOCKET_ENV] === undefined ? null : cowshedSubtreeOwner(cowshed ?? 'cowshed');
|
|
88
108
|
// A command a parent's CPU pool started draws its tokens from that pool. The pool is named by the environment the
|
|
89
109
|
// parent set for this command; a run no parent started has none.
|
|
90
110
|
const inherited = readInheritedCpuPool(process.env);
|
|
91
|
-
return runBoundedExec(run, context, createProcessTreeKiller(), tempVolume,
|
|
111
|
+
return runBoundedExec(run, context, createProcessTreeKiller(), tempVolume, inherited, captureHangDiagnostics, owner);
|
|
92
112
|
}
|
|
93
113
|
/**
|
|
94
114
|
* Nx gives an executor the task's project, target and configuration, not its id; the id is the
|
|
@@ -113,17 +133,15 @@ export function recordContext(context) {
|
|
|
113
133
|
};
|
|
114
134
|
}
|
|
115
135
|
/**
|
|
116
|
-
* Run the command
|
|
117
|
-
*
|
|
118
|
-
* granted: a wait for the host is not the command's time.
|
|
136
|
+
* Run the command, sized to the tokens its parent's CPU pool (`cpu-pool.ts`) lends it; a run that inherits no pool
|
|
137
|
+
* runs it as asked. Both bounds start once it is granted: a wait for the pool is not the command's time.
|
|
119
138
|
*
|
|
120
|
-
* `inherited` is the parent's
|
|
121
|
-
*
|
|
122
|
-
*
|
|
123
|
-
*
|
|
124
|
-
* its own exit, and the run says so.
|
|
139
|
+
* `inherited` is the parent's pool: an enclosing bounded-exec's sub-pool or the Nx run's own (`run-cpu-pool.ts`). A
|
|
140
|
+
* granted run serves a sub-pool of its grant to its own descendants, closed before the grant returns. A grant whose
|
|
141
|
+
* connection closes while the command runs (the lending parent's exit) ends its accounting, never the command: the
|
|
142
|
+
* command runs on over budget to its own exit, and the run says so.
|
|
125
143
|
*/
|
|
126
|
-
export async function runBoundedExec(options, context, killer, tempVolume,
|
|
144
|
+
export async function runBoundedExec(options, context, killer, tempVolume, inherited = null, captureDiagnostics = captureHangDiagnostics, owner = null) {
|
|
127
145
|
const cwd = resolveCwd(options.cwd, context.root);
|
|
128
146
|
const directory = context.record?.directory ?? null;
|
|
129
147
|
// A previous run's verdict and report go first, so neither can be read as this run's.
|
|
@@ -134,13 +152,18 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
|
|
|
134
152
|
const timeoutMs = options.timeoutMs;
|
|
135
153
|
const idleTimeoutMs = options.idleTimeoutMs;
|
|
136
154
|
const killAfterMs = options.killAfterMs ?? DEFAULT_KILL_AFTER_MS;
|
|
155
|
+
const fixture = options.fixtureCommand !== undefined && options.fixtureTimeoutMs !== undefined
|
|
156
|
+
? { command: options.fixtureCommand, timeoutMs: options.fixtureTimeoutMs }
|
|
157
|
+
: null;
|
|
137
158
|
const outputChunks = [];
|
|
138
159
|
const state = {
|
|
139
160
|
settled: false,
|
|
140
161
|
expiry: null,
|
|
141
162
|
forceKillNeeded: false,
|
|
142
163
|
poolCloseError: null,
|
|
164
|
+
retireError: null,
|
|
143
165
|
noSpace: null,
|
|
166
|
+
lockWaits: [],
|
|
144
167
|
};
|
|
145
168
|
const appendStdout = (chunk) => {
|
|
146
169
|
const text = chunk.toString();
|
|
@@ -157,9 +180,55 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
|
|
|
157
180
|
outputChunks.push(text);
|
|
158
181
|
process.stderr.write(text);
|
|
159
182
|
};
|
|
183
|
+
// Told, never thrown, wherever the command is being ended: a diagnostic must neither replace the failure it
|
|
184
|
+
// accompanies nor skip the cleanup that follows (the signal handlers, the CPU grant, the RAM lease, the run's record).
|
|
185
|
+
const tell = (text) => {
|
|
186
|
+
try {
|
|
187
|
+
appendStderr(text);
|
|
188
|
+
}
|
|
189
|
+
catch {
|
|
190
|
+
// Nobody is left to tell.
|
|
191
|
+
}
|
|
192
|
+
};
|
|
193
|
+
// The fixture pair names one segment: a fixture with no build budget, or a budget with no fixture, is a
|
|
194
|
+
// target that does not say what it runs, so it does not run.
|
|
195
|
+
if ((options.fixtureCommand === undefined) !== (options.fixtureTimeoutMs === undefined) ||
|
|
196
|
+
(fixture !== null && !(fixture.timeoutMs > 0))) {
|
|
197
|
+
appendStderr('bounded-exec: fixtureCommand and a positive fixtureTimeoutMs must be named together; refusing to run the command\n');
|
|
198
|
+
if (context.record !== undefined) {
|
|
199
|
+
const { task, directory: recordDirectory } = context.record;
|
|
200
|
+
await writeRecord(recordDirectory, { task: task.id, hash: task.hash, verdict: judgeOperationalFailure(1, null) });
|
|
201
|
+
}
|
|
202
|
+
return { success: false, terminalOutput: outputChunks.join('') };
|
|
203
|
+
}
|
|
204
|
+
// A leg that runs bun runs the Bun its workspace declares, or it does not run. This comes before the RAM lease and the
|
|
205
|
+
// CPU tokens: a refusal that waited in its pool's queue first would say nothing for minutes.
|
|
206
|
+
const runtime = await checkRuntimePin({
|
|
207
|
+
root: context.root,
|
|
208
|
+
command: fixture === null ? requested : `${fixture.command} && ${requested}`,
|
|
209
|
+
env: { ...withoutPoolEnv(process.env), ...options.env },
|
|
210
|
+
cwd,
|
|
211
|
+
pathSource: options.env?.PATH === undefined ? 'the inherited environment' : "the target's env",
|
|
212
|
+
});
|
|
213
|
+
if (runtime !== null) {
|
|
214
|
+
appendStderr(describeRuntimePinRefusal(runtime, context.record?.task.id));
|
|
215
|
+
if (context.record !== undefined) {
|
|
216
|
+
const { task, directory: recordDirectory } = context.record;
|
|
217
|
+
await writeRecord(recordDirectory, { task: task.id, hash: task.hash, verdict: judgeOperationalFailure(1, null) });
|
|
218
|
+
}
|
|
219
|
+
return { success: false, terminalOutput: outputChunks.join('') };
|
|
220
|
+
}
|
|
160
221
|
let lease = null;
|
|
161
|
-
|
|
162
|
-
|
|
222
|
+
let ordinaryTemp = null;
|
|
223
|
+
// Before spawn there is nothing to retire. Once started, only the owner's terminal retirement permits deletion.
|
|
224
|
+
let subtreeRetired = true;
|
|
225
|
+
// A lease is removed when its command ends, so it is taken only where whatever the command leaves
|
|
226
|
+
// in it can be ended by its owner first: the supervisor's subtree retirement, never a census.
|
|
227
|
+
if (tempVolume !== null && owner === null) {
|
|
228
|
+
appendStderr(`RAM temp volume not leased: no cowshed supervisor owns this command's processes (${SUPERVISOR_SOCKET_ENV} is unset); TMPDIR stays ${process.env.TMPDIR ?? '(unset)'}\n`);
|
|
229
|
+
}
|
|
230
|
+
else if (tempVolume !== null && owner !== null) {
|
|
231
|
+
const acquired = await tempVolume.acquire(owner);
|
|
163
232
|
if (!acquired.ok) {
|
|
164
233
|
const message = `${describeRamTempError(acquired.error)}\n`;
|
|
165
234
|
process.stderr.write(message);
|
|
@@ -171,44 +240,118 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
|
|
|
171
240
|
appendStderr(`RAM temp volume: dead lease ${held}\n`);
|
|
172
241
|
}
|
|
173
242
|
for (const reaped of acquired.value.reaped) {
|
|
174
|
-
appendStderr(`RAM temp volume: stopped ${reaped}, which a dead task left
|
|
243
|
+
appendStderr(`RAM temp volume: the supervisor stopped ${reaped}, which a dead task's command left running\n`);
|
|
244
|
+
}
|
|
245
|
+
for (const gap of acquired.value.gaps) {
|
|
246
|
+
appendStderr(`RAM temp volume: the supervisor's view of a dead task's command has a gap: ${gap}\n`);
|
|
247
|
+
}
|
|
248
|
+
for (const retained of acquired.value.retained) {
|
|
249
|
+
appendStderr(`RAM temp volume: dead lease ${retained}\n`);
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
else if (acquired.value.kind === 'unavailable') {
|
|
253
|
+
const unavailable = `RAM temp volume unavailable (${acquired.value.reason}: ${acquired.value.detail})`;
|
|
254
|
+
if (options.env?.TMPDIR !== undefined) {
|
|
255
|
+
appendStderr(`${unavailable}; using target-provided TMPDIR ${options.env.TMPDIR} (medium not inferred)\n`);
|
|
256
|
+
}
|
|
257
|
+
else {
|
|
258
|
+
try {
|
|
259
|
+
ordinaryTemp = await mkdtemp(join(tempVolume.ordinaryParent, `smoo-bounded-exec-${process.pid}-`));
|
|
260
|
+
}
|
|
261
|
+
catch (error) {
|
|
262
|
+
appendStderr(`${unavailable}; could not create ordinary scratch under ${tempVolume.ordinaryParent}: ${error instanceof Error ? error.message : String(error)}\n`);
|
|
263
|
+
return { success: false, terminalOutput: outputChunks.join('') };
|
|
264
|
+
}
|
|
265
|
+
appendStderr(`${unavailable}; using owned ordinary TMPDIR ${ordinaryTemp}\n`);
|
|
175
266
|
}
|
|
176
267
|
}
|
|
177
268
|
else {
|
|
178
269
|
appendStderr(`RAM temp volume unavailable in this sandbox (${acquired.value.detail}); TMPDIR stays ${process.env.TMPDIR ?? '(unset)'}\n`);
|
|
179
270
|
}
|
|
180
271
|
}
|
|
181
|
-
|
|
182
|
-
|
|
272
|
+
// The supervisor's answer to the command's own admission, once it is spawned as an owned subtree root, and once
|
|
273
|
+
// any lease has recorded the handle for whoever settles that lease should this task die first.
|
|
274
|
+
let admission = null;
|
|
275
|
+
const retireSubtree = async () => {
|
|
276
|
+
if (admission === null || owner === null) {
|
|
183
277
|
return;
|
|
184
278
|
}
|
|
185
|
-
const
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
if (reaped?.ok) {
|
|
191
|
-
for (const leftover of reaped.value) {
|
|
192
|
-
appendStderr(`RAM temp volume: stopped ${leftover}, which the command left working in its lease\n`);
|
|
193
|
-
}
|
|
194
|
-
}
|
|
195
|
-
else if (reaped) {
|
|
196
|
-
appendStderr(`${describeRamTempError(reaped.error)}\n`);
|
|
279
|
+
const answer = await admission;
|
|
280
|
+
admission = null;
|
|
281
|
+
if (answer.kind !== 'admitted') {
|
|
282
|
+
appendStderr(`cowshed: the command ran without an owned subtree (${answer.kind === 'refused' ? `supervisor: ${answer.reason}` : answer.detail}); nothing retires what it left running\n`);
|
|
283
|
+
return;
|
|
197
284
|
}
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
285
|
+
// What the command left running is a leak, whichever way the command ended: its supervisor ends exactly the
|
|
286
|
+
// processes its tree records under the command, before any lease goes, and each is said so its owner fixes it.
|
|
287
|
+
const retired = await owner.retire(answer.handle, LEFTOVER_GRACE_MS);
|
|
288
|
+
switch (retired.kind) {
|
|
289
|
+
case 'retired':
|
|
290
|
+
subtreeRetired = true;
|
|
291
|
+
for (const entry of retired.signalled) {
|
|
292
|
+
appendStderr(`cowshed: the supervisor stopped ${describeSignalled(entry)}, which the command left running\n`);
|
|
293
|
+
}
|
|
294
|
+
if (retired.signalledIncomplete) {
|
|
295
|
+
appendStderr("cowshed: the supervisor may have stopped processes of the command's that it could not name\n");
|
|
296
|
+
}
|
|
297
|
+
for (const gap of retired.gaps) {
|
|
298
|
+
appendStderr(`cowshed: the supervisor's view of the command's processes has a gap: ${describeCoverageGap(gap)}\n`);
|
|
299
|
+
}
|
|
300
|
+
return;
|
|
301
|
+
case 'refused':
|
|
302
|
+
appendStderr(`cowshed: the supervisor refused to retire the command's processes (${retired.reason})\n`);
|
|
303
|
+
return;
|
|
304
|
+
case 'failed':
|
|
305
|
+
state.retireError = retired.detail;
|
|
306
|
+
appendStderr(`cowshed: could not retire the command's processes: ${retired.detail}\n`);
|
|
307
|
+
return;
|
|
201
308
|
}
|
|
202
309
|
};
|
|
203
|
-
//
|
|
204
|
-
|
|
310
|
+
// Failed scratch remains evidence: RAM leases are archived; ordinary scratch is retained at its reported path.
|
|
311
|
+
let tempReleased = null;
|
|
312
|
+
const releaseLease = (keep) => {
|
|
313
|
+
tempReleased ??= (async () => {
|
|
314
|
+
await retireSubtree();
|
|
315
|
+
if (ordinaryTemp !== null) {
|
|
316
|
+
const held = ordinaryTemp;
|
|
317
|
+
ordinaryTemp = null;
|
|
318
|
+
if (keep || state.retireError !== null || !subtreeRetired) {
|
|
319
|
+
appendStderr(`Ordinary temp directory retained at ${held}: ${!subtreeRetired ? 'the command subtree was not retired' : 'the run did not pass'}\n`);
|
|
320
|
+
}
|
|
321
|
+
else {
|
|
322
|
+
try {
|
|
323
|
+
await rm(held, { recursive: true, force: true });
|
|
324
|
+
}
|
|
325
|
+
catch (error) {
|
|
326
|
+
appendStderr(`Ordinary temp directory retained at ${held}: cleanup failed: ${error instanceof Error ? error.message : String(error)}\n`);
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
if (lease === null || owner === null) {
|
|
331
|
+
return;
|
|
332
|
+
}
|
|
333
|
+
const held = lease;
|
|
334
|
+
lease = null;
|
|
335
|
+
const released = await tempVolume?.release(held, keep || state.retireError !== null, owner);
|
|
336
|
+
if (released?.ok) {
|
|
337
|
+
for (const retained of released.value) {
|
|
338
|
+
appendStderr(`RAM temp volume: lease ${retained}\n`);
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
else if (released) {
|
|
342
|
+
appendStderr(`${describeRamTempError(released.error)}\n`);
|
|
343
|
+
}
|
|
344
|
+
})();
|
|
345
|
+
return tempReleased;
|
|
346
|
+
};
|
|
347
|
+
// The wait is one span line, as cowshed's own spans read. Every Nx run that inherits no pool serves its own
|
|
348
|
+
// (`run-cpu-pool.ts`), so a task takes its tokens from a parent's pool; a command that inherits none runs exactly as
|
|
205
349
|
// asked and nothing is said.
|
|
206
350
|
const cores = availableParallelism();
|
|
207
351
|
const demand = runnerDemand(requested, options.parallelism, cores, process.env, options.env);
|
|
208
352
|
const asked = Date.now();
|
|
209
|
-
// A
|
|
210
|
-
// A pool
|
|
211
|
-
// relay to the host -- so asking the host beside it would count the run twice; the run fails instead.
|
|
353
|
+
// A pool that answered and refused is a live budget, and running beside it would count the run twice: the run fails
|
|
354
|
+
// instead. A pool whose endpoint is gone (its holder exited) lends nothing, and the run goes on as asked.
|
|
212
355
|
let grant = null;
|
|
213
356
|
let parentRefusal = null;
|
|
214
357
|
if (inherited !== null) {
|
|
@@ -217,32 +360,19 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
|
|
|
217
360
|
grant = fromParent;
|
|
218
361
|
}
|
|
219
362
|
else if (fromParent.cause === 'absent') {
|
|
220
|
-
appendStderr(`cowshed: cpu-
|
|
363
|
+
appendStderr(`cowshed: cpu-pool parent pool absent: ${fromParent.reason}; running the command as asked\n`);
|
|
221
364
|
}
|
|
222
365
|
else {
|
|
223
366
|
parentRefusal = fromParent.reason;
|
|
224
367
|
}
|
|
225
368
|
}
|
|
226
|
-
let unbudgeted = null;
|
|
227
|
-
if (grant === null && parentRefusal === null && budget !== null) {
|
|
228
|
-
const fromHost = await budget.take(demand.want, context.root, requested);
|
|
229
|
-
if (fromHost.granted) {
|
|
230
|
-
grant = fromHost;
|
|
231
|
-
}
|
|
232
|
-
else if (fromHost.cause !== 'absent') {
|
|
233
|
-
unbudgeted = fromHost.reason;
|
|
234
|
-
}
|
|
235
|
-
}
|
|
236
369
|
const waitedMs = Date.now() - asked;
|
|
237
370
|
let cpu;
|
|
238
371
|
let sized = { command: requested, env: {} };
|
|
239
372
|
if (grant !== null) {
|
|
240
373
|
sized = sizeRunner(demand, requested, grant.tokens);
|
|
241
|
-
cpu = { tokens: grant.tokens, want: demand.want, waitedMs
|
|
242
|
-
appendStderr(`cowshed: cpu-
|
|
243
|
-
}
|
|
244
|
-
else if (unbudgeted !== null) {
|
|
245
|
-
appendStderr(`cowshed: cpu-tokens wait done elapsed=${waitedMs}ms status=err runner=${demand.kind} runs unbudgeted: ${unbudgeted}\n`);
|
|
374
|
+
cpu = { tokens: grant.tokens, want: demand.want, waitedMs };
|
|
375
|
+
appendStderr(`cowshed: cpu-pool wait done elapsed=${waitedMs}ms status=ok tokens=${grant.tokens}/${demand.want} runner=${demand.kind}\n`);
|
|
246
376
|
}
|
|
247
377
|
// The run's sub-pool and grant are given back exactly once, however the run ends. The sub-pool closes first, so no
|
|
248
378
|
// descendant still draws on tokens the grant has returned.
|
|
@@ -264,9 +394,9 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
|
|
|
264
394
|
return cpuReleased;
|
|
265
395
|
};
|
|
266
396
|
// Records the run's verdict when it has a task to record as. An `expiry` is set only when a bound ended the run.
|
|
267
|
-
// An `operational` run failed on its CPU
|
|
268
|
-
//
|
|
269
|
-
const recordRun = async (exitCode, expiry, elapsedMs, operational, noSpace) => {
|
|
397
|
+
// An `operational` run failed on its CPU pool (a sub-pool that did not close): an ordinary failure, whatever bound also
|
|
398
|
+
// fired, so it is never judged as a bound.
|
|
399
|
+
const recordRun = async (exitCode, expiry, elapsedMs, operational, noSpace, lockWaits) => {
|
|
270
400
|
if (context.record === undefined) {
|
|
271
401
|
return;
|
|
272
402
|
}
|
|
@@ -274,7 +404,7 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
|
|
|
274
404
|
const report = await readJunitReport(recordDirectory);
|
|
275
405
|
const verdict = operational
|
|
276
406
|
? judgeOperationalFailure(exitCode, report)
|
|
277
|
-
: judgeRun({ exitCode, elapsedMs, expiry, report, noSpace });
|
|
407
|
+
: judgeRun({ exitCode, elapsedMs, expiry, report, noSpace, lockWaits });
|
|
278
408
|
if (verdict.outcome === 'bound' && verdict.bound === 'test') {
|
|
279
409
|
appendStderr(`Every failing test failed only on its per-test timeout: ${verdict.tests.join(', ')}\n`);
|
|
280
410
|
}
|
|
@@ -284,34 +414,18 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
|
|
|
284
414
|
const refuse = async (message) => {
|
|
285
415
|
appendStderr(message);
|
|
286
416
|
await releaseCpu();
|
|
287
|
-
|
|
288
|
-
await
|
|
417
|
+
// The command never started, so its lease holds nothing of its own to keep.
|
|
418
|
+
await releaseLease(false);
|
|
419
|
+
await recordRun(1, null, Date.now() - asked, false, null, []);
|
|
289
420
|
return { success: false, terminalOutput: outputChunks.join('') };
|
|
290
421
|
};
|
|
291
422
|
if (parentRefusal !== null) {
|
|
292
|
-
return refuse(`cowshed: cpu-
|
|
293
|
-
}
|
|
294
|
-
// Descendant work queued for host CPU tokens, as its pool hears it (`host-wait`): while any is held the idle clock
|
|
295
|
-
// is stopped, because a queued job is not an idle one (specs/cowshed/05_gateway.md, "The bounds start at the
|
|
296
|
-
// grant"). The clock it stops is made with the command below; a wait held before then is counted and honoured there.
|
|
297
|
-
let heldWaits = 0;
|
|
298
|
-
let idleClock = null;
|
|
299
|
-
const hostWaits = {
|
|
300
|
-
begin: () => {
|
|
301
|
-
heldWaits += 1;
|
|
302
|
-
if (heldWaits === 1)
|
|
303
|
-
idleClock?.stop();
|
|
304
|
-
},
|
|
305
|
-
end: () => {
|
|
306
|
-
heldWaits -= 1;
|
|
307
|
-
if (heldWaits === 0)
|
|
308
|
-
idleClock?.restart();
|
|
309
|
-
},
|
|
310
|
-
};
|
|
423
|
+
return refuse(`cowshed: cpu-pool parent pool refused: ${parentRefusal}; refusing to run the command\n`);
|
|
424
|
+
}
|
|
311
425
|
// A granted run serves a sub-pool of its own grant, so its descendants draw from that and never from above it.
|
|
312
|
-
// A pool that cannot be set up refuses the run: no command runs
|
|
426
|
+
// A pool that cannot be set up refuses the run: no granted command runs beside its grant.
|
|
313
427
|
if (grant !== null) {
|
|
314
|
-
const created = await createCpuPool(grant.tokens, grant.closed
|
|
428
|
+
const created = await createCpuPool(grant.tokens, grant.closed);
|
|
315
429
|
if (!created.ok) {
|
|
316
430
|
return refuse(`cowshed: cpu pool unavailable: ${created.error}; refusing to run the command\n`);
|
|
317
431
|
}
|
|
@@ -330,26 +444,61 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
|
|
|
330
444
|
...sized.env,
|
|
331
445
|
...(pool?.environment ?? {}),
|
|
332
446
|
};
|
|
333
|
-
|
|
447
|
+
// A fixture rides as segment 0 of the lifecycle. Without the channel (Windows) it is plain shell text ahead
|
|
448
|
+
// of the command: the progress bound runs during it and its own ceiling does not.
|
|
449
|
+
const hooked = process.platform === 'win32'
|
|
450
|
+
? null
|
|
451
|
+
: fixture === null
|
|
452
|
+
? withRunnerLifecycle(command)
|
|
453
|
+
: withFixtureLifecycle(fixture.command, command);
|
|
334
454
|
const lifecycle = hooked === null ? null : createRunnerLifecycle(hooked.runners);
|
|
335
455
|
let lifecycleFailure = null;
|
|
336
456
|
const isTesting = () => lifecycleFailure !== null || (lifecycle?.isTesting() ?? true);
|
|
337
457
|
const startedAt = Date.now();
|
|
458
|
+
// With a supervisor, `cowshed subtree-root` asks it for a handle on itself, writes the answer to the handle
|
|
459
|
+
// descriptor, and execs `/bin/sh -c command` in the same process: the command's pid, group and life are unchanged.
|
|
460
|
+
const shellCommand = hooked?.command ?? (fixture === null ? command : `${fixture.command} && ${command}`);
|
|
461
|
+
const root = owner?.rootArgv(shellCommand);
|
|
338
462
|
let child;
|
|
339
463
|
try {
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
464
|
+
// fd 3 is the runner lifecycle's duplex socket when the command is hooked; the subtree admission takes fd 4, which
|
|
465
|
+
// `cowshed subtree-root` closes before it execs, so fd 3 reaches the shell unchanged.
|
|
466
|
+
child =
|
|
467
|
+
root === undefined
|
|
468
|
+
? spawn(shellCommand, [], {
|
|
469
|
+
cwd,
|
|
470
|
+
env: mergeEnv(lease?.directory ?? ordinaryTemp, env),
|
|
471
|
+
shell: true,
|
|
472
|
+
stdio: hooked === null ? 'pipe' : ['pipe', 'pipe', 'pipe', 'pipe'],
|
|
473
|
+
detached: process.platform !== 'win32',
|
|
474
|
+
windowsHide: true,
|
|
475
|
+
})
|
|
476
|
+
: spawn(root.file, root.args, {
|
|
477
|
+
cwd,
|
|
478
|
+
env: mergeEnv(lease?.directory ?? ordinaryTemp, env),
|
|
479
|
+
stdio: ['pipe', 'pipe', 'pipe', hooked === null ? 'ignore' : 'pipe', 'pipe'],
|
|
480
|
+
detached: process.platform !== 'win32',
|
|
481
|
+
windowsHide: true,
|
|
482
|
+
});
|
|
348
483
|
}
|
|
349
484
|
catch (error) {
|
|
350
485
|
// Nothing started, so nothing runs under the grant: it goes back here, with the RAM lease.
|
|
351
486
|
return refuse(`cowshed: could not start the command: ${error instanceof Error ? error.message : String(error)}\n`);
|
|
352
487
|
}
|
|
488
|
+
subtreeRetired = false;
|
|
489
|
+
if (root !== undefined) {
|
|
490
|
+
const channel = child.stdio[HANDLE_FD];
|
|
491
|
+
const leased = lease;
|
|
492
|
+
admission = readAdmission(channel instanceof Readable ? channel : null).then(async (answer) => {
|
|
493
|
+
if (answer.kind === 'admitted' && leased !== null && tempVolume !== null) {
|
|
494
|
+
const recorded = await tempVolume.record(leased, answer.handle);
|
|
495
|
+
if (!recorded.ok) {
|
|
496
|
+
appendStderr(`${describeRamTempError(recorded.error)}\n`);
|
|
497
|
+
}
|
|
498
|
+
}
|
|
499
|
+
return answer;
|
|
500
|
+
});
|
|
501
|
+
}
|
|
353
502
|
// Progress is a finished test, not output: a runner's SLOW warnings and a hung test's heartbeat keep printing.
|
|
354
503
|
const progress = createRunProgress(command);
|
|
355
504
|
// A write that found no space is said once, in the output, by whatever made it; the volume is read when it is said,
|
|
@@ -359,24 +508,22 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
|
|
|
359
508
|
const held = lease;
|
|
360
509
|
return { line, volume: held === null ? null : ((await tempVolume?.space(held)) ?? null) };
|
|
361
510
|
};
|
|
362
|
-
|
|
511
|
+
// Which bound is armed, and when it expires, is bounds.ts's decision; one timer waits for the earliest of them.
|
|
512
|
+
let bounds = UNARMED;
|
|
513
|
+
let boundTimer;
|
|
514
|
+
// The fixture's own output is compilation, not the command's progress: it is not scanned for finished tests.
|
|
515
|
+
let fixtureDone = fixture === null || lifecycle === null;
|
|
516
|
+
// Whether a segment past the fixture ever announced: without one, the shell's exit is the fixture's.
|
|
517
|
+
let mainStarted = fixture === null || lifecycle === null;
|
|
363
518
|
let captureTimer;
|
|
364
519
|
let idleCapture;
|
|
365
520
|
const cancelCapture = () => {
|
|
366
521
|
clearTimeout(captureTimer);
|
|
367
522
|
idleCapture?.controller.abort();
|
|
368
523
|
};
|
|
369
|
-
|
|
370
|
-
// wait keeps it unspent: the clock is stopped while descendant work is queued for CPU tokens, and runs on from where
|
|
371
|
-
// it stopped once the last wait ends. `queuedMs` is the queued time this interval has not counted, for the report.
|
|
372
|
-
let idleLeftMs = idleTimeoutMs ?? 0;
|
|
373
|
-
let queuedMs = 0;
|
|
374
|
-
let queuedSinceMs = null;
|
|
375
|
-
const scheduleIdle = (limitMs, leftMs) => {
|
|
376
|
-
clearTimeout(idleTimer);
|
|
377
|
-
cancelCapture();
|
|
524
|
+
const scheduleCapture = (expiresAtMs) => {
|
|
378
525
|
const capture = {
|
|
379
|
-
expiresAtMs
|
|
526
|
+
expiresAtMs,
|
|
380
527
|
controller: new AbortController(),
|
|
381
528
|
result: { kind: 'pending', startedAtMs: null },
|
|
382
529
|
};
|
|
@@ -409,65 +556,42 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
|
|
|
409
556
|
};
|
|
410
557
|
}
|
|
411
558
|
});
|
|
412
|
-
}, Math.max(0,
|
|
413
|
-
idleTimer = setTimeout(() => expire({ kind: 'idle', limitMs, tests: progress.inFlight() }, queuedMs), leftMs);
|
|
559
|
+
}, Math.max(0, expiresAtMs - Date.now() - HANG_CAPTURE_MS));
|
|
414
560
|
};
|
|
415
|
-
const
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
if (
|
|
422
|
-
queuedSinceMs = Date.now();
|
|
561
|
+
const expiredAs = (kind) => kind === 'idle'
|
|
562
|
+
? { kind, limitMs: idleTimeoutMs ?? 0, tests: progress.inFlight() }
|
|
563
|
+
: kind === 'fixture'
|
|
564
|
+
? { kind, limitMs: fixture?.timeoutMs ?? 0 }
|
|
565
|
+
: { kind, limitMs: timeoutMs ?? 0 };
|
|
566
|
+
const stepTo = (event) => {
|
|
567
|
+
if (state.settled || state.expiry !== null)
|
|
423
568
|
return;
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
}
|
|
432
|
-
const now = Date.now();
|
|
433
|
-
queuedSinceMs = now;
|
|
434
|
-
if (idleCapture !== undefined) {
|
|
435
|
-
idleLeftMs = Math.max(0, idleCapture.expiresAtMs - now);
|
|
436
|
-
}
|
|
437
|
-
clearTimeout(idleTimer);
|
|
569
|
+
const idleBefore = bounds.idle;
|
|
570
|
+
bounds = stepBounds(bounds, {
|
|
571
|
+
...(idleTimeoutMs === undefined ? {} : { idleMs: idleTimeoutMs }),
|
|
572
|
+
...(fixture === null || lifecycle === null ? {} : { fixtureMs: fixture.timeoutMs }),
|
|
573
|
+
...(timeoutMs === undefined ? {} : { totalMs: timeoutMs }),
|
|
574
|
+
}, event);
|
|
575
|
+
if (bounds.idle !== idleBefore) {
|
|
438
576
|
cancelCapture();
|
|
439
577
|
idleCapture = undefined;
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
queuedMs += waitedMs;
|
|
448
|
-
queuedSinceMs = null;
|
|
449
|
-
if (!isTesting()) {
|
|
450
|
-
appendStderr(`bounded-exec: host CPU wait ended after ${waitedMs}ms; no test runner is active\n`);
|
|
451
|
-
return;
|
|
452
|
-
}
|
|
453
|
-
appendStderr(`bounded-exec: idle clock restarted: no descendant's work is queued for host CPU tokens after ${waitedMs}ms; ${idleLeftMs}ms of idleTimeoutMs=${idleTimeoutMs} left\n`);
|
|
454
|
-
scheduleIdle(idleTimeoutMs, idleLeftMs);
|
|
455
|
-
},
|
|
578
|
+
if (bounds.idle !== null)
|
|
579
|
+
scheduleCapture(bounds.idle);
|
|
580
|
+
}
|
|
581
|
+
clearTimeout(boundTimer);
|
|
582
|
+
const next = earliestBound(bounds);
|
|
583
|
+
if (next !== null)
|
|
584
|
+
boundTimer = setTimeout(() => expire(expiredAs(next.kind)), Math.max(0, next.at - Date.now()));
|
|
456
585
|
};
|
|
457
586
|
// Only the CHILD's own output is read for progress. The diagnostics below go
|
|
458
587
|
// through appendStderr as well, and re-arming on those would let a teardown
|
|
459
588
|
// message extend the very bound that just fired.
|
|
460
589
|
const updateIdle = (finished, wasTesting) => {
|
|
461
|
-
|
|
462
|
-
clearTimeout(idleTimer);
|
|
463
|
-
cancelCapture();
|
|
464
|
-
idleCapture = undefined;
|
|
465
|
-
}
|
|
466
|
-
else if (finished || !wasTesting) {
|
|
467
|
-
armIdleTimer();
|
|
468
|
-
}
|
|
590
|
+
stepTo({ kind: 'progress', at: Date.now(), testing: isTesting(), wasTesting, finished });
|
|
469
591
|
};
|
|
470
592
|
const scanProgress = (stream, text) => {
|
|
593
|
+
if (!fixtureDone)
|
|
594
|
+
return;
|
|
471
595
|
updateIdle(progress.scan(stream, text), isTesting());
|
|
472
596
|
};
|
|
473
597
|
if (lifecycle !== null) {
|
|
@@ -481,7 +605,8 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
|
|
|
481
605
|
appendStderr(`bounded-exec: runner lifecycle failed: ${detail}\n`);
|
|
482
606
|
updateIdle(false, wasTesting);
|
|
483
607
|
};
|
|
484
|
-
createInterface({ input: channel })
|
|
608
|
+
const lines = createInterface({ input: channel });
|
|
609
|
+
lines.on('line', (line) => {
|
|
485
610
|
const wasTesting = isTesting();
|
|
486
611
|
const event = lifecycle.accept(line);
|
|
487
612
|
if (event === null) {
|
|
@@ -489,9 +614,18 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
|
|
|
489
614
|
return;
|
|
490
615
|
}
|
|
491
616
|
updateIdle(false, wasTesting);
|
|
617
|
+
if (event.id === 0 && event.phase === 'terminal' && fixture !== null) {
|
|
618
|
+
fixtureDone = true;
|
|
619
|
+
stepTo({ kind: 'fixture-done' });
|
|
620
|
+
}
|
|
621
|
+
if (event.id > 0)
|
|
622
|
+
mainStarted = true;
|
|
492
623
|
// Tool startup and native lifecycle-bearing stdout wait for their phase acknowledgement.
|
|
493
624
|
channel.write(`${JSON.stringify(event)}\n`);
|
|
494
625
|
});
|
|
626
|
+
// readline re-emits its input's errors on the interface, where one with no listener is thrown: a
|
|
627
|
+
// runner killed with unread lifecycle bytes resets the channel (ECONNRESET on Linux).
|
|
628
|
+
lines.on('error', (error) => failLifecycle(error.message));
|
|
495
629
|
channel.on('error', (error) => failLifecycle(error.message));
|
|
496
630
|
channel.on('end', () => {
|
|
497
631
|
if (!lifecycle.finished())
|
|
@@ -514,22 +648,30 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
|
|
|
514
648
|
state.noSpace = sight(said);
|
|
515
649
|
appendStderr(text);
|
|
516
650
|
});
|
|
517
|
-
|
|
651
|
+
// The graceful pass: SIGTERM to the whole group. A signal that fails is told, not thrown: every caller has more to do
|
|
652
|
+
// after it, `terminate` the force-kill and the signal handler its releases and the signal it gives back.
|
|
653
|
+
const killChildTree = async () => {
|
|
518
654
|
const pid = child.pid;
|
|
519
655
|
if (!pid) {
|
|
520
656
|
return;
|
|
521
657
|
}
|
|
522
|
-
|
|
658
|
+
try {
|
|
659
|
+
await killer.kill(pid, 'SIGTERM');
|
|
660
|
+
}
|
|
661
|
+
catch (error) {
|
|
662
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
663
|
+
tell(`Terminating process group ${pid} failed: ${reason}\n`);
|
|
664
|
+
}
|
|
523
665
|
};
|
|
524
666
|
const onProcessExit = () => {
|
|
525
|
-
void killChildTree(
|
|
667
|
+
void killChildTree();
|
|
526
668
|
};
|
|
527
669
|
const onTerminationSignal = (signal) => {
|
|
528
670
|
removeSignalHandlers();
|
|
529
671
|
cancelCapture();
|
|
530
|
-
void killChildTree(
|
|
672
|
+
void killChildTree()
|
|
531
673
|
.finally(releaseCpu)
|
|
532
|
-
.finally(releaseLease)
|
|
674
|
+
.finally(() => releaseLease(true))
|
|
533
675
|
.finally(() => process.kill(process.pid, signal));
|
|
534
676
|
};
|
|
535
677
|
const onSigint = () => onTerminationSignal('SIGINT');
|
|
@@ -565,22 +707,23 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
|
|
|
565
707
|
// the reader unable to tell a wedged toolchain from a loaded machine — the
|
|
566
708
|
// two have opposite fixes.
|
|
567
709
|
let termination = Promise.resolve();
|
|
568
|
-
|
|
569
|
-
const expire = (bound, queuedMs = 0) => {
|
|
710
|
+
const expire = (bound) => {
|
|
570
711
|
if (state.settled || state.expiry !== null) {
|
|
571
712
|
return;
|
|
572
713
|
}
|
|
573
714
|
state.expiry = bound;
|
|
574
|
-
clearTimeout(
|
|
575
|
-
clearTimeout(idleTimer);
|
|
715
|
+
clearTimeout(boundTimer);
|
|
576
716
|
const elapsedMs = Date.now() - startedAt;
|
|
577
717
|
appendStderr(bound.kind === 'idle'
|
|
578
|
-
? `\nCommand made no progress: no test finished for ${bound.limitMs}ms (idleTimeoutMs=${bound.limitMs})
|
|
718
|
+
? `\nCommand made no progress: no test finished for ${bound.limitMs}ms (idleTimeoutMs=${bound.limitMs}) after ${elapsedMs}ms of runtime; ${bound.tests.length > 0
|
|
579
719
|
? `still running: ${bound.tests.join(', ')}`
|
|
580
720
|
: 'the runner named no test still running'} (cwd=${cwd}): ${command}\n`
|
|
581
|
-
:
|
|
721
|
+
: bound.kind === 'fixture'
|
|
722
|
+
? `\nFixture command timed out after ${elapsedMs}ms (fixtureTimeoutMs=${bound.limitMs}, cwd=${cwd}): ${fixture?.command ?? command}\n`
|
|
723
|
+
: `\nCommand timed out after ${elapsedMs}ms (timeoutMs=${bound.limitMs}, cwd=${cwd}): ${command}\n`);
|
|
582
724
|
if (bound.kind === 'idle') {
|
|
583
725
|
const observed = idleCapture?.result;
|
|
726
|
+
state.lockWaits = observed?.kind === 'captured' ? observed.snapshot.lockWaits : [];
|
|
584
727
|
appendStderr(`Hang diagnostics: runner pid=${child.pid ?? 'unavailable'} pgid=${process.platform === 'win32' ? 'unavailable' : (child.pid ?? 'unavailable')} expiryAtMs=${Date.now()} scheduledExpiryAtMs=${idleCapture?.expiresAtMs ?? 'unavailable'}\n`);
|
|
585
728
|
if (observed?.kind === 'captured') {
|
|
586
729
|
appendStderr(`Hang diagnostics: outcome=${observed.snapshot.outcome} capturedAtMs=${observed.snapshot.capturedAtMs} actualExpiryOffsetMs=${Date.now() - observed.snapshot.capturedAtMs}\n${observed.snapshot.text}`);
|
|
@@ -594,55 +737,83 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
|
|
|
594
737
|
cancelCapture();
|
|
595
738
|
termination = terminate();
|
|
596
739
|
};
|
|
740
|
+
// Whether the group still lists members. A probe that fails proves nothing about the group, so it is told, and the
|
|
741
|
+
// group is taken as listed: a force-kill that was not needed costs nothing, and one that was is not skipped.
|
|
742
|
+
const groupListed = (pid) => {
|
|
743
|
+
try {
|
|
744
|
+
return processGroupHasMembers(pid);
|
|
745
|
+
}
|
|
746
|
+
catch (error) {
|
|
747
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
748
|
+
tell(`Probing process group ${pid} for members failed: ${reason}; taking it as still listing members\n`);
|
|
749
|
+
return true;
|
|
750
|
+
}
|
|
751
|
+
};
|
|
597
752
|
// A timed-out run ends with its whole process group, not with its leader.
|
|
598
753
|
// The graceful signal reaches every member that exists when it is sent, but
|
|
599
754
|
// a member that ignores SIGTERM, or one forked while the signal goes out
|
|
600
755
|
// (macOS does not hold a fork back for a group signal in flight), outlives
|
|
601
756
|
// a leader that exits on it. The grace ends when the leader exits or
|
|
602
757
|
// killAfterMs passes, whichever comes first; whatever is left then is killed.
|
|
758
|
+
// SIGKILL is no different: a member forked while it goes out survives it too,
|
|
759
|
+
// so the force-kill is repeated while the group lists members. What the repeats
|
|
760
|
+
// cannot end is said, and a failure of the probe or of a signal is shown.
|
|
761
|
+
const forceKill = async (pid) => {
|
|
762
|
+
try {
|
|
763
|
+
if (await killProcessGroup(pid, DEADLINE_PATIENCE, (group) => killer.kill(group, 'SIGKILL'))) {
|
|
764
|
+
tell(`${describeMembersLeft(pid)}\n`);
|
|
765
|
+
}
|
|
766
|
+
}
|
|
767
|
+
catch (error) {
|
|
768
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
769
|
+
tell(`Force-kill of process group ${pid} failed: ${reason}\n`);
|
|
770
|
+
}
|
|
771
|
+
};
|
|
603
772
|
const terminate = async () => {
|
|
604
|
-
await
|
|
773
|
+
await killChildTree();
|
|
605
774
|
if (!state.settled && killAfterMs > 0) {
|
|
606
775
|
await settledWithin(exitPromise, killAfterMs);
|
|
607
776
|
}
|
|
608
777
|
if (state.settled) {
|
|
609
|
-
if (child.pid !== undefined &&
|
|
778
|
+
if (child.pid !== undefined && groupListed(child.pid)) {
|
|
610
779
|
state.forceKillNeeded = true;
|
|
611
780
|
appendStderr(`Force-killing what the ${state.expiry === null ? 'command' : 'timed-out command'} left in its process group: ${command}\n`);
|
|
612
|
-
await
|
|
781
|
+
await forceKill(child.pid);
|
|
613
782
|
}
|
|
614
783
|
return;
|
|
615
784
|
}
|
|
616
785
|
state.forceKillNeeded = true;
|
|
617
786
|
appendStderr(`Force-killing ${state.expiry === null ? 'command' : 'timed out command'} after killAfterMs=${killAfterMs}: ${command}\n`);
|
|
618
|
-
|
|
619
|
-
|
|
787
|
+
if (child.pid !== undefined)
|
|
788
|
+
await forceKill(child.pid);
|
|
789
|
+
// The repeats wait for the group to empty, the leader with it. Windows has no process groups, so there they have
|
|
790
|
+
// nothing to wait on once the one signal is sent, and the leader's exit is awaited as it always was.
|
|
791
|
+
if (process.platform === 'win32' || child.pid === undefined) {
|
|
792
|
+
await settledWithin(exitPromise, REAP_AFTER_FORCE_KILL_MS);
|
|
793
|
+
}
|
|
620
794
|
if (!state.settled) {
|
|
621
795
|
resolveExit({ code: 1, signal: 'SIGKILL' });
|
|
622
796
|
}
|
|
623
797
|
};
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
//
|
|
630
|
-
//
|
|
631
|
-
// command runs, is said and changes nothing else; only a bound or a signal stops the command. Our own release after
|
|
632
|
-
// the command settled is not a loss; `cpuReleased` marks it.
|
|
798
|
+
// The fixture's ceiling lives and dies with its segment: the segment's terminal clears it, so it can never
|
|
799
|
+
// fire on a command the fixture already handed over to.
|
|
800
|
+
stepTo({ kind: 'spawn', at: Date.now(), testing: isTesting(), fixture: fixture !== null && lifecycle !== null });
|
|
801
|
+
// A grant's connection is a ledger entry, not the command's owner: the lending parent's exit ends it, and what runs
|
|
802
|
+
// under it runs on over budget until it finishes. Its close, while the command runs, is said and changes nothing
|
|
803
|
+
// else; only a bound or a signal stops the command. Our own release after the command settled is not a loss;
|
|
804
|
+
// `cpuReleased` marks it.
|
|
633
805
|
if (grant !== null) {
|
|
634
806
|
const held = grant;
|
|
635
807
|
void held.closed.then(() => {
|
|
636
808
|
if (state.settled || cpuReleased !== null) {
|
|
637
809
|
return;
|
|
638
810
|
}
|
|
639
|
-
appendStderr(`cowshed: cpu-
|
|
811
|
+
appendStderr(`cowshed: cpu-pool grant ended: the parent pool closed its ${held.tokens}-token grant while the command ran; it runs on over budget until it ends: ${command}\n`);
|
|
640
812
|
});
|
|
641
813
|
}
|
|
642
814
|
const { code: exitCode, signal: exitSignal } = await exitPromise;
|
|
643
815
|
await termination;
|
|
644
|
-
clearTimeout(
|
|
645
|
-
clearTimeout(idleTimer);
|
|
816
|
+
clearTimeout(boundTimer);
|
|
646
817
|
cancelCapture();
|
|
647
818
|
removeSignalHandlers();
|
|
648
819
|
// The runner is gone. Its sub-pool closes first, so no descendant still draws on tokens the grant returns;
|
|
@@ -650,7 +821,9 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
|
|
|
650
821
|
await releaseCpu();
|
|
651
822
|
const code = exitCode ?? signalToExitCode(exitSignal);
|
|
652
823
|
if (code !== 0 && state.expiry === null) {
|
|
653
|
-
appendStderr(
|
|
824
|
+
appendStderr(fixture !== null && !mainStarted
|
|
825
|
+
? `Fixture command exited with status ${code}: ${fixture.command}\n`
|
|
826
|
+
: `Command exited with status ${code}: ${command}\n`);
|
|
654
827
|
}
|
|
655
828
|
if (state.expiry !== null && !state.forceKillNeeded) {
|
|
656
829
|
appendStderr(`Timed out command exited after graceful termination: ${command}\n`);
|
|
@@ -668,11 +841,11 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
|
|
|
668
841
|
appendStderr(`${describeRamTempError(full)}\n`);
|
|
669
842
|
}
|
|
670
843
|
}
|
|
671
|
-
await releaseLease();
|
|
672
|
-
// Broken resource ownership
|
|
673
|
-
// its verdict is an ordinary operational failure, never a successful or silently unbounded run.
|
|
674
|
-
const operationalFailure = state.poolCloseError !== null || lifecycleFailure !== null;
|
|
675
|
-
await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure, seen);
|
|
844
|
+
await releaseLease(code !== 0 || state.expiry !== null || state.poolCloseError !== null || lifecycleFailure !== null);
|
|
845
|
+
// Broken resource ownership, a lost native lifecycle, or leftovers that could not be retired fail even if a shell
|
|
846
|
+
// branch later exited 0: its verdict is an ordinary operational failure, never a successful or silently unbounded run.
|
|
847
|
+
const operationalFailure = state.poolCloseError !== null || lifecycleFailure !== null || state.retireError !== null;
|
|
848
|
+
await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure, seen, state.lockWaits);
|
|
676
849
|
return {
|
|
677
850
|
success: !operationalFailure && state.expiry === null && code === 0,
|
|
678
851
|
terminalOutput: outputChunks.join(''),
|
|
@@ -680,44 +853,11 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
|
|
|
680
853
|
}
|
|
681
854
|
export function createProcessTreeKiller() {
|
|
682
855
|
return {
|
|
683
|
-
kill(pid, signal) {
|
|
684
|
-
|
|
856
|
+
async kill(pid, signal) {
|
|
857
|
+
signalProcessGroup(pid, signal);
|
|
685
858
|
},
|
|
686
859
|
};
|
|
687
860
|
}
|
|
688
|
-
function killProcessGroup(pid, signal) {
|
|
689
|
-
try {
|
|
690
|
-
// On POSIX the executor spawns with detached: true, so the child leads a
|
|
691
|
-
// process group of its own and -pid signals all of it at once, including
|
|
692
|
-
// grandchildren that tree-walk libraries miss when parents die and
|
|
693
|
-
// children get reparented to init. Only the group is named: the leader may
|
|
694
|
-
// already be reaped, and its pid reused by an unrelated process, while the
|
|
695
|
-
// group lives on in its other members. Windows has no process groups.
|
|
696
|
-
process.kill(process.platform === 'win32' ? pid : -pid, signal);
|
|
697
|
-
}
|
|
698
|
-
catch {
|
|
699
|
-
// ESRCH: the group (or the Windows process) has already exited.
|
|
700
|
-
}
|
|
701
|
-
return Promise.resolve();
|
|
702
|
-
}
|
|
703
|
-
/**
|
|
704
|
-
* Whether any process is left in the group `pgid` leads. A group outlives its
|
|
705
|
-
* reaped leader for as long as one member does, and no process can take its id
|
|
706
|
-
* meanwhile. Windows has no process groups, so there is nothing left to name.
|
|
707
|
-
*/
|
|
708
|
-
function processGroupHasMembers(pgid) {
|
|
709
|
-
if (process.platform === 'win32') {
|
|
710
|
-
return false;
|
|
711
|
-
}
|
|
712
|
-
try {
|
|
713
|
-
process.kill(-pgid, 0);
|
|
714
|
-
return true;
|
|
715
|
-
}
|
|
716
|
-
catch (error) {
|
|
717
|
-
// EPERM: a member exists but runs as another user.
|
|
718
|
-
return error instanceof Error && 'code' in error && error.code === 'EPERM';
|
|
719
|
-
}
|
|
720
|
-
}
|
|
721
861
|
function resolveCwd(cwd, root) {
|
|
722
862
|
if (!cwd) {
|
|
723
863
|
return root;
|
|
@@ -737,10 +877,10 @@ function buildCommand(options) {
|
|
|
737
877
|
}
|
|
738
878
|
return parts.join(' ');
|
|
739
879
|
}
|
|
740
|
-
/**
|
|
741
|
-
function mergeEnv(
|
|
880
|
+
/** Owned scratch is the command's TMPDIR unless the target names one itself. */
|
|
881
|
+
function mergeEnv(directory, env) {
|
|
742
882
|
const inherited = withoutPoolEnv(process.env);
|
|
743
|
-
return
|
|
883
|
+
return directory === null ? { ...inherited, ...env } : { ...inherited, TMPDIR: directory, ...env };
|
|
744
884
|
}
|
|
745
885
|
/** The process's environment without the CPU pool's descriptor: a pool and its count only a grant may give. */
|
|
746
886
|
function withoutPoolEnv(env) {
|
|
@@ -770,14 +910,6 @@ function signalToExitCode(signal) {
|
|
|
770
910
|
}
|
|
771
911
|
return EXIT_CODE_BY_SIGNAL[signal] ?? 1;
|
|
772
912
|
}
|
|
773
|
-
async function ignoreKillError(promise) {
|
|
774
|
-
try {
|
|
775
|
-
await promise;
|
|
776
|
-
}
|
|
777
|
-
catch {
|
|
778
|
-
// The process tree may have already exited between timeout and kill.
|
|
779
|
-
}
|
|
780
|
-
}
|
|
781
913
|
/** Resolves when `promise` settles or `ms` passes, whichever is first, leaving no timer behind. */
|
|
782
914
|
async function settledWithin(promise, ms) {
|
|
783
915
|
let timer;
|