@smoothbricks/nx-plugin 0.5.7 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1086 -360
- package/dist/.build-inputs.json +138 -0
- package/dist/bin/smoo-nx-cargo-toolchain.d.ts +3 -0
- package/dist/bin/smoo-nx-cargo-toolchain.d.ts.map +1 -0
- package/dist/bin/smoo-nx-cargo-toolchain.js +8 -0
- package/dist/bin/smoo-nx-forbidden-spellings.d.ts +3 -0
- package/dist/bin/smoo-nx-forbidden-spellings.d.ts.map +1 -0
- package/dist/bin/smoo-nx-forbidden-spellings.js +32 -0
- package/dist/bin/smoo-nx-nextest-timings.d.ts +3 -0
- package/dist/bin/smoo-nx-nextest-timings.d.ts.map +1 -0
- package/dist/bin/smoo-nx-nextest-timings.js +28 -0
- package/dist/bounded-test-policy.d.ts +1 -1
- package/dist/bounded-test-policy.d.ts.map +1 -1
- package/dist/bounded-test-policy.js +14 -15
- package/dist/build-state-output-policy.d.ts +95 -1
- package/dist/build-state-output-policy.d.ts.map +1 -1
- package/dist/build-state-output-policy.js +368 -30
- package/dist/bun-lock-scope.d.ts +44 -0
- package/dist/bun-lock-scope.d.ts.map +1 -0
- package/dist/bun-lock-scope.js +97 -0
- package/dist/cargo-build-script-env.d.ts +49 -0
- package/dist/cargo-build-script-env.d.ts.map +1 -0
- package/dist/cargo-build-script-env.js +324 -0
- package/dist/cargo-closure-input.d.ts +31 -4
- package/dist/cargo-closure-input.d.ts.map +1 -1
- package/dist/cargo-closure-input.js +66 -6
- package/dist/cargo-retention-hook.d.ts +17 -0
- package/dist/cargo-retention-hook.d.ts.map +1 -0
- package/dist/cargo-retention-hook.js +85 -0
- package/dist/cargo-source-hash.d.ts +25 -7
- package/dist/cargo-source-hash.d.ts.map +1 -1
- package/dist/cargo-source-hash.js +108 -43
- package/dist/cargo-toolchain-paths.d.ts +40 -0
- package/dist/cargo-toolchain-paths.d.ts.map +1 -0
- package/dist/cargo-toolchain-paths.js +84 -0
- package/dist/cargo-toolchain-policy.d.ts +25 -8
- package/dist/cargo-toolchain-policy.d.ts.map +1 -1
- package/dist/cargo-toolchain-policy.js +39 -5
- package/dist/cargo-workspace.d.ts +10 -4
- package/dist/cargo-workspace.d.ts.map +1 -1
- package/dist/cargo-workspace.js +65 -95
- package/dist/cross-check-policy.d.ts +9 -9
- package/dist/cross-check-policy.d.ts.map +1 -1
- package/dist/cross-check-policy.js +11 -11
- package/dist/disk-command-policy.d.ts.map +1 -1
- package/dist/disk-command-policy.js +1 -0
- package/dist/ensure-built-receipt.d.ts +13 -5
- package/dist/ensure-built-receipt.d.ts.map +1 -1
- package/dist/ensure-built-receipt.js +126 -47
- package/dist/ensure-built-witness.d.ts +27 -8
- package/dist/ensure-built-witness.d.ts.map +1 -1
- package/dist/ensure-built-witness.js +45 -15
- package/dist/ensure-built.d.ts.map +1 -1
- package/dist/ensure-built.js +9 -1
- package/dist/eslint-config-closure.d.ts +7 -0
- package/dist/eslint-config-closure.d.ts.map +1 -0
- package/dist/eslint-config-closure.js +346 -0
- package/dist/executors/bounded-exec/bounds.d.ts +56 -0
- package/dist/executors/bounded-exec/bounds.d.ts.map +1 -0
- package/dist/executors/bounded-exec/bounds.js +43 -0
- package/dist/executors/bounded-exec/cpu-pool.d.ts +33 -36
- package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool.js +54 -119
- package/dist/executors/bounded-exec/executor.d.ts +19 -12
- package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
- package/dist/executors/bounded-exec/executor.js +371 -239
- package/dist/executors/bounded-exec/gateway-lease.d.ts +22 -19
- package/dist/executors/bounded-exec/gateway-lease.d.ts.map +1 -1
- package/dist/executors/bounded-exec/gateway-lease.js +81 -28
- package/dist/executors/bounded-exec/hang-diagnostics.d.ts +18 -1
- package/dist/executors/bounded-exec/hang-diagnostics.d.ts.map +1 -1
- package/dist/executors/bounded-exec/hang-diagnostics.js +148 -30
- package/dist/executors/bounded-exec/lease-wait.d.ts +43 -0
- package/dist/executors/bounded-exec/lease-wait.d.ts.map +1 -0
- package/dist/executors/bounded-exec/lease-wait.js +68 -0
- package/dist/executors/bounded-exec/lock-wait.d.ts +51 -0
- package/dist/executors/bounded-exec/lock-wait.d.ts.map +1 -0
- package/dist/executors/bounded-exec/lock-wait.js +108 -0
- package/dist/executors/bounded-exec/owned-subtree.d.ts +84 -0
- package/dist/executors/bounded-exec/owned-subtree.d.ts.map +1 -0
- package/dist/executors/bounded-exec/owned-subtree.js +178 -0
- package/dist/executors/bounded-exec/progress.d.ts +4 -1
- package/dist/executors/bounded-exec/progress.d.ts.map +1 -1
- package/dist/executors/bounded-exec/progress.js +5 -2
- package/dist/executors/bounded-exec/ram-attach.d.ts +19 -0
- package/dist/executors/bounded-exec/ram-attach.d.ts.map +1 -0
- package/dist/executors/bounded-exec/ram-attach.js +207 -0
- package/dist/executors/bounded-exec/ram-detach.d.ts +25 -0
- package/dist/executors/bounded-exec/ram-detach.d.ts.map +1 -0
- package/dist/executors/bounded-exec/ram-detach.js +207 -0
- package/dist/executors/bounded-exec/ram-temp-fixtures.d.ts +76 -0
- package/dist/executors/bounded-exec/ram-temp-fixtures.d.ts.map +1 -1
- package/dist/executors/bounded-exec/ram-temp-fixtures.js +296 -0
- package/dist/executors/bounded-exec/ram-temp.d.ts +388 -88
- package/dist/executors/bounded-exec/ram-temp.d.ts.map +1 -1
- package/dist/executors/bounded-exec/ram-temp.js +1213 -399
- package/dist/executors/bounded-exec/run-cpu-pool.d.ts +4 -4
- package/dist/executors/bounded-exec/run-cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/run-cpu-pool.js +20 -30
- package/dist/executors/bounded-exec/runner-hook.js +14 -5
- package/dist/executors/bounded-exec/runner-lifecycle.d.ts +14 -1
- package/dist/executors/bounded-exec/runner-lifecycle.d.ts.map +1 -1
- package/dist/executors/bounded-exec/runner-lifecycle.js +27 -6
- package/dist/executors/bounded-exec/{cpu-tokens.d.ts → runner-sizing.d.ts} +14 -42
- package/dist/executors/bounded-exec/runner-sizing.d.ts.map +1 -0
- package/dist/executors/bounded-exec/{cpu-tokens.js → runner-sizing.js} +5 -35
- package/dist/executors/bounded-exec/runtime-pin.d.ts +73 -0
- package/dist/executors/bounded-exec/runtime-pin.d.ts.map +1 -0
- package/dist/executors/bounded-exec/runtime-pin.js +167 -0
- package/dist/executors/bounded-exec/schema.d.ts +18 -1
- package/dist/executors/bounded-exec/schema.d.ts.map +1 -1
- package/dist/executors/bounded-exec/verdict.d.ts +40 -11
- package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
- package/dist/executors/bounded-exec/verdict.js +88 -19
- package/dist/executors/cargo-zigbuild/executor.d.ts +21 -0
- package/dist/executors/cargo-zigbuild/executor.d.ts.map +1 -1
- package/dist/executors/cargo-zigbuild/executor.js +67 -12
- package/dist/executors/clean-outputs/executor.js +1 -0
- package/dist/executors/scriptc-native/executor.d.ts +79 -0
- package/dist/executors/scriptc-native/executor.d.ts.map +1 -0
- package/dist/executors/scriptc-native/executor.js +367 -0
- package/dist/executors/scriptc-native/ffi-paths.d.ts +9 -0
- package/dist/executors/scriptc-native/ffi-paths.d.ts.map +1 -0
- package/dist/executors/scriptc-native/ffi-paths.js +25 -0
- package/dist/executors/scriptc-native/schema.d.ts +34 -0
- package/dist/executors/scriptc-native/schema.d.ts.map +1 -0
- package/dist/executors/scriptc-native/schema.js +1 -0
- package/dist/executors/typescript-emit/executor.d.ts +3 -1
- package/dist/executors/typescript-emit/executor.d.ts.map +1 -1
- package/dist/executors/typescript-emit/executor.js +164 -33
- package/dist/forbidden-spelling-policy.d.ts +56 -0
- package/dist/forbidden-spelling-policy.d.ts.map +1 -0
- package/dist/forbidden-spelling-policy.js +176 -0
- package/dist/generators/create-package/generator.d.ts.map +1 -1
- package/dist/generators/create-package/generator.js +3 -4
- package/dist/graph-input-contract.d.ts +14 -0
- package/dist/graph-input-contract.d.ts.map +1 -1
- package/dist/graph-input-contract.js +67 -2
- package/dist/index.d.ts +23 -6
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +672 -214
- package/dist/managed-assets.d.ts +2 -0
- package/dist/managed-assets.d.ts.map +1 -1
- package/dist/managed-files/files.d.ts +3 -3
- package/dist/managed-files/files.d.ts.map +1 -1
- package/dist/managed-files/files.js +28 -3
- package/dist/managed-files/managed-format.d.ts.map +1 -1
- package/dist/managed-files/managed-format.js +1 -0
- package/dist/nextest-code-bound.js +1 -0
- package/dist/nextest-timings.d.ts +35 -0
- package/dist/nextest-timings.d.ts.map +1 -0
- package/dist/nextest-timings.js +168 -0
- package/dist/owned-installation.d.ts +6 -5
- package/dist/owned-installation.d.ts.map +1 -1
- package/dist/owned-installation.js +150 -26
- package/dist/package-target-policy.d.ts.map +1 -1
- package/dist/package-target-policy.js +2 -3
- package/dist/process-children.d.ts +7 -0
- package/dist/process-children.d.ts.map +1 -0
- package/dist/process-children.darwin.d.ts +7 -0
- package/dist/process-children.darwin.d.ts.map +1 -0
- package/dist/process-children.darwin.js +13 -0
- package/dist/process-children.js +14 -0
- package/dist/process-children.linux.d.ts +8 -0
- package/dist/process-children.linux.d.ts.map +1 -0
- package/dist/process-children.linux.js +25 -0
- package/dist/process-group.d.ts +108 -0
- package/dist/process-group.d.ts.map +1 -0
- package/dist/process-group.js +171 -0
- package/dist/process-listing.d.ts +7 -0
- package/dist/process-listing.d.ts.map +1 -0
- package/dist/process-listing.js +51 -0
- package/dist/producer-receipts.d.ts +8 -2
- package/dist/producer-receipts.d.ts.map +1 -1
- package/dist/producer-receipts.js +75 -48
- package/dist/task-graph-policy.d.ts +21 -0
- package/dist/task-graph-policy.d.ts.map +1 -0
- package/dist/task-graph-policy.js +91 -0
- package/dist/test-timings.js +4 -4
- package/dist/test-walk-policy.d.ts +14 -5
- package/dist/test-walk-policy.d.ts.map +1 -1
- package/dist/test-walk-policy.js +8 -2
- package/dist/testing.d.ts +73 -28
- package/dist/testing.d.ts.map +1 -1
- package/dist/testing.js +217 -223
- package/dist/tool-bins.d.ts +33 -0
- package/dist/tool-bins.d.ts.map +1 -0
- package/dist/tool-bins.js +72 -0
- package/dist/typecheck-test-policy.d.ts.map +1 -1
- package/dist/typecheck-test-policy.js +3 -4
- package/dist/wait-why-policy.d.ts +28 -0
- package/dist/wait-why-policy.d.ts.map +1 -0
- package/dist/wait-why-policy.js +223 -0
- package/dist/workspace-config-policy.d.ts +14 -8
- package/dist/workspace-config-policy.d.ts.map +1 -1
- package/dist/workspace-config-policy.js +60 -16
- package/dist/workspace-manifest.d.ts +41 -6
- package/dist/workspace-manifest.d.ts.map +1 -1
- package/dist/workspace-manifest.js +57 -0
- package/executors.json +16 -1
- package/loader.js +2 -0
- package/managed/raw/envrc +11 -0
- package/managed/raw/patches/nx@23.2.1.patch +2840 -199
- package/managed/raw/patches/ttsc@0.34.1.patch +119 -0
- package/managed/raw/tooling/direnv/cargo-fetch-bound.ts +245 -0
- package/managed/raw/tooling/direnv/cargo-fetch-run.ts +299 -0
- package/managed/raw/tooling/direnv/compiler-variables.sh +42 -0
- package/managed/raw/tooling/direnv/developer-links.ts +464 -25
- package/managed/raw/tooling/direnv/devenv.smoo.nix +78 -38
- package/managed/raw/tooling/direnv/github-actions-bootstrap.sh +14 -12
- package/managed/raw/tooling/direnv/installed-bun-links.ts +308 -0
- package/managed/raw/tooling/direnv/linked-snapshots.ts +62 -3
- package/managed/raw/tooling/direnv/nx-socket-dir.sh +56 -17
- package/managed/raw/tooling/direnv/repo-path +68 -6
- package/managed/raw/tooling/direnv/secret-references.ts +13 -4
- package/managed/raw/tooling/direnv/setup-environment.ts +128 -537
- package/managed/raw/tooling/direnv/toolchain-stamp.ts +4 -2
- package/managed/raw/tooling/direnv/ttsc-env.sh +9 -0
- package/managed/raw/tooling/direnv/uv-relocation.ts +227 -0
- package/managed/raw/tooling/git-hooks/commit-msg.sh +2 -1
- package/managed/raw/tooling/git-hooks/pre-commit.sh +2 -1
- package/managed/raw/tooling/git-hooks/pre-push.sh +2 -1
- package/managed/templates/github/actions/cache-ttsc-plugins/action.yml +85 -7
- package/managed/templates/github/actions/save-nix-devenv/action.yml +12 -2
- package/managed/templates/github/actions/setup-devenv/action.yml +2 -0
- package/package.json +77 -13
- package/src/bin/smoo-nx-cargo-toolchain.ts +11 -0
- package/src/bin/smoo-nx-forbidden-spellings.ts +33 -0
- package/src/bin/smoo-nx-nextest-timings.ts +31 -0
- package/src/bounded-test-policy.ts +12 -14
- package/src/build-state-output-policy.ts +458 -34
- package/src/bun-lock-scope.ts +108 -0
- package/src/cargo-build-script-env.ts +265 -0
- package/src/cargo-closure-input.ts +74 -7
- package/src/cargo-retention-hook.ts +94 -0
- package/src/cargo-source-hash.ts +138 -41
- package/src/cargo-toolchain-paths.ts +95 -0
- package/src/cargo-toolchain-policy.ts +49 -5
- package/src/cargo-workspace.ts +72 -123
- package/src/cross-check-policy.ts +12 -10
- package/src/disk-command-policy.ts +1 -0
- package/src/ensure-built-receipt.ts +132 -39
- package/src/ensure-built-witness.ts +62 -10
- package/src/ensure-built.ts +11 -1
- package/src/eslint-config-closure.ts +357 -0
- package/src/executors/bounded-exec/bounds.ts +76 -0
- package/src/executors/bounded-exec/cpu-pool.ts +51 -117
- package/src/executors/bounded-exec/executor.ts +423 -259
- package/src/executors/bounded-exec/gateway-lease.ts +82 -33
- package/src/executors/bounded-exec/hang-diagnostics.ts +173 -31
- package/src/executors/bounded-exec/lease-wait.ts +89 -0
- package/src/executors/bounded-exec/lock-wait.ts +141 -0
- package/src/executors/bounded-exec/owned-subtree.ts +235 -0
- package/src/executors/bounded-exec/progress.ts +9 -3
- package/src/executors/bounded-exec/ram-attach.ts +238 -0
- package/src/executors/bounded-exec/ram-detach.ts +226 -0
- package/src/executors/bounded-exec/ram-temp-fixtures.ts +358 -0
- package/src/executors/bounded-exec/ram-temp-registry.txt +440 -0
- package/src/executors/bounded-exec/ram-temp.ts +1497 -460
- package/src/executors/bounded-exec/run-cpu-pool.ts +22 -30
- package/src/executors/bounded-exec/runner-hook.ts +14 -6
- package/src/executors/bounded-exec/runner-lifecycle.ts +31 -8
- package/src/executors/bounded-exec/{cpu-tokens.ts → runner-sizing.ts} +13 -59
- package/src/executors/bounded-exec/runtime-pin.ts +232 -0
- package/src/executors/bounded-exec/schema.json +11 -2
- package/src/executors/bounded-exec/schema.ts +19 -1
- package/src/executors/bounded-exec/verdict.ts +137 -34
- package/src/executors/cargo-zigbuild/executor.ts +82 -8
- package/src/executors/clean-outputs/executor.ts +1 -0
- package/src/executors/scriptc-native/executor.ts +394 -0
- package/src/executors/scriptc-native/ffi-paths.ts +29 -0
- package/src/executors/scriptc-native/schema.json +60 -0
- package/src/executors/scriptc-native/schema.ts +34 -0
- package/src/executors/typescript-emit/executor.ts +205 -38
- package/src/forbidden-spelling-policy.ts +226 -0
- package/src/generators/create-package/generator.ts +7 -7
- package/src/graph-input-contract.ts +61 -2
- package/src/index.ts +788 -224
- package/src/managed-assets.ts +3 -0
- package/src/managed-files/files.ts +28 -3
- package/src/managed-files/managed-format.ts +1 -0
- package/src/nextest-code-bound.ts +1 -0
- package/src/nextest-timings.ts +106 -0
- package/src/owned-installation.ts +150 -23
- package/src/package-target-policy.ts +2 -3
- package/src/process-children.darwin.ts +14 -0
- package/src/process-children.linux.ts +33 -0
- package/src/process-children.ts +22 -0
- package/src/process-group.ts +209 -0
- package/src/process-listing.ts +54 -0
- package/src/producer-receipts.ts +68 -45
- package/src/task-graph-policy.ts +98 -0
- package/src/test-walk-policy.ts +23 -8
- package/src/testing.ts +240 -232
- package/src/tool-bins.ts +108 -0
- package/src/typecheck-test-policy.ts +2 -3
- package/src/wait-why-policy.ts +226 -0
- package/src/workspace-config-policy.ts +63 -15
- package/src/workspace-manifest.ts +89 -6
- package/test-timings.json +122 -98
- package/dist/bin/smoo-nx-failures.d.ts +0 -3
- package/dist/bin/smoo-nx-failures.d.ts.map +0 -1
- package/dist/bin/smoo-nx-failures.js +0 -3
- package/dist/bin/smoo-nx-nextest-extract.d.ts +0 -3
- package/dist/bin/smoo-nx-nextest-extract.d.ts.map +0 -1
- package/dist/bin/smoo-nx-nextest-extract.js +0 -42
- package/dist/executors/bounded-exec/cpu-tokens.d.ts.map +0 -1
- package/dist/failure-summary.d.ts +0 -5
- package/dist/failure-summary.d.ts.map +0 -1
- package/dist/failure-summary.js +0 -118
- package/dist/nextest-extraction.d.ts +0 -117
- package/dist/nextest-extraction.d.ts.map +0 -1
- package/dist/nextest-extraction.js +0 -303
- package/managed/raw/patches/ttsc@0.30.4.patch +0 -2255
- package/src/__tests__/cargo-closure-filesets.ts +0 -94
- package/src/__tests__/cargo-closure-fixture.ts +0 -113
- package/src/__tests__/counted-cargo-process.ts +0 -286
- package/src/__tests__/counted-cargo.ts +0 -713
- package/src/__tests__/directory-change.ts +0 -27
- package/src/__tests__/fixture-cargo-home.ts +0 -36
- package/src/__tests__/fixture-nx-env.ts +0 -193
- package/src/__tests__/owned-installation.ts +0 -13
- package/src/__tests__/past-tick.ts +0 -12
- package/src/__tests__/progress-clock.ts +0 -43
- package/src/__tests__/runtime-input.ts +0 -23
- package/src/bin/smoo-nx-bound-failures.test.ts +0 -127
- package/src/bin/smoo-nx-failures.ts +0 -4
- package/src/bin/smoo-nx-nextest-extract.ts +0 -44
- package/src/bounded-test-policy.test.ts +0 -1029
- package/src/build-state-output-policy.test.ts +0 -115
- package/src/bun-bootstrap.test.ts +0 -102
- package/src/bun-file-packages.test.ts +0 -47
- package/src/cargo-closure-filesets.test.ts +0 -63
- package/src/cargo-closure-input.property.test.ts +0 -50
- package/src/cargo-closure-input.test.ts +0 -506
- package/src/cargo-closure-key.test.ts +0 -240
- package/src/cargo-resolve-cache.test.ts +0 -1181
- package/src/cargo-source-hash.test.ts +0 -707
- package/src/cargo-workspace.test.ts +0 -615
- package/src/cross-check-policy.test.ts +0 -316
- package/src/disk-command-policy.test.ts +0 -61
- package/src/ensure-built-receipt.test.ts +0 -279
- package/src/ensure-built-witness.test.ts +0 -688
- package/src/ensure-built.test.ts +0 -1462
- package/src/executors/all-verdicts/executor.test.ts +0 -135
- package/src/executors/bounded-exec/cargo-progress-host.test.ts +0 -332
- package/src/executors/bounded-exec/cpu-pool.test.ts +0 -684
- package/src/executors/bounded-exec/cpu-tokens.test.ts +0 -519
- package/src/executors/bounded-exec/disk-lease.test.ts +0 -219
- package/src/executors/bounded-exec/executor-host.test.ts +0 -800
- package/src/executors/bounded-exec/executor.test.ts +0 -1819
- package/src/executors/bounded-exec/go-progress-host.test.ts +0 -171
- package/src/executors/bounded-exec/hang-diagnostics.test.ts +0 -291
- package/src/executors/bounded-exec/progress.test.ts +0 -139
- package/src/executors/bounded-exec/ram-temp-held.test.ts +0 -264
- package/src/executors/bounded-exec/ram-temp-host.test.ts +0 -635
- package/src/executors/bounded-exec/ram-temp.test.ts +0 -750
- package/src/executors/bounded-exec/run-cpu-pool.test.ts +0 -79
- package/src/executors/bounded-exec/runner-hook.test.ts +0 -193
- package/src/executors/bounded-exec/runner-lifecycle.test.ts +0 -82
- package/src/executors/bounded-exec/verdict.test.ts +0 -545
- package/src/executors/cargo-zigbuild/executor.test.ts +0 -129
- package/src/executors/clean-outputs/executor.test.ts +0 -20
- package/src/executors/typescript-emit/executor.test.ts +0 -83
- package/src/failure-summary.ts +0 -119
- package/src/generators/bounded-test-targets/generator.test.ts +0 -138
- package/src/generators/bun-test-tracing/generator.test.ts +0 -250
- package/src/generators/create-package/generator.test.ts +0 -190
- package/src/generators/make-public/generator.test.ts +0 -137
- package/src/graph-input-contract.test.ts +0 -379
- package/src/host-controller-wiring.test.ts +0 -31
- package/src/index.test.ts +0 -3321
- package/src/linked-snapshots.nx.test.ts +0 -254
- package/src/managed-files/forgejo-artifact.test.ts +0 -395
- package/src/managed-files/generator.test.ts +0 -243
- package/src/managed-files/github-host-cache-isolation.test.ts +0 -86
- package/src/managed-files/json-edit.test.ts +0 -216
- package/src/managed-files/managed-content.test.ts +0 -166
- package/src/managed-files/managed-format-json.test.ts +0 -308
- package/src/managed-files/repo-path.test.ts +0 -172
- package/src/managed-files/setup-devenv-free-disk.test.ts +0 -95
- package/src/managed-files/trace-db-artifact.test.ts +0 -7
- package/src/managed-files/tree-permissions.test.ts +0 -109
- package/src/managed-files/tree.test.ts +0 -123
- package/src/manifest-hash.test.ts +0 -219
- package/src/nextest-code-bound.test.ts +0 -161
- package/src/nextest-extraction.test.ts +0 -216
- package/src/nextest-extraction.ts +0 -371
- package/src/nextest-partitions.test.ts +0 -501
- package/src/nx-cache-bound.test.ts +0 -109
- package/src/nx-cache-restore-mtime.test.ts +0 -170
- package/src/nx-daemon-claim.test.ts +0 -279
- package/src/nx-daemon-idle-cpu.test.ts +0 -80
- package/src/nx-daemon-idle-policy.test.ts +0 -111
- package/src/nx-daemon-ignore-files.test.ts +0 -210
- package/src/nx-daemon-root-normalization.test.ts +0 -105
- package/src/nx-daemon-start-never.test.ts +0 -292
- package/src/nx-daemon-validity.test.ts +0 -1055
- package/src/nx-dot-input-policy.test.ts +0 -353
- package/src/nx-failure-life-cycles.test.ts +0 -207
- package/src/nx-files-archive-racy.test.ts +0 -97
- package/src/nx-foreign-workspace-env.test.ts +0 -329
- package/src/nx-incomplete-run-summary.test.ts +0 -70
- package/src/nx-plugin-load-errors.test.ts +0 -120
- package/src/nx-project-graph-errors.test.ts +0 -70
- package/src/nx-project-resolution-cache.test.ts +0 -176
- package/src/nx-raw-inputs.test.ts +0 -166
- package/src/nx-resident-graph-workers.test.ts +0 -112
- package/src/nx-run-information.test.ts +0 -118
- package/src/nx-skipped-task-exit.test.ts +0 -180
- package/src/nx-state-location.test.ts +0 -63
- package/src/nx-store-resolution.test.ts +0 -81
- package/src/nx-task-graph-cycles.test.ts +0 -148
- package/src/nx-task-output-lease.test.ts +0 -550
- package/src/nx-use-local.test.ts +0 -105
- package/src/owned-installation.test.ts +0 -105
- package/src/package-target-policy.test.ts +0 -1014
- package/src/producer-receipts.test.ts +0 -285
- package/src/release-config-policy.test.ts +0 -311
- package/src/runtime-input.test.ts +0 -393
- package/src/serial-group-wiring.test.ts +0 -326
- package/src/task-history-namespace.test.ts +0 -181
- package/src/task-results.test.ts +0 -228
- package/src/test-clock-policy.test.ts +0 -338
- package/src/test-program.test.ts +0 -303
- package/src/test-timings.test.ts +0 -240
- package/src/test-walk-policy.test.ts +0 -620
- package/src/testing.test.ts +0 -509
- package/src/typecheck-test-policy.test.ts +0 -1468
- package/src/typescript-resolution.test.ts +0 -76
- package/src/version-actions.test.ts +0 -80
- package/src/workspace-config-policy.test.ts +0 -737
|
@@ -1,42 +1,58 @@
|
|
|
1
1
|
import { type ChildProcess, spawn } from 'node:child_process';
|
|
2
2
|
import { accessSync, constants } from 'node:fs';
|
|
3
|
+
import { mkdtemp, rm } from 'node:fs/promises';
|
|
3
4
|
import { availableParallelism, userInfo } from 'node:os';
|
|
4
5
|
import { delimiter, isAbsolute, join, resolve as resolvePath } from 'node:path';
|
|
5
6
|
import { createInterface } from 'node:readline';
|
|
6
|
-
import { Duplex } from 'node:stream';
|
|
7
|
+
import { Duplex, Readable } from 'node:stream';
|
|
7
8
|
import { workspaceDataDirectoryForWorkspace } from 'nx/src/utils/cache-directory.js';
|
|
8
|
-
|
|
9
|
+
import {
|
|
10
|
+
DEADLINE_PATIENCE,
|
|
11
|
+
describeMembersLeft,
|
|
12
|
+
killProcessGroup,
|
|
13
|
+
processGroupHasMembers,
|
|
14
|
+
signalProcessGroup,
|
|
15
|
+
} from '../../process-group.js';
|
|
16
|
+
import { type BoundEvent, type BoundKind, type Bounds, earliestBound, stepBounds, UNARMED } from './bounds.js';
|
|
9
17
|
import {
|
|
10
18
|
CPU_POOL_ENV,
|
|
11
19
|
type CpuPool,
|
|
12
20
|
createCpuPool,
|
|
13
|
-
type HostWaits,
|
|
14
21
|
type InheritedCpuPool,
|
|
15
22
|
readInheritedCpuPool,
|
|
23
|
+
type TokenGrant,
|
|
16
24
|
takeParentCpuTokens,
|
|
17
25
|
} from './cpu-pool.js';
|
|
18
|
-
import {
|
|
19
|
-
CPU_TOKENS_ENV,
|
|
20
|
-
type CpuBudget,
|
|
21
|
-
gatewayCpuBudget,
|
|
22
|
-
runnerDemand,
|
|
23
|
-
sizeRunner,
|
|
24
|
-
type TokenGrant,
|
|
25
|
-
} from './cpu-tokens.js';
|
|
26
|
+
import { GATEWAY_SOCKET } from './gateway-lease.js';
|
|
26
27
|
import type { HangDiagnostics } from './hang-diagnostics.js';
|
|
27
28
|
import { captureHangDiagnostics, HANG_CAPTURE_MS } from './hang-diagnostics.js';
|
|
29
|
+
import type { LockWait } from './lock-wait.js';
|
|
28
30
|
import { createNoSpaceWatch } from './no-space.js';
|
|
29
|
-
import { createRunProgress, withProgressReport } from './progress.js';
|
|
30
31
|
import {
|
|
31
32
|
COWSHED_EXECUTABLE_ENV,
|
|
33
|
+
cowshedSubtreeOwner,
|
|
34
|
+
HANDLE_FD,
|
|
35
|
+
readAdmission,
|
|
36
|
+
SUPERVISOR_SOCKET_ENV,
|
|
37
|
+
type SubtreeAdmission,
|
|
38
|
+
type SubtreeOwner,
|
|
39
|
+
} from './owned-subtree.js';
|
|
40
|
+
import { createRunProgress, withProgressReport } from './progress.js';
|
|
41
|
+
import {
|
|
32
42
|
createHostCommands,
|
|
43
|
+
describeCoverageGap,
|
|
33
44
|
describeRamTempError,
|
|
45
|
+
describeSignalled,
|
|
46
|
+
LEFTOVER_GRACE_MS,
|
|
47
|
+
PRODUCTION,
|
|
34
48
|
RAM_TEMP_CAPACITY_BYTES,
|
|
35
49
|
type RamTempLease,
|
|
36
50
|
RamTempVolume,
|
|
37
51
|
ramTempPaths,
|
|
38
52
|
} from './ram-temp.js';
|
|
39
|
-
import { createRunnerLifecycle, withRunnerLifecycle } from './runner-lifecycle.js';
|
|
53
|
+
import { createRunnerLifecycle, withFixtureLifecycle, withRunnerLifecycle } from './runner-lifecycle.js';
|
|
54
|
+
import { CPU_TOKENS_ENV, runnerDemand, sizeRunner } from './runner-sizing.js';
|
|
55
|
+
import { checkRuntimePin, describeRuntimePinRefusal } from './runtime-pin.js';
|
|
40
56
|
import type { BoundedExecOptions } from './schema.js';
|
|
41
57
|
import {
|
|
42
58
|
type BoundedExecRecord,
|
|
@@ -70,9 +86,9 @@ const EXIT_CODE_BY_SIGNAL: Partial<Record<NodeJS.Signals, number>> = {
|
|
|
70
86
|
/** The pool descriptor: only a grant may give it, so neither the target's nor the inherited environment passes it on. */
|
|
71
87
|
const POOL_ENV = [CPU_TOKENS_ENV, CPU_POOL_ENV];
|
|
72
88
|
/**
|
|
73
|
-
* The variables
|
|
74
|
-
*
|
|
75
|
-
* sub-pool names its own fifo over it.
|
|
89
|
+
* The variables a grant sets for the command it sizes: no target's environment may set them. A jobserver the executor
|
|
90
|
+
* itself inherits, a parent make's or its cowshed job's, still reaches an ungranted command by Cargo's own rule; a
|
|
91
|
+
* granted one's sub-pool names its own fifo over it.
|
|
76
92
|
*/
|
|
77
93
|
const BUDGET_ENV = [...POOL_ENV, 'CARGO_MAKEFLAGS', 'MAKEFLAGS', 'MFLAGS'];
|
|
78
94
|
|
|
@@ -104,8 +120,12 @@ interface RunState {
|
|
|
104
120
|
forceKillNeeded: boolean;
|
|
105
121
|
/** The sub-pool's close error when it did not close cleanly: the run fails. */
|
|
106
122
|
poolCloseError: string | null;
|
|
123
|
+
/** Why the command's subtree could not be retired, when the supervisor's client failed: the run fails. */
|
|
124
|
+
retireError: string | null;
|
|
107
125
|
/** The first output line that said ENOSPC, with the task's volume read when it arrived. */
|
|
108
126
|
noSpace: Promise<NoSpaceSighting> | null;
|
|
127
|
+
/** The lock waits the expiry's snapshot named: empty unless that capture completed. Judged with the expiry. */
|
|
128
|
+
lockWaits: readonly LockWait[];
|
|
109
129
|
}
|
|
110
130
|
|
|
111
131
|
type GrantedTokens = Extract<TokenGrant, { granted: true }>;
|
|
@@ -115,7 +135,10 @@ export interface ProcessTreeKiller {
|
|
|
115
135
|
}
|
|
116
136
|
|
|
117
137
|
/** Where the command's TMPDIR comes from; `null` leaves the inherited one. */
|
|
118
|
-
export type TempVolume = Pick<
|
|
138
|
+
export type TempVolume = Pick<
|
|
139
|
+
RamTempVolume,
|
|
140
|
+
'acquire' | 'release' | 'fullness' | 'space' | 'record' | 'ordinaryParent'
|
|
141
|
+
>;
|
|
119
142
|
|
|
120
143
|
/** Workspace-provided cowshed launchers, in the order a workspace that provides one names them. */
|
|
121
144
|
const WORKSPACE_COWSHED_LAUNCHERS = [
|
|
@@ -170,10 +193,38 @@ export default function boundedExecExecutor(
|
|
|
170
193
|
return runDefaultBoundedExec(options, recordContext(context));
|
|
171
194
|
}
|
|
172
195
|
|
|
173
|
-
/**
|
|
196
|
+
/**
|
|
197
|
+
* Runs with the normal process, RAM lease and CPU pool owners, with an optional stdout sink, on the user's RAM temp
|
|
198
|
+
* volume where there is one, through the host's own gateway: the Nx executor's runs, never a test's.
|
|
199
|
+
*/
|
|
174
200
|
export function runDefaultBoundedExec(
|
|
175
201
|
options: BoundedExecOptions,
|
|
176
202
|
context: BoundedExecContext,
|
|
203
|
+
): Promise<BoundedExecResult> {
|
|
204
|
+
// Only macOS has `hdiutil ram://`; elsewhere the command keeps the inherited TMPDIR.
|
|
205
|
+
return runBoundedExecWith(options, context, (cowshed) =>
|
|
206
|
+
process.platform === 'darwin'
|
|
207
|
+
? new RamTempVolume(
|
|
208
|
+
ramTempPaths(userInfo(), PRODUCTION),
|
|
209
|
+
RAM_TEMP_CAPACITY_BYTES,
|
|
210
|
+
createHostCommands(GATEWAY_SOCKET, undefined, undefined, cowshed),
|
|
211
|
+
undefined,
|
|
212
|
+
undefined,
|
|
213
|
+
'optional',
|
|
214
|
+
)
|
|
215
|
+
: null,
|
|
216
|
+
);
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
/**
|
|
220
|
+
* The default runner with every owner but its RAM volume, which its caller makes: `volume` makes the volume for the
|
|
221
|
+
* cowshed the run's disk commands go through. The Nx executor makes the user's; a host proof passes a fixture volume
|
|
222
|
+
* (`fixtureRamVolume`), so no test leases, creates or detaches the user's volume.
|
|
223
|
+
*/
|
|
224
|
+
export function runBoundedExecWith(
|
|
225
|
+
options: BoundedExecOptions,
|
|
226
|
+
context: BoundedExecContext,
|
|
227
|
+
volume: (cowshed: string | undefined) => TempVolume | null,
|
|
177
228
|
): Promise<BoundedExecResult> {
|
|
178
229
|
// A target that names the cowshed its disk commands run through names it for this run's own RAM volume work and
|
|
179
230
|
// for its command alike. Nx resolves `{workspaceRoot}/…` in an option to a workspace-relative path, which the
|
|
@@ -188,19 +239,14 @@ export function runDefaultBoundedExec(
|
|
|
188
239
|
);
|
|
189
240
|
const run: BoundedExecOptions =
|
|
190
241
|
cowshed === undefined ? options : { ...options, env: { ...options.env, [COWSHED_EXECUTABLE_ENV]: cowshed } };
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
ramTempPaths(userInfo()),
|
|
196
|
-
RAM_TEMP_CAPACITY_BYTES,
|
|
197
|
-
createHostCommands(undefined, undefined, undefined, cowshed),
|
|
198
|
-
)
|
|
199
|
-
: null;
|
|
242
|
+
const tempVolume = volume(cowshed);
|
|
243
|
+
// The command's processes are owned by the supervisor that serves this job, when one does
|
|
244
|
+
// (specs/cowshed/07_api.md, `JobSubtreeHandle`); with none, nothing here may end what it leaves.
|
|
245
|
+
const owner = process.env[SUPERVISOR_SOCKET_ENV] === undefined ? null : cowshedSubtreeOwner(cowshed ?? 'cowshed');
|
|
200
246
|
// A command a parent's CPU pool started draws its tokens from that pool. The pool is named by the environment the
|
|
201
247
|
// parent set for this command; a run no parent started has none.
|
|
202
248
|
const inherited = readInheritedCpuPool(process.env);
|
|
203
|
-
return runBoundedExec(run, context, createProcessTreeKiller(), tempVolume,
|
|
249
|
+
return runBoundedExec(run, context, createProcessTreeKiller(), tempVolume, inherited, captureHangDiagnostics, owner);
|
|
204
250
|
}
|
|
205
251
|
|
|
206
252
|
/**
|
|
@@ -227,24 +273,22 @@ export function recordContext(context: NxTaskContext): BoundedExecContext {
|
|
|
227
273
|
}
|
|
228
274
|
|
|
229
275
|
/**
|
|
230
|
-
* Run the command
|
|
231
|
-
*
|
|
232
|
-
* granted: a wait for the host is not the command's time.
|
|
276
|
+
* Run the command, sized to the tokens its parent's CPU pool (`cpu-pool.ts`) lends it; a run that inherits no pool
|
|
277
|
+
* runs it as asked. Both bounds start once it is granted: a wait for the pool is not the command's time.
|
|
233
278
|
*
|
|
234
|
-
* `inherited` is the parent's
|
|
235
|
-
*
|
|
236
|
-
*
|
|
237
|
-
*
|
|
238
|
-
* its own exit, and the run says so.
|
|
279
|
+
* `inherited` is the parent's pool: an enclosing bounded-exec's sub-pool or the Nx run's own (`run-cpu-pool.ts`). A
|
|
280
|
+
* granted run serves a sub-pool of its grant to its own descendants, closed before the grant returns. A grant whose
|
|
281
|
+
* connection closes while the command runs (the lending parent's exit) ends its accounting, never the command: the
|
|
282
|
+
* command runs on over budget to its own exit, and the run says so.
|
|
239
283
|
*/
|
|
240
284
|
export async function runBoundedExec(
|
|
241
285
|
options: BoundedExecOptions,
|
|
242
286
|
context: BoundedExecContext,
|
|
243
287
|
killer: ProcessTreeKiller,
|
|
244
288
|
tempVolume: TempVolume | null,
|
|
245
|
-
budget: CpuBudget | null,
|
|
246
289
|
inherited: InheritedCpuPool | null = null,
|
|
247
290
|
captureDiagnostics: typeof captureHangDiagnostics = captureHangDiagnostics,
|
|
291
|
+
owner: SubtreeOwner | null = null,
|
|
248
292
|
): Promise<BoundedExecResult> {
|
|
249
293
|
const cwd = resolveCwd(options.cwd, context.root);
|
|
250
294
|
const directory = context.record?.directory ?? null;
|
|
@@ -256,13 +300,19 @@ export async function runBoundedExec(
|
|
|
256
300
|
const timeoutMs = options.timeoutMs;
|
|
257
301
|
const idleTimeoutMs = options.idleTimeoutMs;
|
|
258
302
|
const killAfterMs = options.killAfterMs ?? DEFAULT_KILL_AFTER_MS;
|
|
303
|
+
const fixture: { command: string; timeoutMs: number } | null =
|
|
304
|
+
options.fixtureCommand !== undefined && options.fixtureTimeoutMs !== undefined
|
|
305
|
+
? { command: options.fixtureCommand, timeoutMs: options.fixtureTimeoutMs }
|
|
306
|
+
: null;
|
|
259
307
|
const outputChunks: string[] = [];
|
|
260
308
|
const state: RunState = {
|
|
261
309
|
settled: false,
|
|
262
310
|
expiry: null,
|
|
263
311
|
forceKillNeeded: false,
|
|
264
312
|
poolCloseError: null,
|
|
313
|
+
retireError: null,
|
|
265
314
|
noSpace: null,
|
|
315
|
+
lockWaits: [],
|
|
266
316
|
};
|
|
267
317
|
const appendStdout = (chunk: Buffer | string): void => {
|
|
268
318
|
const text = chunk.toString();
|
|
@@ -278,10 +328,62 @@ export async function runBoundedExec(
|
|
|
278
328
|
outputChunks.push(text);
|
|
279
329
|
process.stderr.write(text);
|
|
280
330
|
};
|
|
331
|
+
// Told, never thrown, wherever the command is being ended: a diagnostic must neither replace the failure it
|
|
332
|
+
// accompanies nor skip the cleanup that follows (the signal handlers, the CPU grant, the RAM lease, the run's record).
|
|
333
|
+
const tell = (text: string): void => {
|
|
334
|
+
try {
|
|
335
|
+
appendStderr(text);
|
|
336
|
+
} catch {
|
|
337
|
+
// Nobody is left to tell.
|
|
338
|
+
}
|
|
339
|
+
};
|
|
340
|
+
|
|
341
|
+
// The fixture pair names one segment: a fixture with no build budget, or a budget with no fixture, is a
|
|
342
|
+
// target that does not say what it runs, so it does not run.
|
|
343
|
+
if (
|
|
344
|
+
(options.fixtureCommand === undefined) !== (options.fixtureTimeoutMs === undefined) ||
|
|
345
|
+
(fixture !== null && !(fixture.timeoutMs > 0))
|
|
346
|
+
) {
|
|
347
|
+
appendStderr(
|
|
348
|
+
'bounded-exec: fixtureCommand and a positive fixtureTimeoutMs must be named together; refusing to run the command\n',
|
|
349
|
+
);
|
|
350
|
+
if (context.record !== undefined) {
|
|
351
|
+
const { task, directory: recordDirectory } = context.record;
|
|
352
|
+
await writeRecord(recordDirectory, { task: task.id, hash: task.hash, verdict: judgeOperationalFailure(1, null) });
|
|
353
|
+
}
|
|
354
|
+
return { success: false, terminalOutput: outputChunks.join('') };
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
// A leg that runs bun runs the Bun its workspace declares, or it does not run. This comes before the RAM lease and the
|
|
358
|
+
// CPU tokens: a refusal that waited in its pool's queue first would say nothing for minutes.
|
|
359
|
+
const runtime = await checkRuntimePin({
|
|
360
|
+
root: context.root,
|
|
361
|
+
command: fixture === null ? requested : `${fixture.command} && ${requested}`,
|
|
362
|
+
env: { ...withoutPoolEnv(process.env), ...options.env },
|
|
363
|
+
cwd,
|
|
364
|
+
pathSource: options.env?.PATH === undefined ? 'the inherited environment' : "the target's env",
|
|
365
|
+
});
|
|
366
|
+
if (runtime !== null) {
|
|
367
|
+
appendStderr(describeRuntimePinRefusal(runtime, context.record?.task.id));
|
|
368
|
+
if (context.record !== undefined) {
|
|
369
|
+
const { task, directory: recordDirectory } = context.record;
|
|
370
|
+
await writeRecord(recordDirectory, { task: task.id, hash: task.hash, verdict: judgeOperationalFailure(1, null) });
|
|
371
|
+
}
|
|
372
|
+
return { success: false, terminalOutput: outputChunks.join('') };
|
|
373
|
+
}
|
|
281
374
|
|
|
282
375
|
let lease: RamTempLease | null = null;
|
|
283
|
-
|
|
284
|
-
|
|
376
|
+
let ordinaryTemp: string | null = null;
|
|
377
|
+
// Before spawn there is nothing to retire. Once started, only the owner's terminal retirement permits deletion.
|
|
378
|
+
let subtreeRetired = true;
|
|
379
|
+
// A lease is removed when its command ends, so it is taken only where whatever the command leaves
|
|
380
|
+
// in it can be ended by its owner first: the supervisor's subtree retirement, never a census.
|
|
381
|
+
if (tempVolume !== null && owner === null) {
|
|
382
|
+
appendStderr(
|
|
383
|
+
`RAM temp volume not leased: no cowshed supervisor owns this command's processes (${SUPERVISOR_SOCKET_ENV} is unset); TMPDIR stays ${process.env.TMPDIR ?? '(unset)'}\n`,
|
|
384
|
+
);
|
|
385
|
+
} else if (tempVolume !== null && owner !== null) {
|
|
386
|
+
const acquired = await tempVolume.acquire(owner);
|
|
285
387
|
if (!acquired.ok) {
|
|
286
388
|
const message = `${describeRamTempError(acquired.error)}\n`;
|
|
287
389
|
process.stderr.write(message);
|
|
@@ -293,7 +395,28 @@ export async function runBoundedExec(
|
|
|
293
395
|
appendStderr(`RAM temp volume: dead lease ${held}\n`);
|
|
294
396
|
}
|
|
295
397
|
for (const reaped of acquired.value.reaped) {
|
|
296
|
-
appendStderr(`RAM temp volume: stopped ${reaped}, which a dead task left
|
|
398
|
+
appendStderr(`RAM temp volume: the supervisor stopped ${reaped}, which a dead task's command left running\n`);
|
|
399
|
+
}
|
|
400
|
+
for (const gap of acquired.value.gaps) {
|
|
401
|
+
appendStderr(`RAM temp volume: the supervisor's view of a dead task's command has a gap: ${gap}\n`);
|
|
402
|
+
}
|
|
403
|
+
for (const retained of acquired.value.retained) {
|
|
404
|
+
appendStderr(`RAM temp volume: dead lease ${retained}\n`);
|
|
405
|
+
}
|
|
406
|
+
} else if (acquired.value.kind === 'unavailable') {
|
|
407
|
+
const unavailable = `RAM temp volume unavailable (${acquired.value.reason}: ${acquired.value.detail})`;
|
|
408
|
+
if (options.env?.TMPDIR !== undefined) {
|
|
409
|
+
appendStderr(`${unavailable}; using target-provided TMPDIR ${options.env.TMPDIR} (medium not inferred)\n`);
|
|
410
|
+
} else {
|
|
411
|
+
try {
|
|
412
|
+
ordinaryTemp = await mkdtemp(join(tempVolume.ordinaryParent, `smoo-bounded-exec-${process.pid}-`));
|
|
413
|
+
} catch (error) {
|
|
414
|
+
appendStderr(
|
|
415
|
+
`${unavailable}; could not create ordinary scratch under ${tempVolume.ordinaryParent}: ${error instanceof Error ? error.message : String(error)}\n`,
|
|
416
|
+
);
|
|
417
|
+
return { success: false, terminalOutput: outputChunks.join('') };
|
|
418
|
+
}
|
|
419
|
+
appendStderr(`${unavailable}; using owned ordinary TMPDIR ${ordinaryTemp}\n`);
|
|
297
420
|
}
|
|
298
421
|
} else {
|
|
299
422
|
appendStderr(
|
|
@@ -301,37 +424,95 @@ export async function runBoundedExec(
|
|
|
301
424
|
);
|
|
302
425
|
}
|
|
303
426
|
}
|
|
304
|
-
|
|
305
|
-
|
|
427
|
+
// The supervisor's answer to the command's own admission, once it is spawned as an owned subtree root, and once
|
|
428
|
+
// any lease has recorded the handle for whoever settles that lease should this task die first.
|
|
429
|
+
let admission: Promise<SubtreeAdmission> | null = null;
|
|
430
|
+
const retireSubtree = async (): Promise<void> => {
|
|
431
|
+
if (admission === null || owner === null) {
|
|
306
432
|
return;
|
|
307
433
|
}
|
|
308
|
-
const
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
appendStderr(`RAM temp volume: stopped ${leftover}, which the command left working in its lease\n`);
|
|
316
|
-
}
|
|
317
|
-
} else if (reaped) {
|
|
318
|
-
appendStderr(`${describeRamTempError(reaped.error)}\n`);
|
|
434
|
+
const answer = await admission;
|
|
435
|
+
admission = null;
|
|
436
|
+
if (answer.kind !== 'admitted') {
|
|
437
|
+
appendStderr(
|
|
438
|
+
`cowshed: the command ran without an owned subtree (${answer.kind === 'refused' ? `supervisor: ${answer.reason}` : answer.detail}); nothing retires what it left running\n`,
|
|
439
|
+
);
|
|
440
|
+
return;
|
|
319
441
|
}
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
442
|
+
// What the command left running is a leak, whichever way the command ended: its supervisor ends exactly the
|
|
443
|
+
// processes its tree records under the command, before any lease goes, and each is said so its owner fixes it.
|
|
444
|
+
const retired = await owner.retire(answer.handle, LEFTOVER_GRACE_MS);
|
|
445
|
+
switch (retired.kind) {
|
|
446
|
+
case 'retired':
|
|
447
|
+
subtreeRetired = true;
|
|
448
|
+
for (const entry of retired.signalled) {
|
|
449
|
+
appendStderr(`cowshed: the supervisor stopped ${describeSignalled(entry)}, which the command left running\n`);
|
|
450
|
+
}
|
|
451
|
+
if (retired.signalledIncomplete) {
|
|
452
|
+
appendStderr("cowshed: the supervisor may have stopped processes of the command's that it could not name\n");
|
|
453
|
+
}
|
|
454
|
+
for (const gap of retired.gaps) {
|
|
455
|
+
appendStderr(
|
|
456
|
+
`cowshed: the supervisor's view of the command's processes has a gap: ${describeCoverageGap(gap)}\n`,
|
|
457
|
+
);
|
|
458
|
+
}
|
|
459
|
+
return;
|
|
460
|
+
case 'refused':
|
|
461
|
+
appendStderr(`cowshed: the supervisor refused to retire the command's processes (${retired.reason})\n`);
|
|
462
|
+
return;
|
|
463
|
+
case 'failed':
|
|
464
|
+
state.retireError = retired.detail;
|
|
465
|
+
appendStderr(`cowshed: could not retire the command's processes: ${retired.detail}\n`);
|
|
466
|
+
return;
|
|
323
467
|
}
|
|
324
468
|
};
|
|
469
|
+
// Failed scratch remains evidence: RAM leases are archived; ordinary scratch is retained at its reported path.
|
|
470
|
+
let tempReleased: Promise<void> | null = null;
|
|
471
|
+
const releaseLease = (keep: boolean): Promise<void> => {
|
|
472
|
+
tempReleased ??= (async () => {
|
|
473
|
+
await retireSubtree();
|
|
474
|
+
if (ordinaryTemp !== null) {
|
|
475
|
+
const held = ordinaryTemp;
|
|
476
|
+
ordinaryTemp = null;
|
|
477
|
+
if (keep || state.retireError !== null || !subtreeRetired) {
|
|
478
|
+
appendStderr(
|
|
479
|
+
`Ordinary temp directory retained at ${held}: ${!subtreeRetired ? 'the command subtree was not retired' : 'the run did not pass'}\n`,
|
|
480
|
+
);
|
|
481
|
+
} else {
|
|
482
|
+
try {
|
|
483
|
+
await rm(held, { recursive: true, force: true });
|
|
484
|
+
} catch (error) {
|
|
485
|
+
appendStderr(
|
|
486
|
+
`Ordinary temp directory retained at ${held}: cleanup failed: ${error instanceof Error ? error.message : String(error)}\n`,
|
|
487
|
+
);
|
|
488
|
+
}
|
|
489
|
+
}
|
|
490
|
+
}
|
|
491
|
+
if (lease === null || owner === null) {
|
|
492
|
+
return;
|
|
493
|
+
}
|
|
494
|
+
const held = lease;
|
|
495
|
+
lease = null;
|
|
496
|
+
const released = await tempVolume?.release(held, keep || state.retireError !== null, owner);
|
|
497
|
+
if (released?.ok) {
|
|
498
|
+
for (const retained of released.value) {
|
|
499
|
+
appendStderr(`RAM temp volume: lease ${retained}\n`);
|
|
500
|
+
}
|
|
501
|
+
} else if (released) {
|
|
502
|
+
appendStderr(`${describeRamTempError(released.error)}\n`);
|
|
503
|
+
}
|
|
504
|
+
})();
|
|
505
|
+
return tempReleased;
|
|
506
|
+
};
|
|
325
507
|
|
|
326
|
-
// The wait is one span line, as cowshed's own spans read.
|
|
327
|
-
//
|
|
508
|
+
// The wait is one span line, as cowshed's own spans read. Every Nx run that inherits no pool serves its own
|
|
509
|
+
// (`run-cpu-pool.ts`), so a task takes its tokens from a parent's pool; a command that inherits none runs exactly as
|
|
328
510
|
// asked and nothing is said.
|
|
329
511
|
const cores = availableParallelism();
|
|
330
512
|
const demand = runnerDemand(requested, options.parallelism, cores, process.env, options.env);
|
|
331
513
|
const asked = Date.now();
|
|
332
|
-
// A
|
|
333
|
-
// A pool
|
|
334
|
-
// relay to the host -- so asking the host beside it would count the run twice; the run fails instead.
|
|
514
|
+
// A pool that answered and refused is a live budget, and running beside it would count the run twice: the run fails
|
|
515
|
+
// instead. A pool whose endpoint is gone (its holder exited) lends nothing, and the run goes on as asked.
|
|
335
516
|
let grant: GrantedTokens | null = null;
|
|
336
517
|
let parentRefusal: string | null = null;
|
|
337
518
|
if (inherited !== null) {
|
|
@@ -339,32 +520,19 @@ export async function runBoundedExec(
|
|
|
339
520
|
if (fromParent.granted) {
|
|
340
521
|
grant = fromParent;
|
|
341
522
|
} else if (fromParent.cause === 'absent') {
|
|
342
|
-
appendStderr(`cowshed: cpu-
|
|
523
|
+
appendStderr(`cowshed: cpu-pool parent pool absent: ${fromParent.reason}; running the command as asked\n`);
|
|
343
524
|
} else {
|
|
344
525
|
parentRefusal = fromParent.reason;
|
|
345
526
|
}
|
|
346
527
|
}
|
|
347
|
-
let unbudgeted: string | null = null;
|
|
348
|
-
if (grant === null && parentRefusal === null && budget !== null) {
|
|
349
|
-
const fromHost = await budget.take(demand.want, context.root, requested);
|
|
350
|
-
if (fromHost.granted) {
|
|
351
|
-
grant = fromHost;
|
|
352
|
-
} else if (fromHost.cause !== 'absent') {
|
|
353
|
-
unbudgeted = fromHost.reason;
|
|
354
|
-
}
|
|
355
|
-
}
|
|
356
528
|
const waitedMs = Date.now() - asked;
|
|
357
529
|
let cpu: BoundedExecRecord['cpu'];
|
|
358
530
|
let sized: { command: string; env: Record<string, string> } = { command: requested, env: {} };
|
|
359
531
|
if (grant !== null) {
|
|
360
532
|
sized = sizeRunner(demand, requested, grant.tokens);
|
|
361
|
-
cpu = { tokens: grant.tokens, want: demand.want, waitedMs
|
|
362
|
-
appendStderr(
|
|
363
|
-
`cowshed: cpu-tokens wait done elapsed=${waitedMs}ms status=ok ${grant.inherited ? 'inherited=true ' : ''}tokens=${grant.tokens}/${demand.want} runner=${demand.kind}\n`,
|
|
364
|
-
);
|
|
365
|
-
} else if (unbudgeted !== null) {
|
|
533
|
+
cpu = { tokens: grant.tokens, want: demand.want, waitedMs };
|
|
366
534
|
appendStderr(
|
|
367
|
-
`cowshed: cpu-
|
|
535
|
+
`cowshed: cpu-pool wait done elapsed=${waitedMs}ms status=ok tokens=${grant.tokens}/${demand.want} runner=${demand.kind}\n`,
|
|
368
536
|
);
|
|
369
537
|
}
|
|
370
538
|
// The run's sub-pool and grant are given back exactly once, however the run ends. The sub-pool closes first, so no
|
|
@@ -386,14 +554,15 @@ export async function runBoundedExec(
|
|
|
386
554
|
return cpuReleased;
|
|
387
555
|
};
|
|
388
556
|
// Records the run's verdict when it has a task to record as. An `expiry` is set only when a bound ended the run.
|
|
389
|
-
// An `operational` run failed on its CPU
|
|
390
|
-
//
|
|
557
|
+
// An `operational` run failed on its CPU pool (a sub-pool that did not close): an ordinary failure, whatever bound also
|
|
558
|
+
// fired, so it is never judged as a bound.
|
|
391
559
|
const recordRun = async (
|
|
392
560
|
exitCode: number,
|
|
393
561
|
expiry: ExpiredBound | null,
|
|
394
562
|
elapsedMs: number,
|
|
395
563
|
operational: boolean,
|
|
396
564
|
noSpace: NoSpaceSighting | null,
|
|
565
|
+
lockWaits: readonly LockWait[],
|
|
397
566
|
): Promise<void> => {
|
|
398
567
|
if (context.record === undefined) {
|
|
399
568
|
return;
|
|
@@ -402,7 +571,7 @@ export async function runBoundedExec(
|
|
|
402
571
|
const report = await readJunitReport(recordDirectory);
|
|
403
572
|
const verdict = operational
|
|
404
573
|
? judgeOperationalFailure(exitCode, report)
|
|
405
|
-
: judgeRun({ exitCode, elapsedMs, expiry, report, noSpace });
|
|
574
|
+
: judgeRun({ exitCode, elapsedMs, expiry, report, noSpace, lockWaits });
|
|
406
575
|
if (verdict.outcome === 'bound' && verdict.bound === 'test') {
|
|
407
576
|
appendStderr(`Every failing test failed only on its per-test timeout: ${verdict.tests.join(', ')}\n`);
|
|
408
577
|
}
|
|
@@ -412,36 +581,19 @@ export async function runBoundedExec(
|
|
|
412
581
|
const refuse = async (message: string): Promise<BoundedExecResult> => {
|
|
413
582
|
appendStderr(message);
|
|
414
583
|
await releaseCpu();
|
|
415
|
-
|
|
416
|
-
await
|
|
584
|
+
// The command never started, so its lease holds nothing of its own to keep.
|
|
585
|
+
await releaseLease(false);
|
|
586
|
+
await recordRun(1, null, Date.now() - asked, false, null, []);
|
|
417
587
|
return { success: false, terminalOutput: outputChunks.join('') };
|
|
418
588
|
};
|
|
419
589
|
if (parentRefusal !== null) {
|
|
420
|
-
return refuse(
|
|
421
|
-
`cowshed: cpu-tokens parent pool refused: ${parentRefusal}; refusing to run the command without asking the host budget\n`,
|
|
422
|
-
);
|
|
590
|
+
return refuse(`cowshed: cpu-pool parent pool refused: ${parentRefusal}; refusing to run the command\n`);
|
|
423
591
|
}
|
|
424
592
|
|
|
425
|
-
// Descendant work queued for host CPU tokens, as its pool hears it (`host-wait`): while any is held the idle clock
|
|
426
|
-
// is stopped, because a queued job is not an idle one (specs/cowshed/05_gateway.md, "The bounds start at the
|
|
427
|
-
// grant"). The clock it stops is made with the command below; a wait held before then is counted and honoured there.
|
|
428
|
-
let heldWaits = 0;
|
|
429
|
-
let idleClock: { stop(): void; restart(): void } | null = null;
|
|
430
|
-
const hostWaits: HostWaits = {
|
|
431
|
-
begin: () => {
|
|
432
|
-
heldWaits += 1;
|
|
433
|
-
if (heldWaits === 1) idleClock?.stop();
|
|
434
|
-
},
|
|
435
|
-
end: () => {
|
|
436
|
-
heldWaits -= 1;
|
|
437
|
-
if (heldWaits === 0) idleClock?.restart();
|
|
438
|
-
},
|
|
439
|
-
};
|
|
440
|
-
|
|
441
593
|
// A granted run serves a sub-pool of its own grant, so its descendants draw from that and never from above it.
|
|
442
|
-
// A pool that cannot be set up refuses the run: no command runs
|
|
594
|
+
// A pool that cannot be set up refuses the run: no granted command runs beside its grant.
|
|
443
595
|
if (grant !== null) {
|
|
444
|
-
const created = await createCpuPool(grant.tokens, grant.closed
|
|
596
|
+
const created = await createCpuPool(grant.tokens, grant.closed);
|
|
445
597
|
if (!created.ok) {
|
|
446
598
|
return refuse(`cowshed: cpu pool unavailable: ${created.error}; refusing to run the command\n`);
|
|
447
599
|
}
|
|
@@ -461,25 +613,61 @@ export async function runBoundedExec(
|
|
|
461
613
|
...sized.env,
|
|
462
614
|
...(pool?.environment ?? {}),
|
|
463
615
|
};
|
|
464
|
-
|
|
616
|
+
// A fixture rides as segment 0 of the lifecycle. Without the channel (Windows) it is plain shell text ahead
|
|
617
|
+
// of the command: the progress bound runs during it and its own ceiling does not.
|
|
618
|
+
const hooked =
|
|
619
|
+
process.platform === 'win32'
|
|
620
|
+
? null
|
|
621
|
+
: fixture === null
|
|
622
|
+
? withRunnerLifecycle(command)
|
|
623
|
+
: withFixtureLifecycle(fixture.command, command);
|
|
465
624
|
const lifecycle = hooked === null ? null : createRunnerLifecycle(hooked.runners);
|
|
466
625
|
let lifecycleFailure: string | null = null;
|
|
467
626
|
const isTesting = (): boolean => lifecycleFailure !== null || (lifecycle?.isTesting() ?? true);
|
|
468
627
|
const startedAt = Date.now();
|
|
628
|
+
// With a supervisor, `cowshed subtree-root` asks it for a handle on itself, writes the answer to the handle
|
|
629
|
+
// descriptor, and execs `/bin/sh -c command` in the same process: the command's pid, group and life are unchanged.
|
|
630
|
+
const shellCommand = hooked?.command ?? (fixture === null ? command : `${fixture.command} && ${command}`);
|
|
631
|
+
const root = owner?.rootArgv(shellCommand);
|
|
469
632
|
let child: ChildProcess;
|
|
470
633
|
try {
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
634
|
+
// fd 3 is the runner lifecycle's duplex socket when the command is hooked; the subtree admission takes fd 4, which
|
|
635
|
+
// `cowshed subtree-root` closes before it execs, so fd 3 reaches the shell unchanged.
|
|
636
|
+
child =
|
|
637
|
+
root === undefined
|
|
638
|
+
? spawn(shellCommand, [], {
|
|
639
|
+
cwd,
|
|
640
|
+
env: mergeEnv(lease?.directory ?? ordinaryTemp, env),
|
|
641
|
+
shell: true,
|
|
642
|
+
stdio: hooked === null ? 'pipe' : ['pipe', 'pipe', 'pipe', 'pipe'],
|
|
643
|
+
detached: process.platform !== 'win32',
|
|
644
|
+
windowsHide: true,
|
|
645
|
+
})
|
|
646
|
+
: spawn(root.file, root.args, {
|
|
647
|
+
cwd,
|
|
648
|
+
env: mergeEnv(lease?.directory ?? ordinaryTemp, env),
|
|
649
|
+
stdio: ['pipe', 'pipe', 'pipe', hooked === null ? 'ignore' : 'pipe', 'pipe'],
|
|
650
|
+
detached: process.platform !== 'win32',
|
|
651
|
+
windowsHide: true,
|
|
652
|
+
});
|
|
479
653
|
} catch (error) {
|
|
480
654
|
// Nothing started, so nothing runs under the grant: it goes back here, with the RAM lease.
|
|
481
655
|
return refuse(`cowshed: could not start the command: ${error instanceof Error ? error.message : String(error)}\n`);
|
|
482
656
|
}
|
|
657
|
+
subtreeRetired = false;
|
|
658
|
+
if (root !== undefined) {
|
|
659
|
+
const channel = child.stdio[HANDLE_FD];
|
|
660
|
+
const leased = lease;
|
|
661
|
+
admission = readAdmission(channel instanceof Readable ? channel : null).then(async (answer) => {
|
|
662
|
+
if (answer.kind === 'admitted' && leased !== null && tempVolume !== null) {
|
|
663
|
+
const recorded = await tempVolume.record(leased, answer.handle);
|
|
664
|
+
if (!recorded.ok) {
|
|
665
|
+
appendStderr(`${describeRamTempError(recorded.error)}\n`);
|
|
666
|
+
}
|
|
667
|
+
}
|
|
668
|
+
return answer;
|
|
669
|
+
});
|
|
670
|
+
}
|
|
483
671
|
|
|
484
672
|
// Progress is a finished test, not output: a runner's SLOW warnings and a hung test's heartbeat keep printing.
|
|
485
673
|
const progress = createRunProgress(command);
|
|
@@ -490,24 +678,22 @@ export async function runBoundedExec(
|
|
|
490
678
|
const held = lease;
|
|
491
679
|
return { line, volume: held === null ? null : ((await tempVolume?.space(held)) ?? null) };
|
|
492
680
|
};
|
|
493
|
-
|
|
681
|
+
// Which bound is armed, and when it expires, is bounds.ts's decision; one timer waits for the earliest of them.
|
|
682
|
+
let bounds: Bounds = UNARMED;
|
|
683
|
+
let boundTimer: NodeJS.Timeout | undefined;
|
|
684
|
+
// The fixture's own output is compilation, not the command's progress: it is not scanned for finished tests.
|
|
685
|
+
let fixtureDone = fixture === null || lifecycle === null;
|
|
686
|
+
// Whether a segment past the fixture ever announced: without one, the shell's exit is the fixture's.
|
|
687
|
+
let mainStarted = fixture === null || lifecycle === null;
|
|
494
688
|
let captureTimer: NodeJS.Timeout | undefined;
|
|
495
689
|
let idleCapture: { expiresAtMs: number; controller: AbortController; result: IdleDiagnosticResult } | undefined;
|
|
496
690
|
const cancelCapture = (): void => {
|
|
497
691
|
clearTimeout(captureTimer);
|
|
498
692
|
idleCapture?.controller.abort();
|
|
499
693
|
};
|
|
500
|
-
|
|
501
|
-
// wait keeps it unspent: the clock is stopped while descendant work is queued for CPU tokens, and runs on from where
|
|
502
|
-
// it stopped once the last wait ends. `queuedMs` is the queued time this interval has not counted, for the report.
|
|
503
|
-
let idleLeftMs = idleTimeoutMs ?? 0;
|
|
504
|
-
let queuedMs = 0;
|
|
505
|
-
let queuedSinceMs: number | null = null;
|
|
506
|
-
const scheduleIdle = (limitMs: number, leftMs: number): void => {
|
|
507
|
-
clearTimeout(idleTimer);
|
|
508
|
-
cancelCapture();
|
|
694
|
+
const scheduleCapture = (expiresAtMs: number): void => {
|
|
509
695
|
const capture: NonNullable<typeof idleCapture> = {
|
|
510
|
-
expiresAtMs
|
|
696
|
+
expiresAtMs,
|
|
511
697
|
controller: new AbortController(),
|
|
512
698
|
result: { kind: 'pending', startedAtMs: null },
|
|
513
699
|
};
|
|
@@ -545,70 +731,45 @@ export async function runBoundedExec(
|
|
|
545
731
|
},
|
|
546
732
|
);
|
|
547
733
|
},
|
|
548
|
-
Math.max(0,
|
|
734
|
+
Math.max(0, expiresAtMs - Date.now() - HANG_CAPTURE_MS),
|
|
549
735
|
);
|
|
550
|
-
idleTimer = setTimeout(() => expire({ kind: 'idle', limitMs, tests: progress.inFlight() }, queuedMs), leftMs);
|
|
551
736
|
};
|
|
552
|
-
const
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
if (idleCapture !== undefined) {
|
|
572
|
-
idleLeftMs = Math.max(0, idleCapture.expiresAtMs - now);
|
|
573
|
-
}
|
|
574
|
-
clearTimeout(idleTimer);
|
|
737
|
+
const expiredAs = (kind: BoundKind): ExpiredBound =>
|
|
738
|
+
kind === 'idle'
|
|
739
|
+
? { kind, limitMs: idleTimeoutMs ?? 0, tests: progress.inFlight() }
|
|
740
|
+
: kind === 'fixture'
|
|
741
|
+
? { kind, limitMs: fixture?.timeoutMs ?? 0 }
|
|
742
|
+
: { kind, limitMs: timeoutMs ?? 0 };
|
|
743
|
+
const stepTo = (event: BoundEvent): void => {
|
|
744
|
+
if (state.settled || state.expiry !== null) return;
|
|
745
|
+
const idleBefore = bounds.idle;
|
|
746
|
+
bounds = stepBounds(
|
|
747
|
+
bounds,
|
|
748
|
+
{
|
|
749
|
+
...(idleTimeoutMs === undefined ? {} : { idleMs: idleTimeoutMs }),
|
|
750
|
+
...(fixture === null || lifecycle === null ? {} : { fixtureMs: fixture.timeoutMs }),
|
|
751
|
+
...(timeoutMs === undefined ? {} : { totalMs: timeoutMs }),
|
|
752
|
+
},
|
|
753
|
+
event,
|
|
754
|
+
);
|
|
755
|
+
if (bounds.idle !== idleBefore) {
|
|
575
756
|
cancelCapture();
|
|
576
757
|
idleCapture = undefined;
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
if (idleTimeoutMs === undefined || state.settled || state.expiry !== null || queuedSinceMs === null) {
|
|
583
|
-
return;
|
|
584
|
-
}
|
|
585
|
-
const waitedMs = Date.now() - queuedSinceMs;
|
|
586
|
-
queuedMs += waitedMs;
|
|
587
|
-
queuedSinceMs = null;
|
|
588
|
-
if (!isTesting()) {
|
|
589
|
-
appendStderr(`bounded-exec: host CPU wait ended after ${waitedMs}ms; no test runner is active\n`);
|
|
590
|
-
return;
|
|
591
|
-
}
|
|
592
|
-
appendStderr(
|
|
593
|
-
`bounded-exec: idle clock restarted: no descendant's work is queued for host CPU tokens after ${waitedMs}ms; ${idleLeftMs}ms of idleTimeoutMs=${idleTimeoutMs} left\n`,
|
|
594
|
-
);
|
|
595
|
-
scheduleIdle(idleTimeoutMs, idleLeftMs);
|
|
596
|
-
},
|
|
758
|
+
if (bounds.idle !== null) scheduleCapture(bounds.idle);
|
|
759
|
+
}
|
|
760
|
+
clearTimeout(boundTimer);
|
|
761
|
+
const next = earliestBound(bounds);
|
|
762
|
+
if (next !== null) boundTimer = setTimeout(() => expire(expiredAs(next.kind)), Math.max(0, next.at - Date.now()));
|
|
597
763
|
};
|
|
598
764
|
|
|
599
765
|
// Only the CHILD's own output is read for progress. The diagnostics below go
|
|
600
766
|
// through appendStderr as well, and re-arming on those would let a teardown
|
|
601
767
|
// message extend the very bound that just fired.
|
|
602
768
|
const updateIdle = (finished: boolean, wasTesting: boolean): void => {
|
|
603
|
-
|
|
604
|
-
clearTimeout(idleTimer);
|
|
605
|
-
cancelCapture();
|
|
606
|
-
idleCapture = undefined;
|
|
607
|
-
} else if (finished || !wasTesting) {
|
|
608
|
-
armIdleTimer();
|
|
609
|
-
}
|
|
769
|
+
stepTo({ kind: 'progress', at: Date.now(), testing: isTesting(), wasTesting, finished });
|
|
610
770
|
};
|
|
611
771
|
const scanProgress = (stream: 'stdout' | 'stderr', text: string): void => {
|
|
772
|
+
if (!fixtureDone) return;
|
|
612
773
|
updateIdle(progress.scan(stream, text), isTesting());
|
|
613
774
|
};
|
|
614
775
|
if (lifecycle !== null) {
|
|
@@ -620,7 +781,8 @@ export async function runBoundedExec(
|
|
|
620
781
|
appendStderr(`bounded-exec: runner lifecycle failed: ${detail}\n`);
|
|
621
782
|
updateIdle(false, wasTesting);
|
|
622
783
|
};
|
|
623
|
-
createInterface({ input: channel })
|
|
784
|
+
const lines = createInterface({ input: channel });
|
|
785
|
+
lines.on('line', (line) => {
|
|
624
786
|
const wasTesting = isTesting();
|
|
625
787
|
const event = lifecycle.accept(line);
|
|
626
788
|
if (event === null) {
|
|
@@ -628,9 +790,17 @@ export async function runBoundedExec(
|
|
|
628
790
|
return;
|
|
629
791
|
}
|
|
630
792
|
updateIdle(false, wasTesting);
|
|
793
|
+
if (event.id === 0 && event.phase === 'terminal' && fixture !== null) {
|
|
794
|
+
fixtureDone = true;
|
|
795
|
+
stepTo({ kind: 'fixture-done' });
|
|
796
|
+
}
|
|
797
|
+
if (event.id > 0) mainStarted = true;
|
|
631
798
|
// Tool startup and native lifecycle-bearing stdout wait for their phase acknowledgement.
|
|
632
799
|
channel.write(`${JSON.stringify(event)}\n`);
|
|
633
800
|
});
|
|
801
|
+
// readline re-emits its input's errors on the interface, where one with no listener is thrown: a
|
|
802
|
+
// runner killed with unread lifecycle bytes resets the channel (ECONNRESET on Linux).
|
|
803
|
+
lines.on('error', (error) => failLifecycle(error.message));
|
|
634
804
|
channel.on('error', (error) => failLifecycle(error.message));
|
|
635
805
|
channel.on('end', () => {
|
|
636
806
|
if (!lifecycle.finished()) failLifecycle('supervisor channel ended without an owned terminal event');
|
|
@@ -651,23 +821,30 @@ export async function runBoundedExec(
|
|
|
651
821
|
appendStderr(text);
|
|
652
822
|
});
|
|
653
823
|
|
|
654
|
-
|
|
824
|
+
// The graceful pass: SIGTERM to the whole group. A signal that fails is told, not thrown: every caller has more to do
|
|
825
|
+
// after it, `terminate` the force-kill and the signal handler its releases and the signal it gives back.
|
|
826
|
+
const killChildTree = async (): Promise<void> => {
|
|
655
827
|
const pid = child.pid;
|
|
656
828
|
if (!pid) {
|
|
657
829
|
return;
|
|
658
830
|
}
|
|
659
|
-
|
|
831
|
+
try {
|
|
832
|
+
await killer.kill(pid, 'SIGTERM');
|
|
833
|
+
} catch (error) {
|
|
834
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
835
|
+
tell(`Terminating process group ${pid} failed: ${reason}\n`);
|
|
836
|
+
}
|
|
660
837
|
};
|
|
661
838
|
|
|
662
839
|
const onProcessExit = (): void => {
|
|
663
|
-
void killChildTree(
|
|
840
|
+
void killChildTree();
|
|
664
841
|
};
|
|
665
842
|
const onTerminationSignal = (signal: NodeJS.Signals): void => {
|
|
666
843
|
removeSignalHandlers();
|
|
667
844
|
cancelCapture();
|
|
668
|
-
void killChildTree(
|
|
845
|
+
void killChildTree()
|
|
669
846
|
.finally(releaseCpu)
|
|
670
|
-
.finally(releaseLease)
|
|
847
|
+
.finally(() => releaseLease(true))
|
|
671
848
|
.finally(() => process.kill(process.pid, signal));
|
|
672
849
|
};
|
|
673
850
|
const onSigint = (): void => onTerminationSignal('SIGINT');
|
|
@@ -706,28 +883,27 @@ export async function runBoundedExec(
|
|
|
706
883
|
// the reader unable to tell a wedged toolchain from a loaded machine — the
|
|
707
884
|
// two have opposite fixes.
|
|
708
885
|
let termination: Promise<void> = Promise.resolve();
|
|
709
|
-
|
|
710
|
-
const expire = (bound: ExpiredBound, queuedMs = 0): void => {
|
|
886
|
+
const expire = (bound: ExpiredBound): void => {
|
|
711
887
|
if (state.settled || state.expiry !== null) {
|
|
712
888
|
return;
|
|
713
889
|
}
|
|
714
890
|
state.expiry = bound;
|
|
715
|
-
clearTimeout(
|
|
716
|
-
clearTimeout(idleTimer);
|
|
891
|
+
clearTimeout(boundTimer);
|
|
717
892
|
const elapsedMs = Date.now() - startedAt;
|
|
718
893
|
appendStderr(
|
|
719
894
|
bound.kind === 'idle'
|
|
720
|
-
? `\nCommand made no progress: no test finished for ${bound.limitMs}ms (idleTimeoutMs=${bound.limitMs})${
|
|
721
|
-
queuedMs > 0 ? `, not counting ${queuedMs}ms queued for host CPU tokens,` : ''
|
|
722
|
-
} after ${elapsedMs}ms of runtime; ${
|
|
895
|
+
? `\nCommand made no progress: no test finished for ${bound.limitMs}ms (idleTimeoutMs=${bound.limitMs}) after ${elapsedMs}ms of runtime; ${
|
|
723
896
|
bound.tests.length > 0
|
|
724
897
|
? `still running: ${bound.tests.join(', ')}`
|
|
725
898
|
: 'the runner named no test still running'
|
|
726
899
|
} (cwd=${cwd}): ${command}\n`
|
|
727
|
-
:
|
|
900
|
+
: bound.kind === 'fixture'
|
|
901
|
+
? `\nFixture command timed out after ${elapsedMs}ms (fixtureTimeoutMs=${bound.limitMs}, cwd=${cwd}): ${fixture?.command ?? command}\n`
|
|
902
|
+
: `\nCommand timed out after ${elapsedMs}ms (timeoutMs=${bound.limitMs}, cwd=${cwd}): ${command}\n`,
|
|
728
903
|
);
|
|
729
904
|
if (bound.kind === 'idle') {
|
|
730
905
|
const observed = idleCapture?.result;
|
|
906
|
+
state.lockWaits = observed?.kind === 'captured' ? observed.snapshot.lockWaits : [];
|
|
731
907
|
appendStderr(
|
|
732
908
|
`Hang diagnostics: runner pid=${child.pid ?? 'unavailable'} pgid=${
|
|
733
909
|
process.platform === 'win32' ? 'unavailable' : (child.pid ?? 'unavailable')
|
|
@@ -753,24 +929,49 @@ export async function runBoundedExec(
|
|
|
753
929
|
termination = terminate();
|
|
754
930
|
};
|
|
755
931
|
|
|
932
|
+
// Whether the group still lists members. A probe that fails proves nothing about the group, so it is told, and the
|
|
933
|
+
// group is taken as listed: a force-kill that was not needed costs nothing, and one that was is not skipped.
|
|
934
|
+
const groupListed = (pid: number): boolean => {
|
|
935
|
+
try {
|
|
936
|
+
return processGroupHasMembers(pid);
|
|
937
|
+
} catch (error) {
|
|
938
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
939
|
+
tell(`Probing process group ${pid} for members failed: ${reason}; taking it as still listing members\n`);
|
|
940
|
+
return true;
|
|
941
|
+
}
|
|
942
|
+
};
|
|
943
|
+
|
|
756
944
|
// A timed-out run ends with its whole process group, not with its leader.
|
|
757
945
|
// The graceful signal reaches every member that exists when it is sent, but
|
|
758
946
|
// a member that ignores SIGTERM, or one forked while the signal goes out
|
|
759
947
|
// (macOS does not hold a fork back for a group signal in flight), outlives
|
|
760
948
|
// a leader that exits on it. The grace ends when the leader exits or
|
|
761
949
|
// killAfterMs passes, whichever comes first; whatever is left then is killed.
|
|
950
|
+
// SIGKILL is no different: a member forked while it goes out survives it too,
|
|
951
|
+
// so the force-kill is repeated while the group lists members. What the repeats
|
|
952
|
+
// cannot end is said, and a failure of the probe or of a signal is shown.
|
|
953
|
+
const forceKill = async (pid: number): Promise<void> => {
|
|
954
|
+
try {
|
|
955
|
+
if (await killProcessGroup(pid, DEADLINE_PATIENCE, (group) => killer.kill(group, 'SIGKILL'))) {
|
|
956
|
+
tell(`${describeMembersLeft(pid)}\n`);
|
|
957
|
+
}
|
|
958
|
+
} catch (error) {
|
|
959
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
960
|
+
tell(`Force-kill of process group ${pid} failed: ${reason}\n`);
|
|
961
|
+
}
|
|
962
|
+
};
|
|
762
963
|
const terminate = async (): Promise<void> => {
|
|
763
|
-
await
|
|
964
|
+
await killChildTree();
|
|
764
965
|
if (!state.settled && killAfterMs > 0) {
|
|
765
966
|
await settledWithin(exitPromise, killAfterMs);
|
|
766
967
|
}
|
|
767
968
|
if (state.settled) {
|
|
768
|
-
if (child.pid !== undefined &&
|
|
969
|
+
if (child.pid !== undefined && groupListed(child.pid)) {
|
|
769
970
|
state.forceKillNeeded = true;
|
|
770
971
|
appendStderr(
|
|
771
972
|
`Force-killing what the ${state.expiry === null ? 'command' : 'timed-out command'} left in its process group: ${command}\n`,
|
|
772
973
|
);
|
|
773
|
-
await
|
|
974
|
+
await forceKill(child.pid);
|
|
774
975
|
}
|
|
775
976
|
return;
|
|
776
977
|
}
|
|
@@ -778,23 +979,25 @@ export async function runBoundedExec(
|
|
|
778
979
|
appendStderr(
|
|
779
980
|
`Force-killing ${state.expiry === null ? 'command' : 'timed out command'} after killAfterMs=${killAfterMs}: ${command}\n`,
|
|
780
981
|
);
|
|
781
|
-
await
|
|
782
|
-
|
|
982
|
+
if (child.pid !== undefined) await forceKill(child.pid);
|
|
983
|
+
// The repeats wait for the group to empty, the leader with it. Windows has no process groups, so there they have
|
|
984
|
+
// nothing to wait on once the one signal is sent, and the leader's exit is awaited as it always was.
|
|
985
|
+
if (process.platform === 'win32' || child.pid === undefined) {
|
|
986
|
+
await settledWithin(exitPromise, REAP_AFTER_FORCE_KILL_MS);
|
|
987
|
+
}
|
|
783
988
|
if (!state.settled) {
|
|
784
989
|
resolveExit({ code: 1, signal: 'SIGKILL' });
|
|
785
990
|
}
|
|
786
991
|
};
|
|
787
992
|
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
// A wait its pool heard before the command started stops the clock it was not yet there to stop.
|
|
792
|
-
if (heldWaits > 0) idleClock.stop();
|
|
993
|
+
// The fixture's ceiling lives and dies with its segment: the segment's terminal clears it, so it can never
|
|
994
|
+
// fire on a command the fixture already handed over to.
|
|
995
|
+
stepTo({ kind: 'spawn', at: Date.now(), testing: isTesting(), fixture: fixture !== null && lifecycle !== null });
|
|
793
996
|
|
|
794
|
-
// A grant's connection is a ledger entry, not the command's owner
|
|
795
|
-
//
|
|
796
|
-
//
|
|
797
|
-
//
|
|
997
|
+
// A grant's connection is a ledger entry, not the command's owner: the lending parent's exit ends it, and what runs
|
|
998
|
+
// under it runs on over budget until it finishes. Its close, while the command runs, is said and changes nothing
|
|
999
|
+
// else; only a bound or a signal stops the command. Our own release after the command settled is not a loss;
|
|
1000
|
+
// `cpuReleased` marks it.
|
|
798
1001
|
if (grant !== null) {
|
|
799
1002
|
const held = grant;
|
|
800
1003
|
void held.closed.then(() => {
|
|
@@ -802,15 +1005,14 @@ export async function runBoundedExec(
|
|
|
802
1005
|
return;
|
|
803
1006
|
}
|
|
804
1007
|
appendStderr(
|
|
805
|
-
`cowshed: cpu-
|
|
1008
|
+
`cowshed: cpu-pool grant ended: the parent pool closed its ${held.tokens}-token grant while the command ran; it runs on over budget until it ends: ${command}\n`,
|
|
806
1009
|
);
|
|
807
1010
|
});
|
|
808
1011
|
}
|
|
809
1012
|
|
|
810
1013
|
const { code: exitCode, signal: exitSignal } = await exitPromise;
|
|
811
1014
|
await termination;
|
|
812
|
-
clearTimeout(
|
|
813
|
-
clearTimeout(idleTimer);
|
|
1015
|
+
clearTimeout(boundTimer);
|
|
814
1016
|
cancelCapture();
|
|
815
1017
|
removeSignalHandlers();
|
|
816
1018
|
// The runner is gone. Its sub-pool closes first, so no descendant still draws on tokens the grant returns;
|
|
@@ -819,7 +1021,11 @@ export async function runBoundedExec(
|
|
|
819
1021
|
|
|
820
1022
|
const code = exitCode ?? signalToExitCode(exitSignal);
|
|
821
1023
|
if (code !== 0 && state.expiry === null) {
|
|
822
|
-
appendStderr(
|
|
1024
|
+
appendStderr(
|
|
1025
|
+
fixture !== null && !mainStarted
|
|
1026
|
+
? `Fixture command exited with status ${code}: ${fixture.command}\n`
|
|
1027
|
+
: `Command exited with status ${code}: ${command}\n`,
|
|
1028
|
+
);
|
|
823
1029
|
}
|
|
824
1030
|
|
|
825
1031
|
if (state.expiry !== null && !state.forceKillNeeded) {
|
|
@@ -838,13 +1044,12 @@ export async function runBoundedExec(
|
|
|
838
1044
|
appendStderr(`${describeRamTempError(full)}\n`);
|
|
839
1045
|
}
|
|
840
1046
|
}
|
|
841
|
-
await releaseLease();
|
|
842
|
-
|
|
843
|
-
// Broken resource ownership or a lost native lifecycle fails even if a shell branch later exited 0:
|
|
844
|
-
// its verdict is an ordinary operational failure, never a successful or silently unbounded run.
|
|
845
|
-
const operationalFailure = state.poolCloseError !== null || lifecycleFailure !== null;
|
|
846
|
-
await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure, seen);
|
|
1047
|
+
await releaseLease(code !== 0 || state.expiry !== null || state.poolCloseError !== null || lifecycleFailure !== null);
|
|
847
1048
|
|
|
1049
|
+
// Broken resource ownership, a lost native lifecycle, or leftovers that could not be retired fail even if a shell
|
|
1050
|
+
// branch later exited 0: its verdict is an ordinary operational failure, never a successful or silently unbounded run.
|
|
1051
|
+
const operationalFailure = state.poolCloseError !== null || lifecycleFailure !== null || state.retireError !== null;
|
|
1052
|
+
await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure, seen, state.lockWaits);
|
|
848
1053
|
return {
|
|
849
1054
|
success: !operationalFailure && state.expiry === null && code === 0,
|
|
850
1055
|
terminalOutput: outputChunks.join(''),
|
|
@@ -853,45 +1058,12 @@ export async function runBoundedExec(
|
|
|
853
1058
|
|
|
854
1059
|
export function createProcessTreeKiller(): ProcessTreeKiller {
|
|
855
1060
|
return {
|
|
856
|
-
kill(pid, signal) {
|
|
857
|
-
|
|
1061
|
+
async kill(pid, signal) {
|
|
1062
|
+
signalProcessGroup(pid, signal);
|
|
858
1063
|
},
|
|
859
1064
|
};
|
|
860
1065
|
}
|
|
861
1066
|
|
|
862
|
-
function killProcessGroup(pid: number, signal: NodeJS.Signals): Promise<void> {
|
|
863
|
-
try {
|
|
864
|
-
// On POSIX the executor spawns with detached: true, so the child leads a
|
|
865
|
-
// process group of its own and -pid signals all of it at once, including
|
|
866
|
-
// grandchildren that tree-walk libraries miss when parents die and
|
|
867
|
-
// children get reparented to init. Only the group is named: the leader may
|
|
868
|
-
// already be reaped, and its pid reused by an unrelated process, while the
|
|
869
|
-
// group lives on in its other members. Windows has no process groups.
|
|
870
|
-
process.kill(process.platform === 'win32' ? pid : -pid, signal);
|
|
871
|
-
} catch {
|
|
872
|
-
// ESRCH: the group (or the Windows process) has already exited.
|
|
873
|
-
}
|
|
874
|
-
return Promise.resolve();
|
|
875
|
-
}
|
|
876
|
-
|
|
877
|
-
/**
|
|
878
|
-
* Whether any process is left in the group `pgid` leads. A group outlives its
|
|
879
|
-
* reaped leader for as long as one member does, and no process can take its id
|
|
880
|
-
* meanwhile. Windows has no process groups, so there is nothing left to name.
|
|
881
|
-
*/
|
|
882
|
-
function processGroupHasMembers(pgid: number): boolean {
|
|
883
|
-
if (process.platform === 'win32') {
|
|
884
|
-
return false;
|
|
885
|
-
}
|
|
886
|
-
try {
|
|
887
|
-
process.kill(-pgid, 0);
|
|
888
|
-
return true;
|
|
889
|
-
} catch (error) {
|
|
890
|
-
// EPERM: a member exists but runs as another user.
|
|
891
|
-
return error instanceof Error && 'code' in error && error.code === 'EPERM';
|
|
892
|
-
}
|
|
893
|
-
}
|
|
894
|
-
|
|
895
1067
|
function resolveCwd(cwd: string | undefined, root: string): string {
|
|
896
1068
|
if (!cwd) {
|
|
897
1069
|
return root;
|
|
@@ -912,10 +1084,10 @@ function buildCommand(options: BoundedExecOptions): string {
|
|
|
912
1084
|
return parts.join(' ');
|
|
913
1085
|
}
|
|
914
1086
|
|
|
915
|
-
/**
|
|
916
|
-
function mergeEnv(
|
|
1087
|
+
/** Owned scratch is the command's TMPDIR unless the target names one itself. */
|
|
1088
|
+
function mergeEnv(directory: string | null, env: Record<string, string>): NodeJS.ProcessEnv {
|
|
917
1089
|
const inherited = withoutPoolEnv(process.env);
|
|
918
|
-
return
|
|
1090
|
+
return directory === null ? { ...inherited, ...env } : { ...inherited, TMPDIR: directory, ...env };
|
|
919
1091
|
}
|
|
920
1092
|
|
|
921
1093
|
/** The process's environment without the CPU pool's descriptor: a pool and its count only a grant may give. */
|
|
@@ -949,14 +1121,6 @@ function signalToExitCode(signal: NodeJS.Signals | null): number {
|
|
|
949
1121
|
return EXIT_CODE_BY_SIGNAL[signal] ?? 1;
|
|
950
1122
|
}
|
|
951
1123
|
|
|
952
|
-
async function ignoreKillError(promise: Promise<void>): Promise<void> {
|
|
953
|
-
try {
|
|
954
|
-
await promise;
|
|
955
|
-
} catch {
|
|
956
|
-
// The process tree may have already exited between timeout and kill.
|
|
957
|
-
}
|
|
958
|
-
}
|
|
959
|
-
|
|
960
1124
|
/** Resolves when `promise` settles or `ms` passes, whichever is first, leaving no timer behind. */
|
|
961
1125
|
async function settledWithin(promise: Promise<unknown>, ms: number): Promise<void> {
|
|
962
1126
|
let timer: NodeJS.Timeout | undefined;
|