@smoothbricks/nx-plugin 0.5.7 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1097 -360
- package/dist/.build-inputs.json +141 -0
- package/dist/bin/smoo-nx-cargo-toolchain.d.ts +3 -0
- package/dist/bin/smoo-nx-cargo-toolchain.d.ts.map +1 -0
- package/dist/bin/smoo-nx-cargo-toolchain.js +8 -0
- package/dist/bin/smoo-nx-forbidden-spellings.d.ts +3 -0
- package/dist/bin/smoo-nx-forbidden-spellings.d.ts.map +1 -0
- package/dist/bin/smoo-nx-forbidden-spellings.js +32 -0
- package/dist/bin/smoo-nx-nextest-timings.d.ts +3 -0
- package/dist/bin/smoo-nx-nextest-timings.d.ts.map +1 -0
- package/dist/bin/smoo-nx-nextest-timings.js +28 -0
- package/dist/bounded-test-policy.d.ts +1 -1
- package/dist/bounded-test-policy.d.ts.map +1 -1
- package/dist/bounded-test-policy.js +14 -15
- package/dist/build-state-output-policy.d.ts +95 -1
- package/dist/build-state-output-policy.d.ts.map +1 -1
- package/dist/build-state-output-policy.js +368 -30
- package/dist/bun-lock-scope.d.ts +44 -0
- package/dist/bun-lock-scope.d.ts.map +1 -0
- package/dist/bun-lock-scope.js +97 -0
- package/dist/cargo-build-script-env.d.ts +49 -0
- package/dist/cargo-build-script-env.d.ts.map +1 -0
- package/dist/cargo-build-script-env.js +324 -0
- package/dist/cargo-closure-input.d.ts +31 -4
- package/dist/cargo-closure-input.d.ts.map +1 -1
- package/dist/cargo-closure-input.js +66 -6
- package/dist/cargo-retention-hook.d.ts +17 -0
- package/dist/cargo-retention-hook.d.ts.map +1 -0
- package/dist/cargo-retention-hook.js +85 -0
- package/dist/cargo-source-hash.d.ts +25 -7
- package/dist/cargo-source-hash.d.ts.map +1 -1
- package/dist/cargo-source-hash.js +108 -43
- package/dist/cargo-toolchain-paths.d.ts +40 -0
- package/dist/cargo-toolchain-paths.d.ts.map +1 -0
- package/dist/cargo-toolchain-paths.js +84 -0
- package/dist/cargo-toolchain-policy.d.ts +25 -8
- package/dist/cargo-toolchain-policy.d.ts.map +1 -1
- package/dist/cargo-toolchain-policy.js +39 -5
- package/dist/cargo-workspace.d.ts +10 -4
- package/dist/cargo-workspace.d.ts.map +1 -1
- package/dist/cargo-workspace.js +65 -95
- package/dist/cross-check-policy.d.ts +9 -9
- package/dist/cross-check-policy.d.ts.map +1 -1
- package/dist/cross-check-policy.js +11 -11
- package/dist/disk-command-policy.d.ts.map +1 -1
- package/dist/disk-command-policy.js +1 -0
- package/dist/ensure-built-receipt.d.ts +22 -5
- package/dist/ensure-built-receipt.d.ts.map +1 -1
- package/dist/ensure-built-receipt.js +168 -48
- package/dist/ensure-built-witness.d.ts +27 -8
- package/dist/ensure-built-witness.d.ts.map +1 -1
- package/dist/ensure-built-witness.js +45 -15
- package/dist/ensure-built.d.ts.map +1 -1
- package/dist/ensure-built.js +9 -1
- package/dist/eslint-config-closure.d.ts +7 -0
- package/dist/eslint-config-closure.d.ts.map +1 -0
- package/dist/eslint-config-closure.js +346 -0
- package/dist/executors/bounded-exec/bounds.d.ts +56 -0
- package/dist/executors/bounded-exec/bounds.d.ts.map +1 -0
- package/dist/executors/bounded-exec/bounds.js +43 -0
- package/dist/executors/bounded-exec/cpu-pool-thread.d.ts +28 -0
- package/dist/executors/bounded-exec/cpu-pool-thread.d.ts.map +1 -0
- package/dist/executors/bounded-exec/cpu-pool-thread.js +20 -0
- package/dist/executors/bounded-exec/cpu-pool.d.ts +35 -38
- package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool.js +56 -120
- package/dist/executors/bounded-exec/executor.d.ts +19 -12
- package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
- package/dist/executors/bounded-exec/executor.js +389 -239
- package/dist/executors/bounded-exec/gateway-lease.d.ts +23 -19
- package/dist/executors/bounded-exec/gateway-lease.d.ts.map +1 -1
- package/dist/executors/bounded-exec/gateway-lease.js +102 -29
- package/dist/executors/bounded-exec/hang-diagnostics.d.ts +18 -1
- package/dist/executors/bounded-exec/hang-diagnostics.d.ts.map +1 -1
- package/dist/executors/bounded-exec/hang-diagnostics.js +148 -30
- package/dist/executors/bounded-exec/lease-wait.d.ts +64 -0
- package/dist/executors/bounded-exec/lease-wait.d.ts.map +1 -0
- package/dist/executors/bounded-exec/lease-wait.js +72 -0
- package/dist/executors/bounded-exec/lock-wait.d.ts +51 -0
- package/dist/executors/bounded-exec/lock-wait.d.ts.map +1 -0
- package/dist/executors/bounded-exec/lock-wait.js +108 -0
- package/dist/executors/bounded-exec/mbx-pool.d.ts +31 -0
- package/dist/executors/bounded-exec/mbx-pool.d.ts.map +1 -0
- package/dist/executors/bounded-exec/mbx-pool.js +24 -0
- package/dist/executors/bounded-exec/owned-subtree.d.ts +84 -0
- package/dist/executors/bounded-exec/owned-subtree.d.ts.map +1 -0
- package/dist/executors/bounded-exec/owned-subtree.js +178 -0
- package/dist/executors/bounded-exec/progress.d.ts +4 -1
- package/dist/executors/bounded-exec/progress.d.ts.map +1 -1
- package/dist/executors/bounded-exec/progress.js +5 -2
- package/dist/executors/bounded-exec/ram-attach.d.ts +19 -0
- package/dist/executors/bounded-exec/ram-attach.d.ts.map +1 -0
- package/dist/executors/bounded-exec/ram-attach.js +207 -0
- package/dist/executors/bounded-exec/ram-detach.d.ts +25 -0
- package/dist/executors/bounded-exec/ram-detach.d.ts.map +1 -0
- package/dist/executors/bounded-exec/ram-detach.js +207 -0
- package/dist/executors/bounded-exec/ram-temp-fixtures.d.ts +76 -0
- package/dist/executors/bounded-exec/ram-temp-fixtures.d.ts.map +1 -1
- package/dist/executors/bounded-exec/ram-temp-fixtures.js +296 -0
- package/dist/executors/bounded-exec/ram-temp.d.ts +388 -88
- package/dist/executors/bounded-exec/ram-temp.d.ts.map +1 -1
- package/dist/executors/bounded-exec/ram-temp.js +1213 -399
- package/dist/executors/bounded-exec/run-cpu-pool.d.ts +10 -4
- package/dist/executors/bounded-exec/run-cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/run-cpu-pool.js +122 -35
- package/dist/executors/bounded-exec/runner-hook.js +14 -5
- package/dist/executors/bounded-exec/runner-lifecycle.d.ts +14 -1
- package/dist/executors/bounded-exec/runner-lifecycle.d.ts.map +1 -1
- package/dist/executors/bounded-exec/runner-lifecycle.js +27 -6
- package/dist/executors/bounded-exec/{cpu-tokens.d.ts → runner-sizing.d.ts} +14 -42
- package/dist/executors/bounded-exec/runner-sizing.d.ts.map +1 -0
- package/dist/executors/bounded-exec/{cpu-tokens.js → runner-sizing.js} +5 -35
- package/dist/executors/bounded-exec/runtime-pin.d.ts +73 -0
- package/dist/executors/bounded-exec/runtime-pin.d.ts.map +1 -0
- package/dist/executors/bounded-exec/runtime-pin.js +167 -0
- package/dist/executors/bounded-exec/schema.d.ts +18 -1
- package/dist/executors/bounded-exec/schema.d.ts.map +1 -1
- package/dist/executors/bounded-exec/verdict.d.ts +48 -11
- package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
- package/dist/executors/bounded-exec/verdict.js +89 -20
- package/dist/executors/cargo-zigbuild/executor.d.ts +21 -0
- package/dist/executors/cargo-zigbuild/executor.d.ts.map +1 -1
- package/dist/executors/cargo-zigbuild/executor.js +67 -12
- package/dist/executors/clean-outputs/executor.js +1 -0
- package/dist/executors/scriptc-native/executor.d.ts +79 -0
- package/dist/executors/scriptc-native/executor.d.ts.map +1 -0
- package/dist/executors/scriptc-native/executor.js +367 -0
- package/dist/executors/scriptc-native/ffi-paths.d.ts +9 -0
- package/dist/executors/scriptc-native/ffi-paths.d.ts.map +1 -0
- package/dist/executors/scriptc-native/ffi-paths.js +25 -0
- package/dist/executors/scriptc-native/schema.d.ts +34 -0
- package/dist/executors/scriptc-native/schema.d.ts.map +1 -0
- package/dist/executors/scriptc-native/schema.js +1 -0
- package/dist/executors/typescript-emit/emit-overrides.d.ts +14 -0
- package/dist/executors/typescript-emit/emit-overrides.d.ts.map +1 -0
- package/dist/executors/typescript-emit/emit-overrides.js +24 -0
- package/dist/executors/typescript-emit/executor.d.ts +3 -1
- package/dist/executors/typescript-emit/executor.d.ts.map +1 -1
- package/dist/executors/typescript-emit/executor.js +146 -33
- package/dist/forbidden-spelling-policy.d.ts +56 -0
- package/dist/forbidden-spelling-policy.d.ts.map +1 -0
- package/dist/forbidden-spelling-policy.js +176 -0
- package/dist/generators/create-package/generator.d.ts.map +1 -1
- package/dist/generators/create-package/generator.js +3 -4
- package/dist/graph-input-contract.d.ts +14 -0
- package/dist/graph-input-contract.d.ts.map +1 -1
- package/dist/graph-input-contract.js +67 -2
- package/dist/index.d.ts +23 -6
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +672 -214
- package/dist/managed-assets.d.ts +2 -0
- package/dist/managed-assets.d.ts.map +1 -1
- package/dist/managed-files/files.d.ts +3 -3
- package/dist/managed-files/files.d.ts.map +1 -1
- package/dist/managed-files/files.js +28 -3
- package/dist/managed-files/managed-format.d.ts.map +1 -1
- package/dist/managed-files/managed-format.js +1 -0
- package/dist/managed-files/publish-workflow.js +14 -11
- package/dist/nextest-code-bound.js +1 -0
- package/dist/nextest-timings.d.ts +35 -0
- package/dist/nextest-timings.d.ts.map +1 -0
- package/dist/nextest-timings.js +168 -0
- package/dist/owned-installation.d.ts +6 -5
- package/dist/owned-installation.d.ts.map +1 -1
- package/dist/owned-installation.js +150 -26
- package/dist/package-target-policy.d.ts.map +1 -1
- package/dist/package-target-policy.js +2 -3
- package/dist/process-children.d.ts +7 -0
- package/dist/process-children.d.ts.map +1 -0
- package/dist/process-children.darwin.d.ts +7 -0
- package/dist/process-children.darwin.d.ts.map +1 -0
- package/dist/process-children.darwin.js +13 -0
- package/dist/process-children.js +14 -0
- package/dist/process-children.linux.d.ts +8 -0
- package/dist/process-children.linux.d.ts.map +1 -0
- package/dist/process-children.linux.js +25 -0
- package/dist/process-group.d.ts +108 -0
- package/dist/process-group.d.ts.map +1 -0
- package/dist/process-group.js +171 -0
- package/dist/process-listing.d.ts +7 -0
- package/dist/process-listing.d.ts.map +1 -0
- package/dist/process-listing.js +51 -0
- package/dist/producer-receipts.d.ts +8 -2
- package/dist/producer-receipts.d.ts.map +1 -1
- package/dist/producer-receipts.js +514 -47
- package/dist/task-graph-policy.d.ts +21 -0
- package/dist/task-graph-policy.d.ts.map +1 -0
- package/dist/task-graph-policy.js +91 -0
- package/dist/test-timings.js +4 -4
- package/dist/test-walk-policy.d.ts +14 -5
- package/dist/test-walk-policy.d.ts.map +1 -1
- package/dist/test-walk-policy.js +8 -2
- package/dist/testing.d.ts +73 -28
- package/dist/testing.d.ts.map +1 -1
- package/dist/testing.js +217 -223
- package/dist/tool-bins.d.ts +33 -0
- package/dist/tool-bins.d.ts.map +1 -0
- package/dist/tool-bins.js +72 -0
- package/dist/typecheck-test-policy.d.ts.map +1 -1
- package/dist/typecheck-test-policy.js +3 -4
- package/dist/wait-why-policy.d.ts +28 -0
- package/dist/wait-why-policy.d.ts.map +1 -0
- package/dist/wait-why-policy.js +223 -0
- package/dist/workspace-config-policy.d.ts +14 -8
- package/dist/workspace-config-policy.d.ts.map +1 -1
- package/dist/workspace-config-policy.js +60 -16
- package/dist/workspace-manifest.d.ts +41 -6
- package/dist/workspace-manifest.d.ts.map +1 -1
- package/dist/workspace-manifest.js +57 -0
- package/executors.json +16 -1
- package/loader.js +2 -0
- package/managed/raw/envrc +11 -0
- package/managed/raw/patches/nx@23.2.1.patch +2840 -199
- package/managed/raw/patches/ttsc@0.34.1.patch +119 -0
- package/managed/raw/tooling/direnv/cargo-fetch-bound.ts +245 -0
- package/managed/raw/tooling/direnv/cargo-fetch-run.ts +299 -0
- package/managed/raw/tooling/direnv/compiler-variables.sh +42 -0
- package/managed/raw/tooling/direnv/developer-links.ts +464 -25
- package/managed/raw/tooling/direnv/devenv.smoo.nix +78 -38
- package/managed/raw/tooling/direnv/github-actions-bootstrap.sh +14 -12
- package/managed/raw/tooling/direnv/installed-bun-links.ts +308 -0
- package/managed/raw/tooling/direnv/linked-snapshots.ts +62 -3
- package/managed/raw/tooling/direnv/nx-socket-dir.sh +56 -17
- package/managed/raw/tooling/direnv/repo-path +68 -6
- package/managed/raw/tooling/direnv/secret-references.ts +13 -4
- package/managed/raw/tooling/direnv/setup-environment.ts +128 -537
- package/managed/raw/tooling/direnv/toolchain-stamp.ts +4 -2
- package/managed/raw/tooling/direnv/ttsc-env.sh +9 -0
- package/managed/raw/tooling/direnv/uv-relocation.ts +227 -0
- package/managed/raw/tooling/git-hooks/commit-msg.sh +2 -1
- package/managed/raw/tooling/git-hooks/pre-commit.sh +2 -1
- package/managed/raw/tooling/git-hooks/pre-push.sh +2 -1
- package/managed/templates/github/actions/cache-ttsc-plugins/action.yml +85 -7
- package/managed/templates/github/actions/save-nix-devenv/action.yml +12 -2
- package/managed/templates/github/actions/setup-devenv/action.yml +2 -0
- package/package.json +95 -14
- package/src/bin/smoo-nx-cargo-toolchain.ts +11 -0
- package/src/bin/smoo-nx-forbidden-spellings.ts +33 -0
- package/src/bin/smoo-nx-nextest-timings.ts +31 -0
- package/src/bounded-test-policy.ts +12 -14
- package/src/build-state-output-policy.ts +458 -34
- package/src/bun-lock-scope.ts +108 -0
- package/src/cargo-build-script-env.ts +265 -0
- package/src/cargo-closure-input.ts +74 -7
- package/src/cargo-retention-hook.ts +94 -0
- package/src/cargo-source-hash.ts +138 -41
- package/src/cargo-toolchain-paths.ts +95 -0
- package/src/cargo-toolchain-policy.ts +49 -5
- package/src/cargo-workspace.ts +72 -123
- package/src/cross-check-policy.ts +12 -10
- package/src/disk-command-policy.ts +1 -0
- package/src/ensure-built-receipt.ts +173 -39
- package/src/ensure-built-witness.ts +62 -10
- package/src/ensure-built.ts +11 -1
- package/src/eslint-config-closure.ts +357 -0
- package/src/executors/bounded-exec/bounds.ts +76 -0
- package/src/executors/bounded-exec/cpu-pool-thread.ts +43 -0
- package/src/executors/bounded-exec/cpu-pool.ts +55 -120
- package/src/executors/bounded-exec/executor.ts +442 -260
- package/src/executors/bounded-exec/gateway-lease.ts +94 -35
- package/src/executors/bounded-exec/hang-diagnostics.ts +173 -31
- package/src/executors/bounded-exec/lease-wait.ts +116 -0
- package/src/executors/bounded-exec/lock-wait.ts +141 -0
- package/src/executors/bounded-exec/mbx-pool.ts +48 -0
- package/src/executors/bounded-exec/owned-subtree.ts +235 -0
- package/src/executors/bounded-exec/progress.ts +9 -3
- package/src/executors/bounded-exec/ram-attach.ts +238 -0
- package/src/executors/bounded-exec/ram-detach.ts +226 -0
- package/src/executors/bounded-exec/ram-temp-fixtures.ts +358 -0
- package/src/executors/bounded-exec/ram-temp-registry.txt +440 -0
- package/src/executors/bounded-exec/ram-temp.ts +1497 -460
- package/src/executors/bounded-exec/run-cpu-pool.ts +129 -35
- package/src/executors/bounded-exec/runner-hook.ts +14 -6
- package/src/executors/bounded-exec/runner-lifecycle.ts +31 -8
- package/src/executors/bounded-exec/{cpu-tokens.ts → runner-sizing.ts} +13 -59
- package/src/executors/bounded-exec/runtime-pin.ts +232 -0
- package/src/executors/bounded-exec/schema.json +11 -2
- package/src/executors/bounded-exec/schema.ts +19 -1
- package/src/executors/bounded-exec/verdict.ts +145 -35
- package/src/executors/cargo-zigbuild/executor.ts +82 -8
- package/src/executors/clean-outputs/executor.ts +1 -0
- package/src/executors/scriptc-native/executor.ts +394 -0
- package/src/executors/scriptc-native/ffi-paths.ts +29 -0
- package/src/executors/scriptc-native/schema.json +60 -0
- package/src/executors/scriptc-native/schema.ts +34 -0
- package/src/executors/typescript-emit/emit-overrides.ts +24 -0
- package/src/executors/typescript-emit/executor.ts +186 -38
- package/src/forbidden-spelling-policy.ts +226 -0
- package/src/generators/create-package/generator.ts +7 -7
- package/src/graph-input-contract.ts +61 -2
- package/src/index.ts +788 -224
- package/src/managed-assets.ts +3 -0
- package/src/managed-files/files.ts +28 -3
- package/src/managed-files/managed-format.ts +1 -0
- package/src/managed-files/publish-workflow.ts +20 -15
- package/src/nextest-code-bound.ts +1 -0
- package/src/nextest-timings.ts +106 -0
- package/src/owned-installation.ts +150 -23
- package/src/package-target-policy.ts +2 -3
- package/src/process-children.darwin.ts +14 -0
- package/src/process-children.linux.ts +33 -0
- package/src/process-children.ts +22 -0
- package/src/process-group.ts +209 -0
- package/src/process-listing.ts +54 -0
- package/src/producer-receipts.ts +92 -41
- package/src/task-graph-policy.ts +98 -0
- package/src/test-walk-policy.ts +23 -8
- package/src/testing.ts +240 -232
- package/src/tool-bins.ts +108 -0
- package/src/typecheck-test-policy.ts +2 -3
- package/src/wait-why-policy.ts +226 -0
- package/src/workspace-config-policy.ts +63 -15
- package/src/workspace-manifest.ts +89 -6
- package/test-timings.json +124 -99
- package/dist/bin/smoo-nx-failures.d.ts +0 -3
- package/dist/bin/smoo-nx-failures.d.ts.map +0 -1
- package/dist/bin/smoo-nx-failures.js +0 -3
- package/dist/bin/smoo-nx-nextest-extract.d.ts +0 -3
- package/dist/bin/smoo-nx-nextest-extract.d.ts.map +0 -1
- package/dist/bin/smoo-nx-nextest-extract.js +0 -42
- package/dist/executors/bounded-exec/cpu-tokens.d.ts.map +0 -1
- package/dist/failure-summary.d.ts +0 -5
- package/dist/failure-summary.d.ts.map +0 -1
- package/dist/failure-summary.js +0 -118
- package/dist/nextest-extraction.d.ts +0 -117
- package/dist/nextest-extraction.d.ts.map +0 -1
- package/dist/nextest-extraction.js +0 -303
- package/managed/raw/patches/ttsc@0.30.4.patch +0 -2255
- package/src/__tests__/cargo-closure-filesets.ts +0 -94
- package/src/__tests__/cargo-closure-fixture.ts +0 -113
- package/src/__tests__/counted-cargo-process.ts +0 -286
- package/src/__tests__/counted-cargo.ts +0 -713
- package/src/__tests__/directory-change.ts +0 -27
- package/src/__tests__/fixture-cargo-home.ts +0 -36
- package/src/__tests__/fixture-nx-env.ts +0 -193
- package/src/__tests__/owned-installation.ts +0 -13
- package/src/__tests__/past-tick.ts +0 -12
- package/src/__tests__/progress-clock.ts +0 -43
- package/src/__tests__/runtime-input.ts +0 -23
- package/src/bin/smoo-nx-bound-failures.test.ts +0 -127
- package/src/bin/smoo-nx-failures.ts +0 -4
- package/src/bin/smoo-nx-nextest-extract.ts +0 -44
- package/src/bounded-test-policy.test.ts +0 -1029
- package/src/build-state-output-policy.test.ts +0 -115
- package/src/bun-bootstrap.test.ts +0 -102
- package/src/bun-file-packages.test.ts +0 -47
- package/src/cargo-closure-filesets.test.ts +0 -63
- package/src/cargo-closure-input.property.test.ts +0 -50
- package/src/cargo-closure-input.test.ts +0 -506
- package/src/cargo-closure-key.test.ts +0 -240
- package/src/cargo-resolve-cache.test.ts +0 -1181
- package/src/cargo-source-hash.test.ts +0 -707
- package/src/cargo-workspace.test.ts +0 -615
- package/src/cross-check-policy.test.ts +0 -316
- package/src/disk-command-policy.test.ts +0 -61
- package/src/ensure-built-receipt.test.ts +0 -279
- package/src/ensure-built-witness.test.ts +0 -688
- package/src/ensure-built.test.ts +0 -1462
- package/src/executors/all-verdicts/executor.test.ts +0 -135
- package/src/executors/bounded-exec/cargo-progress-host.test.ts +0 -332
- package/src/executors/bounded-exec/cpu-pool.test.ts +0 -684
- package/src/executors/bounded-exec/cpu-tokens.test.ts +0 -519
- package/src/executors/bounded-exec/disk-lease.test.ts +0 -219
- package/src/executors/bounded-exec/executor-host.test.ts +0 -800
- package/src/executors/bounded-exec/executor.test.ts +0 -1819
- package/src/executors/bounded-exec/go-progress-host.test.ts +0 -171
- package/src/executors/bounded-exec/hang-diagnostics.test.ts +0 -291
- package/src/executors/bounded-exec/progress.test.ts +0 -139
- package/src/executors/bounded-exec/ram-temp-held.test.ts +0 -264
- package/src/executors/bounded-exec/ram-temp-host.test.ts +0 -635
- package/src/executors/bounded-exec/ram-temp.test.ts +0 -750
- package/src/executors/bounded-exec/run-cpu-pool.test.ts +0 -79
- package/src/executors/bounded-exec/runner-hook.test.ts +0 -193
- package/src/executors/bounded-exec/runner-lifecycle.test.ts +0 -82
- package/src/executors/bounded-exec/verdict.test.ts +0 -545
- package/src/executors/cargo-zigbuild/executor.test.ts +0 -129
- package/src/executors/clean-outputs/executor.test.ts +0 -20
- package/src/executors/typescript-emit/executor.test.ts +0 -83
- package/src/failure-summary.ts +0 -119
- package/src/generators/bounded-test-targets/generator.test.ts +0 -138
- package/src/generators/bun-test-tracing/generator.test.ts +0 -250
- package/src/generators/create-package/generator.test.ts +0 -190
- package/src/generators/make-public/generator.test.ts +0 -137
- package/src/graph-input-contract.test.ts +0 -379
- package/src/host-controller-wiring.test.ts +0 -31
- package/src/index.test.ts +0 -3321
- package/src/linked-snapshots.nx.test.ts +0 -254
- package/src/managed-files/forgejo-artifact.test.ts +0 -395
- package/src/managed-files/generator.test.ts +0 -243
- package/src/managed-files/github-host-cache-isolation.test.ts +0 -86
- package/src/managed-files/json-edit.test.ts +0 -216
- package/src/managed-files/managed-content.test.ts +0 -166
- package/src/managed-files/managed-format-json.test.ts +0 -308
- package/src/managed-files/repo-path.test.ts +0 -172
- package/src/managed-files/setup-devenv-free-disk.test.ts +0 -95
- package/src/managed-files/trace-db-artifact.test.ts +0 -7
- package/src/managed-files/tree-permissions.test.ts +0 -109
- package/src/managed-files/tree.test.ts +0 -123
- package/src/manifest-hash.test.ts +0 -219
- package/src/nextest-code-bound.test.ts +0 -161
- package/src/nextest-extraction.test.ts +0 -216
- package/src/nextest-extraction.ts +0 -371
- package/src/nextest-partitions.test.ts +0 -501
- package/src/nx-cache-bound.test.ts +0 -109
- package/src/nx-cache-restore-mtime.test.ts +0 -170
- package/src/nx-daemon-claim.test.ts +0 -279
- package/src/nx-daemon-idle-cpu.test.ts +0 -80
- package/src/nx-daemon-idle-policy.test.ts +0 -111
- package/src/nx-daemon-ignore-files.test.ts +0 -210
- package/src/nx-daemon-root-normalization.test.ts +0 -105
- package/src/nx-daemon-start-never.test.ts +0 -292
- package/src/nx-daemon-validity.test.ts +0 -1055
- package/src/nx-dot-input-policy.test.ts +0 -353
- package/src/nx-failure-life-cycles.test.ts +0 -207
- package/src/nx-files-archive-racy.test.ts +0 -97
- package/src/nx-foreign-workspace-env.test.ts +0 -329
- package/src/nx-incomplete-run-summary.test.ts +0 -70
- package/src/nx-plugin-load-errors.test.ts +0 -120
- package/src/nx-project-graph-errors.test.ts +0 -70
- package/src/nx-project-resolution-cache.test.ts +0 -176
- package/src/nx-raw-inputs.test.ts +0 -166
- package/src/nx-resident-graph-workers.test.ts +0 -112
- package/src/nx-run-information.test.ts +0 -118
- package/src/nx-skipped-task-exit.test.ts +0 -180
- package/src/nx-state-location.test.ts +0 -63
- package/src/nx-store-resolution.test.ts +0 -81
- package/src/nx-task-graph-cycles.test.ts +0 -148
- package/src/nx-task-output-lease.test.ts +0 -550
- package/src/nx-use-local.test.ts +0 -105
- package/src/owned-installation.test.ts +0 -105
- package/src/package-target-policy.test.ts +0 -1014
- package/src/producer-receipts.test.ts +0 -285
- package/src/release-config-policy.test.ts +0 -311
- package/src/runtime-input.test.ts +0 -393
- package/src/serial-group-wiring.test.ts +0 -326
- package/src/task-history-namespace.test.ts +0 -181
- package/src/task-results.test.ts +0 -228
- package/src/test-clock-policy.test.ts +0 -338
- package/src/test-program.test.ts +0 -303
- package/src/test-timings.test.ts +0 -240
- package/src/test-walk-policy.test.ts +0 -620
- package/src/testing.test.ts +0 -509
- package/src/typecheck-test-policy.test.ts +0 -1468
- package/src/typescript-resolution.test.ts +0 -76
- package/src/version-actions.test.ts +0 -80
- package/src/workspace-config-policy.test.ts +0 -737
|
@@ -1,17 +1,24 @@
|
|
|
1
1
|
import { spawn } from 'node:child_process';
|
|
2
2
|
import { accessSync, constants } from 'node:fs';
|
|
3
|
+
import { mkdtemp, rm } from 'node:fs/promises';
|
|
3
4
|
import { availableParallelism, userInfo } from 'node:os';
|
|
4
5
|
import { delimiter, isAbsolute, join, resolve as resolvePath } from 'node:path';
|
|
5
6
|
import { createInterface } from 'node:readline';
|
|
6
|
-
import { Duplex } from 'node:stream';
|
|
7
|
+
import { Duplex, Readable } from 'node:stream';
|
|
7
8
|
import { workspaceDataDirectoryForWorkspace } from 'nx/src/utils/cache-directory.js';
|
|
9
|
+
import { DEADLINE_PATIENCE, describeMembersLeft, killProcessGroup, processGroupHasMembers, signalProcessGroup, } from '../../process-group.js';
|
|
10
|
+
import { earliestBound, stepBounds, UNARMED } from './bounds.js';
|
|
8
11
|
import { CPU_POOL_ENV, createCpuPool, readInheritedCpuPool, takeParentCpuTokens, } from './cpu-pool.js';
|
|
9
|
-
import {
|
|
12
|
+
import { GATEWAY_SOCKET } from './gateway-lease.js';
|
|
10
13
|
import { captureHangDiagnostics, HANG_CAPTURE_MS } from './hang-diagnostics.js';
|
|
14
|
+
import { mbxLeaf, mbxPool, withMbxLeaf } from './mbx-pool.js';
|
|
11
15
|
import { createNoSpaceWatch } from './no-space.js';
|
|
16
|
+
import { COWSHED_EXECUTABLE_ENV, cowshedSubtreeOwner, HANDLE_FD, readAdmission, SUPERVISOR_SOCKET_ENV, } from './owned-subtree.js';
|
|
12
17
|
import { createRunProgress, withProgressReport } from './progress.js';
|
|
13
|
-
import {
|
|
14
|
-
import { createRunnerLifecycle, withRunnerLifecycle } from './runner-lifecycle.js';
|
|
18
|
+
import { createHostCommands, describeCoverageGap, describeRamTempError, describeSignalled, LEFTOVER_GRACE_MS, PRODUCTION, RAM_TEMP_CAPACITY_BYTES, RamTempVolume, ramTempPaths, } from './ram-temp.js';
|
|
19
|
+
import { createRunnerLifecycle, withFixtureLifecycle, withRunnerLifecycle } from './runner-lifecycle.js';
|
|
20
|
+
import { CPU_TOKENS_ENV, runnerDemand, sizeRunner } from './runner-sizing.js';
|
|
21
|
+
import { checkRuntimePin, describeRuntimePinRefusal } from './runtime-pin.js';
|
|
15
22
|
import { clearTaskDirectory, describeNoSpace, JUNIT_FILE, JUNIT_REPORT_ENV, judgeOperationalFailure, judgeRun, readJunitReport, taskDirectory, withJunitReport, writeRecord, } from './verdict.js';
|
|
16
23
|
const DEFAULT_KILL_AFTER_MS = 10_000;
|
|
17
24
|
const REAP_AFTER_FORCE_KILL_MS = 2_000;
|
|
@@ -23,9 +30,9 @@ const EXIT_CODE_BY_SIGNAL = {
|
|
|
23
30
|
/** The pool descriptor: only a grant may give it, so neither the target's nor the inherited environment passes it on. */
|
|
24
31
|
const POOL_ENV = [CPU_TOKENS_ENV, CPU_POOL_ENV];
|
|
25
32
|
/**
|
|
26
|
-
* The variables
|
|
27
|
-
*
|
|
28
|
-
* sub-pool names its own fifo over it.
|
|
33
|
+
* The variables a grant sets for the command it sizes: no target's environment may set them. A jobserver the executor
|
|
34
|
+
* itself inherits, a parent make's or its cowshed job's, still reaches an ungranted command by Cargo's own rule; a
|
|
35
|
+
* granted one's sub-pool names its own fifo over it.
|
|
29
36
|
*/
|
|
30
37
|
const BUDGET_ENV = [...POOL_ENV, 'CARGO_MAKEFLAGS', 'MAKEFLAGS', 'MFLAGS'];
|
|
31
38
|
/** Workspace-provided cowshed launchers, in the order a workspace that provides one names them. */
|
|
@@ -73,22 +80,36 @@ export function resolveCowshedExecutable(root, named, inherited, pathValue, cwd,
|
|
|
73
80
|
export default function boundedExecExecutor(options, context) {
|
|
74
81
|
return runDefaultBoundedExec(options, recordContext(context));
|
|
75
82
|
}
|
|
76
|
-
/**
|
|
83
|
+
/**
|
|
84
|
+
* Runs with the normal process, RAM lease and CPU pool owners, with an optional stdout sink, on the user's RAM temp
|
|
85
|
+
* volume where there is one, through the host's own gateway: the Nx executor's runs, never a test's.
|
|
86
|
+
*/
|
|
77
87
|
export function runDefaultBoundedExec(options, context) {
|
|
88
|
+
// Only macOS has `hdiutil ram://`; elsewhere the command keeps the inherited TMPDIR.
|
|
89
|
+
return runBoundedExecWith(options, context, (cowshed) => process.platform === 'darwin'
|
|
90
|
+
? new RamTempVolume(ramTempPaths(userInfo(), PRODUCTION), RAM_TEMP_CAPACITY_BYTES, createHostCommands(GATEWAY_SOCKET, undefined, undefined, cowshed), undefined, undefined, 'optional')
|
|
91
|
+
: null);
|
|
92
|
+
}
|
|
93
|
+
/**
|
|
94
|
+
* The default runner with every owner but its RAM volume, which its caller makes: `volume` makes the volume for the
|
|
95
|
+
* cowshed the run's disk commands go through. The Nx executor makes the user's; a host proof passes a fixture volume
|
|
96
|
+
* (`fixtureRamVolume`), so no test leases, creates or detaches the user's volume.
|
|
97
|
+
*/
|
|
98
|
+
export function runBoundedExecWith(options, context, volume) {
|
|
78
99
|
// A target that names the cowshed its disk commands run through names it for this run's own RAM volume work and
|
|
79
100
|
// for its command alike. Nx resolves `{workspaceRoot}/…` in an option to a workspace-relative path, which the
|
|
80
101
|
// command, starting in another directory, could not find: it gets the absolute one.
|
|
81
102
|
const named = options.env?.[COWSHED_EXECUTABLE_ENV];
|
|
82
103
|
const cowshed = resolveCowshedExecutable(context.root, named, process.env[COWSHED_EXECUTABLE_ENV], process.env.PATH, process.cwd());
|
|
83
104
|
const run = cowshed === undefined ? options : { ...options, env: { ...options.env, [COWSHED_EXECUTABLE_ENV]: cowshed } };
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
105
|
+
const tempVolume = volume(cowshed);
|
|
106
|
+
// The command's processes are owned by the supervisor that serves this job, when one does
|
|
107
|
+
// (specs/cowshed/07_api.md, `JobSubtreeHandle`); with none, nothing here may end what it leaves.
|
|
108
|
+
const owner = process.env[SUPERVISOR_SOCKET_ENV] === undefined ? null : cowshedSubtreeOwner(cowshed ?? 'cowshed');
|
|
88
109
|
// A command a parent's CPU pool started draws its tokens from that pool. The pool is named by the environment the
|
|
89
110
|
// parent set for this command; a run no parent started has none.
|
|
90
111
|
const inherited = readInheritedCpuPool(process.env);
|
|
91
|
-
return runBoundedExec(run, context, createProcessTreeKiller(), tempVolume,
|
|
112
|
+
return runBoundedExec(run, context, createProcessTreeKiller(), tempVolume, inherited, captureHangDiagnostics, owner);
|
|
92
113
|
}
|
|
93
114
|
/**
|
|
94
115
|
* Nx gives an executor the task's project, target and configuration, not its id; the id is the
|
|
@@ -113,17 +134,15 @@ export function recordContext(context) {
|
|
|
113
134
|
};
|
|
114
135
|
}
|
|
115
136
|
/**
|
|
116
|
-
* Run the command
|
|
117
|
-
*
|
|
118
|
-
* granted: a wait for the host is not the command's time.
|
|
137
|
+
* Run the command, sized to the tokens its parent's CPU pool (`cpu-pool.ts`) lends it; a run that inherits no pool
|
|
138
|
+
* runs it as asked. Both bounds start once it is granted: a wait for the pool is not the command's time.
|
|
119
139
|
*
|
|
120
|
-
* `inherited` is the parent's
|
|
121
|
-
*
|
|
122
|
-
*
|
|
123
|
-
*
|
|
124
|
-
* its own exit, and the run says so.
|
|
140
|
+
* `inherited` is the parent's pool: an enclosing bounded-exec's sub-pool or the Nx run's own (`run-cpu-pool.ts`). A
|
|
141
|
+
* granted run serves a sub-pool of its grant to its own descendants, closed before the grant returns. A grant whose
|
|
142
|
+
* connection closes while the command runs (the lending parent's exit) ends its accounting, never the command: the
|
|
143
|
+
* command runs on over budget to its own exit, and the run says so.
|
|
125
144
|
*/
|
|
126
|
-
export async function runBoundedExec(options, context, killer, tempVolume,
|
|
145
|
+
export async function runBoundedExec(options, context, killer, tempVolume, inherited = null, captureDiagnostics = captureHangDiagnostics, owner = null) {
|
|
127
146
|
const cwd = resolveCwd(options.cwd, context.root);
|
|
128
147
|
const directory = context.record?.directory ?? null;
|
|
129
148
|
// A previous run's verdict and report go first, so neither can be read as this run's.
|
|
@@ -134,13 +153,18 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
|
|
|
134
153
|
const timeoutMs = options.timeoutMs;
|
|
135
154
|
const idleTimeoutMs = options.idleTimeoutMs;
|
|
136
155
|
const killAfterMs = options.killAfterMs ?? DEFAULT_KILL_AFTER_MS;
|
|
156
|
+
const fixture = options.fixtureCommand !== undefined && options.fixtureTimeoutMs !== undefined
|
|
157
|
+
? { command: options.fixtureCommand, timeoutMs: options.fixtureTimeoutMs }
|
|
158
|
+
: null;
|
|
137
159
|
const outputChunks = [];
|
|
138
160
|
const state = {
|
|
139
161
|
settled: false,
|
|
140
162
|
expiry: null,
|
|
141
163
|
forceKillNeeded: false,
|
|
142
164
|
poolCloseError: null,
|
|
165
|
+
retireError: null,
|
|
143
166
|
noSpace: null,
|
|
167
|
+
lockWaits: [],
|
|
144
168
|
};
|
|
145
169
|
const appendStdout = (chunk) => {
|
|
146
170
|
const text = chunk.toString();
|
|
@@ -157,9 +181,55 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
|
|
|
157
181
|
outputChunks.push(text);
|
|
158
182
|
process.stderr.write(text);
|
|
159
183
|
};
|
|
184
|
+
// Told, never thrown, wherever the command is being ended: a diagnostic must neither replace the failure it
|
|
185
|
+
// accompanies nor skip the cleanup that follows (the signal handlers, the CPU grant, the RAM lease, the run's record).
|
|
186
|
+
const tell = (text) => {
|
|
187
|
+
try {
|
|
188
|
+
appendStderr(text);
|
|
189
|
+
}
|
|
190
|
+
catch {
|
|
191
|
+
// Nobody is left to tell.
|
|
192
|
+
}
|
|
193
|
+
};
|
|
194
|
+
// The fixture pair names one segment: a fixture with no build budget, or a budget with no fixture, is a
|
|
195
|
+
// target that does not say what it runs, so it does not run.
|
|
196
|
+
if ((options.fixtureCommand === undefined) !== (options.fixtureTimeoutMs === undefined) ||
|
|
197
|
+
(fixture !== null && !(fixture.timeoutMs > 0))) {
|
|
198
|
+
appendStderr('bounded-exec: fixtureCommand and a positive fixtureTimeoutMs must be named together; refusing to run the command\n');
|
|
199
|
+
if (context.record !== undefined) {
|
|
200
|
+
const { task, directory: recordDirectory } = context.record;
|
|
201
|
+
await writeRecord(recordDirectory, { task: task.id, hash: task.hash, verdict: judgeOperationalFailure(1, null) });
|
|
202
|
+
}
|
|
203
|
+
return { success: false, terminalOutput: outputChunks.join('') };
|
|
204
|
+
}
|
|
205
|
+
// A leg that runs bun runs the Bun its workspace declares, or it does not run. This comes before the RAM lease and the
|
|
206
|
+
// CPU tokens: a refusal that waited in its pool's queue first would say nothing for minutes.
|
|
207
|
+
const runtime = await checkRuntimePin({
|
|
208
|
+
root: context.root,
|
|
209
|
+
command: fixture === null ? requested : `${fixture.command} && ${requested}`,
|
|
210
|
+
env: { ...withoutPoolEnv(process.env), ...options.env },
|
|
211
|
+
cwd,
|
|
212
|
+
pathSource: options.env?.PATH === undefined ? 'the inherited environment' : "the target's env",
|
|
213
|
+
});
|
|
214
|
+
if (runtime !== null) {
|
|
215
|
+
appendStderr(describeRuntimePinRefusal(runtime, context.record?.task.id));
|
|
216
|
+
if (context.record !== undefined) {
|
|
217
|
+
const { task, directory: recordDirectory } = context.record;
|
|
218
|
+
await writeRecord(recordDirectory, { task: task.id, hash: task.hash, verdict: judgeOperationalFailure(1, null) });
|
|
219
|
+
}
|
|
220
|
+
return { success: false, terminalOutput: outputChunks.join('') };
|
|
221
|
+
}
|
|
160
222
|
let lease = null;
|
|
161
|
-
|
|
162
|
-
|
|
223
|
+
let ordinaryTemp = null;
|
|
224
|
+
// Before spawn there is nothing to retire. Once started, only the owner's terminal retirement permits deletion.
|
|
225
|
+
let subtreeRetired = true;
|
|
226
|
+
// A lease is removed when its command ends, so it is taken only where whatever the command leaves
|
|
227
|
+
// in it can be ended by its owner first: the supervisor's subtree retirement, never a census.
|
|
228
|
+
if (tempVolume !== null && owner === null) {
|
|
229
|
+
appendStderr(`RAM temp volume not leased: no cowshed supervisor owns this command's processes (${SUPERVISOR_SOCKET_ENV} is unset); TMPDIR stays ${process.env.TMPDIR ?? '(unset)'}\n`);
|
|
230
|
+
}
|
|
231
|
+
else if (tempVolume !== null && owner !== null) {
|
|
232
|
+
const acquired = await tempVolume.acquire(owner);
|
|
163
233
|
if (!acquired.ok) {
|
|
164
234
|
const message = `${describeRamTempError(acquired.error)}\n`;
|
|
165
235
|
process.stderr.write(message);
|
|
@@ -171,78 +241,149 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
|
|
|
171
241
|
appendStderr(`RAM temp volume: dead lease ${held}\n`);
|
|
172
242
|
}
|
|
173
243
|
for (const reaped of acquired.value.reaped) {
|
|
174
|
-
appendStderr(`RAM temp volume: stopped ${reaped}, which a dead task left
|
|
244
|
+
appendStderr(`RAM temp volume: the supervisor stopped ${reaped}, which a dead task's command left running\n`);
|
|
245
|
+
}
|
|
246
|
+
for (const gap of acquired.value.gaps) {
|
|
247
|
+
appendStderr(`RAM temp volume: the supervisor's view of a dead task's command has a gap: ${gap}\n`);
|
|
248
|
+
}
|
|
249
|
+
for (const retained of acquired.value.retained) {
|
|
250
|
+
appendStderr(`RAM temp volume: dead lease ${retained}\n`);
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
else if (acquired.value.kind === 'unavailable') {
|
|
254
|
+
const unavailable = `RAM temp volume unavailable (${acquired.value.reason}: ${acquired.value.detail})`;
|
|
255
|
+
if (options.env?.TMPDIR !== undefined) {
|
|
256
|
+
appendStderr(`${unavailable}; using target-provided TMPDIR ${options.env.TMPDIR} (medium not inferred)\n`);
|
|
257
|
+
}
|
|
258
|
+
else {
|
|
259
|
+
try {
|
|
260
|
+
ordinaryTemp = await mkdtemp(join(tempVolume.ordinaryParent, `smoo-bounded-exec-${process.pid}-`));
|
|
261
|
+
}
|
|
262
|
+
catch (error) {
|
|
263
|
+
appendStderr(`${unavailable}; could not create ordinary scratch under ${tempVolume.ordinaryParent}: ${error instanceof Error ? error.message : String(error)}\n`);
|
|
264
|
+
return { success: false, terminalOutput: outputChunks.join('') };
|
|
265
|
+
}
|
|
266
|
+
appendStderr(`${unavailable}; using owned ordinary TMPDIR ${ordinaryTemp}\n`);
|
|
175
267
|
}
|
|
176
268
|
}
|
|
177
269
|
else {
|
|
178
270
|
appendStderr(`RAM temp volume unavailable in this sandbox (${acquired.value.detail}); TMPDIR stays ${process.env.TMPDIR ?? '(unset)'}\n`);
|
|
179
271
|
}
|
|
180
272
|
}
|
|
181
|
-
|
|
182
|
-
|
|
273
|
+
// The supervisor's answer to the command's own admission, once it is spawned as an owned subtree root, and once
|
|
274
|
+
// any lease has recorded the handle for whoever settles that lease should this task die first.
|
|
275
|
+
let admission = null;
|
|
276
|
+
const retireSubtree = async () => {
|
|
277
|
+
if (admission === null || owner === null) {
|
|
183
278
|
return;
|
|
184
279
|
}
|
|
185
|
-
const
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
if (reaped?.ok) {
|
|
191
|
-
for (const leftover of reaped.value) {
|
|
192
|
-
appendStderr(`RAM temp volume: stopped ${leftover}, which the command left working in its lease\n`);
|
|
193
|
-
}
|
|
194
|
-
}
|
|
195
|
-
else if (reaped) {
|
|
196
|
-
appendStderr(`${describeRamTempError(reaped.error)}\n`);
|
|
280
|
+
const answer = await admission;
|
|
281
|
+
admission = null;
|
|
282
|
+
if (answer.kind !== 'admitted') {
|
|
283
|
+
appendStderr(`cowshed: the command ran without an owned subtree (${answer.kind === 'refused' ? `supervisor: ${answer.reason}` : answer.detail}); nothing retires what it left running\n`);
|
|
284
|
+
return;
|
|
197
285
|
}
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
286
|
+
// What the command left running is a leak, whichever way the command ended: its supervisor ends exactly the
|
|
287
|
+
// processes its tree records under the command, before any lease goes, and each is said so its owner fixes it.
|
|
288
|
+
const retired = await owner.retire(answer.handle, LEFTOVER_GRACE_MS);
|
|
289
|
+
switch (retired.kind) {
|
|
290
|
+
case 'retired':
|
|
291
|
+
subtreeRetired = true;
|
|
292
|
+
for (const entry of retired.signalled) {
|
|
293
|
+
appendStderr(`cowshed: the supervisor stopped ${describeSignalled(entry)}, which the command left running\n`);
|
|
294
|
+
}
|
|
295
|
+
if (retired.signalledIncomplete) {
|
|
296
|
+
appendStderr("cowshed: the supervisor may have stopped processes of the command's that it could not name\n");
|
|
297
|
+
}
|
|
298
|
+
for (const gap of retired.gaps) {
|
|
299
|
+
appendStderr(`cowshed: the supervisor's view of the command's processes has a gap: ${describeCoverageGap(gap)}\n`);
|
|
300
|
+
}
|
|
301
|
+
return;
|
|
302
|
+
case 'refused':
|
|
303
|
+
appendStderr(`cowshed: the supervisor refused to retire the command's processes (${retired.reason})\n`);
|
|
304
|
+
return;
|
|
305
|
+
case 'failed':
|
|
306
|
+
state.retireError = retired.detail;
|
|
307
|
+
appendStderr(`cowshed: could not retire the command's processes: ${retired.detail}\n`);
|
|
308
|
+
return;
|
|
201
309
|
}
|
|
202
310
|
};
|
|
203
|
-
//
|
|
204
|
-
|
|
311
|
+
// Failed scratch remains evidence: RAM leases are archived; ordinary scratch is retained at its reported path.
|
|
312
|
+
let tempReleased = null;
|
|
313
|
+
const releaseLease = (keep) => {
|
|
314
|
+
tempReleased ??= (async () => {
|
|
315
|
+
await retireSubtree();
|
|
316
|
+
if (ordinaryTemp !== null) {
|
|
317
|
+
const held = ordinaryTemp;
|
|
318
|
+
ordinaryTemp = null;
|
|
319
|
+
if (keep || state.retireError !== null || !subtreeRetired) {
|
|
320
|
+
appendStderr(`Ordinary temp directory retained at ${held}: ${!subtreeRetired ? 'the command subtree was not retired' : 'the run did not pass'}\n`);
|
|
321
|
+
}
|
|
322
|
+
else {
|
|
323
|
+
try {
|
|
324
|
+
await rm(held, { recursive: true, force: true });
|
|
325
|
+
}
|
|
326
|
+
catch (error) {
|
|
327
|
+
appendStderr(`Ordinary temp directory retained at ${held}: cleanup failed: ${error instanceof Error ? error.message : String(error)}\n`);
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
if (lease === null || owner === null) {
|
|
332
|
+
return;
|
|
333
|
+
}
|
|
334
|
+
const held = lease;
|
|
335
|
+
lease = null;
|
|
336
|
+
const released = await tempVolume?.release(held, keep || state.retireError !== null, owner);
|
|
337
|
+
if (released?.ok) {
|
|
338
|
+
for (const retained of released.value) {
|
|
339
|
+
appendStderr(`RAM temp volume: lease ${retained}\n`);
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
else if (released) {
|
|
343
|
+
appendStderr(`${describeRamTempError(released.error)}\n`);
|
|
344
|
+
}
|
|
345
|
+
})();
|
|
346
|
+
return tempReleased;
|
|
347
|
+
};
|
|
348
|
+
// The wait is one span line, as cowshed's own spans read. Every Nx run that inherits no pool serves its own
|
|
349
|
+
// (`run-cpu-pool.ts`), so a task takes its tokens from a parent's pool; a command that inherits none runs exactly as
|
|
205
350
|
// asked and nothing is said.
|
|
206
351
|
const cores = availableParallelism();
|
|
207
352
|
const demand = runnerDemand(requested, options.parallelism, cores, process.env, options.env);
|
|
208
353
|
const asked = Date.now();
|
|
209
|
-
// A
|
|
210
|
-
// A pool
|
|
211
|
-
// relay to the host -- so asking the host beside it would count the run twice; the run fails instead.
|
|
354
|
+
// A pool that answered and refused is a live budget, and running beside it would count the run twice: the run fails
|
|
355
|
+
// instead. A pool whose endpoint is gone (its holder exited) lends nothing, and the run goes on as asked.
|
|
212
356
|
let grant = null;
|
|
213
357
|
let parentRefusal = null;
|
|
358
|
+
let parentWait;
|
|
214
359
|
if (inherited !== null) {
|
|
215
360
|
const fromParent = await takeParentCpuTokens(inherited, demand.want);
|
|
216
361
|
if (fromParent.granted) {
|
|
217
362
|
grant = fromParent;
|
|
218
363
|
}
|
|
219
364
|
else if (fromParent.cause === 'absent') {
|
|
220
|
-
appendStderr(`cowshed: cpu-
|
|
365
|
+
appendStderr(`cowshed: cpu-pool parent pool absent: ${fromParent.reason}; running the command as asked\n`);
|
|
221
366
|
}
|
|
222
367
|
else {
|
|
223
368
|
parentRefusal = fromParent.reason;
|
|
224
|
-
|
|
225
|
-
}
|
|
226
|
-
let unbudgeted = null;
|
|
227
|
-
if (grant === null && parentRefusal === null && budget !== null) {
|
|
228
|
-
const fromHost = await budget.take(demand.want, context.root, requested);
|
|
229
|
-
if (fromHost.granted) {
|
|
230
|
-
grant = fromHost;
|
|
231
|
-
}
|
|
232
|
-
else if (fromHost.cause !== 'absent') {
|
|
233
|
-
unbudgeted = fromHost.reason;
|
|
369
|
+
parentWait = fromParent.wait;
|
|
234
370
|
}
|
|
235
371
|
}
|
|
236
372
|
const waitedMs = Date.now() - asked;
|
|
237
373
|
let cpu;
|
|
238
374
|
let sized = { command: requested, env: {} };
|
|
375
|
+
// With mbx the leaf's admission is mbx's one pool, not a parent's tokens: its runner is sized to the job's count,
|
|
376
|
+
// and a non-Cargo leaf holds that much of the pool for its run (specs/cowshed/05_gateway.md, "Parallelism inside a
|
|
377
|
+
// job"). No grant and no sub-pool beside it.
|
|
378
|
+
const mbx = mbxPool(process.env);
|
|
379
|
+
const leaf = mbx === null ? null : mbxLeaf(mbx, demand.kind, demand.want);
|
|
239
380
|
if (grant !== null) {
|
|
240
381
|
sized = sizeRunner(demand, requested, grant.tokens);
|
|
241
|
-
cpu = { tokens: grant.tokens, want: demand.want, waitedMs
|
|
242
|
-
appendStderr(`cowshed: cpu-
|
|
382
|
+
cpu = { tokens: grant.tokens, want: demand.want, waitedMs };
|
|
383
|
+
appendStderr(`cowshed: cpu-pool wait done elapsed=${waitedMs}ms status=ok tokens=${grant.tokens}/${demand.want} runner=${demand.kind}\n`);
|
|
243
384
|
}
|
|
244
|
-
else if (
|
|
245
|
-
|
|
385
|
+
else if (leaf !== null) {
|
|
386
|
+
sized = sizeRunner(demand, requested, demand.want);
|
|
246
387
|
}
|
|
247
388
|
// The run's sub-pool and grant are given back exactly once, however the run ends. The sub-pool closes first, so no
|
|
248
389
|
// descendant still draws on tokens the grant has returned.
|
|
@@ -264,9 +405,9 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
|
|
|
264
405
|
return cpuReleased;
|
|
265
406
|
};
|
|
266
407
|
// Records the run's verdict when it has a task to record as. An `expiry` is set only when a bound ended the run.
|
|
267
|
-
// An `operational` run failed on its CPU
|
|
268
|
-
//
|
|
269
|
-
const recordRun = async (exitCode, expiry, elapsedMs, operational, noSpace) => {
|
|
408
|
+
// An `operational` run failed on its CPU pool (a sub-pool that did not close): an ordinary failure, whatever bound also
|
|
409
|
+
// fired, so it is never judged as a bound.
|
|
410
|
+
const recordRun = async (exitCode, expiry, elapsedMs, operational, noSpace, lockWaits) => {
|
|
270
411
|
if (context.record === undefined) {
|
|
271
412
|
return;
|
|
272
413
|
}
|
|
@@ -274,44 +415,34 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
|
|
|
274
415
|
const report = await readJunitReport(recordDirectory);
|
|
275
416
|
const verdict = operational
|
|
276
417
|
? judgeOperationalFailure(exitCode, report)
|
|
277
|
-
: judgeRun({ exitCode, elapsedMs, expiry, report, noSpace });
|
|
418
|
+
: judgeRun({ exitCode, elapsedMs, expiry, report, noSpace, lockWaits });
|
|
278
419
|
if (verdict.outcome === 'bound' && verdict.bound === 'test') {
|
|
279
420
|
appendStderr(`Every failing test failed only on its per-test timeout: ${verdict.tests.join(', ')}\n`);
|
|
280
421
|
}
|
|
281
|
-
await writeRecord(recordDirectory, {
|
|
422
|
+
await writeRecord(recordDirectory, {
|
|
423
|
+
task: task.id,
|
|
424
|
+
hash: task.hash,
|
|
425
|
+
verdict,
|
|
426
|
+
...(cpu ? { cpu } : {}),
|
|
427
|
+
...(parentWait ? { cpuWait: parentWait } : {}),
|
|
428
|
+
});
|
|
282
429
|
};
|
|
283
430
|
// A run refused before its command starts says why, gives back its grant and RAM, and records the failure.
|
|
284
431
|
const refuse = async (message) => {
|
|
285
432
|
appendStderr(message);
|
|
286
433
|
await releaseCpu();
|
|
287
|
-
|
|
288
|
-
await
|
|
434
|
+
// The command never started, so its lease holds nothing of its own to keep.
|
|
435
|
+
await releaseLease(false);
|
|
436
|
+
await recordRun(1, null, Date.now() - asked, false, null, []);
|
|
289
437
|
return { success: false, terminalOutput: outputChunks.join('') };
|
|
290
438
|
};
|
|
291
439
|
if (parentRefusal !== null) {
|
|
292
|
-
return refuse(`cowshed: cpu-
|
|
293
|
-
}
|
|
294
|
-
// Descendant work queued for host CPU tokens, as its pool hears it (`host-wait`): while any is held the idle clock
|
|
295
|
-
// is stopped, because a queued job is not an idle one (specs/cowshed/05_gateway.md, "The bounds start at the
|
|
296
|
-
// grant"). The clock it stops is made with the command below; a wait held before then is counted and honoured there.
|
|
297
|
-
let heldWaits = 0;
|
|
298
|
-
let idleClock = null;
|
|
299
|
-
const hostWaits = {
|
|
300
|
-
begin: () => {
|
|
301
|
-
heldWaits += 1;
|
|
302
|
-
if (heldWaits === 1)
|
|
303
|
-
idleClock?.stop();
|
|
304
|
-
},
|
|
305
|
-
end: () => {
|
|
306
|
-
heldWaits -= 1;
|
|
307
|
-
if (heldWaits === 0)
|
|
308
|
-
idleClock?.restart();
|
|
309
|
-
},
|
|
310
|
-
};
|
|
440
|
+
return refuse(`cowshed: cpu-pool parent pool refused: ${parentRefusal}; refusing to run the command\n`);
|
|
441
|
+
}
|
|
311
442
|
// A granted run serves a sub-pool of its own grant, so its descendants draw from that and never from above it.
|
|
312
|
-
// A pool that cannot be set up refuses the run: no command runs
|
|
443
|
+
// A pool that cannot be set up refuses the run: no granted command runs beside its grant.
|
|
313
444
|
if (grant !== null) {
|
|
314
|
-
const created = await createCpuPool(grant.tokens, grant.closed
|
|
445
|
+
const created = await createCpuPool(grant.tokens, grant.closed);
|
|
315
446
|
if (!created.ok) {
|
|
316
447
|
return refuse(`cowshed: cpu pool unavailable: ${created.error}; refusing to run the command\n`);
|
|
317
448
|
}
|
|
@@ -330,26 +461,62 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
|
|
|
330
461
|
...sized.env,
|
|
331
462
|
...(pool?.environment ?? {}),
|
|
332
463
|
};
|
|
333
|
-
|
|
464
|
+
// A fixture rides as segment 0 of the lifecycle. Without the channel (Windows) it is plain shell text ahead
|
|
465
|
+
// of the command: the progress bound runs during it and its own ceiling does not.
|
|
466
|
+
const hooked = process.platform === 'win32'
|
|
467
|
+
? null
|
|
468
|
+
: fixture === null
|
|
469
|
+
? withRunnerLifecycle(command)
|
|
470
|
+
: withFixtureLifecycle(fixture.command, command);
|
|
334
471
|
const lifecycle = hooked === null ? null : createRunnerLifecycle(hooked.runners);
|
|
335
472
|
let lifecycleFailure = null;
|
|
336
473
|
const isTesting = () => lifecycleFailure !== null || (lifecycle?.isTesting() ?? true);
|
|
337
474
|
const startedAt = Date.now();
|
|
475
|
+
// With a supervisor, `cowshed subtree-root` asks it for a handle on itself, writes the answer to the handle
|
|
476
|
+
// descriptor, and execs `/bin/sh -c command` in the same process: the command's pid, group and life are unchanged.
|
|
477
|
+
const unadmitted = hooked?.command ?? (fixture === null ? command : `${fixture.command} && ${command}`);
|
|
478
|
+
const shellCommand = leaf === null ? unadmitted : withMbxLeaf(unadmitted, leaf);
|
|
479
|
+
const root = owner?.rootArgv(shellCommand);
|
|
338
480
|
let child;
|
|
339
481
|
try {
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
482
|
+
// fd 3 is the runner lifecycle's duplex socket when the command is hooked; the subtree admission takes fd 4, which
|
|
483
|
+
// `cowshed subtree-root` closes before it execs, so fd 3 reaches the shell unchanged.
|
|
484
|
+
child =
|
|
485
|
+
root === undefined
|
|
486
|
+
? spawn(shellCommand, [], {
|
|
487
|
+
cwd,
|
|
488
|
+
env: mergeEnv(lease?.directory ?? ordinaryTemp, env),
|
|
489
|
+
shell: true,
|
|
490
|
+
stdio: hooked === null ? 'pipe' : ['pipe', 'pipe', 'pipe', 'pipe'],
|
|
491
|
+
detached: process.platform !== 'win32',
|
|
492
|
+
windowsHide: true,
|
|
493
|
+
})
|
|
494
|
+
: spawn(root.file, root.args, {
|
|
495
|
+
cwd,
|
|
496
|
+
env: mergeEnv(lease?.directory ?? ordinaryTemp, env),
|
|
497
|
+
stdio: ['pipe', 'pipe', 'pipe', hooked === null ? 'ignore' : 'pipe', 'pipe'],
|
|
498
|
+
detached: process.platform !== 'win32',
|
|
499
|
+
windowsHide: true,
|
|
500
|
+
});
|
|
348
501
|
}
|
|
349
502
|
catch (error) {
|
|
350
503
|
// Nothing started, so nothing runs under the grant: it goes back here, with the RAM lease.
|
|
351
504
|
return refuse(`cowshed: could not start the command: ${error instanceof Error ? error.message : String(error)}\n`);
|
|
352
505
|
}
|
|
506
|
+
subtreeRetired = false;
|
|
507
|
+
if (root !== undefined) {
|
|
508
|
+
const channel = child.stdio[HANDLE_FD];
|
|
509
|
+
const leased = lease;
|
|
510
|
+
admission = readAdmission(channel instanceof Readable ? channel : null).then(async (answer) => {
|
|
511
|
+
if (answer.kind === 'admitted' && leased !== null && tempVolume !== null) {
|
|
512
|
+
const recorded = await tempVolume.record(leased, answer.handle);
|
|
513
|
+
if (!recorded.ok) {
|
|
514
|
+
appendStderr(`${describeRamTempError(recorded.error)}\n`);
|
|
515
|
+
}
|
|
516
|
+
}
|
|
517
|
+
return answer;
|
|
518
|
+
});
|
|
519
|
+
}
|
|
353
520
|
// Progress is a finished test, not output: a runner's SLOW warnings and a hung test's heartbeat keep printing.
|
|
354
521
|
const progress = createRunProgress(command);
|
|
355
522
|
// A write that found no space is said once, in the output, by whatever made it; the volume is read when it is said,
|
|
@@ -359,24 +526,22 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
|
|
|
359
526
|
const held = lease;
|
|
360
527
|
return { line, volume: held === null ? null : ((await tempVolume?.space(held)) ?? null) };
|
|
361
528
|
};
|
|
362
|
-
|
|
529
|
+
// Which bound is armed, and when it expires, is bounds.ts's decision; one timer waits for the earliest of them.
|
|
530
|
+
let bounds = UNARMED;
|
|
531
|
+
let boundTimer;
|
|
532
|
+
// The fixture's own output is compilation, not the command's progress: it is not scanned for finished tests.
|
|
533
|
+
let fixtureDone = fixture === null || lifecycle === null;
|
|
534
|
+
// Whether a segment past the fixture ever announced: without one, the shell's exit is the fixture's.
|
|
535
|
+
let mainStarted = fixture === null || lifecycle === null;
|
|
363
536
|
let captureTimer;
|
|
364
537
|
let idleCapture;
|
|
365
538
|
const cancelCapture = () => {
|
|
366
539
|
clearTimeout(captureTimer);
|
|
367
540
|
idleCapture?.controller.abort();
|
|
368
541
|
};
|
|
369
|
-
|
|
370
|
-
// wait keeps it unspent: the clock is stopped while descendant work is queued for CPU tokens, and runs on from where
|
|
371
|
-
// it stopped once the last wait ends. `queuedMs` is the queued time this interval has not counted, for the report.
|
|
372
|
-
let idleLeftMs = idleTimeoutMs ?? 0;
|
|
373
|
-
let queuedMs = 0;
|
|
374
|
-
let queuedSinceMs = null;
|
|
375
|
-
const scheduleIdle = (limitMs, leftMs) => {
|
|
376
|
-
clearTimeout(idleTimer);
|
|
377
|
-
cancelCapture();
|
|
542
|
+
const scheduleCapture = (expiresAtMs) => {
|
|
378
543
|
const capture = {
|
|
379
|
-
expiresAtMs
|
|
544
|
+
expiresAtMs,
|
|
380
545
|
controller: new AbortController(),
|
|
381
546
|
result: { kind: 'pending', startedAtMs: null },
|
|
382
547
|
};
|
|
@@ -409,65 +574,42 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
|
|
|
409
574
|
};
|
|
410
575
|
}
|
|
411
576
|
});
|
|
412
|
-
}, Math.max(0,
|
|
413
|
-
idleTimer = setTimeout(() => expire({ kind: 'idle', limitMs, tests: progress.inFlight() }, queuedMs), leftMs);
|
|
577
|
+
}, Math.max(0, expiresAtMs - Date.now() - HANG_CAPTURE_MS));
|
|
414
578
|
};
|
|
415
|
-
const
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
if (
|
|
422
|
-
queuedSinceMs = Date.now();
|
|
579
|
+
const expiredAs = (kind) => kind === 'idle'
|
|
580
|
+
? { kind, limitMs: idleTimeoutMs ?? 0, tests: progress.inFlight() }
|
|
581
|
+
: kind === 'fixture'
|
|
582
|
+
? { kind, limitMs: fixture?.timeoutMs ?? 0 }
|
|
583
|
+
: { kind, limitMs: timeoutMs ?? 0 };
|
|
584
|
+
const stepTo = (event) => {
|
|
585
|
+
if (state.settled || state.expiry !== null)
|
|
423
586
|
return;
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
}
|
|
432
|
-
const now = Date.now();
|
|
433
|
-
queuedSinceMs = now;
|
|
434
|
-
if (idleCapture !== undefined) {
|
|
435
|
-
idleLeftMs = Math.max(0, idleCapture.expiresAtMs - now);
|
|
436
|
-
}
|
|
437
|
-
clearTimeout(idleTimer);
|
|
587
|
+
const idleBefore = bounds.idle;
|
|
588
|
+
bounds = stepBounds(bounds, {
|
|
589
|
+
...(idleTimeoutMs === undefined ? {} : { idleMs: idleTimeoutMs }),
|
|
590
|
+
...(fixture === null || lifecycle === null ? {} : { fixtureMs: fixture.timeoutMs }),
|
|
591
|
+
...(timeoutMs === undefined ? {} : { totalMs: timeoutMs }),
|
|
592
|
+
}, event);
|
|
593
|
+
if (bounds.idle !== idleBefore) {
|
|
438
594
|
cancelCapture();
|
|
439
595
|
idleCapture = undefined;
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
queuedMs += waitedMs;
|
|
448
|
-
queuedSinceMs = null;
|
|
449
|
-
if (!isTesting()) {
|
|
450
|
-
appendStderr(`bounded-exec: host CPU wait ended after ${waitedMs}ms; no test runner is active\n`);
|
|
451
|
-
return;
|
|
452
|
-
}
|
|
453
|
-
appendStderr(`bounded-exec: idle clock restarted: no descendant's work is queued for host CPU tokens after ${waitedMs}ms; ${idleLeftMs}ms of idleTimeoutMs=${idleTimeoutMs} left\n`);
|
|
454
|
-
scheduleIdle(idleTimeoutMs, idleLeftMs);
|
|
455
|
-
},
|
|
596
|
+
if (bounds.idle !== null)
|
|
597
|
+
scheduleCapture(bounds.idle);
|
|
598
|
+
}
|
|
599
|
+
clearTimeout(boundTimer);
|
|
600
|
+
const next = earliestBound(bounds);
|
|
601
|
+
if (next !== null)
|
|
602
|
+
boundTimer = setTimeout(() => expire(expiredAs(next.kind)), Math.max(0, next.at - Date.now()));
|
|
456
603
|
};
|
|
457
604
|
// Only the CHILD's own output is read for progress. The diagnostics below go
|
|
458
605
|
// through appendStderr as well, and re-arming on those would let a teardown
|
|
459
606
|
// message extend the very bound that just fired.
|
|
460
607
|
const updateIdle = (finished, wasTesting) => {
|
|
461
|
-
|
|
462
|
-
clearTimeout(idleTimer);
|
|
463
|
-
cancelCapture();
|
|
464
|
-
idleCapture = undefined;
|
|
465
|
-
}
|
|
466
|
-
else if (finished || !wasTesting) {
|
|
467
|
-
armIdleTimer();
|
|
468
|
-
}
|
|
608
|
+
stepTo({ kind: 'progress', at: Date.now(), testing: isTesting(), wasTesting, finished });
|
|
469
609
|
};
|
|
470
610
|
const scanProgress = (stream, text) => {
|
|
611
|
+
if (!fixtureDone)
|
|
612
|
+
return;
|
|
471
613
|
updateIdle(progress.scan(stream, text), isTesting());
|
|
472
614
|
};
|
|
473
615
|
if (lifecycle !== null) {
|
|
@@ -481,7 +623,8 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
|
|
|
481
623
|
appendStderr(`bounded-exec: runner lifecycle failed: ${detail}\n`);
|
|
482
624
|
updateIdle(false, wasTesting);
|
|
483
625
|
};
|
|
484
|
-
createInterface({ input: channel })
|
|
626
|
+
const lines = createInterface({ input: channel });
|
|
627
|
+
lines.on('line', (line) => {
|
|
485
628
|
const wasTesting = isTesting();
|
|
486
629
|
const event = lifecycle.accept(line);
|
|
487
630
|
if (event === null) {
|
|
@@ -489,9 +632,18 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
|
|
|
489
632
|
return;
|
|
490
633
|
}
|
|
491
634
|
updateIdle(false, wasTesting);
|
|
635
|
+
if (event.id === 0 && event.phase === 'terminal' && fixture !== null) {
|
|
636
|
+
fixtureDone = true;
|
|
637
|
+
stepTo({ kind: 'fixture-done' });
|
|
638
|
+
}
|
|
639
|
+
if (event.id > 0)
|
|
640
|
+
mainStarted = true;
|
|
492
641
|
// Tool startup and native lifecycle-bearing stdout wait for their phase acknowledgement.
|
|
493
642
|
channel.write(`${JSON.stringify(event)}\n`);
|
|
494
643
|
});
|
|
644
|
+
// readline re-emits its input's errors on the interface, where one with no listener is thrown: a
|
|
645
|
+
// runner killed with unread lifecycle bytes resets the channel (ECONNRESET on Linux).
|
|
646
|
+
lines.on('error', (error) => failLifecycle(error.message));
|
|
495
647
|
channel.on('error', (error) => failLifecycle(error.message));
|
|
496
648
|
channel.on('end', () => {
|
|
497
649
|
if (!lifecycle.finished())
|
|
@@ -514,22 +666,30 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
|
|
|
514
666
|
state.noSpace = sight(said);
|
|
515
667
|
appendStderr(text);
|
|
516
668
|
});
|
|
517
|
-
|
|
669
|
+
// The graceful pass: SIGTERM to the whole group. A signal that fails is told, not thrown: every caller has more to do
|
|
670
|
+
// after it, `terminate` the force-kill and the signal handler its releases and the signal it gives back.
|
|
671
|
+
const killChildTree = async () => {
|
|
518
672
|
const pid = child.pid;
|
|
519
673
|
if (!pid) {
|
|
520
674
|
return;
|
|
521
675
|
}
|
|
522
|
-
|
|
676
|
+
try {
|
|
677
|
+
await killer.kill(pid, 'SIGTERM');
|
|
678
|
+
}
|
|
679
|
+
catch (error) {
|
|
680
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
681
|
+
tell(`Terminating process group ${pid} failed: ${reason}\n`);
|
|
682
|
+
}
|
|
523
683
|
};
|
|
524
684
|
const onProcessExit = () => {
|
|
525
|
-
void killChildTree(
|
|
685
|
+
void killChildTree();
|
|
526
686
|
};
|
|
527
687
|
const onTerminationSignal = (signal) => {
|
|
528
688
|
removeSignalHandlers();
|
|
529
689
|
cancelCapture();
|
|
530
|
-
void killChildTree(
|
|
690
|
+
void killChildTree()
|
|
531
691
|
.finally(releaseCpu)
|
|
532
|
-
.finally(releaseLease)
|
|
692
|
+
.finally(() => releaseLease(true))
|
|
533
693
|
.finally(() => process.kill(process.pid, signal));
|
|
534
694
|
};
|
|
535
695
|
const onSigint = () => onTerminationSignal('SIGINT');
|
|
@@ -565,22 +725,23 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
|
|
|
565
725
|
// the reader unable to tell a wedged toolchain from a loaded machine — the
|
|
566
726
|
// two have opposite fixes.
|
|
567
727
|
let termination = Promise.resolve();
|
|
568
|
-
|
|
569
|
-
const expire = (bound, queuedMs = 0) => {
|
|
728
|
+
const expire = (bound) => {
|
|
570
729
|
if (state.settled || state.expiry !== null) {
|
|
571
730
|
return;
|
|
572
731
|
}
|
|
573
732
|
state.expiry = bound;
|
|
574
|
-
clearTimeout(
|
|
575
|
-
clearTimeout(idleTimer);
|
|
733
|
+
clearTimeout(boundTimer);
|
|
576
734
|
const elapsedMs = Date.now() - startedAt;
|
|
577
735
|
appendStderr(bound.kind === 'idle'
|
|
578
|
-
? `\nCommand made no progress: no test finished for ${bound.limitMs}ms (idleTimeoutMs=${bound.limitMs})
|
|
736
|
+
? `\nCommand made no progress: no test finished for ${bound.limitMs}ms (idleTimeoutMs=${bound.limitMs}) after ${elapsedMs}ms of runtime; ${bound.tests.length > 0
|
|
579
737
|
? `still running: ${bound.tests.join(', ')}`
|
|
580
738
|
: 'the runner named no test still running'} (cwd=${cwd}): ${command}\n`
|
|
581
|
-
:
|
|
739
|
+
: bound.kind === 'fixture'
|
|
740
|
+
? `\nFixture command timed out after ${elapsedMs}ms (fixtureTimeoutMs=${bound.limitMs}, cwd=${cwd}): ${fixture?.command ?? command}\n`
|
|
741
|
+
: `\nCommand timed out after ${elapsedMs}ms (timeoutMs=${bound.limitMs}, cwd=${cwd}): ${command}\n`);
|
|
582
742
|
if (bound.kind === 'idle') {
|
|
583
743
|
const observed = idleCapture?.result;
|
|
744
|
+
state.lockWaits = observed?.kind === 'captured' ? observed.snapshot.lockWaits : [];
|
|
584
745
|
appendStderr(`Hang diagnostics: runner pid=${child.pid ?? 'unavailable'} pgid=${process.platform === 'win32' ? 'unavailable' : (child.pid ?? 'unavailable')} expiryAtMs=${Date.now()} scheduledExpiryAtMs=${idleCapture?.expiresAtMs ?? 'unavailable'}\n`);
|
|
585
746
|
if (observed?.kind === 'captured') {
|
|
586
747
|
appendStderr(`Hang diagnostics: outcome=${observed.snapshot.outcome} capturedAtMs=${observed.snapshot.capturedAtMs} actualExpiryOffsetMs=${Date.now() - observed.snapshot.capturedAtMs}\n${observed.snapshot.text}`);
|
|
@@ -594,55 +755,83 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
|
|
|
594
755
|
cancelCapture();
|
|
595
756
|
termination = terminate();
|
|
596
757
|
};
|
|
758
|
+
// Whether the group still lists members. A probe that fails proves nothing about the group, so it is told, and the
|
|
759
|
+
// group is taken as listed: a force-kill that was not needed costs nothing, and one that was is not skipped.
|
|
760
|
+
const groupListed = (pid) => {
|
|
761
|
+
try {
|
|
762
|
+
return processGroupHasMembers(pid);
|
|
763
|
+
}
|
|
764
|
+
catch (error) {
|
|
765
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
766
|
+
tell(`Probing process group ${pid} for members failed: ${reason}; taking it as still listing members\n`);
|
|
767
|
+
return true;
|
|
768
|
+
}
|
|
769
|
+
};
|
|
597
770
|
// A timed-out run ends with its whole process group, not with its leader.
|
|
598
771
|
// The graceful signal reaches every member that exists when it is sent, but
|
|
599
772
|
// a member that ignores SIGTERM, or one forked while the signal goes out
|
|
600
773
|
// (macOS does not hold a fork back for a group signal in flight), outlives
|
|
601
774
|
// a leader that exits on it. The grace ends when the leader exits or
|
|
602
775
|
// killAfterMs passes, whichever comes first; whatever is left then is killed.
|
|
776
|
+
// SIGKILL is no different: a member forked while it goes out survives it too,
|
|
777
|
+
// so the force-kill is repeated while the group lists members. What the repeats
|
|
778
|
+
// cannot end is said, and a failure of the probe or of a signal is shown.
|
|
779
|
+
const forceKill = async (pid) => {
|
|
780
|
+
try {
|
|
781
|
+
if (await killProcessGroup(pid, DEADLINE_PATIENCE, (group) => killer.kill(group, 'SIGKILL'))) {
|
|
782
|
+
tell(`${describeMembersLeft(pid)}\n`);
|
|
783
|
+
}
|
|
784
|
+
}
|
|
785
|
+
catch (error) {
|
|
786
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
787
|
+
tell(`Force-kill of process group ${pid} failed: ${reason}\n`);
|
|
788
|
+
}
|
|
789
|
+
};
|
|
603
790
|
const terminate = async () => {
|
|
604
|
-
await
|
|
791
|
+
await killChildTree();
|
|
605
792
|
if (!state.settled && killAfterMs > 0) {
|
|
606
793
|
await settledWithin(exitPromise, killAfterMs);
|
|
607
794
|
}
|
|
608
795
|
if (state.settled) {
|
|
609
|
-
if (child.pid !== undefined &&
|
|
796
|
+
if (child.pid !== undefined && groupListed(child.pid)) {
|
|
610
797
|
state.forceKillNeeded = true;
|
|
611
798
|
appendStderr(`Force-killing what the ${state.expiry === null ? 'command' : 'timed-out command'} left in its process group: ${command}\n`);
|
|
612
|
-
await
|
|
799
|
+
await forceKill(child.pid);
|
|
613
800
|
}
|
|
614
801
|
return;
|
|
615
802
|
}
|
|
616
803
|
state.forceKillNeeded = true;
|
|
617
804
|
appendStderr(`Force-killing ${state.expiry === null ? 'command' : 'timed out command'} after killAfterMs=${killAfterMs}: ${command}\n`);
|
|
618
|
-
|
|
619
|
-
|
|
805
|
+
if (child.pid !== undefined)
|
|
806
|
+
await forceKill(child.pid);
|
|
807
|
+
// The repeats wait for the group to empty, the leader with it. Windows has no process groups, so there they have
|
|
808
|
+
// nothing to wait on once the one signal is sent, and the leader's exit is awaited as it always was.
|
|
809
|
+
if (process.platform === 'win32' || child.pid === undefined) {
|
|
810
|
+
await settledWithin(exitPromise, REAP_AFTER_FORCE_KILL_MS);
|
|
811
|
+
}
|
|
620
812
|
if (!state.settled) {
|
|
621
813
|
resolveExit({ code: 1, signal: 'SIGKILL' });
|
|
622
814
|
}
|
|
623
815
|
};
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
//
|
|
630
|
-
//
|
|
631
|
-
// command runs, is said and changes nothing else; only a bound or a signal stops the command. Our own release after
|
|
632
|
-
// the command settled is not a loss; `cpuReleased` marks it.
|
|
816
|
+
// The fixture's ceiling lives and dies with its segment: the segment's terminal clears it, so it can never
|
|
817
|
+
// fire on a command the fixture already handed over to.
|
|
818
|
+
stepTo({ kind: 'spawn', at: Date.now(), testing: isTesting(), fixture: fixture !== null && lifecycle !== null });
|
|
819
|
+
// A grant's connection is a ledger entry, not the command's owner: the lending parent's exit ends it, and what runs
|
|
820
|
+
// under it runs on over budget until it finishes. Its close, while the command runs, is said and changes nothing
|
|
821
|
+
// else; only a bound or a signal stops the command. Our own release after the command settled is not a loss;
|
|
822
|
+
// `cpuReleased` marks it.
|
|
633
823
|
if (grant !== null) {
|
|
634
824
|
const held = grant;
|
|
635
825
|
void held.closed.then(() => {
|
|
636
826
|
if (state.settled || cpuReleased !== null) {
|
|
637
827
|
return;
|
|
638
828
|
}
|
|
639
|
-
appendStderr(`cowshed: cpu-
|
|
829
|
+
appendStderr(`cowshed: cpu-pool grant ended: the parent pool closed its ${held.tokens}-token grant while the command ran; it runs on over budget until it ends: ${command}\n`);
|
|
640
830
|
});
|
|
641
831
|
}
|
|
642
832
|
const { code: exitCode, signal: exitSignal } = await exitPromise;
|
|
643
833
|
await termination;
|
|
644
|
-
clearTimeout(
|
|
645
|
-
clearTimeout(idleTimer);
|
|
834
|
+
clearTimeout(boundTimer);
|
|
646
835
|
cancelCapture();
|
|
647
836
|
removeSignalHandlers();
|
|
648
837
|
// The runner is gone. Its sub-pool closes first, so no descendant still draws on tokens the grant returns;
|
|
@@ -650,7 +839,9 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
|
|
|
650
839
|
await releaseCpu();
|
|
651
840
|
const code = exitCode ?? signalToExitCode(exitSignal);
|
|
652
841
|
if (code !== 0 && state.expiry === null) {
|
|
653
|
-
appendStderr(
|
|
842
|
+
appendStderr(fixture !== null && !mainStarted
|
|
843
|
+
? `Fixture command exited with status ${code}: ${fixture.command}\n`
|
|
844
|
+
: `Command exited with status ${code}: ${command}\n`);
|
|
654
845
|
}
|
|
655
846
|
if (state.expiry !== null && !state.forceKillNeeded) {
|
|
656
847
|
appendStderr(`Timed out command exited after graceful termination: ${command}\n`);
|
|
@@ -668,11 +859,11 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
|
|
|
668
859
|
appendStderr(`${describeRamTempError(full)}\n`);
|
|
669
860
|
}
|
|
670
861
|
}
|
|
671
|
-
await releaseLease();
|
|
672
|
-
// Broken resource ownership
|
|
673
|
-
// its verdict is an ordinary operational failure, never a successful or silently unbounded run.
|
|
674
|
-
const operationalFailure = state.poolCloseError !== null || lifecycleFailure !== null;
|
|
675
|
-
await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure, seen);
|
|
862
|
+
await releaseLease(code !== 0 || state.expiry !== null || state.poolCloseError !== null || lifecycleFailure !== null);
|
|
863
|
+
// Broken resource ownership, a lost native lifecycle, or leftovers that could not be retired fail even if a shell
|
|
864
|
+
// branch later exited 0: its verdict is an ordinary operational failure, never a successful or silently unbounded run.
|
|
865
|
+
const operationalFailure = state.poolCloseError !== null || lifecycleFailure !== null || state.retireError !== null;
|
|
866
|
+
await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure, seen, state.lockWaits);
|
|
676
867
|
return {
|
|
677
868
|
success: !operationalFailure && state.expiry === null && code === 0,
|
|
678
869
|
terminalOutput: outputChunks.join(''),
|
|
@@ -680,44 +871,11 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
|
|
|
680
871
|
}
|
|
681
872
|
export function createProcessTreeKiller() {
|
|
682
873
|
return {
|
|
683
|
-
kill(pid, signal) {
|
|
684
|
-
|
|
874
|
+
async kill(pid, signal) {
|
|
875
|
+
signalProcessGroup(pid, signal);
|
|
685
876
|
},
|
|
686
877
|
};
|
|
687
878
|
}
|
|
688
|
-
function killProcessGroup(pid, signal) {
|
|
689
|
-
try {
|
|
690
|
-
// On POSIX the executor spawns with detached: true, so the child leads a
|
|
691
|
-
// process group of its own and -pid signals all of it at once, including
|
|
692
|
-
// grandchildren that tree-walk libraries miss when parents die and
|
|
693
|
-
// children get reparented to init. Only the group is named: the leader may
|
|
694
|
-
// already be reaped, and its pid reused by an unrelated process, while the
|
|
695
|
-
// group lives on in its other members. Windows has no process groups.
|
|
696
|
-
process.kill(process.platform === 'win32' ? pid : -pid, signal);
|
|
697
|
-
}
|
|
698
|
-
catch {
|
|
699
|
-
// ESRCH: the group (or the Windows process) has already exited.
|
|
700
|
-
}
|
|
701
|
-
return Promise.resolve();
|
|
702
|
-
}
|
|
703
|
-
/**
|
|
704
|
-
* Whether any process is left in the group `pgid` leads. A group outlives its
|
|
705
|
-
* reaped leader for as long as one member does, and no process can take its id
|
|
706
|
-
* meanwhile. Windows has no process groups, so there is nothing left to name.
|
|
707
|
-
*/
|
|
708
|
-
function processGroupHasMembers(pgid) {
|
|
709
|
-
if (process.platform === 'win32') {
|
|
710
|
-
return false;
|
|
711
|
-
}
|
|
712
|
-
try {
|
|
713
|
-
process.kill(-pgid, 0);
|
|
714
|
-
return true;
|
|
715
|
-
}
|
|
716
|
-
catch (error) {
|
|
717
|
-
// EPERM: a member exists but runs as another user.
|
|
718
|
-
return error instanceof Error && 'code' in error && error.code === 'EPERM';
|
|
719
|
-
}
|
|
720
|
-
}
|
|
721
879
|
function resolveCwd(cwd, root) {
|
|
722
880
|
if (!cwd) {
|
|
723
881
|
return root;
|
|
@@ -737,10 +895,10 @@ function buildCommand(options) {
|
|
|
737
895
|
}
|
|
738
896
|
return parts.join(' ');
|
|
739
897
|
}
|
|
740
|
-
/**
|
|
741
|
-
function mergeEnv(
|
|
898
|
+
/** Owned scratch is the command's TMPDIR unless the target names one itself. */
|
|
899
|
+
function mergeEnv(directory, env) {
|
|
742
900
|
const inherited = withoutPoolEnv(process.env);
|
|
743
|
-
return
|
|
901
|
+
return directory === null ? { ...inherited, ...env } : { ...inherited, TMPDIR: directory, ...env };
|
|
744
902
|
}
|
|
745
903
|
/** The process's environment without the CPU pool's descriptor: a pool and its count only a grant may give. */
|
|
746
904
|
function withoutPoolEnv(env) {
|
|
@@ -770,14 +928,6 @@ function signalToExitCode(signal) {
|
|
|
770
928
|
}
|
|
771
929
|
return EXIT_CODE_BY_SIGNAL[signal] ?? 1;
|
|
772
930
|
}
|
|
773
|
-
async function ignoreKillError(promise) {
|
|
774
|
-
try {
|
|
775
|
-
await promise;
|
|
776
|
-
}
|
|
777
|
-
catch {
|
|
778
|
-
// The process tree may have already exited between timeout and kill.
|
|
779
|
-
}
|
|
780
|
-
}
|
|
781
931
|
/** Resolves when `promise` settles or `ms` passes, whichever is first, leaving no timer behind. */
|
|
782
932
|
async function settledWithin(promise, ms) {
|
|
783
933
|
let timer;
|