@smoothbricks/nx-plugin 0.4.26 → 0.4.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +994 -117
- package/dist/bin/smoo-nx-bound-failures.d.ts +3 -0
- package/dist/bin/smoo-nx-bound-failures.d.ts.map +1 -0
- package/dist/bin/smoo-nx-bound-failures.js +31 -0
- package/dist/bin/smoo-nx-cargo-hash.js +28 -3
- package/dist/bin/smoo-nx-exec.js +15 -73
- package/dist/bin/smoo-nx-failures.d.ts +3 -0
- package/dist/bin/smoo-nx-failures.d.ts.map +1 -0
- package/dist/bin/smoo-nx-failures.js +3 -0
- package/dist/bin/smoo-nx-nextest-extract.js +3 -2
- package/dist/bin/smoo-nx-test-walks.d.ts +3 -0
- package/dist/bin/smoo-nx-test-walks.d.ts.map +1 -0
- package/dist/bin/smoo-nx-test-walks.js +34 -0
- package/dist/bounded-test-policy.d.ts +7 -1
- package/dist/bounded-test-policy.d.ts.map +1 -1
- package/dist/bounded-test-policy.js +101 -40
- package/dist/build-state-output-policy.d.ts +25 -0
- package/dist/build-state-output-policy.d.ts.map +1 -0
- package/dist/build-state-output-policy.js +64 -0
- package/dist/cargo-closure-input.d.ts +3 -2
- package/dist/cargo-closure-input.d.ts.map +1 -1
- package/dist/cargo-closure-input.js +16 -15
- package/dist/cargo-dev-profile.d.ts +37 -0
- package/dist/cargo-dev-profile.d.ts.map +1 -0
- package/dist/cargo-dev-profile.js +108 -0
- package/dist/cargo-source-hash.d.ts +43 -6
- package/dist/cargo-source-hash.d.ts.map +1 -1
- package/dist/cargo-source-hash.js +309 -73
- package/dist/cargo-toolchain-policy.d.ts +4 -0
- package/dist/cargo-toolchain-policy.d.ts.map +1 -1
- package/dist/cargo-toolchain-policy.js +5 -0
- package/dist/cargo-workspace.d.ts +101 -5
- package/dist/cargo-workspace.d.ts.map +1 -1
- package/dist/cargo-workspace.js +291 -10
- package/dist/cross-check-policy.d.ts +21 -3
- package/dist/cross-check-policy.d.ts.map +1 -1
- package/dist/cross-check-policy.js +24 -6
- package/dist/disk-command-policy.d.ts +7 -0
- package/dist/disk-command-policy.d.ts.map +1 -0
- package/dist/disk-command-policy.js +98 -0
- package/dist/ensure-built-receipt.d.ts +25 -0
- package/dist/ensure-built-receipt.d.ts.map +1 -0
- package/dist/ensure-built-receipt.js +261 -0
- package/dist/ensure-built-witness.d.ts +57 -0
- package/dist/ensure-built-witness.d.ts.map +1 -0
- package/dist/ensure-built-witness.js +412 -0
- package/dist/ensure-built.d.ts +207 -12
- package/dist/ensure-built.d.ts.map +1 -1
- package/dist/ensure-built.js +576 -202
- package/dist/eslint-file-set.d.ts +7 -0
- package/dist/eslint-file-set.d.ts.map +1 -0
- package/dist/eslint-file-set.js +12 -0
- package/dist/executors/bounded-exec/cpu-pool.d.ts +63 -0
- package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -0
- package/dist/executors/bounded-exec/cpu-pool.js +379 -0
- package/dist/executors/bounded-exec/cpu-tokens.d.ts +82 -0
- package/dist/executors/bounded-exec/cpu-tokens.d.ts.map +1 -0
- package/dist/executors/bounded-exec/cpu-tokens.js +114 -0
- package/dist/executors/bounded-exec/disk-lease.d.ts +61 -0
- package/dist/executors/bounded-exec/disk-lease.d.ts.map +1 -0
- package/dist/executors/bounded-exec/disk-lease.js +91 -0
- package/dist/executors/bounded-exec/executor.d.ts +45 -2
- package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
- package/dist/executors/bounded-exec/executor.js +447 -52
- package/dist/executors/bounded-exec/gateway-lease.d.ts +52 -0
- package/dist/executors/bounded-exec/gateway-lease.d.ts.map +1 -0
- package/dist/executors/bounded-exec/gateway-lease.js +143 -0
- package/dist/executors/bounded-exec/hang-diagnostics.d.ts +35 -0
- package/dist/executors/bounded-exec/hang-diagnostics.d.ts.map +1 -0
- package/dist/executors/bounded-exec/hang-diagnostics.js +231 -0
- package/dist/executors/bounded-exec/progress.d.ts +35 -0
- package/dist/executors/bounded-exec/progress.d.ts.map +1 -0
- package/dist/executors/bounded-exec/progress.js +79 -0
- package/dist/executors/bounded-exec/ram-temp-fixtures.d.ts +3 -0
- package/dist/executors/bounded-exec/ram-temp-fixtures.d.ts.map +1 -0
- package/dist/executors/bounded-exec/ram-temp-fixtures.js +12 -0
- package/dist/executors/bounded-exec/ram-temp-held.d.ts +91 -0
- package/dist/executors/bounded-exec/ram-temp-held.d.ts.map +1 -0
- package/dist/executors/bounded-exec/ram-temp-held.js +152 -0
- package/dist/executors/bounded-exec/ram-temp.d.ts +223 -0
- package/dist/executors/bounded-exec/ram-temp.d.ts.map +1 -0
- package/dist/executors/bounded-exec/ram-temp.js +1528 -0
- package/dist/executors/bounded-exec/schema.d.ts +29 -11
- package/dist/executors/bounded-exec/schema.d.ts.map +1 -1
- package/dist/executors/bounded-exec/verdict.d.ts +162 -0
- package/dist/executors/bounded-exec/verdict.d.ts.map +1 -0
- package/dist/executors/bounded-exec/verdict.js +318 -0
- package/dist/executors/shard-timings/executor.d.ts +14 -0
- package/dist/executors/shard-timings/executor.d.ts.map +1 -0
- package/dist/executors/shard-timings/executor.js +25 -0
- package/dist/executors/shard-timings/schema.d.ts +4 -0
- package/dist/executors/shard-timings/schema.d.ts.map +1 -0
- package/dist/executors/shard-timings/schema.js +1 -0
- package/dist/failure-summary.d.ts +5 -0
- package/dist/failure-summary.d.ts.map +1 -0
- package/dist/failure-summary.js +118 -0
- package/dist/generators/create-package/generator.d.ts.map +1 -1
- package/dist/generators/create-package/generator.js +6 -2
- package/dist/index.d.ts +14 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +370 -138
- package/dist/managed-files/ci-workflow.d.ts +13 -0
- package/dist/managed-files/ci-workflow.d.ts.map +1 -1
- package/dist/managed-files/ci-workflow.js +44 -13
- package/dist/managed-files/files.d.ts +7 -1
- package/dist/managed-files/files.d.ts.map +1 -1
- package/dist/managed-files/files.js +62 -8
- package/dist/managed-files/json-edit.d.ts +21 -0
- package/dist/managed-files/json-edit.d.ts.map +1 -0
- package/dist/managed-files/json-edit.js +295 -0
- package/dist/managed-files/managed-format.d.ts +33 -1
- package/dist/managed-files/managed-format.d.ts.map +1 -1
- package/dist/managed-files/managed-format.js +82 -2
- package/dist/managed-files/pr-preview-cleanup-workflow.d.ts +7 -0
- package/dist/managed-files/pr-preview-cleanup-workflow.d.ts.map +1 -1
- package/dist/managed-files/pr-preview-cleanup-workflow.js +68 -4
- package/dist/managed-files/publish-workflow.d.ts.map +1 -1
- package/dist/managed-files/publish-workflow.js +231 -48
- package/dist/managed-files/tree.d.ts +5 -0
- package/dist/managed-files/tree.d.ts.map +1 -1
- package/dist/managed-files/tree.js +5 -3
- package/dist/manifest-hash.d.ts.map +1 -1
- package/dist/manifest-hash.js +2 -5
- package/dist/nextest-code-bound.d.ts +62 -0
- package/dist/nextest-code-bound.d.ts.map +1 -0
- package/dist/nextest-code-bound.js +154 -0
- package/dist/nextest-extraction.d.ts +24 -8
- package/dist/nextest-extraction.d.ts.map +1 -1
- package/dist/nextest-extraction.js +42 -7
- package/dist/package-target-policy.d.ts.map +1 -1
- package/dist/package-target-policy.js +3 -5
- package/dist/release-config-policy.d.ts.map +1 -1
- package/dist/release-config-policy.js +3 -5
- package/dist/runtime-input.d.ts +14 -0
- package/dist/runtime-input.d.ts.map +1 -0
- package/dist/runtime-input.js +15 -0
- package/dist/rust-tokens.d.ts +11 -0
- package/dist/rust-tokens.d.ts.map +1 -0
- package/dist/rust-tokens.js +81 -0
- package/dist/task-results.d.ts +6 -0
- package/dist/task-results.d.ts.map +1 -0
- package/dist/task-results.js +61 -0
- package/dist/test-program.d.ts +54 -0
- package/dist/test-program.d.ts.map +1 -0
- package/dist/test-program.js +292 -0
- package/dist/test-timings.d.ts +71 -0
- package/dist/test-timings.d.ts.map +1 -0
- package/dist/test-timings.js +393 -0
- package/dist/test-walk-policy.d.ts +103 -0
- package/dist/test-walk-policy.d.ts.map +1 -0
- package/dist/test-walk-policy.js +657 -0
- package/dist/testing.d.ts +177 -0
- package/dist/testing.d.ts.map +1 -0
- package/dist/testing.js +824 -0
- package/dist/type-guards.d.ts +3 -0
- package/dist/type-guards.d.ts.map +1 -0
- package/dist/type-guards.js +4 -0
- package/dist/typecheck-test-policy.d.ts +8 -5
- package/dist/typecheck-test-policy.d.ts.map +1 -1
- package/dist/typecheck-test-policy.js +113 -100
- package/dist/workspace-config-policy.d.ts +8 -0
- package/dist/workspace-config-policy.d.ts.map +1 -1
- package/dist/workspace-config-policy.js +175 -63
- package/dist/workspace-index.d.ts +12 -0
- package/dist/workspace-index.d.ts.map +1 -0
- package/dist/workspace-index.js +17 -0
- package/dist/workspace-manifest.d.ts +37 -0
- package/dist/workspace-manifest.d.ts.map +1 -1
- package/executors.json +5 -0
- package/loader.js +13 -11
- package/managed/raw/envrc +33 -67
- package/managed/raw/git-format-staged.yml +30 -35
- package/managed/raw/patches/nx@23.2.1.patch +3484 -0
- package/managed/raw/tooling/direnv/apple-developer.sh +83 -0
- package/managed/raw/tooling/direnv/developer-links.ts +569 -11
- package/managed/raw/tooling/direnv/devenv.smoo.nix +314 -93
- package/managed/raw/tooling/direnv/github-actions-bootstrap.sh +20 -11
- package/managed/raw/tooling/direnv/linked-snapshots.ts +845 -0
- package/managed/raw/tooling/direnv/nx-socket-dir.sh +82 -0
- package/managed/raw/tooling/direnv/repo-path +14 -2
- package/managed/raw/tooling/direnv/setup-environment.ts +1146 -159
- package/managed/raw/tooling/direnv/toolchain-stamp.ts +45 -3
- package/managed/raw/tooling/direnv/ttsc-env.sh +39 -0
- package/managed/raw/tooling/git-hooks/commit-msg.sh +5 -6
- package/managed/raw/tooling/git-hooks/pre-commit.sh +7 -6
- package/managed/raw/tooling/git-hooks/pre-push.sh +20 -15
- package/managed/templates/github/actions/cache-nix-devenv/action.yml +6 -6
- package/managed/templates/github/actions/cache-node-modules/action.yml +6 -4
- package/managed/templates/github/actions/cache-nx/action.yml +1 -1
- package/managed/templates/github/actions/cache-ttsc-plugins/action.yml +1 -1
- package/managed/templates/github/actions/forgejo-artifact.cjs +99 -0
- package/managed/templates/github/actions/forgejo-download-artifact/action.yml +49 -0
- package/managed/templates/github/actions/forgejo-download-artifact/index.cjs +7 -0
- package/managed/templates/github/actions/forgejo-upload-artifact/action.yml +65 -0
- package/managed/templates/github/actions/forgejo-upload-artifact/index.cjs +7 -0
- package/managed/templates/github/actions/setup-devenv/action.yml +21 -3
- package/managed/templates/github/workflows/managed-files.yml +1 -2
- package/nextest.toml +8 -41
- package/package.json +74 -12
- package/runtime-input.sh +81 -0
- package/src/__tests__/counted-cargo.ts +33 -14
- package/src/__tests__/fixture-nx-env.ts +111 -46
- package/src/__tests__/progress-clock.ts +43 -0
- package/src/__tests__/runtime-input.ts +23 -0
- package/src/bin/smoo-nx-bound-failures.test.ts +127 -0
- package/src/bin/smoo-nx-bound-failures.ts +33 -0
- package/src/bin/smoo-nx-cargo-hash.ts +28 -3
- package/src/bin/smoo-nx-exec.ts +16 -74
- package/src/bin/smoo-nx-failures.ts +4 -0
- package/src/bin/smoo-nx-nextest-extract.ts +3 -2
- package/src/bin/smoo-nx-test-walks.ts +42 -0
- package/src/bounded-test-policy.test.ts +268 -21
- package/src/bounded-test-policy.ts +114 -42
- package/src/build-state-output-policy.test.ts +115 -0
- package/src/build-state-output-policy.ts +90 -0
- package/src/bun-bootstrap.test.ts +57 -3
- package/src/bun-file-packages.test.ts +47 -0
- package/src/cargo-closure-input.test.ts +138 -66
- package/src/cargo-closure-input.ts +17 -15
- package/src/cargo-dev-profile.ts +137 -0
- package/src/cargo-resolve-cache.test.ts +70 -1
- package/src/cargo-source-hash.test.ts +193 -97
- package/src/cargo-source-hash.ts +345 -68
- package/src/cargo-toolchain-policy.ts +6 -0
- package/src/cargo-workspace.test.ts +320 -0
- package/src/cargo-workspace.ts +336 -10
- package/src/cross-check-policy.test.ts +59 -1
- package/src/cross-check-policy.ts +25 -6
- package/src/disk-command-policy.test.ts +61 -0
- package/src/disk-command-policy.ts +107 -0
- package/src/ensure-built-receipt.test.ts +184 -0
- package/src/ensure-built-receipt.ts +263 -0
- package/src/ensure-built-witness.test.ts +344 -0
- package/src/ensure-built-witness.ts +252 -0
- package/src/ensure-built.test.ts +1036 -578
- package/src/ensure-built.ts +820 -224
- package/src/eslint-file-set.ts +9 -0
- package/src/executors/bounded-exec/cpu-pool.test.ts +454 -0
- package/src/executors/bounded-exec/cpu-pool.ts +417 -0
- package/src/executors/bounded-exec/cpu-tokens.test.ts +269 -0
- package/src/executors/bounded-exec/cpu-tokens.ts +177 -0
- package/src/executors/bounded-exec/disk-lease.test.ts +185 -0
- package/src/executors/bounded-exec/disk-lease.ts +139 -0
- package/src/executors/bounded-exec/executor-host.test.ts +778 -0
- package/src/executors/bounded-exec/executor.test.ts +1116 -63
- package/src/executors/bounded-exec/executor.ts +565 -67
- package/src/executors/bounded-exec/gateway-lease.ts +176 -0
- package/src/executors/bounded-exec/hang-diagnostics.test.ts +285 -0
- package/src/executors/bounded-exec/hang-diagnostics.ts +283 -0
- package/src/executors/bounded-exec/progress.test.ts +79 -0
- package/src/executors/bounded-exec/progress.ts +118 -0
- package/src/executors/bounded-exec/ram-temp-fixtures.ts +12 -0
- package/src/executors/bounded-exec/ram-temp-held.test.ts +264 -0
- package/src/executors/bounded-exec/ram-temp-held.ts +242 -0
- package/src/executors/bounded-exec/ram-temp-host.test.ts +605 -0
- package/src/executors/bounded-exec/ram-temp.test.ts +674 -0
- package/src/executors/bounded-exec/ram-temp.ts +1918 -0
- package/src/executors/bounded-exec/schema.json +11 -5
- package/src/executors/bounded-exec/schema.ts +27 -11
- package/src/executors/bounded-exec/verdict.test.ts +440 -0
- package/src/executors/bounded-exec/verdict.ts +433 -0
- package/src/executors/shard-timings/executor.ts +36 -0
- package/src/executors/shard-timings/schema.json +21 -0
- package/src/executors/shard-timings/schema.ts +4 -0
- package/src/failure-summary.ts +119 -0
- package/src/generators/bounded-test-targets/generator.test.ts +4 -4
- package/src/generators/bun-test-tracing/generator.test.ts +2 -2
- package/src/generators/create-package/generator.test.ts +6 -0
- package/src/generators/create-package/generator.ts +6 -2
- package/src/host-controller-wiring.test.ts +31 -0
- package/src/index.test.ts +909 -295
- package/src/index.ts +441 -143
- package/src/linked-snapshots.nx.test.ts +172 -0
- package/src/managed-files/ci-workflow.ts +54 -13
- package/src/managed-files/files.ts +65 -9
- package/src/managed-files/forgejo-artifact.test.ts +395 -0
- package/src/managed-files/generator.test.ts +60 -1
- package/src/managed-files/json-edit.test.ts +216 -0
- package/src/managed-files/json-edit.ts +290 -0
- package/src/managed-files/managed-format-json.test.ts +308 -0
- package/src/managed-files/managed-format.ts +90 -2
- package/src/managed-files/pr-preview-cleanup-workflow.ts +69 -3
- package/src/managed-files/publish-workflow.ts +285 -67
- package/src/managed-files/repo-path.test.ts +129 -0
- package/src/managed-files/trace-db-artifact.test.ts +7 -0
- package/src/managed-files/tree.ts +11 -3
- package/src/manifest-hash.ts +2 -6
- package/src/nextest-code-bound.test.ts +155 -0
- package/src/nextest-code-bound.ts +195 -0
- package/src/nextest-extraction.test.ts +37 -5
- package/src/nextest-extraction.ts +51 -9
- package/src/nextest-partitions.test.ts +427 -82
- package/src/nx-cache-bound.test.ts +5 -10
- package/src/nx-cache-restore-mtime.test.ts +170 -0
- package/src/nx-daemon-claim.test.ts +277 -0
- package/src/nx-daemon-idle-cpu.test.ts +81 -0
- package/src/nx-daemon-idle-policy.test.ts +51 -0
- package/src/nx-daemon-ignore-files.test.ts +210 -0
- package/src/nx-daemon-root-normalization.test.ts +105 -0
- package/src/nx-daemon-validity.test.ts +925 -0
- package/src/nx-dot-input-policy.test.ts +353 -0
- package/src/nx-failure-life-cycles.test.ts +207 -0
- package/src/nx-files-archive-racy.test.ts +97 -0
- package/src/nx-foreign-workspace-env.test.ts +329 -0
- package/src/nx-incomplete-run-summary.test.ts +70 -0
- package/src/nx-plugin-load-errors.test.ts +120 -0
- package/src/nx-project-resolution-cache.test.ts +176 -0
- package/src/nx-raw-inputs.test.ts +166 -0
- package/src/nx-resident-graph-workers.test.ts +7 -16
- package/src/nx-run-information.test.ts +118 -0
- package/src/nx-skipped-task-exit.test.ts +180 -0
- package/src/nx-state-location.test.ts +63 -0
- package/src/nx-store-resolution.test.ts +26 -25
- package/src/nx-task-graph-cycles.test.ts +148 -0
- package/src/nx-task-output-lease.test.ts +414 -0
- package/src/nx-use-local.test.ts +103 -0
- package/src/package-target-policy.ts +3 -6
- package/src/release-config-policy.ts +3 -6
- package/src/runtime-input.test.ts +215 -0
- package/src/runtime-input.ts +17 -0
- package/src/rust-tokens.ts +78 -0
- package/src/serial-group-wiring.test.ts +326 -0
- package/src/task-history-namespace.test.ts +87 -89
- package/src/task-results.test.ts +226 -0
- package/src/task-results.ts +75 -0
- package/src/test-program.test.ts +303 -0
- package/src/test-program.ts +341 -0
- package/src/test-timings.test.ts +240 -0
- package/src/test-timings.ts +311 -0
- package/src/test-walk-policy.test.ts +585 -0
- package/src/test-walk-policy.ts +731 -0
- package/src/testing.test.ts +509 -0
- package/src/testing.ts +922 -0
- package/src/type-guards.ts +4 -0
- package/src/typecheck-test-policy.test.ts +496 -11
- package/src/typecheck-test-policy.ts +137 -103
- package/src/workspace-config-policy.test.ts +212 -13
- package/src/workspace-config-policy.ts +101 -73
- package/src/workspace-index.ts +18 -0
- package/src/workspace-manifest.ts +41 -0
- package/test-timings.json +85 -30
- package/managed/raw/tooling/direnv/inherited-devenv.ts +0 -665
- package/managed/raw/tooling/direnv/shared-caches.sh +0 -53
|
@@ -1,18 +1,82 @@
|
|
|
1
|
-
import { spawn } from 'node:child_process';
|
|
2
|
-
import {
|
|
1
|
+
import { type ChildProcess, spawn } from 'node:child_process';
|
|
2
|
+
import { availableParallelism, userInfo } from 'node:os';
|
|
3
|
+
import { isAbsolute, join, resolve as resolvePath } from 'node:path';
|
|
4
|
+
import { workspaceDataDirectoryForWorkspace } from 'nx/src/utils/cache-directory.js';
|
|
3
5
|
|
|
6
|
+
import {
|
|
7
|
+
CPU_POOL_ENV,
|
|
8
|
+
type CpuPool,
|
|
9
|
+
createCpuPool,
|
|
10
|
+
type InheritedCpuPool,
|
|
11
|
+
readInheritedCpuPool,
|
|
12
|
+
takeParentCpuTokens,
|
|
13
|
+
} from './cpu-pool.js';
|
|
14
|
+
import {
|
|
15
|
+
CPU_TOKENS_ENV,
|
|
16
|
+
type CpuBudget,
|
|
17
|
+
gatewayCpuBudget,
|
|
18
|
+
runnerDemand,
|
|
19
|
+
sizeRunner,
|
|
20
|
+
type TokenGrant,
|
|
21
|
+
} from './cpu-tokens.js';
|
|
22
|
+
import type { HangDiagnostics } from './hang-diagnostics.js';
|
|
23
|
+
import { captureHangDiagnostics, HANG_CAPTURE_MS } from './hang-diagnostics.js';
|
|
24
|
+
import { createRunProgress, withProgressReport } from './progress.js';
|
|
25
|
+
import {
|
|
26
|
+
COWSHED_EXECUTABLE_ENV,
|
|
27
|
+
createHostCommands,
|
|
28
|
+
describeRamTempError,
|
|
29
|
+
RAM_TEMP_CAPACITY_BYTES,
|
|
30
|
+
type RamTempLease,
|
|
31
|
+
RamTempVolume,
|
|
32
|
+
ramTempPaths,
|
|
33
|
+
} from './ram-temp.js';
|
|
4
34
|
import type { BoundedExecOptions } from './schema.js';
|
|
35
|
+
import {
|
|
36
|
+
type BoundedExecRecord,
|
|
37
|
+
type BoundedExecTask,
|
|
38
|
+
clearTaskDirectory,
|
|
39
|
+
type ExpiredBound,
|
|
40
|
+
JUNIT_FILE,
|
|
41
|
+
JUNIT_REPORT_ENV,
|
|
42
|
+
judgeOperationalFailure,
|
|
43
|
+
judgeRun,
|
|
44
|
+
readJunitReport,
|
|
45
|
+
taskDirectory,
|
|
46
|
+
withJunitReport,
|
|
47
|
+
writeRecord,
|
|
48
|
+
} from './verdict.js';
|
|
5
49
|
|
|
6
50
|
const DEFAULT_KILL_AFTER_MS = 10_000;
|
|
7
51
|
const REAP_AFTER_FORCE_KILL_MS = 2_000;
|
|
52
|
+
type IdleDiagnosticResult =
|
|
53
|
+
| { kind: 'pending'; startedAtMs: number | null }
|
|
54
|
+
| { kind: 'captured'; snapshot: HangDiagnostics }
|
|
55
|
+
| { kind: 'failed'; detail: string };
|
|
8
56
|
const EXIT_CODE_BY_SIGNAL: Partial<Record<NodeJS.Signals, number>> = {
|
|
9
57
|
SIGHUP: 129,
|
|
10
58
|
SIGINT: 130,
|
|
11
59
|
SIGTERM: 143,
|
|
12
60
|
};
|
|
13
61
|
|
|
62
|
+
/** The variables the CPU budget sets for the command it grants: no target's environment may set them. */
|
|
63
|
+
const BUDGET_ENV = [CPU_TOKENS_ENV, CPU_POOL_ENV];
|
|
64
|
+
|
|
14
65
|
export interface BoundedExecContext {
|
|
15
66
|
root: string;
|
|
67
|
+
/** The Nx task this run executes as, and the directory its verdict and report go to. */
|
|
68
|
+
record?: { task: BoundedExecTask; directory: string };
|
|
69
|
+
/** Owns child stdout when present; otherwise it is retained and forwarded normally. */
|
|
70
|
+
stdout?: (chunk: string) => void;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/** The part of Nx's executor context that names the running task. */
|
|
74
|
+
export interface NxTaskContext {
|
|
75
|
+
root: string;
|
|
76
|
+
projectName?: string;
|
|
77
|
+
targetName?: string;
|
|
78
|
+
configurationName?: string;
|
|
79
|
+
taskGraph?: { tasks: Record<string, { hash?: string }> };
|
|
16
80
|
}
|
|
17
81
|
|
|
18
82
|
export interface BoundedExecResult {
|
|
@@ -20,62 +84,127 @@ export interface BoundedExecResult {
|
|
|
20
84
|
terminalOutput: string;
|
|
21
85
|
}
|
|
22
86
|
|
|
23
|
-
/**
|
|
24
|
-
* Which bound ended the run.
|
|
25
|
-
*
|
|
26
|
-
* `total` is a wall-clock ceiling: it answers "is this run unbounded?".
|
|
27
|
-
* `idle` is a no-progress bound: it answers "is this run wedged?".
|
|
28
|
-
*
|
|
29
|
-
* These are different questions and one number cannot answer both. A ceiling
|
|
30
|
-
* tight enough to catch a wedge promptly is also tight enough to fail a correct
|
|
31
|
-
* run on a busy machine; a ceiling loose enough never to do that cannot catch a
|
|
32
|
-
* wedge promptly. Keeping the two bounds separate is what makes each honest.
|
|
33
|
-
*/
|
|
34
|
-
type ExpiredBound = { kind: 'total' | 'idle'; limitMs: number };
|
|
35
|
-
|
|
36
87
|
interface RunState {
|
|
37
88
|
settled: boolean;
|
|
38
89
|
expiry: ExpiredBound | null;
|
|
39
90
|
forceKillNeeded: boolean;
|
|
91
|
+
/** Set when a borrowed grant closed while the command ran, and the command was stopped as a lost grant. */
|
|
92
|
+
grantLost: boolean;
|
|
93
|
+
/** The sub-pool's close error when it did not close cleanly: the run fails. */
|
|
94
|
+
poolCloseError: string | null;
|
|
40
95
|
}
|
|
41
96
|
|
|
97
|
+
type GrantedTokens = Extract<TokenGrant, { granted: true }>;
|
|
98
|
+
|
|
42
99
|
export interface ProcessTreeKiller {
|
|
43
100
|
kill(pid: number, signal: NodeJS.Signals): Promise<void>;
|
|
44
101
|
}
|
|
45
102
|
|
|
103
|
+
/** Where the command's TMPDIR comes from; `null` leaves the inherited one. */
|
|
104
|
+
export type TempVolume = Pick<RamTempVolume, 'acquire' | 'release' | 'fullness' | 'reap'>;
|
|
105
|
+
|
|
46
106
|
export default function boundedExecExecutor(
|
|
107
|
+
options: BoundedExecOptions,
|
|
108
|
+
context: NxTaskContext,
|
|
109
|
+
): Promise<BoundedExecResult> {
|
|
110
|
+
return runDefaultBoundedExec(options, recordContext(context));
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/** Runs with the normal process, RAM lease and CPU-budget owners, with an optional stdout sink. */
|
|
114
|
+
export function runDefaultBoundedExec(
|
|
47
115
|
options: BoundedExecOptions,
|
|
48
116
|
context: BoundedExecContext,
|
|
49
117
|
): Promise<BoundedExecResult> {
|
|
50
|
-
|
|
118
|
+
// A target that names the cowshed its disk commands run through names it for this run's own RAM volume work and
|
|
119
|
+
// for its command alike. Nx resolves `{workspaceRoot}/…` in an option to a workspace-relative path, which the
|
|
120
|
+
// command, starting in another directory, could not find: it gets the absolute one.
|
|
121
|
+
const named = options.env?.[COWSHED_EXECUTABLE_ENV];
|
|
122
|
+
const cowshed = named === undefined ? undefined : resolvePath(context.root, named);
|
|
123
|
+
const run: BoundedExecOptions =
|
|
124
|
+
cowshed === undefined ? options : { ...options, env: { ...options.env, [COWSHED_EXECUTABLE_ENV]: cowshed } };
|
|
125
|
+
// Only macOS has `hdiutil ram://`; elsewhere the command keeps the inherited TMPDIR.
|
|
126
|
+
const tempVolume =
|
|
127
|
+
process.platform === 'darwin'
|
|
128
|
+
? new RamTempVolume(
|
|
129
|
+
ramTempPaths(userInfo()),
|
|
130
|
+
RAM_TEMP_CAPACITY_BYTES,
|
|
131
|
+
createHostCommands(undefined, undefined, undefined, cowshed),
|
|
132
|
+
)
|
|
133
|
+
: null;
|
|
134
|
+
// A command a parent's CPU pool started draws its tokens from that pool. The pool is named by the environment the
|
|
135
|
+
// parent set for this command; a run no parent started has none.
|
|
136
|
+
const inherited = readInheritedCpuPool(process.env);
|
|
137
|
+
return runBoundedExec(run, context, createProcessTreeKiller(), tempVolume, gatewayCpuBudget(), inherited);
|
|
51
138
|
}
|
|
52
139
|
|
|
140
|
+
/**
|
|
141
|
+
* Nx gives an executor the task's project, target and configuration, not its id; the id is the
|
|
142
|
+
* key of the task graph entry they name. Outside a task graph the run records nothing.
|
|
143
|
+
*/
|
|
144
|
+
export function recordContext(context: NxTaskContext): BoundedExecContext {
|
|
145
|
+
const { root, projectName, targetName, configurationName, taskGraph } = context;
|
|
146
|
+
if (!projectName || !targetName) {
|
|
147
|
+
return { root };
|
|
148
|
+
}
|
|
149
|
+
const id = `${projectName}:${targetName}${configurationName ? `:${configurationName}` : ''}`;
|
|
150
|
+
const task = taskGraph?.tasks[id];
|
|
151
|
+
if (task === undefined) {
|
|
152
|
+
return { root };
|
|
153
|
+
}
|
|
154
|
+
return {
|
|
155
|
+
root,
|
|
156
|
+
record: {
|
|
157
|
+
task: { id, hash: task.hash ?? null },
|
|
158
|
+
directory: taskDirectory(workspaceDataDirectoryForWorkspace(root), id),
|
|
159
|
+
},
|
|
160
|
+
};
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/**
|
|
164
|
+
* Run the command once its runner holds CPU tokens from `budget` (`cpu-tokens.ts`), sized to the
|
|
165
|
+
* grant; `null`, or a budget nobody serves, runs it as asked. Both bounds start once it is
|
|
166
|
+
* granted: a wait for the host is not the command's time.
|
|
167
|
+
*
|
|
168
|
+
* `inherited` is the parent's CPU pool (`cpu-pool.ts`) that lends this run its tokens. The run asks that pool first
|
|
169
|
+
* and the host budget only when the pool refuses. Whatever it is granted, the run serves its own sub-pool to its
|
|
170
|
+
* descendants, closed before the grant returns. A borrowed grant that closes while the command runs stops the command
|
|
171
|
+
* as a lost grant, which is a failure.
|
|
172
|
+
*/
|
|
53
173
|
export async function runBoundedExec(
|
|
54
174
|
options: BoundedExecOptions,
|
|
55
175
|
context: BoundedExecContext,
|
|
56
176
|
killer: ProcessTreeKiller,
|
|
177
|
+
tempVolume: TempVolume | null,
|
|
178
|
+
budget: CpuBudget | null,
|
|
179
|
+
inherited: InheritedCpuPool | null = null,
|
|
180
|
+
captureDiagnostics: typeof captureHangDiagnostics = captureHangDiagnostics,
|
|
57
181
|
): Promise<BoundedExecResult> {
|
|
58
182
|
const cwd = resolveCwd(options.cwd, context.root);
|
|
59
|
-
const
|
|
183
|
+
const directory = context.record?.directory ?? null;
|
|
184
|
+
// A previous run's verdict and report go first, so neither can be read as this run's.
|
|
185
|
+
if (directory !== null) {
|
|
186
|
+
await clearTaskDirectory(directory);
|
|
187
|
+
}
|
|
188
|
+
const requested = buildCommand(options);
|
|
60
189
|
const timeoutMs = options.timeoutMs;
|
|
61
190
|
const idleTimeoutMs = options.idleTimeoutMs;
|
|
62
191
|
const killAfterMs = options.killAfterMs ?? DEFAULT_KILL_AFTER_MS;
|
|
63
|
-
const startedAt = Date.now();
|
|
64
192
|
const outputChunks: string[] = [];
|
|
65
|
-
const state: RunState = {
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
windowsHide: true,
|
|
73
|
-
});
|
|
74
|
-
|
|
193
|
+
const state: RunState = {
|
|
194
|
+
settled: false,
|
|
195
|
+
expiry: null,
|
|
196
|
+
forceKillNeeded: false,
|
|
197
|
+
grantLost: false,
|
|
198
|
+
poolCloseError: null,
|
|
199
|
+
};
|
|
75
200
|
const appendStdout = (chunk: Buffer | string): void => {
|
|
76
201
|
const text = chunk.toString();
|
|
77
|
-
|
|
78
|
-
|
|
202
|
+
if (context.stdout !== undefined) {
|
|
203
|
+
context.stdout(text);
|
|
204
|
+
} else {
|
|
205
|
+
outputChunks.push(text);
|
|
206
|
+
process.stdout.write(text);
|
|
207
|
+
}
|
|
79
208
|
};
|
|
80
209
|
const appendStderr = (chunk: Buffer | string): void => {
|
|
81
210
|
const text = chunk.toString();
|
|
@@ -83,25 +212,259 @@ export async function runBoundedExec(
|
|
|
83
212
|
process.stderr.write(text);
|
|
84
213
|
};
|
|
85
214
|
|
|
215
|
+
let lease: RamTempLease | null = null;
|
|
216
|
+
if (tempVolume !== null) {
|
|
217
|
+
const acquired = await tempVolume.acquire();
|
|
218
|
+
if (!acquired.ok) {
|
|
219
|
+
const message = `${describeRamTempError(acquired.error)}\n`;
|
|
220
|
+
process.stderr.write(message);
|
|
221
|
+
return { success: false, terminalOutput: message };
|
|
222
|
+
}
|
|
223
|
+
if (acquired.value.kind === 'leased') {
|
|
224
|
+
lease = acquired.value.lease;
|
|
225
|
+
for (const held of acquired.value.held) {
|
|
226
|
+
appendStderr(`RAM temp volume: dead lease ${held}\n`);
|
|
227
|
+
}
|
|
228
|
+
for (const reaped of acquired.value.reaped) {
|
|
229
|
+
appendStderr(`RAM temp volume: stopped ${reaped}, which a dead task left working in its lease\n`);
|
|
230
|
+
}
|
|
231
|
+
} else {
|
|
232
|
+
appendStderr(
|
|
233
|
+
`RAM temp volume unavailable in this sandbox (${acquired.value.detail}); TMPDIR stays ${process.env.TMPDIR ?? '(unset)'}\n`,
|
|
234
|
+
);
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
const releaseLease = async (): Promise<void> => {
|
|
238
|
+
if (lease === null) {
|
|
239
|
+
return;
|
|
240
|
+
}
|
|
241
|
+
const held = lease;
|
|
242
|
+
lease = null;
|
|
243
|
+
// What the command left working in its lease is a leak, whichever way the command ended; it
|
|
244
|
+
// is stopped before the lease goes, so the volume can detach, and said, so its owner fixes it.
|
|
245
|
+
const reaped = await tempVolume?.reap(held);
|
|
246
|
+
if (reaped?.ok) {
|
|
247
|
+
for (const leftover of reaped.value) {
|
|
248
|
+
appendStderr(`RAM temp volume: stopped ${leftover}, which the command left working in its lease\n`);
|
|
249
|
+
}
|
|
250
|
+
} else if (reaped) {
|
|
251
|
+
appendStderr(`${describeRamTempError(reaped.error)}\n`);
|
|
252
|
+
}
|
|
253
|
+
const released = await tempVolume?.release(held);
|
|
254
|
+
if (released && !released.ok) {
|
|
255
|
+
appendStderr(`${describeRamTempError(released.error)}\n`);
|
|
256
|
+
}
|
|
257
|
+
};
|
|
258
|
+
|
|
259
|
+
// The wait is one span line, as cowshed's own spans read. With no gateway listening, as on any
|
|
260
|
+
// host without cowshed, the command runs exactly as asked and nothing is said.
|
|
261
|
+
const cores = availableParallelism();
|
|
262
|
+
const demand = runnerDemand(requested, options.parallelism, cores, process.env, options.env);
|
|
263
|
+
const asked = Date.now();
|
|
264
|
+
// A run a parent's CPU pool lent tokens to takes them there. Only an absent pool leaves the host budget to ask.
|
|
265
|
+
// A pool that answered and refused still holds tokens of its own: asking the host then would hold them while waiting
|
|
266
|
+
// on another budget, so the run fails instead.
|
|
267
|
+
let grant: GrantedTokens | null = null;
|
|
268
|
+
let parentRefusal: string | null = null;
|
|
269
|
+
if (inherited !== null) {
|
|
270
|
+
const fromParent = await takeParentCpuTokens(inherited, demand.want, cores);
|
|
271
|
+
if (fromParent.granted) {
|
|
272
|
+
grant = fromParent;
|
|
273
|
+
} else if (fromParent.cause === 'absent') {
|
|
274
|
+
appendStderr(`cowshed: cpu-tokens parent pool refused: ${fromParent.reason}; asking the host budget\n`);
|
|
275
|
+
} else {
|
|
276
|
+
parentRefusal = fromParent.reason;
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
let unbudgeted: string | null = null;
|
|
280
|
+
if (grant === null && parentRefusal === null && budget !== null) {
|
|
281
|
+
const fromHost = await budget.take(demand.want, context.root, requested);
|
|
282
|
+
if (fromHost.granted) {
|
|
283
|
+
grant = fromHost;
|
|
284
|
+
} else if (fromHost.cause !== 'absent') {
|
|
285
|
+
unbudgeted = fromHost.reason;
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
const waitedMs = Date.now() - asked;
|
|
289
|
+
let cpu: BoundedExecRecord['cpu'];
|
|
290
|
+
let sized: { command: string; env: Record<string, string> } = { command: requested, env: {} };
|
|
291
|
+
if (grant !== null) {
|
|
292
|
+
sized = sizeRunner(demand, requested, grant.tokens);
|
|
293
|
+
cpu = { tokens: grant.tokens, want: demand.want, waitedMs, ...(grant.inherited ? { inherited: true } : {}) };
|
|
294
|
+
appendStderr(
|
|
295
|
+
`cowshed: cpu-tokens wait done elapsed=${waitedMs}ms status=ok ${grant.inherited ? 'inherited=true ' : ''}tokens=${grant.tokens}/${demand.want} runner=${demand.kind}\n`,
|
|
296
|
+
);
|
|
297
|
+
} else if (unbudgeted !== null) {
|
|
298
|
+
appendStderr(
|
|
299
|
+
`cowshed: cpu-tokens wait done elapsed=${waitedMs}ms status=err runner=${demand.kind} runs unbudgeted: ${unbudgeted}\n`,
|
|
300
|
+
);
|
|
301
|
+
}
|
|
302
|
+
// The run's sub-pool and grant are given back exactly once, however the run ends. The sub-pool closes first, so no
|
|
303
|
+
// descendant still draws on tokens the grant has returned.
|
|
304
|
+
let pool: CpuPool | null = null;
|
|
305
|
+
let cpuReleased: Promise<void> | null = null;
|
|
306
|
+
const releaseCpu = (): Promise<void> => {
|
|
307
|
+
cpuReleased ??= (async () => {
|
|
308
|
+
try {
|
|
309
|
+
const outcome = await pool?.close();
|
|
310
|
+
if (outcome !== undefined && !outcome.ok) {
|
|
311
|
+
state.poolCloseError = outcome.error;
|
|
312
|
+
appendStderr(`cowshed: cpu pool did not close cleanly: ${outcome.error}\n`);
|
|
313
|
+
}
|
|
314
|
+
} finally {
|
|
315
|
+
grant?.release();
|
|
316
|
+
}
|
|
317
|
+
})();
|
|
318
|
+
return cpuReleased;
|
|
319
|
+
};
|
|
320
|
+
// Records the run's verdict when it has a task to record as. An `expiry` is set only when a bound ended the run.
|
|
321
|
+
// An `operational` run failed on its CPU budget (a lost grant, or a sub-pool that did not close): an ordinary
|
|
322
|
+
// failure, whatever bound also fired, so it is never judged as a bound.
|
|
323
|
+
const recordRun = async (
|
|
324
|
+
exitCode: number,
|
|
325
|
+
expiry: ExpiredBound | null,
|
|
326
|
+
elapsedMs: number,
|
|
327
|
+
operational: boolean,
|
|
328
|
+
): Promise<void> => {
|
|
329
|
+
if (context.record === undefined) {
|
|
330
|
+
return;
|
|
331
|
+
}
|
|
332
|
+
const { task, directory: recordDirectory } = context.record;
|
|
333
|
+
const report = await readJunitReport(recordDirectory);
|
|
334
|
+
const verdict = operational
|
|
335
|
+
? judgeOperationalFailure(exitCode, report)
|
|
336
|
+
: judgeRun({ exitCode, elapsedMs, expiry, report });
|
|
337
|
+
if (verdict.outcome === 'bound' && verdict.bound === 'test') {
|
|
338
|
+
appendStderr(`Every failing test failed only on its per-test timeout: ${verdict.tests.join(', ')}\n`);
|
|
339
|
+
}
|
|
340
|
+
await writeRecord(recordDirectory, { task: task.id, hash: task.hash, verdict, ...(cpu ? { cpu } : {}) });
|
|
341
|
+
};
|
|
342
|
+
// A run refused before its command starts says why, gives back its grant and RAM, and records the failure.
|
|
343
|
+
const refuse = async (message: string): Promise<BoundedExecResult> => {
|
|
344
|
+
appendStderr(message);
|
|
345
|
+
await releaseCpu();
|
|
346
|
+
await releaseLease();
|
|
347
|
+
await recordRun(1, null, Date.now() - asked, false);
|
|
348
|
+
return { success: false, terminalOutput: outputChunks.join('') };
|
|
349
|
+
};
|
|
350
|
+
if (parentRefusal !== null) {
|
|
351
|
+
return refuse(
|
|
352
|
+
`cowshed: cpu-tokens parent pool refused: ${parentRefusal}; refusing to run the command without asking the host budget\n`,
|
|
353
|
+
);
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
// A granted run serves a sub-pool of its own grant, so its descendants draw from that and never from above it.
|
|
357
|
+
// A pool that cannot be set up refuses the run: no command runs unbudgeted.
|
|
358
|
+
if (grant !== null) {
|
|
359
|
+
const created = await createCpuPool(grant.tokens, grant.closed);
|
|
360
|
+
if (!created.ok) {
|
|
361
|
+
return refuse(`cowshed: cpu pool unavailable: ${created.error}; refusing to run the command\n`);
|
|
362
|
+
}
|
|
363
|
+
pool = created.pool;
|
|
364
|
+
}
|
|
365
|
+
const command =
|
|
366
|
+
directory === null
|
|
367
|
+
? withProgressReport(sized.command)
|
|
368
|
+
: await withJunitReport(withProgressReport(sized.command), directory);
|
|
369
|
+
// A runner bounded-exec cannot see in the command (a script that spawns `bun test`) writes its JUnit report to this
|
|
370
|
+
// path when it is set. The grant's sizing outranks the target's own environment: the runner may use what it was
|
|
371
|
+
// granted, not what it was configured with. The sub-pool's environment comes last: a descendant reads only the pool
|
|
372
|
+
// this run serves.
|
|
373
|
+
const env: Record<string, string> = {
|
|
374
|
+
...(directory === null ? {} : { [JUNIT_REPORT_ENV]: join(directory, JUNIT_FILE) }),
|
|
375
|
+
...targetEnv(options.env, appendStderr),
|
|
376
|
+
...sized.env,
|
|
377
|
+
...(pool?.environment ?? {}),
|
|
378
|
+
};
|
|
379
|
+
const startedAt = Date.now();
|
|
380
|
+
let child: ChildProcess;
|
|
381
|
+
try {
|
|
382
|
+
child = spawn(command, [], {
|
|
383
|
+
cwd,
|
|
384
|
+
env: mergeEnv(lease, env),
|
|
385
|
+
shell: true,
|
|
386
|
+
detached: process.platform !== 'win32',
|
|
387
|
+
windowsHide: true,
|
|
388
|
+
});
|
|
389
|
+
} catch (error) {
|
|
390
|
+
// Nothing started, so nothing runs under the grant: it goes back here, with the RAM lease.
|
|
391
|
+
return refuse(`cowshed: could not start the command: ${error instanceof Error ? error.message : String(error)}\n`);
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
// Progress is a finished test, not output: a runner's SLOW warnings and a hung test's heartbeat keep printing.
|
|
395
|
+
const progress = createRunProgress();
|
|
86
396
|
let idleTimer: NodeJS.Timeout | undefined;
|
|
397
|
+
let captureTimer: NodeJS.Timeout | undefined;
|
|
398
|
+
let idleCapture: { expiresAtMs: number; controller: AbortController; result: IdleDiagnosticResult } | undefined;
|
|
399
|
+
const cancelCapture = (): void => {
|
|
400
|
+
clearTimeout(captureTimer);
|
|
401
|
+
idleCapture?.controller.abort();
|
|
402
|
+
};
|
|
87
403
|
const armIdleTimer = (): void => {
|
|
88
404
|
if (idleTimeoutMs === undefined || state.settled || state.expiry !== null) {
|
|
89
405
|
return;
|
|
90
406
|
}
|
|
91
407
|
clearTimeout(idleTimer);
|
|
92
|
-
|
|
408
|
+
cancelCapture();
|
|
409
|
+
const capture: NonNullable<typeof idleCapture> = {
|
|
410
|
+
expiresAtMs: Date.now() + idleTimeoutMs,
|
|
411
|
+
controller: new AbortController(),
|
|
412
|
+
result: { kind: 'pending', startedAtMs: null },
|
|
413
|
+
};
|
|
414
|
+
idleCapture = capture;
|
|
415
|
+
captureTimer = setTimeout(
|
|
416
|
+
() => {
|
|
417
|
+
if (state.settled || state.expiry !== null || idleCapture !== capture) return;
|
|
418
|
+
const pid = child.pid;
|
|
419
|
+
if (pid === undefined) {
|
|
420
|
+
idleCapture.result = { kind: 'failed', detail: 'runner PID unavailable; no holder conclusion' };
|
|
421
|
+
return;
|
|
422
|
+
}
|
|
423
|
+
idleCapture.result = { kind: 'pending', startedAtMs: Date.now() };
|
|
424
|
+
void Promise.resolve()
|
|
425
|
+
.then(() =>
|
|
426
|
+
captureDiagnostics({
|
|
427
|
+
pid,
|
|
428
|
+
pgid: process.platform === 'win32' ? null : pid,
|
|
429
|
+
expiresAtMs: capture.expiresAtMs,
|
|
430
|
+
deadlineMs: Math.max(0, Math.min(HANG_CAPTURE_MS, capture.expiresAtMs - Date.now())),
|
|
431
|
+
signal: capture.controller.signal,
|
|
432
|
+
}),
|
|
433
|
+
)
|
|
434
|
+
.then(
|
|
435
|
+
(snapshot) => {
|
|
436
|
+
if (idleCapture === capture) idleCapture.result = { kind: 'captured', snapshot };
|
|
437
|
+
},
|
|
438
|
+
(error: unknown) => {
|
|
439
|
+
if (idleCapture === capture) {
|
|
440
|
+
idleCapture.result = {
|
|
441
|
+
kind: 'failed',
|
|
442
|
+
detail: error instanceof Error ? error.message : String(error),
|
|
443
|
+
};
|
|
444
|
+
}
|
|
445
|
+
},
|
|
446
|
+
);
|
|
447
|
+
},
|
|
448
|
+
Math.max(0, idleTimeoutMs - HANG_CAPTURE_MS),
|
|
449
|
+
);
|
|
450
|
+
idleTimer = setTimeout(
|
|
451
|
+
() => expire({ kind: 'idle', limitMs: idleTimeoutMs, tests: progress.inFlight() }),
|
|
452
|
+
idleTimeoutMs,
|
|
453
|
+
);
|
|
93
454
|
};
|
|
94
455
|
|
|
95
|
-
// Only the CHILD's own output
|
|
456
|
+
// Only the CHILD's own output is read for progress. The diagnostics below go
|
|
96
457
|
// through appendStderr as well, and re-arming on those would let a teardown
|
|
97
458
|
// message extend the very bound that just fired.
|
|
98
459
|
child.stdout?.on('data', (chunk: Buffer | string) => {
|
|
99
|
-
|
|
100
|
-
|
|
460
|
+
const text = chunk.toString();
|
|
461
|
+
if (progress.scan('stdout', text)) armIdleTimer();
|
|
462
|
+
appendStdout(text);
|
|
101
463
|
});
|
|
102
464
|
child.stderr?.on('data', (chunk: Buffer | string) => {
|
|
103
|
-
|
|
104
|
-
|
|
465
|
+
const text = chunk.toString();
|
|
466
|
+
if (progress.scan('stderr', text)) armIdleTimer();
|
|
467
|
+
appendStderr(text);
|
|
105
468
|
});
|
|
106
469
|
|
|
107
470
|
const killChildTree = async (force: boolean): Promise<void> => {
|
|
@@ -117,7 +480,11 @@ export async function runBoundedExec(
|
|
|
117
480
|
};
|
|
118
481
|
const onTerminationSignal = (signal: NodeJS.Signals): void => {
|
|
119
482
|
removeSignalHandlers();
|
|
120
|
-
|
|
483
|
+
cancelCapture();
|
|
484
|
+
void killChildTree(false)
|
|
485
|
+
.finally(releaseCpu)
|
|
486
|
+
.finally(releaseLease)
|
|
487
|
+
.finally(() => process.kill(process.pid, signal));
|
|
121
488
|
};
|
|
122
489
|
const onSigint = (): void => onTerminationSignal('SIGINT');
|
|
123
490
|
const onSigterm = (): void => onTerminationSignal('SIGTERM');
|
|
@@ -154,8 +521,9 @@ export async function runBoundedExec(
|
|
|
154
521
|
// the report always names which bound did it. A bare "timed out" would leave
|
|
155
522
|
// the reader unable to tell a wedged toolchain from a loaded machine — the
|
|
156
523
|
// two have opposite fixes.
|
|
524
|
+
let termination: Promise<void> = Promise.resolve();
|
|
157
525
|
const expire = (bound: ExpiredBound): void => {
|
|
158
|
-
if (state.settled || state.expiry !== null) {
|
|
526
|
+
if (state.settled || state.expiry !== null || state.grantLost) {
|
|
159
527
|
return;
|
|
160
528
|
}
|
|
161
529
|
state.expiry = bound;
|
|
@@ -164,33 +532,105 @@ export async function runBoundedExec(
|
|
|
164
532
|
const elapsedMs = Date.now() - startedAt;
|
|
165
533
|
appendStderr(
|
|
166
534
|
bound.kind === 'idle'
|
|
167
|
-
? `\nCommand made no progress: no
|
|
535
|
+
? `\nCommand made no progress: no test finished for ${bound.limitMs}ms (idleTimeoutMs=${bound.limitMs}) after ${elapsedMs}ms of runtime; ${
|
|
536
|
+
bound.tests.length > 0
|
|
537
|
+
? `still running: ${bound.tests.join(', ')}`
|
|
538
|
+
: 'the runner named no test still running'
|
|
539
|
+
} (cwd=${cwd}): ${command}\n`
|
|
168
540
|
: `\nCommand timed out after ${elapsedMs}ms (timeoutMs=${bound.limitMs}, cwd=${cwd}): ${command}\n`,
|
|
169
541
|
);
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
542
|
+
if (bound.kind === 'idle') {
|
|
543
|
+
const observed = idleCapture?.result;
|
|
544
|
+
appendStderr(
|
|
545
|
+
`Hang diagnostics: runner pid=${child.pid ?? 'unavailable'} pgid=${
|
|
546
|
+
process.platform === 'win32' ? 'unavailable' : (child.pid ?? 'unavailable')
|
|
547
|
+
} expiryAtMs=${Date.now()} scheduledExpiryAtMs=${idleCapture?.expiresAtMs ?? 'unavailable'}\n`,
|
|
548
|
+
);
|
|
549
|
+
if (observed?.kind === 'captured') {
|
|
550
|
+
appendStderr(
|
|
551
|
+
`Hang diagnostics: outcome=${observed.snapshot.outcome} capturedAtMs=${observed.snapshot.capturedAtMs} actualExpiryOffsetMs=${
|
|
552
|
+
Date.now() - observed.snapshot.capturedAtMs
|
|
553
|
+
}\n${observed.snapshot.text}`,
|
|
554
|
+
);
|
|
555
|
+
} else {
|
|
556
|
+
appendStderr(
|
|
557
|
+
`Hang diagnostics: outcome=incomplete; ${
|
|
558
|
+
observed?.kind === 'failed'
|
|
559
|
+
? `capture failed: ${observed.detail}`
|
|
560
|
+
: `capture incomplete at expiry (startedAtMs=${observed?.startedAtMs ?? 'not started'})`
|
|
561
|
+
}; no empty-holder conclusion\n`,
|
|
562
|
+
);
|
|
174
563
|
}
|
|
175
|
-
|
|
564
|
+
}
|
|
565
|
+
cancelCapture();
|
|
566
|
+
termination = terminate();
|
|
567
|
+
};
|
|
568
|
+
|
|
569
|
+
// A timed-out run ends with its whole process group, not with its leader.
|
|
570
|
+
// The graceful signal reaches every member that exists when it is sent, but
|
|
571
|
+
// a member that ignores SIGTERM, or one forked while the signal goes out
|
|
572
|
+
// (macOS does not hold a fork back for a group signal in flight), outlives
|
|
573
|
+
// a leader that exits on it. The grace ends when the leader exits or
|
|
574
|
+
// killAfterMs passes, whichever comes first; whatever is left then is killed.
|
|
575
|
+
const terminate = async (): Promise<void> => {
|
|
576
|
+
await ignoreKillError(killChildTree(false));
|
|
577
|
+
if (!state.settled && killAfterMs > 0) {
|
|
578
|
+
await settledWithin(exitPromise, killAfterMs);
|
|
579
|
+
}
|
|
580
|
+
if (state.settled) {
|
|
581
|
+
if (child.pid !== undefined && processGroupHasMembers(child.pid)) {
|
|
176
582
|
state.forceKillNeeded = true;
|
|
177
|
-
appendStderr(
|
|
583
|
+
appendStderr(
|
|
584
|
+
`Force-killing what the ${state.expiry === null ? 'command' : 'timed-out command'} left in its process group: ${command}\n`,
|
|
585
|
+
);
|
|
178
586
|
await ignoreKillError(killChildTree(true));
|
|
179
|
-
await delay(REAP_AFTER_FORCE_KILL_MS);
|
|
180
587
|
}
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
588
|
+
return;
|
|
589
|
+
}
|
|
590
|
+
state.forceKillNeeded = true;
|
|
591
|
+
appendStderr(
|
|
592
|
+
`Force-killing ${state.expiry === null ? 'command' : 'timed out command'} after killAfterMs=${killAfterMs}: ${command}\n`,
|
|
593
|
+
);
|
|
594
|
+
await ignoreKillError(killChildTree(true));
|
|
595
|
+
await settledWithin(exitPromise, REAP_AFTER_FORCE_KILL_MS);
|
|
596
|
+
if (!state.settled) {
|
|
597
|
+
resolveExit({ code: 1, signal: 'SIGKILL' });
|
|
598
|
+
}
|
|
185
599
|
};
|
|
186
600
|
|
|
187
|
-
const totalTimer =
|
|
601
|
+
const totalTimer =
|
|
602
|
+
timeoutMs === undefined ? undefined : setTimeout(() => expire({ kind: 'total', limitMs: timeoutMs }), timeoutMs);
|
|
188
603
|
armIdleTimer();
|
|
189
604
|
|
|
605
|
+
// A borrowed grant ends with the ancestor that lent it, and the command runs on tokens that ancestor takes back
|
|
606
|
+
// then. Stopping the command is a lost grant: a failure, not a bound. Our own release after the command settled
|
|
607
|
+
// is not a loss; `cpuReleased` marks it.
|
|
608
|
+
const onGrantLost = (): void => {
|
|
609
|
+
if (state.settled || state.grantLost || cpuReleased !== null) {
|
|
610
|
+
return;
|
|
611
|
+
}
|
|
612
|
+
state.grantLost = true;
|
|
613
|
+
cancelCapture();
|
|
614
|
+
appendStderr(
|
|
615
|
+
`cowshed: cpu-tokens grant lost: its parent pool closed while the command ran; stopping it: ${command}\n`,
|
|
616
|
+
);
|
|
617
|
+
// A bound may already be terminating the still-running command. The loss remains an ordinary
|
|
618
|
+
// operational failure, but that existing termination owns the signals and grace.
|
|
619
|
+
if (state.expiry === null) termination = terminate();
|
|
620
|
+
};
|
|
621
|
+
if (grant?.inherited) {
|
|
622
|
+
void grant.closed.then(onGrantLost);
|
|
623
|
+
}
|
|
624
|
+
|
|
190
625
|
const { code: exitCode, signal: exitSignal } = await exitPromise;
|
|
626
|
+
await termination;
|
|
191
627
|
clearTimeout(totalTimer);
|
|
192
628
|
clearTimeout(idleTimer);
|
|
629
|
+
cancelCapture();
|
|
193
630
|
removeSignalHandlers();
|
|
631
|
+
// The runner is gone. Its sub-pool closes first, so no descendant still draws on tokens the grant returns;
|
|
632
|
+
// then the grant goes back to its parent or the host.
|
|
633
|
+
await releaseCpu();
|
|
194
634
|
|
|
195
635
|
const code = exitCode ?? signalToExitCode(exitSignal);
|
|
196
636
|
if (code !== 0 && state.expiry === null) {
|
|
@@ -201,8 +641,20 @@ export async function runBoundedExec(
|
|
|
201
641
|
appendStderr(`Timed out command exited after graceful termination: ${command}\n`);
|
|
202
642
|
}
|
|
203
643
|
|
|
644
|
+
// A test that dies of ENOSPC rarely says where it was writing; the volume can.
|
|
645
|
+
const full = code !== 0 && lease !== null ? await tempVolume?.fullness(lease) : null;
|
|
646
|
+
if (full) {
|
|
647
|
+
appendStderr(`${describeRamTempError(full)}\n`);
|
|
648
|
+
}
|
|
649
|
+
await releaseLease();
|
|
650
|
+
|
|
651
|
+
// A run whose grant was lost, or whose sub-pool did not close cleanly, fails even when the command exited 0, and
|
|
652
|
+
// whatever bound also fired: its verdict is an ordinary failure, never a bound.
|
|
653
|
+
const operationalFailure = state.grantLost || state.poolCloseError !== null;
|
|
654
|
+
await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure);
|
|
655
|
+
|
|
204
656
|
return {
|
|
205
|
-
success: state.expiry === null && code === 0,
|
|
657
|
+
success: !operationalFailure && state.expiry === null && code === 0,
|
|
206
658
|
terminalOutput: outputChunks.join(''),
|
|
207
659
|
};
|
|
208
660
|
}
|
|
@@ -217,21 +669,35 @@ export function createProcessTreeKiller(): ProcessTreeKiller {
|
|
|
217
669
|
|
|
218
670
|
function killProcessGroup(pid: number, signal: NodeJS.Signals): Promise<void> {
|
|
219
671
|
try {
|
|
220
|
-
//
|
|
221
|
-
//
|
|
222
|
-
//
|
|
223
|
-
//
|
|
224
|
-
//
|
|
225
|
-
process.
|
|
672
|
+
// On POSIX the executor spawns with detached: true, so the child leads a
|
|
673
|
+
// process group of its own and -pid signals all of it at once, including
|
|
674
|
+
// grandchildren that tree-walk libraries miss when parents die and
|
|
675
|
+
// children get reparented to init. Only the group is named: the leader may
|
|
676
|
+
// already be reaped, and its pid reused by an unrelated process, while the
|
|
677
|
+
// group lives on in its other members. Windows has no process groups.
|
|
678
|
+
process.kill(process.platform === 'win32' ? pid : -pid, signal);
|
|
226
679
|
} catch {
|
|
227
|
-
// ESRCH:
|
|
680
|
+
// ESRCH: the group (or the Windows process) has already exited.
|
|
681
|
+
}
|
|
682
|
+
return Promise.resolve();
|
|
683
|
+
}
|
|
684
|
+
|
|
685
|
+
/**
|
|
686
|
+
* Whether any process is left in the group `pgid` leads. A group outlives its
|
|
687
|
+
* reaped leader for as long as one member does, and no process can take its id
|
|
688
|
+
* meanwhile. Windows has no process groups, so there is nothing left to name.
|
|
689
|
+
*/
|
|
690
|
+
function processGroupHasMembers(pgid: number): boolean {
|
|
691
|
+
if (process.platform === 'win32') {
|
|
692
|
+
return false;
|
|
228
693
|
}
|
|
229
694
|
try {
|
|
230
|
-
process.kill(
|
|
231
|
-
|
|
232
|
-
|
|
695
|
+
process.kill(-pgid, 0);
|
|
696
|
+
return true;
|
|
697
|
+
} catch (error) {
|
|
698
|
+
// EPERM: a member exists but runs as another user.
|
|
699
|
+
return error instanceof Error && 'code' in error && error.code === 'EPERM';
|
|
233
700
|
}
|
|
234
|
-
return Promise.resolve();
|
|
235
701
|
}
|
|
236
702
|
|
|
237
703
|
function resolveCwd(cwd: string | undefined, root: string): string {
|
|
@@ -254,8 +720,34 @@ function buildCommand(options: BoundedExecOptions): string {
|
|
|
254
720
|
return parts.join(' ');
|
|
255
721
|
}
|
|
256
722
|
|
|
257
|
-
|
|
258
|
-
|
|
723
|
+
/** The lease's directory is the command's TMPDIR unless the target names one itself. */
|
|
724
|
+
function mergeEnv(lease: RamTempLease | null, env: Record<string, string>): NodeJS.ProcessEnv {
|
|
725
|
+
const inherited = withoutBudgetEnv(process.env);
|
|
726
|
+
return lease === null ? { ...inherited, ...env } : { ...inherited, TMPDIR: lease.directory, ...env };
|
|
727
|
+
}
|
|
728
|
+
|
|
729
|
+
/** The process's environment without the CPU budget's variables: a pool and its count only a grant may give. */
|
|
730
|
+
function withoutBudgetEnv(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv {
|
|
731
|
+
const kept = { ...env };
|
|
732
|
+
for (const name of BUDGET_ENV) {
|
|
733
|
+
delete kept[name];
|
|
734
|
+
}
|
|
735
|
+
return kept;
|
|
736
|
+
}
|
|
737
|
+
|
|
738
|
+
/**
|
|
739
|
+
* The target's environment without the CPU budget's variables: the budget sets those for the command it grants,
|
|
740
|
+
* so a value the target names is ignored, and `say` tells the reader so.
|
|
741
|
+
*/
|
|
742
|
+
function targetEnv(env: Record<string, string> | undefined, say: (text: string) => void): Record<string, string> {
|
|
743
|
+
const kept = { ...env };
|
|
744
|
+
for (const name of BUDGET_ENV) {
|
|
745
|
+
if (name in kept) {
|
|
746
|
+
say(`cowshed: ${name} in the target's env is ignored: the CPU budget sets it\n`);
|
|
747
|
+
delete kept[name];
|
|
748
|
+
}
|
|
749
|
+
}
|
|
750
|
+
return kept;
|
|
259
751
|
}
|
|
260
752
|
|
|
261
753
|
function signalToExitCode(signal: NodeJS.Signals | null): number {
|
|
@@ -273,6 +765,12 @@ async function ignoreKillError(promise: Promise<void>): Promise<void> {
|
|
|
273
765
|
}
|
|
274
766
|
}
|
|
275
767
|
|
|
276
|
-
|
|
277
|
-
|
|
768
|
+
/** Resolves when `promise` settles or `ms` passes, whichever is first, leaving no timer behind. */
|
|
769
|
+
async function settledWithin(promise: Promise<unknown>, ms: number): Promise<void> {
|
|
770
|
+
let timer: NodeJS.Timeout | undefined;
|
|
771
|
+
try {
|
|
772
|
+
await Promise.race([promise, new Promise<void>((resolve) => (timer = setTimeout(resolve, ms)))]);
|
|
773
|
+
} finally {
|
|
774
|
+
clearTimeout(timer);
|
|
775
|
+
}
|
|
278
776
|
}
|