@smoothbricks/nx-plugin 0.4.26 → 0.4.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +994 -117
- package/dist/bin/smoo-nx-bound-failures.d.ts +3 -0
- package/dist/bin/smoo-nx-bound-failures.d.ts.map +1 -0
- package/dist/bin/smoo-nx-bound-failures.js +31 -0
- package/dist/bin/smoo-nx-cargo-hash.js +28 -3
- package/dist/bin/smoo-nx-exec.js +15 -73
- package/dist/bin/smoo-nx-failures.d.ts +3 -0
- package/dist/bin/smoo-nx-failures.d.ts.map +1 -0
- package/dist/bin/smoo-nx-failures.js +3 -0
- package/dist/bin/smoo-nx-nextest-extract.js +3 -2
- package/dist/bin/smoo-nx-test-walks.d.ts +3 -0
- package/dist/bin/smoo-nx-test-walks.d.ts.map +1 -0
- package/dist/bin/smoo-nx-test-walks.js +34 -0
- package/dist/bounded-test-policy.d.ts +7 -1
- package/dist/bounded-test-policy.d.ts.map +1 -1
- package/dist/bounded-test-policy.js +101 -40
- package/dist/build-state-output-policy.d.ts +25 -0
- package/dist/build-state-output-policy.d.ts.map +1 -0
- package/dist/build-state-output-policy.js +64 -0
- package/dist/cargo-closure-input.d.ts +3 -2
- package/dist/cargo-closure-input.d.ts.map +1 -1
- package/dist/cargo-closure-input.js +16 -15
- package/dist/cargo-dev-profile.d.ts +37 -0
- package/dist/cargo-dev-profile.d.ts.map +1 -0
- package/dist/cargo-dev-profile.js +108 -0
- package/dist/cargo-source-hash.d.ts +43 -6
- package/dist/cargo-source-hash.d.ts.map +1 -1
- package/dist/cargo-source-hash.js +309 -73
- package/dist/cargo-toolchain-policy.d.ts +4 -0
- package/dist/cargo-toolchain-policy.d.ts.map +1 -1
- package/dist/cargo-toolchain-policy.js +5 -0
- package/dist/cargo-workspace.d.ts +101 -5
- package/dist/cargo-workspace.d.ts.map +1 -1
- package/dist/cargo-workspace.js +291 -10
- package/dist/cross-check-policy.d.ts +21 -3
- package/dist/cross-check-policy.d.ts.map +1 -1
- package/dist/cross-check-policy.js +24 -6
- package/dist/disk-command-policy.d.ts +7 -0
- package/dist/disk-command-policy.d.ts.map +1 -0
- package/dist/disk-command-policy.js +98 -0
- package/dist/ensure-built-receipt.d.ts +25 -0
- package/dist/ensure-built-receipt.d.ts.map +1 -0
- package/dist/ensure-built-receipt.js +261 -0
- package/dist/ensure-built-witness.d.ts +57 -0
- package/dist/ensure-built-witness.d.ts.map +1 -0
- package/dist/ensure-built-witness.js +412 -0
- package/dist/ensure-built.d.ts +207 -12
- package/dist/ensure-built.d.ts.map +1 -1
- package/dist/ensure-built.js +576 -202
- package/dist/eslint-file-set.d.ts +7 -0
- package/dist/eslint-file-set.d.ts.map +1 -0
- package/dist/eslint-file-set.js +12 -0
- package/dist/executors/bounded-exec/cpu-pool.d.ts +63 -0
- package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -0
- package/dist/executors/bounded-exec/cpu-pool.js +379 -0
- package/dist/executors/bounded-exec/cpu-tokens.d.ts +82 -0
- package/dist/executors/bounded-exec/cpu-tokens.d.ts.map +1 -0
- package/dist/executors/bounded-exec/cpu-tokens.js +114 -0
- package/dist/executors/bounded-exec/disk-lease.d.ts +61 -0
- package/dist/executors/bounded-exec/disk-lease.d.ts.map +1 -0
- package/dist/executors/bounded-exec/disk-lease.js +91 -0
- package/dist/executors/bounded-exec/executor.d.ts +45 -2
- package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
- package/dist/executors/bounded-exec/executor.js +447 -52
- package/dist/executors/bounded-exec/gateway-lease.d.ts +52 -0
- package/dist/executors/bounded-exec/gateway-lease.d.ts.map +1 -0
- package/dist/executors/bounded-exec/gateway-lease.js +143 -0
- package/dist/executors/bounded-exec/hang-diagnostics.d.ts +35 -0
- package/dist/executors/bounded-exec/hang-diagnostics.d.ts.map +1 -0
- package/dist/executors/bounded-exec/hang-diagnostics.js +231 -0
- package/dist/executors/bounded-exec/progress.d.ts +35 -0
- package/dist/executors/bounded-exec/progress.d.ts.map +1 -0
- package/dist/executors/bounded-exec/progress.js +79 -0
- package/dist/executors/bounded-exec/ram-temp-fixtures.d.ts +3 -0
- package/dist/executors/bounded-exec/ram-temp-fixtures.d.ts.map +1 -0
- package/dist/executors/bounded-exec/ram-temp-fixtures.js +12 -0
- package/dist/executors/bounded-exec/ram-temp-held.d.ts +91 -0
- package/dist/executors/bounded-exec/ram-temp-held.d.ts.map +1 -0
- package/dist/executors/bounded-exec/ram-temp-held.js +152 -0
- package/dist/executors/bounded-exec/ram-temp.d.ts +223 -0
- package/dist/executors/bounded-exec/ram-temp.d.ts.map +1 -0
- package/dist/executors/bounded-exec/ram-temp.js +1528 -0
- package/dist/executors/bounded-exec/schema.d.ts +29 -11
- package/dist/executors/bounded-exec/schema.d.ts.map +1 -1
- package/dist/executors/bounded-exec/verdict.d.ts +162 -0
- package/dist/executors/bounded-exec/verdict.d.ts.map +1 -0
- package/dist/executors/bounded-exec/verdict.js +318 -0
- package/dist/executors/shard-timings/executor.d.ts +14 -0
- package/dist/executors/shard-timings/executor.d.ts.map +1 -0
- package/dist/executors/shard-timings/executor.js +25 -0
- package/dist/executors/shard-timings/schema.d.ts +4 -0
- package/dist/executors/shard-timings/schema.d.ts.map +1 -0
- package/dist/executors/shard-timings/schema.js +1 -0
- package/dist/failure-summary.d.ts +5 -0
- package/dist/failure-summary.d.ts.map +1 -0
- package/dist/failure-summary.js +118 -0
- package/dist/generators/create-package/generator.d.ts.map +1 -1
- package/dist/generators/create-package/generator.js +6 -2
- package/dist/index.d.ts +14 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +370 -138
- package/dist/managed-files/ci-workflow.d.ts +13 -0
- package/dist/managed-files/ci-workflow.d.ts.map +1 -1
- package/dist/managed-files/ci-workflow.js +44 -13
- package/dist/managed-files/files.d.ts +7 -1
- package/dist/managed-files/files.d.ts.map +1 -1
- package/dist/managed-files/files.js +62 -8
- package/dist/managed-files/json-edit.d.ts +21 -0
- package/dist/managed-files/json-edit.d.ts.map +1 -0
- package/dist/managed-files/json-edit.js +295 -0
- package/dist/managed-files/managed-format.d.ts +33 -1
- package/dist/managed-files/managed-format.d.ts.map +1 -1
- package/dist/managed-files/managed-format.js +82 -2
- package/dist/managed-files/pr-preview-cleanup-workflow.d.ts +7 -0
- package/dist/managed-files/pr-preview-cleanup-workflow.d.ts.map +1 -1
- package/dist/managed-files/pr-preview-cleanup-workflow.js +68 -4
- package/dist/managed-files/publish-workflow.d.ts.map +1 -1
- package/dist/managed-files/publish-workflow.js +231 -48
- package/dist/managed-files/tree.d.ts +5 -0
- package/dist/managed-files/tree.d.ts.map +1 -1
- package/dist/managed-files/tree.js +5 -3
- package/dist/manifest-hash.d.ts.map +1 -1
- package/dist/manifest-hash.js +2 -5
- package/dist/nextest-code-bound.d.ts +62 -0
- package/dist/nextest-code-bound.d.ts.map +1 -0
- package/dist/nextest-code-bound.js +154 -0
- package/dist/nextest-extraction.d.ts +24 -8
- package/dist/nextest-extraction.d.ts.map +1 -1
- package/dist/nextest-extraction.js +42 -7
- package/dist/package-target-policy.d.ts.map +1 -1
- package/dist/package-target-policy.js +3 -5
- package/dist/release-config-policy.d.ts.map +1 -1
- package/dist/release-config-policy.js +3 -5
- package/dist/runtime-input.d.ts +14 -0
- package/dist/runtime-input.d.ts.map +1 -0
- package/dist/runtime-input.js +15 -0
- package/dist/rust-tokens.d.ts +11 -0
- package/dist/rust-tokens.d.ts.map +1 -0
- package/dist/rust-tokens.js +81 -0
- package/dist/task-results.d.ts +6 -0
- package/dist/task-results.d.ts.map +1 -0
- package/dist/task-results.js +61 -0
- package/dist/test-program.d.ts +54 -0
- package/dist/test-program.d.ts.map +1 -0
- package/dist/test-program.js +292 -0
- package/dist/test-timings.d.ts +71 -0
- package/dist/test-timings.d.ts.map +1 -0
- package/dist/test-timings.js +393 -0
- package/dist/test-walk-policy.d.ts +103 -0
- package/dist/test-walk-policy.d.ts.map +1 -0
- package/dist/test-walk-policy.js +657 -0
- package/dist/testing.d.ts +177 -0
- package/dist/testing.d.ts.map +1 -0
- package/dist/testing.js +824 -0
- package/dist/type-guards.d.ts +3 -0
- package/dist/type-guards.d.ts.map +1 -0
- package/dist/type-guards.js +4 -0
- package/dist/typecheck-test-policy.d.ts +8 -5
- package/dist/typecheck-test-policy.d.ts.map +1 -1
- package/dist/typecheck-test-policy.js +113 -100
- package/dist/workspace-config-policy.d.ts +8 -0
- package/dist/workspace-config-policy.d.ts.map +1 -1
- package/dist/workspace-config-policy.js +175 -63
- package/dist/workspace-index.d.ts +12 -0
- package/dist/workspace-index.d.ts.map +1 -0
- package/dist/workspace-index.js +17 -0
- package/dist/workspace-manifest.d.ts +37 -0
- package/dist/workspace-manifest.d.ts.map +1 -1
- package/executors.json +5 -0
- package/loader.js +13 -11
- package/managed/raw/envrc +33 -67
- package/managed/raw/git-format-staged.yml +30 -35
- package/managed/raw/patches/nx@23.2.1.patch +3484 -0
- package/managed/raw/tooling/direnv/apple-developer.sh +83 -0
- package/managed/raw/tooling/direnv/developer-links.ts +569 -11
- package/managed/raw/tooling/direnv/devenv.smoo.nix +314 -93
- package/managed/raw/tooling/direnv/github-actions-bootstrap.sh +20 -11
- package/managed/raw/tooling/direnv/linked-snapshots.ts +845 -0
- package/managed/raw/tooling/direnv/nx-socket-dir.sh +82 -0
- package/managed/raw/tooling/direnv/repo-path +14 -2
- package/managed/raw/tooling/direnv/setup-environment.ts +962 -148
- package/managed/raw/tooling/direnv/toolchain-stamp.ts +45 -3
- package/managed/raw/tooling/direnv/ttsc-env.sh +39 -0
- package/managed/raw/tooling/git-hooks/commit-msg.sh +5 -6
- package/managed/raw/tooling/git-hooks/pre-commit.sh +7 -6
- package/managed/raw/tooling/git-hooks/pre-push.sh +20 -15
- package/managed/templates/github/actions/cache-nix-devenv/action.yml +6 -6
- package/managed/templates/github/actions/cache-node-modules/action.yml +6 -4
- package/managed/templates/github/actions/cache-nx/action.yml +1 -1
- package/managed/templates/github/actions/cache-ttsc-plugins/action.yml +1 -1
- package/managed/templates/github/actions/forgejo-artifact.cjs +99 -0
- package/managed/templates/github/actions/forgejo-download-artifact/action.yml +49 -0
- package/managed/templates/github/actions/forgejo-download-artifact/index.cjs +7 -0
- package/managed/templates/github/actions/forgejo-upload-artifact/action.yml +65 -0
- package/managed/templates/github/actions/forgejo-upload-artifact/index.cjs +7 -0
- package/managed/templates/github/actions/setup-devenv/action.yml +21 -3
- package/managed/templates/github/workflows/managed-files.yml +1 -2
- package/nextest.toml +8 -41
- package/package.json +74 -12
- package/runtime-input.sh +81 -0
- package/src/__tests__/counted-cargo.ts +33 -14
- package/src/__tests__/fixture-nx-env.ts +111 -46
- package/src/__tests__/progress-clock.ts +43 -0
- package/src/__tests__/runtime-input.ts +23 -0
- package/src/bin/smoo-nx-bound-failures.test.ts +127 -0
- package/src/bin/smoo-nx-bound-failures.ts +33 -0
- package/src/bin/smoo-nx-cargo-hash.ts +28 -3
- package/src/bin/smoo-nx-exec.ts +16 -74
- package/src/bin/smoo-nx-failures.ts +4 -0
- package/src/bin/smoo-nx-nextest-extract.ts +3 -2
- package/src/bin/smoo-nx-test-walks.ts +42 -0
- package/src/bounded-test-policy.test.ts +268 -21
- package/src/bounded-test-policy.ts +114 -42
- package/src/build-state-output-policy.test.ts +115 -0
- package/src/build-state-output-policy.ts +90 -0
- package/src/bun-bootstrap.test.ts +57 -3
- package/src/bun-file-packages.test.ts +47 -0
- package/src/cargo-closure-input.test.ts +138 -66
- package/src/cargo-closure-input.ts +17 -15
- package/src/cargo-dev-profile.ts +137 -0
- package/src/cargo-resolve-cache.test.ts +70 -1
- package/src/cargo-source-hash.test.ts +193 -97
- package/src/cargo-source-hash.ts +345 -68
- package/src/cargo-toolchain-policy.ts +6 -0
- package/src/cargo-workspace.test.ts +320 -0
- package/src/cargo-workspace.ts +336 -10
- package/src/cross-check-policy.test.ts +59 -1
- package/src/cross-check-policy.ts +25 -6
- package/src/disk-command-policy.test.ts +61 -0
- package/src/disk-command-policy.ts +107 -0
- package/src/ensure-built-receipt.test.ts +184 -0
- package/src/ensure-built-receipt.ts +263 -0
- package/src/ensure-built-witness.test.ts +344 -0
- package/src/ensure-built-witness.ts +252 -0
- package/src/ensure-built.test.ts +1036 -578
- package/src/ensure-built.ts +820 -224
- package/src/eslint-file-set.ts +9 -0
- package/src/executors/bounded-exec/cpu-pool.test.ts +454 -0
- package/src/executors/bounded-exec/cpu-pool.ts +417 -0
- package/src/executors/bounded-exec/cpu-tokens.test.ts +269 -0
- package/src/executors/bounded-exec/cpu-tokens.ts +177 -0
- package/src/executors/bounded-exec/disk-lease.test.ts +185 -0
- package/src/executors/bounded-exec/disk-lease.ts +139 -0
- package/src/executors/bounded-exec/executor-host.test.ts +778 -0
- package/src/executors/bounded-exec/executor.test.ts +1116 -63
- package/src/executors/bounded-exec/executor.ts +565 -67
- package/src/executors/bounded-exec/gateway-lease.ts +176 -0
- package/src/executors/bounded-exec/hang-diagnostics.test.ts +285 -0
- package/src/executors/bounded-exec/hang-diagnostics.ts +283 -0
- package/src/executors/bounded-exec/progress.test.ts +79 -0
- package/src/executors/bounded-exec/progress.ts +118 -0
- package/src/executors/bounded-exec/ram-temp-fixtures.ts +12 -0
- package/src/executors/bounded-exec/ram-temp-held.test.ts +264 -0
- package/src/executors/bounded-exec/ram-temp-held.ts +242 -0
- package/src/executors/bounded-exec/ram-temp-host.test.ts +605 -0
- package/src/executors/bounded-exec/ram-temp.test.ts +674 -0
- package/src/executors/bounded-exec/ram-temp.ts +1918 -0
- package/src/executors/bounded-exec/schema.json +11 -5
- package/src/executors/bounded-exec/schema.ts +27 -11
- package/src/executors/bounded-exec/verdict.test.ts +440 -0
- package/src/executors/bounded-exec/verdict.ts +433 -0
- package/src/executors/shard-timings/executor.ts +36 -0
- package/src/executors/shard-timings/schema.json +21 -0
- package/src/executors/shard-timings/schema.ts +4 -0
- package/src/failure-summary.ts +119 -0
- package/src/generators/bounded-test-targets/generator.test.ts +4 -4
- package/src/generators/bun-test-tracing/generator.test.ts +2 -2
- package/src/generators/create-package/generator.test.ts +6 -0
- package/src/generators/create-package/generator.ts +6 -2
- package/src/host-controller-wiring.test.ts +31 -0
- package/src/index.test.ts +909 -295
- package/src/index.ts +441 -143
- package/src/linked-snapshots.nx.test.ts +172 -0
- package/src/managed-files/ci-workflow.ts +54 -13
- package/src/managed-files/files.ts +65 -9
- package/src/managed-files/forgejo-artifact.test.ts +395 -0
- package/src/managed-files/generator.test.ts +60 -1
- package/src/managed-files/json-edit.test.ts +216 -0
- package/src/managed-files/json-edit.ts +290 -0
- package/src/managed-files/managed-format-json.test.ts +308 -0
- package/src/managed-files/managed-format.ts +90 -2
- package/src/managed-files/pr-preview-cleanup-workflow.ts +69 -3
- package/src/managed-files/publish-workflow.ts +285 -67
- package/src/managed-files/repo-path.test.ts +129 -0
- package/src/managed-files/trace-db-artifact.test.ts +7 -0
- package/src/managed-files/tree.ts +11 -3
- package/src/manifest-hash.ts +2 -6
- package/src/nextest-code-bound.test.ts +155 -0
- package/src/nextest-code-bound.ts +195 -0
- package/src/nextest-extraction.test.ts +37 -5
- package/src/nextest-extraction.ts +51 -9
- package/src/nextest-partitions.test.ts +427 -82
- package/src/nx-cache-bound.test.ts +5 -10
- package/src/nx-cache-restore-mtime.test.ts +170 -0
- package/src/nx-daemon-claim.test.ts +277 -0
- package/src/nx-daemon-idle-cpu.test.ts +81 -0
- package/src/nx-daemon-idle-policy.test.ts +51 -0
- package/src/nx-daemon-ignore-files.test.ts +210 -0
- package/src/nx-daemon-root-normalization.test.ts +105 -0
- package/src/nx-daemon-validity.test.ts +925 -0
- package/src/nx-dot-input-policy.test.ts +353 -0
- package/src/nx-failure-life-cycles.test.ts +207 -0
- package/src/nx-files-archive-racy.test.ts +97 -0
- package/src/nx-foreign-workspace-env.test.ts +329 -0
- package/src/nx-incomplete-run-summary.test.ts +70 -0
- package/src/nx-plugin-load-errors.test.ts +120 -0
- package/src/nx-project-resolution-cache.test.ts +176 -0
- package/src/nx-raw-inputs.test.ts +166 -0
- package/src/nx-resident-graph-workers.test.ts +7 -16
- package/src/nx-run-information.test.ts +118 -0
- package/src/nx-skipped-task-exit.test.ts +180 -0
- package/src/nx-state-location.test.ts +63 -0
- package/src/nx-store-resolution.test.ts +26 -25
- package/src/nx-task-graph-cycles.test.ts +148 -0
- package/src/nx-task-output-lease.test.ts +414 -0
- package/src/nx-use-local.test.ts +103 -0
- package/src/package-target-policy.ts +3 -6
- package/src/release-config-policy.ts +3 -6
- package/src/runtime-input.test.ts +215 -0
- package/src/runtime-input.ts +17 -0
- package/src/rust-tokens.ts +78 -0
- package/src/serial-group-wiring.test.ts +326 -0
- package/src/task-history-namespace.test.ts +87 -89
- package/src/task-results.test.ts +226 -0
- package/src/task-results.ts +75 -0
- package/src/test-program.test.ts +303 -0
- package/src/test-program.ts +341 -0
- package/src/test-timings.test.ts +240 -0
- package/src/test-timings.ts +311 -0
- package/src/test-walk-policy.test.ts +585 -0
- package/src/test-walk-policy.ts +731 -0
- package/src/testing.test.ts +509 -0
- package/src/testing.ts +922 -0
- package/src/type-guards.ts +4 -0
- package/src/typecheck-test-policy.test.ts +496 -11
- package/src/typecheck-test-policy.ts +137 -103
- package/src/workspace-config-policy.test.ts +212 -13
- package/src/workspace-config-policy.ts +101 -73
- package/src/workspace-index.ts +18 -0
- package/src/workspace-manifest.ts +41 -0
- package/test-timings.json +85 -30
- package/managed/raw/tooling/direnv/inherited-devenv.ts +0 -665
- package/managed/raw/tooling/direnv/shared-caches.sh +0 -53
|
@@ -0,0 +1,778 @@
|
|
|
1
|
+
import { afterEach, describe, expect, it } from 'bun:test';
|
|
2
|
+
import { spawn } from 'node:child_process';
|
|
3
|
+
import { chmod, mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
|
|
4
|
+
import { createServer, type Socket } from 'node:net';
|
|
5
|
+
import { tmpdir } from 'node:os';
|
|
6
|
+
import { basename, dirname, join } from 'node:path';
|
|
7
|
+
import { fileURLToPath } from 'node:url';
|
|
8
|
+
import { CPU_POOL_ENV, type CpuPool, createCpuPool } from './cpu-pool.js';
|
|
9
|
+
import { CPU_TOKENS_ENV } from './cpu-tokens.js';
|
|
10
|
+
import type { BoundedExecOptions } from './schema.js';
|
|
11
|
+
import { VERDICT_FILE } from './verdict.js';
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* The production runner's built Node artifact, run in its own process, with the real RAM volume and CPU-budget owners.
|
|
15
|
+
* `test-host` already depends on this package's build; these peers must not compile TypeScript inside a test's guard.
|
|
16
|
+
* These proofs need the host (`hdiutil` on darwin), so the pure shards exclude this file.
|
|
17
|
+
*/
|
|
18
|
+
const PACKAGE_ROOT = fileURLToPath(new URL('../../..', import.meta.url));
|
|
19
|
+
const EXECUTOR = join(PACKAGE_ROOT, 'dist/executors/bounded-exec/executor.js');
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Names a child never inherits from the process that starts it: budget markers a bounded-exec parent sets for its
|
|
23
|
+
* children, and the fixture socket names `fixtureNxEnv` drops. TMPDIR stays, because each run keeps the RAM volume it
|
|
24
|
+
* was admitted to (`specs/cowshed/08_testing.md` 208-212).
|
|
25
|
+
*/
|
|
26
|
+
const WITHHELD_ENV = new Set([CPU_TOKENS_ENV, CPU_POOL_ENV, 'NX_SOCKET_DIR', 'NX_DAEMON_SOCKET_DIR']);
|
|
27
|
+
|
|
28
|
+
/** One line the sibling proof's barrier heard: a sibling reporting its start or its end with its granted tokens. */
|
|
29
|
+
type SiblingEvent = { readonly kind: 'start' | 'end'; readonly name: string; readonly tokens: number };
|
|
30
|
+
|
|
31
|
+
const EVENT = /^(start|end) ([a-z]+) ([0-9]+)$/;
|
|
32
|
+
|
|
33
|
+
/** The marker a runner's SIGUSR2 handler prints before the Node resources it still has active, as JSON. */
|
|
34
|
+
const ACTIVE_RESOURCES = 'cowshed-test active-resources ';
|
|
35
|
+
|
|
36
|
+
/** Churn workers a foreground proof runs beside: two, each cycling one lease at a time. */
|
|
37
|
+
const CHURN_NAMES = ['churna', 'churnb'] as const;
|
|
38
|
+
/** Written into a case's root to stop its churn workers; each checks it between runs, never mid-run. */
|
|
39
|
+
const CHURN_STOP = 'churn.stop';
|
|
40
|
+
/** One churn run: `true` on one token, so each lease is a real acquire, run and release. */
|
|
41
|
+
const CHURN_OPTIONS: BoundedExecOptions = { command: 'true', timeoutMs: 20_000, parallelism: 1 };
|
|
42
|
+
/** A churn worker's stdout: a `begin` line before each run it starts, a `ran` line after each run that succeeded. */
|
|
43
|
+
const CHURN_BEGIN = /^cowshed-test churn [a-z]+ begin [0-9]+$/gm;
|
|
44
|
+
const CHURN_RAN = /^cowshed-test churn [a-z]+ ran [0-9]+$/gm;
|
|
45
|
+
/** The stderr line a run logs when its one token came from the inherited parent pool. */
|
|
46
|
+
const INHERITED_GRANT = /inherited=true tokens=1\/1/g;
|
|
47
|
+
|
|
48
|
+
interface ChildRun {
|
|
49
|
+
/** The script's file name, so an attribution names the case its child ran. */
|
|
50
|
+
readonly label: string;
|
|
51
|
+
readonly exited: Promise<number | null>;
|
|
52
|
+
stdout(): string;
|
|
53
|
+
stderr(): string;
|
|
54
|
+
/** Whether the child has exited and closed its output streams. */
|
|
55
|
+
closed(): boolean;
|
|
56
|
+
/**
|
|
57
|
+
* Sends SIGUSR2, which a runner answers on stdout with the Node resources it still has active. The answer is in
|
|
58
|
+
* `stdout()` once the child has closed, unless the runner exited before its handler ran.
|
|
59
|
+
*/
|
|
60
|
+
askResources(): void;
|
|
61
|
+
/** Resolves once `marker` has been printed to stdout; rejects if the child exits without printing it. */
|
|
62
|
+
printed(marker: string): Promise<void>;
|
|
63
|
+
/** Stops a surviving runner after its test-owned pools have revoked all borrowed commands. */
|
|
64
|
+
stop(): Promise<void>;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
interface Barrier {
|
|
68
|
+
readonly port: number;
|
|
69
|
+
events(): readonly SiblingEvent[];
|
|
70
|
+
malformed(): readonly string[];
|
|
71
|
+
/**
|
|
72
|
+
* Resolves once `ready` holds for the events heard so far; rejects if the barrier failed or heard a malformed line.
|
|
73
|
+
*/
|
|
74
|
+
until(ready: (events: readonly SiblingEvent[]) => boolean): Promise<void>;
|
|
75
|
+
go(name: string): void;
|
|
76
|
+
close(): Promise<void>;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** What a foreground proof saw of its churn while its body ran. */
|
|
80
|
+
interface Churn<T> {
|
|
81
|
+
readonly value: T;
|
|
82
|
+
/** Lease attempts in flight at body entry or begun while it ran: traffic sharing the ledger with the foreground. */
|
|
83
|
+
readonly begun: number;
|
|
84
|
+
readonly workers: readonly ChildRun[];
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/** Parent pools a case has opened and not yet closed; afterEach closes what a case left open. */
|
|
88
|
+
const pools = new Set<CpuPool>();
|
|
89
|
+
/** Barriers a case has opened and not yet closed; afterEach closes what a case left open. */
|
|
90
|
+
const barriers = new Set<Barrier>();
|
|
91
|
+
const workspaces: string[] = [];
|
|
92
|
+
const children = new Set<ChildRun>();
|
|
93
|
+
|
|
94
|
+
describe('@smoothbricks/nx-plugin:bounded-exec host proofs', () => {
|
|
95
|
+
afterEach(async () => {
|
|
96
|
+
// A child still open here was not awaited by its case: its guard fired, or the case failed before awaiting it. Its
|
|
97
|
+
// cumulative output is written before anything waits, so a hook timeout below cannot lose it. Each live runner is
|
|
98
|
+
// also asked, by SIGUSR2, what Node still has active; the answer lands in its stdout before the child closes.
|
|
99
|
+
const live = [...children].filter((child) => !child.closed());
|
|
100
|
+
for (const child of live) {
|
|
101
|
+
process.stderr.write(attribution(child, 'at guard teardown'));
|
|
102
|
+
child.askResources();
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
// Pools close first: a command still running under a closed pool is stopped as a lost grant, so its runner exits.
|
|
106
|
+
const outcomes = await Promise.all([...pools].map((pool) => pool.close()));
|
|
107
|
+
pools.clear();
|
|
108
|
+
await Promise.all([...children].map((child) => child.stop()));
|
|
109
|
+
for (const child of live) {
|
|
110
|
+
process.stderr.write(attribution(child, 'after teardown'));
|
|
111
|
+
}
|
|
112
|
+
children.clear();
|
|
113
|
+
await Promise.all([...barriers].map((barrier) => barrier.close()));
|
|
114
|
+
await Promise.all(workspaces.splice(0).map((root) => rm(root, { recursive: true, force: true })));
|
|
115
|
+
for (const outcome of outcomes) {
|
|
116
|
+
if (!outcome.ok) {
|
|
117
|
+
throw new Error(`cpu pool did not close: ${outcome.error}`);
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
});
|
|
121
|
+
|
|
122
|
+
it('runs a nested command on its parent sub-pool, granting it no more than the sub-pool holds', async () => {
|
|
123
|
+
const root = await createRoot();
|
|
124
|
+
await withPool(4, async (pool) => {
|
|
125
|
+
const leafDirectory = join(root, 'leaf');
|
|
126
|
+
const leafScript = await writeRunner(
|
|
127
|
+
root,
|
|
128
|
+
'leaf',
|
|
129
|
+
{ command: `node -e "console.log('leaf')"`, timeoutMs: 20_000, parallelism: 1 },
|
|
130
|
+
leafDirectory,
|
|
131
|
+
);
|
|
132
|
+
const nestedDirectory = join(root, 'nested');
|
|
133
|
+
const nestedCommand = `node ${JSON.stringify(leafScript)}`;
|
|
134
|
+
// Churn draws from this pool beside the nested run. It can delay the nested grant, never widen the pool.
|
|
135
|
+
const churn = await withChurn(root, pool.environment, async () => {
|
|
136
|
+
const run = await startRunner(
|
|
137
|
+
root,
|
|
138
|
+
'nested',
|
|
139
|
+
{ command: nestedCommand, cwd: PACKAGE_ROOT, timeoutMs: 20_000, parallelism: 2 },
|
|
140
|
+
nestedDirectory,
|
|
141
|
+
childEnv(pool.environment),
|
|
142
|
+
);
|
|
143
|
+
await run.exited;
|
|
144
|
+
return run;
|
|
145
|
+
});
|
|
146
|
+
const nested = churn.value;
|
|
147
|
+
expect(await nested.exited, nested.stderr()).toBe(0);
|
|
148
|
+
|
|
149
|
+
// The nested run logs its own grant before its command starts; the leaf's grant reaches the same stderr.
|
|
150
|
+
const granted = [...nested.stderr().matchAll(/inherited=true tokens=(\d+)\//g)].map((match) => Number(match[1]));
|
|
151
|
+
expect(granted).toHaveLength(2);
|
|
152
|
+
const [nestedTokens = 0, leafTokens = 0] = granted;
|
|
153
|
+
expect(nestedTokens).toBeGreaterThanOrEqual(1);
|
|
154
|
+
expect(leafTokens).toBeGreaterThanOrEqual(1);
|
|
155
|
+
expect(leafTokens).toBeLessThanOrEqual(nestedTokens);
|
|
156
|
+
|
|
157
|
+
for (const directory of [nestedDirectory, leafDirectory]) {
|
|
158
|
+
expect(await readFile(join(directory, VERDICT_FILE), 'utf8')).toMatch(/"inherited":\s*true/);
|
|
159
|
+
}
|
|
160
|
+
await expectChurn(churn);
|
|
161
|
+
});
|
|
162
|
+
}, 30_000);
|
|
163
|
+
|
|
164
|
+
it('admits five concurrent RAM leases without blocking the lock holder behind filesystem workers', async () => {
|
|
165
|
+
const root = await createRoot();
|
|
166
|
+
await withPool(1, async (pool) => {
|
|
167
|
+
const script = join(root, 'contended.mjs');
|
|
168
|
+
await writeFile(
|
|
169
|
+
script,
|
|
170
|
+
[
|
|
171
|
+
`import { runDefaultBoundedExec } from ${JSON.stringify(EXECUTOR)};`,
|
|
172
|
+
RESOURCES_HANDLER,
|
|
173
|
+
`const options = ${JSON.stringify({
|
|
174
|
+
command: `node -e "console.log('lease')"`,
|
|
175
|
+
cwd: PACKAGE_ROOT,
|
|
176
|
+
timeoutMs: 20_000,
|
|
177
|
+
parallelism: 1,
|
|
178
|
+
})};`,
|
|
179
|
+
`const context = ${JSON.stringify({ root })};`,
|
|
180
|
+
'const results = await Promise.all(Array.from({ length: 5 }, () => runDefaultBoundedExec(options, context)));',
|
|
181
|
+
"console.log('leases-complete');",
|
|
182
|
+
'process.exitCode = results.every((result) => result.success) ? 0 : 1;',
|
|
183
|
+
'',
|
|
184
|
+
].join('\n'),
|
|
185
|
+
);
|
|
186
|
+
// Five requests leave four contended opens beside the holder. Blocking opens consume all
|
|
187
|
+
// four libuv workers, so even the holder's marker read cannot finish. The one-token pool
|
|
188
|
+
// still serializes actual commands: five lease requests are not five runnable commands.
|
|
189
|
+
const run = startChild(script, childEnv({ ...pool.environment, UV_THREADPOOL_SIZE: '4' }));
|
|
190
|
+
expect(await run.exited, run.stderr()).toBe(0);
|
|
191
|
+
expect(run.stdout().match(/^lease$/gm)).toHaveLength(5);
|
|
192
|
+
expect(run.stdout()).toContain('leases-complete');
|
|
193
|
+
expect([...run.stderr().matchAll(/inherited=true tokens=1\/1/g)]).toHaveLength(5);
|
|
194
|
+
});
|
|
195
|
+
}, 30_000);
|
|
196
|
+
|
|
197
|
+
it('holds a parent pool to its count: three siblings on a pool of two never hold more than two', async () => {
|
|
198
|
+
const root = await createRoot();
|
|
199
|
+
await withPool(2, async (pool) => {
|
|
200
|
+
const barrier = await startBarrier();
|
|
201
|
+
try {
|
|
202
|
+
// Churn draws from this pool beside the siblings. It can delay a sibling's admission, never widen the pool, so
|
|
203
|
+
// the two-token peak below is still the pool's own count.
|
|
204
|
+
const churn = await withChurn(root, pool.environment, async () => {
|
|
205
|
+
const siblings = await Promise.all(
|
|
206
|
+
['a', 'b', 'c'].map((name) =>
|
|
207
|
+
startRunner(
|
|
208
|
+
root,
|
|
209
|
+
name,
|
|
210
|
+
{ command: siblingCommand(barrier.port, name), timeoutMs: 20_000, parallelism: 1 },
|
|
211
|
+
join(root, name),
|
|
212
|
+
childEnv(pool.environment),
|
|
213
|
+
),
|
|
214
|
+
),
|
|
215
|
+
);
|
|
216
|
+
|
|
217
|
+
// Two siblings are admitted together and the third waits for a release. Which two is not fixed, so the proof
|
|
218
|
+
// releases whichever started first, then every other started sibling in turn.
|
|
219
|
+
await barrier.until((events) => namesOf(events, 'start').length >= 2);
|
|
220
|
+
const [first = ''] = namesOf(barrier.events(), 'start');
|
|
221
|
+
barrier.go(first);
|
|
222
|
+
await barrier.until((events) => events.some((event) => event.kind === 'end' && event.name === first));
|
|
223
|
+
await barrier.until((events) => namesOf(events, 'start').length >= 3);
|
|
224
|
+
const ended = new Set(namesOf(barrier.events(), 'end'));
|
|
225
|
+
for (const name of namesOf(barrier.events(), 'start')) {
|
|
226
|
+
if (!ended.has(name)) {
|
|
227
|
+
barrier.go(name);
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
await barrier.until((events) => namesOf(events, 'end').length >= 3);
|
|
231
|
+
return siblings;
|
|
232
|
+
});
|
|
233
|
+
for (const sibling of churn.value) {
|
|
234
|
+
expect(await sibling.exited).toBe(0);
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
const events = barrier.events();
|
|
238
|
+
expect(barrier.malformed()).toEqual([]);
|
|
239
|
+
expect(namesOf(events, 'start').sort()).toEqual(['a', 'b', 'c']);
|
|
240
|
+
expect(namesOf(events, 'end').sort()).toEqual(['a', 'b', 'c']);
|
|
241
|
+
|
|
242
|
+
let held = 0;
|
|
243
|
+
let peak = 0;
|
|
244
|
+
for (const event of events) {
|
|
245
|
+
expect(Number.isSafeInteger(event.tokens)).toBe(true);
|
|
246
|
+
expect(event.tokens).toBeGreaterThanOrEqual(1);
|
|
247
|
+
held += event.kind === 'start' ? event.tokens : -event.tokens;
|
|
248
|
+
expect(held).toBeGreaterThanOrEqual(0);
|
|
249
|
+
expect(held).toBeLessThanOrEqual(2);
|
|
250
|
+
peak = Math.max(peak, held);
|
|
251
|
+
}
|
|
252
|
+
expect(peak).toBe(2);
|
|
253
|
+
expect(held).toBe(0);
|
|
254
|
+
await expectChurn(churn);
|
|
255
|
+
} finally {
|
|
256
|
+
await barrier.close();
|
|
257
|
+
}
|
|
258
|
+
});
|
|
259
|
+
}, 30_000);
|
|
260
|
+
|
|
261
|
+
it('uses a real backing budget after its named pool is absent and returns the allocation', async () => {
|
|
262
|
+
const root = await createRoot();
|
|
263
|
+
const stale = await withPool(1, async (pool) => pool.environment);
|
|
264
|
+
await withPool(1, async (outer) => {
|
|
265
|
+
const options: BoundedExecOptions = {
|
|
266
|
+
command: `node -e "console.log('actual-tokens=' + process.env.${CPU_TOKENS_ENV})"`,
|
|
267
|
+
timeoutMs: 20_000,
|
|
268
|
+
parallelism: 1,
|
|
269
|
+
};
|
|
270
|
+
const directory = join(root, 'stale');
|
|
271
|
+
const context = { root, record: { task: { id: 'stale', hash: null }, directory } };
|
|
272
|
+
const script = join(root, 'stale.mjs');
|
|
273
|
+
await writeFile(
|
|
274
|
+
script,
|
|
275
|
+
[
|
|
276
|
+
`import assert from 'node:assert/strict';`,
|
|
277
|
+
`import { availableParallelism } from 'node:os';`,
|
|
278
|
+
`import { runBoundedExec, createProcessTreeKiller } from ${JSON.stringify(EXECUTOR)};`,
|
|
279
|
+
`import { readInheritedCpuPool, takeParentCpuTokens } from ${JSON.stringify(join(dirname(EXECUTOR), 'cpu-pool.js'))};`,
|
|
280
|
+
RESOURCES_HANDLER,
|
|
281
|
+
`const backing = readInheritedCpuPool(${JSON.stringify(outer.environment)});`,
|
|
282
|
+
`const stale = readInheritedCpuPool(${JSON.stringify({ ...stale, [CPU_TOKENS_ENV]: '977' })});`,
|
|
283
|
+
'assert(backing !== null && stale !== null);',
|
|
284
|
+
`const options = ${JSON.stringify(options)};`,
|
|
285
|
+
`const context = ${JSON.stringify(context)};`,
|
|
286
|
+
'const asked = [];',
|
|
287
|
+
'const allocations = [];',
|
|
288
|
+
'let released = 0;',
|
|
289
|
+
'const budget = { async take(want, checkout, command) {',
|
|
290
|
+
' asked.push({ want, checkout, command });',
|
|
291
|
+
' const grant = await takeParentCpuTokens(backing, want, availableParallelism());',
|
|
292
|
+
' assert(grant.granted, grant.reason);',
|
|
293
|
+
' allocations.push(grant);',
|
|
294
|
+
' return { ...grant, release() { released++; grant.release(); } };',
|
|
295
|
+
'} };',
|
|
296
|
+
'const result = await runBoundedExec(options, context, createProcessTreeKiller(), null, budget, stale);',
|
|
297
|
+
'assert(result.success, result.terminalOutput);',
|
|
298
|
+
'assert.deepEqual(asked, [{ want: 1, checkout: context.root, command: options.command }]);',
|
|
299
|
+
'assert.equal(released, 1);',
|
|
300
|
+
'await allocations[0].closed;',
|
|
301
|
+
'const returned = await takeParentCpuTokens(backing, 1, availableParallelism());',
|
|
302
|
+
'assert(returned.granted, returned.reason);',
|
|
303
|
+
'assert.equal(returned.tokens, 1);',
|
|
304
|
+
'returned.release();',
|
|
305
|
+
'await returned.closed;',
|
|
306
|
+
`console.log('actual-allocation-returned');`,
|
|
307
|
+
'',
|
|
308
|
+
].join('\n'),
|
|
309
|
+
);
|
|
310
|
+
const run = startChild(script, childEnv(outer.environment));
|
|
311
|
+
expect(await run.exited, run.stderr()).toBe(0);
|
|
312
|
+
expect(run.stderr()).toContain('cowshed: cpu-tokens parent pool refused: ');
|
|
313
|
+
expect(run.stderr()).toContain('status=ok inherited=true tokens=1/1');
|
|
314
|
+
expect(run.stdout()).toContain('actual-tokens=1');
|
|
315
|
+
expect(run.stdout()).toContain('actual-allocation-returned');
|
|
316
|
+
expect(run.stdout()).not.toContain('977');
|
|
317
|
+
expect(await readFile(join(directory, VERDICT_FILE), 'utf8')).toMatch(/"tokens":\s*1/);
|
|
318
|
+
});
|
|
319
|
+
}, 30_000);
|
|
320
|
+
|
|
321
|
+
it('stops a command whose parent pool closes under it, and fails its run', async () => {
|
|
322
|
+
const root = await createRoot();
|
|
323
|
+
await withPool(2, async (pool) => {
|
|
324
|
+
const run = await startRunner(
|
|
325
|
+
root,
|
|
326
|
+
'orphan',
|
|
327
|
+
{
|
|
328
|
+
command: `node -e "console.log('started'); require('node:http').createServer().listen(0)"`,
|
|
329
|
+
timeoutMs: 30_000,
|
|
330
|
+
killAfterMs: 2_000,
|
|
331
|
+
parallelism: 1,
|
|
332
|
+
},
|
|
333
|
+
join(root, 'orphan'),
|
|
334
|
+
childEnv(pool.environment),
|
|
335
|
+
);
|
|
336
|
+
await run.printed('started');
|
|
337
|
+
const outcome = await pool.close();
|
|
338
|
+
expect(outcome.ok).toBe(true);
|
|
339
|
+
expect(await run.exited, run.stderr()).toBe(1);
|
|
340
|
+
expect(run.stderr()).toContain('cowshed: cpu-tokens grant lost');
|
|
341
|
+
expect(await readFile(join(root, 'orphan', VERDICT_FILE), 'utf8')).toMatch(/"outcome":\s*"failed"/);
|
|
342
|
+
});
|
|
343
|
+
}, 30_000);
|
|
344
|
+
|
|
345
|
+
it('fails a run whose sub-pool does not close cleanly, even when its command exits 0', async () => {
|
|
346
|
+
const root = await createRoot();
|
|
347
|
+
let stuck = '';
|
|
348
|
+
try {
|
|
349
|
+
await withPool(1, async (pool) => {
|
|
350
|
+
const unremovable = await startRunner(
|
|
351
|
+
root,
|
|
352
|
+
'unremovable',
|
|
353
|
+
{ command: UNREMOVABLE_COMMAND, timeoutMs: 20_000, parallelism: 1 },
|
|
354
|
+
join(root, 'unremovable'),
|
|
355
|
+
childEnv(pool.environment),
|
|
356
|
+
);
|
|
357
|
+
expect(await unremovable.exited, unremovable.stderr()).toBe(1);
|
|
358
|
+
stuck = unremovable.stdout().trim();
|
|
359
|
+
expect(basename(dirname(stuck)).startsWith('cpu-pool-')).toBe(true);
|
|
360
|
+
expect(unremovable.stderr()).toContain('cowshed: cpu pool did not close cleanly: ');
|
|
361
|
+
const record: unknown = JSON.parse(await readFile(join(root, 'unremovable', VERDICT_FILE), 'utf8'));
|
|
362
|
+
expect(record).toHaveProperty('verdict', { outcome: 'failed', exitCode: 1, tests: [] });
|
|
363
|
+
|
|
364
|
+
// The failed run gave its one token back, so a sibling on the same one-token pool is admitted.
|
|
365
|
+
const next = await startRunner(
|
|
366
|
+
root,
|
|
367
|
+
'next',
|
|
368
|
+
{ command: `node -e "console.log(process.env.${CPU_TOKENS_ENV})"`, timeoutMs: 20_000, parallelism: 1 },
|
|
369
|
+
join(root, 'next'),
|
|
370
|
+
childEnv(pool.environment),
|
|
371
|
+
);
|
|
372
|
+
expect(await next.exited, next.stderr()).toBe(0);
|
|
373
|
+
expect(next.stderr()).toContain('inherited=true tokens=1/1');
|
|
374
|
+
});
|
|
375
|
+
} finally {
|
|
376
|
+
// The sub-pool's directory is private to it; the pool could not remove `stuck`, so the proof removes it.
|
|
377
|
+
if (stuck !== '' && basename(dirname(stuck)).startsWith('cpu-pool-')) {
|
|
378
|
+
await chmod(stuck, 0o700);
|
|
379
|
+
await rm(dirname(stuck), { recursive: true, force: true });
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
}, 30_000);
|
|
383
|
+
});
|
|
384
|
+
|
|
385
|
+
async function createRoot(): Promise<string> {
|
|
386
|
+
const root = await mkdtemp(join(tmpdir(), 'smoothbricks-bounded-exec-host-'));
|
|
387
|
+
workspaces.push(root);
|
|
388
|
+
return root;
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
/** The environment a child starts with: this process's, minus the withheld names, plus `extra`. */
|
|
392
|
+
function childEnv(extra: Readonly<Record<string, string>>): NodeJS.ProcessEnv {
|
|
393
|
+
const env: NodeJS.ProcessEnv = {};
|
|
394
|
+
for (const [name, value] of Object.entries(process.env)) {
|
|
395
|
+
if (value !== undefined && !WITHHELD_ENV.has(name)) {
|
|
396
|
+
env[name] = value;
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
return { ...env, ...extra };
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
/** A parent's CPU pool of `tokens`, open while `body` runs. Its upstream never closes on its own. */
|
|
403
|
+
async function withPool<T>(tokens: number, body: (pool: CpuPool) => Promise<T>): Promise<T> {
|
|
404
|
+
const created = await createCpuPool(tokens, new Promise<void>(() => {}));
|
|
405
|
+
if (!created.ok) {
|
|
406
|
+
throw new Error(`cpu pool: ${created.error}`);
|
|
407
|
+
}
|
|
408
|
+
pools.add(created.pool);
|
|
409
|
+
let result: { ok: true; value: T } | { ok: false; error: unknown };
|
|
410
|
+
try {
|
|
411
|
+
result = { ok: true, value: await body(created.pool) };
|
|
412
|
+
} catch (error) {
|
|
413
|
+
result = { ok: false, error };
|
|
414
|
+
}
|
|
415
|
+
const outcome = await created.pool.close();
|
|
416
|
+
pools.delete(created.pool);
|
|
417
|
+
if (!outcome.ok) {
|
|
418
|
+
const failure = new Error(`cpu pool did not close: ${outcome.error}`);
|
|
419
|
+
throw result.ok ? failure : new AggregateError([result.error, failure], 'CPU pool body and cleanup failed');
|
|
420
|
+
}
|
|
421
|
+
if (!result.ok) throw result.error;
|
|
422
|
+
return result.value;
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
/** Installed in each runner first: SIGUSR2 makes it report the Node resources it still has active. Reads only. */
|
|
426
|
+
const RESOURCES_HANDLER = [
|
|
427
|
+
"process.on('SIGUSR2', () => {",
|
|
428
|
+
' const resources = JSON.stringify(process.getActiveResourcesInfo());',
|
|
429
|
+
` process.stdout.write(${JSON.stringify(ACTIVE_RESOURCES)} + resources + '\\n');`,
|
|
430
|
+
'});',
|
|
431
|
+
].join('\n');
|
|
432
|
+
|
|
433
|
+
/** Writes a script that runs `options` through the production runner, recording its verdict in `directory`. */
|
|
434
|
+
async function writeRunner(
|
|
435
|
+
root: string,
|
|
436
|
+
name: string,
|
|
437
|
+
options: BoundedExecOptions,
|
|
438
|
+
directory: string,
|
|
439
|
+
): Promise<string> {
|
|
440
|
+
await mkdir(directory, { recursive: true });
|
|
441
|
+
const context = { root, record: { task: { id: name, hash: null }, directory } };
|
|
442
|
+
const script = join(root, `${name}.mjs`);
|
|
443
|
+
await writeFile(
|
|
444
|
+
script,
|
|
445
|
+
[
|
|
446
|
+
`import { runDefaultBoundedExec } from ${JSON.stringify(EXECUTOR)};`,
|
|
447
|
+
RESOURCES_HANDLER,
|
|
448
|
+
`const result = await runDefaultBoundedExec(${JSON.stringify(options)}, ${JSON.stringify(context)});`,
|
|
449
|
+
'process.exitCode = result.success ? 0 : 1;',
|
|
450
|
+
'',
|
|
451
|
+
].join('\n'),
|
|
452
|
+
);
|
|
453
|
+
return script;
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
async function startRunner(
|
|
457
|
+
root: string,
|
|
458
|
+
name: string,
|
|
459
|
+
options: BoundedExecOptions,
|
|
460
|
+
directory: string,
|
|
461
|
+
env: NodeJS.ProcessEnv,
|
|
462
|
+
): Promise<ChildRun> {
|
|
463
|
+
return startChild(await writeRunner(root, name, options, directory), env);
|
|
464
|
+
}
|
|
465
|
+
|
|
466
|
+
/**
|
|
467
|
+
* Starts `script` with Node against the exact compiled artifact Nx's executor registration names.
|
|
468
|
+
* Stdout and stderr are captured; `exited` resolves once the child has closed both streams.
|
|
469
|
+
*/
|
|
470
|
+
function startChild(script: string, env: NodeJS.ProcessEnv): ChildRun {
|
|
471
|
+
const child = spawn('node', [script], {
|
|
472
|
+
cwd: PACKAGE_ROOT,
|
|
473
|
+
env,
|
|
474
|
+
stdio: ['ignore', 'pipe', 'pipe'],
|
|
475
|
+
});
|
|
476
|
+
let out = '';
|
|
477
|
+
let err = '';
|
|
478
|
+
let closed = false;
|
|
479
|
+
const waiters = new Set<() => void>();
|
|
480
|
+
const wake = (): void => {
|
|
481
|
+
for (const waiter of [...waiters]) {
|
|
482
|
+
waiter();
|
|
483
|
+
}
|
|
484
|
+
};
|
|
485
|
+
child.stdout.on('data', (chunk: Buffer) => {
|
|
486
|
+
out += chunk.toString('utf8');
|
|
487
|
+
wake();
|
|
488
|
+
});
|
|
489
|
+
child.stderr.on('data', (chunk: Buffer) => {
|
|
490
|
+
err += chunk.toString('utf8');
|
|
491
|
+
});
|
|
492
|
+
const exited = new Promise<number | null>((resolve) => {
|
|
493
|
+
const done = (code: number | null): void => {
|
|
494
|
+
closed = true;
|
|
495
|
+
resolve(code);
|
|
496
|
+
wake();
|
|
497
|
+
};
|
|
498
|
+
child.once('error', (error) => {
|
|
499
|
+
err += `${error.message}\n`;
|
|
500
|
+
done(null);
|
|
501
|
+
});
|
|
502
|
+
child.once('close', (code) => done(code));
|
|
503
|
+
});
|
|
504
|
+
const run: ChildRun = {
|
|
505
|
+
label: basename(script),
|
|
506
|
+
exited,
|
|
507
|
+
stdout: () => out,
|
|
508
|
+
stderr: () => err,
|
|
509
|
+
closed: () => closed,
|
|
510
|
+
askResources(): void {
|
|
511
|
+
child.kill('SIGUSR2');
|
|
512
|
+
},
|
|
513
|
+
printed(marker: string): Promise<void> {
|
|
514
|
+
return new Promise<void>((resolve, reject) => {
|
|
515
|
+
const check = (): void => {
|
|
516
|
+
if (out.includes(marker)) {
|
|
517
|
+
waiters.delete(check);
|
|
518
|
+
resolve();
|
|
519
|
+
} else if (closed) {
|
|
520
|
+
waiters.delete(check);
|
|
521
|
+
reject(new Error(`child exited before printing ${JSON.stringify(marker)}; stderr:\n${err}`));
|
|
522
|
+
}
|
|
523
|
+
};
|
|
524
|
+
waiters.add(check);
|
|
525
|
+
check();
|
|
526
|
+
});
|
|
527
|
+
},
|
|
528
|
+
async stop(): Promise<void> {
|
|
529
|
+
if (!closed) child.kill('SIGTERM');
|
|
530
|
+
await exited;
|
|
531
|
+
},
|
|
532
|
+
};
|
|
533
|
+
children.add(run);
|
|
534
|
+
return run;
|
|
535
|
+
}
|
|
536
|
+
|
|
537
|
+
/**
|
|
538
|
+
* What a runner has printed and written so far, named by its script and the stage it is attributed at. Cumulative and
|
|
539
|
+
* untrimmed: a guard flap says what the runner was doing, not just that it was late.
|
|
540
|
+
*/
|
|
541
|
+
function attribution(child: ChildRun, stage: string): string {
|
|
542
|
+
return [
|
|
543
|
+
`cowshed host proof ${stage}: ${child.label} ${child.closed() ? 'closed' : 'still open'}`,
|
|
544
|
+
'--- stdout ---',
|
|
545
|
+
child.stdout(),
|
|
546
|
+
'--- stderr ---',
|
|
547
|
+
child.stderr(),
|
|
548
|
+
`--- end ${child.label} ---`,
|
|
549
|
+
'',
|
|
550
|
+
].join('\n');
|
|
551
|
+
}
|
|
552
|
+
|
|
553
|
+
/**
|
|
554
|
+
* The sibling's command: report `start NAME TOKENS` to the barrier, wait for `go`, then report `end NAME TOKENS`.
|
|
555
|
+
* It is plain `node`, so it needs no preload.
|
|
556
|
+
*/
|
|
557
|
+
function siblingCommand(port: number, name: string): string {
|
|
558
|
+
const tokens = `process.env.${CPU_TOKENS_ENV}`;
|
|
559
|
+
const script = [
|
|
560
|
+
`const s = require('node:net').connect(${port}, '127.0.0.1', () =>`,
|
|
561
|
+
`s.write('start ${name} ' + ${tokens} + '\\n'));`,
|
|
562
|
+
`let got = ''; s.on('data', (d) => { got += d;`,
|
|
563
|
+
`if (got.includes('go\\n')) s.end('end ${name} ' + ${tokens} + '\\n'); });`,
|
|
564
|
+
].join(' ');
|
|
565
|
+
return `node -e ${JSON.stringify(script)}`;
|
|
566
|
+
}
|
|
567
|
+
|
|
568
|
+
/**
|
|
569
|
+
* Writes one churn worker: a loop that runs `CHURN_OPTIONS` through the production runner, one lease after another,
|
|
570
|
+
* until `stopFile` exists. It prints a `begin` line before each run and a `ran` line after each success, so the parent
|
|
571
|
+
* reads its progress from stdout, with no socket and no clock. A failed run exits the worker with the run's output.
|
|
572
|
+
*/
|
|
573
|
+
async function writeChurnWorker(root: string, name: string, stopFile: string): Promise<string> {
|
|
574
|
+
const script = join(root, `${name}.mjs`);
|
|
575
|
+
await writeFile(
|
|
576
|
+
script,
|
|
577
|
+
[
|
|
578
|
+
"import { existsSync } from 'node:fs';",
|
|
579
|
+
`import { runDefaultBoundedExec } from ${JSON.stringify(EXECUTOR)};`,
|
|
580
|
+
RESOURCES_HANDLER,
|
|
581
|
+
`const options = ${JSON.stringify(CHURN_OPTIONS)};`,
|
|
582
|
+
`const context = ${JSON.stringify({ root })};`,
|
|
583
|
+
`for (let run = 1; !existsSync(${JSON.stringify(stopFile)}); run += 1) {`,
|
|
584
|
+
` process.stdout.write('cowshed-test churn ${name} begin ' + run + '\\n');`,
|
|
585
|
+
' const result = await runDefaultBoundedExec(options, context);',
|
|
586
|
+
' if (!result.success) {',
|
|
587
|
+
` process.stderr.write('cowshed host proof churn ${name} failed at run ' + run + ':\\n');`,
|
|
588
|
+
' process.stderr.write(result.terminalOutput);',
|
|
589
|
+
' process.exitCode = 1;',
|
|
590
|
+
' break;',
|
|
591
|
+
' }',
|
|
592
|
+
` process.stdout.write('cowshed-test churn ${name} ran ' + run + '\\n');`,
|
|
593
|
+
'}',
|
|
594
|
+
'',
|
|
595
|
+
].join('\n'),
|
|
596
|
+
);
|
|
597
|
+
return script;
|
|
598
|
+
}
|
|
599
|
+
|
|
600
|
+
/**
|
|
601
|
+
* Runs `body` beside two churn workers that draw from `poolEnvironment`'s parent pool, so churn competes for the same
|
|
602
|
+
* tokens and the same RAM ledger as the foreground. `body` starts only after each worker has completed a run. The
|
|
603
|
+
* workers are stopped and awaited before this returns, whatever `body` did, so no assertion sees a live churn child.
|
|
604
|
+
*/
|
|
605
|
+
async function withChurn<T>(
|
|
606
|
+
root: string,
|
|
607
|
+
poolEnvironment: Readonly<Record<string, string>>,
|
|
608
|
+
body: () => Promise<T>,
|
|
609
|
+
): Promise<Churn<T>> {
|
|
610
|
+
const stopFile = join(root, CHURN_STOP);
|
|
611
|
+
const workers: { readonly name: string; readonly run: ChildRun }[] = [];
|
|
612
|
+
try {
|
|
613
|
+
for (const name of CHURN_NAMES) {
|
|
614
|
+
const script = await writeChurnWorker(root, name, stopFile);
|
|
615
|
+
workers.push({ name, run: startChild(script, childEnv(poolEnvironment)) });
|
|
616
|
+
}
|
|
617
|
+
for (const { name, run } of workers) {
|
|
618
|
+
await run.printed(`cowshed-test churn ${name} ran 1`);
|
|
619
|
+
}
|
|
620
|
+
const marks = workers.map(({ run }) => {
|
|
621
|
+
const output = run.stdout();
|
|
622
|
+
const inFlight = [...output.matchAll(CHURN_BEGIN)].length - [...output.matchAll(CHURN_RAN)].length;
|
|
623
|
+
return { run, mark: output.length, inFlight };
|
|
624
|
+
});
|
|
625
|
+
const value = await body();
|
|
626
|
+
const begun = marks.reduce(
|
|
627
|
+
(sum, { run, mark, inFlight }) => sum + inFlight + [...run.stdout().slice(mark).matchAll(CHURN_BEGIN)].length,
|
|
628
|
+
0,
|
|
629
|
+
);
|
|
630
|
+
return { value, begun, workers: workers.map(({ run }) => run) };
|
|
631
|
+
} finally {
|
|
632
|
+
await writeFile(stopFile, '');
|
|
633
|
+
await Promise.all(workers.map(({ run }) => run.exited));
|
|
634
|
+
}
|
|
635
|
+
}
|
|
636
|
+
|
|
637
|
+
/**
|
|
638
|
+
* Asserts, after `withChurn` has returned, that the churn ran on the parent pool and stopped cleanly. Each worker exited
|
|
639
|
+
* 0 and completed a run, and logged an inherited grant for every run it completed: no churn run took tokens the fixture
|
|
640
|
+
* pool did not hold. At least one lease was in flight at body entry or began while the foreground body was running.
|
|
641
|
+
*/
|
|
642
|
+
async function expectChurn(churn: Churn<unknown>): Promise<void> {
|
|
643
|
+
const report = churn.workers.map((worker) => attribution(worker, 'after the churn stopped')).join('\n');
|
|
644
|
+
expect(churn.begun, report).toBeGreaterThanOrEqual(1);
|
|
645
|
+
for (const worker of churn.workers) {
|
|
646
|
+
expect(await worker.exited, report).toBe(0);
|
|
647
|
+
const ran = [...worker.stdout().matchAll(CHURN_RAN)];
|
|
648
|
+
expect(ran.length, report).toBeGreaterThanOrEqual(1);
|
|
649
|
+
expect([...worker.stderr().matchAll(INHERITED_GRANT)], report).toHaveLength(ran.length);
|
|
650
|
+
}
|
|
651
|
+
}
|
|
652
|
+
|
|
653
|
+
/**
|
|
654
|
+
* A command that makes its sub-pool's directory unremovable: it creates `stuck` under the pool's private directory,
|
|
655
|
+
* puts a file in it, locks it (mode 000), and exits 0. The pool's cleanup cannot enter `stuck`, so its close fails.
|
|
656
|
+
*/
|
|
657
|
+
const UNREMOVABLE_SCRIPT = [
|
|
658
|
+
"const fs = require('node:fs');",
|
|
659
|
+
"const path = require('node:path');",
|
|
660
|
+
"const stuck = path.join(path.dirname(process.env.BOUNDED_EXEC_CPU_POOL), 'stuck');",
|
|
661
|
+
"fs.mkdirSync(stuck); fs.writeFileSync(path.join(stuck, 'held'), ''); fs.chmodSync(stuck, 0o000);",
|
|
662
|
+
'console.log(stuck);',
|
|
663
|
+
].join(' ');
|
|
664
|
+
const UNREMOVABLE_COMMAND = `node -e ${JSON.stringify(UNREMOVABLE_SCRIPT)}`;
|
|
665
|
+
|
|
666
|
+
/** The names of the sibling events of `kind`, in the order the barrier heard them. */
|
|
667
|
+
function namesOf(events: readonly SiblingEvent[], kind: SiblingEvent['kind']): string[] {
|
|
668
|
+
return events.filter((event) => event.kind === kind).map((event) => event.name);
|
|
669
|
+
}
|
|
670
|
+
|
|
671
|
+
/**
|
|
672
|
+
* A TCP barrier the sibling proof drives. It stamps each sibling event in the order it hears it, so the proof never
|
|
673
|
+
* needs a clock: a sibling's `start` and `end` are the only ordering it trusts. A connection error, a server error, or
|
|
674
|
+
* a malformed line fails the barrier, and every pending `until` rejects with that failure.
|
|
675
|
+
*/
|
|
676
|
+
function startBarrier(): Promise<Barrier> {
|
|
677
|
+
const events: SiblingEvent[] = [];
|
|
678
|
+
const malformed: string[] = [];
|
|
679
|
+
const sockets = new Map<string, Socket>();
|
|
680
|
+
const connections = new Set<Socket>();
|
|
681
|
+
const waiters = new Set<() => void>();
|
|
682
|
+
let failure: Error | null = null;
|
|
683
|
+
|
|
684
|
+
const wake = (): void => {
|
|
685
|
+
for (const waiter of [...waiters]) {
|
|
686
|
+
waiter();
|
|
687
|
+
}
|
|
688
|
+
};
|
|
689
|
+
const fail = (error: Error): void => {
|
|
690
|
+
failure ??= error;
|
|
691
|
+
wake();
|
|
692
|
+
};
|
|
693
|
+
const record = (connection: Socket, line: string): void => {
|
|
694
|
+
const match = EVENT.exec(line);
|
|
695
|
+
if (match === null) {
|
|
696
|
+
malformed.push(line);
|
|
697
|
+
fail(new Error(`barrier heard a malformed line: ${JSON.stringify(line)}`));
|
|
698
|
+
return;
|
|
699
|
+
}
|
|
700
|
+
const [, kind = '', name = '', tokens = ''] = match;
|
|
701
|
+
if (kind === 'start') {
|
|
702
|
+
sockets.set(name, connection);
|
|
703
|
+
}
|
|
704
|
+
events.push({ kind: kind === 'start' ? 'start' : 'end', name, tokens: Number(tokens) });
|
|
705
|
+
wake();
|
|
706
|
+
};
|
|
707
|
+
|
|
708
|
+
const server = createServer((connection) => {
|
|
709
|
+
connections.add(connection);
|
|
710
|
+
connection.on('close', () => {
|
|
711
|
+
connections.delete(connection);
|
|
712
|
+
});
|
|
713
|
+
connection.on('error', (error) => {
|
|
714
|
+
fail(new Error(`barrier connection failed: ${error.message}`));
|
|
715
|
+
});
|
|
716
|
+
connection.setEncoding('utf8');
|
|
717
|
+
let pending = '';
|
|
718
|
+
connection.on('data', (chunk: string) => {
|
|
719
|
+
const lines = (pending + chunk).split('\n');
|
|
720
|
+
pending = lines.pop() ?? '';
|
|
721
|
+
for (const line of lines) {
|
|
722
|
+
record(connection, line);
|
|
723
|
+
}
|
|
724
|
+
});
|
|
725
|
+
});
|
|
726
|
+
server.on('error', (error) => {
|
|
727
|
+
fail(new Error(`barrier server failed: ${error.message}`));
|
|
728
|
+
});
|
|
729
|
+
|
|
730
|
+
return new Promise<Barrier>((resolve, reject) => {
|
|
731
|
+
server.once('error', reject);
|
|
732
|
+
server.listen(0, '127.0.0.1', () => {
|
|
733
|
+
const address = server.address();
|
|
734
|
+
if (address === null || typeof address === 'string') {
|
|
735
|
+
reject(new Error('the barrier is not listening on a TCP port'));
|
|
736
|
+
return;
|
|
737
|
+
}
|
|
738
|
+
const barrier: Barrier = {
|
|
739
|
+
port: address.port,
|
|
740
|
+
events: () => events,
|
|
741
|
+
malformed: () => malformed,
|
|
742
|
+
until(ready) {
|
|
743
|
+
return new Promise<void>((done, rejectUntil) => {
|
|
744
|
+
const check = (): void => {
|
|
745
|
+
if (failure !== null) {
|
|
746
|
+
waiters.delete(check);
|
|
747
|
+
rejectUntil(failure);
|
|
748
|
+
} else if (ready(events)) {
|
|
749
|
+
waiters.delete(check);
|
|
750
|
+
done();
|
|
751
|
+
}
|
|
752
|
+
};
|
|
753
|
+
waiters.add(check);
|
|
754
|
+
check();
|
|
755
|
+
});
|
|
756
|
+
},
|
|
757
|
+
go(name) {
|
|
758
|
+
const socket = sockets.get(name);
|
|
759
|
+
if (socket === undefined) {
|
|
760
|
+
throw new Error(`sibling ${JSON.stringify(name)} has not reported its start`);
|
|
761
|
+
}
|
|
762
|
+
socket.write('go\n');
|
|
763
|
+
},
|
|
764
|
+
close() {
|
|
765
|
+
barriers.delete(barrier);
|
|
766
|
+
for (const connection of connections) {
|
|
767
|
+
connection.destroy();
|
|
768
|
+
}
|
|
769
|
+
return new Promise<void>((done, rejectClose) => {
|
|
770
|
+
server.close((error) => (error === undefined ? done() : rejectClose(error)));
|
|
771
|
+
});
|
|
772
|
+
},
|
|
773
|
+
};
|
|
774
|
+
barriers.add(barrier);
|
|
775
|
+
resolve(barrier);
|
|
776
|
+
});
|
|
777
|
+
});
|
|
778
|
+
}
|