@smoothbricks/nx-plugin 0.7.10 → 0.7.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -21
- package/dist/.build-inputs.json +9 -9
- package/dist/executors/bounded-exec/cpu-pool-thread.d.ts +5 -4
- package/dist/executors/bounded-exec/cpu-pool-thread.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool.d.ts +25 -10
- package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool.js +86 -23
- package/dist/executors/bounded-exec/executor.d.ts +5 -0
- package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
- package/dist/executors/bounded-exec/executor.js +44 -5
- package/dist/executors/bounded-exec/host-memory.d.ts +22 -0
- package/dist/executors/bounded-exec/host-memory.d.ts.map +1 -1
- package/dist/executors/bounded-exec/host-memory.darwin.d.ts.map +1 -1
- package/dist/executors/bounded-exec/host-memory.darwin.js +7 -1
- package/dist/executors/bounded-exec/host-memory.js +6 -0
- package/dist/executors/bounded-exec/host-memory.linux.d.ts +3 -1
- package/dist/executors/bounded-exec/host-memory.linux.d.ts.map +1 -1
- package/dist/executors/bounded-exec/host-memory.linux.js +43 -1
- package/dist/executors/bounded-exec/pool-admission.d.ts +31 -7
- package/dist/executors/bounded-exec/pool-admission.d.ts.map +1 -1
- package/dist/executors/bounded-exec/pool-admission.js +44 -8
- package/dist/executors/bounded-exec/run-cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/run-cpu-pool.js +1 -1
- package/dist/executors/bounded-exec/verdict.d.ts +4 -2
- package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
- package/dist/executors/bounded-exec/verdict.js +19 -2
- package/managed/raw/patches/nx@24.0.3.patch +16 -0
- package/managed/raw/patches/ttsc@0.34.1.patch +189 -2
- package/package.json +1 -1
- package/src/executors/bounded-exec/cpu-pool-thread.ts +5 -4
- package/src/executors/bounded-exec/cpu-pool.ts +121 -25
- package/src/executors/bounded-exec/executor.ts +71 -5
- package/src/executors/bounded-exec/host-memory.darwin.ts +7 -1
- package/src/executors/bounded-exec/host-memory.linux.ts +42 -2
- package/src/executors/bounded-exec/host-memory.ts +25 -0
- package/src/executors/bounded-exec/pool-admission.ts +66 -11
- package/src/executors/bounded-exec/run-cpu-pool.ts +3 -3
- package/src/executors/bounded-exec/verdict.ts +25 -5
- package/test-timings.json +2 -0
package/README.md
CHANGED
|
@@ -1160,26 +1160,27 @@ Concrete targets come from concrete files:
|
|
|
1160
1160
|
produced. The emit sets `TTSC_WRITE_IF_CHANGED=1`, with which ttsc writes an output only when its bytes differ (the
|
|
1161
1161
|
managed ttsc patch), so an emit after one source edit leaves every output whose bytes did not change as it was, inode
|
|
1162
1162
|
and mtime included: rewriting identical bytes is file churn that every copy-on-write clone of the checkout pays for.
|
|
1163
|
-
The emit runs with `--verbose`, whose summary lists every file the emit produced
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
record from rustc's dep-info
|
|
1175
|
-
Linux's `cargo-zigbuild` executor
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1163
|
+
The emit runs with `--verbose`, whose summary lists every file the emit produced, and every file a linked plugin
|
|
1164
|
+
reported writing beside them (ttsc's `PluginContext.ReportHostOutput`), on each of ttsc's lanes (its own host, the
|
|
1165
|
+
direct-tsgo lane, the utility-plugin host and typia's), and after a successful emit the executor removes each file
|
|
1166
|
+
under `outDir` that the list does not name, other than its own `.build-inputs.json`, and each directory that leaves
|
|
1167
|
+
empty. A file an earlier build wrote and this one no longer does (a bin moved to another package, a deleted module) is
|
|
1168
|
+
gone before Nx captures the output, so no cache artifact carries it. A summary that lists no files, or fewer or more
|
|
1169
|
+
than it counts, fails the emit rather than read as one that produced nothing, and a list that names a file the emit
|
|
1170
|
+
did not leave prunes nothing and fails the emit, so Nx caches no output it emptied. Execute bits are set only on a bin
|
|
1171
|
+
that lacks them, since a chmod moves the file's ctime even when its mode stays. A failed emit removes nothing, and an
|
|
1172
|
+
`outDir` that holds a TypeScript source is refused before the compiler runs. It writes `.build-inputs.json` into the
|
|
1173
|
+
`outDir`, only when the record changed: the SHA-256 of every file the config's `include` matches less its `exclude`,
|
|
1174
|
+
taken before the emit, and those globs as Git pathspecs. Native targets write the same record from rustc's dep-info
|
|
1175
|
+
through `@smoothbricks/build-inputs`: macOS uses its `smoo-nx-build-inputs` CLI and Linux's `cargo-zigbuild` executor
|
|
1176
|
+
calls that same collector. A record produced under Nx also carries its producer's project, target and optional
|
|
1177
|
+
configuration, taken from the actual task environment. Shell entry uses that identity to judge the selected
|
|
1178
|
+
development receipts, including inferred targets absent from package manifests, and name their actual cached rebuild.
|
|
1179
|
+
It constructs no Nx graph, shadow output registry or second interpolation policy. The record is inside the target's
|
|
1180
|
+
outputs, so a cache hit restores it with them; a live developer link (`@smoothbricks/cli`, "Declared Links") checks it
|
|
1181
|
+
to tell a current build from a stale one by content. The shell entry that bootstraps the plugin
|
|
1182
|
+
(`tooling/direnv/enter-shell.ts`) keeps its build marker under `node_modules/.cache`, not under `outDir`, where the
|
|
1183
|
+
executor would remove it as a file its emit did not write.
|
|
1183
1184
|
- `scriptc-<arch>-<family>` (`scriptc-arm64-macos`, `scriptc-x64-macos`, `scriptc-arm64-linux`, `scriptc-x64-linux`) are
|
|
1184
1185
|
inferred for a package that lists `scriptc` in `devDependencies` (that pin is the compiler the targets run; the
|
|
1185
1186
|
workspace root lists `scriptc` in `trustedDependencies` so its install links the platform compiler) together with each
|
|
@@ -2429,7 +2430,15 @@ the run's memory: the least finite `memory.max` of the run's cgroup and its ance
|
|
|
2429
2430
|
Linux, `hw.memsize` on macOS (`host-memory.ts`). A run whose memory cannot be read fails in `preTasksExecution`, naming
|
|
2430
2431
|
the file or sysctl. A task whose own peak exceeds the run's memory is granted once nothing else holds memory, and its
|
|
2431
2432
|
stderr says so; the wait line names the charge as `memory=<bytes>` or `memory=none`. A granted executor's sub-pool
|
|
2432
|
-
charges no memory.
|
|
2433
|
+
charges no memory. The charge follows the parallelism the task is lent: its record's `cpu.tokens` is the parallelism its
|
|
2434
|
+
peak was measured under, and the pool charges the peak in proportion to the tokens it lends when that is more
|
|
2435
|
+
(`ceil(peak × tokens ÷ measured)`, never less than the peak). A task whose whole ask does not fit is lent the most
|
|
2436
|
+
tokens that do, and its wait line names the bytes charged (`memory=<peak> charged=<bytes>`) when they differ from the
|
|
2437
|
+
peak. Each grant is said on the pool's stderr with the task, its charge, what the holders are charged of the run's
|
|
2438
|
+
memory and what the run's bound holds live (anonymous plus shared memory). A command that died of a SIGKILL this
|
|
2439
|
+
executor did not send, with no bound of its own ended, was killed by the kernel for memory: it runs once more, alone in
|
|
2440
|
+
its run's pool (`"alone":true` in the request), which grants it only while no leaf holds a grant and nothing beside it,
|
|
2441
|
+
and says so. A host CI scope's first run of a task it holds no record of is the `memory=none` case: the managed
|
|
2433
2442
|
`setup-devenv` action starts a scope from the records of the pushes it follows (a pull request's base branch's, a
|
|
2434
2443
|
release workflow's CI's) and takes, at each run, the record of every task the scope lacks
|
|
2435
2444
|
(`specs/cowshed/05_gateway.md`, "Without mbx, every Nx run budgets itself").
|
package/dist/.build-inputs.json
CHANGED
|
@@ -35,15 +35,15 @@
|
|
|
35
35
|
"packages/nx-plugin/src/executors/all-verdicts/executor.ts": "4d3aac65aa2e03a2d24d149c412aa18f2e66a6a36a4e88993122b10e9355c8f0",
|
|
36
36
|
"packages/nx-plugin/src/executors/all-verdicts/schema.ts": "2ca6bd8d2f19d851f291ab6a5227167399b9dc8ed0fda0248b69896d77e471f1",
|
|
37
37
|
"packages/nx-plugin/src/executors/bounded-exec/bounds.ts": "fecc48ea66df1c2ba526dc0322ab7760cfc1a4e85d99a72a7a43f040448970f0",
|
|
38
|
-
"packages/nx-plugin/src/executors/bounded-exec/cpu-pool-thread.ts": "
|
|
39
|
-
"packages/nx-plugin/src/executors/bounded-exec/cpu-pool.ts": "
|
|
38
|
+
"packages/nx-plugin/src/executors/bounded-exec/cpu-pool-thread.ts": "47697a9531df5fd6fcdf4e706fc7e5874bca1717c838496673bf148c691f1e9e",
|
|
39
|
+
"packages/nx-plugin/src/executors/bounded-exec/cpu-pool.ts": "0d7972fb3a6bf3557dddf4b67e50c80c1d37302baecebcf4a078130d8979a803",
|
|
40
40
|
"packages/nx-plugin/src/executors/bounded-exec/disk-lease.ts": "1a3f6a6fa590d9a2550490bb6906dc91b6c05fbef235b025eaf257eb5310e92d",
|
|
41
|
-
"packages/nx-plugin/src/executors/bounded-exec/executor.ts": "
|
|
41
|
+
"packages/nx-plugin/src/executors/bounded-exec/executor.ts": "2e0ae5b31e9863bfd8d27010dbaefcf17997c0d795d031b4a232f20129b2268f",
|
|
42
42
|
"packages/nx-plugin/src/executors/bounded-exec/gateway-lease.ts": "f3f73bd92db30e4857f98fb8bd2b9a75b2a5a2a318f094e075ed803c1f9a0117",
|
|
43
43
|
"packages/nx-plugin/src/executors/bounded-exec/hang-diagnostics.ts": "bcaa9e52384f6c481a9cc2835ba6e8ffe167afa3cb51623fdf8dc3493876f335",
|
|
44
|
-
"packages/nx-plugin/src/executors/bounded-exec/host-memory.darwin.ts": "
|
|
45
|
-
"packages/nx-plugin/src/executors/bounded-exec/host-memory.linux.ts": "
|
|
46
|
-
"packages/nx-plugin/src/executors/bounded-exec/host-memory.ts": "
|
|
44
|
+
"packages/nx-plugin/src/executors/bounded-exec/host-memory.darwin.ts": "ce1c12e3f8ebe06034d2e9ecd332f907c567fb2d0a71f3c1c5acccb687488fc8",
|
|
45
|
+
"packages/nx-plugin/src/executors/bounded-exec/host-memory.linux.ts": "465171b065355df13af73fad9c8101d48f6bb060cf41cb50d8dd272abf693b08",
|
|
46
|
+
"packages/nx-plugin/src/executors/bounded-exec/host-memory.ts": "6d02f33e950c89162deb9131bd3c0819ab19bf54dd600da557d7869491eaf99a",
|
|
47
47
|
"packages/nx-plugin/src/executors/bounded-exec/leaf-peak.ts": "0b1b94992b2cbb9e3759785424ceb202667053c5571edf3a79cdd0b206e54a1d",
|
|
48
48
|
"packages/nx-plugin/src/executors/bounded-exec/lease-wait.ts": "59d8ac94d0b8b75858971c5bb28ee421fca4f7685577274fe8bbaf0c01ba462a",
|
|
49
49
|
"packages/nx-plugin/src/executors/bounded-exec/lock-wait.ts": "b14a23b49759a777f4b3338ced003adaf76837c2c38462458f76a7fef1b95d9e",
|
|
@@ -53,14 +53,14 @@
|
|
|
53
53
|
"packages/nx-plugin/src/executors/bounded-exec/no-space.ts": "ec032968d89f9e853796012705a06c06cfc4da54bf6e6f10a526f98daca3d077",
|
|
54
54
|
"packages/nx-plugin/src/executors/bounded-exec/owned-subtree.ts": "cb5edf2b5fe2c1b5b961f8c0ff16ec7bfae3c22ad75dfd671b74f1a4a865863f",
|
|
55
55
|
"packages/nx-plugin/src/executors/bounded-exec/peak-waiter.ts": "3d9a60f8e61d160cf4151746c160f836954f57aca896127a5ef9df4d09c26180",
|
|
56
|
-
"packages/nx-plugin/src/executors/bounded-exec/pool-admission.ts": "
|
|
56
|
+
"packages/nx-plugin/src/executors/bounded-exec/pool-admission.ts": "0cc84dbda8d2cf472a0ee9ec14ab136a219ab2ed223e9769ef45590efbb4b793",
|
|
57
57
|
"packages/nx-plugin/src/executors/bounded-exec/progress.ts": "ed926f7b4991ebf7a7c8876ab0ea00295934d058c774e3f811963a85d209979b",
|
|
58
58
|
"packages/nx-plugin/src/executors/bounded-exec/ram-attach.ts": "865eb5c277d9ab4b258013030938aeee762b731a7298fa8ad3c5eba3026df77a",
|
|
59
59
|
"packages/nx-plugin/src/executors/bounded-exec/ram-detach.ts": "535d26114d14d0d94db76227ea7ba52b1ae0b590d7dbac22993b6796d255e666",
|
|
60
60
|
"packages/nx-plugin/src/executors/bounded-exec/ram-temp-fixtures.ts": "075da4ecfd1cf7dd98fc8c894599dcf2720faba07ac30b8e5592964178849359",
|
|
61
61
|
"packages/nx-plugin/src/executors/bounded-exec/ram-temp-held.ts": "7c32555bc6dab3f767925b55b68684d33a69393050342d06f055fb81e8e5332b",
|
|
62
62
|
"packages/nx-plugin/src/executors/bounded-exec/ram-temp.ts": "8d66e6ac13c4ccf6172586b9857d5114b48e027bdcb305f94f6828076c1f7eb4",
|
|
63
|
-
"packages/nx-plugin/src/executors/bounded-exec/run-cpu-pool.ts": "
|
|
63
|
+
"packages/nx-plugin/src/executors/bounded-exec/run-cpu-pool.ts": "17161460398004c66f10816325d6391755b3482c1f85c630bd020c4d393f47da",
|
|
64
64
|
"packages/nx-plugin/src/executors/bounded-exec/runner-hook.ts": "606eab5db826e571eb15df618315e01cc773d09fd647beb1c00f6434df58ffad",
|
|
65
65
|
"packages/nx-plugin/src/executors/bounded-exec/runner-lifecycle.ts": "b2a7c901296bb6c139909297bbdbab5bd00e94319c022307d006b0e1293f6454",
|
|
66
66
|
"packages/nx-plugin/src/executors/bounded-exec/runner-sizing.ts": "bffd2ba0f80758944ca682838832bdbe20ae12eb9a4f75084a3df82ae7786145",
|
|
@@ -68,7 +68,7 @@
|
|
|
68
68
|
"packages/nx-plugin/src/executors/bounded-exec/schema.ts": "74eb04eaa8d90d966a98ca2b1b7fffc7d2205a55a1e38958effa641b72c680a3",
|
|
69
69
|
"packages/nx-plugin/src/executors/bounded-exec/test-result-writer.ts": "df7668a11d105a1f88d2187ef8b0edff9db7ba56d8176b5bc35daf1359667637",
|
|
70
70
|
"packages/nx-plugin/src/executors/bounded-exec/test-results.ts": "4f83dc3accd7892f45385796522267adad9401326d0cd9ce4d4e1145b9023577",
|
|
71
|
-
"packages/nx-plugin/src/executors/bounded-exec/verdict.ts": "
|
|
71
|
+
"packages/nx-plugin/src/executors/bounded-exec/verdict.ts": "a752c55663c10632e9440519bf868af6c9525dcb6fc22c1dddfa7cf982ef83f2",
|
|
72
72
|
"packages/nx-plugin/src/executors/cargo-zigbuild/executor.ts": "622a5f36897d9eb1f7465654e88f4291aedee9767603028d20c1e819b91d72f4",
|
|
73
73
|
"packages/nx-plugin/src/executors/cargo-zigbuild/schema.ts": "e5cf3b0cc91a2d003a63ca0c9a93f1ec08263469b8a19dbe338b2dc0c0810747",
|
|
74
74
|
"packages/nx-plugin/src/executors/scriptc-native/executor.ts": "4db61da4444d3a30acd3995b6cf53a4bc403a40fe856d88b2c30ce467badbea8",
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { type CloseOutcome } from './cpu-pool.js';
|
|
2
|
+
import type { RunMemory } from './host-memory.js';
|
|
2
3
|
/**
|
|
3
4
|
* The thread an Nx run's CPU pool runs on (`run-cpu-pool.ts`; specs/cowshed/05_gateway.md, "Every Nx run budgets
|
|
4
5
|
* itself"). The process that serves a run's pool, the Nx daemon when installed plugins load in it or the Nx CLI without
|
|
@@ -7,13 +8,13 @@ import { type CloseOutcome } from './cpu-pool.js';
|
|
|
7
8
|
* jobserver fifo and heartbeat share an event loop that runs nothing else, so the heartbeat proves that the actor which
|
|
8
9
|
* admits and grants is turning.
|
|
9
10
|
*
|
|
10
|
-
* The thread is started with the pool's count and the run's memory. It says once that the pool serves, with the environment a
|
|
11
|
-
* or why it cannot; the host's one message is its close, after which the thread says how the pool closed.
|
|
11
|
+
* The thread is started with the pool's count and the run's memory. It says once that the pool serves, with the environment a
|
|
12
|
+
* task inherits, or why it cannot; the host's one message is its close, after which the thread says how the pool closed.
|
|
12
13
|
*/
|
|
13
|
-
/** What the thread is started with: the run's count, and the run's memory
|
|
14
|
+
/** What the thread is started with: the run's count, and the run's memory its grants are charged against. */
|
|
14
15
|
export interface CpuPoolThreadData {
|
|
15
16
|
readonly tokens: number;
|
|
16
|
-
readonly memory:
|
|
17
|
+
readonly memory: RunMemory;
|
|
17
18
|
}
|
|
18
19
|
/** What the thread says, in order: the pool serves or failed; after the host's close, how the pool closed. */
|
|
19
20
|
export type CpuPoolThreadReport = {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cpu-pool-thread.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/cpu-pool-thread.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,YAAY,EAAiB,MAAM,eAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"cpu-pool-thread.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/cpu-pool-thread.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,YAAY,EAAiB,MAAM,eAAe,CAAC;AACjE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAElD;;;;;;;;;;GAUG;AAEH,6GAA6G;AAC7G,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAC;CAC5B;AAED,8GAA8G;AAC9G,MAAM,MAAM,mBAAmB,GAC3B;IAAE,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IAAC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAAE,GAC1E;IAAE,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GACnD;IAAE,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IAAC,QAAQ,CAAC,OAAO,EAAE,YAAY,CAAA;CAAE,CAAC"}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import typia from 'typia';
|
|
2
2
|
import { type Unleased } from './gateway-lease.js';
|
|
3
|
+
import { type LiveUsageReading, type RunMemory } from './host-memory.js';
|
|
3
4
|
/**
|
|
4
5
|
* A CPU pool: the tokens a process lends the runners it starts, held live rather than inherited as a
|
|
5
6
|
* bare count (specs/cowshed/05_gateway.md, "Host scheduling", "Parallelism inside a job"). An Nx run
|
|
@@ -61,32 +62,46 @@ export interface Oversized {
|
|
|
61
62
|
*/
|
|
62
63
|
export declare function readInheritedCpuPool(environment: Readonly<NodeJS.ProcessEnv>): InheritedCpuPool | null;
|
|
63
64
|
/**
|
|
64
|
-
* Tokens a parent pool lent, held until released, or why there are none. `
|
|
65
|
-
*
|
|
66
|
-
*
|
|
65
|
+
* Tokens a parent pool lent, held until released, or why there are none. `charged` is the bytes the pool charged the
|
|
66
|
+
* grant, null from a pool that charges none. `oversized` names the run's memory when the child's charge alone exceeds
|
|
67
|
+
* it and it was granted because nothing else held memory. `closed` resolves when the grant's connection closes, however
|
|
68
|
+
* it closes, so a sub-pool built on the grant ends with it.
|
|
67
69
|
*/
|
|
68
70
|
export type TokenGrant = {
|
|
69
71
|
granted: true;
|
|
70
72
|
tokens: number;
|
|
73
|
+
charged: number | null;
|
|
71
74
|
oversized: Oversized | null;
|
|
72
75
|
release: () => void;
|
|
73
76
|
closed: Promise<void>;
|
|
74
77
|
} | Unleased;
|
|
78
|
+
/** What a leaf tells the pool beyond its count and its charge. */
|
|
79
|
+
export interface AskContext {
|
|
80
|
+
/** The tokens the task was lent when its recorded peak was measured. */
|
|
81
|
+
readonly parallelism?: number | undefined;
|
|
82
|
+
/** The Nx task the leaf runs, named in the pool's line for the grant. */
|
|
83
|
+
readonly task?: string | undefined;
|
|
84
|
+
/** The kernel killed the command for memory: it is run again, granted only while nothing else holds a grant. */
|
|
85
|
+
readonly alone?: boolean | undefined;
|
|
86
|
+
}
|
|
75
87
|
/**
|
|
76
88
|
* Ask the immediate parent pool for `want` tokens, charged `memory` bytes: the task's last recorded peak, or null when
|
|
77
|
-
* it has none.
|
|
78
|
-
*
|
|
79
|
-
*
|
|
80
|
-
*
|
|
89
|
+
* it has none. `context.parallelism` is the tokens the task was lent when that peak was measured; the pool charges the
|
|
90
|
+
* peak in proportion to the tokens it lends now when that is more, and lends fewer when the run's memory holds no more.
|
|
91
|
+
* The inherited count does not bound the ask: a pool grants at most what it holds. A granted lease is the child's local
|
|
92
|
+
* allocation: its connection is the grant, and closing it returns only that allocation. An absent endpoint grants
|
|
93
|
+
* nothing and refuses nothing; a connected endpoint's refusal or protocol failure must fail admission rather than run
|
|
94
|
+
* the child beside its parent's count.
|
|
81
95
|
*/
|
|
82
|
-
export declare function takeParentCpuTokens(parent: InheritedCpuPool, want: number, memory: number | null): Promise<TokenGrant>;
|
|
96
|
+
export declare function takeParentCpuTokens(parent: InheritedCpuPool, want: number, memory: number | null, context?: AskContext): Promise<TokenGrant>;
|
|
83
97
|
/**
|
|
84
98
|
* Make a pool of `tokens` that lives while `closed` does: when `closed` resolves the pool refuses
|
|
85
99
|
* grants, ends its child connections, drains its jobserver fifo and removes its directory. A run's own
|
|
86
|
-
* pool charges its grants against `memory
|
|
100
|
+
* pool charges its grants against `memory` and says each grant with what the run's bound holds live
|
|
101
|
+
* (`liveUsage`); a sub-pool charges none (null).
|
|
87
102
|
* Setup failures are values; a non-positive or unsafe count is a programmer bug and throws.
|
|
88
103
|
*/
|
|
89
|
-
export declare function createCpuPool(tokens: number, closed: Promise<void>, memory:
|
|
104
|
+
export declare function createCpuPool(tokens: number, closed: Promise<void>, memory: RunMemory | null, liveUsage?: (bound: string) => LiveUsageReading): Promise<{
|
|
90
105
|
ok: true;
|
|
91
106
|
pool: CpuPool;
|
|
92
107
|
} | {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cpu-pool.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/cpu-pool.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,MAAM,OAAO,CAAC;AAE1B,OAAO,EAAa,KAAK,QAAQ,EAAE,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"cpu-pool.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/cpu-pool.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,MAAM,OAAO,CAAC;AAE1B,OAAO,EAAa,KAAK,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9D,OAAO,EAAE,KAAK,gBAAgB,EAAE,KAAK,SAAS,EAAiB,MAAM,kBAAkB,CAAC;AAKxF;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,yGAAyG;AACzG,eAAO,MAAM,YAAY,0BAA0B,CAAC;AAEpD,6FAA6F;AAC7F,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,sDAAsD;IACtD,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED,yFAAyF;AACzF,MAAM,WAAW,OAAO;IACtB,oFAAoF;IACpF,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7C;;;;OAIG;IACH,KAAK,IAAI,OAAO,CAAC,YAAY,CAAC,CAAC;CAChC;AAED,MAAM,MAAM,YAAY,GAAG;IAAE,EAAE,EAAE,IAAI,CAAA;CAAE,GAAG;IAAE,EAAE,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC;AAgBvE,kHAAkH;AAClH,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;CACrD;AA+DD;;;;GAIG;AACH,wBAAgB,oBAAoB,CAAC,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,gBAAgB,GAAG,IAAI,CAOtG;AAED;;;;;GAKG;AACH,MAAM,MAAM,UAAU,GAClB;IACE,OAAO,EAAE,IAAI,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,SAAS,EAAE,SAAS,GAAG,IAAI,CAAC;IAC5B,OAAO,EAAE,MAAM,IAAI,CAAC;IACpB,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;CACvB,GACD,QAAQ,CAAC;AAIb,kEAAkE;AAClE,MAAM,WAAW,UAAU;IACzB,wEAAwE;IACxE,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1C,yEAAyE;IACzE,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACnC,gHAAgH;IAChH,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;CACtC;AAED;;;;;;;;GAQG;AACH,wBAAsB,mBAAmB,CACvC,MAAM,EAAE,gBAAgB,EACxB,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,MAAM,GAAG,IAAI,EACrB,OAAO,GAAE,UAAe,GACvB,OAAO,CAAC,UAAU,CAAC,CA8DrB;AAsJD;;;;;;GAMG;AACH,wBAAsB,aAAa,CACjC,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,EACrB,MAAM,EAAE,SAAS,GAAG,IAAI,EACxB,SAAS,GAAE,CAAC,KAAK,EAAE,MAAM,KAAK,gBAAgC,GAC7D,OAAO,CAAC;IAAE,EAAE,EAAE,IAAI,CAAC;IAAC,IAAI,EAAE,OAAO,CAAA;CAAE,GAAG;IAAE,EAAE,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CA8XrE"}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import * as _isTypeUint32_1 from "typia/lib/internal/_isTypeUint32";
|
|
2
2
|
import * as _isTypeUint64_1 from "typia/lib/internal/_isTypeUint64";
|
|
3
|
+
import * as _stringLengthLte_1 from "typia/lib/internal/_stringLengthLte";
|
|
3
4
|
import * as _jsonStringifyNumber_1 from "typia/lib/internal/_jsonStringifyNumber";
|
|
4
5
|
import * as _jsonStringifyTail_1 from "typia/lib/internal/_jsonStringifyTail";
|
|
5
6
|
import * as _jsonStringifyString_1 from "typia/lib/internal/_jsonStringifyString";
|
|
@@ -13,6 +14,7 @@ import { promisify } from 'node:util';
|
|
|
13
14
|
import typia from 'typia';
|
|
14
15
|
import { ownedSocketDirectory } from '../../testing.js';
|
|
15
16
|
import { holdLease } from './gateway-lease.js';
|
|
17
|
+
import { readLiveUsage } from './host-memory.js';
|
|
16
18
|
import { CpuLeaseWait, HEARTBEAT_MS } from './lease-wait.js';
|
|
17
19
|
import { admit } from './pool-admission.js';
|
|
18
20
|
import { CPU_TOKENS_ENV, parseInheritedTokens } from './runner-sizing.js';
|
|
@@ -45,8 +47,8 @@ import { CPU_TOKENS_ENV, parseInheritedTokens } from './runner-sizing.js';
|
|
|
45
47
|
export const CPU_POOL_ENV = 'BOUNDED_EXEC_CPU_POOL';
|
|
46
48
|
/** Exactly a request's fields: any other field is an invalid request. */
|
|
47
49
|
const isRequest = (() => {
|
|
48
|
-
const _io0 = (input, _exceptionable = true) => "cpu-pool" === input.op && ("number" === typeof input.want && (_isTypeUint32_1._isTypeUint32(input.want) && 1 <= input.want)) && (undefined === input.memory || "number" === typeof input.memory && _isTypeUint64_1._isTypeUint64(input.memory)) && (2 === Object.keys(input).length || Object.keys(input).every(key => {
|
|
49
|
-
if (["op", "want", "memory"].some(prop => key === prop))
|
|
50
|
+
const _io0 = (input, _exceptionable = true) => "cpu-pool" === input.op && ("number" === typeof input.want && (_isTypeUint32_1._isTypeUint32(input.want) && 1 <= input.want)) && (undefined === input.memory || "number" === typeof input.memory && _isTypeUint64_1._isTypeUint64(input.memory)) && (undefined === input.parallelism || "number" === typeof input.parallelism && (_isTypeUint32_1._isTypeUint32(input.parallelism) && 1 <= input.parallelism)) && (undefined === input.task || "string" === typeof input.task && _stringLengthLte_1._stringLengthLte(input.task, 256)) && (undefined === input.alone || "boolean" === typeof input.alone) && (2 === Object.keys(input).length || Object.keys(input).every(key => {
|
|
51
|
+
if (["op", "want", "memory", "parallelism", "task", "alone"].some(prop => key === prop))
|
|
50
52
|
return true;
|
|
51
53
|
const value = input[key];
|
|
52
54
|
if (undefined === value)
|
|
@@ -59,7 +61,7 @@ const stringifyReply = (() => {
|
|
|
59
61
|
const _ip0 = input => true === input["ok"];
|
|
60
62
|
const _so0 = input => `{"ok":${String(input.ok)},"progress":${String(_jsonStringifyNumber_1._jsonStringifyNumber(input.progress))}}`;
|
|
61
63
|
const _so1 = input => `{"ok":${String(input.ok)},"lease":${"\"" + input.lease + "\""},"server":${String(_jsonStringifyNumber_1._jsonStringifyNumber(input.server))}}`;
|
|
62
|
-
const _so2 = input => `{${_jsonStringifyTail_1._jsonStringifyTail(`"ok":${String(input.ok)},"lease":${"\"" + input.lease + "\""},"tokens":${String(_jsonStringifyNumber_1._jsonStringifyNumber(input.tokens))},${undefined === input.oversized ? "" : `"oversized":${undefined !== input.oversized ? _so3(input.oversized) : undefined}`}`)}}`;
|
|
64
|
+
const _so2 = input => `{${_jsonStringifyTail_1._jsonStringifyTail(`"ok":${String(input.ok)},"lease":${"\"" + input.lease + "\""},"tokens":${String(_jsonStringifyNumber_1._jsonStringifyNumber(input.tokens))},${undefined === input.oversized ? "" : `"oversized":${undefined !== input.oversized ? _so3(input.oversized) : undefined},`}${undefined === input.charged ? "" : `"charged":${undefined !== input.charged ? String(_jsonStringifyNumber_1._jsonStringifyNumber(input.charged)) : undefined}`}`)}}`;
|
|
63
65
|
const _so3 = input => `{"memory":${String(_jsonStringifyNumber_1._jsonStringifyNumber(input.memory))}}`;
|
|
64
66
|
const _so4 = input => `{"ok":${String(input.ok)},"code":${"\"" + input.code + "\""},"error":${_jsonStringifyString_1._jsonStringifyString(input.error)}}`;
|
|
65
67
|
const _su0 = input => (() => {
|
|
@@ -74,13 +76,13 @@ const stringifyReply = (() => {
|
|
|
74
76
|
else
|
|
75
77
|
_throwTypeGuardError_1._throwTypeGuardError({
|
|
76
78
|
method: "typia.json.createStringify",
|
|
77
|
-
expected: "(CpuLeaseProgress | CpuLeaseQueued | { ok: true; lease: \"granted\"; tokens: number; oversized?: Oversized | undefined; } | { ok: false; code: \"invalid-request\"; error: string; })",
|
|
79
|
+
expected: "(CpuLeaseProgress | CpuLeaseQueued | { ok: true; lease: \"granted\"; tokens: number; oversized?: Oversized | undefined; charged?: number | undefined; } | { ok: false; code: \"invalid-request\"; error: string; })",
|
|
78
80
|
value: input
|
|
79
81
|
});
|
|
80
82
|
})();
|
|
81
83
|
const _io0 = input => _ip0(input) && "number" === typeof input.progress;
|
|
82
84
|
const _io1 = input => _ip0(input) && "queued" === input.lease && ("number" === typeof input.server && _isTypeUint32_1._isTypeUint32(input.server));
|
|
83
|
-
const _io2 = input => _ip0(input) && "granted" === input.lease && "number" === typeof input.tokens && (undefined === input.oversized || "object" === typeof input.oversized && null !== input.oversized && _io3(input.oversized));
|
|
85
|
+
const _io2 = input => _ip0(input) && "granted" === input.lease && "number" === typeof input.tokens && (undefined === input.oversized || "object" === typeof input.oversized && null !== input.oversized && _io3(input.oversized)) && (undefined === input.charged || "number" === typeof input.charged);
|
|
84
86
|
const _io3 = input => "number" === typeof input.memory && _isTypeUint64_1._isTypeUint64(input.memory);
|
|
85
87
|
const _io4 = input => false === input.ok && "invalid-request" === input.code && "string" === typeof input.error;
|
|
86
88
|
return input => _su0(input);
|
|
@@ -127,18 +129,24 @@ const isOversized = (() => {
|
|
|
127
129
|
})();
|
|
128
130
|
/**
|
|
129
131
|
* Ask the immediate parent pool for `want` tokens, charged `memory` bytes: the task's last recorded peak, or null when
|
|
130
|
-
* it has none.
|
|
131
|
-
*
|
|
132
|
-
*
|
|
133
|
-
*
|
|
132
|
+
* it has none. `context.parallelism` is the tokens the task was lent when that peak was measured; the pool charges the
|
|
133
|
+
* peak in proportion to the tokens it lends now when that is more, and lends fewer when the run's memory holds no more.
|
|
134
|
+
* The inherited count does not bound the ask: a pool grants at most what it holds. A granted lease is the child's local
|
|
135
|
+
* allocation: its connection is the grant, and closing it returns only that allocation. An absent endpoint grants
|
|
136
|
+
* nothing and refuses nothing; a connected endpoint's refusal or protocol failure must fail admission rather than run
|
|
137
|
+
* the child beside its parent's count.
|
|
134
138
|
*/
|
|
135
|
-
export async function takeParentCpuTokens(parent, want, memory) {
|
|
139
|
+
export async function takeParentCpuTokens(parent, want, memory, context = {}) {
|
|
140
|
+
const { parallelism, task, alone } = context;
|
|
136
141
|
if (!Number.isSafeInteger(want) || want < 1) {
|
|
137
142
|
throw new RangeError(`a child asks its parent pool for a positive count, not ${want} (pool ${parent.tokens})`);
|
|
138
143
|
}
|
|
139
144
|
if (memory !== null && (!Number.isSafeInteger(memory) || memory < 0)) {
|
|
140
145
|
throw new RangeError(`a child is charged a whole number of bytes, not ${memory}`);
|
|
141
146
|
}
|
|
147
|
+
if (parallelism !== undefined && (!Number.isSafeInteger(parallelism) || parallelism < 1)) {
|
|
148
|
+
throw new RangeError(`a child's charge is measured under a positive count of tokens, not ${parallelism}`);
|
|
149
|
+
}
|
|
142
150
|
if (parent.socket.length === 0 || parent.socket.includes('\0')) {
|
|
143
151
|
return {
|
|
144
152
|
granted: false,
|
|
@@ -146,12 +154,18 @@ export async function takeParentCpuTokens(parent, want, memory) {
|
|
|
146
154
|
reason: `the parent CPU pool's endpoint ${JSON.stringify(parent.socket)} is not a socket path`,
|
|
147
155
|
};
|
|
148
156
|
}
|
|
149
|
-
const request =
|
|
157
|
+
const request = {
|
|
158
|
+
op: 'cpu-pool',
|
|
159
|
+
want,
|
|
160
|
+
...(memory === null ? {} : { memory, ...(parallelism === undefined ? {} : { parallelism }) }),
|
|
161
|
+
...(task === undefined ? {} : { task }),
|
|
162
|
+
...(alone === true ? { alone: true } : {}),
|
|
163
|
+
};
|
|
150
164
|
const lease = await holdLease(parent.socket, request, 'the parent CPU pool', new CpuLeaseWait());
|
|
151
165
|
if (!lease.granted) {
|
|
152
166
|
return lease;
|
|
153
167
|
}
|
|
154
|
-
const { tokens, oversized } = lease.answer;
|
|
168
|
+
const { tokens, oversized, charged } = lease.answer;
|
|
155
169
|
if (typeof tokens !== 'number' || !Number.isInteger(tokens) || tokens < 1 || tokens > want) {
|
|
156
170
|
lease.release();
|
|
157
171
|
return {
|
|
@@ -168,7 +182,22 @@ export async function takeParentCpuTokens(parent, want, memory) {
|
|
|
168
182
|
reason: `the parent CPU pool named its memory ${JSON.stringify(oversized)} in a grant, which is no byte count`,
|
|
169
183
|
};
|
|
170
184
|
}
|
|
171
|
-
|
|
185
|
+
if (charged !== undefined && !(typeof charged === 'number' && Number.isSafeInteger(charged) && charged >= 0)) {
|
|
186
|
+
lease.release();
|
|
187
|
+
return {
|
|
188
|
+
granted: false,
|
|
189
|
+
cause: 'other',
|
|
190
|
+
reason: `the parent CPU pool charged ${JSON.stringify(charged)} in a grant, which is no byte count`,
|
|
191
|
+
};
|
|
192
|
+
}
|
|
193
|
+
return {
|
|
194
|
+
granted: true,
|
|
195
|
+
tokens,
|
|
196
|
+
charged: charged ?? null,
|
|
197
|
+
oversized: oversized ?? null,
|
|
198
|
+
release: lease.release,
|
|
199
|
+
closed: lease.closed,
|
|
200
|
+
};
|
|
172
201
|
}
|
|
173
202
|
/** A thrown value's message. */
|
|
174
203
|
const describe = (error) => (error instanceof Error ? error.message : String(error));
|
|
@@ -304,10 +333,11 @@ function watchFifo(path, took, failed) {
|
|
|
304
333
|
/**
|
|
305
334
|
* Make a pool of `tokens` that lives while `closed` does: when `closed` resolves the pool refuses
|
|
306
335
|
* grants, ends its child connections, drains its jobserver fifo and removes its directory. A run's own
|
|
307
|
-
* pool charges its grants against `memory
|
|
336
|
+
* pool charges its grants against `memory` and says each grant with what the run's bound holds live
|
|
337
|
+
* (`liveUsage`); a sub-pool charges none (null).
|
|
308
338
|
* Setup failures are values; a non-positive or unsafe count is a programmer bug and throws.
|
|
309
339
|
*/
|
|
310
|
-
export async function createCpuPool(tokens, closed, memory) {
|
|
340
|
+
export async function createCpuPool(tokens, closed, memory, liveUsage = readLiveUsage) {
|
|
311
341
|
if (!Number.isSafeInteger(tokens) || tokens < 1) {
|
|
312
342
|
throw new RangeError(`a CPU pool holds a positive safe integer of tokens, not ${tokens}`);
|
|
313
343
|
}
|
|
@@ -337,6 +367,9 @@ export async function createCpuPool(tokens, closed, memory) {
|
|
|
337
367
|
let slotLent = false;
|
|
338
368
|
// The bytes the children holding grants are charged, against `memory`.
|
|
339
369
|
let heldMemory = 0;
|
|
370
|
+
// The children holding a grant, and whether one of them runs alone.
|
|
371
|
+
let holders = 0;
|
|
372
|
+
let aloneHolder = false;
|
|
340
373
|
// The readability wait while the queue's head finds no token, else undefined.
|
|
341
374
|
let watch;
|
|
342
375
|
let closing = false;
|
|
@@ -417,20 +450,32 @@ export async function createCpuPool(tokens, closed, memory) {
|
|
|
417
450
|
return;
|
|
418
451
|
}
|
|
419
452
|
}
|
|
420
|
-
const admission = admit(queue, inHand + (slotLent ? 0 : 1), memory === null ? null : { capacity: memory, held: heldMemory });
|
|
453
|
+
const admission = admit(queue, inHand + (slotLent ? 0 : 1), memory === null ? null : { capacity: memory.bytes, held: heldMemory }, { holders, alone: aloneHolder });
|
|
421
454
|
queue.splice(0, admission.grants.length);
|
|
422
|
-
for (const { request, tokens: granted, oversized } of admission.grants) {
|
|
455
|
+
for (const { request, tokens: granted, charge, oversized } of admission.grants) {
|
|
423
456
|
// Fifo bytes first, the implicit slot last.
|
|
424
457
|
const bytes = Math.min(granted, inHand);
|
|
425
458
|
inHand -= bytes;
|
|
426
459
|
const slot = granted > bytes;
|
|
427
460
|
if (slot)
|
|
428
461
|
slotLent = true;
|
|
429
|
-
heldMemory +=
|
|
430
|
-
|
|
431
|
-
request.
|
|
432
|
-
|
|
433
|
-
|
|
462
|
+
heldMemory += charge;
|
|
463
|
+
holders += 1;
|
|
464
|
+
if (request.alone === true)
|
|
465
|
+
aloneHolder = true;
|
|
466
|
+
request.child.phase = { kind: 'holding', bytes, slot, memory: charge, alone: request.alone === true };
|
|
467
|
+
request.child.socket.write(frame(memory === null
|
|
468
|
+
? { ok: true, lease: 'granted', tokens: granted }
|
|
469
|
+
: oversized
|
|
470
|
+
? { ok: true, lease: 'granted', tokens: granted, charged: charge, oversized: { memory: memory.bytes } }
|
|
471
|
+
: { ok: true, lease: 'granted', tokens: granted, charged: charge }));
|
|
472
|
+
if (memory !== null) {
|
|
473
|
+
const live = liveUsage(memory.source);
|
|
474
|
+
const held = live.ok
|
|
475
|
+
? `${live.usage.anon + live.usage.shmem} bytes (anon ${live.usage.anon}, shmem ${live.usage.shmem}, ${live.usage.source})`
|
|
476
|
+
: `unread (${live.reason})`;
|
|
477
|
+
process.stderr.write(`bounded-exec: the CPU pool at ${socketPath} granted ${request.task ?? 'a leaf'} ${granted} of ${request.want} tokens charged ${charge} bytes: holders charged ${heldMemory} of ${memory.bytes} bytes, live anon+shmem ${held}\n`);
|
|
478
|
+
}
|
|
434
479
|
}
|
|
435
480
|
if (admission.waiting === 'cpu') {
|
|
436
481
|
watchForTokens();
|
|
@@ -467,6 +512,9 @@ export async function createCpuPool(tokens, closed, memory) {
|
|
|
467
512
|
if (child.phase.slot)
|
|
468
513
|
slotLent = false;
|
|
469
514
|
heldMemory -= child.phase.memory;
|
|
515
|
+
holders -= 1;
|
|
516
|
+
if (child.phase.alone)
|
|
517
|
+
aloneHolder = false;
|
|
470
518
|
if (!closing)
|
|
471
519
|
returnTokens(child.phase.bytes);
|
|
472
520
|
}
|
|
@@ -528,7 +576,22 @@ export async function createCpuPool(tokens, closed, memory) {
|
|
|
528
576
|
}
|
|
529
577
|
const request = parsed;
|
|
530
578
|
child.phase = { kind: 'queued' };
|
|
531
|
-
|
|
579
|
+
const waiter = {
|
|
580
|
+
child,
|
|
581
|
+
want: request.want,
|
|
582
|
+
memory: request.memory ?? 0,
|
|
583
|
+
parallelism: request.parallelism,
|
|
584
|
+
task: request.task,
|
|
585
|
+
alone: request.alone,
|
|
586
|
+
};
|
|
587
|
+
if (request.alone === true) {
|
|
588
|
+
// A leaf the kernel killed has waited once already: it goes ahead of the leaves that have not run.
|
|
589
|
+
const at = queue.findIndex((waiting) => waiting.alone !== true);
|
|
590
|
+
queue.splice(at === -1 ? queue.length : at, 0, waiter);
|
|
591
|
+
}
|
|
592
|
+
else {
|
|
593
|
+
queue.push(waiter);
|
|
594
|
+
}
|
|
532
595
|
socket.write(frame({ ok: true, lease: 'queued', server: process.pid }));
|
|
533
596
|
drain(0);
|
|
534
597
|
});
|
|
@@ -30,6 +30,11 @@ export interface NxTaskContext {
|
|
|
30
30
|
export interface BoundedExecResult {
|
|
31
31
|
success: boolean;
|
|
32
32
|
terminalOutput: string;
|
|
33
|
+
/**
|
|
34
|
+
* The command died of a SIGKILL that no bound of this executor and no signal it sent caused: the kernel's OOM killer,
|
|
35
|
+
* which sends it to a leaf in a run's cgroup. Set on the last run of the command only when it was killed again.
|
|
36
|
+
*/
|
|
37
|
+
killedForMemory?: boolean;
|
|
33
38
|
}
|
|
34
39
|
export interface ProcessTreeKiller {
|
|
35
40
|
kill(pid: number, signal: NodeJS.Signals): Promise<void>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/executor.ts"],"names":[],"mappings":"AA0BA,OAAO,EAIL,KAAK,gBAAgB,EAItB,MAAM,eAAe,CAAC;AAGvB,OAAO,EAAE,sBAAsB,EAAmB,MAAM,uBAAuB,CAAC;AAahF,OAAO,EAA4C,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAG5F,OAAO,EAOL,KAAK,YAAY,EAClB,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EASL,aAAa,EAEd,MAAM,eAAe,CAAC;AAIvB,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAEtD,OAAO,EAGL,KAAK,eAAe,EAgBrB,MAAM,cAAc,CAAC;AAiBtB,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,wFAAwF;IACxF,MAAM,CAAC,EAAE;QAAE,IAAI,EAAE,eAAe,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC;IACtD,uFAAuF;IACvF,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;CAClC;AAED,qEAAqE;AACrE,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,SAAS,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE;YAAE,IAAI,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;CAC1D;AAED,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,OAAO,CAAC;IACjB,cAAc,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/executor.ts"],"names":[],"mappings":"AA0BA,OAAO,EAIL,KAAK,gBAAgB,EAItB,MAAM,eAAe,CAAC;AAGvB,OAAO,EAAE,sBAAsB,EAAmB,MAAM,uBAAuB,CAAC;AAahF,OAAO,EAA4C,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAG5F,OAAO,EAOL,KAAK,YAAY,EAClB,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EASL,aAAa,EAEd,MAAM,eAAe,CAAC;AAIvB,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAEtD,OAAO,EAGL,KAAK,eAAe,EAgBrB,MAAM,cAAc,CAAC;AAiBtB,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,wFAAwF;IACxF,MAAM,CAAC,EAAE;QAAE,IAAI,EAAE,eAAe,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC;IACtD,uFAAuF;IACvF,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;CAClC;AAED,qEAAqE;AACrE,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,SAAS,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE;YAAE,IAAI,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;CAC1D;AAED,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,OAAO,CAAC;IACjB,cAAc,EAAE,MAAM,CAAC;IACvB;;;OAGG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;CAC3B;AAoBD,MAAM,WAAW,iBAAiB;IAChC,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1D;AAED,8EAA8E;AAC9E,MAAM,MAAM,UAAU,GAAG,IAAI,CAC3B,aAAa,EACb,SAAS,GAAG,SAAS,GAAG,UAAU,GAAG,OAAO,GAAG,QAAQ,GAAG,gBAAgB,CAC3E,CAAC;AAQF;;;;;;GAMG;AACH,wBAAgB,wBAAwB,CACtC,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,MAAM,GAAG,SAAS,EACzB,SAAS,EAAE,MAAM,GAAG,SAAS,EAC7B,SAAS,EAAE,MAAM,GAAG,SAAS,EAC7B,GAAG,EAAE,MAAM,EACX,MAAM,GAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAOzB,GACA,MAAM,GAAG,SAAS,CAiBpB;AAED,MAAM,CAAC,OAAO,UAAU,mBAAmB,CACzC,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,aAAa,GACrB,OAAO,CAAC,iBAAiB,CAAC,CAE5B;AAED;;;GAGG;AACH,wBAAgB,qBAAqB,CACnC,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,kBAAkB,GAC1B,OAAO,CAAC,iBAAiB,CAAC,CAc5B;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAChC,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,kBAAkB,EAC3B,MAAM,EAAE,CAAC,OAAO,EAAE,MAAM,GAAG,SAAS,KAAK,UAAU,GAAG,IAAI,GACzD,OAAO,CAAC,iBAAiB,CAAC,CAsB5B;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAAC,OAAO,EAAE,aAAa,GAAG,kBAAkB,CAiBxE;AAED;;;;;;;;GAQG;AACH,wBAAsB,cAAc,CAClC,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,kBAAkB,EAC3B,MAAM,EAAE,iBAAiB,EACzB,UAAU,EAAE,UAAU,GAAG,IAAI,EAC7B,SAAS,GAAE,gBAAgB,GAAG,IAAW,EACzC,kBAAkB,GAAE,OAAO,sBAA+C,EAC1E,KAAK,GAAE,YAAY,GAAG,IAAW,EACjC,SAAS,CAAC,EAAE,UAAU,CAAC,OAAO,gBAAgB,CAAC,CAAC,CAAC,CAAC,GACjD,OAAO,CAAC,iBAAiB,CAAC,CA2B5B;AA8rCD,wBAAgB,uBAAuB,IAAI,iBAAiB,CAM3D"}
|
|
@@ -142,10 +142,35 @@ export function recordContext(context) {
|
|
|
142
142
|
* command runs on over budget to its own exit, and the run says so.
|
|
143
143
|
*/
|
|
144
144
|
export async function runBoundedExec(options, context, killer, tempVolume, inherited = null, captureDiagnostics = captureHangDiagnostics, owner = null, queueHost) {
|
|
145
|
-
const cwd = resolveCwd(options.cwd, context.root);
|
|
146
145
|
const directory = context.record?.directory ?? null;
|
|
147
|
-
// The task's last recorded peak
|
|
146
|
+
// The task's last recorded peak, and the tokens it was lent when the peak was measured, are its charge in its parent's
|
|
147
|
+
// pool, read once before a run's clear removes them: a command run again after a kill is charged by the record the
|
|
148
|
+
// killed run replaced, not by the peak it reached before it was killed.
|
|
148
149
|
const lastPeak = directory !== null && inherited !== null ? await readLastPeak(directory) : null;
|
|
150
|
+
const run = (alone) => runCommand(options, context, killer, tempVolume, inherited, captureDiagnostics, owner, queueHost, {
|
|
151
|
+
lastPeak,
|
|
152
|
+
alone,
|
|
153
|
+
});
|
|
154
|
+
const first = await run(false);
|
|
155
|
+
if (first.killedForMemory !== true || inherited === null) {
|
|
156
|
+
return first;
|
|
157
|
+
}
|
|
158
|
+
// The kernel killed the leaf: a SIGKILL this executor did not send. It runs once more, alone in its run's pool, so
|
|
159
|
+
// what it was killed beside is not there to be killed with it.
|
|
160
|
+
const note = `cowshed: the kernel killed the command for memory (a SIGKILL this executor did not send${context.record === undefined ? '' : `: ${context.record.task.id}`}); running it once more, alone in its run's pool\n`;
|
|
161
|
+
process.stderr.write(note);
|
|
162
|
+
const again = await run(true);
|
|
163
|
+
return {
|
|
164
|
+
success: again.success,
|
|
165
|
+
terminalOutput: `${first.terminalOutput}${note}${again.terminalOutput}`,
|
|
166
|
+
...(again.killedForMemory === true ? { killedForMemory: true } : {}),
|
|
167
|
+
};
|
|
168
|
+
}
|
|
169
|
+
/** One run of the command, from the pool's grant to its record. */
|
|
170
|
+
async function runCommand(options, context, killer, tempVolume, inherited, captureDiagnostics, owner, queueHost, attempt) {
|
|
171
|
+
const cwd = resolveCwd(options.cwd, context.root);
|
|
172
|
+
const directory = context.record?.directory ?? null;
|
|
173
|
+
const { lastPeak } = attempt;
|
|
149
174
|
// A previous run's verdict and report go first, so neither can be read as this run's.
|
|
150
175
|
if (directory !== null) {
|
|
151
176
|
await clearTaskDirectory(directory);
|
|
@@ -164,6 +189,7 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
164
189
|
settled: false,
|
|
165
190
|
expiry: null,
|
|
166
191
|
forceKillNeeded: false,
|
|
192
|
+
sentSignal: false,
|
|
167
193
|
poolCloseError: null,
|
|
168
194
|
retireError: null,
|
|
169
195
|
noSpace: null,
|
|
@@ -230,8 +256,10 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
230
256
|
if (lastPeak !== null && !lastPeak.ok) {
|
|
231
257
|
return refuseUnstarted(`bounded-exec: ${lastPeak.error}; it is this task's memory charge, so the command does not run\n`);
|
|
232
258
|
}
|
|
233
|
-
// A task with no recorded peak is charged nothing (null) until it has one.
|
|
259
|
+
// A task with no recorded peak is charged nothing (null) until it has one. A peak is charged in proportion to the
|
|
260
|
+
// tokens the run is lent when that is more than the run that measured it was lent.
|
|
234
261
|
const charge = lastPeak?.ok === true ? lastPeak.peakBytes : null;
|
|
262
|
+
const chargeParallelism = lastPeak?.ok === true ? lastPeak.parallelism : null;
|
|
235
263
|
let lease = null;
|
|
236
264
|
let ordinaryTemp = null;
|
|
237
265
|
// Before spawn there is nothing to retire. Once started, only the owner's terminal retirement permits deletion.
|
|
@@ -371,7 +399,11 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
371
399
|
let parentRefusal = null;
|
|
372
400
|
let parentWait;
|
|
373
401
|
if (inherited !== null) {
|
|
374
|
-
const fromParent = await takeParentCpuTokens(inherited, demand.want, charge
|
|
402
|
+
const fromParent = await takeParentCpuTokens(inherited, demand.want, charge, {
|
|
403
|
+
parallelism: chargeParallelism ?? undefined,
|
|
404
|
+
alone: attempt.alone ? true : undefined,
|
|
405
|
+
task: context.record?.task.id,
|
|
406
|
+
});
|
|
375
407
|
if (fromParent.granted) {
|
|
376
408
|
grant = fromParent;
|
|
377
409
|
}
|
|
@@ -393,7 +425,7 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
393
425
|
if (grant !== null) {
|
|
394
426
|
sized = sizeRunner(demand, requested, grant.tokens);
|
|
395
427
|
cpu = { tokens: grant.tokens, want: demand.want, waitedMs };
|
|
396
|
-
appendStderr(`cowshed: cpu-pool wait done elapsed=${waitedMs}ms status=ok tokens=${grant.tokens}/${demand.want} runner=${demand.kind} memory=${charge ?? 'none'}\n`);
|
|
428
|
+
appendStderr(`cowshed: cpu-pool wait done elapsed=${waitedMs}ms status=ok tokens=${grant.tokens}/${demand.want} runner=${demand.kind} memory=${charge ?? 'none'}${grant.charged !== null && grant.charged !== charge ? ` charged=${grant.charged}` : ''}\n`);
|
|
397
429
|
if (grant.oversized !== null) {
|
|
398
430
|
appendStderr(`cowshed: cpu-pool granted while nothing else holds memory: this task's last peak of ${charge} bytes alone exceeds the run's ${grant.oversized.memory} bytes\n`);
|
|
399
431
|
}
|
|
@@ -919,6 +951,7 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
919
951
|
return;
|
|
920
952
|
}
|
|
921
953
|
try {
|
|
954
|
+
state.sentSignal = true;
|
|
922
955
|
await killer.kill(pid, 'SIGTERM');
|
|
923
956
|
}
|
|
924
957
|
catch (error) {
|
|
@@ -1037,6 +1070,7 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
1037
1070
|
// cannot end is said, and a failure of the probe or of a signal is shown.
|
|
1038
1071
|
const forceKill = async (pid) => {
|
|
1039
1072
|
try {
|
|
1073
|
+
state.sentSignal = true;
|
|
1040
1074
|
if (await killProcessGroup(pid, DEADLINE_PATIENCE, (group) => killer.kill(group, 'SIGKILL'))) {
|
|
1041
1075
|
tell(`${describeMembersLeft(pid)}\n`);
|
|
1042
1076
|
}
|
|
@@ -1236,9 +1270,14 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
1236
1270
|
// branch later exited 0: its verdict is an ordinary operational failure, never a successful or silently unbounded run.
|
|
1237
1271
|
const operationalFailure = state.poolCloseError !== null || lifecycleFailure !== null || state.retireError !== null;
|
|
1238
1272
|
await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure, seen, state.lockWaits, peakBytes);
|
|
1273
|
+
// A SIGKILL this executor did not send, to a command that no bound of this executor ended, came from outside it. In a
|
|
1274
|
+
// run's cgroup the one that sends it to a leaf is the kernel's OOM killer. The exit code alone says nothing of who
|
|
1275
|
+
// sent it: a bound's own force-kill ends in 137 too, and is why `sentSignal` is read first.
|
|
1276
|
+
const killedForMemory = !state.sentSignal && state.expiry === null && (exitSignal === 'SIGKILL' || code === 137);
|
|
1239
1277
|
return {
|
|
1240
1278
|
success: !operationalFailure && resultFailure === null && state.expiry === null && code === 0,
|
|
1241
1279
|
terminalOutput: outputChunks.join(''),
|
|
1280
|
+
...(killedForMemory ? { killedForMemory: true } : {}),
|
|
1242
1281
|
};
|
|
1243
1282
|
}
|
|
1244
1283
|
export function createProcessTreeKiller() {
|