@smoothbricks/nx-plugin 0.7.11 → 0.7.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -1
- package/dist/.build-inputs.json +9 -9
- package/dist/executors/bounded-exec/cpu-pool-thread.d.ts +5 -4
- package/dist/executors/bounded-exec/cpu-pool-thread.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool.d.ts +25 -10
- package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool.js +86 -23
- package/dist/executors/bounded-exec/executor.d.ts +5 -0
- package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
- package/dist/executors/bounded-exec/executor.js +44 -5
- package/dist/executors/bounded-exec/host-memory.d.ts +22 -0
- package/dist/executors/bounded-exec/host-memory.d.ts.map +1 -1
- package/dist/executors/bounded-exec/host-memory.darwin.d.ts.map +1 -1
- package/dist/executors/bounded-exec/host-memory.darwin.js +7 -1
- package/dist/executors/bounded-exec/host-memory.js +6 -0
- package/dist/executors/bounded-exec/host-memory.linux.d.ts +3 -1
- package/dist/executors/bounded-exec/host-memory.linux.d.ts.map +1 -1
- package/dist/executors/bounded-exec/host-memory.linux.js +43 -1
- package/dist/executors/bounded-exec/pool-admission.d.ts +31 -7
- package/dist/executors/bounded-exec/pool-admission.d.ts.map +1 -1
- package/dist/executors/bounded-exec/pool-admission.js +44 -8
- package/dist/executors/bounded-exec/run-cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/run-cpu-pool.js +1 -1
- package/dist/executors/bounded-exec/verdict.d.ts +4 -2
- package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
- package/dist/executors/bounded-exec/verdict.js +19 -2
- package/managed/raw/patches/nx@24.0.3.patch +16 -0
- package/package.json +1 -1
- package/src/executors/bounded-exec/cpu-pool-thread.ts +5 -4
- package/src/executors/bounded-exec/cpu-pool.ts +121 -25
- package/src/executors/bounded-exec/executor.ts +71 -5
- package/src/executors/bounded-exec/host-memory.darwin.ts +7 -1
- package/src/executors/bounded-exec/host-memory.linux.ts +42 -2
- package/src/executors/bounded-exec/host-memory.ts +25 -0
- package/src/executors/bounded-exec/pool-admission.ts +66 -11
- package/src/executors/bounded-exec/run-cpu-pool.ts +3 -3
- package/src/executors/bounded-exec/verdict.ts +25 -5
- package/test-timings.json +2 -0
package/README.md
CHANGED
|
@@ -2430,7 +2430,15 @@ the run's memory: the least finite `memory.max` of the run's cgroup and its ance
|
|
|
2430
2430
|
Linux, `hw.memsize` on macOS (`host-memory.ts`). A run whose memory cannot be read fails in `preTasksExecution`, naming
|
|
2431
2431
|
the file or sysctl. A task whose own peak exceeds the run's memory is granted once nothing else holds memory, and its
|
|
2432
2432
|
stderr says so; the wait line names the charge as `memory=<bytes>` or `memory=none`. A granted executor's sub-pool
|
|
2433
|
-
charges no memory.
|
|
2433
|
+
charges no memory. The charge follows the parallelism the task is lent: its record's `cpu.tokens` is the parallelism its
|
|
2434
|
+
peak was measured under, and the pool charges the peak in proportion to the tokens it lends when that is more
|
|
2435
|
+
(`ceil(peak × tokens ÷ measured)`, never less than the peak). A task whose whole ask does not fit is lent the most
|
|
2436
|
+
tokens that do, and its wait line names the bytes charged (`memory=<peak> charged=<bytes>`) when they differ from the
|
|
2437
|
+
peak. Each grant is said on the pool's stderr with the task, its charge, what the holders are charged of the run's
|
|
2438
|
+
memory and what the run's bound holds live (anonymous plus shared memory). A command that died of a SIGKILL this
|
|
2439
|
+
executor did not send, with no bound of its own ended, was killed by the kernel for memory: it runs once more, alone in
|
|
2440
|
+
its run's pool (`"alone":true` in the request), which grants it only while no leaf holds a grant and nothing beside it,
|
|
2441
|
+
and says so. A host CI scope's first run of a task it holds no record of is the `memory=none` case: the managed
|
|
2434
2442
|
`setup-devenv` action starts a scope from the records of the pushes it follows (a pull request's base branch's, a
|
|
2435
2443
|
release workflow's CI's) and takes, at each run, the record of every task the scope lacks
|
|
2436
2444
|
(`specs/cowshed/05_gateway.md`, "Without mbx, every Nx run budgets itself").
|
package/dist/.build-inputs.json
CHANGED
|
@@ -35,15 +35,15 @@
|
|
|
35
35
|
"packages/nx-plugin/src/executors/all-verdicts/executor.ts": "4d3aac65aa2e03a2d24d149c412aa18f2e66a6a36a4e88993122b10e9355c8f0",
|
|
36
36
|
"packages/nx-plugin/src/executors/all-verdicts/schema.ts": "2ca6bd8d2f19d851f291ab6a5227167399b9dc8ed0fda0248b69896d77e471f1",
|
|
37
37
|
"packages/nx-plugin/src/executors/bounded-exec/bounds.ts": "fecc48ea66df1c2ba526dc0322ab7760cfc1a4e85d99a72a7a43f040448970f0",
|
|
38
|
-
"packages/nx-plugin/src/executors/bounded-exec/cpu-pool-thread.ts": "
|
|
39
|
-
"packages/nx-plugin/src/executors/bounded-exec/cpu-pool.ts": "
|
|
38
|
+
"packages/nx-plugin/src/executors/bounded-exec/cpu-pool-thread.ts": "47697a9531df5fd6fcdf4e706fc7e5874bca1717c838496673bf148c691f1e9e",
|
|
39
|
+
"packages/nx-plugin/src/executors/bounded-exec/cpu-pool.ts": "0d7972fb3a6bf3557dddf4b67e50c80c1d37302baecebcf4a078130d8979a803",
|
|
40
40
|
"packages/nx-plugin/src/executors/bounded-exec/disk-lease.ts": "1a3f6a6fa590d9a2550490bb6906dc91b6c05fbef235b025eaf257eb5310e92d",
|
|
41
|
-
"packages/nx-plugin/src/executors/bounded-exec/executor.ts": "
|
|
41
|
+
"packages/nx-plugin/src/executors/bounded-exec/executor.ts": "2e0ae5b31e9863bfd8d27010dbaefcf17997c0d795d031b4a232f20129b2268f",
|
|
42
42
|
"packages/nx-plugin/src/executors/bounded-exec/gateway-lease.ts": "f3f73bd92db30e4857f98fb8bd2b9a75b2a5a2a318f094e075ed803c1f9a0117",
|
|
43
43
|
"packages/nx-plugin/src/executors/bounded-exec/hang-diagnostics.ts": "bcaa9e52384f6c481a9cc2835ba6e8ffe167afa3cb51623fdf8dc3493876f335",
|
|
44
|
-
"packages/nx-plugin/src/executors/bounded-exec/host-memory.darwin.ts": "
|
|
45
|
-
"packages/nx-plugin/src/executors/bounded-exec/host-memory.linux.ts": "
|
|
46
|
-
"packages/nx-plugin/src/executors/bounded-exec/host-memory.ts": "
|
|
44
|
+
"packages/nx-plugin/src/executors/bounded-exec/host-memory.darwin.ts": "ce1c12e3f8ebe06034d2e9ecd332f907c567fb2d0a71f3c1c5acccb687488fc8",
|
|
45
|
+
"packages/nx-plugin/src/executors/bounded-exec/host-memory.linux.ts": "465171b065355df13af73fad9c8101d48f6bb060cf41cb50d8dd272abf693b08",
|
|
46
|
+
"packages/nx-plugin/src/executors/bounded-exec/host-memory.ts": "6d02f33e950c89162deb9131bd3c0819ab19bf54dd600da557d7869491eaf99a",
|
|
47
47
|
"packages/nx-plugin/src/executors/bounded-exec/leaf-peak.ts": "0b1b94992b2cbb9e3759785424ceb202667053c5571edf3a79cdd0b206e54a1d",
|
|
48
48
|
"packages/nx-plugin/src/executors/bounded-exec/lease-wait.ts": "59d8ac94d0b8b75858971c5bb28ee421fca4f7685577274fe8bbaf0c01ba462a",
|
|
49
49
|
"packages/nx-plugin/src/executors/bounded-exec/lock-wait.ts": "b14a23b49759a777f4b3338ced003adaf76837c2c38462458f76a7fef1b95d9e",
|
|
@@ -53,14 +53,14 @@
|
|
|
53
53
|
"packages/nx-plugin/src/executors/bounded-exec/no-space.ts": "ec032968d89f9e853796012705a06c06cfc4da54bf6e6f10a526f98daca3d077",
|
|
54
54
|
"packages/nx-plugin/src/executors/bounded-exec/owned-subtree.ts": "cb5edf2b5fe2c1b5b961f8c0ff16ec7bfae3c22ad75dfd671b74f1a4a865863f",
|
|
55
55
|
"packages/nx-plugin/src/executors/bounded-exec/peak-waiter.ts": "3d9a60f8e61d160cf4151746c160f836954f57aca896127a5ef9df4d09c26180",
|
|
56
|
-
"packages/nx-plugin/src/executors/bounded-exec/pool-admission.ts": "
|
|
56
|
+
"packages/nx-plugin/src/executors/bounded-exec/pool-admission.ts": "0cc84dbda8d2cf472a0ee9ec14ab136a219ab2ed223e9769ef45590efbb4b793",
|
|
57
57
|
"packages/nx-plugin/src/executors/bounded-exec/progress.ts": "ed926f7b4991ebf7a7c8876ab0ea00295934d058c774e3f811963a85d209979b",
|
|
58
58
|
"packages/nx-plugin/src/executors/bounded-exec/ram-attach.ts": "865eb5c277d9ab4b258013030938aeee762b731a7298fa8ad3c5eba3026df77a",
|
|
59
59
|
"packages/nx-plugin/src/executors/bounded-exec/ram-detach.ts": "535d26114d14d0d94db76227ea7ba52b1ae0b590d7dbac22993b6796d255e666",
|
|
60
60
|
"packages/nx-plugin/src/executors/bounded-exec/ram-temp-fixtures.ts": "075da4ecfd1cf7dd98fc8c894599dcf2720faba07ac30b8e5592964178849359",
|
|
61
61
|
"packages/nx-plugin/src/executors/bounded-exec/ram-temp-held.ts": "7c32555bc6dab3f767925b55b68684d33a69393050342d06f055fb81e8e5332b",
|
|
62
62
|
"packages/nx-plugin/src/executors/bounded-exec/ram-temp.ts": "8d66e6ac13c4ccf6172586b9857d5114b48e027bdcb305f94f6828076c1f7eb4",
|
|
63
|
-
"packages/nx-plugin/src/executors/bounded-exec/run-cpu-pool.ts": "
|
|
63
|
+
"packages/nx-plugin/src/executors/bounded-exec/run-cpu-pool.ts": "17161460398004c66f10816325d6391755b3482c1f85c630bd020c4d393f47da",
|
|
64
64
|
"packages/nx-plugin/src/executors/bounded-exec/runner-hook.ts": "606eab5db826e571eb15df618315e01cc773d09fd647beb1c00f6434df58ffad",
|
|
65
65
|
"packages/nx-plugin/src/executors/bounded-exec/runner-lifecycle.ts": "b2a7c901296bb6c139909297bbdbab5bd00e94319c022307d006b0e1293f6454",
|
|
66
66
|
"packages/nx-plugin/src/executors/bounded-exec/runner-sizing.ts": "bffd2ba0f80758944ca682838832bdbe20ae12eb9a4f75084a3df82ae7786145",
|
|
@@ -68,7 +68,7 @@
|
|
|
68
68
|
"packages/nx-plugin/src/executors/bounded-exec/schema.ts": "74eb04eaa8d90d966a98ca2b1b7fffc7d2205a55a1e38958effa641b72c680a3",
|
|
69
69
|
"packages/nx-plugin/src/executors/bounded-exec/test-result-writer.ts": "df7668a11d105a1f88d2187ef8b0edff9db7ba56d8176b5bc35daf1359667637",
|
|
70
70
|
"packages/nx-plugin/src/executors/bounded-exec/test-results.ts": "4f83dc3accd7892f45385796522267adad9401326d0cd9ce4d4e1145b9023577",
|
|
71
|
-
"packages/nx-plugin/src/executors/bounded-exec/verdict.ts": "
|
|
71
|
+
"packages/nx-plugin/src/executors/bounded-exec/verdict.ts": "a752c55663c10632e9440519bf868af6c9525dcb6fc22c1dddfa7cf982ef83f2",
|
|
72
72
|
"packages/nx-plugin/src/executors/cargo-zigbuild/executor.ts": "622a5f36897d9eb1f7465654e88f4291aedee9767603028d20c1e819b91d72f4",
|
|
73
73
|
"packages/nx-plugin/src/executors/cargo-zigbuild/schema.ts": "e5cf3b0cc91a2d003a63ca0c9a93f1ec08263469b8a19dbe338b2dc0c0810747",
|
|
74
74
|
"packages/nx-plugin/src/executors/scriptc-native/executor.ts": "4db61da4444d3a30acd3995b6cf53a4bc403a40fe856d88b2c30ce467badbea8",
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { type CloseOutcome } from './cpu-pool.js';
|
|
2
|
+
import type { RunMemory } from './host-memory.js';
|
|
2
3
|
/**
|
|
3
4
|
* The thread an Nx run's CPU pool runs on (`run-cpu-pool.ts`; specs/cowshed/05_gateway.md, "Every Nx run budgets
|
|
4
5
|
* itself"). The process that serves a run's pool, the Nx daemon when installed plugins load in it or the Nx CLI without
|
|
@@ -7,13 +8,13 @@ import { type CloseOutcome } from './cpu-pool.js';
|
|
|
7
8
|
* jobserver fifo and heartbeat share an event loop that runs nothing else, so the heartbeat proves that the actor which
|
|
8
9
|
* admits and grants is turning.
|
|
9
10
|
*
|
|
10
|
-
* The thread is started with the pool's count and the run's memory. It says once that the pool serves, with the environment a
|
|
11
|
-
* or why it cannot; the host's one message is its close, after which the thread says how the pool closed.
|
|
11
|
+
* The thread is started with the pool's count and the run's memory. It says once that the pool serves, with the environment a
|
|
12
|
+
* task inherits, or why it cannot; the host's one message is its close, after which the thread says how the pool closed.
|
|
12
13
|
*/
|
|
13
|
-
/** What the thread is started with: the run's count, and the run's memory
|
|
14
|
+
/** What the thread is started with: the run's count, and the run's memory its grants are charged against. */
|
|
14
15
|
export interface CpuPoolThreadData {
|
|
15
16
|
readonly tokens: number;
|
|
16
|
-
readonly memory:
|
|
17
|
+
readonly memory: RunMemory;
|
|
17
18
|
}
|
|
18
19
|
/** What the thread says, in order: the pool serves or failed; after the host's close, how the pool closed. */
|
|
19
20
|
export type CpuPoolThreadReport = {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cpu-pool-thread.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/cpu-pool-thread.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,YAAY,EAAiB,MAAM,eAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"cpu-pool-thread.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/cpu-pool-thread.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,YAAY,EAAiB,MAAM,eAAe,CAAC;AACjE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAElD;;;;;;;;;;GAUG;AAEH,6GAA6G;AAC7G,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAC;CAC5B;AAED,8GAA8G;AAC9G,MAAM,MAAM,mBAAmB,GAC3B;IAAE,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IAAC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAAE,GAC1E;IAAE,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GACnD;IAAE,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IAAC,QAAQ,CAAC,OAAO,EAAE,YAAY,CAAA;CAAE,CAAC"}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import typia from 'typia';
|
|
2
2
|
import { type Unleased } from './gateway-lease.js';
|
|
3
|
+
import { type LiveUsageReading, type RunMemory } from './host-memory.js';
|
|
3
4
|
/**
|
|
4
5
|
* A CPU pool: the tokens a process lends the runners it starts, held live rather than inherited as a
|
|
5
6
|
* bare count (specs/cowshed/05_gateway.md, "Host scheduling", "Parallelism inside a job"). An Nx run
|
|
@@ -61,32 +62,46 @@ export interface Oversized {
|
|
|
61
62
|
*/
|
|
62
63
|
export declare function readInheritedCpuPool(environment: Readonly<NodeJS.ProcessEnv>): InheritedCpuPool | null;
|
|
63
64
|
/**
|
|
64
|
-
* Tokens a parent pool lent, held until released, or why there are none. `
|
|
65
|
-
*
|
|
66
|
-
*
|
|
65
|
+
* Tokens a parent pool lent, held until released, or why there are none. `charged` is the bytes the pool charged the
|
|
66
|
+
* grant, null from a pool that charges none. `oversized` names the run's memory when the child's charge alone exceeds
|
|
67
|
+
* it and it was granted because nothing else held memory. `closed` resolves when the grant's connection closes, however
|
|
68
|
+
* it closes, so a sub-pool built on the grant ends with it.
|
|
67
69
|
*/
|
|
68
70
|
export type TokenGrant = {
|
|
69
71
|
granted: true;
|
|
70
72
|
tokens: number;
|
|
73
|
+
charged: number | null;
|
|
71
74
|
oversized: Oversized | null;
|
|
72
75
|
release: () => void;
|
|
73
76
|
closed: Promise<void>;
|
|
74
77
|
} | Unleased;
|
|
78
|
+
/** What a leaf tells the pool beyond its count and its charge. */
|
|
79
|
+
export interface AskContext {
|
|
80
|
+
/** The tokens the task was lent when its recorded peak was measured. */
|
|
81
|
+
readonly parallelism?: number | undefined;
|
|
82
|
+
/** The Nx task the leaf runs, named in the pool's line for the grant. */
|
|
83
|
+
readonly task?: string | undefined;
|
|
84
|
+
/** The kernel killed the command for memory: it is run again, granted only while nothing else holds a grant. */
|
|
85
|
+
readonly alone?: boolean | undefined;
|
|
86
|
+
}
|
|
75
87
|
/**
|
|
76
88
|
* Ask the immediate parent pool for `want` tokens, charged `memory` bytes: the task's last recorded peak, or null when
|
|
77
|
-
* it has none.
|
|
78
|
-
*
|
|
79
|
-
*
|
|
80
|
-
*
|
|
89
|
+
* it has none. `context.parallelism` is the tokens the task was lent when that peak was measured; the pool charges the
|
|
90
|
+
* peak in proportion to the tokens it lends now when that is more, and lends fewer when the run's memory holds no more.
|
|
91
|
+
* The inherited count does not bound the ask: a pool grants at most what it holds. A granted lease is the child's local
|
|
92
|
+
* allocation: its connection is the grant, and closing it returns only that allocation. An absent endpoint grants
|
|
93
|
+
* nothing and refuses nothing; a connected endpoint's refusal or protocol failure must fail admission rather than run
|
|
94
|
+
* the child beside its parent's count.
|
|
81
95
|
*/
|
|
82
|
-
export declare function takeParentCpuTokens(parent: InheritedCpuPool, want: number, memory: number | null): Promise<TokenGrant>;
|
|
96
|
+
export declare function takeParentCpuTokens(parent: InheritedCpuPool, want: number, memory: number | null, context?: AskContext): Promise<TokenGrant>;
|
|
83
97
|
/**
|
|
84
98
|
* Make a pool of `tokens` that lives while `closed` does: when `closed` resolves the pool refuses
|
|
85
99
|
* grants, ends its child connections, drains its jobserver fifo and removes its directory. A run's own
|
|
86
|
-
* pool charges its grants against `memory
|
|
100
|
+
* pool charges its grants against `memory` and says each grant with what the run's bound holds live
|
|
101
|
+
* (`liveUsage`); a sub-pool charges none (null).
|
|
87
102
|
* Setup failures are values; a non-positive or unsafe count is a programmer bug and throws.
|
|
88
103
|
*/
|
|
89
|
-
export declare function createCpuPool(tokens: number, closed: Promise<void>, memory:
|
|
104
|
+
export declare function createCpuPool(tokens: number, closed: Promise<void>, memory: RunMemory | null, liveUsage?: (bound: string) => LiveUsageReading): Promise<{
|
|
90
105
|
ok: true;
|
|
91
106
|
pool: CpuPool;
|
|
92
107
|
} | {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cpu-pool.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/cpu-pool.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,MAAM,OAAO,CAAC;AAE1B,OAAO,EAAa,KAAK,QAAQ,EAAE,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"cpu-pool.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/cpu-pool.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,MAAM,OAAO,CAAC;AAE1B,OAAO,EAAa,KAAK,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9D,OAAO,EAAE,KAAK,gBAAgB,EAAE,KAAK,SAAS,EAAiB,MAAM,kBAAkB,CAAC;AAKxF;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,yGAAyG;AACzG,eAAO,MAAM,YAAY,0BAA0B,CAAC;AAEpD,6FAA6F;AAC7F,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,sDAAsD;IACtD,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED,yFAAyF;AACzF,MAAM,WAAW,OAAO;IACtB,oFAAoF;IACpF,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7C;;;;OAIG;IACH,KAAK,IAAI,OAAO,CAAC,YAAY,CAAC,CAAC;CAChC;AAED,MAAM,MAAM,YAAY,GAAG;IAAE,EAAE,EAAE,IAAI,CAAA;CAAE,GAAG;IAAE,EAAE,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC;AAgBvE,kHAAkH;AAClH,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;CACrD;AA+DD;;;;GAIG;AACH,wBAAgB,oBAAoB,CAAC,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,gBAAgB,GAAG,IAAI,CAOtG;AAED;;;;;GAKG;AACH,MAAM,MAAM,UAAU,GAClB;IACE,OAAO,EAAE,IAAI,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,SAAS,EAAE,SAAS,GAAG,IAAI,CAAC;IAC5B,OAAO,EAAE,MAAM,IAAI,CAAC;IACpB,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;CACvB,GACD,QAAQ,CAAC;AAIb,kEAAkE;AAClE,MAAM,WAAW,UAAU;IACzB,wEAAwE;IACxE,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1C,yEAAyE;IACzE,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACnC,gHAAgH;IAChH,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;CACtC;AAED;;;;;;;;GAQG;AACH,wBAAsB,mBAAmB,CACvC,MAAM,EAAE,gBAAgB,EACxB,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,MAAM,GAAG,IAAI,EACrB,OAAO,GAAE,UAAe,GACvB,OAAO,CAAC,UAAU,CAAC,CA8DrB;AAsJD;;;;;;GAMG;AACH,wBAAsB,aAAa,CACjC,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,EACrB,MAAM,EAAE,SAAS,GAAG,IAAI,EACxB,SAAS,GAAE,CAAC,KAAK,EAAE,MAAM,KAAK,gBAAgC,GAC7D,OAAO,CAAC;IAAE,EAAE,EAAE,IAAI,CAAC;IAAC,IAAI,EAAE,OAAO,CAAA;CAAE,GAAG;IAAE,EAAE,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CA8XrE"}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import * as _isTypeUint32_1 from "typia/lib/internal/_isTypeUint32";
|
|
2
2
|
import * as _isTypeUint64_1 from "typia/lib/internal/_isTypeUint64";
|
|
3
|
+
import * as _stringLengthLte_1 from "typia/lib/internal/_stringLengthLte";
|
|
3
4
|
import * as _jsonStringifyNumber_1 from "typia/lib/internal/_jsonStringifyNumber";
|
|
4
5
|
import * as _jsonStringifyTail_1 from "typia/lib/internal/_jsonStringifyTail";
|
|
5
6
|
import * as _jsonStringifyString_1 from "typia/lib/internal/_jsonStringifyString";
|
|
@@ -13,6 +14,7 @@ import { promisify } from 'node:util';
|
|
|
13
14
|
import typia from 'typia';
|
|
14
15
|
import { ownedSocketDirectory } from '../../testing.js';
|
|
15
16
|
import { holdLease } from './gateway-lease.js';
|
|
17
|
+
import { readLiveUsage } from './host-memory.js';
|
|
16
18
|
import { CpuLeaseWait, HEARTBEAT_MS } from './lease-wait.js';
|
|
17
19
|
import { admit } from './pool-admission.js';
|
|
18
20
|
import { CPU_TOKENS_ENV, parseInheritedTokens } from './runner-sizing.js';
|
|
@@ -45,8 +47,8 @@ import { CPU_TOKENS_ENV, parseInheritedTokens } from './runner-sizing.js';
|
|
|
45
47
|
export const CPU_POOL_ENV = 'BOUNDED_EXEC_CPU_POOL';
|
|
46
48
|
/** Exactly a request's fields: any other field is an invalid request. */
|
|
47
49
|
const isRequest = (() => {
|
|
48
|
-
const _io0 = (input, _exceptionable = true) => "cpu-pool" === input.op && ("number" === typeof input.want && (_isTypeUint32_1._isTypeUint32(input.want) && 1 <= input.want)) && (undefined === input.memory || "number" === typeof input.memory && _isTypeUint64_1._isTypeUint64(input.memory)) && (2 === Object.keys(input).length || Object.keys(input).every(key => {
|
|
49
|
-
if (["op", "want", "memory"].some(prop => key === prop))
|
|
50
|
+
const _io0 = (input, _exceptionable = true) => "cpu-pool" === input.op && ("number" === typeof input.want && (_isTypeUint32_1._isTypeUint32(input.want) && 1 <= input.want)) && (undefined === input.memory || "number" === typeof input.memory && _isTypeUint64_1._isTypeUint64(input.memory)) && (undefined === input.parallelism || "number" === typeof input.parallelism && (_isTypeUint32_1._isTypeUint32(input.parallelism) && 1 <= input.parallelism)) && (undefined === input.task || "string" === typeof input.task && _stringLengthLte_1._stringLengthLte(input.task, 256)) && (undefined === input.alone || "boolean" === typeof input.alone) && (2 === Object.keys(input).length || Object.keys(input).every(key => {
|
|
51
|
+
if (["op", "want", "memory", "parallelism", "task", "alone"].some(prop => key === prop))
|
|
50
52
|
return true;
|
|
51
53
|
const value = input[key];
|
|
52
54
|
if (undefined === value)
|
|
@@ -59,7 +61,7 @@ const stringifyReply = (() => {
|
|
|
59
61
|
const _ip0 = input => true === input["ok"];
|
|
60
62
|
const _so0 = input => `{"ok":${String(input.ok)},"progress":${String(_jsonStringifyNumber_1._jsonStringifyNumber(input.progress))}}`;
|
|
61
63
|
const _so1 = input => `{"ok":${String(input.ok)},"lease":${"\"" + input.lease + "\""},"server":${String(_jsonStringifyNumber_1._jsonStringifyNumber(input.server))}}`;
|
|
62
|
-
const _so2 = input => `{${_jsonStringifyTail_1._jsonStringifyTail(`"ok":${String(input.ok)},"lease":${"\"" + input.lease + "\""},"tokens":${String(_jsonStringifyNumber_1._jsonStringifyNumber(input.tokens))},${undefined === input.oversized ? "" : `"oversized":${undefined !== input.oversized ? _so3(input.oversized) : undefined}`}`)}}`;
|
|
64
|
+
const _so2 = input => `{${_jsonStringifyTail_1._jsonStringifyTail(`"ok":${String(input.ok)},"lease":${"\"" + input.lease + "\""},"tokens":${String(_jsonStringifyNumber_1._jsonStringifyNumber(input.tokens))},${undefined === input.oversized ? "" : `"oversized":${undefined !== input.oversized ? _so3(input.oversized) : undefined},`}${undefined === input.charged ? "" : `"charged":${undefined !== input.charged ? String(_jsonStringifyNumber_1._jsonStringifyNumber(input.charged)) : undefined}`}`)}}`;
|
|
63
65
|
const _so3 = input => `{"memory":${String(_jsonStringifyNumber_1._jsonStringifyNumber(input.memory))}}`;
|
|
64
66
|
const _so4 = input => `{"ok":${String(input.ok)},"code":${"\"" + input.code + "\""},"error":${_jsonStringifyString_1._jsonStringifyString(input.error)}}`;
|
|
65
67
|
const _su0 = input => (() => {
|
|
@@ -74,13 +76,13 @@ const stringifyReply = (() => {
|
|
|
74
76
|
else
|
|
75
77
|
_throwTypeGuardError_1._throwTypeGuardError({
|
|
76
78
|
method: "typia.json.createStringify",
|
|
77
|
-
expected: "(CpuLeaseProgress | CpuLeaseQueued | { ok: true; lease: \"granted\"; tokens: number; oversized?: Oversized | undefined; } | { ok: false; code: \"invalid-request\"; error: string; })",
|
|
79
|
+
expected: "(CpuLeaseProgress | CpuLeaseQueued | { ok: true; lease: \"granted\"; tokens: number; oversized?: Oversized | undefined; charged?: number | undefined; } | { ok: false; code: \"invalid-request\"; error: string; })",
|
|
78
80
|
value: input
|
|
79
81
|
});
|
|
80
82
|
})();
|
|
81
83
|
const _io0 = input => _ip0(input) && "number" === typeof input.progress;
|
|
82
84
|
const _io1 = input => _ip0(input) && "queued" === input.lease && ("number" === typeof input.server && _isTypeUint32_1._isTypeUint32(input.server));
|
|
83
|
-
const _io2 = input => _ip0(input) && "granted" === input.lease && "number" === typeof input.tokens && (undefined === input.oversized || "object" === typeof input.oversized && null !== input.oversized && _io3(input.oversized));
|
|
85
|
+
const _io2 = input => _ip0(input) && "granted" === input.lease && "number" === typeof input.tokens && (undefined === input.oversized || "object" === typeof input.oversized && null !== input.oversized && _io3(input.oversized)) && (undefined === input.charged || "number" === typeof input.charged);
|
|
84
86
|
const _io3 = input => "number" === typeof input.memory && _isTypeUint64_1._isTypeUint64(input.memory);
|
|
85
87
|
const _io4 = input => false === input.ok && "invalid-request" === input.code && "string" === typeof input.error;
|
|
86
88
|
return input => _su0(input);
|
|
@@ -127,18 +129,24 @@ const isOversized = (() => {
|
|
|
127
129
|
})();
|
|
128
130
|
/**
|
|
129
131
|
* Ask the immediate parent pool for `want` tokens, charged `memory` bytes: the task's last recorded peak, or null when
|
|
130
|
-
* it has none.
|
|
131
|
-
*
|
|
132
|
-
*
|
|
133
|
-
*
|
|
132
|
+
* it has none. `context.parallelism` is the tokens the task was lent when that peak was measured; the pool charges the
|
|
133
|
+
* peak in proportion to the tokens it lends now when that is more, and lends fewer when the run's memory holds no more.
|
|
134
|
+
* The inherited count does not bound the ask: a pool grants at most what it holds. A granted lease is the child's local
|
|
135
|
+
* allocation: its connection is the grant, and closing it returns only that allocation. An absent endpoint grants
|
|
136
|
+
* nothing and refuses nothing; a connected endpoint's refusal or protocol failure must fail admission rather than run
|
|
137
|
+
* the child beside its parent's count.
|
|
134
138
|
*/
|
|
135
|
-
export async function takeParentCpuTokens(parent, want, memory) {
|
|
139
|
+
export async function takeParentCpuTokens(parent, want, memory, context = {}) {
|
|
140
|
+
const { parallelism, task, alone } = context;
|
|
136
141
|
if (!Number.isSafeInteger(want) || want < 1) {
|
|
137
142
|
throw new RangeError(`a child asks its parent pool for a positive count, not ${want} (pool ${parent.tokens})`);
|
|
138
143
|
}
|
|
139
144
|
if (memory !== null && (!Number.isSafeInteger(memory) || memory < 0)) {
|
|
140
145
|
throw new RangeError(`a child is charged a whole number of bytes, not ${memory}`);
|
|
141
146
|
}
|
|
147
|
+
if (parallelism !== undefined && (!Number.isSafeInteger(parallelism) || parallelism < 1)) {
|
|
148
|
+
throw new RangeError(`a child's charge is measured under a positive count of tokens, not ${parallelism}`);
|
|
149
|
+
}
|
|
142
150
|
if (parent.socket.length === 0 || parent.socket.includes('\0')) {
|
|
143
151
|
return {
|
|
144
152
|
granted: false,
|
|
@@ -146,12 +154,18 @@ export async function takeParentCpuTokens(parent, want, memory) {
|
|
|
146
154
|
reason: `the parent CPU pool's endpoint ${JSON.stringify(parent.socket)} is not a socket path`,
|
|
147
155
|
};
|
|
148
156
|
}
|
|
149
|
-
const request =
|
|
157
|
+
const request = {
|
|
158
|
+
op: 'cpu-pool',
|
|
159
|
+
want,
|
|
160
|
+
...(memory === null ? {} : { memory, ...(parallelism === undefined ? {} : { parallelism }) }),
|
|
161
|
+
...(task === undefined ? {} : { task }),
|
|
162
|
+
...(alone === true ? { alone: true } : {}),
|
|
163
|
+
};
|
|
150
164
|
const lease = await holdLease(parent.socket, request, 'the parent CPU pool', new CpuLeaseWait());
|
|
151
165
|
if (!lease.granted) {
|
|
152
166
|
return lease;
|
|
153
167
|
}
|
|
154
|
-
const { tokens, oversized } = lease.answer;
|
|
168
|
+
const { tokens, oversized, charged } = lease.answer;
|
|
155
169
|
if (typeof tokens !== 'number' || !Number.isInteger(tokens) || tokens < 1 || tokens > want) {
|
|
156
170
|
lease.release();
|
|
157
171
|
return {
|
|
@@ -168,7 +182,22 @@ export async function takeParentCpuTokens(parent, want, memory) {
|
|
|
168
182
|
reason: `the parent CPU pool named its memory ${JSON.stringify(oversized)} in a grant, which is no byte count`,
|
|
169
183
|
};
|
|
170
184
|
}
|
|
171
|
-
|
|
185
|
+
if (charged !== undefined && !(typeof charged === 'number' && Number.isSafeInteger(charged) && charged >= 0)) {
|
|
186
|
+
lease.release();
|
|
187
|
+
return {
|
|
188
|
+
granted: false,
|
|
189
|
+
cause: 'other',
|
|
190
|
+
reason: `the parent CPU pool charged ${JSON.stringify(charged)} in a grant, which is no byte count`,
|
|
191
|
+
};
|
|
192
|
+
}
|
|
193
|
+
return {
|
|
194
|
+
granted: true,
|
|
195
|
+
tokens,
|
|
196
|
+
charged: charged ?? null,
|
|
197
|
+
oversized: oversized ?? null,
|
|
198
|
+
release: lease.release,
|
|
199
|
+
closed: lease.closed,
|
|
200
|
+
};
|
|
172
201
|
}
|
|
173
202
|
/** A thrown value's message. */
|
|
174
203
|
const describe = (error) => (error instanceof Error ? error.message : String(error));
|
|
@@ -304,10 +333,11 @@ function watchFifo(path, took, failed) {
|
|
|
304
333
|
/**
|
|
305
334
|
* Make a pool of `tokens` that lives while `closed` does: when `closed` resolves the pool refuses
|
|
306
335
|
* grants, ends its child connections, drains its jobserver fifo and removes its directory. A run's own
|
|
307
|
-
* pool charges its grants against `memory
|
|
336
|
+
* pool charges its grants against `memory` and says each grant with what the run's bound holds live
|
|
337
|
+
* (`liveUsage`); a sub-pool charges none (null).
|
|
308
338
|
* Setup failures are values; a non-positive or unsafe count is a programmer bug and throws.
|
|
309
339
|
*/
|
|
310
|
-
export async function createCpuPool(tokens, closed, memory) {
|
|
340
|
+
export async function createCpuPool(tokens, closed, memory, liveUsage = readLiveUsage) {
|
|
311
341
|
if (!Number.isSafeInteger(tokens) || tokens < 1) {
|
|
312
342
|
throw new RangeError(`a CPU pool holds a positive safe integer of tokens, not ${tokens}`);
|
|
313
343
|
}
|
|
@@ -337,6 +367,9 @@ export async function createCpuPool(tokens, closed, memory) {
|
|
|
337
367
|
let slotLent = false;
|
|
338
368
|
// The bytes the children holding grants are charged, against `memory`.
|
|
339
369
|
let heldMemory = 0;
|
|
370
|
+
// The children holding a grant, and whether one of them runs alone.
|
|
371
|
+
let holders = 0;
|
|
372
|
+
let aloneHolder = false;
|
|
340
373
|
// The readability wait while the queue's head finds no token, else undefined.
|
|
341
374
|
let watch;
|
|
342
375
|
let closing = false;
|
|
@@ -417,20 +450,32 @@ export async function createCpuPool(tokens, closed, memory) {
|
|
|
417
450
|
return;
|
|
418
451
|
}
|
|
419
452
|
}
|
|
420
|
-
const admission = admit(queue, inHand + (slotLent ? 0 : 1), memory === null ? null : { capacity: memory, held: heldMemory });
|
|
453
|
+
const admission = admit(queue, inHand + (slotLent ? 0 : 1), memory === null ? null : { capacity: memory.bytes, held: heldMemory }, { holders, alone: aloneHolder });
|
|
421
454
|
queue.splice(0, admission.grants.length);
|
|
422
|
-
for (const { request, tokens: granted, oversized } of admission.grants) {
|
|
455
|
+
for (const { request, tokens: granted, charge, oversized } of admission.grants) {
|
|
423
456
|
// Fifo bytes first, the implicit slot last.
|
|
424
457
|
const bytes = Math.min(granted, inHand);
|
|
425
458
|
inHand -= bytes;
|
|
426
459
|
const slot = granted > bytes;
|
|
427
460
|
if (slot)
|
|
428
461
|
slotLent = true;
|
|
429
|
-
heldMemory +=
|
|
430
|
-
|
|
431
|
-
request.
|
|
432
|
-
|
|
433
|
-
|
|
462
|
+
heldMemory += charge;
|
|
463
|
+
holders += 1;
|
|
464
|
+
if (request.alone === true)
|
|
465
|
+
aloneHolder = true;
|
|
466
|
+
request.child.phase = { kind: 'holding', bytes, slot, memory: charge, alone: request.alone === true };
|
|
467
|
+
request.child.socket.write(frame(memory === null
|
|
468
|
+
? { ok: true, lease: 'granted', tokens: granted }
|
|
469
|
+
: oversized
|
|
470
|
+
? { ok: true, lease: 'granted', tokens: granted, charged: charge, oversized: { memory: memory.bytes } }
|
|
471
|
+
: { ok: true, lease: 'granted', tokens: granted, charged: charge }));
|
|
472
|
+
if (memory !== null) {
|
|
473
|
+
const live = liveUsage(memory.source);
|
|
474
|
+
const held = live.ok
|
|
475
|
+
? `${live.usage.anon + live.usage.shmem} bytes (anon ${live.usage.anon}, shmem ${live.usage.shmem}, ${live.usage.source})`
|
|
476
|
+
: `unread (${live.reason})`;
|
|
477
|
+
process.stderr.write(`bounded-exec: the CPU pool at ${socketPath} granted ${request.task ?? 'a leaf'} ${granted} of ${request.want} tokens charged ${charge} bytes: holders charged ${heldMemory} of ${memory.bytes} bytes, live anon+shmem ${held}\n`);
|
|
478
|
+
}
|
|
434
479
|
}
|
|
435
480
|
if (admission.waiting === 'cpu') {
|
|
436
481
|
watchForTokens();
|
|
@@ -467,6 +512,9 @@ export async function createCpuPool(tokens, closed, memory) {
|
|
|
467
512
|
if (child.phase.slot)
|
|
468
513
|
slotLent = false;
|
|
469
514
|
heldMemory -= child.phase.memory;
|
|
515
|
+
holders -= 1;
|
|
516
|
+
if (child.phase.alone)
|
|
517
|
+
aloneHolder = false;
|
|
470
518
|
if (!closing)
|
|
471
519
|
returnTokens(child.phase.bytes);
|
|
472
520
|
}
|
|
@@ -528,7 +576,22 @@ export async function createCpuPool(tokens, closed, memory) {
|
|
|
528
576
|
}
|
|
529
577
|
const request = parsed;
|
|
530
578
|
child.phase = { kind: 'queued' };
|
|
531
|
-
|
|
579
|
+
const waiter = {
|
|
580
|
+
child,
|
|
581
|
+
want: request.want,
|
|
582
|
+
memory: request.memory ?? 0,
|
|
583
|
+
parallelism: request.parallelism,
|
|
584
|
+
task: request.task,
|
|
585
|
+
alone: request.alone,
|
|
586
|
+
};
|
|
587
|
+
if (request.alone === true) {
|
|
588
|
+
// A leaf the kernel killed has waited once already: it goes ahead of the leaves that have not run.
|
|
589
|
+
const at = queue.findIndex((waiting) => waiting.alone !== true);
|
|
590
|
+
queue.splice(at === -1 ? queue.length : at, 0, waiter);
|
|
591
|
+
}
|
|
592
|
+
else {
|
|
593
|
+
queue.push(waiter);
|
|
594
|
+
}
|
|
532
595
|
socket.write(frame({ ok: true, lease: 'queued', server: process.pid }));
|
|
533
596
|
drain(0);
|
|
534
597
|
});
|
|
@@ -30,6 +30,11 @@ export interface NxTaskContext {
|
|
|
30
30
|
export interface BoundedExecResult {
|
|
31
31
|
success: boolean;
|
|
32
32
|
terminalOutput: string;
|
|
33
|
+
/**
|
|
34
|
+
* The command died of a SIGKILL that no bound of this executor and no signal it sent caused: the kernel's OOM killer,
|
|
35
|
+
* which sends it to a leaf in a run's cgroup. Set on the last run of the command only when it was killed again.
|
|
36
|
+
*/
|
|
37
|
+
killedForMemory?: boolean;
|
|
33
38
|
}
|
|
34
39
|
export interface ProcessTreeKiller {
|
|
35
40
|
kill(pid: number, signal: NodeJS.Signals): Promise<void>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/executor.ts"],"names":[],"mappings":"AA0BA,OAAO,EAIL,KAAK,gBAAgB,EAItB,MAAM,eAAe,CAAC;AAGvB,OAAO,EAAE,sBAAsB,EAAmB,MAAM,uBAAuB,CAAC;AAahF,OAAO,EAA4C,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAG5F,OAAO,EAOL,KAAK,YAAY,EAClB,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EASL,aAAa,EAEd,MAAM,eAAe,CAAC;AAIvB,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAEtD,OAAO,EAGL,KAAK,eAAe,EAgBrB,MAAM,cAAc,CAAC;AAiBtB,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,wFAAwF;IACxF,MAAM,CAAC,EAAE;QAAE,IAAI,EAAE,eAAe,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC;IACtD,uFAAuF;IACvF,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;CAClC;AAED,qEAAqE;AACrE,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,SAAS,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE;YAAE,IAAI,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;CAC1D;AAED,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,OAAO,CAAC;IACjB,cAAc,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/executor.ts"],"names":[],"mappings":"AA0BA,OAAO,EAIL,KAAK,gBAAgB,EAItB,MAAM,eAAe,CAAC;AAGvB,OAAO,EAAE,sBAAsB,EAAmB,MAAM,uBAAuB,CAAC;AAahF,OAAO,EAA4C,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAG5F,OAAO,EAOL,KAAK,YAAY,EAClB,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EASL,aAAa,EAEd,MAAM,eAAe,CAAC;AAIvB,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAEtD,OAAO,EAGL,KAAK,eAAe,EAgBrB,MAAM,cAAc,CAAC;AAiBtB,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,wFAAwF;IACxF,MAAM,CAAC,EAAE;QAAE,IAAI,EAAE,eAAe,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC;IACtD,uFAAuF;IACvF,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;CAClC;AAED,qEAAqE;AACrE,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,SAAS,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE;YAAE,IAAI,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;CAC1D;AAED,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,OAAO,CAAC;IACjB,cAAc,EAAE,MAAM,CAAC;IACvB;;;OAGG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;CAC3B;AAoBD,MAAM,WAAW,iBAAiB;IAChC,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1D;AAED,8EAA8E;AAC9E,MAAM,MAAM,UAAU,GAAG,IAAI,CAC3B,aAAa,EACb,SAAS,GAAG,SAAS,GAAG,UAAU,GAAG,OAAO,GAAG,QAAQ,GAAG,gBAAgB,CAC3E,CAAC;AAQF;;;;;;GAMG;AACH,wBAAgB,wBAAwB,CACtC,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,MAAM,GAAG,SAAS,EACzB,SAAS,EAAE,MAAM,GAAG,SAAS,EAC7B,SAAS,EAAE,MAAM,GAAG,SAAS,EAC7B,GAAG,EAAE,MAAM,EACX,MAAM,GAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAOzB,GACA,MAAM,GAAG,SAAS,CAiBpB;AAED,MAAM,CAAC,OAAO,UAAU,mBAAmB,CACzC,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,aAAa,GACrB,OAAO,CAAC,iBAAiB,CAAC,CAE5B;AAED;;;GAGG;AACH,wBAAgB,qBAAqB,CACnC,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,kBAAkB,GAC1B,OAAO,CAAC,iBAAiB,CAAC,CAc5B;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAChC,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,kBAAkB,EAC3B,MAAM,EAAE,CAAC,OAAO,EAAE,MAAM,GAAG,SAAS,KAAK,UAAU,GAAG,IAAI,GACzD,OAAO,CAAC,iBAAiB,CAAC,CAsB5B;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAAC,OAAO,EAAE,aAAa,GAAG,kBAAkB,CAiBxE;AAED;;;;;;;;GAQG;AACH,wBAAsB,cAAc,CAClC,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,kBAAkB,EAC3B,MAAM,EAAE,iBAAiB,EACzB,UAAU,EAAE,UAAU,GAAG,IAAI,EAC7B,SAAS,GAAE,gBAAgB,GAAG,IAAW,EACzC,kBAAkB,GAAE,OAAO,sBAA+C,EAC1E,KAAK,GAAE,YAAY,GAAG,IAAW,EACjC,SAAS,CAAC,EAAE,UAAU,CAAC,OAAO,gBAAgB,CAAC,CAAC,CAAC,CAAC,GACjD,OAAO,CAAC,iBAAiB,CAAC,CA2B5B;AA8rCD,wBAAgB,uBAAuB,IAAI,iBAAiB,CAM3D"}
|
|
@@ -142,10 +142,35 @@ export function recordContext(context) {
|
|
|
142
142
|
* command runs on over budget to its own exit, and the run says so.
|
|
143
143
|
*/
|
|
144
144
|
export async function runBoundedExec(options, context, killer, tempVolume, inherited = null, captureDiagnostics = captureHangDiagnostics, owner = null, queueHost) {
|
|
145
|
-
const cwd = resolveCwd(options.cwd, context.root);
|
|
146
145
|
const directory = context.record?.directory ?? null;
|
|
147
|
-
// The task's last recorded peak
|
|
146
|
+
// The task's last recorded peak, and the tokens it was lent when the peak was measured, are its charge in its parent's
|
|
147
|
+
// pool, read once before a run's clear removes them: a command run again after a kill is charged by the record the
|
|
148
|
+
// killed run replaced, not by the peak it reached before it was killed.
|
|
148
149
|
const lastPeak = directory !== null && inherited !== null ? await readLastPeak(directory) : null;
|
|
150
|
+
const run = (alone) => runCommand(options, context, killer, tempVolume, inherited, captureDiagnostics, owner, queueHost, {
|
|
151
|
+
lastPeak,
|
|
152
|
+
alone,
|
|
153
|
+
});
|
|
154
|
+
const first = await run(false);
|
|
155
|
+
if (first.killedForMemory !== true || inherited === null) {
|
|
156
|
+
return first;
|
|
157
|
+
}
|
|
158
|
+
// The kernel killed the leaf: a SIGKILL this executor did not send. It runs once more, alone in its run's pool, so
|
|
159
|
+
// what it was killed beside is not there to be killed with it.
|
|
160
|
+
const note = `cowshed: the kernel killed the command for memory (a SIGKILL this executor did not send${context.record === undefined ? '' : `: ${context.record.task.id}`}); running it once more, alone in its run's pool\n`;
|
|
161
|
+
process.stderr.write(note);
|
|
162
|
+
const again = await run(true);
|
|
163
|
+
return {
|
|
164
|
+
success: again.success,
|
|
165
|
+
terminalOutput: `${first.terminalOutput}${note}${again.terminalOutput}`,
|
|
166
|
+
...(again.killedForMemory === true ? { killedForMemory: true } : {}),
|
|
167
|
+
};
|
|
168
|
+
}
|
|
169
|
+
/** One run of the command, from the pool's grant to its record. */
|
|
170
|
+
async function runCommand(options, context, killer, tempVolume, inherited, captureDiagnostics, owner, queueHost, attempt) {
|
|
171
|
+
const cwd = resolveCwd(options.cwd, context.root);
|
|
172
|
+
const directory = context.record?.directory ?? null;
|
|
173
|
+
const { lastPeak } = attempt;
|
|
149
174
|
// A previous run's verdict and report go first, so neither can be read as this run's.
|
|
150
175
|
if (directory !== null) {
|
|
151
176
|
await clearTaskDirectory(directory);
|
|
@@ -164,6 +189,7 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
164
189
|
settled: false,
|
|
165
190
|
expiry: null,
|
|
166
191
|
forceKillNeeded: false,
|
|
192
|
+
sentSignal: false,
|
|
167
193
|
poolCloseError: null,
|
|
168
194
|
retireError: null,
|
|
169
195
|
noSpace: null,
|
|
@@ -230,8 +256,10 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
230
256
|
if (lastPeak !== null && !lastPeak.ok) {
|
|
231
257
|
return refuseUnstarted(`bounded-exec: ${lastPeak.error}; it is this task's memory charge, so the command does not run\n`);
|
|
232
258
|
}
|
|
233
|
-
// A task with no recorded peak is charged nothing (null) until it has one.
|
|
259
|
+
// A task with no recorded peak is charged nothing (null) until it has one. A peak is charged in proportion to the
|
|
260
|
+
// tokens the run is lent when that is more than the run that measured it was lent.
|
|
234
261
|
const charge = lastPeak?.ok === true ? lastPeak.peakBytes : null;
|
|
262
|
+
const chargeParallelism = lastPeak?.ok === true ? lastPeak.parallelism : null;
|
|
235
263
|
let lease = null;
|
|
236
264
|
let ordinaryTemp = null;
|
|
237
265
|
// Before spawn there is nothing to retire. Once started, only the owner's terminal retirement permits deletion.
|
|
@@ -371,7 +399,11 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
371
399
|
let parentRefusal = null;
|
|
372
400
|
let parentWait;
|
|
373
401
|
if (inherited !== null) {
|
|
374
|
-
const fromParent = await takeParentCpuTokens(inherited, demand.want, charge
|
|
402
|
+
const fromParent = await takeParentCpuTokens(inherited, demand.want, charge, {
|
|
403
|
+
parallelism: chargeParallelism ?? undefined,
|
|
404
|
+
alone: attempt.alone ? true : undefined,
|
|
405
|
+
task: context.record?.task.id,
|
|
406
|
+
});
|
|
375
407
|
if (fromParent.granted) {
|
|
376
408
|
grant = fromParent;
|
|
377
409
|
}
|
|
@@ -393,7 +425,7 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
393
425
|
if (grant !== null) {
|
|
394
426
|
sized = sizeRunner(demand, requested, grant.tokens);
|
|
395
427
|
cpu = { tokens: grant.tokens, want: demand.want, waitedMs };
|
|
396
|
-
appendStderr(`cowshed: cpu-pool wait done elapsed=${waitedMs}ms status=ok tokens=${grant.tokens}/${demand.want} runner=${demand.kind} memory=${charge ?? 'none'}\n`);
|
|
428
|
+
appendStderr(`cowshed: cpu-pool wait done elapsed=${waitedMs}ms status=ok tokens=${grant.tokens}/${demand.want} runner=${demand.kind} memory=${charge ?? 'none'}${grant.charged !== null && grant.charged !== charge ? ` charged=${grant.charged}` : ''}\n`);
|
|
397
429
|
if (grant.oversized !== null) {
|
|
398
430
|
appendStderr(`cowshed: cpu-pool granted while nothing else holds memory: this task's last peak of ${charge} bytes alone exceeds the run's ${grant.oversized.memory} bytes\n`);
|
|
399
431
|
}
|
|
@@ -919,6 +951,7 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
919
951
|
return;
|
|
920
952
|
}
|
|
921
953
|
try {
|
|
954
|
+
state.sentSignal = true;
|
|
922
955
|
await killer.kill(pid, 'SIGTERM');
|
|
923
956
|
}
|
|
924
957
|
catch (error) {
|
|
@@ -1037,6 +1070,7 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
1037
1070
|
// cannot end is said, and a failure of the probe or of a signal is shown.
|
|
1038
1071
|
const forceKill = async (pid) => {
|
|
1039
1072
|
try {
|
|
1073
|
+
state.sentSignal = true;
|
|
1040
1074
|
if (await killProcessGroup(pid, DEADLINE_PATIENCE, (group) => killer.kill(group, 'SIGKILL'))) {
|
|
1041
1075
|
tell(`${describeMembersLeft(pid)}\n`);
|
|
1042
1076
|
}
|
|
@@ -1236,9 +1270,14 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
1236
1270
|
// branch later exited 0: its verdict is an ordinary operational failure, never a successful or silently unbounded run.
|
|
1237
1271
|
const operationalFailure = state.poolCloseError !== null || lifecycleFailure !== null || state.retireError !== null;
|
|
1238
1272
|
await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure, seen, state.lockWaits, peakBytes);
|
|
1273
|
+
// A SIGKILL this executor did not send, to a command that no bound of this executor ended, came from outside it. In a
|
|
1274
|
+
// run's cgroup the one that sends it to a leaf is the kernel's OOM killer. The exit code alone says nothing of who
|
|
1275
|
+
// sent it: a bound's own force-kill ends in 137 too, and is why `sentSignal` is read first.
|
|
1276
|
+
const killedForMemory = !state.sentSignal && state.expiry === null && (exitSignal === 'SIGKILL' || code === 137);
|
|
1239
1277
|
return {
|
|
1240
1278
|
success: !operationalFailure && resultFailure === null && state.expiry === null && code === 0,
|
|
1241
1279
|
terminalOutput: outputChunks.join(''),
|
|
1280
|
+
...(killedForMemory ? { killedForMemory: true } : {}),
|
|
1242
1281
|
};
|
|
1243
1282
|
}
|
|
1244
1283
|
export function createProcessTreeKiller() {
|
|
@@ -18,7 +18,27 @@ export interface HostMemory {
|
|
|
18
18
|
* listing and its read adds nothing, and a root that has exited holds nothing.
|
|
19
19
|
*/
|
|
20
20
|
treeResidentBytes(root: number): Promise<number>;
|
|
21
|
+
/**
|
|
22
|
+
* What the run's bound holds that the kernel cannot take back, read now: its anonymous and shared memory. `bound` is
|
|
23
|
+
* the `source` of the run's memory (`RunMemory`): the cgroup's `memory.max`, whose `memory.stat` is read, or the
|
|
24
|
+
* host's `MemTotal`, whose `AnonPages` and `Shmem` are. The pool says it on every grant, so what the leaves are not
|
|
25
|
+
* charged for (the baseline, a tmpfs) and what a suite holds beyond its record are measured, not guessed.
|
|
26
|
+
*/
|
|
27
|
+
liveUsage(bound: string): LiveUsageReading;
|
|
28
|
+
}
|
|
29
|
+
/** What a bound holds live, in bytes, and the file it was read from. */
|
|
30
|
+
export interface LiveUsage {
|
|
31
|
+
readonly anon: number;
|
|
32
|
+
readonly shmem: number;
|
|
33
|
+
readonly source: string;
|
|
21
34
|
}
|
|
35
|
+
export type LiveUsageReading = {
|
|
36
|
+
readonly ok: true;
|
|
37
|
+
readonly usage: LiveUsage;
|
|
38
|
+
} | {
|
|
39
|
+
readonly ok: false;
|
|
40
|
+
readonly reason: string;
|
|
41
|
+
};
|
|
22
42
|
/** The run's memory in bytes, and where it was read: the file or sysctl that holds it. */
|
|
23
43
|
export interface RunMemory {
|
|
24
44
|
readonly bytes: number;
|
|
@@ -51,4 +71,6 @@ export declare class RunMemoryUnreadable extends Error {
|
|
|
51
71
|
export declare const HOST_MEMORY: HostMemory | null;
|
|
52
72
|
/** The run's memory on this platform, or why it cannot be read. */
|
|
53
73
|
export declare function readRunMemory(): Promise<RunMemoryReading>;
|
|
74
|
+
/** The run's bound live on this platform, or why it cannot be read: a pool says it, never leaves it out. */
|
|
75
|
+
export declare function readLiveUsage(bound: string): LiveUsageReading;
|
|
54
76
|
//# sourceMappingURL=host-memory.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"host-memory.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/host-memory.ts"],"names":[],"mappings":"AAGA;;;;GAIG;AACH,MAAM,WAAW,UAAU;IACzB,mEAAmE;IACnE,aAAa,IAAI,OAAO,CAAC,gBAAgB,CAAC,CAAC;IAC3C;;;OAGG;IACH,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,MAAM,CAAC;IACtD;;;;;OAKG;IACH,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"host-memory.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/host-memory.ts"],"names":[],"mappings":"AAGA;;;;GAIG;AACH,MAAM,WAAW,UAAU;IACzB,mEAAmE;IACnE,aAAa,IAAI,OAAO,CAAC,gBAAgB,CAAC,CAAC;IAC3C;;;OAGG;IACH,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,MAAM,CAAC;IACtD;;;;;OAKG;IACH,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACjD;;;;;OAKG;IACH,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,gBAAgB,CAAC;CAC5C;AAED,wEAAwE;AACxE,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,MAAM,gBAAgB,GACxB;IAAE,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,SAAS,CAAA;CAAE,GAChD;IAAE,QAAQ,CAAC,EAAE,EAAE,KAAK,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC;AAEpD,0FAA0F;AAC1F,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED,4FAA4F;AAC5F,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,SAAS,EAAE,iBAAiB,CAAC;IACtC;;;OAGG;IACH,QAAQ,CAAC,IAAI,EAAE,YAAY,GAAG,WAAW,GAAG,WAAW,GAAG,sBAAsB,CAAC;IACjF,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,MAAM,gBAAgB,GACxB;IAAE,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAA;CAAE,GACjD;IAAE,QAAQ,CAAC,EAAE,EAAE,KAAK,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,cAAc,CAAA;CAAE,CAAC;AAE3D,sFAAsF;AACtF,qBAAa,mBAAoB,SAAQ,KAAK;IAChC,QAAQ,CAAC,KAAK,EAAE,cAAc;IAA1C,YAAqB,KAAK,EAAE,cAAc,EAKzC;CACF;AAID,kGAAkG;AAClG,eAAO,MAAM,WAAW,EAAE,UAAU,GAAG,IAA0C,CAAC;AAElF,mEAAmE;AACnE,wBAAgB,aAAa,IAAI,OAAO,CAAC,gBAAgB,CAAC,CAazD;AAED,4GAA4G;AAC5G,wBAAgB,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,gBAAgB,CAI7D"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"host-memory.darwin.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/host-memory.darwin.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAoB,MAAM,kBAAkB,CAAC;AAqIrE,eAAO,MAAM,YAAY,EAAE,
|
|
1
|
+
{"version":3,"file":"host-memory.darwin.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/host-memory.darwin.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAoB,MAAM,kBAAkB,CAAC;AAqIrE,eAAO,MAAM,YAAY,EAAE,UAM1B,CAAC"}
|
|
@@ -99,4 +99,10 @@ async function treeResidentBytes(root) {
|
|
|
99
99
|
}
|
|
100
100
|
return total;
|
|
101
101
|
}
|
|
102
|
-
export const darwinMemory = {
|
|
102
|
+
export const darwinMemory = {
|
|
103
|
+
readRunMemory,
|
|
104
|
+
peakBytes: (maxRss) => maxRss,
|
|
105
|
+
treeResidentBytes,
|
|
106
|
+
// macOS has no cgroup, and its anonymous and shared pages are not read from a file the pool may open.
|
|
107
|
+
liveUsage: () => ({ ok: false, reason: 'macOS keeps no anonymous and shared memory of a run to read' }),
|
|
108
|
+
};
|