@smoothbricks/nx-plugin 0.7.12 → 0.7.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +78 -24
- package/dist/.build-inputs.json +20 -12
- package/dist/built-output-packages.d.ts +27 -0
- package/dist/built-output-packages.d.ts.map +1 -0
- package/dist/built-output-packages.js +97 -0
- package/dist/cross-check-policy.d.ts +13 -1
- package/dist/cross-check-policy.d.ts.map +1 -1
- package/dist/cross-check-policy.js +13 -1
- package/dist/executors/bounded-exec/admission-queue.d.ts +68 -0
- package/dist/executors/bounded-exec/admission-queue.d.ts.map +1 -0
- package/dist/executors/bounded-exec/admission-queue.js +131 -0
- package/dist/executors/bounded-exec/cpu-grant.d.ts +14 -0
- package/dist/executors/bounded-exec/cpu-grant.d.ts.map +1 -0
- package/dist/executors/bounded-exec/cpu-grant.js +18 -0
- package/dist/executors/bounded-exec/cpu-pool.d.ts +1 -14
- package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool.js +2 -49
- package/dist/executors/bounded-exec/executor.d.ts +5 -12
- package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
- package/dist/executors/bounded-exec/executor.js +51 -154
- package/dist/executors/bounded-exec/memory-retry.d.ts +39 -0
- package/dist/executors/bounded-exec/memory-retry.d.ts.map +1 -0
- package/dist/executors/bounded-exec/memory-retry.js +33 -0
- package/dist/executors/bounded-exec/parent-grant.d.ts +43 -0
- package/dist/executors/bounded-exec/parent-grant.d.ts.map +1 -0
- package/dist/executors/bounded-exec/parent-grant.js +37 -0
- package/dist/executors/bounded-exec/peak-charge.d.ts +17 -0
- package/dist/executors/bounded-exec/peak-charge.d.ts.map +1 -0
- package/dist/executors/bounded-exec/peak-charge.js +20 -0
- package/dist/executors/bounded-exec/pool-environment.d.ts +25 -0
- package/dist/executors/bounded-exec/pool-environment.d.ts.map +1 -0
- package/dist/executors/bounded-exec/pool-environment.js +34 -0
- package/dist/executors/bounded-exec/run-cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/run-cpu-pool.js +1 -1
- package/dist/executors/bounded-exec/verdict.d.ts +3 -18
- package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
- package/dist/executors/bounded-exec/verdict.js +4 -25
- package/dist/index.d.ts +4 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +62 -21
- package/dist/managed-files/ci-workflow.d.ts +29 -5
- package/dist/managed-files/ci-workflow.d.ts.map +1 -1
- package/dist/managed-files/ci-workflow.js +111 -16
- package/dist/managed-files/context.d.ts.map +1 -1
- package/dist/managed-files/context.js +1 -0
- package/dist/managed-files/files.d.ts +3 -1
- package/dist/managed-files/files.d.ts.map +1 -1
- package/dist/managed-files/files.js +2 -0
- package/dist/managed-files/publish-workflow.d.ts +6 -1
- package/dist/managed-files/publish-workflow.d.ts.map +1 -1
- package/dist/managed-files/publish-workflow.js +16 -11
- package/dist/owned-installation.d.ts.map +1 -1
- package/dist/owned-installation.js +48 -14
- package/dist/typecheck-route.d.ts +28 -0
- package/dist/typecheck-route.d.ts.map +1 -0
- package/dist/typecheck-route.js +109 -0
- package/dist/workspace-manifest.d.ts +50 -5
- package/dist/workspace-manifest.d.ts.map +1 -1
- package/executors.json +10 -0
- package/managed/raw/patches/ttsc@0.34.1.patch +93 -8
- package/managed/raw/tooling/direnv/apple-developer.sh +8 -1
- package/managed/raw/tooling/direnv/compiler-variables.sh +1 -1
- package/managed/raw/tooling/direnv/github-actions-bootstrap.sh +4 -2
- package/managed/raw/tooling/direnv/secret-references.ts +19 -5
- package/managed/raw/tooling/git-hooks/pre-push.sh +20 -10
- package/managed/templates/github/actions/forgejo-artifact.cjs +9 -9
- package/managed/templates/github/actions/setup-devenv/action.yml +53 -0
- package/package.json +2 -2
- package/src/built-output-packages.ts +125 -0
- package/src/cross-check-policy.ts +14 -1
- package/src/executors/bounded-exec/admission-queue.ts +163 -0
- package/src/executors/bounded-exec/cpu-grant.ts +32 -0
- package/src/executors/bounded-exec/cpu-pool.ts +2 -33
- package/src/executors/bounded-exec/executor.ts +60 -201
- package/src/executors/bounded-exec/memory-retry.ts +63 -0
- package/src/executors/bounded-exec/parent-grant.ts +75 -0
- package/src/executors/bounded-exec/peak-charge.ts +25 -0
- package/src/executors/bounded-exec/pool-environment.ts +45 -0
- package/src/executors/bounded-exec/run-cpu-pool.ts +2 -1
- package/src/executors/bounded-exec/verdict.ts +7 -35
- package/src/index.ts +70 -21
- package/src/managed-files/ci-workflow.ts +143 -16
- package/src/managed-files/context.ts +1 -0
- package/src/managed-files/files.ts +5 -0
- package/src/managed-files/publish-workflow.ts +26 -10
- package/src/owned-installation.ts +63 -14
- package/src/typecheck-route.ts +64 -0
- package/src/workspace-manifest.ts +56 -5
- package/test-timings.json +141 -139
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
import { admissionWaitMs, NO_ADMISSION_WAIT, stepAdmissionWait } from './bounds.js';
|
|
2
|
+
import { HANG_CAPTURE_MS } from './hang-diagnostics.js';
|
|
3
|
+
import { MBX_QUEUE_POLL_MS, snapshotMbxQueue } from './mbx-queue.js';
|
|
4
|
+
import { describeAdmissionQueue } from './verdict.js';
|
|
5
|
+
/** One run's view of the admission queue its leaf waits in. */
|
|
6
|
+
export class AdmissionObservation {
|
|
7
|
+
host;
|
|
8
|
+
/** mbx granted the leaf its cores. */
|
|
9
|
+
admitted = false;
|
|
10
|
+
wait = NO_ADMISSION_WAIT;
|
|
11
|
+
requests = [];
|
|
12
|
+
/**
|
|
13
|
+
* Why the last admission snapshot failed, while the one after it has not succeeded; null otherwise. A snapshot only
|
|
14
|
+
* decides whether waiting on mbx admission is credited against the idle bound, so a failed one withholds that credit
|
|
15
|
+
* and ends nothing: the bounds decide, and an idle expiry names it.
|
|
16
|
+
*/
|
|
17
|
+
failure = null;
|
|
18
|
+
/** When the last successful snapshot completed (`Date.now()`): credit runs to it and no further. */
|
|
19
|
+
confirmedAt = null;
|
|
20
|
+
abort = new AbortController();
|
|
21
|
+
timer;
|
|
22
|
+
snapshot = null;
|
|
23
|
+
lastDiagnostic = '';
|
|
24
|
+
lastDiagnosticAt = null;
|
|
25
|
+
lastSnapshotDiagnostics = '';
|
|
26
|
+
constructor(host) {
|
|
27
|
+
this.host = host;
|
|
28
|
+
}
|
|
29
|
+
/** Why the last snapshot failed, while the one after it has not succeeded. */
|
|
30
|
+
get unconfirmed() {
|
|
31
|
+
return this.failure;
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* A reserved leaf's own request on its run's endpoint is its admission wait; a Cargo leaf's comes from mbx's report.
|
|
35
|
+
* `reserved` is whether the leaf holds a reservation of its own.
|
|
36
|
+
*/
|
|
37
|
+
queueWait(reserved) {
|
|
38
|
+
return this.admitted || (this.requests.length === 0 && !reserved)
|
|
39
|
+
? null
|
|
40
|
+
: { waitedMs: admissionWaitMs(this.wait, performance.now()), requests: this.requests };
|
|
41
|
+
}
|
|
42
|
+
/** A reserved leaf's request began to wait on its run's endpoint. */
|
|
43
|
+
waiting() {
|
|
44
|
+
this.wait = stepAdmissionWait(this.wait, { kind: 'snapshot', at: performance.now(), waiting: true });
|
|
45
|
+
}
|
|
46
|
+
/** Only mbx's activity report says whether a Cargo leaf's compiler or test waits in the pool: poll it. */
|
|
47
|
+
poll() {
|
|
48
|
+
this.timer = setInterval(() => {
|
|
49
|
+
void this.refresh(false);
|
|
50
|
+
}, MBX_QUEUE_POLL_MS);
|
|
51
|
+
void this.refresh(false);
|
|
52
|
+
}
|
|
53
|
+
/** The run ended: the wait stops counting and no snapshot is taken or credited again. */
|
|
54
|
+
stop() {
|
|
55
|
+
this.wait = stepAdmissionWait(this.wait, { kind: 'stop', at: performance.now() });
|
|
56
|
+
clearInterval(this.timer);
|
|
57
|
+
this.abort.abort();
|
|
58
|
+
}
|
|
59
|
+
/** The snapshot in flight, if any, has finished. */
|
|
60
|
+
async idle() {
|
|
61
|
+
await this.snapshot;
|
|
62
|
+
}
|
|
63
|
+
unconfirm(detail) {
|
|
64
|
+
if (this.host.over() || this.abort.signal.aborted)
|
|
65
|
+
return;
|
|
66
|
+
if (this.failure !== detail) {
|
|
67
|
+
this.host.tell(`bounded-exec: mbx admission snapshot failed: ${detail}; admission is unconfirmed, so this wait earns no idle credit until a snapshot succeeds\n`);
|
|
68
|
+
}
|
|
69
|
+
this.failure = detail;
|
|
70
|
+
this.requests = [];
|
|
71
|
+
this.wait = stepAdmissionWait(this.wait, { kind: 'snapshot', at: performance.now(), waiting: false });
|
|
72
|
+
// The wait since the last confirmed snapshot is unconfirmed: the idle bound resumes from that snapshot.
|
|
73
|
+
this.host.step({
|
|
74
|
+
kind: 'admission-queue',
|
|
75
|
+
at: this.confirmedAt ?? Date.now(),
|
|
76
|
+
waiting: false,
|
|
77
|
+
testing: this.host.testing(),
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
/** Take a snapshot; `fresh` drains an older poll first. */
|
|
81
|
+
async refresh(fresh) {
|
|
82
|
+
if (fresh && this.snapshot !== null)
|
|
83
|
+
await this.snapshot;
|
|
84
|
+
const pid = this.host.pid();
|
|
85
|
+
if (this.host.over() || this.abort.signal.aborted || pid === undefined)
|
|
86
|
+
return;
|
|
87
|
+
this.snapshot ??= (async () => {
|
|
88
|
+
const snapshot = await snapshotMbxQueue(pid, this.host.environment(), this.abort.signal, this.host.queueHost);
|
|
89
|
+
if (this.host.over() || this.abort.signal.aborted)
|
|
90
|
+
return;
|
|
91
|
+
if (!snapshot.ok) {
|
|
92
|
+
this.unconfirm(snapshot.detail);
|
|
93
|
+
return;
|
|
94
|
+
}
|
|
95
|
+
const diagnostics = snapshot.diagnostics?.join('; ') ?? '';
|
|
96
|
+
if (diagnostics !== '' && diagnostics !== this.lastSnapshotDiagnostics) {
|
|
97
|
+
this.host.tell(`bounded-exec: ${diagnostics}\n`);
|
|
98
|
+
}
|
|
99
|
+
this.lastSnapshotDiagnostics = diagnostics;
|
|
100
|
+
const waiting = snapshot.requests.length > 0;
|
|
101
|
+
const observedAt = Date.now();
|
|
102
|
+
const observedWaitAt = performance.now();
|
|
103
|
+
this.wait = stepAdmissionWait(this.wait, { kind: 'snapshot', at: observedWaitAt, waiting });
|
|
104
|
+
this.failure = null;
|
|
105
|
+
this.confirmedAt = observedAt;
|
|
106
|
+
if (waiting) {
|
|
107
|
+
this.requests = snapshot.requests;
|
|
108
|
+
const positions = JSON.stringify(snapshot.requests);
|
|
109
|
+
if (positions !== this.lastDiagnostic ||
|
|
110
|
+
this.lastDiagnosticAt === null ||
|
|
111
|
+
Date.now() - this.lastDiagnosticAt >= HANG_CAPTURE_MS) {
|
|
112
|
+
const waitedMs = admissionWaitMs(this.wait, observedWaitAt);
|
|
113
|
+
this.host.tell(`cowshed: mbx ${describeAdmissionQueue({ waitedMs, requests: snapshot.requests })}\n`);
|
|
114
|
+
this.lastDiagnostic = positions;
|
|
115
|
+
this.lastDiagnosticAt = Date.now();
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
else {
|
|
119
|
+
this.lastDiagnostic = '';
|
|
120
|
+
}
|
|
121
|
+
this.host.step({ kind: 'admission-queue', at: observedAt, waiting, testing: this.host.testing() });
|
|
122
|
+
})()
|
|
123
|
+
.catch((error) => {
|
|
124
|
+
this.unconfirm(error instanceof Error ? error.message : String(error));
|
|
125
|
+
})
|
|
126
|
+
.finally(() => {
|
|
127
|
+
this.snapshot = null;
|
|
128
|
+
});
|
|
129
|
+
await this.snapshot;
|
|
130
|
+
}
|
|
131
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The CPU grant a run's record names (verdict.ts keeps the record; this is what it says of the pool). The tokens are the
|
|
3
|
+
* parallelism the record's peak was measured under, which the task's next run reads (peak-charge.ts) to charge that
|
|
4
|
+
* peak in proportion to the parallelism it is lent. It is scheduling metadata: a record that names none, or names one
|
|
5
|
+
* that is malformed, changes no verdict a passing run records.
|
|
6
|
+
*/
|
|
7
|
+
export interface CpuGrantRecord {
|
|
8
|
+
readonly tokens: number;
|
|
9
|
+
readonly want: number;
|
|
10
|
+
readonly waitedMs: number;
|
|
11
|
+
}
|
|
12
|
+
/** What a record's `cpu` field holds: no grant, a grant, or something that is no grant. */
|
|
13
|
+
export declare function parseCpuGrant(value: unknown): CpuGrantRecord | undefined | 'malformed';
|
|
14
|
+
//# sourceMappingURL=cpu-grant.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cpu-grant.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/cpu-grant.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;CAC3B;AAED,2FAA2F;AAC3F,wBAAgB,aAAa,CAAC,KAAK,EAAE,OAAO,GAAG,cAAc,GAAG,SAAS,GAAG,WAAW,CAkBtF"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/** What a record's `cpu` field holds: no grant, a grant, or something that is no grant. */
|
|
2
|
+
export function parseCpuGrant(value) {
|
|
3
|
+
if (value === undefined) {
|
|
4
|
+
return undefined;
|
|
5
|
+
}
|
|
6
|
+
if (typeof value !== 'object' || value === null) {
|
|
7
|
+
return 'malformed';
|
|
8
|
+
}
|
|
9
|
+
const tokens = Reflect.get(value, 'tokens');
|
|
10
|
+
const want = Reflect.get(value, 'want');
|
|
11
|
+
const waitedMs = Reflect.get(value, 'waitedMs');
|
|
12
|
+
if (!(typeof tokens === 'number' && Number.isSafeInteger(tokens) && tokens >= 1) ||
|
|
13
|
+
!(typeof want === 'number' && Number.isSafeInteger(want) && want >= 1) ||
|
|
14
|
+
!(typeof waitedMs === 'number' && Number.isFinite(waitedMs) && waitedMs >= 0)) {
|
|
15
|
+
return 'malformed';
|
|
16
|
+
}
|
|
17
|
+
return { tokens, want, waitedMs };
|
|
18
|
+
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import typia from 'typia';
|
|
2
2
|
import { type Unleased } from './gateway-lease.js';
|
|
3
3
|
import { type LiveUsageReading, type RunMemory } from './host-memory.js';
|
|
4
|
+
import { type InheritedCpuPool } from './pool-environment.js';
|
|
4
5
|
/**
|
|
5
6
|
* A CPU pool: the tokens a process lends the runners it starts, held live rather than inherited as a
|
|
6
7
|
* bare count (specs/cowshed/05_gateway.md, "Host scheduling", "Parallelism inside a job"). An Nx run
|
|
@@ -26,14 +27,6 @@ import { type LiveUsageReading, type RunMemory } from './host-memory.js';
|
|
|
26
27
|
* which the holder is not using while it waits on the child it lends it to; release writes the bytes back and
|
|
27
28
|
* frees the slot. A Cargo reading the fifo and a nested runner holding a socket grant so share one count.
|
|
28
29
|
*/
|
|
29
|
-
/** The environment name a child reads its parent's pool endpoint from; its count is `CPU_TOKENS_ENV`. */
|
|
30
|
-
export declare const CPU_POOL_ENV = "BOUNDED_EXEC_CPU_POOL";
|
|
31
|
-
/** An inherited pool descriptor; only a grant from its endpoint proves a live allocation. */
|
|
32
|
-
export interface InheritedCpuPool {
|
|
33
|
-
readonly socket: string;
|
|
34
|
-
/** The count the parent names beside its endpoint. */
|
|
35
|
-
readonly tokens: number;
|
|
36
|
-
}
|
|
37
30
|
/** A running pool: the environment a child inherits to reach it, and how to close it. */
|
|
38
31
|
export interface CpuPool {
|
|
39
32
|
/** The endpoint, the count the pool actually holds, and its make jobserver fifo. */
|
|
@@ -55,12 +48,6 @@ export type CloseOutcome = {
|
|
|
55
48
|
export interface Oversized {
|
|
56
49
|
readonly memory: number & typia.tags.Type<'uint64'>;
|
|
57
50
|
}
|
|
58
|
-
/**
|
|
59
|
-
* The inherited descriptor, or null when malformed or absent. A missing or bare count, or an empty
|
|
60
|
-
* or NUL-bearing endpoint, proves no allocation; the count is a positive whole number. A valid
|
|
61
|
-
* descriptor still needs a live endpoint grant.
|
|
62
|
-
*/
|
|
63
|
-
export declare function readInheritedCpuPool(environment: Readonly<NodeJS.ProcessEnv>): InheritedCpuPool | null;
|
|
64
51
|
/**
|
|
65
52
|
* Tokens a parent pool lent, held until released, or why there are none. `charged` is the bytes the pool charged the
|
|
66
53
|
* grant, null from a pool that charges none. `oversized` names the run's memory when the child's charge alone exceeds
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cpu-pool.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/cpu-pool.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,MAAM,OAAO,CAAC;AAE1B,OAAO,EAAa,KAAK,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9D,OAAO,EAAE,KAAK,gBAAgB,EAAE,KAAK,SAAS,EAAiB,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"cpu-pool.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/cpu-pool.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,MAAM,OAAO,CAAC;AAE1B,OAAO,EAAa,KAAK,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9D,OAAO,EAAE,KAAK,gBAAgB,EAAE,KAAK,SAAS,EAAiB,MAAM,kBAAkB,CAAC;AAGxF,OAAO,EAAE,KAAK,gBAAgB,EAAmB,MAAM,uBAAuB,CAAC;AAE/E;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,yFAAyF;AACzF,MAAM,WAAW,OAAO;IACtB,oFAAoF;IACpF,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7C;;;;OAIG;IACH,KAAK,IAAI,OAAO,CAAC,YAAY,CAAC,CAAC;CAChC;AAED,MAAM,MAAM,YAAY,GAAG;IAAE,EAAE,EAAE,IAAI,CAAA;CAAE,GAAG;IAAE,EAAE,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC;AAgBvE,kHAAkH;AAClH,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;CACrD;AA+DD;;;;;GAKG;AACH,MAAM,MAAM,UAAU,GAClB;IACE,OAAO,EAAE,IAAI,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,SAAS,EAAE,SAAS,GAAG,IAAI,CAAC;IAC5B,OAAO,EAAE,MAAM,IAAI,CAAC;IACpB,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;CACvB,GACD,QAAQ,CAAC;AAIb,kEAAkE;AAClE,MAAM,WAAW,UAAU;IACzB,wEAAwE;IACxE,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1C,yEAAyE;IACzE,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACnC,gHAAgH;IAChH,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;CACtC;AAED;;;;;;;;GAQG;AACH,wBAAsB,mBAAmB,CACvC,MAAM,EAAE,gBAAgB,EACxB,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,MAAM,GAAG,IAAI,EACrB,OAAO,GAAE,UAAe,GACvB,OAAO,CAAC,UAAU,CAAC,CA8DrB;AAsJD;;;;;;GAMG;AACH,wBAAsB,aAAa,CACjC,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,EACrB,MAAM,EAAE,SAAS,GAAG,IAAI,EACxB,SAAS,GAAE,CAAC,KAAK,EAAE,MAAM,KAAK,gBAAgC,GAC7D,OAAO,CAAC;IAAE,EAAE,EAAE,IAAI,CAAC;IAAC,IAAI,EAAE,OAAO,CAAA;CAAE,GAAG;IAAE,EAAE,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CAuXrE"}
|
|
@@ -17,34 +17,7 @@ import { holdLease } from './gateway-lease.js';
|
|
|
17
17
|
import { readLiveUsage } from './host-memory.js';
|
|
18
18
|
import { CpuLeaseWait, HEARTBEAT_MS } from './lease-wait.js';
|
|
19
19
|
import { admit } from './pool-admission.js';
|
|
20
|
-
import {
|
|
21
|
-
/**
|
|
22
|
-
* A CPU pool: the tokens a process lends the runners it starts, held live rather than inherited as a
|
|
23
|
-
* bare count (specs/cowshed/05_gateway.md, "Host scheduling", "Parallelism inside a job"). An Nx run
|
|
24
|
-
* serves one on a thread of its own (`run-cpu-pool.ts`), and every granted bounded-exec serves a
|
|
25
|
-
* sub-pool of its grant to its own descendants, so the runners of one build tree together stay
|
|
26
|
-
* within the tree's count.
|
|
27
|
-
*
|
|
28
|
-
* The pool is a private Unix socket in a directory the process owns: the directory 0700, the socket
|
|
29
|
-
* 0600. A child asks it for a count with one request line, in `holdLease`'s framing. The pool
|
|
30
|
-
* answers `queued` at once and `granted` when the child reaches the head of its FIFO queue. A grant
|
|
31
|
-
* lasts while its connection does, so a child that dies returns its tokens with its socket. A pool
|
|
32
|
-
* grants only from its own count, and a pool whose upstream grant has gone grants nothing.
|
|
33
|
-
*
|
|
34
|
-
* A run's own pool also charges memory (`pool-admission.ts`; specs/cowshed/05_gateway.md, "Without mbx, every Nx run
|
|
35
|
-
* budgets itself"). A request names its task's last recorded peak as `memory`, in bytes, and is granted only while
|
|
36
|
-
* the charges of the leaves holding grants and its own fit the run's memory; a request that names none is charged
|
|
37
|
-
* nothing, as mbx's omitted `--memory` is. A sub-pool charges no memory: its holder's charge covers its tree.
|
|
38
|
-
*
|
|
39
|
-
* The same count is a GNU make jobserver fifo beside the socket, mode 0600, which Cargo, its build
|
|
40
|
-
* scripts and every nested `cargo` read through `CARGO_MAKEFLAGS`/`MAKEFLAGS` (specs/cowshed/05_gateway.md,
|
|
41
|
-
* "Cargo reads its holder's jobserver"). A pool of `g` holds `g − 1` token bytes there; the holder's own
|
|
42
|
-
* command holds the implicit first token. A socket grant takes fifo bytes first and that implicit slot last,
|
|
43
|
-
* which the holder is not using while it waits on the child it lends it to; release writes the bytes back and
|
|
44
|
-
* frees the slot. A Cargo reading the fifo and a nested runner holding a socket grant so share one count.
|
|
45
|
-
*/
|
|
46
|
-
/** The environment name a child reads its parent's pool endpoint from; its count is `CPU_TOKENS_ENV`. */
|
|
47
|
-
export const CPU_POOL_ENV = 'BOUNDED_EXEC_CPU_POOL';
|
|
20
|
+
import { poolEnvironment } from './pool-environment.js';
|
|
48
21
|
/** Exactly a request's fields: any other field is an invalid request. */
|
|
49
22
|
const isRequest = (() => {
|
|
50
23
|
const _io0 = (input, _exceptionable = true) => "cpu-pool" === input.op && ("number" === typeof input.want && (_isTypeUint32_1._isTypeUint32(input.want) && 1 <= input.want)) && (undefined === input.memory || "number" === typeof input.memory && _isTypeUint64_1._isTypeUint64(input.memory)) && (undefined === input.parallelism || "number" === typeof input.parallelism && (_isTypeUint32_1._isTypeUint32(input.parallelism) && 1 <= input.parallelism)) && (undefined === input.task || "string" === typeof input.task && _stringLengthLte_1._stringLengthLte(input.task, 256)) && (undefined === input.alone || "boolean" === typeof input.alone) && (2 === Object.keys(input).length || Object.keys(input).every(key => {
|
|
@@ -103,19 +76,6 @@ const CLOSED_WHILE_MADE = 'the grant that lends this CPU pool closed while the p
|
|
|
103
76
|
const run = promisify(execFile);
|
|
104
77
|
/** One framed reply line. The newline is the framing; every reply goes through here. */
|
|
105
78
|
const frame = (reply) => `${stringifyReply(reply)}\n`;
|
|
106
|
-
/**
|
|
107
|
-
* The inherited descriptor, or null when malformed or absent. A missing or bare count, or an empty
|
|
108
|
-
* or NUL-bearing endpoint, proves no allocation; the count is a positive whole number. A valid
|
|
109
|
-
* descriptor still needs a live endpoint grant.
|
|
110
|
-
*/
|
|
111
|
-
export function readInheritedCpuPool(environment) {
|
|
112
|
-
const socket = environment[CPU_POOL_ENV];
|
|
113
|
-
const tokens = parseInheritedTokens(environment[CPU_TOKENS_ENV]);
|
|
114
|
-
if (socket === undefined || socket.length === 0 || socket.includes('\0') || tokens === null) {
|
|
115
|
-
return null;
|
|
116
|
-
}
|
|
117
|
-
return { socket, tokens };
|
|
118
|
-
}
|
|
119
79
|
const isOversized = (() => {
|
|
120
80
|
const _io0 = (input, _exceptionable = true) => "number" === typeof input.memory && _isTypeUint64_1._isTypeUint64(input.memory) && (1 === Object.keys(input).length || Object.keys(input).every(key => {
|
|
121
81
|
if (["memory"].some(prop => key === prop))
|
|
@@ -693,17 +653,10 @@ export async function createCpuPool(tokens, closed, memory, liveUsage = readLive
|
|
|
693
653
|
if (closing) {
|
|
694
654
|
return abort(CLOSED_WHILE_MADE);
|
|
695
655
|
}
|
|
696
|
-
// Cargo and make read the fifo from here; `-j` with a jobserver names no count of its own.
|
|
697
|
-
const jobserverFlags = `-j --jobserver-auth=fifo:${fifoPath}`;
|
|
698
656
|
return {
|
|
699
657
|
ok: true,
|
|
700
658
|
pool: {
|
|
701
|
-
environment:
|
|
702
|
-
[CPU_POOL_ENV]: socketPath,
|
|
703
|
-
[CPU_TOKENS_ENV]: String(tokens),
|
|
704
|
-
CARGO_MAKEFLAGS: jobserverFlags,
|
|
705
|
-
MAKEFLAGS: jobserverFlags,
|
|
706
|
-
},
|
|
659
|
+
environment: poolEnvironment(socketPath, tokens, fifoPath),
|
|
707
660
|
close: shutdown,
|
|
708
661
|
},
|
|
709
662
|
};
|
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
import { type
|
|
1
|
+
import { type QueueHost } from './admission-queue.js';
|
|
2
2
|
import { captureHangDiagnostics } from './hang-diagnostics.js';
|
|
3
|
-
import {
|
|
3
|
+
import { type RunOutcome } from './memory-retry.js';
|
|
4
4
|
import { type SubtreeOwner } from './owned-subtree.js';
|
|
5
|
+
import { type InheritedCpuPool } from './pool-environment.js';
|
|
5
6
|
import { RamTempVolume } from './ram-temp.js';
|
|
6
7
|
import type { BoundedExecOptions } from './schema.js';
|
|
7
8
|
import { type BoundedExecTask } from './verdict.js';
|
|
@@ -27,15 +28,7 @@ export interface NxTaskContext {
|
|
|
27
28
|
}>;
|
|
28
29
|
};
|
|
29
30
|
}
|
|
30
|
-
export
|
|
31
|
-
success: boolean;
|
|
32
|
-
terminalOutput: string;
|
|
33
|
-
/**
|
|
34
|
-
* The command died of a SIGKILL that no bound of this executor and no signal it sent caused: the kernel's OOM killer,
|
|
35
|
-
* which sends it to a leaf in a run's cgroup. Set on the last run of the command only when it was killed again.
|
|
36
|
-
*/
|
|
37
|
-
killedForMemory?: boolean;
|
|
38
|
-
}
|
|
31
|
+
export type BoundedExecResult = RunOutcome;
|
|
39
32
|
export interface ProcessTreeKiller {
|
|
40
33
|
kill(pid: number, signal: NodeJS.Signals): Promise<void>;
|
|
41
34
|
}
|
|
@@ -75,6 +68,6 @@ export declare function recordContext(context: NxTaskContext): BoundedExecContex
|
|
|
75
68
|
* connection closes while the command runs (the lending parent's exit) ends its accounting, never the command: the
|
|
76
69
|
* command runs on over budget to its own exit, and the run says so.
|
|
77
70
|
*/
|
|
78
|
-
export declare function runBoundedExec(options: BoundedExecOptions, context: BoundedExecContext, killer: ProcessTreeKiller, tempVolume: TempVolume | null, inherited?: InheritedCpuPool | null, captureDiagnostics?: typeof captureHangDiagnostics, owner?: SubtreeOwner | null, queueHost?:
|
|
71
|
+
export declare function runBoundedExec(options: BoundedExecOptions, context: BoundedExecContext, killer: ProcessTreeKiller, tempVolume: TempVolume | null, inherited?: InheritedCpuPool | null, captureDiagnostics?: typeof captureHangDiagnostics, owner?: SubtreeOwner | null, queueHost?: QueueHost): Promise<BoundedExecResult>;
|
|
79
72
|
export declare function createProcessTreeKiller(): ProcessTreeKiller;
|
|
80
73
|
//# sourceMappingURL=executor.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/executor.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/executor.ts"],"names":[],"mappings":"AAeA,OAAO,EAAwB,KAAK,SAAS,EAAE,MAAM,sBAAsB,CAAC;AAK5E,OAAO,EAAE,sBAAsB,EAAmB,MAAM,uBAAuB,CAAC;AAahF,OAAO,EAAqB,KAAK,UAAU,EAA8B,MAAM,mBAAmB,CAAC;AAEnG,OAAO,EAOL,KAAK,YAAY,EAClB,MAAM,oBAAoB,CAAC;AAG5B,OAAO,EAAgB,KAAK,gBAAgB,EAAwB,MAAM,uBAAuB,CAAC;AAElG,OAAO,EASL,aAAa,EAEd,MAAM,eAAe,CAAC;AAIvB,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAEtD,OAAO,EAEL,KAAK,eAAe,EAcrB,MAAM,cAAc,CAAC;AAiBtB,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,wFAAwF;IACxF,MAAM,CAAC,EAAE;QAAE,IAAI,EAAE,eAAe,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC;IACtD,uFAAuF;IACvF,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;CAClC;AAED,qEAAqE;AACrE,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,SAAS,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE;YAAE,IAAI,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;CAC1D;AAED,MAAM,MAAM,iBAAiB,GAAG,UAAU,CAAC;AAkB3C,MAAM,WAAW,iBAAiB;IAChC,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1D;AAED,8EAA8E;AAC9E,MAAM,MAAM,UAAU,GAAG,IAAI,CAC3B,aAAa,EACb,SAAS,GAAG,SAAS,GAAG,UAAU,GAAG,OAAO,GAAG,QAAQ,GAAG,gBAAgB,CAC3E,CAAC;AAQF;;;;;;GAMG;AACH,wBAAgB,wBAAwB,CACtC,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,MAAM,GAAG,SAAS,EACzB,SAAS,EAAE,MAAM,GAAG,SAAS,EAC7B,SAAS,EAAE,MAAM,GAAG,SAAS,EAC7B,GAAG,EAAE,MAAM,EACX,MAAM,GAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAOzB,GACA,MAAM,GAAG,SAAS,CAiBpB;AAED,MAAM,CAAC,OAAO,UAAU,mBAAmB,CACzC,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,aAAa,GACrB,OAAO,CAAC,iBAAiB,CAAC,CAE5B;AAED;;;GAGG;AACH,wBAAgB,qBAAqB,CACnC,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,kBAAkB,GAC1B,OAAO,CAAC,iBAAiB,CAAC,CAc5B;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAChC,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,kBAAkB,EAC3B,MAAM,EAAE,CAAC,OAAO,EAAE,MAAM,GAAG,SAAS,KAAK,UAAU,GAAG,IAAI,GACzD,OAAO,CAAC,iBAAiB,CAAC,CAsB5B;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAAC,OAAO,EAAE,aAAa,GAAG,kBAAkB,CAiBxE;AAED;;;;;;;;GAQG;AACH,wBAAsB,cAAc,CAClC,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,kBAAkB,EAC3B,MAAM,EAAE,iBAAiB,EACzB,UAAU,EAAE,UAAU,GAAG,IAAI,EAC7B,SAAS,GAAE,gBAAgB,GAAG,IAAW,EACzC,kBAAkB,GAAE,OAAO,sBAA+C,EAC1E,KAAK,GAAE,YAAY,GAAG,IAAW,EACjC,SAAS,CAAC,EAAE,SAAS,GACpB,OAAO,CAAC,iBAAiB,CAAC,CAe5B;AAylCD,wBAAgB,uBAAuB,IAAI,iBAAiB,CAM3D"}
|