@smoothbricks/nx-plugin 0.7.4 → 0.7.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +59 -30
- package/dist/.build-inputs.json +15 -8
- package/dist/executors/bounded-exec/cpu-pool-thread.d.ts +3 -2
- package/dist/executors/bounded-exec/cpu-pool-thread.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool-thread.js +1 -1
- package/dist/executors/bounded-exec/cpu-pool.d.ts +24 -10
- package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool.js +96 -52
- package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
- package/dist/executors/bounded-exec/executor.js +192 -82
- package/dist/executors/bounded-exec/host-memory.d.ts +47 -0
- package/dist/executors/bounded-exec/host-memory.d.ts.map +1 -0
- package/dist/executors/bounded-exec/host-memory.darwin.d.ts +3 -0
- package/dist/executors/bounded-exec/host-memory.darwin.d.ts.map +1 -0
- package/dist/executors/bounded-exec/host-memory.darwin.js +40 -0
- package/dist/executors/bounded-exec/host-memory.js +29 -0
- package/dist/executors/bounded-exec/host-memory.linux.d.ts +14 -0
- package/dist/executors/bounded-exec/host-memory.linux.d.ts.map +1 -0
- package/dist/executors/bounded-exec/host-memory.linux.js +104 -0
- package/dist/executors/bounded-exec/leaf-peak.d.ts +55 -0
- package/dist/executors/bounded-exec/leaf-peak.d.ts.map +1 -0
- package/dist/executors/bounded-exec/leaf-peak.js +146 -0
- package/dist/executors/bounded-exec/mbx-pool.d.ts +17 -9
- package/dist/executors/bounded-exec/mbx-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/mbx-pool.js +29 -11
- package/dist/executors/bounded-exec/mbx-reservations.d.ts +41 -0
- package/dist/executors/bounded-exec/mbx-reservations.d.ts.map +1 -0
- package/dist/executors/bounded-exec/mbx-reservations.js +372 -0
- package/dist/executors/bounded-exec/peak-waiter.d.ts +2 -0
- package/dist/executors/bounded-exec/peak-waiter.d.ts.map +1 -0
- package/dist/executors/bounded-exec/peak-waiter.js +53 -0
- package/dist/executors/bounded-exec/pool-admission.d.ts +38 -0
- package/dist/executors/bounded-exec/pool-admission.d.ts.map +1 -0
- package/dist/executors/bounded-exec/pool-admission.js +31 -0
- package/dist/executors/bounded-exec/run-cpu-pool.d.ts +6 -4
- package/dist/executors/bounded-exec/run-cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/run-cpu-pool.js +20 -7
- package/dist/executors/bounded-exec/runner-sizing.d.ts +5 -5
- package/dist/executors/bounded-exec/runner-sizing.d.ts.map +1 -1
- package/dist/executors/bounded-exec/runner-sizing.js +6 -6
- package/dist/executors/bounded-exec/verdict.d.ts +21 -2
- package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
- package/dist/executors/bounded-exec/verdict.js +27 -4
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -1
- package/package.json +1 -1
- package/src/executors/bounded-exec/cpu-pool-thread.ts +4 -3
- package/src/executors/bounded-exec/cpu-pool.ts +111 -49
- package/src/executors/bounded-exec/executor.ts +202 -77
- package/src/executors/bounded-exec/host-memory.darwin.ts +44 -0
- package/src/executors/bounded-exec/host-memory.linux.ts +115 -0
- package/src/executors/bounded-exec/host-memory.ts +70 -0
- package/src/executors/bounded-exec/leaf-peak.ts +151 -0
- package/src/executors/bounded-exec/mbx-pool.ts +33 -13
- package/src/executors/bounded-exec/mbx-reservations.ts +300 -0
- package/src/executors/bounded-exec/peak-waiter.ts +82 -0
- package/src/executors/bounded-exec/pool-admission.ts +64 -0
- package/src/executors/bounded-exec/run-cpu-pool.ts +20 -6
- package/src/executors/bounded-exec/runner-sizing.ts +6 -6
- package/src/executors/bounded-exec/verdict.ts +40 -6
- package/src/index.ts +4 -1
- package/test-timings.json +134 -131
package/README.md
CHANGED
|
@@ -1964,31 +1964,43 @@ waiting for, however long a loaded host makes the work take; one that never retu
|
|
|
1964
1964
|
`idleTimeoutMs`. `timeoutMs`, an absolute ceiling, is for commands that report no tests; a target names at least one of
|
|
1965
1965
|
the two.
|
|
1966
1966
|
|
|
1967
|
-
With mbx admission, a
|
|
1968
|
-
|
|
1969
|
-
|
|
1970
|
-
|
|
1971
|
-
|
|
1972
|
-
|
|
1973
|
-
|
|
1974
|
-
|
|
1975
|
-
`mbx
|
|
1976
|
-
|
|
1977
|
-
|
|
1978
|
-
|
|
1979
|
-
|
|
1980
|
-
|
|
1981
|
-
|
|
1982
|
-
|
|
1983
|
-
|
|
1984
|
-
|
|
1985
|
-
|
|
1986
|
-
|
|
1987
|
-
|
|
1988
|
-
|
|
1989
|
-
|
|
1990
|
-
|
|
1991
|
-
the
|
|
1967
|
+
With mbx admission, a supported Cargo/nextest test leaf inherits a private grant pipe: mbx writes `g` on every admission
|
|
1968
|
+
and `r` on permit drop (`MBX_GRANT_FD=5`). The executor counts overlapping holds: while no hold exists it pauses the
|
|
1969
|
+
test idle budget, resuming the remaining budget on admission; a finished test still resets that budget. Total and
|
|
1970
|
+
fixture ceilings start at the first `g` and do not pause after it. EOF is not a grant or a release; a holder killed
|
|
1971
|
+
without `r` remains counted until command exit.
|
|
1972
|
+
|
|
1973
|
+
A reserved leaf takes its cores from its Nx run's mbx reservation endpoint, never from an `mbx reserve` process of its
|
|
1974
|
+
own (specs/cowshed/05_gateway.md, "An Nx run serves its leaves one reservation endpoint"). The plugin's
|
|
1975
|
+
`preTasksExecution` starts `mbx reservations <socket>` once for a run with mbx's pool and publishes the socket in
|
|
1976
|
+
`MBX_RESERVATIONS_SOCKET`; its `postTasksExecution` ends it. The leaf's shell writes the count it asks for on descriptor
|
|
1977
|
+
5 and reads back the cores granted. The executor asks the endpoint over its process's one connection, says
|
|
1978
|
+
`cowshed: mbx reserve wait start cpus=<asked> priority=<p>` and then
|
|
1979
|
+
`cowshed: mbx reserve granted elapsed=<ms> priority=<p>`, and gives the grant back once the leaf's subtree is retired;
|
|
1980
|
+
the grant starts the leaf's total and fixture ceilings as a first `g` does. The shell runs the command with
|
|
1981
|
+
`MBX_RESERVE_CPUS`, `MBX_SCHED_DISABLE=1` and an empty `MBX_SCHED_DIR`. A refusal fails the leaf, naming mbx's error,
|
|
1982
|
+
and a run with no endpoint refuses its reserved leaves.
|
|
1983
|
+
|
|
1984
|
+
Independently of that spawn-time zero-hold rule, **any live process of a Cargo leaf explicitly queued for mbx admission
|
|
1985
|
+
pauses idle**, even while another of its processes holds capacity or the runner is otherwise unsupported. The executor
|
|
1986
|
+
polls public `mbx stats --scheduler --json --activity` activity, accepting only `kind: "waiter"`, `live: true` rows
|
|
1987
|
+
whose PID is corroborated by bounded, scoped per-PID lineage reads through cowshed's native `processLineages` owner
|
|
1988
|
+
(libproc on macOS, procfs on Linux), never a spawned ps or a process-table census. Names, private waiter files and
|
|
1989
|
+
decision logs prove nothing. Public `queue_position` is preserved, never recomputed by the consumer. A successful
|
|
1990
|
+
snapshot with no task waiters resumes the remaining budget when the lifecycle and hold count permit it; a finished test
|
|
1991
|
+
resets the paused budget. An idle callback drains any in-flight poll and takes a fresh bounded snapshot before expiring.
|
|
1992
|
+
The stats read and every native lineage batch share one five-second monotonic control-plane budget, independent of the
|
|
1993
|
+
idle clock; late or cancelled snapshots never become queue or empty-membership evidence. Snapshot failures are reported
|
|
1994
|
+
operational failures, never manufactured queue/admission evidence or an idle wedge; an unidentifiable foreign legacy row
|
|
1995
|
+
is diagnosed without claiming task membership. Live diagnostics say `queued for admission N s` with the mbx queue
|
|
1996
|
+
position. A command that fails or is cancelled before its first admission records a distinct `queued` verdict with
|
|
1997
|
+
observed wait duration and its last public PID/ticket/position rows, or, for a reserved leaf, its wait on its run's
|
|
1998
|
+
endpoint and no rows. Queue observation adds no deadline and changes neither total nor fixture ceilings. Queue
|
|
1999
|
+
observations take effect at snapshot completion, never backdated across a finished test's budget reset. Observed wait
|
|
2000
|
+
duration freezes when the command settles or cancellation stops observation, before teardown or report reading. The
|
|
2001
|
+
observer requires the version-2 activity-only report; the full statistics document's cache, savings and memory histories
|
|
2002
|
+
are not polling input. The activity projection omits those walks and encodings instead of raising the observer's output
|
|
2003
|
+
or control-plane limits.
|
|
1992
2004
|
|
|
1993
2005
|
The decision to use zero-hold pausing is made before spawn, from the canonical scheduling switch and the supported
|
|
1994
2006
|
command, never from silence on the pipe. The inferred in-place
|
|
@@ -2060,8 +2072,8 @@ and its ceiling does not apply.
|
|
|
2060
2072
|
Under mbx, a reserved leaf whose target names a `unitsCommand` (a shell command printing one line per unit `command`
|
|
2061
2073
|
runs; the plugin's nextest pieces name the same `nextest list -T oneline` of their selection and partition) runs that
|
|
2062
2074
|
command at the end of its fixture segment, under the fixture's ceiling, which then starts at spawn, with the fixture's
|
|
2063
|
-
own output on stderr; the fixture runs before the reservation, not inside it. The leaf then
|
|
2064
|
-
|
|
2075
|
+
own output on stderr; the fixture runs before the reservation, not inside it. The leaf then asks its run's endpoint for
|
|
2076
|
+
`min(units, its runner's count)` cores, at least one, and its reservation says
|
|
2065
2077
|
`cowshed: mbx reserve running cpus=<granted> asked=<asked>`. A count that fails fails the run before anything is
|
|
2066
2078
|
reserved, named as the fixture segment. `unitsCommand` is refused without `fixtureCommand`.
|
|
2067
2079
|
|
|
@@ -2264,9 +2276,9 @@ through its recorded handle, then archived whole, readonly tree and modes includ
|
|
|
2264
2276
|
signalled), beside a dead lease that recorded no handle and one whose handle another live job holds (kept, then
|
|
2265
2277
|
reclaimed once that job ends), a later dead lease whose archive rotates the oldest out and is deleted in the background,
|
|
2266
2278
|
a failed run's own lease archived below the half-volume floor and so deleted with every other archive, and the final
|
|
2267
|
-
detach. Its supervisor is a stand-in speaking the subtree verbs' wire shapes, whose tree is the
|
|
2268
|
-
(`src/__tests__/stand-in-cowshed.ts`). Its volume uses a real private gateway served by `ram_fixture <ns>` on
|
|
2269
|
-
namespace's socket, with `recordedRamAttach` capturing every attached identity and `disk-command --gateway-socket`
|
|
2279
|
+
detach. Its supervisor is a stand-in speaking the subtree verbs' wire shapes, whose tree is the process group its root
|
|
2280
|
+
runs in (`src/__tests__/stand-in-cowshed.ts`). Its volume uses a real private gateway served by `ram_fixture <ns>` on
|
|
2281
|
+
its namespace's socket, with `recordedRamAttach` capturing every attached identity and `disk-command --gateway-socket`
|
|
2270
2282
|
routing native admission. The helper is closed only after exact-owned teardown, which then removes the namespace root;
|
|
2271
2283
|
no test asks the installed gateway. External symlink targets and held directory modes stay unchanged, without duplicate
|
|
2272
2284
|
provision and teardown; its 60 s case bound is unchanged and its target has the progress bound.
|
|
@@ -2403,6 +2415,14 @@ instead of each sizing its runner to the whole host. Its scope is one run. A com
|
|
|
2403
2415
|
as configured, sized only by the ceilings it inherits, and nothing is said; a cowshed job's own jobserver fifo still
|
|
2404
2416
|
reaches it by Cargo's own rule ("Cargo reads its holder's jobserver").
|
|
2405
2417
|
|
|
2418
|
+
The run's pool charges memory too. Each task is charged its last recorded peak (`peakBytes` in its `verdict.json`,
|
|
2419
|
+
below), nothing while it has none, and is granted only while the charges of the tasks holding grants plus its own fit
|
|
2420
|
+
the run's memory: the least finite `memory.max` of the run's cgroup and its ancestors and the host's `MemTotal` on
|
|
2421
|
+
Linux, `hw.memsize` on macOS (`host-memory.ts`). A run whose memory cannot be read fails in `preTasksExecution`, naming
|
|
2422
|
+
the file or sysctl. A task whose own peak exceeds the run's memory is granted once nothing else holds memory, and its
|
|
2423
|
+
stderr says so; the wait line names the charge as `memory=<bytes>` or `memory=none`. A granted executor's sub-pool
|
|
2424
|
+
charges no memory.
|
|
2425
|
+
|
|
2406
2426
|
The pool runs on a worker thread of its own (`cpu-pool-thread.ts`). The process that serves the run, the Nx daemon when
|
|
2407
2427
|
installed plugins load in it or the Nx CLI without one, hashes tasks and builds the project graph in synchronous calls
|
|
2408
2428
|
on its main thread; a pool on that thread wrote no heartbeat behind a 14.2 s `hashPlans`, and every runner queued on it
|
|
@@ -2468,6 +2488,15 @@ task id and the hash it ran at:
|
|
|
2468
2488
|
- `failed`: a test failed on its own, or the command exited non-zero with no report naming a timeout.
|
|
2469
2489
|
- `passed`.
|
|
2470
2490
|
|
|
2491
|
+
The record also holds `peakBytes`, the command's memory peak: the exited command's rusage `ru_maxrss`, the largest
|
|
2492
|
+
resident set any one process of its tree reached, in bytes. Node reaps a child without its rusage, so the command runs
|
|
2493
|
+
under a Bun waiter (`peak-waiter.ts`, on the executor's Bun or the first `bun` on its PATH) that reads it, passes the
|
|
2494
|
+
command its stdio and descriptors 3 to 5, and exits as the command did. The waiter leads the command's process group and
|
|
2495
|
+
names the command's pid when it starts it: that command is the runner a hang snapshot samples, never the waiter.
|
|
2496
|
+
`peakBytes` is null when nothing was measured: the command never ran, the waiter was killed before it reported, or no
|
|
2497
|
+
Bun was found; stderr then says why. The task's next run reads it before clearing the directory, and its run's pool
|
|
2498
|
+
charges it.
|
|
2499
|
+
|
|
2471
2500
|
The report needs no target configuration. When the command runs exactly one `bun test` or `nextest run`, the executor
|
|
2472
2501
|
asks that runner for a JUnit report in the task's directory: `bun test` through `--reporter=junit --reporter-outfile`,
|
|
2473
2502
|
nextest through a one-key tool config (nextest takes the report path only from configuration). A `bun` is any unquoted
|
package/dist/.build-inputs.json
CHANGED
|
@@ -35,33 +35,40 @@
|
|
|
35
35
|
"packages/nx-plugin/src/executors/all-verdicts/executor.ts": "4d3aac65aa2e03a2d24d149c412aa18f2e66a6a36a4e88993122b10e9355c8f0",
|
|
36
36
|
"packages/nx-plugin/src/executors/all-verdicts/schema.ts": "2ca6bd8d2f19d851f291ab6a5227167399b9dc8ed0fda0248b69896d77e471f1",
|
|
37
37
|
"packages/nx-plugin/src/executors/bounded-exec/bounds.ts": "fecc48ea66df1c2ba526dc0322ab7760cfc1a4e85d99a72a7a43f040448970f0",
|
|
38
|
-
"packages/nx-plugin/src/executors/bounded-exec/cpu-pool-thread.ts": "
|
|
39
|
-
"packages/nx-plugin/src/executors/bounded-exec/cpu-pool.ts": "
|
|
38
|
+
"packages/nx-plugin/src/executors/bounded-exec/cpu-pool-thread.ts": "bc7d113a044bb830e4db4c373c2f366fadfbb212353e34ef9a5e7dccf876aee6",
|
|
39
|
+
"packages/nx-plugin/src/executors/bounded-exec/cpu-pool.ts": "0f2085296486ebfc63b356fb92c4ec77670de5ac9877bbaf2772ea3aaabbf9f9",
|
|
40
40
|
"packages/nx-plugin/src/executors/bounded-exec/disk-lease.ts": "1a3f6a6fa590d9a2550490bb6906dc91b6c05fbef235b025eaf257eb5310e92d",
|
|
41
|
-
"packages/nx-plugin/src/executors/bounded-exec/executor.ts": "
|
|
41
|
+
"packages/nx-plugin/src/executors/bounded-exec/executor.ts": "63ea2da25f290b458c2ed29bb83c10aebd1236e4cc4b0f9a5da78e8f23de4614",
|
|
42
42
|
"packages/nx-plugin/src/executors/bounded-exec/gateway-lease.ts": "f3f73bd92db30e4857f98fb8bd2b9a75b2a5a2a318f094e075ed803c1f9a0117",
|
|
43
43
|
"packages/nx-plugin/src/executors/bounded-exec/hang-diagnostics.ts": "bcaa9e52384f6c481a9cc2835ba6e8ffe167afa3cb51623fdf8dc3493876f335",
|
|
44
|
+
"packages/nx-plugin/src/executors/bounded-exec/host-memory.darwin.ts": "a7bfc682d004d073f59abfc1b8637d00a545b26744b9605e942a2de7aef651a9",
|
|
45
|
+
"packages/nx-plugin/src/executors/bounded-exec/host-memory.linux.ts": "f4762c05dcc36e9d2a2059d0723e99e9810be1f647493178626ff070b6b00e13",
|
|
46
|
+
"packages/nx-plugin/src/executors/bounded-exec/host-memory.ts": "d0ee70e3e406e3d48a573875fa8dbc60dc7349d31ddd6888b49e1207eed6b744",
|
|
47
|
+
"packages/nx-plugin/src/executors/bounded-exec/leaf-peak.ts": "3d4934d62f7c37e57ce050be45fefb2bc9f2ca3049bef33670c44fc572324f6d",
|
|
44
48
|
"packages/nx-plugin/src/executors/bounded-exec/lease-wait.ts": "59d8ac94d0b8b75858971c5bb28ee421fca4f7685577274fe8bbaf0c01ba462a",
|
|
45
49
|
"packages/nx-plugin/src/executors/bounded-exec/lock-wait.ts": "b14a23b49759a777f4b3338ced003adaf76837c2c38462458f76a7fef1b95d9e",
|
|
46
|
-
"packages/nx-plugin/src/executors/bounded-exec/mbx-pool.ts": "
|
|
50
|
+
"packages/nx-plugin/src/executors/bounded-exec/mbx-pool.ts": "4ef48c1887c1fb29000cdc581686332db6fd503ee07ccdb5f0241fb033249097",
|
|
47
51
|
"packages/nx-plugin/src/executors/bounded-exec/mbx-queue.ts": "ac309e28e5ee78fa08d628edb00e33532514f7572166770799ae86d468c12b4e",
|
|
52
|
+
"packages/nx-plugin/src/executors/bounded-exec/mbx-reservations.ts": "65087d9fdb3f7e788a127d0a7261701b7cb18b0b520ca5cef4b8f7b6fc33d3c5",
|
|
48
53
|
"packages/nx-plugin/src/executors/bounded-exec/no-space.ts": "ec032968d89f9e853796012705a06c06cfc4da54bf6e6f10a526f98daca3d077",
|
|
49
54
|
"packages/nx-plugin/src/executors/bounded-exec/owned-subtree.ts": "cb5edf2b5fe2c1b5b961f8c0ff16ec7bfae3c22ad75dfd671b74f1a4a865863f",
|
|
55
|
+
"packages/nx-plugin/src/executors/bounded-exec/peak-waiter.ts": "27d51f75f182125b555643cdc14463d41139abd49cba59dccf8a20fcae136c56",
|
|
56
|
+
"packages/nx-plugin/src/executors/bounded-exec/pool-admission.ts": "384819f66b486a4075685d9a5c240011a342966eeb5c39ecafe1656529f16c21",
|
|
50
57
|
"packages/nx-plugin/src/executors/bounded-exec/progress.ts": "ed926f7b4991ebf7a7c8876ab0ea00295934d058c774e3f811963a85d209979b",
|
|
51
58
|
"packages/nx-plugin/src/executors/bounded-exec/ram-attach.ts": "865eb5c277d9ab4b258013030938aeee762b731a7298fa8ad3c5eba3026df77a",
|
|
52
59
|
"packages/nx-plugin/src/executors/bounded-exec/ram-detach.ts": "535d26114d14d0d94db76227ea7ba52b1ae0b590d7dbac22993b6796d255e666",
|
|
53
60
|
"packages/nx-plugin/src/executors/bounded-exec/ram-temp-fixtures.ts": "075da4ecfd1cf7dd98fc8c894599dcf2720faba07ac30b8e5592964178849359",
|
|
54
61
|
"packages/nx-plugin/src/executors/bounded-exec/ram-temp-held.ts": "7c32555bc6dab3f767925b55b68684d33a69393050342d06f055fb81e8e5332b",
|
|
55
62
|
"packages/nx-plugin/src/executors/bounded-exec/ram-temp.ts": "8d66e6ac13c4ccf6172586b9857d5114b48e027bdcb305f94f6828076c1f7eb4",
|
|
56
|
-
"packages/nx-plugin/src/executors/bounded-exec/run-cpu-pool.ts": "
|
|
63
|
+
"packages/nx-plugin/src/executors/bounded-exec/run-cpu-pool.ts": "180dc2b185e22c5280182115ca99d52debd2461e51f971ba284aa9a8bef634fb",
|
|
57
64
|
"packages/nx-plugin/src/executors/bounded-exec/runner-hook.ts": "606eab5db826e571eb15df618315e01cc773d09fd647beb1c00f6434df58ffad",
|
|
58
65
|
"packages/nx-plugin/src/executors/bounded-exec/runner-lifecycle.ts": "b2a7c901296bb6c139909297bbdbab5bd00e94319c022307d006b0e1293f6454",
|
|
59
|
-
"packages/nx-plugin/src/executors/bounded-exec/runner-sizing.ts": "
|
|
66
|
+
"packages/nx-plugin/src/executors/bounded-exec/runner-sizing.ts": "bffd2ba0f80758944ca682838832bdbe20ae12eb9a4f75084a3df82ae7786145",
|
|
60
67
|
"packages/nx-plugin/src/executors/bounded-exec/runtime-pin.ts": "3eb9a8b794da51ffa01723509935f48bfb497fba883518046e74a53209f0eb10",
|
|
61
68
|
"packages/nx-plugin/src/executors/bounded-exec/schema.ts": "74eb04eaa8d90d966a98ca2b1b7fffc7d2205a55a1e38958effa641b72c680a3",
|
|
62
69
|
"packages/nx-plugin/src/executors/bounded-exec/test-result-writer.ts": "df7668a11d105a1f88d2187ef8b0edff9db7ba56d8176b5bc35daf1359667637",
|
|
63
70
|
"packages/nx-plugin/src/executors/bounded-exec/test-results.ts": "4f83dc3accd7892f45385796522267adad9401326d0cd9ce4d4e1145b9023577",
|
|
64
|
-
"packages/nx-plugin/src/executors/bounded-exec/verdict.ts": "
|
|
71
|
+
"packages/nx-plugin/src/executors/bounded-exec/verdict.ts": "7ec4b86fbdacc3b6ef0829a19d76e32858f196c2e741f5e69e3b0e395a03a12f",
|
|
65
72
|
"packages/nx-plugin/src/executors/cargo-zigbuild/executor.ts": "622a5f36897d9eb1f7465654e88f4291aedee9767603028d20c1e819b91d72f4",
|
|
66
73
|
"packages/nx-plugin/src/executors/cargo-zigbuild/schema.ts": "e5cf3b0cc91a2d003a63ca0c9a93f1ec08263469b8a19dbe338b2dc0c0810747",
|
|
67
74
|
"packages/nx-plugin/src/executors/scriptc-native/executor.ts": "4db61da4444d3a30acd3995b6cf53a4bc403a40fe856d88b2c30ce467badbea8",
|
|
@@ -87,7 +94,7 @@
|
|
|
87
94
|
"packages/nx-plugin/src/generators/make-public/schema.d.ts": "5e7bd2f1ce16795df4c919d6d53588ea85b574c84400940b6322275f96f60357",
|
|
88
95
|
"packages/nx-plugin/src/go-tests.ts": "8e356b6ee276e6e836c404353b0ffb5ccec5beb261126f167f730e6e01e9c2b6",
|
|
89
96
|
"packages/nx-plugin/src/graph-input-contract.ts": "8a1aec1294960ae80bcff55a0370c95004a911bf1afe6caa47e666da3baeac43",
|
|
90
|
-
"packages/nx-plugin/src/index.ts": "
|
|
97
|
+
"packages/nx-plugin/src/index.ts": "223dd22b1bc871a24c9a7a7b66e2d496870cac40b52281f7119356a09287a96a",
|
|
91
98
|
"packages/nx-plugin/src/managed-assets.ts": "9cea959c09f3163992e666eecc5a65ec1f1b63c81fc56c5e99088507d02c5e3b",
|
|
92
99
|
"packages/nx-plugin/src/managed-files/ci-workflow.ts": "2d64d689e999af20d65cc21b4803659a835a6eb5340e8d3f896cb0c74e6a0878",
|
|
93
100
|
"packages/nx-plugin/src/managed-files/context.ts": "5b83c000322382cb781df4f415f3759388e69686b76779599572a62a5c4dd28c",
|
|
@@ -7,12 +7,13 @@ import { type CloseOutcome } from './cpu-pool.js';
|
|
|
7
7
|
* jobserver fifo and heartbeat share an event loop that runs nothing else, so the heartbeat proves that the actor which
|
|
8
8
|
* admits and grants is turning.
|
|
9
9
|
*
|
|
10
|
-
* The thread is started with the pool's count. It says once that the pool serves, with the environment a task inherits,
|
|
10
|
+
* The thread is started with the pool's count and the run's memory. It says once that the pool serves, with the environment a task inherits,
|
|
11
11
|
* or why it cannot; the host's one message is its close, after which the thread says how the pool closed.
|
|
12
12
|
*/
|
|
13
|
-
/** What the thread is started with. */
|
|
13
|
+
/** What the thread is started with: the run's count, and the run's memory in bytes its grants are charged against. */
|
|
14
14
|
export interface CpuPoolThreadData {
|
|
15
15
|
readonly tokens: number;
|
|
16
|
+
readonly memory: number;
|
|
16
17
|
}
|
|
17
18
|
/** What the thread says, in order: the pool serves or failed; after the host's close, how the pool closed. */
|
|
18
19
|
export type CpuPoolThreadReport = {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cpu-pool-thread.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/cpu-pool-thread.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,YAAY,EAAiB,MAAM,eAAe,CAAC;AAEjE;;;;;;;;;;GAUG;AAEH,
|
|
1
|
+
{"version":3,"file":"cpu-pool-thread.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/cpu-pool-thread.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,YAAY,EAAiB,MAAM,eAAe,CAAC;AAEjE;;;;;;;;;;GAUG;AAEH,sHAAsH;AACtH,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED,8GAA8G;AAC9G,MAAM,MAAM,mBAAmB,GAC3B;IAAE,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IAAC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAAE,GAC1E;IAAE,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GACnD;IAAE,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IAAC,QAAQ,CAAC,OAAO,EAAE,YAAY,CAAA;CAAE,CAAC"}
|
|
@@ -7,7 +7,7 @@ if (host === null) {
|
|
|
7
7
|
const report = (message) => host.postMessage(message);
|
|
8
8
|
const data = workerData;
|
|
9
9
|
// The pool lives until its host closes it, or with the thread.
|
|
10
|
-
const made = await createCpuPool(data.tokens, new Promise(() => { }));
|
|
10
|
+
const made = await createCpuPool(data.tokens, new Promise(() => { }), data.memory);
|
|
11
11
|
if (made.ok) {
|
|
12
12
|
const { pool } = made;
|
|
13
13
|
host.once('message', () => {
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import typia from 'typia';
|
|
1
2
|
import { type Unleased } from './gateway-lease.js';
|
|
2
3
|
/**
|
|
3
4
|
* A CPU pool: the tokens a process lends the runners it starts, held live rather than inherited as a
|
|
@@ -12,6 +13,11 @@ import { type Unleased } from './gateway-lease.js';
|
|
|
12
13
|
* lasts while its connection does, so a child that dies returns its tokens with its socket. A pool
|
|
13
14
|
* grants only from its own count, and a pool whose upstream grant has gone grants nothing.
|
|
14
15
|
*
|
|
16
|
+
* A run's own pool also charges memory (`pool-admission.ts`; specs/cowshed/05_gateway.md, "Without mbx, every Nx run
|
|
17
|
+
* budgets itself"). A request names its task's last recorded peak as `memory`, in bytes, and is granted only while
|
|
18
|
+
* the charges of the leaves holding grants and its own fit the run's memory; a request that names none is charged
|
|
19
|
+
* nothing, as mbx's omitted `--memory` is. A sub-pool charges no memory: its holder's charge covers its tree.
|
|
20
|
+
*
|
|
15
21
|
* The same count is a GNU make jobserver fifo beside the socket, mode 0600, which Cargo, its build
|
|
16
22
|
* scripts and every nested `cargo` read through `CARGO_MAKEFLAGS`/`MAKEFLAGS` (specs/cowshed/05_gateway.md,
|
|
17
23
|
* "Cargo reads its holder's jobserver"). A pool of `g` holds `g − 1` token bytes there; the holder's own
|
|
@@ -44,6 +50,10 @@ export type CloseOutcome = {
|
|
|
44
50
|
ok: false;
|
|
45
51
|
error: string;
|
|
46
52
|
};
|
|
53
|
+
/** A grant whose charge alone exceeds the run's memory, made while nothing else held memory: the run's memory. */
|
|
54
|
+
export interface Oversized {
|
|
55
|
+
readonly memory: number & typia.tags.Type<'uint64'>;
|
|
56
|
+
}
|
|
47
57
|
/**
|
|
48
58
|
* The inherited descriptor, or null when malformed or absent. A missing or bare count, or an empty
|
|
49
59
|
* or NUL-bearing endpoint, proves no allocation; the count is a positive whole number. A valid
|
|
@@ -51,28 +61,32 @@ export type CloseOutcome = {
|
|
|
51
61
|
*/
|
|
52
62
|
export declare function readInheritedCpuPool(environment: Readonly<NodeJS.ProcessEnv>): InheritedCpuPool | null;
|
|
53
63
|
/**
|
|
54
|
-
* Tokens a parent pool lent, held until released, or why there are none. `
|
|
55
|
-
*
|
|
64
|
+
* Tokens a parent pool lent, held until released, or why there are none. `oversized` names the run's memory when the
|
|
65
|
+
* child's charge alone exceeds it and it was granted because nothing else held memory. `closed` resolves when the
|
|
66
|
+
* grant's connection closes, however it closes, so a sub-pool built on the grant ends with it.
|
|
56
67
|
*/
|
|
57
68
|
export type TokenGrant = {
|
|
58
69
|
granted: true;
|
|
59
70
|
tokens: number;
|
|
71
|
+
oversized: Oversized | null;
|
|
60
72
|
release: () => void;
|
|
61
73
|
closed: Promise<void>;
|
|
62
74
|
} | Unleased;
|
|
63
75
|
/**
|
|
64
|
-
* Ask the immediate parent pool for `want` tokens
|
|
65
|
-
*
|
|
66
|
-
*
|
|
67
|
-
*
|
|
76
|
+
* Ask the immediate parent pool for `want` tokens, charged `memory` bytes: the task's last recorded peak, or null when
|
|
77
|
+
* it has none. The inherited count does not bound the ask: a pool grants at most what it holds. A granted lease is the
|
|
78
|
+
* child's local allocation: its connection is the grant, and closing it returns only that allocation. An absent
|
|
79
|
+
* endpoint grants nothing and refuses nothing; a connected endpoint's refusal or protocol failure must fail admission
|
|
80
|
+
* rather than run the child beside its parent's count.
|
|
68
81
|
*/
|
|
69
|
-
export declare function takeParentCpuTokens(parent: InheritedCpuPool, want: number): Promise<TokenGrant>;
|
|
82
|
+
export declare function takeParentCpuTokens(parent: InheritedCpuPool, want: number, memory: number | null): Promise<TokenGrant>;
|
|
70
83
|
/**
|
|
71
84
|
* Make a pool of `tokens` that lives while `closed` does: when `closed` resolves the pool refuses
|
|
72
|
-
* grants, ends its child connections, drains its jobserver fifo and removes its directory.
|
|
73
|
-
*
|
|
85
|
+
* grants, ends its child connections, drains its jobserver fifo and removes its directory. A run's own
|
|
86
|
+
* pool charges its grants against `memory`, the run's memory in bytes; a sub-pool charges none (null).
|
|
87
|
+
* Setup failures are values; a non-positive or unsafe count is a programmer bug and throws.
|
|
74
88
|
*/
|
|
75
|
-
export declare function createCpuPool(tokens: number, closed: Promise<void
|
|
89
|
+
export declare function createCpuPool(tokens: number, closed: Promise<void>, memory: number | null): Promise<{
|
|
76
90
|
ok: true;
|
|
77
91
|
pool: CpuPool;
|
|
78
92
|
} | {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cpu-pool.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/cpu-pool.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"cpu-pool.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/cpu-pool.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,MAAM,OAAO,CAAC;AAE1B,OAAO,EAAa,KAAK,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAK9D;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,yGAAyG;AACzG,eAAO,MAAM,YAAY,0BAA0B,CAAC;AAEpD,6FAA6F;AAC7F,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,sDAAsD;IACtD,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED,yFAAyF;AACzF,MAAM,WAAW,OAAO;IACtB,oFAAoF;IACpF,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7C;;;;OAIG;IACH,KAAK,IAAI,OAAO,CAAC,YAAY,CAAC,CAAC;CAChC;AAED,MAAM,MAAM,YAAY,GAAG;IAAE,EAAE,EAAE,IAAI,CAAA;CAAE,GAAG;IAAE,EAAE,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC;AAYvE,kHAAkH;AAClH,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;CACrD;AAsDD;;;;GAIG;AACH,wBAAgB,oBAAoB,CAAC,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,gBAAgB,GAAG,IAAI,CAOtG;AAED;;;;GAIG;AACH,MAAM,MAAM,UAAU,GAClB;IAAE,OAAO,EAAE,IAAI,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,SAAS,GAAG,IAAI,CAAC;IAAC,OAAO,EAAE,MAAM,IAAI,CAAC;IAAC,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,CAAA;CAAE,GAC1G,QAAQ,CAAC;AAIb;;;;;;GAMG;AACH,wBAAsB,mBAAmB,CACvC,MAAM,EAAE,gBAAgB,EACxB,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,MAAM,GAAG,IAAI,GACpB,OAAO,CAAC,UAAU,CAAC,CAqCrB;AAsJD;;;;;GAKG;AACH,wBAAsB,aAAa,CACjC,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,EACrB,MAAM,EAAE,MAAM,GAAG,IAAI,GACpB,OAAO,CAAC;IAAE,EAAE,EAAE,IAAI,CAAC;IAAC,IAAI,EAAE,OAAO,CAAA;CAAE,GAAG;IAAE,EAAE,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CA4VrE"}
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import * as _isTypeUint32_1 from "typia/lib/internal/_isTypeUint32";
|
|
2
|
+
import * as _isTypeUint64_1 from "typia/lib/internal/_isTypeUint64";
|
|
2
3
|
import * as _jsonStringifyNumber_1 from "typia/lib/internal/_jsonStringifyNumber";
|
|
4
|
+
import * as _jsonStringifyTail_1 from "typia/lib/internal/_jsonStringifyTail";
|
|
3
5
|
import * as _jsonStringifyString_1 from "typia/lib/internal/_jsonStringifyString";
|
|
4
6
|
import * as _throwTypeGuardError_1 from "typia/lib/internal/_throwTypeGuardError";
|
|
5
7
|
import { execFile } from 'node:child_process';
|
|
@@ -12,6 +14,7 @@ import typia from 'typia';
|
|
|
12
14
|
import { ownedSocketDirectory } from '../../testing.js';
|
|
13
15
|
import { holdLease } from './gateway-lease.js';
|
|
14
16
|
import { CpuLeaseWait, HEARTBEAT_MS } from './lease-wait.js';
|
|
17
|
+
import { admit } from './pool-admission.js';
|
|
15
18
|
import { CPU_TOKENS_ENV, parseInheritedTokens } from './runner-sizing.js';
|
|
16
19
|
/**
|
|
17
20
|
* A CPU pool: the tokens a process lends the runners it starts, held live rather than inherited as a
|
|
@@ -26,6 +29,11 @@ import { CPU_TOKENS_ENV, parseInheritedTokens } from './runner-sizing.js';
|
|
|
26
29
|
* lasts while its connection does, so a child that dies returns its tokens with its socket. A pool
|
|
27
30
|
* grants only from its own count, and a pool whose upstream grant has gone grants nothing.
|
|
28
31
|
*
|
|
32
|
+
* A run's own pool also charges memory (`pool-admission.ts`; specs/cowshed/05_gateway.md, "Without mbx, every Nx run
|
|
33
|
+
* budgets itself"). A request names its task's last recorded peak as `memory`, in bytes, and is granted only while
|
|
34
|
+
* the charges of the leaves holding grants and its own fit the run's memory; a request that names none is charged
|
|
35
|
+
* nothing, as mbx's omitted `--memory` is. A sub-pool charges no memory: its holder's charge covers its tree.
|
|
36
|
+
*
|
|
29
37
|
* The same count is a GNU make jobserver fifo beside the socket, mode 0600, which Cargo, its build
|
|
30
38
|
* scripts and every nested `cargo` read through `CARGO_MAKEFLAGS`/`MAKEFLAGS` (specs/cowshed/05_gateway.md,
|
|
31
39
|
* "Cargo reads its holder's jobserver"). A pool of `g` holds `g − 1` token bytes there; the holder's own
|
|
@@ -37,8 +45,8 @@ import { CPU_TOKENS_ENV, parseInheritedTokens } from './runner-sizing.js';
|
|
|
37
45
|
export const CPU_POOL_ENV = 'BOUNDED_EXEC_CPU_POOL';
|
|
38
46
|
/** Exactly a request's fields: any other field is an invalid request. */
|
|
39
47
|
const isRequest = (() => {
|
|
40
|
-
const _io0 = (input, _exceptionable = true) => "cpu-pool" === input.op && ("number" === typeof input.want && (_isTypeUint32_1._isTypeUint32(input.want) && 1 <= input.want)) && (2 === Object.keys(input).length || Object.keys(input).every(key => {
|
|
41
|
-
if (["op", "want"].some(prop => key === prop))
|
|
48
|
+
const _io0 = (input, _exceptionable = true) => "cpu-pool" === input.op && ("number" === typeof input.want && (_isTypeUint32_1._isTypeUint32(input.want) && 1 <= input.want)) && (undefined === input.memory || "number" === typeof input.memory && _isTypeUint64_1._isTypeUint64(input.memory)) && (2 === Object.keys(input).length || Object.keys(input).every(key => {
|
|
49
|
+
if (["op", "want", "memory"].some(prop => key === prop))
|
|
42
50
|
return true;
|
|
43
51
|
const value = input[key];
|
|
44
52
|
if (undefined === value)
|
|
@@ -51,8 +59,9 @@ const stringifyReply = (() => {
|
|
|
51
59
|
const _ip0 = input => true === input["ok"];
|
|
52
60
|
const _so0 = input => `{"ok":${String(input.ok)},"progress":${String(_jsonStringifyNumber_1._jsonStringifyNumber(input.progress))}}`;
|
|
53
61
|
const _so1 = input => `{"ok":${String(input.ok)},"lease":${"\"" + input.lease + "\""},"server":${String(_jsonStringifyNumber_1._jsonStringifyNumber(input.server))}}`;
|
|
54
|
-
const _so2 = input => `{"ok":${String(input.ok)},"lease":${"\"" + input.lease + "\""},"tokens":${String(_jsonStringifyNumber_1._jsonStringifyNumber(input.tokens))}}`;
|
|
55
|
-
const _so3 = input => `{"
|
|
62
|
+
const _so2 = input => `{${_jsonStringifyTail_1._jsonStringifyTail(`"ok":${String(input.ok)},"lease":${"\"" + input.lease + "\""},"tokens":${String(_jsonStringifyNumber_1._jsonStringifyNumber(input.tokens))},${undefined === input.oversized ? "" : `"oversized":${undefined !== input.oversized ? _so3(input.oversized) : undefined}`}`)}}`;
|
|
63
|
+
const _so3 = input => `{"memory":${String(_jsonStringifyNumber_1._jsonStringifyNumber(input.memory))}}`;
|
|
64
|
+
const _so4 = input => `{"ok":${String(input.ok)},"code":${"\"" + input.code + "\""},"error":${_jsonStringifyString_1._jsonStringifyString(input.error)}}`;
|
|
56
65
|
const _su0 = input => (() => {
|
|
57
66
|
if (undefined !== input.progress)
|
|
58
67
|
return _so0(input);
|
|
@@ -61,18 +70,19 @@ const stringifyReply = (() => {
|
|
|
61
70
|
else if ("granted" === input.lease)
|
|
62
71
|
return _so2(input);
|
|
63
72
|
else if (false === input.ok)
|
|
64
|
-
return
|
|
73
|
+
return _so4(input);
|
|
65
74
|
else
|
|
66
75
|
_throwTypeGuardError_1._throwTypeGuardError({
|
|
67
76
|
method: "typia.json.createStringify",
|
|
68
|
-
expected: "(CpuLeaseProgress | CpuLeaseQueued | { ok: true; lease: \"granted\"; tokens: number; } | { ok: false; code: \"invalid-request\"; error: string; })",
|
|
77
|
+
expected: "(CpuLeaseProgress | CpuLeaseQueued | { ok: true; lease: \"granted\"; tokens: number; oversized?: Oversized | undefined; } | { ok: false; code: \"invalid-request\"; error: string; })",
|
|
69
78
|
value: input
|
|
70
79
|
});
|
|
71
80
|
})();
|
|
72
81
|
const _io0 = input => _ip0(input) && "number" === typeof input.progress;
|
|
73
82
|
const _io1 = input => _ip0(input) && "queued" === input.lease && ("number" === typeof input.server && _isTypeUint32_1._isTypeUint32(input.server));
|
|
74
|
-
const _io2 = input => _ip0(input) && "granted" === input.lease && "number" === typeof input.tokens;
|
|
75
|
-
const _io3 = input =>
|
|
83
|
+
const _io2 = input => _ip0(input) && "granted" === input.lease && "number" === typeof input.tokens && (undefined === input.oversized || "object" === typeof input.oversized && null !== input.oversized && _io3(input.oversized));
|
|
84
|
+
const _io3 = input => "number" === typeof input.memory && _isTypeUint64_1._isTypeUint64(input.memory);
|
|
85
|
+
const _io4 = input => false === input.ok && "invalid-request" === input.code && "string" === typeof input.error;
|
|
76
86
|
return input => _su0(input);
|
|
77
87
|
})();
|
|
78
88
|
/** A request over this many bytes is refused as `invalid-request` (specs/cowshed/05_gateway.md, "Wire"). */
|
|
@@ -104,16 +114,31 @@ export function readInheritedCpuPool(environment) {
|
|
|
104
114
|
}
|
|
105
115
|
return { socket, tokens };
|
|
106
116
|
}
|
|
117
|
+
const isOversized = (() => {
|
|
118
|
+
const _io0 = (input, _exceptionable = true) => "number" === typeof input.memory && _isTypeUint64_1._isTypeUint64(input.memory) && (1 === Object.keys(input).length || Object.keys(input).every(key => {
|
|
119
|
+
if (["memory"].some(prop => key === prop))
|
|
120
|
+
return true;
|
|
121
|
+
const value = input[key];
|
|
122
|
+
if (undefined === value)
|
|
123
|
+
return true;
|
|
124
|
+
return false;
|
|
125
|
+
}));
|
|
126
|
+
return (input, _exceptionable = true) => "object" === typeof input && null !== input && _io0(input, true);
|
|
127
|
+
})();
|
|
107
128
|
/**
|
|
108
|
-
* Ask the immediate parent pool for `want` tokens
|
|
109
|
-
*
|
|
110
|
-
*
|
|
111
|
-
*
|
|
129
|
+
* Ask the immediate parent pool for `want` tokens, charged `memory` bytes: the task's last recorded peak, or null when
|
|
130
|
+
* it has none. The inherited count does not bound the ask: a pool grants at most what it holds. A granted lease is the
|
|
131
|
+
* child's local allocation: its connection is the grant, and closing it returns only that allocation. An absent
|
|
132
|
+
* endpoint grants nothing and refuses nothing; a connected endpoint's refusal or protocol failure must fail admission
|
|
133
|
+
* rather than run the child beside its parent's count.
|
|
112
134
|
*/
|
|
113
|
-
export async function takeParentCpuTokens(parent, want) {
|
|
135
|
+
export async function takeParentCpuTokens(parent, want, memory) {
|
|
114
136
|
if (!Number.isSafeInteger(want) || want < 1) {
|
|
115
137
|
throw new RangeError(`a child asks its parent pool for a positive count, not ${want} (pool ${parent.tokens})`);
|
|
116
138
|
}
|
|
139
|
+
if (memory !== null && (!Number.isSafeInteger(memory) || memory < 0)) {
|
|
140
|
+
throw new RangeError(`a child is charged a whole number of bytes, not ${memory}`);
|
|
141
|
+
}
|
|
117
142
|
if (parent.socket.length === 0 || parent.socket.includes('\0')) {
|
|
118
143
|
return {
|
|
119
144
|
granted: false,
|
|
@@ -121,11 +146,12 @@ export async function takeParentCpuTokens(parent, want) {
|
|
|
121
146
|
reason: `the parent CPU pool's endpoint ${JSON.stringify(parent.socket)} is not a socket path`,
|
|
122
147
|
};
|
|
123
148
|
}
|
|
124
|
-
const
|
|
149
|
+
const request = memory === null ? { op: 'cpu-pool', want } : { op: 'cpu-pool', want, memory };
|
|
150
|
+
const lease = await holdLease(parent.socket, request, 'the parent CPU pool', new CpuLeaseWait());
|
|
125
151
|
if (!lease.granted) {
|
|
126
152
|
return lease;
|
|
127
153
|
}
|
|
128
|
-
const tokens = lease.answer
|
|
154
|
+
const { tokens, oversized } = lease.answer;
|
|
129
155
|
if (typeof tokens !== 'number' || !Number.isInteger(tokens) || tokens < 1 || tokens > want) {
|
|
130
156
|
lease.release();
|
|
131
157
|
return {
|
|
@@ -134,7 +160,15 @@ export async function takeParentCpuTokens(parent, want) {
|
|
|
134
160
|
reason: `the parent CPU pool granted ${JSON.stringify(tokens)} tokens for ${want}`,
|
|
135
161
|
};
|
|
136
162
|
}
|
|
137
|
-
|
|
163
|
+
if (oversized !== undefined && !isOversized(oversized)) {
|
|
164
|
+
lease.release();
|
|
165
|
+
return {
|
|
166
|
+
granted: false,
|
|
167
|
+
cause: 'other',
|
|
168
|
+
reason: `the parent CPU pool named its memory ${JSON.stringify(oversized)} in a grant, which is no byte count`,
|
|
169
|
+
};
|
|
170
|
+
}
|
|
171
|
+
return { granted: true, tokens, oversized: oversized ?? null, release: lease.release, closed: lease.closed };
|
|
138
172
|
}
|
|
139
173
|
/** A thrown value's message. */
|
|
140
174
|
const describe = (error) => (error instanceof Error ? error.message : String(error));
|
|
@@ -269,10 +303,11 @@ function watchFifo(path, took, failed) {
|
|
|
269
303
|
}
|
|
270
304
|
/**
|
|
271
305
|
* Make a pool of `tokens` that lives while `closed` does: when `closed` resolves the pool refuses
|
|
272
|
-
* grants, ends its child connections, drains its jobserver fifo and removes its directory.
|
|
273
|
-
*
|
|
306
|
+
* grants, ends its child connections, drains its jobserver fifo and removes its directory. A run's own
|
|
307
|
+
* pool charges its grants against `memory`, the run's memory in bytes; a sub-pool charges none (null).
|
|
308
|
+
* Setup failures are values; a non-positive or unsafe count is a programmer bug and throws.
|
|
274
309
|
*/
|
|
275
|
-
export async function createCpuPool(tokens, closed) {
|
|
310
|
+
export async function createCpuPool(tokens, closed, memory) {
|
|
276
311
|
if (!Number.isSafeInteger(tokens) || tokens < 1) {
|
|
277
312
|
throw new RangeError(`a CPU pool holds a positive safe integer of tokens, not ${tokens}`);
|
|
278
313
|
}
|
|
@@ -300,6 +335,8 @@ export async function createCpuPool(tokens, closed) {
|
|
|
300
335
|
const scratch = Buffer.allocUnsafe(tokens);
|
|
301
336
|
// The holder's implicit first token, while a socket child holds it.
|
|
302
337
|
let slotLent = false;
|
|
338
|
+
// The bytes the children holding grants are charged, against `memory`.
|
|
339
|
+
let heldMemory = 0;
|
|
303
340
|
// The readability wait while the queue's head finds no token, else undefined.
|
|
304
341
|
let watch;
|
|
305
342
|
let closing = false;
|
|
@@ -353,49 +390,55 @@ export async function createCpuPool(tokens, closed) {
|
|
|
353
390
|
fail(`its jobserver fifo could not be watched for a token: ${watching.error}`);
|
|
354
391
|
}
|
|
355
392
|
};
|
|
356
|
-
// FIFO: the head takes `min(want, available)` as one grant once any token is available,
|
|
357
|
-
//
|
|
358
|
-
//
|
|
359
|
-
// child needs. Whatever bytes are in hand when no head can
|
|
393
|
+
// FIFO (`admit`): the head takes `min(want, available)` as one grant once any token is available and, in a run's
|
|
394
|
+
// pool, its charge fits beside the holders'; available is the fifo bytes it can read without blocking plus the
|
|
395
|
+
// implicit slot while no child holds it. `arrived` is bytes the readability wait already took. A waiting child holds
|
|
396
|
+
// nothing, so no child waits while it holds tokens another child needs. Whatever bytes are in hand when no head can
|
|
397
|
+
// use them go straight back to the fifo. A head that waits for memory watches the fifo for nothing: only a holder's
|
|
398
|
+
// close frees memory, and that close drains again.
|
|
360
399
|
function drain(arrived) {
|
|
400
|
+
if (closing) {
|
|
401
|
+
stopWatching();
|
|
402
|
+
return;
|
|
403
|
+
}
|
|
404
|
+
// A child that half-closed or failed while queued leaves the line. Its `close` finds it gone from the queue.
|
|
405
|
+
for (let at = queue.length - 1; at >= 0; at -= 1) {
|
|
406
|
+
if (queue[at]?.child.socket.writable === false)
|
|
407
|
+
queue.splice(at, 1);
|
|
408
|
+
}
|
|
361
409
|
let inHand = arrived;
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
}
|
|
367
|
-
if (!waiter.child.socket.writable) {
|
|
368
|
-
// A child that half-closed or failed while queued. Its `close` finds it gone from the queue.
|
|
369
|
-
queue.shift();
|
|
370
|
-
continue;
|
|
371
|
-
}
|
|
372
|
-
if (inHand < waiter.want) {
|
|
373
|
-
try {
|
|
374
|
-
inHand += readTokens(jobserver, scratch, waiter.want - inHand);
|
|
375
|
-
}
|
|
376
|
-
catch (error) {
|
|
377
|
-
fail(`its jobserver fifo could not be read: ${describe(error)}`);
|
|
378
|
-
return;
|
|
379
|
-
}
|
|
410
|
+
const wanted = Math.min(queue.reduce((sum, waiter) => sum + waiter.want, 0), scratch.length);
|
|
411
|
+
if (inHand < wanted) {
|
|
412
|
+
try {
|
|
413
|
+
inHand += readTokens(jobserver, scratch, wanted - inHand);
|
|
380
414
|
}
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
watchForTokens();
|
|
415
|
+
catch (error) {
|
|
416
|
+
fail(`its jobserver fifo could not be read: ${describe(error)}`);
|
|
384
417
|
return;
|
|
385
418
|
}
|
|
386
|
-
|
|
419
|
+
}
|
|
420
|
+
const admission = admit(queue, inHand + (slotLent ? 0 : 1), memory === null ? null : { capacity: memory, held: heldMemory });
|
|
421
|
+
queue.splice(0, admission.grants.length);
|
|
422
|
+
for (const { request, tokens: granted, oversized } of admission.grants) {
|
|
387
423
|
// Fifo bytes first, the implicit slot last.
|
|
388
424
|
const bytes = Math.min(granted, inHand);
|
|
389
425
|
inHand -= bytes;
|
|
390
426
|
const slot = granted > bytes;
|
|
391
427
|
if (slot)
|
|
392
428
|
slotLent = true;
|
|
393
|
-
|
|
394
|
-
|
|
429
|
+
heldMemory += request.memory;
|
|
430
|
+
request.child.phase = { kind: 'holding', bytes, slot, memory: request.memory };
|
|
431
|
+
request.child.socket.write(frame(oversized && memory !== null
|
|
432
|
+
? { ok: true, lease: 'granted', tokens: granted, oversized: { memory } }
|
|
433
|
+
: { ok: true, lease: 'granted', tokens: granted }));
|
|
434
|
+
}
|
|
435
|
+
if (admission.waiting === 'cpu') {
|
|
436
|
+
watchForTokens();
|
|
437
|
+
}
|
|
438
|
+
else {
|
|
439
|
+
stopWatching();
|
|
395
440
|
}
|
|
396
|
-
|
|
397
|
-
if (!closing)
|
|
398
|
-
returnTokens(inHand);
|
|
441
|
+
returnTokens(inHand);
|
|
399
442
|
}
|
|
400
443
|
const refuse = (child, error) => {
|
|
401
444
|
child.phase = { kind: 'done' };
|
|
@@ -423,6 +466,7 @@ export async function createCpuPool(tokens, closed) {
|
|
|
423
466
|
// A closing pool drains its fifo whole; until then the bytes go back for the next holder.
|
|
424
467
|
if (child.phase.slot)
|
|
425
468
|
slotLent = false;
|
|
469
|
+
heldMemory -= child.phase.memory;
|
|
426
470
|
if (!closing)
|
|
427
471
|
returnTokens(child.phase.bytes);
|
|
428
472
|
}
|
|
@@ -479,12 +523,12 @@ export async function createCpuPool(tokens, closed) {
|
|
|
479
523
|
parsed = undefined;
|
|
480
524
|
}
|
|
481
525
|
if (!isRequest(parsed)) {
|
|
482
|
-
refuse(child, 'a request is {"op":"cpu-pool","want":<a whole number of tokens, at least 1>}');
|
|
526
|
+
refuse(child, 'a request is {"op":"cpu-pool","want":<a whole number of tokens, at least 1>,"memory"?:<bytes it is charged>}');
|
|
483
527
|
return;
|
|
484
528
|
}
|
|
485
529
|
const request = parsed;
|
|
486
530
|
child.phase = { kind: 'queued' };
|
|
487
|
-
queue.push({ child, want: request.want });
|
|
531
|
+
queue.push({ child, want: request.want, memory: request.memory ?? 0 });
|
|
488
532
|
socket.write(frame({ ok: true, lease: 'queued', server: process.pid }));
|
|
489
533
|
drain(0);
|
|
490
534
|
});
|