@smoothbricks/nx-plugin 0.7.4 → 0.7.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -27
- package/dist/.build-inputs.json +6 -5
- package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
- package/dist/executors/bounded-exec/executor.js +95 -38
- package/dist/executors/bounded-exec/mbx-pool.d.ts +17 -9
- package/dist/executors/bounded-exec/mbx-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/mbx-pool.js +29 -11
- package/dist/executors/bounded-exec/mbx-reservations.d.ts +41 -0
- package/dist/executors/bounded-exec/mbx-reservations.d.ts.map +1 -0
- package/dist/executors/bounded-exec/mbx-reservations.js +372 -0
- package/dist/executors/bounded-exec/runner-sizing.d.ts +5 -5
- package/dist/executors/bounded-exec/runner-sizing.d.ts.map +1 -1
- package/dist/executors/bounded-exec/runner-sizing.js +6 -6
- package/dist/executors/bounded-exec/verdict.d.ts +3 -2
- package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
- package/dist/executors/bounded-exec/verdict.js +6 -3
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -1
- package/package.json +1 -1
- package/src/executors/bounded-exec/executor.ts +100 -37
- package/src/executors/bounded-exec/mbx-pool.ts +33 -13
- package/src/executors/bounded-exec/mbx-reservations.ts +300 -0
- package/src/executors/bounded-exec/runner-sizing.ts +6 -6
- package/src/executors/bounded-exec/verdict.ts +10 -5
- package/src/index.ts +4 -1
- package/test-timings.json +1 -0
package/README.md
CHANGED
|
@@ -1964,31 +1964,43 @@ waiting for, however long a loaded host makes the work take; one that never retu
|
|
|
1964
1964
|
`idleTimeoutMs`. `timeoutMs`, an absolute ceiling, is for commands that report no tests; a target names at least one of
|
|
1965
1965
|
the two.
|
|
1966
1966
|
|
|
1967
|
-
With mbx admission, a
|
|
1968
|
-
|
|
1969
|
-
|
|
1970
|
-
|
|
1971
|
-
|
|
1972
|
-
|
|
1973
|
-
|
|
1974
|
-
|
|
1975
|
-
`mbx
|
|
1976
|
-
|
|
1977
|
-
|
|
1978
|
-
|
|
1979
|
-
|
|
1980
|
-
|
|
1981
|
-
|
|
1982
|
-
|
|
1983
|
-
|
|
1984
|
-
|
|
1985
|
-
|
|
1986
|
-
|
|
1987
|
-
|
|
1988
|
-
|
|
1989
|
-
|
|
1990
|
-
|
|
1991
|
-
the
|
|
1967
|
+
With mbx admission, a supported Cargo/nextest test leaf inherits a private grant pipe: mbx writes `g` on every admission
|
|
1968
|
+
and `r` on permit drop (`MBX_GRANT_FD=5`). The executor counts overlapping holds: while no hold exists it pauses the
|
|
1969
|
+
test idle budget, resuming the remaining budget on admission; a finished test still resets that budget. Total and
|
|
1970
|
+
fixture ceilings start at the first `g` and do not pause after it. EOF is not a grant or a release; a holder killed
|
|
1971
|
+
without `r` remains counted until command exit.
|
|
1972
|
+
|
|
1973
|
+
A reserved leaf takes its cores from its Nx run's mbx reservation endpoint, never from an `mbx reserve` process of its
|
|
1974
|
+
own (specs/cowshed/05_gateway.md, "An Nx run serves its leaves one reservation endpoint"). The plugin's
|
|
1975
|
+
`preTasksExecution` starts `mbx reservations <socket>` once for a run with mbx's pool and publishes the socket in
|
|
1976
|
+
`MBX_RESERVATIONS_SOCKET`; its `postTasksExecution` ends it. The leaf's shell writes the count it asks for on descriptor
|
|
1977
|
+
5 and reads back the cores granted. The executor asks the endpoint over its process's one connection, says
|
|
1978
|
+
`cowshed: mbx reserve wait start cpus=<asked> priority=<p>` and then
|
|
1979
|
+
`cowshed: mbx reserve granted elapsed=<ms> priority=<p>`, and gives the grant back once the leaf's subtree is retired;
|
|
1980
|
+
the grant starts the leaf's total and fixture ceilings as a first `g` does. The shell runs the command with
|
|
1981
|
+
`MBX_RESERVE_CPUS`, `MBX_SCHED_DISABLE=1` and an empty `MBX_SCHED_DIR`. A refusal fails the leaf, naming mbx's error,
|
|
1982
|
+
and a run with no endpoint refuses its reserved leaves.
|
|
1983
|
+
|
|
1984
|
+
Independently of that spawn-time zero-hold rule, **any live process of a Cargo leaf explicitly queued for mbx admission
|
|
1985
|
+
pauses idle**, even while another of its processes holds capacity or the runner is otherwise unsupported. The executor
|
|
1986
|
+
polls public `mbx stats --scheduler --json --activity` activity, accepting only `kind: "waiter"`, `live: true` rows
|
|
1987
|
+
whose PID is corroborated by bounded, scoped per-PID lineage reads through cowshed's native `processLineages` owner
|
|
1988
|
+
(libproc on macOS, procfs on Linux), never a spawned ps or a process-table census. Names, private waiter files and
|
|
1989
|
+
decision logs prove nothing. Public `queue_position` is preserved, never recomputed by the consumer. A successful
|
|
1990
|
+
snapshot with no task waiters resumes the remaining budget when the lifecycle and hold count permit it; a finished test
|
|
1991
|
+
resets the paused budget. An idle callback drains any in-flight poll and takes a fresh bounded snapshot before expiring.
|
|
1992
|
+
The stats read and every native lineage batch share one five-second monotonic control-plane budget, independent of the
|
|
1993
|
+
idle clock; late or cancelled snapshots never become queue or empty-membership evidence. Snapshot failures are reported
|
|
1994
|
+
operational failures, never manufactured queue/admission evidence or an idle wedge; an unidentifiable foreign legacy row
|
|
1995
|
+
is diagnosed without claiming task membership. Live diagnostics say `queued for admission N s` with the mbx queue
|
|
1996
|
+
position. A command that fails or is cancelled before its first admission records a distinct `queued` verdict with
|
|
1997
|
+
observed wait duration and its last public PID/ticket/position rows, or, for a reserved leaf, its wait on its run's
|
|
1998
|
+
endpoint and no rows. Queue observation adds no deadline and changes neither total nor fixture ceilings. Queue
|
|
1999
|
+
observations take effect at snapshot completion, never backdated across a finished test's budget reset. Observed wait
|
|
2000
|
+
duration freezes when the command settles or cancellation stops observation, before teardown or report reading. The
|
|
2001
|
+
observer requires the version-2 activity-only report; the full statistics document's cache, savings and memory histories
|
|
2002
|
+
are not polling input. The activity projection omits those walks and encodings instead of raising the observer's output
|
|
2003
|
+
or control-plane limits.
|
|
1992
2004
|
|
|
1993
2005
|
The decision to use zero-hold pausing is made before spawn, from the canonical scheduling switch and the supported
|
|
1994
2006
|
command, never from silence on the pipe. The inferred in-place
|
|
@@ -2060,8 +2072,8 @@ and its ceiling does not apply.
|
|
|
2060
2072
|
Under mbx, a reserved leaf whose target names a `unitsCommand` (a shell command printing one line per unit `command`
|
|
2061
2073
|
runs; the plugin's nextest pieces name the same `nextest list -T oneline` of their selection and partition) runs that
|
|
2062
2074
|
command at the end of its fixture segment, under the fixture's ceiling, which then starts at spawn, with the fixture's
|
|
2063
|
-
own output on stderr; the fixture runs before the reservation, not inside it. The leaf then
|
|
2064
|
-
|
|
2075
|
+
own output on stderr; the fixture runs before the reservation, not inside it. The leaf then asks its run's endpoint for
|
|
2076
|
+
`min(units, its runner's count)` cores, at least one, and its reservation says
|
|
2065
2077
|
`cowshed: mbx reserve running cpus=<granted> asked=<asked>`. A count that fails fails the run before anything is
|
|
2066
2078
|
reserved, named as the fixture segment. `unitsCommand` is refused without `fixtureCommand`.
|
|
2067
2079
|
|
package/dist/.build-inputs.json
CHANGED
|
@@ -38,13 +38,14 @@
|
|
|
38
38
|
"packages/nx-plugin/src/executors/bounded-exec/cpu-pool-thread.ts": "4b914385f7ab73cfb3afdc75c0e3c462e7c103c5061f7f7720d7b95f6655e32d",
|
|
39
39
|
"packages/nx-plugin/src/executors/bounded-exec/cpu-pool.ts": "a4be1280a0ea4b5aa0f04170ce75ce4b813631264c50fab5cdd05d34926fc890",
|
|
40
40
|
"packages/nx-plugin/src/executors/bounded-exec/disk-lease.ts": "1a3f6a6fa590d9a2550490bb6906dc91b6c05fbef235b025eaf257eb5310e92d",
|
|
41
|
-
"packages/nx-plugin/src/executors/bounded-exec/executor.ts": "
|
|
41
|
+
"packages/nx-plugin/src/executors/bounded-exec/executor.ts": "c84aa4b8f0fb53eb37123808f2c89a605f2919cfa924d22a6141970a602b8482",
|
|
42
42
|
"packages/nx-plugin/src/executors/bounded-exec/gateway-lease.ts": "f3f73bd92db30e4857f98fb8bd2b9a75b2a5a2a318f094e075ed803c1f9a0117",
|
|
43
43
|
"packages/nx-plugin/src/executors/bounded-exec/hang-diagnostics.ts": "bcaa9e52384f6c481a9cc2835ba6e8ffe167afa3cb51623fdf8dc3493876f335",
|
|
44
44
|
"packages/nx-plugin/src/executors/bounded-exec/lease-wait.ts": "59d8ac94d0b8b75858971c5bb28ee421fca4f7685577274fe8bbaf0c01ba462a",
|
|
45
45
|
"packages/nx-plugin/src/executors/bounded-exec/lock-wait.ts": "b14a23b49759a777f4b3338ced003adaf76837c2c38462458f76a7fef1b95d9e",
|
|
46
|
-
"packages/nx-plugin/src/executors/bounded-exec/mbx-pool.ts": "
|
|
46
|
+
"packages/nx-plugin/src/executors/bounded-exec/mbx-pool.ts": "4ef48c1887c1fb29000cdc581686332db6fd503ee07ccdb5f0241fb033249097",
|
|
47
47
|
"packages/nx-plugin/src/executors/bounded-exec/mbx-queue.ts": "ac309e28e5ee78fa08d628edb00e33532514f7572166770799ae86d468c12b4e",
|
|
48
|
+
"packages/nx-plugin/src/executors/bounded-exec/mbx-reservations.ts": "65087d9fdb3f7e788a127d0a7261701b7cb18b0b520ca5cef4b8f7b6fc33d3c5",
|
|
48
49
|
"packages/nx-plugin/src/executors/bounded-exec/no-space.ts": "ec032968d89f9e853796012705a06c06cfc4da54bf6e6f10a526f98daca3d077",
|
|
49
50
|
"packages/nx-plugin/src/executors/bounded-exec/owned-subtree.ts": "cb5edf2b5fe2c1b5b961f8c0ff16ec7bfae3c22ad75dfd671b74f1a4a865863f",
|
|
50
51
|
"packages/nx-plugin/src/executors/bounded-exec/progress.ts": "ed926f7b4991ebf7a7c8876ab0ea00295934d058c774e3f811963a85d209979b",
|
|
@@ -56,12 +57,12 @@
|
|
|
56
57
|
"packages/nx-plugin/src/executors/bounded-exec/run-cpu-pool.ts": "3fc4bfe04862a756d9f74743e399d902a6ce6052a0a3ff4e7edf17f875f33266",
|
|
57
58
|
"packages/nx-plugin/src/executors/bounded-exec/runner-hook.ts": "606eab5db826e571eb15df618315e01cc773d09fd647beb1c00f6434df58ffad",
|
|
58
59
|
"packages/nx-plugin/src/executors/bounded-exec/runner-lifecycle.ts": "b2a7c901296bb6c139909297bbdbab5bd00e94319c022307d006b0e1293f6454",
|
|
59
|
-
"packages/nx-plugin/src/executors/bounded-exec/runner-sizing.ts": "
|
|
60
|
+
"packages/nx-plugin/src/executors/bounded-exec/runner-sizing.ts": "bffd2ba0f80758944ca682838832bdbe20ae12eb9a4f75084a3df82ae7786145",
|
|
60
61
|
"packages/nx-plugin/src/executors/bounded-exec/runtime-pin.ts": "3eb9a8b794da51ffa01723509935f48bfb497fba883518046e74a53209f0eb10",
|
|
61
62
|
"packages/nx-plugin/src/executors/bounded-exec/schema.ts": "74eb04eaa8d90d966a98ca2b1b7fffc7d2205a55a1e38958effa641b72c680a3",
|
|
62
63
|
"packages/nx-plugin/src/executors/bounded-exec/test-result-writer.ts": "df7668a11d105a1f88d2187ef8b0edff9db7ba56d8176b5bc35daf1359667637",
|
|
63
64
|
"packages/nx-plugin/src/executors/bounded-exec/test-results.ts": "4f83dc3accd7892f45385796522267adad9401326d0cd9ce4d4e1145b9023577",
|
|
64
|
-
"packages/nx-plugin/src/executors/bounded-exec/verdict.ts": "
|
|
65
|
+
"packages/nx-plugin/src/executors/bounded-exec/verdict.ts": "1614ba94e488629280215dee1e4182fe893342e1c3e3bb124d946addbf9c4e92",
|
|
65
66
|
"packages/nx-plugin/src/executors/cargo-zigbuild/executor.ts": "622a5f36897d9eb1f7465654e88f4291aedee9767603028d20c1e819b91d72f4",
|
|
66
67
|
"packages/nx-plugin/src/executors/cargo-zigbuild/schema.ts": "e5cf3b0cc91a2d003a63ca0c9a93f1ec08263469b8a19dbe338b2dc0c0810747",
|
|
67
68
|
"packages/nx-plugin/src/executors/scriptc-native/executor.ts": "4db61da4444d3a30acd3995b6cf53a4bc403a40fe856d88b2c30ce467badbea8",
|
|
@@ -87,7 +88,7 @@
|
|
|
87
88
|
"packages/nx-plugin/src/generators/make-public/schema.d.ts": "5e7bd2f1ce16795df4c919d6d53588ea85b574c84400940b6322275f96f60357",
|
|
88
89
|
"packages/nx-plugin/src/go-tests.ts": "8e356b6ee276e6e836c404353b0ffb5ccec5beb261126f167f730e6e01e9c2b6",
|
|
89
90
|
"packages/nx-plugin/src/graph-input-contract.ts": "8a1aec1294960ae80bcff55a0370c95004a911bf1afe6caa47e666da3baeac43",
|
|
90
|
-
"packages/nx-plugin/src/index.ts": "
|
|
91
|
+
"packages/nx-plugin/src/index.ts": "223dd22b1bc871a24c9a7a7b66e2d496870cac40b52281f7119356a09287a96a",
|
|
91
92
|
"packages/nx-plugin/src/managed-assets.ts": "9cea959c09f3163992e666eecc5a65ec1f1b63c81fc56c5e99088507d02c5e3b",
|
|
92
93
|
"packages/nx-plugin/src/managed-files/ci-workflow.ts": "2d64d689e999af20d65cc21b4803659a835a6eb5340e8d3f896cb0c74e6a0878",
|
|
93
94
|
"packages/nx-plugin/src/managed-files/context.ts": "5b83c000322382cb781df4f415f3759388e69686b76779599572a62a5c4dd28c",
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/executor.ts"],"names":[],"mappings":"AA0BA,OAAO,EAIL,KAAK,gBAAgB,EAItB,MAAM,eAAe,CAAC;AAGvB,OAAO,EAAE,sBAAsB,EAAmB,MAAM,uBAAuB,CAAC;
|
|
1
|
+
{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/executor.ts"],"names":[],"mappings":"AA0BA,OAAO,EAIL,KAAK,gBAAgB,EAItB,MAAM,eAAe,CAAC;AAGvB,OAAO,EAAE,sBAAsB,EAAmB,MAAM,uBAAuB,CAAC;AAYhF,OAAO,EAA4C,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAG5F,OAAO,EAOL,KAAK,YAAY,EAClB,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EASL,aAAa,EAEd,MAAM,eAAe,CAAC;AAIvB,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAEtD,OAAO,EAGL,KAAK,eAAe,EAerB,MAAM,cAAc,CAAC;AAuBtB,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,wFAAwF;IACxF,MAAM,CAAC,EAAE;QAAE,IAAI,EAAE,eAAe,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC;IACtD,uFAAuF;IACvF,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;CAClC;AAED,qEAAqE;AACrE,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,SAAS,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE;YAAE,IAAI,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;CAC1D;AAED,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,OAAO,CAAC;IACjB,cAAc,EAAE,MAAM,CAAC;CACxB;AAkBD,MAAM,WAAW,iBAAiB;IAChC,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1D;AAED,8EAA8E;AAC9E,MAAM,MAAM,UAAU,GAAG,IAAI,CAC3B,aAAa,EACb,SAAS,GAAG,SAAS,GAAG,UAAU,GAAG,OAAO,GAAG,QAAQ,GAAG,gBAAgB,CAC3E,CAAC;AAQF;;;;;;GAMG;AACH,wBAAgB,wBAAwB,CACtC,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,MAAM,GAAG,SAAS,EACzB,SAAS,EAAE,MAAM,GAAG,SAAS,EAC7B,SAAS,EAAE,MAAM,GAAG,SAAS,EAC7B,GAAG,EAAE,MAAM,EACX,MAAM,GAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAOzB,GACA,MAAM,GAAG,SAAS,CAiBpB;AAED,MAAM,CAAC,OAAO,UAAU,mBAAmB,CACzC,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,aAAa,GACrB,OAAO,CAAC,iBAAiB,CAAC,CAE5B;AAED;;;GAGG;AACH,wBAAgB,qBAAqB,CACnC,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,kBAAkB,GAC1B,OAAO,CAAC,iBAAiB,CAAC,CAc5B;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAChC,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,kBAAkB,EAC3B,MAAM,EAAE,CAAC,OAAO,EAAE,MAAM,GAAG,SAAS,KAAK,UAAU,GAAG,IAAI,GACzD,OAAO,CAAC,iBAAiB,CAAC,CAsB5B;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAAC,OAAO,EAAE,aAAa,GAAG,kBAAkB,CAiBxE;AAED;;;;;;;;GAQG;AACH,wBAAsB,cAAc,CAClC,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,kBAAkB,EAC3B,MAAM,EAAE,iBAAiB,EACzB,UAAU,EAAE,UAAU,GAAG,IAAI,EAC7B,SAAS,GAAE,gBAAgB,GAAG,IAAW,EACzC,kBAAkB,GAAE,OAAO,sBAA+C,EAC1E,KAAK,GAAE,YAAY,GAAG,IAAW,EACjC,SAAS,CAAC,EAAE,UAAU,CAAC,OAAO,gBAAgB,CAAC,CAAC,CAAC,CAAC,GACjD,OAAO,CAAC,iBAAiB,CAAC,CAwlC5B;AAED,wBAAgB,uBAAuB,IAAI,iBAAiB,CAM3D"}
|
|
@@ -11,8 +11,9 @@ import { admissionWaitMs, earliestBound, NO_ADMISSION_WAIT, stepAdmissionWait, s
|
|
|
11
11
|
import { CPU_POOL_ENV, createCpuPool, readInheritedCpuPool, takeParentCpuTokens, } from './cpu-pool.js';
|
|
12
12
|
import { GATEWAY_SOCKET } from './gateway-lease.js';
|
|
13
13
|
import { captureHangDiagnostics, HANG_CAPTURE_MS } from './hang-diagnostics.js';
|
|
14
|
-
import { MBX_GRANT_FD, mbxIdleGated, mbxLeaf, mbxPool, withCountedReservation, withMbxLeaf } from './mbx-pool.js';
|
|
14
|
+
import { askedCores, MBX_GRANT_FD, mbxIdleGated, mbxLeaf, mbxPool, withCountedReservation, withMbxLeaf, } from './mbx-pool.js';
|
|
15
15
|
import { MBX_QUEUE_POLL_MS, snapshotMbxQueue } from './mbx-queue.js';
|
|
16
|
+
import { RESERVATIONS_ENV, reserveCores } from './mbx-reservations.js';
|
|
16
17
|
import { createNoSpaceWatch } from './no-space.js';
|
|
17
18
|
import { COWSHED_EXECUTABLE_ENV, cowshedSubtreeOwner, HANDLE_FD, readAdmission, SUPERVISOR_SOCKET_ENV, } from './owned-subtree.js';
|
|
18
19
|
import { createRunProgress, withProgressReport, withTestResultReport } from './progress.js';
|
|
@@ -394,8 +395,11 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
394
395
|
sized = sizeRunner(demand, requested, demand.want);
|
|
395
396
|
}
|
|
396
397
|
// The run's sub-pool and grant are given back exactly once, however the run ends. The sub-pool closes first, so no
|
|
397
|
-
// descendant still draws on tokens the grant has returned.
|
|
398
|
+
// descendant still draws on tokens the grant has returned. A reserved leaf's mbx grant covers the leaf's conclusion,
|
|
399
|
+
// its subtree's retirement included (11_shell.md), so it goes back after `releaseLease`.
|
|
398
400
|
let pool = null;
|
|
401
|
+
let reservation = null;
|
|
402
|
+
const releaseReservation = () => reservation?.release();
|
|
399
403
|
let cpuReleased = null;
|
|
400
404
|
const releaseCpu = () => {
|
|
401
405
|
cpuReleased ??= (async () => {
|
|
@@ -423,7 +427,8 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
423
427
|
let unconfirmedAdmission = null;
|
|
424
428
|
/** When the last successful snapshot completed (`Date.now()`): credit runs to it and no further. */
|
|
425
429
|
let confirmedAdmissionAt = null;
|
|
426
|
-
|
|
430
|
+
// A reserved leaf's own request on its run's endpoint is its admission wait; a Cargo leaf's comes from mbx's report.
|
|
431
|
+
const queueWait = () => mbxAdmitted || (lastQueueRequests.length === 0 && reservation === null)
|
|
427
432
|
? null
|
|
428
433
|
: {
|
|
429
434
|
waitedMs: admissionWaitMs(admissionWait, performance.now()),
|
|
@@ -494,6 +499,12 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
494
499
|
if (parentRefusal !== null) {
|
|
495
500
|
return refuse(`cowshed: cpu-pool parent pool refused: ${parentRefusal}; refusing to run the command\n`);
|
|
496
501
|
}
|
|
502
|
+
// A reserved leaf takes its cores from its Nx run's mbx reservation endpoint (mbx-reservations.ts). A run that serves
|
|
503
|
+
// none said why in its own output; the leaf does not run beside the pool.
|
|
504
|
+
const reserved = leaf?.kind === 'reserve' ? { leaf, endpoint: process.env[RESERVATIONS_ENV] ?? '' } : null;
|
|
505
|
+
if (reserved !== null && reserved.endpoint === '') {
|
|
506
|
+
return refuse(`cowshed: a reserved leaf takes its cores from its Nx run's mbx reservation endpoint, and ${RESERVATIONS_ENV} names none (the run's preTasksExecution serves it); refusing to run the command\n`);
|
|
507
|
+
}
|
|
497
508
|
// A granted run serves a sub-pool of its own grant, so its descendants draw from that and never from above it.
|
|
498
509
|
// A pool that cannot be set up refuses the run: no granted command runs beside its grant.
|
|
499
510
|
if (grant !== null) {
|
|
@@ -539,7 +550,8 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
539
550
|
: withMbxLeaf(unadmitted, leaf);
|
|
540
551
|
// Only leaves known to be scheduled can exclude zero-hold time. Others keep lifecycle hang protection.
|
|
541
552
|
const queued = mbxIdleGated(leaf, sized.command, { ...process.env, ...env });
|
|
542
|
-
// fd 4 is unused without a subtree admission, so
|
|
553
|
+
// fd 4 is unused without a subtree admission, so mbx's descriptor takes fd 5 either way: a Cargo leaf's admission
|
|
554
|
+
// stream, or a reserved leaf's ask for cores and their count (mbx-pool.ts, `MBX_GRANT_FD`).
|
|
543
555
|
const grantSlots = leaf !== null ? ['ignore', 'pipe'] : [];
|
|
544
556
|
const rootGrantSlot = leaf !== null ? ['pipe'] : [];
|
|
545
557
|
const root = owner?.rootArgv(shellCommand);
|
|
@@ -550,7 +562,7 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
550
562
|
resultWriter = results?.ok ? results.writer : null;
|
|
551
563
|
try {
|
|
552
564
|
// fd 3 is the runner lifecycle's duplex socket when the command is hooked; the subtree admission takes fd 4, which
|
|
553
|
-
// cowshed closes fd 4 before exec; fd 5
|
|
565
|
+
// cowshed closes fd 4 before exec; fd 5 is a Cargo leaf's mbx admission stream or a reserved leaf's ask and grant.
|
|
554
566
|
child =
|
|
555
567
|
root === undefined
|
|
556
568
|
? spawn(shellCommand, [], {
|
|
@@ -681,7 +693,7 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
681
693
|
const scheduled = earliestBound(checkingIdle ? { ...bounds, idle: null } : bounds);
|
|
682
694
|
if (scheduled?.kind !== kind || scheduled.at !== at)
|
|
683
695
|
return;
|
|
684
|
-
if (kind === 'idle' && leaf
|
|
696
|
+
if (kind === 'idle' && leaf?.kind === 'cargo') {
|
|
685
697
|
if (checkingIdle)
|
|
686
698
|
return;
|
|
687
699
|
checkingIdle = true;
|
|
@@ -738,9 +750,6 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
738
750
|
const refreshQueue = async (fresh) => {
|
|
739
751
|
if (fresh && queueSnapshot !== null)
|
|
740
752
|
await queueSnapshot;
|
|
741
|
-
// A leaf that counts its units before reserving waits in no reservation queue while its fixture builds.
|
|
742
|
-
if (counted && !fixtureDone)
|
|
743
|
-
return;
|
|
744
753
|
if (state.settled || state.expiry !== null || queueAbort.signal.aborted || child.pid === undefined)
|
|
745
754
|
return;
|
|
746
755
|
const pid = child.pid;
|
|
@@ -883,13 +892,14 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
883
892
|
clearTimeout(boundTimer);
|
|
884
893
|
// Preserve explicit live membership before cancellation removes the queued process. A failed bounded observation
|
|
885
894
|
// leaves the admission unconfirmed, never a manufactured queue or admission.
|
|
886
|
-
signalCleanup = (leaf
|
|
895
|
+
signalCleanup = (leaf?.kind !== 'cargo' ? Promise.resolve() : refreshQueue(false))
|
|
887
896
|
.finally(() => {
|
|
888
897
|
stopQueueObservation();
|
|
889
898
|
return killChildTree();
|
|
890
899
|
})
|
|
891
900
|
.finally(releaseCpu)
|
|
892
901
|
.finally(() => releaseLease(true))
|
|
902
|
+
.finally(releaseReservation)
|
|
893
903
|
.finally(async () => recordRun(signalToExitCode(signal), state.expiry, Date.now() - startedAt, state.poolCloseError !== null || lifecycleFailure !== null || state.retireError !== null, state.noSpace === null ? null : await state.noSpace, state.lockWaits))
|
|
894
904
|
.finally(() => process.kill(process.pid, signal));
|
|
895
905
|
};
|
|
@@ -1028,12 +1038,10 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
1028
1038
|
fixtureFirst: counted && lifecycle !== null,
|
|
1029
1039
|
queued,
|
|
1030
1040
|
});
|
|
1031
|
-
// Read every byte after spawn accounting: a chunk can contain several grants and releases. EOF is neither.
|
|
1032
1041
|
if (leaf !== null) {
|
|
1033
1042
|
const channel = child.stdio.at(MBX_GRANT_FD);
|
|
1034
|
-
if (!(channel instanceof
|
|
1035
|
-
throw new Error('an mbx leaf requires its
|
|
1036
|
-
let said = false;
|
|
1043
|
+
if (!(channel instanceof Duplex))
|
|
1044
|
+
throw new Error('an mbx leaf requires its mbx descriptor');
|
|
1037
1045
|
const failGrant = (detail) => {
|
|
1038
1046
|
if (lifecycleFailure !== null || state.expiry !== null || state.settled)
|
|
1039
1047
|
return;
|
|
@@ -1041,34 +1049,82 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
1041
1049
|
appendStderr(`bounded-exec: ${lifecycleFailure}\n`);
|
|
1042
1050
|
termination = terminate();
|
|
1043
1051
|
};
|
|
1044
|
-
channel.on('
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
}
|
|
1054
|
-
|
|
1055
|
-
}
|
|
1056
|
-
else if (byte === 114) {
|
|
1057
|
-
stepTo({ kind: 'released', at: Date.now() });
|
|
1052
|
+
channel.on('error', (error) => failGrant(error.message));
|
|
1053
|
+
if (reserved !== null) {
|
|
1054
|
+
// The shell asks for its cores in one line and reads the count granted (mbx-pool.ts, `reservation`). A refusal
|
|
1055
|
+
// ends the descriptor unanswered: the shell's read fails, and nothing runs.
|
|
1056
|
+
const { leaf: reservedLeaf, endpoint } = reserved;
|
|
1057
|
+
let asked = '';
|
|
1058
|
+
channel.setEncoding('utf8');
|
|
1059
|
+
channel.on('data', (text) => {
|
|
1060
|
+
if (reservation !== null) {
|
|
1061
|
+
failGrant(`the command asked for cores a second time: ${JSON.stringify(text)}`);
|
|
1062
|
+
return;
|
|
1058
1063
|
}
|
|
1059
|
-
|
|
1060
|
-
|
|
1064
|
+
asked += text;
|
|
1065
|
+
const end = asked.indexOf('\n');
|
|
1066
|
+
if (end === -1)
|
|
1067
|
+
return;
|
|
1068
|
+
const line = asked.slice(0, end);
|
|
1069
|
+
const cpus = askedCores(line, reservedLeaf.demand.want);
|
|
1070
|
+
if (cpus === null) {
|
|
1071
|
+
failGrant(`the command asked for ${JSON.stringify(line)} cores, not 1 to ${reservedLeaf.demand.want}`);
|
|
1072
|
+
channel.end();
|
|
1061
1073
|
return;
|
|
1062
1074
|
}
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1075
|
+
admissionWait = stepAdmissionWait(admissionWait, { kind: 'snapshot', at: performance.now(), waiting: true });
|
|
1076
|
+
tell(`cowshed: mbx reserve wait start cpus=${cpus} priority=${reservedLeaf.priority}\n`);
|
|
1077
|
+
const held = reserveCores(endpoint, cpus, reservedLeaf.priority);
|
|
1078
|
+
reservation = held;
|
|
1079
|
+
void held.granted.then((granted) => {
|
|
1080
|
+
if (!granted.ok) {
|
|
1081
|
+
failGrant(`mbx did not grant the reservation: ${granted.error}`);
|
|
1082
|
+
channel.end();
|
|
1083
|
+
return;
|
|
1084
|
+
}
|
|
1085
|
+
// A run that ended while it waited gives the grant back with its CPU (`releaseCpu`).
|
|
1086
|
+
if (state.settled || state.expiry !== null)
|
|
1087
|
+
return;
|
|
1088
|
+
mbxAdmitted = true;
|
|
1089
|
+
tell(`cowshed: mbx reserve granted elapsed=${Date.now() - startedAt}ms priority=${reservedLeaf.priority}\n`);
|
|
1090
|
+
channel.end(`${granted.cpus}\n`);
|
|
1091
|
+
stepTo({ kind: 'granted', at: Date.now(), testing: isTesting(), fixture: !fixtureDone });
|
|
1092
|
+
void granted.lost.then((why) => {
|
|
1093
|
+
if (state.settled || cpuReleased !== null)
|
|
1094
|
+
return;
|
|
1095
|
+
appendStderr(`cowshed: mbx reservation lost while the command ran (${why}); it runs on unheld until it ends\n`);
|
|
1096
|
+
});
|
|
1097
|
+
});
|
|
1098
|
+
});
|
|
1099
|
+
}
|
|
1100
|
+
else {
|
|
1101
|
+
// Read every byte after spawn accounting: a chunk can contain several grants and releases. EOF is neither.
|
|
1102
|
+
let said = false;
|
|
1103
|
+
channel.on('data', (chunk) => {
|
|
1104
|
+
for (const byte of chunk) {
|
|
1105
|
+
if (byte === 103) {
|
|
1106
|
+
if (!said) {
|
|
1107
|
+
mbxAdmitted = true;
|
|
1108
|
+
said = true;
|
|
1109
|
+
tell(`cowshed: mbx cargo granted elapsed=${Date.now() - startedAt}ms\n`);
|
|
1110
|
+
}
|
|
1111
|
+
stepTo({ kind: 'granted', at: Date.now(), testing: isTesting(), fixture: !fixtureDone });
|
|
1112
|
+
}
|
|
1113
|
+
else if (byte === 114) {
|
|
1114
|
+
stepTo({ kind: 'released', at: Date.now() });
|
|
1115
|
+
}
|
|
1116
|
+
else {
|
|
1117
|
+
failGrant(`unexpected byte ${byte}`);
|
|
1118
|
+
return;
|
|
1119
|
+
}
|
|
1120
|
+
}
|
|
1121
|
+
});
|
|
1122
|
+
// Only mbx's activity report says whether a Cargo leaf's compiler or test waits in the pool.
|
|
1123
|
+
queueTimer = setInterval(() => {
|
|
1124
|
+
void refreshQueue(false);
|
|
1125
|
+
}, MBX_QUEUE_POLL_MS);
|
|
1069
1126
|
void refreshQueue(false);
|
|
1070
|
-
}
|
|
1071
|
-
void refreshQueue(false);
|
|
1127
|
+
}
|
|
1072
1128
|
}
|
|
1073
1129
|
// A grant's connection is a ledger entry, not the command's owner: the lending parent's exit ends it, and what runs
|
|
1074
1130
|
// under it runs on over budget until it finishes. Its close, while the command runs, is said and changes nothing
|
|
@@ -1120,6 +1176,7 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
1120
1176
|
}
|
|
1121
1177
|
}
|
|
1122
1178
|
await releaseLease(code !== 0 || state.expiry !== null || state.poolCloseError !== null || lifecycleFailure !== null);
|
|
1179
|
+
releaseReservation();
|
|
1123
1180
|
// Broken resource ownership, a lost native lifecycle, or leftovers that could not be retired fail even if a shell
|
|
1124
1181
|
// branch later exited 0: its verdict is an ordinary operational failure, never a successful or silently unbounded run.
|
|
1125
1182
|
const operationalFailure = state.poolCloseError !== null || lifecycleFailure !== null || state.retireError !== null;
|
|
@@ -10,17 +10,19 @@ export interface MbxPool {
|
|
|
10
10
|
}
|
|
11
11
|
/**
|
|
12
12
|
* The pool the environment names, or `null` where the project has no mbx and cowshed's own pools apply unchanged, or
|
|
13
|
-
* where this run is already inside an mbx reservation:
|
|
14
|
-
* non-Cargo work shares the enclosing grant
|
|
15
|
-
* inside a job"). A second reservation would
|
|
16
|
-
* nested leaf queued behind the very work it
|
|
13
|
+
* where this run is already inside an mbx reservation: whatever holds one hands its work `MBX_SCHED_DISABLE=1` (a
|
|
14
|
+
* reserved leaf's shell, `mbx reserve`, a held cowshed job), and nested non-Cargo work shares the enclosing grant
|
|
15
|
+
* instead of reserving twice (specs/cowshed/05_gateway.md, "Parallelism inside a job"). A second reservation would
|
|
16
|
+
* wait in the pool while its parent holds a grant, so under a full pool the nested leaf queued behind the very work it
|
|
17
|
+
* belongs to.
|
|
17
18
|
*/
|
|
18
19
|
export declare function mbxPool(environment: Readonly<NodeJS.ProcessEnv>): MbxPool | null;
|
|
19
20
|
/**
|
|
20
21
|
* How a leaf enters the pool once. A Cargo leaf that builds runs through mbx, which admits its compilers and supported
|
|
21
|
-
* test work itself, so wrapping it would charge it twice. Any other leaf reserves for its whole run
|
|
22
|
-
*
|
|
23
|
-
* the runner runs on the count it is granted (`sizeRunnerToGrant`). That includes a
|
|
22
|
+
* test work itself, so wrapping it would charge it twice. Any other leaf reserves for its whole run, asking its Nx
|
|
23
|
+
* run's mbx reservation endpoint (mbx-reservations.ts) for at most its runner's count: mbx grants as many of those
|
|
24
|
+
* cores as are free once one is, and the runner runs on the count it is granted (`sizeRunnerToGrant`). That includes a
|
|
25
|
+
* nextest run of test binaries
|
|
24
26
|
* already built (the inferred extracted-archive run, or one from `--archive-file`): it compiles nothing, and beneath a
|
|
25
27
|
* reservation mbx schedules none of its test processes, so they run inside the one grant. Admitted one by one under
|
|
26
28
|
* `MBX_SCHEDULER_TESTS=1`, each queued for a ticket of its own: in a downstream gate on 2026-10-30, codec tests of at
|
|
@@ -55,8 +57,9 @@ export declare function mbxLeaf(pool: MbxPool, demand: Demand, command: string):
|
|
|
55
57
|
*/
|
|
56
58
|
export declare function mbxIdleGated(leaf: MbxLeaf | null, command: string, environment: Readonly<NodeJS.ProcessEnv>): boolean;
|
|
57
59
|
/**
|
|
58
|
-
*
|
|
59
|
-
*
|
|
60
|
+
* The descriptor between a pooled leaf's shell and its executor. For a Cargo leaf mbx writes one byte on it per
|
|
61
|
+
* admission (g) and permit drop (r), every Cargo descendant inheriting it through MBX_GRANT_FD. A reserved leaf's shell
|
|
62
|
+
* writes the cores it asks for on it, one line, and reads back the count the pool granted (`reservation`).
|
|
60
63
|
*/
|
|
61
64
|
export declare const MBX_GRANT_FD = 5;
|
|
62
65
|
/**
|
|
@@ -71,4 +74,9 @@ export declare function withMbxLeaf(command: string, leaf: MbxLeaf | null): stri
|
|
|
71
74
|
* is reserved.
|
|
72
75
|
*/
|
|
73
76
|
export declare function withCountedReservation(fixture: string, command: string, leaf: ReservedLeaf): string;
|
|
77
|
+
/**
|
|
78
|
+
* The cores a reserved leaf's shell asked for on `MBX_GRANT_FD` (`reservation`): a whole count from one to `most`, the
|
|
79
|
+
* most its runner may ask for; anything else is `null`.
|
|
80
|
+
*/
|
|
81
|
+
export declare function askedCores(line: string, most: number): number | null;
|
|
74
82
|
//# sourceMappingURL=mbx-pool.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"mbx-pool.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/mbx-pool.ts"],"names":[],"mappings":"AACA,OAAO,EAEL,KAAK,MAAM,
|
|
1
|
+
{"version":3,"file":"mbx-pool.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/mbx-pool.ts"],"names":[],"mappings":"AACA,OAAO,EAEL,KAAK,MAAM,EAIX,KAAK,cAAc,EAEpB,MAAM,oBAAoB,CAAC;AAG5B;;;;;GAKG;AACH,MAAM,WAAW,OAAO;IACtB,QAAQ,CAAC,QAAQ,EAAE,QAAQ,GAAG,KAAK,CAAC;CACrC;AAED;;;;;;;GAOG;AACH,wBAAgB,OAAO,CAAC,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,OAAO,GAAG,IAAI,CAKhF;AAED;;;;;;;;;;;GAWG;AACH,MAAM,MAAM,OAAO,GACf;IAAE,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAA;CAAE,GAC1B;IACE,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IACzB,QAAQ,CAAC,QAAQ,EAAE,QAAQ,GAAG,KAAK,CAAC;IACpC,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAC;CACjC,CAAC;AAEN,sCAAsC;AACtC,MAAM,MAAM,YAAY,GAAG,OAAO,CAAC,OAAO,EAAE;IAAE,IAAI,EAAE,SAAS,CAAA;CAAE,CAAC,CAAC;AAKjE,+FAA+F;AAC/F,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAE5D;AAED;;;;;;GAMG;AACH,wBAAgB,OAAO,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,GAAG,IAAI,CAoBtF;AAQD;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,OAAO,GAAG,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,OAAO,CAkDrH;AAED;;;;GAIG;AACH,eAAO,MAAM,YAAY,IAAI,CAAC;AAE9B;;;GAGG;AACH,wBAAgB,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,GAAG,IAAI,GAAG,MAAM,CAKzE;AAED;;;;;GAKG;AACH,wBAAgB,sBAAsB,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,GAAG,MAAM,CAInG;AAkBD;;;GAGG;AACH,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAIpE"}
|
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
import { shellTokens } from '../../runtime-input-policy.js';
|
|
2
|
-
import { CPU_TOKENS_ENV, grantedSizing, invocationSpan, WANT_ENV, } from './runner-sizing.js';
|
|
2
|
+
import { CPU_TOKENS_ENV, GRANTED_CPUS_ENV, grantedSizing, invocationSpan, WANT_ENV, } from './runner-sizing.js';
|
|
3
3
|
import { shellQuote } from './verdict.js';
|
|
4
4
|
/**
|
|
5
5
|
* The pool the environment names, or `null` where the project has no mbx and cowshed's own pools apply unchanged, or
|
|
6
|
-
* where this run is already inside an mbx reservation:
|
|
7
|
-
* non-Cargo work shares the enclosing grant
|
|
8
|
-
* inside a job"). A second reservation would
|
|
9
|
-
* nested leaf queued behind the very work it
|
|
6
|
+
* where this run is already inside an mbx reservation: whatever holds one hands its work `MBX_SCHED_DISABLE=1` (a
|
|
7
|
+
* reserved leaf's shell, `mbx reserve`, a held cowshed job), and nested non-Cargo work shares the enclosing grant
|
|
8
|
+
* instead of reserving twice (specs/cowshed/05_gateway.md, "Parallelism inside a job"). A second reservation would
|
|
9
|
+
* wait in the pool while its parent holds a grant, so under a full pool the nested leaf queued behind the very work it
|
|
10
|
+
* belongs to.
|
|
10
11
|
*/
|
|
11
12
|
export function mbxPool(environment) {
|
|
12
13
|
if (environment.MBX_SCHEDULER !== '1' || !environment.MBX_CACHE_DIR || environment.MBX_SCHED_DISABLE === '1') {
|
|
@@ -118,8 +119,9 @@ export function mbxIdleGated(leaf, command, environment) {
|
|
|
118
119
|
}
|
|
119
120
|
}
|
|
120
121
|
/**
|
|
121
|
-
*
|
|
122
|
-
*
|
|
122
|
+
* The descriptor between a pooled leaf's shell and its executor. For a Cargo leaf mbx writes one byte on it per
|
|
123
|
+
* admission (g) and permit drop (r), every Cargo descendant inheriting it through MBX_GRANT_FD. A reserved leaf's shell
|
|
124
|
+
* writes the cores it asks for on it, one line, and reads back the count the pool granted (`reservation`).
|
|
123
125
|
*/
|
|
124
126
|
export const MBX_GRANT_FD = 5;
|
|
125
127
|
/**
|
|
@@ -144,11 +146,27 @@ export function withCountedReservation(fixture, command, leaf) {
|
|
|
144
146
|
return `bounded_exec_units=$(${fixture}) && ${WANT_ENV}=$(printf '%s\\n' "$bounded_exec_units" | { ${count}; }) && ${reservation(command, leaf)}`;
|
|
145
147
|
}
|
|
146
148
|
/**
|
|
147
|
-
* The reservation of `WANT_ENV` cores, set by the caller, that runs `command` once mbx grants it
|
|
148
|
-
*
|
|
149
|
-
*
|
|
149
|
+
* The reservation of `WANT_ENV` cores, set by the caller, that runs `command` once mbx grants it. The shell asks its
|
|
150
|
+
* executor on `MBX_GRANT_FD`, one line naming the count; the executor asks its run's mbx reservation endpoint and
|
|
151
|
+
* answers the cores granted, one line, or ends the descriptor unanswered when mbx refused, so the read fails and
|
|
152
|
+
* nothing runs (executor.ts). The shell then closes the descriptor, which the command never inherits, gives the command
|
|
153
|
+
* what `mbx reserve` gives its own (`GRANTED_CPUS_ENV`, `MBX_SCHED_DISABLE=1` and an empty `MBX_SCHED_DIR`, so mbx work
|
|
154
|
+
* inside runs under the grant), sizes the runner to the grant (`grantedSizing`), says so on stderr and runs the command
|
|
155
|
+
* in the same shell: no process starts for the reservation.
|
|
150
156
|
*/
|
|
151
157
|
function reservation(command, leaf) {
|
|
152
158
|
const said = `printf 'cowshed: mbx reserve running cpus=%s asked=%s\\n' "$${CPU_TOKENS_ENV}" "$${WANT_ENV}" >&2; `;
|
|
153
|
-
|
|
159
|
+
const ask = `printf '%s\\n' "$${WANT_ENV}" >&${MBX_GRANT_FD} && IFS= read -r ${GRANTED_CPUS_ENV} <&${MBX_GRANT_FD} && exec ${MBX_GRANT_FD}>&-`;
|
|
160
|
+
const held = `export ${GRANTED_CPUS_ENV} MBX_SCHED_DISABLE=1 MBX_SCHED_DIR=`;
|
|
161
|
+
return `export ${WANT_ENV} && ${ask} && ${held} && eval ${shellQuote(`${grantedSizing(leaf.demand)}${said}${command}`)}`;
|
|
162
|
+
}
|
|
163
|
+
/**
|
|
164
|
+
* The cores a reserved leaf's shell asked for on `MBX_GRANT_FD` (`reservation`): a whole count from one to `most`, the
|
|
165
|
+
* most its runner may ask for; anything else is `null`.
|
|
166
|
+
*/
|
|
167
|
+
export function askedCores(line, most) {
|
|
168
|
+
if (!/^[1-9]\d{0,9}$/.test(line))
|
|
169
|
+
return null;
|
|
170
|
+
const cores = Number(line);
|
|
171
|
+
return cores <= most ? cores : null;
|
|
154
172
|
}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import type { PostTasksExecutionContext, PreTasksExecutionContext } from 'nx/src/project-graph/plugins/public-api.js';
|
|
2
|
+
/**
|
|
3
|
+
* mbx's reservation endpoint of one Nx run (specs/cowshed/05_gateway.md, "Parallelism inside a job"). A reserved leaf
|
|
4
|
+
* once ran under its own `mbx reserve`, two mbx processes per task whose dynamic loading alone kept the host's vnode
|
|
5
|
+
* table recycling. With mbx's pool, the run's `preTasksExecution` starts `mbx reservations <socket>` once and publishes
|
|
6
|
+
* the socket in `RESERVATIONS_ENV`, which every task inherits; each task's executor asks it for its leaf's cores over
|
|
7
|
+
* one connection per process (`reserveCores`), and the run's `postTasksExecution` closes the endpoint's standard input,
|
|
8
|
+
* which ends it. Admission is `mbx reserve`'s: the same pool, priority and order. A grant lasts until its leaf ends, or
|
|
9
|
+
* until its connection closes, so an executor that dies gives back what it held.
|
|
10
|
+
*/
|
|
11
|
+
export declare const RESERVATIONS_ENV = "MBX_RESERVATIONS_SOCKET";
|
|
12
|
+
/**
|
|
13
|
+
* Serve `context`'s run an mbx reservation endpoint and publish it in `env`, which every task inherits, when the
|
|
14
|
+
* environment names mbx's pool (`mbxPool`). An endpoint that does not start is said, and each reserved leaf of the run
|
|
15
|
+
* then refuses to run, naming `RESERVATIONS_ENV`; a leaf in no pool, or one mbx admits itself, runs as before.
|
|
16
|
+
*/
|
|
17
|
+
export declare function serveRunReservations(context: Pick<PreTasksExecutionContext, 'id'>, env?: NodeJS.ProcessEnv): Promise<void>;
|
|
18
|
+
/** Close the endpoint `context`'s run was served, if it was served one: its standard input's end ends it. */
|
|
19
|
+
export declare function closeRunReservations(context: Pick<PostTasksExecutionContext, 'id'>): Promise<void>;
|
|
20
|
+
/** The pool's answer to one reservation: the cores it holds and a promise of why it was lost, or why there are none. */
|
|
21
|
+
export type Granted = {
|
|
22
|
+
readonly ok: true;
|
|
23
|
+
readonly cpus: number;
|
|
24
|
+
readonly lost: Promise<string>;
|
|
25
|
+
} | {
|
|
26
|
+
readonly ok: false;
|
|
27
|
+
readonly error: string;
|
|
28
|
+
};
|
|
29
|
+
/** One request for cores on a run's endpoint. */
|
|
30
|
+
export interface Reservation {
|
|
31
|
+
/** Settles once the pool grants or refuses it, or its connection ends first. */
|
|
32
|
+
readonly granted: Promise<Granted>;
|
|
33
|
+
/** Gives the grant back, or withdraws the request while it waits. Later calls do nothing. */
|
|
34
|
+
release(): void;
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Ask the endpoint at `socket` for at most `cpus` cores at `priority`, over this process's one connection to it,
|
|
38
|
+
* opened by its first request and kept while another one may come.
|
|
39
|
+
*/
|
|
40
|
+
export declare function reserveCores(socket: string, cpus: number, priority: 'normal' | 'low'): Reservation;
|
|
41
|
+
//# sourceMappingURL=mbx-reservations.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"mbx-reservations.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/mbx-reservations.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,yBAAyB,EAAE,wBAAwB,EAAE,MAAM,4CAA4C,CAAC;AAMtH;;;;;;;;GAQG;AACH,eAAO,MAAM,gBAAgB,4BAA4B,CAAC;AAmB1D;;;;GAIG;AACH,wBAAsB,oBAAoB,CACxC,OAAO,EAAE,IAAI,CAAC,wBAAwB,EAAE,IAAI,CAAC,EAC7C,GAAG,GAAE,MAAM,CAAC,UAAwB,GACnC,OAAO,CAAC,IAAI,CAAC,CAaf;AAED,6GAA6G;AAC7G,wBAAsB,oBAAoB,CAAC,OAAO,EAAE,IAAI,CAAC,yBAAyB,EAAE,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAYxG;AAsED,wHAAwH;AACxH,MAAM,MAAM,OAAO,GACf;IAAE,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC,CAAA;CAAE,GAC5E;IAAE,QAAQ,CAAC,EAAE,EAAE,KAAK,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC;AAEnD,iDAAiD;AACjD,MAAM,WAAW,WAAW;IAC1B,gFAAgF;IAChF,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;IACnC,6FAA6F;IAC7F,OAAO,IAAI,IAAI,CAAC;CACjB;AAgID;;;GAGG;AACH,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,GAAG,KAAK,GAAG,WAAW,CAUlG"}
|