@cleocode/core 2026.10.26 → 2026.10.27-rc.ci.698
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/build-provenance.json +2 -2
- package/dist/docs/docs-ops.d.ts.map +1 -1
- package/dist/docs/docs-ops.js +29 -11
- package/dist/docs/docs-ops.js.map +1 -1
- package/dist/resources/admission-ledger.d.ts +45 -1
- package/dist/resources/admission-ledger.d.ts.map +1 -1
- package/dist/resources/admission-ledger.js +232 -29
- package/dist/resources/admission-ledger.js.map +1 -1
- package/dist/resources/hook-footprint.d.ts +2 -1
- package/dist/resources/hook-footprint.d.ts.map +1 -1
- package/dist/resources/hook-footprint.js +11 -2
- package/dist/resources/hook-footprint.js.map +1 -1
- package/dist/resources/run-admission.d.ts +15 -0
- package/dist/resources/run-admission.d.ts.map +1 -1
- package/dist/resources/run-admission.js.map +1 -1
- package/dist/resources/run-class.d.ts +74 -7
- package/dist/resources/run-class.d.ts.map +1 -1
- package/dist/resources/run-class.js +147 -16
- package/dist/resources/run-class.js.map +1 -1
- package/dist/resources/run-governed.d.ts +33 -2
- package/dist/resources/run-governed.d.ts.map +1 -1
- package/dist/resources/run-governed.js +142 -28
- package/dist/resources/run-governed.js.map +1 -1
- package/dist/tasks/ac-immutability.d.ts +32 -4
- package/dist/tasks/ac-immutability.d.ts.map +1 -1
- package/dist/tasks/ac-immutability.js +62 -6
- package/dist/tasks/ac-immutability.js.map +1 -1
- package/dist/tasks/add.d.ts +3 -0
- package/dist/tasks/add.d.ts.map +1 -1
- package/dist/tasks/add.js +29 -0
- package/dist/tasks/add.js.map +1 -1
- package/dist/tasks/affected-scope.d.ts +10 -4
- package/dist/tasks/affected-scope.d.ts.map +1 -1
- package/dist/tasks/affected-scope.js +85 -40
- package/dist/tasks/affected-scope.js.map +1 -1
- package/dist/tasks/child-disposition.d.ts +18 -1
- package/dist/tasks/child-disposition.d.ts.map +1 -1
- package/dist/tasks/child-disposition.js +26 -1
- package/dist/tasks/child-disposition.js.map +1 -1
- package/dist/tasks/complete.d.ts.map +1 -1
- package/dist/tasks/complete.js +5 -2
- package/dist/tasks/complete.js.map +1 -1
- package/dist/tasks/compute-task-view.d.ts.map +1 -1
- package/dist/tasks/compute-task-view.js +8 -3
- package/dist/tasks/compute-task-view.js.map +1 -1
- package/dist/tasks/done-plan.d.ts.map +1 -1
- package/dist/tasks/done-plan.js +7 -1
- package/dist/tasks/done-plan.js.map +1 -1
- package/dist/tasks/evidence.d.ts +26 -7
- package/dist/tasks/evidence.d.ts.map +1 -1
- package/dist/tasks/evidence.js +46 -8
- package/dist/tasks/evidence.js.map +1 -1
- package/dist/tasks/gate-notes.d.ts +93 -1
- package/dist/tasks/gate-notes.d.ts.map +1 -1
- package/dist/tasks/gate-notes.js +142 -0
- package/dist/tasks/gate-notes.js.map +1 -1
- package/dist/tasks/task-reparent.d.ts.map +1 -1
- package/dist/tasks/task-reparent.js +23 -0
- package/dist/tasks/task-reparent.js.map +1 -1
- package/dist/tasks/update.d.ts.map +1 -1
- package/dist/tasks/update.js +41 -1
- package/dist/tasks/update.js.map +1 -1
- package/dist/tasks/verification-policy.d.ts +7 -24
- package/dist/tasks/verification-policy.d.ts.map +1 -1
- package/dist/tasks/verification-policy.js +12 -29
- package/dist/tasks/verification-policy.js.map +1 -1
- package/dist/validation/engine-ops.d.ts.map +1 -1
- package/dist/validation/engine-ops.js +8 -7
- package/dist/validation/engine-ops.js.map +1 -1
- package/package.json +13 -13
|
@@ -76,10 +76,10 @@ import { getLogger } from '../logger.js';
|
|
|
76
76
|
import { getCleoHome } from '../paths.js';
|
|
77
77
|
import { admissionCapacityBytes, admissionReserveBytes, defaultSingleProcessHeapMb, GIB_PER_WORKER, HEAVY_TOOL_HEAP_MB, heavyToolWorkers, isHeavyTool, isMemoryBoundTool, planHeavyToolEnv, projectSettingOverrides, vitestWorkerCeiling, } from '../tasks/heavy-tool-env.js';
|
|
78
78
|
import { gitHookPath, HOOK_MIN_RUNS, HOOK_PEAK_MARGIN, hookFootprintKey, readHookPeak, scriptFootprint, } from './hook-footprint.js';
|
|
79
|
-
import { pressureScore, ResourceMonitor } from './monitor.js';
|
|
79
|
+
import { classifyPressure, pressureScore, ResourceMonitor } from './monitor.js';
|
|
80
80
|
import { checkMemoryGate, MEMORY_GATE_RETRY_AFTER_MS, } from './pressure-gate.js';
|
|
81
81
|
import { processAncestors, processGroupOf, processGroupsOf } from './run-admission.js';
|
|
82
|
-
import { canonicalForClass, gitHookFor, heavyScriptReason, lightCommandReason, namedTestFileCount,
|
|
82
|
+
import { canonicalForClass, gitHookFor, heavyScriptReason, lightCommandReason, namedTestFileCount, runFootprint, scriptRunOf, } from './run-class.js';
|
|
83
83
|
import { ownProcessStartedAt, systemPidProbe } from './slot-holder.js';
|
|
84
84
|
import { followToolGroups, isProbeableId } from './tool-groups.js';
|
|
85
85
|
import { memoryPressureCause, TOP_MEMORY_COUNT, TOP_MEMORY_FAILED_REUSE_MS, TOP_MEMORY_REUSE_MS, topMemoryConsumers, } from './top-memory.js';
|
|
@@ -282,20 +282,26 @@ scriptPeak = (script) => {
|
|
|
282
282
|
let scope = cls === 'full-build' ? null : runFootprint(argv);
|
|
283
283
|
const sized = cls === 'scoped-build' && scope?.size === 'class';
|
|
284
284
|
// T14050: a node script that hosts a heavy tool (its path names one, or it
|
|
285
|
-
// loads a compiler or test runner in-process) is never light.
|
|
286
|
-
|
|
285
|
+
// loads a compiler or test runner in-process) is never light. T14079: so
|
|
286
|
+
// is a Python or shell script that runs one, or one that cannot be read.
|
|
287
|
+
const run = sized ? scriptRunOf(argv) : null;
|
|
288
|
+
const script = run?.script ?? null;
|
|
287
289
|
const scripted = script !== null ? scriptPeak(script) : null;
|
|
288
290
|
const host = script === null
|
|
289
291
|
? null
|
|
290
292
|
: (heavyScriptReason(script) ??
|
|
291
|
-
(scripted?.heavyTool
|
|
293
|
+
(scripted?.heavyTool
|
|
294
|
+
? `it ${run?.runner === 'node' ? 'loads' : 'runs'} ${scripted.heavyTool}`
|
|
295
|
+
: run?.runner !== 'node' && scripted === null
|
|
296
|
+
? 'it cannot be read'
|
|
297
|
+
: null));
|
|
292
298
|
// T13452: a scoped build of a command that runs no heavy tool (`node x.mjs`,
|
|
293
299
|
// `ls`) is charged light. An explicit test run keeps its class plan.
|
|
294
300
|
const light = sized && host === null ? lightCommandReason(argv) : null;
|
|
295
301
|
if (light !== null)
|
|
296
302
|
scope = { size: 'light', reason: light };
|
|
297
303
|
else if (host !== null)
|
|
298
|
-
scope = { size: 'class', reason:
|
|
304
|
+
scope = { size: 'class', reason: `${run?.runner} ${script} hosts a heavy tool (${host})` };
|
|
299
305
|
// T13452: a hooked git push/commit keeps the class plan, because whatever
|
|
300
306
|
// the hook runs (tests, a build) rides this admission.
|
|
301
307
|
const probe = scope?.size === 'light' ? gitHookFor(argv) : null;
|
|
@@ -468,6 +474,10 @@ function isCachedPressure(v) {
|
|
|
468
474
|
(c.lightShare === undefined ||
|
|
469
475
|
(typeof c.lightShare === 'string' && SHARES.has(c.lightShare))) &&
|
|
470
476
|
typeof c.sampledAtMs === 'number' &&
|
|
477
|
+
(c.backoff === undefined || typeof c.backoff === 'boolean') &&
|
|
478
|
+
(c.memAvailableBytes === undefined ||
|
|
479
|
+
c.memAvailableBytes === null ||
|
|
480
|
+
typeof c.memAvailableBytes === 'number') &&
|
|
471
481
|
(c.reading === null || (typeof c.reading === 'object' && c.reading !== null)));
|
|
472
482
|
}
|
|
473
483
|
function readLedgerDoc(dir, strictRead = false) {
|
|
@@ -831,6 +841,53 @@ export function budgetShare(sample, gateRefuses) {
|
|
|
831
841
|
return 'full';
|
|
832
842
|
}
|
|
833
843
|
const NOTHING_HELD = { used: 0, reserved: 0, running: 0, heavy: 0, exclusive: 0 };
|
|
844
|
+
/**
|
|
845
|
+
* The run a latched-below-refuse pass may start (T14088): the first waiting
|
|
846
|
+
* run in queue order whose charge fits the reclaimable memory, or `null`.
|
|
847
|
+
*
|
|
848
|
+
* @param entries - the ledger.
|
|
849
|
+
* @param reclaimableBytes - the reading's available or reclaimable bytes.
|
|
850
|
+
* @param capacityBytes - the machine budget.
|
|
851
|
+
* @param nowMs - the clock.
|
|
852
|
+
*/
|
|
853
|
+
function firstFittingWaiter(entries, reclaimableBytes, capacityBytes, nowMs) {
|
|
854
|
+
if (typeof reclaimableBytes !== 'number' || !(reclaimableBytes > 0))
|
|
855
|
+
return null;
|
|
856
|
+
return (entries
|
|
857
|
+
.filter((e) => e.state === 'waiting' && charged(e, capacityBytes) <= reclaimableBytes)
|
|
858
|
+
.sort(queueOrder(nowMs))[0] ?? null);
|
|
859
|
+
}
|
|
860
|
+
/**
|
|
861
|
+
* Whether a memory-gate reading is latched shut while pressure sits at or
|
|
862
|
+
* below the refuse line, waiting for the resume line (T14088).
|
|
863
|
+
*
|
|
864
|
+
* @param reading - the gate's refusing reading, or `null` while it admits.
|
|
865
|
+
*/
|
|
866
|
+
export function isLatchedBelowRefuse(reading) {
|
|
867
|
+
return reading?.latched === true && reading.score <= reading.refuseAbove;
|
|
868
|
+
}
|
|
869
|
+
/**
|
|
870
|
+
* The idle rule (T14088, T14098): with nothing of CLEO's admitted, the
|
|
871
|
+
* pressure that keeps the budget closed is someone else's, so the first of
|
|
872
|
+
* `candidates` in queue order whose charge fits the reclaimable memory starts
|
|
873
|
+
* (never a whole-suite run into a nearly full swap). It holds the budget
|
|
874
|
+
* alone until it releases; nothing else starts beside it.
|
|
875
|
+
*
|
|
876
|
+
* @param entries - the ledger.
|
|
877
|
+
* @param ctx - the pass context (capacity, clock, reclaimable memory).
|
|
878
|
+
* @param candidates - the waiters the rule may start.
|
|
879
|
+
*/
|
|
880
|
+
function admitOneWhenIdle(entries, ctx, candidates) {
|
|
881
|
+
const idle = (ctx.foreign?.count ?? 0) === 0 && !entries.some((e) => e.state === 'admitted');
|
|
882
|
+
if (!idle)
|
|
883
|
+
return [...entries];
|
|
884
|
+
const fitting = firstFittingWaiter(candidates, ctx.reclaimableBytes, ctx.capacityBytes, ctx.nowMs);
|
|
885
|
+
if (fitting === null)
|
|
886
|
+
return [...entries];
|
|
887
|
+
const opened = schedulePass(entries.filter((e) => e.state !== 'waiting' || e.id === fitting.id), { ...ctx, share: 'one', lightShare: 'one', latchedBelowRefuse: false, backoff: false });
|
|
888
|
+
const started = opened.find((e) => e.id === fitting.id && e.state === 'admitted');
|
|
889
|
+
return entries.map((e) => e.id === started?.id ? { ...started, admittedBy: 'idle' } : e);
|
|
890
|
+
}
|
|
834
891
|
function charged(entry, capacityBytes) {
|
|
835
892
|
return Math.min(Math.max(0, entry.footprintBytes), capacityBytes);
|
|
836
893
|
}
|
|
@@ -851,7 +908,7 @@ export function shareBudgetBytes(share, capacityBytes) {
|
|
|
851
908
|
: 0;
|
|
852
909
|
}
|
|
853
910
|
/** {@link LedgerBudget} from a ledger doc's cached pressure verdict. */
|
|
854
|
-
function ledgerBudget(pressure, capacityBytes) {
|
|
911
|
+
function ledgerBudget(pressure, capacityBytes, entries = null) {
|
|
855
912
|
const share = pressure?.share ?? 'full';
|
|
856
913
|
const lightShare = pressure?.lightShare ?? share;
|
|
857
914
|
const lightBudgetBytes = shareBudgetBytes(lightShare, capacityBytes);
|
|
@@ -862,9 +919,31 @@ function ledgerBudget(pressure, capacityBytes) {
|
|
|
862
919
|
lightBudgetBytes,
|
|
863
920
|
heavyBudgetBytes: Math.min(lightBudgetBytes, shareBudgetBytes(share, capacityBytes)),
|
|
864
921
|
sampledAtMs: pressure?.sampledAtMs ?? null,
|
|
865
|
-
cause: pressure?.reading ?
|
|
922
|
+
cause: pressure?.reading ? latchCause(pressure.reading, entries, capacityBytes) : null,
|
|
866
923
|
};
|
|
867
924
|
}
|
|
925
|
+
/**
|
|
926
|
+
* {@link gateCause}, and when CLEO holds nothing while the latch waits below
|
|
927
|
+
* the refuse line, that the memory is outside CLEO and one run at a time
|
|
928
|
+
* still starts (T14088).
|
|
929
|
+
*
|
|
930
|
+
* @param reading - the gate's refusing reading.
|
|
931
|
+
* @param entries - the ledger, when known.
|
|
932
|
+
*/
|
|
933
|
+
function latchCause(reading, entries, capacityBytes = admissionCapacityBytes(), nowMs = Date.now()) {
|
|
934
|
+
const cause = gateCause(reading);
|
|
935
|
+
const cleoHoldsNothing = entries !== null && !entries.some((e) => e.state === 'admitted');
|
|
936
|
+
if (!cleoHoldsNothing || !isLatchedBelowRefuse(reading))
|
|
937
|
+
return cause;
|
|
938
|
+
const reclaimable = reading.memAvailableBytes;
|
|
939
|
+
const fitting = firstFittingWaiter(entries ?? [], reclaimable, capacityBytes, nowMs);
|
|
940
|
+
const reclaim = reclaimable === null ? 'unknown' : gib(reclaimable);
|
|
941
|
+
return (`${cause}. CLEO holds nothing, so the latch is held by memory outside CLEO (the largest users above); ` +
|
|
942
|
+
`one run at a time that fits the reclaimable memory is still admitted, in queue order, until pressure resumes ` +
|
|
943
|
+
(fitting === null
|
|
944
|
+
? `(largest fitting run: none; reclaimable ${reclaim})`
|
|
945
|
+
: `(next run that fits: ${fitting.label} ${gib(charged(fitting, capacityBytes))}; reclaimable ${reclaim})`));
|
|
946
|
+
}
|
|
868
947
|
/** The memory gate's refusal in words, with its cause (T13914). */
|
|
869
948
|
function gateCause(reading) {
|
|
870
949
|
return (`memory pressure ${reading.score} (refused above ${reading.refuseAbove}, resumes at ` +
|
|
@@ -879,14 +958,17 @@ function gateCause(reading) {
|
|
|
879
958
|
* @returns the shares and the light/heavy budgets they leave.
|
|
880
959
|
*/
|
|
881
960
|
export function readLedgerBudget(dir = admissionDir(), capacityBytes = admissionCapacityBytes()) {
|
|
882
|
-
|
|
961
|
+
const doc = readLedgerDoc(dir);
|
|
962
|
+
return ledgerBudget(doc.pressure, capacityBytes, doc.entries);
|
|
883
963
|
}
|
|
884
964
|
/**
|
|
885
965
|
* The budget a waiting entry is held to, in words: the capacity when no share
|
|
886
966
|
* narrows it, else the narrowed budget with its share and the capacity, e.g.
|
|
887
967
|
* `24 GiB light budget (light share half from memory pressure; capacity 48 GiB)`.
|
|
968
|
+
* When `usedBytes` is above it, the phrase says the budget was narrowed after
|
|
969
|
+
* those runs were admitted (T13967), so "48 GiB of 24 GiB" is not a puzzle.
|
|
888
970
|
*/
|
|
889
|
-
function budgetPhrase(entry, budget) {
|
|
971
|
+
function budgetPhrase(entry, budget, usedBytes = 0) {
|
|
890
972
|
const heavy = isHeavy(entry);
|
|
891
973
|
const share = heavy ? budget.share : budget.lightShare;
|
|
892
974
|
const bytes = heavy ? budget.heavyBudgetBytes : budget.lightBudgetBytes;
|
|
@@ -894,8 +976,9 @@ function budgetPhrase(entry, budget) {
|
|
|
894
976
|
return gib(budget.capacityBytes);
|
|
895
977
|
const kind = heavy ? 'heavy' : 'light';
|
|
896
978
|
const cause = heavy ? 'memory or CPU pressure' : 'memory pressure';
|
|
979
|
+
const narrowed = usedBytes > bytes ? '; narrowed after the running jobs were admitted, so they hold more' : '';
|
|
897
980
|
return (`${gib(bytes)} ${kind} budget (${kind} share ${share}` +
|
|
898
|
-
`${share === 'one' ? ', one heavy run at a time' : ''} from ${cause}; capacity ${gib(budget.capacityBytes)})`);
|
|
981
|
+
`${share === 'one' ? ', one heavy run at a time' : ''} from ${cause}; capacity ${gib(budget.capacityBytes)}${narrowed})`);
|
|
899
982
|
}
|
|
900
983
|
/**
|
|
901
984
|
* One scheduling pass: admit every waiting entry the budget and the queue
|
|
@@ -925,7 +1008,9 @@ function budgetPhrase(entry, budget) {
|
|
|
925
1008
|
* the budget, and over the capacity too.
|
|
926
1009
|
* - Up to {@link TINY_LANE_SLOTS} tiny runs are admitted from a reserve outside
|
|
927
1010
|
* the pressure share, wherever they wait, while the capacity has room for
|
|
928
|
-
* them
|
|
1011
|
+
* them, or, when holders admitted before the share narrowed hold more than
|
|
1012
|
+
* the narrowed budget, past the capacity unless an exclusive run holds the
|
|
1013
|
+
* machine (T13587, T13967): their bytes do not count against the share budget, and
|
|
929
1014
|
* other runs never use the reserve. Like the tiny lane, the reserve closes
|
|
930
1015
|
* while the oldest non-tiny run waits past {@link TINY_LANE_WINDOWS}
|
|
931
1016
|
* reservation windows or needs the whole capacity (T13591). The memory gate
|
|
@@ -942,8 +1027,30 @@ function budgetPhrase(entry, budget) {
|
|
|
942
1027
|
*/
|
|
943
1028
|
export function schedulePass(entries, ctx) {
|
|
944
1029
|
const lightShare = ctx.lightShare ?? ctx.share;
|
|
945
|
-
if (ctx.share === 'none' && lightShare === 'none')
|
|
946
|
-
|
|
1030
|
+
if (ctx.share === 'none' && lightShare === 'none') {
|
|
1031
|
+
// T14088: latched below the refuse line with nothing of CLEO's running: the
|
|
1032
|
+
// pressure is someone else's, so the idle rule admits one run. Above the
|
|
1033
|
+
// refuse line nothing starts.
|
|
1034
|
+
if (ctx.latchedBelowRefuse !== true)
|
|
1035
|
+
return [...entries];
|
|
1036
|
+
return admitOneWhenIdle(entries, ctx, entries);
|
|
1037
|
+
}
|
|
1038
|
+
if (ctx.backoff === true) {
|
|
1039
|
+
// T14098: pressure cannot pause a never-paused run later, so at backoff it
|
|
1040
|
+
// starts only by the same idle rule. A nested one (its ancestors admitted)
|
|
1041
|
+
// is never held: CLEO is never idle for it. Non-tiny runs queued behind
|
|
1042
|
+
// the first one that fits keep their place, so a stream of pausable runs
|
|
1043
|
+
// cannot starve it; the rest are scheduled as usual.
|
|
1044
|
+
const held = entries.filter((e) => e.state === 'waiting' && e.neverPause === true && e.parentId === undefined);
|
|
1045
|
+
if (held.length > 0) {
|
|
1046
|
+
const heldIds = new Set(held.map((e) => e.id));
|
|
1047
|
+
const head = firstFittingWaiter(held, ctx.reclaimableBytes, ctx.capacityBytes, ctx.nowMs);
|
|
1048
|
+
const order = queueOrder(ctx.nowMs);
|
|
1049
|
+
const waitsBehind = (e) => head !== null && e.state === 'waiting' && !isTiny(e) && order(head, e) < 0;
|
|
1050
|
+
const scheduled = new Map(schedulePass(entries.filter((e) => !heldIds.has(e.id) && !waitsBehind(e)), { ...ctx, backoff: false }).map((e) => [e.id, e]));
|
|
1051
|
+
return admitOneWhenIdle(entries.map((e) => scheduled.get(e.id) ?? e), ctx, held);
|
|
1052
|
+
}
|
|
1053
|
+
}
|
|
947
1054
|
const reservationMs = ctx.reservationMs ?? LEDGER_RESERVATION_MS;
|
|
948
1055
|
const bytesFor = (share) => shareBudgetBytes(share, ctx.capacityBytes);
|
|
949
1056
|
// Light runs see the memory budget. Heavy runs see the narrower of it and
|
|
@@ -959,14 +1066,22 @@ export function schedulePass(entries, ctx) {
|
|
|
959
1066
|
// T13367: admitted tiny runs, for the tiny lane past a blocked head.
|
|
960
1067
|
let tinyRunning = 0;
|
|
961
1068
|
// T13587: up to TINY_LANE_SLOTS tiny runs sit in a reserve outside the
|
|
962
|
-
// pressure share
|
|
963
|
-
//
|
|
1069
|
+
// pressure share. Their bytes are not `used`, so a holder that fills a
|
|
1070
|
+
// halved budget exactly cannot starve a formatter.
|
|
964
1071
|
let reserved = 0;
|
|
965
1072
|
let reservedBytes = 0;
|
|
966
1073
|
const reserveTakes = (e, cost) => isTiny(e) &&
|
|
967
1074
|
e.exclusive !== true &&
|
|
968
1075
|
reserved < TINY_LANE_SLOTS &&
|
|
969
1076
|
used + reservedBytes + cost <= ctx.capacityBytes;
|
|
1077
|
+
// T13967: holders admitted before memory pressure narrowed the light share
|
|
1078
|
+
// can hold the whole capacity, more than the narrowed budget allows; a
|
|
1079
|
+
// two-file formatter must not wait behind them. Only then may a waiting
|
|
1080
|
+
// tiny run take the reserve past the capacity (at most TINY_LANE_SLOTS
|
|
1081
|
+
// gigabytes), and never beside an exclusive run. At a full share the
|
|
1082
|
+
// capacity bounds the reserve as before (T13367), and the memory gate
|
|
1083
|
+
// (`none`) still admits nothing.
|
|
1084
|
+
const narrowedOverfull = () => lightBudget < ctx.capacityBytes && used > lightBudget && exclusiveRunning === 0;
|
|
970
1085
|
const reservedIds = new Set();
|
|
971
1086
|
for (const e of entries) {
|
|
972
1087
|
if (e.state === 'admitted') {
|
|
@@ -1083,7 +1198,10 @@ export function schedulePass(entries, ctx) {
|
|
|
1083
1198
|
const cost = charged(w, ctx.capacityBytes);
|
|
1084
1199
|
// T13587: the tiny reserve admits a tiny run wherever it waits; the memory
|
|
1085
1200
|
// gate (`none`) still admits nothing.
|
|
1086
|
-
if (reserveOpen &&
|
|
1201
|
+
if (reserveOpen &&
|
|
1202
|
+
lightShare !== 'none' &&
|
|
1203
|
+
(reserveTakes(w, cost) ||
|
|
1204
|
+
(isTiny(w) && w.exclusive !== true && reserved < TINY_LANE_SLOTS && narrowedOverfull()))) {
|
|
1087
1205
|
admit.add(w.id);
|
|
1088
1206
|
reserved++;
|
|
1089
1207
|
reservedBytes += cost;
|
|
@@ -1503,11 +1621,19 @@ async function sampleShare(opts, cached, nowMs, probeFor = null) {
|
|
|
1503
1621
|
const reading = gate.refuse && gate.reading !== null
|
|
1504
1622
|
? withTopConsumers(gate.reading, cached, nowMs, opts, probeFor)
|
|
1505
1623
|
: null;
|
|
1624
|
+
const backoff = classifyPressure(sample).state === 'backoff';
|
|
1506
1625
|
return {
|
|
1507
1626
|
share,
|
|
1508
1627
|
lightShare,
|
|
1509
1628
|
reading,
|
|
1510
|
-
sampled: {
|
|
1629
|
+
sampled: {
|
|
1630
|
+
share,
|
|
1631
|
+
lightShare,
|
|
1632
|
+
reading,
|
|
1633
|
+
sampledAtMs: nowMs,
|
|
1634
|
+
backoff,
|
|
1635
|
+
memAvailableBytes: sample.memAvailableBytes,
|
|
1636
|
+
},
|
|
1511
1637
|
};
|
|
1512
1638
|
}
|
|
1513
1639
|
/**
|
|
@@ -1596,9 +1722,18 @@ function foreignLoad(foreign, capacityBytes) {
|
|
|
1596
1722
|
function reapAndSchedule(doc, dead, ctx, sampled) {
|
|
1597
1723
|
const live = doc.entries.filter((e) => dead.entries.get(e.id) !== e.heartbeatAtMs);
|
|
1598
1724
|
const foreign = doc.foreign.filter((f) => !dead.foreign.has(JSON.stringify(f)));
|
|
1725
|
+
const cached = sampled ?? doc.pressure;
|
|
1726
|
+
const reading = cached?.reading;
|
|
1727
|
+
const latchedBelowRefuse = isLatchedBelowRefuse(reading);
|
|
1599
1728
|
return {
|
|
1600
1729
|
...doc,
|
|
1601
|
-
entries: schedulePass(live, {
|
|
1730
|
+
entries: schedulePass(live, {
|
|
1731
|
+
...ctx,
|
|
1732
|
+
foreign: foreignLoad(foreign, ctx.capacityBytes),
|
|
1733
|
+
latchedBelowRefuse,
|
|
1734
|
+
reclaimableBytes: reading?.memAvailableBytes ?? cached?.memAvailableBytes ?? null,
|
|
1735
|
+
backoff: cached?.backoff === true,
|
|
1736
|
+
}),
|
|
1602
1737
|
foreign,
|
|
1603
1738
|
pressure: sampled ?? doc.pressure,
|
|
1604
1739
|
};
|
|
@@ -1658,13 +1793,40 @@ async function admitInner(req, opts) {
|
|
|
1658
1793
|
// that grant is a heavy run's or covers the request. Under a light or
|
|
1659
1794
|
// single-process grant a bigger run queues for itself (T13452), naming the
|
|
1660
1795
|
// parent: schedulePass never makes it wait on its own ancestors.
|
|
1661
|
-
|
|
1796
|
+
// T12994: note whether a `ps` probe failed, so a nested run whose first
|
|
1797
|
+
// detection could not see its parent re-checks on every queue poll instead
|
|
1798
|
+
// of waiting on its own parent until it times out.
|
|
1799
|
+
const baseFacts = opts.facts ?? systemProcessFacts;
|
|
1800
|
+
let factsFailed = false;
|
|
1801
|
+
const facts = {
|
|
1802
|
+
ancestorsOf: (p) => {
|
|
1803
|
+
const a = baseFacts.ancestorsOf(p);
|
|
1804
|
+
if (a === null)
|
|
1805
|
+
factsFailed = true;
|
|
1806
|
+
return a;
|
|
1807
|
+
},
|
|
1808
|
+
groupOf: (p) => {
|
|
1809
|
+
const g = baseFacts.groupOf(p);
|
|
1810
|
+
if (g === null)
|
|
1811
|
+
factsFailed = true;
|
|
1812
|
+
return g;
|
|
1813
|
+
},
|
|
1814
|
+
startedAt: (p) => {
|
|
1815
|
+
const at = baseFacts.startedAt(p);
|
|
1816
|
+
if (at === null)
|
|
1817
|
+
factsFailed = true;
|
|
1818
|
+
return at;
|
|
1819
|
+
},
|
|
1820
|
+
};
|
|
1821
|
+
const rides = (p) => isHeavy(p) || p.footprintBytes >= req.footprintBytes;
|
|
1822
|
+
const parent = enclosingGrant(readLedger(dir), pid, env, facts);
|
|
1662
1823
|
if (parent !== null &&
|
|
1663
|
-
(
|
|
1824
|
+
rides(parent) &&
|
|
1664
1825
|
!opts.signal?.aborted &&
|
|
1665
1826
|
(!opts.wait || timeoutMs <= 0 || now() - t0 < timeoutMs)) {
|
|
1666
1827
|
return { admitted: true, grant: passThrough(admissionToken(parent), true, null) };
|
|
1667
1828
|
}
|
|
1829
|
+
let parentUnknown = parent === null && factsFailed;
|
|
1668
1830
|
const entry = {
|
|
1669
1831
|
id: newId(pid, t0),
|
|
1670
1832
|
nonce: randomBytes(12).toString('hex'),
|
|
@@ -1686,6 +1848,7 @@ async function admitInner(req, opts) {
|
|
|
1686
1848
|
...(req.footprintReason !== undefined ? { footprintReason: req.footprintReason } : {}),
|
|
1687
1849
|
...(req.exclusive === true ? { exclusive: true } : {}),
|
|
1688
1850
|
...(req.urgent === true ? { priority: 'urgent' } : {}),
|
|
1851
|
+
...(req.neverPause === true ? { neverPause: true } : {}),
|
|
1689
1852
|
};
|
|
1690
1853
|
const deadline = t0 + timeoutMs;
|
|
1691
1854
|
const stop = new AbortController();
|
|
@@ -1763,13 +1926,16 @@ async function admitInner(req, opts) {
|
|
|
1763
1926
|
const exclusiveHolder = entry.exclusive === true
|
|
1764
1927
|
? doc.entries.find((e) => e.state === 'admitted' && e.exclusive === true && e.id !== entry.id)
|
|
1765
1928
|
: undefined;
|
|
1929
|
+
const hold = neverPauseHold(doc, entry);
|
|
1766
1930
|
const reason = reading !== null
|
|
1767
|
-
?
|
|
1768
|
-
:
|
|
1769
|
-
?
|
|
1770
|
-
|
|
1771
|
-
|
|
1772
|
-
|
|
1931
|
+
? latchCause(reading, doc.entries, capacityBytes, nowMs)
|
|
1932
|
+
: hold !== null
|
|
1933
|
+
? `${hold}${ahead > 0 ? `; ${ahead} waiting ahead` : ''}`
|
|
1934
|
+
: exclusiveHolder !== undefined
|
|
1935
|
+
? `another full build holds the machine-wide full-build slot (${exclusiveHolder.label} pid ` +
|
|
1936
|
+
`${exclusiveHolder.pid}); one runs at a time, whatever its size${ahead > 0 ? `; ${ahead} waiting ahead` : ''}`
|
|
1937
|
+
: `machine budget in use: ${gib(load.bytes)} of ${budgetPhrase(entry, ledgerBudget(doc.pressure, capacityBytes), load.bytes)} by ${load.runs} run(s); ` +
|
|
1938
|
+
`this run needs ${gib(charged(entry, capacityBytes))}${ahead > 0 ? `; ${ahead} waiting ahead` : ''}`;
|
|
1773
1939
|
return {
|
|
1774
1940
|
reason,
|
|
1775
1941
|
retryAfterMs: reading !== null ? MEMORY_GATE_RETRY_AFTER_MS : 2_000,
|
|
@@ -1792,6 +1958,10 @@ async function admitInner(req, opts) {
|
|
|
1792
1958
|
return { admitted: true, grant: holdGrant(raced, dir, now() - t0, opts) };
|
|
1793
1959
|
return { admitted: false, refusal: refusal(first.doc, first.reading) };
|
|
1794
1960
|
}
|
|
1961
|
+
// T14098: say at once which rule holds a never-paused run at backoff.
|
|
1962
|
+
const hold = first.reading === null ? neverPauseHold(first.doc, first.mine ?? entry) : null;
|
|
1963
|
+
if (hold !== null)
|
|
1964
|
+
opts.notice?.(`waiting: ${hold}`);
|
|
1795
1965
|
let lastPassAt = now();
|
|
1796
1966
|
let lastReportAt = t0;
|
|
1797
1967
|
let lastKeepAliveAt = Number.NEGATIVE_INFINITY;
|
|
@@ -1823,6 +1993,21 @@ async function admitInner(req, opts) {
|
|
|
1823
1993
|
checkQueue();
|
|
1824
1994
|
return { admitted: true, grant: holdGrant(mine, dir, now() - t0, opts) };
|
|
1825
1995
|
}
|
|
1996
|
+
if (parentUnknown) {
|
|
1997
|
+
// T12994: the first detection could not see the process tree; try again.
|
|
1998
|
+
factsFailed = false;
|
|
1999
|
+
const found = enclosingGrant(seen.entries, pid, env, facts);
|
|
2000
|
+
parentUnknown = found === null && factsFailed;
|
|
2001
|
+
if (found !== null && rides(found)) {
|
|
2002
|
+
const raced = await leave();
|
|
2003
|
+
checkQueue();
|
|
2004
|
+
if (raced !== null) {
|
|
2005
|
+
return { admitted: true, grant: holdGrant(raced, dir, now() - t0, opts) };
|
|
2006
|
+
}
|
|
2007
|
+
opts.memoryPressure?.admitted(now() - t0, null);
|
|
2008
|
+
return { admitted: true, grant: passThrough(admissionToken(found), true, null) };
|
|
2009
|
+
}
|
|
2010
|
+
}
|
|
1826
2011
|
if (seen.pressure !== null && isFresh(seen.pressure, now()))
|
|
1827
2012
|
reading = seen.pressure.reading;
|
|
1828
2013
|
if (passDue(mine === undefined, now() - lastPassAt, seen.pressure, now(), opts)) {
|
|
@@ -1894,6 +2079,20 @@ function describeLoad(doc, nowMs, probe) {
|
|
|
1894
2079
|
}
|
|
1895
2080
|
return lines;
|
|
1896
2081
|
}
|
|
2082
|
+
/**
|
|
2083
|
+
* Why a never-paused waiter is held at backoff (T14098), naming the rule, or
|
|
2084
|
+
* `null` when that rule does not hold it.
|
|
2085
|
+
*/
|
|
2086
|
+
function neverPauseHold(doc, mine) {
|
|
2087
|
+
if (mine.neverPause !== true || mine.parentId !== undefined || doc.pressure?.backoff !== true) {
|
|
2088
|
+
return null;
|
|
2089
|
+
}
|
|
2090
|
+
const admitted = doc.entries.filter((e) => e.state === 'admitted').length + doc.foreign.length;
|
|
2091
|
+
const reclaimable = doc.pressure.memAvailableBytes;
|
|
2092
|
+
return (`machine at backoff; this run is never paused, so it starts only when no CLEO run is admitted ` +
|
|
2093
|
+
`and its charge fits the reclaimable memory, one at a time in queue order (${admitted} CLEO run(s) admitted; ` +
|
|
2094
|
+
`reclaimable ${typeof reclaimable === 'number' ? gib(reclaimable) : 'unknown'})`);
|
|
2095
|
+
}
|
|
1897
2096
|
/**
|
|
1898
2097
|
* The holder report a long waiter prints. When the memory gate closed the
|
|
1899
2098
|
* budget, it names the cause (the host's largest memory users, CLEO or not)
|
|
@@ -1904,12 +2103,15 @@ function holderReport(doc, mine, pid, waitedMs, capacityBytes, nowMs, timeoutMs,
|
|
|
1904
2103
|
const used = budgetLoad(doc, capacityBytes).bytes;
|
|
1905
2104
|
const ahead = waitingAhead(doc.entries, mine, nowMs);
|
|
1906
2105
|
const lines = [
|
|
1907
|
-
`still waiting after ${age(waitedMs)} for the machine budget (${gib(used)} of ${budgetPhrase(mine, ledgerBudget(doc.pressure, capacityBytes))} in use, ` +
|
|
2106
|
+
`still waiting after ${age(waitedMs)} for the machine budget (${gib(used)} of ${budgetPhrase(mine, ledgerBudget(doc.pressure, capacityBytes), used)} in use, ` +
|
|
1908
2107
|
`queue position ${ahead + 1}${mine.priority === 'urgent' ? ' (urgent)' : ''}, ${ahead} waiting ahead). Holders: ${describeLoad(doc, nowMs, probe).join(' | ') || 'none'}.`,
|
|
1909
2108
|
];
|
|
1910
2109
|
const reading = doc.pressure?.reading ?? null;
|
|
2110
|
+
const hold = reading === null ? neverPauseHold(doc, mine) : null;
|
|
2111
|
+
if (hold !== null)
|
|
2112
|
+
lines.push(`Held by the backoff rule: ${hold}.`);
|
|
1911
2113
|
if (reading !== null) {
|
|
1912
|
-
lines.push(`The budget is closed by ${
|
|
2114
|
+
lines.push(`The budget is closed by ${latchCause(reading, doc.entries, capacityBytes, nowMs)}. Admission resumes by itself; this wait ` +
|
|
1913
2115
|
`gives up in ${age(Math.max(0, timeoutMs - waitedMs))}.`);
|
|
1914
2116
|
}
|
|
1915
2117
|
const table = holders.length > 0 ? processTable() : null;
|
|
@@ -2002,6 +2204,7 @@ function holdGrant(entry, dir, waitedMs, opts) {
|
|
|
2002
2204
|
ungoverned: null,
|
|
2003
2205
|
waitedMs,
|
|
2004
2206
|
footprintBytes: entry.footprintBytes,
|
|
2207
|
+
...(entry.admittedBy !== undefined ? { admittedBy: entry.admittedBy } : {}),
|
|
2005
2208
|
recordChild: (child) => {
|
|
2006
2209
|
void update((e) => ({ ...e, child }));
|
|
2007
2210
|
},
|