@smoothbricks/nx-plugin 0.7.5 → 0.7.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -3
- package/dist/.build-inputs.json +12 -6
- package/dist/executors/bounded-exec/cpu-pool-thread.d.ts +3 -2
- package/dist/executors/bounded-exec/cpu-pool-thread.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool-thread.js +1 -1
- package/dist/executors/bounded-exec/cpu-pool.d.ts +24 -10
- package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool.js +96 -52
- package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
- package/dist/executors/bounded-exec/executor.js +115 -62
- package/dist/executors/bounded-exec/host-memory.d.ts +54 -0
- package/dist/executors/bounded-exec/host-memory.d.ts.map +1 -0
- package/dist/executors/bounded-exec/host-memory.darwin.d.ts +3 -0
- package/dist/executors/bounded-exec/host-memory.darwin.d.ts.map +1 -0
- package/dist/executors/bounded-exec/host-memory.darwin.js +115 -0
- package/dist/executors/bounded-exec/host-memory.js +29 -0
- package/dist/executors/bounded-exec/host-memory.linux.d.ts +14 -0
- package/dist/executors/bounded-exec/host-memory.linux.d.ts.map +1 -0
- package/dist/executors/bounded-exec/host-memory.linux.js +137 -0
- package/dist/executors/bounded-exec/leaf-peak.d.ts +56 -0
- package/dist/executors/bounded-exec/leaf-peak.d.ts.map +1 -0
- package/dist/executors/bounded-exec/leaf-peak.js +147 -0
- package/dist/executors/bounded-exec/mbx-reservations.d.ts +22 -0
- package/dist/executors/bounded-exec/mbx-reservations.d.ts.map +1 -1
- package/dist/executors/bounded-exec/mbx-reservations.js +22 -0
- package/dist/executors/bounded-exec/peak-waiter.d.ts +2 -0
- package/dist/executors/bounded-exec/peak-waiter.d.ts.map +1 -0
- package/dist/executors/bounded-exec/peak-waiter.js +79 -0
- package/dist/executors/bounded-exec/pool-admission.d.ts +38 -0
- package/dist/executors/bounded-exec/pool-admission.d.ts.map +1 -0
- package/dist/executors/bounded-exec/pool-admission.js +31 -0
- package/dist/executors/bounded-exec/run-cpu-pool.d.ts +6 -4
- package/dist/executors/bounded-exec/run-cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/run-cpu-pool.js +20 -7
- package/dist/executors/bounded-exec/verdict.d.ts +18 -0
- package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
- package/dist/executors/bounded-exec/verdict.js +21 -1
- package/managed/templates/github/actions/setup-devenv/action.yml +31 -2
- package/package.json +2 -2
- package/src/executors/bounded-exec/cpu-pool-thread.ts +4 -3
- package/src/executors/bounded-exec/cpu-pool.ts +111 -49
- package/src/executors/bounded-exec/executor.ts +121 -59
- package/src/executors/bounded-exec/host-memory.darwin.ts +147 -0
- package/src/executors/bounded-exec/host-memory.linux.ts +147 -0
- package/src/executors/bounded-exec/host-memory.ts +77 -0
- package/src/executors/bounded-exec/leaf-peak.ts +152 -0
- package/src/executors/bounded-exec/mbx-reservations.ts +30 -0
- package/src/executors/bounded-exec/peak-waiter.ts +115 -0
- package/src/executors/bounded-exec/pool-admission.ts +64 -0
- package/src/executors/bounded-exec/run-cpu-pool.ts +20 -6
- package/src/executors/bounded-exec/verdict.ts +30 -1
- package/test-timings.json +136 -132
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { spawn } from 'node:child_process';
|
|
2
2
|
import { accessSync, constants } from 'node:fs';
|
|
3
3
|
import { mkdtemp, rm } from 'node:fs/promises';
|
|
4
|
-
import { availableParallelism, userInfo } from 'node:os';
|
|
4
|
+
import { availableParallelism, constants as osConstants, userInfo } from 'node:os';
|
|
5
5
|
import { delimiter, isAbsolute, join, resolve as resolvePath } from 'node:path';
|
|
6
6
|
import { createInterface } from 'node:readline';
|
|
7
7
|
import { Duplex, Readable } from 'node:stream';
|
|
@@ -11,9 +11,10 @@ import { admissionWaitMs, earliestBound, NO_ADMISSION_WAIT, stepAdmissionWait, s
|
|
|
11
11
|
import { CPU_POOL_ENV, createCpuPool, readInheritedCpuPool, takeParentCpuTokens, } from './cpu-pool.js';
|
|
12
12
|
import { GATEWAY_SOCKET } from './gateway-lease.js';
|
|
13
13
|
import { captureHangDiagnostics, HANG_CAPTURE_MS } from './hang-diagnostics.js';
|
|
14
|
+
import { PEAK_FD, peakWaiter, readWaiterReport, underWaiter } from './leaf-peak.js';
|
|
14
15
|
import { askedCores, MBX_GRANT_FD, mbxIdleGated, mbxLeaf, mbxPool, withCountedReservation, withMbxLeaf, } from './mbx-pool.js';
|
|
15
16
|
import { MBX_QUEUE_POLL_MS, snapshotMbxQueue } from './mbx-queue.js';
|
|
16
|
-
import { RESERVATIONS_ENV, reserveCores } from './mbx-reservations.js';
|
|
17
|
+
import { describeUnserved, RESERVATIONS_ENV, reserveCores, runEndpoint } from './mbx-reservations.js';
|
|
17
18
|
import { createNoSpaceWatch } from './no-space.js';
|
|
18
19
|
import { COWSHED_EXECUTABLE_ENV, cowshedSubtreeOwner, HANDLE_FD, readAdmission, SUPERVISOR_SOCKET_ENV, } from './owned-subtree.js';
|
|
19
20
|
import { createRunProgress, withProgressReport, withTestResultReport } from './progress.js';
|
|
@@ -22,14 +23,9 @@ import { createRunnerLifecycle, withFixtureLifecycle, withRunnerLifecycle } from
|
|
|
22
23
|
import { CPU_TOKENS_ENV, runnerDemand, sizeRunner, sizeRunnerToGrant } from './runner-sizing.js';
|
|
23
24
|
import { checkRuntimePin, describeRuntimePinRefusal } from './runtime-pin.js';
|
|
24
25
|
import { createTestResultWriter } from './test-result-writer.js';
|
|
25
|
-
import { BUN_TEST, clearTaskDirectory, describeAdmissionQueue, describeNoSpace, JUNIT_FILE, JUNIT_REPORT_ENV, judgeOperationalFailure, judgeRun, readJunitReport, taskDirectory, withJunitReport, writeRecord, } from './verdict.js';
|
|
26
|
+
import { BUN_TEST, clearTaskDirectory, describeAdmissionQueue, describeNoSpace, JUNIT_FILE, JUNIT_REPORT_ENV, judgeOperationalFailure, judgeRun, readJunitReport, readLastPeak, taskDirectory, withJunitReport, writeRecord, } from './verdict.js';
|
|
26
27
|
const DEFAULT_KILL_AFTER_MS = 10_000;
|
|
27
28
|
const REAP_AFTER_FORCE_KILL_MS = 2_000;
|
|
28
|
-
const EXIT_CODE_BY_SIGNAL = {
|
|
29
|
-
SIGHUP: 129,
|
|
30
|
-
SIGINT: 130,
|
|
31
|
-
SIGTERM: 143,
|
|
32
|
-
};
|
|
33
29
|
/** The pool descriptor: only a grant may give it, so neither the target's nor the inherited environment passes it on. */
|
|
34
30
|
const POOL_ENV = [CPU_TOKENS_ENV, CPU_POOL_ENV];
|
|
35
31
|
/**
|
|
@@ -148,6 +144,8 @@ export function recordContext(context) {
|
|
|
148
144
|
export async function runBoundedExec(options, context, killer, tempVolume, inherited = null, captureDiagnostics = captureHangDiagnostics, owner = null, queueHost) {
|
|
149
145
|
const cwd = resolveCwd(options.cwd, context.root);
|
|
150
146
|
const directory = context.record?.directory ?? null;
|
|
147
|
+
// The task's last recorded peak is its charge in its parent's pool, read before the clear below removes it.
|
|
148
|
+
const lastPeak = directory !== null && inherited !== null ? await readLastPeak(directory) : null;
|
|
151
149
|
// A previous run's verdict and report go first, so neither can be read as this run's.
|
|
152
150
|
if (directory !== null) {
|
|
153
151
|
await clearTaskDirectory(directory);
|
|
@@ -196,17 +194,26 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
196
194
|
// Nobody is left to tell.
|
|
197
195
|
}
|
|
198
196
|
};
|
|
197
|
+
// A run refused before its command exists says why and records an operational failure, and no peak: nothing ran.
|
|
198
|
+
const refuseUnstarted = async (message) => {
|
|
199
|
+
appendStderr(message);
|
|
200
|
+
if (context.record !== undefined) {
|
|
201
|
+
const { task, directory: recordDirectory } = context.record;
|
|
202
|
+
await writeRecord(recordDirectory, {
|
|
203
|
+
task: task.id,
|
|
204
|
+
hash: task.hash,
|
|
205
|
+
verdict: judgeOperationalFailure(1, null),
|
|
206
|
+
peakBytes: null,
|
|
207
|
+
});
|
|
208
|
+
}
|
|
209
|
+
return { success: false, terminalOutput: outputChunks.join('') };
|
|
210
|
+
};
|
|
199
211
|
// The fixture pair names one segment: a fixture with no build budget, or a budget with no fixture, is a
|
|
200
212
|
// target that does not say what it runs, so it does not run. Units are counted at the end of that segment.
|
|
201
213
|
if ((options.fixtureCommand === undefined) !== (options.fixtureTimeoutMs === undefined) ||
|
|
202
214
|
(options.unitsCommand !== undefined && options.fixtureCommand === undefined) ||
|
|
203
215
|
(fixture !== null && !(fixture.timeoutMs > 0))) {
|
|
204
|
-
|
|
205
|
-
if (context.record !== undefined) {
|
|
206
|
-
const { task, directory: recordDirectory } = context.record;
|
|
207
|
-
await writeRecord(recordDirectory, { task: task.id, hash: task.hash, verdict: judgeOperationalFailure(1, null) });
|
|
208
|
-
}
|
|
209
|
-
return { success: false, terminalOutput: outputChunks.join('') };
|
|
216
|
+
return refuseUnstarted('bounded-exec: fixtureCommand and a positive fixtureTimeoutMs must be named together, and unitsCommand only with them; refusing to run the command\n');
|
|
210
217
|
}
|
|
211
218
|
// A leg that runs bun runs the Bun its workspace declares, or it does not run. This comes before the RAM lease and the
|
|
212
219
|
// CPU tokens: a refusal that waited in its pool's queue first would say nothing for minutes.
|
|
@@ -218,13 +225,13 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
218
225
|
pathSource: options.env?.PATH === undefined ? 'the inherited environment' : "the target's env",
|
|
219
226
|
});
|
|
220
227
|
if (runtime !== null) {
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
}
|
|
226
|
-
return { success: false, terminalOutput: outputChunks.join('') };
|
|
228
|
+
return refuseUnstarted(describeRuntimePinRefusal(runtime, context.record?.task.id));
|
|
229
|
+
}
|
|
230
|
+
if (lastPeak !== null && !lastPeak.ok) {
|
|
231
|
+
return refuseUnstarted(`bounded-exec: ${lastPeak.error}; it is this task's memory charge, so the command does not run\n`);
|
|
227
232
|
}
|
|
233
|
+
// A task with no recorded peak is charged nothing (null) until it has one.
|
|
234
|
+
const charge = lastPeak?.ok === true ? lastPeak.peakBytes : null;
|
|
228
235
|
let lease = null;
|
|
229
236
|
let ordinaryTemp = null;
|
|
230
237
|
// Before spawn there is nothing to retire. Once started, only the owner's terminal retirement permits deletion.
|
|
@@ -364,7 +371,7 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
364
371
|
let parentRefusal = null;
|
|
365
372
|
let parentWait;
|
|
366
373
|
if (inherited !== null) {
|
|
367
|
-
const fromParent = await takeParentCpuTokens(inherited, demand.want);
|
|
374
|
+
const fromParent = await takeParentCpuTokens(inherited, demand.want, charge);
|
|
368
375
|
if (fromParent.granted) {
|
|
369
376
|
grant = fromParent;
|
|
370
377
|
}
|
|
@@ -386,7 +393,10 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
386
393
|
if (grant !== null) {
|
|
387
394
|
sized = sizeRunner(demand, requested, grant.tokens);
|
|
388
395
|
cpu = { tokens: grant.tokens, want: demand.want, waitedMs };
|
|
389
|
-
appendStderr(`cowshed: cpu-pool wait done elapsed=${waitedMs}ms status=ok tokens=${grant.tokens}/${demand.want} runner=${demand.kind}\n`);
|
|
396
|
+
appendStderr(`cowshed: cpu-pool wait done elapsed=${waitedMs}ms status=ok tokens=${grant.tokens}/${demand.want} runner=${demand.kind} memory=${charge ?? 'none'}\n`);
|
|
397
|
+
if (grant.oversized !== null) {
|
|
398
|
+
appendStderr(`cowshed: cpu-pool granted while nothing else holds memory: this task's last peak of ${charge} bytes alone exceeds the run's ${grant.oversized.memory} bytes\n`);
|
|
399
|
+
}
|
|
390
400
|
}
|
|
391
401
|
else if (leaf?.kind === 'reserve') {
|
|
392
402
|
sized = { command: sizeRunnerToGrant(leaf.demand, requested), env: {} };
|
|
@@ -444,7 +454,7 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
444
454
|
};
|
|
445
455
|
// Records the run's verdict when it has a task to record as. An `expiry` is set only when a bound ended the run.
|
|
446
456
|
// Observation and ownership failures remain operational even after valid queue evidence; cancellation alone does not.
|
|
447
|
-
const recordRun = async (exitCode, expiry, elapsedMs, operational, noSpace, lockWaits) => {
|
|
457
|
+
const recordRun = async (exitCode, expiry, elapsedMs, operational, noSpace, lockWaits, peakBytes) => {
|
|
448
458
|
if (context.record === undefined) {
|
|
449
459
|
return;
|
|
450
460
|
}
|
|
@@ -485,6 +495,7 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
485
495
|
verdict,
|
|
486
496
|
...(cpu ? { cpu } : {}),
|
|
487
497
|
...(parentWait ? { cpuWait: parentWait } : {}),
|
|
498
|
+
peakBytes,
|
|
488
499
|
});
|
|
489
500
|
};
|
|
490
501
|
// A run refused before its command starts says why, gives back its grant and RAM, and records the failure.
|
|
@@ -493,22 +504,24 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
493
504
|
await releaseCpu();
|
|
494
505
|
// The command never started, so its lease holds nothing of its own to keep.
|
|
495
506
|
await releaseLease(false);
|
|
496
|
-
await recordRun(1, null, Date.now() - asked, false, null, []);
|
|
507
|
+
await recordRun(1, null, Date.now() - asked, false, null, [], null);
|
|
497
508
|
return { success: false, terminalOutput: outputChunks.join('') };
|
|
498
509
|
};
|
|
499
510
|
if (parentRefusal !== null) {
|
|
500
511
|
return refuse(`cowshed: cpu-pool parent pool refused: ${parentRefusal}; refusing to run the command\n`);
|
|
501
512
|
}
|
|
502
513
|
// A reserved leaf takes its cores from its Nx run's mbx reservation endpoint (mbx-reservations.ts). A run that serves
|
|
503
|
-
// none said why in its own output
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
514
|
+
// none said why in its own output, which a gate's failures-only output never shows, so the refusal says it too; the
|
|
515
|
+
// leaf does not run beside the pool.
|
|
516
|
+
const endpoint = leaf?.kind === 'reserve' ? runEndpoint(process.env) : null;
|
|
517
|
+
if (endpoint !== null && endpoint.kind !== 'served') {
|
|
518
|
+
return refuse(`cowshed: a reserved leaf takes its cores from its Nx run's mbx reservation endpoint, and ${RESERVATIONS_ENV} names none: ${describeUnserved(endpoint)}; refusing to run the command\n`);
|
|
507
519
|
}
|
|
520
|
+
const reserved = leaf?.kind === 'reserve' && endpoint?.kind === 'served' ? { leaf, endpoint: endpoint.socket } : null;
|
|
508
521
|
// A granted run serves a sub-pool of its own grant, so its descendants draw from that and never from above it.
|
|
509
522
|
// A pool that cannot be set up refuses the run: no granted command runs beside its grant.
|
|
510
523
|
if (grant !== null) {
|
|
511
|
-
const created = await createCpuPool(grant.tokens, grant.closed);
|
|
524
|
+
const created = await createCpuPool(grant.tokens, grant.closed, null);
|
|
512
525
|
if (!created.ok) {
|
|
513
526
|
return refuse(`cowshed: cpu pool unavailable: ${created.error}; refusing to run the command\n`);
|
|
514
527
|
}
|
|
@@ -555,6 +568,17 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
555
568
|
const grantSlots = leaf !== null ? ['ignore', 'pipe'] : [];
|
|
556
569
|
const rootGrantSlot = leaf !== null ? ['pipe'] : [];
|
|
557
570
|
const root = owner?.rootArgv(shellCommand);
|
|
571
|
+
// The command's own descriptors past stderr, from fd 3: the lifecycle channel, the subtree admission, mbx's grant.
|
|
572
|
+
const descriptors = root === undefined
|
|
573
|
+
? hooked === null && leaf === null
|
|
574
|
+
? []
|
|
575
|
+
: [hooked === null ? 'ignore' : 'pipe', ...grantSlots]
|
|
576
|
+
: [hooked === null ? 'ignore' : 'pipe', 'pipe', ...rootGrantSlot];
|
|
577
|
+
// A run that records its verdict records its command's peak too: the command runs under a waiter that reports it.
|
|
578
|
+
const waiter = context.record === undefined ? null : peakWaiter();
|
|
579
|
+
const measured = waiter?.kind === 'waiter'
|
|
580
|
+
? underWaiter(waiter.runtime, root?.file ?? '/bin/sh', root?.args ?? ['-c', shellCommand], descriptors)
|
|
581
|
+
: null;
|
|
558
582
|
let child;
|
|
559
583
|
const results = directory === null ? null : await createTestResultWriter(directory, { cwd, root: context.root });
|
|
560
584
|
if (results !== null && !results.ok)
|
|
@@ -562,32 +586,48 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
562
586
|
resultWriter = results?.ok ? results.writer : null;
|
|
563
587
|
try {
|
|
564
588
|
// fd 3 is the runner lifecycle's duplex socket when the command is hooked; the subtree admission takes fd 4, which
|
|
565
|
-
// cowshed closes fd 4 before exec; fd 5 is a Cargo leaf's mbx admission stream or a reserved leaf's ask and grant
|
|
589
|
+
// cowshed closes fd 4 before exec; fd 5 is a Cargo leaf's mbx admission stream or a reserved leaf's ask and grant;
|
|
590
|
+
// the waiter, when there is one, passes them on and reports the command's peak on fd 6.
|
|
566
591
|
child =
|
|
567
|
-
|
|
568
|
-
? spawn(
|
|
592
|
+
measured !== null
|
|
593
|
+
? spawn(measured.file, measured.args, {
|
|
569
594
|
cwd,
|
|
570
595
|
env: mergeEnv(lease?.directory ?? ordinaryTemp, env),
|
|
571
|
-
|
|
572
|
-
stdio: hooked === null && leaf === null
|
|
573
|
-
? 'pipe'
|
|
574
|
-
: ['pipe', 'pipe', 'pipe', hooked === null ? 'ignore' : 'pipe', ...grantSlots],
|
|
596
|
+
stdio: measured.stdio,
|
|
575
597
|
detached: process.platform !== 'win32',
|
|
576
598
|
windowsHide: true,
|
|
577
599
|
})
|
|
578
|
-
:
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
600
|
+
: root === undefined
|
|
601
|
+
? spawn(shellCommand, [], {
|
|
602
|
+
cwd,
|
|
603
|
+
env: mergeEnv(lease?.directory ?? ordinaryTemp, env),
|
|
604
|
+
shell: true,
|
|
605
|
+
stdio: descriptors.length === 0 ? 'pipe' : ['pipe', 'pipe', 'pipe', ...descriptors],
|
|
606
|
+
detached: process.platform !== 'win32',
|
|
607
|
+
windowsHide: true,
|
|
608
|
+
})
|
|
609
|
+
: spawn(root.file, root.args, {
|
|
610
|
+
cwd,
|
|
611
|
+
env: mergeEnv(lease?.directory ?? ordinaryTemp, env),
|
|
612
|
+
stdio: ['pipe', 'pipe', 'pipe', ...descriptors],
|
|
613
|
+
detached: process.platform !== 'win32',
|
|
614
|
+
windowsHide: true,
|
|
615
|
+
});
|
|
585
616
|
}
|
|
586
617
|
catch (error) {
|
|
587
618
|
// Nothing started, so nothing runs under the grant: it goes back here, with the RAM lease.
|
|
588
619
|
return refuse(`cowshed: could not start the command: ${error instanceof Error ? error.message : String(error)}\n`);
|
|
589
620
|
}
|
|
590
621
|
subtreeRetired = false;
|
|
622
|
+
const reportChannel = child.stdio.at(PEAK_FD);
|
|
623
|
+
const waiterReport = measured === null ? null : readWaiterReport(reportChannel instanceof Readable ? reportChannel : null);
|
|
624
|
+
// The leaf's runner is its command: under a waiter, which leads the group, the child the waiter names; else the
|
|
625
|
+
// group's leader itself. A hang snapshot names and samples it (specs/cowshed/08_testing.md).
|
|
626
|
+
const runnerPid = waiterReport?.started ?? Promise.resolve(child.pid ?? null);
|
|
627
|
+
let runner = null;
|
|
628
|
+
void runnerPid.then((pid) => {
|
|
629
|
+
runner = pid;
|
|
630
|
+
});
|
|
591
631
|
if (root !== undefined) {
|
|
592
632
|
const channel = child.stdio[HANDLE_FD];
|
|
593
633
|
const leased = lease;
|
|
@@ -643,20 +683,24 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
643
683
|
captureTimer = setTimeout(() => {
|
|
644
684
|
if (state.settled || state.expiry !== null || idleCapture !== capture)
|
|
645
685
|
return;
|
|
646
|
-
const
|
|
647
|
-
if (
|
|
686
|
+
const group = child.pid;
|
|
687
|
+
if (group === undefined) {
|
|
648
688
|
idleCapture.result = { kind: 'failed', detail: 'runner PID unavailable; no holder conclusion' };
|
|
649
689
|
return;
|
|
650
690
|
}
|
|
651
691
|
idleCapture.result = { kind: 'pending', startedAtMs: Date.now() };
|
|
652
|
-
void
|
|
653
|
-
.then(() =>
|
|
654
|
-
pid
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
692
|
+
void runnerPid
|
|
693
|
+
.then((pid) => {
|
|
694
|
+
if (pid === null)
|
|
695
|
+
throw new Error('runner PID unavailable: its waiter named no command; no holder conclusion');
|
|
696
|
+
return captureDiagnostics({
|
|
697
|
+
pid,
|
|
698
|
+
pgid: process.platform === 'win32' ? null : group,
|
|
699
|
+
expiresAtMs: capture.expiresAtMs,
|
|
700
|
+
deadlineMs: Math.max(0, Math.min(HANG_CAPTURE_MS, capture.expiresAtMs - Date.now())),
|
|
701
|
+
signal: capture.controller.signal,
|
|
702
|
+
});
|
|
703
|
+
})
|
|
660
704
|
.then((snapshot) => {
|
|
661
705
|
if (idleCapture === capture)
|
|
662
706
|
idleCapture.result = { kind: 'captured', snapshot };
|
|
@@ -900,7 +944,7 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
900
944
|
.finally(releaseCpu)
|
|
901
945
|
.finally(() => releaseLease(true))
|
|
902
946
|
.finally(releaseReservation)
|
|
903
|
-
.finally(async () => recordRun(signalToExitCode(signal), state.expiry, Date.now() - startedAt, state.poolCloseError !== null || lifecycleFailure !== null || state.retireError !== null, state.noSpace === null ? null : await state.noSpace, state.lockWaits))
|
|
947
|
+
.finally(async () => recordRun(signalToExitCode(signal), state.expiry, Date.now() - startedAt, state.poolCloseError !== null || lifecycleFailure !== null || state.retireError !== null, state.noSpace === null ? null : await state.noSpace, state.lockWaits, null))
|
|
904
948
|
.finally(() => process.kill(process.pid, signal));
|
|
905
949
|
};
|
|
906
950
|
const onSigint = () => onTerminationSignal('SIGINT');
|
|
@@ -957,7 +1001,7 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
957
1001
|
if (bound.kind === 'idle') {
|
|
958
1002
|
const observed = idleCapture?.result;
|
|
959
1003
|
state.lockWaits = observed?.kind === 'captured' ? observed.snapshot.lockWaits : [];
|
|
960
|
-
appendStderr(`Hang diagnostics: runner pid=${
|
|
1004
|
+
appendStderr(`Hang diagnostics: runner pid=${runner ?? 'unavailable'} pgid=${process.platform === 'win32' ? 'unavailable' : (child.pid ?? 'unavailable')} expiryAtMs=${Date.now()} scheduledExpiryAtMs=${idleCapture?.expiresAtMs ?? 'unavailable'}\n`);
|
|
961
1005
|
if (observed?.kind === 'captured') {
|
|
962
1006
|
appendStderr(`Hang diagnostics: outcome=${observed.snapshot.outcome} capturedAtMs=${observed.snapshot.capturedAtMs} actualExpiryOffsetMs=${Date.now() - observed.snapshot.capturedAtMs}\n${observed.snapshot.text}`);
|
|
963
1007
|
}
|
|
@@ -1155,9 +1199,14 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
1155
1199
|
await releaseCpu();
|
|
1156
1200
|
const code = exitCode ?? signalToExitCode(exitSignal);
|
|
1157
1201
|
if (code !== 0 && state.expiry === null) {
|
|
1202
|
+
// No bound of this executor ended it, so a signal it died of came from outside (the kernel's OOM killer sends
|
|
1203
|
+
// SIGKILL): the signal is named, never folded into a bare status.
|
|
1204
|
+
const ended = exitCode === null && exitSignal !== null
|
|
1205
|
+
? `was killed by ${exitSignal} (status ${code})`
|
|
1206
|
+
: `exited with status ${code}`;
|
|
1158
1207
|
appendStderr(fixture !== null && !mainStarted
|
|
1159
|
-
? `Fixture command
|
|
1160
|
-
: `Command
|
|
1208
|
+
? `Fixture command ${ended}: ${segment ?? fixture.command}\n`
|
|
1209
|
+
: `Command ${ended}: ${command}\n`);
|
|
1161
1210
|
}
|
|
1162
1211
|
if (state.expiry !== null && !state.forceKillNeeded) {
|
|
1163
1212
|
appendStderr(`Timed out command exited after graceful termination: ${command}\n`);
|
|
@@ -1177,10 +1226,16 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
1177
1226
|
}
|
|
1178
1227
|
await releaseLease(code !== 0 || state.expiry !== null || state.poolCloseError !== null || lifecycleFailure !== null);
|
|
1179
1228
|
releaseReservation();
|
|
1229
|
+
// The command's peak, as its waiter reported it once both ended; a peak that was not measured is said, never zero.
|
|
1230
|
+
const reported = waiterReport === null ? (waiter?.kind === 'unmeasured' ? waiter : null) : await waiterReport.peak;
|
|
1231
|
+
if (reported?.kind === 'unmeasured') {
|
|
1232
|
+
appendStderr(`bounded-exec: this run's memory peak was not measured (${reported.reason}); its record names none\n`);
|
|
1233
|
+
}
|
|
1234
|
+
const peakBytes = reported?.kind === 'measured' ? reported.bytes : null;
|
|
1180
1235
|
// Broken resource ownership, a lost native lifecycle, or leftovers that could not be retired fail even if a shell
|
|
1181
1236
|
// branch later exited 0: its verdict is an ordinary operational failure, never a successful or silently unbounded run.
|
|
1182
1237
|
const operationalFailure = state.poolCloseError !== null || lifecycleFailure !== null || state.retireError !== null;
|
|
1183
|
-
await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure, seen, state.lockWaits);
|
|
1238
|
+
await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure, seen, state.lockWaits, peakBytes);
|
|
1184
1239
|
return {
|
|
1185
1240
|
success: !operationalFailure && resultFailure === null && state.expiry === null && code === 0,
|
|
1186
1241
|
terminalOutput: outputChunks.join(''),
|
|
@@ -1239,11 +1294,9 @@ function targetEnv(env, say) {
|
|
|
1239
1294
|
}
|
|
1240
1295
|
return kept;
|
|
1241
1296
|
}
|
|
1297
|
+
/** The status a shell gives a command a signal ended: 128 plus the signal's number (SIGKILL 137, SIGTERM 143). */
|
|
1242
1298
|
function signalToExitCode(signal) {
|
|
1243
|
-
|
|
1244
|
-
return 1;
|
|
1245
|
-
}
|
|
1246
|
-
return EXIT_CODE_BY_SIGNAL[signal] ?? 1;
|
|
1299
|
+
return signal === null ? 1 : 128 + osConstants.signals[signal];
|
|
1247
1300
|
}
|
|
1248
1301
|
/** Resolves when `promise` settles or `ms` passes, whichever is first, leaving no timer behind. */
|
|
1249
1302
|
async function settledWithin(promise, ms) {
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The memory facts a run's CPU pool charges by (specs/cowshed/05_gateway.md, "Without mbx, every Nx run budgets
|
|
3
|
+
* itself"; 13_telemetry.md, "Job resource peaks"), read by each platform's own mechanism: `host-memory.linux.ts`,
|
|
4
|
+
* `host-memory.darwin.ts`. Callers read them here and never ask which platform they run on.
|
|
5
|
+
*/
|
|
6
|
+
export interface HostMemory {
|
|
7
|
+
/** The memory the run's pool charges its leaves' peaks against. */
|
|
8
|
+
readRunMemory(): Promise<RunMemoryReading>;
|
|
9
|
+
/**
|
|
10
|
+
* A leaf's peak in bytes, from the `maxRSS` that Bun's `Subprocess.resourceUsage()` on `bunVersion` reports for
|
|
11
|
+
* it: the exited leaf's `ru_maxrss`.
|
|
12
|
+
*/
|
|
13
|
+
peakBytes(maxRss: number, bunVersion: string): number;
|
|
14
|
+
/**
|
|
15
|
+
* What the process tree rooted at `root` holds resident at this instant: the sum over `root` and every descendant the
|
|
16
|
+
* kernel links to it, each process's footprint where the platform keeps one (macOS `ri_phys_footprint`, which counts
|
|
17
|
+
* its compressed pages) and its resident set where it does not (Linux `VmRSS`). A process that exits between the
|
|
18
|
+
* listing and its read adds nothing, and a root that has exited holds nothing.
|
|
19
|
+
*/
|
|
20
|
+
treeResidentBytes(root: number): Promise<number>;
|
|
21
|
+
}
|
|
22
|
+
/** The run's memory in bytes, and where it was read: the file or sysctl that holds it. */
|
|
23
|
+
export interface RunMemory {
|
|
24
|
+
readonly bytes: number;
|
|
25
|
+
readonly source: string;
|
|
26
|
+
}
|
|
27
|
+
/** Why the run's memory could not be read: the file or sysctl, and what reading it gave. */
|
|
28
|
+
export interface RunMemoryError {
|
|
29
|
+
readonly operation: 'read-run-memory';
|
|
30
|
+
/**
|
|
31
|
+
* `unreadable`: reading it failed; `malformed`: it held no byte count; `unmounted`: the run's cgroup v2 hierarchy is
|
|
32
|
+
* mounted nowhere this process can see; `unsupported-platform`: the platform has no reader.
|
|
33
|
+
*/
|
|
34
|
+
readonly code: 'unreadable' | 'malformed' | 'unmounted' | 'unsupported-platform';
|
|
35
|
+
readonly subject: string;
|
|
36
|
+
readonly cause: string;
|
|
37
|
+
}
|
|
38
|
+
export type RunMemoryReading = {
|
|
39
|
+
readonly ok: true;
|
|
40
|
+
readonly memory: RunMemory;
|
|
41
|
+
} | {
|
|
42
|
+
readonly ok: false;
|
|
43
|
+
readonly error: RunMemoryError;
|
|
44
|
+
};
|
|
45
|
+
/** A run whose memory cannot be read does not start: it is not shown to have room. */
|
|
46
|
+
export declare class RunMemoryUnreadable extends Error {
|
|
47
|
+
readonly error: RunMemoryError;
|
|
48
|
+
constructor(error: RunMemoryError);
|
|
49
|
+
}
|
|
50
|
+
/** This platform's mechanisms, or null where it has none: there a leaf's peak goes unmeasured. */
|
|
51
|
+
export declare const HOST_MEMORY: HostMemory | null;
|
|
52
|
+
/** The run's memory on this platform, or why it cannot be read. */
|
|
53
|
+
export declare function readRunMemory(): Promise<RunMemoryReading>;
|
|
54
|
+
//# sourceMappingURL=host-memory.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"host-memory.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/host-memory.ts"],"names":[],"mappings":"AAGA;;;;GAIG;AACH,MAAM,WAAW,UAAU;IACzB,mEAAmE;IACnE,aAAa,IAAI,OAAO,CAAC,gBAAgB,CAAC,CAAC;IAC3C;;;OAGG;IACH,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,MAAM,CAAC;IACtD;;;;;OAKG;IACH,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CAClD;AAED,0FAA0F;AAC1F,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED,4FAA4F;AAC5F,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,SAAS,EAAE,iBAAiB,CAAC;IACtC;;;OAGG;IACH,QAAQ,CAAC,IAAI,EAAE,YAAY,GAAG,WAAW,GAAG,WAAW,GAAG,sBAAsB,CAAC;IACjF,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,MAAM,gBAAgB,GACxB;IAAE,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAA;CAAE,GACjD;IAAE,QAAQ,CAAC,EAAE,EAAE,KAAK,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,cAAc,CAAA;CAAE,CAAC;AAE3D,sFAAsF;AACtF,qBAAa,mBAAoB,SAAQ,KAAK;IAChC,QAAQ,CAAC,KAAK,EAAE,cAAc;IAA1C,YAAqB,KAAK,EAAE,cAAc,EAKzC;CACF;AAID,kGAAkG;AAClG,eAAO,MAAM,WAAW,EAAE,UAAU,GAAG,IAA0C,CAAC;AAElF,mEAAmE;AACnE,wBAAgB,aAAa,IAAI,OAAO,CAAC,gBAAgB,CAAC,CAazD"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"host-memory.darwin.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/host-memory.darwin.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,UAAU,EAAoB,MAAM,kBAAkB,CAAC;AAgJrE,eAAO,MAAM,YAAY,EAAE,UAAgF,CAAC"}
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
import { execFile } from 'node:child_process';
|
|
2
|
+
import { promisify } from 'node:util';
|
|
3
|
+
/**
|
|
4
|
+
* macOS's memory mechanisms. A run's memory is the host's, `hw.memsize`: macOS confines no job to a memory limit of
|
|
5
|
+
* its own. Bun reports a leaf's `ru_maxrss` in bytes, as macOS does.
|
|
6
|
+
*/
|
|
7
|
+
const run = promisify(execFile);
|
|
8
|
+
const HW_MEMSIZE = 'hw.memsize';
|
|
9
|
+
async function readRunMemory() {
|
|
10
|
+
let stdout;
|
|
11
|
+
try {
|
|
12
|
+
({ stdout } = await run('sysctl', ['-n', HW_MEMSIZE]));
|
|
13
|
+
}
|
|
14
|
+
catch (error) {
|
|
15
|
+
return {
|
|
16
|
+
ok: false,
|
|
17
|
+
error: {
|
|
18
|
+
operation: 'read-run-memory',
|
|
19
|
+
code: 'unreadable',
|
|
20
|
+
subject: HW_MEMSIZE,
|
|
21
|
+
cause: error instanceof Error ? error.message : String(error),
|
|
22
|
+
},
|
|
23
|
+
};
|
|
24
|
+
}
|
|
25
|
+
const value = stdout.trim();
|
|
26
|
+
const bytes = Number(value);
|
|
27
|
+
if (!/^\d+$/.test(value) || !Number.isSafeInteger(bytes) || bytes === 0) {
|
|
28
|
+
return {
|
|
29
|
+
ok: false,
|
|
30
|
+
error: {
|
|
31
|
+
operation: 'read-run-memory',
|
|
32
|
+
code: 'malformed',
|
|
33
|
+
subject: HW_MEMSIZE,
|
|
34
|
+
cause: `sysctl printed ${JSON.stringify(value)}, not a byte count`,
|
|
35
|
+
},
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
return { ok: true, memory: { bytes, source: HW_MEMSIZE } };
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Bun's FFI, as the peak waiter reaches it. Node, which runs the executor, loads this module too and has no `bun:ffi`;
|
|
42
|
+
* only the waiter, on Bun, reads a tree's footprint, so the FFI loads on its first read, by a specifier the compiler
|
|
43
|
+
* does not resolve against Node's types.
|
|
44
|
+
*/
|
|
45
|
+
const BUN_FFI = 'bun:ffi';
|
|
46
|
+
const LIBPROC = '/usr/lib/libproc.dylib';
|
|
47
|
+
/** `proc_pid_rusage`'s oldest flavor, whose fields every later one starts with. */
|
|
48
|
+
const RUSAGE_INFO_V0 = 0;
|
|
49
|
+
/** `rusage_info_v0`: a 16-byte uuid, then ten u64 counters; `ri_phys_footprint` is the eighth, u64 index 9. */
|
|
50
|
+
const RUSAGE_INFO_V0_WORDS = 12;
|
|
51
|
+
const PHYS_FOOTPRINT = 9;
|
|
52
|
+
function isBunFfi(module) {
|
|
53
|
+
return (typeof module === 'object' &&
|
|
54
|
+
module !== null &&
|
|
55
|
+
'dlopen' in module &&
|
|
56
|
+
typeof module.dlopen === 'function' &&
|
|
57
|
+
'ptr' in module &&
|
|
58
|
+
typeof module.ptr === 'function');
|
|
59
|
+
}
|
|
60
|
+
function isLibprocSymbols(symbols) {
|
|
61
|
+
return (typeof symbols === 'object' &&
|
|
62
|
+
symbols !== null &&
|
|
63
|
+
'proc_listchildpids' in symbols &&
|
|
64
|
+
typeof symbols.proc_listchildpids === 'function' &&
|
|
65
|
+
'proc_pid_rusage' in symbols &&
|
|
66
|
+
typeof symbols.proc_pid_rusage === 'function');
|
|
67
|
+
}
|
|
68
|
+
let footprints;
|
|
69
|
+
async function loadFootprints() {
|
|
70
|
+
const ffi = await import(BUN_FFI);
|
|
71
|
+
if (!isBunFfi(ffi)) {
|
|
72
|
+
throw new Error(`${BUN_FFI} names no dlopen and ptr: a tree's footprint is read on Bun, by the peak waiter`);
|
|
73
|
+
}
|
|
74
|
+
const { symbols } = ffi.dlopen(LIBPROC, {
|
|
75
|
+
proc_listchildpids: { args: ['i32', 'ptr', 'i32'], returns: 'i32' },
|
|
76
|
+
proc_pid_rusage: { args: ['i32', 'i32', 'ptr'], returns: 'i32' },
|
|
77
|
+
});
|
|
78
|
+
if (!isLibprocSymbols(symbols)) {
|
|
79
|
+
throw new Error(`${LIBPROC} opened without proc_listchildpids and proc_pid_rusage`);
|
|
80
|
+
}
|
|
81
|
+
const usage = new BigUint64Array(RUSAGE_INFO_V0_WORDS);
|
|
82
|
+
const usageAt = ffi.ptr(usage);
|
|
83
|
+
let pids = new Int32Array(256);
|
|
84
|
+
return {
|
|
85
|
+
children(pid) {
|
|
86
|
+
// A full buffer may have cut the list short: it is read again into one twice the size.
|
|
87
|
+
for (;;) {
|
|
88
|
+
const count = symbols.proc_listchildpids(pid, ffi.ptr(pids), pids.byteLength);
|
|
89
|
+
if (count < pids.length)
|
|
90
|
+
return count <= 0 ? [] : Array.from(pids.subarray(0, count));
|
|
91
|
+
pids = new Int32Array(pids.length * 2);
|
|
92
|
+
}
|
|
93
|
+
},
|
|
94
|
+
footprint(pid) {
|
|
95
|
+
return symbols.proc_pid_rusage(pid, RUSAGE_INFO_V0, usageAt) === 0 ? Number(usage[PHYS_FOOTPRINT]) : 0;
|
|
96
|
+
},
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
/** What the tree rooted at `root` holds: each member's `ri_phys_footprint`, the tree walked by `proc_listchildpids`. */
|
|
100
|
+
async function treeResidentBytes(root) {
|
|
101
|
+
footprints ??= loadFootprints();
|
|
102
|
+
const proc = await footprints;
|
|
103
|
+
const seen = new Set();
|
|
104
|
+
const pending = [root];
|
|
105
|
+
let total = 0;
|
|
106
|
+
for (let pid = pending.pop(); pid !== undefined; pid = pending.pop()) {
|
|
107
|
+
if (seen.has(pid))
|
|
108
|
+
continue;
|
|
109
|
+
seen.add(pid);
|
|
110
|
+
total += proc.footprint(pid);
|
|
111
|
+
pending.push(...proc.children(pid));
|
|
112
|
+
}
|
|
113
|
+
return total;
|
|
114
|
+
}
|
|
115
|
+
export const darwinMemory = { readRunMemory, peakBytes: (maxRss) => maxRss, treeResidentBytes };
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { darwinMemory } from './host-memory.darwin.js';
|
|
2
|
+
import { linuxMemory } from './host-memory.linux.js';
|
|
3
|
+
/** A run whose memory cannot be read does not start: it is not shown to have room. */
|
|
4
|
+
export class RunMemoryUnreadable extends Error {
|
|
5
|
+
error;
|
|
6
|
+
constructor(error) {
|
|
7
|
+
super(`bounded-exec: this run's memory cannot be read (${error.operation} ${error.code} ${error.subject}: ${error.cause}); a run whose memory cannot be read is not shown to have room, so it does not start`);
|
|
8
|
+
this.error = error;
|
|
9
|
+
this.name = 'RunMemoryUnreadable';
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
const PLATFORMS = { darwin: darwinMemory, linux: linuxMemory };
|
|
13
|
+
/** This platform's mechanisms, or null where it has none: there a leaf's peak goes unmeasured. */
|
|
14
|
+
export const HOST_MEMORY = PLATFORMS[process.platform] ?? null;
|
|
15
|
+
/** The run's memory on this platform, or why it cannot be read. */
|
|
16
|
+
export function readRunMemory() {
|
|
17
|
+
if (HOST_MEMORY === null) {
|
|
18
|
+
return Promise.resolve({
|
|
19
|
+
ok: false,
|
|
20
|
+
error: {
|
|
21
|
+
operation: 'read-run-memory',
|
|
22
|
+
code: 'unsupported-platform',
|
|
23
|
+
subject: process.platform,
|
|
24
|
+
cause: 'only Linux and macOS have a reader of the run memory',
|
|
25
|
+
},
|
|
26
|
+
});
|
|
27
|
+
}
|
|
28
|
+
return HOST_MEMORY.readRunMemory();
|
|
29
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { HostMemory, RunMemoryReading } from './host-memory.js';
|
|
2
|
+
/** A file's text, or the errno code and message its read failed with. */
|
|
3
|
+
export type FileRead = {
|
|
4
|
+
readonly ok: true;
|
|
5
|
+
readonly text: string;
|
|
6
|
+
} | {
|
|
7
|
+
readonly ok: false;
|
|
8
|
+
readonly code: string;
|
|
9
|
+
readonly message: string;
|
|
10
|
+
};
|
|
11
|
+
/** The run's memory, reading every file through `read`. */
|
|
12
|
+
export declare function linuxRunMemory(read: (path: string) => Promise<FileRead>): Promise<RunMemoryReading>;
|
|
13
|
+
export declare const linuxMemory: HostMemory;
|
|
14
|
+
//# sourceMappingURL=host-memory.linux.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"host-memory.linux.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/host-memory.linux.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,UAAU,EAA6B,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAchG,yEAAyE;AACzE,MAAM,MAAM,QAAQ,GAChB;IAAE,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAC5C;IAAE,QAAQ,CAAC,EAAE,EAAE,KAAK,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC;AA2B5E,2DAA2D;AAC3D,wBAAsB,cAAc,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,QAAQ,CAAC,GAAG,OAAO,CAAC,gBAAgB,CAAC,CA8CzG;AAoDD,eAAO,MAAM,WAAW,EAAE,UAA4F,CAAC"}
|