@isentinel/machine-budget 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.mjs CHANGED
@@ -1,588 +1,14 @@
1
- import { createRequire } from "node:module";
1
+ import { c as costKey, i as openSharedLedger, l as learnedCost, n as formatGb, o as treeProbe, r as nodePlatform, s as isLedgerLocked, t as describeRun } from "./run-text-1kRifnCt.mjs";
2
2
  import process from "node:process";
3
- import { execFileSync } from "node:child_process";
4
- import { mkdirSync, readFileSync, readdirSync } from "node:fs";
5
- import os from "node:os";
6
- import path from "node:path";
7
- import { setTimeout } from "node:timers/promises";
8
- import { type } from "arktype";
9
- import { DatabaseSync } from "node:sqlite";
10
- import { Buffer } from "node:buffer";
11
3
  import { randomUUID } from "node:crypto";
12
4
  import { doNothing } from "remeda";
13
- //#region src/learned-cost.ts
14
- const MINIMUM_SAMPLES = 3;
15
- const PERCENTILE = .9;
16
- /**
17
- * Names a learned cost: the clone's git common directory, the project and the
18
- * pass. All worktrees of one clone share the common directory, so they share
19
- * costs.
20
- *
21
- * @param gitCommonDirectory - The git common directory of the clone.
22
- * @param scope - The project and the pass.
23
- * @returns The key of the cost in the ledger.
24
- */
25
- function costKey(gitCommonDirectory, scope) {
26
- return JSON.stringify([
27
- gitCommonDirectory,
28
- scope.project,
29
- scope.pass
30
- ]);
31
- }
32
- /**
33
- * Estimates the memory of one worker from the peaks of earlier runs.
34
- *
35
- * @param samples - Peak memory of one worker in earlier runs, in bytes.
36
- * @returns About the 90th percentile, or undefined with too few samples.
37
- */
38
- function learnedCost(samples) {
39
- if (samples.length < MINIMUM_SAMPLES) return;
40
- const sorted = samples.toSorted((left, right) => left - right);
41
- return sorted[Math.ceil(sorted.length * PERCENTILE) - 1];
42
- }
43
- //#endregion
44
- //#region src/ledger.ts
45
- const LEDGER_FORMAT = 1;
46
- const BUSY_TIMEOUT_MS = 3e4;
47
- const LAST_SEEN = "last_seen_at";
48
- const SCHEMA = `
49
- CREATE TABLE IF NOT EXISTS sessions (
50
- id TEXT PRIMARY KEY,
51
- pid INTEGER NOT NULL,
52
- started_at INTEGER NOT NULL,
53
- heartbeat_at INTEGER NOT NULL
54
- );
55
- CREATE TABLE IF NOT EXISTS leases (
56
- id TEXT PRIMARY KEY,
57
- session TEXT NOT NULL,
58
- lane TEXT NOT NULL,
59
- memory INTEGER NOT NULL,
60
- threads INTEGER NOT NULL
61
- );
62
- CREATE TABLE IF NOT EXISTS waiters (
63
- id TEXT PRIMARY KEY,
64
- session TEXT NOT NULL,
65
- is_big INTEGER NOT NULL,
66
- since INTEGER NOT NULL
67
- );
68
- CREATE TABLE IF NOT EXISTS costs (
69
- key TEXT NOT NULL,
70
- at INTEGER NOT NULL,
71
- bytes INTEGER NOT NULL
72
- );
73
- CREATE INDEX IF NOT EXISTS costs_by_key ON costs (key, at);
74
- CREATE TABLE IF NOT EXISTS meta (
75
- key TEXT PRIMARY KEY,
76
- value INTEGER NOT NULL
77
- );
78
- `;
79
- const sessionRows = type({
80
- id: "string",
81
- heartbeatAt: "number",
82
- pid: "number",
83
- startedAt: "number"
84
- }).array();
85
- const leaseRows = type({
86
- id: "string",
87
- lane: "'main' | 'small'",
88
- memory: "number",
89
- session: "string",
90
- threads: "number"
91
- }).array();
92
- const waiterRows = type({
93
- id: "string",
94
- isBig: type("0 | 1").pipe((bit) => bit === 1),
95
- session: "string",
96
- since: "number"
97
- }).array();
98
- const valueRows = type({ value: "number" }).array();
99
- /**
100
- * Names the ledger file in a directory. A new format takes a new file, so an
101
- * older client never reads a format it does not know.
102
- *
103
- * @param directory - Where the machine keeps its ledger.
104
- * @returns The path of the ledger file.
105
- */
106
- function ledgerPath(directory) {
107
- return path.join(directory, `ledger-v${LEDGER_FORMAT}.sqlite`);
108
- }
109
- /**
110
- * Opens the ledger, and makes its tables when they do not exist. The
111
- * directory of the file must exist.
112
- *
113
- * @param file - A path from `ledgerPath`, or `:memory:` for a private one.
114
- * @returns A handle that runs transactions on the file.
115
- */
116
- function openLedger(file) {
117
- const database = openDatabase(file);
118
- function query(sql) {
119
- return database.prepare(sql);
120
- }
121
- const transaction = {
122
- ...metaTable(query),
123
- ...sessionTable(query),
124
- ...leaseTable(query),
125
- ...waiterTable(query),
126
- ...costTable(query)
127
- };
128
- return {
129
- close: () => {
130
- database.close();
131
- },
132
- transact: (work) => {
133
- database.exec("BEGIN IMMEDIATE");
134
- try {
135
- const result = work(transaction);
136
- database.exec("COMMIT");
137
- return result;
138
- } catch (err) {
139
- database.exec("ROLLBACK");
140
- throw err;
141
- }
142
- }
143
- };
144
- }
145
- function openDatabase(file) {
146
- const database = new DatabaseSync(file, { timeout: BUSY_TIMEOUT_MS });
147
- database.exec("PRAGMA journal_mode = WAL");
148
- database.exec(SCHEMA);
149
- return database;
150
- }
151
- function parseRows(schema, rows) {
152
- const parsed = schema(rows.map((row) => ({ ...row })));
153
- if (parsed instanceof type.errors) throw new Error(`machine budget ledger holds an invalid row: ${parsed.summary}`);
154
- return parsed;
155
- }
156
- function metaTable(query) {
157
- return {
158
- lastSeenAt: () => {
159
- const rows = query("SELECT value FROM meta WHERE key = ?").all(LAST_SEEN);
160
- return parseRows(valueRows, rows)[0]?.value;
161
- },
162
- markSeen: (at) => {
163
- query("INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT (key) DO UPDATE SET value = excluded.value").run(LAST_SEEN, at);
164
- }
165
- };
166
- }
167
- function sessionTable(query) {
168
- return {
169
- refreshHeartbeats: (at) => {
170
- query("UPDATE sessions SET heartbeat_at = ?").run(at);
171
- },
172
- removeSessions: (ids) => {
173
- for (const id of ids) {
174
- query("DELETE FROM leases WHERE session = ?").run(id);
175
- query("DELETE FROM waiters WHERE session = ?").run(id);
176
- query("DELETE FROM sessions WHERE id = ?").run(id);
177
- }
178
- },
179
- sessions: () => {
180
- const rows = query("SELECT id, heartbeat_at AS heartbeatAt, pid, started_at AS startedAt FROM sessions").all();
181
- return parseRows(sessionRows, rows);
182
- },
183
- upsertSession: ({ id, heartbeatAt, pid, startedAt }) => {
184
- query("INSERT INTO sessions (id, pid, started_at, heartbeat_at) VALUES (?, ?, ?, ?) ON CONFLICT (id) DO UPDATE SET heartbeat_at = excluded.heartbeat_at").run(id, pid, startedAt, heartbeatAt);
185
- }
186
- };
187
- }
188
- function leaseTable(query) {
189
- return {
190
- addLease: ({ id, lane, memory, session, threads }) => {
191
- query("INSERT OR IGNORE INTO leases (id, session, lane, memory, threads) VALUES (?, ?, ?, ?, ?)").run(id, session, lane, memory, threads);
192
- },
193
- leases: () => {
194
- const rows = query("SELECT id, lane, memory, session, threads FROM leases").all();
195
- return parseRows(leaseRows, rows);
196
- },
197
- removeLease: (id) => {
198
- query("DELETE FROM leases WHERE id = ?").run(id);
199
- }
200
- };
201
- }
202
- function waiterTable(query) {
203
- return {
204
- removeWaiter: (id) => {
205
- query("DELETE FROM waiters WHERE id = ?").run(id);
206
- },
207
- upsertWaiter: ({ id, isBig, session, since }) => {
208
- query("INSERT OR IGNORE INTO waiters (id, session, is_big, since) VALUES (?, ?, ?, ?)").run(id, session, isBig ? 1 : 0, since);
209
- },
210
- waiters: () => {
211
- const rows = query("SELECT id, is_big AS isBig, session, since FROM waiters").all();
212
- return parseRows(waiterRows, rows);
213
- }
214
- };
215
- }
216
- function costTable(query) {
217
- return {
218
- addSample: ({ key, at, bytes }) => {
219
- query("INSERT INTO costs (key, at, bytes) VALUES (?, ?, ?)").run(key, at, bytes);
220
- query("DELETE FROM costs WHERE key = ? AND rowid NOT IN (SELECT rowid FROM costs WHERE key = ? ORDER BY at DESC, rowid DESC LIMIT ?)").run(key, key, 20);
221
- },
222
- costSamples: (key) => {
223
- const rows = query("SELECT bytes AS value FROM costs WHERE key = ? ORDER BY at DESC, rowid DESC").all(key);
224
- return parseRows(valueRows, rows).map((row) => row.value);
225
- }
226
- };
227
- }
228
- //#endregion
229
- //#region src/memory-probe.ts
230
- const PAGE_SIZE = /page size of (\d+) bytes/;
231
- const AVAILABLE_PAGES = /* @__PURE__ */ new Set([
232
- "Pages free",
233
- "Pages inactive",
234
- "Pages purgeable"
235
- ]);
236
- const PRESSURE_LEVELS = /* @__PURE__ */ new Map([
237
- ["1", "normal"],
238
- ["2", "warn"],
239
- ["4", "critical"]
240
- ]);
241
- /**
242
- * Reads the memory that can be given now. MacOS counts free, inactive and
243
- * purgeable pages, and reports its pressure level; Windows and Linux report
244
- * free memory, which on Linux is already the kernel's available estimate.
245
- *
246
- * @param sources - The operating system and how to read it.
247
- * @returns The available memory and the pressure level.
248
- */
249
- function readMemory(sources) {
250
- if (sources.platform !== "darwin") return {
251
- availableMemory: sources.freeMemory(),
252
- pressure: "normal"
253
- };
254
- const vmStat = sources.run("vm_stat", []);
255
- const level = sources.run("sysctl", ["-n", "kern.memorystatus_vm_pressure_level"]);
256
- return {
257
- availableMemory: (vmStat === void 0 ? void 0 : parseVmStat(vmStat)) ?? sources.freeMemory(),
258
- pressure: parsePressure(level)
259
- };
260
- }
261
- function parsePressure(level) {
262
- if (level === void 0) return "normal";
263
- return PRESSURE_LEVELS.get(level.trim()) ?? "normal";
264
- }
265
- function parseVmStat(output) {
266
- const pageSize = PAGE_SIZE.exec(output)?.[1];
267
- if (pageSize === void 0) return;
268
- let pages = 0;
269
- for (const line of output.split("\n")) {
270
- const separator = line.indexOf(":");
271
- if (AVAILABLE_PAGES.has(line.slice(0, separator))) pages += Number.parseInt(line.slice(separator + 1), 10);
272
- }
273
- return pages * Number(pageSize);
274
- }
275
- //#endregion
276
- //#region src/tree-linux.ts
277
- const BYTES_PER_KB = 1024;
278
- const PARENT_FIELD = 1;
279
- const PEAK_RESIDENT = /^VmHWM:\s+(\d+) kB$/m;
280
- const RESIDENT = /^VmRSS:\s+(\d+) kB$/m;
281
- /**
282
- * Reads a process tree from `/proc`. Linux keeps no peak commit, so the peak
283
- * of a descendant is its peak resident set, and the peak of the tree is the
284
- * highest sum of resident sets over the samples.
285
- *
286
- * @param root - The pid at the top of the tree.
287
- * @param source - How to read `/proc`.
288
- * @returns The probe, which keeps the peaks of descendants that have exited.
289
- */
290
- function linuxTreeProbe(root, source) {
291
- const peaks = /* @__PURE__ */ new Map();
292
- let treePeak;
293
- return {
294
- descendants: () => peaks,
295
- sample: () => {
296
- let resident = 0;
297
- let isSeen = false;
298
- for (const pid of [root, ...descendantsOf(source, root)]) {
299
- const status = source.read(`/proc/${pid}/status`);
300
- if (status === void 0) continue;
301
- isSeen = true;
302
- resident += bytes(status, RESIDENT);
303
- if (pid !== root) peaks.set(pid, Math.max(peaks.get(pid) ?? 0, bytes(status, PEAK_RESIDENT)));
304
- }
305
- if (isSeen) treePeak = Math.max(treePeak ?? 0, resident);
306
- },
307
- tree: () => treePeak
308
- };
309
- }
310
- function parentOf(source, pid) {
311
- const stat = source.read(`/proc/${pid}/stat`);
312
- if (stat === void 0) return;
313
- return Number(stat.slice(stat.lastIndexOf(")") + 2).split(" ", 2)[PARENT_FIELD]);
314
- }
315
- function descendantsOf(source, root) {
316
- const children = /* @__PURE__ */ new Map();
317
- for (const pid of source.pids()) {
318
- const parent = parentOf(source, pid);
319
- if (parent !== void 0) children.set(parent, [...children.get(parent) ?? [], pid]);
320
- }
321
- const found = [];
322
- const pending = [root];
323
- for (let pid = pending.pop(); pid !== void 0; pid = pending.pop()) {
324
- const next = children.get(pid) ?? [];
325
- found.push(...next);
326
- pending.push(...next);
327
- }
328
- return found;
329
- }
330
- function bytes(status, field) {
331
- return Number(field.exec(status)?.[1] ?? 0) * BYTES_PER_KB;
332
- }
333
- //#endregion
334
- //#region src/tree-windows.ts
335
- const QUERY_LIMITED_ACCESS = 4096;
336
- const BASIC_PROCESS_ID_LIST = 3;
337
- const EXTENDED_LIMIT_INFORMATION = 9;
338
- const EXTENDED_LIMIT_SIZE = 144;
339
- const PEAK_JOB_MEMORY_OFFSET = 136;
340
- const LISTED_COUNT_OFFSET = 4;
341
- const PROCESS_ID_LIST_HEADER = 8;
342
- const PROCESS_ID_SIZE = 8;
343
- const MEMORY_COUNTERS_SIZE = 72;
344
- const PEAK_PAGEFILE_OFFSET = 64;
345
- /**
346
- * Puts the root in a new Job Object. The tree peak is the job's peak commit.
347
- * A descendant's peak is its peak pagefile usage, read through a handle that
348
- * opens when a sample first lists it, so the figure outlives the process.
349
- *
350
- * @param root - The pid at the top of the tree.
351
- * @returns The probe, or undefined when koffi does not load or the root cannot
352
- * join a job.
353
- */
354
- function windowsTreeProbe(root) {
355
- let kernel32;
356
- try {
357
- kernel32 = loadKernel32();
358
- } catch {
359
- return;
360
- }
361
- const job = attach(kernel32, root);
362
- if (job === null) return;
363
- const state = {
364
- handles: /* @__PURE__ */ new Map(),
365
- job,
366
- kernel32,
367
- root
368
- };
369
- return {
370
- close: () => {
371
- closeJob(state);
372
- },
373
- descendants: () => descendantPeaks(state),
374
- sample: () => {
375
- openNewDescendants(state);
376
- },
377
- tree: () => jobPeak(state)
378
- };
379
- }
380
- function loadKernel32() {
381
- const library = createRequire(import.meta.url)("koffi").load("kernel32.dll");
382
- return {
383
- assign: library.func("bool AssignProcessToJobObject(void *job, void *process)"),
384
- close: library.func("bool CloseHandle(void *handle)"),
385
- createJob: library.func("void *CreateJobObjectW(void *attributes, void *name)"),
386
- open: library.func("void *OpenProcess(uint32_t access, bool inherit, uint32_t pid)"),
387
- queryJob: library.func("bool QueryInformationJobObject(void *job, int infoClass, _Out_ uint8_t *info, uint32_t size, void *returned)"),
388
- readCounters: library.func("bool K32GetProcessMemoryInfo(void *process, _Out_ uint8_t *counters, uint32_t size)")
389
- };
390
- }
391
- function attach(kernel32, root) {
392
- const process = kernel32.open(257, false, root);
393
- if (process === null) return null;
394
- const job = kernel32.createJob(null, null);
395
- const isAssigned = job !== null && kernel32.assign(job, process) === true;
396
- kernel32.close(process);
397
- if (!isAssigned && job !== null) kernel32.close(job);
398
- return isAssigned ? job : null;
399
- }
400
- function listedProcesses({ job, kernel32 }) {
401
- const list = Buffer.alloc(8200);
402
- if (kernel32.queryJob(job, BASIC_PROCESS_ID_LIST, list, list.length, null) !== true) return [];
403
- return Array.from({ length: list.readUInt32LE(LISTED_COUNT_OFFSET) }, (_, index) => {
404
- return Number(list.readBigUInt64LE(PROCESS_ID_LIST_HEADER + index * PROCESS_ID_SIZE));
405
- });
406
- }
407
- function openNewDescendants(state) {
408
- for (const pid of listedProcesses(state)) {
409
- if (pid === state.root || state.handles.has(pid)) continue;
410
- const handle = state.kernel32.open(QUERY_LIMITED_ACCESS, false, pid);
411
- if (handle !== null) state.handles.set(pid, handle);
412
- }
413
- }
414
- function descendantPeaks({ handles, kernel32 }) {
415
- const counters = Buffer.alloc(MEMORY_COUNTERS_SIZE);
416
- const peaks = /* @__PURE__ */ new Map();
417
- for (const [pid, handle] of handles) if (kernel32.readCounters(handle, counters, counters.length) === true) peaks.set(pid, Number(counters.readBigUInt64LE(PEAK_PAGEFILE_OFFSET)));
418
- return peaks;
419
- }
420
- function jobPeak({ job, kernel32 }) {
421
- const limits = Buffer.alloc(EXTENDED_LIMIT_SIZE);
422
- if (kernel32.queryJob(job, EXTENDED_LIMIT_INFORMATION, limits, limits.length, null) !== true) return;
423
- return Number(limits.readBigUInt64LE(PEAK_JOB_MEMORY_OFFSET));
424
- }
425
- function closeJob({ handles, job, kernel32 }) {
426
- for (const handle of handles.values()) kernel32.close(handle);
427
- handles.clear();
428
- kernel32.close(job);
429
- }
430
- //#endregion
431
- //#region src/adapters.ts
432
- const COMMAND_TIMEOUT_MS = 5e3;
433
- const LINUX_CLOCK_TICKS_PER_SECOND = 100;
434
- const LINUX_START_TIME_FIELD = 19;
435
- const BOOT_TIME = /^btime (\d+)$/m;
436
- const PID_NAME = /^\d+$/;
437
- /**
438
- * Opens the ledger of this machine in the directory that `ledgerDirectory`
439
- * names for this process, and makes the directory when it does not exist.
440
- *
441
- * @returns The ledger that every run of this user on the machine opens.
442
- */
443
- function openSharedLedger() {
444
- const directory = ledgerDirectory(process.env);
445
- mkdirSync(directory, { recursive: true });
446
- return openLedger(ledgerPath(directory));
447
- }
448
- function ledgerDirectory(environment) {
449
- const configured = environment["MACHINE_BUDGET_DIR"];
450
- if (configured !== void 0 && configured !== "") return configured;
451
- return path.join(os.tmpdir(), `machine-budget-${os.userInfo().username}`);
452
- }
453
- function constrainedMemory() {
454
- return process.constrainedMemory?.() ?? 0;
455
- }
456
- function run(command, args) {
457
- try {
458
- return execFileSync(command, args, {
459
- encoding: "utf8",
460
- stdio: [
461
- "ignore",
462
- "pipe",
463
- "ignore"
464
- ],
465
- timeout: COMMAND_TIMEOUT_MS,
466
- windowsHide: true
467
- });
468
- } catch {
469
- return;
470
- }
471
- }
472
- function isAlive(pid) {
473
- try {
474
- process.kill(pid, 0);
475
- return true;
476
- } catch (err) {
477
- return !(err instanceof Error) || !("code" in err) || err.code !== "ESRCH";
478
- }
479
- }
480
- function linuxStartTime(pid) {
481
- try {
482
- const stat = readFileSync(`/proc/${pid}/stat`, "utf8");
483
- const fields = stat.slice(stat.lastIndexOf(")") + 2).split(" ");
484
- const bootTime = BOOT_TIME.exec(readFileSync("/proc/stat", "utf8"))?.[1];
485
- const ticks = fields[LINUX_START_TIME_FIELD];
486
- if (bootTime === void 0 || ticks === void 0) return;
487
- return (Number(bootTime) + Number(ticks) / LINUX_CLOCK_TICKS_PER_SECOND) * 1e3;
488
- } catch {
489
- return;
490
- }
491
- }
492
- function parseTime(text) {
493
- const time = text === void 0 ? NaN : Date.parse(text.trim());
494
- return Number.isNaN(time) ? void 0 : time;
495
- }
496
- function windowsStartTime(pid) {
497
- return parseTime(run("powershell.exe", [
498
- "-NoProfile",
499
- "-NonInteractive",
500
- "-Command",
501
- `(Get-Process -Id ${pid}).StartTime.ToUniversalTime().ToString('o')`
502
- ]));
503
- }
504
- const START_TIME_READERS = {
505
- darwin: (pid) => parseTime(run("ps", [
506
- "-o",
507
- "lstart=",
508
- "-p",
509
- String(pid)
510
- ])),
511
- linux: linuxStartTime,
512
- win32: windowsStartTime
513
- };
514
- function startTime(pid) {
515
- return START_TIME_READERS[process.platform]?.(pid);
516
- }
517
- /** The real machine and process. */
518
- const nodePlatform = {
519
- every: (milliseconds, work) => {
520
- const timer = setInterval(work, milliseconds);
521
- timer.unref();
522
- return () => {
523
- clearInterval(timer);
524
- };
525
- },
526
- gitCommonDirectory: (workspaceRoot) => {
527
- const output = run("git", [
528
- "-C",
529
- workspaceRoot,
530
- "rev-parse",
531
- "--path-format=absolute",
532
- "--git-common-dir"
533
- ]);
534
- return output === void 0 ? workspaceRoot : output.trim();
535
- },
536
- now: Date.now,
537
- pid: process.pid,
538
- probe: {
539
- isAlive,
540
- startTime
541
- },
542
- readMemory: () => readMemory({
543
- freeMemory: os.freemem,
544
- platform: process.platform,
545
- run
546
- }),
547
- sleepAsync: async (milliseconds) => {
548
- await setTimeout(milliseconds);
549
- },
550
- startedAt: Date.now() - process.uptime() * 1e3,
551
- threads: os.availableParallelism(),
552
- totalMemory: Math.min(os.totalmem(), constrainedMemory() || Infinity)
553
- };
554
- /**
555
- * Chooses how this platform reads a process tree: a Job Object on Windows,
556
- * `/proc` on Linux, and nothing elsewhere.
557
- *
558
- * @param pid - The pid at the top of the tree.
559
- * @returns The probe of this platform, or undefined where it cannot read one.
560
- */
561
- function treeProbe(pid) {
562
- if (process.platform === "win32") return windowsTreeProbe(pid);
563
- return process.platform === "linux" ? linuxTreeProbe(pid, {
564
- pids: procPids,
565
- read: readProc
566
- }) : void 0;
567
- }
568
- function readProc(file) {
569
- try {
570
- return readFileSync(file, "utf8");
571
- } catch {
572
- return;
573
- }
574
- }
575
- function procPids() {
576
- return readdirSync("/proc").filter((name) => PID_NAME.test(name)).map(Number);
577
- }
578
- //#endregion
579
- //#region src/admission.ts
580
- const BYTES_PER_GB$1 = 2 ** 30;
5
+ // src/admission.ts
6
+ const BYTES_PER_GB = 2 ** 30;
581
7
  const MAIN_MEMORY_SHARE = .5;
582
8
  const RESERVE_SHARE = .15;
583
9
  const LANE_MEMORY_SHARE = .15;
584
10
  const LANE_THREADS = 2;
585
- const SMALL_REQUEST_BYTES = 2 * BYTES_PER_GB$1;
11
+ const SMALL_REQUEST_BYTES = 2 * BYTES_PER_GB;
586
12
  const AGING_MS = 6e4;
587
13
  /**
588
14
  * Whether a request takes the small-job lane when the main lane is full.
@@ -594,6 +20,17 @@ function isSmallRequest(request) {
594
20
  return request.minWorkers * request.memoryPerWorker <= SMALL_REQUEST_BYTES;
595
21
  }
596
22
  /**
23
+ * The oldest big request that has waited past the aging limit and started
24
+ * before this one, which every younger request waits behind.
25
+ *
26
+ * @param input - The time, this request's waiter id, and every waiter.
27
+ * @returns The aged waiter, or undefined when none blocks this request.
28
+ */
29
+ function olderAgedWaiter({ now, waiterId, waiters }) {
30
+ const since = waiters.find((waiter) => waiter.id === waiterId)?.since ?? now;
31
+ return waiters.filter((waiter) => waiter.isBig && waiter.since < since && now - waiter.since > AGING_MS).toSorted((a, b) => a.since - b.since)[0];
32
+ }
33
+ /**
597
34
  * Decides one request: the largest worker count that fits, or why it waits.
598
35
  *
599
36
  * @param input - The machine, the leases and waiters, and the request.
@@ -602,7 +39,7 @@ function isSmallRequest(request) {
602
39
  function admit(input) {
603
40
  const { leases, machine, request } = input;
604
41
  const isBig = !isSmallRequest(request);
605
- const isBehindAged = hasOlderAgedWaiter(input);
42
+ const isBehindAged = olderAgedWaiter(input) !== void 0;
606
43
  const mainLeases = leases.filter((lease) => lease.lane === "main");
607
44
  if (isBig && machine.pressure !== "normal" && mainLeases.length > 0) return {
608
45
  kind: "wait",
@@ -647,12 +84,6 @@ function runCapWorkers(input) {
647
84
  const byMemory = Math.floor(limit.memory / runs / request.memoryPerWorker);
648
85
  return Math.min(request.maxWorkers, byMemory, Math.floor(limit.threads / runs));
649
86
  }
650
- function hasOlderAgedWaiter({ now, waiterId, waiters }) {
651
- const since = waiters.find((waiter) => waiter.id === waiterId)?.since ?? now;
652
- return waiters.some((waiter) => {
653
- return waiter.isBig && waiter.since < since && now - waiter.since > AGING_MS;
654
- });
655
- }
656
87
  function activeRuns({ leases, session, waiters }) {
657
88
  const sessions = /* @__PURE__ */ new Set([session]);
658
89
  for (const lease of leases) if (lease.lane === "main") sessions.add(lease.session);
@@ -739,8 +170,7 @@ function fitWorkers(request, fit) {
739
170
  workers
740
171
  };
741
172
  }
742
- //#endregion
743
- //#region src/reclaim.ts
173
+ // src/reclaim.ts
744
174
  /** How often a live session stamps its heartbeat. */
745
175
  const HEARTBEAT_INTERVAL_MS = 5e3;
746
176
  const SUSPECT_MS = 3 * HEARTBEAT_INTERVAL_MS;
@@ -775,11 +205,140 @@ function isDead({ isHeartbeatSkipped, now, probe, session }) {
775
205
  if (startedAt !== void 0 && Math.abs(startedAt - session.startedAt) > START_TOLERANCE_MS) return true;
776
206
  return !isHeartbeatSkipped && age >= STALE_MS;
777
207
  }
778
- //#endregion
779
- //#region src/budget.ts
208
+ // src/prune.ts
209
+ /**
210
+ * Reads the sessions with no write lock, then probes their processes with no
211
+ * transaction open, so a slow probe never holds the ledger lock.
212
+ *
213
+ * @param surveyor - The session that surveys.
214
+ * @returns The sessions and the dead among them.
215
+ */
216
+ function surveySessions({ id, ledger, options }) {
217
+ const { platform } = options;
218
+ const { lastSeenAt, sessions } = ledger.read((reader) => {
219
+ return {
220
+ lastSeenAt: reader.lastSeenAt(),
221
+ sessions: reader.sessions()
222
+ };
223
+ });
224
+ const now = platform.now();
225
+ return {
226
+ ...findDeadSessions({
227
+ lastSeenAt,
228
+ now,
229
+ probe: platform.probe,
230
+ self: id,
231
+ sessions
232
+ }),
233
+ now,
234
+ sessions
235
+ };
236
+ }
237
+ /**
238
+ * Removes the dead sessions of a survey that no run changed since, and marks
239
+ * the ledger seen.
240
+ *
241
+ * @param transaction - The open write transaction.
242
+ * @param pruning - The survey and the time to write.
243
+ */
244
+ function prune(transaction, { now, survey }) {
245
+ const { dead, isHeartbeatSkipped, sessions } = survey;
246
+ const current = new Map(transaction.sessions().map((record) => [record.id, record]));
247
+ const unchanged = sessions.filter((record) => {
248
+ const stored = current.get(record.id);
249
+ return dead.includes(record.id) && stored !== void 0 && isSameSession(record, stored);
250
+ });
251
+ transaction.removeSessions(unchanged.map((record) => record.id));
252
+ if (isHeartbeatSkipped) transaction.refreshHeartbeats(now);
253
+ transaction.markSeen(now);
254
+ }
255
+ /**
256
+ * Prunes, then puts the session and its leases back with a fresh heartbeat.
257
+ *
258
+ * @param transaction - The open write transaction.
259
+ * @param restorer - The session and its survey.
260
+ * @returns The time of the writes.
261
+ */
262
+ function restore(transaction, { session, survey }) {
263
+ const { platform } = session.options;
264
+ const now = platform.now();
265
+ prune(transaction, {
266
+ now,
267
+ survey
268
+ });
269
+ transaction.upsertSession({
270
+ id: session.id,
271
+ heartbeatAt: now,
272
+ pid: platform.pid,
273
+ startedAt: platform.startedAt,
274
+ worktree: session.options.workspaceRoot
275
+ });
276
+ for (const lease of session.held.values()) transaction.addLease(lease);
277
+ return now;
278
+ }
279
+ /**
280
+ * Surveys the sessions, then restores the session in one write transaction.
281
+ *
282
+ * @param session - The session to restore.
283
+ */
284
+ function restoreSession(session) {
285
+ const survey = surveySessions(session);
286
+ session.ledger.transact((transaction) => restore(transaction, {
287
+ session,
288
+ survey
289
+ }));
290
+ }
291
+ function isSameSession(left, right) {
292
+ return left.heartbeatAt === right.heartbeatAt && left.pid === right.pid && left.startedAt === right.startedAt;
293
+ }
294
+ // src/wait-line.ts
295
+ /**
296
+ * Writes the line of a waiting request: its size, its reason, and the runs in
297
+ * its way. Behind an older request, that is the run of the aged request;
298
+ * otherwise each other run that holds leases, the largest first.
299
+ *
300
+ * @param notice - The request, the reason and the ledger rows.
301
+ * @returns The line and its reprint key.
302
+ */
303
+ function waitLine(notice) {
304
+ const { reason, request } = notice;
305
+ const need = formatGb(request.minWorkers * request.memoryPerWorker);
306
+ const { cause, runs } = causeOf(notice);
307
+ return {
308
+ line: `machine budget: waiting for ${need} and ${request.minWorkers} threads (${reason}); ${cause}; machine-budget-status prints the ledger`,
309
+ reprintKey: [reason, ...runs.toSorted()].join(" ")
310
+ };
311
+ }
312
+ function holdersOf({ leases, session: own, sessions }) {
313
+ return sessions.filter((session) => session.id !== own).map((session) => {
314
+ const owned = leases.filter((lease) => lease.session === session.id);
315
+ return {
316
+ memory: owned.reduce((total, lease) => total + lease.memory, 0),
317
+ session,
318
+ threads: owned.reduce((total, lease) => total + lease.threads, 0)
319
+ };
320
+ }).filter(({ threads }) => threads > 0).toSorted((a, b) => b.memory - a.memory);
321
+ }
322
+ function causeOf(notice) {
323
+ const { reason, sessions, waiters } = notice;
324
+ const aged = reason === "older request" ? olderAgedWaiter(notice) : void 0;
325
+ const blocker = waiters.find((waiter) => waiter.id === aged?.id);
326
+ if (blocker !== void 0) return {
327
+ cause: `behind ${describeRun(sessions.find((session) => session.id === blocker.session))} (${formatGb(blocker.memory)}, ${blocker.threads} threads)`,
328
+ runs: [blocker.id]
329
+ };
330
+ const holders = holdersOf(notice);
331
+ const parts = holders.map(({ memory, session, threads }) => {
332
+ return `${describeRun(session)} (${formatGb(memory)}, ${threads} threads)`;
333
+ });
334
+ return {
335
+ cause: parts.length === 0 ? "no run holds a lease" : `held by ${parts.join(", ")}`,
336
+ runs: holders.map(({ session }) => session.id)
337
+ };
338
+ }
339
+ // src/budget.ts
780
340
  const FIRST_RETRY_MS = 250;
781
341
  const LAST_RETRY_MS = 2e3;
782
- const BYTES_PER_GB = 2 ** 30;
783
342
  /**
784
343
  * Joins the machine budget. Each lease is one row in the shared ledger until
785
344
  * it is released, the run closes, or a later run finds the run dead.
@@ -791,9 +350,16 @@ function openBudget(options) {
791
350
  if (isBudgetOff(options.environment)) return unlimitedBudget;
792
351
  return sharedBudget(joinSession(options));
793
352
  }
353
+ function heartbeat(session) {
354
+ try {
355
+ restoreSession(session);
356
+ } catch (err) {
357
+ session.options.log(`machine budget: heartbeat failed: ${String(err)}`);
358
+ }
359
+ }
794
360
  function sharedBudget(session) {
795
361
  const stop = session.options.platform.every(HEARTBEAT_INTERVAL_MS, () => {
796
- session.ledger.transact((transaction) => restore(session, transaction));
362
+ heartbeat(session);
797
363
  });
798
364
  let isClosed = false;
799
365
  return {
@@ -852,33 +418,6 @@ function assertRequest({ maxWorkers, memoryPerWorker, minWorkers }) {
852
418
  if (!Number.isSafeInteger(minWorkers) || minWorkers < 1 || maxWorkers < minWorkers) throw new RangeError(`a lease needs 1 <= minWorkers <= maxWorkers, got ${minWorkers} and ${maxWorkers}`);
853
419
  if (!Number.isSafeInteger(maxWorkers) || !Number.isFinite(memoryPerWorker) || memoryPerWorker <= 0) throw new RangeError(`a lease needs a whole maxWorkers and memoryPerWorker above 0, got ${maxWorkers} and ${memoryPerWorker}`);
854
420
  }
855
- function prune(session, transaction) {
856
- const { platform } = session.options;
857
- const now = platform.now();
858
- const { dead, isHeartbeatSkipped } = findDeadSessions({
859
- lastSeenAt: transaction.lastSeenAt(),
860
- now,
861
- probe: platform.probe,
862
- self: session.id,
863
- sessions: transaction.sessions()
864
- });
865
- transaction.removeSessions(dead);
866
- if (isHeartbeatSkipped) transaction.refreshHeartbeats(now);
867
- transaction.markSeen(now);
868
- return now;
869
- }
870
- function restore(session, transaction) {
871
- const { platform } = session.options;
872
- const now = prune(session, transaction);
873
- transaction.upsertSession({
874
- id: session.id,
875
- heartbeatAt: now,
876
- pid: platform.pid,
877
- startedAt: platform.startedAt
878
- });
879
- for (const lease of session.held.values()) transaction.addLease(lease);
880
- return now;
881
- }
882
421
  function joinSession(options) {
883
422
  const session = {
884
423
  id: randomUUID(),
@@ -887,7 +426,7 @@ function joinSession(options) {
887
426
  options,
888
427
  repository: options.platform.gitCommonDirectory(options.workspaceRoot)
889
428
  };
890
- session.ledger.transact((transaction) => restore(session, transaction));
429
+ restoreSession(session);
891
430
  return session;
892
431
  }
893
432
  function leaveSession(session) {
@@ -898,27 +437,50 @@ function leaveSession(session) {
898
437
  }
899
438
  function workerMemory({ request, session, transaction }) {
900
439
  if (request.cost === void 0) return request.memoryPerWorker;
901
- return learnedCost(transaction.costSamples(costKey(session.repository, request.cost))) ?? request.memoryPerWorker;
902
- }
903
- function formatGb(bytes) {
904
- return `${(bytes / BYTES_PER_GB).toFixed(1)} GB`;
440
+ const samples = transaction.costSamples(costKey(session.repository, request.cost));
441
+ return learnedCost(samples) ?? request.memoryPerWorker;
442
+ }
443
+ function recordWaiter({ session, transaction, waiterId }, { now, request }) {
444
+ transaction.upsertWaiter({
445
+ id: waiterId,
446
+ isBig: !isSmallRequest(request),
447
+ memory: request.minWorkers * request.memoryPerWorker,
448
+ session: session.id,
449
+ since: now,
450
+ threads: request.minWorkers
451
+ });
905
452
  }
906
- function waitLine({ leases, reason, request }) {
907
- const leased = leases.reduce((total, lease) => total + lease.memory, 0);
908
- const runs = new Set(leases.map((lease) => lease.session));
909
- return `machine budget: waiting for ${formatGb(request.minWorkers * request.memoryPerWorker)} and ${request.minWorkers} threads (${reason}); ${formatGb(leased)} leased by ${runs.size} runs`;
453
+ function grant({ session, transaction, waiterId }, { decision, request }) {
454
+ const { lane, workers } = decision;
455
+ const memory = workers * request.memoryPerWorker;
456
+ const lease = {
457
+ id: randomUUID(),
458
+ lane,
459
+ memory,
460
+ session: session.id,
461
+ threads: workers
462
+ };
463
+ transaction.addLease(lease);
464
+ transaction.removeWaiter(waiterId);
465
+ return {
466
+ kind: "grant",
467
+ lease,
468
+ memoryPerWorker: request.memoryPerWorker
469
+ };
910
470
  }
911
471
  function decide(context) {
912
- const { isWaiting, reading, session, transaction, waiterId } = context;
472
+ const { isWaiting, reading, session, survey, transaction, waiterId } = context;
913
473
  const { platform } = session.options;
914
- const now = restore(session, transaction);
474
+ const now = restore(transaction, {
475
+ session,
476
+ survey
477
+ });
915
478
  const request = {
916
479
  ...context.request,
917
480
  memoryPerWorker: workerMemory(context)
918
481
  };
919
- const leases = transaction.leases();
920
- const decision = admit({
921
- leases,
482
+ const input = {
483
+ leases: transaction.leases(),
922
484
  machine: {
923
485
  ...reading,
924
486
  threads: platform.threads,
@@ -929,49 +491,33 @@ function decide(context) {
929
491
  session: session.id,
930
492
  waiterId,
931
493
  waiters: transaction.waiters()
932
- });
933
- if (decision.kind === "wait") {
934
- if (isWaiting) {
935
- const isBig = !isSmallRequest(request);
936
- transaction.upsertWaiter({
937
- id: waiterId,
938
- isBig,
939
- session: session.id,
940
- since: now
941
- });
942
- }
943
- return {
944
- kind: "wait",
945
- line: waitLine({
946
- leases,
947
- reason: decision.reason,
948
- request
949
- })
950
- };
951
- }
952
- const { lane, workers } = decision;
953
- const memory = workers * request.memoryPerWorker;
954
- const lease = {
955
- id: randomUUID(),
956
- lane,
957
- memory,
958
- session: session.id,
959
- threads: workers
960
494
  };
961
- transaction.addLease(lease);
962
- transaction.removeWaiter(waiterId);
495
+ const decision = admit(input);
496
+ if (decision.kind === "grant") return grant(context, {
497
+ decision,
498
+ request
499
+ });
500
+ if (isWaiting) recordWaiter(context, {
501
+ now,
502
+ request
503
+ });
963
504
  return {
964
- kind: "grant",
965
- lease,
966
- memoryPerWorker: request.memoryPerWorker
505
+ kind: "wait",
506
+ ...waitLine({
507
+ ...input,
508
+ ...decision,
509
+ sessions: transaction.sessions()
510
+ })
967
511
  };
968
512
  }
969
513
  function attempt(pending) {
514
+ const survey = surveySessions(pending.session);
970
515
  const reading = pending.session.options.platform.readMemory();
971
516
  return pending.session.ledger.transact((transaction) => {
972
517
  return decide({
973
518
  ...pending,
974
519
  reading,
520
+ survey,
975
521
  transaction
976
522
  });
977
523
  });
@@ -1000,8 +546,13 @@ function readRunCap(session, request) {
1000
546
  }
1001
547
  function releaseLease({ lease, report, request, session }) {
1002
548
  if (!session.held.delete(lease.id)) return;
549
+ const survey = surveySessions(session);
1003
550
  session.ledger.transact((transaction) => {
1004
- const now = prune(session, transaction);
551
+ const now = session.options.platform.now();
552
+ prune(transaction, {
553
+ now,
554
+ survey
555
+ });
1005
556
  transaction.removeLease(lease.id);
1006
557
  if (report !== void 0 && request.cost !== void 0) {
1007
558
  const bytes = report.peakPerWorker ?? Math.round(report.peakMemory / lease.threads);
@@ -1029,6 +580,14 @@ function toLease({ outcome, request, session }) {
1029
580
  workers: lease.threads
1030
581
  };
1031
582
  }
583
+ function attemptUnlessLocked(pending) {
584
+ try {
585
+ return attempt(pending);
586
+ } catch (err) {
587
+ if (isLedgerLocked(err)) return { kind: "locked" };
588
+ throw err;
589
+ }
590
+ }
1032
591
  async function leaseAsync(session, request) {
1033
592
  const pending = {
1034
593
  isWaiting: true,
@@ -1036,19 +595,22 @@ async function leaseAsync(session, request) {
1036
595
  session,
1037
596
  waiterId: randomUUID()
1038
597
  };
598
+ let reprintKey;
1039
599
  for (let delay = FIRST_RETRY_MS;; delay = Math.min(delay * 2, LAST_RETRY_MS)) {
1040
- const outcome = attempt(pending);
600
+ const outcome = attemptUnlessLocked(pending);
1041
601
  if (outcome.kind === "grant") return toLease({
1042
602
  outcome,
1043
603
  request,
1044
604
  session
1045
605
  });
1046
- if (delay === FIRST_RETRY_MS) session.options.log(outcome.line);
606
+ if (outcome.kind === "wait" && outcome.reprintKey !== reprintKey) {
607
+ ({reprintKey} = outcome);
608
+ session.options.log(outcome.line);
609
+ }
1047
610
  await session.options.platform.sleepAsync(delay);
1048
611
  }
1049
612
  }
1050
- //#endregion
1051
- //#region src/tree-watch.ts
613
+ // src/tree-watch.ts
1052
614
  const TREE_SAMPLE_MS = 100;
1053
615
  const BLIND_WATCH = { stop: () => ({
1054
616
  peakMemory: void 0,
@@ -1076,8 +638,7 @@ function watchWith({ every, probe }) {
1076
638
  return result;
1077
639
  } };
1078
640
  }
1079
- //#endregion
1080
- //#region src/index.ts
641
+ // src/index.ts
1081
642
  /**
1082
643
  * Joins the budget that every dev tool on this machine shares. Close it when
1083
644
  * the run ends; a run that dies is taken back by the next run that leases.
@@ -1111,5 +672,4 @@ function watchTree(pid) {
1111
672
  probe: treeProbe(pid)
1112
673
  });
1113
674
  }
1114
- //#endregion
1115
675
  export { openMachineBudget, watchTree };