@isentinel/machine-budget 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -3
- package/bin/machine-budget-status.js +15 -0
- package/dist/admission-DcnnK4uq.d.mts +35 -0
- package/dist/index.d.mts +1 -21
- package/dist/index.mjs +233 -673
- package/dist/run-text-1kRifnCt.mjs +663 -0
- package/dist/status-cli.d.mts +53 -0
- package/dist/status-cli.mjs +51 -0
- package/package.json +5 -1
package/dist/index.mjs
CHANGED
|
@@ -1,588 +1,14 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { c as costKey, i as openSharedLedger, l as learnedCost, n as formatGb, o as treeProbe, r as nodePlatform, s as isLedgerLocked, t as describeRun } from "./run-text-1kRifnCt.mjs";
|
|
2
2
|
import process from "node:process";
|
|
3
|
-
import { execFileSync } from "node:child_process";
|
|
4
|
-
import { mkdirSync, readFileSync, readdirSync } from "node:fs";
|
|
5
|
-
import os from "node:os";
|
|
6
|
-
import path from "node:path";
|
|
7
|
-
import { setTimeout } from "node:timers/promises";
|
|
8
|
-
import { type } from "arktype";
|
|
9
|
-
import { DatabaseSync } from "node:sqlite";
|
|
10
|
-
import { Buffer } from "node:buffer";
|
|
11
3
|
import { randomUUID } from "node:crypto";
|
|
12
4
|
import { doNothing } from "remeda";
|
|
13
|
-
|
|
14
|
-
const
|
|
15
|
-
const PERCENTILE = .9;
|
|
16
|
-
/**
|
|
17
|
-
* Names a learned cost: the clone's git common directory, the project and the
|
|
18
|
-
* pass. All worktrees of one clone share the common directory, so they share
|
|
19
|
-
* costs.
|
|
20
|
-
*
|
|
21
|
-
* @param gitCommonDirectory - The git common directory of the clone.
|
|
22
|
-
* @param scope - The project and the pass.
|
|
23
|
-
* @returns The key of the cost in the ledger.
|
|
24
|
-
*/
|
|
25
|
-
function costKey(gitCommonDirectory, scope) {
|
|
26
|
-
return JSON.stringify([
|
|
27
|
-
gitCommonDirectory,
|
|
28
|
-
scope.project,
|
|
29
|
-
scope.pass
|
|
30
|
-
]);
|
|
31
|
-
}
|
|
32
|
-
/**
|
|
33
|
-
* Estimates the memory of one worker from the peaks of earlier runs.
|
|
34
|
-
*
|
|
35
|
-
* @param samples - Peak memory of one worker in earlier runs, in bytes.
|
|
36
|
-
* @returns About the 90th percentile, or undefined with too few samples.
|
|
37
|
-
*/
|
|
38
|
-
function learnedCost(samples) {
|
|
39
|
-
if (samples.length < MINIMUM_SAMPLES) return;
|
|
40
|
-
const sorted = samples.toSorted((left, right) => left - right);
|
|
41
|
-
return sorted[Math.ceil(sorted.length * PERCENTILE) - 1];
|
|
42
|
-
}
|
|
43
|
-
//#endregion
|
|
44
|
-
//#region src/ledger.ts
|
|
45
|
-
const LEDGER_FORMAT = 1;
|
|
46
|
-
const BUSY_TIMEOUT_MS = 3e4;
|
|
47
|
-
const LAST_SEEN = "last_seen_at";
|
|
48
|
-
const SCHEMA = `
|
|
49
|
-
CREATE TABLE IF NOT EXISTS sessions (
|
|
50
|
-
id TEXT PRIMARY KEY,
|
|
51
|
-
pid INTEGER NOT NULL,
|
|
52
|
-
started_at INTEGER NOT NULL,
|
|
53
|
-
heartbeat_at INTEGER NOT NULL
|
|
54
|
-
);
|
|
55
|
-
CREATE TABLE IF NOT EXISTS leases (
|
|
56
|
-
id TEXT PRIMARY KEY,
|
|
57
|
-
session TEXT NOT NULL,
|
|
58
|
-
lane TEXT NOT NULL,
|
|
59
|
-
memory INTEGER NOT NULL,
|
|
60
|
-
threads INTEGER NOT NULL
|
|
61
|
-
);
|
|
62
|
-
CREATE TABLE IF NOT EXISTS waiters (
|
|
63
|
-
id TEXT PRIMARY KEY,
|
|
64
|
-
session TEXT NOT NULL,
|
|
65
|
-
is_big INTEGER NOT NULL,
|
|
66
|
-
since INTEGER NOT NULL
|
|
67
|
-
);
|
|
68
|
-
CREATE TABLE IF NOT EXISTS costs (
|
|
69
|
-
key TEXT NOT NULL,
|
|
70
|
-
at INTEGER NOT NULL,
|
|
71
|
-
bytes INTEGER NOT NULL
|
|
72
|
-
);
|
|
73
|
-
CREATE INDEX IF NOT EXISTS costs_by_key ON costs (key, at);
|
|
74
|
-
CREATE TABLE IF NOT EXISTS meta (
|
|
75
|
-
key TEXT PRIMARY KEY,
|
|
76
|
-
value INTEGER NOT NULL
|
|
77
|
-
);
|
|
78
|
-
`;
|
|
79
|
-
const sessionRows = type({
|
|
80
|
-
id: "string",
|
|
81
|
-
heartbeatAt: "number",
|
|
82
|
-
pid: "number",
|
|
83
|
-
startedAt: "number"
|
|
84
|
-
}).array();
|
|
85
|
-
const leaseRows = type({
|
|
86
|
-
id: "string",
|
|
87
|
-
lane: "'main' | 'small'",
|
|
88
|
-
memory: "number",
|
|
89
|
-
session: "string",
|
|
90
|
-
threads: "number"
|
|
91
|
-
}).array();
|
|
92
|
-
const waiterRows = type({
|
|
93
|
-
id: "string",
|
|
94
|
-
isBig: type("0 | 1").pipe((bit) => bit === 1),
|
|
95
|
-
session: "string",
|
|
96
|
-
since: "number"
|
|
97
|
-
}).array();
|
|
98
|
-
const valueRows = type({ value: "number" }).array();
|
|
99
|
-
/**
|
|
100
|
-
* Names the ledger file in a directory. A new format takes a new file, so an
|
|
101
|
-
* older client never reads a format it does not know.
|
|
102
|
-
*
|
|
103
|
-
* @param directory - Where the machine keeps its ledger.
|
|
104
|
-
* @returns The path of the ledger file.
|
|
105
|
-
*/
|
|
106
|
-
function ledgerPath(directory) {
|
|
107
|
-
return path.join(directory, `ledger-v${LEDGER_FORMAT}.sqlite`);
|
|
108
|
-
}
|
|
109
|
-
/**
|
|
110
|
-
* Opens the ledger, and makes its tables when they do not exist. The
|
|
111
|
-
* directory of the file must exist.
|
|
112
|
-
*
|
|
113
|
-
* @param file - A path from `ledgerPath`, or `:memory:` for a private one.
|
|
114
|
-
* @returns A handle that runs transactions on the file.
|
|
115
|
-
*/
|
|
116
|
-
function openLedger(file) {
|
|
117
|
-
const database = openDatabase(file);
|
|
118
|
-
function query(sql) {
|
|
119
|
-
return database.prepare(sql);
|
|
120
|
-
}
|
|
121
|
-
const transaction = {
|
|
122
|
-
...metaTable(query),
|
|
123
|
-
...sessionTable(query),
|
|
124
|
-
...leaseTable(query),
|
|
125
|
-
...waiterTable(query),
|
|
126
|
-
...costTable(query)
|
|
127
|
-
};
|
|
128
|
-
return {
|
|
129
|
-
close: () => {
|
|
130
|
-
database.close();
|
|
131
|
-
},
|
|
132
|
-
transact: (work) => {
|
|
133
|
-
database.exec("BEGIN IMMEDIATE");
|
|
134
|
-
try {
|
|
135
|
-
const result = work(transaction);
|
|
136
|
-
database.exec("COMMIT");
|
|
137
|
-
return result;
|
|
138
|
-
} catch (err) {
|
|
139
|
-
database.exec("ROLLBACK");
|
|
140
|
-
throw err;
|
|
141
|
-
}
|
|
142
|
-
}
|
|
143
|
-
};
|
|
144
|
-
}
|
|
145
|
-
function openDatabase(file) {
|
|
146
|
-
const database = new DatabaseSync(file, { timeout: BUSY_TIMEOUT_MS });
|
|
147
|
-
database.exec("PRAGMA journal_mode = WAL");
|
|
148
|
-
database.exec(SCHEMA);
|
|
149
|
-
return database;
|
|
150
|
-
}
|
|
151
|
-
function parseRows(schema, rows) {
|
|
152
|
-
const parsed = schema(rows.map((row) => ({ ...row })));
|
|
153
|
-
if (parsed instanceof type.errors) throw new Error(`machine budget ledger holds an invalid row: ${parsed.summary}`);
|
|
154
|
-
return parsed;
|
|
155
|
-
}
|
|
156
|
-
function metaTable(query) {
|
|
157
|
-
return {
|
|
158
|
-
lastSeenAt: () => {
|
|
159
|
-
const rows = query("SELECT value FROM meta WHERE key = ?").all(LAST_SEEN);
|
|
160
|
-
return parseRows(valueRows, rows)[0]?.value;
|
|
161
|
-
},
|
|
162
|
-
markSeen: (at) => {
|
|
163
|
-
query("INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT (key) DO UPDATE SET value = excluded.value").run(LAST_SEEN, at);
|
|
164
|
-
}
|
|
165
|
-
};
|
|
166
|
-
}
|
|
167
|
-
function sessionTable(query) {
|
|
168
|
-
return {
|
|
169
|
-
refreshHeartbeats: (at) => {
|
|
170
|
-
query("UPDATE sessions SET heartbeat_at = ?").run(at);
|
|
171
|
-
},
|
|
172
|
-
removeSessions: (ids) => {
|
|
173
|
-
for (const id of ids) {
|
|
174
|
-
query("DELETE FROM leases WHERE session = ?").run(id);
|
|
175
|
-
query("DELETE FROM waiters WHERE session = ?").run(id);
|
|
176
|
-
query("DELETE FROM sessions WHERE id = ?").run(id);
|
|
177
|
-
}
|
|
178
|
-
},
|
|
179
|
-
sessions: () => {
|
|
180
|
-
const rows = query("SELECT id, heartbeat_at AS heartbeatAt, pid, started_at AS startedAt FROM sessions").all();
|
|
181
|
-
return parseRows(sessionRows, rows);
|
|
182
|
-
},
|
|
183
|
-
upsertSession: ({ id, heartbeatAt, pid, startedAt }) => {
|
|
184
|
-
query("INSERT INTO sessions (id, pid, started_at, heartbeat_at) VALUES (?, ?, ?, ?) ON CONFLICT (id) DO UPDATE SET heartbeat_at = excluded.heartbeat_at").run(id, pid, startedAt, heartbeatAt);
|
|
185
|
-
}
|
|
186
|
-
};
|
|
187
|
-
}
|
|
188
|
-
function leaseTable(query) {
|
|
189
|
-
return {
|
|
190
|
-
addLease: ({ id, lane, memory, session, threads }) => {
|
|
191
|
-
query("INSERT OR IGNORE INTO leases (id, session, lane, memory, threads) VALUES (?, ?, ?, ?, ?)").run(id, session, lane, memory, threads);
|
|
192
|
-
},
|
|
193
|
-
leases: () => {
|
|
194
|
-
const rows = query("SELECT id, lane, memory, session, threads FROM leases").all();
|
|
195
|
-
return parseRows(leaseRows, rows);
|
|
196
|
-
},
|
|
197
|
-
removeLease: (id) => {
|
|
198
|
-
query("DELETE FROM leases WHERE id = ?").run(id);
|
|
199
|
-
}
|
|
200
|
-
};
|
|
201
|
-
}
|
|
202
|
-
function waiterTable(query) {
|
|
203
|
-
return {
|
|
204
|
-
removeWaiter: (id) => {
|
|
205
|
-
query("DELETE FROM waiters WHERE id = ?").run(id);
|
|
206
|
-
},
|
|
207
|
-
upsertWaiter: ({ id, isBig, session, since }) => {
|
|
208
|
-
query("INSERT OR IGNORE INTO waiters (id, session, is_big, since) VALUES (?, ?, ?, ?)").run(id, session, isBig ? 1 : 0, since);
|
|
209
|
-
},
|
|
210
|
-
waiters: () => {
|
|
211
|
-
const rows = query("SELECT id, is_big AS isBig, session, since FROM waiters").all();
|
|
212
|
-
return parseRows(waiterRows, rows);
|
|
213
|
-
}
|
|
214
|
-
};
|
|
215
|
-
}
|
|
216
|
-
function costTable(query) {
|
|
217
|
-
return {
|
|
218
|
-
addSample: ({ key, at, bytes }) => {
|
|
219
|
-
query("INSERT INTO costs (key, at, bytes) VALUES (?, ?, ?)").run(key, at, bytes);
|
|
220
|
-
query("DELETE FROM costs WHERE key = ? AND rowid NOT IN (SELECT rowid FROM costs WHERE key = ? ORDER BY at DESC, rowid DESC LIMIT ?)").run(key, key, 20);
|
|
221
|
-
},
|
|
222
|
-
costSamples: (key) => {
|
|
223
|
-
const rows = query("SELECT bytes AS value FROM costs WHERE key = ? ORDER BY at DESC, rowid DESC").all(key);
|
|
224
|
-
return parseRows(valueRows, rows).map((row) => row.value);
|
|
225
|
-
}
|
|
226
|
-
};
|
|
227
|
-
}
|
|
228
|
-
//#endregion
|
|
229
|
-
//#region src/memory-probe.ts
|
|
230
|
-
const PAGE_SIZE = /page size of (\d+) bytes/;
|
|
231
|
-
const AVAILABLE_PAGES = /* @__PURE__ */ new Set([
|
|
232
|
-
"Pages free",
|
|
233
|
-
"Pages inactive",
|
|
234
|
-
"Pages purgeable"
|
|
235
|
-
]);
|
|
236
|
-
const PRESSURE_LEVELS = /* @__PURE__ */ new Map([
|
|
237
|
-
["1", "normal"],
|
|
238
|
-
["2", "warn"],
|
|
239
|
-
["4", "critical"]
|
|
240
|
-
]);
|
|
241
|
-
/**
|
|
242
|
-
* Reads the memory that can be given now. MacOS counts free, inactive and
|
|
243
|
-
* purgeable pages, and reports its pressure level; Windows and Linux report
|
|
244
|
-
* free memory, which on Linux is already the kernel's available estimate.
|
|
245
|
-
*
|
|
246
|
-
* @param sources - The operating system and how to read it.
|
|
247
|
-
* @returns The available memory and the pressure level.
|
|
248
|
-
*/
|
|
249
|
-
function readMemory(sources) {
|
|
250
|
-
if (sources.platform !== "darwin") return {
|
|
251
|
-
availableMemory: sources.freeMemory(),
|
|
252
|
-
pressure: "normal"
|
|
253
|
-
};
|
|
254
|
-
const vmStat = sources.run("vm_stat", []);
|
|
255
|
-
const level = sources.run("sysctl", ["-n", "kern.memorystatus_vm_pressure_level"]);
|
|
256
|
-
return {
|
|
257
|
-
availableMemory: (vmStat === void 0 ? void 0 : parseVmStat(vmStat)) ?? sources.freeMemory(),
|
|
258
|
-
pressure: parsePressure(level)
|
|
259
|
-
};
|
|
260
|
-
}
|
|
261
|
-
function parsePressure(level) {
|
|
262
|
-
if (level === void 0) return "normal";
|
|
263
|
-
return PRESSURE_LEVELS.get(level.trim()) ?? "normal";
|
|
264
|
-
}
|
|
265
|
-
function parseVmStat(output) {
|
|
266
|
-
const pageSize = PAGE_SIZE.exec(output)?.[1];
|
|
267
|
-
if (pageSize === void 0) return;
|
|
268
|
-
let pages = 0;
|
|
269
|
-
for (const line of output.split("\n")) {
|
|
270
|
-
const separator = line.indexOf(":");
|
|
271
|
-
if (AVAILABLE_PAGES.has(line.slice(0, separator))) pages += Number.parseInt(line.slice(separator + 1), 10);
|
|
272
|
-
}
|
|
273
|
-
return pages * Number(pageSize);
|
|
274
|
-
}
|
|
275
|
-
//#endregion
|
|
276
|
-
//#region src/tree-linux.ts
|
|
277
|
-
const BYTES_PER_KB = 1024;
|
|
278
|
-
const PARENT_FIELD = 1;
|
|
279
|
-
const PEAK_RESIDENT = /^VmHWM:\s+(\d+) kB$/m;
|
|
280
|
-
const RESIDENT = /^VmRSS:\s+(\d+) kB$/m;
|
|
281
|
-
/**
|
|
282
|
-
* Reads a process tree from `/proc`. Linux keeps no peak commit, so the peak
|
|
283
|
-
* of a descendant is its peak resident set, and the peak of the tree is the
|
|
284
|
-
* highest sum of resident sets over the samples.
|
|
285
|
-
*
|
|
286
|
-
* @param root - The pid at the top of the tree.
|
|
287
|
-
* @param source - How to read `/proc`.
|
|
288
|
-
* @returns The probe, which keeps the peaks of descendants that have exited.
|
|
289
|
-
*/
|
|
290
|
-
function linuxTreeProbe(root, source) {
|
|
291
|
-
const peaks = /* @__PURE__ */ new Map();
|
|
292
|
-
let treePeak;
|
|
293
|
-
return {
|
|
294
|
-
descendants: () => peaks,
|
|
295
|
-
sample: () => {
|
|
296
|
-
let resident = 0;
|
|
297
|
-
let isSeen = false;
|
|
298
|
-
for (const pid of [root, ...descendantsOf(source, root)]) {
|
|
299
|
-
const status = source.read(`/proc/${pid}/status`);
|
|
300
|
-
if (status === void 0) continue;
|
|
301
|
-
isSeen = true;
|
|
302
|
-
resident += bytes(status, RESIDENT);
|
|
303
|
-
if (pid !== root) peaks.set(pid, Math.max(peaks.get(pid) ?? 0, bytes(status, PEAK_RESIDENT)));
|
|
304
|
-
}
|
|
305
|
-
if (isSeen) treePeak = Math.max(treePeak ?? 0, resident);
|
|
306
|
-
},
|
|
307
|
-
tree: () => treePeak
|
|
308
|
-
};
|
|
309
|
-
}
|
|
310
|
-
function parentOf(source, pid) {
|
|
311
|
-
const stat = source.read(`/proc/${pid}/stat`);
|
|
312
|
-
if (stat === void 0) return;
|
|
313
|
-
return Number(stat.slice(stat.lastIndexOf(")") + 2).split(" ", 2)[PARENT_FIELD]);
|
|
314
|
-
}
|
|
315
|
-
function descendantsOf(source, root) {
|
|
316
|
-
const children = /* @__PURE__ */ new Map();
|
|
317
|
-
for (const pid of source.pids()) {
|
|
318
|
-
const parent = parentOf(source, pid);
|
|
319
|
-
if (parent !== void 0) children.set(parent, [...children.get(parent) ?? [], pid]);
|
|
320
|
-
}
|
|
321
|
-
const found = [];
|
|
322
|
-
const pending = [root];
|
|
323
|
-
for (let pid = pending.pop(); pid !== void 0; pid = pending.pop()) {
|
|
324
|
-
const next = children.get(pid) ?? [];
|
|
325
|
-
found.push(...next);
|
|
326
|
-
pending.push(...next);
|
|
327
|
-
}
|
|
328
|
-
return found;
|
|
329
|
-
}
|
|
330
|
-
function bytes(status, field) {
|
|
331
|
-
return Number(field.exec(status)?.[1] ?? 0) * BYTES_PER_KB;
|
|
332
|
-
}
|
|
333
|
-
//#endregion
|
|
334
|
-
//#region src/tree-windows.ts
|
|
335
|
-
const QUERY_LIMITED_ACCESS = 4096;
|
|
336
|
-
const BASIC_PROCESS_ID_LIST = 3;
|
|
337
|
-
const EXTENDED_LIMIT_INFORMATION = 9;
|
|
338
|
-
const EXTENDED_LIMIT_SIZE = 144;
|
|
339
|
-
const PEAK_JOB_MEMORY_OFFSET = 136;
|
|
340
|
-
const LISTED_COUNT_OFFSET = 4;
|
|
341
|
-
const PROCESS_ID_LIST_HEADER = 8;
|
|
342
|
-
const PROCESS_ID_SIZE = 8;
|
|
343
|
-
const MEMORY_COUNTERS_SIZE = 72;
|
|
344
|
-
const PEAK_PAGEFILE_OFFSET = 64;
|
|
345
|
-
/**
|
|
346
|
-
* Puts the root in a new Job Object. The tree peak is the job's peak commit.
|
|
347
|
-
* A descendant's peak is its peak pagefile usage, read through a handle that
|
|
348
|
-
* opens when a sample first lists it, so the figure outlives the process.
|
|
349
|
-
*
|
|
350
|
-
* @param root - The pid at the top of the tree.
|
|
351
|
-
* @returns The probe, or undefined when koffi does not load or the root cannot
|
|
352
|
-
* join a job.
|
|
353
|
-
*/
|
|
354
|
-
function windowsTreeProbe(root) {
|
|
355
|
-
let kernel32;
|
|
356
|
-
try {
|
|
357
|
-
kernel32 = loadKernel32();
|
|
358
|
-
} catch {
|
|
359
|
-
return;
|
|
360
|
-
}
|
|
361
|
-
const job = attach(kernel32, root);
|
|
362
|
-
if (job === null) return;
|
|
363
|
-
const state = {
|
|
364
|
-
handles: /* @__PURE__ */ new Map(),
|
|
365
|
-
job,
|
|
366
|
-
kernel32,
|
|
367
|
-
root
|
|
368
|
-
};
|
|
369
|
-
return {
|
|
370
|
-
close: () => {
|
|
371
|
-
closeJob(state);
|
|
372
|
-
},
|
|
373
|
-
descendants: () => descendantPeaks(state),
|
|
374
|
-
sample: () => {
|
|
375
|
-
openNewDescendants(state);
|
|
376
|
-
},
|
|
377
|
-
tree: () => jobPeak(state)
|
|
378
|
-
};
|
|
379
|
-
}
|
|
380
|
-
function loadKernel32() {
|
|
381
|
-
const library = createRequire(import.meta.url)("koffi").load("kernel32.dll");
|
|
382
|
-
return {
|
|
383
|
-
assign: library.func("bool AssignProcessToJobObject(void *job, void *process)"),
|
|
384
|
-
close: library.func("bool CloseHandle(void *handle)"),
|
|
385
|
-
createJob: library.func("void *CreateJobObjectW(void *attributes, void *name)"),
|
|
386
|
-
open: library.func("void *OpenProcess(uint32_t access, bool inherit, uint32_t pid)"),
|
|
387
|
-
queryJob: library.func("bool QueryInformationJobObject(void *job, int infoClass, _Out_ uint8_t *info, uint32_t size, void *returned)"),
|
|
388
|
-
readCounters: library.func("bool K32GetProcessMemoryInfo(void *process, _Out_ uint8_t *counters, uint32_t size)")
|
|
389
|
-
};
|
|
390
|
-
}
|
|
391
|
-
function attach(kernel32, root) {
|
|
392
|
-
const process = kernel32.open(257, false, root);
|
|
393
|
-
if (process === null) return null;
|
|
394
|
-
const job = kernel32.createJob(null, null);
|
|
395
|
-
const isAssigned = job !== null && kernel32.assign(job, process) === true;
|
|
396
|
-
kernel32.close(process);
|
|
397
|
-
if (!isAssigned && job !== null) kernel32.close(job);
|
|
398
|
-
return isAssigned ? job : null;
|
|
399
|
-
}
|
|
400
|
-
function listedProcesses({ job, kernel32 }) {
|
|
401
|
-
const list = Buffer.alloc(8200);
|
|
402
|
-
if (kernel32.queryJob(job, BASIC_PROCESS_ID_LIST, list, list.length, null) !== true) return [];
|
|
403
|
-
return Array.from({ length: list.readUInt32LE(LISTED_COUNT_OFFSET) }, (_, index) => {
|
|
404
|
-
return Number(list.readBigUInt64LE(PROCESS_ID_LIST_HEADER + index * PROCESS_ID_SIZE));
|
|
405
|
-
});
|
|
406
|
-
}
|
|
407
|
-
function openNewDescendants(state) {
|
|
408
|
-
for (const pid of listedProcesses(state)) {
|
|
409
|
-
if (pid === state.root || state.handles.has(pid)) continue;
|
|
410
|
-
const handle = state.kernel32.open(QUERY_LIMITED_ACCESS, false, pid);
|
|
411
|
-
if (handle !== null) state.handles.set(pid, handle);
|
|
412
|
-
}
|
|
413
|
-
}
|
|
414
|
-
function descendantPeaks({ handles, kernel32 }) {
|
|
415
|
-
const counters = Buffer.alloc(MEMORY_COUNTERS_SIZE);
|
|
416
|
-
const peaks = /* @__PURE__ */ new Map();
|
|
417
|
-
for (const [pid, handle] of handles) if (kernel32.readCounters(handle, counters, counters.length) === true) peaks.set(pid, Number(counters.readBigUInt64LE(PEAK_PAGEFILE_OFFSET)));
|
|
418
|
-
return peaks;
|
|
419
|
-
}
|
|
420
|
-
function jobPeak({ job, kernel32 }) {
|
|
421
|
-
const limits = Buffer.alloc(EXTENDED_LIMIT_SIZE);
|
|
422
|
-
if (kernel32.queryJob(job, EXTENDED_LIMIT_INFORMATION, limits, limits.length, null) !== true) return;
|
|
423
|
-
return Number(limits.readBigUInt64LE(PEAK_JOB_MEMORY_OFFSET));
|
|
424
|
-
}
|
|
425
|
-
function closeJob({ handles, job, kernel32 }) {
|
|
426
|
-
for (const handle of handles.values()) kernel32.close(handle);
|
|
427
|
-
handles.clear();
|
|
428
|
-
kernel32.close(job);
|
|
429
|
-
}
|
|
430
|
-
//#endregion
|
|
431
|
-
//#region src/adapters.ts
|
|
432
|
-
const COMMAND_TIMEOUT_MS = 5e3;
|
|
433
|
-
const LINUX_CLOCK_TICKS_PER_SECOND = 100;
|
|
434
|
-
const LINUX_START_TIME_FIELD = 19;
|
|
435
|
-
const BOOT_TIME = /^btime (\d+)$/m;
|
|
436
|
-
const PID_NAME = /^\d+$/;
|
|
437
|
-
/**
|
|
438
|
-
* Opens the ledger of this machine in the directory that `ledgerDirectory`
|
|
439
|
-
* names for this process, and makes the directory when it does not exist.
|
|
440
|
-
*
|
|
441
|
-
* @returns The ledger that every run of this user on the machine opens.
|
|
442
|
-
*/
|
|
443
|
-
function openSharedLedger() {
|
|
444
|
-
const directory = ledgerDirectory(process.env);
|
|
445
|
-
mkdirSync(directory, { recursive: true });
|
|
446
|
-
return openLedger(ledgerPath(directory));
|
|
447
|
-
}
|
|
448
|
-
function ledgerDirectory(environment) {
|
|
449
|
-
const configured = environment["MACHINE_BUDGET_DIR"];
|
|
450
|
-
if (configured !== void 0 && configured !== "") return configured;
|
|
451
|
-
return path.join(os.tmpdir(), `machine-budget-${os.userInfo().username}`);
|
|
452
|
-
}
|
|
453
|
-
function constrainedMemory() {
|
|
454
|
-
return process.constrainedMemory?.() ?? 0;
|
|
455
|
-
}
|
|
456
|
-
function run(command, args) {
|
|
457
|
-
try {
|
|
458
|
-
return execFileSync(command, args, {
|
|
459
|
-
encoding: "utf8",
|
|
460
|
-
stdio: [
|
|
461
|
-
"ignore",
|
|
462
|
-
"pipe",
|
|
463
|
-
"ignore"
|
|
464
|
-
],
|
|
465
|
-
timeout: COMMAND_TIMEOUT_MS,
|
|
466
|
-
windowsHide: true
|
|
467
|
-
});
|
|
468
|
-
} catch {
|
|
469
|
-
return;
|
|
470
|
-
}
|
|
471
|
-
}
|
|
472
|
-
function isAlive(pid) {
|
|
473
|
-
try {
|
|
474
|
-
process.kill(pid, 0);
|
|
475
|
-
return true;
|
|
476
|
-
} catch (err) {
|
|
477
|
-
return !(err instanceof Error) || !("code" in err) || err.code !== "ESRCH";
|
|
478
|
-
}
|
|
479
|
-
}
|
|
480
|
-
function linuxStartTime(pid) {
|
|
481
|
-
try {
|
|
482
|
-
const stat = readFileSync(`/proc/${pid}/stat`, "utf8");
|
|
483
|
-
const fields = stat.slice(stat.lastIndexOf(")") + 2).split(" ");
|
|
484
|
-
const bootTime = BOOT_TIME.exec(readFileSync("/proc/stat", "utf8"))?.[1];
|
|
485
|
-
const ticks = fields[LINUX_START_TIME_FIELD];
|
|
486
|
-
if (bootTime === void 0 || ticks === void 0) return;
|
|
487
|
-
return (Number(bootTime) + Number(ticks) / LINUX_CLOCK_TICKS_PER_SECOND) * 1e3;
|
|
488
|
-
} catch {
|
|
489
|
-
return;
|
|
490
|
-
}
|
|
491
|
-
}
|
|
492
|
-
function parseTime(text) {
|
|
493
|
-
const time = text === void 0 ? NaN : Date.parse(text.trim());
|
|
494
|
-
return Number.isNaN(time) ? void 0 : time;
|
|
495
|
-
}
|
|
496
|
-
function windowsStartTime(pid) {
|
|
497
|
-
return parseTime(run("powershell.exe", [
|
|
498
|
-
"-NoProfile",
|
|
499
|
-
"-NonInteractive",
|
|
500
|
-
"-Command",
|
|
501
|
-
`(Get-Process -Id ${pid}).StartTime.ToUniversalTime().ToString('o')`
|
|
502
|
-
]));
|
|
503
|
-
}
|
|
504
|
-
const START_TIME_READERS = {
|
|
505
|
-
darwin: (pid) => parseTime(run("ps", [
|
|
506
|
-
"-o",
|
|
507
|
-
"lstart=",
|
|
508
|
-
"-p",
|
|
509
|
-
String(pid)
|
|
510
|
-
])),
|
|
511
|
-
linux: linuxStartTime,
|
|
512
|
-
win32: windowsStartTime
|
|
513
|
-
};
|
|
514
|
-
function startTime(pid) {
|
|
515
|
-
return START_TIME_READERS[process.platform]?.(pid);
|
|
516
|
-
}
|
|
517
|
-
/** The real machine and process. */
|
|
518
|
-
const nodePlatform = {
|
|
519
|
-
every: (milliseconds, work) => {
|
|
520
|
-
const timer = setInterval(work, milliseconds);
|
|
521
|
-
timer.unref();
|
|
522
|
-
return () => {
|
|
523
|
-
clearInterval(timer);
|
|
524
|
-
};
|
|
525
|
-
},
|
|
526
|
-
gitCommonDirectory: (workspaceRoot) => {
|
|
527
|
-
const output = run("git", [
|
|
528
|
-
"-C",
|
|
529
|
-
workspaceRoot,
|
|
530
|
-
"rev-parse",
|
|
531
|
-
"--path-format=absolute",
|
|
532
|
-
"--git-common-dir"
|
|
533
|
-
]);
|
|
534
|
-
return output === void 0 ? workspaceRoot : output.trim();
|
|
535
|
-
},
|
|
536
|
-
now: Date.now,
|
|
537
|
-
pid: process.pid,
|
|
538
|
-
probe: {
|
|
539
|
-
isAlive,
|
|
540
|
-
startTime
|
|
541
|
-
},
|
|
542
|
-
readMemory: () => readMemory({
|
|
543
|
-
freeMemory: os.freemem,
|
|
544
|
-
platform: process.platform,
|
|
545
|
-
run
|
|
546
|
-
}),
|
|
547
|
-
sleepAsync: async (milliseconds) => {
|
|
548
|
-
await setTimeout(milliseconds);
|
|
549
|
-
},
|
|
550
|
-
startedAt: Date.now() - process.uptime() * 1e3,
|
|
551
|
-
threads: os.availableParallelism(),
|
|
552
|
-
totalMemory: Math.min(os.totalmem(), constrainedMemory() || Infinity)
|
|
553
|
-
};
|
|
554
|
-
/**
|
|
555
|
-
* Chooses how this platform reads a process tree: a Job Object on Windows,
|
|
556
|
-
* `/proc` on Linux, and nothing elsewhere.
|
|
557
|
-
*
|
|
558
|
-
* @param pid - The pid at the top of the tree.
|
|
559
|
-
* @returns The probe of this platform, or undefined where it cannot read one.
|
|
560
|
-
*/
|
|
561
|
-
function treeProbe(pid) {
|
|
562
|
-
if (process.platform === "win32") return windowsTreeProbe(pid);
|
|
563
|
-
return process.platform === "linux" ? linuxTreeProbe(pid, {
|
|
564
|
-
pids: procPids,
|
|
565
|
-
read: readProc
|
|
566
|
-
}) : void 0;
|
|
567
|
-
}
|
|
568
|
-
function readProc(file) {
|
|
569
|
-
try {
|
|
570
|
-
return readFileSync(file, "utf8");
|
|
571
|
-
} catch {
|
|
572
|
-
return;
|
|
573
|
-
}
|
|
574
|
-
}
|
|
575
|
-
function procPids() {
|
|
576
|
-
return readdirSync("/proc").filter((name) => PID_NAME.test(name)).map(Number);
|
|
577
|
-
}
|
|
578
|
-
//#endregion
|
|
579
|
-
//#region src/admission.ts
|
|
580
|
-
const BYTES_PER_GB$1 = 2 ** 30;
|
|
5
|
+
// src/admission.ts
|
|
6
|
+
const BYTES_PER_GB = 2 ** 30;
|
|
581
7
|
const MAIN_MEMORY_SHARE = .5;
|
|
582
8
|
const RESERVE_SHARE = .15;
|
|
583
9
|
const LANE_MEMORY_SHARE = .15;
|
|
584
10
|
const LANE_THREADS = 2;
|
|
585
|
-
const SMALL_REQUEST_BYTES = 2 * BYTES_PER_GB
|
|
11
|
+
const SMALL_REQUEST_BYTES = 2 * BYTES_PER_GB;
|
|
586
12
|
const AGING_MS = 6e4;
|
|
587
13
|
/**
|
|
588
14
|
* Whether a request takes the small-job lane when the main lane is full.
|
|
@@ -594,6 +20,17 @@ function isSmallRequest(request) {
|
|
|
594
20
|
return request.minWorkers * request.memoryPerWorker <= SMALL_REQUEST_BYTES;
|
|
595
21
|
}
|
|
596
22
|
/**
|
|
23
|
+
* The oldest big request that has waited past the aging limit and started
|
|
24
|
+
* before this one, which every younger request waits behind.
|
|
25
|
+
*
|
|
26
|
+
* @param input - The time, this request's waiter id, and every waiter.
|
|
27
|
+
* @returns The aged waiter, or undefined when none blocks this request.
|
|
28
|
+
*/
|
|
29
|
+
function olderAgedWaiter({ now, waiterId, waiters }) {
|
|
30
|
+
const since = waiters.find((waiter) => waiter.id === waiterId)?.since ?? now;
|
|
31
|
+
return waiters.filter((waiter) => waiter.isBig && waiter.since < since && now - waiter.since > AGING_MS).toSorted((a, b) => a.since - b.since)[0];
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
597
34
|
* Decides one request: the largest worker count that fits, or why it waits.
|
|
598
35
|
*
|
|
599
36
|
* @param input - The machine, the leases and waiters, and the request.
|
|
@@ -602,7 +39,7 @@ function isSmallRequest(request) {
|
|
|
602
39
|
function admit(input) {
|
|
603
40
|
const { leases, machine, request } = input;
|
|
604
41
|
const isBig = !isSmallRequest(request);
|
|
605
|
-
const isBehindAged =
|
|
42
|
+
const isBehindAged = olderAgedWaiter(input) !== void 0;
|
|
606
43
|
const mainLeases = leases.filter((lease) => lease.lane === "main");
|
|
607
44
|
if (isBig && machine.pressure !== "normal" && mainLeases.length > 0) return {
|
|
608
45
|
kind: "wait",
|
|
@@ -647,12 +84,6 @@ function runCapWorkers(input) {
|
|
|
647
84
|
const byMemory = Math.floor(limit.memory / runs / request.memoryPerWorker);
|
|
648
85
|
return Math.min(request.maxWorkers, byMemory, Math.floor(limit.threads / runs));
|
|
649
86
|
}
|
|
650
|
-
function hasOlderAgedWaiter({ now, waiterId, waiters }) {
|
|
651
|
-
const since = waiters.find((waiter) => waiter.id === waiterId)?.since ?? now;
|
|
652
|
-
return waiters.some((waiter) => {
|
|
653
|
-
return waiter.isBig && waiter.since < since && now - waiter.since > AGING_MS;
|
|
654
|
-
});
|
|
655
|
-
}
|
|
656
87
|
function activeRuns({ leases, session, waiters }) {
|
|
657
88
|
const sessions = /* @__PURE__ */ new Set([session]);
|
|
658
89
|
for (const lease of leases) if (lease.lane === "main") sessions.add(lease.session);
|
|
@@ -739,8 +170,7 @@ function fitWorkers(request, fit) {
|
|
|
739
170
|
workers
|
|
740
171
|
};
|
|
741
172
|
}
|
|
742
|
-
|
|
743
|
-
//#region src/reclaim.ts
|
|
173
|
+
// src/reclaim.ts
|
|
744
174
|
/** How often a live session stamps its heartbeat. */
|
|
745
175
|
const HEARTBEAT_INTERVAL_MS = 5e3;
|
|
746
176
|
const SUSPECT_MS = 3 * HEARTBEAT_INTERVAL_MS;
|
|
@@ -775,11 +205,140 @@ function isDead({ isHeartbeatSkipped, now, probe, session }) {
|
|
|
775
205
|
if (startedAt !== void 0 && Math.abs(startedAt - session.startedAt) > START_TOLERANCE_MS) return true;
|
|
776
206
|
return !isHeartbeatSkipped && age >= STALE_MS;
|
|
777
207
|
}
|
|
778
|
-
|
|
779
|
-
|
|
208
|
+
// src/prune.ts
|
|
209
|
+
/**
|
|
210
|
+
* Reads the sessions with no write lock, then probes their processes with no
|
|
211
|
+
* transaction open, so a slow probe never holds the ledger lock.
|
|
212
|
+
*
|
|
213
|
+
* @param surveyor - The session that surveys.
|
|
214
|
+
* @returns The sessions and the dead among them.
|
|
215
|
+
*/
|
|
216
|
+
function surveySessions({ id, ledger, options }) {
|
|
217
|
+
const { platform } = options;
|
|
218
|
+
const { lastSeenAt, sessions } = ledger.read((reader) => {
|
|
219
|
+
return {
|
|
220
|
+
lastSeenAt: reader.lastSeenAt(),
|
|
221
|
+
sessions: reader.sessions()
|
|
222
|
+
};
|
|
223
|
+
});
|
|
224
|
+
const now = platform.now();
|
|
225
|
+
return {
|
|
226
|
+
...findDeadSessions({
|
|
227
|
+
lastSeenAt,
|
|
228
|
+
now,
|
|
229
|
+
probe: platform.probe,
|
|
230
|
+
self: id,
|
|
231
|
+
sessions
|
|
232
|
+
}),
|
|
233
|
+
now,
|
|
234
|
+
sessions
|
|
235
|
+
};
|
|
236
|
+
}
|
|
237
|
+
/**
|
|
238
|
+
* Removes the dead sessions of a survey that no run changed since, and marks
|
|
239
|
+
* the ledger seen.
|
|
240
|
+
*
|
|
241
|
+
* @param transaction - The open write transaction.
|
|
242
|
+
* @param pruning - The survey and the time to write.
|
|
243
|
+
*/
|
|
244
|
+
function prune(transaction, { now, survey }) {
|
|
245
|
+
const { dead, isHeartbeatSkipped, sessions } = survey;
|
|
246
|
+
const current = new Map(transaction.sessions().map((record) => [record.id, record]));
|
|
247
|
+
const unchanged = sessions.filter((record) => {
|
|
248
|
+
const stored = current.get(record.id);
|
|
249
|
+
return dead.includes(record.id) && stored !== void 0 && isSameSession(record, stored);
|
|
250
|
+
});
|
|
251
|
+
transaction.removeSessions(unchanged.map((record) => record.id));
|
|
252
|
+
if (isHeartbeatSkipped) transaction.refreshHeartbeats(now);
|
|
253
|
+
transaction.markSeen(now);
|
|
254
|
+
}
|
|
255
|
+
/**
|
|
256
|
+
* Prunes, then puts the session and its leases back with a fresh heartbeat.
|
|
257
|
+
*
|
|
258
|
+
* @param transaction - The open write transaction.
|
|
259
|
+
* @param restorer - The session and its survey.
|
|
260
|
+
* @returns The time of the writes.
|
|
261
|
+
*/
|
|
262
|
+
function restore(transaction, { session, survey }) {
|
|
263
|
+
const { platform } = session.options;
|
|
264
|
+
const now = platform.now();
|
|
265
|
+
prune(transaction, {
|
|
266
|
+
now,
|
|
267
|
+
survey
|
|
268
|
+
});
|
|
269
|
+
transaction.upsertSession({
|
|
270
|
+
id: session.id,
|
|
271
|
+
heartbeatAt: now,
|
|
272
|
+
pid: platform.pid,
|
|
273
|
+
startedAt: platform.startedAt,
|
|
274
|
+
worktree: session.options.workspaceRoot
|
|
275
|
+
});
|
|
276
|
+
for (const lease of session.held.values()) transaction.addLease(lease);
|
|
277
|
+
return now;
|
|
278
|
+
}
|
|
279
|
+
/**
|
|
280
|
+
* Surveys the sessions, then restores the session in one write transaction.
|
|
281
|
+
*
|
|
282
|
+
* @param session - The session to restore.
|
|
283
|
+
*/
|
|
284
|
+
function restoreSession(session) {
|
|
285
|
+
const survey = surveySessions(session);
|
|
286
|
+
session.ledger.transact((transaction) => restore(transaction, {
|
|
287
|
+
session,
|
|
288
|
+
survey
|
|
289
|
+
}));
|
|
290
|
+
}
|
|
291
|
+
function isSameSession(left, right) {
|
|
292
|
+
return left.heartbeatAt === right.heartbeatAt && left.pid === right.pid && left.startedAt === right.startedAt;
|
|
293
|
+
}
|
|
294
|
+
// src/wait-line.ts
|
|
295
|
+
/**
|
|
296
|
+
* Writes the line of a waiting request: its size, its reason, and the runs in
|
|
297
|
+
* its way. Behind an older request, that is the run of the aged request;
|
|
298
|
+
* otherwise each other run that holds leases, the largest first.
|
|
299
|
+
*
|
|
300
|
+
* @param notice - The request, the reason and the ledger rows.
|
|
301
|
+
* @returns The line and its reprint key.
|
|
302
|
+
*/
|
|
303
|
+
function waitLine(notice) {
|
|
304
|
+
const { reason, request } = notice;
|
|
305
|
+
const need = formatGb(request.minWorkers * request.memoryPerWorker);
|
|
306
|
+
const { cause, runs } = causeOf(notice);
|
|
307
|
+
return {
|
|
308
|
+
line: `machine budget: waiting for ${need} and ${request.minWorkers} threads (${reason}); ${cause}; machine-budget-status prints the ledger`,
|
|
309
|
+
reprintKey: [reason, ...runs.toSorted()].join(" ")
|
|
310
|
+
};
|
|
311
|
+
}
|
|
312
|
+
function holdersOf({ leases, session: own, sessions }) {
|
|
313
|
+
return sessions.filter((session) => session.id !== own).map((session) => {
|
|
314
|
+
const owned = leases.filter((lease) => lease.session === session.id);
|
|
315
|
+
return {
|
|
316
|
+
memory: owned.reduce((total, lease) => total + lease.memory, 0),
|
|
317
|
+
session,
|
|
318
|
+
threads: owned.reduce((total, lease) => total + lease.threads, 0)
|
|
319
|
+
};
|
|
320
|
+
}).filter(({ threads }) => threads > 0).toSorted((a, b) => b.memory - a.memory);
|
|
321
|
+
}
|
|
322
|
+
function causeOf(notice) {
|
|
323
|
+
const { reason, sessions, waiters } = notice;
|
|
324
|
+
const aged = reason === "older request" ? olderAgedWaiter(notice) : void 0;
|
|
325
|
+
const blocker = waiters.find((waiter) => waiter.id === aged?.id);
|
|
326
|
+
if (blocker !== void 0) return {
|
|
327
|
+
cause: `behind ${describeRun(sessions.find((session) => session.id === blocker.session))} (${formatGb(blocker.memory)}, ${blocker.threads} threads)`,
|
|
328
|
+
runs: [blocker.id]
|
|
329
|
+
};
|
|
330
|
+
const holders = holdersOf(notice);
|
|
331
|
+
const parts = holders.map(({ memory, session, threads }) => {
|
|
332
|
+
return `${describeRun(session)} (${formatGb(memory)}, ${threads} threads)`;
|
|
333
|
+
});
|
|
334
|
+
return {
|
|
335
|
+
cause: parts.length === 0 ? "no run holds a lease" : `held by ${parts.join(", ")}`,
|
|
336
|
+
runs: holders.map(({ session }) => session.id)
|
|
337
|
+
};
|
|
338
|
+
}
|
|
339
|
+
// src/budget.ts
|
|
780
340
|
const FIRST_RETRY_MS = 250;
|
|
781
341
|
const LAST_RETRY_MS = 2e3;
|
|
782
|
-
const BYTES_PER_GB = 2 ** 30;
|
|
783
342
|
/**
|
|
784
343
|
* Joins the machine budget. Each lease is one row in the shared ledger until
|
|
785
344
|
* it is released, the run closes, or a later run finds the run dead.
|
|
@@ -791,9 +350,16 @@ function openBudget(options) {
|
|
|
791
350
|
if (isBudgetOff(options.environment)) return unlimitedBudget;
|
|
792
351
|
return sharedBudget(joinSession(options));
|
|
793
352
|
}
|
|
353
|
+
function heartbeat(session) {
|
|
354
|
+
try {
|
|
355
|
+
restoreSession(session);
|
|
356
|
+
} catch (err) {
|
|
357
|
+
session.options.log(`machine budget: heartbeat failed: ${String(err)}`);
|
|
358
|
+
}
|
|
359
|
+
}
|
|
794
360
|
function sharedBudget(session) {
|
|
795
361
|
const stop = session.options.platform.every(HEARTBEAT_INTERVAL_MS, () => {
|
|
796
|
-
|
|
362
|
+
heartbeat(session);
|
|
797
363
|
});
|
|
798
364
|
let isClosed = false;
|
|
799
365
|
return {
|
|
@@ -852,33 +418,6 @@ function assertRequest({ maxWorkers, memoryPerWorker, minWorkers }) {
|
|
|
852
418
|
if (!Number.isSafeInteger(minWorkers) || minWorkers < 1 || maxWorkers < minWorkers) throw new RangeError(`a lease needs 1 <= minWorkers <= maxWorkers, got ${minWorkers} and ${maxWorkers}`);
|
|
853
419
|
if (!Number.isSafeInteger(maxWorkers) || !Number.isFinite(memoryPerWorker) || memoryPerWorker <= 0) throw new RangeError(`a lease needs a whole maxWorkers and memoryPerWorker above 0, got ${maxWorkers} and ${memoryPerWorker}`);
|
|
854
420
|
}
|
|
855
|
-
function prune(session, transaction) {
|
|
856
|
-
const { platform } = session.options;
|
|
857
|
-
const now = platform.now();
|
|
858
|
-
const { dead, isHeartbeatSkipped } = findDeadSessions({
|
|
859
|
-
lastSeenAt: transaction.lastSeenAt(),
|
|
860
|
-
now,
|
|
861
|
-
probe: platform.probe,
|
|
862
|
-
self: session.id,
|
|
863
|
-
sessions: transaction.sessions()
|
|
864
|
-
});
|
|
865
|
-
transaction.removeSessions(dead);
|
|
866
|
-
if (isHeartbeatSkipped) transaction.refreshHeartbeats(now);
|
|
867
|
-
transaction.markSeen(now);
|
|
868
|
-
return now;
|
|
869
|
-
}
|
|
870
|
-
function restore(session, transaction) {
|
|
871
|
-
const { platform } = session.options;
|
|
872
|
-
const now = prune(session, transaction);
|
|
873
|
-
transaction.upsertSession({
|
|
874
|
-
id: session.id,
|
|
875
|
-
heartbeatAt: now,
|
|
876
|
-
pid: platform.pid,
|
|
877
|
-
startedAt: platform.startedAt
|
|
878
|
-
});
|
|
879
|
-
for (const lease of session.held.values()) transaction.addLease(lease);
|
|
880
|
-
return now;
|
|
881
|
-
}
|
|
882
421
|
function joinSession(options) {
|
|
883
422
|
const session = {
|
|
884
423
|
id: randomUUID(),
|
|
@@ -887,7 +426,7 @@ function joinSession(options) {
|
|
|
887
426
|
options,
|
|
888
427
|
repository: options.platform.gitCommonDirectory(options.workspaceRoot)
|
|
889
428
|
};
|
|
890
|
-
|
|
429
|
+
restoreSession(session);
|
|
891
430
|
return session;
|
|
892
431
|
}
|
|
893
432
|
function leaveSession(session) {
|
|
@@ -898,27 +437,50 @@ function leaveSession(session) {
|
|
|
898
437
|
}
|
|
899
438
|
function workerMemory({ request, session, transaction }) {
|
|
900
439
|
if (request.cost === void 0) return request.memoryPerWorker;
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
440
|
+
const samples = transaction.costSamples(costKey(session.repository, request.cost));
|
|
441
|
+
return learnedCost(samples) ?? request.memoryPerWorker;
|
|
442
|
+
}
|
|
443
|
+
function recordWaiter({ session, transaction, waiterId }, { now, request }) {
|
|
444
|
+
transaction.upsertWaiter({
|
|
445
|
+
id: waiterId,
|
|
446
|
+
isBig: !isSmallRequest(request),
|
|
447
|
+
memory: request.minWorkers * request.memoryPerWorker,
|
|
448
|
+
session: session.id,
|
|
449
|
+
since: now,
|
|
450
|
+
threads: request.minWorkers
|
|
451
|
+
});
|
|
905
452
|
}
|
|
906
|
-
function
|
|
907
|
-
const
|
|
908
|
-
const
|
|
909
|
-
|
|
453
|
+
function grant({ session, transaction, waiterId }, { decision, request }) {
|
|
454
|
+
const { lane, workers } = decision;
|
|
455
|
+
const memory = workers * request.memoryPerWorker;
|
|
456
|
+
const lease = {
|
|
457
|
+
id: randomUUID(),
|
|
458
|
+
lane,
|
|
459
|
+
memory,
|
|
460
|
+
session: session.id,
|
|
461
|
+
threads: workers
|
|
462
|
+
};
|
|
463
|
+
transaction.addLease(lease);
|
|
464
|
+
transaction.removeWaiter(waiterId);
|
|
465
|
+
return {
|
|
466
|
+
kind: "grant",
|
|
467
|
+
lease,
|
|
468
|
+
memoryPerWorker: request.memoryPerWorker
|
|
469
|
+
};
|
|
910
470
|
}
|
|
911
471
|
function decide(context) {
|
|
912
|
-
const { isWaiting, reading, session, transaction, waiterId } = context;
|
|
472
|
+
const { isWaiting, reading, session, survey, transaction, waiterId } = context;
|
|
913
473
|
const { platform } = session.options;
|
|
914
|
-
const now = restore(
|
|
474
|
+
const now = restore(transaction, {
|
|
475
|
+
session,
|
|
476
|
+
survey
|
|
477
|
+
});
|
|
915
478
|
const request = {
|
|
916
479
|
...context.request,
|
|
917
480
|
memoryPerWorker: workerMemory(context)
|
|
918
481
|
};
|
|
919
|
-
const
|
|
920
|
-
|
|
921
|
-
leases,
|
|
482
|
+
const input = {
|
|
483
|
+
leases: transaction.leases(),
|
|
922
484
|
machine: {
|
|
923
485
|
...reading,
|
|
924
486
|
threads: platform.threads,
|
|
@@ -929,49 +491,33 @@ function decide(context) {
|
|
|
929
491
|
session: session.id,
|
|
930
492
|
waiterId,
|
|
931
493
|
waiters: transaction.waiters()
|
|
932
|
-
});
|
|
933
|
-
if (decision.kind === "wait") {
|
|
934
|
-
if (isWaiting) {
|
|
935
|
-
const isBig = !isSmallRequest(request);
|
|
936
|
-
transaction.upsertWaiter({
|
|
937
|
-
id: waiterId,
|
|
938
|
-
isBig,
|
|
939
|
-
session: session.id,
|
|
940
|
-
since: now
|
|
941
|
-
});
|
|
942
|
-
}
|
|
943
|
-
return {
|
|
944
|
-
kind: "wait",
|
|
945
|
-
line: waitLine({
|
|
946
|
-
leases,
|
|
947
|
-
reason: decision.reason,
|
|
948
|
-
request
|
|
949
|
-
})
|
|
950
|
-
};
|
|
951
|
-
}
|
|
952
|
-
const { lane, workers } = decision;
|
|
953
|
-
const memory = workers * request.memoryPerWorker;
|
|
954
|
-
const lease = {
|
|
955
|
-
id: randomUUID(),
|
|
956
|
-
lane,
|
|
957
|
-
memory,
|
|
958
|
-
session: session.id,
|
|
959
|
-
threads: workers
|
|
960
494
|
};
|
|
961
|
-
|
|
962
|
-
|
|
495
|
+
const decision = admit(input);
|
|
496
|
+
if (decision.kind === "grant") return grant(context, {
|
|
497
|
+
decision,
|
|
498
|
+
request
|
|
499
|
+
});
|
|
500
|
+
if (isWaiting) recordWaiter(context, {
|
|
501
|
+
now,
|
|
502
|
+
request
|
|
503
|
+
});
|
|
963
504
|
return {
|
|
964
|
-
kind: "
|
|
965
|
-
|
|
966
|
-
|
|
505
|
+
kind: "wait",
|
|
506
|
+
...waitLine({
|
|
507
|
+
...input,
|
|
508
|
+
...decision,
|
|
509
|
+
sessions: transaction.sessions()
|
|
510
|
+
})
|
|
967
511
|
};
|
|
968
512
|
}
|
|
969
513
|
function attempt(pending) {
|
|
514
|
+
const survey = surveySessions(pending.session);
|
|
970
515
|
const reading = pending.session.options.platform.readMemory();
|
|
971
516
|
return pending.session.ledger.transact((transaction) => {
|
|
972
517
|
return decide({
|
|
973
518
|
...pending,
|
|
974
519
|
reading,
|
|
520
|
+
survey,
|
|
975
521
|
transaction
|
|
976
522
|
});
|
|
977
523
|
});
|
|
@@ -1000,8 +546,13 @@ function readRunCap(session, request) {
|
|
|
1000
546
|
}
|
|
1001
547
|
function releaseLease({ lease, report, request, session }) {
|
|
1002
548
|
if (!session.held.delete(lease.id)) return;
|
|
549
|
+
const survey = surveySessions(session);
|
|
1003
550
|
session.ledger.transact((transaction) => {
|
|
1004
|
-
const now =
|
|
551
|
+
const now = session.options.platform.now();
|
|
552
|
+
prune(transaction, {
|
|
553
|
+
now,
|
|
554
|
+
survey
|
|
555
|
+
});
|
|
1005
556
|
transaction.removeLease(lease.id);
|
|
1006
557
|
if (report !== void 0 && request.cost !== void 0) {
|
|
1007
558
|
const bytes = report.peakPerWorker ?? Math.round(report.peakMemory / lease.threads);
|
|
@@ -1029,6 +580,14 @@ function toLease({ outcome, request, session }) {
|
|
|
1029
580
|
workers: lease.threads
|
|
1030
581
|
};
|
|
1031
582
|
}
|
|
583
|
+
function attemptUnlessLocked(pending) {
|
|
584
|
+
try {
|
|
585
|
+
return attempt(pending);
|
|
586
|
+
} catch (err) {
|
|
587
|
+
if (isLedgerLocked(err)) return { kind: "locked" };
|
|
588
|
+
throw err;
|
|
589
|
+
}
|
|
590
|
+
}
|
|
1032
591
|
async function leaseAsync(session, request) {
|
|
1033
592
|
const pending = {
|
|
1034
593
|
isWaiting: true,
|
|
@@ -1036,19 +595,22 @@ async function leaseAsync(session, request) {
|
|
|
1036
595
|
session,
|
|
1037
596
|
waiterId: randomUUID()
|
|
1038
597
|
};
|
|
598
|
+
let reprintKey;
|
|
1039
599
|
for (let delay = FIRST_RETRY_MS;; delay = Math.min(delay * 2, LAST_RETRY_MS)) {
|
|
1040
|
-
const outcome =
|
|
600
|
+
const outcome = attemptUnlessLocked(pending);
|
|
1041
601
|
if (outcome.kind === "grant") return toLease({
|
|
1042
602
|
outcome,
|
|
1043
603
|
request,
|
|
1044
604
|
session
|
|
1045
605
|
});
|
|
1046
|
-
if (
|
|
606
|
+
if (outcome.kind === "wait" && outcome.reprintKey !== reprintKey) {
|
|
607
|
+
({reprintKey} = outcome);
|
|
608
|
+
session.options.log(outcome.line);
|
|
609
|
+
}
|
|
1047
610
|
await session.options.platform.sleepAsync(delay);
|
|
1048
611
|
}
|
|
1049
612
|
}
|
|
1050
|
-
|
|
1051
|
-
//#region src/tree-watch.ts
|
|
613
|
+
// src/tree-watch.ts
|
|
1052
614
|
const TREE_SAMPLE_MS = 100;
|
|
1053
615
|
const BLIND_WATCH = { stop: () => ({
|
|
1054
616
|
peakMemory: void 0,
|
|
@@ -1076,8 +638,7 @@ function watchWith({ every, probe }) {
|
|
|
1076
638
|
return result;
|
|
1077
639
|
} };
|
|
1078
640
|
}
|
|
1079
|
-
|
|
1080
|
-
//#region src/index.ts
|
|
641
|
+
// src/index.ts
|
|
1081
642
|
/**
|
|
1082
643
|
* Joins the budget that every dev tool on this machine shares. Close it when
|
|
1083
644
|
* the run ends; a run that dies is taken back by the next run that leases.
|
|
@@ -1111,5 +672,4 @@ function watchTree(pid) {
|
|
|
1111
672
|
probe: treeProbe(pid)
|
|
1112
673
|
});
|
|
1113
674
|
}
|
|
1114
|
-
//#endregion
|
|
1115
675
|
export { openMachineBudget, watchTree };
|