@tinycloud/operations 0.5.0-beta.0 → 0.5.0-beta.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/coverage.json +11 -1
- package/dist/artifacts.cjs +235 -43
- package/dist/artifacts.cjs.map +1 -1
- package/dist/artifacts.d.cts +1 -1
- package/dist/artifacts.d.ts +1 -1
- package/dist/artifacts.js +236 -43
- package/dist/artifacts.js.map +1 -1
- package/dist/cli-runtime.cjs +1021 -223
- package/dist/cli-runtime.cjs.map +1 -1
- package/dist/cli-runtime.d.cts +1 -1
- package/dist/cli-runtime.d.ts +1 -1
- package/dist/cli-runtime.js +1020 -222
- package/dist/cli-runtime.js.map +1 -1
- package/dist/{contract-DfyrluLB.d.cts → contract-zRklonbR.d.cts} +6 -3
- package/dist/{contract-DfyrluLB.d.ts → contract-zRklonbR.d.ts} +6 -3
- package/dist/delegation-binding.cjs +242 -43
- package/dist/delegation-binding.cjs.map +1 -1
- package/dist/delegation-binding.d.cts +15 -3
- package/dist/delegation-binding.d.ts +15 -3
- package/dist/delegation-binding.js +242 -43
- package/dist/delegation-binding.js.map +1 -1
- package/dist/index.cjs +1049 -224
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1046 -221
- package/dist/index.js.map +1 -1
- package/dist/profile.cjs +13 -4
- package/dist/profile.cjs.map +1 -1
- package/dist/profile.d.cts +14 -2
- package/dist/profile.d.ts +14 -2
- package/dist/profile.js +13 -4
- package/dist/profile.js.map +1 -1
- package/dist/recovery.cjs +67 -0
- package/dist/recovery.cjs.map +1 -0
- package/dist/recovery.d.cts +38 -0
- package/dist/recovery.d.ts +38 -0
- package/dist/recovery.js +39 -0
- package/dist/recovery.js.map +1 -0
- package/dist/secret-request.cjs +271 -0
- package/dist/secret-request.cjs.map +1 -0
- package/dist/secret-request.d.cts +119 -0
- package/dist/secret-request.d.ts +119 -0
- package/dist/secret-request.js +222 -0
- package/dist/secret-request.js.map +1 -0
- package/dist/state.cjs +232 -40
- package/dist/state.cjs.map +1 -1
- package/dist/state.js +233 -40
- package/dist/state.js.map +1 -1
- package/generated/operations.json +1838 -89
- package/package.json +15 -5
package/dist/state.js
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
// src/state.ts
|
|
2
2
|
import { randomUUID } from "crypto";
|
|
3
3
|
import { AsyncLocalStorage } from "async_hooks";
|
|
4
|
+
import { readFileSync, readlinkSync } from "fs";
|
|
4
5
|
import {
|
|
5
6
|
chmod,
|
|
6
7
|
link,
|
|
8
|
+
lstat,
|
|
7
9
|
mkdir,
|
|
8
10
|
open,
|
|
9
11
|
readFile,
|
|
@@ -15,6 +17,7 @@ import {
|
|
|
15
17
|
unlink,
|
|
16
18
|
writeFile
|
|
17
19
|
} from "fs/promises";
|
|
20
|
+
import { connect, createServer } from "net";
|
|
18
21
|
import { homedir } from "os";
|
|
19
22
|
import { basename, dirname, isAbsolute, join, relative, resolve } from "path";
|
|
20
23
|
var DEFAULT_PROFILE = "default";
|
|
@@ -35,13 +38,171 @@ var TEST_LOCK_MOVED_BARRIER_DIR = "TC_TEST_PROFILE_LOCK_MOVED_BARRIER_DIR";
|
|
|
35
38
|
var TEST_LOCK_TURN_PUBLISH_BARRIER_DIR = "TC_TEST_PROFILE_LOCK_TURN_PUBLISH_BARRIER_DIR";
|
|
36
39
|
var TEST_LOCK_TURN_SETTLE_BARRIER_DIR = "TC_TEST_PROFILE_LOCK_TURN_SETTLE_BARRIER_DIR";
|
|
37
40
|
var TEST_LOCK_TURN_COLLECT_BARRIER_DIR = "TC_TEST_PROFILE_LOCK_TURN_COLLECT_BARRIER_DIR";
|
|
41
|
+
var TEST_LOCK_LIVENESS_BARRIER_DIR = "TC_TEST_PROFILE_LOCK_LIVENESS_BARRIER_DIR";
|
|
38
42
|
function processWideContext(key) {
|
|
39
|
-
var
|
|
43
|
+
var _a3;
|
|
40
44
|
const registry = globalThis;
|
|
41
|
-
return registry[
|
|
45
|
+
return registry[_a3 = Symbol.for(key)] ?? (registry[_a3] = new AsyncLocalStorage());
|
|
42
46
|
}
|
|
43
47
|
var invocationStateRoot = processWideContext("tinycloud.operations.invocationStateRoot.v1");
|
|
44
48
|
var heldProfileLocks = processWideContext("tinycloud.operations.heldProfileLocks.v2");
|
|
49
|
+
var _a;
|
|
50
|
+
var processInstanceId = globalThis[_a = /* @__PURE__ */ Symbol.for("tinycloud.operations.profileLockInstance.v1")] ?? (globalThis[_a] = randomUUID());
|
|
51
|
+
var cachedProcessOrigin;
|
|
52
|
+
function processOrigin() {
|
|
53
|
+
if (cachedProcessOrigin !== void 0) return cachedProcessOrigin;
|
|
54
|
+
if (process.platform !== "linux") return cachedProcessOrigin = null;
|
|
55
|
+
try {
|
|
56
|
+
const bootId = readFileSync("/proc/sys/kernel/random/boot_id", "utf8").trim();
|
|
57
|
+
const pidNamespace = readlinkSync("/proc/self/ns/pid");
|
|
58
|
+
const processStat = readFileSync("/proc/self/stat", "utf8");
|
|
59
|
+
const closingParen = processStat.lastIndexOf(") ");
|
|
60
|
+
const processStartTime = closingParen < 0 ? void 0 : processStat.slice(closingParen + 2).trim().split(/\s+/)[19];
|
|
61
|
+
if (!/^[0-9a-f-]{36}$/.test(bootId) || !/^pid:\[[0-9]+\]$/.test(pidNamespace) || !processStartTime || !/^[0-9]+$/.test(processStartTime)) {
|
|
62
|
+
return cachedProcessOrigin = null;
|
|
63
|
+
}
|
|
64
|
+
return cachedProcessOrigin = { bootId, pidNamespace, processStartTime };
|
|
65
|
+
} catch {
|
|
66
|
+
return cachedProcessOrigin = null;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
function processOwner(liveness) {
|
|
70
|
+
return { instanceId: processInstanceId, ...processOrigin(), ...liveness ? { liveness: liveness.record } : {} };
|
|
71
|
+
}
|
|
72
|
+
var LIVENESS_SOCKET = /^\.live-([0-9a-f]+)-([0-9a-f]+)-[0-9a-f]{8}-(?:[0-9a-f]{4}-){3}[0-9a-f]{12}(\.new)?$/;
|
|
73
|
+
var LIVENESS_PROBE_TIMEOUT_MS = 1e3;
|
|
74
|
+
function socketDirectory(name) {
|
|
75
|
+
const match = LIVENESS_SOCKET.exec(name);
|
|
76
|
+
return match ? `${match[1]}-${match[2]}` : null;
|
|
77
|
+
}
|
|
78
|
+
async function pinDirectory(directory) {
|
|
79
|
+
const handle = await open(directory, "r");
|
|
80
|
+
try {
|
|
81
|
+
const { dev, ino } = await handle.stat({ bigint: true });
|
|
82
|
+
return {
|
|
83
|
+
identity: `${dev.toString(16)}-${ino.toString(16)}`,
|
|
84
|
+
path: (name) => `/proc/self/fd/${handle.fd}/${name}`,
|
|
85
|
+
close: () => handle.close()
|
|
86
|
+
};
|
|
87
|
+
} catch (error) {
|
|
88
|
+
await handle.close();
|
|
89
|
+
throw error;
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
async function bindLiveness(directory) {
|
|
93
|
+
if (!processOrigin()) return null;
|
|
94
|
+
let pinned;
|
|
95
|
+
try {
|
|
96
|
+
pinned = await pinDirectory(directory);
|
|
97
|
+
} catch {
|
|
98
|
+
return null;
|
|
99
|
+
}
|
|
100
|
+
const socket = `.live-${pinned.identity}-${randomUUID()}`;
|
|
101
|
+
const server = createServer((connection) => connection.destroy());
|
|
102
|
+
let bound = false;
|
|
103
|
+
try {
|
|
104
|
+
await new Promise((resolve2, reject) => {
|
|
105
|
+
server.once("error", reject);
|
|
106
|
+
server.listen({ path: pinned.path(`${socket}.new`), exclusive: true }, () => {
|
|
107
|
+
server.off("error", reject);
|
|
108
|
+
resolve2();
|
|
109
|
+
});
|
|
110
|
+
});
|
|
111
|
+
bound = true;
|
|
112
|
+
server.on("error", () => void 0);
|
|
113
|
+
server.unref();
|
|
114
|
+
await rename(pinned.path(`${socket}.new`), pinned.path(socket));
|
|
115
|
+
const stats = await lstat(pinned.path(socket), { bigint: true });
|
|
116
|
+
if (!stats.isSocket()) throw new Error(`${socket} is not a socket`);
|
|
117
|
+
await waitForTestBarrier(TEST_LOCK_LIVENESS_BARRIER_DIR, basename(directory));
|
|
118
|
+
return { record: { socket, dev: String(stats.dev), ino: String(stats.ino) }, path: join(directory, socket), server };
|
|
119
|
+
} catch {
|
|
120
|
+
if (bound) {
|
|
121
|
+
server.close();
|
|
122
|
+
for (const name of [`${socket}.new`, socket]) await unlink(pinned.path(name)).catch(() => void 0);
|
|
123
|
+
}
|
|
124
|
+
return null;
|
|
125
|
+
} finally {
|
|
126
|
+
await pinned.close();
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
async function ensureLiveness(acquisition) {
|
|
130
|
+
if (acquisition.held === void 0) acquisition.held = await bindLiveness(acquisition.directory);
|
|
131
|
+
return acquisition.held;
|
|
132
|
+
}
|
|
133
|
+
async function closeLiveness(held) {
|
|
134
|
+
held.server.close();
|
|
135
|
+
await unlink(held.path).catch(() => void 0);
|
|
136
|
+
}
|
|
137
|
+
function parseLiveness(value) {
|
|
138
|
+
if (typeof value !== "object" || value === null) return null;
|
|
139
|
+
const { socket, dev, ino } = value;
|
|
140
|
+
const name = typeof socket === "string" ? LIVENESS_SOCKET.exec(socket) : null;
|
|
141
|
+
return name !== null && name[3] === void 0 && typeof dev === "string" && /^[0-9]+$/.test(dev) && typeof ino === "string" && /^[0-9]+$/.test(ino) ? { socket: name[0], dev, ino } : null;
|
|
142
|
+
}
|
|
143
|
+
async function controlIn(pinned, acquisition) {
|
|
144
|
+
const own = await ensureLiveness(acquisition);
|
|
145
|
+
if (!own || socketDirectory(own.record.socket) !== pinned.identity) return null;
|
|
146
|
+
const stats = await lstat(pinned.path(own.record.socket), { bigint: true }).catch(() => null);
|
|
147
|
+
return stats?.isSocket() && String(stats.dev) === own.record.dev && String(stats.ino) === own.record.ino ? own : null;
|
|
148
|
+
}
|
|
149
|
+
async function tryConnect(path) {
|
|
150
|
+
return new Promise((resolve2) => {
|
|
151
|
+
const probe = connect({ path });
|
|
152
|
+
const settle = (outcome) => {
|
|
153
|
+
clearTimeout(timer);
|
|
154
|
+
probe.destroy();
|
|
155
|
+
resolve2(outcome);
|
|
156
|
+
};
|
|
157
|
+
const timer = setTimeout(() => settle("other"), LIVENESS_PROBE_TIMEOUT_MS);
|
|
158
|
+
probe.once("connect", () => settle("connected"));
|
|
159
|
+
probe.once("error", (error) => settle(isErrno(error, "ECONNREFUSED") ? "refused" : "other"));
|
|
160
|
+
});
|
|
161
|
+
}
|
|
162
|
+
async function kernelRefuses(pinned, name, control) {
|
|
163
|
+
if (process.versions.bun !== void 0) return false;
|
|
164
|
+
return await tryConnect(pinned.path(name)) === "refused" && await tryConnect(pinned.path(control.record.socket)) === "connected";
|
|
165
|
+
}
|
|
166
|
+
async function probeLiveness(liveness, acquisition) {
|
|
167
|
+
let pinned;
|
|
168
|
+
try {
|
|
169
|
+
pinned = await pinDirectory(acquisition.directory);
|
|
170
|
+
} catch {
|
|
171
|
+
return "unknown";
|
|
172
|
+
}
|
|
173
|
+
try {
|
|
174
|
+
if (socketDirectory(liveness.socket) !== pinned.identity) return "unknown";
|
|
175
|
+
const control = await controlIn(pinned, acquisition);
|
|
176
|
+
if (!control) return "unknown";
|
|
177
|
+
let stats;
|
|
178
|
+
try {
|
|
179
|
+
stats = await lstat(pinned.path(liveness.socket), { bigint: true });
|
|
180
|
+
} catch (error) {
|
|
181
|
+
return isErrno(error, "ENOENT") ? "gone" : "unknown";
|
|
182
|
+
}
|
|
183
|
+
if (!stats.isSocket() || String(stats.dev) !== liveness.dev || String(stats.ino) !== liveness.ino) return "unknown";
|
|
184
|
+
return await kernelRefuses(pinned, liveness.socket, control) ? "refused" : "unknown";
|
|
185
|
+
} finally {
|
|
186
|
+
await pinned.close();
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
async function sweepLiveness(acquisition) {
|
|
190
|
+
if (process.versions.bun !== void 0 || !acquisition.held) return;
|
|
191
|
+
const pinned = await pinDirectory(acquisition.directory);
|
|
192
|
+
try {
|
|
193
|
+
const control = await controlIn(pinned, acquisition);
|
|
194
|
+
if (!control) return;
|
|
195
|
+
for (const name of await readdir(pinned.path("."))) {
|
|
196
|
+
if (name === control.record.socket || socketDirectory(name) !== pinned.identity) continue;
|
|
197
|
+
const stats = await lstat(pinned.path(name)).catch(() => null);
|
|
198
|
+
if (stats?.isSocket() && await kernelRefuses(pinned, name, control)) {
|
|
199
|
+
await unlink(pinned.path(name)).catch(() => void 0);
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
} finally {
|
|
203
|
+
await pinned.close();
|
|
204
|
+
}
|
|
205
|
+
}
|
|
45
206
|
var ProfileLockTimeoutError = class extends Error {
|
|
46
207
|
/** `detail` names the cause when it is not simply another process holding the lock. */
|
|
47
208
|
constructor(profile, timeoutMs, detail) {
|
|
@@ -270,7 +431,18 @@ async function acquireProfileLock(profile, options) {
|
|
|
270
431
|
if (path !== directory || !isErrno(error, "ENOENT")) throw error;
|
|
271
432
|
});
|
|
272
433
|
}
|
|
273
|
-
const
|
|
434
|
+
const liveness = { directory: turnDirectory, held: void 0 };
|
|
435
|
+
const closeAcquisitionLiveness = async () => {
|
|
436
|
+
if (liveness.held) await closeLiveness(liveness.held);
|
|
437
|
+
};
|
|
438
|
+
let taken;
|
|
439
|
+
try {
|
|
440
|
+
taken = await acquireTurn(profile, turnDirectory, deadline, liveness);
|
|
441
|
+
} catch (error) {
|
|
442
|
+
await closeAcquisitionLiveness();
|
|
443
|
+
throw error;
|
|
444
|
+
}
|
|
445
|
+
const { turn, firstWaitedTurn } = taken;
|
|
274
446
|
try {
|
|
275
447
|
if (options.requireProfile && !await exists(profileConfigPath(profile))) {
|
|
276
448
|
await rmdir(directory).catch(() => void 0);
|
|
@@ -280,7 +452,7 @@ async function acquireProfileLock(profile, options) {
|
|
|
280
452
|
const deletedWhileWaiting = typeof deletion?.slot === "number" && deletion.slot >= firstWaitedTurn;
|
|
281
453
|
while (true) {
|
|
282
454
|
const token = randomUUID();
|
|
283
|
-
if (await publishProfileLock(profile, lockPath, token, options.requireProfile === true)) {
|
|
455
|
+
if (await publishProfileLock(profile, lockPath, token, options.requireProfile === true, liveness.held ?? null)) {
|
|
284
456
|
return {
|
|
285
457
|
deletedWhileWaiting,
|
|
286
458
|
turnSlot: turn.slot,
|
|
@@ -288,17 +460,19 @@ async function acquireProfileLock(profile, options) {
|
|
|
288
460
|
try {
|
|
289
461
|
await releaseProfileLock(profile, lockPath, token);
|
|
290
462
|
} finally {
|
|
291
|
-
await releaseTurn(profile, turn);
|
|
463
|
+
await releaseTurn(profile, turn, liveness);
|
|
464
|
+
await closeAcquisitionLiveness();
|
|
292
465
|
}
|
|
293
466
|
}
|
|
294
467
|
};
|
|
295
468
|
}
|
|
296
469
|
await signalTestLockContention(profile);
|
|
297
|
-
if (await recoverStaleLock(profile, lockPath, staleAfterMs)) continue;
|
|
470
|
+
if (await recoverStaleLock(profile, lockPath, liveness, staleAfterMs)) continue;
|
|
298
471
|
await waitOrTimeOut(profile, deadline);
|
|
299
472
|
}
|
|
300
473
|
} catch (error) {
|
|
301
|
-
await releaseTurn(profile, turn);
|
|
474
|
+
await releaseTurn(profile, turn, liveness);
|
|
475
|
+
await closeAcquisitionLiveness();
|
|
302
476
|
throw error;
|
|
303
477
|
}
|
|
304
478
|
}
|
|
@@ -312,12 +486,12 @@ async function waitOrTimeOut(profile, deadline, detail) {
|
|
|
312
486
|
await sleep(Math.min(deadline.retryMs, deadline.timeoutMs - (Date.now() - deadline.startedAt)));
|
|
313
487
|
}
|
|
314
488
|
var TURN = /^(?:0|[1-9][0-9]*)$/;
|
|
315
|
-
var
|
|
316
|
-
var relinquishedTurns = globalThis[
|
|
489
|
+
var _a2;
|
|
490
|
+
var relinquishedTurns = globalThis[_a2 = /* @__PURE__ */ Symbol.for("tinycloud.operations.relinquishedProfileTurns.v1")] ?? (globalThis[_a2] = /* @__PURE__ */ new Set());
|
|
317
491
|
var MARKER_RETRY_DELAYS_MS = [10, 50, 250];
|
|
318
492
|
var SETTLE_RETRY_MS = 1e3;
|
|
319
493
|
var SETTLE_RETRIES = 300;
|
|
320
|
-
async function acquireTurn(profile, directory, deadline) {
|
|
494
|
+
async function acquireTurn(profile, directory, deadline, liveness) {
|
|
321
495
|
let firstWaitedTurn;
|
|
322
496
|
let problem;
|
|
323
497
|
const damaged = (what) => `The turn lock ${directory} ${what}, so no process can take it. If no tc or MCP process is using profile "${profile}", remove that directory.`;
|
|
@@ -342,13 +516,13 @@ async function acquireTurn(profile, directory, deadline) {
|
|
|
342
516
|
}
|
|
343
517
|
firstWaitedTurn ?? (firstWaitedTurn = latest.done ? latest.slot + 1 : latest.slot);
|
|
344
518
|
if (latest.held && latest.done) {
|
|
345
|
-
const turn = await takeTurn(profile, directory, latest);
|
|
519
|
+
const turn = await takeTurn(profile, directory, latest, await ensureLiveness(liveness));
|
|
346
520
|
if (turn !== null) return { turn, firstWaitedTurn };
|
|
347
521
|
await waitOrTimeOut(profile, deadline);
|
|
348
522
|
} else if (latest.done) {
|
|
349
523
|
problem = damaged(`ends with turn ${latest.slot}, which was never granted`);
|
|
350
524
|
await waitOrTimeOut(profile, deadline, problem);
|
|
351
|
-
} else if (latest.pid === process.pid
|
|
525
|
+
} else if (latest.pid === process.pid && relinquishedTurns.has(latest.token) || !await isRecordedProcessAlive(latest, liveness)) {
|
|
352
526
|
await waitForTestBarrier(TEST_LOCK_TURN_SETTLE_BARRIER_DIR, profile);
|
|
353
527
|
await settleAbandonedTurn(directory, latest);
|
|
354
528
|
relinquishedTurns.delete(latest.token);
|
|
@@ -393,6 +567,7 @@ async function readTurn(directory, slot) {
|
|
|
393
567
|
const done = await exists(join(path, `${owner.token}.done`));
|
|
394
568
|
const held = await exists(join(path, `${owner.token}.held`));
|
|
395
569
|
return {
|
|
570
|
+
...owner,
|
|
396
571
|
slot,
|
|
397
572
|
token: owner.token,
|
|
398
573
|
pid: owner.pid,
|
|
@@ -406,11 +581,11 @@ async function isFinishedTurn(directory, slot, token) {
|
|
|
406
581
|
if (turn === "damaged") throw new Error(`Turn ${slot} of the turn lock ${directory} is unreadable.`);
|
|
407
582
|
return turn !== null && turn.token === token && turn.held && turn.done;
|
|
408
583
|
}
|
|
409
|
-
async function takeTurn(profile, directory, latest) {
|
|
584
|
+
async function takeTurn(profile, directory, latest, liveness) {
|
|
410
585
|
const token = randomUUID();
|
|
411
586
|
const slot = latest.slot + 1;
|
|
412
587
|
const path = join(directory, String(slot));
|
|
413
|
-
const owner = { pid: process.pid, createdAt: (/* @__PURE__ */ new Date()).toISOString(), token, after: latest.token };
|
|
588
|
+
const owner = { pid: process.pid, createdAt: (/* @__PURE__ */ new Date()).toISOString(), token, after: latest.token, ...processOwner(liveness) };
|
|
414
589
|
let published = false;
|
|
415
590
|
try {
|
|
416
591
|
published = await publishDirectory(
|
|
@@ -464,7 +639,7 @@ async function markTurn(path, token, kind) {
|
|
|
464
639
|
}
|
|
465
640
|
async function createTurnDirectory(directory) {
|
|
466
641
|
const token = randomUUID();
|
|
467
|
-
const owner = { pid: process.pid, createdAt: (/* @__PURE__ */ new Date()).toISOString(), token, after: null };
|
|
642
|
+
const owner = { pid: process.pid, createdAt: (/* @__PURE__ */ new Date()).toISOString(), token, after: null, ...processOwner(null) };
|
|
468
643
|
await publishDirectory(join(dirname(directory), `.stage-${basename(directory)}-${process.pid}-${token}`), directory, {
|
|
469
644
|
"0/owner.json": `${JSON.stringify(owner, null, 2)}
|
|
470
645
|
`,
|
|
@@ -488,31 +663,39 @@ async function publishDirectory(stage, target, files, beforeRename) {
|
|
|
488
663
|
await rm(stage, { recursive: true, force: true }).catch(() => void 0);
|
|
489
664
|
}
|
|
490
665
|
}
|
|
491
|
-
async function releaseTurn(profile, turn) {
|
|
666
|
+
async function releaseTurn(profile, turn, liveness) {
|
|
492
667
|
try {
|
|
493
668
|
await markTurn(join(turn.directory, String(turn.slot)), turn.token, "done");
|
|
494
669
|
} catch {
|
|
495
670
|
settleOwnTurn(turn.directory, turn.slot, turn.token);
|
|
496
671
|
return;
|
|
497
672
|
}
|
|
498
|
-
await collectTurns(profile, turn).catch(() => void 0);
|
|
673
|
+
await collectTurns(profile, turn, liveness).catch(() => void 0);
|
|
499
674
|
}
|
|
500
|
-
function isAbandonedTurnEntry(name) {
|
|
675
|
+
async function isAbandonedTurnEntry(directory, name, liveness) {
|
|
501
676
|
if (name.startsWith(".trash-")) return true;
|
|
502
677
|
const writer = /^\.(?:stage|deleted\.json)[-.]([0-9]+)[-.]/.exec(name)?.[1];
|
|
503
|
-
|
|
678
|
+
if (writer === void 0) return false;
|
|
679
|
+
if (Number(writer) !== process.pid) return !await isRecordedProcessAlive({ pid: Number(writer) }, liveness);
|
|
680
|
+
return name.startsWith(".stage-") && await isStagedOwnerGone(join(directory, name, "owner.json"), liveness);
|
|
504
681
|
}
|
|
505
|
-
function isAbandonedTurnDirectoryStage(name) {
|
|
682
|
+
async function isAbandonedTurnDirectoryStage(directory, name, liveness) {
|
|
506
683
|
const writer = /^\.stage-.+-([0-9]+)-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/.exec(name)?.[1];
|
|
507
|
-
|
|
684
|
+
if (writer === void 0) return false;
|
|
685
|
+
if (Number(writer) !== process.pid) return !await isRecordedProcessAlive({ pid: Number(writer) }, liveness);
|
|
686
|
+
return isStagedOwnerGone(join(directory, name, "0", "owner.json"), liveness);
|
|
508
687
|
}
|
|
509
|
-
async function
|
|
688
|
+
async function isStagedOwnerGone(recordPath, liveness) {
|
|
689
|
+
const owner = await readJson(recordPath).catch(() => null);
|
|
690
|
+
return typeof owner === "object" && owner !== null && owner.pid === process.pid && !await isRecordedProcessAlive(owner, liveness);
|
|
691
|
+
}
|
|
692
|
+
async function collectTurns(profile, turn, liveness) {
|
|
510
693
|
for (const [directory, abandoned] of [
|
|
511
694
|
[turn.directory, isAbandonedTurnEntry],
|
|
512
695
|
[dirname(turn.directory), isAbandonedTurnDirectoryStage]
|
|
513
696
|
]) {
|
|
514
|
-
for (const name of
|
|
515
|
-
await rm(join(directory, name), { recursive: true, force: true }).catch(() => void 0);
|
|
697
|
+
for (const name of await readdir(directory)) {
|
|
698
|
+
if (await abandoned(directory, name, liveness)) await rm(join(directory, name), { recursive: true, force: true }).catch(() => void 0);
|
|
516
699
|
}
|
|
517
700
|
}
|
|
518
701
|
const older = (await readdir(turn.directory)).filter((name) => TURN.test(name)).map(Number).filter((slot) => slot < turn.slot).sort((left, right) => left - right);
|
|
@@ -530,8 +713,9 @@ async function collectTurns(profile, turn) {
|
|
|
530
713
|
await rm(trash, { recursive: true, force: true }).catch(() => void 0);
|
|
531
714
|
}
|
|
532
715
|
}
|
|
716
|
+
await sweepLiveness(liveness);
|
|
533
717
|
}
|
|
534
|
-
async function publishProfileLock(profile, lockPath, token, requireProfile) {
|
|
718
|
+
async function publishProfileLock(profile, lockPath, token, requireProfile, liveness) {
|
|
535
719
|
try {
|
|
536
720
|
await mkdir(lockPath, { mode: PRIVATE_DIR_MODE });
|
|
537
721
|
} catch (error) {
|
|
@@ -544,7 +728,7 @@ async function publishProfileLock(profile, lockPath, token, requireProfile) {
|
|
|
544
728
|
await waitForTestBarrier(TEST_LOCK_PUBLISH_BARRIER_DIR, profile);
|
|
545
729
|
const ownerPath = join(lockPath, "owner.json");
|
|
546
730
|
const staged = join(dirname(lockPath), `.lock-owner-${token}.tmp`);
|
|
547
|
-
const owner = `${JSON.stringify({ pid: process.pid, createdAt: (/* @__PURE__ */ new Date()).toISOString(), token }, null, 2)}
|
|
731
|
+
const owner = `${JSON.stringify({ pid: process.pid, createdAt: (/* @__PURE__ */ new Date()).toISOString(), token, ...processOwner(liveness) }, null, 2)}
|
|
548
732
|
`;
|
|
549
733
|
try {
|
|
550
734
|
await writeFile(staged, owner, { encoding: "utf8", mode: PRIVATE_FILE_MODE, flag: "wx" });
|
|
@@ -589,7 +773,7 @@ async function signalTestLockContention(profile) {
|
|
|
589
773
|
await writeFile(signalPath, `${profile}
|
|
590
774
|
`, { encoding: "utf8", flag: "wx" }).catch(() => void 0);
|
|
591
775
|
}
|
|
592
|
-
async function recoverOwnerlessLock(profile, lockPath, staleAfterMs) {
|
|
776
|
+
async function recoverOwnerlessLock(profile, lockPath, liveness, staleAfterMs) {
|
|
593
777
|
let aged;
|
|
594
778
|
try {
|
|
595
779
|
const { mtimeMs } = await stat(lockPath);
|
|
@@ -606,7 +790,7 @@ async function recoverOwnerlessLock(profile, lockPath, staleAfterMs) {
|
|
|
606
790
|
if (!onlyOrphanedClaims && (!aged || recoveryFenced(checkedAt, staleAfterMs))) return false;
|
|
607
791
|
for (const name of entries.filter((entry) => RECOVERY_CLAIM.test(entry))) {
|
|
608
792
|
const claimPath = join(lockPath, name);
|
|
609
|
-
if (await hasLiveRecoveryOwner(claimPath)) {
|
|
793
|
+
if (await hasLiveRecoveryOwner(claimPath, liveness)) {
|
|
610
794
|
if (!await exists(join(lockPath, "owner.json"))) await rename(claimPath, join(lockPath, "owner.json"));
|
|
611
795
|
return false;
|
|
612
796
|
}
|
|
@@ -622,7 +806,7 @@ async function recoverOwnerlessLock(profile, lockPath, staleAfterMs) {
|
|
|
622
806
|
await removeAgedOwnerFiles(lockPath, staleAfterMs);
|
|
623
807
|
return true;
|
|
624
808
|
}
|
|
625
|
-
async function hasLiveRecoveryOwner(claimPath) {
|
|
809
|
+
async function hasLiveRecoveryOwner(claimPath, liveness) {
|
|
626
810
|
let contents;
|
|
627
811
|
try {
|
|
628
812
|
contents = await readFile(claimPath, "utf8");
|
|
@@ -636,8 +820,8 @@ async function hasLiveRecoveryOwner(claimPath) {
|
|
|
636
820
|
} catch {
|
|
637
821
|
return false;
|
|
638
822
|
}
|
|
639
|
-
const
|
|
640
|
-
return typeof pid === "number" && Number.isInteger(pid) && pid > 0 &&
|
|
823
|
+
const record = typeof owner === "object" && owner !== null && "pid" in owner ? owner : null;
|
|
824
|
+
return typeof record?.pid === "number" && Number.isInteger(record.pid) && record.pid > 0 && await isRecordedProcessAlive(record, liveness);
|
|
641
825
|
}
|
|
642
826
|
function recoveryFenced(since, staleAfterMs) {
|
|
643
827
|
return performance.now() - since >= staleAfterMs / 2;
|
|
@@ -654,11 +838,11 @@ async function removeAgedOwnerFiles(lockPath, staleAfterMs) {
|
|
|
654
838
|
if (aged) await rm(path, { force: true }).catch(() => void 0);
|
|
655
839
|
}
|
|
656
840
|
}
|
|
657
|
-
async function recoverStaleLock(profile, lockPath, staleAfterMs) {
|
|
841
|
+
async function recoverStaleLock(profile, lockPath, liveness, staleAfterMs) {
|
|
658
842
|
const ownerPath = join(lockPath, "owner.json");
|
|
659
843
|
const observed = await readFile(ownerPath, "utf8").catch(() => null);
|
|
660
|
-
if (observed === null) return recoverOwnerlessLock(profile, lockPath, staleAfterMs);
|
|
661
|
-
if (!await isAbandonedOwner(observed, ownerPath, staleAfterMs)) return false;
|
|
844
|
+
if (observed === null) return recoverOwnerlessLock(profile, lockPath, liveness, staleAfterMs);
|
|
845
|
+
if (!await isAbandonedOwner(observed, ownerPath, liveness, staleAfterMs)) return false;
|
|
662
846
|
await waitForTestBarrier(TEST_LOCK_RECOVERY_BARRIER_DIR, profile);
|
|
663
847
|
const claimPath = join(lockPath, `.recover-${randomUUID()}.json`);
|
|
664
848
|
let moved = false;
|
|
@@ -694,7 +878,7 @@ async function recoverStaleLock(profile, lockPath, staleAfterMs) {
|
|
|
694
878
|
await removeAgedOwnerFiles(lockPath, staleAfterMs);
|
|
695
879
|
return true;
|
|
696
880
|
}
|
|
697
|
-
async function isAbandonedOwner(observed, ownerPath, staleAfterMs) {
|
|
881
|
+
async function isAbandonedOwner(observed, ownerPath, liveness, staleAfterMs) {
|
|
698
882
|
let owner;
|
|
699
883
|
try {
|
|
700
884
|
owner = JSON.parse(observed);
|
|
@@ -702,7 +886,7 @@ async function isAbandonedOwner(observed, ownerPath, staleAfterMs) {
|
|
|
702
886
|
owner = null;
|
|
703
887
|
}
|
|
704
888
|
if (typeof owner === "object" && owner !== null && "pid" in owner) {
|
|
705
|
-
return isStaleOwner(owner, staleAfterMs);
|
|
889
|
+
return isStaleOwner(owner, liveness, staleAfterMs);
|
|
706
890
|
}
|
|
707
891
|
const modifiedAt = await stat(ownerPath).then(({ mtimeMs }) => mtimeMs, () => Number.NaN);
|
|
708
892
|
return Date.now() - modifiedAt >= staleAfterMs;
|
|
@@ -724,14 +908,14 @@ async function waitForTestBarrier(environmentName, profile) {
|
|
|
724
908
|
}
|
|
725
909
|
}
|
|
726
910
|
}
|
|
727
|
-
function isStaleOwner(owner, staleAfterMs) {
|
|
911
|
+
async function isStaleOwner(owner, liveness, staleAfterMs) {
|
|
728
912
|
const now = Date.now();
|
|
729
913
|
const createdAt = typeof owner?.createdAt === "string" ? Date.parse(owner.createdAt) : Number.NaN;
|
|
730
914
|
if (!Number.isFinite(createdAt) || now - createdAt < staleAfterMs) return false;
|
|
731
915
|
if (typeof owner?.pid !== "number" || !Number.isInteger(owner.pid) || owner.pid <= 0) {
|
|
732
916
|
return false;
|
|
733
917
|
}
|
|
734
|
-
return !
|
|
918
|
+
return !await isRecordedProcessAlive(owner, liveness);
|
|
735
919
|
}
|
|
736
920
|
async function exists(path) {
|
|
737
921
|
try {
|
|
@@ -742,13 +926,22 @@ async function exists(path) {
|
|
|
742
926
|
throw error;
|
|
743
927
|
}
|
|
744
928
|
}
|
|
745
|
-
function
|
|
929
|
+
async function isRecordedProcessAlive(owner, acquisition) {
|
|
930
|
+
const pid = owner.pid;
|
|
931
|
+
if (typeof pid !== "number" || !Number.isInteger(pid) || pid <= 0) return true;
|
|
746
932
|
try {
|
|
747
933
|
process.kill(pid, 0);
|
|
748
|
-
return true;
|
|
749
934
|
} catch (error) {
|
|
750
935
|
return !isErrno(error, "ESRCH");
|
|
751
936
|
}
|
|
937
|
+
if (pid !== process.pid || typeof owner.instanceId !== "string" || !/^[0-9a-f]{8}-(?:[0-9a-f]{4}-){3}[0-9a-f]{12}$/.test(owner.instanceId) || owner.instanceId === processInstanceId) return true;
|
|
938
|
+
const origin = processOrigin();
|
|
939
|
+
if (!origin || owner.bootId !== origin.bootId || typeof owner.pidNamespace !== "string" || !/^pid:\[[0-9]+\]$/.test(owner.pidNamespace)) return true;
|
|
940
|
+
if (owner.pidNamespace === origin.pidNamespace) {
|
|
941
|
+
return typeof owner.processStartTime !== "string" || !/^[0-9]+$/.test(owner.processStartTime) || owner.processStartTime === origin.processStartTime;
|
|
942
|
+
}
|
|
943
|
+
const liveness = parseLiveness(owner.liveness);
|
|
944
|
+
return liveness === null || await probeLiveness(liveness, acquisition) === "unknown";
|
|
752
945
|
}
|
|
753
946
|
function validateProfileName(profile) {
|
|
754
947
|
if (!profile || profile === "." || profile === ".." || profile.includes("/") || profile.includes("\\") || profile.includes("\0")) {
|