@tinycloud/operations 0.5.0-beta.4 → 0.5.0-beta.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/coverage.json +1 -0
- package/dist/artifacts.cjs +232 -40
- package/dist/artifacts.cjs.map +1 -1
- package/dist/artifacts.js +233 -40
- package/dist/artifacts.js.map +1 -1
- package/dist/cli-runtime.cjs +230 -40
- package/dist/cli-runtime.cjs.map +1 -1
- package/dist/cli-runtime.js +231 -40
- package/dist/cli-runtime.js.map +1 -1
- package/dist/delegation-binding.cjs +232 -40
- package/dist/delegation-binding.cjs.map +1 -1
- package/dist/delegation-binding.js +233 -40
- package/dist/delegation-binding.js.map +1 -1
- package/dist/index.cjs +230 -40
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +231 -40
- package/dist/index.js.map +1 -1
- package/dist/profile.cjs +8 -4
- package/dist/profile.cjs.map +1 -1
- package/dist/profile.js +9 -4
- package/dist/profile.js.map +1 -1
- package/dist/state.cjs +232 -40
- package/dist/state.cjs.map +1 -1
- package/dist/state.js +233 -40
- package/dist/state.js.map +1 -1
- package/package.json +4 -4
package/dist/state.cjs
CHANGED
|
@@ -56,7 +56,9 @@ __export(state_exports, {
|
|
|
56
56
|
module.exports = __toCommonJS(state_exports);
|
|
57
57
|
var import_node_crypto = require("crypto");
|
|
58
58
|
var import_node_async_hooks = require("async_hooks");
|
|
59
|
+
var import_node_fs = require("fs");
|
|
59
60
|
var import_promises = require("fs/promises");
|
|
61
|
+
var import_node_net = require("net");
|
|
60
62
|
var import_node_os = require("os");
|
|
61
63
|
var import_node_path = require("path");
|
|
62
64
|
var DEFAULT_PROFILE = "default";
|
|
@@ -77,13 +79,171 @@ var TEST_LOCK_MOVED_BARRIER_DIR = "TC_TEST_PROFILE_LOCK_MOVED_BARRIER_DIR";
|
|
|
77
79
|
var TEST_LOCK_TURN_PUBLISH_BARRIER_DIR = "TC_TEST_PROFILE_LOCK_TURN_PUBLISH_BARRIER_DIR";
|
|
78
80
|
var TEST_LOCK_TURN_SETTLE_BARRIER_DIR = "TC_TEST_PROFILE_LOCK_TURN_SETTLE_BARRIER_DIR";
|
|
79
81
|
var TEST_LOCK_TURN_COLLECT_BARRIER_DIR = "TC_TEST_PROFILE_LOCK_TURN_COLLECT_BARRIER_DIR";
|
|
82
|
+
var TEST_LOCK_LIVENESS_BARRIER_DIR = "TC_TEST_PROFILE_LOCK_LIVENESS_BARRIER_DIR";
|
|
80
83
|
function processWideContext(key) {
|
|
81
|
-
var
|
|
84
|
+
var _a3;
|
|
82
85
|
const registry = globalThis;
|
|
83
|
-
return registry[
|
|
86
|
+
return registry[_a3 = Symbol.for(key)] ?? (registry[_a3] = new import_node_async_hooks.AsyncLocalStorage());
|
|
84
87
|
}
|
|
85
88
|
var invocationStateRoot = processWideContext("tinycloud.operations.invocationStateRoot.v1");
|
|
86
89
|
var heldProfileLocks = processWideContext("tinycloud.operations.heldProfileLocks.v2");
|
|
90
|
+
var _a;
|
|
91
|
+
var processInstanceId = globalThis[_a = /* @__PURE__ */ Symbol.for("tinycloud.operations.profileLockInstance.v1")] ?? (globalThis[_a] = (0, import_node_crypto.randomUUID)());
|
|
92
|
+
var cachedProcessOrigin;
|
|
93
|
+
function processOrigin() {
|
|
94
|
+
if (cachedProcessOrigin !== void 0) return cachedProcessOrigin;
|
|
95
|
+
if (process.platform !== "linux") return cachedProcessOrigin = null;
|
|
96
|
+
try {
|
|
97
|
+
const bootId = (0, import_node_fs.readFileSync)("/proc/sys/kernel/random/boot_id", "utf8").trim();
|
|
98
|
+
const pidNamespace = (0, import_node_fs.readlinkSync)("/proc/self/ns/pid");
|
|
99
|
+
const processStat = (0, import_node_fs.readFileSync)("/proc/self/stat", "utf8");
|
|
100
|
+
const closingParen = processStat.lastIndexOf(") ");
|
|
101
|
+
const processStartTime = closingParen < 0 ? void 0 : processStat.slice(closingParen + 2).trim().split(/\s+/)[19];
|
|
102
|
+
if (!/^[0-9a-f-]{36}$/.test(bootId) || !/^pid:\[[0-9]+\]$/.test(pidNamespace) || !processStartTime || !/^[0-9]+$/.test(processStartTime)) {
|
|
103
|
+
return cachedProcessOrigin = null;
|
|
104
|
+
}
|
|
105
|
+
return cachedProcessOrigin = { bootId, pidNamespace, processStartTime };
|
|
106
|
+
} catch {
|
|
107
|
+
return cachedProcessOrigin = null;
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
function processOwner(liveness) {
|
|
111
|
+
return { instanceId: processInstanceId, ...processOrigin(), ...liveness ? { liveness: liveness.record } : {} };
|
|
112
|
+
}
|
|
113
|
+
var LIVENESS_SOCKET = /^\.live-([0-9a-f]+)-([0-9a-f]+)-[0-9a-f]{8}-(?:[0-9a-f]{4}-){3}[0-9a-f]{12}(\.new)?$/;
|
|
114
|
+
var LIVENESS_PROBE_TIMEOUT_MS = 1e3;
|
|
115
|
+
function socketDirectory(name) {
|
|
116
|
+
const match = LIVENESS_SOCKET.exec(name);
|
|
117
|
+
return match ? `${match[1]}-${match[2]}` : null;
|
|
118
|
+
}
|
|
119
|
+
async function pinDirectory(directory) {
|
|
120
|
+
const handle = await (0, import_promises.open)(directory, "r");
|
|
121
|
+
try {
|
|
122
|
+
const { dev, ino } = await handle.stat({ bigint: true });
|
|
123
|
+
return {
|
|
124
|
+
identity: `${dev.toString(16)}-${ino.toString(16)}`,
|
|
125
|
+
path: (name) => `/proc/self/fd/${handle.fd}/${name}`,
|
|
126
|
+
close: () => handle.close()
|
|
127
|
+
};
|
|
128
|
+
} catch (error) {
|
|
129
|
+
await handle.close();
|
|
130
|
+
throw error;
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
async function bindLiveness(directory) {
|
|
134
|
+
if (!processOrigin()) return null;
|
|
135
|
+
let pinned;
|
|
136
|
+
try {
|
|
137
|
+
pinned = await pinDirectory(directory);
|
|
138
|
+
} catch {
|
|
139
|
+
return null;
|
|
140
|
+
}
|
|
141
|
+
const socket = `.live-${pinned.identity}-${(0, import_node_crypto.randomUUID)()}`;
|
|
142
|
+
const server = (0, import_node_net.createServer)((connection) => connection.destroy());
|
|
143
|
+
let bound = false;
|
|
144
|
+
try {
|
|
145
|
+
await new Promise((resolve2, reject) => {
|
|
146
|
+
server.once("error", reject);
|
|
147
|
+
server.listen({ path: pinned.path(`${socket}.new`), exclusive: true }, () => {
|
|
148
|
+
server.off("error", reject);
|
|
149
|
+
resolve2();
|
|
150
|
+
});
|
|
151
|
+
});
|
|
152
|
+
bound = true;
|
|
153
|
+
server.on("error", () => void 0);
|
|
154
|
+
server.unref();
|
|
155
|
+
await (0, import_promises.rename)(pinned.path(`${socket}.new`), pinned.path(socket));
|
|
156
|
+
const stats = await (0, import_promises.lstat)(pinned.path(socket), { bigint: true });
|
|
157
|
+
if (!stats.isSocket()) throw new Error(`${socket} is not a socket`);
|
|
158
|
+
await waitForTestBarrier(TEST_LOCK_LIVENESS_BARRIER_DIR, (0, import_node_path.basename)(directory));
|
|
159
|
+
return { record: { socket, dev: String(stats.dev), ino: String(stats.ino) }, path: (0, import_node_path.join)(directory, socket), server };
|
|
160
|
+
} catch {
|
|
161
|
+
if (bound) {
|
|
162
|
+
server.close();
|
|
163
|
+
for (const name of [`${socket}.new`, socket]) await (0, import_promises.unlink)(pinned.path(name)).catch(() => void 0);
|
|
164
|
+
}
|
|
165
|
+
return null;
|
|
166
|
+
} finally {
|
|
167
|
+
await pinned.close();
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
async function ensureLiveness(acquisition) {
|
|
171
|
+
if (acquisition.held === void 0) acquisition.held = await bindLiveness(acquisition.directory);
|
|
172
|
+
return acquisition.held;
|
|
173
|
+
}
|
|
174
|
+
async function closeLiveness(held) {
|
|
175
|
+
held.server.close();
|
|
176
|
+
await (0, import_promises.unlink)(held.path).catch(() => void 0);
|
|
177
|
+
}
|
|
178
|
+
function parseLiveness(value) {
|
|
179
|
+
if (typeof value !== "object" || value === null) return null;
|
|
180
|
+
const { socket, dev, ino } = value;
|
|
181
|
+
const name = typeof socket === "string" ? LIVENESS_SOCKET.exec(socket) : null;
|
|
182
|
+
return name !== null && name[3] === void 0 && typeof dev === "string" && /^[0-9]+$/.test(dev) && typeof ino === "string" && /^[0-9]+$/.test(ino) ? { socket: name[0], dev, ino } : null;
|
|
183
|
+
}
|
|
184
|
+
async function controlIn(pinned, acquisition) {
|
|
185
|
+
const own = await ensureLiveness(acquisition);
|
|
186
|
+
if (!own || socketDirectory(own.record.socket) !== pinned.identity) return null;
|
|
187
|
+
const stats = await (0, import_promises.lstat)(pinned.path(own.record.socket), { bigint: true }).catch(() => null);
|
|
188
|
+
return stats?.isSocket() && String(stats.dev) === own.record.dev && String(stats.ino) === own.record.ino ? own : null;
|
|
189
|
+
}
|
|
190
|
+
async function tryConnect(path) {
|
|
191
|
+
return new Promise((resolve2) => {
|
|
192
|
+
const probe = (0, import_node_net.connect)({ path });
|
|
193
|
+
const settle = (outcome) => {
|
|
194
|
+
clearTimeout(timer);
|
|
195
|
+
probe.destroy();
|
|
196
|
+
resolve2(outcome);
|
|
197
|
+
};
|
|
198
|
+
const timer = setTimeout(() => settle("other"), LIVENESS_PROBE_TIMEOUT_MS);
|
|
199
|
+
probe.once("connect", () => settle("connected"));
|
|
200
|
+
probe.once("error", (error) => settle(isErrno(error, "ECONNREFUSED") ? "refused" : "other"));
|
|
201
|
+
});
|
|
202
|
+
}
|
|
203
|
+
async function kernelRefuses(pinned, name, control) {
|
|
204
|
+
if (process.versions.bun !== void 0) return false;
|
|
205
|
+
return await tryConnect(pinned.path(name)) === "refused" && await tryConnect(pinned.path(control.record.socket)) === "connected";
|
|
206
|
+
}
|
|
207
|
+
async function probeLiveness(liveness, acquisition) {
|
|
208
|
+
let pinned;
|
|
209
|
+
try {
|
|
210
|
+
pinned = await pinDirectory(acquisition.directory);
|
|
211
|
+
} catch {
|
|
212
|
+
return "unknown";
|
|
213
|
+
}
|
|
214
|
+
try {
|
|
215
|
+
if (socketDirectory(liveness.socket) !== pinned.identity) return "unknown";
|
|
216
|
+
const control = await controlIn(pinned, acquisition);
|
|
217
|
+
if (!control) return "unknown";
|
|
218
|
+
let stats;
|
|
219
|
+
try {
|
|
220
|
+
stats = await (0, import_promises.lstat)(pinned.path(liveness.socket), { bigint: true });
|
|
221
|
+
} catch (error) {
|
|
222
|
+
return isErrno(error, "ENOENT") ? "gone" : "unknown";
|
|
223
|
+
}
|
|
224
|
+
if (!stats.isSocket() || String(stats.dev) !== liveness.dev || String(stats.ino) !== liveness.ino) return "unknown";
|
|
225
|
+
return await kernelRefuses(pinned, liveness.socket, control) ? "refused" : "unknown";
|
|
226
|
+
} finally {
|
|
227
|
+
await pinned.close();
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
async function sweepLiveness(acquisition) {
|
|
231
|
+
if (process.versions.bun !== void 0 || !acquisition.held) return;
|
|
232
|
+
const pinned = await pinDirectory(acquisition.directory);
|
|
233
|
+
try {
|
|
234
|
+
const control = await controlIn(pinned, acquisition);
|
|
235
|
+
if (!control) return;
|
|
236
|
+
for (const name of await (0, import_promises.readdir)(pinned.path("."))) {
|
|
237
|
+
if (name === control.record.socket || socketDirectory(name) !== pinned.identity) continue;
|
|
238
|
+
const stats = await (0, import_promises.lstat)(pinned.path(name)).catch(() => null);
|
|
239
|
+
if (stats?.isSocket() && await kernelRefuses(pinned, name, control)) {
|
|
240
|
+
await (0, import_promises.unlink)(pinned.path(name)).catch(() => void 0);
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
} finally {
|
|
244
|
+
await pinned.close();
|
|
245
|
+
}
|
|
246
|
+
}
|
|
87
247
|
var ProfileLockTimeoutError = class extends Error {
|
|
88
248
|
/** `detail` names the cause when it is not simply another process holding the lock. */
|
|
89
249
|
constructor(profile, timeoutMs, detail) {
|
|
@@ -312,7 +472,18 @@ async function acquireProfileLock(profile, options) {
|
|
|
312
472
|
if (path !== directory || !isErrno(error, "ENOENT")) throw error;
|
|
313
473
|
});
|
|
314
474
|
}
|
|
315
|
-
const
|
|
475
|
+
const liveness = { directory: turnDirectory, held: void 0 };
|
|
476
|
+
const closeAcquisitionLiveness = async () => {
|
|
477
|
+
if (liveness.held) await closeLiveness(liveness.held);
|
|
478
|
+
};
|
|
479
|
+
let taken;
|
|
480
|
+
try {
|
|
481
|
+
taken = await acquireTurn(profile, turnDirectory, deadline, liveness);
|
|
482
|
+
} catch (error) {
|
|
483
|
+
await closeAcquisitionLiveness();
|
|
484
|
+
throw error;
|
|
485
|
+
}
|
|
486
|
+
const { turn, firstWaitedTurn } = taken;
|
|
316
487
|
try {
|
|
317
488
|
if (options.requireProfile && !await exists(profileConfigPath(profile))) {
|
|
318
489
|
await (0, import_promises.rmdir)(directory).catch(() => void 0);
|
|
@@ -322,7 +493,7 @@ async function acquireProfileLock(profile, options) {
|
|
|
322
493
|
const deletedWhileWaiting = typeof deletion?.slot === "number" && deletion.slot >= firstWaitedTurn;
|
|
323
494
|
while (true) {
|
|
324
495
|
const token = (0, import_node_crypto.randomUUID)();
|
|
325
|
-
if (await publishProfileLock(profile, lockPath, token, options.requireProfile === true)) {
|
|
496
|
+
if (await publishProfileLock(profile, lockPath, token, options.requireProfile === true, liveness.held ?? null)) {
|
|
326
497
|
return {
|
|
327
498
|
deletedWhileWaiting,
|
|
328
499
|
turnSlot: turn.slot,
|
|
@@ -330,17 +501,19 @@ async function acquireProfileLock(profile, options) {
|
|
|
330
501
|
try {
|
|
331
502
|
await releaseProfileLock(profile, lockPath, token);
|
|
332
503
|
} finally {
|
|
333
|
-
await releaseTurn(profile, turn);
|
|
504
|
+
await releaseTurn(profile, turn, liveness);
|
|
505
|
+
await closeAcquisitionLiveness();
|
|
334
506
|
}
|
|
335
507
|
}
|
|
336
508
|
};
|
|
337
509
|
}
|
|
338
510
|
await signalTestLockContention(profile);
|
|
339
|
-
if (await recoverStaleLock(profile, lockPath, staleAfterMs)) continue;
|
|
511
|
+
if (await recoverStaleLock(profile, lockPath, liveness, staleAfterMs)) continue;
|
|
340
512
|
await waitOrTimeOut(profile, deadline);
|
|
341
513
|
}
|
|
342
514
|
} catch (error) {
|
|
343
|
-
await releaseTurn(profile, turn);
|
|
515
|
+
await releaseTurn(profile, turn, liveness);
|
|
516
|
+
await closeAcquisitionLiveness();
|
|
344
517
|
throw error;
|
|
345
518
|
}
|
|
346
519
|
}
|
|
@@ -354,12 +527,12 @@ async function waitOrTimeOut(profile, deadline, detail) {
|
|
|
354
527
|
await sleep(Math.min(deadline.retryMs, deadline.timeoutMs - (Date.now() - deadline.startedAt)));
|
|
355
528
|
}
|
|
356
529
|
var TURN = /^(?:0|[1-9][0-9]*)$/;
|
|
357
|
-
var
|
|
358
|
-
var relinquishedTurns = globalThis[
|
|
530
|
+
var _a2;
|
|
531
|
+
var relinquishedTurns = globalThis[_a2 = /* @__PURE__ */ Symbol.for("tinycloud.operations.relinquishedProfileTurns.v1")] ?? (globalThis[_a2] = /* @__PURE__ */ new Set());
|
|
359
532
|
var MARKER_RETRY_DELAYS_MS = [10, 50, 250];
|
|
360
533
|
var SETTLE_RETRY_MS = 1e3;
|
|
361
534
|
var SETTLE_RETRIES = 300;
|
|
362
|
-
async function acquireTurn(profile, directory, deadline) {
|
|
535
|
+
async function acquireTurn(profile, directory, deadline, liveness) {
|
|
363
536
|
let firstWaitedTurn;
|
|
364
537
|
let problem;
|
|
365
538
|
const damaged = (what) => `The turn lock ${directory} ${what}, so no process can take it. If no tc or MCP process is using profile "${profile}", remove that directory.`;
|
|
@@ -384,13 +557,13 @@ async function acquireTurn(profile, directory, deadline) {
|
|
|
384
557
|
}
|
|
385
558
|
firstWaitedTurn ?? (firstWaitedTurn = latest.done ? latest.slot + 1 : latest.slot);
|
|
386
559
|
if (latest.held && latest.done) {
|
|
387
|
-
const turn = await takeTurn(profile, directory, latest);
|
|
560
|
+
const turn = await takeTurn(profile, directory, latest, await ensureLiveness(liveness));
|
|
388
561
|
if (turn !== null) return { turn, firstWaitedTurn };
|
|
389
562
|
await waitOrTimeOut(profile, deadline);
|
|
390
563
|
} else if (latest.done) {
|
|
391
564
|
problem = damaged(`ends with turn ${latest.slot}, which was never granted`);
|
|
392
565
|
await waitOrTimeOut(profile, deadline, problem);
|
|
393
|
-
} else if (latest.pid === process.pid
|
|
566
|
+
} else if (latest.pid === process.pid && relinquishedTurns.has(latest.token) || !await isRecordedProcessAlive(latest, liveness)) {
|
|
394
567
|
await waitForTestBarrier(TEST_LOCK_TURN_SETTLE_BARRIER_DIR, profile);
|
|
395
568
|
await settleAbandonedTurn(directory, latest);
|
|
396
569
|
relinquishedTurns.delete(latest.token);
|
|
@@ -435,6 +608,7 @@ async function readTurn(directory, slot) {
|
|
|
435
608
|
const done = await exists((0, import_node_path.join)(path, `${owner.token}.done`));
|
|
436
609
|
const held = await exists((0, import_node_path.join)(path, `${owner.token}.held`));
|
|
437
610
|
return {
|
|
611
|
+
...owner,
|
|
438
612
|
slot,
|
|
439
613
|
token: owner.token,
|
|
440
614
|
pid: owner.pid,
|
|
@@ -448,11 +622,11 @@ async function isFinishedTurn(directory, slot, token) {
|
|
|
448
622
|
if (turn === "damaged") throw new Error(`Turn ${slot} of the turn lock ${directory} is unreadable.`);
|
|
449
623
|
return turn !== null && turn.token === token && turn.held && turn.done;
|
|
450
624
|
}
|
|
451
|
-
async function takeTurn(profile, directory, latest) {
|
|
625
|
+
async function takeTurn(profile, directory, latest, liveness) {
|
|
452
626
|
const token = (0, import_node_crypto.randomUUID)();
|
|
453
627
|
const slot = latest.slot + 1;
|
|
454
628
|
const path = (0, import_node_path.join)(directory, String(slot));
|
|
455
|
-
const owner = { pid: process.pid, createdAt: (/* @__PURE__ */ new Date()).toISOString(), token, after: latest.token };
|
|
629
|
+
const owner = { pid: process.pid, createdAt: (/* @__PURE__ */ new Date()).toISOString(), token, after: latest.token, ...processOwner(liveness) };
|
|
456
630
|
let published = false;
|
|
457
631
|
try {
|
|
458
632
|
published = await publishDirectory(
|
|
@@ -506,7 +680,7 @@ async function markTurn(path, token, kind) {
|
|
|
506
680
|
}
|
|
507
681
|
async function createTurnDirectory(directory) {
|
|
508
682
|
const token = (0, import_node_crypto.randomUUID)();
|
|
509
|
-
const owner = { pid: process.pid, createdAt: (/* @__PURE__ */ new Date()).toISOString(), token, after: null };
|
|
683
|
+
const owner = { pid: process.pid, createdAt: (/* @__PURE__ */ new Date()).toISOString(), token, after: null, ...processOwner(null) };
|
|
510
684
|
await publishDirectory((0, import_node_path.join)((0, import_node_path.dirname)(directory), `.stage-${(0, import_node_path.basename)(directory)}-${process.pid}-${token}`), directory, {
|
|
511
685
|
"0/owner.json": `${JSON.stringify(owner, null, 2)}
|
|
512
686
|
`,
|
|
@@ -530,31 +704,39 @@ async function publishDirectory(stage, target, files, beforeRename) {
|
|
|
530
704
|
await (0, import_promises.rm)(stage, { recursive: true, force: true }).catch(() => void 0);
|
|
531
705
|
}
|
|
532
706
|
}
|
|
533
|
-
async function releaseTurn(profile, turn) {
|
|
707
|
+
async function releaseTurn(profile, turn, liveness) {
|
|
534
708
|
try {
|
|
535
709
|
await markTurn((0, import_node_path.join)(turn.directory, String(turn.slot)), turn.token, "done");
|
|
536
710
|
} catch {
|
|
537
711
|
settleOwnTurn(turn.directory, turn.slot, turn.token);
|
|
538
712
|
return;
|
|
539
713
|
}
|
|
540
|
-
await collectTurns(profile, turn).catch(() => void 0);
|
|
714
|
+
await collectTurns(profile, turn, liveness).catch(() => void 0);
|
|
541
715
|
}
|
|
542
|
-
function isAbandonedTurnEntry(name) {
|
|
716
|
+
async function isAbandonedTurnEntry(directory, name, liveness) {
|
|
543
717
|
if (name.startsWith(".trash-")) return true;
|
|
544
718
|
const writer = /^\.(?:stage|deleted\.json)[-.]([0-9]+)[-.]/.exec(name)?.[1];
|
|
545
|
-
|
|
719
|
+
if (writer === void 0) return false;
|
|
720
|
+
if (Number(writer) !== process.pid) return !await isRecordedProcessAlive({ pid: Number(writer) }, liveness);
|
|
721
|
+
return name.startsWith(".stage-") && await isStagedOwnerGone((0, import_node_path.join)(directory, name, "owner.json"), liveness);
|
|
546
722
|
}
|
|
547
|
-
function isAbandonedTurnDirectoryStage(name) {
|
|
723
|
+
async function isAbandonedTurnDirectoryStage(directory, name, liveness) {
|
|
548
724
|
const writer = /^\.stage-.+-([0-9]+)-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/.exec(name)?.[1];
|
|
549
|
-
|
|
725
|
+
if (writer === void 0) return false;
|
|
726
|
+
if (Number(writer) !== process.pid) return !await isRecordedProcessAlive({ pid: Number(writer) }, liveness);
|
|
727
|
+
return isStagedOwnerGone((0, import_node_path.join)(directory, name, "0", "owner.json"), liveness);
|
|
550
728
|
}
|
|
551
|
-
async function
|
|
729
|
+
async function isStagedOwnerGone(recordPath, liveness) {
|
|
730
|
+
const owner = await readJson(recordPath).catch(() => null);
|
|
731
|
+
return typeof owner === "object" && owner !== null && owner.pid === process.pid && !await isRecordedProcessAlive(owner, liveness);
|
|
732
|
+
}
|
|
733
|
+
async function collectTurns(profile, turn, liveness) {
|
|
552
734
|
for (const [directory, abandoned] of [
|
|
553
735
|
[turn.directory, isAbandonedTurnEntry],
|
|
554
736
|
[(0, import_node_path.dirname)(turn.directory), isAbandonedTurnDirectoryStage]
|
|
555
737
|
]) {
|
|
556
|
-
for (const name of
|
|
557
|
-
await (0, import_promises.rm)((0, import_node_path.join)(directory, name), { recursive: true, force: true }).catch(() => void 0);
|
|
738
|
+
for (const name of await (0, import_promises.readdir)(directory)) {
|
|
739
|
+
if (await abandoned(directory, name, liveness)) await (0, import_promises.rm)((0, import_node_path.join)(directory, name), { recursive: true, force: true }).catch(() => void 0);
|
|
558
740
|
}
|
|
559
741
|
}
|
|
560
742
|
const older = (await (0, import_promises.readdir)(turn.directory)).filter((name) => TURN.test(name)).map(Number).filter((slot) => slot < turn.slot).sort((left, right) => left - right);
|
|
@@ -572,8 +754,9 @@ async function collectTurns(profile, turn) {
|
|
|
572
754
|
await (0, import_promises.rm)(trash, { recursive: true, force: true }).catch(() => void 0);
|
|
573
755
|
}
|
|
574
756
|
}
|
|
757
|
+
await sweepLiveness(liveness);
|
|
575
758
|
}
|
|
576
|
-
async function publishProfileLock(profile, lockPath, token, requireProfile) {
|
|
759
|
+
async function publishProfileLock(profile, lockPath, token, requireProfile, liveness) {
|
|
577
760
|
try {
|
|
578
761
|
await (0, import_promises.mkdir)(lockPath, { mode: PRIVATE_DIR_MODE });
|
|
579
762
|
} catch (error) {
|
|
@@ -586,7 +769,7 @@ async function publishProfileLock(profile, lockPath, token, requireProfile) {
|
|
|
586
769
|
await waitForTestBarrier(TEST_LOCK_PUBLISH_BARRIER_DIR, profile);
|
|
587
770
|
const ownerPath = (0, import_node_path.join)(lockPath, "owner.json");
|
|
588
771
|
const staged = (0, import_node_path.join)((0, import_node_path.dirname)(lockPath), `.lock-owner-${token}.tmp`);
|
|
589
|
-
const owner = `${JSON.stringify({ pid: process.pid, createdAt: (/* @__PURE__ */ new Date()).toISOString(), token }, null, 2)}
|
|
772
|
+
const owner = `${JSON.stringify({ pid: process.pid, createdAt: (/* @__PURE__ */ new Date()).toISOString(), token, ...processOwner(liveness) }, null, 2)}
|
|
590
773
|
`;
|
|
591
774
|
try {
|
|
592
775
|
await (0, import_promises.writeFile)(staged, owner, { encoding: "utf8", mode: PRIVATE_FILE_MODE, flag: "wx" });
|
|
@@ -631,7 +814,7 @@ async function signalTestLockContention(profile) {
|
|
|
631
814
|
await (0, import_promises.writeFile)(signalPath, `${profile}
|
|
632
815
|
`, { encoding: "utf8", flag: "wx" }).catch(() => void 0);
|
|
633
816
|
}
|
|
634
|
-
async function recoverOwnerlessLock(profile, lockPath, staleAfterMs) {
|
|
817
|
+
async function recoverOwnerlessLock(profile, lockPath, liveness, staleAfterMs) {
|
|
635
818
|
let aged;
|
|
636
819
|
try {
|
|
637
820
|
const { mtimeMs } = await (0, import_promises.stat)(lockPath);
|
|
@@ -648,7 +831,7 @@ async function recoverOwnerlessLock(profile, lockPath, staleAfterMs) {
|
|
|
648
831
|
if (!onlyOrphanedClaims && (!aged || recoveryFenced(checkedAt, staleAfterMs))) return false;
|
|
649
832
|
for (const name of entries.filter((entry) => RECOVERY_CLAIM.test(entry))) {
|
|
650
833
|
const claimPath = (0, import_node_path.join)(lockPath, name);
|
|
651
|
-
if (await hasLiveRecoveryOwner(claimPath)) {
|
|
834
|
+
if (await hasLiveRecoveryOwner(claimPath, liveness)) {
|
|
652
835
|
if (!await exists((0, import_node_path.join)(lockPath, "owner.json"))) await (0, import_promises.rename)(claimPath, (0, import_node_path.join)(lockPath, "owner.json"));
|
|
653
836
|
return false;
|
|
654
837
|
}
|
|
@@ -664,7 +847,7 @@ async function recoverOwnerlessLock(profile, lockPath, staleAfterMs) {
|
|
|
664
847
|
await removeAgedOwnerFiles(lockPath, staleAfterMs);
|
|
665
848
|
return true;
|
|
666
849
|
}
|
|
667
|
-
async function hasLiveRecoveryOwner(claimPath) {
|
|
850
|
+
async function hasLiveRecoveryOwner(claimPath, liveness) {
|
|
668
851
|
let contents;
|
|
669
852
|
try {
|
|
670
853
|
contents = await (0, import_promises.readFile)(claimPath, "utf8");
|
|
@@ -678,8 +861,8 @@ async function hasLiveRecoveryOwner(claimPath) {
|
|
|
678
861
|
} catch {
|
|
679
862
|
return false;
|
|
680
863
|
}
|
|
681
|
-
const
|
|
682
|
-
return typeof pid === "number" && Number.isInteger(pid) && pid > 0 &&
|
|
864
|
+
const record = typeof owner === "object" && owner !== null && "pid" in owner ? owner : null;
|
|
865
|
+
return typeof record?.pid === "number" && Number.isInteger(record.pid) && record.pid > 0 && await isRecordedProcessAlive(record, liveness);
|
|
683
866
|
}
|
|
684
867
|
function recoveryFenced(since, staleAfterMs) {
|
|
685
868
|
return performance.now() - since >= staleAfterMs / 2;
|
|
@@ -696,11 +879,11 @@ async function removeAgedOwnerFiles(lockPath, staleAfterMs) {
|
|
|
696
879
|
if (aged) await (0, import_promises.rm)(path, { force: true }).catch(() => void 0);
|
|
697
880
|
}
|
|
698
881
|
}
|
|
699
|
-
async function recoverStaleLock(profile, lockPath, staleAfterMs) {
|
|
882
|
+
async function recoverStaleLock(profile, lockPath, liveness, staleAfterMs) {
|
|
700
883
|
const ownerPath = (0, import_node_path.join)(lockPath, "owner.json");
|
|
701
884
|
const observed = await (0, import_promises.readFile)(ownerPath, "utf8").catch(() => null);
|
|
702
|
-
if (observed === null) return recoverOwnerlessLock(profile, lockPath, staleAfterMs);
|
|
703
|
-
if (!await isAbandonedOwner(observed, ownerPath, staleAfterMs)) return false;
|
|
885
|
+
if (observed === null) return recoverOwnerlessLock(profile, lockPath, liveness, staleAfterMs);
|
|
886
|
+
if (!await isAbandonedOwner(observed, ownerPath, liveness, staleAfterMs)) return false;
|
|
704
887
|
await waitForTestBarrier(TEST_LOCK_RECOVERY_BARRIER_DIR, profile);
|
|
705
888
|
const claimPath = (0, import_node_path.join)(lockPath, `.recover-${(0, import_node_crypto.randomUUID)()}.json`);
|
|
706
889
|
let moved = false;
|
|
@@ -736,7 +919,7 @@ async function recoverStaleLock(profile, lockPath, staleAfterMs) {
|
|
|
736
919
|
await removeAgedOwnerFiles(lockPath, staleAfterMs);
|
|
737
920
|
return true;
|
|
738
921
|
}
|
|
739
|
-
async function isAbandonedOwner(observed, ownerPath, staleAfterMs) {
|
|
922
|
+
async function isAbandonedOwner(observed, ownerPath, liveness, staleAfterMs) {
|
|
740
923
|
let owner;
|
|
741
924
|
try {
|
|
742
925
|
owner = JSON.parse(observed);
|
|
@@ -744,7 +927,7 @@ async function isAbandonedOwner(observed, ownerPath, staleAfterMs) {
|
|
|
744
927
|
owner = null;
|
|
745
928
|
}
|
|
746
929
|
if (typeof owner === "object" && owner !== null && "pid" in owner) {
|
|
747
|
-
return isStaleOwner(owner, staleAfterMs);
|
|
930
|
+
return isStaleOwner(owner, liveness, staleAfterMs);
|
|
748
931
|
}
|
|
749
932
|
const modifiedAt = await (0, import_promises.stat)(ownerPath).then(({ mtimeMs }) => mtimeMs, () => Number.NaN);
|
|
750
933
|
return Date.now() - modifiedAt >= staleAfterMs;
|
|
@@ -766,14 +949,14 @@ async function waitForTestBarrier(environmentName, profile) {
|
|
|
766
949
|
}
|
|
767
950
|
}
|
|
768
951
|
}
|
|
769
|
-
function isStaleOwner(owner, staleAfterMs) {
|
|
952
|
+
async function isStaleOwner(owner, liveness, staleAfterMs) {
|
|
770
953
|
const now = Date.now();
|
|
771
954
|
const createdAt = typeof owner?.createdAt === "string" ? Date.parse(owner.createdAt) : Number.NaN;
|
|
772
955
|
if (!Number.isFinite(createdAt) || now - createdAt < staleAfterMs) return false;
|
|
773
956
|
if (typeof owner?.pid !== "number" || !Number.isInteger(owner.pid) || owner.pid <= 0) {
|
|
774
957
|
return false;
|
|
775
958
|
}
|
|
776
|
-
return !
|
|
959
|
+
return !await isRecordedProcessAlive(owner, liveness);
|
|
777
960
|
}
|
|
778
961
|
async function exists(path) {
|
|
779
962
|
try {
|
|
@@ -784,13 +967,22 @@ async function exists(path) {
|
|
|
784
967
|
throw error;
|
|
785
968
|
}
|
|
786
969
|
}
|
|
787
|
-
function
|
|
970
|
+
async function isRecordedProcessAlive(owner, acquisition) {
|
|
971
|
+
const pid = owner.pid;
|
|
972
|
+
if (typeof pid !== "number" || !Number.isInteger(pid) || pid <= 0) return true;
|
|
788
973
|
try {
|
|
789
974
|
process.kill(pid, 0);
|
|
790
|
-
return true;
|
|
791
975
|
} catch (error) {
|
|
792
976
|
return !isErrno(error, "ESRCH");
|
|
793
977
|
}
|
|
978
|
+
if (pid !== process.pid || typeof owner.instanceId !== "string" || !/^[0-9a-f]{8}-(?:[0-9a-f]{4}-){3}[0-9a-f]{12}$/.test(owner.instanceId) || owner.instanceId === processInstanceId) return true;
|
|
979
|
+
const origin = processOrigin();
|
|
980
|
+
if (!origin || owner.bootId !== origin.bootId || typeof owner.pidNamespace !== "string" || !/^pid:\[[0-9]+\]$/.test(owner.pidNamespace)) return true;
|
|
981
|
+
if (owner.pidNamespace === origin.pidNamespace) {
|
|
982
|
+
return typeof owner.processStartTime !== "string" || !/^[0-9]+$/.test(owner.processStartTime) || owner.processStartTime === origin.processStartTime;
|
|
983
|
+
}
|
|
984
|
+
const liveness = parseLiveness(owner.liveness);
|
|
985
|
+
return liveness === null || await probeLiveness(liveness, acquisition) === "unknown";
|
|
794
986
|
}
|
|
795
987
|
function validateProfileName(profile) {
|
|
796
988
|
if (!profile || profile === "." || profile === ".." || profile.includes("/") || profile.includes("\\") || profile.includes("\0")) {
|