@tinycloud/operations 0.5.0-beta.4 → 0.5.0-beta.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/coverage.json +1 -0
- package/dist/artifacts.cjs +232 -40
- package/dist/artifacts.cjs.map +1 -1
- package/dist/artifacts.js +233 -40
- package/dist/artifacts.js.map +1 -1
- package/dist/cli-runtime.cjs +230 -40
- package/dist/cli-runtime.cjs.map +1 -1
- package/dist/cli-runtime.js +231 -40
- package/dist/cli-runtime.js.map +1 -1
- package/dist/delegation-binding.cjs +232 -40
- package/dist/delegation-binding.cjs.map +1 -1
- package/dist/delegation-binding.js +233 -40
- package/dist/delegation-binding.js.map +1 -1
- package/dist/index.cjs +230 -40
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +231 -40
- package/dist/index.js.map +1 -1
- package/dist/profile.cjs +8 -4
- package/dist/profile.cjs.map +1 -1
- package/dist/profile.js +9 -4
- package/dist/profile.js.map +1 -1
- package/dist/state.cjs +232 -40
- package/dist/state.cjs.map +1 -1
- package/dist/state.js +233 -40
- package/dist/state.js.map +1 -1
- package/package.json +4 -4
package/coverage.json
CHANGED
|
@@ -45,6 +45,7 @@
|
|
|
45
45
|
{ "command": "replica status", "status": "legacy" },
|
|
46
46
|
{ "command": "replica report", "status": "legacy" },
|
|
47
47
|
{ "command": "replica reset", "status": "legacy" },
|
|
48
|
+
{ "command": "courier [args]", "status": "excluded", "reason": "Delegates argv to @tinycloud/courier; Courier's own agent identity and profile, not a TinyCloud operations surface." },
|
|
48
49
|
{ "command": "space", "status": "legacy" },
|
|
49
50
|
{ "command": "space list", "status": "legacy" },
|
|
50
51
|
{ "command": "space create <name>", "status": "legacy" },
|
package/dist/artifacts.cjs
CHANGED
|
@@ -406,7 +406,9 @@ function canonicalizeCaveats(caveats) {
|
|
|
406
406
|
// src/state.ts
|
|
407
407
|
var import_node_crypto = require("crypto");
|
|
408
408
|
var import_node_async_hooks = require("async_hooks");
|
|
409
|
+
var import_node_fs = require("fs");
|
|
409
410
|
var import_promises = require("fs/promises");
|
|
411
|
+
var import_node_net = require("net");
|
|
410
412
|
var import_node_os = require("os");
|
|
411
413
|
var import_node_path = require("path");
|
|
412
414
|
var DEFAULT_LOCK_TIMEOUT_MS = 2e3;
|
|
@@ -426,13 +428,171 @@ var TEST_LOCK_MOVED_BARRIER_DIR = "TC_TEST_PROFILE_LOCK_MOVED_BARRIER_DIR";
|
|
|
426
428
|
var TEST_LOCK_TURN_PUBLISH_BARRIER_DIR = "TC_TEST_PROFILE_LOCK_TURN_PUBLISH_BARRIER_DIR";
|
|
427
429
|
var TEST_LOCK_TURN_SETTLE_BARRIER_DIR = "TC_TEST_PROFILE_LOCK_TURN_SETTLE_BARRIER_DIR";
|
|
428
430
|
var TEST_LOCK_TURN_COLLECT_BARRIER_DIR = "TC_TEST_PROFILE_LOCK_TURN_COLLECT_BARRIER_DIR";
|
|
431
|
+
var TEST_LOCK_LIVENESS_BARRIER_DIR = "TC_TEST_PROFILE_LOCK_LIVENESS_BARRIER_DIR";
|
|
429
432
|
function processWideContext(key) {
|
|
430
|
-
var
|
|
433
|
+
var _a3;
|
|
431
434
|
const registry = globalThis;
|
|
432
|
-
return registry[
|
|
435
|
+
return registry[_a3 = Symbol.for(key)] ?? (registry[_a3] = new import_node_async_hooks.AsyncLocalStorage());
|
|
433
436
|
}
|
|
434
437
|
var invocationStateRoot = processWideContext("tinycloud.operations.invocationStateRoot.v1");
|
|
435
438
|
var heldProfileLocks = processWideContext("tinycloud.operations.heldProfileLocks.v2");
|
|
439
|
+
var _a;
|
|
440
|
+
var processInstanceId = globalThis[_a = /* @__PURE__ */ Symbol.for("tinycloud.operations.profileLockInstance.v1")] ?? (globalThis[_a] = (0, import_node_crypto.randomUUID)());
|
|
441
|
+
var cachedProcessOrigin;
|
|
442
|
+
function processOrigin() {
|
|
443
|
+
if (cachedProcessOrigin !== void 0) return cachedProcessOrigin;
|
|
444
|
+
if (process.platform !== "linux") return cachedProcessOrigin = null;
|
|
445
|
+
try {
|
|
446
|
+
const bootId = (0, import_node_fs.readFileSync)("/proc/sys/kernel/random/boot_id", "utf8").trim();
|
|
447
|
+
const pidNamespace = (0, import_node_fs.readlinkSync)("/proc/self/ns/pid");
|
|
448
|
+
const processStat = (0, import_node_fs.readFileSync)("/proc/self/stat", "utf8");
|
|
449
|
+
const closingParen = processStat.lastIndexOf(") ");
|
|
450
|
+
const processStartTime = closingParen < 0 ? void 0 : processStat.slice(closingParen + 2).trim().split(/\s+/)[19];
|
|
451
|
+
if (!/^[0-9a-f-]{36}$/.test(bootId) || !/^pid:\[[0-9]+\]$/.test(pidNamespace) || !processStartTime || !/^[0-9]+$/.test(processStartTime)) {
|
|
452
|
+
return cachedProcessOrigin = null;
|
|
453
|
+
}
|
|
454
|
+
return cachedProcessOrigin = { bootId, pidNamespace, processStartTime };
|
|
455
|
+
} catch {
|
|
456
|
+
return cachedProcessOrigin = null;
|
|
457
|
+
}
|
|
458
|
+
}
|
|
459
|
+
function processOwner(liveness) {
|
|
460
|
+
return { instanceId: processInstanceId, ...processOrigin(), ...liveness ? { liveness: liveness.record } : {} };
|
|
461
|
+
}
|
|
462
|
+
var LIVENESS_SOCKET = /^\.live-([0-9a-f]+)-([0-9a-f]+)-[0-9a-f]{8}-(?:[0-9a-f]{4}-){3}[0-9a-f]{12}(\.new)?$/;
|
|
463
|
+
var LIVENESS_PROBE_TIMEOUT_MS = 1e3;
|
|
464
|
+
function socketDirectory(name) {
|
|
465
|
+
const match = LIVENESS_SOCKET.exec(name);
|
|
466
|
+
return match ? `${match[1]}-${match[2]}` : null;
|
|
467
|
+
}
|
|
468
|
+
async function pinDirectory(directory) {
|
|
469
|
+
const handle = await (0, import_promises.open)(directory, "r");
|
|
470
|
+
try {
|
|
471
|
+
const { dev, ino } = await handle.stat({ bigint: true });
|
|
472
|
+
return {
|
|
473
|
+
identity: `${dev.toString(16)}-${ino.toString(16)}`,
|
|
474
|
+
path: (name) => `/proc/self/fd/${handle.fd}/${name}`,
|
|
475
|
+
close: () => handle.close()
|
|
476
|
+
};
|
|
477
|
+
} catch (error) {
|
|
478
|
+
await handle.close();
|
|
479
|
+
throw error;
|
|
480
|
+
}
|
|
481
|
+
}
|
|
482
|
+
async function bindLiveness(directory) {
|
|
483
|
+
if (!processOrigin()) return null;
|
|
484
|
+
let pinned;
|
|
485
|
+
try {
|
|
486
|
+
pinned = await pinDirectory(directory);
|
|
487
|
+
} catch {
|
|
488
|
+
return null;
|
|
489
|
+
}
|
|
490
|
+
const socket = `.live-${pinned.identity}-${(0, import_node_crypto.randomUUID)()}`;
|
|
491
|
+
const server = (0, import_node_net.createServer)((connection) => connection.destroy());
|
|
492
|
+
let bound = false;
|
|
493
|
+
try {
|
|
494
|
+
await new Promise((resolve2, reject) => {
|
|
495
|
+
server.once("error", reject);
|
|
496
|
+
server.listen({ path: pinned.path(`${socket}.new`), exclusive: true }, () => {
|
|
497
|
+
server.off("error", reject);
|
|
498
|
+
resolve2();
|
|
499
|
+
});
|
|
500
|
+
});
|
|
501
|
+
bound = true;
|
|
502
|
+
server.on("error", () => void 0);
|
|
503
|
+
server.unref();
|
|
504
|
+
await (0, import_promises.rename)(pinned.path(`${socket}.new`), pinned.path(socket));
|
|
505
|
+
const stats = await (0, import_promises.lstat)(pinned.path(socket), { bigint: true });
|
|
506
|
+
if (!stats.isSocket()) throw new Error(`${socket} is not a socket`);
|
|
507
|
+
await waitForTestBarrier(TEST_LOCK_LIVENESS_BARRIER_DIR, (0, import_node_path.basename)(directory));
|
|
508
|
+
return { record: { socket, dev: String(stats.dev), ino: String(stats.ino) }, path: (0, import_node_path.join)(directory, socket), server };
|
|
509
|
+
} catch {
|
|
510
|
+
if (bound) {
|
|
511
|
+
server.close();
|
|
512
|
+
for (const name of [`${socket}.new`, socket]) await (0, import_promises.unlink)(pinned.path(name)).catch(() => void 0);
|
|
513
|
+
}
|
|
514
|
+
return null;
|
|
515
|
+
} finally {
|
|
516
|
+
await pinned.close();
|
|
517
|
+
}
|
|
518
|
+
}
|
|
519
|
+
async function ensureLiveness(acquisition) {
|
|
520
|
+
if (acquisition.held === void 0) acquisition.held = await bindLiveness(acquisition.directory);
|
|
521
|
+
return acquisition.held;
|
|
522
|
+
}
|
|
523
|
+
async function closeLiveness(held) {
|
|
524
|
+
held.server.close();
|
|
525
|
+
await (0, import_promises.unlink)(held.path).catch(() => void 0);
|
|
526
|
+
}
|
|
527
|
+
function parseLiveness(value) {
|
|
528
|
+
if (typeof value !== "object" || value === null) return null;
|
|
529
|
+
const { socket, dev, ino } = value;
|
|
530
|
+
const name = typeof socket === "string" ? LIVENESS_SOCKET.exec(socket) : null;
|
|
531
|
+
return name !== null && name[3] === void 0 && typeof dev === "string" && /^[0-9]+$/.test(dev) && typeof ino === "string" && /^[0-9]+$/.test(ino) ? { socket: name[0], dev, ino } : null;
|
|
532
|
+
}
|
|
533
|
+
async function controlIn(pinned, acquisition) {
|
|
534
|
+
const own = await ensureLiveness(acquisition);
|
|
535
|
+
if (!own || socketDirectory(own.record.socket) !== pinned.identity) return null;
|
|
536
|
+
const stats = await (0, import_promises.lstat)(pinned.path(own.record.socket), { bigint: true }).catch(() => null);
|
|
537
|
+
return stats?.isSocket() && String(stats.dev) === own.record.dev && String(stats.ino) === own.record.ino ? own : null;
|
|
538
|
+
}
|
|
539
|
+
async function tryConnect(path) {
|
|
540
|
+
return new Promise((resolve2) => {
|
|
541
|
+
const probe = (0, import_node_net.connect)({ path });
|
|
542
|
+
const settle = (outcome) => {
|
|
543
|
+
clearTimeout(timer);
|
|
544
|
+
probe.destroy();
|
|
545
|
+
resolve2(outcome);
|
|
546
|
+
};
|
|
547
|
+
const timer = setTimeout(() => settle("other"), LIVENESS_PROBE_TIMEOUT_MS);
|
|
548
|
+
probe.once("connect", () => settle("connected"));
|
|
549
|
+
probe.once("error", (error) => settle(isErrno(error, "ECONNREFUSED") ? "refused" : "other"));
|
|
550
|
+
});
|
|
551
|
+
}
|
|
552
|
+
async function kernelRefuses(pinned, name, control) {
|
|
553
|
+
if (process.versions.bun !== void 0) return false;
|
|
554
|
+
return await tryConnect(pinned.path(name)) === "refused" && await tryConnect(pinned.path(control.record.socket)) === "connected";
|
|
555
|
+
}
|
|
556
|
+
async function probeLiveness(liveness, acquisition) {
|
|
557
|
+
let pinned;
|
|
558
|
+
try {
|
|
559
|
+
pinned = await pinDirectory(acquisition.directory);
|
|
560
|
+
} catch {
|
|
561
|
+
return "unknown";
|
|
562
|
+
}
|
|
563
|
+
try {
|
|
564
|
+
if (socketDirectory(liveness.socket) !== pinned.identity) return "unknown";
|
|
565
|
+
const control = await controlIn(pinned, acquisition);
|
|
566
|
+
if (!control) return "unknown";
|
|
567
|
+
let stats;
|
|
568
|
+
try {
|
|
569
|
+
stats = await (0, import_promises.lstat)(pinned.path(liveness.socket), { bigint: true });
|
|
570
|
+
} catch (error) {
|
|
571
|
+
return isErrno(error, "ENOENT") ? "gone" : "unknown";
|
|
572
|
+
}
|
|
573
|
+
if (!stats.isSocket() || String(stats.dev) !== liveness.dev || String(stats.ino) !== liveness.ino) return "unknown";
|
|
574
|
+
return await kernelRefuses(pinned, liveness.socket, control) ? "refused" : "unknown";
|
|
575
|
+
} finally {
|
|
576
|
+
await pinned.close();
|
|
577
|
+
}
|
|
578
|
+
}
|
|
579
|
+
async function sweepLiveness(acquisition) {
|
|
580
|
+
if (process.versions.bun !== void 0 || !acquisition.held) return;
|
|
581
|
+
const pinned = await pinDirectory(acquisition.directory);
|
|
582
|
+
try {
|
|
583
|
+
const control = await controlIn(pinned, acquisition);
|
|
584
|
+
if (!control) return;
|
|
585
|
+
for (const name of await (0, import_promises.readdir)(pinned.path("."))) {
|
|
586
|
+
if (name === control.record.socket || socketDirectory(name) !== pinned.identity) continue;
|
|
587
|
+
const stats = await (0, import_promises.lstat)(pinned.path(name)).catch(() => null);
|
|
588
|
+
if (stats?.isSocket() && await kernelRefuses(pinned, name, control)) {
|
|
589
|
+
await (0, import_promises.unlink)(pinned.path(name)).catch(() => void 0);
|
|
590
|
+
}
|
|
591
|
+
}
|
|
592
|
+
} finally {
|
|
593
|
+
await pinned.close();
|
|
594
|
+
}
|
|
595
|
+
}
|
|
436
596
|
var ProfileLockTimeoutError = class extends Error {
|
|
437
597
|
/** `detail` names the cause when it is not simply another process holding the lock. */
|
|
438
598
|
constructor(profile, timeoutMs, detail) {
|
|
@@ -597,7 +757,18 @@ async function acquireProfileLock(profile, options) {
|
|
|
597
757
|
if (path !== directory || !isErrno(error, "ENOENT")) throw error;
|
|
598
758
|
});
|
|
599
759
|
}
|
|
600
|
-
const
|
|
760
|
+
const liveness = { directory: turnDirectory, held: void 0 };
|
|
761
|
+
const closeAcquisitionLiveness = async () => {
|
|
762
|
+
if (liveness.held) await closeLiveness(liveness.held);
|
|
763
|
+
};
|
|
764
|
+
let taken;
|
|
765
|
+
try {
|
|
766
|
+
taken = await acquireTurn(profile, turnDirectory, deadline, liveness);
|
|
767
|
+
} catch (error) {
|
|
768
|
+
await closeAcquisitionLiveness();
|
|
769
|
+
throw error;
|
|
770
|
+
}
|
|
771
|
+
const { turn, firstWaitedTurn } = taken;
|
|
601
772
|
try {
|
|
602
773
|
if (options.requireProfile && !await exists(profileConfigPath(profile))) {
|
|
603
774
|
await (0, import_promises.rmdir)(directory).catch(() => void 0);
|
|
@@ -607,7 +778,7 @@ async function acquireProfileLock(profile, options) {
|
|
|
607
778
|
const deletedWhileWaiting = typeof deletion?.slot === "number" && deletion.slot >= firstWaitedTurn;
|
|
608
779
|
while (true) {
|
|
609
780
|
const token = (0, import_node_crypto.randomUUID)();
|
|
610
|
-
if (await publishProfileLock(profile, lockPath, token, options.requireProfile === true)) {
|
|
781
|
+
if (await publishProfileLock(profile, lockPath, token, options.requireProfile === true, liveness.held ?? null)) {
|
|
611
782
|
return {
|
|
612
783
|
deletedWhileWaiting,
|
|
613
784
|
turnSlot: turn.slot,
|
|
@@ -615,17 +786,19 @@ async function acquireProfileLock(profile, options) {
|
|
|
615
786
|
try {
|
|
616
787
|
await releaseProfileLock(profile, lockPath, token);
|
|
617
788
|
} finally {
|
|
618
|
-
await releaseTurn(profile, turn);
|
|
789
|
+
await releaseTurn(profile, turn, liveness);
|
|
790
|
+
await closeAcquisitionLiveness();
|
|
619
791
|
}
|
|
620
792
|
}
|
|
621
793
|
};
|
|
622
794
|
}
|
|
623
795
|
await signalTestLockContention(profile);
|
|
624
|
-
if (await recoverStaleLock(profile, lockPath, staleAfterMs)) continue;
|
|
796
|
+
if (await recoverStaleLock(profile, lockPath, liveness, staleAfterMs)) continue;
|
|
625
797
|
await waitOrTimeOut(profile, deadline);
|
|
626
798
|
}
|
|
627
799
|
} catch (error) {
|
|
628
|
-
await releaseTurn(profile, turn);
|
|
800
|
+
await releaseTurn(profile, turn, liveness);
|
|
801
|
+
await closeAcquisitionLiveness();
|
|
629
802
|
throw error;
|
|
630
803
|
}
|
|
631
804
|
}
|
|
@@ -639,12 +812,12 @@ async function waitOrTimeOut(profile, deadline, detail) {
|
|
|
639
812
|
await sleep(Math.min(deadline.retryMs, deadline.timeoutMs - (Date.now() - deadline.startedAt)));
|
|
640
813
|
}
|
|
641
814
|
var TURN = /^(?:0|[1-9][0-9]*)$/;
|
|
642
|
-
var
|
|
643
|
-
var relinquishedTurns = globalThis[
|
|
815
|
+
var _a2;
|
|
816
|
+
var relinquishedTurns = globalThis[_a2 = /* @__PURE__ */ Symbol.for("tinycloud.operations.relinquishedProfileTurns.v1")] ?? (globalThis[_a2] = /* @__PURE__ */ new Set());
|
|
644
817
|
var MARKER_RETRY_DELAYS_MS = [10, 50, 250];
|
|
645
818
|
var SETTLE_RETRY_MS = 1e3;
|
|
646
819
|
var SETTLE_RETRIES = 300;
|
|
647
|
-
async function acquireTurn(profile, directory, deadline) {
|
|
820
|
+
async function acquireTurn(profile, directory, deadline, liveness) {
|
|
648
821
|
let firstWaitedTurn;
|
|
649
822
|
let problem;
|
|
650
823
|
const damaged = (what) => `The turn lock ${directory} ${what}, so no process can take it. If no tc or MCP process is using profile "${profile}", remove that directory.`;
|
|
@@ -669,13 +842,13 @@ async function acquireTurn(profile, directory, deadline) {
|
|
|
669
842
|
}
|
|
670
843
|
firstWaitedTurn ?? (firstWaitedTurn = latest.done ? latest.slot + 1 : latest.slot);
|
|
671
844
|
if (latest.held && latest.done) {
|
|
672
|
-
const turn = await takeTurn(profile, directory, latest);
|
|
845
|
+
const turn = await takeTurn(profile, directory, latest, await ensureLiveness(liveness));
|
|
673
846
|
if (turn !== null) return { turn, firstWaitedTurn };
|
|
674
847
|
await waitOrTimeOut(profile, deadline);
|
|
675
848
|
} else if (latest.done) {
|
|
676
849
|
problem = damaged(`ends with turn ${latest.slot}, which was never granted`);
|
|
677
850
|
await waitOrTimeOut(profile, deadline, problem);
|
|
678
|
-
} else if (latest.pid === process.pid
|
|
851
|
+
} else if (latest.pid === process.pid && relinquishedTurns.has(latest.token) || !await isRecordedProcessAlive(latest, liveness)) {
|
|
679
852
|
await waitForTestBarrier(TEST_LOCK_TURN_SETTLE_BARRIER_DIR, profile);
|
|
680
853
|
await settleAbandonedTurn(directory, latest);
|
|
681
854
|
relinquishedTurns.delete(latest.token);
|
|
@@ -720,6 +893,7 @@ async function readTurn(directory, slot) {
|
|
|
720
893
|
const done = await exists((0, import_node_path.join)(path, `${owner.token}.done`));
|
|
721
894
|
const held = await exists((0, import_node_path.join)(path, `${owner.token}.held`));
|
|
722
895
|
return {
|
|
896
|
+
...owner,
|
|
723
897
|
slot,
|
|
724
898
|
token: owner.token,
|
|
725
899
|
pid: owner.pid,
|
|
@@ -733,11 +907,11 @@ async function isFinishedTurn(directory, slot, token) {
|
|
|
733
907
|
if (turn === "damaged") throw new Error(`Turn ${slot} of the turn lock ${directory} is unreadable.`);
|
|
734
908
|
return turn !== null && turn.token === token && turn.held && turn.done;
|
|
735
909
|
}
|
|
736
|
-
async function takeTurn(profile, directory, latest) {
|
|
910
|
+
async function takeTurn(profile, directory, latest, liveness) {
|
|
737
911
|
const token = (0, import_node_crypto.randomUUID)();
|
|
738
912
|
const slot = latest.slot + 1;
|
|
739
913
|
const path = (0, import_node_path.join)(directory, String(slot));
|
|
740
|
-
const owner = { pid: process.pid, createdAt: (/* @__PURE__ */ new Date()).toISOString(), token, after: latest.token };
|
|
914
|
+
const owner = { pid: process.pid, createdAt: (/* @__PURE__ */ new Date()).toISOString(), token, after: latest.token, ...processOwner(liveness) };
|
|
741
915
|
let published = false;
|
|
742
916
|
try {
|
|
743
917
|
published = await publishDirectory(
|
|
@@ -791,7 +965,7 @@ async function markTurn(path, token, kind) {
|
|
|
791
965
|
}
|
|
792
966
|
async function createTurnDirectory(directory) {
|
|
793
967
|
const token = (0, import_node_crypto.randomUUID)();
|
|
794
|
-
const owner = { pid: process.pid, createdAt: (/* @__PURE__ */ new Date()).toISOString(), token, after: null };
|
|
968
|
+
const owner = { pid: process.pid, createdAt: (/* @__PURE__ */ new Date()).toISOString(), token, after: null, ...processOwner(null) };
|
|
795
969
|
await publishDirectory((0, import_node_path.join)((0, import_node_path.dirname)(directory), `.stage-${(0, import_node_path.basename)(directory)}-${process.pid}-${token}`), directory, {
|
|
796
970
|
"0/owner.json": `${JSON.stringify(owner, null, 2)}
|
|
797
971
|
`,
|
|
@@ -815,31 +989,39 @@ async function publishDirectory(stage, target, files, beforeRename) {
|
|
|
815
989
|
await (0, import_promises.rm)(stage, { recursive: true, force: true }).catch(() => void 0);
|
|
816
990
|
}
|
|
817
991
|
}
|
|
818
|
-
async function releaseTurn(profile, turn) {
|
|
992
|
+
async function releaseTurn(profile, turn, liveness) {
|
|
819
993
|
try {
|
|
820
994
|
await markTurn((0, import_node_path.join)(turn.directory, String(turn.slot)), turn.token, "done");
|
|
821
995
|
} catch {
|
|
822
996
|
settleOwnTurn(turn.directory, turn.slot, turn.token);
|
|
823
997
|
return;
|
|
824
998
|
}
|
|
825
|
-
await collectTurns(profile, turn).catch(() => void 0);
|
|
999
|
+
await collectTurns(profile, turn, liveness).catch(() => void 0);
|
|
826
1000
|
}
|
|
827
|
-
function isAbandonedTurnEntry(name) {
|
|
1001
|
+
async function isAbandonedTurnEntry(directory, name, liveness) {
|
|
828
1002
|
if (name.startsWith(".trash-")) return true;
|
|
829
1003
|
const writer = /^\.(?:stage|deleted\.json)[-.]([0-9]+)[-.]/.exec(name)?.[1];
|
|
830
|
-
|
|
1004
|
+
if (writer === void 0) return false;
|
|
1005
|
+
if (Number(writer) !== process.pid) return !await isRecordedProcessAlive({ pid: Number(writer) }, liveness);
|
|
1006
|
+
return name.startsWith(".stage-") && await isStagedOwnerGone((0, import_node_path.join)(directory, name, "owner.json"), liveness);
|
|
831
1007
|
}
|
|
832
|
-
function isAbandonedTurnDirectoryStage(name) {
|
|
1008
|
+
async function isAbandonedTurnDirectoryStage(directory, name, liveness) {
|
|
833
1009
|
const writer = /^\.stage-.+-([0-9]+)-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/.exec(name)?.[1];
|
|
834
|
-
|
|
1010
|
+
if (writer === void 0) return false;
|
|
1011
|
+
if (Number(writer) !== process.pid) return !await isRecordedProcessAlive({ pid: Number(writer) }, liveness);
|
|
1012
|
+
return isStagedOwnerGone((0, import_node_path.join)(directory, name, "0", "owner.json"), liveness);
|
|
835
1013
|
}
|
|
836
|
-
async function
|
|
1014
|
+
async function isStagedOwnerGone(recordPath, liveness) {
|
|
1015
|
+
const owner = await readJson(recordPath).catch(() => null);
|
|
1016
|
+
return typeof owner === "object" && owner !== null && owner.pid === process.pid && !await isRecordedProcessAlive(owner, liveness);
|
|
1017
|
+
}
|
|
1018
|
+
async function collectTurns(profile, turn, liveness) {
|
|
837
1019
|
for (const [directory, abandoned] of [
|
|
838
1020
|
[turn.directory, isAbandonedTurnEntry],
|
|
839
1021
|
[(0, import_node_path.dirname)(turn.directory), isAbandonedTurnDirectoryStage]
|
|
840
1022
|
]) {
|
|
841
|
-
for (const name of
|
|
842
|
-
await (0, import_promises.rm)((0, import_node_path.join)(directory, name), { recursive: true, force: true }).catch(() => void 0);
|
|
1023
|
+
for (const name of await (0, import_promises.readdir)(directory)) {
|
|
1024
|
+
if (await abandoned(directory, name, liveness)) await (0, import_promises.rm)((0, import_node_path.join)(directory, name), { recursive: true, force: true }).catch(() => void 0);
|
|
843
1025
|
}
|
|
844
1026
|
}
|
|
845
1027
|
const older = (await (0, import_promises.readdir)(turn.directory)).filter((name) => TURN.test(name)).map(Number).filter((slot) => slot < turn.slot).sort((left, right) => left - right);
|
|
@@ -857,8 +1039,9 @@ async function collectTurns(profile, turn) {
|
|
|
857
1039
|
await (0, import_promises.rm)(trash, { recursive: true, force: true }).catch(() => void 0);
|
|
858
1040
|
}
|
|
859
1041
|
}
|
|
1042
|
+
await sweepLiveness(liveness);
|
|
860
1043
|
}
|
|
861
|
-
async function publishProfileLock(profile, lockPath, token, requireProfile) {
|
|
1044
|
+
async function publishProfileLock(profile, lockPath, token, requireProfile, liveness) {
|
|
862
1045
|
try {
|
|
863
1046
|
await (0, import_promises.mkdir)(lockPath, { mode: PRIVATE_DIR_MODE });
|
|
864
1047
|
} catch (error) {
|
|
@@ -871,7 +1054,7 @@ async function publishProfileLock(profile, lockPath, token, requireProfile) {
|
|
|
871
1054
|
await waitForTestBarrier(TEST_LOCK_PUBLISH_BARRIER_DIR, profile);
|
|
872
1055
|
const ownerPath = (0, import_node_path.join)(lockPath, "owner.json");
|
|
873
1056
|
const staged = (0, import_node_path.join)((0, import_node_path.dirname)(lockPath), `.lock-owner-${token}.tmp`);
|
|
874
|
-
const owner = `${JSON.stringify({ pid: process.pid, createdAt: (/* @__PURE__ */ new Date()).toISOString(), token }, null, 2)}
|
|
1057
|
+
const owner = `${JSON.stringify({ pid: process.pid, createdAt: (/* @__PURE__ */ new Date()).toISOString(), token, ...processOwner(liveness) }, null, 2)}
|
|
875
1058
|
`;
|
|
876
1059
|
try {
|
|
877
1060
|
await (0, import_promises.writeFile)(staged, owner, { encoding: "utf8", mode: PRIVATE_FILE_MODE, flag: "wx" });
|
|
@@ -916,7 +1099,7 @@ async function signalTestLockContention(profile) {
|
|
|
916
1099
|
await (0, import_promises.writeFile)(signalPath, `${profile}
|
|
917
1100
|
`, { encoding: "utf8", flag: "wx" }).catch(() => void 0);
|
|
918
1101
|
}
|
|
919
|
-
async function recoverOwnerlessLock(profile, lockPath, staleAfterMs) {
|
|
1102
|
+
async function recoverOwnerlessLock(profile, lockPath, liveness, staleAfterMs) {
|
|
920
1103
|
let aged;
|
|
921
1104
|
try {
|
|
922
1105
|
const { mtimeMs } = await (0, import_promises.stat)(lockPath);
|
|
@@ -933,7 +1116,7 @@ async function recoverOwnerlessLock(profile, lockPath, staleAfterMs) {
|
|
|
933
1116
|
if (!onlyOrphanedClaims && (!aged || recoveryFenced(checkedAt, staleAfterMs))) return false;
|
|
934
1117
|
for (const name of entries.filter((entry) => RECOVERY_CLAIM.test(entry))) {
|
|
935
1118
|
const claimPath = (0, import_node_path.join)(lockPath, name);
|
|
936
|
-
if (await hasLiveRecoveryOwner(claimPath)) {
|
|
1119
|
+
if (await hasLiveRecoveryOwner(claimPath, liveness)) {
|
|
937
1120
|
if (!await exists((0, import_node_path.join)(lockPath, "owner.json"))) await (0, import_promises.rename)(claimPath, (0, import_node_path.join)(lockPath, "owner.json"));
|
|
938
1121
|
return false;
|
|
939
1122
|
}
|
|
@@ -949,7 +1132,7 @@ async function recoverOwnerlessLock(profile, lockPath, staleAfterMs) {
|
|
|
949
1132
|
await removeAgedOwnerFiles(lockPath, staleAfterMs);
|
|
950
1133
|
return true;
|
|
951
1134
|
}
|
|
952
|
-
async function hasLiveRecoveryOwner(claimPath) {
|
|
1135
|
+
async function hasLiveRecoveryOwner(claimPath, liveness) {
|
|
953
1136
|
let contents;
|
|
954
1137
|
try {
|
|
955
1138
|
contents = await (0, import_promises.readFile)(claimPath, "utf8");
|
|
@@ -963,8 +1146,8 @@ async function hasLiveRecoveryOwner(claimPath) {
|
|
|
963
1146
|
} catch {
|
|
964
1147
|
return false;
|
|
965
1148
|
}
|
|
966
|
-
const
|
|
967
|
-
return typeof pid === "number" && Number.isInteger(pid) && pid > 0 &&
|
|
1149
|
+
const record = typeof owner === "object" && owner !== null && "pid" in owner ? owner : null;
|
|
1150
|
+
return typeof record?.pid === "number" && Number.isInteger(record.pid) && record.pid > 0 && await isRecordedProcessAlive(record, liveness);
|
|
968
1151
|
}
|
|
969
1152
|
function recoveryFenced(since, staleAfterMs) {
|
|
970
1153
|
return performance.now() - since >= staleAfterMs / 2;
|
|
@@ -981,11 +1164,11 @@ async function removeAgedOwnerFiles(lockPath, staleAfterMs) {
|
|
|
981
1164
|
if (aged) await (0, import_promises.rm)(path, { force: true }).catch(() => void 0);
|
|
982
1165
|
}
|
|
983
1166
|
}
|
|
984
|
-
async function recoverStaleLock(profile, lockPath, staleAfterMs) {
|
|
1167
|
+
async function recoverStaleLock(profile, lockPath, liveness, staleAfterMs) {
|
|
985
1168
|
const ownerPath = (0, import_node_path.join)(lockPath, "owner.json");
|
|
986
1169
|
const observed = await (0, import_promises.readFile)(ownerPath, "utf8").catch(() => null);
|
|
987
|
-
if (observed === null) return recoverOwnerlessLock(profile, lockPath, staleAfterMs);
|
|
988
|
-
if (!await isAbandonedOwner(observed, ownerPath, staleAfterMs)) return false;
|
|
1170
|
+
if (observed === null) return recoverOwnerlessLock(profile, lockPath, liveness, staleAfterMs);
|
|
1171
|
+
if (!await isAbandonedOwner(observed, ownerPath, liveness, staleAfterMs)) return false;
|
|
989
1172
|
await waitForTestBarrier(TEST_LOCK_RECOVERY_BARRIER_DIR, profile);
|
|
990
1173
|
const claimPath = (0, import_node_path.join)(lockPath, `.recover-${(0, import_node_crypto.randomUUID)()}.json`);
|
|
991
1174
|
let moved = false;
|
|
@@ -1021,7 +1204,7 @@ async function recoverStaleLock(profile, lockPath, staleAfterMs) {
|
|
|
1021
1204
|
await removeAgedOwnerFiles(lockPath, staleAfterMs);
|
|
1022
1205
|
return true;
|
|
1023
1206
|
}
|
|
1024
|
-
async function isAbandonedOwner(observed, ownerPath, staleAfterMs) {
|
|
1207
|
+
async function isAbandonedOwner(observed, ownerPath, liveness, staleAfterMs) {
|
|
1025
1208
|
let owner;
|
|
1026
1209
|
try {
|
|
1027
1210
|
owner = JSON.parse(observed);
|
|
@@ -1029,7 +1212,7 @@ async function isAbandonedOwner(observed, ownerPath, staleAfterMs) {
|
|
|
1029
1212
|
owner = null;
|
|
1030
1213
|
}
|
|
1031
1214
|
if (typeof owner === "object" && owner !== null && "pid" in owner) {
|
|
1032
|
-
return isStaleOwner(owner, staleAfterMs);
|
|
1215
|
+
return isStaleOwner(owner, liveness, staleAfterMs);
|
|
1033
1216
|
}
|
|
1034
1217
|
const modifiedAt = await (0, import_promises.stat)(ownerPath).then(({ mtimeMs }) => mtimeMs, () => Number.NaN);
|
|
1035
1218
|
return Date.now() - modifiedAt >= staleAfterMs;
|
|
@@ -1051,14 +1234,14 @@ async function waitForTestBarrier(environmentName, profile) {
|
|
|
1051
1234
|
}
|
|
1052
1235
|
}
|
|
1053
1236
|
}
|
|
1054
|
-
function isStaleOwner(owner, staleAfterMs) {
|
|
1237
|
+
async function isStaleOwner(owner, liveness, staleAfterMs) {
|
|
1055
1238
|
const now = Date.now();
|
|
1056
1239
|
const createdAt = typeof owner?.createdAt === "string" ? Date.parse(owner.createdAt) : Number.NaN;
|
|
1057
1240
|
if (!Number.isFinite(createdAt) || now - createdAt < staleAfterMs) return false;
|
|
1058
1241
|
if (typeof owner?.pid !== "number" || !Number.isInteger(owner.pid) || owner.pid <= 0) {
|
|
1059
1242
|
return false;
|
|
1060
1243
|
}
|
|
1061
|
-
return !
|
|
1244
|
+
return !await isRecordedProcessAlive(owner, liveness);
|
|
1062
1245
|
}
|
|
1063
1246
|
async function exists(path) {
|
|
1064
1247
|
try {
|
|
@@ -1069,13 +1252,22 @@ async function exists(path) {
|
|
|
1069
1252
|
throw error;
|
|
1070
1253
|
}
|
|
1071
1254
|
}
|
|
1072
|
-
function
|
|
1255
|
+
async function isRecordedProcessAlive(owner, acquisition) {
|
|
1256
|
+
const pid = owner.pid;
|
|
1257
|
+
if (typeof pid !== "number" || !Number.isInteger(pid) || pid <= 0) return true;
|
|
1073
1258
|
try {
|
|
1074
1259
|
process.kill(pid, 0);
|
|
1075
|
-
return true;
|
|
1076
1260
|
} catch (error) {
|
|
1077
1261
|
return !isErrno(error, "ESRCH");
|
|
1078
1262
|
}
|
|
1263
|
+
if (pid !== process.pid || typeof owner.instanceId !== "string" || !/^[0-9a-f]{8}-(?:[0-9a-f]{4}-){3}[0-9a-f]{12}$/.test(owner.instanceId) || owner.instanceId === processInstanceId) return true;
|
|
1264
|
+
const origin = processOrigin();
|
|
1265
|
+
if (!origin || owner.bootId !== origin.bootId || typeof owner.pidNamespace !== "string" || !/^pid:\[[0-9]+\]$/.test(owner.pidNamespace)) return true;
|
|
1266
|
+
if (owner.pidNamespace === origin.pidNamespace) {
|
|
1267
|
+
return typeof owner.processStartTime !== "string" || !/^[0-9]+$/.test(owner.processStartTime) || owner.processStartTime === origin.processStartTime;
|
|
1268
|
+
}
|
|
1269
|
+
const liveness = parseLiveness(owner.liveness);
|
|
1270
|
+
return liveness === null || await probeLiveness(liveness, acquisition) === "unknown";
|
|
1079
1271
|
}
|
|
1080
1272
|
function validateProfileName(profile) {
|
|
1081
1273
|
if (!profile || profile === "." || profile === ".." || profile.includes("/") || profile.includes("\\") || profile.includes("\0")) {
|