@proagentstore/cli 0.4.90 → 0.4.92
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +697 -219
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -1406,6 +1406,462 @@ function parseStatusLine(line) {
|
|
|
1406
1406
|
return status;
|
|
1407
1407
|
}
|
|
1408
1408
|
|
|
1409
|
+
// src/commands/runner/auto-update.ts
|
|
1410
|
+
import { existsSync as existsSync7, mkdirSync as mkdirSync6, readFileSync as readFileSync7, renameSync as renameSync2, writeFileSync as writeFileSync5 } from "fs";
|
|
1411
|
+
import { dirname as dirname3, join as join8 } from "path";
|
|
1412
|
+
|
|
1413
|
+
// src/runner-lock.ts
|
|
1414
|
+
import { createHash, randomUUID as randomUUID2 } from "crypto";
|
|
1415
|
+
import { execFileSync as execFileSync2 } from "child_process";
|
|
1416
|
+
import { closeSync, mkdirSync as mkdirSync5, openSync, readFileSync as readFileSync6, renameSync, unlinkSync as unlinkSync2, writeSync } from "fs";
|
|
1417
|
+
import { homedir as homedir3 } from "os";
|
|
1418
|
+
import { join as join7 } from "path";
|
|
1419
|
+
function configDir(env = process.env) {
|
|
1420
|
+
return env.PAGS_CONFIG_DIR?.trim() || join7(homedir3(), ".config", "proagentstore");
|
|
1421
|
+
}
|
|
1422
|
+
var accountKey = (account) => createHash("sha256").update(account.trim().toLowerCase()).digest("hex").slice(0, 12);
|
|
1423
|
+
var lockPath = (account, env = process.env) => join7(configDir(env), `up-${accountKey(account)}.lock`);
|
|
1424
|
+
function classifyHolder(facts) {
|
|
1425
|
+
const lock = facts.lock;
|
|
1426
|
+
if (!lock) return "unknown";
|
|
1427
|
+
if (lock.v !== 1) return "unknown";
|
|
1428
|
+
if (!lock.pid || !Number.isInteger(lock.pid)) return "unknown";
|
|
1429
|
+
if (!facts.pidAlive) return "dead";
|
|
1430
|
+
if (lock.pidStartedAt && facts.pidStartedAt && lock.pidStartedAt !== facts.pidStartedAt) return "dead";
|
|
1431
|
+
if (facts.healthRsid && facts.healthRsid === lock.rsid) return "live";
|
|
1432
|
+
return "unresponsive";
|
|
1433
|
+
}
|
|
1434
|
+
var isTakeable = (verdict) => verdict === "dead";
|
|
1435
|
+
function holderWhere(lock) {
|
|
1436
|
+
if (lock.launch === "tmux") return `tmux session ${lock.launchDetail || "?"}`;
|
|
1437
|
+
if (lock.launch === "service") return "a background service (launchd/systemd)";
|
|
1438
|
+
if (lock.launch === "headless") return "headless";
|
|
1439
|
+
return lock.launchDetail ? `terminal ${lock.launchDetail}` : "a terminal";
|
|
1440
|
+
}
|
|
1441
|
+
function holderMessage(lock, verdict) {
|
|
1442
|
+
const since = new Date(lock.startedAt).toLocaleString();
|
|
1443
|
+
const lines = [
|
|
1444
|
+
`A \`pags up\` is already running for this account on this machine.`,
|
|
1445
|
+
` pid ${lock.pid}, started ${since}, CLI ${lock.version || "?"}, in ${holderWhere(lock)}.`
|
|
1446
|
+
];
|
|
1447
|
+
if (verdict === "unresponsive") {
|
|
1448
|
+
lines.push(` It is NOT answering its health check \u2014 it may be asleep, stopped, or the machine may be overloaded.`);
|
|
1449
|
+
}
|
|
1450
|
+
if (verdict === "unknown") {
|
|
1451
|
+
lines.push(` Its lock file could not be read (a newer CLI may have written it), so it is assumed to be running.`);
|
|
1452
|
+
}
|
|
1453
|
+
lines.push("", " What you can do:");
|
|
1454
|
+
if (lock.launch === "tmux" && lock.launchDetail) lines.push(` tmux attach -t ${lock.launchDetail} \u2014 go to the one that is running`);
|
|
1455
|
+
lines.push(
|
|
1456
|
+
` pags status \u2014 what it is doing, and anything else running here`,
|
|
1457
|
+
lock.launch === "service" ? ` (this one is a service \u2014 stop its launchd/systemd unit rather than replacing it)` : ` pags up --replace \u2014 stop that one gracefully, then take over`
|
|
1458
|
+
);
|
|
1459
|
+
return lines;
|
|
1460
|
+
}
|
|
1461
|
+
function detectLaunch(env = process.env, isTty = Boolean(process.stdout.isTTY)) {
|
|
1462
|
+
if (env.PAGS_SERVICE === "1") return { launch: "service", detail: env.PAGS_SERVICE_NAME?.trim() || "" };
|
|
1463
|
+
if (env.TMUX) return { launch: "tmux", detail: env.TMUX_PANE?.trim() || env.TMUX.split(",")[0]?.split("/").pop() || "" };
|
|
1464
|
+
if (!isTty) return { launch: "headless", detail: "" };
|
|
1465
|
+
return { launch: "tty", detail: env.TERM_SESSION_ID?.trim() || env.WINDOWID?.trim() || "" };
|
|
1466
|
+
}
|
|
1467
|
+
function pidAlive(pid) {
|
|
1468
|
+
if (!pid || !Number.isInteger(pid) || pid <= 0) return false;
|
|
1469
|
+
try {
|
|
1470
|
+
process.kill(pid, 0);
|
|
1471
|
+
return true;
|
|
1472
|
+
} catch (e) {
|
|
1473
|
+
return e.code === "EPERM";
|
|
1474
|
+
}
|
|
1475
|
+
}
|
|
1476
|
+
function pidStartedAt(pid) {
|
|
1477
|
+
if (!pidAlive(pid)) return "";
|
|
1478
|
+
try {
|
|
1479
|
+
if (process.platform === "win32") {
|
|
1480
|
+
const out = execFileSync2("wmic", ["process", "where", `ProcessId=${pid}`, "get", "CreationDate", "/value"], { encoding: "utf-8", timeout: 4e3, stdio: ["ignore", "pipe", "ignore"] });
|
|
1481
|
+
return (out.match(/CreationDate=(\S+)/)?.[1] ?? "").trim();
|
|
1482
|
+
}
|
|
1483
|
+
return execFileSync2("ps", ["-o", "lstart=", "-p", String(pid)], { encoding: "utf-8", timeout: 4e3, stdio: ["ignore", "pipe", "ignore"] }).trim();
|
|
1484
|
+
} catch {
|
|
1485
|
+
return "";
|
|
1486
|
+
}
|
|
1487
|
+
}
|
|
1488
|
+
async function healthRsid(port, token = "", timeoutMs = 5e3, fetchImpl = fetch) {
|
|
1489
|
+
if (!port) return null;
|
|
1490
|
+
try {
|
|
1491
|
+
const res = await fetchImpl(`http://127.0.0.1:${port}/health`, { headers: token ? { Authorization: `Bearer ${token}` } : {}, signal: AbortSignal.timeout(timeoutMs) });
|
|
1492
|
+
if (!res.ok) return null;
|
|
1493
|
+
const body = await res.json();
|
|
1494
|
+
return typeof body.rsid === "string" && body.rsid ? body.rsid : null;
|
|
1495
|
+
} catch {
|
|
1496
|
+
return null;
|
|
1497
|
+
}
|
|
1498
|
+
}
|
|
1499
|
+
function readLock(path) {
|
|
1500
|
+
try {
|
|
1501
|
+
const parsed = JSON.parse(readFileSync6(path, "utf-8"));
|
|
1502
|
+
return parsed && typeof parsed === "object" && typeof parsed.rsid === "string" ? parsed : null;
|
|
1503
|
+
} catch {
|
|
1504
|
+
return null;
|
|
1505
|
+
}
|
|
1506
|
+
}
|
|
1507
|
+
async function inspectHolder(path, opts = {}) {
|
|
1508
|
+
const lock = readLock(path);
|
|
1509
|
+
if (!lock) return { lock: null, verdict: "unknown" };
|
|
1510
|
+
const alive = pidAlive(lock.pid);
|
|
1511
|
+
const facts = {
|
|
1512
|
+
lock,
|
|
1513
|
+
pidAlive: alive,
|
|
1514
|
+
pidStartedAt: alive ? pidStartedAt(lock.pid) : "",
|
|
1515
|
+
healthRsid: alive ? await healthRsid(lock.port, lock.runnerToken ?? "", opts.healthTimeoutMs ?? 5e3, opts.fetchImpl ?? fetch) : null
|
|
1516
|
+
};
|
|
1517
|
+
return { lock, verdict: classifyHolder(facts) };
|
|
1518
|
+
}
|
|
1519
|
+
async function acquireLock(input) {
|
|
1520
|
+
const env = input.env ?? process.env;
|
|
1521
|
+
const path = lockPath(input.account, env);
|
|
1522
|
+
const rsid = input.rsid ?? randomUUID2();
|
|
1523
|
+
const handoff = input.handoffFrom?.trim() || env.PAGS_LOCK_HANDOFF?.trim() || "";
|
|
1524
|
+
mkdirSync5(configDir(env), { recursive: true });
|
|
1525
|
+
for (let attempt = 0; attempt < 3; attempt++) {
|
|
1526
|
+
const written = tryWrite(path, buildLock(input, rsid, env));
|
|
1527
|
+
if (written) return { ok: true, lock: written, path, tookOver: attempt === 0 ? "none" : "stale" };
|
|
1528
|
+
const { lock: lock2, verdict: verdict2 } = await inspectHolder(path);
|
|
1529
|
+
if (handoff && lock2?.rsid === handoff) {
|
|
1530
|
+
renameAside(path, lock2.pid);
|
|
1531
|
+
const taken = tryWrite(path, buildLock(input, rsid, env));
|
|
1532
|
+
if (taken) return { ok: true, lock: taken, path, tookOver: "handoff" };
|
|
1533
|
+
continue;
|
|
1534
|
+
}
|
|
1535
|
+
if (!isTakeable(verdict2)) return { ok: false, lock: lock2, path, verdict: verdict2 };
|
|
1536
|
+
if (!takeAsideIfStill(path, lock2?.rsid ?? "", lock2?.pid ?? 0)) continue;
|
|
1537
|
+
}
|
|
1538
|
+
const { lock, verdict } = await inspectHolder(path);
|
|
1539
|
+
return { ok: false, lock, path, verdict };
|
|
1540
|
+
}
|
|
1541
|
+
function buildLock(input, rsid, env) {
|
|
1542
|
+
const { launch, detail } = detectLaunch(env);
|
|
1543
|
+
const now = Date.now();
|
|
1544
|
+
return {
|
|
1545
|
+
v: 1,
|
|
1546
|
+
rsid,
|
|
1547
|
+
pid: process.pid,
|
|
1548
|
+
pidStartedAt: pidStartedAt(process.pid),
|
|
1549
|
+
startedAt: now,
|
|
1550
|
+
port: input.port,
|
|
1551
|
+
node: input.node,
|
|
1552
|
+
machineId: input.machineId,
|
|
1553
|
+
account: input.account,
|
|
1554
|
+
version: input.version,
|
|
1555
|
+
launch,
|
|
1556
|
+
launchDetail: detail,
|
|
1557
|
+
childPids: [],
|
|
1558
|
+
nonce: randomUUID2(),
|
|
1559
|
+
runnerToken: input.runnerToken ?? "",
|
|
1560
|
+
heartbeatAt: now
|
|
1561
|
+
};
|
|
1562
|
+
}
|
|
1563
|
+
function tryWrite(path, lock) {
|
|
1564
|
+
let fd = null;
|
|
1565
|
+
try {
|
|
1566
|
+
fd = openSync(path, "wx", 384);
|
|
1567
|
+
writeSync(fd, JSON.stringify(lock, null, 2));
|
|
1568
|
+
return lock;
|
|
1569
|
+
} catch (e) {
|
|
1570
|
+
if (e.code === "EEXIST") return null;
|
|
1571
|
+
throw e;
|
|
1572
|
+
} finally {
|
|
1573
|
+
if (fd !== null) closeSync(fd);
|
|
1574
|
+
}
|
|
1575
|
+
}
|
|
1576
|
+
function renameAside(path, pid) {
|
|
1577
|
+
const aside = `${path}.stale-${pid || "unknown"}`;
|
|
1578
|
+
try {
|
|
1579
|
+
renameSync(path, aside);
|
|
1580
|
+
return aside;
|
|
1581
|
+
} catch {
|
|
1582
|
+
return null;
|
|
1583
|
+
}
|
|
1584
|
+
}
|
|
1585
|
+
function takeAsideIfStill(path, deadRsid, pid) {
|
|
1586
|
+
const aside = renameAside(path, pid);
|
|
1587
|
+
if (!aside) return false;
|
|
1588
|
+
const moved = readLock(aside);
|
|
1589
|
+
if (moved && deadRsid && moved.rsid !== deadRsid) {
|
|
1590
|
+
try {
|
|
1591
|
+
renameSync(aside, path);
|
|
1592
|
+
} catch {
|
|
1593
|
+
}
|
|
1594
|
+
return false;
|
|
1595
|
+
}
|
|
1596
|
+
return true;
|
|
1597
|
+
}
|
|
1598
|
+
function releaseLock(path, rsid) {
|
|
1599
|
+
const current = readLock(path);
|
|
1600
|
+
if (!current || current.rsid !== rsid) return false;
|
|
1601
|
+
try {
|
|
1602
|
+
unlinkSync2(path);
|
|
1603
|
+
return true;
|
|
1604
|
+
} catch {
|
|
1605
|
+
return false;
|
|
1606
|
+
}
|
|
1607
|
+
}
|
|
1608
|
+
|
|
1609
|
+
// src/commands/runner/auto-update.ts
|
|
1610
|
+
function autoUpdateStatusWire(status) {
|
|
1611
|
+
if (!status) return void 0;
|
|
1612
|
+
return {
|
|
1613
|
+
"waiting-for-idle": "waiting_for_idle",
|
|
1614
|
+
running: "installing",
|
|
1615
|
+
"verified-success": "verified_success",
|
|
1616
|
+
failure: "failed"
|
|
1617
|
+
}[status] ?? status;
|
|
1618
|
+
}
|
|
1619
|
+
var statuses = /* @__PURE__ */ new Set([
|
|
1620
|
+
"enabled",
|
|
1621
|
+
"checking",
|
|
1622
|
+
"waiting-for-idle",
|
|
1623
|
+
"running",
|
|
1624
|
+
"restarting",
|
|
1625
|
+
"verified-success",
|
|
1626
|
+
"offline",
|
|
1627
|
+
"unsupported",
|
|
1628
|
+
"failure"
|
|
1629
|
+
]);
|
|
1630
|
+
function parseStatus(value) {
|
|
1631
|
+
if (typeof value !== "string") return void 0;
|
|
1632
|
+
const readable = {
|
|
1633
|
+
waiting_for_idle: "waiting-for-idle",
|
|
1634
|
+
installing: "running",
|
|
1635
|
+
verified_success: "verified-success",
|
|
1636
|
+
failed: "failure"
|
|
1637
|
+
}[value] ?? value;
|
|
1638
|
+
return statuses.has(readable) ? readable : void 0;
|
|
1639
|
+
}
|
|
1640
|
+
function autoUpdatePolicyKey(machineId, env = process.env) {
|
|
1641
|
+
return `${env.PAGS_LOCK_ACCOUNT?.trim() || "default"}:${machineId.trim() || "unidentified"}`;
|
|
1642
|
+
}
|
|
1643
|
+
function autoUpdatePolicyPath(env = process.env) {
|
|
1644
|
+
return env.PAGS_AUTO_UPDATE_POLICY_FILE?.trim() || join8(configDir(env), "auto-update.json");
|
|
1645
|
+
}
|
|
1646
|
+
function parseAutoUpdatePolicy(value) {
|
|
1647
|
+
if (!value || typeof value !== "object") return null;
|
|
1648
|
+
const raw = value;
|
|
1649
|
+
const autoUpdate = raw.autoUpdate ?? raw.auto_update;
|
|
1650
|
+
if (typeof autoUpdate !== "boolean") return null;
|
|
1651
|
+
const status = parseStatus(raw.status ?? raw.autoUpdateStatus ?? raw.auto_update_status);
|
|
1652
|
+
const latestVersion = raw.latestVersion ?? raw.latest_version;
|
|
1653
|
+
const lastAttemptAt = raw.lastAttemptAt ?? raw.last_attempt_at;
|
|
1654
|
+
const reason = raw.reason;
|
|
1655
|
+
return {
|
|
1656
|
+
autoUpdate,
|
|
1657
|
+
...status ? { status } : {},
|
|
1658
|
+
...typeof latestVersion === "string" ? { latestVersion } : {},
|
|
1659
|
+
...typeof lastAttemptAt === "string" ? { lastAttemptAt } : {},
|
|
1660
|
+
...typeof reason === "string" ? { reason } : {}
|
|
1661
|
+
};
|
|
1662
|
+
}
|
|
1663
|
+
function policyFromResponse(value) {
|
|
1664
|
+
if (!value || typeof value !== "object") return null;
|
|
1665
|
+
const raw = value;
|
|
1666
|
+
const direct = raw.autoUpdatePolicy ?? raw.auto_update_policy ?? raw.policy;
|
|
1667
|
+
if (typeof direct === "boolean") {
|
|
1668
|
+
return parseAutoUpdatePolicy({
|
|
1669
|
+
autoUpdate: direct,
|
|
1670
|
+
status: raw.autoUpdateStatus ?? raw.auto_update_status,
|
|
1671
|
+
latestVersion: raw.latestVersion ?? raw.latest_version,
|
|
1672
|
+
lastAttemptAt: raw.lastAttemptAt ?? raw.last_attempt_at,
|
|
1673
|
+
reason: raw.reason
|
|
1674
|
+
});
|
|
1675
|
+
}
|
|
1676
|
+
return parseAutoUpdatePolicy(direct) ?? parseAutoUpdatePolicy(raw.machine && typeof raw.machine === "object" ? raw.machine.autoUpdatePolicy ?? raw.machine.auto_update_policy : void 0);
|
|
1677
|
+
}
|
|
1678
|
+
function loadAutoUpdatePolicy(key, env = process.env) {
|
|
1679
|
+
try {
|
|
1680
|
+
const data = JSON.parse(readFileSync7(autoUpdatePolicyPath(env), "utf8"));
|
|
1681
|
+
if (data.v !== 1 || !data.policies || typeof data.policies !== "object") return null;
|
|
1682
|
+
return parseAutoUpdatePolicy(data.policies[key]);
|
|
1683
|
+
} catch {
|
|
1684
|
+
return null;
|
|
1685
|
+
}
|
|
1686
|
+
}
|
|
1687
|
+
function saveAutoUpdatePolicy(key, policy, env = process.env) {
|
|
1688
|
+
const path = autoUpdatePolicyPath(env);
|
|
1689
|
+
try {
|
|
1690
|
+
let old = { v: 1, policies: {} };
|
|
1691
|
+
if (existsSync7(path)) {
|
|
1692
|
+
const parsed = JSON.parse(readFileSync7(path, "utf8"));
|
|
1693
|
+
if (parsed.v === 1 && parsed.policies && typeof parsed.policies === "object") old = parsed;
|
|
1694
|
+
}
|
|
1695
|
+
mkdirSync6(dirname3(path), { recursive: true });
|
|
1696
|
+
const next = { v: 1, policies: { ...old.policies, [key]: policy } };
|
|
1697
|
+
const temp = `${path}.${process.pid}.tmp`;
|
|
1698
|
+
writeFileSync5(temp, JSON.stringify(next, null, 2), { mode: 384 });
|
|
1699
|
+
renameSync2(temp, path);
|
|
1700
|
+
return true;
|
|
1701
|
+
} catch {
|
|
1702
|
+
return false;
|
|
1703
|
+
}
|
|
1704
|
+
}
|
|
1705
|
+
function nextAutoUpdateDelayMs(failures, random = Math.random) {
|
|
1706
|
+
const base = failures > 0 ? Math.min(60 * 6e4, 6e4 * 2 ** Math.min(failures - 1, 6)) : 6 * 60 * 6e4;
|
|
1707
|
+
const jitter = 0.85 + Math.max(0, Math.min(1, random())) * 0.3;
|
|
1708
|
+
return Math.round(base * jitter);
|
|
1709
|
+
}
|
|
1710
|
+
function policyScheduleAction(previous, next, wasAuthoritative) {
|
|
1711
|
+
if (!next.autoUpdate) return "cancel";
|
|
1712
|
+
if (!wasAuthoritative || !previous.autoUpdate) return "start";
|
|
1713
|
+
return "keep";
|
|
1714
|
+
}
|
|
1715
|
+
function mayAutomaticallyRestart(input) {
|
|
1716
|
+
return input.authoritative && input.enabled && input.workObserved && input.busy.length === 0;
|
|
1717
|
+
}
|
|
1718
|
+
|
|
1719
|
+
// src/commands/runner/auto-update-controller.ts
|
|
1720
|
+
var AutomaticUpdateController = class {
|
|
1721
|
+
constructor(deps) {
|
|
1722
|
+
this.deps = deps;
|
|
1723
|
+
this.setTimer = deps.setTimer ?? ((fn, ms) => setTimeout(fn, ms));
|
|
1724
|
+
this.clearTimer = deps.clearTimer ?? ((timer) => clearTimeout(timer));
|
|
1725
|
+
}
|
|
1726
|
+
deps;
|
|
1727
|
+
timer = null;
|
|
1728
|
+
checking = false;
|
|
1729
|
+
waiting = false;
|
|
1730
|
+
installing = false;
|
|
1731
|
+
failures = 0;
|
|
1732
|
+
setTimer;
|
|
1733
|
+
clearTimer;
|
|
1734
|
+
/** Called only for a true policy transition. `keep` deliberately leaves a due heartbeat timer alone. */
|
|
1735
|
+
onPolicy(action) {
|
|
1736
|
+
if (action === "cancel") return this.cancel();
|
|
1737
|
+
if (action === "start") this.schedule(true);
|
|
1738
|
+
}
|
|
1739
|
+
/** Testable immediate entry point; production reaches it through the scheduled timer. */
|
|
1740
|
+
async runNow() {
|
|
1741
|
+
const current = this.deps.policy();
|
|
1742
|
+
if (this.checking || this.waiting || !current.authoritative || !current.value.autoUpdate) return;
|
|
1743
|
+
this.checking = true;
|
|
1744
|
+
try {
|
|
1745
|
+
if (!await this.deps.refreshPolicy() || !this.enabled()) {
|
|
1746
|
+
this.deps.status("offline", { reason: "The owner policy could not be refreshed; automatic update is deferred." });
|
|
1747
|
+
return;
|
|
1748
|
+
}
|
|
1749
|
+
await this.decide(false);
|
|
1750
|
+
} catch (error) {
|
|
1751
|
+
this.fail(error);
|
|
1752
|
+
} finally {
|
|
1753
|
+
this.checking = false;
|
|
1754
|
+
}
|
|
1755
|
+
}
|
|
1756
|
+
/** Explicit remote updates and unattended checks share this single flight. */
|
|
1757
|
+
async installManually(plan) {
|
|
1758
|
+
return this.install(plan, false);
|
|
1759
|
+
}
|
|
1760
|
+
enabled() {
|
|
1761
|
+
const { value, authoritative } = this.deps.policy();
|
|
1762
|
+
return authoritative && value.autoUpdate;
|
|
1763
|
+
}
|
|
1764
|
+
schedule(initial = false) {
|
|
1765
|
+
if (!this.enabled() || this.timer || this.waiting) return;
|
|
1766
|
+
const delay = initial ? Math.round(5e3 + (this.deps.random ?? Math.random)() * 25e3) : nextAutoUpdateDelayMs(this.failures, this.deps.random);
|
|
1767
|
+
this.timer = this.setTimer(() => {
|
|
1768
|
+
this.timer = null;
|
|
1769
|
+
void this.runNow();
|
|
1770
|
+
}, delay);
|
|
1771
|
+
this.timer.unref?.();
|
|
1772
|
+
}
|
|
1773
|
+
cancel() {
|
|
1774
|
+
if (this.timer) this.clearTimer(this.timer);
|
|
1775
|
+
this.timer = null;
|
|
1776
|
+
}
|
|
1777
|
+
fail(error) {
|
|
1778
|
+
this.failures++;
|
|
1779
|
+
this.deps.status("failure", { reason: error instanceof Error ? error.message : String(error) });
|
|
1780
|
+
this.schedule();
|
|
1781
|
+
}
|
|
1782
|
+
async decide(deferred) {
|
|
1783
|
+
const facts = await this.deps.facts();
|
|
1784
|
+
if (!this.enabled()) return;
|
|
1785
|
+
const plan = this.deps.plan(facts);
|
|
1786
|
+
if (plan.action === "up-to-date") {
|
|
1787
|
+
this.failures = 0;
|
|
1788
|
+
this.deps.status("verified-success", { latestVersion: facts.latest ?? void 0, reason: void 0 });
|
|
1789
|
+
this.schedule();
|
|
1790
|
+
return;
|
|
1791
|
+
}
|
|
1792
|
+
if (plan.action === "refused") {
|
|
1793
|
+
this.deps.status("unsupported", { reason: plan.reason });
|
|
1794
|
+
this.schedule();
|
|
1795
|
+
return;
|
|
1796
|
+
}
|
|
1797
|
+
if (plan.action === "wait") {
|
|
1798
|
+
this.deps.status("waiting-for-idle", { latestVersion: plan.latest, reason: void 0 });
|
|
1799
|
+
this.defer();
|
|
1800
|
+
return;
|
|
1801
|
+
}
|
|
1802
|
+
this.deps.status("running", { latestVersion: plan.latest, reason: void 0 });
|
|
1803
|
+
const restarted = await this.install(plan, true);
|
|
1804
|
+
if (restarted) this.deps.status("restarting", { latestVersion: plan.latest, reason: void 0 });
|
|
1805
|
+
else if (!deferred) this.defer(plan.latest);
|
|
1806
|
+
}
|
|
1807
|
+
defer(latest) {
|
|
1808
|
+
if (this.waiting) return;
|
|
1809
|
+
this.waiting = true;
|
|
1810
|
+
this.timer = this.setTimer(() => {
|
|
1811
|
+
this.timer = null;
|
|
1812
|
+
void (async () => {
|
|
1813
|
+
this.waiting = false;
|
|
1814
|
+
try {
|
|
1815
|
+
if (!await this.deps.refreshPolicy() || !this.enabled()) {
|
|
1816
|
+
this.deps.status("offline", { reason: "The owner policy could not be refreshed; automatic update is deferred." });
|
|
1817
|
+
return;
|
|
1818
|
+
}
|
|
1819
|
+
await this.decide(true);
|
|
1820
|
+
} catch (error) {
|
|
1821
|
+
this.fail(error);
|
|
1822
|
+
}
|
|
1823
|
+
})();
|
|
1824
|
+
}, 15e3);
|
|
1825
|
+
this.timer.unref?.();
|
|
1826
|
+
this.deps.status("waiting-for-idle", latest ? { latestVersion: latest } : void 0);
|
|
1827
|
+
}
|
|
1828
|
+
async install(plan, automatic) {
|
|
1829
|
+
if (this.installing) return false;
|
|
1830
|
+
this.installing = true;
|
|
1831
|
+
try {
|
|
1832
|
+
return await this.deps.installAndRestart(plan, automatic);
|
|
1833
|
+
} finally {
|
|
1834
|
+
this.installing = false;
|
|
1835
|
+
}
|
|
1836
|
+
}
|
|
1837
|
+
};
|
|
1838
|
+
var RelayMutationAdmission = class {
|
|
1839
|
+
draining = false;
|
|
1840
|
+
inFlight = 0;
|
|
1841
|
+
waiters = [];
|
|
1842
|
+
begin() {
|
|
1843
|
+
if (this.draining) return null;
|
|
1844
|
+
this.inFlight++;
|
|
1845
|
+
let released = false;
|
|
1846
|
+
return () => {
|
|
1847
|
+
if (released) return;
|
|
1848
|
+
released = true;
|
|
1849
|
+
this.inFlight--;
|
|
1850
|
+
if (this.inFlight === 0) {
|
|
1851
|
+
for (const wake of this.waiters.splice(0)) wake();
|
|
1852
|
+
}
|
|
1853
|
+
};
|
|
1854
|
+
}
|
|
1855
|
+
async drain() {
|
|
1856
|
+
this.draining = true;
|
|
1857
|
+
if (this.inFlight === 0) return;
|
|
1858
|
+
await new Promise((resolve5) => this.waiters.push(resolve5));
|
|
1859
|
+
}
|
|
1860
|
+
resume() {
|
|
1861
|
+
this.draining = false;
|
|
1862
|
+
}
|
|
1863
|
+
};
|
|
1864
|
+
|
|
1409
1865
|
// src/commands/runner/relay.ts
|
|
1410
1866
|
var MEMBERSHIP_SYNC_PATH = "/pags/membership/sync";
|
|
1411
1867
|
var CLI_CONTROL_PATHS = /* @__PURE__ */ new Set([MEMBERSHIP_SYNC_PATH, RUNNER_UPDATE_PATH]);
|
|
@@ -1415,13 +1871,44 @@ async function connectViaRelay(instanceIds, localUrl, runnerToken, opts, force =
|
|
|
1415
1871
|
if (!pagsToken) throw new Error("PAGS token required for WebSocket relay");
|
|
1416
1872
|
const runnerNode = hostname3();
|
|
1417
1873
|
const machine = loadMachineIdentity(runnerNode);
|
|
1874
|
+
const autoUpdateKey = machine.id ? autoUpdatePolicyKey(machine.id) : "";
|
|
1875
|
+
let autoUpdatePolicy = autoUpdateKey ? loadAutoUpdatePolicy(autoUpdateKey) ?? { autoUpdate: false } : { autoUpdate: false };
|
|
1876
|
+
let autoUpdatePolicyAuthoritative = false;
|
|
1877
|
+
let refreshingAutomaticPolicy = false;
|
|
1878
|
+
let scheduleAutomaticUpdate = () => void 0;
|
|
1879
|
+
const saveAutoUpdateStatus = (status, extra = {}) => {
|
|
1880
|
+
autoUpdatePolicy = {
|
|
1881
|
+
...autoUpdatePolicy,
|
|
1882
|
+
...extra,
|
|
1883
|
+
status,
|
|
1884
|
+
lastAttemptAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
1885
|
+
};
|
|
1886
|
+
if (autoUpdateKey) saveAutoUpdatePolicy(autoUpdateKey, autoUpdatePolicy);
|
|
1887
|
+
};
|
|
1888
|
+
const acceptAutoUpdatePolicy = (response) => {
|
|
1889
|
+
if (!autoUpdateKey) return;
|
|
1890
|
+
const policy = policyFromResponse(response);
|
|
1891
|
+
if (!policy) return;
|
|
1892
|
+
const action = policyScheduleAction(autoUpdatePolicy, policy, autoUpdatePolicyAuthoritative);
|
|
1893
|
+
autoUpdatePolicy = { ...autoUpdatePolicy, ...policy };
|
|
1894
|
+
autoUpdatePolicyAuthoritative = true;
|
|
1895
|
+
if (autoUpdateKey) saveAutoUpdatePolicy(autoUpdateKey, autoUpdatePolicy);
|
|
1896
|
+
if (action === "cancel") {
|
|
1897
|
+
scheduleAutomaticUpdate(false, true);
|
|
1898
|
+
return;
|
|
1899
|
+
}
|
|
1900
|
+
if (refreshingAutomaticPolicy) return;
|
|
1901
|
+
if (action === "start") {
|
|
1902
|
+
scheduleAutomaticUpdate(true);
|
|
1903
|
+
}
|
|
1904
|
+
};
|
|
1418
1905
|
const capabilities = await requestRunner("GET", "/capabilities", { url: localUrl, token: runnerToken, instanceId: instanceIds[0] });
|
|
1419
1906
|
const caps = Array.isArray(capabilities.capabilities) ? capabilities.capabilities.filter((item) => typeof item === "string") : [];
|
|
1420
1907
|
const registered = /* @__PURE__ */ new Set();
|
|
1421
1908
|
let lastRegisterError = "";
|
|
1422
1909
|
const registerRuntime = async (id, forceClaim = force) => {
|
|
1423
1910
|
try {
|
|
1424
|
-
await requestPags("POST", `/v1/instances/${apiPathSegment(id)}/runtime`, opts, {
|
|
1911
|
+
const result = await requestPags("POST", `/v1/instances/${apiPathSegment(id)}/runtime`, opts, {
|
|
1425
1912
|
endpointUrl: localUrl,
|
|
1426
1913
|
token: runnerToken,
|
|
1427
1914
|
placement: "local",
|
|
@@ -1430,8 +1917,14 @@ async function connectViaRelay(instanceIds, localUrl, runnerToken, opts, force =
|
|
|
1430
1917
|
runnerNode,
|
|
1431
1918
|
machineId: machine.id,
|
|
1432
1919
|
machineNames: machine.names,
|
|
1433
|
-
force: forceClaim
|
|
1920
|
+
force: forceClaim,
|
|
1921
|
+
autoUpdateStatus: {
|
|
1922
|
+
status: autoUpdateStatusWire(autoUpdatePolicy.status),
|
|
1923
|
+
latestVersion: autoUpdatePolicy.latestVersion,
|
|
1924
|
+
error: autoUpdatePolicy.reason
|
|
1925
|
+
}
|
|
1434
1926
|
});
|
|
1927
|
+
acceptAutoUpdatePolicy(result);
|
|
1435
1928
|
registered.add(id);
|
|
1436
1929
|
return true;
|
|
1437
1930
|
} catch (e) {
|
|
@@ -1442,6 +1935,17 @@ async function connectViaRelay(instanceIds, localUrl, runnerToken, opts, force =
|
|
|
1442
1935
|
return false;
|
|
1443
1936
|
}
|
|
1444
1937
|
};
|
|
1938
|
+
const refreshAutomaticPolicy = async () => {
|
|
1939
|
+
const id = registered.values().next().value;
|
|
1940
|
+
if (!id) return false;
|
|
1941
|
+
autoUpdatePolicyAuthoritative = false;
|
|
1942
|
+
refreshingAutomaticPolicy = true;
|
|
1943
|
+
try {
|
|
1944
|
+
return await registerRuntime(id) && autoUpdatePolicyAuthoritative;
|
|
1945
|
+
} finally {
|
|
1946
|
+
refreshingAutomaticPolicy = false;
|
|
1947
|
+
}
|
|
1948
|
+
};
|
|
1445
1949
|
const attached = /* @__PURE__ */ new Map();
|
|
1446
1950
|
const processStartedAt = Date.now();
|
|
1447
1951
|
const starts24h = recordRunnerStart(processStartedAt);
|
|
@@ -1455,43 +1959,190 @@ async function connectViaRelay(instanceIds, localUrl, runnerToken, opts, force =
|
|
|
1455
1959
|
};
|
|
1456
1960
|
for (const id of instanceIds) await registerRuntime(id);
|
|
1457
1961
|
const updateFacts = async () => {
|
|
1458
|
-
const
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
|
|
1962
|
+
const [sessionsResult, healthResult] = await Promise.allSettled([
|
|
1963
|
+
requestRunner("GET", "/coding/sessions", {
|
|
1964
|
+
url: localUrl,
|
|
1965
|
+
token: runnerToken,
|
|
1966
|
+
instanceId: instanceIds[0]
|
|
1967
|
+
}),
|
|
1968
|
+
requestRunner("GET", "/health", { url: localUrl, token: runnerToken, instanceId: instanceIds[0] })
|
|
1969
|
+
]);
|
|
1970
|
+
const observationsUnavailable = sessionsResult.status !== "fulfilled" || healthResult.status !== "fulfilled" || !healthResult.value.work;
|
|
1971
|
+
const sessions = sessionsResult.status === "fulfilled" ? sessionsResult.value : { sessions: [] };
|
|
1972
|
+
const health = healthResult.status === "fulfilled" ? healthResult.value : {};
|
|
1973
|
+
const activeCoding = (sessions.sessions ?? []).filter((s) => s.alive && s.runState && s.runState !== "idle").map((s) => s.sessionId);
|
|
1974
|
+
const localRuns = health.work?.localRuns ?? 0;
|
|
1975
|
+
const codingTurns = health.work?.codingTurns ?? 0;
|
|
1463
1976
|
return {
|
|
1464
1977
|
current: CLI_VERSION,
|
|
1465
1978
|
latest: await latestPublishedVersion(),
|
|
1466
1979
|
fromSource: runsFromSource(),
|
|
1467
1980
|
restarter: restarterFrom(process.env),
|
|
1468
|
-
|
|
1981
|
+
// A restart ends browser/application work too. Give these durable names to the same
|
|
1982
|
+
// planner so neither manual nor automatic update can mistake them for an idle machine.
|
|
1983
|
+
busy: observationsUnavailable ? ["runner-work-observation-unavailable"] : [...activeCoding, ...Array.from({ length: codingTurns }, (_v, i) => `coding-turn-${i + 1}`), ...Array.from({ length: localRuns }, (_v, i) => health.work?.detail?.[i] || `local-run-${i + 1}`)]
|
|
1469
1984
|
};
|
|
1470
1985
|
};
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1986
|
+
let updateInstallInFlight = false;
|
|
1987
|
+
let installedPendingRestart = null;
|
|
1988
|
+
const relayMutationAdmission = new RelayMutationAdmission();
|
|
1989
|
+
const installAndRestart = async (plan, automatic = false) => {
|
|
1990
|
+
if (updateInstallInFlight) return false;
|
|
1991
|
+
updateInstallInFlight = true;
|
|
1992
|
+
if (automatic && (!await refreshAutomaticPolicy() || !autoUpdatePolicy.autoUpdate)) {
|
|
1993
|
+
updateInstallInFlight = false;
|
|
1994
|
+
return false;
|
|
1995
|
+
}
|
|
1996
|
+
if (installedPendingRestart !== plan.latest) {
|
|
1997
|
+
writeLine(`Updating ${plan.current} \u2192 ${plan.latest} (runner_update)\u2026`);
|
|
1998
|
+
try {
|
|
1999
|
+
await installVersion(plan.latest);
|
|
2000
|
+
installedPendingRestart = plan.latest;
|
|
2001
|
+
} catch (error) {
|
|
2002
|
+
updateInstallInFlight = false;
|
|
2003
|
+
throw error;
|
|
2004
|
+
}
|
|
2005
|
+
}
|
|
2006
|
+
if (automatic) {
|
|
2007
|
+
await relayMutationAdmission.drain();
|
|
2008
|
+
await new Promise((resolve5) => setTimeout(resolve5, 500));
|
|
2009
|
+
if (!await refreshAutomaticPolicy() || !autoUpdatePolicy.autoUpdate) {
|
|
2010
|
+
relayMutationAdmission.resume();
|
|
2011
|
+
updateInstallInFlight = false;
|
|
2012
|
+
return false;
|
|
2013
|
+
}
|
|
2014
|
+
}
|
|
2015
|
+
const finalFacts = await updateFacts();
|
|
2016
|
+
if (finalFacts.busy.length || automatic && !mayAutomaticallyRestart({ authoritative: autoUpdatePolicyAuthoritative, enabled: autoUpdatePolicy.autoUpdate, workObserved: !finalFacts.busy.includes("runner-work-observation-unavailable"), busy: finalFacts.busy })) {
|
|
2017
|
+
relayMutationAdmission.resume();
|
|
2018
|
+
updateInstallInFlight = false;
|
|
2019
|
+
return false;
|
|
2020
|
+
}
|
|
1474
2021
|
writeLine(`Installed ${plan.latest} \u2014 restarting; every agent re-attaches on the way back up.`);
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
}, 500).unref();
|
|
2022
|
+
for (const id of [...attached.keys()]) detach(id);
|
|
2023
|
+
leaveForRestart(plan.restarter);
|
|
2024
|
+
return true;
|
|
1479
2025
|
};
|
|
1480
2026
|
let updateWaiting = false;
|
|
1481
|
-
const updateWhenIdle = () => {
|
|
2027
|
+
const updateWhenIdle = (automatic = false) => {
|
|
1482
2028
|
if (updateWaiting) return;
|
|
1483
2029
|
updateWaiting = true;
|
|
1484
2030
|
const until = Date.now() + 60 * 6e4;
|
|
1485
2031
|
const tick = () => setTimeout(async () => {
|
|
2032
|
+
if (automatic && (!await refreshAutomaticPolicy() || !autoUpdatePolicy.autoUpdate)) {
|
|
2033
|
+
saveAutoUpdateStatus("offline", { reason: "The owner policy could not be refreshed; automatic update is deferred." });
|
|
2034
|
+
updateWaiting = false;
|
|
2035
|
+
return;
|
|
2036
|
+
}
|
|
1486
2037
|
const plan = planRunnerUpdate(await updateFacts());
|
|
1487
2038
|
if (plan.action === "update") {
|
|
1488
|
-
|
|
2039
|
+
if (automatic && !autoUpdatePolicy.autoUpdate) {
|
|
2040
|
+
updateWaiting = false;
|
|
2041
|
+
return;
|
|
2042
|
+
}
|
|
2043
|
+
if (automatic) saveAutoUpdateStatus("running", { latestVersion: plan.latest, reason: void 0 });
|
|
2044
|
+
const installed = await installAndRestart(plan, automatic).catch((e) => {
|
|
2045
|
+
if (automatic) saveAutoUpdateStatus("failure", { reason: e instanceof Error ? e.message : String(e) });
|
|
2046
|
+
writeError(`runner_update: install failed \u2014 ${e instanceof Error ? e.message : String(e)}`);
|
|
2047
|
+
return false;
|
|
2048
|
+
});
|
|
2049
|
+
if (automatic && !installed && Date.now() < until) {
|
|
2050
|
+
saveAutoUpdateStatus("waiting-for-idle", { latestVersion: plan.latest });
|
|
2051
|
+
tick();
|
|
2052
|
+
} else updateWaiting = false;
|
|
2053
|
+
} else if (plan.action === "wait" && Date.now() < until) {
|
|
2054
|
+
if (automatic) saveAutoUpdateStatus("waiting-for-idle", { latestVersion: plan.latest, reason: void 0 });
|
|
2055
|
+
tick();
|
|
2056
|
+
} else {
|
|
2057
|
+
if (automatic && plan.action === "refused") saveAutoUpdateStatus("unsupported", { reason: plan.reason });
|
|
1489
2058
|
updateWaiting = false;
|
|
1490
|
-
}
|
|
1491
|
-
else updateWaiting = false;
|
|
2059
|
+
}
|
|
1492
2060
|
}, 15e3).unref();
|
|
1493
2061
|
tick();
|
|
1494
2062
|
};
|
|
2063
|
+
let automaticTimer = null;
|
|
2064
|
+
let automaticChecking = false;
|
|
2065
|
+
let automaticFailures = 0;
|
|
2066
|
+
const runAutomaticUpdateCheck = async () => {
|
|
2067
|
+
if (!autoUpdatePolicy.autoUpdate || !autoUpdatePolicyAuthoritative || automaticChecking || updateWaiting) return;
|
|
2068
|
+
automaticChecking = true;
|
|
2069
|
+
try {
|
|
2070
|
+
if (!await refreshAutomaticPolicy() || !autoUpdatePolicy.autoUpdate) {
|
|
2071
|
+
saveAutoUpdateStatus("offline", { reason: "The owner policy could not be refreshed; automatic update is deferred." });
|
|
2072
|
+
scheduleAutomaticUpdate();
|
|
2073
|
+
return;
|
|
2074
|
+
}
|
|
2075
|
+
saveAutoUpdateStatus("checking", { reason: void 0 });
|
|
2076
|
+
const facts = await updateFacts();
|
|
2077
|
+
if (!autoUpdatePolicy.autoUpdate || !autoUpdatePolicyAuthoritative) return;
|
|
2078
|
+
if (!facts.latest) {
|
|
2079
|
+
automaticFailures++;
|
|
2080
|
+
saveAutoUpdateStatus("failure", { reason: "Could not check the trusted npm registry for a newer CLI release." });
|
|
2081
|
+
scheduleAutomaticUpdate();
|
|
2082
|
+
return;
|
|
2083
|
+
}
|
|
2084
|
+
automaticFailures = 0;
|
|
2085
|
+
const plan = planRunnerUpdate(facts);
|
|
2086
|
+
if (plan.action === "up-to-date") {
|
|
2087
|
+
saveAutoUpdateStatus("verified-success", { latestVersion: facts.latest, reason: void 0 });
|
|
2088
|
+
scheduleAutomaticUpdate();
|
|
2089
|
+
return;
|
|
2090
|
+
}
|
|
2091
|
+
if (plan.action === "refused") {
|
|
2092
|
+
saveAutoUpdateStatus("unsupported", { latestVersion: facts.latest, reason: plan.reason });
|
|
2093
|
+
scheduleAutomaticUpdate();
|
|
2094
|
+
return;
|
|
2095
|
+
}
|
|
2096
|
+
if (plan.action === "wait") {
|
|
2097
|
+
saveAutoUpdateStatus("waiting-for-idle", { latestVersion: plan.latest, reason: void 0 });
|
|
2098
|
+
updateWhenIdle(true);
|
|
2099
|
+
return;
|
|
2100
|
+
}
|
|
2101
|
+
if (!autoUpdatePolicy.autoUpdate || !autoUpdatePolicyAuthoritative) return;
|
|
2102
|
+
saveAutoUpdateStatus("running", { latestVersion: plan.latest, reason: void 0 });
|
|
2103
|
+
try {
|
|
2104
|
+
const installed = await installAndRestart(plan, true);
|
|
2105
|
+
if (!installed) {
|
|
2106
|
+
saveAutoUpdateStatus("waiting-for-idle", { latestVersion: plan.latest });
|
|
2107
|
+
updateWhenIdle(true);
|
|
2108
|
+
return;
|
|
2109
|
+
}
|
|
2110
|
+
saveAutoUpdateStatus("restarting", { latestVersion: plan.latest, reason: void 0 });
|
|
2111
|
+
} catch (e) {
|
|
2112
|
+
automaticFailures++;
|
|
2113
|
+
saveAutoUpdateStatus("failure", { latestVersion: plan.latest, reason: e instanceof Error ? e.message : String(e) });
|
|
2114
|
+
scheduleAutomaticUpdate();
|
|
2115
|
+
}
|
|
2116
|
+
} finally {
|
|
2117
|
+
automaticChecking = false;
|
|
2118
|
+
}
|
|
2119
|
+
};
|
|
2120
|
+
scheduleAutomaticUpdate = (initial = false, cancel = false) => {
|
|
2121
|
+
if (cancel || !autoUpdatePolicy.autoUpdate || !autoUpdatePolicyAuthoritative) {
|
|
2122
|
+
if (automaticTimer) clearTimeout(automaticTimer);
|
|
2123
|
+
automaticTimer = null;
|
|
2124
|
+
return;
|
|
2125
|
+
}
|
|
2126
|
+
if (automaticTimer || updateWaiting) return;
|
|
2127
|
+
const delay = initial ? Math.round(5e3 + Math.random() * 25e3) : nextAutoUpdateDelayMs(automaticFailures);
|
|
2128
|
+
automaticTimer = setTimeout(() => {
|
|
2129
|
+
automaticTimer = null;
|
|
2130
|
+
void runAutomaticUpdateCheck();
|
|
2131
|
+
}, delay);
|
|
2132
|
+
automaticTimer.unref();
|
|
2133
|
+
};
|
|
2134
|
+
const automaticUpdateController = new AutomaticUpdateController({
|
|
2135
|
+
policy: () => ({ value: autoUpdatePolicy, authoritative: autoUpdatePolicyAuthoritative }),
|
|
2136
|
+
refreshPolicy: refreshAutomaticPolicy,
|
|
2137
|
+
facts: updateFacts,
|
|
2138
|
+
plan: planRunnerUpdate,
|
|
2139
|
+
// This bridge is deliberately the same function used by explicit runner_update. It owns npm,
|
|
2140
|
+
// a fresh authoritative policy, the mutation drain, final local-work observation and restart.
|
|
2141
|
+
installAndRestart,
|
|
2142
|
+
status: saveAutoUpdateStatus
|
|
2143
|
+
});
|
|
2144
|
+
scheduleAutomaticUpdate = (initial = false, cancel = false) => automaticUpdateController.onPolicy(cancel ? "cancel" : initial ? "start" : "keep");
|
|
2145
|
+
if (autoUpdatePolicyAuthoritative && autoUpdatePolicy.autoUpdate) scheduleAutomaticUpdate(true);
|
|
1495
2146
|
const answerUpdate = async (body) => {
|
|
1496
2147
|
const dryRun = body?.dryRun === true;
|
|
1497
2148
|
const plan = planRunnerUpdate(await updateFacts());
|
|
@@ -1501,7 +2152,7 @@ async function connectViaRelay(instanceIds, localUrl, runnerToken, opts, force =
|
|
|
1501
2152
|
return { status: 200, result: { ...plan, detail: "Restarts itself as soon as these engines finish their turns \u2014 no run is cut off." } };
|
|
1502
2153
|
}
|
|
1503
2154
|
try {
|
|
1504
|
-
await
|
|
2155
|
+
if (!await automaticUpdateController.installManually(plan)) return { status: 409, result: { error: "A runner update is already installing or restarting." } };
|
|
1505
2156
|
} catch (e) {
|
|
1506
2157
|
return { status: 500, result: { error: `npm could not install ${plan.latest}: ${e instanceof Error ? e.message : String(e)}` } };
|
|
1507
2158
|
}
|
|
@@ -1567,7 +2218,10 @@ async function connectViaRelay(instanceIds, localUrl, runnerToken, opts, force =
|
|
|
1567
2218
|
reportRegistration();
|
|
1568
2219
|
},
|
|
1569
2220
|
answerControl,
|
|
1570
|
-
runnerNode
|
|
2221
|
+
runnerNode,
|
|
2222
|
+
// The final automatic-update drain is an admission gate, not merely another poll:
|
|
2223
|
+
// command work starts only when the old runner is still permitted to serve it.
|
|
2224
|
+
() => relayMutationAdmission.begin()
|
|
1571
2225
|
)
|
|
1572
2226
|
);
|
|
1573
2227
|
if (label) writeLine(`Attached agent: ${label}`);
|
|
@@ -1612,7 +2266,17 @@ async function connectViaRelay(instanceIds, localUrl, runnerToken, opts, force =
|
|
|
1612
2266
|
});
|
|
1613
2267
|
for (const id of [...attached.keys()]) {
|
|
1614
2268
|
try {
|
|
1615
|
-
await requestPags("POST", `/v1/instances/${apiPathSegment(id)}/runtime/heartbeat`, opts, {
|
|
2269
|
+
const response = await requestPags("POST", `/v1/instances/${apiPathSegment(id)}/runtime/heartbeat`, opts, {
|
|
2270
|
+
runnerNode,
|
|
2271
|
+
machineId: machine.id,
|
|
2272
|
+
resources,
|
|
2273
|
+
autoUpdateStatus: {
|
|
2274
|
+
status: autoUpdateStatusWire(autoUpdatePolicy.status),
|
|
2275
|
+
latestVersion: autoUpdatePolicy.latestVersion,
|
|
2276
|
+
error: autoUpdatePolicy.reason
|
|
2277
|
+
}
|
|
2278
|
+
});
|
|
2279
|
+
acceptAutoUpdatePolicy(response);
|
|
1616
2280
|
} catch (e) {
|
|
1617
2281
|
failure = e instanceof Error ? e.message : String(e);
|
|
1618
2282
|
}
|
|
@@ -1701,7 +2365,7 @@ async function readSessionResources(localUrl, runnerToken) {
|
|
|
1701
2365
|
}
|
|
1702
2366
|
var RTT_PROBE_PREFIX = "rtt:";
|
|
1703
2367
|
var RTT_ECHO_PREFIX = "rtt-echo:";
|
|
1704
|
-
function openRelaySocket(instanceId, wsBase, mintToken, localUrl, runnerToken, force = false, onConflict, onOpen, onControl, runnerNode = hostname3()) {
|
|
2368
|
+
function openRelaySocket(instanceId, wsBase, mintToken, localUrl, runnerToken, force = false, onConflict, onOpen, onControl, runnerNode = hostname3(), beginMutation = () => () => void 0) {
|
|
1705
2369
|
let backoffMs = 1e3;
|
|
1706
2370
|
let reconnecting = false;
|
|
1707
2371
|
let closed = false;
|
|
@@ -1769,6 +2433,14 @@ function openRelaySocket(instanceId, wsBase, mintToken, localUrl, runnerToken, f
|
|
|
1769
2433
|
return;
|
|
1770
2434
|
}
|
|
1771
2435
|
const method = (cmd.method || "POST").toUpperCase();
|
|
2436
|
+
const releaseMutation = method !== "GET" && method !== "HEAD" ? beginMutation() : null;
|
|
2437
|
+
if (method !== "GET" && method !== "HEAD" && !releaseMutation) {
|
|
2438
|
+
try {
|
|
2439
|
+
ws.send(JSON.stringify({ id: cmd.id, status: 503, result: { error: "Runner is draining for a safe automatic update; retry shortly." } }));
|
|
2440
|
+
} catch {
|
|
2441
|
+
}
|
|
2442
|
+
return;
|
|
2443
|
+
}
|
|
1772
2444
|
const hasBody = method !== "GET" && method !== "HEAD" && cmd.body !== void 0;
|
|
1773
2445
|
try {
|
|
1774
2446
|
const headers = {};
|
|
@@ -1796,6 +2468,8 @@ function openRelaySocket(instanceId, wsBase, mintToken, localUrl, runnerToken, f
|
|
|
1796
2468
|
ws.send(JSON.stringify({ id: cmd.id, status: 500, error: err instanceof Error ? err.message : String(err) }));
|
|
1797
2469
|
} catch {
|
|
1798
2470
|
}
|
|
2471
|
+
} finally {
|
|
2472
|
+
releaseMutation?.();
|
|
1799
2473
|
}
|
|
1800
2474
|
};
|
|
1801
2475
|
ws.onclose = (ev) => {
|
|
@@ -1862,202 +2536,6 @@ function openRelaySocket(instanceId, wsBase, mintToken, localUrl, runnerToken, f
|
|
|
1862
2536
|
};
|
|
1863
2537
|
}
|
|
1864
2538
|
|
|
1865
|
-
// src/runner-lock.ts
|
|
1866
|
-
import { createHash, randomUUID as randomUUID2 } from "crypto";
|
|
1867
|
-
import { execFileSync as execFileSync2 } from "child_process";
|
|
1868
|
-
import { closeSync, mkdirSync as mkdirSync5, openSync, readFileSync as readFileSync6, renameSync, unlinkSync as unlinkSync2, writeSync } from "fs";
|
|
1869
|
-
import { homedir as homedir3 } from "os";
|
|
1870
|
-
import { join as join7 } from "path";
|
|
1871
|
-
function configDir(env = process.env) {
|
|
1872
|
-
return env.PAGS_CONFIG_DIR?.trim() || join7(homedir3(), ".config", "proagentstore");
|
|
1873
|
-
}
|
|
1874
|
-
var accountKey = (account) => createHash("sha256").update(account.trim().toLowerCase()).digest("hex").slice(0, 12);
|
|
1875
|
-
var lockPath = (account, env = process.env) => join7(configDir(env), `up-${accountKey(account)}.lock`);
|
|
1876
|
-
function classifyHolder(facts) {
|
|
1877
|
-
const lock = facts.lock;
|
|
1878
|
-
if (!lock) return "unknown";
|
|
1879
|
-
if (lock.v !== 1) return "unknown";
|
|
1880
|
-
if (!lock.pid || !Number.isInteger(lock.pid)) return "unknown";
|
|
1881
|
-
if (!facts.pidAlive) return "dead";
|
|
1882
|
-
if (lock.pidStartedAt && facts.pidStartedAt && lock.pidStartedAt !== facts.pidStartedAt) return "dead";
|
|
1883
|
-
if (facts.healthRsid && facts.healthRsid === lock.rsid) return "live";
|
|
1884
|
-
return "unresponsive";
|
|
1885
|
-
}
|
|
1886
|
-
var isTakeable = (verdict) => verdict === "dead";
|
|
1887
|
-
function holderWhere(lock) {
|
|
1888
|
-
if (lock.launch === "tmux") return `tmux session ${lock.launchDetail || "?"}`;
|
|
1889
|
-
if (lock.launch === "service") return "a background service (launchd/systemd)";
|
|
1890
|
-
if (lock.launch === "headless") return "headless";
|
|
1891
|
-
return lock.launchDetail ? `terminal ${lock.launchDetail}` : "a terminal";
|
|
1892
|
-
}
|
|
1893
|
-
function holderMessage(lock, verdict) {
|
|
1894
|
-
const since = new Date(lock.startedAt).toLocaleString();
|
|
1895
|
-
const lines = [
|
|
1896
|
-
`A \`pags up\` is already running for this account on this machine.`,
|
|
1897
|
-
` pid ${lock.pid}, started ${since}, CLI ${lock.version || "?"}, in ${holderWhere(lock)}.`
|
|
1898
|
-
];
|
|
1899
|
-
if (verdict === "unresponsive") {
|
|
1900
|
-
lines.push(` It is NOT answering its health check \u2014 it may be asleep, stopped, or the machine may be overloaded.`);
|
|
1901
|
-
}
|
|
1902
|
-
if (verdict === "unknown") {
|
|
1903
|
-
lines.push(` Its lock file could not be read (a newer CLI may have written it), so it is assumed to be running.`);
|
|
1904
|
-
}
|
|
1905
|
-
lines.push("", " What you can do:");
|
|
1906
|
-
if (lock.launch === "tmux" && lock.launchDetail) lines.push(` tmux attach -t ${lock.launchDetail} \u2014 go to the one that is running`);
|
|
1907
|
-
lines.push(
|
|
1908
|
-
` pags status \u2014 what it is doing, and anything else running here`,
|
|
1909
|
-
lock.launch === "service" ? ` (this one is a service \u2014 stop its launchd/systemd unit rather than replacing it)` : ` pags up --replace \u2014 stop that one gracefully, then take over`
|
|
1910
|
-
);
|
|
1911
|
-
return lines;
|
|
1912
|
-
}
|
|
1913
|
-
function detectLaunch(env = process.env, isTty = Boolean(process.stdout.isTTY)) {
|
|
1914
|
-
if (env.PAGS_SERVICE === "1") return { launch: "service", detail: env.PAGS_SERVICE_NAME?.trim() || "" };
|
|
1915
|
-
if (env.TMUX) return { launch: "tmux", detail: env.TMUX_PANE?.trim() || env.TMUX.split(",")[0]?.split("/").pop() || "" };
|
|
1916
|
-
if (!isTty) return { launch: "headless", detail: "" };
|
|
1917
|
-
return { launch: "tty", detail: env.TERM_SESSION_ID?.trim() || env.WINDOWID?.trim() || "" };
|
|
1918
|
-
}
|
|
1919
|
-
function pidAlive(pid) {
|
|
1920
|
-
if (!pid || !Number.isInteger(pid) || pid <= 0) return false;
|
|
1921
|
-
try {
|
|
1922
|
-
process.kill(pid, 0);
|
|
1923
|
-
return true;
|
|
1924
|
-
} catch (e) {
|
|
1925
|
-
return e.code === "EPERM";
|
|
1926
|
-
}
|
|
1927
|
-
}
|
|
1928
|
-
function pidStartedAt(pid) {
|
|
1929
|
-
if (!pidAlive(pid)) return "";
|
|
1930
|
-
try {
|
|
1931
|
-
if (process.platform === "win32") {
|
|
1932
|
-
const out = execFileSync2("wmic", ["process", "where", `ProcessId=${pid}`, "get", "CreationDate", "/value"], { encoding: "utf-8", timeout: 4e3, stdio: ["ignore", "pipe", "ignore"] });
|
|
1933
|
-
return (out.match(/CreationDate=(\S+)/)?.[1] ?? "").trim();
|
|
1934
|
-
}
|
|
1935
|
-
return execFileSync2("ps", ["-o", "lstart=", "-p", String(pid)], { encoding: "utf-8", timeout: 4e3, stdio: ["ignore", "pipe", "ignore"] }).trim();
|
|
1936
|
-
} catch {
|
|
1937
|
-
return "";
|
|
1938
|
-
}
|
|
1939
|
-
}
|
|
1940
|
-
async function healthRsid(port, token = "", timeoutMs = 5e3, fetchImpl = fetch) {
|
|
1941
|
-
if (!port) return null;
|
|
1942
|
-
try {
|
|
1943
|
-
const res = await fetchImpl(`http://127.0.0.1:${port}/health`, { headers: token ? { Authorization: `Bearer ${token}` } : {}, signal: AbortSignal.timeout(timeoutMs) });
|
|
1944
|
-
if (!res.ok) return null;
|
|
1945
|
-
const body = await res.json();
|
|
1946
|
-
return typeof body.rsid === "string" && body.rsid ? body.rsid : null;
|
|
1947
|
-
} catch {
|
|
1948
|
-
return null;
|
|
1949
|
-
}
|
|
1950
|
-
}
|
|
1951
|
-
function readLock(path) {
|
|
1952
|
-
try {
|
|
1953
|
-
const parsed = JSON.parse(readFileSync6(path, "utf-8"));
|
|
1954
|
-
return parsed && typeof parsed === "object" && typeof parsed.rsid === "string" ? parsed : null;
|
|
1955
|
-
} catch {
|
|
1956
|
-
return null;
|
|
1957
|
-
}
|
|
1958
|
-
}
|
|
1959
|
-
async function inspectHolder(path, opts = {}) {
|
|
1960
|
-
const lock = readLock(path);
|
|
1961
|
-
if (!lock) return { lock: null, verdict: "unknown" };
|
|
1962
|
-
const alive = pidAlive(lock.pid);
|
|
1963
|
-
const facts = {
|
|
1964
|
-
lock,
|
|
1965
|
-
pidAlive: alive,
|
|
1966
|
-
pidStartedAt: alive ? pidStartedAt(lock.pid) : "",
|
|
1967
|
-
healthRsid: alive ? await healthRsid(lock.port, lock.runnerToken ?? "", opts.healthTimeoutMs ?? 5e3, opts.fetchImpl ?? fetch) : null
|
|
1968
|
-
};
|
|
1969
|
-
return { lock, verdict: classifyHolder(facts) };
|
|
1970
|
-
}
|
|
1971
|
-
async function acquireLock(input) {
|
|
1972
|
-
const env = input.env ?? process.env;
|
|
1973
|
-
const path = lockPath(input.account, env);
|
|
1974
|
-
const rsid = input.rsid ?? randomUUID2();
|
|
1975
|
-
const handoff = input.handoffFrom?.trim() || env.PAGS_LOCK_HANDOFF?.trim() || "";
|
|
1976
|
-
mkdirSync5(configDir(env), { recursive: true });
|
|
1977
|
-
for (let attempt = 0; attempt < 3; attempt++) {
|
|
1978
|
-
const written = tryWrite(path, buildLock(input, rsid, env));
|
|
1979
|
-
if (written) return { ok: true, lock: written, path, tookOver: attempt === 0 ? "none" : "stale" };
|
|
1980
|
-
const { lock: lock2, verdict: verdict2 } = await inspectHolder(path);
|
|
1981
|
-
if (handoff && lock2?.rsid === handoff) {
|
|
1982
|
-
renameAside(path, lock2.pid);
|
|
1983
|
-
const taken = tryWrite(path, buildLock(input, rsid, env));
|
|
1984
|
-
if (taken) return { ok: true, lock: taken, path, tookOver: "handoff" };
|
|
1985
|
-
continue;
|
|
1986
|
-
}
|
|
1987
|
-
if (!isTakeable(verdict2)) return { ok: false, lock: lock2, path, verdict: verdict2 };
|
|
1988
|
-
if (!takeAsideIfStill(path, lock2?.rsid ?? "", lock2?.pid ?? 0)) continue;
|
|
1989
|
-
}
|
|
1990
|
-
const { lock, verdict } = await inspectHolder(path);
|
|
1991
|
-
return { ok: false, lock, path, verdict };
|
|
1992
|
-
}
|
|
1993
|
-
function buildLock(input, rsid, env) {
|
|
1994
|
-
const { launch, detail } = detectLaunch(env);
|
|
1995
|
-
const now = Date.now();
|
|
1996
|
-
return {
|
|
1997
|
-
v: 1,
|
|
1998
|
-
rsid,
|
|
1999
|
-
pid: process.pid,
|
|
2000
|
-
pidStartedAt: pidStartedAt(process.pid),
|
|
2001
|
-
startedAt: now,
|
|
2002
|
-
port: input.port,
|
|
2003
|
-
node: input.node,
|
|
2004
|
-
machineId: input.machineId,
|
|
2005
|
-
account: input.account,
|
|
2006
|
-
version: input.version,
|
|
2007
|
-
launch,
|
|
2008
|
-
launchDetail: detail,
|
|
2009
|
-
childPids: [],
|
|
2010
|
-
nonce: randomUUID2(),
|
|
2011
|
-
runnerToken: input.runnerToken ?? "",
|
|
2012
|
-
heartbeatAt: now
|
|
2013
|
-
};
|
|
2014
|
-
}
|
|
2015
|
-
function tryWrite(path, lock) {
|
|
2016
|
-
let fd = null;
|
|
2017
|
-
try {
|
|
2018
|
-
fd = openSync(path, "wx", 384);
|
|
2019
|
-
writeSync(fd, JSON.stringify(lock, null, 2));
|
|
2020
|
-
return lock;
|
|
2021
|
-
} catch (e) {
|
|
2022
|
-
if (e.code === "EEXIST") return null;
|
|
2023
|
-
throw e;
|
|
2024
|
-
} finally {
|
|
2025
|
-
if (fd !== null) closeSync(fd);
|
|
2026
|
-
}
|
|
2027
|
-
}
|
|
2028
|
-
function renameAside(path, pid) {
|
|
2029
|
-
const aside = `${path}.stale-${pid || "unknown"}`;
|
|
2030
|
-
try {
|
|
2031
|
-
renameSync(path, aside);
|
|
2032
|
-
return aside;
|
|
2033
|
-
} catch {
|
|
2034
|
-
return null;
|
|
2035
|
-
}
|
|
2036
|
-
}
|
|
2037
|
-
function takeAsideIfStill(path, deadRsid, pid) {
|
|
2038
|
-
const aside = renameAside(path, pid);
|
|
2039
|
-
if (!aside) return false;
|
|
2040
|
-
const moved = readLock(aside);
|
|
2041
|
-
if (moved && deadRsid && moved.rsid !== deadRsid) {
|
|
2042
|
-
try {
|
|
2043
|
-
renameSync(aside, path);
|
|
2044
|
-
} catch {
|
|
2045
|
-
}
|
|
2046
|
-
return false;
|
|
2047
|
-
}
|
|
2048
|
-
return true;
|
|
2049
|
-
}
|
|
2050
|
-
function releaseLock(path, rsid) {
|
|
2051
|
-
const current = readLock(path);
|
|
2052
|
-
if (!current || current.rsid !== rsid) return false;
|
|
2053
|
-
try {
|
|
2054
|
-
unlinkSync2(path);
|
|
2055
|
-
return true;
|
|
2056
|
-
} catch {
|
|
2057
|
-
return false;
|
|
2058
|
-
}
|
|
2059
|
-
}
|
|
2060
|
-
|
|
2061
2539
|
// src/commands/runner/command.ts
|
|
2062
2540
|
import { hostname as hostname4 } from "os";
|
|
2063
2541
|
function collectCapability(value, previous = []) {
|