@indigoai-us/hq-cli 5.345.168 → 5.345.169
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +9 -0
- package/dist/commands/core-checkpoint-native.js +30 -10
- package/dist/commands/feedback.js +11 -3
- package/dist/lib/doctor/checks/sync-health.d.ts +7 -1
- package/dist/lib/doctor/checks/sync-health.js +67 -2
- package/dist/lib/doctor/fix/apply.d.ts +40 -4
- package/dist/lib/doctor/fix/apply.js +183 -69
- package/dist/lib/doctor/fix/process-identity.d.ts +13 -0
- package/dist/lib/doctor/fix/process-identity.js +34 -0
- package/dist/lib/doctor/fix/refresh-sync-handoff.d.ts +201 -0
- package/dist/lib/doctor/fix/refresh-sync-handoff.js +752 -0
- package/dist/lib/doctor/fix/refresh-sync-outcome.d.ts +1 -1
- package/dist/lib/doctor/fix/refresh-sync-outcome.js +13 -0
- package/dist/lib/doctor/types.d.ts +1 -1
- package/dist/lib/doctor/types.js +2 -0
- package/dist/sentry-before-send.js +13 -0
- package/dist/utils/breadcrumb-buffer.js +15 -4
- package/dist/utils/feedback-diagnostics.js +5 -2
- package/dist/utils/feedback-logs.d.ts +7 -0
- package/dist/utils/feedback-logs.js +37 -3
- package/dist/utils/scrub-diagnostics.d.ts +4 -0
- package/dist/utils/scrub-diagnostics.js +95 -0
- package/package.json +7 -7
|
@@ -0,0 +1,752 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Supervised `hq doctor --fix` journal pulls (feedback cd56a501).
|
|
3
|
+
*
|
|
4
|
+
* A pull records `lastSync` only when its whole pass finishes. `--fix` used to
|
|
5
|
+
* kill its pull at 90s, so a large vault never refreshed its journal and the
|
|
6
|
+
* client-health hook reported it every day.
|
|
7
|
+
*
|
|
8
|
+
* Every `--fix` pull now runs under a small supervisor process. The
|
|
9
|
+
* supervisor writes a heartbeat while the pull runs, the pull's pid, and an
|
|
10
|
+
* exit file with the pull's real status when it ends. It stops the pull at
|
|
11
|
+
* {@link REFRESH_BACKGROUND_MAX_MS}. If the pull is still running at the
|
|
12
|
+
* foreground deadline, `--fix` leaves the supervisor running.
|
|
13
|
+
*
|
|
14
|
+
* One pull per HQ root. A run claims the root before it spawns
|
|
15
|
+
* ({@link claimRefreshRun}): holding an O_EXCL spawn lock, it checks that no
|
|
16
|
+
* other pull occupies the root and creates the pull's record exclusively. A
|
|
17
|
+
* second `--fix` started at the same moment either cannot get the lock or
|
|
18
|
+
* sees that record, and does not spawn.
|
|
19
|
+
*
|
|
20
|
+
* State lives in one file per company under
|
|
21
|
+
* `workspace/doctor/refresh-in-progress/`, and each run's output, heartbeat,
|
|
22
|
+
* pid and exit files are named by a per-run id.
|
|
23
|
+
*
|
|
24
|
+
* The next `--fix` harvests finished runs first: it reads the exit file and
|
|
25
|
+
* the log tail, reports the real outcome, and removes the record. A pull whose
|
|
26
|
+
* supervisor died is still alive while its process group is, until the cap.
|
|
27
|
+
* Past the cap the next run removes it as capped, and kills it only when the
|
|
28
|
+
* caller's identity check proves the pid is still that pull.
|
|
29
|
+
*/
|
|
30
|
+
import * as crypto from "node:crypto";
|
|
31
|
+
import * as fs from "node:fs";
|
|
32
|
+
import * as path from "node:path";
|
|
33
|
+
const STATE_DIR = path.join("workspace", "doctor", "refresh-in-progress");
|
|
34
|
+
const RUNS_DIR = path.join("workspace", "doctor", "refresh-runs");
|
|
35
|
+
const SPAWN_LOCK = path.join("workspace", "doctor", "refresh-spawn.lock");
|
|
36
|
+
/** The supervisor stops a background pull after this long. */
|
|
37
|
+
export const REFRESH_BACKGROUND_MAX_MS = 10 * 60 * 1000;
|
|
38
|
+
/** The supervisor refreshes its heartbeat this often. */
|
|
39
|
+
export const REFRESH_HEARTBEAT_INTERVAL_MS = 10_000;
|
|
40
|
+
/** A heartbeat older than this means the supervisor is gone. */
|
|
41
|
+
export const REFRESH_HEARTBEAT_STALE_MS = 60_000;
|
|
42
|
+
/** After SIGTERM, the supervisor waits this long before SIGKILL of the pull's process group. */
|
|
43
|
+
export const REFRESH_KILL_GRACE_MS = 5_000;
|
|
44
|
+
/**
|
|
45
|
+
* A record older than this is over the cap whatever its heartbeat says. The
|
|
46
|
+
* supervisor stops the pull at {@link REFRESH_BACKGROUND_MAX_MS} and kills it
|
|
47
|
+
* {@link REFRESH_KILL_GRACE_MS} later; this adds a heartbeat interval of slack.
|
|
48
|
+
*/
|
|
49
|
+
export const REFRESH_RECORD_MAX_AGE_MS = REFRESH_BACKGROUND_MAX_MS + REFRESH_KILL_GRACE_MS + 15_000;
|
|
50
|
+
export function safeRefreshSlug(slug) {
|
|
51
|
+
return slug.replace(/[^A-Za-z0-9._-]/g, "_").slice(0, 120) || "_";
|
|
52
|
+
}
|
|
53
|
+
/** Fresh per-run file names under workspace/doctor/refresh-runs/. */
|
|
54
|
+
export function newRefreshRunPaths(hqRoot, slug) {
|
|
55
|
+
const id = `${safeRefreshSlug(slug)}.${Date.now()}.${crypto.randomBytes(4).toString("hex")}`;
|
|
56
|
+
return runPaths(hqRoot, id);
|
|
57
|
+
}
|
|
58
|
+
function runPaths(hqRoot, id) {
|
|
59
|
+
const dir = path.join(hqRoot, RUNS_DIR);
|
|
60
|
+
return {
|
|
61
|
+
id,
|
|
62
|
+
log: path.join(dir, `${id}.log`),
|
|
63
|
+
exit: path.join(dir, `${id}.exit`),
|
|
64
|
+
heartbeat: path.join(dir, `${id}.hb`),
|
|
65
|
+
pid: path.join(dir, `${id}.pid`),
|
|
66
|
+
start: path.join(dir, `${id}.start`),
|
|
67
|
+
cancel: path.join(dir, `${id}.cancel`),
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
function recordPath(hqRoot, slug) {
|
|
71
|
+
return path.join(hqRoot, STATE_DIR, `${safeRefreshSlug(slug)}.json`);
|
|
72
|
+
}
|
|
73
|
+
function recordJson(slug, run, nowMs, token) {
|
|
74
|
+
const record = {
|
|
75
|
+
version: 1,
|
|
76
|
+
slug,
|
|
77
|
+
id: run.id,
|
|
78
|
+
startedAtMs: nowMs,
|
|
79
|
+
ownerPid: process.pid,
|
|
80
|
+
token,
|
|
81
|
+
};
|
|
82
|
+
return JSON.stringify(record);
|
|
83
|
+
}
|
|
84
|
+
/** Write (or replace) the record for a pull of `slug`. Tests and recovery only; claims use {@link claimRefreshRun}. */
|
|
85
|
+
export function recordRefreshHandoff(hqRoot, slug, run, nowMs) {
|
|
86
|
+
const file = recordPath(hqRoot, slug);
|
|
87
|
+
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
88
|
+
const tmp = `${file}.${crypto.randomBytes(6).toString("hex")}.tmp`;
|
|
89
|
+
fs.writeFileSync(tmp, recordJson(slug, run, nowMs, crypto.randomBytes(8).toString("hex")), { mode: 0o600 });
|
|
90
|
+
fs.renameSync(tmp, file);
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* Create the record for `slug` only if none exists: the content is written to
|
|
94
|
+
* a temp file and hard-linked into place, which fails if the name is taken,
|
|
95
|
+
* so a reader never sees a partial record and an existing one is never
|
|
96
|
+
* overwritten. Returns false when a record already exists.
|
|
97
|
+
*/
|
|
98
|
+
function createRecordExclusive(hqRoot, slug, run, nowMs, token) {
|
|
99
|
+
const file = recordPath(hqRoot, slug);
|
|
100
|
+
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
101
|
+
const tmp = `${file}.${crypto.randomBytes(6).toString("hex")}.tmp`;
|
|
102
|
+
const json = recordJson(slug, run, nowMs, token);
|
|
103
|
+
fs.writeFileSync(tmp, json, { mode: 0o600, flag: "wx" });
|
|
104
|
+
try {
|
|
105
|
+
fs.linkSync(tmp, file);
|
|
106
|
+
return true;
|
|
107
|
+
}
|
|
108
|
+
catch (error) {
|
|
109
|
+
if (error.code === "EEXIST")
|
|
110
|
+
return false;
|
|
111
|
+
// Hard links are not available on every filesystem (exFAT, FAT, some
|
|
112
|
+
// network drives: EPERM, ENOTSUP, EXDEV, ...). Fall back to an exclusive
|
|
113
|
+
// create of the record itself: still never an overwrite. A reader can
|
|
114
|
+
// briefly see it half-written; harvest leaves young unreadable records
|
|
115
|
+
// alone (see UNREADABLE_RECORD_SETTLE_MS).
|
|
116
|
+
}
|
|
117
|
+
finally {
|
|
118
|
+
fs.rmSync(tmp, { force: true });
|
|
119
|
+
}
|
|
120
|
+
try {
|
|
121
|
+
fs.writeFileSync(file, json, { mode: 0o600, flag: "wx" });
|
|
122
|
+
return true;
|
|
123
|
+
}
|
|
124
|
+
catch (error) {
|
|
125
|
+
if (error.code === "EEXIST")
|
|
126
|
+
return false;
|
|
127
|
+
throw error;
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
/**
|
|
131
|
+
* An unreadable record younger than this may still be mid-write (the
|
|
132
|
+
* exclusive-create fallback), so harvest does not delete it yet.
|
|
133
|
+
*/
|
|
134
|
+
export const UNREADABLE_RECORD_SETTLE_MS = 10_000;
|
|
135
|
+
/** Remove the record for `slug` if it still names `run`. */
|
|
136
|
+
export function clearRefreshRecord(hqRoot, slug, run) {
|
|
137
|
+
const file = recordPath(hqRoot, slug);
|
|
138
|
+
const record = readRecord(file);
|
|
139
|
+
if (record && record.id === run.id)
|
|
140
|
+
fs.rmSync(file, { force: true });
|
|
141
|
+
}
|
|
142
|
+
function readRecord(file) {
|
|
143
|
+
try {
|
|
144
|
+
const parsed = JSON.parse(fs.readFileSync(file, "utf8"));
|
|
145
|
+
if (parsed?.version === 1 &&
|
|
146
|
+
typeof parsed.slug === "string" &&
|
|
147
|
+
typeof parsed.id === "string" &&
|
|
148
|
+
/^[A-Za-z0-9._-]+$/.test(parsed.id) &&
|
|
149
|
+
typeof parsed.startedAtMs === "number" &&
|
|
150
|
+
(parsed.ownerPid === undefined || (typeof parsed.ownerPid === "number" && Number.isInteger(parsed.ownerPid))) &&
|
|
151
|
+
(parsed.token === undefined || typeof parsed.token === "string")) {
|
|
152
|
+
return parsed;
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
catch {
|
|
156
|
+
// unreadable
|
|
157
|
+
}
|
|
158
|
+
return null;
|
|
159
|
+
}
|
|
160
|
+
/** Read a supervisor exit file, or null if the run has not ended. */
|
|
161
|
+
export function readRefreshRunExit(run) {
|
|
162
|
+
try {
|
|
163
|
+
const parsed = JSON.parse(fs.readFileSync(run.exit, "utf8"));
|
|
164
|
+
return {
|
|
165
|
+
status: typeof parsed.status === "number" ? parsed.status : null,
|
|
166
|
+
signal: typeof parsed.signal === "string" ? parsed.signal : null,
|
|
167
|
+
...(parsed.preempted === true ? { preempted: true } : {}),
|
|
168
|
+
...(parsed.capped === true ? { capped: true } : {}),
|
|
169
|
+
};
|
|
170
|
+
}
|
|
171
|
+
catch {
|
|
172
|
+
return null;
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
/** Last 64 KiB of a run's output: enough to classify, bounded in memory. */
|
|
176
|
+
export function readRefreshRunLog(run) {
|
|
177
|
+
try {
|
|
178
|
+
const size = fs.statSync(run.log).size;
|
|
179
|
+
const length = Math.min(size, 64 * 1024);
|
|
180
|
+
const buffer = Buffer.alloc(length);
|
|
181
|
+
const fd = fs.openSync(run.log, "r");
|
|
182
|
+
try {
|
|
183
|
+
fs.readSync(fd, buffer, 0, length, size - length);
|
|
184
|
+
}
|
|
185
|
+
finally {
|
|
186
|
+
fs.closeSync(fd);
|
|
187
|
+
}
|
|
188
|
+
return buffer.toString("utf8");
|
|
189
|
+
}
|
|
190
|
+
catch {
|
|
191
|
+
return "";
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
function readHeartbeat(run) {
|
|
195
|
+
try {
|
|
196
|
+
return fs.readFileSync(run.heartbeat, "utf8");
|
|
197
|
+
}
|
|
198
|
+
catch {
|
|
199
|
+
return null;
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
function heartbeatFresh(run, nowMs) {
|
|
203
|
+
const text = readHeartbeat(run);
|
|
204
|
+
if (text === null)
|
|
205
|
+
return false;
|
|
206
|
+
const beat = Number(text);
|
|
207
|
+
return Number.isFinite(beat) && nowMs - beat <= REFRESH_HEARTBEAT_STALE_MS && beat <= nowMs + 60_000;
|
|
208
|
+
}
|
|
209
|
+
/**
|
|
210
|
+
* True while a capped run is still inside its kill grace period: the
|
|
211
|
+
* supervisor has stopped heartbeating but has not written its exit file yet.
|
|
212
|
+
*/
|
|
213
|
+
function cappedAndStopping(run, nowMs) {
|
|
214
|
+
const text = readHeartbeat(run);
|
|
215
|
+
if (!text?.startsWith("capped:"))
|
|
216
|
+
return false;
|
|
217
|
+
const at = Number(text.slice("capped:".length));
|
|
218
|
+
return Number.isFinite(at) && nowMs - at <= REFRESH_KILL_GRACE_MS + 10_000;
|
|
219
|
+
}
|
|
220
|
+
function readPullPid(run) {
|
|
221
|
+
try {
|
|
222
|
+
const pid = Number(fs.readFileSync(run.pid, "utf8"));
|
|
223
|
+
return Number.isInteger(pid) && pid > 0 ? pid : null;
|
|
224
|
+
}
|
|
225
|
+
catch {
|
|
226
|
+
return null;
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
function processExists(pid) {
|
|
230
|
+
try {
|
|
231
|
+
process.kill(pid, 0);
|
|
232
|
+
return true;
|
|
233
|
+
}
|
|
234
|
+
catch (error) {
|
|
235
|
+
return error.code === "EPERM";
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
/**
|
|
239
|
+
* True when the pull of a run whose supervisor died is still running. On POSIX
|
|
240
|
+
* the pull leads its own process group, so the group is probed; elsewhere the
|
|
241
|
+
* pid. Only called within the record's age cap, which bounds pid reuse.
|
|
242
|
+
*/
|
|
243
|
+
function orphanPullAlive(run) {
|
|
244
|
+
const pid = readPullPid(run);
|
|
245
|
+
if (pid === null)
|
|
246
|
+
return false;
|
|
247
|
+
return process.platform === "win32" ? processExists(pid) : processExists(-pid);
|
|
248
|
+
}
|
|
249
|
+
/** An orphan is only ever killed within this long of its record. */
|
|
250
|
+
export const REFRESH_ORPHAN_KILL_WINDOW_MS = REFRESH_RECORD_MAX_AGE_MS + 5 * 60 * 1000;
|
|
251
|
+
function orphanOf(run, record) {
|
|
252
|
+
const pid = readPullPid(run);
|
|
253
|
+
if (pid === null)
|
|
254
|
+
return null;
|
|
255
|
+
let recordedStart = null;
|
|
256
|
+
try {
|
|
257
|
+
recordedStart = fs.readFileSync(run.start, "utf8").trim() || null;
|
|
258
|
+
}
|
|
259
|
+
catch {
|
|
260
|
+
recordedStart = null;
|
|
261
|
+
}
|
|
262
|
+
return { pid, recordedStart, recordStartedAtMs: record.startedAtMs };
|
|
263
|
+
}
|
|
264
|
+
/** Delete a run's files. */
|
|
265
|
+
export function removeRefreshRun(run) {
|
|
266
|
+
for (const file of [run.log, run.exit, run.heartbeat, run.pid, run.start, run.cancel]) {
|
|
267
|
+
try {
|
|
268
|
+
fs.rmSync(file, { force: true });
|
|
269
|
+
}
|
|
270
|
+
catch {
|
|
271
|
+
// ignore
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
function listRecords(hqRoot) {
|
|
276
|
+
const dir = path.join(hqRoot, STATE_DIR);
|
|
277
|
+
let names;
|
|
278
|
+
try {
|
|
279
|
+
names = fs.readdirSync(dir).filter((name) => name.endsWith(".json"));
|
|
280
|
+
}
|
|
281
|
+
catch {
|
|
282
|
+
return [];
|
|
283
|
+
}
|
|
284
|
+
return names.map((name) => {
|
|
285
|
+
const file = path.join(dir, name);
|
|
286
|
+
return { file, record: readRecord(file) };
|
|
287
|
+
});
|
|
288
|
+
}
|
|
289
|
+
/**
|
|
290
|
+
* Admission occupancy of one recorded run: true while its pull may still be
|
|
291
|
+
* running, including a capped run inside its stop and kill grace window. A
|
|
292
|
+
* new pull is never admitted while this is true.
|
|
293
|
+
*/
|
|
294
|
+
function recordOccupiesRoot(hqRoot, record, nowMs) {
|
|
295
|
+
if (recordAlive(hqRoot, record, nowMs))
|
|
296
|
+
return true;
|
|
297
|
+
if (nowMs - record.startedAtMs > REFRESH_RECORD_MAX_AGE_MS)
|
|
298
|
+
return false;
|
|
299
|
+
const run = runPaths(hqRoot, record.id);
|
|
300
|
+
return !readRefreshRunExit(run) && cappedAndStopping(run, nowMs);
|
|
301
|
+
}
|
|
302
|
+
/**
|
|
303
|
+
* In-progress status of one recorded run (what the journal check shows). A
|
|
304
|
+
* capped run is not in progress, even while it is being stopped.
|
|
305
|
+
*/
|
|
306
|
+
function recordAlive(hqRoot, record, nowMs) {
|
|
307
|
+
const age = nowMs - record.startedAtMs;
|
|
308
|
+
if (age > REFRESH_RECORD_MAX_AGE_MS)
|
|
309
|
+
return false;
|
|
310
|
+
const run = runPaths(hqRoot, record.id);
|
|
311
|
+
if (readRefreshRunExit(run))
|
|
312
|
+
return false;
|
|
313
|
+
if (heartbeatFresh(run, nowMs))
|
|
314
|
+
return true;
|
|
315
|
+
const beat = readHeartbeat(run);
|
|
316
|
+
// Claimed but the supervisor has not written its first heartbeat yet: the
|
|
317
|
+
// claiming process owns it while that process is alive.
|
|
318
|
+
if (beat === null && typeof record.ownerPid === "number" && record.ownerPid > 0 && processExists(record.ownerPid)) {
|
|
319
|
+
return true;
|
|
320
|
+
}
|
|
321
|
+
if (beat?.startsWith("capped:"))
|
|
322
|
+
return false;
|
|
323
|
+
// The supervisor is gone; the pull itself may still be running.
|
|
324
|
+
return orphanPullAlive(run);
|
|
325
|
+
}
|
|
326
|
+
/**
|
|
327
|
+
* Remove every run that is no longer alive and return what each one ended
|
|
328
|
+
* with. `exit` is null when the supervisor died without writing one. A run
|
|
329
|
+
* over the cap is reported as capped, and an orphaned pull still running past
|
|
330
|
+
* the cap is killed. Live runs are left in place. Unreadable records are
|
|
331
|
+
* deleted.
|
|
332
|
+
*/
|
|
333
|
+
export function harvestRefreshHandoffs(hqRoot, nowMs, killOrphan) {
|
|
334
|
+
const harvested = [];
|
|
335
|
+
for (const { file, record } of listRecords(hqRoot)) {
|
|
336
|
+
if (!record) {
|
|
337
|
+
let young = false;
|
|
338
|
+
try {
|
|
339
|
+
young = nowMs - fs.statSync(file).mtimeMs < UNREADABLE_RECORD_SETTLE_MS;
|
|
340
|
+
}
|
|
341
|
+
catch {
|
|
342
|
+
young = false;
|
|
343
|
+
}
|
|
344
|
+
if (!young)
|
|
345
|
+
fs.rmSync(file, { force: true });
|
|
346
|
+
continue;
|
|
347
|
+
}
|
|
348
|
+
const run = runPaths(hqRoot, record.id);
|
|
349
|
+
const exit = readRefreshRunExit(run);
|
|
350
|
+
const overCap = nowMs - record.startedAtMs > REFRESH_RECORD_MAX_AGE_MS;
|
|
351
|
+
if (!exit && !overCap && recordAlive(hqRoot, record, nowMs))
|
|
352
|
+
continue;
|
|
353
|
+
// Capped and being killed: wait for its exit file rather than guess.
|
|
354
|
+
if (!exit && !overCap && cappedAndStopping(run, nowMs))
|
|
355
|
+
continue;
|
|
356
|
+
if (!exit && overCap && killOrphan && nowMs - record.startedAtMs <= REFRESH_ORPHAN_KILL_WINDOW_MS) {
|
|
357
|
+
const orphan = orphanOf(run, record);
|
|
358
|
+
if (orphan) {
|
|
359
|
+
try {
|
|
360
|
+
killOrphan(orphan, nowMs);
|
|
361
|
+
}
|
|
362
|
+
catch {
|
|
363
|
+
// never let a kill attempt break harvest
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
harvested.push({
|
|
368
|
+
slug: record.slug,
|
|
369
|
+
exit: exit ?? (overCap ? { status: null, signal: null, capped: true } : null),
|
|
370
|
+
output: readRefreshRunLog(run),
|
|
371
|
+
});
|
|
372
|
+
fs.rmSync(file, { force: true });
|
|
373
|
+
removeRefreshRun(run);
|
|
374
|
+
}
|
|
375
|
+
return harvested;
|
|
376
|
+
}
|
|
377
|
+
/** Start time of the live pull for `slug`, or null. */
|
|
378
|
+
export function refreshHandoffStartedAt(hqRoot, slug, nowMs) {
|
|
379
|
+
const record = readRecord(recordPath(hqRoot, slug));
|
|
380
|
+
if (!record || record.slug !== slug)
|
|
381
|
+
return null;
|
|
382
|
+
return recordAlive(hqRoot, record, nowMs) ? record.startedAtMs : null;
|
|
383
|
+
}
|
|
384
|
+
/**
|
|
385
|
+
* True when any `--fix` pull for this HQ root still occupies it: running,
|
|
386
|
+
* starting, orphaned, or capped and still inside its stop grace window.
|
|
387
|
+
*/
|
|
388
|
+
export function anyRefreshHandoffAlive(hqRoot, nowMs) {
|
|
389
|
+
return listRecords(hqRoot).some(({ record }) => record !== null && recordOccupiesRoot(hqRoot, record, nowMs));
|
|
390
|
+
}
|
|
391
|
+
function parseSpawnLock(text) {
|
|
392
|
+
try {
|
|
393
|
+
const parsed = JSON.parse(text);
|
|
394
|
+
if (typeof parsed.pid === "number" && Number.isInteger(parsed.pid) && parsed.pid > 0) {
|
|
395
|
+
return { pid: parsed.pid, start: typeof parsed.start === "string" && parsed.start ? parsed.start : null };
|
|
396
|
+
}
|
|
397
|
+
return null;
|
|
398
|
+
}
|
|
399
|
+
catch {
|
|
400
|
+
// Older "<pid>:<ms>:<hex>" form: pid only, no start time.
|
|
401
|
+
const pid = Number(text.split(":")[0]);
|
|
402
|
+
return Number.isInteger(pid) && pid > 0 ? { pid, start: null } : null;
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
export function acquireRefreshSpawnLock(hqRoot, nowMs, identity) {
|
|
406
|
+
const file = path.join(hqRoot, SPAWN_LOCK);
|
|
407
|
+
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
408
|
+
let selfStart = null;
|
|
409
|
+
try {
|
|
410
|
+
selfStart = identity?.selfStart() ?? null;
|
|
411
|
+
}
|
|
412
|
+
catch {
|
|
413
|
+
selfStart = null;
|
|
414
|
+
}
|
|
415
|
+
const token = JSON.stringify({
|
|
416
|
+
pid: process.pid,
|
|
417
|
+
start: selfStart,
|
|
418
|
+
token: crypto.randomBytes(8).toString("hex"),
|
|
419
|
+
at: nowMs,
|
|
420
|
+
});
|
|
421
|
+
for (let attempt = 0; attempt < 2; attempt++) {
|
|
422
|
+
try {
|
|
423
|
+
fs.writeFileSync(file, token, { flag: "wx", mode: 0o600 });
|
|
424
|
+
const stillHeld = () => {
|
|
425
|
+
try {
|
|
426
|
+
return fs.readFileSync(file, "utf8") === token;
|
|
427
|
+
}
|
|
428
|
+
catch {
|
|
429
|
+
return false;
|
|
430
|
+
}
|
|
431
|
+
};
|
|
432
|
+
const release = (() => {
|
|
433
|
+
if (stillHeld())
|
|
434
|
+
fs.rmSync(file, { force: true });
|
|
435
|
+
});
|
|
436
|
+
release.stillHeld = stillHeld;
|
|
437
|
+
return release;
|
|
438
|
+
}
|
|
439
|
+
catch (error) {
|
|
440
|
+
if (error.code !== "EEXIST")
|
|
441
|
+
return null;
|
|
442
|
+
}
|
|
443
|
+
let text = "";
|
|
444
|
+
try {
|
|
445
|
+
text = fs.readFileSync(file, "utf8");
|
|
446
|
+
}
|
|
447
|
+
catch {
|
|
448
|
+
continue;
|
|
449
|
+
}
|
|
450
|
+
const holder = parseSpawnLock(text);
|
|
451
|
+
if (!holder || !ownerProvenGone(holder, identity))
|
|
452
|
+
return null;
|
|
453
|
+
// Take it over atomically: only the process whose rename succeeds removes
|
|
454
|
+
// it. If the renamed file is not the holder we judged (a new holder raced
|
|
455
|
+
// in), put it back and give up.
|
|
456
|
+
const aside = `${file}.stale-${crypto.randomBytes(6).toString("hex")}`;
|
|
457
|
+
try {
|
|
458
|
+
fs.renameSync(file, aside);
|
|
459
|
+
}
|
|
460
|
+
catch {
|
|
461
|
+
return null;
|
|
462
|
+
}
|
|
463
|
+
let moved = "";
|
|
464
|
+
try {
|
|
465
|
+
moved = fs.readFileSync(aside, "utf8");
|
|
466
|
+
}
|
|
467
|
+
catch {
|
|
468
|
+
moved = "";
|
|
469
|
+
}
|
|
470
|
+
if (moved !== text) {
|
|
471
|
+
try {
|
|
472
|
+
fs.linkSync(aside, file);
|
|
473
|
+
}
|
|
474
|
+
catch {
|
|
475
|
+
// someone already holds it again
|
|
476
|
+
}
|
|
477
|
+
fs.rmSync(aside, { force: true });
|
|
478
|
+
return null;
|
|
479
|
+
}
|
|
480
|
+
fs.rmSync(aside, { force: true });
|
|
481
|
+
}
|
|
482
|
+
return null;
|
|
483
|
+
}
|
|
484
|
+
/** A spawn lock still held after this long is reported by `hq doctor`. */
|
|
485
|
+
export const REFRESH_SPAWN_LOCK_WARN_MS = 10 * 60 * 1000;
|
|
486
|
+
/**
|
|
487
|
+
* The current spawn lock holder and how long it has held the lock, or null
|
|
488
|
+
* when no lock is held. Read-only: never removes or changes the lock.
|
|
489
|
+
*/
|
|
490
|
+
export function inspectRefreshSpawnLock(hqRoot, nowMs, identity) {
|
|
491
|
+
let text;
|
|
492
|
+
try {
|
|
493
|
+
text = fs.readFileSync(path.join(hqRoot, SPAWN_LOCK), "utf8");
|
|
494
|
+
}
|
|
495
|
+
catch {
|
|
496
|
+
return null;
|
|
497
|
+
}
|
|
498
|
+
const holder = parseSpawnLock(text);
|
|
499
|
+
if (!holder)
|
|
500
|
+
return null;
|
|
501
|
+
let at = null;
|
|
502
|
+
try {
|
|
503
|
+
const parsed = JSON.parse(text);
|
|
504
|
+
if (typeof parsed.at === "number")
|
|
505
|
+
at = parsed.at;
|
|
506
|
+
}
|
|
507
|
+
catch {
|
|
508
|
+
const legacy = Number(text.split(":")[1]);
|
|
509
|
+
if (Number.isFinite(legacy))
|
|
510
|
+
at = legacy;
|
|
511
|
+
}
|
|
512
|
+
return {
|
|
513
|
+
pid: holder.pid,
|
|
514
|
+
heldMs: at === null ? null : Math.max(0, nowMs - at),
|
|
515
|
+
ownerGone: ownerProvenGone(holder, identity),
|
|
516
|
+
};
|
|
517
|
+
}
|
|
518
|
+
/**
|
|
519
|
+
* Release a spawn lock whose owner is proven gone (pid dead, or start time
|
|
520
|
+
* changed). A live or unverifiable owner's lock is left alone. Uses the same
|
|
521
|
+
* atomic takeover as a claim, then releases at once.
|
|
522
|
+
*/
|
|
523
|
+
export function releaseDeadRefreshSpawnLock(hqRoot, nowMs, identity) {
|
|
524
|
+
const lock = inspectRefreshSpawnLock(hqRoot, nowMs, identity);
|
|
525
|
+
if (!lock?.ownerGone)
|
|
526
|
+
return;
|
|
527
|
+
try {
|
|
528
|
+
acquireRefreshSpawnLock(hqRoot, nowMs, identity)?.();
|
|
529
|
+
}
|
|
530
|
+
catch {
|
|
531
|
+
// best effort
|
|
532
|
+
}
|
|
533
|
+
}
|
|
534
|
+
/** True only when identity proves the lock's owner no longer runs. */
|
|
535
|
+
function ownerProvenGone(holder, identity) {
|
|
536
|
+
if (!processExists(holder.pid))
|
|
537
|
+
return true;
|
|
538
|
+
if (!identity || !holder.start)
|
|
539
|
+
return false;
|
|
540
|
+
let current;
|
|
541
|
+
try {
|
|
542
|
+
current = identity.startOf(holder.pid);
|
|
543
|
+
}
|
|
544
|
+
catch {
|
|
545
|
+
current = null;
|
|
546
|
+
}
|
|
547
|
+
return current !== null && current !== holder.start;
|
|
548
|
+
}
|
|
549
|
+
/**
|
|
550
|
+
* Claim this HQ root for a pull of `slug`; true means spawn.
|
|
551
|
+
*
|
|
552
|
+
* - The spawn lock is required. If another live or unverifiable process holds
|
|
553
|
+
* it, nothing is written and the claim fails.
|
|
554
|
+
* - Under the lock: no other pull may occupy the root, and the record is
|
|
555
|
+
* created exclusively, never overwritten. An existing record for `slug`
|
|
556
|
+
* wins unless it no longer occupies the root (its pull exited, or its
|
|
557
|
+
* owner and pull are proven gone); then it is moved aside atomically and
|
|
558
|
+
* the create is retried once.
|
|
559
|
+
* - The claim holds only if, after writing, the record still carries this
|
|
560
|
+
* claim's token and this process still holds the lock.
|
|
561
|
+
*/
|
|
562
|
+
export function claimRefreshRun(hqRoot, slug, run, nowMs, identity) {
|
|
563
|
+
const release = acquireRefreshSpawnLock(hqRoot, nowMs, identity);
|
|
564
|
+
if (!release)
|
|
565
|
+
return false;
|
|
566
|
+
const token = crypto.randomBytes(8).toString("hex");
|
|
567
|
+
let created = false;
|
|
568
|
+
try {
|
|
569
|
+
if (anyRefreshHandoffAlive(hqRoot, nowMs))
|
|
570
|
+
return false;
|
|
571
|
+
created = createRecordExclusive(hqRoot, slug, run, nowMs, token);
|
|
572
|
+
if (!created) {
|
|
573
|
+
const file = recordPath(hqRoot, slug);
|
|
574
|
+
const existing = readRecord(file);
|
|
575
|
+
if (existing && recordOccupiesRoot(hqRoot, existing, nowMs))
|
|
576
|
+
return false;
|
|
577
|
+
const aside = `${file}.dead-${crypto.randomBytes(6).toString("hex")}`;
|
|
578
|
+
try {
|
|
579
|
+
fs.renameSync(file, aside);
|
|
580
|
+
}
|
|
581
|
+
catch {
|
|
582
|
+
return false;
|
|
583
|
+
}
|
|
584
|
+
const moved = readRecord(aside);
|
|
585
|
+
const sameAsJudged = moved?.id === existing?.id && moved?.token === existing?.token;
|
|
586
|
+
if (!sameAsJudged) {
|
|
587
|
+
// Not the record we judged dead: put it back if the name is free.
|
|
588
|
+
try {
|
|
589
|
+
fs.linkSync(aside, file);
|
|
590
|
+
}
|
|
591
|
+
catch {
|
|
592
|
+
// a new record already took the name
|
|
593
|
+
}
|
|
594
|
+
fs.rmSync(aside, { force: true });
|
|
595
|
+
return false;
|
|
596
|
+
}
|
|
597
|
+
fs.rmSync(aside, { force: true });
|
|
598
|
+
if (existing)
|
|
599
|
+
removeRefreshRun(runPaths(hqRoot, existing.id));
|
|
600
|
+
created = createRecordExclusive(hqRoot, slug, run, nowMs, token);
|
|
601
|
+
if (!created)
|
|
602
|
+
return false;
|
|
603
|
+
}
|
|
604
|
+
const mine = readRecord(recordPath(hqRoot, slug));
|
|
605
|
+
if (!mine || mine.id !== run.id || mine.token !== token || !release.stillHeld()) {
|
|
606
|
+
if (mine && mine.id === run.id && mine.token === token)
|
|
607
|
+
clearRefreshRecord(hqRoot, slug, run);
|
|
608
|
+
return false;
|
|
609
|
+
}
|
|
610
|
+
return true;
|
|
611
|
+
}
|
|
612
|
+
catch {
|
|
613
|
+
if (created)
|
|
614
|
+
clearRefreshRecord(hqRoot, slug, run);
|
|
615
|
+
return false;
|
|
616
|
+
}
|
|
617
|
+
finally {
|
|
618
|
+
release();
|
|
619
|
+
}
|
|
620
|
+
}
|
|
621
|
+
/** How often the supervisor looks for another operation waiting on the lock. */
|
|
622
|
+
export const REFRESH_PREEMPT_POLL_MS = 500;
|
|
623
|
+
/**
|
|
624
|
+
* Supervisor program, run as
|
|
625
|
+
* `node -e SUPERVISOR log exit heartbeat pidFile cancel locksDir maxMs cmd ...args`.
|
|
626
|
+
*
|
|
627
|
+
* It appends the pull's output to `log`, writes the pull's pid to `pidFile`,
|
|
628
|
+
* refreshes `heartbeat`, and writes `{status, signal, preempted, capped}` to
|
|
629
|
+
* `exit` when the pull ends.
|
|
630
|
+
*
|
|
631
|
+
* Stopping: at `maxMs`, when `cancel` exists, or when another live process
|
|
632
|
+
* publishes a waiter for the operation lock the pull holds
|
|
633
|
+
* (`<lock>.wait-<pid>-*`; `sync` holds the lock shared, so only exclusive
|
|
634
|
+
* callers such as rescue and repairs wait), the supervisor stops the pull.
|
|
635
|
+
* On POSIX it sends SIGTERM to the pull's process group, and SIGKILL to the
|
|
636
|
+
* group after {@link REFRESH_KILL_GRACE_MS} in every case, so a grandchild
|
|
637
|
+
* cannot outlive it. SIGTERM is the same stop as Ctrl-C on `hq sync`: the
|
|
638
|
+
* lock's signal handler releases the lock, downloads are staged and renamed
|
|
639
|
+
* into place, and the journal is written by atomic rename. On Windows the
|
|
640
|
+
* stop is `taskkill /T /F` on the tree, which is a forced kill, not a
|
|
641
|
+
* cooperative one; the same staged writes and atomic journal keep completed
|
|
642
|
+
* work.
|
|
643
|
+
*/
|
|
644
|
+
export const REFRESH_SUPERVISOR_SOURCE = `
|
|
645
|
+
const fs = require("node:fs");
|
|
646
|
+
const path = require("node:path");
|
|
647
|
+
const { spawn, spawnSync } = require("node:child_process");
|
|
648
|
+
const [logPath, exitPath, hbPath, pidPath, cancelPath, locksDir, maxMs, cmd, ...args] = process.argv.slice(1);
|
|
649
|
+
const isWin = process.platform === "win32";
|
|
650
|
+
const beat = () => { try { fs.writeFileSync(hbPath, String(Date.now())); } catch {} };
|
|
651
|
+
beat();
|
|
652
|
+
const hb = setInterval(beat, ${REFRESH_HEARTBEAT_INTERVAL_MS});
|
|
653
|
+
let fd;
|
|
654
|
+
try { fd = fs.openSync(logPath, "a", 0o600); } catch { fd = "ignore"; }
|
|
655
|
+
let finished = false;
|
|
656
|
+
let preempted = false;
|
|
657
|
+
let capped = false;
|
|
658
|
+
let stopAt = 0;
|
|
659
|
+
let watch;
|
|
660
|
+
// Its own process group on POSIX, so a stop reaches every process the pull started.
|
|
661
|
+
const child = spawn(cmd, args, { stdio: ["ignore", fd, fd], windowsHide: true, detached: !isWin });
|
|
662
|
+
try { fs.writeFileSync(pidPath, String(child.pid)); } catch {}
|
|
663
|
+
if (!isWin && child.pid) {
|
|
664
|
+
try {
|
|
665
|
+
const r = spawnSync("ps", ["-o", "lstart=", "-p", String(child.pid)], { encoding: "utf8" });
|
|
666
|
+
const started = (r.stdout || "").trim();
|
|
667
|
+
if (started) fs.writeFileSync(path.join(path.dirname(pidPath), path.basename(pidPath, ".pid") + ".start"), started);
|
|
668
|
+
} catch {}
|
|
669
|
+
}
|
|
670
|
+
const killTree = (sig) => {
|
|
671
|
+
if (isWin) {
|
|
672
|
+
try { spawnSync("taskkill", ["/PID", String(child.pid), "/T", "/F"], { stdio: "ignore", windowsHide: true }); } catch {}
|
|
673
|
+
return;
|
|
674
|
+
}
|
|
675
|
+
try { process.kill(-child.pid, sig); } catch {}
|
|
676
|
+
};
|
|
677
|
+
const stop = () => {
|
|
678
|
+
if (stopAt) return;
|
|
679
|
+
stopAt = Date.now();
|
|
680
|
+
killTree("SIGTERM");
|
|
681
|
+
setTimeout(() => killTree("SIGKILL"), ${REFRESH_KILL_GRACE_MS});
|
|
682
|
+
};
|
|
683
|
+
const exitNow = (status) => process.exit(typeof status === "number" ? status : 1);
|
|
684
|
+
const done = (status, signal) => {
|
|
685
|
+
if (finished) return;
|
|
686
|
+
finished = true;
|
|
687
|
+
clearInterval(hb);
|
|
688
|
+
clearInterval(watch);
|
|
689
|
+
clearTimeout(stopper);
|
|
690
|
+
try {
|
|
691
|
+
fs.writeFileSync(exitPath + ".tmp", JSON.stringify({ status, signal, preempted, capped }));
|
|
692
|
+
fs.renameSync(exitPath + ".tmp", exitPath);
|
|
693
|
+
} catch {}
|
|
694
|
+
if (!stopAt) return exitNow(status);
|
|
695
|
+
// A stopped pull's group is always SIGKILLed after the grace period, even
|
|
696
|
+
// when the pull itself exited on SIGTERM, so no grandchild survives.
|
|
697
|
+
const left = Math.max(0, stopAt + ${REFRESH_KILL_GRACE_MS} - Date.now());
|
|
698
|
+
setTimeout(() => { killTree("SIGKILL"); exitNow(status); }, left);
|
|
699
|
+
};
|
|
700
|
+
const stopper = setTimeout(() => {
|
|
701
|
+
capped = true;
|
|
702
|
+
// At the cap the pull is no longer live: stop heartbeating so nothing reads it as in progress.
|
|
703
|
+
clearInterval(hb);
|
|
704
|
+
try { fs.writeFileSync(hbPath, "capped:" + Date.now()); } catch {}
|
|
705
|
+
stop();
|
|
706
|
+
}, Number(maxMs));
|
|
707
|
+
const alive = (pid) => { try { process.kill(pid, 0); return true; } catch (e) { return Boolean(e && e.code === "EPERM"); } };
|
|
708
|
+
let lockName = null;
|
|
709
|
+
const findLock = () => {
|
|
710
|
+
let names;
|
|
711
|
+
try { names = fs.readdirSync(locksDir); } catch { return null; }
|
|
712
|
+
for (const name of names) {
|
|
713
|
+
// A sync holds shared membership: <lock>.shared-<pid>-<hex>.
|
|
714
|
+
const shared = /^(.+\\.lock)\\.shared-(\\d+)-/.exec(name);
|
|
715
|
+
if (shared && Number(shared[2]) === child.pid) return shared[1];
|
|
716
|
+
if (!name.endsWith(".lock")) continue;
|
|
717
|
+
// An exclusive holder writes its record into the lock file itself.
|
|
718
|
+
try {
|
|
719
|
+
const info = JSON.parse(fs.readFileSync(path.join(locksDir, name), "utf8"));
|
|
720
|
+
if (info && info.pid === child.pid) return name;
|
|
721
|
+
} catch {}
|
|
722
|
+
}
|
|
723
|
+
return null;
|
|
724
|
+
};
|
|
725
|
+
const someoneWaiting = () => {
|
|
726
|
+
if (fs.existsSync(cancelPath)) return true;
|
|
727
|
+
if (!lockName) lockName = findLock();
|
|
728
|
+
if (!lockName) return false;
|
|
729
|
+
let names;
|
|
730
|
+
try { names = fs.readdirSync(locksDir); } catch { return false; }
|
|
731
|
+
const prefix = lockName + ".wait-";
|
|
732
|
+
for (const name of names) {
|
|
733
|
+
if (!name.startsWith(prefix)) continue;
|
|
734
|
+
const waiter = Number(name.slice(prefix.length).split("-")[0]);
|
|
735
|
+
if (Number.isInteger(waiter) && waiter > 0 && waiter !== child.pid && waiter !== process.pid && alive(waiter)) return true;
|
|
736
|
+
}
|
|
737
|
+
return false;
|
|
738
|
+
};
|
|
739
|
+
watch = setInterval(() => {
|
|
740
|
+
if (finished || stopAt) return;
|
|
741
|
+
if (someoneWaiting()) {
|
|
742
|
+
preempted = true;
|
|
743
|
+
stop();
|
|
744
|
+
}
|
|
745
|
+
}, ${REFRESH_PREEMPT_POLL_MS});
|
|
746
|
+
child.on("error", (error) => {
|
|
747
|
+
try { fs.appendFileSync(logPath, String((error && error.code) || error) + "\\n"); } catch {}
|
|
748
|
+
done(null, null);
|
|
749
|
+
});
|
|
750
|
+
child.on("exit", (status, signal) => done(status, signal));
|
|
751
|
+
`;
|
|
752
|
+
//# sourceMappingURL=refresh-sync-handoff.js.map
|