@indigoai-us/hq-cli 5.345.166 → 5.345.169
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +13 -0
- package/assets/scaffold/core/scripts/hq-delegate-send.sh +1 -1
- package/dist/command-catalog.generated.d.ts +6 -2
- package/dist/command-catalog.generated.js +7 -2
- package/dist/commands/access.js +2 -0
- package/dist/commands/agents.js +2 -2
- package/dist/commands/core-checkpoint-native.js +30 -10
- package/dist/commands/core-delegate-w1b.js +2 -1
- package/dist/commands/core-dm-bind-native.js +3 -1
- package/dist/commands/dm.d.ts +13 -2
- package/dist/commands/dm.js +42 -9
- package/dist/commands/feedback.js +11 -3
- package/dist/lib/agent-kit/mcp/tools.js +5 -6
- package/dist/lib/doctor/checks/sync-health.d.ts +7 -1
- package/dist/lib/doctor/checks/sync-health.js +67 -2
- package/dist/lib/doctor/fix/apply.d.ts +40 -4
- package/dist/lib/doctor/fix/apply.js +183 -69
- package/dist/lib/doctor/fix/process-identity.d.ts +13 -0
- package/dist/lib/doctor/fix/process-identity.js +34 -0
- package/dist/lib/doctor/fix/refresh-sync-handoff.d.ts +201 -0
- package/dist/lib/doctor/fix/refresh-sync-handoff.js +752 -0
- package/dist/lib/doctor/fix/refresh-sync-outcome.d.ts +1 -1
- package/dist/lib/doctor/fix/refresh-sync-outcome.js +13 -0
- package/dist/lib/doctor/types.d.ts +1 -1
- package/dist/lib/doctor/types.js +2 -0
- package/dist/lib/notify-recipient.d.ts +15 -0
- package/dist/lib/notify-recipient.js +15 -0
- package/dist/lib/secret-scan/repo-config.d.ts +8 -2
- package/dist/lib/secret-scan/repo-config.js +12 -6
- package/dist/lib/secret-scan/staged.d.ts +6 -0
- package/dist/lib/secret-scan/staged.js +21 -3
- package/dist/sentry-before-send.js +13 -0
- package/dist/utils/breadcrumb-buffer.js +15 -4
- package/dist/utils/feedback-diagnostics.js +5 -2
- package/dist/utils/feedback-logs.d.ts +7 -0
- package/dist/utils/feedback-logs.js +37 -3
- package/dist/utils/scrub-diagnostics.d.ts +4 -0
- package/dist/utils/scrub-diagnostics.js +95 -0
- package/package.json +7 -7
|
@@ -42,6 +42,8 @@ import type { CheckResult, DoctorStatus } from "../types.js";
|
|
|
42
42
|
import { type GateProfile } from "../hook-gate-profiles.js";
|
|
43
43
|
import { type FixClass } from "./remediation.js";
|
|
44
44
|
import { type RemovePs1ShimsResult } from "../../windows/ps1-shims.js";
|
|
45
|
+
import { processStartTime } from "./process-identity.js";
|
|
46
|
+
import { type HarvestedRefreshRun, type OrphanPull, type RefreshOwnerIdentity } from "./refresh-sync-handoff.js";
|
|
45
47
|
import { type RefreshSyncOutcome } from "./refresh-sync-outcome.js";
|
|
46
48
|
/** The tree subtrees whose uncommitted changes block a `--fix` run. */
|
|
47
49
|
export declare const HOOK_CONFIG_DIRS: readonly [".claude", ".codex", ".grok"];
|
|
@@ -290,10 +292,45 @@ export declare function unresolvedSyncChecks(findings: readonly CheckResult[], a
|
|
|
290
292
|
/**
|
|
291
293
|
* Default `refresh-sync` repair: a bounded, pull-only, keep-conflicts
|
|
292
294
|
* `hq sync pull --company <slug>` (or `--personal`) against the tree
|
|
293
|
-
* `--fix` is repairing. Never a push, never interactive. A
|
|
294
|
-
*
|
|
295
|
+
* `--fix` is repairing. Never a push, never interactive. A non-zero exit is
|
|
296
|
+
* reported rather than thrown so later plans still run.
|
|
297
|
+
*
|
|
298
|
+
* A pull still running after {@link REFRESH_SYNC_TIMEOUT_MS} is NOT killed
|
|
299
|
+
* (feedback cd56a501). A pull writes `lastSync` only when its pass finishes,
|
|
300
|
+
* so killing it at the deadline left a large vault, or a pull queued behind
|
|
301
|
+
* the sync runner's lock, stale on every run. It is left running detached and
|
|
302
|
+
* recorded in refresh-sync-handoff.ts; the journal check reports it as
|
|
303
|
+
* "refresh in progress" until it finishes.
|
|
304
|
+
*/
|
|
305
|
+
export declare function defaultRefreshSync(target: RefreshSyncTarget, deps?: RefreshSyncSpawnDeps): Promise<RefreshSyncResult>;
|
|
306
|
+
/** Map a finished supervised pull to a {@link RefreshSyncResult}. */
|
|
307
|
+
export declare function harvestedRefreshSyncResult(harvested: HarvestedRefreshRun): RefreshSyncResult;
|
|
308
|
+
export { processStartTime };
|
|
309
|
+
/**
|
|
310
|
+
* Kill an orphaned --fix pull past its cap, but only when the pid still names
|
|
311
|
+
* that pull: its process start time must match the one recorded when it
|
|
312
|
+
* started. A reused pid has a different start time and is left alone. On
|
|
313
|
+
* Windows no start time is recorded, so nothing is killed there.
|
|
314
|
+
*/
|
|
315
|
+
export declare function killOrphanPullIfSame(orphan: OrphanPull, _nowMs: number, probe?: (pid: number) => string | null): void;
|
|
316
|
+
/** Test seams for {@link defaultRefreshSync}. */
|
|
317
|
+
export interface RefreshSyncSpawnDeps {
|
|
318
|
+
command?: string;
|
|
319
|
+
args?: string[];
|
|
320
|
+
timeoutMs?: number;
|
|
321
|
+
maxBackgroundMs?: number;
|
|
322
|
+
nowMs?: () => number;
|
|
323
|
+
/** Directory holding hq-cloud's operation-lock files. */
|
|
324
|
+
locksDir?: string;
|
|
325
|
+
/** Process start-time probe for the spawn lock. */
|
|
326
|
+
identity?: RefreshOwnerIdentity;
|
|
327
|
+
}
|
|
328
|
+
/**
|
|
329
|
+
* How long a `--fix` pull waits for the operation lock. Unchanged from before
|
|
330
|
+
* the hand-off: a background pull must never queue on the lock longer than a
|
|
331
|
+
* foreground one did.
|
|
295
332
|
*/
|
|
296
|
-
export declare
|
|
333
|
+
export declare const REFRESH_SYNC_LOCK_TIMEOUT_SEC = 30;
|
|
297
334
|
/**
|
|
298
335
|
* Default `refresh-manifest` repair: a bounded `hq sync manifest --scope <s>`
|
|
299
336
|
* against the tree `--fix` is repairing. Exit 0 also covers throttled,
|
|
@@ -349,5 +386,4 @@ export declare function addHookIdToGateProfiles(source: string, hookId: string,
|
|
|
349
386
|
* non-object `hooks` value — it is replaced rather than trusted.
|
|
350
387
|
*/
|
|
351
388
|
export declare function registerHookInSettings(settings: unknown, event: string, command: string): Record<string, unknown>;
|
|
352
|
-
export {};
|
|
353
389
|
//# sourceMappingURL=apply.d.ts.map
|
|
@@ -37,9 +37,10 @@
|
|
|
37
37
|
* (the command resolves it once), keeping this module decoupled from the CLI and
|
|
38
38
|
* trivially testable against a fake tree.
|
|
39
39
|
*/
|
|
40
|
-
import { spawnSync } from "node:child_process";
|
|
40
|
+
import { spawn, spawnSync } from "node:child_process";
|
|
41
41
|
import * as fs from "node:fs";
|
|
42
42
|
import * as path from "node:path";
|
|
43
|
+
import { getStateDir } from "@indigoai-us/hq-cloud";
|
|
43
44
|
import { isPersonalJournalSlug } from "../checks/sync-health.js";
|
|
44
45
|
import { createDefaultRegistry } from "../registry.js";
|
|
45
46
|
import { flattenFamilies } from "../report.js";
|
|
@@ -51,6 +52,8 @@ import { HQ_CLI_BINS, removeHqCliPs1Shims } from "../../windows/ps1-shims.js";
|
|
|
51
52
|
import { botDir } from "../../bot/paths.js";
|
|
52
53
|
import { readBotConfig } from "../../bot/config.js";
|
|
53
54
|
import { buildBotDaemonPaths, installedDaemonHqBinary, repairBotDaemonIfStale, rewriteBotDaemonNodePath, } from "../../bot/daemon.js";
|
|
55
|
+
import { DEFAULT_OWNER_IDENTITY, processStartTime } from "./process-identity.js";
|
|
56
|
+
import { REFRESH_BACKGROUND_MAX_MS, REFRESH_SUPERVISOR_SOURCE, anyRefreshHandoffAlive, harvestRefreshHandoffs, releaseDeadRefreshSpawnLock, newRefreshRunPaths, readRefreshRunExit, readRefreshRunLog, claimRefreshRun, clearRefreshRecord, removeRefreshRun, } from "./refresh-sync-handoff.js";
|
|
54
57
|
import { classifyRefreshSyncOutcome, } from "./refresh-sync-outcome.js";
|
|
55
58
|
import { pullOutputShowsCompanyNotResolved, recordCompanyResolution, } from "../company-resolution-state.js";
|
|
56
59
|
import { readRefreshManifestCursor, readRefreshSyncProgress, writeRefreshManifestCursor, writeRefreshSyncCursor, writeRefreshSyncLastRun, } from "./refresh-sync-progress.js";
|
|
@@ -184,6 +187,10 @@ export async function applyFixes(options) {
|
|
|
184
187
|
};
|
|
185
188
|
const refreshSync = async (target) => {
|
|
186
189
|
const result = await performRefreshSync(target);
|
|
190
|
+
// Still running in the background: no evidence about the company yet.
|
|
191
|
+
// The next --fix harvests its real exit before it plans anything.
|
|
192
|
+
if (result.outcome === "pull-handed-off")
|
|
193
|
+
return result;
|
|
187
194
|
if (target.personal)
|
|
188
195
|
return result;
|
|
189
196
|
noteCompanyOutcome(target.slug, result.ok, result.companyNotResolved === true);
|
|
@@ -205,6 +212,17 @@ export async function applyFixes(options) {
|
|
|
205
212
|
// 2. Collect findings, then plan each newly unlocked allowlisted repair.
|
|
206
213
|
// Identity is deliberately the check id + target pair: an unsuccessful
|
|
207
214
|
// repair must not be applied repeatedly in one invocation.
|
|
215
|
+
// Background pulls from an earlier --fix: record how each really ended
|
|
216
|
+
// before planning, then drop the finished records (feedback cd56a501).
|
|
217
|
+
// A spawn lock left by a crashed claimer is released here when its owner is
|
|
218
|
+
// proven gone, even if no journal is stale this run.
|
|
219
|
+
releaseDeadRefreshSpawnLock(hqRoot, Date.now(), DEFAULT_OWNER_IDENTITY);
|
|
220
|
+
for (const harvested of harvestRefreshHandoffs(hqRoot, Date.now(), killOrphanPullIfSame)) {
|
|
221
|
+
const outcome = harvestedRefreshSyncResult(harvested);
|
|
222
|
+
if (harvested.slug === "personal" || isPersonalJournalSlug(harvested.slug))
|
|
223
|
+
continue;
|
|
224
|
+
noteCompanyOutcome(harvested.slug, outcome.ok, outcome.companyNotResolved === true);
|
|
225
|
+
}
|
|
208
226
|
let findings = await collect(hqRoot);
|
|
209
227
|
let lastAttemptedRefreshSyncSlug = readRefreshSyncProgress(hqRoot)?.lastAttemptedSlug ?? null;
|
|
210
228
|
let refreshSyncAppliedThisRun = 0;
|
|
@@ -333,6 +351,11 @@ export async function applyFixes(options) {
|
|
|
333
351
|
// left over and reported "not attempted" rather than as a failed repair.
|
|
334
352
|
if (network && !networkBudgetLeft())
|
|
335
353
|
continue;
|
|
354
|
+
// One background pull per HQ root. While it runs it holds the operation
|
|
355
|
+
// lock, so another pull would only wait on it; leave the rest for the
|
|
356
|
+
// next run.
|
|
357
|
+
if (plan.fixClass === "refresh-sync" && anyRefreshHandoffAlive(hqRoot, Date.now()))
|
|
358
|
+
continue;
|
|
336
359
|
if (plan.fixClass === "refresh-sync") {
|
|
337
360
|
// Persist before starting the bounded pull. If this process is killed,
|
|
338
361
|
// the next invocation resumes after this company instead of starving
|
|
@@ -540,85 +563,176 @@ async function defaultRunChecks(hqRoot) {
|
|
|
540
563
|
/**
|
|
541
564
|
* Default `refresh-sync` repair: a bounded, pull-only, keep-conflicts
|
|
542
565
|
* `hq sync pull --company <slug>` (or `--personal`) against the tree
|
|
543
|
-
* `--fix` is repairing. Never a push, never interactive. A
|
|
544
|
-
*
|
|
566
|
+
* `--fix` is repairing. Never a push, never interactive. A non-zero exit is
|
|
567
|
+
* reported rather than thrown so later plans still run.
|
|
568
|
+
*
|
|
569
|
+
* A pull still running after {@link REFRESH_SYNC_TIMEOUT_MS} is NOT killed
|
|
570
|
+
* (feedback cd56a501). A pull writes `lastSync` only when its pass finishes,
|
|
571
|
+
* so killing it at the deadline left a large vault, or a pull queued behind
|
|
572
|
+
* the sync runner's lock, stale on every run. It is left running detached and
|
|
573
|
+
* recorded in refresh-sync-handoff.ts; the journal check reports it as
|
|
574
|
+
* "refresh in progress" until it finishes.
|
|
545
575
|
*/
|
|
546
|
-
export function defaultRefreshSync(target) {
|
|
576
|
+
export async function defaultRefreshSync(target, deps = {}) {
|
|
547
577
|
const hqBin = process.argv[1];
|
|
548
578
|
const selector = target.personal || isPersonalJournalSlug(target.slug)
|
|
549
579
|
? ["--personal"]
|
|
550
580
|
: ["--company", target.slug];
|
|
551
|
-
const
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
...selector,
|
|
568
|
-
"--hq-root",
|
|
569
|
-
target.hqRoot,
|
|
570
|
-
"--on-conflict",
|
|
571
|
-
"keep",
|
|
572
|
-
"--lock-timeout",
|
|
573
|
-
"30",
|
|
574
|
-
];
|
|
575
|
-
const result = spawnSync(hqBin ? process.execPath : "hq", args, {
|
|
576
|
-
encoding: "utf8",
|
|
577
|
-
timeout: REFRESH_SYNC_TIMEOUT_MS,
|
|
578
|
-
cwd: target.hqRoot,
|
|
579
|
-
});
|
|
580
|
-
if (result.error) {
|
|
581
|
-
const spawnError = result.error;
|
|
582
|
-
const outcome = classifyRefreshSyncOutcome({
|
|
583
|
-
errorCode: spawnError.code,
|
|
584
|
-
...(result.error.message.includes("ETIMEDOUT")
|
|
585
|
-
? { errorCode: "ETIMEDOUT" }
|
|
586
|
-
: {}),
|
|
587
|
-
signal: result.signal,
|
|
588
|
-
status: result.status,
|
|
589
|
-
stdout: result.stdout,
|
|
590
|
-
stderr: result.stderr,
|
|
591
|
-
});
|
|
592
|
-
return Promise.resolve({
|
|
581
|
+
const pullArgs = [
|
|
582
|
+
"sync",
|
|
583
|
+
"pull",
|
|
584
|
+
...selector,
|
|
585
|
+
"--hq-root",
|
|
586
|
+
target.hqRoot,
|
|
587
|
+
"--on-conflict",
|
|
588
|
+
"keep",
|
|
589
|
+
"--lock-timeout",
|
|
590
|
+
String(REFRESH_SYNC_LOCK_TIMEOUT_SEC),
|
|
591
|
+
];
|
|
592
|
+
// Run the same CLI script under this node. Without a script path there is no
|
|
593
|
+
// safe way to start `hq` on Windows (it is a .cmd shim that needs a shell),
|
|
594
|
+
// so only POSIX falls back to `hq` on PATH.
|
|
595
|
+
if (!deps.command && !hqBin && process.platform === "win32") {
|
|
596
|
+
return {
|
|
593
597
|
ok: false,
|
|
594
|
-
outcome,
|
|
595
|
-
message:
|
|
596
|
-
|
|
597
|
-
: `could not start pull for '${target.slug}': ${result.error.message}`,
|
|
598
|
-
});
|
|
598
|
+
outcome: "pull-failed:other",
|
|
599
|
+
message: `could not start pull for '${target.slug}': the hq CLI script path is unknown`,
|
|
600
|
+
};
|
|
599
601
|
}
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
602
|
+
const command = deps.command ?? (hqBin ? process.execPath : "hq");
|
|
603
|
+
const args = deps.args ?? (hqBin ? [hqBin, ...pullArgs] : pullArgs);
|
|
604
|
+
const timeoutMs = deps.timeoutMs ?? REFRESH_SYNC_TIMEOUT_MS;
|
|
605
|
+
const maxBackgroundMs = deps.maxBackgroundMs ?? REFRESH_BACKGROUND_MAX_MS;
|
|
606
|
+
const nowMs = deps.nowMs ?? (() => Date.now());
|
|
607
|
+
// The pull runs under a supervisor that records its real exit status, so a
|
|
608
|
+
// pull left running past the deadline is still classified by the next run.
|
|
609
|
+
const run = newRefreshRunPaths(target.hqRoot, target.slug);
|
|
610
|
+
// One pull per HQ root: claim it (record written under the spawn lock)
|
|
611
|
+
// before spawning, so a concurrent --fix sees this pull and backs off.
|
|
612
|
+
if (!claimRefreshRun(target.hqRoot, target.slug, run, nowMs(), deps.identity ?? DEFAULT_OWNER_IDENTITY)) {
|
|
613
|
+
return {
|
|
608
614
|
ok: false,
|
|
609
|
-
outcome,
|
|
610
|
-
message: `pull
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
+
outcome: "pull-failed:lock-busy",
|
|
616
|
+
message: `another hq doctor --fix pull is already running; '${target.slug}' is left for the next run`,
|
|
617
|
+
};
|
|
618
|
+
}
|
|
619
|
+
let supervisor;
|
|
620
|
+
try {
|
|
621
|
+
fs.mkdirSync(path.dirname(run.log), { recursive: true });
|
|
622
|
+
supervisor = spawn(process.execPath, [
|
|
623
|
+
"-e",
|
|
624
|
+
REFRESH_SUPERVISOR_SOURCE,
|
|
625
|
+
run.log,
|
|
626
|
+
run.exit,
|
|
627
|
+
run.heartbeat,
|
|
628
|
+
run.pid,
|
|
629
|
+
run.cancel,
|
|
630
|
+
deps.locksDir ?? defaultLocksDir(),
|
|
631
|
+
String(maxBackgroundMs),
|
|
632
|
+
command,
|
|
633
|
+
...args,
|
|
634
|
+
], { cwd: target.hqRoot, detached: true, stdio: "ignore", windowsHide: true });
|
|
635
|
+
}
|
|
636
|
+
catch (error) {
|
|
637
|
+
clearRefreshRecord(target.hqRoot, target.slug, run);
|
|
638
|
+
removeRefreshRun(run);
|
|
639
|
+
return {
|
|
640
|
+
ok: false,
|
|
641
|
+
outcome: classifyRefreshSyncOutcome({ errorCode: error.code }),
|
|
642
|
+
message: `could not start pull for '${target.slug}': ${error.message}`,
|
|
643
|
+
};
|
|
615
644
|
}
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
645
|
+
const finished = await new Promise((resolve) => {
|
|
646
|
+
let settled = false;
|
|
647
|
+
const settle = (value) => {
|
|
648
|
+
if (settled)
|
|
649
|
+
return;
|
|
650
|
+
settled = true;
|
|
651
|
+
clearTimeout(timer);
|
|
652
|
+
resolve(value);
|
|
653
|
+
};
|
|
654
|
+
supervisor.once("error", () => settle(true));
|
|
655
|
+
supervisor.once("exit", () => settle(true));
|
|
656
|
+
const timer = setTimeout(() => settle(false), timeoutMs);
|
|
620
657
|
});
|
|
658
|
+
if (!finished) {
|
|
659
|
+
// Leave the pull running on its own. It records lastSync when it
|
|
660
|
+
// finishes; until then the journal check reports "refresh in progress".
|
|
661
|
+
supervisor.removeAllListeners();
|
|
662
|
+
supervisor.unref();
|
|
663
|
+
return {
|
|
664
|
+
ok: false,
|
|
665
|
+
outcome: "pull-handed-off",
|
|
666
|
+
message: `still pulling '${target.slug}' after ${timeoutMs / 1000}s; it keeps running in the background`,
|
|
667
|
+
};
|
|
668
|
+
}
|
|
669
|
+
const result = harvestedRefreshSyncResult({
|
|
670
|
+
slug: target.slug,
|
|
671
|
+
exit: readRefreshRunExit(run),
|
|
672
|
+
output: readRefreshRunLog(run),
|
|
673
|
+
});
|
|
674
|
+
clearRefreshRecord(target.hqRoot, target.slug, run);
|
|
675
|
+
removeRefreshRun(run);
|
|
676
|
+
return result;
|
|
677
|
+
}
|
|
678
|
+
/** Map a finished supervised pull to a {@link RefreshSyncResult}. */
|
|
679
|
+
export function harvestedRefreshSyncResult(harvested) {
|
|
680
|
+
const { slug, exit, output } = harvested;
|
|
681
|
+
if (!exit) {
|
|
682
|
+
return { ok: false, outcome: "pull-failed:other", message: `pull of '${slug}' ended without a result` };
|
|
683
|
+
}
|
|
684
|
+
if (exit.capped) {
|
|
685
|
+
// Stopped at the background time cap. Completed downloads are kept.
|
|
686
|
+
return { ok: false, outcome: "pull-timeout-capped", message: `stopped pulling '${slug}' at the background time limit` };
|
|
687
|
+
}
|
|
688
|
+
if (exit.preempted) {
|
|
689
|
+
// Stopped so a sync the user or the runner started could take the lock.
|
|
690
|
+
// Completed downloads are kept; the next --fix resumes the journal.
|
|
691
|
+
return { ok: false, outcome: "pull-preempted", message: `paused pulling '${slug}' for another sync` };
|
|
692
|
+
}
|
|
693
|
+
if (exit.status === 0 && !exit.signal) {
|
|
694
|
+
return { ok: true, outcome: "pull-succeeded", message: `pulled '${slug}'` };
|
|
695
|
+
}
|
|
696
|
+
const outcome = classifyRefreshSyncOutcome({ status: exit.status, signal: exit.signal, stdout: output });
|
|
697
|
+
const detail = output.trim().split("\n").at(-1) ?? "";
|
|
698
|
+
const personal = slug === "personal" || isPersonalJournalSlug(slug);
|
|
699
|
+
return {
|
|
700
|
+
ok: false,
|
|
701
|
+
outcome,
|
|
702
|
+
message: `pull of '${slug}' exited ${exit.status ?? exit.signal}${detail ? `: ${detail}` : ""}`,
|
|
703
|
+
companyNotResolved: !personal && pullOutputShowsCompanyNotResolved(output),
|
|
704
|
+
};
|
|
621
705
|
}
|
|
706
|
+
export { processStartTime };
|
|
707
|
+
/**
|
|
708
|
+
* Kill an orphaned --fix pull past its cap, but only when the pid still names
|
|
709
|
+
* that pull: its process start time must match the one recorded when it
|
|
710
|
+
* started. A reused pid has a different start time and is left alone. On
|
|
711
|
+
* Windows no start time is recorded, so nothing is killed there.
|
|
712
|
+
*/
|
|
713
|
+
export function killOrphanPullIfSame(orphan, _nowMs, probe = processStartTime) {
|
|
714
|
+
if (process.platform === "win32" || !orphan.recordedStart)
|
|
715
|
+
return;
|
|
716
|
+
const current = probe(orphan.pid);
|
|
717
|
+
if (!current || current !== orphan.recordedStart)
|
|
718
|
+
return;
|
|
719
|
+
try {
|
|
720
|
+
process.kill(-orphan.pid, "SIGKILL");
|
|
721
|
+
}
|
|
722
|
+
catch {
|
|
723
|
+
// gone
|
|
724
|
+
}
|
|
725
|
+
}
|
|
726
|
+
/** hq-cloud keeps operation locks in `<stateDir>/locks`. */
|
|
727
|
+
function defaultLocksDir() {
|
|
728
|
+
return path.join(getStateDir(), "locks");
|
|
729
|
+
}
|
|
730
|
+
/**
|
|
731
|
+
* How long a `--fix` pull waits for the operation lock. Unchanged from before
|
|
732
|
+
* the hand-off: a background pull must never queue on the lock longer than a
|
|
733
|
+
* foreground one did.
|
|
734
|
+
*/
|
|
735
|
+
export const REFRESH_SYNC_LOCK_TIMEOUT_SEC = 30;
|
|
622
736
|
/**
|
|
623
737
|
* Default `refresh-manifest` repair: a bounded `hq sync manifest --scope <s>`
|
|
624
738
|
* against the tree `--fix` is repairing. Exit 0 also covers throttled,
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Process start times, used to tell the owner of a pid from a later process
|
|
3
|
+
* that reused it (feedback cd56a501).
|
|
4
|
+
*/
|
|
5
|
+
import type { RefreshOwnerIdentity } from "./refresh-sync-handoff.js";
|
|
6
|
+
/**
|
|
7
|
+
* A process's start time, or null if it cannot be read: `ps -o lstart=` on
|
|
8
|
+
* POSIX, the process StartTime from PowerShell on Windows.
|
|
9
|
+
*/
|
|
10
|
+
export declare function processStartTime(pid: number): string | null;
|
|
11
|
+
/** Real process-table identity for the refresh spawn lock. */
|
|
12
|
+
export declare const DEFAULT_OWNER_IDENTITY: RefreshOwnerIdentity;
|
|
13
|
+
//# sourceMappingURL=process-identity.d.ts.map
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Process start times, used to tell the owner of a pid from a later process
|
|
3
|
+
* that reused it (feedback cd56a501).
|
|
4
|
+
*/
|
|
5
|
+
import { spawnSync } from "node:child_process";
|
|
6
|
+
/**
|
|
7
|
+
* A process's start time, or null if it cannot be read: `ps -o lstart=` on
|
|
8
|
+
* POSIX, the process StartTime from PowerShell on Windows.
|
|
9
|
+
*/
|
|
10
|
+
export function processStartTime(pid) {
|
|
11
|
+
if (!Number.isInteger(pid) || pid <= 0)
|
|
12
|
+
return null;
|
|
13
|
+
try {
|
|
14
|
+
const result = process.platform === "win32"
|
|
15
|
+
? spawnSync("powershell.exe", [
|
|
16
|
+
"-NoProfile",
|
|
17
|
+
"-NonInteractive",
|
|
18
|
+
"-Command",
|
|
19
|
+
`(Get-Process -Id ${pid} -ErrorAction Stop).StartTime.ToUniversalTime().ToString('o')`,
|
|
20
|
+
], { encoding: "utf8", timeout: 10_000, windowsHide: true })
|
|
21
|
+
: spawnSync("ps", ["-o", "lstart=", "-p", String(pid)], { encoding: "utf8", timeout: 5_000 });
|
|
22
|
+
const text = (result.stdout ?? "").trim();
|
|
23
|
+
return result.status === 0 && text ? text : null;
|
|
24
|
+
}
|
|
25
|
+
catch {
|
|
26
|
+
return null;
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
/** Real process-table identity for the refresh spawn lock. */
|
|
30
|
+
export const DEFAULT_OWNER_IDENTITY = {
|
|
31
|
+
selfStart: () => processStartTime(process.pid),
|
|
32
|
+
startOf: (pid) => processStartTime(pid),
|
|
33
|
+
};
|
|
34
|
+
//# sourceMappingURL=process-identity.js.map
|
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Supervised `hq doctor --fix` journal pulls (feedback cd56a501).
|
|
3
|
+
*
|
|
4
|
+
* A pull records `lastSync` only when its whole pass finishes. `--fix` used to
|
|
5
|
+
* kill its pull at 90s, so a large vault never refreshed its journal and the
|
|
6
|
+
* client-health hook reported it every day.
|
|
7
|
+
*
|
|
8
|
+
* Every `--fix` pull now runs under a small supervisor process. The
|
|
9
|
+
* supervisor writes a heartbeat while the pull runs, the pull's pid, and an
|
|
10
|
+
* exit file with the pull's real status when it ends. It stops the pull at
|
|
11
|
+
* {@link REFRESH_BACKGROUND_MAX_MS}. If the pull is still running at the
|
|
12
|
+
* foreground deadline, `--fix` leaves the supervisor running.
|
|
13
|
+
*
|
|
14
|
+
* One pull per HQ root. A run claims the root before it spawns
|
|
15
|
+
* ({@link claimRefreshRun}): holding an O_EXCL spawn lock, it checks that no
|
|
16
|
+
* other pull occupies the root and creates the pull's record exclusively. A
|
|
17
|
+
* second `--fix` started at the same moment either cannot get the lock or
|
|
18
|
+
* sees that record, and does not spawn.
|
|
19
|
+
*
|
|
20
|
+
* State lives in one file per company under
|
|
21
|
+
* `workspace/doctor/refresh-in-progress/`, and each run's output, heartbeat,
|
|
22
|
+
* pid and exit files are named by a per-run id.
|
|
23
|
+
*
|
|
24
|
+
* The next `--fix` harvests finished runs first: it reads the exit file and
|
|
25
|
+
* the log tail, reports the real outcome, and removes the record. A pull whose
|
|
26
|
+
* supervisor died is still alive while its process group is, until the cap.
|
|
27
|
+
* Past the cap the next run removes it as capped, and kills it only when the
|
|
28
|
+
* caller's identity check proves the pid is still that pull.
|
|
29
|
+
*/
|
|
30
|
+
/** The supervisor stops a background pull after this long. */
|
|
31
|
+
export declare const REFRESH_BACKGROUND_MAX_MS: number;
|
|
32
|
+
/** The supervisor refreshes its heartbeat this often. */
|
|
33
|
+
export declare const REFRESH_HEARTBEAT_INTERVAL_MS = 10000;
|
|
34
|
+
/** A heartbeat older than this means the supervisor is gone. */
|
|
35
|
+
export declare const REFRESH_HEARTBEAT_STALE_MS = 60000;
|
|
36
|
+
/** After SIGTERM, the supervisor waits this long before SIGKILL of the pull's process group. */
|
|
37
|
+
export declare const REFRESH_KILL_GRACE_MS = 5000;
|
|
38
|
+
/**
|
|
39
|
+
* A record older than this is over the cap whatever its heartbeat says. The
|
|
40
|
+
* supervisor stops the pull at {@link REFRESH_BACKGROUND_MAX_MS} and kills it
|
|
41
|
+
* {@link REFRESH_KILL_GRACE_MS} later; this adds a heartbeat interval of slack.
|
|
42
|
+
*/
|
|
43
|
+
export declare const REFRESH_RECORD_MAX_AGE_MS: number;
|
|
44
|
+
/**
|
|
45
|
+
* Reads a process's start time, to tell the owner of a pid from a later
|
|
46
|
+
* process that reused it. Supplied by the caller (it needs a process table
|
|
47
|
+
* probe). `null` means the start time could not be read.
|
|
48
|
+
*/
|
|
49
|
+
export interface RefreshOwnerIdentity {
|
|
50
|
+
/** This process's start time. */
|
|
51
|
+
selfStart: () => string | null;
|
|
52
|
+
/** Start time of `pid`, or null if unreadable. */
|
|
53
|
+
startOf: (pid: number) => string | null;
|
|
54
|
+
}
|
|
55
|
+
/** Files for one supervised pull. */
|
|
56
|
+
export interface RefreshRunPaths {
|
|
57
|
+
id: string;
|
|
58
|
+
log: string;
|
|
59
|
+
exit: string;
|
|
60
|
+
heartbeat: string;
|
|
61
|
+
/** The supervisor writes the pull's pid (its process group on POSIX) here. */
|
|
62
|
+
pid: string;
|
|
63
|
+
/** The supervisor writes the pull's process start time (`ps -o lstart=`) here, on POSIX. */
|
|
64
|
+
start: string;
|
|
65
|
+
/** Create this file to ask the supervisor to stop the pull. */
|
|
66
|
+
cancel: string;
|
|
67
|
+
}
|
|
68
|
+
/** Exit file the supervisor writes when the pull ends. */
|
|
69
|
+
export interface RefreshRunExit {
|
|
70
|
+
status: number | null;
|
|
71
|
+
signal: string | null;
|
|
72
|
+
/** True when the supervisor stopped the pull because another operation waited on the lock. */
|
|
73
|
+
preempted?: boolean;
|
|
74
|
+
/** True when the pull was stopped at the background time cap. */
|
|
75
|
+
capped?: boolean;
|
|
76
|
+
}
|
|
77
|
+
export declare function safeRefreshSlug(slug: string): string;
|
|
78
|
+
/** Fresh per-run file names under workspace/doctor/refresh-runs/. */
|
|
79
|
+
export declare function newRefreshRunPaths(hqRoot: string, slug: string): RefreshRunPaths;
|
|
80
|
+
/** Write (or replace) the record for a pull of `slug`. Tests and recovery only; claims use {@link claimRefreshRun}. */
|
|
81
|
+
export declare function recordRefreshHandoff(hqRoot: string, slug: string, run: RefreshRunPaths, nowMs: number): void;
|
|
82
|
+
/**
|
|
83
|
+
* An unreadable record younger than this may still be mid-write (the
|
|
84
|
+
* exclusive-create fallback), so harvest does not delete it yet.
|
|
85
|
+
*/
|
|
86
|
+
export declare const UNREADABLE_RECORD_SETTLE_MS = 10000;
|
|
87
|
+
/** Remove the record for `slug` if it still names `run`. */
|
|
88
|
+
export declare function clearRefreshRecord(hqRoot: string, slug: string, run: RefreshRunPaths): void;
|
|
89
|
+
/** Read a supervisor exit file, or null if the run has not ended. */
|
|
90
|
+
export declare function readRefreshRunExit(run: RefreshRunPaths): RefreshRunExit | null;
|
|
91
|
+
/** Last 64 KiB of a run's output: enough to classify, bounded in memory. */
|
|
92
|
+
export declare function readRefreshRunLog(run: RefreshRunPaths): string;
|
|
93
|
+
/** What an identity-checked killer needs to know about an orphaned pull. */
|
|
94
|
+
export interface OrphanPull {
|
|
95
|
+
pid: number;
|
|
96
|
+
/** Start time the supervisor recorded for the pull, or null if none. */
|
|
97
|
+
recordedStart: string | null;
|
|
98
|
+
/** When the record was written (the pull started at or before this). */
|
|
99
|
+
recordStartedAtMs: number;
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* Kills an orphaned pull only if it can prove the pid is still that pull.
|
|
103
|
+
* Supplied by the caller (it needs a process table probe); without one,
|
|
104
|
+
* harvest never kills.
|
|
105
|
+
*/
|
|
106
|
+
export type OrphanPullKiller = (orphan: OrphanPull, nowMs: number) => void;
|
|
107
|
+
/** An orphan is only ever killed within this long of its record. */
|
|
108
|
+
export declare const REFRESH_ORPHAN_KILL_WINDOW_MS: number;
|
|
109
|
+
/** Delete a run's files. */
|
|
110
|
+
export declare function removeRefreshRun(run: RefreshRunPaths): void;
|
|
111
|
+
/** A run that has ended (or died) and been removed. */
|
|
112
|
+
export interface HarvestedRefreshRun {
|
|
113
|
+
slug: string;
|
|
114
|
+
exit: RefreshRunExit | null;
|
|
115
|
+
output: string;
|
|
116
|
+
}
|
|
117
|
+
/**
|
|
118
|
+
* Remove every run that is no longer alive and return what each one ended
|
|
119
|
+
* with. `exit` is null when the supervisor died without writing one. A run
|
|
120
|
+
* over the cap is reported as capped, and an orphaned pull still running past
|
|
121
|
+
* the cap is killed. Live runs are left in place. Unreadable records are
|
|
122
|
+
* deleted.
|
|
123
|
+
*/
|
|
124
|
+
export declare function harvestRefreshHandoffs(hqRoot: string, nowMs: number, killOrphan?: OrphanPullKiller): HarvestedRefreshRun[];
|
|
125
|
+
/** Start time of the live pull for `slug`, or null. */
|
|
126
|
+
export declare function refreshHandoffStartedAt(hqRoot: string, slug: string, nowMs: number): number | null;
|
|
127
|
+
/**
|
|
128
|
+
* True when any `--fix` pull for this HQ root still occupies it: running,
|
|
129
|
+
* starting, orphaned, or capped and still inside its stop grace window.
|
|
130
|
+
*/
|
|
131
|
+
export declare function anyRefreshHandoffAlive(hqRoot: string, nowMs: number): boolean;
|
|
132
|
+
/**
|
|
133
|
+
* Take the per-root spawn lock: an O_EXCL file holding this process's pid,
|
|
134
|
+
* its start time and a random token.
|
|
135
|
+
*
|
|
136
|
+
* An existing lock is taken over only when identity proves its owner is gone:
|
|
137
|
+
* the pid is dead, or the pid is alive but its start time differs from the
|
|
138
|
+
* recorded one (the pid was reused). Age never counts. If the owner's start
|
|
139
|
+
* time cannot be verified the lock is left alone and this returns null.
|
|
140
|
+
*/
|
|
141
|
+
/** Releases the spawn lock; `stillHeld` re-reads the lock file for this holder's token. */
|
|
142
|
+
export type SpawnLockRelease = (() => void) & {
|
|
143
|
+
stillHeld: () => boolean;
|
|
144
|
+
};
|
|
145
|
+
export declare function acquireRefreshSpawnLock(hqRoot: string, nowMs: number, identity?: RefreshOwnerIdentity): SpawnLockRelease | null;
|
|
146
|
+
/** A spawn lock still held after this long is reported by `hq doctor`. */
|
|
147
|
+
export declare const REFRESH_SPAWN_LOCK_WARN_MS: number;
|
|
148
|
+
/**
|
|
149
|
+
* The current spawn lock holder and how long it has held the lock, or null
|
|
150
|
+
* when no lock is held. Read-only: never removes or changes the lock.
|
|
151
|
+
*/
|
|
152
|
+
export declare function inspectRefreshSpawnLock(hqRoot: string, nowMs: number, identity?: RefreshOwnerIdentity): {
|
|
153
|
+
pid: number;
|
|
154
|
+
heldMs: number | null;
|
|
155
|
+
ownerGone: boolean;
|
|
156
|
+
} | null;
|
|
157
|
+
/**
|
|
158
|
+
* Release a spawn lock whose owner is proven gone (pid dead, or start time
|
|
159
|
+
* changed). A live or unverifiable owner's lock is left alone. Uses the same
|
|
160
|
+
* atomic takeover as a claim, then releases at once.
|
|
161
|
+
*/
|
|
162
|
+
export declare function releaseDeadRefreshSpawnLock(hqRoot: string, nowMs: number, identity?: RefreshOwnerIdentity): void;
|
|
163
|
+
/**
|
|
164
|
+
* Claim this HQ root for a pull of `slug`; true means spawn.
|
|
165
|
+
*
|
|
166
|
+
* - The spawn lock is required. If another live or unverifiable process holds
|
|
167
|
+
* it, nothing is written and the claim fails.
|
|
168
|
+
* - Under the lock: no other pull may occupy the root, and the record is
|
|
169
|
+
* created exclusively, never overwritten. An existing record for `slug`
|
|
170
|
+
* wins unless it no longer occupies the root (its pull exited, or its
|
|
171
|
+
* owner and pull are proven gone); then it is moved aside atomically and
|
|
172
|
+
* the create is retried once.
|
|
173
|
+
* - The claim holds only if, after writing, the record still carries this
|
|
174
|
+
* claim's token and this process still holds the lock.
|
|
175
|
+
*/
|
|
176
|
+
export declare function claimRefreshRun(hqRoot: string, slug: string, run: RefreshRunPaths, nowMs: number, identity?: RefreshOwnerIdentity): boolean;
|
|
177
|
+
/** How often the supervisor looks for another operation waiting on the lock. */
|
|
178
|
+
export declare const REFRESH_PREEMPT_POLL_MS = 500;
|
|
179
|
+
/**
|
|
180
|
+
* Supervisor program, run as
|
|
181
|
+
* `node -e SUPERVISOR log exit heartbeat pidFile cancel locksDir maxMs cmd ...args`.
|
|
182
|
+
*
|
|
183
|
+
* It appends the pull's output to `log`, writes the pull's pid to `pidFile`,
|
|
184
|
+
* refreshes `heartbeat`, and writes `{status, signal, preempted, capped}` to
|
|
185
|
+
* `exit` when the pull ends.
|
|
186
|
+
*
|
|
187
|
+
* Stopping: at `maxMs`, when `cancel` exists, or when another live process
|
|
188
|
+
* publishes a waiter for the operation lock the pull holds
|
|
189
|
+
* (`<lock>.wait-<pid>-*`; `sync` holds the lock shared, so only exclusive
|
|
190
|
+
* callers such as rescue and repairs wait), the supervisor stops the pull.
|
|
191
|
+
* On POSIX it sends SIGTERM to the pull's process group, and SIGKILL to the
|
|
192
|
+
* group after {@link REFRESH_KILL_GRACE_MS} in every case, so a grandchild
|
|
193
|
+
* cannot outlive it. SIGTERM is the same stop as Ctrl-C on `hq sync`: the
|
|
194
|
+
* lock's signal handler releases the lock, downloads are staged and renamed
|
|
195
|
+
* into place, and the journal is written by atomic rename. On Windows the
|
|
196
|
+
* stop is `taskkill /T /F` on the tree, which is a forced kill, not a
|
|
197
|
+
* cooperative one; the same staged writes and atomic journal keep completed
|
|
198
|
+
* work.
|
|
199
|
+
*/
|
|
200
|
+
export declare const REFRESH_SUPERVISOR_SOURCE = "\nconst fs = require(\"node:fs\");\nconst path = require(\"node:path\");\nconst { spawn, spawnSync } = require(\"node:child_process\");\nconst [logPath, exitPath, hbPath, pidPath, cancelPath, locksDir, maxMs, cmd, ...args] = process.argv.slice(1);\nconst isWin = process.platform === \"win32\";\nconst beat = () => { try { fs.writeFileSync(hbPath, String(Date.now())); } catch {} };\nbeat();\nconst hb = setInterval(beat, 10000);\nlet fd;\ntry { fd = fs.openSync(logPath, \"a\", 0o600); } catch { fd = \"ignore\"; }\nlet finished = false;\nlet preempted = false;\nlet capped = false;\nlet stopAt = 0;\nlet watch;\n// Its own process group on POSIX, so a stop reaches every process the pull started.\nconst child = spawn(cmd, args, { stdio: [\"ignore\", fd, fd], windowsHide: true, detached: !isWin });\ntry { fs.writeFileSync(pidPath, String(child.pid)); } catch {}\nif (!isWin && child.pid) {\n try {\n const r = spawnSync(\"ps\", [\"-o\", \"lstart=\", \"-p\", String(child.pid)], { encoding: \"utf8\" });\n const started = (r.stdout || \"\").trim();\n if (started) fs.writeFileSync(path.join(path.dirname(pidPath), path.basename(pidPath, \".pid\") + \".start\"), started);\n } catch {}\n}\nconst killTree = (sig) => {\n if (isWin) {\n try { spawnSync(\"taskkill\", [\"/PID\", String(child.pid), \"/T\", \"/F\"], { stdio: \"ignore\", windowsHide: true }); } catch {}\n return;\n }\n try { process.kill(-child.pid, sig); } catch {}\n};\nconst stop = () => {\n if (stopAt) return;\n stopAt = Date.now();\n killTree(\"SIGTERM\");\n setTimeout(() => killTree(\"SIGKILL\"), 5000);\n};\nconst exitNow = (status) => process.exit(typeof status === \"number\" ? status : 1);\nconst done = (status, signal) => {\n if (finished) return;\n finished = true;\n clearInterval(hb);\n clearInterval(watch);\n clearTimeout(stopper);\n try {\n fs.writeFileSync(exitPath + \".tmp\", JSON.stringify({ status, signal, preempted, capped }));\n fs.renameSync(exitPath + \".tmp\", exitPath);\n } catch {}\n if (!stopAt) return exitNow(status);\n // A stopped pull's group is always SIGKILLed after the grace period, even\n // when the pull itself exited on SIGTERM, so no grandchild survives.\n const left = Math.max(0, stopAt + 5000 - Date.now());\n setTimeout(() => { killTree(\"SIGKILL\"); exitNow(status); }, left);\n};\nconst stopper = setTimeout(() => {\n capped = true;\n // At the cap the pull is no longer live: stop heartbeating so nothing reads it as in progress.\n clearInterval(hb);\n try { fs.writeFileSync(hbPath, \"capped:\" + Date.now()); } catch {}\n stop();\n}, Number(maxMs));\nconst alive = (pid) => { try { process.kill(pid, 0); return true; } catch (e) { return Boolean(e && e.code === \"EPERM\"); } };\nlet lockName = null;\nconst findLock = () => {\n let names;\n try { names = fs.readdirSync(locksDir); } catch { return null; }\n for (const name of names) {\n // A sync holds shared membership: <lock>.shared-<pid>-<hex>.\n const shared = /^(.+\\.lock)\\.shared-(\\d+)-/.exec(name);\n if (shared && Number(shared[2]) === child.pid) return shared[1];\n if (!name.endsWith(\".lock\")) continue;\n // An exclusive holder writes its record into the lock file itself.\n try {\n const info = JSON.parse(fs.readFileSync(path.join(locksDir, name), \"utf8\"));\n if (info && info.pid === child.pid) return name;\n } catch {}\n }\n return null;\n};\nconst someoneWaiting = () => {\n if (fs.existsSync(cancelPath)) return true;\n if (!lockName) lockName = findLock();\n if (!lockName) return false;\n let names;\n try { names = fs.readdirSync(locksDir); } catch { return false; }\n const prefix = lockName + \".wait-\";\n for (const name of names) {\n if (!name.startsWith(prefix)) continue;\n const waiter = Number(name.slice(prefix.length).split(\"-\")[0]);\n if (Number.isInteger(waiter) && waiter > 0 && waiter !== child.pid && waiter !== process.pid && alive(waiter)) return true;\n }\n return false;\n};\nwatch = setInterval(() => {\n if (finished || stopAt) return;\n if (someoneWaiting()) {\n preempted = true;\n stop();\n }\n}, 500);\nchild.on(\"error\", (error) => {\n try { fs.appendFileSync(logPath, String((error && error.code) || error) + \"\\n\"); } catch {}\n done(null, null);\n});\nchild.on(\"exit\", (status, signal) => done(status, signal));\n";
|
|
201
|
+
//# sourceMappingURL=refresh-sync-handoff.d.ts.map
|