@indigoai-us/hq-cli 5.345.1 → 5.345.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +10 -0
- package/dist/commands/daemon.js +6 -2
- package/dist/commands/lanes-message.js +0 -3
- package/dist/commands/lanes-resume.js +0 -3
- package/dist/commands/lanes.js +1 -4
- package/dist/commands/mcp-serve.js +15 -5
- package/dist/hq-cloud-sync-runner/bin/sync-runner.js +135416 -0
- package/dist/hq-cloud-sync-runner/package.json +1 -0
- package/dist/index.js +2 -1
- package/dist/lib/daemon/hq-child-bootstrap-stage.d.ts +22 -0
- package/dist/lib/daemon/hq-child-bootstrap-stage.js +127 -0
- package/dist/lib/daemon/hq-child-bootstrap.d.mts +3 -0
- package/dist/lib/daemon/hq-child-bootstrap.mjs +208 -0
- package/dist/lib/daemon/hq-cloud.d.ts +5 -0
- package/dist/lib/daemon/hq-cloud.js +14 -0
- package/dist/lib/daemon/run.d.ts +28 -2
- package/dist/lib/daemon/run.js +139 -10
- package/dist/lib/daemon/sync-runner-bundle.d.ts +40 -0
- package/dist/lib/daemon/sync-runner-bundle.js +199 -0
- package/dist/lib/lanes/disk-guard.d.ts +3 -4
- package/dist/lib/lanes/disk-guard.js +7 -6
- package/dist/lib/lanes/reconcile.js +22 -28
- package/dist/lib/lanes/spawn.d.ts +0 -2
- package/dist/lib/lanes/spawn.js +21 -29
- package/dist/lib/lanes/watcher.js +6 -0
- package/dist/utils/package-use-lease.d.ts +6 -0
- package/dist/utils/package-use-lease.js +25 -2
- package/package.json +2 -2
package/dist/lib/daemon/run.js
CHANGED
|
@@ -10,10 +10,11 @@ import * as fs from "node:fs";
|
|
|
10
10
|
import { createRequire } from "node:module";
|
|
11
11
|
import * as os from "node:os";
|
|
12
12
|
import * as path from "node:path";
|
|
13
|
+
import { fileURLToPath } from "node:url";
|
|
13
14
|
import { loadCachedTokens } from "@indigoai-us/hq-cloud";
|
|
14
15
|
import { CLI_VERSION } from "../../cli-version.js";
|
|
15
16
|
import { anywhereRuntimeEnabled } from "../../commands/anywhere-runtime-flag.js";
|
|
16
|
-
import { currentPackageUseUpdateRequested } from "../../utils/package-use-lease.js";
|
|
17
|
+
import { currentPackageUseLeaseInstall, currentPackageUseUpdateRequested } from "../../utils/package-use-lease.js";
|
|
17
18
|
import { isLocalDependencyInstall, resolveRunningInstall } from "../../utils/version-gate.js";
|
|
18
19
|
import { agentKitPaths } from "../agent-kit/paths.js";
|
|
19
20
|
import { readBotConfig } from "../bot/config.js";
|
|
@@ -36,7 +37,9 @@ import { startHqdServer } from "./server.js";
|
|
|
36
37
|
import { heartbeatFile, startHeartbeat } from "./heartbeat.js";
|
|
37
38
|
import { createProtocolTail, ensureProtocolFifo, openProtocolWriter } from "./protocol-tail.js";
|
|
38
39
|
import { takeControlRequests } from "./requests.js";
|
|
39
|
-
import { currentHqCloud, hqCloudRunners } from "./hq-cloud.js";
|
|
40
|
+
import { currentHqCloud, currentHqCloudAfterLeaseRelease, hqCloudRunners } from "./hq-cloud.js";
|
|
41
|
+
import { stageSyncRunnerBundle } from "./sync-runner-bundle.js";
|
|
42
|
+
import { stageHqChildBootstrap } from "./hq-child-bootstrap-stage.js";
|
|
40
43
|
import { runnerSessionName } from "./remote-control.js";
|
|
41
44
|
import { adoptChild, createChildTracker, finishSuccession, handoffEligible, readProcStart, stopOrphans, successionPaths, systemdKillMode, takeSuccession, writeHandoff, } from "./succession.js";
|
|
42
45
|
import { listSyncRunnerProcesses, readWatchRunnerOwnerStatus, syncHeartbeatTimeoutMs, syncRunnerNodeOptions, watchRunnerOwnerStatusPath, } from "./sync-runner.js";
|
|
@@ -73,6 +76,48 @@ export function handoffForPackageUpdateRequest(requested, restartDaemon) {
|
|
|
73
76
|
const LEGACY_PROBE_TTL_MS = 60_000;
|
|
74
77
|
/** A flag read that has not answered by then counts as off. */
|
|
75
78
|
const HANDOFF_FLAG_TIMEOUT_MS = 10_000;
|
|
79
|
+
export function syncRunnerSourceForPlan(stagedRunnerPath, hqCloud, legacyBundledRunnerPath = null) {
|
|
80
|
+
if (stagedRunnerPath)
|
|
81
|
+
return { kind: "staged-sync", runnerPath: stagedRunnerPath };
|
|
82
|
+
if (!hqCloud) {
|
|
83
|
+
return legacyBundledRunnerPath
|
|
84
|
+
? { kind: "bundled-full-package", runnerPath: legacyBundledRunnerPath }
|
|
85
|
+
: { kind: "unavailable", runnerPath: null };
|
|
86
|
+
}
|
|
87
|
+
// The caller has already selected a usable full package; preserve the
|
|
88
|
+
// conventional path here, including in tests with virtual package roots.
|
|
89
|
+
const runnerPath = path.join(hqCloud.root, "dist", "bin", "sync-runner.js");
|
|
90
|
+
return hqCloud.source === "managed"
|
|
91
|
+
? { kind: "managed-full-package", runnerPath }
|
|
92
|
+
: { kind: "bundled-full-package", runnerPath };
|
|
93
|
+
}
|
|
94
|
+
function isPathWithin(root, candidate) {
|
|
95
|
+
if (!root || !path.isAbsolute(candidate))
|
|
96
|
+
return false;
|
|
97
|
+
const relative = path.relative(path.resolve(root), path.resolve(candidate));
|
|
98
|
+
return relative === "" || (!relative.startsWith(`..${path.sep}`) && relative !== ".." && !path.isAbsolute(relative));
|
|
99
|
+
}
|
|
100
|
+
/** Whether every executable in the first service plan remains usable after the CLI package is replaced. */
|
|
101
|
+
export function daemonPlanCanReleasePackageUseLease(plans, replaceablePackageRoots, stagingFailed, hqBootstrapReady = false) {
|
|
102
|
+
if (stagingFailed)
|
|
103
|
+
return false;
|
|
104
|
+
return plans.every((unit) => {
|
|
105
|
+
const command = unit.command;
|
|
106
|
+
if (!command)
|
|
107
|
+
return true;
|
|
108
|
+
if (command.type === "hq")
|
|
109
|
+
return hqBootstrapReady;
|
|
110
|
+
if (command.type === "internal")
|
|
111
|
+
return true;
|
|
112
|
+
const executablePaths = command.type === "node" ? [command.script, ...command.args] : [command.file, ...command.args];
|
|
113
|
+
return executablePaths.every((candidate) => replaceablePackageRoots.every((root) => !isPathWithin(root, candidate)));
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
export function daemonHqChildCommand(nodeBinary, hqBinary, args, bootstrapPath, packagePrefix) {
|
|
117
|
+
return bootstrapPath && packagePrefix
|
|
118
|
+
? [nodeBinary, [bootstrapPath, packagePrefix, CLI_VERSION, ...args]]
|
|
119
|
+
: [nodeBinary, [hqBinary, ...args]];
|
|
120
|
+
}
|
|
76
121
|
export function resolveSyncRunnerPath() {
|
|
77
122
|
try {
|
|
78
123
|
const require = createRequire(import.meta.url);
|
|
@@ -304,6 +349,60 @@ export async function runDaemon(opts) {
|
|
|
304
349
|
const platform = process.platform;
|
|
305
350
|
const paths = daemonPaths(home, env);
|
|
306
351
|
const log = makeLogger(paths, opts.foreground === true);
|
|
352
|
+
const syncRunnerBundleDirectory = opts.syncRunnerBundleDirectory ?? path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../../hq-cloud-sync-runner");
|
|
353
|
+
const cliPackageRoot = opts.cliPackageRoot === undefined ? resolveRunningInstall().packageRoot : opts.cliPackageRoot;
|
|
354
|
+
const cliPackagePrefix = opts.cliPackagePrefix === undefined
|
|
355
|
+
? currentPackageUseLeaseInstall()?.prefix ?? null
|
|
356
|
+
: opts.cliPackagePrefix;
|
|
357
|
+
let stagedSyncBundle = null;
|
|
358
|
+
let bundleStageChecked = false;
|
|
359
|
+
let bundleStageFailed = false;
|
|
360
|
+
let stagedHqChildBootstrap = null;
|
|
361
|
+
let hqChildBootstrapStageChecked = false;
|
|
362
|
+
let hqChildBootstrapStageFailed = false;
|
|
363
|
+
let legacySyncRunnerPathChecked = false;
|
|
364
|
+
let legacySyncRunnerPath = null;
|
|
365
|
+
let packageUseLeaseReleased = false;
|
|
366
|
+
let hqCloudVersionBeforeRelease = null;
|
|
367
|
+
let startupPlanOpen = true;
|
|
368
|
+
const startupPlans = [];
|
|
369
|
+
const getStagedSyncBundle = () => {
|
|
370
|
+
if (bundleStageChecked)
|
|
371
|
+
return stagedSyncBundle;
|
|
372
|
+
bundleStageChecked = true;
|
|
373
|
+
// Older and development installs have no publish-time payload; keep their
|
|
374
|
+
// existing hq-cloud selection path without treating absence as an error.
|
|
375
|
+
if (!fs.existsSync(syncRunnerBundleDirectory))
|
|
376
|
+
return null;
|
|
377
|
+
try {
|
|
378
|
+
stagedSyncBundle = stageSyncRunnerBundle({ payloadDirectory: syncRunnerBundleDirectory, daemonRoot: paths.root });
|
|
379
|
+
return stagedSyncBundle;
|
|
380
|
+
}
|
|
381
|
+
catch {
|
|
382
|
+
bundleStageFailed = true;
|
|
383
|
+
log("warn", "could not stage the packaged sync runner; keeping the package-use lease");
|
|
384
|
+
return null;
|
|
385
|
+
}
|
|
386
|
+
};
|
|
387
|
+
const getStagedHqChildBootstrap = () => {
|
|
388
|
+
if (hqChildBootstrapStageChecked)
|
|
389
|
+
return stagedHqChildBootstrap;
|
|
390
|
+
hqChildBootstrapStageChecked = true;
|
|
391
|
+
if (!cliPackagePrefix)
|
|
392
|
+
return null;
|
|
393
|
+
try {
|
|
394
|
+
stagedHqChildBootstrap = stageHqChildBootstrap({
|
|
395
|
+
sourcePath: opts.hqChildBootstrapSourcePath,
|
|
396
|
+
daemonRoot: paths.root,
|
|
397
|
+
});
|
|
398
|
+
return stagedHqChildBootstrap;
|
|
399
|
+
}
|
|
400
|
+
catch {
|
|
401
|
+
hqChildBootstrapStageFailed = true;
|
|
402
|
+
log("warn", "could not stage the daemon hq bootstrap; keeping the package-use lease");
|
|
403
|
+
return null;
|
|
404
|
+
}
|
|
405
|
+
};
|
|
307
406
|
const lock = acquireDaemonLock(paths.lockPath);
|
|
308
407
|
const lockConflict = daemonLockConflictOutcome(lock, !lock.ok && lock.owner ? anywhereRuntimeEnabled(env) : false);
|
|
309
408
|
if (lockConflict) {
|
|
@@ -374,8 +473,6 @@ export async function runDaemon(opts) {
|
|
|
374
473
|
finishSuccession(succession, tracker.records(), log);
|
|
375
474
|
/** Service processes this daemon spawned, so a handoff can let go of them. */
|
|
376
475
|
const spawned = new Set();
|
|
377
|
-
if (!resolveSyncRunnerPath())
|
|
378
|
-
log("warn", "the hq-cloud sync runner was not found in this CLI install; sync is unavailable");
|
|
379
476
|
const updatable = cliSelfUpdatable();
|
|
380
477
|
const hostName = os.hostname().split(".")[0] || "hq-host";
|
|
381
478
|
let legacyCache = null;
|
|
@@ -435,14 +532,22 @@ export async function runDaemon(opts) {
|
|
|
435
532
|
return `mesh daemon already running (pid ${owner.pid}); waiting for it to exit`;
|
|
436
533
|
};
|
|
437
534
|
let warnedMissingPin = null;
|
|
535
|
+
let warnedMissingSyncRunner = false;
|
|
438
536
|
const daemonSyncRunnerPids = new Set();
|
|
439
537
|
const plan = () => {
|
|
440
538
|
const config = readDaemonConfig(paths.configPath);
|
|
441
539
|
const legacy = legacyUnits();
|
|
442
540
|
const configured = config.hqRoot ?? (env.HQ_ROOT?.trim() || undefined);
|
|
443
541
|
const hqRoot = configured && fs.existsSync(configured) ? configured : undefined;
|
|
444
|
-
|
|
445
|
-
const
|
|
542
|
+
getStagedHqChildBootstrap();
|
|
543
|
+
const staged = getStagedSyncBundle();
|
|
544
|
+
// Before release, preserve today's selection. Later plans can only use the
|
|
545
|
+
// daemon-owned full package; the CLI-bundled tree may have been replaced.
|
|
546
|
+
const hqCloud = packageUseLeaseReleased
|
|
547
|
+
? currentHqCloudAfterLeaseRelease(paths.root, hqCloudVersionBeforeRelease)
|
|
548
|
+
: currentHqCloud(paths.root);
|
|
549
|
+
if (!packageUseLeaseReleased)
|
|
550
|
+
hqCloudVersionBeforeRelease = hqCloud?.version ?? staged?.metadata.version ?? null;
|
|
446
551
|
const missingPin = hqCloud?.pinnedVersionMissing ?? null;
|
|
447
552
|
if (missingPin !== warnedMissingPin) {
|
|
448
553
|
warnedMissingPin = missingPin;
|
|
@@ -450,7 +555,17 @@ export async function runDaemon(opts) {
|
|
|
450
555
|
log("warn", `hq-cloud is pinned to ${missingPin}, which is not installed; running ${hqCloud.version} (${hqCloud.source}) until the updater installs it`);
|
|
451
556
|
}
|
|
452
557
|
}
|
|
453
|
-
|
|
558
|
+
if (!packageUseLeaseReleased && !legacySyncRunnerPathChecked) {
|
|
559
|
+
legacySyncRunnerPath = resolveSyncRunnerPath();
|
|
560
|
+
legacySyncRunnerPathChecked = true;
|
|
561
|
+
}
|
|
562
|
+
const syncSource = syncRunnerSourceForPlan(staged?.runnerPath ?? null, hqCloud, packageUseLeaseReleased ? null : legacySyncRunnerPath);
|
|
563
|
+
if (!syncSource.runnerPath && !warnedMissingSyncRunner) {
|
|
564
|
+
warnedMissingSyncRunner = true;
|
|
565
|
+
log("warn", "the hq-cloud sync runner was not found in this CLI install; sync is unavailable");
|
|
566
|
+
}
|
|
567
|
+
const packageRunners = hqCloud ? hqCloudRunners(hqCloud.root) : { syncRunner: null, heartbeat: null, sessionHost: null };
|
|
568
|
+
const runners = { ...packageRunners, syncRunner: syncSource.runnerPath };
|
|
454
569
|
const services = enabledServices(machineType, config);
|
|
455
570
|
const isSignedIn = signedIn(env, home, log);
|
|
456
571
|
const syncOwner = effectiveSyncOwner(config, machineType);
|
|
@@ -477,7 +592,7 @@ export async function runDaemon(opts) {
|
|
|
477
592
|
}
|
|
478
593
|
}
|
|
479
594
|
}
|
|
480
|
-
|
|
595
|
+
const plans = planUnits(services, {
|
|
481
596
|
machineType,
|
|
482
597
|
platform,
|
|
483
598
|
home,
|
|
@@ -512,13 +627,23 @@ export async function runDaemon(opts) {
|
|
|
512
627
|
tmuxPath: findOnPath("tmux", env),
|
|
513
628
|
hostName,
|
|
514
629
|
});
|
|
630
|
+
if (startupPlanOpen) {
|
|
631
|
+
const replaceablePackageRoots = cliPackageRoot ? [cliPackageRoot] : [];
|
|
632
|
+
if (hqCloud?.source === "bundled")
|
|
633
|
+
replaceablePackageRoots.push(hqCloud.root);
|
|
634
|
+
else if (!hqCloud && legacySyncRunnerPath) {
|
|
635
|
+
replaceablePackageRoots.push(path.dirname(path.dirname(path.dirname(legacySyncRunnerPath))));
|
|
636
|
+
}
|
|
637
|
+
startupPlans.push({ plans, replaceablePackageRoots });
|
|
638
|
+
}
|
|
639
|
+
return plans;
|
|
515
640
|
};
|
|
516
641
|
const spawnUnit = (unit) => {
|
|
517
642
|
const command = unit.command;
|
|
518
643
|
if (!command || command.type === "internal")
|
|
519
644
|
throw new Error(`${unit.id} has no process to start`);
|
|
520
645
|
const [file, args] = command.type === "hq"
|
|
521
|
-
?
|
|
646
|
+
? daemonHqChildCommand(opts.nodeBinary, opts.hqBinary, command.args, stagedHqChildBootstrap?.bootstrapPath ?? null, cliPackagePrefix)
|
|
522
647
|
: command.type === "node"
|
|
523
648
|
? [opts.nodeBinary, [command.script, ...command.args]]
|
|
524
649
|
: [command.file, command.args];
|
|
@@ -831,7 +956,11 @@ export async function runDaemon(opts) {
|
|
|
831
956
|
resolve(1);
|
|
832
957
|
}
|
|
833
958
|
finally {
|
|
834
|
-
|
|
959
|
+
startupPlanOpen = false;
|
|
960
|
+
const startupPlanIsStable = startupPlans.length > 0 && startupPlans.every(({ plans, replaceablePackageRoots }) => daemonPlanCanReleasePackageUseLease(plans, replaceablePackageRoots, bundleStageFailed || hqChildBootstrapStageFailed, Boolean(stagedHqChildBootstrap && cliPackagePrefix)));
|
|
961
|
+
if (startupPlanIsStable) {
|
|
962
|
+
packageUseLeaseReleased = opts.onStartupReadsComplete?.() === true;
|
|
963
|
+
}
|
|
835
964
|
}
|
|
836
965
|
});
|
|
837
966
|
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/** Atomic staging for the sync-runner bundle shipped with hq-cli. */
|
|
2
|
+
import * as fs from "node:fs";
|
|
3
|
+
export declare const SYNC_RUNNER_BUNDLE_RUNNER_FILE = "bin/sync-runner.js";
|
|
4
|
+
export declare const SYNC_RUNNER_BUNDLE_METADATA_FILE = "package.json";
|
|
5
|
+
export declare const SYNC_RUNNER_BUNDLE_DIRECTORY = "hq-cloud-sync-runner";
|
|
6
|
+
export declare const SYNC_RUNNER_BUNDLE_FORMAT_VERSION = 1;
|
|
7
|
+
/** package.json contract that the hq-cli build step emits beside the bundled bin directory. */
|
|
8
|
+
export interface SyncRunnerBundleMetadata {
|
|
9
|
+
name: "hq-cli-daemon-sync-runner";
|
|
10
|
+
type: "module";
|
|
11
|
+
version: string;
|
|
12
|
+
bundleSha256: string;
|
|
13
|
+
formatVersion: typeof SYNC_RUNNER_BUNDLE_FORMAT_VERSION;
|
|
14
|
+
}
|
|
15
|
+
export type SyncRunnerBundleStageErrorCode = "PAYLOAD_MISSING" | "MISSING_SIDECAR" | "INVALID_METADATA" | "VERSION_OUT_OF_RANGE" | "HASH_MISMATCH" | "STAGE_EXISTS_INVALID" | "COPY_FAILED" | "FSYNC_FAILED" | "RENAME_FAILED";
|
|
16
|
+
export declare class SyncRunnerBundleStageError extends Error {
|
|
17
|
+
readonly code: SyncRunnerBundleStageErrorCode;
|
|
18
|
+
constructor(code: SyncRunnerBundleStageErrorCode, message: string, options?: ErrorOptions);
|
|
19
|
+
}
|
|
20
|
+
export interface StagedSyncRunnerBundle {
|
|
21
|
+
directory: string;
|
|
22
|
+
runnerPath: string;
|
|
23
|
+
metadataPath: string;
|
|
24
|
+
metadata: SyncRunnerBundleMetadata;
|
|
25
|
+
}
|
|
26
|
+
export interface StageSyncRunnerBundleOptions {
|
|
27
|
+
/** Directory in the installed hq-cli dist containing both contract files. */
|
|
28
|
+
payloadDirectory: string;
|
|
29
|
+
/** Daemon-owned state root, normally daemonPaths().root. */
|
|
30
|
+
daemonRoot: string;
|
|
31
|
+
/** Test seams for the file copy and atomic directory rename. */
|
|
32
|
+
copyFile?: typeof fs.copyFileSync;
|
|
33
|
+
renameDirectory?: (from: string, to: string) => void;
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Stage one validated runner bundle into an immutable daemon-owned directory.
|
|
37
|
+
* Only a completed, fsynced directory is returned; temp siblings are ignored.
|
|
38
|
+
*/
|
|
39
|
+
export declare function stageSyncRunnerBundle(options: StageSyncRunnerBundleOptions): StagedSyncRunnerBundle;
|
|
40
|
+
//# sourceMappingURL=sync-runner-bundle.d.ts.map
|
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
/** Atomic staging for the sync-runner bundle shipped with hq-cli. */
|
|
2
|
+
import { createHash } from "node:crypto";
|
|
3
|
+
import * as fs from "node:fs";
|
|
4
|
+
import * as path from "node:path";
|
|
5
|
+
import { fileURLToPath } from "node:url";
|
|
6
|
+
import semver from "semver";
|
|
7
|
+
import { ensurePrivateDir } from "./paths.js";
|
|
8
|
+
export const SYNC_RUNNER_BUNDLE_RUNNER_FILE = "bin/sync-runner.js";
|
|
9
|
+
export const SYNC_RUNNER_BUNDLE_METADATA_FILE = "package.json";
|
|
10
|
+
export const SYNC_RUNNER_BUNDLE_DIRECTORY = "hq-cloud-sync-runner";
|
|
11
|
+
export const SYNC_RUNNER_BUNDLE_FORMAT_VERSION = 1;
|
|
12
|
+
export class SyncRunnerBundleStageError extends Error {
|
|
13
|
+
code;
|
|
14
|
+
constructor(code, message, options) {
|
|
15
|
+
super(message, options);
|
|
16
|
+
this.code = code;
|
|
17
|
+
this.name = "SyncRunnerBundleStageError";
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
function fail(code, message, cause) {
|
|
21
|
+
if (cause instanceof SyncRunnerBundleStageError)
|
|
22
|
+
throw cause;
|
|
23
|
+
throw new SyncRunnerBundleStageError(code, message, cause === undefined ? undefined : { cause });
|
|
24
|
+
}
|
|
25
|
+
function readMetadata(file) {
|
|
26
|
+
let raw;
|
|
27
|
+
try {
|
|
28
|
+
raw = fs.readFileSync(file, "utf8");
|
|
29
|
+
}
|
|
30
|
+
catch (error) {
|
|
31
|
+
if (error.code === "ENOENT") {
|
|
32
|
+
fail("MISSING_SIDECAR", "The sync-runner bundle metadata sidecar is missing", error);
|
|
33
|
+
}
|
|
34
|
+
fail("PAYLOAD_MISSING", "Unable to read the sync-runner bundle metadata", error);
|
|
35
|
+
}
|
|
36
|
+
let value;
|
|
37
|
+
try {
|
|
38
|
+
value = JSON.parse(raw);
|
|
39
|
+
}
|
|
40
|
+
catch (error) {
|
|
41
|
+
fail("INVALID_METADATA", "The sync-runner bundle metadata is invalid JSON", error);
|
|
42
|
+
}
|
|
43
|
+
if (!value || typeof value !== "object") {
|
|
44
|
+
fail("INVALID_METADATA", "The sync-runner bundle metadata has an invalid shape");
|
|
45
|
+
}
|
|
46
|
+
const metadata = value;
|
|
47
|
+
if (metadata.name !== "hq-cli-daemon-sync-runner"
|
|
48
|
+
|| metadata.type !== "module"
|
|
49
|
+
|| metadata.formatVersion !== SYNC_RUNNER_BUNDLE_FORMAT_VERSION
|
|
50
|
+
|| typeof metadata.version !== "string"
|
|
51
|
+
|| !semver.valid(metadata.version)
|
|
52
|
+
|| typeof metadata.bundleSha256 !== "string"
|
|
53
|
+
|| !/^[a-f0-9]{64}$/.test(metadata.bundleSha256)) {
|
|
54
|
+
fail("INVALID_METADATA", "The sync-runner bundle metadata has an invalid shape");
|
|
55
|
+
}
|
|
56
|
+
return metadata;
|
|
57
|
+
}
|
|
58
|
+
function hqCloudRange() {
|
|
59
|
+
const cliPackagePath = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../../../package.json");
|
|
60
|
+
try {
|
|
61
|
+
const pkg = JSON.parse(fs.readFileSync(cliPackagePath, "utf8"));
|
|
62
|
+
const range = pkg.dependencies?.["@indigoai-us/hq-cloud"];
|
|
63
|
+
if (typeof range !== "string" || !semver.validRange(range))
|
|
64
|
+
throw new Error("invalid declared range");
|
|
65
|
+
return range;
|
|
66
|
+
}
|
|
67
|
+
catch (error) {
|
|
68
|
+
fail("INVALID_METADATA", "Unable to read the declared hq-cloud version range", error);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
function sha256(file) {
|
|
72
|
+
return createHash("sha256").update(fs.readFileSync(file)).digest("hex");
|
|
73
|
+
}
|
|
74
|
+
function syncFile(file) {
|
|
75
|
+
const fd = fs.openSync(file, "r");
|
|
76
|
+
try {
|
|
77
|
+
fs.fsyncSync(fd);
|
|
78
|
+
}
|
|
79
|
+
finally {
|
|
80
|
+
fs.closeSync(fd);
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
function syncDirectory(directory) {
|
|
84
|
+
const fd = fs.openSync(directory, fs.constants.O_RDONLY);
|
|
85
|
+
try {
|
|
86
|
+
fs.fsyncSync(fd);
|
|
87
|
+
}
|
|
88
|
+
finally {
|
|
89
|
+
fs.closeSync(fd);
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
function readStaged(directory, expected, range) {
|
|
93
|
+
if (!fs.existsSync(directory))
|
|
94
|
+
return null;
|
|
95
|
+
const runnerPath = path.join(directory, SYNC_RUNNER_BUNDLE_RUNNER_FILE);
|
|
96
|
+
const metadataPath = path.join(directory, SYNC_RUNNER_BUNDLE_METADATA_FILE);
|
|
97
|
+
let metadata;
|
|
98
|
+
try {
|
|
99
|
+
metadata = readMetadata(metadataPath);
|
|
100
|
+
if (metadata.formatVersion !== expected.formatVersion
|
|
101
|
+
|| metadata.version !== expected.version
|
|
102
|
+
|| metadata.bundleSha256 !== expected.bundleSha256
|
|
103
|
+
|| metadata.type !== "module"
|
|
104
|
+
|| !semver.satisfies(metadata.version, range)
|
|
105
|
+
|| sha256(runnerPath) !== metadata.bundleSha256)
|
|
106
|
+
fail("STAGE_EXISTS_INVALID", "An existing sync-runner stage is invalid; preserving it");
|
|
107
|
+
}
|
|
108
|
+
catch (error) {
|
|
109
|
+
if (error instanceof SyncRunnerBundleStageError && error.code === "STAGE_EXISTS_INVALID")
|
|
110
|
+
throw error;
|
|
111
|
+
fail("STAGE_EXISTS_INVALID", "An existing sync-runner stage is invalid; preserving it", error);
|
|
112
|
+
}
|
|
113
|
+
return { directory, runnerPath, metadataPath, metadata };
|
|
114
|
+
}
|
|
115
|
+
function stagedLeaf(metadata) {
|
|
116
|
+
const version = metadata.version.replace(/[^A-Za-z0-9.-]/g, "_");
|
|
117
|
+
return `${version}-${metadata.bundleSha256}`;
|
|
118
|
+
}
|
|
119
|
+
/**
|
|
120
|
+
* Stage one validated runner bundle into an immutable daemon-owned directory.
|
|
121
|
+
* Only a completed, fsynced directory is returned; temp siblings are ignored.
|
|
122
|
+
*/
|
|
123
|
+
export function stageSyncRunnerBundle(options) {
|
|
124
|
+
const sourceRunner = path.join(options.payloadDirectory, SYNC_RUNNER_BUNDLE_RUNNER_FILE);
|
|
125
|
+
const sourceMetadata = path.join(options.payloadDirectory, SYNC_RUNNER_BUNDLE_METADATA_FILE);
|
|
126
|
+
const metadata = readMetadata(sourceMetadata);
|
|
127
|
+
const range = hqCloudRange();
|
|
128
|
+
if (!semver.satisfies(metadata.version, range)) {
|
|
129
|
+
fail("VERSION_OUT_OF_RANGE", "The sync-runner bundle version is outside the declared hq-cloud range");
|
|
130
|
+
}
|
|
131
|
+
try {
|
|
132
|
+
if (!fs.statSync(sourceRunner).isFile())
|
|
133
|
+
fail("PAYLOAD_MISSING", "The sync-runner bundle file is missing");
|
|
134
|
+
}
|
|
135
|
+
catch (error) {
|
|
136
|
+
fail("PAYLOAD_MISSING", "Unable to access the sync-runner bundle file", error);
|
|
137
|
+
}
|
|
138
|
+
if (sha256(sourceRunner) !== metadata.bundleSha256) {
|
|
139
|
+
fail("HASH_MISMATCH", "The sync-runner bundle sha256 does not match its sidecar");
|
|
140
|
+
}
|
|
141
|
+
const stageRoot = path.join(options.daemonRoot, "bundles", "sync-runner");
|
|
142
|
+
const selectedDirectory = path.join(stageRoot, stagedLeaf(metadata));
|
|
143
|
+
const existing = readStaged(selectedDirectory, metadata, range);
|
|
144
|
+
if (existing)
|
|
145
|
+
return existing;
|
|
146
|
+
let tempDirectory;
|
|
147
|
+
let failureCode = "COPY_FAILED";
|
|
148
|
+
try {
|
|
149
|
+
ensurePrivateDir(stageRoot);
|
|
150
|
+
tempDirectory = path.join(stageRoot, `.tmp-${process.pid}-${Math.random().toString(16).slice(2)}`);
|
|
151
|
+
fs.mkdirSync(tempDirectory, { mode: 0o700 });
|
|
152
|
+
fs.mkdirSync(path.join(tempDirectory, "bin"), { mode: 0o700 });
|
|
153
|
+
(options.copyFile ?? fs.copyFileSync)(sourceRunner, path.join(tempDirectory, SYNC_RUNNER_BUNDLE_RUNNER_FILE));
|
|
154
|
+
(options.copyFile ?? fs.copyFileSync)(sourceMetadata, path.join(tempDirectory, SYNC_RUNNER_BUNDLE_METADATA_FILE));
|
|
155
|
+
if (sha256(path.join(tempDirectory, SYNC_RUNNER_BUNDLE_RUNNER_FILE)) !== metadata.bundleSha256) {
|
|
156
|
+
fail("HASH_MISMATCH", "The staged sync-runner bundle sha256 does not match its sidecar");
|
|
157
|
+
}
|
|
158
|
+
failureCode = "FSYNC_FAILED";
|
|
159
|
+
syncFile(path.join(tempDirectory, SYNC_RUNNER_BUNDLE_RUNNER_FILE));
|
|
160
|
+
syncFile(path.join(tempDirectory, SYNC_RUNNER_BUNDLE_METADATA_FILE));
|
|
161
|
+
syncDirectory(path.join(tempDirectory, "bin"));
|
|
162
|
+
syncDirectory(tempDirectory);
|
|
163
|
+
failureCode = "RENAME_FAILED";
|
|
164
|
+
try {
|
|
165
|
+
(options.renameDirectory ?? fs.renameSync)(tempDirectory, selectedDirectory);
|
|
166
|
+
}
|
|
167
|
+
catch (error) {
|
|
168
|
+
if (error.code === "EEXIST" || error.code === "ENOTEMPTY") {
|
|
169
|
+
const raced = readStaged(selectedDirectory, metadata, range);
|
|
170
|
+
if (raced) {
|
|
171
|
+
fs.rmSync(tempDirectory, { recursive: true, force: true });
|
|
172
|
+
return raced;
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
fail("RENAME_FAILED", "Unable to atomically select the staged sync-runner bundle", error);
|
|
176
|
+
}
|
|
177
|
+
tempDirectory = undefined;
|
|
178
|
+
failureCode = "FSYNC_FAILED";
|
|
179
|
+
syncDirectory(stageRoot);
|
|
180
|
+
}
|
|
181
|
+
catch (error) {
|
|
182
|
+
if (tempDirectory) {
|
|
183
|
+
try {
|
|
184
|
+
fs.rmSync(tempDirectory, { recursive: true, force: true });
|
|
185
|
+
}
|
|
186
|
+
catch (cleanupError) {
|
|
187
|
+
fail("COPY_FAILED", "Unable to remove the temporary sync-runner stage", new AggregateError([error, cleanupError]));
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
if (error instanceof SyncRunnerBundleStageError)
|
|
191
|
+
throw error;
|
|
192
|
+
fail(failureCode, "Unable to stage the sync-runner bundle", error);
|
|
193
|
+
}
|
|
194
|
+
const staged = readStaged(selectedDirectory, metadata, range);
|
|
195
|
+
if (!staged)
|
|
196
|
+
fail("FSYNC_FAILED", "The completed sync-runner stage failed validation");
|
|
197
|
+
return staged;
|
|
198
|
+
}
|
|
199
|
+
//# sourceMappingURL=sync-runner-bundle.js.map
|
|
@@ -1,14 +1,13 @@
|
|
|
1
1
|
import * as fs from "node:fs";
|
|
2
|
-
import {
|
|
3
|
-
import type { LaneRecord, LanesConfig } from "./types.js";
|
|
4
|
-
export declare const LOW_DISK_GUARD_FLAG = "lanes.low-disk-guard";
|
|
5
|
-
export declare function lowDiskGuardEnabledForLane(hqRoot: string, lane: LaneRecord, readFlag?: (key: string, options: ReadLanesFlagOptions) => Promise<boolean>): Promise<boolean>;
|
|
2
|
+
import type { LaneProvider, LanesConfig } from "./types.js";
|
|
6
3
|
interface DiskStats {
|
|
7
4
|
bsize: number;
|
|
8
5
|
blocks: number;
|
|
9
6
|
bavail: number;
|
|
10
7
|
}
|
|
11
8
|
export type StatFs = (target: string) => DiskStats;
|
|
9
|
+
/** Resolve the enabled provider home a lane can use for its disk preflight. */
|
|
10
|
+
export declare function laneDiskAccount(config: LanesConfig, provider: LaneProvider, preferredAccountId?: string): import("./types.js").AccountConfig | undefined;
|
|
12
11
|
export interface DiskSpaceMeasurement {
|
|
13
12
|
path: string;
|
|
14
13
|
free_bytes: number;
|
|
@@ -3,17 +3,18 @@ import * as path from "node:path";
|
|
|
3
3
|
import { createGzip } from "node:zlib";
|
|
4
4
|
import { createReadStream, createWriteStream } from "node:fs";
|
|
5
5
|
import { pipeline } from "node:stream/promises";
|
|
6
|
-
import { readLanesRegistryFlag } from "./registry-flag.js";
|
|
7
|
-
import { resolveLaneCompany } from "./lane-company.js";
|
|
8
6
|
import { effectiveAccountHome } from "./account-home.js";
|
|
9
7
|
import { lanesConfigPath } from "./config.js";
|
|
10
|
-
export const LOW_DISK_GUARD_FLAG = "lanes.low-disk-guard";
|
|
11
8
|
const DEFAULT_LOW_DISK_FREE_BYTES = 5 * 1024 ** 3;
|
|
12
9
|
const DEFAULT_LOW_DISK_FRACTION = 0.02;
|
|
13
10
|
const EXCLUDED_SESSION_FILES = new Set(["auth.json", ".credentials.json", "thread_history.sqlite"]);
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
11
|
+
/** Resolve the enabled provider home a lane can use for its disk preflight. */
|
|
12
|
+
export function laneDiskAccount(config, provider, preferredAccountId) {
|
|
13
|
+
const accounts = config.providers[provider].accounts;
|
|
14
|
+
const preferred = preferredAccountId ? accounts[preferredAccountId] : undefined;
|
|
15
|
+
if (preferred?.enabled)
|
|
16
|
+
return preferred;
|
|
17
|
+
return Object.values(accounts).find((account) => account.enabled);
|
|
17
18
|
}
|
|
18
19
|
export function measureDiskSpace(target, statfs = (p) => fs.statfsSync(p), minFreeBytes = DEFAULT_LOW_DISK_FREE_BYTES) {
|
|
19
20
|
const stats = statfs(target);
|
|
@@ -23,7 +23,7 @@ import { appendCappedHistory, lanesStoreRoot, parseLaneRecord, parseQuestionReco
|
|
|
23
23
|
import { applyThrottleDecision, decideAdmission, defaultThrottleDeps, emptyThrottleState, evaluateThrottle, isCpuCheckEnabled, readAvailableMemMb, readThrottleState, resolveCpuBusyFraction, sampleCpuBusyObservation, recordCpuBusyObservation, throttleStatePath, writeThrottleState, } from "./throttle.js";
|
|
24
24
|
import { readLanesRegistryFlag, readLanesRegistryFlagFailOff } from "./registry-flag.js";
|
|
25
25
|
import { LANE_HIRE_GRANT_FINISHED_LANES_FLAG, LANE_LEAD_ADMISSION_PRIORITY_FLAG } from "./flag-keys.js";
|
|
26
|
-
import { checkLaneDiskSpace, configuredCodexRolloutHomes,
|
|
26
|
+
import { checkLaneDiskSpace, configuredCodexRolloutHomes, laneDiskAccount, retainCodexRollouts } from "./disk-guard.js";
|
|
27
27
|
import { laneCompanyUidSource, resolveLaneCompany } from "./lane-company.js";
|
|
28
28
|
import { appendDropboxMessage } from "./dropbox.js";
|
|
29
29
|
import { DEFAULT_RUNAWAY_SEARCH_GUARD_CONFIG, listHostProcesses, runRunawayToolGuard, RUNAWAY_TOOL_GUARD_FLAG, } from "./runaway-tool-guard.js";
|
|
@@ -1452,31 +1452,27 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
|
|
|
1452
1452
|
repairCandidate &&
|
|
1453
1453
|
repairCandidate.envelope === null &&
|
|
1454
1454
|
(!repairCandidate.missing || envelopeRepairMode(classified.provider) === "resume")) {
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
|
|
1463
|
-
const
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
}));
|
|
1477
|
-
summary.skipped.push({ kind: "lane", id: classified.lane_id, why: `low_disk: hq=${free.hq.free_bytes} provider=${free.provider.free_bytes}` });
|
|
1478
|
-
continue;
|
|
1479
|
-
}
|
|
1455
|
+
const repairConfig = loadLanesConfig(hqRoot);
|
|
1456
|
+
const repairAccount = laneDiskAccount(repairConfig, classified.provider, classified.account);
|
|
1457
|
+
if (!repairAccount) {
|
|
1458
|
+
throw new LanesError("low_disk_account_unavailable", `Cannot determine provider home for envelope repair of lane ${classified.lane_id}`, { lane_id: classified.lane_id });
|
|
1459
|
+
}
|
|
1460
|
+
const providerHome = effectiveAccountHome(repairAccount, deps.host.hostname).home;
|
|
1461
|
+
const free = checkLaneDiskSpace(hqRoot, providerHome, deps.spawnLaneDeps?.statfs, repairConfig.low_disk_min_free_bytes);
|
|
1462
|
+
if (free.low) {
|
|
1463
|
+
const measuredAt = now.toISOString();
|
|
1464
|
+
updateLane(hqRoot, classified.lane_id, classified.version, "reconcile", (current) => ({
|
|
1465
|
+
...current,
|
|
1466
|
+
state: "queued",
|
|
1467
|
+
queued_reason: "low_disk",
|
|
1468
|
+
queued_details: { reason: "low_disk", hq_free_bytes: free.hq.free_bytes, provider_free_bytes: free.provider.free_bytes, measured_at: measuredAt },
|
|
1469
|
+
queued_at: current.queued_at ?? measuredAt,
|
|
1470
|
+
admission_reserved_at: undefined,
|
|
1471
|
+
terminal_reason: undefined,
|
|
1472
|
+
terminal_details: undefined,
|
|
1473
|
+
}));
|
|
1474
|
+
summary.skipped.push({ kind: "lane", id: classified.lane_id, why: `low_disk: hq=${free.hq.free_bytes} provider=${free.provider.free_bytes}` });
|
|
1475
|
+
continue;
|
|
1480
1476
|
}
|
|
1481
1477
|
const spawnRepair = deps.spawnRepair ??
|
|
1482
1478
|
((root, repairLane, input) => spawnEnvelopeRepair(root, repairLane, input, spawnDepsForHost(deps)));
|
|
@@ -1848,7 +1844,6 @@ async function admitQueued(hqRoot, lanes, summary, deps, now, onlyLaneIds, exclu
|
|
|
1848
1844
|
const spawnFn = deps.spawn ??
|
|
1849
1845
|
(async (root, lane) => spawnLaneWorker(root, lane, {
|
|
1850
1846
|
...spawnDepsForHost(deps),
|
|
1851
|
-
lowDiskGuardEnabled: await lowDiskGuardEnabledForLane(root, lane, deps.readRegistryFlag),
|
|
1852
1847
|
}));
|
|
1853
1848
|
const running = lanes.filter((lane) => (lane.state === "running" ||
|
|
1854
1849
|
hasFreshAdmissionReservation(lane, now.getTime()) ||
|
|
@@ -3292,7 +3287,6 @@ async function resumeAwaiting(hqRoot, lanes, questions, summary, deps, onlyLaneI
|
|
|
3292
3287
|
}
|
|
3293
3288
|
const spawnFn = deps.spawn ?? (async (root, lane) => spawnLaneWorker(root, lane, {
|
|
3294
3289
|
...spawnDepsForHost(deps),
|
|
3295
|
-
lowDiskGuardEnabled: await lowDiskGuardEnabledForLane(root, lane, deps.readRegistryFlag),
|
|
3296
3290
|
}));
|
|
3297
3291
|
const nowMs = deps.now().getTime();
|
|
3298
3292
|
const fallbackAnswerAt = new Date(nowMs).toISOString();
|
|
@@ -75,8 +75,6 @@ export interface SpawnLaneDeps {
|
|
|
75
75
|
hostname?: string;
|
|
76
76
|
/** The caller's host scope for admission checks; reconcile supplies its resolved host. */
|
|
77
77
|
admissionHost?: AdmissionHost;
|
|
78
|
-
/** Resolved per-company by the async command/reconcile boundary. */
|
|
79
|
-
lowDiskGuardEnabled?: boolean;
|
|
80
78
|
/** Injected statfs boundary; production uses node:fs.statfsSync. */
|
|
81
79
|
statfs?: StatFs;
|
|
82
80
|
/** Provider model catalog dependencies, shared with create in tests. */
|