@indigoai-us/hq-cli 5.345.175 → 5.345.176
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +22 -0
- package/dist/command-catalog.generated.d.ts +65 -0
- package/dist/command-catalog.generated.js +80 -0
- package/dist/commands/bot.d.ts +28 -2
- package/dist/commands/bot.js +29 -6
- package/dist/commands/daemon.js +16 -0
- package/dist/commands/lanes-loop.d.ts +4 -1
- package/dist/commands/lanes-loop.js +23 -4
- package/dist/commands/lanes-shared.d.ts +5 -0
- package/dist/commands/lanes.d.ts +5 -7
- package/dist/commands/lanes.js +65 -5
- package/dist/commands/sync-core.d.ts +17 -0
- package/dist/commands/sync-core.js +46 -5
- package/dist/commands/sync-manifest.d.ts +6 -0
- package/dist/commands/sync-manifest.js +13 -0
- package/dist/hq-cloud-sync-runner/bin/sync-runner.js +448 -226
- package/dist/hq-cloud-sync-runner/package.json +1 -1
- package/dist/lib/bot/cli-update.d.ts +36 -0
- package/dist/lib/bot/cli-update.js +160 -42
- package/dist/lib/bot/run.d.ts +21 -0
- package/dist/lib/bot/run.js +127 -10
- package/dist/lib/bot/runtime/claude.d.ts +7 -1
- package/dist/lib/bot/runtime/claude.js +29 -0
- package/dist/lib/bot/runtime/codex.d.ts +5 -0
- package/dist/lib/bot/runtime/codex.js +13 -0
- package/dist/lib/bot/runtime/types.d.ts +10 -0
- package/dist/lib/bot/runtime-sign-in-check.d.ts +32 -0
- package/dist/lib/bot/runtime-sign-in-check.js +53 -0
- package/dist/lib/bot/runtime-sign-in.d.ts +32 -3
- package/dist/lib/bot/runtime-sign-in.js +17 -2
- package/dist/lib/bot/status.d.ts +25 -1
- package/dist/lib/bot/status.js +28 -0
- package/dist/lib/daemon/catalog.d.ts +4 -0
- package/dist/lib/daemon/catalog.js +47 -19
- package/dist/lib/daemon/run.js +1 -0
- package/dist/lib/flag-registry-worker-source.generated.d.ts +1 -1
- package/dist/lib/flag-registry-worker-source.generated.js +1 -1
- package/dist/lib/lanes/loop-disposition.d.ts +44 -0
- package/dist/lib/lanes/loop-disposition.js +302 -0
- package/dist/lib/lanes/loop.d.ts +16 -3
- package/dist/lib/lanes/loop.js +167 -60
- package/dist/lib/lanes/reconcile.d.ts +5 -0
- package/dist/lib/lanes/reconcile.js +79 -4
- package/dist/lib/mesh/live/daemon/work-subscribe.d.ts +3 -0
- package/dist/lib/mesh/live/daemon/work-subscribe.js +41 -12
- package/dist/lib/sync-core/engine-home.d.ts +11 -4
- package/dist/lib/sync-core/engine-home.js +22 -21
- package/dist/lib/sync-core/manifest-scopes.d.ts +15 -0
- package/dist/lib/sync-core/manifest-scopes.js +26 -0
- package/dist/lib/sync-core/reconcile.d.ts +2 -2
- package/dist/lib/sync-core/reconcile.js +70 -46
- package/dist/lib/sync-core/release.d.ts +2 -2
- package/dist/lib/sync-core/release.js +31 -6
- package/dist/lib/sync-core/rollout-state.d.ts +6 -3
- package/dist/lib/sync-core/rollout-state.js +26 -22
- package/package.json +8 -8
package/dist/lib/lanes/loop.js
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
import { createHash, randomUUID } from "node:crypto";
|
|
3
3
|
import * as fs from "node:fs";
|
|
4
4
|
import * as path from "node:path";
|
|
5
|
+
import { isPidRecordAlive, readProcStarttime } from "./spawn.js";
|
|
5
6
|
import { providerSupportsSessionResume } from "./delivery.js";
|
|
6
7
|
export const LOOP_SCHEMA = "hq-lane-loop.v1";
|
|
7
8
|
export const PHASE_ENVELOPE_SCHEMA = "hq-phase-envelope/v1";
|
|
@@ -362,7 +363,9 @@ function nextQueueSequence(runDir) {
|
|
|
362
363
|
atomicJson(sequenceFile, { sequence });
|
|
363
364
|
return sequence;
|
|
364
365
|
}
|
|
365
|
-
function acquireRunnerLease(runDir) {
|
|
366
|
+
function acquireRunnerLease(runDir, identity) {
|
|
367
|
+
const readStarttime = identity?.readStarttime ?? readProcStarttime;
|
|
368
|
+
const starttime = readStarttime(process.pid) ?? undefined;
|
|
366
369
|
const lock = path.join(runDir, "loop", "runner.lock");
|
|
367
370
|
fs.mkdirSync(path.dirname(lock), { recursive: true });
|
|
368
371
|
try {
|
|
@@ -372,27 +375,34 @@ function acquireRunnerLease(runDir) {
|
|
|
372
375
|
if (error.code !== "EEXIST")
|
|
373
376
|
throw error;
|
|
374
377
|
let pid;
|
|
378
|
+
let recordedStarttime;
|
|
375
379
|
try {
|
|
376
380
|
const owner = readJson(path.join(lock, "owner.json"));
|
|
377
|
-
if (isRecord(owner) && typeof owner.pid === "number")
|
|
381
|
+
if (isRecord(owner) && typeof owner.pid === "number") {
|
|
378
382
|
pid = owner.pid;
|
|
383
|
+
recordedStarttime = typeof owner.starttime === "number" ? owner.starttime : undefined;
|
|
384
|
+
}
|
|
379
385
|
}
|
|
380
386
|
catch (readError) {
|
|
381
387
|
if (readError.code !== "ENOENT" && !(readError instanceof SyntaxError))
|
|
382
388
|
throw readError;
|
|
383
389
|
}
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
390
|
+
const alive = pid !== undefined && Number.isSafeInteger(pid) && pid > 0 &&
|
|
391
|
+
isPidRecordAlive({ pid, starttime: recordedStarttime }, identity?.isAlive ?? ((candidate) => {
|
|
392
|
+
try {
|
|
393
|
+
process.kill(candidate, 0);
|
|
394
|
+
return true;
|
|
395
|
+
}
|
|
396
|
+
catch (error) {
|
|
397
|
+
if (error.code === "EPERM")
|
|
398
|
+
return true;
|
|
399
|
+
if (error.code === "ESRCH")
|
|
400
|
+
return false;
|
|
401
|
+
throw error;
|
|
402
|
+
}
|
|
403
|
+
}), readStarttime);
|
|
404
|
+
if (pid === undefined || !Number.isSafeInteger(pid) || pid <= 0)
|
|
405
|
+
throw new Error("Loop runner lease ownership is unknown; explicit recovery required");
|
|
396
406
|
const age = Date.now() - fs.statSync(lock).mtimeMs;
|
|
397
407
|
if (alive || age <= 60_000)
|
|
398
408
|
throw new Error(`Loop runner is already active${pid ? ` (pid ${pid})` : ""}`);
|
|
@@ -423,9 +433,12 @@ function acquireRunnerLease(runDir) {
|
|
|
423
433
|
if (released)
|
|
424
434
|
return;
|
|
425
435
|
released = true;
|
|
436
|
+
process.removeListener("exit", release);
|
|
426
437
|
clearInterval(heartbeat);
|
|
427
438
|
try {
|
|
428
|
-
|
|
439
|
+
const owner = readJson(ownerFile);
|
|
440
|
+
if (isRecord(owner) && owner.pid === process.pid && owner.starttime === starttime)
|
|
441
|
+
fs.rmSync(lock, { recursive: true, force: true });
|
|
429
442
|
}
|
|
430
443
|
catch (error) {
|
|
431
444
|
if (error.code !== "ENOENT")
|
|
@@ -434,27 +447,35 @@ function acquireRunnerLease(runDir) {
|
|
|
434
447
|
};
|
|
435
448
|
const heartbeat = setInterval(() => {
|
|
436
449
|
try {
|
|
437
|
-
atomicJson(ownerFile, { pid: process.pid, heartbeat_at: new Date().toISOString() });
|
|
450
|
+
atomicJson(ownerFile, { pid: process.pid, starttime, heartbeat_at: new Date().toISOString() });
|
|
438
451
|
}
|
|
439
452
|
catch (error) {
|
|
440
453
|
process.stderr.write(`hq lanes: loop lease heartbeat failed: ${error instanceof Error ? error.message : String(error)}\n`);
|
|
441
454
|
}
|
|
442
455
|
}, 15_000);
|
|
443
|
-
atomicJson(ownerFile, { pid: process.pid, heartbeat_at: new Date().toISOString() });
|
|
456
|
+
atomicJson(ownerFile, { pid: process.pid, starttime, heartbeat_at: new Date().toISOString() });
|
|
444
457
|
process.once("exit", release);
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
458
|
+
// A dead controller may have performed external effects before it died.
|
|
459
|
+
// Replaying an active envelope cannot be justified by reclaiming its lease.
|
|
460
|
+
if (inspectLoopRecovery(runDir).kind === "hold") {
|
|
461
|
+
release();
|
|
462
|
+
throw new Error("Loop recovery requires an explicit disposition of interrupted work");
|
|
463
|
+
}
|
|
464
|
+
try {
|
|
465
|
+
// Only this lease owner may finalize the proven local completion window.
|
|
466
|
+
// Never execute it again; a failure leaves the original active file intact.
|
|
467
|
+
for (const file of completedActiveRecovery(runDir).files) {
|
|
468
|
+
const destination = path.join(claimedDir(runDir), path.basename(file));
|
|
469
|
+
if (fs.existsSync(destination))
|
|
470
|
+
throw new Error("Completed loop archive already exists; explicit recovery required");
|
|
471
|
+
fs.mkdirSync(claimedDir(runDir), { recursive: true });
|
|
472
|
+
fs.renameSync(file, destination);
|
|
456
473
|
}
|
|
457
474
|
}
|
|
475
|
+
catch (error) {
|
|
476
|
+
release();
|
|
477
|
+
throw error;
|
|
478
|
+
}
|
|
458
479
|
return release;
|
|
459
480
|
}
|
|
460
481
|
function envelopeFiles(dir) {
|
|
@@ -806,8 +827,8 @@ export function commitIdleLoopStop(runDir, laneId, now = () => new Date()) {
|
|
|
806
827
|
persistStatus(runDir, laneId, { state: "stopped", queue_depth: listLoopQueue(runDir).length, active_envelope_id: null, active_phase: null });
|
|
807
828
|
return true;
|
|
808
829
|
}
|
|
809
|
-
export function setLoopControllerPid(runDir, laneId, pid) {
|
|
810
|
-
persistStatus(runDir, laneId, { pid });
|
|
830
|
+
export function setLoopControllerPid(runDir, laneId, pid, starttime) {
|
|
831
|
+
persistStatus(runDir, laneId, { pid, starttime });
|
|
811
832
|
}
|
|
812
833
|
export function clearLoopControllerPid(runDir, laneId, pid) {
|
|
813
834
|
let currentPid;
|
|
@@ -821,51 +842,129 @@ export function clearLoopControllerPid(runDir, laneId, pid) {
|
|
|
821
842
|
throw error;
|
|
822
843
|
}
|
|
823
844
|
if (currentPid === pid)
|
|
824
|
-
persistStatus(runDir, laneId, { pid: undefined });
|
|
845
|
+
persistStatus(runDir, laneId, { pid: undefined, starttime: undefined });
|
|
825
846
|
}
|
|
826
847
|
/**
|
|
827
848
|
* True when a loop lane's controller is alive with no phase active: the lane
|
|
828
849
|
* finished its last phase and is waiting for the next envelope.
|
|
829
850
|
*/
|
|
830
|
-
export function loopLaneIsIdle(runDir, laneId) {
|
|
851
|
+
export function loopLaneIsIdle(runDir, laneId, isAlive, readStarttime) {
|
|
831
852
|
const status = loopStatus(runDir, laneId);
|
|
832
|
-
return status.state === "waiting" && status.active_envelope_id === null && loopControllerIsAlive(runDir);
|
|
853
|
+
return status.state === "waiting" && status.active_envelope_id === null && loopControllerIsAlive(runDir, isAlive, readStarttime);
|
|
833
854
|
}
|
|
834
|
-
export function loopControllerIsAlive(runDir) {
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
pid
|
|
855
|
+
export function loopControllerIsAlive(runDir, isAlive, readStarttime = readProcStarttime) {
|
|
856
|
+
const records = [];
|
|
857
|
+
for (const file of [loopPath(runDir), path.join(runDir, "loop", "runner.lock", "owner.json")]) {
|
|
858
|
+
try {
|
|
859
|
+
const value = readJson(file);
|
|
860
|
+
if (isRecord(value) && typeof value.pid === "number" && Number.isSafeInteger(value.pid) && value.pid > 0) {
|
|
861
|
+
records.push({ pid: value.pid, starttime: typeof value.starttime === "number" ? value.starttime : undefined });
|
|
862
|
+
}
|
|
863
|
+
}
|
|
864
|
+
catch (error) {
|
|
865
|
+
if (error.code !== "ENOENT" && !(error instanceof SyntaxError))
|
|
866
|
+
throw error;
|
|
867
|
+
}
|
|
840
868
|
}
|
|
841
|
-
|
|
842
|
-
|
|
869
|
+
const probe = isAlive ?? ((candidate) => {
|
|
870
|
+
try {
|
|
871
|
+
process.kill(candidate, 0);
|
|
872
|
+
return true;
|
|
873
|
+
}
|
|
874
|
+
catch (error) {
|
|
875
|
+
if (error.code === "EPERM")
|
|
876
|
+
return true;
|
|
877
|
+
if (error.code === "ESRCH")
|
|
878
|
+
return false;
|
|
843
879
|
throw error;
|
|
844
|
-
|
|
845
|
-
|
|
880
|
+
}
|
|
881
|
+
});
|
|
882
|
+
// A failed second launch may overwrite loop.json before its lease fails.
|
|
883
|
+
// The live lease owner still protects the first controller.
|
|
884
|
+
return records.some(record => isPidRecordAlive(record, probe, readStarttime));
|
|
885
|
+
}
|
|
886
|
+
/** A local terminal result is enough only when no external handoff was promised. */
|
|
887
|
+
function completedActiveRecovery(runDir) {
|
|
888
|
+
const files = [];
|
|
889
|
+
for (const file of envelopeFiles(activeDir(runDir))) {
|
|
890
|
+
const record = fileEnvelope(file);
|
|
891
|
+
if (record.envelope.kind === "stop")
|
|
892
|
+
return { files: [], stopped: true };
|
|
893
|
+
let result;
|
|
846
894
|
try {
|
|
847
|
-
|
|
848
|
-
if (isRecord(owner))
|
|
849
|
-
pid = owner.pid;
|
|
895
|
+
result = readJson(path.join(resultsDir(runDir), resultName(record.envelope_id)));
|
|
850
896
|
}
|
|
851
897
|
catch (error) {
|
|
852
898
|
if (error.code !== "ENOENT" && !(error instanceof SyntaxError))
|
|
853
899
|
throw error;
|
|
854
900
|
}
|
|
901
|
+
if (!isRecord(result) || result.schema_version !== LOOP_SCHEMA || result.envelope_id !== record.envelope_id ||
|
|
902
|
+
!["done", "failed"].includes(String(result.status)) ||
|
|
903
|
+
typeof result.started_at !== "string" || !Number.isFinite(Date.parse(result.started_at)) ||
|
|
904
|
+
typeof result.finished_at !== "string" || !Number.isFinite(Date.parse(result.finished_at)) ||
|
|
905
|
+
Date.parse(result.finished_at) < Date.parse(result.started_at)) {
|
|
906
|
+
return { files: [], reason: "active_phase_outcome_unknown" };
|
|
907
|
+
}
|
|
908
|
+
if (typeof record.envelope.result_path === "string") {
|
|
909
|
+
// Local write happens before the optional external handoff. Do not read
|
|
910
|
+
// arbitrary paths or assume a local result proves that handoff finished.
|
|
911
|
+
return { files: [], reason: "external_completion_unverified" };
|
|
912
|
+
}
|
|
913
|
+
files.push(file);
|
|
855
914
|
}
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
915
|
+
return { files };
|
|
916
|
+
}
|
|
917
|
+
/** Read durable work without changing queue files or assuming effects are replayable. */
|
|
918
|
+
export function inspectLoopRecovery(runDir) {
|
|
919
|
+
const status = loopStatus(runDir, path.basename(runDir));
|
|
920
|
+
if (["parked", "stopped", "failed"].includes(status.state))
|
|
921
|
+
return { kind: "protected" };
|
|
922
|
+
const lease = path.join(runDir, "loop", "runner.lock");
|
|
923
|
+
if (fs.existsSync(lease)) {
|
|
924
|
+
let owner;
|
|
925
|
+
try {
|
|
926
|
+
owner = readJson(path.join(lease, "owner.json"));
|
|
927
|
+
}
|
|
928
|
+
catch (error) {
|
|
929
|
+
if (error.code !== "ENOENT" && !(error instanceof SyntaxError))
|
|
930
|
+
throw error;
|
|
931
|
+
}
|
|
932
|
+
if (!isRecord(owner) || !Number.isSafeInteger(owner.pid) || Number(owner.pid) <= 0) {
|
|
933
|
+
return { kind: "hold", reason: "runner_lease_identity_unknown" };
|
|
934
|
+
}
|
|
861
935
|
}
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
936
|
+
const completed = completedActiveRecovery(runDir);
|
|
937
|
+
if (completed.reason)
|
|
938
|
+
return { kind: "hold", reason: completed.reason };
|
|
939
|
+
if (completed.stopped)
|
|
940
|
+
return { kind: "protected" };
|
|
941
|
+
for (const file of envelopeFiles(claimedDir(runDir))) {
|
|
942
|
+
const record = fileEnvelope(file);
|
|
943
|
+
if (record.envelope.kind === "stop")
|
|
944
|
+
return { kind: "protected" };
|
|
945
|
+
let result;
|
|
946
|
+
try {
|
|
947
|
+
result = readJson(path.join(resultsDir(runDir), resultName(record.envelope_id)));
|
|
948
|
+
}
|
|
949
|
+
catch (error) {
|
|
950
|
+
if (error.code !== "ENOENT" && !(error instanceof SyntaxError))
|
|
951
|
+
throw error;
|
|
952
|
+
}
|
|
953
|
+
if (!isRecord(result) || result.schema_version !== LOOP_SCHEMA || result.envelope_id !== record.envelope_id ||
|
|
954
|
+
!["done", "failed", "stopped"].includes(String(result.status))) {
|
|
955
|
+
return { kind: "hold", reason: "claimed_phase_outcome_unknown" };
|
|
956
|
+
}
|
|
957
|
+
if (result.status === "stopped")
|
|
958
|
+
return { kind: "protected" };
|
|
868
959
|
}
|
|
960
|
+
const pending = listLoopQueue(runDir);
|
|
961
|
+
// A completed phase still needs lease-owned archiving before an idle stop.
|
|
962
|
+
if (completed.files.length > 0)
|
|
963
|
+
return { kind: "pending" };
|
|
964
|
+
// A queued intentional stop remains owned by the ordinary stop path.
|
|
965
|
+
if (pending[0]?.envelope?.kind === "stop")
|
|
966
|
+
return { kind: "protected" };
|
|
967
|
+
return { kind: pending.length > 0 ? "pending" : "empty" };
|
|
869
968
|
}
|
|
870
969
|
function phaseName(envelope, fallback) {
|
|
871
970
|
const value = envelope;
|
|
@@ -911,6 +1010,14 @@ function persistStatus(runDir, laneId, patch) {
|
|
|
911
1010
|
updated_at: new Date().toISOString(),
|
|
912
1011
|
});
|
|
913
1012
|
}
|
|
1013
|
+
/** Reset only after an explicit, durable disposition; this never starts work. */
|
|
1014
|
+
export function resetLoopAfterDisposition(runDir, laneId, decision, hasPending) {
|
|
1015
|
+
persistStatus(runDir, laneId, {
|
|
1016
|
+
state: hasPending ? "waiting" : "failed", active_envelope_id: null, active_phase: null,
|
|
1017
|
+
pid: undefined, starttime: undefined, queue_depth: listLoopQueue(runDir).length,
|
|
1018
|
+
recovery_decision: decision,
|
|
1019
|
+
});
|
|
1020
|
+
}
|
|
914
1021
|
function abortAfter(ms, parent) {
|
|
915
1022
|
const controller = new AbortController();
|
|
916
1023
|
const timer = setTimeout(() => controller.abort(new Error("phase deadline exceeded")), Math.min(Math.max(ms, 1), 2_147_483_647));
|
|
@@ -947,12 +1054,12 @@ export async function runLoopLane(options) {
|
|
|
947
1054
|
const stalls = new Map();
|
|
948
1055
|
let completed = 0;
|
|
949
1056
|
fs.mkdirSync(options.runDir, { recursive: true });
|
|
950
|
-
let releaseRunner = acquireRunnerLease(options.runDir);
|
|
1057
|
+
let releaseRunner = acquireRunnerLease(options.runDir, options.processIdentity);
|
|
951
1058
|
if (readHandoffHold(options.runDir)) {
|
|
952
1059
|
releaseRunner();
|
|
953
1060
|
return { state: loopStatus(options.runDir, options.lane.lane_id).state === "stopped" ? "stopped" : "parked", completed: 0 };
|
|
954
1061
|
}
|
|
955
|
-
persistStatus(options.runDir, options.lane.lane_id, { pid: process.pid });
|
|
1062
|
+
persistStatus(options.runDir, options.lane.lane_id, { pid: process.pid, starttime: (options.processIdentity?.readStarttime ?? readProcStarttime)(process.pid) ?? undefined });
|
|
956
1063
|
try {
|
|
957
1064
|
const previous = readJson(loopPath(options.runDir));
|
|
958
1065
|
if (isRecord(previous) && isRecord(previous.stall_counts)) {
|
|
@@ -1206,7 +1313,7 @@ export async function runLoopLane(options) {
|
|
|
1206
1313
|
const answeredRetryIsPending = envelopeFiles(pendingDir(options.runDir)).some((pendingFile) => path.basename(pendingFile).startsWith("0000000000000-answer-") && fileEnvelope(pendingFile).envelope_id === record.envelope_id);
|
|
1207
1314
|
if (answeredRetryIsPending) {
|
|
1208
1315
|
try {
|
|
1209
|
-
releaseRunner = acquireRunnerLease(options.runDir);
|
|
1316
|
+
releaseRunner = acquireRunnerLease(options.runDir, options.processIdentity);
|
|
1210
1317
|
persistStatus(options.runDir, options.lane.lane_id, { pid: process.pid, state: "waiting", queue_depth: listLoopQueue(options.runDir).length, active_envelope_id: null, active_phase: null });
|
|
1211
1318
|
continue;
|
|
1212
1319
|
}
|
|
@@ -102,6 +102,11 @@ export interface ReconcileDeps {
|
|
|
102
102
|
throttleDeps?: ThrottleDeps;
|
|
103
103
|
livenessDeps?: LivenessDeps;
|
|
104
104
|
isAlive?: (pid: number) => boolean;
|
|
105
|
+
/** Controller identity is independent of worker/watcher liveness. */
|
|
106
|
+
controllerProcessIdentity?: {
|
|
107
|
+
isAlive(pid: number): boolean;
|
|
108
|
+
readStarttime(pid: number): number | null;
|
|
109
|
+
};
|
|
105
110
|
readSessionSenior?: (sessionId: string) => SeniorRef;
|
|
106
111
|
presenceConnected?: boolean;
|
|
107
112
|
env?: NodeJS.ProcessEnv;
|
|
@@ -13,7 +13,7 @@ import { compareLaneQueueOrder, compareLaneQueueOrderWithLeads, leadLaneIds, has
|
|
|
13
13
|
import { accountStatus, isCredentialProven, resolveAccount } from "./accounts.js";
|
|
14
14
|
import { effectiveAccountHome } from "./account-home.js";
|
|
15
15
|
import { enforceRegistryApprovalGate } from "./approval-gate.js";
|
|
16
|
-
import { loopLaneIsIdle, loopStatus } from "./loop.js";
|
|
16
|
+
import { inspectLoopRecovery, loopControllerIsAlive, loopLaneIsIdle, loopStatus } from "./loop.js";
|
|
17
17
|
import { loadLanesConfig, providerQuotaDefaultHoldMs } from "./config.js";
|
|
18
18
|
import { MAX_LANE_DEPTH } from "./constants.js";
|
|
19
19
|
import { laneDepthCap } from "./depth-cap.js";
|
|
@@ -1499,7 +1499,8 @@ export async function runPassBody(hqRoot, deps) {
|
|
|
1499
1499
|
}));
|
|
1500
1500
|
const admissionThrottleDeps = startDeps.throttleDeps ?? defaultThrottleDeps(startDeps.env ?? process.env);
|
|
1501
1501
|
const hasAdmissionCandidates = lanes.some((lane) => laneRunsHere(lane, startDeps.host) &&
|
|
1502
|
-
(lane.state === "queued" || lane.state === "awaiting_input")) ||
|
|
1502
|
+
(lane.state === "queued" || lane.state === "awaiting_input")) || summary.dead_host.some((entry) => entry.reason === "loop_controller_lost" && entry.to === "queued") ||
|
|
1503
|
+
[...keepAliveFlagByCompany.values()].some(Boolean);
|
|
1503
1504
|
const priorThrottleState = readThrottleState(throttleStatePath(hqRoot), admissionThrottleDeps);
|
|
1504
1505
|
const priorThrottleSnapshot = priorThrottleState === "missing" || priorThrottleState === "unreadable"
|
|
1505
1506
|
? undefined
|
|
@@ -1597,6 +1598,8 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
|
|
|
1597
1598
|
const nowIso = now.toISOString();
|
|
1598
1599
|
const isAlive = deps.isAlive ?? isPidAlive;
|
|
1599
1600
|
for (const lane of lanes) {
|
|
1601
|
+
// Historical done lanes require explicit recovery; leftover files alone
|
|
1602
|
+
// do not prove that their work should run again.
|
|
1600
1603
|
if (lane.state !== "running")
|
|
1601
1604
|
continue;
|
|
1602
1605
|
// No HQ process backs a subagent lane, so a missing pid is not a dead
|
|
@@ -1658,7 +1661,7 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
|
|
|
1658
1661
|
// Watcher owns terminal classification of a process that has just exited.
|
|
1659
1662
|
continue;
|
|
1660
1663
|
}
|
|
1661
|
-
if (lane.loop_mode === true && loopLaneIsIdle(runDir, lane.lane_id)) {
|
|
1664
|
+
if (lane.loop_mode === true && loopLaneIsIdle(runDir, lane.lane_id, deps.controllerProcessIdentity?.isAlive, deps.controllerProcessIdentity?.readStarttime)) {
|
|
1662
1665
|
// An idle loop lane has no worker between phases. Its controller is
|
|
1663
1666
|
// alive and waiting for the next envelope, so the lane is not finished.
|
|
1664
1667
|
try {
|
|
@@ -1675,6 +1678,44 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
|
|
|
1675
1678
|
}
|
|
1676
1679
|
continue;
|
|
1677
1680
|
}
|
|
1681
|
+
if (lane.loop_mode === true) {
|
|
1682
|
+
try {
|
|
1683
|
+
// A controller can be alive while a phase starts; never use a stale
|
|
1684
|
+
// phase envelope to terminalize or admit behind that controller.
|
|
1685
|
+
if (loopControllerIsAlive(runDir, deps.controllerProcessIdentity?.isAlive, deps.controllerProcessIdentity?.readStarttime))
|
|
1686
|
+
continue;
|
|
1687
|
+
const recovery = inspectLoopRecovery(runDir);
|
|
1688
|
+
if (recovery.kind === "protected")
|
|
1689
|
+
continue;
|
|
1690
|
+
if (recovery.kind === "hold" || recovery.kind === "pending") {
|
|
1691
|
+
const hasQuestion = loadQuestionRecords(hqRoot, summary).some(question => question.lane_id === lane.lane_id && question.status === "pending");
|
|
1692
|
+
if (hasQuestion)
|
|
1693
|
+
continue;
|
|
1694
|
+
const hold = recovery.kind === "hold" || deps.spawn === undefined;
|
|
1695
|
+
const reason = hold ? "loop_recovery_required" : "loop_controller_lost";
|
|
1696
|
+
const result = casLaneIf(hqRoot, lane.lane_id, current => current.state === "running" && current.loop_mode === true &&
|
|
1697
|
+
laneRunsHere(current, deps.host) && !current.terminal_reason?.startsWith("operator_") &&
|
|
1698
|
+
!["loop_stopped", "story_closed"].includes(current.terminal_reason ?? ""), current => ({ ...current, state: hold ? "interrupted" : "queued",
|
|
1699
|
+
queued_reason: undefined, queued_details: undefined,
|
|
1700
|
+
admission_reserved_at: undefined,
|
|
1701
|
+
terminal_reason: hold ? reason : undefined,
|
|
1702
|
+
terminal_details: hold ? { reason: recovery.reason ?? "controller_admission_unavailable", automatic_replay: false } : { loop_recovery: true },
|
|
1703
|
+
interrupted_at: hold ? nowIso : undefined,
|
|
1704
|
+
last_host_report_at: nowIso,
|
|
1705
|
+
}), deps.dryRun);
|
|
1706
|
+
if (result.changed)
|
|
1707
|
+
summary.dead_host.push({ id: lane.lane_id, from: lane.state, to: result.record.state, reason });
|
|
1708
|
+
continue;
|
|
1709
|
+
}
|
|
1710
|
+
}
|
|
1711
|
+
catch (err) {
|
|
1712
|
+
markDirty(summary, { kind: "lane", id: lane.lane_id,
|
|
1713
|
+
why: `loop recovery inspection failed: ${err instanceof Error ? err.message : String(err)}` });
|
|
1714
|
+
continue;
|
|
1715
|
+
}
|
|
1716
|
+
}
|
|
1717
|
+
if (lane.state !== "running")
|
|
1718
|
+
continue;
|
|
1678
1719
|
const hadPids = workerPid !== null || watcherPid !== null;
|
|
1679
1720
|
if (!hadPids && nowMs - lastHostReportMs(lane) < DEAD_HOST_MS) {
|
|
1680
1721
|
try {
|
|
@@ -2175,9 +2216,40 @@ async function admitQueued(hqRoot, lanes, summary, deps, now, onlyLaneIds, exclu
|
|
|
2175
2216
|
}
|
|
2176
2217
|
const projected = [...running, ...spawned];
|
|
2177
2218
|
const alternateAccount = availableBalancedAccount(hqRoot, lane, projected, config, deps.host, profileMaxConcurrent, now.getTime());
|
|
2178
|
-
|
|
2219
|
+
let admittedLane = alternateAccount !== undefined
|
|
2179
2220
|
? writeAdmissionAccount(hqRoot, lane, alternateAccount, deps.dryRun)
|
|
2180
2221
|
: lane;
|
|
2222
|
+
if (admittedLane.terminal_details?.loop_recovery === true) {
|
|
2223
|
+
const accounts = config.providers[admittedLane.provider].accounts;
|
|
2224
|
+
const company = resolveLaneCompany(hqRoot, admittedLane).company;
|
|
2225
|
+
let account;
|
|
2226
|
+
try {
|
|
2227
|
+
account = resolveAccount(admittedLane.provider, admittedLane.account, config, {
|
|
2228
|
+
hqRoot, hostname: deps.host.hostname,
|
|
2229
|
+
allowedAccounts: Object.keys(accounts).filter((id) => accountMatchesCompany(accounts[id], company, accounts)),
|
|
2230
|
+
}).id;
|
|
2231
|
+
}
|
|
2232
|
+
catch (error) {
|
|
2233
|
+
if (!(error instanceof AccountResolutionError))
|
|
2234
|
+
throw error;
|
|
2235
|
+
writeQueueReason(hqRoot, admittedLane, "account_retry", deps.dryRun);
|
|
2236
|
+
continue;
|
|
2237
|
+
}
|
|
2238
|
+
if (!accountMatchesCompany(accounts[account], company, accounts) && accounts[account]?.company !== undefined) {
|
|
2239
|
+
writeQueueReason(hqRoot, admittedLane, "account_retry", deps.dryRun);
|
|
2240
|
+
continue;
|
|
2241
|
+
}
|
|
2242
|
+
if (admittedLane.account !== account)
|
|
2243
|
+
admittedLane = writeAdmissionAccount(hqRoot, admittedLane, account, deps.dryRun);
|
|
2244
|
+
const status = accountStatus(admittedLane.provider, account, config, {
|
|
2245
|
+
hostname: deps.host.hostname,
|
|
2246
|
+
existsSync: (file) => fs.existsSync(file),
|
|
2247
|
+
});
|
|
2248
|
+
if (!status?.enabled || !isCredentialProven(status)) {
|
|
2249
|
+
writeQueueReason(hqRoot, admittedLane, "account_retry", deps.dryRun);
|
|
2250
|
+
continue;
|
|
2251
|
+
}
|
|
2252
|
+
}
|
|
2181
2253
|
const baseCap = laneAdmissionCapReason(running, admittedLane, config, deps.host, profileMaxConcurrent, now.getTime());
|
|
2182
2254
|
const currentCap = laneAdmissionCapReason([...running, ...spawned], admittedLane, config, deps.host, profileMaxConcurrent, now.getTime());
|
|
2183
2255
|
let blockedBy;
|
|
@@ -2832,6 +2904,9 @@ async function resumeKeepAlive(hqRoot, lanes, questions, summary, deps, now, fla
|
|
|
2832
2904
|
await admitDueCandidates(new Set(queuedDueCandidateIds));
|
|
2833
2905
|
};
|
|
2834
2906
|
for (let originalLane of candidates) {
|
|
2907
|
+
// Recovery requires an explicit effects disposition, even for keep-alive leads.
|
|
2908
|
+
if (originalLane.terminal_reason === "loop_recovery_required")
|
|
2909
|
+
continue;
|
|
2835
2910
|
if (admitDueCandidates !== undefined && dueCandidateIds.size > 0 && !dueCandidateIds.has(originalLane.lane_id)) {
|
|
2836
2911
|
await admitQueuedDueCandidates();
|
|
2837
2912
|
}
|
|
@@ -15,6 +15,9 @@ export interface WorkCacheSubscriber {
|
|
|
15
15
|
stop: () => Promise<void>;
|
|
16
16
|
}
|
|
17
17
|
export { AGENT_INBOX_PUSH_STATE_BASENAME, SYNC_WAKE_STATE_BASENAME } from "../paths.js";
|
|
18
|
+
export type SyncWakeVaults = Record<string, {
|
|
19
|
+
wakeSequence: number;
|
|
20
|
+
}>;
|
|
18
21
|
export declare function startWorkCacheSubscriber(opts: {
|
|
19
22
|
meshRoot: string;
|
|
20
23
|
fetchCredentials?: PersonalCredentialsFetcher;
|
|
@@ -15,22 +15,22 @@ const INBOX_PUSH_HEARTBEAT_MS = 5_000;
|
|
|
15
15
|
const SYNC_WAKE_KEYS = new Set(["v", "type", "vault", "head", "rootId"]);
|
|
16
16
|
const SYNC_WAKE_VAULT = /^(?:prs|cmp|agt)_[0-9A-Za-z]+$/;
|
|
17
17
|
const SYNC_WAKE_ROOT = /^[0-9a-f]{64}$/i;
|
|
18
|
-
function
|
|
18
|
+
function parseSyncWakeEnvelope(payload) {
|
|
19
19
|
let parsed;
|
|
20
20
|
try {
|
|
21
21
|
parsed = JSON.parse(Buffer.isBuffer(payload) ? payload.toString("utf8") : payload);
|
|
22
22
|
}
|
|
23
23
|
catch {
|
|
24
|
-
return
|
|
24
|
+
return null;
|
|
25
25
|
}
|
|
26
26
|
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed))
|
|
27
|
-
return
|
|
27
|
+
return null;
|
|
28
28
|
const message = parsed;
|
|
29
29
|
const keys = Object.keys(message);
|
|
30
30
|
if (keys.length !== SYNC_WAKE_KEYS.size || keys.some((key) => !SYNC_WAKE_KEYS.has(key))) {
|
|
31
|
-
return
|
|
31
|
+
return null;
|
|
32
32
|
}
|
|
33
|
-
|
|
33
|
+
const valid = message.v === 1 &&
|
|
34
34
|
message.type === "vault.head" &&
|
|
35
35
|
typeof message.vault === "string" &&
|
|
36
36
|
SYNC_WAKE_VAULT.test(message.vault) &&
|
|
@@ -38,6 +38,21 @@ function isSyncWakeEnvelope(payload) {
|
|
|
38
38
|
Number(message.head) > 0 &&
|
|
39
39
|
(message.rootId === null ||
|
|
40
40
|
(typeof message.rootId === "string" && SYNC_WAKE_ROOT.test(message.rootId)));
|
|
41
|
+
return valid ? { vault: message.vault } : null;
|
|
42
|
+
}
|
|
43
|
+
function readVaultSequences(raw) {
|
|
44
|
+
if (!raw || typeof raw !== "object" || Array.isArray(raw))
|
|
45
|
+
return {};
|
|
46
|
+
const vaults = {};
|
|
47
|
+
for (const [vault, entry] of Object.entries(raw)) {
|
|
48
|
+
if (!SYNC_WAKE_VAULT.test(vault) || !entry || typeof entry !== "object")
|
|
49
|
+
continue;
|
|
50
|
+
const sequence = entry.wakeSequence;
|
|
51
|
+
if (Number.isSafeInteger(sequence) && Number(sequence) >= 0) {
|
|
52
|
+
vaults[vault] = { wakeSequence: Number(sequence) };
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
return vaults;
|
|
41
56
|
}
|
|
42
57
|
function readWakeSequence(meshRoot, basename, label, onInfo) {
|
|
43
58
|
const target = path.join(meshRoot, basename);
|
|
@@ -45,21 +60,29 @@ function readWakeSequence(meshRoot, basename, label, onInfo) {
|
|
|
45
60
|
const info = fs.lstatSync(target);
|
|
46
61
|
if (!info.isFile() || info.isSymbolicLink()) {
|
|
47
62
|
onInfo?.(`${label} state read ignored (not a regular file)`);
|
|
48
|
-
return { sequence: 0, writable: false };
|
|
63
|
+
return { sequence: 0, writable: false, vaults: {} };
|
|
49
64
|
}
|
|
50
65
|
const parsed = JSON.parse(fs.readFileSync(target, "utf8"));
|
|
51
66
|
if (Number.isSafeInteger(parsed.wakeSequence) && Number(parsed.wakeSequence) >= 0) {
|
|
52
|
-
return {
|
|
67
|
+
return {
|
|
68
|
+
sequence: Number(parsed.wakeSequence),
|
|
69
|
+
writable: true,
|
|
70
|
+
vaults: readVaultSequences(parsed.vaults),
|
|
71
|
+
};
|
|
53
72
|
}
|
|
54
73
|
onInfo?.(`${label} state read ignored (invalid wake sequence)`);
|
|
55
|
-
return { sequence: 0, writable: false };
|
|
74
|
+
return { sequence: 0, writable: false, vaults: {} };
|
|
56
75
|
}
|
|
57
76
|
catch (error) {
|
|
58
77
|
if (error.code !== "ENOENT") {
|
|
59
78
|
const name = error instanceof Error ? error.name : "UnknownError";
|
|
60
79
|
onInfo?.(`${label} state read failed (${name})`);
|
|
61
80
|
}
|
|
62
|
-
return {
|
|
81
|
+
return {
|
|
82
|
+
sequence: 0,
|
|
83
|
+
writable: error.code === "ENOENT",
|
|
84
|
+
vaults: {},
|
|
85
|
+
};
|
|
63
86
|
}
|
|
64
87
|
}
|
|
65
88
|
function writeWakeState(meshRoot, basename, state, durable) {
|
|
@@ -132,19 +155,21 @@ export async function startWorkCacheSubscriber(opts) {
|
|
|
132
155
|
let inboxWakeSequence = readWakeSequence(opts.meshRoot, AGENT_INBOX_PUSH_STATE_BASENAME, "agent inbox push", opts.onInfo).sequence;
|
|
133
156
|
const syncWakeState = readWakeSequence(opts.meshRoot, SYNC_WAKE_STATE_BASENAME, "sync wake", opts.onInfo);
|
|
134
157
|
let syncWakeSequence = syncWakeState.sequence;
|
|
158
|
+
const syncWakeVaults = { ...syncWakeState.vaults };
|
|
135
159
|
const syncStateWritable = syncWakeState.writable;
|
|
136
160
|
const clearInboxHeartbeat = () => {
|
|
137
161
|
if (heartbeatTimer !== null)
|
|
138
162
|
timers.clearTimeout(heartbeatTimer);
|
|
139
163
|
heartbeatTimer = null;
|
|
140
164
|
};
|
|
141
|
-
const persistState = (basename, label, connected, wakeSequence, identity, durable = true) => {
|
|
165
|
+
const persistState = (basename, label, connected, wakeSequence, identity, durable = true, vaults) => {
|
|
142
166
|
try {
|
|
143
167
|
writeWakeState(opts.meshRoot, basename, {
|
|
144
168
|
connected,
|
|
145
169
|
heartbeatAtMs: now().getTime(),
|
|
146
170
|
wakeSequence,
|
|
147
171
|
...(connected && actorUid && identity ? { [identity]: actorUid } : {}),
|
|
172
|
+
...(vaults ? { vaults } : {}),
|
|
148
173
|
}, durable);
|
|
149
174
|
}
|
|
150
175
|
catch (error) {
|
|
@@ -154,7 +179,7 @@ export async function startWorkCacheSubscriber(opts) {
|
|
|
154
179
|
};
|
|
155
180
|
const persistInboxState = (connected, durable = true) => persistState(AGENT_INBOX_PUSH_STATE_BASENAME, "agent inbox push", connected, inboxWakeSequence, "agentUid", durable);
|
|
156
181
|
const persistSyncState = (connected, durable = true) => syncStateWritable
|
|
157
|
-
? persistState(SYNC_WAKE_STATE_BASENAME, "sync wake", connected, syncWakeSequence, undefined, durable)
|
|
182
|
+
? persistState(SYNC_WAKE_STATE_BASENAME, "sync wake", connected, syncWakeSequence, undefined, durable, syncWakeVaults)
|
|
158
183
|
: undefined;
|
|
159
184
|
const scheduleInboxHeartbeat = () => {
|
|
160
185
|
if (stopped || (!inboxSubscribed && !syncSubscribed))
|
|
@@ -287,11 +312,15 @@ export async function startWorkCacheSubscriber(opts) {
|
|
|
287
312
|
return;
|
|
288
313
|
}
|
|
289
314
|
if (syncTopic && _topic === syncTopic && syncSubscribed) {
|
|
290
|
-
|
|
315
|
+
const envelope = parseSyncWakeEnvelope(payload);
|
|
316
|
+
if (!envelope) {
|
|
291
317
|
opts.onInfo?.("sync wake ignored (invalid envelope)");
|
|
292
318
|
return;
|
|
293
319
|
}
|
|
294
320
|
syncWakeSequence += 1;
|
|
321
|
+
syncWakeVaults[envelope.vault] = {
|
|
322
|
+
wakeSequence: (syncWakeVaults[envelope.vault]?.wakeSequence ?? 0) + 1,
|
|
323
|
+
};
|
|
295
324
|
persistSyncState(true);
|
|
296
325
|
return;
|
|
297
326
|
}
|
|
@@ -29,16 +29,23 @@ export declare function writeEngineHome(opts: WriteEngineHomeOptions): {
|
|
|
29
29
|
engineHome: string;
|
|
30
30
|
changed: boolean;
|
|
31
31
|
};
|
|
32
|
-
export interface
|
|
32
|
+
export interface SyncCoreFirstCycle {
|
|
33
33
|
at: string;
|
|
34
|
+
planned: number;
|
|
35
|
+
completed: number;
|
|
36
|
+
pending: number;
|
|
37
|
+
held: number;
|
|
38
|
+
notApplied: number;
|
|
34
39
|
uploads: number;
|
|
35
40
|
downloads: number;
|
|
36
41
|
deletes: number;
|
|
37
42
|
conflicts: number;
|
|
38
|
-
|
|
39
|
-
|
|
43
|
+
deferred: number;
|
|
44
|
+
listed: number;
|
|
45
|
+
truncated: boolean;
|
|
40
46
|
}
|
|
41
|
-
export declare
|
|
47
|
+
export declare const SYNC_CORE_STATUS_WORK_LIMIT = 20;
|
|
48
|
+
export declare function engineFirstCycle(engineHome: string, vaultUid: string, since: string): SyncCoreFirstCycle | null;
|
|
42
49
|
export declare function clearEngineStatus(home: string, vaultUid: string): void;
|
|
43
50
|
export declare function removeEngineOwnership(home: string, vaultUid: string): void;
|
|
44
51
|
export declare function inspectEngineHome(home: string, hqRoot: string | undefined, plan: SyncCoreVaultPlan, platform: NodeJS.Platform): SyncCoreVaultPlan;
|