@indigoai-us/hq-cli 5.345.175 → 5.345.177

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. package/CHANGELOG.md +25 -0
  2. package/dist/command-catalog.generated.d.ts +65 -0
  3. package/dist/command-catalog.generated.js +80 -0
  4. package/dist/commands/bot.d.ts +28 -2
  5. package/dist/commands/bot.js +29 -6
  6. package/dist/commands/daemon.js +16 -0
  7. package/dist/commands/lanes-loop.d.ts +4 -1
  8. package/dist/commands/lanes-loop.js +23 -4
  9. package/dist/commands/lanes-shared.d.ts +5 -0
  10. package/dist/commands/lanes.d.ts +5 -7
  11. package/dist/commands/lanes.js +65 -5
  12. package/dist/commands/sync-core.d.ts +17 -0
  13. package/dist/commands/sync-core.js +46 -5
  14. package/dist/commands/sync-manifest.d.ts +6 -0
  15. package/dist/commands/sync-manifest.js +13 -0
  16. package/dist/hq-cloud-sync-runner/bin/sync-runner.js +448 -226
  17. package/dist/hq-cloud-sync-runner/package.json +1 -1
  18. package/dist/lib/bot/cli-update.d.ts +36 -0
  19. package/dist/lib/bot/cli-update.js +160 -42
  20. package/dist/lib/bot/run.d.ts +21 -0
  21. package/dist/lib/bot/run.js +127 -10
  22. package/dist/lib/bot/runtime/claude.d.ts +7 -1
  23. package/dist/lib/bot/runtime/claude.js +29 -0
  24. package/dist/lib/bot/runtime/codex.d.ts +5 -0
  25. package/dist/lib/bot/runtime/codex.js +13 -0
  26. package/dist/lib/bot/runtime/types.d.ts +10 -0
  27. package/dist/lib/bot/runtime-sign-in-check.d.ts +32 -0
  28. package/dist/lib/bot/runtime-sign-in-check.js +53 -0
  29. package/dist/lib/bot/runtime-sign-in.d.ts +32 -3
  30. package/dist/lib/bot/runtime-sign-in.js +17 -2
  31. package/dist/lib/bot/status.d.ts +25 -1
  32. package/dist/lib/bot/status.js +28 -0
  33. package/dist/lib/daemon/catalog.d.ts +4 -0
  34. package/dist/lib/daemon/catalog.js +47 -19
  35. package/dist/lib/daemon/run.js +1 -0
  36. package/dist/lib/flag-registry-worker-source.generated.d.ts +1 -1
  37. package/dist/lib/flag-registry-worker-source.generated.js +1 -1
  38. package/dist/lib/lanes/loop-disposition.d.ts +44 -0
  39. package/dist/lib/lanes/loop-disposition.js +302 -0
  40. package/dist/lib/lanes/loop.d.ts +16 -3
  41. package/dist/lib/lanes/loop.js +167 -60
  42. package/dist/lib/lanes/reconcile.d.ts +5 -0
  43. package/dist/lib/lanes/reconcile.js +79 -4
  44. package/dist/lib/mesh/live/daemon/work-subscribe.d.ts +3 -0
  45. package/dist/lib/mesh/live/daemon/work-subscribe.js +41 -12
  46. package/dist/lib/sync-core/engine-home.d.ts +11 -4
  47. package/dist/lib/sync-core/engine-home.js +22 -21
  48. package/dist/lib/sync-core/manifest-scopes.d.ts +15 -0
  49. package/dist/lib/sync-core/manifest-scopes.js +26 -0
  50. package/dist/lib/sync-core/reconcile.d.ts +2 -2
  51. package/dist/lib/sync-core/reconcile.js +70 -46
  52. package/dist/lib/sync-core/release.d.ts +2 -2
  53. package/dist/lib/sync-core/release.js +31 -6
  54. package/dist/lib/sync-core/rollout-state.d.ts +6 -3
  55. package/dist/lib/sync-core/rollout-state.js +26 -22
  56. package/package.json +8 -8
@@ -2,6 +2,7 @@
2
2
  import { createHash, randomUUID } from "node:crypto";
3
3
  import * as fs from "node:fs";
4
4
  import * as path from "node:path";
5
+ import { isPidRecordAlive, readProcStarttime } from "./spawn.js";
5
6
  import { providerSupportsSessionResume } from "./delivery.js";
6
7
  export const LOOP_SCHEMA = "hq-lane-loop.v1";
7
8
  export const PHASE_ENVELOPE_SCHEMA = "hq-phase-envelope/v1";
@@ -362,7 +363,9 @@ function nextQueueSequence(runDir) {
362
363
  atomicJson(sequenceFile, { sequence });
363
364
  return sequence;
364
365
  }
365
- function acquireRunnerLease(runDir) {
366
+ function acquireRunnerLease(runDir, identity) {
367
+ const readStarttime = identity?.readStarttime ?? readProcStarttime;
368
+ const starttime = readStarttime(process.pid) ?? undefined;
366
369
  const lock = path.join(runDir, "loop", "runner.lock");
367
370
  fs.mkdirSync(path.dirname(lock), { recursive: true });
368
371
  try {
@@ -372,27 +375,34 @@ function acquireRunnerLease(runDir) {
372
375
  if (error.code !== "EEXIST")
373
376
  throw error;
374
377
  let pid;
378
+ let recordedStarttime;
375
379
  try {
376
380
  const owner = readJson(path.join(lock, "owner.json"));
377
- if (isRecord(owner) && typeof owner.pid === "number")
381
+ if (isRecord(owner) && typeof owner.pid === "number") {
378
382
  pid = owner.pid;
383
+ recordedStarttime = typeof owner.starttime === "number" ? owner.starttime : undefined;
384
+ }
379
385
  }
380
386
  catch (readError) {
381
387
  if (readError.code !== "ENOENT" && !(readError instanceof SyntaxError))
382
388
  throw readError;
383
389
  }
384
- let alive = false;
385
- if (pid !== undefined && Number.isSafeInteger(pid) && pid > 0) {
386
- try {
387
- process.kill(pid, 0);
388
- alive = true;
389
- }
390
- catch (probeError) {
391
- alive = probeError.code === "EPERM";
392
- if (!alive && probeError.code !== "ESRCH")
393
- throw probeError;
394
- }
395
- }
390
+ const alive = pid !== undefined && Number.isSafeInteger(pid) && pid > 0 &&
391
+ isPidRecordAlive({ pid, starttime: recordedStarttime }, identity?.isAlive ?? ((candidate) => {
392
+ try {
393
+ process.kill(candidate, 0);
394
+ return true;
395
+ }
396
+ catch (error) {
397
+ if (error.code === "EPERM")
398
+ return true;
399
+ if (error.code === "ESRCH")
400
+ return false;
401
+ throw error;
402
+ }
403
+ }), readStarttime);
404
+ if (pid === undefined || !Number.isSafeInteger(pid) || pid <= 0)
405
+ throw new Error("Loop runner lease ownership is unknown; explicit recovery required");
396
406
  const age = Date.now() - fs.statSync(lock).mtimeMs;
397
407
  if (alive || age <= 60_000)
398
408
  throw new Error(`Loop runner is already active${pid ? ` (pid ${pid})` : ""}`);
@@ -423,9 +433,12 @@ function acquireRunnerLease(runDir) {
423
433
  if (released)
424
434
  return;
425
435
  released = true;
436
+ process.removeListener("exit", release);
426
437
  clearInterval(heartbeat);
427
438
  try {
428
- fs.rmSync(lock, { recursive: true, force: true });
439
+ const owner = readJson(ownerFile);
440
+ if (isRecord(owner) && owner.pid === process.pid && owner.starttime === starttime)
441
+ fs.rmSync(lock, { recursive: true, force: true });
429
442
  }
430
443
  catch (error) {
431
444
  if (error.code !== "ENOENT")
@@ -434,27 +447,35 @@ function acquireRunnerLease(runDir) {
434
447
  };
435
448
  const heartbeat = setInterval(() => {
436
449
  try {
437
- atomicJson(ownerFile, { pid: process.pid, heartbeat_at: new Date().toISOString() });
450
+ atomicJson(ownerFile, { pid: process.pid, starttime, heartbeat_at: new Date().toISOString() });
438
451
  }
439
452
  catch (error) {
440
453
  process.stderr.write(`hq lanes: loop lease heartbeat failed: ${error instanceof Error ? error.message : String(error)}\n`);
441
454
  }
442
455
  }, 15_000);
443
- atomicJson(ownerFile, { pid: process.pid, heartbeat_at: new Date().toISOString() });
456
+ atomicJson(ownerFile, { pid: process.pid, starttime, heartbeat_at: new Date().toISOString() });
444
457
  process.once("exit", release);
445
- fs.mkdirSync(pendingDir(runDir), { recursive: true });
446
- for (const active of envelopeFiles(activeDir(runDir))) {
447
- const recovered = path.join(pendingDir(runDir), `0000000000000-recovered-${path.basename(active)}`);
448
- try {
449
- fs.renameSync(active, recovered);
450
- }
451
- catch (recoveryError) {
452
- if (recoveryError.code !== "ENOENT") {
453
- release();
454
- throw recoveryError;
455
- }
458
+ // A dead controller may have performed external effects before it died.
459
+ // Replaying an active envelope cannot be justified by reclaiming its lease.
460
+ if (inspectLoopRecovery(runDir).kind === "hold") {
461
+ release();
462
+ throw new Error("Loop recovery requires an explicit disposition of interrupted work");
463
+ }
464
+ try {
465
+ // Only this lease owner may finalize the proven local completion window.
466
+ // Never execute it again; a failure leaves the original active file intact.
467
+ for (const file of completedActiveRecovery(runDir).files) {
468
+ const destination = path.join(claimedDir(runDir), path.basename(file));
469
+ if (fs.existsSync(destination))
470
+ throw new Error("Completed loop archive already exists; explicit recovery required");
471
+ fs.mkdirSync(claimedDir(runDir), { recursive: true });
472
+ fs.renameSync(file, destination);
456
473
  }
457
474
  }
475
+ catch (error) {
476
+ release();
477
+ throw error;
478
+ }
458
479
  return release;
459
480
  }
460
481
  function envelopeFiles(dir) {
@@ -806,8 +827,8 @@ export function commitIdleLoopStop(runDir, laneId, now = () => new Date()) {
806
827
  persistStatus(runDir, laneId, { state: "stopped", queue_depth: listLoopQueue(runDir).length, active_envelope_id: null, active_phase: null });
807
828
  return true;
808
829
  }
809
- export function setLoopControllerPid(runDir, laneId, pid) {
810
- persistStatus(runDir, laneId, { pid });
830
+ export function setLoopControllerPid(runDir, laneId, pid, starttime) {
831
+ persistStatus(runDir, laneId, { pid, starttime });
811
832
  }
812
833
  export function clearLoopControllerPid(runDir, laneId, pid) {
813
834
  let currentPid;
@@ -821,51 +842,129 @@ export function clearLoopControllerPid(runDir, laneId, pid) {
821
842
  throw error;
822
843
  }
823
844
  if (currentPid === pid)
824
- persistStatus(runDir, laneId, { pid: undefined });
845
+ persistStatus(runDir, laneId, { pid: undefined, starttime: undefined });
825
846
  }
826
847
  /**
827
848
  * True when a loop lane's controller is alive with no phase active: the lane
828
849
  * finished its last phase and is waiting for the next envelope.
829
850
  */
830
- export function loopLaneIsIdle(runDir, laneId) {
851
+ export function loopLaneIsIdle(runDir, laneId, isAlive, readStarttime) {
831
852
  const status = loopStatus(runDir, laneId);
832
- return status.state === "waiting" && status.active_envelope_id === null && loopControllerIsAlive(runDir);
853
+ return status.state === "waiting" && status.active_envelope_id === null && loopControllerIsAlive(runDir, isAlive, readStarttime);
833
854
  }
834
- export function loopControllerIsAlive(runDir) {
835
- let pid;
836
- try {
837
- const status = readJson(loopPath(runDir));
838
- if (isRecord(status))
839
- pid = status.pid;
855
+ export function loopControllerIsAlive(runDir, isAlive, readStarttime = readProcStarttime) {
856
+ const records = [];
857
+ for (const file of [loopPath(runDir), path.join(runDir, "loop", "runner.lock", "owner.json")]) {
858
+ try {
859
+ const value = readJson(file);
860
+ if (isRecord(value) && typeof value.pid === "number" && Number.isSafeInteger(value.pid) && value.pid > 0) {
861
+ records.push({ pid: value.pid, starttime: typeof value.starttime === "number" ? value.starttime : undefined });
862
+ }
863
+ }
864
+ catch (error) {
865
+ if (error.code !== "ENOENT" && !(error instanceof SyntaxError))
866
+ throw error;
867
+ }
840
868
  }
841
- catch (error) {
842
- if (error.code !== "ENOENT" && !(error instanceof SyntaxError))
869
+ const probe = isAlive ?? ((candidate) => {
870
+ try {
871
+ process.kill(candidate, 0);
872
+ return true;
873
+ }
874
+ catch (error) {
875
+ if (error.code === "EPERM")
876
+ return true;
877
+ if (error.code === "ESRCH")
878
+ return false;
843
879
  throw error;
844
- }
845
- if (typeof pid !== "number") {
880
+ }
881
+ });
882
+ // A failed second launch may overwrite loop.json before its lease fails.
883
+ // The live lease owner still protects the first controller.
884
+ return records.some(record => isPidRecordAlive(record, probe, readStarttime));
885
+ }
886
+ /** A local terminal result is enough only when no external handoff was promised. */
887
+ function completedActiveRecovery(runDir) {
888
+ const files = [];
889
+ for (const file of envelopeFiles(activeDir(runDir))) {
890
+ const record = fileEnvelope(file);
891
+ if (record.envelope.kind === "stop")
892
+ return { files: [], stopped: true };
893
+ let result;
846
894
  try {
847
- const owner = readJson(path.join(runDir, "loop", "runner.lock", "owner.json"));
848
- if (isRecord(owner))
849
- pid = owner.pid;
895
+ result = readJson(path.join(resultsDir(runDir), resultName(record.envelope_id)));
850
896
  }
851
897
  catch (error) {
852
898
  if (error.code !== "ENOENT" && !(error instanceof SyntaxError))
853
899
  throw error;
854
900
  }
901
+ if (!isRecord(result) || result.schema_version !== LOOP_SCHEMA || result.envelope_id !== record.envelope_id ||
902
+ !["done", "failed"].includes(String(result.status)) ||
903
+ typeof result.started_at !== "string" || !Number.isFinite(Date.parse(result.started_at)) ||
904
+ typeof result.finished_at !== "string" || !Number.isFinite(Date.parse(result.finished_at)) ||
905
+ Date.parse(result.finished_at) < Date.parse(result.started_at)) {
906
+ return { files: [], reason: "active_phase_outcome_unknown" };
907
+ }
908
+ if (typeof record.envelope.result_path === "string") {
909
+ // Local write happens before the optional external handoff. Do not read
910
+ // arbitrary paths or assume a local result proves that handoff finished.
911
+ return { files: [], reason: "external_completion_unverified" };
912
+ }
913
+ files.push(file);
855
914
  }
856
- if (typeof pid !== "number" || !Number.isSafeInteger(pid) || pid <= 0)
857
- return false;
858
- try {
859
- process.kill(pid, 0);
860
- return true;
915
+ return { files };
916
+ }
917
+ /** Read durable work without changing queue files or assuming effects are replayable. */
918
+ export function inspectLoopRecovery(runDir) {
919
+ const status = loopStatus(runDir, path.basename(runDir));
920
+ if (["parked", "stopped", "failed"].includes(status.state))
921
+ return { kind: "protected" };
922
+ const lease = path.join(runDir, "loop", "runner.lock");
923
+ if (fs.existsSync(lease)) {
924
+ let owner;
925
+ try {
926
+ owner = readJson(path.join(lease, "owner.json"));
927
+ }
928
+ catch (error) {
929
+ if (error.code !== "ENOENT" && !(error instanceof SyntaxError))
930
+ throw error;
931
+ }
932
+ if (!isRecord(owner) || !Number.isSafeInteger(owner.pid) || Number(owner.pid) <= 0) {
933
+ return { kind: "hold", reason: "runner_lease_identity_unknown" };
934
+ }
861
935
  }
862
- catch (error) {
863
- if (error.code === "EPERM")
864
- return true;
865
- if (error.code === "ESRCH")
866
- return false;
867
- throw error;
936
+ const completed = completedActiveRecovery(runDir);
937
+ if (completed.reason)
938
+ return { kind: "hold", reason: completed.reason };
939
+ if (completed.stopped)
940
+ return { kind: "protected" };
941
+ for (const file of envelopeFiles(claimedDir(runDir))) {
942
+ const record = fileEnvelope(file);
943
+ if (record.envelope.kind === "stop")
944
+ return { kind: "protected" };
945
+ let result;
946
+ try {
947
+ result = readJson(path.join(resultsDir(runDir), resultName(record.envelope_id)));
948
+ }
949
+ catch (error) {
950
+ if (error.code !== "ENOENT" && !(error instanceof SyntaxError))
951
+ throw error;
952
+ }
953
+ if (!isRecord(result) || result.schema_version !== LOOP_SCHEMA || result.envelope_id !== record.envelope_id ||
954
+ !["done", "failed", "stopped"].includes(String(result.status))) {
955
+ return { kind: "hold", reason: "claimed_phase_outcome_unknown" };
956
+ }
957
+ if (result.status === "stopped")
958
+ return { kind: "protected" };
868
959
  }
960
+ const pending = listLoopQueue(runDir);
961
+ // A completed phase still needs lease-owned archiving before an idle stop.
962
+ if (completed.files.length > 0)
963
+ return { kind: "pending" };
964
+ // A queued intentional stop remains owned by the ordinary stop path.
965
+ if (pending[0]?.envelope?.kind === "stop")
966
+ return { kind: "protected" };
967
+ return { kind: pending.length > 0 ? "pending" : "empty" };
869
968
  }
870
969
  function phaseName(envelope, fallback) {
871
970
  const value = envelope;
@@ -911,6 +1010,14 @@ function persistStatus(runDir, laneId, patch) {
911
1010
  updated_at: new Date().toISOString(),
912
1011
  });
913
1012
  }
1013
+ /** Reset only after an explicit, durable disposition; this never starts work. */
1014
+ export function resetLoopAfterDisposition(runDir, laneId, decision, hasPending) {
1015
+ persistStatus(runDir, laneId, {
1016
+ state: hasPending ? "waiting" : "failed", active_envelope_id: null, active_phase: null,
1017
+ pid: undefined, starttime: undefined, queue_depth: listLoopQueue(runDir).length,
1018
+ recovery_decision: decision,
1019
+ });
1020
+ }
914
1021
  function abortAfter(ms, parent) {
915
1022
  const controller = new AbortController();
916
1023
  const timer = setTimeout(() => controller.abort(new Error("phase deadline exceeded")), Math.min(Math.max(ms, 1), 2_147_483_647));
@@ -947,12 +1054,12 @@ export async function runLoopLane(options) {
947
1054
  const stalls = new Map();
948
1055
  let completed = 0;
949
1056
  fs.mkdirSync(options.runDir, { recursive: true });
950
- let releaseRunner = acquireRunnerLease(options.runDir);
1057
+ let releaseRunner = acquireRunnerLease(options.runDir, options.processIdentity);
951
1058
  if (readHandoffHold(options.runDir)) {
952
1059
  releaseRunner();
953
1060
  return { state: loopStatus(options.runDir, options.lane.lane_id).state === "stopped" ? "stopped" : "parked", completed: 0 };
954
1061
  }
955
- persistStatus(options.runDir, options.lane.lane_id, { pid: process.pid });
1062
+ persistStatus(options.runDir, options.lane.lane_id, { pid: process.pid, starttime: (options.processIdentity?.readStarttime ?? readProcStarttime)(process.pid) ?? undefined });
956
1063
  try {
957
1064
  const previous = readJson(loopPath(options.runDir));
958
1065
  if (isRecord(previous) && isRecord(previous.stall_counts)) {
@@ -1206,7 +1313,7 @@ export async function runLoopLane(options) {
1206
1313
  const answeredRetryIsPending = envelopeFiles(pendingDir(options.runDir)).some((pendingFile) => path.basename(pendingFile).startsWith("0000000000000-answer-") && fileEnvelope(pendingFile).envelope_id === record.envelope_id);
1207
1314
  if (answeredRetryIsPending) {
1208
1315
  try {
1209
- releaseRunner = acquireRunnerLease(options.runDir);
1316
+ releaseRunner = acquireRunnerLease(options.runDir, options.processIdentity);
1210
1317
  persistStatus(options.runDir, options.lane.lane_id, { pid: process.pid, state: "waiting", queue_depth: listLoopQueue(options.runDir).length, active_envelope_id: null, active_phase: null });
1211
1318
  continue;
1212
1319
  }
@@ -102,6 +102,11 @@ export interface ReconcileDeps {
102
102
  throttleDeps?: ThrottleDeps;
103
103
  livenessDeps?: LivenessDeps;
104
104
  isAlive?: (pid: number) => boolean;
105
+ /** Controller identity is independent of worker/watcher liveness. */
106
+ controllerProcessIdentity?: {
107
+ isAlive(pid: number): boolean;
108
+ readStarttime(pid: number): number | null;
109
+ };
105
110
  readSessionSenior?: (sessionId: string) => SeniorRef;
106
111
  presenceConnected?: boolean;
107
112
  env?: NodeJS.ProcessEnv;
@@ -13,7 +13,7 @@ import { compareLaneQueueOrder, compareLaneQueueOrderWithLeads, leadLaneIds, has
13
13
  import { accountStatus, isCredentialProven, resolveAccount } from "./accounts.js";
14
14
  import { effectiveAccountHome } from "./account-home.js";
15
15
  import { enforceRegistryApprovalGate } from "./approval-gate.js";
16
- import { loopLaneIsIdle, loopStatus } from "./loop.js";
16
+ import { inspectLoopRecovery, loopControllerIsAlive, loopLaneIsIdle, loopStatus } from "./loop.js";
17
17
  import { loadLanesConfig, providerQuotaDefaultHoldMs } from "./config.js";
18
18
  import { MAX_LANE_DEPTH } from "./constants.js";
19
19
  import { laneDepthCap } from "./depth-cap.js";
@@ -1499,7 +1499,8 @@ export async function runPassBody(hqRoot, deps) {
1499
1499
  }));
1500
1500
  const admissionThrottleDeps = startDeps.throttleDeps ?? defaultThrottleDeps(startDeps.env ?? process.env);
1501
1501
  const hasAdmissionCandidates = lanes.some((lane) => laneRunsHere(lane, startDeps.host) &&
1502
- (lane.state === "queued" || lane.state === "awaiting_input")) || [...keepAliveFlagByCompany.values()].some(Boolean);
1502
+ (lane.state === "queued" || lane.state === "awaiting_input")) || summary.dead_host.some((entry) => entry.reason === "loop_controller_lost" && entry.to === "queued") ||
1503
+ [...keepAliveFlagByCompany.values()].some(Boolean);
1503
1504
  const priorThrottleState = readThrottleState(throttleStatePath(hqRoot), admissionThrottleDeps);
1504
1505
  const priorThrottleSnapshot = priorThrottleState === "missing" || priorThrottleState === "unreadable"
1505
1506
  ? undefined
@@ -1597,6 +1598,8 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
1597
1598
  const nowIso = now.toISOString();
1598
1599
  const isAlive = deps.isAlive ?? isPidAlive;
1599
1600
  for (const lane of lanes) {
1601
+ // Historical done lanes require explicit recovery; leftover files alone
1602
+ // do not prove that their work should run again.
1600
1603
  if (lane.state !== "running")
1601
1604
  continue;
1602
1605
  // No HQ process backs a subagent lane, so a missing pid is not a dead
@@ -1658,7 +1661,7 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
1658
1661
  // Watcher owns terminal classification of a process that has just exited.
1659
1662
  continue;
1660
1663
  }
1661
- if (lane.loop_mode === true && loopLaneIsIdle(runDir, lane.lane_id)) {
1664
+ if (lane.loop_mode === true && loopLaneIsIdle(runDir, lane.lane_id, deps.controllerProcessIdentity?.isAlive, deps.controllerProcessIdentity?.readStarttime)) {
1662
1665
  // An idle loop lane has no worker between phases. Its controller is
1663
1666
  // alive and waiting for the next envelope, so the lane is not finished.
1664
1667
  try {
@@ -1675,6 +1678,44 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
1675
1678
  }
1676
1679
  continue;
1677
1680
  }
1681
+ if (lane.loop_mode === true) {
1682
+ try {
1683
+ // A controller can be alive while a phase starts; never use a stale
1684
+ // phase envelope to terminalize or admit behind that controller.
1685
+ if (loopControllerIsAlive(runDir, deps.controllerProcessIdentity?.isAlive, deps.controllerProcessIdentity?.readStarttime))
1686
+ continue;
1687
+ const recovery = inspectLoopRecovery(runDir);
1688
+ if (recovery.kind === "protected")
1689
+ continue;
1690
+ if (recovery.kind === "hold" || recovery.kind === "pending") {
1691
+ const hasQuestion = loadQuestionRecords(hqRoot, summary).some(question => question.lane_id === lane.lane_id && question.status === "pending");
1692
+ if (hasQuestion)
1693
+ continue;
1694
+ const hold = recovery.kind === "hold" || deps.spawn === undefined;
1695
+ const reason = hold ? "loop_recovery_required" : "loop_controller_lost";
1696
+ const result = casLaneIf(hqRoot, lane.lane_id, current => current.state === "running" && current.loop_mode === true &&
1697
+ laneRunsHere(current, deps.host) && !current.terminal_reason?.startsWith("operator_") &&
1698
+ !["loop_stopped", "story_closed"].includes(current.terminal_reason ?? ""), current => ({ ...current, state: hold ? "interrupted" : "queued",
1699
+ queued_reason: undefined, queued_details: undefined,
1700
+ admission_reserved_at: undefined,
1701
+ terminal_reason: hold ? reason : undefined,
1702
+ terminal_details: hold ? { reason: recovery.reason ?? "controller_admission_unavailable", automatic_replay: false } : { loop_recovery: true },
1703
+ interrupted_at: hold ? nowIso : undefined,
1704
+ last_host_report_at: nowIso,
1705
+ }), deps.dryRun);
1706
+ if (result.changed)
1707
+ summary.dead_host.push({ id: lane.lane_id, from: lane.state, to: result.record.state, reason });
1708
+ continue;
1709
+ }
1710
+ }
1711
+ catch (err) {
1712
+ markDirty(summary, { kind: "lane", id: lane.lane_id,
1713
+ why: `loop recovery inspection failed: ${err instanceof Error ? err.message : String(err)}` });
1714
+ continue;
1715
+ }
1716
+ }
1717
+ if (lane.state !== "running")
1718
+ continue;
1678
1719
  const hadPids = workerPid !== null || watcherPid !== null;
1679
1720
  if (!hadPids && nowMs - lastHostReportMs(lane) < DEAD_HOST_MS) {
1680
1721
  try {
@@ -2175,9 +2216,40 @@ async function admitQueued(hqRoot, lanes, summary, deps, now, onlyLaneIds, exclu
2175
2216
  }
2176
2217
  const projected = [...running, ...spawned];
2177
2218
  const alternateAccount = availableBalancedAccount(hqRoot, lane, projected, config, deps.host, profileMaxConcurrent, now.getTime());
2178
- const admittedLane = alternateAccount !== undefined
2219
+ let admittedLane = alternateAccount !== undefined
2179
2220
  ? writeAdmissionAccount(hqRoot, lane, alternateAccount, deps.dryRun)
2180
2221
  : lane;
2222
+ if (admittedLane.terminal_details?.loop_recovery === true) {
2223
+ const accounts = config.providers[admittedLane.provider].accounts;
2224
+ const company = resolveLaneCompany(hqRoot, admittedLane).company;
2225
+ let account;
2226
+ try {
2227
+ account = resolveAccount(admittedLane.provider, admittedLane.account, config, {
2228
+ hqRoot, hostname: deps.host.hostname,
2229
+ allowedAccounts: Object.keys(accounts).filter((id) => accountMatchesCompany(accounts[id], company, accounts)),
2230
+ }).id;
2231
+ }
2232
+ catch (error) {
2233
+ if (!(error instanceof AccountResolutionError))
2234
+ throw error;
2235
+ writeQueueReason(hqRoot, admittedLane, "account_retry", deps.dryRun);
2236
+ continue;
2237
+ }
2238
+ if (!accountMatchesCompany(accounts[account], company, accounts) && accounts[account]?.company !== undefined) {
2239
+ writeQueueReason(hqRoot, admittedLane, "account_retry", deps.dryRun);
2240
+ continue;
2241
+ }
2242
+ if (admittedLane.account !== account)
2243
+ admittedLane = writeAdmissionAccount(hqRoot, admittedLane, account, deps.dryRun);
2244
+ const status = accountStatus(admittedLane.provider, account, config, {
2245
+ hostname: deps.host.hostname,
2246
+ existsSync: (file) => fs.existsSync(file),
2247
+ });
2248
+ if (!status?.enabled || !isCredentialProven(status)) {
2249
+ writeQueueReason(hqRoot, admittedLane, "account_retry", deps.dryRun);
2250
+ continue;
2251
+ }
2252
+ }
2181
2253
  const baseCap = laneAdmissionCapReason(running, admittedLane, config, deps.host, profileMaxConcurrent, now.getTime());
2182
2254
  const currentCap = laneAdmissionCapReason([...running, ...spawned], admittedLane, config, deps.host, profileMaxConcurrent, now.getTime());
2183
2255
  let blockedBy;
@@ -2832,6 +2904,9 @@ async function resumeKeepAlive(hqRoot, lanes, questions, summary, deps, now, fla
2832
2904
  await admitDueCandidates(new Set(queuedDueCandidateIds));
2833
2905
  };
2834
2906
  for (let originalLane of candidates) {
2907
+ // Recovery requires an explicit effects disposition, even for keep-alive leads.
2908
+ if (originalLane.terminal_reason === "loop_recovery_required")
2909
+ continue;
2835
2910
  if (admitDueCandidates !== undefined && dueCandidateIds.size > 0 && !dueCandidateIds.has(originalLane.lane_id)) {
2836
2911
  await admitQueuedDueCandidates();
2837
2912
  }
@@ -15,6 +15,9 @@ export interface WorkCacheSubscriber {
15
15
  stop: () => Promise<void>;
16
16
  }
17
17
  export { AGENT_INBOX_PUSH_STATE_BASENAME, SYNC_WAKE_STATE_BASENAME } from "../paths.js";
18
+ export type SyncWakeVaults = Record<string, {
19
+ wakeSequence: number;
20
+ }>;
18
21
  export declare function startWorkCacheSubscriber(opts: {
19
22
  meshRoot: string;
20
23
  fetchCredentials?: PersonalCredentialsFetcher;
@@ -15,22 +15,22 @@ const INBOX_PUSH_HEARTBEAT_MS = 5_000;
15
15
  const SYNC_WAKE_KEYS = new Set(["v", "type", "vault", "head", "rootId"]);
16
16
  const SYNC_WAKE_VAULT = /^(?:prs|cmp|agt)_[0-9A-Za-z]+$/;
17
17
  const SYNC_WAKE_ROOT = /^[0-9a-f]{64}$/i;
18
- function isSyncWakeEnvelope(payload) {
18
+ function parseSyncWakeEnvelope(payload) {
19
19
  let parsed;
20
20
  try {
21
21
  parsed = JSON.parse(Buffer.isBuffer(payload) ? payload.toString("utf8") : payload);
22
22
  }
23
23
  catch {
24
- return false;
24
+ return null;
25
25
  }
26
26
  if (!parsed || typeof parsed !== "object" || Array.isArray(parsed))
27
- return false;
27
+ return null;
28
28
  const message = parsed;
29
29
  const keys = Object.keys(message);
30
30
  if (keys.length !== SYNC_WAKE_KEYS.size || keys.some((key) => !SYNC_WAKE_KEYS.has(key))) {
31
- return false;
31
+ return null;
32
32
  }
33
- return message.v === 1 &&
33
+ const valid = message.v === 1 &&
34
34
  message.type === "vault.head" &&
35
35
  typeof message.vault === "string" &&
36
36
  SYNC_WAKE_VAULT.test(message.vault) &&
@@ -38,6 +38,21 @@ function isSyncWakeEnvelope(payload) {
38
38
  Number(message.head) > 0 &&
39
39
  (message.rootId === null ||
40
40
  (typeof message.rootId === "string" && SYNC_WAKE_ROOT.test(message.rootId)));
41
+ return valid ? { vault: message.vault } : null;
42
+ }
43
+ function readVaultSequences(raw) {
44
+ if (!raw || typeof raw !== "object" || Array.isArray(raw))
45
+ return {};
46
+ const vaults = {};
47
+ for (const [vault, entry] of Object.entries(raw)) {
48
+ if (!SYNC_WAKE_VAULT.test(vault) || !entry || typeof entry !== "object")
49
+ continue;
50
+ const sequence = entry.wakeSequence;
51
+ if (Number.isSafeInteger(sequence) && Number(sequence) >= 0) {
52
+ vaults[vault] = { wakeSequence: Number(sequence) };
53
+ }
54
+ }
55
+ return vaults;
41
56
  }
42
57
  function readWakeSequence(meshRoot, basename, label, onInfo) {
43
58
  const target = path.join(meshRoot, basename);
@@ -45,21 +60,29 @@ function readWakeSequence(meshRoot, basename, label, onInfo) {
45
60
  const info = fs.lstatSync(target);
46
61
  if (!info.isFile() || info.isSymbolicLink()) {
47
62
  onInfo?.(`${label} state read ignored (not a regular file)`);
48
- return { sequence: 0, writable: false };
63
+ return { sequence: 0, writable: false, vaults: {} };
49
64
  }
50
65
  const parsed = JSON.parse(fs.readFileSync(target, "utf8"));
51
66
  if (Number.isSafeInteger(parsed.wakeSequence) && Number(parsed.wakeSequence) >= 0) {
52
- return { sequence: Number(parsed.wakeSequence), writable: true };
67
+ return {
68
+ sequence: Number(parsed.wakeSequence),
69
+ writable: true,
70
+ vaults: readVaultSequences(parsed.vaults),
71
+ };
53
72
  }
54
73
  onInfo?.(`${label} state read ignored (invalid wake sequence)`);
55
- return { sequence: 0, writable: false };
74
+ return { sequence: 0, writable: false, vaults: {} };
56
75
  }
57
76
  catch (error) {
58
77
  if (error.code !== "ENOENT") {
59
78
  const name = error instanceof Error ? error.name : "UnknownError";
60
79
  onInfo?.(`${label} state read failed (${name})`);
61
80
  }
62
- return { sequence: 0, writable: error.code === "ENOENT" };
81
+ return {
82
+ sequence: 0,
83
+ writable: error.code === "ENOENT",
84
+ vaults: {},
85
+ };
63
86
  }
64
87
  }
65
88
  function writeWakeState(meshRoot, basename, state, durable) {
@@ -132,19 +155,21 @@ export async function startWorkCacheSubscriber(opts) {
132
155
  let inboxWakeSequence = readWakeSequence(opts.meshRoot, AGENT_INBOX_PUSH_STATE_BASENAME, "agent inbox push", opts.onInfo).sequence;
133
156
  const syncWakeState = readWakeSequence(opts.meshRoot, SYNC_WAKE_STATE_BASENAME, "sync wake", opts.onInfo);
134
157
  let syncWakeSequence = syncWakeState.sequence;
158
+ const syncWakeVaults = { ...syncWakeState.vaults };
135
159
  const syncStateWritable = syncWakeState.writable;
136
160
  const clearInboxHeartbeat = () => {
137
161
  if (heartbeatTimer !== null)
138
162
  timers.clearTimeout(heartbeatTimer);
139
163
  heartbeatTimer = null;
140
164
  };
141
- const persistState = (basename, label, connected, wakeSequence, identity, durable = true) => {
165
+ const persistState = (basename, label, connected, wakeSequence, identity, durable = true, vaults) => {
142
166
  try {
143
167
  writeWakeState(opts.meshRoot, basename, {
144
168
  connected,
145
169
  heartbeatAtMs: now().getTime(),
146
170
  wakeSequence,
147
171
  ...(connected && actorUid && identity ? { [identity]: actorUid } : {}),
172
+ ...(vaults ? { vaults } : {}),
148
173
  }, durable);
149
174
  }
150
175
  catch (error) {
@@ -154,7 +179,7 @@ export async function startWorkCacheSubscriber(opts) {
154
179
  };
155
180
  const persistInboxState = (connected, durable = true) => persistState(AGENT_INBOX_PUSH_STATE_BASENAME, "agent inbox push", connected, inboxWakeSequence, "agentUid", durable);
156
181
  const persistSyncState = (connected, durable = true) => syncStateWritable
157
- ? persistState(SYNC_WAKE_STATE_BASENAME, "sync wake", connected, syncWakeSequence, undefined, durable)
182
+ ? persistState(SYNC_WAKE_STATE_BASENAME, "sync wake", connected, syncWakeSequence, undefined, durable, syncWakeVaults)
158
183
  : undefined;
159
184
  const scheduleInboxHeartbeat = () => {
160
185
  if (stopped || (!inboxSubscribed && !syncSubscribed))
@@ -287,11 +312,15 @@ export async function startWorkCacheSubscriber(opts) {
287
312
  return;
288
313
  }
289
314
  if (syncTopic && _topic === syncTopic && syncSubscribed) {
290
- if (!isSyncWakeEnvelope(payload)) {
315
+ const envelope = parseSyncWakeEnvelope(payload);
316
+ if (!envelope) {
291
317
  opts.onInfo?.("sync wake ignored (invalid envelope)");
292
318
  return;
293
319
  }
294
320
  syncWakeSequence += 1;
321
+ syncWakeVaults[envelope.vault] = {
322
+ wakeSequence: (syncWakeVaults[envelope.vault]?.wakeSequence ?? 0) + 1,
323
+ };
295
324
  persistSyncState(true);
296
325
  return;
297
326
  }
@@ -29,16 +29,23 @@ export declare function writeEngineHome(opts: WriteEngineHomeOptions): {
29
29
  engineHome: string;
30
30
  changed: boolean;
31
31
  };
32
- export interface SyncCorePlanFigures {
32
+ export interface SyncCoreFirstCycle {
33
33
  at: string;
34
+ planned: number;
35
+ completed: number;
36
+ pending: number;
37
+ held: number;
38
+ notApplied: number;
34
39
  uploads: number;
35
40
  downloads: number;
36
41
  deletes: number;
37
42
  conflicts: number;
38
- remoteRows: number;
39
- localFiles: number;
43
+ deferred: number;
44
+ listed: number;
45
+ truncated: boolean;
40
46
  }
41
- export declare function engineReadOnlyPlan(engineHome: string, vaultUid: string): SyncCorePlanFigures | null;
47
+ export declare const SYNC_CORE_STATUS_WORK_LIMIT = 20;
48
+ export declare function engineFirstCycle(engineHome: string, vaultUid: string, since: string): SyncCoreFirstCycle | null;
42
49
  export declare function clearEngineStatus(home: string, vaultUid: string): void;
43
50
  export declare function removeEngineOwnership(home: string, vaultUid: string): void;
44
51
  export declare function inspectEngineHome(home: string, hqRoot: string | undefined, plan: SyncCoreVaultPlan, platform: NodeJS.Platform): SyncCoreVaultPlan;