@indigoai-us/hq-cli 5.342.0 → 5.342.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -2,6 +2,10 @@
2
2
 
3
3
  ## [Unreleased]
4
4
 
5
+ ## [5.342.1] — 2026-10-04
6
+ - `hq lanes` now continues envelope numbering beyond 999 and records unexpected reconcile admission failures as a distinct queued reason with error details.
7
+ - Reconcile skips configured Codex homes absent on the current host and reports a note; other filesystem errors remain visible as store errors. `--dry-run` no longer compresses rollouts or advances the retention schedule.
8
+
5
9
  ## [5.342.0] — 2026-10-04
6
10
 
7
11
  - Keep `hq integrations call` alive through slow provider calls with bounded async-job polling, safe idempotent retry guidance on timeout, and validation of terminal responses.
@@ -5089,8 +5089,12 @@ async function runList(ctx, opts) {
5089
5089
  ? {
5090
5090
  state: "remote",
5091
5091
  remote_state: lane.state,
5092
- queued_reason: lane.queued_reason === "low_disk" ? lane.queued_reason : undefined,
5093
- queued_details: lane.queued_reason === "low_disk" ? lane.queued_details : undefined,
5092
+ queued_reason: lane.queued_reason === "low_disk" || lane.queued_reason === "admit_failed"
5093
+ ? lane.queued_reason
5094
+ : undefined,
5095
+ queued_details: lane.queued_reason === "low_disk" || lane.queued_reason === "admit_failed"
5096
+ ? lane.queued_details
5097
+ : undefined,
5094
5098
  }
5095
5099
  : {}),
5096
5100
  overdue,
@@ -5142,7 +5146,9 @@ function formatLaneListHuman(lanes) {
5142
5146
  : "no story: " + (lane.no_story_reason ?? "reason not recorded");
5143
5147
  const queueLabel = (lane.state === "queued" || lane.remote_state === "queued") && lane.queued_reason === "low_disk"
5144
5148
  ? ` queued: low_disk (HQ ${lane.queued_details?.reason === "low_disk" ? lane.queued_details.hq_free_bytes : "?"} bytes; provider ${lane.queued_details?.reason === "low_disk" ? lane.queued_details.provider_free_bytes : "?"} bytes free)`
5145
- : "";
5149
+ : (lane.state === "queued" || lane.remote_state === "queued") && lane.queued_reason === "admit_failed"
5150
+ ? ` queued: admit_failed (${lane.queued_details?.reason === "admit_failed" ? `${lane.queued_details.error}: ${lane.queued_details.message}` : "details unavailable"})`
5151
+ : "";
5146
5152
  return lane.lane_id + " " + lane.state + " " + lane.provider + " " + lane.project_id +
5147
5153
  " " + storyLabel + queueLabel;
5148
5154
  });
@@ -5199,9 +5205,11 @@ async function runStatus(ctx, opts) {
5199
5205
  async function runCapacity(ctx) {
5200
5206
  const hqRoot = ctx.hqRoot();
5201
5207
  const host = laneHostFromCtx(ctx);
5208
+ const admissionHost = reconcileHostFromCtx(ctx);
5209
+ const lanes = loadAllLanes(hqRoot);
5202
5210
  const snapshot = buildHostCapacitySnapshot({
5203
5211
  host,
5204
- lanes: loadAllLanes(hqRoot),
5212
+ lanes,
5205
5213
  config: loadLanesConfig(hqRoot),
5206
5214
  existsSync: ctx.existsSync,
5207
5215
  probe: ctx.probeProvider,
@@ -5209,7 +5217,16 @@ async function runCapacity(ctx) {
5209
5217
  hostname: ctx.hostname(),
5210
5218
  now: ctx.now(),
5211
5219
  });
5212
- printJson(snapshot);
5220
+ printJson({
5221
+ ...snapshot,
5222
+ queued_lanes: lanes
5223
+ .filter((lane) => lane.state === "queued" && laneRunsHere(lane, admissionHost))
5224
+ .map((lane) => ({
5225
+ lane_id: lane.lane_id,
5226
+ queued_reason: lane.queued_reason ?? null,
5227
+ ...(lane.queued_details !== undefined ? { queued_details: lane.queued_details } : {}),
5228
+ })),
5229
+ });
5213
5230
  }
5214
5231
  async function runGcRollouts(ctx, opts) {
5215
5232
  const hqRoot = ctx.hqRoot();
@@ -5395,8 +5412,8 @@ async function runShow(ctx, laneId, hasPendingQuestionForLane, resolveLaneTarget
5395
5412
  ...storedLane,
5396
5413
  state: "remote",
5397
5414
  remote_state: storedLane.state,
5398
- queued_reason: storedLane.queued_reason === "low_disk" ? storedLane.queued_reason : undefined,
5399
- queued_details: storedLane.queued_reason === "low_disk" ? storedLane.queued_details : undefined,
5415
+ queued_reason: storedLane.queued_reason,
5416
+ queued_details: storedLane.queued_details,
5400
5417
  }
5401
5418
  : storedLane;
5402
5419
  const askDeliveryFailed = readJsonFile(askDeliveryFailedPath(hqRoot, laneId));
@@ -0,0 +1,3 @@
1
+ /** Numeric ordering for numbered lane envelope filenames, including 1000+. */
2
+ export declare function compareEnvelopeNames(left: string, right: string): number;
3
+ //# sourceMappingURL=envelope-sequence.d.ts.map
@@ -0,0 +1,13 @@
1
+ /** Numeric ordering for numbered lane envelope filenames, including 1000+. */
2
+ export function compareEnvelopeNames(left, right) {
3
+ const leftMatch = /^(\d+)\.json$/.exec(left);
4
+ const rightMatch = /^(\d+)\.json$/.exec(right);
5
+ if (leftMatch && rightMatch) {
6
+ const leftNumber = BigInt(leftMatch[1]);
7
+ const rightNumber = BigInt(rightMatch[1]);
8
+ if (leftNumber !== rightNumber)
9
+ return leftNumber < rightNumber ? -1 : 1;
10
+ }
11
+ return left.localeCompare(right);
12
+ }
13
+ //# sourceMappingURL=envelope-sequence.js.map
@@ -38,6 +38,15 @@ export interface KeepAliveAuditResult {
38
38
  errors: KeepAliveAuditError[];
39
39
  summary: KeepAliveAuditSummary;
40
40
  }
41
+ export interface StopEnvelope {
42
+ name: string;
43
+ mtimeMs: number;
44
+ nextCheckAt?: string;
45
+ }
46
+ export declare function readStopEnvelopes(hqRoot: string, laneId: string): {
47
+ allMtimes: number[];
48
+ stops: StopEnvelope[];
49
+ };
41
50
  export declare function buildKeepAliveAudit(input: {
42
51
  hqRoot: string;
43
52
  now: Date;
@@ -3,6 +3,7 @@ import * as path from "node:path";
3
3
  import { LEAD_LANE_KEEP_ALIVE_BASE_BACKOFF_MS } from "./keep-alive.js";
4
4
  import { LanesError } from "./errors.js";
5
5
  import { laneRunDir } from "./spawn.js";
6
+ import { compareEnvelopeNames } from "./envelope-sequence.js";
6
7
  import { listQuestionIds, readAllLanes, tryReadQuestion } from "./store.js";
7
8
  function readDoneClassifications(hqRoot, laneId) {
8
9
  const root = fs.realpathSync(hqRoot);
@@ -77,7 +78,7 @@ function withinRoot(root, candidate) {
77
78
  const relative = path.relative(root, candidate);
78
79
  return relative !== "" && relative !== ".." && !relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative);
79
80
  }
80
- function readStopEnvelopes(hqRoot, laneId) {
81
+ export function readStopEnvelopes(hqRoot, laneId) {
81
82
  const root = fs.realpathSync(hqRoot);
82
83
  const envelopeDir = path.join(laneRunDir(root, laneId), "envelopes");
83
84
  if (!withinRoot(root, envelopeDir)) {
@@ -89,7 +90,7 @@ function readStopEnvelopes(hqRoot, laneId) {
89
90
  if (!withinRoot(root, realDir)) {
90
91
  throw new LanesError("keep_alive_audit_path_invalid", `Lane envelope directory escapes the HQ root for ${laneId}`, { lane_id: laneId });
91
92
  }
92
- names = fs.readdirSync(realDir).filter((name) => /^\d+\.json$/.test(name)).sort();
93
+ names = fs.readdirSync(realDir).filter((name) => /^\d+\.json$/.test(name)).sort(compareEnvelopeNames);
93
94
  }
94
95
  catch (err) {
95
96
  if (err.code === "ENOENT")
@@ -118,11 +119,15 @@ function readStopEnvelopes(hqRoot, laneId) {
118
119
  if (raw.decision !== "done" || raw.story_complete !== false)
119
120
  continue;
120
121
  stops.push({
122
+ name,
121
123
  mtimeMs: stat.mtimeMs,
122
124
  ...(typeof raw.next_check_at === "string" ? { nextCheckAt: raw.next_check_at } : {}),
123
125
  });
124
126
  }
125
- return { allMtimes: allMtimes.sort((a, b) => a - b), stops: stops.sort((a, b) => a.mtimeMs - b.mtimeMs) };
127
+ return {
128
+ allMtimes: allMtimes.sort((a, b) => a - b),
129
+ stops: stops.sort((a, b) => a.mtimeMs - b.mtimeMs),
130
+ };
126
131
  }
127
132
  function readKeepAliveResumeEvents(hqRoot, laneId) {
128
133
  const root = fs.realpathSync(hqRoot);
@@ -52,6 +52,7 @@ export interface ReconcileSummary {
52
52
  dead_host: ReconcileChange[];
53
53
  heartbeats: string[];
54
54
  skipped: ReconcileSkip[];
55
+ notes?: string[];
55
56
  keep_alive_blocked: Array<{
56
57
  id: string;
57
58
  reason: KeepAliveBlockedReason;
@@ -18,7 +18,7 @@ import { AccountResolutionError, LanesError, isLaneVersionConflict } from "./err
18
18
  import { loadWorkerProfile } from "./profile.js";
19
19
  import { terminalChildResumeHireGrantError, withHireGrantAdmission, } from "./grants.js";
20
20
  import { classifySessionLiveness, defaultLivenessDeps, loadLivenessIndex, readDaemonPresenceConnected, unindexedClaudeSessionLiveness, } from "./liveness.js";
21
- import { ANSWERED_QUESTIONS_FILE, armLaneWatcher, envelopeRepairMode, isPidRecordAlive, laneRunDir, readPidRecord, readProcStarttime, readSpawnManifest, markSpawnFailure, stopLaneProcesses, spawnEnvelopeRepair, spawnLaneWorker, retryLaneOnAlternateAccount, } from "./spawn.js";
21
+ import { ANSWERED_QUESTIONS_FILE, armLaneWatcher, envelopeRepairMode, isPidRecordAlive, laneRunDir, nextEnvelopePath, readPidRecord, readProcStarttime, readSpawnManifest, markSpawnFailure, stopLaneProcesses, spawnEnvelopeRepair, spawnLaneWorker, retryLaneOnAlternateAccount, } from "./spawn.js";
22
22
  import { appendCappedHistory, lanesStoreRoot, parseLaneRecord, parseQuestionRecord, readLane, readQuestion, listQuestionIds, updateLane, updateQuestion, } from "./store.js";
23
23
  import { applyThrottleDecision, decideAdmission, defaultThrottleDeps, emptyThrottleState, evaluateThrottle, isCpuCheckEnabled, readAvailableMemMb, readThrottleState, resolveCpuBusyFraction, sampleCpuBusyObservation, recordCpuBusyObservation, throttleStatePath, writeThrottleState, } from "./throttle.js";
24
24
  import { readLanesRegistryFlag, readLanesRegistryFlagFailOff } from "./registry-flag.js";
@@ -87,6 +87,27 @@ function markDirty(summary, skip) {
87
87
  summary.ok = false;
88
88
  logSkip(skip);
89
89
  }
90
+ function isMissingPathError(error) {
91
+ const code = error?.code;
92
+ return code === "ENOENT" || code === "ENOTDIR";
93
+ }
94
+ function recordMissingCodexHomeNote(summary, home) {
95
+ const note = `Codex rollout retention skipped home absent on this host: ${home}`;
96
+ summary.notes ??= [];
97
+ summary.notes.push(note);
98
+ process.stderr.write(`hq lanes reconcile: note: ${note}\n`);
99
+ }
100
+ function homeIsMissingAfterError(home, error) {
101
+ if (!isMissingPathError(error))
102
+ return false;
103
+ try {
104
+ fs.statSync(home);
105
+ return false;
106
+ }
107
+ catch (statError) {
108
+ return isMissingPathError(statError);
109
+ }
110
+ }
90
111
  function terminalizeModelUnsupported(hqRoot, lane, err) {
91
112
  if (!(err instanceof LanesError) || err.error !== "model_unsupported")
92
113
  return;
@@ -940,14 +961,28 @@ export async function runPassBody(hqRoot, deps) {
940
961
  companyRefs.set(company.uid ?? company.slug, company);
941
962
  }
942
963
  const readFlag = deps.readRegistryFlag ?? readLanesRegistryFlag;
943
- const homes = configuredCodexRolloutHomes(hqRoot, config, deps.host.hostname);
964
+ const configuredHomes = configuredCodexRolloutHomes(hqRoot, config, deps.host.hostname);
944
965
  let retentionEnabled = false;
945
- for (const company of homes.length > 0 ? companyRefs.values() : []) {
966
+ for (const company of configuredHomes.length > 0 ? companyRefs.values() : []) {
946
967
  retentionEnabled ||= await readFlag("lanes.codex-rollout-retention", { company });
947
968
  if (retentionEnabled)
948
969
  break;
949
970
  }
950
971
  if (retentionEnabled) {
972
+ const presentHomes = [];
973
+ for (const home of configuredHomes) {
974
+ try {
975
+ fs.statSync(home);
976
+ presentHomes.push(home);
977
+ }
978
+ catch (err) {
979
+ if (isMissingPathError(err)) {
980
+ recordMissingCodexHomeNote(summary, home);
981
+ continue;
982
+ }
983
+ throw err;
984
+ }
985
+ }
951
986
  const statePath = path.join(hqRoot, "workspace", "lanes-runs", `.rollout-retention-${deps.host.hostname.replaceAll(/[^a-zA-Z0-9_-]/g, "_")}.json`);
952
987
  let lastRunAt;
953
988
  try {
@@ -961,20 +996,38 @@ export async function runPassBody(hqRoot, deps) {
961
996
  if (err.code !== "ENOENT")
962
997
  throw err;
963
998
  }
964
- const lowDisk = homes.some((home) => checkLaneDiskSpace(hqRoot, home, deps.spawnLaneDeps?.statfs, config.low_disk_min_free_bytes).low);
999
+ let lowDisk = false;
1000
+ const retentionHomes = [];
1001
+ for (const home of presentHomes) {
1002
+ try {
1003
+ const disk = checkLaneDiskSpace(hqRoot, home, deps.spawnLaneDeps?.statfs, config.low_disk_min_free_bytes);
1004
+ lowDisk = lowDisk || disk.low;
1005
+ retentionHomes.push(home);
1006
+ }
1007
+ catch (err) {
1008
+ if (homeIsMissingAfterError(home, err)) {
1009
+ recordMissingCodexHomeNote(summary, home);
1010
+ continue;
1011
+ }
1012
+ throw err;
1013
+ }
1014
+ }
965
1015
  if (lowDisk || lastRunAt === undefined || nowMs - lastRunAt >= 60 * 60 * 1000) {
966
1016
  const result = await retainCodexRollouts({
967
- homes,
1017
+ homes: retentionHomes,
968
1018
  lanes,
969
1019
  nowMs,
1020
+ dryRun: deps.dryRun,
970
1021
  compressAfterMs: config.codex_rollout_compress_after_hours * 60 * 60 * 1000,
971
1022
  deleteAfterMs: config.codex_rollout_delete_after_days * 24 * 60 * 60 * 1000,
972
1023
  });
973
1024
  summary.rollout_retention = result;
974
- fs.mkdirSync(path.dirname(statePath), { recursive: true });
975
- const tempPath = `${statePath}.${process.pid}.tmp`;
976
- fs.writeFileSync(tempPath, `${JSON.stringify({ last_run_at: nowIso }, null, 2)}\n`, { flag: "wx" });
977
- fs.renameSync(tempPath, statePath);
1025
+ if (!deps.dryRun) {
1026
+ fs.mkdirSync(path.dirname(statePath), { recursive: true });
1027
+ const tempPath = `${statePath}.${process.pid}.tmp`;
1028
+ fs.writeFileSync(tempPath, `${JSON.stringify({ last_run_at: nowIso }, null, 2)}\n`, { flag: "wx" });
1029
+ fs.renameSync(tempPath, statePath);
1030
+ }
978
1031
  }
979
1032
  }
980
1033
  }
@@ -1848,6 +1901,9 @@ async function admitQueued(hqRoot, lanes, summary, deps, now, onlyLaneIds, exclu
1848
1901
  }
1849
1902
  try {
1850
1903
  if (deps.dryRun) {
1904
+ // Spawn reserves an envelope path. Probe it without writing so a dry
1905
+ // run does not promise admission when the run cannot start.
1906
+ nextEnvelopePath(laneRunDir(hqRoot, lane.lane_id));
1851
1907
  summary.admitted.push({
1852
1908
  id: lane.lane_id,
1853
1909
  from: "queued",
@@ -1907,6 +1963,18 @@ async function admitQueued(hqRoot, lanes, summary, deps, now, onlyLaneIds, exclu
1907
1963
  }
1908
1964
  }
1909
1965
  terminalizeModelUnsupported(hqRoot, lane, err);
1966
+ if (err instanceof LanesError && err.error !== "model_unsupported") {
1967
+ const queueRecorded = deps.dryRun || casLaneIf(hqRoot, lane.lane_id, (current) => current.state === "queued", (current) => ({
1968
+ ...current,
1969
+ queued_reason: "admit_failed",
1970
+ queued_details: { reason: "admit_failed", error: err.error, message: err.message },
1971
+ admission_reserved_at: undefined,
1972
+ }), false).changed;
1973
+ if (queueRecorded) {
1974
+ summary.skipped.push({ kind: "lane", id: lane.lane_id, why: `admit_failed:${err.error}` });
1975
+ continue;
1976
+ }
1977
+ }
1910
1978
  markDirty(summary, {
1911
1979
  kind: "lane",
1912
1980
  id: lane.lane_id,
@@ -983,16 +983,21 @@ function proveDetachedViaPs(pid) {
983
983
  }
984
984
  export function nextEnvelopePath(runDir) {
985
985
  const dir = path.join(runDir, "envelopes");
986
+ let names = new Set();
987
+ try {
988
+ names = new Set(fs.readdirSync(dir));
989
+ }
990
+ catch (err) {
991
+ if (err.code !== "ENOENT")
992
+ throw err;
993
+ }
986
994
  let n = 1;
987
995
  while (true) {
988
996
  const name = `${String(n).padStart(3, "0")}.json`;
989
997
  const filePath = path.join(dir, name);
990
- if (!fs.existsSync(filePath))
998
+ if (!names.has(name))
991
999
  return filePath;
992
1000
  n += 1;
993
- if (n > 999) {
994
- throw new LanesError("envelope_sequence_exhausted", `No free envelope slot under ${dir}`, { dir });
995
- }
996
1001
  }
997
1002
  }
998
1003
  export function envelopeInstruction(envelopePath) {
@@ -623,6 +623,16 @@ export function parseLaneRecord(raw, filePath) {
623
623
  }
624
624
  requireTimestamp(detailsRecord.measured_at, "queued_details.measured_at", filePath);
625
625
  }
626
+ else if (detailsRecord.reason === "admit_failed") {
627
+ if (raw.state !== "queued" || raw.queued_reason !== "admit_failed" ||
628
+ typeof detailsRecord.error !== "string" || detailsRecord.error.trim() === "" ||
629
+ typeof detailsRecord.message !== "string" || detailsRecord.message.trim() === "") {
630
+ throw new LaneSchemaError("queued_details must describe a failed lane admission", {
631
+ path: filePath,
632
+ field: "queued_details",
633
+ });
634
+ }
635
+ }
626
636
  else {
627
637
  throw new LaneSchemaError("queued_details must describe a supported queue condition", {
628
638
  path: filePath,
@@ -177,7 +177,7 @@ export declare const TERMINAL_LANE_STATES: readonly ["done", "failed", "interrup
177
177
  export type TerminalLaneState = (typeof TERMINAL_LANE_STATES)[number];
178
178
  export declare const LANE_PROVIDERS: readonly ["claude", "codex", "grok"];
179
179
  export type LaneProvider = (typeof LANE_PROVIDERS)[number];
180
- export declare const QUEUED_REASONS: readonly ["cap", "provider_cap", "account_cap", "profile_cap", "queue_position", "throttle", "provider_binary_missing", "account_retry", "worker_profile_changed", "low_disk"];
180
+ export declare const QUEUED_REASONS: readonly ["cap", "provider_cap", "account_cap", "profile_cap", "queue_position", "throttle", "provider_binary_missing", "account_retry", "worker_profile_changed", "low_disk", "admit_failed"];
181
181
  export type QueuedReason = (typeof QUEUED_REASONS)[number];
182
182
  export type QueuedDetails = {
183
183
  reason: "worker_profile_changed";
@@ -193,6 +193,10 @@ export type QueuedDetails = {
193
193
  hq_free_bytes: number;
194
194
  provider_free_bytes: number;
195
195
  measured_at: string;
196
+ } | {
197
+ reason: "admit_failed";
198
+ error: string;
199
+ message: string;
196
200
  };
197
201
  export declare function isQueuedReason(value: unknown): value is QueuedReason;
198
202
  export declare const HOST_KINDS: readonly ["local", "outpost"];
@@ -113,6 +113,7 @@ export const QUEUED_REASONS = [
113
113
  "account_retry",
114
114
  "worker_profile_changed",
115
115
  "low_disk",
116
+ "admit_failed",
116
117
  ];
117
118
  export function isQueuedReason(value) {
118
119
  return typeof value === "string" && QUEUED_REASONS.includes(value);
@@ -34,6 +34,7 @@
34
34
  import * as fs from "node:fs";
35
35
  import * as path from "node:path";
36
36
  import { LanesError } from "./errors.js";
37
+ import { compareEnvelopeNames } from "./envelope-sequence.js";
37
38
  import { questionRecordRefusalCode, raiseEnvelopeQuestions, resolveQuestionCompanyWithFallback, } from "./questions.js";
38
39
  import { isTerminalLaneState, ENVELOPE_INVALID_REASON, parseLaneEnvelope, STORY_STATUSES, LANE_PROVIDERS, } from "./types.js";
39
40
  import { laneStories, terminalEnvelopeStoryStatus } from "./stories.js";
@@ -619,8 +620,8 @@ export function readLatestEnvelopeFile(hqRoot, laneId, minName) {
619
620
  names = fs
620
621
  .readdirSync(dir)
621
622
  .filter((name) => name.endsWith(".json"))
622
- .filter((name) => minName === undefined || name >= minName)
623
- .sort();
623
+ .filter((name) => minName === undefined || compareEnvelopeNames(name, minName) >= 0)
624
+ .sort(compareEnvelopeNames);
624
625
  }
625
626
  catch (err) {
626
627
  const code = err.code;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@indigoai-us/hq-cli",
3
- "version": "5.342.0",
3
+ "version": "5.342.1",
4
4
  "description": "HQ by Indigo management CLI — modules and cloud sync",
5
5
  "main": "dist/index.js",
6
6
  "bin": {