@indigoai-us/hq-cli 5.132.0 → 5.133.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -2,6 +2,32 @@
2
2
 
3
3
  ## [Unreleased]
4
4
 
5
+ ## [5.133.0] — 2026-09-22
6
+
7
+ ### Fixed
8
+
9
+ - Lane follow-ups now probe all enabled, logged-in accounts for an unpinned
10
+ model, preserve the later quota reset and its source, expire abandoned pickup
11
+ validation markers with a terminal reason, and parse Codex's ordinal
12
+ `try again at` reset time.
13
+
14
+ - `hq sync push` no longer fails every file when the target vault was
15
+ provisioned on or after 2026-09-19. Those vaults carry an S3 Object Lock
16
+ default retention rule, and S3 refuses a PutObject into them unless the
17
+ request carries a content checksum; the CLI's declared `@indigoai-us/hq-cloud`
18
+ range still allowed a client that sends none, so an affected push reported
19
+ `400 InvalidRequest: Content-MD5 OR x-amz-checksum- HTTP header is required
20
+ for Put Object requests with Object Lock parameters` on every file. The
21
+ dependency floor now starts at the release that attaches the checksum, so no
22
+ install can resolve a client that cannot write to a locked vault. A CI guard
23
+ holds the floor.
24
+
25
+ - Lane creation and re-parenting now allow depth 3. A depth-3 caller and any
26
+ derived depth 4 lane are refused with `max_lane_depth`.
27
+
28
+ - Question ageing now derives its two-advance cap from the depth-3 lane cap, so
29
+ an unanswered worker-lane question reaches the user after two hourly hops.
30
+
5
31
  ## [5.132.0] — 2026-09-22
6
32
 
7
33
  ### Added
@@ -104,6 +130,16 @@
104
130
 
105
131
  ## [Unreleased]
106
132
 
133
+ - `hq mesh daemon install` and `hq mesh daemon run` unload and remove the
134
+ legacy `ai.getindigo.hq-work-mesh-listen` unit (LaunchAgent, systemd user
135
+ unit, or Scheduled Task) so only the daemon subscribes. The daemon writes
136
+ `~/.hq/work-mesh/live-cache.json` and bumps its cursor once per project-view
137
+ push. MQTT reconnect logs once per transition into reconnecting instead of
138
+ on every close. `hq mesh daemon doctor` reports "legacy listener still
139
+ installed" while that unit remains (US-006).
140
+
141
+ ## [Unreleased]
142
+
107
143
  - Held-event repair no longer treats a later event timestamp as proof that
108
144
  metadata belongs to the current session instance. A reused session id stays
109
145
  unresolved when the held event's binding episode does not match the current
@@ -7,14 +7,14 @@
7
7
  import { Command } from "commander";
8
8
  import { type OutpostDispatchResult } from "../lib/lanes/outpost-dispatch.js";
9
9
  import { emitLaneDecisionEvents, type LaneMesh } from "../lib/lanes/mesh-events.js";
10
- import { type LaneCompanyRef, type LaneRecord, type QuestionRecord, type ResolvedLaneWorker, type SeniorRef, type LivenessDeps, type MachineIdentity, type RunCommand, type ThrottleDeps } from "../lib/lanes/index.js";
10
+ import { MAX_LANE_DEPTH, type LaneCompanyRef, type LaneRecord, type QuestionRecord, type ResolvedLaneWorker, type SeniorRef, type LivenessDeps, type MachineIdentity, type RunCommand, type ThrottleDeps } from "../lib/lanes/index.js";
11
11
  import { questionBodyAbsPath } from "../lib/lanes/questions.js";
12
12
  import { type RunCompactionFn } from "../lib/lanes/compaction.js";
13
13
  import { type KnownModelsDeps, type ProbeCommand } from "../lib/lanes/models.js";
14
14
  import { spawnLaneWorker, type KillProcessFn, type ProveDetachedFn, type SpawnProcessFn } from "../lib/lanes/spawn.js";
15
15
  export { spawnLaneWorker };
16
16
  export { questionBodyAbsPath };
17
- export declare const MAX_LANE_DEPTH = 2;
17
+ export { MAX_LANE_DEPTH };
18
18
  export declare const OUTPOST_ID_PATH = "/etc/outpost/outpost-id";
19
19
  export declare const OUTPOST_DIR = "/etc/outpost";
20
20
  export declare const LANES_RUNS_REL: string;
@@ -17,7 +17,7 @@ import { vaultApiFetch } from "../utils/vault-api.js";
17
17
  import { openMeshTransport } from "../lib/mesh/client.js";
18
18
  import { createLaneMesh, emitLaneDecisionEvents, } from "../lib/lanes/mesh-events.js";
19
19
  import { sessionIdFromEnv } from "../lib/mesh/live/session-identity.js";
20
- import { AccountResolutionError, DEFAULT_ACCOUNT_HOMES, LANE_PROVIDERS, LANE_STATES, PROVIDER_HOME_ENV, QUESTION_STATUSES, SENIOR_KINDS, accountMaxRunning, accountStatus, appendCappedHistory, canonicalizeHome, createLane, decideAdmission, defaultLivenessDeps, defaultSystemdUserDir, defaultThrottleDeps, installDrainHook, installReconcileTimer, providerPathEnv, probeDrainHookRegistration, isLanesError, isLaneVersionConflict, isTerminalLaneState, listAccountStatuses, listLaneIds, listQuestionIds, listSessionLiveness, loadLanesConfig, resolveMachineIdentity as lookupMachineIdentity, loadLivenessIndex, parseLaneProvider, parseLaneState, parseLivenessState, parseQuestionStatus, parseSeniorKind, readDaemonPresenceConnected, readLane, retryUpdateLane, readQuestion, resolveAccount, resolveLaneWorker, isLoggedOut, runReconcilePass, sessionLiveness, tryReadLane, uninstallReconcileTimer, updateLane, updateQuestion, writeLanesConfig, appendDropboxMessage, answerDropboxMessage, deliveryRefusalReason, resumeCommandRefusal, dropboxDir, dropboxInboxPath, laneAcceptsDropboxDelivery, laneAcceptsOperatorMessage, laneDeliveryMechanism, hasRecordedSessionId, providerSupportsSessionResume, laneIdForAnswerDelivery, operatorMessageRefusalReason, raiseLaneQuestion, sha256Text, writeQuestionBody, } from "../lib/lanes/index.js";
20
+ import { AccountResolutionError, DEFAULT_ACCOUNT_HOMES, LANE_PROVIDERS, LANE_STATES, PROVIDER_HOME_ENV, QUESTION_STATUSES, SENIOR_KINDS, accountMaxRunning, accountStatus, appendCappedHistory, canonicalizeHome, createLane, decideAdmission, defaultLivenessDeps, defaultSystemdUserDir, defaultThrottleDeps, installDrainHook, installReconcileTimer, providerPathEnv, probeDrainHookRegistration, isLanesError, isLaneVersionConflict, isTerminalLaneState, listAccountStatuses, listLaneIds, listQuestionIds, listSessionLiveness, loadLanesConfig, MAX_LANE_DEPTH, resolveMachineIdentity as lookupMachineIdentity, loadLivenessIndex, parseLaneProvider, parseLaneState, parseLivenessState, parseQuestionStatus, parseSeniorKind, readDaemonPresenceConnected, readLane, retryUpdateLane, readQuestion, resolveAccount, resolveLaneWorker, isLoggedOut, runReconcilePass, sessionLiveness, tryReadLane, uninstallReconcileTimer, updateLane, updateQuestion, writeLanesConfig, appendDropboxMessage, answerDropboxMessage, deliveryRefusalReason, resumeCommandRefusal, dropboxDir, dropboxInboxPath, laneAcceptsDropboxDelivery, laneAcceptsOperatorMessage, laneDeliveryMechanism, hasRecordedSessionId, providerSupportsSessionResume, laneIdForAnswerDelivery, operatorMessageRefusalReason, raiseLaneQuestion, sha256Text, writeQuestionBody, } from "../lib/lanes/index.js";
21
21
  import { questionBodyAbsPath } from "../lib/lanes/questions.js";
22
22
  import { compactionAfterUsage, } from "../lib/lanes/compaction.js";
23
23
  import { providerCompactAfterMs } from "../lib/lanes/config.js";
@@ -29,7 +29,7 @@ import { defaultProbe, validateLaneModel, } from "../lib/lanes/models.js";
29
29
  import { parseCliModel, refuseOutpostSelfTarget, resolveLaneModel, FOREGROUND_ONLY_RULE, STANDING_AUTHORIZATION, spawnLaneWorker, stopLaneProcesses, } from "../lib/lanes/spawn.js";
30
30
  export { spawnLaneWorker };
31
31
  export { questionBodyAbsPath };
32
- export const MAX_LANE_DEPTH = 2;
32
+ export { MAX_LANE_DEPTH };
33
33
  export const OUTPOST_ID_PATH = "/etc/outpost/outpost-id";
34
34
  export const OUTPOST_DIR = "/etc/outpost";
35
35
  export const LANES_RUNS_REL = path.join("workspace", "lanes-runs");
@@ -1092,6 +1092,7 @@ export function pickupFailureUpdater(reasonRaw, details) {
1092
1092
  state: "failed",
1093
1093
  queued_reason: undefined,
1094
1094
  pickup_validation_pending: undefined,
1095
+ pickup_validation_pending_since: undefined,
1095
1096
  terminal_reason: reason,
1096
1097
  terminal_details: details,
1097
1098
  };
@@ -1166,7 +1167,12 @@ export function applyOutpostAccountPickup(hqRoot, laneId, modelDeps = {}) {
1166
1167
  if (current.account === resolved.id && current.pickup_validation_pending === undefined) {
1167
1168
  return current;
1168
1169
  }
1169
- return { ...current, account: resolved.id, pickup_validation_pending: undefined };
1170
+ return {
1171
+ ...current,
1172
+ account: resolved.id,
1173
+ pickup_validation_pending: undefined,
1174
+ pickup_validation_pending_since: undefined,
1175
+ };
1170
1176
  });
1171
1177
  }
1172
1178
  catch (err) {
@@ -1315,8 +1321,8 @@ async function runCreate(ctx, opts) {
1315
1321
  // creates wait on. A pinned create checks the pinned account's home. An
1316
1322
  // unpinned create is not bound to an account here (persist re-resolves the
1317
1323
  // balanced account under the lock, so overlapping creates still spread),
1318
- // so it checks the balanced candidate first and then every other enabled
1319
- // account the locked selection could land on (codex catalogs are per home
1324
+ // so it checks the balanced candidate first and then every other enabled,
1325
+ // logged-in account the locked selection could land on (codex catalogs are per home
1320
1326
  // and can differ). `validatedAccounts` records which accepted the model
1321
1327
  // or had no readable list, and the balanced pick under the lock is
1322
1328
  // restricted to that set (resolveAccount's allowedAccounts): an account
@@ -1344,26 +1350,68 @@ async function runCreate(ctx, opts) {
1344
1350
  : [
1345
1351
  candidate,
1346
1352
  ...Object.keys(accounts)
1347
- .filter((id) => id !== candidate && accounts[id]?.enabled)
1353
+ .filter((id) => id !== candidate &&
1354
+ accounts[id]?.enabled &&
1355
+ accountStatus(provider, id, config, { existsSync: ctx.existsSync }).logged_in === true)
1348
1356
  .sort(),
1349
1357
  ];
1358
+ const checkedAccounts = [];
1359
+ const unsupported = [];
1360
+ const quotaBlocked = [];
1361
+ const quota = loadQuotaState(hqRoot);
1350
1362
  for (const id of toValidate) {
1351
1363
  const home = accounts[id]?.home;
1352
1364
  if (home === undefined)
1353
1365
  continue;
1366
+ checkedAccounts.push(id);
1354
1367
  try {
1355
1368
  validateLaneModel(provider, cliModel, home, modelDeps);
1356
- validatedAccounts.add(id);
1357
1369
  }
1358
1370
  catch (err) {
1359
1371
  if (!(isLanesError(err) && err.error === "model_unsupported"))
1360
1372
  throw err;
1361
- // The account that would run the lane refuses it: refuse now, before
1362
- // the other lists are read. Another account's refusal only drops it
1363
- // from the locked selection.
1364
- if (id === candidate)
1365
- throw err;
1373
+ unsupported.push({ account: id, error: err });
1374
+ continue;
1375
+ }
1376
+ const block = quotaBlockFor(quota, provider, id, cliModel, now);
1377
+ if (block.blocked) {
1378
+ quotaBlocked.push({ account: id, block });
1379
+ continue;
1380
+ }
1381
+ validatedAccounts.add(id);
1382
+ }
1383
+ if (validatedAccounts.size === 0 && quotaBlocked.length > 0) {
1384
+ const blocked = quotaBlocked[0];
1385
+ const model = blocked.block.model ?? cliModel;
1386
+ const checkedSuffix = ` Checked accounts: ${checkedAccounts.join(", ")}`;
1387
+ if (blocked.block.reason === "account_model_unavailable") {
1388
+ throw new AccountResolutionError("account_model_unavailable", `Account ${JSON.stringify(blocked.account)} cannot run model ${model ?? "(no model)"}: ${blocked.block.marker ?? "needs credits"}${checkedSuffix}`, {
1389
+ provider,
1390
+ account: blocked.account,
1391
+ model: model ?? null,
1392
+ marker: blocked.block.marker ?? null,
1393
+ checked_accounts: checkedAccounts,
1394
+ });
1366
1395
  }
1396
+ throw new AccountResolutionError("account_quota_exhausted", `Account ${JSON.stringify(blocked.account)} is out of quota${blocked.block.quota_reset_at ? ` until ${blocked.block.quota_reset_at}` : ""} for model ${model ?? "(no model)"}${checkedSuffix}`, {
1397
+ provider,
1398
+ account: blocked.account,
1399
+ model: model ?? null,
1400
+ quota_reset_at: blocked.block.quota_reset_at ?? null,
1401
+ checked_accounts: checkedAccounts,
1402
+ });
1403
+ }
1404
+ if (validatedAccounts.size === 0 && unsupported.length > 0) {
1405
+ const first = unsupported[0].error;
1406
+ throw new LanesError(first.error, `${first.message} Checked accounts: ${checkedAccounts.join(", ")}`, {
1407
+ ...first.details,
1408
+ checked_accounts: checkedAccounts,
1409
+ account_errors: unsupported.map(({ account, error }) => ({
1410
+ account,
1411
+ accepted: error.details.accepted,
1412
+ source: error.details.source,
1413
+ })),
1414
+ });
1367
1415
  }
1368
1416
  }
1369
1417
  const threadId = company
@@ -1637,7 +1685,7 @@ function parseRemotePickupJson(stdout) {
1637
1685
  }
1638
1686
  return null;
1639
1687
  }
1640
- function ingestPickupPayload(hqRoot, payload) {
1688
+ function ingestPickupPayload(hqRoot, payload, now) {
1641
1689
  const laneId = payload.lane.lane_id;
1642
1690
  if (!laneId) {
1643
1691
  throw new LanesError("outpost_payload_invalid", "Pickup payload lane.lane_id is required", {});
@@ -1646,9 +1694,20 @@ function ingestPickupPayload(hqRoot, payload) {
1646
1694
  // The lane is stored ineligible for admission until the account bind
1647
1695
  // clears the marker (see LaneRecord.pickup_validation_pending).
1648
1696
  const lane = existing === null
1649
- ? createLane(hqRoot, { ...payload.lane, pickup_validation_pending: true })
1650
- : existing.state === "queued" && existing.pickup_validation_pending === undefined
1651
- ? retryUpdateLane(hqRoot, laneId, "command", (current) => current.state === "queued" ? { ...current, pickup_validation_pending: true } : current)
1697
+ ? createLane(hqRoot, {
1698
+ ...payload.lane,
1699
+ updated_at: now.toISOString(),
1700
+ pickup_validation_pending: true,
1701
+ pickup_validation_pending_since: now.toISOString(),
1702
+ })
1703
+ : existing.state === "queued"
1704
+ ? retryUpdateLane(hqRoot, laneId, "command", (current) => current.state === "queued"
1705
+ ? {
1706
+ ...current,
1707
+ pickup_validation_pending: true,
1708
+ pickup_validation_pending_since: now.toISOString(),
1709
+ }
1710
+ : current)
1652
1711
  : existing;
1653
1712
  const dest = runDir(hqRoot, lane.lane_id);
1654
1713
  ensureDir(path.join(dest, "logs"));
@@ -1692,7 +1751,7 @@ async function runPickup(ctx, opts) {
1692
1751
  throw new LanesError("outpost_payload_invalid", `Pickup payload is not JSON: ${err instanceof Error ? err.message : String(err)}`, {});
1693
1752
  }
1694
1753
  const payload = parseOutpostPickupPayload(parsedJson);
1695
- const ingested = ingestPickupPayload(hqRoot, payload);
1754
+ const ingested = ingestPickupPayload(hqRoot, payload, ctx.now());
1696
1755
  if (ingested.host.kind === "outpost") {
1697
1756
  try {
1698
1757
  refuseOutpostSelfTarget(hqRoot, ingested, {
@@ -2373,15 +2432,26 @@ async function runReparent(ctx, laneId, opts) {
2373
2432
  throw new LanesError("demotion_with_pending_questions", "Cannot increase a lane's depth while it has pending questions", { lane_id: laneId, from_depth: oldDepth, to_depth: newDepth });
2374
2433
  }
2375
2434
  const override = { laneId, senior: nextSenior, depth: newDepth };
2435
+ const descendants = [];
2376
2436
  for (const other of loadAllLanes(hqRoot)) {
2377
2437
  if (other.lane_id === laneId)
2378
2438
  continue;
2379
2439
  if (!walkHitsLane(hqRoot, other.senior, laneId, ctx))
2380
2440
  continue;
2381
- const d = derivedLaneDepth(hqRoot, other.senior, ctx, override);
2382
- if (d > MAX_LANE_DEPTH) {
2383
- throw new LanesError("max_lane_depth", `Re-parent would put descendant ${other.lane_id} at depth ${d}`, { lane_id: other.lane_id, depth: d, max: MAX_LANE_DEPTH });
2441
+ const oldDescendantDepth = derivedLaneDepth(hqRoot, other.senior, ctx);
2442
+ const descendantDepth = derivedLaneDepth(hqRoot, other.senior, ctx, override);
2443
+ if (descendantDepth > MAX_LANE_DEPTH) {
2444
+ throw new LanesError("max_lane_depth", `Re-parent would put descendant ${other.lane_id} at depth ${descendantDepth}`, { lane_id: other.lane_id, depth: descendantDepth, max: MAX_LANE_DEPTH });
2384
2445
  }
2446
+ if (descendantDepth > oldDescendantDepth &&
2447
+ pendingQuestionsForLane(hqRoot, other.lane_id).length > 0) {
2448
+ throw new LanesError("demotion_with_pending_questions", "Cannot increase a lane's depth while it has pending questions", {
2449
+ lane_id: other.lane_id,
2450
+ from_depth: oldDescendantDepth,
2451
+ to_depth: descendantDepth,
2452
+ });
2453
+ }
2454
+ descendants.push({ laneId: other.lane_id, depth: descendantDepth });
2385
2455
  }
2386
2456
  const at = ctx.now().toISOString();
2387
2457
  const updated = casUpdateLane(hqRoot, laneId, (current) => {
@@ -2401,6 +2471,13 @@ async function runReparent(ctx, laneId, opts) {
2401
2471
  }),
2402
2472
  };
2403
2473
  });
2474
+ for (const descendant of descendants) {
2475
+ casUpdateLane(hqRoot, descendant.laneId, (current) => {
2476
+ if (current.depth === descendant.depth)
2477
+ return current;
2478
+ return { ...current, depth: descendant.depth };
2479
+ });
2480
+ }
2404
2481
  for (const question of pending) {
2405
2482
  casUpdateQuestion(hqRoot, question.question_id, (current) => {
2406
2483
  const already = current.routing_history.some((entry) => entry.reason === "reparented" && entry.at === at);
@@ -3019,6 +3096,7 @@ async function runReconcile(ctx, opts) {
3019
3096
  throttleDeps: ctx.throttleDeps,
3020
3097
  livenessDeps: livenessDepsFromCtx(ctx),
3021
3098
  presenceConnected: ctx.presenceConnected,
3099
+ mesh: ctx.mesh,
3022
3100
  env: ctx.env,
3023
3101
  });
3024
3102
  printJson(summary);
@@ -1545,7 +1545,10 @@ export function registerMeshCommand(program) {
1545
1545
  }
1546
1546
  async function runDaemonRun(opts = {}) {
1547
1547
  const hqRoot = opts.hqRoot?.trim() || undefined;
1548
- const handle = await runMeshDaemon(hqRoot ? { hqRoot } : {});
1548
+ const handle = await runMeshDaemon({
1549
+ ...(hqRoot ? { hqRoot } : {}),
1550
+ enableWorkCache: true,
1551
+ });
1549
1552
  // Stay alive until signal handlers call stop + exit.
1550
1553
  await new Promise(() => {
1551
1554
  void handle;
@@ -0,0 +1,3 @@
1
+ /** Maximum number of lane levels below the user. */
2
+ export declare const MAX_LANE_DEPTH = 3;
3
+ //# sourceMappingURL=constants.d.ts.map
@@ -0,0 +1,3 @@
1
+ /** Maximum number of lane levels below the user. */
2
+ export const MAX_LANE_DEPTH = 3;
3
+ //# sourceMappingURL=constants.js.map
@@ -2,6 +2,7 @@
2
2
  * Library layer for `hq lanes`. The command module should import from here.
3
3
  */
4
4
  export { AccountResolutionError, LaneNotFoundError, LaneSchemaError, LaneVersionConflict, LanesConfigError, LanesError, WorkerResolutionError, isLaneVersionConflict, isLanesError, } from "./errors.js";
5
+ export { MAX_LANE_DEPTH } from "./constants.js";
5
6
  export { CONFIG_SCHEMA_VERSION, ENVELOPE_DECISIONS, ENVELOPE_SCHEMA_VERSION, HISTORY_CAP, HOST_KINDS, LANE_PROVIDERS, LANE_SCHEMA_VERSION, LANE_STATES, LIVENESS_STATES, QUESTION_BODY_STORES, QUESTION_SCHEMA_VERSION, QUESTION_STATUSES, QUEUED_REASONS, RAISED_VIA, ROUTING_REASONS, SENIOR_KINDS, STORY_STATUSES, TERMINAL_LANE_STATES, isEnvelopeDecision, isHostKind, isLaneProvider, isLaneState, isLivenessState, isQuestionStatus, isSeniorKind, isTerminalLaneState, parseLaneEnvelope, parseLaneProvider, parseLaneState, parseLivenessState, parseMajorSchema, parseQuestionStatus, parseSeniorKind, } from "./types.js";
6
7
  export type { AccountConfig, AttributionHistoryEntry, EnvelopeDecision, EnvelopeQuestion, HostKind, LaneCompanyRef, LaneEnvelope, LaneHost, LaneProvider, LaneRecord, PendingResume, AccountHistoryEntry, LaneState, LanesConfig, LastEnvelopeRef, LivenessState, LivenessWindows, ProviderConfig, QuestionAnswer, QuestionBodyRef, QuestionRecord, QuestionStatus, QueuedReason, RaisedVia, RoutingHistoryEntry, RoutingReason, SeniorHistoryEntry, SeniorKind, SeniorRef, StoryStatus, TerminalLaneState, ThrottleConfig, } from "./types.js";
7
8
  export { DEFAULT_ACCOUNT_HOMES, DEFAULT_COMPACT_AFTER_MS, DEFAULT_CPU_BUSY_PAUSE, DEFAULT_CPU_BUSY_RESUME, DEFAULT_LIVE_WITHIN_MS, DEFAULT_LIVENESS_WINDOWS, DEFAULT_MAX_QUEUED, DEFAULT_MIN_AVAILABLE_MEM_MB, DEFAULT_PROVIDER_MAX_RUNNING, DEFAULT_SILENT_LOST_MS, DEFAULT_TURN_START_LOST_MS, LANES_CONFIG_REL, accountMaxRunning, canonicalizeHome, documentedDefaultConfig, expandHome, lanesConfigPath, loadLanesConfig, parseLanesConfig, providerCompactAfterMs, providerQuotaDefaultHoldMs, DEFAULT_QUOTA_HOLD_MS, writeLanesConfig, } from "./config.js";
@@ -2,6 +2,7 @@
2
2
  * Library layer for `hq lanes`. The command module should import from here.
3
3
  */
4
4
  export { AccountResolutionError, LaneNotFoundError, LaneSchemaError, LaneVersionConflict, LanesConfigError, LanesError, WorkerResolutionError, isLaneVersionConflict, isLanesError, } from "./errors.js";
5
+ export { MAX_LANE_DEPTH } from "./constants.js";
5
6
  export { CONFIG_SCHEMA_VERSION, ENVELOPE_DECISIONS, ENVELOPE_SCHEMA_VERSION, HISTORY_CAP, HOST_KINDS, LANE_PROVIDERS, LANE_SCHEMA_VERSION, LANE_STATES, LIVENESS_STATES, QUESTION_BODY_STORES, QUESTION_SCHEMA_VERSION, QUESTION_STATUSES, QUEUED_REASONS, RAISED_VIA, ROUTING_REASONS, SENIOR_KINDS, STORY_STATUSES, TERMINAL_LANE_STATES, isEnvelopeDecision, isHostKind, isLaneProvider, isLaneState, isLivenessState, isQuestionStatus, isSeniorKind, isTerminalLaneState, parseLaneEnvelope, parseLaneProvider, parseLaneState, parseLivenessState, parseMajorSchema, parseQuestionStatus, parseSeniorKind, } from "./types.js";
6
7
  export { DEFAULT_ACCOUNT_HOMES, DEFAULT_COMPACT_AFTER_MS, DEFAULT_CPU_BUSY_PAUSE, DEFAULT_CPU_BUSY_RESUME, DEFAULT_LIVE_WITHIN_MS, DEFAULT_LIVENESS_WINDOWS, DEFAULT_MAX_QUEUED, DEFAULT_MIN_AVAILABLE_MEM_MB, DEFAULT_PROVIDER_MAX_RUNNING, DEFAULT_SILENT_LOST_MS, DEFAULT_TURN_START_LOST_MS, LANES_CONFIG_REL, accountMaxRunning, canonicalizeHome, documentedDefaultConfig, expandHome, lanesConfigPath, loadLanesConfig, parseLanesConfig, providerCompactAfterMs, providerQuotaDefaultHoldMs, DEFAULT_QUOTA_HOLD_MS, writeLanesConfig, } from "./config.js";
7
8
  export { resolveLanesHqRoot } from "./hq-root.js";
@@ -64,9 +64,12 @@ export declare function applyDefaultQuotaHold(signal: QuotaSignal, defaultHoldMs
64
64
  export declare function quotaStatePath(hqRoot: string): string;
65
65
  export declare function loadQuotaState(hqRoot: string): QuotaState;
66
66
  /**
67
- * Parse "resets Sep 24, 2am (UTC)" into an ISO timestamp. The year is the
68
- * current UTC year, rolled forward when that instant is already past.
69
- * Returns null when the text names no reset.
67
+ * Parse provider reset times such as "resets Sep 24, 2am (UTC)" and Codex's
68
+ * "try again at Sep 14th, 2026 9:24 PM" into an ISO timestamp. Codex's
69
+ * unzoned form is displayed in the provider CLI's local timezone, while the
70
+ * provider `resets` form is UTC when it names `(UTC)`. The year is the current
71
+ * UTC year for an unyear-tagged reset, rolled forward when it is already past.
72
+ * An explicit provider year is kept as stated.
70
73
  */
71
74
  export declare function parseProviderResetTime(text: string, now: Date): string | null;
72
75
  /**
@@ -136,26 +136,36 @@ function writeQuotaState(hqRoot, state) {
136
136
  fs.renameSync(tmp, filePath);
137
137
  }
138
138
  /**
139
- * Parse "resets Sep 24, 2am (UTC)" into an ISO timestamp. The year is the
140
- * current UTC year, rolled forward when that instant is already past.
141
- * Returns null when the text names no reset.
139
+ * Parse provider reset times such as "resets Sep 24, 2am (UTC)" and Codex's
140
+ * "try again at Sep 14th, 2026 9:24 PM" into an ISO timestamp. Codex's
141
+ * unzoned form is displayed in the provider CLI's local timezone, while the
142
+ * provider `resets` form is UTC when it names `(UTC)`. The year is the current
143
+ * UTC year for an unyear-tagged reset, rolled forward when it is already past.
144
+ * An explicit provider year is kept as stated.
142
145
  */
143
146
  export function parseProviderResetTime(text, now) {
144
- const match = /resets\s+([A-Za-z]+)\s+(\d{1,2})(?:,\s*(\d{1,2})(?::(\d{2}))?\s*(am|pm))?(?:\s*\(UTC\))?/i.exec(text);
147
+ const resetMatch = /\bresets\s+([A-Za-z]+)\s+(\d{1,2})(?:st|nd|rd|th)?(?:,\s*(?:(\d{4})\s+)?(\d{1,2})(?::(\d{2}))?\s*(am|pm))?(?:\s*\((UTC)\))?/i.exec(text);
148
+ const codexMatch = /\btry\s+again\s+at\s+([A-Za-z]+)\s+(\d{1,2})(?:st|nd|rd|th)?,\s*(\d{4})\s+(\d{1,2})(?::(\d{2}))?\s*(am|pm)(?:\s*\((UTC)\))?/i.exec(text);
149
+ const match = resetMatch ?? codexMatch;
145
150
  if (!match)
146
151
  return null;
152
+ const codexLocalTime = codexMatch !== null && (match[7] ?? "").toLowerCase() !== "utc";
147
153
  const month = MONTHS[(match[1] ?? "").slice(0, 3).toLowerCase()];
148
154
  if (month === undefined)
149
155
  return null;
150
156
  const day = Number(match[2]);
151
157
  if (!Number.isInteger(day) || day < 1 || day > 31)
152
158
  return null;
159
+ const explicitYear = match[3] !== undefined;
160
+ const parsedYear = explicitYear ? Number(match[3]) : now.getUTCFullYear();
161
+ if (!Number.isInteger(parsedYear) || parsedYear < 1970)
162
+ return null;
153
163
  let hour = 0;
154
164
  let minute = 0;
155
- if (match[3] !== undefined) {
156
- hour = Number(match[3]);
157
- minute = match[4] !== undefined ? Number(match[4]) : 0;
158
- const ap = (match[5] ?? "").toLowerCase();
165
+ if (match[4] !== undefined) {
166
+ hour = Number(match[4]);
167
+ minute = match[5] !== undefined ? Number(match[5]) : 0;
168
+ const ap = (match[6] ?? "").toLowerCase();
159
169
  if (ap === "pm" && hour < 12)
160
170
  hour += 12;
161
171
  if (ap === "am" && hour === 12)
@@ -163,11 +173,32 @@ export function parseProviderResetTime(text, now) {
163
173
  if (hour > 23 || minute > 59)
164
174
  return null;
165
175
  }
166
- let year = now.getUTCFullYear();
167
- let ts = Date.UTC(year, month, day, hour, minute, 0, 0);
168
- if (ts < now.getTime() - 36 * 60 * 60 * 1000) {
176
+ const timestampForYear = (targetYear) => {
177
+ const candidate = codexLocalTime
178
+ ? new Date(targetYear, month, day, hour, minute, 0, 0)
179
+ : new Date(Date.UTC(targetYear, month, day, hour, minute, 0, 0));
180
+ const valid = codexLocalTime
181
+ ? candidate.getFullYear() === targetYear &&
182
+ candidate.getMonth() === month &&
183
+ candidate.getDate() === day &&
184
+ candidate.getHours() === hour &&
185
+ candidate.getMinutes() === minute
186
+ : candidate.getUTCFullYear() === targetYear &&
187
+ candidate.getUTCMonth() === month &&
188
+ candidate.getUTCDate() === day &&
189
+ candidate.getUTCHours() === hour &&
190
+ candidate.getUTCMinutes() === minute;
191
+ return valid ? candidate.getTime() : null;
192
+ };
193
+ let year = parsedYear;
194
+ let ts = timestampForYear(year);
195
+ if (ts === null)
196
+ return null;
197
+ if (!explicitYear && ts < now.getTime() - 36 * 60 * 60 * 1000) {
169
198
  year += 1;
170
- ts = Date.UTC(year, month, day, hour, minute, 0, 0);
199
+ ts = timestampForYear(year);
200
+ if (ts === null)
201
+ return null;
171
202
  }
172
203
  return new Date(ts).toISOString();
173
204
  }
@@ -259,21 +290,40 @@ export function recordAccountQuota(hqRoot, provider, accountId, signal) {
259
290
  next.marker = signal.marker ?? `needs credits for ${signal.model}`;
260
291
  delete next.quota_reset_at;
261
292
  }
262
- if (signal.quota_reset_at && !signal.needs_credits) {
293
+ if (signal.quota_reset_at &&
294
+ !signal.needs_credits &&
295
+ shouldReplaceReset(next.quota_reset_at, signal.quota_reset_at)) {
263
296
  next.quota_reset_at = signal.quota_reset_at;
264
297
  }
265
298
  models[signal.model] = next;
266
299
  }
267
300
  const account = { ...current, models };
268
301
  if (signal.account_wide && signal.quota_reset_at) {
269
- account.all_models_quota_reset_at = signal.quota_reset_at;
270
- account.reset_source = signal.reset_source ?? "provider";
302
+ if (shouldReplaceReset(current.all_models_quota_reset_at, signal.quota_reset_at)) {
303
+ account.all_models_quota_reset_at = signal.quota_reset_at;
304
+ account.reset_source = signal.reset_source ?? "provider";
305
+ }
306
+ else if (current.all_models_quota_reset_at) {
307
+ account.all_models_quota_reset_at = current.all_models_quota_reset_at;
308
+ account.reset_source = current.reset_source ?? "provider";
309
+ }
271
310
  }
272
311
  accounts[accountId] = account;
273
312
  state.providers[provider] = accounts;
274
313
  writeQuotaState(hqRoot, state);
275
314
  return state;
276
315
  }
316
+ function shouldReplaceReset(current, incoming) {
317
+ if (!current)
318
+ return true;
319
+ const currentMs = Date.parse(current);
320
+ const incomingMs = Date.parse(incoming);
321
+ if (!Number.isFinite(currentMs))
322
+ return Number.isFinite(incomingMs);
323
+ if (!Number.isFinite(incomingMs))
324
+ return false;
325
+ return incomingMs > currentMs;
326
+ }
277
327
  function resetStillAhead(iso, now) {
278
328
  if (!iso)
279
329
  return false;
@@ -12,12 +12,15 @@ import type { LaneRecord, LivenessState, QuestionRecord, SeniorRef } from "./typ
12
12
  import { type LaneMesh } from "./mesh-events.js";
13
13
  /** Spec §9: unanswered at the current hop for 60 minutes → one hop. */
14
14
  export declare const QUESTION_AGE_MS: number;
15
- /** Spec §6.6 + §9: v1 permits exactly one aged advance. */
16
- export declare const MAX_AGED_ADVANCES = 1;
15
+ /** Spec §6.6 + §9: a depth-3 worker has two hops to age toward the user. */
16
+ export declare const MAX_AGED_ADVANCES: number;
17
17
  /** Aligns with watcher silence: a host that has not reported for this long is dead. */
18
18
  export declare const DEAD_HOST_MS: number;
19
19
  export declare const RECONCILE_LOCK_STALE_MS = 30000;
20
20
  export declare const RECONCILE_LOCK_TIMEOUT_MS = 5000;
21
+ /** A pickup probe that has not cleared its marker within this window is abandoned. */
22
+ export declare const PICKUP_VALIDATION_MAX_AGE_MS: number;
23
+ export declare const PICKUP_VALIDATION_TIMEOUT_REASON: "pickup_validation_timeout";
21
24
  export declare const RECONCILE_UNIT_NAME = "hq-lanes-reconcile.service";
22
25
  export declare const RECONCILE_TIMER_NAME = "hq-lanes-reconcile.timer";
23
26
  export interface ReconcileSkip {
@@ -9,6 +9,7 @@ import * as fs from "node:fs";
9
9
  import * as path from "node:path";
10
10
  import { parseUserYaml } from "../../utils/user-yaml-error.js";
11
11
  import { accountMaxRunning, loadLanesConfig, providerQuotaDefaultHoldMs } from "./config.js";
12
+ import { MAX_LANE_DEPTH } from "./constants.js";
12
13
  import { LanesError, isLaneVersionConflict } from "./errors.js";
13
14
  import { classifySessionLiveness, defaultLivenessDeps, loadLivenessIndex, readDaemonPresenceConnected, } from "./liveness.js";
14
15
  import { ANSWERED_QUESTIONS_FILE, armLaneWatcher, isPidRecordAlive, laneRunDir, readPidRecord, readSpawnManifest, spawnEnvelopeRepair, spawnLaneWorker, } from "./spawn.js";
@@ -22,12 +23,15 @@ import { decideEnvelopeRepair, envelopeQuestionIds, routeProviderFailure } from
22
23
  import { providerRejectionDetails, readProviderRejection, } from "./provider-rejection.js";
23
24
  /** Spec §9: unanswered at the current hop for 60 minutes → one hop. */
24
25
  export const QUESTION_AGE_MS = 60 * 60 * 1000;
25
- /** Spec §6.6 + §9: v1 permits exactly one aged advance. */
26
- export const MAX_AGED_ADVANCES = 1;
26
+ /** Spec §6.6 + §9: a depth-3 worker has two hops to age toward the user. */
27
+ export const MAX_AGED_ADVANCES = MAX_LANE_DEPTH - 1;
27
28
  /** Aligns with watcher silence: a host that has not reported for this long is dead. */
28
29
  export const DEAD_HOST_MS = SILENCE_MS;
29
30
  export const RECONCILE_LOCK_STALE_MS = 30_000;
30
31
  export const RECONCILE_LOCK_TIMEOUT_MS = 5_000;
32
+ /** A pickup probe that has not cleared its marker within this window is abandoned. */
33
+ export const PICKUP_VALIDATION_MAX_AGE_MS = 15 * 60 * 1000;
34
+ export const PICKUP_VALIDATION_TIMEOUT_REASON = "pickup_validation_timeout";
31
35
  export const RECONCILE_UNIT_NAME = "hq-lanes-reconcile.service";
32
36
  export const RECONCILE_TIMER_NAME = "hq-lanes-reconcile.timer";
33
37
  function logSkip(skip) {
@@ -309,6 +313,89 @@ function casQuestionIf(hqRoot, questionId, shouldWrite, updater, dryRun) {
309
313
  }
310
314
  throw new LanesError("version_conflict", `Exhausted CAS retries for question ${questionId}`, { question_id: questionId });
311
315
  }
316
+ async function emitAgedQuestionAtUser(summary, deps, lane, question) {
317
+ if (deps.dryRun)
318
+ return;
319
+ const mesh = deps.mesh === undefined ? createLaneMesh({ interactive: false }) : deps.mesh;
320
+ if (!mesh || !lane.company || !lane.thread_id)
321
+ return;
322
+ const timeoutMs = deps.meshEmitTimeoutMs ?? 10_000;
323
+ let timer;
324
+ try {
325
+ await Promise.race([
326
+ mesh.appendThreadEvent(lane.company, lane.thread_id, "lane_question_at_user", { lane_id: lane.lane_id, question_id: question.question_id }),
327
+ new Promise((_resolve, reject) => {
328
+ timer = setTimeout(() => reject(new Error(`question-at-user mesh emit timed out after ${timeoutMs}ms`)), timeoutMs);
329
+ }),
330
+ ]);
331
+ }
332
+ catch (err) {
333
+ markDirty(summary, {
334
+ kind: "question",
335
+ id: question.question_id,
336
+ why: `question-at-user mesh emit failed: ${err instanceof Error ? err.message : String(err)}`,
337
+ });
338
+ }
339
+ finally {
340
+ if (timer)
341
+ clearTimeout(timer);
342
+ }
343
+ }
344
+ function pickupValidationPendingAtMs(lane) {
345
+ const raw = lane.pickup_validation_pending_since ?? lane.updated_at;
346
+ const at = Date.parse(raw);
347
+ return Number.isFinite(at) ? at : null;
348
+ }
349
+ function pickupValidationExpired(lane, nowMs) {
350
+ const pendingAtMs = pickupValidationPendingAtMs(lane);
351
+ return pendingAtMs === null || nowMs - pendingAtMs > PICKUP_VALIDATION_MAX_AGE_MS;
352
+ }
353
+ function expireAbandonedPickupValidation(hqRoot, lanes, summary, deps, now) {
354
+ const nowMs = now.getTime();
355
+ for (const lane of lanes) {
356
+ if (lane.state !== "queued" ||
357
+ lane.pickup_validation_pending !== true ||
358
+ !laneRunsHere(lane, deps.host) ||
359
+ !pickupValidationExpired(lane, nowMs)) {
360
+ continue;
361
+ }
362
+ try {
363
+ const result = casLaneIf(hqRoot, lane.lane_id, (current) => current.state === "queued" &&
364
+ current.pickup_validation_pending === true &&
365
+ pickupValidationExpired(current, nowMs), (current) => {
366
+ const pendingAtMs = pickupValidationPendingAtMs(current);
367
+ return {
368
+ ...current,
369
+ state: "failed",
370
+ queued_reason: undefined,
371
+ pickup_validation_pending: undefined,
372
+ pickup_validation_pending_since: undefined,
373
+ terminal_reason: PICKUP_VALIDATION_TIMEOUT_REASON,
374
+ terminal_details: {
375
+ pickup_validation_pending_since: current.pickup_validation_pending_since ?? current.updated_at,
376
+ age_ms: pendingAtMs === null ? null : Math.max(0, nowMs - pendingAtMs),
377
+ max_age_ms: PICKUP_VALIDATION_MAX_AGE_MS,
378
+ },
379
+ };
380
+ }, deps.dryRun);
381
+ if (result.changed) {
382
+ summary.aged.push({
383
+ id: lane.lane_id,
384
+ from: "queued",
385
+ to: "failed",
386
+ reason: PICKUP_VALIDATION_TIMEOUT_REASON,
387
+ });
388
+ }
389
+ }
390
+ catch (err) {
391
+ markDirty(summary, {
392
+ kind: "lane",
393
+ id: lane.lane_id,
394
+ why: `pickup validation expiry failed: ${err instanceof Error ? err.message : String(err)}`,
395
+ });
396
+ }
397
+ }
398
+ }
312
399
  function fileNonEmpty(filePath) {
313
400
  try {
314
401
  return fs.statSync(filePath).size > 0;
@@ -589,6 +676,10 @@ export async function runPassBody(hqRoot, deps) {
589
676
  to: String(nextHop),
590
677
  reason: "unanswered_timeout",
591
678
  });
679
+ if (nextAudience.kind === "user") {
680
+ const originLane = laneById.get(question.lane_id) ?? readLane(hqRoot, question.lane_id);
681
+ await emitAgedQuestionAtUser(summary, deps, originLane, result.record);
682
+ }
592
683
  }
593
684
  }
594
685
  }
@@ -862,6 +953,7 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
862
953
  }
863
954
  }
864
955
  async function admitQueued(hqRoot, lanes, summary, deps, now) {
956
+ expireAbandonedPickupValidation(hqRoot, lanes, summary, deps, now);
865
957
  // A lane whose pickup probe is still out is not admitted: its record is
866
958
  // about to be bound or failed by the pickup, and a worker spawned now
867
959
  // would keep running behind that write.
@@ -257,6 +257,8 @@ export interface LaneRecord {
257
257
  * leave a running worker behind a record the probe result then rewrites.
258
258
  */
259
259
  pickup_validation_pending?: true;
260
+ /** When the pickup validation marker was written. */
261
+ pickup_validation_pending_since?: string;
260
262
  priority: number;
261
263
  worker?: string;
262
264
  /** Registry-relative directory of the resolved worker, when --worker was set. */