@indigoai-us/hq-cli 5.154.0 → 5.156.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -42,7 +42,7 @@
42
42
  * of opening a browser nobody is watching, and the watcher records
43
43
  * `mesh_event_failed` and moves on. No credential file is read here.
44
44
  */
45
- import { appendThreadEvent, clamp, ensureProjectThread, getProjectView, patchStoryStatus, putProjectView, requireToken, registerMeshProject, resolveMeshCompany, } from "../mesh/api.js";
45
+ import { appendThreadEvent, clamp, ensureLaneMeshThread, ensureProjectThread, getProjectView, patchStoryStatus, putProjectView, requireToken, registerMeshProject, resolveMeshCompany, } from "../mesh/api.js";
46
46
  import { LanesError } from "./errors.js";
47
47
  export const LANE_MESH_EVENTS = [
48
48
  "lane_started",
@@ -52,12 +52,17 @@ export const LANE_MESH_EVENTS = [
52
52
  "lane_ask",
53
53
  "lane_blocked",
54
54
  "lane_done",
55
+ "lane_failed",
56
+ "lane_interrupted",
57
+ "lane_cancelled",
58
+ "lane_killed",
55
59
  ];
56
60
  /** Roster kinds this module emits. Subset of the server's `validKinds`. */
57
61
  export const WIRE_EVENT_KINDS = [
58
62
  "run_started",
59
63
  "run_progress",
60
64
  "run_complete",
65
+ "blocked",
61
66
  "question",
62
67
  "note",
63
68
  ];
@@ -94,6 +99,30 @@ function nonEmpty(text, fallback) {
94
99
  export function toWireThreadEvent(event, payload) {
95
100
  const kind = parseLaneMeshEvent(event);
96
101
  const id = identity(payload);
102
+ const laneId = str(payload.lane_id ?? payload.laneId).trim();
103
+ const senior = payload.senior && typeof payload.senior === "object" && !Array.isArray(payload.senior)
104
+ ? payload.senior
105
+ : undefined;
106
+ const lane = laneId
107
+ ? {
108
+ laneId,
109
+ ...(str(payload.story_id ?? payload.storyId) ? { storyId: str(payload.story_id ?? payload.storyId) } : {}),
110
+ ...(str(payload.worker_profile ?? payload.workerProfile) ? { workerProfile: str(payload.worker_profile ?? payload.workerProfile) } : {}),
111
+ ...(str(payload.model) ? { model: str(payload.model) } : {}),
112
+ ...(str(payload.account) ? { account: str(payload.account) } : {}),
113
+ ...(str(payload.parent_lane_id ?? payload.parentLaneId) ? { parentLaneId: str(payload.parent_lane_id ?? payload.parentLaneId) } : {}),
114
+ ...(senior && ["user", "session", "lane"].includes(String(senior.kind))
115
+ ? { senior: { kind: senior.kind, ...(typeof senior.id === "string" ? { id: senior.id } : {}) } }
116
+ : {}),
117
+ laneVersion: Number.isSafeInteger(Number(payload.lane_version ?? payload.laneVersion)) && Number(payload.lane_version ?? payload.laneVersion) > 0
118
+ ? Number(payload.lane_version ?? payload.laneVersion)
119
+ : 1,
120
+ }
121
+ : undefined;
122
+ const laneFields = (laneStatus) => ({
123
+ ...(lane ? { lane } : {}),
124
+ ...(laneStatus ? { laneStatus } : {}),
125
+ });
97
126
  switch (kind) {
98
127
  case "lane_started":
99
128
  return {
@@ -101,6 +130,7 @@ export function toWireThreadEvent(event, payload) {
101
130
  payload: {
102
131
  title: clamp(`lane_started ${str(payload.lane_id) || "unknown-lane"}`, WIRE_ACTIVITY_TEXT_MAX),
103
132
  summary: clamp(`Lane ${id} started`, WIRE_ACTIVITY_TEXT_MAX),
133
+ ...laneFields("running"),
104
134
  },
105
135
  };
106
136
  case "lane_progress": {
@@ -109,29 +139,47 @@ export function toWireThreadEvent(event, payload) {
109
139
  eventKind: "run_progress",
110
140
  payload: {
111
141
  summary: clamp(nonEmpty(`lane_progress ${id}${detail ? `: ${detail}` : ""}`, "lane_progress"), WIRE_ACTIVITY_TEXT_MAX),
142
+ ...laneFields("running"),
112
143
  },
113
144
  };
114
145
  }
115
146
  case "lane_done": {
116
147
  const summary = str(payload.summary);
148
+ const suppliedStatus = str(payload.lane_status ?? payload.laneStatus);
149
+ const laneStatus = isMeshLaneStatus(suppliedStatus) ? suppliedStatus : "done";
117
150
  return {
118
151
  eventKind: "run_complete",
119
152
  payload: {
120
153
  title: clamp(`lane_done ${str(payload.lane_id) || "unknown-lane"}`, WIRE_ACTIVITY_TEXT_MAX),
121
154
  summary: clamp(nonEmpty(`${id}: ${summary}`.replace(/:\s*$/, ""), `lane_done ${id}`), WIRE_ACTIVITY_TEXT_MAX),
155
+ ...laneFields(laneStatus),
122
156
  },
123
157
  };
124
158
  }
125
159
  case "lane_blocked": {
126
- const reason = str(payload.blocked_reason);
127
- // The roster's `blocked` event kind would flip the shared project
128
- // thread, which spec §8 keeps lane events away from. Use the accepted
129
- // audit note kind to preserve a filterable blocked marker.
160
+ const reason = str(payload.blocked_reason ?? payload.reason);
161
+ const suppliedStatus = str(payload.lane_status ?? payload.laneStatus);
162
+ const laneStatus = isMeshLaneStatus(suppliedStatus) ? suppliedStatus : "blocked";
130
163
  return {
131
- eventKind: "note",
164
+ eventKind: "blocked",
132
165
  payload: {
133
- text: clamp(`lane_blocked ${id}${reason ? `: ${reason}` : ""}`, WIRE_NOTE_TEXT_MAX),
134
- noteKind: "audit",
166
+ reason: clamp(`lane_blocked ${id}${reason ? `: ${reason}` : ""}`, WIRE_NOTE_TEXT_MAX),
167
+ ...laneFields(laneStatus),
168
+ },
169
+ };
170
+ }
171
+ case "lane_failed":
172
+ case "lane_interrupted":
173
+ case "lane_cancelled":
174
+ case "lane_killed": {
175
+ const laneStatus = kind.slice("lane_".length);
176
+ const summary = str(payload.summary ?? payload.reason ?? payload.blocked_reason);
177
+ return {
178
+ eventKind: "run_complete",
179
+ payload: {
180
+ title: clamp(`${kind} ${str(payload.lane_id) || "unknown-lane"}`, WIRE_ACTIVITY_TEXT_MAX),
181
+ summary: clamp(nonEmpty(`${id}: ${summary}`.replace(/:\s*$/, ""), `${kind} ${id}`), WIRE_ACTIVITY_TEXT_MAX),
182
+ ...laneFields(laneStatus),
135
183
  },
136
184
  };
137
185
  }
@@ -144,6 +192,7 @@ export function toWireThreadEvent(event, payload) {
144
192
  };
145
193
  if (questionId)
146
194
  out.questionId = questionId;
195
+ Object.assign(out, laneFields());
147
196
  return { eventKind: "question", payload: out };
148
197
  }
149
198
  case "lane_ask": {
@@ -155,6 +204,7 @@ export function toWireThreadEvent(event, payload) {
155
204
  };
156
205
  if (ids[0])
157
206
  out.questionId = ids[0];
207
+ Object.assign(out, laneFields("awaiting_input"));
158
208
  return { eventKind: "question", payload: out };
159
209
  }
160
210
  default: {
@@ -224,12 +274,28 @@ export function createLaneMesh(opts = {}) {
224
274
  return {
225
275
  async ensureProjectThread(company, projectId) {
226
276
  const tok = await token();
227
- const resolved = await resolveMeshCompany(tok, company.slug);
277
+ const resolved = await resolveMeshCompany(tok, company.uid);
228
278
  const ensured = await ensureProjectThread(tok, resolved, projectId, {
229
279
  summary: `HQ lane ${projectId}`,
230
280
  });
231
281
  return ensured.threadId;
232
282
  },
283
+ async ensureLaneThread(company, lane, laneStatus = lane.state, signal) {
284
+ const tok = await token();
285
+ const resolved = await resolveMeshCompany(tok, company.uid);
286
+ const descriptor = {
287
+ laneId: lane.lane_id,
288
+ ...(lane.story_id ? { storyId: lane.story_id } : {}),
289
+ ...(lane.worker_profile ? { workerProfile: lane.worker_profile } : {}),
290
+ ...(lane.model ? { model: lane.model } : {}),
291
+ ...(lane.account ? { account: lane.account } : {}),
292
+ ...(lane.senior.kind === "lane" ? { parentLaneId: lane.senior.id } : {}),
293
+ senior: { kind: lane.senior.kind, id: lane.senior.id },
294
+ laneVersion: lane.version,
295
+ };
296
+ const ensured = await ensureLaneMeshThread(tok, resolved, lane.project_id, descriptor, isMeshLaneStatus(laneStatus) ? laneStatus : "running", signal);
297
+ return ensured.threadId;
298
+ },
233
299
  async appendThreadEvent(company, threadId, event, payload, signal) {
234
300
  const wire = toWireThreadEvent(event, payload);
235
301
  const tok = await token();
@@ -263,34 +329,51 @@ export function laneMeshCompany(lane) {
263
329
  export function laneCanEmit(lane) {
264
330
  return laneMeshCompany(lane) !== undefined && !!lane.thread_id;
265
331
  }
266
- function baseIdentity(lane) {
332
+ export function laneMeshEventIdentity(lane) {
267
333
  const lastMove = lane.account_history?.[lane.account_history.length - 1];
268
334
  return {
269
335
  lane_id: lane.lane_id,
270
336
  project_id: lane.project_id,
271
337
  story_id: lane.story_id,
338
+ worker_profile: lane.worker_profile,
339
+ model: lane.model,
340
+ lane_version: lane.version,
341
+ senior: { kind: lane.senior.kind, id: lane.senior.id },
342
+ ...(lane.senior.kind === "lane" ? { parent_lane_id: lane.senior.id } : {}),
272
343
  ...(lane.account !== undefined ? { account: lane.account } : {}),
273
344
  // A moved lane names the move on every event its run emits (US-023 §8).
274
345
  ...(lastMove !== undefined ? { account_move: { from: lastMove.from, to: lastMove.to, at: lastMove.at } } : {}),
275
346
  };
276
347
  }
348
+ function isMeshLaneStatus(value) {
349
+ return ["queued", "running", "awaiting_input", "blocked", "done", "failed", "interrupted", "cancelled", "killed"].includes(value);
350
+ }
351
+ // hq-pro derives lane thread ids from company + lane identity; opaque ids may
352
+ // be legacy shared project threads and must be migrated before a lane closes.
353
+ const STABLE_LANE_THREAD_ID = /^lane-[0-9a-f]{32}$/;
354
+ async function ensureLaneEventThread(mesh, company, lane, laneStatus, signal) {
355
+ if (mesh.ensureLaneThread && !STABLE_LANE_THREAD_ID.test(lane.thread_id ?? "")) {
356
+ return mesh.ensureLaneThread(company, lane, laneStatus, signal);
357
+ }
358
+ return lane.thread_id;
359
+ }
277
360
  /**
278
361
  * Spec §8 terminal-envelope events. `done` → lane_done and story → review;
279
362
  * `blocked` → lane_blocked; `ask` → lane_ask with the queue ids. The story is
280
363
  * never set to `done` here: promotion belongs to the senior.
281
364
  */
282
365
  export async function emitLaneDecisionEvents(mesh, lane, envelope, opts = {}) {
283
- if (!laneCanEmit(lane))
284
- return;
285
366
  const company = laneMeshCompany(lane);
286
- const threadId = lane.thread_id;
367
+ if (!company)
368
+ return;
369
+ const threadId = await ensureLaneEventThread(mesh, company, lane, lane.state, opts.signal);
287
370
  if (!company || !threadId)
288
371
  return;
289
372
  switch (envelope.decision) {
290
373
  case "done": {
291
374
  const outcomes = await Promise.allSettled([
292
375
  mesh.appendThreadEvent(company, threadId, "lane_done", {
293
- ...baseIdentity(lane),
376
+ ...laneMeshEventIdentity(lane),
294
377
  summary: envelope.summary,
295
378
  }, opts.signal),
296
379
  mesh.patchStoryStatus(company, lane.project_id, lane.story_id, "review"),
@@ -304,7 +387,7 @@ export async function emitLaneDecisionEvents(mesh, lane, envelope, opts = {}) {
304
387
  {
305
388
  const outcomes = await Promise.allSettled([
306
389
  mesh.appendThreadEvent(company, threadId, "lane_blocked", {
307
- ...baseIdentity(lane),
390
+ ...laneMeshEventIdentity(lane),
308
391
  blocked_reason: envelope.blocked_reason ?? "",
309
392
  }),
310
393
  mesh.patchStoryStatus(company, lane.project_id, lane.story_id, "in_progress"),
@@ -322,7 +405,7 @@ export async function emitLaneDecisionEvents(mesh, lane, envelope, opts = {}) {
322
405
  for (const questionId of opts.questionIds ?? []) {
323
406
  try {
324
407
  await mesh.appendThreadEvent(company, threadId, "lane_ask", {
325
- ...baseIdentity(lane),
408
+ ...laneMeshEventIdentity(lane),
326
409
  question_ids: [questionId],
327
410
  }, opts.signal);
328
411
  }
@@ -340,15 +423,28 @@ export async function emitLaneDecisionEvents(mesh, lane, envelope, opts = {}) {
340
423
  }
341
424
  }
342
425
  }
343
- export async function emitLaneProgressEvent(mesh, lane, detail, signal) {
344
- if (!laneCanEmit(lane))
426
+ /** Emit an abnormal terminal state even when the worker produced no envelope. */
427
+ export async function emitLaneTerminalStateEvent(mesh, lane, state, summary, signal) {
428
+ const company = laneMeshCompany(lane);
429
+ if (!company)
345
430
  return;
431
+ const threadId = await ensureLaneEventThread(mesh, company, lane, state, signal);
432
+ if (!company || !threadId)
433
+ return;
434
+ await mesh.appendThreadEvent(company, threadId, `lane_${state}`, {
435
+ ...laneMeshEventIdentity(lane),
436
+ ...(summary ? { summary } : {}),
437
+ }, signal);
438
+ }
439
+ export async function emitLaneProgressEvent(mesh, lane, detail, signal) {
346
440
  const company = laneMeshCompany(lane);
347
- const threadId = lane.thread_id;
441
+ if (!company)
442
+ return;
443
+ const threadId = await ensureLaneEventThread(mesh, company, lane, lane.state, signal);
348
444
  if (!company || !threadId)
349
445
  return;
350
446
  await mesh.appendThreadEvent(company, threadId, "lane_progress", {
351
- ...baseIdentity(lane),
447
+ ...laneMeshEventIdentity(lane),
352
448
  detail,
353
449
  }, signal);
354
450
  }
@@ -37,6 +37,7 @@ export interface ProviderRejection {
37
37
  /** Which event on the stream carried the failure. */
38
38
  event: "codex.turn.failed" | "claude.result.is_error" | "grok.error";
39
39
  }
40
+ export declare function isProviderLoginFailure(failure: ProviderRejection | null): boolean;
40
41
  export declare function capMessage(message: string): string;
41
42
  /**
42
43
  * Whether a later run could succeed without the operator changing anything.
@@ -56,10 +57,9 @@ export declare function retryableFromStatus(status: number | null): boolean;
56
57
  export declare function parseCodexTurnFailed(text: string): ProviderRejection | null;
57
58
  /**
58
59
  * Claude `-p --output-format json`: the last `type: "result"` object with
59
- * `is_error: true` and a numeric `api_error_status`. A result that is an
60
- * error with no status is not a provider rejection: it is either the quota
61
- * text shape quota.ts reads, or the worker's own failure, and keeps the
62
- * existing classification.
60
+ * `is_error: true` and a numeric `api_error_status`, or the CLI's explicit
61
+ * unauthenticated message. Other status-less errors keep the existing
62
+ * classification because they may be quota text or a worker failure.
63
63
  */
64
64
  export declare function parseClaudeResultRejection(text: string): ProviderRejection | null;
65
65
  /**
@@ -24,6 +24,25 @@ export const PROVIDER_REJECTED_REASON = "provider_rejected";
24
24
  export const PROVIDER_REJECTION_MESSAGE_CAP = 2000;
25
25
  /** Status the watcher routes to the quota path instead of `provider_rejected`. */
26
26
  export const QUOTA_STATUS = 429;
27
+ /** Exact auth failures observed from each installed CLI in an empty home. */
28
+ function isProviderLoginMessage(provider, message) {
29
+ switch (provider) {
30
+ case "claude":
31
+ return /\bnot logged in\b/i.test(message);
32
+ case "codex":
33
+ return /\bnot logged in\b/i.test(message) ||
34
+ /\bmissing bearer or basic authentication in header\b/i.test(message);
35
+ case "grok":
36
+ return /^\s*not signed in\b/i.test(message);
37
+ default: {
38
+ const _exhaustive = provider;
39
+ return _exhaustive;
40
+ }
41
+ }
42
+ }
43
+ export function isProviderLoginFailure(failure) {
44
+ return failure !== null && isProviderLoginMessage(failure.provider, failure.message);
45
+ }
27
46
  function isPlainObject(value) {
28
47
  return value !== null && typeof value === "object" && !Array.isArray(value);
29
48
  }
@@ -134,10 +153,9 @@ export function parseCodexTurnFailed(text) {
134
153
  }
135
154
  /**
136
155
  * Claude `-p --output-format json`: the last `type: "result"` object with
137
- * `is_error: true` and a numeric `api_error_status`. A result that is an
138
- * error with no status is not a provider rejection: it is either the quota
139
- * text shape quota.ts reads, or the worker's own failure, and keeps the
140
- * existing classification.
156
+ * `is_error: true` and a numeric `api_error_status`, or the CLI's explicit
157
+ * unauthenticated message. Other status-less errors keep the existing
158
+ * classification because they may be quota text or a worker failure.
141
159
  */
142
160
  export function parseClaudeResultRejection(text) {
143
161
  // The last `result` decides: a success or a status-less error there is
@@ -148,9 +166,9 @@ export function parseClaudeResultRejection(text) {
148
166
  if (rec.is_error !== true)
149
167
  return NOT_REJECTED;
150
168
  const status = statusOf(rec.api_error_status);
151
- if (status === null)
152
- return NOT_REJECTED;
153
169
  const message = typeof rec.result === "string" ? rec.result : "";
170
+ if (status === null && !isProviderLoginMessage("claude", message))
171
+ return NOT_REJECTED;
154
172
  return {
155
173
  reason: PROVIDER_REJECTED_REASON,
156
174
  provider: "claude",
@@ -15,13 +15,13 @@ import { MAX_LANE_DEPTH } from "./constants.js";
15
15
  import { AccountResolutionError, LanesError, isLaneVersionConflict } from "./errors.js";
16
16
  import { loadWorkerProfile } from "./profile.js";
17
17
  import { classifySessionLiveness, defaultLivenessDeps, loadLivenessIndex, readDaemonPresenceConnected, } from "./liveness.js";
18
- import { ANSWERED_QUESTIONS_FILE, armLaneWatcher, isPidRecordAlive, laneRunDir, readPidRecord, readSpawnManifest, markSpawnFailure, spawnEnvelopeRepair, spawnLaneWorker, } from "./spawn.js";
18
+ import { ANSWERED_QUESTIONS_FILE, armLaneWatcher, isPidRecordAlive, laneRunDir, readPidRecord, readSpawnManifest, markSpawnFailure, spawnEnvelopeRepair, spawnLaneWorker, retryLaneOnAlternateAccount, } from "./spawn.js";
19
19
  import { appendCappedHistory, lanesStoreRoot, parseLaneRecord, parseQuestionRecord, readLane, readQuestion, updateLane, updateQuestion, } from "./store.js";
20
20
  import { applyThrottleDecision, decideAdmission, defaultThrottleDeps, emptyThrottleState, evaluateThrottle, isCpuCheckEnabled, readAvailableMemMb, readThrottleState, resolveCpuBusyFraction, throttleStatePath, writeThrottleState, } from "./throttle.js";
21
21
  import { applyDefaultQuotaHold, readQuotaSignal, recordAccountQuota } from "./quota.js";
22
22
  import { answeredQuestionEntries, raiseEnvelopeQuestions } from "./questions.js";
23
23
  import { SILENCE_MS, classifyLaneExit, isPidAlive, readLatestEnvelope, readLatestEnvelopeFile, readLatestEnvelopeRecord, } from "./watcher.js";
24
- import { createLaneMesh, emitLaneDecisionEvents, laneMeshCompany } from "./mesh-events.js";
24
+ import { createLaneMesh, emitLaneDecisionEvents, emitLaneTerminalStateEvent, laneMeshCompany, } from "./mesh-events.js";
25
25
  import { laneStories, storyStatusForEnvelope } from "./stories.js";
26
26
  import { decideEnvelopeRepair, envelopeQuestionIds, routeProviderFailure } from "./watcher.js";
27
27
  import { providerRejectionDetails, readProviderRejection, } from "./provider-rejection.js";
@@ -916,6 +916,11 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
916
916
  const repairedClassification = invalidEnvelope
917
917
  ? classifyLaneExit({ envelope: null, hasPartialOutput: false, invalidEnvelope })
918
918
  : classification;
919
+ const accountRetryReason = quota
920
+ ? "usage_limit"
921
+ : providerRejection && classification.reason === "account_not_logged_in"
922
+ ? "account_unavailable"
923
+ : null;
919
924
  if (!deps.dryRun &&
920
925
  quota &&
921
926
  classified.account &&
@@ -923,6 +928,30 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
923
928
  classification.reason === "account_model_unavailable")) {
924
929
  recordAccountQuota(hqRoot, classified.provider, classified.account, quota);
925
930
  }
931
+ if (!deps.dryRun && accountRetryReason && classified.account) {
932
+ try {
933
+ const recovered = await retryLaneOnAlternateAccount(hqRoot, classified, accountRetryReason, "reconcile", {
934
+ ...(deps.spawnLaneDeps ?? {}),
935
+ admissionHost: deps.host,
936
+ });
937
+ summary.dead_host.push({
938
+ id: lane.lane_id,
939
+ from: "running",
940
+ to: recovered.state,
941
+ reason: accountRetryReason,
942
+ });
943
+ summary.heartbeats.push(`${lane.lane_id}:account-retry`);
944
+ continue;
945
+ }
946
+ catch (err) {
947
+ const message = err instanceof Error ? err.message : String(err);
948
+ process.stderr.write(`hq lanes reconcile: account failover failed for ${lane.lane_id}: ${message}\n`);
949
+ if (readLane(hqRoot, lane.lane_id).state !== "running") {
950
+ summary.errors.push({ message: `${lane.lane_id}: account failover left a non-running record after ${message}` });
951
+ continue;
952
+ }
953
+ }
954
+ }
926
955
  if (!deps.dryRun &&
927
956
  classification.state === "awaiting_input" &&
928
957
  latest &&
@@ -1003,6 +1032,34 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
1003
1032
  }
1004
1033
  }
1005
1034
  }
1035
+ else if (!deps.dryRun &&
1036
+ (result.record.state === "failed" || result.record.state === "interrupted" ||
1037
+ result.record.state === "cancelled" || result.record.state === "killed")) {
1038
+ const mesh = deps.mesh === undefined ? createLaneMesh({ interactive: false }) : deps.mesh;
1039
+ if (mesh) {
1040
+ const meshEmitTimeoutMs = deps.meshEmitTimeoutMs ?? 10_000;
1041
+ let timer;
1042
+ try {
1043
+ await Promise.race([
1044
+ emitLaneTerminalStateEvent(mesh, result.record, result.record.state, result.record.terminal_reason),
1045
+ new Promise((_resolve, reject) => {
1046
+ timer = setTimeout(() => reject(new Error(`terminal mesh emit timed out after ${meshEmitTimeoutMs}ms`)), meshEmitTimeoutMs);
1047
+ }),
1048
+ ]);
1049
+ }
1050
+ catch (err) {
1051
+ markDirty(summary, {
1052
+ kind: "lane",
1053
+ id: lane.lane_id,
1054
+ why: `terminal mesh emit failed: ${err instanceof Error ? err.message : String(err)}`,
1055
+ });
1056
+ }
1057
+ finally {
1058
+ if (timer)
1059
+ clearTimeout(timer);
1060
+ }
1061
+ }
1062
+ }
1006
1063
  }
1007
1064
  }
1008
1065
  catch (err) {
@@ -1031,6 +1088,32 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
1031
1088
  to: "interrupted",
1032
1089
  reason: "host_lost",
1033
1090
  });
1091
+ const mesh = !deps.dryRun
1092
+ ? deps.mesh === undefined ? createLaneMesh({ interactive: false }) : deps.mesh
1093
+ : null;
1094
+ if (mesh) {
1095
+ const meshEmitTimeoutMs = deps.meshEmitTimeoutMs ?? 10_000;
1096
+ let timer;
1097
+ try {
1098
+ await Promise.race([
1099
+ emitLaneTerminalStateEvent(mesh, result.record, "interrupted", "host_lost"),
1100
+ new Promise((_resolve, reject) => {
1101
+ timer = setTimeout(() => reject(new Error(`terminal mesh emit timed out after ${meshEmitTimeoutMs}ms`)), meshEmitTimeoutMs);
1102
+ }),
1103
+ ]);
1104
+ }
1105
+ catch (err) {
1106
+ markDirty(summary, {
1107
+ kind: "lane",
1108
+ id: lane.lane_id,
1109
+ why: `terminal mesh emit failed: ${err instanceof Error ? err.message : String(err)}`,
1110
+ });
1111
+ }
1112
+ finally {
1113
+ if (timer)
1114
+ clearTimeout(timer);
1115
+ }
1116
+ }
1034
1117
  }
1035
1118
  }
1036
1119
  catch (err) {
@@ -12,7 +12,8 @@ import { type RunCompactionFn } from "./compaction.js";
12
12
  import { type KnownModelsDeps } from "./models.js";
13
13
  import { type LaunchTrigger } from "./delivery.js";
14
14
  import { type MachineIdentity } from "./machine-identity.js";
15
- import { type LaneCompanyRef, type LaneProvider, type LaneRecord } from "./types.js";
15
+ import { type AdmissionHost } from "./admission.js";
16
+ import { type LaneCompanyRef, type LaneProvider, type LaneRecord, type AccountHistoryEntry } from "./types.js";
16
17
  export declare const LANES_RUNS_REL: string;
17
18
  /** Written by reconcile before a resume spawn; the worker prompt names this path. */
18
19
  export declare const ANSWERED_QUESTIONS_FILE = "answered-questions.json";
@@ -32,6 +33,9 @@ export declare function resolveLaneModel(input: {
32
33
  accountModel?: string;
33
34
  providerModel?: string;
34
35
  }): string | undefined;
36
+ type AutomaticAccountChangeReason = Exclude<AccountHistoryEntry["reason"], "operator_move">;
37
+ /** Change account only after a provider/runtime account failure, then re-admit. */
38
+ export declare function retryLaneOnAlternateAccount(hqRoot: string, lane: LaneRecord, reason: Extract<AutomaticAccountChangeReason, "usage_limit" | "account_unavailable">, writer?: "watcher" | "reconcile", deps?: SpawnLaneDeps): Promise<LaneRecord>;
35
39
  export declare function appendModelFlag(provider: LaneProvider, args: string[], model: string | undefined): string[];
36
40
  /** Add Codex fast-mode flags before the prompt/positional separator. */
37
41
  export declare function appendFastModeFlags(provider: LaneProvider, args: string[], enabled: boolean | undefined): string[];
@@ -63,8 +67,12 @@ export type ProveDetachedFn = (pid: number) => DetachProof;
63
67
  export type KillProcessFn = (pid: number, signal?: NodeJS.Signals) => void;
64
68
  export interface SpawnLaneDeps {
65
69
  env?: NodeJS.ProcessEnv;
70
+ /** The caller's host scope for admission checks; reconcile supplies its resolved host. */
71
+ admissionHost?: AdmissionHost;
66
72
  /** Provider model catalog dependencies, shared with create in tests. */
67
73
  modelDeps?: KnownModelsDeps;
74
+ /** Create validated this exact account/model candidate before its admission lock. */
75
+ currentModelValidated?: boolean;
68
76
  existsSync?: (p: string) => boolean;
69
77
  spawnProcess?: SpawnProcessFn;
70
78
  proveDetached?: ProveDetachedFn;