@indigoai-us/hq-cli 5.345.55 → 5.345.57

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. package/CHANGELOG.md +18 -0
  2. package/assets/scaffold/core/scripts/derive-trigger-facts.sh +10 -1
  3. package/assets/scaffold/core/scripts/jobs-validate.sh +2 -1
  4. package/assets/scaffold/core/scripts/session-title.sh +18 -15
  5. package/dist/command-catalog.generated.d.ts +124 -0
  6. package/dist/command-catalog.generated.js +155 -0
  7. package/dist/commands/core-delegate-w1b.js +18 -5
  8. package/dist/commands/core-derive-trigger-facts-command.js +14 -0
  9. package/dist/commands/core-derive-trigger-facts-native.d.ts +1 -0
  10. package/dist/commands/core-derive-trigger-facts-native.js +20 -1
  11. package/dist/commands/doctor.d.ts +1 -0
  12. package/dist/commands/doctor.js +15 -2
  13. package/dist/commands/lanes-checks.d.ts +3 -0
  14. package/dist/commands/lanes-checks.js +34 -0
  15. package/dist/commands/lanes-loop.d.ts +12 -0
  16. package/dist/commands/lanes-loop.js +214 -0
  17. package/dist/commands/lanes-questions.js +37 -11
  18. package/dist/commands/lanes-resume.js +19 -1
  19. package/dist/commands/lanes-session-start.d.ts +2 -7
  20. package/dist/commands/lanes-session-start.js +9 -29
  21. package/dist/commands/lanes-shared.d.ts +14 -2
  22. package/dist/commands/lanes.d.ts +3 -1
  23. package/dist/commands/lanes.js +376 -35
  24. package/dist/commands/members.d.ts +4 -0
  25. package/dist/commands/members.js +23 -2
  26. package/dist/lanes-drain-hook.js +6 -2
  27. package/dist/lib/doctor/checks/sync-health.d.ts +2 -0
  28. package/dist/lib/doctor/checks/sync-health.js +47 -23
  29. package/dist/lib/lanes/delivery.d.ts +3 -0
  30. package/dist/lib/lanes/delivery.js +5 -0
  31. package/dist/lib/lanes/dropbox.d.ts +7 -2
  32. package/dist/lib/lanes/dropbox.js +157 -10
  33. package/dist/lib/lanes/flag-keys.d.ts +12 -0
  34. package/dist/lib/lanes/flag-keys.js +12 -0
  35. package/dist/lib/lanes/hooks-install.d.ts +14 -11
  36. package/dist/lib/lanes/hooks-install.js +70 -50
  37. package/dist/lib/lanes/loop.d.ts +158 -0
  38. package/dist/lib/lanes/loop.js +743 -0
  39. package/dist/lib/lanes/outpost-dispatch.d.ts +9 -1
  40. package/dist/lib/lanes/outpost-dispatch.js +21 -7
  41. package/dist/lib/lanes/reaper.d.ts +58 -0
  42. package/dist/lib/lanes/reaper.js +403 -0
  43. package/dist/lib/lanes/reconcile.d.ts +19 -0
  44. package/dist/lib/lanes/reconcile.js +173 -6
  45. package/dist/lib/lanes/spawn.d.ts +4 -0
  46. package/dist/lib/lanes/spawn.js +41 -9
  47. package/dist/lib/lanes/status-fields.d.ts +12 -0
  48. package/dist/lib/lanes/status-fields.js +189 -0
  49. package/dist/lib/lanes/store.js +49 -2
  50. package/dist/lib/lanes/types.d.ts +18 -0
  51. package/dist/lib/lanes/types.js +3 -3
  52. package/dist/lib/lanes/watcher.js +3 -0
  53. package/dist/lib/lanes/worker-continuity.d.ts +18 -0
  54. package/dist/lib/lanes/worker-continuity.js +81 -0
  55. package/dist/lib/lanes/worker-process.d.ts +4 -0
  56. package/dist/lib/lanes/worker-process.js +77 -0
  57. package/dist/main.js +9 -3
  58. package/dist/utils/cli-telemetry.d.ts +4 -1
  59. package/dist/utils/cli-telemetry.js +8 -0
  60. package/dist/utils/jq-windows-compat.d.ts +13 -0
  61. package/dist/utils/jq-windows-compat.js +48 -0
  62. package/dist/utils/run-bundled-script.d.ts +3 -0
  63. package/dist/utils/run-bundled-script.js +9 -80
  64. package/dist/utils/windows-bash.d.ts +5 -0
  65. package/dist/utils/windows-bash.js +75 -0
  66. package/package.json +1 -1
@@ -20,17 +20,18 @@ import { loadWorkerProfile } from "./profile.js";
20
20
  import { terminalChildResumeHireGrantError, withHireGrantAdmission, } from "./grants.js";
21
21
  import { classifySessionLiveness, defaultLivenessDeps, loadLivenessIndex, readDaemonPresenceConnected, unindexedClaudeSessionLiveness, } from "./liveness.js";
22
22
  import { ANSWERED_QUESTIONS_FILE, armLaneWatcher, envelopeRepairMode, isPidRecordAlive, laneRunDir, nextEnvelopePath, readPidRecord, readProcStarttime, readSpawnManifest, markSpawnFailure, stopLaneProcesses, spawnEnvelopeRepair, spawnLaneWorker, retryLaneOnAlternateAccount, } from "./spawn.js";
23
- import { appendCappedHistory, lanesStoreRoot, parseLaneRecord, parseQuestionRecord, readLane, readQuestion, listQuestionIds, updateLane, updateQuestion, } from "./store.js";
23
+ import { appendCappedHistory, lanesStoreRoot, parseLaneRecord, parseQuestionRecord, readLane, retryUpdateLane, readQuestion, listQuestionIds, updateLane, updateQuestion, } from "./store.js";
24
24
  import { applyThrottleDecision, decideAdmission, defaultThrottleDeps, emptyThrottleState, evaluateThrottle, isCpuCheckEnabled, readAvailableMemMb, readThrottleState, resolveCpuBusyFraction, sampleCpuBusyObservation, recordCpuBusyObservation, throttleStatePath, writeThrottleState, } from "./throttle.js";
25
25
  import { readLanesRegistryFlag, readLanesRegistryFlagFailOff, readLanesRegistryFlagFailOpen } from "./registry-flag.js";
26
- import { LANE_HIRE_GRANT_FINISHED_LANES_FLAG, LANE_KEEP_ALIVE_DUE_WAKE_PRECISE_FLAG, LANE_LEAD_ADMISSION_PRIORITY_FLAG, LANE_QUOTA_ACCOUNT_FAILOVER_FLAG, } from "./flag-keys.js";
26
+ import { LANE_BLOCKED_ESCALATION_FLAG, LANE_HIRE_GRANT_FINISHED_LANES_FLAG, LANE_KEEP_ALIVE_DUE_WAKE_PRECISE_FLAG, LANE_LEAD_ADMISSION_PRIORITY_FLAG, LANE_QUOTA_ACCOUNT_FAILOVER_FLAG, } from "./flag-keys.js";
27
27
  import { checkLaneDiskSpace, configuredCodexRolloutHomes, laneDiskAccount, retainCodexRollouts } from "./disk-guard.js";
28
28
  import { laneCompanyUidSource, resolveLaneCompany } from "./lane-company.js";
29
- import { appendDropboxMessage } from "./dropbox.js";
29
+ import { appendDropboxMessageOnce, appendDropboxMessageOnceAtPath } from "./dropbox.js";
30
+ import { monitorSessionDir } from "../monitor.js";
30
31
  import { DEFAULT_RUNAWAY_SEARCH_GUARD_CONFIG, listHostProcesses, runRunawayToolGuard, RUNAWAY_TOOL_GUARD_FLAG, } from "./runaway-tool-guard.js";
31
32
  import { LEAD_LANE_KEEP_ALIVE_FLAG, LEAD_LANE_KEEP_ALIVE_RESUME_BLOCKED_FLAG, LEAD_LANE_KEEP_ALIVE_IMMEDIATE_FIRST_RESUME_FLAG, LEAD_LANE_KEEP_ALIVE_FIRST_RESUME_SETTLE_MS, LEAD_LANE_KEEP_ALIVE_BLOCKED_GRACE_MS, LEAD_LANE_KEEP_ALIVE_BLOCKED_RESUME_TEXT, queueLaneTerminalResume, LEAD_LANE_KEEP_ALIVE_INTERRUPTED_RESUME_TEXT, LEAD_LANE_KEEP_ALIVE_MAX_RESUMES, LEAD_LANE_KEEP_ALIVE_RESUME_TEXT, LANE_SUPERSESSION_LINEAGE_AWARE_FLAG, doneKeepAliveVerdict, dueKeepAliveNextCheckMs, hasPendingKeepAliveWork, keepAliveNextCheckAtMs, keepAliveExclusion, loadRetiredLaneIds, leadLaneKeepAliveBackoffMs, undeliveredAnsweredQuestions, } from "./keep-alive.js";
32
33
  import { isFailedQuotaKeepAliveLane, providerSupportsSessionResume } from "./delivery.js";
33
- import { isIsoDateTime } from "./types.js";
34
+ import { LANE_PROVIDERS, isIsoDateTime, isTerminalLaneState } from "./types.js";
34
35
  import { applyDefaultQuotaHold, clearAccountQuotaAfterSuccessfulTurnBestEffort, loadQuotaState, readQuotaSignal, recordAccountQuota, } from "./quota.js";
35
36
  import { answeredQuestionEntries, questionRecordRefusalCode, raiseEnvelopeQuestions, resolveQuestionCompanyWithFallback, } from "./questions.js";
36
37
  import { SILENCE_MS, appendWatcherEvent, classifyLaneExit, isPidAlive, missingExpectedEnvelopeFile, readLatestEnvelopeFile, readLatestEnvelopeRecord, watcherEventsPath, watcherOutLogPath, writeWatcherDecision, } from "./watcher.js";
@@ -120,6 +121,7 @@ function emptySummary(dryRun) {
120
121
  heartbeats: [],
121
122
  skipped: [],
122
123
  keep_alive_blocked: [],
124
+ blocked_escalations: [],
123
125
  errors: [],
124
126
  heartbeat_at: null,
125
127
  };
@@ -193,6 +195,161 @@ export function currentHopStartedAtMs(question) {
193
195
  const ms = Date.parse(raw);
194
196
  return Number.isFinite(ms) ? ms : Number.NEGATIVE_INFINITY;
195
197
  }
198
+ const BLOCKED_ESCALATION_WINDOW_MS = 6 * 60 * 60 * 1000;
199
+ const BLOCKED_ESCALATION_OPERATOR_SESSION = "fad349d4-38a6-4ae0-b8cc-e5dfd1bc3e9a";
200
+ function seniorKey(senior) {
201
+ return senior.id ? `${senior.kind}:${senior.id}` : senior.kind;
202
+ }
203
+ function recipientIsOnCurrentHost(recipientHost, currentHost) {
204
+ if (!recipientHost)
205
+ return false;
206
+ if (recipientHost.kind === "outpost")
207
+ return currentHost.outpostId === recipientHost.id;
208
+ return currentHost.outpostId === null;
209
+ }
210
+ async function reconcileBlockedEscalations(hqRoot, lanes, summary, deps, now) {
211
+ const env = deps.env ?? process.env;
212
+ const flagReader = deps.readRegistryFlag ?? readLanesRegistryFlagFailOpen;
213
+ const nowMs = now.getTime();
214
+ const findLane = async (laneId) => lanes.find((candidate) => candidate.lane_id === laneId) ??
215
+ (await deps.resolveEscalationLane?.(laneId)) ??
216
+ undefined;
217
+ for (const snapshot of lanes) {
218
+ if (snapshot.state !== "blocked" || !laneRunsHere(snapshot, deps.host))
219
+ continue;
220
+ if (lanes.some((child) => child.senior.kind === "lane" && child.senior.id === snapshot.lane_id && !isTerminalLaneState(child.state)))
221
+ continue;
222
+ let enabled;
223
+ try {
224
+ enabled = await flagReader(LANE_BLOCKED_ESCALATION_FLAG, {
225
+ company: resolveLaneCompany(hqRoot, snapshot).company,
226
+ env,
227
+ defaultValue: true,
228
+ });
229
+ }
230
+ catch (error) {
231
+ markError(summary, error);
232
+ continue;
233
+ }
234
+ if (!enabled)
235
+ continue;
236
+ let lane = readLane(hqRoot, snapshot.lane_id);
237
+ if (!lane.blocked_since) {
238
+ if (!deps.dryRun) {
239
+ lane = retryUpdateLane(hqRoot, lane.lane_id, "reconcile", (current) => current.state === "blocked" && !current.blocked_since
240
+ ? { ...current, blocked_since: now.toISOString() }
241
+ : current);
242
+ }
243
+ continue;
244
+ }
245
+ const blockedSinceMs = Date.parse(lane.blocked_since);
246
+ if (!Number.isFinite(blockedSinceMs) || nowMs - blockedSinceMs < BLOCKED_ESCALATION_WINDOW_MS)
247
+ continue;
248
+ const last = lane.escalations?.at(-1);
249
+ if (last && nowMs - Date.parse(last.at) < BLOCKED_ESCALATION_WINDOW_MS)
250
+ continue;
251
+ const directSenior = lane.senior;
252
+ let recipient = { kind: "session", id: BLOCKED_ESCALATION_OPERATOR_SESSION };
253
+ let recipientHost;
254
+ if (directSenior.kind === "lane" && directSenior.id) {
255
+ let seniorLane;
256
+ try {
257
+ seniorLane = await findLane(directSenior.id);
258
+ }
259
+ catch (error) {
260
+ markError(summary, error);
261
+ continue;
262
+ }
263
+ if (!seniorLane) {
264
+ markError(summary, new LanesError("blocked_escalation_ancestor_unavailable", `Could not resolve direct senior lane ${directSenior.id}`, {
265
+ lane_id: lane.lane_id,
266
+ senior_lane_id: directSenior.id,
267
+ }));
268
+ continue;
269
+ }
270
+ recipient = seniorLane.senior;
271
+ if (recipient.kind === "lane" && recipient.id) {
272
+ try {
273
+ recipientHost = (await findLane(recipient.id))?.host;
274
+ }
275
+ catch (error) {
276
+ markError(summary, error);
277
+ continue;
278
+ }
279
+ if (!recipientHost) {
280
+ markError(summary, new LanesError("blocked_escalation_ancestor_unavailable", `Could not resolve escalation recipient lane ${recipient.id}`, {
281
+ lane_id: lane.lane_id,
282
+ recipient_lane_id: recipient.id,
283
+ }));
284
+ continue;
285
+ }
286
+ }
287
+ else if (recipient.kind !== "session") {
288
+ recipient = { kind: "session", id: BLOCKED_ESCALATION_OPERATOR_SESSION };
289
+ }
290
+ }
291
+ if (recipient.kind === "session")
292
+ recipientHost = { kind: "local", id: "primary" };
293
+ const hours = Math.round(((nowMs - blockedSinceMs) / 3600000) * 100) / 100;
294
+ const blockedReason = lane.last_envelope?.blocked_reason ?? lane.terminal_reason ?? "not recorded";
295
+ const messageId = `blocked-escalation-${lane.lane_id}-${Math.floor((nowMs - blockedSinceMs) / BLOCKED_ESCALATION_WINDOW_MS)}`;
296
+ const text = `Blocked lane escalation\nLane: ${lane.lane_id}\nStory: ${lane.story_id}\nDirect senior: ${seniorKey(directSenior)}\nBlocked since: ${lane.blocked_since}\nHours blocked: ${hours}\nLast blocked_reason: ${blockedReason}`;
297
+ const delivery = {
298
+ lane_id: lane.lane_id,
299
+ project_id: lane.project_id,
300
+ story_id: lane.story_id,
301
+ direct_senior: directSenior,
302
+ recipient,
303
+ recipient_host: recipientHost,
304
+ blocked_since: lane.blocked_since,
305
+ at: now.toISOString(),
306
+ hours,
307
+ blocked_reason: blockedReason,
308
+ message_id: messageId,
309
+ text,
310
+ };
311
+ summary.blocked_escalations.push(delivery);
312
+ if (deps.dryRun)
313
+ continue;
314
+ try {
315
+ const current = readLane(hqRoot, lane.lane_id);
316
+ if (current.state !== "blocked" ||
317
+ current.blocked_since !== lane.blocked_since ||
318
+ (current.escalations?.at(-1) !== undefined &&
319
+ nowMs - Date.parse(current.escalations.at(-1).at) < BLOCKED_ESCALATION_WINDOW_MS))
320
+ continue;
321
+ if (deps.deliverBlockedEscalation && !recipientIsOnCurrentHost(recipientHost, deps.host)) {
322
+ await deps.deliverBlockedEscalation(delivery);
323
+ }
324
+ else if (recipient.kind === "lane" && recipient.id) {
325
+ appendDropboxMessageOnce(hqRoot, recipient.id, { at: delivery.at, message_id: messageId, kind: "blocked_escalation", text });
326
+ }
327
+ else if (recipient.kind === "session" && recipient.id) {
328
+ const dirs = LANE_PROVIDERS.map((provider) => monitorSessionDir(hqRoot, provider, recipient.id))
329
+ .filter((dir) => fs.existsSync(dir));
330
+ if (dirs.length !== 1)
331
+ throw new LanesError("blocked_escalation_recipient_unavailable", `Expected one durable inbox for ${recipient.id}; found ${dirs.length}`);
332
+ appendDropboxMessageOnceAtPath(path.join(dirs[0], "dropbox"), { at: delivery.at, message_id: messageId, kind: "blocked_escalation", text });
333
+ }
334
+ else {
335
+ throw new LanesError("blocked_escalation_recipient_invalid", `Cannot resolve escalation recipient ${seniorKey(recipient)}`);
336
+ }
337
+ retryUpdateLane(hqRoot, lane.lane_id, "reconcile", (latest) => {
338
+ if (latest.state !== "blocked" ||
339
+ latest.blocked_since !== lane.blocked_since ||
340
+ latest.escalations?.some((entry) => entry.at === delivery.at) ||
341
+ (latest.escalations?.at(-1) !== undefined &&
342
+ nowMs - Date.parse(latest.escalations.at(-1).at) < BLOCKED_ESCALATION_WINDOW_MS))
343
+ return latest;
344
+ const entry = { at: delivery.at, to: seniorKey(recipient), hours };
345
+ return { ...latest, escalations: [...(latest.escalations ?? []), entry] };
346
+ });
347
+ }
348
+ catch (error) {
349
+ markError(summary, error);
350
+ }
351
+ }
352
+ }
196
353
  export function shouldAgeQuestion(question, audience, nowMs, ageWindowMs = QUESTION_AGE_MS, maxAgedAdvances = MAX_AGED_ADVANCES) {
197
354
  if (question.status !== "pending")
198
355
  return false;
@@ -203,6 +360,12 @@ export function shouldAgeQuestion(question, audience, nowMs, ageWindowMs = QUEST
203
360
  const started = currentHopStartedAtMs(question);
204
361
  if (started === Number.NEGATIVE_INFINITY)
205
362
  return false;
363
+ if (question.hop === 1 && audience.kind === "session") {
364
+ const seenAt = question.seen_at ? Date.parse(question.seen_at) : Number.NaN;
365
+ if (!Number.isFinite(seenAt) || question.seen_session_id !== audience.id)
366
+ return false;
367
+ return nowMs - Math.max(started, seenAt) >= ageWindowMs;
368
+ }
206
369
  return nowMs - started >= ageWindowMs;
207
370
  }
208
371
  export function shouldAdvanceAudienceEnded(audience, liveness) {
@@ -932,7 +1095,7 @@ async function runRunawayToolGuardPass(hqRoot, lanes, config, deps, now, summary
932
1095
  ((pid, expectedStarttime) => readProcStarttime(pid) === expectedStarttime),
933
1096
  signal: deps.runawayToolProcessSignal ?? ((pid, signal) => process.kill(pid, signal)),
934
1097
  recordEvent: (laneId, event) => appendWatcherEvent(watcherEventsPath(hqRoot, laneId), event, deps.runawayToolEventAppend),
935
- notifyLane: (laneId, text) => appendDropboxMessage(hqRoot, laneId, {
1098
+ notifyLane: (laneId, text) => appendDropboxMessageOnce(hqRoot, laneId, {
936
1099
  at: now.toISOString(),
937
1100
  message_id: `runaway-search-guard:${laneId}:${now.getTime()}`,
938
1101
  kind: "system_notice",
@@ -977,6 +1140,7 @@ export async function runPassBody(hqRoot, deps) {
977
1140
  if (deps.terminalKeepAliveCandidate) {
978
1141
  return runTerminalKeepAliveCandidatePass(hqRoot, lanes, summary, deps, now, deps.terminalKeepAliveCandidate);
979
1142
  }
1143
+ await reconcileBlockedEscalations(hqRoot, lanes, summary, deps, now);
980
1144
  await resumeDueQuotaHeldLanes(hqRoot, lanes, summary, deps, now);
981
1145
  let presenceConnected;
982
1146
  try {
@@ -1134,7 +1298,8 @@ export async function runPassBody(hqRoot, deps) {
1134
1298
  markError(summary, err);
1135
1299
  liveness = null;
1136
1300
  }
1137
- if (shouldAdvanceAudienceEnded(audience, liveness)) {
1301
+ const mustWaitForOperatorDelivery = question.hop === 1 && audience.kind === "session";
1302
+ if (!mustWaitForOperatorDelivery && shouldAdvanceAudienceEnded(audience, liveness)) {
1138
1303
  const nextHop = question.hop + 1;
1139
1304
  const nextAudience = resolveQuestionAudience({ ...question, hop: nextHop }, readLaneSenior, readSessionSenior);
1140
1305
  const result = casQuestionIf(hqRoot, question.question_id, (current) => current.status === "pending" && current.hop === question.hop, (current) => {
@@ -1147,6 +1312,7 @@ export async function runPassBody(hqRoot, deps) {
1147
1312
  return {
1148
1313
  ...current,
1149
1314
  hop: nextHop,
1315
+ delivered_at: null,
1150
1316
  routing_history: appendCappedHistory(current.routing_history, entry),
1151
1317
  };
1152
1318
  }, deps.dryRun);
@@ -1190,6 +1356,7 @@ export async function runPassBody(hqRoot, deps) {
1190
1356
  return {
1191
1357
  ...current,
1192
1358
  hop: current.hop + 1,
1359
+ delivered_at: null,
1193
1360
  routing_history: appendCappedHistory(current.routing_history, entry),
1194
1361
  };
1195
1362
  }, deps.dryRun);
@@ -92,6 +92,8 @@ export interface SpawnLaneDeps {
92
92
  runCompaction?: RunCompactionFn;
93
93
  /** Finds a session transcript under an account home. Injected in tests. */
94
94
  locateTranscript?: (home: string, sessionId: string) => string | null;
95
+ /** Internal loop-lane override for the single provider turn's working directory. */
96
+ workingDirectory?: string;
95
97
  }
96
98
  export interface ProviderLaunch {
97
99
  command: string;
@@ -128,6 +130,8 @@ export declare function readProcStarttimeDetailed(pid: number): {
128
130
  starttime: number | null;
129
131
  error: string | null;
130
132
  };
133
+ /** macOS has no /proc; use ps's process start time as the PID identity token. */
134
+ export declare function parsePsStarttime(value: string): number | null;
131
135
  export declare function readProcStarttime(pid: number): number | null;
132
136
  export declare function parsePidRecord(text: string): PidRecord | null;
133
137
  export declare function readPidRecord(filePath: string): PidRecord | null;
@@ -31,6 +31,7 @@ import { appendCappedHistory, listLaneIds, readLane, retryUpdateLane } from "./s
31
31
  import { managerStatusGateHeartbeat } from "./manager-status-gate.js";
32
32
  import { LANE_QUOTA_ACCOUNT_FAILOVER_FLAG } from "./flag-keys.js";
33
33
  import { readLanesRegistryFlagFailOff } from "./registry-flag.js";
34
+ import { LANE_WORKER_EXIT_PATH_ENV } from "./worker-process.js";
34
35
  import { laneCompanyUidSource, resolveLaneCompany } from "./lane-company.js";
35
36
  import { LEAD_LANE_KEEP_ALIVE_BLOCKED_RESUME_TEXT } from "./keep-alive.js";
36
37
  import { copySessionFiles, planSessionCopy } from "./move-account.js";
@@ -932,6 +933,16 @@ export function readProcStarttimeDetailed(pid) {
932
933
  }
933
934
  catch (err) {
934
935
  const code = err.code;
936
+ if (code === "ENOENT" && process.platform === "darwin") {
937
+ const result = spawnSync("ps", ["-p", String(pid), "-o", "lstart="], {
938
+ encoding: "utf8",
939
+ windowsHide: true,
940
+ });
941
+ const starttime = parsePsStarttime(result.status === 0 ? result.stdout : "");
942
+ return starttime === null
943
+ ? { starttime: null, error: result.status === 0 ? "invalid_ps_starttime" : "ENOENT" }
944
+ : { starttime, error: null };
945
+ }
935
946
  if (code === "ENOENT")
936
947
  return { starttime: null, error: "ENOENT" };
937
948
  process.stderr.write(`hq lanes: failed to read starttime for pid ${pid}: ${err instanceof Error ? err.message : String(err)}\n`);
@@ -941,6 +952,14 @@ export function readProcStarttimeDetailed(pid) {
941
952
  };
942
953
  }
943
954
  }
955
+ /** macOS has no /proc; use ps's process start time as the PID identity token. */
956
+ export function parsePsStarttime(value) {
957
+ const trimmed = value.trim();
958
+ if (!trimmed)
959
+ return null;
960
+ const parsed = Date.parse(trimmed);
961
+ return Number.isFinite(parsed) ? parsed : null;
962
+ }
944
963
  export function readProcStarttime(pid) {
945
964
  return readProcStarttimeDetailed(pid).starttime;
946
965
  }
@@ -1698,6 +1717,17 @@ function spawnDetached(spawnProcess, command, args, env, cwd, stdoutPath, stderr
1698
1717
  closeFd(errFd, stderrPath);
1699
1718
  }
1700
1719
  }
1720
+ function spawnTrackedWorker(spawnProcess, command, args, env, cwd, stdoutPath, stderrPath, exitPath) {
1721
+ try {
1722
+ fs.unlinkSync(exitPath);
1723
+ }
1724
+ catch (error) {
1725
+ if (error.code !== "ENOENT")
1726
+ throw error;
1727
+ }
1728
+ const runnerPath = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../../../dist/lib/lanes/worker-process.js");
1729
+ return spawnDetached(spawnProcess, process.execPath, [runnerPath, command, ...args], { ...env, [LANE_WORKER_EXIT_PATH_ENV]: exitPath }, cwd, stdoutPath, stderrPath);
1730
+ }
1701
1731
  function requireDetached(proveDetached, pid, kind) {
1702
1732
  const proof = proveDetached(pid);
1703
1733
  if (proof.detached)
@@ -1963,13 +1993,15 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
1963
1993
  // state and the caller sees the error. Admission and question resume keep
1964
1994
  // today's behaviour of naming the terminal reason on the record.
1965
1995
  const markPreSpawnFailure = trigger.kind !== "message" && trigger.kind !== "resume";
1966
- const plan = pendingResume?.trigger === "answered_questions"
1967
- ? pendingResume.session_id !== undefined && providerSupportsSessionResume(existing.provider)
1968
- ? { mode: "resume", session_id: pendingResume.session_id }
1969
- : planLaneLaunch(existing, { kind: "answered_questions" })
1970
- : pendingResume?.session_id !== undefined && providerSupportsSessionResume(existing.provider)
1971
- ? { mode: "resume", session_id: pendingResume.session_id }
1972
- : planLaneLaunch(existing, trigger);
1996
+ const plan = pendingResume?.fresh_thread === true
1997
+ ? { mode: "fresh" }
1998
+ : pendingResume?.trigger === "answered_questions"
1999
+ ? pendingResume.session_id !== undefined && providerSupportsSessionResume(existing.provider)
2000
+ ? { mode: "resume", session_id: pendingResume.session_id }
2001
+ : planLaneLaunch(existing, { kind: "answered_questions" })
2002
+ : pendingResume?.session_id !== undefined && providerSupportsSessionResume(existing.provider)
2003
+ ? { mode: "resume", session_id: pendingResume.session_id }
2004
+ : planLaneLaunch(existing, trigger);
1973
2005
  if (existing.host.kind === "outpost") {
1974
2006
  refuseOutpostSelfTarget(hqRoot, existing, deps);
1975
2007
  }
@@ -2572,7 +2604,7 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
2572
2604
  let workerPid;
2573
2605
  let watcherPid;
2574
2606
  try {
2575
- workerPid = spawnDetached(spawnProcess, command.command, command.args, childEnv, hqRoot, path.join(logsDir, "worker.out"), path.join(logsDir, "worker.err"));
2607
+ workerPid = spawnTrackedWorker(spawnProcess, command.command, command.args, childEnv, deps.workingDirectory ?? pendingResume?.working_directory ?? hqRoot, path.join(logsDir, "worker.out"), path.join(logsDir, "worker.err"), path.join(logsDir, "worker-exit.json"));
2576
2608
  writePidRecord(path.join(runDir, "worker.pid"), {
2577
2609
  pid: workerPid,
2578
2610
  starttime: readProcStarttime(workerPid) ?? undefined,
@@ -2881,7 +2913,7 @@ export function spawnEnvelopeRepair(hqRoot, lane, input, deps = {}) {
2881
2913
  const previous = readSpawnManifest(runDir);
2882
2914
  let workerPid;
2883
2915
  try {
2884
- workerPid = spawnDetached(spawnProcess, command.command, command.args, childEnv, hqRoot, path.join(logsDir, "worker.out"), path.join(logsDir, "worker.err"));
2916
+ workerPid = spawnTrackedWorker(spawnProcess, command.command, command.args, childEnv, hqRoot, path.join(logsDir, "worker.out"), path.join(logsDir, "worker.err"), path.join(logsDir, "worker-exit.json"));
2885
2917
  writePidRecord(path.join(runDir, "worker.pid"), {
2886
2918
  pid: workerPid,
2887
2919
  starttime: readProcStarttime(workerPid) ?? undefined,
@@ -0,0 +1,12 @@
1
+ import { type LaneRecord } from "./types.js";
2
+ export interface LaneStatusRowFields {
3
+ last_line: string | null;
4
+ pr: string | null;
5
+ inbox_pending: number | null;
6
+ elapsed_s: number | null;
7
+ exit: number | null;
8
+ }
9
+ export type LaneStatusRow = LaneRecord & Partial<LaneStatusRowFields>;
10
+ /** Bounded, local-only status data. Remote-owned lanes report null for file-backed fields. */
11
+ export declare function readLaneStatusRowFields(hqRoot: string, lane: LaneRecord, nowMs?: number, local?: boolean): LaneStatusRowFields;
12
+ //# sourceMappingURL=status-fields.d.ts.map
@@ -0,0 +1,189 @@
1
+ import * as fs from "node:fs";
2
+ import * as path from "node:path";
3
+ import { assertNoSecretText } from "./text-scan.js";
4
+ import { compareEnvelopeNames } from "./envelope-sequence.js";
5
+ import { isTerminalLaneState } from "./types.js";
6
+ const OUTPUT_TAIL_BYTES = 64 * 1024;
7
+ const INBOX_TAIL_BYTES = 32 * 1024;
8
+ const PR_URL = /https:\/\/github\.com\/[\w.-]+\/[\w.-]+\/pull\/\d+/g;
9
+ function readTail(filePath, maxBytes) {
10
+ let fd;
11
+ try {
12
+ fd = fs.openSync(filePath, "r");
13
+ const stat = fs.fstatSync(fd);
14
+ const size = stat.size;
15
+ const start = Math.max(0, size - maxBytes);
16
+ const buffer = Buffer.alloc(size - start);
17
+ fs.readSync(fd, buffer, 0, buffer.length, start);
18
+ return { text: buffer.toString("utf8"), size, mtimeMs: stat.mtimeMs };
19
+ }
20
+ catch (error) {
21
+ if (error.code !== "ENOENT") {
22
+ process.stderr.write(`hq lanes: could not read status tail ${filePath}: ${error instanceof Error ? error.message : String(error)}\n`);
23
+ }
24
+ return null;
25
+ }
26
+ finally {
27
+ if (fd !== undefined)
28
+ fs.closeSync(fd);
29
+ }
30
+ }
31
+ function candidateLines(value, depth = 0) {
32
+ if (depth > 4 || value === null || value === undefined)
33
+ return [];
34
+ if (typeof value === "string")
35
+ return value.split(/\r?\n/);
36
+ if (Array.isArray(value))
37
+ return value.flatMap((item) => candidateLines(item, depth + 1));
38
+ if (typeof value !== "object")
39
+ return [];
40
+ const record = value;
41
+ const direct = ["text", "content", "message", "last_assistant_message", "lastAssistantMessage"]
42
+ .flatMap((key) => candidateLines(record[key], depth + 1));
43
+ return direct.length > 0 ? direct : ["item", "message", "response", "result"]
44
+ .flatMap((key) => candidateLines(record[key], depth + 1));
45
+ }
46
+ function meaningfulLines(text) {
47
+ const lines = [];
48
+ for (const raw of text.split(/\r?\n/)) {
49
+ // eslint-disable-next-line no-control-regex
50
+ const line = raw.replace(/\x1b\[[0-9;]*m/g, "").trim();
51
+ if (!line || line.startsWith("{") || line.startsWith("[") || line.includes("cache_read_input_tokens"))
52
+ continue;
53
+ lines.push(line);
54
+ }
55
+ return lines;
56
+ }
57
+ function linesFromOutput(text) {
58
+ const output = [];
59
+ for (const raw of text.split(/\r?\n/)) {
60
+ if (!raw.trim())
61
+ continue;
62
+ try {
63
+ const parsed = JSON.parse(raw);
64
+ output.push(...meaningfulLines(candidateLines(parsed).join("\n")));
65
+ }
66
+ catch {
67
+ output.push(...meaningfulLines(raw));
68
+ }
69
+ }
70
+ return output;
71
+ }
72
+ function safeLastLine(lines) {
73
+ for (let i = lines.length - 1; i >= 0; i -= 1) {
74
+ const line = lines[i].trim();
75
+ if (!line)
76
+ continue;
77
+ const clipped = line.length > 160 ? `${line.slice(0, 157)}…` : line;
78
+ try {
79
+ // Same secret detector used by the lane question flow. Drop unsafe output.
80
+ assertNoSecretText("last_line", clipped);
81
+ return clipped;
82
+ }
83
+ catch {
84
+ continue;
85
+ }
86
+ }
87
+ return null;
88
+ }
89
+ function lastPr(text) {
90
+ const matches = [...text.matchAll(PR_URL)];
91
+ return matches.length ? matches[matches.length - 1][0] : null;
92
+ }
93
+ function terminalExit(lane, runDir) {
94
+ if (!isTerminalLaneState(lane.state))
95
+ return null;
96
+ const exitRecord = readTail(path.join(runDir, "logs", "worker-exit.json"), 4 * 1024);
97
+ if (exitRecord !== null) {
98
+ try {
99
+ const parsed = JSON.parse(exitRecord.text);
100
+ if (typeof parsed.exit_code === "number" && Number.isInteger(parsed.exit_code))
101
+ return parsed.exit_code;
102
+ }
103
+ catch {
104
+ process.stderr.write(`hq lanes: worker exit record is invalid for ${lane.lane_id}\n`);
105
+ }
106
+ }
107
+ const row = lane;
108
+ const candidates = [row.worker_exit_code, row.exit_code, row.terminal_details?.exit_code];
109
+ return candidates.find((value) => Number.isInteger(value)) ?? null;
110
+ }
111
+ function startTimeMs(lane, runDir) {
112
+ const row = lane;
113
+ for (const value of [row.worker_started_at, row.started_at]) {
114
+ const parsed = typeof value === "number" ? value * 1000 : typeof value === "string" ? Date.parse(value) : NaN;
115
+ if (Number.isFinite(parsed) && parsed > 0)
116
+ return parsed;
117
+ }
118
+ try {
119
+ return fs.statSync(path.join(runDir, "worker.pid")).mtimeMs;
120
+ }
121
+ catch (error) {
122
+ if (error.code !== "ENOENT") {
123
+ process.stderr.write(`hq lanes: could not read worker start time ${runDir}: ${error instanceof Error ? error.message : String(error)}\n`);
124
+ }
125
+ return null;
126
+ }
127
+ }
128
+ /** Bounded, local-only status data. Remote-owned lanes report null for file-backed fields. */
129
+ export function readLaneStatusRowFields(hqRoot, lane, nowMs = Date.now(), local = true) {
130
+ if (!local)
131
+ return { last_line: null, pr: null, inbox_pending: null, elapsed_s: null, exit: null };
132
+ const runDir = path.join(hqRoot, "workspace", "lanes-runs", lane.lane_id);
133
+ const logsDir = path.join(runDir, "logs");
134
+ const outTail = readTail(path.join(logsDir, "worker.out"), OUTPUT_TAIL_BYTES);
135
+ const messageTail = readTail(path.join(logsDir, "last-message.txt"), OUTPUT_TAIL_BYTES);
136
+ const outputTails = [outTail, messageTail]
137
+ .filter((value) => value !== null)
138
+ .sort((left, right) => left.mtimeMs - right.mtimeMs);
139
+ const lines = outputTails.flatMap(({ text }) => linesFromOutput(text));
140
+ let envelopeTail = null;
141
+ try {
142
+ const names = fs.readdirSync(path.join(runDir, "envelopes"))
143
+ .filter((name) => name.endsWith(".json"))
144
+ .sort(compareEnvelopeNames);
145
+ const latest = names.at(-1);
146
+ if (latest)
147
+ envelopeTail = readTail(path.join(runDir, "envelopes", latest), OUTPUT_TAIL_BYTES);
148
+ }
149
+ catch (error) {
150
+ if (error.code !== "ENOENT") {
151
+ process.stderr.write(`hq lanes: could not read status envelopes for ${lane.lane_id}: ${error instanceof Error ? error.message : String(error)}\n`);
152
+ }
153
+ }
154
+ const prSources = [
155
+ ...outputTails.map(({ text, mtimeMs }) => ({ text, mtimeMs })),
156
+ ...(envelopeTail ? [{ text: envelopeTail.text, mtimeMs: envelopeTail.mtimeMs }] : []),
157
+ ].sort((left, right) => left.mtimeMs - right.mtimeMs);
158
+ const pr = lastPr(prSources.map(({ text }) => text).join("\n"));
159
+ const inboxTail = readTail(path.join(runDir, "dropbox", "inbox.jsonl"), INBOX_TAIL_BYTES);
160
+ const inboxPending = inboxTail === null ? 0
161
+ : inboxTail.size > INBOX_TAIL_BYTES ? null
162
+ : inboxTail.text.split(/\r?\n/).filter((line) => {
163
+ if (!line.trim())
164
+ return false;
165
+ try {
166
+ return JSON.parse(line) !== null;
167
+ }
168
+ catch {
169
+ return false;
170
+ }
171
+ }).length;
172
+ const startedAt = startTimeMs(lane, runDir);
173
+ const terminal = isTerminalLaneState(lane.state);
174
+ const terminalEndAt = [lane.ended_at, lane.updated_at]
175
+ .map((value) => typeof value === "string" ? Date.parse(value) : NaN)
176
+ .find(Number.isFinite);
177
+ const endedAt = terminal ? terminalEndAt ?? nowMs : nowMs;
178
+ const elapsed = startedAt === null || !Number.isFinite(endedAt)
179
+ ? null
180
+ : Math.max(0, Math.floor((endedAt - startedAt) / 1000));
181
+ return {
182
+ last_line: safeLastLine(lines),
183
+ pr,
184
+ inbox_pending: inboxPending,
185
+ elapsed_s: elapsed,
186
+ exit: terminalExit(lane, runDir),
187
+ };
188
+ }
189
+ //# sourceMappingURL=status-fields.js.map