@indigoai-us/hq-cli 5.345.55 → 5.345.57
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/assets/scaffold/core/scripts/derive-trigger-facts.sh +10 -1
- package/assets/scaffold/core/scripts/jobs-validate.sh +2 -1
- package/assets/scaffold/core/scripts/session-title.sh +18 -15
- package/dist/command-catalog.generated.d.ts +124 -0
- package/dist/command-catalog.generated.js +155 -0
- package/dist/commands/core-delegate-w1b.js +18 -5
- package/dist/commands/core-derive-trigger-facts-command.js +14 -0
- package/dist/commands/core-derive-trigger-facts-native.d.ts +1 -0
- package/dist/commands/core-derive-trigger-facts-native.js +20 -1
- package/dist/commands/doctor.d.ts +1 -0
- package/dist/commands/doctor.js +15 -2
- package/dist/commands/lanes-checks.d.ts +3 -0
- package/dist/commands/lanes-checks.js +34 -0
- package/dist/commands/lanes-loop.d.ts +12 -0
- package/dist/commands/lanes-loop.js +214 -0
- package/dist/commands/lanes-questions.js +37 -11
- package/dist/commands/lanes-resume.js +19 -1
- package/dist/commands/lanes-session-start.d.ts +2 -7
- package/dist/commands/lanes-session-start.js +9 -29
- package/dist/commands/lanes-shared.d.ts +14 -2
- package/dist/commands/lanes.d.ts +3 -1
- package/dist/commands/lanes.js +376 -35
- package/dist/commands/members.d.ts +4 -0
- package/dist/commands/members.js +23 -2
- package/dist/lanes-drain-hook.js +6 -2
- package/dist/lib/doctor/checks/sync-health.d.ts +2 -0
- package/dist/lib/doctor/checks/sync-health.js +47 -23
- package/dist/lib/lanes/delivery.d.ts +3 -0
- package/dist/lib/lanes/delivery.js +5 -0
- package/dist/lib/lanes/dropbox.d.ts +7 -2
- package/dist/lib/lanes/dropbox.js +157 -10
- package/dist/lib/lanes/flag-keys.d.ts +12 -0
- package/dist/lib/lanes/flag-keys.js +12 -0
- package/dist/lib/lanes/hooks-install.d.ts +14 -11
- package/dist/lib/lanes/hooks-install.js +70 -50
- package/dist/lib/lanes/loop.d.ts +158 -0
- package/dist/lib/lanes/loop.js +743 -0
- package/dist/lib/lanes/outpost-dispatch.d.ts +9 -1
- package/dist/lib/lanes/outpost-dispatch.js +21 -7
- package/dist/lib/lanes/reaper.d.ts +58 -0
- package/dist/lib/lanes/reaper.js +403 -0
- package/dist/lib/lanes/reconcile.d.ts +19 -0
- package/dist/lib/lanes/reconcile.js +173 -6
- package/dist/lib/lanes/spawn.d.ts +4 -0
- package/dist/lib/lanes/spawn.js +41 -9
- package/dist/lib/lanes/status-fields.d.ts +12 -0
- package/dist/lib/lanes/status-fields.js +189 -0
- package/dist/lib/lanes/store.js +49 -2
- package/dist/lib/lanes/types.d.ts +18 -0
- package/dist/lib/lanes/types.js +3 -3
- package/dist/lib/lanes/watcher.js +3 -0
- package/dist/lib/lanes/worker-continuity.d.ts +18 -0
- package/dist/lib/lanes/worker-continuity.js +81 -0
- package/dist/lib/lanes/worker-process.d.ts +4 -0
- package/dist/lib/lanes/worker-process.js +77 -0
- package/dist/main.js +9 -3
- package/dist/utils/cli-telemetry.d.ts +4 -1
- package/dist/utils/cli-telemetry.js +8 -0
- package/dist/utils/jq-windows-compat.d.ts +13 -0
- package/dist/utils/jq-windows-compat.js +48 -0
- package/dist/utils/run-bundled-script.d.ts +3 -0
- package/dist/utils/run-bundled-script.js +9 -80
- package/dist/utils/windows-bash.d.ts +5 -0
- package/dist/utils/windows-bash.js +75 -0
- package/package.json +1 -1
|
@@ -20,17 +20,18 @@ import { loadWorkerProfile } from "./profile.js";
|
|
|
20
20
|
import { terminalChildResumeHireGrantError, withHireGrantAdmission, } from "./grants.js";
|
|
21
21
|
import { classifySessionLiveness, defaultLivenessDeps, loadLivenessIndex, readDaemonPresenceConnected, unindexedClaudeSessionLiveness, } from "./liveness.js";
|
|
22
22
|
import { ANSWERED_QUESTIONS_FILE, armLaneWatcher, envelopeRepairMode, isPidRecordAlive, laneRunDir, nextEnvelopePath, readPidRecord, readProcStarttime, readSpawnManifest, markSpawnFailure, stopLaneProcesses, spawnEnvelopeRepair, spawnLaneWorker, retryLaneOnAlternateAccount, } from "./spawn.js";
|
|
23
|
-
import { appendCappedHistory, lanesStoreRoot, parseLaneRecord, parseQuestionRecord, readLane, readQuestion, listQuestionIds, updateLane, updateQuestion, } from "./store.js";
|
|
23
|
+
import { appendCappedHistory, lanesStoreRoot, parseLaneRecord, parseQuestionRecord, readLane, retryUpdateLane, readQuestion, listQuestionIds, updateLane, updateQuestion, } from "./store.js";
|
|
24
24
|
import { applyThrottleDecision, decideAdmission, defaultThrottleDeps, emptyThrottleState, evaluateThrottle, isCpuCheckEnabled, readAvailableMemMb, readThrottleState, resolveCpuBusyFraction, sampleCpuBusyObservation, recordCpuBusyObservation, throttleStatePath, writeThrottleState, } from "./throttle.js";
|
|
25
25
|
import { readLanesRegistryFlag, readLanesRegistryFlagFailOff, readLanesRegistryFlagFailOpen } from "./registry-flag.js";
|
|
26
|
-
import { LANE_HIRE_GRANT_FINISHED_LANES_FLAG, LANE_KEEP_ALIVE_DUE_WAKE_PRECISE_FLAG, LANE_LEAD_ADMISSION_PRIORITY_FLAG, LANE_QUOTA_ACCOUNT_FAILOVER_FLAG, } from "./flag-keys.js";
|
|
26
|
+
import { LANE_BLOCKED_ESCALATION_FLAG, LANE_HIRE_GRANT_FINISHED_LANES_FLAG, LANE_KEEP_ALIVE_DUE_WAKE_PRECISE_FLAG, LANE_LEAD_ADMISSION_PRIORITY_FLAG, LANE_QUOTA_ACCOUNT_FAILOVER_FLAG, } from "./flag-keys.js";
|
|
27
27
|
import { checkLaneDiskSpace, configuredCodexRolloutHomes, laneDiskAccount, retainCodexRollouts } from "./disk-guard.js";
|
|
28
28
|
import { laneCompanyUidSource, resolveLaneCompany } from "./lane-company.js";
|
|
29
|
-
import {
|
|
29
|
+
import { appendDropboxMessageOnce, appendDropboxMessageOnceAtPath } from "./dropbox.js";
|
|
30
|
+
import { monitorSessionDir } from "../monitor.js";
|
|
30
31
|
import { DEFAULT_RUNAWAY_SEARCH_GUARD_CONFIG, listHostProcesses, runRunawayToolGuard, RUNAWAY_TOOL_GUARD_FLAG, } from "./runaway-tool-guard.js";
|
|
31
32
|
import { LEAD_LANE_KEEP_ALIVE_FLAG, LEAD_LANE_KEEP_ALIVE_RESUME_BLOCKED_FLAG, LEAD_LANE_KEEP_ALIVE_IMMEDIATE_FIRST_RESUME_FLAG, LEAD_LANE_KEEP_ALIVE_FIRST_RESUME_SETTLE_MS, LEAD_LANE_KEEP_ALIVE_BLOCKED_GRACE_MS, LEAD_LANE_KEEP_ALIVE_BLOCKED_RESUME_TEXT, queueLaneTerminalResume, LEAD_LANE_KEEP_ALIVE_INTERRUPTED_RESUME_TEXT, LEAD_LANE_KEEP_ALIVE_MAX_RESUMES, LEAD_LANE_KEEP_ALIVE_RESUME_TEXT, LANE_SUPERSESSION_LINEAGE_AWARE_FLAG, doneKeepAliveVerdict, dueKeepAliveNextCheckMs, hasPendingKeepAliveWork, keepAliveNextCheckAtMs, keepAliveExclusion, loadRetiredLaneIds, leadLaneKeepAliveBackoffMs, undeliveredAnsweredQuestions, } from "./keep-alive.js";
|
|
32
33
|
import { isFailedQuotaKeepAliveLane, providerSupportsSessionResume } from "./delivery.js";
|
|
33
|
-
import { isIsoDateTime } from "./types.js";
|
|
34
|
+
import { LANE_PROVIDERS, isIsoDateTime, isTerminalLaneState } from "./types.js";
|
|
34
35
|
import { applyDefaultQuotaHold, clearAccountQuotaAfterSuccessfulTurnBestEffort, loadQuotaState, readQuotaSignal, recordAccountQuota, } from "./quota.js";
|
|
35
36
|
import { answeredQuestionEntries, questionRecordRefusalCode, raiseEnvelopeQuestions, resolveQuestionCompanyWithFallback, } from "./questions.js";
|
|
36
37
|
import { SILENCE_MS, appendWatcherEvent, classifyLaneExit, isPidAlive, missingExpectedEnvelopeFile, readLatestEnvelopeFile, readLatestEnvelopeRecord, watcherEventsPath, watcherOutLogPath, writeWatcherDecision, } from "./watcher.js";
|
|
@@ -120,6 +121,7 @@ function emptySummary(dryRun) {
|
|
|
120
121
|
heartbeats: [],
|
|
121
122
|
skipped: [],
|
|
122
123
|
keep_alive_blocked: [],
|
|
124
|
+
blocked_escalations: [],
|
|
123
125
|
errors: [],
|
|
124
126
|
heartbeat_at: null,
|
|
125
127
|
};
|
|
@@ -193,6 +195,161 @@ export function currentHopStartedAtMs(question) {
|
|
|
193
195
|
const ms = Date.parse(raw);
|
|
194
196
|
return Number.isFinite(ms) ? ms : Number.NEGATIVE_INFINITY;
|
|
195
197
|
}
|
|
198
|
+
const BLOCKED_ESCALATION_WINDOW_MS = 6 * 60 * 60 * 1000;
|
|
199
|
+
const BLOCKED_ESCALATION_OPERATOR_SESSION = "fad349d4-38a6-4ae0-b8cc-e5dfd1bc3e9a";
|
|
200
|
+
function seniorKey(senior) {
|
|
201
|
+
return senior.id ? `${senior.kind}:${senior.id}` : senior.kind;
|
|
202
|
+
}
|
|
203
|
+
function recipientIsOnCurrentHost(recipientHost, currentHost) {
|
|
204
|
+
if (!recipientHost)
|
|
205
|
+
return false;
|
|
206
|
+
if (recipientHost.kind === "outpost")
|
|
207
|
+
return currentHost.outpostId === recipientHost.id;
|
|
208
|
+
return currentHost.outpostId === null;
|
|
209
|
+
}
|
|
210
|
+
async function reconcileBlockedEscalations(hqRoot, lanes, summary, deps, now) {
|
|
211
|
+
const env = deps.env ?? process.env;
|
|
212
|
+
const flagReader = deps.readRegistryFlag ?? readLanesRegistryFlagFailOpen;
|
|
213
|
+
const nowMs = now.getTime();
|
|
214
|
+
const findLane = async (laneId) => lanes.find((candidate) => candidate.lane_id === laneId) ??
|
|
215
|
+
(await deps.resolveEscalationLane?.(laneId)) ??
|
|
216
|
+
undefined;
|
|
217
|
+
for (const snapshot of lanes) {
|
|
218
|
+
if (snapshot.state !== "blocked" || !laneRunsHere(snapshot, deps.host))
|
|
219
|
+
continue;
|
|
220
|
+
if (lanes.some((child) => child.senior.kind === "lane" && child.senior.id === snapshot.lane_id && !isTerminalLaneState(child.state)))
|
|
221
|
+
continue;
|
|
222
|
+
let enabled;
|
|
223
|
+
try {
|
|
224
|
+
enabled = await flagReader(LANE_BLOCKED_ESCALATION_FLAG, {
|
|
225
|
+
company: resolveLaneCompany(hqRoot, snapshot).company,
|
|
226
|
+
env,
|
|
227
|
+
defaultValue: true,
|
|
228
|
+
});
|
|
229
|
+
}
|
|
230
|
+
catch (error) {
|
|
231
|
+
markError(summary, error);
|
|
232
|
+
continue;
|
|
233
|
+
}
|
|
234
|
+
if (!enabled)
|
|
235
|
+
continue;
|
|
236
|
+
let lane = readLane(hqRoot, snapshot.lane_id);
|
|
237
|
+
if (!lane.blocked_since) {
|
|
238
|
+
if (!deps.dryRun) {
|
|
239
|
+
lane = retryUpdateLane(hqRoot, lane.lane_id, "reconcile", (current) => current.state === "blocked" && !current.blocked_since
|
|
240
|
+
? { ...current, blocked_since: now.toISOString() }
|
|
241
|
+
: current);
|
|
242
|
+
}
|
|
243
|
+
continue;
|
|
244
|
+
}
|
|
245
|
+
const blockedSinceMs = Date.parse(lane.blocked_since);
|
|
246
|
+
if (!Number.isFinite(blockedSinceMs) || nowMs - blockedSinceMs < BLOCKED_ESCALATION_WINDOW_MS)
|
|
247
|
+
continue;
|
|
248
|
+
const last = lane.escalations?.at(-1);
|
|
249
|
+
if (last && nowMs - Date.parse(last.at) < BLOCKED_ESCALATION_WINDOW_MS)
|
|
250
|
+
continue;
|
|
251
|
+
const directSenior = lane.senior;
|
|
252
|
+
let recipient = { kind: "session", id: BLOCKED_ESCALATION_OPERATOR_SESSION };
|
|
253
|
+
let recipientHost;
|
|
254
|
+
if (directSenior.kind === "lane" && directSenior.id) {
|
|
255
|
+
let seniorLane;
|
|
256
|
+
try {
|
|
257
|
+
seniorLane = await findLane(directSenior.id);
|
|
258
|
+
}
|
|
259
|
+
catch (error) {
|
|
260
|
+
markError(summary, error);
|
|
261
|
+
continue;
|
|
262
|
+
}
|
|
263
|
+
if (!seniorLane) {
|
|
264
|
+
markError(summary, new LanesError("blocked_escalation_ancestor_unavailable", `Could not resolve direct senior lane ${directSenior.id}`, {
|
|
265
|
+
lane_id: lane.lane_id,
|
|
266
|
+
senior_lane_id: directSenior.id,
|
|
267
|
+
}));
|
|
268
|
+
continue;
|
|
269
|
+
}
|
|
270
|
+
recipient = seniorLane.senior;
|
|
271
|
+
if (recipient.kind === "lane" && recipient.id) {
|
|
272
|
+
try {
|
|
273
|
+
recipientHost = (await findLane(recipient.id))?.host;
|
|
274
|
+
}
|
|
275
|
+
catch (error) {
|
|
276
|
+
markError(summary, error);
|
|
277
|
+
continue;
|
|
278
|
+
}
|
|
279
|
+
if (!recipientHost) {
|
|
280
|
+
markError(summary, new LanesError("blocked_escalation_ancestor_unavailable", `Could not resolve escalation recipient lane ${recipient.id}`, {
|
|
281
|
+
lane_id: lane.lane_id,
|
|
282
|
+
recipient_lane_id: recipient.id,
|
|
283
|
+
}));
|
|
284
|
+
continue;
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
else if (recipient.kind !== "session") {
|
|
288
|
+
recipient = { kind: "session", id: BLOCKED_ESCALATION_OPERATOR_SESSION };
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
if (recipient.kind === "session")
|
|
292
|
+
recipientHost = { kind: "local", id: "primary" };
|
|
293
|
+
const hours = Math.round(((nowMs - blockedSinceMs) / 3600000) * 100) / 100;
|
|
294
|
+
const blockedReason = lane.last_envelope?.blocked_reason ?? lane.terminal_reason ?? "not recorded";
|
|
295
|
+
const messageId = `blocked-escalation-${lane.lane_id}-${Math.floor((nowMs - blockedSinceMs) / BLOCKED_ESCALATION_WINDOW_MS)}`;
|
|
296
|
+
const text = `Blocked lane escalation\nLane: ${lane.lane_id}\nStory: ${lane.story_id}\nDirect senior: ${seniorKey(directSenior)}\nBlocked since: ${lane.blocked_since}\nHours blocked: ${hours}\nLast blocked_reason: ${blockedReason}`;
|
|
297
|
+
const delivery = {
|
|
298
|
+
lane_id: lane.lane_id,
|
|
299
|
+
project_id: lane.project_id,
|
|
300
|
+
story_id: lane.story_id,
|
|
301
|
+
direct_senior: directSenior,
|
|
302
|
+
recipient,
|
|
303
|
+
recipient_host: recipientHost,
|
|
304
|
+
blocked_since: lane.blocked_since,
|
|
305
|
+
at: now.toISOString(),
|
|
306
|
+
hours,
|
|
307
|
+
blocked_reason: blockedReason,
|
|
308
|
+
message_id: messageId,
|
|
309
|
+
text,
|
|
310
|
+
};
|
|
311
|
+
summary.blocked_escalations.push(delivery);
|
|
312
|
+
if (deps.dryRun)
|
|
313
|
+
continue;
|
|
314
|
+
try {
|
|
315
|
+
const current = readLane(hqRoot, lane.lane_id);
|
|
316
|
+
if (current.state !== "blocked" ||
|
|
317
|
+
current.blocked_since !== lane.blocked_since ||
|
|
318
|
+
(current.escalations?.at(-1) !== undefined &&
|
|
319
|
+
nowMs - Date.parse(current.escalations.at(-1).at) < BLOCKED_ESCALATION_WINDOW_MS))
|
|
320
|
+
continue;
|
|
321
|
+
if (deps.deliverBlockedEscalation && !recipientIsOnCurrentHost(recipientHost, deps.host)) {
|
|
322
|
+
await deps.deliverBlockedEscalation(delivery);
|
|
323
|
+
}
|
|
324
|
+
else if (recipient.kind === "lane" && recipient.id) {
|
|
325
|
+
appendDropboxMessageOnce(hqRoot, recipient.id, { at: delivery.at, message_id: messageId, kind: "blocked_escalation", text });
|
|
326
|
+
}
|
|
327
|
+
else if (recipient.kind === "session" && recipient.id) {
|
|
328
|
+
const dirs = LANE_PROVIDERS.map((provider) => monitorSessionDir(hqRoot, provider, recipient.id))
|
|
329
|
+
.filter((dir) => fs.existsSync(dir));
|
|
330
|
+
if (dirs.length !== 1)
|
|
331
|
+
throw new LanesError("blocked_escalation_recipient_unavailable", `Expected one durable inbox for ${recipient.id}; found ${dirs.length}`);
|
|
332
|
+
appendDropboxMessageOnceAtPath(path.join(dirs[0], "dropbox"), { at: delivery.at, message_id: messageId, kind: "blocked_escalation", text });
|
|
333
|
+
}
|
|
334
|
+
else {
|
|
335
|
+
throw new LanesError("blocked_escalation_recipient_invalid", `Cannot resolve escalation recipient ${seniorKey(recipient)}`);
|
|
336
|
+
}
|
|
337
|
+
retryUpdateLane(hqRoot, lane.lane_id, "reconcile", (latest) => {
|
|
338
|
+
if (latest.state !== "blocked" ||
|
|
339
|
+
latest.blocked_since !== lane.blocked_since ||
|
|
340
|
+
latest.escalations?.some((entry) => entry.at === delivery.at) ||
|
|
341
|
+
(latest.escalations?.at(-1) !== undefined &&
|
|
342
|
+
nowMs - Date.parse(latest.escalations.at(-1).at) < BLOCKED_ESCALATION_WINDOW_MS))
|
|
343
|
+
return latest;
|
|
344
|
+
const entry = { at: delivery.at, to: seniorKey(recipient), hours };
|
|
345
|
+
return { ...latest, escalations: [...(latest.escalations ?? []), entry] };
|
|
346
|
+
});
|
|
347
|
+
}
|
|
348
|
+
catch (error) {
|
|
349
|
+
markError(summary, error);
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
}
|
|
196
353
|
export function shouldAgeQuestion(question, audience, nowMs, ageWindowMs = QUESTION_AGE_MS, maxAgedAdvances = MAX_AGED_ADVANCES) {
|
|
197
354
|
if (question.status !== "pending")
|
|
198
355
|
return false;
|
|
@@ -203,6 +360,12 @@ export function shouldAgeQuestion(question, audience, nowMs, ageWindowMs = QUEST
|
|
|
203
360
|
const started = currentHopStartedAtMs(question);
|
|
204
361
|
if (started === Number.NEGATIVE_INFINITY)
|
|
205
362
|
return false;
|
|
363
|
+
if (question.hop === 1 && audience.kind === "session") {
|
|
364
|
+
const seenAt = question.seen_at ? Date.parse(question.seen_at) : Number.NaN;
|
|
365
|
+
if (!Number.isFinite(seenAt) || question.seen_session_id !== audience.id)
|
|
366
|
+
return false;
|
|
367
|
+
return nowMs - Math.max(started, seenAt) >= ageWindowMs;
|
|
368
|
+
}
|
|
206
369
|
return nowMs - started >= ageWindowMs;
|
|
207
370
|
}
|
|
208
371
|
export function shouldAdvanceAudienceEnded(audience, liveness) {
|
|
@@ -932,7 +1095,7 @@ async function runRunawayToolGuardPass(hqRoot, lanes, config, deps, now, summary
|
|
|
932
1095
|
((pid, expectedStarttime) => readProcStarttime(pid) === expectedStarttime),
|
|
933
1096
|
signal: deps.runawayToolProcessSignal ?? ((pid, signal) => process.kill(pid, signal)),
|
|
934
1097
|
recordEvent: (laneId, event) => appendWatcherEvent(watcherEventsPath(hqRoot, laneId), event, deps.runawayToolEventAppend),
|
|
935
|
-
notifyLane: (laneId, text) =>
|
|
1098
|
+
notifyLane: (laneId, text) => appendDropboxMessageOnce(hqRoot, laneId, {
|
|
936
1099
|
at: now.toISOString(),
|
|
937
1100
|
message_id: `runaway-search-guard:${laneId}:${now.getTime()}`,
|
|
938
1101
|
kind: "system_notice",
|
|
@@ -977,6 +1140,7 @@ export async function runPassBody(hqRoot, deps) {
|
|
|
977
1140
|
if (deps.terminalKeepAliveCandidate) {
|
|
978
1141
|
return runTerminalKeepAliveCandidatePass(hqRoot, lanes, summary, deps, now, deps.terminalKeepAliveCandidate);
|
|
979
1142
|
}
|
|
1143
|
+
await reconcileBlockedEscalations(hqRoot, lanes, summary, deps, now);
|
|
980
1144
|
await resumeDueQuotaHeldLanes(hqRoot, lanes, summary, deps, now);
|
|
981
1145
|
let presenceConnected;
|
|
982
1146
|
try {
|
|
@@ -1134,7 +1298,8 @@ export async function runPassBody(hqRoot, deps) {
|
|
|
1134
1298
|
markError(summary, err);
|
|
1135
1299
|
liveness = null;
|
|
1136
1300
|
}
|
|
1137
|
-
|
|
1301
|
+
const mustWaitForOperatorDelivery = question.hop === 1 && audience.kind === "session";
|
|
1302
|
+
if (!mustWaitForOperatorDelivery && shouldAdvanceAudienceEnded(audience, liveness)) {
|
|
1138
1303
|
const nextHop = question.hop + 1;
|
|
1139
1304
|
const nextAudience = resolveQuestionAudience({ ...question, hop: nextHop }, readLaneSenior, readSessionSenior);
|
|
1140
1305
|
const result = casQuestionIf(hqRoot, question.question_id, (current) => current.status === "pending" && current.hop === question.hop, (current) => {
|
|
@@ -1147,6 +1312,7 @@ export async function runPassBody(hqRoot, deps) {
|
|
|
1147
1312
|
return {
|
|
1148
1313
|
...current,
|
|
1149
1314
|
hop: nextHop,
|
|
1315
|
+
delivered_at: null,
|
|
1150
1316
|
routing_history: appendCappedHistory(current.routing_history, entry),
|
|
1151
1317
|
};
|
|
1152
1318
|
}, deps.dryRun);
|
|
@@ -1190,6 +1356,7 @@ export async function runPassBody(hqRoot, deps) {
|
|
|
1190
1356
|
return {
|
|
1191
1357
|
...current,
|
|
1192
1358
|
hop: current.hop + 1,
|
|
1359
|
+
delivered_at: null,
|
|
1193
1360
|
routing_history: appendCappedHistory(current.routing_history, entry),
|
|
1194
1361
|
};
|
|
1195
1362
|
}, deps.dryRun);
|
|
@@ -92,6 +92,8 @@ export interface SpawnLaneDeps {
|
|
|
92
92
|
runCompaction?: RunCompactionFn;
|
|
93
93
|
/** Finds a session transcript under an account home. Injected in tests. */
|
|
94
94
|
locateTranscript?: (home: string, sessionId: string) => string | null;
|
|
95
|
+
/** Internal loop-lane override for the single provider turn's working directory. */
|
|
96
|
+
workingDirectory?: string;
|
|
95
97
|
}
|
|
96
98
|
export interface ProviderLaunch {
|
|
97
99
|
command: string;
|
|
@@ -128,6 +130,8 @@ export declare function readProcStarttimeDetailed(pid: number): {
|
|
|
128
130
|
starttime: number | null;
|
|
129
131
|
error: string | null;
|
|
130
132
|
};
|
|
133
|
+
/** macOS has no /proc; use ps's process start time as the PID identity token. */
|
|
134
|
+
export declare function parsePsStarttime(value: string): number | null;
|
|
131
135
|
export declare function readProcStarttime(pid: number): number | null;
|
|
132
136
|
export declare function parsePidRecord(text: string): PidRecord | null;
|
|
133
137
|
export declare function readPidRecord(filePath: string): PidRecord | null;
|
package/dist/lib/lanes/spawn.js
CHANGED
|
@@ -31,6 +31,7 @@ import { appendCappedHistory, listLaneIds, readLane, retryUpdateLane } from "./s
|
|
|
31
31
|
import { managerStatusGateHeartbeat } from "./manager-status-gate.js";
|
|
32
32
|
import { LANE_QUOTA_ACCOUNT_FAILOVER_FLAG } from "./flag-keys.js";
|
|
33
33
|
import { readLanesRegistryFlagFailOff } from "./registry-flag.js";
|
|
34
|
+
import { LANE_WORKER_EXIT_PATH_ENV } from "./worker-process.js";
|
|
34
35
|
import { laneCompanyUidSource, resolveLaneCompany } from "./lane-company.js";
|
|
35
36
|
import { LEAD_LANE_KEEP_ALIVE_BLOCKED_RESUME_TEXT } from "./keep-alive.js";
|
|
36
37
|
import { copySessionFiles, planSessionCopy } from "./move-account.js";
|
|
@@ -932,6 +933,16 @@ export function readProcStarttimeDetailed(pid) {
|
|
|
932
933
|
}
|
|
933
934
|
catch (err) {
|
|
934
935
|
const code = err.code;
|
|
936
|
+
if (code === "ENOENT" && process.platform === "darwin") {
|
|
937
|
+
const result = spawnSync("ps", ["-p", String(pid), "-o", "lstart="], {
|
|
938
|
+
encoding: "utf8",
|
|
939
|
+
windowsHide: true,
|
|
940
|
+
});
|
|
941
|
+
const starttime = parsePsStarttime(result.status === 0 ? result.stdout : "");
|
|
942
|
+
return starttime === null
|
|
943
|
+
? { starttime: null, error: result.status === 0 ? "invalid_ps_starttime" : "ENOENT" }
|
|
944
|
+
: { starttime, error: null };
|
|
945
|
+
}
|
|
935
946
|
if (code === "ENOENT")
|
|
936
947
|
return { starttime: null, error: "ENOENT" };
|
|
937
948
|
process.stderr.write(`hq lanes: failed to read starttime for pid ${pid}: ${err instanceof Error ? err.message : String(err)}\n`);
|
|
@@ -941,6 +952,14 @@ export function readProcStarttimeDetailed(pid) {
|
|
|
941
952
|
};
|
|
942
953
|
}
|
|
943
954
|
}
|
|
955
|
+
/** macOS has no /proc; use ps's process start time as the PID identity token. */
|
|
956
|
+
export function parsePsStarttime(value) {
|
|
957
|
+
const trimmed = value.trim();
|
|
958
|
+
if (!trimmed)
|
|
959
|
+
return null;
|
|
960
|
+
const parsed = Date.parse(trimmed);
|
|
961
|
+
return Number.isFinite(parsed) ? parsed : null;
|
|
962
|
+
}
|
|
944
963
|
export function readProcStarttime(pid) {
|
|
945
964
|
return readProcStarttimeDetailed(pid).starttime;
|
|
946
965
|
}
|
|
@@ -1698,6 +1717,17 @@ function spawnDetached(spawnProcess, command, args, env, cwd, stdoutPath, stderr
|
|
|
1698
1717
|
closeFd(errFd, stderrPath);
|
|
1699
1718
|
}
|
|
1700
1719
|
}
|
|
1720
|
+
function spawnTrackedWorker(spawnProcess, command, args, env, cwd, stdoutPath, stderrPath, exitPath) {
|
|
1721
|
+
try {
|
|
1722
|
+
fs.unlinkSync(exitPath);
|
|
1723
|
+
}
|
|
1724
|
+
catch (error) {
|
|
1725
|
+
if (error.code !== "ENOENT")
|
|
1726
|
+
throw error;
|
|
1727
|
+
}
|
|
1728
|
+
const runnerPath = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../../../dist/lib/lanes/worker-process.js");
|
|
1729
|
+
return spawnDetached(spawnProcess, process.execPath, [runnerPath, command, ...args], { ...env, [LANE_WORKER_EXIT_PATH_ENV]: exitPath }, cwd, stdoutPath, stderrPath);
|
|
1730
|
+
}
|
|
1701
1731
|
function requireDetached(proveDetached, pid, kind) {
|
|
1702
1732
|
const proof = proveDetached(pid);
|
|
1703
1733
|
if (proof.detached)
|
|
@@ -1963,13 +1993,15 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
|
|
|
1963
1993
|
// state and the caller sees the error. Admission and question resume keep
|
|
1964
1994
|
// today's behaviour of naming the terminal reason on the record.
|
|
1965
1995
|
const markPreSpawnFailure = trigger.kind !== "message" && trigger.kind !== "resume";
|
|
1966
|
-
const plan = pendingResume?.
|
|
1967
|
-
?
|
|
1968
|
-
|
|
1969
|
-
|
|
1970
|
-
|
|
1971
|
-
|
|
1972
|
-
:
|
|
1996
|
+
const plan = pendingResume?.fresh_thread === true
|
|
1997
|
+
? { mode: "fresh" }
|
|
1998
|
+
: pendingResume?.trigger === "answered_questions"
|
|
1999
|
+
? pendingResume.session_id !== undefined && providerSupportsSessionResume(existing.provider)
|
|
2000
|
+
? { mode: "resume", session_id: pendingResume.session_id }
|
|
2001
|
+
: planLaneLaunch(existing, { kind: "answered_questions" })
|
|
2002
|
+
: pendingResume?.session_id !== undefined && providerSupportsSessionResume(existing.provider)
|
|
2003
|
+
? { mode: "resume", session_id: pendingResume.session_id }
|
|
2004
|
+
: planLaneLaunch(existing, trigger);
|
|
1973
2005
|
if (existing.host.kind === "outpost") {
|
|
1974
2006
|
refuseOutpostSelfTarget(hqRoot, existing, deps);
|
|
1975
2007
|
}
|
|
@@ -2572,7 +2604,7 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
|
|
|
2572
2604
|
let workerPid;
|
|
2573
2605
|
let watcherPid;
|
|
2574
2606
|
try {
|
|
2575
|
-
workerPid =
|
|
2607
|
+
workerPid = spawnTrackedWorker(spawnProcess, command.command, command.args, childEnv, deps.workingDirectory ?? pendingResume?.working_directory ?? hqRoot, path.join(logsDir, "worker.out"), path.join(logsDir, "worker.err"), path.join(logsDir, "worker-exit.json"));
|
|
2576
2608
|
writePidRecord(path.join(runDir, "worker.pid"), {
|
|
2577
2609
|
pid: workerPid,
|
|
2578
2610
|
starttime: readProcStarttime(workerPid) ?? undefined,
|
|
@@ -2881,7 +2913,7 @@ export function spawnEnvelopeRepair(hqRoot, lane, input, deps = {}) {
|
|
|
2881
2913
|
const previous = readSpawnManifest(runDir);
|
|
2882
2914
|
let workerPid;
|
|
2883
2915
|
try {
|
|
2884
|
-
workerPid =
|
|
2916
|
+
workerPid = spawnTrackedWorker(spawnProcess, command.command, command.args, childEnv, hqRoot, path.join(logsDir, "worker.out"), path.join(logsDir, "worker.err"), path.join(logsDir, "worker-exit.json"));
|
|
2885
2917
|
writePidRecord(path.join(runDir, "worker.pid"), {
|
|
2886
2918
|
pid: workerPid,
|
|
2887
2919
|
starttime: readProcStarttime(workerPid) ?? undefined,
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { type LaneRecord } from "./types.js";
|
|
2
|
+
export interface LaneStatusRowFields {
|
|
3
|
+
last_line: string | null;
|
|
4
|
+
pr: string | null;
|
|
5
|
+
inbox_pending: number | null;
|
|
6
|
+
elapsed_s: number | null;
|
|
7
|
+
exit: number | null;
|
|
8
|
+
}
|
|
9
|
+
export type LaneStatusRow = LaneRecord & Partial<LaneStatusRowFields>;
|
|
10
|
+
/** Bounded, local-only status data. Remote-owned lanes report null for file-backed fields. */
|
|
11
|
+
export declare function readLaneStatusRowFields(hqRoot: string, lane: LaneRecord, nowMs?: number, local?: boolean): LaneStatusRowFields;
|
|
12
|
+
//# sourceMappingURL=status-fields.d.ts.map
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
import * as fs from "node:fs";
|
|
2
|
+
import * as path from "node:path";
|
|
3
|
+
import { assertNoSecretText } from "./text-scan.js";
|
|
4
|
+
import { compareEnvelopeNames } from "./envelope-sequence.js";
|
|
5
|
+
import { isTerminalLaneState } from "./types.js";
|
|
6
|
+
const OUTPUT_TAIL_BYTES = 64 * 1024;
|
|
7
|
+
const INBOX_TAIL_BYTES = 32 * 1024;
|
|
8
|
+
const PR_URL = /https:\/\/github\.com\/[\w.-]+\/[\w.-]+\/pull\/\d+/g;
|
|
9
|
+
function readTail(filePath, maxBytes) {
|
|
10
|
+
let fd;
|
|
11
|
+
try {
|
|
12
|
+
fd = fs.openSync(filePath, "r");
|
|
13
|
+
const stat = fs.fstatSync(fd);
|
|
14
|
+
const size = stat.size;
|
|
15
|
+
const start = Math.max(0, size - maxBytes);
|
|
16
|
+
const buffer = Buffer.alloc(size - start);
|
|
17
|
+
fs.readSync(fd, buffer, 0, buffer.length, start);
|
|
18
|
+
return { text: buffer.toString("utf8"), size, mtimeMs: stat.mtimeMs };
|
|
19
|
+
}
|
|
20
|
+
catch (error) {
|
|
21
|
+
if (error.code !== "ENOENT") {
|
|
22
|
+
process.stderr.write(`hq lanes: could not read status tail ${filePath}: ${error instanceof Error ? error.message : String(error)}\n`);
|
|
23
|
+
}
|
|
24
|
+
return null;
|
|
25
|
+
}
|
|
26
|
+
finally {
|
|
27
|
+
if (fd !== undefined)
|
|
28
|
+
fs.closeSync(fd);
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
function candidateLines(value, depth = 0) {
|
|
32
|
+
if (depth > 4 || value === null || value === undefined)
|
|
33
|
+
return [];
|
|
34
|
+
if (typeof value === "string")
|
|
35
|
+
return value.split(/\r?\n/);
|
|
36
|
+
if (Array.isArray(value))
|
|
37
|
+
return value.flatMap((item) => candidateLines(item, depth + 1));
|
|
38
|
+
if (typeof value !== "object")
|
|
39
|
+
return [];
|
|
40
|
+
const record = value;
|
|
41
|
+
const direct = ["text", "content", "message", "last_assistant_message", "lastAssistantMessage"]
|
|
42
|
+
.flatMap((key) => candidateLines(record[key], depth + 1));
|
|
43
|
+
return direct.length > 0 ? direct : ["item", "message", "response", "result"]
|
|
44
|
+
.flatMap((key) => candidateLines(record[key], depth + 1));
|
|
45
|
+
}
|
|
46
|
+
function meaningfulLines(text) {
|
|
47
|
+
const lines = [];
|
|
48
|
+
for (const raw of text.split(/\r?\n/)) {
|
|
49
|
+
// eslint-disable-next-line no-control-regex
|
|
50
|
+
const line = raw.replace(/\x1b\[[0-9;]*m/g, "").trim();
|
|
51
|
+
if (!line || line.startsWith("{") || line.startsWith("[") || line.includes("cache_read_input_tokens"))
|
|
52
|
+
continue;
|
|
53
|
+
lines.push(line);
|
|
54
|
+
}
|
|
55
|
+
return lines;
|
|
56
|
+
}
|
|
57
|
+
function linesFromOutput(text) {
|
|
58
|
+
const output = [];
|
|
59
|
+
for (const raw of text.split(/\r?\n/)) {
|
|
60
|
+
if (!raw.trim())
|
|
61
|
+
continue;
|
|
62
|
+
try {
|
|
63
|
+
const parsed = JSON.parse(raw);
|
|
64
|
+
output.push(...meaningfulLines(candidateLines(parsed).join("\n")));
|
|
65
|
+
}
|
|
66
|
+
catch {
|
|
67
|
+
output.push(...meaningfulLines(raw));
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
return output;
|
|
71
|
+
}
|
|
72
|
+
function safeLastLine(lines) {
|
|
73
|
+
for (let i = lines.length - 1; i >= 0; i -= 1) {
|
|
74
|
+
const line = lines[i].trim();
|
|
75
|
+
if (!line)
|
|
76
|
+
continue;
|
|
77
|
+
const clipped = line.length > 160 ? `${line.slice(0, 157)}…` : line;
|
|
78
|
+
try {
|
|
79
|
+
// Same secret detector used by the lane question flow. Drop unsafe output.
|
|
80
|
+
assertNoSecretText("last_line", clipped);
|
|
81
|
+
return clipped;
|
|
82
|
+
}
|
|
83
|
+
catch {
|
|
84
|
+
continue;
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
return null;
|
|
88
|
+
}
|
|
89
|
+
function lastPr(text) {
|
|
90
|
+
const matches = [...text.matchAll(PR_URL)];
|
|
91
|
+
return matches.length ? matches[matches.length - 1][0] : null;
|
|
92
|
+
}
|
|
93
|
+
function terminalExit(lane, runDir) {
|
|
94
|
+
if (!isTerminalLaneState(lane.state))
|
|
95
|
+
return null;
|
|
96
|
+
const exitRecord = readTail(path.join(runDir, "logs", "worker-exit.json"), 4 * 1024);
|
|
97
|
+
if (exitRecord !== null) {
|
|
98
|
+
try {
|
|
99
|
+
const parsed = JSON.parse(exitRecord.text);
|
|
100
|
+
if (typeof parsed.exit_code === "number" && Number.isInteger(parsed.exit_code))
|
|
101
|
+
return parsed.exit_code;
|
|
102
|
+
}
|
|
103
|
+
catch {
|
|
104
|
+
process.stderr.write(`hq lanes: worker exit record is invalid for ${lane.lane_id}\n`);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
const row = lane;
|
|
108
|
+
const candidates = [row.worker_exit_code, row.exit_code, row.terminal_details?.exit_code];
|
|
109
|
+
return candidates.find((value) => Number.isInteger(value)) ?? null;
|
|
110
|
+
}
|
|
111
|
+
function startTimeMs(lane, runDir) {
|
|
112
|
+
const row = lane;
|
|
113
|
+
for (const value of [row.worker_started_at, row.started_at]) {
|
|
114
|
+
const parsed = typeof value === "number" ? value * 1000 : typeof value === "string" ? Date.parse(value) : NaN;
|
|
115
|
+
if (Number.isFinite(parsed) && parsed > 0)
|
|
116
|
+
return parsed;
|
|
117
|
+
}
|
|
118
|
+
try {
|
|
119
|
+
return fs.statSync(path.join(runDir, "worker.pid")).mtimeMs;
|
|
120
|
+
}
|
|
121
|
+
catch (error) {
|
|
122
|
+
if (error.code !== "ENOENT") {
|
|
123
|
+
process.stderr.write(`hq lanes: could not read worker start time ${runDir}: ${error instanceof Error ? error.message : String(error)}\n`);
|
|
124
|
+
}
|
|
125
|
+
return null;
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
/** Bounded, local-only status data. Remote-owned lanes report null for file-backed fields. */
|
|
129
|
+
export function readLaneStatusRowFields(hqRoot, lane, nowMs = Date.now(), local = true) {
|
|
130
|
+
if (!local)
|
|
131
|
+
return { last_line: null, pr: null, inbox_pending: null, elapsed_s: null, exit: null };
|
|
132
|
+
const runDir = path.join(hqRoot, "workspace", "lanes-runs", lane.lane_id);
|
|
133
|
+
const logsDir = path.join(runDir, "logs");
|
|
134
|
+
const outTail = readTail(path.join(logsDir, "worker.out"), OUTPUT_TAIL_BYTES);
|
|
135
|
+
const messageTail = readTail(path.join(logsDir, "last-message.txt"), OUTPUT_TAIL_BYTES);
|
|
136
|
+
const outputTails = [outTail, messageTail]
|
|
137
|
+
.filter((value) => value !== null)
|
|
138
|
+
.sort((left, right) => left.mtimeMs - right.mtimeMs);
|
|
139
|
+
const lines = outputTails.flatMap(({ text }) => linesFromOutput(text));
|
|
140
|
+
let envelopeTail = null;
|
|
141
|
+
try {
|
|
142
|
+
const names = fs.readdirSync(path.join(runDir, "envelopes"))
|
|
143
|
+
.filter((name) => name.endsWith(".json"))
|
|
144
|
+
.sort(compareEnvelopeNames);
|
|
145
|
+
const latest = names.at(-1);
|
|
146
|
+
if (latest)
|
|
147
|
+
envelopeTail = readTail(path.join(runDir, "envelopes", latest), OUTPUT_TAIL_BYTES);
|
|
148
|
+
}
|
|
149
|
+
catch (error) {
|
|
150
|
+
if (error.code !== "ENOENT") {
|
|
151
|
+
process.stderr.write(`hq lanes: could not read status envelopes for ${lane.lane_id}: ${error instanceof Error ? error.message : String(error)}\n`);
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
const prSources = [
|
|
155
|
+
...outputTails.map(({ text, mtimeMs }) => ({ text, mtimeMs })),
|
|
156
|
+
...(envelopeTail ? [{ text: envelopeTail.text, mtimeMs: envelopeTail.mtimeMs }] : []),
|
|
157
|
+
].sort((left, right) => left.mtimeMs - right.mtimeMs);
|
|
158
|
+
const pr = lastPr(prSources.map(({ text }) => text).join("\n"));
|
|
159
|
+
const inboxTail = readTail(path.join(runDir, "dropbox", "inbox.jsonl"), INBOX_TAIL_BYTES);
|
|
160
|
+
const inboxPending = inboxTail === null ? 0
|
|
161
|
+
: inboxTail.size > INBOX_TAIL_BYTES ? null
|
|
162
|
+
: inboxTail.text.split(/\r?\n/).filter((line) => {
|
|
163
|
+
if (!line.trim())
|
|
164
|
+
return false;
|
|
165
|
+
try {
|
|
166
|
+
return JSON.parse(line) !== null;
|
|
167
|
+
}
|
|
168
|
+
catch {
|
|
169
|
+
return false;
|
|
170
|
+
}
|
|
171
|
+
}).length;
|
|
172
|
+
const startedAt = startTimeMs(lane, runDir);
|
|
173
|
+
const terminal = isTerminalLaneState(lane.state);
|
|
174
|
+
const terminalEndAt = [lane.ended_at, lane.updated_at]
|
|
175
|
+
.map((value) => typeof value === "string" ? Date.parse(value) : NaN)
|
|
176
|
+
.find(Number.isFinite);
|
|
177
|
+
const endedAt = terminal ? terminalEndAt ?? nowMs : nowMs;
|
|
178
|
+
const elapsed = startedAt === null || !Number.isFinite(endedAt)
|
|
179
|
+
? null
|
|
180
|
+
: Math.max(0, Math.floor((endedAt - startedAt) / 1000));
|
|
181
|
+
return {
|
|
182
|
+
last_line: safeLastLine(lines),
|
|
183
|
+
pr,
|
|
184
|
+
inbox_pending: inboxPending,
|
|
185
|
+
elapsed_s: elapsed,
|
|
186
|
+
exit: terminalExit(lane, runDir),
|
|
187
|
+
};
|
|
188
|
+
}
|
|
189
|
+
//# sourceMappingURL=status-fields.js.map
|