@indigoai-us/hq-cli 5.154.0 → 5.156.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +48 -0
- package/dist/command-catalog.generated.d.ts +4 -1
- package/dist/command-catalog.generated.js +5 -1
- package/dist/commands/lanes.d.ts +3 -11
- package/dist/commands/lanes.js +162 -55
- package/dist/commands/mesh.js +35 -8
- package/dist/lib/lanes/accounts.d.ts +16 -11
- package/dist/lib/lanes/accounts.js +50 -8
- package/dist/lib/lanes/admission-host.d.ts +19 -0
- package/dist/lib/lanes/admission-host.js +33 -0
- package/dist/lib/lanes/config.js +25 -0
- package/dist/lib/lanes/mesh-events.d.ts +7 -3
- package/dist/lib/lanes/mesh-events.js +116 -20
- package/dist/lib/lanes/provider-rejection.d.ts +4 -4
- package/dist/lib/lanes/provider-rejection.js +24 -6
- package/dist/lib/lanes/reconcile.js +85 -2
- package/dist/lib/lanes/spawn.d.ts +9 -1
- package/dist/lib/lanes/spawn.js +420 -14
- package/dist/lib/lanes/types.d.ts +7 -3
- package/dist/lib/lanes/types.js +3 -2
- package/dist/lib/lanes/watcher.d.ts +8 -1
- package/dist/lib/lanes/watcher.js +66 -17
- package/dist/lib/mesh/api.d.ts +27 -1
- package/dist/lib/mesh/api.js +41 -12
- package/dist/lib/mesh/project-board.d.ts +1 -0
- package/dist/lib/mesh/project-board.js +1 -1
- package/package.json +1 -1
|
@@ -42,7 +42,7 @@
|
|
|
42
42
|
* of opening a browser nobody is watching, and the watcher records
|
|
43
43
|
* `mesh_event_failed` and moves on. No credential file is read here.
|
|
44
44
|
*/
|
|
45
|
-
import { appendThreadEvent, clamp, ensureProjectThread, getProjectView, patchStoryStatus, putProjectView, requireToken, registerMeshProject, resolveMeshCompany, } from "../mesh/api.js";
|
|
45
|
+
import { appendThreadEvent, clamp, ensureLaneMeshThread, ensureProjectThread, getProjectView, patchStoryStatus, putProjectView, requireToken, registerMeshProject, resolveMeshCompany, } from "../mesh/api.js";
|
|
46
46
|
import { LanesError } from "./errors.js";
|
|
47
47
|
export const LANE_MESH_EVENTS = [
|
|
48
48
|
"lane_started",
|
|
@@ -52,12 +52,17 @@ export const LANE_MESH_EVENTS = [
|
|
|
52
52
|
"lane_ask",
|
|
53
53
|
"lane_blocked",
|
|
54
54
|
"lane_done",
|
|
55
|
+
"lane_failed",
|
|
56
|
+
"lane_interrupted",
|
|
57
|
+
"lane_cancelled",
|
|
58
|
+
"lane_killed",
|
|
55
59
|
];
|
|
56
60
|
/** Roster kinds this module emits. Subset of the server's `validKinds`. */
|
|
57
61
|
export const WIRE_EVENT_KINDS = [
|
|
58
62
|
"run_started",
|
|
59
63
|
"run_progress",
|
|
60
64
|
"run_complete",
|
|
65
|
+
"blocked",
|
|
61
66
|
"question",
|
|
62
67
|
"note",
|
|
63
68
|
];
|
|
@@ -94,6 +99,30 @@ function nonEmpty(text, fallback) {
|
|
|
94
99
|
export function toWireThreadEvent(event, payload) {
|
|
95
100
|
const kind = parseLaneMeshEvent(event);
|
|
96
101
|
const id = identity(payload);
|
|
102
|
+
const laneId = str(payload.lane_id ?? payload.laneId).trim();
|
|
103
|
+
const senior = payload.senior && typeof payload.senior === "object" && !Array.isArray(payload.senior)
|
|
104
|
+
? payload.senior
|
|
105
|
+
: undefined;
|
|
106
|
+
const lane = laneId
|
|
107
|
+
? {
|
|
108
|
+
laneId,
|
|
109
|
+
...(str(payload.story_id ?? payload.storyId) ? { storyId: str(payload.story_id ?? payload.storyId) } : {}),
|
|
110
|
+
...(str(payload.worker_profile ?? payload.workerProfile) ? { workerProfile: str(payload.worker_profile ?? payload.workerProfile) } : {}),
|
|
111
|
+
...(str(payload.model) ? { model: str(payload.model) } : {}),
|
|
112
|
+
...(str(payload.account) ? { account: str(payload.account) } : {}),
|
|
113
|
+
...(str(payload.parent_lane_id ?? payload.parentLaneId) ? { parentLaneId: str(payload.parent_lane_id ?? payload.parentLaneId) } : {}),
|
|
114
|
+
...(senior && ["user", "session", "lane"].includes(String(senior.kind))
|
|
115
|
+
? { senior: { kind: senior.kind, ...(typeof senior.id === "string" ? { id: senior.id } : {}) } }
|
|
116
|
+
: {}),
|
|
117
|
+
laneVersion: Number.isSafeInteger(Number(payload.lane_version ?? payload.laneVersion)) && Number(payload.lane_version ?? payload.laneVersion) > 0
|
|
118
|
+
? Number(payload.lane_version ?? payload.laneVersion)
|
|
119
|
+
: 1,
|
|
120
|
+
}
|
|
121
|
+
: undefined;
|
|
122
|
+
const laneFields = (laneStatus) => ({
|
|
123
|
+
...(lane ? { lane } : {}),
|
|
124
|
+
...(laneStatus ? { laneStatus } : {}),
|
|
125
|
+
});
|
|
97
126
|
switch (kind) {
|
|
98
127
|
case "lane_started":
|
|
99
128
|
return {
|
|
@@ -101,6 +130,7 @@ export function toWireThreadEvent(event, payload) {
|
|
|
101
130
|
payload: {
|
|
102
131
|
title: clamp(`lane_started ${str(payload.lane_id) || "unknown-lane"}`, WIRE_ACTIVITY_TEXT_MAX),
|
|
103
132
|
summary: clamp(`Lane ${id} started`, WIRE_ACTIVITY_TEXT_MAX),
|
|
133
|
+
...laneFields("running"),
|
|
104
134
|
},
|
|
105
135
|
};
|
|
106
136
|
case "lane_progress": {
|
|
@@ -109,29 +139,47 @@ export function toWireThreadEvent(event, payload) {
|
|
|
109
139
|
eventKind: "run_progress",
|
|
110
140
|
payload: {
|
|
111
141
|
summary: clamp(nonEmpty(`lane_progress ${id}${detail ? `: ${detail}` : ""}`, "lane_progress"), WIRE_ACTIVITY_TEXT_MAX),
|
|
142
|
+
...laneFields("running"),
|
|
112
143
|
},
|
|
113
144
|
};
|
|
114
145
|
}
|
|
115
146
|
case "lane_done": {
|
|
116
147
|
const summary = str(payload.summary);
|
|
148
|
+
const suppliedStatus = str(payload.lane_status ?? payload.laneStatus);
|
|
149
|
+
const laneStatus = isMeshLaneStatus(suppliedStatus) ? suppliedStatus : "done";
|
|
117
150
|
return {
|
|
118
151
|
eventKind: "run_complete",
|
|
119
152
|
payload: {
|
|
120
153
|
title: clamp(`lane_done ${str(payload.lane_id) || "unknown-lane"}`, WIRE_ACTIVITY_TEXT_MAX),
|
|
121
154
|
summary: clamp(nonEmpty(`${id}: ${summary}`.replace(/:\s*$/, ""), `lane_done ${id}`), WIRE_ACTIVITY_TEXT_MAX),
|
|
155
|
+
...laneFields(laneStatus),
|
|
122
156
|
},
|
|
123
157
|
};
|
|
124
158
|
}
|
|
125
159
|
case "lane_blocked": {
|
|
126
|
-
const reason = str(payload.blocked_reason);
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
// audit note kind to preserve a filterable blocked marker.
|
|
160
|
+
const reason = str(payload.blocked_reason ?? payload.reason);
|
|
161
|
+
const suppliedStatus = str(payload.lane_status ?? payload.laneStatus);
|
|
162
|
+
const laneStatus = isMeshLaneStatus(suppliedStatus) ? suppliedStatus : "blocked";
|
|
130
163
|
return {
|
|
131
|
-
eventKind: "
|
|
164
|
+
eventKind: "blocked",
|
|
132
165
|
payload: {
|
|
133
|
-
|
|
134
|
-
|
|
166
|
+
reason: clamp(`lane_blocked ${id}${reason ? `: ${reason}` : ""}`, WIRE_NOTE_TEXT_MAX),
|
|
167
|
+
...laneFields(laneStatus),
|
|
168
|
+
},
|
|
169
|
+
};
|
|
170
|
+
}
|
|
171
|
+
case "lane_failed":
|
|
172
|
+
case "lane_interrupted":
|
|
173
|
+
case "lane_cancelled":
|
|
174
|
+
case "lane_killed": {
|
|
175
|
+
const laneStatus = kind.slice("lane_".length);
|
|
176
|
+
const summary = str(payload.summary ?? payload.reason ?? payload.blocked_reason);
|
|
177
|
+
return {
|
|
178
|
+
eventKind: "run_complete",
|
|
179
|
+
payload: {
|
|
180
|
+
title: clamp(`${kind} ${str(payload.lane_id) || "unknown-lane"}`, WIRE_ACTIVITY_TEXT_MAX),
|
|
181
|
+
summary: clamp(nonEmpty(`${id}: ${summary}`.replace(/:\s*$/, ""), `${kind} ${id}`), WIRE_ACTIVITY_TEXT_MAX),
|
|
182
|
+
...laneFields(laneStatus),
|
|
135
183
|
},
|
|
136
184
|
};
|
|
137
185
|
}
|
|
@@ -144,6 +192,7 @@ export function toWireThreadEvent(event, payload) {
|
|
|
144
192
|
};
|
|
145
193
|
if (questionId)
|
|
146
194
|
out.questionId = questionId;
|
|
195
|
+
Object.assign(out, laneFields());
|
|
147
196
|
return { eventKind: "question", payload: out };
|
|
148
197
|
}
|
|
149
198
|
case "lane_ask": {
|
|
@@ -155,6 +204,7 @@ export function toWireThreadEvent(event, payload) {
|
|
|
155
204
|
};
|
|
156
205
|
if (ids[0])
|
|
157
206
|
out.questionId = ids[0];
|
|
207
|
+
Object.assign(out, laneFields("awaiting_input"));
|
|
158
208
|
return { eventKind: "question", payload: out };
|
|
159
209
|
}
|
|
160
210
|
default: {
|
|
@@ -224,12 +274,28 @@ export function createLaneMesh(opts = {}) {
|
|
|
224
274
|
return {
|
|
225
275
|
async ensureProjectThread(company, projectId) {
|
|
226
276
|
const tok = await token();
|
|
227
|
-
const resolved = await resolveMeshCompany(tok, company.
|
|
277
|
+
const resolved = await resolveMeshCompany(tok, company.uid);
|
|
228
278
|
const ensured = await ensureProjectThread(tok, resolved, projectId, {
|
|
229
279
|
summary: `HQ lane ${projectId}`,
|
|
230
280
|
});
|
|
231
281
|
return ensured.threadId;
|
|
232
282
|
},
|
|
283
|
+
async ensureLaneThread(company, lane, laneStatus = lane.state, signal) {
|
|
284
|
+
const tok = await token();
|
|
285
|
+
const resolved = await resolveMeshCompany(tok, company.uid);
|
|
286
|
+
const descriptor = {
|
|
287
|
+
laneId: lane.lane_id,
|
|
288
|
+
...(lane.story_id ? { storyId: lane.story_id } : {}),
|
|
289
|
+
...(lane.worker_profile ? { workerProfile: lane.worker_profile } : {}),
|
|
290
|
+
...(lane.model ? { model: lane.model } : {}),
|
|
291
|
+
...(lane.account ? { account: lane.account } : {}),
|
|
292
|
+
...(lane.senior.kind === "lane" ? { parentLaneId: lane.senior.id } : {}),
|
|
293
|
+
senior: { kind: lane.senior.kind, id: lane.senior.id },
|
|
294
|
+
laneVersion: lane.version,
|
|
295
|
+
};
|
|
296
|
+
const ensured = await ensureLaneMeshThread(tok, resolved, lane.project_id, descriptor, isMeshLaneStatus(laneStatus) ? laneStatus : "running", signal);
|
|
297
|
+
return ensured.threadId;
|
|
298
|
+
},
|
|
233
299
|
async appendThreadEvent(company, threadId, event, payload, signal) {
|
|
234
300
|
const wire = toWireThreadEvent(event, payload);
|
|
235
301
|
const tok = await token();
|
|
@@ -263,34 +329,51 @@ export function laneMeshCompany(lane) {
|
|
|
263
329
|
export function laneCanEmit(lane) {
|
|
264
330
|
return laneMeshCompany(lane) !== undefined && !!lane.thread_id;
|
|
265
331
|
}
|
|
266
|
-
function
|
|
332
|
+
export function laneMeshEventIdentity(lane) {
|
|
267
333
|
const lastMove = lane.account_history?.[lane.account_history.length - 1];
|
|
268
334
|
return {
|
|
269
335
|
lane_id: lane.lane_id,
|
|
270
336
|
project_id: lane.project_id,
|
|
271
337
|
story_id: lane.story_id,
|
|
338
|
+
worker_profile: lane.worker_profile,
|
|
339
|
+
model: lane.model,
|
|
340
|
+
lane_version: lane.version,
|
|
341
|
+
senior: { kind: lane.senior.kind, id: lane.senior.id },
|
|
342
|
+
...(lane.senior.kind === "lane" ? { parent_lane_id: lane.senior.id } : {}),
|
|
272
343
|
...(lane.account !== undefined ? { account: lane.account } : {}),
|
|
273
344
|
// A moved lane names the move on every event its run emits (US-023 §8).
|
|
274
345
|
...(lastMove !== undefined ? { account_move: { from: lastMove.from, to: lastMove.to, at: lastMove.at } } : {}),
|
|
275
346
|
};
|
|
276
347
|
}
|
|
348
|
+
function isMeshLaneStatus(value) {
|
|
349
|
+
return ["queued", "running", "awaiting_input", "blocked", "done", "failed", "interrupted", "cancelled", "killed"].includes(value);
|
|
350
|
+
}
|
|
351
|
+
// hq-pro derives lane thread ids from company + lane identity; opaque ids may
|
|
352
|
+
// be legacy shared project threads and must be migrated before a lane closes.
|
|
353
|
+
const STABLE_LANE_THREAD_ID = /^lane-[0-9a-f]{32}$/;
|
|
354
|
+
async function ensureLaneEventThread(mesh, company, lane, laneStatus, signal) {
|
|
355
|
+
if (mesh.ensureLaneThread && !STABLE_LANE_THREAD_ID.test(lane.thread_id ?? "")) {
|
|
356
|
+
return mesh.ensureLaneThread(company, lane, laneStatus, signal);
|
|
357
|
+
}
|
|
358
|
+
return lane.thread_id;
|
|
359
|
+
}
|
|
277
360
|
/**
|
|
278
361
|
* Spec §8 terminal-envelope events. `done` → lane_done and story → review;
|
|
279
362
|
* `blocked` → lane_blocked; `ask` → lane_ask with the queue ids. The story is
|
|
280
363
|
* never set to `done` here: promotion belongs to the senior.
|
|
281
364
|
*/
|
|
282
365
|
export async function emitLaneDecisionEvents(mesh, lane, envelope, opts = {}) {
|
|
283
|
-
if (!laneCanEmit(lane))
|
|
284
|
-
return;
|
|
285
366
|
const company = laneMeshCompany(lane);
|
|
286
|
-
|
|
367
|
+
if (!company)
|
|
368
|
+
return;
|
|
369
|
+
const threadId = await ensureLaneEventThread(mesh, company, lane, lane.state, opts.signal);
|
|
287
370
|
if (!company || !threadId)
|
|
288
371
|
return;
|
|
289
372
|
switch (envelope.decision) {
|
|
290
373
|
case "done": {
|
|
291
374
|
const outcomes = await Promise.allSettled([
|
|
292
375
|
mesh.appendThreadEvent(company, threadId, "lane_done", {
|
|
293
|
-
...
|
|
376
|
+
...laneMeshEventIdentity(lane),
|
|
294
377
|
summary: envelope.summary,
|
|
295
378
|
}, opts.signal),
|
|
296
379
|
mesh.patchStoryStatus(company, lane.project_id, lane.story_id, "review"),
|
|
@@ -304,7 +387,7 @@ export async function emitLaneDecisionEvents(mesh, lane, envelope, opts = {}) {
|
|
|
304
387
|
{
|
|
305
388
|
const outcomes = await Promise.allSettled([
|
|
306
389
|
mesh.appendThreadEvent(company, threadId, "lane_blocked", {
|
|
307
|
-
...
|
|
390
|
+
...laneMeshEventIdentity(lane),
|
|
308
391
|
blocked_reason: envelope.blocked_reason ?? "",
|
|
309
392
|
}),
|
|
310
393
|
mesh.patchStoryStatus(company, lane.project_id, lane.story_id, "in_progress"),
|
|
@@ -322,7 +405,7 @@ export async function emitLaneDecisionEvents(mesh, lane, envelope, opts = {}) {
|
|
|
322
405
|
for (const questionId of opts.questionIds ?? []) {
|
|
323
406
|
try {
|
|
324
407
|
await mesh.appendThreadEvent(company, threadId, "lane_ask", {
|
|
325
|
-
...
|
|
408
|
+
...laneMeshEventIdentity(lane),
|
|
326
409
|
question_ids: [questionId],
|
|
327
410
|
}, opts.signal);
|
|
328
411
|
}
|
|
@@ -340,15 +423,28 @@ export async function emitLaneDecisionEvents(mesh, lane, envelope, opts = {}) {
|
|
|
340
423
|
}
|
|
341
424
|
}
|
|
342
425
|
}
|
|
343
|
-
|
|
344
|
-
|
|
426
|
+
/** Emit an abnormal terminal state even when the worker produced no envelope. */
|
|
427
|
+
export async function emitLaneTerminalStateEvent(mesh, lane, state, summary, signal) {
|
|
428
|
+
const company = laneMeshCompany(lane);
|
|
429
|
+
if (!company)
|
|
345
430
|
return;
|
|
431
|
+
const threadId = await ensureLaneEventThread(mesh, company, lane, state, signal);
|
|
432
|
+
if (!company || !threadId)
|
|
433
|
+
return;
|
|
434
|
+
await mesh.appendThreadEvent(company, threadId, `lane_${state}`, {
|
|
435
|
+
...laneMeshEventIdentity(lane),
|
|
436
|
+
...(summary ? { summary } : {}),
|
|
437
|
+
}, signal);
|
|
438
|
+
}
|
|
439
|
+
export async function emitLaneProgressEvent(mesh, lane, detail, signal) {
|
|
346
440
|
const company = laneMeshCompany(lane);
|
|
347
|
-
|
|
441
|
+
if (!company)
|
|
442
|
+
return;
|
|
443
|
+
const threadId = await ensureLaneEventThread(mesh, company, lane, lane.state, signal);
|
|
348
444
|
if (!company || !threadId)
|
|
349
445
|
return;
|
|
350
446
|
await mesh.appendThreadEvent(company, threadId, "lane_progress", {
|
|
351
|
-
...
|
|
447
|
+
...laneMeshEventIdentity(lane),
|
|
352
448
|
detail,
|
|
353
449
|
}, signal);
|
|
354
450
|
}
|
|
@@ -37,6 +37,7 @@ export interface ProviderRejection {
|
|
|
37
37
|
/** Which event on the stream carried the failure. */
|
|
38
38
|
event: "codex.turn.failed" | "claude.result.is_error" | "grok.error";
|
|
39
39
|
}
|
|
40
|
+
export declare function isProviderLoginFailure(failure: ProviderRejection | null): boolean;
|
|
40
41
|
export declare function capMessage(message: string): string;
|
|
41
42
|
/**
|
|
42
43
|
* Whether a later run could succeed without the operator changing anything.
|
|
@@ -56,10 +57,9 @@ export declare function retryableFromStatus(status: number | null): boolean;
|
|
|
56
57
|
export declare function parseCodexTurnFailed(text: string): ProviderRejection | null;
|
|
57
58
|
/**
|
|
58
59
|
* Claude `-p --output-format json`: the last `type: "result"` object with
|
|
59
|
-
* `is_error: true` and a numeric `api_error_status
|
|
60
|
-
*
|
|
61
|
-
*
|
|
62
|
-
* existing classification.
|
|
60
|
+
* `is_error: true` and a numeric `api_error_status`, or the CLI's explicit
|
|
61
|
+
* unauthenticated message. Other status-less errors keep the existing
|
|
62
|
+
* classification because they may be quota text or a worker failure.
|
|
63
63
|
*/
|
|
64
64
|
export declare function parseClaudeResultRejection(text: string): ProviderRejection | null;
|
|
65
65
|
/**
|
|
@@ -24,6 +24,25 @@ export const PROVIDER_REJECTED_REASON = "provider_rejected";
|
|
|
24
24
|
export const PROVIDER_REJECTION_MESSAGE_CAP = 2000;
|
|
25
25
|
/** Status the watcher routes to the quota path instead of `provider_rejected`. */
|
|
26
26
|
export const QUOTA_STATUS = 429;
|
|
27
|
+
/** Exact auth failures observed from each installed CLI in an empty home. */
|
|
28
|
+
function isProviderLoginMessage(provider, message) {
|
|
29
|
+
switch (provider) {
|
|
30
|
+
case "claude":
|
|
31
|
+
return /\bnot logged in\b/i.test(message);
|
|
32
|
+
case "codex":
|
|
33
|
+
return /\bnot logged in\b/i.test(message) ||
|
|
34
|
+
/\bmissing bearer or basic authentication in header\b/i.test(message);
|
|
35
|
+
case "grok":
|
|
36
|
+
return /^\s*not signed in\b/i.test(message);
|
|
37
|
+
default: {
|
|
38
|
+
const _exhaustive = provider;
|
|
39
|
+
return _exhaustive;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
export function isProviderLoginFailure(failure) {
|
|
44
|
+
return failure !== null && isProviderLoginMessage(failure.provider, failure.message);
|
|
45
|
+
}
|
|
27
46
|
function isPlainObject(value) {
|
|
28
47
|
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
29
48
|
}
|
|
@@ -134,10 +153,9 @@ export function parseCodexTurnFailed(text) {
|
|
|
134
153
|
}
|
|
135
154
|
/**
|
|
136
155
|
* Claude `-p --output-format json`: the last `type: "result"` object with
|
|
137
|
-
* `is_error: true` and a numeric `api_error_status
|
|
138
|
-
*
|
|
139
|
-
*
|
|
140
|
-
* existing classification.
|
|
156
|
+
* `is_error: true` and a numeric `api_error_status`, or the CLI's explicit
|
|
157
|
+
* unauthenticated message. Other status-less errors keep the existing
|
|
158
|
+
* classification because they may be quota text or a worker failure.
|
|
141
159
|
*/
|
|
142
160
|
export function parseClaudeResultRejection(text) {
|
|
143
161
|
// The last `result` decides: a success or a status-less error there is
|
|
@@ -148,9 +166,9 @@ export function parseClaudeResultRejection(text) {
|
|
|
148
166
|
if (rec.is_error !== true)
|
|
149
167
|
return NOT_REJECTED;
|
|
150
168
|
const status = statusOf(rec.api_error_status);
|
|
151
|
-
if (status === null)
|
|
152
|
-
return NOT_REJECTED;
|
|
153
169
|
const message = typeof rec.result === "string" ? rec.result : "";
|
|
170
|
+
if (status === null && !isProviderLoginMessage("claude", message))
|
|
171
|
+
return NOT_REJECTED;
|
|
154
172
|
return {
|
|
155
173
|
reason: PROVIDER_REJECTED_REASON,
|
|
156
174
|
provider: "claude",
|
|
@@ -15,13 +15,13 @@ import { MAX_LANE_DEPTH } from "./constants.js";
|
|
|
15
15
|
import { AccountResolutionError, LanesError, isLaneVersionConflict } from "./errors.js";
|
|
16
16
|
import { loadWorkerProfile } from "./profile.js";
|
|
17
17
|
import { classifySessionLiveness, defaultLivenessDeps, loadLivenessIndex, readDaemonPresenceConnected, } from "./liveness.js";
|
|
18
|
-
import { ANSWERED_QUESTIONS_FILE, armLaneWatcher, isPidRecordAlive, laneRunDir, readPidRecord, readSpawnManifest, markSpawnFailure, spawnEnvelopeRepair, spawnLaneWorker, } from "./spawn.js";
|
|
18
|
+
import { ANSWERED_QUESTIONS_FILE, armLaneWatcher, isPidRecordAlive, laneRunDir, readPidRecord, readSpawnManifest, markSpawnFailure, spawnEnvelopeRepair, spawnLaneWorker, retryLaneOnAlternateAccount, } from "./spawn.js";
|
|
19
19
|
import { appendCappedHistory, lanesStoreRoot, parseLaneRecord, parseQuestionRecord, readLane, readQuestion, updateLane, updateQuestion, } from "./store.js";
|
|
20
20
|
import { applyThrottleDecision, decideAdmission, defaultThrottleDeps, emptyThrottleState, evaluateThrottle, isCpuCheckEnabled, readAvailableMemMb, readThrottleState, resolveCpuBusyFraction, throttleStatePath, writeThrottleState, } from "./throttle.js";
|
|
21
21
|
import { applyDefaultQuotaHold, readQuotaSignal, recordAccountQuota } from "./quota.js";
|
|
22
22
|
import { answeredQuestionEntries, raiseEnvelopeQuestions } from "./questions.js";
|
|
23
23
|
import { SILENCE_MS, classifyLaneExit, isPidAlive, readLatestEnvelope, readLatestEnvelopeFile, readLatestEnvelopeRecord, } from "./watcher.js";
|
|
24
|
-
import { createLaneMesh, emitLaneDecisionEvents, laneMeshCompany } from "./mesh-events.js";
|
|
24
|
+
import { createLaneMesh, emitLaneDecisionEvents, emitLaneTerminalStateEvent, laneMeshCompany, } from "./mesh-events.js";
|
|
25
25
|
import { laneStories, storyStatusForEnvelope } from "./stories.js";
|
|
26
26
|
import { decideEnvelopeRepair, envelopeQuestionIds, routeProviderFailure } from "./watcher.js";
|
|
27
27
|
import { providerRejectionDetails, readProviderRejection, } from "./provider-rejection.js";
|
|
@@ -916,6 +916,11 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
|
|
|
916
916
|
const repairedClassification = invalidEnvelope
|
|
917
917
|
? classifyLaneExit({ envelope: null, hasPartialOutput: false, invalidEnvelope })
|
|
918
918
|
: classification;
|
|
919
|
+
const accountRetryReason = quota
|
|
920
|
+
? "usage_limit"
|
|
921
|
+
: providerRejection && classification.reason === "account_not_logged_in"
|
|
922
|
+
? "account_unavailable"
|
|
923
|
+
: null;
|
|
919
924
|
if (!deps.dryRun &&
|
|
920
925
|
quota &&
|
|
921
926
|
classified.account &&
|
|
@@ -923,6 +928,30 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
|
|
|
923
928
|
classification.reason === "account_model_unavailable")) {
|
|
924
929
|
recordAccountQuota(hqRoot, classified.provider, classified.account, quota);
|
|
925
930
|
}
|
|
931
|
+
if (!deps.dryRun && accountRetryReason && classified.account) {
|
|
932
|
+
try {
|
|
933
|
+
const recovered = await retryLaneOnAlternateAccount(hqRoot, classified, accountRetryReason, "reconcile", {
|
|
934
|
+
...(deps.spawnLaneDeps ?? {}),
|
|
935
|
+
admissionHost: deps.host,
|
|
936
|
+
});
|
|
937
|
+
summary.dead_host.push({
|
|
938
|
+
id: lane.lane_id,
|
|
939
|
+
from: "running",
|
|
940
|
+
to: recovered.state,
|
|
941
|
+
reason: accountRetryReason,
|
|
942
|
+
});
|
|
943
|
+
summary.heartbeats.push(`${lane.lane_id}:account-retry`);
|
|
944
|
+
continue;
|
|
945
|
+
}
|
|
946
|
+
catch (err) {
|
|
947
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
948
|
+
process.stderr.write(`hq lanes reconcile: account failover failed for ${lane.lane_id}: ${message}\n`);
|
|
949
|
+
if (readLane(hqRoot, lane.lane_id).state !== "running") {
|
|
950
|
+
summary.errors.push({ message: `${lane.lane_id}: account failover left a non-running record after ${message}` });
|
|
951
|
+
continue;
|
|
952
|
+
}
|
|
953
|
+
}
|
|
954
|
+
}
|
|
926
955
|
if (!deps.dryRun &&
|
|
927
956
|
classification.state === "awaiting_input" &&
|
|
928
957
|
latest &&
|
|
@@ -1003,6 +1032,34 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
|
|
|
1003
1032
|
}
|
|
1004
1033
|
}
|
|
1005
1034
|
}
|
|
1035
|
+
else if (!deps.dryRun &&
|
|
1036
|
+
(result.record.state === "failed" || result.record.state === "interrupted" ||
|
|
1037
|
+
result.record.state === "cancelled" || result.record.state === "killed")) {
|
|
1038
|
+
const mesh = deps.mesh === undefined ? createLaneMesh({ interactive: false }) : deps.mesh;
|
|
1039
|
+
if (mesh) {
|
|
1040
|
+
const meshEmitTimeoutMs = deps.meshEmitTimeoutMs ?? 10_000;
|
|
1041
|
+
let timer;
|
|
1042
|
+
try {
|
|
1043
|
+
await Promise.race([
|
|
1044
|
+
emitLaneTerminalStateEvent(mesh, result.record, result.record.state, result.record.terminal_reason),
|
|
1045
|
+
new Promise((_resolve, reject) => {
|
|
1046
|
+
timer = setTimeout(() => reject(new Error(`terminal mesh emit timed out after ${meshEmitTimeoutMs}ms`)), meshEmitTimeoutMs);
|
|
1047
|
+
}),
|
|
1048
|
+
]);
|
|
1049
|
+
}
|
|
1050
|
+
catch (err) {
|
|
1051
|
+
markDirty(summary, {
|
|
1052
|
+
kind: "lane",
|
|
1053
|
+
id: lane.lane_id,
|
|
1054
|
+
why: `terminal mesh emit failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
1055
|
+
});
|
|
1056
|
+
}
|
|
1057
|
+
finally {
|
|
1058
|
+
if (timer)
|
|
1059
|
+
clearTimeout(timer);
|
|
1060
|
+
}
|
|
1061
|
+
}
|
|
1062
|
+
}
|
|
1006
1063
|
}
|
|
1007
1064
|
}
|
|
1008
1065
|
catch (err) {
|
|
@@ -1031,6 +1088,32 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
|
|
|
1031
1088
|
to: "interrupted",
|
|
1032
1089
|
reason: "host_lost",
|
|
1033
1090
|
});
|
|
1091
|
+
const mesh = !deps.dryRun
|
|
1092
|
+
? deps.mesh === undefined ? createLaneMesh({ interactive: false }) : deps.mesh
|
|
1093
|
+
: null;
|
|
1094
|
+
if (mesh) {
|
|
1095
|
+
const meshEmitTimeoutMs = deps.meshEmitTimeoutMs ?? 10_000;
|
|
1096
|
+
let timer;
|
|
1097
|
+
try {
|
|
1098
|
+
await Promise.race([
|
|
1099
|
+
emitLaneTerminalStateEvent(mesh, result.record, "interrupted", "host_lost"),
|
|
1100
|
+
new Promise((_resolve, reject) => {
|
|
1101
|
+
timer = setTimeout(() => reject(new Error(`terminal mesh emit timed out after ${meshEmitTimeoutMs}ms`)), meshEmitTimeoutMs);
|
|
1102
|
+
}),
|
|
1103
|
+
]);
|
|
1104
|
+
}
|
|
1105
|
+
catch (err) {
|
|
1106
|
+
markDirty(summary, {
|
|
1107
|
+
kind: "lane",
|
|
1108
|
+
id: lane.lane_id,
|
|
1109
|
+
why: `terminal mesh emit failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
1110
|
+
});
|
|
1111
|
+
}
|
|
1112
|
+
finally {
|
|
1113
|
+
if (timer)
|
|
1114
|
+
clearTimeout(timer);
|
|
1115
|
+
}
|
|
1116
|
+
}
|
|
1034
1117
|
}
|
|
1035
1118
|
}
|
|
1036
1119
|
catch (err) {
|
|
@@ -12,7 +12,8 @@ import { type RunCompactionFn } from "./compaction.js";
|
|
|
12
12
|
import { type KnownModelsDeps } from "./models.js";
|
|
13
13
|
import { type LaunchTrigger } from "./delivery.js";
|
|
14
14
|
import { type MachineIdentity } from "./machine-identity.js";
|
|
15
|
-
import { type
|
|
15
|
+
import { type AdmissionHost } from "./admission.js";
|
|
16
|
+
import { type LaneCompanyRef, type LaneProvider, type LaneRecord, type AccountHistoryEntry } from "./types.js";
|
|
16
17
|
export declare const LANES_RUNS_REL: string;
|
|
17
18
|
/** Written by reconcile before a resume spawn; the worker prompt names this path. */
|
|
18
19
|
export declare const ANSWERED_QUESTIONS_FILE = "answered-questions.json";
|
|
@@ -32,6 +33,9 @@ export declare function resolveLaneModel(input: {
|
|
|
32
33
|
accountModel?: string;
|
|
33
34
|
providerModel?: string;
|
|
34
35
|
}): string | undefined;
|
|
36
|
+
type AutomaticAccountChangeReason = Exclude<AccountHistoryEntry["reason"], "operator_move">;
|
|
37
|
+
/** Change account only after a provider/runtime account failure, then re-admit. */
|
|
38
|
+
export declare function retryLaneOnAlternateAccount(hqRoot: string, lane: LaneRecord, reason: Extract<AutomaticAccountChangeReason, "usage_limit" | "account_unavailable">, writer?: "watcher" | "reconcile", deps?: SpawnLaneDeps): Promise<LaneRecord>;
|
|
35
39
|
export declare function appendModelFlag(provider: LaneProvider, args: string[], model: string | undefined): string[];
|
|
36
40
|
/** Add Codex fast-mode flags before the prompt/positional separator. */
|
|
37
41
|
export declare function appendFastModeFlags(provider: LaneProvider, args: string[], enabled: boolean | undefined): string[];
|
|
@@ -63,8 +67,12 @@ export type ProveDetachedFn = (pid: number) => DetachProof;
|
|
|
63
67
|
export type KillProcessFn = (pid: number, signal?: NodeJS.Signals) => void;
|
|
64
68
|
export interface SpawnLaneDeps {
|
|
65
69
|
env?: NodeJS.ProcessEnv;
|
|
70
|
+
/** The caller's host scope for admission checks; reconcile supplies its resolved host. */
|
|
71
|
+
admissionHost?: AdmissionHost;
|
|
66
72
|
/** Provider model catalog dependencies, shared with create in tests. */
|
|
67
73
|
modelDeps?: KnownModelsDeps;
|
|
74
|
+
/** Create validated this exact account/model candidate before its admission lock. */
|
|
75
|
+
currentModelValidated?: boolean;
|
|
68
76
|
existsSync?: (p: string) => boolean;
|
|
69
77
|
spawnProcess?: SpawnProcessFn;
|
|
70
78
|
proveDetached?: ProveDetachedFn;
|