@indigoai-us/hq-cli 5.214.0 → 5.216.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +25 -0
- package/dist/command-catalog.generated.d.ts +22 -0
- package/dist/command-catalog.generated.js +29 -0
- package/dist/commands/lanes.js +106 -64
- package/dist/lib/lanes/index.d.ts +1 -1
- package/dist/lib/lanes/index.js +1 -1
- package/dist/lib/lanes/keep-alive.d.ts +9 -0
- package/dist/lib/lanes/keep-alive.js +101 -0
- package/dist/lib/lanes/mesh-events.d.ts +3 -0
- package/dist/lib/lanes/mesh-events.js +32 -3
- package/dist/lib/lanes/profile.d.ts +8 -4
- package/dist/lib/lanes/profile.js +13 -17
- package/dist/lib/lanes/questions.d.ts +1 -1
- package/dist/lib/lanes/questions.js +2 -2
- package/dist/lib/lanes/reconcile.js +126 -40
- package/dist/lib/lanes/spawn.d.ts +2 -0
- package/dist/lib/lanes/spawn.js +6 -1
- package/dist/lib/lanes/store.js +25 -0
- package/dist/lib/lanes/types.d.ts +6 -0
- package/dist/lib/lanes/types.js +2 -2
- package/dist/lib/lanes/watcher.d.ts +3 -1
- package/dist/lib/lanes/watcher.js +174 -15
- package/package.json +1 -1
|
@@ -229,6 +229,17 @@ function isUnknownProjectStory(error) {
|
|
|
229
229
|
typeof candidate.message === "string" &&
|
|
230
230
|
candidate.message.includes("Unknown story"));
|
|
231
231
|
}
|
|
232
|
+
function isUncodedStoryPatchNotFound(error) {
|
|
233
|
+
if (!error || typeof error !== "object")
|
|
234
|
+
return false;
|
|
235
|
+
const candidate = error;
|
|
236
|
+
// The story-absent route response is exactly `404 Not found` without a code.
|
|
237
|
+
// Keep coded 404s, especially PROJECT_VIEW_NOT_FOUND, on their own paths.
|
|
238
|
+
return (candidate.status === 404 &&
|
|
239
|
+
candidate.code === undefined &&
|
|
240
|
+
typeof candidate.message === "string" &&
|
|
241
|
+
/^404 not found$/i.test(candidate.message));
|
|
242
|
+
}
|
|
232
243
|
function shellQuote(value) {
|
|
233
244
|
return "'" + value.replace(/'/g, "'\\''") + "'";
|
|
234
245
|
}
|
|
@@ -245,6 +256,9 @@ export function storyNotOnBoardError(company, projectId, storyId) {
|
|
|
245
256
|
command,
|
|
246
257
|
});
|
|
247
258
|
}
|
|
259
|
+
export function laneStoryStatusSuppressionKey(company, projectId, storyId) {
|
|
260
|
+
return JSON.stringify([company.uid, projectId, storyId]);
|
|
261
|
+
}
|
|
248
262
|
function isAlreadyRegistered(error) {
|
|
249
263
|
if (!error || typeof error !== "object")
|
|
250
264
|
return false;
|
|
@@ -365,7 +379,7 @@ export function createLaneMesh(opts = {}) {
|
|
|
365
379
|
await patchStoryStatus(tok, company.uid, projectId, storyId, status);
|
|
366
380
|
}
|
|
367
381
|
catch (error) {
|
|
368
|
-
if (isUnknownProjectStory(error)) {
|
|
382
|
+
if (isUnknownProjectStory(error) || isUncodedStoryPatchNotFound(error)) {
|
|
369
383
|
throw storyNotOnBoardError(company, projectId, storyId);
|
|
370
384
|
}
|
|
371
385
|
throw error;
|
|
@@ -437,6 +451,21 @@ export async function emitLaneDecisionEvents(mesh, lane, envelope, opts = {}) {
|
|
|
437
451
|
const threadId = await ensureLaneEventThread(mesh, company, lane, lane.state, opts.signal);
|
|
438
452
|
if (!company || !threadId)
|
|
439
453
|
return;
|
|
454
|
+
const patchStoryStatus = async (status) => {
|
|
455
|
+
const suppressions = opts.storyStatusSuppressions;
|
|
456
|
+
const key = laneStoryStatusSuppressionKey(company, lane.project_id, lane.story_id);
|
|
457
|
+
if (suppressions?.has(key))
|
|
458
|
+
return;
|
|
459
|
+
try {
|
|
460
|
+
await mesh.patchStoryStatus(company, lane.project_id, lane.story_id, status);
|
|
461
|
+
}
|
|
462
|
+
catch (err) {
|
|
463
|
+
if (suppressions && err instanceof LanesError && err.error === "story_not_on_board") {
|
|
464
|
+
suppressions.add(key);
|
|
465
|
+
}
|
|
466
|
+
throw err;
|
|
467
|
+
}
|
|
468
|
+
};
|
|
440
469
|
switch (envelope.decision) {
|
|
441
470
|
case "done": {
|
|
442
471
|
const outcomes = await Promise.allSettled([
|
|
@@ -444,7 +473,7 @@ export async function emitLaneDecisionEvents(mesh, lane, envelope, opts = {}) {
|
|
|
444
473
|
...laneMeshEventIdentity(lane),
|
|
445
474
|
summary: envelope.summary,
|
|
446
475
|
}, opts.signal),
|
|
447
|
-
|
|
476
|
+
patchStoryStatus("review"),
|
|
448
477
|
]);
|
|
449
478
|
const failure = outcomes.find((outcome) => outcome.status === "rejected");
|
|
450
479
|
if (failure?.status === "rejected")
|
|
@@ -458,7 +487,7 @@ export async function emitLaneDecisionEvents(mesh, lane, envelope, opts = {}) {
|
|
|
458
487
|
...laneMeshEventIdentity(lane),
|
|
459
488
|
blocked_reason: envelope.blocked_reason ?? "",
|
|
460
489
|
}),
|
|
461
|
-
|
|
490
|
+
patchStoryStatus("in_progress"),
|
|
462
491
|
]);
|
|
463
492
|
const failure = outcomes.find((outcome) => outcome.status === "rejected");
|
|
464
493
|
if (failure?.status === "rejected")
|
|
@@ -33,16 +33,20 @@ export interface WorkerProfileRuntime {
|
|
|
33
33
|
settingsPath?: string;
|
|
34
34
|
hooksDir?: string;
|
|
35
35
|
}
|
|
36
|
+
export interface LoadWorkerProfileOptions {
|
|
37
|
+
/** Unknown keys may be ignored only by callers with an explicit recovery path. */
|
|
38
|
+
onUnknownMetaFields?: (fields: readonly string[]) => void;
|
|
39
|
+
}
|
|
36
40
|
export declare function personalWorkerProfileDir(hqRoot: string, id: string): string;
|
|
37
41
|
/** A meta.yaml file is the explicit marker for a personal worker profile. */
|
|
38
42
|
export declare function workerProfileMarker(hqRoot: string, id: string): "absent" | "present";
|
|
39
|
-
export declare function loadWorkerProfile(hqRoot: string, id: string): WorkerProfile;
|
|
43
|
+
export declare function loadWorkerProfile(hqRoot: string, id: string, options?: LoadWorkerProfileOptions): WorkerProfile;
|
|
40
44
|
export interface EffectiveLaneKeepAlive {
|
|
41
45
|
enabled: boolean;
|
|
42
|
-
source: "lane" | "
|
|
46
|
+
source: "lane" | "none";
|
|
43
47
|
}
|
|
44
|
-
/** Resolve
|
|
45
|
-
export declare function resolveLaneKeepAlive(
|
|
48
|
+
/** Resolve keep-alive from the lane record; profile metadata only seeds new spawns. */
|
|
49
|
+
export declare function resolveLaneKeepAlive(_hqRoot: string, lane: Pick<LaneRecord, "keep_alive">): EffectiveLaneKeepAlive;
|
|
46
50
|
export declare function workerProfilePrompt(profile: WorkerProfile, delivery?: "hook" | "prompt"): string;
|
|
47
51
|
export declare function ignoredWorkerProfileFields(provider: LaneProvider): string[];
|
|
48
52
|
/**
|
|
@@ -117,15 +117,18 @@ function collectFiles(id, dir, relativeDir) {
|
|
|
117
117
|
visit(absoluteDir, relativeDir);
|
|
118
118
|
return files.sort((a, b) => a.localeCompare(b));
|
|
119
119
|
}
|
|
120
|
-
function parseMeta(id, dir, raw) {
|
|
120
|
+
function parseMeta(id, dir, raw, options = {}) {
|
|
121
121
|
if (!isPlainObject(raw)) {
|
|
122
122
|
throw profileError(id, dir, "meta.yaml must contain an object");
|
|
123
123
|
}
|
|
124
124
|
const unknown = Object.keys(raw).filter((key) => !PROFILE_META_KEYS.has(key));
|
|
125
125
|
if (unknown.length > 0) {
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
126
|
+
if (options.onUnknownMetaFields === undefined) {
|
|
127
|
+
throw profileError(id, dir, `meta.yaml has unknown field(s): ${unknown.join(", ")}`, {
|
|
128
|
+
fields: unknown,
|
|
129
|
+
});
|
|
130
|
+
}
|
|
131
|
+
options.onUnknownMetaFields(unknown);
|
|
129
132
|
}
|
|
130
133
|
const provider = raw.provider;
|
|
131
134
|
if (typeof provider !== "string" || !isLaneProvider(provider.trim())) {
|
|
@@ -305,7 +308,7 @@ export function workerProfileMarker(hqRoot, id) {
|
|
|
305
308
|
return "present";
|
|
306
309
|
return "present";
|
|
307
310
|
}
|
|
308
|
-
export function loadWorkerProfile(hqRoot, id) {
|
|
311
|
+
export function loadWorkerProfile(hqRoot, id, options = {}) {
|
|
309
312
|
const dir = personalWorkerProfileDir(hqRoot, id);
|
|
310
313
|
if (!validProfileId(id)) {
|
|
311
314
|
throw profileError(id, dir, "profile id must be a simple directory name", {
|
|
@@ -338,7 +341,7 @@ export function loadWorkerProfile(hqRoot, id) {
|
|
|
338
341
|
field: PROFILE_META,
|
|
339
342
|
});
|
|
340
343
|
}
|
|
341
|
-
const meta = parseMeta(id, dir, metaRaw);
|
|
344
|
+
const meta = parseMeta(id, dir, metaRaw, options);
|
|
342
345
|
const systemPromptPath = path.join(dir, PROFILE_SYSTEM_PROMPT);
|
|
343
346
|
const systemPrompt = readRequiredText(id, dir, systemPromptPath, PROFILE_SYSTEM_PROMPT);
|
|
344
347
|
if (systemPrompt.trim().length === 0) {
|
|
@@ -370,18 +373,11 @@ export function loadWorkerProfile(hqRoot, id) {
|
|
|
370
373
|
contentHash: hashProfileFiles(dir, hashFiles),
|
|
371
374
|
};
|
|
372
375
|
}
|
|
373
|
-
/** Resolve
|
|
374
|
-
export function resolveLaneKeepAlive(
|
|
375
|
-
|
|
376
|
-
return { enabled: lane.keep_alive, source: "lane" };
|
|
377
|
-
}
|
|
378
|
-
if (lane.worker_profile === undefined) {
|
|
379
|
-
return { enabled: false, source: "none" };
|
|
380
|
-
}
|
|
381
|
-
const profileKeepAlive = loadWorkerProfile(hqRoot, lane.worker_profile).meta.keep_alive;
|
|
382
|
-
return profileKeepAlive === undefined
|
|
376
|
+
/** Resolve keep-alive from the lane record; profile metadata only seeds new spawns. */
|
|
377
|
+
export function resolveLaneKeepAlive(_hqRoot, lane) {
|
|
378
|
+
return lane.keep_alive === undefined
|
|
383
379
|
? { enabled: false, source: "none" }
|
|
384
|
-
: { enabled:
|
|
380
|
+
: { enabled: lane.keep_alive, source: "lane" };
|
|
385
381
|
}
|
|
386
382
|
export function workerProfilePrompt(profile, delivery = "prompt") {
|
|
387
383
|
const parts = [profile.systemPrompt.trimEnd()];
|
|
@@ -71,5 +71,5 @@ export declare function envelopeQuestionId(laneId: string, envelopeRef: string,
|
|
|
71
71
|
* Spec §6.1: each envelope questions[] entry becomes one queue record, via
|
|
72
72
|
* raiseLaneQuestion (the same function the CLI ask path uses).
|
|
73
73
|
*/
|
|
74
|
-
export declare function raiseEnvelopeQuestions(hqRoot: string, lane: LaneRecord, envelope: LaneEnvelope, envelopeRef: string, at: string): QuestionRecord[];
|
|
74
|
+
export declare function raiseEnvelopeQuestions(hqRoot: string, lane: LaneRecord, envelope: LaneEnvelope, envelopeRef: string, at: string, questionIndexes?: readonly number[]): QuestionRecord[];
|
|
75
75
|
//# sourceMappingURL=questions.d.ts.map
|
|
@@ -222,7 +222,7 @@ function envelopeQuestionBody(item) {
|
|
|
222
222
|
* Spec §6.1: each envelope questions[] entry becomes one queue record, via
|
|
223
223
|
* raiseLaneQuestion (the same function the CLI ask path uses).
|
|
224
224
|
*/
|
|
225
|
-
export function raiseEnvelopeQuestions(hqRoot, lane, envelope, envelopeRef, at) {
|
|
225
|
+
export function raiseEnvelopeQuestions(hqRoot, lane, envelope, envelopeRef, at, questionIndexes) {
|
|
226
226
|
const items = envelope.questions ?? [];
|
|
227
227
|
const raised = [];
|
|
228
228
|
for (let i = 0; i < items.length; i += 1) {
|
|
@@ -235,7 +235,7 @@ export function raiseEnvelopeQuestions(hqRoot, lane, envelope, envelopeRef, at)
|
|
|
235
235
|
body: envelopeQuestionBody(item),
|
|
236
236
|
raisedVia: "envelope",
|
|
237
237
|
at,
|
|
238
|
-
questionId: envelopeQuestionId(lane.lane_id, envelopeRef, i),
|
|
238
|
+
questionId: envelopeQuestionId(lane.lane_id, envelopeRef, questionIndexes?.[i] ?? i),
|
|
239
239
|
}));
|
|
240
240
|
}
|
|
241
241
|
return raised;
|
|
@@ -14,14 +14,14 @@ import { effectiveAccountHome } from "./account-home.js";
|
|
|
14
14
|
import { loadLanesConfig, providerQuotaDefaultHoldMs } from "./config.js";
|
|
15
15
|
import { MAX_LANE_DEPTH } from "./constants.js";
|
|
16
16
|
import { AccountResolutionError, LanesError, isLaneVersionConflict } from "./errors.js";
|
|
17
|
+
import { loadWorkerProfile } from "./profile.js";
|
|
17
18
|
import { LANE_HIRE_GRANTS_FLAG, terminalChildResumeHireGrantError, withHireGrantAdmission, } from "./hire-grants.js";
|
|
18
|
-
import { loadWorkerProfile, resolveLaneKeepAlive } from "./profile.js";
|
|
19
19
|
import { classifySessionLiveness, defaultLivenessDeps, loadLivenessIndex, readDaemonPresenceConnected, unindexedClaudeSessionLiveness, } from "./liveness.js";
|
|
20
20
|
import { ANSWERED_QUESTIONS_FILE, armLaneWatcher, envelopeRepairMode, isPidRecordAlive, laneRunDir, readPidRecord, readSpawnManifest, markSpawnFailure, spawnEnvelopeRepair, spawnLaneWorker, retryLaneOnAlternateAccount, } from "./spawn.js";
|
|
21
21
|
import { appendCappedHistory, lanesStoreRoot, parseLaneRecord, parseQuestionRecord, readLane, readQuestion, updateLane, updateQuestion, } from "./store.js";
|
|
22
22
|
import { applyThrottleDecision, decideAdmission, defaultThrottleDeps, emptyThrottleState, evaluateThrottle, isCpuCheckEnabled, readAvailableMemMb, readThrottleState, resolveCpuBusyFraction, throttleStatePath, writeThrottleState, } from "./throttle.js";
|
|
23
23
|
import { readLanesRegistryFlag } from "./registry-flag.js";
|
|
24
|
-
import { LEAD_LANE_KEEP_ALIVE_FLAG, LEAD_LANE_KEEP_ALIVE_MAX_RESUMES, LEAD_LANE_KEEP_ALIVE_RESUME_TEXT, leadLaneKeepAliveBackoffMs, } from "./keep-alive.js";
|
|
24
|
+
import { LEAD_LANE_KEEP_ALIVE_FLAG, LEAD_LANE_KEEP_ALIVE_MAX_RESUMES, LEAD_LANE_KEEP_ALIVE_RESUME_TEXT, keepAliveExclusion, loadRetiredLaneIds, leadLaneKeepAliveBackoffMs, } from "./keep-alive.js";
|
|
25
25
|
import { providerSupportsSessionResume } from "./delivery.js";
|
|
26
26
|
import { applyDefaultQuotaHold, readQuotaSignal, recordAccountQuota } from "./quota.js";
|
|
27
27
|
import { answeredQuestionEntries, raiseEnvelopeQuestions } from "./questions.js";
|
|
@@ -800,18 +800,19 @@ export async function runPassBody(hqRoot, deps) {
|
|
|
800
800
|
const keepAliveFlagCompanies = new Map();
|
|
801
801
|
const keepAliveHireGrantFlagCompanies = new Map();
|
|
802
802
|
for (const lane of lanes) {
|
|
803
|
-
if (lane.state
|
|
804
|
-
(lane.keep_alive !== true &&
|
|
805
|
-
!(lane.keep_alive === undefined && lane.worker_profile !== undefined)) ||
|
|
803
|
+
if (!(lane.state === "done" || lane.state === "killed" || lane.state === "cancelled" || lane.state === "failed") ||
|
|
804
|
+
(lane.keep_alive !== true && lane.worker_profile === undefined) ||
|
|
806
805
|
!laneRunsHere(lane, deps.host) ||
|
|
807
|
-
lane.terminal_reason === PROVIDER_REJECTED_REASON ||
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
continue;
|
|
812
|
-
const envelope = readLatestEnvelopeRecord(hqRoot, lane.lane_id)?.envelope;
|
|
813
|
-
if (!envelope || envelope.decision !== "done" || envelope.story_complete === true)
|
|
806
|
+
(lane.state === "done" && (lane.terminal_reason === PROVIDER_REJECTED_REASON ||
|
|
807
|
+
lane.terminal_reason === "account_not_logged_in" ||
|
|
808
|
+
!providerSupportsSessionResume(lane.provider) ||
|
|
809
|
+
!(lane.provider_session_id ?? lane.last_resume?.session_id))))
|
|
814
810
|
continue;
|
|
811
|
+
if (lane.state === "done") {
|
|
812
|
+
const envelope = readLatestEnvelopeRecord(hqRoot, lane.lane_id)?.envelope;
|
|
813
|
+
if (!envelope || envelope.decision !== "done" || envelope.story_complete === true)
|
|
814
|
+
continue;
|
|
815
|
+
}
|
|
815
816
|
const key = keepAliveCompanyKey(lane.company);
|
|
816
817
|
keepAliveFlagCompanies.set(key, lane.company);
|
|
817
818
|
if (lane.senior.kind === "lane" && lane.senior.id) {
|
|
@@ -1413,80 +1414,164 @@ function recordKeepAliveEvent(hqRoot, laneId, summary, event) {
|
|
|
1413
1414
|
});
|
|
1414
1415
|
}
|
|
1415
1416
|
}
|
|
1417
|
+
function recordKeepAliveSkip(hqRoot, lane, reason, summary, deps, detail) {
|
|
1418
|
+
summary.skipped.push({
|
|
1419
|
+
kind: "lane",
|
|
1420
|
+
id: lane.lane_id,
|
|
1421
|
+
why: `keep_alive_${reason}${detail ? `: ${detail}` : ""}`,
|
|
1422
|
+
});
|
|
1423
|
+
if (deps.dryRun || lane.keep_alive_blocked_reason === reason)
|
|
1424
|
+
return;
|
|
1425
|
+
try {
|
|
1426
|
+
casLaneIf(hqRoot, lane.lane_id, (current) => current.version === lane.version && current.state === lane.state, (current) => ({ ...current, keep_alive_blocked_reason: reason }), false);
|
|
1427
|
+
}
|
|
1428
|
+
catch (err) {
|
|
1429
|
+
markDirty(summary, {
|
|
1430
|
+
kind: "lane",
|
|
1431
|
+
id: lane.lane_id,
|
|
1432
|
+
why: `keep-alive skip state write failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
1433
|
+
});
|
|
1434
|
+
}
|
|
1435
|
+
}
|
|
1436
|
+
function latestEnvelopeMtimeMs(hqRoot, lane, ref) {
|
|
1437
|
+
const filePath = path.join(laneRunDir(hqRoot, lane.lane_id), ref);
|
|
1438
|
+
try {
|
|
1439
|
+
return fs.statSync(filePath).mtimeMs;
|
|
1440
|
+
}
|
|
1441
|
+
catch (err) {
|
|
1442
|
+
throw new LanesError("keep_alive_envelope_timestamp_unavailable", `Cannot stat latest envelope for lane ${lane.lane_id}: ${err instanceof Error ? err.message : String(err)}`, { lane_id: lane.lane_id, path: filePath });
|
|
1443
|
+
}
|
|
1444
|
+
}
|
|
1416
1445
|
function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, hireGrantFlagByCompany) {
|
|
1417
1446
|
const nowMs = now.getTime();
|
|
1418
1447
|
const nowIso = now.toISOString();
|
|
1419
1448
|
const config = loadLanesConfig(hqRoot);
|
|
1449
|
+
const keepAliveEnabled = [...flagByCompany.values()].some((enabled) => enabled);
|
|
1450
|
+
let retiredLaneIds = null;
|
|
1451
|
+
if (keepAliveEnabled) {
|
|
1452
|
+
try {
|
|
1453
|
+
retiredLaneIds = loadRetiredLaneIds(hqRoot);
|
|
1454
|
+
}
|
|
1455
|
+
catch (err) {
|
|
1456
|
+
markError(summary, err);
|
|
1457
|
+
}
|
|
1458
|
+
}
|
|
1420
1459
|
for (const lane of lanes) {
|
|
1421
|
-
|
|
1460
|
+
if (!laneRunsHere(lane, deps.host))
|
|
1461
|
+
continue;
|
|
1462
|
+
if (lane.state === "killed" || lane.state === "cancelled" || lane.state === "failed") {
|
|
1463
|
+
if (lane.keep_alive === true &&
|
|
1464
|
+
flagByCompany.get(keepAliveCompanyKey(lane.company)) === true)
|
|
1465
|
+
recordKeepAliveSkip(hqRoot, lane, "terminal_state", summary, deps);
|
|
1466
|
+
continue;
|
|
1467
|
+
}
|
|
1422
1468
|
if (lane.state !== "done" ||
|
|
1423
|
-
lane.keep_alive ===
|
|
1424
|
-
(lane.keep_alive !== true && !hasRecordedProfileFallback) ||
|
|
1425
|
-
!laneRunsHere(lane, deps.host))
|
|
1469
|
+
(lane.keep_alive !== true && lane.worker_profile === undefined))
|
|
1426
1470
|
continue;
|
|
1427
1471
|
const latest = readLatestEnvelopeRecord(hqRoot, lane.lane_id);
|
|
1428
1472
|
const envelope = latest?.envelope;
|
|
1429
1473
|
if (!envelope || envelope.decision !== "done")
|
|
1430
1474
|
continue;
|
|
1431
|
-
const keepAliveConfigurationUnchanged = (candidate) => lane.keep_alive === true
|
|
1432
|
-
? candidate.keep_alive === true
|
|
1433
|
-
: lane.keep_alive === undefined &&
|
|
1434
|
-
lane.worker_profile !== undefined &&
|
|
1435
|
-
candidate.keep_alive === undefined &&
|
|
1436
|
-
candidate.worker_profile === lane.worker_profile;
|
|
1437
1475
|
if (envelope.story_complete === true) {
|
|
1438
1476
|
if (hasKeepAliveProgress(lane)) {
|
|
1439
1477
|
casLaneIf(hqRoot, lane.lane_id, (current) => current.version === lane.version &&
|
|
1440
1478
|
current.state === "done" &&
|
|
1441
|
-
|
|
1479
|
+
current.keep_alive === lane.keep_alive, (current) => ({ ...clearedKeepAliveProgress(current), keep_alive_blocked_reason: undefined }), deps.dryRun);
|
|
1442
1480
|
}
|
|
1443
1481
|
continue;
|
|
1444
1482
|
}
|
|
1483
|
+
if (flagByCompany.get(keepAliveCompanyKey(lane.company)) !== true)
|
|
1484
|
+
continue;
|
|
1445
1485
|
if (lane.terminal_reason === PROVIDER_REJECTED_REASON ||
|
|
1446
|
-
lane.terminal_reason === "account_not_logged_in"
|
|
1447
|
-
|
|
1448
|
-
!(lane.provider_session_id ?? lane.last_resume?.session_id))
|
|
1486
|
+
lane.terminal_reason === "account_not_logged_in") {
|
|
1487
|
+
recordKeepAliveSkip(hqRoot, lane, "terminal_state", summary, deps, lane.terminal_reason);
|
|
1449
1488
|
continue;
|
|
1450
|
-
|
|
1489
|
+
}
|
|
1490
|
+
if (lane.keep_alive !== true) {
|
|
1491
|
+
recordKeepAliveSkip(hqRoot, lane, "not_opted_in", summary, deps);
|
|
1451
1492
|
continue;
|
|
1452
|
-
|
|
1493
|
+
}
|
|
1494
|
+
const optedInAtMs = Date.parse(lane.keep_alive_opted_in_at ?? "");
|
|
1495
|
+
if (!Number.isFinite(optedInAtMs)) {
|
|
1496
|
+
recordKeepAliveSkip(hqRoot, lane, "missing_opt_in_time", summary, deps);
|
|
1497
|
+
continue;
|
|
1498
|
+
}
|
|
1499
|
+
let envelopeAtMs;
|
|
1453
1500
|
try {
|
|
1454
|
-
|
|
1501
|
+
envelopeAtMs = latestEnvelopeMtimeMs(hqRoot, lane, latest.ref);
|
|
1455
1502
|
}
|
|
1456
1503
|
catch (err) {
|
|
1457
|
-
const message = err instanceof LanesError
|
|
1458
|
-
? `${err.error}: ${err.message}`
|
|
1459
|
-
: err instanceof Error ? err.message : String(err);
|
|
1460
1504
|
markDirty(summary, {
|
|
1461
1505
|
kind: "lane",
|
|
1462
1506
|
id: lane.lane_id,
|
|
1463
|
-
why: `keep-alive
|
|
1507
|
+
why: `keep-alive envelope timestamp unavailable: ${err instanceof Error ? err.message : String(err)}`,
|
|
1464
1508
|
});
|
|
1465
1509
|
continue;
|
|
1466
1510
|
}
|
|
1467
|
-
if (
|
|
1511
|
+
if (envelopeAtMs <= optedInAtMs) {
|
|
1512
|
+
recordKeepAliveSkip(hqRoot, lane, "envelope_before_opt_in", summary, deps);
|
|
1513
|
+
continue;
|
|
1514
|
+
}
|
|
1515
|
+
if (retiredLaneIds === null) {
|
|
1516
|
+
recordKeepAliveSkip(hqRoot, lane, "retired_list_invalid", summary, deps);
|
|
1517
|
+
continue;
|
|
1518
|
+
}
|
|
1519
|
+
const exclusion = keepAliveExclusion(lane, lanes, retiredLaneIds);
|
|
1520
|
+
if (exclusion !== undefined) {
|
|
1521
|
+
recordKeepAliveSkip(hqRoot, lane, exclusion, summary, deps);
|
|
1522
|
+
continue;
|
|
1523
|
+
}
|
|
1524
|
+
if (!providerSupportsSessionResume(lane.provider) ||
|
|
1525
|
+
!(lane.provider_session_id ?? lane.last_resume?.session_id))
|
|
1468
1526
|
continue;
|
|
1469
|
-
const keepAliveStillEnabled = (candidate) =>
|
|
1470
|
-
|
|
1471
|
-
: candidate.keep_alive === undefined &&
|
|
1472
|
-
candidate.worker_profile === lane.worker_profile;
|
|
1527
|
+
const keepAliveStillEnabled = (candidate) => candidate.keep_alive === true &&
|
|
1528
|
+
candidate.keep_alive_opted_in_at === lane.keep_alive_opted_in_at;
|
|
1473
1529
|
const current = readLane(hqRoot, lane.lane_id);
|
|
1474
1530
|
if (current.version !== lane.version ||
|
|
1475
1531
|
current.state !== "done" ||
|
|
1476
1532
|
!keepAliveStillEnabled(current) ||
|
|
1477
1533
|
!laneRunsHere(current, deps.host))
|
|
1478
1534
|
continue;
|
|
1479
|
-
const
|
|
1535
|
+
const currentEnvelopeRecord = readLatestEnvelopeRecord(hqRoot, current.lane_id);
|
|
1536
|
+
const currentEnvelope = currentEnvelopeRecord?.envelope;
|
|
1480
1537
|
if (!currentEnvelope || currentEnvelope.decision !== "done")
|
|
1481
1538
|
continue;
|
|
1482
1539
|
if (currentEnvelope.story_complete === true) {
|
|
1483
1540
|
if (hasKeepAliveProgress(current)) {
|
|
1484
1541
|
casLaneIf(hqRoot, current.lane_id, (latestLane) => latestLane.version === current.version &&
|
|
1485
1542
|
latestLane.state === "done" &&
|
|
1486
|
-
keepAliveStillEnabled(latestLane), (latestLane) => clearedKeepAliveProgress(latestLane), deps.dryRun);
|
|
1543
|
+
keepAliveStillEnabled(latestLane), (latestLane) => ({ ...clearedKeepAliveProgress(latestLane), keep_alive_blocked_reason: undefined }), deps.dryRun);
|
|
1487
1544
|
}
|
|
1488
1545
|
continue;
|
|
1489
1546
|
}
|
|
1547
|
+
if (currentEnvelopeRecord !== null) {
|
|
1548
|
+
let currentEnvelopeAtMs;
|
|
1549
|
+
try {
|
|
1550
|
+
currentEnvelopeAtMs = latestEnvelopeMtimeMs(hqRoot, current, currentEnvelopeRecord.ref);
|
|
1551
|
+
}
|
|
1552
|
+
catch (err) {
|
|
1553
|
+
markDirty(summary, {
|
|
1554
|
+
kind: "lane",
|
|
1555
|
+
id: current.lane_id,
|
|
1556
|
+
why: `keep-alive envelope timestamp unavailable: ${err instanceof Error ? err.message : String(err)}`,
|
|
1557
|
+
});
|
|
1558
|
+
continue;
|
|
1559
|
+
}
|
|
1560
|
+
if (currentEnvelopeAtMs <= Date.parse(current.keep_alive_opted_in_at ?? "")) {
|
|
1561
|
+
recordKeepAliveSkip(hqRoot, current, "envelope_before_opt_in", summary, deps);
|
|
1562
|
+
continue;
|
|
1563
|
+
}
|
|
1564
|
+
}
|
|
1565
|
+
if (retiredLaneIds === null) {
|
|
1566
|
+
recordKeepAliveSkip(hqRoot, current, "retired_list_invalid", summary, deps);
|
|
1567
|
+
continue;
|
|
1568
|
+
}
|
|
1569
|
+
const currentLanes = loadLaneRecords(hqRoot, summary);
|
|
1570
|
+
const currentExclusion = keepAliveExclusion(current, currentLanes, retiredLaneIds);
|
|
1571
|
+
if (currentExclusion !== undefined) {
|
|
1572
|
+
recordKeepAliveSkip(hqRoot, current, currentExclusion, summary, deps);
|
|
1573
|
+
continue;
|
|
1574
|
+
}
|
|
1490
1575
|
const providerSessionId = current.provider_session_id ?? current.last_resume?.session_id;
|
|
1491
1576
|
if (current.terminal_reason === PROVIDER_REJECTED_REASON ||
|
|
1492
1577
|
current.terminal_reason === "account_not_logged_in" ||
|
|
@@ -1567,6 +1652,7 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
|
|
|
1567
1652
|
keep_alive_last_resume_at: nowIso,
|
|
1568
1653
|
keep_alive_next_eligible_at: nextEligibleAt,
|
|
1569
1654
|
keep_alive_exhausted_at: undefined,
|
|
1655
|
+
keep_alive_blocked_reason: undefined,
|
|
1570
1656
|
}), deps.dryRun);
|
|
1571
1657
|
const seniorId = current.senior.kind === "lane" ? current.senior.id : undefined;
|
|
1572
1658
|
let queued;
|
|
@@ -305,6 +305,8 @@ export interface EnvelopeRepairSpawnInput {
|
|
|
305
305
|
rawText: string;
|
|
306
306
|
/** True when this run never wrote its expected envelope file. */
|
|
307
307
|
missing?: boolean;
|
|
308
|
+
/** Repair-only warning hook for profile metadata ignored during recovery. */
|
|
309
|
+
onProfileWarning?: (fields: readonly string[]) => void;
|
|
308
310
|
}
|
|
309
311
|
/** How the spec §4 repair turn reaches the provider (recorded as `envelope_repair.mode`). */
|
|
310
312
|
export type EnvelopeRepairMode = "resume" | "fresh_turn";
|
package/dist/lib/lanes/spawn.js
CHANGED
|
@@ -2152,7 +2152,12 @@ export function spawnEnvelopeRepair(hqRoot, lane, input, deps = {}) {
|
|
|
2152
2152
|
ensureDir(logsDir);
|
|
2153
2153
|
const config = loadLanesConfig(hqRoot);
|
|
2154
2154
|
const profile = lane.worker_profile
|
|
2155
|
-
? loadWorkerProfile(hqRoot, lane.worker_profile
|
|
2155
|
+
? loadWorkerProfile(hqRoot, lane.worker_profile, {
|
|
2156
|
+
onUnknownMetaFields: (fields) => {
|
|
2157
|
+
process.stderr.write(`hq lanes watcher: envelope repair ignored unknown worker profile meta field(s) for ${lane.worker_profile}: ${fields.join(", ")}\n`);
|
|
2158
|
+
input.onProfileWarning?.(fields);
|
|
2159
|
+
},
|
|
2160
|
+
})
|
|
2156
2161
|
: undefined;
|
|
2157
2162
|
const profileDefaultsApply = profile !== undefined && profile.meta.provider === lane.provider;
|
|
2158
2163
|
const profileModel = profileDefaultsApply ? profile.meta.model : undefined;
|
package/dist/lib/lanes/store.js
CHANGED
|
@@ -409,6 +409,25 @@ export function parseLaneRecord(raw, filePath) {
|
|
|
409
409
|
value: raw.keep_alive,
|
|
410
410
|
});
|
|
411
411
|
}
|
|
412
|
+
const keepAliveBlockedReason = raw.keep_alive_blocked_reason;
|
|
413
|
+
const validKeepAliveBlockedReasons = [
|
|
414
|
+
"not_opted_in",
|
|
415
|
+
"missing_opt_in_time",
|
|
416
|
+
"envelope_before_opt_in",
|
|
417
|
+
"retired",
|
|
418
|
+
"superseded",
|
|
419
|
+
"retired_list_invalid",
|
|
420
|
+
"terminal_state",
|
|
421
|
+
];
|
|
422
|
+
if (keepAliveBlockedReason !== undefined &&
|
|
423
|
+
(typeof keepAliveBlockedReason !== "string" ||
|
|
424
|
+
!validKeepAliveBlockedReasons.includes(keepAliveBlockedReason))) {
|
|
425
|
+
throw new LaneSchemaError("keep_alive_blocked_reason is invalid", {
|
|
426
|
+
path: filePath,
|
|
427
|
+
field: "keep_alive_blocked_reason",
|
|
428
|
+
value: keepAliveBlockedReason,
|
|
429
|
+
});
|
|
430
|
+
}
|
|
412
431
|
if (raw.keep_alive_resume_count !== undefined &&
|
|
413
432
|
(typeof raw.keep_alive_resume_count !== "number" ||
|
|
414
433
|
!Number.isSafeInteger(raw.keep_alive_resume_count) ||
|
|
@@ -420,6 +439,7 @@ export function parseLaneRecord(raw, filePath) {
|
|
|
420
439
|
});
|
|
421
440
|
}
|
|
422
441
|
for (const field of [
|
|
442
|
+
"keep_alive_opted_in_at",
|
|
423
443
|
"keep_alive_last_resume_at",
|
|
424
444
|
"keep_alive_next_eligible_at",
|
|
425
445
|
"keep_alive_exhausted_at",
|
|
@@ -450,6 +470,11 @@ export function parseLaneRecord(raw, filePath) {
|
|
|
450
470
|
new Set(raw.pending_question_ids).size !== raw.pending_question_ids.length)) {
|
|
451
471
|
throw new LaneSchemaError("pending_question_ids must be an array of unique non-empty ids", { path: filePath, field: "pending_question_ids" });
|
|
452
472
|
}
|
|
473
|
+
if (raw.envelope_validation_errors !== undefined && (!Array.isArray(raw.envelope_validation_errors) ||
|
|
474
|
+
raw.envelope_validation_errors.length === 0 ||
|
|
475
|
+
raw.envelope_validation_errors.some((error) => typeof error !== "string" || error.trim() === ""))) {
|
|
476
|
+
throw new LaneSchemaError("envelope_validation_errors must be a non-empty array of non-empty strings", { path: filePath, field: "envelope_validation_errors" });
|
|
477
|
+
}
|
|
453
478
|
requireTimestamp(raw.created_at, "created_at", filePath);
|
|
454
479
|
requireTimestamp(raw.updated_at, "updated_at", filePath);
|
|
455
480
|
if (raw.queued_reason !== undefined && !isQueuedReason(raw.queued_reason)) {
|
|
@@ -356,6 +356,10 @@ export interface LaneRecord {
|
|
|
356
356
|
worker_profile?: string;
|
|
357
357
|
/** Whether this lead lane should resume completed turns while its story is open. */
|
|
358
358
|
keep_alive?: boolean;
|
|
359
|
+
/** Time this lane most recently explicitly opted into keep-alive. */
|
|
360
|
+
keep_alive_opted_in_at?: string;
|
|
361
|
+
/** Why reconcile declined to resume this lane most recently. */
|
|
362
|
+
keep_alive_blocked_reason?: "not_opted_in" | "missing_opt_in_time" | "envelope_before_opt_in" | "retired" | "superseded" | "retired_list_invalid" | "terminal_state";
|
|
359
363
|
/** Consecutive automatic keep-alive resumes since the last human reset. */
|
|
360
364
|
keep_alive_resume_count?: number;
|
|
361
365
|
/** Time the most recent automatic keep-alive resume was queued. */
|
|
@@ -428,6 +432,8 @@ export interface LaneRecord {
|
|
|
428
432
|
story_complete_reason?: string;
|
|
429
433
|
/** Copied from the last ingested envelope when present. */
|
|
430
434
|
story_status?: StoryStatus;
|
|
435
|
+
/** Envelope parser failures retained when a usable ask is recovered. */
|
|
436
|
+
envelope_validation_errors?: string[];
|
|
431
437
|
/** Set by the reconcile pass when `--timeout` has passed (spec §5.7). */
|
|
432
438
|
overdue?: boolean;
|
|
433
439
|
overdue_at?: string;
|
package/dist/lib/lanes/types.js
CHANGED
|
@@ -224,7 +224,7 @@ export const LANE_FIELD_WRITERS = {
|
|
|
224
224
|
pickup_validation_pending: ["creator", "command", "spawn", "reconcile"],
|
|
225
225
|
pickup_validation_pending_since: ["creator", "command", "spawn", "reconcile"],
|
|
226
226
|
priority: ["command"], worker: ["creator", "command"], worker_path: ["creator", "command"], worker_pack: ["creator", "command"], worker_profile: ["creator", "command"], worker_profile_hash: ["creator", "command", "spawn"], worker_profile_ignored: ["creator", "command", "spawn"], worker_policy_delivery: ["command", "spawn"],
|
|
227
|
-
keep_alive: ["creator", "command"], keep_alive_resume_count: ["command", "reconcile"], keep_alive_last_resume_at: ["command", "reconcile"], keep_alive_next_eligible_at: ["command", "reconcile"], keep_alive_exhausted_at: ["command", "reconcile"],
|
|
227
|
+
keep_alive: ["creator", "command"], keep_alive_opted_in_at: ["creator", "command"], keep_alive_blocked_reason: ["command", "reconcile"], keep_alive_resume_count: ["command", "reconcile"], keep_alive_last_resume_at: ["command", "reconcile"], keep_alive_next_eligible_at: ["command", "reconcile"], keep_alive_exhausted_at: ["command", "reconcile"],
|
|
228
228
|
provider: ["creator"], account: ["creator", "spawn", "command", "watcher", "reconcile"], account_selection: ["creator", "command"], admission_accounts: ["creator", "command"], account_history: ["command", "spawn", "watcher", "reconcile"], pending_resume: ["command", "spawn", "watcher", "reconcile"], placement_default: ["creator"], host: ["creator", "command"], origin_host: ["creator"], origin_host_source: ["creator"],
|
|
229
229
|
version: ["creator"], created_at: ["creator"], updated_at: ["creator"], attribution_history: ["command"],
|
|
230
230
|
last_envelope: ["watcher", "reconcile"], model: ["creator", "spawn", "command"], model_source: ["creator", "spawn", "command"], effort: ["creator", "spawn", "command"], effort_source: ["creator", "spawn", "command"], fast_mode: ["spawn"],
|
|
@@ -232,7 +232,7 @@ export const LANE_FIELD_WRITERS = {
|
|
|
232
232
|
overdue: ["reconcile"], overdue_at: ["reconcile"], last_host_report_at: ["watcher", "spawn", "reconcile"], stalled: ["watcher"], stalled_at: ["watcher"], stalled_reason: ["watcher"],
|
|
233
233
|
admitted_reason: ["command", "reconcile"], terminal_reason: ["command", "watcher", "spawn", "reconcile"], terminal_details: ["command", "spawn", "watcher", "reconcile"],
|
|
234
234
|
quota_reset_at: ["watcher", "reconcile"], model_quota_marker: ["watcher", "reconcile"], outpost_command_id: ["command"],
|
|
235
|
-
provider_session_id: ["watcher", "command", "spawn"], provider_session_id_absent: ["watcher", "command"], provider_home_at_spawn: ["spawn"], last_resume: ["spawn"], last_run_usage: ["watcher"], envelope_repair: ["watcher", "reconcile"], last_progress_event_at: ["watcher"],
|
|
235
|
+
provider_session_id: ["watcher", "command", "spawn"], provider_session_id_absent: ["watcher", "command"], provider_home_at_spawn: ["spawn"], last_resume: ["spawn"], last_run_usage: ["watcher"], envelope_repair: ["watcher", "reconcile"], envelope_validation_errors: ["watcher"], last_progress_event_at: ["watcher"],
|
|
236
236
|
};
|
|
237
237
|
function isPlainObject(value) {
|
|
238
238
|
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
@@ -143,6 +143,8 @@ export declare function notifySlotFreed(_input: {
|
|
|
143
143
|
export interface ApplyTerminalOpts {
|
|
144
144
|
envelope?: LaneEnvelope | null;
|
|
145
145
|
envelopeRef?: string;
|
|
146
|
+
envelopeValidationErrors?: string[];
|
|
147
|
+
questionIndexes?: readonly number[];
|
|
146
148
|
at?: string;
|
|
147
149
|
quota?: QuotaSignal | null;
|
|
148
150
|
/** Written to `terminal_details` with the model and account the run used. */
|
|
@@ -225,7 +227,7 @@ export interface WatcherLoopDeps {
|
|
|
225
227
|
*/
|
|
226
228
|
export declare function emitMeshSoft(event: string, logDecision: (record: Record<string, unknown>) => void, at: string, fn: (signal: AbortSignal) => Promise<void>, timeoutMs?: number): Promise<boolean>;
|
|
227
229
|
/** Question ids an ask envelope raised, as `raiseEnvelopeQuestions` mints them. */
|
|
228
|
-
export declare function envelopeQuestionIds(laneId: string, envelopeRef: string, envelope: LaneEnvelope): string[];
|
|
230
|
+
export declare function envelopeQuestionIds(laneId: string, envelopeRef: string, envelope: LaneEnvelope, questionIndexes?: readonly number[]): string[];
|
|
229
231
|
export declare function runWatcherLoop(hqRoot: string, laneId: string, deps?: WatcherLoopDeps): Promise<void>;
|
|
230
232
|
export type EnvelopeRepairDecision = {
|
|
231
233
|
action: "spawned";
|