@indigoai-us/hq-cli 5.216.0 → 5.218.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/dist/lib/lanes/keep-alive.d.ts +1 -0
- package/dist/lib/lanes/keep-alive.js +1 -0
- package/dist/lib/lanes/reconcile.d.ts +7 -0
- package/dist/lib/lanes/reconcile.js +261 -77
- package/dist/lib/lanes/store.js +3 -0
- package/dist/lib/lanes/types.d.ts +3 -1
- package/dist/lib/lanes/types.js +1 -1
- package/dist/lib/lanes/watcher.js +1 -0
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,22 @@
|
|
|
2
2
|
|
|
3
3
|
## [Unreleased]
|
|
4
4
|
|
|
5
|
+
## [5.218.0] — 2026-09-26
|
|
6
|
+
|
|
7
|
+
### Fixed
|
|
8
|
+
|
|
9
|
+
- The reconcile summary now lists opted-in lanes it could not resume and why.
|
|
10
|
+
Stderr provides reason-specific guidance and recommends reapplying opt-in only
|
|
11
|
+
when that can help.
|
|
12
|
+
|
|
13
|
+
## [5.217.0] — 2026-09-26
|
|
14
|
+
|
|
15
|
+
### Fixed
|
|
16
|
+
|
|
17
|
+
- Keep-alive resumes a lane interrupted by `exited_with_partial_output` when
|
|
18
|
+
the interruption followed its explicit opt-in. Quota and host-loss
|
|
19
|
+
interruptions, login failures, and stopped lanes remain ineligible.
|
|
20
|
+
|
|
5
21
|
## [5.216.0] — 2026-09-26
|
|
6
22
|
|
|
7
23
|
### Fixed
|
|
@@ -5,6 +5,7 @@ export declare const LEAD_LANE_KEEP_ALIVE_MAX_RESUMES = 24;
|
|
|
5
5
|
export declare const LEAD_LANE_KEEP_ALIVE_BASE_BACKOFF_MS: number;
|
|
6
6
|
export declare const LEAD_LANE_KEEP_ALIVE_MAX_BACKOFF_MS: number;
|
|
7
7
|
export declare const LEAD_LANE_KEEP_ALIVE_RESUME_TEXT = "Automatic keep-alive resume: your last turn ended with decision done and story_complete false. Continue the open work from where you stopped. End with done only when story_complete is true; if you are waiting, use hq lanes ask or stay in the loop.";
|
|
8
|
+
export declare const LEAD_LANE_KEEP_ALIVE_INTERRUPTED_RESUME_TEXT = "Automatic keep-alive resume: your previous turn was interrupted by a provider process exit. Check your last state before continuing. Continue the open work from where you stopped. End with done only when story_complete is true; if you are waiting, use hq lanes ask or stay in the loop.";
|
|
8
9
|
export type KeepAliveBlockedReason = NonNullable<LaneRecord["keep_alive_blocked_reason"]>;
|
|
9
10
|
export declare function retiredLaneRegistryPath(hqRoot: string): string;
|
|
10
11
|
/** Missing registry means no retired lanes; every present invalid file fails closed. */
|
|
@@ -7,6 +7,7 @@ export const LEAD_LANE_KEEP_ALIVE_MAX_RESUMES = 24;
|
|
|
7
7
|
export const LEAD_LANE_KEEP_ALIVE_BASE_BACKOFF_MS = 5 * 60 * 1000;
|
|
8
8
|
export const LEAD_LANE_KEEP_ALIVE_MAX_BACKOFF_MS = 60 * 60 * 1000;
|
|
9
9
|
export const LEAD_LANE_KEEP_ALIVE_RESUME_TEXT = "Automatic keep-alive resume: your last turn ended with decision done and story_complete false. Continue the open work from where you stopped. End with done only when story_complete is true; if you are waiting, use hq lanes ask or stay in the loop.";
|
|
10
|
+
export const LEAD_LANE_KEEP_ALIVE_INTERRUPTED_RESUME_TEXT = "Automatic keep-alive resume: your previous turn was interrupted by a provider process exit. Check your last state before continuing. Continue the open work from where you stopped. End with done only when story_complete is true; if you are waiting, use hq lanes ask or stay in the loop.";
|
|
10
11
|
export function retiredLaneRegistryPath(hqRoot) {
|
|
11
12
|
return path.join(hqRoot, "workspace", "lane-briefs", "juniors.json");
|
|
12
13
|
}
|
|
@@ -9,6 +9,7 @@ import { type LivenessDeps } from "./liveness.js";
|
|
|
9
9
|
import { type EnvelopeRepairSpawnInput, type SpawnLaneDeps } from "./spawn.js";
|
|
10
10
|
import { type ThrottleDeps } from "./throttle.js";
|
|
11
11
|
import { readLanesRegistryFlag } from "./registry-flag.js";
|
|
12
|
+
import { type KeepAliveBlockedReason } from "./keep-alive.js";
|
|
12
13
|
import type { LaneRecord, LivenessState, QueuedReason, QuestionRecord, SeniorRef } from "./types.js";
|
|
13
14
|
import { type LaneMesh } from "./mesh-events.js";
|
|
14
15
|
/** Spec §9: unanswered at the current hop for 60 minutes → one hop. */
|
|
@@ -49,6 +50,10 @@ export interface ReconcileSummary {
|
|
|
49
50
|
dead_host: ReconcileChange[];
|
|
50
51
|
heartbeats: string[];
|
|
51
52
|
skipped: ReconcileSkip[];
|
|
53
|
+
keep_alive_blocked: Array<{
|
|
54
|
+
id: string;
|
|
55
|
+
reason: KeepAliveBlockedReason;
|
|
56
|
+
}>;
|
|
52
57
|
errors: Array<{
|
|
53
58
|
message: string;
|
|
54
59
|
code?: string;
|
|
@@ -78,6 +83,8 @@ export interface ReconcileDeps {
|
|
|
78
83
|
mesh?: LaneMesh | null;
|
|
79
84
|
/** Bound on one terminal mesh emit during dead-host recovery. Default 10 s. */
|
|
80
85
|
meshEmitTimeoutMs?: number;
|
|
86
|
+
/** Fresh keep-alive eligibility read; injected to exercise opt-in changes before queueing. */
|
|
87
|
+
readKeepAliveRecheckLane?: (hqRoot: string, laneId: string) => LaneRecord;
|
|
81
88
|
/**
|
|
82
89
|
* Awaited after lane/question records and liveness are loaded, before any
|
|
83
90
|
* CAS write. Test seam so two in-process passes can share a snapshot.
|
|
@@ -21,11 +21,11 @@ import { ANSWERED_QUESTIONS_FILE, armLaneWatcher, envelopeRepairMode, isPidRecor
|
|
|
21
21
|
import { appendCappedHistory, lanesStoreRoot, parseLaneRecord, parseQuestionRecord, readLane, readQuestion, updateLane, updateQuestion, } from "./store.js";
|
|
22
22
|
import { applyThrottleDecision, decideAdmission, defaultThrottleDeps, emptyThrottleState, evaluateThrottle, isCpuCheckEnabled, readAvailableMemMb, readThrottleState, resolveCpuBusyFraction, throttleStatePath, writeThrottleState, } from "./throttle.js";
|
|
23
23
|
import { readLanesRegistryFlag } from "./registry-flag.js";
|
|
24
|
-
import { LEAD_LANE_KEEP_ALIVE_FLAG, LEAD_LANE_KEEP_ALIVE_MAX_RESUMES, LEAD_LANE_KEEP_ALIVE_RESUME_TEXT, keepAliveExclusion, loadRetiredLaneIds, leadLaneKeepAliveBackoffMs, } from "./keep-alive.js";
|
|
24
|
+
import { LEAD_LANE_KEEP_ALIVE_FLAG, LEAD_LANE_KEEP_ALIVE_INTERRUPTED_RESUME_TEXT, LEAD_LANE_KEEP_ALIVE_MAX_RESUMES, LEAD_LANE_KEEP_ALIVE_RESUME_TEXT, keepAliveExclusion, loadRetiredLaneIds, leadLaneKeepAliveBackoffMs, } from "./keep-alive.js";
|
|
25
25
|
import { providerSupportsSessionResume } from "./delivery.js";
|
|
26
26
|
import { applyDefaultQuotaHold, readQuotaSignal, recordAccountQuota } from "./quota.js";
|
|
27
27
|
import { answeredQuestionEntries, raiseEnvelopeQuestions } from "./questions.js";
|
|
28
|
-
import { SILENCE_MS, classifyLaneExit, isPidAlive, missingExpectedEnvelopeFile, readLatestEnvelopeFile, readLatestEnvelopeRecord, watcherEventsPath, writeWatcherDecision, } from "./watcher.js";
|
|
28
|
+
import { SILENCE_MS, classifyLaneExit, isPidAlive, missingExpectedEnvelopeFile, readLatestEnvelopeFile, readLatestEnvelopeRecord, watcherEventsPath, watcherOutLogPath, writeWatcherDecision, } from "./watcher.js";
|
|
29
29
|
import { createLaneMesh, emitLaneDecisionEvents, emitLaneTerminalStateEvent, laneMeshCompany, } from "./mesh-events.js";
|
|
30
30
|
import { laneStories, storyStatusForEnvelope } from "./stories.js";
|
|
31
31
|
import { decideEnvelopeRepair, envelopeQuestionIds, routeProviderFailure } from "./watcher.js";
|
|
@@ -66,6 +66,7 @@ function emptySummary(dryRun) {
|
|
|
66
66
|
dead_host: [],
|
|
67
67
|
heartbeats: [],
|
|
68
68
|
skipped: [],
|
|
69
|
+
keep_alive_blocked: [],
|
|
69
70
|
errors: [],
|
|
70
71
|
heartbeat_at: null,
|
|
71
72
|
};
|
|
@@ -800,7 +801,7 @@ export async function runPassBody(hqRoot, deps) {
|
|
|
800
801
|
const keepAliveFlagCompanies = new Map();
|
|
801
802
|
const keepAliveHireGrantFlagCompanies = new Map();
|
|
802
803
|
for (const lane of lanes) {
|
|
803
|
-
if (!(lane.state === "done" || lane.state === "killed" || lane.state === "cancelled" || lane.state === "failed") ||
|
|
804
|
+
if (!(lane.state === "done" || lane.state === "interrupted" || lane.state === "killed" || lane.state === "cancelled" || lane.state === "failed") ||
|
|
804
805
|
(lane.keep_alive !== true && lane.worker_profile === undefined) ||
|
|
805
806
|
!laneRunsHere(lane, deps.host) ||
|
|
806
807
|
(lane.state === "done" && (lane.terminal_reason === PROVIDER_REJECTED_REASON ||
|
|
@@ -838,7 +839,7 @@ export async function runPassBody(hqRoot, deps) {
|
|
|
838
839
|
]);
|
|
839
840
|
await withAllLaneAdmissionLocks(hqRoot, async () => {
|
|
840
841
|
const freshLanes = loadLaneRecords(hqRoot, summary);
|
|
841
|
-
|
|
842
|
+
resumeKeepAlive(hqRoot, freshLanes, summary, deps, now, keepAliveFlagByCompany, keepAliveHireGrantFlagByCompany);
|
|
842
843
|
const drainLanes = loadLaneRecords(hqRoot, summary);
|
|
843
844
|
const freshQuestions = loadQuestionRecords(hqRoot, summary);
|
|
844
845
|
await admitQueued(hqRoot, drainLanes, summary, deps, now);
|
|
@@ -1059,6 +1060,7 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
|
|
|
1059
1060
|
state: repairedClassification.state,
|
|
1060
1061
|
queued_reason: undefined,
|
|
1061
1062
|
terminal_reason: repairedClassification.reason ?? "host_lost",
|
|
1063
|
+
interrupted_at: repairedClassification.state === "interrupted" ? nowIso : undefined,
|
|
1062
1064
|
...(providerRejection
|
|
1063
1065
|
? { terminal_details: providerRejectionDetails(providerRejection, current) }
|
|
1064
1066
|
: {}),
|
|
@@ -1175,6 +1177,7 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
|
|
|
1175
1177
|
state: "interrupted",
|
|
1176
1178
|
queued_reason: undefined,
|
|
1177
1179
|
terminal_reason: "host_lost",
|
|
1180
|
+
interrupted_at: nowIso,
|
|
1178
1181
|
}), deps.dryRun);
|
|
1179
1182
|
if (result.changed) {
|
|
1180
1183
|
summary.dead_host.push({
|
|
@@ -1392,7 +1395,8 @@ function keepAliveProgressResetByHuman(lane) {
|
|
|
1392
1395
|
if (count > 0 && !Number.isFinite(lastAutoMs))
|
|
1393
1396
|
return true;
|
|
1394
1397
|
const messageMs = Date.parse(lane.last_message_at ?? "");
|
|
1395
|
-
const resumeMs = lane.last_resume?.text === LEAD_LANE_KEEP_ALIVE_RESUME_TEXT
|
|
1398
|
+
const resumeMs = lane.last_resume?.text === LEAD_LANE_KEEP_ALIVE_RESUME_TEXT ||
|
|
1399
|
+
lane.last_resume?.text === LEAD_LANE_KEEP_ALIVE_INTERRUPTED_RESUME_TEXT
|
|
1396
1400
|
? Number.NaN
|
|
1397
1401
|
: Date.parse(lane.last_resume?.at ?? "");
|
|
1398
1402
|
return Number.isFinite(lastAutoMs) &&
|
|
@@ -1414,12 +1418,38 @@ function recordKeepAliveEvent(hqRoot, laneId, summary, event) {
|
|
|
1414
1418
|
});
|
|
1415
1419
|
}
|
|
1416
1420
|
}
|
|
1421
|
+
function keepAliveSkipGuidance(laneId, reason) {
|
|
1422
|
+
switch (reason) {
|
|
1423
|
+
case "missing_opt_in_time":
|
|
1424
|
+
return `if appropriate, run hq lanes keep-alive ${laneId} --on`;
|
|
1425
|
+
case "not_opted_in":
|
|
1426
|
+
return "explicit opt-in is required before keep-alive can resume this lane";
|
|
1427
|
+
case "envelope_before_opt_in":
|
|
1428
|
+
return `the latest envelope predates opt-in; review the lane with hq lanes show ${laneId}`;
|
|
1429
|
+
case "interruption_before_opt_in":
|
|
1430
|
+
return `the interruption predates opt-in; review the lane with hq lanes show ${laneId}`;
|
|
1431
|
+
case "missing_interruption_time":
|
|
1432
|
+
return "the interruption time is unavailable; inspect the watcher classification before resuming";
|
|
1433
|
+
case "retired":
|
|
1434
|
+
return "the lane is retired; resolve its retired status before resuming";
|
|
1435
|
+
case "superseded":
|
|
1436
|
+
return "a newer lead exists for this project; continue on that lane";
|
|
1437
|
+
case "retired_list_invalid":
|
|
1438
|
+
return "the retired lane list is invalid; repair workspace/lane-briefs/juniors.json before retrying";
|
|
1439
|
+
case "terminal_state":
|
|
1440
|
+
return `the lane is in a terminal state; review it with hq lanes show ${laneId} before starting replacement work`;
|
|
1441
|
+
}
|
|
1442
|
+
}
|
|
1417
1443
|
function recordKeepAliveSkip(hqRoot, lane, reason, summary, deps, detail) {
|
|
1418
1444
|
summary.skipped.push({
|
|
1419
1445
|
kind: "lane",
|
|
1420
1446
|
id: lane.lane_id,
|
|
1421
1447
|
why: `keep_alive_${reason}${detail ? `: ${detail}` : ""}`,
|
|
1422
1448
|
});
|
|
1449
|
+
if (lane.keep_alive === true) {
|
|
1450
|
+
summary.keep_alive_blocked.push({ id: lane.lane_id, reason });
|
|
1451
|
+
process.stderr.write(`hq lanes reconcile: keep-alive blocked for lane ${lane.lane_id} (${reason}); ${keepAliveSkipGuidance(lane.lane_id, reason)}\n`);
|
|
1452
|
+
}
|
|
1423
1453
|
if (deps.dryRun || lane.keep_alive_blocked_reason === reason)
|
|
1424
1454
|
return;
|
|
1425
1455
|
try {
|
|
@@ -1433,6 +1463,36 @@ function recordKeepAliveSkip(hqRoot, lane, reason, summary, deps, detail) {
|
|
|
1433
1463
|
});
|
|
1434
1464
|
}
|
|
1435
1465
|
}
|
|
1466
|
+
function readLatestInterruptedClassificationAt(hqRoot, lane) {
|
|
1467
|
+
const logPath = watcherOutLogPath(hqRoot, lane.lane_id);
|
|
1468
|
+
let body;
|
|
1469
|
+
try {
|
|
1470
|
+
body = fs.readFileSync(logPath, "utf8");
|
|
1471
|
+
}
|
|
1472
|
+
catch (err) {
|
|
1473
|
+
if (err.code === "ENOENT")
|
|
1474
|
+
return undefined;
|
|
1475
|
+
throw err;
|
|
1476
|
+
}
|
|
1477
|
+
let latestClassification;
|
|
1478
|
+
for (const line of body.split(/\r?\n/).filter(Boolean)) {
|
|
1479
|
+
const parsed = JSON.parse(line);
|
|
1480
|
+
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
|
|
1481
|
+
throw new Error("watcher log contains a non-object decision record");
|
|
1482
|
+
}
|
|
1483
|
+
const record = parsed;
|
|
1484
|
+
if (record.event === "classification")
|
|
1485
|
+
latestClassification = record;
|
|
1486
|
+
}
|
|
1487
|
+
const at = latestClassification?.at;
|
|
1488
|
+
if (latestClassification?.state !== "interrupted" ||
|
|
1489
|
+
latestClassification.reason !== lane.terminal_reason ||
|
|
1490
|
+
typeof at !== "string" ||
|
|
1491
|
+
!Number.isFinite(Date.parse(at)) ||
|
|
1492
|
+
new Date(Date.parse(at)).toISOString() !== at)
|
|
1493
|
+
return undefined;
|
|
1494
|
+
return at;
|
|
1495
|
+
}
|
|
1436
1496
|
function latestEnvelopeMtimeMs(hqRoot, lane, ref) {
|
|
1437
1497
|
const filePath = path.join(laneRunDir(hqRoot, lane.lane_id), ref);
|
|
1438
1498
|
try {
|
|
@@ -1442,7 +1502,7 @@ function latestEnvelopeMtimeMs(hqRoot, lane, ref) {
|
|
|
1442
1502
|
throw new LanesError("keep_alive_envelope_timestamp_unavailable", `Cannot stat latest envelope for lane ${lane.lane_id}: ${err instanceof Error ? err.message : String(err)}`, { lane_id: lane.lane_id, path: filePath });
|
|
1443
1503
|
}
|
|
1444
1504
|
}
|
|
1445
|
-
function
|
|
1505
|
+
function resumeKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, hireGrantFlagByCompany) {
|
|
1446
1506
|
const nowMs = now.getTime();
|
|
1447
1507
|
const nowIso = now.toISOString();
|
|
1448
1508
|
const config = loadLanesConfig(hqRoot);
|
|
@@ -1456,34 +1516,97 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
|
|
|
1456
1516
|
markError(summary, err);
|
|
1457
1517
|
}
|
|
1458
1518
|
}
|
|
1459
|
-
for (const
|
|
1460
|
-
if (!laneRunsHere(
|
|
1519
|
+
for (const originalLane of lanes) {
|
|
1520
|
+
if (!laneRunsHere(originalLane, deps.host))
|
|
1461
1521
|
continue;
|
|
1462
|
-
if (
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
|
|
1522
|
+
if (originalLane.state === "killed" ||
|
|
1523
|
+
originalLane.state === "cancelled" ||
|
|
1524
|
+
originalLane.state === "failed") {
|
|
1525
|
+
if (originalLane.keep_alive === true &&
|
|
1526
|
+
flagByCompany.get(keepAliveCompanyKey(originalLane.company)) === true)
|
|
1527
|
+
recordKeepAliveSkip(hqRoot, originalLane, "terminal_state", summary, deps);
|
|
1466
1528
|
continue;
|
|
1467
1529
|
}
|
|
1468
|
-
if (
|
|
1469
|
-
(
|
|
1470
|
-
continue;
|
|
1471
|
-
const latest = readLatestEnvelopeRecord(hqRoot, lane.lane_id);
|
|
1472
|
-
const envelope = latest?.envelope;
|
|
1473
|
-
if (!envelope || envelope.decision !== "done")
|
|
1530
|
+
if ((originalLane.state !== "done" && originalLane.state !== "interrupted") ||
|
|
1531
|
+
(originalLane.keep_alive !== true && originalLane.worker_profile === undefined))
|
|
1474
1532
|
continue;
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1533
|
+
const initialState = originalLane.state;
|
|
1534
|
+
let lane = originalLane;
|
|
1535
|
+
if (initialState === "done") {
|
|
1536
|
+
const envelope = readLatestEnvelopeRecord(hqRoot, lane.lane_id)?.envelope;
|
|
1537
|
+
if (!envelope || envelope.decision !== "done")
|
|
1538
|
+
continue;
|
|
1539
|
+
if (envelope.story_complete === true) {
|
|
1540
|
+
if (hasKeepAliveProgress(lane)) {
|
|
1541
|
+
casLaneIf(hqRoot, lane.lane_id, (current) => current.version === lane.version &&
|
|
1542
|
+
current.state === "done" &&
|
|
1543
|
+
current.keep_alive === lane.keep_alive, (current) => ({ ...clearedKeepAliveProgress(current), keep_alive_blocked_reason: undefined }), deps.dryRun);
|
|
1544
|
+
}
|
|
1545
|
+
continue;
|
|
1480
1546
|
}
|
|
1481
|
-
continue;
|
|
1482
1547
|
}
|
|
1483
1548
|
if (flagByCompany.get(keepAliveCompanyKey(lane.company)) !== true)
|
|
1484
1549
|
continue;
|
|
1485
|
-
|
|
1486
|
-
|
|
1550
|
+
// Persist a stable timestamp for legacy interrupted rows before a later
|
|
1551
|
+
// keep-alive status write can advance their general updated_at clock. The
|
|
1552
|
+
// watcher classification is the interruption time; updated_at is only an
|
|
1553
|
+
// upper bound and can postdate explicit opt-in.
|
|
1554
|
+
if (initialState === "interrupted" && lane.keep_alive === true && lane.interrupted_at === undefined) {
|
|
1555
|
+
let interruptedAt;
|
|
1556
|
+
try {
|
|
1557
|
+
interruptedAt = readLatestInterruptedClassificationAt(hqRoot, lane);
|
|
1558
|
+
}
|
|
1559
|
+
catch (err) {
|
|
1560
|
+
const detail = `watcher classification read failed: ${err instanceof Error ? err.message : String(err)}`;
|
|
1561
|
+
recordKeepAliveSkip(hqRoot, lane, "missing_interruption_time", summary, deps, detail);
|
|
1562
|
+
const skip = summary.skipped[summary.skipped.length - 1];
|
|
1563
|
+
if (skip) {
|
|
1564
|
+
summary.ok = false;
|
|
1565
|
+
logSkip(skip);
|
|
1566
|
+
}
|
|
1567
|
+
continue;
|
|
1568
|
+
}
|
|
1569
|
+
if (interruptedAt === undefined) {
|
|
1570
|
+
recordKeepAliveSkip(hqRoot, lane, "missing_interruption_time", summary, deps, "no latest matching watcher classification event");
|
|
1571
|
+
const skip = summary.skipped[summary.skipped.length - 1];
|
|
1572
|
+
if (skip) {
|
|
1573
|
+
summary.ok = false;
|
|
1574
|
+
logSkip(skip);
|
|
1575
|
+
}
|
|
1576
|
+
continue;
|
|
1577
|
+
}
|
|
1578
|
+
try {
|
|
1579
|
+
const backfill = casLaneIf(hqRoot, lane.lane_id, (current) => current.version === lane.version &&
|
|
1580
|
+
current.state === "interrupted" &&
|
|
1581
|
+
current.keep_alive === true &&
|
|
1582
|
+
current.terminal_reason === lane.terminal_reason &&
|
|
1583
|
+
current.interrupted_at === undefined, (current) => ({ ...current, interrupted_at: interruptedAt }), deps.dryRun);
|
|
1584
|
+
lane = backfill.record;
|
|
1585
|
+
if (lane.interrupted_at === undefined) {
|
|
1586
|
+
recordKeepAliveSkip(hqRoot, lane, "missing_interruption_time", summary, deps, "lane changed before watcher classification timestamp could be recorded");
|
|
1587
|
+
const skip = summary.skipped[summary.skipped.length - 1];
|
|
1588
|
+
if (skip) {
|
|
1589
|
+
summary.ok = false;
|
|
1590
|
+
logSkip(skip);
|
|
1591
|
+
}
|
|
1592
|
+
continue;
|
|
1593
|
+
}
|
|
1594
|
+
}
|
|
1595
|
+
catch (err) {
|
|
1596
|
+
markDirty(summary, {
|
|
1597
|
+
kind: "lane",
|
|
1598
|
+
id: lane.lane_id,
|
|
1599
|
+
why: `keep-alive interruption timestamp backfill failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
1600
|
+
});
|
|
1601
|
+
continue;
|
|
1602
|
+
}
|
|
1603
|
+
}
|
|
1604
|
+
if (initialState === "interrupted" && lane.terminal_reason !== "exited_with_partial_output") {
|
|
1605
|
+
recordKeepAliveSkip(hqRoot, lane, "terminal_state", summary, deps, lane.terminal_reason ?? "missing interruption reason");
|
|
1606
|
+
continue;
|
|
1607
|
+
}
|
|
1608
|
+
if (initialState === "done" && (lane.terminal_reason === PROVIDER_REJECTED_REASON ||
|
|
1609
|
+
lane.terminal_reason === "account_not_logged_in")) {
|
|
1487
1610
|
recordKeepAliveSkip(hqRoot, lane, "terminal_state", summary, deps, lane.terminal_reason);
|
|
1488
1611
|
continue;
|
|
1489
1612
|
}
|
|
@@ -1496,20 +1619,35 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
|
|
|
1496
1619
|
recordKeepAliveSkip(hqRoot, lane, "missing_opt_in_time", summary, deps);
|
|
1497
1620
|
continue;
|
|
1498
1621
|
}
|
|
1499
|
-
let
|
|
1500
|
-
|
|
1501
|
-
|
|
1622
|
+
let eligibilityAtMs;
|
|
1623
|
+
if (initialState === "done") {
|
|
1624
|
+
const latest = readLatestEnvelopeRecord(hqRoot, lane.lane_id);
|
|
1625
|
+
if (!latest || latest.envelope.decision !== "done" || latest.envelope.story_complete === true)
|
|
1626
|
+
continue;
|
|
1627
|
+
try {
|
|
1628
|
+
eligibilityAtMs = latestEnvelopeMtimeMs(hqRoot, lane, latest.ref);
|
|
1629
|
+
}
|
|
1630
|
+
catch (err) {
|
|
1631
|
+
markDirty(summary, {
|
|
1632
|
+
kind: "lane",
|
|
1633
|
+
id: lane.lane_id,
|
|
1634
|
+
why: `keep-alive envelope timestamp unavailable: ${err instanceof Error ? err.message : String(err)}`,
|
|
1635
|
+
});
|
|
1636
|
+
continue;
|
|
1637
|
+
}
|
|
1502
1638
|
}
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
|
|
1508
|
-
}
|
|
1509
|
-
continue;
|
|
1639
|
+
else {
|
|
1640
|
+
eligibilityAtMs = Date.parse(lane.interrupted_at ?? "");
|
|
1641
|
+
if (!Number.isFinite(eligibilityAtMs)) {
|
|
1642
|
+
recordKeepAliveSkip(hqRoot, lane, "missing_interruption_time", summary, deps);
|
|
1643
|
+
continue;
|
|
1644
|
+
}
|
|
1510
1645
|
}
|
|
1511
|
-
|
|
1512
|
-
|
|
1646
|
+
const initialTimeBlockReason = initialState === "done"
|
|
1647
|
+
? "envelope_before_opt_in"
|
|
1648
|
+
: "interruption_before_opt_in";
|
|
1649
|
+
if (eligibilityAtMs <= optedInAtMs) {
|
|
1650
|
+
recordKeepAliveSkip(hqRoot, lane, initialTimeBlockReason, summary, deps);
|
|
1513
1651
|
continue;
|
|
1514
1652
|
}
|
|
1515
1653
|
if (retiredLaneIds === null) {
|
|
@@ -1521,33 +1659,54 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
|
|
|
1521
1659
|
recordKeepAliveSkip(hqRoot, lane, exclusion, summary, deps);
|
|
1522
1660
|
continue;
|
|
1523
1661
|
}
|
|
1662
|
+
const providerSessionId = lane.provider_session_id ?? lane.last_resume?.session_id;
|
|
1524
1663
|
if (!providerSupportsSessionResume(lane.provider) ||
|
|
1525
|
-
|
|
1664
|
+
typeof providerSessionId !== "string" ||
|
|
1665
|
+
providerSessionId.trim().length === 0) {
|
|
1666
|
+
if (initialState === "interrupted") {
|
|
1667
|
+
recordKeepAliveSkip(hqRoot, lane, "terminal_state", summary, deps, "provider session cannot be resumed");
|
|
1668
|
+
}
|
|
1526
1669
|
continue;
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
|
|
1670
|
+
}
|
|
1671
|
+
// This second read is intentionally independent of the pass snapshot. The
|
|
1672
|
+
// final timestamp comparison below closes the opt-in race before the CAS.
|
|
1673
|
+
const rechecked = deps.readKeepAliveRecheckLane?.(hqRoot, lane.lane_id) ?? readLane(hqRoot, lane.lane_id);
|
|
1674
|
+
// Dry-run backfills only in memory. Carry that simulated legacy timestamp
|
|
1675
|
+
// through the fresh read when the stored lane version has not changed.
|
|
1676
|
+
const current = deps.dryRun &&
|
|
1677
|
+
initialState === "interrupted" &&
|
|
1678
|
+
originalLane.interrupted_at === undefined &&
|
|
1679
|
+
lane.interrupted_at !== undefined &&
|
|
1680
|
+
rechecked.version === lane.version &&
|
|
1681
|
+
rechecked.interrupted_at === undefined
|
|
1682
|
+
? { ...rechecked, interrupted_at: lane.interrupted_at }
|
|
1683
|
+
: rechecked;
|
|
1684
|
+
if (current.state !== initialState || !laneRunsHere(current, deps.host))
|
|
1534
1685
|
continue;
|
|
1535
|
-
|
|
1536
|
-
|
|
1537
|
-
if (!currentEnvelope || currentEnvelope.decision !== "done")
|
|
1686
|
+
if (current.keep_alive !== true) {
|
|
1687
|
+
recordKeepAliveSkip(hqRoot, current, "not_opted_in", summary, deps);
|
|
1538
1688
|
continue;
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
latestLane.state === "done" &&
|
|
1543
|
-
keepAliveStillEnabled(latestLane), (latestLane) => ({ ...clearedKeepAliveProgress(latestLane), keep_alive_blocked_reason: undefined }), deps.dryRun);
|
|
1544
|
-
}
|
|
1689
|
+
}
|
|
1690
|
+
if (initialState === "interrupted" && current.terminal_reason !== "exited_with_partial_output") {
|
|
1691
|
+
recordKeepAliveSkip(hqRoot, current, "terminal_state", summary, deps, current.terminal_reason ?? "missing interruption reason");
|
|
1545
1692
|
continue;
|
|
1546
1693
|
}
|
|
1547
|
-
|
|
1548
|
-
|
|
1694
|
+
let currentEligibilityAtMs;
|
|
1695
|
+
if (initialState === "done") {
|
|
1696
|
+
const currentEnvelope = readLatestEnvelopeRecord(hqRoot, current.lane_id);
|
|
1697
|
+
if (!currentEnvelope || currentEnvelope.envelope.decision !== "done")
|
|
1698
|
+
continue;
|
|
1699
|
+
if (currentEnvelope.envelope.story_complete === true) {
|
|
1700
|
+
if (hasKeepAliveProgress(current)) {
|
|
1701
|
+
casLaneIf(hqRoot, current.lane_id, (latestLane) => latestLane.version === current.version &&
|
|
1702
|
+
latestLane.state === "done" &&
|
|
1703
|
+
latestLane.keep_alive === true &&
|
|
1704
|
+
latestLane.keep_alive_opted_in_at === current.keep_alive_opted_in_at, (latestLane) => ({ ...clearedKeepAliveProgress(latestLane), keep_alive_blocked_reason: undefined }), deps.dryRun);
|
|
1705
|
+
}
|
|
1706
|
+
continue;
|
|
1707
|
+
}
|
|
1549
1708
|
try {
|
|
1550
|
-
|
|
1709
|
+
currentEligibilityAtMs = latestEnvelopeMtimeMs(hqRoot, current, currentEnvelope.ref);
|
|
1551
1710
|
}
|
|
1552
1711
|
catch (err) {
|
|
1553
1712
|
markDirty(summary, {
|
|
@@ -1557,11 +1716,26 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
|
|
|
1557
1716
|
});
|
|
1558
1717
|
continue;
|
|
1559
1718
|
}
|
|
1560
|
-
|
|
1561
|
-
|
|
1719
|
+
}
|
|
1720
|
+
else {
|
|
1721
|
+
currentEligibilityAtMs = Date.parse(current.interrupted_at ?? "");
|
|
1722
|
+
if (!Number.isFinite(currentEligibilityAtMs)) {
|
|
1723
|
+
recordKeepAliveSkip(hqRoot, current, "missing_interruption_time", summary, deps);
|
|
1562
1724
|
continue;
|
|
1563
1725
|
}
|
|
1564
1726
|
}
|
|
1727
|
+
const currentOptedInAtMs = Date.parse(current.keep_alive_opted_in_at ?? "");
|
|
1728
|
+
if (!Number.isFinite(currentOptedInAtMs)) {
|
|
1729
|
+
recordKeepAliveSkip(hqRoot, current, "missing_opt_in_time", summary, deps);
|
|
1730
|
+
continue;
|
|
1731
|
+
}
|
|
1732
|
+
const currentTimeBlockReason = initialState === "done"
|
|
1733
|
+
? "envelope_before_opt_in"
|
|
1734
|
+
: "interruption_before_opt_in";
|
|
1735
|
+
if (currentEligibilityAtMs <= currentOptedInAtMs) {
|
|
1736
|
+
recordKeepAliveSkip(hqRoot, current, currentTimeBlockReason, summary, deps);
|
|
1737
|
+
continue;
|
|
1738
|
+
}
|
|
1565
1739
|
if (retiredLaneIds === null) {
|
|
1566
1740
|
recordKeepAliveSkip(hqRoot, current, "retired_list_invalid", summary, deps);
|
|
1567
1741
|
continue;
|
|
@@ -1572,11 +1746,10 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
|
|
|
1572
1746
|
recordKeepAliveSkip(hqRoot, current, currentExclusion, summary, deps);
|
|
1573
1747
|
continue;
|
|
1574
1748
|
}
|
|
1575
|
-
const
|
|
1576
|
-
if (current.
|
|
1577
|
-
|
|
1578
|
-
|
|
1579
|
-
providerSessionId.trim().length === 0)
|
|
1749
|
+
const currentSessionId = current.provider_session_id ?? current.last_resume?.session_id;
|
|
1750
|
+
if (!providerSupportsSessionResume(current.provider) ||
|
|
1751
|
+
typeof currentSessionId !== "string" ||
|
|
1752
|
+
currentSessionId.trim().length === 0)
|
|
1580
1753
|
continue;
|
|
1581
1754
|
const resetByHuman = keepAliveProgressResetByHuman(current);
|
|
1582
1755
|
const attempts = resetByHuman ? 0 : current.keep_alive_resume_count ?? 0;
|
|
@@ -1584,8 +1757,9 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
|
|
|
1584
1757
|
if (current.keep_alive_exhausted_at !== undefined)
|
|
1585
1758
|
continue;
|
|
1586
1759
|
const exhausted = casLaneIf(hqRoot, current.lane_id, (latestLane) => latestLane.version === current.version &&
|
|
1587
|
-
latestLane.state ===
|
|
1588
|
-
|
|
1760
|
+
latestLane.state === current.state &&
|
|
1761
|
+
latestLane.keep_alive === true &&
|
|
1762
|
+
latestLane.keep_alive_opted_in_at === current.keep_alive_opted_in_at &&
|
|
1589
1763
|
(latestLane.keep_alive_resume_count ?? 0) >= LEAD_LANE_KEEP_ALIVE_MAX_RESUMES &&
|
|
1590
1764
|
latestLane.keep_alive_exhausted_at === undefined, (latestLane) => ({ ...latestLane, keep_alive_exhausted_at: nowIso }), deps.dryRun);
|
|
1591
1765
|
if (exhausted.changed && !deps.dryRun) {
|
|
@@ -1593,13 +1767,16 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
|
|
|
1593
1767
|
event: "keep_alive_exhausted",
|
|
1594
1768
|
at: nowIso,
|
|
1595
1769
|
attempt: attempts,
|
|
1770
|
+
resumed_from: current.state,
|
|
1596
1771
|
});
|
|
1597
1772
|
}
|
|
1598
1773
|
continue;
|
|
1599
1774
|
}
|
|
1600
1775
|
const attempt = attempts + 1;
|
|
1601
|
-
const
|
|
1602
|
-
|
|
1776
|
+
const terminalAtMs = initialState === "done"
|
|
1777
|
+
? Date.parse(current.updated_at)
|
|
1778
|
+
: Date.parse(current.interrupted_at ?? "");
|
|
1779
|
+
const attemptEligibleMs = terminalAtMs + leadLaneKeepAliveBackoffMs(attempt);
|
|
1603
1780
|
const storedNextMs = resetByHuman
|
|
1604
1781
|
? Number.NaN
|
|
1605
1782
|
: Date.parse(current.keep_alive_next_eligible_at ?? "");
|
|
@@ -1609,8 +1786,9 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
|
|
|
1609
1786
|
if (nowMs < nextEligibleMs) {
|
|
1610
1787
|
if (resetByHuman && hasKeepAliveProgress(current)) {
|
|
1611
1788
|
casLaneIf(hqRoot, current.lane_id, (latestLane) => latestLane.version === current.version &&
|
|
1612
|
-
latestLane.state ===
|
|
1613
|
-
|
|
1789
|
+
latestLane.state === current.state &&
|
|
1790
|
+
latestLane.keep_alive === true &&
|
|
1791
|
+
latestLane.keep_alive_opted_in_at === current.keep_alive_opted_in_at, (latestLane) => clearedKeepAliveProgress(latestLane), deps.dryRun);
|
|
1614
1792
|
}
|
|
1615
1793
|
continue;
|
|
1616
1794
|
}
|
|
@@ -1628,9 +1806,14 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
|
|
|
1628
1806
|
});
|
|
1629
1807
|
continue;
|
|
1630
1808
|
}
|
|
1809
|
+
const resumedFrom = current.state;
|
|
1810
|
+
const resumeText = resumedFrom === "interrupted"
|
|
1811
|
+
? LEAD_LANE_KEEP_ALIVE_INTERRUPTED_RESUME_TEXT
|
|
1812
|
+
: LEAD_LANE_KEEP_ALIVE_RESUME_TEXT;
|
|
1631
1813
|
const queue = (expectedSeniorId) => casLaneIf(hqRoot, current.lane_id, (latestLane) => latestLane.version === current.version &&
|
|
1632
|
-
latestLane.state ===
|
|
1633
|
-
|
|
1814
|
+
latestLane.state === current.state &&
|
|
1815
|
+
latestLane.keep_alive === true &&
|
|
1816
|
+
latestLane.keep_alive_opted_in_at === current.keep_alive_opted_in_at &&
|
|
1634
1817
|
(expectedSeniorId === undefined ||
|
|
1635
1818
|
(latestLane.senior.kind === "lane" && latestLane.senior.id === expectedSeniorId)), (latestLane) => ({
|
|
1636
1819
|
...latestLane,
|
|
@@ -1641,10 +1824,10 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
|
|
|
1641
1824
|
terminal_reason: undefined,
|
|
1642
1825
|
terminal_details: undefined,
|
|
1643
1826
|
pending_resume: {
|
|
1644
|
-
session_id:
|
|
1827
|
+
session_id: currentSessionId,
|
|
1645
1828
|
trigger: "resume",
|
|
1646
|
-
resumed_from:
|
|
1647
|
-
text:
|
|
1829
|
+
resumed_from: resumedFrom,
|
|
1830
|
+
text: resumeText,
|
|
1648
1831
|
compact: true,
|
|
1649
1832
|
at: nowIso,
|
|
1650
1833
|
},
|
|
@@ -1693,7 +1876,7 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
|
|
|
1693
1876
|
continue;
|
|
1694
1877
|
summary.resumed.push({
|
|
1695
1878
|
id: current.lane_id,
|
|
1696
|
-
from:
|
|
1879
|
+
from: resumedFrom,
|
|
1697
1880
|
to: "queued",
|
|
1698
1881
|
reason: "keep_alive",
|
|
1699
1882
|
});
|
|
@@ -1703,6 +1886,7 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
|
|
|
1703
1886
|
at: nowIso,
|
|
1704
1887
|
attempt,
|
|
1705
1888
|
next_eligible_at: nextEligibleAt,
|
|
1889
|
+
resumed_from: resumedFrom,
|
|
1706
1890
|
});
|
|
1707
1891
|
}
|
|
1708
1892
|
}
|
package/dist/lib/lanes/store.js
CHANGED
|
@@ -414,6 +414,8 @@ export function parseLaneRecord(raw, filePath) {
|
|
|
414
414
|
"not_opted_in",
|
|
415
415
|
"missing_opt_in_time",
|
|
416
416
|
"envelope_before_opt_in",
|
|
417
|
+
"interruption_before_opt_in",
|
|
418
|
+
"missing_interruption_time",
|
|
417
419
|
"retired",
|
|
418
420
|
"superseded",
|
|
419
421
|
"retired_list_invalid",
|
|
@@ -443,6 +445,7 @@ export function parseLaneRecord(raw, filePath) {
|
|
|
443
445
|
"keep_alive_last_resume_at",
|
|
444
446
|
"keep_alive_next_eligible_at",
|
|
445
447
|
"keep_alive_exhausted_at",
|
|
448
|
+
"interrupted_at",
|
|
446
449
|
]) {
|
|
447
450
|
if (raw[field] !== undefined)
|
|
448
451
|
requireTimestamp(raw[field], field, filePath);
|
|
@@ -359,7 +359,7 @@ export interface LaneRecord {
|
|
|
359
359
|
/** Time this lane most recently explicitly opted into keep-alive. */
|
|
360
360
|
keep_alive_opted_in_at?: string;
|
|
361
361
|
/** Why reconcile declined to resume this lane most recently. */
|
|
362
|
-
keep_alive_blocked_reason?: "not_opted_in" | "missing_opt_in_time" | "envelope_before_opt_in" | "retired" | "superseded" | "retired_list_invalid" | "terminal_state";
|
|
362
|
+
keep_alive_blocked_reason?: "not_opted_in" | "missing_opt_in_time" | "envelope_before_opt_in" | "interruption_before_opt_in" | "missing_interruption_time" | "retired" | "superseded" | "retired_list_invalid" | "terminal_state";
|
|
363
363
|
/** Consecutive automatic keep-alive resumes since the last human reset. */
|
|
364
364
|
keep_alive_resume_count?: number;
|
|
365
365
|
/** Time the most recent automatic keep-alive resume was queued. */
|
|
@@ -451,6 +451,8 @@ export interface LaneRecord {
|
|
|
451
451
|
/** Set when the reconcile pass force-admits under the starvation bound. */
|
|
452
452
|
admitted_reason?: string;
|
|
453
453
|
terminal_reason?: string;
|
|
454
|
+
/** Classification time when this lane entered interrupted; stable across later metadata writes. */
|
|
455
|
+
interrupted_at?: string;
|
|
454
456
|
/**
|
|
455
457
|
* Structured detail behind a named `terminal_reason` (dispatch, pickup and
|
|
456
458
|
* spawn failures), the missing-binary search path behind
|
package/dist/lib/lanes/types.js
CHANGED
|
@@ -230,7 +230,7 @@ export const LANE_FIELD_WRITERS = {
|
|
|
230
230
|
last_envelope: ["watcher", "reconcile"], model: ["creator", "spawn", "command"], model_source: ["creator", "spawn", "command"], effort: ["creator", "spawn", "command"], effort_source: ["creator", "spawn", "command"], fast_mode: ["spawn"],
|
|
231
231
|
last_message_at: ["command"], last_compaction: ["spawn"], story_complete: ["watcher", "reconcile"], story_complete_reason: ["watcher", "reconcile"], story_status: ["watcher", "reconcile"],
|
|
232
232
|
overdue: ["reconcile"], overdue_at: ["reconcile"], last_host_report_at: ["watcher", "spawn", "reconcile"], stalled: ["watcher"], stalled_at: ["watcher"], stalled_reason: ["watcher"],
|
|
233
|
-
admitted_reason: ["command", "reconcile"], terminal_reason: ["command", "watcher", "spawn", "reconcile"], terminal_details: ["command", "spawn", "watcher", "reconcile"],
|
|
233
|
+
admitted_reason: ["command", "reconcile"], terminal_reason: ["command", "watcher", "spawn", "reconcile"], interrupted_at: ["watcher", "reconcile"], terminal_details: ["command", "spawn", "watcher", "reconcile"],
|
|
234
234
|
quota_reset_at: ["watcher", "reconcile"], model_quota_marker: ["watcher", "reconcile"], outpost_command_id: ["command"],
|
|
235
235
|
provider_session_id: ["watcher", "command", "spawn"], provider_session_id_absent: ["watcher", "command"], provider_home_at_spawn: ["spawn"], last_resume: ["spawn"], last_run_usage: ["watcher"], envelope_repair: ["watcher", "reconcile"], envelope_validation_errors: ["watcher"], last_progress_event_at: ["watcher"],
|
|
236
236
|
};
|
|
@@ -336,6 +336,7 @@ export function applyTerminalToLaneWithResult(hqRoot, laneId, classification, op
|
|
|
336
336
|
stalled: undefined,
|
|
337
337
|
stalled_at: undefined,
|
|
338
338
|
stalled_reason: undefined,
|
|
339
|
+
interrupted_at: classification.state === "interrupted" ? at : undefined,
|
|
339
340
|
envelope_validation_errors: opts.envelopeValidationErrors,
|
|
340
341
|
};
|
|
341
342
|
changed = true;
|