@indigoai-us/hq-cli 5.216.0 → 5.217.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -0
- package/dist/lib/lanes/keep-alive.d.ts +1 -0
- package/dist/lib/lanes/keep-alive.js +1 -0
- package/dist/lib/lanes/reconcile.d.ts +2 -0
- package/dist/lib/lanes/reconcile.js +234 -77
- package/dist/lib/lanes/store.js +3 -0
- package/dist/lib/lanes/types.d.ts +3 -1
- package/dist/lib/lanes/types.js +1 -1
- package/dist/lib/lanes/watcher.js +1 -0
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,14 @@
|
|
|
2
2
|
|
|
3
3
|
## [Unreleased]
|
|
4
4
|
|
|
5
|
+
## [5.217.0] — 2026-09-26
|
|
6
|
+
|
|
7
|
+
### Fixed
|
|
8
|
+
|
|
9
|
+
- Keep-alive resumes a lane interrupted by `exited_with_partial_output` when
|
|
10
|
+
the interruption followed its explicit opt-in. Quota and host-loss
|
|
11
|
+
interruptions, login failures, and stopped lanes remain ineligible.
|
|
12
|
+
|
|
5
13
|
## [5.216.0] — 2026-09-26
|
|
6
14
|
|
|
7
15
|
### Fixed
|
|
@@ -5,6 +5,7 @@ export declare const LEAD_LANE_KEEP_ALIVE_MAX_RESUMES = 24;
|
|
|
5
5
|
export declare const LEAD_LANE_KEEP_ALIVE_BASE_BACKOFF_MS: number;
|
|
6
6
|
export declare const LEAD_LANE_KEEP_ALIVE_MAX_BACKOFF_MS: number;
|
|
7
7
|
export declare const LEAD_LANE_KEEP_ALIVE_RESUME_TEXT = "Automatic keep-alive resume: your last turn ended with decision done and story_complete false. Continue the open work from where you stopped. End with done only when story_complete is true; if you are waiting, use hq lanes ask or stay in the loop.";
|
|
8
|
+
export declare const LEAD_LANE_KEEP_ALIVE_INTERRUPTED_RESUME_TEXT = "Automatic keep-alive resume: your previous turn was interrupted by a provider process exit. Check your last state before continuing. Continue the open work from where you stopped. End with done only when story_complete is true; if you are waiting, use hq lanes ask or stay in the loop.";
|
|
8
9
|
export type KeepAliveBlockedReason = NonNullable<LaneRecord["keep_alive_blocked_reason"]>;
|
|
9
10
|
export declare function retiredLaneRegistryPath(hqRoot: string): string;
|
|
10
11
|
/** Missing registry means no retired lanes; every present invalid file fails closed. */
|
|
@@ -7,6 +7,7 @@ export const LEAD_LANE_KEEP_ALIVE_MAX_RESUMES = 24;
|
|
|
7
7
|
export const LEAD_LANE_KEEP_ALIVE_BASE_BACKOFF_MS = 5 * 60 * 1000;
|
|
8
8
|
export const LEAD_LANE_KEEP_ALIVE_MAX_BACKOFF_MS = 60 * 60 * 1000;
|
|
9
9
|
export const LEAD_LANE_KEEP_ALIVE_RESUME_TEXT = "Automatic keep-alive resume: your last turn ended with decision done and story_complete false. Continue the open work from where you stopped. End with done only when story_complete is true; if you are waiting, use hq lanes ask or stay in the loop.";
|
|
10
|
+
export const LEAD_LANE_KEEP_ALIVE_INTERRUPTED_RESUME_TEXT = "Automatic keep-alive resume: your previous turn was interrupted by a provider process exit. Check your last state before continuing. Continue the open work from where you stopped. End with done only when story_complete is true; if you are waiting, use hq lanes ask or stay in the loop.";
|
|
10
11
|
export function retiredLaneRegistryPath(hqRoot) {
|
|
11
12
|
return path.join(hqRoot, "workspace", "lane-briefs", "juniors.json");
|
|
12
13
|
}
|
|
@@ -78,6 +78,8 @@ export interface ReconcileDeps {
|
|
|
78
78
|
mesh?: LaneMesh | null;
|
|
79
79
|
/** Bound on one terminal mesh emit during dead-host recovery. Default 10 s. */
|
|
80
80
|
meshEmitTimeoutMs?: number;
|
|
81
|
+
/** Fresh keep-alive eligibility read; injected to exercise opt-in changes before queueing. */
|
|
82
|
+
readKeepAliveRecheckLane?: (hqRoot: string, laneId: string) => LaneRecord;
|
|
81
83
|
/**
|
|
82
84
|
* Awaited after lane/question records and liveness are loaded, before any
|
|
83
85
|
* CAS write. Test seam so two in-process passes can share a snapshot.
|
|
@@ -21,11 +21,11 @@ import { ANSWERED_QUESTIONS_FILE, armLaneWatcher, envelopeRepairMode, isPidRecor
|
|
|
21
21
|
import { appendCappedHistory, lanesStoreRoot, parseLaneRecord, parseQuestionRecord, readLane, readQuestion, updateLane, updateQuestion, } from "./store.js";
|
|
22
22
|
import { applyThrottleDecision, decideAdmission, defaultThrottleDeps, emptyThrottleState, evaluateThrottle, isCpuCheckEnabled, readAvailableMemMb, readThrottleState, resolveCpuBusyFraction, throttleStatePath, writeThrottleState, } from "./throttle.js";
|
|
23
23
|
import { readLanesRegistryFlag } from "./registry-flag.js";
|
|
24
|
-
import { LEAD_LANE_KEEP_ALIVE_FLAG, LEAD_LANE_KEEP_ALIVE_MAX_RESUMES, LEAD_LANE_KEEP_ALIVE_RESUME_TEXT, keepAliveExclusion, loadRetiredLaneIds, leadLaneKeepAliveBackoffMs, } from "./keep-alive.js";
|
|
24
|
+
import { LEAD_LANE_KEEP_ALIVE_FLAG, LEAD_LANE_KEEP_ALIVE_INTERRUPTED_RESUME_TEXT, LEAD_LANE_KEEP_ALIVE_MAX_RESUMES, LEAD_LANE_KEEP_ALIVE_RESUME_TEXT, keepAliveExclusion, loadRetiredLaneIds, leadLaneKeepAliveBackoffMs, } from "./keep-alive.js";
|
|
25
25
|
import { providerSupportsSessionResume } from "./delivery.js";
|
|
26
26
|
import { applyDefaultQuotaHold, readQuotaSignal, recordAccountQuota } from "./quota.js";
|
|
27
27
|
import { answeredQuestionEntries, raiseEnvelopeQuestions } from "./questions.js";
|
|
28
|
-
import { SILENCE_MS, classifyLaneExit, isPidAlive, missingExpectedEnvelopeFile, readLatestEnvelopeFile, readLatestEnvelopeRecord, watcherEventsPath, writeWatcherDecision, } from "./watcher.js";
|
|
28
|
+
import { SILENCE_MS, classifyLaneExit, isPidAlive, missingExpectedEnvelopeFile, readLatestEnvelopeFile, readLatestEnvelopeRecord, watcherEventsPath, watcherOutLogPath, writeWatcherDecision, } from "./watcher.js";
|
|
29
29
|
import { createLaneMesh, emitLaneDecisionEvents, emitLaneTerminalStateEvent, laneMeshCompany, } from "./mesh-events.js";
|
|
30
30
|
import { laneStories, storyStatusForEnvelope } from "./stories.js";
|
|
31
31
|
import { decideEnvelopeRepair, envelopeQuestionIds, routeProviderFailure } from "./watcher.js";
|
|
@@ -800,7 +800,7 @@ export async function runPassBody(hqRoot, deps) {
|
|
|
800
800
|
const keepAliveFlagCompanies = new Map();
|
|
801
801
|
const keepAliveHireGrantFlagCompanies = new Map();
|
|
802
802
|
for (const lane of lanes) {
|
|
803
|
-
if (!(lane.state === "done" || lane.state === "killed" || lane.state === "cancelled" || lane.state === "failed") ||
|
|
803
|
+
if (!(lane.state === "done" || lane.state === "interrupted" || lane.state === "killed" || lane.state === "cancelled" || lane.state === "failed") ||
|
|
804
804
|
(lane.keep_alive !== true && lane.worker_profile === undefined) ||
|
|
805
805
|
!laneRunsHere(lane, deps.host) ||
|
|
806
806
|
(lane.state === "done" && (lane.terminal_reason === PROVIDER_REJECTED_REASON ||
|
|
@@ -838,7 +838,7 @@ export async function runPassBody(hqRoot, deps) {
|
|
|
838
838
|
]);
|
|
839
839
|
await withAllLaneAdmissionLocks(hqRoot, async () => {
|
|
840
840
|
const freshLanes = loadLaneRecords(hqRoot, summary);
|
|
841
|
-
|
|
841
|
+
resumeKeepAlive(hqRoot, freshLanes, summary, deps, now, keepAliveFlagByCompany, keepAliveHireGrantFlagByCompany);
|
|
842
842
|
const drainLanes = loadLaneRecords(hqRoot, summary);
|
|
843
843
|
const freshQuestions = loadQuestionRecords(hqRoot, summary);
|
|
844
844
|
await admitQueued(hqRoot, drainLanes, summary, deps, now);
|
|
@@ -1059,6 +1059,7 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
|
|
|
1059
1059
|
state: repairedClassification.state,
|
|
1060
1060
|
queued_reason: undefined,
|
|
1061
1061
|
terminal_reason: repairedClassification.reason ?? "host_lost",
|
|
1062
|
+
interrupted_at: repairedClassification.state === "interrupted" ? nowIso : undefined,
|
|
1062
1063
|
...(providerRejection
|
|
1063
1064
|
? { terminal_details: providerRejectionDetails(providerRejection, current) }
|
|
1064
1065
|
: {}),
|
|
@@ -1175,6 +1176,7 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
|
|
|
1175
1176
|
state: "interrupted",
|
|
1176
1177
|
queued_reason: undefined,
|
|
1177
1178
|
terminal_reason: "host_lost",
|
|
1179
|
+
interrupted_at: nowIso,
|
|
1178
1180
|
}), deps.dryRun);
|
|
1179
1181
|
if (result.changed) {
|
|
1180
1182
|
summary.dead_host.push({
|
|
@@ -1392,7 +1394,8 @@ function keepAliveProgressResetByHuman(lane) {
|
|
|
1392
1394
|
if (count > 0 && !Number.isFinite(lastAutoMs))
|
|
1393
1395
|
return true;
|
|
1394
1396
|
const messageMs = Date.parse(lane.last_message_at ?? "");
|
|
1395
|
-
const resumeMs = lane.last_resume?.text === LEAD_LANE_KEEP_ALIVE_RESUME_TEXT
|
|
1397
|
+
const resumeMs = lane.last_resume?.text === LEAD_LANE_KEEP_ALIVE_RESUME_TEXT ||
|
|
1398
|
+
lane.last_resume?.text === LEAD_LANE_KEEP_ALIVE_INTERRUPTED_RESUME_TEXT
|
|
1396
1399
|
? Number.NaN
|
|
1397
1400
|
: Date.parse(lane.last_resume?.at ?? "");
|
|
1398
1401
|
return Number.isFinite(lastAutoMs) &&
|
|
@@ -1433,6 +1436,36 @@ function recordKeepAliveSkip(hqRoot, lane, reason, summary, deps, detail) {
|
|
|
1433
1436
|
});
|
|
1434
1437
|
}
|
|
1435
1438
|
}
|
|
1439
|
+
function readLatestInterruptedClassificationAt(hqRoot, lane) {
|
|
1440
|
+
const logPath = watcherOutLogPath(hqRoot, lane.lane_id);
|
|
1441
|
+
let body;
|
|
1442
|
+
try {
|
|
1443
|
+
body = fs.readFileSync(logPath, "utf8");
|
|
1444
|
+
}
|
|
1445
|
+
catch (err) {
|
|
1446
|
+
if (err.code === "ENOENT")
|
|
1447
|
+
return undefined;
|
|
1448
|
+
throw err;
|
|
1449
|
+
}
|
|
1450
|
+
let latestClassification;
|
|
1451
|
+
for (const line of body.split(/\r?\n/).filter(Boolean)) {
|
|
1452
|
+
const parsed = JSON.parse(line);
|
|
1453
|
+
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
|
|
1454
|
+
throw new Error("watcher log contains a non-object decision record");
|
|
1455
|
+
}
|
|
1456
|
+
const record = parsed;
|
|
1457
|
+
if (record.event === "classification")
|
|
1458
|
+
latestClassification = record;
|
|
1459
|
+
}
|
|
1460
|
+
const at = latestClassification?.at;
|
|
1461
|
+
if (latestClassification?.state !== "interrupted" ||
|
|
1462
|
+
latestClassification.reason !== lane.terminal_reason ||
|
|
1463
|
+
typeof at !== "string" ||
|
|
1464
|
+
!Number.isFinite(Date.parse(at)) ||
|
|
1465
|
+
new Date(Date.parse(at)).toISOString() !== at)
|
|
1466
|
+
return undefined;
|
|
1467
|
+
return at;
|
|
1468
|
+
}
|
|
1436
1469
|
function latestEnvelopeMtimeMs(hqRoot, lane, ref) {
|
|
1437
1470
|
const filePath = path.join(laneRunDir(hqRoot, lane.lane_id), ref);
|
|
1438
1471
|
try {
|
|
@@ -1442,7 +1475,7 @@ function latestEnvelopeMtimeMs(hqRoot, lane, ref) {
|
|
|
1442
1475
|
throw new LanesError("keep_alive_envelope_timestamp_unavailable", `Cannot stat latest envelope for lane ${lane.lane_id}: ${err instanceof Error ? err.message : String(err)}`, { lane_id: lane.lane_id, path: filePath });
|
|
1443
1476
|
}
|
|
1444
1477
|
}
|
|
1445
|
-
function
|
|
1478
|
+
function resumeKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, hireGrantFlagByCompany) {
|
|
1446
1479
|
const nowMs = now.getTime();
|
|
1447
1480
|
const nowIso = now.toISOString();
|
|
1448
1481
|
const config = loadLanesConfig(hqRoot);
|
|
@@ -1456,34 +1489,97 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
|
|
|
1456
1489
|
markError(summary, err);
|
|
1457
1490
|
}
|
|
1458
1491
|
}
|
|
1459
|
-
for (const
|
|
1460
|
-
if (!laneRunsHere(
|
|
1492
|
+
for (const originalLane of lanes) {
|
|
1493
|
+
if (!laneRunsHere(originalLane, deps.host))
|
|
1461
1494
|
continue;
|
|
1462
|
-
if (
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
|
|
1495
|
+
if (originalLane.state === "killed" ||
|
|
1496
|
+
originalLane.state === "cancelled" ||
|
|
1497
|
+
originalLane.state === "failed") {
|
|
1498
|
+
if (originalLane.keep_alive === true &&
|
|
1499
|
+
flagByCompany.get(keepAliveCompanyKey(originalLane.company)) === true)
|
|
1500
|
+
recordKeepAliveSkip(hqRoot, originalLane, "terminal_state", summary, deps);
|
|
1466
1501
|
continue;
|
|
1467
1502
|
}
|
|
1468
|
-
if (
|
|
1469
|
-
(
|
|
1503
|
+
if ((originalLane.state !== "done" && originalLane.state !== "interrupted") ||
|
|
1504
|
+
(originalLane.keep_alive !== true && originalLane.worker_profile === undefined))
|
|
1470
1505
|
continue;
|
|
1471
|
-
const
|
|
1472
|
-
|
|
1473
|
-
if (
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1506
|
+
const initialState = originalLane.state;
|
|
1507
|
+
let lane = originalLane;
|
|
1508
|
+
if (initialState === "done") {
|
|
1509
|
+
const envelope = readLatestEnvelopeRecord(hqRoot, lane.lane_id)?.envelope;
|
|
1510
|
+
if (!envelope || envelope.decision !== "done")
|
|
1511
|
+
continue;
|
|
1512
|
+
if (envelope.story_complete === true) {
|
|
1513
|
+
if (hasKeepAliveProgress(lane)) {
|
|
1514
|
+
casLaneIf(hqRoot, lane.lane_id, (current) => current.version === lane.version &&
|
|
1515
|
+
current.state === "done" &&
|
|
1516
|
+
current.keep_alive === lane.keep_alive, (current) => ({ ...clearedKeepAliveProgress(current), keep_alive_blocked_reason: undefined }), deps.dryRun);
|
|
1517
|
+
}
|
|
1518
|
+
continue;
|
|
1480
1519
|
}
|
|
1481
|
-
continue;
|
|
1482
1520
|
}
|
|
1483
1521
|
if (flagByCompany.get(keepAliveCompanyKey(lane.company)) !== true)
|
|
1484
1522
|
continue;
|
|
1485
|
-
|
|
1486
|
-
|
|
1523
|
+
// Persist a stable timestamp for legacy interrupted rows before a later
|
|
1524
|
+
// keep-alive status write can advance their general updated_at clock. The
|
|
1525
|
+
// watcher classification is the interruption time; updated_at is only an
|
|
1526
|
+
// upper bound and can postdate explicit opt-in.
|
|
1527
|
+
if (initialState === "interrupted" && lane.keep_alive === true && lane.interrupted_at === undefined) {
|
|
1528
|
+
let interruptedAt;
|
|
1529
|
+
try {
|
|
1530
|
+
interruptedAt = readLatestInterruptedClassificationAt(hqRoot, lane);
|
|
1531
|
+
}
|
|
1532
|
+
catch (err) {
|
|
1533
|
+
const detail = `watcher classification read failed: ${err instanceof Error ? err.message : String(err)}`;
|
|
1534
|
+
recordKeepAliveSkip(hqRoot, lane, "missing_interruption_time", summary, deps, detail);
|
|
1535
|
+
const skip = summary.skipped[summary.skipped.length - 1];
|
|
1536
|
+
if (skip) {
|
|
1537
|
+
summary.ok = false;
|
|
1538
|
+
logSkip(skip);
|
|
1539
|
+
}
|
|
1540
|
+
continue;
|
|
1541
|
+
}
|
|
1542
|
+
if (interruptedAt === undefined) {
|
|
1543
|
+
recordKeepAliveSkip(hqRoot, lane, "missing_interruption_time", summary, deps, "no latest matching watcher classification event");
|
|
1544
|
+
const skip = summary.skipped[summary.skipped.length - 1];
|
|
1545
|
+
if (skip) {
|
|
1546
|
+
summary.ok = false;
|
|
1547
|
+
logSkip(skip);
|
|
1548
|
+
}
|
|
1549
|
+
continue;
|
|
1550
|
+
}
|
|
1551
|
+
try {
|
|
1552
|
+
const backfill = casLaneIf(hqRoot, lane.lane_id, (current) => current.version === lane.version &&
|
|
1553
|
+
current.state === "interrupted" &&
|
|
1554
|
+
current.keep_alive === true &&
|
|
1555
|
+
current.terminal_reason === lane.terminal_reason &&
|
|
1556
|
+
current.interrupted_at === undefined, (current) => ({ ...current, interrupted_at: interruptedAt }), deps.dryRun);
|
|
1557
|
+
lane = backfill.record;
|
|
1558
|
+
if (lane.interrupted_at === undefined) {
|
|
1559
|
+
recordKeepAliveSkip(hqRoot, lane, "missing_interruption_time", summary, deps, "lane changed before watcher classification timestamp could be recorded");
|
|
1560
|
+
const skip = summary.skipped[summary.skipped.length - 1];
|
|
1561
|
+
if (skip) {
|
|
1562
|
+
summary.ok = false;
|
|
1563
|
+
logSkip(skip);
|
|
1564
|
+
}
|
|
1565
|
+
continue;
|
|
1566
|
+
}
|
|
1567
|
+
}
|
|
1568
|
+
catch (err) {
|
|
1569
|
+
markDirty(summary, {
|
|
1570
|
+
kind: "lane",
|
|
1571
|
+
id: lane.lane_id,
|
|
1572
|
+
why: `keep-alive interruption timestamp backfill failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
1573
|
+
});
|
|
1574
|
+
continue;
|
|
1575
|
+
}
|
|
1576
|
+
}
|
|
1577
|
+
if (initialState === "interrupted" && lane.terminal_reason !== "exited_with_partial_output") {
|
|
1578
|
+
recordKeepAliveSkip(hqRoot, lane, "terminal_state", summary, deps, lane.terminal_reason ?? "missing interruption reason");
|
|
1579
|
+
continue;
|
|
1580
|
+
}
|
|
1581
|
+
if (initialState === "done" && (lane.terminal_reason === PROVIDER_REJECTED_REASON ||
|
|
1582
|
+
lane.terminal_reason === "account_not_logged_in")) {
|
|
1487
1583
|
recordKeepAliveSkip(hqRoot, lane, "terminal_state", summary, deps, lane.terminal_reason);
|
|
1488
1584
|
continue;
|
|
1489
1585
|
}
|
|
@@ -1496,20 +1592,35 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
|
|
|
1496
1592
|
recordKeepAliveSkip(hqRoot, lane, "missing_opt_in_time", summary, deps);
|
|
1497
1593
|
continue;
|
|
1498
1594
|
}
|
|
1499
|
-
let
|
|
1500
|
-
|
|
1501
|
-
|
|
1595
|
+
let eligibilityAtMs;
|
|
1596
|
+
if (initialState === "done") {
|
|
1597
|
+
const latest = readLatestEnvelopeRecord(hqRoot, lane.lane_id);
|
|
1598
|
+
if (!latest || latest.envelope.decision !== "done" || latest.envelope.story_complete === true)
|
|
1599
|
+
continue;
|
|
1600
|
+
try {
|
|
1601
|
+
eligibilityAtMs = latestEnvelopeMtimeMs(hqRoot, lane, latest.ref);
|
|
1602
|
+
}
|
|
1603
|
+
catch (err) {
|
|
1604
|
+
markDirty(summary, {
|
|
1605
|
+
kind: "lane",
|
|
1606
|
+
id: lane.lane_id,
|
|
1607
|
+
why: `keep-alive envelope timestamp unavailable: ${err instanceof Error ? err.message : String(err)}`,
|
|
1608
|
+
});
|
|
1609
|
+
continue;
|
|
1610
|
+
}
|
|
1502
1611
|
}
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
|
|
1508
|
-
}
|
|
1509
|
-
continue;
|
|
1612
|
+
else {
|
|
1613
|
+
eligibilityAtMs = Date.parse(lane.interrupted_at ?? "");
|
|
1614
|
+
if (!Number.isFinite(eligibilityAtMs)) {
|
|
1615
|
+
recordKeepAliveSkip(hqRoot, lane, "missing_interruption_time", summary, deps);
|
|
1616
|
+
continue;
|
|
1617
|
+
}
|
|
1510
1618
|
}
|
|
1511
|
-
|
|
1512
|
-
|
|
1619
|
+
const initialTimeBlockReason = initialState === "done"
|
|
1620
|
+
? "envelope_before_opt_in"
|
|
1621
|
+
: "interruption_before_opt_in";
|
|
1622
|
+
if (eligibilityAtMs <= optedInAtMs) {
|
|
1623
|
+
recordKeepAliveSkip(hqRoot, lane, initialTimeBlockReason, summary, deps);
|
|
1513
1624
|
continue;
|
|
1514
1625
|
}
|
|
1515
1626
|
if (retiredLaneIds === null) {
|
|
@@ -1521,33 +1632,54 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
|
|
|
1521
1632
|
recordKeepAliveSkip(hqRoot, lane, exclusion, summary, deps);
|
|
1522
1633
|
continue;
|
|
1523
1634
|
}
|
|
1635
|
+
const providerSessionId = lane.provider_session_id ?? lane.last_resume?.session_id;
|
|
1524
1636
|
if (!providerSupportsSessionResume(lane.provider) ||
|
|
1525
|
-
|
|
1637
|
+
typeof providerSessionId !== "string" ||
|
|
1638
|
+
providerSessionId.trim().length === 0) {
|
|
1639
|
+
if (initialState === "interrupted") {
|
|
1640
|
+
recordKeepAliveSkip(hqRoot, lane, "terminal_state", summary, deps, "provider session cannot be resumed");
|
|
1641
|
+
}
|
|
1526
1642
|
continue;
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
|
|
1643
|
+
}
|
|
1644
|
+
// This second read is intentionally independent of the pass snapshot. The
|
|
1645
|
+
// final timestamp comparison below closes the opt-in race before the CAS.
|
|
1646
|
+
const rechecked = deps.readKeepAliveRecheckLane?.(hqRoot, lane.lane_id) ?? readLane(hqRoot, lane.lane_id);
|
|
1647
|
+
// Dry-run backfills only in memory. Carry that simulated legacy timestamp
|
|
1648
|
+
// through the fresh read when the stored lane version has not changed.
|
|
1649
|
+
const current = deps.dryRun &&
|
|
1650
|
+
initialState === "interrupted" &&
|
|
1651
|
+
originalLane.interrupted_at === undefined &&
|
|
1652
|
+
lane.interrupted_at !== undefined &&
|
|
1653
|
+
rechecked.version === lane.version &&
|
|
1654
|
+
rechecked.interrupted_at === undefined
|
|
1655
|
+
? { ...rechecked, interrupted_at: lane.interrupted_at }
|
|
1656
|
+
: rechecked;
|
|
1657
|
+
if (current.state !== initialState || !laneRunsHere(current, deps.host))
|
|
1534
1658
|
continue;
|
|
1535
|
-
|
|
1536
|
-
|
|
1537
|
-
if (!currentEnvelope || currentEnvelope.decision !== "done")
|
|
1659
|
+
if (current.keep_alive !== true) {
|
|
1660
|
+
recordKeepAliveSkip(hqRoot, current, "not_opted_in", summary, deps);
|
|
1538
1661
|
continue;
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
latestLane.state === "done" &&
|
|
1543
|
-
keepAliveStillEnabled(latestLane), (latestLane) => ({ ...clearedKeepAliveProgress(latestLane), keep_alive_blocked_reason: undefined }), deps.dryRun);
|
|
1544
|
-
}
|
|
1662
|
+
}
|
|
1663
|
+
if (initialState === "interrupted" && current.terminal_reason !== "exited_with_partial_output") {
|
|
1664
|
+
recordKeepAliveSkip(hqRoot, current, "terminal_state", summary, deps, current.terminal_reason ?? "missing interruption reason");
|
|
1545
1665
|
continue;
|
|
1546
1666
|
}
|
|
1547
|
-
|
|
1548
|
-
|
|
1667
|
+
let currentEligibilityAtMs;
|
|
1668
|
+
if (initialState === "done") {
|
|
1669
|
+
const currentEnvelope = readLatestEnvelopeRecord(hqRoot, current.lane_id);
|
|
1670
|
+
if (!currentEnvelope || currentEnvelope.envelope.decision !== "done")
|
|
1671
|
+
continue;
|
|
1672
|
+
if (currentEnvelope.envelope.story_complete === true) {
|
|
1673
|
+
if (hasKeepAliveProgress(current)) {
|
|
1674
|
+
casLaneIf(hqRoot, current.lane_id, (latestLane) => latestLane.version === current.version &&
|
|
1675
|
+
latestLane.state === "done" &&
|
|
1676
|
+
latestLane.keep_alive === true &&
|
|
1677
|
+
latestLane.keep_alive_opted_in_at === current.keep_alive_opted_in_at, (latestLane) => ({ ...clearedKeepAliveProgress(latestLane), keep_alive_blocked_reason: undefined }), deps.dryRun);
|
|
1678
|
+
}
|
|
1679
|
+
continue;
|
|
1680
|
+
}
|
|
1549
1681
|
try {
|
|
1550
|
-
|
|
1682
|
+
currentEligibilityAtMs = latestEnvelopeMtimeMs(hqRoot, current, currentEnvelope.ref);
|
|
1551
1683
|
}
|
|
1552
1684
|
catch (err) {
|
|
1553
1685
|
markDirty(summary, {
|
|
@@ -1557,11 +1689,26 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
|
|
|
1557
1689
|
});
|
|
1558
1690
|
continue;
|
|
1559
1691
|
}
|
|
1560
|
-
|
|
1561
|
-
|
|
1692
|
+
}
|
|
1693
|
+
else {
|
|
1694
|
+
currentEligibilityAtMs = Date.parse(current.interrupted_at ?? "");
|
|
1695
|
+
if (!Number.isFinite(currentEligibilityAtMs)) {
|
|
1696
|
+
recordKeepAliveSkip(hqRoot, current, "missing_interruption_time", summary, deps);
|
|
1562
1697
|
continue;
|
|
1563
1698
|
}
|
|
1564
1699
|
}
|
|
1700
|
+
const currentOptedInAtMs = Date.parse(current.keep_alive_opted_in_at ?? "");
|
|
1701
|
+
if (!Number.isFinite(currentOptedInAtMs)) {
|
|
1702
|
+
recordKeepAliveSkip(hqRoot, current, "missing_opt_in_time", summary, deps);
|
|
1703
|
+
continue;
|
|
1704
|
+
}
|
|
1705
|
+
const currentTimeBlockReason = initialState === "done"
|
|
1706
|
+
? "envelope_before_opt_in"
|
|
1707
|
+
: "interruption_before_opt_in";
|
|
1708
|
+
if (currentEligibilityAtMs <= currentOptedInAtMs) {
|
|
1709
|
+
recordKeepAliveSkip(hqRoot, current, currentTimeBlockReason, summary, deps);
|
|
1710
|
+
continue;
|
|
1711
|
+
}
|
|
1565
1712
|
if (retiredLaneIds === null) {
|
|
1566
1713
|
recordKeepAliveSkip(hqRoot, current, "retired_list_invalid", summary, deps);
|
|
1567
1714
|
continue;
|
|
@@ -1572,11 +1719,10 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
|
|
|
1572
1719
|
recordKeepAliveSkip(hqRoot, current, currentExclusion, summary, deps);
|
|
1573
1720
|
continue;
|
|
1574
1721
|
}
|
|
1575
|
-
const
|
|
1576
|
-
if (current.
|
|
1577
|
-
|
|
1578
|
-
|
|
1579
|
-
providerSessionId.trim().length === 0)
|
|
1722
|
+
const currentSessionId = current.provider_session_id ?? current.last_resume?.session_id;
|
|
1723
|
+
if (!providerSupportsSessionResume(current.provider) ||
|
|
1724
|
+
typeof currentSessionId !== "string" ||
|
|
1725
|
+
currentSessionId.trim().length === 0)
|
|
1580
1726
|
continue;
|
|
1581
1727
|
const resetByHuman = keepAliveProgressResetByHuman(current);
|
|
1582
1728
|
const attempts = resetByHuman ? 0 : current.keep_alive_resume_count ?? 0;
|
|
@@ -1584,8 +1730,9 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
|
|
|
1584
1730
|
if (current.keep_alive_exhausted_at !== undefined)
|
|
1585
1731
|
continue;
|
|
1586
1732
|
const exhausted = casLaneIf(hqRoot, current.lane_id, (latestLane) => latestLane.version === current.version &&
|
|
1587
|
-
latestLane.state ===
|
|
1588
|
-
|
|
1733
|
+
latestLane.state === current.state &&
|
|
1734
|
+
latestLane.keep_alive === true &&
|
|
1735
|
+
latestLane.keep_alive_opted_in_at === current.keep_alive_opted_in_at &&
|
|
1589
1736
|
(latestLane.keep_alive_resume_count ?? 0) >= LEAD_LANE_KEEP_ALIVE_MAX_RESUMES &&
|
|
1590
1737
|
latestLane.keep_alive_exhausted_at === undefined, (latestLane) => ({ ...latestLane, keep_alive_exhausted_at: nowIso }), deps.dryRun);
|
|
1591
1738
|
if (exhausted.changed && !deps.dryRun) {
|
|
@@ -1593,13 +1740,16 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
|
|
|
1593
1740
|
event: "keep_alive_exhausted",
|
|
1594
1741
|
at: nowIso,
|
|
1595
1742
|
attempt: attempts,
|
|
1743
|
+
resumed_from: current.state,
|
|
1596
1744
|
});
|
|
1597
1745
|
}
|
|
1598
1746
|
continue;
|
|
1599
1747
|
}
|
|
1600
1748
|
const attempt = attempts + 1;
|
|
1601
|
-
const
|
|
1602
|
-
|
|
1749
|
+
const terminalAtMs = initialState === "done"
|
|
1750
|
+
? Date.parse(current.updated_at)
|
|
1751
|
+
: Date.parse(current.interrupted_at ?? "");
|
|
1752
|
+
const attemptEligibleMs = terminalAtMs + leadLaneKeepAliveBackoffMs(attempt);
|
|
1603
1753
|
const storedNextMs = resetByHuman
|
|
1604
1754
|
? Number.NaN
|
|
1605
1755
|
: Date.parse(current.keep_alive_next_eligible_at ?? "");
|
|
@@ -1609,8 +1759,9 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
|
|
|
1609
1759
|
if (nowMs < nextEligibleMs) {
|
|
1610
1760
|
if (resetByHuman && hasKeepAliveProgress(current)) {
|
|
1611
1761
|
casLaneIf(hqRoot, current.lane_id, (latestLane) => latestLane.version === current.version &&
|
|
1612
|
-
latestLane.state ===
|
|
1613
|
-
|
|
1762
|
+
latestLane.state === current.state &&
|
|
1763
|
+
latestLane.keep_alive === true &&
|
|
1764
|
+
latestLane.keep_alive_opted_in_at === current.keep_alive_opted_in_at, (latestLane) => clearedKeepAliveProgress(latestLane), deps.dryRun);
|
|
1614
1765
|
}
|
|
1615
1766
|
continue;
|
|
1616
1767
|
}
|
|
@@ -1628,9 +1779,14 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
|
|
|
1628
1779
|
});
|
|
1629
1780
|
continue;
|
|
1630
1781
|
}
|
|
1782
|
+
const resumedFrom = current.state;
|
|
1783
|
+
const resumeText = resumedFrom === "interrupted"
|
|
1784
|
+
? LEAD_LANE_KEEP_ALIVE_INTERRUPTED_RESUME_TEXT
|
|
1785
|
+
: LEAD_LANE_KEEP_ALIVE_RESUME_TEXT;
|
|
1631
1786
|
const queue = (expectedSeniorId) => casLaneIf(hqRoot, current.lane_id, (latestLane) => latestLane.version === current.version &&
|
|
1632
|
-
latestLane.state ===
|
|
1633
|
-
|
|
1787
|
+
latestLane.state === current.state &&
|
|
1788
|
+
latestLane.keep_alive === true &&
|
|
1789
|
+
latestLane.keep_alive_opted_in_at === current.keep_alive_opted_in_at &&
|
|
1634
1790
|
(expectedSeniorId === undefined ||
|
|
1635
1791
|
(latestLane.senior.kind === "lane" && latestLane.senior.id === expectedSeniorId)), (latestLane) => ({
|
|
1636
1792
|
...latestLane,
|
|
@@ -1641,10 +1797,10 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
|
|
|
1641
1797
|
terminal_reason: undefined,
|
|
1642
1798
|
terminal_details: undefined,
|
|
1643
1799
|
pending_resume: {
|
|
1644
|
-
session_id:
|
|
1800
|
+
session_id: currentSessionId,
|
|
1645
1801
|
trigger: "resume",
|
|
1646
|
-
resumed_from:
|
|
1647
|
-
text:
|
|
1802
|
+
resumed_from: resumedFrom,
|
|
1803
|
+
text: resumeText,
|
|
1648
1804
|
compact: true,
|
|
1649
1805
|
at: nowIso,
|
|
1650
1806
|
},
|
|
@@ -1693,7 +1849,7 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
|
|
|
1693
1849
|
continue;
|
|
1694
1850
|
summary.resumed.push({
|
|
1695
1851
|
id: current.lane_id,
|
|
1696
|
-
from:
|
|
1852
|
+
from: resumedFrom,
|
|
1697
1853
|
to: "queued",
|
|
1698
1854
|
reason: "keep_alive",
|
|
1699
1855
|
});
|
|
@@ -1703,6 +1859,7 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
|
|
|
1703
1859
|
at: nowIso,
|
|
1704
1860
|
attempt,
|
|
1705
1861
|
next_eligible_at: nextEligibleAt,
|
|
1862
|
+
resumed_from: resumedFrom,
|
|
1706
1863
|
});
|
|
1707
1864
|
}
|
|
1708
1865
|
}
|
package/dist/lib/lanes/store.js
CHANGED
|
@@ -414,6 +414,8 @@ export function parseLaneRecord(raw, filePath) {
|
|
|
414
414
|
"not_opted_in",
|
|
415
415
|
"missing_opt_in_time",
|
|
416
416
|
"envelope_before_opt_in",
|
|
417
|
+
"interruption_before_opt_in",
|
|
418
|
+
"missing_interruption_time",
|
|
417
419
|
"retired",
|
|
418
420
|
"superseded",
|
|
419
421
|
"retired_list_invalid",
|
|
@@ -443,6 +445,7 @@ export function parseLaneRecord(raw, filePath) {
|
|
|
443
445
|
"keep_alive_last_resume_at",
|
|
444
446
|
"keep_alive_next_eligible_at",
|
|
445
447
|
"keep_alive_exhausted_at",
|
|
448
|
+
"interrupted_at",
|
|
446
449
|
]) {
|
|
447
450
|
if (raw[field] !== undefined)
|
|
448
451
|
requireTimestamp(raw[field], field, filePath);
|
|
@@ -359,7 +359,7 @@ export interface LaneRecord {
|
|
|
359
359
|
/** Time this lane most recently explicitly opted into keep-alive. */
|
|
360
360
|
keep_alive_opted_in_at?: string;
|
|
361
361
|
/** Why reconcile declined to resume this lane most recently. */
|
|
362
|
-
keep_alive_blocked_reason?: "not_opted_in" | "missing_opt_in_time" | "envelope_before_opt_in" | "retired" | "superseded" | "retired_list_invalid" | "terminal_state";
|
|
362
|
+
keep_alive_blocked_reason?: "not_opted_in" | "missing_opt_in_time" | "envelope_before_opt_in" | "interruption_before_opt_in" | "missing_interruption_time" | "retired" | "superseded" | "retired_list_invalid" | "terminal_state";
|
|
363
363
|
/** Consecutive automatic keep-alive resumes since the last human reset. */
|
|
364
364
|
keep_alive_resume_count?: number;
|
|
365
365
|
/** Time the most recent automatic keep-alive resume was queued. */
|
|
@@ -451,6 +451,8 @@ export interface LaneRecord {
|
|
|
451
451
|
/** Set when the reconcile pass force-admits under the starvation bound. */
|
|
452
452
|
admitted_reason?: string;
|
|
453
453
|
terminal_reason?: string;
|
|
454
|
+
/** Classification time when this lane entered interrupted; stable across later metadata writes. */
|
|
455
|
+
interrupted_at?: string;
|
|
454
456
|
/**
|
|
455
457
|
* Structured detail behind a named `terminal_reason` (dispatch, pickup and
|
|
456
458
|
* spawn failures), the missing-binary search path behind
|
package/dist/lib/lanes/types.js
CHANGED
|
@@ -230,7 +230,7 @@ export const LANE_FIELD_WRITERS = {
|
|
|
230
230
|
last_envelope: ["watcher", "reconcile"], model: ["creator", "spawn", "command"], model_source: ["creator", "spawn", "command"], effort: ["creator", "spawn", "command"], effort_source: ["creator", "spawn", "command"], fast_mode: ["spawn"],
|
|
231
231
|
last_message_at: ["command"], last_compaction: ["spawn"], story_complete: ["watcher", "reconcile"], story_complete_reason: ["watcher", "reconcile"], story_status: ["watcher", "reconcile"],
|
|
232
232
|
overdue: ["reconcile"], overdue_at: ["reconcile"], last_host_report_at: ["watcher", "spawn", "reconcile"], stalled: ["watcher"], stalled_at: ["watcher"], stalled_reason: ["watcher"],
|
|
233
|
-
admitted_reason: ["command", "reconcile"], terminal_reason: ["command", "watcher", "spawn", "reconcile"], terminal_details: ["command", "spawn", "watcher", "reconcile"],
|
|
233
|
+
admitted_reason: ["command", "reconcile"], terminal_reason: ["command", "watcher", "spawn", "reconcile"], interrupted_at: ["watcher", "reconcile"], terminal_details: ["command", "spawn", "watcher", "reconcile"],
|
|
234
234
|
quota_reset_at: ["watcher", "reconcile"], model_quota_marker: ["watcher", "reconcile"], outpost_command_id: ["command"],
|
|
235
235
|
provider_session_id: ["watcher", "command", "spawn"], provider_session_id_absent: ["watcher", "command"], provider_home_at_spawn: ["spawn"], last_resume: ["spawn"], last_run_usage: ["watcher"], envelope_repair: ["watcher", "reconcile"], envelope_validation_errors: ["watcher"], last_progress_event_at: ["watcher"],
|
|
236
236
|
};
|
|
@@ -336,6 +336,7 @@ export function applyTerminalToLaneWithResult(hqRoot, laneId, classification, op
|
|
|
336
336
|
stalled: undefined,
|
|
337
337
|
stalled_at: undefined,
|
|
338
338
|
stalled_reason: undefined,
|
|
339
|
+
interrupted_at: classification.state === "interrupted" ? at : undefined,
|
|
339
340
|
envelope_validation_errors: opts.envelopeValidationErrors,
|
|
340
341
|
};
|
|
341
342
|
changed = true;
|