@indigoai-us/hq-cli 5.204.0 → 5.206.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +25 -0
- package/dist/command-catalog.generated.d.ts +6 -0
- package/dist/command-catalog.generated.js +8 -0
- package/dist/commands/lanes.js +53 -25
- package/dist/lib/lanes/admission.js +6 -0
- package/dist/lib/lanes/hire-grants.d.ts +4 -0
- package/dist/lib/lanes/hire-grants.js +62 -0
- package/dist/lib/lanes/index.d.ts +1 -1
- package/dist/lib/lanes/index.js +1 -1
- package/dist/lib/lanes/keep-alive.d.ts +8 -0
- package/dist/lib/lanes/keep-alive.js +14 -0
- package/dist/lib/lanes/outpost-dispatch.d.ts +1 -0
- package/dist/lib/lanes/outpost-dispatch.js +2 -0
- package/dist/lib/lanes/profile.d.ts +1 -0
- package/dist/lib/lanes/profile.js +8 -0
- package/dist/lib/lanes/provider-rejection.d.ts +1 -1
- package/dist/lib/lanes/provider-rejection.js +15 -5
- package/dist/lib/lanes/reconcile.d.ts +3 -0
- package/dist/lib/lanes/reconcile.js +273 -4
- package/dist/lib/lanes/store.js +25 -0
- package/dist/lib/lanes/types.d.ts +10 -0
- package/dist/lib/lanes/types.js +1 -0
- package/dist/startup-registration.js +2 -0
- package/dist/utils/install-tree-torn.d.ts +22 -7
- package/dist/utils/install-tree-torn.js +108 -20
- package/dist/utils/sentry-fingerprint.js +9 -0
- package/package.json +1 -1
|
@@ -14,18 +14,22 @@ import { effectiveAccountHome } from "./account-home.js";
|
|
|
14
14
|
import { loadLanesConfig, providerQuotaDefaultHoldMs } from "./config.js";
|
|
15
15
|
import { MAX_LANE_DEPTH } from "./constants.js";
|
|
16
16
|
import { AccountResolutionError, LanesError, isLaneVersionConflict } from "./errors.js";
|
|
17
|
+
import { LANE_HIRE_GRANTS_FLAG, terminalChildResumeHireGrantError, withHireGrantAdmission, } from "./hire-grants.js";
|
|
17
18
|
import { loadWorkerProfile } from "./profile.js";
|
|
18
19
|
import { classifySessionLiveness, defaultLivenessDeps, loadLivenessIndex, readDaemonPresenceConnected, unindexedClaudeSessionLiveness, } from "./liveness.js";
|
|
19
20
|
import { ANSWERED_QUESTIONS_FILE, armLaneWatcher, envelopeRepairMode, isPidRecordAlive, laneRunDir, readPidRecord, readSpawnManifest, markSpawnFailure, spawnEnvelopeRepair, spawnLaneWorker, retryLaneOnAlternateAccount, } from "./spawn.js";
|
|
20
21
|
import { appendCappedHistory, lanesStoreRoot, parseLaneRecord, parseQuestionRecord, readLane, readQuestion, updateLane, updateQuestion, } from "./store.js";
|
|
21
22
|
import { applyThrottleDecision, decideAdmission, defaultThrottleDeps, emptyThrottleState, evaluateThrottle, isCpuCheckEnabled, readAvailableMemMb, readThrottleState, resolveCpuBusyFraction, throttleStatePath, writeThrottleState, } from "./throttle.js";
|
|
23
|
+
import { readLanesRegistryFlag } from "./registry-flag.js";
|
|
24
|
+
import { LEAD_LANE_KEEP_ALIVE_FLAG, LEAD_LANE_KEEP_ALIVE_MAX_RESUMES, LEAD_LANE_KEEP_ALIVE_RESUME_TEXT, leadLaneKeepAliveBackoffMs, } from "./keep-alive.js";
|
|
25
|
+
import { providerSupportsSessionResume } from "./delivery.js";
|
|
22
26
|
import { applyDefaultQuotaHold, readQuotaSignal, recordAccountQuota } from "./quota.js";
|
|
23
27
|
import { answeredQuestionEntries, raiseEnvelopeQuestions } from "./questions.js";
|
|
24
|
-
import { SILENCE_MS, classifyLaneExit, isPidAlive, missingExpectedEnvelopeFile, readLatestEnvelopeFile, readLatestEnvelopeRecord, } from "./watcher.js";
|
|
28
|
+
import { SILENCE_MS, classifyLaneExit, isPidAlive, missingExpectedEnvelopeFile, readLatestEnvelopeFile, readLatestEnvelopeRecord, watcherEventsPath, writeWatcherDecision, } from "./watcher.js";
|
|
25
29
|
import { createLaneMesh, emitLaneDecisionEvents, emitLaneTerminalStateEvent, laneMeshCompany, } from "./mesh-events.js";
|
|
26
30
|
import { laneStories, storyStatusForEnvelope } from "./stories.js";
|
|
27
31
|
import { decideEnvelopeRepair, envelopeQuestionIds, routeProviderFailure } from "./watcher.js";
|
|
28
|
-
import { providerRejectionDetails, readProviderRejection, } from "./provider-rejection.js";
|
|
32
|
+
import { PROVIDER_REJECTED_REASON, providerRejectionDetails, readProviderRejection, } from "./provider-rejection.js";
|
|
29
33
|
/** Spec §9: unanswered at the current hop for 60 minutes → one hop. */
|
|
30
34
|
export const QUESTION_AGE_MS = 60 * 60 * 1000;
|
|
31
35
|
/** Spec §6.6 + §9: a depth-3 worker has two hops to age toward the user. */
|
|
@@ -792,11 +796,51 @@ export async function runPassBody(hqRoot, deps) {
|
|
|
792
796
|
});
|
|
793
797
|
}
|
|
794
798
|
}
|
|
799
|
+
const readRegistryFlag = deps.readRegistryFlag ?? readLanesRegistryFlag;
|
|
800
|
+
const keepAliveFlagCompanies = new Map();
|
|
801
|
+
const keepAliveHireGrantFlagCompanies = new Map();
|
|
802
|
+
for (const lane of lanes) {
|
|
803
|
+
if (lane.state !== "done" ||
|
|
804
|
+
lane.keep_alive !== true ||
|
|
805
|
+
!laneRunsHere(lane, deps.host) ||
|
|
806
|
+
lane.terminal_reason === PROVIDER_REJECTED_REASON ||
|
|
807
|
+
lane.terminal_reason === "account_not_logged_in" ||
|
|
808
|
+
!providerSupportsSessionResume(lane.provider) ||
|
|
809
|
+
!(lane.provider_session_id ?? lane.last_resume?.session_id))
|
|
810
|
+
continue;
|
|
811
|
+
const envelope = readLatestEnvelopeRecord(hqRoot, lane.lane_id)?.envelope;
|
|
812
|
+
if (!envelope || envelope.decision !== "done" || envelope.story_complete === true)
|
|
813
|
+
continue;
|
|
814
|
+
const key = keepAliveCompanyKey(lane.company);
|
|
815
|
+
keepAliveFlagCompanies.set(key, lane.company);
|
|
816
|
+
if (lane.senior.kind === "lane" && lane.senior.id) {
|
|
817
|
+
keepAliveHireGrantFlagCompanies.set(key, lane.company);
|
|
818
|
+
}
|
|
819
|
+
}
|
|
820
|
+
const keepAliveFlagByCompany = new Map();
|
|
821
|
+
const keepAliveHireGrantFlagByCompany = new Map();
|
|
822
|
+
const env = deps.env ?? process.env;
|
|
823
|
+
await Promise.all([
|
|
824
|
+
...[...keepAliveFlagCompanies.entries()].map(async ([key, company]) => {
|
|
825
|
+
keepAliveFlagByCompany.set(key, await readRegistryFlag(LEAD_LANE_KEEP_ALIVE_FLAG, {
|
|
826
|
+
company: company ?? undefined,
|
|
827
|
+
env,
|
|
828
|
+
}));
|
|
829
|
+
}),
|
|
830
|
+
...[...keepAliveHireGrantFlagCompanies.entries()].map(async ([key, company]) => {
|
|
831
|
+
keepAliveHireGrantFlagByCompany.set(key, await readRegistryFlag(LANE_HIRE_GRANTS_FLAG, {
|
|
832
|
+
company: company ?? undefined,
|
|
833
|
+
env,
|
|
834
|
+
}));
|
|
835
|
+
}),
|
|
836
|
+
]);
|
|
795
837
|
await withAllLaneAdmissionLocks(hqRoot, async () => {
|
|
796
838
|
const freshLanes = loadLaneRecords(hqRoot, summary);
|
|
839
|
+
resumeDoneKeepAlive(hqRoot, freshLanes, summary, deps, now, keepAliveFlagByCompany, keepAliveHireGrantFlagByCompany);
|
|
840
|
+
const drainLanes = loadLaneRecords(hqRoot, summary);
|
|
797
841
|
const freshQuestions = loadQuestionRecords(hqRoot, summary);
|
|
798
|
-
await admitQueued(hqRoot,
|
|
799
|
-
await resumeAwaiting(hqRoot,
|
|
842
|
+
await admitQueued(hqRoot, drainLanes, summary, deps, now);
|
|
843
|
+
await resumeAwaiting(hqRoot, drainLanes, freshQuestions, summary, deps);
|
|
800
844
|
});
|
|
801
845
|
if (!deps.dryRun && summary.errors.length === 0) {
|
|
802
846
|
try {
|
|
@@ -1322,6 +1366,231 @@ async function admitQueued(hqRoot, lanes, summary, deps, now) {
|
|
|
1322
1366
|
writeThrottleState(filePath, applyThrottleDecision(base, applied, nowMs), throttleDeps);
|
|
1323
1367
|
}
|
|
1324
1368
|
}
|
|
1369
|
+
function keepAliveCompanyKey(company) {
|
|
1370
|
+
return JSON.stringify([company?.uid ?? null, company?.slug ?? null]);
|
|
1371
|
+
}
|
|
1372
|
+
function hasKeepAliveProgress(lane) {
|
|
1373
|
+
return (lane.keep_alive_resume_count ?? 0) !== 0 ||
|
|
1374
|
+
lane.keep_alive_last_resume_at !== undefined ||
|
|
1375
|
+
lane.keep_alive_next_eligible_at !== undefined ||
|
|
1376
|
+
lane.keep_alive_exhausted_at !== undefined;
|
|
1377
|
+
}
|
|
1378
|
+
function clearedKeepAliveProgress(lane) {
|
|
1379
|
+
return {
|
|
1380
|
+
...lane,
|
|
1381
|
+
keep_alive_resume_count: 0,
|
|
1382
|
+
keep_alive_last_resume_at: undefined,
|
|
1383
|
+
keep_alive_next_eligible_at: undefined,
|
|
1384
|
+
keep_alive_exhausted_at: undefined,
|
|
1385
|
+
};
|
|
1386
|
+
}
|
|
1387
|
+
function keepAliveProgressResetByHuman(lane) {
|
|
1388
|
+
const count = lane.keep_alive_resume_count ?? 0;
|
|
1389
|
+
const lastAutoMs = Date.parse(lane.keep_alive_last_resume_at ?? "");
|
|
1390
|
+
if (count > 0 && !Number.isFinite(lastAutoMs))
|
|
1391
|
+
return true;
|
|
1392
|
+
const messageMs = Date.parse(lane.last_message_at ?? "");
|
|
1393
|
+
const resumeMs = lane.last_resume?.text === LEAD_LANE_KEEP_ALIVE_RESUME_TEXT
|
|
1394
|
+
? Number.NaN
|
|
1395
|
+
: Date.parse(lane.last_resume?.at ?? "");
|
|
1396
|
+
return Number.isFinite(lastAutoMs) &&
|
|
1397
|
+
((Number.isFinite(messageMs) && messageMs > lastAutoMs) ||
|
|
1398
|
+
(Number.isFinite(resumeMs) && resumeMs > lastAutoMs));
|
|
1399
|
+
}
|
|
1400
|
+
function recordKeepAliveEvent(hqRoot, laneId, summary, event) {
|
|
1401
|
+
try {
|
|
1402
|
+
writeWatcherDecision(watcherEventsPath(hqRoot, laneId), {
|
|
1403
|
+
lane_id: laneId,
|
|
1404
|
+
...event,
|
|
1405
|
+
});
|
|
1406
|
+
}
|
|
1407
|
+
catch (err) {
|
|
1408
|
+
markDirty(summary, {
|
|
1409
|
+
kind: "lane",
|
|
1410
|
+
id: laneId,
|
|
1411
|
+
why: `keep-alive event write failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
1412
|
+
});
|
|
1413
|
+
}
|
|
1414
|
+
}
|
|
1415
|
+
function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, hireGrantFlagByCompany) {
|
|
1416
|
+
const nowMs = now.getTime();
|
|
1417
|
+
const nowIso = now.toISOString();
|
|
1418
|
+
const config = loadLanesConfig(hqRoot);
|
|
1419
|
+
for (const lane of lanes) {
|
|
1420
|
+
if (lane.state !== "done" ||
|
|
1421
|
+
lane.keep_alive !== true ||
|
|
1422
|
+
!laneRunsHere(lane, deps.host))
|
|
1423
|
+
continue;
|
|
1424
|
+
const latest = readLatestEnvelopeRecord(hqRoot, lane.lane_id);
|
|
1425
|
+
const envelope = latest?.envelope;
|
|
1426
|
+
if (!envelope || envelope.decision !== "done")
|
|
1427
|
+
continue;
|
|
1428
|
+
if (envelope.story_complete === true) {
|
|
1429
|
+
if (hasKeepAliveProgress(lane)) {
|
|
1430
|
+
casLaneIf(hqRoot, lane.lane_id, (current) => current.version === lane.version &&
|
|
1431
|
+
current.state === "done" &&
|
|
1432
|
+
current.keep_alive === true, (current) => clearedKeepAliveProgress(current), deps.dryRun);
|
|
1433
|
+
}
|
|
1434
|
+
continue;
|
|
1435
|
+
}
|
|
1436
|
+
if (lane.terminal_reason === PROVIDER_REJECTED_REASON ||
|
|
1437
|
+
lane.terminal_reason === "account_not_logged_in" ||
|
|
1438
|
+
!providerSupportsSessionResume(lane.provider) ||
|
|
1439
|
+
!(lane.provider_session_id ?? lane.last_resume?.session_id))
|
|
1440
|
+
continue;
|
|
1441
|
+
if (flagByCompany.get(keepAliveCompanyKey(lane.company)) !== true)
|
|
1442
|
+
continue;
|
|
1443
|
+
const current = readLane(hqRoot, lane.lane_id);
|
|
1444
|
+
if (current.version !== lane.version ||
|
|
1445
|
+
current.state !== "done" ||
|
|
1446
|
+
current.keep_alive !== true ||
|
|
1447
|
+
!laneRunsHere(current, deps.host))
|
|
1448
|
+
continue;
|
|
1449
|
+
const currentEnvelope = readLatestEnvelopeRecord(hqRoot, current.lane_id)?.envelope;
|
|
1450
|
+
if (!currentEnvelope || currentEnvelope.decision !== "done")
|
|
1451
|
+
continue;
|
|
1452
|
+
if (currentEnvelope.story_complete === true) {
|
|
1453
|
+
if (hasKeepAliveProgress(current)) {
|
|
1454
|
+
casLaneIf(hqRoot, current.lane_id, (latestLane) => latestLane.version === current.version &&
|
|
1455
|
+
latestLane.state === "done" &&
|
|
1456
|
+
latestLane.keep_alive === true, (latestLane) => clearedKeepAliveProgress(latestLane), deps.dryRun);
|
|
1457
|
+
}
|
|
1458
|
+
continue;
|
|
1459
|
+
}
|
|
1460
|
+
const providerSessionId = current.provider_session_id ?? current.last_resume?.session_id;
|
|
1461
|
+
if (current.terminal_reason === PROVIDER_REJECTED_REASON ||
|
|
1462
|
+
current.terminal_reason === "account_not_logged_in" ||
|
|
1463
|
+
typeof providerSessionId !== "string" ||
|
|
1464
|
+
providerSessionId.trim().length === 0)
|
|
1465
|
+
continue;
|
|
1466
|
+
const resetByHuman = keepAliveProgressResetByHuman(current);
|
|
1467
|
+
const attempts = resetByHuman ? 0 : current.keep_alive_resume_count ?? 0;
|
|
1468
|
+
if (attempts >= LEAD_LANE_KEEP_ALIVE_MAX_RESUMES) {
|
|
1469
|
+
if (current.keep_alive_exhausted_at !== undefined)
|
|
1470
|
+
continue;
|
|
1471
|
+
const exhausted = casLaneIf(hqRoot, current.lane_id, (latestLane) => latestLane.version === current.version &&
|
|
1472
|
+
latestLane.state === "done" &&
|
|
1473
|
+
latestLane.keep_alive === true &&
|
|
1474
|
+
(latestLane.keep_alive_resume_count ?? 0) >= LEAD_LANE_KEEP_ALIVE_MAX_RESUMES &&
|
|
1475
|
+
latestLane.keep_alive_exhausted_at === undefined, (latestLane) => ({ ...latestLane, keep_alive_exhausted_at: nowIso }), deps.dryRun);
|
|
1476
|
+
if (exhausted.changed && !deps.dryRun) {
|
|
1477
|
+
recordKeepAliveEvent(hqRoot, current.lane_id, summary, {
|
|
1478
|
+
event: "keep_alive_exhausted",
|
|
1479
|
+
at: nowIso,
|
|
1480
|
+
attempt: attempts,
|
|
1481
|
+
});
|
|
1482
|
+
}
|
|
1483
|
+
continue;
|
|
1484
|
+
}
|
|
1485
|
+
const attempt = attempts + 1;
|
|
1486
|
+
const doneAtMs = Date.parse(current.updated_at);
|
|
1487
|
+
const attemptEligibleMs = doneAtMs + leadLaneKeepAliveBackoffMs(attempt);
|
|
1488
|
+
const storedNextMs = resetByHuman
|
|
1489
|
+
? Number.NaN
|
|
1490
|
+
: Date.parse(current.keep_alive_next_eligible_at ?? "");
|
|
1491
|
+
const nextEligibleMs = Number.isFinite(storedNextMs)
|
|
1492
|
+
? Math.max(attemptEligibleMs, storedNextMs)
|
|
1493
|
+
: attemptEligibleMs;
|
|
1494
|
+
if (nowMs < nextEligibleMs) {
|
|
1495
|
+
if (resetByHuman && hasKeepAliveProgress(current)) {
|
|
1496
|
+
casLaneIf(hqRoot, current.lane_id, (latestLane) => latestLane.version === current.version &&
|
|
1497
|
+
latestLane.state === "done" &&
|
|
1498
|
+
latestLane.keep_alive === true, (latestLane) => clearedKeepAliveProgress(latestLane), deps.dryRun);
|
|
1499
|
+
}
|
|
1500
|
+
continue;
|
|
1501
|
+
}
|
|
1502
|
+
const nextEligibleAt = new Date(nowMs + leadLaneKeepAliveBackoffMs(attempt + 1)).toISOString();
|
|
1503
|
+
const queuedCount = loadLaneRecords(hqRoot, summary).filter((queuedLane) => queuedLane.state === "queued" &&
|
|
1504
|
+
queuedLane.provider === current.provider &&
|
|
1505
|
+
laneRunsHere(queuedLane, deps.host) &&
|
|
1506
|
+
!hasFreshAdmissionReservation(queuedLane, nowMs)).length;
|
|
1507
|
+
const maxQueued = config.providers[current.provider].max_queued;
|
|
1508
|
+
if (queuedCount >= maxQueued) {
|
|
1509
|
+
summary.skipped.push({
|
|
1510
|
+
kind: "lane",
|
|
1511
|
+
id: current.lane_id,
|
|
1512
|
+
why: `lane_queue_full: ${current.provider} queue has ${queuedCount}/${maxQueued} lanes`,
|
|
1513
|
+
});
|
|
1514
|
+
continue;
|
|
1515
|
+
}
|
|
1516
|
+
const queue = (expectedSeniorId) => casLaneIf(hqRoot, current.lane_id, (latestLane) => latestLane.version === current.version &&
|
|
1517
|
+
latestLane.state === "done" &&
|
|
1518
|
+
latestLane.keep_alive === true &&
|
|
1519
|
+
(expectedSeniorId === undefined ||
|
|
1520
|
+
(latestLane.senior.kind === "lane" && latestLane.senior.id === expectedSeniorId)), (latestLane) => ({
|
|
1521
|
+
...latestLane,
|
|
1522
|
+
state: "queued",
|
|
1523
|
+
queued_reason: undefined,
|
|
1524
|
+
queued_at: nowIso,
|
|
1525
|
+
admission_reserved_at: undefined,
|
|
1526
|
+
terminal_reason: undefined,
|
|
1527
|
+
terminal_details: undefined,
|
|
1528
|
+
pending_resume: {
|
|
1529
|
+
session_id: providerSessionId,
|
|
1530
|
+
trigger: "resume",
|
|
1531
|
+
resumed_from: "done",
|
|
1532
|
+
text: LEAD_LANE_KEEP_ALIVE_RESUME_TEXT,
|
|
1533
|
+
compact: true,
|
|
1534
|
+
at: nowIso,
|
|
1535
|
+
},
|
|
1536
|
+
keep_alive_resume_count: attempt,
|
|
1537
|
+
keep_alive_last_resume_at: nowIso,
|
|
1538
|
+
keep_alive_next_eligible_at: nextEligibleAt,
|
|
1539
|
+
keep_alive_exhausted_at: undefined,
|
|
1540
|
+
}), deps.dryRun);
|
|
1541
|
+
const seniorId = current.senior.kind === "lane" ? current.senior.id : undefined;
|
|
1542
|
+
let queued;
|
|
1543
|
+
if (seniorId &&
|
|
1544
|
+
hireGrantFlagByCompany.get(keepAliveCompanyKey(current.company)) === true) {
|
|
1545
|
+
try {
|
|
1546
|
+
queued = withHireGrantAdmission(hqRoot, seniorId, (snapshot) => {
|
|
1547
|
+
const latestLane = readLane(hqRoot, current.lane_id);
|
|
1548
|
+
if (latestLane.version !== current.version ||
|
|
1549
|
+
latestLane.senior.kind !== "lane" ||
|
|
1550
|
+
latestLane.senior.id !== seniorId)
|
|
1551
|
+
return { changed: false, record: latestLane };
|
|
1552
|
+
const refusal = terminalChildResumeHireGrantError(latestLane, snapshot);
|
|
1553
|
+
if (refusal !== null) {
|
|
1554
|
+
summary.skipped.push({
|
|
1555
|
+
kind: "lane",
|
|
1556
|
+
id: current.lane_id,
|
|
1557
|
+
why: `keep_alive_hire_admission: ${refusal.error}: ${refusal.message}`,
|
|
1558
|
+
});
|
|
1559
|
+
return { changed: false, record: latestLane };
|
|
1560
|
+
}
|
|
1561
|
+
return queue(seniorId);
|
|
1562
|
+
});
|
|
1563
|
+
}
|
|
1564
|
+
catch (err) {
|
|
1565
|
+
markDirty(summary, {
|
|
1566
|
+
kind: "lane",
|
|
1567
|
+
id: current.lane_id,
|
|
1568
|
+
why: `keep-alive hire admission failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
1569
|
+
});
|
|
1570
|
+
continue;
|
|
1571
|
+
}
|
|
1572
|
+
}
|
|
1573
|
+
else {
|
|
1574
|
+
queued = queue(seniorId);
|
|
1575
|
+
}
|
|
1576
|
+
if (!queued.changed)
|
|
1577
|
+
continue;
|
|
1578
|
+
summary.resumed.push({
|
|
1579
|
+
id: current.lane_id,
|
|
1580
|
+
from: "done",
|
|
1581
|
+
to: "queued",
|
|
1582
|
+
reason: "keep_alive",
|
|
1583
|
+
});
|
|
1584
|
+
if (!deps.dryRun) {
|
|
1585
|
+
recordKeepAliveEvent(hqRoot, current.lane_id, summary, {
|
|
1586
|
+
event: "keep_alive_resume",
|
|
1587
|
+
at: nowIso,
|
|
1588
|
+
attempt,
|
|
1589
|
+
next_eligible_at: nextEligibleAt,
|
|
1590
|
+
});
|
|
1591
|
+
}
|
|
1592
|
+
}
|
|
1593
|
+
}
|
|
1325
1594
|
async function resumeAwaiting(hqRoot, lanes, questions, summary, deps, onlyLaneIds) {
|
|
1326
1595
|
const config = loadLanesConfig(hqRoot);
|
|
1327
1596
|
const awaiting = lanes.filter((lane) => lane.state === "awaiting_input" &&
|
package/dist/lib/lanes/store.js
CHANGED
|
@@ -402,6 +402,31 @@ export function parseLaneRecord(raw, filePath) {
|
|
|
402
402
|
value: raw.hire_grant,
|
|
403
403
|
});
|
|
404
404
|
}
|
|
405
|
+
if (raw.keep_alive !== undefined && typeof raw.keep_alive !== "boolean") {
|
|
406
|
+
throw new LaneSchemaError("keep_alive must be a boolean", {
|
|
407
|
+
path: filePath,
|
|
408
|
+
field: "keep_alive",
|
|
409
|
+
value: raw.keep_alive,
|
|
410
|
+
});
|
|
411
|
+
}
|
|
412
|
+
if (raw.keep_alive_resume_count !== undefined &&
|
|
413
|
+
(typeof raw.keep_alive_resume_count !== "number" ||
|
|
414
|
+
!Number.isSafeInteger(raw.keep_alive_resume_count) ||
|
|
415
|
+
raw.keep_alive_resume_count < 0 || raw.keep_alive_resume_count > 24)) {
|
|
416
|
+
throw new LaneSchemaError("keep_alive_resume_count must be an integer from 0 through 24", {
|
|
417
|
+
path: filePath,
|
|
418
|
+
field: "keep_alive_resume_count",
|
|
419
|
+
value: raw.keep_alive_resume_count,
|
|
420
|
+
});
|
|
421
|
+
}
|
|
422
|
+
for (const field of [
|
|
423
|
+
"keep_alive_last_resume_at",
|
|
424
|
+
"keep_alive_next_eligible_at",
|
|
425
|
+
"keep_alive_exhausted_at",
|
|
426
|
+
]) {
|
|
427
|
+
if (raw[field] !== undefined)
|
|
428
|
+
requireTimestamp(raw[field], field, filePath);
|
|
429
|
+
}
|
|
405
430
|
requireString(raw.project_id, "project_id");
|
|
406
431
|
if (typeof raw.story_id !== "string" ||
|
|
407
432
|
(raw.story_id.trim() === "" && raw.no_story_reason === undefined)) {
|
|
@@ -354,6 +354,16 @@ export interface LaneRecord {
|
|
|
354
354
|
worker_pack?: string;
|
|
355
355
|
/** Personal worker profile id applied to every spawn. */
|
|
356
356
|
worker_profile?: string;
|
|
357
|
+
/** Whether this lead lane should resume completed turns while its story is open. */
|
|
358
|
+
keep_alive?: boolean;
|
|
359
|
+
/** Consecutive automatic keep-alive resumes since the last human reset. */
|
|
360
|
+
keep_alive_resume_count?: number;
|
|
361
|
+
/** Time the most recent automatic keep-alive resume was queued. */
|
|
362
|
+
keep_alive_last_resume_at?: string;
|
|
363
|
+
/** Earliest time the next consecutive automatic resume may be queued. */
|
|
364
|
+
keep_alive_next_eligible_at?: string;
|
|
365
|
+
/** Time the automatic resume cap was reached for this run of the story. */
|
|
366
|
+
keep_alive_exhausted_at?: string;
|
|
357
367
|
/** Maximum number of direct child lanes this lane may keep alive; old records default to 0. */
|
|
358
368
|
hire_grant?: number;
|
|
359
369
|
/** Content hash of the profile last applied to this lane. */
|
package/dist/lib/lanes/types.js
CHANGED
|
@@ -224,6 +224,7 @@ export const LANE_FIELD_WRITERS = {
|
|
|
224
224
|
pickup_validation_pending: ["creator", "command", "spawn", "reconcile"],
|
|
225
225
|
pickup_validation_pending_since: ["creator", "command", "spawn", "reconcile"],
|
|
226
226
|
priority: ["command"], worker: ["creator", "command"], worker_path: ["creator", "command"], worker_pack: ["creator", "command"], worker_profile: ["creator", "command"], worker_profile_hash: ["creator", "command", "spawn"], worker_profile_ignored: ["creator", "command", "spawn"], worker_policy_delivery: ["command", "spawn"],
|
|
227
|
+
keep_alive: ["creator", "command"], keep_alive_resume_count: ["command", "reconcile"], keep_alive_last_resume_at: ["command", "reconcile"], keep_alive_next_eligible_at: ["command", "reconcile"], keep_alive_exhausted_at: ["command", "reconcile"],
|
|
227
228
|
provider: ["creator"], account: ["creator", "spawn", "command", "watcher", "reconcile"], account_selection: ["creator", "command"], admission_accounts: ["creator", "command"], account_history: ["command", "spawn", "watcher", "reconcile"], pending_resume: ["command", "spawn", "watcher", "reconcile"], placement_default: ["creator"], host: ["creator", "command"], origin_host: ["creator"], origin_host_source: ["creator"],
|
|
228
229
|
version: ["creator"], created_at: ["creator"], updated_at: ["creator"], attribution_history: ["command"],
|
|
229
230
|
last_envelope: ["watcher", "reconcile"], model: ["creator", "spawn", "command"], model_source: ["creator", "spawn", "command"], effort: ["creator", "spawn", "command"], effort_source: ["creator", "spawn", "command"], fast_mode: ["spawn"],
|
|
@@ -158,6 +158,7 @@ export async function registerCommandsWithRecovery(args) {
|
|
|
158
158
|
waitedMs: result.waitedMs,
|
|
159
159
|
sawLock: result.sawLock,
|
|
160
160
|
sawRetired: result.sawRetired,
|
|
161
|
+
retiredSiblingPresent: result.retiredSiblingPresent,
|
|
161
162
|
sawTreeChange: result.sawTreeChange,
|
|
162
163
|
lockHeldAtEnd: result.lockHeldAtEnd,
|
|
163
164
|
lockedMs: result.lockedMs,
|
|
@@ -188,6 +189,7 @@ export async function registerCommandsWithRecovery(args) {
|
|
|
188
189
|
waitedMs: result.waitedMs,
|
|
189
190
|
sawLock: result.sawLock,
|
|
190
191
|
sawRetired: result.sawRetired,
|
|
192
|
+
retiredSiblingPresent: result.retiredSiblingPresent,
|
|
191
193
|
sawTreeChange: result.sawTreeChange,
|
|
192
194
|
lockTool: result.lockTool,
|
|
193
195
|
});
|
|
@@ -39,8 +39,9 @@
|
|
|
39
39
|
*/
|
|
40
40
|
export type ModuleNotFoundCode = "ERR_MODULE_NOT_FOUND" | "MODULE_NOT_FOUND" | "ENOENT" | "EVAL_THROW";
|
|
41
41
|
/**
|
|
42
|
-
* The closed set of resolution-failure dialects, verified on Node v22.23.1
|
|
43
|
-
*
|
|
42
|
+
* The closed set of resolution-failure dialects, verified on Node v22.23.1 and
|
|
43
|
+
* the observed Node 24 synchronous ESM loader shape (the @sentry/node
|
|
44
|
+
* import-in-the-middle hook does not change the stack transfer):
|
|
44
45
|
* - `esm-path`: ESM importer, an absolute file (or legacyMainResolve dir)
|
|
45
46
|
* target — `err.url` set, or `Cannot find package '<abs>'…`.
|
|
46
47
|
* - `esm-package`: ESM importer, a bare specifier — `Cannot find package
|
|
@@ -62,7 +63,7 @@ export type ModuleNotFoundCode = "ERR_MODULE_NOT_FOUND" | "MODULE_NOT_FOUND" | "
|
|
|
62
63
|
* held in Module._pathCache) and gone at READ, so the CJS
|
|
63
64
|
* loader's source read (defaultLoadImpl/loadSource, or the older
|
|
64
65
|
* Module._extensions..js handler) raises ENOENT with syscall
|
|
65
|
-
* 'open' and a cjs/loader frame but NO
|
|
66
|
+
* 'open' and a cjs/loader frame but NO ESM source-read frame. `err.path`
|
|
66
67
|
* is the vanished file.
|
|
67
68
|
* - `resolve-enoent`: ESM/CJS RESOLVE-time ENOENT (HQ-CLI-1S) — a path component
|
|
68
69
|
* stat'ed OK, then vanished before realpathSync's lstat, so the
|
|
@@ -179,6 +180,13 @@ export interface InstallTreeFs {
|
|
|
179
180
|
existsSync(p: string): boolean;
|
|
180
181
|
statSync(p: string): {
|
|
181
182
|
isDirectory(): boolean;
|
|
183
|
+
isSymbolicLink?(): boolean;
|
|
184
|
+
size?: number;
|
|
185
|
+
mtimeMs?: number;
|
|
186
|
+
};
|
|
187
|
+
lstatSync?(p: string): {
|
|
188
|
+
isDirectory(): boolean;
|
|
189
|
+
isSymbolicLink?(): boolean;
|
|
182
190
|
size?: number;
|
|
183
191
|
mtimeMs?: number;
|
|
184
192
|
};
|
|
@@ -264,6 +272,8 @@ export interface WaitForInstallTreeSettledResult {
|
|
|
264
272
|
sawTreeChange?: boolean;
|
|
265
273
|
/** True when the lock-free tree stayed unchanged for `staleProbeMs`. */
|
|
266
274
|
staleTree?: boolean;
|
|
275
|
+
/** Presence on the final poll; null means the sibling directory was unreadable. */
|
|
276
|
+
retiredSiblingPresent?: boolean | null;
|
|
267
277
|
}
|
|
268
278
|
export declare const DEFAULT_INSTALL_TREE_STALE_PROBE_MS = 1500;
|
|
269
279
|
/**
|
|
@@ -278,10 +288,11 @@ export declare const DEFAULT_INSTALL_TREE_STALE_PROBE_MS = 1500;
|
|
|
278
288
|
* app, the box's update timer, another `hq`) cannot make a legitimately-in-progress
|
|
279
289
|
* reinstall look stale:
|
|
280
290
|
* - UNLOCKED time — wall time observed with no fresh foreign lock — is bounded
|
|
281
|
-
* by `deadlineMs`. If the target is still missing
|
|
282
|
-
*
|
|
283
|
-
* wait returns `staleTree` early. Any observed tree
|
|
284
|
-
* settle deadline in force. `deadlineMs === 0`
|
|
291
|
+
* by `deadlineMs`. If the target is still missing and the observed tree,
|
|
292
|
+
* including retired sibling names and mtimes, stays unchanged for
|
|
293
|
+
* `staleProbeMs`, the wait returns `staleTree` early. Any observed tree
|
|
294
|
+
* change keeps the ordinary settle deadline in force. `deadlineMs === 0`
|
|
295
|
+
* evaluates readiness once.
|
|
285
296
|
* - LOCKED time — wall time observed under a fresh foreign lock held by a live
|
|
286
297
|
* pid — is bounded SEPARATELY by `lockWaitMs` (default the lock contract's own
|
|
287
298
|
* 10-minute staleness window) and does NOT consume the settle budget. When
|
|
@@ -342,6 +353,8 @@ export type InstallTreeTornDiagnostics = {
|
|
|
342
353
|
/** The bounded `tool` of the last fresh foreign lock seen, or "". */
|
|
343
354
|
lockTool: string;
|
|
344
355
|
node: string;
|
|
356
|
+
/** Presence on the final poll when the scope directory could be observed. */
|
|
357
|
+
retiredSiblingPresent?: boolean;
|
|
345
358
|
/** The thrown error's `name` for the eval-throw dialect (HQ-CLI-1T), else "". */
|
|
346
359
|
errorName: string;
|
|
347
360
|
};
|
|
@@ -363,6 +376,8 @@ export interface InstallTreeTornInit {
|
|
|
363
376
|
lockedMs?: number;
|
|
364
377
|
/** The last fresh foreign lock's `tool`, bounded on construction (default ""). */
|
|
365
378
|
lockTool?: string;
|
|
379
|
+
/** Final retired sibling presence, or null when the scope directory was unreadable. */
|
|
380
|
+
retiredSiblingPresent?: boolean | null;
|
|
366
381
|
}
|
|
367
382
|
/**
|
|
368
383
|
* A torn-install failure that stays VISIBLE in Sentry with bounded, hq-derived
|