@indigoai-us/hq-cli 5.203.0 → 5.205.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/dist/command-catalog.generated.d.ts +33 -0
- package/dist/command-catalog.generated.js +43 -0
- package/dist/commands/lanes.d.ts +3 -0
- package/dist/commands/lanes.js +253 -50
- package/dist/lib/lanes/account-home.d.ts +8 -0
- package/dist/lib/lanes/account-home.js +10 -0
- package/dist/lib/lanes/accounts.d.ts +4 -1
- package/dist/lib/lanes/accounts.js +15 -10
- package/dist/lib/lanes/admission.js +6 -0
- package/dist/lib/lanes/claude-account-balance.d.ts +3 -0
- package/dist/lib/lanes/claude-account-balance.js +19 -4
- package/dist/lib/lanes/config.d.ts +2 -1
- package/dist/lib/lanes/config.js +55 -13
- package/dist/lib/lanes/hire-grants.d.ts +4 -0
- package/dist/lib/lanes/hire-grants.js +62 -0
- package/dist/lib/lanes/host-capacity.d.ts +4 -0
- package/dist/lib/lanes/host-capacity.js +8 -1
- package/dist/lib/lanes/index.d.ts +3 -1
- package/dist/lib/lanes/index.js +2 -1
- package/dist/lib/lanes/keep-alive.d.ts +8 -0
- package/dist/lib/lanes/keep-alive.js +14 -0
- package/dist/lib/lanes/liveness.d.ts +1 -0
- package/dist/lib/lanes/liveness.js +3 -1
- package/dist/lib/lanes/outpost-dispatch.d.ts +1 -0
- package/dist/lib/lanes/outpost-dispatch.js +2 -0
- package/dist/lib/lanes/profile.d.ts +1 -0
- package/dist/lib/lanes/profile.js +8 -0
- package/dist/lib/lanes/provider-rejection.d.ts +1 -1
- package/dist/lib/lanes/provider-rejection.js +15 -5
- package/dist/lib/lanes/reconcile.d.ts +3 -0
- package/dist/lib/lanes/reconcile.js +293 -13
- package/dist/lib/lanes/spawn.d.ts +1 -0
- package/dist/lib/lanes/spawn.js +30 -14
- package/dist/lib/lanes/store.js +25 -0
- package/dist/lib/lanes/types.d.ts +15 -0
- package/dist/lib/lanes/types.js +1 -0
- package/package.json +1 -1
|
@@ -28,6 +28,7 @@ const PROFILE_META_KEYS = new Set([
|
|
|
28
28
|
"max_concurrent",
|
|
29
29
|
"source",
|
|
30
30
|
"accounts",
|
|
31
|
+
"keep_alive",
|
|
31
32
|
]);
|
|
32
33
|
function isPlainObject(value) {
|
|
33
34
|
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
@@ -164,6 +165,12 @@ function parseMeta(id, dir, raw) {
|
|
|
164
165
|
value: raw.fast_mode,
|
|
165
166
|
});
|
|
166
167
|
}
|
|
168
|
+
if (raw.keep_alive !== undefined && typeof raw.keep_alive !== "boolean") {
|
|
169
|
+
throw profileError(id, dir, "meta.yaml keep_alive must be a boolean when present", {
|
|
170
|
+
field: "keep_alive",
|
|
171
|
+
value: raw.keep_alive,
|
|
172
|
+
});
|
|
173
|
+
}
|
|
167
174
|
let version;
|
|
168
175
|
if (raw.version !== undefined) {
|
|
169
176
|
if (typeof raw.version !== "string" || raw.version.trim().length === 0) {
|
|
@@ -211,6 +218,7 @@ function parseMeta(id, dir, raw) {
|
|
|
211
218
|
...(account !== undefined ? { account } : {}),
|
|
212
219
|
...(effort !== undefined ? { effort } : {}),
|
|
213
220
|
...(raw.fast_mode !== undefined ? { fast_mode: raw.fast_mode } : {}),
|
|
221
|
+
...(raw.keep_alive !== undefined ? { keep_alive: raw.keep_alive } : {}),
|
|
214
222
|
...(maxConcurrent !== undefined ? { max_concurrent: maxConcurrent } : {}),
|
|
215
223
|
...(source !== undefined ? { source } : {}),
|
|
216
224
|
...(accounts !== undefined ? { accounts } : {}),
|
|
@@ -35,7 +35,7 @@ export interface ProviderRejection {
|
|
|
35
35
|
/** Derived from the status: a transient class (408, 425, 5xx) is retryable; a 4xx is not. */
|
|
36
36
|
retryable: boolean;
|
|
37
37
|
/** Which event on the stream carried the failure. */
|
|
38
|
-
event: "codex.turn.failed" | "claude.result.is_error" | "grok.error";
|
|
38
|
+
event: "codex.turn.failed" | "claude.result.is_error" | "claude.result.login_failure" | "grok.error";
|
|
39
39
|
}
|
|
40
40
|
export declare function isProviderLoginFailure(failure: ProviderRejection | null): boolean;
|
|
41
41
|
export declare function capMessage(message: string): string;
|
|
@@ -28,7 +28,7 @@ export const QUOTA_STATUS = 429;
|
|
|
28
28
|
function isProviderLoginMessage(provider, message) {
|
|
29
29
|
switch (provider) {
|
|
30
30
|
case "claude":
|
|
31
|
-
return
|
|
31
|
+
return /(?:\bnot logged in\b|\bplease run\s+\/login\b)/i.test(message);
|
|
32
32
|
case "codex":
|
|
33
33
|
return /\bnot logged in\b/i.test(message) ||
|
|
34
34
|
/\bmissing bearer or basic authentication in header\b/i.test(message);
|
|
@@ -40,6 +40,11 @@ function isProviderLoginMessage(provider, message) {
|
|
|
40
40
|
}
|
|
41
41
|
}
|
|
42
42
|
}
|
|
43
|
+
/** Successful Claude results can contain worker prose, so only accept its short CLI login banner. */
|
|
44
|
+
function isClaudeSuccessLoginMessage(message) {
|
|
45
|
+
const trimmed = message.trim();
|
|
46
|
+
return trimmed.length <= 80 && /^not logged in\b.{0,40}please run \/login\.?$/i.test(trimmed);
|
|
47
|
+
}
|
|
43
48
|
export function isProviderLoginFailure(failure) {
|
|
44
49
|
return failure !== null && isProviderLoginMessage(failure.provider, failure.message);
|
|
45
50
|
}
|
|
@@ -163,11 +168,14 @@ export function parseClaudeResultRejection(text) {
|
|
|
163
168
|
const found = scanJsonLinesBackward(text, (rec) => {
|
|
164
169
|
if (rec.type !== "result")
|
|
165
170
|
return null;
|
|
166
|
-
|
|
171
|
+
const message = typeof rec.result === "string" ? rec.result : "";
|
|
172
|
+
const loginFailure = rec.is_error === true
|
|
173
|
+
? isProviderLoginMessage("claude", message)
|
|
174
|
+
: isClaudeSuccessLoginMessage(message);
|
|
175
|
+
if (rec.is_error !== true && !loginFailure)
|
|
167
176
|
return NOT_REJECTED;
|
|
168
177
|
const status = statusOf(rec.api_error_status);
|
|
169
|
-
|
|
170
|
-
if (status === null && !isProviderLoginMessage("claude", message))
|
|
178
|
+
if (status === null && !loginFailure)
|
|
171
179
|
return NOT_REJECTED;
|
|
172
180
|
return {
|
|
173
181
|
reason: PROVIDER_REJECTED_REASON,
|
|
@@ -176,7 +184,9 @@ export function parseClaudeResultRejection(text) {
|
|
|
176
184
|
status,
|
|
177
185
|
error_type: typeof rec.subtype === "string" ? rec.subtype : null,
|
|
178
186
|
retryable: retryableFromStatus(status),
|
|
179
|
-
event:
|
|
187
|
+
event: loginFailure && rec.is_error !== true
|
|
188
|
+
? "claude.result.login_failure"
|
|
189
|
+
: "claude.result.is_error",
|
|
180
190
|
};
|
|
181
191
|
});
|
|
182
192
|
return found === NOT_REJECTED ? null : found;
|
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
import { type LivenessDeps } from "./liveness.js";
|
|
9
9
|
import { type EnvelopeRepairSpawnInput, type SpawnLaneDeps } from "./spawn.js";
|
|
10
10
|
import { type ThrottleDeps } from "./throttle.js";
|
|
11
|
+
import { readLanesRegistryFlag } from "./registry-flag.js";
|
|
11
12
|
import type { LaneRecord, LivenessState, QueuedReason, QuestionRecord, SeniorRef } from "./types.js";
|
|
12
13
|
import { type LaneMesh } from "./mesh-events.js";
|
|
13
14
|
/** Spec §9: unanswered at the current hop for 60 minutes → one hop. */
|
|
@@ -71,6 +72,8 @@ export interface ReconcileDeps {
|
|
|
71
72
|
readSessionSenior?: (sessionId: string) => SeniorRef;
|
|
72
73
|
presenceConnected?: boolean;
|
|
73
74
|
env?: NodeJS.ProcessEnv;
|
|
75
|
+
/** Flag reader seam used to assert rollout reads happen before admission locks. */
|
|
76
|
+
readRegistryFlag?: typeof readLanesRegistryFlag;
|
|
74
77
|
/** Detached mesh seam for dead-watcher recovery. */
|
|
75
78
|
mesh?: LaneMesh | null;
|
|
76
79
|
/** Bound on one terminal mesh emit during dead-host recovery. Default 10 s. */
|
|
@@ -10,21 +10,26 @@ import * as path from "node:path";
|
|
|
10
10
|
import { parseUserYaml } from "../../utils/user-yaml-error.js";
|
|
11
11
|
import { compareLaneQueueOrder, hasActiveAdmissionReservation, hasActivePendingResume, laneAdmissionCapReason, laneRunsOnHost, withAllLaneAdmissionLocks, } from "./admission.js";
|
|
12
12
|
import { accountStatus, isCredentialProven, resolveAccount } from "./accounts.js";
|
|
13
|
+
import { effectiveAccountHome } from "./account-home.js";
|
|
13
14
|
import { loadLanesConfig, providerQuotaDefaultHoldMs } from "./config.js";
|
|
14
15
|
import { MAX_LANE_DEPTH } from "./constants.js";
|
|
15
16
|
import { AccountResolutionError, LanesError, isLaneVersionConflict } from "./errors.js";
|
|
17
|
+
import { LANE_HIRE_GRANTS_FLAG, terminalChildResumeHireGrantError, withHireGrantAdmission, } from "./hire-grants.js";
|
|
16
18
|
import { loadWorkerProfile } from "./profile.js";
|
|
17
19
|
import { classifySessionLiveness, defaultLivenessDeps, loadLivenessIndex, readDaemonPresenceConnected, unindexedClaudeSessionLiveness, } from "./liveness.js";
|
|
18
20
|
import { ANSWERED_QUESTIONS_FILE, armLaneWatcher, envelopeRepairMode, isPidRecordAlive, laneRunDir, readPidRecord, readSpawnManifest, markSpawnFailure, spawnEnvelopeRepair, spawnLaneWorker, retryLaneOnAlternateAccount, } from "./spawn.js";
|
|
19
21
|
import { appendCappedHistory, lanesStoreRoot, parseLaneRecord, parseQuestionRecord, readLane, readQuestion, updateLane, updateQuestion, } from "./store.js";
|
|
20
22
|
import { applyThrottleDecision, decideAdmission, defaultThrottleDeps, emptyThrottleState, evaluateThrottle, isCpuCheckEnabled, readAvailableMemMb, readThrottleState, resolveCpuBusyFraction, throttleStatePath, writeThrottleState, } from "./throttle.js";
|
|
23
|
+
import { readLanesRegistryFlag } from "./registry-flag.js";
|
|
24
|
+
import { LEAD_LANE_KEEP_ALIVE_FLAG, LEAD_LANE_KEEP_ALIVE_MAX_RESUMES, LEAD_LANE_KEEP_ALIVE_RESUME_TEXT, leadLaneKeepAliveBackoffMs, } from "./keep-alive.js";
|
|
25
|
+
import { providerSupportsSessionResume } from "./delivery.js";
|
|
21
26
|
import { applyDefaultQuotaHold, readQuotaSignal, recordAccountQuota } from "./quota.js";
|
|
22
27
|
import { answeredQuestionEntries, raiseEnvelopeQuestions } from "./questions.js";
|
|
23
|
-
import { SILENCE_MS, classifyLaneExit, isPidAlive, missingExpectedEnvelopeFile, readLatestEnvelopeFile, readLatestEnvelopeRecord, } from "./watcher.js";
|
|
28
|
+
import { SILENCE_MS, classifyLaneExit, isPidAlive, missingExpectedEnvelopeFile, readLatestEnvelopeFile, readLatestEnvelopeRecord, watcherEventsPath, writeWatcherDecision, } from "./watcher.js";
|
|
24
29
|
import { createLaneMesh, emitLaneDecisionEvents, emitLaneTerminalStateEvent, laneMeshCompany, } from "./mesh-events.js";
|
|
25
30
|
import { laneStories, storyStatusForEnvelope } from "./stories.js";
|
|
26
31
|
import { decideEnvelopeRepair, envelopeQuestionIds, routeProviderFailure } from "./watcher.js";
|
|
27
|
-
import { providerRejectionDetails, readProviderRejection, } from "./provider-rejection.js";
|
|
32
|
+
import { PROVIDER_REJECTED_REASON, providerRejectionDetails, readProviderRejection, } from "./provider-rejection.js";
|
|
28
33
|
/** Spec §9: unanswered at the current hop for 60 minutes → one hop. */
|
|
29
34
|
export const QUESTION_AGE_MS = 60 * 60 * 1000;
|
|
30
35
|
/** Spec §6.6 + §9: a depth-3 worker has two hops to age toward the user. */
|
|
@@ -38,6 +43,13 @@ export const PICKUP_VALIDATION_MAX_AGE_MS = 15 * 60 * 1000;
|
|
|
38
43
|
export const PICKUP_VALIDATION_TIMEOUT_REASON = "pickup_validation_timeout";
|
|
39
44
|
export const RECONCILE_UNIT_NAME = "hq-lanes-reconcile.service";
|
|
40
45
|
export const RECONCILE_TIMER_NAME = "hq-lanes-reconcile.timer";
|
|
46
|
+
function spawnDepsForHost(deps) {
|
|
47
|
+
return {
|
|
48
|
+
...(deps.spawnLaneDeps ?? {}),
|
|
49
|
+
hostname: deps.host.hostname,
|
|
50
|
+
admissionHost: deps.host,
|
|
51
|
+
};
|
|
52
|
+
}
|
|
41
53
|
function logSkip(skip) {
|
|
42
54
|
process.stderr.write(`hq lanes reconcile: skipped ${skip.kind}${skip.id ? ` ${skip.id}` : ""}: ${skip.why}\n`);
|
|
43
55
|
}
|
|
@@ -326,12 +338,16 @@ function availableBalancedAccount(hqRoot, lane, occupying, config, host, profile
|
|
|
326
338
|
continue;
|
|
327
339
|
if (config.providers[lane.provider].accounts[account]?.enabled !== true)
|
|
328
340
|
continue;
|
|
329
|
-
const status = accountStatus(lane.provider, account, config
|
|
341
|
+
const status = accountStatus(lane.provider, account, config, {
|
|
342
|
+
hostname: host.hostname,
|
|
343
|
+
existsSync: (file) => fs.existsSync(file),
|
|
344
|
+
});
|
|
330
345
|
if (!isCredentialProven(status))
|
|
331
346
|
continue;
|
|
332
347
|
try {
|
|
333
348
|
resolveAccount(lane.provider, account, config, {
|
|
334
349
|
hqRoot,
|
|
350
|
+
hostname: host.hostname,
|
|
335
351
|
...(pinnedModel !== undefined ? { model: pinnedModel } : {}),
|
|
336
352
|
});
|
|
337
353
|
}
|
|
@@ -655,7 +671,7 @@ export async function runPassBody(hqRoot, deps) {
|
|
|
655
671
|
};
|
|
656
672
|
const readSessionSenior = deps.readSessionSenior ??
|
|
657
673
|
((sessionId) => defaultReadSessionSenior(hqRoot, sessionId));
|
|
658
|
-
const configuredClaudeHomes = Object.values(config.providers.claude.accounts).map((account) => account.home);
|
|
674
|
+
const configuredClaudeHomes = Object.values(config.providers.claude.accounts).map((account) => effectiveAccountHome(account, deps.host.hostname).home);
|
|
659
675
|
const sessionStateCache = new Map();
|
|
660
676
|
const sessionState = (sessionId) => {
|
|
661
677
|
if (sessionStateCache.has(sessionId)) {
|
|
@@ -780,11 +796,51 @@ export async function runPassBody(hqRoot, deps) {
|
|
|
780
796
|
});
|
|
781
797
|
}
|
|
782
798
|
}
|
|
799
|
+
const readRegistryFlag = deps.readRegistryFlag ?? readLanesRegistryFlag;
|
|
800
|
+
const keepAliveFlagCompanies = new Map();
|
|
801
|
+
const keepAliveHireGrantFlagCompanies = new Map();
|
|
802
|
+
for (const lane of lanes) {
|
|
803
|
+
if (lane.state !== "done" ||
|
|
804
|
+
lane.keep_alive !== true ||
|
|
805
|
+
!laneRunsHere(lane, deps.host) ||
|
|
806
|
+
lane.terminal_reason === PROVIDER_REJECTED_REASON ||
|
|
807
|
+
lane.terminal_reason === "account_not_logged_in" ||
|
|
808
|
+
!providerSupportsSessionResume(lane.provider) ||
|
|
809
|
+
!(lane.provider_session_id ?? lane.last_resume?.session_id))
|
|
810
|
+
continue;
|
|
811
|
+
const envelope = readLatestEnvelopeRecord(hqRoot, lane.lane_id)?.envelope;
|
|
812
|
+
if (!envelope || envelope.decision !== "done" || envelope.story_complete === true)
|
|
813
|
+
continue;
|
|
814
|
+
const key = keepAliveCompanyKey(lane.company);
|
|
815
|
+
keepAliveFlagCompanies.set(key, lane.company);
|
|
816
|
+
if (lane.senior.kind === "lane" && lane.senior.id) {
|
|
817
|
+
keepAliveHireGrantFlagCompanies.set(key, lane.company);
|
|
818
|
+
}
|
|
819
|
+
}
|
|
820
|
+
const keepAliveFlagByCompany = new Map();
|
|
821
|
+
const keepAliveHireGrantFlagByCompany = new Map();
|
|
822
|
+
const env = deps.env ?? process.env;
|
|
823
|
+
await Promise.all([
|
|
824
|
+
...[...keepAliveFlagCompanies.entries()].map(async ([key, company]) => {
|
|
825
|
+
keepAliveFlagByCompany.set(key, await readRegistryFlag(LEAD_LANE_KEEP_ALIVE_FLAG, {
|
|
826
|
+
company: company ?? undefined,
|
|
827
|
+
env,
|
|
828
|
+
}));
|
|
829
|
+
}),
|
|
830
|
+
...[...keepAliveHireGrantFlagCompanies.entries()].map(async ([key, company]) => {
|
|
831
|
+
keepAliveHireGrantFlagByCompany.set(key, await readRegistryFlag(LANE_HIRE_GRANTS_FLAG, {
|
|
832
|
+
company: company ?? undefined,
|
|
833
|
+
env,
|
|
834
|
+
}));
|
|
835
|
+
}),
|
|
836
|
+
]);
|
|
783
837
|
await withAllLaneAdmissionLocks(hqRoot, async () => {
|
|
784
838
|
const freshLanes = loadLaneRecords(hqRoot, summary);
|
|
839
|
+
resumeDoneKeepAlive(hqRoot, freshLanes, summary, deps, now, keepAliveFlagByCompany, keepAliveHireGrantFlagByCompany);
|
|
840
|
+
const drainLanes = loadLaneRecords(hqRoot, summary);
|
|
785
841
|
const freshQuestions = loadQuestionRecords(hqRoot, summary);
|
|
786
|
-
await admitQueued(hqRoot,
|
|
787
|
-
await resumeAwaiting(hqRoot,
|
|
842
|
+
await admitQueued(hqRoot, drainLanes, summary, deps, now);
|
|
843
|
+
await resumeAwaiting(hqRoot, drainLanes, freshQuestions, summary, deps);
|
|
788
844
|
});
|
|
789
845
|
if (!deps.dryRun && summary.errors.length === 0) {
|
|
790
846
|
try {
|
|
@@ -887,7 +943,7 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
|
|
|
887
943
|
}
|
|
888
944
|
if (!watcherAlive && !deps.dryRun) {
|
|
889
945
|
try {
|
|
890
|
-
armLaneWatcher(hqRoot, lane.lane_id, deps
|
|
946
|
+
armLaneWatcher(hqRoot, lane.lane_id, spawnDepsForHost(deps));
|
|
891
947
|
summary.heartbeats.push(`${lane.lane_id}:watcher-rearmed`);
|
|
892
948
|
}
|
|
893
949
|
catch (err) {
|
|
@@ -941,10 +997,10 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
|
|
|
941
997
|
repairCandidate.envelope === null &&
|
|
942
998
|
(!repairCandidate.missing || envelopeRepairMode(classified.provider) === "resume")) {
|
|
943
999
|
const spawnRepair = deps.spawnRepair ??
|
|
944
|
-
((root, repairLane, input) => spawnEnvelopeRepair(root, repairLane, input, deps
|
|
1000
|
+
((root, repairLane, input) => spawnEnvelopeRepair(root, repairLane, input, spawnDepsForHost(deps)));
|
|
945
1001
|
const repair = decideEnvelopeRepair(hqRoot, lane.lane_id, repairCandidate, nowMs, spawnRepair, () => undefined);
|
|
946
1002
|
if (repair.action === "spawned") {
|
|
947
|
-
armLaneWatcher(hqRoot, lane.lane_id, deps
|
|
1003
|
+
armLaneWatcher(hqRoot, lane.lane_id, spawnDepsForHost(deps));
|
|
948
1004
|
summary.heartbeats.push(`${lane.lane_id}:envelope-repair`);
|
|
949
1005
|
continue;
|
|
950
1006
|
}
|
|
@@ -969,8 +1025,7 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
|
|
|
969
1025
|
if (!deps.dryRun && accountRetryReason && classified.account) {
|
|
970
1026
|
try {
|
|
971
1027
|
const recovered = await retryLaneOnAlternateAccount(hqRoot, classified, accountRetryReason, "reconcile", {
|
|
972
|
-
...(deps
|
|
973
|
-
admissionHost: deps.host,
|
|
1028
|
+
...spawnDepsForHost(deps),
|
|
974
1029
|
});
|
|
975
1030
|
summary.dead_host.push({
|
|
976
1031
|
id: lane.lane_id,
|
|
@@ -1197,7 +1252,7 @@ async function admitQueued(hqRoot, lanes, summary, deps, now) {
|
|
|
1197
1252
|
nowMs,
|
|
1198
1253
|
});
|
|
1199
1254
|
const spawnFn = deps.spawn ??
|
|
1200
|
-
((root, lane) => spawnLaneWorker(root, lane, deps
|
|
1255
|
+
((root, lane) => spawnLaneWorker(root, lane, spawnDepsForHost(deps)));
|
|
1201
1256
|
const running = lanes.filter((lane) => (lane.state === "running" ||
|
|
1202
1257
|
hasFreshAdmissionReservation(lane, now.getTime()) ||
|
|
1203
1258
|
hasActivePendingResume(lane, now.getTime())) &&
|
|
@@ -1311,6 +1366,231 @@ async function admitQueued(hqRoot, lanes, summary, deps, now) {
|
|
|
1311
1366
|
writeThrottleState(filePath, applyThrottleDecision(base, applied, nowMs), throttleDeps);
|
|
1312
1367
|
}
|
|
1313
1368
|
}
|
|
1369
|
+
function keepAliveCompanyKey(company) {
|
|
1370
|
+
return JSON.stringify([company?.uid ?? null, company?.slug ?? null]);
|
|
1371
|
+
}
|
|
1372
|
+
function hasKeepAliveProgress(lane) {
|
|
1373
|
+
return (lane.keep_alive_resume_count ?? 0) !== 0 ||
|
|
1374
|
+
lane.keep_alive_last_resume_at !== undefined ||
|
|
1375
|
+
lane.keep_alive_next_eligible_at !== undefined ||
|
|
1376
|
+
lane.keep_alive_exhausted_at !== undefined;
|
|
1377
|
+
}
|
|
1378
|
+
function clearedKeepAliveProgress(lane) {
|
|
1379
|
+
return {
|
|
1380
|
+
...lane,
|
|
1381
|
+
keep_alive_resume_count: 0,
|
|
1382
|
+
keep_alive_last_resume_at: undefined,
|
|
1383
|
+
keep_alive_next_eligible_at: undefined,
|
|
1384
|
+
keep_alive_exhausted_at: undefined,
|
|
1385
|
+
};
|
|
1386
|
+
}
|
|
1387
|
+
function keepAliveProgressResetByHuman(lane) {
|
|
1388
|
+
const count = lane.keep_alive_resume_count ?? 0;
|
|
1389
|
+
const lastAutoMs = Date.parse(lane.keep_alive_last_resume_at ?? "");
|
|
1390
|
+
if (count > 0 && !Number.isFinite(lastAutoMs))
|
|
1391
|
+
return true;
|
|
1392
|
+
const messageMs = Date.parse(lane.last_message_at ?? "");
|
|
1393
|
+
const resumeMs = lane.last_resume?.text === LEAD_LANE_KEEP_ALIVE_RESUME_TEXT
|
|
1394
|
+
? Number.NaN
|
|
1395
|
+
: Date.parse(lane.last_resume?.at ?? "");
|
|
1396
|
+
return Number.isFinite(lastAutoMs) &&
|
|
1397
|
+
((Number.isFinite(messageMs) && messageMs > lastAutoMs) ||
|
|
1398
|
+
(Number.isFinite(resumeMs) && resumeMs > lastAutoMs));
|
|
1399
|
+
}
|
|
1400
|
+
function recordKeepAliveEvent(hqRoot, laneId, summary, event) {
|
|
1401
|
+
try {
|
|
1402
|
+
writeWatcherDecision(watcherEventsPath(hqRoot, laneId), {
|
|
1403
|
+
lane_id: laneId,
|
|
1404
|
+
...event,
|
|
1405
|
+
});
|
|
1406
|
+
}
|
|
1407
|
+
catch (err) {
|
|
1408
|
+
markDirty(summary, {
|
|
1409
|
+
kind: "lane",
|
|
1410
|
+
id: laneId,
|
|
1411
|
+
why: `keep-alive event write failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
1412
|
+
});
|
|
1413
|
+
}
|
|
1414
|
+
}
|
|
1415
|
+
function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, hireGrantFlagByCompany) {
|
|
1416
|
+
const nowMs = now.getTime();
|
|
1417
|
+
const nowIso = now.toISOString();
|
|
1418
|
+
const config = loadLanesConfig(hqRoot);
|
|
1419
|
+
for (const lane of lanes) {
|
|
1420
|
+
if (lane.state !== "done" ||
|
|
1421
|
+
lane.keep_alive !== true ||
|
|
1422
|
+
!laneRunsHere(lane, deps.host))
|
|
1423
|
+
continue;
|
|
1424
|
+
const latest = readLatestEnvelopeRecord(hqRoot, lane.lane_id);
|
|
1425
|
+
const envelope = latest?.envelope;
|
|
1426
|
+
if (!envelope || envelope.decision !== "done")
|
|
1427
|
+
continue;
|
|
1428
|
+
if (envelope.story_complete === true) {
|
|
1429
|
+
if (hasKeepAliveProgress(lane)) {
|
|
1430
|
+
casLaneIf(hqRoot, lane.lane_id, (current) => current.version === lane.version &&
|
|
1431
|
+
current.state === "done" &&
|
|
1432
|
+
current.keep_alive === true, (current) => clearedKeepAliveProgress(current), deps.dryRun);
|
|
1433
|
+
}
|
|
1434
|
+
continue;
|
|
1435
|
+
}
|
|
1436
|
+
if (lane.terminal_reason === PROVIDER_REJECTED_REASON ||
|
|
1437
|
+
lane.terminal_reason === "account_not_logged_in" ||
|
|
1438
|
+
!providerSupportsSessionResume(lane.provider) ||
|
|
1439
|
+
!(lane.provider_session_id ?? lane.last_resume?.session_id))
|
|
1440
|
+
continue;
|
|
1441
|
+
if (flagByCompany.get(keepAliveCompanyKey(lane.company)) !== true)
|
|
1442
|
+
continue;
|
|
1443
|
+
const current = readLane(hqRoot, lane.lane_id);
|
|
1444
|
+
if (current.version !== lane.version ||
|
|
1445
|
+
current.state !== "done" ||
|
|
1446
|
+
current.keep_alive !== true ||
|
|
1447
|
+
!laneRunsHere(current, deps.host))
|
|
1448
|
+
continue;
|
|
1449
|
+
const currentEnvelope = readLatestEnvelopeRecord(hqRoot, current.lane_id)?.envelope;
|
|
1450
|
+
if (!currentEnvelope || currentEnvelope.decision !== "done")
|
|
1451
|
+
continue;
|
|
1452
|
+
if (currentEnvelope.story_complete === true) {
|
|
1453
|
+
if (hasKeepAliveProgress(current)) {
|
|
1454
|
+
casLaneIf(hqRoot, current.lane_id, (latestLane) => latestLane.version === current.version &&
|
|
1455
|
+
latestLane.state === "done" &&
|
|
1456
|
+
latestLane.keep_alive === true, (latestLane) => clearedKeepAliveProgress(latestLane), deps.dryRun);
|
|
1457
|
+
}
|
|
1458
|
+
continue;
|
|
1459
|
+
}
|
|
1460
|
+
const providerSessionId = current.provider_session_id ?? current.last_resume?.session_id;
|
|
1461
|
+
if (current.terminal_reason === PROVIDER_REJECTED_REASON ||
|
|
1462
|
+
current.terminal_reason === "account_not_logged_in" ||
|
|
1463
|
+
typeof providerSessionId !== "string" ||
|
|
1464
|
+
providerSessionId.trim().length === 0)
|
|
1465
|
+
continue;
|
|
1466
|
+
const resetByHuman = keepAliveProgressResetByHuman(current);
|
|
1467
|
+
const attempts = resetByHuman ? 0 : current.keep_alive_resume_count ?? 0;
|
|
1468
|
+
if (attempts >= LEAD_LANE_KEEP_ALIVE_MAX_RESUMES) {
|
|
1469
|
+
if (current.keep_alive_exhausted_at !== undefined)
|
|
1470
|
+
continue;
|
|
1471
|
+
const exhausted = casLaneIf(hqRoot, current.lane_id, (latestLane) => latestLane.version === current.version &&
|
|
1472
|
+
latestLane.state === "done" &&
|
|
1473
|
+
latestLane.keep_alive === true &&
|
|
1474
|
+
(latestLane.keep_alive_resume_count ?? 0) >= LEAD_LANE_KEEP_ALIVE_MAX_RESUMES &&
|
|
1475
|
+
latestLane.keep_alive_exhausted_at === undefined, (latestLane) => ({ ...latestLane, keep_alive_exhausted_at: nowIso }), deps.dryRun);
|
|
1476
|
+
if (exhausted.changed && !deps.dryRun) {
|
|
1477
|
+
recordKeepAliveEvent(hqRoot, current.lane_id, summary, {
|
|
1478
|
+
event: "keep_alive_exhausted",
|
|
1479
|
+
at: nowIso,
|
|
1480
|
+
attempt: attempts,
|
|
1481
|
+
});
|
|
1482
|
+
}
|
|
1483
|
+
continue;
|
|
1484
|
+
}
|
|
1485
|
+
const attempt = attempts + 1;
|
|
1486
|
+
const doneAtMs = Date.parse(current.updated_at);
|
|
1487
|
+
const attemptEligibleMs = doneAtMs + leadLaneKeepAliveBackoffMs(attempt);
|
|
1488
|
+
const storedNextMs = resetByHuman
|
|
1489
|
+
? Number.NaN
|
|
1490
|
+
: Date.parse(current.keep_alive_next_eligible_at ?? "");
|
|
1491
|
+
const nextEligibleMs = Number.isFinite(storedNextMs)
|
|
1492
|
+
? Math.max(attemptEligibleMs, storedNextMs)
|
|
1493
|
+
: attemptEligibleMs;
|
|
1494
|
+
if (nowMs < nextEligibleMs) {
|
|
1495
|
+
if (resetByHuman && hasKeepAliveProgress(current)) {
|
|
1496
|
+
casLaneIf(hqRoot, current.lane_id, (latestLane) => latestLane.version === current.version &&
|
|
1497
|
+
latestLane.state === "done" &&
|
|
1498
|
+
latestLane.keep_alive === true, (latestLane) => clearedKeepAliveProgress(latestLane), deps.dryRun);
|
|
1499
|
+
}
|
|
1500
|
+
continue;
|
|
1501
|
+
}
|
|
1502
|
+
const nextEligibleAt = new Date(nowMs + leadLaneKeepAliveBackoffMs(attempt + 1)).toISOString();
|
|
1503
|
+
const queuedCount = loadLaneRecords(hqRoot, summary).filter((queuedLane) => queuedLane.state === "queued" &&
|
|
1504
|
+
queuedLane.provider === current.provider &&
|
|
1505
|
+
laneRunsHere(queuedLane, deps.host) &&
|
|
1506
|
+
!hasFreshAdmissionReservation(queuedLane, nowMs)).length;
|
|
1507
|
+
const maxQueued = config.providers[current.provider].max_queued;
|
|
1508
|
+
if (queuedCount >= maxQueued) {
|
|
1509
|
+
summary.skipped.push({
|
|
1510
|
+
kind: "lane",
|
|
1511
|
+
id: current.lane_id,
|
|
1512
|
+
why: `lane_queue_full: ${current.provider} queue has ${queuedCount}/${maxQueued} lanes`,
|
|
1513
|
+
});
|
|
1514
|
+
continue;
|
|
1515
|
+
}
|
|
1516
|
+
const queue = (expectedSeniorId) => casLaneIf(hqRoot, current.lane_id, (latestLane) => latestLane.version === current.version &&
|
|
1517
|
+
latestLane.state === "done" &&
|
|
1518
|
+
latestLane.keep_alive === true &&
|
|
1519
|
+
(expectedSeniorId === undefined ||
|
|
1520
|
+
(latestLane.senior.kind === "lane" && latestLane.senior.id === expectedSeniorId)), (latestLane) => ({
|
|
1521
|
+
...latestLane,
|
|
1522
|
+
state: "queued",
|
|
1523
|
+
queued_reason: undefined,
|
|
1524
|
+
queued_at: nowIso,
|
|
1525
|
+
admission_reserved_at: undefined,
|
|
1526
|
+
terminal_reason: undefined,
|
|
1527
|
+
terminal_details: undefined,
|
|
1528
|
+
pending_resume: {
|
|
1529
|
+
session_id: providerSessionId,
|
|
1530
|
+
trigger: "resume",
|
|
1531
|
+
resumed_from: "done",
|
|
1532
|
+
text: LEAD_LANE_KEEP_ALIVE_RESUME_TEXT,
|
|
1533
|
+
compact: true,
|
|
1534
|
+
at: nowIso,
|
|
1535
|
+
},
|
|
1536
|
+
keep_alive_resume_count: attempt,
|
|
1537
|
+
keep_alive_last_resume_at: nowIso,
|
|
1538
|
+
keep_alive_next_eligible_at: nextEligibleAt,
|
|
1539
|
+
keep_alive_exhausted_at: undefined,
|
|
1540
|
+
}), deps.dryRun);
|
|
1541
|
+
const seniorId = current.senior.kind === "lane" ? current.senior.id : undefined;
|
|
1542
|
+
let queued;
|
|
1543
|
+
if (seniorId &&
|
|
1544
|
+
hireGrantFlagByCompany.get(keepAliveCompanyKey(current.company)) === true) {
|
|
1545
|
+
try {
|
|
1546
|
+
queued = withHireGrantAdmission(hqRoot, seniorId, (snapshot) => {
|
|
1547
|
+
const latestLane = readLane(hqRoot, current.lane_id);
|
|
1548
|
+
if (latestLane.version !== current.version ||
|
|
1549
|
+
latestLane.senior.kind !== "lane" ||
|
|
1550
|
+
latestLane.senior.id !== seniorId)
|
|
1551
|
+
return { changed: false, record: latestLane };
|
|
1552
|
+
const refusal = terminalChildResumeHireGrantError(latestLane, snapshot);
|
|
1553
|
+
if (refusal !== null) {
|
|
1554
|
+
summary.skipped.push({
|
|
1555
|
+
kind: "lane",
|
|
1556
|
+
id: current.lane_id,
|
|
1557
|
+
why: `keep_alive_hire_admission: ${refusal.error}: ${refusal.message}`,
|
|
1558
|
+
});
|
|
1559
|
+
return { changed: false, record: latestLane };
|
|
1560
|
+
}
|
|
1561
|
+
return queue(seniorId);
|
|
1562
|
+
});
|
|
1563
|
+
}
|
|
1564
|
+
catch (err) {
|
|
1565
|
+
markDirty(summary, {
|
|
1566
|
+
kind: "lane",
|
|
1567
|
+
id: current.lane_id,
|
|
1568
|
+
why: `keep-alive hire admission failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
1569
|
+
});
|
|
1570
|
+
continue;
|
|
1571
|
+
}
|
|
1572
|
+
}
|
|
1573
|
+
else {
|
|
1574
|
+
queued = queue(seniorId);
|
|
1575
|
+
}
|
|
1576
|
+
if (!queued.changed)
|
|
1577
|
+
continue;
|
|
1578
|
+
summary.resumed.push({
|
|
1579
|
+
id: current.lane_id,
|
|
1580
|
+
from: "done",
|
|
1581
|
+
to: "queued",
|
|
1582
|
+
reason: "keep_alive",
|
|
1583
|
+
});
|
|
1584
|
+
if (!deps.dryRun) {
|
|
1585
|
+
recordKeepAliveEvent(hqRoot, current.lane_id, summary, {
|
|
1586
|
+
event: "keep_alive_resume",
|
|
1587
|
+
at: nowIso,
|
|
1588
|
+
attempt,
|
|
1589
|
+
next_eligible_at: nextEligibleAt,
|
|
1590
|
+
});
|
|
1591
|
+
}
|
|
1592
|
+
}
|
|
1593
|
+
}
|
|
1314
1594
|
async function resumeAwaiting(hqRoot, lanes, questions, summary, deps, onlyLaneIds) {
|
|
1315
1595
|
const config = loadLanesConfig(hqRoot);
|
|
1316
1596
|
const awaiting = lanes.filter((lane) => lane.state === "awaiting_input" &&
|
|
@@ -1336,7 +1616,7 @@ async function resumeAwaiting(hqRoot, lanes, questions, summary, deps, onlyLaneI
|
|
|
1336
1616
|
const throttle = await decideAdmission(hqRoot, config.throttle, throttleDeps);
|
|
1337
1617
|
throttleDecision = throttle.decision;
|
|
1338
1618
|
}
|
|
1339
|
-
const spawnFn = deps.spawn ?? ((root, lane) => spawnLaneWorker(root, lane, deps
|
|
1619
|
+
const spawnFn = deps.spawn ?? ((root, lane) => spawnLaneWorker(root, lane, spawnDepsForHost(deps)));
|
|
1340
1620
|
const nowMs = deps.now().getTime();
|
|
1341
1621
|
const fallbackAnswerAt = new Date(nowMs).toISOString();
|
|
1342
1622
|
const answerAtByLane = new Map();
|
|
@@ -68,6 +68,7 @@ export type ProveDetachedFn = (pid: number) => DetachProof;
|
|
|
68
68
|
export type KillProcessFn = (pid: number, signal?: NodeJS.Signals) => void;
|
|
69
69
|
export interface SpawnLaneDeps {
|
|
70
70
|
env?: NodeJS.ProcessEnv;
|
|
71
|
+
hostname?: string;
|
|
71
72
|
/** The caller's host scope for admission checks; reconcile supplies its resolved host. */
|
|
72
73
|
admissionHost?: AdmissionHost;
|
|
73
74
|
/** Provider model catalog dependencies, shared with create in tests. */
|