@indigoai-us/hq-cli 5.213.0 → 5.215.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +26 -0
- package/dist/command-catalog.generated.d.ts +22 -0
- package/dist/command-catalog.generated.js +29 -0
- package/dist/commands/cloud.d.ts +13 -1
- package/dist/commands/cloud.js +24 -3
- package/dist/commands/lanes.js +68 -12
- package/dist/lib/doctor/checks/sync-health.d.ts +2 -0
- package/dist/lib/doctor/checks/sync-health.js +4 -0
- package/dist/lib/doctor/checks/sync-runtime.d.ts +11 -0
- package/dist/lib/doctor/checks/sync-runtime.js +166 -0
- package/dist/lib/doctor/client-health-sentry.d.ts +1 -1
- package/dist/lib/doctor/client-health-sentry.js +7 -0
- package/dist/lib/doctor/types.d.ts +1 -1
- package/dist/lib/doctor/types.js +5 -0
- package/dist/lib/lanes/keep-alive.d.ts +9 -0
- package/dist/lib/lanes/keep-alive.js +101 -0
- package/dist/lib/lanes/profile.d.ts +3 -3
- package/dist/lib/lanes/profile.js +4 -11
- package/dist/lib/lanes/reconcile.js +126 -40
- package/dist/lib/lanes/store.js +20 -0
- package/dist/lib/lanes/types.d.ts +4 -0
- package/dist/lib/lanes/types.js +1 -1
- package/package.json +2 -2
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,32 @@
|
|
|
2
2
|
|
|
3
3
|
## [Unreleased]
|
|
4
4
|
|
|
5
|
+
## [5.215.0] — 2026-09-26
|
|
6
|
+
|
|
7
|
+
### Fixed
|
|
8
|
+
|
|
9
|
+
- hq-cli 5.211.0 could resume a retired lead from its worker profile's
|
|
10
|
+
`keep_alive` value. Reconcile now requires an explicit lane opt-in written
|
|
11
|
+
after the latest envelope, and skips retired or superseded leads.
|
|
12
|
+
## [5.214.0] — 2026-09-26
|
|
13
|
+
|
|
14
|
+
### Added
|
|
15
|
+
|
|
16
|
+
- `hq lanes keep-alive <lane-id> --on|--off` lets a lane's senior opt in on
|
|
17
|
+
the host that owns the lane.
|
|
18
|
+
|
|
19
|
+
- `hq doctor` now checks the last hour of the sync runner log for stuck
|
|
20
|
+
realtime lag, cloud permission refusals, runner restart loops, device-limit
|
|
21
|
+
backoff, and repeated conflicts. Client health reports send these to Sentry
|
|
22
|
+
as reason codes and counts only.
|
|
23
|
+
|
|
24
|
+
### Fixed
|
|
25
|
+
|
|
26
|
+
- The desktop app's first sync push now uploads files inside private folders.
|
|
27
|
+
The credentials the app hands to `hq sync push --creds-from-stdin` cannot
|
|
28
|
+
write there directly, so the push also passes the vault config to hq-cloud
|
|
29
|
+
6.18.5, which sends those files through presigned uploads.
|
|
30
|
+
|
|
5
31
|
## [5.213.0] — 2026-09-26
|
|
6
32
|
|
|
7
33
|
### Fixed
|
|
@@ -7399,6 +7399,28 @@ export declare const COMMAND_CATALOG: readonly [{
|
|
|
7399
7399
|
readonly description: "JSON on stdout (always)";
|
|
7400
7400
|
}];
|
|
7401
7401
|
readonly subcommands: readonly [];
|
|
7402
|
+
}, {
|
|
7403
|
+
readonly name: "keep-alive";
|
|
7404
|
+
readonly description: "Opt a lane into or out of automatic lead-lane resumes";
|
|
7405
|
+
readonly aliases: readonly [];
|
|
7406
|
+
readonly hidden: false;
|
|
7407
|
+
readonly usage: "[options] <lane-id>";
|
|
7408
|
+
readonly arguments: readonly [{
|
|
7409
|
+
readonly name: "lane-id";
|
|
7410
|
+
readonly required: true;
|
|
7411
|
+
readonly variadic: false;
|
|
7412
|
+
}];
|
|
7413
|
+
readonly options: readonly [{
|
|
7414
|
+
readonly flags: "--on";
|
|
7415
|
+
readonly description: "Enable explicit keep-alive for this lane";
|
|
7416
|
+
}, {
|
|
7417
|
+
readonly flags: "--off";
|
|
7418
|
+
readonly description: "Disable keep-alive for this lane";
|
|
7419
|
+
}, {
|
|
7420
|
+
readonly flags: "--json";
|
|
7421
|
+
readonly description: "Emit JSON on stdout (default)";
|
|
7422
|
+
}];
|
|
7423
|
+
readonly subcommands: readonly [];
|
|
7402
7424
|
}, {
|
|
7403
7425
|
readonly name: "grant";
|
|
7404
7426
|
readonly description: "Set a lane's alive child-lane grant as its senior";
|
|
@@ -9566,6 +9566,35 @@ export const COMMAND_CATALOG = [
|
|
|
9566
9566
|
],
|
|
9567
9567
|
"subcommands": []
|
|
9568
9568
|
},
|
|
9569
|
+
{
|
|
9570
|
+
"name": "keep-alive",
|
|
9571
|
+
"description": "Opt a lane into or out of automatic lead-lane resumes",
|
|
9572
|
+
"aliases": [],
|
|
9573
|
+
"hidden": false,
|
|
9574
|
+
"usage": "[options] <lane-id>",
|
|
9575
|
+
"arguments": [
|
|
9576
|
+
{
|
|
9577
|
+
"name": "lane-id",
|
|
9578
|
+
"required": true,
|
|
9579
|
+
"variadic": false
|
|
9580
|
+
}
|
|
9581
|
+
],
|
|
9582
|
+
"options": [
|
|
9583
|
+
{
|
|
9584
|
+
"flags": "--on",
|
|
9585
|
+
"description": "Enable explicit keep-alive for this lane"
|
|
9586
|
+
},
|
|
9587
|
+
{
|
|
9588
|
+
"flags": "--off",
|
|
9589
|
+
"description": "Disable keep-alive for this lane"
|
|
9590
|
+
},
|
|
9591
|
+
{
|
|
9592
|
+
"flags": "--json",
|
|
9593
|
+
"description": "Emit JSON on stdout (default)"
|
|
9594
|
+
}
|
|
9595
|
+
],
|
|
9596
|
+
"subcommands": []
|
|
9597
|
+
},
|
|
9569
9598
|
{
|
|
9570
9599
|
"name": "grant",
|
|
9571
9600
|
"description": "Set a lane's alive child-lane grant as its senior",
|
package/dist/commands/cloud.d.ts
CHANGED
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
* hq sync status — show local journal summary
|
|
14
14
|
*/
|
|
15
15
|
import { Command } from "commander";
|
|
16
|
-
import { type ConflictStrategy, type MembershipSyncConfig, type SyncMode, type PullScope, type ExplicitGrant } from "@indigoai-us/hq-cloud";
|
|
16
|
+
import { type ConflictStrategy, type EntityContext, type MembershipSyncConfig, type SyncMode, type PullScope, type ExplicitGrant, type ShareOptions, type VaultServiceConfig } from "@indigoai-us/hq-cloud";
|
|
17
17
|
import { type BannerLevel } from "../lib/narrow-hint-banner.js";
|
|
18
18
|
/**
|
|
19
19
|
* Terminal-failure signal for the sync fanout runners. The runners
|
|
@@ -264,6 +264,18 @@ export declare function pullAll(options: PullAllOptions, deps: PullAllDeps): Pro
|
|
|
264
264
|
* deps so tests can drive it without network or filesystem.
|
|
265
265
|
*/
|
|
266
266
|
export declare function pushAll(options: PushAllOptions, deps: PushAllDeps): Promise<PushAllResult>;
|
|
267
|
+
/**
|
|
268
|
+
* Credentials `hq sync push` hands to share(). A pre-vended EntityContext
|
|
269
|
+
* (AppBar's `--creds-from-stdin` first push) cannot write under private
|
|
270
|
+
* folders directly, so it travels with the caller's vault config as
|
|
271
|
+
* `privateFolderVaultConfig`. share() takes exactly one of `vaultConfig` /
|
|
272
|
+
* `entityContext`.
|
|
273
|
+
*/
|
|
274
|
+
export declare function pushShareCredentials(input: {
|
|
275
|
+
entityContext?: EntityContext;
|
|
276
|
+
vaultConfig?: VaultServiceConfig;
|
|
277
|
+
scopeVaultConfig?: VaultServiceConfig;
|
|
278
|
+
}): Pick<ShareOptions, "entityContext" | "vaultConfig" | "privateFolderVaultConfig">;
|
|
267
279
|
/**
|
|
268
280
|
* Resolve the canonical person entity UID for the logged-in user. Used by
|
|
269
281
|
* `hq sync {push,pull,now} --personal` to target the personal vault without
|
package/dist/commands/cloud.js
CHANGED
|
@@ -455,6 +455,24 @@ export async function pushAll(options, deps) {
|
|
|
455
455
|
}
|
|
456
456
|
return result;
|
|
457
457
|
}
|
|
458
|
+
/**
|
|
459
|
+
* Credentials `hq sync push` hands to share(). A pre-vended EntityContext
|
|
460
|
+
* (AppBar's `--creds-from-stdin` first push) cannot write under private
|
|
461
|
+
* folders directly, so it travels with the caller's vault config as
|
|
462
|
+
* `privateFolderVaultConfig`. share() takes exactly one of `vaultConfig` /
|
|
463
|
+
* `entityContext`.
|
|
464
|
+
*/
|
|
465
|
+
export function pushShareCredentials(input) {
|
|
466
|
+
if (input.entityContext) {
|
|
467
|
+
return {
|
|
468
|
+
entityContext: input.entityContext,
|
|
469
|
+
...(input.scopeVaultConfig
|
|
470
|
+
? { privateFolderVaultConfig: input.scopeVaultConfig }
|
|
471
|
+
: {}),
|
|
472
|
+
};
|
|
473
|
+
}
|
|
474
|
+
return input.vaultConfig ? { vaultConfig: input.vaultConfig } : {};
|
|
475
|
+
}
|
|
458
476
|
/**
|
|
459
477
|
* Resolve the canonical person entity UID for the logged-in user. Used by
|
|
460
478
|
* `hq sync {push,pull,now} --personal` to target the personal vault without
|
|
@@ -715,7 +733,7 @@ export function registerCloudCommands(program) {
|
|
|
715
733
|
// 2. default: vend via cached Cognito session (the human CLI path).
|
|
716
734
|
let entityContext;
|
|
717
735
|
let vaultConfig;
|
|
718
|
-
/** Vault API config for membership/grant scope lookups
|
|
736
|
+
/** Vault API config for membership/grant scope lookups and, on the pre-vended `--creds-from-stdin` path, private-folder uploads. */
|
|
719
737
|
let scopeVaultConfig;
|
|
720
738
|
if (options.credsFromStdin) {
|
|
721
739
|
if (process.stdin.isTTY) {
|
|
@@ -798,8 +816,11 @@ export function registerCloudCommands(program) {
|
|
|
798
816
|
company: targetCompany,
|
|
799
817
|
message: options.message,
|
|
800
818
|
onConflict: options.onConflict,
|
|
801
|
-
|
|
802
|
-
|
|
819
|
+
...pushShareCredentials({
|
|
820
|
+
entityContext,
|
|
821
|
+
vaultConfig,
|
|
822
|
+
scopeVaultConfig,
|
|
823
|
+
}),
|
|
803
824
|
hqRoot: options.hqRoot,
|
|
804
825
|
onEvent,
|
|
805
826
|
...(personalMode ? { personalMode: true } : {}),
|
package/dist/commands/lanes.js
CHANGED
|
@@ -36,9 +36,10 @@ import { CompanySelectionError } from "../utils/company-selection-error.js";
|
|
|
36
36
|
import { createLaneMesh, emitLaneDecisionEvents, emitLaneTerminalStateEvent, laneMeshEventIdentity, } from "../lib/lanes/mesh-events.js";
|
|
37
37
|
import { sessionIdFromEnv } from "../lib/mesh/live/session-identity.js";
|
|
38
38
|
import { effectiveAccountHome } from "../lib/lanes/account-home.js";
|
|
39
|
-
import { AccountResolutionError, compareLaneQueueOrder, coveredLaneIds, DEFAULT_ACCOUNT_HOMES, LANE_PROVIDERS, LANE_STATES, PROVIDER_HOME_ENV, QUESTION_STATUSES, SENIOR_KINDS, accountStatus, appendCappedHistory, canonicalizeHome, createLane, createQuestion, decideAdmission, defaultLivenessDeps, defaultSystemdUserDir, defaultThrottleDeps, installDrainHook, installReconcileTimer, providerPathEnv, probeDrainHookRegistration, isLanesError, isLaneVersionConflict, isTerminalLaneState, aliveDirectChildren, assignedChildHireGrants, LANE_HIRE_GRANTS_FLAG, planHireGrantBackfill, terminalChildResumeHireGrantError, withHireGrantAdmission, laneStoryLedgerEnabled, validateProjectStories, buildLaneStatusSnapshot, formatLaneStatusHuman, listAccountStatuses, listLaneIds, listQuestionIds, listSessionLiveness, loadLanesConfig, laneRunsHere, MAX_LANE_DEPTH, MONITOR_ERROR_EXIT_CODE, MONITOR_ENGINES, MONITOR_MAX_TIMEOUT_MS, MONITOR_UNCOVERED_EXIT_CODE, MAX_WATCHED_LANES, monitorReminderText, readLiveMonitorEvidence, resolveMachineIdentity as lookupMachineIdentity, loadLivenessIndex, parseLaneProvider, parseLaneRecord, parseLaneState, parseLivenessState, parseQuestionRecord, parseQuestionStatus, parseSeniorKind, readDaemonPresenceConnected, readLane, retryUpdateLane, syncLaneRecordFromOwner, readQuestion, resolveAccount, resolveLaneWorker, ignoredWorkerProfileFields, loadWorkerProfile, runReconcilePass, wakeAnsweredLaneNow, runLaneWatch, sessionLiveness, tryReadLane, uninstallReconcileTimer, updateLane, updateQuestion, writeLanesConfig, appendDropboxMessage, answerDropboxMessage, deliveryRefusalReason, resumeCommandRefusal, dropboxDir, dropboxInboxPath, laneAcceptsDropboxDelivery, laneAcceptsOperatorMessage, laneDeliveryMechanism, hasActiveAdmissionReservation, hasActivePendingResume, hasRecordedSessionId, hasEarlierEligibleQueuedLane, laneAdmissionCapReason, providerSupportsSessionResume, providerSessionPortable, sessionNotPortableMessage, withLaneAdmissionLock, withLaneHireLocks, laneIdForAnswerDelivery, operatorMessageRefusalReason, raiseLaneQuestion, sha256Text, writeQuestionBody, } from "../lib/lanes/index.js";
|
|
39
|
+
import { AccountResolutionError, compareLaneQueueOrder, coveredLaneIds, DEFAULT_ACCOUNT_HOMES, LANE_PROVIDERS, LANE_STATES, PROVIDER_HOME_ENV, QUESTION_STATUSES, SENIOR_KINDS, accountStatus, appendCappedHistory, canonicalizeHome, createLane, createQuestion, decideAdmission, defaultLivenessDeps, defaultSystemdUserDir, defaultThrottleDeps, installDrainHook, installReconcileTimer, providerPathEnv, probeDrainHookRegistration, isLanesError, isLaneVersionConflict, isTerminalLaneState, aliveDirectChildren, assignedChildHireGrants, LANE_HIRE_GRANTS_FLAG, planHireGrantBackfill, terminalChildResumeHireGrantError, withHireGrantAdmission, laneStoryLedgerEnabled, validateProjectStories, buildLaneStatusSnapshot, formatLaneStatusHuman, listAccountStatuses, listLaneIds, listQuestionIds, listSessionLiveness, loadLanesConfig, laneRunsHere, MAX_LANE_DEPTH, MONITOR_ERROR_EXIT_CODE, MONITOR_ENGINES, MONITOR_MAX_TIMEOUT_MS, MONITOR_UNCOVERED_EXIT_CODE, MAX_WATCHED_LANES, monitorReminderText, readLiveMonitorEvidence, resolveMachineIdentity as lookupMachineIdentity, loadLivenessIndex, parseLaneProvider, parseLaneRecord, parseLaneState, parseLivenessState, parseQuestionRecord, parseQuestionStatus, parseSeniorKind, readDaemonPresenceConnected, readLane, retryUpdateLane, syncLaneRecordFromOwner, readQuestion, resolveAccount, resolveLaneWorker, ignoredWorkerProfileFields, loadWorkerProfile, runReconcilePass, wakeAnsweredLaneNow, runLaneWatch, sessionLiveness, tryReadLane, uninstallReconcileTimer, updateLane, updateQuestion, writeLanesConfig, appendDropboxMessage, answerDropboxMessage, deliveryRefusalReason, resumeCommandRefusal, dropboxDir, dropboxInboxPath, laneAcceptsDropboxDelivery, laneAcceptsOperatorMessage, laneDeliveryMechanism, hasActiveAdmissionReservation, hasActivePendingResume, hasRecordedSessionId, hasEarlierEligibleQueuedLane, laneAdmissionCapReason, providerSupportsSessionResume, providerSessionPortable, sessionNotPortableMessage, withLaneAdmissionLock, withAllLaneAdmissionLocks, withLaneHireLocks, laneIdForAnswerDelivery, operatorMessageRefusalReason, raiseLaneQuestion, sha256Text, writeQuestionBody, } from "../lib/lanes/index.js";
|
|
40
40
|
import { activeLaneStories, laneStories, laneStory, storyWarning, } from "../lib/lanes/stories.js";
|
|
41
41
|
import { resolveLaneKeepAlive } from "../lib/lanes/profile.js";
|
|
42
|
+
import { keepAliveExclusion, loadRetiredLaneIds, } from "../lib/lanes/keep-alive.js";
|
|
42
43
|
import { questionBodyAbsPath } from "../lib/lanes/questions.js";
|
|
43
44
|
import { compactionAfterUsage, } from "../lib/lanes/compaction.js";
|
|
44
45
|
import { assertUniqueAccountHomes, providerCompactAfterMs } from "../lib/lanes/config.js";
|
|
@@ -2935,6 +2936,7 @@ async function runCreate(ctx, opts) {
|
|
|
2935
2936
|
}
|
|
2936
2937
|
: {}),
|
|
2937
2938
|
keep_alive: keepAlive,
|
|
2939
|
+
...(keepAlive ? { keep_alive_opted_in_at: createdAt } : {}),
|
|
2938
2940
|
provider,
|
|
2939
2941
|
account: pinnedAccount,
|
|
2940
2942
|
...(accountSelection !== undefined ? { account_selection: accountSelection } : {}),
|
|
@@ -4493,15 +4495,7 @@ async function runShow(ctx, laneId) {
|
|
|
4493
4495
|
? { ...storedLane, state: "remote", remote_state: storedLane.state, queued_reason: undefined }
|
|
4494
4496
|
: storedLane;
|
|
4495
4497
|
const askDeliveryFailed = readJsonFile(askDeliveryFailedPath(hqRoot, laneId));
|
|
4496
|
-
|
|
4497
|
-
try {
|
|
4498
|
-
keepAliveState = resolveLaneKeepAlive(hqRoot, storedLane);
|
|
4499
|
-
}
|
|
4500
|
-
catch (err) {
|
|
4501
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
4502
|
-
process.stderr.write(`hq lanes show: keep-alive profile could not be loaded for ${laneId}: ${message}\n`);
|
|
4503
|
-
keepAliveState = { enabled: false, source: "none" };
|
|
4504
|
-
}
|
|
4498
|
+
const keepAliveState = resolveLaneKeepAlive(hqRoot, storedLane);
|
|
4505
4499
|
printJson({
|
|
4506
4500
|
ok: true,
|
|
4507
4501
|
action: "show",
|
|
@@ -4529,6 +4523,9 @@ async function runShow(ctx, laneId) {
|
|
|
4529
4523
|
keep_alive_state: {
|
|
4530
4524
|
enabled: keepAliveState.enabled,
|
|
4531
4525
|
source: keepAliveState.source,
|
|
4526
|
+
opted_in_at: storedLane.keep_alive_opted_in_at ?? null,
|
|
4527
|
+
blocked_reason: storedLane.keep_alive_blocked_reason ??
|
|
4528
|
+
(storedLane.keep_alive === true ? null : "not_opted_in"),
|
|
4532
4529
|
consecutive_auto_resumes: lane.keep_alive_resume_count ?? 0,
|
|
4533
4530
|
last_auto_resume_at: lane.keep_alive_last_resume_at ?? null,
|
|
4534
4531
|
next_eligible_at: lane.keep_alive_next_eligible_at ?? null,
|
|
@@ -4658,6 +4655,45 @@ function printGrantResult(hqRoot, updated, hierarchy) {
|
|
|
4658
4655
|
const fields = laneHireFields(updated, hierarchy ?? loadAllLanes(hqRoot));
|
|
4659
4656
|
printJson({ ok: true, action: "grant", lane_id: updated.lane_id, lane: updated, ...fields });
|
|
4660
4657
|
}
|
|
4658
|
+
async function runKeepAlive(ctx, laneId, opts) {
|
|
4659
|
+
if (Boolean(opts.on) === Boolean(opts.off)) {
|
|
4660
|
+
throw new LanesError("keep_alive_mode_invalid", "Pass exactly one of --on or --off", { lane_id: laneId });
|
|
4661
|
+
}
|
|
4662
|
+
const hqRoot = ctx.hqRoot();
|
|
4663
|
+
const updated = await withAllLaneAdmissionLocks(hqRoot, async () => {
|
|
4664
|
+
const lane = readLane(hqRoot, laneId);
|
|
4665
|
+
if (!callerMatchesSenior(ctx, lane.senior)) {
|
|
4666
|
+
throw new LanesError("keep_alive_unauthorized", `Only senior ${seniorKey(lane.senior)} may change lane ${laneId}'s keep-alive setting`, { lane_id: laneId, senior: lane.senior });
|
|
4667
|
+
}
|
|
4668
|
+
if (!laneRunsHere(lane, reconcileHostFromCtx(ctx))) {
|
|
4669
|
+
throw new LanesError("keep_alive_owner_host_required", `Run hq lanes keep-alive ${laneId} on its owner host ${lane.host.kind}:${lane.host.id}`, { lane_id: laneId, host: lane.host });
|
|
4670
|
+
}
|
|
4671
|
+
if (lane.state === "killed" || lane.state === "cancelled" || lane.state === "failed") {
|
|
4672
|
+
throw new LanesError("keep_alive_terminal_lane", `Cannot change keep-alive for lane ${laneId} in terminal state ${lane.state}`, { lane_id: laneId, state: lane.state });
|
|
4673
|
+
}
|
|
4674
|
+
let optedInAt;
|
|
4675
|
+
if (opts.on) {
|
|
4676
|
+
const retiredLaneIds = loadRetiredLaneIds(hqRoot);
|
|
4677
|
+
const exclusion = keepAliveExclusion(lane, loadAllLanes(hqRoot), retiredLaneIds);
|
|
4678
|
+
if (exclusion !== undefined) {
|
|
4679
|
+
throw new LanesError("keep_alive_lane_excluded", `Cannot opt in lane ${laneId}: it is ${exclusion}`, { lane_id: laneId, blocked_reason: exclusion });
|
|
4680
|
+
}
|
|
4681
|
+
optedInAt = ctx.now().toISOString();
|
|
4682
|
+
}
|
|
4683
|
+
return updateLane(hqRoot, laneId, lane.version, "command", (current) => ({
|
|
4684
|
+
...current,
|
|
4685
|
+
keep_alive: Boolean(opts.on),
|
|
4686
|
+
keep_alive_opted_in_at: optedInAt,
|
|
4687
|
+
keep_alive_blocked_reason: undefined,
|
|
4688
|
+
}));
|
|
4689
|
+
});
|
|
4690
|
+
printJson({
|
|
4691
|
+
ok: true,
|
|
4692
|
+
action: "keep-alive",
|
|
4693
|
+
lane_id: updated.lane_id,
|
|
4694
|
+
lane: updated,
|
|
4695
|
+
});
|
|
4696
|
+
}
|
|
4661
4697
|
async function runGrant(ctx, laneId, opts) {
|
|
4662
4698
|
if (opts.hire === undefined) {
|
|
4663
4699
|
throw new LanesError("missing_hire_grant", "--hire is required", {});
|
|
@@ -4950,6 +4986,10 @@ function applyResumeProfileAndEffort(current, next, input) {
|
|
|
4950
4986
|
next.worker_policy_delivery = undefined;
|
|
4951
4987
|
if (profile.meta.keep_alive !== undefined) {
|
|
4952
4988
|
next.keep_alive = profile.meta.keep_alive;
|
|
4989
|
+
next.keep_alive_opted_in_at = profile.meta.keep_alive
|
|
4990
|
+
? input.at ?? new Date().toISOString()
|
|
4991
|
+
: undefined;
|
|
4992
|
+
next.keep_alive_blocked_reason = undefined;
|
|
4953
4993
|
}
|
|
4954
4994
|
if (next.model_source !== "cli") {
|
|
4955
4995
|
if (profile.meta.provider === current.provider && profile.meta.model !== undefined) {
|
|
@@ -4966,8 +5006,13 @@ function applyResumeProfileAndEffort(current, next, input) {
|
|
|
4966
5006
|
next.effort = input.effort.effort;
|
|
4967
5007
|
next.effort_source = input.effort.source;
|
|
4968
5008
|
}
|
|
4969
|
-
if (input.keepAlive !== undefined)
|
|
5009
|
+
if (input.keepAlive !== undefined) {
|
|
4970
5010
|
next.keep_alive = input.keepAlive;
|
|
5011
|
+
next.keep_alive_opted_in_at = input.keepAlive
|
|
5012
|
+
? input.at ?? new Date().toISOString()
|
|
5013
|
+
: undefined;
|
|
5014
|
+
next.keep_alive_blocked_reason = undefined;
|
|
5015
|
+
}
|
|
4971
5016
|
}
|
|
4972
5017
|
function recordResumeSetup(ctx, lane, input) {
|
|
4973
5018
|
if (input.cliModel !== undefined) {
|
|
@@ -4998,7 +5043,7 @@ function recordResumeSetup(ctx, lane, input) {
|
|
|
4998
5043
|
throw new LanesError("lane_resume_setup_conflict", `Lane ${lane.lane_id} changed while resume settings were prepared; run the resume again`, { lane_id: lane.lane_id, prepared_version: lane.version, version: current.version });
|
|
4999
5044
|
}
|
|
5000
5045
|
const next = { ...current };
|
|
5001
|
-
applyResumeProfileAndEffort(current, next, input);
|
|
5046
|
+
applyResumeProfileAndEffort(current, next, { ...input, at: ctx.now().toISOString() });
|
|
5002
5047
|
if (input.cliModel !== undefined) {
|
|
5003
5048
|
next.model = input.cliModel.trim();
|
|
5004
5049
|
next.model_source = "cli";
|
|
@@ -5237,6 +5282,7 @@ async function prepareAccountMove(ctx, lane, input) {
|
|
|
5237
5282
|
applyResumeProfileAndEffort(current, next, {
|
|
5238
5283
|
...(input.workerProfile !== undefined ? { profile: input.workerProfile } : {}),
|
|
5239
5284
|
...(input.effort !== undefined ? { effort: input.effort } : {}),
|
|
5285
|
+
at,
|
|
5240
5286
|
});
|
|
5241
5287
|
if (input.deferTurnStart) {
|
|
5242
5288
|
next.state = current.state;
|
|
@@ -8388,6 +8434,16 @@ export function registerLanesCommand(parent, deps = {}) {
|
|
|
8388
8434
|
.action(wrap(async (laneId, opts) => {
|
|
8389
8435
|
await runGrantOwnerTarget(ctx, laneId, opts);
|
|
8390
8436
|
}));
|
|
8437
|
+
lanes
|
|
8438
|
+
.command("keep-alive")
|
|
8439
|
+
.description("Opt a lane into or out of automatic lead-lane resumes")
|
|
8440
|
+
.argument("<lane-id>", "Lane id")
|
|
8441
|
+
.option("--on", "Enable explicit keep-alive for this lane")
|
|
8442
|
+
.option("--off", "Disable keep-alive for this lane")
|
|
8443
|
+
.option("--json", "Emit JSON on stdout (default)")
|
|
8444
|
+
.action(wrap(async (laneId, opts) => {
|
|
8445
|
+
await runKeepAlive(ctx, laneId, opts);
|
|
8446
|
+
}));
|
|
8391
8447
|
lanes
|
|
8392
8448
|
.command("grant")
|
|
8393
8449
|
.description("Set a lane's alive child-lane grant as its senior")
|
|
@@ -64,6 +64,8 @@ export interface SyncManifestUploadStatus {
|
|
|
64
64
|
}
|
|
65
65
|
/** Injectable dependencies — defaults are the real collectors. */
|
|
66
66
|
export interface SyncHealthDeps {
|
|
67
|
+
/** Runner log and conflict signals; omitted in unit tests of other checks. */
|
|
68
|
+
runtime?: (now: Date) => CheckResult[];
|
|
67
69
|
versions: (hqRoot: string) => SyncVersionInfo;
|
|
68
70
|
journals: () => readonly SyncJournalSummary[];
|
|
69
71
|
now: () => Date;
|
|
@@ -23,6 +23,7 @@
|
|
|
23
23
|
* Every dependency is injectable so the family is unit-testable without an HQ
|
|
24
24
|
* tree, a state dir, or the wall clock.
|
|
25
25
|
*/
|
|
26
|
+
import { defaultSyncRuntimeResults } from "./sync-runtime.js";
|
|
26
27
|
import * as fs from "node:fs";
|
|
27
28
|
import * as path from "node:path";
|
|
28
29
|
// Namespace import, deliberately: hq-cloud is an ESM package, so a static
|
|
@@ -50,6 +51,7 @@ export const UPDATE_CACHE_RELPATH = path.join("workspace", ".hq-update-check", "
|
|
|
50
51
|
function withDefaults(deps, hqRoot) {
|
|
51
52
|
return {
|
|
52
53
|
...deps,
|
|
54
|
+
runtime: deps.runtime ?? (() => []),
|
|
53
55
|
manifestStatuses: deps.manifestStatuses ?? defaultManifestStatuses,
|
|
54
56
|
manifestAvailable: deps.manifestAvailable ?? (() => loadManifestExports() !== null),
|
|
55
57
|
unresolvedScopes: deps.unresolvedScopes ?? defaultUnresolvedScopes,
|
|
@@ -118,6 +120,7 @@ const DEFAULT_DEPS = {
|
|
|
118
120
|
journals: () => hqCloud.listJournals(),
|
|
119
121
|
now: () => new Date(),
|
|
120
122
|
manifestStatuses: defaultManifestStatuses,
|
|
123
|
+
runtime: defaultSyncRuntimeResults,
|
|
121
124
|
};
|
|
122
125
|
/**
|
|
123
126
|
* Map the machine's journal shards onto the manifest scopes hq-cloud keys its
|
|
@@ -190,6 +193,7 @@ export function checkSyncHealth(context, rawDeps = DEFAULT_DEPS) {
|
|
|
190
193
|
...versionResults(context, deps),
|
|
191
194
|
...updateAvailabilityResult(context, deps),
|
|
192
195
|
...journalAndManifestResults(deps),
|
|
196
|
+
...deps.runtime(deps.now()),
|
|
193
197
|
];
|
|
194
198
|
}
|
|
195
199
|
catch (error) {
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { CheckResult } from "../types.js";
|
|
2
|
+
export declare const SYNC_RUNTIME_WINDOW_MS: number;
|
|
3
|
+
export interface SyncRuntimeOptions {
|
|
4
|
+
now: number;
|
|
5
|
+
conflictCount?: number;
|
|
6
|
+
}
|
|
7
|
+
/** Classify the last hour of sync log lines into privacy-safe results. */
|
|
8
|
+
export declare function analyzeSyncRuntimeLog(lines: readonly string[], options: SyncRuntimeOptions): CheckResult[];
|
|
9
|
+
/** Read the machine's sync logs and client health file. */
|
|
10
|
+
export declare function defaultSyncRuntimeResults(now: Date, hqHome?: string): CheckResult[];
|
|
11
|
+
//# sourceMappingURL=sync-runtime.d.ts.map
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Sync runtime checks: problems visible only in the running sync runner's log
|
|
3
|
+
* (realtime lag, receiver permission errors, restart loops, device-limit
|
|
4
|
+
* backoff) plus the conflict counter the desktop app keeps in client health.
|
|
5
|
+
*
|
|
6
|
+
* Results carry only a closed reason code and counts. Log content (paths,
|
|
7
|
+
* tenant ids, AWS error prose) never leaves this module.
|
|
8
|
+
*/
|
|
9
|
+
import * as fs from "node:fs";
|
|
10
|
+
import * as os from "node:os";
|
|
11
|
+
import * as path from "node:path";
|
|
12
|
+
export const SYNC_RUNTIME_WINDOW_MS = 60 * 60_000;
|
|
13
|
+
const LAG_SECONDS = 15 * 60;
|
|
14
|
+
const LAG_MIN_RECEIPTS = 3;
|
|
15
|
+
const RESTART_MIN_STARTS = 4;
|
|
16
|
+
const CONFLICT_MIN_COUNT = 5;
|
|
17
|
+
const LOG_TAIL_BYTES = 8 * 1024 * 1024;
|
|
18
|
+
const TIMESTAMP = /^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z)\s/;
|
|
19
|
+
const PERMISSION = /AccessDenied|not authorized to perform|\b403\b/;
|
|
20
|
+
const DEVICE_LIMIT = /sync device limit exceeded|"sync_device_limit"/;
|
|
21
|
+
function parseRecord(rest) {
|
|
22
|
+
const start = rest.indexOf("{");
|
|
23
|
+
if (start < 0)
|
|
24
|
+
return null;
|
|
25
|
+
try {
|
|
26
|
+
const value = JSON.parse(rest.slice(start));
|
|
27
|
+
return value !== null && typeof value === "object" && !Array.isArray(value)
|
|
28
|
+
? value
|
|
29
|
+
: null;
|
|
30
|
+
}
|
|
31
|
+
catch {
|
|
32
|
+
return null;
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
/** Classify the last hour of sync log lines into privacy-safe results. */
|
|
36
|
+
export function analyzeSyncRuntimeLog(lines, options) {
|
|
37
|
+
const since = options.now - SYNC_RUNTIME_WINDOW_MS;
|
|
38
|
+
let slowReceipts = 0;
|
|
39
|
+
let denied = 0;
|
|
40
|
+
let starts = 0;
|
|
41
|
+
let deviceLimited = 0;
|
|
42
|
+
for (const line of lines) {
|
|
43
|
+
const stamp = TIMESTAMP.exec(line);
|
|
44
|
+
if (!stamp)
|
|
45
|
+
continue;
|
|
46
|
+
const time = Date.parse(stamp[1]);
|
|
47
|
+
if (!Number.isFinite(time) || time < since || time > options.now)
|
|
48
|
+
continue;
|
|
49
|
+
const rest = line.slice(stamp[0].length);
|
|
50
|
+
if (DEVICE_LIMIT.test(rest))
|
|
51
|
+
deviceLimited += 1;
|
|
52
|
+
const record = parseRecord(rest);
|
|
53
|
+
if (!record)
|
|
54
|
+
continue;
|
|
55
|
+
const event = record.event;
|
|
56
|
+
if (event === "fanout.receive") {
|
|
57
|
+
const latency = record.latencySeconds;
|
|
58
|
+
if (typeof latency === "number" && latency > LAG_SECONDS)
|
|
59
|
+
slowReceipts += 1;
|
|
60
|
+
}
|
|
61
|
+
else if (event === "receiver.start") {
|
|
62
|
+
starts += 1;
|
|
63
|
+
}
|
|
64
|
+
if ((record.type === "receiver" || record.type === "error") &&
|
|
65
|
+
PERMISSION.test(JSON.stringify(record.err ?? record.message ?? ""))) {
|
|
66
|
+
denied += 1;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
const conflicts = options.conflictCount ?? 0;
|
|
70
|
+
const checks = [
|
|
71
|
+
{ id: "realtime-lag", reasonCode: "realtime-lag-stuck", failing: slowReceipts >= LAG_MIN_RECEIPTS, count: slowReceipts, label: "incoming changes delayed over 15 minutes" },
|
|
72
|
+
{ id: "receiver-permission", reasonCode: "receiver-permission-denied", failing: denied > 0, count: denied, label: "cloud permission refusals" },
|
|
73
|
+
{ id: "runner-restarts", reasonCode: "runner-restart-loop", failing: starts >= RESTART_MIN_STARTS, count: starts, label: "sync runner starts" },
|
|
74
|
+
{ id: "device-limit", reasonCode: "device-limit-backoff", failing: deviceLimited > 0, count: deviceLimited, label: "device-limit backoffs" },
|
|
75
|
+
{ id: "conflicts", reasonCode: "repeated-conflicts", failing: conflicts >= CONFLICT_MIN_COUNT, count: conflicts, label: "unresolved conflicts" },
|
|
76
|
+
];
|
|
77
|
+
return checks.map((check) => check.failing
|
|
78
|
+
? {
|
|
79
|
+
status: "WARN",
|
|
80
|
+
checkId: `sync.runtime.${check.id}`,
|
|
81
|
+
reasonCode: check.reasonCode,
|
|
82
|
+
message: `${check.count} ${check.label} in the last hour.`,
|
|
83
|
+
remediation: "Run `hq sync status` and check ~/.hq/logs/hq-sync.log.",
|
|
84
|
+
}
|
|
85
|
+
: {
|
|
86
|
+
status: "PASS",
|
|
87
|
+
checkId: `sync.runtime.${check.id}`,
|
|
88
|
+
message: `${check.count} ${check.label} in the last hour.`,
|
|
89
|
+
});
|
|
90
|
+
}
|
|
91
|
+
const LOG_GENERATIONS = 4;
|
|
92
|
+
function readLogTail(file) {
|
|
93
|
+
const fd = fs.openSync(file, "r");
|
|
94
|
+
try {
|
|
95
|
+
const size = fs.fstatSync(fd).size;
|
|
96
|
+
const length = Math.min(size, LOG_TAIL_BYTES);
|
|
97
|
+
const buffer = Buffer.alloc(length);
|
|
98
|
+
fs.readSync(fd, buffer, 0, length, size - length);
|
|
99
|
+
return buffer.toString("utf8").split("\n");
|
|
100
|
+
}
|
|
101
|
+
finally {
|
|
102
|
+
fs.closeSync(fd);
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
function firstTimestamp(lines) {
|
|
106
|
+
for (const line of lines) {
|
|
107
|
+
const stamp = TIMESTAMP.exec(line);
|
|
108
|
+
if (stamp)
|
|
109
|
+
return Date.parse(stamp[1]);
|
|
110
|
+
}
|
|
111
|
+
return null;
|
|
112
|
+
}
|
|
113
|
+
/**
|
|
114
|
+
* Lines from the active log and as many rotated generations as needed to
|
|
115
|
+
* cover the window. Throws when a present log cannot be read.
|
|
116
|
+
*/
|
|
117
|
+
function readWindowLines(logFile, since) {
|
|
118
|
+
const generations = [];
|
|
119
|
+
for (let n = 0; n < LOG_GENERATIONS; n += 1) {
|
|
120
|
+
const file = n === 0 ? logFile : `${logFile}.${n}`;
|
|
121
|
+
if (!fs.existsSync(file))
|
|
122
|
+
break;
|
|
123
|
+
const lines = readLogTail(file);
|
|
124
|
+
generations.unshift(lines);
|
|
125
|
+
const first = firstTimestamp(lines);
|
|
126
|
+
if (first !== null && first <= since)
|
|
127
|
+
break;
|
|
128
|
+
}
|
|
129
|
+
return generations.length > 0 ? generations.flat() : null;
|
|
130
|
+
}
|
|
131
|
+
function readConflictCount(file) {
|
|
132
|
+
try {
|
|
133
|
+
const value = JSON.parse(fs.readFileSync(file, "utf8"));
|
|
134
|
+
const count = value?.conflictCount;
|
|
135
|
+
return typeof count === "number" && Number.isFinite(count) ? count : 0;
|
|
136
|
+
}
|
|
137
|
+
catch {
|
|
138
|
+
return 0;
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
/** Read the machine's sync logs and client health file. */
|
|
142
|
+
export function defaultSyncRuntimeResults(now, hqHome = path.join(os.homedir(), ".hq")) {
|
|
143
|
+
const options = {
|
|
144
|
+
now: now.getTime(),
|
|
145
|
+
conflictCount: readConflictCount(path.join(hqHome, "client-health.json")),
|
|
146
|
+
};
|
|
147
|
+
const logFile = path.join(hqHome, "logs", "hq-sync.log");
|
|
148
|
+
let lines;
|
|
149
|
+
try {
|
|
150
|
+
lines = readWindowLines(logFile, options.now - SYNC_RUNTIME_WINDOW_MS);
|
|
151
|
+
}
|
|
152
|
+
catch {
|
|
153
|
+
return [
|
|
154
|
+
{ status: "UNKNOWN", checkId: "sync.runtime.log", message: "The sync runner log could not be read." },
|
|
155
|
+
...analyzeSyncRuntimeLog([], options).filter((r) => r.checkId === "sync.runtime.conflicts"),
|
|
156
|
+
];
|
|
157
|
+
}
|
|
158
|
+
if (lines === null) {
|
|
159
|
+
return [
|
|
160
|
+
{ status: "NA", checkId: "sync.runtime.log", message: "No sync runner log on this machine." },
|
|
161
|
+
...analyzeSyncRuntimeLog([], options).filter((r) => r.checkId === "sync.runtime.conflicts"),
|
|
162
|
+
];
|
|
163
|
+
}
|
|
164
|
+
return analyzeSyncRuntimeLog(lines, options);
|
|
165
|
+
}
|
|
166
|
+
//# sourceMappingURL=sync-runtime.js.map
|
|
@@ -5,7 +5,7 @@ export declare const CLIENT_HEALTH_INSTALL_ID_FILENAME = "install-id";
|
|
|
5
5
|
export declare const CLIENT_HEALTH_LAST_RESULT_FILENAME = "last-result.json";
|
|
6
6
|
export type ClientHealthFixOutcome = "not_auto_fixable" | "fix_failed" | "fix_timed_out";
|
|
7
7
|
export type ClientHealthStatus = "FAIL" | "WARN";
|
|
8
|
-
export type ClientHealthCheckClass = "sync.update.core" | "sync.journal.personal" | "sync.journal.personal_vault" | "sync.journal.company" | "sync.manifest.personal" | "sync.manifest.personal_vault" | "sync.manifest.company" | "other";
|
|
8
|
+
export type ClientHealthCheckClass = "sync.update.core" | "sync.journal.personal" | "sync.journal.personal_vault" | "sync.journal.company" | "sync.manifest.personal" | "sync.manifest.personal_vault" | "sync.manifest.company" | "sync.runtime" | "other";
|
|
9
9
|
export interface ClientHealthSentryEvent {
|
|
10
10
|
message: string;
|
|
11
11
|
level: "warning";
|
|
@@ -24,6 +24,11 @@ const REASON_CODES = new Set([
|
|
|
24
24
|
"never-uploaded",
|
|
25
25
|
"invalid-upload-time",
|
|
26
26
|
"unresolved-company-uid",
|
|
27
|
+
"realtime-lag-stuck",
|
|
28
|
+
"receiver-permission-denied",
|
|
29
|
+
"runner-restart-loop",
|
|
30
|
+
"device-limit-backoff",
|
|
31
|
+
"repeated-conflicts",
|
|
27
32
|
]);
|
|
28
33
|
const FIX_OUTCOMES = new Set(["not_auto_fixable", "fix_failed", "fix_timed_out"]);
|
|
29
34
|
const INSTALL_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
|
|
@@ -52,6 +57,8 @@ function cleanArchitecture(value) {
|
|
|
52
57
|
export function normalizeClientHealthCheckId(checkId) {
|
|
53
58
|
if (checkId === "sync.update.core")
|
|
54
59
|
return checkId;
|
|
60
|
+
if (checkId === "sync.runtime" || checkId.startsWith("sync.runtime."))
|
|
61
|
+
return "sync.runtime";
|
|
55
62
|
if (checkId === "sync.journal.__hq_personal_vault__")
|
|
56
63
|
return "sync.journal.personal_vault";
|
|
57
64
|
if (checkId === "sync.manifest.__hq_personal_vault__")
|
|
@@ -19,7 +19,7 @@ export declare const DOCTOR_STATUSES: readonly ["PASS", "FAIL", "WARN", "UNTESTE
|
|
|
19
19
|
*/
|
|
20
20
|
export type DoctorStatus = (typeof DOCTOR_STATUSES)[number];
|
|
21
21
|
/** Stable, privacy-safe diagnostics emitted by the sync health family. */
|
|
22
|
-
export declare const SYNC_REASON_CODES: readonly ["core-version-unavailable", "core-update-available", "no-local-tree", "never-synced", "invalid-last-sync", "stale-threshold", "manifest-unavailable", "manifest-state-unavailable", "no-manifest-scopes", "never-uploaded", "invalid-upload-time", "unresolved-company-uid"];
|
|
22
|
+
export declare const SYNC_REASON_CODES: readonly ["core-version-unavailable", "core-update-available", "no-local-tree", "never-synced", "invalid-last-sync", "stale-threshold", "manifest-unavailable", "manifest-state-unavailable", "no-manifest-scopes", "never-uploaded", "invalid-upload-time", "unresolved-company-uid", "realtime-lag-stuck", "receiver-permission-denied", "runner-restart-loop", "device-limit-backoff", "repeated-conflicts"];
|
|
23
23
|
export type SyncReasonCode = (typeof SYNC_REASON_CODES)[number];
|
|
24
24
|
/** Runtime type guard: true only for a member of the fixed vocabulary. */
|
|
25
25
|
export declare function isDoctorStatus(value: unknown): value is DoctorStatus;
|
package/dist/lib/doctor/types.js
CHANGED
|
@@ -35,6 +35,11 @@ export const SYNC_REASON_CODES = [
|
|
|
35
35
|
"never-uploaded",
|
|
36
36
|
"invalid-upload-time",
|
|
37
37
|
"unresolved-company-uid",
|
|
38
|
+
"realtime-lag-stuck",
|
|
39
|
+
"receiver-permission-denied",
|
|
40
|
+
"runner-restart-loop",
|
|
41
|
+
"device-limit-backoff",
|
|
42
|
+
"repeated-conflicts",
|
|
38
43
|
];
|
|
39
44
|
/** Runtime type guard: true only for a member of the fixed vocabulary. */
|
|
40
45
|
export function isDoctorStatus(value) {
|
|
@@ -1,8 +1,17 @@
|
|
|
1
|
+
import type { LaneRecord } from "./types.js";
|
|
1
2
|
export declare const LEAD_LANE_KEEP_ALIVE_FLAG = "lanes.lead-keep-alive";
|
|
2
3
|
export declare const LEAD_LANE_KEEP_ALIVE_DEFAULT_ENABLED = false;
|
|
3
4
|
export declare const LEAD_LANE_KEEP_ALIVE_MAX_RESUMES = 24;
|
|
4
5
|
export declare const LEAD_LANE_KEEP_ALIVE_BASE_BACKOFF_MS: number;
|
|
5
6
|
export declare const LEAD_LANE_KEEP_ALIVE_MAX_BACKOFF_MS: number;
|
|
6
7
|
export declare const LEAD_LANE_KEEP_ALIVE_RESUME_TEXT = "Automatic keep-alive resume: your last turn ended with decision done and story_complete false. Continue the open work from where you stopped. End with done only when story_complete is true; if you are waiting, use hq lanes ask or stay in the loop.";
|
|
8
|
+
export type KeepAliveBlockedReason = NonNullable<LaneRecord["keep_alive_blocked_reason"]>;
|
|
9
|
+
export declare function retiredLaneRegistryPath(hqRoot: string): string;
|
|
10
|
+
/** Missing registry means no retired lanes; every present invalid file fails closed. */
|
|
11
|
+
export declare function loadRetiredLaneIds(hqRoot: string): ReadonlySet<string>;
|
|
12
|
+
/** IDs may be stored bare or with an owner-host suffix after an underscore. */
|
|
13
|
+
export declare function isRetiredLaneId(laneId: string, retiredLaneIds: ReadonlySet<string>): boolean;
|
|
14
|
+
export declare function keepAliveSupersededBy(lane: LaneRecord, lanes: readonly LaneRecord[]): LaneRecord | undefined;
|
|
15
|
+
export declare function keepAliveExclusion(lane: LaneRecord, lanes: readonly LaneRecord[], retiredLaneIds: ReadonlySet<string>): "retired" | "superseded" | undefined;
|
|
7
16
|
export declare function leadLaneKeepAliveBackoffMs(attempt: number): number;
|
|
8
17
|
//# sourceMappingURL=keep-alive.d.ts.map
|
|
@@ -1,9 +1,110 @@
|
|
|
1
|
+
import * as fs from "node:fs";
|
|
2
|
+
import * as path from "node:path";
|
|
3
|
+
import { LanesError } from "./errors.js";
|
|
1
4
|
export const LEAD_LANE_KEEP_ALIVE_FLAG = "lanes.lead-keep-alive";
|
|
2
5
|
export const LEAD_LANE_KEEP_ALIVE_DEFAULT_ENABLED = false;
|
|
3
6
|
export const LEAD_LANE_KEEP_ALIVE_MAX_RESUMES = 24;
|
|
4
7
|
export const LEAD_LANE_KEEP_ALIVE_BASE_BACKOFF_MS = 5 * 60 * 1000;
|
|
5
8
|
export const LEAD_LANE_KEEP_ALIVE_MAX_BACKOFF_MS = 60 * 60 * 1000;
|
|
6
9
|
export const LEAD_LANE_KEEP_ALIVE_RESUME_TEXT = "Automatic keep-alive resume: your last turn ended with decision done and story_complete false. Continue the open work from where you stopped. End with done only when story_complete is true; if you are waiting, use hq lanes ask or stay in the loop.";
|
|
10
|
+
export function retiredLaneRegistryPath(hqRoot) {
|
|
11
|
+
return path.join(hqRoot, "workspace", "lane-briefs", "juniors.json");
|
|
12
|
+
}
|
|
13
|
+
/** Missing registry means no retired lanes; every present invalid file fails closed. */
|
|
14
|
+
export function loadRetiredLaneIds(hqRoot) {
|
|
15
|
+
const filePath = retiredLaneRegistryPath(hqRoot);
|
|
16
|
+
let text;
|
|
17
|
+
try {
|
|
18
|
+
const stat = fs.lstatSync(filePath);
|
|
19
|
+
if (!stat.isFile() || stat.isSymbolicLink()) {
|
|
20
|
+
throw new Error("expected a regular file");
|
|
21
|
+
}
|
|
22
|
+
text = fs.readFileSync(filePath, "utf8");
|
|
23
|
+
}
|
|
24
|
+
catch (err) {
|
|
25
|
+
if (err.code === "ENOENT")
|
|
26
|
+
return new Set();
|
|
27
|
+
throw new LanesError("keep_alive_retired_list_invalid", `Cannot read retired lane list ${filePath}: ${err instanceof Error ? err.message : String(err)}`, { path: filePath });
|
|
28
|
+
}
|
|
29
|
+
let parsed;
|
|
30
|
+
try {
|
|
31
|
+
parsed = JSON.parse(text);
|
|
32
|
+
}
|
|
33
|
+
catch (err) {
|
|
34
|
+
throw new LanesError("keep_alive_retired_list_invalid", `Cannot parse retired lane list ${filePath}: ${err instanceof Error ? err.message : String(err)}`, { path: filePath });
|
|
35
|
+
}
|
|
36
|
+
if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) {
|
|
37
|
+
throw invalidRetiredList(filePath, "root must be an object");
|
|
38
|
+
}
|
|
39
|
+
const retired = parsed.retired;
|
|
40
|
+
if (retired === undefined)
|
|
41
|
+
return new Set();
|
|
42
|
+
if (!Array.isArray(retired))
|
|
43
|
+
throw invalidRetiredList(filePath, "retired must be an array");
|
|
44
|
+
const laneIds = new Set();
|
|
45
|
+
for (const [index, entry] of retired.entries()) {
|
|
46
|
+
if (entry === null || typeof entry !== "object" || Array.isArray(entry)) {
|
|
47
|
+
throw invalidRetiredList(filePath, `retired[${index}] must be an object with a lane id`);
|
|
48
|
+
}
|
|
49
|
+
const laneId = entry.lane;
|
|
50
|
+
if (typeof laneId !== "string" || laneId.trim() === "") {
|
|
51
|
+
throw invalidRetiredList(filePath, `retired[${index}].lane must be a non-empty string`);
|
|
52
|
+
}
|
|
53
|
+
laneIds.add(laneId.trim());
|
|
54
|
+
}
|
|
55
|
+
return laneIds;
|
|
56
|
+
}
|
|
57
|
+
function invalidRetiredList(filePath, message) {
|
|
58
|
+
return new LanesError("keep_alive_retired_list_invalid", `Invalid retired lane list ${filePath}: ${message}`, { path: filePath });
|
|
59
|
+
}
|
|
60
|
+
/** IDs may be stored bare or with an owner-host suffix after an underscore. */
|
|
61
|
+
export function isRetiredLaneId(laneId, retiredLaneIds) {
|
|
62
|
+
for (const retiredId of retiredLaneIds) {
|
|
63
|
+
if (retiredId === laneId ||
|
|
64
|
+
retiredId.startsWith(`${laneId}_`) ||
|
|
65
|
+
laneId.startsWith(`${retiredId}_`))
|
|
66
|
+
return true;
|
|
67
|
+
}
|
|
68
|
+
return false;
|
|
69
|
+
}
|
|
70
|
+
export function keepAliveSupersededBy(lane, lanes) {
|
|
71
|
+
const createdAt = Date.parse(lane.created_at);
|
|
72
|
+
if (!Number.isFinite(createdAt))
|
|
73
|
+
return undefined;
|
|
74
|
+
const lanesById = new Map(lanes.map((candidate) => [candidate.lane_id, candidate]));
|
|
75
|
+
return lanes.find((candidate) => {
|
|
76
|
+
if (candidate.lane_id === lane.lane_id || Date.parse(candidate.created_at) <= createdAt)
|
|
77
|
+
return false;
|
|
78
|
+
const sameCompany = lane.company === null
|
|
79
|
+
? candidate.company === null
|
|
80
|
+
: candidate.company?.uid === lane.company.uid;
|
|
81
|
+
if (!sameCompany)
|
|
82
|
+
return false;
|
|
83
|
+
// Any descendant is part of this lead's work tree, not a replacement lead.
|
|
84
|
+
let parentLaneId = candidate.senior.kind === "lane" ? candidate.senior.id : undefined;
|
|
85
|
+
const visitedLaneIds = new Set();
|
|
86
|
+
while (parentLaneId) {
|
|
87
|
+
if (parentLaneId === lane.lane_id)
|
|
88
|
+
return false;
|
|
89
|
+
if (visitedLaneIds.has(parentLaneId))
|
|
90
|
+
break;
|
|
91
|
+
visitedLaneIds.add(parentLaneId);
|
|
92
|
+
const parent = lanesById.get(parentLaneId);
|
|
93
|
+
parentLaneId = parent?.senior.kind === "lane" ? parent.senior.id : undefined;
|
|
94
|
+
}
|
|
95
|
+
const sameProfile = lane.worker_profile !== undefined &&
|
|
96
|
+
candidate.worker_profile === lane.worker_profile;
|
|
97
|
+
const projectLead = lane.project_id !== undefined &&
|
|
98
|
+
candidate.project_id === lane.project_id &&
|
|
99
|
+
(candidate.hire_grant ?? 0) > 0;
|
|
100
|
+
return sameProfile || projectLead;
|
|
101
|
+
});
|
|
102
|
+
}
|
|
103
|
+
export function keepAliveExclusion(lane, lanes, retiredLaneIds) {
|
|
104
|
+
if (isRetiredLaneId(lane.lane_id, retiredLaneIds))
|
|
105
|
+
return "retired";
|
|
106
|
+
return keepAliveSupersededBy(lane, lanes) === undefined ? undefined : "superseded";
|
|
107
|
+
}
|
|
7
108
|
export function leadLaneKeepAliveBackoffMs(attempt) {
|
|
8
109
|
if (!Number.isSafeInteger(attempt) || attempt < 1) {
|
|
9
110
|
throw new RangeError("keep-alive attempt must be a positive safe integer");
|
|
@@ -39,10 +39,10 @@ export declare function workerProfileMarker(hqRoot: string, id: string): "absent
|
|
|
39
39
|
export declare function loadWorkerProfile(hqRoot: string, id: string): WorkerProfile;
|
|
40
40
|
export interface EffectiveLaneKeepAlive {
|
|
41
41
|
enabled: boolean;
|
|
42
|
-
source: "lane" | "
|
|
42
|
+
source: "lane" | "none";
|
|
43
43
|
}
|
|
44
|
-
/** Resolve
|
|
45
|
-
export declare function resolveLaneKeepAlive(
|
|
44
|
+
/** Resolve keep-alive from the lane record; profile metadata only seeds new spawns. */
|
|
45
|
+
export declare function resolveLaneKeepAlive(_hqRoot: string, lane: Pick<LaneRecord, "keep_alive">): EffectiveLaneKeepAlive;
|
|
46
46
|
export declare function workerProfilePrompt(profile: WorkerProfile, delivery?: "hook" | "prompt"): string;
|
|
47
47
|
export declare function ignoredWorkerProfileFields(provider: LaneProvider): string[];
|
|
48
48
|
/**
|
|
@@ -370,18 +370,11 @@ export function loadWorkerProfile(hqRoot, id) {
|
|
|
370
370
|
contentHash: hashProfileFiles(dir, hashFiles),
|
|
371
371
|
};
|
|
372
372
|
}
|
|
373
|
-
/** Resolve
|
|
374
|
-
export function resolveLaneKeepAlive(
|
|
375
|
-
|
|
376
|
-
return { enabled: lane.keep_alive, source: "lane" };
|
|
377
|
-
}
|
|
378
|
-
if (lane.worker_profile === undefined) {
|
|
379
|
-
return { enabled: false, source: "none" };
|
|
380
|
-
}
|
|
381
|
-
const profileKeepAlive = loadWorkerProfile(hqRoot, lane.worker_profile).meta.keep_alive;
|
|
382
|
-
return profileKeepAlive === undefined
|
|
373
|
+
/** Resolve keep-alive from the lane record; profile metadata only seeds new spawns. */
|
|
374
|
+
export function resolveLaneKeepAlive(_hqRoot, lane) {
|
|
375
|
+
return lane.keep_alive === undefined
|
|
383
376
|
? { enabled: false, source: "none" }
|
|
384
|
-
: { enabled:
|
|
377
|
+
: { enabled: lane.keep_alive, source: "lane" };
|
|
385
378
|
}
|
|
386
379
|
export function workerProfilePrompt(profile, delivery = "prompt") {
|
|
387
380
|
const parts = [profile.systemPrompt.trimEnd()];
|
|
@@ -14,14 +14,14 @@ import { effectiveAccountHome } from "./account-home.js";
|
|
|
14
14
|
import { loadLanesConfig, providerQuotaDefaultHoldMs } from "./config.js";
|
|
15
15
|
import { MAX_LANE_DEPTH } from "./constants.js";
|
|
16
16
|
import { AccountResolutionError, LanesError, isLaneVersionConflict } from "./errors.js";
|
|
17
|
+
import { loadWorkerProfile } from "./profile.js";
|
|
17
18
|
import { LANE_HIRE_GRANTS_FLAG, terminalChildResumeHireGrantError, withHireGrantAdmission, } from "./hire-grants.js";
|
|
18
|
-
import { loadWorkerProfile, resolveLaneKeepAlive } from "./profile.js";
|
|
19
19
|
import { classifySessionLiveness, defaultLivenessDeps, loadLivenessIndex, readDaemonPresenceConnected, unindexedClaudeSessionLiveness, } from "./liveness.js";
|
|
20
20
|
import { ANSWERED_QUESTIONS_FILE, armLaneWatcher, envelopeRepairMode, isPidRecordAlive, laneRunDir, readPidRecord, readSpawnManifest, markSpawnFailure, spawnEnvelopeRepair, spawnLaneWorker, retryLaneOnAlternateAccount, } from "./spawn.js";
|
|
21
21
|
import { appendCappedHistory, lanesStoreRoot, parseLaneRecord, parseQuestionRecord, readLane, readQuestion, updateLane, updateQuestion, } from "./store.js";
|
|
22
22
|
import { applyThrottleDecision, decideAdmission, defaultThrottleDeps, emptyThrottleState, evaluateThrottle, isCpuCheckEnabled, readAvailableMemMb, readThrottleState, resolveCpuBusyFraction, throttleStatePath, writeThrottleState, } from "./throttle.js";
|
|
23
23
|
import { readLanesRegistryFlag } from "./registry-flag.js";
|
|
24
|
-
import { LEAD_LANE_KEEP_ALIVE_FLAG, LEAD_LANE_KEEP_ALIVE_MAX_RESUMES, LEAD_LANE_KEEP_ALIVE_RESUME_TEXT, leadLaneKeepAliveBackoffMs, } from "./keep-alive.js";
|
|
24
|
+
import { LEAD_LANE_KEEP_ALIVE_FLAG, LEAD_LANE_KEEP_ALIVE_MAX_RESUMES, LEAD_LANE_KEEP_ALIVE_RESUME_TEXT, keepAliveExclusion, loadRetiredLaneIds, leadLaneKeepAliveBackoffMs, } from "./keep-alive.js";
|
|
25
25
|
import { providerSupportsSessionResume } from "./delivery.js";
|
|
26
26
|
import { applyDefaultQuotaHold, readQuotaSignal, recordAccountQuota } from "./quota.js";
|
|
27
27
|
import { answeredQuestionEntries, raiseEnvelopeQuestions } from "./questions.js";
|
|
@@ -800,18 +800,19 @@ export async function runPassBody(hqRoot, deps) {
|
|
|
800
800
|
const keepAliveFlagCompanies = new Map();
|
|
801
801
|
const keepAliveHireGrantFlagCompanies = new Map();
|
|
802
802
|
for (const lane of lanes) {
|
|
803
|
-
if (lane.state
|
|
804
|
-
(lane.keep_alive !== true &&
|
|
805
|
-
!(lane.keep_alive === undefined && lane.worker_profile !== undefined)) ||
|
|
803
|
+
if (!(lane.state === "done" || lane.state === "killed" || lane.state === "cancelled" || lane.state === "failed") ||
|
|
804
|
+
(lane.keep_alive !== true && lane.worker_profile === undefined) ||
|
|
806
805
|
!laneRunsHere(lane, deps.host) ||
|
|
807
|
-
lane.terminal_reason === PROVIDER_REJECTED_REASON ||
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
continue;
|
|
812
|
-
const envelope = readLatestEnvelopeRecord(hqRoot, lane.lane_id)?.envelope;
|
|
813
|
-
if (!envelope || envelope.decision !== "done" || envelope.story_complete === true)
|
|
806
|
+
(lane.state === "done" && (lane.terminal_reason === PROVIDER_REJECTED_REASON ||
|
|
807
|
+
lane.terminal_reason === "account_not_logged_in" ||
|
|
808
|
+
!providerSupportsSessionResume(lane.provider) ||
|
|
809
|
+
!(lane.provider_session_id ?? lane.last_resume?.session_id))))
|
|
814
810
|
continue;
|
|
811
|
+
if (lane.state === "done") {
|
|
812
|
+
const envelope = readLatestEnvelopeRecord(hqRoot, lane.lane_id)?.envelope;
|
|
813
|
+
if (!envelope || envelope.decision !== "done" || envelope.story_complete === true)
|
|
814
|
+
continue;
|
|
815
|
+
}
|
|
815
816
|
const key = keepAliveCompanyKey(lane.company);
|
|
816
817
|
keepAliveFlagCompanies.set(key, lane.company);
|
|
817
818
|
if (lane.senior.kind === "lane" && lane.senior.id) {
|
|
@@ -1413,80 +1414,164 @@ function recordKeepAliveEvent(hqRoot, laneId, summary, event) {
|
|
|
1413
1414
|
});
|
|
1414
1415
|
}
|
|
1415
1416
|
}
|
|
1417
|
+
function recordKeepAliveSkip(hqRoot, lane, reason, summary, deps, detail) {
|
|
1418
|
+
summary.skipped.push({
|
|
1419
|
+
kind: "lane",
|
|
1420
|
+
id: lane.lane_id,
|
|
1421
|
+
why: `keep_alive_${reason}${detail ? `: ${detail}` : ""}`,
|
|
1422
|
+
});
|
|
1423
|
+
if (deps.dryRun || lane.keep_alive_blocked_reason === reason)
|
|
1424
|
+
return;
|
|
1425
|
+
try {
|
|
1426
|
+
casLaneIf(hqRoot, lane.lane_id, (current) => current.version === lane.version && current.state === lane.state, (current) => ({ ...current, keep_alive_blocked_reason: reason }), false);
|
|
1427
|
+
}
|
|
1428
|
+
catch (err) {
|
|
1429
|
+
markDirty(summary, {
|
|
1430
|
+
kind: "lane",
|
|
1431
|
+
id: lane.lane_id,
|
|
1432
|
+
why: `keep-alive skip state write failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
1433
|
+
});
|
|
1434
|
+
}
|
|
1435
|
+
}
|
|
1436
|
+
function latestEnvelopeMtimeMs(hqRoot, lane, ref) {
|
|
1437
|
+
const filePath = path.join(laneRunDir(hqRoot, lane.lane_id), ref);
|
|
1438
|
+
try {
|
|
1439
|
+
return fs.statSync(filePath).mtimeMs;
|
|
1440
|
+
}
|
|
1441
|
+
catch (err) {
|
|
1442
|
+
throw new LanesError("keep_alive_envelope_timestamp_unavailable", `Cannot stat latest envelope for lane ${lane.lane_id}: ${err instanceof Error ? err.message : String(err)}`, { lane_id: lane.lane_id, path: filePath });
|
|
1443
|
+
}
|
|
1444
|
+
}
|
|
1416
1445
|
function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, hireGrantFlagByCompany) {
|
|
1417
1446
|
const nowMs = now.getTime();
|
|
1418
1447
|
const nowIso = now.toISOString();
|
|
1419
1448
|
const config = loadLanesConfig(hqRoot);
|
|
1449
|
+
const keepAliveEnabled = [...flagByCompany.values()].some((enabled) => enabled);
|
|
1450
|
+
let retiredLaneIds = null;
|
|
1451
|
+
if (keepAliveEnabled) {
|
|
1452
|
+
try {
|
|
1453
|
+
retiredLaneIds = loadRetiredLaneIds(hqRoot);
|
|
1454
|
+
}
|
|
1455
|
+
catch (err) {
|
|
1456
|
+
markError(summary, err);
|
|
1457
|
+
}
|
|
1458
|
+
}
|
|
1420
1459
|
for (const lane of lanes) {
|
|
1421
|
-
|
|
1460
|
+
if (!laneRunsHere(lane, deps.host))
|
|
1461
|
+
continue;
|
|
1462
|
+
if (lane.state === "killed" || lane.state === "cancelled" || lane.state === "failed") {
|
|
1463
|
+
if (lane.keep_alive === true &&
|
|
1464
|
+
flagByCompany.get(keepAliveCompanyKey(lane.company)) === true)
|
|
1465
|
+
recordKeepAliveSkip(hqRoot, lane, "terminal_state", summary, deps);
|
|
1466
|
+
continue;
|
|
1467
|
+
}
|
|
1422
1468
|
if (lane.state !== "done" ||
|
|
1423
|
-
lane.keep_alive ===
|
|
1424
|
-
(lane.keep_alive !== true && !hasRecordedProfileFallback) ||
|
|
1425
|
-
!laneRunsHere(lane, deps.host))
|
|
1469
|
+
(lane.keep_alive !== true && lane.worker_profile === undefined))
|
|
1426
1470
|
continue;
|
|
1427
1471
|
const latest = readLatestEnvelopeRecord(hqRoot, lane.lane_id);
|
|
1428
1472
|
const envelope = latest?.envelope;
|
|
1429
1473
|
if (!envelope || envelope.decision !== "done")
|
|
1430
1474
|
continue;
|
|
1431
|
-
const keepAliveConfigurationUnchanged = (candidate) => lane.keep_alive === true
|
|
1432
|
-
? candidate.keep_alive === true
|
|
1433
|
-
: lane.keep_alive === undefined &&
|
|
1434
|
-
lane.worker_profile !== undefined &&
|
|
1435
|
-
candidate.keep_alive === undefined &&
|
|
1436
|
-
candidate.worker_profile === lane.worker_profile;
|
|
1437
1475
|
if (envelope.story_complete === true) {
|
|
1438
1476
|
if (hasKeepAliveProgress(lane)) {
|
|
1439
1477
|
casLaneIf(hqRoot, lane.lane_id, (current) => current.version === lane.version &&
|
|
1440
1478
|
current.state === "done" &&
|
|
1441
|
-
|
|
1479
|
+
current.keep_alive === lane.keep_alive, (current) => ({ ...clearedKeepAliveProgress(current), keep_alive_blocked_reason: undefined }), deps.dryRun);
|
|
1442
1480
|
}
|
|
1443
1481
|
continue;
|
|
1444
1482
|
}
|
|
1483
|
+
if (flagByCompany.get(keepAliveCompanyKey(lane.company)) !== true)
|
|
1484
|
+
continue;
|
|
1445
1485
|
if (lane.terminal_reason === PROVIDER_REJECTED_REASON ||
|
|
1446
|
-
lane.terminal_reason === "account_not_logged_in"
|
|
1447
|
-
|
|
1448
|
-
!(lane.provider_session_id ?? lane.last_resume?.session_id))
|
|
1486
|
+
lane.terminal_reason === "account_not_logged_in") {
|
|
1487
|
+
recordKeepAliveSkip(hqRoot, lane, "terminal_state", summary, deps, lane.terminal_reason);
|
|
1449
1488
|
continue;
|
|
1450
|
-
|
|
1489
|
+
}
|
|
1490
|
+
if (lane.keep_alive !== true) {
|
|
1491
|
+
recordKeepAliveSkip(hqRoot, lane, "not_opted_in", summary, deps);
|
|
1451
1492
|
continue;
|
|
1452
|
-
|
|
1493
|
+
}
|
|
1494
|
+
const optedInAtMs = Date.parse(lane.keep_alive_opted_in_at ?? "");
|
|
1495
|
+
if (!Number.isFinite(optedInAtMs)) {
|
|
1496
|
+
recordKeepAliveSkip(hqRoot, lane, "missing_opt_in_time", summary, deps);
|
|
1497
|
+
continue;
|
|
1498
|
+
}
|
|
1499
|
+
let envelopeAtMs;
|
|
1453
1500
|
try {
|
|
1454
|
-
|
|
1501
|
+
envelopeAtMs = latestEnvelopeMtimeMs(hqRoot, lane, latest.ref);
|
|
1455
1502
|
}
|
|
1456
1503
|
catch (err) {
|
|
1457
|
-
const message = err instanceof LanesError
|
|
1458
|
-
? `${err.error}: ${err.message}`
|
|
1459
|
-
: err instanceof Error ? err.message : String(err);
|
|
1460
1504
|
markDirty(summary, {
|
|
1461
1505
|
kind: "lane",
|
|
1462
1506
|
id: lane.lane_id,
|
|
1463
|
-
why: `keep-alive
|
|
1507
|
+
why: `keep-alive envelope timestamp unavailable: ${err instanceof Error ? err.message : String(err)}`,
|
|
1464
1508
|
});
|
|
1465
1509
|
continue;
|
|
1466
1510
|
}
|
|
1467
|
-
if (
|
|
1511
|
+
if (envelopeAtMs <= optedInAtMs) {
|
|
1512
|
+
recordKeepAliveSkip(hqRoot, lane, "envelope_before_opt_in", summary, deps);
|
|
1513
|
+
continue;
|
|
1514
|
+
}
|
|
1515
|
+
if (retiredLaneIds === null) {
|
|
1516
|
+
recordKeepAliveSkip(hqRoot, lane, "retired_list_invalid", summary, deps);
|
|
1517
|
+
continue;
|
|
1518
|
+
}
|
|
1519
|
+
const exclusion = keepAliveExclusion(lane, lanes, retiredLaneIds);
|
|
1520
|
+
if (exclusion !== undefined) {
|
|
1521
|
+
recordKeepAliveSkip(hqRoot, lane, exclusion, summary, deps);
|
|
1522
|
+
continue;
|
|
1523
|
+
}
|
|
1524
|
+
if (!providerSupportsSessionResume(lane.provider) ||
|
|
1525
|
+
!(lane.provider_session_id ?? lane.last_resume?.session_id))
|
|
1468
1526
|
continue;
|
|
1469
|
-
const keepAliveStillEnabled = (candidate) =>
|
|
1470
|
-
|
|
1471
|
-
: candidate.keep_alive === undefined &&
|
|
1472
|
-
candidate.worker_profile === lane.worker_profile;
|
|
1527
|
+
const keepAliveStillEnabled = (candidate) => candidate.keep_alive === true &&
|
|
1528
|
+
candidate.keep_alive_opted_in_at === lane.keep_alive_opted_in_at;
|
|
1473
1529
|
const current = readLane(hqRoot, lane.lane_id);
|
|
1474
1530
|
if (current.version !== lane.version ||
|
|
1475
1531
|
current.state !== "done" ||
|
|
1476
1532
|
!keepAliveStillEnabled(current) ||
|
|
1477
1533
|
!laneRunsHere(current, deps.host))
|
|
1478
1534
|
continue;
|
|
1479
|
-
const
|
|
1535
|
+
const currentEnvelopeRecord = readLatestEnvelopeRecord(hqRoot, current.lane_id);
|
|
1536
|
+
const currentEnvelope = currentEnvelopeRecord?.envelope;
|
|
1480
1537
|
if (!currentEnvelope || currentEnvelope.decision !== "done")
|
|
1481
1538
|
continue;
|
|
1482
1539
|
if (currentEnvelope.story_complete === true) {
|
|
1483
1540
|
if (hasKeepAliveProgress(current)) {
|
|
1484
1541
|
casLaneIf(hqRoot, current.lane_id, (latestLane) => latestLane.version === current.version &&
|
|
1485
1542
|
latestLane.state === "done" &&
|
|
1486
|
-
keepAliveStillEnabled(latestLane), (latestLane) => clearedKeepAliveProgress(latestLane), deps.dryRun);
|
|
1543
|
+
keepAliveStillEnabled(latestLane), (latestLane) => ({ ...clearedKeepAliveProgress(latestLane), keep_alive_blocked_reason: undefined }), deps.dryRun);
|
|
1487
1544
|
}
|
|
1488
1545
|
continue;
|
|
1489
1546
|
}
|
|
1547
|
+
if (currentEnvelopeRecord !== null) {
|
|
1548
|
+
let currentEnvelopeAtMs;
|
|
1549
|
+
try {
|
|
1550
|
+
currentEnvelopeAtMs = latestEnvelopeMtimeMs(hqRoot, current, currentEnvelopeRecord.ref);
|
|
1551
|
+
}
|
|
1552
|
+
catch (err) {
|
|
1553
|
+
markDirty(summary, {
|
|
1554
|
+
kind: "lane",
|
|
1555
|
+
id: current.lane_id,
|
|
1556
|
+
why: `keep-alive envelope timestamp unavailable: ${err instanceof Error ? err.message : String(err)}`,
|
|
1557
|
+
});
|
|
1558
|
+
continue;
|
|
1559
|
+
}
|
|
1560
|
+
if (currentEnvelopeAtMs <= Date.parse(current.keep_alive_opted_in_at ?? "")) {
|
|
1561
|
+
recordKeepAliveSkip(hqRoot, current, "envelope_before_opt_in", summary, deps);
|
|
1562
|
+
continue;
|
|
1563
|
+
}
|
|
1564
|
+
}
|
|
1565
|
+
if (retiredLaneIds === null) {
|
|
1566
|
+
recordKeepAliveSkip(hqRoot, current, "retired_list_invalid", summary, deps);
|
|
1567
|
+
continue;
|
|
1568
|
+
}
|
|
1569
|
+
const currentLanes = loadLaneRecords(hqRoot, summary);
|
|
1570
|
+
const currentExclusion = keepAliveExclusion(current, currentLanes, retiredLaneIds);
|
|
1571
|
+
if (currentExclusion !== undefined) {
|
|
1572
|
+
recordKeepAliveSkip(hqRoot, current, currentExclusion, summary, deps);
|
|
1573
|
+
continue;
|
|
1574
|
+
}
|
|
1490
1575
|
const providerSessionId = current.provider_session_id ?? current.last_resume?.session_id;
|
|
1491
1576
|
if (current.terminal_reason === PROVIDER_REJECTED_REASON ||
|
|
1492
1577
|
current.terminal_reason === "account_not_logged_in" ||
|
|
@@ -1567,6 +1652,7 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
|
|
|
1567
1652
|
keep_alive_last_resume_at: nowIso,
|
|
1568
1653
|
keep_alive_next_eligible_at: nextEligibleAt,
|
|
1569
1654
|
keep_alive_exhausted_at: undefined,
|
|
1655
|
+
keep_alive_blocked_reason: undefined,
|
|
1570
1656
|
}), deps.dryRun);
|
|
1571
1657
|
const seniorId = current.senior.kind === "lane" ? current.senior.id : undefined;
|
|
1572
1658
|
let queued;
|
package/dist/lib/lanes/store.js
CHANGED
|
@@ -409,6 +409,25 @@ export function parseLaneRecord(raw, filePath) {
|
|
|
409
409
|
value: raw.keep_alive,
|
|
410
410
|
});
|
|
411
411
|
}
|
|
412
|
+
const keepAliveBlockedReason = raw.keep_alive_blocked_reason;
|
|
413
|
+
const validKeepAliveBlockedReasons = [
|
|
414
|
+
"not_opted_in",
|
|
415
|
+
"missing_opt_in_time",
|
|
416
|
+
"envelope_before_opt_in",
|
|
417
|
+
"retired",
|
|
418
|
+
"superseded",
|
|
419
|
+
"retired_list_invalid",
|
|
420
|
+
"terminal_state",
|
|
421
|
+
];
|
|
422
|
+
if (keepAliveBlockedReason !== undefined &&
|
|
423
|
+
(typeof keepAliveBlockedReason !== "string" ||
|
|
424
|
+
!validKeepAliveBlockedReasons.includes(keepAliveBlockedReason))) {
|
|
425
|
+
throw new LaneSchemaError("keep_alive_blocked_reason is invalid", {
|
|
426
|
+
path: filePath,
|
|
427
|
+
field: "keep_alive_blocked_reason",
|
|
428
|
+
value: keepAliveBlockedReason,
|
|
429
|
+
});
|
|
430
|
+
}
|
|
412
431
|
if (raw.keep_alive_resume_count !== undefined &&
|
|
413
432
|
(typeof raw.keep_alive_resume_count !== "number" ||
|
|
414
433
|
!Number.isSafeInteger(raw.keep_alive_resume_count) ||
|
|
@@ -420,6 +439,7 @@ export function parseLaneRecord(raw, filePath) {
|
|
|
420
439
|
});
|
|
421
440
|
}
|
|
422
441
|
for (const field of [
|
|
442
|
+
"keep_alive_opted_in_at",
|
|
423
443
|
"keep_alive_last_resume_at",
|
|
424
444
|
"keep_alive_next_eligible_at",
|
|
425
445
|
"keep_alive_exhausted_at",
|
|
@@ -356,6 +356,10 @@ export interface LaneRecord {
|
|
|
356
356
|
worker_profile?: string;
|
|
357
357
|
/** Whether this lead lane should resume completed turns while its story is open. */
|
|
358
358
|
keep_alive?: boolean;
|
|
359
|
+
/** Time this lane most recently explicitly opted into keep-alive. */
|
|
360
|
+
keep_alive_opted_in_at?: string;
|
|
361
|
+
/** Why reconcile declined to resume this lane most recently. */
|
|
362
|
+
keep_alive_blocked_reason?: "not_opted_in" | "missing_opt_in_time" | "envelope_before_opt_in" | "retired" | "superseded" | "retired_list_invalid" | "terminal_state";
|
|
359
363
|
/** Consecutive automatic keep-alive resumes since the last human reset. */
|
|
360
364
|
keep_alive_resume_count?: number;
|
|
361
365
|
/** Time the most recent automatic keep-alive resume was queued. */
|
package/dist/lib/lanes/types.js
CHANGED
|
@@ -224,7 +224,7 @@ export const LANE_FIELD_WRITERS = {
|
|
|
224
224
|
pickup_validation_pending: ["creator", "command", "spawn", "reconcile"],
|
|
225
225
|
pickup_validation_pending_since: ["creator", "command", "spawn", "reconcile"],
|
|
226
226
|
priority: ["command"], worker: ["creator", "command"], worker_path: ["creator", "command"], worker_pack: ["creator", "command"], worker_profile: ["creator", "command"], worker_profile_hash: ["creator", "command", "spawn"], worker_profile_ignored: ["creator", "command", "spawn"], worker_policy_delivery: ["command", "spawn"],
|
|
227
|
-
keep_alive: ["creator", "command"], keep_alive_resume_count: ["command", "reconcile"], keep_alive_last_resume_at: ["command", "reconcile"], keep_alive_next_eligible_at: ["command", "reconcile"], keep_alive_exhausted_at: ["command", "reconcile"],
|
|
227
|
+
keep_alive: ["creator", "command"], keep_alive_opted_in_at: ["creator", "command"], keep_alive_blocked_reason: ["command", "reconcile"], keep_alive_resume_count: ["command", "reconcile"], keep_alive_last_resume_at: ["command", "reconcile"], keep_alive_next_eligible_at: ["command", "reconcile"], keep_alive_exhausted_at: ["command", "reconcile"],
|
|
228
228
|
provider: ["creator"], account: ["creator", "spawn", "command", "watcher", "reconcile"], account_selection: ["creator", "command"], admission_accounts: ["creator", "command"], account_history: ["command", "spawn", "watcher", "reconcile"], pending_resume: ["command", "spawn", "watcher", "reconcile"], placement_default: ["creator"], host: ["creator", "command"], origin_host: ["creator"], origin_host_source: ["creator"],
|
|
229
229
|
version: ["creator"], created_at: ["creator"], updated_at: ["creator"], attribution_history: ["command"],
|
|
230
230
|
last_envelope: ["watcher", "reconcile"], model: ["creator", "spawn", "command"], model_source: ["creator", "spawn", "command"], effort: ["creator", "spawn", "command"], effort_source: ["creator", "spawn", "command"], fast_mode: ["spawn"],
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@indigoai-us/hq-cli",
|
|
3
|
-
"version": "5.
|
|
3
|
+
"version": "5.215.0",
|
|
4
4
|
"description": "HQ by Indigo management CLI — modules and cloud sync",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"bin": {
|
|
@@ -41,7 +41,7 @@
|
|
|
41
41
|
"dependencies": {
|
|
42
42
|
"@aws-sdk/client-iot-data-plane": "^3.1096.0",
|
|
43
43
|
"@aws-sdk/client-s3": "^3.1049.0",
|
|
44
|
-
"@indigoai-us/hq-cloud": "~6.
|
|
44
|
+
"@indigoai-us/hq-cloud": "~6.18.5",
|
|
45
45
|
"@indigoai-us/hq-flags-client": "^0.1.2",
|
|
46
46
|
"@sentry/node": "^10.49.0",
|
|
47
47
|
"@tobilu/qmd": "2.5.3",
|