@indigoai-us/hq-cli 5.259.0 → 5.261.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -0
- package/dist/commands/lanes-message.js +3 -0
- package/dist/commands/lanes.js +19 -1
- package/dist/lib/lanes/blocker-freshness.d.ts +31 -0
- package/dist/lib/lanes/blocker-freshness.js +167 -0
- package/dist/lib/lanes/config.d.ts +2 -0
- package/dist/lib/lanes/config.js +12 -0
- package/dist/lib/lanes/keep-alive.d.ts +13 -5
- package/dist/lib/lanes/keep-alive.js +68 -46
- package/dist/lib/lanes/reconcile.js +27 -9
- package/dist/lib/lanes/types.d.ts +26 -2
- package/dist/lib/lanes/types.js +1 -1
- package/dist/lib/lanes/watcher-main.js +25 -0
- package/dist/lib/lanes/watcher.d.ts +4 -0
- package/dist/lib/lanes/watcher.js +176 -10
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,29 @@
|
|
|
2
2
|
|
|
3
3
|
## [Unreleased]
|
|
4
4
|
|
|
5
|
+
## [5.261.0] — 2026-09-28
|
|
6
|
+
|
|
7
|
+
### Fixed
|
|
8
|
+
|
|
9
|
+
- Standing keep-alive leads now resume stories that remain open. Completed
|
|
10
|
+
stories resume only on standing leads with queued work, and reconcile reports
|
|
11
|
+
why other completed lanes stay idle.
|
|
12
|
+
|
|
13
|
+
## [5.260.0] — 2026-09-28
|
|
14
|
+
|
|
15
|
+
### Added
|
|
16
|
+
|
|
17
|
+
- The `lanes.blocker-freshness` flag checks manager blockers after each turn.
|
|
18
|
+
It resumes once for stale blockers, then alerts the senior on a repeated
|
|
19
|
+
stale set. The window defaults to four hours and can be set with
|
|
20
|
+
`blocker_freshness_hours` in the lanes config.
|
|
21
|
+
|
|
22
|
+
### Fixed
|
|
23
|
+
|
|
24
|
+
- Blocker-freshness escalations now reach session seniors through the existing
|
|
25
|
+
monitor-session inbox. Work Mesh is an optional signal, not delivery, and a
|
|
26
|
+
failed delivery is retried on a later stale turn.
|
|
27
|
+
|
|
5
28
|
## [5.259.0] — 2026-09-28
|
|
6
29
|
|
|
7
30
|
### Fixed
|
|
@@ -92,6 +92,9 @@ export function createLaneMessageHandlers(deps) {
|
|
|
92
92
|
readStarttime: ctx.readStarttime ?? readProcStarttime,
|
|
93
93
|
mesh: null,
|
|
94
94
|
resolveQuestionActiveCompany: () => resolveActiveQuestionCompany(ctx),
|
|
95
|
+
deliverLaneSeniorMessage: async (seniorLaneId, seniorText) => {
|
|
96
|
+
await runMessage(ctx, seniorLaneId, { text: seniorText, silent: true });
|
|
97
|
+
},
|
|
95
98
|
});
|
|
96
99
|
lane = readLane(hqRoot, laneId);
|
|
97
100
|
// This invocation won the race to classify the dead worker, so the
|
package/dist/commands/lanes.js
CHANGED
|
@@ -50,7 +50,7 @@ import { effectiveAccountHome } from "../lib/lanes/account-home.js";
|
|
|
50
50
|
import { AccountResolutionError, compareLaneQueueOrder, LANE_PROVIDERS, LANE_STATES, QUESTION_STATUSES, SENIOR_KINDS, accountStatus, appendCappedHistory, canonicalizeHome, createLane, decideAdmission, defaultLivenessDeps, defaultSystemdUserDir, defaultThrottleDeps, installReconcileTimer, providerPathEnv, probeDrainHookRegistration, isLanesError, isLaneVersionConflict, isTerminalLaneState, aliveDirectChildren, assignedChildHireGrants, hireGrantCapacityError, planHireGrantBackfill, terminalChildResumeHireGrantError, withHireGrantAdmission, laneStoryLedgerEnabled, validateProjectStories, buildLaneStatusSnapshot, formatLaneStatusHuman, listLaneIds, listQuestionIds, loadLanesConfig, laneRunsHere, MAX_LANE_DEPTH, MONITOR_ERROR_EXIT_CODE, readLiveMonitorEvidence, resolveMachineIdentity as lookupMachineIdentity, loadLivenessIndex, parseLaneProvider, parseLaneRecord, parseLaneState, parseQuestionRecord, parseSeniorKind, readLane, retryUpdateLane, syncLaneRecordFromOwner, readQuestion, resolveAccount, resolveLaneWorker, ignoredWorkerProfileFields, loadWorkerProfile, loadWorkerProfileWithRegistry, registryWorkerRecord, runReconcilePass, sessionLiveness, tryReadLane, uninstallReconcileTimer, updateLane, updateQuestion, dropboxDir, laneDeliveryMechanism, hasActiveAdmissionReservation, hasActivePendingResume, hasRecordedSessionId, hasEarlierEligibleQueuedLane, laneAdmissionCapReason, providerSupportsSessionResume, providerSessionPortable, sessionNotPortableMessage, withLaneAdmissionLock, withAllLaneAdmissionLocks, withLaneHireLocks, raiseLaneQuestion, sha256Text, } from "../lib/lanes/index.js";
|
|
51
51
|
import { activeLaneStories, laneStories, laneStory, storyWarning, } from "../lib/lanes/stories.js";
|
|
52
52
|
import { resolveLaneKeepAlive } from "../lib/lanes/profile.js";
|
|
53
|
-
import { keepAliveExclusion, LANE_SUPERSESSION_LINEAGE_AWARE_FLAG, loadRetiredLaneIds, } from "../lib/lanes/keep-alive.js";
|
|
53
|
+
import { doneKeepAliveVerdict, hasPendingKeepAliveWork, keepAliveExclusion, LANE_SUPERSESSION_LINEAGE_AWARE_FLAG, loadRetiredLaneIds, } from "../lib/lanes/keep-alive.js";
|
|
54
54
|
import { questionBodyAbsPath, resolveQuestionCompanyWithFallback, resolveProjectQuestionCompany, } from "../lib/lanes/questions.js";
|
|
55
55
|
import { assertQuestionBodySecretSafe } from "../lib/lanes/questions.js";
|
|
56
56
|
import { compactionAfterUsage, } from "../lib/lanes/compaction.js";
|
|
@@ -59,6 +59,7 @@ import { LanesError } from "../lib/lanes/errors.js";
|
|
|
59
59
|
import { loadQuotaState, quotaBlockFor } from "../lib/lanes/quota.js";
|
|
60
60
|
import { SESSION_MOVE_REASON } from "../lib/lanes/move-account.js";
|
|
61
61
|
import { managerStatusGateEnabledForLane, resolveManagerStatusGateForLane, } from "../lib/lanes/manager-status-gate.js";
|
|
62
|
+
import { blockerFreshnessFingerprint, resolveBlockerFreshnessForLane, } from "../lib/lanes/blocker-freshness.js";
|
|
62
63
|
import { ensureSessionHome } from "../lib/lanes/session-relink.js";
|
|
63
64
|
import { defaultProbe, validateProfileLaneModel, validateLaneModel, } from "../lib/lanes/models.js";
|
|
64
65
|
import { parseCliModel, refuseOutpostSelfTarget, resolveLaneModel, FOREGROUND_ONLY_RULE, STANDING_AUTHORIZATION, spawnLaneWorker, spawnEnvelopeRepairRequest, laneRunDir, readSpawnManifest, stopLaneProcesses, } from "../lib/lanes/spawn.js";
|
|
@@ -4523,8 +4524,15 @@ async function runShow(ctx, laneId) {
|
|
|
4523
4524
|
: storedLane;
|
|
4524
4525
|
const askDeliveryFailed = readJsonFile(askDeliveryFailedPath(hqRoot, laneId));
|
|
4525
4526
|
const keepAliveState = resolveLaneKeepAlive(hqRoot, storedLane);
|
|
4527
|
+
const doneStateVerdict = storedLane.state === "done" && storedLane.keep_alive === true
|
|
4528
|
+
? doneKeepAliveVerdict(storedLane, { decision: storedLane.last_envelope?.decision, story_complete: storedLane.story_complete }, storedLane.keep_alive_standing === true && hasPendingKeepAliveWork(hqRoot, storedLane))
|
|
4529
|
+
: null;
|
|
4526
4530
|
const managerStatusGate = await resolveManagerStatusGateForLane(hqRoot, storedLane, { env: ctx.env });
|
|
4527
4531
|
const managerStatusTarget = managerStatusGate.target;
|
|
4532
|
+
const blockerFreshness = await resolveBlockerFreshnessForLane(hqRoot, storedLane, {
|
|
4533
|
+
env: ctx.env,
|
|
4534
|
+
now: ctx.now().getTime(),
|
|
4535
|
+
});
|
|
4528
4536
|
printJson({
|
|
4529
4537
|
ok: true,
|
|
4530
4538
|
action: "show",
|
|
@@ -4555,6 +4563,7 @@ async function runShow(ctx, laneId) {
|
|
|
4555
4563
|
enabled: keepAliveState.enabled,
|
|
4556
4564
|
source: keepAliveState.source,
|
|
4557
4565
|
standing: storedLane.keep_alive_standing === true,
|
|
4566
|
+
done_state_verdict: doneStateVerdict,
|
|
4558
4567
|
opted_in_at: storedLane.keep_alive_opted_in_at ?? null,
|
|
4559
4568
|
blocked_reason: storedLane.keep_alive_blocked_reason ??
|
|
4560
4569
|
(storedLane.keep_alive === true ? null : "not_opted_in"),
|
|
@@ -4579,6 +4588,15 @@ async function runShow(ctx, laneId) {
|
|
|
4579
4588
|
consecutive_misses: storedLane.status_gate?.consecutive_misses ?? 0,
|
|
4580
4589
|
}
|
|
4581
4590
|
: null,
|
|
4591
|
+
blocker_freshness: blockerFreshness.flagEnabled
|
|
4592
|
+
? {
|
|
4593
|
+
enabled: blockerFreshness.enabled,
|
|
4594
|
+
stale: blockerFreshness.stale.map((blocker) => blocker.label),
|
|
4595
|
+
escalated: blockerFreshness.stale.length > 0 &&
|
|
4596
|
+
storedLane.blocker_freshness_state?.escalated_fingerprint ===
|
|
4597
|
+
blockerFreshnessFingerprint(blockerFreshness.stale),
|
|
4598
|
+
}
|
|
4599
|
+
: null,
|
|
4582
4600
|
last_message_at: lane.last_message_at ?? null,
|
|
4583
4601
|
last_compaction: lane.last_compaction
|
|
4584
4602
|
? { ...lane.last_compaction, after: compactionAfterUsage(storedLane) }
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { type ManagerStatusTarget } from "./manager-status-gate.js";
|
|
2
|
+
import { readLanesRegistryFlag } from "./registry-flag.js";
|
|
3
|
+
import type { LaneRecord } from "./types.js";
|
|
4
|
+
export declare const BLOCKER_FRESHNESS_FLAG = "lanes.blocker-freshness";
|
|
5
|
+
export declare const BLOCKER_FRESHNESS_DEFAULT_HOURS = 4;
|
|
6
|
+
export declare const BLOCKER_FRESHNESS_FUTURE_GRACE_MS: number;
|
|
7
|
+
export interface StaleBlocker {
|
|
8
|
+
key: string;
|
|
9
|
+
label: string;
|
|
10
|
+
}
|
|
11
|
+
export interface BlockerFreshnessResolution {
|
|
12
|
+
/** The company-scoped flag value, kept separate from manager applicability. */
|
|
13
|
+
flagEnabled: boolean;
|
|
14
|
+
/** True only when the flag is on and this lane has a valid manager target. */
|
|
15
|
+
enabled: boolean;
|
|
16
|
+
target?: ManagerStatusTarget;
|
|
17
|
+
statusPath?: string;
|
|
18
|
+
windowHours: number;
|
|
19
|
+
stale: StaleBlocker[];
|
|
20
|
+
error?: string;
|
|
21
|
+
}
|
|
22
|
+
export interface BlockerFreshnessResolutionOptions {
|
|
23
|
+
env?: NodeJS.ProcessEnv;
|
|
24
|
+
now?: number;
|
|
25
|
+
readFlag?: typeof readLanesRegistryFlag;
|
|
26
|
+
onError?: (error: unknown) => void;
|
|
27
|
+
}
|
|
28
|
+
export declare function blockerFreshnessFingerprint(stale: readonly StaleBlocker[]): string;
|
|
29
|
+
/** Resolve the flag before touching the manager mapping or status artifact. */
|
|
30
|
+
export declare function resolveBlockerFreshnessForLane(root: string, lane: LaneRecord, options?: BlockerFreshnessResolutionOptions): Promise<BlockerFreshnessResolution>;
|
|
31
|
+
//# sourceMappingURL=blocker-freshness.d.ts.map
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import * as fs from "node:fs";
|
|
3
|
+
import { LanesError } from "./errors.js";
|
|
4
|
+
import { DEFAULT_BLOCKER_FRESHNESS_HOURS, loadLanesConfig } from "./config.js";
|
|
5
|
+
import { resolveManagerStatusTarget } from "./manager-status-gate.js";
|
|
6
|
+
import { readLanesRegistryFlag } from "./registry-flag.js";
|
|
7
|
+
import { assertNoSecretText } from "./text-scan.js";
|
|
8
|
+
export const BLOCKER_FRESHNESS_FLAG = "lanes.blocker-freshness";
|
|
9
|
+
export const BLOCKER_FRESHNESS_DEFAULT_HOURS = DEFAULT_BLOCKER_FRESHNESS_HOURS;
|
|
10
|
+
export const BLOCKER_FRESHNESS_FUTURE_GRACE_MS = 5 * 60 * 1000;
|
|
11
|
+
const TERMINAL_BLOCKER_STATUSES = new Set([
|
|
12
|
+
"closed",
|
|
13
|
+
"resolved",
|
|
14
|
+
"done",
|
|
15
|
+
"superseded",
|
|
16
|
+
"cancelled",
|
|
17
|
+
]);
|
|
18
|
+
function isObject(value) {
|
|
19
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
20
|
+
}
|
|
21
|
+
function errorCode(error) {
|
|
22
|
+
return error instanceof LanesError ? error.error : "blocker_freshness_unavailable";
|
|
23
|
+
}
|
|
24
|
+
function reportError(lane, error) {
|
|
25
|
+
process.stderr.write(`hq lanes: blocker freshness disabled for ${lane.lane_id}: ${errorCode(error)}\n`);
|
|
26
|
+
}
|
|
27
|
+
function failedResolution(flagEnabled, error, onError) {
|
|
28
|
+
onError(error);
|
|
29
|
+
return {
|
|
30
|
+
flagEnabled,
|
|
31
|
+
enabled: false,
|
|
32
|
+
windowHours: BLOCKER_FRESHNESS_DEFAULT_HOURS,
|
|
33
|
+
stale: [],
|
|
34
|
+
error: errorCode(error),
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
function identityFor(blocker, index) {
|
|
38
|
+
for (const field of ["id", "item", "what", "reason"]) {
|
|
39
|
+
const candidate = blocker[field];
|
|
40
|
+
if (typeof candidate === "string" && candidate.trim().length > 0) {
|
|
41
|
+
const identity = candidate.trim();
|
|
42
|
+
assertNoSecretText(`blocker ${field}`, identity);
|
|
43
|
+
return {
|
|
44
|
+
key: blockerKey(`${field}:${identity}`),
|
|
45
|
+
label: identity.slice(0, 120),
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
const identity = `index:${index + 1}`;
|
|
50
|
+
const label = `blocker ${index + 1}`;
|
|
51
|
+
assertNoSecretText("blocker label", label);
|
|
52
|
+
return { key: blockerKey(identity), label };
|
|
53
|
+
}
|
|
54
|
+
function hasClosingTimestamp(blocker) {
|
|
55
|
+
return [blocker.closed_at, blocker.resolved_at].some((value) => value !== undefined && value !== null && value !== "");
|
|
56
|
+
}
|
|
57
|
+
function blockerKey(label) {
|
|
58
|
+
return createHash("sha256").update(label, "utf8").digest("hex");
|
|
59
|
+
}
|
|
60
|
+
function readBlockers(statusPath) {
|
|
61
|
+
let stat;
|
|
62
|
+
try {
|
|
63
|
+
stat = fs.lstatSync(statusPath);
|
|
64
|
+
}
|
|
65
|
+
catch (error) {
|
|
66
|
+
throw new LanesError("blocker_freshness_status_unreadable", `Cannot inspect manager status ${statusPath}: ${error instanceof Error ? error.message : String(error)}`, { path: statusPath });
|
|
67
|
+
}
|
|
68
|
+
if (!stat.isFile() || stat.isSymbolicLink()) {
|
|
69
|
+
throw new LanesError("blocker_freshness_status_invalid", `Manager status must be a regular file: ${statusPath}`, { path: statusPath });
|
|
70
|
+
}
|
|
71
|
+
let parsed;
|
|
72
|
+
try {
|
|
73
|
+
parsed = JSON.parse(fs.readFileSync(statusPath, "utf8"));
|
|
74
|
+
}
|
|
75
|
+
catch (error) {
|
|
76
|
+
throw new LanesError("blocker_freshness_status_invalid", `Manager status contains invalid JSON: ${statusPath}`, { path: statusPath, cause: error instanceof Error ? error.name : "UnknownError" });
|
|
77
|
+
}
|
|
78
|
+
if (!isObject(parsed) || !Array.isArray(parsed.blockers)) {
|
|
79
|
+
throw new LanesError("blocker_freshness_status_invalid", `Manager status must contain a top-level blockers array: ${statusPath}`, { path: statusPath });
|
|
80
|
+
}
|
|
81
|
+
return parsed.blockers.filter(isObject);
|
|
82
|
+
}
|
|
83
|
+
function isStale(verifiedAt, now, windowHours) {
|
|
84
|
+
if (typeof verifiedAt !== "string")
|
|
85
|
+
return true;
|
|
86
|
+
const verifiedAtMs = Date.parse(verifiedAt);
|
|
87
|
+
if (!Number.isFinite(verifiedAtMs))
|
|
88
|
+
return true;
|
|
89
|
+
if (verifiedAtMs - now > BLOCKER_FRESHNESS_FUTURE_GRACE_MS)
|
|
90
|
+
return true;
|
|
91
|
+
return now - verifiedAtMs > windowHours * 60 * 60 * 1000;
|
|
92
|
+
}
|
|
93
|
+
export function blockerFreshnessFingerprint(stale) {
|
|
94
|
+
const keys = stale.map((item) => item.key).sort();
|
|
95
|
+
return createHash("sha256").update(JSON.stringify(keys), "utf8").digest("hex");
|
|
96
|
+
}
|
|
97
|
+
/** Resolve the flag before touching the manager mapping or status artifact. */
|
|
98
|
+
export async function resolveBlockerFreshnessForLane(root, lane, options = {}) {
|
|
99
|
+
const onError = options.onError ?? ((error) => reportError(lane, error));
|
|
100
|
+
let flagEnabled;
|
|
101
|
+
try {
|
|
102
|
+
flagEnabled = await (options.readFlag ?? readLanesRegistryFlag)(BLOCKER_FRESHNESS_FLAG, {
|
|
103
|
+
company: lane.company ?? lane.mesh_company ?? undefined,
|
|
104
|
+
env: options.env,
|
|
105
|
+
});
|
|
106
|
+
}
|
|
107
|
+
catch (error) {
|
|
108
|
+
return failedResolution(false, error, onError);
|
|
109
|
+
}
|
|
110
|
+
if (!flagEnabled) {
|
|
111
|
+
return {
|
|
112
|
+
flagEnabled: false,
|
|
113
|
+
enabled: false,
|
|
114
|
+
windowHours: BLOCKER_FRESHNESS_DEFAULT_HOURS,
|
|
115
|
+
stale: [],
|
|
116
|
+
};
|
|
117
|
+
}
|
|
118
|
+
let target;
|
|
119
|
+
try {
|
|
120
|
+
target = resolveManagerStatusTarget(root, lane);
|
|
121
|
+
}
|
|
122
|
+
catch (error) {
|
|
123
|
+
return failedResolution(true, error, onError);
|
|
124
|
+
}
|
|
125
|
+
if (!target.manager || target.statusPath === undefined) {
|
|
126
|
+
return {
|
|
127
|
+
flagEnabled: true,
|
|
128
|
+
enabled: false,
|
|
129
|
+
target,
|
|
130
|
+
windowHours: BLOCKER_FRESHNESS_DEFAULT_HOURS,
|
|
131
|
+
stale: [],
|
|
132
|
+
};
|
|
133
|
+
}
|
|
134
|
+
let windowHours;
|
|
135
|
+
let blockers;
|
|
136
|
+
try {
|
|
137
|
+
windowHours = loadLanesConfig(root).blocker_freshness_hours;
|
|
138
|
+
blockers = readBlockers(target.statusPath);
|
|
139
|
+
}
|
|
140
|
+
catch (error) {
|
|
141
|
+
return failedResolution(true, error, onError);
|
|
142
|
+
}
|
|
143
|
+
const now = options.now ?? Date.now();
|
|
144
|
+
const stale = [];
|
|
145
|
+
try {
|
|
146
|
+
for (const [index, blocker] of blockers.entries()) {
|
|
147
|
+
const status = typeof blocker.status === "string" ? blocker.status.trim().toLowerCase() : "";
|
|
148
|
+
if (TERMINAL_BLOCKER_STATUSES.has(status) || hasClosingTimestamp(blocker))
|
|
149
|
+
continue;
|
|
150
|
+
if (!isStale(blocker.verified_at, now, windowHours))
|
|
151
|
+
continue;
|
|
152
|
+
stale.push(identityFor(blocker, index));
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
catch (error) {
|
|
156
|
+
return failedResolution(true, error, onError);
|
|
157
|
+
}
|
|
158
|
+
return {
|
|
159
|
+
flagEnabled: true,
|
|
160
|
+
enabled: true,
|
|
161
|
+
target,
|
|
162
|
+
statusPath: target.statusPath,
|
|
163
|
+
windowHours,
|
|
164
|
+
stale,
|
|
165
|
+
};
|
|
166
|
+
}
|
|
167
|
+
//# sourceMappingURL=blocker-freshness.js.map
|
|
@@ -39,6 +39,8 @@ export declare const DEFAULT_TURN_START_LOST_MS: number;
|
|
|
39
39
|
export declare const DEFAULT_SILENT_LOST_MS: number;
|
|
40
40
|
/** Unanswered session questions move to the next audience after 10 minutes. */
|
|
41
41
|
export declare const DEFAULT_SESSION_QUESTION_TIMEOUT_SECS: number;
|
|
42
|
+
/** Open manager blockers are re-verified every four hours by default. */
|
|
43
|
+
export declare const DEFAULT_BLOCKER_FRESHNESS_HOURS = 4;
|
|
42
44
|
export declare const DEFAULT_TRACKER_CHECKS: TrackerChecksConfig;
|
|
43
45
|
export declare const DEFAULT_LIVENESS_WINDOWS: LivenessWindows;
|
|
44
46
|
export declare const DEFAULT_PROVIDER_MAX_RUNNING: Record<LaneProvider, number>;
|
package/dist/lib/lanes/config.js
CHANGED
|
@@ -43,6 +43,8 @@ export const DEFAULT_TURN_START_LOST_MS = 2 * 60 * 60 * 1000;
|
|
|
43
43
|
export const DEFAULT_SILENT_LOST_MS = 12 * 60 * 60 * 1000;
|
|
44
44
|
/** Unanswered session questions move to the next audience after 10 minutes. */
|
|
45
45
|
export const DEFAULT_SESSION_QUESTION_TIMEOUT_SECS = 10 * 60;
|
|
46
|
+
/** Open manager blockers are re-verified every four hours by default. */
|
|
47
|
+
export const DEFAULT_BLOCKER_FRESHNESS_HOURS = 4;
|
|
46
48
|
export const DEFAULT_TRACKER_CHECKS = {
|
|
47
49
|
owner_question_surface_ms: 10 * 60 * 1000,
|
|
48
50
|
open_question_age_ms: 2 * 60 * 60 * 1000,
|
|
@@ -141,6 +143,14 @@ function positiveInt(value, field, fallback) {
|
|
|
141
143
|
}
|
|
142
144
|
return value;
|
|
143
145
|
}
|
|
146
|
+
function positiveNumber(value, field, fallback) {
|
|
147
|
+
if (value === undefined)
|
|
148
|
+
return fallback;
|
|
149
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value <= 0) {
|
|
150
|
+
throw new LanesConfigError("invalid_config", `${field} must be a positive finite number`, { field, value });
|
|
151
|
+
}
|
|
152
|
+
return value;
|
|
153
|
+
}
|
|
144
154
|
function unitInterval(value, field, fallback) {
|
|
145
155
|
if (value === undefined)
|
|
146
156
|
return fallback;
|
|
@@ -414,6 +424,7 @@ export function documentedDefaultConfig(deps = defaultDeps()) {
|
|
|
414
424
|
liveness: { ...DEFAULT_LIVENESS_WINDOWS },
|
|
415
425
|
tracker: { checks: { ...DEFAULT_TRACKER_CHECKS } },
|
|
416
426
|
session_question_timeout_secs: DEFAULT_SESSION_QUESTION_TIMEOUT_SECS,
|
|
427
|
+
blocker_freshness_hours: DEFAULT_BLOCKER_FRESHNESS_HOURS,
|
|
417
428
|
};
|
|
418
429
|
}
|
|
419
430
|
export function parseLanesConfig(raw, deps = defaultDeps()) {
|
|
@@ -456,6 +467,7 @@ export function parseLanesConfig(raw, deps = defaultDeps()) {
|
|
|
456
467
|
liveness: parseLiveness(raw.liveness),
|
|
457
468
|
tracker: { checks: parseTrackerChecks(raw.tracker) },
|
|
458
469
|
session_question_timeout_secs: positiveInt(raw.session_question_timeout_secs, "session_question_timeout_secs", DEFAULT_SESSION_QUESTION_TIMEOUT_SECS),
|
|
470
|
+
blocker_freshness_hours: positiveNumber(raw.blocker_freshness_hours, "blocker_freshness_hours", DEFAULT_BLOCKER_FRESHNESS_HOURS),
|
|
459
471
|
};
|
|
460
472
|
}
|
|
461
473
|
export function lanesConfigPath(hqRoot) {
|
|
@@ -7,6 +7,16 @@ export declare const LANE_SUPERSESSION_LINEAGE_AWARE_DEFAULT_ENABLED = false;
|
|
|
7
7
|
export declare const LEAD_LANE_KEEP_ALIVE_MAX_RESUMES = 24;
|
|
8
8
|
export declare const LEAD_LANE_KEEP_ALIVE_BASE_BACKOFF_MS: number;
|
|
9
9
|
export declare const LEAD_LANE_KEEP_ALIVE_MAX_BACKOFF_MS: number;
|
|
10
|
+
export interface LaneTerminalResumeResult {
|
|
11
|
+
lane: LaneRecord;
|
|
12
|
+
resumeQueued: boolean;
|
|
13
|
+
resumeSkippedReason?: "open_question" | "blocked" | "miss_limit" | "no_provider_session";
|
|
14
|
+
}
|
|
15
|
+
/** Shared resume transition for terminal gate follow-ups. */
|
|
16
|
+
export declare function queueLaneTerminalResume(current: LaneRecord, at: string, resumeText: string, options?: {
|
|
17
|
+
allowResume?: boolean;
|
|
18
|
+
mergeExisting?: boolean;
|
|
19
|
+
}): LaneTerminalResumeResult;
|
|
10
20
|
/** Persist a manager status miss and queue its one bounded follow-up turn. */
|
|
11
21
|
export declare function queueManagerStatusGateResume(hqRoot: string, laneId: string, entry: Omit<ManagerStatusGateHistoryEntry, "event" | "resume_queued" | "resume_skipped_reason">, resumeText: string): LaneRecord;
|
|
12
22
|
export declare const LEAD_LANE_KEEP_ALIVE_RESUME_TEXT = "Automatic keep-alive resume: your last turn ended with decision done and story_complete false. Continue the open work from where you stopped. End with done only when story_complete is true; if you are waiting, use hq lanes ask or stay in the loop.";
|
|
@@ -17,11 +27,9 @@ export interface KeepAliveDoneEnvelope {
|
|
|
17
27
|
decision?: string;
|
|
18
28
|
story_complete?: boolean;
|
|
19
29
|
}
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
* complete. Watch and reconcile both use this decision.
|
|
24
|
-
*/
|
|
30
|
+
export type DoneKeepAliveVerdict = "eligible" | "standing_idle" | "story_complete";
|
|
31
|
+
/** Shared done-state policy used by watch, reconcile, and the show command. */
|
|
32
|
+
export declare function doneKeepAliveVerdict(lane: Pick<LaneRecord, "keep_alive_standing">, envelope: KeepAliveDoneEnvelope, hasQueuedWork: boolean): DoneKeepAliveVerdict | null;
|
|
25
33
|
export declare function canResumeDoneKeepAliveLane(lane: Pick<LaneRecord, "keep_alive_standing">, envelope: KeepAliveDoneEnvelope, hasQueuedWork: boolean): boolean;
|
|
26
34
|
/** Work signals already persisted for a lane that ended its turn. */
|
|
27
35
|
export declare function hasPendingKeepAliveWork(hqRoot: string, lane: Pick<LaneRecord, "lane_id" | "company" | "pending_resume">, questionRecords?: readonly QuestionRecord[]): boolean;
|
|
@@ -11,6 +11,53 @@ export const LANE_SUPERSESSION_LINEAGE_AWARE_DEFAULT_ENABLED = false;
|
|
|
11
11
|
export const LEAD_LANE_KEEP_ALIVE_MAX_RESUMES = 24;
|
|
12
12
|
export const LEAD_LANE_KEEP_ALIVE_BASE_BACKOFF_MS = 5 * 60 * 1000;
|
|
13
13
|
export const LEAD_LANE_KEEP_ALIVE_MAX_BACKOFF_MS = 60 * 60 * 1000;
|
|
14
|
+
/** Shared resume transition for terminal gate follow-ups. */
|
|
15
|
+
export function queueLaneTerminalResume(current, at, resumeText, options = {}) {
|
|
16
|
+
if (options.mergeExisting && current.pending_resume !== undefined) {
|
|
17
|
+
return {
|
|
18
|
+
lane: {
|
|
19
|
+
...current,
|
|
20
|
+
pending_resume: {
|
|
21
|
+
...current.pending_resume,
|
|
22
|
+
text: `${current.pending_resume.text}\n\n${resumeText}`,
|
|
23
|
+
},
|
|
24
|
+
},
|
|
25
|
+
resumeQueued: true,
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
if (current.state === "blocked")
|
|
29
|
+
return { lane: current, resumeQueued: false, resumeSkippedReason: "blocked" };
|
|
30
|
+
if (current.state === "awaiting_input")
|
|
31
|
+
return { lane: current, resumeQueued: false, resumeSkippedReason: "open_question" };
|
|
32
|
+
if (options.allowResume === false)
|
|
33
|
+
return { lane: current, resumeQueued: false, resumeSkippedReason: "miss_limit" };
|
|
34
|
+
if (current.state !== "done" && current.state !== "interrupted" && current.state !== "failed") {
|
|
35
|
+
return { lane: current, resumeQueued: false };
|
|
36
|
+
}
|
|
37
|
+
if (!current.provider_session_id) {
|
|
38
|
+
return { lane: current, resumeQueued: false, resumeSkippedReason: "no_provider_session" };
|
|
39
|
+
}
|
|
40
|
+
return {
|
|
41
|
+
lane: {
|
|
42
|
+
...current,
|
|
43
|
+
state: "queued",
|
|
44
|
+
queued_reason: undefined,
|
|
45
|
+
queued_at: at,
|
|
46
|
+
admission_reserved_at: undefined,
|
|
47
|
+
terminal_reason: undefined,
|
|
48
|
+
terminal_details: undefined,
|
|
49
|
+
pending_resume: {
|
|
50
|
+
session_id: current.provider_session_id,
|
|
51
|
+
trigger: "resume",
|
|
52
|
+
resumed_from: current.state,
|
|
53
|
+
text: resumeText,
|
|
54
|
+
compact: true,
|
|
55
|
+
at,
|
|
56
|
+
},
|
|
57
|
+
},
|
|
58
|
+
resumeQueued: true,
|
|
59
|
+
};
|
|
60
|
+
}
|
|
14
61
|
/** Persist a manager status miss and queue its one bounded follow-up turn. */
|
|
15
62
|
export function queueManagerStatusGateResume(hqRoot, laneId, entry, resumeText) {
|
|
16
63
|
return retryUpdateLane(hqRoot, laneId, "watcher", (current) => {
|
|
@@ -22,63 +69,38 @@ export function queueManagerStatusGateResume(hqRoot, laneId, entry, resumeText)
|
|
|
22
69
|
return current;
|
|
23
70
|
}
|
|
24
71
|
const consecutiveMisses = gate.consecutive_misses + 1;
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
resumeSkippedReason = "open_question";
|
|
30
|
-
else if (consecutiveMisses > 1)
|
|
31
|
-
resumeSkippedReason = "miss_limit";
|
|
32
|
-
else if (!current.provider_session_id)
|
|
33
|
-
resumeSkippedReason = "no_provider_session";
|
|
34
|
-
const resumeQueued = resumeSkippedReason === undefined &&
|
|
35
|
-
(current.state === "done" || current.state === "interrupted" || current.state === "failed");
|
|
72
|
+
const transitioned = queueLaneTerminalResume({
|
|
73
|
+
...current,
|
|
74
|
+
status_gate: { ...gate, consecutive_misses: consecutiveMisses },
|
|
75
|
+
}, entry.at, resumeText, { allowResume: consecutiveMisses <= 1 });
|
|
36
76
|
const missed = {
|
|
37
77
|
event: "status_missed",
|
|
38
78
|
...entry,
|
|
39
|
-
...(resumeQueued ? { resume_queued: true } : {}),
|
|
40
|
-
...(resumeSkippedReason !== undefined
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
...current,
|
|
44
|
-
status_gate: { ...gate, consecutive_misses: consecutiveMisses },
|
|
45
|
-
status_gate_history: appendCappedHistory(history, missed),
|
|
79
|
+
...(transitioned.resumeQueued ? { resume_queued: true } : {}),
|
|
80
|
+
...(transitioned.resumeSkippedReason !== undefined
|
|
81
|
+
? { resume_skipped_reason: transitioned.resumeSkippedReason }
|
|
82
|
+
: {}),
|
|
46
83
|
};
|
|
47
|
-
if (!resumeQueued || !current.provider_session_id)
|
|
48
|
-
return next;
|
|
49
|
-
const resumedFrom = current.state;
|
|
50
84
|
return {
|
|
51
|
-
...
|
|
52
|
-
|
|
53
|
-
queued_reason: undefined,
|
|
54
|
-
queued_at: entry.at,
|
|
55
|
-
admission_reserved_at: undefined,
|
|
56
|
-
terminal_reason: undefined,
|
|
57
|
-
terminal_details: undefined,
|
|
58
|
-
pending_resume: {
|
|
59
|
-
session_id: current.provider_session_id,
|
|
60
|
-
trigger: "resume",
|
|
61
|
-
resumed_from: resumedFrom,
|
|
62
|
-
text: resumeText,
|
|
63
|
-
compact: true,
|
|
64
|
-
at: entry.at,
|
|
65
|
-
},
|
|
85
|
+
...transitioned.lane,
|
|
86
|
+
status_gate_history: appendCappedHistory(history, missed),
|
|
66
87
|
};
|
|
67
88
|
});
|
|
68
89
|
}
|
|
69
90
|
export const LEAD_LANE_KEEP_ALIVE_RESUME_TEXT = "Automatic keep-alive resume: your last turn ended with decision done and story_complete false. Continue the open work from where you stopped. End with done only when story_complete is true; if you are waiting, use hq lanes ask or stay in the loop.";
|
|
70
91
|
export const LEAD_LANE_KEEP_ALIVE_INTERRUPTED_RESUME_TEXT = "Automatic keep-alive resume: your previous turn was interrupted by a provider process exit. Check your last state before continuing. Continue the open work from where you stopped. End with done only when story_complete is true; if you are waiting, use hq lanes ask or stay in the loop.";
|
|
71
|
-
/**
|
|
72
|
-
|
|
73
|
-
* standing leads wait for queued work, even when each turn marks the story
|
|
74
|
-
* complete. Watch and reconcile both use this decision.
|
|
75
|
-
*/
|
|
76
|
-
export function canResumeDoneKeepAliveLane(lane, envelope, hasQueuedWork) {
|
|
92
|
+
/** Shared done-state policy used by watch, reconcile, and the show command. */
|
|
93
|
+
export function doneKeepAliveVerdict(lane, envelope, hasQueuedWork) {
|
|
77
94
|
if (envelope.decision !== "done")
|
|
78
|
-
return
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
95
|
+
return null;
|
|
96
|
+
if (envelope.story_complete !== true)
|
|
97
|
+
return "eligible";
|
|
98
|
+
if (lane.keep_alive_standing !== true)
|
|
99
|
+
return "story_complete";
|
|
100
|
+
return hasQueuedWork ? "eligible" : "standing_idle";
|
|
101
|
+
}
|
|
102
|
+
export function canResumeDoneKeepAliveLane(lane, envelope, hasQueuedWork) {
|
|
103
|
+
return doneKeepAliveVerdict(lane, envelope, hasQueuedWork) === "eligible";
|
|
82
104
|
}
|
|
83
105
|
/** Work signals already persisted for a lane that ended its turn. */
|
|
84
106
|
export function hasPendingKeepAliveWork(hqRoot, lane, questionRecords) {
|
|
@@ -22,7 +22,7 @@ import { ANSWERED_QUESTIONS_FILE, armLaneWatcher, envelopeRepairMode, isPidRecor
|
|
|
22
22
|
import { appendCappedHistory, lanesStoreRoot, parseLaneRecord, parseQuestionRecord, readLane, readQuestion, updateLane, updateQuestion, } from "./store.js";
|
|
23
23
|
import { applyThrottleDecision, decideAdmission, defaultThrottleDeps, emptyThrottleState, evaluateThrottle, isCpuCheckEnabled, readAvailableMemMb, readThrottleState, resolveCpuBusyFraction, throttleStatePath, writeThrottleState, } from "./throttle.js";
|
|
24
24
|
import { readLanesRegistryFlag } from "./registry-flag.js";
|
|
25
|
-
import { LEAD_LANE_KEEP_ALIVE_FLAG, LEAD_LANE_KEEP_ALIVE_INTERRUPTED_RESUME_TEXT, LEAD_LANE_KEEP_ALIVE_MAX_RESUMES, LEAD_LANE_KEEP_ALIVE_RESUME_TEXT, LANE_SUPERSESSION_LINEAGE_AWARE_FLAG,
|
|
25
|
+
import { LEAD_LANE_KEEP_ALIVE_FLAG, LEAD_LANE_KEEP_ALIVE_INTERRUPTED_RESUME_TEXT, LEAD_LANE_KEEP_ALIVE_MAX_RESUMES, LEAD_LANE_KEEP_ALIVE_RESUME_TEXT, LANE_SUPERSESSION_LINEAGE_AWARE_FLAG, doneKeepAliveVerdict, hasPendingKeepAliveWork, keepAliveExclusion, loadRetiredLaneIds, leadLaneKeepAliveBackoffMs, undeliveredAnsweredQuestions, } from "./keep-alive.js";
|
|
26
26
|
import { providerSupportsSessionResume } from "./delivery.js";
|
|
27
27
|
import { applyDefaultQuotaHold, readQuotaSignal, recordAccountQuota } from "./quota.js";
|
|
28
28
|
import { answeredQuestionEntries, questionRecordRefusalCode, raiseEnvelopeQuestions, resolveQuestionCompanyWithFallback, } from "./questions.js";
|
|
@@ -848,7 +848,7 @@ export async function runPassBody(hqRoot, deps) {
|
|
|
848
848
|
if (!envelope || envelope.decision !== "done")
|
|
849
849
|
continue;
|
|
850
850
|
const hasQueuedWork = lane.keep_alive_standing === true && hasPendingKeepAliveWork(hqRoot, lane, questions);
|
|
851
|
-
if (
|
|
851
|
+
if (doneKeepAliveVerdict(lane, envelope, hasQueuedWork) !== "eligible")
|
|
852
852
|
continue;
|
|
853
853
|
}
|
|
854
854
|
const key = keepAliveCompanyKey(lane.company);
|
|
@@ -1137,10 +1137,8 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
|
|
|
1137
1137
|
if (terminalEnvelope && latest) {
|
|
1138
1138
|
const envelope = terminalEnvelope;
|
|
1139
1139
|
next.story_status = envelope.story_status;
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
next.story_complete_reason = envelope.story_complete_reason;
|
|
1143
|
-
}
|
|
1140
|
+
next.story_complete = envelope.story_complete;
|
|
1141
|
+
next.story_complete_reason = envelope.story_complete_reason;
|
|
1144
1142
|
const storyStatus = envelope.decision === "blocked"
|
|
1145
1143
|
? "blocked"
|
|
1146
1144
|
: envelope.decision === "done"
|
|
@@ -1550,6 +1548,13 @@ function recordKeepAliveSkip(hqRoot, lane, reason, summary, deps, detail) {
|
|
|
1550
1548
|
});
|
|
1551
1549
|
}
|
|
1552
1550
|
}
|
|
1551
|
+
function recordDoneKeepAlivePolicySkip(summary, lane, verdict) {
|
|
1552
|
+
summary.skipped.push({
|
|
1553
|
+
kind: "lane",
|
|
1554
|
+
id: lane.lane_id,
|
|
1555
|
+
why: `keep_alive_${verdict}`,
|
|
1556
|
+
});
|
|
1557
|
+
}
|
|
1553
1558
|
function readLatestInterruptedClassificationAt(hqRoot, lane) {
|
|
1554
1559
|
const logPath = watcherOutLogPath(hqRoot, lane.lane_id);
|
|
1555
1560
|
let body;
|
|
@@ -1624,7 +1629,10 @@ function resumeKeepAlive(hqRoot, lanes, questions, summary, deps, now, flagByCom
|
|
|
1624
1629
|
if (!envelope || envelope.decision !== "done")
|
|
1625
1630
|
continue;
|
|
1626
1631
|
const hasQueuedWork = lane.keep_alive_standing === true && hasPendingKeepAliveWork(hqRoot, lane, questions);
|
|
1627
|
-
|
|
1632
|
+
const verdict = doneKeepAliveVerdict(lane, envelope, hasQueuedWork);
|
|
1633
|
+
if (verdict !== "eligible") {
|
|
1634
|
+
if (verdict !== null)
|
|
1635
|
+
recordDoneKeepAlivePolicySkip(summary, lane, verdict);
|
|
1628
1636
|
if (envelope.story_complete === true && lane.keep_alive_standing !== true && hasKeepAliveProgress(lane)) {
|
|
1629
1637
|
casLaneIf(hqRoot, lane.lane_id, (current) => current.version === lane.version &&
|
|
1630
1638
|
current.state === "done" &&
|
|
@@ -1713,8 +1721,15 @@ function resumeKeepAlive(hqRoot, lanes, questions, summary, deps, now, flagByCom
|
|
|
1713
1721
|
if (!latest || latest.envelope.decision !== "done")
|
|
1714
1722
|
continue;
|
|
1715
1723
|
const hasQueuedWork = lane.keep_alive_standing === true && hasPendingKeepAliveWork(hqRoot, lane, questions);
|
|
1716
|
-
|
|
1724
|
+
const verdict = doneKeepAliveVerdict(lane, latest.envelope, hasQueuedWork);
|
|
1725
|
+
if (verdict !== "eligible") {
|
|
1726
|
+
if (verdict !== null)
|
|
1727
|
+
recordDoneKeepAlivePolicySkip(summary, lane, verdict);
|
|
1728
|
+
if (latest.envelope.story_complete === true && lane.keep_alive_standing !== true && hasKeepAliveProgress(lane)) {
|
|
1729
|
+
casLaneIf(hqRoot, lane.lane_id, (current) => current.version === lane.version && current.state === "done", (current) => ({ ...clearedKeepAliveProgress(current), keep_alive_blocked_reason: undefined }), deps.dryRun);
|
|
1730
|
+
}
|
|
1717
1731
|
continue;
|
|
1732
|
+
}
|
|
1718
1733
|
try {
|
|
1719
1734
|
eligibilityAtMs = latestEnvelopeMtimeMs(hqRoot, lane, latest.ref);
|
|
1720
1735
|
}
|
|
@@ -1788,7 +1803,10 @@ function resumeKeepAlive(hqRoot, lanes, questions, summary, deps, now, flagByCom
|
|
|
1788
1803
|
if (!currentEnvelope || currentEnvelope.envelope.decision !== "done")
|
|
1789
1804
|
continue;
|
|
1790
1805
|
const hasQueuedWork = current.keep_alive_standing === true && hasPendingKeepAliveWork(hqRoot, current, questions);
|
|
1791
|
-
|
|
1806
|
+
const verdict = doneKeepAliveVerdict(current, currentEnvelope.envelope, hasQueuedWork);
|
|
1807
|
+
if (verdict !== "eligible") {
|
|
1808
|
+
if (verdict !== null)
|
|
1809
|
+
recordDoneKeepAlivePolicySkip(summary, current, verdict);
|
|
1792
1810
|
if (currentEnvelope.envelope.story_complete === true && current.keep_alive_standing !== true && hasKeepAliveProgress(current)) {
|
|
1793
1811
|
casLaneIf(hqRoot, current.lane_id, (latestLane) => latestLane.version === current.version &&
|
|
1794
1812
|
latestLane.state === "done" &&
|
|
@@ -351,6 +351,26 @@ export interface ManagerStatusGateHistoryEntry {
|
|
|
351
351
|
resume_queued?: boolean;
|
|
352
352
|
resume_skipped_reason?: "open_question" | "blocked" | "miss_limit" | "no_provider_session";
|
|
353
353
|
}
|
|
354
|
+
export interface BlockerFreshnessState {
|
|
355
|
+
/** Stable hashes of the stale blocker labels seen at the last completed turn. */
|
|
356
|
+
stale_keys: string[];
|
|
357
|
+
stale_labels: string[];
|
|
358
|
+
/** Hash of the last stale set already escalated to the senior. */
|
|
359
|
+
escalated_fingerprint?: string;
|
|
360
|
+
}
|
|
361
|
+
export interface BlockerFreshnessHistoryEntry {
|
|
362
|
+
event: "blocker_stale" | "blocker_stale_escalated";
|
|
363
|
+
turn_id: string;
|
|
364
|
+
path: string;
|
|
365
|
+
at: string;
|
|
366
|
+
labels: string[];
|
|
367
|
+
count: number;
|
|
368
|
+
fingerprint: string;
|
|
369
|
+
resume_queued?: boolean;
|
|
370
|
+
resume_skipped_reason?: "open_question" | "blocked" | "miss_limit" | "no_provider_session";
|
|
371
|
+
delivery_failed?: true;
|
|
372
|
+
}
|
|
373
|
+
export type LaneGateHistoryEntry = ManagerStatusGateHistoryEntry | BlockerFreshnessHistoryEntry;
|
|
354
374
|
export interface LaneStory {
|
|
355
375
|
project_id: string;
|
|
356
376
|
story_id: string;
|
|
@@ -414,7 +434,7 @@ export interface LaneRecord {
|
|
|
414
434
|
worker_profile?: string;
|
|
415
435
|
/** Whether this lead lane should resume completed turns while its story is open. */
|
|
416
436
|
keep_alive?: boolean;
|
|
417
|
-
/** Whether
|
|
437
|
+
/** Whether completed keep-alive stories wait for queued work after a done turn. */
|
|
418
438
|
keep_alive_standing?: boolean;
|
|
419
439
|
/** Time this lane most recently explicitly opted into keep-alive. */
|
|
420
440
|
keep_alive_opted_in_at?: string;
|
|
@@ -493,7 +513,9 @@ export interface LaneRecord {
|
|
|
493
513
|
/** Turn-start baseline used by the optional manager status gate. */
|
|
494
514
|
status_gate?: ManagerStatusGate;
|
|
495
515
|
/** Bounded audit history of manager turns that missed their declared status artifact. */
|
|
496
|
-
status_gate_history?:
|
|
516
|
+
status_gate_history?: LaneGateHistoryEntry[];
|
|
517
|
+
/** Last blocker-freshness state used to detect consecutive stale turns. */
|
|
518
|
+
blocker_freshness_state?: BlockerFreshnessState;
|
|
497
519
|
/** Last automatic compaction of a resumed session (compaction.ts). */
|
|
498
520
|
last_compaction?: LaneCompactionRecord;
|
|
499
521
|
/**
|
|
@@ -833,6 +855,8 @@ export interface LanesConfig {
|
|
|
833
855
|
};
|
|
834
856
|
/** Seconds an unanswered question waits at a session audience before advancing. */
|
|
835
857
|
session_question_timeout_secs: number;
|
|
858
|
+
/** Age in hours after which an open manager blocker must be re-verified. */
|
|
859
|
+
blocker_freshness_hours: number;
|
|
836
860
|
}
|
|
837
861
|
export declare function parseMajorSchema(schemaVersion: string, expectedPrefix: string, expectedMajor: number): void;
|
|
838
862
|
export declare function parseLaneEnvelope(raw: unknown): LaneEnvelope;
|
package/dist/lib/lanes/types.js
CHANGED
|
@@ -228,7 +228,7 @@ export const LANE_FIELD_WRITERS = {
|
|
|
228
228
|
provider: ["creator"], account: ["creator", "spawn", "command", "watcher", "reconcile"], account_selection: ["creator", "command"], admission_accounts: ["creator", "command"], account_history: ["command", "spawn", "watcher", "reconcile"], failed_resume_history: ["command"], pending_resume: ["command", "spawn", "watcher", "reconcile"], placement_default: ["creator"], host: ["creator", "command"], origin_host: ["creator"], origin_host_source: ["creator"],
|
|
229
229
|
version: ["creator"], created_at: ["creator"], updated_at: ["creator"], attribution_history: ["command"],
|
|
230
230
|
last_envelope: ["watcher", "reconcile"], model: ["creator", "spawn", "command"], model_source: ["creator", "spawn", "command"], effort: ["creator", "spawn", "command"], effort_source: ["creator", "spawn", "command"], fast_mode: ["spawn"],
|
|
231
|
-
last_message_at: ["command"], status_gate: ["spawn", "watcher"], status_gate_history: ["watcher"], last_compaction: ["spawn"], story_complete: ["watcher", "reconcile"], story_complete_reason: ["watcher", "reconcile"], story_status: ["watcher", "reconcile"],
|
|
231
|
+
last_message_at: ["command"], status_gate: ["spawn", "watcher"], status_gate_history: ["watcher"], blocker_freshness_state: ["watcher"], last_compaction: ["spawn"], story_complete: ["watcher", "reconcile"], story_complete_reason: ["watcher", "reconcile"], story_status: ["watcher", "reconcile"],
|
|
232
232
|
overdue: ["reconcile"], overdue_at: ["reconcile"], last_host_report_at: ["watcher", "spawn", "reconcile"], stalled: ["watcher"], stalled_at: ["watcher"], stalled_reason: ["watcher"],
|
|
233
233
|
admitted_reason: ["command", "reconcile"], terminal_reason: ["command", "watcher", "spawn", "reconcile"], stop_history: ["command"], interrupted_at: ["watcher", "reconcile"], terminal_details: ["command", "spawn", "watcher", "reconcile"],
|
|
234
234
|
quota_reset_at: ["watcher", "reconcile"], model_quota_marker: ["watcher", "reconcile"], outpost_command_id: ["command"],
|
|
@@ -19,6 +19,7 @@ export function watcherMainDeps(hqRoot, laneId, env = process.env, repairDispatc
|
|
|
19
19
|
return {
|
|
20
20
|
resolveQuestionActiveCompany,
|
|
21
21
|
spawnRepair: (root, lane, input) => spawnEnvelopeRepairViaCurrentCli(root, lane, input, { ...repairDispatchDeps, env }),
|
|
22
|
+
deliverLaneSeniorMessage: async (seniorLaneId, text) => deliverLaneSeniorMessage(cliEntry, seniorLaneId, text),
|
|
22
23
|
recoverQueuedMessages: async () => {
|
|
23
24
|
const result = spawnSync(process.execPath, [cliEntry, "lanes", "_recover-queued-messages", laneId], {
|
|
24
25
|
env,
|
|
@@ -33,6 +34,30 @@ export function watcherMainDeps(hqRoot, laneId, env = process.env, repairDispatc
|
|
|
33
34
|
},
|
|
34
35
|
};
|
|
35
36
|
}
|
|
37
|
+
function deliverLaneSeniorMessage(cliEntry, laneId, text) {
|
|
38
|
+
const result = spawnSync(process.execPath, [cliEntry, "lanes", "message", laneId, "--text", text], {
|
|
39
|
+
env: process.env,
|
|
40
|
+
encoding: "utf8",
|
|
41
|
+
timeout: 120_000,
|
|
42
|
+
});
|
|
43
|
+
if (result.error)
|
|
44
|
+
throw result.error;
|
|
45
|
+
if (result.status !== 0) {
|
|
46
|
+
throw new Error(`senior lane message command exited ${result.status ?? result.signal ?? "without status"}`);
|
|
47
|
+
}
|
|
48
|
+
let response;
|
|
49
|
+
try {
|
|
50
|
+
response = JSON.parse((result.stdout ?? "").trim());
|
|
51
|
+
}
|
|
52
|
+
catch {
|
|
53
|
+
throw new Error("senior lane message command returned invalid JSON");
|
|
54
|
+
}
|
|
55
|
+
if (response === null || typeof response !== "object" || Array.isArray(response) ||
|
|
56
|
+
response.accepted !== true ||
|
|
57
|
+
response.lane_id !== laneId) {
|
|
58
|
+
throw new Error("senior lane message command did not confirm acceptance");
|
|
59
|
+
}
|
|
60
|
+
}
|
|
36
61
|
async function resolveQuestionActiveCompany() {
|
|
37
62
|
try {
|
|
38
63
|
const { company } = await openMeshTransport({});
|
|
@@ -227,8 +227,12 @@ export interface WatcherLoopDeps {
|
|
|
227
227
|
resolveQuestionActiveCompany?: () => Promise<LaneCompanyRef>;
|
|
228
228
|
/** Resume an atomically queued inbox after this watcher wins terminal CAS. */
|
|
229
229
|
recoverQueuedMessages?: (lane: LaneRecord) => Promise<void>;
|
|
230
|
+
/** Deliver a lane-senior escalation through the normal lane-message router. */
|
|
231
|
+
deliverLaneSeniorMessage?: (seniorLaneId: string, text: string) => Promise<void>;
|
|
230
232
|
/** Test seam; production reads the company-scoped hq-flags row. */
|
|
231
233
|
readManagerStatusGateFlag?: typeof readLanesRegistryFlag;
|
|
234
|
+
/** Test seam; production reads the company-scoped hq-flags row. */
|
|
235
|
+
readBlockerFreshnessFlag?: typeof readLanesRegistryFlag;
|
|
232
236
|
}
|
|
233
237
|
/**
|
|
234
238
|
* Wrap one mesh call. A failed emit is logged as `mesh_event_failed` with
|
|
@@ -35,21 +35,23 @@ import * as fs from "node:fs";
|
|
|
35
35
|
import * as path from "node:path";
|
|
36
36
|
import { LanesError } from "./errors.js";
|
|
37
37
|
import { questionRecordRefusalCode, raiseEnvelopeQuestions, resolveQuestionCompanyWithFallback, } from "./questions.js";
|
|
38
|
-
import { isTerminalLaneState, ENVELOPE_INVALID_REASON, parseLaneEnvelope, STORY_STATUSES, } from "./types.js";
|
|
38
|
+
import { isTerminalLaneState, ENVELOPE_INVALID_REASON, parseLaneEnvelope, STORY_STATUSES, LANE_PROVIDERS, } from "./types.js";
|
|
39
39
|
import { laneStories, storyStatusForEnvelope } from "./stories.js";
|
|
40
40
|
import { appendCappedHistory, readLane, retryUpdateLane, withLaneHireLocks } from "./store.js";
|
|
41
41
|
import { enforceRegistryApprovalGate } from "./approval-gate.js";
|
|
42
42
|
import { readWorkerRunResult } from "./delivery.js";
|
|
43
43
|
import { applyDefaultQuotaHold, quotaSignalFromFields, readQuotaSignal, recordAccountQuota, } from "./quota.js";
|
|
44
44
|
import { PROVIDER_REJECTED_REASON, QUOTA_STATUS, isProviderLoginFailure, providerRejectionDetails, readProviderRejection, } from "./provider-rejection.js";
|
|
45
|
-
import { dropboxInboxHasMessages, readToolActivity, toolActivityPath } from "./dropbox.js";
|
|
45
|
+
import { appendDropboxMessageAtPath, dropboxInboxHasMessages, readToolActivity, toolActivityPath } from "./dropbox.js";
|
|
46
46
|
import { loadLanesConfig, providerProgressEventMinIntervalMs, providerQuotaDefaultHoldMs } from "./config.js";
|
|
47
47
|
import { resolveAdmissionHost as resolveProcessAdmissionHost } from "./admission-host.js";
|
|
48
48
|
import { createLaneMesh, emitLaneDecisionEvents, emitLaneProgressEvent, emitLaneTerminalStateEvent, laneStoryStatusSuppressionKey, progressEventDue, } from "./mesh-events.js";
|
|
49
49
|
import { envelopeQuestionId } from "./questions.js";
|
|
50
50
|
import { readLanesRegistryFlag } from "./registry-flag.js";
|
|
51
|
+
import { blockerFreshnessFingerprint, resolveBlockerFreshnessForLane, } from "./blocker-freshness.js";
|
|
51
52
|
import { MANAGER_STATUS_GATE_FLAG, managerStatusWasWritten, readStatusSnapshot, resolveManagerStatusTarget, managerStatusGateEnabledForLane, } from "./manager-status-gate.js";
|
|
52
|
-
import { queueManagerStatusGateResume } from "./keep-alive.js";
|
|
53
|
+
import { queueLaneTerminalResume, queueManagerStatusGateResume } from "./keep-alive.js";
|
|
54
|
+
import { monitorSessionDir } from "../monitor.js";
|
|
53
55
|
import { laneRunDir, missingPidProbe, probePidRecord, readPidRecord, readProcStarttime, readSpawnManifest, envelopeRepairMode, spawnEnvelopeRepair, writePidRecord, retryLaneOnAlternateAccount, } from "./spawn.js";
|
|
54
56
|
/** Spec §5.7: no signal for 20 minutes while the process is still alive. */
|
|
55
57
|
export const SILENCE_MS = 20 * 60 * 1000;
|
|
@@ -403,10 +405,8 @@ export function applyTerminalToLaneWithResult(hqRoot, laneId, classification, op
|
|
|
403
405
|
...(envelope.blocked_reason !== undefined ? { blocked_reason: envelope.blocked_reason } : {}),
|
|
404
406
|
};
|
|
405
407
|
next.story_status = envelope.story_status;
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
next.story_complete_reason = envelope.story_complete_reason;
|
|
409
|
-
}
|
|
408
|
+
next.story_complete = envelope.story_complete;
|
|
409
|
+
next.story_complete_reason = envelope.story_complete_reason;
|
|
410
410
|
const currentProject = current.project_id;
|
|
411
411
|
const currentStory = current.story_id;
|
|
412
412
|
const storyStatus = envelope.decision === "blocked"
|
|
@@ -711,6 +711,171 @@ async function settleManagerStatusGate(hqRoot, lane, manifestTurnId, at, deps, l
|
|
|
711
711
|
}
|
|
712
712
|
return updated;
|
|
713
713
|
}
|
|
714
|
+
async function settleBlockerFreshness(hqRoot, lane, manifestTurnId, at, deps, mesh, logDecision) {
|
|
715
|
+
let errorLogged = false;
|
|
716
|
+
const resolution = await resolveBlockerFreshnessForLane(hqRoot, lane, {
|
|
717
|
+
readFlag: deps.readBlockerFreshnessFlag ?? readLanesRegistryFlag,
|
|
718
|
+
now: Date.parse(at),
|
|
719
|
+
onError: (error) => {
|
|
720
|
+
if (errorLogged)
|
|
721
|
+
return;
|
|
722
|
+
errorLogged = true;
|
|
723
|
+
const code = error instanceof LanesError ? error.error : "blocker_freshness_unavailable";
|
|
724
|
+
process.stderr.write(`hq lanes watcher: blocker freshness disabled for ${lane.lane_id}: ${code}\n`);
|
|
725
|
+
logDecision({ event: "blocker_freshness_disabled", at, error: code });
|
|
726
|
+
},
|
|
727
|
+
});
|
|
728
|
+
if (!resolution.enabled || resolution.statusPath === undefined || resolution.error !== undefined)
|
|
729
|
+
return lane;
|
|
730
|
+
if (manifestTurnId !== undefined && lane.status_gate !== undefined && lane.status_gate.turn_id !== manifestTurnId) {
|
|
731
|
+
logDecision({ event: "blocker_freshness_baseline_mismatch", at, turn_id: lane.status_gate.turn_id });
|
|
732
|
+
return lane;
|
|
733
|
+
}
|
|
734
|
+
if (resolution.stale.length === 0) {
|
|
735
|
+
if (lane.blocker_freshness_state === undefined)
|
|
736
|
+
return lane;
|
|
737
|
+
return retryUpdateLane(hqRoot, lane.lane_id, "watcher", (current) => current.blocker_freshness_state === undefined
|
|
738
|
+
? current
|
|
739
|
+
: { ...current, blocker_freshness_state: undefined });
|
|
740
|
+
}
|
|
741
|
+
const staleKeys = resolution.stale.map((blocker) => blocker.key).sort();
|
|
742
|
+
const staleLabels = resolution.stale.map((blocker) => blocker.label);
|
|
743
|
+
const fingerprint = blockerFreshnessFingerprint(resolution.stale);
|
|
744
|
+
const turnId = manifestTurnId ?? lane.status_gate?.turn_id ?? `unknown-${at}`;
|
|
745
|
+
const resumeText = `These open blockers in ${resolution.statusPath} were not re-verified in the last ${resolution.windowHours}h: ${staleLabels.join(", ")}. Check each one now; set verified_at on the ones that still hold and close the rest, then end the turn.`;
|
|
746
|
+
let deliverEscalation = false;
|
|
747
|
+
let escalationText = "";
|
|
748
|
+
let updated = retryUpdateLane(hqRoot, lane.lane_id, "watcher", (current) => {
|
|
749
|
+
deliverEscalation = false;
|
|
750
|
+
const prior = current.blocker_freshness_state;
|
|
751
|
+
const repeated = prior !== undefined && staleKeys.some((key) => prior.stale_keys.includes(key));
|
|
752
|
+
const history = current.status_gate_history ?? [];
|
|
753
|
+
const alreadyEscalatedThisTurn = history.some((entry) => entry.event === "blocker_stale_escalated" && entry.turn_id === turnId && entry.fingerprint === fingerprint);
|
|
754
|
+
const escalate = repeated && prior?.escalated_fingerprint !== fingerprint && !alreadyEscalatedThisTurn;
|
|
755
|
+
const alreadyEscalatedSet = prior?.escalated_fingerprint === fingerprint;
|
|
756
|
+
const statusResumeQueued = history.some((entry) => entry.event === "status_missed" && entry.turn_id === turnId && entry.resume_queued === true);
|
|
757
|
+
let resumeQueued = false;
|
|
758
|
+
let resumeSkippedReason;
|
|
759
|
+
let nextLane = current;
|
|
760
|
+
if (!escalate && !alreadyEscalatedSet) {
|
|
761
|
+
const resume = queueLaneTerminalResume(current, at, resumeText, {
|
|
762
|
+
mergeExisting: statusResumeQueued,
|
|
763
|
+
});
|
|
764
|
+
resumeQueued = resume.resumeQueued;
|
|
765
|
+
resumeSkippedReason = resume.resumeSkippedReason;
|
|
766
|
+
nextLane = resume.lane;
|
|
767
|
+
}
|
|
768
|
+
const staleEntry = {
|
|
769
|
+
event: "blocker_stale",
|
|
770
|
+
turn_id: turnId,
|
|
771
|
+
path: resolution.statusPath,
|
|
772
|
+
at,
|
|
773
|
+
labels: staleLabels,
|
|
774
|
+
count: resolution.stale.length,
|
|
775
|
+
fingerprint,
|
|
776
|
+
...(resumeQueued ? { resume_queued: true } : {}),
|
|
777
|
+
...(resumeSkippedReason !== undefined ? { resume_skipped_reason: resumeSkippedReason } : {}),
|
|
778
|
+
};
|
|
779
|
+
let nextHistory = history.some((entry) => entry.event === "blocker_stale" && entry.turn_id === turnId)
|
|
780
|
+
? [...history]
|
|
781
|
+
: appendCappedHistory(history, staleEntry);
|
|
782
|
+
let escalatedFingerprint = prior?.escalated_fingerprint;
|
|
783
|
+
if (escalate) {
|
|
784
|
+
const escalatedEntry = {
|
|
785
|
+
event: "blocker_stale_escalated",
|
|
786
|
+
turn_id: turnId,
|
|
787
|
+
path: resolution.statusPath,
|
|
788
|
+
at,
|
|
789
|
+
labels: staleLabels,
|
|
790
|
+
count: resolution.stale.length,
|
|
791
|
+
fingerprint,
|
|
792
|
+
};
|
|
793
|
+
nextHistory = appendCappedHistory(nextHistory, escalatedEntry);
|
|
794
|
+
escalatedFingerprint = fingerprint;
|
|
795
|
+
deliverEscalation = true;
|
|
796
|
+
escalationText = `Blocker freshness escalation for ${current.lane_id}: these blockers remain stale after consecutive turns: ${staleLabels.join(", ")}. Status file: ${resolution.statusPath}.`;
|
|
797
|
+
}
|
|
798
|
+
else if (prior !== undefined && blockerFreshnessFingerprint(prior.stale_keys.map((key, index) => ({ key, label: prior.stale_labels[index] ?? "" }))) !== fingerprint) {
|
|
799
|
+
escalatedFingerprint = undefined;
|
|
800
|
+
}
|
|
801
|
+
return {
|
|
802
|
+
...nextLane,
|
|
803
|
+
blocker_freshness_state: {
|
|
804
|
+
stale_keys: staleKeys,
|
|
805
|
+
stale_labels: staleLabels,
|
|
806
|
+
...(escalatedFingerprint !== undefined ? { escalated_fingerprint: escalatedFingerprint } : {}),
|
|
807
|
+
},
|
|
808
|
+
status_gate_history: nextHistory,
|
|
809
|
+
};
|
|
810
|
+
});
|
|
811
|
+
logDecision({ event: "blocker_stale", at, turn_id: turnId, path: resolution.statusPath, labels: staleLabels, count: staleLabels.length });
|
|
812
|
+
const escalated = updated.status_gate_history?.find((entry) => entry.event === "blocker_stale_escalated" && entry.turn_id === turnId && entry.fingerprint === fingerprint);
|
|
813
|
+
if (escalated && deliverEscalation) {
|
|
814
|
+
try {
|
|
815
|
+
if (lane.senior.kind === "lane" && lane.senior.id) {
|
|
816
|
+
if (deps.deliverLaneSeniorMessage === undefined) {
|
|
817
|
+
throw new LanesError("blocker_freshness_delivery_unavailable", "Lane-senior blocker escalations require the lane-message delivery path", { lane_id: lane.lane_id, senior_lane_id: lane.senior.id });
|
|
818
|
+
}
|
|
819
|
+
await deps.deliverLaneSeniorMessage(lane.senior.id, escalationText);
|
|
820
|
+
}
|
|
821
|
+
else if (lane.senior.kind === "session" && lane.senior.id) {
|
|
822
|
+
const inbox = resolveBlockerFreshnessSessionInbox(hqRoot, lane.senior.id);
|
|
823
|
+
appendDropboxMessageAtPath(inbox, {
|
|
824
|
+
at,
|
|
825
|
+
message_id: `blocker-freshness:${lane.lane_id}:${fingerprint}:${turnId}`,
|
|
826
|
+
kind: "blocker_stale_escalation",
|
|
827
|
+
text: escalationText,
|
|
828
|
+
});
|
|
829
|
+
}
|
|
830
|
+
else {
|
|
831
|
+
throw new LanesError("blocker_freshness_delivery_unavailable", "No durable senior inbox is available for this blocker escalation", { lane_id: lane.lane_id, senior_kind: lane.senior.kind });
|
|
832
|
+
}
|
|
833
|
+
logDecision({ event: "blocker_stale_escalation_delivered", at, turn_id: turnId, senior_kind: lane.senior.kind });
|
|
834
|
+
}
|
|
835
|
+
catch (error) {
|
|
836
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
837
|
+
process.stderr.write(`hq lanes watcher: blocker freshness escalation delivery failed for ${lane.lane_id}: ${message}\n`);
|
|
838
|
+
logDecision({ event: "blocker_stale_escalation_delivery_failed", at, turn_id: turnId, error: message });
|
|
839
|
+
updated = retryUpdateLane(hqRoot, lane.lane_id, "watcher", (current) => {
|
|
840
|
+
const state = current.blocker_freshness_state;
|
|
841
|
+
if (state?.escalated_fingerprint !== fingerprint)
|
|
842
|
+
return current;
|
|
843
|
+
const retryableState = { ...state };
|
|
844
|
+
delete retryableState.escalated_fingerprint;
|
|
845
|
+
return {
|
|
846
|
+
...current,
|
|
847
|
+
blocker_freshness_state: retryableState,
|
|
848
|
+
status_gate_history: (current.status_gate_history ?? []).map((entry) => entry.event === "blocker_stale_escalated"
|
|
849
|
+
&& entry.turn_id === turnId
|
|
850
|
+
&& entry.fingerprint === fingerprint
|
|
851
|
+
? { ...entry, delivery_failed: true }
|
|
852
|
+
: entry),
|
|
853
|
+
};
|
|
854
|
+
});
|
|
855
|
+
}
|
|
856
|
+
if (mesh !== null) {
|
|
857
|
+
await emitMeshSoft("blocker_freshness_escalation_signal", logDecision, at, (signal) => emitLaneProgressEvent(mesh, lane, escalationText, signal));
|
|
858
|
+
}
|
|
859
|
+
}
|
|
860
|
+
return updated;
|
|
861
|
+
}
|
|
862
|
+
function resolveBlockerFreshnessSessionInbox(hqRoot, sessionId) {
|
|
863
|
+
let candidates;
|
|
864
|
+
try {
|
|
865
|
+
candidates = LANE_PROVIDERS
|
|
866
|
+
.map((provider) => monitorSessionDir(hqRoot, provider, sessionId))
|
|
867
|
+
.filter((sessionDir) => fs.existsSync(sessionDir));
|
|
868
|
+
}
|
|
869
|
+
catch (error) {
|
|
870
|
+
throw new LanesError("blocker_freshness_session_inbox_unavailable", `Could not resolve the monitor inbox for session senior: ${error instanceof Error ? error.message : String(error)}`, { session_id: sessionId });
|
|
871
|
+
}
|
|
872
|
+
if (candidates.length !== 1) {
|
|
873
|
+
throw new LanesError("blocker_freshness_session_inbox_unavailable", candidates.length === 0
|
|
874
|
+
? "No monitor-session inbox is registered for the session senior"
|
|
875
|
+
: "The session senior matches more than one monitor-session inbox", { session_id: sessionId, matches: candidates.length });
|
|
876
|
+
}
|
|
877
|
+
return path.join(candidates[0], "dropbox");
|
|
878
|
+
}
|
|
714
879
|
/**
|
|
715
880
|
* Wrap one mesh call. A failed emit is logged as `mesh_event_failed` with
|
|
716
881
|
* the event name and the error message, and never propagates: the lane's
|
|
@@ -1282,11 +1447,12 @@ async function runWatcherLoopOwned(hqRoot, laneId, deps) {
|
|
|
1282
1447
|
}
|
|
1283
1448
|
if (terminalWrite.changed) {
|
|
1284
1449
|
updated = await settleManagerStatusGate(hqRoot, updated, manifest.turn_id, tick.emit.at, deps, logDecision);
|
|
1450
|
+
updated = await settleBlockerFreshness(hqRoot, updated, manifest.turn_id, tick.emit.at, deps, mesh, logDecision);
|
|
1285
1451
|
}
|
|
1286
1452
|
const managerStatusFollowupQueued = manifest.turn_id !== undefined &&
|
|
1287
|
-
updated.status_gate_history?.some((entry) => entry.
|
|
1288
|
-
entry.
|
|
1289
|
-
|
|
1453
|
+
updated.status_gate_history?.some((entry) => entry.turn_id === manifest.turn_id &&
|
|
1454
|
+
((entry.event === "status_missed" && entry.resume_queued === true) ||
|
|
1455
|
+
(entry.event === "blocker_stale" && entry.resume_queued === true))) === true;
|
|
1290
1456
|
if (terminalWrite.changed && deps.recoverQueuedMessages && !managerStatusFollowupQueued) {
|
|
1291
1457
|
try {
|
|
1292
1458
|
if (dropboxInboxHasMessages(hqRoot, laneId)) {
|