@indigoai-us/hq-cli 5.258.0 → 5.260.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +24 -0
- package/assets/scaffold/core/scripts/checkpoint-stop-gate.sh +20 -7
- package/dist/commands/lanes-message.js +3 -0
- package/dist/commands/lanes.js +14 -0
- package/dist/lib/doctor/checks/runtime-health.d.ts +2 -0
- package/dist/lib/doctor/checks/runtime-health.js +13 -3
- package/dist/lib/lanes/blocker-freshness.d.ts +31 -0
- package/dist/lib/lanes/blocker-freshness.js +167 -0
- package/dist/lib/lanes/config.d.ts +2 -0
- package/dist/lib/lanes/config.js +12 -0
- package/dist/lib/lanes/keep-alive.d.ts +10 -0
- package/dist/lib/lanes/keep-alive.js +57 -36
- package/dist/lib/lanes/types.d.ts +25 -1
- package/dist/lib/lanes/types.js +1 -1
- package/dist/lib/lanes/watcher-main.js +25 -0
- package/dist/lib/lanes/watcher.d.ts +4 -0
- package/dist/lib/lanes/watcher.js +174 -6
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,30 @@
|
|
|
2
2
|
|
|
3
3
|
## [Unreleased]
|
|
4
4
|
|
|
5
|
+
## [5.260.0] — 2026-09-28
|
|
6
|
+
|
|
7
|
+
### Added
|
|
8
|
+
|
|
9
|
+
- The `lanes.blocker-freshness` flag checks manager blockers after each turn.
|
|
10
|
+
It resumes once for stale blockers, then alerts the senior on a repeated
|
|
11
|
+
stale set. The window defaults to four hours and can be set with
|
|
12
|
+
`blocker_freshness_hours` in the lanes config.
|
|
13
|
+
|
|
14
|
+
### Fixed
|
|
15
|
+
|
|
16
|
+
- Blocker-freshness escalations now reach session seniors through the existing
|
|
17
|
+
monitor-session inbox. Work Mesh is an optional signal, not delivery, and a
|
|
18
|
+
failed delivery is retried on a later stale turn.
|
|
19
|
+
|
|
20
|
+
## [5.259.0] — 2026-09-28
|
|
21
|
+
|
|
22
|
+
### Fixed
|
|
23
|
+
|
|
24
|
+
- The checkpoint stop gate now recognizes path-qualified `hq core checkpoint`
|
|
25
|
+
commands without treating lookalike text as a checkpoint.
|
|
26
|
+
- `hq doctor --live-runtimes` now invokes Windows npm shims through their `.cmd`
|
|
27
|
+
entry points.
|
|
28
|
+
|
|
5
29
|
## [5.258.0] — 2026-09-28
|
|
6
30
|
|
|
7
31
|
### Fixed
|
|
@@ -350,7 +350,11 @@ exec 3>&2
|
|
|
350
350
|
else ""
|
|
351
351
|
end;
|
|
352
352
|
def checkpoint_command:
|
|
353
|
-
|
|
353
|
+
# Match the actual command at a shell command boundary. A whitespace
|
|
354
|
+
# boundary alone would accept `echo hq core checkpoint`; allow only the
|
|
355
|
+
# shell wrappers and environment assignments that can invoke hq itself.
|
|
356
|
+
# Quoted assignment values and executable paths may contain spaces.
|
|
357
|
+
test("(^\\s*|[;&|()]\\s*|\\n\\s*)(?:(?:command|exec)\\s+)?(?:env\\s+)?([A-Za-z_][A-Za-z0-9_]*=(?:\"(?:\\\\.|[^\"\\\\])*\"|\\u0027[^\\u0027]*\\u0027|[^\\s;&|()]*)\\s+)*(?:(?:command|exec|env)\\s+)?(\"[^\"]*[/\\\\]hq\"|[^\\s;&|()]*[/\\\\]hq|hq)\\s+core\\s+checkpoint(\\s|$)");
|
|
354
358
|
def strip_quoted:
|
|
355
359
|
# Remove quoted spans so flag text inside a --summary value (e.g.
|
|
356
360
|
# --summary "investigated --gate-probe behavior") is never mistaken for
|
|
@@ -362,12 +366,21 @@ exec 3>&2
|
|
|
362
366
|
def checkpoint_flag($flag):
|
|
363
367
|
strip_quoted | test("(^|\\s)" + $flag + "(\\s|$)");
|
|
364
368
|
def pure_checkpoint:
|
|
365
|
-
# The command is ONLY a checkpoint invocation (
|
|
366
|
-
#
|
|
367
|
-
# `git status; hq core checkpoint
|
|
368
|
-
# tool call, so its exemptions
|
|
369
|
-
|
|
370
|
-
|
|
369
|
+
# The command is ONLY a checkpoint invocation (recognized wrappers and
|
|
370
|
+
# env assignments are allowed, but no other command may be joined by a
|
|
371
|
+
# separator). A compound call like `git status; hq core checkpoint
|
|
372
|
+
# --idle` did other work in the same tool call, so its exemptions do not
|
|
373
|
+
# apply.
|
|
374
|
+
# Normalize a quoted or path-qualified executable to hq before
|
|
375
|
+
# stripping quoted argument values, so a path with spaces still gets
|
|
376
|
+
# the same --idle exemptions.
|
|
377
|
+
(capture("^(?<prefix>\\s*(?:(?:command|exec)\\s+)?(?:env\\s+)?([A-Za-z_][A-Za-z0-9_]*=(?:\"(?:\\\\.|[^\"\\\\])*\"|\\u0027[^\\u0027]*\\u0027|[^\\s;&|()]*)\\s+)*(?:(?:command|exec|env)\\s+)?)(?<executable>\"[^\"]*[/\\\\]hq\"|[^\\s;&|()]*[/\\\\]hq|hq)(?<tail>\\s+core\\s+checkpoint(?:\\s+[^;&|\n]*)?)$")? // false) as $parts
|
|
378
|
+
| if $parts == false then false
|
|
379
|
+
else
|
|
380
|
+
($parts.prefix + "hq" + $parts.tail)
|
|
381
|
+
| strip_quoted
|
|
382
|
+
| test("^\\s*(?:(?:command|exec)\\s+)?(?:env\\s+)?([A-Za-z_][A-Za-z0-9_]*=(?:\"(?:\\\\.|[^\"\\\\])*\"|\\u0027[^\\u0027]*\\u0027|[^\\s;&|()]*)\\s+)*(?:(?:command|exec|env)\\s+)?hq\\s+core\\s+checkpoint([\\s][^;&|\n]*)?$")
|
|
383
|
+
end;
|
|
371
384
|
def gate_feedback_user:
|
|
372
385
|
# This gate surfaces its own Stop feedback as a synthetic user row. That
|
|
373
386
|
# row is a turn boundary for the checkpoint requirement, but NOT for
|
|
@@ -92,6 +92,9 @@ export function createLaneMessageHandlers(deps) {
|
|
|
92
92
|
readStarttime: ctx.readStarttime ?? readProcStarttime,
|
|
93
93
|
mesh: null,
|
|
94
94
|
resolveQuestionActiveCompany: () => resolveActiveQuestionCompany(ctx),
|
|
95
|
+
deliverLaneSeniorMessage: async (seniorLaneId, seniorText) => {
|
|
96
|
+
await runMessage(ctx, seniorLaneId, { text: seniorText, silent: true });
|
|
97
|
+
},
|
|
95
98
|
});
|
|
96
99
|
lane = readLane(hqRoot, laneId);
|
|
97
100
|
// This invocation won the race to classify the dead worker, so the
|
package/dist/commands/lanes.js
CHANGED
|
@@ -59,6 +59,7 @@ import { LanesError } from "../lib/lanes/errors.js";
|
|
|
59
59
|
import { loadQuotaState, quotaBlockFor } from "../lib/lanes/quota.js";
|
|
60
60
|
import { SESSION_MOVE_REASON } from "../lib/lanes/move-account.js";
|
|
61
61
|
import { managerStatusGateEnabledForLane, resolveManagerStatusGateForLane, } from "../lib/lanes/manager-status-gate.js";
|
|
62
|
+
import { blockerFreshnessFingerprint, resolveBlockerFreshnessForLane, } from "../lib/lanes/blocker-freshness.js";
|
|
62
63
|
import { ensureSessionHome } from "../lib/lanes/session-relink.js";
|
|
63
64
|
import { defaultProbe, validateProfileLaneModel, validateLaneModel, } from "../lib/lanes/models.js";
|
|
64
65
|
import { parseCliModel, refuseOutpostSelfTarget, resolveLaneModel, FOREGROUND_ONLY_RULE, STANDING_AUTHORIZATION, spawnLaneWorker, spawnEnvelopeRepairRequest, laneRunDir, readSpawnManifest, stopLaneProcesses, } from "../lib/lanes/spawn.js";
|
|
@@ -4525,6 +4526,10 @@ async function runShow(ctx, laneId) {
|
|
|
4525
4526
|
const keepAliveState = resolveLaneKeepAlive(hqRoot, storedLane);
|
|
4526
4527
|
const managerStatusGate = await resolveManagerStatusGateForLane(hqRoot, storedLane, { env: ctx.env });
|
|
4527
4528
|
const managerStatusTarget = managerStatusGate.target;
|
|
4529
|
+
const blockerFreshness = await resolveBlockerFreshnessForLane(hqRoot, storedLane, {
|
|
4530
|
+
env: ctx.env,
|
|
4531
|
+
now: ctx.now().getTime(),
|
|
4532
|
+
});
|
|
4528
4533
|
printJson({
|
|
4529
4534
|
ok: true,
|
|
4530
4535
|
action: "show",
|
|
@@ -4579,6 +4584,15 @@ async function runShow(ctx, laneId) {
|
|
|
4579
4584
|
consecutive_misses: storedLane.status_gate?.consecutive_misses ?? 0,
|
|
4580
4585
|
}
|
|
4581
4586
|
: null,
|
|
4587
|
+
blocker_freshness: blockerFreshness.flagEnabled
|
|
4588
|
+
? {
|
|
4589
|
+
enabled: blockerFreshness.enabled,
|
|
4590
|
+
stale: blockerFreshness.stale.map((blocker) => blocker.label),
|
|
4591
|
+
escalated: blockerFreshness.stale.length > 0 &&
|
|
4592
|
+
storedLane.blocker_freshness_state?.escalated_fingerprint ===
|
|
4593
|
+
blockerFreshnessFingerprint(blockerFreshness.stale),
|
|
4594
|
+
}
|
|
4595
|
+
: null,
|
|
4582
4596
|
last_message_at: lane.last_message_at ?? null,
|
|
4583
4597
|
last_compaction: lane.last_compaction
|
|
4584
4598
|
? { ...lane.last_compaction, after: compactionAfterUsage(storedLane) }
|
|
@@ -80,6 +80,8 @@ export interface RuntimeHealthDeps {
|
|
|
80
80
|
execProbe?: (file: string, args: string[], timeoutMs: number) => Promise<ProbeOutcome>;
|
|
81
81
|
/** Per-probe timeout. Default {@link DEFAULT_PROBE_TIMEOUT_MS}. */
|
|
82
82
|
probeTimeoutMs?: number;
|
|
83
|
+
/** Runtime platform override for the spawn plan (normally process.platform). */
|
|
84
|
+
platform?: NodeJS.Platform;
|
|
83
85
|
}
|
|
84
86
|
/**
|
|
85
87
|
* The AI-runtime-health check family entry. Presence checks always run; the
|
|
@@ -29,6 +29,7 @@ import { spawn } from "node:child_process";
|
|
|
29
29
|
import * as fs from "node:fs";
|
|
30
30
|
import * as os from "node:os";
|
|
31
31
|
import * as path from "node:path";
|
|
32
|
+
import { planCommandSpawn } from "../../../utils/windows-spawn.js";
|
|
32
33
|
/** The id of the AI-runtime-health family. */
|
|
33
34
|
export const RUNTIMES_FAMILY_ID = "runtimes";
|
|
34
35
|
/** Common id prefix for every result this family emits. */
|
|
@@ -101,7 +102,8 @@ export async function checkRuntimeHealth(context, deps = {}) {
|
|
|
101
102
|
async function runRuntimeChecks(context, deps) {
|
|
102
103
|
const resolved = {
|
|
103
104
|
resolveBinary: deps.resolveBinary ?? resolveOnPath,
|
|
104
|
-
execProbe: deps.execProbe ??
|
|
105
|
+
execProbe: deps.execProbe ??
|
|
106
|
+
((file, args, timeoutMs) => defaultExecProbe(file, args, timeoutMs, deps.platform ?? process.platform)),
|
|
105
107
|
timeoutMs: deps.probeTimeoutMs ?? DEFAULT_PROBE_TIMEOUT_MS,
|
|
106
108
|
live: context.liveRuntimes === true,
|
|
107
109
|
};
|
|
@@ -268,11 +270,19 @@ export function resolveOnPath(binary) {
|
|
|
268
270
|
* CLI that reads piped stdin in non-TTY mode — blocks forever on a pipe that
|
|
269
271
|
* never reaches EOF, which turns every probe into a timeout.
|
|
270
272
|
*/
|
|
271
|
-
function defaultExecProbe(file, args, timeoutMs) {
|
|
273
|
+
function defaultExecProbe(file, args, timeoutMs, platform) {
|
|
272
274
|
return new Promise((resolve) => {
|
|
273
|
-
const
|
|
275
|
+
const command = platform === "win32" && !/\.(?:cmd|bat|exe|com)$/i.test(file)
|
|
276
|
+
? `${file}.cmd`
|
|
277
|
+
: file;
|
|
278
|
+
const plan = planCommandSpawn(command, args, platform);
|
|
279
|
+
const child = spawn(plan.cmd, plan.args, {
|
|
274
280
|
cwd: os.tmpdir(),
|
|
275
281
|
stdio: ["ignore", "pipe", "pipe"],
|
|
282
|
+
shell: plan.shell,
|
|
283
|
+
...(plan.windowsVerbatimArguments === undefined
|
|
284
|
+
? {}
|
|
285
|
+
: { windowsVerbatimArguments: plan.windowsVerbatimArguments }),
|
|
276
286
|
});
|
|
277
287
|
const cap = 1024 * 1024;
|
|
278
288
|
let stdout = "";
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { type ManagerStatusTarget } from "./manager-status-gate.js";
|
|
2
|
+
import { readLanesRegistryFlag } from "./registry-flag.js";
|
|
3
|
+
import type { LaneRecord } from "./types.js";
|
|
4
|
+
export declare const BLOCKER_FRESHNESS_FLAG = "lanes.blocker-freshness";
|
|
5
|
+
export declare const BLOCKER_FRESHNESS_DEFAULT_HOURS = 4;
|
|
6
|
+
export declare const BLOCKER_FRESHNESS_FUTURE_GRACE_MS: number;
|
|
7
|
+
export interface StaleBlocker {
|
|
8
|
+
key: string;
|
|
9
|
+
label: string;
|
|
10
|
+
}
|
|
11
|
+
export interface BlockerFreshnessResolution {
|
|
12
|
+
/** The company-scoped flag value, kept separate from manager applicability. */
|
|
13
|
+
flagEnabled: boolean;
|
|
14
|
+
/** True only when the flag is on and this lane has a valid manager target. */
|
|
15
|
+
enabled: boolean;
|
|
16
|
+
target?: ManagerStatusTarget;
|
|
17
|
+
statusPath?: string;
|
|
18
|
+
windowHours: number;
|
|
19
|
+
stale: StaleBlocker[];
|
|
20
|
+
error?: string;
|
|
21
|
+
}
|
|
22
|
+
export interface BlockerFreshnessResolutionOptions {
|
|
23
|
+
env?: NodeJS.ProcessEnv;
|
|
24
|
+
now?: number;
|
|
25
|
+
readFlag?: typeof readLanesRegistryFlag;
|
|
26
|
+
onError?: (error: unknown) => void;
|
|
27
|
+
}
|
|
28
|
+
export declare function blockerFreshnessFingerprint(stale: readonly StaleBlocker[]): string;
|
|
29
|
+
/** Resolve the flag before touching the manager mapping or status artifact. */
|
|
30
|
+
export declare function resolveBlockerFreshnessForLane(root: string, lane: LaneRecord, options?: BlockerFreshnessResolutionOptions): Promise<BlockerFreshnessResolution>;
|
|
31
|
+
//# sourceMappingURL=blocker-freshness.d.ts.map
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import * as fs from "node:fs";
|
|
3
|
+
import { LanesError } from "./errors.js";
|
|
4
|
+
import { DEFAULT_BLOCKER_FRESHNESS_HOURS, loadLanesConfig } from "./config.js";
|
|
5
|
+
import { resolveManagerStatusTarget } from "./manager-status-gate.js";
|
|
6
|
+
import { readLanesRegistryFlag } from "./registry-flag.js";
|
|
7
|
+
import { assertNoSecretText } from "./text-scan.js";
|
|
8
|
+
export const BLOCKER_FRESHNESS_FLAG = "lanes.blocker-freshness";
|
|
9
|
+
export const BLOCKER_FRESHNESS_DEFAULT_HOURS = DEFAULT_BLOCKER_FRESHNESS_HOURS;
|
|
10
|
+
export const BLOCKER_FRESHNESS_FUTURE_GRACE_MS = 5 * 60 * 1000;
|
|
11
|
+
const TERMINAL_BLOCKER_STATUSES = new Set([
|
|
12
|
+
"closed",
|
|
13
|
+
"resolved",
|
|
14
|
+
"done",
|
|
15
|
+
"superseded",
|
|
16
|
+
"cancelled",
|
|
17
|
+
]);
|
|
18
|
+
function isObject(value) {
|
|
19
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
20
|
+
}
|
|
21
|
+
function errorCode(error) {
|
|
22
|
+
return error instanceof LanesError ? error.error : "blocker_freshness_unavailable";
|
|
23
|
+
}
|
|
24
|
+
function reportError(lane, error) {
|
|
25
|
+
process.stderr.write(`hq lanes: blocker freshness disabled for ${lane.lane_id}: ${errorCode(error)}\n`);
|
|
26
|
+
}
|
|
27
|
+
function failedResolution(flagEnabled, error, onError) {
|
|
28
|
+
onError(error);
|
|
29
|
+
return {
|
|
30
|
+
flagEnabled,
|
|
31
|
+
enabled: false,
|
|
32
|
+
windowHours: BLOCKER_FRESHNESS_DEFAULT_HOURS,
|
|
33
|
+
stale: [],
|
|
34
|
+
error: errorCode(error),
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
function identityFor(blocker, index) {
|
|
38
|
+
for (const field of ["id", "item", "what", "reason"]) {
|
|
39
|
+
const candidate = blocker[field];
|
|
40
|
+
if (typeof candidate === "string" && candidate.trim().length > 0) {
|
|
41
|
+
const identity = candidate.trim();
|
|
42
|
+
assertNoSecretText(`blocker ${field}`, identity);
|
|
43
|
+
return {
|
|
44
|
+
key: blockerKey(`${field}:${identity}`),
|
|
45
|
+
label: identity.slice(0, 120),
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
const identity = `index:${index + 1}`;
|
|
50
|
+
const label = `blocker ${index + 1}`;
|
|
51
|
+
assertNoSecretText("blocker label", label);
|
|
52
|
+
return { key: blockerKey(identity), label };
|
|
53
|
+
}
|
|
54
|
+
function hasClosingTimestamp(blocker) {
|
|
55
|
+
return [blocker.closed_at, blocker.resolved_at].some((value) => value !== undefined && value !== null && value !== "");
|
|
56
|
+
}
|
|
57
|
+
function blockerKey(label) {
|
|
58
|
+
return createHash("sha256").update(label, "utf8").digest("hex");
|
|
59
|
+
}
|
|
60
|
+
function readBlockers(statusPath) {
|
|
61
|
+
let stat;
|
|
62
|
+
try {
|
|
63
|
+
stat = fs.lstatSync(statusPath);
|
|
64
|
+
}
|
|
65
|
+
catch (error) {
|
|
66
|
+
throw new LanesError("blocker_freshness_status_unreadable", `Cannot inspect manager status ${statusPath}: ${error instanceof Error ? error.message : String(error)}`, { path: statusPath });
|
|
67
|
+
}
|
|
68
|
+
if (!stat.isFile() || stat.isSymbolicLink()) {
|
|
69
|
+
throw new LanesError("blocker_freshness_status_invalid", `Manager status must be a regular file: ${statusPath}`, { path: statusPath });
|
|
70
|
+
}
|
|
71
|
+
let parsed;
|
|
72
|
+
try {
|
|
73
|
+
parsed = JSON.parse(fs.readFileSync(statusPath, "utf8"));
|
|
74
|
+
}
|
|
75
|
+
catch (error) {
|
|
76
|
+
throw new LanesError("blocker_freshness_status_invalid", `Manager status contains invalid JSON: ${statusPath}`, { path: statusPath, cause: error instanceof Error ? error.name : "UnknownError" });
|
|
77
|
+
}
|
|
78
|
+
if (!isObject(parsed) || !Array.isArray(parsed.blockers)) {
|
|
79
|
+
throw new LanesError("blocker_freshness_status_invalid", `Manager status must contain a top-level blockers array: ${statusPath}`, { path: statusPath });
|
|
80
|
+
}
|
|
81
|
+
return parsed.blockers.filter(isObject);
|
|
82
|
+
}
|
|
83
|
+
function isStale(verifiedAt, now, windowHours) {
|
|
84
|
+
if (typeof verifiedAt !== "string")
|
|
85
|
+
return true;
|
|
86
|
+
const verifiedAtMs = Date.parse(verifiedAt);
|
|
87
|
+
if (!Number.isFinite(verifiedAtMs))
|
|
88
|
+
return true;
|
|
89
|
+
if (verifiedAtMs - now > BLOCKER_FRESHNESS_FUTURE_GRACE_MS)
|
|
90
|
+
return true;
|
|
91
|
+
return now - verifiedAtMs > windowHours * 60 * 60 * 1000;
|
|
92
|
+
}
|
|
93
|
+
export function blockerFreshnessFingerprint(stale) {
|
|
94
|
+
const keys = stale.map((item) => item.key).sort();
|
|
95
|
+
return createHash("sha256").update(JSON.stringify(keys), "utf8").digest("hex");
|
|
96
|
+
}
|
|
97
|
+
/** Resolve the flag before touching the manager mapping or status artifact. */
|
|
98
|
+
export async function resolveBlockerFreshnessForLane(root, lane, options = {}) {
|
|
99
|
+
const onError = options.onError ?? ((error) => reportError(lane, error));
|
|
100
|
+
let flagEnabled;
|
|
101
|
+
try {
|
|
102
|
+
flagEnabled = await (options.readFlag ?? readLanesRegistryFlag)(BLOCKER_FRESHNESS_FLAG, {
|
|
103
|
+
company: lane.company ?? lane.mesh_company ?? undefined,
|
|
104
|
+
env: options.env,
|
|
105
|
+
});
|
|
106
|
+
}
|
|
107
|
+
catch (error) {
|
|
108
|
+
return failedResolution(false, error, onError);
|
|
109
|
+
}
|
|
110
|
+
if (!flagEnabled) {
|
|
111
|
+
return {
|
|
112
|
+
flagEnabled: false,
|
|
113
|
+
enabled: false,
|
|
114
|
+
windowHours: BLOCKER_FRESHNESS_DEFAULT_HOURS,
|
|
115
|
+
stale: [],
|
|
116
|
+
};
|
|
117
|
+
}
|
|
118
|
+
let target;
|
|
119
|
+
try {
|
|
120
|
+
target = resolveManagerStatusTarget(root, lane);
|
|
121
|
+
}
|
|
122
|
+
catch (error) {
|
|
123
|
+
return failedResolution(true, error, onError);
|
|
124
|
+
}
|
|
125
|
+
if (!target.manager || target.statusPath === undefined) {
|
|
126
|
+
return {
|
|
127
|
+
flagEnabled: true,
|
|
128
|
+
enabled: false,
|
|
129
|
+
target,
|
|
130
|
+
windowHours: BLOCKER_FRESHNESS_DEFAULT_HOURS,
|
|
131
|
+
stale: [],
|
|
132
|
+
};
|
|
133
|
+
}
|
|
134
|
+
let windowHours;
|
|
135
|
+
let blockers;
|
|
136
|
+
try {
|
|
137
|
+
windowHours = loadLanesConfig(root).blocker_freshness_hours;
|
|
138
|
+
blockers = readBlockers(target.statusPath);
|
|
139
|
+
}
|
|
140
|
+
catch (error) {
|
|
141
|
+
return failedResolution(true, error, onError);
|
|
142
|
+
}
|
|
143
|
+
const now = options.now ?? Date.now();
|
|
144
|
+
const stale = [];
|
|
145
|
+
try {
|
|
146
|
+
for (const [index, blocker] of blockers.entries()) {
|
|
147
|
+
const status = typeof blocker.status === "string" ? blocker.status.trim().toLowerCase() : "";
|
|
148
|
+
if (TERMINAL_BLOCKER_STATUSES.has(status) || hasClosingTimestamp(blocker))
|
|
149
|
+
continue;
|
|
150
|
+
if (!isStale(blocker.verified_at, now, windowHours))
|
|
151
|
+
continue;
|
|
152
|
+
stale.push(identityFor(blocker, index));
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
catch (error) {
|
|
156
|
+
return failedResolution(true, error, onError);
|
|
157
|
+
}
|
|
158
|
+
return {
|
|
159
|
+
flagEnabled: true,
|
|
160
|
+
enabled: true,
|
|
161
|
+
target,
|
|
162
|
+
statusPath: target.statusPath,
|
|
163
|
+
windowHours,
|
|
164
|
+
stale,
|
|
165
|
+
};
|
|
166
|
+
}
|
|
167
|
+
//# sourceMappingURL=blocker-freshness.js.map
|
|
@@ -39,6 +39,8 @@ export declare const DEFAULT_TURN_START_LOST_MS: number;
|
|
|
39
39
|
export declare const DEFAULT_SILENT_LOST_MS: number;
|
|
40
40
|
/** Unanswered session questions move to the next audience after 10 minutes. */
|
|
41
41
|
export declare const DEFAULT_SESSION_QUESTION_TIMEOUT_SECS: number;
|
|
42
|
+
/** Open manager blockers are re-verified every four hours by default. */
|
|
43
|
+
export declare const DEFAULT_BLOCKER_FRESHNESS_HOURS = 4;
|
|
42
44
|
export declare const DEFAULT_TRACKER_CHECKS: TrackerChecksConfig;
|
|
43
45
|
export declare const DEFAULT_LIVENESS_WINDOWS: LivenessWindows;
|
|
44
46
|
export declare const DEFAULT_PROVIDER_MAX_RUNNING: Record<LaneProvider, number>;
|
package/dist/lib/lanes/config.js
CHANGED
|
@@ -43,6 +43,8 @@ export const DEFAULT_TURN_START_LOST_MS = 2 * 60 * 60 * 1000;
|
|
|
43
43
|
export const DEFAULT_SILENT_LOST_MS = 12 * 60 * 60 * 1000;
|
|
44
44
|
/** Unanswered session questions move to the next audience after 10 minutes. */
|
|
45
45
|
export const DEFAULT_SESSION_QUESTION_TIMEOUT_SECS = 10 * 60;
|
|
46
|
+
/** Open manager blockers are re-verified every four hours by default. */
|
|
47
|
+
export const DEFAULT_BLOCKER_FRESHNESS_HOURS = 4;
|
|
46
48
|
export const DEFAULT_TRACKER_CHECKS = {
|
|
47
49
|
owner_question_surface_ms: 10 * 60 * 1000,
|
|
48
50
|
open_question_age_ms: 2 * 60 * 60 * 1000,
|
|
@@ -141,6 +143,14 @@ function positiveInt(value, field, fallback) {
|
|
|
141
143
|
}
|
|
142
144
|
return value;
|
|
143
145
|
}
|
|
146
|
+
function positiveNumber(value, field, fallback) {
|
|
147
|
+
if (value === undefined)
|
|
148
|
+
return fallback;
|
|
149
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value <= 0) {
|
|
150
|
+
throw new LanesConfigError("invalid_config", `${field} must be a positive finite number`, { field, value });
|
|
151
|
+
}
|
|
152
|
+
return value;
|
|
153
|
+
}
|
|
144
154
|
function unitInterval(value, field, fallback) {
|
|
145
155
|
if (value === undefined)
|
|
146
156
|
return fallback;
|
|
@@ -414,6 +424,7 @@ export function documentedDefaultConfig(deps = defaultDeps()) {
|
|
|
414
424
|
liveness: { ...DEFAULT_LIVENESS_WINDOWS },
|
|
415
425
|
tracker: { checks: { ...DEFAULT_TRACKER_CHECKS } },
|
|
416
426
|
session_question_timeout_secs: DEFAULT_SESSION_QUESTION_TIMEOUT_SECS,
|
|
427
|
+
blocker_freshness_hours: DEFAULT_BLOCKER_FRESHNESS_HOURS,
|
|
417
428
|
};
|
|
418
429
|
}
|
|
419
430
|
export function parseLanesConfig(raw, deps = defaultDeps()) {
|
|
@@ -456,6 +467,7 @@ export function parseLanesConfig(raw, deps = defaultDeps()) {
|
|
|
456
467
|
liveness: parseLiveness(raw.liveness),
|
|
457
468
|
tracker: { checks: parseTrackerChecks(raw.tracker) },
|
|
458
469
|
session_question_timeout_secs: positiveInt(raw.session_question_timeout_secs, "session_question_timeout_secs", DEFAULT_SESSION_QUESTION_TIMEOUT_SECS),
|
|
470
|
+
blocker_freshness_hours: positiveNumber(raw.blocker_freshness_hours, "blocker_freshness_hours", DEFAULT_BLOCKER_FRESHNESS_HOURS),
|
|
459
471
|
};
|
|
460
472
|
}
|
|
461
473
|
export function lanesConfigPath(hqRoot) {
|
|
@@ -7,6 +7,16 @@ export declare const LANE_SUPERSESSION_LINEAGE_AWARE_DEFAULT_ENABLED = false;
|
|
|
7
7
|
export declare const LEAD_LANE_KEEP_ALIVE_MAX_RESUMES = 24;
|
|
8
8
|
export declare const LEAD_LANE_KEEP_ALIVE_BASE_BACKOFF_MS: number;
|
|
9
9
|
export declare const LEAD_LANE_KEEP_ALIVE_MAX_BACKOFF_MS: number;
|
|
10
|
+
export interface LaneTerminalResumeResult {
|
|
11
|
+
lane: LaneRecord;
|
|
12
|
+
resumeQueued: boolean;
|
|
13
|
+
resumeSkippedReason?: "open_question" | "blocked" | "miss_limit" | "no_provider_session";
|
|
14
|
+
}
|
|
15
|
+
/** Shared resume transition for terminal gate follow-ups. */
|
|
16
|
+
export declare function queueLaneTerminalResume(current: LaneRecord, at: string, resumeText: string, options?: {
|
|
17
|
+
allowResume?: boolean;
|
|
18
|
+
mergeExisting?: boolean;
|
|
19
|
+
}): LaneTerminalResumeResult;
|
|
10
20
|
/** Persist a manager status miss and queue its one bounded follow-up turn. */
|
|
11
21
|
export declare function queueManagerStatusGateResume(hqRoot: string, laneId: string, entry: Omit<ManagerStatusGateHistoryEntry, "event" | "resume_queued" | "resume_skipped_reason">, resumeText: string): LaneRecord;
|
|
12
22
|
export declare const LEAD_LANE_KEEP_ALIVE_RESUME_TEXT = "Automatic keep-alive resume: your last turn ended with decision done and story_complete false. Continue the open work from where you stopped. End with done only when story_complete is true; if you are waiting, use hq lanes ask or stay in the loop.";
|
|
@@ -11,6 +11,53 @@ export const LANE_SUPERSESSION_LINEAGE_AWARE_DEFAULT_ENABLED = false;
|
|
|
11
11
|
export const LEAD_LANE_KEEP_ALIVE_MAX_RESUMES = 24;
|
|
12
12
|
export const LEAD_LANE_KEEP_ALIVE_BASE_BACKOFF_MS = 5 * 60 * 1000;
|
|
13
13
|
export const LEAD_LANE_KEEP_ALIVE_MAX_BACKOFF_MS = 60 * 60 * 1000;
|
|
14
|
+
/** Shared resume transition for terminal gate follow-ups. */
|
|
15
|
+
export function queueLaneTerminalResume(current, at, resumeText, options = {}) {
|
|
16
|
+
if (options.mergeExisting && current.pending_resume !== undefined) {
|
|
17
|
+
return {
|
|
18
|
+
lane: {
|
|
19
|
+
...current,
|
|
20
|
+
pending_resume: {
|
|
21
|
+
...current.pending_resume,
|
|
22
|
+
text: `${current.pending_resume.text}\n\n${resumeText}`,
|
|
23
|
+
},
|
|
24
|
+
},
|
|
25
|
+
resumeQueued: true,
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
if (current.state === "blocked")
|
|
29
|
+
return { lane: current, resumeQueued: false, resumeSkippedReason: "blocked" };
|
|
30
|
+
if (current.state === "awaiting_input")
|
|
31
|
+
return { lane: current, resumeQueued: false, resumeSkippedReason: "open_question" };
|
|
32
|
+
if (options.allowResume === false)
|
|
33
|
+
return { lane: current, resumeQueued: false, resumeSkippedReason: "miss_limit" };
|
|
34
|
+
if (current.state !== "done" && current.state !== "interrupted" && current.state !== "failed") {
|
|
35
|
+
return { lane: current, resumeQueued: false };
|
|
36
|
+
}
|
|
37
|
+
if (!current.provider_session_id) {
|
|
38
|
+
return { lane: current, resumeQueued: false, resumeSkippedReason: "no_provider_session" };
|
|
39
|
+
}
|
|
40
|
+
return {
|
|
41
|
+
lane: {
|
|
42
|
+
...current,
|
|
43
|
+
state: "queued",
|
|
44
|
+
queued_reason: undefined,
|
|
45
|
+
queued_at: at,
|
|
46
|
+
admission_reserved_at: undefined,
|
|
47
|
+
terminal_reason: undefined,
|
|
48
|
+
terminal_details: undefined,
|
|
49
|
+
pending_resume: {
|
|
50
|
+
session_id: current.provider_session_id,
|
|
51
|
+
trigger: "resume",
|
|
52
|
+
resumed_from: current.state,
|
|
53
|
+
text: resumeText,
|
|
54
|
+
compact: true,
|
|
55
|
+
at,
|
|
56
|
+
},
|
|
57
|
+
},
|
|
58
|
+
resumeQueued: true,
|
|
59
|
+
};
|
|
60
|
+
}
|
|
14
61
|
/** Persist a manager status miss and queue its one bounded follow-up turn. */
|
|
15
62
|
export function queueManagerStatusGateResume(hqRoot, laneId, entry, resumeText) {
|
|
16
63
|
return retryUpdateLane(hqRoot, laneId, "watcher", (current) => {
|
|
@@ -22,47 +69,21 @@ export function queueManagerStatusGateResume(hqRoot, laneId, entry, resumeText)
|
|
|
22
69
|
return current;
|
|
23
70
|
}
|
|
24
71
|
const consecutiveMisses = gate.consecutive_misses + 1;
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
resumeSkippedReason = "open_question";
|
|
30
|
-
else if (consecutiveMisses > 1)
|
|
31
|
-
resumeSkippedReason = "miss_limit";
|
|
32
|
-
else if (!current.provider_session_id)
|
|
33
|
-
resumeSkippedReason = "no_provider_session";
|
|
34
|
-
const resumeQueued = resumeSkippedReason === undefined &&
|
|
35
|
-
(current.state === "done" || current.state === "interrupted" || current.state === "failed");
|
|
72
|
+
const transitioned = queueLaneTerminalResume({
|
|
73
|
+
...current,
|
|
74
|
+
status_gate: { ...gate, consecutive_misses: consecutiveMisses },
|
|
75
|
+
}, entry.at, resumeText, { allowResume: consecutiveMisses <= 1 });
|
|
36
76
|
const missed = {
|
|
37
77
|
event: "status_missed",
|
|
38
78
|
...entry,
|
|
39
|
-
...(resumeQueued ? { resume_queued: true } : {}),
|
|
40
|
-
...(resumeSkippedReason !== undefined
|
|
79
|
+
...(transitioned.resumeQueued ? { resume_queued: true } : {}),
|
|
80
|
+
...(transitioned.resumeSkippedReason !== undefined
|
|
81
|
+
? { resume_skipped_reason: transitioned.resumeSkippedReason }
|
|
82
|
+
: {}),
|
|
41
83
|
};
|
|
42
|
-
const next = {
|
|
43
|
-
...current,
|
|
44
|
-
status_gate: { ...gate, consecutive_misses: consecutiveMisses },
|
|
45
|
-
status_gate_history: appendCappedHistory(history, missed),
|
|
46
|
-
};
|
|
47
|
-
if (!resumeQueued || !current.provider_session_id)
|
|
48
|
-
return next;
|
|
49
|
-
const resumedFrom = current.state;
|
|
50
84
|
return {
|
|
51
|
-
...
|
|
52
|
-
|
|
53
|
-
queued_reason: undefined,
|
|
54
|
-
queued_at: entry.at,
|
|
55
|
-
admission_reserved_at: undefined,
|
|
56
|
-
terminal_reason: undefined,
|
|
57
|
-
terminal_details: undefined,
|
|
58
|
-
pending_resume: {
|
|
59
|
-
session_id: current.provider_session_id,
|
|
60
|
-
trigger: "resume",
|
|
61
|
-
resumed_from: resumedFrom,
|
|
62
|
-
text: resumeText,
|
|
63
|
-
compact: true,
|
|
64
|
-
at: entry.at,
|
|
65
|
-
},
|
|
85
|
+
...transitioned.lane,
|
|
86
|
+
status_gate_history: appendCappedHistory(history, missed),
|
|
66
87
|
};
|
|
67
88
|
});
|
|
68
89
|
}
|
|
@@ -351,6 +351,26 @@ export interface ManagerStatusGateHistoryEntry {
|
|
|
351
351
|
resume_queued?: boolean;
|
|
352
352
|
resume_skipped_reason?: "open_question" | "blocked" | "miss_limit" | "no_provider_session";
|
|
353
353
|
}
|
|
354
|
+
export interface BlockerFreshnessState {
|
|
355
|
+
/** Stable hashes of the stale blocker labels seen at the last completed turn. */
|
|
356
|
+
stale_keys: string[];
|
|
357
|
+
stale_labels: string[];
|
|
358
|
+
/** Hash of the last stale set already escalated to the senior. */
|
|
359
|
+
escalated_fingerprint?: string;
|
|
360
|
+
}
|
|
361
|
+
export interface BlockerFreshnessHistoryEntry {
|
|
362
|
+
event: "blocker_stale" | "blocker_stale_escalated";
|
|
363
|
+
turn_id: string;
|
|
364
|
+
path: string;
|
|
365
|
+
at: string;
|
|
366
|
+
labels: string[];
|
|
367
|
+
count: number;
|
|
368
|
+
fingerprint: string;
|
|
369
|
+
resume_queued?: boolean;
|
|
370
|
+
resume_skipped_reason?: "open_question" | "blocked" | "miss_limit" | "no_provider_session";
|
|
371
|
+
delivery_failed?: true;
|
|
372
|
+
}
|
|
373
|
+
export type LaneGateHistoryEntry = ManagerStatusGateHistoryEntry | BlockerFreshnessHistoryEntry;
|
|
354
374
|
export interface LaneStory {
|
|
355
375
|
project_id: string;
|
|
356
376
|
story_id: string;
|
|
@@ -493,7 +513,9 @@ export interface LaneRecord {
|
|
|
493
513
|
/** Turn-start baseline used by the optional manager status gate. */
|
|
494
514
|
status_gate?: ManagerStatusGate;
|
|
495
515
|
/** Bounded audit history of manager turns that missed their declared status artifact. */
|
|
496
|
-
status_gate_history?:
|
|
516
|
+
status_gate_history?: LaneGateHistoryEntry[];
|
|
517
|
+
/** Last blocker-freshness state used to detect consecutive stale turns. */
|
|
518
|
+
blocker_freshness_state?: BlockerFreshnessState;
|
|
497
519
|
/** Last automatic compaction of a resumed session (compaction.ts). */
|
|
498
520
|
last_compaction?: LaneCompactionRecord;
|
|
499
521
|
/**
|
|
@@ -833,6 +855,8 @@ export interface LanesConfig {
|
|
|
833
855
|
};
|
|
834
856
|
/** Seconds an unanswered question waits at a session audience before advancing. */
|
|
835
857
|
session_question_timeout_secs: number;
|
|
858
|
+
/** Age in hours after which an open manager blocker must be re-verified. */
|
|
859
|
+
blocker_freshness_hours: number;
|
|
836
860
|
}
|
|
837
861
|
export declare function parseMajorSchema(schemaVersion: string, expectedPrefix: string, expectedMajor: number): void;
|
|
838
862
|
export declare function parseLaneEnvelope(raw: unknown): LaneEnvelope;
|
package/dist/lib/lanes/types.js
CHANGED
|
@@ -228,7 +228,7 @@ export const LANE_FIELD_WRITERS = {
|
|
|
228
228
|
provider: ["creator"], account: ["creator", "spawn", "command", "watcher", "reconcile"], account_selection: ["creator", "command"], admission_accounts: ["creator", "command"], account_history: ["command", "spawn", "watcher", "reconcile"], failed_resume_history: ["command"], pending_resume: ["command", "spawn", "watcher", "reconcile"], placement_default: ["creator"], host: ["creator", "command"], origin_host: ["creator"], origin_host_source: ["creator"],
|
|
229
229
|
version: ["creator"], created_at: ["creator"], updated_at: ["creator"], attribution_history: ["command"],
|
|
230
230
|
last_envelope: ["watcher", "reconcile"], model: ["creator", "spawn", "command"], model_source: ["creator", "spawn", "command"], effort: ["creator", "spawn", "command"], effort_source: ["creator", "spawn", "command"], fast_mode: ["spawn"],
|
|
231
|
-
last_message_at: ["command"], status_gate: ["spawn", "watcher"], status_gate_history: ["watcher"], last_compaction: ["spawn"], story_complete: ["watcher", "reconcile"], story_complete_reason: ["watcher", "reconcile"], story_status: ["watcher", "reconcile"],
|
|
231
|
+
last_message_at: ["command"], status_gate: ["spawn", "watcher"], status_gate_history: ["watcher"], blocker_freshness_state: ["watcher"], last_compaction: ["spawn"], story_complete: ["watcher", "reconcile"], story_complete_reason: ["watcher", "reconcile"], story_status: ["watcher", "reconcile"],
|
|
232
232
|
overdue: ["reconcile"], overdue_at: ["reconcile"], last_host_report_at: ["watcher", "spawn", "reconcile"], stalled: ["watcher"], stalled_at: ["watcher"], stalled_reason: ["watcher"],
|
|
233
233
|
admitted_reason: ["command", "reconcile"], terminal_reason: ["command", "watcher", "spawn", "reconcile"], stop_history: ["command"], interrupted_at: ["watcher", "reconcile"], terminal_details: ["command", "spawn", "watcher", "reconcile"],
|
|
234
234
|
quota_reset_at: ["watcher", "reconcile"], model_quota_marker: ["watcher", "reconcile"], outpost_command_id: ["command"],
|
|
@@ -19,6 +19,7 @@ export function watcherMainDeps(hqRoot, laneId, env = process.env, repairDispatc
|
|
|
19
19
|
return {
|
|
20
20
|
resolveQuestionActiveCompany,
|
|
21
21
|
spawnRepair: (root, lane, input) => spawnEnvelopeRepairViaCurrentCli(root, lane, input, { ...repairDispatchDeps, env }),
|
|
22
|
+
deliverLaneSeniorMessage: async (seniorLaneId, text) => deliverLaneSeniorMessage(cliEntry, seniorLaneId, text),
|
|
22
23
|
recoverQueuedMessages: async () => {
|
|
23
24
|
const result = spawnSync(process.execPath, [cliEntry, "lanes", "_recover-queued-messages", laneId], {
|
|
24
25
|
env,
|
|
@@ -33,6 +34,30 @@ export function watcherMainDeps(hqRoot, laneId, env = process.env, repairDispatc
|
|
|
33
34
|
},
|
|
34
35
|
};
|
|
35
36
|
}
|
|
37
|
+
function deliverLaneSeniorMessage(cliEntry, laneId, text) {
|
|
38
|
+
const result = spawnSync(process.execPath, [cliEntry, "lanes", "message", laneId, "--text", text], {
|
|
39
|
+
env: process.env,
|
|
40
|
+
encoding: "utf8",
|
|
41
|
+
timeout: 120_000,
|
|
42
|
+
});
|
|
43
|
+
if (result.error)
|
|
44
|
+
throw result.error;
|
|
45
|
+
if (result.status !== 0) {
|
|
46
|
+
throw new Error(`senior lane message command exited ${result.status ?? result.signal ?? "without status"}`);
|
|
47
|
+
}
|
|
48
|
+
let response;
|
|
49
|
+
try {
|
|
50
|
+
response = JSON.parse((result.stdout ?? "").trim());
|
|
51
|
+
}
|
|
52
|
+
catch {
|
|
53
|
+
throw new Error("senior lane message command returned invalid JSON");
|
|
54
|
+
}
|
|
55
|
+
if (response === null || typeof response !== "object" || Array.isArray(response) ||
|
|
56
|
+
response.accepted !== true ||
|
|
57
|
+
response.lane_id !== laneId) {
|
|
58
|
+
throw new Error("senior lane message command did not confirm acceptance");
|
|
59
|
+
}
|
|
60
|
+
}
|
|
36
61
|
async function resolveQuestionActiveCompany() {
|
|
37
62
|
try {
|
|
38
63
|
const { company } = await openMeshTransport({});
|
|
@@ -227,8 +227,12 @@ export interface WatcherLoopDeps {
|
|
|
227
227
|
resolveQuestionActiveCompany?: () => Promise<LaneCompanyRef>;
|
|
228
228
|
/** Resume an atomically queued inbox after this watcher wins terminal CAS. */
|
|
229
229
|
recoverQueuedMessages?: (lane: LaneRecord) => Promise<void>;
|
|
230
|
+
/** Deliver a lane-senior escalation through the normal lane-message router. */
|
|
231
|
+
deliverLaneSeniorMessage?: (seniorLaneId: string, text: string) => Promise<void>;
|
|
230
232
|
/** Test seam; production reads the company-scoped hq-flags row. */
|
|
231
233
|
readManagerStatusGateFlag?: typeof readLanesRegistryFlag;
|
|
234
|
+
/** Test seam; production reads the company-scoped hq-flags row. */
|
|
235
|
+
readBlockerFreshnessFlag?: typeof readLanesRegistryFlag;
|
|
232
236
|
}
|
|
233
237
|
/**
|
|
234
238
|
* Wrap one mesh call. A failed emit is logged as `mesh_event_failed` with
|
|
@@ -35,21 +35,23 @@ import * as fs from "node:fs";
|
|
|
35
35
|
import * as path from "node:path";
|
|
36
36
|
import { LanesError } from "./errors.js";
|
|
37
37
|
import { questionRecordRefusalCode, raiseEnvelopeQuestions, resolveQuestionCompanyWithFallback, } from "./questions.js";
|
|
38
|
-
import { isTerminalLaneState, ENVELOPE_INVALID_REASON, parseLaneEnvelope, STORY_STATUSES, } from "./types.js";
|
|
38
|
+
import { isTerminalLaneState, ENVELOPE_INVALID_REASON, parseLaneEnvelope, STORY_STATUSES, LANE_PROVIDERS, } from "./types.js";
|
|
39
39
|
import { laneStories, storyStatusForEnvelope } from "./stories.js";
|
|
40
40
|
import { appendCappedHistory, readLane, retryUpdateLane, withLaneHireLocks } from "./store.js";
|
|
41
41
|
import { enforceRegistryApprovalGate } from "./approval-gate.js";
|
|
42
42
|
import { readWorkerRunResult } from "./delivery.js";
|
|
43
43
|
import { applyDefaultQuotaHold, quotaSignalFromFields, readQuotaSignal, recordAccountQuota, } from "./quota.js";
|
|
44
44
|
import { PROVIDER_REJECTED_REASON, QUOTA_STATUS, isProviderLoginFailure, providerRejectionDetails, readProviderRejection, } from "./provider-rejection.js";
|
|
45
|
-
import { dropboxInboxHasMessages, readToolActivity, toolActivityPath } from "./dropbox.js";
|
|
45
|
+
import { appendDropboxMessageAtPath, dropboxInboxHasMessages, readToolActivity, toolActivityPath } from "./dropbox.js";
|
|
46
46
|
import { loadLanesConfig, providerProgressEventMinIntervalMs, providerQuotaDefaultHoldMs } from "./config.js";
|
|
47
47
|
import { resolveAdmissionHost as resolveProcessAdmissionHost } from "./admission-host.js";
|
|
48
48
|
import { createLaneMesh, emitLaneDecisionEvents, emitLaneProgressEvent, emitLaneTerminalStateEvent, laneStoryStatusSuppressionKey, progressEventDue, } from "./mesh-events.js";
|
|
49
49
|
import { envelopeQuestionId } from "./questions.js";
|
|
50
50
|
import { readLanesRegistryFlag } from "./registry-flag.js";
|
|
51
|
+
import { blockerFreshnessFingerprint, resolveBlockerFreshnessForLane, } from "./blocker-freshness.js";
|
|
51
52
|
import { MANAGER_STATUS_GATE_FLAG, managerStatusWasWritten, readStatusSnapshot, resolveManagerStatusTarget, managerStatusGateEnabledForLane, } from "./manager-status-gate.js";
|
|
52
|
-
import { queueManagerStatusGateResume } from "./keep-alive.js";
|
|
53
|
+
import { queueLaneTerminalResume, queueManagerStatusGateResume } from "./keep-alive.js";
|
|
54
|
+
import { monitorSessionDir } from "../monitor.js";
|
|
53
55
|
import { laneRunDir, missingPidProbe, probePidRecord, readPidRecord, readProcStarttime, readSpawnManifest, envelopeRepairMode, spawnEnvelopeRepair, writePidRecord, retryLaneOnAlternateAccount, } from "./spawn.js";
|
|
54
56
|
/** Spec §5.7: no signal for 20 minutes while the process is still alive. */
|
|
55
57
|
export const SILENCE_MS = 20 * 60 * 1000;
|
|
@@ -711,6 +713,171 @@ async function settleManagerStatusGate(hqRoot, lane, manifestTurnId, at, deps, l
|
|
|
711
713
|
}
|
|
712
714
|
return updated;
|
|
713
715
|
}
|
|
716
|
+
async function settleBlockerFreshness(hqRoot, lane, manifestTurnId, at, deps, mesh, logDecision) {
|
|
717
|
+
let errorLogged = false;
|
|
718
|
+
const resolution = await resolveBlockerFreshnessForLane(hqRoot, lane, {
|
|
719
|
+
readFlag: deps.readBlockerFreshnessFlag ?? readLanesRegistryFlag,
|
|
720
|
+
now: Date.parse(at),
|
|
721
|
+
onError: (error) => {
|
|
722
|
+
if (errorLogged)
|
|
723
|
+
return;
|
|
724
|
+
errorLogged = true;
|
|
725
|
+
const code = error instanceof LanesError ? error.error : "blocker_freshness_unavailable";
|
|
726
|
+
process.stderr.write(`hq lanes watcher: blocker freshness disabled for ${lane.lane_id}: ${code}\n`);
|
|
727
|
+
logDecision({ event: "blocker_freshness_disabled", at, error: code });
|
|
728
|
+
},
|
|
729
|
+
});
|
|
730
|
+
if (!resolution.enabled || resolution.statusPath === undefined || resolution.error !== undefined)
|
|
731
|
+
return lane;
|
|
732
|
+
if (manifestTurnId !== undefined && lane.status_gate !== undefined && lane.status_gate.turn_id !== manifestTurnId) {
|
|
733
|
+
logDecision({ event: "blocker_freshness_baseline_mismatch", at, turn_id: lane.status_gate.turn_id });
|
|
734
|
+
return lane;
|
|
735
|
+
}
|
|
736
|
+
if (resolution.stale.length === 0) {
|
|
737
|
+
if (lane.blocker_freshness_state === undefined)
|
|
738
|
+
return lane;
|
|
739
|
+
return retryUpdateLane(hqRoot, lane.lane_id, "watcher", (current) => current.blocker_freshness_state === undefined
|
|
740
|
+
? current
|
|
741
|
+
: { ...current, blocker_freshness_state: undefined });
|
|
742
|
+
}
|
|
743
|
+
const staleKeys = resolution.stale.map((blocker) => blocker.key).sort();
|
|
744
|
+
const staleLabels = resolution.stale.map((blocker) => blocker.label);
|
|
745
|
+
const fingerprint = blockerFreshnessFingerprint(resolution.stale);
|
|
746
|
+
const turnId = manifestTurnId ?? lane.status_gate?.turn_id ?? `unknown-${at}`;
|
|
747
|
+
const resumeText = `These open blockers in ${resolution.statusPath} were not re-verified in the last ${resolution.windowHours}h: ${staleLabels.join(", ")}. Check each one now; set verified_at on the ones that still hold and close the rest, then end the turn.`;
|
|
748
|
+
let deliverEscalation = false;
|
|
749
|
+
let escalationText = "";
|
|
750
|
+
let updated = retryUpdateLane(hqRoot, lane.lane_id, "watcher", (current) => {
|
|
751
|
+
deliverEscalation = false;
|
|
752
|
+
const prior = current.blocker_freshness_state;
|
|
753
|
+
const repeated = prior !== undefined && staleKeys.some((key) => prior.stale_keys.includes(key));
|
|
754
|
+
const history = current.status_gate_history ?? [];
|
|
755
|
+
const alreadyEscalatedThisTurn = history.some((entry) => entry.event === "blocker_stale_escalated" && entry.turn_id === turnId && entry.fingerprint === fingerprint);
|
|
756
|
+
const escalate = repeated && prior?.escalated_fingerprint !== fingerprint && !alreadyEscalatedThisTurn;
|
|
757
|
+
const alreadyEscalatedSet = prior?.escalated_fingerprint === fingerprint;
|
|
758
|
+
const statusResumeQueued = history.some((entry) => entry.event === "status_missed" && entry.turn_id === turnId && entry.resume_queued === true);
|
|
759
|
+
let resumeQueued = false;
|
|
760
|
+
let resumeSkippedReason;
|
|
761
|
+
let nextLane = current;
|
|
762
|
+
if (!escalate && !alreadyEscalatedSet) {
|
|
763
|
+
const resume = queueLaneTerminalResume(current, at, resumeText, {
|
|
764
|
+
mergeExisting: statusResumeQueued,
|
|
765
|
+
});
|
|
766
|
+
resumeQueued = resume.resumeQueued;
|
|
767
|
+
resumeSkippedReason = resume.resumeSkippedReason;
|
|
768
|
+
nextLane = resume.lane;
|
|
769
|
+
}
|
|
770
|
+
const staleEntry = {
|
|
771
|
+
event: "blocker_stale",
|
|
772
|
+
turn_id: turnId,
|
|
773
|
+
path: resolution.statusPath,
|
|
774
|
+
at,
|
|
775
|
+
labels: staleLabels,
|
|
776
|
+
count: resolution.stale.length,
|
|
777
|
+
fingerprint,
|
|
778
|
+
...(resumeQueued ? { resume_queued: true } : {}),
|
|
779
|
+
...(resumeSkippedReason !== undefined ? { resume_skipped_reason: resumeSkippedReason } : {}),
|
|
780
|
+
};
|
|
781
|
+
let nextHistory = history.some((entry) => entry.event === "blocker_stale" && entry.turn_id === turnId)
|
|
782
|
+
? [...history]
|
|
783
|
+
: appendCappedHistory(history, staleEntry);
|
|
784
|
+
let escalatedFingerprint = prior?.escalated_fingerprint;
|
|
785
|
+
if (escalate) {
|
|
786
|
+
const escalatedEntry = {
|
|
787
|
+
event: "blocker_stale_escalated",
|
|
788
|
+
turn_id: turnId,
|
|
789
|
+
path: resolution.statusPath,
|
|
790
|
+
at,
|
|
791
|
+
labels: staleLabels,
|
|
792
|
+
count: resolution.stale.length,
|
|
793
|
+
fingerprint,
|
|
794
|
+
};
|
|
795
|
+
nextHistory = appendCappedHistory(nextHistory, escalatedEntry);
|
|
796
|
+
escalatedFingerprint = fingerprint;
|
|
797
|
+
deliverEscalation = true;
|
|
798
|
+
escalationText = `Blocker freshness escalation for ${current.lane_id}: these blockers remain stale after consecutive turns: ${staleLabels.join(", ")}. Status file: ${resolution.statusPath}.`;
|
|
799
|
+
}
|
|
800
|
+
else if (prior !== undefined && blockerFreshnessFingerprint(prior.stale_keys.map((key, index) => ({ key, label: prior.stale_labels[index] ?? "" }))) !== fingerprint) {
|
|
801
|
+
escalatedFingerprint = undefined;
|
|
802
|
+
}
|
|
803
|
+
return {
|
|
804
|
+
...nextLane,
|
|
805
|
+
blocker_freshness_state: {
|
|
806
|
+
stale_keys: staleKeys,
|
|
807
|
+
stale_labels: staleLabels,
|
|
808
|
+
...(escalatedFingerprint !== undefined ? { escalated_fingerprint: escalatedFingerprint } : {}),
|
|
809
|
+
},
|
|
810
|
+
status_gate_history: nextHistory,
|
|
811
|
+
};
|
|
812
|
+
});
|
|
813
|
+
logDecision({ event: "blocker_stale", at, turn_id: turnId, path: resolution.statusPath, labels: staleLabels, count: staleLabels.length });
|
|
814
|
+
const escalated = updated.status_gate_history?.find((entry) => entry.event === "blocker_stale_escalated" && entry.turn_id === turnId && entry.fingerprint === fingerprint);
|
|
815
|
+
if (escalated && deliverEscalation) {
|
|
816
|
+
try {
|
|
817
|
+
if (lane.senior.kind === "lane" && lane.senior.id) {
|
|
818
|
+
if (deps.deliverLaneSeniorMessage === undefined) {
|
|
819
|
+
throw new LanesError("blocker_freshness_delivery_unavailable", "Lane-senior blocker escalations require the lane-message delivery path", { lane_id: lane.lane_id, senior_lane_id: lane.senior.id });
|
|
820
|
+
}
|
|
821
|
+
await deps.deliverLaneSeniorMessage(lane.senior.id, escalationText);
|
|
822
|
+
}
|
|
823
|
+
else if (lane.senior.kind === "session" && lane.senior.id) {
|
|
824
|
+
const inbox = resolveBlockerFreshnessSessionInbox(hqRoot, lane.senior.id);
|
|
825
|
+
appendDropboxMessageAtPath(inbox, {
|
|
826
|
+
at,
|
|
827
|
+
message_id: `blocker-freshness:${lane.lane_id}:${fingerprint}:${turnId}`,
|
|
828
|
+
kind: "blocker_stale_escalation",
|
|
829
|
+
text: escalationText,
|
|
830
|
+
});
|
|
831
|
+
}
|
|
832
|
+
else {
|
|
833
|
+
throw new LanesError("blocker_freshness_delivery_unavailable", "No durable senior inbox is available for this blocker escalation", { lane_id: lane.lane_id, senior_kind: lane.senior.kind });
|
|
834
|
+
}
|
|
835
|
+
logDecision({ event: "blocker_stale_escalation_delivered", at, turn_id: turnId, senior_kind: lane.senior.kind });
|
|
836
|
+
}
|
|
837
|
+
catch (error) {
|
|
838
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
839
|
+
process.stderr.write(`hq lanes watcher: blocker freshness escalation delivery failed for ${lane.lane_id}: ${message}\n`);
|
|
840
|
+
logDecision({ event: "blocker_stale_escalation_delivery_failed", at, turn_id: turnId, error: message });
|
|
841
|
+
updated = retryUpdateLane(hqRoot, lane.lane_id, "watcher", (current) => {
|
|
842
|
+
const state = current.blocker_freshness_state;
|
|
843
|
+
if (state?.escalated_fingerprint !== fingerprint)
|
|
844
|
+
return current;
|
|
845
|
+
const retryableState = { ...state };
|
|
846
|
+
delete retryableState.escalated_fingerprint;
|
|
847
|
+
return {
|
|
848
|
+
...current,
|
|
849
|
+
blocker_freshness_state: retryableState,
|
|
850
|
+
status_gate_history: (current.status_gate_history ?? []).map((entry) => entry.event === "blocker_stale_escalated"
|
|
851
|
+
&& entry.turn_id === turnId
|
|
852
|
+
&& entry.fingerprint === fingerprint
|
|
853
|
+
? { ...entry, delivery_failed: true }
|
|
854
|
+
: entry),
|
|
855
|
+
};
|
|
856
|
+
});
|
|
857
|
+
}
|
|
858
|
+
if (mesh !== null) {
|
|
859
|
+
await emitMeshSoft("blocker_freshness_escalation_signal", logDecision, at, (signal) => emitLaneProgressEvent(mesh, lane, escalationText, signal));
|
|
860
|
+
}
|
|
861
|
+
}
|
|
862
|
+
return updated;
|
|
863
|
+
}
|
|
864
|
+
function resolveBlockerFreshnessSessionInbox(hqRoot, sessionId) {
|
|
865
|
+
let candidates;
|
|
866
|
+
try {
|
|
867
|
+
candidates = LANE_PROVIDERS
|
|
868
|
+
.map((provider) => monitorSessionDir(hqRoot, provider, sessionId))
|
|
869
|
+
.filter((sessionDir) => fs.existsSync(sessionDir));
|
|
870
|
+
}
|
|
871
|
+
catch (error) {
|
|
872
|
+
throw new LanesError("blocker_freshness_session_inbox_unavailable", `Could not resolve the monitor inbox for session senior: ${error instanceof Error ? error.message : String(error)}`, { session_id: sessionId });
|
|
873
|
+
}
|
|
874
|
+
if (candidates.length !== 1) {
|
|
875
|
+
throw new LanesError("blocker_freshness_session_inbox_unavailable", candidates.length === 0
|
|
876
|
+
? "No monitor-session inbox is registered for the session senior"
|
|
877
|
+
: "The session senior matches more than one monitor-session inbox", { session_id: sessionId, matches: candidates.length });
|
|
878
|
+
}
|
|
879
|
+
return path.join(candidates[0], "dropbox");
|
|
880
|
+
}
|
|
714
881
|
/**
|
|
715
882
|
* Wrap one mesh call. A failed emit is logged as `mesh_event_failed` with
|
|
716
883
|
* the event name and the error message, and never propagates: the lane's
|
|
@@ -1282,11 +1449,12 @@ async function runWatcherLoopOwned(hqRoot, laneId, deps) {
|
|
|
1282
1449
|
}
|
|
1283
1450
|
if (terminalWrite.changed) {
|
|
1284
1451
|
updated = await settleManagerStatusGate(hqRoot, updated, manifest.turn_id, tick.emit.at, deps, logDecision);
|
|
1452
|
+
updated = await settleBlockerFreshness(hqRoot, updated, manifest.turn_id, tick.emit.at, deps, mesh, logDecision);
|
|
1285
1453
|
}
|
|
1286
1454
|
const managerStatusFollowupQueued = manifest.turn_id !== undefined &&
|
|
1287
|
-
updated.status_gate_history?.some((entry) => entry.
|
|
1288
|
-
entry.
|
|
1289
|
-
|
|
1455
|
+
updated.status_gate_history?.some((entry) => entry.turn_id === manifest.turn_id &&
|
|
1456
|
+
((entry.event === "status_missed" && entry.resume_queued === true) ||
|
|
1457
|
+
(entry.event === "blocker_stale" && entry.resume_queued === true))) === true;
|
|
1290
1458
|
if (terminalWrite.changed && deps.recoverQueuedMessages && !managerStatusFollowupQueued) {
|
|
1291
1459
|
try {
|
|
1292
1460
|
if (dropboxInboxHasMessages(hqRoot, laneId)) {
|