@indigoai-us/hq-cli 5.253.0 → 5.255.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +26 -0
- package/dist/command-catalog.generated.d.ts +2 -2
- package/dist/command-catalog.generated.js +2 -2
- package/dist/commands/db-sql.js +7 -21
- package/dist/commands/lanes-checks.js +1 -31
- package/dist/commands/lanes-message.js +3 -0
- package/dist/commands/lanes-resume.js +3 -0
- package/dist/commands/lanes-shared.d.ts +27 -0
- package/dist/commands/lanes-watch.d.ts +7 -0
- package/dist/commands/lanes-watch.js +215 -0
- package/dist/commands/lanes.d.ts +1 -1
- package/dist/commands/lanes.js +36 -211
- package/dist/lib/db/remote-sql.d.ts +2 -2
- package/dist/lib/db/remote-sql.js +3 -3
- package/dist/lib/lanes/checked-company-path.d.ts +3 -0
- package/dist/lib/lanes/checked-company-path.js +39 -0
- package/dist/lib/lanes/keep-alive.d.ts +3 -1
- package/dist/lib/lanes/keep-alive.js +56 -1
- package/dist/lib/lanes/manager-status-gate.d.ts +34 -0
- package/dist/lib/lanes/manager-status-gate.js +168 -0
- package/dist/lib/lanes/reconcile.d.ts +2 -0
- package/dist/lib/lanes/reconcile.js +45 -9
- package/dist/lib/lanes/spawn.d.ts +3 -0
- package/dist/lib/lanes/spawn.js +9 -0
- package/dist/lib/lanes/types.d.ts +21 -0
- package/dist/lib/lanes/types.js +1 -1
- package/dist/lib/lanes/watcher.d.ts +3 -0
- package/dist/lib/lanes/watcher.js +92 -6
- package/dist/lib/mesh/live/daemon/transcript-watch.js +25 -4
- package/package.json +1 -1
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
import * as fs from "node:fs";
|
|
2
|
+
import * as path from "node:path";
|
|
3
|
+
import { checkCompanyPath } from "./checked-company-path.js";
|
|
4
|
+
import { LanesError } from "./errors.js";
|
|
5
|
+
import { readLanesRegistryFlag } from "./registry-flag.js";
|
|
6
|
+
export const MANAGER_STATUS_GATE_FLAG = "lanes.manager-status-gate";
|
|
7
|
+
function isObject(value) {
|
|
8
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
9
|
+
}
|
|
10
|
+
function invalidMapping(message, filePath, details = {}) {
|
|
11
|
+
throw new LanesError("manager_status_mapping_invalid", message, { path: filePath, ...details });
|
|
12
|
+
}
|
|
13
|
+
function readJuniors(root) {
|
|
14
|
+
const filePath = path.join(root, "workspace", "lane-briefs", "juniors.json");
|
|
15
|
+
let raw;
|
|
16
|
+
try {
|
|
17
|
+
const stat = fs.lstatSync(filePath);
|
|
18
|
+
if (!stat.isFile() || stat.isSymbolicLink())
|
|
19
|
+
invalidMapping("juniors.json must be a regular file", filePath);
|
|
20
|
+
raw = fs.readFileSync(filePath, "utf8");
|
|
21
|
+
}
|
|
22
|
+
catch (err) {
|
|
23
|
+
if (err.code === "ENOENT")
|
|
24
|
+
return { filePath, rows: [] };
|
|
25
|
+
if (err instanceof LanesError)
|
|
26
|
+
throw err;
|
|
27
|
+
throw new LanesError("manager_status_mapping_unreadable", `Cannot read manager mapping ${filePath}: ${err instanceof Error ? err.message : String(err)}`, { path: filePath });
|
|
28
|
+
}
|
|
29
|
+
let parsed;
|
|
30
|
+
try {
|
|
31
|
+
parsed = JSON.parse(raw);
|
|
32
|
+
}
|
|
33
|
+
catch {
|
|
34
|
+
invalidMapping("juniors.json contains invalid JSON", filePath);
|
|
35
|
+
}
|
|
36
|
+
if (!isObject(parsed) || !Array.isArray(parsed.juniors))
|
|
37
|
+
invalidMapping("juniors.json must contain a juniors array", filePath);
|
|
38
|
+
const rows = [];
|
|
39
|
+
for (const [index, row] of parsed.juniors.entries()) {
|
|
40
|
+
if (!isObject(row))
|
|
41
|
+
invalidMapping(`juniors[${index}] must be an object`, filePath, { index });
|
|
42
|
+
rows.push(row);
|
|
43
|
+
}
|
|
44
|
+
return { filePath, rows };
|
|
45
|
+
}
|
|
46
|
+
function matchesLane(rowLane, laneId) {
|
|
47
|
+
const row = rowLane.trim();
|
|
48
|
+
const bareLaneId = laneId.split("_", 1)[0] ?? "";
|
|
49
|
+
return row === laneId || row === bareLaneId;
|
|
50
|
+
}
|
|
51
|
+
function managerStatusGateErrorCode(err) {
|
|
52
|
+
return err instanceof LanesError ? err.error : "manager_status_gate_unavailable";
|
|
53
|
+
}
|
|
54
|
+
function logManagerStatusGateFailure(lane, err) {
|
|
55
|
+
const code = managerStatusGateErrorCode(err);
|
|
56
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
57
|
+
process.stderr.write(`hq lanes: manager status gate disabled for ${lane.lane_id}: ${code}: ${message}\n`);
|
|
58
|
+
}
|
|
59
|
+
/** Resolve manager identity and the single declared status file, if present. */
|
|
60
|
+
export function resolveManagerStatusTarget(root, lane) {
|
|
61
|
+
const directIdentity = lane.story_id === "MANAGER"
|
|
62
|
+
? "story_id"
|
|
63
|
+
: lane.worker_profile?.endsWith("-manager") === true
|
|
64
|
+
? "worker_profile"
|
|
65
|
+
: undefined;
|
|
66
|
+
const registry = readJuniors(root);
|
|
67
|
+
const matches = [];
|
|
68
|
+
for (const [index, row] of registry.rows.entries()) {
|
|
69
|
+
if (typeof row.lane !== "string")
|
|
70
|
+
invalidMapping(`juniors[${index}].lane must be a string`, registry.filePath, { index });
|
|
71
|
+
if (matchesLane(row.lane, lane.lane_id))
|
|
72
|
+
matches.push(row);
|
|
73
|
+
}
|
|
74
|
+
if (matches.length > 1)
|
|
75
|
+
invalidMapping("More than one juniors row maps to this manager lane", registry.filePath, { lane_id: lane.lane_id });
|
|
76
|
+
const row = matches[0];
|
|
77
|
+
if (row === undefined) {
|
|
78
|
+
return directIdentity
|
|
79
|
+
? { manager: true, identified_by: directIdentity, skipped: "no_juniors_row" }
|
|
80
|
+
: { manager: false };
|
|
81
|
+
}
|
|
82
|
+
for (const field of ["team", "lane", "host", "status", "prd"]) {
|
|
83
|
+
if (typeof row[field] !== "string" || row[field].trim() === "") {
|
|
84
|
+
invalidMapping(`Matched juniors row needs a non-empty ${field}`, registry.filePath, { lane_id: lane.lane_id, field });
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
const company = lane.company?.slug ?? lane.mesh_company?.slug ?? "";
|
|
88
|
+
const statusPath = checkCompanyPath(root, company, row.status, registry.filePath);
|
|
89
|
+
return {
|
|
90
|
+
manager: true,
|
|
91
|
+
identified_by: directIdentity ?? "juniors",
|
|
92
|
+
statusPath,
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
export async function resolveManagerStatusGateForLane(root, lane, options = {}) {
|
|
96
|
+
const reportError = options.onError ?? ((err) => logManagerStatusGateFailure(lane, err));
|
|
97
|
+
let enabled;
|
|
98
|
+
try {
|
|
99
|
+
enabled = await (options.readFlag ?? readLanesRegistryFlag)(MANAGER_STATUS_GATE_FLAG, {
|
|
100
|
+
company: lane.company ?? lane.mesh_company ?? undefined,
|
|
101
|
+
env: options.env,
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
catch (err) {
|
|
105
|
+
reportError(err);
|
|
106
|
+
return { enabled: false, error: managerStatusGateErrorCode(err) };
|
|
107
|
+
}
|
|
108
|
+
if (!enabled)
|
|
109
|
+
return { enabled: false };
|
|
110
|
+
try {
|
|
111
|
+
const target = resolveManagerStatusTarget(root, lane);
|
|
112
|
+
return { enabled: target.manager && target.statusPath !== undefined, target };
|
|
113
|
+
}
|
|
114
|
+
catch (err) {
|
|
115
|
+
reportError(err);
|
|
116
|
+
return { enabled: false, error: managerStatusGateErrorCode(err) };
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
export async function managerStatusGateEnabledForLane(root, lane, options = {}) {
|
|
120
|
+
return (await resolveManagerStatusGateForLane(root, lane, options)).enabled;
|
|
121
|
+
}
|
|
122
|
+
export function readStatusSnapshot(filePath) {
|
|
123
|
+
let stat;
|
|
124
|
+
try {
|
|
125
|
+
stat = fs.statSync(filePath);
|
|
126
|
+
}
|
|
127
|
+
catch (err) {
|
|
128
|
+
if (err.code === "ENOENT")
|
|
129
|
+
return { mtime: null, updatedAt: null };
|
|
130
|
+
throw new LanesError("manager_status_unreadable", `Cannot stat manager status ${filePath}: ${err instanceof Error ? err.message : String(err)}`, { path: filePath });
|
|
131
|
+
}
|
|
132
|
+
if (!stat.isFile())
|
|
133
|
+
throw new LanesError("manager_status_invalid", `Manager status path is not a file: ${filePath}`, { path: filePath });
|
|
134
|
+
let updatedAt = null;
|
|
135
|
+
try {
|
|
136
|
+
const parsed = JSON.parse(fs.readFileSync(filePath, "utf8"));
|
|
137
|
+
if (isObject(parsed) && typeof parsed.updated_at === "string")
|
|
138
|
+
updatedAt = parsed.updated_at;
|
|
139
|
+
}
|
|
140
|
+
catch (err) {
|
|
141
|
+
if (!(err instanceof SyntaxError)) {
|
|
142
|
+
if (err.code === "ENOENT")
|
|
143
|
+
return { mtime: null, updatedAt: null };
|
|
144
|
+
throw new LanesError("manager_status_unreadable", `Cannot read manager status ${filePath}: ${err instanceof Error ? err.message : String(err)}`, { path: filePath });
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
return { mtime: stat.mtimeMs, updatedAt };
|
|
148
|
+
}
|
|
149
|
+
export function managerStatusGateHeartbeat(root, lane, turnId, turnStartedAt) {
|
|
150
|
+
const target = resolveManagerStatusTarget(root, lane);
|
|
151
|
+
if (!target.manager || target.statusPath === undefined)
|
|
152
|
+
return undefined;
|
|
153
|
+
const snapshot = readStatusSnapshot(target.statusPath);
|
|
154
|
+
return {
|
|
155
|
+
turn_started_at: turnStartedAt,
|
|
156
|
+
status_mtime_at_start: snapshot.mtime,
|
|
157
|
+
status_updated_at_at_start: snapshot.updatedAt,
|
|
158
|
+
status_path: target.statusPath,
|
|
159
|
+
turn_id: turnId,
|
|
160
|
+
consecutive_misses: lane.status_gate?.consecutive_misses ?? 0,
|
|
161
|
+
};
|
|
162
|
+
}
|
|
163
|
+
export function managerStatusWasWritten(gate, snapshot) {
|
|
164
|
+
const startedAt = Date.parse(gate.turn_started_at);
|
|
165
|
+
return ((snapshot.mtime !== null && Number.isFinite(startedAt) && snapshot.mtime > startedAt) ||
|
|
166
|
+
snapshot.updatedAt !== gate.status_updated_at_at_start);
|
|
167
|
+
}
|
|
168
|
+
//# sourceMappingURL=manager-status-gate.js.map
|
|
@@ -79,6 +79,8 @@ export interface ReconcileDeps {
|
|
|
79
79
|
env?: NodeJS.ProcessEnv;
|
|
80
80
|
/** Flag reader seam used to assert rollout reads happen before admission locks. */
|
|
81
81
|
readRegistryFlag?: typeof readLanesRegistryFlag;
|
|
82
|
+
/** Pre-read manager gate decisions, scoped to each lane for this pass. */
|
|
83
|
+
managerStatusGateByLane?: ReadonlyMap<string, boolean>;
|
|
82
84
|
/** Active-company fallback for questions raised from an unscoped lane. */
|
|
83
85
|
resolveQuestionActiveCompany?: () => Promise<LaneCompanyRef>;
|
|
84
86
|
/** Resume queued messages after this pass wins a local dead-worker CAS. */
|
|
@@ -29,6 +29,7 @@ import { answeredQuestionEntries, questionRecordRefusalCode, raiseEnvelopeQuesti
|
|
|
29
29
|
import { SILENCE_MS, classifyLaneExit, isPidAlive, missingExpectedEnvelopeFile, readLatestEnvelopeFile, readLatestEnvelopeRecord, watcherEventsPath, watcherOutLogPath, writeWatcherDecision, } from "./watcher.js";
|
|
30
30
|
import { createLaneMesh, emitLaneDecisionEvents, emitLaneTerminalStateEvent, laneMeshCompany, } from "./mesh-events.js";
|
|
31
31
|
import { laneStories, storyStatusForEnvelope } from "./stories.js";
|
|
32
|
+
import { resolveManagerStatusGateForLane } from "./manager-status-gate.js";
|
|
32
33
|
import { decideEnvelopeRepair, envelopeQuestionIds, routeProviderFailure } from "./watcher.js";
|
|
33
34
|
import { PROVIDER_REJECTED_REASON, providerRejectionDetails, readProviderRejection, } from "./provider-rejection.js";
|
|
34
35
|
/** Spec §9: unanswered at the current hop for 60 minutes → one hop. */
|
|
@@ -44,11 +45,43 @@ export const PICKUP_VALIDATION_MAX_AGE_MS = 15 * 60 * 1000;
|
|
|
44
45
|
export const PICKUP_VALIDATION_TIMEOUT_REASON = "pickup_validation_timeout";
|
|
45
46
|
export const RECONCILE_UNIT_NAME = "hq-lanes-reconcile.service";
|
|
46
47
|
export const RECONCILE_TIMER_NAME = "hq-lanes-reconcile.timer";
|
|
47
|
-
function spawnDepsForHost(deps) {
|
|
48
|
+
function spawnDepsForHost(deps, lane) {
|
|
48
49
|
return {
|
|
49
50
|
...(deps.spawnLaneDeps ?? {}),
|
|
50
51
|
hostname: deps.host.hostname,
|
|
51
52
|
admissionHost: deps.host,
|
|
53
|
+
...(lane !== undefined && deps.managerStatusGateByLane?.has(lane.lane_id)
|
|
54
|
+
? { managerStatusGateEnabled: deps.managerStatusGateByLane.get(lane.lane_id) === true }
|
|
55
|
+
: {}),
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
async function prepareManagerStatusGateStarts(root, lanes, deps, summary) {
|
|
59
|
+
const enabledByLane = new Map();
|
|
60
|
+
let gateFailureRecorded = false;
|
|
61
|
+
if (!deps.dryRun) {
|
|
62
|
+
await Promise.all(lanes.filter((lane) => laneRunsHere(lane, deps.host)).map(async (lane) => {
|
|
63
|
+
const resolution = await resolveManagerStatusGateForLane(root, lane, {
|
|
64
|
+
env: deps.env,
|
|
65
|
+
readFlag: deps.readRegistryFlag,
|
|
66
|
+
onError: (err) => {
|
|
67
|
+
if (gateFailureRecorded)
|
|
68
|
+
return;
|
|
69
|
+
gateFailureRecorded = true;
|
|
70
|
+
const code = err instanceof LanesError ? err.error : "manager_status_gate_unavailable";
|
|
71
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
72
|
+
summary.skipped.push({
|
|
73
|
+
kind: "lane",
|
|
74
|
+
id: lane.lane_id,
|
|
75
|
+
why: `manager status gate disabled for this reconcile pass: ${code}: ${message}`,
|
|
76
|
+
});
|
|
77
|
+
},
|
|
78
|
+
});
|
|
79
|
+
enabledByLane.set(lane.lane_id, resolution.enabled);
|
|
80
|
+
}));
|
|
81
|
+
}
|
|
82
|
+
return {
|
|
83
|
+
...deps,
|
|
84
|
+
managerStatusGateByLane: enabledByLane,
|
|
52
85
|
};
|
|
53
86
|
}
|
|
54
87
|
function logSkip(skip) {
|
|
@@ -694,8 +727,9 @@ export async function runPassBody(hqRoot, deps) {
|
|
|
694
727
|
: QUESTION_AGE_MS;
|
|
695
728
|
if (deps.afterLoad)
|
|
696
729
|
await deps.afterLoad();
|
|
730
|
+
const startDeps = await prepareManagerStatusGateStarts(hqRoot, lanes, deps, summary);
|
|
697
731
|
markOverdueLanes(hqRoot, lanes, summary, deps, now);
|
|
698
|
-
await heartbeatAndDeadHosts(hqRoot, lanes, summary,
|
|
732
|
+
await heartbeatAndDeadHosts(hqRoot, lanes, summary, startDeps, now);
|
|
699
733
|
for (const question of questions) {
|
|
700
734
|
if (question.status !== "pending")
|
|
701
735
|
continue;
|
|
@@ -838,10 +872,10 @@ export async function runPassBody(hqRoot, deps) {
|
|
|
838
872
|
await withAllLaneAdmissionLocks(hqRoot, async () => {
|
|
839
873
|
const freshLanes = loadLaneRecords(hqRoot, summary);
|
|
840
874
|
const freshQuestions = loadQuestionRecords(hqRoot, summary);
|
|
841
|
-
resumeKeepAlive(hqRoot, freshLanes, freshQuestions, summary,
|
|
875
|
+
resumeKeepAlive(hqRoot, freshLanes, freshQuestions, summary, startDeps, now, keepAliveFlagByCompany, supersessionLineageAwareByCompany);
|
|
842
876
|
const drainLanes = loadLaneRecords(hqRoot, summary);
|
|
843
|
-
await admitQueued(hqRoot, drainLanes, summary,
|
|
844
|
-
await resumeAwaiting(hqRoot, drainLanes, freshQuestions, summary,
|
|
877
|
+
await admitQueued(hqRoot, drainLanes, summary, startDeps, now);
|
|
878
|
+
await resumeAwaiting(hqRoot, drainLanes, freshQuestions, summary, startDeps);
|
|
845
879
|
});
|
|
846
880
|
if (!deps.dryRun && summary.errors.length === 0) {
|
|
847
881
|
try {
|
|
@@ -1024,10 +1058,12 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
|
|
|
1024
1058
|
classification.reason === "account_model_unavailable")) {
|
|
1025
1059
|
recordAccountQuota(hqRoot, classified.provider, classified.account, quota);
|
|
1026
1060
|
}
|
|
1027
|
-
if (!deps.dryRun &&
|
|
1061
|
+
if (!deps.dryRun &&
|
|
1062
|
+
accountRetryReason &&
|
|
1063
|
+
classified.account) {
|
|
1028
1064
|
try {
|
|
1029
1065
|
const recovered = await retryLaneOnAlternateAccount(hqRoot, classified, accountRetryReason, "reconcile", {
|
|
1030
|
-
...spawnDepsForHost(deps),
|
|
1066
|
+
...spawnDepsForHost(deps, classified),
|
|
1031
1067
|
});
|
|
1032
1068
|
summary.dead_host.push({
|
|
1033
1069
|
id: lane.lane_id,
|
|
@@ -1308,7 +1344,7 @@ async function admitQueued(hqRoot, lanes, summary, deps, now) {
|
|
|
1308
1344
|
nowMs,
|
|
1309
1345
|
});
|
|
1310
1346
|
const spawnFn = deps.spawn ??
|
|
1311
|
-
((root, lane) => spawnLaneWorker(root, lane, spawnDepsForHost(deps)));
|
|
1347
|
+
((root, lane) => spawnLaneWorker(root, lane, spawnDepsForHost(deps, lane)));
|
|
1312
1348
|
const running = lanes.filter((lane) => (lane.state === "running" ||
|
|
1313
1349
|
hasFreshAdmissionReservation(lane, now.getTime()) ||
|
|
1314
1350
|
hasActivePendingResume(lane, now.getTime())) &&
|
|
@@ -1977,7 +2013,7 @@ async function resumeAwaiting(hqRoot, lanes, questions, summary, deps, onlyLaneI
|
|
|
1977
2013
|
const throttle = await decideAdmission(hqRoot, config.throttle, throttleDeps);
|
|
1978
2014
|
throttleDecision = throttle.decision;
|
|
1979
2015
|
}
|
|
1980
|
-
const spawnFn = deps.spawn ?? ((root, lane) => spawnLaneWorker(root, lane, spawnDepsForHost(deps)));
|
|
2016
|
+
const spawnFn = deps.spawn ?? ((root, lane) => spawnLaneWorker(root, lane, spawnDepsForHost(deps, lane)));
|
|
1981
2017
|
const nowMs = deps.now().getTime();
|
|
1982
2018
|
const fallbackAnswerAt = new Date(nowMs).toISOString();
|
|
1983
2019
|
const answerAtByLane = new Map();
|
|
@@ -86,6 +86,8 @@ export interface SpawnLaneDeps {
|
|
|
86
86
|
runCompaction?: RunCompactionFn;
|
|
87
87
|
/** Finds a session transcript under an account home. Injected in tests. */
|
|
88
88
|
locateTranscript?: (home: string, sessionId: string) => string | null;
|
|
89
|
+
/** Pre-read by the async command/reconcile boundary before this sync spawn. */
|
|
90
|
+
managerStatusGateEnabled?: boolean;
|
|
89
91
|
}
|
|
90
92
|
export interface ProviderLaunch {
|
|
91
93
|
command: string;
|
|
@@ -254,6 +256,7 @@ export declare function refuseOutpostSelfTarget(hqRoot: string, lane: LaneRecord
|
|
|
254
256
|
export declare function armLaneWatcher(hqRoot: string, laneId: string, deps?: SpawnLaneDeps): number;
|
|
255
257
|
export interface SpawnManifestFile {
|
|
256
258
|
envelope_path?: string;
|
|
259
|
+
turn_id?: string;
|
|
257
260
|
resume?: {
|
|
258
261
|
session_id: string;
|
|
259
262
|
} | null;
|
package/dist/lib/lanes/spawn.js
CHANGED
|
@@ -27,6 +27,7 @@ import { ignoredWorkerProfileFields, loadWorkerProfile, loadWorkerProfileWithReg
|
|
|
27
27
|
import { registryWorkerRecord } from "./registry-persona.js";
|
|
28
28
|
import { resolveMachineIdentity, } from "./machine-identity.js";
|
|
29
29
|
import { appendCappedHistory, listLaneIds, readLane, retryUpdateLane } from "./store.js";
|
|
30
|
+
import { managerStatusGateHeartbeat } from "./manager-status-gate.js";
|
|
30
31
|
import { copySessionFiles, planSessionCopy } from "./move-account.js";
|
|
31
32
|
import { ensureSessionHome } from "./session-relink.js";
|
|
32
33
|
import { MESSAGE_REPLAY_STATE_NAME } from "./message-replay.js";
|
|
@@ -1476,6 +1477,7 @@ function writeSpawnManifest(runDir, input) {
|
|
|
1476
1477
|
watcher_pid: input.watcherPid,
|
|
1477
1478
|
detached: true,
|
|
1478
1479
|
envelope_path: input.envelopePath,
|
|
1480
|
+
...(input.turnId !== undefined ? { turn_id: input.turnId } : {}),
|
|
1479
1481
|
resume: input.resume,
|
|
1480
1482
|
worker_out_offset: input.workerOutOffset,
|
|
1481
1483
|
...(input.fastMode ? { fast_mode: true } : {}),
|
|
@@ -1552,6 +1554,8 @@ export function readSpawnManifest(runDir) {
|
|
|
1552
1554
|
const out = {};
|
|
1553
1555
|
if (typeof rec.envelope_path === "string")
|
|
1554
1556
|
out.envelope_path = rec.envelope_path;
|
|
1557
|
+
if (typeof rec.turn_id === "string")
|
|
1558
|
+
out.turn_id = rec.turn_id;
|
|
1555
1559
|
if (typeof rec.provider_home_at_spawn === "string") {
|
|
1556
1560
|
out.provider_home_at_spawn = rec.provider_home_at_spawn;
|
|
1557
1561
|
}
|
|
@@ -1982,6 +1986,9 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
|
|
|
1982
1986
|
// Claim running before spawning. Two callers that both observe queued must
|
|
1983
1987
|
// not both start processes; the CAS loser returns without spawn.
|
|
1984
1988
|
const nowIso = (deps.now ?? (() => new Date()))().toISOString();
|
|
1989
|
+
const statusGateHeartbeat = deps.managerStatusGateEnabled === true
|
|
1990
|
+
? managerStatusGateHeartbeat(hqRoot, existing, laneTurnId, nowIso)
|
|
1991
|
+
: undefined;
|
|
1985
1992
|
let claimed = false;
|
|
1986
1993
|
const claimedRecord = retryUpdateLane(hqRoot, lane.lane_id, "spawn", (current) => {
|
|
1987
1994
|
claimed = false;
|
|
@@ -2033,6 +2040,7 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
|
|
|
2033
2040
|
}
|
|
2034
2041
|
: { model_source: "none" }),
|
|
2035
2042
|
effort_source: effort.source,
|
|
2043
|
+
status_gate: statusGateHeartbeat,
|
|
2036
2044
|
last_host_report_at: nowIso,
|
|
2037
2045
|
...(profile
|
|
2038
2046
|
? {
|
|
@@ -2167,6 +2175,7 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
|
|
|
2167
2175
|
workerPid,
|
|
2168
2176
|
watcherPid,
|
|
2169
2177
|
envelopePath,
|
|
2178
|
+
turnId: laneTurnId,
|
|
2170
2179
|
resume: resume ? { session_id: resume.session_id } : null,
|
|
2171
2180
|
workerOutOffset,
|
|
2172
2181
|
fastMode,
|
|
@@ -334,6 +334,23 @@ export interface LastEnvelopeRef {
|
|
|
334
334
|
artifacts?: unknown[];
|
|
335
335
|
blocked_reason?: string;
|
|
336
336
|
}
|
|
337
|
+
export interface ManagerStatusGate {
|
|
338
|
+
turn_started_at: string;
|
|
339
|
+
status_mtime_at_start: number | null;
|
|
340
|
+
status_updated_at_at_start: string | null;
|
|
341
|
+
status_path: string;
|
|
342
|
+
turn_id: string;
|
|
343
|
+
consecutive_misses: number;
|
|
344
|
+
}
|
|
345
|
+
export interface ManagerStatusGateHistoryEntry {
|
|
346
|
+
event: "status_missed";
|
|
347
|
+
turn_id: string;
|
|
348
|
+
path: string;
|
|
349
|
+
at: string;
|
|
350
|
+
identified_by: "story_id" | "worker_profile" | "juniors";
|
|
351
|
+
resume_queued?: boolean;
|
|
352
|
+
resume_skipped_reason?: "open_question" | "blocked" | "miss_limit" | "no_provider_session";
|
|
353
|
+
}
|
|
337
354
|
export interface LaneStory {
|
|
338
355
|
project_id: string;
|
|
339
356
|
story_id: string;
|
|
@@ -473,6 +490,10 @@ export interface LaneRecord {
|
|
|
473
490
|
fast_mode?: boolean;
|
|
474
491
|
/** Time of the last operator message accepted for this lane. Drives compaction. */
|
|
475
492
|
last_message_at?: string;
|
|
493
|
+
/** Turn-start baseline used by the optional manager status gate. */
|
|
494
|
+
status_gate?: ManagerStatusGate;
|
|
495
|
+
/** Bounded audit history of manager turns that missed their declared status artifact. */
|
|
496
|
+
status_gate_history?: ManagerStatusGateHistoryEntry[];
|
|
476
497
|
/** Last automatic compaction of a resumed session (compaction.ts). */
|
|
477
498
|
last_compaction?: LaneCompactionRecord;
|
|
478
499
|
/**
|
package/dist/lib/lanes/types.js
CHANGED
|
@@ -228,7 +228,7 @@ export const LANE_FIELD_WRITERS = {
|
|
|
228
228
|
provider: ["creator"], account: ["creator", "spawn", "command", "watcher", "reconcile"], account_selection: ["creator", "command"], admission_accounts: ["creator", "command"], account_history: ["command", "spawn", "watcher", "reconcile"], failed_resume_history: ["command"], pending_resume: ["command", "spawn", "watcher", "reconcile"], placement_default: ["creator"], host: ["creator", "command"], origin_host: ["creator"], origin_host_source: ["creator"],
|
|
229
229
|
version: ["creator"], created_at: ["creator"], updated_at: ["creator"], attribution_history: ["command"],
|
|
230
230
|
last_envelope: ["watcher", "reconcile"], model: ["creator", "spawn", "command"], model_source: ["creator", "spawn", "command"], effort: ["creator", "spawn", "command"], effort_source: ["creator", "spawn", "command"], fast_mode: ["spawn"],
|
|
231
|
-
last_message_at: ["command"], last_compaction: ["spawn"], story_complete: ["watcher", "reconcile"], story_complete_reason: ["watcher", "reconcile"], story_status: ["watcher", "reconcile"],
|
|
231
|
+
last_message_at: ["command"], status_gate: ["spawn", "watcher"], status_gate_history: ["watcher"], last_compaction: ["spawn"], story_complete: ["watcher", "reconcile"], story_complete_reason: ["watcher", "reconcile"], story_status: ["watcher", "reconcile"],
|
|
232
232
|
overdue: ["reconcile"], overdue_at: ["reconcile"], last_host_report_at: ["watcher", "spawn", "reconcile"], stalled: ["watcher"], stalled_at: ["watcher"], stalled_reason: ["watcher"],
|
|
233
233
|
admitted_reason: ["command", "reconcile"], terminal_reason: ["command", "watcher", "spawn", "reconcile"], stop_history: ["command"], interrupted_at: ["watcher", "reconcile"], terminal_details: ["command", "spawn", "watcher", "reconcile"],
|
|
234
234
|
quota_reset_at: ["watcher", "reconcile"], model_quota_marker: ["watcher", "reconcile"], outpost_command_id: ["command"],
|
|
@@ -36,6 +36,7 @@ import { type QuotaSignal } from "./quota.js";
|
|
|
36
36
|
import { type ProviderRejection } from "./provider-rejection.js";
|
|
37
37
|
import type { AdmissionHost } from "./admission.js";
|
|
38
38
|
import { type LaneMesh } from "./mesh-events.js";
|
|
39
|
+
import { readLanesRegistryFlag } from "./registry-flag.js";
|
|
39
40
|
import { type EnvelopeRepairSpawnInput, type SpawnLaneDeps } from "./spawn.js";
|
|
40
41
|
/** Spec §5.7: no signal for 20 minutes while the process is still alive. */
|
|
41
42
|
export declare const SILENCE_MS: number;
|
|
@@ -226,6 +227,8 @@ export interface WatcherLoopDeps {
|
|
|
226
227
|
resolveQuestionActiveCompany?: () => Promise<LaneCompanyRef>;
|
|
227
228
|
/** Resume an atomically queued inbox after this watcher wins terminal CAS. */
|
|
228
229
|
recoverQueuedMessages?: (lane: LaneRecord) => Promise<void>;
|
|
230
|
+
/** Test seam; production reads the company-scoped hq-flags row. */
|
|
231
|
+
readManagerStatusGateFlag?: typeof readLanesRegistryFlag;
|
|
229
232
|
}
|
|
230
233
|
/**
|
|
231
234
|
* Wrap one mesh call. A failed emit is logged as `mesh_event_failed` with
|
|
@@ -47,6 +47,9 @@ import { loadLanesConfig, providerProgressEventMinIntervalMs, providerQuotaDefau
|
|
|
47
47
|
import { resolveAdmissionHost as resolveProcessAdmissionHost } from "./admission-host.js";
|
|
48
48
|
import { createLaneMesh, emitLaneDecisionEvents, emitLaneProgressEvent, emitLaneTerminalStateEvent, laneStoryStatusSuppressionKey, progressEventDue, } from "./mesh-events.js";
|
|
49
49
|
import { envelopeQuestionId } from "./questions.js";
|
|
50
|
+
import { readLanesRegistryFlag } from "./registry-flag.js";
|
|
51
|
+
import { MANAGER_STATUS_GATE_FLAG, managerStatusWasWritten, readStatusSnapshot, resolveManagerStatusTarget, managerStatusGateEnabledForLane, } from "./manager-status-gate.js";
|
|
52
|
+
import { queueManagerStatusGateResume } from "./keep-alive.js";
|
|
50
53
|
import { laneRunDir, missingPidProbe, probePidRecord, readPidRecord, readProcStarttime, readSpawnManifest, envelopeRepairMode, spawnEnvelopeRepair, writePidRecord, retryLaneOnAlternateAccount, } from "./spawn.js";
|
|
51
54
|
/** Spec §5.7: no signal for 20 minutes while the process is still alive. */
|
|
52
55
|
export const SILENCE_MS = 20 * 60 * 1000;
|
|
@@ -639,6 +642,75 @@ export function missingExpectedEnvelopeFile(hqRoot, laneId, envelopePath) {
|
|
|
639
642
|
missing: true,
|
|
640
643
|
};
|
|
641
644
|
}
|
|
645
|
+
async function settleManagerStatusGate(hqRoot, lane, manifestTurnId, at, deps, logDecision) {
|
|
646
|
+
const readFlag = deps.readManagerStatusGateFlag ?? readLanesRegistryFlag;
|
|
647
|
+
let enabled = false;
|
|
648
|
+
try {
|
|
649
|
+
enabled = await readFlag(MANAGER_STATUS_GATE_FLAG, {
|
|
650
|
+
company: lane.company ?? lane.mesh_company ?? undefined,
|
|
651
|
+
});
|
|
652
|
+
}
|
|
653
|
+
catch (err) {
|
|
654
|
+
process.stderr.write(`hq lanes watcher: manager status gate flag read failed for ${lane.lane_id}: ${err instanceof Error ? err.message : String(err)}\n`);
|
|
655
|
+
logDecision({ event: "status_gate_flag_failed", at, error: err instanceof Error ? err.name : "UnknownError" });
|
|
656
|
+
}
|
|
657
|
+
if (!enabled)
|
|
658
|
+
return lane;
|
|
659
|
+
let target;
|
|
660
|
+
try {
|
|
661
|
+
target = resolveManagerStatusTarget(hqRoot, lane);
|
|
662
|
+
}
|
|
663
|
+
catch (err) {
|
|
664
|
+
process.stderr.write(`hq lanes watcher: manager status mapping failed for ${lane.lane_id}: ${err instanceof Error ? err.message : String(err)}\n`);
|
|
665
|
+
logDecision({ event: "status_gate_mapping_failed", at, error: err instanceof Error ? err.message : String(err) });
|
|
666
|
+
return lane;
|
|
667
|
+
}
|
|
668
|
+
const gate = lane.status_gate;
|
|
669
|
+
if (!target.manager || target.statusPath === undefined || gate === undefined)
|
|
670
|
+
return lane;
|
|
671
|
+
if (manifestTurnId !== undefined && manifestTurnId !== gate.turn_id ||
|
|
672
|
+
target.statusPath !== gate.status_path) {
|
|
673
|
+
logDecision({ event: "status_gate_baseline_mismatch", at, turn_id: gate.turn_id });
|
|
674
|
+
return lane;
|
|
675
|
+
}
|
|
676
|
+
let written;
|
|
677
|
+
try {
|
|
678
|
+
written = managerStatusWasWritten(gate, readStatusSnapshot(gate.status_path));
|
|
679
|
+
}
|
|
680
|
+
catch (err) {
|
|
681
|
+
process.stderr.write(`hq lanes watcher: manager status read failed for ${lane.lane_id}: ${err instanceof Error ? err.message : String(err)}\n`);
|
|
682
|
+
logDecision({ event: "status_gate_status_read_failed", at, error: err instanceof Error ? err.message : String(err) });
|
|
683
|
+
written = false;
|
|
684
|
+
}
|
|
685
|
+
if (written) {
|
|
686
|
+
return retryUpdateLane(hqRoot, lane.lane_id, "watcher", (current) => {
|
|
687
|
+
if (current.status_gate?.turn_id !== gate.turn_id || current.status_gate.consecutive_misses === 0)
|
|
688
|
+
return current;
|
|
689
|
+
return { ...current, status_gate: { ...current.status_gate, consecutive_misses: 0 } };
|
|
690
|
+
});
|
|
691
|
+
}
|
|
692
|
+
const resumeText = `Your turn ended without updating ${gate.status_path}. Write your status now: update updated_at, kpis and current_work, then end the turn.`;
|
|
693
|
+
const updated = queueManagerStatusGateResume(hqRoot, lane.lane_id, {
|
|
694
|
+
turn_id: gate.turn_id,
|
|
695
|
+
path: gate.status_path,
|
|
696
|
+
at,
|
|
697
|
+
identified_by: target.identified_by ?? "juniors",
|
|
698
|
+
}, resumeText);
|
|
699
|
+
const miss = updated.status_gate_history?.find((entry) => entry.event === "status_missed" && entry.turn_id === gate.turn_id);
|
|
700
|
+
if (miss) {
|
|
701
|
+
logDecision({
|
|
702
|
+
event: "status_missed",
|
|
703
|
+
at,
|
|
704
|
+
turn_id: gate.turn_id,
|
|
705
|
+
path: gate.status_path,
|
|
706
|
+
consecutive_misses: updated.status_gate?.consecutive_misses ?? 0,
|
|
707
|
+
resume_queued: miss.resume_queued === true,
|
|
708
|
+
resume_skipped_reason: miss.resume_skipped_reason,
|
|
709
|
+
state: updated.state,
|
|
710
|
+
});
|
|
711
|
+
}
|
|
712
|
+
return updated;
|
|
713
|
+
}
|
|
642
714
|
/**
|
|
643
715
|
* Wrap one mesh call. A failed emit is logged as `mesh_event_failed` with
|
|
644
716
|
* the event name and the error message, and never propagates: the lane's
|
|
@@ -820,10 +892,17 @@ async function runWatcherLoopOwned(hqRoot, laneId, deps) {
|
|
|
820
892
|
}
|
|
821
893
|
const progressIntervalMs = deps.progressEventMinIntervalMs ?? readProgressIntervalMs(hqRoot, readLaneSoft(hqRoot, laneId));
|
|
822
894
|
const spawnRepair = deps.spawnRepair ?? spawnEnvelopeRepair;
|
|
823
|
-
const retryAccount = deps.retryAccount ?? ((root, lane, reason) =>
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
895
|
+
const retryAccount = deps.retryAccount ?? (async (root, lane, reason) => {
|
|
896
|
+
const managerStatusGateEnabled = await managerStatusGateEnabledForLane(root, lane, {
|
|
897
|
+
env: deps.retrySpawnDeps?.env,
|
|
898
|
+
readFlag: deps.readManagerStatusGateFlag,
|
|
899
|
+
});
|
|
900
|
+
return retryLaneOnAlternateAccount(root, lane, reason, "watcher", {
|
|
901
|
+
...deps.retrySpawnDeps,
|
|
902
|
+
managerStatusGateEnabled,
|
|
903
|
+
admissionHost: (deps.resolveAdmissionHost ?? resolveProcessAdmissionHost)(),
|
|
904
|
+
});
|
|
905
|
+
});
|
|
827
906
|
let lastSignalMs = now();
|
|
828
907
|
let lastSeenMtime = 0;
|
|
829
908
|
let lastAlive = null;
|
|
@@ -1168,7 +1247,7 @@ async function runWatcherLoopOwned(hqRoot, laneId, deps) {
|
|
|
1168
1247
|
quota,
|
|
1169
1248
|
providerRejection,
|
|
1170
1249
|
});
|
|
1171
|
-
|
|
1250
|
+
let updated = terminalWrite.record;
|
|
1172
1251
|
const effectiveEnvelope = terminalWrite.envelope ?? envelope;
|
|
1173
1252
|
const effectiveClassification = terminalWrite.classification;
|
|
1174
1253
|
logDecision({
|
|
@@ -1201,7 +1280,14 @@ async function runWatcherLoopOwned(hqRoot, laneId, deps) {
|
|
|
1201
1280
|
await emitMeshSoft(`lane_${updated.state}`, logDecision, tick.emit.at, (signal) => emitLaneTerminalStateEvent(mesh, updated, updated.state, updated.terminal_reason ?? effectiveClassification.reason, signal), meshRequestTimeoutMs);
|
|
1202
1281
|
}
|
|
1203
1282
|
}
|
|
1204
|
-
if (terminalWrite.changed
|
|
1283
|
+
if (terminalWrite.changed) {
|
|
1284
|
+
updated = await settleManagerStatusGate(hqRoot, updated, manifest.turn_id, tick.emit.at, deps, logDecision);
|
|
1285
|
+
}
|
|
1286
|
+
const managerStatusFollowupQueued = manifest.turn_id !== undefined &&
|
|
1287
|
+
updated.status_gate_history?.some((entry) => entry.event === "status_missed" &&
|
|
1288
|
+
entry.turn_id === manifest.turn_id &&
|
|
1289
|
+
entry.resume_queued === true) === true;
|
|
1290
|
+
if (terminalWrite.changed && deps.recoverQueuedMessages && !managerStatusFollowupQueued) {
|
|
1205
1291
|
try {
|
|
1206
1292
|
if (dropboxInboxHasMessages(hqRoot, laneId)) {
|
|
1207
1293
|
await deps.recoverQueuedMessages(updated);
|
|
@@ -722,7 +722,7 @@ export class TranscriptWatcher {
|
|
|
722
722
|
}
|
|
723
723
|
if (track.pendingTurn &&
|
|
724
724
|
nowMs - track.lastChangeAtMs >= TRANSCRIPT_TURN_QUIET_MS) {
|
|
725
|
-
if (this.emitKind(track, "turn_end", nowMs)) {
|
|
725
|
+
if (this.emitKind(track, "turn_end", nowMs, transcriptTurnAtMs(track, nowMs))) {
|
|
726
726
|
track.pendingTurn = false;
|
|
727
727
|
result.turns += 1;
|
|
728
728
|
}
|
|
@@ -824,7 +824,7 @@ export class TranscriptWatcher {
|
|
|
824
824
|
}
|
|
825
825
|
if (track.pendingTurn &&
|
|
826
826
|
nowMs - track.lastChangeAtMs >= TRANSCRIPT_TURN_QUIET_MS) {
|
|
827
|
-
if (this.emitKind(track, "turn_end", nowMs)) {
|
|
827
|
+
if (this.emitKind(track, "turn_end", nowMs, transcriptTurnAtMs(track, nowMs))) {
|
|
828
828
|
track.pendingTurn = false;
|
|
829
829
|
result.turns += 1;
|
|
830
830
|
}
|
|
@@ -916,7 +916,9 @@ export class TranscriptWatcher {
|
|
|
916
916
|
}
|
|
917
917
|
return env;
|
|
918
918
|
}
|
|
919
|
-
emitKind(track, kind, nowMs
|
|
919
|
+
emitKind(track, kind, nowMs,
|
|
920
|
+
/** Event time when it differs from detection time (turn_end). */
|
|
921
|
+
atMs = nowMs) {
|
|
920
922
|
let seq;
|
|
921
923
|
try {
|
|
922
924
|
seq = reserveSessionSeq(track.sessionId, 1, this.seqEnv());
|
|
@@ -925,13 +927,18 @@ export class TranscriptWatcher {
|
|
|
925
927
|
return false;
|
|
926
928
|
}
|
|
927
929
|
track.seq = seq;
|
|
930
|
+
// `at` must not go backwards against seq: a transcript time older than the
|
|
931
|
+
// last event this watcher sent for the session is clamped to that event.
|
|
932
|
+
const eventAtMs = track.lastSentAtMs !== undefined && atMs < track.lastSentAtMs
|
|
933
|
+
? track.lastSentAtMs
|
|
934
|
+
: atMs;
|
|
928
935
|
const opts = {
|
|
929
936
|
kind,
|
|
930
937
|
sessionId: track.sessionId,
|
|
931
938
|
harness: track.harness,
|
|
932
939
|
adapterVersion: TRANSCRIPT_ADAPTER_VERSION,
|
|
933
940
|
source: TRANSCRIPT_EVENT_SOURCE,
|
|
934
|
-
at: new Date(
|
|
941
|
+
at: new Date(eventAtMs).toISOString(),
|
|
935
942
|
seq,
|
|
936
943
|
cwd: track.cwd,
|
|
937
944
|
hqRoot: this.deps.hqRoot,
|
|
@@ -942,6 +949,7 @@ export class TranscriptWatcher {
|
|
|
942
949
|
};
|
|
943
950
|
try {
|
|
944
951
|
this.enqueueFn(opts);
|
|
952
|
+
track.lastSentAtMs = eventAtMs;
|
|
945
953
|
return true;
|
|
946
954
|
}
|
|
947
955
|
catch {
|
|
@@ -949,6 +957,19 @@ export class TranscriptWatcher {
|
|
|
949
957
|
}
|
|
950
958
|
}
|
|
951
959
|
}
|
|
960
|
+
/**
|
|
961
|
+
* A transcript turn ended when the transcript last changed (file mtime), not
|
|
962
|
+
* when a backed-off or restarted poll noticed it. Never later than now. A
|
|
963
|
+
* catch-up turn_end sent in the same tick as session_end (restart adoption of
|
|
964
|
+
* an already quiet transcript) therefore carries the earlier transcript time,
|
|
965
|
+
* not the session_end time.
|
|
966
|
+
*/
|
|
967
|
+
function transcriptTurnAtMs(track, nowMs) {
|
|
968
|
+
const mtimeMs = track.lastMtimeMs;
|
|
969
|
+
if (!Number.isFinite(mtimeMs) || mtimeMs <= 0)
|
|
970
|
+
return nowMs;
|
|
971
|
+
return Math.min(mtimeMs, nowMs);
|
|
972
|
+
}
|
|
952
973
|
/**
|
|
953
974
|
* Adaptive interval: busy ticks double up to MAX; quiet ticks return to base.
|
|
954
975
|
*/
|