@indigoai-us/hq-cli 5.293.0 → 5.295.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +39 -0
- package/assets/scaffold/core/scripts/outpost-jobs-reconcile.sh +5 -1
- package/dist/command-catalog.generated.d.ts +54 -33
- package/dist/command-catalog.generated.js +68 -42
- package/dist/commands/auth.d.ts +1 -0
- package/dist/commands/auth.js +76 -1
- package/dist/commands/core-command-catalog.js +3 -2
- package/dist/commands/core-delegate-w1b.d.ts +2 -0
- package/dist/commands/core-delegate-w1b.js +28 -2
- package/dist/commands/core-delegate.d.ts +3 -1
- package/dist/commands/core-delegate.js +35 -5
- package/dist/commands/core-handoff-fast.d.ts +18 -0
- package/dist/commands/core-handoff-fast.js +103 -0
- package/dist/commands/core-handoff-open-steps.d.ts +21 -0
- package/dist/commands/core-handoff-open-steps.js +264 -0
- package/dist/commands/core-handoff.d.ts +4 -2
- package/dist/commands/core-handoff.js +22 -249
- package/dist/commands/core-jobs-native.d.ts +2 -0
- package/dist/commands/core-jobs-native.js +22 -11
- package/dist/commands/core-knowledge-prefs.d.ts +2 -0
- package/dist/commands/core-knowledge-prefs.js +13 -1
- package/dist/commands/core-ontology-native.d.ts +2 -0
- package/dist/commands/core-ontology-native.js +22 -2
- package/dist/commands/core-policy.d.ts +2 -0
- package/dist/commands/core-policy.js +18 -1
- package/dist/commands/core.js +2 -1
- package/dist/commands/daemon.d.ts +2 -1
- package/dist/commands/daemon.js +22 -7
- package/dist/commands/doctor.js +0 -74
- package/dist/commands/lanes-autopilot.d.ts +11 -0
- package/dist/commands/lanes-autopilot.js +336 -0
- package/dist/commands/lanes-message.js +23 -5
- package/dist/commands/lanes-questions.js +8 -5
- package/dist/commands/lanes-resume.js +0 -3
- package/dist/commands/lanes-shared.d.ts +18 -2
- package/dist/commands/lanes-stories.d.ts +14 -0
- package/dist/commands/lanes-stories.js +308 -0
- package/dist/commands/lanes.d.ts +1 -9
- package/dist/commands/lanes.js +216 -481
- package/dist/fast-core.js +53 -25
- package/dist/fast-scaffold.d.ts +15 -7
- package/dist/fast-scaffold.js +167 -17
- package/dist/index.js +1 -10
- package/dist/lib/core-utils/sentry-report.d.ts +0 -19
- package/dist/lib/core-utils/sentry-report.js +8 -61
- package/dist/lib/core-utils/worktree-resume.d.ts +2 -0
- package/dist/lib/core-utils/worktree-resume.js +16 -0
- package/dist/lib/daemon/migrate.d.ts +8 -1
- package/dist/lib/daemon/migrate.js +19 -7
- package/dist/lib/doctor/checks/claude-wiring.js +9 -0
- package/dist/lib/lanes/autopilot-metrics.d.ts +99 -0
- package/dist/lib/lanes/autopilot-metrics.js +301 -0
- package/dist/lib/lanes/flag-keys.d.ts +2 -0
- package/dist/lib/lanes/flag-keys.js +2 -0
- package/dist/lib/lanes/manager-status-gate.d.ts +0 -17
- package/dist/lib/lanes/manager-status-gate.js +2 -43
- package/dist/lib/lanes/outpost-dispatch.d.ts +2 -0
- package/dist/lib/lanes/outpost-dispatch.js +7 -1
- package/dist/lib/lanes/outpost-hire-cache.d.ts +0 -1
- package/dist/lib/lanes/outpost-hire-cache.js +0 -1
- package/dist/lib/lanes/outpost-snapshot-memo.d.ts +6 -0
- package/dist/lib/lanes/outpost-snapshot-memo.js +14 -0
- package/dist/lib/lanes/reconcile.d.ts +2 -3
- package/dist/lib/lanes/reconcile.js +44 -38
- package/dist/lib/lanes/spawn.d.ts +0 -2
- package/dist/lib/lanes/spawn.js +18 -5
- package/dist/lib/lanes/store.d.ts +2 -0
- package/dist/lib/lanes/store.js +54 -3
- package/dist/lib/lanes/types.d.ts +18 -1
- package/dist/lib/lanes/types.js +1 -1
- package/dist/lib/lanes/watcher.d.ts +4 -3
- package/dist/lib/lanes/watcher.js +102 -34
- package/dist/lib/mesh/live/daemon/run.js +18 -1
- package/dist/native-fast-route.d.ts +3 -1
- package/dist/native-fast-route.js +10 -8
- package/dist/utils/cognito-session.d.ts +6 -0
- package/dist/utils/cognito-session.js +9 -0
- package/dist/utils/contribution-table.d.ts +1 -1
- package/dist/utils/fast-stream-errors.d.ts +12 -0
- package/dist/utils/fast-stream-errors.js +18 -0
- package/dist/utils/scaffold-fast-import-guard.d.ts +2 -2
- package/dist/utils/scaffold-fast-import-guard.js +6 -3
- package/dist/utils/self-update.js +31 -0
- package/dist/utils/version-gate.js +22 -3
- package/package.json +1 -1
- package/dist/fast-lanes-list-route.d.ts +0 -8
- package/dist/fast-lanes-list-route.js +0 -32
- package/dist/fast-lanes-list.d.ts +0 -3
- package/dist/fast-lanes-list.js +0 -224
- package/dist/lib/doctor/client-health-sentry-flag.d.ts +0 -23
- package/dist/lib/doctor/client-health-sentry-flag.js +0 -99
- package/dist/lib/doctor/client-health-sentry.d.ts +0 -71
- package/dist/lib/doctor/client-health-sentry.js +0 -286
- package/dist/lib/lanes/outpost-snapshot-cache.d.ts +0 -26
- package/dist/lib/lanes/outpost-snapshot-cache.js +0 -215
- package/dist/lib/lanes/outpost-snapshot-refresh.d.ts +0 -21
- package/dist/lib/lanes/outpost-snapshot-refresh.js +0 -329
|
@@ -32,7 +32,6 @@ import { answeredQuestionEntries, questionRecordRefusalCode, raiseEnvelopeQuesti
|
|
|
32
32
|
import { SILENCE_MS, appendWatcherEvent, classifyLaneExit, isPidAlive, missingExpectedEnvelopeFile, readLatestEnvelopeFile, readLatestEnvelopeRecord, watcherEventsPath, watcherOutLogPath, writeWatcherDecision, } from "./watcher.js";
|
|
33
33
|
import { createLaneMesh, emitLaneDecisionEvents, emitLaneTerminalStateEvent, laneMeshCompany, } from "./mesh-events.js";
|
|
34
34
|
import { activeLaneStories, laneStories, storyStatusForEnvelope } from "./stories.js";
|
|
35
|
-
import { resolveManagerStatusGateForLane } from "./manager-status-gate.js";
|
|
36
35
|
import { decideEnvelopeRepair, envelopeQuestionIds, routeProviderFailure } from "./watcher.js";
|
|
37
36
|
import { PROVIDER_REJECTED_REASON, providerRejectionDetails, readProviderRejection, } from "./provider-rejection.js";
|
|
38
37
|
/** Spec §9: unanswered at the current hop for 60 minutes → one hop. */
|
|
@@ -48,43 +47,11 @@ export const PICKUP_VALIDATION_MAX_AGE_MS = 15 * 60 * 1000;
|
|
|
48
47
|
export const PICKUP_VALIDATION_TIMEOUT_REASON = "pickup_validation_timeout";
|
|
49
48
|
export const RECONCILE_UNIT_NAME = "hq-lanes-reconcile.service";
|
|
50
49
|
export const RECONCILE_TIMER_NAME = "hq-lanes-reconcile.timer";
|
|
51
|
-
function spawnDepsForHost(deps
|
|
50
|
+
function spawnDepsForHost(deps) {
|
|
52
51
|
return {
|
|
53
52
|
...(deps.spawnLaneDeps ?? {}),
|
|
54
53
|
hostname: deps.host.hostname,
|
|
55
54
|
admissionHost: deps.host,
|
|
56
|
-
...(lane !== undefined && deps.managerStatusGateByLane?.has(lane.lane_id)
|
|
57
|
-
? { managerStatusGateEnabled: deps.managerStatusGateByLane.get(lane.lane_id) === true }
|
|
58
|
-
: {}),
|
|
59
|
-
};
|
|
60
|
-
}
|
|
61
|
-
async function prepareManagerStatusGateStarts(root, lanes, deps, summary) {
|
|
62
|
-
const enabledByLane = new Map();
|
|
63
|
-
let gateFailureRecorded = false;
|
|
64
|
-
if (!deps.dryRun) {
|
|
65
|
-
await Promise.all(lanes.filter((lane) => laneRunsHere(lane, deps.host)).map(async (lane) => {
|
|
66
|
-
const resolution = await resolveManagerStatusGateForLane(root, lane, {
|
|
67
|
-
env: deps.env,
|
|
68
|
-
readFlag: deps.readRegistryFlag,
|
|
69
|
-
onError: (err) => {
|
|
70
|
-
if (gateFailureRecorded)
|
|
71
|
-
return;
|
|
72
|
-
gateFailureRecorded = true;
|
|
73
|
-
const code = err instanceof LanesError ? err.error : "manager_status_gate_unavailable";
|
|
74
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
75
|
-
summary.skipped.push({
|
|
76
|
-
kind: "lane",
|
|
77
|
-
id: lane.lane_id,
|
|
78
|
-
why: `manager status gate disabled for this reconcile pass: ${code}: ${message}`,
|
|
79
|
-
});
|
|
80
|
-
},
|
|
81
|
-
});
|
|
82
|
-
enabledByLane.set(lane.lane_id, resolution.enabled);
|
|
83
|
-
}));
|
|
84
|
-
}
|
|
85
|
-
return {
|
|
86
|
-
...deps,
|
|
87
|
-
managerStatusGateByLane: enabledByLane,
|
|
88
55
|
};
|
|
89
56
|
}
|
|
90
57
|
function logSkip(skip) {
|
|
@@ -854,7 +821,7 @@ export async function runPassBody(hqRoot, deps) {
|
|
|
854
821
|
if (deps.afterLoad)
|
|
855
822
|
await deps.afterLoad();
|
|
856
823
|
await runRunawayToolGuardPass(hqRoot, lanes, config, deps, now, summary);
|
|
857
|
-
const startDeps =
|
|
824
|
+
const startDeps = deps;
|
|
858
825
|
markOverdueLanes(hqRoot, lanes, summary, deps, now);
|
|
859
826
|
await heartbeatAndDeadHosts(hqRoot, lanes, summary, startDeps, now);
|
|
860
827
|
for (const question of questions) {
|
|
@@ -1229,7 +1196,7 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
|
|
|
1229
1196
|
classified.account) {
|
|
1230
1197
|
try {
|
|
1231
1198
|
const recovered = await retryLaneOnAlternateAccount(hqRoot, classified, accountRetryReason, "reconcile", {
|
|
1232
|
-
...spawnDepsForHost(deps
|
|
1199
|
+
...spawnDepsForHost(deps),
|
|
1233
1200
|
});
|
|
1234
1201
|
summary.dead_host.push({
|
|
1235
1202
|
id: lane.lane_id,
|
|
@@ -1512,7 +1479,7 @@ async function admitQueued(hqRoot, lanes, summary, deps, now) {
|
|
|
1512
1479
|
nowMs,
|
|
1513
1480
|
});
|
|
1514
1481
|
const spawnFn = deps.spawn ??
|
|
1515
|
-
((root, lane) => spawnLaneWorker(root, lane, spawnDepsForHost(deps
|
|
1482
|
+
((root, lane) => spawnLaneWorker(root, lane, spawnDepsForHost(deps)));
|
|
1516
1483
|
const running = lanes.filter((lane) => (lane.state === "running" ||
|
|
1517
1484
|
hasFreshAdmissionReservation(lane, now.getTime()) ||
|
|
1518
1485
|
hasActivePendingResume(lane, now.getTime())) &&
|
|
@@ -1699,6 +1666,21 @@ function recordKeepAliveEvent(hqRoot, laneId, summary, event) {
|
|
|
1699
1666
|
});
|
|
1700
1667
|
}
|
|
1701
1668
|
}
|
|
1669
|
+
function keepAliveAccountSelectionEvidence(selection) {
|
|
1670
|
+
if (selection === undefined)
|
|
1671
|
+
return undefined;
|
|
1672
|
+
return {
|
|
1673
|
+
account: selection.account,
|
|
1674
|
+
selected_at: selection.selected_at,
|
|
1675
|
+
reason: selection.reason,
|
|
1676
|
+
considered: selection.considered.map((row) => ({
|
|
1677
|
+
account: row.account,
|
|
1678
|
+
live_lanes: row.live_lanes,
|
|
1679
|
+
eligible: row.eligible,
|
|
1680
|
+
reason: row.ineligible_reason ?? null,
|
|
1681
|
+
})),
|
|
1682
|
+
};
|
|
1683
|
+
}
|
|
1702
1684
|
function keepAliveSkipGuidance(laneId, reason) {
|
|
1703
1685
|
switch (reason) {
|
|
1704
1686
|
case "missing_opt_in_time":
|
|
@@ -1892,10 +1874,12 @@ async function resumeBlockedKeepAlive(hqRoot, lane, questions, summary, deps, no
|
|
|
1892
1874
|
return;
|
|
1893
1875
|
}
|
|
1894
1876
|
let current = latest;
|
|
1877
|
+
let accountSelection;
|
|
1895
1878
|
if (current.provider === "claude" && deps.prepareKeepAliveClaudeResume !== undefined) {
|
|
1896
1879
|
try {
|
|
1897
1880
|
const prepared = await deps.prepareKeepAliveClaudeResume(current);
|
|
1898
1881
|
current = prepared.lane;
|
|
1882
|
+
accountSelection = prepared.accountSelection;
|
|
1899
1883
|
}
|
|
1900
1884
|
catch (error) {
|
|
1901
1885
|
markDirty(summary, { kind: "lane", id: lane.lane_id, why: `blocked keep-alive account preparation failed: ${error instanceof Error ? error.message : String(error)}` });
|
|
@@ -1917,8 +1901,16 @@ async function resumeBlockedKeepAlive(hqRoot, lane, questions, summary, deps, no
|
|
|
1917
1901
|
const queued = casLaneIf(hqRoot, latest.lane_id, (latestRecord) => latestRecord.version === expectedVersion && latestRecord.state === "blocked" && latestRecord.last_envelope?.decision === "blocked" && latestRecord.last_envelope.at === terminalAt, (current) => ({
|
|
1918
1902
|
...result.lane,
|
|
1919
1903
|
version: current.version,
|
|
1904
|
+
pending_resume: result.lane.pending_resume === undefined
|
|
1905
|
+
? undefined
|
|
1906
|
+
: { ...result.lane.pending_resume, keep_alive: true },
|
|
1920
1907
|
keep_alive_blocked_resume_streak: priorResumes + 1,
|
|
1921
1908
|
keep_alive_resume_count: (current.keep_alive_resume_count ?? 0) + 1,
|
|
1909
|
+
keep_alive_history: appendCappedHistory(current.keep_alive_history ?? [], {
|
|
1910
|
+
at: now.toISOString(),
|
|
1911
|
+
resumed_from: "blocked",
|
|
1912
|
+
account: current.account ?? null,
|
|
1913
|
+
}),
|
|
1922
1914
|
keep_alive_last_resume_at: now.toISOString(),
|
|
1923
1915
|
status_gate_history: appendCappedHistory(current.status_gate_history ?? [], {
|
|
1924
1916
|
event: "keep_alive_blocked_resumed",
|
|
@@ -1935,6 +1927,9 @@ async function resumeBlockedKeepAlive(hqRoot, lane, questions, summary, deps, no
|
|
|
1935
1927
|
at: now.toISOString(),
|
|
1936
1928
|
attempt: (lane.keep_alive_resume_count ?? 0) + 1,
|
|
1937
1929
|
resumed_from: "blocked",
|
|
1930
|
+
...(accountSelection !== undefined
|
|
1931
|
+
? { account_selection: keepAliveAccountSelectionEvidence(accountSelection) }
|
|
1932
|
+
: {}),
|
|
1938
1933
|
});
|
|
1939
1934
|
}
|
|
1940
1935
|
}
|
|
@@ -2390,6 +2385,7 @@ async function resumeKeepAlive(hqRoot, lanes, questions, summary, deps, now, fla
|
|
|
2390
2385
|
writeAnsweredQuestions(hqRoot, current, questions);
|
|
2391
2386
|
}
|
|
2392
2387
|
let accountSelectionFailure;
|
|
2388
|
+
let accountSelection;
|
|
2393
2389
|
if (current.provider === "claude" && deps.prepareKeepAliveClaudeResume !== undefined) {
|
|
2394
2390
|
const optInBeforePreparation = current.keep_alive_opted_in_at;
|
|
2395
2391
|
let preparedLane;
|
|
@@ -2397,6 +2393,7 @@ async function resumeKeepAlive(hqRoot, lanes, questions, summary, deps, now, fla
|
|
|
2397
2393
|
const prepared = await deps.prepareKeepAliveClaudeResume(current);
|
|
2398
2394
|
preparedLane = prepared.lane;
|
|
2399
2395
|
accountSelectionFailure = prepared.selectionFailure;
|
|
2396
|
+
accountSelection = prepared.accountSelection;
|
|
2400
2397
|
}
|
|
2401
2398
|
catch (err) {
|
|
2402
2399
|
// Account balancing is best-effort for keep-alive. A failure to read
|
|
@@ -2514,12 +2511,18 @@ async function resumeKeepAlive(hqRoot, lanes, questions, summary, deps, now, fla
|
|
|
2514
2511
|
pending_resume: {
|
|
2515
2512
|
session_id: currentSessionId,
|
|
2516
2513
|
trigger: carriesAnsweredQuestions ? "answered_questions" : "resume",
|
|
2514
|
+
keep_alive: true,
|
|
2517
2515
|
resumed_from: resumedFrom,
|
|
2518
2516
|
text: resumeText,
|
|
2519
2517
|
compact: true,
|
|
2520
2518
|
at: nowIso,
|
|
2521
2519
|
},
|
|
2522
2520
|
keep_alive_resume_count: attempt,
|
|
2521
|
+
keep_alive_history: appendCappedHistory(latestLane.keep_alive_history ?? [], {
|
|
2522
|
+
at: nowIso,
|
|
2523
|
+
resumed_from: resumedFrom,
|
|
2524
|
+
account: latestLane.account ?? null,
|
|
2525
|
+
}),
|
|
2523
2526
|
keep_alive_last_resume_at: nowIso,
|
|
2524
2527
|
keep_alive_next_eligible_at: nextEligibleAt,
|
|
2525
2528
|
keep_alive_exhausted_at: undefined,
|
|
@@ -2573,6 +2576,9 @@ async function resumeKeepAlive(hqRoot, lanes, questions, summary, deps, now, fla
|
|
|
2573
2576
|
attempt,
|
|
2574
2577
|
next_eligible_at: nextEligibleAt,
|
|
2575
2578
|
resumed_from: resumedFrom,
|
|
2579
|
+
...(accountSelection !== undefined
|
|
2580
|
+
? { account_selection: keepAliveAccountSelectionEvidence(accountSelection) }
|
|
2581
|
+
: {}),
|
|
2576
2582
|
});
|
|
2577
2583
|
}
|
|
2578
2584
|
}
|
|
@@ -2604,7 +2610,7 @@ async function resumeAwaiting(hqRoot, lanes, questions, summary, deps, onlyLaneI
|
|
|
2604
2610
|
const throttle = await decideAdmission(hqRoot, config.throttle, throttleDeps, deps.cpuBusyFraction, deps.cpuObservation);
|
|
2605
2611
|
throttleDecision = throttle.decision;
|
|
2606
2612
|
}
|
|
2607
|
-
const spawnFn = deps.spawn ?? ((root, lane) => spawnLaneWorker(root, lane, spawnDepsForHost(deps
|
|
2613
|
+
const spawnFn = deps.spawn ?? ((root, lane) => spawnLaneWorker(root, lane, spawnDepsForHost(deps)));
|
|
2608
2614
|
const nowMs = deps.now().getTime();
|
|
2609
2615
|
const fallbackAnswerAt = new Date(nowMs).toISOString();
|
|
2610
2616
|
const answerAtByLane = new Map();
|
|
@@ -88,8 +88,6 @@ export interface SpawnLaneDeps {
|
|
|
88
88
|
runCompaction?: RunCompactionFn;
|
|
89
89
|
/** Finds a session transcript under an account home. Injected in tests. */
|
|
90
90
|
locateTranscript?: (home: string, sessionId: string) => string | null;
|
|
91
|
-
/** Pre-read by the async command/reconcile boundary before this sync spawn. */
|
|
92
|
-
managerStatusGateEnabled?: boolean;
|
|
93
91
|
}
|
|
94
92
|
export interface ProviderLaunch {
|
|
95
93
|
command: string;
|
package/dist/lib/lanes/spawn.js
CHANGED
|
@@ -354,12 +354,13 @@ function archivedResumeMessageText(hqRoot, laneId, messageId) {
|
|
|
354
354
|
}
|
|
355
355
|
function pendingResumeForRetry(hqRoot, lane, sessionId, at) {
|
|
356
356
|
const last = lane.last_resume;
|
|
357
|
+
const keepAlive = last?.keep_alive === true ? { keep_alive: true } : {};
|
|
357
358
|
if (last === undefined || last.session_id !== sessionId) {
|
|
358
|
-
return { session_id: sessionId, trigger: "resume", resumed_from: "interrupted", at };
|
|
359
|
+
return { session_id: sessionId, trigger: "resume", resumed_from: "interrupted", at, ...keepAlive };
|
|
359
360
|
}
|
|
360
361
|
const compact = last.compact === undefined ? {} : { compact: last.compact };
|
|
361
362
|
if (last.trigger === "answered_questions") {
|
|
362
|
-
return { session_id: sessionId, trigger: "answered_questions", at, ...compact };
|
|
363
|
+
return { session_id: sessionId, trigger: "answered_questions", at, ...compact, ...keepAlive };
|
|
363
364
|
}
|
|
364
365
|
if (last.trigger === "message") {
|
|
365
366
|
if (last.message_id === undefined) {
|
|
@@ -376,6 +377,7 @@ function pendingResumeForRetry(hqRoot, lane, sessionId, at) {
|
|
|
376
377
|
message_id: last.message_id,
|
|
377
378
|
at,
|
|
378
379
|
...compact,
|
|
380
|
+
...keepAlive,
|
|
379
381
|
};
|
|
380
382
|
}
|
|
381
383
|
return {
|
|
@@ -386,6 +388,7 @@ function pendingResumeForRetry(hqRoot, lane, sessionId, at) {
|
|
|
386
388
|
...(last.message_id !== undefined ? { message_id: last.message_id } : {}),
|
|
387
389
|
at,
|
|
388
390
|
...compact,
|
|
391
|
+
...keepAlive,
|
|
389
392
|
};
|
|
390
393
|
}
|
|
391
394
|
/** Change account only after a provider/runtime account failure, then re-admit. */
|
|
@@ -1112,6 +1115,8 @@ export function envelopeRepairPrompt(input) {
|
|
|
1112
1115
|
`Missing expected envelope file: ${input.envelopePath}`,
|
|
1113
1116
|
"The previous envelope is stale and must not be reused or copied into this run's result.",
|
|
1114
1117
|
"Printing an envelope in the conversation or writing it to last-message.txt is not enough; write the JSON object directly to the expected file.",
|
|
1118
|
+
"If the task is unfinished, write decision: done, story_complete: false, and set story_status and story_complete_reason to describe the work that remains. The repair turn itself is never a reason for blocked.",
|
|
1119
|
+
"This repair turn only writes the missing envelope; do not resume the task.",
|
|
1115
1120
|
"Schema:",
|
|
1116
1121
|
JSON.stringify(LANE_ENVELOPE_JSON_SCHEMA, null, 2),
|
|
1117
1122
|
`Write the corrected JSON object to ${input.envelopePath}, overwriting it. ` +
|
|
@@ -2000,9 +2005,16 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
|
|
|
2000
2005
|
// Claim running before spawning. Two callers that both observe queued must
|
|
2001
2006
|
// not both start processes; the CAS loser returns without spawn.
|
|
2002
2007
|
const nowIso = (deps.now ?? (() => new Date()))().toISOString();
|
|
2003
|
-
|
|
2004
|
-
|
|
2005
|
-
|
|
2008
|
+
let statusGateHeartbeat;
|
|
2009
|
+
try {
|
|
2010
|
+
statusGateHeartbeat = managerStatusGateHeartbeat(hqRoot, existing, laneTurnId, nowIso);
|
|
2011
|
+
}
|
|
2012
|
+
catch (err) {
|
|
2013
|
+
const code = err instanceof LanesError ? err.error : "manager_status_gate_unavailable";
|
|
2014
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
2015
|
+
process.stderr.write(`hq lanes: manager status gate disabled for ${lane.lane_id}: ${code}: ${message}\n`);
|
|
2016
|
+
statusGateHeartbeat = undefined;
|
|
2017
|
+
}
|
|
2006
2018
|
let claimed = false;
|
|
2007
2019
|
const claimedRecord = retryUpdateLane(hqRoot, lane.lane_id, "spawn", (current) => {
|
|
2008
2020
|
claimed = false;
|
|
@@ -2102,6 +2114,7 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
|
|
|
2102
2114
|
at: nowIso,
|
|
2103
2115
|
account: resolved.id,
|
|
2104
2116
|
trigger: pendingResume.trigger,
|
|
2117
|
+
...(pendingResume.keep_alive === true ? { keep_alive: true } : {}),
|
|
2105
2118
|
...(pendingResume.message_id !== undefined ? { message_id: pendingResume.message_id } : {}),
|
|
2106
2119
|
...(pendingResume.monitor_event === true ? { monitor_event: true } : {}),
|
|
2107
2120
|
...(pendingResume.resumed_from !== undefined ? { resumed_from: pendingResume.resumed_from } : {}),
|
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
* compare-and-set on `version`. Histories are capped at 50 at the append site.
|
|
8
8
|
*/
|
|
9
9
|
import * as fs from "node:fs";
|
|
10
|
+
import { type OutpostIdentity } from "./admission-host.js";
|
|
10
11
|
import { type AttributionHistoryEntry, type LaneRecord, type LaneWriter, type QuestionRecord, type RoutingHistoryEntry, type SeniorHistoryEntry } from "./types.js";
|
|
11
12
|
export declare const LANES_STORE_REL: string;
|
|
12
13
|
export declare const LANE_RECORDS_DIR = "lanes";
|
|
@@ -15,6 +16,7 @@ export declare const PENDING_INDEX_DIR: string;
|
|
|
15
16
|
export interface StoreDeps {
|
|
16
17
|
now: () => Date;
|
|
17
18
|
hostname: () => string;
|
|
19
|
+
outpostIdentity: () => OutpostIdentity;
|
|
18
20
|
mkdirSync: typeof fs.mkdirSync;
|
|
19
21
|
readFileSync: typeof fs.readFileSync;
|
|
20
22
|
writeFileSync: typeof fs.writeFileSync;
|
package/dist/lib/lanes/store.js
CHANGED
|
@@ -14,6 +14,7 @@ import semver from "semver";
|
|
|
14
14
|
import { CLI_VERSION } from "../../cli-version.js";
|
|
15
15
|
import { generateUlid } from "../mesh/live/ulid.js";
|
|
16
16
|
import { resolveLanesHqRoot } from "./hq-root.js";
|
|
17
|
+
import { readOutpostIdentity } from "./admission-host.js";
|
|
17
18
|
import { LaneNotFoundError, LaneSchemaError, LaneVersionConflict, LanesError, isLaneVersionConflict, } from "./errors.js";
|
|
18
19
|
import { HISTORY_CAP, HOST_KINDS, LANE_STORY_STATUSES, LANE_SCHEMA_VERSION, QUESTION_SCHEMA_VERSION, SENIOR_KINDS, isHostKind, isLaneProvider, isQueuedReason, isLaneState, isQuestionStatus, isSeniorKind, isTerminalLaneState, parseMajorSchema, REQUIRED_LANE_RECORD_FIELDS, LANE_FIELD_WRITERS, LANE_WRITERS, } from "./types.js";
|
|
19
20
|
import { laneStories } from "./stories.js";
|
|
@@ -29,6 +30,7 @@ function defaultStoreDeps() {
|
|
|
29
30
|
return {
|
|
30
31
|
now: () => new Date(),
|
|
31
32
|
hostname: () => os.hostname(),
|
|
33
|
+
outpostIdentity: () => readOutpostIdentity(),
|
|
32
34
|
mkdirSync: fs.mkdirSync.bind(fs),
|
|
33
35
|
readFileSync: fs.readFileSync.bind(fs),
|
|
34
36
|
writeFileSync: fs.writeFileSync.bind(fs),
|
|
@@ -749,6 +751,28 @@ export function parseLaneRecord(raw, filePath) {
|
|
|
749
751
|
path: filePath,
|
|
750
752
|
});
|
|
751
753
|
}
|
|
754
|
+
if (raw.keep_alive_history !== undefined) {
|
|
755
|
+
if (!Array.isArray(raw.keep_alive_history) || raw.keep_alive_history.length > HISTORY_CAP) {
|
|
756
|
+
throw new LaneSchemaError(`keep_alive_history must be an array capped at ${HISTORY_CAP} entries`, {
|
|
757
|
+
path: filePath,
|
|
758
|
+
field: "keep_alive_history",
|
|
759
|
+
});
|
|
760
|
+
}
|
|
761
|
+
for (const [index, value] of raw.keep_alive_history.entries()) {
|
|
762
|
+
const field = `keep_alive_history[${index}]`;
|
|
763
|
+
if (!isPlainObject(value) ||
|
|
764
|
+
Object.keys(value).some((key) => !["at", "resumed_from", "account"].includes(key)) ||
|
|
765
|
+
(value.resumed_from !== "done" && value.resumed_from !== "interrupted" && value.resumed_from !== "blocked") ||
|
|
766
|
+
(value.account !== null && typeof value.account !== "string")) {
|
|
767
|
+
throw new LaneSchemaError(`${field} must be a keep-alive resume history entry`, {
|
|
768
|
+
path: filePath,
|
|
769
|
+
field,
|
|
770
|
+
value,
|
|
771
|
+
});
|
|
772
|
+
}
|
|
773
|
+
requireTimestamp(value.at, `${field}.at`, filePath);
|
|
774
|
+
}
|
|
775
|
+
}
|
|
752
776
|
if (raw.stop_history !== undefined) {
|
|
753
777
|
if (!Array.isArray(raw.stop_history)) {
|
|
754
778
|
throw new LaneSchemaError("stop_history must be an array when present", {
|
|
@@ -1238,9 +1262,36 @@ export function readAllLanes(hqRoot, deps = defaultStoreDeps()) {
|
|
|
1238
1262
|
...deps,
|
|
1239
1263
|
conflictSidecarPathsFor: (recordPath) => sidecarsByRecord.get(path.basename(recordPath)) ?? [],
|
|
1240
1264
|
};
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1265
|
+
const records = [];
|
|
1266
|
+
for (const laneId of laneIds) {
|
|
1267
|
+
try {
|
|
1268
|
+
records.push(readLane(hqRoot, laneId, indexedDeps));
|
|
1269
|
+
}
|
|
1270
|
+
catch (err) {
|
|
1271
|
+
if (!(err instanceof LaneSchemaError))
|
|
1272
|
+
throw err;
|
|
1273
|
+
const filePath = laneRecordPath(hqRoot, laneId);
|
|
1274
|
+
const raw = readJsonFile(filePath, indexedDeps);
|
|
1275
|
+
const host = raw !== null && typeof raw === "object" && !Array.isArray(raw)
|
|
1276
|
+
? raw.host
|
|
1277
|
+
: undefined;
|
|
1278
|
+
const hostRecord = host !== null && typeof host === "object" && !Array.isArray(host)
|
|
1279
|
+
? host
|
|
1280
|
+
: undefined;
|
|
1281
|
+
const outpostId = hostRecord?.kind === "outpost" &&
|
|
1282
|
+
typeof hostRecord.id === "string" && hostRecord.id.trim() !== ""
|
|
1283
|
+
? hostRecord.id
|
|
1284
|
+
: undefined;
|
|
1285
|
+
const identity = indexedDeps.outpostIdentity();
|
|
1286
|
+
const isRemoteMirror = outpostId !== undefined && (identity.kind === "not_outpost" ||
|
|
1287
|
+
(identity.kind === "id" && identity.id !== outpostId));
|
|
1288
|
+
if (!isRemoteMirror)
|
|
1289
|
+
throw err;
|
|
1290
|
+
const field = typeof err.details.field === "string" ? err.details.field : "unknown_field";
|
|
1291
|
+
process.stderr.write(`lanes store: skipping malformed mirrored lane ${laneId} field ${field}: ${err.message}\n`);
|
|
1292
|
+
}
|
|
1293
|
+
}
|
|
1294
|
+
return records.sort((a, b) => Date.parse(a.created_at) - Date.parse(b.created_at) ||
|
|
1244
1295
|
a.lane_id.localeCompare(b.lane_id));
|
|
1245
1296
|
}
|
|
1246
1297
|
export function tryReadQuestion(hqRoot, questionId, deps = defaultStoreDeps()) {
|
|
@@ -407,6 +407,11 @@ export interface KeepAliveBlockedHistoryEntry {
|
|
|
407
407
|
resume_skipped_reason?: "open_question" | "blocked_grace";
|
|
408
408
|
resume_queued?: boolean;
|
|
409
409
|
}
|
|
410
|
+
export interface KeepAliveResumeHistoryEntry {
|
|
411
|
+
at: string;
|
|
412
|
+
resumed_from: "done" | "interrupted" | "blocked";
|
|
413
|
+
account: string | null;
|
|
414
|
+
}
|
|
410
415
|
export interface KrFreshnessHistoryEntry {
|
|
411
416
|
event: "kr_stalled";
|
|
412
417
|
turn_id: string;
|
|
@@ -418,7 +423,13 @@ export interface KrFreshnessHistoryEntry {
|
|
|
418
423
|
resume_queued?: boolean;
|
|
419
424
|
resume_skipped_reason?: "open_question" | "blocked" | "miss_limit" | "no_provider_session";
|
|
420
425
|
}
|
|
421
|
-
export
|
|
426
|
+
export interface EnvelopeRepairContinueHistoryEntry {
|
|
427
|
+
event: "envelope_repair_continued";
|
|
428
|
+
at: string;
|
|
429
|
+
envelope_ref: string;
|
|
430
|
+
session_id: string;
|
|
431
|
+
}
|
|
432
|
+
export type LaneGateHistoryEntry = ManagerStatusGateHistoryEntry | BlockerFreshnessHistoryEntry | KrFreshnessHistoryEntry | EnvelopeRepairContinueHistoryEntry | KeepAliveBlockedHistoryEntry;
|
|
422
433
|
export interface LaneStory {
|
|
423
434
|
project_id: string;
|
|
424
435
|
story_id: string;
|
|
@@ -491,6 +502,8 @@ export interface LaneRecord {
|
|
|
491
502
|
keep_alive_blocked_reason?: "not_opted_in" | "missing_opt_in_time" | "envelope_before_opt_in" | "interruption_before_opt_in" | "missing_interruption_time" | "retired" | "superseded" | "retired_list_invalid" | "terminal_state";
|
|
492
503
|
/** Consecutive automatic keep-alive resumes since the last human reset. */
|
|
493
504
|
keep_alive_resume_count?: number;
|
|
505
|
+
/** Bounded audit trail of automatic keep-alive resumes; human resets never clear it. */
|
|
506
|
+
keep_alive_history?: KeepAliveResumeHistoryEntry[];
|
|
494
507
|
/** Blocked keep-alive resumes in the current blocked episode. */
|
|
495
508
|
keep_alive_blocked_resume_streak?: number;
|
|
496
509
|
/** Time the most recent automatic keep-alive resume was queued. */
|
|
@@ -700,6 +713,8 @@ export interface PendingResume {
|
|
|
700
713
|
/** Fresh answered-question launches need no previous session id. */
|
|
701
714
|
session_id?: string;
|
|
702
715
|
trigger: "answered_questions" | "message" | "resume";
|
|
716
|
+
/** True when this queued resume was initiated automatically by keep-alive. */
|
|
717
|
+
keep_alive?: true;
|
|
703
718
|
resumed_from?: "interrupted" | "done" | "blocked" | "failed";
|
|
704
719
|
from_failed?: true;
|
|
705
720
|
/** Senior that authorized a failed-lane resume, rechecked at the claim CAS. */
|
|
@@ -727,6 +742,8 @@ export interface LaneResumeRecord {
|
|
|
727
742
|
/** Account the resumed run was bound to. */
|
|
728
743
|
account?: string;
|
|
729
744
|
trigger: "answered_questions" | "message" | "resume";
|
|
745
|
+
/** True when this was an automatic keep-alive resume. */
|
|
746
|
+
keep_alive?: true;
|
|
730
747
|
message_id?: string;
|
|
731
748
|
monitor_event?: true;
|
|
732
749
|
/**
|
package/dist/lib/lanes/types.js
CHANGED
|
@@ -225,7 +225,7 @@ export const LANE_FIELD_WRITERS = {
|
|
|
225
225
|
pickup_validation_pending: ["creator", "command", "spawn", "reconcile"],
|
|
226
226
|
pickup_validation_pending_since: ["creator", "command", "spawn", "reconcile"],
|
|
227
227
|
priority: ["command"], worker: ["creator", "command"], worker_path: ["creator", "command"], worker_pack: ["creator", "command"], worker_profile: ["creator", "command"], worker_profile_hash: ["creator", "command", "spawn"], registry_worker: ["creator", "command", "spawn"], registry_worker_approval_history: ["command", "watcher", "reconcile"], worker_profile_repin_history: ["command"], worker_profile_ignored: ["creator", "command", "spawn"], worker_policy_delivery: ["command", "spawn"],
|
|
228
|
-
keep_alive: ["creator", "command"], keep_alive_standing: ["command"], keep_alive_opted_in_at: ["creator", "command"], keep_alive_blocked_reason: ["command", "reconcile"], keep_alive_resume_count: ["command", "reconcile"], keep_alive_blocked_resume_streak: ["reconcile", "spawn", "watcher"], keep_alive_last_resume_at: ["command", "reconcile"], keep_alive_next_eligible_at: ["command", "reconcile"], keep_alive_exhausted_at: ["command", "reconcile"],
|
|
228
|
+
keep_alive: ["creator", "command"], keep_alive_standing: ["command"], keep_alive_opted_in_at: ["creator", "command"], keep_alive_blocked_reason: ["command", "reconcile"], keep_alive_resume_count: ["command", "reconcile"], keep_alive_history: ["reconcile"], keep_alive_blocked_resume_streak: ["reconcile", "spawn", "watcher"], keep_alive_last_resume_at: ["command", "reconcile"], keep_alive_next_eligible_at: ["command", "reconcile"], keep_alive_exhausted_at: ["command", "reconcile"],
|
|
229
229
|
provider: ["creator"], account: ["creator", "spawn", "command", "watcher", "reconcile"], account_selection: ["creator", "command"], admission_accounts: ["creator", "command"], account_history: ["command", "spawn", "watcher", "reconcile"], failed_resume_history: ["command"], pending_resume: ["command", "spawn", "watcher", "reconcile"], placement_default: ["creator"], host: ["creator", "command"], origin_host: ["creator"], origin_host_source: ["creator"],
|
|
230
230
|
version: ["creator"], created_at: ["creator"], updated_at: ["creator"], ended_at: ["reconcile"], envelope_summary: ["reconcile"], attribution_history: ["command"],
|
|
231
231
|
last_envelope: ["watcher", "reconcile"], model: ["creator", "spawn", "command"], model_source: ["creator", "spawn", "command"], effort: ["creator", "spawn", "command"], effort_source: ["creator", "spawn", "command"], fast_mode: ["spawn"],
|
|
@@ -41,6 +41,7 @@ import { type EnvelopeRepairSpawnInput, type SpawnLaneDeps } from "./spawn.js";
|
|
|
41
41
|
/** Spec §5.7: no signal for 20 minutes while the process is still alive. */
|
|
42
42
|
export declare const SILENCE_MS: number;
|
|
43
43
|
export declare const WATCHER_POLL_MS = 5000;
|
|
44
|
+
export declare const ENVELOPE_REPAIR_CONTINUE_TEXT = "continue the task; the previous turn ended without an envelope.";
|
|
44
45
|
/** Synced last_host_report_at cadence. Reconcile dead-host TTL is 20 minutes. */
|
|
45
46
|
export declare const WATCHER_HEARTBEAT_MS: number;
|
|
46
47
|
/**
|
|
@@ -49,7 +50,7 @@ export declare const WATCHER_HEARTBEAT_MS: number;
|
|
|
49
50
|
* once the file would exceed this, the oldest complete lines are dropped.
|
|
50
51
|
*/
|
|
51
52
|
export declare const WATCHER_OUT_MAX_BYTES: number;
|
|
52
|
-
export declare const WATCHER_DECISION_EVENTS: readonly ["liveness_transition", "silence_deadline", "envelope_observed", "envelope_invalid", "envelope_repair", "classification", "terminal_write", "message_recovery", "message_recovery_failed", "message_undeliverable", "mesh_event", "mesh_event_failed"];
|
|
53
|
+
export declare const WATCHER_DECISION_EVENTS: readonly ["liveness_transition", "silence_deadline", "envelope_observed", "envelope_invalid", "envelope_repair", "envelope_repair_continue", "classification", "terminal_write", "message_recovery", "message_recovery_failed", "message_undeliverable", "mesh_event", "mesh_event_failed"];
|
|
53
54
|
/** Terminal reason when the envelope failed to parse after the repair turn (spec §4). */
|
|
54
55
|
export { ENVELOPE_INVALID_REASON };
|
|
55
56
|
export type WatcherDecisionEvent = (typeof WATCHER_DECISION_EVENTS)[number];
|
|
@@ -232,11 +233,11 @@ export interface WatcherLoopDeps {
|
|
|
232
233
|
/** Deliver a lane-senior escalation through the normal lane-message router. */
|
|
233
234
|
deliverLaneSeniorMessage?: (seniorLaneId: string, text: string) => Promise<void>;
|
|
234
235
|
/** Test seam; production reads the company-scoped hq-flags row. */
|
|
235
|
-
readManagerStatusGateFlag?: typeof readLanesRegistryFlag;
|
|
236
|
-
/** Test seam; production reads the company-scoped hq-flags row. */
|
|
237
236
|
readBlockerFreshnessFlag?: typeof readLanesRegistryFlag;
|
|
238
237
|
/** Test seam; production reads the company-scoped hq-flags row. */
|
|
239
238
|
readKrFreshnessFlag?: typeof readLanesRegistryFlag;
|
|
239
|
+
/** Test seam; production reads the company-scoped hq-flags row. */
|
|
240
|
+
readEnvelopeRepairContinueFlag?: typeof readLanesRegistryFlag;
|
|
240
241
|
}
|
|
241
242
|
/**
|
|
242
243
|
* Wrap one mesh call. A failed emit is logged as `mesh_event_failed` with
|