@indigoai-us/hq-cli 5.279.0 → 5.281.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +25 -0
- package/assets/scaffold/core/scripts/knowledge-prefs.sh +55 -0
- package/assets/scaffold/core/scripts/ontology-audience-grant.sh +81 -0
- package/assets/scaffold/core/scripts/ontology-brief.mjs +93 -0
- package/assets/scaffold/core/scripts/ontology-candidate.sh +100 -0
- package/assets/scaffold/core/scripts/ontology-garden.mjs +238 -0
- package/assets/scaffold/core/scripts/ontology-source-worklist.sh +79 -0
- package/assets/scaffold/core/scripts/source-yaml-validate.sh +36 -0
- package/dist/command-catalog.generated.d.ts +80 -2
- package/dist/command-catalog.generated.js +98 -2
- package/dist/commands/auth.js +40 -2
- package/dist/commands/core-ontology-native.d.ts +16 -0
- package/dist/commands/core-ontology-native.js +924 -0
- package/dist/commands/core-ontology.js +11 -0
- package/dist/commands/core.js +9 -0
- package/dist/commands/group-grants.js +3 -2
- package/dist/commands/lanes-shared.d.ts +5 -0
- package/dist/commands/lanes.d.ts +3 -1
- package/dist/commands/lanes.js +368 -31
- package/dist/lib/lanes/outpost-dispatch.d.ts +3 -1
- package/dist/lib/lanes/outpost-dispatch.js +4 -0
- package/dist/lib/lanes/store.js +11 -0
- package/dist/lib/lanes/types.d.ts +4 -0
- package/dist/lib/lanes/types.js +1 -1
- package/package.json +1 -1
package/dist/commands/lanes.js
CHANGED
|
@@ -36,7 +36,7 @@ import { createOutpostProfileRepinContext, parseOutpostProfileRepinContext, } fr
|
|
|
36
36
|
import { createOutpostRegistryWorkerApprovalContext, parseOutpostRegistryWorkerApprovalContext, } from "../lib/lanes/outpost-registry-worker-approval.js";
|
|
37
37
|
import { OUTPOST_DIR, OUTPOST_ID_PATH, readOutpostIdentity, resolveAdmissionHost, } from "../lib/lanes/admission-host.js";
|
|
38
38
|
export { OUTPOST_DIR, OUTPOST_ID_PATH, readOutpostIdentity } from "../lib/lanes/admission-host.js";
|
|
39
|
-
import { buildOutpostCapacityCommand, buildOutpostIdentityBackfillCommand, buildOutpostLaneCommand, buildOutpostQuestionListCommand, buildOutpostPickupCommand, createOutpostsClient, dispatchErrorMessage, dispatchOutpostPickup, buildOutpostLaneListCommand, outpostRejectedEffortOption, parseOutpostPickupPayload, stageOutpostPayload, } from "../lib/lanes/outpost-dispatch.js";
|
|
39
|
+
import { buildOutpostCapacityCommand, buildOutpostIdentityBackfillCommand, buildOutpostLaneCommand, buildOutpostQuestionListCommand, buildOutpostPickupCommand, createOutpostsClient, dispatchErrorMessage, dispatchOutpostPickup, buildOutpostLaneListCommand, outpostRejectedEffortOption, parseOutpostPickupPayload, PICKUP_WAIT_MS, stageOutpostPayload, } from "../lib/lanes/outpost-dispatch.js";
|
|
40
40
|
import { outpostHireSummaryCacheKey, readOutpostHireSummaryCache, writeOutpostHireSummaryCache, } from "../lib/lanes/outpost-hire-cache.js";
|
|
41
41
|
import { buildHostCapacitySnapshot, chooseHostForLane, parseHostCapacitySnapshot, } from "../lib/lanes/host-capacity.js";
|
|
42
42
|
import { BALANCE_MOVABLE_STATES, selectClaudeAccount, } from "../lib/lanes/claude-balance.js";
|
|
@@ -61,6 +61,7 @@ import { assertQuestionBodySecretSafe } from "../lib/lanes/questions.js";
|
|
|
61
61
|
import { compactionAfterUsage, } from "../lib/lanes/compaction.js";
|
|
62
62
|
import { parseCliEffort, resolveLaneEffort, } from "../lib/lanes/effort.js";
|
|
63
63
|
import { LanesError } from "../lib/lanes/errors.js";
|
|
64
|
+
import { readLatestEnvelopeRecord } from "../lib/lanes/watcher.js";
|
|
64
65
|
import { loadQuotaState, quotaBlockFor } from "../lib/lanes/quota.js";
|
|
65
66
|
import { SESSION_MOVE_REASON } from "../lib/lanes/move-account.js";
|
|
66
67
|
import { managerStatusGateEnabledForLane, resolveManagerStatusGateForLane, } from "../lib/lanes/manager-status-gate.js";
|
|
@@ -74,6 +75,8 @@ export { spawnLaneWorker };
|
|
|
74
75
|
export { questionBodyAbsPath };
|
|
75
76
|
export { MAX_LANE_DEPTH };
|
|
76
77
|
export const LANES_RUNS_REL = path.join("workspace", "lanes-runs");
|
|
78
|
+
export const LANE_REMOTE_MIRROR_SYNC_FLAG = "lanes.remote-done-mirror";
|
|
79
|
+
const REMOTE_MIRROR_REFRESH_BATCH_SIZE = 100;
|
|
77
80
|
/** The lane → mesh seam. Defined in lib/lanes/mesh-events.ts; the watcher shares it. */
|
|
78
81
|
export { emitLaneDecisionEvents };
|
|
79
82
|
function withoutProviderHome(value) {
|
|
@@ -159,6 +162,10 @@ async function defaultListOutpostIds() {
|
|
|
159
162
|
const rows = await defaultOutpostsClient().list();
|
|
160
163
|
return rows.map((row) => row.outpostId);
|
|
161
164
|
}
|
|
165
|
+
async function defaultListOutpostRoutes() {
|
|
166
|
+
const rows = await defaultOutpostsClient().list();
|
|
167
|
+
return rows.map(({ outpostId, instanceName }) => ({ outpostId, instanceName }));
|
|
168
|
+
}
|
|
162
169
|
async function defaultStageOutpostPayload(outpostId, bytes) {
|
|
163
170
|
return stageOutpostPayload(defaultOutpostsClient(), outpostId, bytes);
|
|
164
171
|
}
|
|
@@ -218,6 +225,9 @@ function resolveDeps(deps = {}) {
|
|
|
218
225
|
};
|
|
219
226
|
const bindProcessCwd = deps.hqRoot === undefined;
|
|
220
227
|
const resolveRoot = deps.hqRoot ?? (() => resolveLanesHqRoot({ env, cwd: cwd() }));
|
|
228
|
+
const listOutpostRoutes = deps.listOutpostRoutes ?? (deps.listOutpostIds
|
|
229
|
+
? async () => (await deps.listOutpostIds()).map((outpostId) => ({ outpostId }))
|
|
230
|
+
: defaultListOutpostRoutes);
|
|
221
231
|
const hqRoot = () => {
|
|
222
232
|
callerCwd();
|
|
223
233
|
const root = resolveRoot();
|
|
@@ -236,6 +246,7 @@ function resolveDeps(deps = {}) {
|
|
|
236
246
|
resolveCompany: deps.resolveCompany ?? defaultResolveCompany,
|
|
237
247
|
resolveActiveCompany: deps.resolveActiveCompany ?? defaultResolveActiveCompany,
|
|
238
248
|
listOutpostIds: deps.listOutpostIds ?? defaultListOutpostIds,
|
|
249
|
+
listOutpostRoutes,
|
|
239
250
|
readOutpostIdentity: deps.readOutpostIdentity ?? (() => readOutpostIdentity()),
|
|
240
251
|
spawnLogin: deps.spawnLogin ?? defaultSpawnLogin,
|
|
241
252
|
throttleDeps: deps.throttleDeps ?? defaultThrottleDeps(env),
|
|
@@ -611,6 +622,7 @@ async function fetchOwnerLaneHireFields(ctx, lane) {
|
|
|
611
622
|
}
|
|
612
623
|
const outpostId = lane.host.id;
|
|
613
624
|
const hqBinary = outpostCliOverride(ctx, outpostId);
|
|
625
|
+
const waitMs = outpostSingleLaneWaitMs(ctx.env);
|
|
614
626
|
let dispatched;
|
|
615
627
|
try {
|
|
616
628
|
dispatched = await ctx.dispatchOutpostExec({
|
|
@@ -620,12 +632,15 @@ async function fetchOwnerLaneHireFields(ctx, lane) {
|
|
|
620
632
|
laneId: lane.lane_id,
|
|
621
633
|
hqBinary,
|
|
622
634
|
}),
|
|
623
|
-
timeoutSeconds:
|
|
624
|
-
waitMs
|
|
635
|
+
timeoutSeconds: Math.ceil(waitMs / 1000),
|
|
636
|
+
waitMs,
|
|
625
637
|
});
|
|
626
638
|
}
|
|
627
639
|
catch (err) {
|
|
628
|
-
|
|
640
|
+
const message = dispatchErrorMessage(err);
|
|
641
|
+
throw new LanesError("outpost_lane_hire_summary_unavailable", /timed out waiting for outpost pickup/i.test(message)
|
|
642
|
+
? outpostPickupTimeoutMessage(outpostId, waitMs, `reading hire summary for lane ${lane.lane_id}`)
|
|
643
|
+
: `Could not read hire summary for lane ${lane.lane_id} from Outpost ${outpostId}: ${message}`, { lane_id: lane.lane_id, outpost_id: outpostId });
|
|
629
644
|
}
|
|
630
645
|
if (dispatched.done !== true || dispatched.status !== "Success" || dispatched.exitCode !== 0) {
|
|
631
646
|
throw new LanesError("outpost_lane_hire_summary_unavailable", `Outpost ${outpostId} did not confirm the hire summary for lane ${lane.lane_id}`, { lane_id: lane.lane_id, outpost_id: outpostId, command_id: dispatched.commandId });
|
|
@@ -702,7 +717,7 @@ async function dispatchOutpostLaneList(ctx, outpostId, query, timeoutSeconds, wa
|
|
|
702
717
|
timeoutSeconds,
|
|
703
718
|
waitMs,
|
|
704
719
|
});
|
|
705
|
-
if (!rejectedLaneListFlag(filtered, query))
|
|
720
|
+
if (query.strictSnapshot || !rejectedLaneListFlag(filtered, query))
|
|
706
721
|
return filtered;
|
|
707
722
|
return ctx.dispatchOutpostExec({
|
|
708
723
|
outpostId,
|
|
@@ -808,8 +823,6 @@ async function batchOwnerLaneHireFields(ctx, lanes) {
|
|
|
808
823
|
writeOutpostHireSummaryCache(ctx.hqRoot(), cacheUpdates, ctx.now().getTime(), cached);
|
|
809
824
|
return outcomes;
|
|
810
825
|
}
|
|
811
|
-
const OUTPOST_HIRE_SUMMARY_TIMEOUT_SECONDS = 20;
|
|
812
|
-
const OUTPOST_HIRE_SUMMARY_WAIT_MS = 15_000;
|
|
813
826
|
const OUTPOST_HIRE_SUMMARY_CONCURRENCY = 4;
|
|
814
827
|
async function laneHirePresentation(ctx, lane, lanes) {
|
|
815
828
|
const local = laneHireFields(lane, lanes);
|
|
@@ -864,6 +877,7 @@ async function withResumeHireAdmission(ctx, lane, fn, grantContext) {
|
|
|
864
877
|
!listLaneIds(hqRoot).includes(lane.senior.id)) {
|
|
865
878
|
throw new LanesError("hire_grant_context_required", `Resume of Outpost lane ${lane.lane_id} requires the primary senior's hire-grant context`, { lane_id: lane.lane_id, senior_lane_id: lane.senior.id });
|
|
866
879
|
}
|
|
880
|
+
await refreshRemoteLaneMirrors(ctx, hqRoot, loadAllLanes(hqRoot), new Set([lane.senior.id]));
|
|
867
881
|
return withHireGrantAdmission(hqRoot, lane.senior.id, (snapshot) => {
|
|
868
882
|
const current = readLane(hqRoot, lane.lane_id);
|
|
869
883
|
if (grantContext !== undefined &&
|
|
@@ -886,6 +900,28 @@ async function withResumeHireAdmission(ctx, lane, fn, grantContext) {
|
|
|
886
900
|
}
|
|
887
901
|
const CAPACITY_PROBE_TIMEOUT_MS = 15_000;
|
|
888
902
|
const MAX_PLACEMENT_OUTPOSTS = 16;
|
|
903
|
+
const OUTPOST_HIRE_SUMMARY_TIMEOUT_SECONDS = 20;
|
|
904
|
+
const OUTPOST_HIRE_SUMMARY_WAIT_MS = 15_000;
|
|
905
|
+
const OUTPOST_SINGLE_LANE_WAIT_ENV = "HQ_LANES_OUTPOST_SINGLE_LANE_WAIT_MS";
|
|
906
|
+
const OUTPOST_SINGLE_LANE_WAIT_MAX_MS = 600_000;
|
|
907
|
+
function outpostSingleLaneWaitMs(env) {
|
|
908
|
+
const raw = env[OUTPOST_SINGLE_LANE_WAIT_ENV]?.trim() || String(PICKUP_WAIT_MS);
|
|
909
|
+
const value = /^\d+$/.test(raw) ? Number(raw) : Number.NaN;
|
|
910
|
+
if (!Number.isSafeInteger(value) ||
|
|
911
|
+
value < CAPACITY_PROBE_TIMEOUT_MS ||
|
|
912
|
+
value > OUTPOST_SINGLE_LANE_WAIT_MAX_MS) {
|
|
913
|
+
throw new LanesError("outpost_wait_invalid", `${OUTPOST_SINGLE_LANE_WAIT_ENV} must be an integer from ${CAPACITY_PROBE_TIMEOUT_MS} to ${OUTPOST_SINGLE_LANE_WAIT_MAX_MS} milliseconds`, { env: OUTPOST_SINGLE_LANE_WAIT_ENV });
|
|
914
|
+
}
|
|
915
|
+
return value;
|
|
916
|
+
}
|
|
917
|
+
function outpostRouteMatchesHost(route, host) {
|
|
918
|
+
return typeof route.instanceName === "string" &&
|
|
919
|
+
route.instanceName.trim() !== "" &&
|
|
920
|
+
sanitizeHostComponent(route.instanceName.trim()) === sanitizeHostComponent(host.trim());
|
|
921
|
+
}
|
|
922
|
+
function outpostPickupTimeoutMessage(outpostId, waitMs, operation) {
|
|
923
|
+
return `Outpost ${outpostId} pickup timed out after ${waitMs}ms while ${operation}. Raise ${OUTPOST_SINGLE_LANE_WAIT_ENV} (milliseconds) and retry.`;
|
|
924
|
+
}
|
|
889
925
|
function outpostIdentityUnknownError(message, expectedId, details = {}) {
|
|
890
926
|
const expected = expectedId ?? "<expected-outpost-id>";
|
|
891
927
|
const command = `hq lanes outpost-identity backfill --id ${expected}`;
|
|
@@ -1022,17 +1058,24 @@ function normalizedLaneListQuery(query) {
|
|
|
1022
1058
|
...(query.active ? { active: true } : {}),
|
|
1023
1059
|
...(query.hierarchy === undefined ? {} : { hierarchy: query.hierarchy }),
|
|
1024
1060
|
...(query.snapshot ? { snapshot: true } : {}),
|
|
1061
|
+
...(query.strictSnapshot ? { strictSnapshot: true } : {}),
|
|
1025
1062
|
};
|
|
1026
1063
|
}
|
|
1027
1064
|
async function readOutpostLaneRecordsUncached(ctx, outpostId, query) {
|
|
1065
|
+
const targetedLookup = query.hierarchy !== undefined || query.laneIds !== undefined;
|
|
1066
|
+
const waitMs = targetedLookup ? outpostSingleLaneWaitMs(ctx.env) : CAPACITY_PROBE_TIMEOUT_MS;
|
|
1028
1067
|
let result;
|
|
1029
1068
|
try {
|
|
1030
|
-
result = await dispatchOutpostLaneList(ctx, outpostId, query,
|
|
1069
|
+
result = await dispatchOutpostLaneList(ctx, outpostId, query, Math.ceil(waitMs / 1000), waitMs);
|
|
1031
1070
|
}
|
|
1032
1071
|
catch (err) {
|
|
1072
|
+
const message = dispatchErrorMessage(err);
|
|
1073
|
+
if (/timed out waiting for outpost pickup/i.test(message)) {
|
|
1074
|
+
throw new LanesError("outpost_lane_list_unavailable", outpostPickupTimeoutMessage(outpostId, waitMs, "reading the requested lane"), { outpost_id: outpostId, wait_ms: waitMs, wait_env: OUTPOST_SINGLE_LANE_WAIT_ENV });
|
|
1075
|
+
}
|
|
1033
1076
|
if (isLanesError(err))
|
|
1034
1077
|
throw err;
|
|
1035
|
-
throw new LanesError("outpost_lane_list_unavailable", `Could not read lanes from Outpost ${outpostId}: ${
|
|
1078
|
+
throw new LanesError("outpost_lane_list_unavailable", `Could not read lanes from Outpost ${outpostId}: ${message}`, { outpost_id: outpostId });
|
|
1036
1079
|
}
|
|
1037
1080
|
if (result.done !== true || result.status !== "Success" || result.exitCode !== 0) {
|
|
1038
1081
|
throw new LanesError("outpost_lane_list_unavailable", `Outpost ${outpostId} did not return a successful lane list`, { outpost_id: outpostId, status: result.status, done: result.done ?? false, exit_code: result.exitCode });
|
|
@@ -1106,6 +1149,156 @@ async function readOutpostLaneRecords(ctx, outpostId, requestedQuery = {}) {
|
|
|
1106
1149
|
byOutpost.set(outpostId, { queryKey, promise });
|
|
1107
1150
|
return promise;
|
|
1108
1151
|
}
|
|
1152
|
+
/** Refresh only primary-created, non-terminal Outpost mirrors in one bounded owner query per Outpost. */
|
|
1153
|
+
async function refreshRemoteLaneMirrors(ctx, hqRoot, records, parentLaneIds) {
|
|
1154
|
+
const result = {
|
|
1155
|
+
updated: new Map(),
|
|
1156
|
+
unverifiedRemote: new Map(),
|
|
1157
|
+
};
|
|
1158
|
+
const host = reconcileHostFromCtx(ctx);
|
|
1159
|
+
if (host.outpostId !== null || parentLaneIds?.size === 0)
|
|
1160
|
+
return result;
|
|
1161
|
+
let primaryIdentity;
|
|
1162
|
+
const originMatchesThisPrimary = (lane) => {
|
|
1163
|
+
if (lane.origin_host_source !== "imds") {
|
|
1164
|
+
return lane.origin_host === host.hostname;
|
|
1165
|
+
}
|
|
1166
|
+
primaryIdentity ??= ctx.resolveMachineIdentity();
|
|
1167
|
+
if (primaryIdentity.status !== "ok")
|
|
1168
|
+
return false;
|
|
1169
|
+
const primaryImdsId = primaryIdentity.imds ??
|
|
1170
|
+
(primaryIdentity.source === "imds" ? primaryIdentity.id : null);
|
|
1171
|
+
return primaryImdsId !== null && primaryImdsId === lane.origin_host;
|
|
1172
|
+
};
|
|
1173
|
+
const candidates = records
|
|
1174
|
+
.filter((lane) => lane.host.kind === "outpost" &&
|
|
1175
|
+
originMatchesThisPrimary(lane) &&
|
|
1176
|
+
(parentLaneIds === undefined || (lane.senior.kind === "lane" &&
|
|
1177
|
+
lane.senior.id !== undefined &&
|
|
1178
|
+
parentLaneIds.has(lane.senior.id))) &&
|
|
1179
|
+
!isTerminalLaneState(lane.state))
|
|
1180
|
+
.sort((a, b) => a.lane_id.localeCompare(b.lane_id));
|
|
1181
|
+
if (candidates.length === 0)
|
|
1182
|
+
return result;
|
|
1183
|
+
const enabledByCompany = new Map();
|
|
1184
|
+
const enabledCandidates = [];
|
|
1185
|
+
for (const lane of candidates) {
|
|
1186
|
+
const companyKey = JSON.stringify([lane.company?.uid ?? null, lane.company?.slug ?? null]);
|
|
1187
|
+
let enabled = enabledByCompany.get(companyKey);
|
|
1188
|
+
if (enabled === undefined) {
|
|
1189
|
+
enabled = await readLanesRegistryFlag(LANE_REMOTE_MIRROR_SYNC_FLAG, {
|
|
1190
|
+
company: lane.company,
|
|
1191
|
+
env: ctx.env,
|
|
1192
|
+
});
|
|
1193
|
+
enabledByCompany.set(companyKey, enabled);
|
|
1194
|
+
}
|
|
1195
|
+
if (enabled)
|
|
1196
|
+
enabledCandidates.push(lane);
|
|
1197
|
+
}
|
|
1198
|
+
if (enabledCandidates.length === 0)
|
|
1199
|
+
return result;
|
|
1200
|
+
const byOutpost = new Map();
|
|
1201
|
+
for (const lane of enabledCandidates) {
|
|
1202
|
+
const group = byOutpost.get(lane.host.id) ?? [];
|
|
1203
|
+
group.push(lane);
|
|
1204
|
+
byOutpost.set(lane.host.id, group);
|
|
1205
|
+
}
|
|
1206
|
+
for (const [outpostId, group] of byOutpost) {
|
|
1207
|
+
const batch = group.slice(0, REMOTE_MIRROR_REFRESH_BATCH_SIZE);
|
|
1208
|
+
const overflow = group.slice(REMOTE_MIRROR_REFRESH_BATCH_SIZE);
|
|
1209
|
+
if (overflow.length > 0) {
|
|
1210
|
+
const ids = overflow.map((lane) => lane.lane_id);
|
|
1211
|
+
for (const lane of overflow) {
|
|
1212
|
+
result.unverifiedRemote.set(lane.lane_id, "refresh_batch_limit");
|
|
1213
|
+
}
|
|
1214
|
+
process.stderr.write(`hq lanes: unverified remote mirrors from Outpost ${outpostId} remain counted; refresh batch limit left lanes ${ids.join(", ")} unread\n`);
|
|
1215
|
+
}
|
|
1216
|
+
let ownerRecords;
|
|
1217
|
+
try {
|
|
1218
|
+
ownerRecords = await readOutpostLaneRecords(ctx, outpostId, {
|
|
1219
|
+
laneIds: batch.map((lane) => lane.lane_id),
|
|
1220
|
+
snapshot: true,
|
|
1221
|
+
strictSnapshot: true,
|
|
1222
|
+
});
|
|
1223
|
+
}
|
|
1224
|
+
catch (error) {
|
|
1225
|
+
const detail = isLanesError(error)
|
|
1226
|
+
? `${error.error}: ${error.message}`
|
|
1227
|
+
: dispatchErrorMessage(error);
|
|
1228
|
+
for (const lane of batch)
|
|
1229
|
+
result.unverifiedRemote.set(lane.lane_id, detail);
|
|
1230
|
+
process.stderr.write(`hq lanes: unverified remote mirrors from Outpost ${outpostId} remain counted: ${detail}; lanes ${batch.map((lane) => lane.lane_id).join(", ")}\n`);
|
|
1231
|
+
continue;
|
|
1232
|
+
}
|
|
1233
|
+
for (const mirror of batch) {
|
|
1234
|
+
const matches = ownerRecords.filter((lane) => lane.lane_id === mirror.lane_id);
|
|
1235
|
+
const owner = matches[0];
|
|
1236
|
+
const ownerIsHostedByOutpost = owner !== undefined && ((owner.host.kind === "outpost" && owner.host.id === outpostId) ||
|
|
1237
|
+
(owner.host.kind === "local" && owner.host.id.trim() !== ""));
|
|
1238
|
+
if (matches.length !== 1 ||
|
|
1239
|
+
owner === undefined ||
|
|
1240
|
+
!ownerIsHostedByOutpost ||
|
|
1241
|
+
(owner.company?.uid ?? null) !== (mirror.company?.uid ?? null) ||
|
|
1242
|
+
(owner.company?.slug ?? null) !== (mirror.company?.slug ?? null)) {
|
|
1243
|
+
const detail = matches.length === 0
|
|
1244
|
+
? "owner did not return the requested lane"
|
|
1245
|
+
: matches.length > 1
|
|
1246
|
+
? "owner returned duplicate lane rows"
|
|
1247
|
+
: "owner lane identity did not match its mirror";
|
|
1248
|
+
result.unverifiedRemote.set(mirror.lane_id, detail);
|
|
1249
|
+
process.stderr.write(`hq lanes: unverified remote mirror ${mirror.lane_id} from Outpost ${outpostId} remains counted: ${detail}\n`);
|
|
1250
|
+
continue;
|
|
1251
|
+
}
|
|
1252
|
+
const endedAt = isTerminalLaneState(owner.state)
|
|
1253
|
+
? owner.last_envelope?.at ?? owner.updated_at
|
|
1254
|
+
: undefined;
|
|
1255
|
+
const envelopeSummary = isTerminalLaneState(owner.state)
|
|
1256
|
+
? owner.envelope_summary
|
|
1257
|
+
: undefined;
|
|
1258
|
+
const before = readLane(hqRoot, mirror.lane_id);
|
|
1259
|
+
if (before.host.kind !== "outpost" || before.host.id !== outpostId ||
|
|
1260
|
+
!originMatchesThisPrimary(before) || isTerminalLaneState(before.state)) {
|
|
1261
|
+
result.updated.set(mirror.lane_id, before);
|
|
1262
|
+
continue;
|
|
1263
|
+
}
|
|
1264
|
+
const needsUpdate = before.state !== owner.state ||
|
|
1265
|
+
before.queued_reason !== (owner.state === "queued" ? owner.queued_reason : undefined) ||
|
|
1266
|
+
before.ended_at !== endedAt ||
|
|
1267
|
+
before.envelope_summary !== envelopeSummary ||
|
|
1268
|
+
(isTerminalLaneState(owner.state) && (JSON.stringify(before.last_envelope) !== JSON.stringify(owner.last_envelope) ||
|
|
1269
|
+
before.story_complete !== owner.story_complete ||
|
|
1270
|
+
before.story_complete_reason !== owner.story_complete_reason ||
|
|
1271
|
+
before.story_status !== owner.story_status ||
|
|
1272
|
+
before.terminal_reason !== owner.terminal_reason));
|
|
1273
|
+
if (!needsUpdate) {
|
|
1274
|
+
result.updated.set(mirror.lane_id, before);
|
|
1275
|
+
continue;
|
|
1276
|
+
}
|
|
1277
|
+
const updated = retryUpdateLane(hqRoot, mirror.lane_id, "reconcile", (current) => {
|
|
1278
|
+
if (current.host.kind !== "outpost" || current.host.id !== outpostId ||
|
|
1279
|
+
!originMatchesThisPrimary(current) || isTerminalLaneState(current.state))
|
|
1280
|
+
return current;
|
|
1281
|
+
return {
|
|
1282
|
+
...current,
|
|
1283
|
+
state: owner.state,
|
|
1284
|
+
queued_reason: owner.state === "queued" ? owner.queued_reason : undefined,
|
|
1285
|
+
ended_at: endedAt,
|
|
1286
|
+
envelope_summary: envelopeSummary,
|
|
1287
|
+
...(isTerminalLaneState(owner.state) ? {
|
|
1288
|
+
last_envelope: owner.last_envelope,
|
|
1289
|
+
story_complete: owner.story_complete,
|
|
1290
|
+
story_complete_reason: owner.story_complete_reason,
|
|
1291
|
+
story_status: owner.story_status,
|
|
1292
|
+
terminal_reason: owner.terminal_reason,
|
|
1293
|
+
terminal_details: owner.terminal_details,
|
|
1294
|
+
} : {}),
|
|
1295
|
+
};
|
|
1296
|
+
});
|
|
1297
|
+
result.updated.set(mirror.lane_id, updated);
|
|
1298
|
+
}
|
|
1299
|
+
}
|
|
1300
|
+
return result;
|
|
1301
|
+
}
|
|
1109
1302
|
async function resolveCommandLaneTarget(ctx, laneId, options = {}) {
|
|
1110
1303
|
try {
|
|
1111
1304
|
return { lane: readLane(ctx.hqRoot(), laneId), remoteOnly: false };
|
|
@@ -1119,17 +1312,34 @@ async function resolveCommandLaneTarget(ctx, laneId, options = {}) {
|
|
|
1119
1312
|
if (!hostSuffix || sanitizeHostComponent(hostSuffix) !== hostSuffix) {
|
|
1120
1313
|
throw commandLaneNotFound(laneId, options.outpostHint === true);
|
|
1121
1314
|
}
|
|
1122
|
-
let
|
|
1315
|
+
let rawOutpostRoutes;
|
|
1123
1316
|
try {
|
|
1124
|
-
|
|
1317
|
+
rawOutpostRoutes = await ctx.listOutpostRoutes();
|
|
1125
1318
|
}
|
|
1126
1319
|
catch (err) {
|
|
1127
1320
|
throw new LanesError("outpost_list_unavailable", `Could not resolve registered Outposts for lane ${laneId}: ${dispatchErrorMessage(err)}`, { lane_id: laneId });
|
|
1128
1321
|
}
|
|
1129
|
-
if (!Array.isArray(
|
|
1322
|
+
if (!Array.isArray(rawOutpostRoutes)) {
|
|
1130
1323
|
throw new LanesError("outpost_list_invalid", "Registered Outpost list is not an array", { lane_id: laneId });
|
|
1131
1324
|
}
|
|
1132
|
-
const
|
|
1325
|
+
const outpostRoutes = rawOutpostRoutes.map((raw, index) => {
|
|
1326
|
+
if (raw === null || typeof raw !== "object" || Array.isArray(raw) ||
|
|
1327
|
+
typeof raw.outpostId !== "string" ||
|
|
1328
|
+
!raw.outpostId ||
|
|
1329
|
+
(raw.instanceName !== undefined &&
|
|
1330
|
+
typeof raw.instanceName !== "string")) {
|
|
1331
|
+
throw new LanesError("outpost_list_invalid", `Registered Outpost row ${index} is malformed`, {
|
|
1332
|
+
lane_id: laneId,
|
|
1333
|
+
index,
|
|
1334
|
+
});
|
|
1335
|
+
}
|
|
1336
|
+
const row = raw;
|
|
1337
|
+
return {
|
|
1338
|
+
outpostId: row.outpostId.trim(),
|
|
1339
|
+
...(typeof row.instanceName === "string" ? { instanceName: row.instanceName.trim() } : {}),
|
|
1340
|
+
};
|
|
1341
|
+
});
|
|
1342
|
+
const outpostIds = outpostRoutes.map(({ outpostId }) => outpostId);
|
|
1133
1343
|
if (outpostIds.length > MAX_PLACEMENT_OUTPOSTS ||
|
|
1134
1344
|
outpostIds.some((id) => id === "") ||
|
|
1135
1345
|
new Set(outpostIds).size !== outpostIds.length) {
|
|
@@ -1142,9 +1352,23 @@ async function resolveCommandLaneTarget(ctx, laneId, options = {}) {
|
|
|
1142
1352
|
if (requestedOutpostId !== undefined && !outpostIds.includes(requestedOutpostId)) {
|
|
1143
1353
|
throw new LanesError("outpost_not_registered", `Outpost ${requestedOutpostId} is not registered`, { lane_id: laneId, outpost_id: requestedOutpostId });
|
|
1144
1354
|
}
|
|
1355
|
+
const hostRoutes = outpostRoutes.filter((route) => outpostRouteMatchesHost(route, hostSuffix));
|
|
1356
|
+
const identity = ctx.readOutpostIdentity();
|
|
1357
|
+
const localHostRoutes = outpostRoutes.filter((route) => outpostRouteMatchesHost(route, ctx.hostname()));
|
|
1358
|
+
if (localHostRoutes.length > 1 || (identity.kind === "id" && localHostRoutes.some((route) => route.outpostId !== identity.id))) {
|
|
1359
|
+
throw new LanesError("outpost_host_ambiguous", "Registered Outpost host identity conflicts with this host's identity", { lane_id: laneId, local_outpost_id: identity.kind === "id" ? identity.id : undefined });
|
|
1360
|
+
}
|
|
1361
|
+
const localOutpostId = identity.kind === "id" ? identity.id : localHostRoutes[0]?.outpostId;
|
|
1362
|
+
const hostOutpostIds = hostRoutes.map(({ outpostId }) => outpostId);
|
|
1363
|
+
const hostOutpostIdSet = new Set(hostOutpostIds);
|
|
1145
1364
|
const matches = [];
|
|
1146
1365
|
const failedOutpostIds = [];
|
|
1147
|
-
const lookupOutpostIds = requestedOutpostId
|
|
1366
|
+
const lookupOutpostIds = requestedOutpostId !== undefined
|
|
1367
|
+
? [requestedOutpostId]
|
|
1368
|
+
: [
|
|
1369
|
+
...hostOutpostIds.filter((id) => id !== localOutpostId),
|
|
1370
|
+
...outpostIds.filter((id) => id !== localOutpostId && !hostOutpostIdSet.has(id)),
|
|
1371
|
+
];
|
|
1148
1372
|
for (const outpostId of lookupOutpostIds) {
|
|
1149
1373
|
let ownerLanes;
|
|
1150
1374
|
try {
|
|
@@ -1156,6 +1380,8 @@ async function resolveCommandLaneTarget(ctx, laneId, options = {}) {
|
|
|
1156
1380
|
catch (err) {
|
|
1157
1381
|
if (requestedOutpostId !== undefined)
|
|
1158
1382
|
throw err;
|
|
1383
|
+
if (isLanesError(err) && err.error === "outpost_wait_invalid")
|
|
1384
|
+
throw err;
|
|
1159
1385
|
failedOutpostIds.push(outpostId);
|
|
1160
1386
|
continue;
|
|
1161
1387
|
}
|
|
@@ -2964,6 +3190,9 @@ async function runCreate(ctx, opts) {
|
|
|
2964
3190
|
: {}),
|
|
2965
3191
|
attribution_history: [],
|
|
2966
3192
|
};
|
|
3193
|
+
if (senior.kind === "lane" && senior.id) {
|
|
3194
|
+
await refreshRemoteLaneMirrors(ctx, hqRoot, loadAllLanes(hqRoot), new Set([senior.id]));
|
|
3195
|
+
}
|
|
2967
3196
|
const lane = senior.kind === "lane"
|
|
2968
3197
|
? withHireGrantAdmission(hqRoot, senior.id, ({ parent, children, parentGrant, availableChildGrant }) => {
|
|
2969
3198
|
if (isTerminalLaneState(parent.state)) {
|
|
@@ -3547,6 +3776,7 @@ async function dispatchOutpostLaneCommand(ctx, lane, action, options, behavior =
|
|
|
3547
3776
|
throw new LanesError("outpost_dispatch_failed", `Lane ${lane.lane_id} has no Outpost host to dispatch to`, { lane_id: lane.lane_id });
|
|
3548
3777
|
}
|
|
3549
3778
|
const outpostId = lane.host.id;
|
|
3779
|
+
const waitMs = outpostSingleLaneWaitMs(ctx.env);
|
|
3550
3780
|
const profileRepinContext = action === "resume" &&
|
|
3551
3781
|
options.worker !== undefined &&
|
|
3552
3782
|
callerMatchesSenior(ctx, lane.senior)
|
|
@@ -3583,6 +3813,8 @@ async function dispatchOutpostLaneCommand(ctx, lane, action, options, behavior =
|
|
|
3583
3813
|
try {
|
|
3584
3814
|
const dispatchCommand = (contexts) => ctx.dispatchOutpostExec({
|
|
3585
3815
|
outpostId,
|
|
3816
|
+
timeoutSeconds: Math.ceil(waitMs / 1000),
|
|
3817
|
+
waitMs,
|
|
3586
3818
|
command: buildOutpostLaneCommand({
|
|
3587
3819
|
action: action === "grant"
|
|
3588
3820
|
? "grant-target"
|
|
@@ -3636,7 +3868,10 @@ async function dispatchOutpostLaneCommand(ctx, lane, action, options, behavior =
|
|
|
3636
3868
|
}
|
|
3637
3869
|
}
|
|
3638
3870
|
catch (err) {
|
|
3639
|
-
|
|
3871
|
+
const message = dispatchErrorMessage(err);
|
|
3872
|
+
throw new LanesError("outpost_dispatch_failed", /timed out waiting for outpost pickup/i.test(message)
|
|
3873
|
+
? outpostPickupTimeoutMessage(outpostId, waitMs, `dispatching ${action} for lane ${lane.lane_id}`)
|
|
3874
|
+
: `Could not dispatch ${action} for lane ${lane.lane_id} to Outpost ${outpostId}: ${message}`, { lane_id: lane.lane_id, outpost_id: outpostId, wait_ms: waitMs, wait_env: OUTPOST_SINGLE_LANE_WAIT_ENV });
|
|
3640
3875
|
}
|
|
3641
3876
|
if (grantContextRejected) {
|
|
3642
3877
|
throw new LanesError("outpost_cli_upgrade_required", `Outpost ${outpostId} needs an hq-cli release containing this fix before it can resume lane ${lane.lane_id} under its primary-hosted senior`, {
|
|
@@ -3834,6 +4069,7 @@ function parseReparentDescendantIds(value, laneId, outpostId) {
|
|
|
3834
4069
|
return ids;
|
|
3835
4070
|
}
|
|
3836
4071
|
async function readOutpostLaneSnapshot(ctx, outpostId, laneId) {
|
|
4072
|
+
const waitMs = outpostSingleLaneWaitMs(ctx.env);
|
|
3837
4073
|
let dispatched;
|
|
3838
4074
|
try {
|
|
3839
4075
|
dispatched = await ctx.dispatchOutpostExec({
|
|
@@ -3843,12 +4079,15 @@ async function readOutpostLaneSnapshot(ctx, outpostId, laneId) {
|
|
|
3843
4079
|
laneId,
|
|
3844
4080
|
hqBinary: outpostCliOverride(ctx, outpostId),
|
|
3845
4081
|
}),
|
|
3846
|
-
timeoutSeconds:
|
|
3847
|
-
waitMs
|
|
4082
|
+
timeoutSeconds: Math.ceil(waitMs / 1000),
|
|
4083
|
+
waitMs,
|
|
3848
4084
|
});
|
|
3849
4085
|
}
|
|
3850
4086
|
catch (err) {
|
|
3851
|
-
|
|
4087
|
+
const message = dispatchErrorMessage(err);
|
|
4088
|
+
throw new LanesError("outpost_lane_sync_unavailable", /timed out waiting for outpost pickup/i.test(message)
|
|
4089
|
+
? outpostPickupTimeoutMessage(outpostId, waitMs, `reading lane ${laneId}`)
|
|
4090
|
+
: `Could not refresh descendant lane ${laneId} from Outpost ${outpostId}: ${message}`, { lane_id: laneId, outpost_id: outpostId });
|
|
3852
4091
|
}
|
|
3853
4092
|
if (dispatched.done !== true || dispatched.status !== "Success" || dispatched.exitCode !== 0) {
|
|
3854
4093
|
throw new LanesError("outpost_lane_sync_unavailable", `Outpost ${outpostId} did not confirm the snapshot for descendant lane ${laneId}`, { lane_id: laneId, outpost_id: outpostId, command_id: dispatched.commandId });
|
|
@@ -4147,7 +4386,7 @@ async function runList(ctx, opts) {
|
|
|
4147
4386
|
const hierarchyIds = opts.hierarchy === undefined
|
|
4148
4387
|
? undefined
|
|
4149
4388
|
: new Set(selectOutpostHireGrantHierarchy(opts.hierarchy, laneSource).map((lane) => lane.lane_id));
|
|
4150
|
-
|
|
4389
|
+
let selectedLanes = laneSource
|
|
4151
4390
|
.filter((lane) => {
|
|
4152
4391
|
if (requestedLaneIds.length > 0 && !requestedIds.has(lane.lane_id))
|
|
4153
4392
|
return false;
|
|
@@ -4173,9 +4412,27 @@ async function runList(ctx, opts) {
|
|
|
4173
4412
|
})
|
|
4174
4413
|
.sort(compareLaneQueueOrder);
|
|
4175
4414
|
if (opts.snapshot) {
|
|
4176
|
-
printJson({
|
|
4415
|
+
printJson({
|
|
4416
|
+
ok: true,
|
|
4417
|
+
action: "list",
|
|
4418
|
+
lanes: selectedLanes.map((lane) => {
|
|
4419
|
+
if (!isTerminalLaneState(lane.state))
|
|
4420
|
+
return lane;
|
|
4421
|
+
const envelope = readLatestEnvelopeRecord(hqRoot, lane.lane_id)?.envelope;
|
|
4422
|
+
return envelope?.summary ? { ...lane, envelope_summary: envelope.summary } : lane;
|
|
4423
|
+
}),
|
|
4424
|
+
});
|
|
4177
4425
|
return;
|
|
4178
4426
|
}
|
|
4427
|
+
const remoteRefresh = await refreshRemoteLaneMirrors(ctx, hqRoot, laneSource, new Set(selectedLanes
|
|
4428
|
+
.filter((lane) => !isTerminalLaneState(lane.state))
|
|
4429
|
+
.map((lane) => lane.lane_id)));
|
|
4430
|
+
for (const [laneId, updated] of remoteRefresh.updated) {
|
|
4431
|
+
const index = laneSource.findIndex((lane) => lane.lane_id === laneId);
|
|
4432
|
+
if (index >= 0)
|
|
4433
|
+
laneSource[index] = updated;
|
|
4434
|
+
}
|
|
4435
|
+
selectedLanes = selectedLanes.map((lane) => remoteRefresh.updated.get(lane.lane_id) ?? lane);
|
|
4179
4436
|
const now = ctx.now();
|
|
4180
4437
|
const host = reconcileHostFromCtx(ctx);
|
|
4181
4438
|
const allLanes = laneSource;
|
|
@@ -4232,6 +4489,7 @@ async function runList(ctx, opts) {
|
|
|
4232
4489
|
return {
|
|
4233
4490
|
...lane,
|
|
4234
4491
|
...hireByLane.get(lane.lane_id),
|
|
4492
|
+
...(remoteRefresh.unverifiedRemote.has(lane.lane_id) ? { unverified_remote: true } : {}),
|
|
4235
4493
|
stories: laneStories(lane),
|
|
4236
4494
|
story_ids: laneStories(lane).map((story) => story.story_id),
|
|
4237
4495
|
no_story_reason: lane.no_story_reason ?? null,
|
|
@@ -4268,6 +4526,9 @@ async function runList(ctx, opts) {
|
|
|
4268
4526
|
ok: true,
|
|
4269
4527
|
action: "list",
|
|
4270
4528
|
lanes,
|
|
4529
|
+
...(remoteRefresh.unverifiedRemote.size > 0
|
|
4530
|
+
? { unverified_remote_lanes: [...remoteRefresh.unverifiedRemote.keys()] }
|
|
4531
|
+
: {}),
|
|
4271
4532
|
...(host_capacity !== undefined ? { host_capacity } : {}),
|
|
4272
4533
|
};
|
|
4273
4534
|
if (opts.human)
|
|
@@ -4687,6 +4948,13 @@ async function applyHireGrant(ctx, laneId, hireGrant, authorization, grantContex
|
|
|
4687
4948
|
? [initial.senior.id]
|
|
4688
4949
|
: []),
|
|
4689
4950
|
];
|
|
4951
|
+
if (authorization === "senior") {
|
|
4952
|
+
const parentLaneIds = new Set([initial.lane_id]);
|
|
4953
|
+
if (initial.senior.kind === "lane" && initial.senior.id) {
|
|
4954
|
+
parentLaneIds.add(initial.senior.id);
|
|
4955
|
+
}
|
|
4956
|
+
await refreshRemoteLaneMirrors(ctx, hqRoot, loadAllLanes(hqRoot), parentLaneIds);
|
|
4957
|
+
}
|
|
4690
4958
|
const updated = ctx.withGrantHireLocks(hqRoot, lockedLaneIds, () => {
|
|
4691
4959
|
const current = readLane(hqRoot, laneId);
|
|
4692
4960
|
const currentCallerAuthorization = authorization === "senior"
|
|
@@ -6140,6 +6408,12 @@ async function reparentOnOwner(ctx, laneId, opts) {
|
|
|
6140
6408
|
const nextSenior = parseSeniorRef(opts.senior);
|
|
6141
6409
|
assertSeniorExists(hqRoot, nextSenior, ctx);
|
|
6142
6410
|
const initial = readLane(hqRoot, laneId);
|
|
6411
|
+
const refreshParents = new Set();
|
|
6412
|
+
if (initial.senior.kind === "lane" && initial.senior.id)
|
|
6413
|
+
refreshParents.add(initial.senior.id);
|
|
6414
|
+
if (nextSenior.kind === "lane" && nextSenior.id)
|
|
6415
|
+
refreshParents.add(nextSenior.id);
|
|
6416
|
+
await refreshRemoteLaneMirrors(ctx, hqRoot, loadAllLanes(hqRoot), refreshParents);
|
|
6143
6417
|
const lockIds = [
|
|
6144
6418
|
laneId,
|
|
6145
6419
|
...(initial.senior.kind === "lane" && initial.senior.id ? [initial.senior.id] : []),
|
|
@@ -6299,12 +6573,58 @@ const CANONICAL_LOG_STREAMS = [
|
|
|
6299
6573
|
];
|
|
6300
6574
|
async function runLogs(ctx, laneId, opts) {
|
|
6301
6575
|
const hqRoot = ctx.hqRoot();
|
|
6302
|
-
readLane(hqRoot, laneId);
|
|
6303
|
-
const logsDir = path.join(runDir(hqRoot, laneId), "logs");
|
|
6576
|
+
const lane = readLane(hqRoot, laneId);
|
|
6304
6577
|
const tail = parseOptionalInt(opts.tail, "--tail");
|
|
6305
6578
|
if (tail === 0) {
|
|
6306
6579
|
throw new LanesError("invalid_integer", "--tail must be a positive integer", { flag: "--tail", value: opts.tail });
|
|
6307
6580
|
}
|
|
6581
|
+
if (lane.host.kind === "outpost" &&
|
|
6582
|
+
!laneRunsHere(lane, reconcileHostFromCtx(ctx)) &&
|
|
6583
|
+
await readLanesRegistryFlag(LANE_REMOTE_MIRROR_SYNC_FLAG, {
|
|
6584
|
+
company: lane.company,
|
|
6585
|
+
env: ctx.env,
|
|
6586
|
+
})) {
|
|
6587
|
+
const outpostId = lane.host.id;
|
|
6588
|
+
const waitMs = outpostSingleLaneWaitMs(ctx.env);
|
|
6589
|
+
let dispatched;
|
|
6590
|
+
try {
|
|
6591
|
+
dispatched = await ctx.dispatchOutpostExec({
|
|
6592
|
+
outpostId,
|
|
6593
|
+
command: buildOutpostLaneCommand({
|
|
6594
|
+
action: "logs",
|
|
6595
|
+
laneId,
|
|
6596
|
+
hqBinary: outpostCliOverride(ctx, outpostId),
|
|
6597
|
+
...(tail === undefined ? {} : { tail: String(tail) }),
|
|
6598
|
+
json: true,
|
|
6599
|
+
}),
|
|
6600
|
+
timeoutSeconds: Math.ceil(waitMs / 1000),
|
|
6601
|
+
waitMs,
|
|
6602
|
+
});
|
|
6603
|
+
}
|
|
6604
|
+
catch (error) {
|
|
6605
|
+
throw new LanesError("outpost_lane_logs_unavailable", `Could not read logs for lane ${laneId} from Outpost ${outpostId}: ${dispatchErrorMessage(error)}`, { lane_id: laneId, outpost_id: outpostId, wait_ms: waitMs, wait_env: OUTPOST_SINGLE_LANE_WAIT_ENV });
|
|
6606
|
+
}
|
|
6607
|
+
if (dispatched.done !== true || dispatched.status !== "Success" || dispatched.exitCode !== 0) {
|
|
6608
|
+
throw new LanesError("outpost_lane_logs_unavailable", `Outpost ${outpostId} did not return logs for lane ${laneId}`, { lane_id: laneId, outpost_id: outpostId, status: dispatched.status, exit_code: dispatched.exitCode });
|
|
6609
|
+
}
|
|
6610
|
+
let body;
|
|
6611
|
+
try {
|
|
6612
|
+
body = JSON.parse(dispatched.stdout);
|
|
6613
|
+
}
|
|
6614
|
+
catch (error) {
|
|
6615
|
+
throw new LanesError("outpost_lane_logs_invalid", `Outpost ${outpostId} returned invalid logs JSON for lane ${laneId}: ${dispatchErrorMessage(error)}`, { lane_id: laneId, outpost_id: outpostId });
|
|
6616
|
+
}
|
|
6617
|
+
if (body === null || typeof body !== "object" || Array.isArray(body) ||
|
|
6618
|
+
body.ok !== true ||
|
|
6619
|
+
body.action !== "logs" ||
|
|
6620
|
+
body.lane_id !== laneId ||
|
|
6621
|
+
!Array.isArray(body.streams)) {
|
|
6622
|
+
throw new LanesError("outpost_lane_logs_invalid", `Outpost ${outpostId} returned a malformed logs payload for lane ${laneId}`, { lane_id: laneId, outpost_id: outpostId });
|
|
6623
|
+
}
|
|
6624
|
+
printJson(body);
|
|
6625
|
+
return;
|
|
6626
|
+
}
|
|
6627
|
+
const logsDir = path.join(runDir(hqRoot, laneId), "logs");
|
|
6308
6628
|
const names = new Set(CANONICAL_LOG_STREAMS);
|
|
6309
6629
|
if (fs.existsSync(logsDir)) {
|
|
6310
6630
|
for (const name of fs.readdirSync(logsDir)) {
|
|
@@ -6373,22 +6693,27 @@ function parseRemoteCancelledQuestionIds(value, laneId, outpostId, commandId) {
|
|
|
6373
6693
|
}
|
|
6374
6694
|
return value;
|
|
6375
6695
|
}
|
|
6376
|
-
async function readOutpostTerminalLane(ctx, laneId, outpostId, stopCommandId) {
|
|
6696
|
+
async function readOutpostTerminalLane(ctx, laneId, outpostId, stopCommandId, waitMs) {
|
|
6377
6697
|
let dispatched;
|
|
6378
6698
|
try {
|
|
6379
6699
|
dispatched = await ctx.dispatchOutpostExec({
|
|
6380
6700
|
outpostId,
|
|
6381
6701
|
command: buildOutpostLaneCommand({ action: "show", laneId }),
|
|
6382
|
-
timeoutSeconds:
|
|
6383
|
-
waitMs
|
|
6702
|
+
timeoutSeconds: Math.ceil(waitMs / 1000),
|
|
6703
|
+
waitMs,
|
|
6384
6704
|
});
|
|
6385
6705
|
}
|
|
6386
6706
|
catch (err) {
|
|
6387
|
-
|
|
6707
|
+
const message = dispatchErrorMessage(err);
|
|
6708
|
+
throw new LanesError("outpost_delivery_unconfirmed", /timed out waiting for outpost pickup/i.test(message)
|
|
6709
|
+
? outpostPickupTimeoutMessage(outpostId, waitMs, `confirming terminal lane ${laneId}`)
|
|
6710
|
+
: `Could not read terminal lane ${laneId} from Outpost ${outpostId}: ${message}`, {
|
|
6388
6711
|
lane_id: laneId,
|
|
6389
6712
|
outpost_id: outpostId,
|
|
6390
6713
|
command_id: stopCommandId,
|
|
6391
6714
|
sync_command_id: null,
|
|
6715
|
+
wait_ms: waitMs,
|
|
6716
|
+
wait_env: OUTPOST_SINGLE_LANE_WAIT_ENV,
|
|
6392
6717
|
});
|
|
6393
6718
|
}
|
|
6394
6719
|
const commandId = dispatched.commandId?.trim();
|
|
@@ -6457,17 +6782,21 @@ async function dispatchOutpostLaneStop(ctx, lane, reason, quiet = false) {
|
|
|
6457
6782
|
throw new LanesError("outpost_dispatch_failed", `Lane ${lane.lane_id} has no Outpost host to dispatch stop to`, { lane_id: lane.lane_id });
|
|
6458
6783
|
}
|
|
6459
6784
|
const outpostId = lane.host.id;
|
|
6785
|
+
const waitMs = outpostSingleLaneWaitMs(ctx.env);
|
|
6460
6786
|
let dispatched;
|
|
6461
6787
|
try {
|
|
6462
6788
|
dispatched = await ctx.dispatchOutpostExec({
|
|
6463
6789
|
outpostId,
|
|
6464
6790
|
command: buildOutpostLaneCommand({ action: "stop", laneId: lane.lane_id, reason }),
|
|
6465
|
-
timeoutSeconds:
|
|
6466
|
-
waitMs
|
|
6791
|
+
timeoutSeconds: Math.ceil(waitMs / 1000),
|
|
6792
|
+
waitMs,
|
|
6467
6793
|
});
|
|
6468
6794
|
}
|
|
6469
6795
|
catch (err) {
|
|
6470
|
-
|
|
6796
|
+
const message = dispatchErrorMessage(err);
|
|
6797
|
+
throw new LanesError("outpost_dispatch_failed", /timed out waiting for outpost pickup/i.test(message)
|
|
6798
|
+
? outpostPickupTimeoutMessage(outpostId, waitMs, `stopping lane ${lane.lane_id}`)
|
|
6799
|
+
: `Could not dispatch stop for lane ${lane.lane_id} to Outpost ${outpostId}: ${message}`, { lane_id: lane.lane_id, outpost_id: outpostId, wait_ms: waitMs, wait_env: OUTPOST_SINGLE_LANE_WAIT_ENV });
|
|
6471
6800
|
}
|
|
6472
6801
|
if (reason !== undefined && outpostRejectedStopReasonOption(dispatched)) {
|
|
6473
6802
|
throw new LanesError("outpost_cli_upgrade_required", `Outpost ${outpostId} needs an hq-cli release containing stop reason support before it can stop lane ${lane.lane_id}`, {
|
|
@@ -6515,7 +6844,7 @@ async function dispatchOutpostLaneStop(ctx, lane, reason, quiet = false) {
|
|
|
6515
6844
|
}
|
|
6516
6845
|
state = remote.state;
|
|
6517
6846
|
alreadyTerminal = true;
|
|
6518
|
-
const owner = await readOutpostTerminalLane(ctx, lane.lane_id, outpostId, commandId);
|
|
6847
|
+
const owner = await readOutpostTerminalLane(ctx, lane.lane_id, outpostId, commandId, waitMs);
|
|
6519
6848
|
if (owner.lane.state !== state) {
|
|
6520
6849
|
throw new LanesError("outpost_delivery_unconfirmed", `Outpost ${outpostId} returned a different terminal state for lane ${lane.lane_id}`, {
|
|
6521
6850
|
lane_id: lane.lane_id,
|
|
@@ -6969,6 +7298,9 @@ function writeAnswerDeliveryFailed(hqRoot, laneId, questionId, err, at) {
|
|
|
6969
7298
|
async function runReconcile(ctx, opts, recoverQueuedMessages) {
|
|
6970
7299
|
const fromEnv = ctx.env.HQ_ROOT?.trim();
|
|
6971
7300
|
const hqRoot = fromEnv && ctx.existsSync(fromEnv) ? fromEnv : ctx.hqRoot();
|
|
7301
|
+
const mirrorRefresh = opts.dryRun
|
|
7302
|
+
? { updated: new Map(), unverifiedRemote: new Map() }
|
|
7303
|
+
: await refreshRemoteLaneMirrors(ctx, hqRoot, loadAllLanes(hqRoot));
|
|
6972
7304
|
const summary = await runReconcilePass(hqRoot, {
|
|
6973
7305
|
now: ctx.now,
|
|
6974
7306
|
host: reconcileHostFromCtx(ctx),
|
|
@@ -7084,7 +7416,12 @@ async function runReconcile(ctx, opts, recoverQueuedMessages) {
|
|
|
7084
7416
|
}
|
|
7085
7417
|
},
|
|
7086
7418
|
});
|
|
7087
|
-
printJson(
|
|
7419
|
+
printJson({
|
|
7420
|
+
...summary,
|
|
7421
|
+
...(mirrorRefresh.unverifiedRemote.size > 0
|
|
7422
|
+
? { unverified_remote_lanes: [...mirrorRefresh.unverifiedRemote.keys()] }
|
|
7423
|
+
: {}),
|
|
7424
|
+
});
|
|
7088
7425
|
if (!summary.ok) {
|
|
7089
7426
|
process.exit(1);
|
|
7090
7427
|
}
|