@indigoai-us/hq-cli 5.280.0 → 5.282.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +31 -0
- package/assets/scaffold/core/scripts/knowledge-prefs.sh +55 -0
- package/assets/scaffold/core/scripts/ontology-audience-grant.sh +81 -0
- package/assets/scaffold/core/scripts/ontology-brief.mjs +93 -0
- package/assets/scaffold/core/scripts/ontology-candidate.sh +100 -0
- package/assets/scaffold/core/scripts/ontology-garden.mjs +238 -0
- package/assets/scaffold/core/scripts/ontology-source-worklist.sh +79 -0
- package/assets/scaffold/core/scripts/source-yaml-validate.sh +36 -0
- package/dist/command-catalog.generated.d.ts +80 -2
- package/dist/command-catalog.generated.js +98 -2
- package/dist/commands/auth.js +40 -2
- package/dist/commands/core-ontology-native.d.ts +16 -0
- package/dist/commands/core-ontology-native.js +924 -0
- package/dist/commands/core-ontology.js +11 -0
- package/dist/commands/core.js +9 -0
- package/dist/commands/group-grants.js +3 -2
- package/dist/commands/lanes-message.js +1 -0
- package/dist/commands/lanes-move.js +2 -0
- package/dist/commands/lanes-resume.js +1 -0
- package/dist/commands/lanes-shared.d.ts +6 -0
- package/dist/commands/lanes.d.ts +1 -0
- package/dist/commands/lanes.js +283 -13
- package/dist/lib/lanes/keep-alive.js +1 -0
- package/dist/lib/lanes/outpost-dispatch.d.ts +3 -1
- package/dist/lib/lanes/outpost-dispatch.js +4 -0
- package/dist/lib/lanes/reconcile.d.ts +7 -1
- package/dist/lib/lanes/reconcile.js +88 -16
- package/dist/lib/lanes/spawn.js +5 -0
- package/dist/lib/lanes/store.js +38 -0
- package/dist/lib/lanes/throttle.d.ts +29 -2
- package/dist/lib/lanes/throttle.js +145 -5
- package/dist/lib/lanes/types.d.ts +12 -1
- package/dist/lib/lanes/types.js +3 -2
- package/dist/lib/lanes/watcher.js +1 -0
- package/package.json +1 -1
|
@@ -20,6 +20,7 @@ import { buildEntityIndex } from "../lib/checkpoint-ontology/tree.js";
|
|
|
20
20
|
import { MERGE_COPIES_FLAG, mergeCopies, } from "../lib/checkpoint-ontology/merge-copies.js";
|
|
21
21
|
import { callDeleteEndpoint } from "./files.js";
|
|
22
22
|
import { resolveVaultCredential } from "../utils/resolve-vault-credential.js";
|
|
23
|
+
import { registerCoreOntologyNativeCommands } from "./core-ontology-native.js";
|
|
23
24
|
export const ONTOLOGY_PAYLOAD_FILE = "payload.json";
|
|
24
25
|
class OntologyUsageError extends Error {
|
|
25
26
|
}
|
|
@@ -267,5 +268,15 @@ export function registerCoreOntologyCommand(core) {
|
|
|
267
268
|
}
|
|
268
269
|
});
|
|
269
270
|
describeCoreCommand(apply, { execution: "native", root: "live" });
|
|
271
|
+
registerCoreOntologyNativeCommands(ontology, () => {
|
|
272
|
+
const argv = process.argv.slice(2);
|
|
273
|
+
const coreIndex = argv.indexOf("core");
|
|
274
|
+
const ontologyIndex = argv.indexOf("ontology", coreIndex + 1);
|
|
275
|
+
const rootOptionIndex = argv.indexOf("--hq-root", coreIndex + 1);
|
|
276
|
+
const groupRoot = rootOptionIndex >= 0 && (ontologyIndex < 0 || rootOptionIndex < ontologyIndex)
|
|
277
|
+
? argv[rootOptionIndex + 1]
|
|
278
|
+
: undefined;
|
|
279
|
+
return groupRoot ? rootFor({ hqRoot: groupRoot }) : process.env.HQ_ROOT || rootFor({});
|
|
280
|
+
});
|
|
270
281
|
}
|
|
271
282
|
//# sourceMappingURL=core-ontology.js.map
|
package/dist/commands/core.js
CHANGED
|
@@ -222,6 +222,15 @@ export const SCAFFOLD_ONLY_ASSETS = [
|
|
|
222
222
|
{ asset: "core/scripts/hq-delegate-secrets.sh", root: "live", summary: "Rollback shell for native delegate secrets" },
|
|
223
223
|
{ asset: "core/scripts/hq-delegate-transfer.sh", root: "live", summary: "Rollback shell for native delegate transfer" },
|
|
224
224
|
{ asset: "core/scripts/lib/secret-patterns.sh", root: "live", summary: "Secret-pattern library sourced by the delegate rollback shells" },
|
|
225
|
+
// Retained rollback companions for the gated native ontology commands. These
|
|
226
|
+
// are loose HQ-tree assets, but `hq core ontology` dispatches natively.
|
|
227
|
+
{ asset: "core/scripts/ontology-audience-grant.sh", root: "live", summary: "Rollback companion for native ontology audience grant" },
|
|
228
|
+
{ asset: "core/scripts/ontology-brief.mjs", root: "live", summary: "Rollback companion for native ontology brief" },
|
|
229
|
+
{ asset: "core/scripts/ontology-candidate.sh", root: "live", summary: "Rollback companion for native ontology candidate" },
|
|
230
|
+
{ asset: "core/scripts/ontology-garden.mjs", root: "live", summary: "Rollback companion for native ontology garden" },
|
|
231
|
+
{ asset: "core/scripts/ontology-source-worklist.sh", root: "live", summary: "Rollback companion for native ontology source worklist" },
|
|
232
|
+
{ asset: "core/scripts/source-yaml-validate.sh", root: "cwd", summary: "Rollback companion for native source YAML validation" },
|
|
233
|
+
{ asset: "core/scripts/knowledge-prefs.sh", root: "live", summary: "Sibling dependency for ontology candidate rollback companion" },
|
|
225
234
|
...NATIVE_UTILITY_COMMANDS.map(({ asset, root, summary }) => ({
|
|
226
235
|
asset,
|
|
227
236
|
root,
|
|
@@ -242,8 +242,9 @@ export function registerGroupGrantsCommand(program) {
|
|
|
242
242
|
});
|
|
243
243
|
grants
|
|
244
244
|
.command("outbound")
|
|
245
|
-
.description("List grants a source-company group holds on other companies"
|
|
246
|
-
|
|
245
|
+
.description("List grants a source-company group holds on other companies; " +
|
|
246
|
+
"requires --group <groupId>.")
|
|
247
|
+
.requiredOption("--group <groupId>", "Group id to inspect (required)")
|
|
247
248
|
.action(async (opts) => {
|
|
248
249
|
try {
|
|
249
250
|
const token = await ensureCognitoToken();
|
|
@@ -156,6 +156,7 @@ export function createLaneMoveHandlers(deps) {
|
|
|
156
156
|
host: target,
|
|
157
157
|
account,
|
|
158
158
|
queued_reason: undefined,
|
|
159
|
+
queued_details: undefined,
|
|
159
160
|
admission_reserved_at: undefined,
|
|
160
161
|
}));
|
|
161
162
|
}, ctx.sleep);
|
|
@@ -218,6 +219,7 @@ export function createLaneMoveHandlers(deps) {
|
|
|
218
219
|
host: before.host,
|
|
219
220
|
account: before.account,
|
|
220
221
|
queued_reason: before.queued_reason,
|
|
222
|
+
queued_details: before.queued_details,
|
|
221
223
|
admission_reserved_at: before.admission_reserved_at,
|
|
222
224
|
}));
|
|
223
225
|
return true;
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { LaneCompanyRef, LanePlacementPolicy, LaneRecord, LanesConfig, QuestionAnswer, QuestionBody, QuestionRecord, SeniorRef, AccountHistoryEntry, LaneAccountSelection, PendingResume, QueuedReason, ResolvedLaneWorker, LivenessDeps, MachineIdentity, MonitorEvidenceReader, ReconcileHost, RunCommand, ThrottleDeps, withLaneHireLocks } from "../lib/lanes/index.js";
|
|
2
2
|
import type { OutpostIdentity } from "../lib/lanes/admission-host.js";
|
|
3
3
|
import type { OutpostDispatchResult } from "../lib/lanes/outpost-dispatch.js";
|
|
4
|
+
import type { CpuBusySource } from "../lib/lanes/throttle.js";
|
|
4
5
|
import type { ProbeCommand } from "../lib/lanes/models.js";
|
|
5
6
|
import type { KillProcessFn, ProveDetachedFn, SpawnProcessFn } from "../lib/lanes/spawn.js";
|
|
6
7
|
import type { WatcherLoopDeps } from "../lib/lanes/watcher.js";
|
|
@@ -81,6 +82,11 @@ export interface AdmissionResult {
|
|
|
81
82
|
action: "start" | "queue" | "refuse";
|
|
82
83
|
reason?: QueuedReason;
|
|
83
84
|
error?: LanesError;
|
|
85
|
+
throttle?: {
|
|
86
|
+
last_busy: number | null;
|
|
87
|
+
busy_source: CpuBusySource | undefined;
|
|
88
|
+
sampled_at: string | undefined;
|
|
89
|
+
};
|
|
84
90
|
}
|
|
85
91
|
export interface PrepareAccountMoveInput {
|
|
86
92
|
account: string;
|
package/dist/commands/lanes.d.ts
CHANGED
|
@@ -25,6 +25,7 @@ export { spawnLaneWorker };
|
|
|
25
25
|
export { questionBodyAbsPath };
|
|
26
26
|
export { MAX_LANE_DEPTH };
|
|
27
27
|
export declare const LANES_RUNS_REL: string;
|
|
28
|
+
export declare const LANE_REMOTE_MIRROR_SYNC_FLAG = "lanes.remote-done-mirror";
|
|
28
29
|
/** The lane → mesh seam. Defined in lib/lanes/mesh-events.ts; the watcher shares it. */
|
|
29
30
|
export { emitLaneDecisionEvents };
|
|
30
31
|
export interface LanesCommandDeps {
|
package/dist/commands/lanes.js
CHANGED
|
@@ -61,6 +61,7 @@ import { assertQuestionBodySecretSafe } from "../lib/lanes/questions.js";
|
|
|
61
61
|
import { compactionAfterUsage, } from "../lib/lanes/compaction.js";
|
|
62
62
|
import { parseCliEffort, resolveLaneEffort, } from "../lib/lanes/effort.js";
|
|
63
63
|
import { LanesError } from "../lib/lanes/errors.js";
|
|
64
|
+
import { readLatestEnvelopeRecord } from "../lib/lanes/watcher.js";
|
|
64
65
|
import { loadQuotaState, quotaBlockFor } from "../lib/lanes/quota.js";
|
|
65
66
|
import { SESSION_MOVE_REASON } from "../lib/lanes/move-account.js";
|
|
66
67
|
import { managerStatusGateEnabledForLane, resolveManagerStatusGateForLane, } from "../lib/lanes/manager-status-gate.js";
|
|
@@ -74,6 +75,8 @@ export { spawnLaneWorker };
|
|
|
74
75
|
export { questionBodyAbsPath };
|
|
75
76
|
export { MAX_LANE_DEPTH };
|
|
76
77
|
export const LANES_RUNS_REL = path.join("workspace", "lanes-runs");
|
|
78
|
+
export const LANE_REMOTE_MIRROR_SYNC_FLAG = "lanes.remote-done-mirror";
|
|
79
|
+
const REMOTE_MIRROR_REFRESH_BATCH_SIZE = 100;
|
|
77
80
|
/** The lane → mesh seam. Defined in lib/lanes/mesh-events.ts; the watcher shares it. */
|
|
78
81
|
export { emitLaneDecisionEvents };
|
|
79
82
|
function withoutProviderHome(value) {
|
|
@@ -714,7 +717,7 @@ async function dispatchOutpostLaneList(ctx, outpostId, query, timeoutSeconds, wa
|
|
|
714
717
|
timeoutSeconds,
|
|
715
718
|
waitMs,
|
|
716
719
|
});
|
|
717
|
-
if (!rejectedLaneListFlag(filtered, query))
|
|
720
|
+
if (query.strictSnapshot || !rejectedLaneListFlag(filtered, query))
|
|
718
721
|
return filtered;
|
|
719
722
|
return ctx.dispatchOutpostExec({
|
|
720
723
|
outpostId,
|
|
@@ -874,6 +877,7 @@ async function withResumeHireAdmission(ctx, lane, fn, grantContext) {
|
|
|
874
877
|
!listLaneIds(hqRoot).includes(lane.senior.id)) {
|
|
875
878
|
throw new LanesError("hire_grant_context_required", `Resume of Outpost lane ${lane.lane_id} requires the primary senior's hire-grant context`, { lane_id: lane.lane_id, senior_lane_id: lane.senior.id });
|
|
876
879
|
}
|
|
880
|
+
await refreshRemoteLaneMirrors(ctx, hqRoot, loadAllLanes(hqRoot), new Set([lane.senior.id]));
|
|
877
881
|
return withHireGrantAdmission(hqRoot, lane.senior.id, (snapshot) => {
|
|
878
882
|
const current = readLane(hqRoot, lane.lane_id);
|
|
879
883
|
if (grantContext !== undefined &&
|
|
@@ -1054,6 +1058,7 @@ function normalizedLaneListQuery(query) {
|
|
|
1054
1058
|
...(query.active ? { active: true } : {}),
|
|
1055
1059
|
...(query.hierarchy === undefined ? {} : { hierarchy: query.hierarchy }),
|
|
1056
1060
|
...(query.snapshot ? { snapshot: true } : {}),
|
|
1061
|
+
...(query.strictSnapshot ? { strictSnapshot: true } : {}),
|
|
1057
1062
|
};
|
|
1058
1063
|
}
|
|
1059
1064
|
async function readOutpostLaneRecordsUncached(ctx, outpostId, query) {
|
|
@@ -1144,6 +1149,156 @@ async function readOutpostLaneRecords(ctx, outpostId, requestedQuery = {}) {
|
|
|
1144
1149
|
byOutpost.set(outpostId, { queryKey, promise });
|
|
1145
1150
|
return promise;
|
|
1146
1151
|
}
|
|
1152
|
+
/** Refresh only primary-created, non-terminal Outpost mirrors in one bounded owner query per Outpost. */
|
|
1153
|
+
async function refreshRemoteLaneMirrors(ctx, hqRoot, records, parentLaneIds) {
|
|
1154
|
+
const result = {
|
|
1155
|
+
updated: new Map(),
|
|
1156
|
+
unverifiedRemote: new Map(),
|
|
1157
|
+
};
|
|
1158
|
+
const host = reconcileHostFromCtx(ctx);
|
|
1159
|
+
if (host.outpostId !== null || parentLaneIds?.size === 0)
|
|
1160
|
+
return result;
|
|
1161
|
+
let primaryIdentity;
|
|
1162
|
+
const originMatchesThisPrimary = (lane) => {
|
|
1163
|
+
if (lane.origin_host_source !== "imds") {
|
|
1164
|
+
return lane.origin_host === host.hostname;
|
|
1165
|
+
}
|
|
1166
|
+
primaryIdentity ??= ctx.resolveMachineIdentity();
|
|
1167
|
+
if (primaryIdentity.status !== "ok")
|
|
1168
|
+
return false;
|
|
1169
|
+
const primaryImdsId = primaryIdentity.imds ??
|
|
1170
|
+
(primaryIdentity.source === "imds" ? primaryIdentity.id : null);
|
|
1171
|
+
return primaryImdsId !== null && primaryImdsId === lane.origin_host;
|
|
1172
|
+
};
|
|
1173
|
+
const candidates = records
|
|
1174
|
+
.filter((lane) => lane.host.kind === "outpost" &&
|
|
1175
|
+
originMatchesThisPrimary(lane) &&
|
|
1176
|
+
(parentLaneIds === undefined || (lane.senior.kind === "lane" &&
|
|
1177
|
+
lane.senior.id !== undefined &&
|
|
1178
|
+
parentLaneIds.has(lane.senior.id))) &&
|
|
1179
|
+
!isTerminalLaneState(lane.state))
|
|
1180
|
+
.sort((a, b) => a.lane_id.localeCompare(b.lane_id));
|
|
1181
|
+
if (candidates.length === 0)
|
|
1182
|
+
return result;
|
|
1183
|
+
const enabledByCompany = new Map();
|
|
1184
|
+
const enabledCandidates = [];
|
|
1185
|
+
for (const lane of candidates) {
|
|
1186
|
+
const companyKey = JSON.stringify([lane.company?.uid ?? null, lane.company?.slug ?? null]);
|
|
1187
|
+
let enabled = enabledByCompany.get(companyKey);
|
|
1188
|
+
if (enabled === undefined) {
|
|
1189
|
+
enabled = await readLanesRegistryFlag(LANE_REMOTE_MIRROR_SYNC_FLAG, {
|
|
1190
|
+
company: lane.company,
|
|
1191
|
+
env: ctx.env,
|
|
1192
|
+
});
|
|
1193
|
+
enabledByCompany.set(companyKey, enabled);
|
|
1194
|
+
}
|
|
1195
|
+
if (enabled)
|
|
1196
|
+
enabledCandidates.push(lane);
|
|
1197
|
+
}
|
|
1198
|
+
if (enabledCandidates.length === 0)
|
|
1199
|
+
return result;
|
|
1200
|
+
const byOutpost = new Map();
|
|
1201
|
+
for (const lane of enabledCandidates) {
|
|
1202
|
+
const group = byOutpost.get(lane.host.id) ?? [];
|
|
1203
|
+
group.push(lane);
|
|
1204
|
+
byOutpost.set(lane.host.id, group);
|
|
1205
|
+
}
|
|
1206
|
+
for (const [outpostId, group] of byOutpost) {
|
|
1207
|
+
const batch = group.slice(0, REMOTE_MIRROR_REFRESH_BATCH_SIZE);
|
|
1208
|
+
const overflow = group.slice(REMOTE_MIRROR_REFRESH_BATCH_SIZE);
|
|
1209
|
+
if (overflow.length > 0) {
|
|
1210
|
+
const ids = overflow.map((lane) => lane.lane_id);
|
|
1211
|
+
for (const lane of overflow) {
|
|
1212
|
+
result.unverifiedRemote.set(lane.lane_id, "refresh_batch_limit");
|
|
1213
|
+
}
|
|
1214
|
+
process.stderr.write(`hq lanes: unverified remote mirrors from Outpost ${outpostId} remain counted; refresh batch limit left lanes ${ids.join(", ")} unread\n`);
|
|
1215
|
+
}
|
|
1216
|
+
let ownerRecords;
|
|
1217
|
+
try {
|
|
1218
|
+
ownerRecords = await readOutpostLaneRecords(ctx, outpostId, {
|
|
1219
|
+
laneIds: batch.map((lane) => lane.lane_id),
|
|
1220
|
+
snapshot: true,
|
|
1221
|
+
strictSnapshot: true,
|
|
1222
|
+
});
|
|
1223
|
+
}
|
|
1224
|
+
catch (error) {
|
|
1225
|
+
const detail = isLanesError(error)
|
|
1226
|
+
? `${error.error}: ${error.message}`
|
|
1227
|
+
: dispatchErrorMessage(error);
|
|
1228
|
+
for (const lane of batch)
|
|
1229
|
+
result.unverifiedRemote.set(lane.lane_id, detail);
|
|
1230
|
+
process.stderr.write(`hq lanes: unverified remote mirrors from Outpost ${outpostId} remain counted: ${detail}; lanes ${batch.map((lane) => lane.lane_id).join(", ")}\n`);
|
|
1231
|
+
continue;
|
|
1232
|
+
}
|
|
1233
|
+
for (const mirror of batch) {
|
|
1234
|
+
const matches = ownerRecords.filter((lane) => lane.lane_id === mirror.lane_id);
|
|
1235
|
+
const owner = matches[0];
|
|
1236
|
+
const ownerIsHostedByOutpost = owner !== undefined && ((owner.host.kind === "outpost" && owner.host.id === outpostId) ||
|
|
1237
|
+
(owner.host.kind === "local" && owner.host.id.trim() !== ""));
|
|
1238
|
+
if (matches.length !== 1 ||
|
|
1239
|
+
owner === undefined ||
|
|
1240
|
+
!ownerIsHostedByOutpost ||
|
|
1241
|
+
(owner.company?.uid ?? null) !== (mirror.company?.uid ?? null) ||
|
|
1242
|
+
(owner.company?.slug ?? null) !== (mirror.company?.slug ?? null)) {
|
|
1243
|
+
const detail = matches.length === 0
|
|
1244
|
+
? "owner did not return the requested lane"
|
|
1245
|
+
: matches.length > 1
|
|
1246
|
+
? "owner returned duplicate lane rows"
|
|
1247
|
+
: "owner lane identity did not match its mirror";
|
|
1248
|
+
result.unverifiedRemote.set(mirror.lane_id, detail);
|
|
1249
|
+
process.stderr.write(`hq lanes: unverified remote mirror ${mirror.lane_id} from Outpost ${outpostId} remains counted: ${detail}\n`);
|
|
1250
|
+
continue;
|
|
1251
|
+
}
|
|
1252
|
+
const endedAt = isTerminalLaneState(owner.state)
|
|
1253
|
+
? owner.last_envelope?.at ?? owner.updated_at
|
|
1254
|
+
: undefined;
|
|
1255
|
+
const envelopeSummary = isTerminalLaneState(owner.state)
|
|
1256
|
+
? owner.envelope_summary
|
|
1257
|
+
: undefined;
|
|
1258
|
+
const before = readLane(hqRoot, mirror.lane_id);
|
|
1259
|
+
if (before.host.kind !== "outpost" || before.host.id !== outpostId ||
|
|
1260
|
+
!originMatchesThisPrimary(before) || isTerminalLaneState(before.state)) {
|
|
1261
|
+
result.updated.set(mirror.lane_id, before);
|
|
1262
|
+
continue;
|
|
1263
|
+
}
|
|
1264
|
+
const needsUpdate = before.state !== owner.state ||
|
|
1265
|
+
before.queued_reason !== (owner.state === "queued" ? owner.queued_reason : undefined) ||
|
|
1266
|
+
before.ended_at !== endedAt ||
|
|
1267
|
+
before.envelope_summary !== envelopeSummary ||
|
|
1268
|
+
(isTerminalLaneState(owner.state) && (JSON.stringify(before.last_envelope) !== JSON.stringify(owner.last_envelope) ||
|
|
1269
|
+
before.story_complete !== owner.story_complete ||
|
|
1270
|
+
before.story_complete_reason !== owner.story_complete_reason ||
|
|
1271
|
+
before.story_status !== owner.story_status ||
|
|
1272
|
+
before.terminal_reason !== owner.terminal_reason));
|
|
1273
|
+
if (!needsUpdate) {
|
|
1274
|
+
result.updated.set(mirror.lane_id, before);
|
|
1275
|
+
continue;
|
|
1276
|
+
}
|
|
1277
|
+
const updated = retryUpdateLane(hqRoot, mirror.lane_id, "reconcile", (current) => {
|
|
1278
|
+
if (current.host.kind !== "outpost" || current.host.id !== outpostId ||
|
|
1279
|
+
!originMatchesThisPrimary(current) || isTerminalLaneState(current.state))
|
|
1280
|
+
return current;
|
|
1281
|
+
return {
|
|
1282
|
+
...current,
|
|
1283
|
+
state: owner.state,
|
|
1284
|
+
queued_reason: owner.state === "queued" ? owner.queued_reason : undefined,
|
|
1285
|
+
ended_at: endedAt,
|
|
1286
|
+
envelope_summary: envelopeSummary,
|
|
1287
|
+
...(isTerminalLaneState(owner.state) ? {
|
|
1288
|
+
last_envelope: owner.last_envelope,
|
|
1289
|
+
story_complete: owner.story_complete,
|
|
1290
|
+
story_complete_reason: owner.story_complete_reason,
|
|
1291
|
+
story_status: owner.story_status,
|
|
1292
|
+
terminal_reason: owner.terminal_reason,
|
|
1293
|
+
terminal_details: owner.terminal_details,
|
|
1294
|
+
} : {}),
|
|
1295
|
+
};
|
|
1296
|
+
});
|
|
1297
|
+
result.updated.set(mirror.lane_id, updated);
|
|
1298
|
+
}
|
|
1299
|
+
}
|
|
1300
|
+
return result;
|
|
1301
|
+
}
|
|
1147
1302
|
async function resolveCommandLaneTarget(ctx, laneId, options = {}) {
|
|
1148
1303
|
try {
|
|
1149
1304
|
return { lane: readLane(ctx.hqRoot(), laneId), remoteOnly: false };
|
|
@@ -1802,11 +1957,16 @@ async function applyThrottle(hqRoot, config, ctx, current, provider, pinnedAccou
|
|
|
1802
1957
|
if (current.action !== "start" &&
|
|
1803
1958
|
!(current.action === "queue" && current.reason === "queue_position"))
|
|
1804
1959
|
return current;
|
|
1805
|
-
const { decision } = await decideAdmission(hqRoot, config.throttle, ctx.throttleDeps);
|
|
1960
|
+
const { decision, state } = await decideAdmission(hqRoot, config.throttle, ctx.throttleDeps);
|
|
1806
1961
|
if (decision.admit)
|
|
1807
1962
|
return current;
|
|
1963
|
+
const throttle = {
|
|
1964
|
+
last_busy: state.last_busy ?? null,
|
|
1965
|
+
busy_source: state.busy_source,
|
|
1966
|
+
sampled_at: state.sampled_at,
|
|
1967
|
+
};
|
|
1808
1968
|
if (current.action === "queue")
|
|
1809
|
-
return { action: "queue", reason: "throttle" };
|
|
1969
|
+
return { action: "queue", reason: "throttle", throttle };
|
|
1810
1970
|
const providerCfg = config.providers[provider];
|
|
1811
1971
|
const host = options.host ?? reconcileHostFromCtx(ctx);
|
|
1812
1972
|
const nowMs = options.nowMs ?? ctx.now().getTime();
|
|
@@ -1838,7 +1998,7 @@ async function applyThrottle(hqRoot, config, ctx, current, provider, pinnedAccou
|
|
|
1838
1998
|
}),
|
|
1839
1999
|
};
|
|
1840
2000
|
}
|
|
1841
|
-
return { action: "queue", reason: "throttle" };
|
|
2001
|
+
return { action: "queue", reason: "throttle", throttle };
|
|
1842
2002
|
}
|
|
1843
2003
|
async function assessLaneAdmission(ctx, lane, options) {
|
|
1844
2004
|
const hqRoot = ctx.hqRoot();
|
|
@@ -2196,6 +2356,7 @@ function markDispatchFailure(hqRoot, laneId, details) {
|
|
|
2196
2356
|
...current,
|
|
2197
2357
|
state: "failed",
|
|
2198
2358
|
queued_reason: undefined,
|
|
2359
|
+
queued_details: undefined,
|
|
2199
2360
|
terminal_reason: DISPATCH_TERMINAL_REASON,
|
|
2200
2361
|
terminal_details: details,
|
|
2201
2362
|
};
|
|
@@ -2219,6 +2380,7 @@ export function pickupFailureUpdater(reasonRaw, details) {
|
|
|
2219
2380
|
...current,
|
|
2220
2381
|
state: "failed",
|
|
2221
2382
|
queued_reason: undefined,
|
|
2383
|
+
queued_details: undefined,
|
|
2222
2384
|
pickup_validation_pending: undefined,
|
|
2223
2385
|
pickup_validation_pending_since: undefined,
|
|
2224
2386
|
terminal_reason: reason,
|
|
@@ -3035,6 +3197,9 @@ async function runCreate(ctx, opts) {
|
|
|
3035
3197
|
: {}),
|
|
3036
3198
|
attribution_history: [],
|
|
3037
3199
|
};
|
|
3200
|
+
if (senior.kind === "lane" && senior.id) {
|
|
3201
|
+
await refreshRemoteLaneMirrors(ctx, hqRoot, loadAllLanes(hqRoot), new Set([senior.id]));
|
|
3202
|
+
}
|
|
3038
3203
|
const lane = senior.kind === "lane"
|
|
3039
3204
|
? withHireGrantAdmission(hqRoot, senior.id, ({ parent, children, parentGrant, availableChildGrant }) => {
|
|
3040
3205
|
if (isTerminalLaneState(parent.state)) {
|
|
@@ -3083,6 +3248,7 @@ async function runCreate(ctx, opts) {
|
|
|
3083
3248
|
...current,
|
|
3084
3249
|
state: "failed",
|
|
3085
3250
|
queued_reason: undefined,
|
|
3251
|
+
queued_details: undefined,
|
|
3086
3252
|
terminal_reason: "mesh_thread_registration_failed",
|
|
3087
3253
|
}));
|
|
3088
3254
|
}
|
|
@@ -3227,6 +3393,7 @@ async function dispatchCreatedOutpostLane(ctx, input) {
|
|
|
3227
3393
|
...current,
|
|
3228
3394
|
state: "cancelled",
|
|
3229
3395
|
queued_reason: undefined,
|
|
3396
|
+
queued_details: undefined,
|
|
3230
3397
|
terminal_reason: "outpost_self_target",
|
|
3231
3398
|
outpost_command_id: dispatched.commandId,
|
|
3232
3399
|
}));
|
|
@@ -4139,6 +4306,7 @@ async function runPickup(ctx, opts) {
|
|
|
4139
4306
|
...live,
|
|
4140
4307
|
state: "queued",
|
|
4141
4308
|
queued_reason: admission.reason ?? "throttle",
|
|
4309
|
+
queued_details: undefined,
|
|
4142
4310
|
queued_at: live.queued_at ?? ctx.now().toISOString(),
|
|
4143
4311
|
};
|
|
4144
4312
|
});
|
|
@@ -4228,7 +4396,7 @@ async function runList(ctx, opts) {
|
|
|
4228
4396
|
const hierarchyIds = opts.hierarchy === undefined
|
|
4229
4397
|
? undefined
|
|
4230
4398
|
: new Set(selectOutpostHireGrantHierarchy(opts.hierarchy, laneSource).map((lane) => lane.lane_id));
|
|
4231
|
-
|
|
4399
|
+
let selectedLanes = laneSource
|
|
4232
4400
|
.filter((lane) => {
|
|
4233
4401
|
if (requestedLaneIds.length > 0 && !requestedIds.has(lane.lane_id))
|
|
4234
4402
|
return false;
|
|
@@ -4254,9 +4422,27 @@ async function runList(ctx, opts) {
|
|
|
4254
4422
|
})
|
|
4255
4423
|
.sort(compareLaneQueueOrder);
|
|
4256
4424
|
if (opts.snapshot) {
|
|
4257
|
-
printJson({
|
|
4425
|
+
printJson({
|
|
4426
|
+
ok: true,
|
|
4427
|
+
action: "list",
|
|
4428
|
+
lanes: selectedLanes.map((lane) => {
|
|
4429
|
+
if (!isTerminalLaneState(lane.state))
|
|
4430
|
+
return lane;
|
|
4431
|
+
const envelope = readLatestEnvelopeRecord(hqRoot, lane.lane_id)?.envelope;
|
|
4432
|
+
return envelope?.summary ? { ...lane, envelope_summary: envelope.summary } : lane;
|
|
4433
|
+
}),
|
|
4434
|
+
});
|
|
4258
4435
|
return;
|
|
4259
4436
|
}
|
|
4437
|
+
const remoteRefresh = await refreshRemoteLaneMirrors(ctx, hqRoot, laneSource, new Set(selectedLanes
|
|
4438
|
+
.filter((lane) => !isTerminalLaneState(lane.state))
|
|
4439
|
+
.map((lane) => lane.lane_id)));
|
|
4440
|
+
for (const [laneId, updated] of remoteRefresh.updated) {
|
|
4441
|
+
const index = laneSource.findIndex((lane) => lane.lane_id === laneId);
|
|
4442
|
+
if (index >= 0)
|
|
4443
|
+
laneSource[index] = updated;
|
|
4444
|
+
}
|
|
4445
|
+
selectedLanes = selectedLanes.map((lane) => remoteRefresh.updated.get(lane.lane_id) ?? lane);
|
|
4260
4446
|
const now = ctx.now();
|
|
4261
4447
|
const host = reconcileHostFromCtx(ctx);
|
|
4262
4448
|
const allLanes = laneSource;
|
|
@@ -4313,11 +4499,12 @@ async function runList(ctx, opts) {
|
|
|
4313
4499
|
return {
|
|
4314
4500
|
...lane,
|
|
4315
4501
|
...hireByLane.get(lane.lane_id),
|
|
4502
|
+
...(remoteRefresh.unverifiedRemote.has(lane.lane_id) ? { unverified_remote: true } : {}),
|
|
4316
4503
|
stories: laneStories(lane),
|
|
4317
4504
|
story_ids: laneStories(lane).map((story) => story.story_id),
|
|
4318
4505
|
no_story_reason: lane.no_story_reason ?? null,
|
|
4319
4506
|
...(remoteQueued
|
|
4320
|
-
? { state: "remote", remote_state: lane.state, queued_reason: undefined }
|
|
4507
|
+
? { state: "remote", remote_state: lane.state, queued_reason: undefined, queued_details: undefined }
|
|
4321
4508
|
: {}),
|
|
4322
4509
|
overdue: Boolean(lane.overdue) || isOverdue(hqRoot, lane.lane_id, now),
|
|
4323
4510
|
stalled: Boolean(lane.stalled),
|
|
@@ -4349,6 +4536,9 @@ async function runList(ctx, opts) {
|
|
|
4349
4536
|
ok: true,
|
|
4350
4537
|
action: "list",
|
|
4351
4538
|
lanes,
|
|
4539
|
+
...(remoteRefresh.unverifiedRemote.size > 0
|
|
4540
|
+
? { unverified_remote_lanes: [...remoteRefresh.unverifiedRemote.keys()] }
|
|
4541
|
+
: {}),
|
|
4352
4542
|
...(host_capacity !== undefined ? { host_capacity } : {}),
|
|
4353
4543
|
};
|
|
4354
4544
|
if (opts.human)
|
|
@@ -4552,7 +4742,7 @@ async function runShow(ctx, laneId) {
|
|
|
4552
4742
|
storedLane.host.kind === "outpost" &&
|
|
4553
4743
|
!laneRunsHere(storedLane, reconcileHostFromCtx(ctx));
|
|
4554
4744
|
const lane = remoteQueued
|
|
4555
|
-
? { ...storedLane, state: "remote", remote_state: storedLane.state, queued_reason: undefined }
|
|
4745
|
+
? { ...storedLane, state: "remote", remote_state: storedLane.state, queued_reason: undefined, queued_details: undefined }
|
|
4556
4746
|
: storedLane;
|
|
4557
4747
|
const askDeliveryFailed = readJsonFile(askDeliveryFailedPath(hqRoot, laneId));
|
|
4558
4748
|
const keepAliveState = resolveLaneKeepAlive(hqRoot, storedLane);
|
|
@@ -4768,6 +4958,13 @@ async function applyHireGrant(ctx, laneId, hireGrant, authorization, grantContex
|
|
|
4768
4958
|
? [initial.senior.id]
|
|
4769
4959
|
: []),
|
|
4770
4960
|
];
|
|
4961
|
+
if (authorization === "senior") {
|
|
4962
|
+
const parentLaneIds = new Set([initial.lane_id]);
|
|
4963
|
+
if (initial.senior.kind === "lane" && initial.senior.id) {
|
|
4964
|
+
parentLaneIds.add(initial.senior.id);
|
|
4965
|
+
}
|
|
4966
|
+
await refreshRemoteLaneMirrors(ctx, hqRoot, loadAllLanes(hqRoot), parentLaneIds);
|
|
4967
|
+
}
|
|
4771
4968
|
const updated = ctx.withGrantHireLocks(hqRoot, lockedLaneIds, () => {
|
|
4772
4969
|
const current = readLane(hqRoot, laneId);
|
|
4773
4970
|
const currentCallerAuthorization = authorization === "senior"
|
|
@@ -5262,6 +5459,7 @@ async function runAdmit(ctx, laneId, opts) {
|
|
|
5262
5459
|
});
|
|
5263
5460
|
}
|
|
5264
5461
|
let blockedBy;
|
|
5462
|
+
let throttleTelemetry;
|
|
5265
5463
|
if (opts.force) {
|
|
5266
5464
|
const callerAuthorization = forceAdmissionAuthorization(ctx, lane);
|
|
5267
5465
|
if (!callerAuthorization.authorized) {
|
|
@@ -5269,9 +5467,14 @@ async function runAdmit(ctx, laneId, opts) {
|
|
|
5269
5467
|
}
|
|
5270
5468
|
const config = loadLanesConfig(hqRoot);
|
|
5271
5469
|
const lanes = loadAllLanes(hqRoot).filter((row) => row.lane_id !== laneId && laneRunsHere(row, host));
|
|
5272
|
-
const { decision } = await decideAdmission(hqRoot, config.throttle, ctx.throttleDeps);
|
|
5470
|
+
const { decision, state } = await decideAdmission(hqRoot, config.throttle, ctx.throttleDeps);
|
|
5273
5471
|
if (!decision.admit) {
|
|
5274
5472
|
blockedBy = "throttle";
|
|
5473
|
+
throttleTelemetry = {
|
|
5474
|
+
last_busy: state.last_busy ?? null,
|
|
5475
|
+
busy_source: state.busy_source,
|
|
5476
|
+
sampled_at: state.sampled_at,
|
|
5477
|
+
};
|
|
5275
5478
|
}
|
|
5276
5479
|
else if (hasEarlierEligibleQueuedLane(lanes, lane, config, host, (row) => row.worker_profile === undefined
|
|
5277
5480
|
? undefined
|
|
@@ -5284,6 +5487,7 @@ async function runAdmit(ctx, laneId, opts) {
|
|
|
5284
5487
|
queueAt: lane.queued_at ?? lane.created_at,
|
|
5285
5488
|
alreadyQueued: true,
|
|
5286
5489
|
});
|
|
5490
|
+
throttleTelemetry = admission.throttle;
|
|
5287
5491
|
if (admission.action === "refuse" && admission.error) {
|
|
5288
5492
|
throw admission.error;
|
|
5289
5493
|
}
|
|
@@ -5300,7 +5504,7 @@ async function runAdmit(ctx, laneId, opts) {
|
|
|
5300
5504
|
state: current.state,
|
|
5301
5505
|
});
|
|
5302
5506
|
}
|
|
5303
|
-
return { ...current, queued_reason: blockedBy };
|
|
5507
|
+
return { ...current, queued_reason: blockedBy, queued_details: undefined };
|
|
5304
5508
|
});
|
|
5305
5509
|
printJson({
|
|
5306
5510
|
ok: true,
|
|
@@ -5308,6 +5512,9 @@ async function runAdmit(ctx, laneId, opts) {
|
|
|
5308
5512
|
lane_id: laneId,
|
|
5309
5513
|
admitted: false,
|
|
5310
5514
|
blocked_by: blockedBy,
|
|
5515
|
+
...(blockedBy === "throttle" && throttleTelemetry !== undefined
|
|
5516
|
+
? throttleTelemetry
|
|
5517
|
+
: {}),
|
|
5311
5518
|
lane: updated,
|
|
5312
5519
|
});
|
|
5313
5520
|
return;
|
|
@@ -5675,6 +5882,7 @@ async function prepareAccountMove(ctx, lane, input) {
|
|
|
5675
5882
|
else if (queued) {
|
|
5676
5883
|
next.state = "queued";
|
|
5677
5884
|
next.queued_reason = admission?.reason ?? "throttle";
|
|
5885
|
+
next.queued_details = undefined;
|
|
5678
5886
|
next.queued_at = at;
|
|
5679
5887
|
}
|
|
5680
5888
|
else {
|
|
@@ -6221,6 +6429,12 @@ async function reparentOnOwner(ctx, laneId, opts) {
|
|
|
6221
6429
|
const nextSenior = parseSeniorRef(opts.senior);
|
|
6222
6430
|
assertSeniorExists(hqRoot, nextSenior, ctx);
|
|
6223
6431
|
const initial = readLane(hqRoot, laneId);
|
|
6432
|
+
const refreshParents = new Set();
|
|
6433
|
+
if (initial.senior.kind === "lane" && initial.senior.id)
|
|
6434
|
+
refreshParents.add(initial.senior.id);
|
|
6435
|
+
if (nextSenior.kind === "lane" && nextSenior.id)
|
|
6436
|
+
refreshParents.add(nextSenior.id);
|
|
6437
|
+
await refreshRemoteLaneMirrors(ctx, hqRoot, loadAllLanes(hqRoot), refreshParents);
|
|
6224
6438
|
const lockIds = [
|
|
6225
6439
|
laneId,
|
|
6226
6440
|
...(initial.senior.kind === "lane" && initial.senior.id ? [initial.senior.id] : []),
|
|
@@ -6380,12 +6594,58 @@ const CANONICAL_LOG_STREAMS = [
|
|
|
6380
6594
|
];
|
|
6381
6595
|
async function runLogs(ctx, laneId, opts) {
|
|
6382
6596
|
const hqRoot = ctx.hqRoot();
|
|
6383
|
-
readLane(hqRoot, laneId);
|
|
6384
|
-
const logsDir = path.join(runDir(hqRoot, laneId), "logs");
|
|
6597
|
+
const lane = readLane(hqRoot, laneId);
|
|
6385
6598
|
const tail = parseOptionalInt(opts.tail, "--tail");
|
|
6386
6599
|
if (tail === 0) {
|
|
6387
6600
|
throw new LanesError("invalid_integer", "--tail must be a positive integer", { flag: "--tail", value: opts.tail });
|
|
6388
6601
|
}
|
|
6602
|
+
if (lane.host.kind === "outpost" &&
|
|
6603
|
+
!laneRunsHere(lane, reconcileHostFromCtx(ctx)) &&
|
|
6604
|
+
await readLanesRegistryFlag(LANE_REMOTE_MIRROR_SYNC_FLAG, {
|
|
6605
|
+
company: lane.company,
|
|
6606
|
+
env: ctx.env,
|
|
6607
|
+
})) {
|
|
6608
|
+
const outpostId = lane.host.id;
|
|
6609
|
+
const waitMs = outpostSingleLaneWaitMs(ctx.env);
|
|
6610
|
+
let dispatched;
|
|
6611
|
+
try {
|
|
6612
|
+
dispatched = await ctx.dispatchOutpostExec({
|
|
6613
|
+
outpostId,
|
|
6614
|
+
command: buildOutpostLaneCommand({
|
|
6615
|
+
action: "logs",
|
|
6616
|
+
laneId,
|
|
6617
|
+
hqBinary: outpostCliOverride(ctx, outpostId),
|
|
6618
|
+
...(tail === undefined ? {} : { tail: String(tail) }),
|
|
6619
|
+
json: true,
|
|
6620
|
+
}),
|
|
6621
|
+
timeoutSeconds: Math.ceil(waitMs / 1000),
|
|
6622
|
+
waitMs,
|
|
6623
|
+
});
|
|
6624
|
+
}
|
|
6625
|
+
catch (error) {
|
|
6626
|
+
throw new LanesError("outpost_lane_logs_unavailable", `Could not read logs for lane ${laneId} from Outpost ${outpostId}: ${dispatchErrorMessage(error)}`, { lane_id: laneId, outpost_id: outpostId, wait_ms: waitMs, wait_env: OUTPOST_SINGLE_LANE_WAIT_ENV });
|
|
6627
|
+
}
|
|
6628
|
+
if (dispatched.done !== true || dispatched.status !== "Success" || dispatched.exitCode !== 0) {
|
|
6629
|
+
throw new LanesError("outpost_lane_logs_unavailable", `Outpost ${outpostId} did not return logs for lane ${laneId}`, { lane_id: laneId, outpost_id: outpostId, status: dispatched.status, exit_code: dispatched.exitCode });
|
|
6630
|
+
}
|
|
6631
|
+
let body;
|
|
6632
|
+
try {
|
|
6633
|
+
body = JSON.parse(dispatched.stdout);
|
|
6634
|
+
}
|
|
6635
|
+
catch (error) {
|
|
6636
|
+
throw new LanesError("outpost_lane_logs_invalid", `Outpost ${outpostId} returned invalid logs JSON for lane ${laneId}: ${dispatchErrorMessage(error)}`, { lane_id: laneId, outpost_id: outpostId });
|
|
6637
|
+
}
|
|
6638
|
+
if (body === null || typeof body !== "object" || Array.isArray(body) ||
|
|
6639
|
+
body.ok !== true ||
|
|
6640
|
+
body.action !== "logs" ||
|
|
6641
|
+
body.lane_id !== laneId ||
|
|
6642
|
+
!Array.isArray(body.streams)) {
|
|
6643
|
+
throw new LanesError("outpost_lane_logs_invalid", `Outpost ${outpostId} returned a malformed logs payload for lane ${laneId}`, { lane_id: laneId, outpost_id: outpostId });
|
|
6644
|
+
}
|
|
6645
|
+
printJson(body);
|
|
6646
|
+
return;
|
|
6647
|
+
}
|
|
6648
|
+
const logsDir = path.join(runDir(hqRoot, laneId), "logs");
|
|
6389
6649
|
const names = new Set(CANONICAL_LOG_STREAMS);
|
|
6390
6650
|
if (fs.existsSync(logsDir)) {
|
|
6391
6651
|
for (const name of fs.readdirSync(logsDir)) {
|
|
@@ -6705,6 +6965,7 @@ async function runStop(ctx, laneId, reason, quiet = false) {
|
|
|
6705
6965
|
...current,
|
|
6706
6966
|
state: stopCommittedState(current.state),
|
|
6707
6967
|
queued_reason: undefined,
|
|
6968
|
+
queued_details: undefined,
|
|
6708
6969
|
...(operatorReason === undefined ? {} : {
|
|
6709
6970
|
stop_history: [
|
|
6710
6971
|
...(current.stop_history ?? []),
|
|
@@ -6779,6 +7040,7 @@ async function runInterrupt(ctx, laneId, reason) {
|
|
|
6779
7040
|
...current,
|
|
6780
7041
|
state: "interrupted",
|
|
6781
7042
|
queued_reason: undefined,
|
|
7043
|
+
queued_details: undefined,
|
|
6782
7044
|
interrupted_at: at,
|
|
6783
7045
|
terminal_reason: "operator_interrupt",
|
|
6784
7046
|
...(operatorReason === undefined ? {} : {
|
|
@@ -7059,6 +7321,9 @@ function writeAnswerDeliveryFailed(hqRoot, laneId, questionId, err, at) {
|
|
|
7059
7321
|
async function runReconcile(ctx, opts, recoverQueuedMessages) {
|
|
7060
7322
|
const fromEnv = ctx.env.HQ_ROOT?.trim();
|
|
7061
7323
|
const hqRoot = fromEnv && ctx.existsSync(fromEnv) ? fromEnv : ctx.hqRoot();
|
|
7324
|
+
const mirrorRefresh = opts.dryRun
|
|
7325
|
+
? { updated: new Map(), unverifiedRemote: new Map() }
|
|
7326
|
+
: await refreshRemoteLaneMirrors(ctx, hqRoot, loadAllLanes(hqRoot));
|
|
7062
7327
|
const summary = await runReconcilePass(hqRoot, {
|
|
7063
7328
|
now: ctx.now,
|
|
7064
7329
|
host: reconcileHostFromCtx(ctx),
|
|
@@ -7174,7 +7439,12 @@ async function runReconcile(ctx, opts, recoverQueuedMessages) {
|
|
|
7174
7439
|
}
|
|
7175
7440
|
},
|
|
7176
7441
|
});
|
|
7177
|
-
printJson(
|
|
7442
|
+
printJson({
|
|
7443
|
+
...summary,
|
|
7444
|
+
...(mirrorRefresh.unverifiedRemote.size > 0
|
|
7445
|
+
? { unverified_remote_lanes: [...mirrorRefresh.unverifiedRemote.keys()] }
|
|
7446
|
+
: {}),
|
|
7447
|
+
});
|
|
7178
7448
|
if (!summary.ok) {
|
|
7179
7449
|
process.exit(1);
|
|
7180
7450
|
}
|