@indigoai-us/hq-cli 5.287.0 → 5.289.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +48 -5
- package/assets/bot-workers/setup/skills/explain-hq.md +35 -0
- package/assets/bot-workers/setup/worker.yaml +25 -23
- package/dist/command-catalog.generated.d.ts +3 -0
- package/dist/command-catalog.generated.js +4 -0
- package/dist/commands/agent-enroll.js +10 -2
- package/dist/commands/agents.d.ts +1 -0
- package/dist/commands/agents.js +12 -4
- package/dist/commands/core-ontology.d.ts +4 -1
- package/dist/commands/core-ontology.js +26 -0
- package/dist/commands/lanes-resume.js +41 -2
- package/dist/commands/lanes-shared.d.ts +2 -0
- package/dist/commands/lanes.js +204 -6
- package/dist/lanes-drain-hook.d.ts +12 -0
- package/dist/lanes-drain-hook.js +59 -55
- package/dist/lib/bot/run.js +92 -2
- package/dist/lib/bot/setup-explain.d.ts +68 -0
- package/dist/lib/bot/setup-explain.js +141 -0
- package/dist/lib/checkpoint-ontology/merge-copies.d.ts +10 -1
- package/dist/lib/checkpoint-ontology/merge-copies.js +17 -2
- package/dist/lib/checkpoint-ontology/plan.d.ts +0 -88
- package/dist/lib/checkpoint-ontology/plan.js +80 -107
- package/dist/lib/lanes/dropbox.d.ts +1 -2
- package/dist/lib/lanes/dropbox.js +10 -111
- package/dist/lib/lanes/message-replay.d.ts +8 -0
- package/dist/lib/lanes/message-replay.js +119 -0
- package/dist/lib/lanes/store.js +30 -12
- package/dist/lib/lanes/types.d.ts +6 -2
- package/dist/lib/lanes/types.js +1 -1
- package/dist/lib/plan-limit-nag.js +11 -0
- package/dist/main.js +17 -4
- package/dist/utils/cli-telemetry.d.ts +21 -0
- package/dist/utils/cli-telemetry.js +101 -1
- package/dist/utils/cognito-refresh-sentry.d.ts +3 -0
- package/dist/utils/cognito-refresh-sentry.js +49 -0
- package/dist/utils/cognito-session.js +55 -0
- package/package.json +1 -1
package/dist/commands/lanes.js
CHANGED
|
@@ -79,6 +79,7 @@ export { MAX_LANE_DEPTH };
|
|
|
79
79
|
export const LANES_RUNS_REL = path.join("workspace", "lanes-runs");
|
|
80
80
|
export const LANE_REMOTE_MIRROR_SYNC_FLAG = "lanes.remote-done-mirror";
|
|
81
81
|
const REMOTE_MIRROR_REFRESH_BATCH_SIZE = 100;
|
|
82
|
+
const REMOTE_LANE_NOT_FOUND_GRACE_MS = 15 * 60_000;
|
|
82
83
|
/** The lane → mesh seam. Defined in lib/lanes/mesh-events.ts; the watcher shares it. */
|
|
83
84
|
export { emitLaneDecisionEvents };
|
|
84
85
|
export { LANE_OUTPOST_SNAPSHOT_CACHE_FLAG };
|
|
@@ -1129,6 +1130,7 @@ function normalizedLaneListQuery(query) {
|
|
|
1129
1130
|
...(hierarchy === undefined ? {} : { hierarchy }),
|
|
1130
1131
|
...(query.snapshot ? { snapshot: true } : {}),
|
|
1131
1132
|
...(query.strictSnapshot ? { strictSnapshot: true } : {}),
|
|
1133
|
+
...(query.includeForeignHostRows ? { includeForeignHostRows: true } : {}),
|
|
1132
1134
|
};
|
|
1133
1135
|
}
|
|
1134
1136
|
async function readOutpostLaneRecordsUncached(ctx, outpostId, query, dispatchOptions = {}) {
|
|
@@ -1191,7 +1193,7 @@ async function readOutpostLaneRecordsUncached(ctx, outpostId, query, dispatchOpt
|
|
|
1191
1193
|
// Outpost's own store, so preserve those local identities for accounting.
|
|
1192
1194
|
const hasLocalHostIdentity = rowHost.kind === "local" &&
|
|
1193
1195
|
typeof rowHost.id === "string" && rowHost.id.trim() !== "";
|
|
1194
|
-
if (!belongsToOutpost && !hasLocalHostIdentity)
|
|
1196
|
+
if (!belongsToOutpost && !hasLocalHostIdentity && !query.includeForeignHostRows)
|
|
1195
1197
|
continue;
|
|
1196
1198
|
try {
|
|
1197
1199
|
lanes.push(parseLaneRecord(normalizeLaneRecordView(raw)));
|
|
@@ -1216,6 +1218,7 @@ async function refreshRemoteLaneMirrors(ctx, hqRoot, records, parentLaneIds) {
|
|
|
1216
1218
|
const result = {
|
|
1217
1219
|
updated: new Map(),
|
|
1218
1220
|
unverifiedRemote: new Map(),
|
|
1221
|
+
missingOnOwner: new Map(),
|
|
1219
1222
|
};
|
|
1220
1223
|
const host = reconcileHostFromCtx(ctx);
|
|
1221
1224
|
if (ctx.env.HQ_LANES_REMOTE_READ === "1" || parentLaneIds?.size === 0)
|
|
@@ -1285,6 +1288,7 @@ async function refreshRemoteLaneMirrors(ctx, hqRoot, records, parentLaneIds) {
|
|
|
1285
1288
|
laneIds: batch.map((lane) => lane.lane_id),
|
|
1286
1289
|
snapshot: true,
|
|
1287
1290
|
strictSnapshot: true,
|
|
1291
|
+
includeForeignHostRows: true,
|
|
1288
1292
|
});
|
|
1289
1293
|
}
|
|
1290
1294
|
catch (error) {
|
|
@@ -1306,6 +1310,8 @@ async function refreshRemoteLaneMirrors(ctx, hqRoot, records, parentLaneIds) {
|
|
|
1306
1310
|
!ownerIsHostedByOutpost ||
|
|
1307
1311
|
(owner.company?.uid ?? null) !== (mirror.company?.uid ?? null) ||
|
|
1308
1312
|
(owner.company?.slug ?? null) !== (mirror.company?.slug ?? null)) {
|
|
1313
|
+
if (matches.length === 0)
|
|
1314
|
+
result.missingOnOwner.set(mirror.lane_id, mirror);
|
|
1309
1315
|
const detail = matches.length === 0
|
|
1310
1316
|
? "owner did not return the requested lane"
|
|
1311
1317
|
: matches.length > 1
|
|
@@ -1329,6 +1335,7 @@ async function refreshRemoteLaneMirrors(ctx, hqRoot, records, parentLaneIds) {
|
|
|
1329
1335
|
}
|
|
1330
1336
|
const needsUpdate = before.state !== owner.state ||
|
|
1331
1337
|
before.queued_reason !== (owner.state === "queued" ? owner.queued_reason : undefined) ||
|
|
1338
|
+
JSON.stringify(before.queued_details) !== JSON.stringify(owner.state === "queued" ? owner.queued_details : undefined) ||
|
|
1332
1339
|
before.ended_at !== endedAt ||
|
|
1333
1340
|
before.envelope_summary !== envelopeSummary ||
|
|
1334
1341
|
(isTerminalLaneState(owner.state) && (JSON.stringify(before.last_envelope) !== JSON.stringify(owner.last_envelope) ||
|
|
@@ -1348,6 +1355,7 @@ async function refreshRemoteLaneMirrors(ctx, hqRoot, records, parentLaneIds) {
|
|
|
1348
1355
|
...current,
|
|
1349
1356
|
state: owner.state,
|
|
1350
1357
|
queued_reason: owner.state === "queued" ? owner.queued_reason : undefined,
|
|
1358
|
+
queued_details: owner.state === "queued" ? owner.queued_details : undefined,
|
|
1351
1359
|
ended_at: endedAt,
|
|
1352
1360
|
envelope_summary: envelopeSummary,
|
|
1353
1361
|
...(isTerminalLaneState(owner.state) ? {
|
|
@@ -2445,6 +2453,47 @@ function markDispatchFailure(hqRoot, laneId, details) {
|
|
|
2445
2453
|
};
|
|
2446
2454
|
});
|
|
2447
2455
|
}
|
|
2456
|
+
function retainQueuedOutpostAfterUnknownDelivery(hqRoot, laneId, outpostId, commandId) {
|
|
2457
|
+
return casUpdateLane(hqRoot, laneId, (current) => {
|
|
2458
|
+
if (current.state !== "queued" ||
|
|
2459
|
+
current.host.kind !== "outpost" ||
|
|
2460
|
+
current.host.id !== outpostId)
|
|
2461
|
+
return current;
|
|
2462
|
+
return {
|
|
2463
|
+
...current,
|
|
2464
|
+
queued_details: {
|
|
2465
|
+
reason: "outpost_dispatch_unknown",
|
|
2466
|
+
outcome_unknown: true,
|
|
2467
|
+
command_id: commandId,
|
|
2468
|
+
},
|
|
2469
|
+
};
|
|
2470
|
+
});
|
|
2471
|
+
}
|
|
2472
|
+
function markRemoteLaneLost(hqRoot, laneId, outpostId, at, writer, operatorReason, expectedVersion) {
|
|
2473
|
+
return retryUpdateLane(hqRoot, laneId, writer, (current) => {
|
|
2474
|
+
if (current.host.kind !== "outpost" ||
|
|
2475
|
+
current.host.id !== outpostId ||
|
|
2476
|
+
(expectedVersion !== undefined && current.version !== expectedVersion) ||
|
|
2477
|
+
isTerminalLaneState(current.state))
|
|
2478
|
+
return current;
|
|
2479
|
+
return {
|
|
2480
|
+
...current,
|
|
2481
|
+
state: "failed",
|
|
2482
|
+
queued_reason: undefined,
|
|
2483
|
+
queued_details: undefined,
|
|
2484
|
+
terminal_reason: "remote_lost",
|
|
2485
|
+
terminal_details: {
|
|
2486
|
+
outpost_id: outpostId,
|
|
2487
|
+
owner_error: "lane_not_found",
|
|
2488
|
+
...(operatorReason === undefined ? {} : { operator_reason: operatorReason }),
|
|
2489
|
+
},
|
|
2490
|
+
stop_history: [
|
|
2491
|
+
...(current.stop_history ?? []),
|
|
2492
|
+
{ at, reason: "remote_lost" },
|
|
2493
|
+
],
|
|
2494
|
+
};
|
|
2495
|
+
});
|
|
2496
|
+
}
|
|
2448
2497
|
/**
|
|
2449
2498
|
* The CAS updater a pickup failure applies. A lane that is already terminal
|
|
2450
2499
|
* for any reason (a cancel that landed after the caller's last read, an
|
|
@@ -3528,6 +3577,22 @@ async function dispatchCreatedOutpostLane(ctx, input) {
|
|
|
3528
3577
|
exit_code: dispatched.exitCode,
|
|
3529
3578
|
});
|
|
3530
3579
|
}
|
|
3580
|
+
const commandId = dispatched.commandId?.trim();
|
|
3581
|
+
if (dispatched.done !== true || !commandId) {
|
|
3582
|
+
const details = {
|
|
3583
|
+
outpost_id: outpostId,
|
|
3584
|
+
lane_id: lane.lane_id,
|
|
3585
|
+
command_id: commandId || null,
|
|
3586
|
+
status: dispatched.status,
|
|
3587
|
+
done: dispatched.done ?? false,
|
|
3588
|
+
exit_code: dispatched.exitCode,
|
|
3589
|
+
outcome_unknown: true,
|
|
3590
|
+
};
|
|
3591
|
+
throw new LanesError("outpost_delivery_unconfirmed", `Outpost ${outpostId} did not confirm pickup for lane ${lane.lane_id}; the queued lane will be reconciled against its owner`, {
|
|
3592
|
+
...details,
|
|
3593
|
+
state: retainQueuedOutpostAfterUnknownDelivery(hqRoot, lane.lane_id, outpostId, commandId || null).state,
|
|
3594
|
+
});
|
|
3595
|
+
}
|
|
3531
3596
|
const stdoutTrimmed = dispatched.stdout.trim();
|
|
3532
3597
|
if (remote === null && stdoutTrimmed !== "" && dispatched.status !== "Success") {
|
|
3533
3598
|
if (preserveQueuedOnFailure) {
|
|
@@ -3606,6 +3671,9 @@ async function dispatchCreatedOutpostLane(ctx, input) {
|
|
|
3606
3671
|
catch (err) {
|
|
3607
3672
|
if (preserveQueuedOnFailure)
|
|
3608
3673
|
throw err;
|
|
3674
|
+
if (isLanesError(err) && err.error === "outpost_delivery_unconfirmed" && err.details.outcome_unknown === true) {
|
|
3675
|
+
throw err;
|
|
3676
|
+
}
|
|
3609
3677
|
if (isLanesError(err) &&
|
|
3610
3678
|
err.error === DISPATCH_TERMINAL_REASON &&
|
|
3611
3679
|
readLane(hqRoot, lane.lane_id).state === "failed") {
|
|
@@ -3792,6 +3860,29 @@ function authorizeResumeProfileRepin(ctx, lane, profile, encodedContext) {
|
|
|
3792
3860
|
: workerProfileHashMismatch(lane, profile);
|
|
3793
3861
|
}
|
|
3794
3862
|
}
|
|
3863
|
+
if (lane.state === "queued" && lane.queued_reason === "worker_profile_changed" && profile !== undefined) {
|
|
3864
|
+
// The hash is for the executing Outpost's profile copy, not the primary
|
|
3865
|
+
// host's copy. Only the lane senior (or its authorized owner dispatch) may
|
|
3866
|
+
// clear this hold after resolving that local copy.
|
|
3867
|
+
if (lane.host.kind !== "outpost" || lane.worker_profile !== profile.id) {
|
|
3868
|
+
throw new LanesError("worker_profile_hash_mismatch", `Queued profile recovery for lane ${lane.lane_id} must use its original Outpost worker profile`, {
|
|
3869
|
+
lane_id: lane.lane_id,
|
|
3870
|
+
worker: profile.id,
|
|
3871
|
+
lane_worker: lane.worker_profile ?? null,
|
|
3872
|
+
outpost: lane.host.kind === "outpost" ? lane.host.id : null,
|
|
3873
|
+
});
|
|
3874
|
+
}
|
|
3875
|
+
if (ssmOwnerDispatch) {
|
|
3876
|
+
if (context === undefined)
|
|
3877
|
+
throw workerProfileHashMismatch(lane, profile);
|
|
3878
|
+
}
|
|
3879
|
+
else {
|
|
3880
|
+
const callerAuthorization = callerSeniorAuthorization(ctx, lane.senior);
|
|
3881
|
+
if (!callerAuthorization.authorized) {
|
|
3882
|
+
throw workerProfileHashMismatch(lane, profile, callerAuthorization);
|
|
3883
|
+
}
|
|
3884
|
+
}
|
|
3885
|
+
}
|
|
3795
3886
|
if (profile === undefined ||
|
|
3796
3887
|
(lane.host.kind !== "outpost" && !(ssmOwnerDispatch && laneGrantTargetHostedHere(ctx, lane))) ||
|
|
3797
3888
|
lane.worker_profile_hash === profile.profile.contentHash) {
|
|
@@ -4059,7 +4150,13 @@ async function dispatchOutpostLaneCommand(ctx, lane, action, options, behavior =
|
|
|
4059
4150
|
remote.lane !== null &&
|
|
4060
4151
|
typeof remote.lane === "object" &&
|
|
4061
4152
|
!Array.isArray(remote.lane) &&
|
|
4062
|
-
remote.lane.state === "queued")
|
|
4153
|
+
remote.lane.state === "queued") ||
|
|
4154
|
+
(remote.delivery_status === "queued_for_reconcile" &&
|
|
4155
|
+
remote.lane !== null &&
|
|
4156
|
+
typeof remote.lane === "object" &&
|
|
4157
|
+
!Array.isArray(remote.lane) &&
|
|
4158
|
+
remote.lane.state === "queued" &&
|
|
4159
|
+
remote.lane.queued_reason !== "worker_profile_changed")));
|
|
4063
4160
|
if (!resultConfirmed) {
|
|
4064
4161
|
throw new LanesError("outpost_delivery_unconfirmed", `Outpost ${outpostId} completed exec without a confirmed ${action} result for lane ${lane.lane_id}`, {
|
|
4065
4162
|
lane_id: lane.lane_id,
|
|
@@ -4522,7 +4619,7 @@ async function runList(ctx, opts, fastRouteFlags) {
|
|
|
4522
4619
|
}
|
|
4523
4620
|
const remoteRead = ctx.env[REMOTE_LANE_READ_ENV] === "1";
|
|
4524
4621
|
const remoteRefresh = remoteRead
|
|
4525
|
-
? { updated: new Map(), unverifiedRemote: new Map() }
|
|
4622
|
+
? { updated: new Map(), unverifiedRemote: new Map(), missingOnOwner: new Map() }
|
|
4526
4623
|
: await refreshRemoteLaneMirrors(ctx, hqRoot, laneSource, new Set(selectedLanes
|
|
4527
4624
|
.filter((lane) => !isTerminalLaneState(lane.state))
|
|
4528
4625
|
.map((lane) => lane.lane_id)));
|
|
@@ -5853,6 +5950,7 @@ function recordResumeSetup(ctx, lane, input) {
|
|
|
5853
5950
|
input.keepAlive === undefined &&
|
|
5854
5951
|
input.cliModel === undefined &&
|
|
5855
5952
|
input.meshCompany === undefined &&
|
|
5953
|
+
input.clearWorkerProfileChangedQueue !== true &&
|
|
5856
5954
|
lane.keep_alive_blocked_reason !== "terminal_state")
|
|
5857
5955
|
return lane;
|
|
5858
5956
|
return casUpdateLane(ctx.hqRoot(), lane.lane_id, (current) => {
|
|
@@ -5862,12 +5960,28 @@ function recordResumeSetup(ctx, lane, input) {
|
|
|
5862
5960
|
if (input.profileRepin !== undefined) {
|
|
5863
5961
|
assertResumeProfileRepinStillAuthorized(ctx, current, input.profileRepin);
|
|
5864
5962
|
}
|
|
5963
|
+
if (input.clearWorkerProfileChangedQueue === true && (current.state !== "queued" ||
|
|
5964
|
+
current.queued_reason !== "worker_profile_changed" ||
|
|
5965
|
+
input.profile === undefined ||
|
|
5966
|
+
current.worker_profile !== input.profile.id ||
|
|
5967
|
+
(current.worker_profile_hash !== input.profile.profile.contentHash && input.profileRepin === undefined))) {
|
|
5968
|
+
throw new LanesError("lane_resume_setup_conflict", `Lane ${lane.lane_id} is no longer eligible for worker-profile recovery; run the recovery command again`, {
|
|
5969
|
+
lane_id: lane.lane_id,
|
|
5970
|
+
state: current.state,
|
|
5971
|
+
queued_reason: current.queued_reason ?? null,
|
|
5972
|
+
});
|
|
5973
|
+
}
|
|
5865
5974
|
const next = { ...current };
|
|
5866
5975
|
// An operator resume takes the lane out of the terminal state reconcile
|
|
5867
5976
|
// recorded, so keep-alive may continue it again after this turn.
|
|
5868
5977
|
if (next.keep_alive_blocked_reason === "terminal_state")
|
|
5869
5978
|
next.keep_alive_blocked_reason = undefined;
|
|
5870
5979
|
applyResumeProfileAndEffort(current, next, { ...input, at: ctx.now().toISOString() });
|
|
5980
|
+
if (input.clearWorkerProfileChangedQueue === true) {
|
|
5981
|
+
next.queued_reason = undefined;
|
|
5982
|
+
next.queued_details = undefined;
|
|
5983
|
+
next.admission_reserved_at = undefined;
|
|
5984
|
+
}
|
|
5871
5985
|
if (input.cliModel !== undefined) {
|
|
5872
5986
|
next.model = input.cliModel.trim();
|
|
5873
5987
|
next.model_source = "cli";
|
|
@@ -6989,11 +7103,21 @@ async function readOutpostTerminalLane(ctx, laneId, outpostId, stopCommandId, wa
|
|
|
6989
7103
|
commandId,
|
|
6990
7104
|
};
|
|
6991
7105
|
}
|
|
7106
|
+
function isDefiniteOutpostLaneNotFound(dispatched, laneId, ownerError) {
|
|
7107
|
+
return dispatched.done === true &&
|
|
7108
|
+
Boolean(dispatched.commandId?.trim()) &&
|
|
7109
|
+
dispatched.status === "Failed" &&
|
|
7110
|
+
dispatched.exitCode !== null &&
|
|
7111
|
+
dispatched.exitCode !== 0 &&
|
|
7112
|
+
ownerError?.error === "lane_not_found" &&
|
|
7113
|
+
ownerError.details.lane_id === laneId;
|
|
7114
|
+
}
|
|
6992
7115
|
async function dispatchOutpostLaneStop(ctx, lane, reason, quiet = false) {
|
|
6993
7116
|
if (lane.host.kind !== "outpost") {
|
|
6994
7117
|
throw new LanesError("outpost_dispatch_failed", `Lane ${lane.lane_id} has no Outpost host to dispatch stop to`, { lane_id: lane.lane_id });
|
|
6995
7118
|
}
|
|
6996
7119
|
const outpostId = lane.host.id;
|
|
7120
|
+
const hqRoot = ctx.hqRoot();
|
|
6997
7121
|
const waitMs = outpostSingleLaneWaitMs(ctx.env);
|
|
6998
7122
|
let dispatched;
|
|
6999
7123
|
try {
|
|
@@ -7017,7 +7141,36 @@ async function dispatchOutpostLaneStop(ctx, lane, reason, quiet = false) {
|
|
|
7017
7141
|
required_option: "--reason",
|
|
7018
7142
|
});
|
|
7019
7143
|
}
|
|
7144
|
+
const commandId = dispatched.commandId?.trim();
|
|
7020
7145
|
const ownerError = ownerErrorFromOutput(dispatched.stdout);
|
|
7146
|
+
if (isDefiniteOutpostLaneNotFound(dispatched, lane.lane_id, ownerError)) {
|
|
7147
|
+
const at = ctx.now().toISOString();
|
|
7148
|
+
const updated = markRemoteLaneLost(ctx.hqRoot(), lane.lane_id, outpostId, at, "command", reason);
|
|
7149
|
+
if (updated.terminal_reason !== "remote_lost") {
|
|
7150
|
+
throw new LanesError("already_terminal", `Lane ${lane.lane_id} became terminal before its missing Outpost record could be marked remote_lost`, { lane_id: lane.lane_id, state: updated.state });
|
|
7151
|
+
}
|
|
7152
|
+
const cancelled = cancelPendingQuestionsForLane(hqRoot, lane.lane_id, at, "remote_lost");
|
|
7153
|
+
await emitLaneTerminalStateSoft(ctx, updated);
|
|
7154
|
+
if (!quiet)
|
|
7155
|
+
printJson({
|
|
7156
|
+
ok: true,
|
|
7157
|
+
action: "stop",
|
|
7158
|
+
reason: "remote_lost",
|
|
7159
|
+
lane: updated,
|
|
7160
|
+
terminal: true,
|
|
7161
|
+
resumable: false,
|
|
7162
|
+
cancelled_questions: cancelled.map((question) => question.question_id),
|
|
7163
|
+
outpost_id: outpostId,
|
|
7164
|
+
outpost_command_id: commandId,
|
|
7165
|
+
outpost_result: {
|
|
7166
|
+
state: "failed",
|
|
7167
|
+
reason: "remote_lost",
|
|
7168
|
+
status: dispatched.status,
|
|
7169
|
+
exit_code: dispatched.exitCode,
|
|
7170
|
+
},
|
|
7171
|
+
});
|
|
7172
|
+
return;
|
|
7173
|
+
}
|
|
7021
7174
|
if (ownerError) {
|
|
7022
7175
|
throw new LanesError("outpost_dispatch_failed", `Outpost ${outpostId} rejected the stop command: ${ownerError.message}`, {
|
|
7023
7176
|
lane_id: lane.lane_id,
|
|
@@ -7029,7 +7182,6 @@ async function dispatchOutpostLaneStop(ctx, lane, reason, quiet = false) {
|
|
|
7029
7182
|
owner_error: ownerError,
|
|
7030
7183
|
});
|
|
7031
7184
|
}
|
|
7032
|
-
const commandId = dispatched.commandId?.trim();
|
|
7033
7185
|
if (dispatched.done !== true || !commandId) {
|
|
7034
7186
|
throw new LanesError("outpost_delivery_unconfirmed", `Outpost ${outpostId} did not return a completed stop command for lane ${lane.lane_id}`, {
|
|
7035
7187
|
lane_id: lane.lane_id,
|
|
@@ -7513,8 +7665,42 @@ async function runReconcile(ctx, opts, recoverQueuedMessages) {
|
|
|
7513
7665
|
const fromEnv = ctx.env.HQ_ROOT?.trim();
|
|
7514
7666
|
const hqRoot = fromEnv && ctx.existsSync(fromEnv) ? fromEnv : ctx.hqRoot();
|
|
7515
7667
|
const mirrorRefresh = opts.dryRun
|
|
7516
|
-
? {
|
|
7668
|
+
? {
|
|
7669
|
+
updated: new Map(),
|
|
7670
|
+
unverifiedRemote: new Map(),
|
|
7671
|
+
missingOnOwner: new Map(),
|
|
7672
|
+
}
|
|
7517
7673
|
: await refreshRemoteLaneMirrors(ctx, hqRoot, loadAllLanes(hqRoot));
|
|
7674
|
+
const remoteLost = [];
|
|
7675
|
+
const remoteLostErrors = [];
|
|
7676
|
+
if (!opts.dryRun && mirrorRefresh.missingOnOwner.size > 0) {
|
|
7677
|
+
const at = ctx.now().toISOString();
|
|
7678
|
+
const nowMs = Date.parse(at);
|
|
7679
|
+
for (const [laneId, mirror] of mirrorRefresh.missingOnOwner) {
|
|
7680
|
+
const createdAtMs = Date.parse(mirror.created_at);
|
|
7681
|
+
if (!Number.isFinite(createdAtMs) ||
|
|
7682
|
+
nowMs - createdAtMs < REMOTE_LANE_NOT_FOUND_GRACE_MS)
|
|
7683
|
+
continue;
|
|
7684
|
+
try {
|
|
7685
|
+
const updated = markRemoteLaneLost(hqRoot, laneId, mirror.host.id, at, "reconcile", undefined, mirror.version);
|
|
7686
|
+
if (updated.terminal_reason !== "remote_lost") {
|
|
7687
|
+
if (updated.version !== mirror.version) {
|
|
7688
|
+
mirrorRefresh.unverifiedRemote.set(laneId, "mirror changed after the owner snapshot");
|
|
7689
|
+
}
|
|
7690
|
+
continue;
|
|
7691
|
+
}
|
|
7692
|
+
remoteLost.push({ id: laneId, reason: "remote_lost" });
|
|
7693
|
+
cancelPendingQuestionsForLane(hqRoot, laneId, at, "remote_lost");
|
|
7694
|
+
await emitLaneTerminalStateSoft(ctx, updated);
|
|
7695
|
+
}
|
|
7696
|
+
catch (err) {
|
|
7697
|
+
remoteLostErrors.push({
|
|
7698
|
+
laneId,
|
|
7699
|
+
message: err instanceof Error ? err.message : String(err),
|
|
7700
|
+
});
|
|
7701
|
+
}
|
|
7702
|
+
}
|
|
7703
|
+
}
|
|
7518
7704
|
const summary = await runReconcilePass(hqRoot, {
|
|
7519
7705
|
now: ctx.now,
|
|
7520
7706
|
host: reconcileHostFromCtx(ctx),
|
|
@@ -7630,10 +7816,22 @@ async function runReconcile(ctx, opts, recoverQueuedMessages) {
|
|
|
7630
7816
|
}
|
|
7631
7817
|
},
|
|
7632
7818
|
});
|
|
7819
|
+
for (const failure of remoteLostErrors) {
|
|
7820
|
+
summary.errors.push({
|
|
7821
|
+
code: "remote_lane_terminalize_failed",
|
|
7822
|
+
message: `Could not mark remote lane ${failure.laneId} lost: ${failure.message}`,
|
|
7823
|
+
});
|
|
7824
|
+
summary.ok = false;
|
|
7825
|
+
}
|
|
7826
|
+
const terminalizedRemoteIds = new Set(remoteLost.map(({ id }) => id));
|
|
7633
7827
|
printJson({
|
|
7634
7828
|
...summary,
|
|
7829
|
+
...(remoteLost.length > 0 ? { remote_lost: remoteLost } : {}),
|
|
7635
7830
|
...(mirrorRefresh.unverifiedRemote.size > 0
|
|
7636
|
-
? {
|
|
7831
|
+
? {
|
|
7832
|
+
unverified_remote_lanes: [...mirrorRefresh.unverifiedRemote.keys()]
|
|
7833
|
+
.filter((id) => !terminalizedRemoteIds.has(id)),
|
|
7834
|
+
}
|
|
7637
7835
|
: {}),
|
|
7638
7836
|
});
|
|
7639
7837
|
if (!summary.ok) {
|
|
@@ -1,2 +1,14 @@
|
|
|
1
|
+
import type { LaneDrainHookInput } from "./lib/lanes/dropbox.js";
|
|
2
|
+
interface DrainHookResult {
|
|
3
|
+
exitCode: number;
|
|
4
|
+
stdout: string;
|
|
5
|
+
stderr: string;
|
|
6
|
+
}
|
|
7
|
+
type DropboxModule = typeof import("./lib/lanes/dropbox.js");
|
|
8
|
+
export interface DrainHookRunnerDeps {
|
|
9
|
+
/** Test seam for proving the steady-state path does not load the drainer. */
|
|
10
|
+
loadDropbox?: () => Promise<Pick<DropboxModule, "runDrainHook">>;
|
|
11
|
+
}
|
|
12
|
+
export declare function runLaneDrainHook(env: NodeJS.ProcessEnv, input?: LaneDrainHookInput, inputError?: string, deps?: DrainHookRunnerDeps): Promise<DrainHookResult>;
|
|
1
13
|
export {};
|
|
2
14
|
//# sourceMappingURL=lanes-drain-hook.d.ts.map
|
package/dist/lanes-drain-hook.js
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { randomBytes } from "node:crypto";
|
|
2
2
|
import * as fs from "node:fs";
|
|
3
3
|
import * as path from "node:path";
|
|
4
|
+
import { fileURLToPath } from "node:url";
|
|
5
|
+
import { MESSAGE_REPLAY_STATE_NAME, readMessageReplayState, updateMessageReplayTranscript, writeMessageReplayState, } from "./lib/lanes/message-replay.js";
|
|
4
6
|
function parseHookInput(text) {
|
|
5
7
|
if (text.trim() === "")
|
|
6
8
|
return {};
|
|
@@ -72,38 +74,34 @@ function dropboxMayNeedDrain(box) {
|
|
|
72
74
|
}
|
|
73
75
|
return false;
|
|
74
76
|
}
|
|
75
|
-
function replayMayNeedWork(box, turnId, input) {
|
|
77
|
+
function replayMayNeedWork(box, turnId, input, env) {
|
|
76
78
|
if (!/^[A-Za-z0-9-]{1,80}$/.test(turnId))
|
|
77
79
|
return false;
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
let
|
|
81
|
-
const statePath = path.join(box, "message-compaction-replay.json");
|
|
80
|
+
const statePath = path.join(box, MESSAGE_REPLAY_STATE_NAME);
|
|
81
|
+
let state;
|
|
82
|
+
let hasLegacyFlag = false;
|
|
82
83
|
try {
|
|
83
|
-
|
|
84
|
+
state = readMessageReplayState(statePath, turnId, () => {
|
|
85
|
+
hasLegacyFlag = true;
|
|
86
|
+
});
|
|
84
87
|
}
|
|
85
88
|
catch {
|
|
86
89
|
// Keep the existing named replay error behavior for missing or unreadable state.
|
|
87
90
|
return true;
|
|
88
91
|
}
|
|
92
|
+
if (hasLegacyFlag || !input.transcript_path)
|
|
93
|
+
return hasLegacyFlag;
|
|
94
|
+
if (!state.provider)
|
|
95
|
+
return true;
|
|
89
96
|
try {
|
|
90
|
-
const
|
|
91
|
-
if (
|
|
97
|
+
const update = updateMessageReplayTranscript(state, input.transcript_path, state.provider, env);
|
|
98
|
+
if (update === "compacted")
|
|
92
99
|
return true;
|
|
93
|
-
|
|
94
|
-
if (state.schema_version !== 1 ||
|
|
95
|
-
state.turn_id !== turnId ||
|
|
96
|
-
(state.last_compaction_marker !== null &&
|
|
97
|
-
(!Number.isSafeInteger(state.last_compaction_marker) || Number(state.last_compaction_marker) < 0)) ||
|
|
98
|
-
(state.transcript_offset !== undefined &&
|
|
99
|
-
(!Number.isSafeInteger(state.transcript_offset) || Number(state.transcript_offset) < 0)) ||
|
|
100
|
-
(state.transcript_path !== undefined && typeof state.transcript_path !== "string") ||
|
|
101
|
-
state.flag_enabled !== undefined) {
|
|
102
|
-
return true;
|
|
103
|
-
}
|
|
100
|
+
writeMessageReplayState(statePath, state);
|
|
104
101
|
return false;
|
|
105
102
|
}
|
|
106
103
|
catch {
|
|
104
|
+
// The full path preserves the existing named replay error reporting.
|
|
107
105
|
return true;
|
|
108
106
|
}
|
|
109
107
|
}
|
|
@@ -125,52 +123,42 @@ function recordToolActivity(hqRoot, laneId) {
|
|
|
125
123
|
return `hq-lanes-drain: failed to record tool activity: ${err instanceof Error ? err.message : String(err)}\n`;
|
|
126
124
|
}
|
|
127
125
|
}
|
|
128
|
-
function finish(result
|
|
126
|
+
function finish(result) {
|
|
129
127
|
if (result.stdout)
|
|
130
128
|
process.stdout.write(result.stdout);
|
|
131
|
-
if (
|
|
132
|
-
process.stderr.write(
|
|
129
|
+
if (result.stderr)
|
|
130
|
+
process.stderr.write(result.stderr);
|
|
133
131
|
process.exitCode = result.exitCode;
|
|
134
132
|
}
|
|
135
|
-
async function runFullPath(input,
|
|
136
|
-
const
|
|
137
|
-
|
|
133
|
+
async function runFullPath(env, input, deps) {
|
|
134
|
+
const loadDropbox = deps.loadDropbox ?? (() => import("./lib/lanes/dropbox.js"));
|
|
135
|
+
const { runDrainHook } = await loadDropbox();
|
|
136
|
+
return runDrainHook(env, input);
|
|
138
137
|
}
|
|
139
|
-
async function
|
|
140
|
-
const
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
let inputError = "";
|
|
146
|
-
try {
|
|
147
|
-
input = parseHookInput(text);
|
|
148
|
-
}
|
|
149
|
-
catch (err) {
|
|
150
|
-
inputError = `hq-lanes-drain: ${err instanceof Error ? err.message : String(err)}\n`;
|
|
151
|
-
}
|
|
152
|
-
const laneId = process.env.HQ_LANE_ID ?? "";
|
|
138
|
+
export async function runLaneDrainHook(env, input = {}, inputError = "", deps = {}) {
|
|
139
|
+
const withInputError = (result) => ({
|
|
140
|
+
...result,
|
|
141
|
+
stderr: `${inputError}${result.stderr}`,
|
|
142
|
+
});
|
|
143
|
+
const laneId = env.HQ_LANE_ID ?? "";
|
|
153
144
|
if (laneId === "") {
|
|
154
|
-
|
|
155
|
-
return;
|
|
145
|
+
return withInputError({ exitCode: 0, stdout: "", stderr: "" });
|
|
156
146
|
}
|
|
157
|
-
const hqRoot =
|
|
147
|
+
const hqRoot = env.HQ_ROOT ?? "";
|
|
158
148
|
if (hqRoot === "" || !laneIdIsSafe(laneId)) {
|
|
159
|
-
await runFullPath(input,
|
|
160
|
-
return;
|
|
149
|
+
return withInputError(await runFullPath(env, input, deps));
|
|
161
150
|
}
|
|
162
151
|
const box = path.join(hqRoot, "workspace", "lanes-runs", laneId, "dropbox");
|
|
163
152
|
let noWork = false;
|
|
164
153
|
try {
|
|
165
154
|
noWork = !dropboxMayNeedDrain(box) &&
|
|
166
|
-
!replayMayNeedWork(box,
|
|
155
|
+
!replayMayNeedWork(box, env.HQ_LANE_TURN_ID ?? "", input, env);
|
|
167
156
|
}
|
|
168
157
|
catch {
|
|
169
158
|
// Filesystem uncertainty must use the full path so it retains its named errors.
|
|
170
159
|
}
|
|
171
160
|
if (!noWork) {
|
|
172
|
-
await runFullPath(input,
|
|
173
|
-
return;
|
|
161
|
+
return withInputError(await runFullPath(env, input, deps));
|
|
174
162
|
}
|
|
175
163
|
let stderr = recordToolActivity(hqRoot, laneId);
|
|
176
164
|
const directories = [
|
|
@@ -186,14 +174,30 @@ async function main() {
|
|
|
186
174
|
catch (err) {
|
|
187
175
|
const message = `Failed to open drop box ${box}: ${err instanceof Error ? err.message : String(err)}`;
|
|
188
176
|
stderr += `hq-lanes-drain: ${message}\n`;
|
|
189
|
-
|
|
190
|
-
|
|
177
|
+
return withInputError({ exitCode: 1, stdout: "", stderr });
|
|
178
|
+
}
|
|
179
|
+
return withInputError({ exitCode: 0, stdout: "", stderr });
|
|
180
|
+
}
|
|
181
|
+
async function main() {
|
|
182
|
+
const chunks = [];
|
|
183
|
+
for await (const chunk of process.stdin)
|
|
184
|
+
chunks.push(Buffer.from(chunk));
|
|
185
|
+
const text = Buffer.concat(chunks).toString("utf8");
|
|
186
|
+
let input = {};
|
|
187
|
+
let inputError = "";
|
|
188
|
+
try {
|
|
189
|
+
input = parseHookInput(text);
|
|
191
190
|
}
|
|
192
|
-
|
|
191
|
+
catch (err) {
|
|
192
|
+
inputError = `hq-lanes-drain: ${err instanceof Error ? err.message : String(err)}\n`;
|
|
193
|
+
}
|
|
194
|
+
finish(await runLaneDrainHook(process.env, input, inputError));
|
|
195
|
+
}
|
|
196
|
+
if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
|
197
|
+
main().catch((err) => {
|
|
198
|
+
process.stderr.write(`hq-lanes-drain: hook_runner_failed: ${err instanceof Error ? err.message : String(err)}\n`);
|
|
199
|
+
// A hook runner failure must not stop the provider's requested tool call.
|
|
200
|
+
process.exitCode = 0;
|
|
201
|
+
});
|
|
193
202
|
}
|
|
194
|
-
main().catch((err) => {
|
|
195
|
-
process.stderr.write(`hq-lanes-drain: hook_runner_failed: ${err instanceof Error ? err.message : String(err)}\n`);
|
|
196
|
-
// A hook runner failure must not stop the provider's requested tool call.
|
|
197
|
-
process.exitCode = 0;
|
|
198
|
-
});
|
|
199
203
|
//# sourceMappingURL=lanes-drain-hook.js.map
|