@indigoai-us/hq-cli 5.233.0 → 5.234.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +13 -0
- package/dist/command-catalog.generated.d.ts +10 -1
- package/dist/command-catalog.generated.js +13 -1
- package/dist/commands/agents.js +10 -2
- package/dist/commands/lanes.js +146 -3
- package/dist/lib/lanes/delivery.d.ts +9 -3
- package/dist/lib/lanes/delivery.js +11 -6
- package/dist/lib/lanes/outpost-dispatch.d.ts +2 -0
- package/dist/lib/lanes/outpost-dispatch.js +3 -0
- package/dist/lib/lanes/spawn.js +20 -0
- package/dist/lib/lanes/store.js +5 -0
- package/dist/lib/lanes/types.d.ts +12 -0
- package/dist/lib/lanes/types.js +1 -1
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,19 @@
|
|
|
2
2
|
|
|
3
3
|
## [Unreleased]
|
|
4
4
|
|
|
5
|
+
## [5.234.0] — 2026-09-27
|
|
6
|
+
|
|
7
|
+
### Added
|
|
8
|
+
|
|
9
|
+
- A lane's senior can explicitly resume a failed lane's recorded provider
|
|
10
|
+
session with `hq lanes resume --from-failed --reason`; the secret-scanned
|
|
11
|
+
reason and recovered failure are recorded in lane history. Failed lanes
|
|
12
|
+
continue to refuse ordinary messages. (#927)
|
|
13
|
+
|
|
14
|
+
### Fixed
|
|
15
|
+
|
|
16
|
+
- `hq agents stop` says the stop was accepted and shows the status the API returns, instead of saying the agent has already stopped. (#922)
|
|
17
|
+
|
|
5
18
|
## [5.233.0] — 2026-09-27
|
|
6
19
|
|
|
7
20
|
### Added
|
|
@@ -8047,7 +8047,7 @@ export declare const COMMAND_CATALOG: readonly [{
|
|
|
8047
8047
|
readonly subcommands: readonly [];
|
|
8048
8048
|
}, {
|
|
8049
8049
|
readonly name: "resume";
|
|
8050
|
-
readonly description: "Resume an interrupted, done or blocked lane on its recorded provider session";
|
|
8050
|
+
readonly description: "Resume an interrupted, done or blocked lane on its recorded provider session; recover failed lanes with the senior-only --from-failed option";
|
|
8051
8051
|
readonly aliases: readonly [];
|
|
8052
8052
|
readonly hidden: false;
|
|
8053
8053
|
readonly usage: "[options] <lane-id>";
|
|
@@ -8059,6 +8059,12 @@ export declare const COMMAND_CATALOG: readonly [{
|
|
|
8059
8059
|
readonly options: readonly [{
|
|
8060
8060
|
readonly flags: "--text <msg>";
|
|
8061
8061
|
readonly description: "Operator message for the resumed turn";
|
|
8062
|
+
}, {
|
|
8063
|
+
readonly flags: "--from-failed";
|
|
8064
|
+
readonly description: "Explicitly resume a failed lane as its senior";
|
|
8065
|
+
}, {
|
|
8066
|
+
readonly flags: "--reason <text>";
|
|
8067
|
+
readonly description: "Required explanation for resuming from failed";
|
|
8062
8068
|
}, {
|
|
8063
8069
|
readonly flags: "--project <slug>";
|
|
8064
8070
|
readonly description: "Re-point project attribution from this resume";
|
|
@@ -8080,6 +8086,9 @@ export declare const COMMAND_CATALOG: readonly [{
|
|
|
8080
8086
|
}, {
|
|
8081
8087
|
readonly flags: "--grant-context <base64url>";
|
|
8082
8088
|
readonly description: "";
|
|
8089
|
+
}, {
|
|
8090
|
+
readonly flags: "--failed-resume-context <base64url>";
|
|
8091
|
+
readonly description: "";
|
|
8083
8092
|
}, {
|
|
8084
8093
|
readonly flags: "--keep-alive";
|
|
8085
8094
|
readonly description: "Automatically resume done lead lanes while their story remains open";
|
|
@@ -10408,7 +10408,7 @@ export const COMMAND_CATALOG = [
|
|
|
10408
10408
|
},
|
|
10409
10409
|
{
|
|
10410
10410
|
"name": "resume",
|
|
10411
|
-
"description": "Resume an interrupted, done or blocked lane on its recorded provider session",
|
|
10411
|
+
"description": "Resume an interrupted, done or blocked lane on its recorded provider session; recover failed lanes with the senior-only --from-failed option",
|
|
10412
10412
|
"aliases": [],
|
|
10413
10413
|
"hidden": false,
|
|
10414
10414
|
"usage": "[options] <lane-id>",
|
|
@@ -10424,6 +10424,14 @@ export const COMMAND_CATALOG = [
|
|
|
10424
10424
|
"flags": "--text <msg>",
|
|
10425
10425
|
"description": "Operator message for the resumed turn"
|
|
10426
10426
|
},
|
|
10427
|
+
{
|
|
10428
|
+
"flags": "--from-failed",
|
|
10429
|
+
"description": "Explicitly resume a failed lane as its senior"
|
|
10430
|
+
},
|
|
10431
|
+
{
|
|
10432
|
+
"flags": "--reason <text>",
|
|
10433
|
+
"description": "Required explanation for resuming from failed"
|
|
10434
|
+
},
|
|
10427
10435
|
{
|
|
10428
10436
|
"flags": "--project <slug>",
|
|
10429
10437
|
"description": "Re-point project attribution from this resume"
|
|
@@ -10452,6 +10460,10 @@ export const COMMAND_CATALOG = [
|
|
|
10452
10460
|
"flags": "--grant-context <base64url>",
|
|
10453
10461
|
"description": ""
|
|
10454
10462
|
},
|
|
10463
|
+
{
|
|
10464
|
+
"flags": "--failed-resume-context <base64url>",
|
|
10465
|
+
"description": ""
|
|
10466
|
+
},
|
|
10455
10467
|
{
|
|
10456
10468
|
"flags": "--keep-alive",
|
|
10457
10469
|
"description": "Automatically resume done lead lanes while their story remains open"
|
package/dist/commands/agents.js
CHANGED
|
@@ -1819,8 +1819,16 @@ export function registerAgentsCommand(program) {
|
|
|
1819
1819
|
.action(async function (agentUid) {
|
|
1820
1820
|
try {
|
|
1821
1821
|
const token = (await resolveVaultCredential()).token;
|
|
1822
|
-
await startStopAgent(token, agentUid, action);
|
|
1823
|
-
|
|
1822
|
+
const result = await startStopAgent(token, agentUid, action);
|
|
1823
|
+
const runtimeStatus = result.runtime?.status;
|
|
1824
|
+
const successMessage = action === "start"
|
|
1825
|
+
? `Started agent ${agentUid}.`
|
|
1826
|
+
: runtimeStatus === "terminated"
|
|
1827
|
+
? `Stopped agent ${agentUid}.`
|
|
1828
|
+
: typeof runtimeStatus === "string"
|
|
1829
|
+
? `Stop accepted for agent ${agentUid} (status: ${runtimeStatus}).`
|
|
1830
|
+
: `Stop accepted for agent ${agentUid}.`;
|
|
1831
|
+
console.log(chalk.green(successMessage));
|
|
1824
1832
|
}
|
|
1825
1833
|
catch (err) {
|
|
1826
1834
|
fail(err);
|
package/dist/commands/lanes.js
CHANGED
|
@@ -3626,6 +3626,10 @@ function outpostRejectedGrantContextOption(result) {
|
|
|
3626
3626
|
return result.exitCode !== 0 &&
|
|
3627
3627
|
/unknown option\s+(?:['"])?--grant-context(?:['"])?(?:\s|$)/i.test(`${result.stderr}\n${result.stdout}`);
|
|
3628
3628
|
}
|
|
3629
|
+
function outpostRejectedFailedResumeOption(result) {
|
|
3630
|
+
return result.exitCode !== 0 &&
|
|
3631
|
+
/unknown option\s+(?:['"])?--(?:from-failed|failed-resume-context)(?:['"])?(?:\s|$)/i.test(`${result.stderr}\n${result.stdout}`);
|
|
3632
|
+
}
|
|
3629
3633
|
function verifyOutpostProfileContent(hqRoot, lane) {
|
|
3630
3634
|
if (lane.host.kind !== "outpost" || lane.worker_profile === undefined)
|
|
3631
3635
|
return;
|
|
@@ -3648,6 +3652,70 @@ function resumeProfileRepinActor(ctx) {
|
|
|
3648
3652
|
const sessionId = sessionIdFromEnv(ctx.env);
|
|
3649
3653
|
return sessionId ? `session:${sessionId}` : "ssm-owner-dispatch";
|
|
3650
3654
|
}
|
|
3655
|
+
function createOutpostFailedResumeContext(ctx, lane) {
|
|
3656
|
+
return Buffer.from(JSON.stringify({
|
|
3657
|
+
version: 1,
|
|
3658
|
+
lane_id: lane.lane_id,
|
|
3659
|
+
expected_senior: lane.senior,
|
|
3660
|
+
actor: resumeProfileRepinActor(ctx),
|
|
3661
|
+
})).toString("base64url");
|
|
3662
|
+
}
|
|
3663
|
+
function parseOutpostFailedResumeContext(encoded, laneId) {
|
|
3664
|
+
try {
|
|
3665
|
+
if (!/^[A-Za-z0-9_-]{1,4096}$/.test(encoded))
|
|
3666
|
+
throw new Error("invalid encoding");
|
|
3667
|
+
const value = JSON.parse(Buffer.from(encoded, "base64url").toString("utf8"));
|
|
3668
|
+
if (value === null || typeof value !== "object" || Array.isArray(value))
|
|
3669
|
+
throw new Error("invalid payload");
|
|
3670
|
+
const payload = value;
|
|
3671
|
+
const seniorValue = payload.expected_senior;
|
|
3672
|
+
if (seniorValue === null || typeof seniorValue !== "object" || Array.isArray(seniorValue))
|
|
3673
|
+
throw new Error("invalid senior");
|
|
3674
|
+
const senior = seniorValue;
|
|
3675
|
+
let expectedSenior;
|
|
3676
|
+
if (senior.kind === "user") {
|
|
3677
|
+
expectedSenior = { kind: "user" };
|
|
3678
|
+
}
|
|
3679
|
+
else if ((senior.kind === "lane" || senior.kind === "session") &&
|
|
3680
|
+
typeof senior.id === "string" && senior.id.trim().length > 0) {
|
|
3681
|
+
expectedSenior = { kind: senior.kind, id: senior.id };
|
|
3682
|
+
}
|
|
3683
|
+
else {
|
|
3684
|
+
throw new Error("invalid senior");
|
|
3685
|
+
}
|
|
3686
|
+
if (payload.version !== 1 ||
|
|
3687
|
+
payload.lane_id !== laneId ||
|
|
3688
|
+
typeof payload.actor !== "string" ||
|
|
3689
|
+
payload.actor.trim().length === 0) {
|
|
3690
|
+
throw new Error("invalid payload");
|
|
3691
|
+
}
|
|
3692
|
+
return { expectedSenior, actor: payload.actor };
|
|
3693
|
+
}
|
|
3694
|
+
catch {
|
|
3695
|
+
throw new LanesError("failed_resume_authorization_invalid", "Failed-resume authorization context is invalid", { lane_id: laneId });
|
|
3696
|
+
}
|
|
3697
|
+
}
|
|
3698
|
+
function authorizeFailedResume(ctx, lane, encodedContext) {
|
|
3699
|
+
if (encodedContext !== undefined) {
|
|
3700
|
+
if (!ctx.isSsmOwnerDispatch()) {
|
|
3701
|
+
throw new LanesError("failed_resume_authorization_invalid", "Failed-resume authorization context is only valid on an Outpost owner dispatch", { lane_id: lane.lane_id });
|
|
3702
|
+
}
|
|
3703
|
+
const context = parseOutpostFailedResumeContext(encodedContext, lane.lane_id);
|
|
3704
|
+
if (lane.host.kind !== "outpost" ||
|
|
3705
|
+
!sameHireGrantSenior(lane.senior, context.expectedSenior)) {
|
|
3706
|
+
throw new LanesError("failed_resume_senior_changed", `Lane ${lane.lane_id}'s senior changed after its failed-resume request was authorized`, {
|
|
3707
|
+
lane_id: lane.lane_id,
|
|
3708
|
+
expected_senior: context.expectedSenior,
|
|
3709
|
+
current_senior: lane.senior,
|
|
3710
|
+
});
|
|
3711
|
+
}
|
|
3712
|
+
return context.actor;
|
|
3713
|
+
}
|
|
3714
|
+
if (!callerMatchesSenior(ctx, lane.senior)) {
|
|
3715
|
+
throw new LanesError("failed_resume_unauthorized", "Only the lane's senior may resume a failed lane with --from-failed", { lane_id: lane.lane_id });
|
|
3716
|
+
}
|
|
3717
|
+
return resumeProfileRepinActor(ctx);
|
|
3718
|
+
}
|
|
3651
3719
|
function workerProfileHashMismatch(lane, profile) {
|
|
3652
3720
|
return new LanesError("worker_profile_hash_mismatch", `Worker profile ${JSON.stringify(profile.id)} on Outpost ${lane.host.id} does not match the hash recorded by the lane`, {
|
|
3653
3721
|
lane_id: lane.lane_id,
|
|
@@ -3757,6 +3825,9 @@ async function dispatchOutpostLaneCommand(ctx, lane, action, options) {
|
|
|
3757
3825
|
actor: resumeProfileRepinActor(ctx),
|
|
3758
3826
|
})
|
|
3759
3827
|
: undefined;
|
|
3828
|
+
const failedResumeContext = action === "resume" && options.fromFailed === true
|
|
3829
|
+
? createOutpostFailedResumeContext(ctx, lane)
|
|
3830
|
+
: undefined;
|
|
3760
3831
|
let grantContext;
|
|
3761
3832
|
let grantHierarchy;
|
|
3762
3833
|
if (action === "grant") {
|
|
@@ -3776,6 +3847,7 @@ async function dispatchOutpostLaneCommand(ctx, lane, action, options) {
|
|
|
3776
3847
|
}
|
|
3777
3848
|
let dispatched;
|
|
3778
3849
|
let grantContextRejected = false;
|
|
3850
|
+
let failedResumeOptionRejected = false;
|
|
3779
3851
|
try {
|
|
3780
3852
|
const dispatchCommand = (contexts) => ctx.dispatchOutpostExec({
|
|
3781
3853
|
outpostId,
|
|
@@ -3797,15 +3869,17 @@ async function dispatchOutpostLaneCommand(ctx, lane, action, options) {
|
|
|
3797
3869
|
effort: options.effort,
|
|
3798
3870
|
worker: options.worker,
|
|
3799
3871
|
keepAlive: options.keepAlive,
|
|
3872
|
+
fromFailed: options.fromFailed,
|
|
3800
3873
|
hire: options.hire,
|
|
3801
3874
|
senior: options.senior,
|
|
3802
3875
|
reason: options.reason,
|
|
3803
3876
|
grantContext: contexts.grantContext,
|
|
3804
3877
|
profileRepinContext: contexts.profileRepinContext,
|
|
3878
|
+
failedResumeContext: contexts.failedResumeContext,
|
|
3805
3879
|
compact: options.compact,
|
|
3806
3880
|
}),
|
|
3807
3881
|
});
|
|
3808
|
-
let contexts = { grantContext, profileRepinContext };
|
|
3882
|
+
let contexts = { grantContext, profileRepinContext, failedResumeContext };
|
|
3809
3883
|
dispatched = await dispatchCommand(contexts);
|
|
3810
3884
|
for (let retry = 0; action === "resume" && retry < 2; retry += 1) {
|
|
3811
3885
|
if (contexts.grantContext !== undefined && outpostRejectedGrantContextOption(dispatched)) {
|
|
@@ -3816,6 +3890,10 @@ async function dispatchOutpostLaneCommand(ctx, lane, action, options) {
|
|
|
3816
3890
|
outpostRejectedProfileRepinOption(dispatched)) {
|
|
3817
3891
|
contexts = { ...contexts, profileRepinContext: undefined };
|
|
3818
3892
|
}
|
|
3893
|
+
else if (options.fromFailed === true && outpostRejectedFailedResumeOption(dispatched)) {
|
|
3894
|
+
failedResumeOptionRejected = true;
|
|
3895
|
+
break;
|
|
3896
|
+
}
|
|
3819
3897
|
else {
|
|
3820
3898
|
break;
|
|
3821
3899
|
}
|
|
@@ -3832,6 +3910,13 @@ async function dispatchOutpostLaneCommand(ctx, lane, action, options) {
|
|
|
3832
3910
|
required_option: "--grant-context",
|
|
3833
3911
|
});
|
|
3834
3912
|
}
|
|
3913
|
+
if (failedResumeOptionRejected) {
|
|
3914
|
+
throw new LanesError("outpost_cli_upgrade_required", `Outpost ${outpostId} needs an hq-cli release containing failed-lane resume support before it can resume lane ${lane.lane_id}`, {
|
|
3915
|
+
lane_id: lane.lane_id,
|
|
3916
|
+
outpost_id: outpostId,
|
|
3917
|
+
required_option: "--from-failed",
|
|
3918
|
+
});
|
|
3919
|
+
}
|
|
3835
3920
|
const ownerError = ownerErrorFromOutput(dispatched.stdout);
|
|
3836
3921
|
if (options.effort !== undefined && outpostRejectedEffortOption(dispatched)) {
|
|
3837
3922
|
throw new LanesError("outpost_effort_unsupported", `Outpost ${outpostId} rejected --effort; update hq-cli on that Outpost before changing a lane's effort remotely`, {
|
|
@@ -3848,6 +3933,7 @@ async function dispatchOutpostLaneCommand(ctx, lane, action, options) {
|
|
|
3848
3933
|
if (ownerError) {
|
|
3849
3934
|
if (action === "resume" &&
|
|
3850
3935
|
(ownerError.error === "worker_profile_hash_mismatch" ||
|
|
3936
|
+
ownerError.error.startsWith("failed_resume_") ||
|
|
3851
3937
|
ownerError.error.startsWith("hire_grant_"))) {
|
|
3852
3938
|
throw new LanesError(ownerError.error, ownerError.message, {
|
|
3853
3939
|
...ownerError.details,
|
|
@@ -5047,6 +5133,7 @@ async function runShow(ctx, laneId) {
|
|
|
5047
5133
|
last_resume: lane.last_resume ?? null,
|
|
5048
5134
|
account: lane.account ?? null,
|
|
5049
5135
|
account_history: lane.account_history ?? [],
|
|
5136
|
+
failed_resume_history: lane.failed_resume_history ?? [],
|
|
5050
5137
|
pending_resume: lane.pending_resume ?? null,
|
|
5051
5138
|
last_run_usage: lane.last_run_usage ?? null,
|
|
5052
5139
|
model: lane.model ?? null,
|
|
@@ -6631,6 +6718,30 @@ function laneWithRecoveredProviderSession(hqRoot, lane) {
|
|
|
6631
6718
|
async function runResume(ctx, laneId, opts) {
|
|
6632
6719
|
const hqRoot = ctx.hqRoot();
|
|
6633
6720
|
let lane = readLane(hqRoot, laneId);
|
|
6721
|
+
const failedResumeReason = opts.reason?.trim();
|
|
6722
|
+
if (opts.reason !== undefined && opts.fromFailed !== true) {
|
|
6723
|
+
throw new LanesError("failed_resume_reason_requires_flag", "--reason on hq lanes resume requires --from-failed", { lane_id: laneId });
|
|
6724
|
+
}
|
|
6725
|
+
if (opts.fromFailed === true && !failedResumeReason) {
|
|
6726
|
+
throw new LanesError("failed_resume_reason_required", "--from-failed requires a non-empty --reason", { lane_id: laneId });
|
|
6727
|
+
}
|
|
6728
|
+
if (opts.reason !== undefined)
|
|
6729
|
+
assertNoSecretText("resume reason", opts.reason);
|
|
6730
|
+
if (opts.failedResumeContext !== undefined && opts.fromFailed !== true) {
|
|
6731
|
+
throw new LanesError("failed_resume_authorization_invalid", "Failed-resume authorization context requires --from-failed", { lane_id: laneId });
|
|
6732
|
+
}
|
|
6733
|
+
const failedResumeActor = opts.fromFailed === true
|
|
6734
|
+
? authorizeFailedResume(ctx, lane, opts.failedResumeContext)
|
|
6735
|
+
: undefined;
|
|
6736
|
+
if (opts.fromFailed === true && lane.state !== "failed") {
|
|
6737
|
+
throw new LanesError("lane_not_failed", `Lane ${laneId} is ${lane.state}; --from-failed only applies to a failed lane`, { lane_id: laneId, state: lane.state });
|
|
6738
|
+
}
|
|
6739
|
+
const failedResumeExpectedSenior = opts.fromFailed === true
|
|
6740
|
+
? { ...lane.senior }
|
|
6741
|
+
: undefined;
|
|
6742
|
+
const recoveredFailure = opts.fromFailed === true
|
|
6743
|
+
? lane.terminal_reason ?? "failure_reason_unavailable"
|
|
6744
|
+
: undefined;
|
|
6634
6745
|
const hireGrantContext = opts.grantContext === undefined
|
|
6635
6746
|
? undefined
|
|
6636
6747
|
: (() => {
|
|
@@ -6671,7 +6782,7 @@ async function runResume(ctx, laneId, opts) {
|
|
|
6671
6782
|
if (lane.state === "queued") {
|
|
6672
6783
|
throw new LanesError("lane_not_resumable", `Lane ${laneId} is queued; use hq lanes admit ${laneId} to request admission`, { lane_id: laneId, state: lane.state, admission_command: `hq lanes admit ${laneId}` });
|
|
6673
6784
|
}
|
|
6674
|
-
const refusal = resumeCommandRefusal(lane);
|
|
6785
|
+
const refusal = resumeCommandRefusal(lane, { allowFailed: opts.fromFailed === true });
|
|
6675
6786
|
if (refusal !== null) {
|
|
6676
6787
|
throw new LanesError(refusal.error, refusal.message, {
|
|
6677
6788
|
lane_id: laneId,
|
|
@@ -6741,6 +6852,10 @@ async function runResume(ctx, laneId, opts) {
|
|
|
6741
6852
|
pending: {
|
|
6742
6853
|
trigger: "resume",
|
|
6743
6854
|
resumed_from: resumedFrom,
|
|
6855
|
+
...(opts.fromFailed === true ? { from_failed: true } : {}),
|
|
6856
|
+
...(failedResumeExpectedSenior !== undefined
|
|
6857
|
+
? { expected_senior: failedResumeExpectedSenior }
|
|
6858
|
+
: {}),
|
|
6744
6859
|
...(text !== undefined ? { text } : {}),
|
|
6745
6860
|
...(messageId !== undefined ? { message_id: messageId } : {}),
|
|
6746
6861
|
compact: opts.compact !== false,
|
|
@@ -6766,6 +6881,10 @@ async function runResume(ctx, laneId, opts) {
|
|
|
6766
6881
|
pending: {
|
|
6767
6882
|
trigger: "resume",
|
|
6768
6883
|
resumed_from: resumedFrom,
|
|
6884
|
+
...(opts.fromFailed === true ? { from_failed: true } : {}),
|
|
6885
|
+
...(failedResumeExpectedSenior !== undefined
|
|
6886
|
+
? { expected_senior: failedResumeExpectedSenior }
|
|
6887
|
+
: {}),
|
|
6769
6888
|
...(text !== undefined ? { text } : {}),
|
|
6770
6889
|
...(messageId !== undefined ? { message_id: messageId } : {}),
|
|
6771
6890
|
compact: opts.compact !== false,
|
|
@@ -6826,6 +6945,10 @@ async function runResume(ctx, laneId, opts) {
|
|
|
6826
6945
|
session_id: sessionId,
|
|
6827
6946
|
trigger: "resume",
|
|
6828
6947
|
resumed_from: resumedFrom,
|
|
6948
|
+
...(opts.fromFailed === true ? { from_failed: true } : {}),
|
|
6949
|
+
...(failedResumeExpectedSenior !== undefined
|
|
6950
|
+
? { expected_senior: failedResumeExpectedSenior }
|
|
6951
|
+
: {}),
|
|
6829
6952
|
...(text !== undefined ? { text } : {}),
|
|
6830
6953
|
...(messageId !== undefined ? { message_id: messageId } : {}),
|
|
6831
6954
|
compact: opts.compact !== false,
|
|
@@ -6853,6 +6976,10 @@ async function runResume(ctx, laneId, opts) {
|
|
|
6853
6976
|
trigger: {
|
|
6854
6977
|
kind: "resume",
|
|
6855
6978
|
from: resumedFrom,
|
|
6979
|
+
...(opts.fromFailed === true ? { fromFailed: true } : {}),
|
|
6980
|
+
...(failedResumeExpectedSenior !== undefined
|
|
6981
|
+
? { expectedSenior: failedResumeExpectedSenior }
|
|
6982
|
+
: {}),
|
|
6856
6983
|
...(text !== undefined ? { text } : {}),
|
|
6857
6984
|
...(messageId !== undefined ? { message_id: messageId } : {}),
|
|
6858
6985
|
},
|
|
@@ -6875,6 +7002,19 @@ async function runResume(ctx, laneId, opts) {
|
|
|
6875
7002
|
});
|
|
6876
7003
|
}
|
|
6877
7004
|
let next = resumed;
|
|
7005
|
+
if (failedResumeActor !== undefined &&
|
|
7006
|
+
failedResumeReason !== undefined &&
|
|
7007
|
+
recoveredFailure !== undefined) {
|
|
7008
|
+
next = casUpdateLane(hqRoot, laneId, (current) => ({
|
|
7009
|
+
...current,
|
|
7010
|
+
failed_resume_history: appendCappedHistory(current.failed_resume_history ?? [], {
|
|
7011
|
+
at,
|
|
7012
|
+
actor: failedResumeActor,
|
|
7013
|
+
reason: failedResumeReason,
|
|
7014
|
+
failure: recoveredFailure,
|
|
7015
|
+
}),
|
|
7016
|
+
}));
|
|
7017
|
+
}
|
|
6878
7018
|
if (text !== undefined) {
|
|
6879
7019
|
// An operator text moves the compaction reference forward, as a message does.
|
|
6880
7020
|
next = casUpdateLane(hqRoot, laneId, (current) => ({
|
|
@@ -9656,9 +9796,11 @@ export function registerLanesCommand(parent, deps = {}) {
|
|
|
9656
9796
|
}));
|
|
9657
9797
|
lanes
|
|
9658
9798
|
.command("resume")
|
|
9659
|
-
.description("Resume an interrupted, done or blocked lane on its recorded provider session")
|
|
9799
|
+
.description("Resume an interrupted, done or blocked lane on its recorded provider session; recover failed lanes with the senior-only --from-failed option")
|
|
9660
9800
|
.argument("<lane-id>", "Lane id")
|
|
9661
9801
|
.option("--text <msg>", "Operator message for the resumed turn")
|
|
9802
|
+
.option("--from-failed", "Explicitly resume a failed lane as its senior")
|
|
9803
|
+
.option("--reason <text>", "Required explanation for resuming from failed")
|
|
9662
9804
|
.option("--project <slug>", "Re-point project attribution from this resume")
|
|
9663
9805
|
.option("--story <id>", "Re-point story attribution from this resume")
|
|
9664
9806
|
.option("--account <id>", "Move the lane's session to this account of its provider before resuming")
|
|
@@ -9667,6 +9809,7 @@ export function registerLanesCommand(parent, deps = {}) {
|
|
|
9667
9809
|
.option("--worker <profile>", "Attach a personal worker profile for the resumed turn")
|
|
9668
9810
|
.addOption(new Option("--profile-repin-context <base64url>").hideHelp())
|
|
9669
9811
|
.addOption(new Option("--grant-context <base64url>").hideHelp())
|
|
9812
|
+
.addOption(new Option("--failed-resume-context <base64url>").hideHelp())
|
|
9670
9813
|
.option("--keep-alive", "Automatically resume done lead lanes while their story remains open")
|
|
9671
9814
|
.option("--mesh-company <slug>", "Attach Work Mesh attribution after verifying company membership")
|
|
9672
9815
|
.option("--no-compact", "Skip the resume-time compaction check for this resume")
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
* module rather than testing provider or state inline. A guard bound at one
|
|
18
18
|
* site and not another is the defect class this branch has already shipped.
|
|
19
19
|
*/
|
|
20
|
-
import { type LaneProvider, type LaneRecord, type LaneState } from "./types.js";
|
|
20
|
+
import { type LaneProvider, type LaneRecord, type LaneState, type SeniorRef } from "./types.js";
|
|
21
21
|
/** Which providers can continue an exited worker from a recorded session id. */
|
|
22
22
|
export declare const PROVIDER_SESSION_RESUME: Record<LaneProvider, boolean>;
|
|
23
23
|
export declare function providerSupportsSessionResume(provider: LaneProvider): boolean;
|
|
@@ -45,7 +45,8 @@ export declare function sessionNotPortableMessage(provider: LaneProvider): strin
|
|
|
45
45
|
* reconcile pass resumes it once every question is answered. `killed` and
|
|
46
46
|
* `cancelled` were stopped on purpose by the operator; a message must not
|
|
47
47
|
* silently restart them. `failed` means the run ended without an envelope or
|
|
48
|
-
* never started; the
|
|
48
|
+
* never started; only the lane's senior can explicitly recover it with
|
|
49
|
+
* `hq lanes resume <lane-id> --from-failed --reason <text>`. `interrupted` exited with
|
|
49
50
|
* partial output and no envelope, so the lane's state was never closed
|
|
50
51
|
* cleanly; only the explicit `hq lanes resume` opt-in continues it
|
|
51
52
|
* (RESUME_COMMAND_STATES). `queued` and `running` take the drop box, not a
|
|
@@ -139,6 +140,9 @@ export type LaunchTrigger = {
|
|
|
139
140
|
kind: "resume";
|
|
140
141
|
/** Lane state at the moment of the command; recorded as `last_resume.resumed_from`. */
|
|
141
142
|
from: ResumeCommandState | "failed";
|
|
143
|
+
fromFailed?: true;
|
|
144
|
+
/** Senior that authorized a failed-lane resume, rechecked at the claim CAS. */
|
|
145
|
+
expectedSenior?: SeniorRef;
|
|
142
146
|
text?: string;
|
|
143
147
|
message_id?: string;
|
|
144
148
|
};
|
|
@@ -152,7 +156,9 @@ export type LaunchPlan = {
|
|
|
152
156
|
* Refusal for `hq lanes resume`, in precedence order: wrong state, provider
|
|
153
157
|
* without a resume path, no recorded session. Null when the resume can go.
|
|
154
158
|
*/
|
|
155
|
-
export declare function resumeCommandRefusal(lane: DeliveryLane
|
|
159
|
+
export declare function resumeCommandRefusal(lane: DeliveryLane, options?: {
|
|
160
|
+
allowFailed?: boolean;
|
|
161
|
+
}): {
|
|
156
162
|
error: "lane_not_resumable" | "resume_unsupported" | "no_provider_session";
|
|
157
163
|
message: string;
|
|
158
164
|
} | null;
|
|
@@ -65,7 +65,8 @@ export function sessionNotPortableMessage(provider) {
|
|
|
65
65
|
* reconcile pass resumes it once every question is answered. `killed` and
|
|
66
66
|
* `cancelled` were stopped on purpose by the operator; a message must not
|
|
67
67
|
* silently restart them. `failed` means the run ended without an envelope or
|
|
68
|
-
* never started; the
|
|
68
|
+
* never started; only the lane's senior can explicitly recover it with
|
|
69
|
+
* `hq lanes resume <lane-id> --from-failed --reason <text>`. `interrupted` exited with
|
|
69
70
|
* partial output and no envelope, so the lane's state was never closed
|
|
70
71
|
* cleanly; only the explicit `hq lanes resume` opt-in continues it
|
|
71
72
|
* (RESUME_COMMAND_STATES). `queued` and `running` take the drop box, not a
|
|
@@ -137,7 +138,7 @@ export function deliveryRefusalReason(lane) {
|
|
|
137
138
|
case "interrupted":
|
|
138
139
|
return `Lane is ${state}. The run did not close cleanly and the operator has not decided what to do with it. Resuming from ${state} needs an explicit opt-in: hq lanes resume <lane-id> [--text <message>].`;
|
|
139
140
|
case "failed":
|
|
140
|
-
return `Lane is ${state}.
|
|
141
|
+
return `Lane is ${state}. A plain message does not restart it. Its senior can resume the recorded session with: hq lanes resume <lane-id> --from-failed --reason <text>.`;
|
|
141
142
|
default: {
|
|
142
143
|
const _exhaustive = state;
|
|
143
144
|
return _exhaustive;
|
|
@@ -345,8 +346,10 @@ export const LAUNCH_TRIGGERS = ["admit", "answered_questions", "message", "resum
|
|
|
345
346
|
* Refusal for `hq lanes resume`, in precedence order: wrong state, provider
|
|
346
347
|
* without a resume path, no recorded session. Null when the resume can go.
|
|
347
348
|
*/
|
|
348
|
-
export function resumeCommandRefusal(lane) {
|
|
349
|
-
if (!isResumeCommandState(lane.state) &&
|
|
349
|
+
export function resumeCommandRefusal(lane, options = {}) {
|
|
350
|
+
if (!isResumeCommandState(lane.state) &&
|
|
351
|
+
!isFailedEnvelopeRepairLane(lane) &&
|
|
352
|
+
!(options.allowFailed === true && lane.state === "failed")) {
|
|
350
353
|
return {
|
|
351
354
|
error: "lane_not_resumable",
|
|
352
355
|
message: `Lane is ${lane.state}. hq lanes resume continues a lane in ${RESUME_COMMAND_STATES.join(", ")}; ${lane.state === "awaiting_input"
|
|
@@ -395,7 +398,7 @@ export function planLaneLaunch(lane, trigger) {
|
|
|
395
398
|
return { mode: "resume", session_id: lane.provider_session_id };
|
|
396
399
|
}
|
|
397
400
|
case "resume": {
|
|
398
|
-
const refusal = resumeCommandRefusal(lane);
|
|
401
|
+
const refusal = resumeCommandRefusal(lane, { allowFailed: trigger.fromFailed === true });
|
|
399
402
|
if (refusal !== null || !hasRecordedSessionId(lane)) {
|
|
400
403
|
throw new LanesError(refusal?.error ?? "no_provider_session", refusal?.message ?? `Lane is ${lane.state} and has no recorded provider session`, { state: lane.state, provider: lane.provider });
|
|
401
404
|
}
|
|
@@ -418,7 +421,9 @@ export function claimableStatesFor(trigger, lane) {
|
|
|
418
421
|
case "message":
|
|
419
422
|
return envelopeRepairFailed ? [...RESUMABLE_LANE_STATES, "failed"] : RESUMABLE_LANE_STATES;
|
|
420
423
|
case "resume":
|
|
421
|
-
return
|
|
424
|
+
return trigger.fromFailed === true || envelopeRepairFailed
|
|
425
|
+
? [...RESUME_COMMAND_STATES, "failed"]
|
|
426
|
+
: RESUME_COMMAND_STATES;
|
|
422
427
|
default: {
|
|
423
428
|
const _exhaustive = trigger;
|
|
424
429
|
return _exhaustive;
|
|
@@ -151,12 +151,15 @@ export function buildOutpostLaneCommand(input) {
|
|
|
151
151
|
["--hire", input.hire],
|
|
152
152
|
["--grant-context", input.grantContext],
|
|
153
153
|
["--profile-repin-context", input.profileRepinContext],
|
|
154
|
+
["--failed-resume-context", input.failedResumeContext],
|
|
154
155
|
["--senior", input.senior],
|
|
155
156
|
["--reason", input.reason],
|
|
156
157
|
]) {
|
|
157
158
|
if (value !== undefined)
|
|
158
159
|
args.push(flag, value);
|
|
159
160
|
}
|
|
161
|
+
if (input.action === "resume" && input.fromFailed === true)
|
|
162
|
+
args.push("--from-failed");
|
|
160
163
|
if (input.action === "resume" && input.keepAlive === true)
|
|
161
164
|
args.push("--keep-alive");
|
|
162
165
|
if (input.compact === false)
|
package/dist/lib/lanes/spawn.js
CHANGED
|
@@ -1576,6 +1576,9 @@ function laneLaunchResult(lane) {
|
|
|
1576
1576
|
const { provider_home_at_spawn: _providerHomeAtSpawn, ...safe } = lane;
|
|
1577
1577
|
return safe;
|
|
1578
1578
|
}
|
|
1579
|
+
function sameSeniorRef(a, b) {
|
|
1580
|
+
return a.kind === b.kind && a.id === b.id;
|
|
1581
|
+
}
|
|
1579
1582
|
function inferTrigger(lane) {
|
|
1580
1583
|
if (lane.state === "awaiting_input")
|
|
1581
1584
|
return { kind: "answered_questions" };
|
|
@@ -1631,6 +1634,12 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
|
|
|
1631
1634
|
// starting fresh from the brief.
|
|
1632
1635
|
const pendingResume = launch.trigger === undefined && existing.state === "queued" ? existing.pending_resume : undefined;
|
|
1633
1636
|
const trigger = launch.trigger ?? inferTrigger(existing);
|
|
1637
|
+
const failedResumeClaim = (trigger.kind === "resume" && trigger.fromFailed === true) || pendingResume?.from_failed === true;
|
|
1638
|
+
const failedResumeExpectedSenior = trigger.kind === "resume" && trigger.fromFailed === true
|
|
1639
|
+
? trigger.expectedSenior
|
|
1640
|
+
: pendingResume?.from_failed === true
|
|
1641
|
+
? pendingResume.expected_senior
|
|
1642
|
+
: undefined;
|
|
1634
1643
|
// A message into a finished lane must not turn that lane `failed` when the
|
|
1635
1644
|
// account cannot be bound: nothing has started yet, so the record keeps its
|
|
1636
1645
|
// state and the caller sees the error. Admission and question resume keep
|
|
@@ -1971,6 +1980,15 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
|
|
|
1971
1980
|
if (!claimableStatesFor(trigger, current).includes(current.state)) {
|
|
1972
1981
|
throw new LanesError("spawn_state_mismatch", `Cannot mark ${current.lane_id} running from ${current.state}`, { lane_id: current.lane_id, state: current.state, trigger: trigger.kind });
|
|
1973
1982
|
}
|
|
1983
|
+
if (failedResumeClaim &&
|
|
1984
|
+
(failedResumeExpectedSenior === undefined ||
|
|
1985
|
+
!sameSeniorRef(current.senior, failedResumeExpectedSenior))) {
|
|
1986
|
+
throw new LanesError("failed_resume_senior_changed", `Lane ${current.lane_id}'s senior changed after its failed-resume request was authorized`, {
|
|
1987
|
+
lane_id: current.lane_id,
|
|
1988
|
+
expected_senior: failedResumeExpectedSenior ?? null,
|
|
1989
|
+
current_senior: current.senior,
|
|
1990
|
+
});
|
|
1991
|
+
}
|
|
1974
1992
|
claimed = true;
|
|
1975
1993
|
const next = {
|
|
1976
1994
|
...current,
|
|
@@ -2044,6 +2062,7 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
|
|
|
2044
2062
|
...(pendingResume.message_id !== undefined ? { message_id: pendingResume.message_id } : {}),
|
|
2045
2063
|
...(pendingResume.monitor_event === true ? { monitor_event: true } : {}),
|
|
2046
2064
|
...(pendingResume.resumed_from !== undefined ? { resumed_from: pendingResume.resumed_from } : {}),
|
|
2065
|
+
...(pendingResume.from_failed === true ? { from_failed: true } : {}),
|
|
2047
2066
|
...(pendingResume.text !== undefined ? { text: pendingResume.text } : {}),
|
|
2048
2067
|
...(pendingResume.compact !== undefined ? { compact: pendingResume.compact } : {}),
|
|
2049
2068
|
};
|
|
@@ -2065,6 +2084,7 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
|
|
|
2065
2084
|
...(trigger.kind === "resume"
|
|
2066
2085
|
? {
|
|
2067
2086
|
resumed_from: trigger.from,
|
|
2087
|
+
...(trigger.fromFailed === true ? { from_failed: true } : {}),
|
|
2068
2088
|
...(trigger.text !== undefined ? { text: trigger.text } : {}),
|
|
2069
2089
|
...(trigger.message_id !== undefined ? { message_id: trigger.message_id } : {}),
|
|
2070
2090
|
}
|
package/dist/lib/lanes/store.js
CHANGED
|
@@ -888,6 +888,11 @@ export function mergeLaneRecords(live, sidecars) {
|
|
|
888
888
|
worker_profile_repin_history: unionHistory(all.map((record) => record.worker_profile_repin_history ?? []), entryIdentity, base.worker_profile_repin_history ?? []),
|
|
889
889
|
}
|
|
890
890
|
: {}),
|
|
891
|
+
...(all.some((record) => record.failed_resume_history !== undefined)
|
|
892
|
+
? {
|
|
893
|
+
failed_resume_history: unionHistory(all.map((record) => record.failed_resume_history ?? []), entryIdentity, base.failed_resume_history ?? []),
|
|
894
|
+
}
|
|
895
|
+
: {}),
|
|
891
896
|
stories: mergeLaneStories(all, base),
|
|
892
897
|
};
|
|
893
898
|
}
|
|
@@ -400,6 +400,8 @@ export interface LaneRecord {
|
|
|
400
400
|
* like the other histories.
|
|
401
401
|
*/
|
|
402
402
|
account_history?: AccountHistoryEntry[];
|
|
403
|
+
/** Bounded audit history for explicitly authorized resumes from failed. */
|
|
404
|
+
failed_resume_history?: FailedResumeHistoryEntry[];
|
|
403
405
|
/**
|
|
404
406
|
* Set by an account move (`--account`) until the spawn claims the lane:
|
|
405
407
|
* the turn the move was asked to run. While set it reserves a slot on the
|
|
@@ -537,11 +539,20 @@ export interface AccountHistoryEntry {
|
|
|
537
539
|
at: string;
|
|
538
540
|
reason: "first_assignment" | "model_change" | "usage_limit" | "account_unavailable" | "recorded_account_logged_out" | "load_balance" | "operator_move";
|
|
539
541
|
}
|
|
542
|
+
export interface FailedResumeHistoryEntry {
|
|
543
|
+
at: string;
|
|
544
|
+
actor: string;
|
|
545
|
+
reason: string;
|
|
546
|
+
failure: string;
|
|
547
|
+
}
|
|
540
548
|
export interface PendingResume {
|
|
541
549
|
/** Fresh answered-question launches need no previous session id. */
|
|
542
550
|
session_id?: string;
|
|
543
551
|
trigger: "answered_questions" | "message" | "resume";
|
|
544
552
|
resumed_from?: "interrupted" | "done" | "blocked" | "failed";
|
|
553
|
+
from_failed?: true;
|
|
554
|
+
/** Senior that authorized a failed-lane resume, rechecked at the claim CAS. */
|
|
555
|
+
expected_senior?: SeniorRef;
|
|
545
556
|
text?: string;
|
|
546
557
|
message_id?: string;
|
|
547
558
|
/** The text is already framed as untrusted monitor data, not an operator instruction. */
|
|
@@ -572,6 +583,7 @@ export interface LaneResumeRecord {
|
|
|
572
583
|
* resumed it (`interrupted`, `done` or `blocked`) and the text they sent.
|
|
573
584
|
*/
|
|
574
585
|
resumed_from?: "interrupted" | "done" | "blocked" | "failed";
|
|
586
|
+
from_failed?: true;
|
|
575
587
|
text?: string;
|
|
576
588
|
/** False when the continued turn explicitly skipped compaction. */
|
|
577
589
|
compact?: boolean;
|
package/dist/lib/lanes/types.js
CHANGED
|
@@ -225,7 +225,7 @@ export const LANE_FIELD_WRITERS = {
|
|
|
225
225
|
pickup_validation_pending_since: ["creator", "command", "spawn", "reconcile"],
|
|
226
226
|
priority: ["command"], worker: ["creator", "command"], worker_path: ["creator", "command"], worker_pack: ["creator", "command"], worker_profile: ["creator", "command"], worker_profile_hash: ["creator", "command", "spawn"], worker_profile_repin_history: ["command"], worker_profile_ignored: ["creator", "command", "spawn"], worker_policy_delivery: ["command", "spawn"],
|
|
227
227
|
keep_alive: ["creator", "command"], keep_alive_standing: ["command"], keep_alive_opted_in_at: ["creator", "command"], keep_alive_blocked_reason: ["command", "reconcile"], keep_alive_resume_count: ["command", "reconcile"], keep_alive_last_resume_at: ["command", "reconcile"], keep_alive_next_eligible_at: ["command", "reconcile"], keep_alive_exhausted_at: ["command", "reconcile"],
|
|
228
|
-
provider: ["creator"], account: ["creator", "spawn", "command", "watcher", "reconcile"], account_selection: ["creator", "command"], admission_accounts: ["creator", "command"], account_history: ["command", "spawn", "watcher", "reconcile"], pending_resume: ["command", "spawn", "watcher", "reconcile"], placement_default: ["creator"], host: ["creator", "command"], origin_host: ["creator"], origin_host_source: ["creator"],
|
|
228
|
+
provider: ["creator"], account: ["creator", "spawn", "command", "watcher", "reconcile"], account_selection: ["creator", "command"], admission_accounts: ["creator", "command"], account_history: ["command", "spawn", "watcher", "reconcile"], failed_resume_history: ["command"], pending_resume: ["command", "spawn", "watcher", "reconcile"], placement_default: ["creator"], host: ["creator", "command"], origin_host: ["creator"], origin_host_source: ["creator"],
|
|
229
229
|
version: ["creator"], created_at: ["creator"], updated_at: ["creator"], attribution_history: ["command"],
|
|
230
230
|
last_envelope: ["watcher", "reconcile"], model: ["creator", "spawn", "command"], model_source: ["creator", "spawn", "command"], effort: ["creator", "spawn", "command"], effort_source: ["creator", "spawn", "command"], fast_mode: ["spawn"],
|
|
231
231
|
last_message_at: ["command"], last_compaction: ["spawn"], story_complete: ["watcher", "reconcile"], story_complete_reason: ["watcher", "reconcile"], story_status: ["watcher", "reconcile"],
|