@indigoai-us/hq-cli 5.232.0 → 5.234.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/dist/command-catalog.generated.d.ts +31 -1
- package/dist/command-catalog.generated.js +40 -1
- package/dist/commands/agents.js +10 -2
- package/dist/commands/lanes.js +186 -3
- package/dist/lib/lanes/delivery.d.ts +9 -3
- package/dist/lib/lanes/delivery.js +11 -6
- package/dist/lib/lanes/outpost-dispatch.d.ts +2 -0
- package/dist/lib/lanes/outpost-dispatch.js +3 -0
- package/dist/lib/lanes/spawn.js +20 -0
- package/dist/lib/lanes/store.js +5 -0
- package/dist/lib/lanes/types.d.ts +12 -0
- package/dist/lib/lanes/types.js +1 -1
- package/dist/lib/lanes/wait.d.ts +39 -0
- package/dist/lib/lanes/wait.js +159 -0
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,27 @@
|
|
|
2
2
|
|
|
3
3
|
## [Unreleased]
|
|
4
4
|
|
|
5
|
+
## [5.234.0] — 2026-09-27
|
|
6
|
+
|
|
7
|
+
### Added
|
|
8
|
+
|
|
9
|
+
- A lane's senior can explicitly resume a failed lane's recorded provider
|
|
10
|
+
session with `hq lanes resume --from-failed --reason`; the secret-scanned
|
|
11
|
+
reason and recovered failure are recorded in lane history. Failed lanes
|
|
12
|
+
continue to refuse ordinary messages. (#927)
|
|
13
|
+
|
|
14
|
+
### Fixed
|
|
15
|
+
|
|
16
|
+
- `hq agents stop` says the stop was accepted and shows the status the API returns, instead of saying the agent has already stopped. (#922)
|
|
17
|
+
|
|
18
|
+
## [5.233.0] — 2026-09-27
|
|
19
|
+
|
|
20
|
+
### Added
|
|
21
|
+
|
|
22
|
+
- `hq lanes wait` exits on the first selected envelope, pending-question, or
|
|
23
|
+
state change across the requested lanes. It uses a startup baseline and a
|
|
24
|
+
bounded two-second poll interval.
|
|
25
|
+
|
|
5
26
|
## [5.232.0] — 2026-09-27
|
|
6
27
|
|
|
7
28
|
### Added
|
|
@@ -7773,6 +7773,27 @@ export declare const COMMAND_CATALOG: readonly [{
|
|
|
7773
7773
|
readonly description: "JSON on stdout (always)";
|
|
7774
7774
|
}];
|
|
7775
7775
|
readonly subcommands: readonly [];
|
|
7776
|
+
}, {
|
|
7777
|
+
readonly name: "wait";
|
|
7778
|
+
readonly description: "Wait for the first requested change across lanes";
|
|
7779
|
+
readonly aliases: readonly [];
|
|
7780
|
+
readonly hidden: false;
|
|
7781
|
+
readonly usage: "[options]";
|
|
7782
|
+
readonly arguments: readonly [];
|
|
7783
|
+
readonly options: readonly [{
|
|
7784
|
+
readonly flags: "--any <lane-id...>";
|
|
7785
|
+
readonly description: "Lane ids to monitor";
|
|
7786
|
+
}, {
|
|
7787
|
+
readonly flags: "--for <change-type>";
|
|
7788
|
+
readonly description: "envelope | question | state";
|
|
7789
|
+
}, {
|
|
7790
|
+
readonly flags: "--timeout <seconds>";
|
|
7791
|
+
readonly description: "Positive timeout in seconds";
|
|
7792
|
+
}, {
|
|
7793
|
+
readonly flags: "--json";
|
|
7794
|
+
readonly description: "JSON output";
|
|
7795
|
+
}];
|
|
7796
|
+
readonly subcommands: readonly [];
|
|
7776
7797
|
}, {
|
|
7777
7798
|
readonly name: "watch";
|
|
7778
7799
|
readonly description: "Watch lane state and question changes until all lanes are terminal";
|
|
@@ -8026,7 +8047,7 @@ export declare const COMMAND_CATALOG: readonly [{
|
|
|
8026
8047
|
readonly subcommands: readonly [];
|
|
8027
8048
|
}, {
|
|
8028
8049
|
readonly name: "resume";
|
|
8029
|
-
readonly description: "Resume an interrupted, done or blocked lane on its recorded provider session";
|
|
8050
|
+
readonly description: "Resume an interrupted, done or blocked lane on its recorded provider session; recover failed lanes with the senior-only --from-failed option";
|
|
8030
8051
|
readonly aliases: readonly [];
|
|
8031
8052
|
readonly hidden: false;
|
|
8032
8053
|
readonly usage: "[options] <lane-id>";
|
|
@@ -8038,6 +8059,12 @@ export declare const COMMAND_CATALOG: readonly [{
|
|
|
8038
8059
|
readonly options: readonly [{
|
|
8039
8060
|
readonly flags: "--text <msg>";
|
|
8040
8061
|
readonly description: "Operator message for the resumed turn";
|
|
8062
|
+
}, {
|
|
8063
|
+
readonly flags: "--from-failed";
|
|
8064
|
+
readonly description: "Explicitly resume a failed lane as its senior";
|
|
8065
|
+
}, {
|
|
8066
|
+
readonly flags: "--reason <text>";
|
|
8067
|
+
readonly description: "Required explanation for resuming from failed";
|
|
8041
8068
|
}, {
|
|
8042
8069
|
readonly flags: "--project <slug>";
|
|
8043
8070
|
readonly description: "Re-point project attribution from this resume";
|
|
@@ -8059,6 +8086,9 @@ export declare const COMMAND_CATALOG: readonly [{
|
|
|
8059
8086
|
}, {
|
|
8060
8087
|
readonly flags: "--grant-context <base64url>";
|
|
8061
8088
|
readonly description: "";
|
|
8089
|
+
}, {
|
|
8090
|
+
readonly flags: "--failed-resume-context <base64url>";
|
|
8091
|
+
readonly description: "";
|
|
8062
8092
|
}, {
|
|
8063
8093
|
readonly flags: "--keep-alive";
|
|
8064
8094
|
readonly description: "Automatically resume done lead lanes while their story remains open";
|
|
@@ -10051,6 +10051,33 @@ export const COMMAND_CATALOG = [
|
|
|
10051
10051
|
],
|
|
10052
10052
|
"subcommands": []
|
|
10053
10053
|
},
|
|
10054
|
+
{
|
|
10055
|
+
"name": "wait",
|
|
10056
|
+
"description": "Wait for the first requested change across lanes",
|
|
10057
|
+
"aliases": [],
|
|
10058
|
+
"hidden": false,
|
|
10059
|
+
"usage": "[options]",
|
|
10060
|
+
"arguments": [],
|
|
10061
|
+
"options": [
|
|
10062
|
+
{
|
|
10063
|
+
"flags": "--any <lane-id...>",
|
|
10064
|
+
"description": "Lane ids to monitor"
|
|
10065
|
+
},
|
|
10066
|
+
{
|
|
10067
|
+
"flags": "--for <change-type>",
|
|
10068
|
+
"description": "envelope | question | state"
|
|
10069
|
+
},
|
|
10070
|
+
{
|
|
10071
|
+
"flags": "--timeout <seconds>",
|
|
10072
|
+
"description": "Positive timeout in seconds"
|
|
10073
|
+
},
|
|
10074
|
+
{
|
|
10075
|
+
"flags": "--json",
|
|
10076
|
+
"description": "JSON output"
|
|
10077
|
+
}
|
|
10078
|
+
],
|
|
10079
|
+
"subcommands": []
|
|
10080
|
+
},
|
|
10054
10081
|
{
|
|
10055
10082
|
"name": "watch",
|
|
10056
10083
|
"description": "Watch lane state and question changes until all lanes are terminal",
|
|
@@ -10381,7 +10408,7 @@ export const COMMAND_CATALOG = [
|
|
|
10381
10408
|
},
|
|
10382
10409
|
{
|
|
10383
10410
|
"name": "resume",
|
|
10384
|
-
"description": "Resume an interrupted, done or blocked lane on its recorded provider session",
|
|
10411
|
+
"description": "Resume an interrupted, done or blocked lane on its recorded provider session; recover failed lanes with the senior-only --from-failed option",
|
|
10385
10412
|
"aliases": [],
|
|
10386
10413
|
"hidden": false,
|
|
10387
10414
|
"usage": "[options] <lane-id>",
|
|
@@ -10397,6 +10424,14 @@ export const COMMAND_CATALOG = [
|
|
|
10397
10424
|
"flags": "--text <msg>",
|
|
10398
10425
|
"description": "Operator message for the resumed turn"
|
|
10399
10426
|
},
|
|
10427
|
+
{
|
|
10428
|
+
"flags": "--from-failed",
|
|
10429
|
+
"description": "Explicitly resume a failed lane as its senior"
|
|
10430
|
+
},
|
|
10431
|
+
{
|
|
10432
|
+
"flags": "--reason <text>",
|
|
10433
|
+
"description": "Required explanation for resuming from failed"
|
|
10434
|
+
},
|
|
10400
10435
|
{
|
|
10401
10436
|
"flags": "--project <slug>",
|
|
10402
10437
|
"description": "Re-point project attribution from this resume"
|
|
@@ -10425,6 +10460,10 @@ export const COMMAND_CATALOG = [
|
|
|
10425
10460
|
"flags": "--grant-context <base64url>",
|
|
10426
10461
|
"description": ""
|
|
10427
10462
|
},
|
|
10463
|
+
{
|
|
10464
|
+
"flags": "--failed-resume-context <base64url>",
|
|
10465
|
+
"description": ""
|
|
10466
|
+
},
|
|
10428
10467
|
{
|
|
10429
10468
|
"flags": "--keep-alive",
|
|
10430
10469
|
"description": "Automatically resume done lead lanes while their story remains open"
|
package/dist/commands/agents.js
CHANGED
|
@@ -1819,8 +1819,16 @@ export function registerAgentsCommand(program) {
|
|
|
1819
1819
|
.action(async function (agentUid) {
|
|
1820
1820
|
try {
|
|
1821
1821
|
const token = (await resolveVaultCredential()).token;
|
|
1822
|
-
await startStopAgent(token, agentUid, action);
|
|
1823
|
-
|
|
1822
|
+
const result = await startStopAgent(token, agentUid, action);
|
|
1823
|
+
const runtimeStatus = result.runtime?.status;
|
|
1824
|
+
const successMessage = action === "start"
|
|
1825
|
+
? `Started agent ${agentUid}.`
|
|
1826
|
+
: runtimeStatus === "terminated"
|
|
1827
|
+
? `Stopped agent ${agentUid}.`
|
|
1828
|
+
: typeof runtimeStatus === "string"
|
|
1829
|
+
? `Stop accepted for agent ${agentUid} (status: ${runtimeStatus}).`
|
|
1830
|
+
: `Stop accepted for agent ${agentUid}.`;
|
|
1831
|
+
console.log(chalk.green(successMessage));
|
|
1824
1832
|
}
|
|
1825
1833
|
catch (err) {
|
|
1826
1834
|
fail(err);
|
package/dist/commands/lanes.js
CHANGED
|
@@ -52,6 +52,7 @@ import { compactionAfterUsage, } from "../lib/lanes/compaction.js";
|
|
|
52
52
|
import { assertUniqueAccountHomes, providerCompactAfterMs } from "../lib/lanes/config.js";
|
|
53
53
|
import { parseCliEffort, resolveLaneEffort, } from "../lib/lanes/effort.js";
|
|
54
54
|
import { LanesError } from "../lib/lanes/errors.js";
|
|
55
|
+
import { LANE_WAIT_INVALID_INPUT_EXIT_CODE, LANE_WAIT_TIMEOUT_EXIT_CODE, runLaneWait, } from "../lib/lanes/wait.js";
|
|
55
56
|
import { dispatchLaneDelivery } from "../lib/lanes/delivery-core.js";
|
|
56
57
|
import { dropboxInboxHasMessages, drainDropbox, formatDropboxMessages, formatMonitorEvent, requeueDropboxMessages, } from "../lib/lanes/dropbox.js";
|
|
57
58
|
import { isPidAlive, runWatcherLoop, WATCHER_OUT_MAX_BYTES, watcherOutLogPath, writeWatcherDecision, } from "../lib/lanes/watcher.js";
|
|
@@ -2127,6 +2128,37 @@ function yieldEventLoop() {
|
|
|
2127
2128
|
setImmediate(resolve);
|
|
2128
2129
|
});
|
|
2129
2130
|
}
|
|
2131
|
+
function laneWaitHumanLine(result) {
|
|
2132
|
+
const details = Object.entries(result.change)
|
|
2133
|
+
.filter(([key]) => key !== "type")
|
|
2134
|
+
.map(([key, value]) => `${key}=${String(value)}`)
|
|
2135
|
+
.join(" ");
|
|
2136
|
+
return `lane=${result.lane_id} change=${result.change.type}${details ? ` ${details}` : ""} at=${result.at}`;
|
|
2137
|
+
}
|
|
2138
|
+
async function runWait(ctx, opts) {
|
|
2139
|
+
const timeoutSeconds = opts.timeout === undefined ? Number.NaN : Number(opts.timeout);
|
|
2140
|
+
try {
|
|
2141
|
+
const result = await runLaneWait(ctx.hqRoot(), opts.any ?? [], opts.for ?? [], timeoutSeconds * 1000, { now: ctx.now, sleep: ctx.sleep });
|
|
2142
|
+
if (result) {
|
|
2143
|
+
console.log(opts.json ? JSON.stringify(result) : laneWaitHumanLine(result));
|
|
2144
|
+
return 0;
|
|
2145
|
+
}
|
|
2146
|
+
console.log(opts.json
|
|
2147
|
+
? JSON.stringify({ ok: false, error: "wait_timeout" })
|
|
2148
|
+
: "wait_timeout");
|
|
2149
|
+
return LANE_WAIT_TIMEOUT_EXIT_CODE;
|
|
2150
|
+
}
|
|
2151
|
+
catch (err) {
|
|
2152
|
+
if (!isLanesError(err) || err.error !== "wait_invalid_input")
|
|
2153
|
+
throw err;
|
|
2154
|
+
const output = { ok: false, error: err.error, message: err.message };
|
|
2155
|
+
if (opts.json)
|
|
2156
|
+
console.log(JSON.stringify(output));
|
|
2157
|
+
else
|
|
2158
|
+
process.stderr.write(`${err.message}\n`);
|
|
2159
|
+
return LANE_WAIT_INVALID_INPUT_EXIT_CODE;
|
|
2160
|
+
}
|
|
2161
|
+
}
|
|
2130
2162
|
function withCasRetry(fn, attempts = 5) {
|
|
2131
2163
|
let last;
|
|
2132
2164
|
for (let i = 0; i < attempts; i += 1) {
|
|
@@ -3594,6 +3626,10 @@ function outpostRejectedGrantContextOption(result) {
|
|
|
3594
3626
|
return result.exitCode !== 0 &&
|
|
3595
3627
|
/unknown option\s+(?:['"])?--grant-context(?:['"])?(?:\s|$)/i.test(`${result.stderr}\n${result.stdout}`);
|
|
3596
3628
|
}
|
|
3629
|
+
function outpostRejectedFailedResumeOption(result) {
|
|
3630
|
+
return result.exitCode !== 0 &&
|
|
3631
|
+
/unknown option\s+(?:['"])?--(?:from-failed|failed-resume-context)(?:['"])?(?:\s|$)/i.test(`${result.stderr}\n${result.stdout}`);
|
|
3632
|
+
}
|
|
3597
3633
|
function verifyOutpostProfileContent(hqRoot, lane) {
|
|
3598
3634
|
if (lane.host.kind !== "outpost" || lane.worker_profile === undefined)
|
|
3599
3635
|
return;
|
|
@@ -3616,6 +3652,70 @@ function resumeProfileRepinActor(ctx) {
|
|
|
3616
3652
|
const sessionId = sessionIdFromEnv(ctx.env);
|
|
3617
3653
|
return sessionId ? `session:${sessionId}` : "ssm-owner-dispatch";
|
|
3618
3654
|
}
|
|
3655
|
+
function createOutpostFailedResumeContext(ctx, lane) {
|
|
3656
|
+
return Buffer.from(JSON.stringify({
|
|
3657
|
+
version: 1,
|
|
3658
|
+
lane_id: lane.lane_id,
|
|
3659
|
+
expected_senior: lane.senior,
|
|
3660
|
+
actor: resumeProfileRepinActor(ctx),
|
|
3661
|
+
})).toString("base64url");
|
|
3662
|
+
}
|
|
3663
|
+
function parseOutpostFailedResumeContext(encoded, laneId) {
|
|
3664
|
+
try {
|
|
3665
|
+
if (!/^[A-Za-z0-9_-]{1,4096}$/.test(encoded))
|
|
3666
|
+
throw new Error("invalid encoding");
|
|
3667
|
+
const value = JSON.parse(Buffer.from(encoded, "base64url").toString("utf8"));
|
|
3668
|
+
if (value === null || typeof value !== "object" || Array.isArray(value))
|
|
3669
|
+
throw new Error("invalid payload");
|
|
3670
|
+
const payload = value;
|
|
3671
|
+
const seniorValue = payload.expected_senior;
|
|
3672
|
+
if (seniorValue === null || typeof seniorValue !== "object" || Array.isArray(seniorValue))
|
|
3673
|
+
throw new Error("invalid senior");
|
|
3674
|
+
const senior = seniorValue;
|
|
3675
|
+
let expectedSenior;
|
|
3676
|
+
if (senior.kind === "user") {
|
|
3677
|
+
expectedSenior = { kind: "user" };
|
|
3678
|
+
}
|
|
3679
|
+
else if ((senior.kind === "lane" || senior.kind === "session") &&
|
|
3680
|
+
typeof senior.id === "string" && senior.id.trim().length > 0) {
|
|
3681
|
+
expectedSenior = { kind: senior.kind, id: senior.id };
|
|
3682
|
+
}
|
|
3683
|
+
else {
|
|
3684
|
+
throw new Error("invalid senior");
|
|
3685
|
+
}
|
|
3686
|
+
if (payload.version !== 1 ||
|
|
3687
|
+
payload.lane_id !== laneId ||
|
|
3688
|
+
typeof payload.actor !== "string" ||
|
|
3689
|
+
payload.actor.trim().length === 0) {
|
|
3690
|
+
throw new Error("invalid payload");
|
|
3691
|
+
}
|
|
3692
|
+
return { expectedSenior, actor: payload.actor };
|
|
3693
|
+
}
|
|
3694
|
+
catch {
|
|
3695
|
+
throw new LanesError("failed_resume_authorization_invalid", "Failed-resume authorization context is invalid", { lane_id: laneId });
|
|
3696
|
+
}
|
|
3697
|
+
}
|
|
3698
|
+
function authorizeFailedResume(ctx, lane, encodedContext) {
|
|
3699
|
+
if (encodedContext !== undefined) {
|
|
3700
|
+
if (!ctx.isSsmOwnerDispatch()) {
|
|
3701
|
+
throw new LanesError("failed_resume_authorization_invalid", "Failed-resume authorization context is only valid on an Outpost owner dispatch", { lane_id: lane.lane_id });
|
|
3702
|
+
}
|
|
3703
|
+
const context = parseOutpostFailedResumeContext(encodedContext, lane.lane_id);
|
|
3704
|
+
if (lane.host.kind !== "outpost" ||
|
|
3705
|
+
!sameHireGrantSenior(lane.senior, context.expectedSenior)) {
|
|
3706
|
+
throw new LanesError("failed_resume_senior_changed", `Lane ${lane.lane_id}'s senior changed after its failed-resume request was authorized`, {
|
|
3707
|
+
lane_id: lane.lane_id,
|
|
3708
|
+
expected_senior: context.expectedSenior,
|
|
3709
|
+
current_senior: lane.senior,
|
|
3710
|
+
});
|
|
3711
|
+
}
|
|
3712
|
+
return context.actor;
|
|
3713
|
+
}
|
|
3714
|
+
if (!callerMatchesSenior(ctx, lane.senior)) {
|
|
3715
|
+
throw new LanesError("failed_resume_unauthorized", "Only the lane's senior may resume a failed lane with --from-failed", { lane_id: lane.lane_id });
|
|
3716
|
+
}
|
|
3717
|
+
return resumeProfileRepinActor(ctx);
|
|
3718
|
+
}
|
|
3619
3719
|
function workerProfileHashMismatch(lane, profile) {
|
|
3620
3720
|
return new LanesError("worker_profile_hash_mismatch", `Worker profile ${JSON.stringify(profile.id)} on Outpost ${lane.host.id} does not match the hash recorded by the lane`, {
|
|
3621
3721
|
lane_id: lane.lane_id,
|
|
@@ -3725,6 +3825,9 @@ async function dispatchOutpostLaneCommand(ctx, lane, action, options) {
|
|
|
3725
3825
|
actor: resumeProfileRepinActor(ctx),
|
|
3726
3826
|
})
|
|
3727
3827
|
: undefined;
|
|
3828
|
+
const failedResumeContext = action === "resume" && options.fromFailed === true
|
|
3829
|
+
? createOutpostFailedResumeContext(ctx, lane)
|
|
3830
|
+
: undefined;
|
|
3728
3831
|
let grantContext;
|
|
3729
3832
|
let grantHierarchy;
|
|
3730
3833
|
if (action === "grant") {
|
|
@@ -3744,6 +3847,7 @@ async function dispatchOutpostLaneCommand(ctx, lane, action, options) {
|
|
|
3744
3847
|
}
|
|
3745
3848
|
let dispatched;
|
|
3746
3849
|
let grantContextRejected = false;
|
|
3850
|
+
let failedResumeOptionRejected = false;
|
|
3747
3851
|
try {
|
|
3748
3852
|
const dispatchCommand = (contexts) => ctx.dispatchOutpostExec({
|
|
3749
3853
|
outpostId,
|
|
@@ -3765,15 +3869,17 @@ async function dispatchOutpostLaneCommand(ctx, lane, action, options) {
|
|
|
3765
3869
|
effort: options.effort,
|
|
3766
3870
|
worker: options.worker,
|
|
3767
3871
|
keepAlive: options.keepAlive,
|
|
3872
|
+
fromFailed: options.fromFailed,
|
|
3768
3873
|
hire: options.hire,
|
|
3769
3874
|
senior: options.senior,
|
|
3770
3875
|
reason: options.reason,
|
|
3771
3876
|
grantContext: contexts.grantContext,
|
|
3772
3877
|
profileRepinContext: contexts.profileRepinContext,
|
|
3878
|
+
failedResumeContext: contexts.failedResumeContext,
|
|
3773
3879
|
compact: options.compact,
|
|
3774
3880
|
}),
|
|
3775
3881
|
});
|
|
3776
|
-
let contexts = { grantContext, profileRepinContext };
|
|
3882
|
+
let contexts = { grantContext, profileRepinContext, failedResumeContext };
|
|
3777
3883
|
dispatched = await dispatchCommand(contexts);
|
|
3778
3884
|
for (let retry = 0; action === "resume" && retry < 2; retry += 1) {
|
|
3779
3885
|
if (contexts.grantContext !== undefined && outpostRejectedGrantContextOption(dispatched)) {
|
|
@@ -3784,6 +3890,10 @@ async function dispatchOutpostLaneCommand(ctx, lane, action, options) {
|
|
|
3784
3890
|
outpostRejectedProfileRepinOption(dispatched)) {
|
|
3785
3891
|
contexts = { ...contexts, profileRepinContext: undefined };
|
|
3786
3892
|
}
|
|
3893
|
+
else if (options.fromFailed === true && outpostRejectedFailedResumeOption(dispatched)) {
|
|
3894
|
+
failedResumeOptionRejected = true;
|
|
3895
|
+
break;
|
|
3896
|
+
}
|
|
3787
3897
|
else {
|
|
3788
3898
|
break;
|
|
3789
3899
|
}
|
|
@@ -3800,6 +3910,13 @@ async function dispatchOutpostLaneCommand(ctx, lane, action, options) {
|
|
|
3800
3910
|
required_option: "--grant-context",
|
|
3801
3911
|
});
|
|
3802
3912
|
}
|
|
3913
|
+
if (failedResumeOptionRejected) {
|
|
3914
|
+
throw new LanesError("outpost_cli_upgrade_required", `Outpost ${outpostId} needs an hq-cli release containing failed-lane resume support before it can resume lane ${lane.lane_id}`, {
|
|
3915
|
+
lane_id: lane.lane_id,
|
|
3916
|
+
outpost_id: outpostId,
|
|
3917
|
+
required_option: "--from-failed",
|
|
3918
|
+
});
|
|
3919
|
+
}
|
|
3803
3920
|
const ownerError = ownerErrorFromOutput(dispatched.stdout);
|
|
3804
3921
|
if (options.effort !== undefined && outpostRejectedEffortOption(dispatched)) {
|
|
3805
3922
|
throw new LanesError("outpost_effort_unsupported", `Outpost ${outpostId} rejected --effort; update hq-cli on that Outpost before changing a lane's effort remotely`, {
|
|
@@ -3816,6 +3933,7 @@ async function dispatchOutpostLaneCommand(ctx, lane, action, options) {
|
|
|
3816
3933
|
if (ownerError) {
|
|
3817
3934
|
if (action === "resume" &&
|
|
3818
3935
|
(ownerError.error === "worker_profile_hash_mismatch" ||
|
|
3936
|
+
ownerError.error.startsWith("failed_resume_") ||
|
|
3819
3937
|
ownerError.error.startsWith("hire_grant_"))) {
|
|
3820
3938
|
throw new LanesError(ownerError.error, ownerError.message, {
|
|
3821
3939
|
...ownerError.details,
|
|
@@ -5015,6 +5133,7 @@ async function runShow(ctx, laneId) {
|
|
|
5015
5133
|
last_resume: lane.last_resume ?? null,
|
|
5016
5134
|
account: lane.account ?? null,
|
|
5017
5135
|
account_history: lane.account_history ?? [],
|
|
5136
|
+
failed_resume_history: lane.failed_resume_history ?? [],
|
|
5018
5137
|
pending_resume: lane.pending_resume ?? null,
|
|
5019
5138
|
last_run_usage: lane.last_run_usage ?? null,
|
|
5020
5139
|
model: lane.model ?? null,
|
|
@@ -6599,6 +6718,30 @@ function laneWithRecoveredProviderSession(hqRoot, lane) {
|
|
|
6599
6718
|
async function runResume(ctx, laneId, opts) {
|
|
6600
6719
|
const hqRoot = ctx.hqRoot();
|
|
6601
6720
|
let lane = readLane(hqRoot, laneId);
|
|
6721
|
+
const failedResumeReason = opts.reason?.trim();
|
|
6722
|
+
if (opts.reason !== undefined && opts.fromFailed !== true) {
|
|
6723
|
+
throw new LanesError("failed_resume_reason_requires_flag", "--reason on hq lanes resume requires --from-failed", { lane_id: laneId });
|
|
6724
|
+
}
|
|
6725
|
+
if (opts.fromFailed === true && !failedResumeReason) {
|
|
6726
|
+
throw new LanesError("failed_resume_reason_required", "--from-failed requires a non-empty --reason", { lane_id: laneId });
|
|
6727
|
+
}
|
|
6728
|
+
if (opts.reason !== undefined)
|
|
6729
|
+
assertNoSecretText("resume reason", opts.reason);
|
|
6730
|
+
if (opts.failedResumeContext !== undefined && opts.fromFailed !== true) {
|
|
6731
|
+
throw new LanesError("failed_resume_authorization_invalid", "Failed-resume authorization context requires --from-failed", { lane_id: laneId });
|
|
6732
|
+
}
|
|
6733
|
+
const failedResumeActor = opts.fromFailed === true
|
|
6734
|
+
? authorizeFailedResume(ctx, lane, opts.failedResumeContext)
|
|
6735
|
+
: undefined;
|
|
6736
|
+
if (opts.fromFailed === true && lane.state !== "failed") {
|
|
6737
|
+
throw new LanesError("lane_not_failed", `Lane ${laneId} is ${lane.state}; --from-failed only applies to a failed lane`, { lane_id: laneId, state: lane.state });
|
|
6738
|
+
}
|
|
6739
|
+
const failedResumeExpectedSenior = opts.fromFailed === true
|
|
6740
|
+
? { ...lane.senior }
|
|
6741
|
+
: undefined;
|
|
6742
|
+
const recoveredFailure = opts.fromFailed === true
|
|
6743
|
+
? lane.terminal_reason ?? "failure_reason_unavailable"
|
|
6744
|
+
: undefined;
|
|
6602
6745
|
const hireGrantContext = opts.grantContext === undefined
|
|
6603
6746
|
? undefined
|
|
6604
6747
|
: (() => {
|
|
@@ -6639,7 +6782,7 @@ async function runResume(ctx, laneId, opts) {
|
|
|
6639
6782
|
if (lane.state === "queued") {
|
|
6640
6783
|
throw new LanesError("lane_not_resumable", `Lane ${laneId} is queued; use hq lanes admit ${laneId} to request admission`, { lane_id: laneId, state: lane.state, admission_command: `hq lanes admit ${laneId}` });
|
|
6641
6784
|
}
|
|
6642
|
-
const refusal = resumeCommandRefusal(lane);
|
|
6785
|
+
const refusal = resumeCommandRefusal(lane, { allowFailed: opts.fromFailed === true });
|
|
6643
6786
|
if (refusal !== null) {
|
|
6644
6787
|
throw new LanesError(refusal.error, refusal.message, {
|
|
6645
6788
|
lane_id: laneId,
|
|
@@ -6709,6 +6852,10 @@ async function runResume(ctx, laneId, opts) {
|
|
|
6709
6852
|
pending: {
|
|
6710
6853
|
trigger: "resume",
|
|
6711
6854
|
resumed_from: resumedFrom,
|
|
6855
|
+
...(opts.fromFailed === true ? { from_failed: true } : {}),
|
|
6856
|
+
...(failedResumeExpectedSenior !== undefined
|
|
6857
|
+
? { expected_senior: failedResumeExpectedSenior }
|
|
6858
|
+
: {}),
|
|
6712
6859
|
...(text !== undefined ? { text } : {}),
|
|
6713
6860
|
...(messageId !== undefined ? { message_id: messageId } : {}),
|
|
6714
6861
|
compact: opts.compact !== false,
|
|
@@ -6734,6 +6881,10 @@ async function runResume(ctx, laneId, opts) {
|
|
|
6734
6881
|
pending: {
|
|
6735
6882
|
trigger: "resume",
|
|
6736
6883
|
resumed_from: resumedFrom,
|
|
6884
|
+
...(opts.fromFailed === true ? { from_failed: true } : {}),
|
|
6885
|
+
...(failedResumeExpectedSenior !== undefined
|
|
6886
|
+
? { expected_senior: failedResumeExpectedSenior }
|
|
6887
|
+
: {}),
|
|
6737
6888
|
...(text !== undefined ? { text } : {}),
|
|
6738
6889
|
...(messageId !== undefined ? { message_id: messageId } : {}),
|
|
6739
6890
|
compact: opts.compact !== false,
|
|
@@ -6794,6 +6945,10 @@ async function runResume(ctx, laneId, opts) {
|
|
|
6794
6945
|
session_id: sessionId,
|
|
6795
6946
|
trigger: "resume",
|
|
6796
6947
|
resumed_from: resumedFrom,
|
|
6948
|
+
...(opts.fromFailed === true ? { from_failed: true } : {}),
|
|
6949
|
+
...(failedResumeExpectedSenior !== undefined
|
|
6950
|
+
? { expected_senior: failedResumeExpectedSenior }
|
|
6951
|
+
: {}),
|
|
6797
6952
|
...(text !== undefined ? { text } : {}),
|
|
6798
6953
|
...(messageId !== undefined ? { message_id: messageId } : {}),
|
|
6799
6954
|
compact: opts.compact !== false,
|
|
@@ -6821,6 +6976,10 @@ async function runResume(ctx, laneId, opts) {
|
|
|
6821
6976
|
trigger: {
|
|
6822
6977
|
kind: "resume",
|
|
6823
6978
|
from: resumedFrom,
|
|
6979
|
+
...(opts.fromFailed === true ? { fromFailed: true } : {}),
|
|
6980
|
+
...(failedResumeExpectedSenior !== undefined
|
|
6981
|
+
? { expectedSenior: failedResumeExpectedSenior }
|
|
6982
|
+
: {}),
|
|
6824
6983
|
...(text !== undefined ? { text } : {}),
|
|
6825
6984
|
...(messageId !== undefined ? { message_id: messageId } : {}),
|
|
6826
6985
|
},
|
|
@@ -6843,6 +7002,19 @@ async function runResume(ctx, laneId, opts) {
|
|
|
6843
7002
|
});
|
|
6844
7003
|
}
|
|
6845
7004
|
let next = resumed;
|
|
7005
|
+
if (failedResumeActor !== undefined &&
|
|
7006
|
+
failedResumeReason !== undefined &&
|
|
7007
|
+
recoveredFailure !== undefined) {
|
|
7008
|
+
next = casUpdateLane(hqRoot, laneId, (current) => ({
|
|
7009
|
+
...current,
|
|
7010
|
+
failed_resume_history: appendCappedHistory(current.failed_resume_history ?? [], {
|
|
7011
|
+
at,
|
|
7012
|
+
actor: failedResumeActor,
|
|
7013
|
+
reason: failedResumeReason,
|
|
7014
|
+
failure: recoveredFailure,
|
|
7015
|
+
}),
|
|
7016
|
+
}));
|
|
7017
|
+
}
|
|
6846
7018
|
if (text !== undefined) {
|
|
6847
7019
|
// An operator text moves the compaction reference forward, as a message does.
|
|
6848
7020
|
next = casUpdateLane(hqRoot, laneId, (current) => ({
|
|
@@ -9492,6 +9664,14 @@ export function registerLanesCommand(parent, deps = {}) {
|
|
|
9492
9664
|
.action(wrap(async (laneId, opts) => {
|
|
9493
9665
|
await runAdmit(ctx, laneId, opts);
|
|
9494
9666
|
}));
|
|
9667
|
+
lanes
|
|
9668
|
+
.command("wait")
|
|
9669
|
+
.description("Wait for the first requested change across lanes")
|
|
9670
|
+
.option("--any <lane-id...>", "Lane ids to monitor")
|
|
9671
|
+
.option("--for <change-type>", "envelope | question | state", collectRepeatable, [])
|
|
9672
|
+
.option("--timeout <seconds>", "Positive timeout in seconds")
|
|
9673
|
+
.option("--json", "JSON output")
|
|
9674
|
+
.action(wrapText(async (opts) => runWait(ctx, opts)));
|
|
9495
9675
|
lanes
|
|
9496
9676
|
.command("watch")
|
|
9497
9677
|
.description("Watch lane state and question changes until all lanes are terminal")
|
|
@@ -9616,9 +9796,11 @@ export function registerLanesCommand(parent, deps = {}) {
|
|
|
9616
9796
|
}));
|
|
9617
9797
|
lanes
|
|
9618
9798
|
.command("resume")
|
|
9619
|
-
.description("Resume an interrupted, done or blocked lane on its recorded provider session")
|
|
9799
|
+
.description("Resume an interrupted, done or blocked lane on its recorded provider session; recover failed lanes with the senior-only --from-failed option")
|
|
9620
9800
|
.argument("<lane-id>", "Lane id")
|
|
9621
9801
|
.option("--text <msg>", "Operator message for the resumed turn")
|
|
9802
|
+
.option("--from-failed", "Explicitly resume a failed lane as its senior")
|
|
9803
|
+
.option("--reason <text>", "Required explanation for resuming from failed")
|
|
9622
9804
|
.option("--project <slug>", "Re-point project attribution from this resume")
|
|
9623
9805
|
.option("--story <id>", "Re-point story attribution from this resume")
|
|
9624
9806
|
.option("--account <id>", "Move the lane's session to this account of its provider before resuming")
|
|
@@ -9627,6 +9809,7 @@ export function registerLanesCommand(parent, deps = {}) {
|
|
|
9627
9809
|
.option("--worker <profile>", "Attach a personal worker profile for the resumed turn")
|
|
9628
9810
|
.addOption(new Option("--profile-repin-context <base64url>").hideHelp())
|
|
9629
9811
|
.addOption(new Option("--grant-context <base64url>").hideHelp())
|
|
9812
|
+
.addOption(new Option("--failed-resume-context <base64url>").hideHelp())
|
|
9630
9813
|
.option("--keep-alive", "Automatically resume done lead lanes while their story remains open")
|
|
9631
9814
|
.option("--mesh-company <slug>", "Attach Work Mesh attribution after verifying company membership")
|
|
9632
9815
|
.option("--no-compact", "Skip the resume-time compaction check for this resume")
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
* module rather than testing provider or state inline. A guard bound at one
|
|
18
18
|
* site and not another is the defect class this branch has already shipped.
|
|
19
19
|
*/
|
|
20
|
-
import { type LaneProvider, type LaneRecord, type LaneState } from "./types.js";
|
|
20
|
+
import { type LaneProvider, type LaneRecord, type LaneState, type SeniorRef } from "./types.js";
|
|
21
21
|
/** Which providers can continue an exited worker from a recorded session id. */
|
|
22
22
|
export declare const PROVIDER_SESSION_RESUME: Record<LaneProvider, boolean>;
|
|
23
23
|
export declare function providerSupportsSessionResume(provider: LaneProvider): boolean;
|
|
@@ -45,7 +45,8 @@ export declare function sessionNotPortableMessage(provider: LaneProvider): strin
|
|
|
45
45
|
* reconcile pass resumes it once every question is answered. `killed` and
|
|
46
46
|
* `cancelled` were stopped on purpose by the operator; a message must not
|
|
47
47
|
* silently restart them. `failed` means the run ended without an envelope or
|
|
48
|
-
* never started; the
|
|
48
|
+
* never started; only the lane's senior can explicitly recover it with
|
|
49
|
+
* `hq lanes resume <lane-id> --from-failed --reason <text>`. `interrupted` exited with
|
|
49
50
|
* partial output and no envelope, so the lane's state was never closed
|
|
50
51
|
* cleanly; only the explicit `hq lanes resume` opt-in continues it
|
|
51
52
|
* (RESUME_COMMAND_STATES). `queued` and `running` take the drop box, not a
|
|
@@ -139,6 +140,9 @@ export type LaunchTrigger = {
|
|
|
139
140
|
kind: "resume";
|
|
140
141
|
/** Lane state at the moment of the command; recorded as `last_resume.resumed_from`. */
|
|
141
142
|
from: ResumeCommandState | "failed";
|
|
143
|
+
fromFailed?: true;
|
|
144
|
+
/** Senior that authorized a failed-lane resume, rechecked at the claim CAS. */
|
|
145
|
+
expectedSenior?: SeniorRef;
|
|
142
146
|
text?: string;
|
|
143
147
|
message_id?: string;
|
|
144
148
|
};
|
|
@@ -152,7 +156,9 @@ export type LaunchPlan = {
|
|
|
152
156
|
* Refusal for `hq lanes resume`, in precedence order: wrong state, provider
|
|
153
157
|
* without a resume path, no recorded session. Null when the resume can go.
|
|
154
158
|
*/
|
|
155
|
-
export declare function resumeCommandRefusal(lane: DeliveryLane
|
|
159
|
+
export declare function resumeCommandRefusal(lane: DeliveryLane, options?: {
|
|
160
|
+
allowFailed?: boolean;
|
|
161
|
+
}): {
|
|
156
162
|
error: "lane_not_resumable" | "resume_unsupported" | "no_provider_session";
|
|
157
163
|
message: string;
|
|
158
164
|
} | null;
|
|
@@ -65,7 +65,8 @@ export function sessionNotPortableMessage(provider) {
|
|
|
65
65
|
* reconcile pass resumes it once every question is answered. `killed` and
|
|
66
66
|
* `cancelled` were stopped on purpose by the operator; a message must not
|
|
67
67
|
* silently restart them. `failed` means the run ended without an envelope or
|
|
68
|
-
* never started; the
|
|
68
|
+
* never started; only the lane's senior can explicitly recover it with
|
|
69
|
+
* `hq lanes resume <lane-id> --from-failed --reason <text>`. `interrupted` exited with
|
|
69
70
|
* partial output and no envelope, so the lane's state was never closed
|
|
70
71
|
* cleanly; only the explicit `hq lanes resume` opt-in continues it
|
|
71
72
|
* (RESUME_COMMAND_STATES). `queued` and `running` take the drop box, not a
|
|
@@ -137,7 +138,7 @@ export function deliveryRefusalReason(lane) {
|
|
|
137
138
|
case "interrupted":
|
|
138
139
|
return `Lane is ${state}. The run did not close cleanly and the operator has not decided what to do with it. Resuming from ${state} needs an explicit opt-in: hq lanes resume <lane-id> [--text <message>].`;
|
|
139
140
|
case "failed":
|
|
140
|
-
return `Lane is ${state}.
|
|
141
|
+
return `Lane is ${state}. A plain message does not restart it. Its senior can resume the recorded session with: hq lanes resume <lane-id> --from-failed --reason <text>.`;
|
|
141
142
|
default: {
|
|
142
143
|
const _exhaustive = state;
|
|
143
144
|
return _exhaustive;
|
|
@@ -345,8 +346,10 @@ export const LAUNCH_TRIGGERS = ["admit", "answered_questions", "message", "resum
|
|
|
345
346
|
* Refusal for `hq lanes resume`, in precedence order: wrong state, provider
|
|
346
347
|
* without a resume path, no recorded session. Null when the resume can go.
|
|
347
348
|
*/
|
|
348
|
-
export function resumeCommandRefusal(lane) {
|
|
349
|
-
if (!isResumeCommandState(lane.state) &&
|
|
349
|
+
export function resumeCommandRefusal(lane, options = {}) {
|
|
350
|
+
if (!isResumeCommandState(lane.state) &&
|
|
351
|
+
!isFailedEnvelopeRepairLane(lane) &&
|
|
352
|
+
!(options.allowFailed === true && lane.state === "failed")) {
|
|
350
353
|
return {
|
|
351
354
|
error: "lane_not_resumable",
|
|
352
355
|
message: `Lane is ${lane.state}. hq lanes resume continues a lane in ${RESUME_COMMAND_STATES.join(", ")}; ${lane.state === "awaiting_input"
|
|
@@ -395,7 +398,7 @@ export function planLaneLaunch(lane, trigger) {
|
|
|
395
398
|
return { mode: "resume", session_id: lane.provider_session_id };
|
|
396
399
|
}
|
|
397
400
|
case "resume": {
|
|
398
|
-
const refusal = resumeCommandRefusal(lane);
|
|
401
|
+
const refusal = resumeCommandRefusal(lane, { allowFailed: trigger.fromFailed === true });
|
|
399
402
|
if (refusal !== null || !hasRecordedSessionId(lane)) {
|
|
400
403
|
throw new LanesError(refusal?.error ?? "no_provider_session", refusal?.message ?? `Lane is ${lane.state} and has no recorded provider session`, { state: lane.state, provider: lane.provider });
|
|
401
404
|
}
|
|
@@ -418,7 +421,9 @@ export function claimableStatesFor(trigger, lane) {
|
|
|
418
421
|
case "message":
|
|
419
422
|
return envelopeRepairFailed ? [...RESUMABLE_LANE_STATES, "failed"] : RESUMABLE_LANE_STATES;
|
|
420
423
|
case "resume":
|
|
421
|
-
return
|
|
424
|
+
return trigger.fromFailed === true || envelopeRepairFailed
|
|
425
|
+
? [...RESUME_COMMAND_STATES, "failed"]
|
|
426
|
+
: RESUME_COMMAND_STATES;
|
|
422
427
|
default: {
|
|
423
428
|
const _exhaustive = trigger;
|
|
424
429
|
return _exhaustive;
|
|
@@ -151,12 +151,15 @@ export function buildOutpostLaneCommand(input) {
|
|
|
151
151
|
["--hire", input.hire],
|
|
152
152
|
["--grant-context", input.grantContext],
|
|
153
153
|
["--profile-repin-context", input.profileRepinContext],
|
|
154
|
+
["--failed-resume-context", input.failedResumeContext],
|
|
154
155
|
["--senior", input.senior],
|
|
155
156
|
["--reason", input.reason],
|
|
156
157
|
]) {
|
|
157
158
|
if (value !== undefined)
|
|
158
159
|
args.push(flag, value);
|
|
159
160
|
}
|
|
161
|
+
if (input.action === "resume" && input.fromFailed === true)
|
|
162
|
+
args.push("--from-failed");
|
|
160
163
|
if (input.action === "resume" && input.keepAlive === true)
|
|
161
164
|
args.push("--keep-alive");
|
|
162
165
|
if (input.compact === false)
|
package/dist/lib/lanes/spawn.js
CHANGED
|
@@ -1576,6 +1576,9 @@ function laneLaunchResult(lane) {
|
|
|
1576
1576
|
const { provider_home_at_spawn: _providerHomeAtSpawn, ...safe } = lane;
|
|
1577
1577
|
return safe;
|
|
1578
1578
|
}
|
|
1579
|
+
function sameSeniorRef(a, b) {
|
|
1580
|
+
return a.kind === b.kind && a.id === b.id;
|
|
1581
|
+
}
|
|
1579
1582
|
function inferTrigger(lane) {
|
|
1580
1583
|
if (lane.state === "awaiting_input")
|
|
1581
1584
|
return { kind: "answered_questions" };
|
|
@@ -1631,6 +1634,12 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
|
|
|
1631
1634
|
// starting fresh from the brief.
|
|
1632
1635
|
const pendingResume = launch.trigger === undefined && existing.state === "queued" ? existing.pending_resume : undefined;
|
|
1633
1636
|
const trigger = launch.trigger ?? inferTrigger(existing);
|
|
1637
|
+
const failedResumeClaim = (trigger.kind === "resume" && trigger.fromFailed === true) || pendingResume?.from_failed === true;
|
|
1638
|
+
const failedResumeExpectedSenior = trigger.kind === "resume" && trigger.fromFailed === true
|
|
1639
|
+
? trigger.expectedSenior
|
|
1640
|
+
: pendingResume?.from_failed === true
|
|
1641
|
+
? pendingResume.expected_senior
|
|
1642
|
+
: undefined;
|
|
1634
1643
|
// A message into a finished lane must not turn that lane `failed` when the
|
|
1635
1644
|
// account cannot be bound: nothing has started yet, so the record keeps its
|
|
1636
1645
|
// state and the caller sees the error. Admission and question resume keep
|
|
@@ -1971,6 +1980,15 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
|
|
|
1971
1980
|
if (!claimableStatesFor(trigger, current).includes(current.state)) {
|
|
1972
1981
|
throw new LanesError("spawn_state_mismatch", `Cannot mark ${current.lane_id} running from ${current.state}`, { lane_id: current.lane_id, state: current.state, trigger: trigger.kind });
|
|
1973
1982
|
}
|
|
1983
|
+
if (failedResumeClaim &&
|
|
1984
|
+
(failedResumeExpectedSenior === undefined ||
|
|
1985
|
+
!sameSeniorRef(current.senior, failedResumeExpectedSenior))) {
|
|
1986
|
+
throw new LanesError("failed_resume_senior_changed", `Lane ${current.lane_id}'s senior changed after its failed-resume request was authorized`, {
|
|
1987
|
+
lane_id: current.lane_id,
|
|
1988
|
+
expected_senior: failedResumeExpectedSenior ?? null,
|
|
1989
|
+
current_senior: current.senior,
|
|
1990
|
+
});
|
|
1991
|
+
}
|
|
1974
1992
|
claimed = true;
|
|
1975
1993
|
const next = {
|
|
1976
1994
|
...current,
|
|
@@ -2044,6 +2062,7 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
|
|
|
2044
2062
|
...(pendingResume.message_id !== undefined ? { message_id: pendingResume.message_id } : {}),
|
|
2045
2063
|
...(pendingResume.monitor_event === true ? { monitor_event: true } : {}),
|
|
2046
2064
|
...(pendingResume.resumed_from !== undefined ? { resumed_from: pendingResume.resumed_from } : {}),
|
|
2065
|
+
...(pendingResume.from_failed === true ? { from_failed: true } : {}),
|
|
2047
2066
|
...(pendingResume.text !== undefined ? { text: pendingResume.text } : {}),
|
|
2048
2067
|
...(pendingResume.compact !== undefined ? { compact: pendingResume.compact } : {}),
|
|
2049
2068
|
};
|
|
@@ -2065,6 +2084,7 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
|
|
|
2065
2084
|
...(trigger.kind === "resume"
|
|
2066
2085
|
? {
|
|
2067
2086
|
resumed_from: trigger.from,
|
|
2087
|
+
...(trigger.fromFailed === true ? { from_failed: true } : {}),
|
|
2068
2088
|
...(trigger.text !== undefined ? { text: trigger.text } : {}),
|
|
2069
2089
|
...(trigger.message_id !== undefined ? { message_id: trigger.message_id } : {}),
|
|
2070
2090
|
}
|
package/dist/lib/lanes/store.js
CHANGED
|
@@ -888,6 +888,11 @@ export function mergeLaneRecords(live, sidecars) {
|
|
|
888
888
|
worker_profile_repin_history: unionHistory(all.map((record) => record.worker_profile_repin_history ?? []), entryIdentity, base.worker_profile_repin_history ?? []),
|
|
889
889
|
}
|
|
890
890
|
: {}),
|
|
891
|
+
...(all.some((record) => record.failed_resume_history !== undefined)
|
|
892
|
+
? {
|
|
893
|
+
failed_resume_history: unionHistory(all.map((record) => record.failed_resume_history ?? []), entryIdentity, base.failed_resume_history ?? []),
|
|
894
|
+
}
|
|
895
|
+
: {}),
|
|
891
896
|
stories: mergeLaneStories(all, base),
|
|
892
897
|
};
|
|
893
898
|
}
|
|
@@ -400,6 +400,8 @@ export interface LaneRecord {
|
|
|
400
400
|
* like the other histories.
|
|
401
401
|
*/
|
|
402
402
|
account_history?: AccountHistoryEntry[];
|
|
403
|
+
/** Bounded audit history for explicitly authorized resumes from failed. */
|
|
404
|
+
failed_resume_history?: FailedResumeHistoryEntry[];
|
|
403
405
|
/**
|
|
404
406
|
* Set by an account move (`--account`) until the spawn claims the lane:
|
|
405
407
|
* the turn the move was asked to run. While set it reserves a slot on the
|
|
@@ -537,11 +539,20 @@ export interface AccountHistoryEntry {
|
|
|
537
539
|
at: string;
|
|
538
540
|
reason: "first_assignment" | "model_change" | "usage_limit" | "account_unavailable" | "recorded_account_logged_out" | "load_balance" | "operator_move";
|
|
539
541
|
}
|
|
542
|
+
export interface FailedResumeHistoryEntry {
|
|
543
|
+
at: string;
|
|
544
|
+
actor: string;
|
|
545
|
+
reason: string;
|
|
546
|
+
failure: string;
|
|
547
|
+
}
|
|
540
548
|
export interface PendingResume {
|
|
541
549
|
/** Fresh answered-question launches need no previous session id. */
|
|
542
550
|
session_id?: string;
|
|
543
551
|
trigger: "answered_questions" | "message" | "resume";
|
|
544
552
|
resumed_from?: "interrupted" | "done" | "blocked" | "failed";
|
|
553
|
+
from_failed?: true;
|
|
554
|
+
/** Senior that authorized a failed-lane resume, rechecked at the claim CAS. */
|
|
555
|
+
expected_senior?: SeniorRef;
|
|
545
556
|
text?: string;
|
|
546
557
|
message_id?: string;
|
|
547
558
|
/** The text is already framed as untrusted monitor data, not an operator instruction. */
|
|
@@ -572,6 +583,7 @@ export interface LaneResumeRecord {
|
|
|
572
583
|
* resumed it (`interrupted`, `done` or `blocked`) and the text they sent.
|
|
573
584
|
*/
|
|
574
585
|
resumed_from?: "interrupted" | "done" | "blocked" | "failed";
|
|
586
|
+
from_failed?: true;
|
|
575
587
|
text?: string;
|
|
576
588
|
/** False when the continued turn explicitly skipped compaction. */
|
|
577
589
|
compact?: boolean;
|
package/dist/lib/lanes/types.js
CHANGED
|
@@ -225,7 +225,7 @@ export const LANE_FIELD_WRITERS = {
|
|
|
225
225
|
pickup_validation_pending_since: ["creator", "command", "spawn", "reconcile"],
|
|
226
226
|
priority: ["command"], worker: ["creator", "command"], worker_path: ["creator", "command"], worker_pack: ["creator", "command"], worker_profile: ["creator", "command"], worker_profile_hash: ["creator", "command", "spawn"], worker_profile_repin_history: ["command"], worker_profile_ignored: ["creator", "command", "spawn"], worker_policy_delivery: ["command", "spawn"],
|
|
227
227
|
keep_alive: ["creator", "command"], keep_alive_standing: ["command"], keep_alive_opted_in_at: ["creator", "command"], keep_alive_blocked_reason: ["command", "reconcile"], keep_alive_resume_count: ["command", "reconcile"], keep_alive_last_resume_at: ["command", "reconcile"], keep_alive_next_eligible_at: ["command", "reconcile"], keep_alive_exhausted_at: ["command", "reconcile"],
|
|
228
|
-
provider: ["creator"], account: ["creator", "spawn", "command", "watcher", "reconcile"], account_selection: ["creator", "command"], admission_accounts: ["creator", "command"], account_history: ["command", "spawn", "watcher", "reconcile"], pending_resume: ["command", "spawn", "watcher", "reconcile"], placement_default: ["creator"], host: ["creator", "command"], origin_host: ["creator"], origin_host_source: ["creator"],
|
|
228
|
+
provider: ["creator"], account: ["creator", "spawn", "command", "watcher", "reconcile"], account_selection: ["creator", "command"], admission_accounts: ["creator", "command"], account_history: ["command", "spawn", "watcher", "reconcile"], failed_resume_history: ["command"], pending_resume: ["command", "spawn", "watcher", "reconcile"], placement_default: ["creator"], host: ["creator", "command"], origin_host: ["creator"], origin_host_source: ["creator"],
|
|
229
229
|
version: ["creator"], created_at: ["creator"], updated_at: ["creator"], attribution_history: ["command"],
|
|
230
230
|
last_envelope: ["watcher", "reconcile"], model: ["creator", "spawn", "command"], model_source: ["creator", "spawn", "command"], effort: ["creator", "spawn", "command"], effort_source: ["creator", "spawn", "command"], fast_mode: ["spawn"],
|
|
231
231
|
last_message_at: ["command"], last_compaction: ["spawn"], story_complete: ["watcher", "reconcile"], story_complete_reason: ["watcher", "reconcile"], story_status: ["watcher", "reconcile"],
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import type { LaneRecord, QuestionRecord } from "./types.js";
|
|
2
|
+
export declare const LANE_WAIT_POLL_INTERVAL_MS = 2000;
|
|
3
|
+
export declare const LANE_WAIT_MAX_LANES = 64;
|
|
4
|
+
export declare const LANE_WAIT_TIMEOUT_EXIT_CODE = 4;
|
|
5
|
+
export declare const LANE_WAIT_INVALID_INPUT_EXIT_CODE = 2;
|
|
6
|
+
export declare const LANE_WAIT_CHANGE_TYPES: readonly ["envelope", "question", "state"];
|
|
7
|
+
export type LaneWaitChangeType = (typeof LANE_WAIT_CHANGE_TYPES)[number];
|
|
8
|
+
export type LaneWaitChange = {
|
|
9
|
+
type: "envelope";
|
|
10
|
+
ref: string;
|
|
11
|
+
decision: "done" | "ask" | "blocked";
|
|
12
|
+
} | {
|
|
13
|
+
type: "question";
|
|
14
|
+
question_id: string;
|
|
15
|
+
} | {
|
|
16
|
+
type: "state";
|
|
17
|
+
from: string;
|
|
18
|
+
to: string;
|
|
19
|
+
};
|
|
20
|
+
export interface LaneWaitResult {
|
|
21
|
+
ok: true;
|
|
22
|
+
lane_id: string;
|
|
23
|
+
change: LaneWaitChange;
|
|
24
|
+
at: string;
|
|
25
|
+
}
|
|
26
|
+
export interface LaneWaitDeps {
|
|
27
|
+
readLane?: (hqRoot: string, laneId: string) => LaneRecord | null;
|
|
28
|
+
listQuestionIds?: (hqRoot: string) => string[];
|
|
29
|
+
readQuestion?: (hqRoot: string, questionId: string) => QuestionRecord | null;
|
|
30
|
+
now?: () => Date;
|
|
31
|
+
sleep?: (ms: number) => Promise<void>;
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Wait for the first requested change after taking a read-only baseline.
|
|
35
|
+
* Each poll reads every selected lane once and runs no lane-store mutation or
|
|
36
|
+
* lock. Question records are scanned in one pass when question changes matter.
|
|
37
|
+
*/
|
|
38
|
+
export declare function runLaneWait(hqRoot: string, laneIds: readonly string[], changeTypes: readonly string[], timeoutMs: number, deps?: LaneWaitDeps): Promise<LaneWaitResult | null>;
|
|
39
|
+
//# sourceMappingURL=wait.d.ts.map
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
import { listQuestionIds, readLane, readQuestion } from "./store.js";
|
|
2
|
+
import { LanesError } from "./errors.js";
|
|
3
|
+
export const LANE_WAIT_POLL_INTERVAL_MS = 2_000;
|
|
4
|
+
export const LANE_WAIT_MAX_LANES = 64;
|
|
5
|
+
export const LANE_WAIT_TIMEOUT_EXIT_CODE = 4;
|
|
6
|
+
export const LANE_WAIT_INVALID_INPUT_EXIT_CODE = 2;
|
|
7
|
+
export const LANE_WAIT_CHANGE_TYPES = ["envelope", "question", "state"];
|
|
8
|
+
function defaultSleep(ms) {
|
|
9
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
10
|
+
}
|
|
11
|
+
function invalidInput(message, details = {}) {
|
|
12
|
+
throw new LanesError("wait_invalid_input", message, details);
|
|
13
|
+
}
|
|
14
|
+
function validateInput(laneIds, changeTypes, timeoutMs) {
|
|
15
|
+
const normalizedLaneIds = laneIds.map((laneId) => laneId.trim());
|
|
16
|
+
if (normalizedLaneIds.length === 0 || normalizedLaneIds.some((laneId) => laneId.length === 0)) {
|
|
17
|
+
invalidInput("hq lanes wait requires at least one non-empty lane id", { lane_ids: laneIds });
|
|
18
|
+
}
|
|
19
|
+
if (normalizedLaneIds.some((laneId) => laneId === "." || laneId === ".." || /[\\/]/.test(laneId))) {
|
|
20
|
+
invalidInput("Lane ids must not contain path separators", { lane_ids: laneIds });
|
|
21
|
+
}
|
|
22
|
+
const uniqueLaneIds = [...new Set(normalizedLaneIds)];
|
|
23
|
+
if (uniqueLaneIds.length > LANE_WAIT_MAX_LANES) {
|
|
24
|
+
invalidInput(`hq lanes wait accepts at most ${LANE_WAIT_MAX_LANES} lane ids`, {
|
|
25
|
+
max: LANE_WAIT_MAX_LANES,
|
|
26
|
+
});
|
|
27
|
+
}
|
|
28
|
+
if (changeTypes.length === 0 || changeTypes.some((kind) => !LANE_WAIT_CHANGE_TYPES.includes(kind))) {
|
|
29
|
+
invalidInput(`hq lanes wait requires one or more --for values: ${LANE_WAIT_CHANGE_TYPES.join(", ")}`, { accepted: [...LANE_WAIT_CHANGE_TYPES], values: changeTypes });
|
|
30
|
+
}
|
|
31
|
+
if (!Number.isFinite(timeoutMs) || timeoutMs <= 0 || !Number.isSafeInteger(Math.ceil(timeoutMs))) {
|
|
32
|
+
invalidInput("hq lanes wait requires a positive finite timeout", { timeout_ms: timeoutMs });
|
|
33
|
+
}
|
|
34
|
+
return {
|
|
35
|
+
laneIds: uniqueLaneIds,
|
|
36
|
+
changeTypes: [...new Set(changeTypes)],
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
function snapshotLanes(hqRoot, laneIds, changeTypes, deps) {
|
|
40
|
+
const readLaneRecord = deps.readLane ?? readLane;
|
|
41
|
+
const lanes = new Map();
|
|
42
|
+
for (const laneId of laneIds) {
|
|
43
|
+
const lane = readLaneRecord(hqRoot, laneId);
|
|
44
|
+
if (!lane) {
|
|
45
|
+
throw new LanesError("wait_lane_unreadable", `Lane ${laneId} could not be read while waiting`, { lane_id: laneId });
|
|
46
|
+
}
|
|
47
|
+
lanes.set(laneId, lane);
|
|
48
|
+
}
|
|
49
|
+
const pendingQuestions = new Map();
|
|
50
|
+
if (changeTypes.includes("question")) {
|
|
51
|
+
const listIds = deps.listQuestionIds ?? listQuestionIds;
|
|
52
|
+
const readQuestionRecord = deps.readQuestion ?? readQuestion;
|
|
53
|
+
for (const questionId of new Set(listIds(hqRoot))) {
|
|
54
|
+
const question = readQuestionRecord(hqRoot, questionId);
|
|
55
|
+
if (!question) {
|
|
56
|
+
throw new LanesError("wait_question_unreadable", `Question ${questionId} disappeared while waiting`, { question_id: questionId });
|
|
57
|
+
}
|
|
58
|
+
if (question.status === "pending" && lanes.has(question.lane_id)) {
|
|
59
|
+
pendingQuestions.set(questionId, question);
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
return { lanes, pendingQuestions };
|
|
64
|
+
}
|
|
65
|
+
function timestamp(value, fallbackMs) {
|
|
66
|
+
const parsed = value === undefined ? Number.NaN : Date.parse(value);
|
|
67
|
+
const atMs = Number.isFinite(parsed) ? parsed : fallbackMs;
|
|
68
|
+
return { at: new Date(atMs).toISOString(), atMs };
|
|
69
|
+
}
|
|
70
|
+
function envelopeSignature(lane) {
|
|
71
|
+
const envelope = lane.last_envelope;
|
|
72
|
+
return envelope === undefined
|
|
73
|
+
? undefined
|
|
74
|
+
: `${envelope.ref}\0${envelope.decision}\0${envelope.at}`;
|
|
75
|
+
}
|
|
76
|
+
function candidatesForSnapshot(previous, current, laneIds, changeTypes, observedAtMs) {
|
|
77
|
+
const candidates = [];
|
|
78
|
+
laneIds.forEach((laneId, laneOrder) => {
|
|
79
|
+
const before = previous.lanes.get(laneId);
|
|
80
|
+
const after = current.lanes.get(laneId);
|
|
81
|
+
if (!before || !after)
|
|
82
|
+
return;
|
|
83
|
+
changeTypes.forEach((type, typeOrder) => {
|
|
84
|
+
if (type === "envelope" && envelopeSignature(before) !== envelopeSignature(after)) {
|
|
85
|
+
const envelope = after.last_envelope;
|
|
86
|
+
if (envelope) {
|
|
87
|
+
const when = timestamp(envelope.at, observedAtMs);
|
|
88
|
+
candidates.push({
|
|
89
|
+
laneId,
|
|
90
|
+
change: { type, ref: envelope.ref, decision: envelope.decision },
|
|
91
|
+
...when,
|
|
92
|
+
laneOrder,
|
|
93
|
+
typeOrder,
|
|
94
|
+
});
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
if (type === "state" && before.state !== after.state) {
|
|
98
|
+
const when = timestamp(after.updated_at, observedAtMs);
|
|
99
|
+
candidates.push({
|
|
100
|
+
laneId,
|
|
101
|
+
change: { type, from: before.state, to: after.state },
|
|
102
|
+
...when,
|
|
103
|
+
laneOrder,
|
|
104
|
+
typeOrder,
|
|
105
|
+
});
|
|
106
|
+
}
|
|
107
|
+
if (type === "question") {
|
|
108
|
+
for (const [questionId, question] of current.pendingQuestions) {
|
|
109
|
+
if (question.lane_id !== laneId || previous.pendingQuestions.has(questionId))
|
|
110
|
+
continue;
|
|
111
|
+
const when = timestamp(question.asked_at, observedAtMs);
|
|
112
|
+
candidates.push({
|
|
113
|
+
laneId,
|
|
114
|
+
change: { type, question_id: questionId },
|
|
115
|
+
...when,
|
|
116
|
+
laneOrder,
|
|
117
|
+
typeOrder,
|
|
118
|
+
});
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
});
|
|
122
|
+
});
|
|
123
|
+
return candidates.sort((a, b) => a.atMs - b.atMs || a.laneOrder - b.laneOrder || a.typeOrder - b.typeOrder);
|
|
124
|
+
}
|
|
125
|
+
/**
|
|
126
|
+
* Wait for the first requested change after taking a read-only baseline.
|
|
127
|
+
* Each poll reads every selected lane once and runs no lane-store mutation or
|
|
128
|
+
* lock. Question records are scanned in one pass when question changes matter.
|
|
129
|
+
*/
|
|
130
|
+
export async function runLaneWait(hqRoot, laneIds, changeTypes, timeoutMs, deps = {}) {
|
|
131
|
+
const input = validateInput(laneIds, changeTypes, timeoutMs);
|
|
132
|
+
const now = deps.now ?? (() => new Date());
|
|
133
|
+
const sleep = deps.sleep ?? defaultSleep;
|
|
134
|
+
const startedAtMs = now().getTime();
|
|
135
|
+
const deadlineMs = startedAtMs + timeoutMs;
|
|
136
|
+
let previous = snapshotLanes(hqRoot, input.laneIds, input.changeTypes, deps);
|
|
137
|
+
while (true) {
|
|
138
|
+
const remainingMs = deadlineMs - now().getTime();
|
|
139
|
+
if (remainingMs <= 0)
|
|
140
|
+
return null;
|
|
141
|
+
await sleep(Math.min(LANE_WAIT_POLL_INTERVAL_MS, remainingMs));
|
|
142
|
+
const observedAtMs = now().getTime();
|
|
143
|
+
const current = snapshotLanes(hqRoot, input.laneIds, input.changeTypes, deps);
|
|
144
|
+
const candidates = candidatesForSnapshot(previous, current, input.laneIds, input.changeTypes, observedAtMs);
|
|
145
|
+
const first = candidates.find((candidate) => candidate.atMs <= deadlineMs);
|
|
146
|
+
if (first) {
|
|
147
|
+
return {
|
|
148
|
+
ok: true,
|
|
149
|
+
lane_id: first.laneId,
|
|
150
|
+
change: first.change,
|
|
151
|
+
at: first.at,
|
|
152
|
+
};
|
|
153
|
+
}
|
|
154
|
+
if (observedAtMs >= deadlineMs)
|
|
155
|
+
return null;
|
|
156
|
+
previous = current;
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
//# sourceMappingURL=wait.js.map
|