@indigoai-us/hq-cli 5.192.0 → 5.192.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/dist/commands/lanes.js +190 -3
- package/dist/lib/lanes/admission.js +2 -0
- package/dist/lib/lanes/index.d.ts +2 -2
- package/dist/lib/lanes/index.js +1 -1
- package/dist/lib/lanes/outpost-dispatch.d.ts +7 -0
- package/dist/lib/lanes/outpost-dispatch.js +18 -0
- package/dist/lib/lanes/reconcile.d.ts +10 -1
- package/dist/lib/lanes/reconcile.js +112 -6
- package/dist/lib/lanes/spawn.js +7 -3
- package/dist/lib/lanes/types.d.ts +2 -1
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,17 @@
|
|
|
2
2
|
|
|
3
3
|
## [Unreleased]
|
|
4
4
|
|
|
5
|
+
## [5.192.1] — 2026-09-25
|
|
6
|
+
|
|
7
|
+
### Fixed
|
|
8
|
+
|
|
9
|
+
- `hq lanes questions answer` now resumes an asking lane as soon as its final
|
|
10
|
+
pending question is answered. When capacity or throttling prevents a start,
|
|
11
|
+
hq-cli queues the lane with an `answered_questions` resume and the answer
|
|
12
|
+
timestamp. It sends Outpost answers to the lane's owner and reports any
|
|
13
|
+
unanswered sibling questions or a named wake error. After the owner confirms
|
|
14
|
+
the answer, the primary's question copy is marked answered as well.
|
|
15
|
+
|
|
5
16
|
## [5.192.0] — 2026-09-25
|
|
6
17
|
|
|
7
18
|
### Added
|
package/dist/commands/lanes.js
CHANGED
|
@@ -21,7 +21,7 @@ import { createOutpostHireGrantContext, parseOutpostHireGrantContext, sameHireGr
|
|
|
21
21
|
import { OUTPOST_DIR, OUTPOST_ID_PATH, readOutpostIdentity, resolveAdmissionHost, } from "../lib/lanes/admission-host.js";
|
|
22
22
|
export { OUTPOST_DIR, OUTPOST_ID_PATH, readOutpostIdentity } from "../lib/lanes/admission-host.js";
|
|
23
23
|
export { LANE_HIRE_GRANTS_FLAG };
|
|
24
|
-
import { buildOutpostCapacityCommand, buildOutpostBalanceMoveCommand, buildOutpostIdentityBackfillCommand, buildOutpostLaneCommand, buildOutpostMoveTargetCommand, buildOutpostPickupCommand, createOutpostsClient, dispatchErrorMessage, dispatchOutpostPickup, buildOutpostLaneListCommand, outpostRejectedEffortOption, parseOutpostPickupPayload, stageOutpostPayload, } from "../lib/lanes/outpost-dispatch.js";
|
|
24
|
+
import { buildOutpostCapacityCommand, buildOutpostBalanceMoveCommand, buildOutpostIdentityBackfillCommand, buildOutpostLaneCommand, buildOutpostMoveTargetCommand, buildOutpostQuestionAnswerCommand, buildOutpostPickupCommand, createOutpostsClient, dispatchErrorMessage, dispatchOutpostPickup, buildOutpostLaneListCommand, outpostRejectedEffortOption, parseOutpostPickupPayload, stageOutpostPayload, } from "../lib/lanes/outpost-dispatch.js";
|
|
25
25
|
import { buildHostCapacitySnapshot, chooseHostForLane, parseHostCapacitySnapshot, } from "../lib/lanes/host-capacity.js";
|
|
26
26
|
import { BALANCE_MOVABLE_STATES, CLAUDE_ACCOUNT_BALANCE_FLAG, accountLoadsForReport, assertBalanceApplyEnabled, planClaudeBalanceMoves, selectClaudeAccount, } from "../lib/lanes/claude-account-balance.js";
|
|
27
27
|
import { parseUserYaml } from "../utils/user-yaml-error.js";
|
|
@@ -30,7 +30,7 @@ import { vaultApiFetch } from "../utils/vault-api.js";
|
|
|
30
30
|
import { openMeshTransport } from "../lib/mesh/client.js";
|
|
31
31
|
import { createLaneMesh, emitLaneDecisionEvents, emitLaneTerminalStateEvent, laneMeshEventIdentity, } from "../lib/lanes/mesh-events.js";
|
|
32
32
|
import { sessionIdFromEnv } from "../lib/mesh/live/session-identity.js";
|
|
33
|
-
import { AccountResolutionError, compareLaneQueueOrder, coveredLaneIds, DEFAULT_ACCOUNT_HOMES, LANE_PROVIDERS, LANE_STATES, PROVIDER_HOME_ENV, QUESTION_STATUSES, SENIOR_KINDS, accountStatus, appendCappedHistory, canonicalizeHome, createLane, decideAdmission, defaultLivenessDeps, defaultSystemdUserDir, defaultThrottleDeps, installDrainHook, installReconcileTimer, providerPathEnv, probeDrainHookRegistration, isLanesError, isLaneVersionConflict, isTerminalLaneState, aliveDirectChildren, assignedChildHireGrants, LANE_HIRE_GRANTS_FLAG, planHireGrantBackfill, withHireGrantAdmission, laneStoryLedgerEnabled, validateProjectStories, buildLaneStatusSnapshot, formatLaneStatusHuman, listAccountStatuses, listLaneIds, listQuestionIds, listSessionLiveness, loadLanesConfig, laneRunsHere, MAX_LANE_DEPTH, MONITOR_ERROR_EXIT_CODE, MONITOR_ENGINES, MONITOR_MAX_TIMEOUT_MS, MONITOR_UNCOVERED_EXIT_CODE, MAX_WATCHED_LANES, monitorReminderText, readLiveMonitorEvidence, resolveMachineIdentity as lookupMachineIdentity, loadLivenessIndex, parseLaneProvider, parseLaneRecord, parseLaneState, parseLivenessState, parseQuestionStatus, parseSeniorKind, readDaemonPresenceConnected, readLane, retryUpdateLane, syncLaneRecordFromOwner, readQuestion, resolveAccount, resolveLaneWorker, ignoredWorkerProfileFields, loadWorkerProfile, runReconcilePass, runLaneWatch, sessionLiveness, tryReadLane, uninstallReconcileTimer, updateLane, updateQuestion, writeLanesConfig, appendDropboxMessage, answerDropboxMessage, deliveryRefusalReason, resumeCommandRefusal, dropboxDir, dropboxInboxPath, laneAcceptsDropboxDelivery, laneAcceptsOperatorMessage, laneDeliveryMechanism, hasActiveAdmissionReservation, hasActivePendingResume, hasRecordedSessionId, hasEarlierEligibleQueuedLane, laneAdmissionCapReason, providerSupportsSessionResume, withLaneAdmissionLock, withLaneHireLocks, laneIdForAnswerDelivery, operatorMessageRefusalReason, raiseLaneQuestion, sha256Text, writeQuestionBody, } from "../lib/lanes/index.js";
|
|
33
|
+
import { AccountResolutionError, compareLaneQueueOrder, coveredLaneIds, DEFAULT_ACCOUNT_HOMES, LANE_PROVIDERS, LANE_STATES, PROVIDER_HOME_ENV, QUESTION_STATUSES, SENIOR_KINDS, accountStatus, appendCappedHistory, canonicalizeHome, createLane, decideAdmission, defaultLivenessDeps, defaultSystemdUserDir, defaultThrottleDeps, installDrainHook, installReconcileTimer, providerPathEnv, probeDrainHookRegistration, isLanesError, isLaneVersionConflict, isTerminalLaneState, aliveDirectChildren, assignedChildHireGrants, LANE_HIRE_GRANTS_FLAG, planHireGrantBackfill, withHireGrantAdmission, laneStoryLedgerEnabled, validateProjectStories, buildLaneStatusSnapshot, formatLaneStatusHuman, listAccountStatuses, listLaneIds, listQuestionIds, listSessionLiveness, loadLanesConfig, laneRunsHere, MAX_LANE_DEPTH, MONITOR_ERROR_EXIT_CODE, MONITOR_ENGINES, MONITOR_MAX_TIMEOUT_MS, MONITOR_UNCOVERED_EXIT_CODE, MAX_WATCHED_LANES, monitorReminderText, readLiveMonitorEvidence, resolveMachineIdentity as lookupMachineIdentity, loadLivenessIndex, parseLaneProvider, parseLaneRecord, parseLaneState, parseLivenessState, parseQuestionRecord, parseQuestionStatus, parseSeniorKind, readDaemonPresenceConnected, readLane, retryUpdateLane, syncLaneRecordFromOwner, readQuestion, resolveAccount, resolveLaneWorker, ignoredWorkerProfileFields, loadWorkerProfile, runReconcilePass, wakeAnsweredLaneNow, runLaneWatch, sessionLiveness, tryReadLane, uninstallReconcileTimer, updateLane, updateQuestion, writeLanesConfig, appendDropboxMessage, answerDropboxMessage, deliveryRefusalReason, resumeCommandRefusal, dropboxDir, dropboxInboxPath, laneAcceptsDropboxDelivery, laneAcceptsOperatorMessage, laneDeliveryMechanism, hasActiveAdmissionReservation, hasActivePendingResume, hasRecordedSessionId, hasEarlierEligibleQueuedLane, laneAdmissionCapReason, providerSupportsSessionResume, withLaneAdmissionLock, withLaneHireLocks, laneIdForAnswerDelivery, operatorMessageRefusalReason, raiseLaneQuestion, sha256Text, writeQuestionBody, } from "../lib/lanes/index.js";
|
|
34
34
|
import { activeLaneStories, laneStories, laneStory, storyWarning, } from "../lib/lanes/stories.js";
|
|
35
35
|
import { questionBodyAbsPath } from "../lib/lanes/questions.js";
|
|
36
36
|
import { compactionAfterUsage, } from "../lib/lanes/compaction.js";
|
|
@@ -3208,6 +3208,139 @@ async function dispatchOutpostLaneCommand(ctx, lane, action, options) {
|
|
|
3208
3208
|
outpost_transport: "outposts_exec",
|
|
3209
3209
|
});
|
|
3210
3210
|
}
|
|
3211
|
+
async function dispatchOutpostQuestionAnswer(ctx, lane, questionId, options) {
|
|
3212
|
+
if (lane.host.kind !== "outpost") {
|
|
3213
|
+
throw new LanesError("outpost_dispatch_failed", `Lane ${lane.lane_id} has no Outpost host to answer its question`, { lane_id: lane.lane_id, question_id: questionId });
|
|
3214
|
+
}
|
|
3215
|
+
const outpostId = lane.host.id;
|
|
3216
|
+
let dispatched;
|
|
3217
|
+
try {
|
|
3218
|
+
dispatched = await ctx.dispatchOutpostExec({
|
|
3219
|
+
outpostId,
|
|
3220
|
+
command: buildOutpostQuestionAnswerCommand({
|
|
3221
|
+
questionId,
|
|
3222
|
+
choice: options.choice,
|
|
3223
|
+
freeform: options.freeform,
|
|
3224
|
+
notes: options.notes,
|
|
3225
|
+
}),
|
|
3226
|
+
});
|
|
3227
|
+
}
|
|
3228
|
+
catch (err) {
|
|
3229
|
+
throw new LanesError("outpost_dispatch_failed", `Could not dispatch question ${questionId} to Outpost ${outpostId}: ${dispatchErrorMessage(err)}`, { lane_id: lane.lane_id, question_id: questionId, outpost_id: outpostId });
|
|
3230
|
+
}
|
|
3231
|
+
if (dispatched.done !== true || dispatched.status !== "Success" || dispatched.exitCode !== 0) {
|
|
3232
|
+
throw new LanesError("outpost_dispatch_failed", `Outpost ${outpostId} did not confirm the answer for question ${questionId}`, {
|
|
3233
|
+
lane_id: lane.lane_id,
|
|
3234
|
+
question_id: questionId,
|
|
3235
|
+
outpost_id: outpostId,
|
|
3236
|
+
command_id: dispatched.commandId,
|
|
3237
|
+
status: dispatched.status,
|
|
3238
|
+
done: dispatched.done ?? false,
|
|
3239
|
+
exit_code: dispatched.exitCode,
|
|
3240
|
+
});
|
|
3241
|
+
}
|
|
3242
|
+
const remote = parseRemoteJson(dispatched.stdout, "Outpost question answer");
|
|
3243
|
+
if (remote?.error !== undefined) {
|
|
3244
|
+
throw new LanesError("outpost_remote_command_failed", typeof remote.message === "string"
|
|
3245
|
+
? remote.message
|
|
3246
|
+
: `Outpost ${outpostId} rejected the answer for question ${questionId}`, {
|
|
3247
|
+
lane_id: lane.lane_id,
|
|
3248
|
+
question_id: questionId,
|
|
3249
|
+
outpost_id: outpostId,
|
|
3250
|
+
command_id: dispatched.commandId,
|
|
3251
|
+
remote_error: remote.error,
|
|
3252
|
+
});
|
|
3253
|
+
}
|
|
3254
|
+
const remoteQuestion = remote?.question !== null && typeof remote?.question === "object"
|
|
3255
|
+
? remote.question
|
|
3256
|
+
: null;
|
|
3257
|
+
const awaitingWakeConfirmed = lane.state !== "awaiting_input" ||
|
|
3258
|
+
(remote?.wake_status === "resumed" && remote.lane_state === "running") ||
|
|
3259
|
+
(remote?.wake_status === "queued" && remote.lane_state === "queued") ||
|
|
3260
|
+
(remote?.wake_status === "awaiting_answers" && remote.lane_state === "awaiting_input");
|
|
3261
|
+
if (remote?.ok !== true ||
|
|
3262
|
+
remote.action !== "questions.answer" ||
|
|
3263
|
+
remoteQuestion?.question_id !== questionId ||
|
|
3264
|
+
remoteQuestion.lane_id !== lane.lane_id ||
|
|
3265
|
+
remoteQuestion.status !== "answered" ||
|
|
3266
|
+
typeof remote.delivery !== "string" ||
|
|
3267
|
+
!awaitingWakeConfirmed) {
|
|
3268
|
+
throw new LanesError("outpost_delivery_unconfirmed", `Outpost ${outpostId} completed exec without confirming the answer and wake status for question ${questionId}`, {
|
|
3269
|
+
lane_id: lane.lane_id,
|
|
3270
|
+
question_id: questionId,
|
|
3271
|
+
outpost_id: outpostId,
|
|
3272
|
+
command_id: dispatched.commandId,
|
|
3273
|
+
});
|
|
3274
|
+
}
|
|
3275
|
+
let ownerQuestion;
|
|
3276
|
+
try {
|
|
3277
|
+
ownerQuestion = parseQuestionRecord(remoteQuestion);
|
|
3278
|
+
}
|
|
3279
|
+
catch (err) {
|
|
3280
|
+
throw new LanesError("outpost_delivery_unconfirmed", `Outpost ${outpostId} did not return a valid answered question record for ${questionId}: ${dispatchErrorMessage(err)}`, {
|
|
3281
|
+
lane_id: lane.lane_id,
|
|
3282
|
+
question_id: questionId,
|
|
3283
|
+
outpost_id: outpostId,
|
|
3284
|
+
command_id: dispatched.commandId,
|
|
3285
|
+
});
|
|
3286
|
+
}
|
|
3287
|
+
const ownerAnswer = ownerQuestion.answer;
|
|
3288
|
+
if (!isQuestionAnswer(ownerAnswer)) {
|
|
3289
|
+
throw new LanesError("outpost_delivery_unconfirmed", `Outpost ${outpostId} did not return the confirmed answer for question ${questionId}`, {
|
|
3290
|
+
lane_id: lane.lane_id,
|
|
3291
|
+
question_id: questionId,
|
|
3292
|
+
outpost_id: outpostId,
|
|
3293
|
+
command_id: dispatched.commandId,
|
|
3294
|
+
});
|
|
3295
|
+
}
|
|
3296
|
+
const syncedQuestion = casUpdateQuestion(ctx.hqRoot(), questionId, (current) => {
|
|
3297
|
+
if (current.lane_id !== lane.lane_id) {
|
|
3298
|
+
throw new LanesError("outpost_question_sync_conflict", `Primary question ${questionId} belongs to a different lane than Outpost ${outpostId} confirmed`, { lane_id: lane.lane_id, question_id: questionId, outpost_id: outpostId });
|
|
3299
|
+
}
|
|
3300
|
+
if (current.status === "pending") {
|
|
3301
|
+
return { ...current, status: "answered", answer: ownerAnswer };
|
|
3302
|
+
}
|
|
3303
|
+
if (current.status === "answered" && sameQuestionAnswer(current.answer, ownerAnswer)) {
|
|
3304
|
+
return current;
|
|
3305
|
+
}
|
|
3306
|
+
throw new LanesError("outpost_question_sync_conflict", `Primary question ${questionId} is ${current.status} or has a different answer than Outpost ${outpostId} confirmed`, { lane_id: lane.lane_id, question_id: questionId, outpost_id: outpostId });
|
|
3307
|
+
});
|
|
3308
|
+
printJson({
|
|
3309
|
+
...remote,
|
|
3310
|
+
lane_id: lane.lane_id,
|
|
3311
|
+
question: syncedQuestion,
|
|
3312
|
+
outpost_id: outpostId,
|
|
3313
|
+
outpost_command_id: dispatched.commandId,
|
|
3314
|
+
outpost_transport: "outposts_exec",
|
|
3315
|
+
});
|
|
3316
|
+
}
|
|
3317
|
+
function isQuestionAnswer(value) {
|
|
3318
|
+
if (value === null || typeof value !== "object")
|
|
3319
|
+
return false;
|
|
3320
|
+
const answer = value;
|
|
3321
|
+
return (typeof answer.choice === "string" &&
|
|
3322
|
+
answer.choice.trim() !== "" &&
|
|
3323
|
+
(answer.answer_kind === undefined ||
|
|
3324
|
+
answer.answer_kind === "option" ||
|
|
3325
|
+
answer.answer_kind === "freeform") &&
|
|
3326
|
+
(answer.notes_ref === undefined || typeof answer.notes_ref === "string") &&
|
|
3327
|
+
typeof answer.answered_by === "string" &&
|
|
3328
|
+
answer.answered_by.trim() !== "" &&
|
|
3329
|
+
typeof answer.answered_at === "string" &&
|
|
3330
|
+
Number.isFinite(Date.parse(answer.answered_at)) &&
|
|
3331
|
+
typeof answer.answered_level === "number" &&
|
|
3332
|
+
Number.isInteger(answer.answered_level) &&
|
|
3333
|
+
answer.answered_level > 0);
|
|
3334
|
+
}
|
|
3335
|
+
function sameQuestionAnswer(left, right) {
|
|
3336
|
+
return (left !== undefined &&
|
|
3337
|
+
left.choice === right.choice &&
|
|
3338
|
+
left.answer_kind === right.answer_kind &&
|
|
3339
|
+
left.notes_ref === right.notes_ref &&
|
|
3340
|
+
left.answered_by === right.answered_by &&
|
|
3341
|
+
left.answered_at === right.answered_at &&
|
|
3342
|
+
left.answered_level === right.answered_level);
|
|
3343
|
+
}
|
|
3211
3344
|
function ingestPickupPayload(hqRoot, payload, now) {
|
|
3212
3345
|
const laneId = payload.lane.lane_id;
|
|
3213
3346
|
if (!laneId) {
|
|
@@ -6464,9 +6597,16 @@ async function runQuestionsShow(ctx, questionId) {
|
|
|
6464
6597
|
async function runQuestionsAnswer(ctx, questionId, opts) {
|
|
6465
6598
|
const hqRoot = ctx.hqRoot();
|
|
6466
6599
|
const question = readQuestion(hqRoot, questionId);
|
|
6467
|
-
throwIfQuestionNotPending(question);
|
|
6468
6600
|
await yieldEventLoop();
|
|
6469
6601
|
const askingLane = readLane(hqRoot, laneIdForAnswerDelivery(question));
|
|
6602
|
+
if (askingLane.host.kind === "outpost" && !laneIsHostedHere(ctx, askingLane)) {
|
|
6603
|
+
await dispatchOutpostQuestionAnswer(ctx, askingLane, questionId, opts);
|
|
6604
|
+
return;
|
|
6605
|
+
}
|
|
6606
|
+
throwIfQuestionNotPending(question);
|
|
6607
|
+
if (askingLane.state === "awaiting_input") {
|
|
6608
|
+
assertLaneRunsHereForOperatorMutation(ctx, askingLane, "wake after answering");
|
|
6609
|
+
}
|
|
6470
6610
|
if (laneAcceptsDropboxDelivery(askingLane)) {
|
|
6471
6611
|
assertLaneRunsHereForOperatorMutation(ctx, askingLane, "deliver an answer to");
|
|
6472
6612
|
}
|
|
@@ -6511,6 +6651,53 @@ async function runQuestionsAnswer(ctx, questionId, opts) {
|
|
|
6511
6651
|
},
|
|
6512
6652
|
};
|
|
6513
6653
|
});
|
|
6654
|
+
const latestAskingLane = readLane(hqRoot, askingLane.lane_id);
|
|
6655
|
+
if (latestAskingLane.state === "awaiting_input") {
|
|
6656
|
+
let wake;
|
|
6657
|
+
try {
|
|
6658
|
+
wake = await wakeAnsweredLaneNow(hqRoot, askingLane.lane_id, {
|
|
6659
|
+
now: ctx.now,
|
|
6660
|
+
host: reconcileHostFromCtx(ctx),
|
|
6661
|
+
dryRun: false,
|
|
6662
|
+
spawnLaneDeps: {
|
|
6663
|
+
env: ctx.env,
|
|
6664
|
+
existsSync: ctx.existsSync,
|
|
6665
|
+
spawnProcess: ctx.spawnProcess,
|
|
6666
|
+
proveDetached: ctx.proveDetached,
|
|
6667
|
+
killProcess: ctx.killProcess,
|
|
6668
|
+
now: ctx.now,
|
|
6669
|
+
resolveMachineIdentity: ctx.resolveMachineIdentity,
|
|
6670
|
+
runCompaction: ctx.runCompaction,
|
|
6671
|
+
locateTranscript: ctx.locateTranscript,
|
|
6672
|
+
},
|
|
6673
|
+
throttleDeps: ctx.throttleDeps,
|
|
6674
|
+
livenessDeps: livenessDepsFromCtx(ctx),
|
|
6675
|
+
presenceConnected: ctx.presenceConnected,
|
|
6676
|
+
mesh: ctx.mesh,
|
|
6677
|
+
env: ctx.env,
|
|
6678
|
+
});
|
|
6679
|
+
}
|
|
6680
|
+
catch (err) {
|
|
6681
|
+
throw new LanesError("question_wake_failed", `Question ${questionId} was recorded, but lane ${askingLane.lane_id} could not be woken: ${dispatchErrorMessage(err)}`, { lane_id: askingLane.lane_id, question_id: questionId });
|
|
6682
|
+
}
|
|
6683
|
+
if (wake.status === "failed") {
|
|
6684
|
+
throw new LanesError("question_wake_failed", `Question ${questionId} was recorded, but lane ${askingLane.lane_id} could not be woken: ${wake.error}`, { lane_id: askingLane.lane_id, question_id: questionId, wake_error: wake.error });
|
|
6685
|
+
}
|
|
6686
|
+
printJson({
|
|
6687
|
+
ok: true,
|
|
6688
|
+
action: "questions.answer",
|
|
6689
|
+
lane_id: askingLane.lane_id,
|
|
6690
|
+
question: updated,
|
|
6691
|
+
delivery: wake.status === "awaiting_answers" ? "awaiting_other_answers" : wake.status,
|
|
6692
|
+
wake_status: wake.status,
|
|
6693
|
+
lane_state: wake.lane_state,
|
|
6694
|
+
...(wake.queued_reason !== undefined ? { queued_reason: wake.queued_reason } : {}),
|
|
6695
|
+
...(wake.remaining_question_ids !== undefined
|
|
6696
|
+
? { remaining_question_ids: wake.remaining_question_ids }
|
|
6697
|
+
: {}),
|
|
6698
|
+
});
|
|
6699
|
+
return;
|
|
6700
|
+
}
|
|
6514
6701
|
const delivery = deliverAnswerToAskingLane(ctx, updated, choice, at);
|
|
6515
6702
|
printJson({
|
|
6516
6703
|
ok: true,
|
|
@@ -135,6 +135,8 @@ export function hasActivePendingResume(lane, nowMs) {
|
|
|
135
135
|
const pending = lane.pending_resume;
|
|
136
136
|
if (!pending)
|
|
137
137
|
return false;
|
|
138
|
+
if (lane.state === "queued" && pending.trigger === "answered_questions")
|
|
139
|
+
return false;
|
|
138
140
|
const reservedAt = Date.parse(pending.at);
|
|
139
141
|
const ageMs = nowMs - reservedAt;
|
|
140
142
|
return Number.isFinite(reservedAt) && ageMs >= 0 && ageMs < ADMISSION_RESERVATION_GRACE_MS;
|
|
@@ -42,8 +42,8 @@ export { AUTH_JSON_NAME, CLAUDE_CREDENTIALS_NAME, PROVIDER_CREDENTIAL_FILES, PRO
|
|
|
42
42
|
export type { AccountResolveContext, AccountStatus, AuthState, ResolvedAccount, } from "./accounts.js";
|
|
43
43
|
export { LIVE_WITHIN_MS, LIVENESS_EVENT_KINDS, SILENT_LOST_MS, TURN_START_LOST_MS, buildLivenessIndexFromSpool, classifySessionLiveness, defaultLivenessDeps, defaultSpoolPath, ingestSpoolLine, listSessionLiveness, livenessIndexPath, loadLivenessIndex, readDaemonPresenceConnected, readLivenessIndex, rebuildLivenessIndex, sessionLiveness, } from "./liveness.js";
|
|
44
44
|
export type { LivenessDeps, LivenessEventKind, LivenessIndex, SessionIndexEntry, } from "./liveness.js";
|
|
45
|
-
export { DEAD_HOST_MS, MAX_AGED_ADVANCES, QUESTION_AGE_MS, RECONCILE_TIMER_NAME, RECONCILE_UNIT_NAME, agedAdvanceCount, currentHopStartedAtMs, isDeadlinePassed, laneRunsHere, readTimeoutDeadline, reconcileHeartbeatPath, resolveQuestionAudience, runReconcilePass, shouldAdvanceAudienceEnded, shouldAgeQuestion, timeoutPath, } from "./reconcile.js";
|
|
46
|
-
export type { ReconcileChange, ReconcileDeps, ReconcileHost, ReconcileSkip, ReconcileSummary, } from "./reconcile.js";
|
|
45
|
+
export { DEAD_HOST_MS, MAX_AGED_ADVANCES, QUESTION_AGE_MS, RECONCILE_TIMER_NAME, RECONCILE_UNIT_NAME, wakeAnsweredLaneNow, agedAdvanceCount, currentHopStartedAtMs, isDeadlinePassed, laneRunsHere, readTimeoutDeadline, reconcileHeartbeatPath, resolveQuestionAudience, runReconcilePass, shouldAdvanceAudienceEnded, shouldAgeQuestion, timeoutPath, } from "./reconcile.js";
|
|
46
|
+
export type { AnsweredLaneWakeResult, ReconcileChange, ReconcileDeps, ReconcileHost, ReconcileSkip, ReconcileSummary, } from "./reconcile.js";
|
|
47
47
|
export { RECONCILE_TIMER_INTERVAL, defaultSystemdUserDir, installReconcileTimer, providerPathEnv, renderReconcileService, renderReconcileTimer, uninstallReconcileTimer, } from "./reconcile-install.js";
|
|
48
48
|
export type { ProviderPathEnv, ReconcileInstallOpts, ReconcileInstallResult, RunCommand, RunCommandResult, } from "./reconcile-install.js";
|
|
49
49
|
export { unitPathEnv } from "./reconcile-install.js";
|
package/dist/lib/lanes/index.js
CHANGED
|
@@ -23,7 +23,7 @@ export { LANE_MESH_EVENTS, WIRE_EVENT_KINDS, createLaneMesh, emitLaneDecisionEve
|
|
|
23
23
|
export { CPU_SAMPLE_MS, LANES_CPU_BUSY_OVERRIDE, LANES_CPU_CHECK, PROC_MEMINFO, STARVATION_MAX_DEFERRALS, STARVATION_MAX_MS, applyThrottleDecision, busyFractionFromDeltas, cpuTimesSnapshot, decideAdmission, defaultThrottleDeps, emptyThrottleState, evaluateThrottle, isCpuCheckEnabled, parseMemAvailableKb, readAvailableMemMb, readThrottleState, resolveCpuBusyFraction, sampleCpuBusyFraction, starvationBoundReached, throttleStatePath, writeThrottleState, } from "./throttle.js";
|
|
24
24
|
export { AUTH_JSON_NAME, CLAUDE_CREDENTIALS_NAME, PROVIDER_CREDENTIAL_FILES, PROVIDER_HOME_ENV, accountEnv, accountStatus, authJsonPath, authStateFor, credentialPaths, hasCredentialFile, isCredentialProven, isLoggedOut, isOccupyingLaneState, lastStartedAt, listAccountStatuses, occupyingLaneCount, resolveAccount, runningLaneCount, selectBalancedAccount, } from "./accounts.js";
|
|
25
25
|
export { LIVE_WITHIN_MS, LIVENESS_EVENT_KINDS, SILENT_LOST_MS, TURN_START_LOST_MS, buildLivenessIndexFromSpool, classifySessionLiveness, defaultLivenessDeps, defaultSpoolPath, ingestSpoolLine, listSessionLiveness, livenessIndexPath, loadLivenessIndex, readDaemonPresenceConnected, readLivenessIndex, rebuildLivenessIndex, sessionLiveness, } from "./liveness.js";
|
|
26
|
-
export { DEAD_HOST_MS, MAX_AGED_ADVANCES, QUESTION_AGE_MS, RECONCILE_TIMER_NAME, RECONCILE_UNIT_NAME, agedAdvanceCount, currentHopStartedAtMs, isDeadlinePassed, laneRunsHere, readTimeoutDeadline, reconcileHeartbeatPath, resolveQuestionAudience, runReconcilePass, shouldAdvanceAudienceEnded, shouldAgeQuestion, timeoutPath, } from "./reconcile.js";
|
|
26
|
+
export { DEAD_HOST_MS, MAX_AGED_ADVANCES, QUESTION_AGE_MS, RECONCILE_TIMER_NAME, RECONCILE_UNIT_NAME, wakeAnsweredLaneNow, agedAdvanceCount, currentHopStartedAtMs, isDeadlinePassed, laneRunsHere, readTimeoutDeadline, reconcileHeartbeatPath, resolveQuestionAudience, runReconcilePass, shouldAdvanceAudienceEnded, shouldAgeQuestion, timeoutPath, } from "./reconcile.js";
|
|
27
27
|
export { RECONCILE_TIMER_INTERVAL, defaultSystemdUserDir, installReconcileTimer, providerPathEnv, renderReconcileService, renderReconcileTimer, uninstallReconcileTimer, } from "./reconcile-install.js";
|
|
28
28
|
export { unitPathEnv } from "./reconcile-install.js";
|
|
29
29
|
export { PROVIDER_REJECTED_REASON, PROVIDER_REJECTION_MESSAGE_CAP, parseClaudeResultRejection, parseCodexTurnFailed, parseGrokErrorOutput, parseProviderRejection, readProviderRejection, retryableFromStatus, } from "./provider-rejection.js";
|
|
@@ -41,6 +41,13 @@ export interface OutpostLaneCommand {
|
|
|
41
41
|
grantContext?: string;
|
|
42
42
|
compact?: boolean;
|
|
43
43
|
}
|
|
44
|
+
export declare function buildOutpostQuestionAnswerCommand(input: {
|
|
45
|
+
questionId: string;
|
|
46
|
+
choice?: string;
|
|
47
|
+
freeform?: string;
|
|
48
|
+
notes?: string;
|
|
49
|
+
hqBinary?: string;
|
|
50
|
+
}): string;
|
|
44
51
|
export interface OutpostExecPort {
|
|
45
52
|
stageExecInput: (outpostId?: string) => Promise<{
|
|
46
53
|
putUrl: string;
|
|
@@ -10,6 +10,24 @@ import { OutpostsClient, joinCommandParts, withRemoteHqDir } from "@indigoai-us/
|
|
|
10
10
|
import { LanesError } from "./errors.js";
|
|
11
11
|
export const PICKUP_EXEC_TIMEOUT_SECONDS = 120;
|
|
12
12
|
export const PICKUP_WAIT_MS = PICKUP_EXEC_TIMEOUT_SECONDS * 1000;
|
|
13
|
+
export function buildOutpostQuestionAnswerCommand(input) {
|
|
14
|
+
const args = [
|
|
15
|
+
input.hqBinary?.trim() || "hq",
|
|
16
|
+
"lanes",
|
|
17
|
+
"questions",
|
|
18
|
+
"answer",
|
|
19
|
+
input.questionId,
|
|
20
|
+
];
|
|
21
|
+
for (const [flag, value] of [
|
|
22
|
+
["--choice", input.choice],
|
|
23
|
+
["--freeform", input.freeform],
|
|
24
|
+
["--notes", input.notes],
|
|
25
|
+
]) {
|
|
26
|
+
if (value !== undefined)
|
|
27
|
+
args.push(flag, value);
|
|
28
|
+
}
|
|
29
|
+
return withRemoteHqDir(joinCommandParts(args));
|
|
30
|
+
}
|
|
13
31
|
export function parseOutpostPickupPayload(raw) {
|
|
14
32
|
if (raw === null || typeof raw !== "object" || Array.isArray(raw)) {
|
|
15
33
|
throw new LanesError("outpost_payload_invalid", "Outpost pickup payload must be a JSON object", {});
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
import { type LivenessDeps } from "./liveness.js";
|
|
9
9
|
import { type EnvelopeRepairSpawnInput, type SpawnLaneDeps } from "./spawn.js";
|
|
10
10
|
import { type ThrottleDeps } from "./throttle.js";
|
|
11
|
-
import type { LaneRecord, LivenessState, QuestionRecord, SeniorRef } from "./types.js";
|
|
11
|
+
import type { LaneRecord, LivenessState, QueuedReason, QuestionRecord, SeniorRef } from "./types.js";
|
|
12
12
|
import { type LaneMesh } from "./mesh-events.js";
|
|
13
13
|
/** Spec §9: unanswered at the current hop for 60 minutes → one hop. */
|
|
14
14
|
export declare const QUESTION_AGE_MS: number;
|
|
@@ -102,4 +102,13 @@ export declare function runPassBody(hqRoot: string, deps: ReconcileDeps): Promis
|
|
|
102
102
|
* `ok: false` rather than being swallowed.
|
|
103
103
|
*/
|
|
104
104
|
export declare function runReconcilePass(hqRoot: string, deps: ReconcileDeps): Promise<ReconcileSummary>;
|
|
105
|
+
export interface AnsweredLaneWakeResult {
|
|
106
|
+
status: "resumed" | "queued" | "awaiting_answers" | "failed";
|
|
107
|
+
lane_state: LaneRecord["state"];
|
|
108
|
+
queued_reason?: QueuedReason;
|
|
109
|
+
remaining_question_ids?: string[];
|
|
110
|
+
error?: string;
|
|
111
|
+
}
|
|
112
|
+
/** Run the reconcile answer-resume path for one lane before questions answer returns. */
|
|
113
|
+
export declare function wakeAnsweredLaneNow(hqRoot: string, laneId: string, deps: ReconcileDeps): Promise<AnsweredLaneWakeResult>;
|
|
105
114
|
//# sourceMappingURL=reconcile.d.ts.map
|
|
@@ -1284,9 +1284,11 @@ async function admitQueued(hqRoot, lanes, summary, deps, now) {
|
|
|
1284
1284
|
writeThrottleState(filePath, applyThrottleDecision(base, applied, nowMs), throttleDeps);
|
|
1285
1285
|
}
|
|
1286
1286
|
}
|
|
1287
|
-
async function resumeAwaiting(hqRoot, lanes, questions, summary, deps) {
|
|
1287
|
+
async function resumeAwaiting(hqRoot, lanes, questions, summary, deps, onlyLaneIds) {
|
|
1288
1288
|
const config = loadLanesConfig(hqRoot);
|
|
1289
|
-
const awaiting = lanes.filter((lane) => lane.state === "awaiting_input" &&
|
|
1289
|
+
const awaiting = lanes.filter((lane) => lane.state === "awaiting_input" &&
|
|
1290
|
+
laneRunsHere(lane, deps.host) &&
|
|
1291
|
+
(onlyLaneIds === undefined || onlyLaneIds.has(lane.lane_id)));
|
|
1290
1292
|
if (awaiting.length === 0)
|
|
1291
1293
|
return;
|
|
1292
1294
|
const throttleDeps = deps.throttleDeps ?? defaultThrottleDeps(deps.env ?? process.env);
|
|
@@ -1307,10 +1309,24 @@ async function resumeAwaiting(hqRoot, lanes, questions, summary, deps) {
|
|
|
1307
1309
|
const throttle = await decideAdmission(hqRoot, config.throttle, throttleDeps);
|
|
1308
1310
|
throttleDecision = throttle.decision;
|
|
1309
1311
|
}
|
|
1310
|
-
if (!throttleDecision.admit)
|
|
1311
|
-
return;
|
|
1312
1312
|
const spawnFn = deps.spawn ?? ((root, lane) => spawnLaneWorker(root, lane, deps.spawnLaneDeps));
|
|
1313
1313
|
const nowMs = deps.now().getTime();
|
|
1314
|
+
const fallbackAnswerAt = new Date(nowMs).toISOString();
|
|
1315
|
+
const answerAtByLane = new Map();
|
|
1316
|
+
for (const question of questions) {
|
|
1317
|
+
if (question.status !== "answered")
|
|
1318
|
+
continue;
|
|
1319
|
+
const at = question.answer?.answered_at ?? question.asked_at;
|
|
1320
|
+
const previous = answerAtByLane.get(question.origin.id);
|
|
1321
|
+
if (previous === undefined || Date.parse(at) > Date.parse(previous)) {
|
|
1322
|
+
answerAtByLane.set(question.origin.id, at);
|
|
1323
|
+
}
|
|
1324
|
+
}
|
|
1325
|
+
const latestAnswerAt = (lane) => answerAtByLane.get(lane.lane_id) ?? fallbackAnswerAt;
|
|
1326
|
+
awaiting.sort((a, b) => {
|
|
1327
|
+
const byAnswerTime = Date.parse(latestAnswerAt(a)) - Date.parse(latestAnswerAt(b));
|
|
1328
|
+
return byAnswerTime || compareLaneQueueOrder(a, b);
|
|
1329
|
+
});
|
|
1314
1330
|
const capacityLanes = lanes.map((lane) => {
|
|
1315
1331
|
try {
|
|
1316
1332
|
return readLane(hqRoot, lane.lane_id);
|
|
@@ -1336,7 +1352,55 @@ async function resumeAwaiting(hqRoot, lanes, questions, summary, deps) {
|
|
|
1336
1352
|
if (current.worker_profile !== undefined) {
|
|
1337
1353
|
profileMaxConcurrent = loadWorkerProfile(hqRoot, current.worker_profile).meta.max_concurrent;
|
|
1338
1354
|
}
|
|
1339
|
-
|
|
1355
|
+
const capReason = laneAdmissionCapReason(running, current, config, deps.host, profileMaxConcurrent, nowMs);
|
|
1356
|
+
const answeredAt = latestAnswerAt(current);
|
|
1357
|
+
const queueCandidate = { ...current, state: "queued", queued_at: answeredAt };
|
|
1358
|
+
const earlierQueued = capacityLanes.some((queued) => queued.lane_id !== current.lane_id &&
|
|
1359
|
+
queued.state === "queued" &&
|
|
1360
|
+
queued.provider === current.provider &&
|
|
1361
|
+
laneRunsHere(queued, deps.host) &&
|
|
1362
|
+
compareLaneQueueOrder(queued, queueCandidate) < 0);
|
|
1363
|
+
const queueReason = !throttleDecision.admit
|
|
1364
|
+
? "throttle"
|
|
1365
|
+
: capReason ?? (earlierQueued ? "queue_position" : undefined);
|
|
1366
|
+
if (queueReason !== undefined) {
|
|
1367
|
+
const queuedCount = capacityLanes.filter((queued) => queued.state === "queued" &&
|
|
1368
|
+
queued.provider === current.provider &&
|
|
1369
|
+
laneRunsHere(queued, deps.host) &&
|
|
1370
|
+
!hasFreshAdmissionReservation(queued, nowMs)).length;
|
|
1371
|
+
const maxQueued = config.providers[current.provider].max_queued;
|
|
1372
|
+
if (queuedCount >= maxQueued) {
|
|
1373
|
+
summary.skipped.push({
|
|
1374
|
+
kind: "lane",
|
|
1375
|
+
id: current.lane_id,
|
|
1376
|
+
why: `lane_queue_full: ${current.provider} queue has ${queuedCount}/${maxQueued} lanes`,
|
|
1377
|
+
});
|
|
1378
|
+
continue;
|
|
1379
|
+
}
|
|
1380
|
+
}
|
|
1381
|
+
if (!deps.dryRun)
|
|
1382
|
+
writeAnsweredQuestions(hqRoot, current, questions);
|
|
1383
|
+
if (queueReason !== undefined) {
|
|
1384
|
+
const queued = casLaneIf(hqRoot, current.lane_id, (latest) => latest.state === "awaiting_input", (latest) => ({
|
|
1385
|
+
...latest,
|
|
1386
|
+
state: "queued",
|
|
1387
|
+
queued_reason: queueReason,
|
|
1388
|
+
queued_at: answeredAt,
|
|
1389
|
+
pending_resume: {
|
|
1390
|
+
...(latest.provider_session_id
|
|
1391
|
+
? { session_id: latest.provider_session_id }
|
|
1392
|
+
: {}),
|
|
1393
|
+
trigger: "answered_questions",
|
|
1394
|
+
at: answeredAt,
|
|
1395
|
+
},
|
|
1396
|
+
}), deps.dryRun);
|
|
1397
|
+
summary.resumed.push({
|
|
1398
|
+
id: queued.record.lane_id,
|
|
1399
|
+
from: "awaiting_input",
|
|
1400
|
+
to: "queued",
|
|
1401
|
+
reason: queueReason,
|
|
1402
|
+
});
|
|
1403
|
+
capacityLanes.push(queued.record);
|
|
1340
1404
|
continue;
|
|
1341
1405
|
}
|
|
1342
1406
|
if (deps.dryRun) {
|
|
@@ -1348,7 +1412,6 @@ async function resumeAwaiting(hqRoot, lanes, questions, summary, deps) {
|
|
|
1348
1412
|
running.push({ ...current, state: "running" });
|
|
1349
1413
|
continue;
|
|
1350
1414
|
}
|
|
1351
|
-
writeAnsweredQuestions(hqRoot, lane, questions);
|
|
1352
1415
|
const started = spawnFn(hqRoot, lane);
|
|
1353
1416
|
if (started.state === "running")
|
|
1354
1417
|
running.push(started);
|
|
@@ -1376,4 +1439,47 @@ async function resumeAwaiting(hqRoot, lanes, questions, summary, deps) {
|
|
|
1376
1439
|
export async function runReconcilePass(hqRoot, deps) {
|
|
1377
1440
|
return withReconcileLock(hqRoot, async () => runPassBody(hqRoot, deps));
|
|
1378
1441
|
}
|
|
1442
|
+
/** Run the reconcile answer-resume path for one lane before questions answer returns. */
|
|
1443
|
+
export async function wakeAnsweredLaneNow(hqRoot, laneId, deps) {
|
|
1444
|
+
const summary = emptySummary(deps.dryRun);
|
|
1445
|
+
await withReconcileLock(hqRoot, async () => {
|
|
1446
|
+
await withAllLaneAdmissionLocks(hqRoot, async () => {
|
|
1447
|
+
const lanes = loadLaneRecords(hqRoot, summary);
|
|
1448
|
+
const questions = loadQuestionRecords(hqRoot, summary);
|
|
1449
|
+
if (!lanes.some((lane) => lane.lane_id === laneId)) {
|
|
1450
|
+
throw new LanesError("lane_not_found", `Lane ${laneId} was not found`, { lane_id: laneId });
|
|
1451
|
+
}
|
|
1452
|
+
await resumeAwaiting(hqRoot, lanes, questions, summary, deps, new Set([laneId]));
|
|
1453
|
+
});
|
|
1454
|
+
});
|
|
1455
|
+
const latest = readLane(hqRoot, laneId);
|
|
1456
|
+
const laneQuestions = loadQuestionRecords(hqRoot, emptySummary(false)).filter((question) => question.origin.id === laneId);
|
|
1457
|
+
const remaining = laneQuestions
|
|
1458
|
+
.filter((question) => question.status === "pending")
|
|
1459
|
+
.map((question) => question.question_id)
|
|
1460
|
+
.sort();
|
|
1461
|
+
if (latest.state === "running") {
|
|
1462
|
+
return { status: "resumed", lane_state: latest.state };
|
|
1463
|
+
}
|
|
1464
|
+
if (latest.state === "queued" && latest.pending_resume?.trigger === "answered_questions") {
|
|
1465
|
+
return {
|
|
1466
|
+
status: "queued",
|
|
1467
|
+
lane_state: latest.state,
|
|
1468
|
+
...(latest.queued_reason !== undefined ? { queued_reason: latest.queued_reason } : {}),
|
|
1469
|
+
};
|
|
1470
|
+
}
|
|
1471
|
+
if (latest.state === "awaiting_input" && remaining.length > 0) {
|
|
1472
|
+
return {
|
|
1473
|
+
status: "awaiting_answers",
|
|
1474
|
+
lane_state: latest.state,
|
|
1475
|
+
remaining_question_ids: remaining,
|
|
1476
|
+
};
|
|
1477
|
+
}
|
|
1478
|
+
const failure = summary.skipped.find((skip) => skip.kind === "lane" && skip.id === laneId);
|
|
1479
|
+
return {
|
|
1480
|
+
status: "failed",
|
|
1481
|
+
lane_state: latest.state,
|
|
1482
|
+
error: failure?.why ?? `Lane ${laneId} did not leave awaiting_input after its questions were answered`,
|
|
1483
|
+
};
|
|
1484
|
+
}
|
|
1379
1485
|
//# sourceMappingURL=reconcile.js.map
|
package/dist/lib/lanes/spawn.js
CHANGED
|
@@ -1553,9 +1553,13 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
|
|
|
1553
1553
|
// state and the caller sees the error. Admission and question resume keep
|
|
1554
1554
|
// today's behaviour of naming the terminal reason on the record.
|
|
1555
1555
|
const markPreSpawnFailure = trigger.kind !== "message" && trigger.kind !== "resume";
|
|
1556
|
-
const plan = pendingResume
|
|
1557
|
-
?
|
|
1558
|
-
|
|
1556
|
+
const plan = pendingResume?.trigger === "answered_questions"
|
|
1557
|
+
? pendingResume.session_id !== undefined && providerSupportsSessionResume(existing.provider)
|
|
1558
|
+
? { mode: "resume", session_id: pendingResume.session_id }
|
|
1559
|
+
: planLaneLaunch(existing, { kind: "answered_questions" })
|
|
1560
|
+
: pendingResume?.session_id !== undefined && providerSupportsSessionResume(existing.provider)
|
|
1561
|
+
? { mode: "resume", session_id: pendingResume.session_id }
|
|
1562
|
+
: planLaneLaunch(existing, trigger);
|
|
1559
1563
|
if (existing.host.kind === "outpost") {
|
|
1560
1564
|
refuseOutpostSelfTarget(hqRoot, existing, deps);
|
|
1561
1565
|
}
|
|
@@ -508,7 +508,8 @@ export interface AccountHistoryEntry {
|
|
|
508
508
|
reason: "first_assignment" | "model_change" | "usage_limit" | "account_unavailable" | "operator_move";
|
|
509
509
|
}
|
|
510
510
|
export interface PendingResume {
|
|
511
|
-
|
|
511
|
+
/** Fresh answered-question launches need no previous session id. */
|
|
512
|
+
session_id?: string;
|
|
512
513
|
trigger: "answered_questions" | "message" | "resume";
|
|
513
514
|
resumed_from?: "interrupted" | "done" | "blocked" | "failed";
|
|
514
515
|
text?: string;
|