@indigoai-us/hq-cli 5.148.0 → 5.150.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +30 -0
- package/assets/scaffold/core/scripts/checkpoint-stop-gate.sh +8 -11
- package/dist/command-catalog.generated.d.ts +148 -0
- package/dist/command-catalog.generated.js +193 -0
- package/dist/commands/lanes.js +1000 -268
- package/dist/commands/mesh.js +82 -11
- package/dist/lib/lanes/admission.d.ts +24 -0
- package/dist/lib/lanes/admission.js +193 -0
- package/dist/lib/lanes/delivery.js +5 -3
- package/dist/lib/lanes/index.d.ts +5 -2
- package/dist/lib/lanes/index.js +3 -1
- package/dist/lib/lanes/mesh-events.d.ts +8 -0
- package/dist/lib/lanes/mesh-events.js +83 -6
- package/dist/lib/lanes/outpost-dispatch.d.ts +12 -0
- package/dist/lib/lanes/outpost-dispatch.js +17 -0
- package/dist/lib/lanes/reconcile.js +194 -55
- package/dist/lib/lanes/session-relink.d.ts +25 -0
- package/dist/lib/lanes/session-relink.js +216 -0
- package/dist/lib/lanes/spawn.d.ts +1 -0
- package/dist/lib/lanes/spawn.js +52 -5
- package/dist/lib/lanes/store.js +156 -1
- package/dist/lib/lanes/stories.d.ts +17 -0
- package/dist/lib/lanes/stories.js +73 -0
- package/dist/lib/lanes/types.d.ts +28 -1
- package/dist/lib/lanes/types.js +24 -6
- package/dist/lib/lanes/watcher.js +69 -9
- package/dist/lib/mesh/live/daemon/doctor.d.ts +27 -0
- package/dist/lib/mesh/live/daemon/doctor.js +84 -0
- package/dist/lib/mesh/live/daemon/index.d.ts +3 -3
- package/dist/lib/mesh/live/daemon/index.js +2 -2
- package/dist/lib/mesh/live/daemon/install.d.ts +40 -0
- package/dist/lib/mesh/live/daemon/install.js +88 -0
- package/dist/lib/mesh/live/daemon/run.d.ts +2 -0
- package/dist/lib/mesh/live/daemon/run.js +12 -1
- package/dist/lib/mesh/live/daemon/state.d.ts +4 -0
- package/dist/lib/mesh/project-board.d.ts +15 -0
- package/dist/lib/mesh/project-board.js +84 -0
- package/dist/lib/work-context/prd-sync.d.ts +5 -1
- package/dist/lib/work-context/prd-sync.js +22 -1
- package/package.json +5 -3
- package/scripts/mesh-daemon-if-stale.mjs +16 -0
package/dist/commands/lanes.js
CHANGED
|
@@ -12,22 +12,24 @@ import * as os from "node:os";
|
|
|
12
12
|
import * as path from "node:path";
|
|
13
13
|
import { ensureCognitoToken } from "../utils/cognito-session.js";
|
|
14
14
|
import { resolveLanesHqRoot } from "../lib/lanes/hq-root.js";
|
|
15
|
-
import { buildOutpostPickupCommand, createOutpostsClient, dispatchErrorMessage, dispatchOutpostPickup, parseOutpostPickupPayload, stageOutpostPayload, } from "../lib/lanes/outpost-dispatch.js";
|
|
15
|
+
import { buildOutpostLaneCommand, buildOutpostPickupCommand, createOutpostsClient, dispatchErrorMessage, dispatchOutpostPickup, parseOutpostPickupPayload, stageOutpostPayload, } from "../lib/lanes/outpost-dispatch.js";
|
|
16
16
|
import { parseUserYaml } from "../utils/user-yaml-error.js";
|
|
17
17
|
import { vaultApiFetch } from "../utils/vault-api.js";
|
|
18
18
|
import { openMeshTransport } from "../lib/mesh/client.js";
|
|
19
19
|
import { createLaneMesh, emitLaneDecisionEvents, } from "../lib/lanes/mesh-events.js";
|
|
20
20
|
import { sessionIdFromEnv } from "../lib/mesh/live/session-identity.js";
|
|
21
|
-
import { AccountResolutionError, coveredLaneIds, DEFAULT_ACCOUNT_HOMES, LANE_PROVIDERS, LANE_STATES, PROVIDER_HOME_ENV, QUESTION_STATUSES, SENIOR_KINDS,
|
|
21
|
+
import { AccountResolutionError, compareLaneQueueOrder, coveredLaneIds, DEFAULT_ACCOUNT_HOMES, LANE_PROVIDERS, LANE_STATES, PROVIDER_HOME_ENV, QUESTION_STATUSES, SENIOR_KINDS, accountStatus, appendCappedHistory, canonicalizeHome, createLane, decideAdmission, defaultLivenessDeps, defaultSystemdUserDir, defaultThrottleDeps, installDrainHook, installReconcileTimer, providerPathEnv, probeDrainHookRegistration, isLanesError, isLaneVersionConflict, isTerminalLaneState, listAccountStatuses, listLaneIds, listQuestionIds, listSessionLiveness, loadLanesConfig, laneRunsHere, MAX_LANE_DEPTH, MONITOR_ERROR_EXIT_CODE, MONITOR_ENGINES, MONITOR_MAX_TIMEOUT_MS, MONITOR_UNCOVERED_EXIT_CODE, MAX_WATCHED_LANES, monitorReminderText, readLiveMonitorEvidence, resolveMachineIdentity as lookupMachineIdentity, loadLivenessIndex, parseLaneProvider, parseLaneState, parseLivenessState, parseQuestionStatus, parseSeniorKind, readDaemonPresenceConnected, readLane, retryUpdateLane, readQuestion, resolveAccount, resolveLaneWorker, ignoredWorkerProfileFields, loadWorkerProfile, isLoggedOut, runReconcilePass, runLaneWatch, sessionLiveness, tryReadLane, uninstallReconcileTimer, updateLane, updateQuestion, writeLanesConfig, appendDropboxMessage, answerDropboxMessage, deliveryRefusalReason, resumeCommandRefusal, dropboxDir, dropboxInboxPath, laneAcceptsDropboxDelivery, laneAcceptsOperatorMessage, laneDeliveryMechanism, hasActiveAdmissionReservation, hasActivePendingResume, hasRecordedSessionId, hasEarlierEligibleQueuedLane, laneAdmissionCapReason, providerSupportsSessionResume, withLaneAdmissionLock, laneIdForAnswerDelivery, operatorMessageRefusalReason, raiseLaneQuestion, sha256Text, writeQuestionBody, } from "../lib/lanes/index.js";
|
|
22
|
+
import { activeLaneStories, laneStories, laneStory, storyWarning, } from "../lib/lanes/stories.js";
|
|
22
23
|
import { questionBodyAbsPath } from "../lib/lanes/questions.js";
|
|
23
24
|
import { compactionAfterUsage, } from "../lib/lanes/compaction.js";
|
|
24
25
|
import { providerCompactAfterMs } from "../lib/lanes/config.js";
|
|
25
26
|
import { parseCliEffort, resolveLaneEffort } from "../lib/lanes/effort.js";
|
|
26
27
|
import { LanesError } from "../lib/lanes/errors.js";
|
|
27
28
|
import { accountQuotaView, loadQuotaState, quotaBlockFor } from "../lib/lanes/quota.js";
|
|
28
|
-
import { SESSION_MOVE_REASON
|
|
29
|
+
import { SESSION_MOVE_REASON } from "../lib/lanes/move-account.js";
|
|
30
|
+
import { ensureSessionHome } from "../lib/lanes/session-relink.js";
|
|
29
31
|
import { defaultProbe, validateProfileLaneModel, validateLaneModel, } from "../lib/lanes/models.js";
|
|
30
|
-
import { parseCliModel, refuseOutpostSelfTarget, resolveLaneModel, FOREGROUND_ONLY_RULE, STANDING_AUTHORIZATION, spawnLaneWorker, stopLaneProcesses, } from "../lib/lanes/spawn.js";
|
|
32
|
+
import { parseCliModel, refuseOutpostSelfTarget, resolveLaneModel, FOREGROUND_ONLY_RULE, STANDING_AUTHORIZATION, spawnLaneWorker, laneRunDir, readSpawnManifest, stopLaneProcesses, } from "../lib/lanes/spawn.js";
|
|
31
33
|
export { spawnLaneWorker };
|
|
32
34
|
export { questionBodyAbsPath };
|
|
33
35
|
export { MAX_LANE_DEPTH };
|
|
@@ -40,8 +42,21 @@ const VENDOR_LOGIN = {
|
|
|
40
42
|
grok: { command: "grok", args: ["login"] },
|
|
41
43
|
};
|
|
42
44
|
export { emitLaneDecisionEvents };
|
|
45
|
+
function withoutProviderHome(value) {
|
|
46
|
+
if (Array.isArray(value))
|
|
47
|
+
return value.map(withoutProviderHome);
|
|
48
|
+
if (value === null || typeof value !== "object")
|
|
49
|
+
return value;
|
|
50
|
+
const safe = {};
|
|
51
|
+
for (const [key, nested] of Object.entries(value)) {
|
|
52
|
+
if (key === "provider_home_at_spawn")
|
|
53
|
+
continue;
|
|
54
|
+
safe[key] = withoutProviderHome(nested);
|
|
55
|
+
}
|
|
56
|
+
return safe;
|
|
57
|
+
}
|
|
43
58
|
function printJson(value) {
|
|
44
|
-
console.log(JSON.stringify(value, null, 2));
|
|
59
|
+
console.log(JSON.stringify(withoutProviderHome(value), null, 2));
|
|
45
60
|
}
|
|
46
61
|
function failJson(err) {
|
|
47
62
|
if (isLanesError(err)) {
|
|
@@ -825,36 +840,25 @@ function isOverdue(hqRoot, laneId, now) {
|
|
|
825
840
|
return false;
|
|
826
841
|
return now.getTime() > at;
|
|
827
842
|
}
|
|
828
|
-
function compareQueueOrder(a, b) {
|
|
829
|
-
if (b.priority !== a.priority)
|
|
830
|
-
return b.priority - a.priority;
|
|
831
|
-
return a.created_at.localeCompare(b.created_at);
|
|
832
|
-
}
|
|
833
843
|
function admitCreate(input) {
|
|
834
|
-
const { hqRoot, provider, pinnedAccount, profileId, profileMaxConcurrent, priority, createdAt, config, } = input;
|
|
844
|
+
const { hqRoot, provider, pinnedAccount, profileId, profileMaxConcurrent, host, priority, createdAt, nowMs, config, } = input;
|
|
835
845
|
const providerCfg = config.providers[provider];
|
|
836
846
|
const lanes = loadAllLanes(hqRoot).filter((lane) => lane.lane_id !== input.excludeLaneId);
|
|
837
|
-
const
|
|
838
|
-
const
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
847
|
+
const localLanes = lanes.filter((lane) => laneRunsHere(lane, host));
|
|
848
|
+
const runningProvider = lanes.filter((lane) => lane.provider === provider &&
|
|
849
|
+
(lane.state === "running" ||
|
|
850
|
+
hasActiveAdmissionReservation(lane, nowMs) ||
|
|
851
|
+
hasActivePendingResume(lane, nowMs)) &&
|
|
852
|
+
laneRunsHere(lane, host));
|
|
853
|
+
const queuedProvider = localLanes.filter((lane) => lane.provider === provider &&
|
|
854
|
+
lane.state === "queued" &&
|
|
855
|
+
!hasActiveAdmissionReservation(lane, nowMs));
|
|
845
856
|
const runningCount = pinnedAccount
|
|
846
857
|
? runningProvider.filter((lane) => lane.account === pinnedAccount).length
|
|
847
858
|
: runningProvider.length;
|
|
848
859
|
const providerRunning = runningProvider.length;
|
|
849
|
-
const queuedCount =
|
|
860
|
+
const queuedCount = queuedProvider.length;
|
|
850
861
|
const maxQueued = providerCfg.max_queued;
|
|
851
|
-
const profileRunning = profileId === undefined
|
|
852
|
-
? 0
|
|
853
|
-
: lanes.filter((lane) => lane.worker_profile === profileId && lane.state === "running").length;
|
|
854
|
-
const atCap = providerRunning >= providerCfg.max_running ||
|
|
855
|
-
(pinnedAccount !== undefined &&
|
|
856
|
-
runningCount >= accountMaxRunning(providerCfg, pinnedAccount)) ||
|
|
857
|
-
(profileMaxConcurrent !== undefined && profileRunning >= profileMaxConcurrent);
|
|
858
862
|
const synthetic = {
|
|
859
863
|
schema_version: "hq-lane.v1",
|
|
860
864
|
lane_id: "__incoming__",
|
|
@@ -868,19 +872,24 @@ function admitCreate(input) {
|
|
|
868
872
|
priority,
|
|
869
873
|
provider,
|
|
870
874
|
account: pinnedAccount,
|
|
871
|
-
host:
|
|
875
|
+
host: host.outpostId
|
|
876
|
+
? { kind: "outpost", id: host.outpostId }
|
|
877
|
+
: { kind: "local", id: host.hostname },
|
|
878
|
+
...(profileId !== undefined ? { worker_profile: profileId } : {}),
|
|
872
879
|
version: 0,
|
|
873
880
|
created_at: createdAt,
|
|
874
881
|
updated_at: createdAt,
|
|
875
882
|
attribution_history: [],
|
|
876
883
|
};
|
|
877
|
-
const
|
|
878
|
-
const
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
884
|
+
const capReason = laneAdmissionCapReason(localLanes, synthetic, config, host, profileMaxConcurrent, nowMs);
|
|
885
|
+
const earlierEligible = !capReason && hasEarlierEligibleQueuedLane(localLanes, synthetic, config, host, (lane) => lane.worker_profile === undefined
|
|
886
|
+
? undefined
|
|
887
|
+
: loadWorkerProfile(hqRoot, lane.worker_profile).meta.max_concurrent, nowMs);
|
|
888
|
+
const queuedReason = capReason ??
|
|
889
|
+
(earlierEligible ? "queue_position" : undefined);
|
|
890
|
+
if (!queuedReason)
|
|
882
891
|
return { action: "start" };
|
|
883
|
-
if (queuedCount >= maxQueued) {
|
|
892
|
+
if (!input.alreadyQueued && queuedCount >= maxQueued) {
|
|
884
893
|
return {
|
|
885
894
|
action: "refuse",
|
|
886
895
|
error: new LanesError("lane_queue_full", `Lane queue is full for ${pinnedAccount ? `account ${pinnedAccount}` : provider}`, {
|
|
@@ -891,27 +900,38 @@ function admitCreate(input) {
|
|
|
891
900
|
}),
|
|
892
901
|
};
|
|
893
902
|
}
|
|
894
|
-
|
|
895
|
-
return { action: "queue", reason: isHead ? "cap" : "queue_position" };
|
|
896
|
-
}
|
|
897
|
-
return { action: "queue", reason: "queue_position" };
|
|
903
|
+
return { action: "queue", reason: queuedReason };
|
|
898
904
|
}
|
|
899
|
-
async function applyThrottle(hqRoot, config, ctx, current, provider, pinnedAccount) {
|
|
900
|
-
if (current.action !== "start"
|
|
905
|
+
async function applyThrottle(hqRoot, config, ctx, current, provider, pinnedAccount, options = {}) {
|
|
906
|
+
if (current.action !== "start" &&
|
|
907
|
+
!(current.action === "queue" && current.reason === "queue_position"))
|
|
901
908
|
return current;
|
|
902
909
|
const { decision } = await decideAdmission(hqRoot, config.throttle, ctx.throttleDeps);
|
|
903
910
|
if (decision.admit)
|
|
904
911
|
return current;
|
|
912
|
+
if (current.action === "queue")
|
|
913
|
+
return { action: "queue", reason: "throttle" };
|
|
905
914
|
const providerCfg = config.providers[provider];
|
|
915
|
+
const host = options.host ?? reconcileHostFromCtx(ctx);
|
|
916
|
+
const nowMs = options.nowMs ?? ctx.now().getTime();
|
|
906
917
|
const queuedCount = loadAllLanes(hqRoot).filter((lane) => {
|
|
907
|
-
if (lane.
|
|
918
|
+
if (lane.lane_id === options.excludeLaneId ||
|
|
919
|
+
lane.provider !== provider ||
|
|
920
|
+
lane.state !== "queued" ||
|
|
921
|
+
hasActiveAdmissionReservation(lane, nowMs) ||
|
|
922
|
+
!laneRunsHere(lane, host))
|
|
908
923
|
return false;
|
|
909
924
|
if (pinnedAccount)
|
|
910
925
|
return lane.account === pinnedAccount;
|
|
911
926
|
return !lane.account;
|
|
912
927
|
}).length;
|
|
913
|
-
if (queuedCount >= providerCfg.max_queued) {
|
|
914
|
-
const running = loadAllLanes(hqRoot).filter((lane) => lane.
|
|
928
|
+
if (!options.alreadyQueued && queuedCount >= providerCfg.max_queued) {
|
|
929
|
+
const running = loadAllLanes(hqRoot).filter((lane) => lane.lane_id !== options.excludeLaneId &&
|
|
930
|
+
lane.provider === provider &&
|
|
931
|
+
(lane.state === "running" ||
|
|
932
|
+
hasActiveAdmissionReservation(lane, nowMs) ||
|
|
933
|
+
hasActivePendingResume(lane, nowMs)) &&
|
|
934
|
+
laneRunsHere(lane, host)).length;
|
|
915
935
|
return {
|
|
916
936
|
action: "refuse",
|
|
917
937
|
error: new LanesError("lane_queue_full", `Lane queue is full for ${pinnedAccount ? `account ${pinnedAccount}` : provider}`, {
|
|
@@ -924,6 +944,45 @@ async function applyThrottle(hqRoot, config, ctx, current, provider, pinnedAccou
|
|
|
924
944
|
}
|
|
925
945
|
return { action: "queue", reason: "throttle" };
|
|
926
946
|
}
|
|
947
|
+
async function assessLaneAdmission(ctx, lane, options) {
|
|
948
|
+
const hqRoot = ctx.hqRoot();
|
|
949
|
+
const config = loadLanesConfig(hqRoot);
|
|
950
|
+
const account = options.account ?? lane.account;
|
|
951
|
+
const profile = lane.worker_profile
|
|
952
|
+
? loadWorkerProfile(hqRoot, lane.worker_profile)
|
|
953
|
+
: undefined;
|
|
954
|
+
const nowMs = ctx.now().getTime();
|
|
955
|
+
let admission = admitCreate({
|
|
956
|
+
hqRoot,
|
|
957
|
+
provider: lane.provider,
|
|
958
|
+
pinnedAccount: account,
|
|
959
|
+
...(profile
|
|
960
|
+
? { profileId: profile.id, profileMaxConcurrent: profile.meta.max_concurrent }
|
|
961
|
+
: {}),
|
|
962
|
+
host: reconcileHostFromCtx(ctx),
|
|
963
|
+
priority: lane.priority,
|
|
964
|
+
createdAt: options.queueAt,
|
|
965
|
+
nowMs,
|
|
966
|
+
config,
|
|
967
|
+
excludeLaneId: lane.lane_id,
|
|
968
|
+
alreadyQueued: options.alreadyQueued,
|
|
969
|
+
});
|
|
970
|
+
if (admission.action === "start" ||
|
|
971
|
+
(admission.action === "queue" && admission.reason === "queue_position")) {
|
|
972
|
+
admission = await applyThrottle(hqRoot, config, ctx, admission, lane.provider, account, {
|
|
973
|
+
host: reconcileHostFromCtx(ctx),
|
|
974
|
+
excludeLaneId: lane.lane_id,
|
|
975
|
+
alreadyQueued: options.alreadyQueued,
|
|
976
|
+
nowMs,
|
|
977
|
+
});
|
|
978
|
+
}
|
|
979
|
+
return admission;
|
|
980
|
+
}
|
|
981
|
+
function assertLaneRunsHereForAdmission(ctx, lane) {
|
|
982
|
+
if (laneRunsHere(lane, reconcileHostFromCtx(ctx)))
|
|
983
|
+
return;
|
|
984
|
+
throw new LanesError("lane_remote", `Lane ${lane.lane_id} is hosted remotely; run the admission command on its Outpost`, { lane_id: lane.lane_id, host: lane.host });
|
|
985
|
+
}
|
|
927
986
|
function assertKnownOutpost(requested, known) {
|
|
928
987
|
if (!known.includes(requested)) {
|
|
929
988
|
throw new LanesError("unknown_outpost", `Unknown Outpost ${JSON.stringify(requested)}. Accepted: ${known.join(", ") || "(none)"}`, { accepted: known, value: requested });
|
|
@@ -939,8 +998,6 @@ function copyHookRails(fromHome, toHome) {
|
|
|
939
998
|
function railsPresent(home) {
|
|
940
999
|
return fs.existsSync(path.join(home, "hooks"));
|
|
941
1000
|
}
|
|
942
|
-
const ADMISSION_LOCK_STALE_MS = 30_000;
|
|
943
|
-
const ADMISSION_LOCK_TIMEOUT_MS = 5_000;
|
|
944
1001
|
/**
|
|
945
1002
|
* Spec §5.1: `hq lanes ask` from inside a lane can hit a binary or dist
|
|
946
1003
|
* being replaced mid-call (ENOENT / ETXTBSY / a module that no longer
|
|
@@ -954,83 +1011,6 @@ function yieldEventLoop() {
|
|
|
954
1011
|
setImmediate(resolve);
|
|
955
1012
|
});
|
|
956
1013
|
}
|
|
957
|
-
function admissionLockPath(hqRoot, provider, pinnedAccount) {
|
|
958
|
-
const name = pinnedAccount
|
|
959
|
-
? `${provider}-${pinnedAccount}`
|
|
960
|
-
: `${provider}-unpinned`;
|
|
961
|
-
return path.join(hqRoot, LANES_RUNS_REL, ".admission-locks", `${name}.lock`);
|
|
962
|
-
}
|
|
963
|
-
async function withAdmissionLock(lockPath, fn, sleep) {
|
|
964
|
-
ensureDir(path.dirname(lockPath));
|
|
965
|
-
const started = Date.now();
|
|
966
|
-
while (true) {
|
|
967
|
-
let fd;
|
|
968
|
-
try {
|
|
969
|
-
fd = fs.openSync(lockPath, "wx");
|
|
970
|
-
try {
|
|
971
|
-
fs.writeSync(fd, Buffer.from(`${process.pid}\n${Date.now()}\n`, "utf8"));
|
|
972
|
-
}
|
|
973
|
-
catch (err) {
|
|
974
|
-
fs.closeSync(fd);
|
|
975
|
-
fd = undefined;
|
|
976
|
-
throw err;
|
|
977
|
-
}
|
|
978
|
-
try {
|
|
979
|
-
await yieldEventLoop();
|
|
980
|
-
return await fn();
|
|
981
|
-
}
|
|
982
|
-
finally {
|
|
983
|
-
try {
|
|
984
|
-
fs.closeSync(fd);
|
|
985
|
-
}
|
|
986
|
-
catch (err) {
|
|
987
|
-
process.stderr.write(`hq lanes: admission lock close failed for ${lockPath}: ${err instanceof Error ? err.message : String(err)}\n`);
|
|
988
|
-
}
|
|
989
|
-
try {
|
|
990
|
-
fs.unlinkSync(lockPath);
|
|
991
|
-
}
|
|
992
|
-
catch (err) {
|
|
993
|
-
const code = err.code;
|
|
994
|
-
if (code !== "ENOENT") {
|
|
995
|
-
process.stderr.write(`hq lanes: admission lock unlink failed for ${lockPath}: ${err instanceof Error ? err.message : String(err)}\n`);
|
|
996
|
-
}
|
|
997
|
-
}
|
|
998
|
-
}
|
|
999
|
-
}
|
|
1000
|
-
catch (err) {
|
|
1001
|
-
const code = err.code;
|
|
1002
|
-
if (code !== "EEXIST")
|
|
1003
|
-
throw err;
|
|
1004
|
-
if (Date.now() - started > ADMISSION_LOCK_TIMEOUT_MS) {
|
|
1005
|
-
throw new LanesError("store_lock_timeout", `Timed out waiting for admission lock ${lockPath}`, { path: lockPath });
|
|
1006
|
-
}
|
|
1007
|
-
try {
|
|
1008
|
-
const raw = fs.readFileSync(lockPath, "utf8");
|
|
1009
|
-
const tsLine = raw.split("\n")[1];
|
|
1010
|
-
const ts = tsLine ? Number(tsLine) : NaN;
|
|
1011
|
-
if (Number.isFinite(ts) && Date.now() - ts > ADMISSION_LOCK_STALE_MS) {
|
|
1012
|
-
try {
|
|
1013
|
-
fs.unlinkSync(lockPath);
|
|
1014
|
-
}
|
|
1015
|
-
catch (unlinkErr) {
|
|
1016
|
-
const unlinkCode = unlinkErr.code;
|
|
1017
|
-
if (unlinkCode !== "ENOENT")
|
|
1018
|
-
throw unlinkErr;
|
|
1019
|
-
}
|
|
1020
|
-
continue;
|
|
1021
|
-
}
|
|
1022
|
-
}
|
|
1023
|
-
catch (statErr) {
|
|
1024
|
-
const statCode = statErr.code;
|
|
1025
|
-
if (statCode !== "ENOENT") {
|
|
1026
|
-
process.stderr.write(`hq lanes: stale admission-lock probe failed for ${lockPath}: ${statErr instanceof Error ? statErr.message : String(statErr)}\n`);
|
|
1027
|
-
}
|
|
1028
|
-
}
|
|
1029
|
-
await sleep(10);
|
|
1030
|
-
await yieldEventLoop();
|
|
1031
|
-
}
|
|
1032
|
-
}
|
|
1033
|
-
}
|
|
1034
1014
|
function withCasRetry(fn, attempts = 5) {
|
|
1035
1015
|
let last;
|
|
1036
1016
|
for (let i = 0; i < attempts; i += 1) {
|
|
@@ -1196,6 +1176,26 @@ async function emitLaneMeshEvent(ctx, company, threadId, event, payload) {
|
|
|
1196
1176
|
return;
|
|
1197
1177
|
await ctx.mesh.appendThreadEvent(company, threadId, event, payload);
|
|
1198
1178
|
}
|
|
1179
|
+
async function emitLaneStartedForStories(ctx, company, lane, threadId) {
|
|
1180
|
+
if (!threadId)
|
|
1181
|
+
return;
|
|
1182
|
+
const failures = [];
|
|
1183
|
+
for (const story of laneStories(lane)) {
|
|
1184
|
+
try {
|
|
1185
|
+
await emitLaneMeshEvent(ctx, company, threadId, "lane_started", {
|
|
1186
|
+
lane_id: lane.lane_id,
|
|
1187
|
+
project_id: story.project_id,
|
|
1188
|
+
story_id: story.story_id,
|
|
1189
|
+
});
|
|
1190
|
+
await ctx.mesh.patchStoryStatus(company, story.project_id, story.story_id, "in_progress");
|
|
1191
|
+
}
|
|
1192
|
+
catch (err) {
|
|
1193
|
+
failures.push(err);
|
|
1194
|
+
}
|
|
1195
|
+
}
|
|
1196
|
+
if (failures.length > 0)
|
|
1197
|
+
throw failures[0];
|
|
1198
|
+
}
|
|
1199
1199
|
export const PICKUP_TERMINAL_REASONS = [
|
|
1200
1200
|
"account_not_logged_in",
|
|
1201
1201
|
"unknown_account",
|
|
@@ -1375,6 +1375,7 @@ async function runCreate(ctx, opts) {
|
|
|
1375
1375
|
const meshCompanyFlag = opts.meshCompany?.trim();
|
|
1376
1376
|
const project = opts.project?.trim();
|
|
1377
1377
|
const story = opts.story?.trim();
|
|
1378
|
+
const additionalStories = (opts.additionalStory ?? []).map((value) => value.trim());
|
|
1378
1379
|
const briefFile = opts.briefFile?.trim();
|
|
1379
1380
|
if (!project) {
|
|
1380
1381
|
throw new LanesError("missing_project", "--project is required", {});
|
|
@@ -1382,6 +1383,16 @@ async function runCreate(ctx, opts) {
|
|
|
1382
1383
|
if (!story) {
|
|
1383
1384
|
throw new LanesError("missing_story", "--story is required", {});
|
|
1384
1385
|
}
|
|
1386
|
+
if (additionalStories.some((value) => value.length === 0)) {
|
|
1387
|
+
throw new LanesError("missing_story", "--additional-story values must be non-empty", {});
|
|
1388
|
+
}
|
|
1389
|
+
const storyIds = [story, ...additionalStories];
|
|
1390
|
+
if (new Set(storyIds).size !== storyIds.length) {
|
|
1391
|
+
throw new LanesError("duplicate_story", "A lane cannot carry the same project/story more than once", {
|
|
1392
|
+
project_id: project,
|
|
1393
|
+
stories: storyIds,
|
|
1394
|
+
});
|
|
1395
|
+
}
|
|
1385
1396
|
if (!briefFile) {
|
|
1386
1397
|
throw new LanesError("missing_brief_file", "--brief-file is required", {});
|
|
1387
1398
|
}
|
|
@@ -1620,6 +1631,13 @@ async function runCreate(ctx, opts) {
|
|
|
1620
1631
|
}
|
|
1621
1632
|
}
|
|
1622
1633
|
const allowedAccounts = validatedAccounts.size > 0 ? [...validatedAccounts] : profileAccounts;
|
|
1634
|
+
const balancedAdmissionAccounts = !outpostId && pinnedAccount === undefined
|
|
1635
|
+
? [...new Set(allowedAccounts ??
|
|
1636
|
+
Object.entries(config.providers[provider].accounts)
|
|
1637
|
+
.filter(([id, account]) => account.enabled &&
|
|
1638
|
+
(profileAccounts === undefined || profileAccounts.includes(id)))
|
|
1639
|
+
.map(([id]) => id))]
|
|
1640
|
+
: undefined;
|
|
1623
1641
|
const threadId = meshCompany
|
|
1624
1642
|
? await ctx.mesh.ensureProjectThread(meshCompany, project)
|
|
1625
1643
|
: undefined;
|
|
@@ -1658,6 +1676,7 @@ async function runCreate(ctx, opts) {
|
|
|
1658
1676
|
validation = "pending";
|
|
1659
1677
|
}
|
|
1660
1678
|
else {
|
|
1679
|
+
const admissionNowMs = ctx.now().getTime();
|
|
1661
1680
|
admission = admitCreate({
|
|
1662
1681
|
hqRoot,
|
|
1663
1682
|
provider,
|
|
@@ -1666,14 +1685,16 @@ async function runCreate(ctx, opts) {
|
|
|
1666
1685
|
...(profile?.meta.max_concurrent !== undefined
|
|
1667
1686
|
? { profileMaxConcurrent: profile.meta.max_concurrent }
|
|
1668
1687
|
: {}),
|
|
1688
|
+
host: reconcileHostFromCtx(ctx),
|
|
1669
1689
|
priority,
|
|
1670
1690
|
createdAt,
|
|
1691
|
+
nowMs: admissionNowMs,
|
|
1671
1692
|
config,
|
|
1672
1693
|
});
|
|
1673
1694
|
if (admission.action === "refuse" && admission.error) {
|
|
1674
1695
|
throw admission.error;
|
|
1675
1696
|
}
|
|
1676
|
-
admission = await applyThrottle(hqRoot, config, ctx, admission, provider, pinnedAccount);
|
|
1697
|
+
admission = await applyThrottle(hqRoot, config, ctx, admission, provider, pinnedAccount, { nowMs: admissionNowMs });
|
|
1677
1698
|
if (admission.action === "refuse" && admission.error) {
|
|
1678
1699
|
throw admission.error;
|
|
1679
1700
|
}
|
|
@@ -1682,6 +1703,7 @@ async function runCreate(ctx, opts) {
|
|
|
1682
1703
|
? { kind: "outpost", id: outpostId }
|
|
1683
1704
|
: { kind: "local", id: ctx.hostname() };
|
|
1684
1705
|
const queuedReason = admission.action === "queue" ? admission.reason : undefined;
|
|
1706
|
+
const admissionReservedAt = admission.action === "start" ? ctx.now().toISOString() : undefined;
|
|
1685
1707
|
const lane = createLane(hqRoot, {
|
|
1686
1708
|
company,
|
|
1687
1709
|
...(company === null && meshCompany
|
|
@@ -1689,12 +1711,22 @@ async function runCreate(ctx, opts) {
|
|
|
1689
1711
|
: {}),
|
|
1690
1712
|
project_id: project,
|
|
1691
1713
|
story_id: story,
|
|
1714
|
+
stories: storyIds.map((storyId) => ({
|
|
1715
|
+
project_id: project,
|
|
1716
|
+
story_id: storyId,
|
|
1717
|
+
status: "in_progress",
|
|
1718
|
+
added_at: createdAt,
|
|
1719
|
+
updated_at: createdAt,
|
|
1720
|
+
})),
|
|
1692
1721
|
thread_id: threadId,
|
|
1693
1722
|
depth: laneDepth,
|
|
1694
1723
|
senior,
|
|
1695
1724
|
senior_history: [],
|
|
1696
1725
|
state: "queued",
|
|
1697
1726
|
queued_reason: queuedReason,
|
|
1727
|
+
...(admissionReservedAt !== undefined
|
|
1728
|
+
? { admission_reserved_at: admissionReservedAt }
|
|
1729
|
+
: {}),
|
|
1698
1730
|
priority,
|
|
1699
1731
|
worker: resolvedWorker?.id ?? opts.worker,
|
|
1700
1732
|
...(resolvedWorker ? { worker_path: resolvedWorker.path } : {}),
|
|
@@ -1708,6 +1740,9 @@ async function runCreate(ctx, opts) {
|
|
|
1708
1740
|
: {}),
|
|
1709
1741
|
provider,
|
|
1710
1742
|
account: pinnedAccount,
|
|
1743
|
+
...(balancedAdmissionAccounts !== undefined
|
|
1744
|
+
? { admission_accounts: balancedAdmissionAccounts }
|
|
1745
|
+
: {}),
|
|
1711
1746
|
...(model !== undefined ? { model } : {}),
|
|
1712
1747
|
model_source: cliModel !== undefined
|
|
1713
1748
|
? "cli"
|
|
@@ -1734,7 +1769,7 @@ async function runCreate(ctx, opts) {
|
|
|
1734
1769
|
persisted = await persist();
|
|
1735
1770
|
}
|
|
1736
1771
|
else {
|
|
1737
|
-
persisted = await
|
|
1772
|
+
persisted = await withLaneAdmissionLock(hqRoot, provider, persist, ctx.sleep);
|
|
1738
1773
|
}
|
|
1739
1774
|
const { lane, admission, validation } = persisted;
|
|
1740
1775
|
const dest = runDir(hqRoot, lane.lane_id);
|
|
@@ -1771,14 +1806,8 @@ async function runCreate(ctx, opts) {
|
|
|
1771
1806
|
runCompaction: ctx.runCompaction,
|
|
1772
1807
|
locateTranscript: ctx.locateTranscript,
|
|
1773
1808
|
});
|
|
1774
|
-
if (meshCompany)
|
|
1775
|
-
await
|
|
1776
|
-
lane_id: started.lane_id,
|
|
1777
|
-
project_id: project,
|
|
1778
|
-
story_id: story,
|
|
1779
|
-
});
|
|
1780
|
-
await ctx.mesh.patchStoryStatus(meshCompany, project, story, "in_progress");
|
|
1781
|
-
}
|
|
1809
|
+
if (meshCompany)
|
|
1810
|
+
await emitLaneStartedForStories(ctx, meshCompany, started, threadId);
|
|
1782
1811
|
}
|
|
1783
1812
|
else if (outpostId) {
|
|
1784
1813
|
started = await dispatchCreatedOutpostLane(ctx, {
|
|
@@ -1810,6 +1839,8 @@ async function runCreate(ctx, opts) {
|
|
|
1810
1839
|
company: started.company,
|
|
1811
1840
|
project_id: started.project_id,
|
|
1812
1841
|
story_id: started.story_id,
|
|
1842
|
+
stories: laneStories(started),
|
|
1843
|
+
story_warning: storyWarning(started) ?? null,
|
|
1813
1844
|
thread_id: started.thread_id ?? null,
|
|
1814
1845
|
senior: started.senior,
|
|
1815
1846
|
depth: started.depth,
|
|
@@ -1906,6 +1937,9 @@ async function dispatchCreatedOutpostLane(ctx, input) {
|
|
|
1906
1937
|
}
|
|
1907
1938
|
}
|
|
1908
1939
|
function parseRemotePickupJson(stdout) {
|
|
1940
|
+
return parseRemoteJson(stdout, "Outpost pickup");
|
|
1941
|
+
}
|
|
1942
|
+
function parseRemoteJson(stdout, surface) {
|
|
1909
1943
|
const trimmed = stdout.trim();
|
|
1910
1944
|
if (!trimmed)
|
|
1911
1945
|
return null;
|
|
@@ -1914,14 +1948,92 @@ function parseRemotePickupJson(stdout) {
|
|
|
1914
1948
|
if (parsed !== null && typeof parsed === "object" && !Array.isArray(parsed)) {
|
|
1915
1949
|
return parsed;
|
|
1916
1950
|
}
|
|
1917
|
-
process.stderr.write(
|
|
1951
|
+
process.stderr.write(`hq lanes: ${surface} stdout is not a JSON object\n`);
|
|
1918
1952
|
}
|
|
1919
1953
|
catch (err) {
|
|
1920
|
-
process.stderr.write(`hq lanes: failed to parse
|
|
1954
|
+
process.stderr.write(`hq lanes: failed to parse ${surface} JSON: ${err instanceof Error ? err.message : String(err)}\n`);
|
|
1921
1955
|
return null;
|
|
1922
1956
|
}
|
|
1923
1957
|
return null;
|
|
1924
1958
|
}
|
|
1959
|
+
function laneIsHostedHere(ctx, lane) {
|
|
1960
|
+
if (lane.host.kind !== "outpost")
|
|
1961
|
+
return true;
|
|
1962
|
+
const identity = ctx.readOutpostIdentity();
|
|
1963
|
+
if (identity.kind === "unknown") {
|
|
1964
|
+
throw new LanesError("outpost_identity_unknown", `Cannot determine whether this process is Outpost ${lane.host.id}; refusing local lane delivery`, { lane_id: lane.lane_id, outpost_id: lane.host.id });
|
|
1965
|
+
}
|
|
1966
|
+
return identity.kind === "id" && identity.id === lane.host.id;
|
|
1967
|
+
}
|
|
1968
|
+
async function dispatchOutpostLaneCommand(ctx, lane, action, options) {
|
|
1969
|
+
if (lane.host.kind !== "outpost") {
|
|
1970
|
+
throw new LanesError("outpost_dispatch_failed", `Lane ${lane.lane_id} has no Outpost host to dispatch to`, { lane_id: lane.lane_id });
|
|
1971
|
+
}
|
|
1972
|
+
const outpostId = lane.host.id;
|
|
1973
|
+
let dispatched;
|
|
1974
|
+
try {
|
|
1975
|
+
dispatched = await ctx.dispatchOutpostExec({
|
|
1976
|
+
outpostId,
|
|
1977
|
+
command: buildOutpostLaneCommand({
|
|
1978
|
+
action,
|
|
1979
|
+
laneId: lane.lane_id,
|
|
1980
|
+
text: options.text,
|
|
1981
|
+
project: options.project,
|
|
1982
|
+
story: options.story,
|
|
1983
|
+
account: options.account,
|
|
1984
|
+
model: options.model,
|
|
1985
|
+
compact: options.compact,
|
|
1986
|
+
}),
|
|
1987
|
+
});
|
|
1988
|
+
}
|
|
1989
|
+
catch (err) {
|
|
1990
|
+
throw new LanesError("outpost_dispatch_failed", `Could not dispatch ${action} for lane ${lane.lane_id} to Outpost ${outpostId}: ${dispatchErrorMessage(err)}`, { lane_id: lane.lane_id, outpost_id: outpostId });
|
|
1991
|
+
}
|
|
1992
|
+
if (dispatched.done !== true ||
|
|
1993
|
+
dispatched.status !== "Success" ||
|
|
1994
|
+
dispatched.exitCode !== 0) {
|
|
1995
|
+
throw new LanesError("outpost_dispatch_failed", `Outpost ${outpostId} did not confirm the ${action} command`, {
|
|
1996
|
+
lane_id: lane.lane_id,
|
|
1997
|
+
outpost_id: outpostId,
|
|
1998
|
+
command_id: dispatched.commandId,
|
|
1999
|
+
status: dispatched.status,
|
|
2000
|
+
done: dispatched.done ?? false,
|
|
2001
|
+
exit_code: dispatched.exitCode,
|
|
2002
|
+
});
|
|
2003
|
+
}
|
|
2004
|
+
const remote = parseRemoteJson(dispatched.stdout, "Outpost lane command");
|
|
2005
|
+
if (remote?.error !== undefined) {
|
|
2006
|
+
throw new LanesError("outpost_remote_command_failed", typeof remote.message === "string"
|
|
2007
|
+
? remote.message
|
|
2008
|
+
: `Outpost ${outpostId} rejected the ${action} command`, {
|
|
2009
|
+
lane_id: lane.lane_id,
|
|
2010
|
+
outpost_id: outpostId,
|
|
2011
|
+
command_id: dispatched.commandId,
|
|
2012
|
+
remote_error: remote.error,
|
|
2013
|
+
});
|
|
2014
|
+
}
|
|
2015
|
+
const resultConfirmed = remote !== null &&
|
|
2016
|
+
remote.ok === true &&
|
|
2017
|
+
remote.action === action &&
|
|
2018
|
+
remote.lane_id === lane.lane_id &&
|
|
2019
|
+
(action === "message"
|
|
2020
|
+
? remote.accepted === true
|
|
2021
|
+
: remote.delivery_status === "worker_spawned" ||
|
|
2022
|
+
remote.delivery_status === "queued_on_target_account");
|
|
2023
|
+
if (!resultConfirmed) {
|
|
2024
|
+
throw new LanesError("outpost_delivery_unconfirmed", `Outpost ${outpostId} completed exec without a confirmed ${action} result for lane ${lane.lane_id}`, {
|
|
2025
|
+
lane_id: lane.lane_id,
|
|
2026
|
+
outpost_id: outpostId,
|
|
2027
|
+
command_id: dispatched.commandId,
|
|
2028
|
+
});
|
|
2029
|
+
}
|
|
2030
|
+
printJson({
|
|
2031
|
+
...remote,
|
|
2032
|
+
outpost_id: outpostId,
|
|
2033
|
+
outpost_command_id: dispatched.commandId,
|
|
2034
|
+
outpost_transport: "outposts_exec",
|
|
2035
|
+
});
|
|
2036
|
+
}
|
|
1925
2037
|
function ingestPickupPayload(hqRoot, payload, now) {
|
|
1926
2038
|
const laneId = payload.lane.lane_id;
|
|
1927
2039
|
if (!laneId) {
|
|
@@ -2030,38 +2142,54 @@ async function runPickup(ctx, opts) {
|
|
|
2030
2142
|
});
|
|
2031
2143
|
return;
|
|
2032
2144
|
}
|
|
2033
|
-
|
|
2034
|
-
|
|
2035
|
-
hqRoot,
|
|
2036
|
-
|
|
2037
|
-
|
|
2038
|
-
|
|
2039
|
-
|
|
2040
|
-
|
|
2041
|
-
|
|
2042
|
-
|
|
2043
|
-
|
|
2044
|
-
|
|
2045
|
-
|
|
2046
|
-
|
|
2047
|
-
|
|
2048
|
-
|
|
2049
|
-
|
|
2050
|
-
|
|
2051
|
-
|
|
2052
|
-
|
|
2053
|
-
|
|
2054
|
-
|
|
2055
|
-
return {
|
|
2056
|
-
...current,
|
|
2057
|
-
state: "queued",
|
|
2058
|
-
queued_reason: admission.reason,
|
|
2059
|
-
};
|
|
2145
|
+
let selfTargetCancelled = false;
|
|
2146
|
+
const started = await withLaneAdmissionLock(hqRoot, afterAccount.provider, async () => {
|
|
2147
|
+
const current = readLane(hqRoot, afterAccount.lane_id);
|
|
2148
|
+
const config = loadLanesConfig(hqRoot);
|
|
2149
|
+
const nowMs = ctx.now().getTime();
|
|
2150
|
+
let admission = admitCreate({
|
|
2151
|
+
hqRoot,
|
|
2152
|
+
provider: current.provider,
|
|
2153
|
+
pinnedAccount: current.account,
|
|
2154
|
+
...(current.worker_profile !== undefined
|
|
2155
|
+
? {
|
|
2156
|
+
profileId: current.worker_profile,
|
|
2157
|
+
profileMaxConcurrent: loadWorkerProfile(hqRoot, current.worker_profile).meta.max_concurrent,
|
|
2158
|
+
}
|
|
2159
|
+
: {}),
|
|
2160
|
+
host: reconcileHostFromCtx(ctx),
|
|
2161
|
+
priority: current.priority,
|
|
2162
|
+
createdAt: current.queued_at ?? current.created_at,
|
|
2163
|
+
nowMs,
|
|
2164
|
+
config,
|
|
2165
|
+
excludeLaneId: current.lane_id,
|
|
2166
|
+
alreadyQueued: true,
|
|
2060
2167
|
});
|
|
2061
|
-
|
|
2062
|
-
|
|
2168
|
+
if (admission.action === "refuse" && admission.error)
|
|
2169
|
+
throw admission.error;
|
|
2170
|
+
admission = await applyThrottle(hqRoot, config, ctx, admission, current.provider, current.account, { host: reconcileHostFromCtx(ctx), excludeLaneId: current.lane_id, alreadyQueued: true, nowMs });
|
|
2171
|
+
if (admission.action === "refuse" && admission.error)
|
|
2172
|
+
throw admission.error;
|
|
2173
|
+
if (admission.action === "queue") {
|
|
2174
|
+
return casUpdateLane(hqRoot, current.lane_id, (live) => {
|
|
2175
|
+
if (live.version !== current.version || isTerminalLaneState(live.state)) {
|
|
2176
|
+
throw new LanesError("lane_admission_conflict", `Lane ${current.lane_id} changed while pickup admission was checked`, {
|
|
2177
|
+
lane_id: current.lane_id,
|
|
2178
|
+
prepared_version: current.version,
|
|
2179
|
+
version: live.version,
|
|
2180
|
+
state: live.state,
|
|
2181
|
+
});
|
|
2182
|
+
}
|
|
2183
|
+
return {
|
|
2184
|
+
...live,
|
|
2185
|
+
state: "queued",
|
|
2186
|
+
queued_reason: admission.reason ?? "throttle",
|
|
2187
|
+
queued_at: live.queued_at ?? ctx.now().toISOString(),
|
|
2188
|
+
};
|
|
2189
|
+
});
|
|
2190
|
+
}
|
|
2063
2191
|
try {
|
|
2064
|
-
|
|
2192
|
+
return spawnLaneWorker(hqRoot, current, {
|
|
2065
2193
|
env: ctx.env,
|
|
2066
2194
|
modelDeps: {
|
|
2067
2195
|
readFileSync: ctx.readFileSync,
|
|
@@ -2080,19 +2208,22 @@ async function runPickup(ctx, opts) {
|
|
|
2080
2208
|
}
|
|
2081
2209
|
catch (err) {
|
|
2082
2210
|
if (isLanesError(err) && err.error === "outpost_self_target") {
|
|
2083
|
-
|
|
2084
|
-
|
|
2085
|
-
ok: true,
|
|
2086
|
-
action: "pickup",
|
|
2087
|
-
lane_id: cancelled.lane_id,
|
|
2088
|
-
state: cancelled.state,
|
|
2089
|
-
error: "outpost_self_target",
|
|
2090
|
-
lane: cancelled,
|
|
2091
|
-
});
|
|
2092
|
-
return;
|
|
2211
|
+
selfTargetCancelled = true;
|
|
2212
|
+
return readLane(hqRoot, afterAccount.lane_id);
|
|
2093
2213
|
}
|
|
2094
2214
|
throw err;
|
|
2095
2215
|
}
|
|
2216
|
+
}, ctx.sleep);
|
|
2217
|
+
if (selfTargetCancelled) {
|
|
2218
|
+
printJson({
|
|
2219
|
+
ok: true,
|
|
2220
|
+
action: "pickup",
|
|
2221
|
+
lane_id: started.lane_id,
|
|
2222
|
+
state: started.state,
|
|
2223
|
+
error: "outpost_self_target",
|
|
2224
|
+
lane: started,
|
|
2225
|
+
});
|
|
2226
|
+
return;
|
|
2096
2227
|
}
|
|
2097
2228
|
printJson({
|
|
2098
2229
|
ok: true,
|
|
@@ -2112,14 +2243,16 @@ async function runList(ctx, opts) {
|
|
|
2112
2243
|
company = await ctx.resolveCompany(opts.company);
|
|
2113
2244
|
}
|
|
2114
2245
|
const now = ctx.now();
|
|
2246
|
+
const host = reconcileHostFromCtx(ctx);
|
|
2115
2247
|
const lanes = loadAllLanes(hqRoot)
|
|
2116
2248
|
.filter((lane) => {
|
|
2117
2249
|
if (state && lane.state !== state)
|
|
2118
2250
|
return false;
|
|
2119
|
-
if (opts.project
|
|
2120
|
-
|
|
2121
|
-
|
|
2251
|
+
if ((opts.project || opts.story) &&
|
|
2252
|
+
!laneStories(lane).some((story) => (!opts.project || story.project_id === opts.project) &&
|
|
2253
|
+
(!opts.story || story.story_id === opts.story))) {
|
|
2122
2254
|
return false;
|
|
2255
|
+
}
|
|
2123
2256
|
if (company && !recordMatchesCompany(lane, company)) {
|
|
2124
2257
|
return false;
|
|
2125
2258
|
}
|
|
@@ -2127,28 +2260,42 @@ async function runList(ctx, opts) {
|
|
|
2127
2260
|
return false;
|
|
2128
2261
|
return true;
|
|
2129
2262
|
})
|
|
2130
|
-
.sort(
|
|
2131
|
-
.map((lane) =>
|
|
2132
|
-
|
|
2133
|
-
|
|
2134
|
-
|
|
2135
|
-
|
|
2136
|
-
|
|
2137
|
-
|
|
2138
|
-
|
|
2139
|
-
|
|
2140
|
-
|
|
2141
|
-
|
|
2142
|
-
|
|
2143
|
-
|
|
2144
|
-
|
|
2145
|
-
|
|
2146
|
-
|
|
2263
|
+
.sort(compareLaneQueueOrder)
|
|
2264
|
+
.map((lane) => {
|
|
2265
|
+
const remoteQueued = lane.state === "queued" &&
|
|
2266
|
+
lane.host.kind === "outpost" &&
|
|
2267
|
+
!laneRunsHere(lane, host);
|
|
2268
|
+
return {
|
|
2269
|
+
...lane,
|
|
2270
|
+
...(remoteQueued
|
|
2271
|
+
? { state: "remote", remote_state: lane.state, queued_reason: undefined }
|
|
2272
|
+
: {}),
|
|
2273
|
+
overdue: Boolean(lane.overdue) || isOverdue(hqRoot, lane.lane_id, now),
|
|
2274
|
+
stalled: Boolean(lane.stalled),
|
|
2275
|
+
// Listing is observational. Reconcile owns durable liveness transitions.
|
|
2276
|
+
senior_lost: lane.state === "running" && lane.senior.kind === "session" && lane.senior.id
|
|
2277
|
+
? (() => {
|
|
2278
|
+
const state = sessionLiveness(hqRoot, lane.senior.id, {
|
|
2279
|
+
nowMs: now.getTime(),
|
|
2280
|
+
deps: livenessDepsFromCtx(ctx),
|
|
2281
|
+
presenceConnected: ctx.presenceConnected,
|
|
2282
|
+
});
|
|
2283
|
+
return state === null || state === "ended" || state === "lost";
|
|
2284
|
+
})()
|
|
2285
|
+
: false,
|
|
2286
|
+
};
|
|
2287
|
+
});
|
|
2147
2288
|
printJson({ ok: true, action: "list", lanes });
|
|
2148
2289
|
}
|
|
2149
2290
|
async function runShow(ctx, laneId) {
|
|
2150
2291
|
const hqRoot = ctx.hqRoot();
|
|
2151
|
-
const
|
|
2292
|
+
const storedLane = readLane(hqRoot, laneId);
|
|
2293
|
+
const remoteQueued = storedLane.state === "queued" &&
|
|
2294
|
+
storedLane.host.kind === "outpost" &&
|
|
2295
|
+
!laneRunsHere(storedLane, reconcileHostFromCtx(ctx));
|
|
2296
|
+
const lane = remoteQueued
|
|
2297
|
+
? { ...storedLane, state: "remote", remote_state: storedLane.state, queued_reason: undefined }
|
|
2298
|
+
: storedLane;
|
|
2152
2299
|
const askDeliveryFailed = readJsonFile(askDeliveryFailedPath(hqRoot, laneId));
|
|
2153
2300
|
printJson({
|
|
2154
2301
|
ok: true,
|
|
@@ -2159,7 +2306,7 @@ async function runShow(ctx, laneId) {
|
|
|
2159
2306
|
stalled_at: lane.stalled_at ?? null,
|
|
2160
2307
|
stalled_reason: lane.stalled_reason ?? null,
|
|
2161
2308
|
ask_delivery_failed: askDeliveryFailed,
|
|
2162
|
-
delivery: laneDeliveryMechanism(
|
|
2309
|
+
delivery: laneDeliveryMechanism(storedLane),
|
|
2163
2310
|
provider_session_id: lane.provider_session_id ?? null,
|
|
2164
2311
|
provider_session_id_absent: lane.provider_session_id_absent ?? null,
|
|
2165
2312
|
envelope_repair_mode: lane.envelope_repair?.mode ?? null,
|
|
@@ -2175,10 +2322,136 @@ async function runShow(ctx, laneId) {
|
|
|
2175
2322
|
terminal_details: lane.terminal_details ?? null,
|
|
2176
2323
|
last_message_at: lane.last_message_at ?? null,
|
|
2177
2324
|
last_compaction: lane.last_compaction
|
|
2178
|
-
? { ...lane.last_compaction, after: compactionAfterUsage(
|
|
2325
|
+
? { ...lane.last_compaction, after: compactionAfterUsage(storedLane) }
|
|
2179
2326
|
: null,
|
|
2180
2327
|
});
|
|
2181
2328
|
}
|
|
2329
|
+
function forceAdmissionAuthorized(ctx, lane) {
|
|
2330
|
+
const sessionId = sessionIdFromEnv(ctx.env);
|
|
2331
|
+
const callerLaneId = ctx.env.HQ_LANE_ID?.trim() || undefined;
|
|
2332
|
+
if (lane.senior.kind === "lane") {
|
|
2333
|
+
return callerLaneId !== undefined && callerLaneId === lane.senior.id;
|
|
2334
|
+
}
|
|
2335
|
+
if (!sessionId || callerLaneId !== undefined)
|
|
2336
|
+
return false;
|
|
2337
|
+
if (lane.senior.kind === "session")
|
|
2338
|
+
return sessionId === lane.senior.id;
|
|
2339
|
+
if (lane.senior.kind !== "user")
|
|
2340
|
+
return false;
|
|
2341
|
+
const loaded = loadSessionMeta(ctx.hqRoot(), sessionId, ctx);
|
|
2342
|
+
if (loaded.status !== "ok")
|
|
2343
|
+
return false;
|
|
2344
|
+
const senior = loaded.doc.senior;
|
|
2345
|
+
return senior === "user" ||
|
|
2346
|
+
(senior !== null && typeof senior === "object" && !Array.isArray(senior) &&
|
|
2347
|
+
senior.kind === "user");
|
|
2348
|
+
}
|
|
2349
|
+
async function runAdmit(ctx, laneId, opts) {
|
|
2350
|
+
const hqRoot = ctx.hqRoot();
|
|
2351
|
+
let lane = readLane(hqRoot, laneId);
|
|
2352
|
+
if (lane.state !== "queued") {
|
|
2353
|
+
throw new LanesError("lane_not_queued", `Lane ${laneId} is ${lane.state}; only a queued lane can be admitted`, {
|
|
2354
|
+
lane_id: laneId,
|
|
2355
|
+
state: lane.state,
|
|
2356
|
+
});
|
|
2357
|
+
}
|
|
2358
|
+
const host = reconcileHostFromCtx(ctx);
|
|
2359
|
+
if (!laneRunsHere(lane, host)) {
|
|
2360
|
+
printJson({ ok: true, action: "admit", lane_id: laneId, admitted: false, blocked_by: "remote", lane });
|
|
2361
|
+
return;
|
|
2362
|
+
}
|
|
2363
|
+
if (lane.pickup_validation_pending === true) {
|
|
2364
|
+
printJson({ ok: true, action: "admit", lane_id: laneId, admitted: false, blocked_by: "pickup_validation", lane });
|
|
2365
|
+
return;
|
|
2366
|
+
}
|
|
2367
|
+
await withLaneAdmissionLock(hqRoot, lane.provider, async () => {
|
|
2368
|
+
lane = readLane(hqRoot, laneId);
|
|
2369
|
+
if (lane.state !== "queued") {
|
|
2370
|
+
throw new LanesError("lane_not_queued", `Lane ${laneId} changed to ${lane.state} before admission`, {
|
|
2371
|
+
lane_id: laneId,
|
|
2372
|
+
state: lane.state,
|
|
2373
|
+
});
|
|
2374
|
+
}
|
|
2375
|
+
let blockedBy;
|
|
2376
|
+
if (opts.force) {
|
|
2377
|
+
if (!forceAdmissionAuthorized(ctx, lane)) {
|
|
2378
|
+
throw new LanesError("force_not_authorized", `Only lane ${lane.senior.kind === "lane" ? lane.senior.id : lane.senior.kind === "session" ? lane.senior.id : "senior/operator"} may force admission of ${laneId}`, { lane_id: laneId, senior: lane.senior });
|
|
2379
|
+
}
|
|
2380
|
+
const config = loadLanesConfig(hqRoot);
|
|
2381
|
+
const lanes = loadAllLanes(hqRoot).filter((row) => row.lane_id !== laneId && laneRunsHere(row, host));
|
|
2382
|
+
const { decision } = await decideAdmission(hqRoot, config.throttle, ctx.throttleDeps);
|
|
2383
|
+
if (!decision.admit) {
|
|
2384
|
+
blockedBy = "throttle";
|
|
2385
|
+
}
|
|
2386
|
+
else if (hasEarlierEligibleQueuedLane(lanes, lane, config, host, (row) => row.worker_profile === undefined
|
|
2387
|
+
? undefined
|
|
2388
|
+
: loadWorkerProfile(hqRoot, row.worker_profile).meta.max_concurrent)) {
|
|
2389
|
+
blockedBy = "queue_position";
|
|
2390
|
+
}
|
|
2391
|
+
}
|
|
2392
|
+
else {
|
|
2393
|
+
const admission = await assessLaneAdmission(ctx, lane, {
|
|
2394
|
+
queueAt: lane.queued_at ?? lane.created_at,
|
|
2395
|
+
alreadyQueued: true,
|
|
2396
|
+
});
|
|
2397
|
+
if (admission.action === "refuse" && admission.error) {
|
|
2398
|
+
throw admission.error;
|
|
2399
|
+
}
|
|
2400
|
+
if (admission.action === "queue")
|
|
2401
|
+
blockedBy = admission.reason ?? "throttle";
|
|
2402
|
+
}
|
|
2403
|
+
if (blockedBy !== undefined) {
|
|
2404
|
+
const updated = casUpdateLane(hqRoot, laneId, (current) => {
|
|
2405
|
+
if (current.version !== lane.version || current.state !== "queued") {
|
|
2406
|
+
throw new LanesError("lane_admission_conflict", `Lane ${laneId} changed while admission was checked`, {
|
|
2407
|
+
lane_id: laneId,
|
|
2408
|
+
prepared_version: lane.version,
|
|
2409
|
+
version: current.version,
|
|
2410
|
+
state: current.state,
|
|
2411
|
+
});
|
|
2412
|
+
}
|
|
2413
|
+
return { ...current, queued_reason: blockedBy };
|
|
2414
|
+
});
|
|
2415
|
+
printJson({
|
|
2416
|
+
ok: true,
|
|
2417
|
+
action: "admit",
|
|
2418
|
+
lane_id: laneId,
|
|
2419
|
+
admitted: false,
|
|
2420
|
+
blocked_by: blockedBy,
|
|
2421
|
+
lane: updated,
|
|
2422
|
+
});
|
|
2423
|
+
return;
|
|
2424
|
+
}
|
|
2425
|
+
const started = spawnLaneWorker(hqRoot, lane, {
|
|
2426
|
+
env: ctx.env,
|
|
2427
|
+
modelDeps: {
|
|
2428
|
+
readFileSync: ctx.readFileSync,
|
|
2429
|
+
probe: ctx.probeProvider,
|
|
2430
|
+
env: ctx.env,
|
|
2431
|
+
},
|
|
2432
|
+
existsSync: ctx.existsSync,
|
|
2433
|
+
spawnProcess: ctx.spawnProcess,
|
|
2434
|
+
proveDetached: ctx.proveDetached,
|
|
2435
|
+
killProcess: ctx.killProcess,
|
|
2436
|
+
now: ctx.now,
|
|
2437
|
+
resolveMachineIdentity: ctx.resolveMachineIdentity,
|
|
2438
|
+
runCompaction: ctx.runCompaction,
|
|
2439
|
+
locateTranscript: ctx.locateTranscript,
|
|
2440
|
+
});
|
|
2441
|
+
const admitted = opts.force && started.state === "running"
|
|
2442
|
+
? casUpdateLane(hqRoot, laneId, (current) => ({ ...current, admitted_reason: "operator_force" }))
|
|
2443
|
+
: started;
|
|
2444
|
+
printJson({
|
|
2445
|
+
ok: true,
|
|
2446
|
+
action: "admit",
|
|
2447
|
+
lane_id: laneId,
|
|
2448
|
+
admitted: admitted.state === "running",
|
|
2449
|
+
forced: Boolean(opts.force),
|
|
2450
|
+
blocked_by: admitted.state === "running" ? null : admitted.queued_reason ?? null,
|
|
2451
|
+
lane: admitted,
|
|
2452
|
+
});
|
|
2453
|
+
}, ctx.sleep);
|
|
2454
|
+
}
|
|
2182
2455
|
/**
|
|
2183
2456
|
* US-023: move the lane's provider session to `--account` (same provider)
|
|
2184
2457
|
* before a resume or a message resume. Every check runs before anything is
|
|
@@ -2191,6 +2464,7 @@ async function prepareAccountMove(ctx, lane, input) {
|
|
|
2191
2464
|
const hqRoot = ctx.hqRoot();
|
|
2192
2465
|
const now = ctx.now();
|
|
2193
2466
|
const laneId = lane.lane_id;
|
|
2467
|
+
assertLaneRunsHereForAdmission(ctx, lane);
|
|
2194
2468
|
if (lane.state === "running") {
|
|
2195
2469
|
throw new LanesError("lane_running", `Lane ${laneId} is running; stop it or wait for it to exit before moving its session`, {
|
|
2196
2470
|
lane_id: laneId,
|
|
@@ -2222,6 +2496,9 @@ async function prepareAccountMove(ctx, lane, input) {
|
|
|
2222
2496
|
if (!providerSupportsSessionResume(lane.provider)) {
|
|
2223
2497
|
throw new LanesError("provider_session_not_portable", `Provider ${lane.provider} has no session resume, so a lane cannot move between its accounts`, { lane_id: laneId, provider: lane.provider });
|
|
2224
2498
|
}
|
|
2499
|
+
if (lane.provider === "grok") {
|
|
2500
|
+
throw new LanesError("provider_session_not_portable", `Provider ${lane.provider} has no session resume, so a lane cannot move between its accounts`, { lane_id: laneId, provider: lane.provider });
|
|
2501
|
+
}
|
|
2225
2502
|
if (!hasRecordedSessionId(lane)) {
|
|
2226
2503
|
throw new LanesError("no_provider_session", `Lane ${laneId} has no recorded provider session id, so there is no session to move`, { lane_id: laneId, provider: lane.provider });
|
|
2227
2504
|
}
|
|
@@ -2282,12 +2559,14 @@ async function prepareAccountMove(ctx, lane, input) {
|
|
|
2282
2559
|
throw new AccountResolutionError("unknown_account", `Lane ${laneId} records account ${JSON.stringify(lane.account)} which is not configured for ${lane.provider}`, { provider: lane.provider, account: lane.account ?? "", enabled: [] });
|
|
2283
2560
|
}
|
|
2284
2561
|
// Checks done. Copy the session files, then write the record.
|
|
2285
|
-
const copy =
|
|
2286
|
-
|
|
2287
|
-
targetHome: targetCfg.home,
|
|
2288
|
-
cwd: hqRoot,
|
|
2562
|
+
const copy = ensureSessionHome({
|
|
2563
|
+
provider: lane.provider,
|
|
2289
2564
|
sessionId,
|
|
2290
|
-
|
|
2565
|
+
cwd: hqRoot,
|
|
2566
|
+
targetHome: targetCfg.home,
|
|
2567
|
+
registeredHomes: Object.values(providerCfg.accounts).map((account) => account.home),
|
|
2568
|
+
providerHomeAtSpawn: lane.provider_home_at_spawn ?? sourceCfg.home,
|
|
2569
|
+
});
|
|
2291
2570
|
const at = now.toISOString();
|
|
2292
2571
|
const from = lane.account ?? null;
|
|
2293
2572
|
// The capacity decision and the record write run under the target
|
|
@@ -2296,16 +2575,14 @@ async function prepareAccountMove(ctx, lane, input) {
|
|
|
2296
2575
|
// pointed at the account whose spawn has not claimed it yet
|
|
2297
2576
|
// (`pending_resume` set by this write, cleared by the claim), so two moves
|
|
2298
2577
|
// into one free slot cannot both start.
|
|
2299
|
-
return
|
|
2300
|
-
const
|
|
2301
|
-
|
|
2302
|
-
|
|
2303
|
-
|
|
2304
|
-
|
|
2305
|
-
|
|
2306
|
-
const
|
|
2307
|
-
const atCap = onProvider.length >= providerCfg.max_running ||
|
|
2308
|
-
onProvider.filter((row) => row.account === target).length >= accountMaxRunning(providerCfg, target);
|
|
2578
|
+
return withLaneAdmissionLock(hqRoot, lane.provider, async () => {
|
|
2579
|
+
const admission = await assessLaneAdmission(ctx, lane, {
|
|
2580
|
+
account: target,
|
|
2581
|
+
queueAt: at,
|
|
2582
|
+
});
|
|
2583
|
+
if (admission.action === "refuse" && admission.error)
|
|
2584
|
+
throw admission.error;
|
|
2585
|
+
const queued = admission.action === "queue";
|
|
2309
2586
|
const moved = casUpdateLane(hqRoot, laneId, (current) => {
|
|
2310
2587
|
// The copy and every check above were made from `lane`; a record
|
|
2311
2588
|
// that moved on since (another move, a spawn, a cancel) invalidates
|
|
@@ -2317,6 +2594,9 @@ async function prepareAccountMove(ctx, lane, input) {
|
|
|
2317
2594
|
const next = {
|
|
2318
2595
|
...current,
|
|
2319
2596
|
account: target,
|
|
2597
|
+
admission_accounts: undefined,
|
|
2598
|
+
provider_session_id: sessionId,
|
|
2599
|
+
provider_session_id_absent: undefined,
|
|
2320
2600
|
...(input.cliModel !== undefined
|
|
2321
2601
|
? { model: input.cliModel, model_source: "cli" }
|
|
2322
2602
|
: {}),
|
|
@@ -2328,13 +2608,17 @@ async function prepareAccountMove(ctx, lane, input) {
|
|
|
2328
2608
|
}),
|
|
2329
2609
|
pending_resume: { session_id: sessionId, at, ...input.pending },
|
|
2330
2610
|
};
|
|
2331
|
-
if (
|
|
2611
|
+
if (queued) {
|
|
2332
2612
|
next.state = "queued";
|
|
2333
|
-
next.queued_reason = "
|
|
2613
|
+
next.queued_reason = admission.reason ?? "throttle";
|
|
2614
|
+
next.queued_at = at;
|
|
2615
|
+
}
|
|
2616
|
+
else {
|
|
2617
|
+
next.queued_at = undefined;
|
|
2334
2618
|
}
|
|
2335
2619
|
return next;
|
|
2336
2620
|
});
|
|
2337
|
-
return { lane: moved, from, to: target, copy, queued
|
|
2621
|
+
return { lane: moved, from, to: target, copy, queued };
|
|
2338
2622
|
}, ctx.sleep);
|
|
2339
2623
|
}
|
|
2340
2624
|
async function runMessage(ctx, laneId, opts) {
|
|
@@ -2344,6 +2628,11 @@ async function runMessage(ctx, laneId, opts) {
|
|
|
2344
2628
|
throw new LanesError("missing_text", "--text is required", {});
|
|
2345
2629
|
}
|
|
2346
2630
|
let lane = readLane(hqRoot, laneId);
|
|
2631
|
+
if (lane.host.kind === "outpost" && !laneIsHostedHere(ctx, lane)) {
|
|
2632
|
+
await dispatchOutpostLaneCommand(ctx, lane, "message", opts);
|
|
2633
|
+
return;
|
|
2634
|
+
}
|
|
2635
|
+
lane = laneWithRecoveredProviderSession(hqRoot, lane);
|
|
2347
2636
|
// An account move is refused on its own terms first: a running lane and a
|
|
2348
2637
|
// provider without session resume get the move's named errors, not the
|
|
2349
2638
|
// generic delivery refusal below.
|
|
@@ -2384,6 +2673,7 @@ async function runMessage(ctx, laneId, opts) {
|
|
|
2384
2673
|
};
|
|
2385
2674
|
let resumed = null;
|
|
2386
2675
|
let move = null;
|
|
2676
|
+
let queuedOnAdmission = false;
|
|
2387
2677
|
if (opts.account !== undefined) {
|
|
2388
2678
|
// A moved message is always a resume on the target account: a running
|
|
2389
2679
|
// lane (the dropbox mechanism) is refused inside prepareAccountMove.
|
|
@@ -2408,31 +2698,70 @@ async function runMessage(ctx, laneId, opts) {
|
|
|
2408
2698
|
appendDropboxMessage(hqRoot, laneId, dropboxMessage);
|
|
2409
2699
|
}
|
|
2410
2700
|
else {
|
|
2411
|
-
// Exited
|
|
2412
|
-
//
|
|
2413
|
-
|
|
2414
|
-
|
|
2415
|
-
|
|
2416
|
-
|
|
2417
|
-
|
|
2418
|
-
|
|
2419
|
-
|
|
2420
|
-
|
|
2421
|
-
|
|
2701
|
+
// Exited worker with a recorded session: queue the message when capacity
|
|
2702
|
+
// is occupied, otherwise start its next turn under the shared admission lock.
|
|
2703
|
+
assertLaneRunsHereForAdmission(ctx, lane);
|
|
2704
|
+
resumed = await withLaneAdmissionLock(hqRoot, lane.provider, async () => {
|
|
2705
|
+
const admission = await assessLaneAdmission(ctx, lane, { queueAt: at });
|
|
2706
|
+
if (admission.action === "refuse" && admission.error)
|
|
2707
|
+
throw admission.error;
|
|
2708
|
+
if (admission.action === "queue") {
|
|
2709
|
+
const sessionId = lane.provider_session_id;
|
|
2710
|
+
if (!sessionId) {
|
|
2711
|
+
throw new LanesError("no_provider_session", `Lane ${laneId} has no provider session id to queue this message resume`, {
|
|
2712
|
+
lane_id: laneId,
|
|
2713
|
+
provider: lane.provider,
|
|
2714
|
+
});
|
|
2715
|
+
}
|
|
2716
|
+
archiveResumeMessage(hqRoot, laneId, dropboxMessage);
|
|
2717
|
+
queuedOnAdmission = true;
|
|
2718
|
+
return casUpdateLane(hqRoot, laneId, (current) => {
|
|
2719
|
+
if (current.version !== lane.version || current.state !== lane.state) {
|
|
2720
|
+
throw new LanesError("lane_admission_conflict", `Lane ${laneId} changed while message admission was checked`, {
|
|
2721
|
+
lane_id: laneId,
|
|
2722
|
+
prepared_version: lane.version,
|
|
2723
|
+
version: current.version,
|
|
2724
|
+
state: current.state,
|
|
2725
|
+
});
|
|
2726
|
+
}
|
|
2727
|
+
return {
|
|
2728
|
+
...current,
|
|
2729
|
+
state: "queued",
|
|
2730
|
+
queued_reason: admission.reason ?? "throttle",
|
|
2731
|
+
queued_at: at,
|
|
2732
|
+
pending_resume: {
|
|
2733
|
+
session_id: sessionId,
|
|
2734
|
+
trigger: "message",
|
|
2735
|
+
text,
|
|
2736
|
+
message_id: messageId,
|
|
2737
|
+
compact: opts.compact !== false,
|
|
2738
|
+
at,
|
|
2739
|
+
},
|
|
2740
|
+
};
|
|
2741
|
+
});
|
|
2742
|
+
}
|
|
2743
|
+
const started = spawnLaneWorker(hqRoot, lane, {
|
|
2422
2744
|
env: ctx.env,
|
|
2423
|
-
|
|
2424
|
-
|
|
2425
|
-
|
|
2426
|
-
|
|
2427
|
-
|
|
2428
|
-
|
|
2429
|
-
|
|
2430
|
-
|
|
2431
|
-
|
|
2432
|
-
|
|
2433
|
-
|
|
2434
|
-
|
|
2435
|
-
|
|
2745
|
+
modelDeps: {
|
|
2746
|
+
readFileSync: ctx.readFileSync,
|
|
2747
|
+
probe: ctx.probeProvider,
|
|
2748
|
+
env: ctx.env,
|
|
2749
|
+
},
|
|
2750
|
+
existsSync: ctx.existsSync,
|
|
2751
|
+
spawnProcess: ctx.spawnProcess,
|
|
2752
|
+
proveDetached: ctx.proveDetached,
|
|
2753
|
+
killProcess: ctx.killProcess,
|
|
2754
|
+
now: ctx.now,
|
|
2755
|
+
resolveMachineIdentity: ctx.resolveMachineIdentity,
|
|
2756
|
+
runCompaction: ctx.runCompaction,
|
|
2757
|
+
locateTranscript: ctx.locateTranscript,
|
|
2758
|
+
}, {
|
|
2759
|
+
trigger: { kind: "message", text, message_id: messageId },
|
|
2760
|
+
compact: opts.compact !== false,
|
|
2761
|
+
});
|
|
2762
|
+
archiveResumeMessage(hqRoot, laneId, dropboxMessage);
|
|
2763
|
+
return started;
|
|
2764
|
+
}, ctx.sleep);
|
|
2436
2765
|
}
|
|
2437
2766
|
// Every accepted message moves the compaction reference forward.
|
|
2438
2767
|
let next = casUpdateLane(hqRoot, laneId, (current) => ({
|
|
@@ -2448,6 +2777,18 @@ async function runMessage(ctx, laneId, opts) {
|
|
|
2448
2777
|
...current,
|
|
2449
2778
|
project_id: projectId,
|
|
2450
2779
|
story_id: storyId,
|
|
2780
|
+
stories: laneStory(current, projectId, storyId)
|
|
2781
|
+
? laneStories(current)
|
|
2782
|
+
: [
|
|
2783
|
+
...laneStories(current),
|
|
2784
|
+
{
|
|
2785
|
+
project_id: projectId,
|
|
2786
|
+
story_id: storyId,
|
|
2787
|
+
status: "in_progress",
|
|
2788
|
+
added_at: at,
|
|
2789
|
+
updated_at: at,
|
|
2790
|
+
},
|
|
2791
|
+
],
|
|
2451
2792
|
attribution_history: appendCappedHistory(current.attribution_history, {
|
|
2452
2793
|
project_id: projectId,
|
|
2453
2794
|
story_id: storyId,
|
|
@@ -2466,9 +2807,11 @@ async function runMessage(ctx, laneId, opts) {
|
|
|
2466
2807
|
const drainHook = drainHookStatusFor(ctx, next);
|
|
2467
2808
|
const deliveryStatus = move?.queued
|
|
2468
2809
|
? "queued_on_target_account"
|
|
2469
|
-
:
|
|
2470
|
-
? "
|
|
2471
|
-
: "
|
|
2810
|
+
: queuedOnAdmission
|
|
2811
|
+
? "queued_on_admission"
|
|
2812
|
+
: mechanism === "dropbox"
|
|
2813
|
+
? "awaiting_next_tool_call"
|
|
2814
|
+
: "worker_spawned";
|
|
2472
2815
|
const warning = mechanism === "dropbox" && drainHook !== null && !drainHook.registered
|
|
2473
2816
|
? `The drain hook is not registered in ${drainHook.path}; the running worker will not receive this message. It stays in the inbox and is delivered on the lane's next start.`
|
|
2474
2817
|
: mechanism === "dropbox" && next.stalled === true
|
|
@@ -2493,6 +2836,8 @@ async function runMessage(ctx, laneId, opts) {
|
|
|
2493
2836
|
account_move: move ? accountMoveSummary(move) : null,
|
|
2494
2837
|
project_id: next.project_id,
|
|
2495
2838
|
story_id: next.story_id,
|
|
2839
|
+
stories: laneStories(next),
|
|
2840
|
+
story_warning: storyWarning(next) ?? null,
|
|
2496
2841
|
lane: next,
|
|
2497
2842
|
});
|
|
2498
2843
|
}
|
|
@@ -2508,6 +2853,30 @@ function accountMoveSummary(move) {
|
|
|
2508
2853
|
target_dir: move.copy.target_dir,
|
|
2509
2854
|
};
|
|
2510
2855
|
}
|
|
2856
|
+
function laneWithRecoveredProviderSession(hqRoot, lane) {
|
|
2857
|
+
if (lane.provider === "grok")
|
|
2858
|
+
return lane;
|
|
2859
|
+
const manifest = readSpawnManifest(laneRunDir(hqRoot, lane.lane_id));
|
|
2860
|
+
const sessionId = (typeof lane.provider_session_id === "string" && lane.provider_session_id.trim()
|
|
2861
|
+
? lane.provider_session_id
|
|
2862
|
+
: undefined) ??
|
|
2863
|
+
(typeof lane.last_resume?.session_id === "string" && lane.last_resume.session_id.trim()
|
|
2864
|
+
? lane.last_resume.session_id
|
|
2865
|
+
: undefined) ??
|
|
2866
|
+
(typeof manifest.resume?.session_id === "string" && manifest.resume.session_id.trim()
|
|
2867
|
+
? manifest.resume.session_id
|
|
2868
|
+
: undefined);
|
|
2869
|
+
const providerHome = lane.provider_home_at_spawn ?? manifest.provider_home_at_spawn;
|
|
2870
|
+
return {
|
|
2871
|
+
...lane,
|
|
2872
|
+
...(lane.provider_session_id === undefined && sessionId !== undefined
|
|
2873
|
+
? { provider_session_id: sessionId, provider_session_id_absent: undefined }
|
|
2874
|
+
: {}),
|
|
2875
|
+
...(lane.provider_home_at_spawn === undefined && providerHome !== undefined
|
|
2876
|
+
? { provider_home_at_spawn: providerHome }
|
|
2877
|
+
: {}),
|
|
2878
|
+
};
|
|
2879
|
+
}
|
|
2511
2880
|
/**
|
|
2512
2881
|
* `hq lanes resume <lane-id> [--text <msg>]`: the explicit opt-in that
|
|
2513
2882
|
* continues a lane in `interrupted`, `done` or `blocked` on its recorded
|
|
@@ -2520,6 +2889,11 @@ function accountMoveSummary(move) {
|
|
|
2520
2889
|
async function runResume(ctx, laneId, opts) {
|
|
2521
2890
|
const hqRoot = ctx.hqRoot();
|
|
2522
2891
|
let lane = readLane(hqRoot, laneId);
|
|
2892
|
+
if (lane.host.kind === "outpost" && !laneIsHostedHere(ctx, lane)) {
|
|
2893
|
+
await dispatchOutpostLaneCommand(ctx, lane, "resume", opts);
|
|
2894
|
+
return;
|
|
2895
|
+
}
|
|
2896
|
+
lane = laneWithRecoveredProviderSession(hqRoot, lane);
|
|
2523
2897
|
if (opts.model !== undefined && opts.account === undefined) {
|
|
2524
2898
|
throw new LanesError("model_requires_account", "--model on hq lanes resume is only accepted together with --account", {});
|
|
2525
2899
|
}
|
|
@@ -2532,6 +2906,9 @@ async function runResume(ctx, laneId, opts) {
|
|
|
2532
2906
|
if (opts.account !== undefined && !providerSupportsSessionResume(lane.provider)) {
|
|
2533
2907
|
throw new LanesError("provider_session_not_portable", `Provider ${lane.provider} has no session resume, so a lane cannot move between its accounts`, { lane_id: laneId, provider: lane.provider });
|
|
2534
2908
|
}
|
|
2909
|
+
if (lane.state === "queued") {
|
|
2910
|
+
throw new LanesError("lane_not_resumable", `Lane ${laneId} is queued; use hq lanes admit ${laneId} to request admission`, { lane_id: laneId, state: lane.state, admission_command: `hq lanes admit ${laneId}` });
|
|
2911
|
+
}
|
|
2535
2912
|
const refusal = resumeCommandRefusal(lane);
|
|
2536
2913
|
if (refusal !== null) {
|
|
2537
2914
|
throw new LanesError(refusal.error, refusal.message, {
|
|
@@ -2541,6 +2918,7 @@ async function runResume(ctx, laneId, opts) {
|
|
|
2541
2918
|
message: refusal.message,
|
|
2542
2919
|
});
|
|
2543
2920
|
}
|
|
2921
|
+
assertLaneRunsHereForAdmission(ctx, lane);
|
|
2544
2922
|
const resumedFrom = lane.state;
|
|
2545
2923
|
const text = opts.text?.trim() ? opts.text : undefined;
|
|
2546
2924
|
const at = ctx.now().toISOString();
|
|
@@ -2563,32 +2941,77 @@ async function runResume(ctx, laneId, opts) {
|
|
|
2563
2941
|
});
|
|
2564
2942
|
lane = move.lane;
|
|
2565
2943
|
}
|
|
2566
|
-
|
|
2567
|
-
|
|
2568
|
-
|
|
2569
|
-
|
|
2570
|
-
|
|
2571
|
-
|
|
2572
|
-
|
|
2944
|
+
let queuedOnAdmission = false;
|
|
2945
|
+
let resumed;
|
|
2946
|
+
if (move?.queued) {
|
|
2947
|
+
resumed = lane;
|
|
2948
|
+
}
|
|
2949
|
+
else {
|
|
2950
|
+
resumed = await withLaneAdmissionLock(hqRoot, lane.provider, async () => {
|
|
2951
|
+
const admission = await assessLaneAdmission(ctx, lane, { queueAt: at });
|
|
2952
|
+
if (admission.action === "refuse" && admission.error)
|
|
2953
|
+
throw admission.error;
|
|
2954
|
+
if (admission.action === "queue") {
|
|
2955
|
+
const sessionId = lane.provider_session_id;
|
|
2956
|
+
if (!sessionId) {
|
|
2957
|
+
throw new LanesError("no_provider_session", `Lane ${laneId} has no provider session id to queue this resume`, {
|
|
2958
|
+
lane_id: laneId,
|
|
2959
|
+
provider: lane.provider,
|
|
2960
|
+
});
|
|
2961
|
+
}
|
|
2962
|
+
queuedOnAdmission = true;
|
|
2963
|
+
return casUpdateLane(hqRoot, laneId, (current) => {
|
|
2964
|
+
if (current.version !== lane.version || current.state !== lane.state) {
|
|
2965
|
+
throw new LanesError("lane_admission_conflict", `Lane ${laneId} changed while resume admission was checked`, {
|
|
2966
|
+
lane_id: laneId,
|
|
2967
|
+
prepared_version: lane.version,
|
|
2968
|
+
version: current.version,
|
|
2969
|
+
state: current.state,
|
|
2970
|
+
});
|
|
2971
|
+
}
|
|
2972
|
+
return {
|
|
2973
|
+
...current,
|
|
2974
|
+
state: "queued",
|
|
2975
|
+
queued_reason: admission.reason ?? "throttle",
|
|
2976
|
+
queued_at: at,
|
|
2977
|
+
pending_resume: {
|
|
2978
|
+
session_id: sessionId,
|
|
2979
|
+
trigger: "resume",
|
|
2980
|
+
resumed_from: resumedFrom,
|
|
2981
|
+
...(text !== undefined ? { text } : {}),
|
|
2982
|
+
...(messageId !== undefined ? { message_id: messageId } : {}),
|
|
2983
|
+
compact: opts.compact !== false,
|
|
2984
|
+
at,
|
|
2985
|
+
},
|
|
2986
|
+
};
|
|
2987
|
+
});
|
|
2988
|
+
}
|
|
2989
|
+
return spawnLaneWorker(hqRoot, lane, {
|
|
2573
2990
|
env: ctx.env,
|
|
2574
|
-
|
|
2575
|
-
|
|
2576
|
-
|
|
2577
|
-
|
|
2578
|
-
|
|
2579
|
-
|
|
2580
|
-
|
|
2581
|
-
|
|
2582
|
-
|
|
2583
|
-
|
|
2584
|
-
|
|
2585
|
-
|
|
2586
|
-
|
|
2587
|
-
|
|
2588
|
-
|
|
2589
|
-
|
|
2590
|
-
|
|
2591
|
-
|
|
2991
|
+
modelDeps: {
|
|
2992
|
+
readFileSync: ctx.readFileSync,
|
|
2993
|
+
probe: ctx.probeProvider,
|
|
2994
|
+
env: ctx.env,
|
|
2995
|
+
},
|
|
2996
|
+
existsSync: ctx.existsSync,
|
|
2997
|
+
spawnProcess: ctx.spawnProcess,
|
|
2998
|
+
proveDetached: ctx.proveDetached,
|
|
2999
|
+
killProcess: ctx.killProcess,
|
|
3000
|
+
now: ctx.now,
|
|
3001
|
+
resolveMachineIdentity: ctx.resolveMachineIdentity,
|
|
3002
|
+
runCompaction: ctx.runCompaction,
|
|
3003
|
+
locateTranscript: ctx.locateTranscript,
|
|
3004
|
+
}, {
|
|
3005
|
+
trigger: {
|
|
3006
|
+
kind: "resume",
|
|
3007
|
+
from: resumedFrom,
|
|
3008
|
+
...(text !== undefined ? { text } : {}),
|
|
3009
|
+
...(messageId !== undefined ? { message_id: messageId } : {}),
|
|
3010
|
+
},
|
|
3011
|
+
compact: opts.compact !== false,
|
|
3012
|
+
});
|
|
3013
|
+
}, ctx.sleep);
|
|
3014
|
+
}
|
|
2592
3015
|
if (text !== undefined && messageId !== undefined) {
|
|
2593
3016
|
// Same audit copy a message resume leaves; nothing drains it. Archive only
|
|
2594
3017
|
// after a successful spawn so a concurrent resume cannot leave a phantom.
|
|
@@ -2618,6 +3041,18 @@ async function runResume(ctx, laneId, opts) {
|
|
|
2618
3041
|
...current,
|
|
2619
3042
|
project_id: projectId,
|
|
2620
3043
|
story_id: storyId,
|
|
3044
|
+
stories: laneStory(current, projectId, storyId)
|
|
3045
|
+
? laneStories(current)
|
|
3046
|
+
: [
|
|
3047
|
+
...laneStories(current),
|
|
3048
|
+
{
|
|
3049
|
+
project_id: projectId,
|
|
3050
|
+
story_id: storyId,
|
|
3051
|
+
status: "in_progress",
|
|
3052
|
+
added_at: at,
|
|
3053
|
+
updated_at: at,
|
|
3054
|
+
},
|
|
3055
|
+
],
|
|
2621
3056
|
attribution_history: appendCappedHistory(current.attribution_history, {
|
|
2622
3057
|
project_id: projectId,
|
|
2623
3058
|
story_id: storyId,
|
|
@@ -2635,7 +3070,11 @@ async function runResume(ctx, laneId, opts) {
|
|
|
2635
3070
|
message_id: messageId ?? null,
|
|
2636
3071
|
text: text ?? null,
|
|
2637
3072
|
delivery: "resume",
|
|
2638
|
-
delivery_status: move?.queued
|
|
3073
|
+
delivery_status: move?.queued
|
|
3074
|
+
? "queued_on_target_account"
|
|
3075
|
+
: queuedOnAdmission
|
|
3076
|
+
? "queued_on_admission"
|
|
3077
|
+
: "worker_spawned",
|
|
2639
3078
|
account_move: move ? accountMoveSummary(move) : null,
|
|
2640
3079
|
resumed_session_id: resumed.last_resume?.session_id ?? null,
|
|
2641
3080
|
last_resume: next.last_resume ?? null,
|
|
@@ -2644,9 +3083,245 @@ async function runResume(ctx, laneId, opts) {
|
|
|
2644
3083
|
: null,
|
|
2645
3084
|
project_id: next.project_id,
|
|
2646
3085
|
story_id: next.story_id,
|
|
3086
|
+
stories: laneStories(next),
|
|
3087
|
+
story_warning: storyWarning(next) ?? null,
|
|
2647
3088
|
lane: next,
|
|
2648
3089
|
});
|
|
2649
3090
|
}
|
|
3091
|
+
function storyCommandTarget(lane, storyId, project) {
|
|
3092
|
+
const projectId = project?.trim() || lane.project_id;
|
|
3093
|
+
const normalizedStoryId = storyId.trim();
|
|
3094
|
+
if (!projectId) {
|
|
3095
|
+
throw new LanesError("missing_project", "--project is required", {});
|
|
3096
|
+
}
|
|
3097
|
+
if (!normalizedStoryId) {
|
|
3098
|
+
throw new LanesError("missing_story", "<story-id> is required", {});
|
|
3099
|
+
}
|
|
3100
|
+
const story = laneStory(lane, projectId, normalizedStoryId);
|
|
3101
|
+
if (!story) {
|
|
3102
|
+
throw new LanesError("story_not_on_lane", `Story ${projectId}/${normalizedStoryId} is not carried by lane ${lane.lane_id}`, { lane_id: lane.lane_id, project_id: projectId, story_id: normalizedStoryId });
|
|
3103
|
+
}
|
|
3104
|
+
return { projectId, storyId: normalizedStoryId, story };
|
|
3105
|
+
}
|
|
3106
|
+
function updateCarriedStory(lane, projectId, storyId, status, at, extras = {}) {
|
|
3107
|
+
const stories = laneStories(lane).map((story) => story.project_id === projectId && story.story_id === storyId
|
|
3108
|
+
? {
|
|
3109
|
+
...story,
|
|
3110
|
+
status,
|
|
3111
|
+
updated_at: at,
|
|
3112
|
+
blocked_reason: extras.blocked_reason,
|
|
3113
|
+
verified_at: extras.verified_at,
|
|
3114
|
+
verified_by: extras.verified_by,
|
|
3115
|
+
}
|
|
3116
|
+
: story);
|
|
3117
|
+
return {
|
|
3118
|
+
...lane,
|
|
3119
|
+
project_id: projectId,
|
|
3120
|
+
story_id: storyId,
|
|
3121
|
+
stories,
|
|
3122
|
+
};
|
|
3123
|
+
}
|
|
3124
|
+
function addCarriedStory(lane, projectId, storyId, at) {
|
|
3125
|
+
if (laneStory(lane, projectId, storyId)) {
|
|
3126
|
+
throw new LanesError("story_already_on_lane", `Story ${projectId}/${storyId} is already carried by lane ${lane.lane_id}`, {
|
|
3127
|
+
lane_id: lane.lane_id,
|
|
3128
|
+
project_id: projectId,
|
|
3129
|
+
story_id: storyId,
|
|
3130
|
+
});
|
|
3131
|
+
}
|
|
3132
|
+
return {
|
|
3133
|
+
...lane,
|
|
3134
|
+
project_id: projectId,
|
|
3135
|
+
story_id: storyId,
|
|
3136
|
+
stories: [
|
|
3137
|
+
...laneStories(lane),
|
|
3138
|
+
{ project_id: projectId, story_id: storyId, status: "in_progress", added_at: at, updated_at: at },
|
|
3139
|
+
],
|
|
3140
|
+
};
|
|
3141
|
+
}
|
|
3142
|
+
function callerMayVerifyStory(ctx, lane) {
|
|
3143
|
+
const callerLaneId = ctx.env.HQ_LANE_ID?.trim();
|
|
3144
|
+
if (callerLaneId) {
|
|
3145
|
+
return lane.senior.kind === "lane" && lane.senior.id === callerLaneId;
|
|
3146
|
+
}
|
|
3147
|
+
if (lane.senior.kind === "user")
|
|
3148
|
+
return true;
|
|
3149
|
+
if (lane.senior.kind === "lane")
|
|
3150
|
+
return false;
|
|
3151
|
+
return sessionIdFromEnv(ctx.env) === lane.senior.id;
|
|
3152
|
+
}
|
|
3153
|
+
async function ensureStoryThread(ctx, lane, projectId) {
|
|
3154
|
+
const company = lane.mesh_company ?? lane.company;
|
|
3155
|
+
if (!company)
|
|
3156
|
+
return lane;
|
|
3157
|
+
const threadId = await ctx.mesh.ensureProjectThread(company, projectId);
|
|
3158
|
+
if (threadId === lane.thread_id)
|
|
3159
|
+
return lane;
|
|
3160
|
+
return casUpdateLane(ctx.hqRoot(), lane.lane_id, (current) => ({
|
|
3161
|
+
...current,
|
|
3162
|
+
thread_id: threadId,
|
|
3163
|
+
}));
|
|
3164
|
+
}
|
|
3165
|
+
async function emitStoryTransition(ctx, lane, projectId, storyId, event, status, payload = {}) {
|
|
3166
|
+
const company = lane.mesh_company ?? lane.company;
|
|
3167
|
+
if (!company)
|
|
3168
|
+
return;
|
|
3169
|
+
const ready = await ensureStoryThread(ctx, lane, projectId);
|
|
3170
|
+
if (!ready.thread_id)
|
|
3171
|
+
return;
|
|
3172
|
+
await emitLaneMeshEvent(ctx, company, ready.thread_id, event, {
|
|
3173
|
+
lane_id: ready.lane_id,
|
|
3174
|
+
project_id: projectId,
|
|
3175
|
+
story_id: storyId,
|
|
3176
|
+
...payload,
|
|
3177
|
+
});
|
|
3178
|
+
await ctx.mesh.patchStoryStatus(company, projectId, storyId, status === "blocked" ? "in_progress" : status);
|
|
3179
|
+
}
|
|
3180
|
+
function sameCommittedStory(current, expected, projectId, storyId) {
|
|
3181
|
+
const currentStory = laneStory(current, projectId, storyId);
|
|
3182
|
+
const expectedStory = laneStory(expected, projectId, storyId);
|
|
3183
|
+
return (current.project_id === expected.project_id &&
|
|
3184
|
+
current.story_id === expected.story_id &&
|
|
3185
|
+
JSON.stringify(currentStory) === JSON.stringify(expectedStory));
|
|
3186
|
+
}
|
|
3187
|
+
async function commitStoryTransition(ctx, before, next, projectId, storyId, event, status, payload = {}) {
|
|
3188
|
+
try {
|
|
3189
|
+
await emitStoryTransition(ctx, next, projectId, storyId, event, status, payload);
|
|
3190
|
+
}
|
|
3191
|
+
catch (err) {
|
|
3192
|
+
try {
|
|
3193
|
+
const current = readLane(ctx.hqRoot(), next.lane_id);
|
|
3194
|
+
if (!sameCommittedStory(current, next, projectId, storyId)) {
|
|
3195
|
+
throw new LanesError("story_transition_rollback_conflict", `Story ${projectId}/${storyId} changed while mesh publication failed`, { lane_id: next.lane_id, project_id: projectId, story_id: storyId });
|
|
3196
|
+
}
|
|
3197
|
+
updateLane(ctx.hqRoot(), next.lane_id, current.version, "command", (latest) => {
|
|
3198
|
+
if (!sameCommittedStory(latest, next, projectId, storyId)) {
|
|
3199
|
+
throw new LanesError("story_transition_rollback_conflict", `Story ${projectId}/${storyId} changed before rollback`, { lane_id: next.lane_id, project_id: projectId, story_id: storyId });
|
|
3200
|
+
}
|
|
3201
|
+
return {
|
|
3202
|
+
...latest,
|
|
3203
|
+
project_id: before.project_id,
|
|
3204
|
+
story_id: before.story_id,
|
|
3205
|
+
stories: before.stories,
|
|
3206
|
+
};
|
|
3207
|
+
});
|
|
3208
|
+
}
|
|
3209
|
+
catch (rollbackErr) {
|
|
3210
|
+
process.stderr.write(`hq lanes: story transition rollback failed for ${next.lane_id}/${projectId}/${storyId}: ${rollbackErr instanceof Error ? rollbackErr.message : String(rollbackErr)}\n`);
|
|
3211
|
+
}
|
|
3212
|
+
throw err;
|
|
3213
|
+
}
|
|
3214
|
+
}
|
|
3215
|
+
async function runStoryList(ctx, laneId) {
|
|
3216
|
+
const lane = readLane(ctx.hqRoot(), laneId);
|
|
3217
|
+
printJson({
|
|
3218
|
+
ok: true,
|
|
3219
|
+
action: "story.list",
|
|
3220
|
+
lane_id: laneId,
|
|
3221
|
+
stories: laneStories(lane),
|
|
3222
|
+
active_stories: activeLaneStories(lane),
|
|
3223
|
+
story_warning: storyWarning(lane) ?? null,
|
|
3224
|
+
});
|
|
3225
|
+
}
|
|
3226
|
+
async function runStoryAdd(ctx, laneId, storyId, opts) {
|
|
3227
|
+
const hqRoot = ctx.hqRoot();
|
|
3228
|
+
let lane = readLane(hqRoot, laneId);
|
|
3229
|
+
const projectId = opts.project?.trim() || lane.project_id;
|
|
3230
|
+
if (!projectId)
|
|
3231
|
+
throw new LanesError("missing_project", "--project is required", {});
|
|
3232
|
+
const normalizedStoryId = storyId.trim();
|
|
3233
|
+
if (!normalizedStoryId)
|
|
3234
|
+
throw new LanesError("missing_story", "<story-id> is required", {});
|
|
3235
|
+
const at = ctx.now().toISOString();
|
|
3236
|
+
lane = await ensureStoryThread(ctx, lane, projectId);
|
|
3237
|
+
const before = lane;
|
|
3238
|
+
const next = casUpdateLane(hqRoot, laneId, (current) => addCarriedStory(current, projectId, normalizedStoryId, at));
|
|
3239
|
+
await commitStoryTransition(ctx, before, next, projectId, normalizedStoryId, "lane_started", "in_progress");
|
|
3240
|
+
const finalLane = readLane(hqRoot, laneId);
|
|
3241
|
+
printJson({
|
|
3242
|
+
ok: true,
|
|
3243
|
+
action: "story.add",
|
|
3244
|
+
lane_id: laneId,
|
|
3245
|
+
project_id: projectId,
|
|
3246
|
+
story_id: normalizedStoryId,
|
|
3247
|
+
status: "in_progress",
|
|
3248
|
+
story_warning: storyWarning(finalLane) ?? null,
|
|
3249
|
+
lane: finalLane,
|
|
3250
|
+
});
|
|
3251
|
+
}
|
|
3252
|
+
async function runStoryStart(ctx, laneId, storyId, opts) {
|
|
3253
|
+
const hqRoot = ctx.hqRoot();
|
|
3254
|
+
let lane = readLane(hqRoot, laneId);
|
|
3255
|
+
const target = storyCommandTarget(lane, storyId, opts.project);
|
|
3256
|
+
if (target.story.status === "done") {
|
|
3257
|
+
throw new LanesError("story_already_done", `Story ${target.projectId}/${target.storyId} is done and cannot be started again`, {
|
|
3258
|
+
lane_id: laneId,
|
|
3259
|
+
project_id: target.projectId,
|
|
3260
|
+
story_id: target.storyId,
|
|
3261
|
+
});
|
|
3262
|
+
}
|
|
3263
|
+
const at = ctx.now().toISOString();
|
|
3264
|
+
lane = await ensureStoryThread(ctx, lane, target.projectId);
|
|
3265
|
+
const before = lane;
|
|
3266
|
+
const next = casUpdateLane(hqRoot, laneId, (current) => updateCarriedStory(current, target.projectId, target.storyId, "in_progress", at));
|
|
3267
|
+
await commitStoryTransition(ctx, before, next, target.projectId, target.storyId, "lane_started", "in_progress");
|
|
3268
|
+
const finalLane = readLane(hqRoot, laneId);
|
|
3269
|
+
printJson({ ok: true, action: "story.start", lane_id: laneId, project_id: target.projectId, story_id: target.storyId, status: "in_progress", story_warning: storyWarning(finalLane) ?? null, lane: finalLane });
|
|
3270
|
+
}
|
|
3271
|
+
async function runStoryBlock(ctx, laneId, storyId, opts) {
|
|
3272
|
+
const hqRoot = ctx.hqRoot();
|
|
3273
|
+
let lane = readLane(hqRoot, laneId);
|
|
3274
|
+
const target = storyCommandTarget(lane, storyId, opts.project);
|
|
3275
|
+
if (target.story.status === "done") {
|
|
3276
|
+
throw new LanesError("story_already_done", `Story ${target.projectId}/${target.storyId} is done and cannot be blocked again`, {
|
|
3277
|
+
lane_id: laneId,
|
|
3278
|
+
project_id: target.projectId,
|
|
3279
|
+
story_id: target.storyId,
|
|
3280
|
+
});
|
|
3281
|
+
}
|
|
3282
|
+
const reason = opts.reason?.trim();
|
|
3283
|
+
if (!reason)
|
|
3284
|
+
throw new LanesError("missing_block_reason", "--reason is required", {});
|
|
3285
|
+
const at = ctx.now().toISOString();
|
|
3286
|
+
lane = await ensureStoryThread(ctx, lane, target.projectId);
|
|
3287
|
+
const before = lane;
|
|
3288
|
+
const next = casUpdateLane(hqRoot, laneId, (current) => updateCarriedStory(current, target.projectId, target.storyId, "blocked", at, { blocked_reason: reason }));
|
|
3289
|
+
await commitStoryTransition(ctx, before, next, target.projectId, target.storyId, "lane_blocked", "blocked", { blocked_reason: reason });
|
|
3290
|
+
const finalLane = readLane(hqRoot, laneId);
|
|
3291
|
+
printJson({ ok: true, action: "story.block", lane_id: laneId, project_id: target.projectId, story_id: target.storyId, status: "blocked", blocked_reason: reason, story_warning: storyWarning(finalLane) ?? null, lane: finalLane });
|
|
3292
|
+
}
|
|
3293
|
+
async function runStoryDone(ctx, laneId, storyId, opts) {
|
|
3294
|
+
const hqRoot = ctx.hqRoot();
|
|
3295
|
+
let lane = readLane(hqRoot, laneId);
|
|
3296
|
+
if (!opts.verified) {
|
|
3297
|
+
throw new LanesError("senior_verification_required", "Only a senior may mark a story done; pass --verified after reviewing or merging it", { lane_id: laneId });
|
|
3298
|
+
}
|
|
3299
|
+
if (!callerMayVerifyStory(ctx, lane)) {
|
|
3300
|
+
throw new LanesError("senior_verification_required", `Caller is not the senior for lane ${laneId}`, {
|
|
3301
|
+
lane_id: laneId,
|
|
3302
|
+
senior: lane.senior,
|
|
3303
|
+
});
|
|
3304
|
+
}
|
|
3305
|
+
const target = storyCommandTarget(lane, storyId, opts.project);
|
|
3306
|
+
if (target.story.status !== "review") {
|
|
3307
|
+
throw new LanesError("story_not_ready_for_verification", `Story ${target.projectId}/${target.storyId} must be in review before senior verification`, {
|
|
3308
|
+
lane_id: laneId,
|
|
3309
|
+
project_id: target.projectId,
|
|
3310
|
+
story_id: target.storyId,
|
|
3311
|
+
status: target.story.status,
|
|
3312
|
+
});
|
|
3313
|
+
}
|
|
3314
|
+
const at = ctx.now().toISOString();
|
|
3315
|
+
lane = await ensureStoryThread(ctx, lane, target.projectId);
|
|
3316
|
+
const before = lane;
|
|
3317
|
+
const next = casUpdateLane(hqRoot, laneId, (current) => updateCarriedStory(current, target.projectId, target.storyId, "done", at, {
|
|
3318
|
+
verified_at: at,
|
|
3319
|
+
verified_by: current.senior,
|
|
3320
|
+
}));
|
|
3321
|
+
await commitStoryTransition(ctx, before, next, target.projectId, target.storyId, "lane_done", "done", { summary: "Senior verification complete" });
|
|
3322
|
+
const finalLane = readLane(hqRoot, laneId);
|
|
3323
|
+
printJson({ ok: true, action: "story.done", lane_id: laneId, project_id: target.projectId, story_id: target.storyId, status: "done", verified: true, story_warning: storyWarning(finalLane) ?? null, lane: finalLane });
|
|
3324
|
+
}
|
|
2650
3325
|
/**
|
|
2651
3326
|
* Registration state of the drain hook in the lane's account home, read the
|
|
2652
3327
|
* way the provider reads it. Null when the lane has no bound account yet
|
|
@@ -3443,6 +4118,7 @@ export function registerLanesCommand(parent, deps = {}) {
|
|
|
3443
4118
|
.option("--mesh-company <slug|uid>", "Explicit Work Mesh company for a personal lane; membership is verified")
|
|
3444
4119
|
.requiredOption("--project <slug>", "HQ project slug")
|
|
3445
4120
|
.requiredOption("--story <id>", "Story id from the project's prd.json")
|
|
4121
|
+
.option("--additional-story <id>", "Additional story carried by this lane; repeatable and discouraged when active", collectRepeatable)
|
|
3446
4122
|
.requiredOption("--brief-file <path>", "Path to the lane brief")
|
|
3447
4123
|
.option("--worker <name>", "HQ worker id from the registry")
|
|
3448
4124
|
.option("--provider <name>", "claude | codex | grok")
|
|
@@ -3468,6 +4144,14 @@ export function registerLanesCommand(parent, deps = {}) {
|
|
|
3468
4144
|
.action(wrap(async (opts) => {
|
|
3469
4145
|
await runList(ctx, opts);
|
|
3470
4146
|
}));
|
|
4147
|
+
lanes
|
|
4148
|
+
.command("admit")
|
|
4149
|
+
.description("Try to admit the oldest eligible queued lane")
|
|
4150
|
+
.argument("<lane-id>", "Lane id")
|
|
4151
|
+
.option("--force", "Senior/operator override for provider, account, and profile caps")
|
|
4152
|
+
.action(wrap(async (laneId, opts) => {
|
|
4153
|
+
await runAdmit(ctx, laneId, opts);
|
|
4154
|
+
}));
|
|
3471
4155
|
lanes
|
|
3472
4156
|
.command("watch")
|
|
3473
4157
|
.description("Watch lane state and question changes until all lanes are terminal")
|
|
@@ -3489,6 +4173,54 @@ export function registerLanesCommand(parent, deps = {}) {
|
|
|
3489
4173
|
.action(wrap(async (laneId) => {
|
|
3490
4174
|
await runShow(ctx, laneId);
|
|
3491
4175
|
}));
|
|
4176
|
+
const stories = lanes
|
|
4177
|
+
.command("story")
|
|
4178
|
+
.description("Track several project stories on one lane");
|
|
4179
|
+
stories
|
|
4180
|
+
.command("list")
|
|
4181
|
+
.description("List the stories carried by a lane")
|
|
4182
|
+
.argument("<lane-id>", "Lane id")
|
|
4183
|
+
.action(wrap(async (laneId) => {
|
|
4184
|
+
await runStoryList(ctx, laneId);
|
|
4185
|
+
}));
|
|
4186
|
+
stories
|
|
4187
|
+
.command("add")
|
|
4188
|
+
.description("Add and start another story on a lane")
|
|
4189
|
+
.argument("<lane-id>", "Lane id")
|
|
4190
|
+
.argument("<story-id>", "Story id")
|
|
4191
|
+
.option("--project <slug>", "Project id; defaults to the lane's current project")
|
|
4192
|
+
.action(wrap(async (laneId, storyId, opts) => {
|
|
4193
|
+
await runStoryAdd(ctx, laneId, storyId, opts);
|
|
4194
|
+
}));
|
|
4195
|
+
stories
|
|
4196
|
+
.command("start")
|
|
4197
|
+
.description("Start or resume a carried story")
|
|
4198
|
+
.argument("<lane-id>", "Lane id")
|
|
4199
|
+
.argument("<story-id>", "Story id")
|
|
4200
|
+
.option("--project <slug>", "Project id; defaults to the lane's current project")
|
|
4201
|
+
.action(wrap(async (laneId, storyId, opts) => {
|
|
4202
|
+
await runStoryStart(ctx, laneId, storyId, opts);
|
|
4203
|
+
}));
|
|
4204
|
+
stories
|
|
4205
|
+
.command("block")
|
|
4206
|
+
.description("Mark one carried story blocked")
|
|
4207
|
+
.argument("<lane-id>", "Lane id")
|
|
4208
|
+
.argument("<story-id>", "Story id")
|
|
4209
|
+
.requiredOption("--reason <text>", "Why the story is blocked")
|
|
4210
|
+
.option("--project <slug>", "Project id; defaults to the lane's current project")
|
|
4211
|
+
.action(wrap(async (laneId, storyId, opts) => {
|
|
4212
|
+
await runStoryBlock(ctx, laneId, storyId, opts);
|
|
4213
|
+
}));
|
|
4214
|
+
stories
|
|
4215
|
+
.command("done")
|
|
4216
|
+
.description("Mark a reviewed story done after senior verification")
|
|
4217
|
+
.argument("<lane-id>", "Lane id")
|
|
4218
|
+
.argument("<story-id>", "Story id")
|
|
4219
|
+
.requiredOption("--verified", "Confirm senior review or merge")
|
|
4220
|
+
.option("--project <slug>", "Project id; defaults to the lane's current project")
|
|
4221
|
+
.action(wrap(async (laneId, storyId, opts) => {
|
|
4222
|
+
await runStoryDone(ctx, laneId, storyId, opts);
|
|
4223
|
+
}));
|
|
3492
4224
|
lanes
|
|
3493
4225
|
.command("message")
|
|
3494
4226
|
.description("Send a follow-up message to a lane")
|