atris 3.74.28 → 3.75.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/atris/skills/calendar/SKILL.md +9 -0
- package/atris/skills/drive/SKILL.md +7 -0
- package/atris/skills/email-agent/SKILL.md +7 -0
- package/atris/skills/magic-inbox/SKILL.md +7 -0
- package/atris/skills/slides/SKILL.md +7 -0
- package/bin/atris.js +14 -2
- package/commands/engine.js +2 -2
- package/commands/integrations.js +539 -54
- package/commands/mission.js +2 -2
- package/commands/rating.js +14 -4
- package/commands/worktree.js +39 -12
- package/lib/fleet.js +584 -81
- package/lib/gm-season.js +31 -9
- package/lib/known-commands.js +1 -1
- package/lib/release-kind.js +31 -7
- package/lib/task-db.js +185 -26
- package/lib/worktree-space.js +4 -45
- package/package.json +1 -1
package/lib/fleet.js
CHANGED
|
@@ -33,7 +33,7 @@ const {
|
|
|
33
33
|
buildOneLapValidatorPrompt,
|
|
34
34
|
parseOneLapValidatorVerdict,
|
|
35
35
|
} = require('./one-lap-validator');
|
|
36
|
-
const { listWorktrees, defaultMainlineBase, gitFailureDetail, gitNetworkEnvironment } = require('../commands/worktree');
|
|
36
|
+
const { listWorktrees, defaultMainlineBase, gitFailureDetail, gitNetworkEnvironment, lowDiskRefusal } = require('../commands/worktree');
|
|
37
37
|
const { isConductorStatusLine } = require('./conductor-artifacts');
|
|
38
38
|
const { matchLessons } = require('./lesson-preflight');
|
|
39
39
|
const { sleepSync } = require('./sleep-sync');
|
|
@@ -44,7 +44,7 @@ const { buildVerifiedScoutPack, appendVerifiedScoutPack } = require('./dispatch-
|
|
|
44
44
|
const { runPrepPass, withPrepBrief, prepRecordFields } = require('./roster-prep');
|
|
45
45
|
const { mapNotesPath } = require('./map-refs');
|
|
46
46
|
const { SHARED_BRIEF_RULES } = require('./brief-rules');
|
|
47
|
-
const { PROOF_NOT_SAVED } = require('./release-kind');
|
|
47
|
+
const { PROOF_NOT_SAVED, COULD_NOT_START } = require('./release-kind');
|
|
48
48
|
const {
|
|
49
49
|
applySecretGrantEnvironment,
|
|
50
50
|
buildGatewaySupervisorScript,
|
|
@@ -53,18 +53,27 @@ const {
|
|
|
53
53
|
} = require('./secret-gateway');
|
|
54
54
|
const { appendEngineLiveLogChunk, createEngineLiveLog, engineTerminalReason } = require('./engine-job-lifecycle');
|
|
55
55
|
const { recordEngineResource } = require('./engine-resource');
|
|
56
|
+
const {
|
|
57
|
+
taskTagTokens,
|
|
58
|
+
isDecisionHoldTag,
|
|
59
|
+
isDecisionTask,
|
|
60
|
+
DECISION_REFUSE_REASON,
|
|
61
|
+
} = require('./task-decision');
|
|
56
62
|
|
|
57
63
|
// Lanes a fleet may never staff on its own: the human keeps irreversible
|
|
58
64
|
// calls. Mirrors the autoland denied lanes.
|
|
59
65
|
const DENIED_TAGS = ['billing', 'money', 'payments', 'deploy', 'security', 'customer', 'external', 'feedback'];
|
|
60
66
|
|
|
61
|
-
// The protected lane a task sits in
|
|
62
|
-
//
|
|
63
|
-
//
|
|
67
|
+
// The protected lane a task sits in, or null. This is the one place that
|
|
68
|
+
// decides protection: staffing (isSafeLane), the ship gate, the dispatch flight
|
|
69
|
+
// and the builder prompt all ask it, so a task one of them holds back as
|
|
70
|
+
// protected is never one that another would merge. A task's tags are its tag,
|
|
71
|
+
// its tags list, the tags `atris task tag` adds later and hashtags in its
|
|
72
|
+
// title (all through taskTagTokens), plus the todo tags it was imported with.
|
|
64
73
|
function protectedLaneOfTask(task) {
|
|
65
74
|
if (!task || typeof task !== 'object') return null;
|
|
66
75
|
const todoTags = task.metadata && Array.isArray(task.metadata.todo_tags) ? task.metadata.todo_tags : [];
|
|
67
|
-
const tags = [task
|
|
76
|
+
const tags = [...taskTagTokens(task), ...todoTags.filter(Boolean).map((tag) => String(tag).trim().toLowerCase())];
|
|
68
77
|
return tags.find((tag) => DENIED_TAGS.includes(tag)) || null;
|
|
69
78
|
}
|
|
70
79
|
|
|
@@ -126,7 +135,8 @@ function shellQuote(value) {
|
|
|
126
135
|
|
|
127
136
|
// The bounded prompt every engine gets: isolated worktree, MAP first, the
|
|
128
137
|
// task's own check, one landing order per mode. Yolo: the engine lands it
|
|
129
|
-
// itself with worktree ship
|
|
138
|
+
// itself with worktree ship, unless the task is in a protected lane. Otherwise:
|
|
139
|
+
// commit only, the conductor lands it.
|
|
130
140
|
function buildFleetPrompt(task, { worktreePath, yolo = false } = {}) {
|
|
131
141
|
const ref = task.display_id || task.id || 'TASK';
|
|
132
142
|
const title = String(task.title || '').trim();
|
|
@@ -134,11 +144,27 @@ function buildFleetPrompt(task, { worktreePath, yolo = false } = {}) {
|
|
|
134
144
|
const done = titleDone || String((task.metadata && task.metadata.done_looks_like) || '').trim();
|
|
135
145
|
const check = taskDispatchCheck(task, worktreePath || process.cwd());
|
|
136
146
|
const shipMessage = title.split(/[.:](?:\s|$)/)[0].replace(/"/g, "'").slice(0, 90) || ref;
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
147
|
+
// A protected-lane task is never shipped by its builder, yolo or not. The
|
|
148
|
+
// builder commits and stops, and the dispatch flight ships that commit once,
|
|
149
|
+
// without --merge, so a pull request opens and a person lands it. A builder
|
|
150
|
+
// that shipped too would open the pull request before the flight's own
|
|
151
|
+
// check, and if the flight's check or ship then failed, the claim would be
|
|
152
|
+
// released with that pull request open and unrecorded. Yolo still gives the
|
|
153
|
+
// builder its network for running checks.
|
|
154
|
+
const lane = yolo ? protectedLaneOfTask(task) : null;
|
|
155
|
+
let commitRule;
|
|
156
|
+
if (lane) {
|
|
157
|
+
commitRule = `This task is in the protected ${lane} lane, so a person lands it, not you. Commit on the current branch with a plain-English message. Do not push, do not run \`atris worktree ship\`, and do not create branches; the conductor ships your commit for a person to review.`;
|
|
158
|
+
} else if (yolo) {
|
|
159
|
+
commitRule = `Land it yourself. Commit on the current branch with a plain-English message, then run: atris worktree ship --message "${shipMessage} (${ref})" --verify ${shellQuote(check)} --merge, and report the PR URL. That ship command is the only push allowed. If it reports a conflict or a failed check, stop and report; never resolve conflicts yourself.`;
|
|
160
|
+
} else {
|
|
161
|
+
commitRule = 'Commit on the current branch with a clear message. Do not push and do not create branches; the conductor lands your commit.';
|
|
162
|
+
}
|
|
163
|
+
// The guard refuses a checkout that has edits in it, so it can only pass at
|
|
164
|
+
// the start. A builder that re-ran it in its review step, after its edits,
|
|
165
|
+
// took the failure as a stop order and left finished work uncommitted.
|
|
140
166
|
const lines = [
|
|
141
|
-
'
|
|
167
|
+
'Before your first edit, run `atris worktree guard` once; if it fails then, stop immediately, report back, and do not edit anything. It checks for a clean start, so it fails once edits exist: never run it again after editing.',
|
|
142
168
|
'',
|
|
143
169
|
ATRIS_BUILD_PROCESS_PREAMBLE,
|
|
144
170
|
'',
|
|
@@ -162,7 +188,7 @@ function buildFleetPrompt(task, { worktreePath, yolo = false } = {}) {
|
|
|
162
188
|
`- ${commitRule}`,
|
|
163
189
|
...METHOD_KERNEL.map((rule) => `- ${rule}`),
|
|
164
190
|
'',
|
|
165
|
-
`Final report (plain text): files changed, check command + result, commit sha${yolo ? ', PR URL' : ''} (or say what failed and why).`
|
|
191
|
+
`Final report (plain text): files changed, check command + result, commit sha${yolo && !lane ? ', PR URL' : ''} (or say what failed and why).`
|
|
166
192
|
);
|
|
167
193
|
const lessonFiles = [
|
|
168
194
|
...(Array.isArray(task.files) ? task.files : []),
|
|
@@ -926,10 +952,18 @@ async function dispatchEntryWithRestaff({
|
|
|
926
952
|
defaultPin = null,
|
|
927
953
|
job = 'build',
|
|
928
954
|
prepAsk = undefined,
|
|
955
|
+
beforeEngine = null,
|
|
929
956
|
}) {
|
|
930
957
|
const attempts = [];
|
|
931
|
-
|
|
932
|
-
|
|
958
|
+
try {
|
|
959
|
+
const result = await dispatchEntryAttempts({ entry, engine, root, dispatch, installedEngines, restaffState, scoutAsk, clock, defaultPin, attempts, job, prepAsk, beforeEngine });
|
|
960
|
+
return { ...result, attempts };
|
|
961
|
+
} catch (err) {
|
|
962
|
+
// One job's attempt blowing up (a retry brief that cannot be written, a store that throws) is that job's
|
|
963
|
+
// trouble. It comes back as a job whose attempt failed, for the flight to let its claim go where it failed,
|
|
964
|
+
// and the run goes on to its cleanup instead of ending with the other jobs' claims still held.
|
|
965
|
+
return { not_started: 'dispatch', detail: String(err && err.message || err), attempts };
|
|
966
|
+
}
|
|
933
967
|
}
|
|
934
968
|
|
|
935
969
|
// One prep pass per prep job per task: a backup whose line asks for the same
|
|
@@ -968,6 +1002,10 @@ async function dispatchEntryAttempts({
|
|
|
968
1002
|
attempts = [],
|
|
969
1003
|
job = 'build',
|
|
970
1004
|
prepAsk = undefined,
|
|
1005
|
+
// Called right before an engine is asked, every time one is: the assigned engine, a retry on another engine
|
|
1006
|
+
// after a dead one, a handover to the next worker (all after any scout, prep, and brief), so the flight can
|
|
1007
|
+
// check it still holds the claim and mark it started. If it throws, that engine is not asked.
|
|
1008
|
+
beforeEngine = null,
|
|
971
1009
|
}) {
|
|
972
1010
|
const basePrompt = entry.prompt || buildFleetPrompt(entry.task, { worktreePath: entry.worktreePath, yolo: entry.yolo });
|
|
973
1011
|
const prepFor = prepPassCache({ entry, root, clock, prepAsk, job });
|
|
@@ -989,16 +1027,39 @@ async function dispatchEntryAttempts({
|
|
|
989
1027
|
// A worker whose roster line asks for prep gets its brief first.
|
|
990
1028
|
const prep = ranWith.prep ? await prepFor(ranWith.prep, scouted) : null;
|
|
991
1029
|
const prompt = withPrepBrief(scouted, prep);
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1030
|
+
let brief;
|
|
1031
|
+
try {
|
|
1032
|
+
brief = captureDispatchBrief({
|
|
1033
|
+
root,
|
|
1034
|
+
task: entry.task,
|
|
1035
|
+
engine: engineName,
|
|
1036
|
+
worktreePath: entry.worktreePath,
|
|
1037
|
+
prompt,
|
|
1038
|
+
missionId: entry.mission_id || entry.missionId || '',
|
|
1039
|
+
author: entry.author || '',
|
|
1040
|
+
yolo: entry.yolo,
|
|
1041
|
+
});
|
|
1042
|
+
} catch (err) {
|
|
1043
|
+
// Before any engine has run, a brief that cannot be written means the job never started: hand that
|
|
1044
|
+
// back, so the flight pauses it and lets the claim go as one that could not start. After an engine has
|
|
1045
|
+
// run, it fails the flight as it always did.
|
|
1046
|
+
if (attempts.length) throw err;
|
|
1047
|
+
return { not_started: 'brief_capture', detail: String(err && err.message || err) };
|
|
1048
|
+
}
|
|
1049
|
+
// An engine is about to be asked, the assigned one or a later one (a retry, a handover), and each start gets
|
|
1050
|
+
// the same check on the claim as it stands now: a claim can be lost between one attempt and the next. A scout
|
|
1051
|
+
// or prep pass before this point never counts as the build starting. A mark that cannot be saved means the
|
|
1052
|
+
// start did not happen, so the job is handed back as not started and the engine is not asked. A mark refused
|
|
1053
|
+
// because the flight no longer holds the claim (claimLost, see markClaimStarted) is handed back as such: the
|
|
1054
|
+
// job is someone else's now. The mark of an attempt already made stands (marking twice adds nothing).
|
|
1055
|
+
if (beforeEngine) {
|
|
1056
|
+
try {
|
|
1057
|
+
beforeEngine(entry);
|
|
1058
|
+
} catch (err) {
|
|
1059
|
+
if (err && err.claimLost) return { not_started: 'claim_lost', detail: String(err.message || err), ...(err.unnamed ? { unnamed: true } : {}) };
|
|
1060
|
+
return { not_started: 'prepare', detail: String(err && err.message || err) };
|
|
1061
|
+
}
|
|
1062
|
+
}
|
|
1002
1063
|
const startedMs = clock();
|
|
1003
1064
|
let runResult;
|
|
1004
1065
|
try {
|
|
@@ -1019,6 +1080,8 @@ async function dispatchEntryAttempts({
|
|
|
1019
1080
|
};
|
|
1020
1081
|
|
|
1021
1082
|
const first = await runOnce(engine);
|
|
1083
|
+
// No engine was asked, so there is no engine health to record and no one to hand the job to.
|
|
1084
|
+
if (first.not_started) return first;
|
|
1022
1085
|
if (!first.max_seconds && entry.maxSeconds) first.max_seconds = entry.maxSeconds;
|
|
1023
1086
|
const deadEngine = detectDeadEngineDispatch(first);
|
|
1024
1087
|
recordDispatchEngineHealth(first, deadEngine, root);
|
|
@@ -1043,6 +1106,9 @@ async function dispatchEntryAttempts({
|
|
|
1043
1106
|
|
|
1044
1107
|
restaffState.used = true;
|
|
1045
1108
|
const fallbackResult = await runOnce(fallback);
|
|
1109
|
+
// The start check refused the retry (the claim is not this run's any more): no engine was asked, so the job is
|
|
1110
|
+
// handed back as not started, and there is no engine health to record.
|
|
1111
|
+
if (fallbackResult.not_started) return fallbackResult;
|
|
1046
1112
|
recordDispatchEngineHealth(fallbackResult, detectDeadEngineDispatch(fallbackResult), root);
|
|
1047
1113
|
return {
|
|
1048
1114
|
...fallbackResult,
|
|
@@ -1103,6 +1169,9 @@ async function handOverStalledDispatch({ entry, engine, first, root, runOnce, jo
|
|
|
1103
1169
|
prep: next.roster_prep || '',
|
|
1104
1170
|
};
|
|
1105
1171
|
const result = await runOnce(next.id, pin);
|
|
1172
|
+
// The start check refused this handover (see dispatchEntryAttempts): no engine was asked, so the job is
|
|
1173
|
+
// handed back as not started and no one else is tried.
|
|
1174
|
+
if (result.not_started) return result;
|
|
1106
1175
|
const dead = detectDeadEngineDispatch(result);
|
|
1107
1176
|
recordDispatchEngineHealth(result, dead, root);
|
|
1108
1177
|
last = result;
|
|
@@ -1358,20 +1427,6 @@ function dispatchToEngine({ task, engine, worktreePath, root = process.cwd(), ti
|
|
|
1358
1427
|
// ---------------------------------------------------------------------------
|
|
1359
1428
|
// T2: staffing
|
|
1360
1429
|
|
|
1361
|
-
const {
|
|
1362
|
-
taskTagTokens,
|
|
1363
|
-
isDecisionHoldTag,
|
|
1364
|
-
isDecisionTask,
|
|
1365
|
-
DECISION_REFUSE_REASON,
|
|
1366
|
-
} = require('./task-decision');
|
|
1367
|
-
|
|
1368
|
-
function taskTags(task) {
|
|
1369
|
-
// Tags added after creation live in metadata.tags (`atris task tag`); a
|
|
1370
|
-
// fleet that only read task.tags/title hashtags would ignore an owner-hold
|
|
1371
|
-
// flag stamped on a live task and keep restaffing it (CLI-879).
|
|
1372
|
-
return taskTagTokens(task);
|
|
1373
|
-
}
|
|
1374
|
-
|
|
1375
1430
|
// A task flagged for a human decision is never fleet-staffable, whatever its
|
|
1376
1431
|
// lane. Mirrors the sweep's needs-human hold so both loops agree. Also honors
|
|
1377
1432
|
// the clearer `decision` tag so policy questions stay off autonomous lanes.
|
|
@@ -1379,10 +1434,10 @@ function isHumanHoldTag(tag) {
|
|
|
1379
1434
|
return isDecisionHoldTag(tag);
|
|
1380
1435
|
}
|
|
1381
1436
|
|
|
1437
|
+
// The fleet may staff a task only when no person has to decide it and it sits
|
|
1438
|
+
// in no protected lane. Protection is protectedLaneOfTask's call alone.
|
|
1382
1439
|
function isSafeLane(task) {
|
|
1383
|
-
|
|
1384
|
-
const tags = taskTags(task);
|
|
1385
|
-
return !tags.some((t) => DENIED_TAGS.includes(t));
|
|
1440
|
+
return !isDecisionTask(task) && !protectedLaneOfTask(task);
|
|
1386
1441
|
}
|
|
1387
1442
|
|
|
1388
1443
|
// Rough file-surface guess from a task's text: paths and bare `commands/x.js`
|
|
@@ -1689,6 +1744,10 @@ module.exports = {
|
|
|
1689
1744
|
shipPullRequest,
|
|
1690
1745
|
DEAD_ENGINE_OUTPUT_PATTERNS,
|
|
1691
1746
|
releasePausedClaimsWithoutWork,
|
|
1747
|
+
dispatchEntryWithRestaff,
|
|
1748
|
+
flightExitCode,
|
|
1749
|
+
markClaimStarted,
|
|
1750
|
+
releaseOwnClaim,
|
|
1692
1751
|
reviewOnlyEngineEnvironment,
|
|
1693
1752
|
runInReapedProcessGroup,
|
|
1694
1753
|
shipWithRetry,
|
|
@@ -1713,12 +1772,10 @@ module.exports = {
|
|
|
1713
1772
|
assertIsolatedWorktree,
|
|
1714
1773
|
buildEngineCommand,
|
|
1715
1774
|
isSafeLane,
|
|
1716
|
-
taskTags,
|
|
1717
1775
|
nextInstalledFleetEngine,
|
|
1718
1776
|
rankFleetEnginesDetailed,
|
|
1719
1777
|
dispatchToEngine,
|
|
1720
1778
|
recordDispatchOutcome,
|
|
1721
|
-
taskTags,
|
|
1722
1779
|
isHumanHoldTag,
|
|
1723
1780
|
isDecisionTask,
|
|
1724
1781
|
DECISION_REFUSE_REASON,
|
|
@@ -3208,6 +3265,9 @@ async function runFleetFlight({
|
|
|
3208
3265
|
scoutAsk = null,
|
|
3209
3266
|
clock = Date.now,
|
|
3210
3267
|
prepAsk = undefined,
|
|
3268
|
+
// The disk check before anything is claimed (see lowDiskRefusal). Tests inject both.
|
|
3269
|
+
freeSpace = undefined,
|
|
3270
|
+
spaceGuard = undefined,
|
|
3211
3271
|
} = {}) {
|
|
3212
3272
|
const cli = ownCli || defaultOwnCli(root);
|
|
3213
3273
|
const landTarget = resolveDispatchLandTarget(root, checkoutBase);
|
|
@@ -3257,6 +3317,12 @@ async function runFleetFlight({
|
|
|
3257
3317
|
return flight;
|
|
3258
3318
|
}
|
|
3259
3319
|
|
|
3320
|
+
// One look at the disk before anything is claimed or written to any job, with worktree start's own
|
|
3321
|
+
// check and reaping. Under its floor every start would refuse anyway, and each refused start used to
|
|
3322
|
+
// let a claim go that the rating then scored as a drop.
|
|
3323
|
+
const lowDisk = lowDiskRefusal({ root, agent: 'fleet', task: 'flight', freeSpace, spaceGuard });
|
|
3324
|
+
if (lowDisk) return stopForLowDisk(flight, lowDisk, staffed.map((s) => ({ task: s.task.display_id, engine: s.engine })), log);
|
|
3325
|
+
|
|
3260
3326
|
const liveLogPath = startLiveFlight(root, receiptPath, flight);
|
|
3261
3327
|
|
|
3262
3328
|
// Claim + cut a worktree per assignment, then dispatch all in parallel.
|
|
@@ -3291,8 +3357,25 @@ async function runFleetFlight({
|
|
|
3291
3357
|
// jobs this flight claimed are in them, so a listed job it refused cannot be mistaken for one.
|
|
3292
3358
|
const claimedRefs = new Map();
|
|
3293
3359
|
const claimedActors = new Map();
|
|
3360
|
+
// The claim this flight made on each job, remembered from the claim write itself (see claimJob).
|
|
3361
|
+
const claims = claimLedger();
|
|
3362
|
+
// A claim let go before the engine ran is recorded in this process, on the claim this flight made, as one
|
|
3363
|
+
// that could not start. Any other claim the flight lets go (the engine was asked) is an ordinary release.
|
|
3364
|
+
const letGoUnstarted = ({ ref, actor, reason }) => releaseOwnClaim({ taskDb: leaseDb, db: leaseConnection, root, ref, actor, reason, claimEventId: claims.eventId(ref), neverStarted: true });
|
|
3365
|
+
const letGoAsked = ({ ref, actor, reason }) => releaseOwnClaim({ taskDb: leaseDb, db: leaseConnection, root, ref, actor, reason, claimEventId: claims.eventId(ref) });
|
|
3366
|
+
// The start-failure releases the store failed, for the closing pass to try again (see releaseUnstartedDispatchClaim).
|
|
3367
|
+
const releaseFailures = new Map();
|
|
3368
|
+
// Right before the assigned engine is asked, the claim is marked started.
|
|
3369
|
+
const markStarted = (entry) => markClaimStarted({
|
|
3370
|
+
taskDb: leaseDb,
|
|
3371
|
+
db: leaseConnection,
|
|
3372
|
+
claims,
|
|
3373
|
+
ref: claimedRefs.get(String(entry.task.display_id)) || entry.task.display_id,
|
|
3374
|
+
actor: claimedActors.get(String(entry.task.display_id)) || `fleet-${entry.engine}`,
|
|
3375
|
+
task: entry.task.display_id,
|
|
3376
|
+
});
|
|
3294
3377
|
const leaseTimer = setInterval(() => {
|
|
3295
|
-
for (const [id, holder] of leased) leaseDb.renewLease(leaseConnection, { id, holder, pid: process.pid });
|
|
3378
|
+
for (const [id, holder] of leased) leaseDb.renewLease(leaseConnection, { id, holder, pid: process.pid, claimEventId: claims.eventId(id) });
|
|
3296
3379
|
}, 5 * 60000);
|
|
3297
3380
|
leaseTimer.unref();
|
|
3298
3381
|
try {
|
|
@@ -3308,7 +3391,7 @@ async function runFleetFlight({
|
|
|
3308
3391
|
// The row the list named is the row this claims, leases, proves and lets go of: named once,
|
|
3309
3392
|
// by its row id, at every step (see jobRef).
|
|
3310
3393
|
const rowRef = jobRef(entry.task, entry.task.display_id);
|
|
3311
|
-
const claimed = cli
|
|
3394
|
+
const claimed = claimJob({ cli, taskDb: leaseDb, db: leaseConnection, root, ref: rowRef, actor: `fleet-${entry.engine}` });
|
|
3312
3395
|
if (!claimed || claimed.status !== 0) {
|
|
3313
3396
|
const detail = String(claimed && (claimed.stderr || claimed.stdout) || 'claim failed').trim().slice(0, 300);
|
|
3314
3397
|
flight.paused.push({ task: entry.task.display_id, engine: entry.engine, stage: 'claim', detail });
|
|
@@ -3319,20 +3402,67 @@ async function runFleetFlight({
|
|
|
3319
3402
|
const holder = `fleet-${entry.engine}`;
|
|
3320
3403
|
claimedRefs.set(String(entry.task.display_id), rowRef);
|
|
3321
3404
|
claimedActors.set(String(entry.task.display_id), holder);
|
|
3322
|
-
|
|
3323
|
-
|
|
3324
|
-
|
|
3325
|
-
const
|
|
3326
|
-
|
|
3327
|
-
|
|
3328
|
-
|
|
3329
|
-
|
|
3405
|
+
claims.remember(rowRef, claimed.eventId);
|
|
3406
|
+
// From here until the engine is asked, a job that cannot start is let go where it fails, as one that could
|
|
3407
|
+
// not start, so a failure later in the flight cannot leave its claim to run out as a miss.
|
|
3408
|
+
const cannotStart = (stage, detail) => {
|
|
3409
|
+
const released = releaseUnstartedDispatchClaim(cli, { taskId: entry.task.display_id, ref: rowRef, actor: holder, detail, stage, release: letGoUnstarted, failures: releaseFailures, log });
|
|
3410
|
+
flight.paused.push({ task: entry.task.display_id, engine: entry.engine, stage, detail, claim_released: released.released });
|
|
3411
|
+
};
|
|
3412
|
+
try {
|
|
3413
|
+
const leaseSet = leaseDb.setLease(leaseConnection, { id: leaseId, holder, claimEventId: claims.eventId(leaseId) });
|
|
3414
|
+
if (!leaseSet && (!ownCli || leaseDb.getTask(leaseConnection, leaseId))) throw new Error('could not set claim lease');
|
|
3415
|
+
if (leaseSet) leased.set(leaseId, holder);
|
|
3416
|
+
const started = cli(['worktree', 'start', '--agent', entry.engine, '--task', `fleet-${String(entry.task.display_id).toLowerCase()}`, ...startBaseArgs]);
|
|
3417
|
+
const wt = (started.stdout.match(/next: cd (.+)/) || [])[1];
|
|
3418
|
+
if (!wt) {
|
|
3419
|
+
cannotStart('worktree_start', started.stderr.slice(0, 200));
|
|
3420
|
+
continue;
|
|
3421
|
+
}
|
|
3422
|
+
prepared.push({ ...entry, worktreePath: wt.trim() });
|
|
3423
|
+
log(` ${entry.engine.padEnd(8)} building ${entry.task.display_id} in ${path.basename(wt.trim())}`);
|
|
3424
|
+
} catch (err) {
|
|
3425
|
+
cannotStart('prepare', String(err && err.message || err).slice(0, 200));
|
|
3426
|
+
log(` ${entry.engine.padEnd(8)} ✗ ${entry.task.display_id} prepare failed`);
|
|
3330
3427
|
}
|
|
3331
|
-
prepared.push({ ...entry, worktreePath: wt.trim() });
|
|
3332
|
-
log(` ${entry.engine.padEnd(8)} building ${entry.task.display_id} in ${path.basename(wt.trim())}`);
|
|
3333
3428
|
}
|
|
3334
3429
|
|
|
3335
|
-
|
|
3430
|
+
// A job that never reached its engine (its brief could not be written, or its start could not be marked)
|
|
3431
|
+
// pauses and its claim goes back as one that could not start, here, where it failed, while the flight's
|
|
3432
|
+
// other jobs go on. The jobs that did start carry on.
|
|
3433
|
+
const letGoNotStarted = (entry, result) => {
|
|
3434
|
+
const taskLabel = entry.task.display_id;
|
|
3435
|
+
const stage = result.not_started;
|
|
3436
|
+
const ref = claimedRefs.get(String(taskLabel)) || taskLabel;
|
|
3437
|
+
if (stage === 'claim_lost') {
|
|
3438
|
+
// The job is another run's now (see markClaimStarted). The flight lets go of nothing, writes nothing to it, and
|
|
3439
|
+
// stops renewing a lease on it.
|
|
3440
|
+
leased.delete(ref);
|
|
3441
|
+
flight.paused.push({ task: taskLabel, engine: entry.engine, stage, detail: result.unnamed ? UNNAMED_CLAIM_DETAIL : LOST_CLAIM_DETAIL, worktree: entry.worktreePath, claim_released: false });
|
|
3442
|
+
log(result.unnamed ? unnamedClaimLine(taskLabel) : claimLostLine(taskLabel));
|
|
3443
|
+
return;
|
|
3444
|
+
}
|
|
3445
|
+
// Nothing in here may end the run: the other jobs still have their cleanup to come.
|
|
3446
|
+
let detail = result.detail;
|
|
3447
|
+
let released = false;
|
|
3448
|
+
try {
|
|
3449
|
+
({ released, detail } = releaseUnstartedDispatchClaim(cli, {
|
|
3450
|
+
taskId: taskLabel,
|
|
3451
|
+
ref,
|
|
3452
|
+
actor: claimedActors.get(String(taskLabel)) || `fleet-${entry.engine}`,
|
|
3453
|
+
detail: result.detail,
|
|
3454
|
+
stage,
|
|
3455
|
+
release: letGoUnstarted,
|
|
3456
|
+
failures: releaseFailures,
|
|
3457
|
+
log,
|
|
3458
|
+
}));
|
|
3459
|
+
} catch (err) {
|
|
3460
|
+
detail = `${String(result.detail || '').slice(0, 120)}. claim release failed: ${String(err && err.message || err).slice(0, 80)}`;
|
|
3461
|
+
}
|
|
3462
|
+
flight.paused.push({ task: taskLabel, engine: entry.engine, stage, detail, worktree: entry.worktreePath, claim_released: released });
|
|
3463
|
+
log(` ${entry.engine.padEnd(8)} ✗ ${taskLabel} ${NOT_STARTED_WORDS[stage] || 'prepare'} failed`);
|
|
3464
|
+
};
|
|
3465
|
+
const settled = await Promise.all(prepared.map((entry) =>
|
|
3336
3466
|
dispatchEntryWithRestaff({
|
|
3337
3467
|
entry,
|
|
3338
3468
|
engine: entry.engine,
|
|
@@ -3344,8 +3474,13 @@ async function runFleetFlight({
|
|
|
3344
3474
|
clock,
|
|
3345
3475
|
defaultPin: fleetPrepPin(entry.engine, root),
|
|
3346
3476
|
prepAsk,
|
|
3347
|
-
|
|
3477
|
+
beforeEngine: markStarted,
|
|
3478
|
+
}).then((result) => {
|
|
3479
|
+
if (result.not_started) letGoNotStarted(entry, result);
|
|
3480
|
+
return { entry, result };
|
|
3481
|
+
})
|
|
3348
3482
|
));
|
|
3483
|
+
const results = settled.filter(({ result }) => !result.not_started);
|
|
3349
3484
|
flight.results = results.map(({ entry, result }) => {
|
|
3350
3485
|
const row = { task: entry.task.display_id, engine: result.engine || entry.engine, exitCode: result.exitCode };
|
|
3351
3486
|
if (result.brief_id) row.brief_id = result.brief_id;
|
|
@@ -3438,6 +3573,8 @@ async function runFleetFlight({
|
|
|
3438
3573
|
refByTask: claimedRefs,
|
|
3439
3574
|
baseRef: landTarget,
|
|
3440
3575
|
log,
|
|
3576
|
+
release: letGoAsked,
|
|
3577
|
+
failures: releaseFailures,
|
|
3441
3578
|
});
|
|
3442
3579
|
|
|
3443
3580
|
recordFlightRosterRuns(root, flight, results, { job: 'build', source: 'fleet' });
|
|
@@ -3452,7 +3589,7 @@ async function runFleetFlight({
|
|
|
3452
3589
|
clearInterval(leaseTimer);
|
|
3453
3590
|
for (const [id, holder] of leased) {
|
|
3454
3591
|
const row = leaseDb.getTask(leaseConnection, id);
|
|
3455
|
-
if (row && row.status !== 'claimed') leaseDb.clearLease(leaseConnection, { id, holder });
|
|
3592
|
+
if (row && row.status !== 'claimed') leaseDb.clearLease(leaseConnection, { id, holder, claimEventId: claims.eventId(id) });
|
|
3456
3593
|
}
|
|
3457
3594
|
}
|
|
3458
3595
|
}
|
|
@@ -3472,7 +3609,7 @@ function releaseReason(lead, detail) {
|
|
|
3472
3609
|
}
|
|
3473
3610
|
|
|
3474
3611
|
// Pauses where the engine never got to finish its turn.
|
|
3475
|
-
const ENGINE_DID_NOT_FINISH = new Set(['worktree_start', 'remote_quarantine', 'prepare', 'build']);
|
|
3612
|
+
const ENGINE_DID_NOT_FINISH = new Set(['worktree_start', 'remote_quarantine', 'prepare', 'brief_capture', 'build']);
|
|
3476
3613
|
|
|
3477
3614
|
// The word a flight uses to name a job to the task command at every step: its row id when the
|
|
3478
3615
|
// task command (or the task list) gave one, else the display id the flight has. The display id a
|
|
@@ -3496,17 +3633,225 @@ function jobOutsideWorkspace(taskDb, task, workspace) {
|
|
|
3496
3633
|
|
|
3497
3634
|
const namesAnotherWorkspace = (displayId) => `${displayId} names a job in another workspace`;
|
|
3498
3635
|
|
|
3636
|
+
// What a flight does when the disk is under worktree start's floor. It has claimed nothing and written
|
|
3637
|
+
// nothing to any job, so it says why in one plain line (the free space and the floor, in worktree start's
|
|
3638
|
+
// own words) and hands back a failed flight that names the refusal. flightExitCode turns that into the
|
|
3639
|
+
// exit code worktree start itself gives. jobs are the rows the flight would have worked on.
|
|
3640
|
+
function stopForLowDisk(flight, refusal, jobs, log) {
|
|
3641
|
+
log(` refusing: ${refusal.message}`);
|
|
3642
|
+
flight.refusal = { reason: 'low_disk', detail: refusal.message, free_bytes: refusal.free, needs_bytes: refusal.needs };
|
|
3643
|
+
flight.paused = jobs.map((job) => ({ ...job, stage: 'low_disk', detail: refusal.message }));
|
|
3644
|
+
flight.status = 'failed';
|
|
3645
|
+
flight.finished_at = new Date().toISOString();
|
|
3646
|
+
return flight;
|
|
3647
|
+
}
|
|
3648
|
+
|
|
3649
|
+
// The exit code for a finished flight: 2 when it stopped on a low disk, as worktree start does, else 1
|
|
3650
|
+
// when any job paused, else 0.
|
|
3651
|
+
function flightExitCode(flight) {
|
|
3652
|
+
if (flight && flight.refusal && flight.refusal.reason === 'low_disk') return 2;
|
|
3653
|
+
return flight && flight.paused && flight.paused.length > 0 ? 1 : 0;
|
|
3654
|
+
}
|
|
3655
|
+
|
|
3656
|
+
// The two files people read, drawn again from the saved state the way the task command does after
|
|
3657
|
+
// every release and every claim: the task list file, and atris/TODO.md of the flight's own workspace.
|
|
3658
|
+
// The write is already saved, so a file that cannot be written does not undo it, and one failing does
|
|
3659
|
+
// not stop the other. The task command is loaded only here: this file is read on every start, and only
|
|
3660
|
+
// the paths that write to a job need it.
|
|
3661
|
+
function redrawTaskFiles({ taskDb, db, root }) {
|
|
3662
|
+
for (const [what, refresh] of [
|
|
3663
|
+
['task list file', () => require('../commands/task').writeDefaultProjection(taskDb, db, { root })],
|
|
3664
|
+
['TODO.md', () => require('../commands/task').autoRenderTodoFromDb(taskDb.workspaceRoot(root))],
|
|
3665
|
+
]) {
|
|
3666
|
+
try { refresh(); } catch (err) {
|
|
3667
|
+
if (process.env.ATRIS_DEBUG) process.stderr.write(`${what} refresh failed: ${String(err?.message ?? err).replace(/[\r\n]+/g, ' ')}\n`);
|
|
3668
|
+
}
|
|
3669
|
+
}
|
|
3670
|
+
}
|
|
3671
|
+
|
|
3672
|
+
// Which claim this flight made on each job, by the row id it claimed. A mark or a release checks the job is
|
|
3673
|
+
// still held under that claim before it writes anything: another run can take a job over under the same
|
|
3674
|
+
// builder name (a lease that ran out, then a second flight), and a late release from this flight must not
|
|
3675
|
+
// land on, or stamp, a claim that belongs to someone else. The name of the claim comes from the claim write
|
|
3676
|
+
// itself (see claimJob) and is remembered the moment it is made; it is never looked up afterwards, because
|
|
3677
|
+
// a look-up cannot tell this flight's claim from one made just after it. A job claimed through the task
|
|
3678
|
+
// command (a stand-in task plane, a job shown with no row id) has no name to remember.
|
|
3679
|
+
function claimLedger() {
|
|
3680
|
+
const made = new Map();
|
|
3681
|
+
return {
|
|
3682
|
+
remember(ref, eventId) {
|
|
3683
|
+
if (ref && eventId) made.set(String(ref), String(eventId));
|
|
3684
|
+
},
|
|
3685
|
+
// The event that is this flight's claim on the job, or '' when none was remembered.
|
|
3686
|
+
eventId(ref) {
|
|
3687
|
+
return made.get(String(ref)) || '';
|
|
3688
|
+
},
|
|
3689
|
+
};
|
|
3690
|
+
}
|
|
3691
|
+
|
|
3692
|
+
// True when the task store holds a row under this name. It decides whether a flight may act on a job without a
|
|
3693
|
+
// claim to name, so a store that cannot be read counts as holding it: doubt means it may not.
|
|
3694
|
+
function storeHoldsJob(taskDb, db, ref) {
|
|
3695
|
+
if (!ref) return false;
|
|
3696
|
+
try {
|
|
3697
|
+
return Boolean(taskDb.getTask(db, ref));
|
|
3698
|
+
} catch {
|
|
3699
|
+
return true;
|
|
3700
|
+
}
|
|
3701
|
+
}
|
|
3702
|
+
|
|
3703
|
+
// How a flight claims a job. A job the task store holds a row for is claimed in the flight's own process,
|
|
3704
|
+
// through the store it already holds for leases, and the claim's name (its claimed event) comes from the claim
|
|
3705
|
+
// write itself: claimTask saves the row and the event in one transaction and answers with the event, so there is
|
|
3706
|
+
// no moment in which the flight has claimed a job but cannot say which claim is its own. What the task command
|
|
3707
|
+
// did besides the write stays: the check on the actor's name, the plain failure text, the two files people
|
|
3708
|
+
// read drawn again (see redrawTaskFiles), and the clearing of the claims that have run out, which every task
|
|
3709
|
+
// command begins with, so a job whose old claim is dead is cleared and claimed in one go. A job the store holds
|
|
3710
|
+
// no row for (a stand-in task plane, a job shown with no row id) has no claim to name and is claimed through the
|
|
3711
|
+
// task command, as it always was.
|
|
3712
|
+
//
|
|
3713
|
+
// It answers like the task command (status 0, or status 1 and why), with eventId when it claimed in process. A
|
|
3714
|
+
// claim that landed but came back with no name is refused: the flight cannot tell later whether the job is
|
|
3715
|
+
// still its own, so it builds nothing on it and leaves it alone.
|
|
3716
|
+
function claimJob({ cli, taskDb, db, root, ref, actor, minutes = 60 }) {
|
|
3717
|
+
if (!storeHoldsJob(taskDb, db, ref)) {
|
|
3718
|
+
return cli(['task', 'claim', ref, '--as', actor, '--lease-minutes', String(minutes), '--lease-pid', String(process.pid)]);
|
|
3719
|
+
}
|
|
3720
|
+
const check = require('./review-integrity').validateActor(actor, { root });
|
|
3721
|
+
if (!check.ok) {
|
|
3722
|
+
return {
|
|
3723
|
+
status: 1,
|
|
3724
|
+
stdout: '',
|
|
3725
|
+
stderr: check.reason === 'reserved_actor'
|
|
3726
|
+
? `atris task claim: reserved_actor: '${actor}' is a system actor and cannot be used with --as`
|
|
3727
|
+
: `atris task claim: actor_not_on_roster: '${actor}' is not a workspace member or engine (actor validation is enforced)`,
|
|
3728
|
+
};
|
|
3729
|
+
}
|
|
3730
|
+
// The step the task command takes first, with its reach (every claim in the store that has run out) and the
|
|
3731
|
+
// history it writes (a claim_expired entry for each), because it is the command's own function. It never throws.
|
|
3732
|
+
taskDb.sweepExpiredLeases();
|
|
3733
|
+
let result;
|
|
3734
|
+
try {
|
|
3735
|
+
result = taskDb.claimTask(db, { id: ref, claimedBy: actor, leaseMinutes: minutes, leasePid: process.pid });
|
|
3736
|
+
} catch (err) {
|
|
3737
|
+
return { status: 1, stdout: '', stderr: `claim failed: ${String(err?.message ?? err).trim() || 'unknown'}` };
|
|
3738
|
+
}
|
|
3739
|
+
if (!result.claimed) {
|
|
3740
|
+
return { status: 1, stdout: '', stderr: `claim failed: ${result.reason}${result.claimed_by ? ` (held by ${result.claimed_by})` : ''}` };
|
|
3741
|
+
}
|
|
3742
|
+
redrawTaskFiles({ taskDb, db, root });
|
|
3743
|
+
const eventId = result.event && result.event.event_id;
|
|
3744
|
+
if (!eventId) return { status: 1, stdout: '', stderr: UNNAMED_CLAIM_DETAIL, unnamed: true };
|
|
3745
|
+
return { status: 0, stdout: `claimed ${ref} as ${actor}`, stderr: '', eventId };
|
|
3746
|
+
}
|
|
3747
|
+
|
|
3748
|
+
// The one plain line for a job another run has claimed since this flight made its own claim.
|
|
3749
|
+
const claimTakenOverLine = (task) => ` left ${task} alone: another run claimed it after this one did`;
|
|
3750
|
+
|
|
3751
|
+
// The one plain line for a job whose claim this flight no longer holds when it is about to ask the engine.
|
|
3752
|
+
const LOST_CLAIM_DETAIL = 'this run no longer holds the claim, so the engine was not started and the job was left alone';
|
|
3753
|
+
const claimLostLine = (task) => ` ${task}: ${LOST_CLAIM_DETAIL}`;
|
|
3754
|
+
|
|
3755
|
+
// The one plain line for a job the flight has claimed but cannot name the claim of: it cannot tell whether the job
|
|
3756
|
+
// is still its own, so it writes nothing to it, releases nothing, and starts no engine on it.
|
|
3757
|
+
const UNNAMED_CLAIM_DETAIL = 'this run has no record of which claim is its own, so the job was left alone';
|
|
3758
|
+
const unnamedClaimLine = (task) => ` ${task}: ${UNNAMED_CLAIM_DETAIL}`;
|
|
3759
|
+
|
|
3760
|
+
// What a flight calls the step a job failed at, in the one line it logs when it lets that job's claim go.
|
|
3761
|
+
const NOT_STARTED_WORDS = { brief_capture: 'brief capture', dispatch: 'attempt', prepare: 'prepare' };
|
|
3762
|
+
|
|
3763
|
+
// Right before an engine is asked, the first or a later one (a retry, a handover), the flight marks the claim it
|
|
3764
|
+
// made as started (see markBuildStarted in task-db.js). That is what keeps the could-not-start kind honest: the
|
|
3765
|
+
// store and the rating take it only for a claim with no mark. A job the store does not hold under this holder (a
|
|
3766
|
+
// stand-in task plane) has no claim to mark and goes on. A mark refused for a claim the flight remembered means
|
|
3767
|
+
// the claim is not the flight's any more (it was let go, or taken over, even by a run of the same name): the
|
|
3768
|
+
// engine is not asked, and the error says so (claimLost) for the flight to stop on, writing nothing more. A job
|
|
3769
|
+
// the store holds that the flight has no name for the claim of stops the same way (claimLost and unnamed).
|
|
3770
|
+
// A mark that cannot be saved throws like any store trouble, and the engine is not asked either.
|
|
3771
|
+
function markClaimStarted({ taskDb, db, claims, ref, actor, task }) {
|
|
3772
|
+
const claimEventId = claims.eventId(ref);
|
|
3773
|
+
if (!claimEventId && storeHoldsJob(taskDb, db, ref)) {
|
|
3774
|
+
// A job the store holds is claimed in process, so a flight with no name for its claim cannot say the job is
|
|
3775
|
+
// still its own: the engine is not asked, and nothing is written.
|
|
3776
|
+
const unnamed = new Error(`${task}: ${UNNAMED_CLAIM_DETAIL}`);
|
|
3777
|
+
unnamed.claimLost = true;
|
|
3778
|
+
unnamed.unnamed = true;
|
|
3779
|
+
throw unnamed;
|
|
3780
|
+
}
|
|
3781
|
+
const result = taskDb.markBuildStarted(db, { id: ref, actor, claimEventId });
|
|
3782
|
+
if (!result.marked && claimEventId) {
|
|
3783
|
+
const lost = new Error(`the claim on ${task} is no longer this run's (${result.reason})`);
|
|
3784
|
+
lost.claimLost = true;
|
|
3785
|
+
throw lost;
|
|
3786
|
+
}
|
|
3787
|
+
return result;
|
|
3788
|
+
}
|
|
3789
|
+
|
|
3790
|
+
// A flight lets go of a claim in its own process, on the exact claim it made, through the task store it already
|
|
3791
|
+
// holds for leases. ref is the row id the claim named and claimEventId the claim event the flight remembered
|
|
3792
|
+
// (see claimLedger): the store lets go only if the job is still held under that claim, in the same write, so a
|
|
3793
|
+
// job another run took over since, even under the same builder name and in the same millisecond, is never
|
|
3794
|
+
// released by a late call of this flight's. `atris task release` takes no kind and no claim, because any worker
|
|
3795
|
+
// holding a failed job could pass one to excuse its own miss; only the flights say these things, here.
|
|
3796
|
+
//
|
|
3797
|
+
// neverStarted asks for the could-not-start kind (lib/release-kind.js): the build machine could not start the
|
|
3798
|
+
// job, so it is nobody's miss. The store keeps that honest: it drops the kind from a claim marked started (see
|
|
3799
|
+
// markClaimStarted), and the release then goes through as an ordinary one, so a claim is never left held.
|
|
3800
|
+
//
|
|
3801
|
+
// It answers like the task command (status 0, or status 1 and why), with a word more: not_ours when the job is
|
|
3802
|
+
// not held under the flight's claim any more (nothing was written, nothing is retried, and the caller says so),
|
|
3803
|
+
// final when it is already let go or gone, retryable when the store failed (the closing pass tries once more),
|
|
3804
|
+
// unnamed when the store holds the job but the flight has no name for its claim (nothing is written and nothing
|
|
3805
|
+
// is retried: a release that names no claim could land on anyone's, so the job is left alone and the caller says
|
|
3806
|
+
// so). Null only when the store holds no row for the job (a job shown with no row id, a stand-in task plane): the
|
|
3807
|
+
// caller then lets it go through the task command, as a plain release, the way it always did.
|
|
3808
|
+
function releaseOwnClaim({ taskDb, db, root, ref, actor, reason, claimEventId = '', neverStarted = false }) {
|
|
3809
|
+
if (!claimEventId) {
|
|
3810
|
+
return storeHoldsJob(taskDb, db, ref) ? { status: 1, stdout: '', stderr: UNNAMED_CLAIM_DETAIL, unnamed: true } : null;
|
|
3811
|
+
}
|
|
3812
|
+
let result;
|
|
3813
|
+
try {
|
|
3814
|
+
result = taskDb.releaseTask(db, { id: ref, actor, reason, claimEventId, ...(neverStarted ? { kind: COULD_NOT_START } : {}) });
|
|
3815
|
+
} catch (err) {
|
|
3816
|
+
return { status: 1, stdout: '', stderr: String(err?.message ?? err).trim() || 'unknown', retryable: true };
|
|
3817
|
+
}
|
|
3818
|
+
if (!result.released) {
|
|
3819
|
+
const stderr = `release failed: ${result.reason}${result.claimed_by ? ` (held by ${result.claimed_by})` : ''}`;
|
|
3820
|
+
if (result.reason === 'claim_changed' || result.reason === 'held_by_other') return { status: 1, stdout: '', stderr, not_ours: true };
|
|
3821
|
+
if (result.reason === 'not_found' || String(result.reason).startsWith('already_')) return { status: 1, stdout: '', stderr, final: true };
|
|
3822
|
+
return { status: 1, stdout: '', stderr, retryable: true };
|
|
3823
|
+
}
|
|
3824
|
+
redrawTaskFiles({ taskDb, db, root });
|
|
3825
|
+
return { status: 0, stdout: 'released', stderr: '' };
|
|
3826
|
+
}
|
|
3827
|
+
|
|
3499
3828
|
// A claim taken before the engine starts must not stay held if the flight
|
|
3500
3829
|
// refuses or errors first. Release through the same task plane the claim used,
|
|
3501
|
-
// naming the job by ref: the row id the claim named.
|
|
3502
|
-
|
|
3503
|
-
|
|
3830
|
+
// naming the job by ref: the row id the claim named. release, when a flight gives
|
|
3831
|
+
// one, lets it go in process as a job that could not start (see releaseOwnClaim); it
|
|
3832
|
+
// answers null for a claim it cannot record, and the task plane lets that go. A
|
|
3833
|
+
// claim another run has taken over since is left alone, with one plain line. A
|
|
3834
|
+
// release the store failed (retryable) is noted in failures, by the job's name, for the
|
|
3835
|
+
// closing pass to try again.
|
|
3836
|
+
function releaseUnstartedDispatchClaim(cli, { taskId, ref = taskId, actor, detail, stage, release = null, failures = null, log = () => {} }) {
|
|
3837
|
+
const reason = releaseReason(`paused at ${pausedStageLabel(stage)}`, detail);
|
|
3838
|
+
const released = (release && release({ ref, actor, reason })) || cli(['task', 'release', ref, '--as', actor, '--reason', reason]);
|
|
3504
3839
|
const releasedOk = Boolean(released && released.status === 0);
|
|
3840
|
+
const notOurs = Boolean(released && released.not_ours);
|
|
3841
|
+
const unnamed = Boolean(released && released.unnamed);
|
|
3842
|
+
if (notOurs) log(claimTakenOverLine(taskId));
|
|
3843
|
+
if (unnamed) log(unnamedClaimLine(taskId));
|
|
3505
3844
|
const suffix = releasedOk
|
|
3506
3845
|
? 'claim released, safe to retry'
|
|
3507
|
-
:
|
|
3846
|
+
: notOurs
|
|
3847
|
+
? 'claim left alone, another run claimed it after this one did'
|
|
3848
|
+
: unnamed
|
|
3849
|
+
? 'claim left alone, this run has no record of which claim is its own'
|
|
3850
|
+
: `claim release failed: ${String(released && (released.stderr || released.stdout) || 'unknown').trim().slice(0, 80)}`;
|
|
3508
3851
|
const base = String(detail || '').trim().slice(0, 200);
|
|
3509
|
-
|
|
3852
|
+
const text = base ? `${base}. ${suffix}` : suffix;
|
|
3853
|
+
if (failures && released && released.retryable) failures.set(String(taskId), { ref, actor, reason, base, detail: text, release });
|
|
3854
|
+
return { released: releasedOk, detail: text };
|
|
3510
3855
|
}
|
|
3511
3856
|
|
|
3512
3857
|
// A pause with no work must not keep the task. On 2026-08-21, 49 bug tasks sat
|
|
@@ -3524,14 +3869,44 @@ function releaseUnstartedDispatchClaim(cli, { taskId, ref = taskId, actor, detai
|
|
|
3524
3869
|
// refByTask names each claimed job by the row id the flight claimed it with, so the release
|
|
3525
3870
|
// lands on that row even if the display id a person reads now means another job. A job with
|
|
3526
3871
|
// no entry there is named by the display id in its pause row.
|
|
3527
|
-
|
|
3872
|
+
//
|
|
3873
|
+
// release, when a flight gives one, lets a claim go in the flight's own process, on the exact claim it
|
|
3874
|
+
// made (see releaseOwnClaim); a job it answers null for is let go through the task command. A job another
|
|
3875
|
+
// run has claimed since is left alone, with one plain line, and its row says the claim was not released.
|
|
3876
|
+
// A release the store failed is tried once more, and a claim that still cannot be let go says so in one
|
|
3877
|
+
// plain line.
|
|
3878
|
+
//
|
|
3879
|
+
// failures holds the start-failure releases that failed at the point of failure (see
|
|
3880
|
+
// releaseUnstartedDispatchClaim): a row that says its claim was not released, with an entry there, gets its
|
|
3881
|
+
// release tried again here, the way it was first tried.
|
|
3882
|
+
function releasePausedClaimsWithoutWork(cli, { paused, worktreeByTask, actorByTask, refByTask = new Map(), baseRef = '', log = () => {}, keepClaimsWithWork = true, release = null, failures = null }) {
|
|
3883
|
+
const letGo = (how, { ref, actor, reason }) => (how && how({ ref, actor, reason })) || cli(['task', 'release', ref, '--as', actor, '--reason', reason]);
|
|
3884
|
+
const saysNo = (taskId, released) => log(` could not release the claim on ${taskId}: ${String(released && (released.stderr || released.stdout) || 'unknown').replace(/\s+/g, ' ').trim().slice(0, 80)}. it stays until its lease runs out, unless the task record is repaired first.`);
|
|
3528
3885
|
for (const row of paused || []) {
|
|
3529
3886
|
// No claim (the job was never found, was refused as another workspace's, or the claim
|
|
3530
|
-
// failed),
|
|
3531
|
-
//
|
|
3532
|
-
|
|
3533
|
-
if (row.
|
|
3887
|
+
// failed), a claim the flight lost before its engine ran (it is the other run's now), work
|
|
3888
|
+
// already landed, or work waiting on a pull request for a person (the flight let that claim go
|
|
3889
|
+
// itself when saving its proof failed).
|
|
3890
|
+
if (row.stage === 'claim' || row.stage === 'task_lookup' || row.stage === 'other_workspace' || row.stage === 'claim_lost' || row.landed === true || row.held_for_review === true) continue;
|
|
3534
3891
|
const taskId = String(row.task || '');
|
|
3892
|
+
const failed = row.claim_released === false && failures ? failures.get(taskId) : null;
|
|
3893
|
+
if (failed) {
|
|
3894
|
+
failures.delete(taskId);
|
|
3895
|
+
const released = letGo(failed.release, failed);
|
|
3896
|
+
if (released && released.status === 0) {
|
|
3897
|
+
row.claim_released = true;
|
|
3898
|
+
if (row.detail === failed.detail) row.detail = failed.base ? `${failed.base}. claim released, safe to retry` : 'claim released, safe to retry';
|
|
3899
|
+
log(` released claim on ${taskId} on a second try`);
|
|
3900
|
+
} else if (released && released.not_ours) {
|
|
3901
|
+
log(claimTakenOverLine(taskId));
|
|
3902
|
+
} else if (released && released.unnamed) {
|
|
3903
|
+
log(unnamedClaimLine(taskId));
|
|
3904
|
+
} else if (!(released && released.final)) {
|
|
3905
|
+
saysNo(taskId, released);
|
|
3906
|
+
}
|
|
3907
|
+
continue;
|
|
3908
|
+
}
|
|
3909
|
+
if (row.claim_released !== undefined) continue;
|
|
3535
3910
|
const actor = actorByTask.get(taskId);
|
|
3536
3911
|
if (!taskId || !actor) continue;
|
|
3537
3912
|
const worktreePath = row.worktree || worktreeByTask.get(taskId) || '';
|
|
@@ -3543,9 +3918,19 @@ function releasePausedClaimsWithoutWork(cli, { paused, worktreeByTask, actorByTa
|
|
|
3543
3918
|
const reason = noWork && !ENGINE_DID_NOT_FINISH.has(row.stage)
|
|
3544
3919
|
? 'engine produced no work'
|
|
3545
3920
|
: releaseReason(`paused at ${pausedStageLabel(row.stage)}`, row.stage === 'build' ? row.reason || row.detail : row.detail || row.reason);
|
|
3546
|
-
const
|
|
3921
|
+
const ref = refByTask.get(taskId) || taskId;
|
|
3922
|
+
let released = letGo(release, { ref, actor, reason });
|
|
3923
|
+
if (released && released.retryable) released = letGo(release, { ref, actor, reason });
|
|
3924
|
+
if (released && (released.not_ours || released.unnamed)) {
|
|
3925
|
+
row.claim_released = false;
|
|
3926
|
+
log(released.unnamed ? unnamedClaimLine(taskId) : claimTakenOverLine(taskId));
|
|
3927
|
+
continue;
|
|
3928
|
+
}
|
|
3547
3929
|
row.claim_released = Boolean(released && released.status === 0);
|
|
3548
|
-
if (!row.claim_released)
|
|
3930
|
+
if (!row.claim_released) {
|
|
3931
|
+
if (released && released.retryable) saysNo(taskId, released);
|
|
3932
|
+
continue;
|
|
3933
|
+
}
|
|
3549
3934
|
log(keepClaimsWithWork
|
|
3550
3935
|
? ` released claim on ${taskId} (engine produced no work)`
|
|
3551
3936
|
: ` released claim on ${taskId} so a retry can pick it up${worktreePath ? `; any work stays in ${path.basename(worktreePath)}` : ''}`);
|
|
@@ -3572,14 +3957,26 @@ function releasePausedClaimsWithoutWork(cli, { paused, worktreeByTask, actorByTa
|
|
|
3572
3957
|
// same ids, so it is never looked up again. With no row id there is nothing exact to
|
|
3573
3958
|
// release, and nothing is.
|
|
3574
3959
|
//
|
|
3960
|
+
// It also names the claim it made (claimEventId, see claimLedger). A job another run has claimed since is
|
|
3961
|
+
// left alone, with one plain line: the pull request is still open for a person, and the claim is not this
|
|
3962
|
+
// flight's to put the kind on. With no name for its claim, the flight cannot say the claim is its own, so it
|
|
3963
|
+
// writes nothing and says so in one plain line.
|
|
3964
|
+
//
|
|
3575
3965
|
// Tried once: if the release fails too, the row says so.
|
|
3576
|
-
function releaseClaimOfUnsavedPullRequest({ taskDb, db, root, row, rowId, actor, log = () => {} }) {
|
|
3966
|
+
function releaseClaimOfUnsavedPullRequest({ taskDb, db, root, row, rowId, actor, log = () => {}, claimEventId = '' }) {
|
|
3577
3967
|
const reason = releaseReason(`PR ${row.pull_request.url} is open but its proof could not be saved`, row.detail);
|
|
3578
3968
|
let released = false;
|
|
3969
|
+
let notOurs = false;
|
|
3970
|
+
let unnamed = false;
|
|
3579
3971
|
let failure = 'unknown';
|
|
3580
3972
|
try {
|
|
3581
3973
|
if (!rowId) {
|
|
3582
3974
|
failure = 'the flight has no row id for it';
|
|
3975
|
+
} else if (!claimEventId && storeHoldsJob(taskDb, db, rowId)) {
|
|
3976
|
+
// The store holds the job and the flight has no name for its claim: a release that names no claim
|
|
3977
|
+
// could land on anyone's, so the job is left alone. A row the store does not know goes on to the
|
|
3978
|
+
// release below, which answers not_found, as it always did.
|
|
3979
|
+
unnamed = true;
|
|
3583
3980
|
} else {
|
|
3584
3981
|
const result = taskDb.releaseTask(db, {
|
|
3585
3982
|
id: rowId,
|
|
@@ -3587,9 +3984,11 @@ function releaseClaimOfUnsavedPullRequest({ taskDb, db, root, row, rowId, actor,
|
|
|
3587
3984
|
reason,
|
|
3588
3985
|
kind: PROOF_NOT_SAVED,
|
|
3589
3986
|
pullRequest: row.pull_request,
|
|
3987
|
+
claimEventId,
|
|
3590
3988
|
});
|
|
3591
3989
|
if (!result.released) {
|
|
3592
3990
|
failure = `release failed: ${result.reason}${result.claimed_by ? ` (held by ${result.claimed_by})` : ''}`;
|
|
3991
|
+
notOurs = result.reason === 'claim_changed' || result.reason === 'held_by_other';
|
|
3593
3992
|
} else {
|
|
3594
3993
|
released = true;
|
|
3595
3994
|
// The two files people read are drawn again from the saved state, the way the
|
|
@@ -3614,7 +4013,11 @@ function releaseClaimOfUnsavedPullRequest({ taskDb, db, root, row, rowId, actor,
|
|
|
3614
4013
|
row.claim_released = released;
|
|
3615
4014
|
log(row.claim_released
|
|
3616
4015
|
? ` released the claim on ${row.task} with the pull request link as the reason`
|
|
3617
|
-
:
|
|
4016
|
+
: notOurs
|
|
4017
|
+
? claimTakenOverLine(row.task)
|
|
4018
|
+
: unnamed
|
|
4019
|
+
? unnamedClaimLine(row.task)
|
|
4020
|
+
: ` could not release the claim on ${row.task}: ${failure.replace(/\s+/g, ' ').trim().slice(0, 80)}. it stays until its lease runs out, unless the task record is repaired first.`);
|
|
3618
4021
|
return row.claim_released;
|
|
3619
4022
|
}
|
|
3620
4023
|
|
|
@@ -3688,6 +4091,9 @@ async function runDispatchFlight({
|
|
|
3688
4091
|
// Extra house rules a caller appends to the builder's brief (atris idea
|
|
3689
4092
|
// tick sends the four values and the no-force-push, no-hook-skip rules).
|
|
3690
4093
|
brief = '',
|
|
4094
|
+
// The disk check before anything is claimed (see lowDiskRefusal). Tests inject both.
|
|
4095
|
+
freeSpace = undefined,
|
|
4096
|
+
spaceGuard = undefined,
|
|
3691
4097
|
} = {}) {
|
|
3692
4098
|
if (!engine) throw new Error('runDispatchFlight: engine is required');
|
|
3693
4099
|
if (!DISPATCH_CAPABLE.includes(engine)) {
|
|
@@ -3710,8 +4116,44 @@ async function runDispatchFlight({
|
|
|
3710
4116
|
}
|
|
3711
4117
|
|
|
3712
4118
|
const cli = ownCli || defaultOwnCli(root);
|
|
4119
|
+
// One look at the disk before anything is claimed or written to any job, not even the sweep of expired
|
|
4120
|
+
// claims below: worktree start's own check and reaping. Under its floor every start would refuse anyway,
|
|
4121
|
+
// and each refused start used to let a claim go that the rating then scored as a drop. A sealed review-only
|
|
4122
|
+
// flight builds in the temp folder, which that check leaves alone by default, so it is asked about that
|
|
4123
|
+
// folder with the check on.
|
|
4124
|
+
const sealedReview = reviewOnly && !dispatcher;
|
|
4125
|
+
const lowDisk = lowDiskRefusal({
|
|
4126
|
+
root,
|
|
4127
|
+
agent: engine,
|
|
4128
|
+
task: 'dispatch',
|
|
4129
|
+
freeSpace,
|
|
4130
|
+
spaceGuard: sealedReview ? (spaceGuard ?? true) : spaceGuard,
|
|
4131
|
+
...(sealedReview ? { target: os.tmpdir() } : {}),
|
|
4132
|
+
});
|
|
4133
|
+
if (lowDisk) {
|
|
4134
|
+
return stopForLowDisk({
|
|
4135
|
+
schema: 'atris.dispatch_receipt.v1',
|
|
4136
|
+
at: new Date().toISOString(),
|
|
4137
|
+
root,
|
|
4138
|
+
engine,
|
|
4139
|
+
tasks: ids,
|
|
4140
|
+
results: [],
|
|
4141
|
+
landed: [],
|
|
4142
|
+
ready: [],
|
|
4143
|
+
held_for_review: [],
|
|
4144
|
+
paused: [],
|
|
4145
|
+
}, lowDisk, ids.map((task) => ({ task })), log);
|
|
4146
|
+
}
|
|
3713
4147
|
const leaseDb = require('./task-db');
|
|
3714
4148
|
const leaseConnection = leaseDb.open();
|
|
4149
|
+
// The claim this flight made on each job, remembered from the claim write itself (see claimJob).
|
|
4150
|
+
const claims = claimLedger();
|
|
4151
|
+
// A claim let go before the engine ran is recorded in this process, on the claim this flight made, as one
|
|
4152
|
+
// that could not start. Any other claim the flight lets go (the engine was asked) is an ordinary release.
|
|
4153
|
+
const letGoUnstarted = ({ ref, actor, reason }) => releaseOwnClaim({ taskDb: leaseDb, db: leaseConnection, root, ref, actor, reason, claimEventId: claims.eventId(ref), neverStarted: true });
|
|
4154
|
+
const letGoAsked = ({ ref, actor, reason }) => releaseOwnClaim({ taskDb: leaseDb, db: leaseConnection, root, ref, actor, reason, claimEventId: claims.eventId(ref) });
|
|
4155
|
+
// The start-failure releases the store failed, for the closing pass to try again (see releaseUnstartedDispatchClaim).
|
|
4156
|
+
const releaseFailures = new Map();
|
|
3715
4157
|
// The workspace this flight works in, spelled the way the store spells it. It is read once, before
|
|
3716
4158
|
// the flight writes anything into its folder.
|
|
3717
4159
|
const workspace = leaseDb.workspaceRoot(root);
|
|
@@ -3764,8 +4206,17 @@ async function runDispatchFlight({
|
|
|
3764
4206
|
// What each claimed job is called to the task command, by the id the caller gave. Only jobs this
|
|
3765
4207
|
// flight claimed are in it.
|
|
3766
4208
|
const claimedRefs = new Map();
|
|
4209
|
+
// Right before the assigned engine is asked, the claim is marked started.
|
|
4210
|
+
const markStarted = (entry) => markClaimStarted({
|
|
4211
|
+
taskDb: leaseDb,
|
|
4212
|
+
db: leaseConnection,
|
|
4213
|
+
claims,
|
|
4214
|
+
ref: claimedRefs.get(String(entry.taskId)) || entry.taskId,
|
|
4215
|
+
actor: taskActor,
|
|
4216
|
+
task: entry.taskId,
|
|
4217
|
+
});
|
|
3767
4218
|
const leaseTimer = setInterval(() => {
|
|
3768
|
-
for (const id of leasedIds) leaseDb.renewLease(leaseConnection, { id, holder: taskActor, pid: process.pid });
|
|
4219
|
+
for (const id of leasedIds) leaseDb.renewLease(leaseConnection, { id, holder: taskActor, pid: process.pid, claimEventId: claims.eventId(id) });
|
|
3769
4220
|
}, 5 * 60000);
|
|
3770
4221
|
leaseTimer.unref();
|
|
3771
4222
|
try {
|
|
@@ -3789,17 +4240,18 @@ async function runDispatchFlight({
|
|
|
3789
4240
|
// named once, by its row id, at every step (see jobRef). Log lines and the receipt keep saying
|
|
3790
4241
|
// the display id.
|
|
3791
4242
|
const rowRef = jobRef(task, taskId);
|
|
3792
|
-
const claimed =
|
|
4243
|
+
const claimed = claimJob({ cli, taskDb: leaseDb, db: leaseConnection, root, ref: rowRef, actor: taskActor });
|
|
3793
4244
|
if (!claimed || claimed.status !== 0) {
|
|
3794
4245
|
const detail = String(claimed && (claimed.stderr || claimed.stdout) || 'claim failed').trim().slice(0, 300);
|
|
3795
4246
|
flight.paused.push({ task: taskId, stage: 'claim', detail });
|
|
3796
|
-
log(` x ${taskId} claim failed`);
|
|
4247
|
+
log(claimed && claimed.unnamed ? unnamedClaimLine(taskId) : ` x ${taskId} claim failed`);
|
|
3797
4248
|
continue;
|
|
3798
4249
|
}
|
|
3799
4250
|
claimedRefs.set(String(taskId), rowRef);
|
|
4251
|
+
claims.remember(rowRef, claimed.eventId);
|
|
3800
4252
|
try {
|
|
3801
4253
|
const leaseId = rowRef;
|
|
3802
|
-
const leased = leaseDb.setLease(leaseConnection, { id: leaseId, holder: taskActor });
|
|
4254
|
+
const leased = leaseDb.setLease(leaseConnection, { id: leaseId, holder: taskActor, claimEventId: claims.eventId(leaseId) });
|
|
3803
4255
|
if (!leased && (!ownCli || leaseDb.getTask(leaseConnection, leaseId))) throw new Error('could not set claim lease');
|
|
3804
4256
|
if (leased) leasedIds.add(leaseId);
|
|
3805
4257
|
let landingWorktreePath = '';
|
|
@@ -3820,6 +4272,9 @@ async function runDispatchFlight({
|
|
|
3820
4272
|
actor: taskActor,
|
|
3821
4273
|
stage: 'worktree_start',
|
|
3822
4274
|
detail: String(started.stderr || '').slice(0, 200),
|
|
4275
|
+
release: letGoUnstarted,
|
|
4276
|
+
failures: releaseFailures,
|
|
4277
|
+
log,
|
|
3823
4278
|
});
|
|
3824
4279
|
flight.paused.push({ task: taskId, stage: 'worktree_start', detail: released.detail, claim_released: released.released });
|
|
3825
4280
|
log(` ✗ ${taskId} worktree start failed`);
|
|
@@ -3834,6 +4289,9 @@ async function runDispatchFlight({
|
|
|
3834
4289
|
actor: taskActor,
|
|
3835
4290
|
stage: 'remote_quarantine',
|
|
3836
4291
|
detail: headTail(String(remoteBoundary && remoteBoundary.detail || 'could not prepare a sealed review sandbox')),
|
|
4292
|
+
release: letGoUnstarted,
|
|
4293
|
+
failures: releaseFailures,
|
|
4294
|
+
log,
|
|
3837
4295
|
});
|
|
3838
4296
|
flight.paused.push({
|
|
3839
4297
|
task: taskId,
|
|
@@ -3879,6 +4337,9 @@ async function runDispatchFlight({
|
|
|
3879
4337
|
actor: taskActor,
|
|
3880
4338
|
stage: 'prepare',
|
|
3881
4339
|
detail: String(err && err.message || err).slice(0, 200),
|
|
4340
|
+
release: letGoUnstarted,
|
|
4341
|
+
failures: releaseFailures,
|
|
4342
|
+
log,
|
|
3882
4343
|
});
|
|
3883
4344
|
flight.paused.push({ task: taskId, stage: 'prepare', detail: released.detail, claim_released: released.released });
|
|
3884
4345
|
log(` ✗ ${taskId} prepare failed`);
|
|
@@ -3913,7 +4374,41 @@ async function runDispatchFlight({
|
|
|
3913
4374
|
log(` fable handoff started: receipt ${path.relative(root, receiptPath)}`);
|
|
3914
4375
|
}
|
|
3915
4376
|
|
|
3916
|
-
|
|
4377
|
+
// A job that never reached its engine (its brief could not be written, or its start could not be marked)
|
|
4378
|
+
// pauses and its claim goes back as one that could not start, here, where it failed, while the flight's
|
|
4379
|
+
// other jobs go on. The jobs that did start carry on.
|
|
4380
|
+
const letGoNotStarted = (entry, result) => {
|
|
4381
|
+
const stage = result.not_started;
|
|
4382
|
+
const ref = claimedRefs.get(String(entry.taskId)) || entry.taskId;
|
|
4383
|
+
if (stage === 'claim_lost') {
|
|
4384
|
+
// The job is another run's now (see markClaimStarted). The flight lets go of nothing, writes nothing to it, and
|
|
4385
|
+
// stops renewing a lease on it.
|
|
4386
|
+
leasedIds.delete(ref);
|
|
4387
|
+
flight.paused.push({ task: entry.taskId, stage, detail: result.unnamed ? UNNAMED_CLAIM_DETAIL : LOST_CLAIM_DETAIL, worktree: entry.worktreePath, claim_released: false });
|
|
4388
|
+
log(result.unnamed ? unnamedClaimLine(entry.taskId) : claimLostLine(entry.taskId));
|
|
4389
|
+
return;
|
|
4390
|
+
}
|
|
4391
|
+
// Nothing in here may end the run: the other jobs still have their cleanup to come.
|
|
4392
|
+
let detail = result.detail;
|
|
4393
|
+
let released = false;
|
|
4394
|
+
try {
|
|
4395
|
+
({ released, detail } = releaseUnstartedDispatchClaim(cli, {
|
|
4396
|
+
taskId: entry.taskId,
|
|
4397
|
+
ref,
|
|
4398
|
+
actor: taskActor,
|
|
4399
|
+
stage,
|
|
4400
|
+
detail: result.detail,
|
|
4401
|
+
release: letGoUnstarted,
|
|
4402
|
+
failures: releaseFailures,
|
|
4403
|
+
log,
|
|
4404
|
+
}));
|
|
4405
|
+
} catch (err) {
|
|
4406
|
+
detail = `${String(result.detail || '').slice(0, 120)}. claim release failed: ${String(err && err.message || err).slice(0, 80)}`;
|
|
4407
|
+
}
|
|
4408
|
+
flight.paused.push({ task: entry.taskId, stage, detail, worktree: entry.worktreePath, claim_released: released });
|
|
4409
|
+
log(` ✗ ${entry.taskId} ${NOT_STARTED_WORDS[stage] || 'prepare'} failed`);
|
|
4410
|
+
};
|
|
4411
|
+
const settled = await Promise.all(prepared.map((entry) => {
|
|
3917
4412
|
const startedAtMs = Date.now();
|
|
3918
4413
|
return dispatchEntryWithRestaff({
|
|
3919
4414
|
entry,
|
|
@@ -3927,19 +4422,22 @@ async function runDispatchFlight({
|
|
|
3927
4422
|
defaultPin: { model, effort, maxSeconds: capSeconds(maxSeconds), ...(prep ? { prep } : {}) },
|
|
3928
4423
|
job: rosterJob,
|
|
3929
4424
|
prepAsk,
|
|
4425
|
+
beforeEngine: markStarted,
|
|
3930
4426
|
}).then((result) => {
|
|
4427
|
+
if (result.not_started) letGoNotStarted(entry, result);
|
|
3931
4428
|
const completedAtMs = Date.now();
|
|
3932
4429
|
return {
|
|
3933
4430
|
entry,
|
|
3934
4431
|
result,
|
|
3935
4432
|
startedAtMs,
|
|
3936
4433
|
completedAtMs,
|
|
3937
|
-
candidate: enforceRemoteBoundary && !detectDeadEngineDispatch(result)
|
|
4434
|
+
candidate: enforceRemoteBoundary && !result.not_started && !detectDeadEngineDispatch(result)
|
|
3938
4435
|
? reviewCandidateSnapshot(entry.remoteBoundary)
|
|
3939
4436
|
: null,
|
|
3940
4437
|
};
|
|
3941
4438
|
});
|
|
3942
4439
|
}));
|
|
4440
|
+
const results = settled.filter(({ result }) => !result.not_started);
|
|
3943
4441
|
flight.results = results.map(({ entry, result, candidate, startedAtMs, completedAtMs }) => {
|
|
3944
4442
|
const row = {
|
|
3945
4443
|
task: entry.taskId,
|
|
@@ -4482,7 +4980,10 @@ async function runDispatchFlight({
|
|
|
4482
4980
|
stampDispatchBrief(root, result.brief_id, 'partial', `opened a pull request for ${entry.taskId}, but task ready failed`);
|
|
4483
4981
|
log(' the pull request is open, but saving its proof failed. repair the task record before starting more work.');
|
|
4484
4982
|
const unsaved = flight.paused.find((row) => row.task === entry.taskId && row.stage === 'task_ready' && row.held_for_review === true);
|
|
4485
|
-
if (unsaved)
|
|
4983
|
+
if (unsaved) {
|
|
4984
|
+
const rowId = entry.task && entry.task.id;
|
|
4985
|
+
releaseClaimOfUnsavedPullRequest({ taskDb: leaseDb, db: leaseConnection, root, row: unsaved, rowId, actor: taskActor, log, claimEventId: claims.eventId(rowId) });
|
|
4986
|
+
}
|
|
4486
4987
|
continue;
|
|
4487
4988
|
}
|
|
4488
4989
|
stampDispatchBrief(root, result.brief_id, 'pass', `held ${entry.taskId} for a person to land`);
|
|
@@ -4531,6 +5032,8 @@ async function runDispatchFlight({
|
|
|
4531
5032
|
baseRef: landTarget,
|
|
4532
5033
|
log,
|
|
4533
5034
|
keepClaimsWithWork: proofOnly,
|
|
5035
|
+
release: letGoAsked,
|
|
5036
|
+
failures: releaseFailures,
|
|
4534
5037
|
});
|
|
4535
5038
|
|
|
4536
5039
|
recordFlightRosterRuns(root, flight, results, {
|
|
@@ -4557,7 +5060,7 @@ async function runDispatchFlight({
|
|
|
4557
5060
|
clearInterval(leaseTimer);
|
|
4558
5061
|
for (const id of leasedIds) {
|
|
4559
5062
|
const row = leaseDb.getTask(leaseConnection, id);
|
|
4560
|
-
if (row && row.status !== 'claimed') leaseDb.clearLease(leaseConnection, { id, holder: taskActor });
|
|
5063
|
+
if (row && row.status !== 'claimed') leaseDb.clearLease(leaseConnection, { id, holder: taskActor, claimEventId: claims.eventId(id) });
|
|
4561
5064
|
}
|
|
4562
5065
|
}
|
|
4563
5066
|
}
|